2023-09-23T18:43:04.138 INFO:root:teuthology version: 0.0.1.dev194+g54e62bc 2023-09-23T18:43:04.139 DEBUG:teuthology.run:Teuthology command: teuthology --name teuthology-2023-09-23_14:23:10-upgrade:pacific-x-reef-distro-default-smithi --owner scheduled_teuthology@teuthology --description upgrade:pacific-x/stress-split/{0-distro/ubuntu_20.04 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} --verbose --archive /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:pacific-x-reef-distro-default-smithi/7401794 -- /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:pacific-x-reef-distro-default-smithi/7401794/orig.config.yaml 2023-09-23T18:43:04.205 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-09-23T18:43:04.385 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:pacific-x-reef-distro-default-smithi/7401794 branch: reef description: upgrade:pacific-x/stress-split/{0-distro/ubuntu_20.04 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} email: ceph-qa@ceph.io first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' first_in_suite: false job_id: '7401794' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' name: teuthology-2023-09-23_14:23:10-upgrade:pacific-x-reef-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: ubuntu os_version: '20.04' overrides: admin_socket: branch: reef ceph: conf: global: mon election default strategy: 3 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 kernel: hwe: true workunit: branch: reef sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 owner: scheduled_teuthology@teuthology priority: 100 repo: https://github.com/ceph/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' seed: 1860 sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: pacific clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' suite: upgrade:pacific-x suite_branch: reef suite_path: /home/teuthworker/src/github.com_ceph_ceph_bb0daeff97f020f686596fca6eba20c310e26fb7/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph.git suite_sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 targets: smithi016.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFyT/r7xuKfmg50w0OdQebr4A/ptJo1Zk6DyZNZNZM/q5rW+sznh+wvHqeT3e4YtCdZLaXyvMms8o8u8H5nXOYM= smithi092.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFyT/r7xuKfmg50w0OdQebr4A/ptJo1Zk6DyZNZNZM/q5rW+sznh+wvHqeT3e4YtCdZLaXyvMms8o8u8H5nXOYM= tasks: - install: branch: pacific exclude_packages: - ceph-volume - cephadm: cephadm_branch: pacific cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:pacific - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 54e62bcbac4e53d9685e08328b790d3b20d71cae timestamp: 2023-09-23_14:23:10 tube: smithi user: teuthology verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2830 2023-09-23T18:43:04.385 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph_bb0daeff97f020f686596fca6eba20c310e26fb7/qa; will attempt to use it 2023-09-23T18:43:04.386 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph_bb0daeff97f020f686596fca6eba20c310e26fb7/qa/tasks 2023-09-23T18:43:04.386 INFO:teuthology.run_tasks:Running task internal.check_packages... 2023-09-23T18:43:04.388 INFO:teuthology.task.internal:Checking packages... 2023-09-23T18:43:04.412 INFO:teuthology.task.internal:Checking packages for os_type 'ubuntu', flavor 'default' and ceph hash 'bb0daeff97f020f686596fca6eba20c310e26fb7' 2023-09-23T18:43:04.413 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T18:43:04.413 INFO:teuthology.packaging:ref: None 2023-09-23T18:43:04.413 INFO:teuthology.packaging:tag: None 2023-09-23T18:43:04.413 INFO:teuthology.packaging:branch: reef 2023-09-23T18:43:04.413 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T18:43:04.413 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=reef 2023-09-23T18:43:04.758 INFO:teuthology.task.internal:Found packages for ceph version 18.2.0-460-gbb0daeff-1focal 2023-09-23T18:43:04.758 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2023-09-23T18:43:04.765 INFO:teuthology.task.internal:no buildpackages task found 2023-09-23T18:43:04.765 INFO:teuthology.run_tasks:Running task internal.save_config... 2023-09-23T18:43:04.781 INFO:teuthology.task.internal:Saving configuration 2023-09-23T18:43:04.802 INFO:teuthology.run_tasks:Running task internal.check_lock... 2023-09-23T18:43:04.808 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-09-23T18:43:04.833 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi016.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:pacific-x-reef-distro-default-smithi/7401794', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '20.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-09-23 18:36:55.411726', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGETOy3xONfbx7kUFCNiUtRLNfkcrcEsWzPdq6Np/6Bw'} 2023-09-23T18:43:04.856 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi092.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:pacific-x-reef-distro-default-smithi/7401794', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '20.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-09-23 18:36:55.413131', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGETOy3xONfbx7kUFCNiUtRLNfkcrcEsWzPdq6Np/6Bw'} 2023-09-23T18:43:04.856 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2023-09-23T18:43:04.862 INFO:teuthology.task.internal:roles: ubuntu@smithi016.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2023-09-23T18:43:04.862 INFO:teuthology.task.internal:roles: ubuntu@smithi092.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2023-09-23T18:43:04.862 INFO:teuthology.run_tasks:Running task console_log... 2023-09-23T18:43:04.935 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f708573d1f0>, signals=[15]) 2023-09-23T18:43:04.936 INFO:teuthology.run_tasks:Running task internal.connect... 2023-09-23T18:43:04.942 INFO:teuthology.task.internal:Opening connections... 2023-09-23T18:43:04.943 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi016.front.sepia.ceph.com 2023-09-23T18:43:04.944 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi016.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T18:43:04.994 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi092.front.sepia.ceph.com 2023-09-23T18:43:04.995 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi092.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T18:43:05.043 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2023-09-23T18:43:05.050 DEBUG:teuthology.orchestra.run.smithi016:> uname -m 2023-09-23T18:43:05.075 INFO:teuthology.orchestra.run.smithi016.stdout:x86_64 2023-09-23T18:43:05.076 DEBUG:teuthology.orchestra.run.smithi016:> cat /etc/os-release 2023-09-23T18:43:05.082 INFO:teuthology.orchestra.run.smithi016.stdout:NAME="Ubuntu" 2023-09-23T18:43:05.082 INFO:teuthology.orchestra.run.smithi016.stdout:VERSION="20.04.4 LTS (Focal Fossa)" 2023-09-23T18:43:05.082 INFO:teuthology.orchestra.run.smithi016.stdout:ID=ubuntu 2023-09-23T18:43:05.082 INFO:teuthology.orchestra.run.smithi016.stdout:ID_LIKE=debian 2023-09-23T18:43:05.082 INFO:teuthology.orchestra.run.smithi016.stdout:PRETTY_NAME="Ubuntu 20.04.4 LTS" 2023-09-23T18:43:05.083 INFO:teuthology.orchestra.run.smithi016.stdout:VERSION_ID="20.04" 2023-09-23T18:43:05.083 INFO:teuthology.orchestra.run.smithi016.stdout:HOME_URL="https://www.ubuntu.com/" 2023-09-23T18:43:05.083 INFO:teuthology.orchestra.run.smithi016.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2023-09-23T18:43:05.083 INFO:teuthology.orchestra.run.smithi016.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2023-09-23T18:43:05.083 INFO:teuthology.orchestra.run.smithi016.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2023-09-23T18:43:05.083 INFO:teuthology.orchestra.run.smithi016.stdout:VERSION_CODENAME=focal 2023-09-23T18:43:05.083 INFO:teuthology.orchestra.run.smithi016.stdout:UBUNTU_CODENAME=focal 2023-09-23T18:43:05.084 INFO:teuthology.lock.ops:Updating smithi016.front.sepia.ceph.com on lock server 2023-09-23T18:43:05.121 DEBUG:teuthology.orchestra.run.smithi092:> uname -m 2023-09-23T18:43:05.128 INFO:teuthology.orchestra.run.smithi092.stdout:x86_64 2023-09-23T18:43:05.128 DEBUG:teuthology.orchestra.run.smithi092:> cat /etc/os-release 2023-09-23T18:43:05.133 INFO:teuthology.orchestra.run.smithi092.stdout:NAME="Ubuntu" 2023-09-23T18:43:05.134 INFO:teuthology.orchestra.run.smithi092.stdout:VERSION="20.04.4 LTS (Focal Fossa)" 2023-09-23T18:43:05.134 INFO:teuthology.orchestra.run.smithi092.stdout:ID=ubuntu 2023-09-23T18:43:05.134 INFO:teuthology.orchestra.run.smithi092.stdout:ID_LIKE=debian 2023-09-23T18:43:05.134 INFO:teuthology.orchestra.run.smithi092.stdout:PRETTY_NAME="Ubuntu 20.04.4 LTS" 2023-09-23T18:43:05.134 INFO:teuthology.orchestra.run.smithi092.stdout:VERSION_ID="20.04" 2023-09-23T18:43:05.134 INFO:teuthology.orchestra.run.smithi092.stdout:HOME_URL="https://www.ubuntu.com/" 2023-09-23T18:43:05.134 INFO:teuthology.orchestra.run.smithi092.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2023-09-23T18:43:05.134 INFO:teuthology.orchestra.run.smithi092.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2023-09-23T18:43:05.134 INFO:teuthology.orchestra.run.smithi092.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2023-09-23T18:43:05.135 INFO:teuthology.orchestra.run.smithi092.stdout:VERSION_CODENAME=focal 2023-09-23T18:43:05.135 INFO:teuthology.orchestra.run.smithi092.stdout:UBUNTU_CODENAME=focal 2023-09-23T18:43:05.135 INFO:teuthology.lock.ops:Updating smithi092.front.sepia.ceph.com on lock server 2023-09-23T18:43:05.162 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2023-09-23T18:43:05.171 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2023-09-23T18:43:05.175 INFO:teuthology.task.internal:Checking for old test directory... 2023-09-23T18:43:05.175 DEBUG:teuthology.orchestra.run.smithi016:> test '!' -e /home/ubuntu/cephtest 2023-09-23T18:43:05.178 DEBUG:teuthology.orchestra.run.smithi092:> test '!' -e /home/ubuntu/cephtest 2023-09-23T18:43:05.183 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2023-09-23T18:43:05.189 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2023-09-23T18:43:05.189 DEBUG:teuthology.orchestra.run.smithi016:> test -z $(ls -A /var/lib/ceph) 2023-09-23T18:43:05.224 DEBUG:teuthology.orchestra.run.smithi092:> test -z $(ls -A /var/lib/ceph) 2023-09-23T18:43:05.244 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2023-09-23T18:43:05.297 INFO:teuthology.run_tasks:Running task kernel... 2023-09-23T18:43:05.312 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2023-09-23T18:43:05.312 INFO:teuthology.task.kernel:normalize config orig: {'hwe': True} 2023-09-23T18:43:05.312 DEBUG:teuthology.task.kernel:normalized overrides {'mon.a': {'hwe': True}, 'mon.c': {'hwe': True}, 'mgr.y': {'hwe': True}, 'osd.0': {'hwe': True}, 'osd.1': {'hwe': True}, 'osd.2': {'hwe': True}, 'osd.3': {'hwe': True}, 'client.0': {'hwe': True}, 'node-exporter.a': {'hwe': True}, 'alertmanager.a': {'hwe': True}, 'mon.b': {'hwe': True}, 'mgr.x': {'hwe': True}, 'osd.4': {'hwe': True}, 'osd.5': {'hwe': True}, 'osd.6': {'hwe': True}, 'osd.7': {'hwe': True}, 'client.1': {'hwe': True}, 'prometheus.a': {'hwe': True}, 'grafana.a': {'hwe': True}, 'node-exporter.b': {'hwe': True}} 2023-09-23T18:43:05.313 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': True, 'sha1': 'distro', 'hwe': True}, 'mon.b': {'kdb': True, 'sha1': 'distro', 'hwe': True}}, timeout 300 2023-09-23T18:43:05.313 DEBUG:teuthology.orchestra.run.smithi016:> test -f /run/.containerenv -o -f /.dockerenv 2023-09-23T18:43:05.313 DEBUG:teuthology.orchestra.run.smithi092:> test -f /run/.containerenv -o -f /.dockerenv 2023-09-23T18:43:05.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:05.319 DEBUG:teuthology.orchestra.run.smithi016:> uname -r 2023-09-23T18:43:05.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:05.320 DEBUG:teuthology.orchestra.run.smithi092:> uname -r 2023-09-23T18:43:05.368 INFO:teuthology.orchestra.run.smithi016.stdout:5.4.0-124-generic 2023-09-23T18:43:05.368 INFO:teuthology.task.kernel:Running kernel on smithi016: 5.4.0-124-generic 2023-09-23T18:43:05.368 DEBUG:teuthology.orchestra.run.smithi016:> sudo apt-get clean 2023-09-23T18:43:05.369 INFO:teuthology.orchestra.run.smithi092.stdout:5.4.0-124-generic 2023-09-23T18:43:05.370 INFO:teuthology.task.kernel:Running kernel on smithi092: 5.4.0-124-generic 2023-09-23T18:43:05.370 DEBUG:teuthology.orchestra.run.smithi092:> sudo apt-get clean 2023-09-23T18:43:05.476 DEBUG:teuthology.orchestra.run.smithi092:> sudo apt-get update 2023-09-23T18:43:05.482 DEBUG:teuthology.orchestra.run.smithi016:> sudo apt-get update 2023-09-23T18:43:05.647 INFO:teuthology.orchestra.run.smithi016.stdout:Get:1 http://security.ubuntu.com/ubuntu focal-security InRelease [114 kB] 2023-09-23T18:43:05.671 INFO:teuthology.orchestra.run.smithi092.stdout:Hit:1 http://archive.ubuntu.com/ubuntu focal InRelease 2023-09-23T18:43:05.675 INFO:teuthology.orchestra.run.smithi016.stdout:Hit:2 http://archive.ubuntu.com/ubuntu focal InRelease 2023-09-23T18:43:05.682 INFO:teuthology.orchestra.run.smithi016.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal-updates InRelease [114 kB] 2023-09-23T18:43:05.692 INFO:teuthology.orchestra.run.smithi092.stdout:Get:2 http://archive.ubuntu.com/ubuntu focal-updates InRelease [114 kB] 2023-09-23T18:43:05.762 INFO:teuthology.orchestra.run.smithi092.stdout:Get:3 http://security.ubuntu.com/ubuntu focal-security InRelease [114 kB] 2023-09-23T18:43:05.798 INFO:teuthology.orchestra.run.smithi016.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal-backports InRelease [108 kB] 2023-09-23T18:43:05.799 INFO:teuthology.orchestra.run.smithi092.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal-backports InRelease [108 kB] 2023-09-23T18:43:06.054 INFO:teuthology.orchestra.run.smithi016.stdout:Get:5 http://security.ubuntu.com/ubuntu focal-security/main i386 Packages [652 kB] 2023-09-23T18:43:06.072 INFO:teuthology.orchestra.run.smithi092.stdout:Get:5 http://archive.ubuntu.com/ubuntu focal-updates/main i386 Packages [884 kB] 2023-09-23T18:43:06.132 INFO:teuthology.orchestra.run.smithi016.stdout:Get:6 http://security.ubuntu.com/ubuntu focal-security/main amd64 Packages [2,464 kB] 2023-09-23T18:43:06.173 INFO:teuthology.orchestra.run.smithi016.stdout:Get:7 http://security.ubuntu.com/ubuntu focal-security/main Translation-en [385 kB] 2023-09-23T18:43:06.176 INFO:teuthology.orchestra.run.smithi016.stdout:Get:8 http://security.ubuntu.com/ubuntu focal-security/main amd64 c-n-f Metadata [13.2 kB] 2023-09-23T18:43:06.176 INFO:teuthology.orchestra.run.smithi016.stdout:Get:9 http://security.ubuntu.com/ubuntu focal-security/restricted i386 Packages [32.3 kB] 2023-09-23T18:43:06.177 INFO:teuthology.orchestra.run.smithi016.stdout:Get:10 http://security.ubuntu.com/ubuntu focal-security/restricted amd64 Packages [2,210 kB] 2023-09-23T18:43:06.191 INFO:teuthology.orchestra.run.smithi016.stdout:Get:11 http://security.ubuntu.com/ubuntu focal-security/restricted Translation-en [309 kB] 2023-09-23T18:43:06.193 INFO:teuthology.orchestra.run.smithi016.stdout:Get:12 http://security.ubuntu.com/ubuntu focal-security/restricted amd64 c-n-f Metadata [580 B] 2023-09-23T18:43:06.193 INFO:teuthology.orchestra.run.smithi016.stdout:Get:13 http://security.ubuntu.com/ubuntu focal-security/universe amd64 Packages [886 kB] 2023-09-23T18:43:06.199 INFO:teuthology.orchestra.run.smithi016.stdout:Get:14 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 Packages [2,852 kB] 2023-09-23T18:43:06.199 INFO:teuthology.orchestra.run.smithi016.stdout:Get:15 http://security.ubuntu.com/ubuntu focal-security/universe i386 Packages [615 kB] 2023-09-23T18:43:06.203 INFO:teuthology.orchestra.run.smithi016.stdout:Get:16 http://security.ubuntu.com/ubuntu focal-security/universe Translation-en [185 kB] 2023-09-23T18:43:06.205 INFO:teuthology.orchestra.run.smithi016.stdout:Get:17 http://security.ubuntu.com/ubuntu focal-security/universe amd64 c-n-f Metadata [19.1 kB] 2023-09-23T18:43:06.207 INFO:teuthology.orchestra.run.smithi016.stdout:Get:18 http://security.ubuntu.com/ubuntu focal-security/multiverse i386 Packages [7,196 B] 2023-09-23T18:43:06.207 INFO:teuthology.orchestra.run.smithi016.stdout:Get:19 http://security.ubuntu.com/ubuntu focal-security/multiverse amd64 Packages [23.6 kB] 2023-09-23T18:43:06.208 INFO:teuthology.orchestra.run.smithi016.stdout:Get:20 http://security.ubuntu.com/ubuntu focal-security/multiverse Translation-en [5,504 B] 2023-09-23T18:43:06.214 INFO:teuthology.orchestra.run.smithi092.stdout:Get:6 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 Packages [2,852 kB] 2023-09-23T18:43:06.224 INFO:teuthology.orchestra.run.smithi016.stdout:Get:21 http://security.ubuntu.com/ubuntu focal-security/multiverse amd64 c-n-f Metadata [548 B] 2023-09-23T18:43:06.285 INFO:teuthology.orchestra.run.smithi092.stdout:Get:7 http://archive.ubuntu.com/ubuntu focal-updates/main Translation-en [468 kB] 2023-09-23T18:43:06.291 INFO:teuthology.orchestra.run.smithi092.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 c-n-f Metadata [17.1 kB] 2023-09-23T18:43:06.291 INFO:teuthology.orchestra.run.smithi092.stdout:Get:9 http://archive.ubuntu.com/ubuntu focal-updates/restricted i386 Packages [33.7 kB] 2023-09-23T18:43:06.292 INFO:teuthology.orchestra.run.smithi092.stdout:Get:10 http://archive.ubuntu.com/ubuntu focal-updates/restricted amd64 Packages [2,327 kB] 2023-09-23T18:43:06.317 INFO:teuthology.orchestra.run.smithi092.stdout:Get:11 http://archive.ubuntu.com/ubuntu focal-updates/restricted Translation-en [325 kB] 2023-09-23T18:43:06.320 INFO:teuthology.orchestra.run.smithi092.stdout:Get:12 http://archive.ubuntu.com/ubuntu focal-updates/restricted amd64 c-n-f Metadata [576 B] 2023-09-23T18:43:06.320 INFO:teuthology.orchestra.run.smithi092.stdout:Get:13 http://archive.ubuntu.com/ubuntu focal-updates/universe i386 Packages [748 kB] 2023-09-23T18:43:06.328 INFO:teuthology.orchestra.run.smithi092.stdout:Get:14 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 Packages [1,117 kB] 2023-09-23T18:43:06.341 INFO:teuthology.orchestra.run.smithi092.stdout:Get:15 http://archive.ubuntu.com/ubuntu focal-updates/universe Translation-en [267 kB] 2023-09-23T18:43:06.343 INFO:teuthology.orchestra.run.smithi092.stdout:Get:16 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 c-n-f Metadata [25.6 kB] 2023-09-23T18:43:06.344 INFO:teuthology.orchestra.run.smithi092.stdout:Get:17 http://archive.ubuntu.com/ubuntu focal-updates/multiverse amd64 Packages [25.8 kB] 2023-09-23T18:43:06.344 INFO:teuthology.orchestra.run.smithi092.stdout:Get:18 http://archive.ubuntu.com/ubuntu focal-updates/multiverse i386 Packages [8,436 B] 2023-09-23T18:43:06.344 INFO:teuthology.orchestra.run.smithi092.stdout:Get:19 http://archive.ubuntu.com/ubuntu focal-updates/multiverse Translation-en [7,484 B] 2023-09-23T18:43:06.363 INFO:teuthology.orchestra.run.smithi092.stdout:Get:20 http://archive.ubuntu.com/ubuntu focal-updates/multiverse amd64 c-n-f Metadata [620 B] 2023-09-23T18:43:06.363 INFO:teuthology.orchestra.run.smithi092.stdout:Get:21 http://archive.ubuntu.com/ubuntu focal-backports/main amd64 Packages [45.7 kB] 2023-09-23T18:43:06.364 INFO:teuthology.orchestra.run.smithi092.stdout:Get:22 http://archive.ubuntu.com/ubuntu focal-backports/main i386 Packages [36.1 kB] 2023-09-23T18:43:06.364 INFO:teuthology.orchestra.run.smithi092.stdout:Get:23 http://archive.ubuntu.com/ubuntu focal-backports/main amd64 c-n-f Metadata [1,420 B] 2023-09-23T18:43:06.364 INFO:teuthology.orchestra.run.smithi092.stdout:Get:24 http://archive.ubuntu.com/ubuntu focal-backports/universe amd64 Packages [25.0 kB] 2023-09-23T18:43:06.378 INFO:teuthology.orchestra.run.smithi016.stdout:Get:22 http://archive.ubuntu.com/ubuntu focal-updates/main i386 Packages [884 kB] 2023-09-23T18:43:06.383 INFO:teuthology.orchestra.run.smithi092.stdout:Get:25 http://archive.ubuntu.com/ubuntu focal-backports/universe i386 Packages [13.8 kB] 2023-09-23T18:43:06.383 INFO:teuthology.orchestra.run.smithi092.stdout:Get:26 http://archive.ubuntu.com/ubuntu focal-backports/universe amd64 c-n-f Metadata [880 B] 2023-09-23T18:43:06.387 INFO:teuthology.orchestra.run.smithi092.stdout:Get:27 http://security.ubuntu.com/ubuntu focal-security/main amd64 Packages [2,464 kB] 2023-09-23T18:43:06.389 INFO:teuthology.orchestra.run.smithi016.stdout:Get:23 http://archive.ubuntu.com/ubuntu focal-updates/main Translation-en [468 kB] 2023-09-23T18:43:06.394 INFO:teuthology.orchestra.run.smithi016.stdout:Get:24 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 c-n-f Metadata [17.1 kB] 2023-09-23T18:43:06.394 INFO:teuthology.orchestra.run.smithi016.stdout:Get:25 http://archive.ubuntu.com/ubuntu focal-updates/restricted amd64 Packages [2,327 kB] 2023-09-23T18:43:06.420 INFO:teuthology.orchestra.run.smithi016.stdout:Get:26 http://archive.ubuntu.com/ubuntu focal-updates/restricted i386 Packages [33.7 kB] 2023-09-23T18:43:06.420 INFO:teuthology.orchestra.run.smithi016.stdout:Get:27 http://archive.ubuntu.com/ubuntu focal-updates/restricted Translation-en [325 kB] 2023-09-23T18:43:06.423 INFO:teuthology.orchestra.run.smithi016.stdout:Get:28 http://archive.ubuntu.com/ubuntu focal-updates/restricted amd64 c-n-f Metadata [576 B] 2023-09-23T18:43:06.423 INFO:teuthology.orchestra.run.smithi016.stdout:Get:29 http://archive.ubuntu.com/ubuntu focal-updates/universe i386 Packages [748 kB] 2023-09-23T18:43:06.429 INFO:teuthology.orchestra.run.smithi016.stdout:Get:30 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 Packages [1,117 kB] 2023-09-23T18:43:06.439 INFO:teuthology.orchestra.run.smithi016.stdout:Get:31 http://archive.ubuntu.com/ubuntu focal-updates/universe Translation-en [267 kB] 2023-09-23T18:43:06.441 INFO:teuthology.orchestra.run.smithi016.stdout:Get:32 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 c-n-f Metadata [25.6 kB] 2023-09-23T18:43:06.441 INFO:teuthology.orchestra.run.smithi016.stdout:Get:33 http://archive.ubuntu.com/ubuntu focal-updates/multiverse amd64 Packages [25.8 kB] 2023-09-23T18:43:06.442 INFO:teuthology.orchestra.run.smithi016.stdout:Get:34 http://archive.ubuntu.com/ubuntu focal-updates/multiverse i386 Packages [8,436 B] 2023-09-23T18:43:06.458 INFO:teuthology.orchestra.run.smithi016.stdout:Get:35 http://archive.ubuntu.com/ubuntu focal-updates/multiverse Translation-en [7,484 B] 2023-09-23T18:43:06.459 INFO:teuthology.orchestra.run.smithi016.stdout:Get:36 http://archive.ubuntu.com/ubuntu focal-updates/multiverse amd64 c-n-f Metadata [620 B] 2023-09-23T18:43:06.459 INFO:teuthology.orchestra.run.smithi016.stdout:Get:37 http://archive.ubuntu.com/ubuntu focal-backports/main amd64 Packages [45.7 kB] 2023-09-23T18:43:06.459 INFO:teuthology.orchestra.run.smithi016.stdout:Get:38 http://archive.ubuntu.com/ubuntu focal-backports/main i386 Packages [36.1 kB] 2023-09-23T18:43:06.460 INFO:teuthology.orchestra.run.smithi016.stdout:Get:39 http://archive.ubuntu.com/ubuntu focal-backports/main amd64 c-n-f Metadata [1,420 B] 2023-09-23T18:43:06.478 INFO:teuthology.orchestra.run.smithi016.stdout:Get:40 http://archive.ubuntu.com/ubuntu focal-backports/universe i386 Packages [13.8 kB] 2023-09-23T18:43:06.479 INFO:teuthology.orchestra.run.smithi016.stdout:Get:41 http://archive.ubuntu.com/ubuntu focal-backports/universe amd64 Packages [25.0 kB] 2023-09-23T18:43:06.479 INFO:teuthology.orchestra.run.smithi016.stdout:Get:42 http://archive.ubuntu.com/ubuntu focal-backports/universe amd64 c-n-f Metadata [880 B] 2023-09-23T18:43:06.864 INFO:teuthology.orchestra.run.smithi092.stdout:Get:28 http://security.ubuntu.com/ubuntu focal-security/main i386 Packages [652 kB] 2023-09-23T18:43:07.120 INFO:teuthology.orchestra.run.smithi092.stdout:Get:29 http://security.ubuntu.com/ubuntu focal-security/main Translation-en [385 kB] 2023-09-23T18:43:07.121 INFO:teuthology.orchestra.run.smithi092.stdout:Get:30 http://security.ubuntu.com/ubuntu focal-security/main amd64 c-n-f Metadata [13.2 kB] 2023-09-23T18:43:07.121 INFO:teuthology.orchestra.run.smithi092.stdout:Get:31 http://security.ubuntu.com/ubuntu focal-security/restricted i386 Packages [32.3 kB] 2023-09-23T18:43:07.121 INFO:teuthology.orchestra.run.smithi092.stdout:Get:32 http://security.ubuntu.com/ubuntu focal-security/restricted amd64 Packages [2,210 kB] 2023-09-23T18:43:07.121 INFO:teuthology.orchestra.run.smithi092.stdout:Get:33 http://security.ubuntu.com/ubuntu focal-security/restricted Translation-en [309 kB] 2023-09-23T18:43:07.121 INFO:teuthology.orchestra.run.smithi092.stdout:Get:34 http://security.ubuntu.com/ubuntu focal-security/restricted amd64 c-n-f Metadata [580 B] 2023-09-23T18:43:07.121 INFO:teuthology.orchestra.run.smithi092.stdout:Get:35 http://security.ubuntu.com/ubuntu focal-security/universe i386 Packages [615 kB] 2023-09-23T18:43:07.121 INFO:teuthology.orchestra.run.smithi092.stdout:Get:36 http://security.ubuntu.com/ubuntu focal-security/universe amd64 Packages [886 kB] 2023-09-23T18:43:07.167 INFO:teuthology.orchestra.run.smithi092.stdout:Get:37 http://security.ubuntu.com/ubuntu focal-security/universe Translation-en [185 kB] 2023-09-23T18:43:07.583 INFO:teuthology.orchestra.run.smithi092.stdout:Get:38 http://security.ubuntu.com/ubuntu focal-security/universe amd64 c-n-f Metadata [19.1 kB] 2023-09-23T18:43:07.589 INFO:teuthology.orchestra.run.smithi092.stdout:Get:39 http://security.ubuntu.com/ubuntu focal-security/multiverse i386 Packages [7,196 B] 2023-09-23T18:43:07.591 INFO:teuthology.orchestra.run.smithi092.stdout:Get:40 http://security.ubuntu.com/ubuntu focal-security/multiverse amd64 Packages [23.6 kB] 2023-09-23T18:43:07.599 INFO:teuthology.orchestra.run.smithi092.stdout:Get:41 http://security.ubuntu.com/ubuntu focal-security/multiverse Translation-en [5,504 B] 2023-09-23T18:43:07.601 INFO:teuthology.orchestra.run.smithi092.stdout:Get:42 http://security.ubuntu.com/ubuntu focal-security/multiverse amd64 c-n-f Metadata [548 B] 2023-09-23T18:43:10.007 INFO:teuthology.orchestra.run.smithi092.stdout:Fetched 17.4 MB in 3s (6,740 kB/s) 2023-09-23T18:43:10.096 INFO:teuthology.orchestra.run.smithi016.stdout:Fetched 17.4 MB in 3s (6,705 kB/s) 2023-09-23T18:43:11.449 INFO:teuthology.orchestra.run.smithi092.stdout:Reading package lists... 2023-09-23T18:43:11.475 DEBUG:teuthology.orchestra.run.smithi092:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic-hwe-20.04 2023-09-23T18:43:11.554 INFO:teuthology.orchestra.run.smithi092.stdout:Reading package lists... 2023-09-23T18:43:11.570 INFO:teuthology.orchestra.run.smithi016.stdout:Reading package lists... 2023-09-23T18:43:11.595 DEBUG:teuthology.orchestra.run.smithi016:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic-hwe-20.04 2023-09-23T18:43:11.675 INFO:teuthology.orchestra.run.smithi016.stdout:Reading package lists... 2023-09-23T18:43:11.760 INFO:teuthology.orchestra.run.smithi092.stdout:Building dependency tree... 2023-09-23T18:43:11.760 INFO:teuthology.orchestra.run.smithi092.stdout:Reading state information... 2023-09-23T18:43:11.885 INFO:teuthology.orchestra.run.smithi016.stdout:Building dependency tree... 2023-09-23T18:43:11.886 INFO:teuthology.orchestra.run.smithi016.stdout:Reading state information... 2023-09-23T18:43:11.927 INFO:teuthology.orchestra.run.smithi092.stdout:The following additional packages will be installed: 2023-09-23T18:43:11.930 INFO:teuthology.orchestra.run.smithi092.stdout: libdbus-glib-1-2 libevdev2 libimobiledevice6 libplist3 libupower-glib3 2023-09-23T18:43:11.930 INFO:teuthology.orchestra.run.smithi092.stdout: libusbmuxd6 linux-image-5.15.0-84-generic linux-modules-5.15.0-84-generic 2023-09-23T18:43:11.931 INFO:teuthology.orchestra.run.smithi092.stdout: linux-modules-extra-5.15.0-84-generic thermald upower usbmuxd 2023-09-23T18:43:11.932 INFO:teuthology.orchestra.run.smithi092.stdout:Suggested packages: 2023-09-23T18:43:11.932 INFO:teuthology.orchestra.run.smithi092.stdout: libusbmuxd-tools fdutils linux-doc | linux-hwe-5.15-source-5.15.0 2023-09-23T18:43:11.932 INFO:teuthology.orchestra.run.smithi092.stdout: linux-hwe-5.15-tools linux-headers-5.15.0-84-generic 2023-09-23T18:43:11.984 INFO:teuthology.orchestra.run.smithi092.stdout:The following NEW packages will be installed: 2023-09-23T18:43:11.984 INFO:teuthology.orchestra.run.smithi092.stdout: libdbus-glib-1-2 libevdev2 libimobiledevice6 libplist3 libupower-glib3 2023-09-23T18:43:11.984 INFO:teuthology.orchestra.run.smithi092.stdout: libusbmuxd6 linux-image-5.15.0-84-generic linux-image-generic-hwe-20.04 2023-09-23T18:43:11.985 INFO:teuthology.orchestra.run.smithi092.stdout: linux-modules-5.15.0-84-generic linux-modules-extra-5.15.0-84-generic 2023-09-23T18:43:11.985 INFO:teuthology.orchestra.run.smithi092.stdout: thermald upower usbmuxd 2023-09-23T18:43:12.040 INFO:teuthology.orchestra.run.smithi092.stdout:0 upgraded, 13 newly installed, 0 to remove and 266 not upgraded. 2023-09-23T18:43:12.040 INFO:teuthology.orchestra.run.smithi092.stdout:Need to get 90.9 MB of archives. 2023-09-23T18:43:12.040 INFO:teuthology.orchestra.run.smithi092.stdout:After this operation, 497 MB of additional disk space will be used. 2023-09-23T18:43:12.040 INFO:teuthology.orchestra.run.smithi092.stdout:Get:1 http://archive.ubuntu.com/ubuntu focal/main amd64 libdbus-glib-1-2 amd64 0.110-5fakssync1 [59.1 kB] 2023-09-23T18:43:12.068 INFO:teuthology.orchestra.run.smithi016.stdout:The following additional packages will be installed: 2023-09-23T18:43:12.068 INFO:teuthology.orchestra.run.smithi016.stdout: libdbus-glib-1-2 libevdev2 libimobiledevice6 libplist3 libupower-glib3 2023-09-23T18:43:12.068 INFO:teuthology.orchestra.run.smithi016.stdout: libusbmuxd6 linux-image-5.15.0-84-generic linux-modules-5.15.0-84-generic 2023-09-23T18:43:12.068 INFO:teuthology.orchestra.run.smithi016.stdout: linux-modules-extra-5.15.0-84-generic thermald upower usbmuxd 2023-09-23T18:43:12.069 INFO:teuthology.orchestra.run.smithi016.stdout:Suggested packages: 2023-09-23T18:43:12.069 INFO:teuthology.orchestra.run.smithi016.stdout: libusbmuxd-tools fdutils linux-doc | linux-hwe-5.15-source-5.15.0 2023-09-23T18:43:12.069 INFO:teuthology.orchestra.run.smithi016.stdout: linux-hwe-5.15-tools linux-headers-5.15.0-84-generic 2023-09-23T18:43:12.097 INFO:teuthology.orchestra.run.smithi092.stdout:Get:2 http://archive.ubuntu.com/ubuntu focal/main amd64 libplist3 amd64 2.1.0-4build2 [31.6 kB] 2023-09-23T18:43:12.103 INFO:teuthology.orchestra.run.smithi092.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal/main amd64 libusbmuxd6 amd64 2.0.1-2 [19.1 kB] 2023-09-23T18:43:12.107 INFO:teuthology.orchestra.run.smithi092.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal/main amd64 libimobiledevice6 amd64 1.2.1~git20191129.9f79242-1build1 [65.2 kB] 2023-09-23T18:43:12.121 INFO:teuthology.orchestra.run.smithi016.stdout:The following NEW packages will be installed: 2023-09-23T18:43:12.121 INFO:teuthology.orchestra.run.smithi092.stdout:Get:5 http://archive.ubuntu.com/ubuntu focal/main amd64 libupower-glib3 amd64 0.99.11-1build2 [43.2 kB] 2023-09-23T18:43:12.122 INFO:teuthology.orchestra.run.smithi016.stdout: libdbus-glib-1-2 libevdev2 libimobiledevice6 libplist3 libupower-glib3 2023-09-23T18:43:12.122 INFO:teuthology.orchestra.run.smithi016.stdout: libusbmuxd6 linux-image-5.15.0-84-generic linux-image-generic-hwe-20.04 2023-09-23T18:43:12.122 INFO:teuthology.orchestra.run.smithi016.stdout: linux-modules-5.15.0-84-generic linux-modules-extra-5.15.0-84-generic 2023-09-23T18:43:12.122 INFO:teuthology.orchestra.run.smithi016.stdout: thermald upower usbmuxd 2023-09-23T18:43:12.129 INFO:teuthology.orchestra.run.smithi092.stdout:Get:6 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-modules-5.15.0-84-generic amd64 5.15.0-84.93~20.04.1 [20.9 MB] 2023-09-23T18:43:12.304 INFO:teuthology.orchestra.run.smithi016.stdout:0 upgraded, 13 newly installed, 0 to remove and 266 not upgraded. 2023-09-23T18:43:12.305 INFO:teuthology.orchestra.run.smithi016.stdout:Need to get 90.9 MB of archives. 2023-09-23T18:43:12.305 INFO:teuthology.orchestra.run.smithi016.stdout:After this operation, 497 MB of additional disk space will be used. 2023-09-23T18:43:12.305 INFO:teuthology.orchestra.run.smithi016.stdout:Get:1 http://archive.ubuntu.com/ubuntu focal/main amd64 libdbus-glib-1-2 amd64 0.110-5fakssync1 [59.1 kB] 2023-09-23T18:43:12.425 INFO:teuthology.orchestra.run.smithi092.stdout:Get:7 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-image-5.15.0-84-generic amd64 5.15.0-84.93~20.04.1 [11.4 MB] 2023-09-23T18:43:12.520 INFO:teuthology.orchestra.run.smithi092.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-modules-extra-5.15.0-84-generic amd64 5.15.0-84.93~20.04.1 [58.0 MB] 2023-09-23T18:43:12.576 INFO:teuthology.orchestra.run.smithi016.stdout:Get:2 http://archive.ubuntu.com/ubuntu focal/main amd64 libplist3 amd64 2.1.0-4build2 [31.6 kB] 2023-09-23T18:43:12.608 INFO:teuthology.orchestra.run.smithi016.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal/main amd64 libusbmuxd6 amd64 2.0.1-2 [19.1 kB] 2023-09-23T18:43:12.625 INFO:teuthology.orchestra.run.smithi016.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal/main amd64 libimobiledevice6 amd64 1.2.1~git20191129.9f79242-1build1 [65.2 kB] 2023-09-23T18:43:12.698 INFO:teuthology.orchestra.run.smithi016.stdout:Get:5 http://archive.ubuntu.com/ubuntu focal/main amd64 libupower-glib3 amd64 0.99.11-1build2 [43.2 kB] 2023-09-23T18:43:12.735 INFO:teuthology.orchestra.run.smithi016.stdout:Get:6 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-modules-5.15.0-84-generic amd64 5.15.0-84.93~20.04.1 [20.9 MB] 2023-09-23T18:43:13.181 INFO:teuthology.orchestra.run.smithi092.stdout:Get:9 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-image-generic-hwe-20.04 amd64 5.15.0.84.93~20.04.42 [2,692 B] 2023-09-23T18:43:13.256 INFO:teuthology.orchestra.run.smithi092.stdout:Get:10 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 libevdev2 amd64 1.9.0+dfsg-1ubuntu0.2 [31.6 kB] 2023-09-23T18:43:13.257 INFO:teuthology.orchestra.run.smithi092.stdout:Get:11 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 thermald amd64 1.9.1-1ubuntu0.6 [233 kB] 2023-09-23T18:43:13.263 INFO:teuthology.orchestra.run.smithi092.stdout:Get:12 http://archive.ubuntu.com/ubuntu focal/main amd64 upower amd64 0.99.11-1build2 [104 kB] 2023-09-23T18:43:13.265 INFO:teuthology.orchestra.run.smithi092.stdout:Get:13 http://archive.ubuntu.com/ubuntu focal/main amd64 usbmuxd amd64 1.1.1~git20191130.9af2b12-1 [38.4 kB] 2023-09-23T18:43:13.684 INFO:teuthology.orchestra.run.smithi016.stdout:Get:7 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-image-5.15.0-84-generic amd64 5.15.0-84.93~20.04.1 [11.4 MB] 2023-09-23T18:43:13.921 INFO:teuthology.orchestra.run.smithi092.stdout:Fetched 90.9 MB in 1s (71.4 MB/s) 2023-09-23T18:43:13.971 INFO:teuthology.orchestra.run.smithi016.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-modules-extra-5.15.0-84-generic amd64 5.15.0-84.93~20.04.1 [58.0 MB] 2023-09-23T18:43:14.126 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package libdbus-glib-1-2:amd64. 2023-09-23T18:43:15.894 INFO:teuthology.orchestra.run.smithi092.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 104373 files and directories currently installed.) 2023-09-23T18:43:15.899 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../00-libdbus-glib-1-2_0.110-5fakssync1_amd64.deb ... 2023-09-23T18:43:15.926 INFO:teuthology.orchestra.run.smithi016.stdout:Get:9 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-image-generic-hwe-20.04 amd64 5.15.0.84.93~20.04.42 [2,692 B] 2023-09-23T18:43:15.926 INFO:teuthology.orchestra.run.smithi016.stdout:Get:10 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 libevdev2 amd64 1.9.0+dfsg-1ubuntu0.2 [31.6 kB] 2023-09-23T18:43:15.927 INFO:teuthology.orchestra.run.smithi016.stdout:Get:11 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 thermald amd64 1.9.1-1ubuntu0.6 [233 kB] 2023-09-23T18:43:16.023 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking libdbus-glib-1-2:amd64 (0.110-5fakssync1) ... 2023-09-23T18:43:16.095 INFO:teuthology.orchestra.run.smithi016.stdout:Get:12 http://archive.ubuntu.com/ubuntu focal/main amd64 upower amd64 0.99.11-1build2 [104 kB] 2023-09-23T18:43:16.393 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package libplist3:amd64. 2023-09-23T18:43:16.407 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../01-libplist3_2.1.0-4build2_amd64.deb ... 2023-09-23T18:43:16.439 INFO:teuthology.orchestra.run.smithi016.stdout:Get:13 http://archive.ubuntu.com/ubuntu focal/main amd64 usbmuxd amd64 1.1.1~git20191130.9af2b12-1 [38.4 kB] 2023-09-23T18:43:16.451 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking libplist3:amd64 (2.1.0-4build2) ... 2023-09-23T18:43:16.746 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package libusbmuxd6:amd64. 2023-09-23T18:43:16.759 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../02-libusbmuxd6_2.0.1-2_amd64.deb ... 2023-09-23T18:43:16.796 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking libusbmuxd6:amd64 (2.0.1-2) ... 2023-09-23T18:43:17.049 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package libimobiledevice6:amd64. 2023-09-23T18:43:17.065 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../03-libimobiledevice6_1.2.1~git20191129.9f79242-1build1_amd64.deb ... 2023-09-23T18:43:17.107 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking libimobiledevice6:amd64 (1.2.1~git20191129.9f79242-1build1) ... 2023-09-23T18:43:17.136 INFO:teuthology.orchestra.run.smithi016.stdout:Fetched 90.9 MB in 4s (21.0 MB/s) 2023-09-23T18:43:17.373 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package libdbus-glib-1-2:amd64. 2023-09-23T18:43:17.485 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package libupower-glib3:amd64. 2023-09-23T18:43:17.498 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../04-libupower-glib3_0.99.11-1build2_amd64.deb ... 2023-09-23T18:43:17.535 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking libupower-glib3:amd64 (0.99.11-1build2) ... 2023-09-23T18:43:17.838 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package linux-modules-5.15.0-84-generic. 2023-09-23T18:43:17.854 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../05-linux-modules-5.15.0-84-generic_5.15.0-84.93~20.04.1_amd64.deb ... 2023-09-23T18:43:17.897 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking linux-modules-5.15.0-84-generic (5.15.0-84.93~20.04.1) ... 2023-09-23T18:43:19.097 INFO:teuthology.orchestra.run.smithi016.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 104373 files and directories currently installed.) 2023-09-23T18:43:19.102 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../00-libdbus-glib-1-2_0.110-5fakssync1_amd64.deb ... 2023-09-23T18:43:19.270 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking libdbus-glib-1-2:amd64 (0.110-5fakssync1) ... 2023-09-23T18:43:19.690 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package libplist3:amd64. 2023-09-23T18:43:19.704 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../01-libplist3_2.1.0-4build2_amd64.deb ... 2023-09-23T18:43:19.756 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking libplist3:amd64 (2.1.0-4build2) ... 2023-09-23T18:43:20.126 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package libusbmuxd6:amd64. 2023-09-23T18:43:20.140 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../02-libusbmuxd6_2.0.1-2_amd64.deb ... 2023-09-23T18:43:20.204 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking libusbmuxd6:amd64 (2.0.1-2) ... 2023-09-23T18:43:20.406 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package linux-image-5.15.0-84-generic. 2023-09-23T18:43:20.420 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../06-linux-image-5.15.0-84-generic_5.15.0-84.93~20.04.1_amd64.deb ... 2023-09-23T18:43:20.521 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking linux-image-5.15.0-84-generic (5.15.0-84.93~20.04.1) ... 2023-09-23T18:43:20.557 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package libimobiledevice6:amd64. 2023-09-23T18:43:20.573 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../03-libimobiledevice6_1.2.1~git20191129.9f79242-1build1_amd64.deb ... 2023-09-23T18:43:20.621 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking libimobiledevice6:amd64 (1.2.1~git20191129.9f79242-1build1) ... 2023-09-23T18:43:20.990 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package libupower-glib3:amd64. 2023-09-23T18:43:21.003 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../04-libupower-glib3_0.99.11-1build2_amd64.deb ... 2023-09-23T18:43:21.026 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package linux-modules-extra-5.15.0-84-generic. 2023-09-23T18:43:21.040 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../07-linux-modules-extra-5.15.0-84-generic_5.15.0-84.93~20.04.1_amd64.deb ... 2023-09-23T18:43:21.040 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking libupower-glib3:amd64 (0.99.11-1build2) ... 2023-09-23T18:43:21.084 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking linux-modules-extra-5.15.0-84-generic (5.15.0-84.93~20.04.1) ... 2023-09-23T18:43:21.377 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package linux-modules-5.15.0-84-generic. 2023-09-23T18:43:21.391 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../05-linux-modules-5.15.0-84-generic_5.15.0-84.93~20.04.1_amd64.deb ... 2023-09-23T18:43:21.435 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking linux-modules-5.15.0-84-generic (5.15.0-84.93~20.04.1) ... 2023-09-23T18:43:24.044 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package linux-image-5.15.0-84-generic. 2023-09-23T18:43:24.057 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../06-linux-image-5.15.0-84-generic_5.15.0-84.93~20.04.1_amd64.deb ... 2023-09-23T18:43:24.157 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking linux-image-5.15.0-84-generic (5.15.0-84.93~20.04.1) ... 2023-09-23T18:43:24.664 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package linux-modules-extra-5.15.0-84-generic. 2023-09-23T18:43:24.678 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../07-linux-modules-extra-5.15.0-84-generic_5.15.0-84.93~20.04.1_amd64.deb ... 2023-09-23T18:43:24.722 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking linux-modules-extra-5.15.0-84-generic (5.15.0-84.93~20.04.1) ... 2023-09-23T18:43:27.919 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package linux-image-generic-hwe-20.04. 2023-09-23T18:43:27.934 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../08-linux-image-generic-hwe-20.04_5.15.0.84.93~20.04.42_amd64.deb ... 2023-09-23T18:43:27.977 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking linux-image-generic-hwe-20.04 (5.15.0.84.93~20.04.42) ... 2023-09-23T18:43:28.314 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package libevdev2:amd64. 2023-09-23T18:43:28.328 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../09-libevdev2_1.9.0+dfsg-1ubuntu0.2_amd64.deb ... 2023-09-23T18:43:28.372 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking libevdev2:amd64 (1.9.0+dfsg-1ubuntu0.2) ... 2023-09-23T18:43:28.766 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package thermald. 2023-09-23T18:43:28.780 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../10-thermald_1.9.1-1ubuntu0.6_amd64.deb ... 2023-09-23T18:43:28.825 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking thermald (1.9.1-1ubuntu0.6) ... 2023-09-23T18:43:29.371 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package upower. 2023-09-23T18:43:29.385 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../11-upower_0.99.11-1build2_amd64.deb ... 2023-09-23T18:43:29.451 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking upower (0.99.11-1build2) ... 2023-09-23T18:43:29.851 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package usbmuxd. 2023-09-23T18:43:29.864 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../12-usbmuxd_1.1.1~git20191130.9af2b12-1_amd64.deb ... 2023-09-23T18:43:29.909 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking usbmuxd (1.1.1~git20191130.9af2b12-1) ... 2023-09-23T18:43:30.209 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up libplist3:amd64 (2.1.0-4build2) ... 2023-09-23T18:43:30.330 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up libdbus-glib-1-2:amd64 (0.110-5fakssync1) ... 2023-09-23T18:43:30.456 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up libusbmuxd6:amd64 (2.0.1-2) ... 2023-09-23T18:43:30.582 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up libupower-glib3:amd64 (0.99.11-1build2) ... 2023-09-23T18:43:30.709 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up libimobiledevice6:amd64 (1.2.1~git20191129.9f79242-1build1) ... 2023-09-23T18:43:30.826 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up libevdev2:amd64 (1.9.0+dfsg-1ubuntu0.2) ... 2023-09-23T18:43:30.952 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up upower (0.99.11-1build2) ... 2023-09-23T18:43:31.548 INFO:teuthology.orchestra.run.smithi092.stdout:upower.service is a disabled or a static unit, not starting it. 2023-09-23T18:43:31.598 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up usbmuxd (1.1.1~git20191130.9af2b12-1) ... 2023-09-23T18:43:31.761 INFO:teuthology.orchestra.run.smithi092.stdout:Warning: The home dir /var/lib/usbmux you specified can't be accessed: No such file or directory 2023-09-23T18:43:31.771 INFO:teuthology.orchestra.run.smithi092.stdout:Adding system user `usbmux' (UID 114) ... 2023-09-23T18:43:31.772 INFO:teuthology.orchestra.run.smithi092.stdout:Adding new user `usbmux' (UID 114) with group `plugdev' ... 2023-09-23T18:43:31.832 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package linux-image-generic-hwe-20.04. 2023-09-23T18:43:31.847 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../08-linux-image-generic-hwe-20.04_5.15.0.84.93~20.04.42_amd64.deb ... 2023-09-23T18:43:31.890 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking linux-image-generic-hwe-20.04 (5.15.0.84.93~20.04.42) ... 2023-09-23T18:43:32.235 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package libevdev2:amd64. 2023-09-23T18:43:32.249 INFO:teuthology.orchestra.run.smithi092.stdout:Not creating home directory `/var/lib/usbmux'. 2023-09-23T18:43:32.251 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../09-libevdev2_1.9.0+dfsg-1ubuntu0.2_amd64.deb ... 2023-09-23T18:43:32.293 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking libevdev2:amd64 (1.9.0+dfsg-1ubuntu0.2) ... 2023-09-23T18:43:32.471 INFO:teuthology.orchestra.run.smithi092.stdout:usbmuxd.service is a disabled or a static unit, not starting it. 2023-09-23T18:43:32.512 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up thermald (1.9.1-1ubuntu0.6) ... 2023-09-23T18:43:32.672 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package thermald. 2023-09-23T18:43:32.689 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../10-thermald_1.9.1-1ubuntu0.6_amd64.deb ... 2023-09-23T18:43:32.730 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking thermald (1.9.1-1ubuntu0.6) ... 2023-09-23T18:43:32.824 INFO:teuthology.orchestra.run.smithi092.stdout:Created symlink /etc/systemd/system/dbus-org.freedesktop.thermald.service -> /lib/systemd/system/thermald.service. 2023-09-23T18:43:32.824 INFO:teuthology.orchestra.run.smithi092.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/thermald.service -> /lib/systemd/system/thermald.service. 2023-09-23T18:43:33.293 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package upower. 2023-09-23T18:43:33.300 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up linux-image-5.15.0-84-generic (5.15.0-84.93~20.04.1) ... 2023-09-23T18:43:33.308 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../11-upower_0.99.11-1build2_amd64.deb ... 2023-09-23T18:43:33.370 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking upower (0.99.11-1build2) ... 2023-09-23T18:43:33.797 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package usbmuxd. 2023-09-23T18:43:33.812 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../12-usbmuxd_1.1.1~git20191130.9af2b12-1_amd64.deb ... 2023-09-23T18:43:33.881 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking usbmuxd (1.1.1~git20191130.9af2b12-1) ... 2023-09-23T18:43:34.180 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up libplist3:amd64 (2.1.0-4build2) ... 2023-09-23T18:43:34.335 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up libdbus-glib-1-2:amd64 (0.110-5fakssync1) ... 2023-09-23T18:43:34.461 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up libusbmuxd6:amd64 (2.0.1-2) ... 2023-09-23T18:43:34.608 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up libupower-glib3:amd64 (0.99.11-1build2) ... 2023-09-23T18:43:34.622 INFO:teuthology.orchestra.run.smithi092.stdout:I: /vmlinuz is now a symlink to boot/vmlinuz-5.15.0-84-generic 2023-09-23T18:43:34.622 INFO:teuthology.orchestra.run.smithi092.stdout:I: /initrd.img is now a symlink to boot/initrd.img-5.15.0-84-generic 2023-09-23T18:43:34.738 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up libimobiledevice6:amd64 (1.2.1~git20191129.9f79242-1build1) ... 2023-09-23T18:43:34.785 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up linux-modules-extra-5.15.0-84-generic (5.15.0-84.93~20.04.1) ... 2023-09-23T18:43:34.864 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up libevdev2:amd64 (1.9.0+dfsg-1ubuntu0.2) ... 2023-09-23T18:43:34.986 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up upower (0.99.11-1build2) ... 2023-09-23T18:43:35.527 INFO:teuthology.orchestra.run.smithi016.stdout:upower.service is a disabled or a static unit, not starting it. 2023-09-23T18:43:35.578 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up usbmuxd (1.1.1~git20191130.9af2b12-1) ... 2023-09-23T18:43:35.773 INFO:teuthology.orchestra.run.smithi016.stdout:Warning: The home dir /var/lib/usbmux you specified can't be accessed: No such file or directory 2023-09-23T18:43:35.781 INFO:teuthology.orchestra.run.smithi016.stdout:Adding system user `usbmux' (UID 114) ... 2023-09-23T18:43:35.781 INFO:teuthology.orchestra.run.smithi016.stdout:Adding new user `usbmux' (UID 114) with group `plugdev' ... 2023-09-23T18:43:36.237 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up linux-image-generic-hwe-20.04 (5.15.0.84.93~20.04.42) ... 2023-09-23T18:43:36.278 INFO:teuthology.orchestra.run.smithi016.stdout:Not creating home directory `/var/lib/usbmux'. 2023-09-23T18:43:36.438 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up linux-modules-5.15.0-84-generic (5.15.0-84.93~20.04.1) ... 2023-09-23T18:43:36.502 INFO:teuthology.orchestra.run.smithi016.stdout:usbmuxd.service is a disabled or a static unit, not starting it. 2023-09-23T18:43:36.575 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up thermald (1.9.1-1ubuntu0.6) ... 2023-09-23T18:43:36.964 INFO:teuthology.orchestra.run.smithi016.stdout:Created symlink /etc/systemd/system/dbus-org.freedesktop.thermald.service -> /lib/systemd/system/thermald.service. 2023-09-23T18:43:36.964 INFO:teuthology.orchestra.run.smithi016.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/thermald.service -> /lib/systemd/system/thermald.service. 2023-09-23T18:43:37.590 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up linux-image-5.15.0-84-generic (5.15.0-84.93~20.04.1) ... 2023-09-23T18:43:37.822 INFO:teuthology.orchestra.run.smithi092.stdout:Processing triggers for libc-bin (2.31-0ubuntu9.9) ... 2023-09-23T18:43:38.078 INFO:teuthology.orchestra.run.smithi092.stdout:Processing triggers for man-db (2.9.1-1) ... 2023-09-23T18:43:38.823 INFO:teuthology.orchestra.run.smithi092.stdout:Processing triggers for dbus (1.12.16-2ubuntu2.2) ... 2023-09-23T18:43:38.914 INFO:teuthology.orchestra.run.smithi092.stdout:Processing triggers for linux-image-5.15.0-84-generic (5.15.0-84.93~20.04.1) ... 2023-09-23T18:43:38.975 INFO:teuthology.orchestra.run.smithi092.stdout:/etc/kernel/postinst.d/initramfs-tools: 2023-09-23T18:43:38.975 INFO:teuthology.orchestra.run.smithi092.stdout:update-initramfs: Generating /boot/initrd.img-5.15.0-84-generic 2023-09-23T18:43:39.012 INFO:teuthology.orchestra.run.smithi016.stdout:I: /vmlinuz is now a symlink to boot/vmlinuz-5.15.0-84-generic 2023-09-23T18:43:39.012 INFO:teuthology.orchestra.run.smithi016.stdout:I: /initrd.img is now a symlink to boot/initrd.img-5.15.0-84-generic 2023-09-23T18:43:39.424 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up linux-modules-extra-5.15.0-84-generic (5.15.0-84.93~20.04.1) ... 2023-09-23T18:43:40.859 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up linux-image-generic-hwe-20.04 (5.15.0.84.93~20.04.42) ... 2023-09-23T18:43:40.985 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up linux-modules-5.15.0-84-generic (5.15.0-84.93~20.04.1) ... 2023-09-23T18:43:42.403 INFO:teuthology.orchestra.run.smithi016.stdout:Processing triggers for libc-bin (2.31-0ubuntu9.9) ... 2023-09-23T18:43:42.700 INFO:teuthology.orchestra.run.smithi016.stdout:Processing triggers for man-db (2.9.1-1) ... 2023-09-23T18:43:43.512 INFO:teuthology.orchestra.run.smithi016.stdout:Processing triggers for dbus (1.12.16-2ubuntu2.2) ... 2023-09-23T18:43:43.637 INFO:teuthology.orchestra.run.smithi016.stdout:Processing triggers for linux-image-5.15.0-84-generic (5.15.0-84.93~20.04.1) ... 2023-09-23T18:43:43.721 INFO:teuthology.orchestra.run.smithi016.stdout:/etc/kernel/postinst.d/initramfs-tools: 2023-09-23T18:43:43.722 INFO:teuthology.orchestra.run.smithi016.stdout:update-initramfs: Generating /boot/initrd.img-5.15.0-84-generic 2023-09-23T18:44:04.271 INFO:teuthology.orchestra.run.smithi092.stdout:/etc/kernel/postinst.d/zz-update-grub: 2023-09-23T18:44:04.272 INFO:teuthology.orchestra.run.smithi092.stdout:Sourcing file `/etc/default/grub' 2023-09-23T18:44:04.293 INFO:teuthology.orchestra.run.smithi092.stdout:Sourcing file `/etc/default/grub.d/init-select.cfg' 2023-09-23T18:44:04.324 INFO:teuthology.orchestra.run.smithi092.stdout:Generating grub configuration file ... 2023-09-23T18:44:04.617 INFO:teuthology.orchestra.run.smithi092.stdout:Found linux image: /boot/vmlinuz-5.15.0-84-generic 2023-09-23T18:44:04.638 INFO:teuthology.orchestra.run.smithi092.stdout:Found initrd image: /boot/initrd.img-5.15.0-84-generic 2023-09-23T18:44:05.028 INFO:teuthology.orchestra.run.smithi092.stdout:Found linux image: /boot/vmlinuz-5.4.0-124-generic 2023-09-23T18:44:05.039 INFO:teuthology.orchestra.run.smithi092.stdout:Found initrd image: /boot/initrd.img-5.4.0-124-generic 2023-09-23T18:44:07.700 INFO:teuthology.orchestra.run.smithi092.stdout:done 2023-09-23T18:44:08.104 DEBUG:teuthology.orchestra.run.smithi092:> dpkg -s linux-image-generic-hwe-20.04 2023-09-23T18:44:08.154 INFO:teuthology.orchestra.run.smithi092.stdout:Package: linux-image-generic-hwe-20.04 2023-09-23T18:44:08.154 INFO:teuthology.orchestra.run.smithi092.stdout:Status: install ok installed 2023-09-23T18:44:08.154 INFO:teuthology.orchestra.run.smithi092.stdout:Priority: optional 2023-09-23T18:44:08.154 INFO:teuthology.orchestra.run.smithi092.stdout:Section: kernel 2023-09-23T18:44:08.154 INFO:teuthology.orchestra.run.smithi092.stdout:Installed-Size: 20 2023-09-23T18:44:08.154 INFO:teuthology.orchestra.run.smithi092.stdout:Maintainer: Ubuntu Kernel Team 2023-09-23T18:44:08.155 INFO:teuthology.orchestra.run.smithi092.stdout:Architecture: amd64 2023-09-23T18:44:08.155 INFO:teuthology.orchestra.run.smithi092.stdout:Source: linux-meta-hwe-5.15 2023-09-23T18:44:08.155 INFO:teuthology.orchestra.run.smithi092.stdout:Version: 5.15.0.84.93~20.04.42 2023-09-23T18:44:08.155 INFO:teuthology.orchestra.run.smithi092.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.1), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-84), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.1) 2023-09-23T18:44:08.155 INFO:teuthology.orchestra.run.smithi092.stdout:Depends: linux-image-5.15.0-84-generic, linux-modules-extra-5.15.0-84-generic, linux-firmware, intel-microcode, amd64-microcode 2023-09-23T18:44:08.155 INFO:teuthology.orchestra.run.smithi092.stdout:Recommends: thermald 2023-09-23T18:44:08.155 INFO:teuthology.orchestra.run.smithi092.stdout:Description: Generic Linux kernel image 2023-09-23T18:44:08.155 INFO:teuthology.orchestra.run.smithi092.stdout: This package will always depend on the latest generic kernel image 2023-09-23T18:44:08.155 INFO:teuthology.orchestra.run.smithi092.stdout: available. 2023-09-23T18:44:08.155 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.4.0-124-generic Expected: 5.15.0-84-generic 2023-09-23T18:44:08.156 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2023-09-23T18:44:08.156 DEBUG:teuthology.task.kernel:src is distro, skipping download 2023-09-23T18:44:08.156 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2023-09-23T18:44:08.156 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi092.front.sepia.ceph.com, path=None, version=distro) 2023-09-23T18:44:08.156 DEBUG:teuthology.orchestra.run.smithi092:> sudo apt-get clean 2023-09-23T18:44:08.248 DEBUG:teuthology.orchestra.run.smithi092:> sudo apt-get update 2023-09-23T18:44:08.382 INFO:teuthology.orchestra.run.smithi092.stdout:Hit:1 http://archive.ubuntu.com/ubuntu focal InRelease 2023-09-23T18:44:08.388 INFO:teuthology.orchestra.run.smithi092.stdout:Hit:2 http://archive.ubuntu.com/ubuntu focal-updates InRelease 2023-09-23T18:44:08.405 INFO:teuthology.orchestra.run.smithi092.stdout:Hit:3 http://archive.ubuntu.com/ubuntu focal-backports InRelease 2023-09-23T18:44:08.501 INFO:teuthology.orchestra.run.smithi092.stdout:Hit:4 http://security.ubuntu.com/ubuntu focal-security InRelease 2023-09-23T18:44:09.968 INFO:teuthology.orchestra.run.smithi016.stdout:/etc/kernel/postinst.d/zz-update-grub: 2023-09-23T18:44:09.968 INFO:teuthology.orchestra.run.smithi016.stdout:Sourcing file `/etc/default/grub' 2023-09-23T18:44:09.987 INFO:teuthology.orchestra.run.smithi016.stdout:Sourcing file `/etc/default/grub.d/init-select.cfg' 2023-09-23T18:44:10.006 INFO:teuthology.orchestra.run.smithi016.stdout:Generating grub configuration file ... 2023-09-23T18:44:10.134 INFO:teuthology.orchestra.run.smithi092.stdout:Reading package lists... 2023-09-23T18:44:10.158 DEBUG:teuthology.orchestra.run.smithi092:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic-hwe-20.04 2023-09-23T18:44:10.226 INFO:teuthology.orchestra.run.smithi092.stdout:Reading package lists... 2023-09-23T18:44:10.425 INFO:teuthology.orchestra.run.smithi092.stdout:Building dependency tree... 2023-09-23T18:44:10.426 INFO:teuthology.orchestra.run.smithi092.stdout:Reading state information... 2023-09-23T18:44:10.429 INFO:teuthology.orchestra.run.smithi016.stdout:Found linux image: /boot/vmlinuz-5.15.0-84-generic 2023-09-23T18:44:10.484 INFO:teuthology.orchestra.run.smithi016.stdout:Found initrd image: /boot/initrd.img-5.15.0-84-generic 2023-09-23T18:44:10.640 INFO:teuthology.orchestra.run.smithi092.stdout:linux-image-generic-hwe-20.04 is already the newest version (5.15.0.84.93~20.04.42). 2023-09-23T18:44:10.640 INFO:teuthology.orchestra.run.smithi092.stdout:0 upgraded, 0 newly installed, 0 to remove and 266 not upgraded. 2023-09-23T18:44:10.642 DEBUG:teuthology.orchestra.run.smithi092:> dpkg -s linux-image-generic-hwe-20.04 2023-09-23T18:44:10.662 INFO:teuthology.orchestra.run.smithi092.stdout:Package: linux-image-generic-hwe-20.04 2023-09-23T18:44:10.662 INFO:teuthology.orchestra.run.smithi092.stdout:Status: install ok installed 2023-09-23T18:44:10.663 INFO:teuthology.orchestra.run.smithi092.stdout:Priority: optional 2023-09-23T18:44:10.663 INFO:teuthology.orchestra.run.smithi092.stdout:Section: kernel 2023-09-23T18:44:10.663 INFO:teuthology.orchestra.run.smithi092.stdout:Installed-Size: 20 2023-09-23T18:44:10.663 INFO:teuthology.orchestra.run.smithi092.stdout:Maintainer: Ubuntu Kernel Team 2023-09-23T18:44:10.663 INFO:teuthology.orchestra.run.smithi092.stdout:Architecture: amd64 2023-09-23T18:44:10.663 INFO:teuthology.orchestra.run.smithi092.stdout:Source: linux-meta-hwe-5.15 2023-09-23T18:44:10.663 INFO:teuthology.orchestra.run.smithi092.stdout:Version: 5.15.0.84.93~20.04.42 2023-09-23T18:44:10.664 INFO:teuthology.orchestra.run.smithi092.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.1), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-84), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.1) 2023-09-23T18:44:10.664 INFO:teuthology.orchestra.run.smithi092.stdout:Depends: linux-image-5.15.0-84-generic, linux-modules-extra-5.15.0-84-generic, linux-firmware, intel-microcode, amd64-microcode 2023-09-23T18:44:10.664 INFO:teuthology.orchestra.run.smithi092.stdout:Recommends: thermald 2023-09-23T18:44:10.664 INFO:teuthology.orchestra.run.smithi092.stdout:Description: Generic Linux kernel image 2023-09-23T18:44:10.664 INFO:teuthology.orchestra.run.smithi092.stdout: This package will always depend on the latest generic kernel image 2023-09-23T18:44:10.664 INFO:teuthology.orchestra.run.smithi092.stdout: available. 2023-09-23T18:44:10.664 DEBUG:teuthology.orchestra.run.smithi092:> mktemp 2023-09-23T18:44:10.669 INFO:teuthology.orchestra.run.smithi092.stdout:/tmp/tmp.oHgqplrRFi 2023-09-23T18:44:10.670 DEBUG:teuthology.orchestra.run.smithi092:> sudo cp /boot/grub/grub.cfg /tmp/tmp.oHgqplrRFi 2023-09-23T18:44:10.726 DEBUG:teuthology.orchestra.run.smithi092:> sudo chmod 0666 /tmp/tmp.oHgqplrRFi 2023-09-23T18:44:10.767 DEBUG:teuthology.orchestra.remote:smithi092:/tmp/tmp.oHgqplrRFi is 9KB 2023-09-23T18:44:10.781 DEBUG:teuthology.orchestra.run.smithi092:> rm -fr /tmp/tmp.oHgqplrRFi 2023-09-23T18:44:10.788 DEBUG:teuthology.orchestra.run.smithi092:> sudo rm -f -- /etc/grub.d/01_ceph_kernel 2023-09-23T18:44:10.846 DEBUG:teuthology.orchestra.run.smithi092:> set -ex 2023-09-23T18:44:10.846 DEBUG:teuthology.orchestra.run.smithi092:> sudo dd of=/etc/grub.d/01_ceph_kernel 2023-09-23T18:44:10.846 DEBUG:teuthology.orchestra.run.smithi092:> sudo chmod 755 /etc/grub.d/01_ceph_kernel 2023-09-23T18:44:10.847 INFO:teuthology.orchestra.run.smithi016.stdout:Found linux image: /boot/vmlinuz-5.4.0-124-generic 2023-09-23T18:44:10.858 INFO:teuthology.orchestra.run.smithi016.stdout:Found initrd image: /boot/initrd.img-5.4.0-124-generic 2023-09-23T18:44:10.876 INFO:teuthology.task.kernel:Distro Kernel Version: 5.15.0-84-generic 2023-09-23T18:44:10.876 DEBUG:teuthology.orchestra.run.smithi092:> sudo update-grub 2023-09-23T18:44:12.053 INFO:teuthology.orchestra.run.smithi092.stderr:Sourcing file `/etc/default/grub' 2023-09-23T18:44:12.064 INFO:teuthology.orchestra.run.smithi092.stderr:Sourcing file `/etc/default/grub.d/init-select.cfg' 2023-09-23T18:44:12.081 INFO:teuthology.orchestra.run.smithi092.stderr:Generating grub configuration file ... 2023-09-23T18:44:12.402 INFO:teuthology.orchestra.run.smithi092.stderr:Found linux image: /boot/vmlinuz-5.15.0-84-generic 2023-09-23T18:44:12.413 INFO:teuthology.orchestra.run.smithi092.stderr:Found initrd image: /boot/initrd.img-5.15.0-84-generic 2023-09-23T18:44:12.768 INFO:teuthology.orchestra.run.smithi092.stderr:Found linux image: /boot/vmlinuz-5.4.0-124-generic 2023-09-23T18:44:12.780 INFO:teuthology.orchestra.run.smithi092.stderr:Found initrd image: /boot/initrd.img-5.4.0-124-generic 2023-09-23T18:44:13.768 INFO:teuthology.orchestra.run.smithi016.stdout:done 2023-09-23T18:44:14.213 DEBUG:teuthology.orchestra.run.smithi016:> dpkg -s linux-image-generic-hwe-20.04 2023-09-23T18:44:14.265 INFO:teuthology.orchestra.run.smithi016.stdout:Package: linux-image-generic-hwe-20.04 2023-09-23T18:44:14.265 INFO:teuthology.orchestra.run.smithi016.stdout:Status: install ok installed 2023-09-23T18:44:14.266 INFO:teuthology.orchestra.run.smithi016.stdout:Priority: optional 2023-09-23T18:44:14.266 INFO:teuthology.orchestra.run.smithi016.stdout:Section: kernel 2023-09-23T18:44:14.266 INFO:teuthology.orchestra.run.smithi016.stdout:Installed-Size: 20 2023-09-23T18:44:14.266 INFO:teuthology.orchestra.run.smithi016.stdout:Maintainer: Ubuntu Kernel Team 2023-09-23T18:44:14.266 INFO:teuthology.orchestra.run.smithi016.stdout:Architecture: amd64 2023-09-23T18:44:14.266 INFO:teuthology.orchestra.run.smithi016.stdout:Source: linux-meta-hwe-5.15 2023-09-23T18:44:14.266 INFO:teuthology.orchestra.run.smithi016.stdout:Version: 5.15.0.84.93~20.04.42 2023-09-23T18:44:14.266 INFO:teuthology.orchestra.run.smithi016.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.1), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-84), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.1) 2023-09-23T18:44:14.266 INFO:teuthology.orchestra.run.smithi016.stdout:Depends: linux-image-5.15.0-84-generic, linux-modules-extra-5.15.0-84-generic, linux-firmware, intel-microcode, amd64-microcode 2023-09-23T18:44:14.266 INFO:teuthology.orchestra.run.smithi016.stdout:Recommends: thermald 2023-09-23T18:44:14.267 INFO:teuthology.orchestra.run.smithi016.stdout:Description: Generic Linux kernel image 2023-09-23T18:44:14.267 INFO:teuthology.orchestra.run.smithi016.stdout: This package will always depend on the latest generic kernel image 2023-09-23T18:44:14.267 INFO:teuthology.orchestra.run.smithi016.stdout: available. 2023-09-23T18:44:14.267 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.4.0-124-generic Expected: 5.15.0-84-generic 2023-09-23T18:44:14.267 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2023-09-23T18:44:14.267 DEBUG:teuthology.task.kernel:src is distro, skipping download 2023-09-23T18:44:14.267 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2023-09-23T18:44:14.267 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi016.front.sepia.ceph.com, path=None, version=distro) 2023-09-23T18:44:14.267 DEBUG:teuthology.orchestra.run.smithi016:> sudo apt-get clean 2023-09-23T18:44:14.351 DEBUG:teuthology.orchestra.run.smithi016:> sudo apt-get update 2023-09-23T18:44:14.436 INFO:teuthology.orchestra.run.smithi092.stderr:done 2023-09-23T18:44:14.438 DEBUG:teuthology.orchestra.run.smithi092:> sudo shutdown -r now 2023-09-23T18:44:14.479 INFO:teuthology.orchestra.run.smithi016.stdout:Hit:1 http://archive.ubuntu.com/ubuntu focal InRelease 2023-09-23T18:44:14.479 INFO:teuthology.orchestra.run.smithi016.stdout:Hit:2 http://security.ubuntu.com/ubuntu focal-security InRelease 2023-09-23T18:44:14.484 INFO:teuthology.orchestra.run.smithi016.stdout:Hit:3 http://archive.ubuntu.com/ubuntu focal-updates InRelease 2023-09-23T18:44:14.501 INFO:teuthology.orchestra.run.smithi016.stdout:Hit:4 http://archive.ubuntu.com/ubuntu focal-backports InRelease 2023-09-23T18:44:16.261 INFO:teuthology.orchestra.run.smithi016.stdout:Reading package lists... 2023-09-23T18:44:16.287 DEBUG:teuthology.orchestra.run.smithi016:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic-hwe-20.04 2023-09-23T18:44:16.353 INFO:teuthology.orchestra.run.smithi016.stdout:Reading package lists... 2023-09-23T18:44:16.558 INFO:teuthology.orchestra.run.smithi016.stdout:Building dependency tree... 2023-09-23T18:44:16.558 INFO:teuthology.orchestra.run.smithi016.stdout:Reading state information... 2023-09-23T18:44:16.782 INFO:teuthology.orchestra.run.smithi016.stdout:linux-image-generic-hwe-20.04 is already the newest version (5.15.0.84.93~20.04.42). 2023-09-23T18:44:16.782 INFO:teuthology.orchestra.run.smithi016.stdout:0 upgraded, 0 newly installed, 0 to remove and 266 not upgraded. 2023-09-23T18:44:16.783 DEBUG:teuthology.orchestra.run.smithi016:> dpkg -s linux-image-generic-hwe-20.04 2023-09-23T18:44:16.800 INFO:teuthology.orchestra.run.smithi016.stdout:Package: linux-image-generic-hwe-20.04 2023-09-23T18:44:16.801 INFO:teuthology.orchestra.run.smithi016.stdout:Status: install ok installed 2023-09-23T18:44:16.801 INFO:teuthology.orchestra.run.smithi016.stdout:Priority: optional 2023-09-23T18:44:16.801 INFO:teuthology.orchestra.run.smithi016.stdout:Section: kernel 2023-09-23T18:44:16.801 INFO:teuthology.orchestra.run.smithi016.stdout:Installed-Size: 20 2023-09-23T18:44:16.801 INFO:teuthology.orchestra.run.smithi016.stdout:Maintainer: Ubuntu Kernel Team 2023-09-23T18:44:16.802 INFO:teuthology.orchestra.run.smithi016.stdout:Architecture: amd64 2023-09-23T18:44:16.802 INFO:teuthology.orchestra.run.smithi016.stdout:Source: linux-meta-hwe-5.15 2023-09-23T18:44:16.802 INFO:teuthology.orchestra.run.smithi016.stdout:Version: 5.15.0.84.93~20.04.42 2023-09-23T18:44:16.802 INFO:teuthology.orchestra.run.smithi016.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.1), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-84), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.1) 2023-09-23T18:44:16.802 INFO:teuthology.orchestra.run.smithi016.stdout:Depends: linux-image-5.15.0-84-generic, linux-modules-extra-5.15.0-84-generic, linux-firmware, intel-microcode, amd64-microcode 2023-09-23T18:44:16.802 INFO:teuthology.orchestra.run.smithi016.stdout:Recommends: thermald 2023-09-23T18:44:16.802 INFO:teuthology.orchestra.run.smithi016.stdout:Description: Generic Linux kernel image 2023-09-23T18:44:16.803 INFO:teuthology.orchestra.run.smithi016.stdout: This package will always depend on the latest generic kernel image 2023-09-23T18:44:16.803 INFO:teuthology.orchestra.run.smithi016.stdout: available. 2023-09-23T18:44:16.803 DEBUG:teuthology.orchestra.run.smithi016:> mktemp 2023-09-23T18:44:16.810 INFO:teuthology.orchestra.run.smithi016.stdout:/tmp/tmp.bxbmFcwYTx 2023-09-23T18:44:16.811 DEBUG:teuthology.orchestra.run.smithi016:> sudo cp /boot/grub/grub.cfg /tmp/tmp.bxbmFcwYTx 2023-09-23T18:44:16.824 DEBUG:teuthology.orchestra.run.smithi016:> sudo chmod 0666 /tmp/tmp.bxbmFcwYTx 2023-09-23T18:44:16.912 DEBUG:teuthology.orchestra.remote:smithi016:/tmp/tmp.bxbmFcwYTx is 9KB 2023-09-23T18:44:16.925 DEBUG:teuthology.orchestra.run.smithi016:> rm -fr /tmp/tmp.bxbmFcwYTx 2023-09-23T18:44:16.931 DEBUG:teuthology.orchestra.run.smithi016:> sudo rm -f -- /etc/grub.d/01_ceph_kernel 2023-09-23T18:44:16.991 DEBUG:teuthology.orchestra.run.smithi016:> set -ex 2023-09-23T18:44:16.991 DEBUG:teuthology.orchestra.run.smithi016:> sudo dd of=/etc/grub.d/01_ceph_kernel 2023-09-23T18:44:16.991 DEBUG:teuthology.orchestra.run.smithi016:> sudo chmod 755 /etc/grub.d/01_ceph_kernel 2023-09-23T18:44:17.020 INFO:teuthology.task.kernel:Distro Kernel Version: 5.15.0-84-generic 2023-09-23T18:44:17.020 DEBUG:teuthology.orchestra.run.smithi016:> sudo update-grub 2023-09-23T18:44:18.483 INFO:teuthology.orchestra.run.smithi016.stderr:Sourcing file `/etc/default/grub' 2023-09-23T18:44:18.494 INFO:teuthology.orchestra.run.smithi016.stderr:Sourcing file `/etc/default/grub.d/init-select.cfg' 2023-09-23T18:44:18.505 INFO:teuthology.orchestra.run.smithi016.stderr:Generating grub configuration file ... 2023-09-23T18:44:18.765 INFO:teuthology.orchestra.run.smithi016.stderr:Found linux image: /boot/vmlinuz-5.15.0-84-generic 2023-09-23T18:44:18.777 INFO:teuthology.orchestra.run.smithi016.stderr:Found initrd image: /boot/initrd.img-5.15.0-84-generic 2023-09-23T18:44:19.155 INFO:teuthology.orchestra.run.smithi016.stderr:Found linux image: /boot/vmlinuz-5.4.0-124-generic 2023-09-23T18:44:19.166 INFO:teuthology.orchestra.run.smithi016.stderr:Found initrd image: /boot/initrd.img-5.4.0-124-generic 2023-09-23T18:44:20.814 INFO:teuthology.orchestra.run.smithi016.stderr:done 2023-09-23T18:44:20.816 DEBUG:teuthology.orchestra.run.smithi016:> sudo shutdown -r now 2023-09-23T18:44:44.446 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2023-09-23T18:44:44.446 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi092.front.sepia.ceph.com' 2023-09-23T18:44:44.447 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi092.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T18:44:50.824 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2023-09-23T18:44:50.825 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi016.front.sepia.ceph.com' 2023-09-23T18:44:50.825 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi016.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T18:45:02.974 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.92 2023-09-23T18:45:09.374 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.16 2023-09-23T18:45:11.977 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi092.front.sepia.ceph.com' 2023-09-23T18:45:11.978 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi092.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T18:45:15.034 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.92 2023-09-23T18:45:18.378 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi016.front.sepia.ceph.com' 2023-09-23T18:45:18.381 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi016.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T18:45:27.044 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi092.front.sepia.ceph.com' 2023-09-23T18:45:27.044 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi092.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T18:45:58.764 DEBUG:teuthology.orchestra.run.smithi092:> true 2023-09-23T18:45:59.321 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi092.front.sepia.ceph.com' 2023-09-23T18:45:59.321 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.15.0-84-generic"... 2023-09-23T18:45:59.321 DEBUG:teuthology.orchestra.run.smithi092:> uname -r 2023-09-23T18:45:59.369 INFO:teuthology.orchestra.run.smithi092.stdout:5.15.0-84-generic 2023-09-23T18:45:59.369 DEBUG:teuthology.task.kernel:current kernel version is 5.15.0-84-generic vs 5.15.0-84-generic 2023-09-23T18:45:59.370 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2023-09-23T18:45:59.370 DEBUG:teuthology.task.kernel:Distro of this test job: ubuntu 2023-09-23T18:46:00.370 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2023-09-23T18:46:00.371 DEBUG:teuthology.orchestra.run.smithi092:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-09-23T18:46:00.725 INFO:teuthology.orchestra.run.smithi092.stdout:ttyS1 2023-09-23T18:46:00.758 DEBUG:teuthology.parallel:result is None 2023-09-23T18:46:18.383 DEBUG:teuthology.orchestra.remote:timed out 2023-09-23T18:46:30.385 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi016.front.sepia.ceph.com' 2023-09-23T18:46:30.385 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi016.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T18:46:30.618 DEBUG:teuthology.orchestra.run.smithi016:> true 2023-09-23T18:46:31.191 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi016.front.sepia.ceph.com' 2023-09-23T18:46:31.191 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.15.0-84-generic"... 2023-09-23T18:46:31.192 DEBUG:teuthology.orchestra.run.smithi016:> uname -r 2023-09-23T18:46:31.239 INFO:teuthology.orchestra.run.smithi016.stdout:5.15.0-84-generic 2023-09-23T18:46:31.239 DEBUG:teuthology.task.kernel:current kernel version is 5.15.0-84-generic vs 5.15.0-84-generic 2023-09-23T18:46:31.240 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2023-09-23T18:46:31.240 DEBUG:teuthology.task.kernel:Distro of this test job: ubuntu 2023-09-23T18:46:32.241 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2023-09-23T18:46:32.241 DEBUG:teuthology.orchestra.run.smithi016:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-09-23T18:46:32.311 INFO:teuthology.orchestra.run.smithi016.stdout:ttyS1 2023-09-23T18:46:32.335 DEBUG:teuthology.parallel:result is None 2023-09-23T18:46:32.335 INFO:teuthology.run_tasks:Running task internal.base... 2023-09-23T18:46:32.343 INFO:teuthology.task.internal:Creating test directory... 2023-09-23T18:46:32.343 DEBUG:teuthology.orchestra.run.smithi016:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-09-23T18:46:32.345 DEBUG:teuthology.orchestra.run.smithi092:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-09-23T18:46:32.352 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2023-09-23T18:46:32.359 INFO:teuthology.run_tasks:Running task internal.archive... 2023-09-23T18:46:32.365 INFO:teuthology.task.internal:Creating archive directory... 2023-09-23T18:46:32.365 DEBUG:teuthology.orchestra.run.smithi016:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-09-23T18:46:32.394 DEBUG:teuthology.orchestra.run.smithi092:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-09-23T18:46:32.483 INFO:teuthology.run_tasks:Running task internal.coredump... 2023-09-23T18:46:32.491 INFO:teuthology.task.internal:Enabling coredump saving... 2023-09-23T18:46:32.492 DEBUG:teuthology.orchestra.run.smithi016:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-09-23T18:46:32.494 DEBUG:teuthology.orchestra.run.smithi092:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-09-23T18:46:32.509 INFO:teuthology.orchestra.run.smithi016.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-23T18:46:32.516 INFO:teuthology.orchestra.run.smithi092.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-23T18:46:32.516 INFO:teuthology.orchestra.run.smithi016.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-23T18:46:32.523 INFO:teuthology.orchestra.run.smithi092.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-23T18:46:32.524 INFO:teuthology.run_tasks:Running task internal.sudo... 2023-09-23T18:46:32.531 INFO:teuthology.task.internal:Configuring sudo... 2023-09-23T18:46:32.531 DEBUG:teuthology.orchestra.run.smithi016:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-09-23T18:46:32.562 DEBUG:teuthology.orchestra.run.smithi092:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-09-23T18:46:32.584 INFO:teuthology.run_tasks:Running task internal.syslog... 2023-09-23T18:46:32.592 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2023-09-23T18:46:32.593 DEBUG:teuthology.orchestra.run.smithi016:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-09-23T18:46:32.626 DEBUG:teuthology.orchestra.run.smithi092:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-09-23T18:46:32.634 DEBUG:teuthology.orchestra.run.smithi016:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-23T18:46:32.679 DEBUG:teuthology.orchestra.run.smithi016:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-23T18:46:32.727 DEBUG:teuthology.orchestra.run.smithi016:> set -ex 2023-09-23T18:46:32.727 DEBUG:teuthology.orchestra.run.smithi016:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-09-23T18:46:32.787 DEBUG:teuthology.orchestra.run.smithi092:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-23T18:46:32.795 DEBUG:teuthology.orchestra.run.smithi092:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-23T18:46:32.842 DEBUG:teuthology.orchestra.run.smithi092:> set -ex 2023-09-23T18:46:32.842 DEBUG:teuthology.orchestra.run.smithi092:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-09-23T18:46:32.904 DEBUG:teuthology.orchestra.run.smithi016:> sudo service rsyslog restart 2023-09-23T18:46:32.906 DEBUG:teuthology.orchestra.run.smithi092:> sudo service rsyslog restart 2023-09-23T18:46:32.995 INFO:teuthology.run_tasks:Running task internal.timer... 2023-09-23T18:46:33.003 INFO:teuthology.task.internal:Starting timer... 2023-09-23T18:46:33.003 INFO:teuthology.run_tasks:Running task pcp... 2023-09-23T18:46:33.022 INFO:teuthology.run_tasks:Running task selinux... 2023-09-23T18:46:33.031 DEBUG:teuthology.task.selinux:Excluding smithi016: OS 'ubuntu' does not support SELinux 2023-09-23T18:46:33.031 DEBUG:teuthology.task.selinux:Excluding smithi092: OS 'ubuntu' does not support SELinux 2023-09-23T18:46:33.031 DEBUG:teuthology.task.selinux:Getting current SELinux state 2023-09-23T18:46:33.031 DEBUG:teuthology.task.selinux:Existing SELinux modes: {} 2023-09-23T18:46:33.031 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2023-09-23T18:46:33.031 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2023-09-23T18:46:33.040 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2023-09-23T18:46:33.045 INFO:teuthology.repo_utils:Fetching main from origin 2023-09-23T18:46:33.229 INFO:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2023-09-23T18:46:33.319 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2023-09-23T18:46:33.320 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi016.front.sepia.ceph.com,smithi092.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2023-09-23T18:52:12.244 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi016.front.sepia.ceph.com'), Remote(name='ubuntu@smithi092.front.sepia.ceph.com')] 2023-09-23T18:52:12.245 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi016.front.sepia.ceph.com' 2023-09-23T18:52:12.246 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi016.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T18:52:12.344 DEBUG:teuthology.orchestra.run.smithi016:> true 2023-09-23T18:52:12.472 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi016.front.sepia.ceph.com' 2023-09-23T18:52:12.472 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi092.front.sepia.ceph.com' 2023-09-23T18:52:12.473 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi092.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T18:52:12.550 DEBUG:teuthology.orchestra.run.smithi092:> true 2023-09-23T18:52:12.672 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi092.front.sepia.ceph.com' 2023-09-23T18:52:12.672 INFO:teuthology.run_tasks:Running task clock... 2023-09-23T18:52:12.682 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2023-09-23T18:52:12.682 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-09-23T18:52:12.683 DEBUG:teuthology.orchestra.run.smithi016:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-09-23T18:52:12.685 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-09-23T18:52:12.685 DEBUG:teuthology.orchestra.run.smithi092:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-09-23T18:52:12.717 INFO:teuthology.orchestra.run.smithi016.stdout:23 Sep 18:52:12 ntpd[22477]: ntpd 4.2.8p12@1.3728-o (1): Starting 2023-09-23T18:52:12.717 INFO:teuthology.orchestra.run.smithi016.stdout:23 Sep 18:52:12 ntpd[22477]: Command line: ntpd -gq 2023-09-23T18:52:12.718 INFO:teuthology.orchestra.run.smithi016.stdout:23 Sep 18:52:12 ntpd[22477]: proto: precision = 0.103 usec (-23) 2023-09-23T18:52:12.718 INFO:teuthology.orchestra.run.smithi016.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED. 2023-09-23T18:52:12.719 INFO:teuthology.orchestra.run.smithi016.stdout:23 Sep 18:52:12 ntpd[22477]: restrict 0.0.0.0: KOD does nothing without LIMITED. 2023-09-23T18:52:12.719 INFO:teuthology.orchestra.run.smithi016.stderr:restrict ::: KOD does nothing without LIMITED. 2023-09-23T18:52:12.720 INFO:teuthology.orchestra.run.smithi016.stdout:23 Sep 18:52:12 ntpd[22477]: restrict ::: KOD does nothing without LIMITED. 2023-09-23T18:52:12.720 INFO:teuthology.orchestra.run.smithi016.stdout:23 Sep 18:52:12 ntpd[22477]: Listen and drop on 0 v6wildcard [::]:123 2023-09-23T18:52:12.720 INFO:teuthology.orchestra.run.smithi016.stdout:23 Sep 18:52:12 ntpd[22477]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2023-09-23T18:52:12.720 INFO:teuthology.orchestra.run.smithi016.stdout:23 Sep 18:52:12 ntpd[22477]: Listen normally on 2 lo 127.0.0.1:123 2023-09-23T18:52:12.720 INFO:teuthology.orchestra.run.smithi016.stdout:23 Sep 18:52:12 ntpd[22477]: Listen normally on 3 ens1f0 172.21.15.16:123 2023-09-23T18:52:12.720 INFO:teuthology.orchestra.run.smithi016.stdout:23 Sep 18:52:12 ntpd[22477]: Listen normally on 4 lo [::1]:123 2023-09-23T18:52:12.721 INFO:teuthology.orchestra.run.smithi016.stdout:23 Sep 18:52:12 ntpd[22477]: Listen normally on 5 ens1f0 [fe80::ec4:7aff:febd:1564%2]:123 2023-09-23T18:52:12.721 INFO:teuthology.orchestra.run.smithi016.stdout:23 Sep 18:52:12 ntpd[22477]: Listening on routing socket on fd #22 for interface updates 2023-09-23T18:52:12.747 INFO:teuthology.orchestra.run.smithi092.stdout:23 Sep 18:52:12 ntpd[22544]: ntpd 4.2.8p12@1.3728-o (1): Starting 2023-09-23T18:52:12.747 INFO:teuthology.orchestra.run.smithi092.stdout:23 Sep 18:52:12 ntpd[22544]: Command line: ntpd -gq 2023-09-23T18:52:12.748 INFO:teuthology.orchestra.run.smithi092.stdout:23 Sep 18:52:12 ntpd[22544]: proto: precision = 0.100 usec (-23) 2023-09-23T18:52:12.748 INFO:teuthology.orchestra.run.smithi092.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED. 2023-09-23T18:52:12.749 INFO:teuthology.orchestra.run.smithi092.stdout:23 Sep 18:52:12 ntpd[22544]: restrict 0.0.0.0: KOD does nothing without LIMITED. 2023-09-23T18:52:12.749 INFO:teuthology.orchestra.run.smithi092.stderr:restrict ::: KOD does nothing without LIMITED. 2023-09-23T18:52:12.750 INFO:teuthology.orchestra.run.smithi092.stdout:23 Sep 18:52:12 ntpd[22544]: restrict ::: KOD does nothing without LIMITED. 2023-09-23T18:52:12.750 INFO:teuthology.orchestra.run.smithi092.stdout:23 Sep 18:52:12 ntpd[22544]: Listen and drop on 0 v6wildcard [::]:123 2023-09-23T18:52:12.750 INFO:teuthology.orchestra.run.smithi092.stdout:23 Sep 18:52:12 ntpd[22544]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2023-09-23T18:52:12.750 INFO:teuthology.orchestra.run.smithi092.stdout:23 Sep 18:52:12 ntpd[22544]: Listen normally on 2 lo 127.0.0.1:123 2023-09-23T18:52:12.750 INFO:teuthology.orchestra.run.smithi092.stdout:23 Sep 18:52:12 ntpd[22544]: Listen normally on 3 enp3s0f1 172.21.15.92:123 2023-09-23T18:52:12.750 INFO:teuthology.orchestra.run.smithi092.stdout:23 Sep 18:52:12 ntpd[22544]: Listen normally on 4 lo [::1]:123 2023-09-23T18:52:12.750 INFO:teuthology.orchestra.run.smithi092.stdout:23 Sep 18:52:12 ntpd[22544]: Listen normally on 5 enp3s0f1 [fe80::ec4:7aff:fe8f:cd45%5]:123 2023-09-23T18:52:12.750 INFO:teuthology.orchestra.run.smithi092.stdout:23 Sep 18:52:12 ntpd[22544]: Listening on routing socket on fd #22 for interface updates 2023-09-23T18:52:13.718 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:13 ntpd[22477]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:52:13.719 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:13 ntpd[22477]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:52:13.719 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:13 ntpd[22477]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:52:13.719 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:13 ntpd[22477]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:52:13.720 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:13 ntpd[22477]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:52:13.720 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:13 ntpd[22477]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:52:13.720 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:13 ntpd[22477]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-09-23T18:52:13.720 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:13 ntpd[22477]: can't open /var/log/ntpstats/peerstats.20230923: Permission denied 2023-09-23T18:52:13.720 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:13 ntpd[22477]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:52:13.720 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:13 ntpd[22477]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:52:13.720 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:13 ntpd[22477]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-09-23T18:52:13.720 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:13 ntpd[22477]: can't open /var/log/ntpstats/peerstats.20230923: Permission denied 2023-09-23T18:52:14.718 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:14 ntpd[22477]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:52:14.718 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:14 ntpd[22477]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:52:14.718 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:14 ntpd[22477]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:52:14.718 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:14 ntpd[22477]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:52:14.719 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:14 ntpd[22477]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-09-23T18:52:14.719 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:14 ntpd[22477]: can't open /var/log/ntpstats/peerstats.20230923: Permission denied 2023-09-23T18:52:14.748 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:14 ntpd[22544]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:52:14.748 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:14 ntpd[22544]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:52:14.748 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:14 ntpd[22544]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:52:14.749 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:14 ntpd[22544]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:52:14.749 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:14 ntpd[22544]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:52:14.749 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:14 ntpd[22544]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:52:14.749 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:14 ntpd[22544]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-09-23T18:52:14.749 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:14 ntpd[22544]: can't open /var/log/ntpstats/peerstats.20230923: Permission denied 2023-09-23T18:52:15.719 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:15 ntpd[22477]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:52:15.719 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:15 ntpd[22477]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:52:15.719 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:15 ntpd[22477]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:52:15.719 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:15 ntpd[22477]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:52:15.719 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:15 ntpd[22477]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:52:15.719 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:15 ntpd[22477]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:52:15.720 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:15 ntpd[22477]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:52:15.720 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:15 ntpd[22477]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:52:15.720 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:15 ntpd[22477]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-09-23T18:52:15.720 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:15 ntpd[22477]: can't open /var/log/ntpstats/peerstats.20230923: Permission denied 2023-09-23T18:52:15.720 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:15 ntpd[22477]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:52:15.720 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:15 ntpd[22477]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:52:15.720 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:15 ntpd[22477]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-09-23T18:52:15.720 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:15 ntpd[22477]: can't open /var/log/ntpstats/peerstats.20230923: Permission denied 2023-09-23T18:52:15.748 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:15 ntpd[22544]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:52:15.748 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:15 ntpd[22544]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:52:15.748 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:15 ntpd[22544]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:52:15.748 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:15 ntpd[22544]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:52:15.748 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:15 ntpd[22544]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-09-23T18:52:15.748 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:15 ntpd[22544]: can't open /var/log/ntpstats/peerstats.20230923: Permission denied 2023-09-23T18:52:16.718 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:16 ntpd[22477]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:52:16.718 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:16 ntpd[22477]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:52:16.718 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:16 ntpd[22477]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:52:16.719 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:16 ntpd[22477]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:52:16.719 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:16 ntpd[22477]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-09-23T18:52:16.719 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:16 ntpd[22477]: can't open /var/log/ntpstats/peerstats.20230923: Permission denied 2023-09-23T18:52:16.748 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:16 ntpd[22544]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:52:16.748 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:16 ntpd[22544]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:52:16.748 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:16 ntpd[22544]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:52:16.748 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:16 ntpd[22544]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:52:16.748 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:16 ntpd[22544]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-09-23T18:52:16.749 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:16 ntpd[22544]: can't open /var/log/ntpstats/peerstats.20230923: Permission denied 2023-09-23T18:52:17.718 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:17 ntpd[22477]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:52:17.719 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:17 ntpd[22477]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:52:17.719 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:17 ntpd[22477]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:52:17.719 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:17 ntpd[22477]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:52:17.719 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:17 ntpd[22477]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:52:17.719 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:17 ntpd[22477]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:52:17.719 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:17 ntpd[22477]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-09-23T18:52:17.719 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:17 ntpd[22477]: can't open /var/log/ntpstats/peerstats.20230923: Permission denied 2023-09-23T18:52:17.719 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:17 ntpd[22477]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:52:17.719 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:17 ntpd[22477]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:52:17.720 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:17 ntpd[22477]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-09-23T18:52:17.720 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:17 ntpd[22477]: can't open /var/log/ntpstats/peerstats.20230923: Permission denied 2023-09-23T18:52:17.748 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:17 ntpd[22544]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:52:17.748 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:17 ntpd[22544]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:52:17.748 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:17 ntpd[22544]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:52:17.748 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:17 ntpd[22544]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:52:17.749 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:17 ntpd[22544]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:52:17.749 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:17 ntpd[22544]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:52:17.749 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:17 ntpd[22544]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-09-23T18:52:17.749 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:17 ntpd[22544]: can't open /var/log/ntpstats/peerstats.20230923: Permission denied 2023-09-23T18:52:17.749 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:17 ntpd[22544]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:52:17.749 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:17 ntpd[22544]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:52:17.749 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:17 ntpd[22544]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-09-23T18:52:17.749 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:17 ntpd[22544]: can't open /var/log/ntpstats/peerstats.20230923: Permission denied 2023-09-23T18:52:18.718 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:18 ntpd[22477]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:52:18.718 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:18 ntpd[22477]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:52:18.719 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:18 ntpd[22477]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:52:18.719 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:18 ntpd[22477]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:52:18.719 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:18 ntpd[22477]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-09-23T18:52:18.719 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:18 ntpd[22477]: can't open /var/log/ntpstats/peerstats.20230923: Permission denied 2023-09-23T18:52:18.748 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:18 ntpd[22544]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:52:18.748 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:18 ntpd[22544]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:52:18.748 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:18 ntpd[22544]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:52:18.748 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:18 ntpd[22544]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:52:18.748 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:18 ntpd[22544]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-09-23T18:52:18.748 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:18 ntpd[22544]: can't open /var/log/ntpstats/peerstats.20230923: Permission denied 2023-09-23T18:52:19.718 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:19 ntpd[22477]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:52:19.719 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:19 ntpd[22477]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:52:19.719 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:19 ntpd[22477]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:52:19.719 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:19 ntpd[22477]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:52:19.719 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:19 ntpd[22477]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:52:19.719 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:19 ntpd[22477]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:52:19.719 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:19 ntpd[22477]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-09-23T18:52:19.719 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:19 ntpd[22477]: can't open /var/log/ntpstats/peerstats.20230923: Permission denied 2023-09-23T18:52:19.720 INFO:teuthology.orchestra.run.smithi016.stdout:23 Sep 18:52:19 ntpd[22477]: ntpd: time slew -0.002174 s 2023-09-23T18:52:19.720 INFO:teuthology.orchestra.run.smithi016.stdout:ntpd: time slew -0.002174s 2023-09-23T18:52:19.720 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:19 ntpd[22477]: couldn't unlink /var/log/ntpstats/loopstats: Permission denied 2023-09-23T18:52:19.720 INFO:teuthology.orchestra.run.smithi016.stderr:23 Sep 18:52:19 ntpd[22477]: can't open /var/log/ntpstats/loopstats.20230923: Permission denied 2023-09-23T18:52:19.748 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:19 ntpd[22544]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:52:19.749 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:19 ntpd[22544]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:52:19.749 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:19 ntpd[22544]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:52:19.749 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:19 ntpd[22544]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:52:19.749 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:19 ntpd[22544]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:52:19.749 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:19 ntpd[22544]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:52:19.749 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:19 ntpd[22544]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-09-23T18:52:19.749 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:19 ntpd[22544]: can't open /var/log/ntpstats/peerstats.20230923: Permission denied 2023-09-23T18:52:19.749 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:19 ntpd[22544]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:52:19.750 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:19 ntpd[22544]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:52:19.750 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:19 ntpd[22544]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-09-23T18:52:19.750 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:19 ntpd[22544]: can't open /var/log/ntpstats/peerstats.20230923: Permission denied 2023-09-23T18:52:19.778 INFO:teuthology.orchestra.run.smithi016.stdout: remote refid st t when poll reach delay offset jitter 2023-09-23T18:52:19.778 INFO:teuthology.orchestra.run.smithi016.stdout:============================================================================== 2023-09-23T18:52:19.778 INFO:teuthology.orchestra.run.smithi016.stdout: hv01.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-09-23T18:52:19.778 INFO:teuthology.orchestra.run.smithi016.stdout: hv02.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-09-23T18:52:19.779 INFO:teuthology.orchestra.run.smithi016.stdout: hv03.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-09-23T18:52:19.779 INFO:teuthology.orchestra.run.smithi016.stdout: hv04.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-09-23T18:52:20.748 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:20 ntpd[22544]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:52:20.748 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:20 ntpd[22544]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:52:20.748 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:20 ntpd[22544]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:52:20.748 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:20 ntpd[22544]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:52:20.748 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:20 ntpd[22544]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-09-23T18:52:20.748 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:20 ntpd[22544]: can't open /var/log/ntpstats/peerstats.20230923: Permission denied 2023-09-23T18:52:20.749 INFO:teuthology.orchestra.run.smithi092.stdout:23 Sep 18:52:20 ntpd[22544]: ntpd: time slew -0.004364 s 2023-09-23T18:52:20.749 INFO:teuthology.orchestra.run.smithi092.stdout:ntpd: time slew -0.004364s 2023-09-23T18:52:20.749 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:20 ntpd[22544]: couldn't unlink /var/log/ntpstats/loopstats: Permission denied 2023-09-23T18:52:20.749 INFO:teuthology.orchestra.run.smithi092.stderr:23 Sep 18:52:20 ntpd[22544]: can't open /var/log/ntpstats/loopstats.20230923: Permission denied 2023-09-23T18:52:20.810 INFO:teuthology.orchestra.run.smithi092.stdout: remote refid st t when poll reach delay offset jitter 2023-09-23T18:52:20.811 INFO:teuthology.orchestra.run.smithi092.stdout:============================================================================== 2023-09-23T18:52:20.811 INFO:teuthology.orchestra.run.smithi092.stdout: hv01.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-09-23T18:52:20.811 INFO:teuthology.orchestra.run.smithi092.stdout: hv02.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-09-23T18:52:20.811 INFO:teuthology.orchestra.run.smithi092.stdout: hv03.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-09-23T18:52:20.811 INFO:teuthology.orchestra.run.smithi092.stdout: hv04.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-09-23T18:52:20.812 INFO:teuthology.run_tasks:Running task install... 2023-09-23T18:52:20.821 DEBUG:teuthology.task.install:project ceph 2023-09-23T18:52:20.821 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'}} 2023-09-23T18:52:20.822 DEBUG:teuthology.task.install:config {'branch': 'pacific', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'} 2023-09-23T18:52:20.822 INFO:teuthology.task.install:Using flavor: default 2023-09-23T18:52:20.827 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2023-09-23T18:52:20.827 INFO:teuthology.task.install:extra packages: [] 2023-09-23T18:52:20.828 DEBUG:teuthology.orchestra.run.smithi016:> sudo apt-key list | grep Ceph 2023-09-23T18:52:20.828 DEBUG:teuthology.orchestra.run.smithi092:> sudo apt-key list | grep Ceph 2023-09-23T18:52:20.932 INFO:teuthology.orchestra.run.smithi016.stderr:Warning: apt-key output should not be parsed (stdout is not a terminal) 2023-09-23T18:52:20.965 INFO:teuthology.orchestra.run.smithi092.stderr:Warning: apt-key output should not be parsed (stdout is not a terminal) 2023-09-23T18:52:20.991 INFO:teuthology.orchestra.run.smithi016.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2023-09-23T18:52:20.991 INFO:teuthology.orchestra.run.smithi016.stdout:uid [ unknown] Ceph.com (release key) 2023-09-23T18:52:20.992 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse on remote deb x86_64 2023-09-23T18:52:20.992 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T18:52:20.992 INFO:teuthology.packaging:ref: None 2023-09-23T18:52:20.992 INFO:teuthology.packaging:tag: None 2023-09-23T18:52:20.992 INFO:teuthology.packaging:branch: pacific 2023-09-23T18:52:20.992 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T18:52:20.993 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=pacific 2023-09-23T18:52:21.023 INFO:teuthology.orchestra.run.smithi092.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2023-09-23T18:52:21.023 INFO:teuthology.orchestra.run.smithi092.stdout:uid [ unknown] Ceph.com (release key) 2023-09-23T18:52:21.024 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse on remote deb x86_64 2023-09-23T18:52:21.024 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T18:52:21.024 INFO:teuthology.packaging:ref: None 2023-09-23T18:52:21.024 INFO:teuthology.packaging:tag: None 2023-09-23T18:52:21.024 INFO:teuthology.packaging:branch: pacific 2023-09-23T18:52:21.024 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T18:52:21.024 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=pacific 2023-09-23T18:52:21.342 INFO:teuthology.task.install.deb:Pulling from https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default/ 2023-09-23T18:52:21.342 INFO:teuthology.task.install.deb:Package version is 16.2.14-37-g2e1f1015-1focal 2023-09-23T18:52:21.357 INFO:teuthology.task.install.deb:Pulling from https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default/ 2023-09-23T18:52:21.357 INFO:teuthology.task.install.deb:Package version is 16.2.14-37-g2e1f1015-1focal 2023-09-23T18:52:21.473 DEBUG:teuthology.orchestra.run.smithi016:> set -ex 2023-09-23T18:52:21.473 DEBUG:teuthology.orchestra.run.smithi016:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2023-09-23T18:52:21.489 DEBUG:teuthology.orchestra.run.smithi016:> sudo apt-get update 2023-09-23T18:52:21.526 DEBUG:teuthology.orchestra.run.smithi092:> set -ex 2023-09-23T18:52:21.526 DEBUG:teuthology.orchestra.run.smithi092:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2023-09-23T18:52:21.542 DEBUG:teuthology.orchestra.run.smithi092:> sudo apt-get update 2023-09-23T18:52:21.699 INFO:teuthology.orchestra.run.smithi016.stdout:Hit:1 http://archive.ubuntu.com/ubuntu focal InRelease 2023-09-23T18:52:21.699 INFO:teuthology.orchestra.run.smithi016.stdout:Hit:2 http://security.ubuntu.com/ubuntu focal-security InRelease 2023-09-23T18:52:21.704 INFO:teuthology.orchestra.run.smithi016.stdout:Hit:3 http://archive.ubuntu.com/ubuntu focal-updates InRelease 2023-09-23T18:52:21.722 INFO:teuthology.orchestra.run.smithi016.stdout:Hit:4 http://archive.ubuntu.com/ubuntu focal-backports InRelease 2023-09-23T18:52:21.731 INFO:teuthology.orchestra.run.smithi092.stdout:Hit:1 http://archive.ubuntu.com/ubuntu focal InRelease 2023-09-23T18:52:21.731 INFO:teuthology.orchestra.run.smithi092.stdout:Hit:2 http://security.ubuntu.com/ubuntu focal-security InRelease 2023-09-23T18:52:21.735 INFO:teuthology.orchestra.run.smithi092.stdout:Hit:3 http://archive.ubuntu.com/ubuntu focal-updates InRelease 2023-09-23T18:52:21.752 INFO:teuthology.orchestra.run.smithi092.stdout:Hit:4 http://archive.ubuntu.com/ubuntu focal-backports InRelease 2023-09-23T18:52:21.787 INFO:teuthology.orchestra.run.smithi016.stdout:Ign:5 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal InRelease 2023-09-23T18:52:21.824 INFO:teuthology.orchestra.run.smithi016.stdout:Get:6 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal Release [7,680 B] 2023-09-23T18:52:21.842 INFO:teuthology.orchestra.run.smithi092.stdout:Ign:5 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal InRelease 2023-09-23T18:52:21.861 INFO:teuthology.orchestra.run.smithi016.stdout:Ign:7 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal Release.gpg 2023-09-23T18:52:21.878 INFO:teuthology.orchestra.run.smithi092.stdout:Get:6 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal Release [7,680 B] 2023-09-23T18:52:21.897 INFO:teuthology.orchestra.run.smithi016.stdout:Get:8 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main i386 Packages [3,104 B] 2023-09-23T18:52:21.915 INFO:teuthology.orchestra.run.smithi092.stdout:Ign:7 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal Release.gpg 2023-09-23T18:52:21.952 INFO:teuthology.orchestra.run.smithi092.stdout:Get:8 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main i386 Packages [3,104 B] 2023-09-23T18:52:21.957 INFO:teuthology.orchestra.run.smithi016.stdout:Get:9 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 Packages [17.2 kB] 2023-09-23T18:52:22.010 INFO:teuthology.orchestra.run.smithi092.stdout:Get:9 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 Packages [17.2 kB] 2023-09-23T18:52:22.346 INFO:teuthology.orchestra.run.smithi092.stdout:Fetched 28.0 kB in 1s (46.9 kB/s) 2023-09-23T18:52:22.366 INFO:teuthology.orchestra.run.smithi016.stdout:Fetched 28.0 kB in 1s (41.9 kB/s) 2023-09-23T18:52:23.392 INFO:teuthology.orchestra.run.smithi092.stdout:Reading package lists... 2023-09-23T18:52:23.421 DEBUG:teuthology.orchestra.run.smithi092:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=16.2.14-37-g2e1f1015-1focal cephadm=16.2.14-37-g2e1f1015-1focal ceph-mds=16.2.14-37-g2e1f1015-1focal ceph-mgr=16.2.14-37-g2e1f1015-1focal ceph-common=16.2.14-37-g2e1f1015-1focal ceph-fuse=16.2.14-37-g2e1f1015-1focal ceph-test=16.2.14-37-g2e1f1015-1focal radosgw=16.2.14-37-g2e1f1015-1focal python3-rados=16.2.14-37-g2e1f1015-1focal python3-rgw=16.2.14-37-g2e1f1015-1focal python3-cephfs=16.2.14-37-g2e1f1015-1focal python3-rbd=16.2.14-37-g2e1f1015-1focal libcephfs2=16.2.14-37-g2e1f1015-1focal libcephfs-dev=16.2.14-37-g2e1f1015-1focal librados2=16.2.14-37-g2e1f1015-1focal librbd1=16.2.14-37-g2e1f1015-1focal rbd-fuse=16.2.14-37-g2e1f1015-1focal 2023-09-23T18:52:23.438 INFO:teuthology.orchestra.run.smithi016.stdout:Reading package lists... 2023-09-23T18:52:23.468 DEBUG:teuthology.orchestra.run.smithi016:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=16.2.14-37-g2e1f1015-1focal cephadm=16.2.14-37-g2e1f1015-1focal ceph-mds=16.2.14-37-g2e1f1015-1focal ceph-mgr=16.2.14-37-g2e1f1015-1focal ceph-common=16.2.14-37-g2e1f1015-1focal ceph-fuse=16.2.14-37-g2e1f1015-1focal ceph-test=16.2.14-37-g2e1f1015-1focal radosgw=16.2.14-37-g2e1f1015-1focal python3-rados=16.2.14-37-g2e1f1015-1focal python3-rgw=16.2.14-37-g2e1f1015-1focal python3-cephfs=16.2.14-37-g2e1f1015-1focal python3-rbd=16.2.14-37-g2e1f1015-1focal libcephfs2=16.2.14-37-g2e1f1015-1focal libcephfs-dev=16.2.14-37-g2e1f1015-1focal librados2=16.2.14-37-g2e1f1015-1focal librbd1=16.2.14-37-g2e1f1015-1focal rbd-fuse=16.2.14-37-g2e1f1015-1focal 2023-09-23T18:52:23.491 INFO:teuthology.orchestra.run.smithi092.stdout:Reading package lists... 2023-09-23T18:52:23.541 INFO:teuthology.orchestra.run.smithi016.stdout:Reading package lists... 2023-09-23T18:52:23.691 INFO:teuthology.orchestra.run.smithi092.stdout:Building dependency tree... 2023-09-23T18:52:23.691 INFO:teuthology.orchestra.run.smithi092.stdout:Reading state information... 2023-09-23T18:52:23.746 INFO:teuthology.orchestra.run.smithi016.stdout:Building dependency tree... 2023-09-23T18:52:23.747 INFO:teuthology.orchestra.run.smithi016.stdout:Reading state information... 2023-09-23T18:52:23.864 INFO:teuthology.orchestra.run.smithi092.stdout:The following packages were automatically installed and are no longer required: 2023-09-23T18:52:23.865 INFO:teuthology.orchestra.run.smithi092.stdout: libboost-iostreams1.71.0 libboost-thread1.71.0 2023-09-23T18:52:23.865 INFO:teuthology.orchestra.run.smithi092.stdout:Use 'sudo apt autoremove' to remove them. 2023-09-23T18:52:23.865 INFO:teuthology.orchestra.run.smithi092.stdout:The following additional packages will be installed: 2023-09-23T18:52:23.866 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-local 2023-09-23T18:52:23.867 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon ceph-osd jq libjaeger 2023-09-23T18:52:23.867 INFO:teuthology.orchestra.run.smithi092.stdout: libjq1 libjs-sphinxdoc libjs-underscore liblbfgsb0 libleveldb1d 2023-09-23T18:52:23.867 INFO:teuthology.orchestra.run.smithi092.stdout: liblttng-ust0 liblua5.3-0 liboath0 libonig5 librabbitmq4 libradosstriper1 2023-09-23T18:52:23.867 INFO:teuthology.orchestra.run.smithi092.stdout: librdkafka1 librgw2 libsqlite3-mod-ceph nvme-cli python-pastedeploy-tpl 2023-09-23T18:52:23.867 INFO:teuthology.orchestra.run.smithi092.stdout: python3-atomicwrites python3-attr python3-bcrypt python3-blinker python3-bs4 2023-09-23T18:52:23.868 INFO:teuthology.orchestra.run.smithi092.stdout: python3-cachetools python3-ceph-argparse python3-ceph-common 2023-09-23T18:52:23.868 INFO:teuthology.orchestra.run.smithi092.stdout: python3-cherrypy3 python3-cryptography python3-decorator python3-google-auth 2023-09-23T18:52:23.868 INFO:teuthology.orchestra.run.smithi092.stdout: python3-html5lib python3-jinja2 python3-joblib python3-jwt 2023-09-23T18:52:23.868 INFO:teuthology.orchestra.run.smithi092.stdout: python3-kubernetes python3-logutils python3-lxml python3-mako 2023-09-23T18:52:23.868 INFO:teuthology.orchestra.run.smithi092.stdout: python3-markupsafe python3-oauthlib python3-openssl python3-packaging 2023-09-23T18:52:23.868 INFO:teuthology.orchestra.run.smithi092.stdout: python3-paste python3-pastedeploy python3-pastescript python3-pecan 2023-09-23T18:52:23.868 INFO:teuthology.orchestra.run.smithi092.stdout: python3-pluggy python3-prettytable python3-psutil python3-py python3-pyasn1 2023-09-23T18:52:23.868 INFO:teuthology.orchestra.run.smithi092.stdout: python3-pyasn1-modules python3-pyinotify python3-pytest python3-repoze.lru 2023-09-23T18:52:23.868 INFO:teuthology.orchestra.run.smithi092.stdout: python3-requests-oauthlib python3-routes python3-rsa python3-scipy 2023-09-23T18:52:23.868 INFO:teuthology.orchestra.run.smithi092.stdout: python3-simplegeneric python3-simplejson python3-singledispatch 2023-09-23T18:52:23.868 INFO:teuthology.orchestra.run.smithi092.stdout: python3-sklearn python3-sklearn-lib python3-soupsieve python3-tempita 2023-09-23T18:52:23.869 INFO:teuthology.orchestra.run.smithi092.stdout: python3-waitress python3-wcwidth python3-webencodings python3-webob 2023-09-23T18:52:23.869 INFO:teuthology.orchestra.run.smithi092.stdout: python3-websocket python3-webtest python3-werkzeug socat xmlstarlet 2023-09-23T18:52:23.869 INFO:teuthology.orchestra.run.smithi092.stdout:Suggested packages: 2023-09-23T18:52:23.869 INFO:teuthology.orchestra.run.smithi092.stdout: python3-influxdb python-attr-doc python-blinker-doc python-cryptography-doc 2023-09-23T18:52:23.869 INFO:teuthology.orchestra.run.smithi092.stdout: python3-cryptography-vectors python3-genshi python-jinja2-doc python3-crypto 2023-09-23T18:52:23.869 INFO:teuthology.orchestra.run.smithi092.stdout: python3-lxml-dbg python-lxml-doc python3-beaker python-mako-doc 2023-09-23T18:52:23.870 INFO:teuthology.orchestra.run.smithi092.stdout: python-openssl-doc python3-openssl-dbg httpd-wsgi libapache2-mod-python 2023-09-23T18:52:23.870 INFO:teuthology.orchestra.run.smithi092.stdout: libapache2-mod-scgi libjs-mochikit python-pecan-doc python-psutil-doc 2023-09-23T18:52:23.870 INFO:teuthology.orchestra.run.smithi092.stdout: subversion python-pyinotify-doc python-scipy-doc python3-dap 2023-09-23T18:52:23.870 INFO:teuthology.orchestra.run.smithi092.stdout: python-sklearn-doc ipython3 python-waitress-doc python-webob-doc 2023-09-23T18:52:23.870 INFO:teuthology.orchestra.run.smithi092.stdout: python-webtest-doc python-werkzeug-doc python3-termcolor python3-watchdog 2023-09-23T18:52:23.870 INFO:teuthology.orchestra.run.smithi092.stdout:Recommended packages: 2023-09-23T18:52:23.870 INFO:teuthology.orchestra.run.smithi092.stdout: btrfs-tools 2023-09-23T18:52:23.939 INFO:teuthology.orchestra.run.smithi016.stdout:The following packages were automatically installed and are no longer required: 2023-09-23T18:52:23.940 INFO:teuthology.orchestra.run.smithi016.stdout: libboost-iostreams1.71.0 libboost-thread1.71.0 2023-09-23T18:52:23.940 INFO:teuthology.orchestra.run.smithi016.stdout:Use 'sudo apt autoremove' to remove them. 2023-09-23T18:52:23.940 INFO:teuthology.orchestra.run.smithi016.stdout:The following additional packages will be installed: 2023-09-23T18:52:23.940 INFO:teuthology.orchestra.run.smithi016.stdout: ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-local 2023-09-23T18:52:23.941 INFO:teuthology.orchestra.run.smithi016.stdout: ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon ceph-osd jq libjaeger 2023-09-23T18:52:23.941 INFO:teuthology.orchestra.run.smithi016.stdout: libjq1 libjs-sphinxdoc libjs-underscore liblbfgsb0 libleveldb1d 2023-09-23T18:52:23.942 INFO:teuthology.orchestra.run.smithi016.stdout: liblttng-ust0 liblua5.3-0 liboath0 libonig5 librabbitmq4 libradosstriper1 2023-09-23T18:52:23.942 INFO:teuthology.orchestra.run.smithi016.stdout: librdkafka1 librgw2 libsqlite3-mod-ceph nvme-cli python-pastedeploy-tpl 2023-09-23T18:52:23.942 INFO:teuthology.orchestra.run.smithi016.stdout: python3-atomicwrites python3-attr python3-bcrypt python3-blinker python3-bs4 2023-09-23T18:52:23.942 INFO:teuthology.orchestra.run.smithi016.stdout: python3-cachetools python3-ceph-argparse python3-ceph-common 2023-09-23T18:52:23.942 INFO:teuthology.orchestra.run.smithi016.stdout: python3-cherrypy3 python3-cryptography python3-decorator python3-google-auth 2023-09-23T18:52:23.942 INFO:teuthology.orchestra.run.smithi016.stdout: python3-html5lib python3-jinja2 python3-joblib python3-jwt 2023-09-23T18:52:23.942 INFO:teuthology.orchestra.run.smithi016.stdout: python3-kubernetes python3-logutils python3-lxml python3-mako 2023-09-23T18:52:23.942 INFO:teuthology.orchestra.run.smithi016.stdout: python3-markupsafe python3-oauthlib python3-openssl python3-packaging 2023-09-23T18:52:23.942 INFO:teuthology.orchestra.run.smithi016.stdout: python3-paste python3-pastedeploy python3-pastescript python3-pecan 2023-09-23T18:52:23.942 INFO:teuthology.orchestra.run.smithi016.stdout: python3-pluggy python3-prettytable python3-psutil python3-py python3-pyasn1 2023-09-23T18:52:23.943 INFO:teuthology.orchestra.run.smithi016.stdout: python3-pyasn1-modules python3-pyinotify python3-pytest python3-repoze.lru 2023-09-23T18:52:23.943 INFO:teuthology.orchestra.run.smithi016.stdout: python3-requests-oauthlib python3-routes python3-rsa python3-scipy 2023-09-23T18:52:23.943 INFO:teuthology.orchestra.run.smithi016.stdout: python3-simplegeneric python3-simplejson python3-singledispatch 2023-09-23T18:52:23.943 INFO:teuthology.orchestra.run.smithi016.stdout: python3-sklearn python3-sklearn-lib python3-soupsieve python3-tempita 2023-09-23T18:52:23.943 INFO:teuthology.orchestra.run.smithi016.stdout: python3-waitress python3-wcwidth python3-webencodings python3-webob 2023-09-23T18:52:23.943 INFO:teuthology.orchestra.run.smithi016.stdout: python3-websocket python3-webtest python3-werkzeug socat xmlstarlet 2023-09-23T18:52:23.943 INFO:teuthology.orchestra.run.smithi016.stdout:Suggested packages: 2023-09-23T18:52:23.943 INFO:teuthology.orchestra.run.smithi016.stdout: python3-influxdb python-attr-doc python-blinker-doc python-cryptography-doc 2023-09-23T18:52:23.944 INFO:teuthology.orchestra.run.smithi016.stdout: python3-cryptography-vectors python3-genshi python-jinja2-doc python3-crypto 2023-09-23T18:52:23.944 INFO:teuthology.orchestra.run.smithi016.stdout: python3-lxml-dbg python-lxml-doc python3-beaker python-mako-doc 2023-09-23T18:52:23.944 INFO:teuthology.orchestra.run.smithi016.stdout: python-openssl-doc python3-openssl-dbg httpd-wsgi libapache2-mod-python 2023-09-23T18:52:23.944 INFO:teuthology.orchestra.run.smithi016.stdout: libapache2-mod-scgi libjs-mochikit python-pecan-doc python-psutil-doc 2023-09-23T18:52:23.944 INFO:teuthology.orchestra.run.smithi016.stdout: subversion python-pyinotify-doc python-scipy-doc python3-dap 2023-09-23T18:52:23.944 INFO:teuthology.orchestra.run.smithi016.stdout: python-sklearn-doc ipython3 python-waitress-doc python-webob-doc 2023-09-23T18:52:23.944 INFO:teuthology.orchestra.run.smithi016.stdout: python-webtest-doc python-werkzeug-doc python3-termcolor python3-watchdog 2023-09-23T18:52:23.944 INFO:teuthology.orchestra.run.smithi016.stdout:Recommended packages: 2023-09-23T18:52:23.944 INFO:teuthology.orchestra.run.smithi016.stdout: btrfs-tools 2023-09-23T18:52:23.979 INFO:teuthology.orchestra.run.smithi092.stdout:The following NEW packages will be installed: 2023-09-23T18:52:23.979 INFO:teuthology.orchestra.run.smithi092.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm 2023-09-23T18:52:23.979 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-k8sevents 2023-09-23T18:52:23.979 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-mgr-modules-core ceph-mon ceph-osd ceph-test cephadm jq libcephfs-dev 2023-09-23T18:52:23.979 INFO:teuthology.orchestra.run.smithi092.stdout: libcephfs2 libjaeger libjq1 libjs-sphinxdoc libjs-underscore liblbfgsb0 2023-09-23T18:52:23.979 INFO:teuthology.orchestra.run.smithi092.stdout: libleveldb1d liblttng-ust0 liblua5.3-0 liboath0 libonig5 librabbitmq4 2023-09-23T18:52:23.980 INFO:teuthology.orchestra.run.smithi092.stdout: libradosstriper1 librdkafka1 librgw2 libsqlite3-mod-ceph nvme-cli 2023-09-23T18:52:23.981 INFO:teuthology.orchestra.run.smithi092.stdout: python-pastedeploy-tpl python3-atomicwrites python3-attr python3-bcrypt 2023-09-23T18:52:23.981 INFO:teuthology.orchestra.run.smithi092.stdout: python3-blinker python3-bs4 python3-cachetools python3-ceph-argparse 2023-09-23T18:52:23.981 INFO:teuthology.orchestra.run.smithi092.stdout: python3-ceph-common python3-cephfs python3-cherrypy3 python3-cryptography 2023-09-23T18:52:23.981 INFO:teuthology.orchestra.run.smithi092.stdout: python3-decorator python3-google-auth python3-html5lib python3-jinja2 2023-09-23T18:52:23.981 INFO:teuthology.orchestra.run.smithi092.stdout: python3-joblib python3-jwt python3-kubernetes python3-logutils python3-lxml 2023-09-23T18:52:23.981 INFO:teuthology.orchestra.run.smithi092.stdout: python3-mako python3-markupsafe python3-oauthlib python3-openssl 2023-09-23T18:52:23.981 INFO:teuthology.orchestra.run.smithi092.stdout: python3-packaging python3-paste python3-pastedeploy python3-pastescript 2023-09-23T18:52:23.981 INFO:teuthology.orchestra.run.smithi092.stdout: python3-pecan python3-pluggy python3-prettytable python3-psutil python3-py 2023-09-23T18:52:23.981 INFO:teuthology.orchestra.run.smithi092.stdout: python3-pyasn1 python3-pyasn1-modules python3-pyinotify python3-pytest 2023-09-23T18:52:23.981 INFO:teuthology.orchestra.run.smithi092.stdout: python3-rados python3-rbd python3-repoze.lru python3-requests-oauthlib 2023-09-23T18:52:23.981 INFO:teuthology.orchestra.run.smithi092.stdout: python3-rgw python3-routes python3-rsa python3-scipy python3-simplegeneric 2023-09-23T18:52:23.982 INFO:teuthology.orchestra.run.smithi092.stdout: python3-simplejson python3-singledispatch python3-sklearn 2023-09-23T18:52:23.982 INFO:teuthology.orchestra.run.smithi092.stdout: python3-sklearn-lib python3-soupsieve python3-tempita python3-waitress 2023-09-23T18:52:23.982 INFO:teuthology.orchestra.run.smithi092.stdout: python3-wcwidth python3-webencodings python3-webob python3-websocket 2023-09-23T18:52:23.982 INFO:teuthology.orchestra.run.smithi092.stdout: python3-webtest python3-werkzeug radosgw rbd-fuse socat xmlstarlet 2023-09-23T18:52:23.982 INFO:teuthology.orchestra.run.smithi092.stdout:The following packages will be upgraded: 2023-09-23T18:52:23.982 INFO:teuthology.orchestra.run.smithi092.stdout: librados2 librbd1 2023-09-23T18:52:24.046 INFO:teuthology.orchestra.run.smithi092.stdout:2 upgraded, 98 newly installed, 0 to remove and 248 not upgraded. 2023-09-23T18:52:24.046 INFO:teuthology.orchestra.run.smithi092.stdout:Need to get 174 MB of archives. 2023-09-23T18:52:24.046 INFO:teuthology.orchestra.run.smithi092.stdout:After this operation, 804 MB of additional disk space will be used. 2023-09-23T18:52:24.046 INFO:teuthology.orchestra.run.smithi092.stdout:Get:1 http://archive.ubuntu.com/ubuntu focal/main amd64 liblttng-ust0 amd64 2.11.0-1 [160 kB] 2023-09-23T18:52:24.059 INFO:teuthology.orchestra.run.smithi016.stdout:The following NEW packages will be installed: 2023-09-23T18:52:24.059 INFO:teuthology.orchestra.run.smithi016.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm 2023-09-23T18:52:24.059 INFO:teuthology.orchestra.run.smithi016.stdout: ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-k8sevents 2023-09-23T18:52:24.059 INFO:teuthology.orchestra.run.smithi016.stdout: ceph-mgr-modules-core ceph-mon ceph-osd ceph-test cephadm jq libcephfs-dev 2023-09-23T18:52:24.060 INFO:teuthology.orchestra.run.smithi016.stdout: libcephfs2 libjaeger libjq1 libjs-sphinxdoc libjs-underscore liblbfgsb0 2023-09-23T18:52:24.060 INFO:teuthology.orchestra.run.smithi016.stdout: libleveldb1d liblttng-ust0 liblua5.3-0 liboath0 libonig5 librabbitmq4 2023-09-23T18:52:24.060 INFO:teuthology.orchestra.run.smithi016.stdout: libradosstriper1 librdkafka1 librgw2 libsqlite3-mod-ceph nvme-cli 2023-09-23T18:52:24.060 INFO:teuthology.orchestra.run.smithi016.stdout: python-pastedeploy-tpl python3-atomicwrites python3-attr python3-bcrypt 2023-09-23T18:52:24.060 INFO:teuthology.orchestra.run.smithi016.stdout: python3-blinker python3-bs4 python3-cachetools python3-ceph-argparse 2023-09-23T18:52:24.061 INFO:teuthology.orchestra.run.smithi016.stdout: python3-ceph-common python3-cephfs python3-cherrypy3 python3-cryptography 2023-09-23T18:52:24.061 INFO:teuthology.orchestra.run.smithi016.stdout: python3-decorator python3-google-auth python3-html5lib python3-jinja2 2023-09-23T18:52:24.061 INFO:teuthology.orchestra.run.smithi016.stdout: python3-joblib python3-jwt python3-kubernetes python3-logutils python3-lxml 2023-09-23T18:52:24.061 INFO:teuthology.orchestra.run.smithi016.stdout: python3-mako python3-markupsafe python3-oauthlib python3-openssl 2023-09-23T18:52:24.061 INFO:teuthology.orchestra.run.smithi016.stdout: python3-packaging python3-paste python3-pastedeploy python3-pastescript 2023-09-23T18:52:24.061 INFO:teuthology.orchestra.run.smithi016.stdout: python3-pecan python3-pluggy python3-prettytable python3-psutil python3-py 2023-09-23T18:52:24.061 INFO:teuthology.orchestra.run.smithi016.stdout: python3-pyasn1 python3-pyasn1-modules python3-pyinotify python3-pytest 2023-09-23T18:52:24.061 INFO:teuthology.orchestra.run.smithi016.stdout: python3-rados python3-rbd python3-repoze.lru python3-requests-oauthlib 2023-09-23T18:52:24.061 INFO:teuthology.orchestra.run.smithi016.stdout: python3-rgw python3-routes python3-rsa python3-scipy python3-simplegeneric 2023-09-23T18:52:24.061 INFO:teuthology.orchestra.run.smithi016.stdout: python3-simplejson python3-singledispatch python3-sklearn 2023-09-23T18:52:24.061 INFO:teuthology.orchestra.run.smithi016.stdout: python3-sklearn-lib python3-soupsieve python3-tempita python3-waitress 2023-09-23T18:52:24.061 INFO:teuthology.orchestra.run.smithi016.stdout: python3-wcwidth python3-webencodings python3-webob python3-websocket 2023-09-23T18:52:24.061 INFO:teuthology.orchestra.run.smithi016.stdout: python3-webtest python3-werkzeug radosgw rbd-fuse socat xmlstarlet 2023-09-23T18:52:24.062 INFO:teuthology.orchestra.run.smithi016.stdout:The following packages will be upgraded: 2023-09-23T18:52:24.062 INFO:teuthology.orchestra.run.smithi016.stdout: librados2 librbd1 2023-09-23T18:52:24.106 INFO:teuthology.orchestra.run.smithi092.stdout:Get:2 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 libjaeger amd64 16.2.14-37-g2e1f1015-1focal [3,952 B] 2023-09-23T18:52:24.127 INFO:teuthology.orchestra.run.smithi092.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-prettytable all 0.7.2-5 [20.1 kB] 2023-09-23T18:52:24.128 INFO:teuthology.orchestra.run.smithi092.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal/main amd64 liblua5.3-0 amd64 5.3.3-1.1ubuntu2 [116 kB] 2023-09-23T18:52:24.129 INFO:teuthology.orchestra.run.smithi016.stdout:2 upgraded, 98 newly installed, 0 to remove and 248 not upgraded. 2023-09-23T18:52:24.129 INFO:teuthology.orchestra.run.smithi016.stdout:Need to get 174 MB of archives. 2023-09-23T18:52:24.129 INFO:teuthology.orchestra.run.smithi016.stdout:After this operation, 804 MB of additional disk space will be used. 2023-09-23T18:52:24.129 INFO:teuthology.orchestra.run.smithi016.stdout:Get:1 http://archive.ubuntu.com/ubuntu focal/main amd64 liblttng-ust0 amd64 2.11.0-1 [160 kB] 2023-09-23T18:52:24.137 INFO:teuthology.orchestra.run.smithi092.stdout:Get:5 http://archive.ubuntu.com/ubuntu focal/main amd64 librabbitmq4 amd64 0.10.0-1 [35.8 kB] 2023-09-23T18:52:24.139 INFO:teuthology.orchestra.run.smithi092.stdout:Get:6 http://archive.ubuntu.com/ubuntu focal/universe amd64 librdkafka1 amd64 1.2.1-1ubuntu1 [410 kB] 2023-09-23T18:52:24.151 INFO:teuthology.orchestra.run.smithi092.stdout:Get:7 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 librbd1 amd64 16.2.14-37-g2e1f1015-1focal [3,385 kB] 2023-09-23T18:52:24.166 INFO:teuthology.orchestra.run.smithi092.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal/main amd64 libleveldb1d amd64 1.22-3ubuntu2 [139 kB] 2023-09-23T18:52:24.170 INFO:teuthology.orchestra.run.smithi092.stdout:Get:9 http://archive.ubuntu.com/ubuntu focal/main amd64 liboath0 amd64 2.6.1-1.3 [44.7 kB] 2023-09-23T18:52:24.172 INFO:teuthology.orchestra.run.smithi092.stdout:Get:10 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-cryptography amd64 2.8-3ubuntu0.1 [211 kB] 2023-09-23T18:52:24.178 INFO:teuthology.orchestra.run.smithi092.stdout:Get:11 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-openssl all 19.0.0-1build1 [43.3 kB] 2023-09-23T18:52:24.179 INFO:teuthology.orchestra.run.smithi092.stdout:Get:12 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-bcrypt amd64 3.1.7-2ubuntu1 [30.4 kB] 2023-09-23T18:52:24.180 INFO:teuthology.orchestra.run.smithi092.stdout:Get:13 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-cherrypy3 all 8.9.1-6 [160 kB] 2023-09-23T18:52:24.183 INFO:teuthology.orchestra.run.smithi092.stdout:Get:14 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-jwt all 1.7.1-2ubuntu2.1 [18.0 kB] 2023-09-23T18:52:24.184 INFO:teuthology.orchestra.run.smithi092.stdout:Get:15 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-logutils all 0.3.3-7 [17.3 kB] 2023-09-23T18:52:24.184 INFO:teuthology.orchestra.run.smithi092.stdout:Get:16 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-markupsafe amd64 1.1.0-1build2 [13.9 kB] 2023-09-23T18:52:24.185 INFO:teuthology.orchestra.run.smithi092.stdout:Get:17 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-mako all 1.1.0+ds1-1ubuntu2.1 [59.2 kB] 2023-09-23T18:52:24.201 INFO:teuthology.orchestra.run.smithi092.stdout:Get:18 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-simplegeneric all 0.8.1-3 [11.3 kB] 2023-09-23T18:52:24.202 INFO:teuthology.orchestra.run.smithi092.stdout:Get:19 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-singledispatch all 3.4.0.3-2build1 [7,128 B] 2023-09-23T18:52:24.203 INFO:teuthology.orchestra.run.smithi092.stdout:Get:20 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webob all 1:1.8.5-2 [84.4 kB] 2023-09-23T18:52:24.205 INFO:teuthology.orchestra.run.smithi092.stdout:Get:21 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-soupsieve all 1.9.5+dfsg-1 [29.1 kB] 2023-09-23T18:52:24.206 INFO:teuthology.orchestra.run.smithi092.stdout:Get:22 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-bs4 all 4.8.2-1 [83.0 kB] 2023-09-23T18:52:24.208 INFO:teuthology.orchestra.run.smithi092.stdout:Get:23 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-waitress all 1.4.1-1ubuntu0.1 [81.4 kB] 2023-09-23T18:52:24.217 INFO:teuthology.orchestra.run.smithi092.stdout:Get:24 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-tempita all 0.5.2-6 [14.8 kB] 2023-09-23T18:52:24.218 INFO:teuthology.orchestra.run.smithi092.stdout:Get:25 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-paste all 3.4.0+dfsg1-1 [456 kB] 2023-09-23T18:52:24.223 INFO:teuthology.orchestra.run.smithi092.stdout:Get:26 http://archive.ubuntu.com/ubuntu focal/main amd64 python-pastedeploy-tpl all 2.1.0-1 [5,016 B] 2023-09-23T18:52:24.223 INFO:teuthology.orchestra.run.smithi092.stdout:Get:27 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pastedeploy all 2.1.0-1 [26.5 kB] 2023-09-23T18:52:24.224 INFO:teuthology.orchestra.run.smithi016.stdout:Get:2 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-prettytable all 0.7.2-5 [20.1 kB] 2023-09-23T18:52:24.226 INFO:teuthology.orchestra.run.smithi016.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal/main amd64 liblua5.3-0 amd64 5.3.3-1.1ubuntu2 [116 kB] 2023-09-23T18:52:24.234 INFO:teuthology.orchestra.run.smithi092.stdout:Get:28 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webtest all 2.0.32-2 [28.2 kB] 2023-09-23T18:52:24.235 INFO:teuthology.orchestra.run.smithi092.stdout:Get:29 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pecan all 1.3.3-3ubuntu1 [85.8 kB] 2023-09-23T18:52:24.236 INFO:teuthology.orchestra.run.smithi092.stdout:Get:30 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-werkzeug all 0.16.1+dfsg1-2ubuntu0.1 [183 kB] 2023-09-23T18:52:24.236 INFO:teuthology.orchestra.run.smithi016.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal/main amd64 librabbitmq4 amd64 0.10.0-1 [35.8 kB] 2023-09-23T18:52:24.238 INFO:teuthology.orchestra.run.smithi092.stdout:Get:31 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-jinja2 all 2.10.1-2 [95.5 kB] 2023-09-23T18:52:24.239 INFO:teuthology.orchestra.run.smithi092.stdout:Get:32 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB] 2023-09-23T18:52:24.240 INFO:teuthology.orchestra.run.smithi092.stdout:Get:33 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 libjs-underscore all 1.9.1~dfsg-1ubuntu0.20.04.1 [99.5 kB] 2023-09-23T18:52:24.244 INFO:teuthology.orchestra.run.smithi016.stdout:Get:5 http://archive.ubuntu.com/ubuntu focal/universe amd64 librdkafka1 amd64 1.2.1-1ubuntu1 [410 kB] 2023-09-23T18:52:24.249 INFO:teuthology.orchestra.run.smithi016.stdout:Get:6 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 libjaeger amd64 16.2.14-37-g2e1f1015-1focal [3,952 B] 2023-09-23T18:52:24.251 INFO:teuthology.orchestra.run.smithi092.stdout:Get:34 http://archive.ubuntu.com/ubuntu focal/main amd64 libjs-sphinxdoc all 1.8.5-7ubuntu3 [97.1 kB] 2023-09-23T18:52:24.252 INFO:teuthology.orchestra.run.smithi092.stdout:Get:35 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-routes all 2.4.1-2 [88.1 kB] 2023-09-23T18:52:24.254 INFO:teuthology.orchestra.run.smithi092.stdout:Get:36 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-decorator all 4.4.2-0ubuntu1 [10.3 kB] 2023-09-23T18:52:24.254 INFO:teuthology.orchestra.run.smithi092.stdout:Get:37 http://archive.ubuntu.com/ubuntu focal/universe amd64 liblbfgsb0 amd64 3.0+dfsg.3-7build1 [29.5 kB] 2023-09-23T18:52:24.268 INFO:teuthology.orchestra.run.smithi092.stdout:Get:38 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-scipy amd64 1.3.3-3build1 [11.1 MB] 2023-09-23T18:52:24.271 INFO:teuthology.orchestra.run.smithi016.stdout:Get:7 http://archive.ubuntu.com/ubuntu focal/main amd64 libleveldb1d amd64 1.22-3ubuntu2 [139 kB] 2023-09-23T18:52:24.276 INFO:teuthology.orchestra.run.smithi016.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal/main amd64 liboath0 amd64 2.6.1-1.3 [44.7 kB] 2023-09-23T18:52:24.278 INFO:teuthology.orchestra.run.smithi016.stdout:Get:9 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-cryptography amd64 2.8-3ubuntu0.1 [211 kB] 2023-09-23T18:52:24.285 INFO:teuthology.orchestra.run.smithi016.stdout:Get:10 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-openssl all 19.0.0-1build1 [43.3 kB] 2023-09-23T18:52:24.286 INFO:teuthology.orchestra.run.smithi016.stdout:Get:11 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-bcrypt amd64 3.1.7-2ubuntu1 [30.4 kB] 2023-09-23T18:52:24.287 INFO:teuthology.orchestra.run.smithi016.stdout:Get:12 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-cherrypy3 all 8.9.1-6 [160 kB] 2023-09-23T18:52:24.291 INFO:teuthology.orchestra.run.smithi016.stdout:Get:13 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-jwt all 1.7.1-2ubuntu2.1 [18.0 kB] 2023-09-23T18:52:24.292 INFO:teuthology.orchestra.run.smithi016.stdout:Get:14 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-logutils all 0.3.3-7 [17.3 kB] 2023-09-23T18:52:24.292 INFO:teuthology.orchestra.run.smithi016.stdout:Get:15 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-markupsafe amd64 1.1.0-1build2 [13.9 kB] 2023-09-23T18:52:24.304 INFO:teuthology.orchestra.run.smithi016.stdout:Get:16 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-mako all 1.1.0+ds1-1ubuntu2.1 [59.2 kB] 2023-09-23T18:52:24.306 INFO:teuthology.orchestra.run.smithi016.stdout:Get:17 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-simplegeneric all 0.8.1-3 [11.3 kB] 2023-09-23T18:52:24.307 INFO:teuthology.orchestra.run.smithi016.stdout:Get:18 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-singledispatch all 3.4.0.3-2build1 [7,128 B] 2023-09-23T18:52:24.324 INFO:teuthology.orchestra.run.smithi016.stdout:Get:19 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 librbd1 amd64 16.2.14-37-g2e1f1015-1focal [3,385 kB] 2023-09-23T18:52:24.324 INFO:teuthology.orchestra.run.smithi016.stdout:Get:20 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webob all 1:1.8.5-2 [84.4 kB] 2023-09-23T18:52:24.326 INFO:teuthology.orchestra.run.smithi016.stdout:Get:21 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-soupsieve all 1.9.5+dfsg-1 [29.1 kB] 2023-09-23T18:52:24.327 INFO:teuthology.orchestra.run.smithi016.stdout:Get:22 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-bs4 all 4.8.2-1 [83.0 kB] 2023-09-23T18:52:24.329 INFO:teuthology.orchestra.run.smithi016.stdout:Get:23 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-waitress all 1.4.1-1ubuntu0.1 [81.4 kB] 2023-09-23T18:52:24.332 INFO:teuthology.orchestra.run.smithi016.stdout:Get:24 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-tempita all 0.5.2-6 [14.8 kB] 2023-09-23T18:52:24.333 INFO:teuthology.orchestra.run.smithi016.stdout:Get:25 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-paste all 3.4.0+dfsg1-1 [456 kB] 2023-09-23T18:52:24.337 INFO:teuthology.orchestra.run.smithi016.stdout:Get:26 http://archive.ubuntu.com/ubuntu focal/main amd64 python-pastedeploy-tpl all 2.1.0-1 [5,016 B] 2023-09-23T18:52:24.344 INFO:teuthology.orchestra.run.smithi016.stdout:Get:27 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pastedeploy all 2.1.0-1 [26.5 kB] 2023-09-23T18:52:24.345 INFO:teuthology.orchestra.run.smithi016.stdout:Get:28 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webtest all 2.0.32-2 [28.2 kB] 2023-09-23T18:52:24.345 INFO:teuthology.orchestra.run.smithi016.stdout:Get:29 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pecan all 1.3.3-3ubuntu1 [85.8 kB] 2023-09-23T18:52:24.350 INFO:teuthology.orchestra.run.smithi092.stdout:Get:39 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-sklearn-lib amd64 0.22.2.post1+dfsg-5 [2,358 kB] 2023-09-23T18:52:24.361 INFO:teuthology.orchestra.run.smithi092.stdout:Get:40 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-joblib all 0.14.0-3 [192 kB] 2023-09-23T18:52:24.363 INFO:teuthology.orchestra.run.smithi092.stdout:Get:41 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-sklearn all 0.22.2.post1+dfsg-5 [1,773 kB] 2023-09-23T18:52:24.364 INFO:teuthology.orchestra.run.smithi016.stdout:Get:30 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-werkzeug all 0.16.1+dfsg1-2ubuntu0.1 [183 kB] 2023-09-23T18:52:24.368 INFO:teuthology.orchestra.run.smithi016.stdout:Get:31 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-jinja2 all 2.10.1-2 [95.5 kB] 2023-09-23T18:52:24.370 INFO:teuthology.orchestra.run.smithi016.stdout:Get:32 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB] 2023-09-23T18:52:24.371 INFO:teuthology.orchestra.run.smithi016.stdout:Get:33 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 libjs-underscore all 1.9.1~dfsg-1ubuntu0.20.04.1 [99.5 kB] 2023-09-23T18:52:24.374 INFO:teuthology.orchestra.run.smithi016.stdout:Get:34 http://archive.ubuntu.com/ubuntu focal/main amd64 libjs-sphinxdoc all 1.8.5-7ubuntu3 [97.1 kB] 2023-09-23T18:52:24.374 INFO:teuthology.orchestra.run.smithi092.stdout:Get:42 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-cachetools all 4.0.0-1 [9,820 B] 2023-09-23T18:52:24.374 INFO:teuthology.orchestra.run.smithi092.stdout:Get:43 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyasn1 all 0.4.2-3build1 [46.7 kB] 2023-09-23T18:52:24.375 INFO:teuthology.orchestra.run.smithi016.stdout:Get:35 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-routes all 2.4.1-2 [88.1 kB] 2023-09-23T18:52:24.376 INFO:teuthology.orchestra.run.smithi016.stdout:Get:36 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-decorator all 4.4.2-0ubuntu1 [10.3 kB] 2023-09-23T18:52:24.383 INFO:teuthology.orchestra.run.smithi016.stdout:Get:37 http://archive.ubuntu.com/ubuntu focal/universe amd64 liblbfgsb0 amd64 3.0+dfsg.3-7build1 [29.5 kB] 2023-09-23T18:52:24.384 INFO:teuthology.orchestra.run.smithi016.stdout:Get:38 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-scipy amd64 1.3.3-3build1 [11.1 MB] 2023-09-23T18:52:24.434 INFO:teuthology.orchestra.run.smithi092.stdout:Get:44 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyasn1-modules all 0.2.1-0.2build1 [32.9 kB] 2023-09-23T18:52:24.434 INFO:teuthology.orchestra.run.smithi092.stdout:Get:45 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-rsa all 4.0-3 [24.7 kB] 2023-09-23T18:52:24.435 INFO:teuthology.orchestra.run.smithi092.stdout:Get:46 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-google-auth all 1.5.1-2 [35.6 kB] 2023-09-23T18:52:24.435 INFO:teuthology.orchestra.run.smithi092.stdout:Get:47 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-blinker all 1.4+dfsg1-0.3ubuntu1 [13.2 kB] 2023-09-23T18:52:24.435 INFO:teuthology.orchestra.run.smithi092.stdout:Get:48 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-oauthlib all 3.1.0-1ubuntu2 [84.8 kB] 2023-09-23T18:52:24.436 INFO:teuthology.orchestra.run.smithi092.stdout:Get:49 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-requests-oauthlib all 1.0.0-1.1build1 [16.8 kB] 2023-09-23T18:52:24.437 INFO:teuthology.orchestra.run.smithi092.stdout:Get:50 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-websocket all 0.53.0-2ubuntu1 [32.3 kB] 2023-09-23T18:52:24.454 INFO:teuthology.orchestra.run.smithi092.stdout:Get:51 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-kubernetes all 7.0.0~a1-2 [271 kB] 2023-09-23T18:52:24.459 INFO:teuthology.orchestra.run.smithi092.stdout:Get:52 http://archive.ubuntu.com/ubuntu focal/universe amd64 libonig5 amd64 6.9.4-1 [142 kB] 2023-09-23T18:52:24.462 INFO:teuthology.orchestra.run.smithi092.stdout:Get:53 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 libjq1 amd64 1.6-1ubuntu0.20.04.1 [121 kB] 2023-09-23T18:52:24.464 INFO:teuthology.orchestra.run.smithi092.stdout:Get:54 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 jq amd64 1.6-1ubuntu0.20.04.1 [50.2 kB] 2023-09-23T18:52:24.465 INFO:teuthology.orchestra.run.smithi092.stdout:Get:55 http://archive.ubuntu.com/ubuntu focal/main amd64 socat amd64 1.7.3.3-2 [323 kB] 2023-09-23T18:52:24.467 INFO:teuthology.orchestra.run.smithi092.stdout:Get:56 http://archive.ubuntu.com/ubuntu focal/universe amd64 xmlstarlet amd64 1.6.1-2build1 [267 kB] 2023-09-23T18:52:24.469 INFO:teuthology.orchestra.run.smithi092.stdout:Get:57 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 nvme-cli amd64 1.9-1ubuntu0.1 [283 kB] 2023-09-23T18:52:24.472 INFO:teuthology.orchestra.run.smithi092.stdout:Get:58 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-atomicwrites all 1.1.5-2build1 [7,016 B] 2023-09-23T18:52:24.472 INFO:teuthology.orchestra.run.smithi092.stdout:Get:59 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-attr all 19.3.0-2 [33.9 kB] 2023-09-23T18:52:24.472 INFO:teuthology.orchestra.run.smithi092.stdout:Get:60 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webencodings all 0.5.1-1ubuntu1 [11.0 kB] 2023-09-23T18:52:24.487 INFO:teuthology.orchestra.run.smithi092.stdout:Get:61 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-html5lib all 1.0.1-2 [84.3 kB] 2023-09-23T18:52:24.488 INFO:teuthology.orchestra.run.smithi092.stdout:Get:62 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-lxml amd64 4.5.0-1ubuntu0.5 [1,384 kB] 2023-09-23T18:52:24.504 INFO:teuthology.orchestra.run.smithi092.stdout:Get:63 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-packaging all 20.3-1 [26.8 kB] 2023-09-23T18:52:24.505 INFO:teuthology.orchestra.run.smithi092.stdout:Get:64 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pastescript all 2.0.2-4 [54.6 kB] 2023-09-23T18:52:24.505 INFO:teuthology.orchestra.run.smithi092.stdout:Get:65 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-pluggy all 0.13.0-2 [18.4 kB] 2023-09-23T18:52:24.505 INFO:teuthology.orchestra.run.smithi092.stdout:Get:66 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-psutil amd64 5.5.1-1ubuntu4 [141 kB] 2023-09-23T18:52:24.506 INFO:teuthology.orchestra.run.smithi092.stdout:Get:67 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 python3-py all 1.8.1-1ubuntu0.1 [65.6 kB] 2023-09-23T18:52:24.507 INFO:teuthology.orchestra.run.smithi092.stdout:Get:68 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyinotify all 0.9.6-1.2ubuntu1 [24.8 kB] 2023-09-23T18:52:24.507 INFO:teuthology.orchestra.run.smithi092.stdout:Get:69 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-wcwidth all 0.1.8+dfsg1-3 [17.4 kB] 2023-09-23T18:52:24.507 INFO:teuthology.orchestra.run.smithi092.stdout:Get:70 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-pytest all 4.6.9-1 [178 kB] 2023-09-23T18:52:24.519 INFO:teuthology.orchestra.run.smithi016.stdout:Get:39 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-sklearn-lib amd64 0.22.2.post1+dfsg-5 [2,358 kB] 2023-09-23T18:52:24.520 INFO:teuthology.orchestra.run.smithi092.stdout:Get:71 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-simplejson amd64 3.16.0-2ubuntu2 [50.8 kB] 2023-09-23T18:52:24.523 INFO:teuthology.orchestra.run.smithi092.stdout:Get:72 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 librados2 amd64 16.2.14-37-g2e1f1015-1focal [3,391 kB] 2023-09-23T18:52:24.534 INFO:teuthology.orchestra.run.smithi016.stdout:Get:40 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-joblib all 0.14.0-3 [192 kB] 2023-09-23T18:52:24.536 INFO:teuthology.orchestra.run.smithi016.stdout:Get:41 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-sklearn all 0.22.2.post1+dfsg-5 [1,773 kB] 2023-09-23T18:52:24.550 INFO:teuthology.orchestra.run.smithi016.stdout:Get:42 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-cachetools all 4.0.0-1 [9,820 B] 2023-09-23T18:52:24.550 INFO:teuthology.orchestra.run.smithi016.stdout:Get:43 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyasn1 all 0.4.2-3build1 [46.7 kB] 2023-09-23T18:52:24.553 INFO:teuthology.orchestra.run.smithi016.stdout:Get:44 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyasn1-modules all 0.2.1-0.2build1 [32.9 kB] 2023-09-23T18:52:24.556 INFO:teuthology.orchestra.run.smithi016.stdout:Get:45 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-rsa all 4.0-3 [24.7 kB] 2023-09-23T18:52:24.557 INFO:teuthology.orchestra.run.smithi016.stdout:Get:46 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-google-auth all 1.5.1-2 [35.6 kB] 2023-09-23T18:52:24.557 INFO:teuthology.orchestra.run.smithi016.stdout:Get:47 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-blinker all 1.4+dfsg1-0.3ubuntu1 [13.2 kB] 2023-09-23T18:52:24.557 INFO:teuthology.orchestra.run.smithi016.stdout:Get:48 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-oauthlib all 3.1.0-1ubuntu2 [84.8 kB] 2023-09-23T18:52:24.560 INFO:teuthology.orchestra.run.smithi016.stdout:Get:49 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-requests-oauthlib all 1.0.0-1.1build1 [16.8 kB] 2023-09-23T18:52:24.561 INFO:teuthology.orchestra.run.smithi016.stdout:Get:50 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-websocket all 0.53.0-2ubuntu1 [32.3 kB] 2023-09-23T18:52:24.580 INFO:teuthology.orchestra.run.smithi016.stdout:Get:51 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-kubernetes all 7.0.0~a1-2 [271 kB] 2023-09-23T18:52:24.585 INFO:teuthology.orchestra.run.smithi016.stdout:Get:52 http://archive.ubuntu.com/ubuntu focal/universe amd64 libonig5 amd64 6.9.4-1 [142 kB] 2023-09-23T18:52:24.587 INFO:teuthology.orchestra.run.smithi016.stdout:Get:53 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 libjq1 amd64 1.6-1ubuntu0.20.04.1 [121 kB] 2023-09-23T18:52:24.589 INFO:teuthology.orchestra.run.smithi016.stdout:Get:54 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 jq amd64 1.6-1ubuntu0.20.04.1 [50.2 kB] 2023-09-23T18:52:24.590 INFO:teuthology.orchestra.run.smithi016.stdout:Get:55 http://archive.ubuntu.com/ubuntu focal/main amd64 socat amd64 1.7.3.3-2 [323 kB] 2023-09-23T18:52:24.593 INFO:teuthology.orchestra.run.smithi016.stdout:Get:56 http://archive.ubuntu.com/ubuntu focal/universe amd64 xmlstarlet amd64 1.6.1-2build1 [267 kB] 2023-09-23T18:52:24.596 INFO:teuthology.orchestra.run.smithi016.stdout:Get:57 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 nvme-cli amd64 1.9-1ubuntu0.1 [283 kB] 2023-09-23T18:52:24.599 INFO:teuthology.orchestra.run.smithi016.stdout:Get:58 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-atomicwrites all 1.1.5-2build1 [7,016 B] 2023-09-23T18:52:24.601 INFO:teuthology.orchestra.run.smithi016.stdout:Get:59 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-attr all 19.3.0-2 [33.9 kB] 2023-09-23T18:52:24.601 INFO:teuthology.orchestra.run.smithi016.stdout:Get:60 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webencodings all 0.5.1-1ubuntu1 [11.0 kB] 2023-09-23T18:52:24.621 INFO:teuthology.orchestra.run.smithi016.stdout:Get:61 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-html5lib all 1.0.1-2 [84.3 kB] 2023-09-23T18:52:24.623 INFO:teuthology.orchestra.run.smithi016.stdout:Get:62 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-lxml amd64 4.5.0-1ubuntu0.5 [1,384 kB] 2023-09-23T18:52:24.641 INFO:teuthology.orchestra.run.smithi016.stdout:Get:63 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-packaging all 20.3-1 [26.8 kB] 2023-09-23T18:52:24.641 INFO:teuthology.orchestra.run.smithi016.stdout:Get:64 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pastescript all 2.0.2-4 [54.6 kB] 2023-09-23T18:52:24.641 INFO:teuthology.orchestra.run.smithi016.stdout:Get:65 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-pluggy all 0.13.0-2 [18.4 kB] 2023-09-23T18:52:24.642 INFO:teuthology.orchestra.run.smithi016.stdout:Get:66 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-psutil amd64 5.5.1-1ubuntu4 [141 kB] 2023-09-23T18:52:24.643 INFO:teuthology.orchestra.run.smithi016.stdout:Get:67 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 python3-py all 1.8.1-1ubuntu0.1 [65.6 kB] 2023-09-23T18:52:24.644 INFO:teuthology.orchestra.run.smithi016.stdout:Get:68 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyinotify all 0.9.6-1.2ubuntu1 [24.8 kB] 2023-09-23T18:52:24.644 INFO:teuthology.orchestra.run.smithi016.stdout:Get:69 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-wcwidth all 0.1.8+dfsg1-3 [17.4 kB] 2023-09-23T18:52:24.644 INFO:teuthology.orchestra.run.smithi016.stdout:Get:70 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-pytest all 4.6.9-1 [178 kB] 2023-09-23T18:52:24.661 INFO:teuthology.orchestra.run.smithi016.stdout:Get:71 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-simplejson amd64 3.16.0-2ubuntu2 [50.8 kB] 2023-09-23T18:52:24.710 INFO:teuthology.orchestra.run.smithi092.stdout:Get:73 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 libcephfs2 amd64 16.2.14-37-g2e1f1015-1focal [729 kB] 2023-09-23T18:52:24.748 INFO:teuthology.orchestra.run.smithi092.stdout:Get:74 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 python3-rados amd64 16.2.14-37-g2e1f1015-1focal [413 kB] 2023-09-23T18:52:24.767 INFO:teuthology.orchestra.run.smithi092.stdout:Get:75 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 python3-ceph-argparse all 16.2.14-37-g2e1f1015-1focal [22.2 kB] 2023-09-23T18:52:24.768 INFO:teuthology.orchestra.run.smithi092.stdout:Get:76 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 python3-cephfs amd64 16.2.14-37-g2e1f1015-1focal [212 kB] 2023-09-23T18:52:24.772 INFO:teuthology.orchestra.run.smithi092.stdout:Get:77 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 python3-ceph-common all 16.2.14-37-g2e1f1015-1focal [38.4 kB] 2023-09-23T18:52:24.774 INFO:teuthology.orchestra.run.smithi092.stdout:Get:78 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 python3-rbd amd64 16.2.14-37-g2e1f1015-1focal [386 kB] 2023-09-23T18:52:24.790 INFO:teuthology.orchestra.run.smithi092.stdout:Get:79 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 librgw2 amd64 16.2.14-37-g2e1f1015-1focal [3,689 kB] 2023-09-23T18:52:24.934 INFO:teuthology.orchestra.run.smithi092.stdout:Get:80 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 python3-rgw amd64 16.2.14-37-g2e1f1015-1focal [116 kB] 2023-09-23T18:52:24.939 INFO:teuthology.orchestra.run.smithi092.stdout:Get:81 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 libradosstriper1 amd64 16.2.14-37-g2e1f1015-1focal [448 kB] 2023-09-23T18:52:24.957 INFO:teuthology.orchestra.run.smithi092.stdout:Get:82 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph-common amd64 16.2.14-37-g2e1f1015-1focal [22.6 MB] 2023-09-23T18:52:25.046 INFO:teuthology.orchestra.run.smithi016.stdout:Get:72 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 librados2 amd64 16.2.14-37-g2e1f1015-1focal [3,391 kB] 2023-09-23T18:52:25.610 INFO:teuthology.orchestra.run.smithi016.stdout:Get:73 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 libcephfs2 amd64 16.2.14-37-g2e1f1015-1focal [729 kB] 2023-09-23T18:52:25.732 INFO:teuthology.orchestra.run.smithi016.stdout:Get:74 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 python3-rados amd64 16.2.14-37-g2e1f1015-1focal [413 kB] 2023-09-23T18:52:25.801 INFO:teuthology.orchestra.run.smithi016.stdout:Get:75 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 python3-ceph-argparse all 16.2.14-37-g2e1f1015-1focal [22.2 kB] 2023-09-23T18:52:25.803 INFO:teuthology.orchestra.run.smithi016.stdout:Get:76 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 python3-cephfs amd64 16.2.14-37-g2e1f1015-1focal [212 kB] 2023-09-23T18:52:25.838 INFO:teuthology.orchestra.run.smithi016.stdout:Get:77 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 python3-ceph-common all 16.2.14-37-g2e1f1015-1focal [38.4 kB] 2023-09-23T18:52:25.844 INFO:teuthology.orchestra.run.smithi016.stdout:Get:78 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 python3-rbd amd64 16.2.14-37-g2e1f1015-1focal [386 kB] 2023-09-23T18:52:25.909 INFO:teuthology.orchestra.run.smithi016.stdout:Get:79 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 librgw2 amd64 16.2.14-37-g2e1f1015-1focal [3,689 kB] 2023-09-23T18:52:25.985 INFO:teuthology.orchestra.run.smithi092.stdout:Get:83 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph-base amd64 16.2.14-37-g2e1f1015-1focal [5,913 kB] 2023-09-23T18:52:26.173 INFO:teuthology.orchestra.run.smithi092.stdout:Get:84 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-modules-core all 16.2.14-37-g2e1f1015-1focal [204 kB] 2023-09-23T18:52:26.174 INFO:teuthology.orchestra.run.smithi092.stdout:Get:85 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 libsqlite3-mod-ceph amd64 16.2.14-37-g2e1f1015-1focal [126 kB] 2023-09-23T18:52:26.175 INFO:teuthology.orchestra.run.smithi092.stdout:Get:86 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr amd64 16.2.14-37-g2e1f1015-1focal [1,536 kB] 2023-09-23T18:52:26.225 INFO:teuthology.orchestra.run.smithi092.stdout:Get:87 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph-mon amd64 16.2.14-37-g2e1f1015-1focal [6,995 kB] 2023-09-23T18:52:26.451 INFO:teuthology.orchestra.run.smithi016.stdout:Get:80 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 python3-rgw amd64 16.2.14-37-g2e1f1015-1focal [116 kB] 2023-09-23T18:52:26.477 INFO:teuthology.orchestra.run.smithi016.stdout:Get:81 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 libradosstriper1 amd64 16.2.14-37-g2e1f1015-1focal [448 kB] 2023-09-23T18:52:26.505 INFO:teuthology.orchestra.run.smithi092.stdout:Get:88 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph-osd amd64 16.2.14-37-g2e1f1015-1focal [26.2 MB] 2023-09-23T18:52:26.548 INFO:teuthology.orchestra.run.smithi016.stdout:Get:82 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph-common amd64 16.2.14-37-g2e1f1015-1focal [22.6 MB] 2023-09-23T18:52:27.690 INFO:teuthology.orchestra.run.smithi092.stdout:Get:89 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph amd64 16.2.14-37-g2e1f1015-1focal [4,048 B] 2023-09-23T18:52:27.691 INFO:teuthology.orchestra.run.smithi092.stdout:Get:90 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph-fuse amd64 16.2.14-37-g2e1f1015-1focal [834 kB] 2023-09-23T18:52:27.696 INFO:teuthology.orchestra.run.smithi092.stdout:Get:91 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph-mds amd64 16.2.14-37-g2e1f1015-1focal [2,363 kB] 2023-09-23T18:52:27.735 INFO:teuthology.orchestra.run.smithi092.stdout:Get:92 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 cephadm amd64 16.2.14-37-g2e1f1015-1focal [76.4 kB] 2023-09-23T18:52:27.737 INFO:teuthology.orchestra.run.smithi092.stdout:Get:93 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-cephadm all 16.2.14-37-g2e1f1015-1focal [100 kB] 2023-09-23T18:52:27.740 INFO:teuthology.orchestra.run.smithi092.stdout:Get:94 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-dashboard all 16.2.14-37-g2e1f1015-1focal [2,823 kB] 2023-09-23T18:52:27.852 INFO:teuthology.orchestra.run.smithi092.stdout:Get:95 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-diskprediction-local all 16.2.14-37-g2e1f1015-1focal [7,891 kB] 2023-09-23T18:52:28.185 INFO:teuthology.orchestra.run.smithi092.stdout:Get:96 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-k8sevents all 16.2.14-37-g2e1f1015-1focal [4,164 B] 2023-09-23T18:52:28.186 INFO:teuthology.orchestra.run.smithi092.stdout:Get:97 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph-test amd64 16.2.14-37-g2e1f1015-1focal [49.3 MB] 2023-09-23T18:52:30.034 INFO:teuthology.orchestra.run.smithi016.stdout:Get:83 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph-base amd64 16.2.14-37-g2e1f1015-1focal [5,913 kB] 2023-09-23T18:52:30.489 INFO:teuthology.orchestra.run.smithi092.stdout:Get:98 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 libcephfs-dev amd64 16.2.14-37-g2e1f1015-1focal [21.5 kB] 2023-09-23T18:52:30.489 INFO:teuthology.orchestra.run.smithi092.stdout:Get:99 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 radosgw amd64 16.2.14-37-g2e1f1015-1focal [11.3 MB] 2023-09-23T18:52:30.601 INFO:teuthology.orchestra.run.smithi016.stdout:Get:84 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-modules-core all 16.2.14-37-g2e1f1015-1focal [204 kB] 2023-09-23T18:52:30.640 INFO:teuthology.orchestra.run.smithi016.stdout:Get:85 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 libsqlite3-mod-ceph amd64 16.2.14-37-g2e1f1015-1focal [126 kB] 2023-09-23T18:52:30.647 INFO:teuthology.orchestra.run.smithi016.stdout:Get:86 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr amd64 16.2.14-37-g2e1f1015-1focal [1,536 kB] 2023-09-23T18:52:30.829 INFO:teuthology.orchestra.run.smithi016.stdout:Get:87 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph-mon amd64 16.2.14-37-g2e1f1015-1focal [6,995 kB] 2023-09-23T18:52:30.871 INFO:teuthology.orchestra.run.smithi092.stdout:Get:100 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 rbd-fuse amd64 16.2.14-37-g2e1f1015-1focal [78.6 kB] 2023-09-23T18:52:31.219 INFO:teuthology.orchestra.run.smithi016.stdout:Get:88 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph-osd amd64 16.2.14-37-g2e1f1015-1focal [26.2 MB] 2023-09-23T18:52:31.514 INFO:teuthology.orchestra.run.smithi092.stdout:Fetched 174 MB in 7s (25.2 MB/s) 2023-09-23T18:52:31.589 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package libjaeger. 2023-09-23T18:52:31.633 INFO:teuthology.orchestra.run.smithi092.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 112648 files and directories currently installed.) 2023-09-23T18:52:31.637 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../00-libjaeger_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:52:31.679 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking libjaeger (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:52:32.007 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package liblttng-ust0:amd64. 2023-09-23T18:52:32.022 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../01-liblttng-ust0_2.11.0-1_amd64.deb ... 2023-09-23T18:52:32.107 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking liblttng-ust0:amd64 (2.11.0-1) ... 2023-09-23T18:52:32.443 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../02-librbd1_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:52:32.569 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking librbd1 (16.2.14-37-g2e1f1015-1focal) over (15.2.17-0ubuntu0.20.04.4) ... 2023-09-23T18:52:32.795 INFO:teuthology.orchestra.run.smithi016.stdout:Get:89 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph amd64 16.2.14-37-g2e1f1015-1focal [4,048 B] 2023-09-23T18:52:32.795 INFO:teuthology.orchestra.run.smithi016.stdout:Get:90 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph-fuse amd64 16.2.14-37-g2e1f1015-1focal [834 kB] 2023-09-23T18:52:32.801 INFO:teuthology.orchestra.run.smithi016.stdout:Get:91 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph-mds amd64 16.2.14-37-g2e1f1015-1focal [2,363 kB] 2023-09-23T18:52:32.912 INFO:teuthology.orchestra.run.smithi016.stdout:Get:92 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 cephadm amd64 16.2.14-37-g2e1f1015-1focal [76.4 kB] 2023-09-23T18:52:32.938 INFO:teuthology.orchestra.run.smithi016.stdout:Get:93 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-cephadm all 16.2.14-37-g2e1f1015-1focal [100 kB] 2023-09-23T18:52:32.941 INFO:teuthology.orchestra.run.smithi016.stdout:Get:94 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-dashboard all 16.2.14-37-g2e1f1015-1focal [2,823 kB] 2023-09-23T18:52:33.089 INFO:teuthology.orchestra.run.smithi016.stdout:Get:95 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-diskprediction-local all 16.2.14-37-g2e1f1015-1focal [7,891 kB] 2023-09-23T18:52:33.231 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../03-librados2_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:52:33.357 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking librados2 (16.2.14-37-g2e1f1015-1focal) over (15.2.17-0ubuntu0.20.04.4) ... 2023-09-23T18:52:33.532 INFO:teuthology.orchestra.run.smithi016.stdout:Get:96 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-k8sevents all 16.2.14-37-g2e1f1015-1focal [4,164 B] 2023-09-23T18:52:33.532 INFO:teuthology.orchestra.run.smithi016.stdout:Get:97 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph-test amd64 16.2.14-37-g2e1f1015-1focal [49.3 MB] 2023-09-23T18:52:34.019 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package libcephfs2. 2023-09-23T18:52:34.032 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../04-libcephfs2_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:52:34.077 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking libcephfs2 (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:52:34.413 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-rados. 2023-09-23T18:52:34.426 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../05-python3-rados_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:52:34.463 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-rados (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:52:34.808 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-ceph-argparse. 2023-09-23T18:52:34.823 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../06-python3-ceph-argparse_16.2.14-37-g2e1f1015-1focal_all.deb ... 2023-09-23T18:52:34.866 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-ceph-argparse (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:52:35.152 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-cephfs. 2023-09-23T18:52:35.167 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../07-python3-cephfs_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:52:35.210 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-cephfs (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:52:35.521 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-ceph-common. 2023-09-23T18:52:35.535 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../08-python3-ceph-common_16.2.14-37-g2e1f1015-1focal_all.deb ... 2023-09-23T18:52:35.580 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-ceph-common (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:52:35.908 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-prettytable. 2023-09-23T18:52:35.921 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../09-python3-prettytable_0.7.2-5_all.deb ... 2023-09-23T18:52:35.966 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-prettytable (0.7.2-5) ... 2023-09-23T18:52:36.210 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-rbd. 2023-09-23T18:52:36.223 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../10-python3-rbd_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:52:36.261 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-rbd (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:52:36.518 INFO:teuthology.orchestra.run.smithi016.stdout:Get:98 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 libcephfs-dev amd64 16.2.14-37-g2e1f1015-1focal [21.5 kB] 2023-09-23T18:52:36.519 INFO:teuthology.orchestra.run.smithi016.stdout:Get:99 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 radosgw amd64 16.2.14-37-g2e1f1015-1focal [11.3 MB] 2023-09-23T18:52:36.597 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package liblua5.3-0:amd64. 2023-09-23T18:52:36.610 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../11-liblua5.3-0_5.3.3-1.1ubuntu2_amd64.deb ... 2023-09-23T18:52:36.648 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking liblua5.3-0:amd64 (5.3.3-1.1ubuntu2) ... 2023-09-23T18:52:36.960 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package librabbitmq4:amd64. 2023-09-23T18:52:36.973 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../12-librabbitmq4_0.10.0-1_amd64.deb ... 2023-09-23T18:52:37.018 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking librabbitmq4:amd64 (0.10.0-1) ... 2023-09-23T18:52:37.050 INFO:teuthology.orchestra.run.smithi016.stdout:Get:100 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 rbd-fuse amd64 16.2.14-37-g2e1f1015-1focal [78.6 kB] 2023-09-23T18:52:37.335 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package librdkafka1:amd64. 2023-09-23T18:52:37.351 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../13-librdkafka1_1.2.1-1ubuntu1_amd64.deb ... 2023-09-23T18:52:37.393 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking librdkafka1:amd64 (1.2.1-1ubuntu1) ... 2023-09-23T18:52:37.713 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package librgw2. 2023-09-23T18:52:37.726 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../14-librgw2_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:52:37.729 INFO:teuthology.orchestra.run.smithi016.stdout:Fetched 174 MB in 13s (13.4 MB/s) 2023-09-23T18:52:37.771 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking librgw2 (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:52:37.820 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package libjaeger. 2023-09-23T18:52:37.867 INFO:teuthology.orchestra.run.smithi016.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 112648 files and directories currently installed.) 2023-09-23T18:52:37.871 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../00-libjaeger_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:52:37.910 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking libjaeger (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:52:38.230 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package liblttng-ust0:amd64. 2023-09-23T18:52:38.244 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../01-liblttng-ust0_2.11.0-1_amd64.deb ... 2023-09-23T18:52:38.313 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking liblttng-ust0:amd64 (2.11.0-1) ... 2023-09-23T18:52:38.366 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-rgw. 2023-09-23T18:52:38.379 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../15-python3-rgw_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:52:38.424 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-rgw (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:52:38.655 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../02-librbd1_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:52:38.727 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package libleveldb1d:amd64. 2023-09-23T18:52:38.741 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../16-libleveldb1d_1.22-3ubuntu2_amd64.deb ... 2023-09-23T18:52:38.785 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking libleveldb1d:amd64 (1.22-3ubuntu2) ... 2023-09-23T18:52:38.809 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking librbd1 (16.2.14-37-g2e1f1015-1focal) over (15.2.17-0ubuntu0.20.04.4) ... 2023-09-23T18:52:39.121 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package liboath0. 2023-09-23T18:52:39.135 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../17-liboath0_2.6.1-1.3_amd64.deb ... 2023-09-23T18:52:39.180 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking liboath0 (2.6.1-1.3) ... 2023-09-23T18:52:39.491 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package libradosstriper1. 2023-09-23T18:52:39.506 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../18-libradosstriper1_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:52:39.510 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../03-librados2_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:52:39.549 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking libradosstriper1 (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:52:39.638 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking librados2 (16.2.14-37-g2e1f1015-1focal) over (15.2.17-0ubuntu0.20.04.4) ... 2023-09-23T18:52:39.995 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package ceph-common. 2023-09-23T18:52:40.008 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../19-ceph-common_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:52:40.053 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking ceph-common (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:52:40.301 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package libcephfs2. 2023-09-23T18:52:40.315 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../04-libcephfs2_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:52:40.359 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking libcephfs2 (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:52:40.763 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-rados. 2023-09-23T18:52:40.772 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../05-python3-rados_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:52:40.899 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-rados (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:52:41.256 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-ceph-argparse. 2023-09-23T18:52:41.271 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../06-python3-ceph-argparse_16.2.14-37-g2e1f1015-1focal_all.deb ... 2023-09-23T18:52:41.314 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-ceph-argparse (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:52:41.617 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-cephfs. 2023-09-23T18:52:41.630 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../07-python3-cephfs_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:52:41.667 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-cephfs (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:52:41.978 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-ceph-common. 2023-09-23T18:52:41.991 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../08-python3-ceph-common_16.2.14-37-g2e1f1015-1focal_all.deb ... 2023-09-23T18:52:42.064 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-ceph-common (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:52:42.319 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-prettytable. 2023-09-23T18:52:42.333 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../09-python3-prettytable_0.7.2-5_all.deb ... 2023-09-23T18:52:42.358 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package ceph-base. 2023-09-23T18:52:42.371 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../20-ceph-base_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:52:42.378 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-prettytable (0.7.2-5) ... 2023-09-23T18:52:42.426 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking ceph-base (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:52:42.627 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-rbd. 2023-09-23T18:52:42.642 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../10-python3-rbd_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:52:42.686 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-rbd (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:52:43.031 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package liblua5.3-0:amd64. 2023-09-23T18:52:43.044 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../11-liblua5.3-0_5.3.3-1.1ubuntu2_amd64.deb ... 2023-09-23T18:52:43.085 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking liblua5.3-0:amd64 (5.3.3-1.1ubuntu2) ... 2023-09-23T18:52:43.422 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package librabbitmq4:amd64. 2023-09-23T18:52:43.436 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../12-librabbitmq4_0.10.0-1_amd64.deb ... 2023-09-23T18:52:43.463 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-cryptography. 2023-09-23T18:52:43.472 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking librabbitmq4:amd64 (0.10.0-1) ... 2023-09-23T18:52:43.476 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../21-python3-cryptography_2.8-3ubuntu0.1_amd64.deb ... 2023-09-23T18:52:43.512 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-cryptography (2.8-3ubuntu0.1) ... 2023-09-23T18:52:43.817 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package librdkafka1:amd64. 2023-09-23T18:52:43.831 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../13-librdkafka1_1.2.1-1ubuntu1_amd64.deb ... 2023-09-23T18:52:43.875 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking librdkafka1:amd64 (1.2.1-1ubuntu1) ... 2023-09-23T18:52:43.900 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-openssl. 2023-09-23T18:52:43.913 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../22-python3-openssl_19.0.0-1build1_all.deb ... 2023-09-23T18:52:43.958 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-openssl (19.0.0-1build1) ... 2023-09-23T18:52:44.202 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2023-09-23T18:52:44.211 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package librgw2. 2023-09-23T18:52:44.215 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../23-ceph-mgr-modules-core_16.2.14-37-g2e1f1015-1focal_all.deb ... 2023-09-23T18:52:44.225 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../14-librgw2_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:52:44.252 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking ceph-mgr-modules-core (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:52:44.295 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking librgw2 (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:52:44.567 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-bcrypt. 2023-09-23T18:52:44.580 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../24-python3-bcrypt_3.1.7-2ubuntu1_amd64.deb ... 2023-09-23T18:52:44.624 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-bcrypt (3.1.7-2ubuntu1) ... 2023-09-23T18:52:44.914 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-rgw. 2023-09-23T18:52:44.927 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-cherrypy3. 2023-09-23T18:52:44.928 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../15-python3-rgw_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:52:44.943 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../25-python3-cherrypy3_8.9.1-6_all.deb ... 2023-09-23T18:52:44.981 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-rgw (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:52:44.985 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-cherrypy3 (8.9.1-6) ... 2023-09-23T18:52:45.306 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-jwt. 2023-09-23T18:52:45.308 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package libleveldb1d:amd64. 2023-09-23T18:52:45.319 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../26-python3-jwt_1.7.1-2ubuntu2.1_all.deb ... 2023-09-23T18:52:45.322 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../16-libleveldb1d_1.22-3ubuntu2_amd64.deb ... 2023-09-23T18:52:45.358 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking libleveldb1d:amd64 (1.22-3ubuntu2) ... 2023-09-23T18:52:45.381 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-jwt (1.7.1-2ubuntu2.1) ... 2023-09-23T18:52:45.692 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-logutils. 2023-09-23T18:52:45.707 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../27-python3-logutils_0.3.3-7_all.deb ... 2023-09-23T18:52:45.711 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package liboath0. 2023-09-23T18:52:45.725 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../17-liboath0_2.6.1-1.3_amd64.deb ... 2023-09-23T18:52:45.750 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-logutils (0.3.3-7) ... 2023-09-23T18:52:45.769 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking liboath0 (2.6.1-1.3) ... 2023-09-23T18:52:46.053 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-markupsafe. 2023-09-23T18:52:46.068 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../28-python3-markupsafe_1.1.0-1build2_amd64.deb ... 2023-09-23T18:52:46.073 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package libradosstriper1. 2023-09-23T18:52:46.087 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../18-libradosstriper1_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:52:46.111 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-markupsafe (1.1.0-1build2) ... 2023-09-23T18:52:46.123 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking libradosstriper1 (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:52:46.414 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-mako. 2023-09-23T18:52:46.428 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../29-python3-mako_1.1.0+ds1-1ubuntu2.1_all.deb ... 2023-09-23T18:52:46.473 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-mako (1.1.0+ds1-1ubuntu2.1) ... 2023-09-23T18:52:46.584 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package ceph-common. 2023-09-23T18:52:46.599 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../19-ceph-common_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:52:46.642 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking ceph-common (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:52:46.784 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-simplegeneric. 2023-09-23T18:52:46.798 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../30-python3-simplegeneric_0.8.1-3_all.deb ... 2023-09-23T18:52:46.842 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-simplegeneric (0.8.1-3) ... 2023-09-23T18:52:47.129 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-singledispatch. 2023-09-23T18:52:47.143 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../31-python3-singledispatch_3.4.0.3-2build1_all.deb ... 2023-09-23T18:52:47.187 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-singledispatch (3.4.0.3-2build1) ... 2023-09-23T18:52:47.482 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-webob. 2023-09-23T18:52:47.495 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../32-python3-webob_1%3a1.8.5-2_all.deb ... 2023-09-23T18:52:47.532 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-webob (1:1.8.5-2) ... 2023-09-23T18:52:47.852 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-soupsieve. 2023-09-23T18:52:47.867 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../33-python3-soupsieve_1.9.5+dfsg-1_all.deb ... 2023-09-23T18:52:47.910 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-soupsieve (1.9.5+dfsg-1) ... 2023-09-23T18:52:48.212 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-bs4. 2023-09-23T18:52:48.225 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../34-python3-bs4_4.8.2-1_all.deb ... 2023-09-23T18:52:48.271 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-bs4 (4.8.2-1) ... 2023-09-23T18:52:48.599 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-waitress. 2023-09-23T18:52:48.613 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../35-python3-waitress_1.4.1-1ubuntu0.1_all.deb ... 2023-09-23T18:52:48.652 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-waitress (1.4.1-1ubuntu0.1) ... 2023-09-23T18:52:48.907 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package ceph-base. 2023-09-23T18:52:48.920 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../20-ceph-base_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:52:48.944 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-tempita. 2023-09-23T18:52:48.957 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../36-python3-tempita_0.5.2-6_all.deb ... 2023-09-23T18:52:48.966 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking ceph-base (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:52:49.002 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-tempita (0.5.2-6) ... 2023-09-23T18:52:49.255 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-paste. 2023-09-23T18:52:49.268 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../37-python3-paste_3.4.0+dfsg1-1_all.deb ... 2023-09-23T18:52:49.313 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-paste (3.4.0+dfsg1-1) ... 2023-09-23T18:52:49.625 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python-pastedeploy-tpl. 2023-09-23T18:52:49.638 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../38-python-pastedeploy-tpl_2.1.0-1_all.deb ... 2023-09-23T18:52:49.674 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python-pastedeploy-tpl (2.1.0-1) ... 2023-09-23T18:52:49.944 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-cryptography. 2023-09-23T18:52:49.952 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-pastedeploy. 2023-09-23T18:52:49.959 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../21-python3-cryptography_2.8-3ubuntu0.1_amd64.deb ... 2023-09-23T18:52:49.967 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../39-python3-pastedeploy_2.1.0-1_all.deb ... 2023-09-23T18:52:50.002 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-cryptography (2.8-3ubuntu0.1) ... 2023-09-23T18:52:50.010 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-pastedeploy (2.1.0-1) ... 2023-09-23T18:52:50.297 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-webtest. 2023-09-23T18:52:50.311 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../40-python3-webtest_2.0.32-2_all.deb ... 2023-09-23T18:52:50.354 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-webtest (2.0.32-2) ... 2023-09-23T18:52:50.498 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-openssl. 2023-09-23T18:52:50.511 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../22-python3-openssl_19.0.0-1build1_all.deb ... 2023-09-23T18:52:50.557 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-openssl (19.0.0-1build1) ... 2023-09-23T18:52:50.666 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-pecan. 2023-09-23T18:52:50.680 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../41-python3-pecan_1.3.3-3ubuntu1_all.deb ... 2023-09-23T18:52:50.724 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-pecan (1.3.3-3ubuntu1) ... 2023-09-23T18:52:50.826 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2023-09-23T18:52:50.842 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../23-ceph-mgr-modules-core_16.2.14-37-g2e1f1015-1focal_all.deb ... 2023-09-23T18:52:50.885 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking ceph-mgr-modules-core (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:52:51.088 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-werkzeug. 2023-09-23T18:52:51.102 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../42-python3-werkzeug_0.16.1+dfsg1-2ubuntu0.1_all.deb ... 2023-09-23T18:52:51.145 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-werkzeug (0.16.1+dfsg1-2ubuntu0.1) ... 2023-09-23T18:52:51.257 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-bcrypt. 2023-09-23T18:52:51.271 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../24-python3-bcrypt_3.1.7-2ubuntu1_amd64.deb ... 2023-09-23T18:52:51.306 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-bcrypt (3.1.7-2ubuntu1) ... 2023-09-23T18:52:51.499 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package libsqlite3-mod-ceph. 2023-09-23T18:52:51.515 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../43-libsqlite3-mod-ceph_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:52:51.557 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking libsqlite3-mod-ceph (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:52:51.626 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-cherrypy3. 2023-09-23T18:52:51.639 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../25-python3-cherrypy3_8.9.1-6_all.deb ... 2023-09-23T18:52:51.684 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-cherrypy3 (8.9.1-6) ... 2023-09-23T18:52:51.894 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package ceph-mgr. 2023-09-23T18:52:51.907 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../44-ceph-mgr_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:52:51.951 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking ceph-mgr (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:52:52.004 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-jwt. 2023-09-23T18:52:52.019 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../26-python3-jwt_1.7.1-2ubuntu2.1_all.deb ... 2023-09-23T18:52:52.062 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-jwt (1.7.1-2ubuntu2.1) ... 2023-09-23T18:52:52.374 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-logutils. 2023-09-23T18:52:52.388 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../27-python3-logutils_0.3.3-7_all.deb ... 2023-09-23T18:52:52.413 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package ceph-mon. 2023-09-23T18:52:52.427 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../45-ceph-mon_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:52:52.433 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-logutils (0.3.3-7) ... 2023-09-23T18:52:52.471 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking ceph-mon (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:52:52.735 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-markupsafe. 2023-09-23T18:52:52.749 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../28-python3-markupsafe_1.1.0-1build2_amd64.deb ... 2023-09-23T18:52:52.793 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-markupsafe (1.1.0-1build2) ... 2023-09-23T18:52:53.079 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-mako. 2023-09-23T18:52:53.095 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../29-python3-mako_1.1.0+ds1-1ubuntu2.1_all.deb ... 2023-09-23T18:52:53.137 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-mako (1.1.0+ds1-1ubuntu2.1) ... 2023-09-23T18:52:53.441 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-simplegeneric. 2023-09-23T18:52:53.457 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../30-python3-simplegeneric_0.8.1-3_all.deb ... 2023-09-23T18:52:53.491 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-simplegeneric (0.8.1-3) ... 2023-09-23T18:52:53.534 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package ceph-osd. 2023-09-23T18:52:53.547 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../46-ceph-osd_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:52:53.583 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking ceph-osd (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:52:53.794 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-singledispatch. 2023-09-23T18:52:53.808 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../31-python3-singledispatch_3.4.0.3-2build1_all.deb ... 2023-09-23T18:52:53.852 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-singledispatch (3.4.0.3-2build1) ... 2023-09-23T18:52:54.155 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-webob. 2023-09-23T18:52:54.169 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../32-python3-webob_1%3a1.8.5-2_all.deb ... 2023-09-23T18:52:54.214 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-webob (1:1.8.5-2) ... 2023-09-23T18:52:54.550 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-soupsieve. 2023-09-23T18:52:54.563 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../33-python3-soupsieve_1.9.5+dfsg-1_all.deb ... 2023-09-23T18:52:54.599 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-soupsieve (1.9.5+dfsg-1) ... 2023-09-23T18:52:54.869 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-bs4. 2023-09-23T18:52:54.884 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../34-python3-bs4_4.8.2-1_all.deb ... 2023-09-23T18:52:54.927 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-bs4 (4.8.2-1) ... 2023-09-23T18:52:55.389 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-waitress. 2023-09-23T18:52:55.403 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../35-python3-waitress_1.4.1-1ubuntu0.1_all.deb ... 2023-09-23T18:52:55.459 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-waitress (1.4.1-1ubuntu0.1) ... 2023-09-23T18:52:55.767 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-tempita. 2023-09-23T18:52:55.782 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../36-python3-tempita_0.5.2-6_all.deb ... 2023-09-23T18:52:55.817 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-tempita (0.5.2-6) ... 2023-09-23T18:52:56.120 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-paste. 2023-09-23T18:52:56.133 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../37-python3-paste_3.4.0+dfsg1-1_all.deb ... 2023-09-23T18:52:56.178 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-paste (3.4.0+dfsg1-1) ... 2023-09-23T18:52:56.722 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package ceph. 2023-09-23T18:52:56.735 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../47-ceph_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:52:56.780 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking ceph (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:52:57.091 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package ceph-fuse. 2023-09-23T18:52:57.107 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../48-ceph-fuse_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:52:57.149 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking ceph-fuse (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:52:57.340 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python-pastedeploy-tpl. 2023-09-23T18:52:57.355 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../38-python-pastedeploy-tpl_2.1.0-1_all.deb ... 2023-09-23T18:52:57.398 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python-pastedeploy-tpl (2.1.0-1) ... 2023-09-23T18:52:57.545 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package ceph-mds. 2023-09-23T18:52:57.558 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../49-ceph-mds_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:52:57.602 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking ceph-mds (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:52:57.684 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-pastedeploy. 2023-09-23T18:52:57.699 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../39-python3-pastedeploy_2.1.0-1_all.deb ... 2023-09-23T18:52:57.742 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-pastedeploy (2.1.0-1) ... 2023-09-23T18:52:58.020 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-webtest. 2023-09-23T18:52:58.035 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../40-python3-webtest_2.0.32-2_all.deb ... 2023-09-23T18:52:58.078 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-webtest (2.0.32-2) ... 2023-09-23T18:52:58.114 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package cephadm. 2023-09-23T18:52:58.128 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../50-cephadm_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:52:58.172 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking cephadm (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:52:58.398 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-pecan. 2023-09-23T18:52:58.412 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../41-python3-pecan_1.3.3-3ubuntu1_all.deb ... 2023-09-23T18:52:58.456 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-pecan (1.3.3-3ubuntu1) ... 2023-09-23T18:52:58.484 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-jinja2. 2023-09-23T18:52:58.497 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../51-python3-jinja2_2.10.1-2_all.deb ... 2023-09-23T18:52:58.542 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-jinja2 (2.10.1-2) ... 2023-09-23T18:52:58.828 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-werkzeug. 2023-09-23T18:52:58.841 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../42-python3-werkzeug_0.16.1+dfsg1-2ubuntu0.1_all.deb ... 2023-09-23T18:52:58.861 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package ceph-mgr-cephadm. 2023-09-23T18:52:58.876 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../52-ceph-mgr-cephadm_16.2.14-37-g2e1f1015-1focal_all.deb ... 2023-09-23T18:52:58.886 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-werkzeug (0.16.1+dfsg1-2ubuntu0.1) ... 2023-09-23T18:52:58.920 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking ceph-mgr-cephadm (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:52:59.231 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package libsqlite3-mod-ceph. 2023-09-23T18:52:59.245 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../43-libsqlite3-mod-ceph_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:52:59.256 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-repoze.lru. 2023-09-23T18:52:59.271 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../53-python3-repoze.lru_0.7-2_all.deb ... 2023-09-23T18:52:59.289 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking libsqlite3-mod-ceph (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:52:59.314 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-repoze.lru (0.7-2) ... 2023-09-23T18:52:59.550 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package libjs-underscore. 2023-09-23T18:52:59.564 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../54-libjs-underscore_1.9.1~dfsg-1ubuntu0.20.04.1_all.deb ... 2023-09-23T18:52:59.609 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking libjs-underscore (1.9.1~dfsg-1ubuntu0.20.04.1) ... 2023-09-23T18:52:59.617 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package ceph-mgr. 2023-09-23T18:52:59.632 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../44-ceph-mgr_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:52:59.675 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking ceph-mgr (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:52:59.878 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package libjs-sphinxdoc. 2023-09-23T18:52:59.892 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../55-libjs-sphinxdoc_1.8.5-7ubuntu3_all.deb ... 2023-09-23T18:52:59.936 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking libjs-sphinxdoc (1.8.5-7ubuntu3) ... 2023-09-23T18:53:00.129 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package ceph-mon. 2023-09-23T18:53:00.142 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../45-ceph-mon_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:53:00.178 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking ceph-mon (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:00.238 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-routes. 2023-09-23T18:53:00.252 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../56-python3-routes_2.4.1-2_all.deb ... 2023-09-23T18:53:00.299 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-routes (2.4.1-2) ... 2023-09-23T18:53:00.618 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2023-09-23T18:53:00.632 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../57-ceph-mgr-dashboard_16.2.14-37-g2e1f1015-1focal_all.deb ... 2023-09-23T18:53:00.676 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking ceph-mgr-dashboard (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:01.274 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package ceph-osd. 2023-09-23T18:53:01.288 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../46-ceph-osd_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:53:01.332 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking ceph-osd (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:01.625 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-decorator. 2023-09-23T18:53:01.639 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../58-python3-decorator_4.4.2-0ubuntu1_all.deb ... 2023-09-23T18:53:01.675 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-decorator (4.4.2-0ubuntu1) ... 2023-09-23T18:53:01.969 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package liblbfgsb0:amd64. 2023-09-23T18:53:01.984 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../59-liblbfgsb0_3.0+dfsg.3-7build1_amd64.deb ... 2023-09-23T18:53:02.027 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking liblbfgsb0:amd64 (3.0+dfsg.3-7build1) ... 2023-09-23T18:53:02.338 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-scipy. 2023-09-23T18:53:02.351 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../60-python3-scipy_1.3.3-3build1_amd64.deb ... 2023-09-23T18:53:02.396 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-scipy (1.3.3-3build1) ... 2023-09-23T18:53:03.696 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-sklearn-lib. 2023-09-23T18:53:03.710 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../61-python3-sklearn-lib_0.22.2.post1+dfsg-5_amd64.deb ... 2023-09-23T18:53:03.754 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-sklearn-lib (0.22.2.post1+dfsg-5) ... 2023-09-23T18:53:03.954 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package ceph. 2023-09-23T18:53:03.968 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../47-ceph_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:53:04.012 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking ceph (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:04.283 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-joblib. 2023-09-23T18:53:04.299 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../62-python3-joblib_0.14.0-3_all.deb ... 2023-09-23T18:53:04.331 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package ceph-fuse. 2023-09-23T18:53:04.341 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-joblib (0.14.0-3) ... 2023-09-23T18:53:04.345 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../48-ceph-fuse_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:53:04.390 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking ceph-fuse (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:04.711 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-sklearn. 2023-09-23T18:53:04.727 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../63-python3-sklearn_0.22.2.post1+dfsg-5_all.deb ... 2023-09-23T18:53:04.769 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-sklearn (0.22.2.post1+dfsg-5) ... 2023-09-23T18:53:04.785 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package ceph-mds. 2023-09-23T18:53:04.798 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../49-ceph-mds_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:53:04.843 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking ceph-mds (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:05.379 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package cephadm. 2023-09-23T18:53:05.393 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../50-cephadm_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:53:05.437 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking cephadm (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:05.469 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2023-09-23T18:53:05.484 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../64-ceph-mgr-diskprediction-local_16.2.14-37-g2e1f1015-1focal_all.deb ... 2023-09-23T18:53:05.527 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking ceph-mgr-diskprediction-local (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:05.774 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-jinja2. 2023-09-23T18:53:05.787 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../51-python3-jinja2_2.10.1-2_all.deb ... 2023-09-23T18:53:05.840 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-jinja2 (2.10.1-2) ... 2023-09-23T18:53:06.218 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package ceph-mgr-cephadm. 2023-09-23T18:53:06.232 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../52-ceph-mgr-cephadm_16.2.14-37-g2e1f1015-1focal_all.deb ... 2023-09-23T18:53:06.276 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking ceph-mgr-cephadm (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:06.630 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-repoze.lru. 2023-09-23T18:53:06.644 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../53-python3-repoze.lru_0.7-2_all.deb ... 2023-09-23T18:53:06.663 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-cachetools. 2023-09-23T18:53:06.679 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../65-python3-cachetools_4.0.0-1_all.deb ... 2023-09-23T18:53:06.687 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-repoze.lru (0.7-2) ... 2023-09-23T18:53:06.721 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-cachetools (4.0.0-1) ... 2023-09-23T18:53:06.915 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package libjs-underscore. 2023-09-23T18:53:06.931 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../54-libjs-underscore_1.9.1~dfsg-1ubuntu0.20.04.1_all.deb ... 2023-09-23T18:53:06.974 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking libjs-underscore (1.9.1~dfsg-1ubuntu0.20.04.1) ... 2023-09-23T18:53:07.023 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-pyasn1. 2023-09-23T18:53:07.037 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../66-python3-pyasn1_0.4.2-3build1_all.deb ... 2023-09-23T18:53:07.082 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-pyasn1 (0.4.2-3build1) ... 2023-09-23T18:53:07.234 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package libjs-sphinxdoc. 2023-09-23T18:53:07.248 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../55-libjs-sphinxdoc_1.8.5-7ubuntu3_all.deb ... 2023-09-23T18:53:07.301 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking libjs-sphinxdoc (1.8.5-7ubuntu3) ... 2023-09-23T18:53:07.443 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-pyasn1-modules. 2023-09-23T18:53:07.459 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../67-python3-pyasn1-modules_0.2.1-0.2build1_all.deb ... 2023-09-23T18:53:07.501 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-pyasn1-modules (0.2.1-0.2build1) ... 2023-09-23T18:53:07.612 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-routes. 2023-09-23T18:53:07.627 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../56-python3-routes_2.4.1-2_all.deb ... 2023-09-23T18:53:07.670 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-routes (2.4.1-2) ... 2023-09-23T18:53:07.812 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-rsa. 2023-09-23T18:53:07.828 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../68-python3-rsa_4.0-3_all.deb ... 2023-09-23T18:53:07.870 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-rsa (4.0-3) ... 2023-09-23T18:53:07.998 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2023-09-23T18:53:08.012 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../57-ceph-mgr-dashboard_16.2.14-37-g2e1f1015-1focal_all.deb ... 2023-09-23T18:53:08.056 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking ceph-mgr-dashboard (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:08.182 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-google-auth. 2023-09-23T18:53:08.195 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../69-python3-google-auth_1.5.1-2_all.deb ... 2023-09-23T18:53:08.240 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-google-auth (1.5.1-2) ... 2023-09-23T18:53:08.568 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-blinker. 2023-09-23T18:53:08.583 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../70-python3-blinker_1.4+dfsg1-0.3ubuntu1_all.deb ... 2023-09-23T18:53:08.626 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-blinker (1.4+dfsg1-0.3ubuntu1) ... 2023-09-23T18:53:08.912 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-oauthlib. 2023-09-23T18:53:08.927 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../71-python3-oauthlib_3.1.0-1ubuntu2_all.deb ... 2023-09-23T18:53:08.970 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-oauthlib (3.1.0-1ubuntu2) ... 2023-09-23T18:53:08.989 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-decorator. 2023-09-23T18:53:09.004 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../58-python3-decorator_4.4.2-0ubuntu1_all.deb ... 2023-09-23T18:53:09.056 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-decorator (4.4.2-0ubuntu1) ... 2023-09-23T18:53:09.307 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-requests-oauthlib. 2023-09-23T18:53:09.320 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../72-python3-requests-oauthlib_1.0.0-1.1build1_all.deb ... 2023-09-23T18:53:09.358 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package liblbfgsb0:amd64. 2023-09-23T18:53:09.364 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-requests-oauthlib (1.0.0-1.1build1) ... 2023-09-23T18:53:09.373 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../59-liblbfgsb0_3.0+dfsg.3-7build1_amd64.deb ... 2023-09-23T18:53:09.416 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking liblbfgsb0:amd64 (3.0+dfsg.3-7build1) ... 2023-09-23T18:53:09.676 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-websocket. 2023-09-23T18:53:09.689 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../73-python3-websocket_0.53.0-2ubuntu1_all.deb ... 2023-09-23T18:53:09.734 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-websocket (0.53.0-2ubuntu1) ... 2023-09-23T18:53:09.761 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-scipy. 2023-09-23T18:53:09.776 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../60-python3-scipy_1.3.3-3build1_amd64.deb ... 2023-09-23T18:53:09.819 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-scipy (1.3.3-3build1) ... 2023-09-23T18:53:10.045 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-kubernetes. 2023-09-23T18:53:10.060 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../74-python3-kubernetes_7.0.0~a1-2_all.deb ... 2023-09-23T18:53:10.103 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-kubernetes (7.0.0~a1-2) ... 2023-09-23T18:53:10.650 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2023-09-23T18:53:10.663 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../75-ceph-mgr-k8sevents_16.2.14-37-g2e1f1015-1focal_all.deb ... 2023-09-23T18:53:10.708 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking ceph-mgr-k8sevents (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:11.027 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package libonig5:amd64. 2023-09-23T18:53:11.043 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../76-libonig5_6.9.4-1_amd64.deb ... 2023-09-23T18:53:11.085 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking libonig5:amd64 (6.9.4-1) ... 2023-09-23T18:53:11.147 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-sklearn-lib. 2023-09-23T18:53:11.163 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../61-python3-sklearn-lib_0.22.2.post1+dfsg-5_amd64.deb ... 2023-09-23T18:53:11.205 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-sklearn-lib (0.22.2.post1+dfsg-5) ... 2023-09-23T18:53:11.463 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package libjq1:amd64. 2023-09-23T18:53:11.477 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../77-libjq1_1.6-1ubuntu0.20.04.1_amd64.deb ... 2023-09-23T18:53:11.521 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking libjq1:amd64 (1.6-1ubuntu0.20.04.1) ... 2023-09-23T18:53:11.751 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-joblib. 2023-09-23T18:53:11.764 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../62-python3-joblib_0.14.0-3_all.deb ... 2023-09-23T18:53:11.809 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-joblib (0.14.0-3) ... 2023-09-23T18:53:12.188 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-sklearn. 2023-09-23T18:53:12.198 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package jq. 2023-09-23T18:53:12.201 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../63-python3-sklearn_0.22.2.post1+dfsg-5_all.deb ... 2023-09-23T18:53:12.212 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../78-jq_1.6-1ubuntu0.20.04.1_amd64.deb ... 2023-09-23T18:53:12.245 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-sklearn (0.22.2.post1+dfsg-5) ... 2023-09-23T18:53:12.256 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking jq (1.6-1ubuntu0.20.04.1) ... 2023-09-23T18:53:12.509 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package socat. 2023-09-23T18:53:12.524 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../79-socat_1.7.3.3-2_amd64.deb ... 2023-09-23T18:53:12.567 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking socat (1.7.3.3-2) ... 2023-09-23T18:53:12.845 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package xmlstarlet. 2023-09-23T18:53:12.859 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../80-xmlstarlet_1.6.1-2build1_amd64.deb ... 2023-09-23T18:53:12.894 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking xmlstarlet (1.6.1-2build1) ... 2023-09-23T18:53:13.293 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package ceph-test. 2023-09-23T18:53:13.306 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../81-ceph-test_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:53:13.351 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking ceph-test (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:13.388 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2023-09-23T18:53:13.403 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../64-ceph-mgr-diskprediction-local_16.2.14-37-g2e1f1015-1focal_all.deb ... 2023-09-23T18:53:13.446 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking ceph-mgr-diskprediction-local (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:14.549 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-cachetools. 2023-09-23T18:53:14.564 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../65-python3-cachetools_4.0.0-1_all.deb ... 2023-09-23T18:53:14.607 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-cachetools (4.0.0-1) ... 2023-09-23T18:53:14.910 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-pyasn1. 2023-09-23T18:53:14.924 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../66-python3-pyasn1_0.4.2-3build1_all.deb ... 2023-09-23T18:53:14.968 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-pyasn1 (0.4.2-3build1) ... 2023-09-23T18:53:15.297 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-pyasn1-modules. 2023-09-23T18:53:15.312 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../67-python3-pyasn1-modules_0.2.1-0.2build1_all.deb ... 2023-09-23T18:53:15.355 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-pyasn1-modules (0.2.1-0.2build1) ... 2023-09-23T18:53:15.658 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-rsa. 2023-09-23T18:53:15.672 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../68-python3-rsa_4.0-3_all.deb ... 2023-09-23T18:53:15.715 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-rsa (4.0-3) ... 2023-09-23T18:53:16.027 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-google-auth. 2023-09-23T18:53:16.043 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../69-python3-google-auth_1.5.1-2_all.deb ... 2023-09-23T18:53:16.085 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-google-auth (1.5.1-2) ... 2023-09-23T18:53:16.389 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-blinker. 2023-09-23T18:53:16.403 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../70-python3-blinker_1.4+dfsg1-0.3ubuntu1_all.deb ... 2023-09-23T18:53:16.438 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-blinker (1.4+dfsg1-0.3ubuntu1) ... 2023-09-23T18:53:16.741 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-oauthlib. 2023-09-23T18:53:16.750 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../71-python3-oauthlib_3.1.0-1ubuntu2_all.deb ... 2023-09-23T18:53:16.816 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-oauthlib (3.1.0-1ubuntu2) ... 2023-09-23T18:53:17.087 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-requests-oauthlib. 2023-09-23T18:53:17.101 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../72-python3-requests-oauthlib_1.0.0-1.1build1_all.deb ... 2023-09-23T18:53:17.145 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-requests-oauthlib (1.0.0-1.1build1) ... 2023-09-23T18:53:17.456 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-websocket. 2023-09-23T18:53:17.470 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../73-python3-websocket_0.53.0-2ubuntu1_all.deb ... 2023-09-23T18:53:17.514 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-websocket (0.53.0-2ubuntu1) ... 2023-09-23T18:53:17.579 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package libcephfs-dev. 2023-09-23T18:53:17.595 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../82-libcephfs-dev_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:53:17.637 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking libcephfs-dev (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:17.842 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-kubernetes. 2023-09-23T18:53:17.857 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../74-python3-kubernetes_7.0.0~a1-2_all.deb ... 2023-09-23T18:53:17.900 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-kubernetes (7.0.0~a1-2) ... 2023-09-23T18:53:18.049 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package nvme-cli. 2023-09-23T18:53:18.063 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../83-nvme-cli_1.9-1ubuntu0.1_amd64.deb ... 2023-09-23T18:53:18.122 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking nvme-cli (1.9-1ubuntu0.1) ... 2023-09-23T18:53:18.447 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2023-09-23T18:53:18.463 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../75-ceph-mgr-k8sevents_16.2.14-37-g2e1f1015-1focal_all.deb ... 2023-09-23T18:53:18.493 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-atomicwrites. 2023-09-23T18:53:18.502 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../84-python3-atomicwrites_1.1.5-2build1_all.deb ... 2023-09-23T18:53:18.505 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking ceph-mgr-k8sevents (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:18.543 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-atomicwrites (1.1.5-2build1) ... 2023-09-23T18:53:18.808 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package libonig5:amd64. 2023-09-23T18:53:18.822 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../76-libonig5_6.9.4-1_amd64.deb ... 2023-09-23T18:53:18.829 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-attr. 2023-09-23T18:53:18.838 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../85-python3-attr_19.3.0-2_all.deb ... 2023-09-23T18:53:18.866 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking libonig5:amd64 (6.9.4-1) ... 2023-09-23T18:53:18.879 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-attr (19.3.0-2) ... 2023-09-23T18:53:19.215 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-webencodings. 2023-09-23T18:53:19.224 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../86-python3-webencodings_0.5.1-1ubuntu1_all.deb ... 2023-09-23T18:53:19.227 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package libjq1:amd64. 2023-09-23T18:53:19.243 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../77-libjq1_1.6-1ubuntu0.20.04.1_amd64.deb ... 2023-09-23T18:53:19.264 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-webencodings (0.5.1-1ubuntu1) ... 2023-09-23T18:53:19.286 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking libjq1:amd64 (1.6-1ubuntu0.20.04.1) ... 2023-09-23T18:53:19.547 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package jq. 2023-09-23T18:53:19.560 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../78-jq_1.6-1ubuntu0.20.04.1_amd64.deb ... 2023-09-23T18:53:19.584 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-html5lib. 2023-09-23T18:53:19.598 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../87-python3-html5lib_1.0.1-2_all.deb ... 2023-09-23T18:53:19.604 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking jq (1.6-1ubuntu0.20.04.1) ... 2023-09-23T18:53:19.642 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-html5lib (1.0.1-2) ... 2023-09-23T18:53:19.841 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package socat. 2023-09-23T18:53:19.855 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../79-socat_1.7.3.3-2_amd64.deb ... 2023-09-23T18:53:19.890 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking socat (1.7.3.3-2) ... 2023-09-23T18:53:19.971 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-lxml:amd64. 2023-09-23T18:53:19.985 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../88-python3-lxml_4.5.0-1ubuntu0.5_amd64.deb ... 2023-09-23T18:53:20.028 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-lxml:amd64 (4.5.0-1ubuntu0.5) ... 2023-09-23T18:53:20.219 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package xmlstarlet. 2023-09-23T18:53:20.235 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../80-xmlstarlet_1.6.1-2build1_amd64.deb ... 2023-09-23T18:53:20.277 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking xmlstarlet (1.6.1-2build1) ... 2023-09-23T18:53:20.498 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-packaging. 2023-09-23T18:53:20.512 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../89-python3-packaging_20.3-1_all.deb ... 2023-09-23T18:53:20.556 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-packaging (20.3-1) ... 2023-09-23T18:53:20.725 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package ceph-test. 2023-09-23T18:53:20.740 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../81-ceph-test_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:53:20.783 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking ceph-test (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:20.851 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-pastescript. 2023-09-23T18:53:20.865 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../90-python3-pastescript_2.0.2-4_all.deb ... 2023-09-23T18:53:20.909 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-pastescript (2.0.2-4) ... 2023-09-23T18:53:21.250 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-pluggy. 2023-09-23T18:53:21.263 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../91-python3-pluggy_0.13.0-2_all.deb ... 2023-09-23T18:53:21.308 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-pluggy (0.13.0-2) ... 2023-09-23T18:53:21.619 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-psutil. 2023-09-23T18:53:21.635 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../92-python3-psutil_5.5.1-1ubuntu4_amd64.deb ... 2023-09-23T18:53:21.677 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-psutil (5.5.1-1ubuntu4) ... 2023-09-23T18:53:21.988 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-py. 2023-09-23T18:53:21.997 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../93-python3-py_1.8.1-1ubuntu0.1_all.deb ... 2023-09-23T18:53:22.038 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-py (1.8.1-1ubuntu0.1) ... 2023-09-23T18:53:22.373 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-pyinotify. 2023-09-23T18:53:22.386 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../94-python3-pyinotify_0.9.6-1.2ubuntu1_all.deb ... 2023-09-23T18:53:22.431 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-pyinotify (0.9.6-1.2ubuntu1) ... 2023-09-23T18:53:22.750 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-wcwidth. 2023-09-23T18:53:22.764 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../95-python3-wcwidth_0.1.8+dfsg1-3_all.deb ... 2023-09-23T18:53:22.809 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-wcwidth (0.1.8+dfsg1-3) ... 2023-09-23T18:53:23.111 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-pytest. 2023-09-23T18:53:23.127 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../96-python3-pytest_4.6.9-1_all.deb ... 2023-09-23T18:53:23.169 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-pytest (4.6.9-1) ... 2023-09-23T18:53:23.527 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package python3-simplejson. 2023-09-23T18:53:23.536 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../97-python3-simplejson_3.16.0-2ubuntu2_amd64.deb ... 2023-09-23T18:53:23.581 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking python3-simplejson (3.16.0-2ubuntu2) ... 2023-09-23T18:53:24.018 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package radosgw. 2023-09-23T18:53:24.032 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../98-radosgw_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:53:24.076 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking radosgw (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:25.053 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package libcephfs-dev. 2023-09-23T18:53:25.068 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../82-libcephfs-dev_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:53:25.112 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking libcephfs-dev (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:25.246 INFO:teuthology.orchestra.run.smithi092.stdout:Selecting previously unselected package rbd-fuse. 2023-09-23T18:53:25.260 INFO:teuthology.orchestra.run.smithi092.stdout:Preparing to unpack .../99-rbd-fuse_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:53:25.303 INFO:teuthology.orchestra.run.smithi092.stdout:Unpacking rbd-fuse (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:25.541 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package nvme-cli. 2023-09-23T18:53:25.556 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../83-nvme-cli_1.9-1ubuntu0.1_amd64.deb ... 2023-09-23T18:53:25.613 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-attr (19.3.0-2) ... 2023-09-23T18:53:25.614 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking nvme-cli (1.9-1ubuntu0.1) ... 2023-09-23T18:53:25.908 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up libleveldb1d:amd64 (1.22-3ubuntu2) ... 2023-09-23T18:53:25.947 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-atomicwrites. 2023-09-23T18:53:25.963 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../84-python3-atomicwrites_1.1.5-2build1_all.deb ... 2023-09-23T18:53:25.997 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-atomicwrites (1.1.5-2build1) ... 2023-09-23T18:53:26.034 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up librabbitmq4:amd64 (0.10.0-1) ... 2023-09-23T18:53:26.160 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up nvme-cli (1.9-1ubuntu0.1) ... 2023-09-23T18:53:26.300 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-attr. 2023-09-23T18:53:26.314 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../85-python3-attr_19.3.0-2_all.deb ... 2023-09-23T18:53:26.353 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up cephadm (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:26.358 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-attr (19.3.0-2) ... 2023-09-23T18:53:26.686 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-webencodings. 2023-09-23T18:53:26.701 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../86-python3-webencodings_0.5.1-1ubuntu1_all.deb ... 2023-09-23T18:53:26.744 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-webencodings (0.5.1-1ubuntu1) ... 2023-09-23T18:53:26.948 INFO:teuthology.orchestra.run.smithi092.stdout:Adding system user cephadm....done 2023-09-23T18:53:27.047 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-html5lib. 2023-09-23T18:53:27.064 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../87-python3-html5lib_1.0.1-2_all.deb ... 2023-09-23T18:53:27.100 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-waitress (1.4.1-1ubuntu0.1) ... 2023-09-23T18:53:27.106 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-html5lib (1.0.1-2) ... 2023-09-23T18:53:27.452 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-repoze.lru (0.7-2) ... 2023-09-23T18:53:27.467 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-lxml:amd64. 2023-09-23T18:53:27.484 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../88-python3-lxml_4.5.0-1ubuntu0.5_amd64.deb ... 2023-09-23T18:53:27.550 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-lxml:amd64 (4.5.0-1ubuntu0.5) ... 2023-09-23T18:53:27.737 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up liboath0 (2.6.1-1.3) ... 2023-09-23T18:53:27.862 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-py (1.8.1-1ubuntu0.1) ... 2023-09-23T18:53:28.033 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-packaging. 2023-09-23T18:53:28.047 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../89-python3-packaging_20.3-1_all.deb ... 2023-09-23T18:53:28.091 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-packaging (20.3-1) ... 2023-09-23T18:53:28.172 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-joblib (0.14.0-3) ... 2023-09-23T18:53:28.394 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-pastescript. 2023-09-23T18:53:28.409 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../90-python3-pastescript_2.0.2-4_all.deb ... 2023-09-23T18:53:28.452 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-pastescript (2.0.2-4) ... 2023-09-23T18:53:28.565 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-jwt (1.7.1-2ubuntu2.1) ... 2023-09-23T18:53:28.772 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-pluggy. 2023-09-23T18:53:28.786 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../91-python3-pluggy_0.13.0-2_all.deb ... 2023-09-23T18:53:28.830 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-pluggy (0.13.0-2) ... 2023-09-23T18:53:28.858 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-cachetools (4.0.0-1) ... 2023-09-23T18:53:29.125 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-psutil. 2023-09-23T18:53:29.139 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../92-python3-psutil_5.5.1-1ubuntu4_amd64.deb ... 2023-09-23T18:53:29.142 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-bcrypt (3.1.7-2ubuntu1) ... 2023-09-23T18:53:29.182 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-psutil (5.5.1-1ubuntu4) ... 2023-09-23T18:53:29.427 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-pyinotify (0.9.6-1.2ubuntu1) ... 2023-09-23T18:53:29.536 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-py. 2023-09-23T18:53:29.550 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../93-python3-py_1.8.1-1ubuntu0.1_all.deb ... 2023-09-23T18:53:29.594 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-py (1.8.1-1ubuntu0.1) ... 2023-09-23T18:53:29.711 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-ceph-argparse (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:29.897 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-pyinotify. 2023-09-23T18:53:29.911 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../94-python3-pyinotify_0.9.6-1.2ubuntu1_all.deb ... 2023-09-23T18:53:29.955 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-pyinotify (0.9.6-1.2ubuntu1) ... 2023-09-23T18:53:30.004 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-sklearn-lib (0.22.2.post1+dfsg-5) ... 2023-09-23T18:53:30.131 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-markupsafe (1.1.0-1build2) ... 2023-09-23T18:53:30.267 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-wcwidth. 2023-09-23T18:53:30.282 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../95-python3-wcwidth_0.1.8+dfsg1-3_all.deb ... 2023-09-23T18:53:30.325 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-wcwidth (0.1.8+dfsg1-3) ... 2023-09-23T18:53:30.416 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-webencodings (0.5.1-1ubuntu1) ... 2023-09-23T18:53:30.629 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-pytest. 2023-09-23T18:53:30.643 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../96-python3-pytest_4.6.9-1_all.deb ... 2023-09-23T18:53:30.686 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-pytest (4.6.9-1) ... 2023-09-23T18:53:30.700 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-psutil (5.5.1-1ubuntu4) ... 2023-09-23T18:53:31.007 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package python3-simplejson. 2023-09-23T18:53:31.021 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../97-python3-simplejson_3.16.0-2ubuntu2_amd64.deb ... 2023-09-23T18:53:31.065 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking python3-simplejson (3.16.0-2ubuntu2) ... 2023-09-23T18:53:31.076 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-atomicwrites (1.1.5-2build1) ... 2023-09-23T18:53:31.344 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-simplejson (3.16.0-2ubuntu2) ... 2023-09-23T18:53:31.510 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package radosgw. 2023-09-23T18:53:31.524 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../98-radosgw_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:53:31.568 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking radosgw (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:31.678 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-decorator (4.4.2-0ubuntu1) ... 2023-09-23T18:53:31.979 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up libjaeger (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:32.105 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-jinja2 (2.10.1-2) ... 2023-09-23T18:53:32.431 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-tempita (0.5.2-6) ... 2023-09-23T18:53:32.740 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-packaging (20.3-1) ... 2023-09-23T18:53:32.763 INFO:teuthology.orchestra.run.smithi016.stdout:Selecting previously unselected package rbd-fuse. 2023-09-23T18:53:32.779 INFO:teuthology.orchestra.run.smithi016.stdout:Preparing to unpack .../99-rbd-fuse_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:53:32.821 INFO:teuthology.orchestra.run.smithi016.stdout:Unpacking rbd-fuse (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:33.033 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python-pastedeploy-tpl (2.1.0-1) ... 2023-09-23T18:53:33.109 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-attr (19.3.0-2) ... 2023-09-23T18:53:33.159 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-wcwidth (0.1.8+dfsg1-3) ... 2023-09-23T18:53:33.409 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up libleveldb1d:amd64 (1.22-3ubuntu2) ... 2023-09-23T18:53:33.443 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-paste (3.4.0+dfsg1-1) ... 2023-09-23T18:53:33.536 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up librabbitmq4:amd64 (0.10.0-1) ... 2023-09-23T18:53:33.662 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up nvme-cli (1.9-1ubuntu0.1) ... 2023-09-23T18:53:33.828 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-werkzeug (0.16.1+dfsg1-2ubuntu0.1) ... 2023-09-23T18:53:33.846 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up cephadm (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:34.179 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-cryptography (2.8-3ubuntu0.1) ... 2023-09-23T18:53:34.449 INFO:teuthology.orchestra.run.smithi016.stdout:Adding system user cephadm....done 2023-09-23T18:53:34.531 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up socat (1.7.3.3-2) ... 2023-09-23T18:53:34.603 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-waitress (1.4.1-1ubuntu0.1) ... 2023-09-23T18:53:34.665 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-ceph-common (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:34.971 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-repoze.lru (0.7-2) ... 2023-09-23T18:53:34.975 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-html5lib (1.0.1-2) ... 2023-09-23T18:53:35.272 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up liboath0 (2.6.1-1.3) ... 2023-09-23T18:53:35.318 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up librdkafka1:amd64 (1.2.1-1ubuntu1) ... 2023-09-23T18:53:35.398 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-py (1.8.1-1ubuntu0.1) ... 2023-09-23T18:53:35.444 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up xmlstarlet (1.6.1-2build1) ... 2023-09-23T18:53:35.570 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-pluggy (0.13.0-2) ... 2023-09-23T18:53:35.725 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-joblib (0.14.0-3) ... 2023-09-23T18:53:35.854 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-lxml:amd64 (4.5.0-1ubuntu0.5) ... 2023-09-23T18:53:36.155 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up liblua5.3-0:amd64 (5.3.3-1.1ubuntu2) ... 2023-09-23T18:53:36.170 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-jwt (1.7.1-2ubuntu2.1) ... 2023-09-23T18:53:36.283 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-pyasn1 (0.4.2-3build1) ... 2023-09-23T18:53:36.455 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-cachetools (4.0.0-1) ... 2023-09-23T18:53:36.610 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-singledispatch (3.4.0.3-2build1) ... 2023-09-23T18:53:36.764 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-bcrypt (3.1.7-2ubuntu1) ... 2023-09-23T18:53:36.894 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-logutils (0.3.3-7) ... 2023-09-23T18:53:37.057 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-pyinotify (0.9.6-1.2ubuntu1) ... 2023-09-23T18:53:37.187 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up liblttng-ust0:amd64 (2.11.0-1) ... 2023-09-23T18:53:37.313 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-simplegeneric (0.8.1-3) ... 2023-09-23T18:53:37.358 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-ceph-argparse (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:37.605 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-prettytable (0.7.2-5) ... 2023-09-23T18:53:37.668 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-sklearn-lib (0.22.2.post1+dfsg-5) ... 2023-09-23T18:53:37.802 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-markupsafe (1.1.0-1build2) ... 2023-09-23T18:53:37.898 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-soupsieve (1.9.5+dfsg-1) ... 2023-09-23T18:53:38.087 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-webencodings (0.5.1-1ubuntu1) ... 2023-09-23T18:53:38.199 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-websocket (0.53.0-2ubuntu1) ... 2023-09-23T18:53:38.371 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-psutil (5.5.1-1ubuntu4) ... 2023-09-23T18:53:38.668 INFO:teuthology.orchestra.run.smithi092.stdout:update-alternatives: using /usr/bin/python3-wsdump to provide /usr/bin/wsdump (wsdump) in auto mode 2023-09-23T18:53:38.747 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-atomicwrites (1.1.5-2build1) ... 2023-09-23T18:53:38.910 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up libonig5:amd64 (6.9.4-1) ... 2023-09-23T18:53:39.032 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-simplejson (3.16.0-2ubuntu2) ... 2023-09-23T18:53:39.036 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-mako (1.1.0+ds1-1ubuntu2.1) ... 2023-09-23T18:53:39.345 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-cherrypy3 (8.9.1-6) ... 2023-09-23T18:53:39.392 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-decorator (4.4.2-0ubuntu1) ... 2023-09-23T18:53:39.684 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up libjaeger (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:39.689 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up libjs-underscore (1.9.1~dfsg-1ubuntu0.20.04.1) ... 2023-09-23T18:53:39.815 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up liblbfgsb0:amd64 (3.0+dfsg.3-7build1) ... 2023-09-23T18:53:39.819 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-jinja2 (2.10.1-2) ... 2023-09-23T18:53:39.966 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-webob (1:1.8.5-2) ... 2023-09-23T18:53:40.170 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-tempita (0.5.2-6) ... 2023-09-23T18:53:40.300 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-blinker (1.4+dfsg1-0.3ubuntu1) ... 2023-09-23T18:53:40.463 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-packaging (20.3-1) ... 2023-09-23T18:53:40.576 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up librados2 (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:40.702 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-scipy (1.3.3-3build1) ... 2023-09-23T18:53:40.747 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python-pastedeploy-tpl (2.1.0-1) ... 2023-09-23T18:53:40.865 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-wcwidth (0.1.8+dfsg1-3) ... 2023-09-23T18:53:41.149 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-paste (3.4.0+dfsg1-1) ... 2023-09-23T18:53:41.559 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-werkzeug (0.16.1+dfsg1-2ubuntu0.1) ... 2023-09-23T18:53:41.936 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-cryptography (2.8-3ubuntu0.1) ... 2023-09-23T18:53:42.192 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up librgw2 (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:42.309 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up libsqlite3-mod-ceph (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:42.313 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up socat (1.7.3.3-2) ... 2023-09-23T18:53:42.431 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-ceph-common (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:42.435 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up libjq1:amd64 (1.6-1ubuntu0.20.04.1) ... 2023-09-23T18:53:42.561 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-pytest (4.6.9-1) ... 2023-09-23T18:53:42.732 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-html5lib (1.0.1-2) ... 2023-09-23T18:53:43.067 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up librdkafka1:amd64 (1.2.1-1ubuntu1) ... 2023-09-23T18:53:43.193 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up xmlstarlet (1.6.1-2build1) ... 2023-09-23T18:53:43.212 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-openssl (19.0.0-1build1) ... 2023-09-23T18:53:43.327 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-pluggy (0.13.0-2) ... 2023-09-23T18:53:43.522 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-pastedeploy (2.1.0-1) ... 2023-09-23T18:53:43.823 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-bs4 (4.8.2-1) ... 2023-09-23T18:53:43.879 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-lxml:amd64 (4.5.0-1ubuntu0.5) ... 2023-09-23T18:53:44.161 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up libcephfs2 (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:44.188 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up liblua5.3-0:amd64 (5.3.3-1.1ubuntu2) ... 2023-09-23T18:53:44.287 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-oauthlib (3.1.0-1ubuntu2) ... 2023-09-23T18:53:44.314 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-pyasn1 (0.4.2-3build1) ... 2023-09-23T18:53:44.605 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up libradosstriper1 (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:44.641 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-singledispatch (3.4.0.3-2build1) ... 2023-09-23T18:53:44.731 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-pyasn1-modules (0.2.1-0.2build1) ... 2023-09-23T18:53:44.925 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-logutils (0.3.3-7) ... 2023-09-23T18:53:45.057 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-sklearn (0.22.2.post1+dfsg-5) ... 2023-09-23T18:53:45.226 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up liblttng-ust0:amd64 (2.11.0-1) ... 2023-09-23T18:53:45.361 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-simplegeneric (0.8.1-3) ... 2023-09-23T18:53:45.670 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-prettytable (0.7.2-5) ... 2023-09-23T18:53:45.963 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-soupsieve (1.9.5+dfsg-1) ... 2023-09-23T18:53:46.071 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up librbd1 (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:46.196 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up libjs-sphinxdoc (1.8.5-7ubuntu3) ... 2023-09-23T18:53:46.274 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-websocket (0.53.0-2ubuntu1) ... 2023-09-23T18:53:46.322 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up jq (1.6-1ubuntu0.20.04.1) ... 2023-09-23T18:53:46.448 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-rsa (4.0-3) ... 2023-09-23T18:53:46.617 INFO:teuthology.orchestra.run.smithi016.stdout:update-alternatives: using /usr/bin/python3-wsdump to provide /usr/bin/wsdump (wsdump) in auto mode 2023-09-23T18:53:46.750 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-webtest (2.0.32-2) ... 2023-09-23T18:53:46.876 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up libonig5:amd64 (6.9.4-1) ... 2023-09-23T18:53:47.011 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-mako (1.1.0+ds1-1ubuntu2.1) ... 2023-09-23T18:53:47.043 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up ceph-mgr-modules-core (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:47.169 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up ceph-fuse (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:47.320 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-cherrypy3 (8.9.1-6) ... 2023-09-23T18:53:47.383 INFO:teuthology.orchestra.run.smithi092.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2023-09-23T18:53:47.383 INFO:teuthology.orchestra.run.smithi092.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2023-09-23T18:53:47.672 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up libjs-underscore (1.9.1~dfsg-1ubuntu0.20.04.1) ... 2023-09-23T18:53:47.798 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up liblbfgsb0:amd64 (3.0+dfsg.3-7build1) ... 2023-09-23T18:53:47.828 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up libcephfs-dev (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:47.915 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-webob (1:1.8.5-2) ... 2023-09-23T18:53:47.954 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-requests-oauthlib (1.0.0-1.1build1) ... 2023-09-23T18:53:48.234 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-pastescript (2.0.2-4) ... 2023-09-23T18:53:48.242 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-blinker (1.4+dfsg1-0.3ubuntu1) ... 2023-09-23T18:53:48.535 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up librados2 (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:48.543 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-pecan (1.3.3-3ubuntu1) ... 2023-09-23T18:53:48.670 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-scipy (1.3.3-3build1) ... 2023-09-23T18:53:48.904 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-rados (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:49.030 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-rbd (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:49.156 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up rbd-fuse (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:49.293 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-routes (2.4.1-2) ... 2023-09-23T18:53:49.574 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-google-auth (1.5.1-2) ... 2023-09-23T18:53:49.972 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-rgw (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:50.097 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-cephfs (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:50.192 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up librgw2 (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:50.314 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up libsqlite3-mod-ceph (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:50.382 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up python3-kubernetes (7.0.0~a1-2) ... 2023-09-23T18:53:50.441 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up libjq1:amd64 (1.6-1ubuntu0.20.04.1) ... 2023-09-23T18:53:50.567 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-pytest (4.6.9-1) ... 2023-09-23T18:53:50.944 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-openssl (19.0.0-1build1) ... 2023-09-23T18:53:51.245 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-pastedeploy (2.1.0-1) ... 2023-09-23T18:53:51.310 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up ceph-common (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:51.547 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-bs4 (4.8.2-1) ... 2023-09-23T18:53:51.853 INFO:teuthology.orchestra.run.smithi092.stdout:Adding group ceph....done 2023-09-23T18:53:51.890 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up libcephfs2 (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:51.999 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-oauthlib (3.1.0-1ubuntu2) ... 2023-09-23T18:53:52.272 INFO:teuthology.orchestra.run.smithi092.stdout:Adding system user ceph....done 2023-09-23T18:53:52.326 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up libradosstriper1 (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:52.449 INFO:teuthology.orchestra.run.smithi092.stdout:Setting system user ceph properties....done 2023-09-23T18:53:52.451 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-pyasn1-modules (0.2.1-0.2build1) ... 2023-09-23T18:53:52.467 INFO:teuthology.orchestra.run.smithi092.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2023-09-23T18:53:52.613 INFO:teuthology.orchestra.run.smithi092.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /lib/systemd/system/ceph.target. 2023-09-23T18:53:52.786 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-sklearn (0.22.2.post1+dfsg-5) ... 2023-09-23T18:53:52.935 INFO:teuthology.orchestra.run.smithi092.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service -> /lib/systemd/system/rbdmap.service. 2023-09-23T18:53:53.394 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up ceph-test (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:53.520 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up radosgw (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:53.785 INFO:teuthology.orchestra.run.smithi092.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2023-09-23T18:53:53.786 INFO:teuthology.orchestra.run.smithi092.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2023-09-23T18:53:53.885 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up librbd1 (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:54.010 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up libjs-sphinxdoc (1.8.5-7ubuntu3) ... 2023-09-23T18:53:54.136 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up jq (1.6-1ubuntu0.20.04.1) ... 2023-09-23T18:53:54.258 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-rsa (4.0-3) ... 2023-09-23T18:53:54.438 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up ceph-base (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:54.551 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-webtest (2.0.32-2) ... 2023-09-23T18:53:54.828 INFO:teuthology.orchestra.run.smithi092.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service -> /lib/systemd/system/ceph-crash.service. 2023-09-23T18:53:54.861 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up ceph-mgr-modules-core (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:54.987 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up ceph-fuse (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:55.215 INFO:teuthology.orchestra.run.smithi016.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2023-09-23T18:53:55.216 INFO:teuthology.orchestra.run.smithi016.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2023-09-23T18:53:55.307 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up ceph-mds (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:55.534 INFO:teuthology.orchestra.run.smithi092.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2023-09-23T18:53:55.535 INFO:teuthology.orchestra.run.smithi092.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2023-09-23T18:53:55.697 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up libcephfs-dev (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:55.815 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-requests-oauthlib (1.0.0-1.1build1) ... 2023-09-23T18:53:56.008 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up ceph-mgr (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:56.117 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-pastescript (2.0.2-4) ... 2023-09-23T18:53:56.237 INFO:teuthology.orchestra.run.smithi092.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2023-09-23T18:53:56.237 INFO:teuthology.orchestra.run.smithi092.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2023-09-23T18:53:56.459 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-pecan (1.3.3-3ubuntu1) ... 2023-09-23T18:53:56.702 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up ceph-osd (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:56.822 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-rados (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:56.948 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-rbd (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:57.074 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up rbd-fuse (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:57.191 INFO:teuthology.orchestra.run.smithi092.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2023-09-23T18:53:57.191 INFO:teuthology.orchestra.run.smithi092.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2023-09-23T18:53:57.200 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-routes (2.4.1-2) ... 2023-09-23T18:53:57.510 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-google-auth (1.5.1-2) ... 2023-09-23T18:53:57.644 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up ceph-mgr-k8sevents (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:57.771 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up ceph-mgr-diskprediction-local (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:57.906 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-rgw (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:57.915 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up ceph-mon (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:58.032 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-cephfs (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:58.133 INFO:teuthology.orchestra.run.smithi092.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2023-09-23T18:53:58.133 INFO:teuthology.orchestra.run.smithi092.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2023-09-23T18:53:58.324 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up python3-kubernetes (7.0.0~a1-2) ... 2023-09-23T18:53:58.618 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up ceph-mgr-cephadm (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:58.761 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up ceph (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:58.887 INFO:teuthology.orchestra.run.smithi092.stdout:Setting up ceph-mgr-dashboard (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:59.030 INFO:teuthology.orchestra.run.smithi092.stdout:Processing triggers for systemd (245.4-4ubuntu3.17) ... 2023-09-23T18:53:59.303 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up ceph-common (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:53:59.323 INFO:teuthology.orchestra.run.smithi092.stdout:Processing triggers for man-db (2.9.1-1) ... 2023-09-23T18:53:59.856 INFO:teuthology.orchestra.run.smithi016.stdout:Adding group ceph....done 2023-09-23T18:54:00.250 INFO:teuthology.orchestra.run.smithi016.stdout:Adding system user ceph....done 2023-09-23T18:54:00.426 INFO:teuthology.orchestra.run.smithi016.stdout:Setting system user ceph properties....done 2023-09-23T18:54:00.443 INFO:teuthology.orchestra.run.smithi016.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2023-09-23T18:54:00.600 INFO:teuthology.orchestra.run.smithi016.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /lib/systemd/system/ceph.target. 2023-09-23T18:54:00.909 INFO:teuthology.orchestra.run.smithi016.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service -> /lib/systemd/system/rbdmap.service. 2023-09-23T18:54:01.397 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up ceph-test (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:54:01.507 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up radosgw (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:54:01.781 INFO:teuthology.orchestra.run.smithi016.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2023-09-23T18:54:01.781 INFO:teuthology.orchestra.run.smithi016.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2023-09-23T18:54:02.254 INFO:teuthology.orchestra.run.smithi092.stdout:Processing triggers for libc-bin (2.31-0ubuntu9.9) ... 2023-09-23T18:54:02.476 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up ceph-base (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:54:02.699 INFO:teuthology.orchestra.run.smithi092.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2023-09-23T18:54:02.701 DEBUG:teuthology.parallel:result is None 2023-09-23T18:54:02.864 INFO:teuthology.orchestra.run.smithi016.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service -> /lib/systemd/system/ceph-crash.service. 2023-09-23T18:54:03.339 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up ceph-mds (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:54:03.593 INFO:teuthology.orchestra.run.smithi016.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2023-09-23T18:54:03.594 INFO:teuthology.orchestra.run.smithi016.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2023-09-23T18:54:04.049 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up ceph-mgr (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:54:04.276 INFO:teuthology.orchestra.run.smithi016.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2023-09-23T18:54:04.276 INFO:teuthology.orchestra.run.smithi016.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2023-09-23T18:54:04.742 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up ceph-osd (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:54:05.241 INFO:teuthology.orchestra.run.smithi016.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2023-09-23T18:54:05.241 INFO:teuthology.orchestra.run.smithi016.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2023-09-23T18:54:05.710 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up ceph-mgr-k8sevents (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:54:05.836 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up ceph-mgr-diskprediction-local (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:54:05.982 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up ceph-mon (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:54:06.208 INFO:teuthology.orchestra.run.smithi016.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2023-09-23T18:54:06.208 INFO:teuthology.orchestra.run.smithi016.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2023-09-23T18:54:06.658 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up ceph-mgr-cephadm (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:54:06.801 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up ceph (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:54:06.927 INFO:teuthology.orchestra.run.smithi016.stdout:Setting up ceph-mgr-dashboard (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:54:07.096 INFO:teuthology.orchestra.run.smithi016.stdout:Processing triggers for systemd (245.4-4ubuntu3.17) ... 2023-09-23T18:54:07.388 INFO:teuthology.orchestra.run.smithi016.stdout:Processing triggers for man-db (2.9.1-1) ... 2023-09-23T18:54:10.615 INFO:teuthology.orchestra.run.smithi016.stdout:Processing triggers for libc-bin (2.31-0ubuntu9.9) ... 2023-09-23T18:54:11.085 INFO:teuthology.orchestra.run.smithi016.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2023-09-23T18:54:11.088 DEBUG:teuthology.parallel:result is None 2023-09-23T18:54:11.089 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T18:54:11.089 INFO:teuthology.packaging:ref: None 2023-09-23T18:54:11.089 INFO:teuthology.packaging:tag: None 2023-09-23T18:54:11.089 INFO:teuthology.packaging:branch: pacific 2023-09-23T18:54:11.089 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T18:54:11.089 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=pacific 2023-09-23T18:54:11.417 DEBUG:teuthology.orchestra.run.smithi016:> dpkg-query -W -f '${Version}' ceph 2023-09-23T18:54:11.440 INFO:teuthology.orchestra.run.smithi016.stdout:16.2.14-37-g2e1f1015-1focal 2023-09-23T18:54:11.441 INFO:teuthology.packaging:The installed version of ceph is 16.2.14-37-g2e1f1015-1focal 2023-09-23T18:54:11.441 INFO:teuthology.task.install:The correct ceph version 16.2.14-37-g2e1f1015-1focal is installed. 2023-09-23T18:54:11.441 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T18:54:11.441 INFO:teuthology.packaging:ref: None 2023-09-23T18:54:11.441 INFO:teuthology.packaging:tag: None 2023-09-23T18:54:11.441 INFO:teuthology.packaging:branch: pacific 2023-09-23T18:54:11.441 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T18:54:11.442 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=pacific 2023-09-23T18:54:11.664 DEBUG:teuthology.orchestra.run.smithi092:> dpkg-query -W -f '${Version}' ceph 2023-09-23T18:54:11.687 INFO:teuthology.orchestra.run.smithi092.stdout:16.2.14-37-g2e1f1015-1focal 2023-09-23T18:54:11.687 INFO:teuthology.packaging:The installed version of ceph is 16.2.14-37-g2e1f1015-1focal 2023-09-23T18:54:11.688 INFO:teuthology.task.install:The correct ceph version 16.2.14-37-g2e1f1015-1focal is installed. 2023-09-23T18:54:11.688 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2023-09-23T18:54:11.688 DEBUG:teuthology.orchestra.run.smithi016:> set -ex 2023-09-23T18:54:11.688 DEBUG:teuthology.orchestra.run.smithi016:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-09-23T18:54:11.706 DEBUG:teuthology.orchestra.run.smithi092:> set -ex 2023-09-23T18:54:11.706 DEBUG:teuthology.orchestra.run.smithi092:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-09-23T18:54:11.743 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2023-09-23T18:54:11.743 DEBUG:teuthology.orchestra.run.smithi016:> set -ex 2023-09-23T18:54:11.743 DEBUG:teuthology.orchestra.run.smithi016:> sudo dd of=/usr/bin/daemon-helper 2023-09-23T18:54:11.769 DEBUG:teuthology.orchestra.run.smithi016:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-09-23T18:54:11.826 DEBUG:teuthology.orchestra.run.smithi092:> set -ex 2023-09-23T18:54:11.826 DEBUG:teuthology.orchestra.run.smithi092:> sudo dd of=/usr/bin/daemon-helper 2023-09-23T18:54:11.840 DEBUG:teuthology.orchestra.run.smithi092:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-09-23T18:54:11.901 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2023-09-23T18:54:11.901 DEBUG:teuthology.orchestra.run.smithi016:> set -ex 2023-09-23T18:54:11.901 DEBUG:teuthology.orchestra.run.smithi016:> sudo dd of=/usr/bin/adjust-ulimits 2023-09-23T18:54:11.916 DEBUG:teuthology.orchestra.run.smithi016:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-09-23T18:54:11.973 DEBUG:teuthology.orchestra.run.smithi092:> set -ex 2023-09-23T18:54:11.974 DEBUG:teuthology.orchestra.run.smithi092:> sudo dd of=/usr/bin/adjust-ulimits 2023-09-23T18:54:11.989 DEBUG:teuthology.orchestra.run.smithi092:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-09-23T18:54:12.049 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2023-09-23T18:54:12.049 DEBUG:teuthology.orchestra.run.smithi016:> set -ex 2023-09-23T18:54:12.049 DEBUG:teuthology.orchestra.run.smithi016:> sudo dd of=/usr/bin/stdin-killer 2023-09-23T18:54:12.066 DEBUG:teuthology.orchestra.run.smithi016:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-09-23T18:54:12.126 DEBUG:teuthology.orchestra.run.smithi092:> set -ex 2023-09-23T18:54:12.126 DEBUG:teuthology.orchestra.run.smithi092:> sudo dd of=/usr/bin/stdin-killer 2023-09-23T18:54:12.142 DEBUG:teuthology.orchestra.run.smithi092:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-09-23T18:54:12.199 INFO:teuthology.run_tasks:Running task cephadm... 2023-09-23T18:54:12.274 INFO:tasks.cephadm:Config: {'cephadm_branch': 'pacific', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:pacific', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'} 2023-09-23T18:54:12.274 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:pacific 2023-09-23T18:54:12.275 INFO:tasks.cephadm:Cluster fsid is 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 2023-09-23T18:54:12.275 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2023-09-23T18:54:12.275 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.16', 'mon.c': '[v2:172.21.15.16:3301,v1:172.21.15.16:6790]', 'mon.b': '172.21.15.92'} 2023-09-23T18:54:12.275 INFO:tasks.cephadm:First mon is mon.a on smithi016 2023-09-23T18:54:12.275 INFO:tasks.cephadm:First mgr is y 2023-09-23T18:54:12.275 INFO:tasks.cephadm:Normalizing hostnames... 2023-09-23T18:54:12.276 DEBUG:teuthology.orchestra.run.smithi016:> sudo hostname $(hostname -s) 2023-09-23T18:54:12.291 DEBUG:teuthology.orchestra.run.smithi092:> sudo hostname $(hostname -s) 2023-09-23T18:54:12.305 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref pacific)... 2023-09-23T18:54:12.305 DEBUG:teuthology.orchestra.run.smithi016:> curl --silent https://raw.githubusercontent.com/ceph/ceph/pacific/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-09-23T18:54:12.589 INFO:teuthology.orchestra.run.smithi016.stdout:-rw-rw-r-- 1 ubuntu ubuntu 350873 Sep 23 18:54 /home/ubuntu/cephtest/cephadm 2023-09-23T18:54:12.589 DEBUG:teuthology.orchestra.run.smithi092:> curl --silent https://raw.githubusercontent.com/ceph/ceph/pacific/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-09-23T18:54:12.743 INFO:teuthology.orchestra.run.smithi092.stdout:-rw-rw-r-- 1 ubuntu ubuntu 350873 Sep 23 18:54 /home/ubuntu/cephtest/cephadm 2023-09-23T18:54:12.744 DEBUG:teuthology.orchestra.run.smithi016:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-09-23T18:54:12.753 DEBUG:teuthology.orchestra.run.smithi092:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-09-23T18:54:12.771 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:pacific on all hosts... 2023-09-23T18:54:12.771 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific pull 2023-09-23T18:54:12.796 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific pull 2023-09-23T18:54:13.071 INFO:teuthology.orchestra.run.smithi016.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:pacific... 2023-09-23T18:54:13.080 INFO:teuthology.orchestra.run.smithi092.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:pacific... 2023-09-23T18:55:09.268 INFO:teuthology.orchestra.run.smithi092.stdout:{ 2023-09-23T18:55:09.269 INFO:teuthology.orchestra.run.smithi092.stdout: "ceph_version": "ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)", 2023-09-23T18:55:09.269 INFO:teuthology.orchestra.run.smithi092.stdout: "image_id": "1e6a756174a8dbc0d5ced5aeb4272e8f04dc2045a9196dffb3df976d1caca13e", 2023-09-23T18:55:09.269 INFO:teuthology.orchestra.run.smithi092.stdout: "repo_digests": [ 2023-09-23T18:55:09.269 INFO:teuthology.orchestra.run.smithi092.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:396076557f19bac7d00d097a6db258821cfaec622cd332710d7957388aedf72b" 2023-09-23T18:55:09.269 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T18:55:09.270 INFO:teuthology.orchestra.run.smithi092.stdout:} 2023-09-23T18:55:11.016 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-09-23T18:55:11.016 INFO:teuthology.orchestra.run.smithi016.stdout: "ceph_version": "ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)", 2023-09-23T18:55:11.016 INFO:teuthology.orchestra.run.smithi016.stdout: "image_id": "1e6a756174a8dbc0d5ced5aeb4272e8f04dc2045a9196dffb3df976d1caca13e", 2023-09-23T18:55:11.016 INFO:teuthology.orchestra.run.smithi016.stdout: "repo_digests": [ 2023-09-23T18:55:11.016 INFO:teuthology.orchestra.run.smithi016.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:396076557f19bac7d00d097a6db258821cfaec622cd332710d7957388aedf72b" 2023-09-23T18:55:11.016 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T18:55:11.016 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-09-23T18:55:11.042 DEBUG:teuthology.orchestra.run.smithi016:> sudo mkdir -p /etc/ceph 2023-09-23T18:55:11.054 DEBUG:teuthology.orchestra.run.smithi092:> sudo mkdir -p /etc/ceph 2023-09-23T18:55:11.070 DEBUG:teuthology.orchestra.run.smithi016:> sudo chmod 777 /etc/ceph 2023-09-23T18:55:11.111 DEBUG:teuthology.orchestra.run.smithi092:> sudo chmod 777 /etc/ceph 2023-09-23T18:55:11.129 INFO:tasks.cephadm:Writing seed config... 2023-09-23T18:55:11.130 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2023-09-23T18:55:11.130 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2023-09-23T18:55:11.130 INFO:tasks.cephadm: override: [osd] debug ms = 1 2023-09-23T18:55:11.130 INFO:tasks.cephadm: override: [osd] debug osd = 20 2023-09-23T18:55:11.130 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2023-09-23T18:55:11.130 INFO:tasks.cephadm: override: [global] mon election default strategy = 3 2023-09-23T18:55:11.130 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2023-09-23T18:55:11.130 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2023-09-23T18:55:11.131 INFO:tasks.cephadm: override: [mon] debug mon = 20 2023-09-23T18:55:11.131 INFO:tasks.cephadm: override: [mon] debug ms = 1 2023-09-23T18:55:11.131 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2023-09-23T18:55:11.131 DEBUG:teuthology.orchestra.run.smithi016:> set -ex 2023-09-23T18:55:11.131 DEBUG:teuthology.orchestra.run.smithi016:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2023-09-23T18:55:11.161 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 mon election default strategy = 3 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2023-09-23T18:55:11.162 DEBUG:teuthology.orchestra.run.smithi016:mon.a> sudo journalctl -f -n 0 -u ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@mon.a.service 2023-09-23T18:55:11.204 DEBUG:teuthology.orchestra.run.smithi016:mgr.y> sudo journalctl -f -n 0 -u ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@mgr.y.service 2023-09-23T18:55:11.248 INFO:tasks.cephadm:Bootstrapping... 2023-09-23T18:55:11.248 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific -v bootstrap --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.16 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2023-09-23T18:55:11.456 INFO:teuthology.orchestra.run.smithi016.stderr:-------------------------------------------------------------------------------- 2023-09-23T18:55:11.456 INFO:teuthology.orchestra.run.smithi016.stderr:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:pacific', '-v', 'bootstrap', '--fsid', '95ce9eb6-5a42-11ee-8db4-212e2dc638e7', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.16', '--skip-admin-label'] 2023-09-23T18:55:11.456 INFO:teuthology.orchestra.run.smithi016.stderr:Verifying podman|docker is present... 2023-09-23T18:55:11.456 INFO:teuthology.orchestra.run.smithi016.stderr:Verifying lvm2 is present... 2023-09-23T18:55:11.456 INFO:teuthology.orchestra.run.smithi016.stderr:Verifying time synchronization is in place... 2023-09-23T18:55:11.460 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2023-09-23T18:55:11.466 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: inactive 2023-09-23T18:55:11.472 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: Failed to get unit file state for chronyd.service: No such file or directory 2023-09-23T18:55:11.476 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: inactive 2023-09-23T18:55:11.481 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: masked 2023-09-23T18:55:11.486 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: inactive 2023-09-23T18:55:11.491 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: Failed to get unit file state for ntpd.service: No such file or directory 2023-09-23T18:55:11.496 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: inactive 2023-09-23T18:55:11.503 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: enabled 2023-09-23T18:55:11.508 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: active 2023-09-23T18:55:11.509 INFO:teuthology.orchestra.run.smithi016.stderr:Unit ntp.service is enabled and running 2023-09-23T18:55:11.510 INFO:teuthology.orchestra.run.smithi016.stderr:Repeating the final host check... 2023-09-23T18:55:11.510 INFO:teuthology.orchestra.run.smithi016.stderr:docker (/usr/bin/docker) is present 2023-09-23T18:55:11.510 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl is present 2023-09-23T18:55:11.510 INFO:teuthology.orchestra.run.smithi016.stderr:lvcreate is present 2023-09-23T18:55:11.516 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2023-09-23T18:55:11.521 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: inactive 2023-09-23T18:55:11.528 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: Failed to get unit file state for chronyd.service: No such file or directory 2023-09-23T18:55:11.535 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: inactive 2023-09-23T18:55:11.538 INFO:journalctl@ceph.mon.a.smithi016.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-09-23T18:55:11.540 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: masked 2023-09-23T18:55:11.546 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: inactive 2023-09-23T18:55:11.554 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: Failed to get unit file state for ntpd.service: No such file or directory 2023-09-23T18:55:11.559 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: inactive 2023-09-23T18:55:11.568 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: enabled 2023-09-23T18:55:11.575 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: active 2023-09-23T18:55:11.577 INFO:teuthology.orchestra.run.smithi016.stderr:Unit ntp.service is enabled and running 2023-09-23T18:55:11.577 INFO:teuthology.orchestra.run.smithi016.stderr:Host looks OK 2023-09-23T18:55:11.577 INFO:teuthology.orchestra.run.smithi016.stderr:Cluster fsid: 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 2023-09-23T18:55:11.577 INFO:teuthology.orchestra.run.smithi016.stderr:Acquiring lock 139865091145680 on /run/cephadm/95ce9eb6-5a42-11ee-8db4-212e2dc638e7.lock 2023-09-23T18:55:11.577 INFO:teuthology.orchestra.run.smithi016.stderr:Lock 139865091145680 acquired on /run/cephadm/95ce9eb6-5a42-11ee-8db4-212e2dc638e7.lock 2023-09-23T18:55:11.578 INFO:teuthology.orchestra.run.smithi016.stderr:Verifying IP 172.21.15.16 port 3300 ... 2023-09-23T18:55:11.578 INFO:teuthology.orchestra.run.smithi016.stderr:Verifying IP 172.21.15.16 port 6789 ... 2023-09-23T18:55:11.578 INFO:teuthology.orchestra.run.smithi016.stderr:Base mon IP(s) is [172.21.15.16:3300, 172.21.15.16:6789], mon addrv is [v2:172.21.15.16:3300,v1:172.21.15.16:6789] 2023-09-23T18:55:11.581 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/sbin/ip: default via 172.21.15.254 dev ens1f0 2023-09-23T18:55:11.581 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/sbin/ip: 172.17.0.0/16 dev docker0 proto kernel scope link src 172.17.0.1 linkdown 2023-09-23T18:55:11.582 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/sbin/ip: 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.16 2023-09-23T18:55:11.585 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2023-09-23T18:55:11.586 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/sbin/ip: fe80::/64 dev ens1f0 proto kernel metric 256 pref medium 2023-09-23T18:55:11.586 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 1024 expires 1769sec hoplimit 64 pref medium 2023-09-23T18:55:11.590 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2023-09-23T18:55:11.590 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/sbin/ip: inet6 ::1/128 scope host 2023-09-23T18:55:11.590 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/sbin/ip: valid_lft forever preferred_lft forever 2023-09-23T18:55:11.590 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/sbin/ip: 2: ens1f0: mtu 1500 state UP qlen 1000 2023-09-23T18:55:11.590 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/sbin/ip: inet6 fe80::ec4:7aff:febd:1564/64 scope link 2023-09-23T18:55:11.590 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/sbin/ip: valid_lft forever preferred_lft forever 2023-09-23T18:55:11.591 INFO:teuthology.orchestra.run.smithi016.stderr:Mon IP `172.21.15.16` is in CIDR network `172.21.0.0/20` 2023-09-23T18:55:11.591 INFO:teuthology.orchestra.run.smithi016.stderr:Mon IP `172.21.15.16` is in CIDR network `172.21.0.0/20` 2023-09-23T18:55:11.591 INFO:teuthology.orchestra.run.smithi016.stderr:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2023-09-23T18:55:11.591 INFO:teuthology.orchestra.run.smithi016.stderr:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2023-09-23T18:55:11.592 INFO:teuthology.orchestra.run.smithi016.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:pacific... 2023-09-23T18:55:11.675 INFO:journalctl@ceph.mgr.y.smithi016.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-09-23T18:55:11.836 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/docker: pacific: Pulling from ceph-ci/ceph 2023-09-23T18:55:11.866 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/docker: Digest: sha256:396076557f19bac7d00d097a6db258821cfaec622cd332710d7957388aedf72b 2023-09-23T18:55:11.867 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/docker: Status: Image is up to date for quay.ceph.io/ceph-ci/ceph:pacific 2023-09-23T18:55:11.899 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/docker: quay.ceph.io/ceph-ci/ceph:pacific 2023-09-23T18:55:12.821 INFO:teuthology.orchestra.run.smithi016.stderr:ceph: ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable) 2023-09-23T18:55:13.354 INFO:teuthology.orchestra.run.smithi016.stderr:Ceph version: ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable) 2023-09-23T18:55:13.354 INFO:teuthology.orchestra.run.smithi016.stderr:Extracting ceph user uid/gid from container image... 2023-09-23T18:55:14.137 INFO:teuthology.orchestra.run.smithi016.stderr:stat: 167 167 2023-09-23T18:55:14.723 INFO:teuthology.orchestra.run.smithi016.stderr:Creating initial keys... 2023-09-23T18:55:15.532 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-authtool: AQCTNA9lfyuDHxAA5pqEAUfBhvu+sLnduXbOUA== 2023-09-23T18:55:16.845 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-authtool: AQCUNA9lIY4vMhAA6ns7W5BCpLn/3IsH8CNwnQ== 2023-09-23T18:55:18.221 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-authtool: AQCWNA9lKPL3DBAASeSl5KLTOJ8Z66NnBN5qxA== 2023-09-23T18:55:18.763 INFO:teuthology.orchestra.run.smithi016.stderr:Creating initial monmap... 2023-09-23T18:55:19.604 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2023-09-23T18:55:19.604 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 2023-09-23T18:55:19.604 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-09-23T18:55:20.158 INFO:teuthology.orchestra.run.smithi016.stderr:monmaptool for a [v2:172.21.15.16:3300,v1:172.21.15.16:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2023-09-23T18:55:20.158 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/monmaptool: set fsid to 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 2023-09-23T18:55:20.158 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-09-23T18:55:20.158 INFO:teuthology.orchestra.run.smithi016.stderr: 2023-09-23T18:55:20.159 INFO:teuthology.orchestra.run.smithi016.stderr:Creating mon... 2023-09-23T18:55:20.992 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.981+0000 7f9e6ff2d880 0 set uid:gid to 167:167 (ceph:ceph) 2023-09-23T18:55:20.992 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.981+0000 7f9e6ff2d880 1 imported monmap: 2023-09-23T18:55:20.992 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: epoch 0 2023-09-23T18:55:20.992 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 2023-09-23T18:55:20.993 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: last_changed 2023-09-23T18:55:19.600161+0000 2023-09-23T18:55:20.993 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: created 2023-09-23T18:55:19.600161+0000 2023-09-23T18:55:20.993 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: min_mon_release 0 (unknown) 2023-09-23T18:55:20.993 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: election_strategy: 1 2023-09-23T18:55:20.994 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: 0: [v2:172.21.15.16:3300/0,v1:172.21.15.16:6789/0] mon.a 2023-09-23T18:55:20.995 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: 2023-09-23T18:55:20.995 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.981+0000 7f9e6ff2d880 0 /usr/bin/ceph-mon: set fsid to 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 2023-09-23T18:55:20.995 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: RocksDB version: 6.8.1 2023-09-23T18:55:20.995 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: 2023-09-23T18:55:20.995 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-09-23T18:55:20.995 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Compile date Sep 21 2023 2023-09-23T18:55:20.996 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: DB SUMMARY 2023-09-23T18:55:20.996 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: 2023-09-23T18:55:20.996 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 0, files: 2023-09-23T18:55:20.997 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: 2023-09-23T18:55:20.997 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 2023-09-23T18:55:20.997 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: 2023-09-23T18:55:20.997 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.error_if_exists: 0 2023-09-23T18:55:20.997 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.create_if_missing: 1 2023-09-23T18:55:21.000 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.paranoid_checks: 1 2023-09-23T18:55:21.000 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.env: 0x5623d37c1080 2023-09-23T18:55:21.000 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.fs: Posix File System 2023-09-23T18:55:21.000 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.info_log: 0x5623d46b1aa0 2023-09-23T18:55:21.000 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.max_file_opening_threads: 16 2023-09-23T18:55:21.000 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.statistics: (nil) 2023-09-23T18:55:21.000 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.use_fsync: 0 2023-09-23T18:55:21.000 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.max_log_file_size: 0 2023-09-23T18:55:21.001 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T18:55:21.001 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T18:55:21.001 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.keep_log_file_num: 1000 2023-09-23T18:55:21.001 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.recycle_log_file_num: 0 2023-09-23T18:55:21.001 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.allow_fallocate: 1 2023-09-23T18:55:21.001 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.allow_mmap_reads: 0 2023-09-23T18:55:21.001 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.allow_mmap_writes: 0 2023-09-23T18:55:21.001 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.use_direct_reads: 0 2023-09-23T18:55:21.001 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T18:55:21.001 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.create_missing_column_families: 0 2023-09-23T18:55:21.001 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.db_log_dir: 2023-09-23T18:55:21.001 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2023-09-23T18:55:21.002 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T18:55:21.002 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.max_subcompactions: 1 2023-09-23T18:55:21.002 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.max_background_flushes: -1 2023-09-23T18:55:21.002 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T18:55:21.002 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T18:55:21.002 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T18:55:21.002 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T18:55:21.022 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T18:55:21.022 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.advise_random_on_open: 1 2023-09-23T18:55:21.022 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.db_write_buffer_size: 0 2023-09-23T18:55:21.022 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.write_buffer_manager: 0x5623d46b9890 2023-09-23T18:55:21.022 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T18:55:21.022 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-09-23T18:55:21.022 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T18:55:21.022 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T18:55:21.023 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.rate_limiter: (nil) 2023-09-23T18:55:21.023 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T18:55:21.023 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.wal_recovery_mode: 2 2023-09-23T18:55:21.023 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.enable_thread_tracking: 0 2023-09-23T18:55:21.023 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.enable_pipelined_write: 0 2023-09-23T18:55:21.023 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.unordered_write: 0 2023-09-23T18:55:21.023 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T18:55:21.023 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T18:55:21.023 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T18:55:21.023 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T18:55:21.023 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.row_cache: None 2023-09-23T18:55:21.024 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.wal_filter: None 2023-09-23T18:55:21.024 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T18:55:21.024 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.allow_ingest_behind: 0 2023-09-23T18:55:21.024 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.preserve_deletes: 0 2023-09-23T18:55:21.024 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.two_write_queues: 0 2023-09-23T18:55:21.024 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.manual_wal_flush: 0 2023-09-23T18:55:21.024 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.atomic_flush: 0 2023-09-23T18:55:21.024 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T18:55:21.024 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T18:55:21.024 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T18:55:21.024 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.log_readahead_size: 0 2023-09-23T18:55:21.024 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.sst_file_checksum_func: Unknown 2023-09-23T18:55:21.024 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.max_background_jobs: 2 2023-09-23T18:55:21.025 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.max_background_compactions: -1 2023-09-23T18:55:21.025 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T18:55:21.025 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T18:55:21.025 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T18:55:21.025 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.max_total_wal_size: 0 2023-09-23T18:55:21.025 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T18:55:21.025 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T18:55:21.025 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T18:55:21.025 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T18:55:21.025 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.max_open_files: -1 2023-09-23T18:55:21.025 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.bytes_per_sync: 0 2023-09-23T18:55:21.025 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T18:55:21.026 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T18:55:21.026 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.compaction_readahead_size: 0 2023-09-23T18:55:21.026 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Compression algorithms supported: 2023-09-23T18:55:21.026 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T18:55:21.026 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: kZSTD supported: 0 2023-09-23T18:55:21.026 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: kXpressCompression supported: 0 2023-09-23T18:55:21.026 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: kLZ4HCCompression supported: 1 2023-09-23T18:55:21.026 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: kLZ4Compression supported: 1 2023-09-23T18:55:21.026 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: kBZip2Compression supported: 0 2023-09-23T18:55:21.026 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: kZlibCompression supported: 1 2023-09-23T18:55:21.026 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: kSnappyCompression supported: 1 2023-09-23T18:55:21.026 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T18:55:21.027 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: [db_impl/db_impl_open.cc:273] Creating manifest 1 2023-09-23T18:55:21.027 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: 2023-09-23T18:55:21.027 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000001 2023-09-23T18:55:21.027 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: 2023-09-23T18:55:21.027 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2023-09-23T18:55:21.027 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: 2023-09-23T18:55:21.027 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T18:55:21.027 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.merge_operator: 2023-09-23T18:55:21.027 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.compaction_filter: None 2023-09-23T18:55:21.027 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.compaction_filter_factory: None 2023-09-23T18:55:21.027 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T18:55:21.027 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T18:55:21.028 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5623d45c7d20) 2023-09-23T18:55:21.028 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: cache_index_and_filter_blocks: 1 2023-09-23T18:55:21.028 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T18:55:21.028 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T18:55:21.028 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: pin_top_level_index_and_filter: 1 2023-09-23T18:55:21.028 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: index_type: 0 2023-09-23T18:55:21.028 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: data_block_index_type: 0 2023-09-23T18:55:21.028 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: index_shortening: 1 2023-09-23T18:55:21.028 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: data_block_hash_table_util_ratio: 0.750000 2023-09-23T18:55:21.028 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: hash_index_allow_collision: 1 2023-09-23T18:55:21.028 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: checksum: 1 2023-09-23T18:55:21.028 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: no_block_cache: 0 2023-09-23T18:55:21.029 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: block_cache: 0x5623d45fed10 2023-09-23T18:55:21.029 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: block_cache_name: BinnedLRUCache 2023-09-23T18:55:21.029 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: block_cache_options: 2023-09-23T18:55:21.029 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: capacity : 536870912 2023-09-23T18:55:21.029 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: num_shard_bits : 4 2023-09-23T18:55:21.029 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: strict_capacity_limit : 0 2023-09-23T18:55:21.029 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: high_pri_pool_ratio: 0.000 2023-09-23T18:55:21.029 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: block_cache_compressed: (nil) 2023-09-23T18:55:21.029 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: persistent_cache: (nil) 2023-09-23T18:55:21.029 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: block_size: 4096 2023-09-23T18:55:21.029 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: block_size_deviation: 10 2023-09-23T18:55:21.029 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: block_restart_interval: 16 2023-09-23T18:55:21.030 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: index_block_restart_interval: 1 2023-09-23T18:55:21.030 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: metadata_block_size: 4096 2023-09-23T18:55:21.030 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: partition_filters: 0 2023-09-23T18:55:21.030 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: use_delta_encoding: 1 2023-09-23T18:55:21.030 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: filter_policy: rocksdb.BuiltinBloomFilter 2023-09-23T18:55:21.030 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: whole_key_filtering: 1 2023-09-23T18:55:21.030 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: verify_compression: 0 2023-09-23T18:55:21.030 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: read_amp_bytes_per_bit: 0 2023-09-23T18:55:21.030 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: format_version: 2 2023-09-23T18:55:21.030 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: enable_index_compression: 1 2023-09-23T18:55:21.030 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: block_align: 0 2023-09-23T18:55:21.030 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: 2023-09-23T18:55:21.031 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.write_buffer_size: 33554432 2023-09-23T18:55:21.031 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.max_write_buffer_number: 2 2023-09-23T18:55:21.031 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.compression: NoCompression 2023-09-23T18:55:21.031 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.bottommost_compression: Disabled 2023-09-23T18:55:21.031 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.prefix_extractor: nullptr 2023-09-23T18:55:21.031 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T18:55:21.031 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.num_levels: 7 2023-09-23T18:55:21.031 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T18:55:21.031 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T18:55:21.031 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T18:55:21.031 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T18:55:21.031 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T18:55:21.032 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T18:55:21.032 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T18:55:21.032 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T18:55:21.032 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T18:55:21.032 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T18:55:21.032 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.compression_opts.level: 32767 2023-09-23T18:55:21.032 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.compression_opts.strategy: 0 2023-09-23T18:55:21.032 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T18:55:21.032 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T18:55:21.032 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.compression_opts.enabled: false 2023-09-23T18:55:21.032 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T18:55:21.032 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T18:55:21.032 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T18:55:21.033 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.target_file_size_base: 67108864 2023-09-23T18:55:21.033 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T18:55:21.033 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T18:55:21.033 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T18:55:21.033 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T18:55:21.033 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T18:55:21.033 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T18:55:21.033 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T18:55:21.033 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T18:55:21.033 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T18:55:21.033 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T18:55:21.033 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T18:55:21.033 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T18:55:21.034 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T18:55:21.034 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.arena_block_size: 4194304 2023-09-23T18:55:21.034 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T18:55:21.034 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T18:55:21.034 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-09-23T18:55:21.034 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.disable_auto_compactions: 0 2023-09-23T18:55:21.034 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T18:55:21.034 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T18:55:21.034 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T18:55:21.034 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T18:55:21.034 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T18:55:21.034 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T18:55:21.035 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T18:55:21.035 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T18:55:21.035 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T18:55:21.035 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T18:55:21.035 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.table_properties_collectors: 2023-09-23T18:55:21.035 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.inplace_update_support: 0 2023-09-23T18:55:21.035 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T18:55:21.035 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T18:55:21.035 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T18:55:21.035 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T18:55:21.035 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.bloom_locality: 0 2023-09-23T18:55:21.035 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.max_successive_merges: 0 2023-09-23T18:55:21.036 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T18:55:21.036 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.paranoid_file_checks: 0 2023-09-23T18:55:21.036 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.force_consistency_checks: 0 2023-09-23T18:55:21.036 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.report_bg_io_stats: 0 2023-09-23T18:55:21.036 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.ttl: 2592000 2023-09-23T18:55:21.036 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.985+0000 7f9e6ff2d880 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T18:55:21.036 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.989+0000 7f9e6ff2d880 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T18:55:21.036 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: 2023-09-23T18:55:21.036 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.989+0000 7f9e6ff2d880 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 0 2023-09-23T18:55:21.036 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: 2023-09-23T18:55:21.036 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.989+0000 7f9e6ff2d880 4 rocksdb: DB pointer 0x5623d46c7800 2023-09-23T18:55:21.036 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.989+0000 7f9e58db6700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2023-09-23T18:55:21.037 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.989+0000 7f9e58db6700 4 rocksdb: [db_impl/db_impl.cc:851] 2023-09-23T18:55:21.037 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: ** DB Stats ** 2023-09-23T18:55:21.037 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T18:55:21.037 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T18:55:21.037 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T18:55:21.037 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T18:55:21.037 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T18:55:21.037 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-09-23T18:55:21.037 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T18:55:21.037 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: 2023-09-23T18:55:21.037 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-09-23T18:55:21.037 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T18:55:21.038 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T18:55:21.038 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-09-23T18:55:21.038 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-09-23T18:55:21.038 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: 2023-09-23T18:55:21.038 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-09-23T18:55:21.038 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T18:55:21.038 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T18:55:21.038 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T18:55:21.038 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T18:55:21.038 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T18:55:21.038 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T18:55:21.038 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T18:55:21.039 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: AddFile(Keys): cumulative 0, interval 0 2023-09-23T18:55:21.039 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T18:55:21.039 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T18:55:21.039 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T18:55:21.039 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: 2023-09-23T18:55:21.039 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: ** File Read Latency Histogram By Level [default] ** 2023-09-23T18:55:21.039 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: 2023-09-23T18:55:21.039 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-09-23T18:55:21.039 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T18:55:21.039 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T18:55:21.039 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-09-23T18:55:21.039 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-09-23T18:55:21.039 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: 2023-09-23T18:55:21.040 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-09-23T18:55:21.040 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T18:55:21.040 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T18:55:21.040 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T18:55:21.040 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T18:55:21.040 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T18:55:21.040 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T18:55:21.040 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T18:55:21.040 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: AddFile(Keys): cumulative 0, interval 0 2023-09-23T18:55:21.040 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T18:55:21.040 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T18:55:21.040 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T18:55:21.041 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: 2023-09-23T18:55:21.041 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: ** File Read Latency Histogram By Level [default] ** 2023-09-23T18:55:21.041 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: 2023-09-23T18:55:21.041 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.993+0000 7f9e6ff2d880 4 rocksdb: [db_impl/db_impl.cc:397] Shutdown: canceling all background work 2023-09-23T18:55:21.041 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.993+0000 7f9e6ff2d880 4 rocksdb: [db_impl/db_impl.cc:573] Shutdown complete 2023-09-23T18:55:21.041 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:55:20.993+0000 7f9e6ff2d880 0 /usr/bin/ceph-mon: created monfs at /var/lib/ceph/mon/ceph-a for mon.a 2023-09-23T18:55:21.472 INFO:teuthology.orchestra.run.smithi016.stderr:create mon.a on 2023-09-23T18:55:21.497 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: Removed /etc/systemd/system/multi-user.target.wants/ceph.target. 2023-09-23T18:55:21.704 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /etc/systemd/system/ceph.target. 2023-09-23T18:55:21.910 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7.target -> /etc/systemd/system/ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7.target. 2023-09-23T18:55:21.910 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7.target -> /etc/systemd/system/ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7.target. 2023-09-23T18:55:22.324 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: Failed to reset failed state of unit ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@mon.a.service: Unit ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@mon.a.service not loaded. 2023-09-23T18:55:22.331 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: Created symlink /etc/systemd/system/ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7.target.wants/ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@mon.a.service -> /etc/systemd/system/ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@.service. 2023-09-23T18:55:22.550 INFO:teuthology.orchestra.run.smithi016.stderr:firewalld does not appear to be present 2023-09-23T18:55:22.551 INFO:teuthology.orchestra.run.smithi016.stderr:Not possible to enable service . firewalld.service is not available 2023-09-23T18:55:22.551 INFO:teuthology.orchestra.run.smithi016.stderr:Waiting for mon to start... 2023-09-23T18:55:22.551 INFO:teuthology.orchestra.run.smithi016.stderr:Waiting for mon... 2023-09-23T18:55:22.609 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:22 smithi016 systemd[1]: Started Ceph mon.a for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7. 2023-09-23T18:55:24.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:23 smithi016 bash[26711]: cluster 2023-09-23T18:55:23.803111+0000 mon.a (mon.0) 0 : [INF] mkfs 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 2023-09-23T18:55:24.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:23 smithi016 bash[26711]: cluster 2023-09-23T18:55:23.798849+0000 mon.a (mon.0) 1 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2023-09-23T18:55:24.043 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: cluster: 2023-09-23T18:55:24.045 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: id: 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 2023-09-23T18:55:24.045 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: health: HEALTH_OK 2023-09-23T18:55:24.045 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: 2023-09-23T18:55:24.045 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: services: 2023-09-23T18:55:24.046 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: mon: 1 daemons, quorum a (age 0.23738s) 2023-09-23T18:55:24.046 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: mgr: no daemons active 2023-09-23T18:55:24.046 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2023-09-23T18:55:24.046 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: 2023-09-23T18:55:24.046 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: data: 2023-09-23T18:55:24.046 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2023-09-23T18:55:24.046 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2023-09-23T18:55:24.046 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2023-09-23T18:55:24.046 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: pgs: 2023-09-23T18:55:24.047 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: 2023-09-23T18:55:24.639 INFO:teuthology.orchestra.run.smithi016.stderr:mon is available 2023-09-23T18:55:24.639 INFO:teuthology.orchestra.run.smithi016.stderr:Assimilating anything we can from ceph.conf... 2023-09-23T18:55:25.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:24 smithi016 bash[26711]: cluster 2023-09-23T18:55:23.802137+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2023-09-23T18:55:25.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:24 smithi016 bash[26711]: cluster 2023-09-23T18:55:23.802286+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e1: 1 mons at {a=[v2:172.21.15.16:3300/0,v1:172.21.15.16:6789/0]} removed_ranks: {} 2023-09-23T18:55:25.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:24 smithi016 bash[26711]: cluster 2023-09-23T18:55:23.804852+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap 2023-09-23T18:55:25.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:24 smithi016 bash[26711]: cluster 2023-09-23T18:55:23.807418+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2023-09-23T18:55:25.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:24 smithi016 bash[26711]: cluster 2023-09-23T18:55:23.807973+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e1: no daemons active 2023-09-23T18:55:25.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:24 smithi016 bash[26711]: audit 2023-09-23T18:55:24.039430+0000 mon.a (mon.0) 7 : audit [DBG] from='client.? 172.21.15.16:0/810053025' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2023-09-23T18:55:25.686 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: 2023-09-23T18:55:25.686 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: [global] 2023-09-23T18:55:25.687 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: fsid = 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 2023-09-23T18:55:25.687 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: log_to_journald = false 2023-09-23T18:55:25.687 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.16:3300,v1:172.21.15.16:6789] 2023-09-23T18:55:25.687 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2023-09-23T18:55:25.687 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2023-09-23T18:55:25.687 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2023-09-23T18:55:25.687 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2023-09-23T18:55:25.688 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: 2023-09-23T18:55:25.688 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: [mgr] 2023-09-23T18:55:25.688 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2023-09-23T18:55:25.688 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: 2023-09-23T18:55:25.688 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: [osd] 2023-09-23T18:55:25.688 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2023-09-23T18:55:25.688 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2023-09-23T18:55:26.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:25 smithi016 bash[26711]: audit 2023-09-23T18:55:25.676655+0000 mon.a (mon.0) 8 : audit [INF] from='client.? 172.21.15.16:0/1797695249' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-09-23T18:55:26.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:25 smithi016 bash[26711]: audit 2023-09-23T18:55:25.681414+0000 mon.a (mon.0) 9 : audit [INF] from='client.? 172.21.15.16:0/1797695249' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2023-09-23T18:55:26.227 INFO:teuthology.orchestra.run.smithi016.stderr:Generating new minimal ceph.conf... 2023-09-23T18:55:27.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:27 smithi016 bash[26711]: audit 2023-09-23T18:55:27.286313+0000 mon.a (mon.0) 10 : audit [DBG] from='client.? 172.21.15.16:0/1711080087' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:55:27.825 INFO:teuthology.orchestra.run.smithi016.stderr:Restarting the monitor... 2023-09-23T18:55:28.138 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:27 smithi016 systemd[1]: Stopping Ceph mon.a for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7... 2023-09-23T18:55:28.139 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:27 smithi016 bash[27096]: Error response from daemon: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-mon.a 2023-09-23T18:55:28.139 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:28 smithi016 bash[26711]: debug 2023-09-23T18:55:28.001+0000 7fb4685c1700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-09-23T18:55:28.139 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:28 smithi016 bash[26711]: debug 2023-09-23T18:55:28.001+0000 7fb4685c1700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2023-09-23T18:55:28.634 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:28 smithi016 bash[27132]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-mon-a 2023-09-23T18:55:28.660 INFO:teuthology.orchestra.run.smithi016.stderr:Setting mon public_network to 172.21.0.0/20 2023-09-23T18:55:28.721 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:28 smithi016 bash[27196]: Error response from daemon: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-mon-a 2023-09-23T18:55:28.721 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:28 smithi016 systemd[1]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@mon.a.service: Succeeded. 2023-09-23T18:55:28.721 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:28 smithi016 systemd[1]: Stopped Ceph mon.a for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7. 2023-09-23T18:55:28.721 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:28 smithi016 systemd[1]: Started Ceph mon.a for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7. 2023-09-23T18:55:30.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.733+0000 7f3892c77880 0 set uid:gid to 167:167 (ceph:ceph) 2023-09-23T18:55:30.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.733+0000 7f3892c77880 0 ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable), process ceph-mon, pid 7 2023-09-23T18:55:30.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.733+0000 7f3892c77880 0 pidfile_write: ignore empty --pid-file 2023-09-23T18:55:30.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 0 load: jerasure load: lrc load: isa 2023-09-23T18:55:30.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: RocksDB version: 6.8.1 2023-09-23T18:55:30.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-09-23T18:55:30.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Compile date Sep 21 2023 2023-09-23T18:55:30.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: DB SUMMARY 2023-09-23T18:55:30.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: CURRENT file: CURRENT 2023-09-23T18:55:30.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: IDENTITY file: IDENTITY 2023-09-23T18:55:30.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: MANIFEST file: MANIFEST-000005 size: 131 Bytes 2023-09-23T18:55:30.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000004.sst 2023-09-23T18:55:30.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000006.log size: 86699 ; 2023-09-23T18:55:30.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.error_if_exists: 0 2023-09-23T18:55:30.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.create_if_missing: 0 2023-09-23T18:55:30.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.paranoid_checks: 1 2023-09-23T18:55:30.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.env: 0x560bff3c1080 2023-09-23T18:55:30.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.fs: Posix File System 2023-09-23T18:55:30.039 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.info_log: 0x560c016fda80 2023-09-23T18:55:30.039 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.max_file_opening_threads: 16 2023-09-23T18:55:30.039 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.statistics: (nil) 2023-09-23T18:55:30.039 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.use_fsync: 0 2023-09-23T18:55:30.039 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.max_log_file_size: 0 2023-09-23T18:55:30.039 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T18:55:30.039 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T18:55:30.039 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.keep_log_file_num: 1000 2023-09-23T18:55:30.039 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.recycle_log_file_num: 0 2023-09-23T18:55:30.040 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.allow_fallocate: 1 2023-09-23T18:55:30.040 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.allow_mmap_reads: 0 2023-09-23T18:55:30.040 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.allow_mmap_writes: 0 2023-09-23T18:55:30.040 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.use_direct_reads: 0 2023-09-23T18:55:30.040 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T18:55:30.040 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.create_missing_column_families: 0 2023-09-23T18:55:30.040 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.db_log_dir: 2023-09-23T18:55:30.040 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2023-09-23T18:55:30.040 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T18:55:30.041 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.max_subcompactions: 1 2023-09-23T18:55:30.041 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.max_background_flushes: -1 2023-09-23T18:55:30.041 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T18:55:30.041 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T18:55:30.041 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T18:55:30.041 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T18:55:30.041 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T18:55:30.041 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.advise_random_on_open: 1 2023-09-23T18:55:30.041 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.db_write_buffer_size: 0 2023-09-23T18:55:30.041 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.write_buffer_manager: 0x560c023fa330 2023-09-23T18:55:30.041 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T18:55:30.042 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-09-23T18:55:30.042 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T18:55:30.042 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T18:55:30.042 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.rate_limiter: (nil) 2023-09-23T18:55:30.042 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T18:55:30.042 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.wal_recovery_mode: 2 2023-09-23T18:55:30.042 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.enable_thread_tracking: 0 2023-09-23T18:55:30.042 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.enable_pipelined_write: 0 2023-09-23T18:55:30.042 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.unordered_write: 0 2023-09-23T18:55:30.042 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T18:55:30.042 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T18:55:30.043 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T18:55:30.043 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T18:55:30.043 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.row_cache: None 2023-09-23T18:55:30.043 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.wal_filter: None 2023-09-23T18:55:30.043 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T18:55:30.043 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.allow_ingest_behind: 0 2023-09-23T18:55:30.043 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.preserve_deletes: 0 2023-09-23T18:55:30.043 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.two_write_queues: 0 2023-09-23T18:55:30.043 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.manual_wal_flush: 0 2023-09-23T18:55:30.043 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.atomic_flush: 0 2023-09-23T18:55:30.043 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T18:55:30.044 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T18:55:30.044 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T18:55:30.044 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.log_readahead_size: 0 2023-09-23T18:55:30.044 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.sst_file_checksum_func: Unknown 2023-09-23T18:55:30.044 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.max_background_jobs: 2 2023-09-23T18:55:30.044 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.max_background_compactions: -1 2023-09-23T18:55:30.044 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T18:55:30.044 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T18:55:30.044 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T18:55:30.044 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.max_total_wal_size: 0 2023-09-23T18:55:30.044 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T18:55:30.044 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T18:55:30.046 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T18:55:30.046 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T18:55:30.046 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.max_open_files: -1 2023-09-23T18:55:30.046 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.bytes_per_sync: 0 2023-09-23T18:55:30.046 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T18:55:30.046 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T18:55:30.046 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.compaction_readahead_size: 0 2023-09-23T18:55:30.046 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Compression algorithms supported: 2023-09-23T18:55:30.046 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T18:55:30.047 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: kZSTD supported: 0 2023-09-23T18:55:30.047 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: kXpressCompression supported: 0 2023-09-23T18:55:30.047 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: kLZ4HCCompression supported: 1 2023-09-23T18:55:30.047 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: kLZ4Compression supported: 1 2023-09-23T18:55:30.047 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: kBZip2Compression supported: 0 2023-09-23T18:55:30.047 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: kZlibCompression supported: 1 2023-09-23T18:55:30.047 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: kSnappyCompression supported: 1 2023-09-23T18:55:30.047 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T18:55:30.047 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000005 2023-09-23T18:55:30.047 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2023-09-23T18:55:30.048 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T18:55:30.048 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.merge_operator: 2023-09-23T18:55:30.048 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.compaction_filter: None 2023-09-23T18:55:30.048 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.compaction_filter_factory: None 2023-09-23T18:55:30.048 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T18:55:30.048 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T18:55:30.048 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x560c016abd18) 2023-09-23T18:55:30.048 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: cache_index_and_filter_blocks: 1 2023-09-23T18:55:30.048 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T18:55:30.048 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T18:55:30.048 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: pin_top_level_index_and_filter: 1 2023-09-23T18:55:30.049 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: index_type: 0 2023-09-23T18:55:30.049 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: data_block_index_type: 0 2023-09-23T18:55:30.049 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: index_shortening: 1 2023-09-23T18:55:30.049 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: data_block_hash_table_util_ratio: 0.750000 2023-09-23T18:55:30.049 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: hash_index_allow_collision: 1 2023-09-23T18:55:30.049 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: checksum: 1 2023-09-23T18:55:30.049 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: no_block_cache: 0 2023-09-23T18:55:30.049 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: block_cache: 0x560c016e2f10 2023-09-23T18:55:30.049 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: block_cache_name: BinnedLRUCache 2023-09-23T18:55:30.049 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: block_cache_options: 2023-09-23T18:55:30.049 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: capacity : 536870912 2023-09-23T18:55:30.049 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: num_shard_bits : 4 2023-09-23T18:55:30.050 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: strict_capacity_limit : 0 2023-09-23T18:55:30.050 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: high_pri_pool_ratio: 0.000 2023-09-23T18:55:30.050 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: block_cache_compressed: (nil) 2023-09-23T18:55:30.050 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: persistent_cache: (nil) 2023-09-23T18:55:30.050 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: block_size: 4096 2023-09-23T18:55:30.050 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: block_size_deviation: 10 2023-09-23T18:55:30.050 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: block_restart_interval: 16 2023-09-23T18:55:30.050 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: index_block_restart_interval: 1 2023-09-23T18:55:30.050 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: metadata_block_size: 4096 2023-09-23T18:55:30.050 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: partition_filters: 0 2023-09-23T18:55:30.050 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: use_delta_encoding: 1 2023-09-23T18:55:30.050 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: filter_policy: rocksdb.BuiltinBloomFilter 2023-09-23T18:55:30.050 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: whole_key_filtering: 1 2023-09-23T18:55:30.051 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: verify_compression: 0 2023-09-23T18:55:30.051 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: read_amp_bytes_per_bit: 0 2023-09-23T18:55:30.051 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: format_version: 2 2023-09-23T18:55:30.051 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: enable_index_compression: 1 2023-09-23T18:55:30.051 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: block_align: 0 2023-09-23T18:55:30.051 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.write_buffer_size: 33554432 2023-09-23T18:55:30.051 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.max_write_buffer_number: 2 2023-09-23T18:55:30.051 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.compression: NoCompression 2023-09-23T18:55:30.051 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.bottommost_compression: Disabled 2023-09-23T18:55:30.051 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.prefix_extractor: nullptr 2023-09-23T18:55:30.051 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T18:55:30.051 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.num_levels: 7 2023-09-23T18:55:30.052 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T18:55:30.052 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T18:55:30.052 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T18:55:30.052 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T18:55:30.052 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T18:55:30.052 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T18:55:30.052 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T18:55:30.052 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T18:55:30.052 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T18:55:30.052 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T18:55:30.052 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.compression_opts.level: 32767 2023-09-23T18:55:30.052 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.compression_opts.strategy: 0 2023-09-23T18:55:30.053 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T18:55:30.053 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T18:55:30.053 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.compression_opts.enabled: false 2023-09-23T18:55:30.053 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T18:55:30.053 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T18:55:30.053 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T18:55:30.053 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.target_file_size_base: 67108864 2023-09-23T18:55:30.053 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T18:55:30.053 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T18:55:30.053 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T18:55:30.053 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T18:55:30.054 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T18:55:30.054 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T18:55:30.054 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T18:55:30.054 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T18:55:30.054 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T18:55:30.054 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T18:55:30.054 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T18:55:30.054 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T18:55:30.054 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T18:55:30.054 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.arena_block_size: 4194304 2023-09-23T18:55:30.054 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T18:55:30.054 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T18:55:30.055 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-09-23T18:55:30.055 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.disable_auto_compactions: 0 2023-09-23T18:55:30.055 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T18:55:30.055 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T18:55:30.055 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T18:55:30.055 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T18:55:30.055 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T18:55:30.055 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T18:55:30.055 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T18:55:30.055 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T18:55:30.055 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T18:55:30.055 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T18:55:30.056 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.table_properties_collectors: 2023-09-23T18:55:30.056 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.inplace_update_support: 0 2023-09-23T18:55:30.056 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T18:55:30.056 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T18:55:30.056 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T18:55:30.056 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T18:55:30.056 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.bloom_locality: 0 2023-09-23T18:55:30.056 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.max_successive_merges: 0 2023-09-23T18:55:30.056 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T18:55:30.056 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.paranoid_file_checks: 0 2023-09-23T18:55:30.056 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.force_consistency_checks: 0 2023-09-23T18:55:30.056 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.report_bg_io_stats: 0 2023-09-23T18:55:30.057 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.ttl: 2592000 2023-09-23T18:55:30.057 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.737+0000 7f3892c77880 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T18:55:30.057 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.741+0000 7f3892c77880 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 5, log_number is 4,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T18:55:30.057 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.741+0000 7f3892c77880 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 4 2023-09-23T18:55:30.057 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.741+0000 7f3892c77880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1695495329744968, "job": 1, "event": "recovery_started", "log_files": [6]} 2023-09-23T18:55:30.057 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.741+0000 7f3892c77880 4 rocksdb: [db_impl/db_impl_open.cc:760] Recovering log #6 mode 2 2023-09-23T18:55:30.057 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.741+0000 7f3892c77880 3 rocksdb: [le/block_based/filter_policy.cc:584] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-09-23T18:55:30.057 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.741+0000 7f3892c77880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1695495329746215, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 7, "file_size": 83299, "table_properties": {"data_size": 81572, "index_size": 332, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 0, "index_value_is_delta_encoded": 0, "filter_size": 581, "raw_key_size": 9829, "raw_average_key_size": 47, "raw_value_size": 75893, "raw_average_value_size": 368, "num_data_blocks": 11, "num_entries": 206, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1695495329, "oldest_key_time": 3, "file_creation_time": 0}} 2023-09-23T18:55:30.057 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.741+0000 7f3892c77880 4 rocksdb: [version_set.cc:3826] Creating manifest 8 2023-09-23T18:55:30.057 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.741+0000 7f3892c77880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1695495329746773, "job": 1, "event": "recovery_finished"} 2023-09-23T18:55:30.057 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.741+0000 7f3892c77880 4 rocksdb: DB pointer 0x560c017ab800 2023-09-23T18:55:30.057 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.741+0000 7f3879a7f700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2023-09-23T18:55:30.058 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.741+0000 7f3879a7f700 4 rocksdb: [db_impl/db_impl.cc:851] 2023-09-23T18:55:30.058 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: ** DB Stats ** 2023-09-23T18:55:30.058 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T18:55:30.058 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T18:55:30.058 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T18:55:30.058 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T18:55:30.058 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T18:55:30.058 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-09-23T18:55:30.058 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T18:55:30.058 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: ** Compaction Stats [default] ** 2023-09-23T18:55:30.058 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T18:55:30.059 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T18:55:30.059 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: L0 2/0 83.00 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 101.6 0.00 0.00 1 0.001 0 0 2023-09-23T18:55:30.059 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: Sum 2/0 83.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 101.6 0.00 0.00 1 0.001 0 0 2023-09-23T18:55:30.059 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 101.6 0.00 0.00 1 0.001 0 0 2023-09-23T18:55:30.059 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: ** Compaction Stats [default] ** 2023-09-23T18:55:30.059 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T18:55:30.059 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T18:55:30.059 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 101.6 0.00 0.00 1 0.001 0 0 2023-09-23T18:55:30.059 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T18:55:30.059 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T18:55:30.059 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T18:55:30.059 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T18:55:30.060 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T18:55:30.060 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: AddFile(Keys): cumulative 0, interval 0 2023-09-23T18:55:30.060 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: Cumulative compaction: 0.00 GB write, 18.24 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T18:55:30.060 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: Interval compaction: 0.00 GB write, 18.24 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T18:55:30.060 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T18:55:30.060 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: ** File Read Latency Histogram By Level [default] ** 2023-09-23T18:55:30.060 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: ** Compaction Stats [default] ** 2023-09-23T18:55:30.060 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T18:55:30.060 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T18:55:30.060 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: L0 2/0 83.00 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 101.6 0.00 0.00 1 0.001 0 0 2023-09-23T18:55:30.060 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: Sum 2/0 83.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 101.6 0.00 0.00 1 0.001 0 0 2023-09-23T18:55:30.060 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-09-23T18:55:30.061 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: ** Compaction Stats [default] ** 2023-09-23T18:55:30.061 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T18:55:30.061 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T18:55:30.061 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 101.6 0.00 0.00 1 0.001 0 0 2023-09-23T18:55:30.061 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T18:55:30.061 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T18:55:30.061 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T18:55:30.061 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T18:55:30.061 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T18:55:30.061 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: AddFile(Keys): cumulative 0, interval 0 2023-09-23T18:55:30.061 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: Cumulative compaction: 0.00 GB write, 17.90 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T18:55:30.062 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T18:55:30.062 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T18:55:30.062 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: ** File Read Latency Histogram By Level [default] ** 2023-09-23T18:55:30.062 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.745+0000 7f3892c77880 0 starting mon.a rank 0 at public addrs [v2:172.21.15.16:3300/0,v1:172.21.15.16:6789/0] at bind addrs [v2:172.21.15.16:3300/0,v1:172.21.15.16:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 2023-09-23T18:55:30.062 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.745+0000 7f3892c77880 1 mon.a@-1(???) e1 preinit fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 2023-09-23T18:55:30.062 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.745+0000 7f3892c77880 0 mon.a@-1(???).mds e1 new map 2023-09-23T18:55:30.062 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.745+0000 7f3892c77880 0 mon.a@-1(???).mds e1 print_map 2023-09-23T18:55:30.062 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: e1 2023-09-23T18:55:30.062 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T18:55:30.062 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T18:55:30.062 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: legacy client fscid: -1 2023-09-23T18:55:30.063 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: 2023-09-23T18:55:30.063 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: No filesystems configured 2023-09-23T18:55:30.063 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.745+0000 7f3892c77880 0 mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2023-09-23T18:55:30.063 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.745+0000 7f3892c77880 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T18:55:30.063 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.745+0000 7f3892c77880 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T18:55:30.063 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.745+0000 7f3892c77880 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T18:55:30.063 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: audit 2023-09-23T18:55:27.286313+0000 mon.a (mon.0) 10 : audit [DBG] from='client.? 172.21.15.16:0/1711080087' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:55:30.063 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:29 smithi016 bash[27237]: debug 2023-09-23T18:55:29.745+0000 7f3892c77880 1 mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2023-09-23T18:55:30.659 INFO:teuthology.orchestra.run.smithi016.stderr:Wrote config to /etc/ceph/ceph.conf 2023-09-23T18:55:30.660 INFO:teuthology.orchestra.run.smithi016.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2023-09-23T18:55:30.660 INFO:teuthology.orchestra.run.smithi016.stderr:Creating mgr... 2023-09-23T18:55:30.660 INFO:teuthology.orchestra.run.smithi016.stderr:Verifying port 9283 ... 2023-09-23T18:55:30.893 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: Failed to reset failed state of unit ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@mgr.y.service: Unit ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@mgr.y.service not loaded. 2023-09-23T18:55:30.901 INFO:teuthology.orchestra.run.smithi016.stderr:systemctl: Created symlink /etc/systemd/system/ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7.target.wants/ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@mgr.y.service -> /etc/systemd/system/ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@.service. 2023-09-23T18:55:31.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:30 smithi016 bash[27237]: cluster 2023-09-23T18:55:29.773102+0000 mon.a (mon.0) 1 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2023-09-23T18:55:31.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:30 smithi016 bash[27237]: cluster 2023-09-23T18:55:29.773226+0000 mon.a (mon.0) 2 : cluster [DBG] monmap e1: 1 mons at {a=[v2:172.21.15.16:3300/0,v1:172.21.15.16:6789/0]} removed_ranks: {} 2023-09-23T18:55:31.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:30 smithi016 bash[27237]: cluster 2023-09-23T18:55:29.774251+0000 mon.a (mon.0) 3 : cluster [DBG] fsmap 2023-09-23T18:55:31.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:30 smithi016 bash[27237]: cluster 2023-09-23T18:55:29.774327+0000 mon.a (mon.0) 4 : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2023-09-23T18:55:31.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:30 smithi016 bash[27237]: cluster 2023-09-23T18:55:29.775124+0000 mon.a (mon.0) 5 : cluster [DBG] mgrmap e1: no daemons active 2023-09-23T18:55:31.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:30 smithi016 bash[27237]: audit 2023-09-23T18:55:30.008660+0000 mon.a (mon.0) 6 : audit [INF] from='client.? 172.21.15.16:0/767608337' entity='client.admin' 2023-09-23T18:55:31.111 INFO:teuthology.orchestra.run.smithi016.stderr:firewalld does not appear to be present 2023-09-23T18:55:31.111 INFO:teuthology.orchestra.run.smithi016.stderr:Not possible to enable service . firewalld.service is not available 2023-09-23T18:55:31.112 INFO:teuthology.orchestra.run.smithi016.stderr:firewalld does not appear to be present 2023-09-23T18:55:31.112 INFO:teuthology.orchestra.run.smithi016.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2023-09-23T18:55:31.112 INFO:teuthology.orchestra.run.smithi016.stderr:Waiting for mgr to start... 2023-09-23T18:55:31.112 INFO:teuthology.orchestra.run.smithi016.stderr:Waiting for mgr... 2023-09-23T18:55:32.518 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: 2023-09-23T18:55:32.518 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: { 2023-09-23T18:55:32.518 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "fsid": "95ce9eb6-5a42-11ee-8db4-212e2dc638e7", 2023-09-23T18:55:32.518 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "health": { 2023-09-23T18:55:32.519 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-09-23T18:55:32.519 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "checks": {}, 2023-09-23T18:55:32.519 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "mutes": [] 2023-09-23T18:55:32.519 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: }, 2023-09-23T18:55:32.520 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-09-23T18:55:32.520 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "quorum": [ 2023-09-23T18:55:32.520 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: 0 2023-09-23T18:55:32.520 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: ], 2023-09-23T18:55:32.520 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "quorum_names": [ 2023-09-23T18:55:32.520 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "a" 2023-09-23T18:55:32.521 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: ], 2023-09-23T18:55:32.521 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "quorum_age": 2, 2023-09-23T18:55:32.521 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "monmap": { 2023-09-23T18:55:32.522 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "epoch": 1, 2023-09-23T18:55:32.522 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-09-23T18:55:32.522 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_mons": 1 2023-09-23T18:55:32.522 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: }, 2023-09-23T18:55:32.522 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "osdmap": { 2023-09-23T18:55:32.522 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "epoch": 1, 2023-09-23T18:55:32.522 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_osds": 0, 2023-09-23T18:55:32.522 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-09-23T18:55:32.522 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-09-23T18:55:32.522 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-09-23T18:55:32.526 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-09-23T18:55:32.526 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-09-23T18:55:32.526 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: }, 2023-09-23T18:55:32.526 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "pgmap": { 2023-09-23T18:55:32.526 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-09-23T18:55:32.526 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-09-23T18:55:32.526 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_pools": 0, 2023-09-23T18:55:32.526 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_objects": 0, 2023-09-23T18:55:32.526 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-09-23T18:55:32.526 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-09-23T18:55:32.527 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-09-23T18:55:32.527 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "bytes_total": 0 2023-09-23T18:55:32.527 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: }, 2023-09-23T18:55:32.527 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "fsmap": { 2023-09-23T18:55:32.527 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "epoch": 1, 2023-09-23T18:55:32.527 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "by_rank": [], 2023-09-23T18:55:32.527 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "up:standby": 0 2023-09-23T18:55:32.527 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: }, 2023-09-23T18:55:32.527 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "mgrmap": { 2023-09-23T18:55:32.527 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "available": false, 2023-09-23T18:55:32.527 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-09-23T18:55:32.527 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "modules": [ 2023-09-23T18:55:32.527 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "iostat", 2023-09-23T18:55:32.528 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "nfs", 2023-09-23T18:55:32.528 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "restful" 2023-09-23T18:55:32.528 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: ], 2023-09-23T18:55:32.528 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "services": {} 2023-09-23T18:55:32.528 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: }, 2023-09-23T18:55:32.528 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "servicemap": { 2023-09-23T18:55:32.528 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "epoch": 1, 2023-09-23T18:55:32.528 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "modified": "2023-09-23T18:55:23.803637+0000", 2023-09-23T18:55:32.528 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "services": {} 2023-09-23T18:55:32.528 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: }, 2023-09-23T18:55:32.528 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "progress_events": {} 2023-09-23T18:55:32.528 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: } 2023-09-23T18:55:32.694 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:32 smithi016 bash[27237]: audit 2023-09-23T18:55:32.512687+0000 mon.a (mon.0) 7 : audit [DBG] from='client.? 172.21.15.16:0/2241777562' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-09-23T18:55:32.695 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:32 smithi016 bash[27503]: debug 2023-09-23T18:55:32.525+0000 7f9ccbf28680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T18:55:33.108 INFO:teuthology.orchestra.run.smithi016.stderr:mgr not available, waiting (1/15)... 2023-09-23T18:55:33.286 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:33 smithi016 bash[27503]: debug 2023-09-23T18:55:33.025+0000 7f9ccbf28680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T18:55:34.286 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:34 smithi016 bash[27503]: debug 2023-09-23T18:55:34.001+0000 7f9ccbf28680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T18:55:34.286 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:34 smithi016 bash[27503]: debug 2023-09-23T18:55:34.101+0000 7f9ccbf28680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T18:55:34.786 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:34 smithi016 bash[27503]: debug 2023-09-23T18:55:34.361+0000 7f9ccbf28680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T18:55:35.810 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:35 smithi016 bash[27503]: debug 2023-09-23T18:55:35.537+0000 7f9ccbf28680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T18:55:35.810 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:35 smithi016 bash[27503]: debug 2023-09-23T18:55:35.705+0000 7f9ccbf28680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T18:55:36.147 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: 2023-09-23T18:55:36.148 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: { 2023-09-23T18:55:36.148 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "fsid": "95ce9eb6-5a42-11ee-8db4-212e2dc638e7", 2023-09-23T18:55:36.148 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "health": { 2023-09-23T18:55:36.148 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-09-23T18:55:36.148 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "checks": {}, 2023-09-23T18:55:36.148 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "mutes": [] 2023-09-23T18:55:36.148 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: }, 2023-09-23T18:55:36.148 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-09-23T18:55:36.148 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "quorum": [ 2023-09-23T18:55:36.148 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: 0 2023-09-23T18:55:36.148 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: ], 2023-09-23T18:55:36.149 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "quorum_names": [ 2023-09-23T18:55:36.149 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "a" 2023-09-23T18:55:36.149 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: ], 2023-09-23T18:55:36.149 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "quorum_age": 6, 2023-09-23T18:55:36.149 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "monmap": { 2023-09-23T18:55:36.149 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "epoch": 1, 2023-09-23T18:55:36.149 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-09-23T18:55:36.149 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_mons": 1 2023-09-23T18:55:36.149 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: }, 2023-09-23T18:55:36.150 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "osdmap": { 2023-09-23T18:55:36.150 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "epoch": 1, 2023-09-23T18:55:36.151 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_osds": 0, 2023-09-23T18:55:36.151 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-09-23T18:55:36.151 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-09-23T18:55:36.151 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-09-23T18:55:36.151 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-09-23T18:55:36.151 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-09-23T18:55:36.151 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: }, 2023-09-23T18:55:36.151 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "pgmap": { 2023-09-23T18:55:36.154 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-09-23T18:55:36.154 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-09-23T18:55:36.154 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_pools": 0, 2023-09-23T18:55:36.154 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_objects": 0, 2023-09-23T18:55:36.155 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-09-23T18:55:36.155 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-09-23T18:55:36.155 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-09-23T18:55:36.155 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "bytes_total": 0 2023-09-23T18:55:36.155 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: }, 2023-09-23T18:55:36.155 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "fsmap": { 2023-09-23T18:55:36.155 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "epoch": 1, 2023-09-23T18:55:36.155 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "by_rank": [], 2023-09-23T18:55:36.155 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "up:standby": 0 2023-09-23T18:55:36.155 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: }, 2023-09-23T18:55:36.155 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "mgrmap": { 2023-09-23T18:55:36.155 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "available": false, 2023-09-23T18:55:36.156 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-09-23T18:55:36.156 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "modules": [ 2023-09-23T18:55:36.156 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "iostat", 2023-09-23T18:55:36.156 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "nfs", 2023-09-23T18:55:36.156 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "restful" 2023-09-23T18:55:36.156 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: ], 2023-09-23T18:55:36.156 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "services": {} 2023-09-23T18:55:36.156 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: }, 2023-09-23T18:55:36.156 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "servicemap": { 2023-09-23T18:55:36.156 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "epoch": 1, 2023-09-23T18:55:36.156 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "modified": "2023-09-23T18:55:23.803637+0000", 2023-09-23T18:55:36.157 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "services": {} 2023-09-23T18:55:36.157 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: }, 2023-09-23T18:55:36.157 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "progress_events": {} 2023-09-23T18:55:36.157 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: } 2023-09-23T18:55:36.158 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:35 smithi016 bash[27503]: debug 2023-09-23T18:55:35.801+0000 7f9ccbf28680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T18:55:36.158 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:35 smithi016 bash[27503]: debug 2023-09-23T18:55:35.921+0000 7f9ccbf28680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T18:55:36.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:36 smithi016 bash[27237]: audit 2023-09-23T18:55:36.141011+0000 mon.a (mon.0) 8 : audit [DBG] from='client.? 172.21.15.16:0/2316380450' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-09-23T18:55:36.536 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:36 smithi016 bash[27503]: debug 2023-09-23T18:55:36.149+0000 7f9ccbf28680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T18:55:36.671 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:36 smithi016 bash[27503]: debug 2023-09-23T18:55:36.537+0000 7f9ccbf28680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T18:55:36.846 INFO:teuthology.orchestra.run.smithi016.stderr:mgr not available, waiting (2/15)... 2023-09-23T18:55:37.036 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:36 smithi016 bash[27503]: debug 2023-09-23T18:55:36.661+0000 7f9ccbf28680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T18:55:38.029 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:37 smithi016 bash[27503]: debug 2023-09-23T18:55:37.561+0000 7f9ccbf28680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T18:55:38.029 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:37 smithi016 bash[27503]: debug 2023-09-23T18:55:37.669+0000 7f9ccbf28680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T18:55:38.029 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:37 smithi016 bash[27503]: debug 2023-09-23T18:55:37.789+0000 7f9ccbf28680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T18:55:38.030 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:38 smithi016 bash[27503]: debug 2023-09-23T18:55:38.021+0000 7f9ccbf28680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T18:55:38.673 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:38 smithi016 bash[27503]: debug 2023-09-23T18:55:38.317+0000 7f9ccbf28680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T18:55:38.673 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:38 smithi016 bash[27503]: debug 2023-09-23T18:55:38.489+0000 7f9ccbf28680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T18:55:39.000 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:38 smithi016 bash[27503]: debug 2023-09-23T18:55:38.665+0000 7f9ccbf28680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T18:55:39.001 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:38 smithi016 bash[27503]: debug 2023-09-23T18:55:38.889+0000 7f9ccbf28680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T18:55:39.286 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:38 smithi016 bash[27503]: debug 2023-09-23T18:55:38.993+0000 7f9ccbf28680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T18:55:39.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:39 smithi016 bash[27237]: cluster 2023-09-23T18:55:39.017914+0000 mon.a (mon.0) 9 : cluster [INF] Activating manager daemon y 2023-09-23T18:55:39.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:39 smithi016 bash[27237]: cluster 2023-09-23T18:55:39.024568+0000 mon.a (mon.0) 10 : cluster [DBG] mgrmap e2: y(active, starting, since 0.00674411s) 2023-09-23T18:55:39.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:39 smithi016 bash[27237]: audit 2023-09-23T18:55:39.026046+0000 mon.a (mon.0) 11 : audit [DBG] from='mgr.14100 172.21.15.16:0/1558931977' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T18:55:39.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:39 smithi016 bash[27237]: audit 2023-09-23T18:55:39.026255+0000 mon.a (mon.0) 12 : audit [DBG] from='mgr.14100 172.21.15.16:0/1558931977' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T18:55:39.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:39 smithi016 bash[27237]: audit 2023-09-23T18:55:39.026456+0000 mon.a (mon.0) 13 : audit [DBG] from='mgr.14100 172.21.15.16:0/1558931977' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T18:55:39.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:39 smithi016 bash[27237]: audit 2023-09-23T18:55:39.026650+0000 mon.a (mon.0) 14 : audit [DBG] from='mgr.14100 172.21.15.16:0/1558931977' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T18:55:39.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:39 smithi016 bash[27237]: audit 2023-09-23T18:55:39.027480+0000 mon.a (mon.0) 15 : audit [DBG] from='mgr.14100 172.21.15.16:0/1558931977' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T18:55:39.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:39 smithi016 bash[27237]: cluster 2023-09-23T18:55:39.051761+0000 mon.a (mon.0) 16 : cluster [INF] Manager daemon y is now available 2023-09-23T18:55:40.040 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: 2023-09-23T18:55:40.040 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: { 2023-09-23T18:55:40.040 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "fsid": "95ce9eb6-5a42-11ee-8db4-212e2dc638e7", 2023-09-23T18:55:40.041 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "health": { 2023-09-23T18:55:40.041 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-09-23T18:55:40.041 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "checks": {}, 2023-09-23T18:55:40.041 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "mutes": [] 2023-09-23T18:55:40.041 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: }, 2023-09-23T18:55:40.041 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-09-23T18:55:40.041 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "quorum": [ 2023-09-23T18:55:40.041 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: 0 2023-09-23T18:55:40.041 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: ], 2023-09-23T18:55:40.041 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "quorum_names": [ 2023-09-23T18:55:40.041 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "a" 2023-09-23T18:55:40.041 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: ], 2023-09-23T18:55:40.042 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "quorum_age": 10, 2023-09-23T18:55:40.042 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "monmap": { 2023-09-23T18:55:40.042 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "epoch": 1, 2023-09-23T18:55:40.042 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-09-23T18:55:40.042 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_mons": 1 2023-09-23T18:55:40.042 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: }, 2023-09-23T18:55:40.042 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "osdmap": { 2023-09-23T18:55:40.042 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "epoch": 1, 2023-09-23T18:55:40.042 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_osds": 0, 2023-09-23T18:55:40.042 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-09-23T18:55:40.042 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-09-23T18:55:40.043 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-09-23T18:55:40.043 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-09-23T18:55:40.043 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-09-23T18:55:40.043 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: }, 2023-09-23T18:55:40.043 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "pgmap": { 2023-09-23T18:55:40.043 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-09-23T18:55:40.043 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-09-23T18:55:40.043 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_pools": 0, 2023-09-23T18:55:40.043 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_objects": 0, 2023-09-23T18:55:40.043 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-09-23T18:55:40.043 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-09-23T18:55:40.043 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-09-23T18:55:40.044 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "bytes_total": 0 2023-09-23T18:55:40.044 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: }, 2023-09-23T18:55:40.044 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "fsmap": { 2023-09-23T18:55:40.044 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "epoch": 1, 2023-09-23T18:55:40.044 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "by_rank": [], 2023-09-23T18:55:40.044 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "up:standby": 0 2023-09-23T18:55:40.044 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: }, 2023-09-23T18:55:40.044 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "mgrmap": { 2023-09-23T18:55:40.044 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "available": true, 2023-09-23T18:55:40.044 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-09-23T18:55:40.044 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "modules": [ 2023-09-23T18:55:40.044 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "iostat", 2023-09-23T18:55:40.045 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "nfs", 2023-09-23T18:55:40.045 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "restful" 2023-09-23T18:55:40.045 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: ], 2023-09-23T18:55:40.045 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "services": {} 2023-09-23T18:55:40.045 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: }, 2023-09-23T18:55:40.045 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "servicemap": { 2023-09-23T18:55:40.045 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "epoch": 1, 2023-09-23T18:55:40.045 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "modified": "2023-09-23T18:55:23.803637+0000", 2023-09-23T18:55:40.045 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "services": {} 2023-09-23T18:55:40.045 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: }, 2023-09-23T18:55:40.045 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "progress_events": {} 2023-09-23T18:55:40.046 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: } 2023-09-23T18:55:40.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:40 smithi016 bash[27237]: audit 2023-09-23T18:55:39.068600+0000 mon.a (mon.0) 17 : audit [INF] from='mgr.14100 172.21.15.16:0/1558931977' entity='mgr.y' 2023-09-23T18:55:40.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:40 smithi016 bash[27237]: audit 2023-09-23T18:55:39.075295+0000 mon.a (mon.0) 18 : audit [INF] from='mgr.14100 172.21.15.16:0/1558931977' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T18:55:40.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:40 smithi016 bash[27237]: audit 2023-09-23T18:55:39.080454+0000 mon.a (mon.0) 19 : audit [INF] from='mgr.14100 172.21.15.16:0/1558931977' entity='mgr.y' 2023-09-23T18:55:40.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:40 smithi016 bash[27237]: audit 2023-09-23T18:55:39.080643+0000 mon.a (mon.0) 20 : audit [INF] from='mgr.14100 172.21.15.16:0/1558931977' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T18:55:40.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:40 smithi016 bash[27237]: audit 2023-09-23T18:55:39.082867+0000 mon.a (mon.0) 21 : audit [INF] from='mgr.14100 172.21.15.16:0/1558931977' entity='mgr.y' 2023-09-23T18:55:40.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:40 smithi016 bash[27237]: cluster 2023-09-23T18:55:40.029013+0000 mon.a (mon.0) 22 : cluster [DBG] mgrmap e3: y(active, since 1.01119s) 2023-09-23T18:55:40.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:40 smithi016 bash[27237]: audit 2023-09-23T18:55:40.031721+0000 mon.a (mon.0) 23 : audit [DBG] from='client.? 172.21.15.16:0/2505585307' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-09-23T18:55:40.620 INFO:teuthology.orchestra.run.smithi016.stderr:mgr is available 2023-09-23T18:55:41.861 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: 2023-09-23T18:55:41.862 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: [global] 2023-09-23T18:55:41.862 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: fsid = 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 2023-09-23T18:55:41.863 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: log_to_journald = false 2023-09-23T18:55:41.863 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2023-09-23T18:55:41.863 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2023-09-23T18:55:41.863 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2023-09-23T18:55:41.863 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2023-09-23T18:55:41.863 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: 2023-09-23T18:55:41.863 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: [mgr] 2023-09-23T18:55:41.864 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2023-09-23T18:55:41.864 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: 2023-09-23T18:55:41.864 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: [osd] 2023-09-23T18:55:41.864 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2023-09-23T18:55:41.864 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2023-09-23T18:55:42.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:42 smithi016 bash[27237]: cluster 2023-09-23T18:55:41.076683+0000 mon.a (mon.0) 24 : cluster [DBG] mgrmap e4: y(active, since 2s) 2023-09-23T18:55:42.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:42 smithi016 bash[27237]: audit 2023-09-23T18:55:41.856486+0000 mon.a (mon.0) 25 : audit [INF] from='client.? 172.21.15.16:0/614782819' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-09-23T18:55:42.383 INFO:teuthology.orchestra.run.smithi016.stderr:Enabling cephadm module... 2023-09-23T18:55:44.112 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:43 smithi016 bash[27237]: audit 2023-09-23T18:55:43.737478+0000 mon.a (mon.0) 26 : audit [INF] from='client.? 172.21.15.16:0/2155467532' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2023-09-23T18:55:44.112 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:43 smithi016 bash[27503]: ignoring --setuser ceph since I am not root 2023-09-23T18:55:44.112 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:43 smithi016 bash[27503]: ignoring --setgroup ceph since I am not root 2023-09-23T18:55:44.112 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:43 smithi016 bash[27503]: debug 2023-09-23T18:55:43.993+0000 7f124b833680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T18:55:44.785 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:44 smithi016 bash[27503]: debug 2023-09-23T18:55:44.105+0000 7f124b833680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T18:55:44.785 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:44 smithi016 bash[27503]: debug 2023-09-23T18:55:44.577+0000 7f124b833680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T18:55:45.097 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:44 smithi016 bash[27237]: audit 2023-09-23T18:55:43.793765+0000 mon.a (mon.0) 27 : audit [INF] from='client.? 172.21.15.16:0/2155467532' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2023-09-23T18:55:45.097 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:44 smithi016 bash[27237]: cluster 2023-09-23T18:55:43.793867+0000 mon.a (mon.0) 28 : cluster [DBG] mgrmap e5: y(active, since 4s) 2023-09-23T18:55:45.785 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:45 smithi016 bash[27503]: debug 2023-09-23T18:55:45.525+0000 7f124b833680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T18:55:45.785 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:45 smithi016 bash[27503]: debug 2023-09-23T18:55:45.629+0000 7f124b833680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T18:55:45.802 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: { 2023-09-23T18:55:45.803 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "epoch": 5, 2023-09-23T18:55:45.803 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "available": true, 2023-09-23T18:55:45.803 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "active_name": "y", 2023-09-23T18:55:45.803 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_standby": 0 2023-09-23T18:55:45.803 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: } 2023-09-23T18:55:46.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:45 smithi016 bash[27237]: audit 2023-09-23T18:55:45.798916+0000 mon.a (mon.0) 29 : audit [DBG] from='client.? 172.21.15.16:0/2355180201' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-09-23T18:55:46.285 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:45 smithi016 bash[27503]: debug 2023-09-23T18:55:45.889+0000 7f124b833680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T18:55:46.379 INFO:teuthology.orchestra.run.smithi016.stderr:Waiting for the mgr to restart... 2023-09-23T18:55:46.379 INFO:teuthology.orchestra.run.smithi016.stderr:Waiting for mgr epoch 5... 2023-09-23T18:55:47.357 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:47 smithi016 bash[27503]: debug 2023-09-23T18:55:47.065+0000 7f124b833680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T18:55:47.357 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:47 smithi016 bash[27503]: debug 2023-09-23T18:55:47.245+0000 7f124b833680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T18:55:47.698 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:47 smithi016 bash[27503]: debug 2023-09-23T18:55:47.349+0000 7f124b833680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T18:55:47.698 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:47 smithi016 bash[27503]: debug 2023-09-23T18:55:47.465+0000 7f124b833680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T18:55:48.035 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:47 smithi016 bash[27503]: debug 2023-09-23T18:55:47.689+0000 7f124b833680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T18:55:48.535 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:48 smithi016 bash[27503]: debug 2023-09-23T18:55:48.073+0000 7f124b833680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T18:55:48.535 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:48 smithi016 bash[27503]: debug 2023-09-23T18:55:48.197+0000 7f124b833680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T18:55:49.535 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:49 smithi016 bash[27503]: debug 2023-09-23T18:55:49.089+0000 7f124b833680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T18:55:49.535 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:49 smithi016 bash[27503]: debug 2023-09-23T18:55:49.193+0000 7f124b833680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T18:55:49.535 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:49 smithi016 bash[27503]: debug 2023-09-23T18:55:49.317+0000 7f124b833680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T18:55:49.849 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:49 smithi016 bash[27503]: debug 2023-09-23T18:55:49.545+0000 7f124b833680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T18:55:50.199 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:49 smithi016 bash[27503]: debug 2023-09-23T18:55:49.841+0000 7f124b833680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T18:55:50.199 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:50 smithi016 bash[27503]: debug 2023-09-23T18:55:50.013+0000 7f124b833680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T18:55:50.534 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:50 smithi016 bash[27503]: debug 2023-09-23T18:55:50.193+0000 7f124b833680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T18:55:50.534 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:50 smithi016 bash[27503]: debug 2023-09-23T18:55:50.417+0000 7f124b833680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T18:55:50.534 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:50 smithi016 bash[27503]: debug 2023-09-23T18:55:50.525+0000 7f124b833680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T18:55:50.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:50 smithi016 bash[27237]: cluster 2023-09-23T18:55:50.550510+0000 mon.a (mon.0) 30 : cluster [INF] Active manager daemon y restarted 2023-09-23T18:55:50.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:50 smithi016 bash[27237]: cluster 2023-09-23T18:55:50.550941+0000 mon.a (mon.0) 31 : cluster [INF] Activating manager daemon y 2023-09-23T18:55:50.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:50 smithi016 bash[27237]: cluster 2023-09-23T18:55:50.556434+0000 mon.a (mon.0) 32 : cluster [DBG] osdmap e2: 0 total, 0 up, 0 in 2023-09-23T18:55:50.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:50 smithi016 bash[27237]: cluster 2023-09-23T18:55:50.556627+0000 mon.a (mon.0) 33 : cluster [DBG] mgrmap e6: y(active, starting, since 0.00584119s) 2023-09-23T18:55:50.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:50 smithi016 bash[27237]: audit 2023-09-23T18:55:50.558745+0000 mon.a (mon.0) 34 : audit [DBG] from='mgr.14116 172.21.15.16:0/2443957845' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T18:55:50.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:50 smithi016 bash[27237]: audit 2023-09-23T18:55:50.559806+0000 mon.a (mon.0) 35 : audit [DBG] from='mgr.14116 172.21.15.16:0/2443957845' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T18:55:50.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:50 smithi016 bash[27237]: audit 2023-09-23T18:55:50.560802+0000 mon.a (mon.0) 36 : audit [DBG] from='mgr.14116 172.21.15.16:0/2443957845' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T18:55:50.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:50 smithi016 bash[27237]: audit 2023-09-23T18:55:50.561010+0000 mon.a (mon.0) 37 : audit [DBG] from='mgr.14116 172.21.15.16:0/2443957845' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T18:55:50.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:50 smithi016 bash[27237]: audit 2023-09-23T18:55:50.561211+0000 mon.a (mon.0) 38 : audit [DBG] from='mgr.14116 172.21.15.16:0/2443957845' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T18:55:50.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:50 smithi016 bash[27237]: cluster 2023-09-23T18:55:50.564253+0000 mon.a (mon.0) 39 : cluster [INF] Manager daemon y is now available 2023-09-23T18:55:50.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:50 smithi016 bash[27237]: audit 2023-09-23T18:55:50.583896+0000 mon.a (mon.0) 40 : audit [INF] from='mgr.14116 172.21.15.16:0/2443957845' entity='mgr.y' 2023-09-23T18:55:50.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:50 smithi016 bash[27237]: audit 2023-09-23T18:55:50.707446+0000 mon.a (mon.0) 41 : audit [INF] from='mgr.14116 172.21.15.16:0/2443957845' entity='mgr.y' 2023-09-23T18:55:50.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:50 smithi016 bash[27237]: audit 2023-09-23T18:55:50.709345+0000 mon.a (mon.0) 42 : audit [DBG] from='mgr.14116 172.21.15.16:0/2443957845' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:55:50.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:50 smithi016 bash[27237]: audit 2023-09-23T18:55:50.717377+0000 mon.a (mon.0) 43 : audit [DBG] from='mgr.14116 172.21.15.16:0/2443957845' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:55:50.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:50 smithi016 bash[27237]: audit 2023-09-23T18:55:50.720692+0000 mon.a (mon.0) 44 : audit [DBG] from='mgr.14116 172.21.15.16:0/2443957845' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:55:50.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:50 smithi016 bash[27237]: audit 2023-09-23T18:55:50.737317+0000 mon.a (mon.0) 45 : audit [INF] from='mgr.14116 172.21.15.16:0/2443957845' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T18:55:50.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:50 smithi016 bash[27237]: audit 2023-09-23T18:55:50.740778+0000 mon.a (mon.0) 46 : audit [INF] from='mgr.14116 172.21.15.16:0/2443957845' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T18:55:51.578 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: { 2023-09-23T18:55:51.578 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2023-09-23T18:55:51.578 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "initialized": true 2023-09-23T18:55:51.578 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: } 2023-09-23T18:55:52.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:51 smithi016 bash[27237]: cephadm 2023-09-23T18:55:50.578429+0000 mgr.y (mgr.14116) 1 : cephadm [INF] Found migration_current of "None". Setting to last migration. 2023-09-23T18:55:52.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:51 smithi016 bash[27237]: cluster 2023-09-23T18:55:51.560579+0000 mon.a (mon.0) 47 : cluster [DBG] mgrmap e7: y(active, since 1.00979s) 2023-09-23T18:55:52.199 INFO:teuthology.orchestra.run.smithi016.stderr:mgr epoch 5 is available 2023-09-23T18:55:52.199 INFO:teuthology.orchestra.run.smithi016.stderr:Setting orchestrator backend to cephadm... 2023-09-23T18:55:53.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:52 smithi016 bash[27237]: audit 2023-09-23T18:55:51.560335+0000 mgr.y (mgr.14116) 2 : audit [DBG] from='client.14120 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-09-23T18:55:53.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:52 smithi016 bash[27237]: audit 2023-09-23T18:55:51.574522+0000 mgr.y (mgr.14116) 3 : audit [DBG] from='client.14120 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-09-23T18:55:54.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:53 smithi016 bash[27237]: cluster 2023-09-23T18:55:52.768653+0000 mon.a (mon.0) 48 : cluster [DBG] mgrmap e8: y(active, since 2s) 2023-09-23T18:55:54.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:53 smithi016 bash[27237]: audit 2023-09-23T18:55:53.488896+0000 mon.a (mon.0) 49 : audit [INF] from='mgr.14116 172.21.15.16:0/2443957845' entity='mgr.y' 2023-09-23T18:55:54.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:53 smithi016 bash[27237]: audit 2023-09-23T18:55:53.546403+0000 mon.a (mon.0) 50 : audit [DBG] from='mgr.14116 172.21.15.16:0/2443957845' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:55:55.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:54 smithi016 bash[27237]: audit 2023-09-23T18:55:53.478738+0000 mgr.y (mgr.14116) 4 : audit [DBG] from='client.14126 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:55:55.339 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: value unchanged 2023-09-23T18:55:55.915 INFO:teuthology.orchestra.run.smithi016.stderr:Generating ssh key... 2023-09-23T18:55:56.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:55 smithi016 bash[27237]: audit 2023-09-23T18:55:55.336697+0000 mgr.y (mgr.14116) 5 : audit [DBG] from='client.14128 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:55:58.158 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:57 smithi016 bash[27237]: audit 2023-09-23T18:55:57.131221+0000 mgr.y (mgr.14116) 6 : audit [DBG] from='client.14130 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:55:58.158 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:57 smithi016 bash[27237]: cephadm 2023-09-23T18:55:57.131682+0000 mgr.y (mgr.14116) 7 : cephadm [INF] Generating ssh key... 2023-09-23T18:55:58.159 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:57 smithi016 bash[27503]: Generating public/private rsa key pair. 2023-09-23T18:55:58.159 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:57 smithi016 bash[27503]: Your identification has been saved in /tmp/tmpbgbbdsdh/key. 2023-09-23T18:55:58.159 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:57 smithi016 bash[27503]: Your public key has been saved in /tmp/tmpbgbbdsdh/key.pub. 2023-09-23T18:55:58.159 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:57 smithi016 bash[27503]: The key fingerprint is: 2023-09-23T18:55:58.159 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:57 smithi016 bash[27503]: SHA256:dkSeiJDlOnSKT/ynMckokgCMrElJwzwiy9EGNHS0eto ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7 2023-09-23T18:55:58.159 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:57 smithi016 bash[27503]: The key's randomart image is: 2023-09-23T18:55:58.159 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:57 smithi016 bash[27503]: +---[RSA 3072]----+ 2023-09-23T18:55:58.160 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:57 smithi016 bash[27503]: |=B+o.o. . | 2023-09-23T18:55:58.160 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:57 smithi016 bash[27503]: |B==ooo . + . | 2023-09-23T18:55:58.160 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:57 smithi016 bash[27503]: |**+.. + . + | 2023-09-23T18:55:58.160 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:57 smithi016 bash[27503]: |=o.+ + . | 2023-09-23T18:55:58.160 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:57 smithi016 bash[27503]: |+...* S . | 2023-09-23T18:55:58.160 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:57 smithi016 bash[27503]: |..+o = o . | 2023-09-23T18:55:58.160 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:57 smithi016 bash[27503]: |o..Eo * . | 2023-09-23T18:55:58.160 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:57 smithi016 bash[27503]: | . . = | 2023-09-23T18:55:58.160 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:57 smithi016 bash[27503]: | . | 2023-09-23T18:55:58.160 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:55:57 smithi016 bash[27503]: +----[SHA256]-----+ 2023-09-23T18:55:59.199 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:58 smithi016 bash[27237]: audit 2023-09-23T18:55:57.922042+0000 mon.a (mon.0) 51 : audit [INF] from='mgr.14116 172.21.15.16:0/2443957845' entity='mgr.y' 2023-09-23T18:55:59.199 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:55:58 smithi016 bash[27237]: audit 2023-09-23T18:55:57.925396+0000 mon.a (mon.0) 52 : audit [INF] from='mgr.14116 172.21.15.16:0/2443957845' entity='mgr.y' 2023-09-23T18:55:59.743 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC1Ln4aD8dkQmujYRI636YNhS4TSpnCbvGdnbnj6ghO69mFuUAkmG2IojeXBHNrjJ04Yr/IdculdWaFr+Qn00b/eHryEAcIaQt9i25csGP+/AS5YDWgAEiCphtN72gBSZPc7oj99H/lMzqF12zamGXwRDq67/0Z/9cjJnVrUcx5C/SmAKl7/v+WbAl4oxIciCBQosk4MBlZy9ixHMh6T5Zj2pnJVyNXYkjbHD7AjUnuCRdpKbaopBJXKzgvJDOiCMrSPWkxzx15OneaJOcTUvZaapiKrAspvpQGirayXGsfFlpBuKZ5xdMa9L18PvZzUCayEPCgSQFeD5RpP9epF63jwGpD+qU7n8itecIEMHVGQt53YmBAQEIzs7Bt3+yBKwWl9T0wcbDSnBr+qkyzsvwps3OC4g5tuuHJ4XDkrABCpwEUrYEV6tqYI6V/iUe5SkmO3z8aYAIYvqz2jTOH/QO8DfCtL3iBrqquJdpEeRMI7tkKs9Fazxdgpd0LqfAki9M= ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7 2023-09-23T18:56:00.304 INFO:teuthology.orchestra.run.smithi016.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2023-09-23T18:56:00.304 INFO:teuthology.orchestra.run.smithi016.stderr:Adding key to root@localhost authorized_keys... 2023-09-23T18:56:00.305 INFO:teuthology.orchestra.run.smithi016.stderr:Adding host smithi016... 2023-09-23T18:56:01.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:00 smithi016 bash[27237]: audit 2023-09-23T18:55:59.740168+0000 mgr.y (mgr.14116) 8 : audit [DBG] from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:56:01.898 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:56:01 smithi016 bash[27503]: Warning: Permanently added '172.21.15.16' (ECDSA) to the list of known hosts. 2023-09-23T18:56:02.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:01 smithi016 bash[27237]: audit 2023-09-23T18:56:01.501957+0000 mgr.y (mgr.14116) 9 : audit [DBG] from='client.14134 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi016", "addr": "172.21.15.16", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:56:02.333 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: Added host 'smithi016' with addr '172.21.15.16' 2023-09-23T18:56:03.190 INFO:teuthology.orchestra.run.smithi016.stderr:Deploying unmanaged mon service... 2023-09-23T18:56:03.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:03 smithi016 bash[27237]: audit 2023-09-23T18:56:02.329962+0000 mon.a (mon.0) 53 : audit [INF] from='mgr.14116 172.21.15.16:0/2443957845' entity='mgr.y' 2023-09-23T18:56:03.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:03 smithi016 bash[27237]: cephadm 2023-09-23T18:56:02.330664+0000 mgr.y (mgr.14116) 10 : cephadm [INF] Added host smithi016 2023-09-23T18:56:03.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:03 smithi016 bash[27237]: audit 2023-09-23T18:56:02.333618+0000 mon.a (mon.0) 54 : audit [DBG] from='mgr.14116 172.21.15.16:0/2443957845' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:56:05.082 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: Scheduled mon update... 2023-09-23T18:56:05.810 INFO:teuthology.orchestra.run.smithi016.stderr:Deploying unmanaged mgr service... 2023-09-23T18:56:06.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:06 smithi016 bash[27237]: audit 2023-09-23T18:56:05.072408+0000 mgr.y (mgr.14116) 11 : audit [DBG] from='client.14136 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:56:06.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:06 smithi016 bash[27237]: cephadm 2023-09-23T18:56:05.074150+0000 mgr.y (mgr.14116) 12 : cephadm [INF] Saving service mon spec with placement count:5 2023-09-23T18:56:06.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:06 smithi016 bash[27237]: audit 2023-09-23T18:56:05.078032+0000 mon.a (mon.0) 55 : audit [INF] from='mgr.14116 172.21.15.16:0/2443957845' entity='mgr.y' 2023-09-23T18:56:06.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:06 smithi016 bash[27237]: audit 2023-09-23T18:56:05.406235+0000 mon.a (mon.0) 56 : audit [INF] from='mgr.14116 172.21.15.16:0/2443957845' entity='mgr.y' 2023-09-23T18:56:06.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:06 smithi016 bash[27237]: audit 2023-09-23T18:56:05.671552+0000 mon.a (mon.0) 57 : audit [INF] from='mgr.14116 172.21.15.16:0/2443957845' entity='mgr.y' 2023-09-23T18:56:07.132 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: Scheduled mgr update... 2023-09-23T18:56:08.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:08 smithi016 bash[27237]: audit 2023-09-23T18:56:07.122224+0000 mgr.y (mgr.14116) 13 : audit [DBG] from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:56:08.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:08 smithi016 bash[27237]: cephadm 2023-09-23T18:56:07.124608+0000 mgr.y (mgr.14116) 14 : cephadm [INF] Saving service mgr spec with placement count:2 2023-09-23T18:56:08.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:08 smithi016 bash[27237]: audit 2023-09-23T18:56:07.129482+0000 mon.a (mon.0) 58 : audit [INF] from='mgr.14116 172.21.15.16:0/2443957845' entity='mgr.y' 2023-09-23T18:56:10.442 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:10 smithi016 bash[27237]: audit 2023-09-23T18:56:09.143502+0000 mon.a (mon.0) 59 : audit [INF] from='client.? 172.21.15.16:0/1580870698' entity='client.admin' 2023-09-23T18:56:11.710 INFO:teuthology.orchestra.run.smithi016.stderr:Enabling the dashboard module... 2023-09-23T18:56:12.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:11 smithi016 bash[27237]: cluster 2023-09-23T18:56:10.562755+0000 mgr.y (mgr.14116) 15 : cluster [DBG] pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:56:12.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:11 smithi016 bash[27237]: audit 2023-09-23T18:56:10.713795+0000 mon.a (mon.0) 60 : audit [INF] from='mgr.14116 172.21.15.16:0/2443957845' entity='mgr.y' 2023-09-23T18:56:12.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:11 smithi016 bash[27237]: audit 2023-09-23T18:56:11.065806+0000 mon.a (mon.0) 61 : audit [INF] from='client.? 172.21.15.16:0/3785954296' entity='client.admin' 2023-09-23T18:56:13.989 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:13 smithi016 bash[27237]: cluster 2023-09-23T18:56:12.563152+0000 mgr.y (mgr.14116) 16 : cluster [DBG] pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:56:13.990 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:13 smithi016 bash[27237]: audit 2023-09-23T18:56:13.128433+0000 mon.a (mon.0) 62 : audit [INF] from='client.? 172.21.15.16:0/1493268665' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2023-09-23T18:56:13.990 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:56:13 smithi016 bash[27503]: ignoring --setuser ceph since I am not root 2023-09-23T18:56:13.990 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:56:13 smithi016 bash[27503]: ignoring --setgroup ceph since I am not root 2023-09-23T18:56:13.990 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:56:13 smithi016 bash[27503]: debug 2023-09-23T18:56:13.930+0000 7f860dfc7680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T18:56:14.284 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:56:14 smithi016 bash[27503]: debug 2023-09-23T18:56:14.034+0000 7f860dfc7680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T18:56:14.725 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:56:14 smithi016 bash[27503]: debug 2023-09-23T18:56:14.522+0000 7f860dfc7680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T18:56:15.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:14 smithi016 bash[27237]: audit 2023-09-23T18:56:13.721558+0000 mon.a (mon.0) 63 : audit [INF] from='client.? 172.21.15.16:0/1493268665' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2023-09-23T18:56:15.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:14 smithi016 bash[27237]: cluster 2023-09-23T18:56:13.721760+0000 mon.a (mon.0) 64 : cluster [DBG] mgrmap e9: y(active, since 23s) 2023-09-23T18:56:15.784 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:56:15 smithi016 bash[27503]: debug 2023-09-23T18:56:15.510+0000 7f860dfc7680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T18:56:15.784 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:56:15 smithi016 bash[27503]: debug 2023-09-23T18:56:15.614+0000 7f860dfc7680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T18:56:15.827 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: { 2023-09-23T18:56:15.827 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "epoch": 9, 2023-09-23T18:56:15.827 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "available": true, 2023-09-23T18:56:15.827 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "active_name": "y", 2023-09-23T18:56:15.828 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "num_standby": 0 2023-09-23T18:56:15.828 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: } 2023-09-23T18:56:16.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:15 smithi016 bash[27237]: audit 2023-09-23T18:56:15.824893+0000 mon.a (mon.0) 65 : audit [DBG] from='client.? 172.21.15.16:0/3001608262' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-09-23T18:56:16.284 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:56:15 smithi016 bash[27503]: debug 2023-09-23T18:56:15.870+0000 7f860dfc7680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T18:56:16.431 INFO:teuthology.orchestra.run.smithi016.stderr:Waiting for the mgr to restart... 2023-09-23T18:56:16.431 INFO:teuthology.orchestra.run.smithi016.stderr:Waiting for mgr epoch 9... 2023-09-23T18:56:17.216 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:56:17 smithi016 bash[27503]: debug 2023-09-23T18:56:17.042+0000 7f860dfc7680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T18:56:17.522 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:56:17 smithi016 bash[27503]: debug 2023-09-23T18:56:17.210+0000 7f860dfc7680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T18:56:17.522 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:56:17 smithi016 bash[27503]: debug 2023-09-23T18:56:17.306+0000 7f860dfc7680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T18:56:17.522 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:56:17 smithi016 bash[27503]: debug 2023-09-23T18:56:17.426+0000 7f860dfc7680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T18:56:17.783 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:56:17 smithi016 bash[27503]: debug 2023-09-23T18:56:17.650+0000 7f860dfc7680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T18:56:18.181 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:56:18 smithi016 bash[27503]: debug 2023-09-23T18:56:18.042+0000 7f860dfc7680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T18:56:18.534 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:56:18 smithi016 bash[27503]: debug 2023-09-23T18:56:18.178+0000 7f860dfc7680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T18:56:19.533 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:56:19 smithi016 bash[27503]: debug 2023-09-23T18:56:19.106+0000 7f860dfc7680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T18:56:19.534 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:56:19 smithi016 bash[27503]: debug 2023-09-23T18:56:19.214+0000 7f860dfc7680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T18:56:19.534 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:56:19 smithi016 bash[27503]: debug 2023-09-23T18:56:19.338+0000 7f860dfc7680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T18:56:19.870 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:56:19 smithi016 bash[27503]: debug 2023-09-23T18:56:19.566+0000 7f860dfc7680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T18:56:20.218 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:56:19 smithi016 bash[27503]: debug 2023-09-23T18:56:19.866+0000 7f860dfc7680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T18:56:20.218 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:56:20 smithi016 bash[27503]: debug 2023-09-23T18:56:20.034+0000 7f860dfc7680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T18:56:20.533 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:56:20 smithi016 bash[27503]: debug 2023-09-23T18:56:20.214+0000 7f860dfc7680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T18:56:20.534 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:56:20 smithi016 bash[27503]: debug 2023-09-23T18:56:20.434+0000 7f860dfc7680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T18:56:20.838 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:56:20 smithi016 bash[27503]: debug 2023-09-23T18:56:20.550+0000 7f860dfc7680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T18:56:20.838 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:56:20 smithi016 bash[27503]: Warning: Permanently added '172.21.15.16' (ECDSA) to the list of known hosts. 2023-09-23T18:56:20.838 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:20 smithi016 bash[27237]: cluster 2023-09-23T18:56:20.574585+0000 mon.a (mon.0) 66 : cluster [INF] Active manager daemon y restarted 2023-09-23T18:56:20.838 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:20 smithi016 bash[27237]: cluster 2023-09-23T18:56:20.575027+0000 mon.a (mon.0) 67 : cluster [INF] Activating manager daemon y 2023-09-23T18:56:20.838 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:20 smithi016 bash[27237]: cluster 2023-09-23T18:56:20.581332+0000 mon.a (mon.0) 68 : cluster [DBG] osdmap e3: 0 total, 0 up, 0 in 2023-09-23T18:56:20.838 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:20 smithi016 bash[27237]: cluster 2023-09-23T18:56:20.581619+0000 mon.a (mon.0) 69 : cluster [DBG] mgrmap e10: y(active, starting, since 0.00676268s) 2023-09-23T18:56:20.838 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:20 smithi016 bash[27237]: audit 2023-09-23T18:56:20.582654+0000 mon.a (mon.0) 70 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T18:56:20.839 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:20 smithi016 bash[27237]: audit 2023-09-23T18:56:20.583891+0000 mon.a (mon.0) 71 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T18:56:20.839 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:20 smithi016 bash[27237]: audit 2023-09-23T18:56:20.584810+0000 mon.a (mon.0) 72 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T18:56:20.839 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:20 smithi016 bash[27237]: audit 2023-09-23T18:56:20.585076+0000 mon.a (mon.0) 73 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T18:56:20.839 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:20 smithi016 bash[27237]: audit 2023-09-23T18:56:20.585323+0000 mon.a (mon.0) 74 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T18:56:20.839 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:20 smithi016 bash[27237]: cluster 2023-09-23T18:56:20.608031+0000 mon.a (mon.0) 75 : cluster [INF] Manager daemon y is now available 2023-09-23T18:56:20.839 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:20 smithi016 bash[27237]: audit 2023-09-23T18:56:20.744162+0000 mon.a (mon.0) 76 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:56:20.839 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:20 smithi016 bash[27237]: audit 2023-09-23T18:56:20.747210+0000 mon.a (mon.0) 77 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:56:21.603 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: { 2023-09-23T18:56:21.604 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2023-09-23T18:56:21.604 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: "initialized": true 2023-09-23T18:56:21.604 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: } 2023-09-23T18:56:22.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:21 smithi016 bash[27237]: audit 2023-09-23T18:56:20.792064+0000 mon.a (mon.0) 78 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T18:56:22.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:21 smithi016 bash[27237]: audit 2023-09-23T18:56:20.796863+0000 mon.a (mon.0) 79 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T18:56:22.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:21 smithi016 bash[27237]: cluster 2023-09-23T18:56:21.585796+0000 mon.a (mon.0) 80 : cluster [DBG] mgrmap e11: y(active, since 1.01094s) 2023-09-23T18:56:22.236 INFO:teuthology.orchestra.run.smithi016.stderr:mgr epoch 9 is available 2023-09-23T18:56:22.236 INFO:teuthology.orchestra.run.smithi016.stderr:Generating a dashboard self-signed certificate... 2023-09-23T18:56:23.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:22 smithi016 bash[27237]: audit 2023-09-23T18:56:21.585076+0000 mgr.y (mgr.14146) 1 : audit [DBG] from='client.14150 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-09-23T18:56:23.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:22 smithi016 bash[27237]: audit 2023-09-23T18:56:21.601559+0000 mgr.y (mgr.14146) 2 : audit [DBG] from='client.14150 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-09-23T18:56:23.617 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: Self-signed certificate created 2023-09-23T18:56:24.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:23 smithi016 bash[27237]: cluster 2023-09-23T18:56:22.771554+0000 mon.a (mon.0) 81 : cluster [DBG] mgrmap e12: y(active, since 2s) 2023-09-23T18:56:24.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:23 smithi016 bash[27237]: audit 2023-09-23T18:56:23.610508+0000 mon.a (mon.0) 82 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:56:24.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:23 smithi016 bash[27237]: audit 2023-09-23T18:56:23.614809+0000 mon.a (mon.0) 83 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:56:24.282 INFO:teuthology.orchestra.run.smithi016.stderr:Creating initial admin user... 2023-09-23T18:56:25.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:24 smithi016 bash[27237]: audit 2023-09-23T18:56:23.515807+0000 mgr.y (mgr.14146) 3 : audit [DBG] from='client.14156 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:56:25.850 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$yRwxZa7AkK0T279eTqT7hO5Ht7gLKEKGsL68OMa7/aaJIZzqLSzY6", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1695495385, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2023-09-23T18:56:26.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:25 smithi016 bash[27237]: audit 2023-09-23T18:56:25.589271+0000 mgr.y (mgr.14146) 4 : audit [DBG] from='client.14158 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:56:26.473 INFO:teuthology.orchestra.run.smithi016.stderr:Fetching dashboard port number... 2023-09-23T18:56:27.150 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:26 smithi016 bash[27237]: audit 2023-09-23T18:56:25.846323+0000 mon.a (mon.0) 84 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:56:27.810 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: 8443 2023-09-23T18:56:28.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:27 smithi016 bash[27237]: cluster 2023-09-23T18:56:26.848546+0000 mon.a (mon.0) 85 : cluster [DBG] mgrmap e13: y(active, since 6s) 2023-09-23T18:56:28.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:27 smithi016 bash[27237]: audit 2023-09-23T18:56:27.592217+0000 mon.a (mon.0) 86 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:56:28.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:27 smithi016 bash[27237]: audit 2023-09-23T18:56:27.807732+0000 mon.a (mon.0) 87 : audit [DBG] from='client.? 172.21.15.16:0/1151064052' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2023-09-23T18:56:28.454 INFO:teuthology.orchestra.run.smithi016.stderr:firewalld does not appear to be present 2023-09-23T18:56:28.455 INFO:teuthology.orchestra.run.smithi016.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2023-09-23T18:56:28.458 INFO:teuthology.orchestra.run.smithi016.stderr:Ceph Dashboard is now available at: 2023-09-23T18:56:28.458 INFO:teuthology.orchestra.run.smithi016.stderr: 2023-09-23T18:56:28.458 INFO:teuthology.orchestra.run.smithi016.stderr: URL: https://smithi016.front.sepia.ceph.com:8443/ 2023-09-23T18:56:28.458 INFO:teuthology.orchestra.run.smithi016.stderr: User: admin 2023-09-23T18:56:28.458 INFO:teuthology.orchestra.run.smithi016.stderr: Password: fyabrmdf2g 2023-09-23T18:56:28.458 INFO:teuthology.orchestra.run.smithi016.stderr: 2023-09-23T18:56:28.459 INFO:teuthology.orchestra.run.smithi016.stderr:Enabling autotune for osd_memory_target 2023-09-23T18:56:31.915 INFO:teuthology.orchestra.run.smithi016.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2023-09-23T18:56:32.541 INFO:teuthology.orchestra.run.smithi016.stderr:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2023-09-23T18:56:32.542 INFO:teuthology.orchestra.run.smithi016.stderr: 2023-09-23T18:56:32.542 INFO:teuthology.orchestra.run.smithi016.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2023-09-23T18:56:32.542 INFO:teuthology.orchestra.run.smithi016.stderr: 2023-09-23T18:56:32.542 INFO:teuthology.orchestra.run.smithi016.stderr:Or, if you are only running a single cluster on this host: 2023-09-23T18:56:32.542 INFO:teuthology.orchestra.run.smithi016.stderr: 2023-09-23T18:56:32.542 INFO:teuthology.orchestra.run.smithi016.stderr: sudo /home/ubuntu/cephtest/cephadm shell 2023-09-23T18:56:32.542 INFO:teuthology.orchestra.run.smithi016.stderr: 2023-09-23T18:56:32.542 INFO:teuthology.orchestra.run.smithi016.stderr:Please consider enabling telemetry to help improve Ceph: 2023-09-23T18:56:32.542 INFO:teuthology.orchestra.run.smithi016.stderr: 2023-09-23T18:56:32.542 INFO:teuthology.orchestra.run.smithi016.stderr: ceph telemetry on 2023-09-23T18:56:32.542 INFO:teuthology.orchestra.run.smithi016.stderr: 2023-09-23T18:56:32.542 INFO:teuthology.orchestra.run.smithi016.stderr:For more information see: 2023-09-23T18:56:32.543 INFO:teuthology.orchestra.run.smithi016.stderr: 2023-09-23T18:56:32.543 INFO:teuthology.orchestra.run.smithi016.stderr: https://docs.ceph.com/en/pacific/mgr/telemetry/ 2023-09-23T18:56:32.543 INFO:teuthology.orchestra.run.smithi016.stderr: 2023-09-23T18:56:32.543 INFO:teuthology.orchestra.run.smithi016.stderr:Bootstrap complete. 2023-09-23T18:56:32.579 INFO:tasks.cephadm:Fetching config... 2023-09-23T18:56:32.580 DEBUG:teuthology.orchestra.run.smithi016:> set -ex 2023-09-23T18:56:32.580 DEBUG:teuthology.orchestra.run.smithi016:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2023-09-23T18:56:32.585 INFO:tasks.cephadm:Fetching client.admin keyring... 2023-09-23T18:56:32.585 DEBUG:teuthology.orchestra.run.smithi016:> set -ex 2023-09-23T18:56:32.585 DEBUG:teuthology.orchestra.run.smithi016:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2023-09-23T18:56:32.636 INFO:tasks.cephadm:Fetching mon keyring... 2023-09-23T18:56:32.636 DEBUG:teuthology.orchestra.run.smithi016:> set -ex 2023-09-23T18:56:32.636 DEBUG:teuthology.orchestra.run.smithi016:> sudo dd if=/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/keyring of=/dev/stdout 2023-09-23T18:56:32.690 INFO:tasks.cephadm:Fetching pub ssh key... 2023-09-23T18:56:32.690 DEBUG:teuthology.orchestra.run.smithi016:> set -ex 2023-09-23T18:56:32.691 DEBUG:teuthology.orchestra.run.smithi016:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2023-09-23T18:56:32.739 INFO:tasks.cephadm:Installing pub ssh key for root users... 2023-09-23T18:56:32.740 DEBUG:teuthology.orchestra.run.smithi016:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC1Ln4aD8dkQmujYRI636YNhS4TSpnCbvGdnbnj6ghO69mFuUAkmG2IojeXBHNrjJ04Yr/IdculdWaFr+Qn00b/eHryEAcIaQt9i25csGP+/AS5YDWgAEiCphtN72gBSZPc7oj99H/lMzqF12zamGXwRDq67/0Z/9cjJnVrUcx5C/SmAKl7/v+WbAl4oxIciCBQosk4MBlZy9ixHMh6T5Zj2pnJVyNXYkjbHD7AjUnuCRdpKbaopBJXKzgvJDOiCMrSPWkxzx15OneaJOcTUvZaapiKrAspvpQGirayXGsfFlpBuKZ5xdMa9L18PvZzUCayEPCgSQFeD5RpP9epF63jwGpD+qU7n8itecIEMHVGQt53YmBAQEIzs7Bt3+yBKwWl9T0wcbDSnBr+qkyzsvwps3OC4g5tuuHJ4XDkrABCpwEUrYEV6tqYI6V/iUe5SkmO3z8aYAIYvqz2jTOH/QO8DfCtL3iBrqquJdpEeRMI7tkKs9Fazxdgpd0LqfAki9M= ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-09-23T18:56:32.803 INFO:teuthology.orchestra.run.smithi016.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC1Ln4aD8dkQmujYRI636YNhS4TSpnCbvGdnbnj6ghO69mFuUAkmG2IojeXBHNrjJ04Yr/IdculdWaFr+Qn00b/eHryEAcIaQt9i25csGP+/AS5YDWgAEiCphtN72gBSZPc7oj99H/lMzqF12zamGXwRDq67/0Z/9cjJnVrUcx5C/SmAKl7/v+WbAl4oxIciCBQosk4MBlZy9ixHMh6T5Zj2pnJVyNXYkjbHD7AjUnuCRdpKbaopBJXKzgvJDOiCMrSPWkxzx15OneaJOcTUvZaapiKrAspvpQGirayXGsfFlpBuKZ5xdMa9L18PvZzUCayEPCgSQFeD5RpP9epF63jwGpD+qU7n8itecIEMHVGQt53YmBAQEIzs7Bt3+yBKwWl9T0wcbDSnBr+qkyzsvwps3OC4g5tuuHJ4XDkrABCpwEUrYEV6tqYI6V/iUe5SkmO3z8aYAIYvqz2jTOH/QO8DfCtL3iBrqquJdpEeRMI7tkKs9Fazxdgpd0LqfAki9M= ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7 2023-09-23T18:56:32.810 DEBUG:teuthology.orchestra.run.smithi092:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC1Ln4aD8dkQmujYRI636YNhS4TSpnCbvGdnbnj6ghO69mFuUAkmG2IojeXBHNrjJ04Yr/IdculdWaFr+Qn00b/eHryEAcIaQt9i25csGP+/AS5YDWgAEiCphtN72gBSZPc7oj99H/lMzqF12zamGXwRDq67/0Z/9cjJnVrUcx5C/SmAKl7/v+WbAl4oxIciCBQosk4MBlZy9ixHMh6T5Zj2pnJVyNXYkjbHD7AjUnuCRdpKbaopBJXKzgvJDOiCMrSPWkxzx15OneaJOcTUvZaapiKrAspvpQGirayXGsfFlpBuKZ5xdMa9L18PvZzUCayEPCgSQFeD5RpP9epF63jwGpD+qU7n8itecIEMHVGQt53YmBAQEIzs7Bt3+yBKwWl9T0wcbDSnBr+qkyzsvwps3OC4g5tuuHJ4XDkrABCpwEUrYEV6tqYI6V/iUe5SkmO3z8aYAIYvqz2jTOH/QO8DfCtL3iBrqquJdpEeRMI7tkKs9Fazxdgpd0LqfAki9M= ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-09-23T18:56:32.834 INFO:teuthology.orchestra.run.smithi092.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC1Ln4aD8dkQmujYRI636YNhS4TSpnCbvGdnbnj6ghO69mFuUAkmG2IojeXBHNrjJ04Yr/IdculdWaFr+Qn00b/eHryEAcIaQt9i25csGP+/AS5YDWgAEiCphtN72gBSZPc7oj99H/lMzqF12zamGXwRDq67/0Z/9cjJnVrUcx5C/SmAKl7/v+WbAl4oxIciCBQosk4MBlZy9ixHMh6T5Zj2pnJVyNXYkjbHD7AjUnuCRdpKbaopBJXKzgvJDOiCMrSPWkxzx15OneaJOcTUvZaapiKrAspvpQGirayXGsfFlpBuKZ5xdMa9L18PvZzUCayEPCgSQFeD5RpP9epF63jwGpD+qU7n8itecIEMHVGQt53YmBAQEIzs7Bt3+yBKwWl9T0wcbDSnBr+qkyzsvwps3OC4g5tuuHJ4XDkrABCpwEUrYEV6tqYI6V/iUe5SkmO3z8aYAIYvqz2jTOH/QO8DfCtL3iBrqquJdpEeRMI7tkKs9Fazxdgpd0LqfAki9M= ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7 2023-09-23T18:56:32.843 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2023-09-23T18:56:33.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:32 smithi016 bash[27237]: audit 2023-09-23T18:56:31.909403+0000 mon.a (mon.0) 88 : audit [INF] from='client.? 172.21.15.16:0/4163483376' entity='client.admin' 2023-09-23T18:56:38.469 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2023-09-23T18:56:38.469 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph orch client-keyring set client.admin '*' --mode 0755 2023-09-23T18:56:39.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:38 smithi016 bash[27237]: audit 2023-09-23T18:56:37.772348+0000 mon.a (mon.0) 89 : audit [INF] from='client.? 172.21.15.16:0/2642926086' entity='client.admin' 2023-09-23T18:56:40.783 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:40 smithi016 bash[27237]: audit 2023-09-23T18:56:39.552934+0000 mon.a (mon.0) 90 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:56:40.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:40 smithi016 bash[27237]: audit 2023-09-23T18:56:39.850511+0000 mon.a (mon.0) 91 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi016", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:56:40.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:40 smithi016 bash[27237]: audit 2023-09-23T18:56:39.856190+0000 mon.a (mon.0) 92 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:56:40.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:40 smithi016 bash[27237]: audit 2023-09-23T18:56:39.858110+0000 mon.a (mon.0) 93 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:56:40.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:40 smithi016 bash[27237]: audit 2023-09-23T18:56:39.862128+0000 mon.a (mon.0) 94 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:56:41.920 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:41 smithi016 bash[27237]: cluster 2023-09-23T18:56:40.586699+0000 mgr.y (mgr.14146) 5 : cluster [DBG] pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:56:42.327 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi092 2023-09-23T18:56:42.327 DEBUG:teuthology.orchestra.run.smithi092:> set -ex 2023-09-23T18:56:42.327 DEBUG:teuthology.orchestra.run.smithi092:> dd of=/etc/ceph/ceph.conf 2023-09-23T18:56:42.335 DEBUG:teuthology.orchestra.run.smithi092:> set -ex 2023-09-23T18:56:42.335 DEBUG:teuthology.orchestra.run.smithi092:> dd of=/etc/ceph/ceph.client.admin.keyring 2023-09-23T18:56:42.382 INFO:tasks.cephadm:Adding host smithi092 to orchestrator... 2023-09-23T18:56:42.383 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph orch host add smithi092 2023-09-23T18:56:43.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:42 smithi016 bash[27237]: audit 2023-09-23T18:56:41.636043+0000 mon.a (mon.0) 95 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:56:43.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:42 smithi016 bash[27237]: audit 2023-09-23T18:56:41.682645+0000 mon.a (mon.0) 96 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:56:43.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:42 smithi016 bash[27237]: audit 2023-09-23T18:56:41.685113+0000 mon.a (mon.0) 97 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:56:43.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:42 smithi016 bash[27237]: audit 2023-09-23T18:56:41.686594+0000 mon.a (mon.0) 98 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:56:43.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:42 smithi016 bash[27237]: cephadm 2023-09-23T18:56:41.688837+0000 mgr.y (mgr.14146) 7 : cephadm [INF] Updating smithi016:/etc/ceph/ceph.conf 2023-09-23T18:56:43.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:42 smithi016 bash[27237]: cephadm 2023-09-23T18:56:41.737097+0000 mgr.y (mgr.14146) 8 : cephadm [INF] Updating smithi016:/etc/ceph/ceph.client.admin.keyring 2023-09-23T18:56:43.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:42 smithi016 bash[27237]: audit 2023-09-23T18:56:41.782309+0000 mon.a (mon.0) 99 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:56:43.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:42 smithi016 bash[27237]: audit 2023-09-23T18:56:41.785460+0000 mon.a (mon.0) 100 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:56:44.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:43 smithi016 bash[27237]: cluster 2023-09-23T18:56:42.587033+0000 mgr.y (mgr.14146) 9 : cluster [DBG] pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:56:45.771 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:56:45 smithi016 bash[27503]: Warning: Permanently added 'smithi092,172.21.15.92' (ECDSA) to the list of known hosts. 2023-09-23T18:56:46.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:45 smithi016 bash[27237]: cluster 2023-09-23T18:56:44.587443+0000 mgr.y (mgr.14146) 10 : cluster [DBG] pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:56:46.221 INFO:teuthology.orchestra.run.smithi016.stdout:Added host 'smithi092' with addr '172.21.15.92' 2023-09-23T18:56:46.533 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:56:46 smithi016 bash[27503]: Warning: Permanently added '172.21.15.92' (ECDSA) to the list of known hosts. 2023-09-23T18:56:46.888 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph orch host ls --format=json 2023-09-23T18:56:47.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:46 smithi016 bash[27237]: audit 2023-09-23T18:56:45.292559+0000 mgr.y (mgr.14146) 11 : audit [DBG] from='client.14170 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi092", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:56:47.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:46 smithi016 bash[27237]: audit 2023-09-23T18:56:46.218429+0000 mon.a (mon.0) 101 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:56:47.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:46 smithi016 bash[27237]: audit 2023-09-23T18:56:46.263495+0000 mon.a (mon.0) 102 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:56:48.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:47 smithi016 bash[27237]: cephadm 2023-09-23T18:56:46.219260+0000 mgr.y (mgr.14146) 12 : cephadm [INF] Added host smithi092 2023-09-23T18:56:48.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:47 smithi016 bash[27237]: cluster 2023-09-23T18:56:46.587979+0000 mgr.y (mgr.14146) 13 : cluster [DBG] pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:56:48.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:47 smithi016 bash[27237]: audit 2023-09-23T18:56:46.822475+0000 mon.a (mon.0) 103 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:56:49.364 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:49 smithi016 bash[27237]: audit 2023-09-23T18:56:48.077599+0000 mon.a (mon.0) 104 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:56:49.365 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:49 smithi016 bash[27237]: cluster 2023-09-23T18:56:48.588404+0000 mgr.y (mgr.14146) 14 : cluster [DBG] pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:56:49.736 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-09-23T18:56:49.737 INFO:teuthology.orchestra.run.smithi016.stdout:[{"addr": "172.21.15.16", "hostname": "smithi016", "labels": [], "status": ""}, {"addr": "172.21.15.92", "hostname": "smithi092", "labels": [], "status": ""}] 2023-09-23T18:56:50.364 INFO:tasks.cephadm:Setting crush tunables to default 2023-09-23T18:56:50.364 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd crush tunables default 2023-09-23T18:56:51.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:50 smithi016 bash[27237]: audit 2023-09-23T18:56:49.734879+0000 mgr.y (mgr.14146) 15 : audit [DBG] from='client.14172 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:56:52.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:51 smithi016 bash[27237]: cluster 2023-09-23T18:56:50.588788+0000 mgr.y (mgr.14146) 16 : cluster [DBG] pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:56:54.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:53 smithi016 bash[27237]: cluster 2023-09-23T18:56:52.589329+0000 mgr.y (mgr.14146) 17 : cluster [DBG] pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:56:54.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:53 smithi016 bash[27237]: audit 2023-09-23T18:56:53.136673+0000 mon.a (mon.0) 105 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:56:54.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:53 smithi016 bash[27237]: audit 2023-09-23T18:56:53.218829+0000 mon.a (mon.0) 106 : audit [INF] from='client.? 172.21.15.16:0/3474018838' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-09-23T18:56:54.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:53 smithi016 bash[27237]: audit 2023-09-23T18:56:53.418993+0000 mon.a (mon.0) 107 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:56:54.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:53 smithi016 bash[27237]: audit 2023-09-23T18:56:53.420187+0000 mon.a (mon.0) 108 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi092", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:56:54.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:53 smithi016 bash[27237]: audit 2023-09-23T18:56:53.421829+0000 mon.a (mon.0) 109 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:56:54.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:53 smithi016 bash[27237]: audit 2023-09-23T18:56:53.423335+0000 mon.a (mon.0) 110 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:56:54.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:53 smithi016 bash[27237]: audit 2023-09-23T18:56:53.472617+0000 mon.a (mon.0) 111 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:56:54.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:53 smithi016 bash[27237]: audit 2023-09-23T18:56:53.477094+0000 mon.a (mon.0) 112 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:56:54.145 INFO:teuthology.orchestra.run.smithi016.stderr:adjusted tunables profile to default 2023-09-23T18:56:54.814 INFO:tasks.cephadm:Adding mon.a on smithi016 2023-09-23T18:56:54.814 INFO:tasks.cephadm:Adding mon.c on smithi016 2023-09-23T18:56:54.814 INFO:tasks.cephadm:Adding mon.b on smithi092 2023-09-23T18:56:54.815 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph orch apply mon '3;smithi016:172.21.15.16=a;smithi016:[v2:172.21.15.16:3301,v1:172.21.15.16:6790]=c;smithi092:172.21.15.92=b' 2023-09-23T18:56:55.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:54 smithi016 bash[27237]: cephadm 2023-09-23T18:56:53.425401+0000 mgr.y (mgr.14146) 18 : cephadm [INF] Updating smithi092:/etc/ceph/ceph.conf 2023-09-23T18:56:55.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:54 smithi016 bash[27237]: cephadm 2023-09-23T18:56:53.442657+0000 mgr.y (mgr.14146) 19 : cephadm [INF] Updating smithi092:/etc/ceph/ceph.client.admin.keyring 2023-09-23T18:56:55.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:54 smithi016 bash[27237]: audit 2023-09-23T18:56:54.138238+0000 mon.a (mon.0) 113 : audit [INF] from='client.? 172.21.15.16:0/3474018838' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-09-23T18:56:55.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:54 smithi016 bash[27237]: cluster 2023-09-23T18:56:54.138416+0000 mon.a (mon.0) 114 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-09-23T18:56:55.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:54 smithi016 bash[27237]: cluster 2023-09-23T18:56:54.589814+0000 mgr.y (mgr.14146) 20 : cluster [DBG] pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:56:57.906 INFO:teuthology.orchestra.run.smithi092.stdout:Scheduled mon update... 2023-09-23T18:56:58.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:57 smithi016 bash[27237]: cluster 2023-09-23T18:56:56.590247+0000 mgr.y (mgr.14146) 21 : cluster [DBG] pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:56:58.773 DEBUG:teuthology.orchestra.run.smithi016:mon.c> sudo journalctl -f -n 0 -u ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@mon.c.service 2023-09-23T18:56:58.775 DEBUG:teuthology.orchestra.run.smithi092:mon.b> sudo journalctl -f -n 0 -u ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@mon.b.service 2023-09-23T18:56:58.779 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-09-23T18:56:58.779 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph mon dump -f json 2023-09-23T18:56:59.021 INFO:journalctl@ceph.mon.c.smithi016.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-09-23T18:56:59.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:58 smithi016 bash[27237]: audit 2023-09-23T18:56:57.893907+0000 mgr.y (mgr.14146) 22 : audit [DBG] from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi016:172.21.15.16=a;smithi016:[v2:172.21.15.16:3301,v1:172.21.15.16:6790]=c;smithi092:172.21.15.92=b", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:56:59.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:58 smithi016 bash[27237]: cephadm 2023-09-23T18:56:57.900052+0000 mgr.y (mgr.14146) 23 : cephadm [INF] Saving service mon spec with placement smithi016:172.21.15.16=a;smithi016:[v2:172.21.15.16:3301,v1:172.21.15.16:6790]=c;smithi092:172.21.15.92=b;count:3 2023-09-23T18:56:59.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:58 smithi016 bash[27237]: audit 2023-09-23T18:56:57.904864+0000 mon.a (mon.0) 115 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:56:59.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:58 smithi016 bash[27237]: audit 2023-09-23T18:56:57.955995+0000 mon.a (mon.0) 116 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:56:59.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:58 smithi016 bash[27237]: audit 2023-09-23T18:56:57.959824+0000 mon.a (mon.0) 117 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:56:59.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:58 smithi016 bash[27237]: audit 2023-09-23T18:56:57.961458+0000 mon.a (mon.0) 118 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:56:59.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:58 smithi016 bash[27237]: audit 2023-09-23T18:56:57.967148+0000 mon.a (mon.0) 119 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:56:59.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:58 smithi016 bash[27237]: audit 2023-09-23T18:56:57.969837+0000 mon.a (mon.0) 120 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T18:56:59.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:58 smithi016 bash[27237]: audit 2023-09-23T18:56:57.971055+0000 mon.a (mon.0) 121 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:56:59.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:58 smithi016 bash[27237]: cephadm 2023-09-23T18:56:57.972100+0000 mgr.y (mgr.14146) 24 : cephadm [INF] Deploying daemon mon.b on smithi092 2023-09-23T18:56:59.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:56:58 smithi016 bash[27237]: cluster 2023-09-23T18:56:58.590653+0000 mgr.y (mgr.14146) 25 : cluster [DBG] pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:56:59.631 INFO:journalctl@ceph.mon.b.smithi092.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-09-23T18:57:02.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:01 smithi016 bash[27237]: cluster 2023-09-23T18:57:00.591048+0000 mgr.y (mgr.14146) 26 : cluster [DBG] pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:03.343 INFO:teuthology.orchestra.run.smithi092.stdout: 2023-09-23T18:57:03.343 INFO:teuthology.orchestra.run.smithi092.stdout:{"epoch":1,"fsid":"95ce9eb6-5a42-11ee-8db4-212e2dc638e7","modified":"2023-09-23T18:55:19.600161Z","created":"2023-09-23T18:55:19.600161Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:3300","nonce":0},{"type":"v1","addr":"172.21.15.16:6789","nonce":0}]},"addr":"172.21.15.16:6789/0","public_addr":"172.21.15.16:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-09-23T18:57:03.346 INFO:teuthology.orchestra.run.smithi092.stderr:dumped monmap epoch 1 2023-09-23T18:57:04.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:03 smithi016 bash[27237]: cluster 2023-09-23T18:57:02.591467+0000 mgr.y (mgr.14146) 27 : cluster [DBG] pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:04.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:03 smithi016 bash[27237]: audit 2023-09-23T18:57:03.342335+0000 mon.a (mon.0) 122 : audit [DBG] from='client.? 172.21.15.92:0/2102316039' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T18:57:04.976 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-09-23T18:57:04.976 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph mon dump -f json 2023-09-23T18:57:05.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:05 smithi092 bash[27556]: debug 2023-09-23T18:57:05.133+0000 7f3f62042700 1 mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-09-23T18:57:09.282 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:09 smithi016 bash[33624]: debug 2023-09-23T18:57:09.102+0000 7f954298b700 1 mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-09-23T18:57:10.533 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:10 smithi016 bash[27237]: audit 2023-09-23T18:57:05.149637+0000 mon.a (mon.0) 127 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T18:57:10.533 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:10 smithi016 bash[27237]: audit 2023-09-23T18:57:05.149854+0000 mon.a (mon.0) 128 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:57:10.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:10 smithi016 bash[27237]: cluster 2023-09-23T18:57:05.150023+0000 mon.a (mon.0) 129 : cluster [INF] mon.a calling monitor election 2023-09-23T18:57:10.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:10 smithi016 bash[27237]: audit 2023-09-23T18:57:06.145266+0000 mon.a (mon.0) 130 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:57:10.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:10 smithi016 bash[27237]: cluster 2023-09-23T18:57:06.592473+0000 mgr.y (mgr.14146) 30 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:10.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:10 smithi016 bash[27237]: audit 2023-09-23T18:57:07.145383+0000 mon.a (mon.0) 131 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:57:10.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:10 smithi016 bash[27237]: cluster 2023-09-23T18:57:07.147253+0000 mon.b (mon.1) 1 : cluster [INF] mon.b calling monitor election 2023-09-23T18:57:10.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:10 smithi016 bash[27237]: audit 2023-09-23T18:57:08.145186+0000 mon.a (mon.0) 132 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:57:10.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:10 smithi016 bash[27237]: cluster 2023-09-23T18:57:08.592919+0000 mgr.y (mgr.14146) 31 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:10.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:10 smithi016 bash[27237]: audit 2023-09-23T18:57:09.111781+0000 mon.a (mon.0) 133 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:57:10.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:10 smithi016 bash[27237]: audit 2023-09-23T18:57:09.145893+0000 mon.a (mon.0) 134 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:57:10.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:10 smithi016 bash[27237]: audit 2023-09-23T18:57:10.112600+0000 mon.a (mon.0) 135 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:57:10.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:10 smithi016 bash[27237]: audit 2023-09-23T18:57:10.145536+0000 mon.a (mon.0) 136 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:57:10.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:10 smithi016 bash[27237]: cluster 2023-09-23T18:57:10.174379+0000 mon.a (mon.0) 137 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T18:57:10.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:10 smithi016 bash[27237]: cluster 2023-09-23T18:57:10.185109+0000 mon.a (mon.0) 138 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.16:3300/0,v1:172.21.15.16:6789/0],b=[v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0]} removed_ranks: {} 2023-09-23T18:57:10.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:10 smithi016 bash[27237]: cluster 2023-09-23T18:57:10.205158+0000 mon.a (mon.0) 139 : cluster [DBG] fsmap 2023-09-23T18:57:10.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:10 smithi016 bash[27237]: cluster 2023-09-23T18:57:10.205224+0000 mon.a (mon.0) 140 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-09-23T18:57:10.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:10 smithi016 bash[27237]: cluster 2023-09-23T18:57:10.205579+0000 mon.a (mon.0) 141 : cluster [DBG] mgrmap e13: y(active, since 49s) 2023-09-23T18:57:10.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:10 smithi016 bash[27237]: cluster 2023-09-23T18:57:10.212000+0000 mon.a (mon.0) 142 : cluster [INF] overall HEALTH_OK 2023-09-23T18:57:10.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:10 smithi016 bash[27237]: audit 2023-09-23T18:57:10.215429+0000 mon.a (mon.0) 143 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:57:10.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:10 smithi016 bash[27237]: audit 2023-09-23T18:57:10.217430+0000 mon.a (mon.0) 144 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:57:10.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:10 smithi092 bash[27556]: audit 2023-09-23T18:57:05.149637+0000 mon.a (mon.0) 127 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T18:57:10.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:10 smithi092 bash[27556]: audit 2023-09-23T18:57:05.149854+0000 mon.a (mon.0) 128 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:57:10.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:10 smithi092 bash[27556]: cluster 2023-09-23T18:57:05.150023+0000 mon.a (mon.0) 129 : cluster [INF] mon.a calling monitor election 2023-09-23T18:57:10.541 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:10 smithi092 bash[27556]: audit 2023-09-23T18:57:06.145266+0000 mon.a (mon.0) 130 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:57:10.541 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:10 smithi092 bash[27556]: cluster 2023-09-23T18:57:06.592473+0000 mgr.y (mgr.14146) 30 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:10.541 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:10 smithi092 bash[27556]: audit 2023-09-23T18:57:07.145383+0000 mon.a (mon.0) 131 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:57:10.541 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:10 smithi092 bash[27556]: cluster 2023-09-23T18:57:07.147253+0000 mon.b (mon.1) 1 : cluster [INF] mon.b calling monitor election 2023-09-23T18:57:10.541 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:10 smithi092 bash[27556]: audit 2023-09-23T18:57:08.145186+0000 mon.a (mon.0) 132 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:57:10.541 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:10 smithi092 bash[27556]: cluster 2023-09-23T18:57:08.592919+0000 mgr.y (mgr.14146) 31 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:10.541 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:10 smithi092 bash[27556]: audit 2023-09-23T18:57:09.111781+0000 mon.a (mon.0) 133 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:57:10.541 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:10 smithi092 bash[27556]: audit 2023-09-23T18:57:09.145893+0000 mon.a (mon.0) 134 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:57:10.541 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:10 smithi092 bash[27556]: audit 2023-09-23T18:57:10.112600+0000 mon.a (mon.0) 135 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:57:10.541 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:10 smithi092 bash[27556]: audit 2023-09-23T18:57:10.145536+0000 mon.a (mon.0) 136 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:57:10.541 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:10 smithi092 bash[27556]: cluster 2023-09-23T18:57:10.174379+0000 mon.a (mon.0) 137 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T18:57:10.542 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:10 smithi092 bash[27556]: cluster 2023-09-23T18:57:10.185109+0000 mon.a (mon.0) 138 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.16:3300/0,v1:172.21.15.16:6789/0],b=[v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0]} removed_ranks: {} 2023-09-23T18:57:10.542 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:10 smithi092 bash[27556]: cluster 2023-09-23T18:57:10.205158+0000 mon.a (mon.0) 139 : cluster [DBG] fsmap 2023-09-23T18:57:10.542 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:10 smithi092 bash[27556]: cluster 2023-09-23T18:57:10.205224+0000 mon.a (mon.0) 140 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-09-23T18:57:10.542 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:10 smithi092 bash[27556]: cluster 2023-09-23T18:57:10.205579+0000 mon.a (mon.0) 141 : cluster [DBG] mgrmap e13: y(active, since 49s) 2023-09-23T18:57:10.542 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:10 smithi092 bash[27556]: cluster 2023-09-23T18:57:10.212000+0000 mon.a (mon.0) 142 : cluster [INF] overall HEALTH_OK 2023-09-23T18:57:10.542 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:10 smithi092 bash[27556]: audit 2023-09-23T18:57:10.215429+0000 mon.a (mon.0) 143 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:57:10.542 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:10 smithi092 bash[27556]: audit 2023-09-23T18:57:10.217430+0000 mon.a (mon.0) 144 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:57:10.845 INFO:teuthology.orchestra.run.smithi092.stdout: 2023-09-23T18:57:10.845 INFO:teuthology.orchestra.run.smithi092.stdout:{"epoch":2,"fsid":"95ce9eb6-5a42-11ee-8db4-212e2dc638e7","modified":"2023-09-23T18:57:05.145189Z","created":"2023-09-23T18:55:19.600161Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:3300","nonce":0},{"type":"v1","addr":"172.21.15.16:6789","nonce":0}]},"addr":"172.21.15.16:6789/0","public_addr":"172.21.15.16:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:3300","nonce":0},{"type":"v1","addr":"172.21.15.92:6789","nonce":0}]},"addr":"172.21.15.92:6789/0","public_addr":"172.21.15.92:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2023-09-23T18:57:10.850 INFO:teuthology.orchestra.run.smithi092.stderr:dumped monmap epoch 2 2023-09-23T18:57:11.532 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 18:57:11 smithi016 bash[27503]: debug 2023-09-23T18:57:11.142+0000 7f85c17cd700 -1 mgr.server handle_report got status from non-daemon mon.b 2023-09-23T18:57:12.505 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-09-23T18:57:12.505 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph mon dump -f json 2023-09-23T18:57:16.465 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[27237]: audit 2023-09-23T18:57:11.119188+0000 mon.a (mon.0) 148 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T18:57:16.465 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[27237]: cluster 2023-09-23T18:57:11.119517+0000 mon.a (mon.0) 149 : cluster [INF] mon.a calling monitor election 2023-09-23T18:57:16.465 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[27237]: cluster 2023-09-23T18:57:11.124244+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2023-09-23T18:57:16.465 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[27237]: audit 2023-09-23T18:57:11.137765+0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:57:16.465 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[27237]: audit 2023-09-23T18:57:11.138079+0000 mon.a (mon.0) 151 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:57:16.465 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[27237]: audit 2023-09-23T18:57:12.112974+0000 mon.a (mon.0) 152 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:57:16.466 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[27237]: cluster 2023-09-23T18:57:12.593779+0000 mgr.y (mgr.14146) 33 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:16.466 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[27237]: audit 2023-09-23T18:57:13.113388+0000 mon.a (mon.0) 153 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:57:16.466 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[27237]: cluster 2023-09-23T18:57:13.115528+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-09-23T18:57:16.466 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[27237]: audit 2023-09-23T18:57:14.113099+0000 mon.a (mon.0) 154 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:57:16.466 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[27237]: cluster 2023-09-23T18:57:14.594204+0000 mgr.y (mgr.14146) 34 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:16.466 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[27237]: audit 2023-09-23T18:57:15.113412+0000 mon.a (mon.0) 155 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:57:16.466 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[27237]: audit 2023-09-23T18:57:16.113659+0000 mon.a (mon.0) 156 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:57:16.466 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[27237]: cluster 2023-09-23T18:57:16.158158+0000 mon.a (mon.0) 157 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T18:57:16.466 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[27237]: cluster 2023-09-23T18:57:16.165590+0000 mon.a (mon.0) 158 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.16:3300/0,v1:172.21.15.16:6789/0],b=[v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0],c=[v2:172.21.15.16:3301/0,v1:172.21.15.16:6790/0]} removed_ranks: {} 2023-09-23T18:57:16.466 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[27237]: cluster 2023-09-23T18:57:16.185031+0000 mon.a (mon.0) 159 : cluster [DBG] fsmap 2023-09-23T18:57:16.466 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[27237]: cluster 2023-09-23T18:57:16.185087+0000 mon.a (mon.0) 160 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-09-23T18:57:16.466 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[27237]: cluster 2023-09-23T18:57:16.185403+0000 mon.a (mon.0) 161 : cluster [DBG] mgrmap e13: y(active, since 55s) 2023-09-23T18:57:16.467 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[27237]: cluster 2023-09-23T18:57:16.191579+0000 mon.a (mon.0) 162 : cluster [INF] overall HEALTH_OK 2023-09-23T18:57:16.467 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[27237]: audit 2023-09-23T18:57:16.195256+0000 mon.a (mon.0) 163 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:57:16.467 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[27237]: audit 2023-09-23T18:57:16.198660+0000 mon.a (mon.0) 164 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:57:16.467 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[27237]: audit 2023-09-23T18:57:16.199823+0000 mon.a (mon.0) 165 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:57:16.467 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[27237]: audit 2023-09-23T18:57:16.201216+0000 mon.a (mon.0) 166 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:57:16.467 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[27237]: audit 2023-09-23T18:57:16.227302+0000 mon.a (mon.0) 167 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:57:16.467 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[27237]: audit 2023-09-23T18:57:16.232585+0000 mon.a (mon.0) 168 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:57:16.467 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[33624]: audit 2023-09-23T18:57:05.149637+0000 mon.a (mon.0) 127 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T18:57:16.468 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[33624]: audit 2023-09-23T18:57:05.149854+0000 mon.a (mon.0) 128 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:57:16.468 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[33624]: cluster 2023-09-23T18:57:05.150023+0000 mon.a (mon.0) 129 : cluster [INF] mon.a calling monitor election 2023-09-23T18:57:16.468 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[33624]: audit 2023-09-23T18:57:06.145266+0000 mon.a (mon.0) 130 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:57:16.468 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[33624]: cluster 2023-09-23T18:57:06.592473+0000 mgr.y (mgr.14146) 30 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:16.468 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[33624]: audit 2023-09-23T18:57:07.145383+0000 mon.a (mon.0) 131 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:57:16.468 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[33624]: cluster 2023-09-23T18:57:07.147253+0000 mon.b (mon.1) 1 : cluster [INF] mon.b calling monitor election 2023-09-23T18:57:16.468 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[33624]: audit 2023-09-23T18:57:08.145186+0000 mon.a (mon.0) 132 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:57:16.468 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[33624]: cluster 2023-09-23T18:57:08.592919+0000 mgr.y (mgr.14146) 31 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:16.468 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[33624]: audit 2023-09-23T18:57:09.111781+0000 mon.a (mon.0) 133 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:57:16.468 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[33624]: audit 2023-09-23T18:57:09.145893+0000 mon.a (mon.0) 134 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:57:16.468 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[33624]: audit 2023-09-23T18:57:10.112600+0000 mon.a (mon.0) 135 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:57:16.469 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[33624]: audit 2023-09-23T18:57:10.145536+0000 mon.a (mon.0) 136 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:57:16.469 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[33624]: cluster 2023-09-23T18:57:10.174379+0000 mon.a (mon.0) 137 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T18:57:16.469 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[33624]: cluster 2023-09-23T18:57:10.185109+0000 mon.a (mon.0) 138 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.16:3300/0,v1:172.21.15.16:6789/0],b=[v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0]} removed_ranks: {} 2023-09-23T18:57:16.469 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[33624]: cluster 2023-09-23T18:57:10.205158+0000 mon.a (mon.0) 139 : cluster [DBG] fsmap 2023-09-23T18:57:16.469 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[33624]: cluster 2023-09-23T18:57:10.205224+0000 mon.a (mon.0) 140 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-09-23T18:57:16.469 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[33624]: cluster 2023-09-23T18:57:10.205579+0000 mon.a (mon.0) 141 : cluster [DBG] mgrmap e13: y(active, since 49s) 2023-09-23T18:57:16.469 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[33624]: cluster 2023-09-23T18:57:10.212000+0000 mon.a (mon.0) 142 : cluster [INF] overall HEALTH_OK 2023-09-23T18:57:16.469 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[33624]: audit 2023-09-23T18:57:10.215429+0000 mon.a (mon.0) 143 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:57:16.469 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[33624]: audit 2023-09-23T18:57:10.217430+0000 mon.a (mon.0) 144 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:57:16.469 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[33624]: audit 2023-09-23T18:57:11.119188+0000 mon.a (mon.0) 148 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T18:57:16.469 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[33624]: cluster 2023-09-23T18:57:11.119517+0000 mon.a (mon.0) 149 : cluster [INF] mon.a calling monitor election 2023-09-23T18:57:16.469 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[33624]: cluster 2023-09-23T18:57:11.124244+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2023-09-23T18:57:16.469 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[33624]: audit 2023-09-23T18:57:11.137765+0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:57:16.470 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[33624]: audit 2023-09-23T18:57:11.138079+0000 mon.a (mon.0) 151 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:57:16.470 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[33624]: audit 2023-09-23T18:57:12.112974+0000 mon.a (mon.0) 152 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:57:16.470 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[33624]: cluster 2023-09-23T18:57:12.593779+0000 mgr.y (mgr.14146) 33 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:16.470 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[33624]: audit 2023-09-23T18:57:13.113388+0000 mon.a (mon.0) 153 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:57:16.470 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[33624]: cluster 2023-09-23T18:57:13.115528+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-09-23T18:57:16.470 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[33624]: audit 2023-09-23T18:57:14.113099+0000 mon.a (mon.0) 154 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:57:16.470 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[33624]: cluster 2023-09-23T18:57:14.594204+0000 mgr.y (mgr.14146) 34 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:16.470 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[33624]: audit 2023-09-23T18:57:15.113412+0000 mon.a (mon.0) 155 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:57:16.470 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[33624]: audit 2023-09-23T18:57:16.113659+0000 mon.a (mon.0) 156 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:57:16.470 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[33624]: cluster 2023-09-23T18:57:16.158158+0000 mon.a (mon.0) 157 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T18:57:16.470 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[33624]: cluster 2023-09-23T18:57:16.165590+0000 mon.a (mon.0) 158 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.16:3300/0,v1:172.21.15.16:6789/0],b=[v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0],c=[v2:172.21.15.16:3301/0,v1:172.21.15.16:6790/0]} removed_ranks: {} 2023-09-23T18:57:16.470 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[33624]: cluster 2023-09-23T18:57:16.185031+0000 mon.a (mon.0) 159 : cluster [DBG] fsmap 2023-09-23T18:57:16.471 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[33624]: cluster 2023-09-23T18:57:16.185087+0000 mon.a (mon.0) 160 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-09-23T18:57:16.471 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[33624]: cluster 2023-09-23T18:57:16.185403+0000 mon.a (mon.0) 161 : cluster [DBG] mgrmap e13: y(active, since 55s) 2023-09-23T18:57:16.471 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[33624]: cluster 2023-09-23T18:57:16.191579+0000 mon.a (mon.0) 162 : cluster [INF] overall HEALTH_OK 2023-09-23T18:57:16.471 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[33624]: audit 2023-09-23T18:57:16.195256+0000 mon.a (mon.0) 163 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:57:16.471 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[33624]: audit 2023-09-23T18:57:16.198660+0000 mon.a (mon.0) 164 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:57:16.471 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[33624]: audit 2023-09-23T18:57:16.199823+0000 mon.a (mon.0) 165 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:57:16.471 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[33624]: audit 2023-09-23T18:57:16.201216+0000 mon.a (mon.0) 166 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:57:16.471 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[33624]: audit 2023-09-23T18:57:16.227302+0000 mon.a (mon.0) 167 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:57:16.471 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:16 smithi016 bash[33624]: audit 2023-09-23T18:57:16.232585+0000 mon.a (mon.0) 168 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:57:16.541 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:16 smithi092 bash[27556]: audit 2023-09-23T18:57:11.119188+0000 mon.a (mon.0) 148 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T18:57:16.541 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:16 smithi092 bash[27556]: cluster 2023-09-23T18:57:11.119517+0000 mon.a (mon.0) 149 : cluster [INF] mon.a calling monitor election 2023-09-23T18:57:16.541 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:16 smithi092 bash[27556]: cluster 2023-09-23T18:57:11.124244+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2023-09-23T18:57:16.541 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:16 smithi092 bash[27556]: audit 2023-09-23T18:57:11.137765+0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:57:16.541 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:16 smithi092 bash[27556]: audit 2023-09-23T18:57:11.138079+0000 mon.a (mon.0) 151 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:57:16.541 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:16 smithi092 bash[27556]: audit 2023-09-23T18:57:12.112974+0000 mon.a (mon.0) 152 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:57:16.541 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:16 smithi092 bash[27556]: cluster 2023-09-23T18:57:12.593779+0000 mgr.y (mgr.14146) 33 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:16.541 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:16 smithi092 bash[27556]: audit 2023-09-23T18:57:13.113388+0000 mon.a (mon.0) 153 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:57:16.541 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:16 smithi092 bash[27556]: cluster 2023-09-23T18:57:13.115528+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-09-23T18:57:16.542 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:16 smithi092 bash[27556]: audit 2023-09-23T18:57:14.113099+0000 mon.a (mon.0) 154 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:57:16.542 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:16 smithi092 bash[27556]: cluster 2023-09-23T18:57:14.594204+0000 mgr.y (mgr.14146) 34 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:16.542 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:16 smithi092 bash[27556]: audit 2023-09-23T18:57:15.113412+0000 mon.a (mon.0) 155 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:57:16.542 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:16 smithi092 bash[27556]: audit 2023-09-23T18:57:16.113659+0000 mon.a (mon.0) 156 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:57:16.542 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:16 smithi092 bash[27556]: cluster 2023-09-23T18:57:16.158158+0000 mon.a (mon.0) 157 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T18:57:16.542 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:16 smithi092 bash[27556]: cluster 2023-09-23T18:57:16.165590+0000 mon.a (mon.0) 158 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.16:3300/0,v1:172.21.15.16:6789/0],b=[v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0],c=[v2:172.21.15.16:3301/0,v1:172.21.15.16:6790/0]} removed_ranks: {} 2023-09-23T18:57:16.542 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:16 smithi092 bash[27556]: cluster 2023-09-23T18:57:16.185031+0000 mon.a (mon.0) 159 : cluster [DBG] fsmap 2023-09-23T18:57:16.542 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:16 smithi092 bash[27556]: cluster 2023-09-23T18:57:16.185087+0000 mon.a (mon.0) 160 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-09-23T18:57:16.542 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:16 smithi092 bash[27556]: cluster 2023-09-23T18:57:16.185403+0000 mon.a (mon.0) 161 : cluster [DBG] mgrmap e13: y(active, since 55s) 2023-09-23T18:57:16.542 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:16 smithi092 bash[27556]: cluster 2023-09-23T18:57:16.191579+0000 mon.a (mon.0) 162 : cluster [INF] overall HEALTH_OK 2023-09-23T18:57:16.542 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:16 smithi092 bash[27556]: audit 2023-09-23T18:57:16.195256+0000 mon.a (mon.0) 163 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:57:16.542 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:16 smithi092 bash[27556]: audit 2023-09-23T18:57:16.198660+0000 mon.a (mon.0) 164 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:57:16.543 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:16 smithi092 bash[27556]: audit 2023-09-23T18:57:16.199823+0000 mon.a (mon.0) 165 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:57:16.543 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:16 smithi092 bash[27556]: audit 2023-09-23T18:57:16.201216+0000 mon.a (mon.0) 166 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:57:16.543 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:16 smithi092 bash[27556]: audit 2023-09-23T18:57:16.227302+0000 mon.a (mon.0) 167 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:57:16.543 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:16 smithi092 bash[27556]: audit 2023-09-23T18:57:16.232585+0000 mon.a (mon.0) 168 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:57:16.725 INFO:teuthology.orchestra.run.smithi092.stdout: 2023-09-23T18:57:16.725 INFO:teuthology.orchestra.run.smithi092.stdout:{"epoch":3,"fsid":"95ce9eb6-5a42-11ee-8db4-212e2dc638e7","modified":"2023-09-23T18:57:11.113400Z","created":"2023-09-23T18:55:19.600161Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:3300","nonce":0},{"type":"v1","addr":"172.21.15.16:6789","nonce":0}]},"addr":"172.21.15.16:6789/0","public_addr":"172.21.15.16:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:3300","nonce":0},{"type":"v1","addr":"172.21.15.92:6789","nonce":0}]},"addr":"172.21.15.92:6789/0","public_addr":"172.21.15.92:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:3301","nonce":0},{"type":"v1","addr":"172.21.15.16:6790","nonce":0}]},"addr":"172.21.15.16:6790/0","public_addr":"172.21.15.16:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2023-09-23T18:57:16.728 INFO:teuthology.orchestra.run.smithi092.stderr:dumped monmap epoch 3 2023-09-23T18:57:17.373 INFO:tasks.cephadm:Generating final ceph.conf file... 2023-09-23T18:57:17.374 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph config generate-minimal-conf 2023-09-23T18:57:17.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:17 smithi016 bash[33624]: cephadm 2023-09-23T18:57:16.203075+0000 mgr.y (mgr.14146) 35 : cephadm [INF] Updating smithi016:/etc/ceph/ceph.conf 2023-09-23T18:57:17.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:17 smithi016 bash[33624]: cephadm 2023-09-23T18:57:16.203502+0000 mgr.y (mgr.14146) 36 : cephadm [INF] Updating smithi092:/etc/ceph/ceph.conf 2023-09-23T18:57:17.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:17 smithi016 bash[33624]: audit 2023-09-23T18:57:16.236949+0000 mon.a (mon.0) 169 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:57:17.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:17 smithi016 bash[33624]: cephadm 2023-09-23T18:57:16.239931+0000 mgr.y (mgr.14146) 37 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2023-09-23T18:57:17.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:17 smithi016 bash[33624]: audit 2023-09-23T18:57:16.244716+0000 mon.a (mon.0) 170 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T18:57:17.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:17 smithi016 bash[33624]: audit 2023-09-23T18:57:16.248210+0000 mon.a (mon.0) 171 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T18:57:17.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:17 smithi016 bash[33624]: audit 2023-09-23T18:57:16.249164+0000 mon.a (mon.0) 172 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:57:17.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:17 smithi016 bash[33624]: cephadm 2023-09-23T18:57:16.250054+0000 mgr.y (mgr.14146) 38 : cephadm [INF] Reconfiguring daemon mon.a on smithi016 2023-09-23T18:57:17.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:17 smithi016 bash[33624]: cluster 2023-09-23T18:57:16.594510+0000 mgr.y (mgr.14146) 39 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:17.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:17 smithi016 bash[33624]: audit 2023-09-23T18:57:16.724485+0000 mon.a (mon.0) 173 : audit [DBG] from='client.? 172.21.15.92:0/3797314709' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T18:57:17.504 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:17 smithi016 bash[33624]: audit 2023-09-23T18:57:17.113651+0000 mon.a (mon.0) 174 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:57:17.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:17 smithi016 bash[27237]: cephadm 2023-09-23T18:57:16.203075+0000 mgr.y (mgr.14146) 35 : cephadm [INF] Updating smithi016:/etc/ceph/ceph.conf 2023-09-23T18:57:17.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:17 smithi016 bash[27237]: cephadm 2023-09-23T18:57:16.203502+0000 mgr.y (mgr.14146) 36 : cephadm [INF] Updating smithi092:/etc/ceph/ceph.conf 2023-09-23T18:57:17.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:17 smithi016 bash[27237]: audit 2023-09-23T18:57:16.236949+0000 mon.a (mon.0) 169 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:57:17.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:17 smithi016 bash[27237]: cephadm 2023-09-23T18:57:16.239931+0000 mgr.y (mgr.14146) 37 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2023-09-23T18:57:17.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:17 smithi016 bash[27237]: audit 2023-09-23T18:57:16.244716+0000 mon.a (mon.0) 170 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T18:57:17.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:17 smithi016 bash[27237]: audit 2023-09-23T18:57:16.248210+0000 mon.a (mon.0) 171 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T18:57:17.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:17 smithi016 bash[27237]: audit 2023-09-23T18:57:16.249164+0000 mon.a (mon.0) 172 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:57:17.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:17 smithi016 bash[27237]: cephadm 2023-09-23T18:57:16.250054+0000 mgr.y (mgr.14146) 38 : cephadm [INF] Reconfiguring daemon mon.a on smithi016 2023-09-23T18:57:17.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:17 smithi016 bash[27237]: cluster 2023-09-23T18:57:16.594510+0000 mgr.y (mgr.14146) 39 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:17.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:17 smithi016 bash[27237]: audit 2023-09-23T18:57:16.724485+0000 mon.a (mon.0) 173 : audit [DBG] from='client.? 172.21.15.92:0/3797314709' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T18:57:17.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:17 smithi016 bash[27237]: audit 2023-09-23T18:57:17.113651+0000 mon.a (mon.0) 174 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:57:17.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:17 smithi092 bash[27556]: cephadm 2023-09-23T18:57:16.203075+0000 mgr.y (mgr.14146) 35 : cephadm [INF] Updating smithi016:/etc/ceph/ceph.conf 2023-09-23T18:57:17.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:17 smithi092 bash[27556]: cephadm 2023-09-23T18:57:16.203502+0000 mgr.y (mgr.14146) 36 : cephadm [INF] Updating smithi092:/etc/ceph/ceph.conf 2023-09-23T18:57:17.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:17 smithi092 bash[27556]: audit 2023-09-23T18:57:16.236949+0000 mon.a (mon.0) 169 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:57:17.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:17 smithi092 bash[27556]: cephadm 2023-09-23T18:57:16.239931+0000 mgr.y (mgr.14146) 37 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2023-09-23T18:57:17.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:17 smithi092 bash[27556]: audit 2023-09-23T18:57:16.244716+0000 mon.a (mon.0) 170 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T18:57:17.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:17 smithi092 bash[27556]: audit 2023-09-23T18:57:16.248210+0000 mon.a (mon.0) 171 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T18:57:17.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:17 smithi092 bash[27556]: audit 2023-09-23T18:57:16.249164+0000 mon.a (mon.0) 172 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:57:17.541 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:17 smithi092 bash[27556]: cephadm 2023-09-23T18:57:16.250054+0000 mgr.y (mgr.14146) 38 : cephadm [INF] Reconfiguring daemon mon.a on smithi016 2023-09-23T18:57:17.541 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:17 smithi092 bash[27556]: cluster 2023-09-23T18:57:16.594510+0000 mgr.y (mgr.14146) 39 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:17.541 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:17 smithi092 bash[27556]: audit 2023-09-23T18:57:16.724485+0000 mon.a (mon.0) 173 : audit [DBG] from='client.? 172.21.15.92:0/3797314709' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T18:57:17.541 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:17 smithi092 bash[27556]: audit 2023-09-23T18:57:17.113651+0000 mon.a (mon.0) 174 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:57:19.782 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:19 smithi016 bash[33624]: audit 2023-09-23T18:57:18.466621+0000 mon.a (mon.0) 175 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:57:19.783 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:19 smithi016 bash[33624]: cephadm 2023-09-23T18:57:18.467673+0000 mgr.y (mgr.14146) 40 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2023-09-23T18:57:19.783 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:19 smithi016 bash[33624]: audit 2023-09-23T18:57:18.468082+0000 mon.a (mon.0) 176 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T18:57:19.783 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:19 smithi016 bash[33624]: audit 2023-09-23T18:57:18.469221+0000 mon.a (mon.0) 177 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T18:57:19.783 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:19 smithi016 bash[33624]: audit 2023-09-23T18:57:18.470431+0000 mon.a (mon.0) 178 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:57:19.783 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:19 smithi016 bash[33624]: cephadm 2023-09-23T18:57:18.471413+0000 mgr.y (mgr.14146) 41 : cephadm [INF] Reconfiguring daemon mon.c on smithi016 2023-09-23T18:57:19.783 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:19 smithi016 bash[33624]: cluster 2023-09-23T18:57:18.594904+0000 mgr.y (mgr.14146) 42 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:19.783 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:19 smithi016 bash[27237]: audit 2023-09-23T18:57:18.466621+0000 mon.a (mon.0) 175 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:57:19.783 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:19 smithi016 bash[27237]: cephadm 2023-09-23T18:57:18.467673+0000 mgr.y (mgr.14146) 40 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2023-09-23T18:57:19.783 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:19 smithi016 bash[27237]: audit 2023-09-23T18:57:18.468082+0000 mon.a (mon.0) 176 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T18:57:19.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:19 smithi016 bash[27237]: audit 2023-09-23T18:57:18.469221+0000 mon.a (mon.0) 177 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T18:57:19.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:19 smithi016 bash[27237]: audit 2023-09-23T18:57:18.470431+0000 mon.a (mon.0) 178 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:57:19.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:19 smithi016 bash[27237]: cephadm 2023-09-23T18:57:18.471413+0000 mgr.y (mgr.14146) 41 : cephadm [INF] Reconfiguring daemon mon.c on smithi016 2023-09-23T18:57:19.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:19 smithi016 bash[27237]: cluster 2023-09-23T18:57:18.594904+0000 mgr.y (mgr.14146) 42 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:19.790 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:19 smithi092 bash[27556]: audit 2023-09-23T18:57:18.466621+0000 mon.a (mon.0) 175 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:57:19.790 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:19 smithi092 bash[27556]: cephadm 2023-09-23T18:57:18.467673+0000 mgr.y (mgr.14146) 40 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2023-09-23T18:57:19.790 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:19 smithi092 bash[27556]: audit 2023-09-23T18:57:18.468082+0000 mon.a (mon.0) 176 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T18:57:19.790 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:19 smithi092 bash[27556]: audit 2023-09-23T18:57:18.469221+0000 mon.a (mon.0) 177 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T18:57:19.790 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:19 smithi092 bash[27556]: audit 2023-09-23T18:57:18.470431+0000 mon.a (mon.0) 178 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:57:19.790 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:19 smithi092 bash[27556]: cephadm 2023-09-23T18:57:18.471413+0000 mgr.y (mgr.14146) 41 : cephadm [INF] Reconfiguring daemon mon.c on smithi016 2023-09-23T18:57:19.790 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:19 smithi092 bash[27556]: cluster 2023-09-23T18:57:18.594904+0000 mgr.y (mgr.14146) 42 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:21.856 INFO:teuthology.orchestra.run.smithi016.stdout:# minimal ceph.conf for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 2023-09-23T18:57:21.856 INFO:teuthology.orchestra.run.smithi016.stdout:[global] 2023-09-23T18:57:21.856 INFO:teuthology.orchestra.run.smithi016.stdout: fsid = 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 2023-09-23T18:57:21.856 INFO:teuthology.orchestra.run.smithi016.stdout: mon_host = [v2:172.21.15.16:3300/0,v1:172.21.15.16:6789/0] [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] [v2:172.21.15.16:3301/0,v1:172.21.15.16:6790/0] 2023-09-23T18:57:22.032 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:21 smithi016 bash[33624]: cluster 2023-09-23T18:57:20.595366+0000 mgr.y (mgr.14146) 43 : cluster [DBG] pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:22.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:21 smithi016 bash[33624]: audit 2023-09-23T18:57:21.403675+0000 mon.a (mon.0) 179 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:57:22.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:21 smithi016 bash[33624]: audit 2023-09-23T18:57:21.404888+0000 mon.a (mon.0) 180 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T18:57:22.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:21 smithi016 bash[33624]: audit 2023-09-23T18:57:21.406064+0000 mon.a (mon.0) 181 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T18:57:22.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:21 smithi016 bash[33624]: audit 2023-09-23T18:57:21.407287+0000 mon.a (mon.0) 182 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:57:22.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:21 smithi016 bash[27237]: cluster 2023-09-23T18:57:20.595366+0000 mgr.y (mgr.14146) 43 : cluster [DBG] pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:22.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:21 smithi016 bash[27237]: audit 2023-09-23T18:57:21.403675+0000 mon.a (mon.0) 179 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:57:22.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:21 smithi016 bash[27237]: audit 2023-09-23T18:57:21.404888+0000 mon.a (mon.0) 180 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T18:57:22.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:21 smithi016 bash[27237]: audit 2023-09-23T18:57:21.406064+0000 mon.a (mon.0) 181 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T18:57:22.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:21 smithi016 bash[27237]: audit 2023-09-23T18:57:21.407287+0000 mon.a (mon.0) 182 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:57:22.042 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:21 smithi092 bash[27556]: cluster 2023-09-23T18:57:20.595366+0000 mgr.y (mgr.14146) 43 : cluster [DBG] pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:22.042 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:21 smithi092 bash[27556]: audit 2023-09-23T18:57:21.403675+0000 mon.a (mon.0) 179 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:57:22.042 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:21 smithi092 bash[27556]: audit 2023-09-23T18:57:21.404888+0000 mon.a (mon.0) 180 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T18:57:22.042 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:21 smithi092 bash[27556]: audit 2023-09-23T18:57:21.406064+0000 mon.a (mon.0) 181 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T18:57:22.043 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:21 smithi092 bash[27556]: audit 2023-09-23T18:57:21.407287+0000 mon.a (mon.0) 182 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:57:22.562 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2023-09-23T18:57:22.562 DEBUG:teuthology.orchestra.run.smithi016:> set -ex 2023-09-23T18:57:22.562 DEBUG:teuthology.orchestra.run.smithi016:> sudo dd of=/etc/ceph/ceph.conf 2023-09-23T18:57:22.577 DEBUG:teuthology.orchestra.run.smithi016:> set -ex 2023-09-23T18:57:22.577 DEBUG:teuthology.orchestra.run.smithi016:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-09-23T18:57:22.634 DEBUG:teuthology.orchestra.run.smithi092:> set -ex 2023-09-23T18:57:22.634 DEBUG:teuthology.orchestra.run.smithi092:> sudo dd of=/etc/ceph/ceph.conf 2023-09-23T18:57:22.647 DEBUG:teuthology.orchestra.run.smithi092:> set -ex 2023-09-23T18:57:22.647 DEBUG:teuthology.orchestra.run.smithi092:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-09-23T18:57:22.704 INFO:tasks.cephadm:Adding mgr.y on smithi016 2023-09-23T18:57:22.704 INFO:tasks.cephadm:Adding mgr.x on smithi092 2023-09-23T18:57:22.704 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph orch apply mgr '2;smithi016=y;smithi092=x' 2023-09-23T18:57:23.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:22 smithi016 bash[27237]: cephadm 2023-09-23T18:57:21.404520+0000 mgr.y (mgr.14146) 44 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2023-09-23T18:57:23.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:22 smithi016 bash[27237]: cephadm 2023-09-23T18:57:21.408384+0000 mgr.y (mgr.14146) 45 : cephadm [INF] Reconfiguring daemon mon.b on smithi092 2023-09-23T18:57:23.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:22 smithi016 bash[27237]: audit 2023-09-23T18:57:21.855617+0000 mon.b (mon.1) 3 : audit [DBG] from='client.? 172.21.15.16:0/2902288067' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:57:23.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:22 smithi016 bash[33624]: cephadm 2023-09-23T18:57:21.404520+0000 mgr.y (mgr.14146) 44 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2023-09-23T18:57:23.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:22 smithi016 bash[33624]: cephadm 2023-09-23T18:57:21.408384+0000 mgr.y (mgr.14146) 45 : cephadm [INF] Reconfiguring daemon mon.b on smithi092 2023-09-23T18:57:23.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:22 smithi016 bash[33624]: audit 2023-09-23T18:57:21.855617+0000 mon.b (mon.1) 3 : audit [DBG] from='client.? 172.21.15.16:0/2902288067' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:57:23.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:22 smithi092 bash[27556]: cephadm 2023-09-23T18:57:21.404520+0000 mgr.y (mgr.14146) 44 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2023-09-23T18:57:23.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:22 smithi092 bash[27556]: cephadm 2023-09-23T18:57:21.408384+0000 mgr.y (mgr.14146) 45 : cephadm [INF] Reconfiguring daemon mon.b on smithi092 2023-09-23T18:57:23.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:22 smithi092 bash[27556]: audit 2023-09-23T18:57:21.855617+0000 mon.b (mon.1) 3 : audit [DBG] from='client.? 172.21.15.16:0/2902288067' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:57:24.028 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:23 smithi092 bash[27556]: cluster 2023-09-23T18:57:22.595843+0000 mgr.y (mgr.14146) 46 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:24.028 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:23 smithi092 bash[27556]: audit 2023-09-23T18:57:23.486388+0000 mon.a (mon.0) 183 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:57:24.028 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:23 smithi092 bash[27556]: audit 2023-09-23T18:57:23.488457+0000 mon.a (mon.0) 184 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:57:24.028 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:23 smithi092 bash[27556]: audit 2023-09-23T18:57:23.490845+0000 mon.a (mon.0) 185 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:57:24.028 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:23 smithi092 bash[27556]: audit 2023-09-23T18:57:23.492264+0000 mon.a (mon.0) 186 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:57:24.028 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:23 smithi092 bash[27556]: audit 2023-09-23T18:57:23.498255+0000 mon.a (mon.0) 187 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:57:24.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:23 smithi016 bash[27237]: cluster 2023-09-23T18:57:22.595843+0000 mgr.y (mgr.14146) 46 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:24.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:23 smithi016 bash[27237]: audit 2023-09-23T18:57:23.486388+0000 mon.a (mon.0) 183 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:57:24.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:23 smithi016 bash[27237]: audit 2023-09-23T18:57:23.488457+0000 mon.a (mon.0) 184 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:57:24.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:23 smithi016 bash[27237]: audit 2023-09-23T18:57:23.490845+0000 mon.a (mon.0) 185 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:57:24.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:23 smithi016 bash[27237]: audit 2023-09-23T18:57:23.492264+0000 mon.a (mon.0) 186 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:57:24.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:23 smithi016 bash[27237]: audit 2023-09-23T18:57:23.498255+0000 mon.a (mon.0) 187 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:57:24.283 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:23 smithi016 bash[33624]: cluster 2023-09-23T18:57:22.595843+0000 mgr.y (mgr.14146) 46 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:24.283 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:23 smithi016 bash[33624]: audit 2023-09-23T18:57:23.486388+0000 mon.a (mon.0) 183 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:57:24.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:23 smithi016 bash[33624]: audit 2023-09-23T18:57:23.488457+0000 mon.a (mon.0) 184 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:57:24.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:23 smithi016 bash[33624]: audit 2023-09-23T18:57:23.490845+0000 mon.a (mon.0) 185 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:57:24.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:23 smithi016 bash[33624]: audit 2023-09-23T18:57:23.492264+0000 mon.a (mon.0) 186 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:57:24.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:23 smithi016 bash[33624]: audit 2023-09-23T18:57:23.498255+0000 mon.a (mon.0) 187 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:57:25.600 INFO:teuthology.orchestra.run.smithi092.stdout:Scheduled mgr update... 2023-09-23T18:57:26.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:25 smithi016 bash[27237]: cluster 2023-09-23T18:57:24.596303+0000 mgr.y (mgr.14146) 47 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:26.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:25 smithi016 bash[27237]: audit 2023-09-23T18:57:25.598680+0000 mon.a (mon.0) 188 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:57:26.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:25 smithi016 bash[27237]: audit 2023-09-23T18:57:25.621493+0000 mon.a (mon.0) 189 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:57:26.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:25 smithi016 bash[27237]: audit 2023-09-23T18:57:25.624810+0000 mon.a (mon.0) 190 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:57:26.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:25 smithi016 bash[27237]: audit 2023-09-23T18:57:25.626401+0000 mon.a (mon.0) 191 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:57:26.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:25 smithi016 bash[27237]: audit 2023-09-23T18:57:25.634349+0000 mon.a (mon.0) 192 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:57:26.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:25 smithi016 bash[27237]: audit 2023-09-23T18:57:25.636637+0000 mon.a (mon.0) 193 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T18:57:26.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:25 smithi016 bash[27237]: audit 2023-09-23T18:57:25.640063+0000 mon.a (mon.0) 194 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-09-23T18:57:26.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:25 smithi016 bash[27237]: audit 2023-09-23T18:57:25.641158+0000 mon.a (mon.0) 195 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T18:57:26.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:25 smithi016 bash[27237]: audit 2023-09-23T18:57:25.642377+0000 mon.a (mon.0) 196 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:57:26.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:25 smithi016 bash[33624]: cluster 2023-09-23T18:57:24.596303+0000 mgr.y (mgr.14146) 47 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:26.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:25 smithi016 bash[33624]: audit 2023-09-23T18:57:25.598680+0000 mon.a (mon.0) 188 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:57:26.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:25 smithi016 bash[33624]: audit 2023-09-23T18:57:25.621493+0000 mon.a (mon.0) 189 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:57:26.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:25 smithi016 bash[33624]: audit 2023-09-23T18:57:25.624810+0000 mon.a (mon.0) 190 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:57:26.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:25 smithi016 bash[33624]: audit 2023-09-23T18:57:25.626401+0000 mon.a (mon.0) 191 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:57:26.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:25 smithi016 bash[33624]: audit 2023-09-23T18:57:25.634349+0000 mon.a (mon.0) 192 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:57:26.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:25 smithi016 bash[33624]: audit 2023-09-23T18:57:25.636637+0000 mon.a (mon.0) 193 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T18:57:26.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:25 smithi016 bash[33624]: audit 2023-09-23T18:57:25.640063+0000 mon.a (mon.0) 194 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-09-23T18:57:26.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:25 smithi016 bash[33624]: audit 2023-09-23T18:57:25.641158+0000 mon.a (mon.0) 195 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T18:57:26.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:25 smithi016 bash[33624]: audit 2023-09-23T18:57:25.642377+0000 mon.a (mon.0) 196 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:57:26.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:25 smithi092 bash[27556]: cluster 2023-09-23T18:57:24.596303+0000 mgr.y (mgr.14146) 47 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:26.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:25 smithi092 bash[27556]: audit 2023-09-23T18:57:25.598680+0000 mon.a (mon.0) 188 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:57:26.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:25 smithi092 bash[27556]: audit 2023-09-23T18:57:25.621493+0000 mon.a (mon.0) 189 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:57:26.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:25 smithi092 bash[27556]: audit 2023-09-23T18:57:25.624810+0000 mon.a (mon.0) 190 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:57:26.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:25 smithi092 bash[27556]: audit 2023-09-23T18:57:25.626401+0000 mon.a (mon.0) 191 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:57:26.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:25 smithi092 bash[27556]: audit 2023-09-23T18:57:25.634349+0000 mon.a (mon.0) 192 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:57:26.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:25 smithi092 bash[27556]: audit 2023-09-23T18:57:25.636637+0000 mon.a (mon.0) 193 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T18:57:26.041 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:25 smithi092 bash[27556]: audit 2023-09-23T18:57:25.640063+0000 mon.a (mon.0) 194 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-09-23T18:57:26.041 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:25 smithi092 bash[27556]: audit 2023-09-23T18:57:25.641158+0000 mon.a (mon.0) 195 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T18:57:26.041 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:25 smithi092 bash[27556]: audit 2023-09-23T18:57:25.642377+0000 mon.a (mon.0) 196 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:57:26.409 DEBUG:teuthology.orchestra.run.smithi092:mgr.x> sudo journalctl -f -n 0 -u ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@mgr.x.service 2023-09-23T18:57:26.411 INFO:tasks.cephadm:Deploying OSDs... 2023-09-23T18:57:26.411 DEBUG:teuthology.orchestra.run.smithi016:> set -ex 2023-09-23T18:57:26.411 DEBUG:teuthology.orchestra.run.smithi016:> dd if=/scratch_devs of=/dev/stdout 2023-09-23T18:57:26.419 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-09-23T18:57:26.419 DEBUG:teuthology.orchestra.run.smithi016:> stat /dev/vg_nvme/lv_1 2023-09-23T18:57:26.471 INFO:teuthology.orchestra.run.smithi016.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-09-23T18:57:26.471 INFO:teuthology.orchestra.run.smithi016.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T18:57:26.471 INFO:teuthology.orchestra.run.smithi016.stdout:Device: 5h/5d Inode: 495 Links: 1 2023-09-23T18:57:26.471 INFO:teuthology.orchestra.run.smithi016.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T18:57:26.471 INFO:teuthology.orchestra.run.smithi016.stdout:Access: 2023-09-23 18:50:40.402111794 +0000 2023-09-23T18:57:26.471 INFO:teuthology.orchestra.run.smithi016.stdout:Modify: 2023-09-23 18:50:40.070119451 +0000 2023-09-23T18:57:26.471 INFO:teuthology.orchestra.run.smithi016.stdout:Change: 2023-09-23 18:50:40.070119451 +0000 2023-09-23T18:57:26.471 INFO:teuthology.orchestra.run.smithi016.stdout: Birth: - 2023-09-23T18:57:26.472 DEBUG:teuthology.orchestra.run.smithi016:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-09-23T18:57:26.526 INFO:teuthology.orchestra.run.smithi016.stderr:1+0 records in 2023-09-23T18:57:26.526 INFO:teuthology.orchestra.run.smithi016.stderr:1+0 records out 2023-09-23T18:57:26.526 INFO:teuthology.orchestra.run.smithi016.stderr:512 bytes copied, 0.000281457 s, 1.8 MB/s 2023-09-23T18:57:26.527 DEBUG:teuthology.orchestra.run.smithi016:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-09-23T18:57:26.576 DEBUG:teuthology.orchestra.run.smithi016:> stat /dev/vg_nvme/lv_2 2023-09-23T18:57:26.627 INFO:teuthology.orchestra.run.smithi016.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-09-23T18:57:26.627 INFO:teuthology.orchestra.run.smithi016.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T18:57:26.627 INFO:teuthology.orchestra.run.smithi016.stdout:Device: 5h/5d Inode: 502 Links: 1 2023-09-23T18:57:26.627 INFO:teuthology.orchestra.run.smithi016.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T18:57:26.627 INFO:teuthology.orchestra.run.smithi016.stdout:Access: 2023-09-23 18:50:40.938099433 +0000 2023-09-23T18:57:26.627 INFO:teuthology.orchestra.run.smithi016.stdout:Modify: 2023-09-23 18:50:40.622106721 +0000 2023-09-23T18:57:26.627 INFO:teuthology.orchestra.run.smithi016.stdout:Change: 2023-09-23 18:50:40.622106721 +0000 2023-09-23T18:57:26.627 INFO:teuthology.orchestra.run.smithi016.stdout: Birth: - 2023-09-23T18:57:26.628 DEBUG:teuthology.orchestra.run.smithi016:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-09-23T18:57:26.686 INFO:teuthology.orchestra.run.smithi016.stderr:1+0 records in 2023-09-23T18:57:26.686 INFO:teuthology.orchestra.run.smithi016.stderr:1+0 records out 2023-09-23T18:57:26.687 INFO:teuthology.orchestra.run.smithi016.stderr:512 bytes copied, 0.000306031 s, 1.7 MB/s 2023-09-23T18:57:26.688 DEBUG:teuthology.orchestra.run.smithi016:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-09-23T18:57:26.741 DEBUG:teuthology.orchestra.run.smithi016:> stat /dev/vg_nvme/lv_3 2023-09-23T18:57:26.785 INFO:journalctl@ceph.mgr.x.smithi092.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-09-23T18:57:26.791 INFO:teuthology.orchestra.run.smithi016.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-09-23T18:57:26.791 INFO:teuthology.orchestra.run.smithi016.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T18:57:26.791 INFO:teuthology.orchestra.run.smithi016.stdout:Device: 5h/5d Inode: 505 Links: 1 2023-09-23T18:57:26.791 INFO:teuthology.orchestra.run.smithi016.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T18:57:26.791 INFO:teuthology.orchestra.run.smithi016.stdout:Access: 2023-09-23 18:50:41.470087165 +0000 2023-09-23T18:57:26.791 INFO:teuthology.orchestra.run.smithi016.stdout:Modify: 2023-09-23 18:50:41.142094729 +0000 2023-09-23T18:57:26.791 INFO:teuthology.orchestra.run.smithi016.stdout:Change: 2023-09-23 18:50:41.142094729 +0000 2023-09-23T18:57:26.792 INFO:teuthology.orchestra.run.smithi016.stdout: Birth: - 2023-09-23T18:57:26.793 DEBUG:teuthology.orchestra.run.smithi016:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-09-23T18:57:26.846 INFO:teuthology.orchestra.run.smithi016.stderr:1+0 records in 2023-09-23T18:57:26.847 INFO:teuthology.orchestra.run.smithi016.stderr:1+0 records out 2023-09-23T18:57:26.847 INFO:teuthology.orchestra.run.smithi016.stderr:512 bytes copied, 0.000306561 s, 1.7 MB/s 2023-09-23T18:57:26.848 DEBUG:teuthology.orchestra.run.smithi016:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-09-23T18:57:26.900 DEBUG:teuthology.orchestra.run.smithi016:> stat /dev/vg_nvme/lv_4 2023-09-23T18:57:26.951 INFO:teuthology.orchestra.run.smithi016.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-09-23T18:57:26.951 INFO:teuthology.orchestra.run.smithi016.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T18:57:26.951 INFO:teuthology.orchestra.run.smithi016.stdout:Device: 5h/5d Inode: 513 Links: 1 2023-09-23T18:57:26.951 INFO:teuthology.orchestra.run.smithi016.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T18:57:26.951 INFO:teuthology.orchestra.run.smithi016.stdout:Access: 2023-09-23 18:50:41.982075358 +0000 2023-09-23T18:57:26.951 INFO:teuthology.orchestra.run.smithi016.stdout:Modify: 2023-09-23 18:50:41.674082461 +0000 2023-09-23T18:57:26.951 INFO:teuthology.orchestra.run.smithi016.stdout:Change: 2023-09-23 18:50:41.674082461 +0000 2023-09-23T18:57:26.951 INFO:teuthology.orchestra.run.smithi016.stdout: Birth: - 2023-09-23T18:57:26.952 DEBUG:teuthology.orchestra.run.smithi016:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-09-23T18:57:27.006 INFO:teuthology.orchestra.run.smithi016.stderr:1+0 records in 2023-09-23T18:57:27.006 INFO:teuthology.orchestra.run.smithi016.stderr:1+0 records out 2023-09-23T18:57:27.007 INFO:teuthology.orchestra.run.smithi016.stderr:512 bytes copied, 0.000258472 s, 2.0 MB/s 2023-09-23T18:57:27.007 DEBUG:teuthology.orchestra.run.smithi016:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-09-23T18:57:27.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:26 smithi092 bash[27556]: audit 2023-09-23T18:57:25.588915+0000 mgr.y (mgr.14146) 48 : audit [DBG] from='client.24109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi016=y;smithi092=x", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:57:27.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:26 smithi092 bash[27556]: cephadm 2023-09-23T18:57:25.592429+0000 mgr.y (mgr.14146) 49 : cephadm [INF] Saving service mgr spec with placement smithi016=y;smithi092=x;count:2 2023-09-23T18:57:27.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:26 smithi092 bash[27556]: cephadm 2023-09-23T18:57:25.643353+0000 mgr.y (mgr.14146) 50 : cephadm [INF] Deploying daemon mgr.x on smithi092 2023-09-23T18:57:27.056 DEBUG:teuthology.orchestra.run.smithi092:> set -ex 2023-09-23T18:57:27.056 DEBUG:teuthology.orchestra.run.smithi092:> dd if=/scratch_devs of=/dev/stdout 2023-09-23T18:57:27.064 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-09-23T18:57:27.064 DEBUG:teuthology.orchestra.run.smithi092:> stat /dev/vg_nvme/lv_1 2023-09-23T18:57:27.115 INFO:teuthology.orchestra.run.smithi092.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-09-23T18:57:27.115 INFO:teuthology.orchestra.run.smithi092.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T18:57:27.115 INFO:teuthology.orchestra.run.smithi092.stdout:Device: 5h/5d Inode: 496 Links: 1 2023-09-23T18:57:27.115 INFO:teuthology.orchestra.run.smithi092.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T18:57:27.116 INFO:teuthology.orchestra.run.smithi092.stdout:Access: 2023-09-23 18:50:42.190010731 +0000 2023-09-23T18:57:27.116 INFO:teuthology.orchestra.run.smithi092.stdout:Modify: 2023-09-23 18:50:41.878017923 +0000 2023-09-23T18:57:27.116 INFO:teuthology.orchestra.run.smithi092.stdout:Change: 2023-09-23 18:50:41.878017923 +0000 2023-09-23T18:57:27.116 INFO:teuthology.orchestra.run.smithi092.stdout: Birth: - 2023-09-23T18:57:27.116 DEBUG:teuthology.orchestra.run.smithi092:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-09-23T18:57:27.168 INFO:teuthology.orchestra.run.smithi092.stderr:1+0 records in 2023-09-23T18:57:27.168 INFO:teuthology.orchestra.run.smithi092.stderr:1+0 records out 2023-09-23T18:57:27.168 INFO:teuthology.orchestra.run.smithi092.stderr:512 bytes copied, 0.000290103 s, 1.8 MB/s 2023-09-23T18:57:27.169 DEBUG:teuthology.orchestra.run.smithi092:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-09-23T18:57:27.219 DEBUG:teuthology.orchestra.run.smithi092:> stat /dev/vg_nvme/lv_2 2023-09-23T18:57:27.267 INFO:teuthology.orchestra.run.smithi092.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-09-23T18:57:27.267 INFO:teuthology.orchestra.run.smithi092.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T18:57:27.267 INFO:teuthology.orchestra.run.smithi092.stdout:Device: 5h/5d Inode: 499 Links: 1 2023-09-23T18:57:27.267 INFO:teuthology.orchestra.run.smithi092.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T18:57:27.267 INFO:teuthology.orchestra.run.smithi092.stdout:Access: 2023-09-23 18:50:42.741998009 +0000 2023-09-23T18:57:27.267 INFO:teuthology.orchestra.run.smithi092.stdout:Modify: 2023-09-23 18:50:42.434005107 +0000 2023-09-23T18:57:27.267 INFO:teuthology.orchestra.run.smithi092.stdout:Change: 2023-09-23 18:50:42.434005107 +0000 2023-09-23T18:57:27.267 INFO:teuthology.orchestra.run.smithi092.stdout: Birth: - 2023-09-23T18:57:27.267 DEBUG:teuthology.orchestra.run.smithi092:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-09-23T18:57:27.282 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:26 smithi016 bash[27237]: audit 2023-09-23T18:57:25.588915+0000 mgr.y (mgr.14146) 48 : audit [DBG] from='client.24109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi016=y;smithi092=x", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:57:27.282 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:26 smithi016 bash[27237]: cephadm 2023-09-23T18:57:25.592429+0000 mgr.y (mgr.14146) 49 : cephadm [INF] Saving service mgr spec with placement smithi016=y;smithi092=x;count:2 2023-09-23T18:57:27.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:26 smithi016 bash[27237]: cephadm 2023-09-23T18:57:25.643353+0000 mgr.y (mgr.14146) 50 : cephadm [INF] Deploying daemon mgr.x on smithi092 2023-09-23T18:57:27.283 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:26 smithi016 bash[33624]: audit 2023-09-23T18:57:25.588915+0000 mgr.y (mgr.14146) 48 : audit [DBG] from='client.24109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi016=y;smithi092=x", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:57:27.283 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:26 smithi016 bash[33624]: cephadm 2023-09-23T18:57:25.592429+0000 mgr.y (mgr.14146) 49 : cephadm [INF] Saving service mgr spec with placement smithi016=y;smithi092=x;count:2 2023-09-23T18:57:27.283 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:26 smithi016 bash[33624]: cephadm 2023-09-23T18:57:25.643353+0000 mgr.y (mgr.14146) 50 : cephadm [INF] Deploying daemon mgr.x on smithi092 2023-09-23T18:57:27.326 INFO:teuthology.orchestra.run.smithi092.stderr:1+0 records in 2023-09-23T18:57:27.326 INFO:teuthology.orchestra.run.smithi092.stderr:1+0 records out 2023-09-23T18:57:27.326 INFO:teuthology.orchestra.run.smithi092.stderr:512 bytes copied, 0.000328136 s, 1.6 MB/s 2023-09-23T18:57:27.327 DEBUG:teuthology.orchestra.run.smithi092:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-09-23T18:57:27.376 DEBUG:teuthology.orchestra.run.smithi092:> stat /dev/vg_nvme/lv_3 2023-09-23T18:57:27.427 INFO:teuthology.orchestra.run.smithi092.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-09-23T18:57:27.427 INFO:teuthology.orchestra.run.smithi092.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T18:57:27.427 INFO:teuthology.orchestra.run.smithi092.stdout:Device: 5h/5d Inode: 505 Links: 1 2023-09-23T18:57:27.427 INFO:teuthology.orchestra.run.smithi092.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T18:57:27.427 INFO:teuthology.orchestra.run.smithi092.stdout:Access: 2023-09-23 18:50:43.273985746 +0000 2023-09-23T18:57:27.427 INFO:teuthology.orchestra.run.smithi092.stdout:Modify: 2023-09-23 18:50:42.965992844 +0000 2023-09-23T18:57:27.427 INFO:teuthology.orchestra.run.smithi092.stdout:Change: 2023-09-23 18:50:42.965992844 +0000 2023-09-23T18:57:27.427 INFO:teuthology.orchestra.run.smithi092.stdout: Birth: - 2023-09-23T18:57:27.428 DEBUG:teuthology.orchestra.run.smithi092:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-09-23T18:57:27.485 INFO:teuthology.orchestra.run.smithi092.stderr:1+0 records in 2023-09-23T18:57:27.485 INFO:teuthology.orchestra.run.smithi092.stderr:1+0 records out 2023-09-23T18:57:27.486 INFO:teuthology.orchestra.run.smithi092.stderr:512 bytes copied, 0.000369224 s, 1.4 MB/s 2023-09-23T18:57:27.486 DEBUG:teuthology.orchestra.run.smithi092:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-09-23T18:57:27.536 DEBUG:teuthology.orchestra.run.smithi092:> stat /dev/vg_nvme/lv_4 2023-09-23T18:57:27.591 INFO:teuthology.orchestra.run.smithi092.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-09-23T18:57:27.591 INFO:teuthology.orchestra.run.smithi092.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T18:57:27.591 INFO:teuthology.orchestra.run.smithi092.stdout:Device: 5h/5d Inode: 513 Links: 1 2023-09-23T18:57:27.591 INFO:teuthology.orchestra.run.smithi092.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T18:57:27.591 INFO:teuthology.orchestra.run.smithi092.stdout:Access: 2023-09-23 18:50:43.781974036 +0000 2023-09-23T18:57:27.591 INFO:teuthology.orchestra.run.smithi092.stdout:Modify: 2023-09-23 18:50:43.469981227 +0000 2023-09-23T18:57:27.592 INFO:teuthology.orchestra.run.smithi092.stdout:Change: 2023-09-23 18:50:43.469981227 +0000 2023-09-23T18:57:27.592 INFO:teuthology.orchestra.run.smithi092.stdout: Birth: - 2023-09-23T18:57:27.592 DEBUG:teuthology.orchestra.run.smithi092:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-09-23T18:57:27.646 INFO:teuthology.orchestra.run.smithi092.stderr:1+0 records in 2023-09-23T18:57:27.646 INFO:teuthology.orchestra.run.smithi092.stderr:1+0 records out 2023-09-23T18:57:27.646 INFO:teuthology.orchestra.run.smithi092.stderr:512 bytes copied, 0.000406331 s, 1.3 MB/s 2023-09-23T18:57:27.647 DEBUG:teuthology.orchestra.run.smithi092:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-09-23T18:57:27.694 INFO:tasks.cephadm:Deploying osd.0 on smithi016 with /dev/vg_nvme/lv_4... 2023-09-23T18:57:27.695 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-09-23T18:57:27.915 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:27 smithi092 bash[27556]: cluster 2023-09-23T18:57:26.596656+0000 mgr.y (mgr.14146) 51 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:28.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:27 smithi016 bash[27237]: cluster 2023-09-23T18:57:26.596656+0000 mgr.y (mgr.14146) 51 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:28.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:27 smithi016 bash[33624]: cluster 2023-09-23T18:57:26.596656+0000 mgr.y (mgr.14146) 51 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:28.955 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 18:57:28 smithi092 bash[28593]: debug 2023-09-23T18:57:28.792+0000 7faf1b5ca700 1 -- 172.21.15.92:0/1325655108 <== mon.1 v2:172.21.15.92:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55b917aa04e0 con 0x55b917b94800 2023-09-23T18:57:29.230 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:28 smithi016 bash[33624]: audit 2023-09-23T18:57:27.948717+0000 mon.a (mon.0) 197 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:57:29.230 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:28 smithi016 bash[33624]: audit 2023-09-23T18:57:27.951807+0000 mon.a (mon.0) 198 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:57:29.230 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:28 smithi016 bash[33624]: cluster 2023-09-23T18:57:28.597062+0000 mgr.y (mgr.14146) 52 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:29.231 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:28 smithi016 bash[27237]: audit 2023-09-23T18:57:27.948717+0000 mon.a (mon.0) 197 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:57:29.231 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:28 smithi016 bash[27237]: audit 2023-09-23T18:57:27.951807+0000 mon.a (mon.0) 198 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:57:29.231 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:28 smithi016 bash[27237]: cluster 2023-09-23T18:57:28.597062+0000 mgr.y (mgr.14146) 52 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:29.290 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 18:57:28 smithi092 bash[28593]: debug 2023-09-23T18:57:28.956+0000 7faf2bbaf680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T18:57:29.290 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 18:57:29 smithi092 bash[28593]: debug 2023-09-23T18:57:29.060+0000 7faf2bbaf680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T18:57:29.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:28 smithi092 bash[27556]: audit 2023-09-23T18:57:27.948717+0000 mon.a (mon.0) 197 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:57:29.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:28 smithi092 bash[27556]: audit 2023-09-23T18:57:27.951807+0000 mon.a (mon.0) 198 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:57:29.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:28 smithi092 bash[27556]: cluster 2023-09-23T18:57:28.597062+0000 mgr.y (mgr.14146) 52 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:30.040 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 18:57:29 smithi092 bash[28593]: debug 2023-09-23T18:57:29.536+0000 7faf2bbaf680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T18:57:30.790 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 18:57:30 smithi092 bash[28593]: debug 2023-09-23T18:57:30.480+0000 7faf2bbaf680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T18:57:30.790 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 18:57:30 smithi092 bash[28593]: debug 2023-09-23T18:57:30.584+0000 7faf2bbaf680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T18:57:31.093 INFO:teuthology.orchestra.run.smithi016.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-09-23T18:57:31.093 INFO:teuthology.orchestra.run.smithi016.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-09-23T18:57:31.094 INFO:teuthology.orchestra.run.smithi016.stderr: stderr: 10+0 records in 2023-09-23T18:57:31.094 INFO:teuthology.orchestra.run.smithi016.stderr:10+0 records out 2023-09-23T18:57:31.094 INFO:teuthology.orchestra.run.smithi016.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0146563 s, 715 MB/s 2023-09-23T18:57:31.094 INFO:teuthology.orchestra.run.smithi016.stderr:--> Zapping successful for: 2023-09-23T18:57:31.290 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 18:57:30 smithi092 bash[28593]: debug 2023-09-23T18:57:30.860+0000 7faf2bbaf680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T18:57:31.749 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi016:vg_nvme/lv_4 2023-09-23T18:57:32.032 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:31 smithi016 bash[27237]: cluster 2023-09-23T18:57:30.597499+0000 mgr.y (mgr.14146) 53 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:32.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:31 smithi016 bash[27237]: audit 2023-09-23T18:57:30.810261+0000 mon.a (mon.0) 199 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:57:32.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:31 smithi016 bash[33624]: cluster 2023-09-23T18:57:30.597499+0000 mgr.y (mgr.14146) 53 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:32.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:31 smithi016 bash[33624]: audit 2023-09-23T18:57:30.810261+0000 mon.a (mon.0) 199 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:57:32.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:31 smithi092 bash[27556]: cluster 2023-09-23T18:57:30.597499+0000 mgr.y (mgr.14146) 53 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:32.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:31 smithi092 bash[27556]: audit 2023-09-23T18:57:30.810261+0000 mon.a (mon.0) 199 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:57:32.040 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 18:57:32 smithi092 bash[28593]: debug 2023-09-23T18:57:32.008+0000 7faf2bbaf680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T18:57:32.540 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 18:57:32 smithi092 bash[28593]: debug 2023-09-23T18:57:32.172+0000 7faf2bbaf680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T18:57:32.540 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 18:57:32 smithi092 bash[28593]: debug 2023-09-23T18:57:32.272+0000 7faf2bbaf680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T18:57:32.540 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 18:57:32 smithi092 bash[28593]: debug 2023-09-23T18:57:32.384+0000 7faf2bbaf680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T18:57:32.992 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 18:57:32 smithi092 bash[28593]: debug 2023-09-23T18:57:32.604+0000 7faf2bbaf680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T18:57:33.290 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 18:57:32 smithi092 bash[28593]: debug 2023-09-23T18:57:32.984+0000 7faf2bbaf680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T18:57:33.290 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 18:57:33 smithi092 bash[28593]: debug 2023-09-23T18:57:33.104+0000 7faf2bbaf680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T18:57:34.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:33 smithi016 bash[27237]: cluster 2023-09-23T18:57:32.597913+0000 mgr.y (mgr.14146) 54 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:34.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:33 smithi016 bash[27237]: audit 2023-09-23T18:57:32.963549+0000 mon.a (mon.0) 200 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:57:34.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:33 smithi016 bash[27237]: audit 2023-09-23T18:57:32.964879+0000 mon.a (mon.0) 201 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:57:34.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:33 smithi016 bash[27237]: audit 2023-09-23T18:57:32.966608+0000 mon.a (mon.0) 202 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:57:34.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:33 smithi016 bash[27237]: audit 2023-09-23T18:57:32.974211+0000 mon.a (mon.0) 203 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:57:34.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:33 smithi016 bash[27237]: audit 2023-09-23T18:57:32.977480+0000 mon.a (mon.0) 204 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T18:57:34.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:33 smithi016 bash[27237]: audit 2023-09-23T18:57:32.978740+0000 mon.a (mon.0) 205 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T18:57:34.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:33 smithi016 bash[27237]: audit 2023-09-23T18:57:32.979751+0000 mon.a (mon.0) 206 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:57:34.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:33 smithi016 bash[33624]: cluster 2023-09-23T18:57:32.597913+0000 mgr.y (mgr.14146) 54 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:34.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:33 smithi016 bash[33624]: audit 2023-09-23T18:57:32.963549+0000 mon.a (mon.0) 200 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:57:34.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:33 smithi016 bash[33624]: audit 2023-09-23T18:57:32.964879+0000 mon.a (mon.0) 201 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:57:34.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:33 smithi016 bash[33624]: audit 2023-09-23T18:57:32.966608+0000 mon.a (mon.0) 202 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:57:34.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:33 smithi016 bash[33624]: audit 2023-09-23T18:57:32.974211+0000 mon.a (mon.0) 203 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:57:34.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:33 smithi016 bash[33624]: audit 2023-09-23T18:57:32.977480+0000 mon.a (mon.0) 204 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T18:57:34.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:33 smithi016 bash[33624]: audit 2023-09-23T18:57:32.978740+0000 mon.a (mon.0) 205 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T18:57:34.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:33 smithi016 bash[33624]: audit 2023-09-23T18:57:32.979751+0000 mon.a (mon.0) 206 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:57:34.039 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 18:57:33 smithi092 bash[28593]: debug 2023-09-23T18:57:33.956+0000 7faf2bbaf680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T18:57:34.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:33 smithi092 bash[27556]: cluster 2023-09-23T18:57:32.597913+0000 mgr.y (mgr.14146) 54 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:34.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:33 smithi092 bash[27556]: audit 2023-09-23T18:57:32.963549+0000 mon.a (mon.0) 200 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:57:34.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:33 smithi092 bash[27556]: audit 2023-09-23T18:57:32.964879+0000 mon.a (mon.0) 201 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:57:34.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:33 smithi092 bash[27556]: audit 2023-09-23T18:57:32.966608+0000 mon.a (mon.0) 202 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:57:34.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:33 smithi092 bash[27556]: audit 2023-09-23T18:57:32.974211+0000 mon.a (mon.0) 203 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:57:34.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:33 smithi092 bash[27556]: audit 2023-09-23T18:57:32.977480+0000 mon.a (mon.0) 204 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T18:57:34.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:33 smithi092 bash[27556]: audit 2023-09-23T18:57:32.978740+0000 mon.a (mon.0) 205 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T18:57:34.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:33 smithi092 bash[27556]: audit 2023-09-23T18:57:32.979751+0000 mon.a (mon.0) 206 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:57:34.389 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 18:57:34 smithi092 bash[28593]: debug 2023-09-23T18:57:34.056+0000 7faf2bbaf680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T18:57:34.389 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 18:57:34 smithi092 bash[28593]: debug 2023-09-23T18:57:34.168+0000 7faf2bbaf680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T18:57:34.389 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 18:57:34 smithi092 bash[28593]: debug 2023-09-23T18:57:34.384+0000 7faf2bbaf680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T18:57:35.015 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:34 smithi092 bash[27556]: cephadm 2023-09-23T18:57:32.977037+0000 mgr.y (mgr.14146) 55 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2023-09-23T18:57:35.015 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:34 smithi092 bash[27556]: cephadm 2023-09-23T18:57:32.980646+0000 mgr.y (mgr.14146) 56 : cephadm [INF] Reconfiguring daemon mgr.y on smithi016 2023-09-23T18:57:35.015 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:34 smithi092 bash[27556]: cluster 2023-09-23T18:57:34.598232+0000 mgr.y (mgr.14146) 57 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:35.016 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 18:57:34 smithi092 bash[28593]: debug 2023-09-23T18:57:34.672+0000 7faf2bbaf680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T18:57:35.016 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 18:57:34 smithi092 bash[28593]: debug 2023-09-23T18:57:34.836+0000 7faf2bbaf680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T18:57:35.016 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 18:57:35 smithi092 bash[28593]: debug 2023-09-23T18:57:35.008+0000 7faf2bbaf680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T18:57:35.032 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:34 smithi016 bash[27237]: cephadm 2023-09-23T18:57:32.977037+0000 mgr.y (mgr.14146) 55 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2023-09-23T18:57:35.032 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:34 smithi016 bash[27237]: cephadm 2023-09-23T18:57:32.980646+0000 mgr.y (mgr.14146) 56 : cephadm [INF] Reconfiguring daemon mgr.y on smithi016 2023-09-23T18:57:35.032 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:34 smithi016 bash[27237]: cluster 2023-09-23T18:57:34.598232+0000 mgr.y (mgr.14146) 57 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:35.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:34 smithi016 bash[33624]: cephadm 2023-09-23T18:57:32.977037+0000 mgr.y (mgr.14146) 55 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2023-09-23T18:57:35.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:34 smithi016 bash[33624]: cephadm 2023-09-23T18:57:32.980646+0000 mgr.y (mgr.14146) 56 : cephadm [INF] Reconfiguring daemon mgr.y on smithi016 2023-09-23T18:57:35.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:34 smithi016 bash[33624]: cluster 2023-09-23T18:57:34.598232+0000 mgr.y (mgr.14146) 57 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:35.290 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 18:57:35 smithi092 bash[28593]: debug 2023-09-23T18:57:35.220+0000 7faf2bbaf680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T18:57:35.790 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 18:57:35 smithi092 bash[28593]: debug 2023-09-23T18:57:35.328+0000 7faf2bbaf680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T18:57:36.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:35 smithi016 bash[27237]: audit 2023-09-23T18:57:35.169115+0000 mgr.y (mgr.14146) 58 : audit [DBG] from='client.14211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi016:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:57:36.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:35 smithi016 bash[27237]: audit 2023-09-23T18:57:35.175077+0000 mon.a (mon.0) 207 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T18:57:36.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:35 smithi016 bash[27237]: audit 2023-09-23T18:57:35.178059+0000 mon.a (mon.0) 208 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T18:57:36.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:35 smithi016 bash[27237]: audit 2023-09-23T18:57:35.178870+0000 mon.a (mon.0) 209 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:57:36.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:35 smithi016 bash[27237]: cluster 2023-09-23T18:57:35.355896+0000 mon.a (mon.0) 210 : cluster [DBG] Standby manager daemon x started 2023-09-23T18:57:36.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:35 smithi016 bash[27237]: audit 2023-09-23T18:57:35.358990+0000 mon.b (mon.1) 4 : audit [DBG] from='mgr.? 172.21.15.92:0/3052341051' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T18:57:36.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:35 smithi016 bash[27237]: audit 2023-09-23T18:57:35.359863+0000 mon.b (mon.1) 5 : audit [DBG] from='mgr.? 172.21.15.92:0/3052341051' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T18:57:36.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:35 smithi016 bash[27237]: audit 2023-09-23T18:57:35.361862+0000 mon.b (mon.1) 6 : audit [DBG] from='mgr.? 172.21.15.92:0/3052341051' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T18:57:36.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:35 smithi016 bash[27237]: audit 2023-09-23T18:57:35.362410+0000 mon.b (mon.1) 7 : audit [DBG] from='mgr.? 172.21.15.92:0/3052341051' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T18:57:36.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:35 smithi016 bash[27237]: audit 2023-09-23T18:57:35.460015+0000 mon.a (mon.0) 211 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:57:36.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:35 smithi016 bash[27237]: audit 2023-09-23T18:57:35.462005+0000 mon.a (mon.0) 212 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:57:36.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:35 smithi016 bash[27237]: audit 2023-09-23T18:57:35.464834+0000 mon.a (mon.0) 213 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:57:36.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:35 smithi016 bash[27237]: audit 2023-09-23T18:57:35.466142+0000 mon.a (mon.0) 214 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:57:36.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:35 smithi016 bash[27237]: audit 2023-09-23T18:57:35.471452+0000 mon.a (mon.0) 215 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:57:36.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:35 smithi016 bash[33624]: audit 2023-09-23T18:57:35.169115+0000 mgr.y (mgr.14146) 58 : audit [DBG] from='client.14211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi016:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:57:36.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:35 smithi016 bash[33624]: audit 2023-09-23T18:57:35.175077+0000 mon.a (mon.0) 207 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T18:57:36.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:35 smithi016 bash[33624]: audit 2023-09-23T18:57:35.178059+0000 mon.a (mon.0) 208 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T18:57:36.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:35 smithi016 bash[33624]: audit 2023-09-23T18:57:35.178870+0000 mon.a (mon.0) 209 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:57:36.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:35 smithi016 bash[33624]: cluster 2023-09-23T18:57:35.355896+0000 mon.a (mon.0) 210 : cluster [DBG] Standby manager daemon x started 2023-09-23T18:57:36.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:35 smithi016 bash[33624]: audit 2023-09-23T18:57:35.358990+0000 mon.b (mon.1) 4 : audit [DBG] from='mgr.? 172.21.15.92:0/3052341051' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T18:57:36.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:35 smithi016 bash[33624]: audit 2023-09-23T18:57:35.359863+0000 mon.b (mon.1) 5 : audit [DBG] from='mgr.? 172.21.15.92:0/3052341051' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T18:57:36.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:35 smithi016 bash[33624]: audit 2023-09-23T18:57:35.361862+0000 mon.b (mon.1) 6 : audit [DBG] from='mgr.? 172.21.15.92:0/3052341051' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T18:57:36.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:35 smithi016 bash[33624]: audit 2023-09-23T18:57:35.362410+0000 mon.b (mon.1) 7 : audit [DBG] from='mgr.? 172.21.15.92:0/3052341051' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T18:57:36.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:35 smithi016 bash[33624]: audit 2023-09-23T18:57:35.460015+0000 mon.a (mon.0) 211 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:57:36.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:35 smithi016 bash[33624]: audit 2023-09-23T18:57:35.462005+0000 mon.a (mon.0) 212 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:57:36.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:35 smithi016 bash[33624]: audit 2023-09-23T18:57:35.464834+0000 mon.a (mon.0) 213 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:57:36.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:35 smithi016 bash[33624]: audit 2023-09-23T18:57:35.466142+0000 mon.a (mon.0) 214 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:57:36.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:35 smithi016 bash[33624]: audit 2023-09-23T18:57:35.471452+0000 mon.a (mon.0) 215 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:57:36.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:35 smithi092 bash[27556]: audit 2023-09-23T18:57:35.169115+0000 mgr.y (mgr.14146) 58 : audit [DBG] from='client.14211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi016:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:57:36.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:35 smithi092 bash[27556]: audit 2023-09-23T18:57:35.175077+0000 mon.a (mon.0) 207 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T18:57:36.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:35 smithi092 bash[27556]: audit 2023-09-23T18:57:35.178059+0000 mon.a (mon.0) 208 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T18:57:36.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:35 smithi092 bash[27556]: audit 2023-09-23T18:57:35.178870+0000 mon.a (mon.0) 209 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:57:36.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:35 smithi092 bash[27556]: cluster 2023-09-23T18:57:35.355896+0000 mon.a (mon.0) 210 : cluster [DBG] Standby manager daemon x started 2023-09-23T18:57:36.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:35 smithi092 bash[27556]: audit 2023-09-23T18:57:35.358990+0000 mon.b (mon.1) 4 : audit [DBG] from='mgr.? 172.21.15.92:0/3052341051' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T18:57:36.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:35 smithi092 bash[27556]: audit 2023-09-23T18:57:35.359863+0000 mon.b (mon.1) 5 : audit [DBG] from='mgr.? 172.21.15.92:0/3052341051' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T18:57:36.291 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:35 smithi092 bash[27556]: audit 2023-09-23T18:57:35.361862+0000 mon.b (mon.1) 6 : audit [DBG] from='mgr.? 172.21.15.92:0/3052341051' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T18:57:36.291 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:35 smithi092 bash[27556]: audit 2023-09-23T18:57:35.362410+0000 mon.b (mon.1) 7 : audit [DBG] from='mgr.? 172.21.15.92:0/3052341051' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T18:57:36.291 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:35 smithi092 bash[27556]: audit 2023-09-23T18:57:35.460015+0000 mon.a (mon.0) 211 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:57:36.291 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:35 smithi092 bash[27556]: audit 2023-09-23T18:57:35.462005+0000 mon.a (mon.0) 212 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:57:36.291 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:35 smithi092 bash[27556]: audit 2023-09-23T18:57:35.464834+0000 mon.a (mon.0) 213 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:57:36.291 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:35 smithi092 bash[27556]: audit 2023-09-23T18:57:35.466142+0000 mon.a (mon.0) 214 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:57:36.291 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:35 smithi092 bash[27556]: audit 2023-09-23T18:57:35.471452+0000 mon.a (mon.0) 215 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:57:37.282 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:36 smithi016 bash[27237]: cluster 2023-09-23T18:57:35.808120+0000 mon.a (mon.0) 216 : cluster [DBG] mgrmap e14: y(active, since 75s), standbys: x 2023-09-23T18:57:37.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:36 smithi016 bash[27237]: audit 2023-09-23T18:57:35.808342+0000 mon.a (mon.0) 217 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T18:57:37.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:36 smithi016 bash[27237]: cluster 2023-09-23T18:57:36.598634+0000 mgr.y (mgr.14146) 59 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:37.283 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:36 smithi016 bash[33624]: cluster 2023-09-23T18:57:35.808120+0000 mon.a (mon.0) 216 : cluster [DBG] mgrmap e14: y(active, since 75s), standbys: x 2023-09-23T18:57:37.283 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:36 smithi016 bash[33624]: audit 2023-09-23T18:57:35.808342+0000 mon.a (mon.0) 217 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T18:57:37.283 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:36 smithi016 bash[33624]: cluster 2023-09-23T18:57:36.598634+0000 mgr.y (mgr.14146) 59 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:37.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:36 smithi092 bash[27556]: cluster 2023-09-23T18:57:35.808120+0000 mon.a (mon.0) 216 : cluster [DBG] mgrmap e14: y(active, since 75s), standbys: x 2023-09-23T18:57:37.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:36 smithi092 bash[27556]: audit 2023-09-23T18:57:35.808342+0000 mon.a (mon.0) 217 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T18:57:37.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:36 smithi092 bash[27556]: cluster 2023-09-23T18:57:36.598634+0000 mgr.y (mgr.14146) 59 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:39.937 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:39 smithi016 bash[33624]: cluster 2023-09-23T18:57:38.599096+0000 mgr.y (mgr.14146) 60 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:39.937 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:39 smithi016 bash[27237]: cluster 2023-09-23T18:57:38.599096+0000 mgr.y (mgr.14146) 60 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:40.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:39 smithi092 bash[27556]: cluster 2023-09-23T18:57:38.599096+0000 mgr.y (mgr.14146) 60 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:42.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:41 smithi092 bash[27556]: cluster 2023-09-23T18:57:40.599557+0000 mgr.y (mgr.14146) 61 : cluster [DBG] pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:42.149 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:41 smithi016 bash[27237]: cluster 2023-09-23T18:57:40.599557+0000 mgr.y (mgr.14146) 61 : cluster [DBG] pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:42.150 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:41 smithi016 bash[33624]: cluster 2023-09-23T18:57:40.599557+0000 mgr.y (mgr.14146) 61 : cluster [DBG] pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:44.282 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:43 smithi016 bash[27237]: cluster 2023-09-23T18:57:42.600018+0000 mgr.y (mgr.14146) 62 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:44.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:43 smithi016 bash[27237]: audit 2023-09-23T18:57:43.094048+0000 mon.c (mon.2) 2 : audit [INF] from='client.? 172.21.15.16:0/1418445478' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a4e8ed09-1195-4878-928a-13881bc04dc7"}]: dispatch 2023-09-23T18:57:44.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:43 smithi016 bash[27237]: audit 2023-09-23T18:57:43.094710+0000 mon.a (mon.0) 218 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a4e8ed09-1195-4878-928a-13881bc04dc7"}]: dispatch 2023-09-23T18:57:44.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:43 smithi016 bash[27237]: audit 2023-09-23T18:57:43.101918+0000 mon.a (mon.0) 219 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a4e8ed09-1195-4878-928a-13881bc04dc7"}]': finished 2023-09-23T18:57:44.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:43 smithi016 bash[27237]: cluster 2023-09-23T18:57:43.102044+0000 mon.a (mon.0) 220 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2023-09-23T18:57:44.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:43 smithi016 bash[27237]: audit 2023-09-23T18:57:43.102265+0000 mon.a (mon.0) 221 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T18:57:44.283 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:43 smithi016 bash[33624]: cluster 2023-09-23T18:57:42.600018+0000 mgr.y (mgr.14146) 62 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:44.283 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:43 smithi016 bash[33624]: audit 2023-09-23T18:57:43.094048+0000 mon.c (mon.2) 2 : audit [INF] from='client.? 172.21.15.16:0/1418445478' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a4e8ed09-1195-4878-928a-13881bc04dc7"}]: dispatch 2023-09-23T18:57:44.283 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:43 smithi016 bash[33624]: audit 2023-09-23T18:57:43.094710+0000 mon.a (mon.0) 218 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a4e8ed09-1195-4878-928a-13881bc04dc7"}]: dispatch 2023-09-23T18:57:44.283 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:43 smithi016 bash[33624]: audit 2023-09-23T18:57:43.101918+0000 mon.a (mon.0) 219 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a4e8ed09-1195-4878-928a-13881bc04dc7"}]': finished 2023-09-23T18:57:44.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:43 smithi016 bash[33624]: cluster 2023-09-23T18:57:43.102044+0000 mon.a (mon.0) 220 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2023-09-23T18:57:44.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:43 smithi016 bash[33624]: audit 2023-09-23T18:57:43.102265+0000 mon.a (mon.0) 221 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T18:57:44.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:43 smithi092 bash[27556]: cluster 2023-09-23T18:57:42.600018+0000 mgr.y (mgr.14146) 62 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:44.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:43 smithi092 bash[27556]: audit 2023-09-23T18:57:43.094048+0000 mon.c (mon.2) 2 : audit [INF] from='client.? 172.21.15.16:0/1418445478' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a4e8ed09-1195-4878-928a-13881bc04dc7"}]: dispatch 2023-09-23T18:57:44.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:43 smithi092 bash[27556]: audit 2023-09-23T18:57:43.094710+0000 mon.a (mon.0) 218 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a4e8ed09-1195-4878-928a-13881bc04dc7"}]: dispatch 2023-09-23T18:57:44.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:43 smithi092 bash[27556]: audit 2023-09-23T18:57:43.101918+0000 mon.a (mon.0) 219 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a4e8ed09-1195-4878-928a-13881bc04dc7"}]': finished 2023-09-23T18:57:44.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:43 smithi092 bash[27556]: cluster 2023-09-23T18:57:43.102044+0000 mon.a (mon.0) 220 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2023-09-23T18:57:44.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:43 smithi092 bash[27556]: audit 2023-09-23T18:57:43.102265+0000 mon.a (mon.0) 221 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T18:57:45.282 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:44 smithi016 bash[27237]: audit 2023-09-23T18:57:43.814292+0000 mon.a (mon.0) 222 : audit [DBG] from='client.? 172.21.15.16:0/2373769302' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T18:57:45.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:44 smithi016 bash[27237]: cluster 2023-09-23T18:57:44.600515+0000 mgr.y (mgr.14146) 63 : cluster [DBG] pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:45.283 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:44 smithi016 bash[33624]: audit 2023-09-23T18:57:43.814292+0000 mon.a (mon.0) 222 : audit [DBG] from='client.? 172.21.15.16:0/2373769302' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T18:57:45.283 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:44 smithi016 bash[33624]: cluster 2023-09-23T18:57:44.600515+0000 mgr.y (mgr.14146) 63 : cluster [DBG] pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:45.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:44 smithi092 bash[27556]: audit 2023-09-23T18:57:43.814292+0000 mon.a (mon.0) 222 : audit [DBG] from='client.? 172.21.15.16:0/2373769302' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T18:57:45.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:44 smithi092 bash[27556]: cluster 2023-09-23T18:57:44.600515+0000 mgr.y (mgr.14146) 63 : cluster [DBG] pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:48.032 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:47 smithi016 bash[27237]: cluster 2023-09-23T18:57:46.600829+0000 mgr.y (mgr.14146) 64 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:48.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:47 smithi016 bash[33624]: cluster 2023-09-23T18:57:46.600829+0000 mgr.y (mgr.14146) 64 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:48.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:47 smithi092 bash[27556]: cluster 2023-09-23T18:57:46.600829+0000 mgr.y (mgr.14146) 64 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:50.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:49 smithi016 bash[27237]: cluster 2023-09-23T18:57:48.601281+0000 mgr.y (mgr.14146) 65 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:50.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:49 smithi016 bash[33624]: cluster 2023-09-23T18:57:48.601281+0000 mgr.y (mgr.14146) 65 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:50.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:49 smithi092 bash[27556]: cluster 2023-09-23T18:57:48.601281+0000 mgr.y (mgr.14146) 65 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:52.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:51 smithi092 bash[27556]: cluster 2023-09-23T18:57:50.601826+0000 mgr.y (mgr.14146) 66 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:52.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:51 smithi016 bash[27237]: cluster 2023-09-23T18:57:50.601826+0000 mgr.y (mgr.14146) 66 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:52.283 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:51 smithi016 bash[33624]: cluster 2023-09-23T18:57:50.601826+0000 mgr.y (mgr.14146) 66 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:53.002 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:52 smithi016 bash[27237]: cluster 2023-09-23T18:57:52.602350+0000 mgr.y (mgr.14146) 67 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:53.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:52 smithi016 bash[33624]: cluster 2023-09-23T18:57:52.602350+0000 mgr.y (mgr.14146) 67 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:53.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:52 smithi092 bash[27556]: cluster 2023-09-23T18:57:52.602350+0000 mgr.y (mgr.14146) 67 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:56.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:55 smithi016 bash[27237]: cluster 2023-09-23T18:57:54.602882+0000 mgr.y (mgr.14146) 68 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:56.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:55 smithi016 bash[27237]: audit 2023-09-23T18:57:55.482590+0000 mon.a (mon.0) 223 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T18:57:56.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:55 smithi016 bash[27237]: audit 2023-09-23T18:57:55.484329+0000 mon.a (mon.0) 224 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:57:56.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:55 smithi016 bash[33624]: cluster 2023-09-23T18:57:54.602882+0000 mgr.y (mgr.14146) 68 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:56.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:55 smithi016 bash[33624]: audit 2023-09-23T18:57:55.482590+0000 mon.a (mon.0) 223 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T18:57:56.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:55 smithi016 bash[33624]: audit 2023-09-23T18:57:55.484329+0000 mon.a (mon.0) 224 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:57:56.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:55 smithi092 bash[27556]: cluster 2023-09-23T18:57:54.602882+0000 mgr.y (mgr.14146) 68 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:56.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:55 smithi092 bash[27556]: audit 2023-09-23T18:57:55.482590+0000 mon.a (mon.0) 223 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T18:57:56.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:55 smithi092 bash[27556]: audit 2023-09-23T18:57:55.484329+0000 mon.a (mon.0) 224 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:57:57.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:56 smithi016 bash[27237]: cephadm 2023-09-23T18:57:55.485411+0000 mgr.y (mgr.14146) 69 : cephadm [INF] Deploying daemon osd.0 on smithi016 2023-09-23T18:57:57.283 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:56 smithi016 bash[33624]: cephadm 2023-09-23T18:57:55.485411+0000 mgr.y (mgr.14146) 69 : cephadm [INF] Deploying daemon osd.0 on smithi016 2023-09-23T18:57:57.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:56 smithi092 bash[27556]: cephadm 2023-09-23T18:57:55.485411+0000 mgr.y (mgr.14146) 69 : cephadm [INF] Deploying daemon osd.0 on smithi016 2023-09-23T18:57:58.133 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:57 smithi016 bash[27237]: cluster 2023-09-23T18:57:56.603315+0000 mgr.y (mgr.14146) 70 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:58.133 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:57 smithi016 bash[33624]: cluster 2023-09-23T18:57:56.603315+0000 mgr.y (mgr.14146) 70 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:58.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:57 smithi092 bash[27556]: cluster 2023-09-23T18:57:56.603315+0000 mgr.y (mgr.14146) 70 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:59.032 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:57:58 smithi016 bash[27237]: cluster 2023-09-23T18:57:58.603636+0000 mgr.y (mgr.14146) 71 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:59.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:57:58 smithi016 bash[33624]: cluster 2023-09-23T18:57:58.603636+0000 mgr.y (mgr.14146) 71 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:57:59.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:57:58 smithi092 bash[27556]: cluster 2023-09-23T18:57:58.603636+0000 mgr.y (mgr.14146) 71 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:58:00.783 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:00 smithi016 bash[27237]: audit 2023-09-23T18:57:59.650543+0000 mon.a (mon.0) 225 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:58:00.783 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:00 smithi016 bash[27237]: audit 2023-09-23T18:57:59.653404+0000 mon.a (mon.0) 226 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:58:00.783 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:00 smithi016 bash[33624]: audit 2023-09-23T18:57:59.650543+0000 mon.a (mon.0) 225 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:58:00.783 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:00 smithi016 bash[33624]: audit 2023-09-23T18:57:59.653404+0000 mon.a (mon.0) 226 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:58:01.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:00 smithi092 bash[27556]: audit 2023-09-23T18:57:59.650543+0000 mon.a (mon.0) 225 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:58:01.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:00 smithi092 bash[27556]: audit 2023-09-23T18:57:59.653404+0000 mon.a (mon.0) 226 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:58:02.282 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:01 smithi016 bash[27237]: cluster 2023-09-23T18:58:00.604177+0000 mgr.y (mgr.14146) 72 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:58:02.283 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:01 smithi016 bash[33624]: cluster 2023-09-23T18:58:00.604177+0000 mgr.y (mgr.14146) 72 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:58:02.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:01 smithi092 bash[27556]: cluster 2023-09-23T18:58:00.604177+0000 mgr.y (mgr.14146) 72 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:58:03.032 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:02 smithi016 bash[27237]: cluster 2023-09-23T18:58:02.604648+0000 mgr.y (mgr.14146) 73 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:58:03.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:02 smithi016 bash[33624]: cluster 2023-09-23T18:58:02.604648+0000 mgr.y (mgr.14146) 73 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:58:03.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:02 smithi092 bash[27556]: cluster 2023-09-23T18:58:02.604648+0000 mgr.y (mgr.14146) 73 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:58:06.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:05 smithi016 bash[27237]: cluster 2023-09-23T18:58:04.605051+0000 mgr.y (mgr.14146) 74 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:58:06.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:05 smithi016 bash[27237]: audit 2023-09-23T18:58:04.779446+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:58:06.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:05 smithi016 bash[27237]: audit 2023-09-23T18:58:05.052030+0000 mon.a (mon.0) 228 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:58:06.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:05 smithi016 bash[27237]: audit 2023-09-23T18:58:05.053009+0000 mon.a (mon.0) 229 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:58:06.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:05 smithi016 bash[27237]: audit 2023-09-23T18:58:05.058399+0000 mon.a (mon.0) 230 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:58:06.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:05 smithi016 bash[33624]: cluster 2023-09-23T18:58:04.605051+0000 mgr.y (mgr.14146) 74 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:58:06.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:05 smithi016 bash[33624]: audit 2023-09-23T18:58:04.779446+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:58:06.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:05 smithi016 bash[33624]: audit 2023-09-23T18:58:05.052030+0000 mon.a (mon.0) 228 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:58:06.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:05 smithi016 bash[33624]: audit 2023-09-23T18:58:05.053009+0000 mon.a (mon.0) 229 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:58:06.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:05 smithi016 bash[33624]: audit 2023-09-23T18:58:05.058399+0000 mon.a (mon.0) 230 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:58:06.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:05 smithi092 bash[27556]: cluster 2023-09-23T18:58:04.605051+0000 mgr.y (mgr.14146) 74 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:58:06.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:05 smithi092 bash[27556]: audit 2023-09-23T18:58:04.779446+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:58:06.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:05 smithi092 bash[27556]: audit 2023-09-23T18:58:05.052030+0000 mon.a (mon.0) 228 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:58:06.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:05 smithi092 bash[27556]: audit 2023-09-23T18:58:05.053009+0000 mon.a (mon.0) 229 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:58:06.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:05 smithi092 bash[27556]: audit 2023-09-23T18:58:05.058399+0000 mon.a (mon.0) 230 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:58:07.032 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:06 smithi016 bash[27237]: audit 2023-09-23T18:58:06.360793+0000 mon.c (mon.2) 3 : audit [INF] from='osd.0 [v2:172.21.15.16:6802/2337435219,v1:172.21.15.16:6803/2337435219]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T18:58:07.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:06 smithi016 bash[27237]: audit 2023-09-23T18:58:06.361364+0000 mon.a (mon.0) 231 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T18:58:07.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:06 smithi016 bash[33624]: audit 2023-09-23T18:58:06.360793+0000 mon.c (mon.2) 3 : audit [INF] from='osd.0 [v2:172.21.15.16:6802/2337435219,v1:172.21.15.16:6803/2337435219]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T18:58:07.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:06 smithi016 bash[33624]: audit 2023-09-23T18:58:06.361364+0000 mon.a (mon.0) 231 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T18:58:07.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:06 smithi092 bash[27556]: audit 2023-09-23T18:58:06.360793+0000 mon.c (mon.2) 3 : audit [INF] from='osd.0 [v2:172.21.15.16:6802/2337435219,v1:172.21.15.16:6803/2337435219]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T18:58:07.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:06 smithi092 bash[27556]: audit 2023-09-23T18:58:06.361364+0000 mon.a (mon.0) 231 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T18:58:08.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:07 smithi016 bash[27237]: cluster 2023-09-23T18:58:06.605498+0000 mgr.y (mgr.14146) 75 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:58:08.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:07 smithi016 bash[27237]: audit 2023-09-23T18:58:06.790707+0000 mon.a (mon.0) 232 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T18:58:08.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:07 smithi016 bash[27237]: cluster 2023-09-23T18:58:06.790822+0000 mon.a (mon.0) 233 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2023-09-23T18:58:08.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:07 smithi016 bash[27237]: audit 2023-09-23T18:58:06.791034+0000 mon.a (mon.0) 234 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T18:58:08.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:07 smithi016 bash[27237]: audit 2023-09-23T18:58:06.791753+0000 mon.c (mon.2) 4 : audit [INF] from='osd.0 [v2:172.21.15.16:6802/2337435219,v1:172.21.15.16:6803/2337435219]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-09-23T18:58:08.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:07 smithi016 bash[27237]: audit 2023-09-23T18:58:06.792160+0000 mon.a (mon.0) 235 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-09-23T18:58:08.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:07 smithi016 bash[33624]: cluster 2023-09-23T18:58:06.605498+0000 mgr.y (mgr.14146) 75 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:58:08.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:07 smithi016 bash[33624]: audit 2023-09-23T18:58:06.790707+0000 mon.a (mon.0) 232 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T18:58:08.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:07 smithi016 bash[33624]: cluster 2023-09-23T18:58:06.790822+0000 mon.a (mon.0) 233 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2023-09-23T18:58:08.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:07 smithi016 bash[33624]: audit 2023-09-23T18:58:06.791034+0000 mon.a (mon.0) 234 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T18:58:08.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:07 smithi016 bash[33624]: audit 2023-09-23T18:58:06.791753+0000 mon.c (mon.2) 4 : audit [INF] from='osd.0 [v2:172.21.15.16:6802/2337435219,v1:172.21.15.16:6803/2337435219]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-09-23T18:58:08.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:07 smithi016 bash[33624]: audit 2023-09-23T18:58:06.792160+0000 mon.a (mon.0) 235 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-09-23T18:58:08.142 INFO:teuthology.orchestra.run.smithi016.stdout:Created osd(s) 0 on host 'smithi016' 2023-09-23T18:58:08.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:07 smithi092 bash[27556]: cluster 2023-09-23T18:58:06.605498+0000 mgr.y (mgr.14146) 75 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:58:08.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:07 smithi092 bash[27556]: audit 2023-09-23T18:58:06.790707+0000 mon.a (mon.0) 232 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T18:58:08.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:07 smithi092 bash[27556]: cluster 2023-09-23T18:58:06.790822+0000 mon.a (mon.0) 233 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2023-09-23T18:58:08.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:07 smithi092 bash[27556]: audit 2023-09-23T18:58:06.791034+0000 mon.a (mon.0) 234 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T18:58:08.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:07 smithi092 bash[27556]: audit 2023-09-23T18:58:06.791753+0000 mon.c (mon.2) 4 : audit [INF] from='osd.0 [v2:172.21.15.16:6802/2337435219,v1:172.21.15.16:6803/2337435219]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-09-23T18:58:08.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:07 smithi092 bash[27556]: audit 2023-09-23T18:58:06.792160+0000 mon.a (mon.0) 235 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-09-23T18:58:08.825 DEBUG:teuthology.orchestra.run.smithi016:osd.0> sudo journalctl -f -n 0 -u ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@osd.0.service 2023-09-23T18:58:08.828 INFO:tasks.cephadm:Deploying osd.1 on smithi016 with /dev/vg_nvme/lv_3... 2023-09-23T18:58:08.828 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-09-23T18:58:09.068 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:08 smithi016 bash[33624]: audit 2023-09-23T18:58:07.795459+0000 mon.a (mon.0) 236 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]': finished 2023-09-23T18:58:09.068 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:08 smithi016 bash[33624]: cluster 2023-09-23T18:58:07.795662+0000 mon.a (mon.0) 237 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2023-09-23T18:58:09.068 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:08 smithi016 bash[33624]: audit 2023-09-23T18:58:07.796368+0000 mon.a (mon.0) 238 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T18:58:09.069 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:08 smithi016 bash[33624]: audit 2023-09-23T18:58:07.801590+0000 mon.a (mon.0) 239 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T18:58:09.069 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:08 smithi016 bash[33624]: audit 2023-09-23T18:58:08.140147+0000 mon.a (mon.0) 240 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:58:09.069 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:08 smithi016 bash[33624]: audit 2023-09-23T18:58:08.182858+0000 mon.a (mon.0) 241 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:58:09.069 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:08 smithi016 bash[33624]: cluster 2023-09-23T18:58:08.606008+0000 mgr.y (mgr.14146) 76 : cluster [DBG] pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:58:09.069 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:08 smithi016 bash[27237]: audit 2023-09-23T18:58:07.795459+0000 mon.a (mon.0) 236 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]': finished 2023-09-23T18:58:09.069 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:08 smithi016 bash[27237]: cluster 2023-09-23T18:58:07.795662+0000 mon.a (mon.0) 237 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2023-09-23T18:58:09.069 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:08 smithi016 bash[27237]: audit 2023-09-23T18:58:07.796368+0000 mon.a (mon.0) 238 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T18:58:09.069 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:08 smithi016 bash[27237]: audit 2023-09-23T18:58:07.801590+0000 mon.a (mon.0) 239 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T18:58:09.069 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:08 smithi016 bash[27237]: audit 2023-09-23T18:58:08.140147+0000 mon.a (mon.0) 240 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:58:09.069 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:08 smithi016 bash[27237]: audit 2023-09-23T18:58:08.182858+0000 mon.a (mon.0) 241 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:58:09.070 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:08 smithi016 bash[27237]: cluster 2023-09-23T18:58:08.606008+0000 mgr.y (mgr.14146) 76 : cluster [DBG] pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:58:09.257 INFO:journalctl@ceph.osd.0.smithi016.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-09-23T18:58:09.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:08 smithi092 bash[27556]: audit 2023-09-23T18:58:07.795459+0000 mon.a (mon.0) 236 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]': finished 2023-09-23T18:58:09.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:08 smithi092 bash[27556]: cluster 2023-09-23T18:58:07.795662+0000 mon.a (mon.0) 237 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2023-09-23T18:58:09.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:08 smithi092 bash[27556]: audit 2023-09-23T18:58:07.796368+0000 mon.a (mon.0) 238 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T18:58:09.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:08 smithi092 bash[27556]: audit 2023-09-23T18:58:07.801590+0000 mon.a (mon.0) 239 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T18:58:09.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:08 smithi092 bash[27556]: audit 2023-09-23T18:58:08.140147+0000 mon.a (mon.0) 240 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:58:09.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:08 smithi092 bash[27556]: audit 2023-09-23T18:58:08.182858+0000 mon.a (mon.0) 241 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:58:09.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:08 smithi092 bash[27556]: cluster 2023-09-23T18:58:08.606008+0000 mgr.y (mgr.14146) 76 : cluster [DBG] pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:58:10.166 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:09 smithi016 bash[27237]: cluster 2023-09-23T18:58:07.335879+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T18:58:10.167 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:09 smithi016 bash[27237]: cluster 2023-09-23T18:58:07.335974+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T18:58:10.167 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:09 smithi016 bash[27237]: audit 2023-09-23T18:58:08.806172+0000 mon.a (mon.0) 242 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T18:58:10.167 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:09 smithi016 bash[27237]: cluster 2023-09-23T18:58:08.810632+0000 mon.a (mon.0) 243 : cluster [INF] osd.0 [v2:172.21.15.16:6802/2337435219,v1:172.21.15.16:6803/2337435219] boot 2023-09-23T18:58:10.167 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:09 smithi016 bash[27237]: cluster 2023-09-23T18:58:08.810737+0000 mon.a (mon.0) 244 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2023-09-23T18:58:10.167 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:09 smithi016 bash[27237]: audit 2023-09-23T18:58:08.810937+0000 mon.a (mon.0) 245 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T18:58:10.167 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:09 smithi016 bash[33624]: cluster 2023-09-23T18:58:07.335879+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T18:58:10.167 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:09 smithi016 bash[33624]: cluster 2023-09-23T18:58:07.335974+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T18:58:10.167 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:09 smithi016 bash[33624]: audit 2023-09-23T18:58:08.806172+0000 mon.a (mon.0) 242 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T18:58:10.167 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:09 smithi016 bash[33624]: cluster 2023-09-23T18:58:08.810632+0000 mon.a (mon.0) 243 : cluster [INF] osd.0 [v2:172.21.15.16:6802/2337435219,v1:172.21.15.16:6803/2337435219] boot 2023-09-23T18:58:10.167 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:09 smithi016 bash[33624]: cluster 2023-09-23T18:58:08.810737+0000 mon.a (mon.0) 244 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2023-09-23T18:58:10.168 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:09 smithi016 bash[33624]: audit 2023-09-23T18:58:08.810937+0000 mon.a (mon.0) 245 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T18:58:10.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:09 smithi092 bash[27556]: cluster 2023-09-23T18:58:07.335879+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T18:58:10.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:09 smithi092 bash[27556]: cluster 2023-09-23T18:58:07.335974+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T18:58:10.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:09 smithi092 bash[27556]: audit 2023-09-23T18:58:08.806172+0000 mon.a (mon.0) 242 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T18:58:10.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:09 smithi092 bash[27556]: cluster 2023-09-23T18:58:08.810632+0000 mon.a (mon.0) 243 : cluster [INF] osd.0 [v2:172.21.15.16:6802/2337435219,v1:172.21.15.16:6803/2337435219] boot 2023-09-23T18:58:10.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:09 smithi092 bash[27556]: cluster 2023-09-23T18:58:08.810737+0000 mon.a (mon.0) 244 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2023-09-23T18:58:10.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:09 smithi092 bash[27556]: audit 2023-09-23T18:58:08.810937+0000 mon.a (mon.0) 245 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T18:58:11.533 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:11 smithi016 bash[27237]: cluster 2023-09-23T18:58:10.147511+0000 mon.a (mon.0) 246 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2023-09-23T18:58:11.533 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:11 smithi016 bash[27237]: cluster 2023-09-23T18:58:10.606527+0000 mgr.y (mgr.14146) 77 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:11.533 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:11 smithi016 bash[33624]: cluster 2023-09-23T18:58:10.147511+0000 mon.a (mon.0) 246 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2023-09-23T18:58:11.533 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:11 smithi016 bash[33624]: cluster 2023-09-23T18:58:10.606527+0000 mgr.y (mgr.14146) 77 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:11.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:11 smithi092 bash[27556]: cluster 2023-09-23T18:58:10.147511+0000 mon.a (mon.0) 246 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2023-09-23T18:58:11.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:11 smithi092 bash[27556]: cluster 2023-09-23T18:58:10.606527+0000 mgr.y (mgr.14146) 77 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:11.999 INFO:teuthology.orchestra.run.smithi016.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-09-23T18:58:11.999 INFO:teuthology.orchestra.run.smithi016.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-09-23T18:58:11.999 INFO:teuthology.orchestra.run.smithi016.stderr: stderr: 10+0 records in 2023-09-23T18:58:11.999 INFO:teuthology.orchestra.run.smithi016.stderr:10+0 records out 2023-09-23T18:58:11.999 INFO:teuthology.orchestra.run.smithi016.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0125477 s, 836 MB/s 2023-09-23T18:58:11.999 INFO:teuthology.orchestra.run.smithi016.stderr:--> Zapping successful for: 2023-09-23T18:58:12.692 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi016:vg_nvme/lv_3 2023-09-23T18:58:14.027 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:13 smithi016 bash[27237]: cluster 2023-09-23T18:58:12.607040+0000 mgr.y (mgr.14146) 78 : cluster [DBG] pgmap v55: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:14.027 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:13 smithi016 bash[33624]: cluster 2023-09-23T18:58:12.607040+0000 mgr.y (mgr.14146) 78 : cluster [DBG] pgmap v55: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:14.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:13 smithi092 bash[27556]: cluster 2023-09-23T18:58:12.607040+0000 mgr.y (mgr.14146) 78 : cluster [DBG] pgmap v55: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:16.283 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:15 smithi016 bash[33624]: cluster 2023-09-23T18:58:14.607449+0000 mgr.y (mgr.14146) 79 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:16.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:15 smithi016 bash[27237]: cluster 2023-09-23T18:58:14.607449+0000 mgr.y (mgr.14146) 79 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:16.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:15 smithi092 bash[27556]: cluster 2023-09-23T18:58:14.607449+0000 mgr.y (mgr.14146) 79 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:17.219 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:16 smithi016 bash[33624]: cluster 2023-09-23T18:58:16.607787+0000 mgr.y (mgr.14146) 80 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:17.219 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:16 smithi016 bash[33624]: audit 2023-09-23T18:58:16.798402+0000 mon.a (mon.0) 247 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T18:58:17.219 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:16 smithi016 bash[33624]: audit 2023-09-23T18:58:16.801963+0000 mon.a (mon.0) 248 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T18:58:17.219 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:16 smithi016 bash[33624]: audit 2023-09-23T18:58:16.802893+0000 mon.a (mon.0) 249 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:58:17.220 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:16 smithi016 bash[27237]: cluster 2023-09-23T18:58:16.607787+0000 mgr.y (mgr.14146) 80 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:17.220 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:16 smithi016 bash[27237]: audit 2023-09-23T18:58:16.798402+0000 mon.a (mon.0) 247 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T18:58:17.220 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:16 smithi016 bash[27237]: audit 2023-09-23T18:58:16.801963+0000 mon.a (mon.0) 248 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T18:58:17.220 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:16 smithi016 bash[27237]: audit 2023-09-23T18:58:16.802893+0000 mon.a (mon.0) 249 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:58:17.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:16 smithi092 bash[27556]: cluster 2023-09-23T18:58:16.607787+0000 mgr.y (mgr.14146) 80 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:17.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:16 smithi092 bash[27556]: audit 2023-09-23T18:58:16.798402+0000 mon.a (mon.0) 247 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T18:58:17.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:16 smithi092 bash[27556]: audit 2023-09-23T18:58:16.801963+0000 mon.a (mon.0) 248 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T18:58:17.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:16 smithi092 bash[27556]: audit 2023-09-23T18:58:16.802893+0000 mon.a (mon.0) 249 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:58:18.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:17 smithi016 bash[27237]: audit 2023-09-23T18:58:16.794117+0000 mgr.y (mgr.14146) 81 : audit [DBG] from='client.14238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi016:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:58:18.283 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:17 smithi016 bash[33624]: audit 2023-09-23T18:58:16.794117+0000 mgr.y (mgr.14146) 81 : audit [DBG] from='client.14238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi016:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:58:18.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:17 smithi092 bash[27556]: audit 2023-09-23T18:58:16.794117+0000 mgr.y (mgr.14146) 81 : audit [DBG] from='client.14238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi016:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:58:19.533 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:19 smithi016 bash[33624]: cephadm 2023-09-23T18:58:18.077810+0000 mgr.y (mgr.14146) 82 : cephadm [INF] Detected new or changed devices on smithi016 2023-09-23T18:58:19.533 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:19 smithi016 bash[33624]: audit 2023-09-23T18:58:18.085775+0000 mon.a (mon.0) 250 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:58:19.533 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:19 smithi016 bash[33624]: audit 2023-09-23T18:58:18.087339+0000 mon.a (mon.0) 251 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:58:19.533 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:19 smithi016 bash[33624]: cephadm 2023-09-23T18:58:18.088291+0000 mgr.y (mgr.14146) 83 : cephadm [INF] Adjusting osd_memory_target on smithi016 to 16245M 2023-09-23T18:58:19.533 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:19 smithi016 bash[33624]: audit 2023-09-23T18:58:18.093003+0000 mon.a (mon.0) 252 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:58:19.533 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:19 smithi016 bash[33624]: audit 2023-09-23T18:58:18.094427+0000 mon.a (mon.0) 253 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:58:19.533 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:19 smithi016 bash[33624]: audit 2023-09-23T18:58:18.095756+0000 mon.a (mon.0) 254 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:58:19.533 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:19 smithi016 bash[33624]: audit 2023-09-23T18:58:18.101679+0000 mon.a (mon.0) 255 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:58:19.533 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:19 smithi016 bash[33624]: cluster 2023-09-23T18:58:18.608283+0000 mgr.y (mgr.14146) 84 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:19.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:19 smithi016 bash[27237]: cephadm 2023-09-23T18:58:18.077810+0000 mgr.y (mgr.14146) 82 : cephadm [INF] Detected new or changed devices on smithi016 2023-09-23T18:58:19.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:19 smithi016 bash[27237]: audit 2023-09-23T18:58:18.085775+0000 mon.a (mon.0) 250 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:58:19.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:19 smithi016 bash[27237]: audit 2023-09-23T18:58:18.087339+0000 mon.a (mon.0) 251 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:58:19.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:19 smithi016 bash[27237]: cephadm 2023-09-23T18:58:18.088291+0000 mgr.y (mgr.14146) 83 : cephadm [INF] Adjusting osd_memory_target on smithi016 to 16245M 2023-09-23T18:58:19.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:19 smithi016 bash[27237]: audit 2023-09-23T18:58:18.093003+0000 mon.a (mon.0) 252 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:58:19.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:19 smithi016 bash[27237]: audit 2023-09-23T18:58:18.094427+0000 mon.a (mon.0) 253 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:58:19.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:19 smithi016 bash[27237]: audit 2023-09-23T18:58:18.095756+0000 mon.a (mon.0) 254 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:58:19.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:19 smithi016 bash[27237]: audit 2023-09-23T18:58:18.101679+0000 mon.a (mon.0) 255 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:58:19.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:19 smithi016 bash[27237]: cluster 2023-09-23T18:58:18.608283+0000 mgr.y (mgr.14146) 84 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:19.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:19 smithi092 bash[27556]: cephadm 2023-09-23T18:58:18.077810+0000 mgr.y (mgr.14146) 82 : cephadm [INF] Detected new or changed devices on smithi016 2023-09-23T18:58:19.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:19 smithi092 bash[27556]: audit 2023-09-23T18:58:18.085775+0000 mon.a (mon.0) 250 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:58:19.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:19 smithi092 bash[27556]: audit 2023-09-23T18:58:18.087339+0000 mon.a (mon.0) 251 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:58:19.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:19 smithi092 bash[27556]: cephadm 2023-09-23T18:58:18.088291+0000 mgr.y (mgr.14146) 83 : cephadm [INF] Adjusting osd_memory_target on smithi016 to 16245M 2023-09-23T18:58:19.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:19 smithi092 bash[27556]: audit 2023-09-23T18:58:18.093003+0000 mon.a (mon.0) 252 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:58:19.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:19 smithi092 bash[27556]: audit 2023-09-23T18:58:18.094427+0000 mon.a (mon.0) 253 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:58:19.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:19 smithi092 bash[27556]: audit 2023-09-23T18:58:18.095756+0000 mon.a (mon.0) 254 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:58:19.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:19 smithi092 bash[27556]: audit 2023-09-23T18:58:18.101679+0000 mon.a (mon.0) 255 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:58:19.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:19 smithi092 bash[27556]: cluster 2023-09-23T18:58:18.608283+0000 mgr.y (mgr.14146) 84 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:22.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:21 smithi016 bash[27237]: cluster 2023-09-23T18:58:20.608774+0000 mgr.y (mgr.14146) 85 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:22.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:21 smithi016 bash[33624]: cluster 2023-09-23T18:58:20.608774+0000 mgr.y (mgr.14146) 85 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:22.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:21 smithi092 bash[27556]: cluster 2023-09-23T18:58:20.608774+0000 mgr.y (mgr.14146) 85 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:23.282 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:22 smithi016 bash[33624]: cluster 2023-09-23T18:58:22.609233+0000 mgr.y (mgr.14146) 86 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:23.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:22 smithi016 bash[27237]: cluster 2023-09-23T18:58:22.609233+0000 mgr.y (mgr.14146) 86 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:23.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:22 smithi092 bash[27556]: cluster 2023-09-23T18:58:22.609233+0000 mgr.y (mgr.14146) 86 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:26.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:25 smithi016 bash[27237]: cluster 2023-09-23T18:58:24.609712+0000 mgr.y (mgr.14146) 87 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:26.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:25 smithi016 bash[27237]: audit 2023-09-23T18:58:24.870827+0000 mon.a (mon.0) 256 : audit [INF] from='client.? 172.21.15.16:0/2946385046' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c5ff2879-0d1c-481a-b847-248433c12aa4"}]: dispatch 2023-09-23T18:58:26.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:25 smithi016 bash[27237]: audit 2023-09-23T18:58:24.878493+0000 mon.a (mon.0) 257 : audit [INF] from='client.? 172.21.15.16:0/2946385046' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c5ff2879-0d1c-481a-b847-248433c12aa4"}]': finished 2023-09-23T18:58:26.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:25 smithi016 bash[27237]: cluster 2023-09-23T18:58:24.878604+0000 mon.a (mon.0) 258 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2023-09-23T18:58:26.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:25 smithi016 bash[27237]: audit 2023-09-23T18:58:24.878784+0000 mon.a (mon.0) 259 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T18:58:26.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:25 smithi016 bash[27237]: audit 2023-09-23T18:58:25.612763+0000 mon.a (mon.0) 260 : audit [DBG] from='client.? 172.21.15.16:0/657078758' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T18:58:26.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:25 smithi016 bash[33624]: cluster 2023-09-23T18:58:24.609712+0000 mgr.y (mgr.14146) 87 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:26.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:25 smithi016 bash[33624]: audit 2023-09-23T18:58:24.870827+0000 mon.a (mon.0) 256 : audit [INF] from='client.? 172.21.15.16:0/2946385046' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c5ff2879-0d1c-481a-b847-248433c12aa4"}]: dispatch 2023-09-23T18:58:26.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:25 smithi016 bash[33624]: audit 2023-09-23T18:58:24.878493+0000 mon.a (mon.0) 257 : audit [INF] from='client.? 172.21.15.16:0/2946385046' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c5ff2879-0d1c-481a-b847-248433c12aa4"}]': finished 2023-09-23T18:58:26.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:25 smithi016 bash[33624]: cluster 2023-09-23T18:58:24.878604+0000 mon.a (mon.0) 258 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2023-09-23T18:58:26.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:25 smithi016 bash[33624]: audit 2023-09-23T18:58:24.878784+0000 mon.a (mon.0) 259 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T18:58:26.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:25 smithi016 bash[33624]: audit 2023-09-23T18:58:25.612763+0000 mon.a (mon.0) 260 : audit [DBG] from='client.? 172.21.15.16:0/657078758' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T18:58:26.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:25 smithi092 bash[27556]: cluster 2023-09-23T18:58:24.609712+0000 mgr.y (mgr.14146) 87 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:26.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:25 smithi092 bash[27556]: audit 2023-09-23T18:58:24.870827+0000 mon.a (mon.0) 256 : audit [INF] from='client.? 172.21.15.16:0/2946385046' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c5ff2879-0d1c-481a-b847-248433c12aa4"}]: dispatch 2023-09-23T18:58:26.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:25 smithi092 bash[27556]: audit 2023-09-23T18:58:24.878493+0000 mon.a (mon.0) 257 : audit [INF] from='client.? 172.21.15.16:0/2946385046' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c5ff2879-0d1c-481a-b847-248433c12aa4"}]': finished 2023-09-23T18:58:26.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:25 smithi092 bash[27556]: cluster 2023-09-23T18:58:24.878604+0000 mon.a (mon.0) 258 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2023-09-23T18:58:26.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:25 smithi092 bash[27556]: audit 2023-09-23T18:58:24.878784+0000 mon.a (mon.0) 259 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T18:58:26.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:25 smithi092 bash[27556]: audit 2023-09-23T18:58:25.612763+0000 mon.a (mon.0) 260 : audit [DBG] from='client.? 172.21.15.16:0/657078758' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T18:58:27.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:26 smithi016 bash[27237]: cluster 2023-09-23T18:58:26.610112+0000 mgr.y (mgr.14146) 88 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:27.283 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:26 smithi016 bash[33624]: cluster 2023-09-23T18:58:26.610112+0000 mgr.y (mgr.14146) 88 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:27.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:26 smithi092 bash[27556]: cluster 2023-09-23T18:58:26.610112+0000 mgr.y (mgr.14146) 88 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:30.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:29 smithi016 bash[33624]: cluster 2023-09-23T18:58:28.610521+0000 mgr.y (mgr.14146) 89 : cluster [DBG] pgmap v64: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:30.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:29 smithi016 bash[27237]: cluster 2023-09-23T18:58:28.610521+0000 mgr.y (mgr.14146) 89 : cluster [DBG] pgmap v64: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:30.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:29 smithi092 bash[27556]: cluster 2023-09-23T18:58:28.610521+0000 mgr.y (mgr.14146) 89 : cluster [DBG] pgmap v64: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:32.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:31 smithi016 bash[27237]: cluster 2023-09-23T18:58:30.610956+0000 mgr.y (mgr.14146) 90 : cluster [DBG] pgmap v65: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:32.283 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:31 smithi016 bash[33624]: cluster 2023-09-23T18:58:30.610956+0000 mgr.y (mgr.14146) 90 : cluster [DBG] pgmap v65: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:32.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:31 smithi092 bash[27556]: cluster 2023-09-23T18:58:30.610956+0000 mgr.y (mgr.14146) 90 : cluster [DBG] pgmap v65: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:33.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:32 smithi016 bash[27237]: cluster 2023-09-23T18:58:32.611437+0000 mgr.y (mgr.14146) 91 : cluster [DBG] pgmap v66: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:33.283 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:32 smithi016 bash[33624]: cluster 2023-09-23T18:58:32.611437+0000 mgr.y (mgr.14146) 91 : cluster [DBG] pgmap v66: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:33.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:32 smithi092 bash[27556]: cluster 2023-09-23T18:58:32.611437+0000 mgr.y (mgr.14146) 91 : cluster [DBG] pgmap v66: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:36.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:35 smithi016 bash[27237]: cluster 2023-09-23T18:58:34.611936+0000 mgr.y (mgr.14146) 92 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:36.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:35 smithi016 bash[33624]: cluster 2023-09-23T18:58:34.611936+0000 mgr.y (mgr.14146) 92 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:36.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:35 smithi092 bash[27556]: cluster 2023-09-23T18:58:34.611936+0000 mgr.y (mgr.14146) 92 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:37.121 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:36 smithi016 bash[27237]: cluster 2023-09-23T18:58:36.612492+0000 mgr.y (mgr.14146) 93 : cluster [DBG] pgmap v68: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:37.121 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:36 smithi016 bash[33624]: cluster 2023-09-23T18:58:36.612492+0000 mgr.y (mgr.14146) 93 : cluster [DBG] pgmap v68: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:37.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:36 smithi092 bash[27556]: cluster 2023-09-23T18:58:36.612492+0000 mgr.y (mgr.14146) 93 : cluster [DBG] pgmap v68: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:38.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:37 smithi016 bash[27237]: audit 2023-09-23T18:58:37.603734+0000 mon.a (mon.0) 261 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T18:58:38.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:37 smithi016 bash[27237]: audit 2023-09-23T18:58:37.605349+0000 mon.a (mon.0) 262 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:58:38.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:37 smithi016 bash[27237]: cephadm 2023-09-23T18:58:37.606706+0000 mgr.y (mgr.14146) 94 : cephadm [INF] Deploying daemon osd.1 on smithi016 2023-09-23T18:58:38.283 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:37 smithi016 bash[33624]: audit 2023-09-23T18:58:37.603734+0000 mon.a (mon.0) 261 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T18:58:38.283 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:37 smithi016 bash[33624]: audit 2023-09-23T18:58:37.605349+0000 mon.a (mon.0) 262 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:58:38.283 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:37 smithi016 bash[33624]: cephadm 2023-09-23T18:58:37.606706+0000 mgr.y (mgr.14146) 94 : cephadm [INF] Deploying daemon osd.1 on smithi016 2023-09-23T18:58:38.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:37 smithi092 bash[27556]: audit 2023-09-23T18:58:37.603734+0000 mon.a (mon.0) 261 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T18:58:38.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:37 smithi092 bash[27556]: audit 2023-09-23T18:58:37.605349+0000 mon.a (mon.0) 262 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:58:38.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:37 smithi092 bash[27556]: cephadm 2023-09-23T18:58:37.606706+0000 mgr.y (mgr.14146) 94 : cephadm [INF] Deploying daemon osd.1 on smithi016 2023-09-23T18:58:39.021 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:38 smithi016 bash[33624]: cluster 2023-09-23T18:58:38.612896+0000 mgr.y (mgr.14146) 95 : cluster [DBG] pgmap v69: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:39.021 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:38 smithi016 bash[27237]: cluster 2023-09-23T18:58:38.612896+0000 mgr.y (mgr.14146) 95 : cluster [DBG] pgmap v69: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:39.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:38 smithi092 bash[27556]: cluster 2023-09-23T18:58:38.612896+0000 mgr.y (mgr.14146) 95 : cluster [DBG] pgmap v69: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:41.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:40 smithi016 bash[33624]: cluster 2023-09-23T18:58:40.613361+0000 mgr.y (mgr.14146) 96 : cluster [DBG] pgmap v70: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:41.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:40 smithi016 bash[27237]: cluster 2023-09-23T18:58:40.613361+0000 mgr.y (mgr.14146) 96 : cluster [DBG] pgmap v70: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:41.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:40 smithi092 bash[27556]: cluster 2023-09-23T18:58:40.613361+0000 mgr.y (mgr.14146) 96 : cluster [DBG] pgmap v70: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:42.783 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:42 smithi016 bash[27237]: audit 2023-09-23T18:58:41.667472+0000 mon.a (mon.0) 263 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:58:42.783 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:42 smithi016 bash[27237]: audit 2023-09-23T18:58:41.680073+0000 mon.a (mon.0) 264 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:58:42.783 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:42 smithi016 bash[33624]: audit 2023-09-23T18:58:41.667472+0000 mon.a (mon.0) 263 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:58:42.783 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:42 smithi016 bash[33624]: audit 2023-09-23T18:58:41.680073+0000 mon.a (mon.0) 264 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:58:43.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:42 smithi092 bash[27556]: audit 2023-09-23T18:58:41.667472+0000 mon.a (mon.0) 263 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:58:43.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:42 smithi092 bash[27556]: audit 2023-09-23T18:58:41.680073+0000 mon.a (mon.0) 264 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:58:44.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:43 smithi016 bash[27237]: cluster 2023-09-23T18:58:42.613716+0000 mgr.y (mgr.14146) 97 : cluster [DBG] pgmap v71: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:44.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:43 smithi016 bash[33624]: cluster 2023-09-23T18:58:42.613716+0000 mgr.y (mgr.14146) 97 : cluster [DBG] pgmap v71: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:44.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:43 smithi092 bash[27556]: cluster 2023-09-23T18:58:42.613716+0000 mgr.y (mgr.14146) 97 : cluster [DBG] pgmap v71: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:46.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:45 smithi016 bash[27237]: cluster 2023-09-23T18:58:44.614123+0000 mgr.y (mgr.14146) 98 : cluster [DBG] pgmap v72: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:46.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:45 smithi016 bash[33624]: cluster 2023-09-23T18:58:44.614123+0000 mgr.y (mgr.14146) 98 : cluster [DBG] pgmap v72: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:46.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:45 smithi092 bash[27556]: cluster 2023-09-23T18:58:44.614123+0000 mgr.y (mgr.14146) 98 : cluster [DBG] pgmap v72: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:48.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:47 smithi016 bash[27237]: cluster 2023-09-23T18:58:46.614456+0000 mgr.y (mgr.14146) 99 : cluster [DBG] pgmap v73: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:48.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:47 smithi016 bash[27237]: audit 2023-09-23T18:58:46.878998+0000 mon.a (mon.0) 265 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:58:48.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:47 smithi016 bash[27237]: audit 2023-09-23T18:58:46.880102+0000 mon.a (mon.0) 266 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:58:48.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:47 smithi016 bash[27237]: audit 2023-09-23T18:58:46.881216+0000 mon.a (mon.0) 267 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:58:48.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:47 smithi016 bash[27237]: audit 2023-09-23T18:58:46.887201+0000 mon.a (mon.0) 268 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:58:48.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:47 smithi016 bash[33624]: cluster 2023-09-23T18:58:46.614456+0000 mgr.y (mgr.14146) 99 : cluster [DBG] pgmap v73: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:48.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:47 smithi016 bash[33624]: audit 2023-09-23T18:58:46.878998+0000 mon.a (mon.0) 265 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:58:48.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:47 smithi016 bash[33624]: audit 2023-09-23T18:58:46.880102+0000 mon.a (mon.0) 266 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:58:48.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:47 smithi016 bash[33624]: audit 2023-09-23T18:58:46.881216+0000 mon.a (mon.0) 267 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:58:48.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:47 smithi016 bash[33624]: audit 2023-09-23T18:58:46.887201+0000 mon.a (mon.0) 268 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:58:48.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:47 smithi092 bash[27556]: cluster 2023-09-23T18:58:46.614456+0000 mgr.y (mgr.14146) 99 : cluster [DBG] pgmap v73: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:48.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:47 smithi092 bash[27556]: audit 2023-09-23T18:58:46.878998+0000 mon.a (mon.0) 265 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:58:48.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:47 smithi092 bash[27556]: audit 2023-09-23T18:58:46.880102+0000 mon.a (mon.0) 266 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:58:48.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:47 smithi092 bash[27556]: audit 2023-09-23T18:58:46.881216+0000 mon.a (mon.0) 267 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:58:48.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:47 smithi092 bash[27556]: audit 2023-09-23T18:58:46.887201+0000 mon.a (mon.0) 268 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:58:49.016 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:48 smithi016 bash[27237]: audit 2023-09-23T18:58:48.303021+0000 mon.c (mon.2) 5 : audit [INF] from='osd.1 [v2:172.21.15.16:6810/3893879931,v1:172.21.15.16:6811/3893879931]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T18:58:49.016 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:48 smithi016 bash[27237]: audit 2023-09-23T18:58:48.303629+0000 mon.a (mon.0) 269 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T18:58:49.017 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:48 smithi016 bash[33624]: audit 2023-09-23T18:58:48.303021+0000 mon.c (mon.2) 5 : audit [INF] from='osd.1 [v2:172.21.15.16:6810/3893879931,v1:172.21.15.16:6811/3893879931]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T18:58:49.017 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:48 smithi016 bash[33624]: audit 2023-09-23T18:58:48.303629+0000 mon.a (mon.0) 269 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T18:58:49.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:48 smithi092 bash[27556]: audit 2023-09-23T18:58:48.303021+0000 mon.c (mon.2) 5 : audit [INF] from='osd.1 [v2:172.21.15.16:6810/3893879931,v1:172.21.15.16:6811/3893879931]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T18:58:49.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:48 smithi092 bash[27556]: audit 2023-09-23T18:58:48.303629+0000 mon.a (mon.0) 269 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T18:58:50.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:49 smithi016 bash[33624]: cluster 2023-09-23T18:58:48.614917+0000 mgr.y (mgr.14146) 100 : cluster [DBG] pgmap v74: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:50.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:49 smithi016 bash[33624]: audit 2023-09-23T18:58:48.693719+0000 mon.a (mon.0) 270 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T18:58:50.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:49 smithi016 bash[33624]: cluster 2023-09-23T18:58:48.693778+0000 mon.a (mon.0) 271 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2023-09-23T18:58:50.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:49 smithi016 bash[33624]: audit 2023-09-23T18:58:48.693979+0000 mon.a (mon.0) 272 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T18:58:50.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:49 smithi016 bash[33624]: audit 2023-09-23T18:58:48.694739+0000 mon.c (mon.2) 6 : audit [INF] from='osd.1 [v2:172.21.15.16:6810/3893879931,v1:172.21.15.16:6811/3893879931]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-09-23T18:58:50.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:49 smithi016 bash[33624]: audit 2023-09-23T18:58:48.695251+0000 mon.a (mon.0) 273 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-09-23T18:58:50.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:49 smithi016 bash[27237]: cluster 2023-09-23T18:58:48.614917+0000 mgr.y (mgr.14146) 100 : cluster [DBG] pgmap v74: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:50.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:49 smithi016 bash[27237]: audit 2023-09-23T18:58:48.693719+0000 mon.a (mon.0) 270 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T18:58:50.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:49 smithi016 bash[27237]: cluster 2023-09-23T18:58:48.693778+0000 mon.a (mon.0) 271 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2023-09-23T18:58:50.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:49 smithi016 bash[27237]: audit 2023-09-23T18:58:48.693979+0000 mon.a (mon.0) 272 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T18:58:50.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:49 smithi016 bash[27237]: audit 2023-09-23T18:58:48.694739+0000 mon.c (mon.2) 6 : audit [INF] from='osd.1 [v2:172.21.15.16:6810/3893879931,v1:172.21.15.16:6811/3893879931]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-09-23T18:58:50.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:49 smithi016 bash[27237]: audit 2023-09-23T18:58:48.695251+0000 mon.a (mon.0) 273 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-09-23T18:58:50.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:49 smithi092 bash[27556]: cluster 2023-09-23T18:58:48.614917+0000 mgr.y (mgr.14146) 100 : cluster [DBG] pgmap v74: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:50.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:49 smithi092 bash[27556]: audit 2023-09-23T18:58:48.693719+0000 mon.a (mon.0) 270 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T18:58:50.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:49 smithi092 bash[27556]: cluster 2023-09-23T18:58:48.693778+0000 mon.a (mon.0) 271 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2023-09-23T18:58:50.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:49 smithi092 bash[27556]: audit 2023-09-23T18:58:48.693979+0000 mon.a (mon.0) 272 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T18:58:50.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:49 smithi092 bash[27556]: audit 2023-09-23T18:58:48.694739+0000 mon.c (mon.2) 6 : audit [INF] from='osd.1 [v2:172.21.15.16:6810/3893879931,v1:172.21.15.16:6811/3893879931]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-09-23T18:58:50.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:49 smithi092 bash[27556]: audit 2023-09-23T18:58:48.695251+0000 mon.a (mon.0) 273 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-09-23T18:58:50.373 INFO:teuthology.orchestra.run.smithi016.stdout:Created osd(s) 1 on host 'smithi016' 2023-09-23T18:58:50.974 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:50 smithi016 bash[27237]: audit 2023-09-23T18:58:49.696494+0000 mon.a (mon.0) 274 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]': finished 2023-09-23T18:58:50.974 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:50 smithi016 bash[27237]: cluster 2023-09-23T18:58:49.696580+0000 mon.a (mon.0) 275 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2023-09-23T18:58:50.974 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:50 smithi016 bash[27237]: audit 2023-09-23T18:58:49.697008+0000 mon.a (mon.0) 276 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T18:58:50.974 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:50 smithi016 bash[27237]: audit 2023-09-23T18:58:49.702553+0000 mon.a (mon.0) 277 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T18:58:50.974 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:50 smithi016 bash[27237]: audit 2023-09-23T18:58:50.367001+0000 mon.a (mon.0) 278 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:58:50.974 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:50 smithi016 bash[27237]: audit 2023-09-23T18:58:50.370423+0000 mon.a (mon.0) 279 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:58:50.975 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:50 smithi016 bash[33624]: audit 2023-09-23T18:58:49.696494+0000 mon.a (mon.0) 274 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]': finished 2023-09-23T18:58:50.975 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:50 smithi016 bash[33624]: cluster 2023-09-23T18:58:49.696580+0000 mon.a (mon.0) 275 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2023-09-23T18:58:50.975 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:50 smithi016 bash[33624]: audit 2023-09-23T18:58:49.697008+0000 mon.a (mon.0) 276 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T18:58:50.975 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:50 smithi016 bash[33624]: audit 2023-09-23T18:58:49.702553+0000 mon.a (mon.0) 277 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T18:58:50.975 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:50 smithi016 bash[33624]: audit 2023-09-23T18:58:50.367001+0000 mon.a (mon.0) 278 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:58:50.975 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:50 smithi016 bash[33624]: audit 2023-09-23T18:58:50.370423+0000 mon.a (mon.0) 279 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:58:50.976 DEBUG:teuthology.orchestra.run.smithi016:osd.1> sudo journalctl -f -n 0 -u ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@osd.1.service 2023-09-23T18:58:50.979 INFO:tasks.cephadm:Deploying osd.2 on smithi016 with /dev/vg_nvme/lv_2... 2023-09-23T18:58:50.979 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-09-23T18:58:51.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:50 smithi092 bash[27556]: audit 2023-09-23T18:58:49.696494+0000 mon.a (mon.0) 274 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]': finished 2023-09-23T18:58:51.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:50 smithi092 bash[27556]: cluster 2023-09-23T18:58:49.696580+0000 mon.a (mon.0) 275 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2023-09-23T18:58:51.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:50 smithi092 bash[27556]: audit 2023-09-23T18:58:49.697008+0000 mon.a (mon.0) 276 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T18:58:51.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:50 smithi092 bash[27556]: audit 2023-09-23T18:58:49.702553+0000 mon.a (mon.0) 277 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T18:58:51.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:50 smithi092 bash[27556]: audit 2023-09-23T18:58:50.367001+0000 mon.a (mon.0) 278 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:58:51.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:50 smithi092 bash[27556]: audit 2023-09-23T18:58:50.370423+0000 mon.a (mon.0) 279 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:58:51.068 INFO:journalctl@ceph.osd.1.smithi016.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-09-23T18:58:52.010 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:51 smithi016 bash[27237]: cluster 2023-09-23T18:58:49.342115+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T18:58:52.011 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:51 smithi016 bash[27237]: cluster 2023-09-23T18:58:49.342208+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T18:58:52.011 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:51 smithi016 bash[27237]: cluster 2023-09-23T18:58:50.615397+0000 mgr.y (mgr.14146) 101 : cluster [DBG] pgmap v77: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:52.011 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:51 smithi016 bash[27237]: audit 2023-09-23T18:58:50.707723+0000 mon.a (mon.0) 280 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T18:58:52.011 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:51 smithi016 bash[27237]: cluster 2023-09-23T18:58:50.711205+0000 mon.a (mon.0) 281 : cluster [INF] osd.1 [v2:172.21.15.16:6810/3893879931,v1:172.21.15.16:6811/3893879931] boot 2023-09-23T18:58:52.011 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:51 smithi016 bash[27237]: cluster 2023-09-23T18:58:50.711252+0000 mon.a (mon.0) 282 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2023-09-23T18:58:52.011 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:51 smithi016 bash[27237]: audit 2023-09-23T18:58:50.711421+0000 mon.a (mon.0) 283 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T18:58:52.011 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:51 smithi016 bash[33624]: cluster 2023-09-23T18:58:49.342115+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T18:58:52.011 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:51 smithi016 bash[33624]: cluster 2023-09-23T18:58:49.342208+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T18:58:52.012 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:51 smithi016 bash[33624]: cluster 2023-09-23T18:58:50.615397+0000 mgr.y (mgr.14146) 101 : cluster [DBG] pgmap v77: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:52.012 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:51 smithi016 bash[33624]: audit 2023-09-23T18:58:50.707723+0000 mon.a (mon.0) 280 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T18:58:52.012 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:51 smithi016 bash[33624]: cluster 2023-09-23T18:58:50.711205+0000 mon.a (mon.0) 281 : cluster [INF] osd.1 [v2:172.21.15.16:6810/3893879931,v1:172.21.15.16:6811/3893879931] boot 2023-09-23T18:58:52.012 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:51 smithi016 bash[33624]: cluster 2023-09-23T18:58:50.711252+0000 mon.a (mon.0) 282 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2023-09-23T18:58:52.012 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:51 smithi016 bash[33624]: audit 2023-09-23T18:58:50.711421+0000 mon.a (mon.0) 283 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T18:58:52.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:51 smithi092 bash[27556]: cluster 2023-09-23T18:58:49.342115+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T18:58:52.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:51 smithi092 bash[27556]: cluster 2023-09-23T18:58:49.342208+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T18:58:52.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:51 smithi092 bash[27556]: cluster 2023-09-23T18:58:50.615397+0000 mgr.y (mgr.14146) 101 : cluster [DBG] pgmap v77: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:58:52.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:51 smithi092 bash[27556]: audit 2023-09-23T18:58:50.707723+0000 mon.a (mon.0) 280 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T18:58:52.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:51 smithi092 bash[27556]: cluster 2023-09-23T18:58:50.711205+0000 mon.a (mon.0) 281 : cluster [INF] osd.1 [v2:172.21.15.16:6810/3893879931,v1:172.21.15.16:6811/3893879931] boot 2023-09-23T18:58:52.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:51 smithi092 bash[27556]: cluster 2023-09-23T18:58:50.711252+0000 mon.a (mon.0) 282 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2023-09-23T18:58:52.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:51 smithi092 bash[27556]: audit 2023-09-23T18:58:50.711421+0000 mon.a (mon.0) 283 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T18:58:53.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:52 smithi016 bash[27237]: cluster 2023-09-23T18:58:51.716726+0000 mon.a (mon.0) 284 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2023-09-23T18:58:53.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:52 smithi016 bash[33624]: cluster 2023-09-23T18:58:51.716726+0000 mon.a (mon.0) 284 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2023-09-23T18:58:53.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:52 smithi092 bash[27556]: cluster 2023-09-23T18:58:51.716726+0000 mon.a (mon.0) 284 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2023-09-23T18:58:53.783 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:53 smithi016 bash[33624]: cluster 2023-09-23T18:58:52.615857+0000 mgr.y (mgr.14146) 102 : cluster [DBG] pgmap v80: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:58:53.783 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:53 smithi016 bash[27237]: cluster 2023-09-23T18:58:52.615857+0000 mgr.y (mgr.14146) 102 : cluster [DBG] pgmap v80: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:58:54.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:53 smithi092 bash[27556]: cluster 2023-09-23T18:58:52.615857+0000 mgr.y (mgr.14146) 102 : cluster [DBG] pgmap v80: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:58:54.224 INFO:teuthology.orchestra.run.smithi016.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-09-23T18:58:54.224 INFO:teuthology.orchestra.run.smithi016.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-09-23T18:58:54.224 INFO:teuthology.orchestra.run.smithi016.stderr: stderr: 10+0 records in 2023-09-23T18:58:54.224 INFO:teuthology.orchestra.run.smithi016.stderr:10+0 records out 2023-09-23T18:58:54.224 INFO:teuthology.orchestra.run.smithi016.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.013281 s, 790 MB/s 2023-09-23T18:58:54.224 INFO:teuthology.orchestra.run.smithi016.stderr:--> Zapping successful for: 2023-09-23T18:58:54.894 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi016:vg_nvme/lv_2 2023-09-23T18:58:56.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:55 smithi016 bash[33624]: cluster 2023-09-23T18:58:54.616419+0000 mgr.y (mgr.14146) 103 : cluster [DBG] pgmap v81: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:58:56.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:55 smithi016 bash[27237]: cluster 2023-09-23T18:58:54.616419+0000 mgr.y (mgr.14146) 103 : cluster [DBG] pgmap v81: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:58:56.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:55 smithi092 bash[27556]: cluster 2023-09-23T18:58:54.616419+0000 mgr.y (mgr.14146) 103 : cluster [DBG] pgmap v81: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:58:58.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:57 smithi016 bash[27237]: cluster 2023-09-23T18:58:56.616786+0000 mgr.y (mgr.14146) 104 : cluster [DBG] pgmap v82: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:58:58.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:57 smithi016 bash[33624]: cluster 2023-09-23T18:58:56.616786+0000 mgr.y (mgr.14146) 104 : cluster [DBG] pgmap v82: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:58:58.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:57 smithi092 bash[27556]: cluster 2023-09-23T18:58:56.616786+0000 mgr.y (mgr.14146) 104 : cluster [DBG] pgmap v82: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:59:00.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:59 smithi016 bash[33624]: cluster 2023-09-23T18:58:58.617139+0000 mgr.y (mgr.14146) 105 : cluster [DBG] pgmap v83: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:59:00.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:59 smithi016 bash[33624]: audit 2023-09-23T18:58:58.820017+0000 mon.a (mon.0) 285 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T18:59:00.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:59 smithi016 bash[33624]: audit 2023-09-23T18:58:58.825487+0000 mon.a (mon.0) 286 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T18:59:00.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:58:59 smithi016 bash[33624]: audit 2023-09-23T18:58:58.826690+0000 mon.a (mon.0) 287 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:59:00.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:59 smithi016 bash[27237]: cluster 2023-09-23T18:58:58.617139+0000 mgr.y (mgr.14146) 105 : cluster [DBG] pgmap v83: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:59:00.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:59 smithi016 bash[27237]: audit 2023-09-23T18:58:58.820017+0000 mon.a (mon.0) 285 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T18:59:00.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:59 smithi016 bash[27237]: audit 2023-09-23T18:58:58.825487+0000 mon.a (mon.0) 286 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T18:59:00.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:58:59 smithi016 bash[27237]: audit 2023-09-23T18:58:58.826690+0000 mon.a (mon.0) 287 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:59:00.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:59 smithi092 bash[27556]: cluster 2023-09-23T18:58:58.617139+0000 mgr.y (mgr.14146) 105 : cluster [DBG] pgmap v83: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:59:00.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:59 smithi092 bash[27556]: audit 2023-09-23T18:58:58.820017+0000 mon.a (mon.0) 285 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T18:59:00.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:59 smithi092 bash[27556]: audit 2023-09-23T18:58:58.825487+0000 mon.a (mon.0) 286 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T18:59:00.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:58:59 smithi092 bash[27556]: audit 2023-09-23T18:58:58.826690+0000 mon.a (mon.0) 287 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:59:01.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:00 smithi016 bash[27237]: audit 2023-09-23T18:58:58.814197+0000 mgr.y (mgr.14146) 106 : audit [DBG] from='client.24176 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi016:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:59:01.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:00 smithi016 bash[27237]: audit 2023-09-23T18:59:00.142741+0000 mon.a (mon.0) 288 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:59:01.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:00 smithi016 bash[27237]: audit 2023-09-23T18:59:00.144193+0000 mon.a (mon.0) 289 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:59:01.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:00 smithi016 bash[27237]: audit 2023-09-23T18:59:00.145373+0000 mon.a (mon.0) 290 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:59:01.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:00 smithi016 bash[27237]: audit 2023-09-23T18:59:00.152517+0000 mon.a (mon.0) 291 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:59:01.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:00 smithi016 bash[27237]: audit 2023-09-23T18:59:00.153861+0000 mon.a (mon.0) 292 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:59:01.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:00 smithi016 bash[27237]: audit 2023-09-23T18:59:00.154930+0000 mon.a (mon.0) 293 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:59:01.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:00 smithi016 bash[27237]: audit 2023-09-23T18:59:00.160274+0000 mon.a (mon.0) 294 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:59:01.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:00 smithi016 bash[33624]: audit 2023-09-23T18:58:58.814197+0000 mgr.y (mgr.14146) 106 : audit [DBG] from='client.24176 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi016:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:59:01.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:00 smithi016 bash[33624]: audit 2023-09-23T18:59:00.142741+0000 mon.a (mon.0) 288 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:59:01.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:00 smithi016 bash[33624]: audit 2023-09-23T18:59:00.144193+0000 mon.a (mon.0) 289 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:59:01.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:00 smithi016 bash[33624]: audit 2023-09-23T18:59:00.145373+0000 mon.a (mon.0) 290 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:59:01.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:00 smithi016 bash[33624]: audit 2023-09-23T18:59:00.152517+0000 mon.a (mon.0) 291 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:59:01.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:00 smithi016 bash[33624]: audit 2023-09-23T18:59:00.153861+0000 mon.a (mon.0) 292 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:59:01.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:00 smithi016 bash[33624]: audit 2023-09-23T18:59:00.154930+0000 mon.a (mon.0) 293 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:59:01.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:00 smithi016 bash[33624]: audit 2023-09-23T18:59:00.160274+0000 mon.a (mon.0) 294 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:59:01.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:00 smithi092 bash[27556]: audit 2023-09-23T18:58:58.814197+0000 mgr.y (mgr.14146) 106 : audit [DBG] from='client.24176 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi016:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:59:01.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:00 smithi092 bash[27556]: audit 2023-09-23T18:59:00.142741+0000 mon.a (mon.0) 288 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:59:01.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:00 smithi092 bash[27556]: audit 2023-09-23T18:59:00.144193+0000 mon.a (mon.0) 289 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:59:01.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:00 smithi092 bash[27556]: audit 2023-09-23T18:59:00.145373+0000 mon.a (mon.0) 290 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:59:01.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:00 smithi092 bash[27556]: audit 2023-09-23T18:59:00.152517+0000 mon.a (mon.0) 291 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:59:01.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:00 smithi092 bash[27556]: audit 2023-09-23T18:59:00.153861+0000 mon.a (mon.0) 292 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:59:01.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:00 smithi092 bash[27556]: audit 2023-09-23T18:59:00.154930+0000 mon.a (mon.0) 293 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:59:01.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:00 smithi092 bash[27556]: audit 2023-09-23T18:59:00.160274+0000 mon.a (mon.0) 294 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:59:02.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:01 smithi016 bash[27237]: cephadm 2023-09-23T18:59:00.135100+0000 mgr.y (mgr.14146) 107 : cephadm [INF] Detected new or changed devices on smithi016 2023-09-23T18:59:02.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:01 smithi016 bash[27237]: cephadm 2023-09-23T18:59:00.146227+0000 mgr.y (mgr.14146) 108 : cephadm [INF] Adjusting osd_memory_target on smithi016 to 8122M 2023-09-23T18:59:02.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:01 smithi016 bash[27237]: cluster 2023-09-23T18:59:00.617596+0000 mgr.y (mgr.14146) 109 : cluster [DBG] pgmap v84: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:59:02.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:01 smithi016 bash[33624]: cephadm 2023-09-23T18:59:00.135100+0000 mgr.y (mgr.14146) 107 : cephadm [INF] Detected new or changed devices on smithi016 2023-09-23T18:59:02.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:01 smithi016 bash[33624]: cephadm 2023-09-23T18:59:00.146227+0000 mgr.y (mgr.14146) 108 : cephadm [INF] Adjusting osd_memory_target on smithi016 to 8122M 2023-09-23T18:59:02.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:01 smithi016 bash[33624]: cluster 2023-09-23T18:59:00.617596+0000 mgr.y (mgr.14146) 109 : cluster [DBG] pgmap v84: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:59:02.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:01 smithi092 bash[27556]: cephadm 2023-09-23T18:59:00.135100+0000 mgr.y (mgr.14146) 107 : cephadm [INF] Detected new or changed devices on smithi016 2023-09-23T18:59:02.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:01 smithi092 bash[27556]: cephadm 2023-09-23T18:59:00.146227+0000 mgr.y (mgr.14146) 108 : cephadm [INF] Adjusting osd_memory_target on smithi016 to 8122M 2023-09-23T18:59:02.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:01 smithi092 bash[27556]: cluster 2023-09-23T18:59:00.617596+0000 mgr.y (mgr.14146) 109 : cluster [DBG] pgmap v84: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:59:04.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:03 smithi016 bash[27237]: cluster 2023-09-23T18:59:02.618107+0000 mgr.y (mgr.14146) 110 : cluster [DBG] pgmap v85: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:59:04.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:03 smithi016 bash[33624]: cluster 2023-09-23T18:59:02.618107+0000 mgr.y (mgr.14146) 110 : cluster [DBG] pgmap v85: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:59:04.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:03 smithi092 bash[27556]: cluster 2023-09-23T18:59:02.618107+0000 mgr.y (mgr.14146) 110 : cluster [DBG] pgmap v85: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:59:06.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:05 smithi016 bash[33624]: cluster 2023-09-23T18:59:04.618607+0000 mgr.y (mgr.14146) 111 : cluster [DBG] pgmap v86: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:59:06.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:05 smithi016 bash[27237]: cluster 2023-09-23T18:59:04.618607+0000 mgr.y (mgr.14146) 111 : cluster [DBG] pgmap v86: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:59:06.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:05 smithi092 bash[27556]: cluster 2023-09-23T18:59:04.618607+0000 mgr.y (mgr.14146) 111 : cluster [DBG] pgmap v86: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:59:08.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:07 smithi016 bash[27237]: cluster 2023-09-23T18:59:06.619010+0000 mgr.y (mgr.14146) 112 : cluster [DBG] pgmap v87: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:59:08.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:07 smithi016 bash[27237]: audit 2023-09-23T18:59:06.815884+0000 mon.a (mon.0) 295 : audit [INF] from='client.? 172.21.15.16:0/2548459436' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7974ce4d-9ed8-4a72-b44b-d500b34da8cf"}]: dispatch 2023-09-23T18:59:08.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:07 smithi016 bash[27237]: audit 2023-09-23T18:59:06.823356+0000 mon.a (mon.0) 296 : audit [INF] from='client.? 172.21.15.16:0/2548459436' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7974ce4d-9ed8-4a72-b44b-d500b34da8cf"}]': finished 2023-09-23T18:59:08.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:07 smithi016 bash[27237]: cluster 2023-09-23T18:59:06.823490+0000 mon.a (mon.0) 297 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2023-09-23T18:59:08.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:07 smithi016 bash[27237]: audit 2023-09-23T18:59:06.823667+0000 mon.a (mon.0) 298 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T18:59:08.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:07 smithi016 bash[27237]: audit 2023-09-23T18:59:07.544464+0000 mon.c (mon.2) 7 : audit [DBG] from='client.? 172.21.15.16:0/1877928715' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T18:59:08.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:07 smithi016 bash[33624]: cluster 2023-09-23T18:59:06.619010+0000 mgr.y (mgr.14146) 112 : cluster [DBG] pgmap v87: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:59:08.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:07 smithi016 bash[33624]: audit 2023-09-23T18:59:06.815884+0000 mon.a (mon.0) 295 : audit [INF] from='client.? 172.21.15.16:0/2548459436' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7974ce4d-9ed8-4a72-b44b-d500b34da8cf"}]: dispatch 2023-09-23T18:59:08.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:07 smithi016 bash[33624]: audit 2023-09-23T18:59:06.823356+0000 mon.a (mon.0) 296 : audit [INF] from='client.? 172.21.15.16:0/2548459436' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7974ce4d-9ed8-4a72-b44b-d500b34da8cf"}]': finished 2023-09-23T18:59:08.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:07 smithi016 bash[33624]: cluster 2023-09-23T18:59:06.823490+0000 mon.a (mon.0) 297 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2023-09-23T18:59:08.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:07 smithi016 bash[33624]: audit 2023-09-23T18:59:06.823667+0000 mon.a (mon.0) 298 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T18:59:08.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:07 smithi016 bash[33624]: audit 2023-09-23T18:59:07.544464+0000 mon.c (mon.2) 7 : audit [DBG] from='client.? 172.21.15.16:0/1877928715' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T18:59:08.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:07 smithi092 bash[27556]: cluster 2023-09-23T18:59:06.619010+0000 mgr.y (mgr.14146) 112 : cluster [DBG] pgmap v87: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:59:08.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:07 smithi092 bash[27556]: audit 2023-09-23T18:59:06.815884+0000 mon.a (mon.0) 295 : audit [INF] from='client.? 172.21.15.16:0/2548459436' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7974ce4d-9ed8-4a72-b44b-d500b34da8cf"}]: dispatch 2023-09-23T18:59:08.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:07 smithi092 bash[27556]: audit 2023-09-23T18:59:06.823356+0000 mon.a (mon.0) 296 : audit [INF] from='client.? 172.21.15.16:0/2548459436' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7974ce4d-9ed8-4a72-b44b-d500b34da8cf"}]': finished 2023-09-23T18:59:08.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:07 smithi092 bash[27556]: cluster 2023-09-23T18:59:06.823490+0000 mon.a (mon.0) 297 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2023-09-23T18:59:08.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:07 smithi092 bash[27556]: audit 2023-09-23T18:59:06.823667+0000 mon.a (mon.0) 298 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T18:59:08.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:07 smithi092 bash[27556]: audit 2023-09-23T18:59:07.544464+0000 mon.c (mon.2) 7 : audit [DBG] from='client.? 172.21.15.16:0/1877928715' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T18:59:10.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:09 smithi016 bash[27237]: cluster 2023-09-23T18:59:08.619469+0000 mgr.y (mgr.14146) 113 : cluster [DBG] pgmap v89: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:59:10.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:09 smithi016 bash[33624]: cluster 2023-09-23T18:59:08.619469+0000 mgr.y (mgr.14146) 113 : cluster [DBG] pgmap v89: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:59:10.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:09 smithi092 bash[27556]: cluster 2023-09-23T18:59:08.619469+0000 mgr.y (mgr.14146) 113 : cluster [DBG] pgmap v89: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:59:11.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:10 smithi016 bash[27237]: cluster 2023-09-23T18:59:10.619974+0000 mgr.y (mgr.14146) 114 : cluster [DBG] pgmap v90: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:59:11.283 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:10 smithi016 bash[33624]: cluster 2023-09-23T18:59:10.619974+0000 mgr.y (mgr.14146) 114 : cluster [DBG] pgmap v90: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:59:11.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:10 smithi092 bash[27556]: cluster 2023-09-23T18:59:10.619974+0000 mgr.y (mgr.14146) 114 : cluster [DBG] pgmap v90: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:59:14.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:13 smithi016 bash[27237]: cluster 2023-09-23T18:59:12.620507+0000 mgr.y (mgr.14146) 115 : cluster [DBG] pgmap v91: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:59:14.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:13 smithi016 bash[33624]: cluster 2023-09-23T18:59:12.620507+0000 mgr.y (mgr.14146) 115 : cluster [DBG] pgmap v91: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:59:14.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:13 smithi092 bash[27556]: cluster 2023-09-23T18:59:12.620507+0000 mgr.y (mgr.14146) 115 : cluster [DBG] pgmap v91: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:59:16.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:15 smithi016 bash[27237]: cluster 2023-09-23T18:59:14.621009+0000 mgr.y (mgr.14146) 116 : cluster [DBG] pgmap v92: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:59:16.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:15 smithi016 bash[33624]: cluster 2023-09-23T18:59:14.621009+0000 mgr.y (mgr.14146) 116 : cluster [DBG] pgmap v92: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:59:16.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:15 smithi092 bash[27556]: cluster 2023-09-23T18:59:14.621009+0000 mgr.y (mgr.14146) 116 : cluster [DBG] pgmap v92: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:59:17.936 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:17 smithi016 bash[27237]: cluster 2023-09-23T18:59:16.621511+0000 mgr.y (mgr.14146) 117 : cluster [DBG] pgmap v93: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:59:17.936 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:17 smithi016 bash[33624]: cluster 2023-09-23T18:59:16.621511+0000 mgr.y (mgr.14146) 117 : cluster [DBG] pgmap v93: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:59:18.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:17 smithi092 bash[27556]: cluster 2023-09-23T18:59:16.621511+0000 mgr.y (mgr.14146) 117 : cluster [DBG] pgmap v93: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:59:19.783 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:19 smithi016 bash[27237]: cluster 2023-09-23T18:59:18.622024+0000 mgr.y (mgr.14146) 118 : cluster [DBG] pgmap v94: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:59:19.783 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:19 smithi016 bash[27237]: audit 2023-09-23T18:59:19.237611+0000 mon.a (mon.0) 299 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T18:59:19.783 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:19 smithi016 bash[27237]: audit 2023-09-23T18:59:19.239189+0000 mon.a (mon.0) 300 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:59:19.783 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:19 smithi016 bash[33624]: cluster 2023-09-23T18:59:18.622024+0000 mgr.y (mgr.14146) 118 : cluster [DBG] pgmap v94: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:59:19.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:19 smithi016 bash[33624]: audit 2023-09-23T18:59:19.237611+0000 mon.a (mon.0) 299 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T18:59:19.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:19 smithi016 bash[33624]: audit 2023-09-23T18:59:19.239189+0000 mon.a (mon.0) 300 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:59:20.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:19 smithi092 bash[27556]: cluster 2023-09-23T18:59:18.622024+0000 mgr.y (mgr.14146) 118 : cluster [DBG] pgmap v94: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:59:20.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:19 smithi092 bash[27556]: audit 2023-09-23T18:59:19.237611+0000 mon.a (mon.0) 299 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T18:59:20.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:19 smithi092 bash[27556]: audit 2023-09-23T18:59:19.239189+0000 mon.a (mon.0) 300 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:59:21.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:20 smithi016 bash[27237]: cephadm 2023-09-23T18:59:19.240483+0000 mgr.y (mgr.14146) 119 : cephadm [INF] Deploying daemon osd.2 on smithi016 2023-09-23T18:59:21.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:20 smithi016 bash[33624]: cephadm 2023-09-23T18:59:19.240483+0000 mgr.y (mgr.14146) 119 : cephadm [INF] Deploying daemon osd.2 on smithi016 2023-09-23T18:59:21.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:20 smithi092 bash[27556]: cephadm 2023-09-23T18:59:19.240483+0000 mgr.y (mgr.14146) 119 : cephadm [INF] Deploying daemon osd.2 on smithi016 2023-09-23T18:59:22.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:21 smithi016 bash[33624]: cluster 2023-09-23T18:59:20.622503+0000 mgr.y (mgr.14146) 120 : cluster [DBG] pgmap v95: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:59:22.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:21 smithi016 bash[27237]: cluster 2023-09-23T18:59:20.622503+0000 mgr.y (mgr.14146) 120 : cluster [DBG] pgmap v95: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:59:22.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:21 smithi092 bash[27556]: cluster 2023-09-23T18:59:20.622503+0000 mgr.y (mgr.14146) 120 : cluster [DBG] pgmap v95: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:59:23.783 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:23 smithi016 bash[27237]: cluster 2023-09-23T18:59:22.623026+0000 mgr.y (mgr.14146) 121 : cluster [DBG] pgmap v96: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:59:23.783 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:23 smithi016 bash[27237]: audit 2023-09-23T18:59:23.431324+0000 mon.a (mon.0) 301 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:59:23.783 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:23 smithi016 bash[27237]: audit 2023-09-23T18:59:23.440641+0000 mon.a (mon.0) 302 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:59:23.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:23 smithi016 bash[33624]: cluster 2023-09-23T18:59:22.623026+0000 mgr.y (mgr.14146) 121 : cluster [DBG] pgmap v96: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:59:23.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:23 smithi016 bash[33624]: audit 2023-09-23T18:59:23.431324+0000 mon.a (mon.0) 301 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:59:23.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:23 smithi016 bash[33624]: audit 2023-09-23T18:59:23.440641+0000 mon.a (mon.0) 302 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:59:24.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:23 smithi092 bash[27556]: cluster 2023-09-23T18:59:22.623026+0000 mgr.y (mgr.14146) 121 : cluster [DBG] pgmap v96: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:59:24.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:23 smithi092 bash[27556]: audit 2023-09-23T18:59:23.431324+0000 mon.a (mon.0) 301 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:59:24.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:23 smithi092 bash[27556]: audit 2023-09-23T18:59:23.440641+0000 mon.a (mon.0) 302 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:59:26.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:25 smithi016 bash[27237]: cluster 2023-09-23T18:59:24.623429+0000 mgr.y (mgr.14146) 122 : cluster [DBG] pgmap v97: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:59:26.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:25 smithi016 bash[33624]: cluster 2023-09-23T18:59:24.623429+0000 mgr.y (mgr.14146) 122 : cluster [DBG] pgmap v97: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:59:26.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:25 smithi092 bash[27556]: cluster 2023-09-23T18:59:24.623429+0000 mgr.y (mgr.14146) 122 : cluster [DBG] pgmap v97: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:59:28.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:27 smithi016 bash[27237]: cluster 2023-09-23T18:59:26.623896+0000 mgr.y (mgr.14146) 123 : cluster [DBG] pgmap v98: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:59:28.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:27 smithi016 bash[33624]: cluster 2023-09-23T18:59:26.623896+0000 mgr.y (mgr.14146) 123 : cluster [DBG] pgmap v98: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:59:28.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:27 smithi092 bash[27556]: cluster 2023-09-23T18:59:26.623896+0000 mgr.y (mgr.14146) 123 : cluster [DBG] pgmap v98: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:59:29.783 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:29 smithi016 bash[27237]: cluster 2023-09-23T18:59:28.624349+0000 mgr.y (mgr.14146) 124 : cluster [DBG] pgmap v99: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:59:29.783 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:29 smithi016 bash[27237]: audit 2023-09-23T18:59:28.665948+0000 mon.a (mon.0) 303 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:59:29.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:29 smithi016 bash[27237]: audit 2023-09-23T18:59:28.962821+0000 mon.a (mon.0) 304 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:59:29.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:29 smithi016 bash[27237]: audit 2023-09-23T18:59:28.964060+0000 mon.a (mon.0) 305 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:59:29.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:29 smithi016 bash[27237]: audit 2023-09-23T18:59:28.970960+0000 mon.a (mon.0) 306 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:59:29.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:29 smithi016 bash[33624]: cluster 2023-09-23T18:59:28.624349+0000 mgr.y (mgr.14146) 124 : cluster [DBG] pgmap v99: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:59:29.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:29 smithi016 bash[33624]: audit 2023-09-23T18:59:28.665948+0000 mon.a (mon.0) 303 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:59:29.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:29 smithi016 bash[33624]: audit 2023-09-23T18:59:28.962821+0000 mon.a (mon.0) 304 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:59:29.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:29 smithi016 bash[33624]: audit 2023-09-23T18:59:28.964060+0000 mon.a (mon.0) 305 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:59:29.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:29 smithi016 bash[33624]: audit 2023-09-23T18:59:28.970960+0000 mon.a (mon.0) 306 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:59:30.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:29 smithi092 bash[27556]: cluster 2023-09-23T18:59:28.624349+0000 mgr.y (mgr.14146) 124 : cluster [DBG] pgmap v99: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:59:30.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:29 smithi092 bash[27556]: audit 2023-09-23T18:59:28.665948+0000 mon.a (mon.0) 303 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:59:30.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:29 smithi092 bash[27556]: audit 2023-09-23T18:59:28.962821+0000 mon.a (mon.0) 304 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:59:30.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:29 smithi092 bash[27556]: audit 2023-09-23T18:59:28.964060+0000 mon.a (mon.0) 305 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:59:30.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:29 smithi092 bash[27556]: audit 2023-09-23T18:59:28.970960+0000 mon.a (mon.0) 306 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:59:31.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:30 smithi016 bash[27237]: audit 2023-09-23T18:59:29.955434+0000 mon.a (mon.0) 307 : audit [INF] from='osd.2 [v2:172.21.15.16:6818/568840676,v1:172.21.15.16:6819/568840676]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T18:59:31.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:30 smithi016 bash[27237]: cluster 2023-09-23T18:59:30.624847+0000 mgr.y (mgr.14146) 125 : cluster [DBG] pgmap v100: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:59:31.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:30 smithi016 bash[33624]: audit 2023-09-23T18:59:29.955434+0000 mon.a (mon.0) 307 : audit [INF] from='osd.2 [v2:172.21.15.16:6818/568840676,v1:172.21.15.16:6819/568840676]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T18:59:31.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:30 smithi016 bash[33624]: cluster 2023-09-23T18:59:30.624847+0000 mgr.y (mgr.14146) 125 : cluster [DBG] pgmap v100: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:59:31.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:30 smithi092 bash[27556]: audit 2023-09-23T18:59:29.955434+0000 mon.a (mon.0) 307 : audit [INF] from='osd.2 [v2:172.21.15.16:6818/568840676,v1:172.21.15.16:6819/568840676]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T18:59:31.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:30 smithi092 bash[27556]: cluster 2023-09-23T18:59:30.624847+0000 mgr.y (mgr.14146) 125 : cluster [DBG] pgmap v100: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:59:31.981 INFO:teuthology.orchestra.run.smithi016.stdout:Created osd(s) 2 on host 'smithi016' 2023-09-23T18:59:32.231 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:31 smithi016 bash[33624]: audit 2023-09-23T18:59:30.823541+0000 mon.a (mon.0) 308 : audit [INF] from='osd.2 [v2:172.21.15.16:6818/568840676,v1:172.21.15.16:6819/568840676]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T18:59:32.231 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:31 smithi016 bash[33624]: cluster 2023-09-23T18:59:30.823666+0000 mon.a (mon.0) 309 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2023-09-23T18:59:32.231 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:31 smithi016 bash[33624]: audit 2023-09-23T18:59:30.823876+0000 mon.a (mon.0) 310 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T18:59:32.231 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:31 smithi016 bash[33624]: audit 2023-09-23T18:59:30.824460+0000 mon.a (mon.0) 311 : audit [INF] from='osd.2 [v2:172.21.15.16:6818/568840676,v1:172.21.15.16:6819/568840676]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-09-23T18:59:32.231 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:31 smithi016 bash[27237]: audit 2023-09-23T18:59:30.823541+0000 mon.a (mon.0) 308 : audit [INF] from='osd.2 [v2:172.21.15.16:6818/568840676,v1:172.21.15.16:6819/568840676]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T18:59:32.231 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:31 smithi016 bash[27237]: cluster 2023-09-23T18:59:30.823666+0000 mon.a (mon.0) 309 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2023-09-23T18:59:32.232 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:31 smithi016 bash[27237]: audit 2023-09-23T18:59:30.823876+0000 mon.a (mon.0) 310 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T18:59:32.232 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:31 smithi016 bash[27237]: audit 2023-09-23T18:59:30.824460+0000 mon.a (mon.0) 311 : audit [INF] from='osd.2 [v2:172.21.15.16:6818/568840676,v1:172.21.15.16:6819/568840676]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-09-23T18:59:32.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:31 smithi092 bash[27556]: audit 2023-09-23T18:59:30.823541+0000 mon.a (mon.0) 308 : audit [INF] from='osd.2 [v2:172.21.15.16:6818/568840676,v1:172.21.15.16:6819/568840676]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T18:59:32.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:31 smithi092 bash[27556]: cluster 2023-09-23T18:59:30.823666+0000 mon.a (mon.0) 309 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2023-09-23T18:59:32.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:31 smithi092 bash[27556]: audit 2023-09-23T18:59:30.823876+0000 mon.a (mon.0) 310 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T18:59:32.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:31 smithi092 bash[27556]: audit 2023-09-23T18:59:30.824460+0000 mon.a (mon.0) 311 : audit [INF] from='osd.2 [v2:172.21.15.16:6818/568840676,v1:172.21.15.16:6819/568840676]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-09-23T18:59:32.629 DEBUG:teuthology.orchestra.run.smithi016:osd.2> sudo journalctl -f -n 0 -u ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@osd.2.service 2023-09-23T18:59:32.632 INFO:tasks.cephadm:Deploying osd.3 on smithi016 with /dev/vg_nvme/lv_1... 2023-09-23T18:59:32.632 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-09-23T18:59:32.862 INFO:journalctl@ceph.osd.2.smithi016.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-09-23T18:59:32.879 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:32 smithi016 bash[27237]: audit 2023-09-23T18:59:31.827605+0000 mon.a (mon.0) 312 : audit [INF] from='osd.2 [v2:172.21.15.16:6818/568840676,v1:172.21.15.16:6819/568840676]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]': finished 2023-09-23T18:59:32.879 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:32 smithi016 bash[27237]: cluster 2023-09-23T18:59:31.827737+0000 mon.a (mon.0) 313 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2023-09-23T18:59:32.879 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:32 smithi016 bash[27237]: audit 2023-09-23T18:59:31.828442+0000 mon.a (mon.0) 314 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T18:59:32.879 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:32 smithi016 bash[27237]: audit 2023-09-23T18:59:31.833948+0000 mon.a (mon.0) 315 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T18:59:32.880 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:32 smithi016 bash[27237]: audit 2023-09-23T18:59:31.978966+0000 mon.a (mon.0) 316 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:59:32.880 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:32 smithi016 bash[27237]: audit 2023-09-23T18:59:31.994625+0000 mon.a (mon.0) 317 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:59:32.880 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:32 smithi016 bash[27237]: cluster 2023-09-23T18:59:32.625331+0000 mgr.y (mgr.14146) 126 : cluster [DBG] pgmap v103: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:59:32.880 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:32 smithi016 bash[33624]: audit 2023-09-23T18:59:31.827605+0000 mon.a (mon.0) 312 : audit [INF] from='osd.2 [v2:172.21.15.16:6818/568840676,v1:172.21.15.16:6819/568840676]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]': finished 2023-09-23T18:59:32.880 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:32 smithi016 bash[33624]: cluster 2023-09-23T18:59:31.827737+0000 mon.a (mon.0) 313 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2023-09-23T18:59:32.880 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:32 smithi016 bash[33624]: audit 2023-09-23T18:59:31.828442+0000 mon.a (mon.0) 314 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T18:59:32.880 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:32 smithi016 bash[33624]: audit 2023-09-23T18:59:31.833948+0000 mon.a (mon.0) 315 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T18:59:32.880 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:32 smithi016 bash[33624]: audit 2023-09-23T18:59:31.978966+0000 mon.a (mon.0) 316 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:59:32.880 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:32 smithi016 bash[33624]: audit 2023-09-23T18:59:31.994625+0000 mon.a (mon.0) 317 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:59:32.881 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:32 smithi016 bash[33624]: cluster 2023-09-23T18:59:32.625331+0000 mgr.y (mgr.14146) 126 : cluster [DBG] pgmap v103: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:59:33.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:32 smithi092 bash[27556]: audit 2023-09-23T18:59:31.827605+0000 mon.a (mon.0) 312 : audit [INF] from='osd.2 [v2:172.21.15.16:6818/568840676,v1:172.21.15.16:6819/568840676]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]': finished 2023-09-23T18:59:33.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:32 smithi092 bash[27556]: cluster 2023-09-23T18:59:31.827737+0000 mon.a (mon.0) 313 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2023-09-23T18:59:33.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:32 smithi092 bash[27556]: audit 2023-09-23T18:59:31.828442+0000 mon.a (mon.0) 314 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T18:59:33.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:32 smithi092 bash[27556]: audit 2023-09-23T18:59:31.833948+0000 mon.a (mon.0) 315 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T18:59:33.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:32 smithi092 bash[27556]: audit 2023-09-23T18:59:31.978966+0000 mon.a (mon.0) 316 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:59:33.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:32 smithi092 bash[27556]: audit 2023-09-23T18:59:31.994625+0000 mon.a (mon.0) 317 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:59:33.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:32 smithi092 bash[27556]: cluster 2023-09-23T18:59:32.625331+0000 mgr.y (mgr.14146) 126 : cluster [DBG] pgmap v103: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:59:34.030 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:33 smithi016 bash[27237]: cluster 2023-09-23T18:59:30.939891+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T18:59:34.030 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:33 smithi016 bash[27237]: cluster 2023-09-23T18:59:30.939981+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T18:59:34.030 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:33 smithi016 bash[27237]: audit 2023-09-23T18:59:32.837682+0000 mon.a (mon.0) 318 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T18:59:34.030 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:33 smithi016 bash[27237]: cluster 2023-09-23T18:59:32.841374+0000 mon.a (mon.0) 319 : cluster [INF] osd.2 [v2:172.21.15.16:6818/568840676,v1:172.21.15.16:6819/568840676] boot 2023-09-23T18:59:34.030 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:33 smithi016 bash[27237]: cluster 2023-09-23T18:59:32.841423+0000 mon.a (mon.0) 320 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2023-09-23T18:59:34.030 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:33 smithi016 bash[27237]: audit 2023-09-23T18:59:32.841826+0000 mon.a (mon.0) 321 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T18:59:34.030 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:33 smithi016 bash[27237]: audit 2023-09-23T18:59:32.842343+0000 mon.a (mon.0) 322 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-09-23T18:59:34.031 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:33 smithi016 bash[33624]: cluster 2023-09-23T18:59:30.939891+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T18:59:34.031 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:33 smithi016 bash[33624]: cluster 2023-09-23T18:59:30.939981+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T18:59:34.031 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:33 smithi016 bash[33624]: audit 2023-09-23T18:59:32.837682+0000 mon.a (mon.0) 318 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T18:59:34.031 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:33 smithi016 bash[33624]: cluster 2023-09-23T18:59:32.841374+0000 mon.a (mon.0) 319 : cluster [INF] osd.2 [v2:172.21.15.16:6818/568840676,v1:172.21.15.16:6819/568840676] boot 2023-09-23T18:59:34.031 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:33 smithi016 bash[33624]: cluster 2023-09-23T18:59:32.841423+0000 mon.a (mon.0) 320 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2023-09-23T18:59:34.031 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:33 smithi016 bash[33624]: audit 2023-09-23T18:59:32.841826+0000 mon.a (mon.0) 321 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T18:59:34.031 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:33 smithi016 bash[33624]: audit 2023-09-23T18:59:32.842343+0000 mon.a (mon.0) 322 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-09-23T18:59:34.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:33 smithi092 bash[27556]: cluster 2023-09-23T18:59:30.939891+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T18:59:34.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:33 smithi092 bash[27556]: cluster 2023-09-23T18:59:30.939981+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T18:59:34.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:33 smithi092 bash[27556]: audit 2023-09-23T18:59:32.837682+0000 mon.a (mon.0) 318 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T18:59:34.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:33 smithi092 bash[27556]: cluster 2023-09-23T18:59:32.841374+0000 mon.a (mon.0) 319 : cluster [INF] osd.2 [v2:172.21.15.16:6818/568840676,v1:172.21.15.16:6819/568840676] boot 2023-09-23T18:59:34.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:33 smithi092 bash[27556]: cluster 2023-09-23T18:59:32.841423+0000 mon.a (mon.0) 320 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2023-09-23T18:59:34.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:33 smithi092 bash[27556]: audit 2023-09-23T18:59:32.841826+0000 mon.a (mon.0) 321 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T18:59:34.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:33 smithi092 bash[27556]: audit 2023-09-23T18:59:32.842343+0000 mon.a (mon.0) 322 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-09-23T18:59:35.144 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:34 smithi016 bash[27237]: audit 2023-09-23T18:59:33.845482+0000 mon.a (mon.0) 323 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-09-23T18:59:35.144 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:34 smithi016 bash[27237]: cluster 2023-09-23T18:59:33.845556+0000 mon.a (mon.0) 324 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2023-09-23T18:59:35.144 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:34 smithi016 bash[27237]: audit 2023-09-23T18:59:33.846654+0000 mon.a (mon.0) 325 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-09-23T18:59:35.144 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:34 smithi016 bash[27237]: cluster 2023-09-23T18:59:34.625915+0000 mgr.y (mgr.14146) 127 : cluster [DBG] pgmap v106: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:59:35.144 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:34 smithi016 bash[33624]: audit 2023-09-23T18:59:33.845482+0000 mon.a (mon.0) 323 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-09-23T18:59:35.144 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:34 smithi016 bash[33624]: cluster 2023-09-23T18:59:33.845556+0000 mon.a (mon.0) 324 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2023-09-23T18:59:35.145 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:34 smithi016 bash[33624]: audit 2023-09-23T18:59:33.846654+0000 mon.a (mon.0) 325 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-09-23T18:59:35.145 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:34 smithi016 bash[33624]: cluster 2023-09-23T18:59:34.625915+0000 mgr.y (mgr.14146) 127 : cluster [DBG] pgmap v106: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:59:35.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:34 smithi092 bash[27556]: audit 2023-09-23T18:59:33.845482+0000 mon.a (mon.0) 323 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-09-23T18:59:35.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:34 smithi092 bash[27556]: cluster 2023-09-23T18:59:33.845556+0000 mon.a (mon.0) 324 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2023-09-23T18:59:35.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:34 smithi092 bash[27556]: audit 2023-09-23T18:59:33.846654+0000 mon.a (mon.0) 325 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-09-23T18:59:35.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:34 smithi092 bash[27556]: cluster 2023-09-23T18:59:34.625915+0000 mgr.y (mgr.14146) 127 : cluster [DBG] pgmap v106: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:59:35.889 INFO:teuthology.orchestra.run.smithi016.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-09-23T18:59:35.889 INFO:teuthology.orchestra.run.smithi016.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-09-23T18:59:35.889 INFO:teuthology.orchestra.run.smithi016.stderr: stderr: 10+0 records in 2023-09-23T18:59:35.889 INFO:teuthology.orchestra.run.smithi016.stderr:10+0 records out 2023-09-23T18:59:35.889 INFO:teuthology.orchestra.run.smithi016.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0135588 s, 773 MB/s 2023-09-23T18:59:35.890 INFO:teuthology.orchestra.run.smithi016.stderr:--> Zapping successful for: 2023-09-23T18:59:36.139 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:35 smithi016 bash[27237]: audit 2023-09-23T18:59:34.854988+0000 mon.a (mon.0) 326 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-09-23T18:59:36.139 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:35 smithi016 bash[27237]: cluster 2023-09-23T18:59:34.855106+0000 mon.a (mon.0) 327 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2023-09-23T18:59:36.139 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:35 smithi016 bash[33624]: audit 2023-09-23T18:59:34.854988+0000 mon.a (mon.0) 326 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-09-23T18:59:36.139 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:35 smithi016 bash[33624]: cluster 2023-09-23T18:59:34.855106+0000 mon.a (mon.0) 327 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2023-09-23T18:59:36.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:35 smithi092 bash[27556]: audit 2023-09-23T18:59:34.854988+0000 mon.a (mon.0) 326 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-09-23T18:59:36.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:35 smithi092 bash[27556]: cluster 2023-09-23T18:59:34.855106+0000 mon.a (mon.0) 327 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2023-09-23T18:59:36.642 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi016:vg_nvme/lv_1 2023-09-23T18:59:37.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:36 smithi016 bash[27237]: cluster 2023-09-23T18:59:35.861313+0000 mon.a (mon.0) 328 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2023-09-23T18:59:37.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:36 smithi016 bash[27237]: cluster 2023-09-23T18:59:36.626396+0000 mgr.y (mgr.14146) 128 : cluster [DBG] pgmap v109: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:59:37.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:36 smithi016 bash[33624]: cluster 2023-09-23T18:59:35.861313+0000 mon.a (mon.0) 328 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2023-09-23T18:59:37.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:36 smithi016 bash[33624]: cluster 2023-09-23T18:59:36.626396+0000 mgr.y (mgr.14146) 128 : cluster [DBG] pgmap v109: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:59:37.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:36 smithi092 bash[27556]: cluster 2023-09-23T18:59:35.861313+0000 mon.a (mon.0) 328 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2023-09-23T18:59:37.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:36 smithi092 bash[27556]: cluster 2023-09-23T18:59:36.626396+0000 mgr.y (mgr.14146) 128 : cluster [DBG] pgmap v109: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:59:40.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:39 smithi016 bash[27237]: cluster 2023-09-23T18:59:38.626934+0000 mgr.y (mgr.14146) 129 : cluster [DBG] pgmap v110: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:59:40.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:39 smithi016 bash[33624]: cluster 2023-09-23T18:59:38.626934+0000 mgr.y (mgr.14146) 129 : cluster [DBG] pgmap v110: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:59:40.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:39 smithi092 bash[27556]: cluster 2023-09-23T18:59:38.626934+0000 mgr.y (mgr.14146) 129 : cluster [DBG] pgmap v110: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:59:41.179 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:40 smithi016 bash[27237]: cluster 2023-09-23T18:59:40.627433+0000 mgr.y (mgr.14146) 130 : cluster [DBG] pgmap v111: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:59:41.179 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:40 smithi016 bash[33624]: cluster 2023-09-23T18:59:40.627433+0000 mgr.y (mgr.14146) 130 : cluster [DBG] pgmap v111: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:59:41.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:40 smithi092 bash[27556]: cluster 2023-09-23T18:59:40.627433+0000 mgr.y (mgr.14146) 130 : cluster [DBG] pgmap v111: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:59:42.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:41 smithi016 bash[27237]: audit 2023-09-23T18:59:40.812559+0000 mgr.y (mgr.14146) 131 : audit [DBG] from='client.24203 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi016:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:59:42.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:41 smithi016 bash[27237]: audit 2023-09-23T18:59:40.819541+0000 mon.a (mon.0) 329 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T18:59:42.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:41 smithi016 bash[27237]: audit 2023-09-23T18:59:40.824756+0000 mon.a (mon.0) 330 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T18:59:42.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:41 smithi016 bash[27237]: audit 2023-09-23T18:59:40.825638+0000 mon.a (mon.0) 331 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:59:42.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:41 smithi016 bash[33624]: audit 2023-09-23T18:59:40.812559+0000 mgr.y (mgr.14146) 131 : audit [DBG] from='client.24203 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi016:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:59:42.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:41 smithi016 bash[33624]: audit 2023-09-23T18:59:40.819541+0000 mon.a (mon.0) 329 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T18:59:42.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:41 smithi016 bash[33624]: audit 2023-09-23T18:59:40.824756+0000 mon.a (mon.0) 330 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T18:59:42.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:41 smithi016 bash[33624]: audit 2023-09-23T18:59:40.825638+0000 mon.a (mon.0) 331 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:59:42.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:41 smithi092 bash[27556]: audit 2023-09-23T18:59:40.812559+0000 mgr.y (mgr.14146) 131 : audit [DBG] from='client.24203 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi016:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:59:42.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:41 smithi092 bash[27556]: audit 2023-09-23T18:59:40.819541+0000 mon.a (mon.0) 329 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T18:59:42.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:41 smithi092 bash[27556]: audit 2023-09-23T18:59:40.824756+0000 mon.a (mon.0) 330 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T18:59:42.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:41 smithi092 bash[27556]: audit 2023-09-23T18:59:40.825638+0000 mon.a (mon.0) 331 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:59:43.533 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:43 smithi016 bash[27237]: cephadm 2023-09-23T18:59:42.043254+0000 mgr.y (mgr.14146) 132 : cephadm [INF] Detected new or changed devices on smithi016 2023-09-23T18:59:43.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:43 smithi016 bash[27237]: audit 2023-09-23T18:59:42.051963+0000 mon.a (mon.0) 332 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:59:43.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:43 smithi016 bash[27237]: audit 2023-09-23T18:59:42.053563+0000 mon.a (mon.0) 333 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:59:43.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:43 smithi016 bash[27237]: audit 2023-09-23T18:59:42.054806+0000 mon.a (mon.0) 334 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:59:43.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:43 smithi016 bash[27237]: audit 2023-09-23T18:59:42.055976+0000 mon.a (mon.0) 335 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:59:43.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:43 smithi016 bash[27237]: cephadm 2023-09-23T18:59:42.056790+0000 mgr.y (mgr.14146) 133 : cephadm [INF] Adjusting osd_memory_target on smithi016 to 5415M 2023-09-23T18:59:43.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:43 smithi016 bash[27237]: audit 2023-09-23T18:59:42.061681+0000 mon.a (mon.0) 336 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:59:43.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:43 smithi016 bash[27237]: audit 2023-09-23T18:59:42.063533+0000 mon.a (mon.0) 337 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:59:43.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:43 smithi016 bash[27237]: audit 2023-09-23T18:59:42.064744+0000 mon.a (mon.0) 338 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:59:43.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:43 smithi016 bash[27237]: audit 2023-09-23T18:59:42.071264+0000 mon.a (mon.0) 339 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:59:43.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:43 smithi016 bash[27237]: cluster 2023-09-23T18:59:42.627957+0000 mgr.y (mgr.14146) 134 : cluster [DBG] pgmap v112: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:59:43.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:43 smithi016 bash[33624]: cephadm 2023-09-23T18:59:42.043254+0000 mgr.y (mgr.14146) 132 : cephadm [INF] Detected new or changed devices on smithi016 2023-09-23T18:59:43.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:43 smithi016 bash[33624]: audit 2023-09-23T18:59:42.051963+0000 mon.a (mon.0) 332 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:59:43.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:43 smithi016 bash[33624]: audit 2023-09-23T18:59:42.053563+0000 mon.a (mon.0) 333 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:59:43.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:43 smithi016 bash[33624]: audit 2023-09-23T18:59:42.054806+0000 mon.a (mon.0) 334 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:59:43.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:43 smithi016 bash[33624]: audit 2023-09-23T18:59:42.055976+0000 mon.a (mon.0) 335 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:59:43.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:43 smithi016 bash[33624]: cephadm 2023-09-23T18:59:42.056790+0000 mgr.y (mgr.14146) 133 : cephadm [INF] Adjusting osd_memory_target on smithi016 to 5415M 2023-09-23T18:59:43.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:43 smithi016 bash[33624]: audit 2023-09-23T18:59:42.061681+0000 mon.a (mon.0) 336 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:59:43.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:43 smithi016 bash[33624]: audit 2023-09-23T18:59:42.063533+0000 mon.a (mon.0) 337 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:59:43.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:43 smithi016 bash[33624]: audit 2023-09-23T18:59:42.064744+0000 mon.a (mon.0) 338 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:59:43.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:43 smithi016 bash[33624]: audit 2023-09-23T18:59:42.071264+0000 mon.a (mon.0) 339 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:59:43.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:43 smithi016 bash[33624]: cluster 2023-09-23T18:59:42.627957+0000 mgr.y (mgr.14146) 134 : cluster [DBG] pgmap v112: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:59:43.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:43 smithi092 bash[27556]: cephadm 2023-09-23T18:59:42.043254+0000 mgr.y (mgr.14146) 132 : cephadm [INF] Detected new or changed devices on smithi016 2023-09-23T18:59:43.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:43 smithi092 bash[27556]: audit 2023-09-23T18:59:42.051963+0000 mon.a (mon.0) 332 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:59:43.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:43 smithi092 bash[27556]: audit 2023-09-23T18:59:42.053563+0000 mon.a (mon.0) 333 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:59:43.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:43 smithi092 bash[27556]: audit 2023-09-23T18:59:42.054806+0000 mon.a (mon.0) 334 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:59:43.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:43 smithi092 bash[27556]: audit 2023-09-23T18:59:42.055976+0000 mon.a (mon.0) 335 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:59:43.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:43 smithi092 bash[27556]: cephadm 2023-09-23T18:59:42.056790+0000 mgr.y (mgr.14146) 133 : cephadm [INF] Adjusting osd_memory_target on smithi016 to 5415M 2023-09-23T18:59:43.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:43 smithi092 bash[27556]: audit 2023-09-23T18:59:42.061681+0000 mon.a (mon.0) 336 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:59:43.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:43 smithi092 bash[27556]: audit 2023-09-23T18:59:42.063533+0000 mon.a (mon.0) 337 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:59:43.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:43 smithi092 bash[27556]: audit 2023-09-23T18:59:42.064744+0000 mon.a (mon.0) 338 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:59:43.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:43 smithi092 bash[27556]: audit 2023-09-23T18:59:42.071264+0000 mon.a (mon.0) 339 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T18:59:43.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:43 smithi092 bash[27556]: cluster 2023-09-23T18:59:42.627957+0000 mgr.y (mgr.14146) 134 : cluster [DBG] pgmap v112: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:59:46.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:45 smithi016 bash[27237]: cluster 2023-09-23T18:59:44.628505+0000 mgr.y (mgr.14146) 135 : cluster [DBG] pgmap v113: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:59:46.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:45 smithi016 bash[33624]: cluster 2023-09-23T18:59:44.628505+0000 mgr.y (mgr.14146) 135 : cluster [DBG] pgmap v113: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:59:46.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:45 smithi092 bash[27556]: cluster 2023-09-23T18:59:44.628505+0000 mgr.y (mgr.14146) 135 : cluster [DBG] pgmap v113: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:59:47.690 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:47 smithi016 bash[27237]: cluster 2023-09-23T18:59:46.629102+0000 mgr.y (mgr.14146) 136 : cluster [DBG] pgmap v114: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:59:47.999 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:47 smithi016 bash[33624]: cluster 2023-09-23T18:59:46.629102+0000 mgr.y (mgr.14146) 136 : cluster [DBG] pgmap v114: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:59:48.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:47 smithi092 bash[27556]: cluster 2023-09-23T18:59:46.629102+0000 mgr.y (mgr.14146) 136 : cluster [DBG] pgmap v114: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:59:50.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:49 smithi016 bash[27237]: cluster 2023-09-23T18:59:48.629689+0000 mgr.y (mgr.14146) 137 : cluster [DBG] pgmap v115: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:59:50.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:49 smithi016 bash[27237]: audit 2023-09-23T18:59:48.951247+0000 mon.c (mon.2) 8 : audit [INF] from='client.? 172.21.15.16:0/2356823733' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f5e0d57b-d629-4453-a527-ad77e6c210ad"}]: dispatch 2023-09-23T18:59:50.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:49 smithi016 bash[27237]: audit 2023-09-23T18:59:48.951995+0000 mon.a (mon.0) 340 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f5e0d57b-d629-4453-a527-ad77e6c210ad"}]: dispatch 2023-09-23T18:59:50.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:49 smithi016 bash[27237]: audit 2023-09-23T18:59:48.960491+0000 mon.a (mon.0) 341 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f5e0d57b-d629-4453-a527-ad77e6c210ad"}]': finished 2023-09-23T18:59:50.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:49 smithi016 bash[27237]: cluster 2023-09-23T18:59:48.960568+0000 mon.a (mon.0) 342 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2023-09-23T18:59:50.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:49 smithi016 bash[27237]: audit 2023-09-23T18:59:48.960744+0000 mon.a (mon.0) 343 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T18:59:50.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:49 smithi016 bash[27237]: audit 2023-09-23T18:59:49.679157+0000 mon.a (mon.0) 344 : audit [DBG] from='client.? 172.21.15.16:0/2974441877' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T18:59:50.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:49 smithi016 bash[33624]: cluster 2023-09-23T18:59:48.629689+0000 mgr.y (mgr.14146) 137 : cluster [DBG] pgmap v115: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:59:50.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:49 smithi016 bash[33624]: audit 2023-09-23T18:59:48.951247+0000 mon.c (mon.2) 8 : audit [INF] from='client.? 172.21.15.16:0/2356823733' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f5e0d57b-d629-4453-a527-ad77e6c210ad"}]: dispatch 2023-09-23T18:59:50.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:49 smithi016 bash[33624]: audit 2023-09-23T18:59:48.951995+0000 mon.a (mon.0) 340 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f5e0d57b-d629-4453-a527-ad77e6c210ad"}]: dispatch 2023-09-23T18:59:50.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:49 smithi016 bash[33624]: audit 2023-09-23T18:59:48.960491+0000 mon.a (mon.0) 341 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f5e0d57b-d629-4453-a527-ad77e6c210ad"}]': finished 2023-09-23T18:59:50.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:49 smithi016 bash[33624]: cluster 2023-09-23T18:59:48.960568+0000 mon.a (mon.0) 342 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2023-09-23T18:59:50.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:49 smithi016 bash[33624]: audit 2023-09-23T18:59:48.960744+0000 mon.a (mon.0) 343 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T18:59:50.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:49 smithi016 bash[33624]: audit 2023-09-23T18:59:49.679157+0000 mon.a (mon.0) 344 : audit [DBG] from='client.? 172.21.15.16:0/2974441877' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T18:59:50.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:49 smithi092 bash[27556]: cluster 2023-09-23T18:59:48.629689+0000 mgr.y (mgr.14146) 137 : cluster [DBG] pgmap v115: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:59:50.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:49 smithi092 bash[27556]: audit 2023-09-23T18:59:48.951247+0000 mon.c (mon.2) 8 : audit [INF] from='client.? 172.21.15.16:0/2356823733' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f5e0d57b-d629-4453-a527-ad77e6c210ad"}]: dispatch 2023-09-23T18:59:50.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:49 smithi092 bash[27556]: audit 2023-09-23T18:59:48.951995+0000 mon.a (mon.0) 340 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f5e0d57b-d629-4453-a527-ad77e6c210ad"}]: dispatch 2023-09-23T18:59:50.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:49 smithi092 bash[27556]: audit 2023-09-23T18:59:48.960491+0000 mon.a (mon.0) 341 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f5e0d57b-d629-4453-a527-ad77e6c210ad"}]': finished 2023-09-23T18:59:50.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:49 smithi092 bash[27556]: cluster 2023-09-23T18:59:48.960568+0000 mon.a (mon.0) 342 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2023-09-23T18:59:50.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:49 smithi092 bash[27556]: audit 2023-09-23T18:59:48.960744+0000 mon.a (mon.0) 343 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T18:59:50.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:49 smithi092 bash[27556]: audit 2023-09-23T18:59:49.679157+0000 mon.a (mon.0) 344 : audit [DBG] from='client.? 172.21.15.16:0/2974441877' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T18:59:51.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:50 smithi016 bash[27237]: cluster 2023-09-23T18:59:50.630209+0000 mgr.y (mgr.14146) 138 : cluster [DBG] pgmap v117: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:59:51.283 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:50 smithi016 bash[33624]: cluster 2023-09-23T18:59:50.630209+0000 mgr.y (mgr.14146) 138 : cluster [DBG] pgmap v117: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:59:51.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:50 smithi092 bash[27556]: cluster 2023-09-23T18:59:50.630209+0000 mgr.y (mgr.14146) 138 : cluster [DBG] pgmap v117: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:59:54.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:53 smithi016 bash[27237]: cluster 2023-09-23T18:59:52.630778+0000 mgr.y (mgr.14146) 139 : cluster [DBG] pgmap v118: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:59:54.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:53 smithi016 bash[33624]: cluster 2023-09-23T18:59:52.630778+0000 mgr.y (mgr.14146) 139 : cluster [DBG] pgmap v118: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:59:54.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:53 smithi092 bash[27556]: cluster 2023-09-23T18:59:52.630778+0000 mgr.y (mgr.14146) 139 : cluster [DBG] pgmap v118: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:59:56.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:55 smithi016 bash[27237]: cluster 2023-09-23T18:59:54.631293+0000 mgr.y (mgr.14146) 140 : cluster [DBG] pgmap v119: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:59:56.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:55 smithi016 bash[33624]: cluster 2023-09-23T18:59:54.631293+0000 mgr.y (mgr.14146) 140 : cluster [DBG] pgmap v119: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:59:56.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:55 smithi092 bash[27556]: cluster 2023-09-23T18:59:54.631293+0000 mgr.y (mgr.14146) 140 : cluster [DBG] pgmap v119: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:59:58.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:57 smithi016 bash[33624]: cluster 2023-09-23T18:59:56.631940+0000 mgr.y (mgr.14146) 141 : cluster [DBG] pgmap v120: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:59:58.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:57 smithi016 bash[27237]: cluster 2023-09-23T18:59:56.631940+0000 mgr.y (mgr.14146) 141 : cluster [DBG] pgmap v120: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:59:58.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:57 smithi092 bash[27556]: cluster 2023-09-23T18:59:56.631940+0000 mgr.y (mgr.14146) 141 : cluster [DBG] pgmap v120: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T19:00:00.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 18:59:59 smithi016 bash[27237]: cluster 2023-09-23T18:59:58.632489+0000 mgr.y (mgr.14146) 142 : cluster [DBG] pgmap v121: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T19:00:00.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 18:59:59 smithi016 bash[33624]: cluster 2023-09-23T18:59:58.632489+0000 mgr.y (mgr.14146) 142 : cluster [DBG] pgmap v121: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T19:00:00.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 18:59:59 smithi092 bash[27556]: cluster 2023-09-23T18:59:58.632489+0000 mgr.y (mgr.14146) 142 : cluster [DBG] pgmap v121: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T19:00:01.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:00 smithi016 bash[27237]: cluster 2023-09-23T19:00:00.000192+0000 mon.a (mon.0) 345 : cluster [INF] overall HEALTH_OK 2023-09-23T19:00:01.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:00 smithi016 bash[27237]: cluster 2023-09-23T19:00:00.632969+0000 mgr.y (mgr.14146) 143 : cluster [DBG] pgmap v122: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T19:00:01.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:00 smithi016 bash[33624]: cluster 2023-09-23T19:00:00.000192+0000 mon.a (mon.0) 345 : cluster [INF] overall HEALTH_OK 2023-09-23T19:00:01.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:00 smithi016 bash[33624]: cluster 2023-09-23T19:00:00.632969+0000 mgr.y (mgr.14146) 143 : cluster [DBG] pgmap v122: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T19:00:01.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:00 smithi092 bash[27556]: cluster 2023-09-23T19:00:00.000192+0000 mon.a (mon.0) 345 : cluster [INF] overall HEALTH_OK 2023-09-23T19:00:01.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:00 smithi092 bash[27556]: cluster 2023-09-23T19:00:00.632969+0000 mgr.y (mgr.14146) 143 : cluster [DBG] pgmap v122: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T19:00:02.106 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:01 smithi016 bash[27237]: audit 2023-09-23T19:00:01.063477+0000 mon.a (mon.0) 346 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T19:00:02.106 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:01 smithi016 bash[27237]: audit 2023-09-23T19:00:01.065053+0000 mon.a (mon.0) 347 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:00:02.106 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:01 smithi016 bash[27237]: cephadm 2023-09-23T19:00:01.066169+0000 mgr.y (mgr.14146) 144 : cephadm [INF] Deploying daemon osd.3 on smithi016 2023-09-23T19:00:02.106 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:01 smithi016 bash[33624]: audit 2023-09-23T19:00:01.063477+0000 mon.a (mon.0) 346 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T19:00:02.107 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:01 smithi016 bash[33624]: audit 2023-09-23T19:00:01.065053+0000 mon.a (mon.0) 347 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:00:02.107 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:01 smithi016 bash[33624]: cephadm 2023-09-23T19:00:01.066169+0000 mgr.y (mgr.14146) 144 : cephadm [INF] Deploying daemon osd.3 on smithi016 2023-09-23T19:00:02.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:01 smithi092 bash[27556]: audit 2023-09-23T19:00:01.063477+0000 mon.a (mon.0) 346 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T19:00:02.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:01 smithi092 bash[27556]: audit 2023-09-23T19:00:01.065053+0000 mon.a (mon.0) 347 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:00:02.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:01 smithi092 bash[27556]: cephadm 2023-09-23T19:00:01.066169+0000 mgr.y (mgr.14146) 144 : cephadm [INF] Deploying daemon osd.3 on smithi016 2023-09-23T19:00:03.283 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:02 smithi016 bash[33624]: cluster 2023-09-23T19:00:02.633497+0000 mgr.y (mgr.14146) 145 : cluster [DBG] pgmap v123: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T19:00:03.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:02 smithi016 bash[27237]: cluster 2023-09-23T19:00:02.633497+0000 mgr.y (mgr.14146) 145 : cluster [DBG] pgmap v123: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T19:00:03.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:02 smithi092 bash[27556]: cluster 2023-09-23T19:00:02.633497+0000 mgr.y (mgr.14146) 145 : cluster [DBG] pgmap v123: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T19:00:06.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:05 smithi016 bash[27237]: cluster 2023-09-23T19:00:04.634040+0000 mgr.y (mgr.14146) 146 : cluster [DBG] pgmap v124: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T19:00:06.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:05 smithi016 bash[27237]: audit 2023-09-23T19:00:05.350084+0000 mon.a (mon.0) 348 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:00:06.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:05 smithi016 bash[27237]: audit 2023-09-23T19:00:05.357590+0000 mon.a (mon.0) 349 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:00:06.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:05 smithi016 bash[33624]: cluster 2023-09-23T19:00:04.634040+0000 mgr.y (mgr.14146) 146 : cluster [DBG] pgmap v124: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T19:00:06.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:05 smithi016 bash[33624]: audit 2023-09-23T19:00:05.350084+0000 mon.a (mon.0) 348 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:00:06.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:05 smithi016 bash[33624]: audit 2023-09-23T19:00:05.357590+0000 mon.a (mon.0) 349 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:00:06.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:05 smithi092 bash[27556]: cluster 2023-09-23T19:00:04.634040+0000 mgr.y (mgr.14146) 146 : cluster [DBG] pgmap v124: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T19:00:06.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:05 smithi092 bash[27556]: audit 2023-09-23T19:00:05.350084+0000 mon.a (mon.0) 348 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:00:06.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:05 smithi092 bash[27556]: audit 2023-09-23T19:00:05.357590+0000 mon.a (mon.0) 349 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:00:08.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:07 smithi016 bash[33624]: cluster 2023-09-23T19:00:06.634560+0000 mgr.y (mgr.14146) 147 : cluster [DBG] pgmap v125: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T19:00:08.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:07 smithi016 bash[27237]: cluster 2023-09-23T19:00:06.634560+0000 mgr.y (mgr.14146) 147 : cluster [DBG] pgmap v125: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T19:00:08.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:07 smithi092 bash[27556]: cluster 2023-09-23T19:00:06.634560+0000 mgr.y (mgr.14146) 147 : cluster [DBG] pgmap v125: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T19:00:10.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:09 smithi016 bash[27237]: cluster 2023-09-23T19:00:08.635147+0000 mgr.y (mgr.14146) 148 : cluster [DBG] pgmap v126: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T19:00:10.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:09 smithi016 bash[33624]: cluster 2023-09-23T19:00:08.635147+0000 mgr.y (mgr.14146) 148 : cluster [DBG] pgmap v126: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T19:00:10.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:09 smithi092 bash[27556]: cluster 2023-09-23T19:00:08.635147+0000 mgr.y (mgr.14146) 148 : cluster [DBG] pgmap v126: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T19:00:11.965 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:11 smithi016 bash[27237]: cluster 2023-09-23T19:00:10.635714+0000 mgr.y (mgr.14146) 149 : cluster [DBG] pgmap v127: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T19:00:11.965 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:11 smithi016 bash[27237]: audit 2023-09-23T19:00:10.651006+0000 mon.a (mon.0) 350 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:00:11.965 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:11 smithi016 bash[27237]: audit 2023-09-23T19:00:10.652358+0000 mon.a (mon.0) 351 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:00:11.965 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:11 smithi016 bash[27237]: audit 2023-09-23T19:00:10.653821+0000 mon.a (mon.0) 352 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:00:11.965 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:11 smithi016 bash[27237]: audit 2023-09-23T19:00:10.660193+0000 mon.a (mon.0) 353 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:00:11.965 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:11 smithi016 bash[33624]: cluster 2023-09-23T19:00:10.635714+0000 mgr.y (mgr.14146) 149 : cluster [DBG] pgmap v127: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T19:00:11.966 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:11 smithi016 bash[33624]: audit 2023-09-23T19:00:10.651006+0000 mon.a (mon.0) 350 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:00:11.966 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:11 smithi016 bash[33624]: audit 2023-09-23T19:00:10.652358+0000 mon.a (mon.0) 351 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:00:11.966 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:11 smithi016 bash[33624]: audit 2023-09-23T19:00:10.653821+0000 mon.a (mon.0) 352 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:00:11.966 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:11 smithi016 bash[33624]: audit 2023-09-23T19:00:10.660193+0000 mon.a (mon.0) 353 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:00:12.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:11 smithi092 bash[27556]: cluster 2023-09-23T19:00:10.635714+0000 mgr.y (mgr.14146) 149 : cluster [DBG] pgmap v127: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T19:00:12.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:11 smithi092 bash[27556]: audit 2023-09-23T19:00:10.651006+0000 mon.a (mon.0) 350 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:00:12.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:11 smithi092 bash[27556]: audit 2023-09-23T19:00:10.652358+0000 mon.a (mon.0) 351 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:00:12.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:11 smithi092 bash[27556]: audit 2023-09-23T19:00:10.653821+0000 mon.a (mon.0) 352 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:00:12.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:11 smithi092 bash[27556]: audit 2023-09-23T19:00:10.660193+0000 mon.a (mon.0) 353 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:00:13.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:12 smithi016 bash[27237]: audit 2023-09-23T19:00:11.967832+0000 mon.c (mon.2) 9 : audit [INF] from='osd.3 [v2:172.21.15.16:6826/204922667,v1:172.21.15.16:6827/204922667]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T19:00:13.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:12 smithi016 bash[27237]: audit 2023-09-23T19:00:11.968298+0000 mon.a (mon.0) 354 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T19:00:13.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:12 smithi016 bash[33624]: audit 2023-09-23T19:00:11.967832+0000 mon.c (mon.2) 9 : audit [INF] from='osd.3 [v2:172.21.15.16:6826/204922667,v1:172.21.15.16:6827/204922667]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T19:00:13.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:12 smithi016 bash[33624]: audit 2023-09-23T19:00:11.968298+0000 mon.a (mon.0) 354 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T19:00:13.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:12 smithi092 bash[27556]: audit 2023-09-23T19:00:11.967832+0000 mon.c (mon.2) 9 : audit [INF] from='osd.3 [v2:172.21.15.16:6826/204922667,v1:172.21.15.16:6827/204922667]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T19:00:13.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:12 smithi092 bash[27556]: audit 2023-09-23T19:00:11.968298+0000 mon.a (mon.0) 354 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T19:00:14.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:13 smithi016 bash[27237]: cluster 2023-09-23T19:00:12.636177+0000 mgr.y (mgr.14146) 150 : cluster [DBG] pgmap v128: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T19:00:14.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:13 smithi016 bash[27237]: audit 2023-09-23T19:00:12.665022+0000 mon.a (mon.0) 355 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T19:00:14.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:13 smithi016 bash[27237]: cluster 2023-09-23T19:00:12.665167+0000 mon.a (mon.0) 356 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2023-09-23T19:00:14.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:13 smithi016 bash[27237]: audit 2023-09-23T19:00:12.665426+0000 mon.a (mon.0) 357 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:00:14.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:13 smithi016 bash[27237]: audit 2023-09-23T19:00:12.666136+0000 mon.c (mon.2) 10 : audit [INF] from='osd.3 [v2:172.21.15.16:6826/204922667,v1:172.21.15.16:6827/204922667]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-09-23T19:00:14.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:13 smithi016 bash[27237]: audit 2023-09-23T19:00:12.666661+0000 mon.a (mon.0) 358 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-09-23T19:00:14.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:13 smithi016 bash[33624]: cluster 2023-09-23T19:00:12.636177+0000 mgr.y (mgr.14146) 150 : cluster [DBG] pgmap v128: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T19:00:14.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:13 smithi016 bash[33624]: audit 2023-09-23T19:00:12.665022+0000 mon.a (mon.0) 355 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T19:00:14.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:13 smithi016 bash[33624]: cluster 2023-09-23T19:00:12.665167+0000 mon.a (mon.0) 356 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2023-09-23T19:00:14.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:13 smithi016 bash[33624]: audit 2023-09-23T19:00:12.665426+0000 mon.a (mon.0) 357 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:00:14.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:13 smithi016 bash[33624]: audit 2023-09-23T19:00:12.666136+0000 mon.c (mon.2) 10 : audit [INF] from='osd.3 [v2:172.21.15.16:6826/204922667,v1:172.21.15.16:6827/204922667]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-09-23T19:00:14.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:13 smithi016 bash[33624]: audit 2023-09-23T19:00:12.666661+0000 mon.a (mon.0) 358 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-09-23T19:00:14.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:13 smithi092 bash[27556]: cluster 2023-09-23T19:00:12.636177+0000 mgr.y (mgr.14146) 150 : cluster [DBG] pgmap v128: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T19:00:14.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:13 smithi092 bash[27556]: audit 2023-09-23T19:00:12.665022+0000 mon.a (mon.0) 355 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T19:00:14.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:13 smithi092 bash[27556]: cluster 2023-09-23T19:00:12.665167+0000 mon.a (mon.0) 356 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2023-09-23T19:00:14.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:13 smithi092 bash[27556]: audit 2023-09-23T19:00:12.665426+0000 mon.a (mon.0) 357 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:00:14.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:13 smithi092 bash[27556]: audit 2023-09-23T19:00:12.666136+0000 mon.c (mon.2) 10 : audit [INF] from='osd.3 [v2:172.21.15.16:6826/204922667,v1:172.21.15.16:6827/204922667]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-09-23T19:00:14.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:13 smithi092 bash[27556]: audit 2023-09-23T19:00:12.666661+0000 mon.a (mon.0) 358 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-09-23T19:00:14.053 INFO:teuthology.orchestra.run.smithi016.stdout:Created osd(s) 3 on host 'smithi016' 2023-09-23T19:00:14.684 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:14 smithi016 bash[27237]: audit 2023-09-23T19:00:13.668808+0000 mon.a (mon.0) 359 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]': finished 2023-09-23T19:00:14.684 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:14 smithi016 bash[27237]: cluster 2023-09-23T19:00:13.668914+0000 mon.a (mon.0) 360 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2023-09-23T19:00:14.684 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:14 smithi016 bash[27237]: audit 2023-09-23T19:00:13.669493+0000 mon.a (mon.0) 361 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:00:14.684 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:14 smithi016 bash[27237]: audit 2023-09-23T19:00:13.675776+0000 mon.a (mon.0) 362 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:00:14.684 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:14 smithi016 bash[27237]: audit 2023-09-23T19:00:14.051158+0000 mon.a (mon.0) 363 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:00:14.684 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:14 smithi016 bash[27237]: audit 2023-09-23T19:00:14.085849+0000 mon.a (mon.0) 364 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:00:14.746 DEBUG:teuthology.orchestra.run.smithi016:osd.3> sudo journalctl -f -n 0 -u ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@osd.3.service 2023-09-23T19:00:14.749 INFO:tasks.cephadm:Deploying osd.4 on smithi092 with /dev/vg_nvme/lv_4... 2023-09-23T19:00:14.749 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-09-23T19:00:14.877 INFO:journalctl@ceph.osd.3.smithi016.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-09-23T19:00:14.946 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:14 smithi092 bash[27556]: audit 2023-09-23T19:00:13.668808+0000 mon.a (mon.0) 359 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]': finished 2023-09-23T19:00:14.946 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:14 smithi092 bash[27556]: cluster 2023-09-23T19:00:13.668914+0000 mon.a (mon.0) 360 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2023-09-23T19:00:14.946 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:14 smithi092 bash[27556]: audit 2023-09-23T19:00:13.669493+0000 mon.a (mon.0) 361 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:00:14.947 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:14 smithi092 bash[27556]: audit 2023-09-23T19:00:13.675776+0000 mon.a (mon.0) 362 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:00:14.947 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:14 smithi092 bash[27556]: audit 2023-09-23T19:00:14.051158+0000 mon.a (mon.0) 363 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:00:14.947 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:14 smithi092 bash[27556]: audit 2023-09-23T19:00:14.085849+0000 mon.a (mon.0) 364 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:00:15.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:14 smithi016 bash[33624]: audit 2023-09-23T19:00:13.668808+0000 mon.a (mon.0) 359 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]': finished 2023-09-23T19:00:15.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:14 smithi016 bash[33624]: cluster 2023-09-23T19:00:13.668914+0000 mon.a (mon.0) 360 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2023-09-23T19:00:15.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:14 smithi016 bash[33624]: audit 2023-09-23T19:00:13.669493+0000 mon.a (mon.0) 361 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:00:15.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:14 smithi016 bash[33624]: audit 2023-09-23T19:00:13.675776+0000 mon.a (mon.0) 362 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:00:15.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:14 smithi016 bash[33624]: audit 2023-09-23T19:00:14.051158+0000 mon.a (mon.0) 363 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:00:15.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:14 smithi016 bash[33624]: audit 2023-09-23T19:00:14.085849+0000 mon.a (mon.0) 364 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:00:15.965 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:15 smithi092 bash[27556]: cluster 2023-09-23T19:00:12.926259+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T19:00:15.965 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:15 smithi092 bash[27556]: cluster 2023-09-23T19:00:12.926359+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T19:00:15.965 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:15 smithi092 bash[27556]: cluster 2023-09-23T19:00:14.636580+0000 mgr.y (mgr.14146) 151 : cluster [DBG] pgmap v131: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T19:00:15.965 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:15 smithi092 bash[27556]: cluster 2023-09-23T19:00:14.678127+0000 mon.a (mon.0) 365 : cluster [INF] osd.3 [v2:172.21.15.16:6826/204922667,v1:172.21.15.16:6827/204922667] boot 2023-09-23T19:00:15.966 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:15 smithi092 bash[27556]: cluster 2023-09-23T19:00:14.678196+0000 mon.a (mon.0) 366 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2023-09-23T19:00:15.966 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:15 smithi092 bash[27556]: audit 2023-09-23T19:00:14.678856+0000 mon.a (mon.0) 367 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:00:16.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:15 smithi016 bash[27237]: cluster 2023-09-23T19:00:12.926259+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T19:00:16.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:15 smithi016 bash[27237]: cluster 2023-09-23T19:00:12.926359+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T19:00:16.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:15 smithi016 bash[27237]: cluster 2023-09-23T19:00:14.636580+0000 mgr.y (mgr.14146) 151 : cluster [DBG] pgmap v131: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T19:00:16.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:15 smithi016 bash[27237]: cluster 2023-09-23T19:00:14.678127+0000 mon.a (mon.0) 365 : cluster [INF] osd.3 [v2:172.21.15.16:6826/204922667,v1:172.21.15.16:6827/204922667] boot 2023-09-23T19:00:16.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:15 smithi016 bash[27237]: cluster 2023-09-23T19:00:14.678196+0000 mon.a (mon.0) 366 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2023-09-23T19:00:16.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:15 smithi016 bash[27237]: audit 2023-09-23T19:00:14.678856+0000 mon.a (mon.0) 367 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:00:16.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:15 smithi016 bash[33624]: cluster 2023-09-23T19:00:12.926259+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T19:00:16.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:15 smithi016 bash[33624]: cluster 2023-09-23T19:00:12.926359+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T19:00:16.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:15 smithi016 bash[33624]: cluster 2023-09-23T19:00:14.636580+0000 mgr.y (mgr.14146) 151 : cluster [DBG] pgmap v131: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T19:00:16.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:15 smithi016 bash[33624]: cluster 2023-09-23T19:00:14.678127+0000 mon.a (mon.0) 365 : cluster [INF] osd.3 [v2:172.21.15.16:6826/204922667,v1:172.21.15.16:6827/204922667] boot 2023-09-23T19:00:16.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:15 smithi016 bash[33624]: cluster 2023-09-23T19:00:14.678196+0000 mon.a (mon.0) 366 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2023-09-23T19:00:16.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:15 smithi016 bash[33624]: audit 2023-09-23T19:00:14.678856+0000 mon.a (mon.0) 367 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:00:17.100 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:16 smithi092 bash[27556]: cluster 2023-09-23T19:00:15.831662+0000 mon.a (mon.0) 368 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2023-09-23T19:00:17.100 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:16 smithi092 bash[27556]: cluster 2023-09-23T19:00:16.637028+0000 mgr.y (mgr.14146) 152 : cluster [DBG] pgmap v134: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:17.283 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:16 smithi016 bash[33624]: cluster 2023-09-23T19:00:15.831662+0000 mon.a (mon.0) 368 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2023-09-23T19:00:17.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:16 smithi016 bash[33624]: cluster 2023-09-23T19:00:16.637028+0000 mgr.y (mgr.14146) 152 : cluster [DBG] pgmap v134: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:17.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:16 smithi016 bash[27237]: cluster 2023-09-23T19:00:15.831662+0000 mon.a (mon.0) 368 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2023-09-23T19:00:17.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:16 smithi016 bash[27237]: cluster 2023-09-23T19:00:16.637028+0000 mgr.y (mgr.14146) 152 : cluster [DBG] pgmap v134: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:17.839 INFO:teuthology.orchestra.run.smithi092.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-09-23T19:00:17.839 INFO:teuthology.orchestra.run.smithi092.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-09-23T19:00:17.839 INFO:teuthology.orchestra.run.smithi092.stderr: stderr: 10+0 records in 2023-09-23T19:00:17.839 INFO:teuthology.orchestra.run.smithi092.stderr:10+0 records out 2023-09-23T19:00:17.839 INFO:teuthology.orchestra.run.smithi092.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0180435 s, 581 MB/s 2023-09-23T19:00:17.839 INFO:teuthology.orchestra.run.smithi092.stderr:--> Zapping successful for: 2023-09-23T19:00:18.465 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi092:vg_nvme/lv_4 2023-09-23T19:00:20.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:19 smithi016 bash[27237]: cluster 2023-09-23T19:00:18.637612+0000 mgr.y (mgr.14146) 153 : cluster [DBG] pgmap v135: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:20.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:19 smithi016 bash[33624]: cluster 2023-09-23T19:00:18.637612+0000 mgr.y (mgr.14146) 153 : cluster [DBG] pgmap v135: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:20.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:19 smithi092 bash[27556]: cluster 2023-09-23T19:00:18.637612+0000 mgr.y (mgr.14146) 153 : cluster [DBG] pgmap v135: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:21.221 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:20 smithi016 bash[33624]: cluster 2023-09-23T19:00:20.638166+0000 mgr.y (mgr.14146) 154 : cluster [DBG] pgmap v136: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:21.221 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:20 smithi016 bash[27237]: cluster 2023-09-23T19:00:20.638166+0000 mgr.y (mgr.14146) 154 : cluster [DBG] pgmap v136: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:21.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:20 smithi092 bash[27556]: cluster 2023-09-23T19:00:20.638166+0000 mgr.y (mgr.14146) 154 : cluster [DBG] pgmap v136: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:22.241 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:21 smithi016 bash[33624]: audit 2023-09-23T19:00:21.292781+0000 mgr.y (mgr.14146) 155 : audit [DBG] from='client.24220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi092:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:00:22.241 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:21 smithi016 bash[33624]: audit 2023-09-23T19:00:21.299389+0000 mon.a (mon.0) 369 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T19:00:22.241 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:21 smithi016 bash[33624]: audit 2023-09-23T19:00:21.304994+0000 mon.a (mon.0) 370 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T19:00:22.241 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:21 smithi016 bash[33624]: audit 2023-09-23T19:00:21.306088+0000 mon.a (mon.0) 371 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:00:22.242 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:21 smithi016 bash[27237]: audit 2023-09-23T19:00:21.292781+0000 mgr.y (mgr.14146) 155 : audit [DBG] from='client.24220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi092:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:00:22.242 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:21 smithi016 bash[27237]: audit 2023-09-23T19:00:21.299389+0000 mon.a (mon.0) 369 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T19:00:22.242 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:21 smithi016 bash[27237]: audit 2023-09-23T19:00:21.304994+0000 mon.a (mon.0) 370 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T19:00:22.242 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:21 smithi016 bash[27237]: audit 2023-09-23T19:00:21.306088+0000 mon.a (mon.0) 371 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:00:22.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:21 smithi092 bash[27556]: audit 2023-09-23T19:00:21.292781+0000 mgr.y (mgr.14146) 155 : audit [DBG] from='client.24220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi092:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:00:22.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:21 smithi092 bash[27556]: audit 2023-09-23T19:00:21.299389+0000 mon.a (mon.0) 369 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T19:00:22.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:21 smithi092 bash[27556]: audit 2023-09-23T19:00:21.304994+0000 mon.a (mon.0) 370 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T19:00:22.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:21 smithi092 bash[27556]: audit 2023-09-23T19:00:21.306088+0000 mon.a (mon.0) 371 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:00:23.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:22 smithi016 bash[27237]: cluster 2023-09-23T19:00:22.638746+0000 mgr.y (mgr.14146) 156 : cluster [DBG] pgmap v137: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:23.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:22 smithi016 bash[33624]: cluster 2023-09-23T19:00:22.638746+0000 mgr.y (mgr.14146) 156 : cluster [DBG] pgmap v137: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:23.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:22 smithi092 bash[27556]: cluster 2023-09-23T19:00:22.638746+0000 mgr.y (mgr.14146) 156 : cluster [DBG] pgmap v137: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:24.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:23 smithi016 bash[33624]: cephadm 2023-09-23T19:00:22.961385+0000 mgr.y (mgr.14146) 157 : cephadm [INF] Detected new or changed devices on smithi016 2023-09-23T19:00:24.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:23 smithi016 bash[33624]: audit 2023-09-23T19:00:22.969884+0000 mon.a (mon.0) 372 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:00:24.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:23 smithi016 bash[33624]: audit 2023-09-23T19:00:22.971746+0000 mon.a (mon.0) 373 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:00:24.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:23 smithi016 bash[33624]: audit 2023-09-23T19:00:22.973238+0000 mon.a (mon.0) 374 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:00:24.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:23 smithi016 bash[33624]: audit 2023-09-23T19:00:22.974640+0000 mon.a (mon.0) 375 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:00:24.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:23 smithi016 bash[33624]: audit 2023-09-23T19:00:22.976084+0000 mon.a (mon.0) 376 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:00:24.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:23 smithi016 bash[33624]: cephadm 2023-09-23T19:00:22.977090+0000 mgr.y (mgr.14146) 158 : cephadm [INF] Adjusting osd_memory_target on smithi016 to 4061M 2023-09-23T19:00:24.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:23 smithi016 bash[33624]: audit 2023-09-23T19:00:22.982016+0000 mon.a (mon.0) 377 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:00:24.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:23 smithi016 bash[33624]: audit 2023-09-23T19:00:22.983765+0000 mon.a (mon.0) 378 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:00:24.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:23 smithi016 bash[33624]: audit 2023-09-23T19:00:22.984941+0000 mon.a (mon.0) 379 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:00:24.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:23 smithi016 bash[33624]: audit 2023-09-23T19:00:22.991564+0000 mon.a (mon.0) 380 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:00:24.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:23 smithi016 bash[27237]: cephadm 2023-09-23T19:00:22.961385+0000 mgr.y (mgr.14146) 157 : cephadm [INF] Detected new or changed devices on smithi016 2023-09-23T19:00:24.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:23 smithi016 bash[27237]: audit 2023-09-23T19:00:22.969884+0000 mon.a (mon.0) 372 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:00:24.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:23 smithi016 bash[27237]: audit 2023-09-23T19:00:22.971746+0000 mon.a (mon.0) 373 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:00:24.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:23 smithi016 bash[27237]: audit 2023-09-23T19:00:22.973238+0000 mon.a (mon.0) 374 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:00:24.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:23 smithi016 bash[27237]: audit 2023-09-23T19:00:22.974640+0000 mon.a (mon.0) 375 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:00:24.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:23 smithi016 bash[27237]: audit 2023-09-23T19:00:22.976084+0000 mon.a (mon.0) 376 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:00:24.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:23 smithi016 bash[27237]: cephadm 2023-09-23T19:00:22.977090+0000 mgr.y (mgr.14146) 158 : cephadm [INF] Adjusting osd_memory_target on smithi016 to 4061M 2023-09-23T19:00:24.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:23 smithi016 bash[27237]: audit 2023-09-23T19:00:22.982016+0000 mon.a (mon.0) 377 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:00:24.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:23 smithi016 bash[27237]: audit 2023-09-23T19:00:22.983765+0000 mon.a (mon.0) 378 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:00:24.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:23 smithi016 bash[27237]: audit 2023-09-23T19:00:22.984941+0000 mon.a (mon.0) 379 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:00:24.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:23 smithi016 bash[27237]: audit 2023-09-23T19:00:22.991564+0000 mon.a (mon.0) 380 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:00:24.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:23 smithi092 bash[27556]: cephadm 2023-09-23T19:00:22.961385+0000 mgr.y (mgr.14146) 157 : cephadm [INF] Detected new or changed devices on smithi016 2023-09-23T19:00:24.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:23 smithi092 bash[27556]: audit 2023-09-23T19:00:22.969884+0000 mon.a (mon.0) 372 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:00:24.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:23 smithi092 bash[27556]: audit 2023-09-23T19:00:22.971746+0000 mon.a (mon.0) 373 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:00:24.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:23 smithi092 bash[27556]: audit 2023-09-23T19:00:22.973238+0000 mon.a (mon.0) 374 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:00:24.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:23 smithi092 bash[27556]: audit 2023-09-23T19:00:22.974640+0000 mon.a (mon.0) 375 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:00:24.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:23 smithi092 bash[27556]: audit 2023-09-23T19:00:22.976084+0000 mon.a (mon.0) 376 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:00:24.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:23 smithi092 bash[27556]: cephadm 2023-09-23T19:00:22.977090+0000 mgr.y (mgr.14146) 158 : cephadm [INF] Adjusting osd_memory_target on smithi016 to 4061M 2023-09-23T19:00:24.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:23 smithi092 bash[27556]: audit 2023-09-23T19:00:22.982016+0000 mon.a (mon.0) 377 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:00:24.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:23 smithi092 bash[27556]: audit 2023-09-23T19:00:22.983765+0000 mon.a (mon.0) 378 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:00:24.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:23 smithi092 bash[27556]: audit 2023-09-23T19:00:22.984941+0000 mon.a (mon.0) 379 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:00:24.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:23 smithi092 bash[27556]: audit 2023-09-23T19:00:22.991564+0000 mon.a (mon.0) 380 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:00:25.283 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:24 smithi016 bash[33624]: cluster 2023-09-23T19:00:24.639291+0000 mgr.y (mgr.14146) 159 : cluster [DBG] pgmap v138: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:25.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:24 smithi016 bash[27237]: cluster 2023-09-23T19:00:24.639291+0000 mgr.y (mgr.14146) 159 : cluster [DBG] pgmap v138: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:25.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:24 smithi092 bash[27556]: cluster 2023-09-23T19:00:24.639291+0000 mgr.y (mgr.14146) 159 : cluster [DBG] pgmap v138: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:28.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:27 smithi016 bash[33624]: cluster 2023-09-23T19:00:26.639815+0000 mgr.y (mgr.14146) 160 : cluster [DBG] pgmap v139: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:28.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:27 smithi016 bash[27237]: cluster 2023-09-23T19:00:26.639815+0000 mgr.y (mgr.14146) 160 : cluster [DBG] pgmap v139: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:28.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:27 smithi092 bash[27556]: cluster 2023-09-23T19:00:26.639815+0000 mgr.y (mgr.14146) 160 : cluster [DBG] pgmap v139: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:30.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:29 smithi016 bash[27237]: cluster 2023-09-23T19:00:28.640377+0000 mgr.y (mgr.14146) 161 : cluster [DBG] pgmap v140: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:30.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:29 smithi016 bash[27237]: audit 2023-09-23T19:00:29.392255+0000 mon.a (mon.0) 381 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "55a8dab2-6393-44ab-943e-cb7ab2cb7c31"}]: dispatch 2023-09-23T19:00:30.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:29 smithi016 bash[27237]: audit 2023-09-23T19:00:29.393789+0000 mon.b (mon.1) 8 : audit [INF] from='client.? 172.21.15.92:0/1497858503' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "55a8dab2-6393-44ab-943e-cb7ab2cb7c31"}]: dispatch 2023-09-23T19:00:30.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:29 smithi016 bash[27237]: audit 2023-09-23T19:00:29.400354+0000 mon.a (mon.0) 382 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "55a8dab2-6393-44ab-943e-cb7ab2cb7c31"}]': finished 2023-09-23T19:00:30.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:29 smithi016 bash[27237]: cluster 2023-09-23T19:00:29.400492+0000 mon.a (mon.0) 383 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2023-09-23T19:00:30.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:29 smithi016 bash[27237]: audit 2023-09-23T19:00:29.400723+0000 mon.a (mon.0) 384 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T19:00:30.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:29 smithi016 bash[33624]: cluster 2023-09-23T19:00:28.640377+0000 mgr.y (mgr.14146) 161 : cluster [DBG] pgmap v140: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:30.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:29 smithi016 bash[33624]: audit 2023-09-23T19:00:29.392255+0000 mon.a (mon.0) 381 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "55a8dab2-6393-44ab-943e-cb7ab2cb7c31"}]: dispatch 2023-09-23T19:00:30.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:29 smithi016 bash[33624]: audit 2023-09-23T19:00:29.393789+0000 mon.b (mon.1) 8 : audit [INF] from='client.? 172.21.15.92:0/1497858503' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "55a8dab2-6393-44ab-943e-cb7ab2cb7c31"}]: dispatch 2023-09-23T19:00:30.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:29 smithi016 bash[33624]: audit 2023-09-23T19:00:29.400354+0000 mon.a (mon.0) 382 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "55a8dab2-6393-44ab-943e-cb7ab2cb7c31"}]': finished 2023-09-23T19:00:30.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:29 smithi016 bash[33624]: cluster 2023-09-23T19:00:29.400492+0000 mon.a (mon.0) 383 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2023-09-23T19:00:30.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:29 smithi016 bash[33624]: audit 2023-09-23T19:00:29.400723+0000 mon.a (mon.0) 384 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T19:00:30.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:29 smithi092 bash[27556]: cluster 2023-09-23T19:00:28.640377+0000 mgr.y (mgr.14146) 161 : cluster [DBG] pgmap v140: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:30.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:29 smithi092 bash[27556]: audit 2023-09-23T19:00:29.392255+0000 mon.a (mon.0) 381 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "55a8dab2-6393-44ab-943e-cb7ab2cb7c31"}]: dispatch 2023-09-23T19:00:30.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:29 smithi092 bash[27556]: audit 2023-09-23T19:00:29.393789+0000 mon.b (mon.1) 8 : audit [INF] from='client.? 172.21.15.92:0/1497858503' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "55a8dab2-6393-44ab-943e-cb7ab2cb7c31"}]: dispatch 2023-09-23T19:00:30.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:29 smithi092 bash[27556]: audit 2023-09-23T19:00:29.400354+0000 mon.a (mon.0) 382 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "55a8dab2-6393-44ab-943e-cb7ab2cb7c31"}]': finished 2023-09-23T19:00:30.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:29 smithi092 bash[27556]: cluster 2023-09-23T19:00:29.400492+0000 mon.a (mon.0) 383 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2023-09-23T19:00:30.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:29 smithi092 bash[27556]: audit 2023-09-23T19:00:29.400723+0000 mon.a (mon.0) 384 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T19:00:31.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:30 smithi016 bash[27237]: audit 2023-09-23T19:00:30.110878+0000 mon.a (mon.0) 385 : audit [DBG] from='client.? 172.21.15.92:0/957361237' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T19:00:31.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:30 smithi016 bash[27237]: cluster 2023-09-23T19:00:30.640909+0000 mgr.y (mgr.14146) 162 : cluster [DBG] pgmap v142: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:31.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:30 smithi016 bash[33624]: audit 2023-09-23T19:00:30.110878+0000 mon.a (mon.0) 385 : audit [DBG] from='client.? 172.21.15.92:0/957361237' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T19:00:31.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:30 smithi016 bash[33624]: cluster 2023-09-23T19:00:30.640909+0000 mgr.y (mgr.14146) 162 : cluster [DBG] pgmap v142: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:31.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:30 smithi092 bash[27556]: audit 2023-09-23T19:00:30.110878+0000 mon.a (mon.0) 385 : audit [DBG] from='client.? 172.21.15.92:0/957361237' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T19:00:31.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:30 smithi092 bash[27556]: cluster 2023-09-23T19:00:30.640909+0000 mgr.y (mgr.14146) 162 : cluster [DBG] pgmap v142: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:34.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:33 smithi016 bash[27237]: cluster 2023-09-23T19:00:32.641422+0000 mgr.y (mgr.14146) 163 : cluster [DBG] pgmap v143: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:34.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:33 smithi016 bash[33624]: cluster 2023-09-23T19:00:32.641422+0000 mgr.y (mgr.14146) 163 : cluster [DBG] pgmap v143: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:34.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:33 smithi092 bash[27556]: cluster 2023-09-23T19:00:32.641422+0000 mgr.y (mgr.14146) 163 : cluster [DBG] pgmap v143: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:36.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:35 smithi016 bash[27237]: cluster 2023-09-23T19:00:34.641951+0000 mgr.y (mgr.14146) 164 : cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:36.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:35 smithi016 bash[33624]: cluster 2023-09-23T19:00:34.641951+0000 mgr.y (mgr.14146) 164 : cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:36.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:35 smithi092 bash[27556]: cluster 2023-09-23T19:00:34.641951+0000 mgr.y (mgr.14146) 164 : cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:38.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:37 smithi016 bash[27237]: cluster 2023-09-23T19:00:36.642519+0000 mgr.y (mgr.14146) 165 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:38.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:37 smithi016 bash[33624]: cluster 2023-09-23T19:00:36.642519+0000 mgr.y (mgr.14146) 165 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:38.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:37 smithi092 bash[27556]: cluster 2023-09-23T19:00:36.642519+0000 mgr.y (mgr.14146) 165 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:40.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:39 smithi016 bash[27237]: cluster 2023-09-23T19:00:38.643052+0000 mgr.y (mgr.14146) 166 : cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:40.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:39 smithi016 bash[33624]: cluster 2023-09-23T19:00:38.643052+0000 mgr.y (mgr.14146) 166 : cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:40.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:39 smithi092 bash[27556]: cluster 2023-09-23T19:00:38.643052+0000 mgr.y (mgr.14146) 166 : cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:41.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:40 smithi092 bash[27556]: cluster 2023-09-23T19:00:40.643589+0000 mgr.y (mgr.14146) 167 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:41.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:40 smithi016 bash[27237]: cluster 2023-09-23T19:00:40.643589+0000 mgr.y (mgr.14146) 167 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:41.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:40 smithi016 bash[33624]: cluster 2023-09-23T19:00:40.643589+0000 mgr.y (mgr.14146) 167 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:42.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:41 smithi016 bash[27237]: audit 2023-09-23T19:00:41.847817+0000 mon.a (mon.0) 386 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T19:00:42.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:41 smithi016 bash[27237]: audit 2023-09-23T19:00:41.849638+0000 mon.a (mon.0) 387 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:00:42.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:41 smithi016 bash[33624]: audit 2023-09-23T19:00:41.847817+0000 mon.a (mon.0) 386 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T19:00:42.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:41 smithi016 bash[33624]: audit 2023-09-23T19:00:41.849638+0000 mon.a (mon.0) 387 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:00:42.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:41 smithi092 bash[27556]: audit 2023-09-23T19:00:41.847817+0000 mon.a (mon.0) 386 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T19:00:42.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:41 smithi092 bash[27556]: audit 2023-09-23T19:00:41.849638+0000 mon.a (mon.0) 387 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:00:43.275 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:42 smithi092 bash[27556]: cephadm 2023-09-23T19:00:41.851225+0000 mgr.y (mgr.14146) 168 : cephadm [INF] Deploying daemon osd.4 on smithi092 2023-09-23T19:00:43.275 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:42 smithi092 bash[27556]: cluster 2023-09-23T19:00:42.644154+0000 mgr.y (mgr.14146) 169 : cluster [DBG] pgmap v148: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:43.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:42 smithi016 bash[27237]: cephadm 2023-09-23T19:00:41.851225+0000 mgr.y (mgr.14146) 168 : cephadm [INF] Deploying daemon osd.4 on smithi092 2023-09-23T19:00:43.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:42 smithi016 bash[27237]: cluster 2023-09-23T19:00:42.644154+0000 mgr.y (mgr.14146) 169 : cluster [DBG] pgmap v148: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:43.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:42 smithi016 bash[33624]: cephadm 2023-09-23T19:00:41.851225+0000 mgr.y (mgr.14146) 168 : cephadm [INF] Deploying daemon osd.4 on smithi092 2023-09-23T19:00:43.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:42 smithi016 bash[33624]: cluster 2023-09-23T19:00:42.644154+0000 mgr.y (mgr.14146) 169 : cluster [DBG] pgmap v148: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:45.707 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:45 smithi092 bash[27556]: cluster 2023-09-23T19:00:44.644771+0000 mgr.y (mgr.14146) 170 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:46.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:45 smithi016 bash[27237]: cluster 2023-09-23T19:00:44.644771+0000 mgr.y (mgr.14146) 170 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:46.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:45 smithi016 bash[33624]: cluster 2023-09-23T19:00:44.644771+0000 mgr.y (mgr.14146) 170 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:47.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:46 smithi016 bash[27237]: audit 2023-09-23T19:00:45.963491+0000 mon.a (mon.0) 388 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:00:47.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:46 smithi016 bash[27237]: audit 2023-09-23T19:00:45.973819+0000 mon.a (mon.0) 389 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:00:47.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:46 smithi016 bash[27237]: cluster 2023-09-23T19:00:46.645456+0000 mgr.y (mgr.14146) 171 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:47.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:46 smithi016 bash[33624]: audit 2023-09-23T19:00:45.963491+0000 mon.a (mon.0) 388 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:00:47.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:46 smithi016 bash[33624]: audit 2023-09-23T19:00:45.973819+0000 mon.a (mon.0) 389 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:00:47.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:46 smithi016 bash[33624]: cluster 2023-09-23T19:00:46.645456+0000 mgr.y (mgr.14146) 171 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:47.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:46 smithi092 bash[27556]: audit 2023-09-23T19:00:45.963491+0000 mon.a (mon.0) 388 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:00:47.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:46 smithi092 bash[27556]: audit 2023-09-23T19:00:45.973819+0000 mon.a (mon.0) 389 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:00:47.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:46 smithi092 bash[27556]: cluster 2023-09-23T19:00:46.645456+0000 mgr.y (mgr.14146) 171 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:49.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:49 smithi092 bash[27556]: cluster 2023-09-23T19:00:48.646002+0000 mgr.y (mgr.14146) 172 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:50.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:49 smithi016 bash[27237]: cluster 2023-09-23T19:00:48.646002+0000 mgr.y (mgr.14146) 172 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:50.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:49 smithi016 bash[33624]: cluster 2023-09-23T19:00:48.646002+0000 mgr.y (mgr.14146) 172 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:51.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:50 smithi092 bash[27556]: cluster 2023-09-23T19:00:50.646529+0000 mgr.y (mgr.14146) 173 : cluster [DBG] pgmap v152: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:51.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:50 smithi016 bash[27237]: cluster 2023-09-23T19:00:50.646529+0000 mgr.y (mgr.14146) 173 : cluster [DBG] pgmap v152: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:51.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:50 smithi016 bash[33624]: cluster 2023-09-23T19:00:50.646529+0000 mgr.y (mgr.14146) 173 : cluster [DBG] pgmap v152: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:52.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:52 smithi092 bash[27556]: audit 2023-09-23T19:00:51.047951+0000 mon.a (mon.0) 390 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:00:52.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:52 smithi092 bash[27556]: audit 2023-09-23T19:00:51.329837+0000 mon.a (mon.0) 391 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:00:52.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:52 smithi092 bash[27556]: audit 2023-09-23T19:00:51.331940+0000 mon.a (mon.0) 392 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:00:52.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:52 smithi092 bash[27556]: audit 2023-09-23T19:00:51.340700+0000 mon.a (mon.0) 393 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:00:52.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:52 smithi016 bash[27237]: audit 2023-09-23T19:00:51.047951+0000 mon.a (mon.0) 390 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:00:52.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:52 smithi016 bash[27237]: audit 2023-09-23T19:00:51.329837+0000 mon.a (mon.0) 391 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:00:52.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:52 smithi016 bash[27237]: audit 2023-09-23T19:00:51.331940+0000 mon.a (mon.0) 392 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:00:52.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:52 smithi016 bash[27237]: audit 2023-09-23T19:00:51.340700+0000 mon.a (mon.0) 393 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:00:52.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:52 smithi016 bash[33624]: audit 2023-09-23T19:00:51.047951+0000 mon.a (mon.0) 390 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:00:52.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:52 smithi016 bash[33624]: audit 2023-09-23T19:00:51.329837+0000 mon.a (mon.0) 391 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:00:52.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:52 smithi016 bash[33624]: audit 2023-09-23T19:00:51.331940+0000 mon.a (mon.0) 392 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:00:52.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:52 smithi016 bash[33624]: audit 2023-09-23T19:00:51.340700+0000 mon.a (mon.0) 393 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:00:53.325 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:53 smithi092 bash[27556]: audit 2023-09-23T19:00:52.501845+0000 mon.a (mon.0) 394 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T19:00:53.325 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:53 smithi092 bash[27556]: audit 2023-09-23T19:00:52.503547+0000 mon.b (mon.1) 9 : audit [INF] from='osd.4 [v2:172.21.15.92:6800/99745912,v1:172.21.15.92:6801/99745912]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T19:00:53.325 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:53 smithi092 bash[27556]: cluster 2023-09-23T19:00:52.647080+0000 mgr.y (mgr.14146) 174 : cluster [DBG] pgmap v153: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:53.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:53 smithi016 bash[27237]: audit 2023-09-23T19:00:52.501845+0000 mon.a (mon.0) 394 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T19:00:53.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:53 smithi016 bash[27237]: audit 2023-09-23T19:00:52.503547+0000 mon.b (mon.1) 9 : audit [INF] from='osd.4 [v2:172.21.15.92:6800/99745912,v1:172.21.15.92:6801/99745912]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T19:00:53.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:53 smithi016 bash[27237]: cluster 2023-09-23T19:00:52.647080+0000 mgr.y (mgr.14146) 174 : cluster [DBG] pgmap v153: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:53.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:53 smithi016 bash[33624]: audit 2023-09-23T19:00:52.501845+0000 mon.a (mon.0) 394 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T19:00:53.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:53 smithi016 bash[33624]: audit 2023-09-23T19:00:52.503547+0000 mon.b (mon.1) 9 : audit [INF] from='osd.4 [v2:172.21.15.92:6800/99745912,v1:172.21.15.92:6801/99745912]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T19:00:53.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:53 smithi016 bash[33624]: cluster 2023-09-23T19:00:52.647080+0000 mgr.y (mgr.14146) 174 : cluster [DBG] pgmap v153: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:54.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:54 smithi092 bash[27556]: audit 2023-09-23T19:00:53.058452+0000 mon.a (mon.0) 395 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-09-23T19:00:54.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:54 smithi092 bash[27556]: cluster 2023-09-23T19:00:53.058543+0000 mon.a (mon.0) 396 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2023-09-23T19:00:54.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:54 smithi092 bash[27556]: audit 2023-09-23T19:00:53.058923+0000 mon.a (mon.0) 397 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T19:00:54.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:54 smithi092 bash[27556]: audit 2023-09-23T19:00:53.060612+0000 mon.a (mon.0) 398 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi092", "root=default"]}]: dispatch 2023-09-23T19:00:54.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:54 smithi092 bash[27556]: audit 2023-09-23T19:00:53.062445+0000 mon.b (mon.1) 10 : audit [INF] from='osd.4 [v2:172.21.15.92:6800/99745912,v1:172.21.15.92:6801/99745912]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi092", "root=default"]}]: dispatch 2023-09-23T19:00:54.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:54 smithi016 bash[27237]: audit 2023-09-23T19:00:53.058452+0000 mon.a (mon.0) 395 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-09-23T19:00:54.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:54 smithi016 bash[27237]: cluster 2023-09-23T19:00:53.058543+0000 mon.a (mon.0) 396 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2023-09-23T19:00:54.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:54 smithi016 bash[27237]: audit 2023-09-23T19:00:53.058923+0000 mon.a (mon.0) 397 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T19:00:54.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:54 smithi016 bash[27237]: audit 2023-09-23T19:00:53.060612+0000 mon.a (mon.0) 398 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi092", "root=default"]}]: dispatch 2023-09-23T19:00:54.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:54 smithi016 bash[27237]: audit 2023-09-23T19:00:53.062445+0000 mon.b (mon.1) 10 : audit [INF] from='osd.4 [v2:172.21.15.92:6800/99745912,v1:172.21.15.92:6801/99745912]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi092", "root=default"]}]: dispatch 2023-09-23T19:00:54.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:54 smithi016 bash[33624]: audit 2023-09-23T19:00:53.058452+0000 mon.a (mon.0) 395 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-09-23T19:00:54.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:54 smithi016 bash[33624]: cluster 2023-09-23T19:00:53.058543+0000 mon.a (mon.0) 396 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2023-09-23T19:00:54.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:54 smithi016 bash[33624]: audit 2023-09-23T19:00:53.058923+0000 mon.a (mon.0) 397 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T19:00:54.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:54 smithi016 bash[33624]: audit 2023-09-23T19:00:53.060612+0000 mon.a (mon.0) 398 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi092", "root=default"]}]: dispatch 2023-09-23T19:00:54.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:54 smithi016 bash[33624]: audit 2023-09-23T19:00:53.062445+0000 mon.b (mon.1) 10 : audit [INF] from='osd.4 [v2:172.21.15.92:6800/99745912,v1:172.21.15.92:6801/99745912]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi092", "root=default"]}]: dispatch 2023-09-23T19:00:54.662 INFO:teuthology.orchestra.run.smithi092.stdout:Created osd(s) 4 on host 'smithi092' 2023-09-23T19:00:55.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:55 smithi092 bash[27556]: audit 2023-09-23T19:00:54.062483+0000 mon.a (mon.0) 399 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi092", "root=default"]}]': finished 2023-09-23T19:00:55.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:55 smithi092 bash[27556]: cluster 2023-09-23T19:00:54.062764+0000 mon.a (mon.0) 400 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2023-09-23T19:00:55.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:55 smithi092 bash[27556]: audit 2023-09-23T19:00:54.063778+0000 mon.a (mon.0) 401 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T19:00:55.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:55 smithi092 bash[27556]: audit 2023-09-23T19:00:54.069270+0000 mon.a (mon.0) 402 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T19:00:55.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:55 smithi092 bash[27556]: cluster 2023-09-23T19:00:54.647614+0000 mgr.y (mgr.14146) 175 : cluster [DBG] pgmap v156: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:55.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:55 smithi092 bash[27556]: audit 2023-09-23T19:00:54.659513+0000 mon.a (mon.0) 403 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:00:55.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:55 smithi092 bash[27556]: audit 2023-09-23T19:00:54.667058+0000 mon.a (mon.0) 404 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:00:55.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:55 smithi092 bash[27556]: cluster 2023-09-23T19:00:54.842304+0000 mon.a (mon.0) 405 : cluster [INF] osd.4 [v2:172.21.15.92:6800/99745912,v1:172.21.15.92:6801/99745912] boot 2023-09-23T19:00:55.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:55 smithi092 bash[27556]: cluster 2023-09-23T19:00:54.842395+0000 mon.a (mon.0) 406 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2023-09-23T19:00:55.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:55 smithi092 bash[27556]: audit 2023-09-23T19:00:54.842933+0000 mon.a (mon.0) 407 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T19:00:55.336 DEBUG:teuthology.orchestra.run.smithi092:osd.4> sudo journalctl -f -n 0 -u ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@osd.4.service 2023-09-23T19:00:55.338 INFO:tasks.cephadm:Deploying osd.5 on smithi092 with /dev/vg_nvme/lv_3... 2023-09-23T19:00:55.338 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-09-23T19:00:55.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:55 smithi016 bash[27237]: audit 2023-09-23T19:00:54.062483+0000 mon.a (mon.0) 399 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi092", "root=default"]}]': finished 2023-09-23T19:00:55.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:55 smithi016 bash[27237]: cluster 2023-09-23T19:00:54.062764+0000 mon.a (mon.0) 400 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2023-09-23T19:00:55.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:55 smithi016 bash[27237]: audit 2023-09-23T19:00:54.063778+0000 mon.a (mon.0) 401 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T19:00:55.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:55 smithi016 bash[27237]: audit 2023-09-23T19:00:54.069270+0000 mon.a (mon.0) 402 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T19:00:55.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:55 smithi016 bash[27237]: cluster 2023-09-23T19:00:54.647614+0000 mgr.y (mgr.14146) 175 : cluster [DBG] pgmap v156: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:55.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:55 smithi016 bash[27237]: audit 2023-09-23T19:00:54.659513+0000 mon.a (mon.0) 403 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:00:55.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:55 smithi016 bash[27237]: audit 2023-09-23T19:00:54.667058+0000 mon.a (mon.0) 404 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:00:55.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:55 smithi016 bash[27237]: cluster 2023-09-23T19:00:54.842304+0000 mon.a (mon.0) 405 : cluster [INF] osd.4 [v2:172.21.15.92:6800/99745912,v1:172.21.15.92:6801/99745912] boot 2023-09-23T19:00:55.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:55 smithi016 bash[27237]: cluster 2023-09-23T19:00:54.842395+0000 mon.a (mon.0) 406 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2023-09-23T19:00:55.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:55 smithi016 bash[27237]: audit 2023-09-23T19:00:54.842933+0000 mon.a (mon.0) 407 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T19:00:55.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:55 smithi016 bash[33624]: audit 2023-09-23T19:00:54.062483+0000 mon.a (mon.0) 399 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi092", "root=default"]}]': finished 2023-09-23T19:00:55.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:55 smithi016 bash[33624]: cluster 2023-09-23T19:00:54.062764+0000 mon.a (mon.0) 400 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2023-09-23T19:00:55.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:55 smithi016 bash[33624]: audit 2023-09-23T19:00:54.063778+0000 mon.a (mon.0) 401 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T19:00:55.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:55 smithi016 bash[33624]: audit 2023-09-23T19:00:54.069270+0000 mon.a (mon.0) 402 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T19:00:55.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:55 smithi016 bash[33624]: cluster 2023-09-23T19:00:54.647614+0000 mgr.y (mgr.14146) 175 : cluster [DBG] pgmap v156: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T19:00:55.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:55 smithi016 bash[33624]: audit 2023-09-23T19:00:54.659513+0000 mon.a (mon.0) 403 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:00:55.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:55 smithi016 bash[33624]: audit 2023-09-23T19:00:54.667058+0000 mon.a (mon.0) 404 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:00:55.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:55 smithi016 bash[33624]: cluster 2023-09-23T19:00:54.842304+0000 mon.a (mon.0) 405 : cluster [INF] osd.4 [v2:172.21.15.92:6800/99745912,v1:172.21.15.92:6801/99745912] boot 2023-09-23T19:00:55.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:55 smithi016 bash[33624]: cluster 2023-09-23T19:00:54.842395+0000 mon.a (mon.0) 406 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2023-09-23T19:00:55.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:55 smithi016 bash[33624]: audit 2023-09-23T19:00:54.842933+0000 mon.a (mon.0) 407 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T19:00:55.980 INFO:journalctl@ceph.osd.4.smithi092.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-09-23T19:00:56.392 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:56 smithi092 bash[27556]: cluster 2023-09-23T19:00:53.466636+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T19:00:56.392 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:56 smithi092 bash[27556]: cluster 2023-09-23T19:00:53.466689+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T19:00:56.392 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:56 smithi092 bash[27556]: cluster 2023-09-23T19:00:55.852053+0000 mon.a (mon.0) 408 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2023-09-23T19:00:56.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:56 smithi016 bash[27237]: cluster 2023-09-23T19:00:53.466636+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T19:00:56.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:56 smithi016 bash[27237]: cluster 2023-09-23T19:00:53.466689+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T19:00:56.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:56 smithi016 bash[27237]: cluster 2023-09-23T19:00:55.852053+0000 mon.a (mon.0) 408 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2023-09-23T19:00:56.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:56 smithi016 bash[33624]: cluster 2023-09-23T19:00:53.466636+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T19:00:56.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:56 smithi016 bash[33624]: cluster 2023-09-23T19:00:53.466689+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T19:00:56.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:56 smithi016 bash[33624]: cluster 2023-09-23T19:00:55.852053+0000 mon.a (mon.0) 408 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2023-09-23T19:00:57.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:57 smithi016 bash[27237]: cluster 2023-09-23T19:00:56.648146+0000 mgr.y (mgr.14146) 176 : cluster [DBG] pgmap v159: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:00:57.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:57 smithi016 bash[27237]: cluster 2023-09-23T19:00:56.855260+0000 mon.a (mon.0) 409 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2023-09-23T19:00:57.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:57 smithi016 bash[33624]: cluster 2023-09-23T19:00:56.648146+0000 mgr.y (mgr.14146) 176 : cluster [DBG] pgmap v159: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:00:57.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:57 smithi016 bash[33624]: cluster 2023-09-23T19:00:56.855260+0000 mon.a (mon.0) 409 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2023-09-23T19:00:57.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:57 smithi092 bash[27556]: cluster 2023-09-23T19:00:56.648146+0000 mgr.y (mgr.14146) 176 : cluster [DBG] pgmap v159: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:00:57.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:57 smithi092 bash[27556]: cluster 2023-09-23T19:00:56.855260+0000 mon.a (mon.0) 409 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2023-09-23T19:00:58.751 INFO:teuthology.orchestra.run.smithi092.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-09-23T19:00:58.751 INFO:teuthology.orchestra.run.smithi092.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-09-23T19:00:58.751 INFO:teuthology.orchestra.run.smithi092.stderr: stderr: 10+0 records in 2023-09-23T19:00:58.751 INFO:teuthology.orchestra.run.smithi092.stderr:10+0 records out 2023-09-23T19:00:58.752 INFO:teuthology.orchestra.run.smithi092.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0173689 s, 604 MB/s 2023-09-23T19:00:58.752 INFO:teuthology.orchestra.run.smithi092.stderr:--> Zapping successful for: 2023-09-23T19:00:59.477 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi092:vg_nvme/lv_3 2023-09-23T19:00:59.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:00:59 smithi092 bash[27556]: cluster 2023-09-23T19:00:58.648702+0000 mgr.y (mgr.14146) 177 : cluster [DBG] pgmap v161: 1 pgs: 1 peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:00.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:00:59 smithi016 bash[33624]: cluster 2023-09-23T19:00:58.648702+0000 mgr.y (mgr.14146) 177 : cluster [DBG] pgmap v161: 1 pgs: 1 peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:00.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:00:59 smithi016 bash[27237]: cluster 2023-09-23T19:00:58.648702+0000 mgr.y (mgr.14146) 177 : cluster [DBG] pgmap v161: 1 pgs: 1 peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:01.203 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:00 smithi092 bash[27556]: cluster 2023-09-23T19:01:00.649261+0000 mgr.y (mgr.14146) 178 : cluster [DBG] pgmap v162: 1 pgs: 1 peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:01.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:00 smithi016 bash[27237]: cluster 2023-09-23T19:01:00.649261+0000 mgr.y (mgr.14146) 178 : cluster [DBG] pgmap v162: 1 pgs: 1 peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:01.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:00 smithi016 bash[33624]: cluster 2023-09-23T19:01:00.649261+0000 mgr.y (mgr.14146) 178 : cluster [DBG] pgmap v162: 1 pgs: 1 peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:03.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:03 smithi092 bash[27556]: cluster 2023-09-23T19:01:02.649851+0000 mgr.y (mgr.14146) 179 : cluster [DBG] pgmap v163: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:04.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:03 smithi016 bash[27237]: cluster 2023-09-23T19:01:02.649851+0000 mgr.y (mgr.14146) 179 : cluster [DBG] pgmap v163: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:04.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:03 smithi016 bash[33624]: cluster 2023-09-23T19:01:02.649851+0000 mgr.y (mgr.14146) 179 : cluster [DBG] pgmap v163: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:04.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:04 smithi092 bash[27556]: audit 2023-09-23T19:01:03.988853+0000 mon.a (mon.0) 410 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T19:01:04.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:04 smithi092 bash[27556]: audit 2023-09-23T19:01:03.994406+0000 mon.a (mon.0) 411 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T19:01:04.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:04 smithi092 bash[27556]: audit 2023-09-23T19:01:03.996249+0000 mon.a (mon.0) 412 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:01:04.969 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:04 smithi016 bash[33624]: audit 2023-09-23T19:01:03.988853+0000 mon.a (mon.0) 410 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T19:01:04.969 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:04 smithi016 bash[33624]: audit 2023-09-23T19:01:03.994406+0000 mon.a (mon.0) 411 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T19:01:04.969 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:04 smithi016 bash[33624]: audit 2023-09-23T19:01:03.996249+0000 mon.a (mon.0) 412 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:01:04.970 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:04 smithi016 bash[27237]: audit 2023-09-23T19:01:03.988853+0000 mon.a (mon.0) 410 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T19:01:04.970 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:04 smithi016 bash[27237]: audit 2023-09-23T19:01:03.994406+0000 mon.a (mon.0) 411 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T19:01:04.970 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:04 smithi016 bash[27237]: audit 2023-09-23T19:01:03.996249+0000 mon.a (mon.0) 412 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:01:06.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:05 smithi016 bash[27237]: audit 2023-09-23T19:01:03.982701+0000 mgr.y (mgr.14146) 180 : audit [DBG] from='client.24247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi092:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:01:06.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:05 smithi016 bash[27237]: cluster 2023-09-23T19:01:04.650379+0000 mgr.y (mgr.14146) 181 : cluster [DBG] pgmap v164: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:06.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:05 smithi016 bash[27237]: audit 2023-09-23T19:01:05.241029+0000 mon.a (mon.0) 413 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:01:06.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:05 smithi016 bash[27237]: audit 2023-09-23T19:01:05.242712+0000 mon.a (mon.0) 414 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:01:06.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:05 smithi016 bash[27237]: audit 2023-09-23T19:01:05.248859+0000 mon.a (mon.0) 415 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:01:06.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:05 smithi016 bash[27237]: audit 2023-09-23T19:01:05.250475+0000 mon.a (mon.0) 416 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:01:06.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:05 smithi016 bash[27237]: audit 2023-09-23T19:01:05.251698+0000 mon.a (mon.0) 417 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:01:06.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:05 smithi016 bash[27237]: audit 2023-09-23T19:01:05.258079+0000 mon.a (mon.0) 418 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:01:06.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:05 smithi016 bash[33624]: audit 2023-09-23T19:01:03.982701+0000 mgr.y (mgr.14146) 180 : audit [DBG] from='client.24247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi092:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:01:06.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:05 smithi016 bash[33624]: cluster 2023-09-23T19:01:04.650379+0000 mgr.y (mgr.14146) 181 : cluster [DBG] pgmap v164: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:06.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:05 smithi016 bash[33624]: audit 2023-09-23T19:01:05.241029+0000 mon.a (mon.0) 413 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:01:06.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:05 smithi016 bash[33624]: audit 2023-09-23T19:01:05.242712+0000 mon.a (mon.0) 414 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:01:06.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:05 smithi016 bash[33624]: audit 2023-09-23T19:01:05.248859+0000 mon.a (mon.0) 415 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:01:06.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:05 smithi016 bash[33624]: audit 2023-09-23T19:01:05.250475+0000 mon.a (mon.0) 416 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:01:06.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:05 smithi016 bash[33624]: audit 2023-09-23T19:01:05.251698+0000 mon.a (mon.0) 417 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:01:06.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:05 smithi016 bash[33624]: audit 2023-09-23T19:01:05.258079+0000 mon.a (mon.0) 418 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:01:06.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:05 smithi092 bash[27556]: audit 2023-09-23T19:01:03.982701+0000 mgr.y (mgr.14146) 180 : audit [DBG] from='client.24247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi092:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:01:06.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:05 smithi092 bash[27556]: cluster 2023-09-23T19:01:04.650379+0000 mgr.y (mgr.14146) 181 : cluster [DBG] pgmap v164: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:06.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:05 smithi092 bash[27556]: audit 2023-09-23T19:01:05.241029+0000 mon.a (mon.0) 413 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:01:06.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:05 smithi092 bash[27556]: audit 2023-09-23T19:01:05.242712+0000 mon.a (mon.0) 414 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:01:06.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:05 smithi092 bash[27556]: audit 2023-09-23T19:01:05.248859+0000 mon.a (mon.0) 415 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:01:06.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:05 smithi092 bash[27556]: audit 2023-09-23T19:01:05.250475+0000 mon.a (mon.0) 416 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:01:06.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:05 smithi092 bash[27556]: audit 2023-09-23T19:01:05.251698+0000 mon.a (mon.0) 417 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:01:06.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:05 smithi092 bash[27556]: audit 2023-09-23T19:01:05.258079+0000 mon.a (mon.0) 418 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:01:07.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:06 smithi016 bash[33624]: cephadm 2023-09-23T19:01:05.231810+0000 mgr.y (mgr.14146) 182 : cephadm [INF] Detected new or changed devices on smithi092 2023-09-23T19:01:07.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:06 smithi016 bash[33624]: cephadm 2023-09-23T19:01:05.243821+0000 mgr.y (mgr.14146) 183 : cephadm [INF] Adjusting osd_memory_target on smithi092 to 17265M 2023-09-23T19:01:07.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:06 smithi016 bash[27237]: cephadm 2023-09-23T19:01:05.231810+0000 mgr.y (mgr.14146) 182 : cephadm [INF] Detected new or changed devices on smithi092 2023-09-23T19:01:07.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:06 smithi016 bash[27237]: cephadm 2023-09-23T19:01:05.243821+0000 mgr.y (mgr.14146) 183 : cephadm [INF] Adjusting osd_memory_target on smithi092 to 17265M 2023-09-23T19:01:07.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:06 smithi092 bash[27556]: cephadm 2023-09-23T19:01:05.231810+0000 mgr.y (mgr.14146) 182 : cephadm [INF] Detected new or changed devices on smithi092 2023-09-23T19:01:07.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:06 smithi092 bash[27556]: cephadm 2023-09-23T19:01:05.243821+0000 mgr.y (mgr.14146) 183 : cephadm [INF] Adjusting osd_memory_target on smithi092 to 17265M 2023-09-23T19:01:08.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:07 smithi016 bash[27237]: cluster 2023-09-23T19:01:06.651058+0000 mgr.y (mgr.14146) 184 : cluster [DBG] pgmap v165: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:08.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:07 smithi016 bash[33624]: cluster 2023-09-23T19:01:06.651058+0000 mgr.y (mgr.14146) 184 : cluster [DBG] pgmap v165: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:08.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:07 smithi092 bash[27556]: cluster 2023-09-23T19:01:06.651058+0000 mgr.y (mgr.14146) 184 : cluster [DBG] pgmap v165: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:09.928 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:09 smithi092 bash[27556]: cluster 2023-09-23T19:01:08.651644+0000 mgr.y (mgr.14146) 185 : cluster [DBG] pgmap v166: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:10.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:09 smithi016 bash[27237]: cluster 2023-09-23T19:01:08.651644+0000 mgr.y (mgr.14146) 185 : cluster [DBG] pgmap v166: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:10.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:09 smithi016 bash[33624]: cluster 2023-09-23T19:01:08.651644+0000 mgr.y (mgr.14146) 185 : cluster [DBG] pgmap v166: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:11.137 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:10 smithi092 bash[27556]: cluster 2023-09-23T19:01:10.652262+0000 mgr.y (mgr.14146) 186 : cluster [DBG] pgmap v167: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:11.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:10 smithi016 bash[27237]: cluster 2023-09-23T19:01:10.652262+0000 mgr.y (mgr.14146) 186 : cluster [DBG] pgmap v167: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:11.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:10 smithi016 bash[33624]: cluster 2023-09-23T19:01:10.652262+0000 mgr.y (mgr.14146) 186 : cluster [DBG] pgmap v167: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:12.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:12 smithi016 bash[27237]: audit 2023-09-23T19:01:12.083853+0000 mon.a (mon.0) 419 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b29b73b1-51d7-4484-bd79-2334c650cb62"}]: dispatch 2023-09-23T19:01:12.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:12 smithi016 bash[27237]: audit 2023-09-23T19:01:12.086072+0000 mon.b (mon.1) 11 : audit [INF] from='client.? 172.21.15.92:0/2178538977' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b29b73b1-51d7-4484-bd79-2334c650cb62"}]: dispatch 2023-09-23T19:01:12.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:12 smithi016 bash[27237]: audit 2023-09-23T19:01:12.092038+0000 mon.a (mon.0) 420 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b29b73b1-51d7-4484-bd79-2334c650cb62"}]': finished 2023-09-23T19:01:12.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:12 smithi016 bash[27237]: cluster 2023-09-23T19:01:12.092141+0000 mon.a (mon.0) 421 : cluster [DBG] osdmap e33: 6 total, 5 up, 6 in 2023-09-23T19:01:12.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:12 smithi016 bash[27237]: audit 2023-09-23T19:01:12.092432+0000 mon.a (mon.0) 422 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:01:12.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:12 smithi016 bash[33624]: audit 2023-09-23T19:01:12.083853+0000 mon.a (mon.0) 419 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b29b73b1-51d7-4484-bd79-2334c650cb62"}]: dispatch 2023-09-23T19:01:12.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:12 smithi016 bash[33624]: audit 2023-09-23T19:01:12.086072+0000 mon.b (mon.1) 11 : audit [INF] from='client.? 172.21.15.92:0/2178538977' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b29b73b1-51d7-4484-bd79-2334c650cb62"}]: dispatch 2023-09-23T19:01:12.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:12 smithi016 bash[33624]: audit 2023-09-23T19:01:12.092038+0000 mon.a (mon.0) 420 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b29b73b1-51d7-4484-bd79-2334c650cb62"}]': finished 2023-09-23T19:01:12.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:12 smithi016 bash[33624]: cluster 2023-09-23T19:01:12.092141+0000 mon.a (mon.0) 421 : cluster [DBG] osdmap e33: 6 total, 5 up, 6 in 2023-09-23T19:01:12.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:12 smithi016 bash[33624]: audit 2023-09-23T19:01:12.092432+0000 mon.a (mon.0) 422 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:01:12.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:12 smithi092 bash[27556]: audit 2023-09-23T19:01:12.083853+0000 mon.a (mon.0) 419 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b29b73b1-51d7-4484-bd79-2334c650cb62"}]: dispatch 2023-09-23T19:01:12.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:12 smithi092 bash[27556]: audit 2023-09-23T19:01:12.086072+0000 mon.b (mon.1) 11 : audit [INF] from='client.? 172.21.15.92:0/2178538977' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b29b73b1-51d7-4484-bd79-2334c650cb62"}]: dispatch 2023-09-23T19:01:12.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:12 smithi092 bash[27556]: audit 2023-09-23T19:01:12.092038+0000 mon.a (mon.0) 420 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b29b73b1-51d7-4484-bd79-2334c650cb62"}]': finished 2023-09-23T19:01:12.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:12 smithi092 bash[27556]: cluster 2023-09-23T19:01:12.092141+0000 mon.a (mon.0) 421 : cluster [DBG] osdmap e33: 6 total, 5 up, 6 in 2023-09-23T19:01:12.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:12 smithi092 bash[27556]: audit 2023-09-23T19:01:12.092432+0000 mon.a (mon.0) 422 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:01:13.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:13 smithi016 bash[27237]: cluster 2023-09-23T19:01:12.652796+0000 mgr.y (mgr.14146) 187 : cluster [DBG] pgmap v169: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:13.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:13 smithi016 bash[27237]: audit 2023-09-23T19:01:12.790542+0000 mon.b (mon.1) 12 : audit [DBG] from='client.? 172.21.15.92:0/3250042868' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T19:01:13.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:13 smithi016 bash[33624]: cluster 2023-09-23T19:01:12.652796+0000 mgr.y (mgr.14146) 187 : cluster [DBG] pgmap v169: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:13.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:13 smithi016 bash[33624]: audit 2023-09-23T19:01:12.790542+0000 mon.b (mon.1) 12 : audit [DBG] from='client.? 172.21.15.92:0/3250042868' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T19:01:13.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:13 smithi092 bash[27556]: cluster 2023-09-23T19:01:12.652796+0000 mgr.y (mgr.14146) 187 : cluster [DBG] pgmap v169: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:13.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:13 smithi092 bash[27556]: audit 2023-09-23T19:01:12.790542+0000 mon.b (mon.1) 12 : audit [DBG] from='client.? 172.21.15.92:0/3250042868' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T19:01:15.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:15 smithi092 bash[27556]: cluster 2023-09-23T19:01:14.653337+0000 mgr.y (mgr.14146) 188 : cluster [DBG] pgmap v170: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:16.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:15 smithi016 bash[33624]: cluster 2023-09-23T19:01:14.653337+0000 mgr.y (mgr.14146) 188 : cluster [DBG] pgmap v170: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:16.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:15 smithi016 bash[27237]: cluster 2023-09-23T19:01:14.653337+0000 mgr.y (mgr.14146) 188 : cluster [DBG] pgmap v170: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:18.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:17 smithi016 bash[33624]: cluster 2023-09-23T19:01:16.653893+0000 mgr.y (mgr.14146) 189 : cluster [DBG] pgmap v171: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:18.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:17 smithi016 bash[27237]: cluster 2023-09-23T19:01:16.653893+0000 mgr.y (mgr.14146) 189 : cluster [DBG] pgmap v171: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:18.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:17 smithi092 bash[27556]: cluster 2023-09-23T19:01:16.653893+0000 mgr.y (mgr.14146) 189 : cluster [DBG] pgmap v171: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:20.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:19 smithi016 bash[27237]: cluster 2023-09-23T19:01:18.654463+0000 mgr.y (mgr.14146) 190 : cluster [DBG] pgmap v172: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:20.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:19 smithi016 bash[33624]: cluster 2023-09-23T19:01:18.654463+0000 mgr.y (mgr.14146) 190 : cluster [DBG] pgmap v172: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:20.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:19 smithi092 bash[27556]: cluster 2023-09-23T19:01:18.654463+0000 mgr.y (mgr.14146) 190 : cluster [DBG] pgmap v172: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:21.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:20 smithi092 bash[27556]: cluster 2023-09-23T19:01:20.654985+0000 mgr.y (mgr.14146) 191 : cluster [DBG] pgmap v173: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:21.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:20 smithi016 bash[27237]: cluster 2023-09-23T19:01:20.654985+0000 mgr.y (mgr.14146) 191 : cluster [DBG] pgmap v173: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:21.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:20 smithi016 bash[33624]: cluster 2023-09-23T19:01:20.654985+0000 mgr.y (mgr.14146) 191 : cluster [DBG] pgmap v173: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:23.983 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:23 smithi092 bash[27556]: cluster 2023-09-23T19:01:22.655548+0000 mgr.y (mgr.14146) 192 : cluster [DBG] pgmap v174: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:24.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:23 smithi016 bash[27237]: cluster 2023-09-23T19:01:22.655548+0000 mgr.y (mgr.14146) 192 : cluster [DBG] pgmap v174: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:24.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:23 smithi016 bash[33624]: cluster 2023-09-23T19:01:22.655548+0000 mgr.y (mgr.14146) 192 : cluster [DBG] pgmap v174: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:25.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:24 smithi016 bash[27237]: audit 2023-09-23T19:01:24.478670+0000 mon.a (mon.0) 423 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T19:01:25.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:24 smithi016 bash[27237]: audit 2023-09-23T19:01:24.480489+0000 mon.a (mon.0) 424 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:01:25.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:24 smithi016 bash[33624]: audit 2023-09-23T19:01:24.478670+0000 mon.a (mon.0) 423 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T19:01:25.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:24 smithi016 bash[33624]: audit 2023-09-23T19:01:24.480489+0000 mon.a (mon.0) 424 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:01:25.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:24 smithi092 bash[27556]: audit 2023-09-23T19:01:24.478670+0000 mon.a (mon.0) 423 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T19:01:25.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:24 smithi092 bash[27556]: audit 2023-09-23T19:01:24.480489+0000 mon.a (mon.0) 424 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:01:25.948 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:25 smithi092 bash[27556]: cephadm 2023-09-23T19:01:24.481915+0000 mgr.y (mgr.14146) 193 : cephadm [INF] Deploying daemon osd.5 on smithi092 2023-09-23T19:01:25.949 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:25 smithi092 bash[27556]: cluster 2023-09-23T19:01:24.656063+0000 mgr.y (mgr.14146) 194 : cluster [DBG] pgmap v175: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:26.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:25 smithi016 bash[33624]: cephadm 2023-09-23T19:01:24.481915+0000 mgr.y (mgr.14146) 193 : cephadm [INF] Deploying daemon osd.5 on smithi092 2023-09-23T19:01:26.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:25 smithi016 bash[33624]: cluster 2023-09-23T19:01:24.656063+0000 mgr.y (mgr.14146) 194 : cluster [DBG] pgmap v175: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:26.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:25 smithi016 bash[27237]: cephadm 2023-09-23T19:01:24.481915+0000 mgr.y (mgr.14146) 193 : cephadm [INF] Deploying daemon osd.5 on smithi092 2023-09-23T19:01:26.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:25 smithi016 bash[27237]: cluster 2023-09-23T19:01:24.656063+0000 mgr.y (mgr.14146) 194 : cluster [DBG] pgmap v175: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:27.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:27 smithi092 bash[27556]: cluster 2023-09-23T19:01:26.656582+0000 mgr.y (mgr.14146) 195 : cluster [DBG] pgmap v176: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:28.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:27 smithi016 bash[27237]: cluster 2023-09-23T19:01:26.656582+0000 mgr.y (mgr.14146) 195 : cluster [DBG] pgmap v176: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:28.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:27 smithi016 bash[33624]: cluster 2023-09-23T19:01:26.656582+0000 mgr.y (mgr.14146) 195 : cluster [DBG] pgmap v176: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:29.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:29 smithi092 bash[27556]: cluster 2023-09-23T19:01:28.657071+0000 mgr.y (mgr.14146) 196 : cluster [DBG] pgmap v177: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:29.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:29 smithi092 bash[27556]: audit 2023-09-23T19:01:28.662028+0000 mon.a (mon.0) 425 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:01:29.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:29 smithi092 bash[27556]: audit 2023-09-23T19:01:28.694845+0000 mon.a (mon.0) 426 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:01:30.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:29 smithi016 bash[27237]: cluster 2023-09-23T19:01:28.657071+0000 mgr.y (mgr.14146) 196 : cluster [DBG] pgmap v177: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:30.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:29 smithi016 bash[27237]: audit 2023-09-23T19:01:28.662028+0000 mon.a (mon.0) 425 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:01:30.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:29 smithi016 bash[27237]: audit 2023-09-23T19:01:28.694845+0000 mon.a (mon.0) 426 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:01:30.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:29 smithi016 bash[33624]: cluster 2023-09-23T19:01:28.657071+0000 mgr.y (mgr.14146) 196 : cluster [DBG] pgmap v177: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:30.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:29 smithi016 bash[33624]: audit 2023-09-23T19:01:28.662028+0000 mon.a (mon.0) 425 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:01:30.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:29 smithi016 bash[33624]: audit 2023-09-23T19:01:28.694845+0000 mon.a (mon.0) 426 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:01:31.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:30 smithi016 bash[27237]: cluster 2023-09-23T19:01:30.657595+0000 mgr.y (mgr.14146) 197 : cluster [DBG] pgmap v178: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:31.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:30 smithi016 bash[33624]: cluster 2023-09-23T19:01:30.657595+0000 mgr.y (mgr.14146) 197 : cluster [DBG] pgmap v178: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:31.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:30 smithi092 bash[27556]: cluster 2023-09-23T19:01:30.657595+0000 mgr.y (mgr.14146) 197 : cluster [DBG] pgmap v178: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:34.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:33 smithi016 bash[27237]: cluster 2023-09-23T19:01:32.658154+0000 mgr.y (mgr.14146) 198 : cluster [DBG] pgmap v179: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:34.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:33 smithi016 bash[33624]: cluster 2023-09-23T19:01:32.658154+0000 mgr.y (mgr.14146) 198 : cluster [DBG] pgmap v179: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:34.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:33 smithi092 bash[27556]: cluster 2023-09-23T19:01:32.658154+0000 mgr.y (mgr.14146) 198 : cluster [DBG] pgmap v179: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:35.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:34 smithi092 bash[27556]: audit 2023-09-23T19:01:33.800884+0000 mon.a (mon.0) 427 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:01:35.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:34 smithi092 bash[27556]: audit 2023-09-23T19:01:33.802247+0000 mon.a (mon.0) 428 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:01:35.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:34 smithi092 bash[27556]: audit 2023-09-23T19:01:33.803584+0000 mon.a (mon.0) 429 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:01:35.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:34 smithi092 bash[27556]: audit 2023-09-23T19:01:33.810124+0000 mon.a (mon.0) 430 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:01:35.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:34 smithi016 bash[27237]: audit 2023-09-23T19:01:33.800884+0000 mon.a (mon.0) 427 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:01:35.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:34 smithi016 bash[27237]: audit 2023-09-23T19:01:33.802247+0000 mon.a (mon.0) 428 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:01:35.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:34 smithi016 bash[27237]: audit 2023-09-23T19:01:33.803584+0000 mon.a (mon.0) 429 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:01:35.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:34 smithi016 bash[27237]: audit 2023-09-23T19:01:33.810124+0000 mon.a (mon.0) 430 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:01:35.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:34 smithi016 bash[33624]: audit 2023-09-23T19:01:33.800884+0000 mon.a (mon.0) 427 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:01:35.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:34 smithi016 bash[33624]: audit 2023-09-23T19:01:33.802247+0000 mon.a (mon.0) 428 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:01:35.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:34 smithi016 bash[33624]: audit 2023-09-23T19:01:33.803584+0000 mon.a (mon.0) 429 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:01:35.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:34 smithi016 bash[33624]: audit 2023-09-23T19:01:33.810124+0000 mon.a (mon.0) 430 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:01:36.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:35 smithi016 bash[33624]: cluster 2023-09-23T19:01:34.658697+0000 mgr.y (mgr.14146) 199 : cluster [DBG] pgmap v180: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:36.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:35 smithi016 bash[33624]: audit 2023-09-23T19:01:35.367958+0000 mon.a (mon.0) 431 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T19:01:36.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:35 smithi016 bash[33624]: audit 2023-09-23T19:01:35.370507+0000 mon.b (mon.1) 13 : audit [INF] from='osd.5 [v2:172.21.15.92:6808/1987214502,v1:172.21.15.92:6809/1987214502]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T19:01:36.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:35 smithi016 bash[27237]: cluster 2023-09-23T19:01:34.658697+0000 mgr.y (mgr.14146) 199 : cluster [DBG] pgmap v180: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:36.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:35 smithi016 bash[27237]: audit 2023-09-23T19:01:35.367958+0000 mon.a (mon.0) 431 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T19:01:36.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:35 smithi016 bash[27237]: audit 2023-09-23T19:01:35.370507+0000 mon.b (mon.1) 13 : audit [INF] from='osd.5 [v2:172.21.15.92:6808/1987214502,v1:172.21.15.92:6809/1987214502]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T19:01:36.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:35 smithi092 bash[27556]: cluster 2023-09-23T19:01:34.658697+0000 mgr.y (mgr.14146) 199 : cluster [DBG] pgmap v180: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:36.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:35 smithi092 bash[27556]: audit 2023-09-23T19:01:35.367958+0000 mon.a (mon.0) 431 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T19:01:36.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:35 smithi092 bash[27556]: audit 2023-09-23T19:01:35.370507+0000 mon.b (mon.1) 13 : audit [INF] from='osd.5 [v2:172.21.15.92:6808/1987214502,v1:172.21.15.92:6809/1987214502]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T19:01:37.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:36 smithi092 bash[27556]: audit 2023-09-23T19:01:35.857658+0000 mon.a (mon.0) 432 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-09-23T19:01:37.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:36 smithi092 bash[27556]: cluster 2023-09-23T19:01:35.857848+0000 mon.a (mon.0) 433 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2023-09-23T19:01:37.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:36 smithi092 bash[27556]: audit 2023-09-23T19:01:35.858110+0000 mon.a (mon.0) 434 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:01:37.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:36 smithi092 bash[27556]: audit 2023-09-23T19:01:35.859371+0000 mon.a (mon.0) 435 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi092", "root=default"]}]: dispatch 2023-09-23T19:01:37.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:36 smithi092 bash[27556]: audit 2023-09-23T19:01:35.861968+0000 mon.b (mon.1) 14 : audit [INF] from='osd.5 [v2:172.21.15.92:6808/1987214502,v1:172.21.15.92:6809/1987214502]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi092", "root=default"]}]: dispatch 2023-09-23T19:01:37.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:36 smithi092 bash[27556]: cluster 2023-09-23T19:01:36.659223+0000 mgr.y (mgr.14146) 200 : cluster [DBG] pgmap v182: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:37.188 INFO:teuthology.orchestra.run.smithi092.stdout:Created osd(s) 5 on host 'smithi092' 2023-09-23T19:01:37.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:36 smithi016 bash[27237]: audit 2023-09-23T19:01:35.857658+0000 mon.a (mon.0) 432 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-09-23T19:01:37.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:36 smithi016 bash[27237]: cluster 2023-09-23T19:01:35.857848+0000 mon.a (mon.0) 433 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2023-09-23T19:01:37.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:36 smithi016 bash[27237]: audit 2023-09-23T19:01:35.858110+0000 mon.a (mon.0) 434 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:01:37.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:36 smithi016 bash[27237]: audit 2023-09-23T19:01:35.859371+0000 mon.a (mon.0) 435 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi092", "root=default"]}]: dispatch 2023-09-23T19:01:37.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:36 smithi016 bash[27237]: audit 2023-09-23T19:01:35.861968+0000 mon.b (mon.1) 14 : audit [INF] from='osd.5 [v2:172.21.15.92:6808/1987214502,v1:172.21.15.92:6809/1987214502]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi092", "root=default"]}]: dispatch 2023-09-23T19:01:37.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:36 smithi016 bash[27237]: cluster 2023-09-23T19:01:36.659223+0000 mgr.y (mgr.14146) 200 : cluster [DBG] pgmap v182: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:37.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:36 smithi016 bash[33624]: audit 2023-09-23T19:01:35.857658+0000 mon.a (mon.0) 432 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-09-23T19:01:37.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:36 smithi016 bash[33624]: cluster 2023-09-23T19:01:35.857848+0000 mon.a (mon.0) 433 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2023-09-23T19:01:37.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:36 smithi016 bash[33624]: audit 2023-09-23T19:01:35.858110+0000 mon.a (mon.0) 434 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:01:37.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:36 smithi016 bash[33624]: audit 2023-09-23T19:01:35.859371+0000 mon.a (mon.0) 435 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi092", "root=default"]}]: dispatch 2023-09-23T19:01:37.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:36 smithi016 bash[33624]: audit 2023-09-23T19:01:35.861968+0000 mon.b (mon.1) 14 : audit [INF] from='osd.5 [v2:172.21.15.92:6808/1987214502,v1:172.21.15.92:6809/1987214502]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi092", "root=default"]}]: dispatch 2023-09-23T19:01:37.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:36 smithi016 bash[33624]: cluster 2023-09-23T19:01:36.659223+0000 mgr.y (mgr.14146) 200 : cluster [DBG] pgmap v182: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:37.891 DEBUG:teuthology.orchestra.run.smithi092:osd.5> sudo journalctl -f -n 0 -u ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@osd.5.service 2023-09-23T19:01:37.894 INFO:tasks.cephadm:Deploying osd.6 on smithi092 with /dev/vg_nvme/lv_2... 2023-09-23T19:01:37.894 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-09-23T19:01:38.136 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:37 smithi092 bash[27556]: audit 2023-09-23T19:01:36.863847+0000 mon.a (mon.0) 436 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi092", "root=default"]}]': finished 2023-09-23T19:01:38.136 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:37 smithi092 bash[27556]: cluster 2023-09-23T19:01:36.863961+0000 mon.a (mon.0) 437 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2023-09-23T19:01:38.136 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:37 smithi092 bash[27556]: audit 2023-09-23T19:01:36.864150+0000 mon.a (mon.0) 438 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:01:38.136 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:37 smithi092 bash[27556]: audit 2023-09-23T19:01:36.871166+0000 mon.a (mon.0) 439 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:01:38.136 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:37 smithi092 bash[27556]: audit 2023-09-23T19:01:37.184999+0000 mon.a (mon.0) 440 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:01:38.136 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:37 smithi092 bash[27556]: audit 2023-09-23T19:01:37.187709+0000 mon.a (mon.0) 441 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:01:38.150 INFO:journalctl@ceph.osd.5.smithi092.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-09-23T19:01:38.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:37 smithi016 bash[27237]: audit 2023-09-23T19:01:36.863847+0000 mon.a (mon.0) 436 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi092", "root=default"]}]': finished 2023-09-23T19:01:38.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:37 smithi016 bash[27237]: cluster 2023-09-23T19:01:36.863961+0000 mon.a (mon.0) 437 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2023-09-23T19:01:38.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:37 smithi016 bash[27237]: audit 2023-09-23T19:01:36.864150+0000 mon.a (mon.0) 438 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:01:38.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:37 smithi016 bash[27237]: audit 2023-09-23T19:01:36.871166+0000 mon.a (mon.0) 439 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:01:38.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:37 smithi016 bash[27237]: audit 2023-09-23T19:01:37.184999+0000 mon.a (mon.0) 440 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:01:38.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:37 smithi016 bash[27237]: audit 2023-09-23T19:01:37.187709+0000 mon.a (mon.0) 441 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:01:38.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:37 smithi016 bash[33624]: audit 2023-09-23T19:01:36.863847+0000 mon.a (mon.0) 436 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi092", "root=default"]}]': finished 2023-09-23T19:01:38.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:37 smithi016 bash[33624]: cluster 2023-09-23T19:01:36.863961+0000 mon.a (mon.0) 437 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2023-09-23T19:01:38.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:37 smithi016 bash[33624]: audit 2023-09-23T19:01:36.864150+0000 mon.a (mon.0) 438 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:01:38.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:37 smithi016 bash[33624]: audit 2023-09-23T19:01:36.871166+0000 mon.a (mon.0) 439 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:01:38.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:37 smithi016 bash[33624]: audit 2023-09-23T19:01:37.184999+0000 mon.a (mon.0) 440 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:01:38.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:37 smithi016 bash[33624]: audit 2023-09-23T19:01:37.187709+0000 mon.a (mon.0) 441 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:01:39.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:38 smithi016 bash[27237]: cluster 2023-09-23T19:01:36.332866+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T19:01:39.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:38 smithi016 bash[27237]: cluster 2023-09-23T19:01:36.332952+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T19:01:39.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:38 smithi016 bash[27237]: cluster 2023-09-23T19:01:37.872822+0000 mon.a (mon.0) 442 : cluster [INF] osd.5 [v2:172.21.15.92:6808/1987214502,v1:172.21.15.92:6809/1987214502] boot 2023-09-23T19:01:39.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:38 smithi016 bash[27237]: cluster 2023-09-23T19:01:37.872884+0000 mon.a (mon.0) 443 : cluster [DBG] osdmap e36: 6 total, 6 up, 6 in 2023-09-23T19:01:39.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:38 smithi016 bash[27237]: audit 2023-09-23T19:01:37.873064+0000 mon.a (mon.0) 444 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:01:39.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:38 smithi016 bash[27237]: cluster 2023-09-23T19:01:38.659776+0000 mgr.y (mgr.14146) 201 : cluster [DBG] pgmap v185: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:39.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:38 smithi016 bash[33624]: cluster 2023-09-23T19:01:36.332866+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T19:01:39.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:38 smithi016 bash[33624]: cluster 2023-09-23T19:01:36.332952+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T19:01:39.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:38 smithi016 bash[33624]: cluster 2023-09-23T19:01:37.872822+0000 mon.a (mon.0) 442 : cluster [INF] osd.5 [v2:172.21.15.92:6808/1987214502,v1:172.21.15.92:6809/1987214502] boot 2023-09-23T19:01:39.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:38 smithi016 bash[33624]: cluster 2023-09-23T19:01:37.872884+0000 mon.a (mon.0) 443 : cluster [DBG] osdmap e36: 6 total, 6 up, 6 in 2023-09-23T19:01:39.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:38 smithi016 bash[33624]: audit 2023-09-23T19:01:37.873064+0000 mon.a (mon.0) 444 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:01:39.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:38 smithi016 bash[33624]: cluster 2023-09-23T19:01:38.659776+0000 mgr.y (mgr.14146) 201 : cluster [DBG] pgmap v185: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:39.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:38 smithi092 bash[27556]: cluster 2023-09-23T19:01:36.332866+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T19:01:39.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:38 smithi092 bash[27556]: cluster 2023-09-23T19:01:36.332952+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T19:01:39.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:38 smithi092 bash[27556]: cluster 2023-09-23T19:01:37.872822+0000 mon.a (mon.0) 442 : cluster [INF] osd.5 [v2:172.21.15.92:6808/1987214502,v1:172.21.15.92:6809/1987214502] boot 2023-09-23T19:01:39.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:38 smithi092 bash[27556]: cluster 2023-09-23T19:01:37.872884+0000 mon.a (mon.0) 443 : cluster [DBG] osdmap e36: 6 total, 6 up, 6 in 2023-09-23T19:01:39.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:38 smithi092 bash[27556]: audit 2023-09-23T19:01:37.873064+0000 mon.a (mon.0) 444 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:01:39.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:38 smithi092 bash[27556]: cluster 2023-09-23T19:01:38.659776+0000 mgr.y (mgr.14146) 201 : cluster [DBG] pgmap v185: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T19:01:40.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:39 smithi016 bash[33624]: cluster 2023-09-23T19:01:38.883475+0000 mon.a (mon.0) 445 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2023-09-23T19:01:40.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:39 smithi016 bash[27237]: cluster 2023-09-23T19:01:38.883475+0000 mon.a (mon.0) 445 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2023-09-23T19:01:40.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:39 smithi092 bash[27556]: cluster 2023-09-23T19:01:38.883475+0000 mon.a (mon.0) 445 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2023-09-23T19:01:41.229 INFO:teuthology.orchestra.run.smithi092.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-09-23T19:01:41.229 INFO:teuthology.orchestra.run.smithi092.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-09-23T19:01:41.230 INFO:teuthology.orchestra.run.smithi092.stderr: stderr: 10+0 records in 2023-09-23T19:01:41.230 INFO:teuthology.orchestra.run.smithi092.stderr:10+0 records out 2023-09-23T19:01:41.230 INFO:teuthology.orchestra.run.smithi092.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0147572 s, 711 MB/s 2023-09-23T19:01:41.230 INFO:teuthology.orchestra.run.smithi092.stderr:--> Zapping successful for: 2023-09-23T19:01:41.275 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:40 smithi092 bash[27556]: cluster 2023-09-23T19:01:39.885637+0000 mon.a (mon.0) 446 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2023-09-23T19:01:41.275 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:40 smithi092 bash[27556]: cluster 2023-09-23T19:01:40.660354+0000 mgr.y (mgr.14146) 202 : cluster [DBG] pgmap v188: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:01:41.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:40 smithi016 bash[27237]: cluster 2023-09-23T19:01:39.885637+0000 mon.a (mon.0) 446 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2023-09-23T19:01:41.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:40 smithi016 bash[27237]: cluster 2023-09-23T19:01:40.660354+0000 mgr.y (mgr.14146) 202 : cluster [DBG] pgmap v188: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:01:41.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:40 smithi016 bash[33624]: cluster 2023-09-23T19:01:39.885637+0000 mon.a (mon.0) 446 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2023-09-23T19:01:41.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:40 smithi016 bash[33624]: cluster 2023-09-23T19:01:40.660354+0000 mgr.y (mgr.14146) 202 : cluster [DBG] pgmap v188: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:01:41.928 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi092:vg_nvme/lv_2 2023-09-23T19:01:43.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:43 smithi092 bash[27556]: cluster 2023-09-23T19:01:42.660924+0000 mgr.y (mgr.14146) 203 : cluster [DBG] pgmap v189: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:01:44.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:43 smithi016 bash[27237]: cluster 2023-09-23T19:01:42.660924+0000 mgr.y (mgr.14146) 203 : cluster [DBG] pgmap v189: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:01:44.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:43 smithi016 bash[33624]: cluster 2023-09-23T19:01:42.660924+0000 mgr.y (mgr.14146) 203 : cluster [DBG] pgmap v189: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:01:46.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:45 smithi016 bash[27237]: cluster 2023-09-23T19:01:44.661523+0000 mgr.y (mgr.14146) 204 : cluster [DBG] pgmap v190: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:01:46.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:45 smithi016 bash[33624]: cluster 2023-09-23T19:01:44.661523+0000 mgr.y (mgr.14146) 204 : cluster [DBG] pgmap v190: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:01:46.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:45 smithi092 bash[27556]: cluster 2023-09-23T19:01:44.661523+0000 mgr.y (mgr.14146) 204 : cluster [DBG] pgmap v190: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:01:47.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:46 smithi016 bash[27237]: audit 2023-09-23T19:01:46.009710+0000 mon.a (mon.0) 447 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T19:01:47.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:46 smithi016 bash[27237]: audit 2023-09-23T19:01:46.015370+0000 mon.a (mon.0) 448 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T19:01:47.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:46 smithi016 bash[27237]: audit 2023-09-23T19:01:46.016803+0000 mon.a (mon.0) 449 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:01:47.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:46 smithi016 bash[33624]: audit 2023-09-23T19:01:46.009710+0000 mon.a (mon.0) 447 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T19:01:47.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:46 smithi016 bash[33624]: audit 2023-09-23T19:01:46.015370+0000 mon.a (mon.0) 448 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T19:01:47.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:46 smithi016 bash[33624]: audit 2023-09-23T19:01:46.016803+0000 mon.a (mon.0) 449 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:01:47.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:46 smithi092 bash[27556]: audit 2023-09-23T19:01:46.009710+0000 mon.a (mon.0) 447 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T19:01:47.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:46 smithi092 bash[27556]: audit 2023-09-23T19:01:46.015370+0000 mon.a (mon.0) 448 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T19:01:47.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:46 smithi092 bash[27556]: audit 2023-09-23T19:01:46.016803+0000 mon.a (mon.0) 449 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:01:48.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:47 smithi016 bash[27237]: audit 2023-09-23T19:01:46.003376+0000 mgr.y (mgr.14146) 205 : audit [DBG] from='client.24274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi092:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:01:48.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:47 smithi016 bash[27237]: cluster 2023-09-23T19:01:46.662063+0000 mgr.y (mgr.14146) 206 : cluster [DBG] pgmap v191: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:01:48.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:47 smithi016 bash[27237]: audit 2023-09-23T19:01:47.350058+0000 mon.a (mon.0) 450 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:01:48.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:47 smithi016 bash[27237]: audit 2023-09-23T19:01:47.351518+0000 mon.a (mon.0) 451 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:01:48.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:47 smithi016 bash[27237]: audit 2023-09-23T19:01:47.352840+0000 mon.a (mon.0) 452 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:01:48.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:47 smithi016 bash[27237]: audit 2023-09-23T19:01:47.359803+0000 mon.a (mon.0) 453 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:01:48.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:47 smithi016 bash[27237]: audit 2023-09-23T19:01:47.361454+0000 mon.a (mon.0) 454 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:01:48.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:47 smithi016 bash[27237]: audit 2023-09-23T19:01:47.362876+0000 mon.a (mon.0) 455 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:01:48.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:47 smithi016 bash[27237]: audit 2023-09-23T19:01:47.370056+0000 mon.a (mon.0) 456 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:01:48.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:47 smithi016 bash[33624]: audit 2023-09-23T19:01:46.003376+0000 mgr.y (mgr.14146) 205 : audit [DBG] from='client.24274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi092:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:01:48.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:47 smithi016 bash[33624]: cluster 2023-09-23T19:01:46.662063+0000 mgr.y (mgr.14146) 206 : cluster [DBG] pgmap v191: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:01:48.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:47 smithi016 bash[33624]: audit 2023-09-23T19:01:47.350058+0000 mon.a (mon.0) 450 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:01:48.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:47 smithi016 bash[33624]: audit 2023-09-23T19:01:47.351518+0000 mon.a (mon.0) 451 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:01:48.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:47 smithi016 bash[33624]: audit 2023-09-23T19:01:47.352840+0000 mon.a (mon.0) 452 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:01:48.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:47 smithi016 bash[33624]: audit 2023-09-23T19:01:47.359803+0000 mon.a (mon.0) 453 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:01:48.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:47 smithi016 bash[33624]: audit 2023-09-23T19:01:47.361454+0000 mon.a (mon.0) 454 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:01:48.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:47 smithi016 bash[33624]: audit 2023-09-23T19:01:47.362876+0000 mon.a (mon.0) 455 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:01:48.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:47 smithi016 bash[33624]: audit 2023-09-23T19:01:47.370056+0000 mon.a (mon.0) 456 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:01:48.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:47 smithi092 bash[27556]: audit 2023-09-23T19:01:46.003376+0000 mgr.y (mgr.14146) 205 : audit [DBG] from='client.24274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi092:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:01:48.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:47 smithi092 bash[27556]: cluster 2023-09-23T19:01:46.662063+0000 mgr.y (mgr.14146) 206 : cluster [DBG] pgmap v191: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:01:48.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:47 smithi092 bash[27556]: audit 2023-09-23T19:01:47.350058+0000 mon.a (mon.0) 450 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:01:48.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:47 smithi092 bash[27556]: audit 2023-09-23T19:01:47.351518+0000 mon.a (mon.0) 451 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:01:48.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:47 smithi092 bash[27556]: audit 2023-09-23T19:01:47.352840+0000 mon.a (mon.0) 452 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:01:48.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:47 smithi092 bash[27556]: audit 2023-09-23T19:01:47.359803+0000 mon.a (mon.0) 453 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:01:48.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:47 smithi092 bash[27556]: audit 2023-09-23T19:01:47.361454+0000 mon.a (mon.0) 454 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:01:48.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:47 smithi092 bash[27556]: audit 2023-09-23T19:01:47.362876+0000 mon.a (mon.0) 455 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:01:48.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:47 smithi092 bash[27556]: audit 2023-09-23T19:01:47.370056+0000 mon.a (mon.0) 456 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:01:49.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:48 smithi016 bash[27237]: cephadm 2023-09-23T19:01:47.338773+0000 mgr.y (mgr.14146) 207 : cephadm [INF] Detected new or changed devices on smithi092 2023-09-23T19:01:49.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:48 smithi016 bash[27237]: cephadm 2023-09-23T19:01:47.353818+0000 mgr.y (mgr.14146) 208 : cephadm [INF] Adjusting osd_memory_target on smithi092 to 8632M 2023-09-23T19:01:49.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:48 smithi016 bash[33624]: cephadm 2023-09-23T19:01:47.338773+0000 mgr.y (mgr.14146) 207 : cephadm [INF] Detected new or changed devices on smithi092 2023-09-23T19:01:49.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:48 smithi016 bash[33624]: cephadm 2023-09-23T19:01:47.353818+0000 mgr.y (mgr.14146) 208 : cephadm [INF] Adjusting osd_memory_target on smithi092 to 8632M 2023-09-23T19:01:49.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:48 smithi092 bash[27556]: cephadm 2023-09-23T19:01:47.338773+0000 mgr.y (mgr.14146) 207 : cephadm [INF] Detected new or changed devices on smithi092 2023-09-23T19:01:49.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:48 smithi092 bash[27556]: cephadm 2023-09-23T19:01:47.353818+0000 mgr.y (mgr.14146) 208 : cephadm [INF] Adjusting osd_memory_target on smithi092 to 8632M 2023-09-23T19:01:50.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:49 smithi016 bash[27237]: cluster 2023-09-23T19:01:48.662609+0000 mgr.y (mgr.14146) 209 : cluster [DBG] pgmap v192: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:01:50.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:49 smithi016 bash[33624]: cluster 2023-09-23T19:01:48.662609+0000 mgr.y (mgr.14146) 209 : cluster [DBG] pgmap v192: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:01:50.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:49 smithi092 bash[27556]: cluster 2023-09-23T19:01:48.662609+0000 mgr.y (mgr.14146) 209 : cluster [DBG] pgmap v192: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:01:51.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:50 smithi092 bash[27556]: cluster 2023-09-23T19:01:50.663176+0000 mgr.y (mgr.14146) 210 : cluster [DBG] pgmap v193: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:01:51.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:50 smithi016 bash[27237]: cluster 2023-09-23T19:01:50.663176+0000 mgr.y (mgr.14146) 210 : cluster [DBG] pgmap v193: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:01:51.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:50 smithi016 bash[33624]: cluster 2023-09-23T19:01:50.663176+0000 mgr.y (mgr.14146) 210 : cluster [DBG] pgmap v193: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:01:54.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:53 smithi016 bash[27237]: cluster 2023-09-23T19:01:52.663731+0000 mgr.y (mgr.14146) 211 : cluster [DBG] pgmap v194: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:01:54.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:53 smithi016 bash[33624]: cluster 2023-09-23T19:01:52.663731+0000 mgr.y (mgr.14146) 211 : cluster [DBG] pgmap v194: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:01:54.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:53 smithi092 bash[27556]: cluster 2023-09-23T19:01:52.663731+0000 mgr.y (mgr.14146) 211 : cluster [DBG] pgmap v194: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:01:55.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:54 smithi016 bash[27237]: audit 2023-09-23T19:01:54.300282+0000 mon.a (mon.0) 457 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9fe16cc9-f0d3-4b79-bc68-38305c55b551"}]: dispatch 2023-09-23T19:01:55.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:54 smithi016 bash[27237]: audit 2023-09-23T19:01:54.302867+0000 mon.b (mon.1) 15 : audit [INF] from='client.? 172.21.15.92:0/107340977' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9fe16cc9-f0d3-4b79-bc68-38305c55b551"}]: dispatch 2023-09-23T19:01:55.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:54 smithi016 bash[27237]: audit 2023-09-23T19:01:54.308894+0000 mon.a (mon.0) 458 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9fe16cc9-f0d3-4b79-bc68-38305c55b551"}]': finished 2023-09-23T19:01:55.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:54 smithi016 bash[27237]: cluster 2023-09-23T19:01:54.309000+0000 mon.a (mon.0) 459 : cluster [DBG] osdmap e39: 7 total, 6 up, 7 in 2023-09-23T19:01:55.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:54 smithi016 bash[27237]: audit 2023-09-23T19:01:54.309306+0000 mon.a (mon.0) 460 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:01:55.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:54 smithi016 bash[33624]: audit 2023-09-23T19:01:54.300282+0000 mon.a (mon.0) 457 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9fe16cc9-f0d3-4b79-bc68-38305c55b551"}]: dispatch 2023-09-23T19:01:55.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:54 smithi016 bash[33624]: audit 2023-09-23T19:01:54.302867+0000 mon.b (mon.1) 15 : audit [INF] from='client.? 172.21.15.92:0/107340977' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9fe16cc9-f0d3-4b79-bc68-38305c55b551"}]: dispatch 2023-09-23T19:01:55.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:54 smithi016 bash[33624]: audit 2023-09-23T19:01:54.308894+0000 mon.a (mon.0) 458 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9fe16cc9-f0d3-4b79-bc68-38305c55b551"}]': finished 2023-09-23T19:01:55.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:54 smithi016 bash[33624]: cluster 2023-09-23T19:01:54.309000+0000 mon.a (mon.0) 459 : cluster [DBG] osdmap e39: 7 total, 6 up, 7 in 2023-09-23T19:01:55.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:54 smithi016 bash[33624]: audit 2023-09-23T19:01:54.309306+0000 mon.a (mon.0) 460 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:01:55.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:54 smithi092 bash[27556]: audit 2023-09-23T19:01:54.300282+0000 mon.a (mon.0) 457 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9fe16cc9-f0d3-4b79-bc68-38305c55b551"}]: dispatch 2023-09-23T19:01:55.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:54 smithi092 bash[27556]: audit 2023-09-23T19:01:54.302867+0000 mon.b (mon.1) 15 : audit [INF] from='client.? 172.21.15.92:0/107340977' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9fe16cc9-f0d3-4b79-bc68-38305c55b551"}]: dispatch 2023-09-23T19:01:55.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:54 smithi092 bash[27556]: audit 2023-09-23T19:01:54.308894+0000 mon.a (mon.0) 458 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9fe16cc9-f0d3-4b79-bc68-38305c55b551"}]': finished 2023-09-23T19:01:55.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:54 smithi092 bash[27556]: cluster 2023-09-23T19:01:54.309000+0000 mon.a (mon.0) 459 : cluster [DBG] osdmap e39: 7 total, 6 up, 7 in 2023-09-23T19:01:55.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:54 smithi092 bash[27556]: audit 2023-09-23T19:01:54.309306+0000 mon.a (mon.0) 460 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:01:56.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:55 smithi016 bash[33624]: cluster 2023-09-23T19:01:54.664274+0000 mgr.y (mgr.14146) 212 : cluster [DBG] pgmap v196: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:01:56.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:55 smithi016 bash[33624]: audit 2023-09-23T19:01:55.015683+0000 mon.b (mon.1) 16 : audit [DBG] from='client.? 172.21.15.92:0/3318986725' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T19:01:56.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:55 smithi016 bash[27237]: cluster 2023-09-23T19:01:54.664274+0000 mgr.y (mgr.14146) 212 : cluster [DBG] pgmap v196: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:01:56.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:55 smithi016 bash[27237]: audit 2023-09-23T19:01:55.015683+0000 mon.b (mon.1) 16 : audit [DBG] from='client.? 172.21.15.92:0/3318986725' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T19:01:56.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:55 smithi092 bash[27556]: cluster 2023-09-23T19:01:54.664274+0000 mgr.y (mgr.14146) 212 : cluster [DBG] pgmap v196: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:01:56.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:55 smithi092 bash[27556]: audit 2023-09-23T19:01:55.015683+0000 mon.b (mon.1) 16 : audit [DBG] from='client.? 172.21.15.92:0/3318986725' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T19:01:57.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:57 smithi092 bash[27556]: cluster 2023-09-23T19:01:56.664841+0000 mgr.y (mgr.14146) 213 : cluster [DBG] pgmap v197: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:01:58.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:57 smithi016 bash[27237]: cluster 2023-09-23T19:01:56.664841+0000 mgr.y (mgr.14146) 213 : cluster [DBG] pgmap v197: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:01:58.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:57 smithi016 bash[33624]: cluster 2023-09-23T19:01:56.664841+0000 mgr.y (mgr.14146) 213 : cluster [DBG] pgmap v197: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:02:00.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:01:59 smithi016 bash[27237]: cluster 2023-09-23T19:01:58.665397+0000 mgr.y (mgr.14146) 214 : cluster [DBG] pgmap v198: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:02:00.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:01:59 smithi016 bash[33624]: cluster 2023-09-23T19:01:58.665397+0000 mgr.y (mgr.14146) 214 : cluster [DBG] pgmap v198: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:02:00.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:01:59 smithi092 bash[27556]: cluster 2023-09-23T19:01:58.665397+0000 mgr.y (mgr.14146) 214 : cluster [DBG] pgmap v198: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:02:01.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:00 smithi016 bash[27237]: cluster 2023-09-23T19:02:00.665997+0000 mgr.y (mgr.14146) 215 : cluster [DBG] pgmap v199: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:02:01.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:00 smithi016 bash[33624]: cluster 2023-09-23T19:02:00.665997+0000 mgr.y (mgr.14146) 215 : cluster [DBG] pgmap v199: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:02:01.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:00 smithi092 bash[27556]: cluster 2023-09-23T19:02:00.665997+0000 mgr.y (mgr.14146) 215 : cluster [DBG] pgmap v199: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:02:04.033 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:03 smithi092 bash[27556]: cluster 2023-09-23T19:02:02.666546+0000 mgr.y (mgr.14146) 216 : cluster [DBG] pgmap v200: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:02:04.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:03 smithi016 bash[27237]: cluster 2023-09-23T19:02:02.666546+0000 mgr.y (mgr.14146) 216 : cluster [DBG] pgmap v200: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:02:04.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:03 smithi016 bash[33624]: cluster 2023-09-23T19:02:02.666546+0000 mgr.y (mgr.14146) 216 : cluster [DBG] pgmap v200: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:02:06.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:05 smithi016 bash[27237]: cluster 2023-09-23T19:02:04.667115+0000 mgr.y (mgr.14146) 217 : cluster [DBG] pgmap v201: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:02:06.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:05 smithi016 bash[33624]: cluster 2023-09-23T19:02:04.667115+0000 mgr.y (mgr.14146) 217 : cluster [DBG] pgmap v201: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:02:06.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:05 smithi092 bash[27556]: cluster 2023-09-23T19:02:04.667115+0000 mgr.y (mgr.14146) 217 : cluster [DBG] pgmap v201: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:02:08.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:07 smithi016 bash[27237]: cluster 2023-09-23T19:02:06.667639+0000 mgr.y (mgr.14146) 218 : cluster [DBG] pgmap v202: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:02:08.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:07 smithi016 bash[27237]: audit 2023-09-23T19:02:06.735627+0000 mon.a (mon.0) 461 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T19:02:08.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:07 smithi016 bash[27237]: audit 2023-09-23T19:02:06.737172+0000 mon.a (mon.0) 462 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:02:08.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:07 smithi016 bash[27237]: cephadm 2023-09-23T19:02:06.738256+0000 mgr.y (mgr.14146) 219 : cephadm [INF] Deploying daemon osd.6 on smithi092 2023-09-23T19:02:08.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:07 smithi016 bash[33624]: cluster 2023-09-23T19:02:06.667639+0000 mgr.y (mgr.14146) 218 : cluster [DBG] pgmap v202: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:02:08.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:07 smithi016 bash[33624]: audit 2023-09-23T19:02:06.735627+0000 mon.a (mon.0) 461 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T19:02:08.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:07 smithi016 bash[33624]: audit 2023-09-23T19:02:06.737172+0000 mon.a (mon.0) 462 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:02:08.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:07 smithi016 bash[33624]: cephadm 2023-09-23T19:02:06.738256+0000 mgr.y (mgr.14146) 219 : cephadm [INF] Deploying daemon osd.6 on smithi092 2023-09-23T19:02:08.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:07 smithi092 bash[27556]: cluster 2023-09-23T19:02:06.667639+0000 mgr.y (mgr.14146) 218 : cluster [DBG] pgmap v202: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:02:08.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:07 smithi092 bash[27556]: audit 2023-09-23T19:02:06.735627+0000 mon.a (mon.0) 461 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T19:02:08.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:07 smithi092 bash[27556]: audit 2023-09-23T19:02:06.737172+0000 mon.a (mon.0) 462 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:02:08.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:07 smithi092 bash[27556]: cephadm 2023-09-23T19:02:06.738256+0000 mgr.y (mgr.14146) 219 : cephadm [INF] Deploying daemon osd.6 on smithi092 2023-09-23T19:02:10.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:09 smithi016 bash[27237]: cluster 2023-09-23T19:02:08.668199+0000 mgr.y (mgr.14146) 220 : cluster [DBG] pgmap v203: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:02:10.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:09 smithi016 bash[33624]: cluster 2023-09-23T19:02:08.668199+0000 mgr.y (mgr.14146) 220 : cluster [DBG] pgmap v203: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:02:10.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:09 smithi092 bash[27556]: cluster 2023-09-23T19:02:08.668199+0000 mgr.y (mgr.14146) 220 : cluster [DBG] pgmap v203: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:02:11.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:10 smithi016 bash[27237]: cluster 2023-09-23T19:02:10.668796+0000 mgr.y (mgr.14146) 221 : cluster [DBG] pgmap v204: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:02:11.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:10 smithi016 bash[33624]: cluster 2023-09-23T19:02:10.668796+0000 mgr.y (mgr.14146) 221 : cluster [DBG] pgmap v204: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:02:11.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:10 smithi092 bash[27556]: cluster 2023-09-23T19:02:10.668796+0000 mgr.y (mgr.14146) 221 : cluster [DBG] pgmap v204: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:02:12.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:11 smithi016 bash[33624]: audit 2023-09-23T19:02:10.939818+0000 mon.a (mon.0) 463 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:02:12.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:11 smithi016 bash[33624]: audit 2023-09-23T19:02:10.956140+0000 mon.a (mon.0) 464 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:02:12.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:11 smithi016 bash[27237]: audit 2023-09-23T19:02:10.939818+0000 mon.a (mon.0) 463 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:02:12.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:11 smithi016 bash[27237]: audit 2023-09-23T19:02:10.956140+0000 mon.a (mon.0) 464 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:02:12.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:11 smithi092 bash[27556]: audit 2023-09-23T19:02:10.939818+0000 mon.a (mon.0) 463 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:02:12.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:11 smithi092 bash[27556]: audit 2023-09-23T19:02:10.956140+0000 mon.a (mon.0) 464 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:02:13.200 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:12 smithi092 bash[27556]: cluster 2023-09-23T19:02:12.669345+0000 mgr.y (mgr.14146) 222 : cluster [DBG] pgmap v205: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:02:13.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:12 smithi016 bash[27237]: cluster 2023-09-23T19:02:12.669345+0000 mgr.y (mgr.14146) 222 : cluster [DBG] pgmap v205: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:02:13.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:12 smithi016 bash[33624]: cluster 2023-09-23T19:02:12.669345+0000 mgr.y (mgr.14146) 222 : cluster [DBG] pgmap v205: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:02:16.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:15 smithi016 bash[27237]: cluster 2023-09-23T19:02:14.669904+0000 mgr.y (mgr.14146) 223 : cluster [DBG] pgmap v206: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:02:16.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:15 smithi016 bash[33624]: cluster 2023-09-23T19:02:14.669904+0000 mgr.y (mgr.14146) 223 : cluster [DBG] pgmap v206: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:02:16.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:15 smithi092 bash[27556]: cluster 2023-09-23T19:02:14.669904+0000 mgr.y (mgr.14146) 223 : cluster [DBG] pgmap v206: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:02:17.503 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:17 smithi092 bash[27556]: audit 2023-09-23T19:02:16.108645+0000 mon.a (mon.0) 465 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:02:17.503 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:17 smithi092 bash[27556]: audit 2023-09-23T19:02:16.387576+0000 mon.a (mon.0) 466 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:02:17.504 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:17 smithi092 bash[27556]: audit 2023-09-23T19:02:16.389563+0000 mon.a (mon.0) 467 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:02:17.504 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:17 smithi092 bash[27556]: audit 2023-09-23T19:02:16.399418+0000 mon.a (mon.0) 468 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:02:17.504 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:17 smithi092 bash[27556]: cluster 2023-09-23T19:02:16.670426+0000 mgr.y (mgr.14146) 224 : cluster [DBG] pgmap v207: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:02:17.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:17 smithi016 bash[27237]: audit 2023-09-23T19:02:16.108645+0000 mon.a (mon.0) 465 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:02:17.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:17 smithi016 bash[27237]: audit 2023-09-23T19:02:16.387576+0000 mon.a (mon.0) 466 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:02:17.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:17 smithi016 bash[27237]: audit 2023-09-23T19:02:16.389563+0000 mon.a (mon.0) 467 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:02:17.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:17 smithi016 bash[27237]: audit 2023-09-23T19:02:16.399418+0000 mon.a (mon.0) 468 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:02:17.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:17 smithi016 bash[27237]: cluster 2023-09-23T19:02:16.670426+0000 mgr.y (mgr.14146) 224 : cluster [DBG] pgmap v207: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:02:17.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:17 smithi016 bash[33624]: audit 2023-09-23T19:02:16.108645+0000 mon.a (mon.0) 465 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:02:17.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:17 smithi016 bash[33624]: audit 2023-09-23T19:02:16.387576+0000 mon.a (mon.0) 466 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:02:17.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:17 smithi016 bash[33624]: audit 2023-09-23T19:02:16.389563+0000 mon.a (mon.0) 467 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:02:17.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:17 smithi016 bash[33624]: audit 2023-09-23T19:02:16.399418+0000 mon.a (mon.0) 468 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:02:17.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:17 smithi016 bash[33624]: cluster 2023-09-23T19:02:16.670426+0000 mgr.y (mgr.14146) 224 : cluster [DBG] pgmap v207: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:02:18.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:18 smithi016 bash[27237]: audit 2023-09-23T19:02:17.589280+0000 mon.a (mon.0) 469 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T19:02:18.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:18 smithi016 bash[27237]: audit 2023-09-23T19:02:17.592226+0000 mon.b (mon.1) 17 : audit [INF] from='osd.6 [v2:172.21.15.92:6816/2820798178,v1:172.21.15.92:6817/2820798178]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T19:02:18.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:18 smithi016 bash[33624]: audit 2023-09-23T19:02:17.589280+0000 mon.a (mon.0) 469 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T19:02:18.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:18 smithi016 bash[33624]: audit 2023-09-23T19:02:17.592226+0000 mon.b (mon.1) 17 : audit [INF] from='osd.6 [v2:172.21.15.92:6816/2820798178,v1:172.21.15.92:6817/2820798178]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T19:02:18.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:18 smithi092 bash[27556]: audit 2023-09-23T19:02:17.589280+0000 mon.a (mon.0) 469 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T19:02:18.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:18 smithi092 bash[27556]: audit 2023-09-23T19:02:17.592226+0000 mon.b (mon.1) 17 : audit [INF] from='osd.6 [v2:172.21.15.92:6816/2820798178,v1:172.21.15.92:6817/2820798178]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T19:02:19.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:19 smithi092 bash[27556]: audit 2023-09-23T19:02:18.403825+0000 mon.a (mon.0) 470 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-09-23T19:02:19.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:19 smithi092 bash[27556]: cluster 2023-09-23T19:02:18.403934+0000 mon.a (mon.0) 471 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2023-09-23T19:02:19.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:19 smithi092 bash[27556]: audit 2023-09-23T19:02:18.404578+0000 mon.a (mon.0) 472 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:02:19.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:19 smithi092 bash[27556]: audit 2023-09-23T19:02:18.409084+0000 mon.b (mon.1) 18 : audit [INF] from='osd.6 [v2:172.21.15.92:6816/2820798178,v1:172.21.15.92:6817/2820798178]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi092", "root=default"]}]: dispatch 2023-09-23T19:02:19.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:19 smithi092 bash[27556]: audit 2023-09-23T19:02:18.410735+0000 mon.a (mon.0) 473 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi092", "root=default"]}]: dispatch 2023-09-23T19:02:19.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:19 smithi092 bash[27556]: cluster 2023-09-23T19:02:18.671008+0000 mgr.y (mgr.14146) 225 : cluster [DBG] pgmap v209: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:02:19.747 INFO:teuthology.orchestra.run.smithi092.stdout:Created osd(s) 6 on host 'smithi092' 2023-09-23T19:02:19.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:19 smithi016 bash[27237]: audit 2023-09-23T19:02:18.403825+0000 mon.a (mon.0) 470 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-09-23T19:02:19.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:19 smithi016 bash[27237]: cluster 2023-09-23T19:02:18.403934+0000 mon.a (mon.0) 471 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2023-09-23T19:02:19.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:19 smithi016 bash[27237]: audit 2023-09-23T19:02:18.404578+0000 mon.a (mon.0) 472 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:02:19.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:19 smithi016 bash[27237]: audit 2023-09-23T19:02:18.409084+0000 mon.b (mon.1) 18 : audit [INF] from='osd.6 [v2:172.21.15.92:6816/2820798178,v1:172.21.15.92:6817/2820798178]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi092", "root=default"]}]: dispatch 2023-09-23T19:02:19.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:19 smithi016 bash[27237]: audit 2023-09-23T19:02:18.410735+0000 mon.a (mon.0) 473 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi092", "root=default"]}]: dispatch 2023-09-23T19:02:19.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:19 smithi016 bash[27237]: cluster 2023-09-23T19:02:18.671008+0000 mgr.y (mgr.14146) 225 : cluster [DBG] pgmap v209: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:02:19.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:19 smithi016 bash[33624]: audit 2023-09-23T19:02:18.403825+0000 mon.a (mon.0) 470 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-09-23T19:02:19.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:19 smithi016 bash[33624]: cluster 2023-09-23T19:02:18.403934+0000 mon.a (mon.0) 471 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2023-09-23T19:02:19.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:19 smithi016 bash[33624]: audit 2023-09-23T19:02:18.404578+0000 mon.a (mon.0) 472 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:02:19.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:19 smithi016 bash[33624]: audit 2023-09-23T19:02:18.409084+0000 mon.b (mon.1) 18 : audit [INF] from='osd.6 [v2:172.21.15.92:6816/2820798178,v1:172.21.15.92:6817/2820798178]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi092", "root=default"]}]: dispatch 2023-09-23T19:02:19.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:19 smithi016 bash[33624]: audit 2023-09-23T19:02:18.410735+0000 mon.a (mon.0) 473 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi092", "root=default"]}]: dispatch 2023-09-23T19:02:19.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:19 smithi016 bash[33624]: cluster 2023-09-23T19:02:18.671008+0000 mgr.y (mgr.14146) 225 : cluster [DBG] pgmap v209: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T19:02:20.412 DEBUG:teuthology.orchestra.run.smithi092:osd.6> sudo journalctl -f -n 0 -u ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@osd.6.service 2023-09-23T19:02:20.414 INFO:tasks.cephadm:Deploying osd.7 on smithi092 with /dev/vg_nvme/lv_1... 2023-09-23T19:02:20.415 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-09-23T19:02:20.453 INFO:journalctl@ceph.osd.6.smithi092.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-09-23T19:02:20.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:20 smithi016 bash[27237]: audit 2023-09-23T19:02:19.413828+0000 mon.a (mon.0) 474 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi092", "root=default"]}]': finished 2023-09-23T19:02:20.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:20 smithi016 bash[27237]: cluster 2023-09-23T19:02:19.413989+0000 mon.a (mon.0) 475 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2023-09-23T19:02:20.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:20 smithi016 bash[27237]: audit 2023-09-23T19:02:19.414999+0000 mon.a (mon.0) 476 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:02:20.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:20 smithi016 bash[27237]: audit 2023-09-23T19:02:19.421926+0000 mon.a (mon.0) 477 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:02:20.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:20 smithi016 bash[27237]: audit 2023-09-23T19:02:19.743489+0000 mon.a (mon.0) 478 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:02:20.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:20 smithi016 bash[27237]: audit 2023-09-23T19:02:19.777083+0000 mon.a (mon.0) 479 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:02:20.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:20 smithi016 bash[27237]: cluster 2023-09-23T19:02:19.867290+0000 mon.a (mon.0) 480 : cluster [INF] osd.6 [v2:172.21.15.92:6816/2820798178,v1:172.21.15.92:6817/2820798178] boot 2023-09-23T19:02:20.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:20 smithi016 bash[27237]: cluster 2023-09-23T19:02:19.867387+0000 mon.a (mon.0) 481 : cluster [DBG] osdmap e42: 7 total, 7 up, 7 in 2023-09-23T19:02:20.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:20 smithi016 bash[27237]: audit 2023-09-23T19:02:19.867881+0000 mon.a (mon.0) 482 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:02:20.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:20 smithi016 bash[33624]: audit 2023-09-23T19:02:19.413828+0000 mon.a (mon.0) 474 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi092", "root=default"]}]': finished 2023-09-23T19:02:20.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:20 smithi016 bash[33624]: cluster 2023-09-23T19:02:19.413989+0000 mon.a (mon.0) 475 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2023-09-23T19:02:20.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:20 smithi016 bash[33624]: audit 2023-09-23T19:02:19.414999+0000 mon.a (mon.0) 476 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:02:20.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:20 smithi016 bash[33624]: audit 2023-09-23T19:02:19.421926+0000 mon.a (mon.0) 477 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:02:20.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:20 smithi016 bash[33624]: audit 2023-09-23T19:02:19.743489+0000 mon.a (mon.0) 478 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:02:20.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:20 smithi016 bash[33624]: audit 2023-09-23T19:02:19.777083+0000 mon.a (mon.0) 479 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:02:20.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:20 smithi016 bash[33624]: cluster 2023-09-23T19:02:19.867290+0000 mon.a (mon.0) 480 : cluster [INF] osd.6 [v2:172.21.15.92:6816/2820798178,v1:172.21.15.92:6817/2820798178] boot 2023-09-23T19:02:20.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:20 smithi016 bash[33624]: cluster 2023-09-23T19:02:19.867387+0000 mon.a (mon.0) 481 : cluster [DBG] osdmap e42: 7 total, 7 up, 7 in 2023-09-23T19:02:20.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:20 smithi016 bash[33624]: audit 2023-09-23T19:02:19.867881+0000 mon.a (mon.0) 482 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:02:20.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:20 smithi092 bash[27556]: audit 2023-09-23T19:02:19.413828+0000 mon.a (mon.0) 474 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi092", "root=default"]}]': finished 2023-09-23T19:02:20.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:20 smithi092 bash[27556]: cluster 2023-09-23T19:02:19.413989+0000 mon.a (mon.0) 475 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2023-09-23T19:02:20.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:20 smithi092 bash[27556]: audit 2023-09-23T19:02:19.414999+0000 mon.a (mon.0) 476 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:02:20.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:20 smithi092 bash[27556]: audit 2023-09-23T19:02:19.421926+0000 mon.a (mon.0) 477 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:02:20.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:20 smithi092 bash[27556]: audit 2023-09-23T19:02:19.743489+0000 mon.a (mon.0) 478 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:02:20.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:20 smithi092 bash[27556]: audit 2023-09-23T19:02:19.777083+0000 mon.a (mon.0) 479 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:02:20.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:20 smithi092 bash[27556]: cluster 2023-09-23T19:02:19.867290+0000 mon.a (mon.0) 480 : cluster [INF] osd.6 [v2:172.21.15.92:6816/2820798178,v1:172.21.15.92:6817/2820798178] boot 2023-09-23T19:02:20.790 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:20 smithi092 bash[27556]: cluster 2023-09-23T19:02:19.867387+0000 mon.a (mon.0) 481 : cluster [DBG] osdmap e42: 7 total, 7 up, 7 in 2023-09-23T19:02:20.790 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:20 smithi092 bash[27556]: audit 2023-09-23T19:02:19.867881+0000 mon.a (mon.0) 482 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:02:21.712 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:21 smithi092 bash[27556]: cluster 2023-09-23T19:02:18.593767+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T19:02:21.712 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:21 smithi092 bash[27556]: cluster 2023-09-23T19:02:18.593849+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T19:02:21.712 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:21 smithi092 bash[27556]: cluster 2023-09-23T19:02:20.671630+0000 mgr.y (mgr.14146) 226 : cluster [DBG] pgmap v212: 1 pgs: 1 peering; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:21.712 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:21 smithi092 bash[27556]: cluster 2023-09-23T19:02:20.876539+0000 mon.a (mon.0) 483 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2023-09-23T19:02:21.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:21 smithi016 bash[27237]: cluster 2023-09-23T19:02:18.593767+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T19:02:21.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:21 smithi016 bash[27237]: cluster 2023-09-23T19:02:18.593849+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T19:02:21.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:21 smithi016 bash[27237]: cluster 2023-09-23T19:02:20.671630+0000 mgr.y (mgr.14146) 226 : cluster [DBG] pgmap v212: 1 pgs: 1 peering; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:21.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:21 smithi016 bash[27237]: cluster 2023-09-23T19:02:20.876539+0000 mon.a (mon.0) 483 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2023-09-23T19:02:21.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:21 smithi016 bash[33624]: cluster 2023-09-23T19:02:18.593767+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T19:02:21.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:21 smithi016 bash[33624]: cluster 2023-09-23T19:02:18.593849+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T19:02:21.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:21 smithi016 bash[33624]: cluster 2023-09-23T19:02:20.671630+0000 mgr.y (mgr.14146) 226 : cluster [DBG] pgmap v212: 1 pgs: 1 peering; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:21.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:21 smithi016 bash[33624]: cluster 2023-09-23T19:02:20.876539+0000 mon.a (mon.0) 483 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2023-09-23T19:02:22.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:22 smithi092 bash[27556]: cluster 2023-09-23T19:02:21.420440+0000 mon.a (mon.0) 484 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T19:02:22.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:22 smithi092 bash[27556]: cluster 2023-09-23T19:02:21.878866+0000 mon.a (mon.0) 485 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2023-09-23T19:02:22.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:22 smithi016 bash[33624]: cluster 2023-09-23T19:02:21.420440+0000 mon.a (mon.0) 484 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T19:02:22.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:22 smithi016 bash[33624]: cluster 2023-09-23T19:02:21.878866+0000 mon.a (mon.0) 485 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2023-09-23T19:02:22.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:22 smithi016 bash[27237]: cluster 2023-09-23T19:02:21.420440+0000 mon.a (mon.0) 484 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T19:02:22.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:22 smithi016 bash[27237]: cluster 2023-09-23T19:02:21.878866+0000 mon.a (mon.0) 485 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2023-09-23T19:02:23.673 INFO:teuthology.orchestra.run.smithi092.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-09-23T19:02:23.673 INFO:teuthology.orchestra.run.smithi092.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-09-23T19:02:23.673 INFO:teuthology.orchestra.run.smithi092.stderr: stderr: 10+0 records in 2023-09-23T19:02:23.673 INFO:teuthology.orchestra.run.smithi092.stderr:10+0 records out 2023-09-23T19:02:23.673 INFO:teuthology.orchestra.run.smithi092.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0151623 s, 692 MB/s 2023-09-23T19:02:23.673 INFO:teuthology.orchestra.run.smithi092.stderr:--> Zapping successful for: 2023-09-23T19:02:23.737 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:23 smithi092 bash[27556]: cluster 2023-09-23T19:02:22.672426+0000 mgr.y (mgr.14146) 227 : cluster [DBG] pgmap v215: 1 pgs: 1 peering; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:23.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:23 smithi016 bash[33624]: cluster 2023-09-23T19:02:22.672426+0000 mgr.y (mgr.14146) 227 : cluster [DBG] pgmap v215: 1 pgs: 1 peering; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:23.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:23 smithi016 bash[27237]: cluster 2023-09-23T19:02:22.672426+0000 mgr.y (mgr.14146) 227 : cluster [DBG] pgmap v215: 1 pgs: 1 peering; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:24.387 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi092:vg_nvme/lv_1 2023-09-23T19:02:26.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:25 smithi016 bash[27237]: cluster 2023-09-23T19:02:24.673063+0000 mgr.y (mgr.14146) 228 : cluster [DBG] pgmap v216: 1 pgs: 1 peering; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:26.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:25 smithi016 bash[33624]: cluster 2023-09-23T19:02:24.673063+0000 mgr.y (mgr.14146) 228 : cluster [DBG] pgmap v216: 1 pgs: 1 peering; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:26.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:25 smithi092 bash[27556]: cluster 2023-09-23T19:02:24.673063+0000 mgr.y (mgr.14146) 228 : cluster [DBG] pgmap v216: 1 pgs: 1 peering; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:27.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:26 smithi016 bash[27237]: cluster 2023-09-23T19:02:26.728743+0000 mon.a (mon.0) 486 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T19:02:27.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:26 smithi016 bash[27237]: cluster 2023-09-23T19:02:26.728808+0000 mon.a (mon.0) 487 : cluster [INF] Cluster is now healthy 2023-09-23T19:02:27.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:26 smithi016 bash[33624]: cluster 2023-09-23T19:02:26.728743+0000 mon.a (mon.0) 486 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T19:02:27.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:26 smithi016 bash[33624]: cluster 2023-09-23T19:02:26.728808+0000 mon.a (mon.0) 487 : cluster [INF] Cluster is now healthy 2023-09-23T19:02:27.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:26 smithi092 bash[27556]: cluster 2023-09-23T19:02:26.728743+0000 mon.a (mon.0) 486 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T19:02:27.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:26 smithi092 bash[27556]: cluster 2023-09-23T19:02:26.728808+0000 mon.a (mon.0) 487 : cluster [INF] Cluster is now healthy 2023-09-23T19:02:28.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:27 smithi092 bash[27556]: cluster 2023-09-23T19:02:26.673626+0000 mgr.y (mgr.14146) 229 : cluster [DBG] pgmap v217: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:28.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:27 smithi016 bash[33624]: cluster 2023-09-23T19:02:26.673626+0000 mgr.y (mgr.14146) 229 : cluster [DBG] pgmap v217: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:28.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:27 smithi016 bash[27237]: cluster 2023-09-23T19:02:26.673626+0000 mgr.y (mgr.14146) 229 : cluster [DBG] pgmap v217: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:29.134 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:28 smithi092 bash[27556]: audit 2023-09-23T19:02:28.593681+0000 mon.a (mon.0) 488 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T19:02:29.135 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:28 smithi092 bash[27556]: audit 2023-09-23T19:02:28.599229+0000 mon.a (mon.0) 489 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T19:02:29.135 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:28 smithi092 bash[27556]: audit 2023-09-23T19:02:28.600741+0000 mon.a (mon.0) 490 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:02:29.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:28 smithi016 bash[27237]: audit 2023-09-23T19:02:28.593681+0000 mon.a (mon.0) 488 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T19:02:29.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:28 smithi016 bash[27237]: audit 2023-09-23T19:02:28.599229+0000 mon.a (mon.0) 489 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T19:02:29.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:28 smithi016 bash[27237]: audit 2023-09-23T19:02:28.600741+0000 mon.a (mon.0) 490 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:02:29.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:28 smithi016 bash[33624]: audit 2023-09-23T19:02:28.593681+0000 mon.a (mon.0) 488 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T19:02:29.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:28 smithi016 bash[33624]: audit 2023-09-23T19:02:28.599229+0000 mon.a (mon.0) 489 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T19:02:29.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:28 smithi016 bash[33624]: audit 2023-09-23T19:02:28.600741+0000 mon.a (mon.0) 490 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:02:30.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:29 smithi016 bash[33624]: audit 2023-09-23T19:02:28.587102+0000 mgr.y (mgr.14146) 230 : audit [DBG] from='client.24301 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi092:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:02:30.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:29 smithi016 bash[33624]: cluster 2023-09-23T19:02:28.674152+0000 mgr.y (mgr.14146) 231 : cluster [DBG] pgmap v218: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:30.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:29 smithi016 bash[27237]: audit 2023-09-23T19:02:28.587102+0000 mgr.y (mgr.14146) 230 : audit [DBG] from='client.24301 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi092:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:02:30.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:29 smithi016 bash[27237]: cluster 2023-09-23T19:02:28.674152+0000 mgr.y (mgr.14146) 231 : cluster [DBG] pgmap v218: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:30.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:29 smithi092 bash[27556]: audit 2023-09-23T19:02:28.587102+0000 mgr.y (mgr.14146) 230 : audit [DBG] from='client.24301 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi092:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:02:30.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:29 smithi092 bash[27556]: cluster 2023-09-23T19:02:28.674152+0000 mgr.y (mgr.14146) 231 : cluster [DBG] pgmap v218: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:31.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:30 smithi016 bash[33624]: cephadm 2023-09-23T19:02:29.859194+0000 mgr.y (mgr.14146) 232 : cephadm [INF] Detected new or changed devices on smithi092 2023-09-23T19:02:31.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:30 smithi016 bash[33624]: audit 2023-09-23T19:02:29.868827+0000 mon.a (mon.0) 491 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:02:31.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:30 smithi016 bash[33624]: audit 2023-09-23T19:02:29.873531+0000 mon.a (mon.0) 492 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:02:31.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:30 smithi016 bash[33624]: audit 2023-09-23T19:02:29.874687+0000 mon.a (mon.0) 493 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:02:31.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:30 smithi016 bash[33624]: audit 2023-09-23T19:02:29.875795+0000 mon.a (mon.0) 494 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:02:31.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:30 smithi016 bash[33624]: cephadm 2023-09-23T19:02:29.876564+0000 mgr.y (mgr.14146) 233 : cephadm [INF] Adjusting osd_memory_target on smithi092 to 5755M 2023-09-23T19:02:31.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:30 smithi016 bash[33624]: audit 2023-09-23T19:02:29.880431+0000 mon.a (mon.0) 495 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:02:31.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:30 smithi016 bash[33624]: audit 2023-09-23T19:02:29.881970+0000 mon.a (mon.0) 496 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:02:31.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:30 smithi016 bash[33624]: audit 2023-09-23T19:02:29.883328+0000 mon.a (mon.0) 497 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:02:31.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:30 smithi016 bash[33624]: audit 2023-09-23T19:02:29.889839+0000 mon.a (mon.0) 498 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:02:31.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:30 smithi016 bash[33624]: cluster 2023-09-23T19:02:30.674704+0000 mgr.y (mgr.14146) 234 : cluster [DBG] pgmap v219: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:31.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:30 smithi016 bash[27237]: cephadm 2023-09-23T19:02:29.859194+0000 mgr.y (mgr.14146) 232 : cephadm [INF] Detected new or changed devices on smithi092 2023-09-23T19:02:31.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:30 smithi016 bash[27237]: audit 2023-09-23T19:02:29.868827+0000 mon.a (mon.0) 491 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:02:31.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:30 smithi016 bash[27237]: audit 2023-09-23T19:02:29.873531+0000 mon.a (mon.0) 492 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:02:31.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:30 smithi016 bash[27237]: audit 2023-09-23T19:02:29.874687+0000 mon.a (mon.0) 493 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:02:31.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:30 smithi016 bash[27237]: audit 2023-09-23T19:02:29.875795+0000 mon.a (mon.0) 494 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:02:31.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:30 smithi016 bash[27237]: cephadm 2023-09-23T19:02:29.876564+0000 mgr.y (mgr.14146) 233 : cephadm [INF] Adjusting osd_memory_target on smithi092 to 5755M 2023-09-23T19:02:31.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:30 smithi016 bash[27237]: audit 2023-09-23T19:02:29.880431+0000 mon.a (mon.0) 495 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:02:31.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:30 smithi016 bash[27237]: audit 2023-09-23T19:02:29.881970+0000 mon.a (mon.0) 496 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:02:31.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:30 smithi016 bash[27237]: audit 2023-09-23T19:02:29.883328+0000 mon.a (mon.0) 497 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:02:31.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:30 smithi016 bash[27237]: audit 2023-09-23T19:02:29.889839+0000 mon.a (mon.0) 498 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:02:31.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:30 smithi016 bash[27237]: cluster 2023-09-23T19:02:30.674704+0000 mgr.y (mgr.14146) 234 : cluster [DBG] pgmap v219: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:31.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:30 smithi092 bash[27556]: cephadm 2023-09-23T19:02:29.859194+0000 mgr.y (mgr.14146) 232 : cephadm [INF] Detected new or changed devices on smithi092 2023-09-23T19:02:31.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:30 smithi092 bash[27556]: audit 2023-09-23T19:02:29.868827+0000 mon.a (mon.0) 491 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:02:31.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:30 smithi092 bash[27556]: audit 2023-09-23T19:02:29.873531+0000 mon.a (mon.0) 492 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:02:31.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:30 smithi092 bash[27556]: audit 2023-09-23T19:02:29.874687+0000 mon.a (mon.0) 493 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:02:31.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:30 smithi092 bash[27556]: audit 2023-09-23T19:02:29.875795+0000 mon.a (mon.0) 494 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:02:31.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:30 smithi092 bash[27556]: cephadm 2023-09-23T19:02:29.876564+0000 mgr.y (mgr.14146) 233 : cephadm [INF] Adjusting osd_memory_target on smithi092 to 5755M 2023-09-23T19:02:31.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:30 smithi092 bash[27556]: audit 2023-09-23T19:02:29.880431+0000 mon.a (mon.0) 495 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:02:31.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:30 smithi092 bash[27556]: audit 2023-09-23T19:02:29.881970+0000 mon.a (mon.0) 496 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:02:31.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:30 smithi092 bash[27556]: audit 2023-09-23T19:02:29.883328+0000 mon.a (mon.0) 497 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:02:31.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:30 smithi092 bash[27556]: audit 2023-09-23T19:02:29.889839+0000 mon.a (mon.0) 498 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:02:31.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:30 smithi092 bash[27556]: cluster 2023-09-23T19:02:30.674704+0000 mgr.y (mgr.14146) 234 : cluster [DBG] pgmap v219: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:34.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:33 smithi016 bash[27237]: cluster 2023-09-23T19:02:32.675278+0000 mgr.y (mgr.14146) 235 : cluster [DBG] pgmap v220: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:34.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:33 smithi016 bash[33624]: cluster 2023-09-23T19:02:32.675278+0000 mgr.y (mgr.14146) 235 : cluster [DBG] pgmap v220: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:34.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:33 smithi092 bash[27556]: cluster 2023-09-23T19:02:32.675278+0000 mgr.y (mgr.14146) 235 : cluster [DBG] pgmap v220: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:36.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:35 smithi016 bash[27237]: cluster 2023-09-23T19:02:34.675839+0000 mgr.y (mgr.14146) 236 : cluster [DBG] pgmap v221: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:36.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:35 smithi016 bash[33624]: cluster 2023-09-23T19:02:34.675839+0000 mgr.y (mgr.14146) 236 : cluster [DBG] pgmap v221: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:36.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:35 smithi092 bash[27556]: cluster 2023-09-23T19:02:34.675839+0000 mgr.y (mgr.14146) 236 : cluster [DBG] pgmap v221: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:37.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:36 smithi016 bash[27237]: audit 2023-09-23T19:02:36.616878+0000 mon.a (mon.0) 499 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d086b076-127f-42de-bee9-157bf579d92d"}]: dispatch 2023-09-23T19:02:37.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:36 smithi016 bash[27237]: audit 2023-09-23T19:02:36.620169+0000 mon.b (mon.1) 19 : audit [INF] from='client.? 172.21.15.92:0/688375605' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d086b076-127f-42de-bee9-157bf579d92d"}]: dispatch 2023-09-23T19:02:37.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:36 smithi016 bash[27237]: audit 2023-09-23T19:02:36.624395+0000 mon.a (mon.0) 500 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d086b076-127f-42de-bee9-157bf579d92d"}]': finished 2023-09-23T19:02:37.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:36 smithi016 bash[27237]: cluster 2023-09-23T19:02:36.624492+0000 mon.a (mon.0) 501 : cluster [DBG] osdmap e45: 8 total, 7 up, 8 in 2023-09-23T19:02:37.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:36 smithi016 bash[27237]: audit 2023-09-23T19:02:36.624731+0000 mon.a (mon.0) 502 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:02:37.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:36 smithi016 bash[33624]: audit 2023-09-23T19:02:36.616878+0000 mon.a (mon.0) 499 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d086b076-127f-42de-bee9-157bf579d92d"}]: dispatch 2023-09-23T19:02:37.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:36 smithi016 bash[33624]: audit 2023-09-23T19:02:36.620169+0000 mon.b (mon.1) 19 : audit [INF] from='client.? 172.21.15.92:0/688375605' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d086b076-127f-42de-bee9-157bf579d92d"}]: dispatch 2023-09-23T19:02:37.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:36 smithi016 bash[33624]: audit 2023-09-23T19:02:36.624395+0000 mon.a (mon.0) 500 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d086b076-127f-42de-bee9-157bf579d92d"}]': finished 2023-09-23T19:02:37.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:36 smithi016 bash[33624]: cluster 2023-09-23T19:02:36.624492+0000 mon.a (mon.0) 501 : cluster [DBG] osdmap e45: 8 total, 7 up, 8 in 2023-09-23T19:02:37.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:36 smithi016 bash[33624]: audit 2023-09-23T19:02:36.624731+0000 mon.a (mon.0) 502 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:02:37.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:36 smithi092 bash[27556]: audit 2023-09-23T19:02:36.616878+0000 mon.a (mon.0) 499 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d086b076-127f-42de-bee9-157bf579d92d"}]: dispatch 2023-09-23T19:02:37.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:36 smithi092 bash[27556]: audit 2023-09-23T19:02:36.620169+0000 mon.b (mon.1) 19 : audit [INF] from='client.? 172.21.15.92:0/688375605' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d086b076-127f-42de-bee9-157bf579d92d"}]: dispatch 2023-09-23T19:02:37.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:36 smithi092 bash[27556]: audit 2023-09-23T19:02:36.624395+0000 mon.a (mon.0) 500 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d086b076-127f-42de-bee9-157bf579d92d"}]': finished 2023-09-23T19:02:37.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:36 smithi092 bash[27556]: cluster 2023-09-23T19:02:36.624492+0000 mon.a (mon.0) 501 : cluster [DBG] osdmap e45: 8 total, 7 up, 8 in 2023-09-23T19:02:37.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:36 smithi092 bash[27556]: audit 2023-09-23T19:02:36.624731+0000 mon.a (mon.0) 502 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:02:38.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:37 smithi016 bash[27237]: cluster 2023-09-23T19:02:36.676416+0000 mgr.y (mgr.14146) 237 : cluster [DBG] pgmap v223: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:38.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:37 smithi016 bash[27237]: audit 2023-09-23T19:02:37.343451+0000 mon.b (mon.1) 20 : audit [DBG] from='client.? 172.21.15.92:0/4219056996' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T19:02:38.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:37 smithi016 bash[33624]: cluster 2023-09-23T19:02:36.676416+0000 mgr.y (mgr.14146) 237 : cluster [DBG] pgmap v223: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:38.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:37 smithi016 bash[33624]: audit 2023-09-23T19:02:37.343451+0000 mon.b (mon.1) 20 : audit [DBG] from='client.? 172.21.15.92:0/4219056996' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T19:02:38.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:37 smithi092 bash[27556]: cluster 2023-09-23T19:02:36.676416+0000 mgr.y (mgr.14146) 237 : cluster [DBG] pgmap v223: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:38.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:37 smithi092 bash[27556]: audit 2023-09-23T19:02:37.343451+0000 mon.b (mon.1) 20 : audit [DBG] from='client.? 172.21.15.92:0/4219056996' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T19:02:40.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:39 smithi016 bash[27237]: cluster 2023-09-23T19:02:38.676967+0000 mgr.y (mgr.14146) 238 : cluster [DBG] pgmap v224: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:40.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:39 smithi016 bash[33624]: cluster 2023-09-23T19:02:38.676967+0000 mgr.y (mgr.14146) 238 : cluster [DBG] pgmap v224: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:40.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:39 smithi092 bash[27556]: cluster 2023-09-23T19:02:38.676967+0000 mgr.y (mgr.14146) 238 : cluster [DBG] pgmap v224: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:41.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:40 smithi016 bash[33624]: cluster 2023-09-23T19:02:40.677586+0000 mgr.y (mgr.14146) 239 : cluster [DBG] pgmap v225: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:41.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:40 smithi016 bash[27237]: cluster 2023-09-23T19:02:40.677586+0000 mgr.y (mgr.14146) 239 : cluster [DBG] pgmap v225: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:41.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:40 smithi092 bash[27556]: cluster 2023-09-23T19:02:40.677586+0000 mgr.y (mgr.14146) 239 : cluster [DBG] pgmap v225: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:44.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:43 smithi016 bash[27237]: cluster 2023-09-23T19:02:42.678134+0000 mgr.y (mgr.14146) 240 : cluster [DBG] pgmap v226: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:44.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:43 smithi016 bash[33624]: cluster 2023-09-23T19:02:42.678134+0000 mgr.y (mgr.14146) 240 : cluster [DBG] pgmap v226: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:44.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:43 smithi092 bash[27556]: cluster 2023-09-23T19:02:42.678134+0000 mgr.y (mgr.14146) 240 : cluster [DBG] pgmap v226: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:46.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:45 smithi016 bash[27237]: cluster 2023-09-23T19:02:44.678694+0000 mgr.y (mgr.14146) 241 : cluster [DBG] pgmap v227: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:46.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:45 smithi016 bash[33624]: cluster 2023-09-23T19:02:44.678694+0000 mgr.y (mgr.14146) 241 : cluster [DBG] pgmap v227: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:46.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:45 smithi092 bash[27556]: cluster 2023-09-23T19:02:44.678694+0000 mgr.y (mgr.14146) 241 : cluster [DBG] pgmap v227: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:48.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:47 smithi016 bash[27237]: cluster 2023-09-23T19:02:46.679269+0000 mgr.y (mgr.14146) 242 : cluster [DBG] pgmap v228: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:48.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:47 smithi016 bash[33624]: cluster 2023-09-23T19:02:46.679269+0000 mgr.y (mgr.14146) 242 : cluster [DBG] pgmap v228: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:48.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:47 smithi092 bash[27556]: cluster 2023-09-23T19:02:46.679269+0000 mgr.y (mgr.14146) 242 : cluster [DBG] pgmap v228: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:50.017 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:49 smithi092 bash[27556]: cluster 2023-09-23T19:02:48.679803+0000 mgr.y (mgr.14146) 243 : cluster [DBG] pgmap v229: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:50.017 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:49 smithi092 bash[27556]: audit 2023-09-23T19:02:48.976370+0000 mon.a (mon.0) 503 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T19:02:50.017 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:49 smithi092 bash[27556]: audit 2023-09-23T19:02:48.978300+0000 mon.a (mon.0) 504 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:02:50.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:49 smithi016 bash[27237]: cluster 2023-09-23T19:02:48.679803+0000 mgr.y (mgr.14146) 243 : cluster [DBG] pgmap v229: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:50.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:49 smithi016 bash[27237]: audit 2023-09-23T19:02:48.976370+0000 mon.a (mon.0) 503 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T19:02:50.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:49 smithi016 bash[27237]: audit 2023-09-23T19:02:48.978300+0000 mon.a (mon.0) 504 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:02:50.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:49 smithi016 bash[33624]: cluster 2023-09-23T19:02:48.679803+0000 mgr.y (mgr.14146) 243 : cluster [DBG] pgmap v229: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:50.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:49 smithi016 bash[33624]: audit 2023-09-23T19:02:48.976370+0000 mon.a (mon.0) 503 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T19:02:50.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:49 smithi016 bash[33624]: audit 2023-09-23T19:02:48.978300+0000 mon.a (mon.0) 504 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:02:51.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:50 smithi016 bash[27237]: cephadm 2023-09-23T19:02:48.980025+0000 mgr.y (mgr.14146) 244 : cephadm [INF] Deploying daemon osd.7 on smithi092 2023-09-23T19:02:51.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:50 smithi016 bash[33624]: cephadm 2023-09-23T19:02:48.980025+0000 mgr.y (mgr.14146) 244 : cephadm [INF] Deploying daemon osd.7 on smithi092 2023-09-23T19:02:51.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:50 smithi092 bash[27556]: cephadm 2023-09-23T19:02:48.980025+0000 mgr.y (mgr.14146) 244 : cephadm [INF] Deploying daemon osd.7 on smithi092 2023-09-23T19:02:52.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:51 smithi016 bash[27237]: cluster 2023-09-23T19:02:50.680423+0000 mgr.y (mgr.14146) 245 : cluster [DBG] pgmap v230: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:52.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:51 smithi016 bash[33624]: cluster 2023-09-23T19:02:50.680423+0000 mgr.y (mgr.14146) 245 : cluster [DBG] pgmap v230: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:52.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:51 smithi092 bash[27556]: cluster 2023-09-23T19:02:50.680423+0000 mgr.y (mgr.14146) 245 : cluster [DBG] pgmap v230: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:54.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:53 smithi016 bash[27237]: cluster 2023-09-23T19:02:52.680968+0000 mgr.y (mgr.14146) 246 : cluster [DBG] pgmap v231: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:54.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:53 smithi016 bash[27237]: audit 2023-09-23T19:02:53.224454+0000 mon.a (mon.0) 505 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:02:54.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:53 smithi016 bash[27237]: audit 2023-09-23T19:02:53.224945+0000 mon.a (mon.0) 506 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:02:54.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:53 smithi016 bash[33624]: cluster 2023-09-23T19:02:52.680968+0000 mgr.y (mgr.14146) 246 : cluster [DBG] pgmap v231: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:54.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:53 smithi016 bash[33624]: audit 2023-09-23T19:02:53.224454+0000 mon.a (mon.0) 505 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:02:54.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:53 smithi016 bash[33624]: audit 2023-09-23T19:02:53.224945+0000 mon.a (mon.0) 506 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:02:54.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:53 smithi092 bash[27556]: cluster 2023-09-23T19:02:52.680968+0000 mgr.y (mgr.14146) 246 : cluster [DBG] pgmap v231: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:54.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:53 smithi092 bash[27556]: audit 2023-09-23T19:02:53.224454+0000 mon.a (mon.0) 505 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:02:54.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:53 smithi092 bash[27556]: audit 2023-09-23T19:02:53.224945+0000 mon.a (mon.0) 506 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:02:56.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:55 smithi016 bash[27237]: cluster 2023-09-23T19:02:54.681647+0000 mgr.y (mgr.14146) 247 : cluster [DBG] pgmap v232: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:56.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:55 smithi016 bash[33624]: cluster 2023-09-23T19:02:54.681647+0000 mgr.y (mgr.14146) 247 : cluster [DBG] pgmap v232: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:56.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:55 smithi092 bash[27556]: cluster 2023-09-23T19:02:54.681647+0000 mgr.y (mgr.14146) 247 : cluster [DBG] pgmap v232: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:58.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:57 smithi016 bash[27237]: cluster 2023-09-23T19:02:56.682255+0000 mgr.y (mgr.14146) 248 : cluster [DBG] pgmap v233: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:58.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:57 smithi016 bash[33624]: cluster 2023-09-23T19:02:56.682255+0000 mgr.y (mgr.14146) 248 : cluster [DBG] pgmap v233: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:58.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:57 smithi092 bash[27556]: cluster 2023-09-23T19:02:56.682255+0000 mgr.y (mgr.14146) 248 : cluster [DBG] pgmap v233: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:59.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:59 smithi092 bash[27556]: audit 2023-09-23T19:02:58.442859+0000 mon.a (mon.0) 507 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:02:59.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:59 smithi092 bash[27556]: audit 2023-09-23T19:02:58.444474+0000 mon.a (mon.0) 508 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:02:59.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:59 smithi092 bash[27556]: audit 2023-09-23T19:02:58.446131+0000 mon.a (mon.0) 509 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:02:59.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:59 smithi092 bash[27556]: audit 2023-09-23T19:02:58.452436+0000 mon.a (mon.0) 510 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:02:59.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:02:59 smithi092 bash[27556]: cluster 2023-09-23T19:02:58.682805+0000 mgr.y (mgr.14146) 249 : cluster [DBG] pgmap v234: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:59.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:59 smithi016 bash[27237]: audit 2023-09-23T19:02:58.442859+0000 mon.a (mon.0) 507 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:02:59.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:59 smithi016 bash[27237]: audit 2023-09-23T19:02:58.444474+0000 mon.a (mon.0) 508 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:02:59.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:59 smithi016 bash[27237]: audit 2023-09-23T19:02:58.446131+0000 mon.a (mon.0) 509 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:02:59.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:59 smithi016 bash[27237]: audit 2023-09-23T19:02:58.452436+0000 mon.a (mon.0) 510 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:02:59.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:02:59 smithi016 bash[27237]: cluster 2023-09-23T19:02:58.682805+0000 mgr.y (mgr.14146) 249 : cluster [DBG] pgmap v234: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:02:59.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:59 smithi016 bash[33624]: audit 2023-09-23T19:02:58.442859+0000 mon.a (mon.0) 507 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:02:59.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:59 smithi016 bash[33624]: audit 2023-09-23T19:02:58.444474+0000 mon.a (mon.0) 508 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:02:59.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:59 smithi016 bash[33624]: audit 2023-09-23T19:02:58.446131+0000 mon.a (mon.0) 509 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:02:59.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:59 smithi016 bash[33624]: audit 2023-09-23T19:02:58.452436+0000 mon.a (mon.0) 510 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:02:59.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:02:59 smithi016 bash[33624]: cluster 2023-09-23T19:02:58.682805+0000 mgr.y (mgr.14146) 249 : cluster [DBG] pgmap v234: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:03:00.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:00 smithi016 bash[27237]: audit 2023-09-23T19:02:59.790041+0000 mon.a (mon.0) 511 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T19:03:00.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:00 smithi016 bash[27237]: audit 2023-09-23T19:02:59.793499+0000 mon.b (mon.1) 21 : audit [INF] from='osd.7 [v2:172.21.15.92:6824/98624761,v1:172.21.15.92:6825/98624761]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T19:03:00.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:00 smithi016 bash[33624]: audit 2023-09-23T19:02:59.790041+0000 mon.a (mon.0) 511 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T19:03:00.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:00 smithi016 bash[33624]: audit 2023-09-23T19:02:59.793499+0000 mon.b (mon.1) 21 : audit [INF] from='osd.7 [v2:172.21.15.92:6824/98624761,v1:172.21.15.92:6825/98624761]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T19:03:00.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:00 smithi092 bash[27556]: audit 2023-09-23T19:02:59.790041+0000 mon.a (mon.0) 511 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T19:03:00.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:00 smithi092 bash[27556]: audit 2023-09-23T19:02:59.793499+0000 mon.b (mon.1) 21 : audit [INF] from='osd.7 [v2:172.21.15.92:6824/98624761,v1:172.21.15.92:6825/98624761]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T19:03:01.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:01 smithi092 bash[27556]: audit 2023-09-23T19:03:00.464658+0000 mon.a (mon.0) 512 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-09-23T19:03:01.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:01 smithi092 bash[27556]: cluster 2023-09-23T19:03:00.464762+0000 mon.a (mon.0) 513 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2023-09-23T19:03:01.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:01 smithi092 bash[27556]: audit 2023-09-23T19:03:00.464984+0000 mon.a (mon.0) 514 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:03:01.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:01 smithi092 bash[27556]: audit 2023-09-23T19:03:00.466519+0000 mon.a (mon.0) 515 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi092", "root=default"]}]: dispatch 2023-09-23T19:03:01.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:01 smithi092 bash[27556]: audit 2023-09-23T19:03:00.470176+0000 mon.b (mon.1) 22 : audit [INF] from='osd.7 [v2:172.21.15.92:6824/98624761,v1:172.21.15.92:6825/98624761]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi092", "root=default"]}]: dispatch 2023-09-23T19:03:01.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:01 smithi092 bash[27556]: cluster 2023-09-23T19:03:00.683362+0000 mgr.y (mgr.14146) 250 : cluster [DBG] pgmap v236: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:03:01.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:01 smithi016 bash[27237]: audit 2023-09-23T19:03:00.464658+0000 mon.a (mon.0) 512 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-09-23T19:03:01.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:01 smithi016 bash[27237]: cluster 2023-09-23T19:03:00.464762+0000 mon.a (mon.0) 513 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2023-09-23T19:03:01.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:01 smithi016 bash[27237]: audit 2023-09-23T19:03:00.464984+0000 mon.a (mon.0) 514 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:03:01.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:01 smithi016 bash[27237]: audit 2023-09-23T19:03:00.466519+0000 mon.a (mon.0) 515 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi092", "root=default"]}]: dispatch 2023-09-23T19:03:01.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:01 smithi016 bash[27237]: audit 2023-09-23T19:03:00.470176+0000 mon.b (mon.1) 22 : audit [INF] from='osd.7 [v2:172.21.15.92:6824/98624761,v1:172.21.15.92:6825/98624761]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi092", "root=default"]}]: dispatch 2023-09-23T19:03:01.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:01 smithi016 bash[27237]: cluster 2023-09-23T19:03:00.683362+0000 mgr.y (mgr.14146) 250 : cluster [DBG] pgmap v236: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:03:01.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:01 smithi016 bash[33624]: audit 2023-09-23T19:03:00.464658+0000 mon.a (mon.0) 512 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-09-23T19:03:01.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:01 smithi016 bash[33624]: cluster 2023-09-23T19:03:00.464762+0000 mon.a (mon.0) 513 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2023-09-23T19:03:01.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:01 smithi016 bash[33624]: audit 2023-09-23T19:03:00.464984+0000 mon.a (mon.0) 514 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:03:01.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:01 smithi016 bash[33624]: audit 2023-09-23T19:03:00.466519+0000 mon.a (mon.0) 515 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi092", "root=default"]}]: dispatch 2023-09-23T19:03:01.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:01 smithi016 bash[33624]: audit 2023-09-23T19:03:00.470176+0000 mon.b (mon.1) 22 : audit [INF] from='osd.7 [v2:172.21.15.92:6824/98624761,v1:172.21.15.92:6825/98624761]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi092", "root=default"]}]: dispatch 2023-09-23T19:03:01.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:01 smithi016 bash[33624]: cluster 2023-09-23T19:03:00.683362+0000 mgr.y (mgr.14146) 250 : cluster [DBG] pgmap v236: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T19:03:02.034 INFO:teuthology.orchestra.run.smithi092.stdout:Created osd(s) 7 on host 'smithi092' 2023-09-23T19:03:02.755 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:02 smithi092 bash[27556]: audit 2023-09-23T19:03:01.468960+0000 mon.a (mon.0) 516 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi092", "root=default"]}]': finished 2023-09-23T19:03:02.756 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:02 smithi092 bash[27556]: cluster 2023-09-23T19:03:01.469236+0000 mon.a (mon.0) 517 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2023-09-23T19:03:02.756 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:02 smithi092 bash[27556]: audit 2023-09-23T19:03:01.470163+0000 mon.a (mon.0) 518 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:03:02.756 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:02 smithi092 bash[27556]: audit 2023-09-23T19:03:01.476225+0000 mon.a (mon.0) 519 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:03:02.756 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:02 smithi092 bash[27556]: audit 2023-09-23T19:03:02.030258+0000 mon.a (mon.0) 520 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:03:02.756 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:02 smithi092 bash[27556]: audit 2023-09-23T19:03:02.030797+0000 mon.a (mon.0) 521 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:03:02.757 DEBUG:teuthology.orchestra.run.smithi092:osd.7> sudo journalctl -f -n 0 -u ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@osd.7.service 2023-09-23T19:03:02.760 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2023-09-23T19:03:02.760 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd stat -f json 2023-09-23T19:03:02.774 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:02 smithi016 bash[27237]: audit 2023-09-23T19:03:01.468960+0000 mon.a (mon.0) 516 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi092", "root=default"]}]': finished 2023-09-23T19:03:02.774 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:02 smithi016 bash[27237]: cluster 2023-09-23T19:03:01.469236+0000 mon.a (mon.0) 517 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2023-09-23T19:03:02.774 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:02 smithi016 bash[27237]: audit 2023-09-23T19:03:01.470163+0000 mon.a (mon.0) 518 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:03:02.774 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:02 smithi016 bash[27237]: audit 2023-09-23T19:03:01.476225+0000 mon.a (mon.0) 519 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:03:02.774 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:02 smithi016 bash[27237]: audit 2023-09-23T19:03:02.030258+0000 mon.a (mon.0) 520 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:03:02.774 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:02 smithi016 bash[27237]: audit 2023-09-23T19:03:02.030797+0000 mon.a (mon.0) 521 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:03:02.775 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:02 smithi016 bash[33624]: audit 2023-09-23T19:03:01.468960+0000 mon.a (mon.0) 516 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi092", "root=default"]}]': finished 2023-09-23T19:03:02.775 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:02 smithi016 bash[33624]: cluster 2023-09-23T19:03:01.469236+0000 mon.a (mon.0) 517 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2023-09-23T19:03:02.775 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:02 smithi016 bash[33624]: audit 2023-09-23T19:03:01.470163+0000 mon.a (mon.0) 518 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:03:02.775 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:02 smithi016 bash[33624]: audit 2023-09-23T19:03:01.476225+0000 mon.a (mon.0) 519 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:03:02.775 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:02 smithi016 bash[33624]: audit 2023-09-23T19:03:02.030258+0000 mon.a (mon.0) 520 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:03:02.775 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:02 smithi016 bash[33624]: audit 2023-09-23T19:03:02.030797+0000 mon.a (mon.0) 521 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:03:02.893 INFO:journalctl@ceph.osd.7.smithi092.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-09-23T19:03:03.481 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:03 smithi016 bash[27237]: cluster 2023-09-23T19:03:00.817346+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T19:03:03.482 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:03 smithi016 bash[27237]: cluster 2023-09-23T19:03:00.817443+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T19:03:03.482 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:03 smithi016 bash[27237]: cluster 2023-09-23T19:03:02.472111+0000 mon.a (mon.0) 522 : cluster [INF] osd.7 [v2:172.21.15.92:6824/98624761,v1:172.21.15.92:6825/98624761] boot 2023-09-23T19:03:03.482 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:03 smithi016 bash[27237]: cluster 2023-09-23T19:03:02.472204+0000 mon.a (mon.0) 523 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2023-09-23T19:03:03.482 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:03 smithi016 bash[27237]: audit 2023-09-23T19:03:02.472936+0000 mon.a (mon.0) 524 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:03:03.482 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:03 smithi016 bash[27237]: cluster 2023-09-23T19:03:02.683918+0000 mgr.y (mgr.14146) 251 : cluster [DBG] pgmap v239: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:03.482 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:03 smithi016 bash[27237]: cluster 2023-09-23T19:03:03.473508+0000 mon.a (mon.0) 525 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2023-09-23T19:03:03.482 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:03 smithi016 bash[33624]: cluster 2023-09-23T19:03:00.817346+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T19:03:03.483 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:03 smithi016 bash[33624]: cluster 2023-09-23T19:03:00.817443+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T19:03:03.483 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:03 smithi016 bash[33624]: cluster 2023-09-23T19:03:02.472111+0000 mon.a (mon.0) 522 : cluster [INF] osd.7 [v2:172.21.15.92:6824/98624761,v1:172.21.15.92:6825/98624761] boot 2023-09-23T19:03:03.483 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:03 smithi016 bash[33624]: cluster 2023-09-23T19:03:02.472204+0000 mon.a (mon.0) 523 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2023-09-23T19:03:03.483 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:03 smithi016 bash[33624]: audit 2023-09-23T19:03:02.472936+0000 mon.a (mon.0) 524 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:03:03.483 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:03 smithi016 bash[33624]: cluster 2023-09-23T19:03:02.683918+0000 mgr.y (mgr.14146) 251 : cluster [DBG] pgmap v239: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:03.483 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:03 smithi016 bash[33624]: cluster 2023-09-23T19:03:03.473508+0000 mon.a (mon.0) 525 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2023-09-23T19:03:03.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:03 smithi092 bash[27556]: cluster 2023-09-23T19:03:00.817346+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T19:03:03.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:03 smithi092 bash[27556]: cluster 2023-09-23T19:03:00.817443+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T19:03:03.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:03 smithi092 bash[27556]: cluster 2023-09-23T19:03:02.472111+0000 mon.a (mon.0) 522 : cluster [INF] osd.7 [v2:172.21.15.92:6824/98624761,v1:172.21.15.92:6825/98624761] boot 2023-09-23T19:03:03.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:03 smithi092 bash[27556]: cluster 2023-09-23T19:03:02.472204+0000 mon.a (mon.0) 523 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2023-09-23T19:03:03.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:03 smithi092 bash[27556]: audit 2023-09-23T19:03:02.472936+0000 mon.a (mon.0) 524 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:03:03.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:03 smithi092 bash[27556]: cluster 2023-09-23T19:03:02.683918+0000 mgr.y (mgr.14146) 251 : cluster [DBG] pgmap v239: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:03.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:03 smithi092 bash[27556]: cluster 2023-09-23T19:03:03.473508+0000 mon.a (mon.0) 525 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2023-09-23T19:03:05.759 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-09-23T19:03:05.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:05 smithi016 bash[33624]: cluster 2023-09-23T19:03:04.476035+0000 mon.a (mon.0) 526 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2023-09-23T19:03:05.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:05 smithi016 bash[33624]: cluster 2023-09-23T19:03:04.684493+0000 mgr.y (mgr.14146) 252 : cluster [DBG] pgmap v242: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:05.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:05 smithi016 bash[27237]: cluster 2023-09-23T19:03:04.476035+0000 mon.a (mon.0) 526 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2023-09-23T19:03:05.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:05 smithi016 bash[27237]: cluster 2023-09-23T19:03:04.684493+0000 mgr.y (mgr.14146) 252 : cluster [DBG] pgmap v242: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:05.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:05 smithi092 bash[27556]: cluster 2023-09-23T19:03:04.476035+0000 mon.a (mon.0) 526 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2023-09-23T19:03:05.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:05 smithi092 bash[27556]: cluster 2023-09-23T19:03:04.684493+0000 mgr.y (mgr.14146) 252 : cluster [DBG] pgmap v242: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:06.604 INFO:teuthology.orchestra.run.smithi016.stdout:{"epoch":50,"num_osds":8,"num_up_osds":8,"osd_up_since":1695495782,"num_in_osds":8,"osd_in_since":1695495756,"num_remapped_pgs":0} 2023-09-23T19:03:06.605 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T19:03:06.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:06 smithi016 bash[27237]: audit 2023-09-23T19:03:05.756857+0000 mon.c (mon.2) 11 : audit [DBG] from='client.? 172.21.15.16:0/1009597085' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-23T19:03:06.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:06 smithi016 bash[33624]: audit 2023-09-23T19:03:05.756857+0000 mon.c (mon.2) 11 : audit [DBG] from='client.? 172.21.15.16:0/1009597085' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-23T19:03:06.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:06 smithi092 bash[27556]: audit 2023-09-23T19:03:05.756857+0000 mon.c (mon.2) 11 : audit [DBG] from='client.? 172.21.15.16:0/1009597085' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-23T19:03:07.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:07 smithi016 bash[27237]: cluster 2023-09-23T19:03:06.685088+0000 mgr.y (mgr.14146) 253 : cluster [DBG] pgmap v243: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:07.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:07 smithi016 bash[33624]: cluster 2023-09-23T19:03:06.685088+0000 mgr.y (mgr.14146) 253 : cluster [DBG] pgmap v243: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:07.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:07 smithi092 bash[27556]: cluster 2023-09-23T19:03:06.685088+0000 mgr.y (mgr.14146) 253 : cluster [DBG] pgmap v243: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:10.013 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:09 smithi092 bash[27556]: cluster 2023-09-23T19:03:08.685667+0000 mgr.y (mgr.14146) 254 : cluster [DBG] pgmap v244: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:10.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:09 smithi016 bash[33624]: cluster 2023-09-23T19:03:08.685667+0000 mgr.y (mgr.14146) 254 : cluster [DBG] pgmap v244: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:10.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:09 smithi016 bash[27237]: cluster 2023-09-23T19:03:08.685667+0000 mgr.y (mgr.14146) 254 : cluster [DBG] pgmap v244: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:11.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:10 smithi016 bash[27237]: cluster 2023-09-23T19:03:10.686251+0000 mgr.y (mgr.14146) 255 : cluster [DBG] pgmap v245: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:11.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:10 smithi016 bash[33624]: cluster 2023-09-23T19:03:10.686251+0000 mgr.y (mgr.14146) 255 : cluster [DBG] pgmap v245: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:11.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:10 smithi092 bash[27556]: cluster 2023-09-23T19:03:10.686251+0000 mgr.y (mgr.14146) 255 : cluster [DBG] pgmap v245: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:11.364 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:03:12.149 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:11 smithi016 bash[27237]: cephadm 2023-09-23T19:03:10.871491+0000 mgr.y (mgr.14146) 256 : cephadm [INF] Detected new or changed devices on smithi092 2023-09-23T19:03:12.149 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:11 smithi016 bash[27237]: audit 2023-09-23T19:03:10.882612+0000 mon.a (mon.0) 527 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:03:12.149 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:11 smithi016 bash[27237]: audit 2023-09-23T19:03:10.884618+0000 mon.a (mon.0) 528 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:03:12.149 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:11 smithi016 bash[27237]: audit 2023-09-23T19:03:10.889546+0000 mon.a (mon.0) 529 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:03:12.149 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:11 smithi016 bash[27237]: audit 2023-09-23T19:03:10.890842+0000 mon.a (mon.0) 530 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:03:12.150 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:11 smithi016 bash[27237]: audit 2023-09-23T19:03:10.892039+0000 mon.a (mon.0) 531 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:03:12.150 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:11 smithi016 bash[27237]: cephadm 2023-09-23T19:03:10.892987+0000 mgr.y (mgr.14146) 257 : cephadm [INF] Adjusting osd_memory_target on smithi092 to 4316M 2023-09-23T19:03:12.150 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:11 smithi016 bash[27237]: audit 2023-09-23T19:03:10.900105+0000 mon.a (mon.0) 532 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:03:12.150 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:11 smithi016 bash[27237]: audit 2023-09-23T19:03:10.901799+0000 mon.a (mon.0) 533 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:03:12.150 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:11 smithi016 bash[27237]: audit 2023-09-23T19:03:10.902895+0000 mon.a (mon.0) 534 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:03:12.150 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:11 smithi016 bash[27237]: audit 2023-09-23T19:03:10.909806+0000 mon.a (mon.0) 535 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:03:12.151 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:11 smithi016 bash[33624]: cephadm 2023-09-23T19:03:10.871491+0000 mgr.y (mgr.14146) 256 : cephadm [INF] Detected new or changed devices on smithi092 2023-09-23T19:03:12.151 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:11 smithi016 bash[33624]: audit 2023-09-23T19:03:10.882612+0000 mon.a (mon.0) 527 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:03:12.151 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:11 smithi016 bash[33624]: audit 2023-09-23T19:03:10.884618+0000 mon.a (mon.0) 528 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:03:12.151 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:11 smithi016 bash[33624]: audit 2023-09-23T19:03:10.889546+0000 mon.a (mon.0) 529 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:03:12.151 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:11 smithi016 bash[33624]: audit 2023-09-23T19:03:10.890842+0000 mon.a (mon.0) 530 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:03:12.151 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:11 smithi016 bash[33624]: audit 2023-09-23T19:03:10.892039+0000 mon.a (mon.0) 531 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:03:12.152 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:11 smithi016 bash[33624]: cephadm 2023-09-23T19:03:10.892987+0000 mgr.y (mgr.14146) 257 : cephadm [INF] Adjusting osd_memory_target on smithi092 to 4316M 2023-09-23T19:03:12.152 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:11 smithi016 bash[33624]: audit 2023-09-23T19:03:10.900105+0000 mon.a (mon.0) 532 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:03:12.152 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:11 smithi016 bash[33624]: audit 2023-09-23T19:03:10.901799+0000 mon.a (mon.0) 533 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:03:12.152 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:11 smithi016 bash[33624]: audit 2023-09-23T19:03:10.902895+0000 mon.a (mon.0) 534 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:03:12.152 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:11 smithi016 bash[33624]: audit 2023-09-23T19:03:10.909806+0000 mon.a (mon.0) 535 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:03:12.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:11 smithi092 bash[27556]: cephadm 2023-09-23T19:03:10.871491+0000 mgr.y (mgr.14146) 256 : cephadm [INF] Detected new or changed devices on smithi092 2023-09-23T19:03:12.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:11 smithi092 bash[27556]: audit 2023-09-23T19:03:10.882612+0000 mon.a (mon.0) 527 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:03:12.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:11 smithi092 bash[27556]: audit 2023-09-23T19:03:10.884618+0000 mon.a (mon.0) 528 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:03:12.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:11 smithi092 bash[27556]: audit 2023-09-23T19:03:10.889546+0000 mon.a (mon.0) 529 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:03:12.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:11 smithi092 bash[27556]: audit 2023-09-23T19:03:10.890842+0000 mon.a (mon.0) 530 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:03:12.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:11 smithi092 bash[27556]: audit 2023-09-23T19:03:10.892039+0000 mon.a (mon.0) 531 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:03:12.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:11 smithi092 bash[27556]: cephadm 2023-09-23T19:03:10.892987+0000 mgr.y (mgr.14146) 257 : cephadm [INF] Adjusting osd_memory_target on smithi092 to 4316M 2023-09-23T19:03:12.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:11 smithi092 bash[27556]: audit 2023-09-23T19:03:10.900105+0000 mon.a (mon.0) 532 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:03:12.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:11 smithi092 bash[27556]: audit 2023-09-23T19:03:10.901799+0000 mon.a (mon.0) 533 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:03:12.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:11 smithi092 bash[27556]: audit 2023-09-23T19:03:10.902895+0000 mon.a (mon.0) 534 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:03:12.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:11 smithi092 bash[27556]: audit 2023-09-23T19:03:10.909806+0000 mon.a (mon.0) 535 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:03:13.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:12 smithi016 bash[33624]: cluster 2023-09-23T19:03:12.686819+0000 mgr.y (mgr.14146) 258 : cluster [DBG] pgmap v246: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:13.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:12 smithi016 bash[27237]: cluster 2023-09-23T19:03:12.686819+0000 mgr.y (mgr.14146) 258 : cluster [DBG] pgmap v246: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:13.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:12 smithi092 bash[27556]: cluster 2023-09-23T19:03:12.686819+0000 mgr.y (mgr.14146) 258 : cluster [DBG] pgmap v246: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:14.329 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-09-23T19:03:14.329 INFO:teuthology.orchestra.run.smithi016.stdout:{"epoch":50,"fsid":"95ce9eb6-5a42-11ee-8db4-212e2dc638e7","created":"2023-09-23T18:55:23.802934+0000","modified":"2023-09-23T19:03:04.468411+0000","last_up_change":"2023-09-23T19:03:02.463512+0000","last_in_change":"2023-09-23T19:02:36.618072+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-09-23T18:59:33.315416+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}}],"osds":[{"osd":0,"uuid":"a4e8ed09-1195-4878-928a-13881bc04dc7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6802","nonce":2337435219},{"type":"v1","addr":"172.21.15.16:6803","nonce":2337435219}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6804","nonce":2337435219},{"type":"v1","addr":"172.21.15.16:6805","nonce":2337435219}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6808","nonce":2337435219},{"type":"v1","addr":"172.21.15.16:6809","nonce":2337435219}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6806","nonce":2337435219},{"type":"v1","addr":"172.21.15.16:6807","nonce":2337435219}]},"public_addr":"172.21.15.16:6803/2337435219","cluster_addr":"172.21.15.16:6805/2337435219","heartbeat_back_addr":"172.21.15.16:6809/2337435219","heartbeat_front_addr":"172.21.15.16:6807/2337435219","state":["exists","up"]},{"osd":1,"uuid":"c5ff2879-0d1c-481a-b847-248433c12aa4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6810","nonce":3893879931},{"type":"v1","addr":"172.21.15.16:6811","nonce":3893879931}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6812","nonce":3893879931},{"type":"v1","addr":"172.21.15.16:6813","nonce":3893879931}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6816","nonce":3893879931},{"type":"v1","addr":"172.21.15.16:6817","nonce":3893879931}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6814","nonce":3893879931},{"type":"v1","addr":"172.21.15.16:6815","nonce":3893879931}]},"public_addr":"172.21.15.16:6811/3893879931","cluster_addr":"172.21.15.16:6813/3893879931","heartbeat_back_addr":"172.21.15.16:6817/3893879931","heartbeat_front_addr":"172.21.15.16:6815/3893879931","state":["exists","up"]},{"osd":2,"uuid":"7974ce4d-9ed8-4a72-b44b-d500b34da8cf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6818","nonce":568840676},{"type":"v1","addr":"172.21.15.16:6819","nonce":568840676}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6820","nonce":568840676},{"type":"v1","addr":"172.21.15.16:6821","nonce":568840676}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6824","nonce":568840676},{"type":"v1","addr":"172.21.15.16:6825","nonce":568840676}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6822","nonce":568840676},{"type":"v1","addr":"172.21.15.16:6823","nonce":568840676}]},"public_addr":"172.21.15.16:6819/568840676","cluster_addr":"172.21.15.16:6821/568840676","heartbeat_back_addr":"172.21.15.16:6825/568840676","heartbeat_front_addr":"172.21.15.16:6823/568840676","state":["exists","up"]},{"osd":3,"uuid":"f5e0d57b-d629-4453-a527-ad77e6c210ad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6826","nonce":204922667},{"type":"v1","addr":"172.21.15.16:6827","nonce":204922667}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6828","nonce":204922667},{"type":"v1","addr":"172.21.15.16:6829","nonce":204922667}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6832","nonce":204922667},{"type":"v1","addr":"172.21.15.16:6833","nonce":204922667}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6830","nonce":204922667},{"type":"v1","addr":"172.21.15.16:6831","nonce":204922667}]},"public_addr":"172.21.15.16:6827/204922667","cluster_addr":"172.21.15.16:6829/204922667","heartbeat_back_addr":"172.21.15.16:6833/204922667","heartbeat_front_addr":"172.21.15.16:6831/204922667","state":["exists","up"]},{"osd":4,"uuid":"55a8dab2-6393-44ab-943e-cb7ab2cb7c31","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6800","nonce":99745912},{"type":"v1","addr":"172.21.15.92:6801","nonce":99745912}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6802","nonce":99745912},{"type":"v1","addr":"172.21.15.92:6803","nonce":99745912}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6806","nonce":99745912},{"type":"v1","addr":"172.21.15.92:6807","nonce":99745912}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6804","nonce":99745912},{"type":"v1","addr":"172.21.15.92:6805","nonce":99745912}]},"public_addr":"172.21.15.92:6801/99745912","cluster_addr":"172.21.15.92:6803/99745912","heartbeat_back_addr":"172.21.15.92:6807/99745912","heartbeat_front_addr":"172.21.15.92:6805/99745912","state":["exists","up"]},{"osd":5,"uuid":"b29b73b1-51d7-4484-bd79-2334c650cb62","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":37,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6808","nonce":1987214502},{"type":"v1","addr":"172.21.15.92:6809","nonce":1987214502}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6810","nonce":1987214502},{"type":"v1","addr":"172.21.15.92:6811","nonce":1987214502}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6814","nonce":1987214502},{"type":"v1","addr":"172.21.15.92:6815","nonce":1987214502}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6812","nonce":1987214502},{"type":"v1","addr":"172.21.15.92:6813","nonce":1987214502}]},"public_addr":"172.21.15.92:6809/1987214502","cluster_addr":"172.21.15.92:6811/1987214502","heartbeat_back_addr":"172.21.15.92:6815/1987214502","heartbeat_front_addr":"172.21.15.92:6813/1987214502","state":["exists","up"]},{"osd":6,"uuid":"9fe16cc9-f0d3-4b79-bc68-38305c55b551","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6816","nonce":2820798178},{"type":"v1","addr":"172.21.15.92:6817","nonce":2820798178}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6818","nonce":2820798178},{"type":"v1","addr":"172.21.15.92:6819","nonce":2820798178}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6822","nonce":2820798178},{"type":"v1","addr":"172.21.15.92:6823","nonce":2820798178}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6820","nonce":2820798178},{"type":"v1","addr":"172.21.15.92:6821","nonce":2820798178}]},"public_addr":"172.21.15.92:6817/2820798178","cluster_addr":"172.21.15.92:6819/2820798178","heartbeat_back_addr":"172.21.15.92:6823/2820798178","heartbeat_front_addr":"172.21.15.92:6821/2820798178","state":["exists","up"]},{"osd":7,"uuid":"d086b076-127f-42de-bee9-157bf579d92d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6824","nonce":98624761},{"type":"v1","addr":"172.21.15.92:6825","nonce":98624761}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6826","nonce":98624761},{"type":"v1","addr":"172.21.15.92:6827","nonce":98624761}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6830","nonce":98624761},{"type":"v1","addr":"172.21.15.92:6831","nonce":98624761}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6828","nonce":98624761},{"type":"v1","addr":"172.21.15.92:6829","nonce":98624761}]},"public_addr":"172.21.15.92:6825/98624761","cluster_addr":"172.21.15.92:6827/98624761","heartbeat_back_addr":"172.21.15.92:6831/98624761","heartbeat_front_addr":"172.21.15.92:6829/98624761","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:58:07.335980+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:58:49.342213+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:59:30.939986+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:00:12.926364+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:00:53.466692+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:01:36.332957+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:02:18.593854+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:03:00.817448+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.16:6801/2756103674":"2023-09-24T18:56:20.574625+0000","172.21.15.16:0/2843459289":"2023-09-24T18:56:20.574625+0000","172.21.15.16:0/272242693":"2023-09-24T18:56:20.574625+0000","172.21.15.16:0/79169563":"2023-09-24T18:55:50.550546+0000","172.21.15.16:6800/2615282850":"2023-09-24T18:55:50.550546+0000","172.21.15.16:0/349446544":"2023-09-24T18:55:50.550546+0000","172.21.15.16:6800/2756103674":"2023-09-24T18:56:20.574625+0000","172.21.15.16:6801/2615282850":"2023-09-24T18:55:50.550546+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T19:03:14.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:14 smithi016 bash[33624]: audit 2023-09-23T19:03:14.324951+0000 mon.a (mon.0) 536 : audit [DBG] from='client.? 172.21.15.16:0/3130827727' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T19:03:14.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:14 smithi016 bash[27237]: audit 2023-09-23T19:03:14.324951+0000 mon.a (mon.0) 536 : audit [DBG] from='client.? 172.21.15.16:0/3130827727' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T19:03:14.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:14 smithi092 bash[27556]: audit 2023-09-23T19:03:14.324951+0000 mon.a (mon.0) 536 : audit [DBG] from='client.? 172.21.15.16:0/3130827727' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T19:03:15.042 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': 'device_health_metrics', 'create_time': '2023-09-23T18:59:33.315416+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr_devicehealth': {}}}] 2023-09-23T19:03:15.042 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd pool get device_health_metrics pg_num 2023-09-23T19:03:15.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:15 smithi016 bash[27237]: cluster 2023-09-23T19:03:14.687274+0000 mgr.y (mgr.14146) 259 : cluster [DBG] pgmap v247: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:15.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:15 smithi016 bash[33624]: cluster 2023-09-23T19:03:14.687274+0000 mgr.y (mgr.14146) 259 : cluster [DBG] pgmap v247: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:15.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:15 smithi092 bash[27556]: cluster 2023-09-23T19:03:14.687274+0000 mgr.y (mgr.14146) 259 : cluster [DBG] pgmap v247: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:18.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:17 smithi016 bash[27237]: cluster 2023-09-23T19:03:16.687848+0000 mgr.y (mgr.14146) 260 : cluster [DBG] pgmap v248: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:18.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:17 smithi016 bash[33624]: cluster 2023-09-23T19:03:16.687848+0000 mgr.y (mgr.14146) 260 : cluster [DBG] pgmap v248: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:18.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:17 smithi092 bash[27556]: cluster 2023-09-23T19:03:16.687848+0000 mgr.y (mgr.14146) 260 : cluster [DBG] pgmap v248: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:19.805 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:03:20.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:19 smithi016 bash[33624]: cluster 2023-09-23T19:03:18.688469+0000 mgr.y (mgr.14146) 261 : cluster [DBG] pgmap v249: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:20.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:19 smithi016 bash[27237]: cluster 2023-09-23T19:03:18.688469+0000 mgr.y (mgr.14146) 261 : cluster [DBG] pgmap v249: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:20.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:19 smithi092 bash[27556]: cluster 2023-09-23T19:03:18.688469+0000 mgr.y (mgr.14146) 261 : cluster [DBG] pgmap v249: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:21.150 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:20 smithi016 bash[33624]: cluster 2023-09-23T19:03:20.689046+0000 mgr.y (mgr.14146) 262 : cluster [DBG] pgmap v250: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:21.150 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:20 smithi016 bash[27237]: cluster 2023-09-23T19:03:20.689046+0000 mgr.y (mgr.14146) 262 : cluster [DBG] pgmap v250: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:21.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:20 smithi092 bash[27556]: cluster 2023-09-23T19:03:20.689046+0000 mgr.y (mgr.14146) 262 : cluster [DBG] pgmap v250: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:22.872 INFO:teuthology.orchestra.run.smithi016.stdout:pg_num: 1 2023-09-23T19:03:23.564 INFO:tasks.cephadm:Adding prometheus.a on smithi092 2023-09-23T19:03:23.564 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph orch apply prometheus '1;smithi092=a' 2023-09-23T19:03:24.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:23 smithi016 bash[27237]: cluster 2023-09-23T19:03:22.689557+0000 mgr.y (mgr.14146) 263 : cluster [DBG] pgmap v251: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:24.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:23 smithi016 bash[27237]: audit 2023-09-23T19:03:22.868907+0000 mon.c (mon.2) 12 : audit [DBG] from='client.? 172.21.15.16:0/1302092465' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2023-09-23T19:03:24.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:23 smithi016 bash[33624]: cluster 2023-09-23T19:03:22.689557+0000 mgr.y (mgr.14146) 263 : cluster [DBG] pgmap v251: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:24.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:23 smithi016 bash[33624]: audit 2023-09-23T19:03:22.868907+0000 mon.c (mon.2) 12 : audit [DBG] from='client.? 172.21.15.16:0/1302092465' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2023-09-23T19:03:24.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:23 smithi092 bash[27556]: cluster 2023-09-23T19:03:22.689557+0000 mgr.y (mgr.14146) 263 : cluster [DBG] pgmap v251: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:24.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:23 smithi092 bash[27556]: audit 2023-09-23T19:03:22.868907+0000 mon.c (mon.2) 12 : audit [DBG] from='client.? 172.21.15.16:0/1302092465' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2023-09-23T19:03:26.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:25 smithi016 bash[27237]: cluster 2023-09-23T19:03:24.690098+0000 mgr.y (mgr.14146) 264 : cluster [DBG] pgmap v252: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:26.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:25 smithi016 bash[33624]: cluster 2023-09-23T19:03:24.690098+0000 mgr.y (mgr.14146) 264 : cluster [DBG] pgmap v252: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:26.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:25 smithi092 bash[27556]: cluster 2023-09-23T19:03:24.690098+0000 mgr.y (mgr.14146) 264 : cluster [DBG] pgmap v252: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:26.637 INFO:teuthology.orchestra.run.smithi092.stdout:Scheduled prometheus update... 2023-09-23T19:03:27.419 DEBUG:teuthology.orchestra.run.smithi092:prometheus.a> sudo journalctl -f -n 0 -u ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@prometheus.a.service 2023-09-23T19:03:27.421 INFO:tasks.cephadm:Adding node-exporter.a on smithi016 2023-09-23T19:03:27.421 INFO:tasks.cephadm:Adding node-exporter.b on smithi092 2023-09-23T19:03:27.421 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph orch apply node-exporter '2;smithi016=a;smithi092=b' 2023-09-23T19:03:27.622 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-09-23T19:03:27.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:27 smithi092 bash[27556]: audit 2023-09-23T19:03:26.621767+0000 mgr.y (mgr.14146) 265 : audit [DBG] from='client.24346 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi092=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:03:27.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:27 smithi092 bash[27556]: cephadm 2023-09-23T19:03:26.625167+0000 mgr.y (mgr.14146) 266 : cephadm [INF] Saving service prometheus spec with placement smithi092=a;count:1 2023-09-23T19:03:27.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:27 smithi092 bash[27556]: audit 2023-09-23T19:03:26.633876+0000 mon.a (mon.0) 537 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:03:27.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:27 smithi092 bash[27556]: audit 2023-09-23T19:03:26.676419+0000 mon.a (mon.0) 538 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:03:27.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:27 smithi092 bash[27556]: cluster 2023-09-23T19:03:26.690657+0000 mgr.y (mgr.14146) 267 : cluster [DBG] pgmap v253: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:27.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:27 smithi092 bash[27556]: audit 2023-09-23T19:03:26.969863+0000 mon.a (mon.0) 539 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:03:27.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:27 smithi092 bash[27556]: audit 2023-09-23T19:03:26.971222+0000 mon.a (mon.0) 540 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:03:27.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:27 smithi092 bash[27556]: audit 2023-09-23T19:03:26.980365+0000 mon.a (mon.0) 541 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:03:27.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:27 smithi092 bash[27556]: audit 2023-09-23T19:03:26.987299+0000 mon.a (mon.0) 542 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-09-23T19:03:28.030 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:27 smithi016 bash[27237]: audit 2023-09-23T19:03:26.621767+0000 mgr.y (mgr.14146) 265 : audit [DBG] from='client.24346 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi092=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:03:28.030 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:27 smithi016 bash[27237]: cephadm 2023-09-23T19:03:26.625167+0000 mgr.y (mgr.14146) 266 : cephadm [INF] Saving service prometheus spec with placement smithi092=a;count:1 2023-09-23T19:03:28.030 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:27 smithi016 bash[27237]: audit 2023-09-23T19:03:26.633876+0000 mon.a (mon.0) 537 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:03:28.030 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:27 smithi016 bash[27237]: audit 2023-09-23T19:03:26.676419+0000 mon.a (mon.0) 538 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:03:28.030 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:27 smithi016 bash[27237]: cluster 2023-09-23T19:03:26.690657+0000 mgr.y (mgr.14146) 267 : cluster [DBG] pgmap v253: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:28.031 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:27 smithi016 bash[27237]: audit 2023-09-23T19:03:26.969863+0000 mon.a (mon.0) 539 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:03:28.031 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:27 smithi016 bash[27237]: audit 2023-09-23T19:03:26.971222+0000 mon.a (mon.0) 540 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:03:28.031 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:27 smithi016 bash[27237]: audit 2023-09-23T19:03:26.980365+0000 mon.a (mon.0) 541 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:03:28.031 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:27 smithi016 bash[27237]: audit 2023-09-23T19:03:26.987299+0000 mon.a (mon.0) 542 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-09-23T19:03:28.031 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:27 smithi016 bash[33624]: audit 2023-09-23T19:03:26.621767+0000 mgr.y (mgr.14146) 265 : audit [DBG] from='client.24346 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi092=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:03:28.031 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:27 smithi016 bash[33624]: cephadm 2023-09-23T19:03:26.625167+0000 mgr.y (mgr.14146) 266 : cephadm [INF] Saving service prometheus spec with placement smithi092=a;count:1 2023-09-23T19:03:28.031 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:27 smithi016 bash[33624]: audit 2023-09-23T19:03:26.633876+0000 mon.a (mon.0) 537 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:03:28.031 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:27 smithi016 bash[33624]: audit 2023-09-23T19:03:26.676419+0000 mon.a (mon.0) 538 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:03:28.031 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:27 smithi016 bash[33624]: cluster 2023-09-23T19:03:26.690657+0000 mgr.y (mgr.14146) 267 : cluster [DBG] pgmap v253: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:28.032 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:27 smithi016 bash[33624]: audit 2023-09-23T19:03:26.969863+0000 mon.a (mon.0) 539 : audit [DBG] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:03:28.032 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:27 smithi016 bash[33624]: audit 2023-09-23T19:03:26.971222+0000 mon.a (mon.0) 540 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:03:28.032 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:27 smithi016 bash[33624]: audit 2023-09-23T19:03:26.980365+0000 mon.a (mon.0) 541 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' 2023-09-23T19:03:28.032 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:27 smithi016 bash[33624]: audit 2023-09-23T19:03:26.987299+0000 mon.a (mon.0) 542 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-09-23T19:03:28.285 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:03:28 smithi016 bash[27503]: ignoring --setuser ceph since I am not root 2023-09-23T19:03:28.285 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:03:28 smithi016 bash[27503]: ignoring --setgroup ceph since I am not root 2023-09-23T19:03:28.285 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:03:28 smithi016 bash[27503]: debug 2023-09-23T19:03:28.034+0000 7f0eaf98e700 1 -- 172.21.15.16:0/2237694957 <== mon.0 v2:172.21.15.16:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5627682ac4e0 con 0x5627683a0c00 2023-09-23T19:03:28.285 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:03:28 smithi016 bash[27503]: debug 2023-09-23T19:03:28.194+0000 7f0ebfb55680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T19:03:28.287 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:03:28 smithi092 bash[28593]: ignoring --setuser ceph since I am not root 2023-09-23T19:03:28.287 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:03:28 smithi092 bash[28593]: ignoring --setgroup ceph since I am not root 2023-09-23T19:03:28.287 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:03:28 smithi092 bash[28593]: debug 2023-09-23T19:03:28.028+0000 7f2262962700 1 -- 172.21.15.92:0/2371616679 <== mon.1 v2:172.21.15.92:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55805d0724e0 con 0x55805d167000 2023-09-23T19:03:28.288 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:03:28 smithi092 bash[28593]: debug 2023-09-23T19:03:28.192+0000 7f2272b29680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T19:03:28.609 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:03:28 smithi092 bash[28593]: debug 2023-09-23T19:03:28.296+0000 7f2272b29680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T19:03:28.778 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:03:28 smithi016 bash[27503]: debug 2023-09-23T19:03:28.302+0000 7f0ebfb55680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T19:03:28.778 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:03:28 smithi016 bash[27503]: debug 2023-09-23T19:03:28.770+0000 7f0ebfb55680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T19:03:28.984 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:03:28 smithi092 bash[28593]: debug 2023-09-23T19:03:28.760+0000 7f2272b29680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T19:03:29.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:28 smithi016 bash[27237]: audit 2023-09-23T19:03:27.988587+0000 mon.a (mon.0) 543 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-09-23T19:03:29.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:28 smithi016 bash[27237]: cluster 2023-09-23T19:03:27.988666+0000 mon.a (mon.0) 544 : cluster [DBG] mgrmap e15: y(active, since 7m), standbys: x 2023-09-23T19:03:29.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:28 smithi016 bash[33624]: audit 2023-09-23T19:03:27.988587+0000 mon.a (mon.0) 543 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-09-23T19:03:29.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:28 smithi016 bash[33624]: cluster 2023-09-23T19:03:27.988666+0000 mon.a (mon.0) 544 : cluster [DBG] mgrmap e15: y(active, since 7m), standbys: x 2023-09-23T19:03:29.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:28 smithi092 bash[27556]: audit 2023-09-23T19:03:27.988587+0000 mon.a (mon.0) 543 : audit [INF] from='mgr.14146 172.21.15.16:0/3506904000' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-09-23T19:03:29.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:28 smithi092 bash[27556]: cluster 2023-09-23T19:03:27.988666+0000 mon.a (mon.0) 544 : cluster [DBG] mgrmap e15: y(active, since 7m), standbys: x 2023-09-23T19:03:30.035 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:03:29 smithi016 bash[27503]: debug 2023-09-23T19:03:29.734+0000 7f0ebfb55680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T19:03:30.035 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:03:29 smithi016 bash[27503]: debug 2023-09-23T19:03:29.834+0000 7f0ebfb55680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T19:03:30.037 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:03:29 smithi092 bash[28593]: debug 2023-09-23T19:03:29.684+0000 7f2272b29680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T19:03:30.037 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:03:29 smithi092 bash[28593]: debug 2023-09-23T19:03:29.784+0000 7f2272b29680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T19:03:30.535 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:03:30 smithi016 bash[27503]: debug 2023-09-23T19:03:30.094+0000 7f0ebfb55680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T19:03:30.537 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:03:30 smithi092 bash[28593]: debug 2023-09-23T19:03:30.064+0000 7f2272b29680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T19:03:31.501 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:03:31 smithi092 bash[28593]: debug 2023-09-23T19:03:31.236+0000 7f2272b29680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T19:03:31.501 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:03:31 smithi092 bash[28593]: debug 2023-09-23T19:03:31.396+0000 7f2272b29680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T19:03:31.563 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:03:31 smithi016 bash[27503]: debug 2023-09-23T19:03:31.282+0000 7f0ebfb55680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T19:03:31.563 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:03:31 smithi016 bash[27503]: debug 2023-09-23T19:03:31.454+0000 7f0ebfb55680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T19:03:31.788 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:03:31 smithi092 bash[28593]: debug 2023-09-23T19:03:31.496+0000 7f2272b29680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T19:03:31.788 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:03:31 smithi092 bash[28593]: debug 2023-09-23T19:03:31.612+0000 7f2272b29680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T19:03:31.914 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:03:31 smithi016 bash[27503]: debug 2023-09-23T19:03:31.558+0000 7f0ebfb55680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T19:03:31.914 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:03:31 smithi016 bash[27503]: debug 2023-09-23T19:03:31.678+0000 7f0ebfb55680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T19:03:32.202 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:03:31 smithi092 bash[28593]: debug 2023-09-23T19:03:31.832+0000 7f2272b29680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T19:03:32.284 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:03:31 smithi016 bash[27503]: debug 2023-09-23T19:03:31.906+0000 7f0ebfb55680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T19:03:32.538 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:03:32 smithi092 bash[28593]: debug 2023-09-23T19:03:32.200+0000 7f2272b29680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T19:03:32.538 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:03:32 smithi092 bash[28593]: debug 2023-09-23T19:03:32.324+0000 7f2272b29680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T19:03:32.785 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:03:32 smithi016 bash[27503]: debug 2023-09-23T19:03:32.298+0000 7f0ebfb55680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T19:03:32.785 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:03:32 smithi016 bash[27503]: debug 2023-09-23T19:03:32.426+0000 7f0ebfb55680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T19:03:33.538 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:03:33 smithi092 bash[28593]: debug 2023-09-23T19:03:33.196+0000 7f2272b29680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T19:03:33.538 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:03:33 smithi092 bash[28593]: debug 2023-09-23T19:03:33.296+0000 7f2272b29680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T19:03:33.538 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:03:33 smithi092 bash[28593]: debug 2023-09-23T19:03:33.412+0000 7f2272b29680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T19:03:33.785 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:03:33 smithi016 bash[27503]: debug 2023-09-23T19:03:33.330+0000 7f0ebfb55680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T19:03:33.785 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:03:33 smithi016 bash[27503]: debug 2023-09-23T19:03:33.438+0000 7f0ebfb55680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T19:03:33.785 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:03:33 smithi016 bash[27503]: debug 2023-09-23T19:03:33.562+0000 7f0ebfb55680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T19:03:33.939 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:03:33 smithi092 bash[28593]: debug 2023-09-23T19:03:33.636+0000 7f2272b29680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T19:03:34.109 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:03:33 smithi016 bash[27503]: debug 2023-09-23T19:03:33.798+0000 7f0ebfb55680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T19:03:34.280 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:03:33 smithi092 bash[28593]: debug 2023-09-23T19:03:33.936+0000 7f2272b29680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T19:03:34.280 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:03:34 smithi092 bash[28593]: debug 2023-09-23T19:03:34.104+0000 7f2272b29680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T19:03:34.280 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:03:34 smithi092 bash[28593]: debug 2023-09-23T19:03:34.276+0000 7f2272b29680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T19:03:34.482 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:03:34 smithi016 bash[27503]: debug 2023-09-23T19:03:34.102+0000 7f0ebfb55680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T19:03:34.482 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:03:34 smithi016 bash[27503]: debug 2023-09-23T19:03:34.286+0000 7f0ebfb55680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T19:03:34.537 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:03:34 smithi092 bash[28593]: debug 2023-09-23T19:03:34.492+0000 7f2272b29680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T19:03:34.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:34 smithi016 bash[33624]: cluster 2023-09-23T19:03:34.619042+0000 mon.a (mon.0) 545 : cluster [DBG] Standby manager daemon x restarted 2023-09-23T19:03:34.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:34 smithi016 bash[33624]: cluster 2023-09-23T19:03:34.619184+0000 mon.a (mon.0) 546 : cluster [DBG] Standby manager daemon x started 2023-09-23T19:03:34.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:34 smithi016 bash[33624]: audit 2023-09-23T19:03:34.628129+0000 mon.b (mon.1) 23 : audit [DBG] from='mgr.? 172.21.15.92:0/3274547196' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T19:03:34.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:34 smithi016 bash[33624]: audit 2023-09-23T19:03:34.630219+0000 mon.b (mon.1) 24 : audit [DBG] from='mgr.? 172.21.15.92:0/3274547196' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T19:03:34.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:34 smithi016 bash[33624]: audit 2023-09-23T19:03:34.633064+0000 mon.b (mon.1) 25 : audit [DBG] from='mgr.? 172.21.15.92:0/3274547196' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T19:03:34.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:34 smithi016 bash[33624]: audit 2023-09-23T19:03:34.634820+0000 mon.b (mon.1) 26 : audit [DBG] from='mgr.? 172.21.15.92:0/3274547196' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T19:03:34.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:34 smithi016 bash[27237]: cluster 2023-09-23T19:03:34.619042+0000 mon.a (mon.0) 545 : cluster [DBG] Standby manager daemon x restarted 2023-09-23T19:03:34.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:34 smithi016 bash[27237]: cluster 2023-09-23T19:03:34.619184+0000 mon.a (mon.0) 546 : cluster [DBG] Standby manager daemon x started 2023-09-23T19:03:34.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:34 smithi016 bash[27237]: audit 2023-09-23T19:03:34.628129+0000 mon.b (mon.1) 23 : audit [DBG] from='mgr.? 172.21.15.92:0/3274547196' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T19:03:34.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:34 smithi016 bash[27237]: audit 2023-09-23T19:03:34.630219+0000 mon.b (mon.1) 24 : audit [DBG] from='mgr.? 172.21.15.92:0/3274547196' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T19:03:34.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:34 smithi016 bash[27237]: audit 2023-09-23T19:03:34.633064+0000 mon.b (mon.1) 25 : audit [DBG] from='mgr.? 172.21.15.92:0/3274547196' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T19:03:34.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:34 smithi016 bash[27237]: audit 2023-09-23T19:03:34.634820+0000 mon.b (mon.1) 26 : audit [DBG] from='mgr.? 172.21.15.92:0/3274547196' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T19:03:34.786 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:03:34 smithi016 bash[27503]: debug 2023-09-23T19:03:34.474+0000 7f0ebfb55680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T19:03:34.786 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:03:34 smithi016 bash[27503]: debug 2023-09-23T19:03:34.706+0000 7f0ebfb55680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T19:03:35.038 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:03:34 smithi092 bash[28593]: debug 2023-09-23T19:03:34.600+0000 7f2272b29680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T19:03:35.038 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:03:34 smithi092 bash[28593]: [23/Sep/2023:19:03:34] ENGINE Bus STARTING 2023-09-23T19:03:35.038 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:03:34 smithi092 bash[28593]: CherryPy Checker: 2023-09-23T19:03:35.038 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:03:34 smithi092 bash[28593]: The Application mounted at '' has an empty config. 2023-09-23T19:03:35.038 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:03:34 smithi092 bash[28593]: [23/Sep/2023:19:03:34] ENGINE Serving on http://:::9283 2023-09-23T19:03:35.038 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:03:34 smithi092 bash[28593]: [23/Sep/2023:19:03:34] ENGINE Bus STARTED 2023-09-23T19:03:35.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:34 smithi092 bash[27556]: cluster 2023-09-23T19:03:34.619042+0000 mon.a (mon.0) 545 : cluster [DBG] Standby manager daemon x restarted 2023-09-23T19:03:35.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:34 smithi092 bash[27556]: cluster 2023-09-23T19:03:34.619184+0000 mon.a (mon.0) 546 : cluster [DBG] Standby manager daemon x started 2023-09-23T19:03:35.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:34 smithi092 bash[27556]: audit 2023-09-23T19:03:34.628129+0000 mon.b (mon.1) 23 : audit [DBG] from='mgr.? 172.21.15.92:0/3274547196' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T19:03:35.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:34 smithi092 bash[27556]: audit 2023-09-23T19:03:34.630219+0000 mon.b (mon.1) 24 : audit [DBG] from='mgr.? 172.21.15.92:0/3274547196' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T19:03:35.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:34 smithi092 bash[27556]: audit 2023-09-23T19:03:34.633064+0000 mon.b (mon.1) 25 : audit [DBG] from='mgr.? 172.21.15.92:0/3274547196' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T19:03:35.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:34 smithi092 bash[27556]: audit 2023-09-23T19:03:34.634820+0000 mon.b (mon.1) 26 : audit [DBG] from='mgr.? 172.21.15.92:0/3274547196' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T19:03:35.107 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:03:34 smithi016 bash[27503]: debug 2023-09-23T19:03:34.822+0000 7f0ebfb55680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T19:03:35.535 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[27503]: Warning: Permanently added '172.21.15.92' (ECDSA) to the list of known hosts. 2023-09-23T19:03:35.535 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[27503]: [23/Sep/2023:19:03:35] ENGINE Bus STARTING 2023-09-23T19:03:35.535 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[27503]: Warning: Permanently added '172.21.15.16' (ECDSA) to the list of known hosts. 2023-09-23T19:03:35.535 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[27503]: CherryPy Checker: 2023-09-23T19:03:35.535 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[27503]: The Application mounted at '' has an empty config. 2023-09-23T19:03:35.535 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[27503]: [23/Sep/2023:19:03:35] ENGINE Serving on http://:::9283 2023-09-23T19:03:35.535 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[27503]: [23/Sep/2023:19:03:35] ENGINE Bus STARTED 2023-09-23T19:03:35.921 INFO:teuthology.orchestra.run.smithi092.stdout:Scheduled node-exporter update... 2023-09-23T19:03:35.979 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:35 smithi092 bash[27556]: cluster 2023-09-23T19:03:34.679975+0000 mon.a (mon.0) 547 : cluster [DBG] mgrmap e16: y(active, since 7m), standbys: x 2023-09-23T19:03:35.979 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:35 smithi092 bash[27556]: cluster 2023-09-23T19:03:34.846370+0000 mon.a (mon.0) 548 : cluster [INF] Active manager daemon y restarted 2023-09-23T19:03:35.979 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:35 smithi092 bash[27556]: cluster 2023-09-23T19:03:34.847266+0000 mon.a (mon.0) 549 : cluster [INF] Activating manager daemon y 2023-09-23T19:03:35.979 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:35 smithi092 bash[27556]: cluster 2023-09-23T19:03:34.859148+0000 mon.a (mon.0) 550 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2023-09-23T19:03:35.979 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:35 smithi092 bash[27556]: cluster 2023-09-23T19:03:34.859545+0000 mon.a (mon.0) 551 : cluster [DBG] mgrmap e17: y(active, starting, since 0.0125965s), standbys: x 2023-09-23T19:03:35.979 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:35 smithi092 bash[27556]: audit 2023-09-23T19:03:34.860790+0000 mon.c (mon.2) 13 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:03:35.979 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:35 smithi092 bash[27556]: audit 2023-09-23T19:03:34.861149+0000 mon.c (mon.2) 14 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:03:35.979 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:35 smithi092 bash[27556]: audit 2023-09-23T19:03:34.861609+0000 mon.c (mon.2) 15 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:03:35.980 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:35 smithi092 bash[27556]: audit 2023-09-23T19:03:34.863002+0000 mon.c (mon.2) 16 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T19:03:35.980 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:35 smithi092 bash[27556]: audit 2023-09-23T19:03:34.863219+0000 mon.c (mon.2) 17 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T19:03:35.980 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:35 smithi092 bash[27556]: audit 2023-09-23T19:03:34.863453+0000 mon.c (mon.2) 18 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T19:03:35.980 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:35 smithi092 bash[27556]: audit 2023-09-23T19:03:34.863783+0000 mon.c (mon.2) 19 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T19:03:35.980 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:35 smithi092 bash[27556]: audit 2023-09-23T19:03:34.864008+0000 mon.c (mon.2) 20 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T19:03:35.980 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:35 smithi092 bash[27556]: audit 2023-09-23T19:03:34.864351+0000 mon.c (mon.2) 21 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:03:35.980 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:35 smithi092 bash[27556]: audit 2023-09-23T19:03:34.864649+0000 mon.c (mon.2) 22 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T19:03:35.980 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:35 smithi092 bash[27556]: audit 2023-09-23T19:03:34.864939+0000 mon.c (mon.2) 23 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:03:35.980 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:35 smithi092 bash[27556]: audit 2023-09-23T19:03:34.865170+0000 mon.c (mon.2) 24 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:03:35.980 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:35 smithi092 bash[27556]: audit 2023-09-23T19:03:34.865443+0000 mon.c (mon.2) 25 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:03:35.980 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:35 smithi092 bash[27556]: audit 2023-09-23T19:03:34.865754+0000 mon.c (mon.2) 26 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T19:03:35.980 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:35 smithi092 bash[27556]: audit 2023-09-23T19:03:34.865977+0000 mon.c (mon.2) 27 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T19:03:35.981 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:35 smithi092 bash[27556]: audit 2023-09-23T19:03:34.866470+0000 mon.c (mon.2) 28 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T19:03:35.981 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:35 smithi092 bash[27556]: cluster 2023-09-23T19:03:34.893955+0000 mon.a (mon.0) 552 : cluster [INF] Manager daemon y is now available 2023-09-23T19:03:35.981 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:35 smithi092 bash[27556]: audit 2023-09-23T19:03:35.035971+0000 mon.c (mon.2) 29 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:03:35.981 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:35 smithi092 bash[27556]: audit 2023-09-23T19:03:35.039070+0000 mon.c (mon.2) 30 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:03:35.981 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:35 smithi092 bash[27556]: audit 2023-09-23T19:03:35.098430+0000 mon.a (mon.0) 553 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:03:35.981 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:35 smithi092 bash[27556]: audit 2023-09-23T19:03:35.116636+0000 mon.c (mon.2) 31 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T19:03:35.981 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:35 smithi092 bash[27556]: audit 2023-09-23T19:03:35.117223+0000 mon.a (mon.0) 554 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T19:03:35.981 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:35 smithi092 bash[27556]: audit 2023-09-23T19:03:35.130468+0000 mon.c (mon.2) 32 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T19:03:35.981 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:35 smithi092 bash[27556]: audit 2023-09-23T19:03:35.131152+0000 mon.a (mon.0) 555 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T19:03:36.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[27237]: cluster 2023-09-23T19:03:34.679975+0000 mon.a (mon.0) 547 : cluster [DBG] mgrmap e16: y(active, since 7m), standbys: x 2023-09-23T19:03:36.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[27237]: cluster 2023-09-23T19:03:34.846370+0000 mon.a (mon.0) 548 : cluster [INF] Active manager daemon y restarted 2023-09-23T19:03:36.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[27237]: cluster 2023-09-23T19:03:34.847266+0000 mon.a (mon.0) 549 : cluster [INF] Activating manager daemon y 2023-09-23T19:03:36.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[27237]: cluster 2023-09-23T19:03:34.859148+0000 mon.a (mon.0) 550 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2023-09-23T19:03:36.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[27237]: cluster 2023-09-23T19:03:34.859545+0000 mon.a (mon.0) 551 : cluster [DBG] mgrmap e17: y(active, starting, since 0.0125965s), standbys: x 2023-09-23T19:03:36.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[27237]: audit 2023-09-23T19:03:34.860790+0000 mon.c (mon.2) 13 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:03:36.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[27237]: audit 2023-09-23T19:03:34.861149+0000 mon.c (mon.2) 14 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:03:36.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[27237]: audit 2023-09-23T19:03:34.861609+0000 mon.c (mon.2) 15 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:03:36.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[27237]: audit 2023-09-23T19:03:34.863002+0000 mon.c (mon.2) 16 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T19:03:36.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[27237]: audit 2023-09-23T19:03:34.863219+0000 mon.c (mon.2) 17 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T19:03:36.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[27237]: audit 2023-09-23T19:03:34.863453+0000 mon.c (mon.2) 18 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T19:03:36.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[27237]: audit 2023-09-23T19:03:34.863783+0000 mon.c (mon.2) 19 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T19:03:36.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[27237]: audit 2023-09-23T19:03:34.864008+0000 mon.c (mon.2) 20 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T19:03:36.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[27237]: audit 2023-09-23T19:03:34.864351+0000 mon.c (mon.2) 21 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:03:36.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[27237]: audit 2023-09-23T19:03:34.864649+0000 mon.c (mon.2) 22 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T19:03:36.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[27237]: audit 2023-09-23T19:03:34.864939+0000 mon.c (mon.2) 23 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:03:36.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[27237]: audit 2023-09-23T19:03:34.865170+0000 mon.c (mon.2) 24 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:03:36.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[27237]: audit 2023-09-23T19:03:34.865443+0000 mon.c (mon.2) 25 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:03:36.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[27237]: audit 2023-09-23T19:03:34.865754+0000 mon.c (mon.2) 26 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T19:03:36.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[27237]: audit 2023-09-23T19:03:34.865977+0000 mon.c (mon.2) 27 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T19:03:36.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[33624]: cluster 2023-09-23T19:03:34.679975+0000 mon.a (mon.0) 547 : cluster [DBG] mgrmap e16: y(active, since 7m), standbys: x 2023-09-23T19:03:36.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[33624]: cluster 2023-09-23T19:03:34.846370+0000 mon.a (mon.0) 548 : cluster [INF] Active manager daemon y restarted 2023-09-23T19:03:36.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[33624]: cluster 2023-09-23T19:03:34.847266+0000 mon.a (mon.0) 549 : cluster [INF] Activating manager daemon y 2023-09-23T19:03:36.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[33624]: cluster 2023-09-23T19:03:34.859148+0000 mon.a (mon.0) 550 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2023-09-23T19:03:36.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[33624]: cluster 2023-09-23T19:03:34.859545+0000 mon.a (mon.0) 551 : cluster [DBG] mgrmap e17: y(active, starting, since 0.0125965s), standbys: x 2023-09-23T19:03:36.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[33624]: audit 2023-09-23T19:03:34.860790+0000 mon.c (mon.2) 13 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:03:36.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[33624]: audit 2023-09-23T19:03:34.861149+0000 mon.c (mon.2) 14 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:03:36.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[33624]: audit 2023-09-23T19:03:34.861609+0000 mon.c (mon.2) 15 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:03:36.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[33624]: audit 2023-09-23T19:03:34.863002+0000 mon.c (mon.2) 16 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T19:03:36.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[33624]: audit 2023-09-23T19:03:34.863219+0000 mon.c (mon.2) 17 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T19:03:36.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[33624]: audit 2023-09-23T19:03:34.863453+0000 mon.c (mon.2) 18 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T19:03:36.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[33624]: audit 2023-09-23T19:03:34.863783+0000 mon.c (mon.2) 19 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T19:03:36.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[33624]: audit 2023-09-23T19:03:34.864008+0000 mon.c (mon.2) 20 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T19:03:36.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[33624]: audit 2023-09-23T19:03:34.864351+0000 mon.c (mon.2) 21 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:03:36.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[33624]: audit 2023-09-23T19:03:34.864649+0000 mon.c (mon.2) 22 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T19:03:36.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[33624]: audit 2023-09-23T19:03:34.864939+0000 mon.c (mon.2) 23 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:03:36.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[33624]: audit 2023-09-23T19:03:34.865170+0000 mon.c (mon.2) 24 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:03:36.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[33624]: audit 2023-09-23T19:03:34.865443+0000 mon.c (mon.2) 25 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:03:36.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[33624]: audit 2023-09-23T19:03:34.865754+0000 mon.c (mon.2) 26 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T19:03:36.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[33624]: audit 2023-09-23T19:03:34.865977+0000 mon.c (mon.2) 27 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T19:03:36.040 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[27237]: audit 2023-09-23T19:03:34.866470+0000 mon.c (mon.2) 28 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T19:03:36.040 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[27237]: cluster 2023-09-23T19:03:34.893955+0000 mon.a (mon.0) 552 : cluster [INF] Manager daemon y is now available 2023-09-23T19:03:36.040 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[27237]: audit 2023-09-23T19:03:35.035971+0000 mon.c (mon.2) 29 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:03:36.040 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[27237]: audit 2023-09-23T19:03:35.039070+0000 mon.c (mon.2) 30 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:03:36.040 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[27237]: audit 2023-09-23T19:03:35.098430+0000 mon.a (mon.0) 553 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:03:36.040 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[27237]: audit 2023-09-23T19:03:35.116636+0000 mon.c (mon.2) 31 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T19:03:36.040 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[27237]: audit 2023-09-23T19:03:35.117223+0000 mon.a (mon.0) 554 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T19:03:36.040 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[27237]: audit 2023-09-23T19:03:35.130468+0000 mon.c (mon.2) 32 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T19:03:36.040 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[27237]: audit 2023-09-23T19:03:35.131152+0000 mon.a (mon.0) 555 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T19:03:36.041 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[33624]: audit 2023-09-23T19:03:34.866470+0000 mon.c (mon.2) 28 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T19:03:36.041 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[33624]: cluster 2023-09-23T19:03:34.893955+0000 mon.a (mon.0) 552 : cluster [INF] Manager daemon y is now available 2023-09-23T19:03:36.041 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[33624]: audit 2023-09-23T19:03:35.035971+0000 mon.c (mon.2) 29 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:03:36.041 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[33624]: audit 2023-09-23T19:03:35.039070+0000 mon.c (mon.2) 30 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:03:36.041 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[33624]: audit 2023-09-23T19:03:35.098430+0000 mon.a (mon.0) 553 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:03:36.041 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[33624]: audit 2023-09-23T19:03:35.116636+0000 mon.c (mon.2) 31 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T19:03:36.041 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[33624]: audit 2023-09-23T19:03:35.117223+0000 mon.a (mon.0) 554 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T19:03:36.041 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[33624]: audit 2023-09-23T19:03:35.130468+0000 mon.c (mon.2) 32 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T19:03:36.041 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:35 smithi016 bash[33624]: audit 2023-09-23T19:03:35.131152+0000 mon.a (mon.0) 555 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T19:03:36.771 DEBUG:teuthology.orchestra.run.smithi016:node-exporter.a> sudo journalctl -f -n 0 -u ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@node-exporter.a.service 2023-09-23T19:03:36.774 DEBUG:teuthology.orchestra.run.smithi092:node-exporter.b> sudo journalctl -f -n 0 -u ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@node-exporter.b.service 2023-09-23T19:03:36.776 INFO:tasks.cephadm:Adding alertmanager.a on smithi016 2023-09-23T19:03:36.776 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph orch apply alertmanager '1;smithi016=a' 2023-09-23T19:03:36.919 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-09-23T19:03:36.923 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-09-23T19:03:37.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:36 smithi016 bash[27237]: cluster 2023-09-23T19:03:35.902720+0000 mon.a (mon.0) 556 : cluster [DBG] mgrmap e18: y(active, since 1.05576s), standbys: x 2023-09-23T19:03:37.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:36 smithi016 bash[27237]: cephadm 2023-09-23T19:03:35.913536+0000 mgr.y (mgr.14454) 3 : cephadm [INF] Saving service node-exporter spec with placement smithi016=a;smithi092=b;count:2 2023-09-23T19:03:37.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:36 smithi016 bash[27237]: audit 2023-09-23T19:03:35.917804+0000 mon.a (mon.0) 557 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:03:37.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:36 smithi016 bash[33624]: cluster 2023-09-23T19:03:35.902720+0000 mon.a (mon.0) 556 : cluster [DBG] mgrmap e18: y(active, since 1.05576s), standbys: x 2023-09-23T19:03:37.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:36 smithi016 bash[33624]: cephadm 2023-09-23T19:03:35.913536+0000 mgr.y (mgr.14454) 3 : cephadm [INF] Saving service node-exporter spec with placement smithi016=a;smithi092=b;count:2 2023-09-23T19:03:37.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:36 smithi016 bash[33624]: audit 2023-09-23T19:03:35.917804+0000 mon.a (mon.0) 557 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:03:37.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:36 smithi092 bash[27556]: cluster 2023-09-23T19:03:35.902720+0000 mon.a (mon.0) 556 : cluster [DBG] mgrmap e18: y(active, since 1.05576s), standbys: x 2023-09-23T19:03:37.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:36 smithi092 bash[27556]: cephadm 2023-09-23T19:03:35.913536+0000 mgr.y (mgr.14454) 3 : cephadm [INF] Saving service node-exporter spec with placement smithi016=a;smithi092=b;count:2 2023-09-23T19:03:37.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:36 smithi092 bash[27556]: audit 2023-09-23T19:03:35.917804+0000 mon.a (mon.0) 557 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:03:38.227 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:37 smithi092 bash[27556]: cluster 2023-09-23T19:03:36.865039+0000 mgr.y (mgr.14454) 4 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:38.227 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:37 smithi092 bash[27556]: cluster 2023-09-23T19:03:36.926478+0000 mon.a (mon.0) 558 : cluster [DBG] mgrmap e19: y(active, since 2s), standbys: x 2023-09-23T19:03:38.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:37 smithi016 bash[27237]: cluster 2023-09-23T19:03:36.865039+0000 mgr.y (mgr.14454) 4 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:38.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:37 smithi016 bash[27237]: cluster 2023-09-23T19:03:36.926478+0000 mon.a (mon.0) 558 : cluster [DBG] mgrmap e19: y(active, since 2s), standbys: x 2023-09-23T19:03:38.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:37 smithi016 bash[33624]: cluster 2023-09-23T19:03:36.865039+0000 mgr.y (mgr.14454) 4 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:38.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:37 smithi016 bash[33624]: cluster 2023-09-23T19:03:36.926478+0000 mon.a (mon.0) 558 : cluster [DBG] mgrmap e19: y(active, since 2s), standbys: x 2023-09-23T19:03:39.871 INFO:teuthology.orchestra.run.smithi092.stdout:Scheduled alertmanager update... 2023-09-23T19:03:40.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:39 smithi016 bash[33624]: cluster 2023-09-23T19:03:38.865704+0000 mgr.y (mgr.14454) 5 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:40.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:39 smithi016 bash[33624]: audit 2023-09-23T19:03:39.866608+0000 mon.a (mon.0) 559 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:03:40.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:39 smithi016 bash[27237]: cluster 2023-09-23T19:03:38.865704+0000 mgr.y (mgr.14454) 5 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:40.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:39 smithi016 bash[27237]: audit 2023-09-23T19:03:39.866608+0000 mon.a (mon.0) 559 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:03:40.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:39 smithi092 bash[27556]: cluster 2023-09-23T19:03:38.865704+0000 mgr.y (mgr.14454) 5 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:40.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:39 smithi092 bash[27556]: audit 2023-09-23T19:03:39.866608+0000 mon.a (mon.0) 559 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:03:40.590 DEBUG:teuthology.orchestra.run.smithi016:alertmanager.a> sudo journalctl -f -n 0 -u ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@alertmanager.a.service 2023-09-23T19:03:40.593 INFO:tasks.cephadm:Adding grafana.a on smithi092 2023-09-23T19:03:40.593 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph orch apply grafana '1;smithi092=a' 2023-09-23T19:03:40.724 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-09-23T19:03:41.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:40 smithi016 bash[27237]: audit 2023-09-23T19:03:39.856570+0000 mgr.y (mgr.14454) 6 : audit [DBG] from='client.24382 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi016=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:03:41.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:40 smithi016 bash[27237]: cephadm 2023-09-23T19:03:39.859978+0000 mgr.y (mgr.14454) 7 : cephadm [INF] Saving service alertmanager spec with placement smithi016=a;count:1 2023-09-23T19:03:41.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:40 smithi016 bash[27237]: audit 2023-09-23T19:03:40.739307+0000 mon.a (mon.0) 560 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:03:41.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:40 smithi016 bash[33624]: audit 2023-09-23T19:03:39.856570+0000 mgr.y (mgr.14454) 6 : audit [DBG] from='client.24382 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi016=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:03:41.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:40 smithi016 bash[33624]: cephadm 2023-09-23T19:03:39.859978+0000 mgr.y (mgr.14454) 7 : cephadm [INF] Saving service alertmanager spec with placement smithi016=a;count:1 2023-09-23T19:03:41.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:40 smithi016 bash[33624]: audit 2023-09-23T19:03:40.739307+0000 mon.a (mon.0) 560 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:03:41.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:40 smithi092 bash[27556]: audit 2023-09-23T19:03:39.856570+0000 mgr.y (mgr.14454) 6 : audit [DBG] from='client.24382 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi016=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:03:41.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:40 smithi092 bash[27556]: cephadm 2023-09-23T19:03:39.859978+0000 mgr.y (mgr.14454) 7 : cephadm [INF] Saving service alertmanager spec with placement smithi016=a;count:1 2023-09-23T19:03:41.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:40 smithi092 bash[27556]: audit 2023-09-23T19:03:40.739307+0000 mon.a (mon.0) 560 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:03:42.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:41 smithi016 bash[27237]: cluster 2023-09-23T19:03:40.866192+0000 mgr.y (mgr.14454) 8 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:42.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:41 smithi016 bash[27237]: audit 2023-09-23T19:03:41.491103+0000 mon.a (mon.0) 561 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:03:42.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:41 smithi016 bash[27237]: audit 2023-09-23T19:03:41.779848+0000 mon.c (mon.2) 33 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:03:42.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:41 smithi016 bash[27237]: audit 2023-09-23T19:03:41.781825+0000 mon.c (mon.2) 34 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:03:42.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:41 smithi016 bash[27237]: audit 2023-09-23T19:03:41.845659+0000 mon.a (mon.0) 562 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:03:42.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:41 smithi016 bash[27237]: audit 2023-09-23T19:03:41.850137+0000 mon.a (mon.0) 563 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:03:42.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:41 smithi016 bash[27237]: audit 2023-09-23T19:03:41.856896+0000 mon.a (mon.0) 564 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:03:42.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:41 smithi016 bash[33624]: cluster 2023-09-23T19:03:40.866192+0000 mgr.y (mgr.14454) 8 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:42.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:41 smithi016 bash[33624]: audit 2023-09-23T19:03:41.491103+0000 mon.a (mon.0) 561 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:03:42.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:41 smithi016 bash[33624]: audit 2023-09-23T19:03:41.779848+0000 mon.c (mon.2) 33 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:03:42.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:41 smithi016 bash[33624]: audit 2023-09-23T19:03:41.781825+0000 mon.c (mon.2) 34 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:03:42.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:41 smithi016 bash[33624]: audit 2023-09-23T19:03:41.845659+0000 mon.a (mon.0) 562 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:03:42.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:41 smithi016 bash[33624]: audit 2023-09-23T19:03:41.850137+0000 mon.a (mon.0) 563 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:03:42.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:41 smithi016 bash[33624]: audit 2023-09-23T19:03:41.856896+0000 mon.a (mon.0) 564 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:03:42.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:41 smithi092 bash[27556]: cluster 2023-09-23T19:03:40.866192+0000 mgr.y (mgr.14454) 8 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:42.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:41 smithi092 bash[27556]: audit 2023-09-23T19:03:41.491103+0000 mon.a (mon.0) 561 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:03:42.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:41 smithi092 bash[27556]: audit 2023-09-23T19:03:41.779848+0000 mon.c (mon.2) 33 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:03:42.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:41 smithi092 bash[27556]: audit 2023-09-23T19:03:41.781825+0000 mon.c (mon.2) 34 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:03:42.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:41 smithi092 bash[27556]: audit 2023-09-23T19:03:41.845659+0000 mon.a (mon.0) 562 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:03:42.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:41 smithi092 bash[27556]: audit 2023-09-23T19:03:41.850137+0000 mon.a (mon.0) 563 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:03:42.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:41 smithi092 bash[27556]: audit 2023-09-23T19:03:41.856896+0000 mon.a (mon.0) 564 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:03:43.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:42 smithi016 bash[27237]: cephadm 2023-09-23T19:03:41.784714+0000 mgr.y (mgr.14454) 9 : cephadm [INF] Updating smithi016:/etc/ceph/ceph.conf 2023-09-23T19:03:43.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:42 smithi016 bash[27237]: cephadm 2023-09-23T19:03:41.785502+0000 mgr.y (mgr.14454) 10 : cephadm [INF] Updating smithi092:/etc/ceph/ceph.conf 2023-09-23T19:03:43.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:42 smithi016 bash[27237]: cephadm 2023-09-23T19:03:41.803060+0000 mgr.y (mgr.14454) 11 : cephadm [INF] Updating smithi092:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:03:43.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:42 smithi016 bash[27237]: cephadm 2023-09-23T19:03:41.809358+0000 mgr.y (mgr.14454) 12 : cephadm [INF] Updating smithi016:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:03:43.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:42 smithi016 bash[27237]: cephadm 2023-09-23T19:03:41.882575+0000 mgr.y (mgr.14454) 13 : cephadm [INF] Deploying daemon prometheus.a on smithi092 2023-09-23T19:03:43.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:42 smithi016 bash[33624]: cephadm 2023-09-23T19:03:41.784714+0000 mgr.y (mgr.14454) 9 : cephadm [INF] Updating smithi016:/etc/ceph/ceph.conf 2023-09-23T19:03:43.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:42 smithi016 bash[33624]: cephadm 2023-09-23T19:03:41.785502+0000 mgr.y (mgr.14454) 10 : cephadm [INF] Updating smithi092:/etc/ceph/ceph.conf 2023-09-23T19:03:43.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:42 smithi016 bash[33624]: cephadm 2023-09-23T19:03:41.803060+0000 mgr.y (mgr.14454) 11 : cephadm [INF] Updating smithi092:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:03:43.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:42 smithi016 bash[33624]: cephadm 2023-09-23T19:03:41.809358+0000 mgr.y (mgr.14454) 12 : cephadm [INF] Updating smithi016:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:03:43.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:42 smithi016 bash[33624]: cephadm 2023-09-23T19:03:41.882575+0000 mgr.y (mgr.14454) 13 : cephadm [INF] Deploying daemon prometheus.a on smithi092 2023-09-23T19:03:43.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:42 smithi092 bash[27556]: cephadm 2023-09-23T19:03:41.784714+0000 mgr.y (mgr.14454) 9 : cephadm [INF] Updating smithi016:/etc/ceph/ceph.conf 2023-09-23T19:03:43.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:42 smithi092 bash[27556]: cephadm 2023-09-23T19:03:41.785502+0000 mgr.y (mgr.14454) 10 : cephadm [INF] Updating smithi092:/etc/ceph/ceph.conf 2023-09-23T19:03:43.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:42 smithi092 bash[27556]: cephadm 2023-09-23T19:03:41.803060+0000 mgr.y (mgr.14454) 11 : cephadm [INF] Updating smithi092:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:03:43.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:42 smithi092 bash[27556]: cephadm 2023-09-23T19:03:41.809358+0000 mgr.y (mgr.14454) 12 : cephadm [INF] Updating smithi016:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:03:43.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:42 smithi092 bash[27556]: cephadm 2023-09-23T19:03:41.882575+0000 mgr.y (mgr.14454) 13 : cephadm [INF] Deploying daemon prometheus.a on smithi092 2023-09-23T19:03:43.917 INFO:teuthology.orchestra.run.smithi092.stdout:Scheduled grafana update... 2023-09-23T19:03:44.209 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:43 smithi092 bash[27556]: cluster 2023-09-23T19:03:42.866794+0000 mgr.y (mgr.14454) 14 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:44.209 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:43 smithi092 bash[27556]: audit 2023-09-23T19:03:43.913065+0000 mon.a (mon.0) 565 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:03:44.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:43 smithi016 bash[27237]: cluster 2023-09-23T19:03:42.866794+0000 mgr.y (mgr.14454) 14 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:44.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:43 smithi016 bash[27237]: audit 2023-09-23T19:03:43.913065+0000 mon.a (mon.0) 565 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:03:44.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:43 smithi016 bash[33624]: cluster 2023-09-23T19:03:42.866794+0000 mgr.y (mgr.14454) 14 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:44.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:43 smithi016 bash[33624]: audit 2023-09-23T19:03:43.913065+0000 mon.a (mon.0) 565 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:03:45.202 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:44 smithi092 bash[27556]: audit 2023-09-23T19:03:43.902065+0000 mgr.y (mgr.14454) 15 : audit [DBG] from='client.24388 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi092=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:03:45.203 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:44 smithi092 bash[27556]: cephadm 2023-09-23T19:03:43.905648+0000 mgr.y (mgr.14454) 16 : cephadm [INF] Saving service grafana spec with placement smithi092=a;count:1 2023-09-23T19:03:45.204 DEBUG:teuthology.orchestra.run.smithi092:grafana.a> sudo journalctl -f -n 0 -u ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@grafana.a.service 2023-09-23T19:03:45.207 INFO:tasks.cephadm:Setting up client nodes... 2023-09-23T19:03:45.208 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-09-23T19:03:45.224 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:44 smithi016 bash[27237]: audit 2023-09-23T19:03:43.902065+0000 mgr.y (mgr.14454) 15 : audit [DBG] from='client.24388 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi092=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:03:45.224 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:44 smithi016 bash[27237]: cephadm 2023-09-23T19:03:43.905648+0000 mgr.y (mgr.14454) 16 : cephadm [INF] Saving service grafana spec with placement smithi092=a;count:1 2023-09-23T19:03:45.225 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:44 smithi016 bash[33624]: audit 2023-09-23T19:03:43.902065+0000 mgr.y (mgr.14454) 15 : audit [DBG] from='client.24388 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi092=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:03:45.225 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:44 smithi016 bash[33624]: cephadm 2023-09-23T19:03:43.905648+0000 mgr.y (mgr.14454) 16 : cephadm [INF] Saving service grafana spec with placement smithi092=a;count:1 2023-09-23T19:03:45.324 INFO:journalctl@ceph.grafana.a.smithi092.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-09-23T19:03:46.235 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:45 smithi016 bash[27237]: cluster 2023-09-23T19:03:44.867386+0000 mgr.y (mgr.14454) 17 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:46.236 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:45 smithi016 bash[33624]: cluster 2023-09-23T19:03:44.867386+0000 mgr.y (mgr.14454) 17 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:46.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:45 smithi092 bash[27556]: cluster 2023-09-23T19:03:44.867386+0000 mgr.y (mgr.14454) 17 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:48.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:47 smithi016 bash[33624]: cluster 2023-09-23T19:03:46.867873+0000 mgr.y (mgr.14454) 18 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:48.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:47 smithi016 bash[27237]: cluster 2023-09-23T19:03:46.867873+0000 mgr.y (mgr.14454) 18 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:48.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:47 smithi092 bash[27556]: cluster 2023-09-23T19:03:46.867873+0000 mgr.y (mgr.14454) 18 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:48.453 INFO:teuthology.orchestra.run.smithi016.stdout:[client.0] 2023-09-23T19:03:48.453 INFO:teuthology.orchestra.run.smithi016.stdout: key = AQCUNg9l03J0GhAA/8MeQ9ZlUKG4YaTZKOPitQ== 2023-09-23T19:03:49.191 DEBUG:teuthology.orchestra.run.smithi016:> set -ex 2023-09-23T19:03:49.192 DEBUG:teuthology.orchestra.run.smithi016:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2023-09-23T19:03:49.192 DEBUG:teuthology.orchestra.run.smithi016:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2023-09-23T19:03:49.247 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:48 smithi016 bash[27237]: audit 2023-09-23T19:03:48.442831+0000 mon.c (mon.2) 35 : audit [INF] from='client.? 172.21.15.16:0/2553491979' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T19:03:49.247 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:48 smithi016 bash[27237]: audit 2023-09-23T19:03:48.443514+0000 mon.a (mon.0) 566 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T19:03:49.247 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:48 smithi016 bash[27237]: audit 2023-09-23T19:03:48.450146+0000 mon.a (mon.0) 567 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T19:03:49.247 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:48 smithi016 bash[33624]: audit 2023-09-23T19:03:48.442831+0000 mon.c (mon.2) 35 : audit [INF] from='client.? 172.21.15.16:0/2553491979' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T19:03:49.248 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:48 smithi016 bash[33624]: audit 2023-09-23T19:03:48.443514+0000 mon.a (mon.0) 566 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T19:03:49.248 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:48 smithi016 bash[33624]: audit 2023-09-23T19:03:48.450146+0000 mon.a (mon.0) 567 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T19:03:49.259 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-09-23T19:03:49.271 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:48 smithi092 bash[27556]: audit 2023-09-23T19:03:48.442831+0000 mon.c (mon.2) 35 : audit [INF] from='client.? 172.21.15.16:0/2553491979' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T19:03:49.271 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:48 smithi092 bash[27556]: audit 2023-09-23T19:03:48.443514+0000 mon.a (mon.0) 566 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T19:03:49.272 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:48 smithi092 bash[27556]: audit 2023-09-23T19:03:48.450146+0000 mon.a (mon.0) 567 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T19:03:50.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:49 smithi016 bash[33624]: cluster 2023-09-23T19:03:48.868598+0000 mgr.y (mgr.14454) 19 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:50.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:49 smithi016 bash[27237]: cluster 2023-09-23T19:03:48.868598+0000 mgr.y (mgr.14454) 19 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:50.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:49 smithi092 bash[27556]: cluster 2023-09-23T19:03:48.868598+0000 mgr.y (mgr.14454) 19 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:52.145 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:51 smithi092 bash[27556]: cluster 2023-09-23T19:03:50.869254+0000 mgr.y (mgr.14454) 20 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:52.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:51 smithi016 bash[33624]: cluster 2023-09-23T19:03:50.869254+0000 mgr.y (mgr.14454) 20 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:52.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:51 smithi016 bash[27237]: cluster 2023-09-23T19:03:50.869254+0000 mgr.y (mgr.14454) 20 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:54.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:53 smithi016 bash[27237]: cluster 2023-09-23T19:03:52.869913+0000 mgr.y (mgr.14454) 21 : cluster [DBG] pgmap v12: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:54.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:53 smithi016 bash[33624]: cluster 2023-09-23T19:03:52.869913+0000 mgr.y (mgr.14454) 21 : cluster [DBG] pgmap v12: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:54.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:53 smithi092 bash[27556]: cluster 2023-09-23T19:03:52.869913+0000 mgr.y (mgr.14454) 21 : cluster [DBG] pgmap v12: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:54.958 INFO:teuthology.orchestra.run.smithi092.stdout:[client.1] 2023-09-23T19:03:54.958 INFO:teuthology.orchestra.run.smithi092.stdout: key = AQCaNg9ls/6DOBAATTuOI3GYMeF0/iMb7srybA== 2023-09-23T19:03:55.964 DEBUG:teuthology.orchestra.run.smithi092:> set -ex 2023-09-23T19:03:55.964 DEBUG:teuthology.orchestra.run.smithi092:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2023-09-23T19:03:55.964 DEBUG:teuthology.orchestra.run.smithi092:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2023-09-23T19:03:56.018 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:55 smithi092 bash[27556]: cluster 2023-09-23T19:03:54.870500+0000 mgr.y (mgr.14454) 22 : cluster [DBG] pgmap v13: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:56.018 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:55 smithi092 bash[27556]: audit 2023-09-23T19:03:54.947847+0000 mon.a (mon.0) 568 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T19:03:56.018 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:55 smithi092 bash[27556]: audit 2023-09-23T19:03:54.952095+0000 mon.b (mon.1) 27 : audit [INF] from='client.? 172.21.15.92:0/3706363035' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T19:03:56.018 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:55 smithi092 bash[27556]: audit 2023-09-23T19:03:54.954809+0000 mon.a (mon.0) 569 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T19:03:56.026 INFO:tasks.cephadm:Waiting for OSDs to come up 2023-09-23T19:03:56.027 DEBUG:teuthology.orchestra.run.smithi016:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2023-09-23T19:03:56.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:55 smithi016 bash[27237]: cluster 2023-09-23T19:03:54.870500+0000 mgr.y (mgr.14454) 22 : cluster [DBG] pgmap v13: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:56.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:55 smithi016 bash[27237]: audit 2023-09-23T19:03:54.947847+0000 mon.a (mon.0) 568 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T19:03:56.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:55 smithi016 bash[27237]: audit 2023-09-23T19:03:54.952095+0000 mon.b (mon.1) 27 : audit [INF] from='client.? 172.21.15.92:0/3706363035' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T19:03:56.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:55 smithi016 bash[27237]: audit 2023-09-23T19:03:54.954809+0000 mon.a (mon.0) 569 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T19:03:56.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:55 smithi016 bash[33624]: cluster 2023-09-23T19:03:54.870500+0000 mgr.y (mgr.14454) 22 : cluster [DBG] pgmap v13: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:56.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:55 smithi016 bash[33624]: audit 2023-09-23T19:03:54.947847+0000 mon.a (mon.0) 568 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T19:03:56.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:55 smithi016 bash[33624]: audit 2023-09-23T19:03:54.952095+0000 mon.b (mon.1) 27 : audit [INF] from='client.? 172.21.15.92:0/3706363035' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T19:03:56.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:55 smithi016 bash[33624]: audit 2023-09-23T19:03:54.954809+0000 mon.a (mon.0) 569 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T19:03:56.287 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:03:56 smithi092 systemd[1]: Started Ceph prometheus.a for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7. 2023-09-23T19:03:56.318 INFO:teuthology.misc.health.smithi016.stdout: 2023-09-23T19:03:56.319 INFO:teuthology.misc.health.smithi016.stdout:{"epoch":51,"fsid":"95ce9eb6-5a42-11ee-8db4-212e2dc638e7","created":"2023-09-23T18:55:23.802934+0000","modified":"2023-09-23T19:03:34.846576+0000","last_up_change":"2023-09-23T19:03:02.463512+0000","last_in_change":"2023-09-23T19:02:36.618072+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-09-23T18:59:33.315416+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}}],"osds":[{"osd":0,"uuid":"a4e8ed09-1195-4878-928a-13881bc04dc7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6802","nonce":2337435219},{"type":"v1","addr":"172.21.15.16:6803","nonce":2337435219}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6804","nonce":2337435219},{"type":"v1","addr":"172.21.15.16:6805","nonce":2337435219}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6808","nonce":2337435219},{"type":"v1","addr":"172.21.15.16:6809","nonce":2337435219}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6806","nonce":2337435219},{"type":"v1","addr":"172.21.15.16:6807","nonce":2337435219}]},"public_addr":"172.21.15.16:6803/2337435219","cluster_addr":"172.21.15.16:6805/2337435219","heartbeat_back_addr":"172.21.15.16:6809/2337435219","heartbeat_front_addr":"172.21.15.16:6807/2337435219","state":["exists","up"]},{"osd":1,"uuid":"c5ff2879-0d1c-481a-b847-248433c12aa4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6810","nonce":3893879931},{"type":"v1","addr":"172.21.15.16:6811","nonce":3893879931}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6812","nonce":3893879931},{"type":"v1","addr":"172.21.15.16:6813","nonce":3893879931}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6816","nonce":3893879931},{"type":"v1","addr":"172.21.15.16:6817","nonce":3893879931}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6814","nonce":3893879931},{"type":"v1","addr":"172.21.15.16:6815","nonce":3893879931}]},"public_addr":"172.21.15.16:6811/3893879931","cluster_addr":"172.21.15.16:6813/3893879931","heartbeat_back_addr":"172.21.15.16:6817/3893879931","heartbeat_front_addr":"172.21.15.16:6815/3893879931","state":["exists","up"]},{"osd":2,"uuid":"7974ce4d-9ed8-4a72-b44b-d500b34da8cf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6818","nonce":568840676},{"type":"v1","addr":"172.21.15.16:6819","nonce":568840676}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6820","nonce":568840676},{"type":"v1","addr":"172.21.15.16:6821","nonce":568840676}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6824","nonce":568840676},{"type":"v1","addr":"172.21.15.16:6825","nonce":568840676}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6822","nonce":568840676},{"type":"v1","addr":"172.21.15.16:6823","nonce":568840676}]},"public_addr":"172.21.15.16:6819/568840676","cluster_addr":"172.21.15.16:6821/568840676","heartbeat_back_addr":"172.21.15.16:6825/568840676","heartbeat_front_addr":"172.21.15.16:6823/568840676","state":["exists","up"]},{"osd":3,"uuid":"f5e0d57b-d629-4453-a527-ad77e6c210ad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6826","nonce":204922667},{"type":"v1","addr":"172.21.15.16:6827","nonce":204922667}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6828","nonce":204922667},{"type":"v1","addr":"172.21.15.16:6829","nonce":204922667}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6832","nonce":204922667},{"type":"v1","addr":"172.21.15.16:6833","nonce":204922667}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6830","nonce":204922667},{"type":"v1","addr":"172.21.15.16:6831","nonce":204922667}]},"public_addr":"172.21.15.16:6827/204922667","cluster_addr":"172.21.15.16:6829/204922667","heartbeat_back_addr":"172.21.15.16:6833/204922667","heartbeat_front_addr":"172.21.15.16:6831/204922667","state":["exists","up"]},{"osd":4,"uuid":"55a8dab2-6393-44ab-943e-cb7ab2cb7c31","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6800","nonce":99745912},{"type":"v1","addr":"172.21.15.92:6801","nonce":99745912}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6802","nonce":99745912},{"type":"v1","addr":"172.21.15.92:6803","nonce":99745912}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6806","nonce":99745912},{"type":"v1","addr":"172.21.15.92:6807","nonce":99745912}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6804","nonce":99745912},{"type":"v1","addr":"172.21.15.92:6805","nonce":99745912}]},"public_addr":"172.21.15.92:6801/99745912","cluster_addr":"172.21.15.92:6803/99745912","heartbeat_back_addr":"172.21.15.92:6807/99745912","heartbeat_front_addr":"172.21.15.92:6805/99745912","state":["exists","up"]},{"osd":5,"uuid":"b29b73b1-51d7-4484-bd79-2334c650cb62","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":37,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6808","nonce":1987214502},{"type":"v1","addr":"172.21.15.92:6809","nonce":1987214502}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6810","nonce":1987214502},{"type":"v1","addr":"172.21.15.92:6811","nonce":1987214502}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6814","nonce":1987214502},{"type":"v1","addr":"172.21.15.92:6815","nonce":1987214502}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6812","nonce":1987214502},{"type":"v1","addr":"172.21.15.92:6813","nonce":1987214502}]},"public_addr":"172.21.15.92:6809/1987214502","cluster_addr":"172.21.15.92:6811/1987214502","heartbeat_back_addr":"172.21.15.92:6815/1987214502","heartbeat_front_addr":"172.21.15.92:6813/1987214502","state":["exists","up"]},{"osd":6,"uuid":"9fe16cc9-f0d3-4b79-bc68-38305c55b551","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6816","nonce":2820798178},{"type":"v1","addr":"172.21.15.92:6817","nonce":2820798178}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6818","nonce":2820798178},{"type":"v1","addr":"172.21.15.92:6819","nonce":2820798178}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6822","nonce":2820798178},{"type":"v1","addr":"172.21.15.92:6823","nonce":2820798178}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6820","nonce":2820798178},{"type":"v1","addr":"172.21.15.92:6821","nonce":2820798178}]},"public_addr":"172.21.15.92:6817/2820798178","cluster_addr":"172.21.15.92:6819/2820798178","heartbeat_back_addr":"172.21.15.92:6823/2820798178","heartbeat_front_addr":"172.21.15.92:6821/2820798178","state":["exists","up"]},{"osd":7,"uuid":"d086b076-127f-42de-bee9-157bf579d92d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6824","nonce":98624761},{"type":"v1","addr":"172.21.15.92:6825","nonce":98624761}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6826","nonce":98624761},{"type":"v1","addr":"172.21.15.92:6827","nonce":98624761}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6830","nonce":98624761},{"type":"v1","addr":"172.21.15.92:6831","nonce":98624761}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6828","nonce":98624761},{"type":"v1","addr":"172.21.15.92:6829","nonce":98624761}]},"public_addr":"172.21.15.92:6825/98624761","cluster_addr":"172.21.15.92:6827/98624761","heartbeat_back_addr":"172.21.15.92:6831/98624761","heartbeat_front_addr":"172.21.15.92:6829/98624761","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:58:07.335980+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:58:49.342213+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:59:30.939986+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:00:12.926364+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:00:53.466692+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:01:36.332957+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:02:18.593854+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:03:00.817448+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.16:6801/1259630240":"2023-09-24T19:03:34.846451+0000","172.21.15.16:6800/1259630240":"2023-09-24T19:03:34.846451+0000","172.21.15.16:0/3767844203":"2023-09-24T19:03:34.846451+0000","172.21.15.16:6801/2756103674":"2023-09-24T18:56:20.574625+0000","172.21.15.16:0/2843459289":"2023-09-24T18:56:20.574625+0000","172.21.15.16:0/272242693":"2023-09-24T18:56:20.574625+0000","172.21.15.16:0/79169563":"2023-09-24T18:55:50.550546+0000","172.21.15.16:6800/2615282850":"2023-09-24T18:55:50.550546+0000","172.21.15.16:0/349446544":"2023-09-24T18:55:50.550546+0000","172.21.15.16:0/2669353350":"2023-09-24T19:03:34.846451+0000","172.21.15.16:6800/2756103674":"2023-09-24T18:56:20.574625+0000","172.21.15.16:6801/2615282850":"2023-09-24T18:55:50.550546+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T19:03:56.335 DEBUG:teuthology.misc:8 of 8 OSDs are up 2023-09-23T19:03:56.335 INFO:tasks.cephadm:Creating RBD pool 2023-09-23T19:03:56.335 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- sudo ceph --cluster ceph osd pool create rbd 8 2023-09-23T19:03:57.022 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:03:56 smithi016 systemd[1]: Started Ceph node-exporter.a for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7. 2023-09-23T19:03:57.284 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:03:57 smithi016 bash[52296]: Unable to find image 'quay.io/prometheus/node-exporter:v1.3.1' locally 2023-09-23T19:03:57.522 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:57 smithi092 bash[27556]: audit 2023-09-23T19:03:56.276179+0000 mon.a (mon.0) 570 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:03:57.522 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:57 smithi092 bash[27556]: cephadm 2023-09-23T19:03:56.280284+0000 mgr.y (mgr.14454) 23 : cephadm [INF] Deploying daemon node-exporter.a on smithi016 2023-09-23T19:03:57.523 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:57 smithi092 bash[27556]: audit 2023-09-23T19:03:56.314650+0000 mon.c (mon.2) 36 : audit [DBG] from='client.? 172.21.15.16:0/2892568181' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T19:03:57.523 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:57 smithi092 bash[27556]: cluster 2023-09-23T19:03:56.871066+0000 mgr.y (mgr.14454) 24 : cluster [DBG] pgmap v14: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:57.523 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:57 smithi092 bash[27556]: audit 2023-09-23T19:03:57.008204+0000 mon.a (mon.0) 571 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:03:57.523 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:57 smithi092 bash[27556]: cephadm 2023-09-23T19:03:57.010509+0000 mgr.y (mgr.14454) 25 : cephadm [INF] Deploying daemon node-exporter.b on smithi092 2023-09-23T19:03:57.523 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:03:57 smithi092 bash[46844]: ts=2023-09-23T19:03:57.393Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2023-09-23T19:03:57.523 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:03:57 smithi092 bash[46844]: ts=2023-09-23T19:03:57.393Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2023-09-23T19:03:57.523 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:03:57 smithi092 bash[46844]: ts=2023-09-23T19:03:57.393Z caller=main.go:518 level=info host_details="(Linux 5.15.0-84-generic #93~20.04.1-Ubuntu SMP Wed Sep 6 16:15:40 UTC 2023 x86_64 smithi092 (none))" 2023-09-23T19:03:57.523 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:03:57 smithi092 bash[46844]: ts=2023-09-23T19:03:57.393Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-09-23T19:03:57.523 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:03:57 smithi092 bash[46844]: ts=2023-09-23T19:03:57.393Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-09-23T19:03:57.524 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:03:57 smithi092 bash[46844]: ts=2023-09-23T19:03:57.395Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2023-09-23T19:03:57.524 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:03:57 smithi092 bash[46844]: ts=2023-09-23T19:03:57.395Z caller=main.go:923 level=info msg="Starting TSDB ..." 2023-09-23T19:03:57.524 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:03:57 smithi092 bash[46844]: ts=2023-09-23T19:03:57.397Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2023-09-23T19:03:57.524 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:03:57 smithi092 bash[46844]: ts=2023-09-23T19:03:57.401Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-09-23T19:03:57.524 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:03:57 smithi092 bash[46844]: ts=2023-09-23T19:03:57.401Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.025µs 2023-09-23T19:03:57.524 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:03:57 smithi092 bash[46844]: ts=2023-09-23T19:03:57.401Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-09-23T19:03:57.524 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:03:57 smithi092 bash[46844]: ts=2023-09-23T19:03:57.401Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2023-09-23T19:03:57.524 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:03:57 smithi092 bash[46844]: ts=2023-09-23T19:03:57.402Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=32.749µs wal_replay_duration=205.703µs total_replay_duration=257.675µs 2023-09-23T19:03:57.524 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:03:57 smithi092 bash[46844]: ts=2023-09-23T19:03:57.403Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2023-09-23T19:03:57.524 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:03:57 smithi092 bash[46844]: ts=2023-09-23T19:03:57.403Z caller=main.go:947 level=info msg="TSDB started" 2023-09-23T19:03:57.524 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:03:57 smithi092 bash[46844]: ts=2023-09-23T19:03:57.403Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-09-23T19:03:57.525 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:03:57 smithi092 bash[46844]: ts=2023-09-23T19:03:57.427Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=24.196159ms db_storage=649ns remote_storage=1.431µs web_handler=358ns query_engine=663ns scrape=278.264µs scrape_sd=26.777µs notify=1.122µs notify_sd=2.721µs rules=23.625202ms 2023-09-23T19:03:57.525 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:03:57 smithi092 bash[46844]: ts=2023-09-23T19:03:57.427Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-09-23T19:03:57.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:57 smithi016 bash[27237]: audit 2023-09-23T19:03:56.276179+0000 mon.a (mon.0) 570 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:03:57.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:57 smithi016 bash[27237]: cephadm 2023-09-23T19:03:56.280284+0000 mgr.y (mgr.14454) 23 : cephadm [INF] Deploying daemon node-exporter.a on smithi016 2023-09-23T19:03:57.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:57 smithi016 bash[27237]: audit 2023-09-23T19:03:56.314650+0000 mon.c (mon.2) 36 : audit [DBG] from='client.? 172.21.15.16:0/2892568181' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T19:03:57.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:57 smithi016 bash[27237]: cluster 2023-09-23T19:03:56.871066+0000 mgr.y (mgr.14454) 24 : cluster [DBG] pgmap v14: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:57.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:57 smithi016 bash[27237]: audit 2023-09-23T19:03:57.008204+0000 mon.a (mon.0) 571 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:03:57.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:57 smithi016 bash[27237]: cephadm 2023-09-23T19:03:57.010509+0000 mgr.y (mgr.14454) 25 : cephadm [INF] Deploying daemon node-exporter.b on smithi092 2023-09-23T19:03:57.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:57 smithi016 bash[33624]: audit 2023-09-23T19:03:56.276179+0000 mon.a (mon.0) 570 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:03:57.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:57 smithi016 bash[33624]: cephadm 2023-09-23T19:03:56.280284+0000 mgr.y (mgr.14454) 23 : cephadm [INF] Deploying daemon node-exporter.a on smithi016 2023-09-23T19:03:57.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:57 smithi016 bash[33624]: audit 2023-09-23T19:03:56.314650+0000 mon.c (mon.2) 36 : audit [DBG] from='client.? 172.21.15.16:0/2892568181' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T19:03:57.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:57 smithi016 bash[33624]: cluster 2023-09-23T19:03:56.871066+0000 mgr.y (mgr.14454) 24 : cluster [DBG] pgmap v14: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:03:57.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:57 smithi016 bash[33624]: audit 2023-09-23T19:03:57.008204+0000 mon.a (mon.0) 571 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:03:57.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:57 smithi016 bash[33624]: cephadm 2023-09-23T19:03:57.010509+0000 mgr.y (mgr.14454) 25 : cephadm [INF] Deploying daemon node-exporter.b on smithi092 2023-09-23T19:03:57.787 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:03:57 smithi092 systemd[1]: Started Ceph node-exporter.b for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7. 2023-09-23T19:03:57.837 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:03:57 smithi016 bash[52296]: v1.3.1: Pulling from prometheus/node-exporter 2023-09-23T19:03:58.152 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:03:57 smithi016 bash[52296]: aa2a8d90b84c: Pulling fs layer 2023-09-23T19:03:58.153 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:03:57 smithi016 bash[52296]: b45d31ee2d7f: Pulling fs layer 2023-09-23T19:03:58.153 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:03:57 smithi016 bash[52296]: b5db1e299295: Pulling fs layer 2023-09-23T19:03:58.287 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:03:57 smithi092 bash[47010]: Unable to find image 'quay.io/prometheus/node-exporter:v1.3.1' locally 2023-09-23T19:03:58.535 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:03:58 smithi016 bash[52296]: b45d31ee2d7f: Verifying Checksum 2023-09-23T19:03:58.535 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:03:58 smithi016 bash[52296]: b45d31ee2d7f: Download complete 2023-09-23T19:03:58.535 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:03:58 smithi016 bash[52296]: aa2a8d90b84c: Verifying Checksum 2023-09-23T19:03:58.535 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:03:58 smithi016 bash[52296]: aa2a8d90b84c: Download complete 2023-09-23T19:03:58.535 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:03:58 smithi016 bash[52296]: b5db1e299295: Verifying Checksum 2023-09-23T19:03:58.535 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:03:58 smithi016 bash[52296]: b5db1e299295: Download complete 2023-09-23T19:03:58.732 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:03:58 smithi092 bash[47010]: v1.3.1: Pulling from prometheus/node-exporter 2023-09-23T19:03:58.733 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:03:58 smithi092 bash[47010]: aa2a8d90b84c: Pulling fs layer 2023-09-23T19:03:58.733 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:03:58 smithi092 bash[47010]: b45d31ee2d7f: Pulling fs layer 2023-09-23T19:03:58.733 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:03:58 smithi092 bash[47010]: b5db1e299295: Pulling fs layer 2023-09-23T19:03:59.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:58 smithi092 bash[27556]: audit 2023-09-23T19:03:57.784892+0000 mon.a (mon.0) 572 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:03:59.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:58 smithi092 bash[27556]: cephadm 2023-09-23T19:03:57.797354+0000 mgr.y (mgr.14454) 26 : cephadm [INF] Deploying daemon alertmanager.a on smithi016 2023-09-23T19:03:59.038 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:03:58 smithi092 bash[47010]: aa2a8d90b84c: Verifying Checksum 2023-09-23T19:03:59.038 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:03:58 smithi092 bash[47010]: aa2a8d90b84c: Download complete 2023-09-23T19:03:59.038 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:03:58 smithi092 bash[47010]: b45d31ee2d7f: Verifying Checksum 2023-09-23T19:03:59.038 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:03:58 smithi092 bash[47010]: b45d31ee2d7f: Download complete 2023-09-23T19:03:59.038 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:03:58 smithi092 bash[47010]: b5db1e299295: Verifying Checksum 2023-09-23T19:03:59.038 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:03:58 smithi092 bash[47010]: b5db1e299295: Download complete 2023-09-23T19:03:59.038 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:03:58 smithi092 bash[47010]: aa2a8d90b84c: Pull complete 2023-09-23T19:03:59.058 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:58 smithi016 bash[33624]: audit 2023-09-23T19:03:57.784892+0000 mon.a (mon.0) 572 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:03:59.058 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:58 smithi016 bash[33624]: cephadm 2023-09-23T19:03:57.797354+0000 mgr.y (mgr.14454) 26 : cephadm [INF] Deploying daemon alertmanager.a on smithi016 2023-09-23T19:03:59.059 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:58 smithi016 bash[27237]: audit 2023-09-23T19:03:57.784892+0000 mon.a (mon.0) 572 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:03:59.059 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:58 smithi016 bash[27237]: cephadm 2023-09-23T19:03:57.797354+0000 mgr.y (mgr.14454) 26 : cephadm [INF] Deploying daemon alertmanager.a on smithi016 2023-09-23T19:03:59.059 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:03:58 smithi016 bash[52296]: aa2a8d90b84c: Pull complete 2023-09-23T19:03:59.710 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:03:59 smithi092 bash[47010]: b45d31ee2d7f: Pull complete 2023-09-23T19:04:00.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:03:59 smithi092 bash[27556]: cluster 2023-09-23T19:03:58.871645+0000 mgr.y (mgr.14454) 27 : cluster [DBG] pgmap v15: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:00.038 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:03:59 smithi092 bash[47010]: b5db1e299295: Pull complete 2023-09-23T19:04:00.038 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:03:59 smithi092 bash[47010]: Digest: sha256:f2269e73124dd0f60a7d19a2ce1264d33d08a985aed0ee6b0b89d0be470592cd 2023-09-23T19:04:00.038 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:03:59 smithi092 bash[47010]: Status: Downloaded newer image for quay.io/prometheus/node-exporter:v1.3.1 2023-09-23T19:04:00.084 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:03:59 smithi016 bash[27237]: cluster 2023-09-23T19:03:58.871645+0000 mgr.y (mgr.14454) 27 : cluster [DBG] pgmap v15: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:00.084 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:03:59 smithi016 bash[33624]: cluster 2023-09-23T19:03:58.871645+0000 mgr.y (mgr.14454) 27 : cluster [DBG] pgmap v15: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:00.535 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:04:00 smithi016 bash[52296]: b45d31ee2d7f: Pull complete 2023-09-23T19:04:00.805 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:04:00 smithi016 bash[52296]: b5db1e299295: Pull complete 2023-09-23T19:04:01.285 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:04:00 smithi016 bash[52296]: Digest: sha256:f2269e73124dd0f60a7d19a2ce1264d33d08a985aed0ee6b0b89d0be470592cd 2023-09-23T19:04:01.285 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:04:00 smithi016 bash[52296]: Status: Downloaded newer image for quay.io/prometheus/node-exporter:v1.3.1 2023-09-23T19:04:01.452 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:04:01 smithi092 bash[47010]: ts=2023-09-23T19:04:01.214Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-09-23T19:04:01.452 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:04:01 smithi092 bash[47010]: ts=2023-09-23T19:04:01.214Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-09-23T19:04:01.452 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:04:01 smithi092 bash[47010]: ts=2023-09-23T19:04:01.214Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-09-23T19:04:01.452 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:04:01 smithi092 bash[47010]: ts=2023-09-23T19:04:01.214Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-09-23T19:04:01.452 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:04:01 smithi092 bash[47010]: ts=2023-09-23T19:04:01.215Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-09-23T19:04:01.452 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:04:01 smithi092 bash[47010]: ts=2023-09-23T19:04:01.215Z caller=node_exporter.go:115 level=info collector=arp 2023-09-23T19:04:01.453 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:04:01 smithi092 bash[47010]: ts=2023-09-23T19:04:01.215Z caller=node_exporter.go:115 level=info collector=bcache 2023-09-23T19:04:01.453 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:04:01 smithi092 bash[47010]: ts=2023-09-23T19:04:01.215Z caller=node_exporter.go:115 level=info collector=bonding 2023-09-23T19:04:01.453 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:04:01 smithi092 bash[47010]: ts=2023-09-23T19:04:01.215Z caller=node_exporter.go:115 level=info collector=btrfs 2023-09-23T19:04:01.453 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:04:01 smithi092 bash[47010]: ts=2023-09-23T19:04:01.215Z caller=node_exporter.go:115 level=info collector=conntrack 2023-09-23T19:04:01.453 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:04:01 smithi092 bash[47010]: ts=2023-09-23T19:04:01.215Z caller=node_exporter.go:115 level=info collector=cpu 2023-09-23T19:04:01.453 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:04:01 smithi092 bash[47010]: ts=2023-09-23T19:04:01.215Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-09-23T19:04:01.453 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:04:01 smithi092 bash[47010]: ts=2023-09-23T19:04:01.215Z caller=node_exporter.go:115 level=info collector=diskstats 2023-09-23T19:04:01.453 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:04:01 smithi092 bash[47010]: ts=2023-09-23T19:04:01.215Z caller=node_exporter.go:115 level=info collector=dmi 2023-09-23T19:04:01.453 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:04:01 smithi092 bash[47010]: ts=2023-09-23T19:04:01.215Z caller=node_exporter.go:115 level=info collector=edac 2023-09-23T19:04:01.453 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:04:01 smithi092 bash[47010]: ts=2023-09-23T19:04:01.215Z caller=node_exporter.go:115 level=info collector=entropy 2023-09-23T19:04:01.453 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:04:01 smithi092 bash[47010]: ts=2023-09-23T19:04:01.215Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-09-23T19:04:01.453 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:04:01 smithi092 bash[47010]: ts=2023-09-23T19:04:01.215Z caller=node_exporter.go:115 level=info collector=filefd 2023-09-23T19:04:01.454 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:04:01 smithi092 bash[47010]: ts=2023-09-23T19:04:01.215Z caller=node_exporter.go:115 level=info collector=filesystem 2023-09-23T19:04:01.454 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:04:01 smithi092 bash[47010]: ts=2023-09-23T19:04:01.215Z caller=node_exporter.go:115 level=info collector=hwmon 2023-09-23T19:04:01.454 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:04:01 smithi092 bash[47010]: ts=2023-09-23T19:04:01.215Z caller=node_exporter.go:115 level=info collector=infiniband 2023-09-23T19:04:01.454 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:04:01 smithi092 bash[47010]: ts=2023-09-23T19:04:01.215Z caller=node_exporter.go:115 level=info collector=ipvs 2023-09-23T19:04:01.454 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:04:01 smithi092 bash[47010]: ts=2023-09-23T19:04:01.215Z caller=node_exporter.go:115 level=info collector=loadavg 2023-09-23T19:04:01.454 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:04:01 smithi092 bash[47010]: ts=2023-09-23T19:04:01.215Z caller=node_exporter.go:115 level=info collector=mdadm 2023-09-23T19:04:01.454 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:04:01 smithi092 bash[47010]: ts=2023-09-23T19:04:01.215Z caller=node_exporter.go:115 level=info collector=meminfo 2023-09-23T19:04:01.454 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:04:01 smithi092 bash[47010]: ts=2023-09-23T19:04:01.215Z caller=node_exporter.go:115 level=info collector=netclass 2023-09-23T19:04:01.454 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:04:01 smithi092 bash[47010]: ts=2023-09-23T19:04:01.215Z caller=node_exporter.go:115 level=info collector=netdev 2023-09-23T19:04:01.454 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:04:01 smithi092 bash[47010]: ts=2023-09-23T19:04:01.215Z caller=node_exporter.go:115 level=info collector=netstat 2023-09-23T19:04:01.454 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:04:01 smithi092 bash[47010]: ts=2023-09-23T19:04:01.215Z caller=node_exporter.go:115 level=info collector=nfs 2023-09-23T19:04:01.454 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:04:01 smithi092 bash[47010]: ts=2023-09-23T19:04:01.215Z caller=node_exporter.go:115 level=info collector=nfsd 2023-09-23T19:04:01.455 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:04:01 smithi092 bash[47010]: ts=2023-09-23T19:04:01.215Z caller=node_exporter.go:115 level=info collector=nvme 2023-09-23T19:04:01.455 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:04:01 smithi092 bash[47010]: ts=2023-09-23T19:04:01.215Z caller=node_exporter.go:115 level=info collector=os 2023-09-23T19:04:01.455 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:04:01 smithi092 bash[47010]: ts=2023-09-23T19:04:01.216Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-09-23T19:04:01.455 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:04:01 smithi092 bash[47010]: ts=2023-09-23T19:04:01.216Z caller=node_exporter.go:115 level=info collector=pressure 2023-09-23T19:04:01.455 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:04:01 smithi092 bash[47010]: ts=2023-09-23T19:04:01.216Z caller=node_exporter.go:115 level=info collector=rapl 2023-09-23T19:04:01.455 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:04:01 smithi092 bash[47010]: ts=2023-09-23T19:04:01.216Z caller=node_exporter.go:115 level=info collector=schedstat 2023-09-23T19:04:01.455 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:04:01 smithi092 bash[47010]: ts=2023-09-23T19:04:01.216Z caller=node_exporter.go:115 level=info collector=sockstat 2023-09-23T19:04:01.455 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:04:01 smithi092 bash[47010]: ts=2023-09-23T19:04:01.216Z caller=node_exporter.go:115 level=info collector=softnet 2023-09-23T19:04:01.455 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:04:01 smithi092 bash[47010]: ts=2023-09-23T19:04:01.216Z caller=node_exporter.go:115 level=info collector=stat 2023-09-23T19:04:01.455 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:04:01 smithi092 bash[47010]: ts=2023-09-23T19:04:01.216Z caller=node_exporter.go:115 level=info collector=tapestats 2023-09-23T19:04:01.455 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:04:01 smithi092 bash[47010]: ts=2023-09-23T19:04:01.216Z caller=node_exporter.go:115 level=info collector=textfile 2023-09-23T19:04:01.455 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:04:01 smithi092 bash[47010]: ts=2023-09-23T19:04:01.216Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-09-23T19:04:01.456 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:04:01 smithi092 bash[47010]: ts=2023-09-23T19:04:01.216Z caller=node_exporter.go:115 level=info collector=time 2023-09-23T19:04:01.456 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:04:01 smithi092 bash[47010]: ts=2023-09-23T19:04:01.216Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-09-23T19:04:01.456 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:04:01 smithi092 bash[47010]: ts=2023-09-23T19:04:01.216Z caller=node_exporter.go:115 level=info collector=uname 2023-09-23T19:04:01.456 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:04:01 smithi092 bash[47010]: ts=2023-09-23T19:04:01.216Z caller=node_exporter.go:115 level=info collector=vmstat 2023-09-23T19:04:01.456 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:04:01 smithi092 bash[47010]: ts=2023-09-23T19:04:01.216Z caller=node_exporter.go:115 level=info collector=xfs 2023-09-23T19:04:01.456 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:04:01 smithi092 bash[47010]: ts=2023-09-23T19:04:01.216Z caller=node_exporter.go:115 level=info collector=zfs 2023-09-23T19:04:01.456 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:04:01 smithi092 bash[47010]: ts=2023-09-23T19:04:01.216Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-09-23T19:04:01.456 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:04:01 smithi092 bash[47010]: ts=2023-09-23T19:04:01.216Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-09-23T19:04:01.504 INFO:teuthology.orchestra.run.smithi016.stderr:pool 'rbd' created 2023-09-23T19:04:01.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:01 smithi016 bash[27237]: audit 2023-09-23T19:04:00.082976+0000 mon.a (mon.0) 573 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:04:01.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:01 smithi016 bash[33624]: audit 2023-09-23T19:04:00.082976+0000 mon.a (mon.0) 573 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:04:01.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:01 smithi092 bash[27556]: audit 2023-09-23T19:04:00.082976+0000 mon.a (mon.0) 573 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:04:02.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:02 smithi016 bash[27237]: cluster 2023-09-23T19:04:00.872194+0000 mgr.y (mgr.14454) 28 : cluster [DBG] pgmap v16: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:02.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:02 smithi016 bash[27237]: audit 2023-09-23T19:04:00.971332+0000 mon.a (mon.0) 574 : audit [INF] from='client.? 172.21.15.16:0/2757623864' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-09-23T19:04:02.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:02 smithi016 bash[27237]: audit 2023-09-23T19:04:01.496619+0000 mon.a (mon.0) 575 : audit [INF] from='client.? 172.21.15.16:0/2757623864' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-09-23T19:04:02.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:02 smithi016 bash[27237]: cluster 2023-09-23T19:04:01.496714+0000 mon.a (mon.0) 576 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2023-09-23T19:04:02.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:02 smithi016 bash[27237]: audit 2023-09-23T19:04:01.497543+0000 mon.c (mon.2) 37 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-09-23T19:04:02.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:02 smithi016 bash[27237]: audit 2023-09-23T19:04:01.498577+0000 mon.a (mon.0) 577 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-09-23T19:04:02.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:02 smithi016 bash[27237]: audit 2023-09-23T19:04:01.500357+0000 mon.c (mon.2) 38 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-09-23T19:04:02.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:02 smithi016 bash[27237]: audit 2023-09-23T19:04:01.500902+0000 mon.a (mon.0) 578 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-09-23T19:04:02.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:02 smithi016 bash[33624]: cluster 2023-09-23T19:04:00.872194+0000 mgr.y (mgr.14454) 28 : cluster [DBG] pgmap v16: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:02.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:02 smithi016 bash[33624]: audit 2023-09-23T19:04:00.971332+0000 mon.a (mon.0) 574 : audit [INF] from='client.? 172.21.15.16:0/2757623864' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-09-23T19:04:02.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:02 smithi016 bash[33624]: audit 2023-09-23T19:04:01.496619+0000 mon.a (mon.0) 575 : audit [INF] from='client.? 172.21.15.16:0/2757623864' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-09-23T19:04:02.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:02 smithi016 bash[33624]: cluster 2023-09-23T19:04:01.496714+0000 mon.a (mon.0) 576 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2023-09-23T19:04:02.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:02 smithi016 bash[33624]: audit 2023-09-23T19:04:01.497543+0000 mon.c (mon.2) 37 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-09-23T19:04:02.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:02 smithi016 bash[33624]: audit 2023-09-23T19:04:01.498577+0000 mon.a (mon.0) 577 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-09-23T19:04:02.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:02 smithi016 bash[33624]: audit 2023-09-23T19:04:01.500357+0000 mon.c (mon.2) 38 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-09-23T19:04:02.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:02 smithi016 bash[33624]: audit 2023-09-23T19:04:01.500902+0000 mon.a (mon.0) 578 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-09-23T19:04:02.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:02 smithi092 bash[27556]: cluster 2023-09-23T19:04:00.872194+0000 mgr.y (mgr.14454) 28 : cluster [DBG] pgmap v16: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:02.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:02 smithi092 bash[27556]: audit 2023-09-23T19:04:00.971332+0000 mon.a (mon.0) 574 : audit [INF] from='client.? 172.21.15.16:0/2757623864' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-09-23T19:04:02.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:02 smithi092 bash[27556]: audit 2023-09-23T19:04:01.496619+0000 mon.a (mon.0) 575 : audit [INF] from='client.? 172.21.15.16:0/2757623864' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-09-23T19:04:02.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:02 smithi092 bash[27556]: cluster 2023-09-23T19:04:01.496714+0000 mon.a (mon.0) 576 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2023-09-23T19:04:02.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:02 smithi092 bash[27556]: audit 2023-09-23T19:04:01.497543+0000 mon.c (mon.2) 37 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-09-23T19:04:02.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:02 smithi092 bash[27556]: audit 2023-09-23T19:04:01.498577+0000 mon.a (mon.0) 577 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-09-23T19:04:02.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:02 smithi092 bash[27556]: audit 2023-09-23T19:04:01.500357+0000 mon.c (mon.2) 38 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-09-23T19:04:02.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:02 smithi092 bash[27556]: audit 2023-09-23T19:04:01.500902+0000 mon.a (mon.0) 578 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-09-23T19:04:03.509 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- sudo ceph --cluster ceph osd pool application enable rbd rbd --yes-i-really-mean-it 2023-09-23T19:04:03.771 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[27237]: audit 2023-09-23T19:04:02.496775+0000 mon.a (mon.0) 579 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-09-23T19:04:03.771 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[27237]: cluster 2023-09-23T19:04:02.496915+0000 mon.a (mon.0) 580 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2023-09-23T19:04:03.771 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[27237]: cluster 2023-09-23T19:04:02.872770+0000 mgr.y (mgr.14454) 29 : cluster [DBG] pgmap v19: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:03.772 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[33624]: audit 2023-09-23T19:04:02.496775+0000 mon.a (mon.0) 579 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-09-23T19:04:03.772 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[33624]: cluster 2023-09-23T19:04:02.496915+0000 mon.a (mon.0) 580 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2023-09-23T19:04:03.772 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[33624]: cluster 2023-09-23T19:04:02.872770+0000 mgr.y (mgr.14454) 29 : cluster [DBG] pgmap v19: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:03.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:03 smithi092 bash[27556]: audit 2023-09-23T19:04:02.496775+0000 mon.a (mon.0) 579 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-09-23T19:04:03.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:03 smithi092 bash[27556]: cluster 2023-09-23T19:04:02.496915+0000 mon.a (mon.0) 580 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2023-09-23T19:04:03.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:03 smithi092 bash[27556]: cluster 2023-09-23T19:04:02.872770+0000 mgr.y (mgr.14454) 29 : cluster [DBG] pgmap v19: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:04.036 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[52296]: ts=2023-09-23T19:04:03.805Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-09-23T19:04:04.036 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[52296]: ts=2023-09-23T19:04:03.805Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-09-23T19:04:04.036 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[52296]: ts=2023-09-23T19:04:03.805Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-09-23T19:04:04.036 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[52296]: ts=2023-09-23T19:04:03.805Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-09-23T19:04:04.036 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[52296]: ts=2023-09-23T19:04:03.808Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-09-23T19:04:04.036 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[52296]: ts=2023-09-23T19:04:03.808Z caller=node_exporter.go:115 level=info collector=arp 2023-09-23T19:04:04.036 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[52296]: ts=2023-09-23T19:04:03.808Z caller=node_exporter.go:115 level=info collector=bcache 2023-09-23T19:04:04.037 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[52296]: ts=2023-09-23T19:04:03.808Z caller=node_exporter.go:115 level=info collector=bonding 2023-09-23T19:04:04.037 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[52296]: ts=2023-09-23T19:04:03.808Z caller=node_exporter.go:115 level=info collector=btrfs 2023-09-23T19:04:04.037 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[52296]: ts=2023-09-23T19:04:03.808Z caller=node_exporter.go:115 level=info collector=conntrack 2023-09-23T19:04:04.037 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[52296]: ts=2023-09-23T19:04:03.808Z caller=node_exporter.go:115 level=info collector=cpu 2023-09-23T19:04:04.037 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[52296]: ts=2023-09-23T19:04:03.808Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-09-23T19:04:04.037 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[52296]: ts=2023-09-23T19:04:03.808Z caller=node_exporter.go:115 level=info collector=diskstats 2023-09-23T19:04:04.037 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[52296]: ts=2023-09-23T19:04:03.808Z caller=node_exporter.go:115 level=info collector=dmi 2023-09-23T19:04:04.037 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[52296]: ts=2023-09-23T19:04:03.808Z caller=node_exporter.go:115 level=info collector=edac 2023-09-23T19:04:04.037 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[52296]: ts=2023-09-23T19:04:03.808Z caller=node_exporter.go:115 level=info collector=entropy 2023-09-23T19:04:04.037 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[52296]: ts=2023-09-23T19:04:03.808Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-09-23T19:04:04.038 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[52296]: ts=2023-09-23T19:04:03.808Z caller=node_exporter.go:115 level=info collector=filefd 2023-09-23T19:04:04.038 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[52296]: ts=2023-09-23T19:04:03.808Z caller=node_exporter.go:115 level=info collector=filesystem 2023-09-23T19:04:04.038 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[52296]: ts=2023-09-23T19:04:03.808Z caller=node_exporter.go:115 level=info collector=hwmon 2023-09-23T19:04:04.038 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[52296]: ts=2023-09-23T19:04:03.808Z caller=node_exporter.go:115 level=info collector=infiniband 2023-09-23T19:04:04.038 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[52296]: ts=2023-09-23T19:04:03.808Z caller=node_exporter.go:115 level=info collector=ipvs 2023-09-23T19:04:04.038 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[52296]: ts=2023-09-23T19:04:03.808Z caller=node_exporter.go:115 level=info collector=loadavg 2023-09-23T19:04:04.038 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[52296]: ts=2023-09-23T19:04:03.808Z caller=node_exporter.go:115 level=info collector=mdadm 2023-09-23T19:04:04.038 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[52296]: ts=2023-09-23T19:04:03.808Z caller=node_exporter.go:115 level=info collector=meminfo 2023-09-23T19:04:04.038 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[52296]: ts=2023-09-23T19:04:03.808Z caller=node_exporter.go:115 level=info collector=netclass 2023-09-23T19:04:04.038 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[52296]: ts=2023-09-23T19:04:03.808Z caller=node_exporter.go:115 level=info collector=netdev 2023-09-23T19:04:04.039 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[52296]: ts=2023-09-23T19:04:03.808Z caller=node_exporter.go:115 level=info collector=netstat 2023-09-23T19:04:04.039 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[52296]: ts=2023-09-23T19:04:03.808Z caller=node_exporter.go:115 level=info collector=nfs 2023-09-23T19:04:04.039 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[52296]: ts=2023-09-23T19:04:03.808Z caller=node_exporter.go:115 level=info collector=nfsd 2023-09-23T19:04:04.039 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[52296]: ts=2023-09-23T19:04:03.808Z caller=node_exporter.go:115 level=info collector=nvme 2023-09-23T19:04:04.039 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[52296]: ts=2023-09-23T19:04:03.808Z caller=node_exporter.go:115 level=info collector=os 2023-09-23T19:04:04.039 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[52296]: ts=2023-09-23T19:04:03.808Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-09-23T19:04:04.039 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[52296]: ts=2023-09-23T19:04:03.808Z caller=node_exporter.go:115 level=info collector=pressure 2023-09-23T19:04:04.039 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[52296]: ts=2023-09-23T19:04:03.808Z caller=node_exporter.go:115 level=info collector=rapl 2023-09-23T19:04:04.039 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[52296]: ts=2023-09-23T19:04:03.808Z caller=node_exporter.go:115 level=info collector=schedstat 2023-09-23T19:04:04.039 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[52296]: ts=2023-09-23T19:04:03.808Z caller=node_exporter.go:115 level=info collector=sockstat 2023-09-23T19:04:04.039 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[52296]: ts=2023-09-23T19:04:03.808Z caller=node_exporter.go:115 level=info collector=softnet 2023-09-23T19:04:04.040 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[52296]: ts=2023-09-23T19:04:03.808Z caller=node_exporter.go:115 level=info collector=stat 2023-09-23T19:04:04.040 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[52296]: ts=2023-09-23T19:04:03.808Z caller=node_exporter.go:115 level=info collector=tapestats 2023-09-23T19:04:04.040 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[52296]: ts=2023-09-23T19:04:03.808Z caller=node_exporter.go:115 level=info collector=textfile 2023-09-23T19:04:04.040 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[52296]: ts=2023-09-23T19:04:03.808Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-09-23T19:04:04.040 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[52296]: ts=2023-09-23T19:04:03.808Z caller=node_exporter.go:115 level=info collector=time 2023-09-23T19:04:04.040 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[52296]: ts=2023-09-23T19:04:03.808Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-09-23T19:04:04.040 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[52296]: ts=2023-09-23T19:04:03.808Z caller=node_exporter.go:115 level=info collector=uname 2023-09-23T19:04:04.040 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[52296]: ts=2023-09-23T19:04:03.808Z caller=node_exporter.go:115 level=info collector=vmstat 2023-09-23T19:04:04.040 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[52296]: ts=2023-09-23T19:04:03.808Z caller=node_exporter.go:115 level=info collector=xfs 2023-09-23T19:04:04.040 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[52296]: ts=2023-09-23T19:04:03.808Z caller=node_exporter.go:115 level=info collector=zfs 2023-09-23T19:04:04.041 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[52296]: ts=2023-09-23T19:04:03.808Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-09-23T19:04:04.041 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:04:03 smithi016 bash[52296]: ts=2023-09-23T19:04:03.809Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-09-23T19:04:04.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:04 smithi016 bash[33624]: cluster 2023-09-23T19:04:03.507553+0000 mon.a (mon.0) 581 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2023-09-23T19:04:04.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:04 smithi016 bash[27237]: cluster 2023-09-23T19:04:03.507553+0000 mon.a (mon.0) 581 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2023-09-23T19:04:04.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:04 smithi092 bash[27556]: cluster 2023-09-23T19:04:03.507553+0000 mon.a (mon.0) 581 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2023-09-23T19:04:06.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:05 smithi016 bash[27237]: cluster 2023-09-23T19:04:04.873269+0000 mgr.y (mgr.14454) 30 : cluster [DBG] pgmap v21: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:06.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:05 smithi016 bash[33624]: cluster 2023-09-23T19:04:04.873269+0000 mgr.y (mgr.14454) 30 : cluster [DBG] pgmap v21: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:06.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:05 smithi092 bash[27556]: cluster 2023-09-23T19:04:04.873269+0000 mgr.y (mgr.14454) 30 : cluster [DBG] pgmap v21: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:08.224 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:07 smithi016 bash[27237]: cluster 2023-09-23T19:04:06.873917+0000 mgr.y (mgr.14454) 31 : cluster [DBG] pgmap v22: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:08.224 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:07 smithi016 bash[33624]: cluster 2023-09-23T19:04:06.873917+0000 mgr.y (mgr.14454) 31 : cluster [DBG] pgmap v22: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:08.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:07 smithi092 bash[27556]: cluster 2023-09-23T19:04:06.873917+0000 mgr.y (mgr.14454) 31 : cluster [DBG] pgmap v22: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:08.925 INFO:teuthology.orchestra.run.smithi016.stderr:enabled application 'rbd' on pool 'rbd' 2023-09-23T19:04:09.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:08 smithi016 bash[27237]: audit 2023-09-23T19:04:08.617647+0000 mon.c (mon.2) 39 : audit [INF] from='client.? 172.21.15.16:0/4206491' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:04:09.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:08 smithi016 bash[27237]: audit 2023-09-23T19:04:08.618323+0000 mon.a (mon.0) 582 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:04:09.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:08 smithi016 bash[33624]: audit 2023-09-23T19:04:08.617647+0000 mon.c (mon.2) 39 : audit [INF] from='client.? 172.21.15.16:0/4206491' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:04:09.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:08 smithi016 bash[33624]: audit 2023-09-23T19:04:08.618323+0000 mon.a (mon.0) 582 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:04:09.287 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:04:09 smithi092 bash[28593]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:04:09] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T19:04:09.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:08 smithi092 bash[27556]: audit 2023-09-23T19:04:08.617647+0000 mon.c (mon.2) 39 : audit [INF] from='client.? 172.21.15.16:0/4206491' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:04:09.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:08 smithi092 bash[27556]: audit 2023-09-23T19:04:08.618323+0000 mon.a (mon.0) 582 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:04:09.870 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-09-23T19:04:09.870 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-09-23T19:04:09.870 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph mgr dump --format=json 2023-09-23T19:04:10.224 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:04:10 smithi016 bash[27503]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:04:10] "GET /metrics HTTP/1.1" 200 155641 "" "Prometheus/2.33.4" 2023-09-23T19:04:10.225 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:09 smithi016 bash[27237]: cluster 2023-09-23T19:04:08.874504+0000 mgr.y (mgr.14454) 32 : cluster [DBG] pgmap v23: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:10.225 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:09 smithi016 bash[27237]: audit 2023-09-23T19:04:08.918613+0000 mon.a (mon.0) 583 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-09-23T19:04:10.225 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:09 smithi016 bash[27237]: cluster 2023-09-23T19:04:08.918725+0000 mon.a (mon.0) 584 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2023-09-23T19:04:10.225 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:09 smithi016 bash[33624]: cluster 2023-09-23T19:04:08.874504+0000 mgr.y (mgr.14454) 32 : cluster [DBG] pgmap v23: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:10.225 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:09 smithi016 bash[33624]: audit 2023-09-23T19:04:08.918613+0000 mon.a (mon.0) 583 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-09-23T19:04:10.226 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:09 smithi016 bash[33624]: cluster 2023-09-23T19:04:08.918725+0000 mon.a (mon.0) 584 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2023-09-23T19:04:10.226 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Sep 23 19:04:10 smithi016 systemd[1]: Started Ceph alertmanager.a for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7. 2023-09-23T19:04:10.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:09 smithi092 bash[27556]: cluster 2023-09-23T19:04:08.874504+0000 mgr.y (mgr.14454) 32 : cluster [DBG] pgmap v23: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:10.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:09 smithi092 bash[27556]: audit 2023-09-23T19:04:08.918613+0000 mon.a (mon.0) 583 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-09-23T19:04:10.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:09 smithi092 bash[27556]: cluster 2023-09-23T19:04:08.918725+0000 mon.a (mon.0) 584 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2023-09-23T19:04:11.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:11 smithi016 bash[27237]: audit 2023-09-23T19:04:10.194907+0000 mon.a (mon.0) 585 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:04:11.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:11 smithi016 bash[27237]: audit 2023-09-23T19:04:10.198107+0000 mon.c (mon.2) 40 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:04:11.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:11 smithi016 bash[27237]: cluster 2023-09-23T19:04:10.875116+0000 mgr.y (mgr.14454) 33 : cluster [DBG] pgmap v25: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:11.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:11 smithi016 bash[33624]: audit 2023-09-23T19:04:10.194907+0000 mon.a (mon.0) 585 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:04:11.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:11 smithi016 bash[33624]: audit 2023-09-23T19:04:10.198107+0000 mon.c (mon.2) 40 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:04:11.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:11 smithi016 bash[33624]: cluster 2023-09-23T19:04:10.875116+0000 mgr.y (mgr.14454) 33 : cluster [DBG] pgmap v25: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:11.536 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Sep 23 19:04:11 smithi016 bash[53025]: level=info ts=2023-09-23T19:04:11.204Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2023-09-23T19:04:11.536 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Sep 23 19:04:11 smithi016 bash[53025]: level=info ts=2023-09-23T19:04:11.205Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2023-09-23T19:04:11.536 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Sep 23 19:04:11 smithi016 bash[53025]: level=info ts=2023-09-23T19:04:11.208Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.16 port=9094 2023-09-23T19:04:11.536 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Sep 23 19:04:11 smithi016 bash[53025]: level=info ts=2023-09-23T19:04:11.209Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-09-23T19:04:11.536 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Sep 23 19:04:11 smithi016 bash[53025]: level=info ts=2023-09-23T19:04:11.254Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T19:04:11.536 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Sep 23 19:04:11 smithi016 bash[53025]: level=info ts=2023-09-23T19:04:11.255Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T19:04:11.537 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Sep 23 19:04:11 smithi016 bash[53025]: level=info ts=2023-09-23T19:04:11.257Z caller=main.go:518 msg=Listening address=:9093 2023-09-23T19:04:11.537 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Sep 23 19:04:11 smithi016 bash[53025]: level=info ts=2023-09-23T19:04:11.258Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2023-09-23T19:04:11.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:11 smithi092 bash[27556]: audit 2023-09-23T19:04:10.194907+0000 mon.a (mon.0) 585 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:04:11.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:11 smithi092 bash[27556]: audit 2023-09-23T19:04:10.198107+0000 mon.c (mon.2) 40 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:04:11.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:11 smithi092 bash[27556]: cluster 2023-09-23T19:04:10.875116+0000 mgr.y (mgr.14454) 33 : cluster [DBG] pgmap v25: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:13.535 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Sep 23 19:04:13 smithi016 bash[53025]: level=info ts=2023-09-23T19:04:13.210Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000929241s 2023-09-23T19:04:14.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:13 smithi016 bash[27237]: cluster 2023-09-23T19:04:12.875830+0000 mgr.y (mgr.14454) 34 : cluster [DBG] pgmap v26: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:14.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:13 smithi016 bash[33624]: cluster 2023-09-23T19:04:12.875830+0000 mgr.y (mgr.14454) 34 : cluster [DBG] pgmap v26: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:14.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:13 smithi092 bash[27556]: cluster 2023-09-23T19:04:12.875830+0000 mgr.y (mgr.14454) 34 : cluster [DBG] pgmap v26: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:15.672 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:04:16.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:15 smithi016 bash[33624]: cluster 2023-09-23T19:04:14.876529+0000 mgr.y (mgr.14454) 35 : cluster [DBG] pgmap v27: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:16.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:15 smithi016 bash[33624]: audit 2023-09-23T19:04:15.100977+0000 mon.a (mon.0) 586 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:04:16.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:15 smithi016 bash[33624]: audit 2023-09-23T19:04:15.832431+0000 mon.a (mon.0) 587 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:04:16.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:15 smithi016 bash[27237]: cluster 2023-09-23T19:04:14.876529+0000 mgr.y (mgr.14454) 35 : cluster [DBG] pgmap v27: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:16.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:15 smithi016 bash[27237]: audit 2023-09-23T19:04:15.100977+0000 mon.a (mon.0) 586 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:04:16.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:15 smithi016 bash[27237]: audit 2023-09-23T19:04:15.832431+0000 mon.a (mon.0) 587 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:04:16.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:15 smithi092 bash[27556]: cluster 2023-09-23T19:04:14.876529+0000 mgr.y (mgr.14454) 35 : cluster [DBG] pgmap v27: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:16.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:15 smithi092 bash[27556]: audit 2023-09-23T19:04:15.100977+0000 mon.a (mon.0) 586 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:04:16.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:15 smithi092 bash[27556]: audit 2023-09-23T19:04:15.832431+0000 mon.a (mon.0) 587 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:04:17.035 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:04:16 smithi016 bash[27503]: [23/Sep/2023:19:04:16] ENGINE Bus STOPPING 2023-09-23T19:04:17.035 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:04:16 smithi016 bash[27503]: [23/Sep/2023:19:04:16] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T19:04:17.035 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:04:16 smithi016 bash[27503]: [23/Sep/2023:19:04:16] ENGINE Bus STOPPED 2023-09-23T19:04:17.035 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:04:16 smithi016 bash[27503]: [23/Sep/2023:19:04:16] ENGINE Bus STARTING 2023-09-23T19:04:17.036 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:04:16 smithi016 bash[27503]: [23/Sep/2023:19:04:16] ENGINE Serving on http://:::9283 2023-09-23T19:04:17.036 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:04:16 smithi016 bash[27503]: [23/Sep/2023:19:04:16] ENGINE Bus STARTED 2023-09-23T19:04:18.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:17 smithi016 bash[27237]: audit 2023-09-23T19:04:16.534106+0000 mon.a (mon.0) 588 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:04:18.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:17 smithi016 bash[27237]: audit 2023-09-23T19:04:16.537243+0000 mon.c (mon.2) 41 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:04:18.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:17 smithi016 bash[27237]: audit 2023-09-23T19:04:16.538701+0000 mon.c (mon.2) 42 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:04:18.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:17 smithi016 bash[27237]: audit 2023-09-23T19:04:16.544733+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:04:18.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:17 smithi016 bash[27237]: audit 2023-09-23T19:04:16.689205+0000 mon.a (mon.0) 590 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:04:18.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:17 smithi016 bash[27237]: audit 2023-09-23T19:04:16.694981+0000 mon.a (mon.0) 591 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:04:18.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:17 smithi016 bash[27237]: audit 2023-09-23T19:04:16.699942+0000 mon.c (mon.2) 43 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T19:04:18.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:17 smithi016 bash[27237]: audit 2023-09-23T19:04:16.700513+0000 mgr.y (mgr.14454) 36 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T19:04:18.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:17 smithi016 bash[27237]: audit 2023-09-23T19:04:16.706262+0000 mon.a (mon.0) 592 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:04:18.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:17 smithi016 bash[27237]: cephadm 2023-09-23T19:04:16.713616+0000 mgr.y (mgr.14454) 37 : cephadm [INF] Deploying daemon grafana.a on smithi092 2023-09-23T19:04:18.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:17 smithi016 bash[27237]: cluster 2023-09-23T19:04:16.877287+0000 mgr.y (mgr.14454) 38 : cluster [DBG] pgmap v28: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:18.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:17 smithi016 bash[33624]: audit 2023-09-23T19:04:16.534106+0000 mon.a (mon.0) 588 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:04:18.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:17 smithi016 bash[33624]: audit 2023-09-23T19:04:16.537243+0000 mon.c (mon.2) 41 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:04:18.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:17 smithi016 bash[33624]: audit 2023-09-23T19:04:16.538701+0000 mon.c (mon.2) 42 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:04:18.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:17 smithi016 bash[33624]: audit 2023-09-23T19:04:16.544733+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:04:18.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:17 smithi016 bash[33624]: audit 2023-09-23T19:04:16.689205+0000 mon.a (mon.0) 590 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:04:18.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:17 smithi016 bash[33624]: audit 2023-09-23T19:04:16.694981+0000 mon.a (mon.0) 591 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:04:18.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:17 smithi016 bash[33624]: audit 2023-09-23T19:04:16.699942+0000 mon.c (mon.2) 43 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T19:04:18.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:17 smithi016 bash[33624]: audit 2023-09-23T19:04:16.700513+0000 mgr.y (mgr.14454) 36 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T19:04:18.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:17 smithi016 bash[33624]: audit 2023-09-23T19:04:16.706262+0000 mon.a (mon.0) 592 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:04:18.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:17 smithi016 bash[33624]: cephadm 2023-09-23T19:04:16.713616+0000 mgr.y (mgr.14454) 37 : cephadm [INF] Deploying daemon grafana.a on smithi092 2023-09-23T19:04:18.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:17 smithi016 bash[33624]: cluster 2023-09-23T19:04:16.877287+0000 mgr.y (mgr.14454) 38 : cluster [DBG] pgmap v28: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:18.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:17 smithi092 bash[27556]: audit 2023-09-23T19:04:16.534106+0000 mon.a (mon.0) 588 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:04:18.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:17 smithi092 bash[27556]: audit 2023-09-23T19:04:16.537243+0000 mon.c (mon.2) 41 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:04:18.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:17 smithi092 bash[27556]: audit 2023-09-23T19:04:16.538701+0000 mon.c (mon.2) 42 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:04:18.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:17 smithi092 bash[27556]: audit 2023-09-23T19:04:16.544733+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:04:18.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:17 smithi092 bash[27556]: audit 2023-09-23T19:04:16.689205+0000 mon.a (mon.0) 590 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:04:18.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:17 smithi092 bash[27556]: audit 2023-09-23T19:04:16.694981+0000 mon.a (mon.0) 591 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:04:18.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:17 smithi092 bash[27556]: audit 2023-09-23T19:04:16.699942+0000 mon.c (mon.2) 43 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T19:04:18.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:17 smithi092 bash[27556]: audit 2023-09-23T19:04:16.700513+0000 mgr.y (mgr.14454) 36 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T19:04:18.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:17 smithi092 bash[27556]: audit 2023-09-23T19:04:16.706262+0000 mon.a (mon.0) 592 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:04:18.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:17 smithi092 bash[27556]: cephadm 2023-09-23T19:04:16.713616+0000 mgr.y (mgr.14454) 37 : cephadm [INF] Deploying daemon grafana.a on smithi092 2023-09-23T19:04:18.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:17 smithi092 bash[27556]: cluster 2023-09-23T19:04:16.877287+0000 mgr.y (mgr.14454) 38 : cluster [DBG] pgmap v28: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:18.866 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-09-23T19:04:19.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:18 smithi016 bash[27237]: audit 2023-09-23T19:04:18.853076+0000 mon.a (mon.0) 593 : audit [DBG] from='client.? 172.21.15.16:0/364347617' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T19:04:19.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:18 smithi016 bash[33624]: audit 2023-09-23T19:04:18.853076+0000 mon.a (mon.0) 593 : audit [DBG] from='client.? 172.21.15.16:0/364347617' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T19:04:19.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:18 smithi092 bash[27556]: audit 2023-09-23T19:04:18.853076+0000 mon.a (mon.0) 593 : audit [DBG] from='client.? 172.21.15.16:0/364347617' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T19:04:19.287 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:04:19 smithi092 bash[28593]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:04:19] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T19:04:19.552 INFO:teuthology.orchestra.run.smithi016.stdout:{"epoch":19,"active_gid":14454,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6800","nonce":1501185908},{"type":"v1","addr":"172.21.15.16:6801","nonce":1501185908}]},"active_addr":"172.21.15.16:6801/1501185908","active_change":"2023-09-23T19:03:34.846934+0000","active_mgr_features":4540138314316775423,"available":true,"standbys":[{"gid":24352,"name":"x","mgr_features":4540138314316775423,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.16:8443/","prometheus":"http://172.21.15.16:9283/"},"always_on_modules":{"nautilus":["balancer","crash","devicehealth","orchestrator_cli","progress","rbd_support","status","volumes"],"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":51,"active_clients":[{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.16:0","nonce":4152413938}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.16:0","nonce":3765291224}]}]} 2023-09-23T19:04:19.557 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-09-23T19:04:19.557 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-09-23T19:04:19.558 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T19:04:20.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:19 smithi016 bash[27237]: cluster 2023-09-23T19:04:18.877877+0000 mgr.y (mgr.14454) 39 : cluster [DBG] pgmap v29: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:20.286 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:04:20 smithi016 bash[27503]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:04:20] "GET /metrics HTTP/1.1" 200 155641 "" "Prometheus/2.33.4" 2023-09-23T19:04:20.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:19 smithi016 bash[33624]: cluster 2023-09-23T19:04:18.877877+0000 mgr.y (mgr.14454) 39 : cluster [DBG] pgmap v29: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:20.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:19 smithi092 bash[27556]: cluster 2023-09-23T19:04:18.877877+0000 mgr.y (mgr.14454) 39 : cluster [DBG] pgmap v29: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:21.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:21 smithi016 bash[27237]: audit 2023-09-23T19:04:20.112311+0000 mon.a (mon.0) 594 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:04:21.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:21 smithi016 bash[27237]: cluster 2023-09-23T19:04:20.878391+0000 mgr.y (mgr.14454) 40 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:21.536 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Sep 23 19:04:21 smithi016 bash[53025]: level=info ts=2023-09-23T19:04:21.213Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.004129512s 2023-09-23T19:04:21.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:21 smithi016 bash[33624]: audit 2023-09-23T19:04:20.112311+0000 mon.a (mon.0) 594 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:04:21.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:21 smithi016 bash[33624]: cluster 2023-09-23T19:04:20.878391+0000 mgr.y (mgr.14454) 40 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:21.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:21 smithi092 bash[27556]: audit 2023-09-23T19:04:20.112311+0000 mon.a (mon.0) 594 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:04:21.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:21 smithi092 bash[27556]: cluster 2023-09-23T19:04:20.878391+0000 mgr.y (mgr.14454) 40 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:24.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:23 smithi016 bash[27237]: cluster 2023-09-23T19:04:22.879125+0000 mgr.y (mgr.14454) 41 : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:24.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:23 smithi016 bash[33624]: cluster 2023-09-23T19:04:22.879125+0000 mgr.y (mgr.14454) 41 : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:24.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:23 smithi092 bash[27556]: cluster 2023-09-23T19:04:22.879125+0000 mgr.y (mgr.14454) 41 : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:24.337 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:04:26.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:25 smithi016 bash[27237]: cluster 2023-09-23T19:04:24.879750+0000 mgr.y (mgr.14454) 42 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:26.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:25 smithi016 bash[33624]: cluster 2023-09-23T19:04:24.879750+0000 mgr.y (mgr.14454) 42 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:26.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:25 smithi092 bash[27556]: cluster 2023-09-23T19:04:24.879750+0000 mgr.y (mgr.14454) 42 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:27.329 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-09-23T19:04:27.330 INFO:teuthology.orchestra.run.smithi016.stdout:{"epoch":55,"fsid":"95ce9eb6-5a42-11ee-8db4-212e2dc638e7","created":"2023-09-23T18:55:23.802934+0000","modified":"2023-09-23T19:04:08.904669+0000","last_up_change":"2023-09-23T19:03:02.463512+0000","last_in_change":"2023-09-23T19:02:36.618072+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-09-23T18:59:33.315416+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T19:04:01.440427+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"a4e8ed09-1195-4878-928a-13881bc04dc7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6802","nonce":2337435219},{"type":"v1","addr":"172.21.15.16:6803","nonce":2337435219}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6804","nonce":2337435219},{"type":"v1","addr":"172.21.15.16:6805","nonce":2337435219}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6808","nonce":2337435219},{"type":"v1","addr":"172.21.15.16:6809","nonce":2337435219}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6806","nonce":2337435219},{"type":"v1","addr":"172.21.15.16:6807","nonce":2337435219}]},"public_addr":"172.21.15.16:6803/2337435219","cluster_addr":"172.21.15.16:6805/2337435219","heartbeat_back_addr":"172.21.15.16:6809/2337435219","heartbeat_front_addr":"172.21.15.16:6807/2337435219","state":["exists","up"]},{"osd":1,"uuid":"c5ff2879-0d1c-481a-b847-248433c12aa4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6810","nonce":3893879931},{"type":"v1","addr":"172.21.15.16:6811","nonce":3893879931}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6812","nonce":3893879931},{"type":"v1","addr":"172.21.15.16:6813","nonce":3893879931}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6816","nonce":3893879931},{"type":"v1","addr":"172.21.15.16:6817","nonce":3893879931}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6814","nonce":3893879931},{"type":"v1","addr":"172.21.15.16:6815","nonce":3893879931}]},"public_addr":"172.21.15.16:6811/3893879931","cluster_addr":"172.21.15.16:6813/3893879931","heartbeat_back_addr":"172.21.15.16:6817/3893879931","heartbeat_front_addr":"172.21.15.16:6815/3893879931","state":["exists","up"]},{"osd":2,"uuid":"7974ce4d-9ed8-4a72-b44b-d500b34da8cf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6818","nonce":568840676},{"type":"v1","addr":"172.21.15.16:6819","nonce":568840676}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6820","nonce":568840676},{"type":"v1","addr":"172.21.15.16:6821","nonce":568840676}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6824","nonce":568840676},{"type":"v1","addr":"172.21.15.16:6825","nonce":568840676}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6822","nonce":568840676},{"type":"v1","addr":"172.21.15.16:6823","nonce":568840676}]},"public_addr":"172.21.15.16:6819/568840676","cluster_addr":"172.21.15.16:6821/568840676","heartbeat_back_addr":"172.21.15.16:6825/568840676","heartbeat_front_addr":"172.21.15.16:6823/568840676","state":["exists","up"]},{"osd":3,"uuid":"f5e0d57b-d629-4453-a527-ad77e6c210ad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6826","nonce":204922667},{"type":"v1","addr":"172.21.15.16:6827","nonce":204922667}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6828","nonce":204922667},{"type":"v1","addr":"172.21.15.16:6829","nonce":204922667}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6832","nonce":204922667},{"type":"v1","addr":"172.21.15.16:6833","nonce":204922667}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6830","nonce":204922667},{"type":"v1","addr":"172.21.15.16:6831","nonce":204922667}]},"public_addr":"172.21.15.16:6827/204922667","cluster_addr":"172.21.15.16:6829/204922667","heartbeat_back_addr":"172.21.15.16:6833/204922667","heartbeat_front_addr":"172.21.15.16:6831/204922667","state":["exists","up"]},{"osd":4,"uuid":"55a8dab2-6393-44ab-943e-cb7ab2cb7c31","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6800","nonce":99745912},{"type":"v1","addr":"172.21.15.92:6801","nonce":99745912}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6802","nonce":99745912},{"type":"v1","addr":"172.21.15.92:6803","nonce":99745912}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6806","nonce":99745912},{"type":"v1","addr":"172.21.15.92:6807","nonce":99745912}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6804","nonce":99745912},{"type":"v1","addr":"172.21.15.92:6805","nonce":99745912}]},"public_addr":"172.21.15.92:6801/99745912","cluster_addr":"172.21.15.92:6803/99745912","heartbeat_back_addr":"172.21.15.92:6807/99745912","heartbeat_front_addr":"172.21.15.92:6805/99745912","state":["exists","up"]},{"osd":5,"uuid":"b29b73b1-51d7-4484-bd79-2334c650cb62","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6808","nonce":1987214502},{"type":"v1","addr":"172.21.15.92:6809","nonce":1987214502}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6810","nonce":1987214502},{"type":"v1","addr":"172.21.15.92:6811","nonce":1987214502}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6814","nonce":1987214502},{"type":"v1","addr":"172.21.15.92:6815","nonce":1987214502}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6812","nonce":1987214502},{"type":"v1","addr":"172.21.15.92:6813","nonce":1987214502}]},"public_addr":"172.21.15.92:6809/1987214502","cluster_addr":"172.21.15.92:6811/1987214502","heartbeat_back_addr":"172.21.15.92:6815/1987214502","heartbeat_front_addr":"172.21.15.92:6813/1987214502","state":["exists","up"]},{"osd":6,"uuid":"9fe16cc9-f0d3-4b79-bc68-38305c55b551","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6816","nonce":2820798178},{"type":"v1","addr":"172.21.15.92:6817","nonce":2820798178}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6818","nonce":2820798178},{"type":"v1","addr":"172.21.15.92:6819","nonce":2820798178}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6822","nonce":2820798178},{"type":"v1","addr":"172.21.15.92:6823","nonce":2820798178}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6820","nonce":2820798178},{"type":"v1","addr":"172.21.15.92:6821","nonce":2820798178}]},"public_addr":"172.21.15.92:6817/2820798178","cluster_addr":"172.21.15.92:6819/2820798178","heartbeat_back_addr":"172.21.15.92:6823/2820798178","heartbeat_front_addr":"172.21.15.92:6821/2820798178","state":["exists","up"]},{"osd":7,"uuid":"d086b076-127f-42de-bee9-157bf579d92d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6824","nonce":98624761},{"type":"v1","addr":"172.21.15.92:6825","nonce":98624761}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6826","nonce":98624761},{"type":"v1","addr":"172.21.15.92:6827","nonce":98624761}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6830","nonce":98624761},{"type":"v1","addr":"172.21.15.92:6831","nonce":98624761}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6828","nonce":98624761},{"type":"v1","addr":"172.21.15.92:6829","nonce":98624761}]},"public_addr":"172.21.15.92:6825/98624761","cluster_addr":"172.21.15.92:6827/98624761","heartbeat_back_addr":"172.21.15.92:6831/98624761","heartbeat_front_addr":"172.21.15.92:6829/98624761","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:58:07.335980+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:58:49.342213+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:59:30.939986+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:00:12.926364+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:00:53.466692+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:01:36.332957+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:02:18.593854+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:03:00.817448+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.16:6801/1259630240":"2023-09-24T19:03:34.846451+0000","172.21.15.16:6800/1259630240":"2023-09-24T19:03:34.846451+0000","172.21.15.16:0/3767844203":"2023-09-24T19:03:34.846451+0000","172.21.15.16:6801/2756103674":"2023-09-24T18:56:20.574625+0000","172.21.15.16:0/2843459289":"2023-09-24T18:56:20.574625+0000","172.21.15.16:0/272242693":"2023-09-24T18:56:20.574625+0000","172.21.15.16:0/79169563":"2023-09-24T18:55:50.550546+0000","172.21.15.16:6800/2615282850":"2023-09-24T18:55:50.550546+0000","172.21.15.16:0/349446544":"2023-09-24T18:55:50.550546+0000","172.21.15.16:0/2669353350":"2023-09-24T19:03:34.846451+0000","172.21.15.16:6800/2756103674":"2023-09-24T18:56:20.574625+0000","172.21.15.16:6801/2615282850":"2023-09-24T18:55:50.550546+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T19:04:28.030 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-09-23T19:04:28.030 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T19:04:28.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:27 smithi016 bash[27237]: cluster 2023-09-23T19:04:26.880294+0000 mgr.y (mgr.14454) 43 : cluster [DBG] pgmap v33: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:28.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:27 smithi016 bash[27237]: audit 2023-09-23T19:04:27.325174+0000 mon.c (mon.2) 44 : audit [DBG] from='client.? 172.21.15.16:0/2197586948' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T19:04:28.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:27 smithi016 bash[33624]: cluster 2023-09-23T19:04:26.880294+0000 mgr.y (mgr.14454) 43 : cluster [DBG] pgmap v33: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:28.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:27 smithi016 bash[33624]: audit 2023-09-23T19:04:27.325174+0000 mon.c (mon.2) 44 : audit [DBG] from='client.? 172.21.15.16:0/2197586948' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T19:04:28.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:27 smithi092 bash[27556]: cluster 2023-09-23T19:04:26.880294+0000 mgr.y (mgr.14454) 43 : cluster [DBG] pgmap v33: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:28.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:27 smithi092 bash[27556]: audit 2023-09-23T19:04:27.325174+0000 mon.c (mon.2) 44 : audit [DBG] from='client.? 172.21.15.16:0/2197586948' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T19:04:29.538 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:04:29 smithi092 bash[28593]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:04:29] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T19:04:30.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:29 smithi016 bash[27237]: cluster 2023-09-23T19:04:28.880991+0000 mgr.y (mgr.14454) 44 : cluster [DBG] pgmap v34: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:30.285 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:04:30 smithi016 bash[27503]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:04:30] "GET /metrics HTTP/1.1" 200 155652 "" "Prometheus/2.33.4" 2023-09-23T19:04:30.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:29 smithi016 bash[33624]: cluster 2023-09-23T19:04:28.880991+0000 mgr.y (mgr.14454) 44 : cluster [DBG] pgmap v34: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:30.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:29 smithi092 bash[27556]: cluster 2023-09-23T19:04:28.880991+0000 mgr.y (mgr.14454) 44 : cluster [DBG] pgmap v34: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:32.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:31 smithi016 bash[27237]: cluster 2023-09-23T19:04:30.881644+0000 mgr.y (mgr.14454) 45 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:32.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:31 smithi016 bash[33624]: cluster 2023-09-23T19:04:30.881644+0000 mgr.y (mgr.14454) 45 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:32.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:31 smithi092 bash[27556]: cluster 2023-09-23T19:04:30.881644+0000 mgr.y (mgr.14454) 45 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:32.806 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:04:33.973 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:33 smithi016 bash[27237]: cluster 2023-09-23T19:04:32.882380+0000 mgr.y (mgr.14454) 46 : cluster [DBG] pgmap v36: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:33.973 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:33 smithi016 bash[33624]: cluster 2023-09-23T19:04:32.882380+0000 mgr.y (mgr.14454) 46 : cluster [DBG] pgmap v36: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:34.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:33 smithi092 bash[27556]: cluster 2023-09-23T19:04:32.882380+0000 mgr.y (mgr.14454) 46 : cluster [DBG] pgmap v36: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:35.864 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-09-23T19:04:35.864 INFO:teuthology.orchestra.run.smithi016.stdout:{"epoch":55,"fsid":"95ce9eb6-5a42-11ee-8db4-212e2dc638e7","created":"2023-09-23T18:55:23.802934+0000","modified":"2023-09-23T19:04:08.904669+0000","last_up_change":"2023-09-23T19:03:02.463512+0000","last_in_change":"2023-09-23T19:02:36.618072+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-09-23T18:59:33.315416+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T19:04:01.440427+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"a4e8ed09-1195-4878-928a-13881bc04dc7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6802","nonce":2337435219},{"type":"v1","addr":"172.21.15.16:6803","nonce":2337435219}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6804","nonce":2337435219},{"type":"v1","addr":"172.21.15.16:6805","nonce":2337435219}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6808","nonce":2337435219},{"type":"v1","addr":"172.21.15.16:6809","nonce":2337435219}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6806","nonce":2337435219},{"type":"v1","addr":"172.21.15.16:6807","nonce":2337435219}]},"public_addr":"172.21.15.16:6803/2337435219","cluster_addr":"172.21.15.16:6805/2337435219","heartbeat_back_addr":"172.21.15.16:6809/2337435219","heartbeat_front_addr":"172.21.15.16:6807/2337435219","state":["exists","up"]},{"osd":1,"uuid":"c5ff2879-0d1c-481a-b847-248433c12aa4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6810","nonce":3893879931},{"type":"v1","addr":"172.21.15.16:6811","nonce":3893879931}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6812","nonce":3893879931},{"type":"v1","addr":"172.21.15.16:6813","nonce":3893879931}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6816","nonce":3893879931},{"type":"v1","addr":"172.21.15.16:6817","nonce":3893879931}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6814","nonce":3893879931},{"type":"v1","addr":"172.21.15.16:6815","nonce":3893879931}]},"public_addr":"172.21.15.16:6811/3893879931","cluster_addr":"172.21.15.16:6813/3893879931","heartbeat_back_addr":"172.21.15.16:6817/3893879931","heartbeat_front_addr":"172.21.15.16:6815/3893879931","state":["exists","up"]},{"osd":2,"uuid":"7974ce4d-9ed8-4a72-b44b-d500b34da8cf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6818","nonce":568840676},{"type":"v1","addr":"172.21.15.16:6819","nonce":568840676}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6820","nonce":568840676},{"type":"v1","addr":"172.21.15.16:6821","nonce":568840676}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6824","nonce":568840676},{"type":"v1","addr":"172.21.15.16:6825","nonce":568840676}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6822","nonce":568840676},{"type":"v1","addr":"172.21.15.16:6823","nonce":568840676}]},"public_addr":"172.21.15.16:6819/568840676","cluster_addr":"172.21.15.16:6821/568840676","heartbeat_back_addr":"172.21.15.16:6825/568840676","heartbeat_front_addr":"172.21.15.16:6823/568840676","state":["exists","up"]},{"osd":3,"uuid":"f5e0d57b-d629-4453-a527-ad77e6c210ad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6826","nonce":204922667},{"type":"v1","addr":"172.21.15.16:6827","nonce":204922667}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6828","nonce":204922667},{"type":"v1","addr":"172.21.15.16:6829","nonce":204922667}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6832","nonce":204922667},{"type":"v1","addr":"172.21.15.16:6833","nonce":204922667}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6830","nonce":204922667},{"type":"v1","addr":"172.21.15.16:6831","nonce":204922667}]},"public_addr":"172.21.15.16:6827/204922667","cluster_addr":"172.21.15.16:6829/204922667","heartbeat_back_addr":"172.21.15.16:6833/204922667","heartbeat_front_addr":"172.21.15.16:6831/204922667","state":["exists","up"]},{"osd":4,"uuid":"55a8dab2-6393-44ab-943e-cb7ab2cb7c31","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6800","nonce":99745912},{"type":"v1","addr":"172.21.15.92:6801","nonce":99745912}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6802","nonce":99745912},{"type":"v1","addr":"172.21.15.92:6803","nonce":99745912}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6806","nonce":99745912},{"type":"v1","addr":"172.21.15.92:6807","nonce":99745912}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6804","nonce":99745912},{"type":"v1","addr":"172.21.15.92:6805","nonce":99745912}]},"public_addr":"172.21.15.92:6801/99745912","cluster_addr":"172.21.15.92:6803/99745912","heartbeat_back_addr":"172.21.15.92:6807/99745912","heartbeat_front_addr":"172.21.15.92:6805/99745912","state":["exists","up"]},{"osd":5,"uuid":"b29b73b1-51d7-4484-bd79-2334c650cb62","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6808","nonce":1987214502},{"type":"v1","addr":"172.21.15.92:6809","nonce":1987214502}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6810","nonce":1987214502},{"type":"v1","addr":"172.21.15.92:6811","nonce":1987214502}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6814","nonce":1987214502},{"type":"v1","addr":"172.21.15.92:6815","nonce":1987214502}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6812","nonce":1987214502},{"type":"v1","addr":"172.21.15.92:6813","nonce":1987214502}]},"public_addr":"172.21.15.92:6809/1987214502","cluster_addr":"172.21.15.92:6811/1987214502","heartbeat_back_addr":"172.21.15.92:6815/1987214502","heartbeat_front_addr":"172.21.15.92:6813/1987214502","state":["exists","up"]},{"osd":6,"uuid":"9fe16cc9-f0d3-4b79-bc68-38305c55b551","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6816","nonce":2820798178},{"type":"v1","addr":"172.21.15.92:6817","nonce":2820798178}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6818","nonce":2820798178},{"type":"v1","addr":"172.21.15.92:6819","nonce":2820798178}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6822","nonce":2820798178},{"type":"v1","addr":"172.21.15.92:6823","nonce":2820798178}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6820","nonce":2820798178},{"type":"v1","addr":"172.21.15.92:6821","nonce":2820798178}]},"public_addr":"172.21.15.92:6817/2820798178","cluster_addr":"172.21.15.92:6819/2820798178","heartbeat_back_addr":"172.21.15.92:6823/2820798178","heartbeat_front_addr":"172.21.15.92:6821/2820798178","state":["exists","up"]},{"osd":7,"uuid":"d086b076-127f-42de-bee9-157bf579d92d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6824","nonce":98624761},{"type":"v1","addr":"172.21.15.92:6825","nonce":98624761}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6826","nonce":98624761},{"type":"v1","addr":"172.21.15.92:6827","nonce":98624761}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6830","nonce":98624761},{"type":"v1","addr":"172.21.15.92:6831","nonce":98624761}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6828","nonce":98624761},{"type":"v1","addr":"172.21.15.92:6829","nonce":98624761}]},"public_addr":"172.21.15.92:6825/98624761","cluster_addr":"172.21.15.92:6827/98624761","heartbeat_back_addr":"172.21.15.92:6831/98624761","heartbeat_front_addr":"172.21.15.92:6829/98624761","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:58:07.335980+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:58:49.342213+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:59:30.939986+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:00:12.926364+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:00:53.466692+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:01:36.332957+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:02:18.593854+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:03:00.817448+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.16:6801/1259630240":"2023-09-24T19:03:34.846451+0000","172.21.15.16:6800/1259630240":"2023-09-24T19:03:34.846451+0000","172.21.15.16:0/3767844203":"2023-09-24T19:03:34.846451+0000","172.21.15.16:6801/2756103674":"2023-09-24T18:56:20.574625+0000","172.21.15.16:0/2843459289":"2023-09-24T18:56:20.574625+0000","172.21.15.16:0/272242693":"2023-09-24T18:56:20.574625+0000","172.21.15.16:0/79169563":"2023-09-24T18:55:50.550546+0000","172.21.15.16:6800/2615282850":"2023-09-24T18:55:50.550546+0000","172.21.15.16:0/349446544":"2023-09-24T18:55:50.550546+0000","172.21.15.16:0/2669353350":"2023-09-24T19:03:34.846451+0000","172.21.15.16:6800/2756103674":"2023-09-24T18:56:20.574625+0000","172.21.15.16:6801/2615282850":"2023-09-24T18:55:50.550546+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T19:04:36.248 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:35 smithi092 bash[27556]: cluster 2023-09-23T19:04:34.882998+0000 mgr.y (mgr.14454) 47 : cluster [DBG] pgmap v37: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:36.248 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:35 smithi092 bash[27556]: audit 2023-09-23T19:04:35.859165+0000 mon.c (mon.2) 45 : audit [DBG] from='client.? 172.21.15.16:0/1394475188' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T19:04:36.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:35 smithi016 bash[27237]: cluster 2023-09-23T19:04:34.882998+0000 mgr.y (mgr.14454) 47 : cluster [DBG] pgmap v37: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:36.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:35 smithi016 bash[27237]: audit 2023-09-23T19:04:35.859165+0000 mon.c (mon.2) 45 : audit [DBG] from='client.? 172.21.15.16:0/1394475188' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T19:04:36.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:35 smithi016 bash[33624]: cluster 2023-09-23T19:04:34.882998+0000 mgr.y (mgr.14454) 47 : cluster [DBG] pgmap v37: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:36.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:35 smithi016 bash[33624]: audit 2023-09-23T19:04:35.859165+0000 mon.c (mon.2) 45 : audit [DBG] from='client.? 172.21.15.16:0/1394475188' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T19:04:36.547 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-09-23T19:04:36.548 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-09-23T19:04:36.548 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-09-23T19:04:36.548 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-09-23T19:04:36.549 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-09-23T19:04:36.549 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-09-23T19:04:36.550 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-09-23T19:04:36.550 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-09-23T19:04:38.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:37 smithi016 bash[27237]: cluster 2023-09-23T19:04:36.883812+0000 mgr.y (mgr.14454) 48 : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:38.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:37 smithi016 bash[33624]: cluster 2023-09-23T19:04:36.883812+0000 mgr.y (mgr.14454) 48 : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:38.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:37 smithi092 bash[27556]: cluster 2023-09-23T19:04:36.883812+0000 mgr.y (mgr.14454) 48 : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:39.287 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:04:39 smithi092 bash[28593]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:04:39] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T19:04:39.288 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:38 smithi092 systemd[1]: Started Ceph grafana.a for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7. 2023-09-23T19:04:39.982 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:39 smithi092 bash[27556]: audit 2023-09-23T19:04:38.877383+0000 mon.a (mon.0) 595 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:04:39.982 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:39 smithi092 bash[27556]: audit 2023-09-23T19:04:38.881867+0000 mon.c (mon.2) 46 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:04:39.982 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:39 smithi092 bash[27556]: cluster 2023-09-23T19:04:38.885137+0000 mgr.y (mgr.14454) 49 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:39.983 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:39 smithi092 bash[47618]: t=2023-09-23T19:04:39+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2023-09-23T19:04:39.983 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:39 smithi092 bash[47618]: t=2023-09-23T19:04:39+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2023-09-23T19:04:39.983 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:39 smithi092 bash[47618]: t=2023-09-23T19:04:39+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2023-09-23T19:04:39.983 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:39 smithi092 bash[47618]: t=2023-09-23T19:04:39+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2023-09-23T19:04:39.983 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:39 smithi092 bash[47618]: t=2023-09-23T19:04:39+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2023-09-23T19:04:39.983 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:39 smithi092 bash[47618]: t=2023-09-23T19:04:39+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2023-09-23T19:04:39.983 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:39 smithi092 bash[47618]: t=2023-09-23T19:04:39+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-09-23T19:04:39.983 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:39 smithi092 bash[47618]: t=2023-09-23T19:04:39+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-09-23T19:04:39.983 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:39 smithi092 bash[47618]: t=2023-09-23T19:04:39+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2023-09-23T19:04:39.983 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:39 smithi092 bash[47618]: t=2023-09-23T19:04:39+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2023-09-23T19:04:39.984 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:39 smithi092 bash[47618]: t=2023-09-23T19:04:39+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2023-09-23T19:04:39.984 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:39 smithi092 bash[47618]: t=2023-09-23T19:04:39+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2023-09-23T19:04:39.984 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:39 smithi092 bash[47618]: t=2023-09-23T19:04:39+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2023-09-23T19:04:39.984 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:39 smithi092 bash[47618]: t=2023-09-23T19:04:39+0000 lvl=info msg="App mode production" logger=settings 2023-09-23T19:04:39.984 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:39 smithi092 bash[47618]: t=2023-09-23T19:04:39+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2023-09-23T19:04:39.984 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:39 smithi092 bash[47618]: t=2023-09-23T19:04:39+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-09-23T19:04:39.984 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:39 smithi092 bash[47618]: t=2023-09-23T19:04:39+0000 lvl=info msg="Starting DB migrations" logger=migrator 2023-09-23T19:04:39.984 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:39 smithi092 bash[47618]: t=2023-09-23T19:04:39+0000 lvl=info msg="Executing migration" logger=migrator id="create migration_log table" 2023-09-23T19:04:39.984 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:39 smithi092 bash[47618]: t=2023-09-23T19:04:39+0000 lvl=info msg="Executing migration" logger=migrator id="create user table" 2023-09-23T19:04:39.984 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:39 smithi092 bash[47618]: t=2023-09-23T19:04:39+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.login" 2023-09-23T19:04:39.985 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:39 smithi092 bash[47618]: t=2023-09-23T19:04:39+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.email" 2023-09-23T19:04:40.259 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:40 smithi092 bash[47618]: t=2023-09-23T19:04:40+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_login - v1" 2023-09-23T19:04:40.259 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:40 smithi092 bash[47618]: t=2023-09-23T19:04:40+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_email - v1" 2023-09-23T19:04:40.259 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:40 smithi092 bash[47618]: t=2023-09-23T19:04:40+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table user to user_v1 - v1" 2023-09-23T19:04:40.259 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:40 smithi092 bash[47618]: t=2023-09-23T19:04:40+0000 lvl=info msg="Executing migration" logger=migrator id="create user table v2" 2023-09-23T19:04:40.259 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:40 smithi092 bash[47618]: t=2023-09-23T19:04:40+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_login - v2" 2023-09-23T19:04:40.285 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:04:40 smithi016 bash[27503]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:04:40] "GET /metrics HTTP/1.1" 200 155652 "" "Prometheus/2.33.4" 2023-09-23T19:04:40.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:39 smithi016 bash[27237]: audit 2023-09-23T19:04:38.877383+0000 mon.a (mon.0) 595 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:04:40.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:39 smithi016 bash[27237]: audit 2023-09-23T19:04:38.881867+0000 mon.c (mon.2) 46 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:04:40.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:39 smithi016 bash[27237]: cluster 2023-09-23T19:04:38.885137+0000 mgr.y (mgr.14454) 49 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:40.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:39 smithi016 bash[33624]: audit 2023-09-23T19:04:38.877383+0000 mon.a (mon.0) 595 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:04:40.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:39 smithi016 bash[33624]: audit 2023-09-23T19:04:38.881867+0000 mon.c (mon.2) 46 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:04:40.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:39 smithi016 bash[33624]: cluster 2023-09-23T19:04:38.885137+0000 mgr.y (mgr.14454) 49 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:40.528 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:40 smithi092 bash[47618]: t=2023-09-23T19:04:40+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_email - v2" 2023-09-23T19:04:40.528 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:40 smithi092 bash[47618]: t=2023-09-23T19:04:40+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2023-09-23T19:04:40.528 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:40 smithi092 bash[47618]: t=2023-09-23T19:04:40+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table user_v1" 2023-09-23T19:04:40.528 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:40 smithi092 bash[47618]: t=2023-09-23T19:04:40+0000 lvl=info msg="Executing migration" logger=migrator id="Add column help_flags1 to user table" 2023-09-23T19:04:40.529 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:40 smithi092 bash[47618]: t=2023-09-23T19:04:40+0000 lvl=info msg="Executing migration" logger=migrator id="Update user table charset" 2023-09-23T19:04:40.529 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:40 smithi092 bash[47618]: t=2023-09-23T19:04:40+0000 lvl=info msg="Executing migration" logger=migrator id="Add last_seen_at column to user" 2023-09-23T19:04:40.529 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:40 smithi092 bash[47618]: t=2023-09-23T19:04:40+0000 lvl=info msg="Executing migration" logger=migrator id="Add missing user data" 2023-09-23T19:04:40.529 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:40 smithi092 bash[47618]: t=2023-09-23T19:04:40+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_disabled column to user" 2023-09-23T19:04:40.529 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:40 smithi092 bash[47618]: t=2023-09-23T19:04:40+0000 lvl=info msg="Executing migration" logger=migrator id="Add index user.login/user.email" 2023-09-23T19:04:40.787 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:40 smithi092 bash[47618]: t=2023-09-23T19:04:40+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_service_account column to user" 2023-09-23T19:04:40.788 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:40 smithi092 bash[47618]: t=2023-09-23T19:04:40+0000 lvl=info msg="Executing migration" logger=migrator id="create temp user table v1-7" 2023-09-23T19:04:40.788 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:40 smithi092 bash[47618]: t=2023-09-23T19:04:40+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v1-7" 2023-09-23T19:04:40.788 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:40 smithi092 bash[47618]: t=2023-09-23T19:04:40+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v1-7" 2023-09-23T19:04:40.788 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:40 smithi092 bash[47618]: t=2023-09-23T19:04:40+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v1-7" 2023-09-23T19:04:40.788 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:40 smithi092 bash[47618]: t=2023-09-23T19:04:40+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v1-7" 2023-09-23T19:04:40.788 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:40 smithi092 bash[47618]: t=2023-09-23T19:04:40+0000 lvl=info msg="Executing migration" logger=migrator id="Update temp_user table charset" 2023-09-23T19:04:40.788 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:40 smithi092 bash[47618]: t=2023-09-23T19:04:40+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_email - v1" 2023-09-23T19:04:41.084 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:40 smithi092 bash[47618]: t=2023-09-23T19:04:40+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_org_id - v1" 2023-09-23T19:04:41.084 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:40 smithi092 bash[47618]: t=2023-09-23T19:04:40+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_code - v1" 2023-09-23T19:04:41.084 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:40 smithi092 bash[47618]: t=2023-09-23T19:04:40+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_status - v1" 2023-09-23T19:04:41.084 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:40 smithi092 bash[47618]: t=2023-09-23T19:04:40+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2023-09-23T19:04:41.084 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:40 smithi092 bash[47618]: t=2023-09-23T19:04:40+0000 lvl=info msg="Executing migration" logger=migrator id="create temp_user v2" 2023-09-23T19:04:41.084 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:40 smithi092 bash[47618]: t=2023-09-23T19:04:40+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v2" 2023-09-23T19:04:41.084 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:40 smithi092 bash[47618]: t=2023-09-23T19:04:40+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v2" 2023-09-23T19:04:41.085 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:41 smithi092 bash[47618]: t=2023-09-23T19:04:41+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v2" 2023-09-23T19:04:41.085 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:41 smithi092 bash[47618]: t=2023-09-23T19:04:41+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v2" 2023-09-23T19:04:41.085 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:41 smithi092 bash[47618]: t=2023-09-23T19:04:41+0000 lvl=info msg="Executing migration" logger=migrator id="copy temp_user v1 to v2" 2023-09-23T19:04:41.085 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:41 smithi092 bash[47618]: t=2023-09-23T19:04:41+0000 lvl=info msg="Executing migration" logger=migrator id="drop temp_user_tmp_qwerty" 2023-09-23T19:04:41.336 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:41 smithi092 bash[27556]: audit 2023-09-23T19:04:40.132717+0000 mon.a (mon.0) 596 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:04:41.336 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:41 smithi092 bash[27556]: cluster 2023-09-23T19:04:40.885657+0000 mgr.y (mgr.14454) 50 : cluster [DBG] pgmap v40: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:41.336 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:41 smithi092 bash[47618]: t=2023-09-23T19:04:41+0000 lvl=info msg="Executing migration" logger=migrator id="Set created for temp users that will otherwise prematurely expire" 2023-09-23T19:04:41.337 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:41 smithi092 bash[47618]: t=2023-09-23T19:04:41+0000 lvl=info msg="Executing migration" logger=migrator id="create star table" 2023-09-23T19:04:41.337 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:41 smithi092 bash[47618]: t=2023-09-23T19:04:41+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index star.user_id_dashboard_id" 2023-09-23T19:04:41.337 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:41 smithi092 bash[47618]: t=2023-09-23T19:04:41+0000 lvl=info msg="Executing migration" logger=migrator id="create org table v1" 2023-09-23T19:04:41.337 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:41 smithi092 bash[47618]: t=2023-09-23T19:04:41+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_name - v1" 2023-09-23T19:04:41.337 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:41 smithi092 bash[47618]: t=2023-09-23T19:04:41+0000 lvl=info msg="Executing migration" logger=migrator id="create org_user table v1" 2023-09-23T19:04:41.442 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:04:41.443 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:04:41.444 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:04:41.444 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:04:41.444 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:04:41.452 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:04:41.452 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:04:41.457 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:04:41.476 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:41 smithi016 bash[33624]: audit 2023-09-23T19:04:40.132717+0000 mon.a (mon.0) 596 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:04:41.476 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:41 smithi016 bash[33624]: cluster 2023-09-23T19:04:40.885657+0000 mgr.y (mgr.14454) 50 : cluster [DBG] pgmap v40: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:41.477 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:41 smithi016 bash[27237]: audit 2023-09-23T19:04:40.132717+0000 mon.a (mon.0) 596 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:04:41.477 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:41 smithi016 bash[27237]: cluster 2023-09-23T19:04:40.885657+0000 mgr.y (mgr.14454) 50 : cluster [DBG] pgmap v40: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:41.606 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:41 smithi092 bash[47618]: t=2023-09-23T19:04:41+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_org_id - v1" 2023-09-23T19:04:41.606 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:41 smithi092 bash[47618]: t=2023-09-23T19:04:41+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_user_org_id_user_id - v1" 2023-09-23T19:04:41.606 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:41 smithi092 bash[47618]: t=2023-09-23T19:04:41+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_user_id - v1" 2023-09-23T19:04:41.606 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:41 smithi092 bash[47618]: t=2023-09-23T19:04:41+0000 lvl=info msg="Executing migration" logger=migrator id="Update org table charset" 2023-09-23T19:04:41.606 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:41 smithi092 bash[47618]: t=2023-09-23T19:04:41+0000 lvl=info msg="Executing migration" logger=migrator id="Update org_user table charset" 2023-09-23T19:04:41.607 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:41 smithi092 bash[47618]: t=2023-09-23T19:04:41+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate all Read Only Viewers to Viewers" 2023-09-23T19:04:41.607 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:41 smithi092 bash[47618]: t=2023-09-23T19:04:41+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard table" 2023-09-23T19:04:41.607 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:41 smithi092 bash[47618]: t=2023-09-23T19:04:41+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard.account_id" 2023-09-23T19:04:41.607 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:41 smithi092 bash[47618]: t=2023-09-23T19:04:41+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_account_id_slug" 2023-09-23T19:04:41.607 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:41 smithi092 bash[47618]: t=2023-09-23T19:04:41+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_tag table" 2023-09-23T19:04:41.875 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:41 smithi092 bash[47618]: t=2023-09-23T19:04:41+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_tag.dasboard_id_term" 2023-09-23T19:04:41.875 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:41 smithi092 bash[47618]: t=2023-09-23T19:04:41+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2023-09-23T19:04:41.875 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:41 smithi092 bash[47618]: t=2023-09-23T19:04:41+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard to dashboard_v1 - v1" 2023-09-23T19:04:41.875 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:41 smithi092 bash[47618]: t=2023-09-23T19:04:41+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard v2" 2023-09-23T19:04:41.875 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:41 smithi092 bash[47618]: t=2023-09-23T19:04:41+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_org_id - v2" 2023-09-23T19:04:41.876 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:41 smithi092 bash[47618]: t=2023-09-23T19:04:41+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_org_id_slug - v2" 2023-09-23T19:04:41.876 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:41 smithi092 bash[47618]: t=2023-09-23T19:04:41+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard v1 to v2" 2023-09-23T19:04:41.876 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:41 smithi092 bash[47618]: t=2023-09-23T19:04:41+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_v1" 2023-09-23T19:04:42.161 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:41 smithi092 bash[47618]: t=2023-09-23T19:04:41+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard.data to mediumtext v1" 2023-09-23T19:04:42.162 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:41 smithi092 bash[47618]: t=2023-09-23T19:04:41+0000 lvl=info msg="Executing migration" logger=migrator id="Add column updated_by in dashboard - v2" 2023-09-23T19:04:42.162 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:41 smithi092 bash[47618]: t=2023-09-23T19:04:41+0000 lvl=info msg="Executing migration" logger=migrator id="Add column created_by in dashboard - v2" 2023-09-23T19:04:42.162 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:41 smithi092 bash[47618]: t=2023-09-23T19:04:41+0000 lvl=info msg="Executing migration" logger=migrator id="Add column gnetId in dashboard" 2023-09-23T19:04:42.162 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:42 smithi092 bash[47618]: t=2023-09-23T19:04:42+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for gnetId in dashboard" 2023-09-23T19:04:42.162 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:42 smithi092 bash[47618]: t=2023-09-23T19:04:42+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_id in dashboard" 2023-09-23T19:04:42.162 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:42 smithi092 bash[47618]: t=2023-09-23T19:04:42+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for plugin_id in dashboard" 2023-09-23T19:04:42.162 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:42 smithi092 bash[47618]: t=2023-09-23T19:04:42+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_id in dashboard_tag" 2023-09-23T19:04:42.162 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:42 smithi092 bash[47618]: t=2023-09-23T19:04:42+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard table charset" 2023-09-23T19:04:42.162 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:42 smithi092 bash[47618]: t=2023-09-23T19:04:42+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_tag table charset" 2023-09-23T19:04:42.431 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:42 smithi092 bash[47618]: t=2023-09-23T19:04:42+0000 lvl=info msg="Executing migration" logger=migrator id="Add column folder_id in dashboard" 2023-09-23T19:04:42.431 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:42 smithi092 bash[47618]: t=2023-09-23T19:04:42+0000 lvl=info msg="Executing migration" logger=migrator id="Add column isFolder in dashboard" 2023-09-23T19:04:42.431 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:42 smithi092 bash[47618]: t=2023-09-23T19:04:42+0000 lvl=info msg="Executing migration" logger=migrator id="Add column has_acl in dashboard" 2023-09-23T19:04:42.431 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:42 smithi092 bash[47618]: t=2023-09-23T19:04:42+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in dashboard" 2023-09-23T19:04:42.431 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:42 smithi092 bash[47618]: t=2023-09-23T19:04:42+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in dashboard" 2023-09-23T19:04:42.431 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:42 smithi092 bash[47618]: t=2023-09-23T19:04:42+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index dashboard_org_id_uid" 2023-09-23T19:04:42.431 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:42 smithi092 bash[47618]: t=2023-09-23T19:04:42+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_slug" 2023-09-23T19:04:42.432 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:42 smithi092 bash[47618]: t=2023-09-23T19:04:42+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard title length" 2023-09-23T19:04:42.432 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:42 smithi092 bash[47618]: t=2023-09-23T19:04:42+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index for dashboard_org_id_title_folder_id" 2023-09-23T19:04:42.700 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:42 smithi092 bash[47618]: t=2023-09-23T19:04:42+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning" 2023-09-23T19:04:42.700 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:42 smithi092 bash[47618]: t=2023-09-23T19:04:42+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2023-09-23T19:04:42.700 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:42 smithi092 bash[47618]: t=2023-09-23T19:04:42+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning v2" 2023-09-23T19:04:42.700 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:42 smithi092 bash[47618]: t=2023-09-23T19:04:42+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2023-09-23T19:04:42.700 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:42 smithi092 bash[47618]: t=2023-09-23T19:04:42+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2023-09-23T19:04:42.700 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:42 smithi092 bash[47618]: t=2023-09-23T19:04:42+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard_provisioning v1 to v2" 2023-09-23T19:04:42.700 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:42 smithi092 bash[47618]: t=2023-09-23T19:04:42+0000 lvl=info msg="Executing migration" logger=migrator id="drop dashboard_provisioning_tmp_qwerty" 2023-09-23T19:04:42.980 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:42 smithi092 bash[47618]: t=2023-09-23T19:04:42+0000 lvl=info msg="Executing migration" logger=migrator id="Add check_sum column" 2023-09-23T19:04:42.981 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:42 smithi092 bash[47618]: t=2023-09-23T19:04:42+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_title" 2023-09-23T19:04:42.981 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:42 smithi092 bash[47618]: t=2023-09-23T19:04:42+0000 lvl=info msg="Executing migration" logger=migrator id="delete tags for deleted dashboards" 2023-09-23T19:04:42.981 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:42 smithi092 bash[47618]: t=2023-09-23T19:04:42+0000 lvl=info msg="Executing migration" logger=migrator id="delete stars for deleted dashboards" 2023-09-23T19:04:42.981 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:42 smithi092 bash[47618]: t=2023-09-23T19:04:42+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_is_folder" 2023-09-23T19:04:42.981 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:42 smithi092 bash[47618]: t=2023-09-23T19:04:42+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table" 2023-09-23T19:04:42.981 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:42 smithi092 bash[47618]: t=2023-09-23T19:04:42+0000 lvl=info msg="Executing migration" logger=migrator id="add index data_source.account_id" 2023-09-23T19:04:42.981 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:42 smithi092 bash[47618]: t=2023-09-23T19:04:42+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index data_source.account_id_name" 2023-09-23T19:04:42.981 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:42 smithi092 bash[47618]: t=2023-09-23T19:04:42+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_data_source_account_id - v1" 2023-09-23T19:04:43.247 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:43 smithi092 bash[47618]: t=2023-09-23T19:04:43+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_data_source_account_id_name - v1" 2023-09-23T19:04:43.247 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:43 smithi092 bash[47618]: t=2023-09-23T19:04:43+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table data_source to data_source_v1 - v1" 2023-09-23T19:04:43.247 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:43 smithi092 bash[47618]: t=2023-09-23T19:04:43+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table v2" 2023-09-23T19:04:43.247 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:43 smithi092 bash[47618]: t=2023-09-23T19:04:43+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_data_source_org_id - v2" 2023-09-23T19:04:43.247 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:43 smithi092 bash[47618]: t=2023-09-23T19:04:43+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_data_source_org_id_name - v2" 2023-09-23T19:04:43.248 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:43 smithi092 bash[47618]: t=2023-09-23T19:04:43+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2023-09-23T19:04:43.248 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:43 smithi092 bash[47618]: t=2023-09-23T19:04:43+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table data_source_v1 #2" 2023-09-23T19:04:43.248 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:43 smithi092 bash[47618]: t=2023-09-23T19:04:43+0000 lvl=info msg="Executing migration" logger=migrator id="Add column with_credentials" 2023-09-23T19:04:43.248 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:43 smithi092 bash[47618]: t=2023-09-23T19:04:43+0000 lvl=info msg="Executing migration" logger=migrator id="Add secure json data column" 2023-09-23T19:04:43.517 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:43 smithi092 bash[47618]: t=2023-09-23T19:04:43+0000 lvl=info msg="Executing migration" logger=migrator id="Update data_source table charset" 2023-09-23T19:04:43.517 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:43 smithi092 bash[47618]: t=2023-09-23T19:04:43+0000 lvl=info msg="Executing migration" logger=migrator id="Update initial version to 1" 2023-09-23T19:04:43.517 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:43 smithi092 bash[47618]: t=2023-09-23T19:04:43+0000 lvl=info msg="Executing migration" logger=migrator id="Add read_only data column" 2023-09-23T19:04:43.517 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:43 smithi092 bash[47618]: t=2023-09-23T19:04:43+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate logging ds to loki ds" 2023-09-23T19:04:43.517 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:43 smithi092 bash[47618]: t=2023-09-23T19:04:43+0000 lvl=info msg="Executing migration" logger=migrator id="Update json_data with nulls" 2023-09-23T19:04:43.517 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:43 smithi092 bash[47618]: t=2023-09-23T19:04:43+0000 lvl=info msg="Executing migration" logger=migrator id="Add uid column" 2023-09-23T19:04:43.517 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:43 smithi092 bash[47618]: t=2023-09-23T19:04:43+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid value" 2023-09-23T19:04:43.517 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:43 smithi092 bash[47618]: t=2023-09-23T19:04:43+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index datasource_org_id_uid" 2023-09-23T19:04:43.786 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:43 smithi092 bash[47618]: t=2023-09-23T19:04:43+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index datasource_org_id_is_default" 2023-09-23T19:04:43.787 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:43 smithi092 bash[47618]: t=2023-09-23T19:04:43+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table" 2023-09-23T19:04:43.787 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:43 smithi092 bash[47618]: t=2023-09-23T19:04:43+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id" 2023-09-23T19:04:43.787 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:43 smithi092 bash[47618]: t=2023-09-23T19:04:43+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.key" 2023-09-23T19:04:43.787 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:43 smithi092 bash[47618]: t=2023-09-23T19:04:43+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id_name" 2023-09-23T19:04:43.787 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:43 smithi092 bash[47618]: t=2023-09-23T19:04:43+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_api_key_account_id - v1" 2023-09-23T19:04:43.787 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:43 smithi092 bash[47618]: t=2023-09-23T19:04:43+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_key - v1" 2023-09-23T19:04:43.787 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:43 smithi092 bash[47618]: t=2023-09-23T19:04:43+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_account_id_name - v1" 2023-09-23T19:04:44.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:43 smithi092 bash[27556]: cluster 2023-09-23T19:04:42.886342+0000 mgr.y (mgr.14454) 51 : cluster [DBG] pgmap v41: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:44.038 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:43 smithi092 bash[47618]: t=2023-09-23T19:04:43+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table api_key to api_key_v1 - v1" 2023-09-23T19:04:44.038 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:43 smithi092 bash[47618]: t=2023-09-23T19:04:43+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table v2" 2023-09-23T19:04:44.038 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:43 smithi092 bash[47618]: t=2023-09-23T19:04:43+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_api_key_org_id - v2" 2023-09-23T19:04:44.038 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:43 smithi092 bash[47618]: t=2023-09-23T19:04:43+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_key - v2" 2023-09-23T19:04:44.038 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:43 smithi092 bash[47618]: t=2023-09-23T19:04:43+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_org_id_name - v2" 2023-09-23T19:04:44.038 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:44 smithi092 bash[47618]: t=2023-09-23T19:04:44+0000 lvl=info msg="Executing migration" logger=migrator id="copy api_key v1 to v2" 2023-09-23T19:04:44.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:43 smithi016 bash[33624]: cluster 2023-09-23T19:04:42.886342+0000 mgr.y (mgr.14454) 51 : cluster [DBG] pgmap v41: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:44.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:43 smithi016 bash[27237]: cluster 2023-09-23T19:04:42.886342+0000 mgr.y (mgr.14454) 51 : cluster [DBG] pgmap v41: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:44.324 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:44 smithi092 bash[47618]: t=2023-09-23T19:04:44+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table api_key_v1" 2023-09-23T19:04:44.324 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:44 smithi092 bash[47618]: t=2023-09-23T19:04:44+0000 lvl=info msg="Executing migration" logger=migrator id="Update api_key table charset" 2023-09-23T19:04:44.325 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:44 smithi092 bash[47618]: t=2023-09-23T19:04:44+0000 lvl=info msg="Executing migration" logger=migrator id="Add expires to api_key table" 2023-09-23T19:04:44.325 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:44 smithi092 bash[47618]: t=2023-09-23T19:04:44+0000 lvl=info msg="Executing migration" logger=migrator id="Add service account foreign key" 2023-09-23T19:04:44.325 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:44 smithi092 bash[47618]: t=2023-09-23T19:04:44+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v4" 2023-09-23T19:04:44.325 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:44 smithi092 bash[47618]: t=2023-09-23T19:04:44+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_snapshot_v4 #1" 2023-09-23T19:04:44.325 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:44 smithi092 bash[47618]: t=2023-09-23T19:04:44+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v5 #2" 2023-09-23T19:04:44.586 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:44 smithi092 bash[47618]: t=2023-09-23T19:04:44+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_key - v5" 2023-09-23T19:04:44.586 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:44 smithi092 bash[47618]: t=2023-09-23T19:04:44+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_delete_key - v5" 2023-09-23T19:04:44.586 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:44 smithi092 bash[47618]: t=2023-09-23T19:04:44+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_snapshot_user_id - v5" 2023-09-23T19:04:44.586 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:44 smithi092 bash[47618]: t=2023-09-23T19:04:44+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_snapshot to mediumtext v2" 2023-09-23T19:04:44.586 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:44 smithi092 bash[47618]: t=2023-09-23T19:04:44+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_snapshot table charset" 2023-09-23T19:04:44.586 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:44 smithi092 bash[47618]: t=2023-09-23T19:04:44+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external_delete_url to dashboard_snapshots table" 2023-09-23T19:04:44.872 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:44 smithi092 bash[47618]: t=2023-09-23T19:04:44+0000 lvl=info msg="Executing migration" logger=migrator id="Add encrypted dashboard json column" 2023-09-23T19:04:44.872 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:44 smithi092 bash[47618]: t=2023-09-23T19:04:44+0000 lvl=info msg="Executing migration" logger=migrator id="Change dashboard_encrypted column to MEDIUMBLOB" 2023-09-23T19:04:44.872 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:44 smithi092 bash[47618]: t=2023-09-23T19:04:44+0000 lvl=info msg="Executing migration" logger=migrator id="create quota table v1" 2023-09-23T19:04:44.872 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:44 smithi092 bash[47618]: t=2023-09-23T19:04:44+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_quota_org_id_user_id_target - v1" 2023-09-23T19:04:44.873 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:44 smithi092 bash[47618]: t=2023-09-23T19:04:44+0000 lvl=info msg="Executing migration" logger=migrator id="Update quota table charset" 2023-09-23T19:04:44.873 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:44 smithi092 bash[47618]: t=2023-09-23T19:04:44+0000 lvl=info msg="Executing migration" logger=migrator id="create plugin_setting table" 2023-09-23T19:04:44.873 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:44 smithi092 bash[47618]: t=2023-09-23T19:04:44+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2023-09-23T19:04:44.873 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:44 smithi092 bash[47618]: t=2023-09-23T19:04:44+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_version to plugin_settings" 2023-09-23T19:04:44.873 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:44 smithi092 bash[47618]: t=2023-09-23T19:04:44+0000 lvl=info msg="Executing migration" logger=migrator id="Update plugin_setting table charset" 2023-09-23T19:04:45.133 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:44 smithi092 bash[47618]: t=2023-09-23T19:04:44+0000 lvl=info msg="Executing migration" logger=migrator id="create session table" 2023-09-23T19:04:45.133 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:44 smithi092 bash[47618]: t=2023-09-23T19:04:44+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist table" 2023-09-23T19:04:45.133 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:44 smithi092 bash[47618]: t=2023-09-23T19:04:44+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist_item table" 2023-09-23T19:04:45.133 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:45 smithi092 bash[47618]: t=2023-09-23T19:04:45+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist table v2" 2023-09-23T19:04:45.133 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:45 smithi092 bash[47618]: t=2023-09-23T19:04:45+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist item table v2" 2023-09-23T19:04:45.134 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:45 smithi092 bash[47618]: t=2023-09-23T19:04:45+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist table charset" 2023-09-23T19:04:45.134 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:45 smithi092 bash[47618]: t=2023-09-23T19:04:45+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist_item table charset" 2023-09-23T19:04:45.134 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:45 smithi092 bash[47618]: t=2023-09-23T19:04:45+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v2" 2023-09-23T19:04:45.134 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:45 smithi092 bash[47618]: t=2023-09-23T19:04:45+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v3" 2023-09-23T19:04:45.386 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:45 smithi092 bash[27556]: audit 2023-09-23T19:04:44.227321+0000 mon.a (mon.0) 597 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:04:45.386 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:45 smithi092 bash[27556]: audit 2023-09-23T19:04:44.678687+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:04:45.386 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:45 smithi092 bash[27556]: cluster 2023-09-23T19:04:44.886952+0000 mgr.y (mgr.14454) 52 : cluster [DBG] pgmap v42: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:45.386 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:45 smithi092 bash[27556]: audit 2023-09-23T19:04:44.941739+0000 mon.c (mon.2) 47 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:04:45.386 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:45 smithi092 bash[27556]: audit 2023-09-23T19:04:44.942823+0000 mon.c (mon.2) 48 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:04:45.386 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:45 smithi092 bash[27556]: audit 2023-09-23T19:04:44.949263+0000 mon.a (mon.0) 599 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:04:45.386 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:45 smithi092 bash[27556]: cephadm 2023-09-23T19:04:44.953920+0000 mgr.y (mgr.14454) 53 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T19:04:45.386 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:45 smithi092 bash[27556]: cephadm 2023-09-23T19:04:44.957845+0000 mgr.y (mgr.14454) 54 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi016 2023-09-23T19:04:45.387 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:45 smithi092 bash[47618]: t=2023-09-23T19:04:45+0000 lvl=info msg="Executing migration" logger=migrator id="create preferences table v3" 2023-09-23T19:04:45.387 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:45 smithi092 bash[47618]: t=2023-09-23T19:04:45+0000 lvl=info msg="Executing migration" logger=migrator id="Update preferences table charset" 2023-09-23T19:04:45.387 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:45 smithi092 bash[47618]: t=2023-09-23T19:04:45+0000 lvl=info msg="Executing migration" logger=migrator id="Add column team_id in preferences" 2023-09-23T19:04:45.387 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:45 smithi092 bash[47618]: t=2023-09-23T19:04:45+0000 lvl=info msg="Executing migration" logger=migrator id="Update team_id column values in preferences" 2023-09-23T19:04:45.387 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:45 smithi092 bash[47618]: t=2023-09-23T19:04:45+0000 lvl=info msg="Executing migration" logger=migrator id="Add column week_start in preferences" 2023-09-23T19:04:45.387 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:45 smithi092 bash[47618]: t=2023-09-23T19:04:45+0000 lvl=info msg="Executing migration" logger=migrator id="create alert table v1" 2023-09-23T19:04:45.387 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:45 smithi092 bash[47618]: t=2023-09-23T19:04:45+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert org_id & id " 2023-09-23T19:04:45.387 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:45 smithi092 bash[47618]: t=2023-09-23T19:04:45+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert state" 2023-09-23T19:04:45.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:45 smithi016 bash[33624]: audit 2023-09-23T19:04:44.227321+0000 mon.a (mon.0) 597 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:04:45.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:45 smithi016 bash[33624]: audit 2023-09-23T19:04:44.678687+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:04:45.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:45 smithi016 bash[33624]: cluster 2023-09-23T19:04:44.886952+0000 mgr.y (mgr.14454) 52 : cluster [DBG] pgmap v42: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:45.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:45 smithi016 bash[33624]: audit 2023-09-23T19:04:44.941739+0000 mon.c (mon.2) 47 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:04:45.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:45 smithi016 bash[33624]: audit 2023-09-23T19:04:44.942823+0000 mon.c (mon.2) 48 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:04:45.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:45 smithi016 bash[33624]: audit 2023-09-23T19:04:44.949263+0000 mon.a (mon.0) 599 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:04:45.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:45 smithi016 bash[33624]: cephadm 2023-09-23T19:04:44.953920+0000 mgr.y (mgr.14454) 53 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T19:04:45.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:45 smithi016 bash[33624]: cephadm 2023-09-23T19:04:44.957845+0000 mgr.y (mgr.14454) 54 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi016 2023-09-23T19:04:45.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:45 smithi016 bash[27237]: audit 2023-09-23T19:04:44.227321+0000 mon.a (mon.0) 597 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:04:45.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:45 smithi016 bash[27237]: audit 2023-09-23T19:04:44.678687+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:04:45.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:45 smithi016 bash[27237]: cluster 2023-09-23T19:04:44.886952+0000 mgr.y (mgr.14454) 52 : cluster [DBG] pgmap v42: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:45.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:45 smithi016 bash[27237]: audit 2023-09-23T19:04:44.941739+0000 mon.c (mon.2) 47 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:04:45.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:45 smithi016 bash[27237]: audit 2023-09-23T19:04:44.942823+0000 mon.c (mon.2) 48 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:04:45.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:45 smithi016 bash[27237]: audit 2023-09-23T19:04:44.949263+0000 mon.a (mon.0) 599 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:04:45.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:45 smithi016 bash[27237]: cephadm 2023-09-23T19:04:44.953920+0000 mgr.y (mgr.14454) 53 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T19:04:45.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:45 smithi016 bash[27237]: cephadm 2023-09-23T19:04:44.957845+0000 mgr.y (mgr.14454) 54 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi016 2023-09-23T19:04:45.655 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:45 smithi092 bash[47618]: t=2023-09-23T19:04:45+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert dashboard_id" 2023-09-23T19:04:45.655 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:45 smithi092 bash[47618]: t=2023-09-23T19:04:45+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v1" 2023-09-23T19:04:45.655 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:45 smithi092 bash[47618]: t=2023-09-23T19:04:45+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_rule_tag.alert_id_tag_id" 2023-09-23T19:04:45.655 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:45 smithi092 bash[47618]: t=2023-09-23T19:04:45+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2023-09-23T19:04:45.655 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:45 smithi092 bash[47618]: t=2023-09-23T19:04:45+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2023-09-23T19:04:45.655 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:45 smithi092 bash[47618]: t=2023-09-23T19:04:45+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v2" 2023-09-23T19:04:45.655 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:45 smithi092 bash[47618]: t=2023-09-23T19:04:45+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2023-09-23T19:04:45.656 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:45 smithi092 bash[47618]: t=2023-09-23T19:04:45+0000 lvl=info msg="Executing migration" logger=migrator id="copy alert_rule_tag v1 to v2" 2023-09-23T19:04:45.656 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:45 smithi092 bash[47618]: t=2023-09-23T19:04:45+0000 lvl=info msg="Executing migration" logger=migrator id="drop table alert_rule_tag_v1" 2023-09-23T19:04:45.941 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:45 smithi092 bash[47618]: t=2023-09-23T19:04:45+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification table v1" 2023-09-23T19:04:45.941 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:45 smithi092 bash[47618]: t=2023-09-23T19:04:45+0000 lvl=info msg="Executing migration" logger=migrator id="Add column is_default" 2023-09-23T19:04:45.941 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:45 smithi092 bash[47618]: t=2023-09-23T19:04:45+0000 lvl=info msg="Executing migration" logger=migrator id="Add column frequency" 2023-09-23T19:04:45.941 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:45 smithi092 bash[47618]: t=2023-09-23T19:04:45+0000 lvl=info msg="Executing migration" logger=migrator id="Add column send_reminder" 2023-09-23T19:04:45.941 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:45 smithi092 bash[47618]: t=2023-09-23T19:04:45+0000 lvl=info msg="Executing migration" logger=migrator id="Add column disable_resolve_message" 2023-09-23T19:04:45.942 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:45 smithi092 bash[47618]: t=2023-09-23T19:04:45+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification org_id & name" 2023-09-23T19:04:45.942 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:45 smithi092 bash[47618]: t=2023-09-23T19:04:45+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert table charset" 2023-09-23T19:04:46.193 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:45 smithi092 bash[47618]: t=2023-09-23T19:04:45+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert_notification table charset" 2023-09-23T19:04:46.193 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:45 smithi092 bash[47618]: t=2023-09-23T19:04:45+0000 lvl=info msg="Executing migration" logger=migrator id="create notification_journal table v1" 2023-09-23T19:04:46.194 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:45 smithi092 bash[47618]: t=2023-09-23T19:04:45+0000 lvl=info msg="Executing migration" logger=migrator id="add index notification_journal org_id & alert_id & notifier_id" 2023-09-23T19:04:46.194 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:46 smithi092 bash[47618]: t=2023-09-23T19:04:46+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_notification_journal" 2023-09-23T19:04:46.194 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:46 smithi092 bash[47618]: t=2023-09-23T19:04:46+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification_state table v1" 2023-09-23T19:04:46.194 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:46 smithi092 bash[47618]: t=2023-09-23T19:04:46+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification_state org_id & alert_id & notifier_id" 2023-09-23T19:04:46.194 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:46 smithi092 bash[47618]: t=2023-09-23T19:04:46+0000 lvl=info msg="Executing migration" logger=migrator id="Add for to alert table" 2023-09-23T19:04:46.194 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:46 smithi092 bash[47618]: t=2023-09-23T19:04:46+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in alert_notification" 2023-09-23T19:04:46.194 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:46 smithi092 bash[47618]: t=2023-09-23T19:04:46+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in alert_notification" 2023-09-23T19:04:46.194 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:46 smithi092 bash[47618]: t=2023-09-23T19:04:46+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_notification_org_id_uid" 2023-09-23T19:04:46.480 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:46 smithi092 bash[47618]: t=2023-09-23T19:04:46+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_name" 2023-09-23T19:04:46.480 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:46 smithi092 bash[47618]: t=2023-09-23T19:04:46+0000 lvl=info msg="Executing migration" logger=migrator id="Add column secure_settings in alert_notification" 2023-09-23T19:04:46.480 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:46 smithi092 bash[47618]: t=2023-09-23T19:04:46+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert.settings to mediumtext" 2023-09-23T19:04:46.480 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:46 smithi092 bash[47618]: t=2023-09-23T19:04:46+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_notification_state_alert_id" 2023-09-23T19:04:46.480 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:46 smithi092 bash[47618]: t=2023-09-23T19:04:46+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_rule_tag_alert_id" 2023-09-23T19:04:46.480 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:46 smithi092 bash[47618]: t=2023-09-23T19:04:46+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old annotation table v4" 2023-09-23T19:04:46.480 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:46 smithi092 bash[47618]: t=2023-09-23T19:04:46+0000 lvl=info msg="Executing migration" logger=migrator id="create annotation table v5" 2023-09-23T19:04:46.480 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:46 smithi092 bash[47618]: t=2023-09-23T19:04:46+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 0 v3" 2023-09-23T19:04:46.480 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:46 smithi092 bash[47618]: t=2023-09-23T19:04:46+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 1 v3" 2023-09-23T19:04:46.741 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:46 smithi092 bash[47618]: t=2023-09-23T19:04:46+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 2 v3" 2023-09-23T19:04:46.741 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:46 smithi092 bash[47618]: t=2023-09-23T19:04:46+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 3 v3" 2023-09-23T19:04:46.741 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:46 smithi092 bash[47618]: t=2023-09-23T19:04:46+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 4 v3" 2023-09-23T19:04:46.741 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:46 smithi092 bash[47618]: t=2023-09-23T19:04:46+0000 lvl=info msg="Executing migration" logger=migrator id="Update annotation table charset" 2023-09-23T19:04:46.741 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:46 smithi092 bash[47618]: t=2023-09-23T19:04:46+0000 lvl=info msg="Executing migration" logger=migrator id="Add column region_id to annotation table" 2023-09-23T19:04:46.742 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:46 smithi092 bash[47618]: t=2023-09-23T19:04:46+0000 lvl=info msg="Executing migration" logger=migrator id="Drop category_id index" 2023-09-23T19:04:46.742 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:46 smithi092 bash[47618]: t=2023-09-23T19:04:46+0000 lvl=info msg="Executing migration" logger=migrator id="Add column tags to annotation table" 2023-09-23T19:04:46.742 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:46 smithi092 bash[47618]: t=2023-09-23T19:04:46+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v2" 2023-09-23T19:04:46.742 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:46 smithi092 bash[47618]: t=2023-09-23T19:04:46+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index annotation_tag.annotation_id_tag_id" 2023-09-23T19:04:46.742 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:46 smithi092 bash[47618]: t=2023-09-23T19:04:46+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2023-09-23T19:04:47.002 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:46 smithi092 bash[47618]: t=2023-09-23T19:04:46+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table annotation_tag to annotation_tag_v2 - v2" 2023-09-23T19:04:47.002 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:46 smithi092 bash[47618]: t=2023-09-23T19:04:46+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v3" 2023-09-23T19:04:47.002 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:46 smithi092 bash[47618]: t=2023-09-23T19:04:46+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2023-09-23T19:04:47.002 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:46 smithi092 bash[47618]: t=2023-09-23T19:04:46+0000 lvl=info msg="Executing migration" logger=migrator id="copy annotation_tag v2 to v3" 2023-09-23T19:04:47.002 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:46 smithi092 bash[47618]: t=2023-09-23T19:04:46+0000 lvl=info msg="Executing migration" logger=migrator id="drop table annotation_tag_v2" 2023-09-23T19:04:47.002 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:46 smithi092 bash[47618]: t=2023-09-23T19:04:46+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert annotations and set TEXT to empty" 2023-09-23T19:04:47.002 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:46 smithi092 bash[47618]: t=2023-09-23T19:04:46+0000 lvl=info msg="Executing migration" logger=migrator id="Add created time to annotation table" 2023-09-23T19:04:47.002 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:47 smithi092 bash[47618]: t=2023-09-23T19:04:47+0000 lvl=info msg="Executing migration" logger=migrator id="Add updated time to annotation table" 2023-09-23T19:04:47.271 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:47 smithi092 bash[47618]: t=2023-09-23T19:04:47+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for created in annotation table" 2023-09-23T19:04:47.271 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:47 smithi092 bash[47618]: t=2023-09-23T19:04:47+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for updated in annotation table" 2023-09-23T19:04:47.271 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:47 smithi092 bash[47618]: t=2023-09-23T19:04:47+0000 lvl=info msg="Executing migration" logger=migrator id="Convert existing annotations from seconds to milliseconds" 2023-09-23T19:04:47.271 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:47 smithi092 bash[47618]: t=2023-09-23T19:04:47+0000 lvl=info msg="Executing migration" logger=migrator id="Add epoch_end column" 2023-09-23T19:04:47.271 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:47 smithi092 bash[47618]: t=2023-09-23T19:04:47+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for epoch_end" 2023-09-23T19:04:47.271 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:47 smithi092 bash[47618]: t=2023-09-23T19:04:47+0000 lvl=info msg="Executing migration" logger=migrator id="Make epoch_end the same as epoch" 2023-09-23T19:04:47.272 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:47 smithi092 bash[47618]: t=2023-09-23T19:04:47+0000 lvl=info msg="Executing migration" logger=migrator id="Move region to single row" 2023-09-23T19:04:47.272 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:47 smithi092 bash[47618]: t=2023-09-23T19:04:47+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch from annotation table" 2023-09-23T19:04:47.537 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:47 smithi092 bash[47618]: t=2023-09-23T19:04:47+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2023-09-23T19:04:47.538 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:47 smithi092 bash[47618]: t=2023-09-23T19:04:47+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2023-09-23T19:04:47.538 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:47 smithi092 bash[47618]: t=2023-09-23T19:04:47+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_epoch_end_epoch on annotation table" 2023-09-23T19:04:47.538 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:47 smithi092 bash[47618]: t=2023-09-23T19:04:47+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch_epoch_end from annotation table" 2023-09-23T19:04:47.538 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:47 smithi092 bash[47618]: t=2023-09-23T19:04:47+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for alert_id on annotation table" 2023-09-23T19:04:47.538 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:47 smithi092 bash[47618]: t=2023-09-23T19:04:47+0000 lvl=info msg="Executing migration" logger=migrator id="create test_data table" 2023-09-23T19:04:47.538 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:47 smithi092 bash[47618]: t=2023-09-23T19:04:47+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_version table v1" 2023-09-23T19:04:47.818 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:47 smithi092 bash[47618]: t=2023-09-23T19:04:47+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_version.dashboard_id" 2023-09-23T19:04:47.818 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:47 smithi092 bash[47618]: t=2023-09-23T19:04:47+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2023-09-23T19:04:47.818 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:47 smithi092 bash[47618]: t=2023-09-23T19:04:47+0000 lvl=info msg="Executing migration" logger=migrator id="Set dashboard version to 1 where 0" 2023-09-23T19:04:47.818 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:47 smithi092 bash[47618]: t=2023-09-23T19:04:47+0000 lvl=info msg="Executing migration" logger=migrator id="save existing dashboard data in dashboard_version table v1" 2023-09-23T19:04:47.818 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:47 smithi092 bash[47618]: t=2023-09-23T19:04:47+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_version.data to mediumtext v1" 2023-09-23T19:04:47.818 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:47 smithi092 bash[47618]: t=2023-09-23T19:04:47+0000 lvl=info msg="Executing migration" logger=migrator id="create team table" 2023-09-23T19:04:47.818 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:47 smithi092 bash[47618]: t=2023-09-23T19:04:47+0000 lvl=info msg="Executing migration" logger=migrator id="add index team.org_id" 2023-09-23T19:04:47.819 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:47 smithi092 bash[47618]: t=2023-09-23T19:04:47+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_org_id_name" 2023-09-23T19:04:47.819 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:47 smithi092 bash[47618]: t=2023-09-23T19:04:47+0000 lvl=info msg="Executing migration" logger=migrator id="create team member table" 2023-09-23T19:04:47.819 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:47 smithi092 bash[47618]: t=2023-09-23T19:04:47+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.org_id" 2023-09-23T19:04:48.047 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:47 smithi016 bash[27237]: cluster 2023-09-23T19:04:46.887616+0000 mgr.y (mgr.14454) 55 : cluster [DBG] pgmap v43: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:48.048 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:47 smithi016 bash[33624]: cluster 2023-09-23T19:04:46.887616+0000 mgr.y (mgr.14454) 55 : cluster [DBG] pgmap v43: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:48.078 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:47 smithi092 bash[27556]: cluster 2023-09-23T19:04:46.887616+0000 mgr.y (mgr.14454) 55 : cluster [DBG] pgmap v43: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:48.079 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:47 smithi092 bash[47618]: t=2023-09-23T19:04:47+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_member_org_id_team_id_user_id" 2023-09-23T19:04:48.079 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:47 smithi092 bash[47618]: t=2023-09-23T19:04:47+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.team_id" 2023-09-23T19:04:48.079 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:47 smithi092 bash[47618]: t=2023-09-23T19:04:47+0000 lvl=info msg="Executing migration" logger=migrator id="Add column email to team table" 2023-09-23T19:04:48.079 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:47 smithi092 bash[47618]: t=2023-09-23T19:04:47+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external to team_member table" 2023-09-23T19:04:48.079 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:47 smithi092 bash[47618]: t=2023-09-23T19:04:47+0000 lvl=info msg="Executing migration" logger=migrator id="Add column permission to team_member table" 2023-09-23T19:04:48.079 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:48 smithi092 bash[47618]: t=2023-09-23T19:04:48+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard acl table" 2023-09-23T19:04:48.079 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:48 smithi092 bash[47618]: t=2023-09-23T19:04:48+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_dashboard_id" 2023-09-23T19:04:48.080 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:48 smithi092 bash[47618]: t=2023-09-23T19:04:48+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_user_id" 2023-09-23T19:04:48.080 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:48 smithi092 bash[47618]: t=2023-09-23T19:04:48+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_team_id" 2023-09-23T19:04:48.364 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:48 smithi092 bash[47618]: t=2023-09-23T19:04:48+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_user_id" 2023-09-23T19:04:48.365 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:48 smithi092 bash[47618]: t=2023-09-23T19:04:48+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_team_id" 2023-09-23T19:04:48.365 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:48 smithi092 bash[47618]: t=2023-09-23T19:04:48+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_org_id_role" 2023-09-23T19:04:48.365 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:48 smithi092 bash[47618]: t=2023-09-23T19:04:48+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_permission" 2023-09-23T19:04:48.365 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:48 smithi092 bash[47618]: t=2023-09-23T19:04:48+0000 lvl=info msg="Executing migration" logger=migrator id="save default acl rules in dashboard_acl table" 2023-09-23T19:04:48.365 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:48 smithi092 bash[47618]: t=2023-09-23T19:04:48+0000 lvl=info msg="Executing migration" logger=migrator id="delete acl rules for deleted dashboards and folders" 2023-09-23T19:04:48.365 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:48 smithi092 bash[47618]: t=2023-09-23T19:04:48+0000 lvl=info msg="Executing migration" logger=migrator id="create tag table" 2023-09-23T19:04:48.634 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:48 smithi092 bash[47618]: t=2023-09-23T19:04:48+0000 lvl=info msg="Executing migration" logger=migrator id="add index tag.key_value" 2023-09-23T19:04:48.634 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:48 smithi092 bash[47618]: t=2023-09-23T19:04:48+0000 lvl=info msg="Executing migration" logger=migrator id="create login attempt table" 2023-09-23T19:04:48.634 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:48 smithi092 bash[47618]: t=2023-09-23T19:04:48+0000 lvl=info msg="Executing migration" logger=migrator id="add index login_attempt.username" 2023-09-23T19:04:48.634 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:48 smithi092 bash[47618]: t=2023-09-23T19:04:48+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_login_attempt_username - v1" 2023-09-23T19:04:48.634 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:48 smithi092 bash[47618]: t=2023-09-23T19:04:48+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2023-09-23T19:04:48.634 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:48 smithi092 bash[47618]: t=2023-09-23T19:04:48+0000 lvl=info msg="Executing migration" logger=migrator id="create login_attempt v2" 2023-09-23T19:04:48.634 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:48 smithi092 bash[47618]: t=2023-09-23T19:04:48+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_login_attempt_username - v2" 2023-09-23T19:04:48.634 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:48 smithi092 bash[47618]: t=2023-09-23T19:04:48+0000 lvl=info msg="Executing migration" logger=migrator id="copy login_attempt v1 to v2" 2023-09-23T19:04:48.886 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:48 smithi092 bash[47618]: t=2023-09-23T19:04:48+0000 lvl=info msg="Executing migration" logger=migrator id="drop login_attempt_tmp_qwerty" 2023-09-23T19:04:48.886 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:48 smithi092 bash[47618]: t=2023-09-23T19:04:48+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth table" 2023-09-23T19:04:48.886 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:48 smithi092 bash[47618]: t=2023-09-23T19:04:48+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_user_auth_auth_module_auth_id - v1" 2023-09-23T19:04:48.886 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:48 smithi092 bash[47618]: t=2023-09-23T19:04:48+0000 lvl=info msg="Executing migration" logger=migrator id="alter user_auth.auth_id to length 190" 2023-09-23T19:04:48.886 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:48 smithi092 bash[47618]: t=2023-09-23T19:04:48+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth access token to user_auth" 2023-09-23T19:04:48.886 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:48 smithi092 bash[47618]: t=2023-09-23T19:04:48+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth refresh token to user_auth" 2023-09-23T19:04:48.886 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:48 smithi092 bash[47618]: t=2023-09-23T19:04:48+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth token type to user_auth" 2023-09-23T19:04:49.172 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:04:49 smithi092 bash[28593]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:04:49] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T19:04:49.173 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:48 smithi092 bash[47618]: t=2023-09-23T19:04:48+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth expiry to user_auth" 2023-09-23T19:04:49.173 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:48 smithi092 bash[47618]: t=2023-09-23T19:04:48+0000 lvl=info msg="Executing migration" logger=migrator id="Add index to user_id column in user_auth" 2023-09-23T19:04:49.173 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:48 smithi092 bash[47618]: t=2023-09-23T19:04:48+0000 lvl=info msg="Executing migration" logger=migrator id="create server_lock table" 2023-09-23T19:04:49.173 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:48 smithi092 bash[47618]: t=2023-09-23T19:04:48+0000 lvl=info msg="Executing migration" logger=migrator id="add index server_lock.operation_uid" 2023-09-23T19:04:49.173 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:49 smithi092 bash[47618]: t=2023-09-23T19:04:49+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth token table" 2023-09-23T19:04:49.173 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:49 smithi092 bash[47618]: t=2023-09-23T19:04:49+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.auth_token" 2023-09-23T19:04:49.173 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:49 smithi092 bash[47618]: t=2023-09-23T19:04:49+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.prev_auth_token" 2023-09-23T19:04:49.173 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:49 smithi092 bash[47618]: t=2023-09-23T19:04:49+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_auth_token.user_id" 2023-09-23T19:04:49.173 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:49 smithi092 bash[47618]: t=2023-09-23T19:04:49+0000 lvl=info msg="Executing migration" logger=migrator id="Add revoked_at to the user auth token" 2023-09-23T19:04:49.425 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:49 smithi092 bash[47618]: t=2023-09-23T19:04:49+0000 lvl=info msg="Executing migration" logger=migrator id="create cache_data table" 2023-09-23T19:04:49.425 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:49 smithi092 bash[47618]: t=2023-09-23T19:04:49+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index cache_data.cache_key" 2023-09-23T19:04:49.425 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:49 smithi092 bash[47618]: t=2023-09-23T19:04:49+0000 lvl=info msg="Executing migration" logger=migrator id="create short_url table v1" 2023-09-23T19:04:49.425 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:49 smithi092 bash[47618]: t=2023-09-23T19:04:49+0000 lvl=info msg="Executing migration" logger=migrator id="add index short_url.org_id-uid" 2023-09-23T19:04:49.425 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:49 smithi092 bash[47618]: t=2023-09-23T19:04:49+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition table" 2023-09-23T19:04:49.426 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:49 smithi092 bash[47618]: t=2023-09-23T19:04:49+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition table" 2023-09-23T19:04:49.426 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:49 smithi092 bash[47618]: t=2023-09-23T19:04:49+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and title columns" 2023-09-23T19:04:49.426 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:49 smithi092 bash[47618]: t=2023-09-23T19:04:49+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and uid columns" 2023-09-23T19:04:49.694 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:49 smithi092 bash[47618]: t=2023-09-23T19:04:49+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition table data column to mediumtext in mysql" 2023-09-23T19:04:49.694 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:49 smithi092 bash[47618]: t=2023-09-23T19:04:49+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and title columns" 2023-09-23T19:04:49.694 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:49 smithi092 bash[47618]: t=2023-09-23T19:04:49+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and uid columns" 2023-09-23T19:04:49.694 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:49 smithi092 bash[47618]: t=2023-09-23T19:04:49+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and title columns" 2023-09-23T19:04:49.695 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:49 smithi092 bash[47618]: t=2023-09-23T19:04:49+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and uid columns" 2023-09-23T19:04:49.695 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:49 smithi092 bash[47618]: t=2023-09-23T19:04:49+0000 lvl=info msg="Executing migration" logger=migrator id="Add column paused in alert_definition" 2023-09-23T19:04:49.695 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:49 smithi092 bash[47618]: t=2023-09-23T19:04:49+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition table" 2023-09-23T19:04:49.695 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:49 smithi092 bash[47618]: t=2023-09-23T19:04:49+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition_version table" 2023-09-23T19:04:49.695 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:49 smithi092 bash[47618]: t=2023-09-23T19:04:49+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition_version table" 2023-09-23T19:04:49.951 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:49 smithi092 bash[47618]: t=2023-09-23T19:04:49+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_id and version columns" 2023-09-23T19:04:49.951 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:49 smithi092 bash[47618]: t=2023-09-23T19:04:49+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_uid and version columns" 2023-09-23T19:04:49.951 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:49 smithi092 bash[47618]: t=2023-09-23T19:04:49+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition_version table data column to mediumtext in mysql" 2023-09-23T19:04:49.951 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:49 smithi092 bash[47618]: t=2023-09-23T19:04:49+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition_version table" 2023-09-23T19:04:49.951 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:49 smithi092 bash[47618]: t=2023-09-23T19:04:49+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_instance table" 2023-09-23T19:04:49.951 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:49 smithi092 bash[47618]: t=2023-09-23T19:04:49+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2023-09-23T19:04:49.951 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:49 smithi092 bash[47618]: t=2023-09-23T19:04:49+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, current_state columns" 2023-09-23T19:04:50.207 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:49 smithi092 bash[27556]: cluster 2023-09-23T19:04:48.888218+0000 mgr.y (mgr.14454) 56 : cluster [DBG] pgmap v44: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:50.207 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:49 smithi092 bash[47618]: t=2023-09-23T19:04:49+0000 lvl=info msg="Executing migration" logger=migrator id="add column current_state_end to alert_instance" 2023-09-23T19:04:50.207 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:50 smithi092 bash[47618]: t=2023-09-23T19:04:49+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, def_uid, current_state on alert_instance" 2023-09-23T19:04:50.207 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:50 smithi092 bash[47618]: t=2023-09-23T19:04:50+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, current_state on alert_instance" 2023-09-23T19:04:50.208 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:50 smithi092 bash[47618]: t=2023-09-23T19:04:50+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_org_id to rule_org_id in alert_instance" 2023-09-23T19:04:50.208 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:50 smithi092 bash[47618]: t=2023-09-23T19:04:50+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_uid to rule_uid in alert_instance" 2023-09-23T19:04:50.208 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:50 smithi092 bash[47618]: t=2023-09-23T19:04:50+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, rule_uid, current_state on alert_instance" 2023-09-23T19:04:50.208 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:50 smithi092 bash[47618]: t=2023-09-23T19:04:50+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, current_state on alert_instance" 2023-09-23T19:04:50.208 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:50 smithi092 bash[47618]: t=2023-09-23T19:04:50+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule table" 2023-09-23T19:04:50.267 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:49 smithi016 bash[33624]: cluster 2023-09-23T19:04:48.888218+0000 mgr.y (mgr.14454) 56 : cluster [DBG] pgmap v44: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:50.267 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:49 smithi016 bash[27237]: cluster 2023-09-23T19:04:48.888218+0000 mgr.y (mgr.14454) 56 : cluster [DBG] pgmap v44: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:50.267 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:04:50 smithi016 bash[27503]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:04:50] "GET /metrics HTTP/1.1" 200 155652 "" "Prometheus/2.33.4" 2023-09-23T19:04:50.485 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:50 smithi092 bash[47618]: t=2023-09-23T19:04:50+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and title columns" 2023-09-23T19:04:50.485 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:50 smithi092 bash[47618]: t=2023-09-23T19:04:50+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and uid columns" 2023-09-23T19:04:50.486 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:50 smithi092 bash[47618]: t=2023-09-23T19:04:50+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2023-09-23T19:04:50.486 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:50 smithi092 bash[47618]: t=2023-09-23T19:04:50+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule table data column to mediumtext in mysql" 2023-09-23T19:04:50.486 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:50 smithi092 bash[47618]: t=2023-09-23T19:04:50+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule" 2023-09-23T19:04:50.486 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:50 smithi092 bash[47618]: t=2023-09-23T19:04:50+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule" 2023-09-23T19:04:50.486 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:50 smithi092 bash[47618]: t=2023-09-23T19:04:50+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule" 2023-09-23T19:04:50.486 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:50 smithi092 bash[47618]: t=2023-09-23T19:04:50+0000 lvl=info msg="Executing migration" logger=migrator id="remove unique index from alert_rule on org_id, title columns" 2023-09-23T19:04:50.486 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:50 smithi092 bash[47618]: t=2023-09-23T19:04:50+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespase_uid and title columns" 2023-09-23T19:04:50.738 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:50 smithi092 bash[47618]: t=2023-09-23T19:04:50+0000 lvl=info msg="Executing migration" logger=migrator id="add dashboard_uid column to alert_rule" 2023-09-23T19:04:50.738 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:50 smithi092 bash[47618]: t=2023-09-23T19:04:50+0000 lvl=info msg="Executing migration" logger=migrator id="add panel_id column to alert_rule" 2023-09-23T19:04:50.738 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:50 smithi092 bash[47618]: t=2023-09-23T19:04:50+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2023-09-23T19:04:50.738 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:50 smithi092 bash[47618]: t=2023-09-23T19:04:50+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule_version table" 2023-09-23T19:04:50.738 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:50 smithi092 bash[47618]: t=2023-09-23T19:04:50+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2023-09-23T19:04:50.738 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:50 smithi092 bash[47618]: t=2023-09-23T19:04:50+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2023-09-23T19:04:50.738 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:50 smithi092 bash[47618]: t=2023-09-23T19:04:50+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule_version table data column to mediumtext in mysql" 2023-09-23T19:04:50.999 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:50 smithi092 bash[47618]: t=2023-09-23T19:04:50+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule_version" 2023-09-23T19:04:50.999 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:50 smithi092 bash[47618]: t=2023-09-23T19:04:50+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule_version" 2023-09-23T19:04:50.999 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:50 smithi092 bash[47618]: t=2023-09-23T19:04:50+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule_version" 2023-09-23T19:04:50.999 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:50 smithi092 bash[47618]: t=2023-09-23T19:04:50+0000 lvl=info msg="Executing migration" logger=migrator id=create_alert_configuration_table 2023-09-23T19:04:50.999 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:50 smithi092 bash[47618]: t=2023-09-23T19:04:50+0000 lvl=info msg="Executing migration" logger=migrator id="Add column default in alert_configuration" 2023-09-23T19:04:50.999 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:50 smithi092 bash[47618]: t=2023-09-23T19:04:50+0000 lvl=info msg="Executing migration" logger=migrator id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2023-09-23T19:04:50.999 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:50 smithi092 bash[47618]: t=2023-09-23T19:04:50+0000 lvl=info msg="Executing migration" logger=migrator id="add column org_id in alert_configuration" 2023-09-23T19:04:50.999 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:50 smithi092 bash[47618]: t=2023-09-23T19:04:50+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_configuration table on org_id column" 2023-09-23T19:04:50.999 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:51 smithi092 bash[47618]: t=2023-09-23T19:04:51+0000 lvl=info msg="Executing migration" logger=migrator id=create_ngalert_configuration_table 2023-09-23T19:04:51.277 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:51 smithi092 bash[47618]: t=2023-09-23T19:04:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index in ngalert_configuration on org_id column" 2023-09-23T19:04:51.277 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:51 smithi092 bash[47618]: t=2023-09-23T19:04:51+0000 lvl=info msg="Executing migration" logger=migrator id="clear migration entry \"remove unified alerting data\"" 2023-09-23T19:04:51.277 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:51 smithi092 bash[47618]: t=2023-09-23T19:04:51+0000 lvl=info msg="Executing migration" logger=migrator id="move dashboard alerts to unified alerting" 2023-09-23T19:04:51.277 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:51 smithi092 bash[47618]: t=2023-09-23T19:04:51+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element table v1" 2023-09-23T19:04:51.277 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:51 smithi092 bash[47618]: t=2023-09-23T19:04:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element org_id-folder_id-name-kind" 2023-09-23T19:04:51.277 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:51 smithi092 bash[47618]: t=2023-09-23T19:04:51+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element_connection table v1" 2023-09-23T19:04:51.277 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:51 smithi092 bash[47618]: t=2023-09-23T19:04:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element_connection element_id-kind-connection_id" 2023-09-23T19:04:51.277 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:51 smithi092 bash[47618]: t=2023-09-23T19:04:51+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index library_element org_id_uid" 2023-09-23T19:04:51.277 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:51 smithi092 bash[47618]: t=2023-09-23T19:04:51+0000 lvl=info msg="Executing migration" logger=migrator id="clone move dashboard alerts to unified alerting" 2023-09-23T19:04:51.278 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:51 smithi092 bash[47618]: t=2023-09-23T19:04:51+0000 lvl=info msg="Executing migration" logger=migrator id="create data_keys table" 2023-09-23T19:04:51.278 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:51 smithi092 bash[47618]: t=2023-09-23T19:04:51+0000 lvl=info msg="Executing migration" logger=migrator id="create kv_store table v1" 2023-09-23T19:04:51.537 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:51 smithi092 bash[47618]: t=2023-09-23T19:04:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index kv_store.org_id-namespace-key" 2023-09-23T19:04:51.537 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:51 smithi092 bash[47618]: t=2023-09-23T19:04:51+0000 lvl=info msg="Executing migration" logger=migrator id="update dashboard_uid and panel_id from existing annotations" 2023-09-23T19:04:51.538 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:51 smithi092 bash[47618]: t=2023-09-23T19:04:51+0000 lvl=info msg="Executing migration" logger=migrator id="create permission table" 2023-09-23T19:04:51.538 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:51 smithi092 bash[47618]: t=2023-09-23T19:04:51+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index permission.role_id" 2023-09-23T19:04:51.538 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:51 smithi092 bash[47618]: t=2023-09-23T19:04:51+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_id_action_scope" 2023-09-23T19:04:51.538 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:51 smithi092 bash[47618]: t=2023-09-23T19:04:51+0000 lvl=info msg="Executing migration" logger=migrator id="create role table" 2023-09-23T19:04:51.538 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:51 smithi092 bash[47618]: t=2023-09-23T19:04:51+0000 lvl=info msg="Executing migration" logger=migrator id="add column display_name" 2023-09-23T19:04:51.832 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:51 smithi092 bash[47618]: t=2023-09-23T19:04:51+0000 lvl=info msg="Executing migration" logger=migrator id="add column group_name" 2023-09-23T19:04:51.832 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:51 smithi092 bash[47618]: t=2023-09-23T19:04:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index role.org_id" 2023-09-23T19:04:51.832 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:51 smithi092 bash[47618]: t=2023-09-23T19:04:51+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_org_id_name" 2023-09-23T19:04:51.832 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:51 smithi092 bash[47618]: t=2023-09-23T19:04:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index role_org_id_uid" 2023-09-23T19:04:51.833 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:51 smithi092 bash[47618]: t=2023-09-23T19:04:51+0000 lvl=info msg="Executing migration" logger=migrator id="create team role table" 2023-09-23T19:04:51.833 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:51 smithi092 bash[47618]: t=2023-09-23T19:04:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.org_id" 2023-09-23T19:04:51.833 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:51 smithi092 bash[47618]: t=2023-09-23T19:04:51+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_role_org_id_team_id_role_id" 2023-09-23T19:04:51.833 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:51 smithi092 bash[47618]: t=2023-09-23T19:04:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.team_id" 2023-09-23T19:04:51.833 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:51 smithi092 bash[47618]: t=2023-09-23T19:04:51+0000 lvl=info msg="Executing migration" logger=migrator id="create user role table" 2023-09-23T19:04:52.118 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:51 smithi092 bash[27556]: cluster 2023-09-23T19:04:50.888783+0000 mgr.y (mgr.14454) 57 : cluster [DBG] pgmap v45: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:52.119 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:51 smithi092 bash[47618]: t=2023-09-23T19:04:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.org_id" 2023-09-23T19:04:52.119 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:51 smithi092 bash[47618]: t=2023-09-23T19:04:51+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_role_org_id_user_id_role_id" 2023-09-23T19:04:52.119 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:51 smithi092 bash[47618]: t=2023-09-23T19:04:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.user_id" 2023-09-23T19:04:52.119 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:51 smithi092 bash[47618]: t=2023-09-23T19:04:51+0000 lvl=info msg="Executing migration" logger=migrator id="create builtin role table" 2023-09-23T19:04:52.119 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:51 smithi092 bash[47618]: t=2023-09-23T19:04:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.role_id" 2023-09-23T19:04:52.119 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:51 smithi092 bash[47618]: t=2023-09-23T19:04:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.name" 2023-09-23T19:04:52.119 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:52 smithi092 bash[47618]: t=2023-09-23T19:04:52+0000 lvl=info msg="Executing migration" logger=migrator id="Add column org_id to builtin_role table" 2023-09-23T19:04:52.119 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:52 smithi092 bash[47618]: t=2023-09-23T19:04:52+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.org_id" 2023-09-23T19:04:52.119 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:52 smithi092 bash[47618]: t=2023-09-23T19:04:52+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_org_id_role_id_role" 2023-09-23T19:04:52.119 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:52 smithi092 bash[47618]: t=2023-09-23T19:04:52+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index role_org_id_uid" 2023-09-23T19:04:52.119 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:52 smithi092 bash[47618]: t=2023-09-23T19:04:52+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role.uid" 2023-09-23T19:04:52.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:51 smithi016 bash[27237]: cluster 2023-09-23T19:04:50.888783+0000 mgr.y (mgr.14454) 57 : cluster [DBG] pgmap v45: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:52.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:51 smithi016 bash[33624]: cluster 2023-09-23T19:04:50.888783+0000 mgr.y (mgr.14454) 57 : cluster [DBG] pgmap v45: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:52.457 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:52 smithi092 bash[47618]: t=2023-09-23T19:04:52+0000 lvl=info msg="Executing migration" logger=migrator id="create seed assignment table" 2023-09-23T19:04:52.457 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:52 smithi092 bash[47618]: t=2023-09-23T19:04:52+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_role_name" 2023-09-23T19:04:52.458 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:52 smithi092 bash[47618]: t=2023-09-23T19:04:52+0000 lvl=info msg="migrations completed" logger=migrator performed=381 skipped=0 duration=12.308599175s 2023-09-23T19:04:52.458 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:52 smithi092 bash[47618]: t=2023-09-23T19:04:52+0000 lvl=info msg="Created default organization" logger=sqlstore 2023-09-23T19:04:52.458 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:52 smithi092 bash[47618]: t=2023-09-23T19:04:52+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2023-09-23T19:04:52.458 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:52 smithi092 bash[47618]: t=2023-09-23T19:04:52+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2023-09-23T19:04:52.458 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:52 smithi092 bash[47618]: t=2023-09-23T19:04:52+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2023-09-23T19:04:52.458 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:52 smithi092 bash[47618]: t=2023-09-23T19:04:52+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2023-09-23T19:04:52.458 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:52 smithi092 bash[47618]: t=2023-09-23T19:04:52+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2023-09-23T19:04:52.458 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:52 smithi092 bash[47618]: t=2023-09-23T19:04:52+0000 lvl=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" logger=provisioning.datasources filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2023-09-23T19:04:52.458 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:52 smithi092 bash[47618]: t=2023-09-23T19:04:52+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2023-09-23T19:04:52.787 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:52 smithi092 bash[47618]: t=2023-09-23T19:04:52+0000 lvl=info msg="warming cache for startup" logger=ngalert 2023-09-23T19:04:52.787 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:52 smithi092 bash[47618]: t=2023-09-23T19:04:52+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2023-09-23T19:04:52.787 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:04:52 smithi092 bash[47618]: t=2023-09-23T19:04:52+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2023-09-23T19:04:54.273 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:53 smithi016 bash[27237]: cluster 2023-09-23T19:04:52.889360+0000 mgr.y (mgr.14454) 58 : cluster [DBG] pgmap v46: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:54.273 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:53 smithi016 bash[33624]: cluster 2023-09-23T19:04:52.889360+0000 mgr.y (mgr.14454) 58 : cluster [DBG] pgmap v46: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:54.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:53 smithi092 bash[27556]: cluster 2023-09-23T19:04:52.889360+0000 mgr.y (mgr.14454) 58 : cluster [DBG] pgmap v46: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:56.171 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:55 smithi016 bash[27237]: cluster 2023-09-23T19:04:54.890003+0000 mgr.y (mgr.14454) 59 : cluster [DBG] pgmap v47: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:56.171 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:55 smithi016 bash[33624]: cluster 2023-09-23T19:04:54.890003+0000 mgr.y (mgr.14454) 59 : cluster [DBG] pgmap v47: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:56.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:55 smithi092 bash[27556]: cluster 2023-09-23T19:04:54.890003+0000 mgr.y (mgr.14454) 59 : cluster [DBG] pgmap v47: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:57.004 INFO:teuthology.orchestra.run.smithi016.stdout:77309411392 2023-09-23T19:04:57.004 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-09-23T19:04:57.535 INFO:teuthology.orchestra.run.smithi016.stdout:55834574923 2023-09-23T19:04:57.536 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-09-23T19:04:58.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:57 smithi016 bash[33624]: cluster 2023-09-23T19:04:56.890764+0000 mgr.y (mgr.14454) 60 : cluster [DBG] pgmap v48: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:58.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:57 smithi016 bash[27237]: cluster 2023-09-23T19:04:56.890764+0000 mgr.y (mgr.14454) 60 : cluster [DBG] pgmap v48: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:58.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:57 smithi092 bash[27556]: cluster 2023-09-23T19:04:56.890764+0000 mgr.y (mgr.14454) 60 : cluster [DBG] pgmap v48: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:04:59.106 INFO:teuthology.orchestra.run.smithi016.stdout:34359738451 2023-09-23T19:04:59.106 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-09-23T19:04:59.154 INFO:teuthology.orchestra.run.smithi016.stdout:107374182458 2023-09-23T19:04:59.154 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-09-23T19:04:59.186 INFO:teuthology.orchestra.run.smithi016.stdout:154618822696 2023-09-23T19:04:59.186 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-09-23T19:04:59.537 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:04:59 smithi092 bash[28593]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:04:59] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T19:05:00.230 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:04:59 smithi016 bash[33624]: cluster 2023-09-23T19:04:58.891359+0000 mgr.y (mgr.14454) 61 : cluster [DBG] pgmap v49: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:00.231 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:05:00 smithi016 bash[27503]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:05:00] "GET /metrics HTTP/1.1" 200 155656 "" "Prometheus/2.33.4" 2023-09-23T19:05:00.231 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:04:59 smithi016 bash[27237]: cluster 2023-09-23T19:04:58.891359+0000 mgr.y (mgr.14454) 61 : cluster [DBG] pgmap v49: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:00.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:04:59 smithi092 bash[27556]: cluster 2023-09-23T19:04:58.891359+0000 mgr.y (mgr.14454) 61 : cluster [DBG] pgmap v49: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:00.572 INFO:teuthology.orchestra.run.smithi016.stdout:128849018930 2023-09-23T19:05:00.572 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-09-23T19:05:01.464 INFO:teuthology.orchestra.run.smithi016.stdout:180388626465 2023-09-23T19:05:01.464 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-09-23T19:05:01.465 INFO:teuthology.orchestra.run.smithi016.stdout:206158430233 2023-09-23T19:05:01.466 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-09-23T19:05:01.745 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Sep 23 19:05:01 smithi016 systemd[1]: Stopping Ceph alertmanager.a for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7... 2023-09-23T19:05:01.745 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Sep 23 19:05:01 smithi016 bash[56327]: Error response from daemon: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-alertmanager.a 2023-09-23T19:05:01.746 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Sep 23 19:05:01 smithi016 bash[53025]: level=info ts=2023-09-23T19:05:01.693Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2023-09-23T19:05:02.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:01 smithi016 bash[27237]: cluster 2023-09-23T19:05:00.891925+0000 mgr.y (mgr.14454) 62 : cluster [DBG] pgmap v50: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:02.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:01 smithi016 bash[33624]: cluster 2023-09-23T19:05:00.891925+0000 mgr.y (mgr.14454) 62 : cluster [DBG] pgmap v50: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:02.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:01 smithi092 bash[27556]: cluster 2023-09-23T19:05:00.891925+0000 mgr.y (mgr.14454) 62 : cluster [DBG] pgmap v50: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:02.465 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Sep 23 19:05:02 smithi016 bash[56361]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-alertmanager-a 2023-09-23T19:05:02.466 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Sep 23 19:05:02 smithi016 bash[56543]: Error response from daemon: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-alertmanager-a 2023-09-23T19:05:02.466 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Sep 23 19:05:02 smithi016 systemd[1]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-09-23T19:05:02.466 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Sep 23 19:05:02 smithi016 systemd[1]: Stopped Ceph alertmanager.a for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7. 2023-09-23T19:05:02.466 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Sep 23 19:05:02 smithi016 systemd[1]: Started Ceph alertmanager.a for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7. 2023-09-23T19:05:03.704 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:03 smithi092 bash[27556]: audit 2023-09-23T19:05:02.435103+0000 mon.a (mon.0) 600 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:05:03.704 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:03 smithi092 bash[27556]: cephadm 2023-09-23T19:05:02.437953+0000 mgr.y (mgr.14454) 63 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T19:05:03.704 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:03 smithi092 bash[27556]: cephadm 2023-09-23T19:05:02.443260+0000 mgr.y (mgr.14454) 64 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi092 2023-09-23T19:05:03.705 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:03 smithi092 bash[27556]: cluster 2023-09-23T19:05:02.892357+0000 mgr.y (mgr.14454) 65 : cluster [DBG] pgmap v51: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:03.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:03 smithi016 bash[33624]: audit 2023-09-23T19:05:02.435103+0000 mon.a (mon.0) 600 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:05:03.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:03 smithi016 bash[33624]: cephadm 2023-09-23T19:05:02.437953+0000 mgr.y (mgr.14454) 63 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T19:05:03.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:03 smithi016 bash[33624]: cephadm 2023-09-23T19:05:02.443260+0000 mgr.y (mgr.14454) 64 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi092 2023-09-23T19:05:03.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:03 smithi016 bash[33624]: cluster 2023-09-23T19:05:02.892357+0000 mgr.y (mgr.14454) 65 : cluster [DBG] pgmap v51: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:03.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:03 smithi016 bash[27237]: audit 2023-09-23T19:05:02.435103+0000 mon.a (mon.0) 600 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:05:03.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:03 smithi016 bash[27237]: cephadm 2023-09-23T19:05:02.437953+0000 mgr.y (mgr.14454) 63 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T19:05:03.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:03 smithi016 bash[27237]: cephadm 2023-09-23T19:05:02.443260+0000 mgr.y (mgr.14454) 64 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi092 2023-09-23T19:05:03.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:03 smithi016 bash[27237]: cluster 2023-09-23T19:05:02.892357+0000 mgr.y (mgr.14454) 65 : cluster [DBG] pgmap v51: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:03.786 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Sep 23 19:05:03 smithi016 bash[56570]: level=info ts=2023-09-23T19:05:03.467Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2023-09-23T19:05:03.786 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Sep 23 19:05:03 smithi016 bash[56570]: level=info ts=2023-09-23T19:05:03.467Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2023-09-23T19:05:03.787 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Sep 23 19:05:03 smithi016 bash[56570]: level=info ts=2023-09-23T19:05:03.470Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.16 port=9094 2023-09-23T19:05:03.787 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Sep 23 19:05:03 smithi016 bash[56570]: level=info ts=2023-09-23T19:05:03.471Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-09-23T19:05:03.787 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Sep 23 19:05:03 smithi016 bash[56570]: level=info ts=2023-09-23T19:05:03.525Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T19:05:03.787 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Sep 23 19:05:03 smithi016 bash[56570]: level=info ts=2023-09-23T19:05:03.525Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T19:05:03.787 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Sep 23 19:05:03 smithi016 bash[56570]: level=info ts=2023-09-23T19:05:03.530Z caller=main.go:518 msg=Listening address=:9093 2023-09-23T19:05:03.787 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Sep 23 19:05:03 smithi016 bash[56570]: level=info ts=2023-09-23T19:05:03.530Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2023-09-23T19:05:03.910 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:05:03.911 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:05:03.912 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:05:03.912 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:05:03.917 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:05:05.682 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Sep 23 19:05:05 smithi016 bash[56570]: level=info ts=2023-09-23T19:05:05.471Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000054042s 2023-09-23T19:05:05.928 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:05:05.935 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:05:05.935 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:05:06.222 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:05 smithi092 bash[27556]: cluster 2023-09-23T19:05:04.892693+0000 mgr.y (mgr.14454) 66 : cluster [DBG] pgmap v52: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:06.223 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:05:06 smithi092 systemd[1]: Stopping Ceph prometheus.a for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7... 2023-09-23T19:05:06.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:05 smithi016 bash[27237]: cluster 2023-09-23T19:05:04.892693+0000 mgr.y (mgr.14454) 66 : cluster [DBG] pgmap v52: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:06.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:05 smithi016 bash[33624]: cluster 2023-09-23T19:05:04.892693+0000 mgr.y (mgr.14454) 66 : cluster [DBG] pgmap v52: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:06.537 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:05:06 smithi092 bash[48205]: Error response from daemon: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-prometheus.a 2023-09-23T19:05:06.538 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:05:06 smithi092 bash[46844]: ts=2023-09-23T19:05:06.357Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-09-23T19:05:06.538 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:05:06 smithi092 bash[46844]: ts=2023-09-23T19:05:06.357Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-09-23T19:05:06.538 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:05:06 smithi092 bash[46844]: ts=2023-09-23T19:05:06.357Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-09-23T19:05:06.538 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:05:06 smithi092 bash[46844]: ts=2023-09-23T19:05:06.357Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-09-23T19:05:06.538 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:05:06 smithi092 bash[46844]: ts=2023-09-23T19:05:06.357Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-09-23T19:05:06.538 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:05:06 smithi092 bash[46844]: ts=2023-09-23T19:05:06.358Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-09-23T19:05:06.538 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:05:06 smithi092 bash[46844]: ts=2023-09-23T19:05:06.358Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-09-23T19:05:06.538 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:05:06 smithi092 bash[46844]: ts=2023-09-23T19:05:06.358Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-09-23T19:05:06.538 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:05:06 smithi092 bash[46844]: ts=2023-09-23T19:05:06.358Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-09-23T19:05:06.538 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:05:06 smithi092 bash[46844]: ts=2023-09-23T19:05:06.358Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-09-23T19:05:06.539 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:05:06 smithi092 bash[46844]: ts=2023-09-23T19:05:06.359Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-09-23T19:05:06.539 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:05:06 smithi092 bash[46844]: ts=2023-09-23T19:05:06.359Z caller=main.go:1066 level=info msg="See you next time!" 2023-09-23T19:05:07.196 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:05:06 smithi092 bash[48241]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-prometheus-a 2023-09-23T19:05:07.196 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:05:07 smithi092 bash[48306]: Error response from daemon: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-prometheus-a 2023-09-23T19:05:07.196 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:05:07 smithi092 systemd[1]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@prometheus.a.service: Succeeded. 2023-09-23T19:05:07.196 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:05:07 smithi092 systemd[1]: Stopped Ceph prometheus.a for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7. 2023-09-23T19:05:07.196 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:05:07 smithi092 systemd[1]: Started Ceph prometheus.a for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7. 2023-09-23T19:05:07.285 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:05:07 smithi016 bash[27503]: [23/Sep/2023:19:05:07] ENGINE Bus STOPPING 2023-09-23T19:05:07.285 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:05:07 smithi016 bash[27503]: [23/Sep/2023:19:05:07] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T19:05:07.285 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:05:07 smithi016 bash[27503]: [23/Sep/2023:19:05:07] ENGINE Bus STOPPED 2023-09-23T19:05:07.285 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:05:07 smithi016 bash[27503]: [23/Sep/2023:19:05:07] ENGINE Bus STARTING 2023-09-23T19:05:07.505 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:05:07 smithi016 bash[27503]: [23/Sep/2023:19:05:07] ENGINE Serving on http://:::9283 2023-09-23T19:05:07.505 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:05:07 smithi016 bash[27503]: [23/Sep/2023:19:05:07] ENGINE Bus STARTED 2023-09-23T19:05:07.505 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:05:07 smithi016 bash[27503]: [23/Sep/2023:19:05:07] ENGINE Bus STOPPING 2023-09-23T19:05:07.505 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:05:07 smithi016 bash[27503]: [23/Sep/2023:19:05:07] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T19:05:07.505 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:05:07 smithi016 bash[27503]: [23/Sep/2023:19:05:07] ENGINE Bus STOPPED 2023-09-23T19:05:07.505 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:05:07 smithi016 bash[27503]: [23/Sep/2023:19:05:07] ENGINE Bus STARTING 2023-09-23T19:05:07.505 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:05:07 smithi016 bash[27503]: [23/Sep/2023:19:05:07] ENGINE Serving on http://:::9283 2023-09-23T19:05:07.505 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:05:07 smithi016 bash[27503]: [23/Sep/2023:19:05:07] ENGINE Bus STARTED 2023-09-23T19:05:07.505 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:05:07 smithi016 bash[27503]: [23/Sep/2023:19:05:07] ENGINE Bus STOPPING 2023-09-23T19:05:07.506 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:05:07 smithi016 bash[27503]: [23/Sep/2023:19:05:07] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T19:05:07.506 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:05:07 smithi016 bash[27503]: [23/Sep/2023:19:05:07] ENGINE Bus STOPPED 2023-09-23T19:05:07.506 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:05:07 smithi016 bash[27503]: [23/Sep/2023:19:05:07] ENGINE Bus STARTING 2023-09-23T19:05:07.785 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:05:07 smithi016 bash[27503]: [23/Sep/2023:19:05:07] ENGINE Serving on http://:::9283 2023-09-23T19:05:07.785 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:05:07 smithi016 bash[27503]: [23/Sep/2023:19:05:07] ENGINE Bus STARTED 2023-09-23T19:05:08.264 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:05:08 smithi092 bash[48334]: ts=2023-09-23T19:05:08.130Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2023-09-23T19:05:08.264 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:05:08 smithi092 bash[48334]: ts=2023-09-23T19:05:08.130Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2023-09-23T19:05:08.264 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:05:08 smithi092 bash[48334]: ts=2023-09-23T19:05:08.131Z caller=main.go:518 level=info host_details="(Linux 5.15.0-84-generic #93~20.04.1-Ubuntu SMP Wed Sep 6 16:15:40 UTC 2023 x86_64 smithi092 (none))" 2023-09-23T19:05:08.264 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:05:08 smithi092 bash[48334]: ts=2023-09-23T19:05:08.131Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-09-23T19:05:08.265 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:05:08 smithi092 bash[48334]: ts=2023-09-23T19:05:08.131Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-09-23T19:05:08.265 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:05:08 smithi092 bash[48334]: ts=2023-09-23T19:05:08.133Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2023-09-23T19:05:08.265 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:05:08 smithi092 bash[48334]: ts=2023-09-23T19:05:08.133Z caller=main.go:923 level=info msg="Starting TSDB ..." 2023-09-23T19:05:08.265 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:05:08 smithi092 bash[48334]: ts=2023-09-23T19:05:08.135Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2023-09-23T19:05:08.265 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:05:08 smithi092 bash[48334]: ts=2023-09-23T19:05:08.138Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-09-23T19:05:08.265 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:05:08 smithi092 bash[48334]: ts=2023-09-23T19:05:08.138Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=12.315µs 2023-09-23T19:05:08.265 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:05:08 smithi092 bash[48334]: ts=2023-09-23T19:05:08.138Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-09-23T19:05:08.265 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:05:08 smithi092 bash[48334]: ts=2023-09-23T19:05:08.266Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2023-09-23T19:05:08.265 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:07 smithi092 bash[27556]: cluster 2023-09-23T19:05:06.893324+0000 mgr.y (mgr.14454) 67 : cluster [DBG] pgmap v53: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:08.265 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:07 smithi092 bash[27556]: audit 2023-09-23T19:05:07.166259+0000 mon.a (mon.0) 601 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:05:08.266 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:07 smithi092 bash[27556]: audit 2023-09-23T19:05:07.170654+0000 mon.c (mon.2) 49 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T19:05:08.266 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:07 smithi092 bash[27556]: audit 2023-09-23T19:05:07.172409+0000 mon.c (mon.2) 50 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi016.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T19:05:08.266 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:07 smithi092 bash[27556]: audit 2023-09-23T19:05:07.177466+0000 mon.a (mon.0) 602 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:05:08.266 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:07 smithi092 bash[27556]: audit 2023-09-23T19:05:07.183277+0000 mon.c (mon.2) 51 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T19:05:08.266 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:07 smithi092 bash[27556]: audit 2023-09-23T19:05:07.190890+0000 mon.c (mon.2) 52 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi092.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T19:05:08.266 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:07 smithi092 bash[27556]: audit 2023-09-23T19:05:07.198140+0000 mon.a (mon.0) 603 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:05:08.266 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:07 smithi092 bash[27556]: audit 2023-09-23T19:05:07.205721+0000 mon.c (mon.2) 53 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T19:05:08.266 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:07 smithi092 bash[27556]: audit 2023-09-23T19:05:07.209695+0000 mon.c (mon.2) 54 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi092.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T19:05:08.266 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:07 smithi092 bash[27556]: audit 2023-09-23T19:05:07.219604+0000 mon.a (mon.0) 604 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:05:08.266 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:07 smithi092 bash[27556]: audit 2023-09-23T19:05:07.224808+0000 mon.c (mon.2) 55 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:05:08.323 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:07 smithi016 bash[27237]: cluster 2023-09-23T19:05:06.893324+0000 mgr.y (mgr.14454) 67 : cluster [DBG] pgmap v53: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:08.323 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:07 smithi016 bash[27237]: audit 2023-09-23T19:05:07.166259+0000 mon.a (mon.0) 601 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:05:08.323 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:07 smithi016 bash[27237]: audit 2023-09-23T19:05:07.170654+0000 mon.c (mon.2) 49 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T19:05:08.323 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:07 smithi016 bash[27237]: audit 2023-09-23T19:05:07.172409+0000 mon.c (mon.2) 50 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi016.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T19:05:08.323 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:07 smithi016 bash[27237]: audit 2023-09-23T19:05:07.177466+0000 mon.a (mon.0) 602 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:05:08.323 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:07 smithi016 bash[27237]: audit 2023-09-23T19:05:07.183277+0000 mon.c (mon.2) 51 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T19:05:08.324 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:07 smithi016 bash[27237]: audit 2023-09-23T19:05:07.190890+0000 mon.c (mon.2) 52 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi092.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T19:05:08.324 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:07 smithi016 bash[27237]: audit 2023-09-23T19:05:07.198140+0000 mon.a (mon.0) 603 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:05:08.324 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:07 smithi016 bash[27237]: audit 2023-09-23T19:05:07.205721+0000 mon.c (mon.2) 53 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T19:05:08.324 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:07 smithi016 bash[27237]: audit 2023-09-23T19:05:07.209695+0000 mon.c (mon.2) 54 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi092.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T19:05:08.324 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:07 smithi016 bash[27237]: audit 2023-09-23T19:05:07.219604+0000 mon.a (mon.0) 604 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:05:08.324 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:07 smithi016 bash[27237]: audit 2023-09-23T19:05:07.224808+0000 mon.c (mon.2) 55 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:05:08.324 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:07 smithi016 bash[33624]: cluster 2023-09-23T19:05:06.893324+0000 mgr.y (mgr.14454) 67 : cluster [DBG] pgmap v53: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:08.324 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:07 smithi016 bash[33624]: audit 2023-09-23T19:05:07.166259+0000 mon.a (mon.0) 601 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:05:08.325 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:07 smithi016 bash[33624]: audit 2023-09-23T19:05:07.170654+0000 mon.c (mon.2) 49 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T19:05:08.325 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:07 smithi016 bash[33624]: audit 2023-09-23T19:05:07.172409+0000 mon.c (mon.2) 50 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi016.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T19:05:08.325 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:07 smithi016 bash[33624]: audit 2023-09-23T19:05:07.177466+0000 mon.a (mon.0) 602 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:05:08.325 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:07 smithi016 bash[33624]: audit 2023-09-23T19:05:07.183277+0000 mon.c (mon.2) 51 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T19:05:08.325 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:07 smithi016 bash[33624]: audit 2023-09-23T19:05:07.190890+0000 mon.c (mon.2) 52 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi092.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T19:05:08.325 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:07 smithi016 bash[33624]: audit 2023-09-23T19:05:07.198140+0000 mon.a (mon.0) 603 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:05:08.325 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:07 smithi016 bash[33624]: audit 2023-09-23T19:05:07.205721+0000 mon.c (mon.2) 53 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T19:05:08.325 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:07 smithi016 bash[33624]: audit 2023-09-23T19:05:07.209695+0000 mon.c (mon.2) 54 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi092.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T19:05:08.325 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:07 smithi016 bash[33624]: audit 2023-09-23T19:05:07.219604+0000 mon.a (mon.0) 604 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:05:08.325 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:07 smithi016 bash[33624]: audit 2023-09-23T19:05:07.224808+0000 mon.c (mon.2) 55 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:05:08.537 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:05:08 smithi092 bash[48334]: ts=2023-09-23T19:05:08.267Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2023-09-23T19:05:08.538 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:05:08 smithi092 bash[48334]: ts=2023-09-23T19:05:08.267Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=41.967µs wal_replay_duration=128.626207ms total_replay_duration=128.693844ms 2023-09-23T19:05:08.538 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:05:08 smithi092 bash[48334]: ts=2023-09-23T19:05:08.270Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2023-09-23T19:05:08.538 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:05:08 smithi092 bash[48334]: ts=2023-09-23T19:05:08.270Z caller=main.go:947 level=info msg="TSDB started" 2023-09-23T19:05:08.538 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:05:08 smithi092 bash[48334]: ts=2023-09-23T19:05:08.270Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-09-23T19:05:08.538 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:05:08 smithi092 bash[48334]: ts=2023-09-23T19:05:08.300Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=29.993995ms db_storage=1.622µs remote_storage=2.267µs web_handler=580ns query_engine=1.223µs scrape=502.947µs scrape_sd=48.236µs notify=45.274µs notify_sd=12.377µs rules=28.773655ms 2023-09-23T19:05:08.538 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:05:08 smithi092 bash[48334]: ts=2023-09-23T19:05:08.301Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-09-23T19:05:09.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:08 smithi016 bash[27237]: audit 2023-09-23T19:05:07.171144+0000 mgr.y (mgr.14454) 68 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T19:05:09.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:08 smithi016 bash[27237]: audit 2023-09-23T19:05:07.172754+0000 mgr.y (mgr.14454) 69 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi016.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T19:05:09.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:08 smithi016 bash[27237]: audit 2023-09-23T19:05:07.183928+0000 mgr.y (mgr.14454) 70 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T19:05:09.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:08 smithi016 bash[27237]: audit 2023-09-23T19:05:07.191337+0000 mgr.y (mgr.14454) 71 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi092.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T19:05:09.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:08 smithi016 bash[27237]: audit 2023-09-23T19:05:07.206187+0000 mgr.y (mgr.14454) 72 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T19:05:09.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:08 smithi016 bash[27237]: audit 2023-09-23T19:05:07.210196+0000 mgr.y (mgr.14454) 73 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi092.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T19:05:09.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:08 smithi016 bash[33624]: audit 2023-09-23T19:05:07.171144+0000 mgr.y (mgr.14454) 68 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T19:05:09.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:08 smithi016 bash[33624]: audit 2023-09-23T19:05:07.172754+0000 mgr.y (mgr.14454) 69 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi016.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T19:05:09.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:08 smithi016 bash[33624]: audit 2023-09-23T19:05:07.183928+0000 mgr.y (mgr.14454) 70 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T19:05:09.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:08 smithi016 bash[33624]: audit 2023-09-23T19:05:07.191337+0000 mgr.y (mgr.14454) 71 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi092.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T19:05:09.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:08 smithi016 bash[33624]: audit 2023-09-23T19:05:07.206187+0000 mgr.y (mgr.14454) 72 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T19:05:09.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:08 smithi016 bash[33624]: audit 2023-09-23T19:05:07.210196+0000 mgr.y (mgr.14454) 73 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi092.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T19:05:09.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:08 smithi092 bash[27556]: audit 2023-09-23T19:05:07.171144+0000 mgr.y (mgr.14454) 68 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T19:05:09.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:08 smithi092 bash[27556]: audit 2023-09-23T19:05:07.172754+0000 mgr.y (mgr.14454) 69 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi016.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T19:05:09.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:08 smithi092 bash[27556]: audit 2023-09-23T19:05:07.183928+0000 mgr.y (mgr.14454) 70 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T19:05:09.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:08 smithi092 bash[27556]: audit 2023-09-23T19:05:07.191337+0000 mgr.y (mgr.14454) 71 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi092.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T19:05:09.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:08 smithi092 bash[27556]: audit 2023-09-23T19:05:07.206187+0000 mgr.y (mgr.14454) 72 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T19:05:09.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:08 smithi092 bash[27556]: audit 2023-09-23T19:05:07.210196+0000 mgr.y (mgr.14454) 73 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi092.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T19:05:10.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:09 smithi016 bash[27237]: cluster 2023-09-23T19:05:08.893768+0000 mgr.y (mgr.14454) 74 : cluster [DBG] pgmap v54: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:10.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:09 smithi016 bash[33624]: cluster 2023-09-23T19:05:08.893768+0000 mgr.y (mgr.14454) 74 : cluster [DBG] pgmap v54: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:10.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:09 smithi092 bash[27556]: cluster 2023-09-23T19:05:08.893768+0000 mgr.y (mgr.14454) 74 : cluster [DBG] pgmap v54: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:12.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:11 smithi016 bash[27237]: cluster 2023-09-23T19:05:10.894282+0000 mgr.y (mgr.14454) 75 : cluster [DBG] pgmap v55: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:12.249 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:11 smithi016 bash[33624]: cluster 2023-09-23T19:05:10.894282+0000 mgr.y (mgr.14454) 75 : cluster [DBG] pgmap v55: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:12.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:11 smithi092 bash[27556]: cluster 2023-09-23T19:05:10.894282+0000 mgr.y (mgr.14454) 75 : cluster [DBG] pgmap v55: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:13.846 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Sep 23 19:05:13 smithi016 bash[56570]: level=info ts=2023-09-23T19:05:13.473Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.00251002s 2023-09-23T19:05:14.232 INFO:teuthology.orchestra.run.smithi016.stdout:107374182462 2023-09-23T19:05:14.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:14 smithi016 bash[33624]: cluster 2023-09-23T19:05:12.894910+0000 mgr.y (mgr.14454) 76 : cluster [DBG] pgmap v56: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:14.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:14 smithi016 bash[33624]: audit 2023-09-23T19:05:13.098359+0000 mon.a (mon.0) 605 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:05:14.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:13 smithi016 bash[27237]: cluster 2023-09-23T19:05:12.894910+0000 mgr.y (mgr.14454) 76 : cluster [DBG] pgmap v56: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:14.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:13 smithi016 bash[27237]: audit 2023-09-23T19:05:13.098359+0000 mon.a (mon.0) 605 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:05:14.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:13 smithi092 bash[27556]: cluster 2023-09-23T19:05:12.894910+0000 mgr.y (mgr.14454) 76 : cluster [DBG] pgmap v56: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:14.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:13 smithi092 bash[27556]: audit 2023-09-23T19:05:13.098359+0000 mon.a (mon.0) 605 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:05:15.194 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:14 smithi016 bash[27237]: audit 2023-09-23T19:05:14.229683+0000 mon.a (mon.0) 606 : audit [DBG] from='client.? 172.21.15.16:0/2994051766' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T19:05:15.195 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:14 smithi016 bash[33624]: audit 2023-09-23T19:05:14.229683+0000 mon.a (mon.0) 606 : audit [DBG] from='client.? 172.21.15.16:0/2994051766' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T19:05:15.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:14 smithi092 bash[27556]: audit 2023-09-23T19:05:14.229683+0000 mon.a (mon.0) 606 : audit [DBG] from='client.? 172.21.15.16:0/2994051766' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T19:05:15.571 INFO:teuthology.orchestra.run.smithi016.stdout:34359738454 2023-09-23T19:05:16.172 INFO:teuthology.orchestra.run.smithi016.stdout:77309411396 2023-09-23T19:05:16.242 INFO:teuthology.orchestra.run.smithi016.stdout:154618822700 2023-09-23T19:05:16.281 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:15 smithi016 bash[33624]: cluster 2023-09-23T19:05:14.895507+0000 mgr.y (mgr.14454) 77 : cluster [DBG] pgmap v57: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:16.281 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:15 smithi016 bash[33624]: audit 2023-09-23T19:05:15.568716+0000 mon.c (mon.2) 56 : audit [DBG] from='client.? 172.21.15.16:0/3928598556' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T19:05:16.282 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:15 smithi016 bash[27237]: cluster 2023-09-23T19:05:14.895507+0000 mgr.y (mgr.14454) 77 : cluster [DBG] pgmap v57: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:16.282 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:15 smithi016 bash[27237]: audit 2023-09-23T19:05:15.568716+0000 mon.c (mon.2) 56 : audit [DBG] from='client.? 172.21.15.16:0/3928598556' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T19:05:16.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:15 smithi092 bash[27556]: cluster 2023-09-23T19:05:14.895507+0000 mgr.y (mgr.14454) 77 : cluster [DBG] pgmap v57: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:16.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:15 smithi092 bash[27556]: audit 2023-09-23T19:05:15.568716+0000 mon.c (mon.2) 56 : audit [DBG] from='client.? 172.21.15.16:0/3928598556' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T19:05:17.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:16 smithi016 bash[27237]: audit 2023-09-23T19:05:16.169867+0000 mon.a (mon.0) 607 : audit [DBG] from='client.? 172.21.15.16:0/266454147' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T19:05:17.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:16 smithi016 bash[27237]: audit 2023-09-23T19:05:16.239983+0000 mon.c (mon.2) 57 : audit [DBG] from='client.? 172.21.15.16:0/434349096' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T19:05:17.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:16 smithi016 bash[33624]: audit 2023-09-23T19:05:16.169867+0000 mon.a (mon.0) 607 : audit [DBG] from='client.? 172.21.15.16:0/266454147' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T19:05:17.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:16 smithi016 bash[33624]: audit 2023-09-23T19:05:16.239983+0000 mon.c (mon.2) 57 : audit [DBG] from='client.? 172.21.15.16:0/434349096' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T19:05:17.287 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182458 got 107374182462 for osd.3 2023-09-23T19:05:17.287 DEBUG:teuthology.parallel:result is None 2023-09-23T19:05:17.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:16 smithi092 bash[27556]: audit 2023-09-23T19:05:16.169867+0000 mon.a (mon.0) 607 : audit [DBG] from='client.? 172.21.15.16:0/266454147' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T19:05:17.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:16 smithi092 bash[27556]: audit 2023-09-23T19:05:16.239983+0000 mon.c (mon.2) 57 : audit [DBG] from='client.? 172.21.15.16:0/434349096' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T19:05:17.488 INFO:teuthology.orchestra.run.smithi016.stdout:55834574928 2023-09-23T19:05:18.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:17 smithi092 bash[27556]: cluster 2023-09-23T19:05:16.896180+0000 mgr.y (mgr.14454) 78 : cluster [DBG] pgmap v58: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:18.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:17 smithi092 bash[27556]: audit 2023-09-23T19:05:17.485286+0000 mon.c (mon.2) 58 : audit [DBG] from='client.? 172.21.15.16:0/2568434750' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T19:05:18.349 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:17 smithi016 bash[33624]: cluster 2023-09-23T19:05:16.896180+0000 mgr.y (mgr.14454) 78 : cluster [DBG] pgmap v58: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:18.349 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:17 smithi016 bash[33624]: audit 2023-09-23T19:05:17.485286+0000 mon.c (mon.2) 58 : audit [DBG] from='client.? 172.21.15.16:0/2568434750' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T19:05:18.349 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:17 smithi016 bash[27237]: cluster 2023-09-23T19:05:16.896180+0000 mgr.y (mgr.14454) 78 : cluster [DBG] pgmap v58: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:18.349 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:17 smithi016 bash[27237]: audit 2023-09-23T19:05:17.485286+0000 mon.c (mon.2) 58 : audit [DBG] from='client.? 172.21.15.16:0/2568434750' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T19:05:18.525 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738451 got 34359738454 for osd.0 2023-09-23T19:05:18.525 DEBUG:teuthology.parallel:result is None 2023-09-23T19:05:18.832 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411392 got 77309411396 for osd.2 2023-09-23T19:05:18.832 DEBUG:teuthology.parallel:result is None 2023-09-23T19:05:18.974 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618822696 got 154618822700 for osd.5 2023-09-23T19:05:18.974 DEBUG:teuthology.parallel:result is None 2023-09-23T19:05:19.537 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:05:19 smithi092 bash[28593]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:05:19] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T19:05:19.978 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574923 got 55834574928 for osd.1 2023-09-23T19:05:19.979 DEBUG:teuthology.parallel:result is None 2023-09-23T19:05:19.980 INFO:teuthology.orchestra.run.smithi016.stdout:206158430237 2023-09-23T19:05:20.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:19 smithi016 bash[27237]: cluster 2023-09-23T19:05:18.896597+0000 mgr.y (mgr.14454) 79 : cluster [DBG] pgmap v59: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:20.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:19 smithi016 bash[27237]: audit 2023-09-23T19:05:19.978154+0000 mon.a (mon.0) 608 : audit [DBG] from='client.? 172.21.15.16:0/1881650484' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T19:05:20.285 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:05:20 smithi016 bash[27503]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:05:20] "GET /metrics HTTP/1.1" 200 155653 "" "Prometheus/2.33.4" 2023-09-23T19:05:20.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:19 smithi016 bash[33624]: cluster 2023-09-23T19:05:18.896597+0000 mgr.y (mgr.14454) 79 : cluster [DBG] pgmap v59: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:20.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:19 smithi016 bash[33624]: audit 2023-09-23T19:05:19.978154+0000 mon.a (mon.0) 608 : audit [DBG] from='client.? 172.21.15.16:0/1881650484' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T19:05:20.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:19 smithi092 bash[27556]: cluster 2023-09-23T19:05:18.896597+0000 mgr.y (mgr.14454) 79 : cluster [DBG] pgmap v59: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:20.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:19 smithi092 bash[27556]: audit 2023-09-23T19:05:19.978154+0000 mon.a (mon.0) 608 : audit [DBG] from='client.? 172.21.15.16:0/1881650484' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T19:05:20.447 INFO:teuthology.orchestra.run.smithi016.stdout:180388626469 2023-09-23T19:05:20.526 INFO:teuthology.orchestra.run.smithi016.stdout:128849018934 2023-09-23T19:05:21.011 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430233 got 206158430237 for osd.7 2023-09-23T19:05:21.011 DEBUG:teuthology.parallel:result is None 2023-09-23T19:05:21.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:21 smithi016 bash[27237]: audit 2023-09-23T19:05:20.444615+0000 mon.a (mon.0) 609 : audit [DBG] from='client.? 172.21.15.16:0/1904470302' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T19:05:21.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:21 smithi016 bash[27237]: audit 2023-09-23T19:05:20.523135+0000 mon.c (mon.2) 59 : audit [DBG] from='client.? 172.21.15.16:0/1741535631' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T19:05:21.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:21 smithi016 bash[33624]: audit 2023-09-23T19:05:20.444615+0000 mon.a (mon.0) 609 : audit [DBG] from='client.? 172.21.15.16:0/1904470302' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T19:05:21.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:21 smithi016 bash[33624]: audit 2023-09-23T19:05:20.523135+0000 mon.c (mon.2) 59 : audit [DBG] from='client.? 172.21.15.16:0/1741535631' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T19:05:21.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:21 smithi092 bash[27556]: audit 2023-09-23T19:05:20.444615+0000 mon.a (mon.0) 609 : audit [DBG] from='client.? 172.21.15.16:0/1904470302' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T19:05:21.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:21 smithi092 bash[27556]: audit 2023-09-23T19:05:20.523135+0000 mon.c (mon.2) 59 : audit [DBG] from='client.? 172.21.15.16:0/1741535631' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T19:05:21.976 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388626465 got 180388626469 for osd.6 2023-09-23T19:05:21.976 DEBUG:teuthology.parallel:result is None 2023-09-23T19:05:22.065 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018930 got 128849018934 for osd.4 2023-09-23T19:05:22.066 DEBUG:teuthology.parallel:result is None 2023-09-23T19:05:22.066 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-09-23T19:05:22.066 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T19:05:22.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:22 smithi016 bash[27237]: cluster 2023-09-23T19:05:20.896948+0000 mgr.y (mgr.14454) 80 : cluster [DBG] pgmap v60: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:22.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:22 smithi016 bash[27237]: audit 2023-09-23T19:05:21.163958+0000 mon.a (mon.0) 610 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:05:22.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:22 smithi016 bash[27237]: audit 2023-09-23T19:05:21.165676+0000 mon.c (mon.2) 60 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:05:22.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:22 smithi016 bash[27237]: audit 2023-09-23T19:05:21.166655+0000 mon.c (mon.2) 61 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:05:22.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:22 smithi016 bash[27237]: audit 2023-09-23T19:05:21.172662+0000 mon.a (mon.0) 611 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:05:22.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:22 smithi016 bash[27237]: audit 2023-09-23T19:05:21.177362+0000 mon.c (mon.2) 62 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:05:22.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:22 smithi016 bash[33624]: cluster 2023-09-23T19:05:20.896948+0000 mgr.y (mgr.14454) 80 : cluster [DBG] pgmap v60: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:22.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:22 smithi016 bash[33624]: audit 2023-09-23T19:05:21.163958+0000 mon.a (mon.0) 610 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:05:22.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:22 smithi016 bash[33624]: audit 2023-09-23T19:05:21.165676+0000 mon.c (mon.2) 60 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:05:22.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:22 smithi016 bash[33624]: audit 2023-09-23T19:05:21.166655+0000 mon.c (mon.2) 61 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:05:22.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:22 smithi016 bash[33624]: audit 2023-09-23T19:05:21.172662+0000 mon.a (mon.0) 611 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:05:22.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:22 smithi016 bash[33624]: audit 2023-09-23T19:05:21.177362+0000 mon.c (mon.2) 62 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:05:22.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:22 smithi092 bash[27556]: cluster 2023-09-23T19:05:20.896948+0000 mgr.y (mgr.14454) 80 : cluster [DBG] pgmap v60: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:22.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:22 smithi092 bash[27556]: audit 2023-09-23T19:05:21.163958+0000 mon.a (mon.0) 610 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:05:22.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:22 smithi092 bash[27556]: audit 2023-09-23T19:05:21.165676+0000 mon.c (mon.2) 60 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:05:22.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:22 smithi092 bash[27556]: audit 2023-09-23T19:05:21.166655+0000 mon.c (mon.2) 61 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:05:22.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:22 smithi092 bash[27556]: audit 2023-09-23T19:05:21.172662+0000 mon.a (mon.0) 611 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:05:22.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:22 smithi092 bash[27556]: audit 2023-09-23T19:05:21.177362+0000 mon.c (mon.2) 62 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:05:24.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:24 smithi016 bash[33624]: cluster 2023-09-23T19:05:22.897592+0000 mgr.y (mgr.14454) 81 : cluster [DBG] pgmap v61: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:24.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:24 smithi016 bash[27237]: cluster 2023-09-23T19:05:22.897592+0000 mgr.y (mgr.14454) 81 : cluster [DBG] pgmap v61: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:24.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:24 smithi092 bash[27556]: cluster 2023-09-23T19:05:22.897592+0000 mgr.y (mgr.14454) 81 : cluster [DBG] pgmap v61: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:26.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:26 smithi016 bash[33624]: cluster 2023-09-23T19:05:24.898180+0000 mgr.y (mgr.14454) 82 : cluster [DBG] pgmap v62: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:26.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:26 smithi016 bash[27237]: cluster 2023-09-23T19:05:24.898180+0000 mgr.y (mgr.14454) 82 : cluster [DBG] pgmap v62: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:26.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:26 smithi092 bash[27556]: cluster 2023-09-23T19:05:24.898180+0000 mgr.y (mgr.14454) 82 : cluster [DBG] pgmap v62: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:26.848 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:05:28.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:28 smithi016 bash[27237]: cluster 2023-09-23T19:05:26.898867+0000 mgr.y (mgr.14454) 83 : cluster [DBG] pgmap v63: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:28.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:28 smithi016 bash[27237]: audit 2023-09-23T19:05:27.008389+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:05:28.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:28 smithi016 bash[27237]: audit 2023-09-23T19:05:27.010589+0000 mon.c (mon.2) 63 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:05:28.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:28 smithi016 bash[27237]: audit 2023-09-23T19:05:27.011948+0000 mon.c (mon.2) 64 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:05:28.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:28 smithi016 bash[27237]: audit 2023-09-23T19:05:27.022521+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:05:28.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:28 smithi016 bash[33624]: cluster 2023-09-23T19:05:26.898867+0000 mgr.y (mgr.14454) 83 : cluster [DBG] pgmap v63: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:28.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:28 smithi016 bash[33624]: audit 2023-09-23T19:05:27.008389+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:05:28.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:28 smithi016 bash[33624]: audit 2023-09-23T19:05:27.010589+0000 mon.c (mon.2) 63 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:05:28.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:28 smithi016 bash[33624]: audit 2023-09-23T19:05:27.011948+0000 mon.c (mon.2) 64 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:05:28.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:28 smithi016 bash[33624]: audit 2023-09-23T19:05:27.022521+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:05:28.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:28 smithi092 bash[27556]: cluster 2023-09-23T19:05:26.898867+0000 mgr.y (mgr.14454) 83 : cluster [DBG] pgmap v63: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:28.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:28 smithi092 bash[27556]: audit 2023-09-23T19:05:27.008389+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:05:28.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:28 smithi092 bash[27556]: audit 2023-09-23T19:05:27.010589+0000 mon.c (mon.2) 63 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:05:28.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:28 smithi092 bash[27556]: audit 2023-09-23T19:05:27.011948+0000 mon.c (mon.2) 64 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:05:28.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:28 smithi092 bash[27556]: audit 2023-09-23T19:05:27.022521+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:05:29.537 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:05:29 smithi092 bash[28593]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:05:29] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T19:05:29.749 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-09-23T19:05:29.751 INFO:teuthology.orchestra.run.smithi016.stderr:dumped all 2023-09-23T19:05:30.072 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:30 smithi016 bash[33624]: cluster 2023-09-23T19:05:28.899456+0000 mgr.y (mgr.14454) 84 : cluster [DBG] pgmap v64: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:30.073 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:30 smithi016 bash[27237]: cluster 2023-09-23T19:05:28.899456+0000 mgr.y (mgr.14454) 84 : cluster [DBG] pgmap v64: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:30.285 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:05:30 smithi016 bash[27503]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:05:30] "GET /metrics HTTP/1.1" 200 155654 "" "Prometheus/2.33.4" 2023-09-23T19:05:30.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:30 smithi092 bash[27556]: cluster 2023-09-23T19:05:28.899456+0000 mgr.y (mgr.14454) 84 : cluster [DBG] pgmap v64: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:30.414 INFO:teuthology.orchestra.run.smithi016.stdout:{"pg_ready":true,"pg_map":{"version":64,"stamp":"2023-09-23T19:05:28.899178+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2380980,"kb_used_data":4212,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":747580236,"statfs":{"total":767960285184,"available":765522161664,"internally_reserved":0,"allocated":4313088,"data_stored":1763175,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.000000"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-09-23T19:04:03.609606+0000","last_change":"2023-09-23T19:04:02.507717+0000","last_active":"2023-09-23T19:04:03.609606+0000","last_peered":"2023-09-23T19:04:03.609606+0000","last_clean":"2023-09-23T19:04:03.609606+0000","last_became_active":"2023-09-23T19:04:02.507042+0000","last_became_peered":"2023-09-23T19:04:02.507042+0000","last_unstale":"2023-09-23T19:04:03.609606+0000","last_undegraded":"2023-09-23T19:04:03.609606+0000","last_fullsized":"2023-09-23T19:04:03.609606+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-09-23T19:04:03.603569+0000","last_change":"2023-09-23T19:04:02.707894+0000","last_active":"2023-09-23T19:04:03.603569+0000","last_peered":"2023-09-23T19:04:03.603569+0000","last_clean":"2023-09-23T19:04:03.603569+0000","last_became_active":"2023-09-23T19:04:02.707071+0000","last_became_peered":"2023-09-23T19:04:02.707071+0000","last_unstale":"2023-09-23T19:04:03.603569+0000","last_undegraded":"2023-09-23T19:04:03.603569+0000","last_fullsized":"2023-09-23T19:04:03.603569+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-09-23T19:04:03.610292+0000","last_change":"2023-09-23T19:04:02.713482+0000","last_active":"2023-09-23T19:04:03.610292+0000","last_peered":"2023-09-23T19:04:03.610292+0000","last_clean":"2023-09-23T19:04:03.610292+0000","last_became_active":"2023-09-23T19:04:02.712471+0000","last_became_peered":"2023-09-23T19:04:02.712471+0000","last_unstale":"2023-09-23T19:04:03.610292+0000","last_undegraded":"2023-09-23T19:04:03.610292+0000","last_fullsized":"2023-09-23T19:04:03.610292+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-09-23T19:04:03.610042+0000","last_change":"2023-09-23T19:04:02.712803+0000","last_active":"2023-09-23T19:04:03.610042+0000","last_peered":"2023-09-23T19:04:03.610042+0000","last_clean":"2023-09-23T19:04:03.610042+0000","last_became_active":"2023-09-23T19:04:02.711952+0000","last_became_peered":"2023-09-23T19:04:02.711952+0000","last_unstale":"2023-09-23T19:04:03.610042+0000","last_undegraded":"2023-09-23T19:04:03.610042+0000","last_fullsized":"2023-09-23T19:04:03.610042+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":15,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-09-23T19:04:03.610239+0000","last_change":"2023-09-23T19:03:04.495032+0000","last_active":"2023-09-23T19:04:03.610239+0000","last_peered":"2023-09-23T19:04:03.610239+0000","last_clean":"2023-09-23T19:04:03.610239+0000","last_became_active":"2023-09-23T19:03:04.494463+0000","last_became_peered":"2023-09-23T19:03:04.494463+0000","last_unstale":"2023-09-23T19:04:03.610239+0000","last_undegraded":"2023-09-23T19:04:03.610239+0000","last_fullsized":"2023-09-23T19:04:03.610239+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:59:33.840751+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:59:33.840751+0000","last_clean_scrub_stamp":"2023-09-23T18:59:33.840751+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-09-23T19:04:03.610045+0000","last_change":"2023-09-23T19:04:02.509889+0000","last_active":"2023-09-23T19:04:03.610045+0000","last_peered":"2023-09-23T19:04:03.610045+0000","last_clean":"2023-09-23T19:04:03.610045+0000","last_became_active":"2023-09-23T19:04:02.509458+0000","last_became_peered":"2023-09-23T19:04:02.509458+0000","last_unstale":"2023-09-23T19:04:03.610045+0000","last_undegraded":"2023-09-23T19:04:03.610045+0000","last_fullsized":"2023-09-23T19:04:03.610045+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-09-23T19:04:03.505472+0000","last_change":"2023-09-23T19:04:02.707164+0000","last_active":"2023-09-23T19:04:03.505472+0000","last_peered":"2023-09-23T19:04:03.505472+0000","last_clean":"2023-09-23T19:04:03.505472+0000","last_became_active":"2023-09-23T19:04:02.706747+0000","last_became_peered":"2023-09-23T19:04:02.706747+0000","last_unstale":"2023-09-23T19:04:03.505472+0000","last_undegraded":"2023-09-23T19:04:03.505472+0000","last_fullsized":"2023-09-23T19:04:03.505472+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-09-23T19:04:03.610031+0000","last_change":"2023-09-23T19:04:02.509914+0000","last_active":"2023-09-23T19:04:03.610031+0000","last_peered":"2023-09-23T19:04:03.610031+0000","last_clean":"2023-09-23T19:04:03.610031+0000","last_became_active":"2023-09-23T19:04:02.509532+0000","last_became_peered":"2023-09-23T19:04:02.509532+0000","last_unstale":"2023-09-23T19:04:03.610031+0000","last_undegraded":"2023-09-23T19:04:03.610031+0000","last_fullsized":"2023-09-23T19:04:03.610031+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-09-23T19:04:03.603476+0000","last_change":"2023-09-23T19:04:02.707302+0000","last_active":"2023-09-23T19:04:03.603476+0000","last_peered":"2023-09-23T19:04:03.603476+0000","last_clean":"2023-09-23T19:04:03.603476+0000","last_became_active":"2023-09-23T19:04:02.706904+0000","last_became_peered":"2023-09-23T19:04:02.706904+0000","last_unstale":"2023-09-23T19:04:03.603476+0000","last_undegraded":"2023-09-23T19:04:03.603476+0000","last_fullsized":"2023-09-23T19:04:03.603476+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430239,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:05:07 2023","interfaces":[{"interface":"back","average":{"1min":0.84399999999999997,"5min":0.91800000000000004,"15min":0.93000000000000005},"min":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":1.2729999999999999,"5min":1.4019999999999999,"15min":1.4019999999999999},"last":0.94299999999999995},{"interface":"front","average":{"1min":0.89300000000000002,"5min":0.90300000000000002,"15min":0.90500000000000003},"min":{"1min":0.58899999999999997,"5min":0.57699999999999996,"15min":0.57699999999999996},"max":{"1min":1.1000000000000001,"5min":1.552,"15min":1.552},"last":0.75900000000000001}]},{"osd":1,"last update":"Sat Sep 23 19:05:07 2023","interfaces":[{"interface":"back","average":{"1min":0.84099999999999997,"5min":0.89500000000000002,"15min":0.90400000000000003},"min":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.224,"5min":1.224,"15min":1.224},"last":0.90400000000000003},{"interface":"front","average":{"1min":0.96299999999999997,"5min":0.96799999999999997,"15min":0.96899999999999997},"min":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"max":{"1min":1.323,"5min":1.498,"15min":1.498},"last":0.96699999999999997}]},{"osd":2,"last update":"Sat Sep 23 19:05:07 2023","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":0.90700000000000003,"15min":0.90400000000000003},"min":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"max":{"1min":1.129,"5min":1.2010000000000001,"15min":1.2010000000000001},"last":0.79800000000000004},{"interface":"front","average":{"1min":0.90100000000000002,"5min":0.96499999999999997,"15min":0.97599999999999998},"min":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.202,"5min":1.417,"15min":1.417},"last":1.194}]},{"osd":3,"last update":"Sat Sep 23 19:05:07 2023","interfaces":[{"interface":"back","average":{"1min":0.94799999999999995,"5min":1.0449999999999999,"15min":1.0609999999999999},"min":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"max":{"1min":1.2310000000000001,"5min":1.4450000000000001,"15min":1.4450000000000001},"last":1.155},{"interface":"front","average":{"1min":0.93100000000000005,"5min":1.089,"15min":1.115},"min":{"1min":0.59799999999999998,"5min":0.58999999999999997,"15min":0.58999999999999997},"max":{"1min":1.2989999999999999,"5min":3.3780000000000001,"15min":3.3780000000000001},"last":1.05}]},{"osd":4,"last update":"Sat Sep 23 19:05:07 2023","interfaces":[{"interface":"back","average":{"1min":0.90700000000000003,"5min":0.88300000000000001,"15min":0.879},"min":{"1min":0.53000000000000003,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":1.2689999999999999,"5min":1.4139999999999999,"15min":1.4139999999999999},"last":1.292},{"interface":"front","average":{"1min":0.97499999999999998,"5min":0.95899999999999996,"15min":0.95699999999999996},"min":{"1min":0.56699999999999995,"5min":0.54500000000000004,"15min":0.54500000000000004},"max":{"1min":1.448,"5min":1.5720000000000001,"15min":1.5720000000000001},"last":1.2470000000000001}]},{"osd":5,"last update":"Sat Sep 23 19:05:07 2023","interfaces":[{"interface":"back","average":{"1min":0.95399999999999996,"5min":0.95299999999999996,"15min":0.95299999999999996},"min":{"1min":0.51400000000000001,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":1.2430000000000001,"5min":1.472,"15min":1.472},"last":1.1439999999999999},{"interface":"front","average":{"1min":1.0309999999999999,"5min":1.0309999999999999,"15min":1.0309999999999999},"min":{"1min":0.72399999999999998,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.343,"5min":1.343,"15min":1.343},"last":1.2170000000000001}]},{"osd":6,"last update":"Sat Sep 23 19:05:07 2023","interfaces":[{"interface":"back","average":{"1min":1.141,"5min":1.0640000000000001,"15min":1.0509999999999999},"min":{"1min":0.83599999999999997,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":2.0339999999999998,"5min":2.0339999999999998,"15min":2.0339999999999998},"last":1.111},{"interface":"front","average":{"1min":1.1339999999999999,"5min":1.0900000000000001,"15min":1.083},"min":{"1min":0.76300000000000001,"5min":0.53000000000000003,"15min":0.53000000000000003},"max":{"1min":1.472,"5min":1.4730000000000001,"15min":1.4730000000000001},"last":1.081}]}]},{"osd":6,"up_from":42,"seq":180388626471,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297620,"kb_used_data":524,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447532,"statfs":{"total":95995035648,"available":95690272768,"internally_reserved":0,"allocated":536576,"data_stored":220200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:04:27 2023","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":0.91100000000000003,"15min":0.92300000000000004},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":1.343,"5min":1.4179999999999999,"15min":1.4179999999999999},"last":0.88100000000000001},{"interface":"front","average":{"1min":0.78600000000000003,"5min":0.89300000000000002,"15min":0.91100000000000003},"min":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":0.997,"5min":1.109,"15min":1.109},"last":1.0309999999999999}]},{"osd":1,"last update":"Sat Sep 23 19:04:27 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":1.0860000000000001,"15min":1.1020000000000001},"min":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"max":{"1min":1.286,"5min":1.4379999999999999,"15min":1.4379999999999999},"last":1.004},{"interface":"front","average":{"1min":0.97199999999999998,"5min":1.0469999999999999,"15min":1.0589999999999999},"min":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"max":{"1min":1.3879999999999999,"5min":1.3879999999999999,"15min":1.3879999999999999},"last":0.96199999999999997}]},{"osd":2,"last update":"Sat Sep 23 19:04:27 2023","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":0.94999999999999996,"15min":0.96599999999999997},"min":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.363,"5min":1.363,"15min":1.363},"last":0.79400000000000004},{"interface":"front","average":{"1min":0.85399999999999998,"5min":0.93400000000000005,"15min":0.94799999999999995},"min":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":1.218,"5min":1.3129999999999999,"15min":1.3129999999999999},"last":1.0600000000000001}]},{"osd":3,"last update":"Sat Sep 23 19:04:27 2023","interfaces":[{"interface":"back","average":{"1min":0.90700000000000003,"5min":1.0229999999999999,"15min":1.0429999999999999},"min":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":1.0900000000000001,"5min":1.5669999999999999,"15min":1.5669999999999999},"last":1.1459999999999999},{"interface":"front","average":{"1min":0.93100000000000005,"5min":1.006,"15min":1.018},"min":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"max":{"1min":1.446,"5min":1.5940000000000001,"15min":1.5940000000000001},"last":0.871}]},{"osd":4,"last update":"Sat Sep 23 19:04:27 2023","interfaces":[{"interface":"back","average":{"1min":0.78200000000000003,"5min":0.90100000000000002,"15min":0.92100000000000004},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":1.296,"5min":1.296,"15min":1.296},"last":0.85399999999999998},{"interface":"front","average":{"1min":0.89200000000000002,"5min":0.93100000000000005,"15min":0.93700000000000006},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":1.2729999999999999,"5min":1.742,"15min":1.742},"last":1.087}]},{"osd":5,"last update":"Sat Sep 23 19:04:27 2023","interfaces":[{"interface":"back","average":{"1min":0.81699999999999995,"5min":0.96499999999999997,"15min":0.98899999999999999},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":1.153,"5min":1.3720000000000001,"15min":1.3720000000000001},"last":0.93600000000000005},{"interface":"front","average":{"1min":0.94999999999999996,"5min":1.0780000000000001,"15min":1.1000000000000001},"min":{"1min":0.501,"5min":0.501,"15min":0.501},"max":{"1min":1.2150000000000001,"5min":1.7,"15min":1.7},"last":1.2010000000000001}]},{"osd":7,"last update":"Sat Sep 23 19:05:03 2023","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":0.92900000000000005,"15min":0.92200000000000004},"min":{"1min":0.71599999999999997,"5min":0.52600000000000002,"15min":0.52600000000000002},"max":{"1min":1.3640000000000001,"5min":1.474,"15min":1.474},"last":1.129},{"interface":"front","average":{"1min":1.105,"5min":1.0509999999999999,"15min":1.042},"min":{"1min":0.627,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.385,"5min":1.647,"15min":1.647},"last":1.2430000000000001}]}]},{"osd":1,"up_from":13,"seq":55834574930,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:04:59 2023","interfaces":[{"interface":"back","average":{"1min":0.70099999999999996,"5min":0.71199999999999997,"15min":0.62},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":1.1220000000000001,"5min":1.167,"15min":1.167},"last":0.71499999999999997},{"interface":"front","average":{"1min":0.749,"5min":0.77700000000000002,"15min":0.63100000000000001},"min":{"1min":0.434,"5min":0.434,"15min":0.35999999999999999},"max":{"1min":1.202,"5min":1.26,"15min":1.26},"last":0.76500000000000001}]},{"osd":2,"last update":"Sat Sep 23 19:04:40 2023","interfaces":[{"interface":"back","average":{"1min":0.79800000000000004,"5min":0.78400000000000003,"15min":0.70999999999999996},"min":{"1min":0.53400000000000003,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.0669999999999999,"5min":1.0820000000000001,"15min":1.0820000000000001},"last":1.0009999999999999},{"interface":"front","average":{"1min":0.77400000000000002,"5min":0.76400000000000001,"15min":0.69899999999999995},"min":{"1min":0.54700000000000004,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":0.98699999999999999,"5min":1.0800000000000001,"15min":1.0800000000000001},"last":0.84199999999999997}]},{"osd":3,"last update":"Sat Sep 23 19:04:29 2023","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":0.85399999999999998,"15min":0.84399999999999997},"min":{"1min":0.504,"5min":0.504,"15min":0.504},"max":{"1min":1.153,"5min":1.21,"15min":1.21},"last":0.95299999999999996},{"interface":"front","average":{"1min":0.91200000000000003,"5min":0.86399999999999999,"15min":0.83199999999999996},"min":{"1min":0.56799999999999995,"5min":0.56000000000000005,"15min":0.56000000000000005},"max":{"1min":1.1499999999999999,"5min":1.1499999999999999,"15min":1.1499999999999999},"last":1.1499999999999999}]},{"osd":4,"last update":"Sat Sep 23 19:04:59 2023","interfaces":[{"interface":"back","average":{"1min":0.81899999999999995,"5min":0.90400000000000003,"15min":0.91000000000000003},"min":{"1min":0.55600000000000005,"5min":0.54500000000000004,"15min":0.54500000000000004},"max":{"1min":1.25,"5min":1.2849999999999999,"15min":1.2849999999999999},"last":1.081},{"interface":"front","average":{"1min":0.95399999999999996,"5min":1.014,"15min":1.0409999999999999},"min":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"max":{"1min":1.2989999999999999,"5min":1.524,"15min":1.524},"last":1.0529999999999999}]},{"osd":5,"last update":"Sat Sep 23 19:04:44 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":1.0069999999999999,"15min":1.0189999999999999},"min":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"max":{"1min":1.181,"5min":1.2869999999999999,"15min":1.2869999999999999},"last":1.1870000000000001},{"interface":"front","average":{"1min":1.097,"5min":1.101,"15min":1.091},"min":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"max":{"1min":1.379,"5min":1.425,"15min":1.425},"last":1.2430000000000001}]},{"osd":6,"last update":"Sat Sep 23 19:04:24 2023","interfaces":[{"interface":"back","average":{"1min":1.0409999999999999,"5min":1.048,"15min":1.0489999999999999},"min":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"max":{"1min":1.319,"5min":1.46,"15min":1.46},"last":1.264},{"interface":"front","average":{"1min":1.149,"5min":1.1539999999999999,"15min":1.155},"min":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"max":{"1min":1.357,"5min":1.389,"15min":1.389},"last":1.3169999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:05:09 2023","interfaces":[{"interface":"back","average":{"1min":1.008,"5min":1.1180000000000001,"15min":1.1359999999999999},"min":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"max":{"1min":1.288,"5min":1.4690000000000001,"15min":1.4690000000000001},"last":1.3620000000000001},{"interface":"front","average":{"1min":0.98599999999999999,"5min":1.073,"15min":1.087},"min":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"max":{"1min":1.359,"5min":1.359,"15min":1.359},"last":1.3440000000000001}]}]},{"osd":0,"up_from":8,"seq":34359738457,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 19:05:04 2023","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":0.69499999999999995,"15min":0.623},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.97099999999999997,"5min":2.0979999999999999,"15min":2.0979999999999999},"last":0.75900000000000001},{"interface":"front","average":{"1min":0.68400000000000005,"5min":0.72599999999999998,"15min":0.61799999999999999},"min":{"1min":0.47199999999999998,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":1.024,"5min":1.351,"15min":1.351},"last":0.68300000000000005}]},{"osd":2,"last update":"Sat Sep 23 19:04:42 2023","interfaces":[{"interface":"back","average":{"1min":0.84099999999999997,"5min":0.79600000000000004,"15min":0.745},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":1.3009999999999999,"5min":1.546,"15min":1.546},"last":0.79500000000000004},{"interface":"front","average":{"1min":0.84099999999999997,"5min":0.77300000000000002,"15min":0.72499999999999998},"min":{"1min":0.53900000000000003,"5min":0.47799999999999998,"15min":0.47799999999999998},"max":{"1min":1.1579999999999999,"5min":1.1579999999999999,"15min":1.1579999999999999},"last":0.91900000000000004}]},{"osd":3,"last update":"Sat Sep 23 19:05:18 2023","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":0.86899999999999999,"15min":0.91800000000000004},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":1.1599999999999999,"5min":3.1840000000000002,"15min":3.1840000000000002},"last":0.94799999999999995},{"interface":"front","average":{"1min":0.76000000000000001,"5min":0.85599999999999998,"15min":0.81599999999999995},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":1.3220000000000001,"5min":1.5600000000000001,"15min":1.5600000000000001},"last":0.82999999999999996}]},{"osd":4,"last update":"Sat Sep 23 19:05:04 2023","interfaces":[{"interface":"back","average":{"1min":0.98699999999999999,"5min":0.97699999999999998,"15min":0.93300000000000005},"min":{"1min":0.72499999999999998,"5min":0.64300000000000002,"15min":0.64300000000000002},"max":{"1min":1.3180000000000001,"5min":1.373,"15min":1.373},"last":1.0269999999999999},{"interface":"front","average":{"1min":0.95299999999999996,"5min":0.99399999999999999,"15min":0.96499999999999997},"min":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"max":{"1min":1.4530000000000001,"5min":1.4530000000000001,"15min":1.4530000000000001},"last":1.0760000000000001}]},{"osd":5,"last update":"Sat Sep 23 19:04:42 2023","interfaces":[{"interface":"back","average":{"1min":1.093,"5min":1.0940000000000001,"15min":1.095},"min":{"1min":0.83999999999999997,"5min":0.78300000000000003,"15min":0.78300000000000003},"max":{"1min":1.403,"5min":1.409,"15min":1.409},"last":0.98199999999999998},{"interface":"front","average":{"1min":0.98099999999999998,"5min":0.996,"15min":0.996},"min":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"max":{"1min":1.3859999999999999,"5min":1.3859999999999999,"15min":1.3859999999999999},"last":1.1339999999999999}]},{"osd":6,"last update":"Sat Sep 23 19:05:23 2023","interfaces":[{"interface":"back","average":{"1min":0.90300000000000002,"5min":1.0449999999999999,"15min":1.0800000000000001},"min":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"max":{"1min":1.1850000000000001,"5min":1.502,"15min":1.502},"last":1.0469999999999999},{"interface":"front","average":{"1min":1.034,"5min":1.149,"15min":1.179},"min":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"max":{"1min":1.3580000000000001,"5min":1.476,"15min":1.476},"last":1.264}]},{"osd":7,"last update":"Sat Sep 23 19:05:04 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":1.129,"15min":1.133},"min":{"1min":0.629,"5min":0.629,"15min":0.629},"max":{"1min":1.655,"5min":1.655,"15min":1.655},"last":1.24},{"interface":"front","average":{"1min":1.0149999999999999,"5min":1.0780000000000001,"15min":1.0880000000000001},"min":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"max":{"1min":1.595,"5min":1.595,"15min":1.595},"last":1.2}]}]},{"osd":2,"up_from":18,"seq":77309411399,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:04:39 2023","interfaces":[{"interface":"back","average":{"1min":0.84999999999999998,"5min":0.74199999999999999,"15min":0.67300000000000004},"min":{"1min":0.49299999999999999,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":1.8979999999999999,"5min":1.8979999999999999,"15min":1.8979999999999999},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.80700000000000005,"5min":0.78300000000000003,"15min":0.70099999999999996},"min":{"1min":0.55300000000000005,"5min":0.495,"15min":0.495},"max":{"1min":1.651,"5min":1.651,"15min":1.651},"last":0.78200000000000003}]},{"osd":1,"last update":"Sat Sep 23 19:04:39 2023","interfaces":[{"interface":"back","average":{"1min":0.80600000000000005,"5min":0.77100000000000002,"15min":0.72699999999999998},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":1.4490000000000001,"5min":1.4490000000000001,"15min":1.4490000000000001},"last":0.81399999999999995},{"interface":"front","average":{"1min":0.90900000000000003,"5min":0.82899999999999996,"15min":0.746},"min":{"1min":0.63400000000000001,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":1.788,"5min":1.788,"15min":1.788},"last":0.84699999999999998}]},{"osd":3,"last update":"Sat Sep 23 19:05:24 2023","interfaces":[{"interface":"back","average":{"1min":0.72999999999999998,"5min":0.79700000000000004,"15min":0.77000000000000002},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":1.748,"5min":1.748,"15min":1.748},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.80300000000000005,"5min":0.86599999999999999,"15min":0.81000000000000005},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":1.4690000000000001,"5min":1.4690000000000001,"15min":1.4690000000000001},"last":0.73599999999999999}]},{"osd":4,"last update":"Sat Sep 23 19:05:01 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":0.99299999999999999,"15min":0.98799999999999999},"min":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"max":{"1min":1.3129999999999999,"5min":1.335,"15min":1.335},"last":0.98399999999999999},{"interface":"front","average":{"1min":0.93999999999999995,"5min":0.99399999999999999,"15min":0.98399999999999999},"min":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"max":{"1min":1.363,"5min":1.554,"15min":1.554},"last":0.90000000000000002}]},{"osd":5,"last update":"Sat Sep 23 19:04:45 2023","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":1.0620000000000001,"15min":1.0509999999999999},"min":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"max":{"1min":1.494,"5min":1.635,"15min":1.635},"last":1.0429999999999999},{"interface":"front","average":{"1min":1.022,"5min":1.087,"15min":1.095},"min":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"max":{"1min":1.3899999999999999,"5min":1.581,"15min":1.581},"last":0.91400000000000003}]},{"osd":6,"last update":"Sat Sep 23 19:05:24 2023","interfaces":[{"interface":"back","average":{"1min":1.0009999999999999,"5min":1.0900000000000001,"15min":1.1220000000000001},"min":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"max":{"1min":1.5,"5min":1.5,"15min":1.5},"last":0.87},{"interface":"front","average":{"1min":1.0580000000000001,"5min":1.1259999999999999,"15min":1.1459999999999999},"min":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"max":{"1min":1.679,"5min":1.679,"15min":1.679},"last":1.262}]},{"osd":7,"last update":"Sat Sep 23 19:05:06 2023","interfaces":[{"interface":"back","average":{"1min":1.129,"5min":1.165,"15min":1.171},"min":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"max":{"1min":1.5649999999999999,"5min":1.659,"15min":1.659},"last":1.3220000000000001},{"interface":"front","average":{"1min":1.0589999999999999,"5min":1.1839999999999999,"15min":1.2050000000000001},"min":{"1min":0.621,"5min":0.621,"15min":0.621},"max":{"1min":1.5760000000000001,"5min":1.6890000000000001,"15min":1.6890000000000001},"last":0.96199999999999997}]}]},{"osd":3,"up_from":25,"seq":107374182465,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:05:25 2023","interfaces":[{"interface":"back","average":{"1min":0.70099999999999996,"5min":0.68999999999999995,"15min":0.66500000000000004},"min":{"1min":0.54900000000000004,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":0.878,"5min":1.286,"15min":1.286},"last":0.74199999999999999},{"interface":"front","average":{"1min":0.69499999999999995,"5min":0.751,"15min":0.75800000000000001},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.95499999999999996,"5min":1.1120000000000001,"15min":1.1120000000000001},"last":0.70899999999999996}]},{"osd":1,"last update":"Sat Sep 23 19:05:25 2023","interfaces":[{"interface":"back","average":{"1min":0.751,"5min":0.78900000000000003,"15min":0.78600000000000003},"min":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":0.95899999999999996,"5min":1.119,"15min":1.119},"last":0.79400000000000004},{"interface":"front","average":{"1min":0.69699999999999995,"5min":0.749,"15min":0.75700000000000001},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.94599999999999995,"5min":1.3180000000000001,"15min":1.3180000000000001},"last":0.77600000000000002}]},{"osd":2,"last update":"Sat Sep 23 19:05:25 2023","interfaces":[{"interface":"back","average":{"1min":0.80200000000000005,"5min":0.85799999999999998,"15min":0.86699999999999999},"min":{"1min":0.59699999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":1.1930000000000001,"5min":1.4339999999999999,"15min":1.4339999999999999},"last":0.90000000000000002},{"interface":"front","average":{"1min":0.80800000000000005,"5min":0.84399999999999997,"15min":0.82499999999999996},"min":{"1min":0.55500000000000005,"5min":0.505,"15min":0.505},"max":{"1min":1.284,"5min":1.3700000000000001,"15min":1.3700000000000001},"last":0.82599999999999996}]},{"osd":4,"last update":"Sat Sep 23 19:05:00 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":0.98799999999999999,"15min":0.98799999999999999},"min":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"max":{"1min":1.3160000000000001,"5min":1.7250000000000001,"15min":1.7250000000000001},"last":0.97099999999999997},{"interface":"front","average":{"1min":0.95799999999999996,"5min":0.98199999999999998,"15min":0.99099999999999999},"min":{"1min":0.63,"5min":0.63,"15min":0.63},"max":{"1min":1.413,"5min":1.413,"15min":1.413},"last":1.212}]},{"osd":5,"last update":"Sat Sep 23 19:04:49 2023","interfaces":[{"interface":"back","average":{"1min":1,"5min":1.0629999999999999,"15min":1.0669999999999999},"min":{"1min":0.74399999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"max":{"1min":1.256,"5min":1.9319999999999999,"15min":1.9319999999999999},"last":1.018},{"interface":"front","average":{"1min":0.99199999999999999,"5min":1.0369999999999999,"15min":1.028},"min":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"max":{"1min":1.2370000000000001,"5min":1.417,"15min":1.417},"last":1.2649999999999999}]},{"osd":6,"last update":"Sat Sep 23 19:05:25 2023","interfaces":[{"interface":"back","average":{"1min":1.0669999999999999,"5min":1.0960000000000001,"15min":1.1220000000000001},"min":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"max":{"1min":1.363,"5min":1.4179999999999999,"15min":1.4179999999999999},"last":0.99399999999999999},{"interface":"front","average":{"1min":1.0980000000000001,"5min":1.21,"15min":1.256},"min":{"1min":0.84999999999999998,"5min":0.78100000000000003,"15min":0.78100000000000003},"max":{"1min":1.3680000000000001,"5min":2.1960000000000002,"15min":2.1960000000000002},"last":1.2370000000000001}]},{"osd":7,"last update":"Sat Sep 23 19:05:07 2023","interfaces":[{"interface":"back","average":{"1min":1.127,"5min":1.1100000000000001,"15min":1.107},"min":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"max":{"1min":1.486,"5min":1.486,"15min":1.486},"last":1.3089999999999999},{"interface":"front","average":{"1min":1.0900000000000001,"5min":1.139,"15min":1.147},"min":{"1min":0.72499999999999998,"5min":0.70499999999999996,"15min":0.70499999999999996},"max":{"1min":1.395,"5min":1.516,"15min":1.516},"last":1.341}]}]},{"osd":4,"up_from":30,"seq":128849018936,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:05:07 2023","interfaces":[{"interface":"back","average":{"1min":0.92900000000000005,"5min":0.97099999999999997,"15min":0.96499999999999997},"min":{"1min":0.57699999999999996,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":1.2709999999999999,"5min":1.343,"15min":1.343},"last":0.86399999999999999},{"interface":"front","average":{"1min":0.92700000000000005,"5min":0.95599999999999996,"15min":0.96999999999999997},"min":{"1min":0.55200000000000005,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.2450000000000001,"5min":1.2769999999999999,"15min":1.2769999999999999},"last":0.92700000000000005}]},{"osd":1,"last update":"Sat Sep 23 19:05:07 2023","interfaces":[{"interface":"back","average":{"1min":0.871,"5min":0.85199999999999998,"15min":0.85899999999999999},"min":{"1min":0.60899999999999999,"5min":0.58799999999999997,"15min":0.58799999999999997},"max":{"1min":1.0780000000000001,"5min":1.23,"15min":1.23},"last":0.81999999999999995},{"interface":"front","average":{"1min":0.86799999999999999,"5min":0.90200000000000002,"15min":0.89000000000000001},"min":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"max":{"1min":1.4099999999999999,"5min":1.4099999999999999,"15min":1.4099999999999999},"last":1.1339999999999999}]},{"osd":2,"last update":"Sat Sep 23 19:05:07 2023","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":0.874,"15min":0.85799999999999998},"min":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.343,"5min":1.343,"15min":1.343},"last":1.0900000000000001},{"interface":"front","average":{"1min":0.82999999999999996,"5min":0.874,"15min":0.87},"min":{"1min":0.626,"5min":0.59499999999999997,"15min":0.59499999999999997},"max":{"1min":1.1180000000000001,"5min":1.2869999999999999,"15min":1.2869999999999999},"last":1.113}]},{"osd":3,"last update":"Sat Sep 23 19:05:07 2023","interfaces":[{"interface":"back","average":{"1min":0.86399999999999999,"5min":0.89200000000000002,"15min":0.90500000000000003},"min":{"1min":0.61499999999999999,"5min":0.56200000000000006,"15min":0.56200000000000006},"max":{"1min":1.0589999999999999,"5min":1.2589999999999999,"15min":1.2589999999999999},"last":0.66500000000000004},{"interface":"front","average":{"1min":0.88400000000000001,"5min":0.89600000000000002,"15min":0.88600000000000001},"min":{"1min":0.65200000000000002,"5min":0.60199999999999998,"15min":0.60199999999999998},"max":{"1min":1.1220000000000001,"5min":1.335,"15min":1.335},"last":1.054}]},{"osd":5,"last update":"Sat Sep 23 19:04:48 2023","interfaces":[{"interface":"back","average":{"1min":0.81200000000000006,"5min":0.78700000000000003,"15min":0.753},"min":{"1min":0.49199999999999999,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":1.1930000000000001,"5min":1.373,"15min":1.373},"last":1.0009999999999999},{"interface":"front","average":{"1min":0.85199999999999998,"5min":0.88200000000000001,"15min":0.878},"min":{"1min":0.39500000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":1.2,"5min":1.407,"15min":1.407},"last":1.0349999999999999}]},{"osd":6,"last update":"Sat Sep 23 19:05:22 2023","interfaces":[{"interface":"back","average":{"1min":0.93700000000000006,"5min":0.94999999999999996,"15min":0.95599999999999996},"min":{"1min":0.59499999999999997,"5min":0.58099999999999996,"15min":0.58099999999999996},"max":{"1min":1.23,"5min":1.4430000000000001,"15min":1.4430000000000001},"last":1.2190000000000001},{"interface":"front","average":{"1min":0.97599999999999998,"5min":0.95299999999999996,"15min":0.94999999999999996},"min":{"1min":0.46999999999999997,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":1.294,"5min":1.492,"15min":1.492},"last":0.99199999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:05:07 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":0.94199999999999995,"15min":0.94799999999999995},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":1.1830000000000001,"5min":1.327,"15min":1.327},"last":0.96499999999999997},{"interface":"front","average":{"1min":1.0069999999999999,"5min":1.0129999999999999,"15min":1.014},"min":{"1min":0.54000000000000004,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":1.4890000000000001,"5min":1.51,"15min":1.51},"last":1.179}]}]},{"osd":5,"up_from":36,"seq":154618822703,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297616,"kb_used_data":520,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447536,"statfs":{"total":95995035648,"available":95690276864,"internally_reserved":0,"allocated":532480,"data_stored":219885,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:04:41 2023","interfaces":[{"interface":"back","average":{"1min":0.88,"5min":0.90700000000000003,"15min":0.91700000000000004},"min":{"1min":0.66400000000000003,"5min":0.64500000000000002,"15min":0.64500000000000002},"max":{"1min":1.2110000000000001,"5min":1.2110000000000001,"15min":1.2110000000000001},"last":0.88900000000000001},{"interface":"front","average":{"1min":0.85599999999999998,"5min":0.86899999999999999,"15min":0.873},"min":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"max":{"1min":1.2410000000000001,"5min":1.4710000000000001,"15min":1.4710000000000001},"last":1.109}]},{"osd":1,"last update":"Sat Sep 23 19:04:41 2023","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":0.91100000000000003,"15min":0.90500000000000003},"min":{"1min":0.69099999999999995,"5min":0.60599999999999998,"15min":0.60599999999999998},"max":{"1min":1.413,"5min":1.413,"15min":1.413},"last":0.83599999999999997},{"interface":"front","average":{"1min":0.80500000000000005,"5min":0.86599999999999999,"15min":0.87},"min":{"1min":0.5,"5min":0.5,"15min":0.5},"max":{"1min":1.3160000000000001,"5min":1.3160000000000001,"15min":1.3160000000000001},"last":1.266}]},{"osd":2,"last update":"Sat Sep 23 19:04:41 2023","interfaces":[{"interface":"back","average":{"1min":0.871,"5min":0.88400000000000001,"15min":0.88500000000000001},"min":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"max":{"1min":1.0940000000000001,"5min":1.4399999999999999,"15min":1.4399999999999999},"last":0.69699999999999995},{"interface":"front","average":{"1min":0.93999999999999995,"5min":0.96699999999999997,"15min":0.96999999999999997},"min":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"max":{"1min":1.3779999999999999,"5min":1.3779999999999999,"15min":1.3779999999999999},"last":1.042}]},{"osd":3,"last update":"Sat Sep 23 19:04:41 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":0.89100000000000001,"15min":0.88900000000000001},"min":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"max":{"1min":1.1020000000000001,"5min":1.3,"15min":1.3},"last":0.81399999999999995},{"interface":"front","average":{"1min":1.008,"5min":1,"15min":1.0049999999999999},"min":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"max":{"1min":1.3360000000000001,"5min":1.3360000000000001,"15min":1.3360000000000001},"last":0.98599999999999999}]},{"osd":4,"last update":"Sat Sep 23 19:04:41 2023","interfaces":[{"interface":"back","average":{"1min":0.80800000000000005,"5min":0.80700000000000005,"15min":0.80600000000000005},"min":{"1min":0.51400000000000001,"5min":0.504,"15min":0.504},"max":{"1min":1.655,"5min":1.655,"15min":1.655},"last":1.3600000000000001},{"interface":"front","average":{"1min":0.92500000000000004,"5min":0.87,"15min":0.85799999999999998},"min":{"1min":0.57799999999999996,"5min":0.57299999999999995,"15min":0.57299999999999995},"max":{"1min":2.415,"5min":2.415,"15min":2.415},"last":1.2470000000000001}]},{"osd":6,"last update":"Sat Sep 23 19:04:23 2023","interfaces":[{"interface":"back","average":{"1min":1.044,"5min":0.95199999999999996,"15min":0.93600000000000005},"min":{"1min":0.58999999999999997,"5min":0.48999999999999999,"15min":0.48999999999999999},"max":{"1min":2.0699999999999998,"5min":2.0699999999999998,"15min":2.0699999999999998},"last":1.4530000000000001},{"interface":"front","average":{"1min":1.103,"5min":1.0329999999999999,"15min":1.0209999999999999},"min":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"max":{"1min":2.4769999999999999,"5min":2.4769999999999999,"15min":2.4769999999999999},"last":1.508}]},{"osd":7,"last update":"Sat Sep 23 19:05:06 2023","interfaces":[{"interface":"back","average":{"1min":1.1060000000000001,"5min":1.038,"15min":1.0269999999999999},"min":{"1min":0.68400000000000005,"5min":0.63800000000000001,"15min":0.63800000000000001},"max":{"1min":2.5049999999999999,"5min":2.5049999999999999,"15min":2.5049999999999999},"last":1.595},{"interface":"front","average":{"1min":1.181,"5min":1.0649999999999999,"15min":1.0449999999999999},"min":{"1min":0.65400000000000003,"5min":0.54500000000000004,"15min":0.54500000000000004},"max":{"1min":2.4340000000000002,"5min":2.4340000000000002,"15min":2.4340000000000002},"last":1.399}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T19:05:30.417 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T19:05:31.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:31 smithi016 bash[27237]: audit 2023-09-23T19:05:29.736285+0000 mgr.y (mgr.14454) 85 : audit [DBG] from='client.24530 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:05:31.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:31 smithi016 bash[33624]: audit 2023-09-23T19:05:29.736285+0000 mgr.y (mgr.14454) 85 : audit [DBG] from='client.24530 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:05:31.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:31 smithi092 bash[27556]: audit 2023-09-23T19:05:29.736285+0000 mgr.y (mgr.14454) 85 : audit [DBG] from='client.24530 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:05:32.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:32 smithi016 bash[27237]: cluster 2023-09-23T19:05:30.900034+0000 mgr.y (mgr.14454) 86 : cluster [DBG] pgmap v65: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:32.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:32 smithi016 bash[33624]: cluster 2023-09-23T19:05:30.900034+0000 mgr.y (mgr.14454) 86 : cluster [DBG] pgmap v65: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:32.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:32 smithi092 bash[27556]: cluster 2023-09-23T19:05:30.900034+0000 mgr.y (mgr.14454) 86 : cluster [DBG] pgmap v65: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:34.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:34 smithi016 bash[27237]: cluster 2023-09-23T19:05:32.900777+0000 mgr.y (mgr.14454) 87 : cluster [DBG] pgmap v66: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:34.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:34 smithi016 bash[33624]: cluster 2023-09-23T19:05:32.900777+0000 mgr.y (mgr.14454) 87 : cluster [DBG] pgmap v66: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:34.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:34 smithi092 bash[27556]: cluster 2023-09-23T19:05:32.900777+0000 mgr.y (mgr.14454) 87 : cluster [DBG] pgmap v66: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:35.193 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:05:36.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:36 smithi016 bash[33624]: cluster 2023-09-23T19:05:34.901334+0000 mgr.y (mgr.14454) 88 : cluster [DBG] pgmap v67: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:36.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:36 smithi016 bash[27237]: cluster 2023-09-23T19:05:34.901334+0000 mgr.y (mgr.14454) 88 : cluster [DBG] pgmap v67: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:36.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:36 smithi092 bash[27556]: cluster 2023-09-23T19:05:34.901334+0000 mgr.y (mgr.14454) 88 : cluster [DBG] pgmap v67: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:38.099 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-09-23T19:05:38.101 INFO:teuthology.orchestra.run.smithi016.stderr:dumped all 2023-09-23T19:05:38.299 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:38 smithi016 bash[33624]: cluster 2023-09-23T19:05:36.901897+0000 mgr.y (mgr.14454) 89 : cluster [DBG] pgmap v68: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:38.299 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:38 smithi016 bash[27237]: cluster 2023-09-23T19:05:36.901897+0000 mgr.y (mgr.14454) 89 : cluster [DBG] pgmap v68: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:38.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:38 smithi092 bash[27556]: cluster 2023-09-23T19:05:36.901897+0000 mgr.y (mgr.14454) 89 : cluster [DBG] pgmap v68: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:38.802 INFO:teuthology.orchestra.run.smithi016.stdout:{"pg_ready":true,"pg_map":{"version":68,"stamp":"2023-09-23T19:05:36.901582+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2380980,"kb_used_data":4212,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":747580236,"statfs":{"total":767960285184,"available":765522161664,"internally_reserved":0,"allocated":4313088,"data_stored":1763175,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.000000"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-09-23T19:04:03.609606+0000","last_change":"2023-09-23T19:04:02.507717+0000","last_active":"2023-09-23T19:04:03.609606+0000","last_peered":"2023-09-23T19:04:03.609606+0000","last_clean":"2023-09-23T19:04:03.609606+0000","last_became_active":"2023-09-23T19:04:02.507042+0000","last_became_peered":"2023-09-23T19:04:02.507042+0000","last_unstale":"2023-09-23T19:04:03.609606+0000","last_undegraded":"2023-09-23T19:04:03.609606+0000","last_fullsized":"2023-09-23T19:04:03.609606+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-09-23T19:04:03.603569+0000","last_change":"2023-09-23T19:04:02.707894+0000","last_active":"2023-09-23T19:04:03.603569+0000","last_peered":"2023-09-23T19:04:03.603569+0000","last_clean":"2023-09-23T19:04:03.603569+0000","last_became_active":"2023-09-23T19:04:02.707071+0000","last_became_peered":"2023-09-23T19:04:02.707071+0000","last_unstale":"2023-09-23T19:04:03.603569+0000","last_undegraded":"2023-09-23T19:04:03.603569+0000","last_fullsized":"2023-09-23T19:04:03.603569+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-09-23T19:04:03.610292+0000","last_change":"2023-09-23T19:04:02.713482+0000","last_active":"2023-09-23T19:04:03.610292+0000","last_peered":"2023-09-23T19:04:03.610292+0000","last_clean":"2023-09-23T19:04:03.610292+0000","last_became_active":"2023-09-23T19:04:02.712471+0000","last_became_peered":"2023-09-23T19:04:02.712471+0000","last_unstale":"2023-09-23T19:04:03.610292+0000","last_undegraded":"2023-09-23T19:04:03.610292+0000","last_fullsized":"2023-09-23T19:04:03.610292+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-09-23T19:04:03.610042+0000","last_change":"2023-09-23T19:04:02.712803+0000","last_active":"2023-09-23T19:04:03.610042+0000","last_peered":"2023-09-23T19:04:03.610042+0000","last_clean":"2023-09-23T19:04:03.610042+0000","last_became_active":"2023-09-23T19:04:02.711952+0000","last_became_peered":"2023-09-23T19:04:02.711952+0000","last_unstale":"2023-09-23T19:04:03.610042+0000","last_undegraded":"2023-09-23T19:04:03.610042+0000","last_fullsized":"2023-09-23T19:04:03.610042+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":15,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-09-23T19:04:03.610239+0000","last_change":"2023-09-23T19:03:04.495032+0000","last_active":"2023-09-23T19:04:03.610239+0000","last_peered":"2023-09-23T19:04:03.610239+0000","last_clean":"2023-09-23T19:04:03.610239+0000","last_became_active":"2023-09-23T19:03:04.494463+0000","last_became_peered":"2023-09-23T19:03:04.494463+0000","last_unstale":"2023-09-23T19:04:03.610239+0000","last_undegraded":"2023-09-23T19:04:03.610239+0000","last_fullsized":"2023-09-23T19:04:03.610239+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:59:33.840751+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:59:33.840751+0000","last_clean_scrub_stamp":"2023-09-23T18:59:33.840751+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-09-23T19:04:03.610045+0000","last_change":"2023-09-23T19:04:02.509889+0000","last_active":"2023-09-23T19:04:03.610045+0000","last_peered":"2023-09-23T19:04:03.610045+0000","last_clean":"2023-09-23T19:04:03.610045+0000","last_became_active":"2023-09-23T19:04:02.509458+0000","last_became_peered":"2023-09-23T19:04:02.509458+0000","last_unstale":"2023-09-23T19:04:03.610045+0000","last_undegraded":"2023-09-23T19:04:03.610045+0000","last_fullsized":"2023-09-23T19:04:03.610045+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-09-23T19:04:03.505472+0000","last_change":"2023-09-23T19:04:02.707164+0000","last_active":"2023-09-23T19:04:03.505472+0000","last_peered":"2023-09-23T19:04:03.505472+0000","last_clean":"2023-09-23T19:04:03.505472+0000","last_became_active":"2023-09-23T19:04:02.706747+0000","last_became_peered":"2023-09-23T19:04:02.706747+0000","last_unstale":"2023-09-23T19:04:03.505472+0000","last_undegraded":"2023-09-23T19:04:03.505472+0000","last_fullsized":"2023-09-23T19:04:03.505472+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-09-23T19:04:03.610031+0000","last_change":"2023-09-23T19:04:02.509914+0000","last_active":"2023-09-23T19:04:03.610031+0000","last_peered":"2023-09-23T19:04:03.610031+0000","last_clean":"2023-09-23T19:04:03.610031+0000","last_became_active":"2023-09-23T19:04:02.509532+0000","last_became_peered":"2023-09-23T19:04:02.509532+0000","last_unstale":"2023-09-23T19:04:03.610031+0000","last_undegraded":"2023-09-23T19:04:03.610031+0000","last_fullsized":"2023-09-23T19:04:03.610031+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-09-23T19:04:03.603476+0000","last_change":"2023-09-23T19:04:02.707302+0000","last_active":"2023-09-23T19:04:03.603476+0000","last_peered":"2023-09-23T19:04:03.603476+0000","last_clean":"2023-09-23T19:04:03.603476+0000","last_became_active":"2023-09-23T19:04:02.706904+0000","last_became_peered":"2023-09-23T19:04:02.706904+0000","last_unstale":"2023-09-23T19:04:03.603476+0000","last_undegraded":"2023-09-23T19:04:03.603476+0000","last_fullsized":"2023-09-23T19:04:03.603476+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430241,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:05:07 2023","interfaces":[{"interface":"back","average":{"1min":0.84399999999999997,"5min":0.91800000000000004,"15min":0.93000000000000005},"min":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":1.2729999999999999,"5min":1.4019999999999999,"15min":1.4019999999999999},"last":0.90400000000000003},{"interface":"front","average":{"1min":0.89300000000000002,"5min":0.90300000000000002,"15min":0.90500000000000003},"min":{"1min":0.58899999999999997,"5min":0.57699999999999996,"15min":0.57699999999999996},"max":{"1min":1.1000000000000001,"5min":1.552,"15min":1.552},"last":0.749}]},{"osd":1,"last update":"Sat Sep 23 19:05:07 2023","interfaces":[{"interface":"back","average":{"1min":0.84099999999999997,"5min":0.89500000000000002,"15min":0.90400000000000003},"min":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.224,"5min":1.224,"15min":1.224},"last":0.96599999999999997},{"interface":"front","average":{"1min":0.96299999999999997,"5min":0.96799999999999997,"15min":0.96899999999999997},"min":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"max":{"1min":1.323,"5min":1.498,"15min":1.498},"last":0.94299999999999995}]},{"osd":2,"last update":"Sat Sep 23 19:05:07 2023","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":0.90700000000000003,"15min":0.90400000000000003},"min":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"max":{"1min":1.129,"5min":1.2010000000000001,"15min":1.2010000000000001},"last":0.98999999999999999},{"interface":"front","average":{"1min":0.90100000000000002,"5min":0.96499999999999997,"15min":0.97599999999999998},"min":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.202,"5min":1.417,"15min":1.417},"last":1.056}]},{"osd":3,"last update":"Sat Sep 23 19:05:07 2023","interfaces":[{"interface":"back","average":{"1min":0.94799999999999995,"5min":1.0449999999999999,"15min":1.0609999999999999},"min":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"max":{"1min":1.2310000000000001,"5min":1.4450000000000001,"15min":1.4450000000000001},"last":0.76900000000000002},{"interface":"front","average":{"1min":0.93100000000000005,"5min":1.089,"15min":1.115},"min":{"1min":0.59799999999999998,"5min":0.58999999999999997,"15min":0.58999999999999997},"max":{"1min":1.2989999999999999,"5min":3.3780000000000001,"15min":3.3780000000000001},"last":0.86499999999999999}]},{"osd":4,"last update":"Sat Sep 23 19:05:07 2023","interfaces":[{"interface":"back","average":{"1min":0.90700000000000003,"5min":0.88300000000000001,"15min":0.879},"min":{"1min":0.53000000000000003,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":1.2689999999999999,"5min":1.4139999999999999,"15min":1.4139999999999999},"last":0.82899999999999996},{"interface":"front","average":{"1min":0.97499999999999998,"5min":0.95899999999999996,"15min":0.95699999999999996},"min":{"1min":0.56699999999999995,"5min":0.54500000000000004,"15min":0.54500000000000004},"max":{"1min":1.448,"5min":1.5720000000000001,"15min":1.5720000000000001},"last":0.68000000000000005}]},{"osd":5,"last update":"Sat Sep 23 19:05:07 2023","interfaces":[{"interface":"back","average":{"1min":0.95399999999999996,"5min":0.95299999999999996,"15min":0.95299999999999996},"min":{"1min":0.51400000000000001,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":1.2430000000000001,"5min":1.472,"15min":1.472},"last":1.1060000000000001},{"interface":"front","average":{"1min":1.0309999999999999,"5min":1.0309999999999999,"15min":1.0309999999999999},"min":{"1min":0.72399999999999998,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.343,"5min":1.343,"15min":1.343},"last":0.60199999999999998}]},{"osd":6,"last update":"Sat Sep 23 19:05:07 2023","interfaces":[{"interface":"back","average":{"1min":1.141,"5min":1.0640000000000001,"15min":1.0509999999999999},"min":{"1min":0.83599999999999997,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":2.0339999999999998,"5min":2.0339999999999998,"15min":2.0339999999999998},"last":1.016},{"interface":"front","average":{"1min":1.1339999999999999,"5min":1.0900000000000001,"15min":1.083},"min":{"1min":0.76300000000000001,"5min":0.53000000000000003,"15min":0.53000000000000003},"max":{"1min":1.472,"5min":1.4730000000000001,"15min":1.4730000000000001},"last":1.0349999999999999}]}]},{"osd":6,"up_from":42,"seq":180388626473,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297620,"kb_used_data":524,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447532,"statfs":{"total":95995035648,"available":95690272768,"internally_reserved":0,"allocated":536576,"data_stored":220200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:05:28 2023","interfaces":[{"interface":"back","average":{"1min":0.75,"5min":0.875,"15min":0.91100000000000003},"min":{"1min":0.51900000000000002,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":1.149,"5min":1.4179999999999999,"15min":1.4179999999999999},"last":1.107},{"interface":"front","average":{"1min":0.82099999999999995,"5min":0.873,"15min":0.90400000000000003},"min":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.111,"5min":1.111,"15min":1.111},"last":0.83399999999999996}]},{"osd":1,"last update":"Sat Sep 23 19:05:28 2023","interfaces":[{"interface":"back","average":{"1min":0.88100000000000001,"5min":1.04,"15min":1.0860000000000001},"min":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":1.2190000000000001,"5min":1.4379999999999999,"15min":1.4379999999999999},"last":1.0589999999999999},{"interface":"front","average":{"1min":0.92100000000000004,"5min":1.018,"15min":1.05},"min":{"1min":0.65200000000000002,"5min":0.50900000000000001,"15min":0.50900000000000001},"max":{"1min":1.1479999999999999,"5min":1.3879999999999999,"15min":1.3879999999999999},"last":1.0760000000000001}]},{"osd":2,"last update":"Sat Sep 23 19:05:28 2023","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":0.92400000000000004,"15min":0.95699999999999996},"min":{"1min":0.56399999999999995,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.133,"5min":1.363,"15min":1.363},"last":0.99399999999999999},{"interface":"front","average":{"1min":0.80700000000000005,"5min":0.90500000000000003,"15min":0.93799999999999994},"min":{"1min":0.58299999999999996,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":1.0600000000000001,"5min":1.3129999999999999,"15min":1.3129999999999999},"last":1.228}]},{"osd":3,"last update":"Sat Sep 23 19:05:28 2023","interfaces":[{"interface":"back","average":{"1min":0.91500000000000004,"5min":0.996,"15min":1.034},"min":{"1min":0.63,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":1.2689999999999999,"5min":1.5669999999999999,"15min":1.5669999999999999},"last":0.93700000000000006},{"interface":"front","average":{"1min":0.93899999999999995,"5min":0.98899999999999999,"15min":1.0129999999999999},"min":{"1min":0.65400000000000003,"5min":0.56299999999999994,"15min":0.56299999999999994},"max":{"1min":1.208,"5min":1.5940000000000001,"15min":1.5940000000000001},"last":1.2070000000000001}]},{"osd":4,"last update":"Sat Sep 23 19:05:28 2023","interfaces":[{"interface":"back","average":{"1min":0.85799999999999998,"5min":0.88600000000000001,"15min":0.91600000000000004},"min":{"1min":0.52700000000000002,"5min":0.443,"15min":0.443},"max":{"1min":1.302,"5min":1.302,"15min":1.302},"last":1.0129999999999999},{"interface":"front","average":{"1min":0.96699999999999997,"5min":0.93600000000000005,"15min":0.93899999999999995},"min":{"1min":0.46200000000000002,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":1.2629999999999999,"5min":1.742,"15min":1.742},"last":1.25}]},{"osd":5,"last update":"Sat Sep 23 19:05:28 2023","interfaces":[{"interface":"back","average":{"1min":0.89400000000000002,"5min":0.94299999999999995,"15min":0.98199999999999998},"min":{"1min":0.68700000000000006,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":1.083,"5min":1.3720000000000001,"15min":1.3720000000000001},"last":1.1879999999999999},{"interface":"front","average":{"1min":1.0169999999999999,"5min":1.0600000000000001,"15min":1.0940000000000001},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":1.4019999999999999,"5min":1.7,"15min":1.7},"last":1.302}]},{"osd":7,"last update":"Sat Sep 23 19:05:03 2023","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":0.92900000000000005,"15min":0.92200000000000004},"min":{"1min":0.71599999999999997,"5min":0.52600000000000002,"15min":0.52600000000000002},"max":{"1min":1.3640000000000001,"5min":1.474,"15min":1.474},"last":1.1319999999999999},{"interface":"front","average":{"1min":1.105,"5min":1.0509999999999999,"15min":1.042},"min":{"1min":0.627,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.385,"5min":1.647,"15min":1.647},"last":1.161}]}]},{"osd":1,"up_from":13,"seq":55834574932,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:04:59 2023","interfaces":[{"interface":"back","average":{"1min":0.70099999999999996,"5min":0.71199999999999997,"15min":0.62},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":1.1220000000000001,"5min":1.167,"15min":1.167},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.749,"5min":0.77700000000000002,"15min":0.63100000000000001},"min":{"1min":0.434,"5min":0.434,"15min":0.35999999999999999},"max":{"1min":1.202,"5min":1.26,"15min":1.26},"last":0.86199999999999999}]},{"osd":2,"last update":"Sat Sep 23 19:04:40 2023","interfaces":[{"interface":"back","average":{"1min":0.79800000000000004,"5min":0.78400000000000003,"15min":0.70999999999999996},"min":{"1min":0.53400000000000003,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.0669999999999999,"5min":1.0820000000000001,"15min":1.0820000000000001},"last":0.92200000000000004},{"interface":"front","average":{"1min":0.77400000000000002,"5min":0.76400000000000001,"15min":0.69899999999999995},"min":{"1min":0.54700000000000004,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":0.98699999999999999,"5min":1.0800000000000001,"15min":1.0800000000000001},"last":0.76400000000000001}]},{"osd":3,"last update":"Sat Sep 23 19:05:34 2023","interfaces":[{"interface":"back","average":{"1min":0.74099999999999999,"5min":0.83499999999999996,"15min":0.83699999999999997},"min":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.0289999999999999,"5min":1.21,"15min":1.21},"last":0.70099999999999996},{"interface":"front","average":{"1min":0.73799999999999999,"5min":0.84899999999999998,"15min":0.82699999999999996},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":1.1499999999999999,"5min":1.1499999999999999,"15min":1.1499999999999999},"last":0.98099999999999998}]},{"osd":4,"last update":"Sat Sep 23 19:04:59 2023","interfaces":[{"interface":"back","average":{"1min":0.81899999999999995,"5min":0.90400000000000003,"15min":0.91000000000000003},"min":{"1min":0.55600000000000005,"5min":0.54500000000000004,"15min":0.54500000000000004},"max":{"1min":1.25,"5min":1.2849999999999999,"15min":1.2849999999999999},"last":0.80700000000000005},{"interface":"front","average":{"1min":0.95399999999999996,"5min":1.014,"15min":1.0409999999999999},"min":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"max":{"1min":1.2989999999999999,"5min":1.524,"15min":1.524},"last":1.093}]},{"osd":5,"last update":"Sat Sep 23 19:04:44 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":1.0069999999999999,"15min":1.0189999999999999},"min":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"max":{"1min":1.181,"5min":1.2869999999999999,"15min":1.2869999999999999},"last":0.89000000000000001},{"interface":"front","average":{"1min":1.097,"5min":1.101,"15min":1.091},"min":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"max":{"1min":1.379,"5min":1.425,"15min":1.425},"last":0.96699999999999997}]},{"osd":6,"last update":"Sat Sep 23 19:05:28 2023","interfaces":[{"interface":"back","average":{"1min":0.95599999999999996,"5min":1.0289999999999999,"15min":1.0429999999999999},"min":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"max":{"1min":1.3240000000000001,"5min":1.46,"15min":1.46},"last":1.0489999999999999},{"interface":"front","average":{"1min":1.0509999999999999,"5min":1.133,"15min":1.1479999999999999},"min":{"1min":0.621,"5min":0.621,"15min":0.621},"max":{"1min":1.3169999999999999,"5min":1.389,"15min":1.389},"last":1.216}]},{"osd":7,"last update":"Sat Sep 23 19:05:09 2023","interfaces":[{"interface":"back","average":{"1min":1.008,"5min":1.1180000000000001,"15min":1.1359999999999999},"min":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"max":{"1min":1.288,"5min":1.4690000000000001,"15min":1.4690000000000001},"last":1.069},{"interface":"front","average":{"1min":0.98599999999999999,"5min":1.073,"15min":1.087},"min":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"max":{"1min":1.359,"5min":1.359,"15min":1.359},"last":1.1879999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738459,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 19:05:04 2023","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":0.69499999999999995,"15min":0.623},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.97099999999999997,"5min":2.0979999999999999,"15min":2.0979999999999999},"last":0.66500000000000004},{"interface":"front","average":{"1min":0.68400000000000005,"5min":0.72599999999999998,"15min":0.61799999999999999},"min":{"1min":0.47199999999999998,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":1.024,"5min":1.351,"15min":1.351},"last":0.70599999999999996}]},{"osd":2,"last update":"Sat Sep 23 19:04:42 2023","interfaces":[{"interface":"back","average":{"1min":0.84099999999999997,"5min":0.79600000000000004,"15min":0.745},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":1.3009999999999999,"5min":1.546,"15min":1.546},"last":0.90100000000000002},{"interface":"front","average":{"1min":0.84099999999999997,"5min":0.77300000000000002,"15min":0.72499999999999998},"min":{"1min":0.53900000000000003,"5min":0.47799999999999998,"15min":0.47799999999999998},"max":{"1min":1.1579999999999999,"5min":1.1579999999999999,"15min":1.1579999999999999},"last":0.82799999999999996}]},{"osd":3,"last update":"Sat Sep 23 19:05:18 2023","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":0.86899999999999999,"15min":0.91800000000000004},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":1.1599999999999999,"5min":3.1840000000000002,"15min":3.1840000000000002},"last":0.99299999999999999},{"interface":"front","average":{"1min":0.76000000000000001,"5min":0.85599999999999998,"15min":0.81599999999999995},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":1.3220000000000001,"5min":1.5600000000000001,"15min":1.5600000000000001},"last":0.94799999999999995}]},{"osd":4,"last update":"Sat Sep 23 19:05:04 2023","interfaces":[{"interface":"back","average":{"1min":0.98699999999999999,"5min":0.97699999999999998,"15min":0.93300000000000005},"min":{"1min":0.72499999999999998,"5min":0.64300000000000002,"15min":0.64300000000000002},"max":{"1min":1.3180000000000001,"5min":1.373,"15min":1.373},"last":1.131},{"interface":"front","average":{"1min":0.95299999999999996,"5min":0.99399999999999999,"15min":0.96499999999999997},"min":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"max":{"1min":1.4530000000000001,"5min":1.4530000000000001,"15min":1.4530000000000001},"last":0.88700000000000001}]},{"osd":5,"last update":"Sat Sep 23 19:04:42 2023","interfaces":[{"interface":"back","average":{"1min":1.093,"5min":1.0940000000000001,"15min":1.095},"min":{"1min":0.83999999999999997,"5min":0.78300000000000003,"15min":0.78300000000000003},"max":{"1min":1.403,"5min":1.409,"15min":1.409},"last":1.0329999999999999},{"interface":"front","average":{"1min":0.98099999999999998,"5min":0.996,"15min":0.996},"min":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"max":{"1min":1.3859999999999999,"5min":1.3859999999999999,"15min":1.3859999999999999},"last":1.175}]},{"osd":6,"last update":"Sat Sep 23 19:05:23 2023","interfaces":[{"interface":"back","average":{"1min":0.90300000000000002,"5min":1.0449999999999999,"15min":1.0800000000000001},"min":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"max":{"1min":1.1850000000000001,"5min":1.502,"15min":1.502},"last":1.054},{"interface":"front","average":{"1min":1.034,"5min":1.149,"15min":1.179},"min":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"max":{"1min":1.3580000000000001,"5min":1.476,"15min":1.476},"last":1.228}]},{"osd":7,"last update":"Sat Sep 23 19:05:04 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":1.129,"15min":1.133},"min":{"1min":0.629,"5min":0.629,"15min":0.629},"max":{"1min":1.655,"5min":1.655,"15min":1.655},"last":1.115},{"interface":"front","average":{"1min":1.0149999999999999,"5min":1.0780000000000001,"15min":1.0880000000000001},"min":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"max":{"1min":1.595,"5min":1.595,"15min":1.595},"last":1.258}]}]},{"osd":2,"up_from":18,"seq":77309411401,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:04:39 2023","interfaces":[{"interface":"back","average":{"1min":0.84999999999999998,"5min":0.74199999999999999,"15min":0.67300000000000004},"min":{"1min":0.49299999999999999,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":1.8979999999999999,"5min":1.8979999999999999,"15min":1.8979999999999999},"last":0.93999999999999995},{"interface":"front","average":{"1min":0.80700000000000005,"5min":0.78300000000000003,"15min":0.70099999999999996},"min":{"1min":0.55300000000000005,"5min":0.495,"15min":0.495},"max":{"1min":1.651,"5min":1.651,"15min":1.651},"last":0.70899999999999996}]},{"osd":1,"last update":"Sat Sep 23 19:04:39 2023","interfaces":[{"interface":"back","average":{"1min":0.80600000000000005,"5min":0.77100000000000002,"15min":0.72699999999999998},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":1.4490000000000001,"5min":1.4490000000000001,"15min":1.4490000000000001},"last":0.86899999999999999},{"interface":"front","average":{"1min":0.90900000000000003,"5min":0.82899999999999996,"15min":0.746},"min":{"1min":0.63400000000000001,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":1.788,"5min":1.788,"15min":1.788},"last":0.746}]},{"osd":3,"last update":"Sat Sep 23 19:05:24 2023","interfaces":[{"interface":"back","average":{"1min":0.72999999999999998,"5min":0.79700000000000004,"15min":0.77000000000000002},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":1.748,"5min":1.748,"15min":1.748},"last":0.80300000000000005},{"interface":"front","average":{"1min":0.80300000000000005,"5min":0.86599999999999999,"15min":0.81000000000000005},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":1.4690000000000001,"5min":1.4690000000000001,"15min":1.4690000000000001},"last":0.91000000000000003}]},{"osd":4,"last update":"Sat Sep 23 19:05:01 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":0.99299999999999999,"15min":0.98799999999999999},"min":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"max":{"1min":1.3129999999999999,"5min":1.335,"15min":1.335},"last":0.84599999999999997},{"interface":"front","average":{"1min":0.93999999999999995,"5min":0.99399999999999999,"15min":0.98399999999999999},"min":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"max":{"1min":1.363,"5min":1.554,"15min":1.554},"last":1.3799999999999999}]},{"osd":5,"last update":"Sat Sep 23 19:04:45 2023","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":1.0620000000000001,"15min":1.0509999999999999},"min":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"max":{"1min":1.494,"5min":1.635,"15min":1.635},"last":1.4350000000000001},{"interface":"front","average":{"1min":1.022,"5min":1.087,"15min":1.095},"min":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"max":{"1min":1.3899999999999999,"5min":1.581,"15min":1.581},"last":1.323}]},{"osd":6,"last update":"Sat Sep 23 19:05:24 2023","interfaces":[{"interface":"back","average":{"1min":1.0009999999999999,"5min":1.0900000000000001,"15min":1.1220000000000001},"min":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"max":{"1min":1.5,"5min":1.5,"15min":1.5},"last":1.2310000000000001},{"interface":"front","average":{"1min":1.0580000000000001,"5min":1.1259999999999999,"15min":1.1459999999999999},"min":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"max":{"1min":1.679,"5min":1.679,"15min":1.679},"last":1.048}]},{"osd":7,"last update":"Sat Sep 23 19:05:06 2023","interfaces":[{"interface":"back","average":{"1min":1.129,"5min":1.165,"15min":1.171},"min":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"max":{"1min":1.5649999999999999,"5min":1.659,"15min":1.659},"last":1.0980000000000001},{"interface":"front","average":{"1min":1.0589999999999999,"5min":1.1839999999999999,"15min":1.2050000000000001},"min":{"1min":0.621,"5min":0.621,"15min":0.621},"max":{"1min":1.5760000000000001,"5min":1.6890000000000001,"15min":1.6890000000000001},"last":1.482}]}]},{"osd":3,"up_from":25,"seq":107374182467,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:05:25 2023","interfaces":[{"interface":"back","average":{"1min":0.70099999999999996,"5min":0.68999999999999995,"15min":0.66500000000000004},"min":{"1min":0.54900000000000004,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":0.878,"5min":1.286,"15min":1.286},"last":0.622},{"interface":"front","average":{"1min":0.69499999999999995,"5min":0.751,"15min":0.75800000000000001},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.95499999999999996,"5min":1.1120000000000001,"15min":1.1120000000000001},"last":0.65000000000000002}]},{"osd":1,"last update":"Sat Sep 23 19:05:25 2023","interfaces":[{"interface":"back","average":{"1min":0.751,"5min":0.78900000000000003,"15min":0.78600000000000003},"min":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":0.95899999999999996,"5min":1.119,"15min":1.119},"last":0.72699999999999998},{"interface":"front","average":{"1min":0.69699999999999995,"5min":0.749,"15min":0.75700000000000001},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.94599999999999995,"5min":1.3180000000000001,"15min":1.3180000000000001},"last":0.745}]},{"osd":2,"last update":"Sat Sep 23 19:05:25 2023","interfaces":[{"interface":"back","average":{"1min":0.80200000000000005,"5min":0.85799999999999998,"15min":0.86699999999999999},"min":{"1min":0.59699999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":1.1930000000000001,"5min":1.4339999999999999,"15min":1.4339999999999999},"last":0.89700000000000002},{"interface":"front","average":{"1min":0.80800000000000005,"5min":0.84399999999999997,"15min":0.82499999999999996},"min":{"1min":0.55500000000000005,"5min":0.505,"15min":0.505},"max":{"1min":1.284,"5min":1.3700000000000001,"15min":1.3700000000000001},"last":0.70499999999999996}]},{"osd":4,"last update":"Sat Sep 23 19:05:00 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":0.98799999999999999,"15min":0.98799999999999999},"min":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"max":{"1min":1.3160000000000001,"5min":1.7250000000000001,"15min":1.7250000000000001},"last":1.2090000000000001},{"interface":"front","average":{"1min":0.95799999999999996,"5min":0.98199999999999998,"15min":0.99099999999999999},"min":{"1min":0.63,"5min":0.63,"15min":0.63},"max":{"1min":1.413,"5min":1.413,"15min":1.413},"last":0.84699999999999998}]},{"osd":5,"last update":"Sat Sep 23 19:04:49 2023","interfaces":[{"interface":"back","average":{"1min":1,"5min":1.0629999999999999,"15min":1.0669999999999999},"min":{"1min":0.74399999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"max":{"1min":1.256,"5min":1.9319999999999999,"15min":1.9319999999999999},"last":1.0649999999999999},{"interface":"front","average":{"1min":0.99199999999999999,"5min":1.0369999999999999,"15min":1.028},"min":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"max":{"1min":1.2370000000000001,"5min":1.417,"15min":1.417},"last":0.78700000000000003}]},{"osd":6,"last update":"Sat Sep 23 19:05:25 2023","interfaces":[{"interface":"back","average":{"1min":1.0669999999999999,"5min":1.0960000000000001,"15min":1.1220000000000001},"min":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"max":{"1min":1.363,"5min":1.4179999999999999,"15min":1.4179999999999999},"last":0.93700000000000006},{"interface":"front","average":{"1min":1.0980000000000001,"5min":1.21,"15min":1.256},"min":{"1min":0.84999999999999998,"5min":0.78100000000000003,"15min":0.78100000000000003},"max":{"1min":1.3680000000000001,"5min":2.1960000000000002,"15min":2.1960000000000002},"last":1.145}]},{"osd":7,"last update":"Sat Sep 23 19:05:07 2023","interfaces":[{"interface":"back","average":{"1min":1.127,"5min":1.1100000000000001,"15min":1.107},"min":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"max":{"1min":1.486,"5min":1.486,"15min":1.486},"last":0.998},{"interface":"front","average":{"1min":1.0900000000000001,"5min":1.139,"15min":1.147},"min":{"1min":0.72499999999999998,"5min":0.70499999999999996,"15min":0.70499999999999996},"max":{"1min":1.395,"5min":1.516,"15min":1.516},"last":1.042}]}]},{"osd":4,"up_from":30,"seq":128849018938,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:05:07 2023","interfaces":[{"interface":"back","average":{"1min":0.92900000000000005,"5min":0.97099999999999997,"15min":0.96499999999999997},"min":{"1min":0.57699999999999996,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":1.2709999999999999,"5min":1.343,"15min":1.343},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.92700000000000005,"5min":0.95599999999999996,"15min":0.96999999999999997},"min":{"1min":0.55200000000000005,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.2450000000000001,"5min":1.2769999999999999,"15min":1.2769999999999999},"last":0.64800000000000002}]},{"osd":1,"last update":"Sat Sep 23 19:05:07 2023","interfaces":[{"interface":"back","average":{"1min":0.871,"5min":0.85199999999999998,"15min":0.85899999999999999},"min":{"1min":0.60899999999999999,"5min":0.58799999999999997,"15min":0.58799999999999997},"max":{"1min":1.0780000000000001,"5min":1.23,"15min":1.23},"last":0.69699999999999995},{"interface":"front","average":{"1min":0.86799999999999999,"5min":0.90200000000000002,"15min":0.89000000000000001},"min":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"max":{"1min":1.4099999999999999,"5min":1.4099999999999999,"15min":1.4099999999999999},"last":0.81999999999999995}]},{"osd":2,"last update":"Sat Sep 23 19:05:07 2023","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":0.874,"15min":0.85799999999999998},"min":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.343,"5min":1.343,"15min":1.343},"last":0.92100000000000004},{"interface":"front","average":{"1min":0.82999999999999996,"5min":0.874,"15min":0.87},"min":{"1min":0.626,"5min":0.59499999999999997,"15min":0.59499999999999997},"max":{"1min":1.1180000000000001,"5min":1.2869999999999999,"15min":1.2869999999999999},"last":1.0669999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:05:07 2023","interfaces":[{"interface":"back","average":{"1min":0.86399999999999999,"5min":0.89200000000000002,"15min":0.90500000000000003},"min":{"1min":0.61499999999999999,"5min":0.56200000000000006,"15min":0.56200000000000006},"max":{"1min":1.0589999999999999,"5min":1.2589999999999999,"15min":1.2589999999999999},"last":0.96899999999999997},{"interface":"front","average":{"1min":0.88400000000000001,"5min":0.89600000000000002,"15min":0.88600000000000001},"min":{"1min":0.65200000000000002,"5min":0.60199999999999998,"15min":0.60199999999999998},"max":{"1min":1.1220000000000001,"5min":1.335,"15min":1.335},"last":0.77300000000000002}]},{"osd":5,"last update":"Sat Sep 23 19:04:48 2023","interfaces":[{"interface":"back","average":{"1min":0.81200000000000006,"5min":0.78700000000000003,"15min":0.753},"min":{"1min":0.49199999999999999,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":1.1930000000000001,"5min":1.373,"15min":1.373},"last":1.109},{"interface":"front","average":{"1min":0.85199999999999998,"5min":0.88200000000000001,"15min":0.878},"min":{"1min":0.39500000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":1.2,"5min":1.407,"15min":1.407},"last":1.173}]},{"osd":6,"last update":"Sat Sep 23 19:05:22 2023","interfaces":[{"interface":"back","average":{"1min":0.93700000000000006,"5min":0.94999999999999996,"15min":0.95599999999999996},"min":{"1min":0.59499999999999997,"5min":0.58099999999999996,"15min":0.58099999999999996},"max":{"1min":1.23,"5min":1.4430000000000001,"15min":1.4430000000000001},"last":1.0349999999999999},{"interface":"front","average":{"1min":0.97599999999999998,"5min":0.95299999999999996,"15min":0.94999999999999996},"min":{"1min":0.46999999999999997,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":1.294,"5min":1.492,"15min":1.492},"last":1.226}]},{"osd":7,"last update":"Sat Sep 23 19:05:07 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":0.94199999999999995,"15min":0.94799999999999995},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":1.1830000000000001,"5min":1.327,"15min":1.327},"last":1},{"interface":"front","average":{"1min":1.0069999999999999,"5min":1.0129999999999999,"15min":1.014},"min":{"1min":0.54000000000000004,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":1.4890000000000001,"5min":1.51,"15min":1.51},"last":1.1990000000000001}]}]},{"osd":5,"up_from":36,"seq":154618822705,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297616,"kb_used_data":520,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447536,"statfs":{"total":95995035648,"available":95690276864,"internally_reserved":0,"allocated":532480,"data_stored":219885,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:04:41 2023","interfaces":[{"interface":"back","average":{"1min":0.88,"5min":0.90700000000000003,"15min":0.91700000000000004},"min":{"1min":0.66400000000000003,"5min":0.64500000000000002,"15min":0.64500000000000002},"max":{"1min":1.2110000000000001,"5min":1.2110000000000001,"15min":1.2110000000000001},"last":0.999},{"interface":"front","average":{"1min":0.85599999999999998,"5min":0.86899999999999999,"15min":0.873},"min":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"max":{"1min":1.2410000000000001,"5min":1.4710000000000001,"15min":1.4710000000000001},"last":0.70299999999999996}]},{"osd":1,"last update":"Sat Sep 23 19:04:41 2023","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":0.91100000000000003,"15min":0.90500000000000003},"min":{"1min":0.69099999999999995,"5min":0.60599999999999998,"15min":0.60599999999999998},"max":{"1min":1.413,"5min":1.413,"15min":1.413},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.80500000000000005,"5min":0.86599999999999999,"15min":0.87},"min":{"1min":0.5,"5min":0.5,"15min":0.5},"max":{"1min":1.3160000000000001,"5min":1.3160000000000001,"15min":1.3160000000000001},"last":0.89100000000000001}]},{"osd":2,"last update":"Sat Sep 23 19:04:41 2023","interfaces":[{"interface":"back","average":{"1min":0.871,"5min":0.88400000000000001,"15min":0.88500000000000001},"min":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"max":{"1min":1.0940000000000001,"5min":1.4399999999999999,"15min":1.4399999999999999},"last":1.127},{"interface":"front","average":{"1min":0.93999999999999995,"5min":0.96699999999999997,"15min":0.96999999999999997},"min":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"max":{"1min":1.3779999999999999,"5min":1.3779999999999999,"15min":1.3779999999999999},"last":0.92900000000000005}]},{"osd":3,"last update":"Sat Sep 23 19:04:41 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":0.89100000000000001,"15min":0.88900000000000001},"min":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"max":{"1min":1.1020000000000001,"5min":1.3,"15min":1.3},"last":1.0369999999999999},{"interface":"front","average":{"1min":1.008,"5min":1,"15min":1.0049999999999999},"min":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"max":{"1min":1.3360000000000001,"5min":1.3360000000000001,"15min":1.3360000000000001},"last":1.0600000000000001}]},{"osd":4,"last update":"Sat Sep 23 19:04:41 2023","interfaces":[{"interface":"back","average":{"1min":0.80800000000000005,"5min":0.80700000000000005,"15min":0.80600000000000005},"min":{"1min":0.51400000000000001,"5min":0.504,"15min":0.504},"max":{"1min":1.655,"5min":1.655,"15min":1.655},"last":0.80900000000000005},{"interface":"front","average":{"1min":0.92500000000000004,"5min":0.87,"15min":0.85799999999999998},"min":{"1min":0.57799999999999996,"5min":0.57299999999999995,"15min":0.57299999999999995},"max":{"1min":2.415,"5min":2.415,"15min":2.415},"last":0.95899999999999996}]},{"osd":6,"last update":"Sat Sep 23 19:05:27 2023","interfaces":[{"interface":"back","average":{"1min":1.054,"5min":0.97699999999999998,"15min":0.94499999999999995},"min":{"1min":0.70099999999999996,"5min":0.48999999999999999,"15min":0.48999999999999999},"max":{"1min":1.4530000000000001,"5min":2.0699999999999998,"15min":2.0699999999999998},"last":1.1040000000000001},{"interface":"front","average":{"1min":1.1259999999999999,"5min":1.0549999999999999,"15min":1.0289999999999999},"min":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"max":{"1min":1.5669999999999999,"5min":2.4769999999999999,"15min":2.4769999999999999},"last":0.75900000000000001}]},{"osd":7,"last update":"Sat Sep 23 19:05:06 2023","interfaces":[{"interface":"back","average":{"1min":1.1060000000000001,"5min":1.038,"15min":1.0269999999999999},"min":{"1min":0.68400000000000005,"5min":0.63800000000000001,"15min":0.63800000000000001},"max":{"1min":2.5049999999999999,"5min":2.5049999999999999,"15min":2.5049999999999999},"last":1.075},{"interface":"front","average":{"1min":1.181,"5min":1.0649999999999999,"15min":1.0449999999999999},"min":{"1min":0.65400000000000003,"5min":0.54500000000000004,"15min":0.54500000000000004},"max":{"1min":2.4340000000000002,"5min":2.4340000000000002,"15min":2.4340000000000002},"last":1.151}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T19:05:38.804 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-09-23T19:05:38.804 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-09-23T19:05:38.804 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-09-23T19:05:38.805 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph health --format=json 2023-09-23T19:05:39.537 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:05:39 smithi092 bash[28593]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:05:39] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T19:05:40.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:40 smithi016 bash[27237]: audit 2023-09-23T19:05:38.087818+0000 mgr.y (mgr.14454) 90 : audit [DBG] from='client.14637 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:05:40.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:40 smithi016 bash[27237]: cluster 2023-09-23T19:05:38.902451+0000 mgr.y (mgr.14454) 91 : cluster [DBG] pgmap v69: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:40.535 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:05:40 smithi016 bash[27503]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:05:40] "GET /metrics HTTP/1.1" 200 155655 "" "Prometheus/2.33.4" 2023-09-23T19:05:40.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:40 smithi016 bash[33624]: audit 2023-09-23T19:05:38.087818+0000 mgr.y (mgr.14454) 90 : audit [DBG] from='client.14637 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:05:40.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:40 smithi016 bash[33624]: cluster 2023-09-23T19:05:38.902451+0000 mgr.y (mgr.14454) 91 : cluster [DBG] pgmap v69: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:40.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:40 smithi092 bash[27556]: audit 2023-09-23T19:05:38.087818+0000 mgr.y (mgr.14454) 90 : audit [DBG] from='client.14637 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:05:40.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:40 smithi092 bash[27556]: cluster 2023-09-23T19:05:38.902451+0000 mgr.y (mgr.14454) 91 : cluster [DBG] pgmap v69: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:42.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:42 smithi016 bash[27237]: cluster 2023-09-23T19:05:40.903010+0000 mgr.y (mgr.14454) 92 : cluster [DBG] pgmap v70: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:42.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:42 smithi016 bash[33624]: cluster 2023-09-23T19:05:40.903010+0000 mgr.y (mgr.14454) 92 : cluster [DBG] pgmap v70: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:42.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:42 smithi092 bash[27556]: cluster 2023-09-23T19:05:40.903010+0000 mgr.y (mgr.14454) 92 : cluster [DBG] pgmap v70: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:43.586 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:05:44.518 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:44 smithi016 bash[27237]: cluster 2023-09-23T19:05:42.903682+0000 mgr.y (mgr.14454) 93 : cluster [DBG] pgmap v71: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:44.519 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:44 smithi016 bash[33624]: cluster 2023-09-23T19:05:42.903682+0000 mgr.y (mgr.14454) 93 : cluster [DBG] pgmap v71: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:44.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:44 smithi092 bash[27556]: cluster 2023-09-23T19:05:42.903682+0000 mgr.y (mgr.14454) 93 : cluster [DBG] pgmap v71: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:46.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:46 smithi016 bash[27237]: cluster 2023-09-23T19:05:44.904315+0000 mgr.y (mgr.14454) 94 : cluster [DBG] pgmap v72: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:46.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:46 smithi016 bash[33624]: cluster 2023-09-23T19:05:44.904315+0000 mgr.y (mgr.14454) 94 : cluster [DBG] pgmap v72: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:46.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:46 smithi092 bash[27556]: cluster 2023-09-23T19:05:44.904315+0000 mgr.y (mgr.14454) 94 : cluster [DBG] pgmap v72: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:46.676 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-09-23T19:05:47.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:47 smithi016 bash[27237]: audit 2023-09-23T19:05:46.673101+0000 mon.a (mon.0) 614 : audit [DBG] from='client.? 172.21.15.16:0/144452704' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T19:05:47.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:47 smithi016 bash[33624]: audit 2023-09-23T19:05:46.673101+0000 mon.a (mon.0) 614 : audit [DBG] from='client.? 172.21.15.16:0/144452704' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T19:05:47.485 INFO:teuthology.orchestra.run.smithi016.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-09-23T19:05:47.485 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-09-23T19:05:47.485 INFO:tasks.cephadm:Setup complete, yielding 2023-09-23T19:05:47.486 INFO:teuthology.run_tasks:Running task cephadm.shell... 2023-09-23T19:05:47.495 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi016.front.sepia.ceph.com 2023-09-23T19:05:47.495 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- bash -c 'ceph fs volume create foo' 2023-09-23T19:05:47.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:47 smithi092 bash[27556]: audit 2023-09-23T19:05:46.673101+0000 mon.a (mon.0) 614 : audit [DBG] from='client.? 172.21.15.16:0/144452704' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T19:05:48.511 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:48 smithi016 bash[27237]: cluster 2023-09-23T19:05:46.904995+0000 mgr.y (mgr.14454) 95 : cluster [DBG] pgmap v73: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:48.511 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:48 smithi016 bash[33624]: cluster 2023-09-23T19:05:46.904995+0000 mgr.y (mgr.14454) 95 : cluster [DBG] pgmap v73: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:48.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:48 smithi092 bash[27556]: cluster 2023-09-23T19:05:46.904995+0000 mgr.y (mgr.14454) 95 : cluster [DBG] pgmap v73: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:49.537 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:05:49 smithi092 bash[28593]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:05:49] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T19:05:50.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:50 smithi016 bash[27237]: cluster 2023-09-23T19:05:48.905605+0000 mgr.y (mgr.14454) 96 : cluster [DBG] pgmap v74: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:50.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:50 smithi016 bash[33624]: cluster 2023-09-23T19:05:48.905605+0000 mgr.y (mgr.14454) 96 : cluster [DBG] pgmap v74: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:50.535 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:05:50 smithi016 bash[27503]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:05:50] "GET /metrics HTTP/1.1" 200 155655 "" "Prometheus/2.33.4" 2023-09-23T19:05:50.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:50 smithi092 bash[27556]: cluster 2023-09-23T19:05:48.905605+0000 mgr.y (mgr.14454) 96 : cluster [DBG] pgmap v74: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:51.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:51 smithi016 bash[33624]: audit 2023-09-23T19:05:50.522695+0000 mon.c (mon.2) 65 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-09-23T19:05:51.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:51 smithi016 bash[33624]: audit 2023-09-23T19:05:50.523445+0000 mon.a (mon.0) 615 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-09-23T19:05:51.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:51 smithi016 bash[27237]: audit 2023-09-23T19:05:50.522695+0000 mon.c (mon.2) 65 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-09-23T19:05:51.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:51 smithi016 bash[27237]: audit 2023-09-23T19:05:50.523445+0000 mon.a (mon.0) 615 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-09-23T19:05:51.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:51 smithi092 bash[27556]: audit 2023-09-23T19:05:50.522695+0000 mon.c (mon.2) 65 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-09-23T19:05:51.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:51 smithi092 bash[27556]: audit 2023-09-23T19:05:50.523445+0000 mon.a (mon.0) 615 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-09-23T19:05:52.397 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:52 smithi016 bash[33624]: audit 2023-09-23T19:05:50.521704+0000 mgr.y (mgr.14454) 97 : audit [DBG] from='client.14649 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:05:52.397 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:52 smithi016 bash[33624]: cluster 2023-09-23T19:05:50.906135+0000 mgr.y (mgr.14454) 98 : cluster [DBG] pgmap v75: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:52.397 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:52 smithi016 bash[33624]: audit 2023-09-23T19:05:51.078659+0000 mon.a (mon.0) 616 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-09-23T19:05:52.397 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:52 smithi016 bash[33624]: cluster 2023-09-23T19:05:51.078773+0000 mon.a (mon.0) 617 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2023-09-23T19:05:52.397 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:52 smithi016 bash[33624]: audit 2023-09-23T19:05:51.083793+0000 mon.c (mon.2) 66 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-09-23T19:05:52.397 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:52 smithi016 bash[33624]: audit 2023-09-23T19:05:51.084227+0000 mon.a (mon.0) 618 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-09-23T19:05:52.398 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:52 smithi016 bash[27237]: audit 2023-09-23T19:05:50.521704+0000 mgr.y (mgr.14454) 97 : audit [DBG] from='client.14649 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:05:52.398 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:52 smithi016 bash[27237]: cluster 2023-09-23T19:05:50.906135+0000 mgr.y (mgr.14454) 98 : cluster [DBG] pgmap v75: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:52.398 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:52 smithi016 bash[27237]: audit 2023-09-23T19:05:51.078659+0000 mon.a (mon.0) 616 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-09-23T19:05:52.398 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:52 smithi016 bash[27237]: cluster 2023-09-23T19:05:51.078773+0000 mon.a (mon.0) 617 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2023-09-23T19:05:52.398 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:52 smithi016 bash[27237]: audit 2023-09-23T19:05:51.083793+0000 mon.c (mon.2) 66 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-09-23T19:05:52.398 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:52 smithi016 bash[27237]: audit 2023-09-23T19:05:51.084227+0000 mon.a (mon.0) 618 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-09-23T19:05:52.398 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:52 smithi016 bash[27237]: debug 2023-09-23T19:05:52.112+0000 7f387c284700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-09-23T19:05:52.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:52 smithi092 bash[27556]: audit 2023-09-23T19:05:50.521704+0000 mgr.y (mgr.14454) 97 : audit [DBG] from='client.14649 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:05:52.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:52 smithi092 bash[27556]: cluster 2023-09-23T19:05:50.906135+0000 mgr.y (mgr.14454) 98 : cluster [DBG] pgmap v75: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:52.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:52 smithi092 bash[27556]: audit 2023-09-23T19:05:51.078659+0000 mon.a (mon.0) 616 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-09-23T19:05:52.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:52 smithi092 bash[27556]: cluster 2023-09-23T19:05:51.078773+0000 mon.a (mon.0) 617 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2023-09-23T19:05:52.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:52 smithi092 bash[27556]: audit 2023-09-23T19:05:51.083793+0000 mon.c (mon.2) 66 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-09-23T19:05:52.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:52 smithi092 bash[27556]: audit 2023-09-23T19:05:51.084227+0000 mon.a (mon.0) 618 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-09-23T19:05:53.247 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2023-09-23T19:05:53.439 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:53 smithi016 bash[27237]: audit 2023-09-23T19:05:52.085076+0000 mon.a (mon.0) 619 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-09-23T19:05:53.439 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:53 smithi016 bash[27237]: cluster 2023-09-23T19:05:52.085475+0000 mon.a (mon.0) 620 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2023-09-23T19:05:53.439 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:53 smithi016 bash[27237]: audit 2023-09-23T19:05:52.108827+0000 mon.c (mon.2) 67 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-09-23T19:05:53.439 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:53 smithi016 bash[27237]: audit 2023-09-23T19:05:52.116253+0000 mon.a (mon.0) 621 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-09-23T19:05:53.440 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:53 smithi016 bash[27237]: cluster 2023-09-23T19:05:52.117201+0000 mon.a (mon.0) 622 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-09-23T19:05:53.440 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:53 smithi016 bash[27237]: cluster 2023-09-23T19:05:52.117248+0000 mon.a (mon.0) 623 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-09-23T19:05:53.440 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:53 smithi016 bash[27237]: cluster 2023-09-23T19:05:52.122782+0000 mon.a (mon.0) 624 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2023-09-23T19:05:53.440 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:53 smithi016 bash[27237]: audit 2023-09-23T19:05:52.123036+0000 mon.a (mon.0) 625 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-09-23T19:05:53.440 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:53 smithi016 bash[27237]: cluster 2023-09-23T19:05:52.123175+0000 mon.a (mon.0) 626 : cluster [DBG] fsmap foo:0 2023-09-23T19:05:53.440 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:53 smithi016 bash[27237]: cephadm 2023-09-23T19:05:52.135384+0000 mgr.y (mgr.14454) 99 : cephadm [INF] Saving service mds.foo spec with placement count:2 2023-09-23T19:05:53.440 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:53 smithi016 bash[27237]: audit 2023-09-23T19:05:52.142242+0000 mon.a (mon.0) 627 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:05:53.440 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:53 smithi016 bash[27237]: audit 2023-09-23T19:05:52.151339+0000 mon.c (mon.2) 68 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:05:53.440 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:53 smithi016 bash[27237]: audit 2023-09-23T19:05:52.444734+0000 mon.c (mon.2) 69 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:05:53.440 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:53 smithi016 bash[27237]: audit 2023-09-23T19:05:52.446367+0000 mon.c (mon.2) 70 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:05:53.440 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:53 smithi016 bash[27237]: audit 2023-09-23T19:05:52.453729+0000 mon.a (mon.0) 628 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:05:53.440 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:53 smithi016 bash[27237]: audit 2023-09-23T19:05:52.463683+0000 mon.a (mon.0) 629 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:05:53.441 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:53 smithi016 bash[27237]: audit 2023-09-23T19:05:52.465733+0000 mon.c (mon.2) 71 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi016.uncssv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T19:05:53.441 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:53 smithi016 bash[27237]: audit 2023-09-23T19:05:52.466150+0000 mon.a (mon.0) 630 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi016.uncssv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T19:05:53.441 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:53 smithi016 bash[27237]: audit 2023-09-23T19:05:52.469008+0000 mon.a (mon.0) 631 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi016.uncssv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T19:05:53.441 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:53 smithi016 bash[27237]: audit 2023-09-23T19:05:52.470711+0000 mon.c (mon.2) 72 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:05:53.441 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:53 smithi016 bash[27237]: cephadm 2023-09-23T19:05:52.471722+0000 mgr.y (mgr.14454) 100 : cephadm [INF] Deploying daemon mds.foo.smithi016.uncssv on smithi016 2023-09-23T19:05:53.441 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:53 smithi016 bash[27237]: cluster 2023-09-23T19:05:52.906776+0000 mgr.y (mgr.14454) 101 : cluster [DBG] pgmap v79: 73 pgs: 64 unknown, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:53.441 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:53 smithi016 bash[33624]: audit 2023-09-23T19:05:52.085076+0000 mon.a (mon.0) 619 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-09-23T19:05:53.441 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:53 smithi016 bash[33624]: cluster 2023-09-23T19:05:52.085475+0000 mon.a (mon.0) 620 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2023-09-23T19:05:53.442 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:53 smithi016 bash[33624]: audit 2023-09-23T19:05:52.108827+0000 mon.c (mon.2) 67 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-09-23T19:05:53.442 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:53 smithi016 bash[33624]: audit 2023-09-23T19:05:52.116253+0000 mon.a (mon.0) 621 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-09-23T19:05:53.442 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:53 smithi016 bash[33624]: cluster 2023-09-23T19:05:52.117201+0000 mon.a (mon.0) 622 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-09-23T19:05:53.442 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:53 smithi016 bash[33624]: cluster 2023-09-23T19:05:52.117248+0000 mon.a (mon.0) 623 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-09-23T19:05:53.442 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:53 smithi016 bash[33624]: cluster 2023-09-23T19:05:52.122782+0000 mon.a (mon.0) 624 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2023-09-23T19:05:53.442 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:53 smithi016 bash[33624]: audit 2023-09-23T19:05:52.123036+0000 mon.a (mon.0) 625 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-09-23T19:05:53.442 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:53 smithi016 bash[33624]: cluster 2023-09-23T19:05:52.123175+0000 mon.a (mon.0) 626 : cluster [DBG] fsmap foo:0 2023-09-23T19:05:53.442 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:53 smithi016 bash[33624]: cephadm 2023-09-23T19:05:52.135384+0000 mgr.y (mgr.14454) 99 : cephadm [INF] Saving service mds.foo spec with placement count:2 2023-09-23T19:05:53.442 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:53 smithi016 bash[33624]: audit 2023-09-23T19:05:52.142242+0000 mon.a (mon.0) 627 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:05:53.442 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:53 smithi016 bash[33624]: audit 2023-09-23T19:05:52.151339+0000 mon.c (mon.2) 68 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:05:53.443 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:53 smithi016 bash[33624]: audit 2023-09-23T19:05:52.444734+0000 mon.c (mon.2) 69 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:05:53.443 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:53 smithi016 bash[33624]: audit 2023-09-23T19:05:52.446367+0000 mon.c (mon.2) 70 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:05:53.443 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:53 smithi016 bash[33624]: audit 2023-09-23T19:05:52.453729+0000 mon.a (mon.0) 628 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:05:53.443 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:53 smithi016 bash[33624]: audit 2023-09-23T19:05:52.463683+0000 mon.a (mon.0) 629 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:05:53.443 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:53 smithi016 bash[33624]: audit 2023-09-23T19:05:52.465733+0000 mon.c (mon.2) 71 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi016.uncssv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T19:05:53.443 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:53 smithi016 bash[33624]: audit 2023-09-23T19:05:52.466150+0000 mon.a (mon.0) 630 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi016.uncssv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T19:05:53.443 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:53 smithi016 bash[33624]: audit 2023-09-23T19:05:52.469008+0000 mon.a (mon.0) 631 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi016.uncssv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T19:05:53.443 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:53 smithi016 bash[33624]: audit 2023-09-23T19:05:52.470711+0000 mon.c (mon.2) 72 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:05:53.443 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:53 smithi016 bash[33624]: cephadm 2023-09-23T19:05:52.471722+0000 mgr.y (mgr.14454) 100 : cephadm [INF] Deploying daemon mds.foo.smithi016.uncssv on smithi016 2023-09-23T19:05:53.443 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:53 smithi016 bash[33624]: cluster 2023-09-23T19:05:52.906776+0000 mgr.y (mgr.14454) 101 : cluster [DBG] pgmap v79: 73 pgs: 64 unknown, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:53.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:53 smithi092 bash[27556]: audit 2023-09-23T19:05:52.085076+0000 mon.a (mon.0) 619 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-09-23T19:05:53.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:53 smithi092 bash[27556]: cluster 2023-09-23T19:05:52.085475+0000 mon.a (mon.0) 620 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2023-09-23T19:05:53.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:53 smithi092 bash[27556]: audit 2023-09-23T19:05:52.108827+0000 mon.c (mon.2) 67 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-09-23T19:05:53.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:53 smithi092 bash[27556]: audit 2023-09-23T19:05:52.116253+0000 mon.a (mon.0) 621 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-09-23T19:05:53.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:53 smithi092 bash[27556]: cluster 2023-09-23T19:05:52.117201+0000 mon.a (mon.0) 622 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-09-23T19:05:53.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:53 smithi092 bash[27556]: cluster 2023-09-23T19:05:52.117248+0000 mon.a (mon.0) 623 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-09-23T19:05:53.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:53 smithi092 bash[27556]: cluster 2023-09-23T19:05:52.122782+0000 mon.a (mon.0) 624 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2023-09-23T19:05:53.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:53 smithi092 bash[27556]: audit 2023-09-23T19:05:52.123036+0000 mon.a (mon.0) 625 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-09-23T19:05:53.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:53 smithi092 bash[27556]: cluster 2023-09-23T19:05:52.123175+0000 mon.a (mon.0) 626 : cluster [DBG] fsmap foo:0 2023-09-23T19:05:53.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:53 smithi092 bash[27556]: cephadm 2023-09-23T19:05:52.135384+0000 mgr.y (mgr.14454) 99 : cephadm [INF] Saving service mds.foo spec with placement count:2 2023-09-23T19:05:53.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:53 smithi092 bash[27556]: audit 2023-09-23T19:05:52.142242+0000 mon.a (mon.0) 627 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:05:53.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:53 smithi092 bash[27556]: audit 2023-09-23T19:05:52.151339+0000 mon.c (mon.2) 68 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:05:53.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:53 smithi092 bash[27556]: audit 2023-09-23T19:05:52.444734+0000 mon.c (mon.2) 69 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:05:53.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:53 smithi092 bash[27556]: audit 2023-09-23T19:05:52.446367+0000 mon.c (mon.2) 70 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:05:53.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:53 smithi092 bash[27556]: audit 2023-09-23T19:05:52.453729+0000 mon.a (mon.0) 628 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:05:53.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:53 smithi092 bash[27556]: audit 2023-09-23T19:05:52.463683+0000 mon.a (mon.0) 629 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:05:53.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:53 smithi092 bash[27556]: audit 2023-09-23T19:05:52.465733+0000 mon.c (mon.2) 71 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi016.uncssv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T19:05:53.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:53 smithi092 bash[27556]: audit 2023-09-23T19:05:52.466150+0000 mon.a (mon.0) 630 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi016.uncssv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T19:05:53.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:53 smithi092 bash[27556]: audit 2023-09-23T19:05:52.469008+0000 mon.a (mon.0) 631 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi016.uncssv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T19:05:53.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:53 smithi092 bash[27556]: audit 2023-09-23T19:05:52.470711+0000 mon.c (mon.2) 72 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:05:53.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:53 smithi092 bash[27556]: cephadm 2023-09-23T19:05:52.471722+0000 mgr.y (mgr.14454) 100 : cephadm [INF] Deploying daemon mds.foo.smithi016.uncssv on smithi016 2023-09-23T19:05:53.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:53 smithi092 bash[27556]: cluster 2023-09-23T19:05:52.906776+0000 mgr.y (mgr.14454) 101 : cluster [DBG] pgmap v79: 73 pgs: 64 unknown, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:54.446 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:54 smithi016 bash[27237]: cluster 2023-09-23T19:05:53.126648+0000 mon.a (mon.0) 632 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2023-09-23T19:05:54.447 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:54 smithi016 bash[33624]: cluster 2023-09-23T19:05:53.126648+0000 mon.a (mon.0) 632 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2023-09-23T19:05:54.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:54 smithi092 bash[27556]: cluster 2023-09-23T19:05:53.126648+0000 mon.a (mon.0) 632 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2023-09-23T19:05:56.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:55 smithi016 bash[33624]: cluster 2023-09-23T19:05:54.907420+0000 mgr.y (mgr.14454) 102 : cluster [DBG] pgmap v81: 73 pgs: 64 unknown, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:56.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:55 smithi016 bash[33624]: audit 2023-09-23T19:05:55.792403+0000 mon.a (mon.0) 633 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:05:56.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:55 smithi016 bash[33624]: audit 2023-09-23T19:05:55.794507+0000 mon.c (mon.2) 73 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi092.vbpqks", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T19:05:56.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:55 smithi016 bash[33624]: audit 2023-09-23T19:05:55.795049+0000 mon.a (mon.0) 634 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi092.vbpqks", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T19:05:56.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:55 smithi016 bash[33624]: audit 2023-09-23T19:05:55.799194+0000 mon.a (mon.0) 635 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi092.vbpqks", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T19:05:56.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:55 smithi016 bash[33624]: audit 2023-09-23T19:05:55.801535+0000 mon.c (mon.2) 74 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:05:56.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:55 smithi016 bash[27237]: cluster 2023-09-23T19:05:54.907420+0000 mgr.y (mgr.14454) 102 : cluster [DBG] pgmap v81: 73 pgs: 64 unknown, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:56.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:55 smithi016 bash[27237]: audit 2023-09-23T19:05:55.792403+0000 mon.a (mon.0) 633 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:05:56.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:55 smithi016 bash[27237]: audit 2023-09-23T19:05:55.794507+0000 mon.c (mon.2) 73 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi092.vbpqks", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T19:05:56.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:55 smithi016 bash[27237]: audit 2023-09-23T19:05:55.795049+0000 mon.a (mon.0) 634 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi092.vbpqks", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T19:05:56.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:55 smithi016 bash[27237]: audit 2023-09-23T19:05:55.799194+0000 mon.a (mon.0) 635 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi092.vbpqks", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T19:05:56.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:55 smithi016 bash[27237]: audit 2023-09-23T19:05:55.801535+0000 mon.c (mon.2) 74 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:05:56.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:55 smithi092 bash[27556]: cluster 2023-09-23T19:05:54.907420+0000 mgr.y (mgr.14454) 102 : cluster [DBG] pgmap v81: 73 pgs: 64 unknown, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:56.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:55 smithi092 bash[27556]: audit 2023-09-23T19:05:55.792403+0000 mon.a (mon.0) 633 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:05:56.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:55 smithi092 bash[27556]: audit 2023-09-23T19:05:55.794507+0000 mon.c (mon.2) 73 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi092.vbpqks", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T19:05:56.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:55 smithi092 bash[27556]: audit 2023-09-23T19:05:55.795049+0000 mon.a (mon.0) 634 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi092.vbpqks", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T19:05:56.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:55 smithi092 bash[27556]: audit 2023-09-23T19:05:55.799194+0000 mon.a (mon.0) 635 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi092.vbpqks", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T19:05:56.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:55 smithi092 bash[27556]: audit 2023-09-23T19:05:55.801535+0000 mon.c (mon.2) 74 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:05:57.188 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:56 smithi092 bash[27556]: cephadm 2023-09-23T19:05:55.802528+0000 mgr.y (mgr.14454) 103 : cephadm [INF] Deploying daemon mds.foo.smithi092.vbpqks on smithi092 2023-09-23T19:05:57.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:56 smithi016 bash[27237]: cephadm 2023-09-23T19:05:55.802528+0000 mgr.y (mgr.14454) 103 : cephadm [INF] Deploying daemon mds.foo.smithi092.vbpqks on smithi092 2023-09-23T19:05:57.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:56 smithi016 bash[33624]: cephadm 2023-09-23T19:05:55.802528+0000 mgr.y (mgr.14454) 103 : cephadm [INF] Deploying daemon mds.foo.smithi092.vbpqks on smithi092 2023-09-23T19:05:58.174 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2023-09-23T19:05:58.225 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:57 smithi092 bash[27556]: cluster 2023-09-23T19:05:56.908043+0000 mgr.y (mgr.14454) 104 : cluster [DBG] pgmap v82: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:58.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:57 smithi016 bash[27237]: cluster 2023-09-23T19:05:56.908043+0000 mgr.y (mgr.14454) 104 : cluster [DBG] pgmap v82: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:58.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:57 smithi016 bash[33624]: cluster 2023-09-23T19:05:56.908043+0000 mgr.y (mgr.14454) 104 : cluster [DBG] pgmap v82: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:05:59.212 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:58 smithi016 bash[27237]: audit 2023-09-23T19:05:57.947964+0000 mon.c (mon.2) 75 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi016.uncssv"}]: dispatch 2023-09-23T19:05:59.213 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:58 smithi016 bash[27237]: cluster 2023-09-23T19:05:57.949379+0000 mon.a (mon.0) 636 : cluster [DBG] mds.? [v2:172.21.15.16:6834/161572387,v1:172.21.15.16:6835/161572387] up:boot 2023-09-23T19:05:59.213 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:58 smithi016 bash[27237]: cluster 2023-09-23T19:05:57.949507+0000 mon.a (mon.0) 637 : cluster [INF] daemon mds.foo.smithi016.uncssv assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-09-23T19:05:59.213 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:58 smithi016 bash[27237]: cluster 2023-09-23T19:05:57.949658+0000 mon.a (mon.0) 638 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-09-23T19:05:59.213 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:58 smithi016 bash[27237]: cluster 2023-09-23T19:05:57.949726+0000 mon.a (mon.0) 639 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-09-23T19:05:59.213 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:58 smithi016 bash[27237]: cluster 2023-09-23T19:05:57.949783+0000 mon.a (mon.0) 640 : cluster [INF] Cluster is now healthy 2023-09-23T19:05:59.213 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:58 smithi016 bash[27237]: cluster 2023-09-23T19:05:57.950402+0000 mon.a (mon.0) 641 : cluster [DBG] fsmap foo:0 1 up:standby 2023-09-23T19:05:59.213 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:58 smithi016 bash[27237]: cluster 2023-09-23T19:05:57.956402+0000 mon.a (mon.0) 642 : cluster [DBG] fsmap foo:1 {0=foo.smithi016.uncssv=up:creating} 2023-09-23T19:05:59.213 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:58 smithi016 bash[27237]: cluster 2023-09-23T19:05:57.974171+0000 mon.a (mon.0) 643 : cluster [INF] daemon mds.foo.smithi016.uncssv is now active in filesystem foo as rank 0 2023-09-23T19:05:59.213 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:58 smithi016 bash[27237]: audit 2023-09-23T19:05:58.199482+0000 mon.a (mon.0) 644 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:05:59.213 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:58 smithi016 bash[27237]: audit 2023-09-23T19:05:58.203142+0000 mon.c (mon.2) 76 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:05:59.214 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:58 smithi016 bash[33624]: audit 2023-09-23T19:05:57.947964+0000 mon.c (mon.2) 75 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi016.uncssv"}]: dispatch 2023-09-23T19:05:59.214 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:58 smithi016 bash[33624]: cluster 2023-09-23T19:05:57.949379+0000 mon.a (mon.0) 636 : cluster [DBG] mds.? [v2:172.21.15.16:6834/161572387,v1:172.21.15.16:6835/161572387] up:boot 2023-09-23T19:05:59.214 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:58 smithi016 bash[33624]: cluster 2023-09-23T19:05:57.949507+0000 mon.a (mon.0) 637 : cluster [INF] daemon mds.foo.smithi016.uncssv assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-09-23T19:05:59.214 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:58 smithi016 bash[33624]: cluster 2023-09-23T19:05:57.949658+0000 mon.a (mon.0) 638 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-09-23T19:05:59.214 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:58 smithi016 bash[33624]: cluster 2023-09-23T19:05:57.949726+0000 mon.a (mon.0) 639 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-09-23T19:05:59.214 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:58 smithi016 bash[33624]: cluster 2023-09-23T19:05:57.949783+0000 mon.a (mon.0) 640 : cluster [INF] Cluster is now healthy 2023-09-23T19:05:59.214 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:58 smithi016 bash[33624]: cluster 2023-09-23T19:05:57.950402+0000 mon.a (mon.0) 641 : cluster [DBG] fsmap foo:0 1 up:standby 2023-09-23T19:05:59.214 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:58 smithi016 bash[33624]: cluster 2023-09-23T19:05:57.956402+0000 mon.a (mon.0) 642 : cluster [DBG] fsmap foo:1 {0=foo.smithi016.uncssv=up:creating} 2023-09-23T19:05:59.214 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:58 smithi016 bash[33624]: cluster 2023-09-23T19:05:57.974171+0000 mon.a (mon.0) 643 : cluster [INF] daemon mds.foo.smithi016.uncssv is now active in filesystem foo as rank 0 2023-09-23T19:05:59.215 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:58 smithi016 bash[33624]: audit 2023-09-23T19:05:58.199482+0000 mon.a (mon.0) 644 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:05:59.215 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:58 smithi016 bash[33624]: audit 2023-09-23T19:05:58.203142+0000 mon.c (mon.2) 76 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:05:59.287 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:05:59 smithi092 bash[28593]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:05:59] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T19:05:59.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:58 smithi092 bash[27556]: audit 2023-09-23T19:05:57.947964+0000 mon.c (mon.2) 75 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi016.uncssv"}]: dispatch 2023-09-23T19:05:59.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:58 smithi092 bash[27556]: cluster 2023-09-23T19:05:57.949379+0000 mon.a (mon.0) 636 : cluster [DBG] mds.? [v2:172.21.15.16:6834/161572387,v1:172.21.15.16:6835/161572387] up:boot 2023-09-23T19:05:59.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:58 smithi092 bash[27556]: cluster 2023-09-23T19:05:57.949507+0000 mon.a (mon.0) 637 : cluster [INF] daemon mds.foo.smithi016.uncssv assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-09-23T19:05:59.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:58 smithi092 bash[27556]: cluster 2023-09-23T19:05:57.949658+0000 mon.a (mon.0) 638 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-09-23T19:05:59.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:58 smithi092 bash[27556]: cluster 2023-09-23T19:05:57.949726+0000 mon.a (mon.0) 639 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-09-23T19:05:59.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:58 smithi092 bash[27556]: cluster 2023-09-23T19:05:57.949783+0000 mon.a (mon.0) 640 : cluster [INF] Cluster is now healthy 2023-09-23T19:05:59.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:58 smithi092 bash[27556]: cluster 2023-09-23T19:05:57.950402+0000 mon.a (mon.0) 641 : cluster [DBG] fsmap foo:0 1 up:standby 2023-09-23T19:05:59.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:58 smithi092 bash[27556]: cluster 2023-09-23T19:05:57.956402+0000 mon.a (mon.0) 642 : cluster [DBG] fsmap foo:1 {0=foo.smithi016.uncssv=up:creating} 2023-09-23T19:05:59.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:58 smithi092 bash[27556]: cluster 2023-09-23T19:05:57.974171+0000 mon.a (mon.0) 643 : cluster [INF] daemon mds.foo.smithi016.uncssv is now active in filesystem foo as rank 0 2023-09-23T19:05:59.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:58 smithi092 bash[27556]: audit 2023-09-23T19:05:58.199482+0000 mon.a (mon.0) 644 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:05:59.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:58 smithi092 bash[27556]: audit 2023-09-23T19:05:58.203142+0000 mon.c (mon.2) 76 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:06:00.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:59 smithi016 bash[27237]: cluster 2023-09-23T19:05:58.908533+0000 mgr.y (mgr.14454) 105 : cluster [DBG] pgmap v83: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:00.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:59 smithi016 bash[27237]: cluster 2023-09-23T19:05:58.964955+0000 mon.a (mon.0) 645 : cluster [DBG] mds.? [v2:172.21.15.16:6834/161572387,v1:172.21.15.16:6835/161572387] up:active 2023-09-23T19:06:00.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:59 smithi016 bash[27237]: cluster 2023-09-23T19:05:58.965054+0000 mon.a (mon.0) 646 : cluster [DBG] fsmap foo:1 {0=foo.smithi016.uncssv=up:active} 2023-09-23T19:06:00.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:59 smithi016 bash[27237]: cluster 2023-09-23T19:05:59.935268+0000 mon.a (mon.0) 647 : cluster [DBG] mds.? [v2:172.21.15.92:6832/1118427195,v1:172.21.15.92:6833/1118427195] up:boot 2023-09-23T19:06:00.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:59 smithi016 bash[27237]: cluster 2023-09-23T19:05:59.935318+0000 mon.a (mon.0) 648 : cluster [DBG] fsmap foo:1 {0=foo.smithi016.uncssv=up:active} 1 up:standby 2023-09-23T19:06:00.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:05:59 smithi016 bash[27237]: audit 2023-09-23T19:05:59.941865+0000 mon.c (mon.2) 77 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi092.vbpqks"}]: dispatch 2023-09-23T19:06:00.287 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:06:00 smithi016 bash[27503]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:06:00] "GET /metrics HTTP/1.1" 200 155660 "" "Prometheus/2.33.4" 2023-09-23T19:06:00.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:59 smithi016 bash[33624]: cluster 2023-09-23T19:05:58.908533+0000 mgr.y (mgr.14454) 105 : cluster [DBG] pgmap v83: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:00.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:59 smithi016 bash[33624]: cluster 2023-09-23T19:05:58.964955+0000 mon.a (mon.0) 645 : cluster [DBG] mds.? [v2:172.21.15.16:6834/161572387,v1:172.21.15.16:6835/161572387] up:active 2023-09-23T19:06:00.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:59 smithi016 bash[33624]: cluster 2023-09-23T19:05:58.965054+0000 mon.a (mon.0) 646 : cluster [DBG] fsmap foo:1 {0=foo.smithi016.uncssv=up:active} 2023-09-23T19:06:00.288 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:59 smithi016 bash[33624]: cluster 2023-09-23T19:05:59.935268+0000 mon.a (mon.0) 647 : cluster [DBG] mds.? [v2:172.21.15.92:6832/1118427195,v1:172.21.15.92:6833/1118427195] up:boot 2023-09-23T19:06:00.288 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:59 smithi016 bash[33624]: cluster 2023-09-23T19:05:59.935318+0000 mon.a (mon.0) 648 : cluster [DBG] fsmap foo:1 {0=foo.smithi016.uncssv=up:active} 1 up:standby 2023-09-23T19:06:00.288 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:05:59 smithi016 bash[33624]: audit 2023-09-23T19:05:59.941865+0000 mon.c (mon.2) 77 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi092.vbpqks"}]: dispatch 2023-09-23T19:06:00.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:59 smithi092 bash[27556]: cluster 2023-09-23T19:05:58.908533+0000 mgr.y (mgr.14454) 105 : cluster [DBG] pgmap v83: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:00.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:59 smithi092 bash[27556]: cluster 2023-09-23T19:05:58.964955+0000 mon.a (mon.0) 645 : cluster [DBG] mds.? [v2:172.21.15.16:6834/161572387,v1:172.21.15.16:6835/161572387] up:active 2023-09-23T19:06:00.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:59 smithi092 bash[27556]: cluster 2023-09-23T19:05:58.965054+0000 mon.a (mon.0) 646 : cluster [DBG] fsmap foo:1 {0=foo.smithi016.uncssv=up:active} 2023-09-23T19:06:00.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:59 smithi092 bash[27556]: cluster 2023-09-23T19:05:59.935268+0000 mon.a (mon.0) 647 : cluster [DBG] mds.? [v2:172.21.15.92:6832/1118427195,v1:172.21.15.92:6833/1118427195] up:boot 2023-09-23T19:06:00.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:59 smithi092 bash[27556]: cluster 2023-09-23T19:05:59.935318+0000 mon.a (mon.0) 648 : cluster [DBG] fsmap foo:1 {0=foo.smithi016.uncssv=up:active} 1 up:standby 2023-09-23T19:06:00.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:05:59 smithi092 bash[27556]: audit 2023-09-23T19:05:59.941865+0000 mon.c (mon.2) 77 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi092.vbpqks"}]: dispatch 2023-09-23T19:06:01.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:06:01 smithi016 bash[33624]: audit 2023-09-23T19:06:00.177598+0000 mon.a (mon.0) 649 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:06:01.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:06:01 smithi016 bash[33624]: cluster 2023-09-23T19:06:00.909165+0000 mgr.y (mgr.14454) 106 : cluster [DBG] pgmap v84: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:01.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:06:01 smithi016 bash[27237]: audit 2023-09-23T19:06:00.177598+0000 mon.a (mon.0) 649 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:06:01.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:06:01 smithi016 bash[27237]: cluster 2023-09-23T19:06:00.909165+0000 mgr.y (mgr.14454) 106 : cluster [DBG] pgmap v84: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:01.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:06:01 smithi092 bash[27556]: audit 2023-09-23T19:06:00.177598+0000 mon.a (mon.0) 649 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:06:01.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:06:01 smithi092 bash[27556]: cluster 2023-09-23T19:06:00.909165+0000 mgr.y (mgr.14454) 106 : cluster [DBG] pgmap v84: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:02.072 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-09-23T19:06:02.080 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-09-23T19:06:02.080 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-09-23T19:06:02.081 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph mgr dump --format=json 2023-09-23T19:06:04.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:06:03 smithi016 bash[33624]: cluster 2023-09-23T19:06:02.910339+0000 mgr.y (mgr.14454) 107 : cluster [DBG] pgmap v85: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:04.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:06:03 smithi016 bash[27237]: cluster 2023-09-23T19:06:02.910339+0000 mgr.y (mgr.14454) 107 : cluster [DBG] pgmap v85: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:04.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:06:03 smithi092 bash[27556]: cluster 2023-09-23T19:06:02.910339+0000 mgr.y (mgr.14454) 107 : cluster [DBG] pgmap v85: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:05.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:06:05 smithi016 bash[27237]: audit 2023-09-23T19:06:04.250410+0000 mon.a (mon.0) 650 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:06:05.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:06:05 smithi016 bash[27237]: audit 2023-09-23T19:06:04.535096+0000 mon.a (mon.0) 651 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:06:05.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:06:05 smithi016 bash[27237]: audit 2023-09-23T19:06:04.538581+0000 mon.c (mon.2) 78 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:06:05.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:06:05 smithi016 bash[27237]: audit 2023-09-23T19:06:04.539991+0000 mon.c (mon.2) 79 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:06:05.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:06:05 smithi016 bash[27237]: audit 2023-09-23T19:06:04.547238+0000 mon.a (mon.0) 652 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:06:05.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:06:05 smithi016 bash[27237]: audit 2023-09-23T19:06:04.556218+0000 mon.c (mon.2) 80 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:06:05.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:06:05 smithi016 bash[27237]: audit 2023-09-23T19:06:04.558408+0000 mon.c (mon.2) 81 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:06:05.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:06:05 smithi016 bash[27237]: audit 2023-09-23T19:06:04.559617+0000 mon.c (mon.2) 82 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:06:05.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:06:05 smithi016 bash[27237]: audit 2023-09-23T19:06:04.567289+0000 mon.a (mon.0) 653 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:06:05.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:06:05 smithi016 bash[27237]: cluster 2023-09-23T19:06:04.911059+0000 mgr.y (mgr.14454) 108 : cluster [DBG] pgmap v86: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:05.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:06:05 smithi016 bash[33624]: audit 2023-09-23T19:06:04.250410+0000 mon.a (mon.0) 650 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:06:05.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:06:05 smithi016 bash[33624]: audit 2023-09-23T19:06:04.535096+0000 mon.a (mon.0) 651 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:06:05.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:06:05 smithi016 bash[33624]: audit 2023-09-23T19:06:04.538581+0000 mon.c (mon.2) 78 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:06:05.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:06:05 smithi016 bash[33624]: audit 2023-09-23T19:06:04.539991+0000 mon.c (mon.2) 79 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:06:05.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:06:05 smithi016 bash[33624]: audit 2023-09-23T19:06:04.547238+0000 mon.a (mon.0) 652 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:06:05.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:06:05 smithi016 bash[33624]: audit 2023-09-23T19:06:04.556218+0000 mon.c (mon.2) 80 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:06:05.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:06:05 smithi016 bash[33624]: audit 2023-09-23T19:06:04.558408+0000 mon.c (mon.2) 81 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:06:05.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:06:05 smithi016 bash[33624]: audit 2023-09-23T19:06:04.559617+0000 mon.c (mon.2) 82 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:06:05.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:06:05 smithi016 bash[33624]: audit 2023-09-23T19:06:04.567289+0000 mon.a (mon.0) 653 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:06:05.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:06:05 smithi016 bash[33624]: cluster 2023-09-23T19:06:04.911059+0000 mgr.y (mgr.14454) 108 : cluster [DBG] pgmap v86: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:05.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:06:05 smithi092 bash[27556]: audit 2023-09-23T19:06:04.250410+0000 mon.a (mon.0) 650 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:06:05.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:06:05 smithi092 bash[27556]: audit 2023-09-23T19:06:04.535096+0000 mon.a (mon.0) 651 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:06:05.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:06:05 smithi092 bash[27556]: audit 2023-09-23T19:06:04.538581+0000 mon.c (mon.2) 78 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:06:05.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:06:05 smithi092 bash[27556]: audit 2023-09-23T19:06:04.539991+0000 mon.c (mon.2) 79 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:06:05.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:06:05 smithi092 bash[27556]: audit 2023-09-23T19:06:04.547238+0000 mon.a (mon.0) 652 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:06:05.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:06:05 smithi092 bash[27556]: audit 2023-09-23T19:06:04.556218+0000 mon.c (mon.2) 80 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:06:05.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:06:05 smithi092 bash[27556]: audit 2023-09-23T19:06:04.558408+0000 mon.c (mon.2) 81 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:06:05.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:06:05 smithi092 bash[27556]: audit 2023-09-23T19:06:04.559617+0000 mon.c (mon.2) 82 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:06:05.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:06:05 smithi092 bash[27556]: audit 2023-09-23T19:06:04.567289+0000 mon.a (mon.0) 653 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:06:05.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:06:05 smithi092 bash[27556]: cluster 2023-09-23T19:06:04.911059+0000 mgr.y (mgr.14454) 108 : cluster [DBG] pgmap v86: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:06.624 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:06:08.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:06:08 smithi016 bash[33624]: cluster 2023-09-23T19:06:06.912212+0000 mgr.y (mgr.14454) 109 : cluster [DBG] pgmap v87: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T19:06:08.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:06:07 smithi016 bash[27237]: cluster 2023-09-23T19:06:06.912212+0000 mgr.y (mgr.14454) 109 : cluster [DBG] pgmap v87: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T19:06:08.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:06:07 smithi092 bash[27556]: cluster 2023-09-23T19:06:06.912212+0000 mgr.y (mgr.14454) 109 : cluster [DBG] pgmap v87: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T19:06:09.537 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:06:09 smithi092 bash[28593]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:06:09] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T19:06:09.597 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-09-23T19:06:10.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:06:09 smithi016 bash[33624]: cluster 2023-09-23T19:06:08.912885+0000 mgr.y (mgr.14454) 110 : cluster [DBG] pgmap v88: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T19:06:10.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:06:09 smithi016 bash[33624]: audit 2023-09-23T19:06:09.584354+0000 mon.a (mon.0) 654 : audit [DBG] from='client.? 172.21.15.16:0/2061288348' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T19:06:10.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:06:09 smithi016 bash[27237]: cluster 2023-09-23T19:06:08.912885+0000 mgr.y (mgr.14454) 110 : cluster [DBG] pgmap v88: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T19:06:10.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:06:09 smithi016 bash[27237]: audit 2023-09-23T19:06:09.584354+0000 mon.a (mon.0) 654 : audit [DBG] from='client.? 172.21.15.16:0/2061288348' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T19:06:10.286 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:06:10 smithi016 bash[27503]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:06:10] "GET /metrics HTTP/1.1" 200 191346 "" "Prometheus/2.33.4" 2023-09-23T19:06:10.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:06:09 smithi092 bash[27556]: cluster 2023-09-23T19:06:08.912885+0000 mgr.y (mgr.14454) 110 : cluster [DBG] pgmap v88: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T19:06:10.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:06:09 smithi092 bash[27556]: audit 2023-09-23T19:06:09.584354+0000 mon.a (mon.0) 654 : audit [DBG] from='client.? 172.21.15.16:0/2061288348' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T19:06:10.307 INFO:teuthology.orchestra.run.smithi016.stdout:{"epoch":19,"active_gid":14454,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6800","nonce":1501185908},{"type":"v1","addr":"172.21.15.16:6801","nonce":1501185908}]},"active_addr":"172.21.15.16:6801/1501185908","active_change":"2023-09-23T19:03:34.846934+0000","active_mgr_features":4540138314316775423,"available":true,"standbys":[{"gid":24352,"name":"x","mgr_features":4540138314316775423,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.16:8443/","prometheus":"http://172.21.15.16:9283/"},"always_on_modules":{"nautilus":["balancer","crash","devicehealth","orchestrator_cli","progress","rbd_support","status","volumes"],"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":51,"active_clients":[{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.16:0","nonce":4152413938}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.16:0","nonce":3765291224}]}]} 2023-09-23T19:06:10.312 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-09-23T19:06:10.312 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-09-23T19:06:10.312 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T19:06:11.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:06:11 smithi016 bash[27237]: audit 2023-09-23T19:06:10.200344+0000 mon.a (mon.0) 655 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:06:11.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:06:11 smithi016 bash[27237]: cluster 2023-09-23T19:06:10.913542+0000 mgr.y (mgr.14454) 111 : cluster [DBG] pgmap v89: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T19:06:11.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:06:11 smithi016 bash[33624]: audit 2023-09-23T19:06:10.200344+0000 mon.a (mon.0) 655 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:06:11.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:06:11 smithi016 bash[33624]: cluster 2023-09-23T19:06:10.913542+0000 mgr.y (mgr.14454) 111 : cluster [DBG] pgmap v89: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T19:06:11.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:06:11 smithi092 bash[27556]: audit 2023-09-23T19:06:10.200344+0000 mon.a (mon.0) 655 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:06:11.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:06:11 smithi092 bash[27556]: cluster 2023-09-23T19:06:10.913542+0000 mgr.y (mgr.14454) 111 : cluster [DBG] pgmap v89: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T19:06:14.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:06:13 smithi016 bash[27237]: cluster 2023-09-23T19:06:12.914666+0000 mgr.y (mgr.14454) 112 : cluster [DBG] pgmap v90: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T19:06:14.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:06:13 smithi016 bash[33624]: cluster 2023-09-23T19:06:12.914666+0000 mgr.y (mgr.14454) 112 : cluster [DBG] pgmap v90: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T19:06:14.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:06:13 smithi092 bash[27556]: cluster 2023-09-23T19:06:12.914666+0000 mgr.y (mgr.14454) 112 : cluster [DBG] pgmap v90: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T19:06:15.105 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:06:16.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:06:15 smithi016 bash[27237]: cluster 2023-09-23T19:06:14.915330+0000 mgr.y (mgr.14454) 113 : cluster [DBG] pgmap v91: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T19:06:16.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:06:15 smithi016 bash[33624]: cluster 2023-09-23T19:06:14.915330+0000 mgr.y (mgr.14454) 113 : cluster [DBG] pgmap v91: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T19:06:16.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:06:15 smithi092 bash[27556]: cluster 2023-09-23T19:06:14.915330+0000 mgr.y (mgr.14454) 113 : cluster [DBG] pgmap v91: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T19:06:18.078 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-09-23T19:06:18.078 INFO:teuthology.orchestra.run.smithi016.stdout:{"epoch":59,"fsid":"95ce9eb6-5a42-11ee-8db4-212e2dc638e7","created":"2023-09-23T18:55:23.802934+0000","modified":"2023-09-23T19:05:53.118919+0000","last_up_change":"2023-09-23T19:03:02.463512+0000","last_in_change":"2023-09-23T19:02:36.618072+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-09-23T18:59:33.315416+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T19:04:01.440427+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T19:05:50.982524+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T19:05:51.538373+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"a4e8ed09-1195-4878-928a-13881bc04dc7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6802","nonce":2337435219},{"type":"v1","addr":"172.21.15.16:6803","nonce":2337435219}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6804","nonce":2337435219},{"type":"v1","addr":"172.21.15.16:6805","nonce":2337435219}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6808","nonce":2337435219},{"type":"v1","addr":"172.21.15.16:6809","nonce":2337435219}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6806","nonce":2337435219},{"type":"v1","addr":"172.21.15.16:6807","nonce":2337435219}]},"public_addr":"172.21.15.16:6803/2337435219","cluster_addr":"172.21.15.16:6805/2337435219","heartbeat_back_addr":"172.21.15.16:6809/2337435219","heartbeat_front_addr":"172.21.15.16:6807/2337435219","state":["exists","up"]},{"osd":1,"uuid":"c5ff2879-0d1c-481a-b847-248433c12aa4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6810","nonce":3893879931},{"type":"v1","addr":"172.21.15.16:6811","nonce":3893879931}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6812","nonce":3893879931},{"type":"v1","addr":"172.21.15.16:6813","nonce":3893879931}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6816","nonce":3893879931},{"type":"v1","addr":"172.21.15.16:6817","nonce":3893879931}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6814","nonce":3893879931},{"type":"v1","addr":"172.21.15.16:6815","nonce":3893879931}]},"public_addr":"172.21.15.16:6811/3893879931","cluster_addr":"172.21.15.16:6813/3893879931","heartbeat_back_addr":"172.21.15.16:6817/3893879931","heartbeat_front_addr":"172.21.15.16:6815/3893879931","state":["exists","up"]},{"osd":2,"uuid":"7974ce4d-9ed8-4a72-b44b-d500b34da8cf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6818","nonce":568840676},{"type":"v1","addr":"172.21.15.16:6819","nonce":568840676}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6820","nonce":568840676},{"type":"v1","addr":"172.21.15.16:6821","nonce":568840676}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6824","nonce":568840676},{"type":"v1","addr":"172.21.15.16:6825","nonce":568840676}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6822","nonce":568840676},{"type":"v1","addr":"172.21.15.16:6823","nonce":568840676}]},"public_addr":"172.21.15.16:6819/568840676","cluster_addr":"172.21.15.16:6821/568840676","heartbeat_back_addr":"172.21.15.16:6825/568840676","heartbeat_front_addr":"172.21.15.16:6823/568840676","state":["exists","up"]},{"osd":3,"uuid":"f5e0d57b-d629-4453-a527-ad77e6c210ad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6826","nonce":204922667},{"type":"v1","addr":"172.21.15.16:6827","nonce":204922667}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6828","nonce":204922667},{"type":"v1","addr":"172.21.15.16:6829","nonce":204922667}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6832","nonce":204922667},{"type":"v1","addr":"172.21.15.16:6833","nonce":204922667}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6830","nonce":204922667},{"type":"v1","addr":"172.21.15.16:6831","nonce":204922667}]},"public_addr":"172.21.15.16:6827/204922667","cluster_addr":"172.21.15.16:6829/204922667","heartbeat_back_addr":"172.21.15.16:6833/204922667","heartbeat_front_addr":"172.21.15.16:6831/204922667","state":["exists","up"]},{"osd":4,"uuid":"55a8dab2-6393-44ab-943e-cb7ab2cb7c31","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6800","nonce":99745912},{"type":"v1","addr":"172.21.15.92:6801","nonce":99745912}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6802","nonce":99745912},{"type":"v1","addr":"172.21.15.92:6803","nonce":99745912}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6806","nonce":99745912},{"type":"v1","addr":"172.21.15.92:6807","nonce":99745912}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6804","nonce":99745912},{"type":"v1","addr":"172.21.15.92:6805","nonce":99745912}]},"public_addr":"172.21.15.92:6801/99745912","cluster_addr":"172.21.15.92:6803/99745912","heartbeat_back_addr":"172.21.15.92:6807/99745912","heartbeat_front_addr":"172.21.15.92:6805/99745912","state":["exists","up"]},{"osd":5,"uuid":"b29b73b1-51d7-4484-bd79-2334c650cb62","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6808","nonce":1987214502},{"type":"v1","addr":"172.21.15.92:6809","nonce":1987214502}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6810","nonce":1987214502},{"type":"v1","addr":"172.21.15.92:6811","nonce":1987214502}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6814","nonce":1987214502},{"type":"v1","addr":"172.21.15.92:6815","nonce":1987214502}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6812","nonce":1987214502},{"type":"v1","addr":"172.21.15.92:6813","nonce":1987214502}]},"public_addr":"172.21.15.92:6809/1987214502","cluster_addr":"172.21.15.92:6811/1987214502","heartbeat_back_addr":"172.21.15.92:6815/1987214502","heartbeat_front_addr":"172.21.15.92:6813/1987214502","state":["exists","up"]},{"osd":6,"uuid":"9fe16cc9-f0d3-4b79-bc68-38305c55b551","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6816","nonce":2820798178},{"type":"v1","addr":"172.21.15.92:6817","nonce":2820798178}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6818","nonce":2820798178},{"type":"v1","addr":"172.21.15.92:6819","nonce":2820798178}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6822","nonce":2820798178},{"type":"v1","addr":"172.21.15.92:6823","nonce":2820798178}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6820","nonce":2820798178},{"type":"v1","addr":"172.21.15.92:6821","nonce":2820798178}]},"public_addr":"172.21.15.92:6817/2820798178","cluster_addr":"172.21.15.92:6819/2820798178","heartbeat_back_addr":"172.21.15.92:6823/2820798178","heartbeat_front_addr":"172.21.15.92:6821/2820798178","state":["exists","up"]},{"osd":7,"uuid":"d086b076-127f-42de-bee9-157bf579d92d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6824","nonce":98624761},{"type":"v1","addr":"172.21.15.92:6825","nonce":98624761}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6826","nonce":98624761},{"type":"v1","addr":"172.21.15.92:6827","nonce":98624761}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6830","nonce":98624761},{"type":"v1","addr":"172.21.15.92:6831","nonce":98624761}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6828","nonce":98624761},{"type":"v1","addr":"172.21.15.92:6829","nonce":98624761}]},"public_addr":"172.21.15.92:6825/98624761","cluster_addr":"172.21.15.92:6827/98624761","heartbeat_back_addr":"172.21.15.92:6831/98624761","heartbeat_front_addr":"172.21.15.92:6829/98624761","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:58:07.335980+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:58:49.342213+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:59:30.939986+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:00:12.926364+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:00:53.466692+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:01:36.332957+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:02:18.593854+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:03:00.817448+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.16:6801/1259630240":"2023-09-24T19:03:34.846451+0000","172.21.15.16:6800/1259630240":"2023-09-24T19:03:34.846451+0000","172.21.15.16:0/3767844203":"2023-09-24T19:03:34.846451+0000","172.21.15.16:6801/2756103674":"2023-09-24T18:56:20.574625+0000","172.21.15.16:0/2843459289":"2023-09-24T18:56:20.574625+0000","172.21.15.16:0/272242693":"2023-09-24T18:56:20.574625+0000","172.21.15.16:0/79169563":"2023-09-24T18:55:50.550546+0000","172.21.15.16:6800/2615282850":"2023-09-24T18:55:50.550546+0000","172.21.15.16:0/349446544":"2023-09-24T18:55:50.550546+0000","172.21.15.16:0/2669353350":"2023-09-24T19:03:34.846451+0000","172.21.15.16:6800/2756103674":"2023-09-24T18:56:20.574625+0000","172.21.15.16:6801/2615282850":"2023-09-24T18:55:50.550546+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T19:06:18.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:06:17 smithi016 bash[27237]: cluster 2023-09-23T19:06:16.916494+0000 mgr.y (mgr.14454) 114 : cluster [DBG] pgmap v92: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T19:06:18.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:06:17 smithi016 bash[33624]: cluster 2023-09-23T19:06:16.916494+0000 mgr.y (mgr.14454) 114 : cluster [DBG] pgmap v92: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T19:06:18.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:06:17 smithi092 bash[27556]: cluster 2023-09-23T19:06:16.916494+0000 mgr.y (mgr.14454) 114 : cluster [DBG] pgmap v92: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T19:06:18.808 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-09-23T19:06:18.809 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T19:06:19.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:06:18 smithi016 bash[27237]: audit 2023-09-23T19:06:18.072975+0000 mon.c (mon.2) 83 : audit [DBG] from='client.? 172.21.15.16:0/1533078711' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T19:06:19.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:06:18 smithi016 bash[33624]: audit 2023-09-23T19:06:18.072975+0000 mon.c (mon.2) 83 : audit [DBG] from='client.? 172.21.15.16:0/1533078711' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T19:06:19.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:06:18 smithi092 bash[27556]: audit 2023-09-23T19:06:18.072975+0000 mon.c (mon.2) 83 : audit [DBG] from='client.? 172.21.15.16:0/1533078711' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T19:06:19.287 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:06:19 smithi092 bash[28593]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:06:19] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T19:06:20.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:06:19 smithi016 bash[27237]: cluster 2023-09-23T19:06:18.917211+0000 mgr.y (mgr.14454) 115 : cluster [DBG] pgmap v93: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:20.285 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:06:20 smithi016 bash[27503]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:06:20] "GET /metrics HTTP/1.1" 200 191346 "" "Prometheus/2.33.4" 2023-09-23T19:06:20.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:06:19 smithi016 bash[33624]: cluster 2023-09-23T19:06:18.917211+0000 mgr.y (mgr.14454) 115 : cluster [DBG] pgmap v93: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:20.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:06:19 smithi092 bash[27556]: cluster 2023-09-23T19:06:18.917211+0000 mgr.y (mgr.14454) 115 : cluster [DBG] pgmap v93: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:22.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:06:21 smithi016 bash[27237]: cluster 2023-09-23T19:06:20.917876+0000 mgr.y (mgr.14454) 116 : cluster [DBG] pgmap v94: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:22.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:06:21 smithi016 bash[33624]: cluster 2023-09-23T19:06:20.917876+0000 mgr.y (mgr.14454) 116 : cluster [DBG] pgmap v94: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:22.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:06:22 smithi092 bash[27556]: cluster 2023-09-23T19:06:20.917876+0000 mgr.y (mgr.14454) 116 : cluster [DBG] pgmap v94: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:23.591 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:06:24.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:06:24 smithi016 bash[27237]: cluster 2023-09-23T19:06:22.918963+0000 mgr.y (mgr.14454) 117 : cluster [DBG] pgmap v95: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:24.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:06:24 smithi016 bash[33624]: cluster 2023-09-23T19:06:22.918963+0000 mgr.y (mgr.14454) 117 : cluster [DBG] pgmap v95: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:24.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:06:24 smithi092 bash[27556]: cluster 2023-09-23T19:06:22.918963+0000 mgr.y (mgr.14454) 117 : cluster [DBG] pgmap v95: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:26.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:06:26 smithi016 bash[27237]: cluster 2023-09-23T19:06:24.919614+0000 mgr.y (mgr.14454) 118 : cluster [DBG] pgmap v96: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:26.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:06:26 smithi016 bash[33624]: cluster 2023-09-23T19:06:24.919614+0000 mgr.y (mgr.14454) 118 : cluster [DBG] pgmap v96: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:26.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:06:26 smithi092 bash[27556]: cluster 2023-09-23T19:06:24.919614+0000 mgr.y (mgr.14454) 118 : cluster [DBG] pgmap v96: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:26.412 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-09-23T19:06:26.412 INFO:teuthology.orchestra.run.smithi016.stdout:{"epoch":59,"fsid":"95ce9eb6-5a42-11ee-8db4-212e2dc638e7","created":"2023-09-23T18:55:23.802934+0000","modified":"2023-09-23T19:05:53.118919+0000","last_up_change":"2023-09-23T19:03:02.463512+0000","last_in_change":"2023-09-23T19:02:36.618072+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-09-23T18:59:33.315416+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T19:04:01.440427+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T19:05:50.982524+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T19:05:51.538373+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"a4e8ed09-1195-4878-928a-13881bc04dc7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6802","nonce":2337435219},{"type":"v1","addr":"172.21.15.16:6803","nonce":2337435219}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6804","nonce":2337435219},{"type":"v1","addr":"172.21.15.16:6805","nonce":2337435219}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6808","nonce":2337435219},{"type":"v1","addr":"172.21.15.16:6809","nonce":2337435219}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6806","nonce":2337435219},{"type":"v1","addr":"172.21.15.16:6807","nonce":2337435219}]},"public_addr":"172.21.15.16:6803/2337435219","cluster_addr":"172.21.15.16:6805/2337435219","heartbeat_back_addr":"172.21.15.16:6809/2337435219","heartbeat_front_addr":"172.21.15.16:6807/2337435219","state":["exists","up"]},{"osd":1,"uuid":"c5ff2879-0d1c-481a-b847-248433c12aa4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6810","nonce":3893879931},{"type":"v1","addr":"172.21.15.16:6811","nonce":3893879931}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6812","nonce":3893879931},{"type":"v1","addr":"172.21.15.16:6813","nonce":3893879931}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6816","nonce":3893879931},{"type":"v1","addr":"172.21.15.16:6817","nonce":3893879931}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6814","nonce":3893879931},{"type":"v1","addr":"172.21.15.16:6815","nonce":3893879931}]},"public_addr":"172.21.15.16:6811/3893879931","cluster_addr":"172.21.15.16:6813/3893879931","heartbeat_back_addr":"172.21.15.16:6817/3893879931","heartbeat_front_addr":"172.21.15.16:6815/3893879931","state":["exists","up"]},{"osd":2,"uuid":"7974ce4d-9ed8-4a72-b44b-d500b34da8cf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6818","nonce":568840676},{"type":"v1","addr":"172.21.15.16:6819","nonce":568840676}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6820","nonce":568840676},{"type":"v1","addr":"172.21.15.16:6821","nonce":568840676}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6824","nonce":568840676},{"type":"v1","addr":"172.21.15.16:6825","nonce":568840676}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6822","nonce":568840676},{"type":"v1","addr":"172.21.15.16:6823","nonce":568840676}]},"public_addr":"172.21.15.16:6819/568840676","cluster_addr":"172.21.15.16:6821/568840676","heartbeat_back_addr":"172.21.15.16:6825/568840676","heartbeat_front_addr":"172.21.15.16:6823/568840676","state":["exists","up"]},{"osd":3,"uuid":"f5e0d57b-d629-4453-a527-ad77e6c210ad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6826","nonce":204922667},{"type":"v1","addr":"172.21.15.16:6827","nonce":204922667}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6828","nonce":204922667},{"type":"v1","addr":"172.21.15.16:6829","nonce":204922667}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6832","nonce":204922667},{"type":"v1","addr":"172.21.15.16:6833","nonce":204922667}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6830","nonce":204922667},{"type":"v1","addr":"172.21.15.16:6831","nonce":204922667}]},"public_addr":"172.21.15.16:6827/204922667","cluster_addr":"172.21.15.16:6829/204922667","heartbeat_back_addr":"172.21.15.16:6833/204922667","heartbeat_front_addr":"172.21.15.16:6831/204922667","state":["exists","up"]},{"osd":4,"uuid":"55a8dab2-6393-44ab-943e-cb7ab2cb7c31","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6800","nonce":99745912},{"type":"v1","addr":"172.21.15.92:6801","nonce":99745912}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6802","nonce":99745912},{"type":"v1","addr":"172.21.15.92:6803","nonce":99745912}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6806","nonce":99745912},{"type":"v1","addr":"172.21.15.92:6807","nonce":99745912}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6804","nonce":99745912},{"type":"v1","addr":"172.21.15.92:6805","nonce":99745912}]},"public_addr":"172.21.15.92:6801/99745912","cluster_addr":"172.21.15.92:6803/99745912","heartbeat_back_addr":"172.21.15.92:6807/99745912","heartbeat_front_addr":"172.21.15.92:6805/99745912","state":["exists","up"]},{"osd":5,"uuid":"b29b73b1-51d7-4484-bd79-2334c650cb62","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6808","nonce":1987214502},{"type":"v1","addr":"172.21.15.92:6809","nonce":1987214502}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6810","nonce":1987214502},{"type":"v1","addr":"172.21.15.92:6811","nonce":1987214502}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6814","nonce":1987214502},{"type":"v1","addr":"172.21.15.92:6815","nonce":1987214502}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6812","nonce":1987214502},{"type":"v1","addr":"172.21.15.92:6813","nonce":1987214502}]},"public_addr":"172.21.15.92:6809/1987214502","cluster_addr":"172.21.15.92:6811/1987214502","heartbeat_back_addr":"172.21.15.92:6815/1987214502","heartbeat_front_addr":"172.21.15.92:6813/1987214502","state":["exists","up"]},{"osd":6,"uuid":"9fe16cc9-f0d3-4b79-bc68-38305c55b551","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6816","nonce":2820798178},{"type":"v1","addr":"172.21.15.92:6817","nonce":2820798178}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6818","nonce":2820798178},{"type":"v1","addr":"172.21.15.92:6819","nonce":2820798178}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6822","nonce":2820798178},{"type":"v1","addr":"172.21.15.92:6823","nonce":2820798178}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6820","nonce":2820798178},{"type":"v1","addr":"172.21.15.92:6821","nonce":2820798178}]},"public_addr":"172.21.15.92:6817/2820798178","cluster_addr":"172.21.15.92:6819/2820798178","heartbeat_back_addr":"172.21.15.92:6823/2820798178","heartbeat_front_addr":"172.21.15.92:6821/2820798178","state":["exists","up"]},{"osd":7,"uuid":"d086b076-127f-42de-bee9-157bf579d92d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6824","nonce":98624761},{"type":"v1","addr":"172.21.15.92:6825","nonce":98624761}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6826","nonce":98624761},{"type":"v1","addr":"172.21.15.92:6827","nonce":98624761}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6830","nonce":98624761},{"type":"v1","addr":"172.21.15.92:6831","nonce":98624761}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6828","nonce":98624761},{"type":"v1","addr":"172.21.15.92:6829","nonce":98624761}]},"public_addr":"172.21.15.92:6825/98624761","cluster_addr":"172.21.15.92:6827/98624761","heartbeat_back_addr":"172.21.15.92:6831/98624761","heartbeat_front_addr":"172.21.15.92:6829/98624761","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:58:07.335980+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:58:49.342213+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:59:30.939986+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:00:12.926364+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:00:53.466692+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:01:36.332957+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:02:18.593854+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:03:00.817448+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.16:6801/1259630240":"2023-09-24T19:03:34.846451+0000","172.21.15.16:6800/1259630240":"2023-09-24T19:03:34.846451+0000","172.21.15.16:0/3767844203":"2023-09-24T19:03:34.846451+0000","172.21.15.16:6801/2756103674":"2023-09-24T18:56:20.574625+0000","172.21.15.16:0/2843459289":"2023-09-24T18:56:20.574625+0000","172.21.15.16:0/272242693":"2023-09-24T18:56:20.574625+0000","172.21.15.16:0/79169563":"2023-09-24T18:55:50.550546+0000","172.21.15.16:6800/2615282850":"2023-09-24T18:55:50.550546+0000","172.21.15.16:0/349446544":"2023-09-24T18:55:50.550546+0000","172.21.15.16:0/2669353350":"2023-09-24T19:03:34.846451+0000","172.21.15.16:6800/2756103674":"2023-09-24T18:56:20.574625+0000","172.21.15.16:6801/2615282850":"2023-09-24T18:55:50.550546+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T19:06:27.226 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-09-23T19:06:27.227 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-09-23T19:06:27.227 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-09-23T19:06:27.228 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-09-23T19:06:27.228 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-09-23T19:06:27.229 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-09-23T19:06:27.229 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-09-23T19:06:27.229 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-09-23T19:06:27.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:06:27 smithi016 bash[27237]: audit 2023-09-23T19:06:26.407037+0000 mon.c (mon.2) 84 : audit [DBG] from='client.? 172.21.15.16:0/521174525' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T19:06:27.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:06:27 smithi016 bash[33624]: audit 2023-09-23T19:06:26.407037+0000 mon.c (mon.2) 84 : audit [DBG] from='client.? 172.21.15.16:0/521174525' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T19:06:27.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:06:27 smithi092 bash[27556]: audit 2023-09-23T19:06:26.407037+0000 mon.c (mon.2) 84 : audit [DBG] from='client.? 172.21.15.16:0/521174525' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T19:06:28.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:06:28 smithi016 bash[27237]: cluster 2023-09-23T19:06:26.920764+0000 mgr.y (mgr.14454) 119 : cluster [DBG] pgmap v97: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:28.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:06:28 smithi016 bash[33624]: cluster 2023-09-23T19:06:26.920764+0000 mgr.y (mgr.14454) 119 : cluster [DBG] pgmap v97: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:28.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:06:28 smithi092 bash[27556]: cluster 2023-09-23T19:06:26.920764+0000 mgr.y (mgr.14454) 119 : cluster [DBG] pgmap v97: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:29.537 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:06:29 smithi092 bash[28593]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:06:29] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T19:06:30.285 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:06:30 smithi016 bash[27503]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:06:30] "GET /metrics HTTP/1.1" 200 191343 "" "Prometheus/2.33.4" 2023-09-23T19:06:30.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:06:30 smithi016 bash[33624]: cluster 2023-09-23T19:06:28.921435+0000 mgr.y (mgr.14454) 120 : cluster [DBG] pgmap v98: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:30.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:06:30 smithi016 bash[27237]: cluster 2023-09-23T19:06:28.921435+0000 mgr.y (mgr.14454) 120 : cluster [DBG] pgmap v98: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:30.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:06:30 smithi092 bash[27556]: cluster 2023-09-23T19:06:28.921435+0000 mgr.y (mgr.14454) 120 : cluster [DBG] pgmap v98: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:32.125 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:06:32.126 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:06:32.126 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:06:32.127 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:06:32.127 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:06:32.128 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:06:32.134 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:06:32.136 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:06:32.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:06:32 smithi016 bash[27237]: cluster 2023-09-23T19:06:30.922110+0000 mgr.y (mgr.14454) 121 : cluster [DBG] pgmap v99: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:32.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:06:32 smithi016 bash[33624]: cluster 2023-09-23T19:06:30.922110+0000 mgr.y (mgr.14454) 121 : cluster [DBG] pgmap v99: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:32.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:06:32 smithi092 bash[27556]: cluster 2023-09-23T19:06:30.922110+0000 mgr.y (mgr.14454) 121 : cluster [DBG] pgmap v99: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:34.236 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:06:34 smithi016 bash[33624]: cluster 2023-09-23T19:06:32.923248+0000 mgr.y (mgr.14454) 122 : cluster [DBG] pgmap v100: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:34.236 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:06:34 smithi016 bash[27237]: cluster 2023-09-23T19:06:32.923248+0000 mgr.y (mgr.14454) 122 : cluster [DBG] pgmap v100: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:34.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:06:34 smithi092 bash[27556]: cluster 2023-09-23T19:06:32.923248+0000 mgr.y (mgr.14454) 122 : cluster [DBG] pgmap v100: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:35.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:06:35 smithi016 bash[33624]: audit 2023-09-23T19:06:34.909102+0000 mon.c (mon.2) 85 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2023-09-23T19:06:35.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:06:35 smithi016 bash[33624]: audit 2023-09-23T19:06:34.909479+0000 mon.c (mon.2) 86 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2023-09-23T19:06:35.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:06:35 smithi016 bash[33624]: audit 2023-09-23T19:06:34.909863+0000 mon.a (mon.0) 656 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2023-09-23T19:06:35.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:06:35 smithi016 bash[33624]: audit 2023-09-23T19:06:34.909991+0000 mon.c (mon.2) 87 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2023-09-23T19:06:35.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:06:35 smithi016 bash[33624]: audit 2023-09-23T19:06:34.910737+0000 mon.a (mon.0) 657 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2023-09-23T19:06:35.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:06:35 smithi016 bash[33624]: audit 2023-09-23T19:06:34.911066+0000 mon.a (mon.0) 658 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2023-09-23T19:06:35.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:06:35 smithi016 bash[27237]: audit 2023-09-23T19:06:34.909102+0000 mon.c (mon.2) 85 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2023-09-23T19:06:35.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:06:35 smithi016 bash[27237]: audit 2023-09-23T19:06:34.909479+0000 mon.c (mon.2) 86 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2023-09-23T19:06:35.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:06:35 smithi016 bash[27237]: audit 2023-09-23T19:06:34.909863+0000 mon.a (mon.0) 656 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2023-09-23T19:06:35.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:06:35 smithi016 bash[27237]: audit 2023-09-23T19:06:34.909991+0000 mon.c (mon.2) 87 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2023-09-23T19:06:35.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:06:35 smithi016 bash[27237]: audit 2023-09-23T19:06:34.910737+0000 mon.a (mon.0) 657 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2023-09-23T19:06:35.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:06:35 smithi016 bash[27237]: audit 2023-09-23T19:06:34.911066+0000 mon.a (mon.0) 658 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2023-09-23T19:06:35.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:06:35 smithi092 bash[27556]: audit 2023-09-23T19:06:34.909102+0000 mon.c (mon.2) 85 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2023-09-23T19:06:35.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:06:35 smithi092 bash[27556]: audit 2023-09-23T19:06:34.909479+0000 mon.c (mon.2) 86 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2023-09-23T19:06:35.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:06:35 smithi092 bash[27556]: audit 2023-09-23T19:06:34.909863+0000 mon.a (mon.0) 656 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2023-09-23T19:06:35.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:06:35 smithi092 bash[27556]: audit 2023-09-23T19:06:34.909991+0000 mon.c (mon.2) 87 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2023-09-23T19:06:35.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:06:35 smithi092 bash[27556]: audit 2023-09-23T19:06:34.910737+0000 mon.a (mon.0) 657 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2023-09-23T19:06:35.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:06:35 smithi092 bash[27556]: audit 2023-09-23T19:06:34.911066+0000 mon.a (mon.0) 658 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2023-09-23T19:06:36.380 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:06:36 smithi016 bash[27237]: cluster 2023-09-23T19:06:34.923880+0000 mgr.y (mgr.14454) 123 : cluster [DBG] pgmap v101: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:36.380 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:06:36 smithi016 bash[27237]: audit 2023-09-23T19:06:35.043797+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2023-09-23T19:06:36.380 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:06:36 smithi016 bash[27237]: audit 2023-09-23T19:06:35.043979+0000 mon.a (mon.0) 660 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]': finished 2023-09-23T19:06:36.380 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:06:36 smithi016 bash[27237]: audit 2023-09-23T19:06:35.044294+0000 mon.a (mon.0) 661 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2023-09-23T19:06:36.380 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:06:36 smithi016 bash[27237]: cluster 2023-09-23T19:06:35.044456+0000 mon.a (mon.0) 662 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2023-09-23T19:06:36.381 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:06:36 smithi016 bash[33624]: cluster 2023-09-23T19:06:34.923880+0000 mgr.y (mgr.14454) 123 : cluster [DBG] pgmap v101: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:36.381 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:06:36 smithi016 bash[33624]: audit 2023-09-23T19:06:35.043797+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2023-09-23T19:06:36.381 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:06:36 smithi016 bash[33624]: audit 2023-09-23T19:06:35.043979+0000 mon.a (mon.0) 660 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]': finished 2023-09-23T19:06:36.381 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:06:36 smithi016 bash[33624]: audit 2023-09-23T19:06:35.044294+0000 mon.a (mon.0) 661 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2023-09-23T19:06:36.381 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:06:36 smithi016 bash[33624]: cluster 2023-09-23T19:06:35.044456+0000 mon.a (mon.0) 662 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2023-09-23T19:06:36.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:06:36 smithi092 bash[27556]: cluster 2023-09-23T19:06:34.923880+0000 mgr.y (mgr.14454) 123 : cluster [DBG] pgmap v101: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:36.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:06:36 smithi092 bash[27556]: audit 2023-09-23T19:06:35.043797+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2023-09-23T19:06:36.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:06:36 smithi092 bash[27556]: audit 2023-09-23T19:06:35.043979+0000 mon.a (mon.0) 660 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]': finished 2023-09-23T19:06:36.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:06:36 smithi092 bash[27556]: audit 2023-09-23T19:06:35.044294+0000 mon.a (mon.0) 661 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2023-09-23T19:06:36.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:06:36 smithi092 bash[27556]: cluster 2023-09-23T19:06:35.044456+0000 mon.a (mon.0) 662 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2023-09-23T19:06:37.193 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:06:37 smithi016 bash[27237]: cluster 2023-09-23T19:06:36.043552+0000 mon.a (mon.0) 663 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2023-09-23T19:06:37.193 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:06:37 smithi016 bash[33624]: cluster 2023-09-23T19:06:36.043552+0000 mon.a (mon.0) 663 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2023-09-23T19:06:37.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:06:37 smithi092 bash[27556]: cluster 2023-09-23T19:06:36.043552+0000 mon.a (mon.0) 663 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2023-09-23T19:06:38.195 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:06:38 smithi016 bash[27237]: cluster 2023-09-23T19:06:36.924436+0000 mgr.y (mgr.14454) 124 : cluster [DBG] pgmap v104: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:38.196 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:06:38 smithi016 bash[33624]: cluster 2023-09-23T19:06:36.924436+0000 mgr.y (mgr.14454) 124 : cluster [DBG] pgmap v104: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:38.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:06:38 smithi092 bash[27556]: cluster 2023-09-23T19:06:36.924436+0000 mgr.y (mgr.14454) 124 : cluster [DBG] pgmap v104: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:39.537 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:06:39 smithi092 bash[28593]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:06:39] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T19:06:40.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:06:40 smithi016 bash[33624]: cluster 2023-09-23T19:06:38.924944+0000 mgr.y (mgr.14454) 125 : cluster [DBG] pgmap v105: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:40.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:06:40 smithi016 bash[27237]: cluster 2023-09-23T19:06:38.924944+0000 mgr.y (mgr.14454) 125 : cluster [DBG] pgmap v105: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:40.286 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:06:40 smithi016 bash[27503]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:06:40] "GET /metrics HTTP/1.1" 200 191342 "" "Prometheus/2.33.4" 2023-09-23T19:06:40.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:06:40 smithi092 bash[27556]: cluster 2023-09-23T19:06:38.924944+0000 mgr.y (mgr.14454) 125 : cluster [DBG] pgmap v105: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:42.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:06:42 smithi016 bash[33624]: cluster 2023-09-23T19:06:40.925488+0000 mgr.y (mgr.14454) 126 : cluster [DBG] pgmap v106: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:42.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:06:42 smithi016 bash[27237]: cluster 2023-09-23T19:06:40.925488+0000 mgr.y (mgr.14454) 126 : cluster [DBG] pgmap v106: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:42.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:06:42 smithi092 bash[27556]: cluster 2023-09-23T19:06:40.925488+0000 mgr.y (mgr.14454) 126 : cluster [DBG] pgmap v106: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:44.173 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:06:44 smithi016 bash[33624]: cluster 2023-09-23T19:06:42.926522+0000 mgr.y (mgr.14454) 127 : cluster [DBG] pgmap v107: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:44.174 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:06:44 smithi016 bash[27237]: cluster 2023-09-23T19:06:42.926522+0000 mgr.y (mgr.14454) 127 : cluster [DBG] pgmap v107: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:44.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:06:44 smithi092 bash[27556]: cluster 2023-09-23T19:06:42.926522+0000 mgr.y (mgr.14454) 127 : cluster [DBG] pgmap v107: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:45.414 INFO:teuthology.orchestra.run.smithi016.stdout:206158430255 2023-09-23T19:06:45.414 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-09-23T19:06:45.465 INFO:teuthology.orchestra.run.smithi016.stdout:77309411415 2023-09-23T19:06:45.466 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-09-23T19:06:46.124 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:06:46 smithi016 bash[33624]: cluster 2023-09-23T19:06:44.927253+0000 mgr.y (mgr.14454) 128 : cluster [DBG] pgmap v108: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:46.124 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:06:46 smithi016 bash[27237]: cluster 2023-09-23T19:06:44.927253+0000 mgr.y (mgr.14454) 128 : cluster [DBG] pgmap v108: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:46.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:06:46 smithi092 bash[27556]: cluster 2023-09-23T19:06:44.927253+0000 mgr.y (mgr.14454) 128 : cluster [DBG] pgmap v108: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:47.301 INFO:teuthology.orchestra.run.smithi016.stdout:180388626487 2023-09-23T19:06:47.301 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-09-23T19:06:47.972 INFO:teuthology.orchestra.run.smithi016.stdout:107374182481 2023-09-23T19:06:47.972 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-09-23T19:06:48.166 INFO:teuthology.orchestra.run.smithi016.stdout:154618822719 2023-09-23T19:06:48.167 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-09-23T19:06:48.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:06:48 smithi016 bash[27237]: cluster 2023-09-23T19:06:46.928385+0000 mgr.y (mgr.14454) 129 : cluster [DBG] pgmap v109: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:48.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:06:48 smithi016 bash[33624]: cluster 2023-09-23T19:06:46.928385+0000 mgr.y (mgr.14454) 129 : cluster [DBG] pgmap v109: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:48.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:06:48 smithi092 bash[27556]: cluster 2023-09-23T19:06:46.928385+0000 mgr.y (mgr.14454) 129 : cluster [DBG] pgmap v109: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:49.143 INFO:teuthology.orchestra.run.smithi016.stdout:128849018953 2023-09-23T19:06:49.143 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-09-23T19:06:49.198 INFO:teuthology.orchestra.run.smithi016.stdout:55834574947 2023-09-23T19:06:49.199 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-09-23T19:06:49.211 INFO:teuthology.orchestra.run.smithi016.stdout:34359738474 2023-09-23T19:06:49.212 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-09-23T19:06:49.537 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:06:49 smithi092 bash[28593]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:06:49] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T19:06:50.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:06:50 smithi016 bash[27237]: cluster 2023-09-23T19:06:48.929101+0000 mgr.y (mgr.14454) 130 : cluster [DBG] pgmap v110: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:50.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:06:50 smithi016 bash[33624]: cluster 2023-09-23T19:06:48.929101+0000 mgr.y (mgr.14454) 130 : cluster [DBG] pgmap v110: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:50.536 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:06:50 smithi016 bash[27503]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:06:50] "GET /metrics HTTP/1.1" 200 191342 "" "Prometheus/2.33.4" 2023-09-23T19:06:50.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:06:50 smithi092 bash[27556]: cluster 2023-09-23T19:06:48.929101+0000 mgr.y (mgr.14454) 130 : cluster [DBG] pgmap v110: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:51.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:06:51 smithi016 bash[27237]: cluster 2023-09-23T19:06:50.929963+0000 mgr.y (mgr.14454) 131 : cluster [DBG] pgmap v111: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:51.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:06:51 smithi016 bash[33624]: cluster 2023-09-23T19:06:50.929963+0000 mgr.y (mgr.14454) 131 : cluster [DBG] pgmap v111: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:51.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:06:51 smithi092 bash[27556]: cluster 2023-09-23T19:06:50.929963+0000 mgr.y (mgr.14454) 131 : cluster [DBG] pgmap v111: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:52.266 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:06:52.266 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:06:52.267 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:06:52.269 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:06:52.270 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:06:53.282 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:06:53.283 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:06:53.283 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:06:54.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:06:53 smithi016 bash[27237]: cluster 2023-09-23T19:06:52.930412+0000 mgr.y (mgr.14454) 132 : cluster [DBG] pgmap v112: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:54.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:06:53 smithi016 bash[33624]: cluster 2023-09-23T19:06:52.930412+0000 mgr.y (mgr.14454) 132 : cluster [DBG] pgmap v112: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:54.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:06:53 smithi092 bash[27556]: cluster 2023-09-23T19:06:52.930412+0000 mgr.y (mgr.14454) 132 : cluster [DBG] pgmap v112: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:56.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:06:55 smithi092 bash[27556]: cluster 2023-09-23T19:06:54.930767+0000 mgr.y (mgr.14454) 133 : cluster [DBG] pgmap v113: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:56.320 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:06:55 smithi016 bash[27237]: cluster 2023-09-23T19:06:54.930767+0000 mgr.y (mgr.14454) 133 : cluster [DBG] pgmap v113: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:56.320 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:06:55 smithi016 bash[33624]: cluster 2023-09-23T19:06:54.930767+0000 mgr.y (mgr.14454) 133 : cluster [DBG] pgmap v113: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:58.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:06:57 smithi016 bash[27237]: cluster 2023-09-23T19:06:56.931543+0000 mgr.y (mgr.14454) 134 : cluster [DBG] pgmap v114: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:58.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:06:57 smithi016 bash[33624]: cluster 2023-09-23T19:06:56.931543+0000 mgr.y (mgr.14454) 134 : cluster [DBG] pgmap v114: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:58.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:06:58 smithi092 bash[27556]: cluster 2023-09-23T19:06:56.931543+0000 mgr.y (mgr.14454) 134 : cluster [DBG] pgmap v114: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:06:59.537 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:06:59 smithi092 bash[28593]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:06:59] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T19:07:00.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:00 smithi092 bash[27556]: cluster 2023-09-23T19:06:58.932128+0000 mgr.y (mgr.14454) 135 : cluster [DBG] pgmap v115: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:00.384 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:00 smithi016 bash[27237]: cluster 2023-09-23T19:06:58.932128+0000 mgr.y (mgr.14454) 135 : cluster [DBG] pgmap v115: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:00.385 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:07:00 smithi016 bash[27503]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:07:00] "GET /metrics HTTP/1.1" 200 191363 "" "Prometheus/2.33.4" 2023-09-23T19:07:00.385 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:00 smithi016 bash[33624]: cluster 2023-09-23T19:06:58.932128+0000 mgr.y (mgr.14454) 135 : cluster [DBG] pgmap v115: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:02.049 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:02 smithi016 bash[27237]: cluster 2023-09-23T19:07:00.933012+0000 mgr.y (mgr.14454) 136 : cluster [DBG] pgmap v116: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:02.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:02 smithi092 bash[27556]: cluster 2023-09-23T19:07:00.933012+0000 mgr.y (mgr.14454) 136 : cluster [DBG] pgmap v116: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:02.622 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:02 smithi016 bash[33624]: cluster 2023-09-23T19:07:00.933012+0000 mgr.y (mgr.14454) 136 : cluster [DBG] pgmap v116: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:03.128 INFO:teuthology.orchestra.run.smithi016.stdout:206158430259 2023-09-23T19:07:04.101 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:04 smithi016 bash[27237]: cluster 2023-09-23T19:07:02.933572+0000 mgr.y (mgr.14454) 137 : cluster [DBG] pgmap v117: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:04.101 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:04 smithi016 bash[27237]: audit 2023-09-23T19:07:03.125447+0000 mon.a (mon.0) 664 : audit [DBG] from='client.? 172.21.15.16:0/1940484959' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T19:07:04.203 INFO:teuthology.orchestra.run.smithi016.stdout:77309411419 2023-09-23T19:07:04.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:04 smithi092 bash[27556]: cluster 2023-09-23T19:07:02.933572+0000 mgr.y (mgr.14454) 137 : cluster [DBG] pgmap v117: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:04.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:04 smithi092 bash[27556]: audit 2023-09-23T19:07:03.125447+0000 mon.a (mon.0) 664 : audit [DBG] from='client.? 172.21.15.16:0/1940484959' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T19:07:04.382 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:04 smithi016 bash[33624]: cluster 2023-09-23T19:07:02.933572+0000 mgr.y (mgr.14454) 137 : cluster [DBG] pgmap v117: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:04.382 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:04 smithi016 bash[33624]: audit 2023-09-23T19:07:03.125447+0000 mon.a (mon.0) 664 : audit [DBG] from='client.? 172.21.15.16:0/1940484959' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T19:07:05.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:05 smithi016 bash[27237]: audit 2023-09-23T19:07:04.200883+0000 mon.c (mon.2) 88 : audit [DBG] from='client.? 172.21.15.16:0/4061598228' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T19:07:05.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:05 smithi016 bash[27237]: audit 2023-09-23T19:07:04.576835+0000 mon.c (mon.2) 89 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:07:05.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:05 smithi016 bash[27237]: audit 2023-09-23T19:07:04.818412+0000 mon.a (mon.0) 665 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:07:05.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:05 smithi016 bash[27237]: audit 2023-09-23T19:07:04.842688+0000 mon.a (mon.0) 666 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:07:05.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:05 smithi016 bash[33624]: audit 2023-09-23T19:07:04.200883+0000 mon.c (mon.2) 88 : audit [DBG] from='client.? 172.21.15.16:0/4061598228' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T19:07:05.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:05 smithi016 bash[33624]: audit 2023-09-23T19:07:04.576835+0000 mon.c (mon.2) 89 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:07:05.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:05 smithi016 bash[33624]: audit 2023-09-23T19:07:04.818412+0000 mon.a (mon.0) 665 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:07:05.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:05 smithi016 bash[33624]: audit 2023-09-23T19:07:04.842688+0000 mon.a (mon.0) 666 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:07:05.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:05 smithi092 bash[27556]: audit 2023-09-23T19:07:04.200883+0000 mon.c (mon.2) 88 : audit [DBG] from='client.? 172.21.15.16:0/4061598228' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T19:07:05.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:05 smithi092 bash[27556]: audit 2023-09-23T19:07:04.576835+0000 mon.c (mon.2) 89 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:07:05.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:05 smithi092 bash[27556]: audit 2023-09-23T19:07:04.818412+0000 mon.a (mon.0) 665 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:07:05.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:05 smithi092 bash[27556]: audit 2023-09-23T19:07:04.842688+0000 mon.a (mon.0) 666 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:07:06.272 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:06 smithi016 bash[27237]: cluster 2023-09-23T19:07:04.934027+0000 mgr.y (mgr.14454) 138 : cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:06.272 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:06 smithi016 bash[27237]: audit 2023-09-23T19:07:05.127013+0000 mon.c (mon.2) 90 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:07:06.272 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:06 smithi016 bash[27237]: audit 2023-09-23T19:07:05.135201+0000 mon.c (mon.2) 91 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:07:06.272 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:06 smithi016 bash[27237]: audit 2023-09-23T19:07:05.142913+0000 mon.a (mon.0) 667 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:07:06.273 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:06 smithi016 bash[33624]: cluster 2023-09-23T19:07:04.934027+0000 mgr.y (mgr.14454) 138 : cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:06.273 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:06 smithi016 bash[33624]: audit 2023-09-23T19:07:05.127013+0000 mon.c (mon.2) 90 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:07:06.273 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:06 smithi016 bash[33624]: audit 2023-09-23T19:07:05.135201+0000 mon.c (mon.2) 91 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:07:06.273 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:06 smithi016 bash[33624]: audit 2023-09-23T19:07:05.142913+0000 mon.a (mon.0) 667 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:07:06.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:06 smithi092 bash[27556]: cluster 2023-09-23T19:07:04.934027+0000 mgr.y (mgr.14454) 138 : cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:06.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:06 smithi092 bash[27556]: audit 2023-09-23T19:07:05.127013+0000 mon.c (mon.2) 90 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:07:06.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:06 smithi092 bash[27556]: audit 2023-09-23T19:07:05.135201+0000 mon.c (mon.2) 91 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:07:06.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:06 smithi092 bash[27556]: audit 2023-09-23T19:07:05.142913+0000 mon.a (mon.0) 667 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:07:06.475 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430255 got 206158430259 for osd.7 2023-09-23T19:07:06.475 DEBUG:teuthology.parallel:result is None 2023-09-23T19:07:06.604 INFO:teuthology.orchestra.run.smithi016.stdout:154618822723 2023-09-23T19:07:06.642 INFO:teuthology.orchestra.run.smithi016.stdout:180388626491 2023-09-23T19:07:07.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:07 smithi016 bash[27237]: audit 2023-09-23T19:07:06.601977+0000 mon.c (mon.2) 92 : audit [DBG] from='client.? 172.21.15.16:0/3000514167' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T19:07:07.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:07 smithi016 bash[27237]: audit 2023-09-23T19:07:06.639165+0000 mon.a (mon.0) 668 : audit [DBG] from='client.? 172.21.15.16:0/2400931901' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T19:07:07.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:07 smithi016 bash[33624]: audit 2023-09-23T19:07:06.601977+0000 mon.c (mon.2) 92 : audit [DBG] from='client.? 172.21.15.16:0/3000514167' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T19:07:07.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:07 smithi016 bash[33624]: audit 2023-09-23T19:07:06.639165+0000 mon.a (mon.0) 668 : audit [DBG] from='client.? 172.21.15.16:0/2400931901' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T19:07:07.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:07 smithi092 bash[27556]: audit 2023-09-23T19:07:06.601977+0000 mon.c (mon.2) 92 : audit [DBG] from='client.? 172.21.15.16:0/3000514167' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T19:07:07.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:07 smithi092 bash[27556]: audit 2023-09-23T19:07:06.639165+0000 mon.a (mon.0) 668 : audit [DBG] from='client.? 172.21.15.16:0/2400931901' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T19:07:07.350 INFO:teuthology.orchestra.run.smithi016.stdout:107374182486 2023-09-23T19:07:07.768 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411415 got 77309411419 for osd.2 2023-09-23T19:07:07.768 DEBUG:teuthology.parallel:result is None 2023-09-23T19:07:07.848 INFO:teuthology.orchestra.run.smithi016.stdout:55834574951 2023-09-23T19:07:08.130 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:08 smithi016 bash[27237]: cluster 2023-09-23T19:07:06.935083+0000 mgr.y (mgr.14454) 139 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:08.130 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:08 smithi016 bash[27237]: audit 2023-09-23T19:07:07.347453+0000 mon.a (mon.0) 669 : audit [DBG] from='client.? 172.21.15.16:0/667009039' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T19:07:08.130 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:08 smithi016 bash[27237]: audit 2023-09-23T19:07:07.845725+0000 mon.c (mon.2) 93 : audit [DBG] from='client.? 172.21.15.16:0/3867353613' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T19:07:08.130 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:08 smithi016 bash[33624]: cluster 2023-09-23T19:07:06.935083+0000 mgr.y (mgr.14454) 139 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:08.130 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:08 smithi016 bash[33624]: audit 2023-09-23T19:07:07.347453+0000 mon.a (mon.0) 669 : audit [DBG] from='client.? 172.21.15.16:0/667009039' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T19:07:08.131 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:08 smithi016 bash[33624]: audit 2023-09-23T19:07:07.845725+0000 mon.c (mon.2) 93 : audit [DBG] from='client.? 172.21.15.16:0/3867353613' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T19:07:08.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:08 smithi092 bash[27556]: cluster 2023-09-23T19:07:06.935083+0000 mgr.y (mgr.14454) 139 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:08.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:08 smithi092 bash[27556]: audit 2023-09-23T19:07:07.347453+0000 mon.a (mon.0) 669 : audit [DBG] from='client.? 172.21.15.16:0/667009039' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T19:07:08.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:08 smithi092 bash[27556]: audit 2023-09-23T19:07:07.845725+0000 mon.c (mon.2) 93 : audit [DBG] from='client.? 172.21.15.16:0/3867353613' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T19:07:09.162 INFO:teuthology.orchestra.run.smithi016.stdout:128849018957 2023-09-23T19:07:09.399 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618822719 got 154618822723 for osd.5 2023-09-23T19:07:09.399 DEBUG:teuthology.parallel:result is None 2023-09-23T19:07:09.400 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388626487 got 180388626491 for osd.6 2023-09-23T19:07:09.400 DEBUG:teuthology.parallel:result is None 2023-09-23T19:07:09.537 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:07:09 smithi092 bash[28593]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:07:09] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T19:07:10.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:10 smithi016 bash[33624]: cluster 2023-09-23T19:07:08.935711+0000 mgr.y (mgr.14454) 140 : cluster [DBG] pgmap v120: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:10.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:10 smithi016 bash[33624]: audit 2023-09-23T19:07:09.160017+0000 mon.a (mon.0) 670 : audit [DBG] from='client.? 172.21.15.16:0/1186002212' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T19:07:10.285 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:07:10 smithi016 bash[27503]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:07:10] "GET /metrics HTTP/1.1" 200 191358 "" "Prometheus/2.33.4" 2023-09-23T19:07:10.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:10 smithi016 bash[27237]: cluster 2023-09-23T19:07:08.935711+0000 mgr.y (mgr.14454) 140 : cluster [DBG] pgmap v120: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:10.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:10 smithi016 bash[27237]: audit 2023-09-23T19:07:09.160017+0000 mon.a (mon.0) 670 : audit [DBG] from='client.? 172.21.15.16:0/1186002212' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T19:07:10.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:10 smithi092 bash[27556]: cluster 2023-09-23T19:07:08.935711+0000 mgr.y (mgr.14454) 140 : cluster [DBG] pgmap v120: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:10.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:10 smithi092 bash[27556]: audit 2023-09-23T19:07:09.160017+0000 mon.a (mon.0) 670 : audit [DBG] from='client.? 172.21.15.16:0/1186002212' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T19:07:10.424 INFO:teuthology.orchestra.run.smithi016.stdout:34359738478 2023-09-23T19:07:10.654 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182481 got 107374182486 for osd.3 2023-09-23T19:07:10.654 DEBUG:teuthology.parallel:result is None 2023-09-23T19:07:11.026 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574947 got 55834574951 for osd.1 2023-09-23T19:07:11.026 DEBUG:teuthology.parallel:result is None 2023-09-23T19:07:11.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:11 smithi016 bash[33624]: audit 2023-09-23T19:07:10.421282+0000 mon.c (mon.2) 94 : audit [DBG] from='client.? 172.21.15.16:0/1224619792' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T19:07:11.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:11 smithi016 bash[27237]: audit 2023-09-23T19:07:10.421282+0000 mon.c (mon.2) 94 : audit [DBG] from='client.? 172.21.15.16:0/1224619792' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T19:07:11.328 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018953 got 128849018957 for osd.4 2023-09-23T19:07:11.328 DEBUG:teuthology.parallel:result is None 2023-09-23T19:07:11.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:11 smithi092 bash[27556]: audit 2023-09-23T19:07:10.421282+0000 mon.c (mon.2) 94 : audit [DBG] from='client.? 172.21.15.16:0/1224619792' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T19:07:11.800 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738474 got 34359738478 for osd.0 2023-09-23T19:07:11.801 DEBUG:teuthology.parallel:result is None 2023-09-23T19:07:11.801 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-09-23T19:07:11.801 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T19:07:12.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:12 smithi016 bash[27237]: cluster 2023-09-23T19:07:10.936689+0000 mgr.y (mgr.14454) 141 : cluster [DBG] pgmap v121: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:12.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:12 smithi016 bash[33624]: cluster 2023-09-23T19:07:10.936689+0000 mgr.y (mgr.14454) 141 : cluster [DBG] pgmap v121: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:12.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:12 smithi092 bash[27556]: cluster 2023-09-23T19:07:10.936689+0000 mgr.y (mgr.14454) 141 : cluster [DBG] pgmap v121: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:14.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:14 smithi016 bash[27237]: cluster 2023-09-23T19:07:12.937303+0000 mgr.y (mgr.14454) 142 : cluster [DBG] pgmap v122: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:14.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:14 smithi016 bash[33624]: cluster 2023-09-23T19:07:12.937303+0000 mgr.y (mgr.14454) 142 : cluster [DBG] pgmap v122: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:14.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:14 smithi092 bash[27556]: cluster 2023-09-23T19:07:12.937303+0000 mgr.y (mgr.14454) 142 : cluster [DBG] pgmap v122: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:16.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:16 smithi016 bash[27237]: cluster 2023-09-23T19:07:14.937944+0000 mgr.y (mgr.14454) 143 : cluster [DBG] pgmap v123: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:16.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:16 smithi016 bash[33624]: cluster 2023-09-23T19:07:14.937944+0000 mgr.y (mgr.14454) 143 : cluster [DBG] pgmap v123: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:16.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:16 smithi092 bash[27556]: cluster 2023-09-23T19:07:14.937944+0000 mgr.y (mgr.14454) 143 : cluster [DBG] pgmap v123: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:16.581 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:07:18.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:18 smithi016 bash[27237]: cluster 2023-09-23T19:07:16.939092+0000 mgr.y (mgr.14454) 144 : cluster [DBG] pgmap v124: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:18.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:18 smithi016 bash[33624]: cluster 2023-09-23T19:07:16.939092+0000 mgr.y (mgr.14454) 144 : cluster [DBG] pgmap v124: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:18.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:18 smithi092 bash[27556]: cluster 2023-09-23T19:07:16.939092+0000 mgr.y (mgr.14454) 144 : cluster [DBG] pgmap v124: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:19.469 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-09-23T19:07:19.472 INFO:teuthology.orchestra.run.smithi016.stderr:dumped all 2023-09-23T19:07:19.536 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:07:19 smithi092 bash[28593]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:07:19] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T19:07:20.238 INFO:teuthology.orchestra.run.smithi016.stdout:{"pg_ready":true,"pg_map":{"version":125,"stamp":"2023-09-23T19:07:18.939401+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2381652,"kb_used_data":4884,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":747579564,"statfs":{"total":767960285184,"available":765521473536,"internally_reserved":0,"allocated":5001216,"data_stored":2111397,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.005118"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":21,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T19:06:36.673002+0000","last_change":"2023-09-23T19:06:36.673002+0000","last_active":"2023-09-23T19:06:36.673002+0000","last_peered":"2023-09-23T19:06:36.673002+0000","last_clean":"2023-09-23T19:06:36.673002+0000","last_became_active":"2023-09-23T19:06:36.671911+0000","last_became_peered":"2023-09-23T19:06:36.671911+0000","last_unstale":"2023-09-23T19:06:36.673002+0000","last_undegraded":"2023-09-23T19:06:36.673002+0000","last_fullsized":"2023-09-23T19:06:36.673002+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":17,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.041899+0000","last_change":"2023-09-23T19:05:52.099329+0000","last_active":"2023-09-23T19:06:35.041899+0000","last_peered":"2023-09-23T19:06:35.041899+0000","last_clean":"2023-09-23T19:06:35.041899+0000","last_became_active":"2023-09-23T19:05:52.098983+0000","last_became_peered":"2023-09-23T19:05:52.098983+0000","last_unstale":"2023-09-23T19:06:35.041899+0000","last_undegraded":"2023-09-23T19:06:35.041899+0000","last_fullsized":"2023-09-23T19:06:35.041899+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.046914+0000","last_change":"2023-09-23T19:05:52.132004+0000","last_active":"2023-09-23T19:06:35.046914+0000","last_peered":"2023-09-23T19:06:35.046914+0000","last_clean":"2023-09-23T19:06:35.046914+0000","last_became_active":"2023-09-23T19:05:52.131655+0000","last_became_peered":"2023-09-23T19:05:52.131655+0000","last_unstale":"2023-09-23T19:06:35.046914+0000","last_undegraded":"2023-09-23T19:06:35.046914+0000","last_fullsized":"2023-09-23T19:06:35.046914+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.047507+0000","last_change":"2023-09-23T19:05:52.100466+0000","last_active":"2023-09-23T19:06:35.047507+0000","last_peered":"2023-09-23T19:06:35.047507+0000","last_clean":"2023-09-23T19:06:35.047507+0000","last_became_active":"2023-09-23T19:05:52.100107+0000","last_became_peered":"2023-09-23T19:05:52.100107+0000","last_unstale":"2023-09-23T19:06:35.047507+0000","last_undegraded":"2023-09-23T19:06:35.047507+0000","last_fullsized":"2023-09-23T19:06:35.047507+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.048567+0000","last_change":"2023-09-23T19:05:52.139495+0000","last_active":"2023-09-23T19:06:35.048567+0000","last_peered":"2023-09-23T19:06:35.048567+0000","last_clean":"2023-09-23T19:06:35.048567+0000","last_became_active":"2023-09-23T19:05:52.139186+0000","last_became_peered":"2023-09-23T19:05:52.139186+0000","last_unstale":"2023-09-23T19:06:35.048567+0000","last_undegraded":"2023-09-23T19:06:35.048567+0000","last_fullsized":"2023-09-23T19:06:35.048567+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":15,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.052666+0000","last_change":"2023-09-23T19:05:52.107570+0000","last_active":"2023-09-23T19:06:35.052666+0000","last_peered":"2023-09-23T19:06:35.052666+0000","last_clean":"2023-09-23T19:06:35.052666+0000","last_became_active":"2023-09-23T19:05:52.107215+0000","last_became_peered":"2023-09-23T19:05:52.107215+0000","last_unstale":"2023-09-23T19:06:35.052666+0000","last_undegraded":"2023-09-23T19:06:35.052666+0000","last_fullsized":"2023-09-23T19:06:35.052666+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":21,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T19:06:36.793265+0000","last_change":"2023-09-23T19:06:36.793265+0000","last_active":"2023-09-23T19:06:36.793265+0000","last_peered":"2023-09-23T19:06:36.793265+0000","last_clean":"2023-09-23T19:06:36.793265+0000","last_became_active":"2023-09-23T19:06:36.792450+0000","last_became_peered":"2023-09-23T19:06:36.792450+0000","last_unstale":"2023-09-23T19:06:36.793265+0000","last_undegraded":"2023-09-23T19:06:36.793265+0000","last_fullsized":"2023-09-23T19:06:36.793265+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":15,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.052298+0000","last_change":"2023-09-23T19:05:52.108768+0000","last_active":"2023-09-23T19:06:35.052298+0000","last_peered":"2023-09-23T19:06:35.052298+0000","last_clean":"2023-09-23T19:06:35.052298+0000","last_became_active":"2023-09-23T19:05:52.108406+0000","last_became_peered":"2023-09-23T19:05:52.108406+0000","last_unstale":"2023-09-23T19:06:35.052298+0000","last_undegraded":"2023-09-23T19:06:35.052298+0000","last_fullsized":"2023-09-23T19:06:35.052298+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.041042+0000","last_change":"2023-09-23T19:05:52.133083+0000","last_active":"2023-09-23T19:06:35.041042+0000","last_peered":"2023-09-23T19:06:35.041042+0000","last_clean":"2023-09-23T19:06:35.041042+0000","last_became_active":"2023-09-23T19:05:52.132694+0000","last_became_peered":"2023-09-23T19:05:52.132694+0000","last_unstale":"2023-09-23T19:06:35.041042+0000","last_undegraded":"2023-09-23T19:06:35.041042+0000","last_fullsized":"2023-09-23T19:06:35.041042+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.041013+0000","last_change":"2023-09-23T19:05:52.105004+0000","last_active":"2023-09-23T19:06:35.041013+0000","last_peered":"2023-09-23T19:06:35.041013+0000","last_clean":"2023-09-23T19:06:35.041013+0000","last_became_active":"2023-09-23T19:05:52.104345+0000","last_became_peered":"2023-09-23T19:05:52.104345+0000","last_unstale":"2023-09-23T19:06:35.041013+0000","last_undegraded":"2023-09-23T19:06:35.041013+0000","last_fullsized":"2023-09-23T19:06:35.041013+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.047745+0000","last_change":"2023-09-23T19:05:52.138620+0000","last_active":"2023-09-23T19:06:35.047745+0000","last_peered":"2023-09-23T19:06:35.047745+0000","last_clean":"2023-09-23T19:06:35.047745+0000","last_became_active":"2023-09-23T19:05:52.138243+0000","last_became_peered":"2023-09-23T19:05:52.138243+0000","last_unstale":"2023-09-23T19:06:35.047745+0000","last_undegraded":"2023-09-23T19:06:35.047745+0000","last_fullsized":"2023-09-23T19:06:35.047745+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.041551+0000","last_change":"2023-09-23T19:05:52.106314+0000","last_active":"2023-09-23T19:06:35.041551+0000","last_peered":"2023-09-23T19:06:35.041551+0000","last_clean":"2023-09-23T19:06:35.041551+0000","last_became_active":"2023-09-23T19:05:52.106027+0000","last_became_peered":"2023-09-23T19:05:52.106027+0000","last_unstale":"2023-09-23T19:06:35.041551+0000","last_undegraded":"2023-09-23T19:06:35.041551+0000","last_fullsized":"2023-09-23T19:06:35.041551+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":17,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.053508+0000","last_change":"2023-09-23T19:04:02.507717+0000","last_active":"2023-09-23T19:06:35.053508+0000","last_peered":"2023-09-23T19:06:35.053508+0000","last_clean":"2023-09-23T19:06:35.053508+0000","last_became_active":"2023-09-23T19:04:02.507042+0000","last_became_peered":"2023-09-23T19:04:02.507042+0000","last_unstale":"2023-09-23T19:06:35.053508+0000","last_undegraded":"2023-09-23T19:06:35.053508+0000","last_fullsized":"2023-09-23T19:06:35.053508+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.046661+0000","last_change":"2023-09-23T19:05:52.783427+0000","last_active":"2023-09-23T19:06:35.046661+0000","last_peered":"2023-09-23T19:06:35.046661+0000","last_clean":"2023-09-23T19:06:35.046661+0000","last_became_active":"2023-09-23T19:05:52.783088+0000","last_became_peered":"2023-09-23T19:05:52.783088+0000","last_unstale":"2023-09-23T19:06:35.046661+0000","last_undegraded":"2023-09-23T19:06:35.046661+0000","last_fullsized":"2023-09-23T19:06:35.046661+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":17,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.045887+0000","last_change":"2023-09-23T19:04:02.707894+0000","last_active":"2023-09-23T19:06:35.045887+0000","last_peered":"2023-09-23T19:06:35.045887+0000","last_clean":"2023-09-23T19:06:35.045887+0000","last_became_active":"2023-09-23T19:04:02.707071+0000","last_became_peered":"2023-09-23T19:04:02.707071+0000","last_unstale":"2023-09-23T19:06:35.045887+0000","last_undegraded":"2023-09-23T19:06:35.045887+0000","last_fullsized":"2023-09-23T19:06:35.045887+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":15,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.047641+0000","last_change":"2023-09-23T19:05:52.112053+0000","last_active":"2023-09-23T19:06:35.047641+0000","last_peered":"2023-09-23T19:06:35.047641+0000","last_clean":"2023-09-23T19:06:35.047641+0000","last_became_active":"2023-09-23T19:05:52.111626+0000","last_became_peered":"2023-09-23T19:05:52.111626+0000","last_unstale":"2023-09-23T19:06:35.047641+0000","last_undegraded":"2023-09-23T19:06:35.047641+0000","last_fullsized":"2023-09-23T19:06:35.047641+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.040820+0000","last_change":"2023-09-23T19:05:52.783290+0000","last_active":"2023-09-23T19:06:35.040820+0000","last_peered":"2023-09-23T19:06:35.040820+0000","last_clean":"2023-09-23T19:06:35.040820+0000","last_became_active":"2023-09-23T19:05:52.782947+0000","last_became_peered":"2023-09-23T19:05:52.782947+0000","last_unstale":"2023-09-23T19:06:35.040820+0000","last_undegraded":"2023-09-23T19:06:35.040820+0000","last_fullsized":"2023-09-23T19:06:35.040820+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.046052+0000","last_change":"2023-09-23T19:05:52.098066+0000","last_active":"2023-09-23T19:06:35.046052+0000","last_peered":"2023-09-23T19:06:35.046052+0000","last_clean":"2023-09-23T19:06:35.046052+0000","last_became_active":"2023-09-23T19:05:52.097675+0000","last_became_peered":"2023-09-23T19:05:52.097675+0000","last_unstale":"2023-09-23T19:06:35.046052+0000","last_undegraded":"2023-09-23T19:06:35.046052+0000","last_fullsized":"2023-09-23T19:06:35.046052+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":17,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.947445+0000","last_change":"2023-09-23T19:04:02.713482+0000","last_active":"2023-09-23T19:06:35.947445+0000","last_peered":"2023-09-23T19:06:35.947445+0000","last_clean":"2023-09-23T19:06:35.947445+0000","last_became_active":"2023-09-23T19:04:02.712471+0000","last_became_peered":"2023-09-23T19:04:02.712471+0000","last_unstale":"2023-09-23T19:06:35.947445+0000","last_undegraded":"2023-09-23T19:06:35.947445+0000","last_fullsized":"2023-09-23T19:06:35.947445+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.041562+0000","last_change":"2023-09-23T19:05:52.150124+0000","last_active":"2023-09-23T19:06:35.041562+0000","last_peered":"2023-09-23T19:06:35.041562+0000","last_clean":"2023-09-23T19:06:35.041562+0000","last_became_active":"2023-09-23T19:05:52.149782+0000","last_became_peered":"2023-09-23T19:05:52.149782+0000","last_unstale":"2023-09-23T19:06:35.041562+0000","last_undegraded":"2023-09-23T19:06:35.041562+0000","last_fullsized":"2023-09-23T19:06:35.041562+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.040694+0000","last_change":"2023-09-23T19:05:52.105960+0000","last_active":"2023-09-23T19:06:35.040694+0000","last_peered":"2023-09-23T19:06:35.040694+0000","last_clean":"2023-09-23T19:06:35.040694+0000","last_became_active":"2023-09-23T19:05:52.105531+0000","last_became_peered":"2023-09-23T19:05:52.105531+0000","last_unstale":"2023-09-23T19:06:35.040694+0000","last_undegraded":"2023-09-23T19:06:35.040694+0000","last_fullsized":"2023-09-23T19:06:35.040694+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":17,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.947064+0000","last_change":"2023-09-23T19:04:02.712803+0000","last_active":"2023-09-23T19:06:35.947064+0000","last_peered":"2023-09-23T19:06:35.947064+0000","last_clean":"2023-09-23T19:06:35.947064+0000","last_became_active":"2023-09-23T19:04:02.711952+0000","last_became_peered":"2023-09-23T19:04:02.711952+0000","last_unstale":"2023-09-23T19:06:35.947064+0000","last_undegraded":"2023-09-23T19:06:35.947064+0000","last_fullsized":"2023-09-23T19:06:35.947064+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.053400+0000","last_change":"2023-09-23T19:05:52.786771+0000","last_active":"2023-09-23T19:06:35.053400+0000","last_peered":"2023-09-23T19:06:35.053400+0000","last_clean":"2023-09-23T19:06:35.053400+0000","last_became_active":"2023-09-23T19:05:52.785797+0000","last_became_peered":"2023-09-23T19:05:52.785797+0000","last_unstale":"2023-09-23T19:06:35.053400+0000","last_undegraded":"2023-09-23T19:06:35.053400+0000","last_fullsized":"2023-09-23T19:06:35.053400+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.047140+0000","last_change":"2023-09-23T19:05:52.107913+0000","last_active":"2023-09-23T19:06:35.047140+0000","last_peered":"2023-09-23T19:06:35.047140+0000","last_clean":"2023-09-23T19:06:35.047140+0000","last_became_active":"2023-09-23T19:05:52.107561+0000","last_became_peered":"2023-09-23T19:05:52.107561+0000","last_unstale":"2023-09-23T19:06:35.047140+0000","last_undegraded":"2023-09-23T19:06:35.047140+0000","last_fullsized":"2023-09-23T19:06:35.047140+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":21,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.947057+0000","last_change":"2023-09-23T19:03:04.495032+0000","last_active":"2023-09-23T19:06:35.947057+0000","last_peered":"2023-09-23T19:06:35.947057+0000","last_clean":"2023-09-23T19:06:35.947057+0000","last_became_active":"2023-09-23T19:03:04.494463+0000","last_became_peered":"2023-09-23T19:03:04.494463+0000","last_unstale":"2023-09-23T19:06:35.947057+0000","last_undegraded":"2023-09-23T19:06:35.947057+0000","last_fullsized":"2023-09-23T19:06:35.947057+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:59:33.840751+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:59:33.840751+0000","last_clean_scrub_stamp":"2023-09-23T18:59:33.840751+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":17,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.052198+0000","last_change":"2023-09-23T19:04:02.509889+0000","last_active":"2023-09-23T19:06:35.052198+0000","last_peered":"2023-09-23T19:06:35.052198+0000","last_clean":"2023-09-23T19:06:35.052198+0000","last_became_active":"2023-09-23T19:04:02.509458+0000","last_became_peered":"2023-09-23T19:04:02.509458+0000","last_unstale":"2023-09-23T19:06:35.052198+0000","last_undegraded":"2023-09-23T19:06:35.052198+0000","last_fullsized":"2023-09-23T19:06:35.052198+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.046240+0000","last_change":"2023-09-23T19:05:52.138724+0000","last_active":"2023-09-23T19:06:35.046240+0000","last_peered":"2023-09-23T19:06:35.046240+0000","last_clean":"2023-09-23T19:06:35.046240+0000","last_became_active":"2023-09-23T19:05:52.136411+0000","last_became_peered":"2023-09-23T19:05:52.136411+0000","last_unstale":"2023-09-23T19:06:35.046240+0000","last_undegraded":"2023-09-23T19:06:35.046240+0000","last_fullsized":"2023-09-23T19:06:35.046240+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.045265+0000","last_change":"2023-09-23T19:05:52.093138+0000","last_active":"2023-09-23T19:06:35.045265+0000","last_peered":"2023-09-23T19:06:35.045265+0000","last_clean":"2023-09-23T19:06:35.045265+0000","last_became_active":"2023-09-23T19:05:52.092797+0000","last_became_peered":"2023-09-23T19:05:52.092797+0000","last_unstale":"2023-09-23T19:06:35.045265+0000","last_undegraded":"2023-09-23T19:06:35.045265+0000","last_fullsized":"2023-09-23T19:06:35.045265+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":17,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.040695+0000","last_change":"2023-09-23T19:04:02.707164+0000","last_active":"2023-09-23T19:06:35.040695+0000","last_peered":"2023-09-23T19:06:35.040695+0000","last_clean":"2023-09-23T19:06:35.040695+0000","last_became_active":"2023-09-23T19:04:02.706747+0000","last_became_peered":"2023-09-23T19:04:02.706747+0000","last_unstale":"2023-09-23T19:06:35.040695+0000","last_undegraded":"2023-09-23T19:06:35.040695+0000","last_fullsized":"2023-09-23T19:06:35.040695+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.045920+0000","last_change":"2023-09-23T19:05:52.137816+0000","last_active":"2023-09-23T19:06:35.045920+0000","last_peered":"2023-09-23T19:06:35.045920+0000","last_clean":"2023-09-23T19:06:35.045920+0000","last_became_active":"2023-09-23T19:05:52.137278+0000","last_became_peered":"2023-09-23T19:05:52.137278+0000","last_unstale":"2023-09-23T19:06:35.045920+0000","last_undegraded":"2023-09-23T19:06:35.045920+0000","last_fullsized":"2023-09-23T19:06:35.045920+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.047806+0000","last_change":"2023-09-23T19:05:52.108160+0000","last_active":"2023-09-23T19:06:35.047806+0000","last_peered":"2023-09-23T19:06:35.047806+0000","last_clean":"2023-09-23T19:06:35.047806+0000","last_became_active":"2023-09-23T19:05:52.107802+0000","last_became_peered":"2023-09-23T19:05:52.107802+0000","last_unstale":"2023-09-23T19:06:35.047806+0000","last_undegraded":"2023-09-23T19:06:35.047806+0000","last_fullsized":"2023-09-23T19:06:35.047806+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":17,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.052124+0000","last_change":"2023-09-23T19:04:02.509914+0000","last_active":"2023-09-23T19:06:35.052124+0000","last_peered":"2023-09-23T19:06:35.052124+0000","last_clean":"2023-09-23T19:06:35.052124+0000","last_became_active":"2023-09-23T19:04:02.509532+0000","last_became_peered":"2023-09-23T19:04:02.509532+0000","last_unstale":"2023-09-23T19:06:35.052124+0000","last_undegraded":"2023-09-23T19:06:35.052124+0000","last_fullsized":"2023-09-23T19:06:35.052124+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":10,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T19:06:36.052412+0000","last_change":"2023-09-23T19:06:36.052412+0000","last_active":"2023-09-23T19:06:36.052412+0000","last_peered":"2023-09-23T19:06:36.052412+0000","last_clean":"2023-09-23T19:06:36.052412+0000","last_became_active":"2023-09-23T19:06:36.052036+0000","last_became_peered":"2023-09-23T19:06:36.052036+0000","last_unstale":"2023-09-23T19:06:36.052412+0000","last_undegraded":"2023-09-23T19:06:36.052412+0000","last_fullsized":"2023-09-23T19:06:36.052412+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.052524+0000","last_change":"2023-09-23T19:05:52.115013+0000","last_active":"2023-09-23T19:06:35.052524+0000","last_peered":"2023-09-23T19:06:35.052524+0000","last_clean":"2023-09-23T19:06:35.052524+0000","last_became_active":"2023-09-23T19:05:52.114762+0000","last_became_peered":"2023-09-23T19:05:52.114762+0000","last_unstale":"2023-09-23T19:06:35.052524+0000","last_undegraded":"2023-09-23T19:06:35.052524+0000","last_fullsized":"2023-09-23T19:06:35.052524+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":17,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.045795+0000","last_change":"2023-09-23T19:04:02.707302+0000","last_active":"2023-09-23T19:06:35.045795+0000","last_peered":"2023-09-23T19:06:35.045795+0000","last_clean":"2023-09-23T19:06:35.045795+0000","last_became_active":"2023-09-23T19:04:02.706904+0000","last_became_peered":"2023-09-23T19:04:02.706904+0000","last_unstale":"2023-09-23T19:06:35.045795+0000","last_undegraded":"2023-09-23T19:06:35.045795+0000","last_fullsized":"2023-09-23T19:06:35.045795+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.046282+0000","last_change":"2023-09-23T19:05:52.142253+0000","last_active":"2023-09-23T19:06:35.046282+0000","last_peered":"2023-09-23T19:06:35.046282+0000","last_clean":"2023-09-23T19:06:35.046282+0000","last_became_active":"2023-09-23T19:05:52.141613+0000","last_became_peered":"2023-09-23T19:05:52.141613+0000","last_unstale":"2023-09-23T19:06:35.046282+0000","last_undegraded":"2023-09-23T19:06:35.046282+0000","last_fullsized":"2023-09-23T19:06:35.046282+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.046701+0000","last_change":"2023-09-23T19:05:52.110960+0000","last_active":"2023-09-23T19:06:35.046701+0000","last_peered":"2023-09-23T19:06:35.046701+0000","last_clean":"2023-09-23T19:06:35.046701+0000","last_became_active":"2023-09-23T19:05:52.110444+0000","last_became_peered":"2023-09-23T19:05:52.110444+0000","last_unstale":"2023-09-23T19:06:35.046701+0000","last_undegraded":"2023-09-23T19:06:35.046701+0000","last_fullsized":"2023-09-23T19:06:35.046701+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.048963+0000","last_change":"2023-09-23T19:05:52.146508+0000","last_active":"2023-09-23T19:06:35.048963+0000","last_peered":"2023-09-23T19:06:35.048963+0000","last_clean":"2023-09-23T19:06:35.048963+0000","last_became_active":"2023-09-23T19:05:52.146192+0000","last_became_peered":"2023-09-23T19:05:52.146192+0000","last_unstale":"2023-09-23T19:06:35.048963+0000","last_undegraded":"2023-09-23T19:06:35.048963+0000","last_fullsized":"2023-09-23T19:06:35.048963+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.047974+0000","last_change":"2023-09-23T19:05:52.107974+0000","last_active":"2023-09-23T19:06:35.047974+0000","last_peered":"2023-09-23T19:06:35.047974+0000","last_clean":"2023-09-23T19:06:35.047974+0000","last_became_active":"2023-09-23T19:05:52.107596+0000","last_became_peered":"2023-09-23T19:05:52.107596+0000","last_unstale":"2023-09-23T19:06:35.047974+0000","last_undegraded":"2023-09-23T19:06:35.047974+0000","last_fullsized":"2023-09-23T19:06:35.047974+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.049605+0000","last_change":"2023-09-23T19:05:52.145950+0000","last_active":"2023-09-23T19:06:35.049605+0000","last_peered":"2023-09-23T19:06:35.049605+0000","last_clean":"2023-09-23T19:06:35.049605+0000","last_became_active":"2023-09-23T19:05:52.145147+0000","last_became_peered":"2023-09-23T19:05:52.145147+0000","last_unstale":"2023-09-23T19:06:35.049605+0000","last_undegraded":"2023-09-23T19:06:35.049605+0000","last_fullsized":"2023-09-23T19:06:35.049605+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.053189+0000","last_change":"2023-09-23T19:05:52.110940+0000","last_active":"2023-09-23T19:06:35.053189+0000","last_peered":"2023-09-23T19:06:35.053189+0000","last_clean":"2023-09-23T19:06:35.053189+0000","last_became_active":"2023-09-23T19:05:52.110544+0000","last_became_peered":"2023-09-23T19:05:52.110544+0000","last_unstale":"2023-09-23T19:06:35.053189+0000","last_undegraded":"2023-09-23T19:06:35.053189+0000","last_fullsized":"2023-09-23T19:06:35.053189+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.048126+0000","last_change":"2023-09-23T19:05:52.138696+0000","last_active":"2023-09-23T19:06:35.048126+0000","last_peered":"2023-09-23T19:06:35.048126+0000","last_clean":"2023-09-23T19:06:35.048126+0000","last_became_active":"2023-09-23T19:05:52.138272+0000","last_became_peered":"2023-09-23T19:05:52.138272+0000","last_unstale":"2023-09-23T19:06:35.048126+0000","last_undegraded":"2023-09-23T19:06:35.048126+0000","last_fullsized":"2023-09-23T19:06:35.048126+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":15,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.047312+0000","last_change":"2023-09-23T19:05:52.110010+0000","last_active":"2023-09-23T19:06:35.047312+0000","last_peered":"2023-09-23T19:06:35.047312+0000","last_clean":"2023-09-23T19:06:35.047312+0000","last_became_active":"2023-09-23T19:05:52.109644+0000","last_became_peered":"2023-09-23T19:05:52.109644+0000","last_unstale":"2023-09-23T19:06:35.047312+0000","last_undegraded":"2023-09-23T19:06:35.047312+0000","last_fullsized":"2023-09-23T19:06:35.047312+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.041171+0000","last_change":"2023-09-23T19:05:52.139974+0000","last_active":"2023-09-23T19:06:35.041171+0000","last_peered":"2023-09-23T19:06:35.041171+0000","last_clean":"2023-09-23T19:06:35.041171+0000","last_became_active":"2023-09-23T19:05:52.139627+0000","last_became_peered":"2023-09-23T19:05:52.139627+0000","last_unstale":"2023-09-23T19:06:35.041171+0000","last_undegraded":"2023-09-23T19:06:35.041171+0000","last_fullsized":"2023-09-23T19:06:35.041171+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.052766+0000","last_change":"2023-09-23T19:05:52.110253+0000","last_active":"2023-09-23T19:06:35.052766+0000","last_peered":"2023-09-23T19:06:35.052766+0000","last_clean":"2023-09-23T19:06:35.052766+0000","last_became_active":"2023-09-23T19:05:52.109817+0000","last_became_peered":"2023-09-23T19:05:52.109817+0000","last_unstale":"2023-09-23T19:06:35.052766+0000","last_undegraded":"2023-09-23T19:06:35.052766+0000","last_fullsized":"2023-09-23T19:06:35.052766+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.053084+0000","last_change":"2023-09-23T19:05:52.786764+0000","last_active":"2023-09-23T19:06:35.053084+0000","last_peered":"2023-09-23T19:06:35.053084+0000","last_clean":"2023-09-23T19:06:35.053084+0000","last_became_active":"2023-09-23T19:05:52.785936+0000","last_became_peered":"2023-09-23T19:05:52.785936+0000","last_unstale":"2023-09-23T19:06:35.053084+0000","last_undegraded":"2023-09-23T19:06:35.053084+0000","last_fullsized":"2023-09-23T19:06:35.053084+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.947610+0000","last_change":"2023-09-23T19:05:52.114108+0000","last_active":"2023-09-23T19:06:35.947610+0000","last_peered":"2023-09-23T19:06:35.947610+0000","last_clean":"2023-09-23T19:06:35.947610+0000","last_became_active":"2023-09-23T19:05:52.113609+0000","last_became_peered":"2023-09-23T19:05:52.113609+0000","last_unstale":"2023-09-23T19:06:35.947610+0000","last_undegraded":"2023-09-23T19:06:35.947610+0000","last_fullsized":"2023-09-23T19:06:35.947610+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.047458+0000","last_change":"2023-09-23T19:05:52.146957+0000","last_active":"2023-09-23T19:06:35.047458+0000","last_peered":"2023-09-23T19:06:35.047458+0000","last_clean":"2023-09-23T19:06:35.047458+0000","last_became_active":"2023-09-23T19:05:52.145201+0000","last_became_peered":"2023-09-23T19:05:52.145201+0000","last_unstale":"2023-09-23T19:06:35.047458+0000","last_undegraded":"2023-09-23T19:06:35.047458+0000","last_fullsized":"2023-09-23T19:06:35.047458+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":15,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.947587+0000","last_change":"2023-09-23T19:05:52.113304+0000","last_active":"2023-09-23T19:06:35.947587+0000","last_peered":"2023-09-23T19:06:35.947587+0000","last_clean":"2023-09-23T19:06:35.947587+0000","last_became_active":"2023-09-23T19:05:52.112945+0000","last_became_peered":"2023-09-23T19:05:52.112945+0000","last_unstale":"2023-09-23T19:06:35.947587+0000","last_undegraded":"2023-09-23T19:06:35.947587+0000","last_fullsized":"2023-09-23T19:06:35.947587+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.051772+0000","last_change":"2023-09-23T19:05:52.789323+0000","last_active":"2023-09-23T19:06:35.051772+0000","last_peered":"2023-09-23T19:06:35.051772+0000","last_clean":"2023-09-23T19:06:35.051772+0000","last_became_active":"2023-09-23T19:05:52.788820+0000","last_became_peered":"2023-09-23T19:05:52.788820+0000","last_unstale":"2023-09-23T19:06:35.051772+0000","last_undegraded":"2023-09-23T19:06:35.051772+0000","last_fullsized":"2023-09-23T19:06:35.051772+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.947900+0000","last_change":"2023-09-23T19:05:52.112710+0000","last_active":"2023-09-23T19:06:35.947900+0000","last_peered":"2023-09-23T19:06:35.947900+0000","last_clean":"2023-09-23T19:06:35.947900+0000","last_became_active":"2023-09-23T19:05:52.112362+0000","last_became_peered":"2023-09-23T19:05:52.112362+0000","last_unstale":"2023-09-23T19:06:35.947900+0000","last_undegraded":"2023-09-23T19:06:35.947900+0000","last_fullsized":"2023-09-23T19:06:35.947900+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.047556+0000","last_change":"2023-09-23T19:05:52.780101+0000","last_active":"2023-09-23T19:06:35.047556+0000","last_peered":"2023-09-23T19:06:35.047556+0000","last_clean":"2023-09-23T19:06:35.047556+0000","last_became_active":"2023-09-23T19:05:52.779607+0000","last_became_peered":"2023-09-23T19:05:52.779607+0000","last_unstale":"2023-09-23T19:06:35.047556+0000","last_undegraded":"2023-09-23T19:06:35.047556+0000","last_fullsized":"2023-09-23T19:06:35.047556+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":21,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.052286+0000","last_change":"2023-09-23T19:05:52.103917+0000","last_active":"2023-09-23T19:06:35.052286+0000","last_peered":"2023-09-23T19:06:35.052286+0000","last_clean":"2023-09-23T19:06:35.052286+0000","last_became_active":"2023-09-23T19:05:52.103551+0000","last_became_peered":"2023-09-23T19:05:52.103551+0000","last_unstale":"2023-09-23T19:06:35.052286+0000","last_undegraded":"2023-09-23T19:06:35.052286+0000","last_fullsized":"2023-09-23T19:06:35.052286+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.040773+0000","last_change":"2023-09-23T19:05:52.785109+0000","last_active":"2023-09-23T19:06:35.040773+0000","last_peered":"2023-09-23T19:06:35.040773+0000","last_clean":"2023-09-23T19:06:35.040773+0000","last_became_active":"2023-09-23T19:05:52.784196+0000","last_became_peered":"2023-09-23T19:05:52.784196+0000","last_unstale":"2023-09-23T19:06:35.040773+0000","last_undegraded":"2023-09-23T19:06:35.040773+0000","last_fullsized":"2023-09-23T19:06:35.040773+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.946866+0000","last_change":"2023-09-23T19:05:52.114186+0000","last_active":"2023-09-23T19:06:35.946866+0000","last_peered":"2023-09-23T19:06:35.946866+0000","last_clean":"2023-09-23T19:06:35.946866+0000","last_became_active":"2023-09-23T19:05:52.113635+0000","last_became_peered":"2023-09-23T19:05:52.113635+0000","last_unstale":"2023-09-23T19:06:35.946866+0000","last_undegraded":"2023-09-23T19:06:35.946866+0000","last_fullsized":"2023-09-23T19:06:35.946866+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.052419+0000","last_change":"2023-09-23T19:05:52.789241+0000","last_active":"2023-09-23T19:06:35.052419+0000","last_peered":"2023-09-23T19:06:35.052419+0000","last_clean":"2023-09-23T19:06:35.052419+0000","last_became_active":"2023-09-23T19:05:52.788902+0000","last_became_peered":"2023-09-23T19:05:52.788902+0000","last_unstale":"2023-09-23T19:06:35.052419+0000","last_undegraded":"2023-09-23T19:06:35.052419+0000","last_fullsized":"2023-09-23T19:06:35.052419+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.040778+0000","last_change":"2023-09-23T19:05:52.105482+0000","last_active":"2023-09-23T19:06:35.040778+0000","last_peered":"2023-09-23T19:06:35.040778+0000","last_clean":"2023-09-23T19:06:35.040778+0000","last_became_active":"2023-09-23T19:05:52.105125+0000","last_became_peered":"2023-09-23T19:05:52.105125+0000","last_unstale":"2023-09-23T19:06:35.040778+0000","last_undegraded":"2023-09-23T19:06:35.040778+0000","last_fullsized":"2023-09-23T19:06:35.040778+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.047352+0000","last_change":"2023-09-23T19:05:52.781046+0000","last_active":"2023-09-23T19:06:35.047352+0000","last_peered":"2023-09-23T19:06:35.047352+0000","last_clean":"2023-09-23T19:06:35.047352+0000","last_became_active":"2023-09-23T19:05:52.780115+0000","last_became_peered":"2023-09-23T19:05:52.780115+0000","last_unstale":"2023-09-23T19:06:35.047352+0000","last_undegraded":"2023-09-23T19:06:35.047352+0000","last_fullsized":"2023-09-23T19:06:35.047352+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":15,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.947239+0000","last_change":"2023-09-23T19:05:52.111942+0000","last_active":"2023-09-23T19:06:35.947239+0000","last_peered":"2023-09-23T19:06:35.947239+0000","last_clean":"2023-09-23T19:06:35.947239+0000","last_became_active":"2023-09-23T19:05:52.111546+0000","last_became_peered":"2023-09-23T19:05:52.111546+0000","last_unstale":"2023-09-23T19:06:35.947239+0000","last_undegraded":"2023-09-23T19:06:35.947239+0000","last_fullsized":"2023-09-23T19:06:35.947239+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.048024+0000","last_change":"2023-09-23T19:05:52.145659+0000","last_active":"2023-09-23T19:06:35.048024+0000","last_peered":"2023-09-23T19:06:35.048024+0000","last_clean":"2023-09-23T19:06:35.048024+0000","last_became_active":"2023-09-23T19:05:52.145156+0000","last_became_peered":"2023-09-23T19:05:52.145156+0000","last_unstale":"2023-09-23T19:06:35.048024+0000","last_undegraded":"2023-09-23T19:06:35.048024+0000","last_fullsized":"2023-09-23T19:06:35.048024+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"59'6","reported_seq":19,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.047943+0000","last_change":"2023-09-23T19:05:52.107727+0000","last_active":"2023-09-23T19:06:35.047943+0000","last_peered":"2023-09-23T19:06:35.047943+0000","last_clean":"2023-09-23T19:06:35.047943+0000","last_became_active":"2023-09-23T19:05:52.107378+0000","last_became_peered":"2023-09-23T19:05:52.107378+0000","last_unstale":"2023-09-23T19:06:35.047943+0000","last_undegraded":"2023-09-23T19:06:35.047943+0000","last_fullsized":"2023-09-23T19:06:35.047943+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.045107+0000","last_change":"2023-09-23T19:05:52.157630+0000","last_active":"2023-09-23T19:06:35.045107+0000","last_peered":"2023-09-23T19:06:35.045107+0000","last_clean":"2023-09-23T19:06:35.045107+0000","last_became_active":"2023-09-23T19:05:52.157146+0000","last_became_peered":"2023-09-23T19:05:52.157146+0000","last_unstale":"2023-09-23T19:06:35.045107+0000","last_undegraded":"2023-09-23T19:06:35.045107+0000","last_fullsized":"2023-09-23T19:06:35.045107+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":16,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.947421+0000","last_change":"2023-09-23T19:05:52.115890+0000","last_active":"2023-09-23T19:06:35.947421+0000","last_peered":"2023-09-23T19:06:35.947421+0000","last_clean":"2023-09-23T19:06:35.947421+0000","last_became_active":"2023-09-23T19:05:52.115522+0000","last_became_peered":"2023-09-23T19:05:52.115522+0000","last_unstale":"2023-09-23T19:06:35.947421+0000","last_undegraded":"2023-09-23T19:06:35.947421+0000","last_fullsized":"2023-09-23T19:06:35.947421+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.047066+0000","last_change":"2023-09-23T19:05:52.780040+0000","last_active":"2023-09-23T19:06:35.047066+0000","last_peered":"2023-09-23T19:06:35.047066+0000","last_clean":"2023-09-23T19:06:35.047066+0000","last_became_active":"2023-09-23T19:05:52.779615+0000","last_became_peered":"2023-09-23T19:05:52.779615+0000","last_unstale":"2023-09-23T19:06:35.047066+0000","last_undegraded":"2023-09-23T19:06:35.047066+0000","last_fullsized":"2023-09-23T19:06:35.047066+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.052430+0000","last_change":"2023-09-23T19:05:52.102495+0000","last_active":"2023-09-23T19:06:35.052430+0000","last_peered":"2023-09-23T19:06:35.052430+0000","last_clean":"2023-09-23T19:06:35.052430+0000","last_became_active":"2023-09-23T19:05:52.102154+0000","last_became_peered":"2023-09-23T19:05:52.102154+0000","last_unstale":"2023-09-23T19:06:35.052430+0000","last_undegraded":"2023-09-23T19:06:35.052430+0000","last_fullsized":"2023-09-23T19:06:35.052430+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.046794+0000","last_change":"2023-09-23T19:05:52.139061+0000","last_active":"2023-09-23T19:06:35.046794+0000","last_peered":"2023-09-23T19:06:35.046794+0000","last_clean":"2023-09-23T19:06:35.046794+0000","last_became_active":"2023-09-23T19:05:52.138768+0000","last_became_peered":"2023-09-23T19:05:52.138768+0000","last_unstale":"2023-09-23T19:06:35.046794+0000","last_undegraded":"2023-09-23T19:06:35.046794+0000","last_fullsized":"2023-09-23T19:06:35.046794+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.041815+0000","last_change":"2023-09-23T19:05:52.106669+0000","last_active":"2023-09-23T19:06:35.041815+0000","last_peered":"2023-09-23T19:06:35.041815+0000","last_clean":"2023-09-23T19:06:35.041815+0000","last_became_active":"2023-09-23T19:05:52.106135+0000","last_became_peered":"2023-09-23T19:05:52.106135+0000","last_unstale":"2023-09-23T19:06:35.041815+0000","last_undegraded":"2023-09-23T19:06:35.041815+0000","last_fullsized":"2023-09-23T19:06:35.041815+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.053476+0000","last_change":"2023-09-23T19:05:52.145491+0000","last_active":"2023-09-23T19:06:35.053476+0000","last_peered":"2023-09-23T19:06:35.053476+0000","last_clean":"2023-09-23T19:06:35.053476+0000","last_became_active":"2023-09-23T19:05:52.145161+0000","last_became_peered":"2023-09-23T19:05:52.145161+0000","last_unstale":"2023-09-23T19:06:35.053476+0000","last_undegraded":"2023-09-23T19:06:35.053476+0000","last_fullsized":"2023-09-23T19:06:35.053476+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.046906+0000","last_change":"2023-09-23T19:05:52.106404+0000","last_active":"2023-09-23T19:06:35.046906+0000","last_peered":"2023-09-23T19:06:35.046906+0000","last_clean":"2023-09-23T19:06:35.046906+0000","last_became_active":"2023-09-23T19:05:52.106034+0000","last_became_peered":"2023-09-23T19:05:52.106034+0000","last_unstale":"2023-09-23T19:06:35.046906+0000","last_undegraded":"2023-09-23T19:06:35.046906+0000","last_fullsized":"2023-09-23T19:06:35.046906+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.041399+0000","last_change":"2023-09-23T19:05:52.781495+0000","last_active":"2023-09-23T19:06:35.041399+0000","last_peered":"2023-09-23T19:06:35.041399+0000","last_clean":"2023-09-23T19:06:35.041399+0000","last_became_active":"2023-09-23T19:05:52.780462+0000","last_became_peered":"2023-09-23T19:05:52.780462+0000","last_unstale":"2023-09-23T19:06:35.041399+0000","last_undegraded":"2023-09-23T19:06:35.041399+0000","last_fullsized":"2023-09-23T19:06:35.041399+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.044339+0000","last_change":"2023-09-23T19:05:52.104072+0000","last_active":"2023-09-23T19:06:35.044339+0000","last_peered":"2023-09-23T19:06:35.044339+0000","last_clean":"2023-09-23T19:06:35.044339+0000","last_became_active":"2023-09-23T19:05:52.103698+0000","last_became_peered":"2023-09-23T19:05:52.103698+0000","last_unstale":"2023-09-23T19:06:35.044339+0000","last_undegraded":"2023-09-23T19:06:35.044339+0000","last_fullsized":"2023-09-23T19:06:35.044339+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.047442+0000","last_change":"2023-09-23T19:05:52.140190+0000","last_active":"2023-09-23T19:06:35.047442+0000","last_peered":"2023-09-23T19:06:35.047442+0000","last_clean":"2023-09-23T19:06:35.047442+0000","last_became_active":"2023-09-23T19:05:52.139863+0000","last_became_peered":"2023-09-23T19:05:52.139863+0000","last_unstale":"2023-09-23T19:06:35.047442+0000","last_undegraded":"2023-09-23T19:06:35.047442+0000","last_fullsized":"2023-09-23T19:06:35.047442+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"59'1","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.047503+0000","last_change":"2023-09-23T19:05:52.099985+0000","last_active":"2023-09-23T19:06:35.047503+0000","last_peered":"2023-09-23T19:06:35.047503+0000","last_clean":"2023-09-23T19:06:35.047503+0000","last_became_active":"2023-09-23T19:05:52.098695+0000","last_became_peered":"2023-09-23T19:05:52.098695+0000","last_unstale":"2023-09-23T19:06:35.047503+0000","last_undegraded":"2023-09-23T19:06:35.047503+0000","last_fullsized":"2023-09-23T19:06:35.047503+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430262,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297724,"kb_used_data":628,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447428,"statfs":{"total":95995035648,"available":95690166272,"internally_reserved":0,"allocated":643072,"data_stored":264930,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:07:10 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":0.90500000000000003,"15min":0.92600000000000005},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":1.1859999999999999,"5min":1.4019999999999999,"15min":1.4019999999999999},"last":0.66500000000000004},{"interface":"front","average":{"1min":0.85999999999999999,"5min":0.89300000000000002,"15min":0.90100000000000002},"min":{"1min":0.58099999999999996,"5min":0.56799999999999995,"15min":0.56799999999999995},"max":{"1min":1.415,"5min":1.552,"15min":1.552},"last":0.83999999999999997}]},{"osd":1,"last update":"Sat Sep 23 19:07:10 2023","interfaces":[{"interface":"back","average":{"1min":0.86299999999999999,"5min":0.88700000000000001,"15min":0.90100000000000002},"min":{"1min":0.55000000000000004,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.1379999999999999,"5min":1.226,"15min":1.226},"last":0.90800000000000003},{"interface":"front","average":{"1min":0.90500000000000003,"5min":0.94499999999999995,"15min":0.96099999999999997},"min":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":1.2230000000000001,"5min":1.498,"15min":1.498},"last":0.59699999999999998}]},{"osd":2,"last update":"Sat Sep 23 19:07:10 2023","interfaces":[{"interface":"back","average":{"1min":0.87,"5min":0.91200000000000003,"15min":0.90600000000000003},"min":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"max":{"1min":1.151,"5min":1.2010000000000001,"15min":1.2010000000000001},"last":0.73799999999999999},{"interface":"front","average":{"1min":0.93999999999999995,"5min":0.95299999999999996,"15min":0.97199999999999998},"min":{"1min":0.66100000000000003,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.198,"5min":1.417,"15min":1.417},"last":1.006}]},{"osd":3,"last update":"Sat Sep 23 19:07:10 2023","interfaces":[{"interface":"back","average":{"1min":0.89500000000000002,"5min":0.98499999999999999,"15min":1.0409999999999999},"min":{"1min":0.623,"5min":0.623,"15min":0.623},"max":{"1min":1.3049999999999999,"5min":1.4450000000000001,"15min":1.4450000000000001},"last":0.873},{"interface":"front","average":{"1min":0.97499999999999998,"5min":1.0289999999999999,"15min":1.095},"min":{"1min":0.65500000000000003,"5min":0.58999999999999997,"15min":0.58999999999999997},"max":{"1min":1.248,"5min":3.3780000000000001,"15min":3.3780000000000001},"last":0.70499999999999996}]},{"osd":4,"last update":"Sat Sep 23 19:07:10 2023","interfaces":[{"interface":"back","average":{"1min":0.93700000000000006,"5min":0.90900000000000003,"15min":0.88800000000000001},"min":{"1min":0.66200000000000003,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":1.242,"5min":1.4139999999999999,"15min":1.4139999999999999},"last":0.95799999999999996},{"interface":"front","average":{"1min":0.99299999999999999,"5min":0.97599999999999998,"15min":0.96199999999999997},"min":{"1min":0.67500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"max":{"1min":1.2370000000000001,"5min":1.5720000000000001,"15min":1.5720000000000001},"last":0.94299999999999995}]},{"osd":5,"last update":"Sat Sep 23 19:07:10 2023","interfaces":[{"interface":"back","average":{"1min":1,"5min":0.98199999999999998,"15min":0.96199999999999997},"min":{"1min":0.751,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":1.288,"5min":1.494,"15min":1.494},"last":1.087},{"interface":"front","average":{"1min":1.0589999999999999,"5min":1.0489999999999999,"15min":1.0369999999999999},"min":{"1min":0.71099999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.27,"5min":1.3779999999999999,"15min":1.3779999999999999},"last":1.038}]},{"osd":6,"last update":"Sat Sep 23 19:07:10 2023","interfaces":[{"interface":"back","average":{"1min":1.115,"5min":1.087,"15min":1.0589999999999999},"min":{"1min":0.80600000000000005,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":1.5900000000000001,"5min":2.0339999999999998,"15min":2.0339999999999998},"last":1.1319999999999999},{"interface":"front","average":{"1min":1.0940000000000001,"5min":1.109,"15min":1.089},"min":{"1min":0.68600000000000005,"5min":0.53000000000000003,"15min":0.53000000000000003},"max":{"1min":1.389,"5min":1.657,"15min":1.657},"last":1.1060000000000001}]}]},{"osd":6,"up_from":42,"seq":180388626494,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297708,"kb_used_data":612,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447444,"statfs":{"total":95995035648,"available":95690182656,"internally_reserved":0,"allocated":626688,"data_stored":264469,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:06:30 2023","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":0.873,"15min":0.91000000000000003},"min":{"1min":0.60399999999999998,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":1.1990000000000001,"5min":1.4179999999999999,"15min":1.4179999999999999},"last":0.85699999999999998},{"interface":"front","average":{"1min":0.90700000000000003,"5min":0.87,"15min":0.90300000000000002},"min":{"1min":0.66900000000000004,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.194,"5min":1.194,"15min":1.194},"last":0.98799999999999999}]},{"osd":1,"last update":"Sat Sep 23 19:06:30 2023","interfaces":[{"interface":"back","average":{"1min":1.0629999999999999,"5min":1.0309999999999999,"15min":1.083},"min":{"1min":0.60899999999999999,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":1.349,"5min":1.4379999999999999,"15min":1.4379999999999999},"last":1.0720000000000001},{"interface":"front","average":{"1min":1.05,"5min":1.0149999999999999,"15min":1.0489999999999999},"min":{"1min":0.875,"5min":0.50900000000000001,"15min":0.50900000000000001},"max":{"1min":1.2330000000000001,"5min":1.3879999999999999,"15min":1.3879999999999999},"last":0.81599999999999995}]},{"osd":2,"last update":"Sat Sep 23 19:06:30 2023","interfaces":[{"interface":"back","average":{"1min":0.89800000000000002,"5min":0.90900000000000003,"15min":0.95199999999999996},"min":{"1min":0.64500000000000002,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.1519999999999999,"5min":1.363,"15min":1.363},"last":0.91600000000000004},{"interface":"front","average":{"1min":1.028,"5min":0.91900000000000004,"15min":0.94299999999999995},"min":{"1min":0.871,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":1.228,"5min":1.3129999999999999,"15min":1.3129999999999999},"last":0.78200000000000003}]},{"osd":3,"last update":"Sat Sep 23 19:06:30 2023","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":0.97699999999999998,"15min":1.0269999999999999},"min":{"1min":0.55400000000000005,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":1.2170000000000001,"5min":1.5669999999999999,"15min":1.5669999999999999},"last":1.008},{"interface":"front","average":{"1min":0.97399999999999998,"5min":0.97799999999999998,"15min":1.0089999999999999},"min":{"1min":0.72699999999999998,"5min":0.56299999999999994,"15min":0.56299999999999994},"max":{"1min":1.2070000000000001,"5min":1.5940000000000001,"15min":1.5940000000000001},"last":1.056}]},{"osd":4,"last update":"Sat Sep 23 19:06:30 2023","interfaces":[{"interface":"back","average":{"1min":0.97299999999999998,"5min":0.89500000000000002,"15min":0.91900000000000004},"min":{"1min":0.72699999999999998,"5min":0.443,"15min":0.443},"max":{"1min":1.171,"5min":1.302,"15min":1.302},"last":1.0289999999999999},{"interface":"front","average":{"1min":0.99099999999999999,"5min":0.94599999999999995,"15min":0.94199999999999995},"min":{"1min":0.72699999999999998,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":1.2989999999999999,"5min":1.742,"15min":1.742},"last":0.72799999999999998}]},{"osd":5,"last update":"Sat Sep 23 19:06:30 2023","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":0.95199999999999996,"15min":0.98499999999999999},"min":{"1min":0.73799999999999999,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":1.268,"5min":1.3720000000000001,"15min":1.3720000000000001},"last":0.95999999999999996},{"interface":"front","average":{"1min":1.1699999999999999,"5min":1.071,"15min":1.097},"min":{"1min":0.97799999999999998,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":1.3799999999999999,"5min":1.7,"15min":1.7},"last":0.93999999999999995}]},{"osd":7,"last update":"Sat Sep 23 19:07:06 2023","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":0.98399999999999999,"15min":0.93999999999999995},"min":{"1min":0.84799999999999998,"5min":0.52600000000000002,"15min":0.52600000000000002},"max":{"1min":1.494,"5min":1.494,"15min":1.494},"last":0.88200000000000001},{"interface":"front","average":{"1min":1.2130000000000001,"5min":1.109,"15min":1.0609999999999999},"min":{"1min":0.81799999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"max":{"1min":1.823,"5min":1.823,"15min":1.823},"last":1.143}]}]},{"osd":1,"up_from":13,"seq":55834574953,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297696,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447456,"statfs":{"total":95995035648,"available":95690194944,"internally_reserved":0,"allocated":614400,"data_stored":263178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:07:06 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.68700000000000006,"15min":0.629},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.90200000000000002,"5min":1.2130000000000001,"15min":1.2130000000000001},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.70499999999999996,"5min":0.76600000000000001,"15min":0.65200000000000002},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":1.087,"5min":1.2809999999999999,"15min":1.2809999999999999},"last":0.94999999999999996}]},{"osd":2,"last update":"Sat Sep 23 19:06:44 2023","interfaces":[{"interface":"back","average":{"1min":0.79900000000000004,"5min":0.80400000000000005,"15min":0.72399999999999998},"min":{"1min":0.37,"5min":0.36899999999999999,"15min":0.34699999999999998},"max":{"1min":1.0589999999999999,"5min":1.1319999999999999,"15min":1.1319999999999999},"last":0.68700000000000006},{"interface":"front","average":{"1min":0.76000000000000001,"5min":0.76100000000000001,"15min":0.70599999999999996},"min":{"1min":0.46700000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":1.125,"5min":1.159,"15min":1.159},"last":0.61499999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:06:34 2023","interfaces":[{"interface":"back","average":{"1min":0.83599999999999997,"5min":0.83399999999999996,"15min":0.83699999999999997},"min":{"1min":0.50600000000000001,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.1699999999999999,"5min":1.21,"15min":1.21},"last":1.018},{"interface":"front","average":{"1min":0.85499999999999998,"5min":0.85599999999999998,"15min":0.82999999999999996},"min":{"1min":0.46700000000000003,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":1.0760000000000001,"5min":1.1499999999999999,"15min":1.1499999999999999},"last":0.56499999999999995}]},{"osd":4,"last update":"Sat Sep 23 19:07:06 2023","interfaces":[{"interface":"back","average":{"1min":0.84399999999999997,"5min":0.89300000000000002,"15min":0.90700000000000003},"min":{"1min":0.64800000000000002,"5min":0.54500000000000004,"15min":0.54500000000000004},"max":{"1min":1.1519999999999999,"5min":1.2849999999999999,"15min":1.2849999999999999},"last":0.85299999999999998},{"interface":"front","average":{"1min":0.92500000000000004,"5min":0.98199999999999998,"15min":1.03},"min":{"1min":0.622,"5min":0.61199999999999999,"15min":0.61199999999999999},"max":{"1min":1.373,"5min":1.373,"15min":1.524},"last":0.747}]},{"osd":5,"last update":"Sat Sep 23 19:06:49 2023","interfaces":[{"interface":"back","average":{"1min":0.93899999999999995,"5min":0.97499999999999998,"15min":1.0089999999999999},"min":{"1min":0.59699999999999998,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":1.276,"5min":1.411,"15min":1.411},"last":0.78900000000000003},{"interface":"front","average":{"1min":1.0580000000000001,"5min":1.0880000000000001,"15min":1.087},"min":{"1min":0.80600000000000005,"5min":0.66300000000000003,"15min":0.66300000000000003},"max":{"1min":1.3999999999999999,"5min":1.425,"15min":1.425},"last":1.071}]},{"osd":6,"last update":"Sat Sep 23 19:06:31 2023","interfaces":[{"interface":"back","average":{"1min":1.0129999999999999,"5min":1.022,"15min":1.04},"min":{"1min":0.66100000000000003,"5min":0.55200000000000005,"15min":0.55200000000000005},"max":{"1min":1.23,"5min":1.46,"15min":1.46},"last":0.91100000000000003},{"interface":"front","average":{"1min":1.137,"5min":1.129,"15min":1.147},"min":{"1min":0.94499999999999995,"5min":0.621,"15min":0.621},"max":{"1min":1.4710000000000001,"5min":1.4710000000000001,"15min":1.4710000000000001},"last":1.1559999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:07:10 2023","interfaces":[{"interface":"back","average":{"1min":1.0309999999999999,"5min":1.091,"15min":1.127},"min":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"max":{"1min":1.4490000000000001,"5min":1.6259999999999999,"15min":1.6259999999999999},"last":1.2110000000000001},{"interface":"front","average":{"1min":1.022,"5min":1.0620000000000001,"15min":1.0840000000000001},"min":{"1min":0.75900000000000001,"5min":0.68100000000000005,"15min":0.68100000000000005},"max":{"1min":1.4490000000000001,"5min":1.4490000000000001,"15min":1.4490000000000001},"last":0.92500000000000004}]}]},{"osd":0,"up_from":8,"seq":34359738480,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297704,"kb_used_data":608,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447448,"statfs":{"total":95995035648,"available":95690186752,"internally_reserved":0,"allocated":622592,"data_stored":263754,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 19:07:08 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":0.68100000000000005,"15min":0.63200000000000001},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.93799999999999994,"5min":1.0649999999999999,"15min":2.0979999999999999},"last":0.71799999999999997},{"interface":"front","average":{"1min":0.68100000000000005,"5min":0.72399999999999998,"15min":0.63500000000000001},"min":{"1min":0.35899999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":1.1020000000000001,"5min":1.1279999999999999,"15min":1.351},"last":0.67900000000000005}]},{"osd":2,"last update":"Sat Sep 23 19:06:50 2023","interfaces":[{"interface":"back","average":{"1min":0.79100000000000004,"5min":0.79500000000000004,"15min":0.747},"min":{"1min":0.38100000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":1.4339999999999999,"5min":1.4339999999999999,"15min":1.546},"last":0.79300000000000004},{"interface":"front","average":{"1min":0.74299999999999999,"5min":0.77800000000000002,"15min":0.72899999999999998},"min":{"1min":0.48499999999999999,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":1.2030000000000001,"5min":1.2030000000000001,"15min":1.2030000000000001},"last":0.93400000000000005}]},{"osd":3,"last update":"Sat Sep 23 19:06:23 2023","interfaces":[{"interface":"back","average":{"1min":0.86699999999999999,"5min":0.85399999999999998,"15min":0.91300000000000003},"min":{"1min":0.66000000000000003,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":1.1839999999999999,"5min":1.2629999999999999,"15min":3.1840000000000002},"last":0.84799999999999998},{"interface":"front","average":{"1min":0.84099999999999997,"5min":0.86499999999999999,"15min":0.81899999999999995},"min":{"1min":0.55100000000000005,"5min":0.377,"15min":0.377},"max":{"1min":1.381,"5min":1.5600000000000001,"15min":1.5600000000000001},"last":1.1719999999999999}]},{"osd":4,"last update":"Sat Sep 23 19:07:08 2023","interfaces":[{"interface":"back","average":{"1min":0.94599999999999995,"5min":0.99399999999999999,"15min":0.93799999999999994},"min":{"1min":0.68999999999999995,"5min":0.53000000000000003,"15min":0.53000000000000003},"max":{"1min":1.323,"5min":1.373,"15min":1.373},"last":0.90600000000000003},{"interface":"front","average":{"1min":1.018,"5min":1.008,"15min":0.96999999999999997},"min":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.649,"5min":1.649,"15min":1.649},"last":1.091}]},{"osd":5,"last update":"Sat Sep 23 19:06:50 2023","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":1.0549999999999999,"15min":1.0820000000000001},"min":{"1min":0.70899999999999996,"5min":0.61099999999999999,"15min":0.61099999999999999},"max":{"1min":1.492,"5min":1.492,"15min":1.492},"last":1.218},{"interface":"front","average":{"1min":0.98099999999999998,"5min":0.98699999999999999,"15min":0.99299999999999999},"min":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"max":{"1min":1.2290000000000001,"5min":1.3859999999999999,"15min":1.3859999999999999},"last":1.1919999999999999}]},{"osd":6,"last update":"Sat Sep 23 19:06:28 2023","interfaces":[{"interface":"back","average":{"1min":0.999,"5min":1.0249999999999999,"15min":1.073},"min":{"1min":0.69399999999999995,"5min":0.60799999999999998,"15min":0.60799999999999998},"max":{"1min":1.272,"5min":1.502,"15min":1.502},"last":1.1299999999999999},{"interface":"front","average":{"1min":1.159,"5min":1.1419999999999999,"15min":1.1759999999999999},"min":{"1min":0.90100000000000002,"5min":0.73099999999999998,"15min":0.73099999999999998},"max":{"1min":1.5289999999999999,"5min":1.5289999999999999,"15min":1.5289999999999999},"last":1.1499999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:07:08 2023","interfaces":[{"interface":"back","average":{"1min":1.0820000000000001,"5min":1.1060000000000001,"15min":1.125},"min":{"1min":0.67000000000000004,"5min":0.626,"15min":0.626},"max":{"1min":1.5840000000000001,"5min":1.655,"15min":1.655},"last":1.323},{"interface":"front","average":{"1min":1.0069999999999999,"5min":1.05,"15min":1.079},"min":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":1.377,"5min":1.595,"15min":1.595},"last":1.038}]}]},{"osd":2,"up_from":18,"seq":77309411422,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297704,"kb_used_data":608,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447448,"statfs":{"total":95995035648,"available":95690186752,"internally_reserved":0,"allocated":622592,"data_stored":263766,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:06:41 2023","interfaces":[{"interface":"back","average":{"1min":0.71099999999999997,"5min":0.74199999999999999,"15min":0.67700000000000005},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.97599999999999998,"5min":1.8979999999999999,"15min":1.8979999999999999},"last":0.93799999999999994},{"interface":"front","average":{"1min":0.72099999999999997,"5min":0.76600000000000001,"15min":0.70399999999999996},"min":{"1min":0.47099999999999997,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":1.089,"5min":1.651,"15min":1.651},"last":0.66700000000000004}]},{"osd":1,"last update":"Sat Sep 23 19:06:41 2023","interfaces":[{"interface":"back","average":{"1min":0.69999999999999996,"5min":0.74199999999999999,"15min":0.72199999999999998},"min":{"1min":0.434,"5min":0.317,"15min":0.317},"max":{"1min":1.0469999999999999,"5min":1.4490000000000001,"15min":1.4490000000000001},"last":0.753},{"interface":"front","average":{"1min":0.69599999999999995,"5min":0.81499999999999995,"15min":0.74399999999999999},"min":{"1min":0.378,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.032,"5min":1.788,"15min":1.788},"last":0.78000000000000003}]},{"osd":3,"last update":"Sat Sep 23 19:06:27 2023","interfaces":[{"interface":"back","average":{"1min":0.79900000000000004,"5min":0.80600000000000005,"15min":0.77300000000000002},"min":{"1min":0.52800000000000002,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":1.1659999999999999,"5min":1.748,"15min":1.748},"last":0.89500000000000002},{"interface":"front","average":{"1min":0.82699999999999996,"5min":0.875,"15min":0.81299999999999994},"min":{"1min":0.61599999999999999,"5min":0.443,"15min":0.443},"max":{"1min":1.1599999999999999,"5min":1.4690000000000001,"15min":1.4690000000000001},"last":0.98799999999999999}]},{"osd":4,"last update":"Sat Sep 23 19:07:05 2023","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":0.98299999999999998,"15min":0.98499999999999999},"min":{"1min":0.71099999999999997,"5min":0.60799999999999998,"15min":0.60799999999999998},"max":{"1min":1.3129999999999999,"5min":1.335,"15min":1.335},"last":1.111},{"interface":"front","average":{"1min":1.018,"5min":0.98699999999999999,"15min":0.98199999999999998},"min":{"1min":0.623,"5min":0.623,"15min":0.623},"max":{"1min":1.3580000000000001,"5min":1.554,"15min":1.554},"last":1.05}]},{"osd":5,"last update":"Sat Sep 23 19:06:51 2023","interfaces":[{"interface":"back","average":{"1min":1.0760000000000001,"5min":1.0549999999999999,"15min":1.0489999999999999},"min":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"max":{"1min":1.476,"5min":1.635,"15min":1.635},"last":1.125},{"interface":"front","average":{"1min":1.0800000000000001,"5min":1.0629999999999999,"15min":1.087},"min":{"1min":0.79800000000000004,"5min":0.65500000000000003,"15min":0.65500000000000003},"max":{"1min":1.4239999999999999,"5min":1.581,"15min":1.581},"last":1.0269999999999999}]},{"osd":6,"last update":"Sat Sep 23 19:06:27 2023","interfaces":[{"interface":"back","average":{"1min":1.103,"5min":1.083,"15min":1.119},"min":{"1min":0.873,"5min":0.68000000000000005,"15min":0.68000000000000005},"max":{"1min":1.3580000000000001,"5min":1.5,"15min":1.5},"last":1.163},{"interface":"front","average":{"1min":1.1200000000000001,"5min":1.119,"15min":1.143},"min":{"1min":0.90500000000000003,"5min":0.76300000000000001,"15min":0.76300000000000001},"max":{"1min":1.5960000000000001,"5min":1.679,"15min":1.679},"last":1.1519999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:07:11 2023","interfaces":[{"interface":"back","average":{"1min":1.0700000000000001,"5min":1.1259999999999999,"15min":1.1579999999999999},"min":{"1min":0.81599999999999995,"5min":0.73499999999999999,"15min":0.73499999999999999},"max":{"1min":1.518,"5min":1.659,"15min":1.659},"last":1.071},{"interface":"front","average":{"1min":1.091,"5min":1.131,"15min":1.1870000000000001},"min":{"1min":0.73799999999999999,"5min":0.621,"15min":0.621},"max":{"1min":1.492,"5min":1.6890000000000001,"15min":1.6890000000000001},"last":1.2090000000000001}]}]},{"osd":3,"up_from":25,"seq":107374182488,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297700,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447452,"statfs":{"total":95995035648,"available":95690190848,"internally_reserved":0,"allocated":618496,"data_stored":263268,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:06:25 2023","interfaces":[{"interface":"back","average":{"1min":0.69099999999999995,"5min":0.69699999999999995,"15min":0.66700000000000004},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":1.0389999999999999,"5min":1.286,"15min":1.286},"last":0.93500000000000005},{"interface":"front","average":{"1min":0.77100000000000002,"5min":0.753,"15min":0.75900000000000001},"min":{"1min":0.52400000000000002,"5min":0.436,"15min":0.436},"max":{"1min":1.0269999999999999,"5min":1.1120000000000001,"15min":1.1120000000000001},"last":0.96099999999999997}]},{"osd":1,"last update":"Sat Sep 23 19:06:25 2023","interfaces":[{"interface":"back","average":{"1min":0.751,"5min":0.78200000000000003,"15min":0.78400000000000003},"min":{"1min":0.52000000000000002,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":1.2270000000000001,"5min":1.2270000000000001,"15min":1.2270000000000001},"last":0.76700000000000002},{"interface":"front","average":{"1min":0.72099999999999997,"5min":0.74099999999999999,"15min":0.754},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":1.1579999999999999,"5min":1.3180000000000001,"15min":1.3180000000000001},"last":0.89500000000000002}]},{"osd":2,"last update":"Sat Sep 23 19:06:25 2023","interfaces":[{"interface":"back","average":{"1min":0.753,"5min":0.83399999999999996,"15min":0.85899999999999999},"min":{"1min":0.438,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":1.0840000000000001,"5min":1.4339999999999999,"15min":1.4339999999999999},"last":1.0009999999999999},{"interface":"front","average":{"1min":0.78400000000000003,"5min":0.83799999999999997,"15min":0.82299999999999995},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":1.0920000000000001,"5min":1.3700000000000001,"15min":1.3700000000000001},"last":1.026}]},{"osd":4,"last update":"Sat Sep 23 19:07:03 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":1.002,"15min":0.99299999999999999},"min":{"1min":0.69499999999999995,"5min":0.64300000000000002,"15min":0.64300000000000002},"max":{"1min":1.5700000000000001,"5min":1.7250000000000001,"15min":1.7250000000000001},"last":1.1359999999999999},{"interface":"front","average":{"1min":1.0169999999999999,"5min":0.99299999999999999,"15min":0.995},"min":{"1min":0.70699999999999996,"5min":0.63,"15min":0.63},"max":{"1min":1.536,"5min":1.536,"15min":1.536},"last":1.087}]},{"osd":5,"last update":"Sat Sep 23 19:06:53 2023","interfaces":[{"interface":"back","average":{"1min":1.095,"5min":1.0640000000000001,"15min":1.0680000000000001},"min":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"max":{"1min":1.5820000000000001,"5min":1.9319999999999999,"15min":1.9319999999999999},"last":1.2090000000000001},{"interface":"front","average":{"1min":1.056,"5min":1.054,"15min":1.034},"min":{"1min":0.59799999999999998,"5min":0.58599999999999997,"15min":0.58599999999999997},"max":{"1min":1.603,"5min":1.603,"15min":1.603},"last":1.1890000000000001}]},{"osd":6,"last update":"Sat Sep 23 19:06:25 2023","interfaces":[{"interface":"back","average":{"1min":1.121,"5min":1.093,"15min":1.121},"min":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"max":{"1min":1.526,"5min":1.526,"15min":1.526},"last":1.117},{"interface":"front","average":{"1min":1.125,"5min":1.179,"15min":1.246},"min":{"1min":0.628,"5min":0.628,"15min":0.628},"max":{"1min":1.4359999999999999,"5min":2.1960000000000002,"15min":2.1960000000000002},"last":1.399}]},{"osd":7,"last update":"Sat Sep 23 19:07:13 2023","interfaces":[{"interface":"back","average":{"1min":1.1519999999999999,"5min":1.131,"15min":1.1140000000000001},"min":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"max":{"1min":1.6679999999999999,"5min":1.6679999999999999,"15min":1.6679999999999999},"last":1.4570000000000001},{"interface":"front","average":{"1min":1.1100000000000001,"5min":1.131,"15min":1.145},"min":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"max":{"1min":1.52,"5min":1.5600000000000001,"15min":1.5600000000000001},"last":1.478}]}]},{"osd":4,"up_from":30,"seq":128849018959,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297720,"kb_used_data":624,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447432,"statfs":{"total":95995035648,"available":95690170368,"internally_reserved":0,"allocated":638976,"data_stored":264860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:07:10 2023","interfaces":[{"interface":"back","average":{"1min":1.04,"5min":0.97899999999999998,"15min":0.96799999999999997},"min":{"1min":0.60099999999999998,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":1.2709999999999999,"5min":1.343,"15min":1.343},"last":1.2649999999999999},{"interface":"front","average":{"1min":0.98299999999999998,"5min":0.96599999999999997,"15min":0.97299999999999998},"min":{"1min":0.64900000000000002,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.304,"5min":1.304,"15min":1.304},"last":1.216}]},{"osd":1,"last update":"Sat Sep 23 19:07:10 2023","interfaces":[{"interface":"back","average":{"1min":0.84699999999999998,"5min":0.85099999999999998,"15min":0.85899999999999999},"min":{"1min":0.495,"5min":0.495,"15min":0.495},"max":{"1min":1.0900000000000001,"5min":1.23,"15min":1.23},"last":1.1559999999999999},{"interface":"front","average":{"1min":0.91500000000000004,"5min":0.92400000000000004,"15min":0.89800000000000002},"min":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"max":{"1min":1.2190000000000001,"5min":1.4099999999999999,"15min":1.4099999999999999},"last":0.77100000000000002}]},{"osd":2,"last update":"Sat Sep 23 19:07:10 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":0.91300000000000003,"15min":0.871},"min":{"1min":0.67800000000000005,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.3380000000000001,"5min":1.343,"15min":1.343},"last":1.339},{"interface":"front","average":{"1min":0.89600000000000002,"5min":0.89700000000000002,"15min":0.877},"min":{"1min":0.629,"5min":0.59499999999999997,"15min":0.59499999999999997},"max":{"1min":1.1890000000000001,"5min":1.3089999999999999,"15min":1.3089999999999999},"last":0.95599999999999996}]},{"osd":3,"last update":"Sat Sep 23 19:07:10 2023","interfaces":[{"interface":"back","average":{"1min":0.93500000000000005,"5min":0.90600000000000003,"15min":0.91000000000000003},"min":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"max":{"1min":1.268,"5min":1.268,"15min":1.268},"last":1.125},{"interface":"front","average":{"1min":0.92600000000000005,"5min":0.91300000000000003,"15min":0.89100000000000001},"min":{"1min":0.61499999999999999,"5min":0.60199999999999998,"15min":0.60199999999999998},"max":{"1min":1.2709999999999999,"5min":1.2709999999999999,"15min":1.335},"last":0.85899999999999999}]},{"osd":5,"last update":"Sat Sep 23 19:06:54 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":0.88600000000000001,"15min":0.78600000000000003},"min":{"1min":0.52200000000000002,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":1.3560000000000001,"5min":1.373,"15min":1.373},"last":1.1879999999999999},{"interface":"front","average":{"1min":1.038,"5min":0.94999999999999996,"15min":0.90000000000000002},"min":{"1min":0.65400000000000003,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":1.327,"5min":1.407,"15min":1.407},"last":1.294}]},{"osd":6,"last update":"Sat Sep 23 19:06:23 2023","interfaces":[{"interface":"back","average":{"1min":1.046,"5min":0.96799999999999997,"15min":0.96199999999999997},"min":{"1min":0.60499999999999998,"5min":0.58099999999999996,"15min":0.58099999999999996},"max":{"1min":1.3320000000000001,"5min":1.4430000000000001,"15min":1.4430000000000001},"last":1.085},{"interface":"front","average":{"1min":1.1080000000000001,"5min":0.98399999999999999,"15min":0.95999999999999996},"min":{"1min":0.82599999999999996,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":1.377,"5min":1.492,"15min":1.492},"last":1.0629999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:07:10 2023","interfaces":[{"interface":"back","average":{"1min":1.0429999999999999,"5min":0.96299999999999997,"15min":0.95499999999999996},"min":{"1min":0.70799999999999996,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":1.595,"5min":1.595,"15min":1.595},"last":1.028},{"interface":"front","average":{"1min":1.1779999999999999,"5min":1.0600000000000001,"15min":1.03},"min":{"1min":0.87,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":1.403,"5min":1.51,"15min":1.51},"last":0.999}]}]},{"osd":5,"up_from":36,"seq":154618822726,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297696,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447456,"statfs":{"total":95995035648,"available":95690194944,"internally_reserved":0,"allocated":614400,"data_stored":263172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:06:46 2023","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":0.89000000000000001,"15min":0.91100000000000003},"min":{"1min":0.65500000000000003,"5min":0.48999999999999999,"15min":0.48999999999999999},"max":{"1min":1.161,"5min":1.2110000000000001,"15min":1.2110000000000001},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.91400000000000003,"5min":0.88,"15min":0.876},"min":{"1min":0.64900000000000002,"5min":0.54300000000000004,"15min":0.54300000000000004},"max":{"1min":1.282,"5min":1.4710000000000001,"15min":1.4710000000000001},"last":0.73399999999999999}]},{"osd":1,"last update":"Sat Sep 23 19:06:46 2023","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":0.92100000000000004,"15min":0.90900000000000003},"min":{"1min":0.59599999999999997,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.3220000000000001,"5min":1.413,"15min":1.413},"last":1.1140000000000001},{"interface":"front","average":{"1min":0.89700000000000002,"5min":0.874,"15min":0.873},"min":{"1min":0.57799999999999996,"5min":0.5,"15min":0.5},"max":{"1min":1.222,"5min":1.3160000000000001,"15min":1.3160000000000001},"last":0.66700000000000004}]},{"osd":2,"last update":"Sat Sep 23 19:06:46 2023","interfaces":[{"interface":"back","average":{"1min":0.90100000000000002,"5min":0.88800000000000001,"15min":0.88600000000000001},"min":{"1min":0.54900000000000004,"5min":0.52200000000000002,"15min":0.52200000000000002},"max":{"1min":1.2809999999999999,"5min":1.4399999999999999,"15min":1.4399999999999999},"last":1.1459999999999999},{"interface":"front","average":{"1min":0.93000000000000005,"5min":0.95199999999999996,"15min":0.96499999999999997},"min":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"max":{"1min":1.319,"5min":1.3779999999999999,"15min":1.3779999999999999},"last":1.1739999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:06:46 2023","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":0.92300000000000004,"15min":0.89900000000000002},"min":{"1min":0.68000000000000005,"5min":0.61199999999999999,"15min":0.61199999999999999},"max":{"1min":1.21,"5min":1.3,"15min":1.3},"last":0.81499999999999995},{"interface":"front","average":{"1min":0.995,"5min":1.0009999999999999,"15min":1.0049999999999999},"min":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"max":{"1min":1.2609999999999999,"5min":1.3360000000000001,"15min":1.3360000000000001},"last":1.216}]},{"osd":4,"last update":"Sat Sep 23 19:06:46 2023","interfaces":[{"interface":"back","average":{"1min":0.88800000000000001,"5min":0.85199999999999998,"15min":0.82099999999999995},"min":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":1.2030000000000001,"5min":1.655,"15min":1.655},"last":0.86299999999999999},{"interface":"front","average":{"1min":1,"5min":0.92500000000000004,"15min":0.877},"min":{"1min":0.68200000000000005,"5min":0.57299999999999995,"15min":0.57299999999999995},"max":{"1min":1.3400000000000001,"5min":2.415,"15min":2.415},"last":1.048}]},{"osd":6,"last update":"Sat Sep 23 19:06:28 2023","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":0.97899999999999998,"15min":0.94499999999999995},"min":{"1min":0.58699999999999997,"5min":0.48999999999999999,"15min":0.48999999999999999},"max":{"1min":1.151,"5min":2.0699999999999998,"15min":2.0699999999999998},"last":1.258},{"interface":"front","average":{"1min":1.0489999999999999,"5min":1.0620000000000001,"15min":1.0309999999999999},"min":{"1min":0.71499999999999997,"5min":0.54800000000000004,"15min":0.54800000000000004},"max":{"1min":1.3600000000000001,"5min":2.4769999999999999,"15min":2.4769999999999999},"last":0.90800000000000003}]},{"osd":7,"last update":"Sat Sep 23 19:07:12 2023","interfaces":[{"interface":"back","average":{"1min":1.0960000000000001,"5min":1.0589999999999999,"15min":1.034},"min":{"1min":0.82699999999999996,"5min":0.63300000000000001,"15min":0.63300000000000001},"max":{"1min":1.401,"5min":2.5049999999999999,"15min":2.5049999999999999},"last":0.97999999999999998},{"interface":"front","average":{"1min":1.155,"5min":1.111,"15min":1.0609999999999999},"min":{"1min":0.84799999999999998,"5min":0.54500000000000004,"15min":0.54500000000000004},"max":{"1min":1.3879999999999999,"5min":2.4340000000000002,"15min":2.4340000000000002},"last":1.2969999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T19:07:20.243 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T19:07:20.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:20 smithi016 bash[27237]: cluster 2023-09-23T19:07:18.939735+0000 mgr.y (mgr.14454) 145 : cluster [DBG] pgmap v125: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:20.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:20 smithi016 bash[33624]: cluster 2023-09-23T19:07:18.939735+0000 mgr.y (mgr.14454) 145 : cluster [DBG] pgmap v125: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:20.536 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:07:20 smithi016 bash[27503]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:07:20] "GET /metrics HTTP/1.1" 200 191358 "" "Prometheus/2.33.4" 2023-09-23T19:07:20.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:20 smithi092 bash[27556]: cluster 2023-09-23T19:07:18.939735+0000 mgr.y (mgr.14454) 145 : cluster [DBG] pgmap v125: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:21.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:21 smithi016 bash[27237]: audit 2023-09-23T19:07:19.448918+0000 mgr.y (mgr.14454) 146 : audit [DBG] from='client.14793 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:07:21.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:21 smithi016 bash[33624]: audit 2023-09-23T19:07:19.448918+0000 mgr.y (mgr.14454) 146 : audit [DBG] from='client.14793 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:07:21.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:21 smithi092 bash[27556]: audit 2023-09-23T19:07:19.448918+0000 mgr.y (mgr.14454) 146 : audit [DBG] from='client.14793 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:07:22.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:22 smithi016 bash[27237]: cluster 2023-09-23T19:07:20.940725+0000 mgr.y (mgr.14454) 147 : cluster [DBG] pgmap v126: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:22.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:22 smithi016 bash[33624]: cluster 2023-09-23T19:07:20.940725+0000 mgr.y (mgr.14454) 147 : cluster [DBG] pgmap v126: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:22.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:22 smithi092 bash[27556]: cluster 2023-09-23T19:07:20.940725+0000 mgr.y (mgr.14454) 147 : cluster [DBG] pgmap v126: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:24.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:24 smithi016 bash[27237]: cluster 2023-09-23T19:07:22.941338+0000 mgr.y (mgr.14454) 148 : cluster [DBG] pgmap v127: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:24.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:24 smithi016 bash[33624]: cluster 2023-09-23T19:07:22.941338+0000 mgr.y (mgr.14454) 148 : cluster [DBG] pgmap v127: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:24.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:24 smithi092 bash[27556]: cluster 2023-09-23T19:07:22.941338+0000 mgr.y (mgr.14454) 148 : cluster [DBG] pgmap v127: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:25.070 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:07:26.206 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:26 smithi016 bash[27237]: cluster 2023-09-23T19:07:24.942113+0000 mgr.y (mgr.14454) 149 : cluster [DBG] pgmap v128: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:26.206 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:26 smithi016 bash[33624]: cluster 2023-09-23T19:07:24.942113+0000 mgr.y (mgr.14454) 149 : cluster [DBG] pgmap v128: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:26.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:26 smithi092 bash[27556]: cluster 2023-09-23T19:07:24.942113+0000 mgr.y (mgr.14454) 149 : cluster [DBG] pgmap v128: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:28.053 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-09-23T19:07:28.060 INFO:teuthology.orchestra.run.smithi016.stderr:dumped all 2023-09-23T19:07:28.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:28 smithi016 bash[27237]: cluster 2023-09-23T19:07:26.943255+0000 mgr.y (mgr.14454) 150 : cluster [DBG] pgmap v129: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:28.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:28 smithi016 bash[33624]: cluster 2023-09-23T19:07:26.943255+0000 mgr.y (mgr.14454) 150 : cluster [DBG] pgmap v129: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:28.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:28 smithi092 bash[27556]: cluster 2023-09-23T19:07:26.943255+0000 mgr.y (mgr.14454) 150 : cluster [DBG] pgmap v129: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:28.784 INFO:teuthology.orchestra.run.smithi016.stdout:{"pg_ready":true,"pg_map":{"version":129,"stamp":"2023-09-23T19:07:26.942463+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2381652,"kb_used_data":4884,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":747579564,"statfs":{"total":767960285184,"available":765521473536,"internally_reserved":0,"allocated":5001216,"data_stored":2111397,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004853"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":21,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T19:06:36.673002+0000","last_change":"2023-09-23T19:06:36.673002+0000","last_active":"2023-09-23T19:06:36.673002+0000","last_peered":"2023-09-23T19:06:36.673002+0000","last_clean":"2023-09-23T19:06:36.673002+0000","last_became_active":"2023-09-23T19:06:36.671911+0000","last_became_peered":"2023-09-23T19:06:36.671911+0000","last_unstale":"2023-09-23T19:06:36.673002+0000","last_undegraded":"2023-09-23T19:06:36.673002+0000","last_fullsized":"2023-09-23T19:06:36.673002+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":17,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.041899+0000","last_change":"2023-09-23T19:05:52.099329+0000","last_active":"2023-09-23T19:06:35.041899+0000","last_peered":"2023-09-23T19:06:35.041899+0000","last_clean":"2023-09-23T19:06:35.041899+0000","last_became_active":"2023-09-23T19:05:52.098983+0000","last_became_peered":"2023-09-23T19:05:52.098983+0000","last_unstale":"2023-09-23T19:06:35.041899+0000","last_undegraded":"2023-09-23T19:06:35.041899+0000","last_fullsized":"2023-09-23T19:06:35.041899+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.046914+0000","last_change":"2023-09-23T19:05:52.132004+0000","last_active":"2023-09-23T19:06:35.046914+0000","last_peered":"2023-09-23T19:06:35.046914+0000","last_clean":"2023-09-23T19:06:35.046914+0000","last_became_active":"2023-09-23T19:05:52.131655+0000","last_became_peered":"2023-09-23T19:05:52.131655+0000","last_unstale":"2023-09-23T19:06:35.046914+0000","last_undegraded":"2023-09-23T19:06:35.046914+0000","last_fullsized":"2023-09-23T19:06:35.046914+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.047507+0000","last_change":"2023-09-23T19:05:52.100466+0000","last_active":"2023-09-23T19:06:35.047507+0000","last_peered":"2023-09-23T19:06:35.047507+0000","last_clean":"2023-09-23T19:06:35.047507+0000","last_became_active":"2023-09-23T19:05:52.100107+0000","last_became_peered":"2023-09-23T19:05:52.100107+0000","last_unstale":"2023-09-23T19:06:35.047507+0000","last_undegraded":"2023-09-23T19:06:35.047507+0000","last_fullsized":"2023-09-23T19:06:35.047507+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.048567+0000","last_change":"2023-09-23T19:05:52.139495+0000","last_active":"2023-09-23T19:06:35.048567+0000","last_peered":"2023-09-23T19:06:35.048567+0000","last_clean":"2023-09-23T19:06:35.048567+0000","last_became_active":"2023-09-23T19:05:52.139186+0000","last_became_peered":"2023-09-23T19:05:52.139186+0000","last_unstale":"2023-09-23T19:06:35.048567+0000","last_undegraded":"2023-09-23T19:06:35.048567+0000","last_fullsized":"2023-09-23T19:06:35.048567+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":15,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.052666+0000","last_change":"2023-09-23T19:05:52.107570+0000","last_active":"2023-09-23T19:06:35.052666+0000","last_peered":"2023-09-23T19:06:35.052666+0000","last_clean":"2023-09-23T19:06:35.052666+0000","last_became_active":"2023-09-23T19:05:52.107215+0000","last_became_peered":"2023-09-23T19:05:52.107215+0000","last_unstale":"2023-09-23T19:06:35.052666+0000","last_undegraded":"2023-09-23T19:06:35.052666+0000","last_fullsized":"2023-09-23T19:06:35.052666+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":21,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T19:06:36.793265+0000","last_change":"2023-09-23T19:06:36.793265+0000","last_active":"2023-09-23T19:06:36.793265+0000","last_peered":"2023-09-23T19:06:36.793265+0000","last_clean":"2023-09-23T19:06:36.793265+0000","last_became_active":"2023-09-23T19:06:36.792450+0000","last_became_peered":"2023-09-23T19:06:36.792450+0000","last_unstale":"2023-09-23T19:06:36.793265+0000","last_undegraded":"2023-09-23T19:06:36.793265+0000","last_fullsized":"2023-09-23T19:06:36.793265+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":15,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.052298+0000","last_change":"2023-09-23T19:05:52.108768+0000","last_active":"2023-09-23T19:06:35.052298+0000","last_peered":"2023-09-23T19:06:35.052298+0000","last_clean":"2023-09-23T19:06:35.052298+0000","last_became_active":"2023-09-23T19:05:52.108406+0000","last_became_peered":"2023-09-23T19:05:52.108406+0000","last_unstale":"2023-09-23T19:06:35.052298+0000","last_undegraded":"2023-09-23T19:06:35.052298+0000","last_fullsized":"2023-09-23T19:06:35.052298+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.041042+0000","last_change":"2023-09-23T19:05:52.133083+0000","last_active":"2023-09-23T19:06:35.041042+0000","last_peered":"2023-09-23T19:06:35.041042+0000","last_clean":"2023-09-23T19:06:35.041042+0000","last_became_active":"2023-09-23T19:05:52.132694+0000","last_became_peered":"2023-09-23T19:05:52.132694+0000","last_unstale":"2023-09-23T19:06:35.041042+0000","last_undegraded":"2023-09-23T19:06:35.041042+0000","last_fullsized":"2023-09-23T19:06:35.041042+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.041013+0000","last_change":"2023-09-23T19:05:52.105004+0000","last_active":"2023-09-23T19:06:35.041013+0000","last_peered":"2023-09-23T19:06:35.041013+0000","last_clean":"2023-09-23T19:06:35.041013+0000","last_became_active":"2023-09-23T19:05:52.104345+0000","last_became_peered":"2023-09-23T19:05:52.104345+0000","last_unstale":"2023-09-23T19:06:35.041013+0000","last_undegraded":"2023-09-23T19:06:35.041013+0000","last_fullsized":"2023-09-23T19:06:35.041013+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.047745+0000","last_change":"2023-09-23T19:05:52.138620+0000","last_active":"2023-09-23T19:06:35.047745+0000","last_peered":"2023-09-23T19:06:35.047745+0000","last_clean":"2023-09-23T19:06:35.047745+0000","last_became_active":"2023-09-23T19:05:52.138243+0000","last_became_peered":"2023-09-23T19:05:52.138243+0000","last_unstale":"2023-09-23T19:06:35.047745+0000","last_undegraded":"2023-09-23T19:06:35.047745+0000","last_fullsized":"2023-09-23T19:06:35.047745+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.041551+0000","last_change":"2023-09-23T19:05:52.106314+0000","last_active":"2023-09-23T19:06:35.041551+0000","last_peered":"2023-09-23T19:06:35.041551+0000","last_clean":"2023-09-23T19:06:35.041551+0000","last_became_active":"2023-09-23T19:05:52.106027+0000","last_became_peered":"2023-09-23T19:05:52.106027+0000","last_unstale":"2023-09-23T19:06:35.041551+0000","last_undegraded":"2023-09-23T19:06:35.041551+0000","last_fullsized":"2023-09-23T19:06:35.041551+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":17,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.053508+0000","last_change":"2023-09-23T19:04:02.507717+0000","last_active":"2023-09-23T19:06:35.053508+0000","last_peered":"2023-09-23T19:06:35.053508+0000","last_clean":"2023-09-23T19:06:35.053508+0000","last_became_active":"2023-09-23T19:04:02.507042+0000","last_became_peered":"2023-09-23T19:04:02.507042+0000","last_unstale":"2023-09-23T19:06:35.053508+0000","last_undegraded":"2023-09-23T19:06:35.053508+0000","last_fullsized":"2023-09-23T19:06:35.053508+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.046661+0000","last_change":"2023-09-23T19:05:52.783427+0000","last_active":"2023-09-23T19:06:35.046661+0000","last_peered":"2023-09-23T19:06:35.046661+0000","last_clean":"2023-09-23T19:06:35.046661+0000","last_became_active":"2023-09-23T19:05:52.783088+0000","last_became_peered":"2023-09-23T19:05:52.783088+0000","last_unstale":"2023-09-23T19:06:35.046661+0000","last_undegraded":"2023-09-23T19:06:35.046661+0000","last_fullsized":"2023-09-23T19:06:35.046661+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":17,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.045887+0000","last_change":"2023-09-23T19:04:02.707894+0000","last_active":"2023-09-23T19:06:35.045887+0000","last_peered":"2023-09-23T19:06:35.045887+0000","last_clean":"2023-09-23T19:06:35.045887+0000","last_became_active":"2023-09-23T19:04:02.707071+0000","last_became_peered":"2023-09-23T19:04:02.707071+0000","last_unstale":"2023-09-23T19:06:35.045887+0000","last_undegraded":"2023-09-23T19:06:35.045887+0000","last_fullsized":"2023-09-23T19:06:35.045887+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":15,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.047641+0000","last_change":"2023-09-23T19:05:52.112053+0000","last_active":"2023-09-23T19:06:35.047641+0000","last_peered":"2023-09-23T19:06:35.047641+0000","last_clean":"2023-09-23T19:06:35.047641+0000","last_became_active":"2023-09-23T19:05:52.111626+0000","last_became_peered":"2023-09-23T19:05:52.111626+0000","last_unstale":"2023-09-23T19:06:35.047641+0000","last_undegraded":"2023-09-23T19:06:35.047641+0000","last_fullsized":"2023-09-23T19:06:35.047641+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.040820+0000","last_change":"2023-09-23T19:05:52.783290+0000","last_active":"2023-09-23T19:06:35.040820+0000","last_peered":"2023-09-23T19:06:35.040820+0000","last_clean":"2023-09-23T19:06:35.040820+0000","last_became_active":"2023-09-23T19:05:52.782947+0000","last_became_peered":"2023-09-23T19:05:52.782947+0000","last_unstale":"2023-09-23T19:06:35.040820+0000","last_undegraded":"2023-09-23T19:06:35.040820+0000","last_fullsized":"2023-09-23T19:06:35.040820+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.046052+0000","last_change":"2023-09-23T19:05:52.098066+0000","last_active":"2023-09-23T19:06:35.046052+0000","last_peered":"2023-09-23T19:06:35.046052+0000","last_clean":"2023-09-23T19:06:35.046052+0000","last_became_active":"2023-09-23T19:05:52.097675+0000","last_became_peered":"2023-09-23T19:05:52.097675+0000","last_unstale":"2023-09-23T19:06:35.046052+0000","last_undegraded":"2023-09-23T19:06:35.046052+0000","last_fullsized":"2023-09-23T19:06:35.046052+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":17,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.947445+0000","last_change":"2023-09-23T19:04:02.713482+0000","last_active":"2023-09-23T19:06:35.947445+0000","last_peered":"2023-09-23T19:06:35.947445+0000","last_clean":"2023-09-23T19:06:35.947445+0000","last_became_active":"2023-09-23T19:04:02.712471+0000","last_became_peered":"2023-09-23T19:04:02.712471+0000","last_unstale":"2023-09-23T19:06:35.947445+0000","last_undegraded":"2023-09-23T19:06:35.947445+0000","last_fullsized":"2023-09-23T19:06:35.947445+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.041562+0000","last_change":"2023-09-23T19:05:52.150124+0000","last_active":"2023-09-23T19:06:35.041562+0000","last_peered":"2023-09-23T19:06:35.041562+0000","last_clean":"2023-09-23T19:06:35.041562+0000","last_became_active":"2023-09-23T19:05:52.149782+0000","last_became_peered":"2023-09-23T19:05:52.149782+0000","last_unstale":"2023-09-23T19:06:35.041562+0000","last_undegraded":"2023-09-23T19:06:35.041562+0000","last_fullsized":"2023-09-23T19:06:35.041562+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.040694+0000","last_change":"2023-09-23T19:05:52.105960+0000","last_active":"2023-09-23T19:06:35.040694+0000","last_peered":"2023-09-23T19:06:35.040694+0000","last_clean":"2023-09-23T19:06:35.040694+0000","last_became_active":"2023-09-23T19:05:52.105531+0000","last_became_peered":"2023-09-23T19:05:52.105531+0000","last_unstale":"2023-09-23T19:06:35.040694+0000","last_undegraded":"2023-09-23T19:06:35.040694+0000","last_fullsized":"2023-09-23T19:06:35.040694+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":17,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.947064+0000","last_change":"2023-09-23T19:04:02.712803+0000","last_active":"2023-09-23T19:06:35.947064+0000","last_peered":"2023-09-23T19:06:35.947064+0000","last_clean":"2023-09-23T19:06:35.947064+0000","last_became_active":"2023-09-23T19:04:02.711952+0000","last_became_peered":"2023-09-23T19:04:02.711952+0000","last_unstale":"2023-09-23T19:06:35.947064+0000","last_undegraded":"2023-09-23T19:06:35.947064+0000","last_fullsized":"2023-09-23T19:06:35.947064+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.053400+0000","last_change":"2023-09-23T19:05:52.786771+0000","last_active":"2023-09-23T19:06:35.053400+0000","last_peered":"2023-09-23T19:06:35.053400+0000","last_clean":"2023-09-23T19:06:35.053400+0000","last_became_active":"2023-09-23T19:05:52.785797+0000","last_became_peered":"2023-09-23T19:05:52.785797+0000","last_unstale":"2023-09-23T19:06:35.053400+0000","last_undegraded":"2023-09-23T19:06:35.053400+0000","last_fullsized":"2023-09-23T19:06:35.053400+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.047140+0000","last_change":"2023-09-23T19:05:52.107913+0000","last_active":"2023-09-23T19:06:35.047140+0000","last_peered":"2023-09-23T19:06:35.047140+0000","last_clean":"2023-09-23T19:06:35.047140+0000","last_became_active":"2023-09-23T19:05:52.107561+0000","last_became_peered":"2023-09-23T19:05:52.107561+0000","last_unstale":"2023-09-23T19:06:35.047140+0000","last_undegraded":"2023-09-23T19:06:35.047140+0000","last_fullsized":"2023-09-23T19:06:35.047140+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":21,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.947057+0000","last_change":"2023-09-23T19:03:04.495032+0000","last_active":"2023-09-23T19:06:35.947057+0000","last_peered":"2023-09-23T19:06:35.947057+0000","last_clean":"2023-09-23T19:06:35.947057+0000","last_became_active":"2023-09-23T19:03:04.494463+0000","last_became_peered":"2023-09-23T19:03:04.494463+0000","last_unstale":"2023-09-23T19:06:35.947057+0000","last_undegraded":"2023-09-23T19:06:35.947057+0000","last_fullsized":"2023-09-23T19:06:35.947057+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:59:33.840751+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:59:33.840751+0000","last_clean_scrub_stamp":"2023-09-23T18:59:33.840751+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":17,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.052198+0000","last_change":"2023-09-23T19:04:02.509889+0000","last_active":"2023-09-23T19:06:35.052198+0000","last_peered":"2023-09-23T19:06:35.052198+0000","last_clean":"2023-09-23T19:06:35.052198+0000","last_became_active":"2023-09-23T19:04:02.509458+0000","last_became_peered":"2023-09-23T19:04:02.509458+0000","last_unstale":"2023-09-23T19:06:35.052198+0000","last_undegraded":"2023-09-23T19:06:35.052198+0000","last_fullsized":"2023-09-23T19:06:35.052198+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.046240+0000","last_change":"2023-09-23T19:05:52.138724+0000","last_active":"2023-09-23T19:06:35.046240+0000","last_peered":"2023-09-23T19:06:35.046240+0000","last_clean":"2023-09-23T19:06:35.046240+0000","last_became_active":"2023-09-23T19:05:52.136411+0000","last_became_peered":"2023-09-23T19:05:52.136411+0000","last_unstale":"2023-09-23T19:06:35.046240+0000","last_undegraded":"2023-09-23T19:06:35.046240+0000","last_fullsized":"2023-09-23T19:06:35.046240+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.045265+0000","last_change":"2023-09-23T19:05:52.093138+0000","last_active":"2023-09-23T19:06:35.045265+0000","last_peered":"2023-09-23T19:06:35.045265+0000","last_clean":"2023-09-23T19:06:35.045265+0000","last_became_active":"2023-09-23T19:05:52.092797+0000","last_became_peered":"2023-09-23T19:05:52.092797+0000","last_unstale":"2023-09-23T19:06:35.045265+0000","last_undegraded":"2023-09-23T19:06:35.045265+0000","last_fullsized":"2023-09-23T19:06:35.045265+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":17,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.040695+0000","last_change":"2023-09-23T19:04:02.707164+0000","last_active":"2023-09-23T19:06:35.040695+0000","last_peered":"2023-09-23T19:06:35.040695+0000","last_clean":"2023-09-23T19:06:35.040695+0000","last_became_active":"2023-09-23T19:04:02.706747+0000","last_became_peered":"2023-09-23T19:04:02.706747+0000","last_unstale":"2023-09-23T19:06:35.040695+0000","last_undegraded":"2023-09-23T19:06:35.040695+0000","last_fullsized":"2023-09-23T19:06:35.040695+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.045920+0000","last_change":"2023-09-23T19:05:52.137816+0000","last_active":"2023-09-23T19:06:35.045920+0000","last_peered":"2023-09-23T19:06:35.045920+0000","last_clean":"2023-09-23T19:06:35.045920+0000","last_became_active":"2023-09-23T19:05:52.137278+0000","last_became_peered":"2023-09-23T19:05:52.137278+0000","last_unstale":"2023-09-23T19:06:35.045920+0000","last_undegraded":"2023-09-23T19:06:35.045920+0000","last_fullsized":"2023-09-23T19:06:35.045920+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.047806+0000","last_change":"2023-09-23T19:05:52.108160+0000","last_active":"2023-09-23T19:06:35.047806+0000","last_peered":"2023-09-23T19:06:35.047806+0000","last_clean":"2023-09-23T19:06:35.047806+0000","last_became_active":"2023-09-23T19:05:52.107802+0000","last_became_peered":"2023-09-23T19:05:52.107802+0000","last_unstale":"2023-09-23T19:06:35.047806+0000","last_undegraded":"2023-09-23T19:06:35.047806+0000","last_fullsized":"2023-09-23T19:06:35.047806+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":17,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.052124+0000","last_change":"2023-09-23T19:04:02.509914+0000","last_active":"2023-09-23T19:06:35.052124+0000","last_peered":"2023-09-23T19:06:35.052124+0000","last_clean":"2023-09-23T19:06:35.052124+0000","last_became_active":"2023-09-23T19:04:02.509532+0000","last_became_peered":"2023-09-23T19:04:02.509532+0000","last_unstale":"2023-09-23T19:06:35.052124+0000","last_undegraded":"2023-09-23T19:06:35.052124+0000","last_fullsized":"2023-09-23T19:06:35.052124+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":10,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T19:06:36.052412+0000","last_change":"2023-09-23T19:06:36.052412+0000","last_active":"2023-09-23T19:06:36.052412+0000","last_peered":"2023-09-23T19:06:36.052412+0000","last_clean":"2023-09-23T19:06:36.052412+0000","last_became_active":"2023-09-23T19:06:36.052036+0000","last_became_peered":"2023-09-23T19:06:36.052036+0000","last_unstale":"2023-09-23T19:06:36.052412+0000","last_undegraded":"2023-09-23T19:06:36.052412+0000","last_fullsized":"2023-09-23T19:06:36.052412+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.052524+0000","last_change":"2023-09-23T19:05:52.115013+0000","last_active":"2023-09-23T19:06:35.052524+0000","last_peered":"2023-09-23T19:06:35.052524+0000","last_clean":"2023-09-23T19:06:35.052524+0000","last_became_active":"2023-09-23T19:05:52.114762+0000","last_became_peered":"2023-09-23T19:05:52.114762+0000","last_unstale":"2023-09-23T19:06:35.052524+0000","last_undegraded":"2023-09-23T19:06:35.052524+0000","last_fullsized":"2023-09-23T19:06:35.052524+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":17,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.045795+0000","last_change":"2023-09-23T19:04:02.707302+0000","last_active":"2023-09-23T19:06:35.045795+0000","last_peered":"2023-09-23T19:06:35.045795+0000","last_clean":"2023-09-23T19:06:35.045795+0000","last_became_active":"2023-09-23T19:04:02.706904+0000","last_became_peered":"2023-09-23T19:04:02.706904+0000","last_unstale":"2023-09-23T19:06:35.045795+0000","last_undegraded":"2023-09-23T19:06:35.045795+0000","last_fullsized":"2023-09-23T19:06:35.045795+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.046282+0000","last_change":"2023-09-23T19:05:52.142253+0000","last_active":"2023-09-23T19:06:35.046282+0000","last_peered":"2023-09-23T19:06:35.046282+0000","last_clean":"2023-09-23T19:06:35.046282+0000","last_became_active":"2023-09-23T19:05:52.141613+0000","last_became_peered":"2023-09-23T19:05:52.141613+0000","last_unstale":"2023-09-23T19:06:35.046282+0000","last_undegraded":"2023-09-23T19:06:35.046282+0000","last_fullsized":"2023-09-23T19:06:35.046282+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.046701+0000","last_change":"2023-09-23T19:05:52.110960+0000","last_active":"2023-09-23T19:06:35.046701+0000","last_peered":"2023-09-23T19:06:35.046701+0000","last_clean":"2023-09-23T19:06:35.046701+0000","last_became_active":"2023-09-23T19:05:52.110444+0000","last_became_peered":"2023-09-23T19:05:52.110444+0000","last_unstale":"2023-09-23T19:06:35.046701+0000","last_undegraded":"2023-09-23T19:06:35.046701+0000","last_fullsized":"2023-09-23T19:06:35.046701+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.048963+0000","last_change":"2023-09-23T19:05:52.146508+0000","last_active":"2023-09-23T19:06:35.048963+0000","last_peered":"2023-09-23T19:06:35.048963+0000","last_clean":"2023-09-23T19:06:35.048963+0000","last_became_active":"2023-09-23T19:05:52.146192+0000","last_became_peered":"2023-09-23T19:05:52.146192+0000","last_unstale":"2023-09-23T19:06:35.048963+0000","last_undegraded":"2023-09-23T19:06:35.048963+0000","last_fullsized":"2023-09-23T19:06:35.048963+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.047974+0000","last_change":"2023-09-23T19:05:52.107974+0000","last_active":"2023-09-23T19:06:35.047974+0000","last_peered":"2023-09-23T19:06:35.047974+0000","last_clean":"2023-09-23T19:06:35.047974+0000","last_became_active":"2023-09-23T19:05:52.107596+0000","last_became_peered":"2023-09-23T19:05:52.107596+0000","last_unstale":"2023-09-23T19:06:35.047974+0000","last_undegraded":"2023-09-23T19:06:35.047974+0000","last_fullsized":"2023-09-23T19:06:35.047974+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.049605+0000","last_change":"2023-09-23T19:05:52.145950+0000","last_active":"2023-09-23T19:06:35.049605+0000","last_peered":"2023-09-23T19:06:35.049605+0000","last_clean":"2023-09-23T19:06:35.049605+0000","last_became_active":"2023-09-23T19:05:52.145147+0000","last_became_peered":"2023-09-23T19:05:52.145147+0000","last_unstale":"2023-09-23T19:06:35.049605+0000","last_undegraded":"2023-09-23T19:06:35.049605+0000","last_fullsized":"2023-09-23T19:06:35.049605+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.053189+0000","last_change":"2023-09-23T19:05:52.110940+0000","last_active":"2023-09-23T19:06:35.053189+0000","last_peered":"2023-09-23T19:06:35.053189+0000","last_clean":"2023-09-23T19:06:35.053189+0000","last_became_active":"2023-09-23T19:05:52.110544+0000","last_became_peered":"2023-09-23T19:05:52.110544+0000","last_unstale":"2023-09-23T19:06:35.053189+0000","last_undegraded":"2023-09-23T19:06:35.053189+0000","last_fullsized":"2023-09-23T19:06:35.053189+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.048126+0000","last_change":"2023-09-23T19:05:52.138696+0000","last_active":"2023-09-23T19:06:35.048126+0000","last_peered":"2023-09-23T19:06:35.048126+0000","last_clean":"2023-09-23T19:06:35.048126+0000","last_became_active":"2023-09-23T19:05:52.138272+0000","last_became_peered":"2023-09-23T19:05:52.138272+0000","last_unstale":"2023-09-23T19:06:35.048126+0000","last_undegraded":"2023-09-23T19:06:35.048126+0000","last_fullsized":"2023-09-23T19:06:35.048126+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":15,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.047312+0000","last_change":"2023-09-23T19:05:52.110010+0000","last_active":"2023-09-23T19:06:35.047312+0000","last_peered":"2023-09-23T19:06:35.047312+0000","last_clean":"2023-09-23T19:06:35.047312+0000","last_became_active":"2023-09-23T19:05:52.109644+0000","last_became_peered":"2023-09-23T19:05:52.109644+0000","last_unstale":"2023-09-23T19:06:35.047312+0000","last_undegraded":"2023-09-23T19:06:35.047312+0000","last_fullsized":"2023-09-23T19:06:35.047312+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.041171+0000","last_change":"2023-09-23T19:05:52.139974+0000","last_active":"2023-09-23T19:06:35.041171+0000","last_peered":"2023-09-23T19:06:35.041171+0000","last_clean":"2023-09-23T19:06:35.041171+0000","last_became_active":"2023-09-23T19:05:52.139627+0000","last_became_peered":"2023-09-23T19:05:52.139627+0000","last_unstale":"2023-09-23T19:06:35.041171+0000","last_undegraded":"2023-09-23T19:06:35.041171+0000","last_fullsized":"2023-09-23T19:06:35.041171+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.052766+0000","last_change":"2023-09-23T19:05:52.110253+0000","last_active":"2023-09-23T19:06:35.052766+0000","last_peered":"2023-09-23T19:06:35.052766+0000","last_clean":"2023-09-23T19:06:35.052766+0000","last_became_active":"2023-09-23T19:05:52.109817+0000","last_became_peered":"2023-09-23T19:05:52.109817+0000","last_unstale":"2023-09-23T19:06:35.052766+0000","last_undegraded":"2023-09-23T19:06:35.052766+0000","last_fullsized":"2023-09-23T19:06:35.052766+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.053084+0000","last_change":"2023-09-23T19:05:52.786764+0000","last_active":"2023-09-23T19:06:35.053084+0000","last_peered":"2023-09-23T19:06:35.053084+0000","last_clean":"2023-09-23T19:06:35.053084+0000","last_became_active":"2023-09-23T19:05:52.785936+0000","last_became_peered":"2023-09-23T19:05:52.785936+0000","last_unstale":"2023-09-23T19:06:35.053084+0000","last_undegraded":"2023-09-23T19:06:35.053084+0000","last_fullsized":"2023-09-23T19:06:35.053084+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.947610+0000","last_change":"2023-09-23T19:05:52.114108+0000","last_active":"2023-09-23T19:06:35.947610+0000","last_peered":"2023-09-23T19:06:35.947610+0000","last_clean":"2023-09-23T19:06:35.947610+0000","last_became_active":"2023-09-23T19:05:52.113609+0000","last_became_peered":"2023-09-23T19:05:52.113609+0000","last_unstale":"2023-09-23T19:06:35.947610+0000","last_undegraded":"2023-09-23T19:06:35.947610+0000","last_fullsized":"2023-09-23T19:06:35.947610+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.047458+0000","last_change":"2023-09-23T19:05:52.146957+0000","last_active":"2023-09-23T19:06:35.047458+0000","last_peered":"2023-09-23T19:06:35.047458+0000","last_clean":"2023-09-23T19:06:35.047458+0000","last_became_active":"2023-09-23T19:05:52.145201+0000","last_became_peered":"2023-09-23T19:05:52.145201+0000","last_unstale":"2023-09-23T19:06:35.047458+0000","last_undegraded":"2023-09-23T19:06:35.047458+0000","last_fullsized":"2023-09-23T19:06:35.047458+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":15,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.947587+0000","last_change":"2023-09-23T19:05:52.113304+0000","last_active":"2023-09-23T19:06:35.947587+0000","last_peered":"2023-09-23T19:06:35.947587+0000","last_clean":"2023-09-23T19:06:35.947587+0000","last_became_active":"2023-09-23T19:05:52.112945+0000","last_became_peered":"2023-09-23T19:05:52.112945+0000","last_unstale":"2023-09-23T19:06:35.947587+0000","last_undegraded":"2023-09-23T19:06:35.947587+0000","last_fullsized":"2023-09-23T19:06:35.947587+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.051772+0000","last_change":"2023-09-23T19:05:52.789323+0000","last_active":"2023-09-23T19:06:35.051772+0000","last_peered":"2023-09-23T19:06:35.051772+0000","last_clean":"2023-09-23T19:06:35.051772+0000","last_became_active":"2023-09-23T19:05:52.788820+0000","last_became_peered":"2023-09-23T19:05:52.788820+0000","last_unstale":"2023-09-23T19:06:35.051772+0000","last_undegraded":"2023-09-23T19:06:35.051772+0000","last_fullsized":"2023-09-23T19:06:35.051772+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.947900+0000","last_change":"2023-09-23T19:05:52.112710+0000","last_active":"2023-09-23T19:06:35.947900+0000","last_peered":"2023-09-23T19:06:35.947900+0000","last_clean":"2023-09-23T19:06:35.947900+0000","last_became_active":"2023-09-23T19:05:52.112362+0000","last_became_peered":"2023-09-23T19:05:52.112362+0000","last_unstale":"2023-09-23T19:06:35.947900+0000","last_undegraded":"2023-09-23T19:06:35.947900+0000","last_fullsized":"2023-09-23T19:06:35.947900+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.047556+0000","last_change":"2023-09-23T19:05:52.780101+0000","last_active":"2023-09-23T19:06:35.047556+0000","last_peered":"2023-09-23T19:06:35.047556+0000","last_clean":"2023-09-23T19:06:35.047556+0000","last_became_active":"2023-09-23T19:05:52.779607+0000","last_became_peered":"2023-09-23T19:05:52.779607+0000","last_unstale":"2023-09-23T19:06:35.047556+0000","last_undegraded":"2023-09-23T19:06:35.047556+0000","last_fullsized":"2023-09-23T19:06:35.047556+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":21,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.052286+0000","last_change":"2023-09-23T19:05:52.103917+0000","last_active":"2023-09-23T19:06:35.052286+0000","last_peered":"2023-09-23T19:06:35.052286+0000","last_clean":"2023-09-23T19:06:35.052286+0000","last_became_active":"2023-09-23T19:05:52.103551+0000","last_became_peered":"2023-09-23T19:05:52.103551+0000","last_unstale":"2023-09-23T19:06:35.052286+0000","last_undegraded":"2023-09-23T19:06:35.052286+0000","last_fullsized":"2023-09-23T19:06:35.052286+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.040773+0000","last_change":"2023-09-23T19:05:52.785109+0000","last_active":"2023-09-23T19:06:35.040773+0000","last_peered":"2023-09-23T19:06:35.040773+0000","last_clean":"2023-09-23T19:06:35.040773+0000","last_became_active":"2023-09-23T19:05:52.784196+0000","last_became_peered":"2023-09-23T19:05:52.784196+0000","last_unstale":"2023-09-23T19:06:35.040773+0000","last_undegraded":"2023-09-23T19:06:35.040773+0000","last_fullsized":"2023-09-23T19:06:35.040773+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.946866+0000","last_change":"2023-09-23T19:05:52.114186+0000","last_active":"2023-09-23T19:06:35.946866+0000","last_peered":"2023-09-23T19:06:35.946866+0000","last_clean":"2023-09-23T19:06:35.946866+0000","last_became_active":"2023-09-23T19:05:52.113635+0000","last_became_peered":"2023-09-23T19:05:52.113635+0000","last_unstale":"2023-09-23T19:06:35.946866+0000","last_undegraded":"2023-09-23T19:06:35.946866+0000","last_fullsized":"2023-09-23T19:06:35.946866+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.052419+0000","last_change":"2023-09-23T19:05:52.789241+0000","last_active":"2023-09-23T19:06:35.052419+0000","last_peered":"2023-09-23T19:06:35.052419+0000","last_clean":"2023-09-23T19:06:35.052419+0000","last_became_active":"2023-09-23T19:05:52.788902+0000","last_became_peered":"2023-09-23T19:05:52.788902+0000","last_unstale":"2023-09-23T19:06:35.052419+0000","last_undegraded":"2023-09-23T19:06:35.052419+0000","last_fullsized":"2023-09-23T19:06:35.052419+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.040778+0000","last_change":"2023-09-23T19:05:52.105482+0000","last_active":"2023-09-23T19:06:35.040778+0000","last_peered":"2023-09-23T19:06:35.040778+0000","last_clean":"2023-09-23T19:06:35.040778+0000","last_became_active":"2023-09-23T19:05:52.105125+0000","last_became_peered":"2023-09-23T19:05:52.105125+0000","last_unstale":"2023-09-23T19:06:35.040778+0000","last_undegraded":"2023-09-23T19:06:35.040778+0000","last_fullsized":"2023-09-23T19:06:35.040778+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.047352+0000","last_change":"2023-09-23T19:05:52.781046+0000","last_active":"2023-09-23T19:06:35.047352+0000","last_peered":"2023-09-23T19:06:35.047352+0000","last_clean":"2023-09-23T19:06:35.047352+0000","last_became_active":"2023-09-23T19:05:52.780115+0000","last_became_peered":"2023-09-23T19:05:52.780115+0000","last_unstale":"2023-09-23T19:06:35.047352+0000","last_undegraded":"2023-09-23T19:06:35.047352+0000","last_fullsized":"2023-09-23T19:06:35.047352+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":15,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.947239+0000","last_change":"2023-09-23T19:05:52.111942+0000","last_active":"2023-09-23T19:06:35.947239+0000","last_peered":"2023-09-23T19:06:35.947239+0000","last_clean":"2023-09-23T19:06:35.947239+0000","last_became_active":"2023-09-23T19:05:52.111546+0000","last_became_peered":"2023-09-23T19:05:52.111546+0000","last_unstale":"2023-09-23T19:06:35.947239+0000","last_undegraded":"2023-09-23T19:06:35.947239+0000","last_fullsized":"2023-09-23T19:06:35.947239+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.048024+0000","last_change":"2023-09-23T19:05:52.145659+0000","last_active":"2023-09-23T19:06:35.048024+0000","last_peered":"2023-09-23T19:06:35.048024+0000","last_clean":"2023-09-23T19:06:35.048024+0000","last_became_active":"2023-09-23T19:05:52.145156+0000","last_became_peered":"2023-09-23T19:05:52.145156+0000","last_unstale":"2023-09-23T19:06:35.048024+0000","last_undegraded":"2023-09-23T19:06:35.048024+0000","last_fullsized":"2023-09-23T19:06:35.048024+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"59'6","reported_seq":19,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.047943+0000","last_change":"2023-09-23T19:05:52.107727+0000","last_active":"2023-09-23T19:06:35.047943+0000","last_peered":"2023-09-23T19:06:35.047943+0000","last_clean":"2023-09-23T19:06:35.047943+0000","last_became_active":"2023-09-23T19:05:52.107378+0000","last_became_peered":"2023-09-23T19:05:52.107378+0000","last_unstale":"2023-09-23T19:06:35.047943+0000","last_undegraded":"2023-09-23T19:06:35.047943+0000","last_fullsized":"2023-09-23T19:06:35.047943+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.045107+0000","last_change":"2023-09-23T19:05:52.157630+0000","last_active":"2023-09-23T19:06:35.045107+0000","last_peered":"2023-09-23T19:06:35.045107+0000","last_clean":"2023-09-23T19:06:35.045107+0000","last_became_active":"2023-09-23T19:05:52.157146+0000","last_became_peered":"2023-09-23T19:05:52.157146+0000","last_unstale":"2023-09-23T19:06:35.045107+0000","last_undegraded":"2023-09-23T19:06:35.045107+0000","last_fullsized":"2023-09-23T19:06:35.045107+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":16,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.947421+0000","last_change":"2023-09-23T19:05:52.115890+0000","last_active":"2023-09-23T19:06:35.947421+0000","last_peered":"2023-09-23T19:06:35.947421+0000","last_clean":"2023-09-23T19:06:35.947421+0000","last_became_active":"2023-09-23T19:05:52.115522+0000","last_became_peered":"2023-09-23T19:05:52.115522+0000","last_unstale":"2023-09-23T19:06:35.947421+0000","last_undegraded":"2023-09-23T19:06:35.947421+0000","last_fullsized":"2023-09-23T19:06:35.947421+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.047066+0000","last_change":"2023-09-23T19:05:52.780040+0000","last_active":"2023-09-23T19:06:35.047066+0000","last_peered":"2023-09-23T19:06:35.047066+0000","last_clean":"2023-09-23T19:06:35.047066+0000","last_became_active":"2023-09-23T19:05:52.779615+0000","last_became_peered":"2023-09-23T19:05:52.779615+0000","last_unstale":"2023-09-23T19:06:35.047066+0000","last_undegraded":"2023-09-23T19:06:35.047066+0000","last_fullsized":"2023-09-23T19:06:35.047066+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.052430+0000","last_change":"2023-09-23T19:05:52.102495+0000","last_active":"2023-09-23T19:06:35.052430+0000","last_peered":"2023-09-23T19:06:35.052430+0000","last_clean":"2023-09-23T19:06:35.052430+0000","last_became_active":"2023-09-23T19:05:52.102154+0000","last_became_peered":"2023-09-23T19:05:52.102154+0000","last_unstale":"2023-09-23T19:06:35.052430+0000","last_undegraded":"2023-09-23T19:06:35.052430+0000","last_fullsized":"2023-09-23T19:06:35.052430+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.046794+0000","last_change":"2023-09-23T19:05:52.139061+0000","last_active":"2023-09-23T19:06:35.046794+0000","last_peered":"2023-09-23T19:06:35.046794+0000","last_clean":"2023-09-23T19:06:35.046794+0000","last_became_active":"2023-09-23T19:05:52.138768+0000","last_became_peered":"2023-09-23T19:05:52.138768+0000","last_unstale":"2023-09-23T19:06:35.046794+0000","last_undegraded":"2023-09-23T19:06:35.046794+0000","last_fullsized":"2023-09-23T19:06:35.046794+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.041815+0000","last_change":"2023-09-23T19:05:52.106669+0000","last_active":"2023-09-23T19:06:35.041815+0000","last_peered":"2023-09-23T19:06:35.041815+0000","last_clean":"2023-09-23T19:06:35.041815+0000","last_became_active":"2023-09-23T19:05:52.106135+0000","last_became_peered":"2023-09-23T19:05:52.106135+0000","last_unstale":"2023-09-23T19:06:35.041815+0000","last_undegraded":"2023-09-23T19:06:35.041815+0000","last_fullsized":"2023-09-23T19:06:35.041815+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.053476+0000","last_change":"2023-09-23T19:05:52.145491+0000","last_active":"2023-09-23T19:06:35.053476+0000","last_peered":"2023-09-23T19:06:35.053476+0000","last_clean":"2023-09-23T19:06:35.053476+0000","last_became_active":"2023-09-23T19:05:52.145161+0000","last_became_peered":"2023-09-23T19:05:52.145161+0000","last_unstale":"2023-09-23T19:06:35.053476+0000","last_undegraded":"2023-09-23T19:06:35.053476+0000","last_fullsized":"2023-09-23T19:06:35.053476+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.046906+0000","last_change":"2023-09-23T19:05:52.106404+0000","last_active":"2023-09-23T19:06:35.046906+0000","last_peered":"2023-09-23T19:06:35.046906+0000","last_clean":"2023-09-23T19:06:35.046906+0000","last_became_active":"2023-09-23T19:05:52.106034+0000","last_became_peered":"2023-09-23T19:05:52.106034+0000","last_unstale":"2023-09-23T19:06:35.046906+0000","last_undegraded":"2023-09-23T19:06:35.046906+0000","last_fullsized":"2023-09-23T19:06:35.046906+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.041399+0000","last_change":"2023-09-23T19:05:52.781495+0000","last_active":"2023-09-23T19:06:35.041399+0000","last_peered":"2023-09-23T19:06:35.041399+0000","last_clean":"2023-09-23T19:06:35.041399+0000","last_became_active":"2023-09-23T19:05:52.780462+0000","last_became_peered":"2023-09-23T19:05:52.780462+0000","last_unstale":"2023-09-23T19:06:35.041399+0000","last_undegraded":"2023-09-23T19:06:35.041399+0000","last_fullsized":"2023-09-23T19:06:35.041399+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.044339+0000","last_change":"2023-09-23T19:05:52.104072+0000","last_active":"2023-09-23T19:06:35.044339+0000","last_peered":"2023-09-23T19:06:35.044339+0000","last_clean":"2023-09-23T19:06:35.044339+0000","last_became_active":"2023-09-23T19:05:52.103698+0000","last_became_peered":"2023-09-23T19:05:52.103698+0000","last_unstale":"2023-09-23T19:06:35.044339+0000","last_undegraded":"2023-09-23T19:06:35.044339+0000","last_fullsized":"2023-09-23T19:06:35.044339+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.047442+0000","last_change":"2023-09-23T19:05:52.140190+0000","last_active":"2023-09-23T19:06:35.047442+0000","last_peered":"2023-09-23T19:06:35.047442+0000","last_clean":"2023-09-23T19:06:35.047442+0000","last_became_active":"2023-09-23T19:05:52.139863+0000","last_became_peered":"2023-09-23T19:05:52.139863+0000","last_unstale":"2023-09-23T19:06:35.047442+0000","last_undegraded":"2023-09-23T19:06:35.047442+0000","last_fullsized":"2023-09-23T19:06:35.047442+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"59'1","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T19:06:35.047503+0000","last_change":"2023-09-23T19:05:52.099985+0000","last_active":"2023-09-23T19:06:35.047503+0000","last_peered":"2023-09-23T19:06:35.047503+0000","last_clean":"2023-09-23T19:06:35.047503+0000","last_became_active":"2023-09-23T19:05:52.098695+0000","last_became_peered":"2023-09-23T19:05:52.098695+0000","last_unstale":"2023-09-23T19:06:35.047503+0000","last_undegraded":"2023-09-23T19:06:35.047503+0000","last_fullsized":"2023-09-23T19:06:35.047503+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430264,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297724,"kb_used_data":628,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447428,"statfs":{"total":95995035648,"available":95690166272,"internally_reserved":0,"allocated":643072,"data_stored":264930,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:07:10 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":0.90500000000000003,"15min":0.92600000000000005},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":1.1859999999999999,"5min":1.4019999999999999,"15min":1.4019999999999999},"last":0.90800000000000003},{"interface":"front","average":{"1min":0.85999999999999999,"5min":0.89300000000000002,"15min":0.90100000000000002},"min":{"1min":0.58099999999999996,"5min":0.56799999999999995,"15min":0.56799999999999995},"max":{"1min":1.415,"5min":1.552,"15min":1.552},"last":1.0189999999999999}]},{"osd":1,"last update":"Sat Sep 23 19:07:10 2023","interfaces":[{"interface":"back","average":{"1min":0.86299999999999999,"5min":0.88700000000000001,"15min":0.90100000000000002},"min":{"1min":0.55000000000000004,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.1379999999999999,"5min":1.226,"15min":1.226},"last":1.149},{"interface":"front","average":{"1min":0.90500000000000003,"5min":0.94499999999999995,"15min":0.96099999999999997},"min":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":1.2230000000000001,"5min":1.498,"15min":1.498},"last":0.93300000000000005}]},{"osd":2,"last update":"Sat Sep 23 19:07:10 2023","interfaces":[{"interface":"back","average":{"1min":0.87,"5min":0.91200000000000003,"15min":0.90600000000000003},"min":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"max":{"1min":1.151,"5min":1.2010000000000001,"15min":1.2010000000000001},"last":1.1839999999999999},{"interface":"front","average":{"1min":0.93999999999999995,"5min":0.95299999999999996,"15min":0.97199999999999998},"min":{"1min":0.66100000000000003,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.198,"5min":1.417,"15min":1.417},"last":0.95299999999999996}]},{"osd":3,"last update":"Sat Sep 23 19:07:10 2023","interfaces":[{"interface":"back","average":{"1min":0.89500000000000002,"5min":0.98499999999999999,"15min":1.0409999999999999},"min":{"1min":0.623,"5min":0.623,"15min":0.623},"max":{"1min":1.3049999999999999,"5min":1.4450000000000001,"15min":1.4450000000000001},"last":1.1359999999999999},{"interface":"front","average":{"1min":0.97499999999999998,"5min":1.0289999999999999,"15min":1.095},"min":{"1min":0.65500000000000003,"5min":0.58999999999999997,"15min":0.58999999999999997},"max":{"1min":1.248,"5min":3.3780000000000001,"15min":3.3780000000000001},"last":0.76800000000000002}]},{"osd":4,"last update":"Sat Sep 23 19:07:10 2023","interfaces":[{"interface":"back","average":{"1min":0.93700000000000006,"5min":0.90900000000000003,"15min":0.88800000000000001},"min":{"1min":0.66200000000000003,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":1.242,"5min":1.4139999999999999,"15min":1.4139999999999999},"last":0.66200000000000003},{"interface":"front","average":{"1min":0.99299999999999999,"5min":0.97599999999999998,"15min":0.96199999999999997},"min":{"1min":0.67500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"max":{"1min":1.2370000000000001,"5min":1.5720000000000001,"15min":1.5720000000000001},"last":0.99399999999999999}]},{"osd":5,"last update":"Sat Sep 23 19:07:10 2023","interfaces":[{"interface":"back","average":{"1min":1,"5min":0.98199999999999998,"15min":0.96199999999999997},"min":{"1min":0.751,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":1.288,"5min":1.494,"15min":1.494},"last":1.0429999999999999},{"interface":"front","average":{"1min":1.0589999999999999,"5min":1.0489999999999999,"15min":1.0369999999999999},"min":{"1min":0.71099999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.27,"5min":1.3779999999999999,"15min":1.3779999999999999},"last":0.84099999999999997}]},{"osd":6,"last update":"Sat Sep 23 19:07:10 2023","interfaces":[{"interface":"back","average":{"1min":1.115,"5min":1.087,"15min":1.0589999999999999},"min":{"1min":0.80600000000000005,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":1.5900000000000001,"5min":2.0339999999999998,"15min":2.0339999999999998},"last":0.97999999999999998},{"interface":"front","average":{"1min":1.0940000000000001,"5min":1.109,"15min":1.089},"min":{"1min":0.68600000000000005,"5min":0.53000000000000003,"15min":0.53000000000000003},"max":{"1min":1.389,"5min":1.657,"15min":1.657},"last":1.087}]}]},{"osd":6,"up_from":42,"seq":180388626496,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297708,"kb_used_data":612,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447444,"statfs":{"total":95995035648,"available":95690182656,"internally_reserved":0,"allocated":626688,"data_stored":264469,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:06:30 2023","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":0.873,"15min":0.91000000000000003},"min":{"1min":0.60399999999999998,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":1.1990000000000001,"5min":1.4179999999999999,"15min":1.4179999999999999},"last":1.0529999999999999},{"interface":"front","average":{"1min":0.90700000000000003,"5min":0.87,"15min":0.90300000000000002},"min":{"1min":0.66900000000000004,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.194,"5min":1.194,"15min":1.194},"last":0.78600000000000003}]},{"osd":1,"last update":"Sat Sep 23 19:06:30 2023","interfaces":[{"interface":"back","average":{"1min":1.0629999999999999,"5min":1.0309999999999999,"15min":1.083},"min":{"1min":0.60899999999999999,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":1.349,"5min":1.4379999999999999,"15min":1.4379999999999999},"last":1.1339999999999999},{"interface":"front","average":{"1min":1.05,"5min":1.0149999999999999,"15min":1.0489999999999999},"min":{"1min":0.875,"5min":0.50900000000000001,"15min":0.50900000000000001},"max":{"1min":1.2330000000000001,"5min":1.3879999999999999,"15min":1.3879999999999999},"last":1.0920000000000001}]},{"osd":2,"last update":"Sat Sep 23 19:06:30 2023","interfaces":[{"interface":"back","average":{"1min":0.89800000000000002,"5min":0.90900000000000003,"15min":0.95199999999999996},"min":{"1min":0.64500000000000002,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.1519999999999999,"5min":1.363,"15min":1.363},"last":0.85699999999999998},{"interface":"front","average":{"1min":1.028,"5min":0.91900000000000004,"15min":0.94299999999999995},"min":{"1min":0.871,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":1.228,"5min":1.3129999999999999,"15min":1.3129999999999999},"last":1.004}]},{"osd":3,"last update":"Sat Sep 23 19:06:30 2023","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":0.97699999999999998,"15min":1.0269999999999999},"min":{"1min":0.55400000000000005,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":1.2170000000000001,"5min":1.5669999999999999,"15min":1.5669999999999999},"last":0.76400000000000001},{"interface":"front","average":{"1min":0.97399999999999998,"5min":0.97799999999999998,"15min":1.0089999999999999},"min":{"1min":0.72699999999999998,"5min":0.56299999999999994,"15min":0.56299999999999994},"max":{"1min":1.2070000000000001,"5min":1.5940000000000001,"15min":1.5940000000000001},"last":0.98299999999999998}]},{"osd":4,"last update":"Sat Sep 23 19:06:30 2023","interfaces":[{"interface":"back","average":{"1min":0.97299999999999998,"5min":0.89500000000000002,"15min":0.91900000000000004},"min":{"1min":0.72699999999999998,"5min":0.443,"15min":0.443},"max":{"1min":1.171,"5min":1.302,"15min":1.302},"last":0.81000000000000005},{"interface":"front","average":{"1min":0.99099999999999999,"5min":0.94599999999999995,"15min":0.94199999999999995},"min":{"1min":0.72699999999999998,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":1.2989999999999999,"5min":1.742,"15min":1.742},"last":0.94999999999999996}]},{"osd":5,"last update":"Sat Sep 23 19:06:30 2023","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":0.95199999999999996,"15min":0.98499999999999999},"min":{"1min":0.73799999999999999,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":1.268,"5min":1.3720000000000001,"15min":1.3720000000000001},"last":1.1579999999999999},{"interface":"front","average":{"1min":1.1699999999999999,"5min":1.071,"15min":1.097},"min":{"1min":0.97799999999999998,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":1.3799999999999999,"5min":1.7,"15min":1.7},"last":1.1830000000000001}]},{"osd":7,"last update":"Sat Sep 23 19:07:06 2023","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":0.98399999999999999,"15min":0.93999999999999995},"min":{"1min":0.84799999999999998,"5min":0.52600000000000002,"15min":0.52600000000000002},"max":{"1min":1.494,"5min":1.494,"15min":1.494},"last":1.2729999999999999},{"interface":"front","average":{"1min":1.2130000000000001,"5min":1.109,"15min":1.0609999999999999},"min":{"1min":0.81799999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"max":{"1min":1.823,"5min":1.823,"15min":1.823},"last":1.224}]}]},{"osd":1,"up_from":13,"seq":55834574955,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297696,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447456,"statfs":{"total":95995035648,"available":95690194944,"internally_reserved":0,"allocated":614400,"data_stored":263178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:07:06 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.68700000000000006,"15min":0.629},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.90200000000000002,"5min":1.2130000000000001,"15min":1.2130000000000001},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.70499999999999996,"5min":0.76600000000000001,"15min":0.65200000000000002},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":1.087,"5min":1.2809999999999999,"15min":1.2809999999999999},"last":1.097}]},{"osd":2,"last update":"Sat Sep 23 19:06:44 2023","interfaces":[{"interface":"back","average":{"1min":0.79900000000000004,"5min":0.80400000000000005,"15min":0.72399999999999998},"min":{"1min":0.37,"5min":0.36899999999999999,"15min":0.34699999999999998},"max":{"1min":1.0589999999999999,"5min":1.1319999999999999,"15min":1.1319999999999999},"last":1.0149999999999999},{"interface":"front","average":{"1min":0.76000000000000001,"5min":0.76100000000000001,"15min":0.70599999999999996},"min":{"1min":0.46700000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":1.125,"5min":1.159,"15min":1.159},"last":0.92500000000000004}]},{"osd":3,"last update":"Sat Sep 23 19:06:34 2023","interfaces":[{"interface":"back","average":{"1min":0.83599999999999997,"5min":0.83399999999999996,"15min":0.83699999999999997},"min":{"1min":0.50600000000000001,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.1699999999999999,"5min":1.21,"15min":1.21},"last":0.89600000000000002},{"interface":"front","average":{"1min":0.85499999999999998,"5min":0.85599999999999998,"15min":0.82999999999999996},"min":{"1min":0.46700000000000003,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":1.0760000000000001,"5min":1.1499999999999999,"15min":1.1499999999999999},"last":1.1200000000000001}]},{"osd":4,"last update":"Sat Sep 23 19:07:06 2023","interfaces":[{"interface":"back","average":{"1min":0.84399999999999997,"5min":0.89300000000000002,"15min":0.90700000000000003},"min":{"1min":0.64800000000000002,"5min":0.54500000000000004,"15min":0.54500000000000004},"max":{"1min":1.1519999999999999,"5min":1.2849999999999999,"15min":1.2849999999999999},"last":0.93999999999999995},{"interface":"front","average":{"1min":0.92500000000000004,"5min":0.98199999999999998,"15min":1.03},"min":{"1min":0.622,"5min":0.61199999999999999,"15min":0.61199999999999999},"max":{"1min":1.373,"5min":1.373,"15min":1.524},"last":1.0449999999999999}]},{"osd":5,"last update":"Sat Sep 23 19:06:49 2023","interfaces":[{"interface":"back","average":{"1min":0.93899999999999995,"5min":0.97499999999999998,"15min":1.0089999999999999},"min":{"1min":0.59699999999999998,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":1.276,"5min":1.411,"15min":1.411},"last":0.98099999999999998},{"interface":"front","average":{"1min":1.0580000000000001,"5min":1.0880000000000001,"15min":1.087},"min":{"1min":0.80600000000000005,"5min":0.66300000000000003,"15min":0.66300000000000003},"max":{"1min":1.3999999999999999,"5min":1.425,"15min":1.425},"last":1.2050000000000001}]},{"osd":6,"last update":"Sat Sep 23 19:06:31 2023","interfaces":[{"interface":"back","average":{"1min":1.0129999999999999,"5min":1.022,"15min":1.04},"min":{"1min":0.66100000000000003,"5min":0.55200000000000005,"15min":0.55200000000000005},"max":{"1min":1.23,"5min":1.46,"15min":1.46},"last":0.86199999999999999},{"interface":"front","average":{"1min":1.137,"5min":1.129,"15min":1.147},"min":{"1min":0.94499999999999995,"5min":0.621,"15min":0.621},"max":{"1min":1.4710000000000001,"5min":1.4710000000000001,"15min":1.4710000000000001},"last":1.256}]},{"osd":7,"last update":"Sat Sep 23 19:07:10 2023","interfaces":[{"interface":"back","average":{"1min":1.0309999999999999,"5min":1.091,"15min":1.127},"min":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"max":{"1min":1.4490000000000001,"5min":1.6259999999999999,"15min":1.6259999999999999},"last":1.1499999999999999},{"interface":"front","average":{"1min":1.022,"5min":1.0620000000000001,"15min":1.0840000000000001},"min":{"1min":0.75900000000000001,"5min":0.68100000000000005,"15min":0.68100000000000005},"max":{"1min":1.4490000000000001,"5min":1.4490000000000001,"15min":1.4490000000000001},"last":1.194}]}]},{"osd":0,"up_from":8,"seq":34359738482,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297704,"kb_used_data":608,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447448,"statfs":{"total":95995035648,"available":95690186752,"internally_reserved":0,"allocated":622592,"data_stored":263754,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 19:07:08 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":0.68100000000000005,"15min":0.63200000000000001},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.93799999999999994,"5min":1.0649999999999999,"15min":2.0979999999999999},"last":0.68799999999999994},{"interface":"front","average":{"1min":0.68100000000000005,"5min":0.72399999999999998,"15min":0.63500000000000001},"min":{"1min":0.35899999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":1.1020000000000001,"5min":1.1279999999999999,"15min":1.351},"last":0.60199999999999998}]},{"osd":2,"last update":"Sat Sep 23 19:06:50 2023","interfaces":[{"interface":"back","average":{"1min":0.79100000000000004,"5min":0.79500000000000004,"15min":0.747},"min":{"1min":0.38100000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":1.4339999999999999,"5min":1.4339999999999999,"15min":1.546},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.74299999999999999,"5min":0.77800000000000002,"15min":0.72899999999999998},"min":{"1min":0.48499999999999999,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":1.2030000000000001,"5min":1.2030000000000001,"15min":1.2030000000000001},"last":0.78400000000000003}]},{"osd":3,"last update":"Sat Sep 23 19:07:24 2023","interfaces":[{"interface":"back","average":{"1min":0.84699999999999998,"5min":0.85299999999999998,"15min":0.90700000000000003},"min":{"1min":0.42199999999999999,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":1.3149999999999999,"5min":1.3149999999999999,"15min":3.1840000000000002},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.874,"5min":0.86499999999999999,"15min":0.82399999999999995},"min":{"1min":0.443,"5min":0.377,"15min":0.377},"max":{"1min":1.2569999999999999,"5min":1.5600000000000001,"15min":1.5600000000000001},"last":0.93500000000000005}]},{"osd":4,"last update":"Sat Sep 23 19:07:08 2023","interfaces":[{"interface":"back","average":{"1min":0.94599999999999995,"5min":0.99399999999999999,"15min":0.93799999999999994},"min":{"1min":0.68999999999999995,"5min":0.53000000000000003,"15min":0.53000000000000003},"max":{"1min":1.323,"5min":1.373,"15min":1.373},"last":0.86599999999999999},{"interface":"front","average":{"1min":1.018,"5min":1.008,"15min":0.96999999999999997},"min":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.649,"5min":1.649,"15min":1.649},"last":1.02}]},{"osd":5,"last update":"Sat Sep 23 19:06:50 2023","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":1.0549999999999999,"15min":1.0820000000000001},"min":{"1min":0.70899999999999996,"5min":0.61099999999999999,"15min":0.61099999999999999},"max":{"1min":1.492,"5min":1.492,"15min":1.492},"last":1.0760000000000001},{"interface":"front","average":{"1min":0.98099999999999998,"5min":0.98699999999999999,"15min":0.99299999999999999},"min":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"max":{"1min":1.2290000000000001,"5min":1.3859999999999999,"15min":1.3859999999999999},"last":0.72699999999999998}]},{"osd":6,"last update":"Sat Sep 23 19:06:28 2023","interfaces":[{"interface":"back","average":{"1min":0.999,"5min":1.0249999999999999,"15min":1.073},"min":{"1min":0.69399999999999995,"5min":0.60799999999999998,"15min":0.60799999999999998},"max":{"1min":1.272,"5min":1.502,"15min":1.502},"last":0.89700000000000002},{"interface":"front","average":{"1min":1.159,"5min":1.1419999999999999,"15min":1.1759999999999999},"min":{"1min":0.90100000000000002,"5min":0.73099999999999998,"15min":0.73099999999999998},"max":{"1min":1.5289999999999999,"5min":1.5289999999999999,"15min":1.5289999999999999},"last":0.96699999999999997}]},{"osd":7,"last update":"Sat Sep 23 19:07:08 2023","interfaces":[{"interface":"back","average":{"1min":1.0820000000000001,"5min":1.1060000000000001,"15min":1.125},"min":{"1min":0.67000000000000004,"5min":0.626,"15min":0.626},"max":{"1min":1.5840000000000001,"5min":1.655,"15min":1.655},"last":1.1220000000000001},{"interface":"front","average":{"1min":1.0069999999999999,"5min":1.05,"15min":1.079},"min":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":1.377,"5min":1.595,"15min":1.595},"last":0.65600000000000003}]}]},{"osd":2,"up_from":18,"seq":77309411424,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297704,"kb_used_data":608,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447448,"statfs":{"total":95995035648,"available":95690186752,"internally_reserved":0,"allocated":622592,"data_stored":263766,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:06:41 2023","interfaces":[{"interface":"back","average":{"1min":0.71099999999999997,"5min":0.74199999999999999,"15min":0.67700000000000005},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.97599999999999998,"5min":1.8979999999999999,"15min":1.8979999999999999},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.72099999999999997,"5min":0.76600000000000001,"15min":0.70399999999999996},"min":{"1min":0.47099999999999997,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":1.089,"5min":1.651,"15min":1.651},"last":0.92300000000000004}]},{"osd":1,"last update":"Sat Sep 23 19:06:41 2023","interfaces":[{"interface":"back","average":{"1min":0.69999999999999996,"5min":0.74199999999999999,"15min":0.72199999999999998},"min":{"1min":0.434,"5min":0.317,"15min":0.317},"max":{"1min":1.0469999999999999,"5min":1.4490000000000001,"15min":1.4490000000000001},"last":0.88900000000000001},{"interface":"front","average":{"1min":0.69599999999999995,"5min":0.81499999999999995,"15min":0.74399999999999999},"min":{"1min":0.378,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.032,"5min":1.788,"15min":1.788},"last":0.58799999999999997}]},{"osd":3,"last update":"Sat Sep 23 19:06:27 2023","interfaces":[{"interface":"back","average":{"1min":0.79900000000000004,"5min":0.80600000000000005,"15min":0.77300000000000002},"min":{"1min":0.52800000000000002,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":1.1659999999999999,"5min":1.748,"15min":1.748},"last":0.82699999999999996},{"interface":"front","average":{"1min":0.82699999999999996,"5min":0.875,"15min":0.81299999999999994},"min":{"1min":0.61599999999999999,"5min":0.443,"15min":0.443},"max":{"1min":1.1599999999999999,"5min":1.4690000000000001,"15min":1.4690000000000001},"last":0.96399999999999997}]},{"osd":4,"last update":"Sat Sep 23 19:07:05 2023","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":0.98299999999999998,"15min":0.98499999999999999},"min":{"1min":0.71099999999999997,"5min":0.60799999999999998,"15min":0.60799999999999998},"max":{"1min":1.3129999999999999,"5min":1.335,"15min":1.335},"last":1.1699999999999999},{"interface":"front","average":{"1min":1.018,"5min":0.98699999999999999,"15min":0.98199999999999998},"min":{"1min":0.623,"5min":0.623,"15min":0.623},"max":{"1min":1.3580000000000001,"5min":1.554,"15min":1.554},"last":0.76700000000000002}]},{"osd":5,"last update":"Sat Sep 23 19:06:51 2023","interfaces":[{"interface":"back","average":{"1min":1.0760000000000001,"5min":1.0549999999999999,"15min":1.0489999999999999},"min":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"max":{"1min":1.476,"5min":1.635,"15min":1.635},"last":1.052},{"interface":"front","average":{"1min":1.0800000000000001,"5min":1.0629999999999999,"15min":1.087},"min":{"1min":0.79800000000000004,"5min":0.65500000000000003,"15min":0.65500000000000003},"max":{"1min":1.4239999999999999,"5min":1.581,"15min":1.581},"last":1.2470000000000001}]},{"osd":6,"last update":"Sat Sep 23 19:06:27 2023","interfaces":[{"interface":"back","average":{"1min":1.103,"5min":1.083,"15min":1.119},"min":{"1min":0.873,"5min":0.68000000000000005,"15min":0.68000000000000005},"max":{"1min":1.3580000000000001,"5min":1.5,"15min":1.5},"last":1.1399999999999999},{"interface":"front","average":{"1min":1.1200000000000001,"5min":1.119,"15min":1.143},"min":{"1min":0.90500000000000003,"5min":0.76300000000000001,"15min":0.76300000000000001},"max":{"1min":1.5960000000000001,"5min":1.679,"15min":1.679},"last":1.1100000000000001}]},{"osd":7,"last update":"Sat Sep 23 19:07:11 2023","interfaces":[{"interface":"back","average":{"1min":1.0700000000000001,"5min":1.1259999999999999,"15min":1.1579999999999999},"min":{"1min":0.81599999999999995,"5min":0.73499999999999999,"15min":0.73499999999999999},"max":{"1min":1.518,"5min":1.659,"15min":1.659},"last":1.2689999999999999},{"interface":"front","average":{"1min":1.091,"5min":1.131,"15min":1.1870000000000001},"min":{"1min":0.73799999999999999,"5min":0.621,"15min":0.621},"max":{"1min":1.492,"5min":1.6890000000000001,"15min":1.6890000000000001},"last":1.196}]}]},{"osd":3,"up_from":25,"seq":107374182490,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297700,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447452,"statfs":{"total":95995035648,"available":95690190848,"internally_reserved":0,"allocated":618496,"data_stored":263268,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:06:25 2023","interfaces":[{"interface":"back","average":{"1min":0.69099999999999995,"5min":0.69699999999999995,"15min":0.66700000000000004},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":1.0389999999999999,"5min":1.286,"15min":1.286},"last":0.70299999999999996},{"interface":"front","average":{"1min":0.77100000000000002,"5min":0.753,"15min":0.75900000000000001},"min":{"1min":0.52400000000000002,"5min":0.436,"15min":0.436},"max":{"1min":1.0269999999999999,"5min":1.1120000000000001,"15min":1.1120000000000001},"last":0.76500000000000001}]},{"osd":1,"last update":"Sat Sep 23 19:06:25 2023","interfaces":[{"interface":"back","average":{"1min":0.751,"5min":0.78200000000000003,"15min":0.78400000000000003},"min":{"1min":0.52000000000000002,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":1.2270000000000001,"5min":1.2270000000000001,"15min":1.2270000000000001},"last":0.90800000000000003},{"interface":"front","average":{"1min":0.72099999999999997,"5min":0.74099999999999999,"15min":0.754},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":1.1579999999999999,"5min":1.3180000000000001,"15min":1.3180000000000001},"last":0.81799999999999995}]},{"osd":2,"last update":"Sat Sep 23 19:06:25 2023","interfaces":[{"interface":"back","average":{"1min":0.753,"5min":0.83399999999999996,"15min":0.85899999999999999},"min":{"1min":0.438,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":1.0840000000000001,"5min":1.4339999999999999,"15min":1.4339999999999999},"last":0.88100000000000001},{"interface":"front","average":{"1min":0.78400000000000003,"5min":0.83799999999999997,"15min":0.82299999999999995},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":1.0920000000000001,"5min":1.3700000000000001,"15min":1.3700000000000001},"last":1.1040000000000001}]},{"osd":4,"last update":"Sat Sep 23 19:07:03 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":1.002,"15min":0.99299999999999999},"min":{"1min":0.69499999999999995,"5min":0.64300000000000002,"15min":0.64300000000000002},"max":{"1min":1.5700000000000001,"5min":1.7250000000000001,"15min":1.7250000000000001},"last":1.1359999999999999},{"interface":"front","average":{"1min":1.0169999999999999,"5min":0.99299999999999999,"15min":0.995},"min":{"1min":0.70699999999999996,"5min":0.63,"15min":0.63},"max":{"1min":1.536,"5min":1.536,"15min":1.536},"last":1.0449999999999999}]},{"osd":5,"last update":"Sat Sep 23 19:06:53 2023","interfaces":[{"interface":"back","average":{"1min":1.095,"5min":1.0640000000000001,"15min":1.0680000000000001},"min":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"max":{"1min":1.5820000000000001,"5min":1.9319999999999999,"15min":1.9319999999999999},"last":0.95499999999999996},{"interface":"front","average":{"1min":1.056,"5min":1.054,"15min":1.034},"min":{"1min":0.59799999999999998,"5min":0.58599999999999997,"15min":0.58599999999999997},"max":{"1min":1.603,"5min":1.603,"15min":1.603},"last":1.1659999999999999}]},{"osd":6,"last update":"Sat Sep 23 19:06:25 2023","interfaces":[{"interface":"back","average":{"1min":1.121,"5min":1.093,"15min":1.121},"min":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"max":{"1min":1.526,"5min":1.526,"15min":1.526},"last":1.0249999999999999},{"interface":"front","average":{"1min":1.125,"5min":1.179,"15min":1.246},"min":{"1min":0.628,"5min":0.628,"15min":0.628},"max":{"1min":1.4359999999999999,"5min":2.1960000000000002,"15min":2.1960000000000002},"last":1.238}]},{"osd":7,"last update":"Sat Sep 23 19:07:13 2023","interfaces":[{"interface":"back","average":{"1min":1.1519999999999999,"5min":1.131,"15min":1.1140000000000001},"min":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"max":{"1min":1.6679999999999999,"5min":1.6679999999999999,"15min":1.6679999999999999},"last":1.194},{"interface":"front","average":{"1min":1.1100000000000001,"5min":1.131,"15min":1.145},"min":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"max":{"1min":1.52,"5min":1.5600000000000001,"15min":1.5600000000000001},"last":1.3100000000000001}]}]},{"osd":4,"up_from":30,"seq":128849018961,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297720,"kb_used_data":624,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447432,"statfs":{"total":95995035648,"available":95690170368,"internally_reserved":0,"allocated":638976,"data_stored":264860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:07:10 2023","interfaces":[{"interface":"back","average":{"1min":1.04,"5min":0.97899999999999998,"15min":0.96799999999999997},"min":{"1min":0.60099999999999998,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":1.2709999999999999,"5min":1.343,"15min":1.343},"last":0.68600000000000005},{"interface":"front","average":{"1min":0.98299999999999998,"5min":0.96599999999999997,"15min":0.97299999999999998},"min":{"1min":0.64900000000000002,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.304,"5min":1.304,"15min":1.304},"last":0.80100000000000005}]},{"osd":1,"last update":"Sat Sep 23 19:07:10 2023","interfaces":[{"interface":"back","average":{"1min":0.84699999999999998,"5min":0.85099999999999998,"15min":0.85899999999999999},"min":{"1min":0.495,"5min":0.495,"15min":0.495},"max":{"1min":1.0900000000000001,"5min":1.23,"15min":1.23},"last":0.76000000000000001},{"interface":"front","average":{"1min":0.91500000000000004,"5min":0.92400000000000004,"15min":0.89800000000000002},"min":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"max":{"1min":1.2190000000000001,"5min":1.4099999999999999,"15min":1.4099999999999999},"last":0.93999999999999995}]},{"osd":2,"last update":"Sat Sep 23 19:07:10 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":0.91300000000000003,"15min":0.871},"min":{"1min":0.67800000000000005,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.3380000000000001,"5min":1.343,"15min":1.343},"last":0.85499999999999998},{"interface":"front","average":{"1min":0.89600000000000002,"5min":0.89700000000000002,"15min":0.877},"min":{"1min":0.629,"5min":0.59499999999999997,"15min":0.59499999999999997},"max":{"1min":1.1890000000000001,"5min":1.3089999999999999,"15min":1.3089999999999999},"last":0.70699999999999996}]},{"osd":3,"last update":"Sat Sep 23 19:07:10 2023","interfaces":[{"interface":"back","average":{"1min":0.93500000000000005,"5min":0.90600000000000003,"15min":0.91000000000000003},"min":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"max":{"1min":1.268,"5min":1.268,"15min":1.268},"last":0.88900000000000001},{"interface":"front","average":{"1min":0.92600000000000005,"5min":0.91300000000000003,"15min":0.89100000000000001},"min":{"1min":0.61499999999999999,"5min":0.60199999999999998,"15min":0.60199999999999998},"max":{"1min":1.2709999999999999,"5min":1.2709999999999999,"15min":1.335},"last":1.143}]},{"osd":5,"last update":"Sat Sep 23 19:06:54 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":0.88600000000000001,"15min":0.78600000000000003},"min":{"1min":0.52200000000000002,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":1.3560000000000001,"5min":1.373,"15min":1.373},"last":1.0169999999999999},{"interface":"front","average":{"1min":1.038,"5min":0.94999999999999996,"15min":0.90000000000000002},"min":{"1min":0.65400000000000003,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":1.327,"5min":1.407,"15min":1.407},"last":1.111}]},{"osd":6,"last update":"Sat Sep 23 19:07:25 2023","interfaces":[{"interface":"back","average":{"1min":1.105,"5min":0.997,"15min":0.97099999999999997},"min":{"1min":0.78000000000000003,"5min":0.58099999999999996,"15min":0.58099999999999996},"max":{"1min":1.284,"5min":1.4430000000000001,"15min":1.4430000000000001},"last":0.78000000000000003},{"interface":"front","average":{"1min":1.1100000000000001,"5min":1.0169999999999999,"15min":0.97099999999999997},"min":{"1min":0.66900000000000004,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":1.625,"5min":1.625,"15min":1.625},"last":0.97599999999999998}]},{"osd":7,"last update":"Sat Sep 23 19:07:10 2023","interfaces":[{"interface":"back","average":{"1min":1.0429999999999999,"5min":0.96299999999999997,"15min":0.95499999999999996},"min":{"1min":0.70799999999999996,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":1.595,"5min":1.595,"15min":1.595},"last":1.052},{"interface":"front","average":{"1min":1.1779999999999999,"5min":1.0600000000000001,"15min":1.03},"min":{"1min":0.87,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":1.403,"5min":1.51,"15min":1.51},"last":1.0840000000000001}]}]},{"osd":5,"up_from":36,"seq":154618822728,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297696,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447456,"statfs":{"total":95995035648,"available":95690194944,"internally_reserved":0,"allocated":614400,"data_stored":263172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:06:46 2023","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":0.89000000000000001,"15min":0.91100000000000003},"min":{"1min":0.65500000000000003,"5min":0.48999999999999999,"15min":0.48999999999999999},"max":{"1min":1.161,"5min":1.2110000000000001,"15min":1.2110000000000001},"last":1.2669999999999999},{"interface":"front","average":{"1min":0.91400000000000003,"5min":0.88,"15min":0.876},"min":{"1min":0.64900000000000002,"5min":0.54300000000000004,"15min":0.54300000000000004},"max":{"1min":1.282,"5min":1.4710000000000001,"15min":1.4710000000000001},"last":0.98399999999999999}]},{"osd":1,"last update":"Sat Sep 23 19:06:46 2023","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":0.92100000000000004,"15min":0.90900000000000003},"min":{"1min":0.59599999999999997,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.3220000000000001,"5min":1.413,"15min":1.413},"last":1.1559999999999999},{"interface":"front","average":{"1min":0.89700000000000002,"5min":0.874,"15min":0.873},"min":{"1min":0.57799999999999996,"5min":0.5,"15min":0.5},"max":{"1min":1.222,"5min":1.3160000000000001,"15min":1.3160000000000001},"last":0.79100000000000004}]},{"osd":2,"last update":"Sat Sep 23 19:06:46 2023","interfaces":[{"interface":"back","average":{"1min":0.90100000000000002,"5min":0.88800000000000001,"15min":0.88600000000000001},"min":{"1min":0.54900000000000004,"5min":0.52200000000000002,"15min":0.52200000000000002},"max":{"1min":1.2809999999999999,"5min":1.4399999999999999,"15min":1.4399999999999999},"last":0.871},{"interface":"front","average":{"1min":0.93000000000000005,"5min":0.95199999999999996,"15min":0.96499999999999997},"min":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"max":{"1min":1.319,"5min":1.3779999999999999,"15min":1.3779999999999999},"last":0.84599999999999997}]},{"osd":3,"last update":"Sat Sep 23 19:06:46 2023","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":0.92300000000000004,"15min":0.89900000000000002},"min":{"1min":0.68000000000000005,"5min":0.61199999999999999,"15min":0.61199999999999999},"max":{"1min":1.21,"5min":1.3,"15min":1.3},"last":1.1240000000000001},{"interface":"front","average":{"1min":0.995,"5min":1.0009999999999999,"15min":1.0049999999999999},"min":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"max":{"1min":1.2609999999999999,"5min":1.3360000000000001,"15min":1.3360000000000001},"last":1.056}]},{"osd":4,"last update":"Sat Sep 23 19:06:46 2023","interfaces":[{"interface":"back","average":{"1min":0.88800000000000001,"5min":0.85199999999999998,"15min":0.82099999999999995},"min":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":1.2030000000000001,"5min":1.655,"15min":1.655},"last":1.0860000000000001},{"interface":"front","average":{"1min":1,"5min":0.92500000000000004,"15min":0.877},"min":{"1min":0.68200000000000005,"5min":0.57299999999999995,"15min":0.57299999999999995},"max":{"1min":1.3400000000000001,"5min":2.415,"15min":2.415},"last":1.026}]},{"osd":6,"last update":"Sat Sep 23 19:06:28 2023","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":0.97899999999999998,"15min":0.94499999999999995},"min":{"1min":0.58699999999999997,"5min":0.48999999999999999,"15min":0.48999999999999999},"max":{"1min":1.151,"5min":2.0699999999999998,"15min":2.0699999999999998},"last":1.2190000000000001},{"interface":"front","average":{"1min":1.0489999999999999,"5min":1.0620000000000001,"15min":1.0309999999999999},"min":{"1min":0.71499999999999997,"5min":0.54800000000000004,"15min":0.54800000000000004},"max":{"1min":1.3600000000000001,"5min":2.4769999999999999,"15min":2.4769999999999999},"last":1.2050000000000001}]},{"osd":7,"last update":"Sat Sep 23 19:07:12 2023","interfaces":[{"interface":"back","average":{"1min":1.0960000000000001,"5min":1.0589999999999999,"15min":1.034},"min":{"1min":0.82699999999999996,"5min":0.63300000000000001,"15min":0.63300000000000001},"max":{"1min":1.401,"5min":2.5049999999999999,"15min":2.5049999999999999},"last":1.2509999999999999},{"interface":"front","average":{"1min":1.155,"5min":1.111,"15min":1.0609999999999999},"min":{"1min":0.84799999999999998,"5min":0.54500000000000004,"15min":0.54500000000000004},"max":{"1min":1.3879999999999999,"5min":2.4340000000000002,"15min":2.4340000000000002},"last":1.3129999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T19:07:28.789 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-09-23T19:07:28.789 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-09-23T19:07:28.789 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-09-23T19:07:28.789 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph health --format=json 2023-09-23T19:07:29.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:29 smithi016 bash[33624]: audit 2023-09-23T19:07:28.034761+0000 mgr.y (mgr.14454) 151 : audit [DBG] from='client.24698 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:07:29.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:29 smithi016 bash[27237]: audit 2023-09-23T19:07:28.034761+0000 mgr.y (mgr.14454) 151 : audit [DBG] from='client.24698 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:07:29.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:29 smithi092 bash[27556]: audit 2023-09-23T19:07:28.034761+0000 mgr.y (mgr.14454) 151 : audit [DBG] from='client.24698 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:07:29.537 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:07:29 smithi092 bash[28593]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:07:29] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T19:07:30.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:30 smithi016 bash[27237]: cluster 2023-09-23T19:07:28.943971+0000 mgr.y (mgr.14454) 152 : cluster [DBG] pgmap v130: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:30.535 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:07:30 smithi016 bash[27503]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:07:30] "GET /metrics HTTP/1.1" 200 191363 "" "Prometheus/2.33.4" 2023-09-23T19:07:30.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:30 smithi016 bash[33624]: cluster 2023-09-23T19:07:28.943971+0000 mgr.y (mgr.14454) 152 : cluster [DBG] pgmap v130: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:30.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:30 smithi092 bash[27556]: cluster 2023-09-23T19:07:28.943971+0000 mgr.y (mgr.14454) 152 : cluster [DBG] pgmap v130: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:31.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:31 smithi016 bash[27237]: cluster 2023-09-23T19:07:30.945015+0000 mgr.y (mgr.14454) 153 : cluster [DBG] pgmap v131: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:31.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:31 smithi016 bash[33624]: cluster 2023-09-23T19:07:30.945015+0000 mgr.y (mgr.14454) 153 : cluster [DBG] pgmap v131: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:31.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:31 smithi092 bash[27556]: cluster 2023-09-23T19:07:30.945015+0000 mgr.y (mgr.14454) 153 : cluster [DBG] pgmap v131: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:33.608 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:07:34.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:34 smithi016 bash[27237]: cluster 2023-09-23T19:07:32.945620+0000 mgr.y (mgr.14454) 154 : cluster [DBG] pgmap v132: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:34.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:34 smithi016 bash[33624]: cluster 2023-09-23T19:07:32.945620+0000 mgr.y (mgr.14454) 154 : cluster [DBG] pgmap v132: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:34.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:34 smithi092 bash[27556]: cluster 2023-09-23T19:07:32.945620+0000 mgr.y (mgr.14454) 154 : cluster [DBG] pgmap v132: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:36.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:36 smithi016 bash[27237]: cluster 2023-09-23T19:07:34.946258+0000 mgr.y (mgr.14454) 155 : cluster [DBG] pgmap v133: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:36.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:36 smithi016 bash[33624]: cluster 2023-09-23T19:07:34.946258+0000 mgr.y (mgr.14454) 155 : cluster [DBG] pgmap v133: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:36.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:36 smithi092 bash[27556]: cluster 2023-09-23T19:07:34.946258+0000 mgr.y (mgr.14454) 155 : cluster [DBG] pgmap v133: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:36.451 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-09-23T19:07:37.184 INFO:teuthology.orchestra.run.smithi016.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-09-23T19:07:37.184 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-09-23T19:07:37.185 INFO:teuthology.run_tasks:Running task print... 2023-09-23T19:07:37.193 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2023-09-23T19:07:37.193 INFO:teuthology.run_tasks:Running task parallel... 2023-09-23T19:07:37.204 INFO:teuthology.task.parallel:starting parallel... 2023-09-23T19:07:37.204 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-09-23T19:07:37.205 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2023-09-23T19:07:37.205 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T19:07:37.206 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T19:07:37.207 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2023-09-23T19:07:37.207 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_0 16 2023-09-23T19:07:37.207 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-09-23T19:07:37.207 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2023-09-23T19:07:37.208 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi016.front.sepia.ceph.com 2023-09-23T19:07:37.208 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2023-09-23T19:07:37.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:37 smithi016 bash[27237]: audit 2023-09-23T19:07:36.448029+0000 mon.a (mon.0) 671 : audit [DBG] from='client.? 172.21.15.16:0/2571439677' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T19:07:37.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:37 smithi016 bash[33624]: audit 2023-09-23T19:07:36.448029+0000 mon.a (mon.0) 671 : audit [DBG] from='client.? 172.21.15.16:0/2571439677' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T19:07:37.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:37 smithi092 bash[27556]: audit 2023-09-23T19:07:36.448029+0000 mon.a (mon.0) 671 : audit [DBG] from='client.? 172.21.15.16:0/2571439677' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T19:07:38.279 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:38 smithi016 bash[33624]: cluster 2023-09-23T19:07:36.946978+0000 mgr.y (mgr.14454) 156 : cluster [DBG] pgmap v134: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:38.279 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:38 smithi016 bash[27237]: cluster 2023-09-23T19:07:36.946978+0000 mgr.y (mgr.14454) 156 : cluster [DBG] pgmap v134: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:38.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:38 smithi092 bash[27556]: cluster 2023-09-23T19:07:36.946978+0000 mgr.y (mgr.14454) 156 : cluster [DBG] pgmap v134: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:39.537 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:07:39 smithi092 bash[28593]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:07:39] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T19:07:40.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:40 smithi016 bash[33624]: cluster 2023-09-23T19:07:38.947600+0000 mgr.y (mgr.14454) 157 : cluster [DBG] pgmap v135: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:40.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:40 smithi016 bash[27237]: cluster 2023-09-23T19:07:38.947600+0000 mgr.y (mgr.14454) 157 : cluster [DBG] pgmap v135: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:40.286 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:07:40 smithi016 bash[27503]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:07:40] "GET /metrics HTTP/1.1" 200 191382 "" "Prometheus/2.33.4" 2023-09-23T19:07:40.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:40 smithi092 bash[27556]: cluster 2023-09-23T19:07:38.947600+0000 mgr.y (mgr.14454) 157 : cluster [DBG] pgmap v135: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:40.786 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:07:40 smithi016 bash[27503]: [23/Sep/2023:19:07:40] ENGINE Bus STOPPING 2023-09-23T19:07:40.786 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:07:40 smithi016 bash[27503]: [23/Sep/2023:19:07:40] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T19:07:40.786 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:07:40 smithi016 bash[27503]: [23/Sep/2023:19:07:40] ENGINE Bus STOPPED 2023-09-23T19:07:40.786 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:07:40 smithi016 bash[27503]: [23/Sep/2023:19:07:40] ENGINE Bus STARTING 2023-09-23T19:07:40.786 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:07:40 smithi016 bash[27503]: [23/Sep/2023:19:07:40] ENGINE Serving on http://:::9283 2023-09-23T19:07:40.786 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:07:40 smithi016 bash[27503]: [23/Sep/2023:19:07:40] ENGINE Bus STARTED 2023-09-23T19:07:41.061 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph config set global log_to_journald false --force' 2023-09-23T19:07:41.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:41 smithi016 bash[27237]: audit 2023-09-23T19:07:40.350752+0000 mon.a (mon.0) 672 : audit [INF] from='client.? ' entity='client.admin' 2023-09-23T19:07:41.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:41 smithi016 bash[27237]: audit 2023-09-23T19:07:40.367418+0000 mon.c (mon.2) 95 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:07:41.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:41 smithi016 bash[27237]: cluster 2023-09-23T19:07:40.948785+0000 mgr.y (mgr.14454) 158 : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:41.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:41 smithi016 bash[33624]: audit 2023-09-23T19:07:40.350752+0000 mon.a (mon.0) 672 : audit [INF] from='client.? ' entity='client.admin' 2023-09-23T19:07:41.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:41 smithi016 bash[33624]: audit 2023-09-23T19:07:40.367418+0000 mon.c (mon.2) 95 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:07:41.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:41 smithi016 bash[33624]: cluster 2023-09-23T19:07:40.948785+0000 mgr.y (mgr.14454) 158 : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:41.786 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:41 smithi092 bash[27556]: audit 2023-09-23T19:07:40.350752+0000 mon.a (mon.0) 672 : audit [INF] from='client.? ' entity='client.admin' 2023-09-23T19:07:41.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:41 smithi092 bash[27556]: audit 2023-09-23T19:07:40.367418+0000 mon.c (mon.2) 95 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:07:41.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:41 smithi092 bash[27556]: cluster 2023-09-23T19:07:40.948785+0000 mgr.y (mgr.14454) 158 : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:42.561 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:07:44.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:44 smithi016 bash[27237]: cluster 2023-09-23T19:07:42.949415+0000 mgr.y (mgr.14454) 159 : cluster [DBG] pgmap v137: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:44.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:44 smithi016 bash[33624]: cluster 2023-09-23T19:07:42.949415+0000 mgr.y (mgr.14454) 159 : cluster [DBG] pgmap v137: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:44.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:44 smithi092 bash[27556]: cluster 2023-09-23T19:07:42.949415+0000 mgr.y (mgr.14454) 159 : cluster [DBG] pgmap v137: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:44.910 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:07:44 smithi016 bash[27503]: [23/Sep/2023:19:07:44] ENGINE Bus STOPPING 2023-09-23T19:07:44.911 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:07:44 smithi016 bash[27503]: [23/Sep/2023:19:07:44] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T19:07:44.911 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:07:44 smithi016 bash[27503]: [23/Sep/2023:19:07:44] ENGINE Bus STOPPED 2023-09-23T19:07:44.911 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:07:44 smithi016 bash[27503]: [23/Sep/2023:19:07:44] ENGINE Bus STARTING 2023-09-23T19:07:44.911 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:07:44 smithi016 bash[27503]: [23/Sep/2023:19:07:44] ENGINE Serving on http://:::9283 2023-09-23T19:07:44.911 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:07:44 smithi016 bash[27503]: [23/Sep/2023:19:07:44] ENGINE Bus STARTED 2023-09-23T19:07:45.938 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2023-09-23T19:07:46.126 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:45 smithi016 bash[33624]: audit 2023-09-23T19:07:44.788126+0000 mon.a (mon.0) 673 : audit [INF] from='client.? ' entity='client.admin' 2023-09-23T19:07:46.126 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:45 smithi016 bash[33624]: cluster 2023-09-23T19:07:44.950012+0000 mgr.y (mgr.14454) 160 : cluster [DBG] pgmap v138: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:46.126 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:45 smithi016 bash[27237]: audit 2023-09-23T19:07:44.788126+0000 mon.a (mon.0) 673 : audit [INF] from='client.? ' entity='client.admin' 2023-09-23T19:07:46.127 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:45 smithi016 bash[27237]: cluster 2023-09-23T19:07:44.950012+0000 mgr.y (mgr.14454) 160 : cluster [DBG] pgmap v138: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:46.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:45 smithi092 bash[27556]: audit 2023-09-23T19:07:44.788126+0000 mon.a (mon.0) 673 : audit [INF] from='client.? ' entity='client.admin' 2023-09-23T19:07:46.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:45 smithi092 bash[27556]: cluster 2023-09-23T19:07:44.950012+0000 mgr.y (mgr.14454) 160 : cluster [DBG] pgmap v138: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:47.333 INFO:teuthology.orchestra.run.smithi016.stderr:pool 'unique_pool_0' created 2023-09-23T19:07:47.355 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:47 smithi016 bash[33624]: audit 2023-09-23T19:07:46.312429+0000 mon.a (mon.0) 674 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:07:47.355 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:47 smithi016 bash[33624]: audit 2023-09-23T19:07:46.819267+0000 mon.c (mon.2) 96 : audit [INF] from='client.? 172.21.15.16:0/1643414565' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-23T19:07:47.355 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:47 smithi016 bash[33624]: audit 2023-09-23T19:07:46.819726+0000 mon.a (mon.0) 675 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-23T19:07:47.356 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:47 smithi016 bash[33624]: cluster 2023-09-23T19:07:46.950719+0000 mgr.y (mgr.14454) 161 : cluster [DBG] pgmap v139: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:47.356 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:47 smithi016 bash[33624]: audit 2023-09-23T19:07:47.302675+0000 mon.a (mon.0) 676 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:07:47.356 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:47 smithi016 bash[33624]: audit 2023-09-23T19:07:47.304851+0000 mon.c (mon.2) 97 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:07:47.356 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:47 smithi016 bash[33624]: audit 2023-09-23T19:07:47.305946+0000 mon.c (mon.2) 98 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:07:47.356 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:47 smithi016 bash[27237]: audit 2023-09-23T19:07:46.312429+0000 mon.a (mon.0) 674 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:07:47.356 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:47 smithi016 bash[27237]: audit 2023-09-23T19:07:46.819267+0000 mon.c (mon.2) 96 : audit [INF] from='client.? 172.21.15.16:0/1643414565' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-23T19:07:47.356 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:47 smithi016 bash[27237]: audit 2023-09-23T19:07:46.819726+0000 mon.a (mon.0) 675 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-23T19:07:47.356 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:47 smithi016 bash[27237]: cluster 2023-09-23T19:07:46.950719+0000 mgr.y (mgr.14454) 161 : cluster [DBG] pgmap v139: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:47.357 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:47 smithi016 bash[27237]: audit 2023-09-23T19:07:47.302675+0000 mon.a (mon.0) 676 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:07:47.357 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:47 smithi016 bash[27237]: audit 2023-09-23T19:07:47.304851+0000 mon.c (mon.2) 97 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:07:47.357 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:47 smithi016 bash[27237]: audit 2023-09-23T19:07:47.305946+0000 mon.c (mon.2) 98 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:07:47.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:47 smithi092 bash[27556]: audit 2023-09-23T19:07:46.312429+0000 mon.a (mon.0) 674 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:07:47.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:47 smithi092 bash[27556]: audit 2023-09-23T19:07:46.819267+0000 mon.c (mon.2) 96 : audit [INF] from='client.? 172.21.15.16:0/1643414565' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-23T19:07:47.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:47 smithi092 bash[27556]: audit 2023-09-23T19:07:46.819726+0000 mon.a (mon.0) 675 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-23T19:07:47.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:47 smithi092 bash[27556]: cluster 2023-09-23T19:07:46.950719+0000 mgr.y (mgr.14454) 161 : cluster [DBG] pgmap v139: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:47.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:47 smithi092 bash[27556]: audit 2023-09-23T19:07:47.302675+0000 mon.a (mon.0) 676 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:07:47.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:47 smithi092 bash[27556]: audit 2023-09-23T19:07:47.304851+0000 mon.c (mon.2) 97 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:07:47.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:47 smithi092 bash[27556]: audit 2023-09-23T19:07:47.305946+0000 mon.c (mon.2) 98 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:07:48.384 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2023-09-23T19:07:48.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:48 smithi016 bash[27237]: audit 2023-09-23T19:07:47.311047+0000 mon.a (mon.0) 677 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:07:48.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:48 smithi016 bash[27237]: audit 2023-09-23T19:07:47.321877+0000 mon.a (mon.0) 678 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-09-23T19:07:48.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:48 smithi016 bash[27237]: cluster 2023-09-23T19:07:47.321990+0000 mon.a (mon.0) 679 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2023-09-23T19:07:48.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:48 smithi016 bash[27237]: audit 2023-09-23T19:07:47.322193+0000 mon.c (mon.2) 99 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:07:48.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:48 smithi016 bash[27237]: audit 2023-09-23T19:07:47.325605+0000 mon.c (mon.2) 100 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:07:48.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:48 smithi016 bash[27237]: audit 2023-09-23T19:07:47.327961+0000 mon.c (mon.2) 101 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:07:48.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:48 smithi016 bash[27237]: audit 2023-09-23T19:07:47.333673+0000 mon.a (mon.0) 680 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:07:48.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:48 smithi016 bash[33624]: audit 2023-09-23T19:07:47.311047+0000 mon.a (mon.0) 677 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:07:48.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:48 smithi016 bash[33624]: audit 2023-09-23T19:07:47.321877+0000 mon.a (mon.0) 678 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-09-23T19:07:48.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:48 smithi016 bash[33624]: cluster 2023-09-23T19:07:47.321990+0000 mon.a (mon.0) 679 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2023-09-23T19:07:48.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:48 smithi016 bash[33624]: audit 2023-09-23T19:07:47.322193+0000 mon.c (mon.2) 99 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:07:48.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:48 smithi016 bash[33624]: audit 2023-09-23T19:07:47.325605+0000 mon.c (mon.2) 100 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:07:48.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:48 smithi016 bash[33624]: audit 2023-09-23T19:07:47.327961+0000 mon.c (mon.2) 101 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:07:48.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:48 smithi016 bash[33624]: audit 2023-09-23T19:07:47.333673+0000 mon.a (mon.0) 680 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:07:48.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:48 smithi092 bash[27556]: audit 2023-09-23T19:07:47.311047+0000 mon.a (mon.0) 677 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:07:48.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:48 smithi092 bash[27556]: audit 2023-09-23T19:07:47.321877+0000 mon.a (mon.0) 678 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-09-23T19:07:48.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:48 smithi092 bash[27556]: cluster 2023-09-23T19:07:47.321990+0000 mon.a (mon.0) 679 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2023-09-23T19:07:48.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:48 smithi092 bash[27556]: audit 2023-09-23T19:07:47.322193+0000 mon.c (mon.2) 99 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:07:48.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:48 smithi092 bash[27556]: audit 2023-09-23T19:07:47.325605+0000 mon.c (mon.2) 100 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:07:48.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:48 smithi092 bash[27556]: audit 2023-09-23T19:07:47.327961+0000 mon.c (mon.2) 101 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:07:48.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:48 smithi092 bash[27556]: audit 2023-09-23T19:07:47.333673+0000 mon.a (mon.0) 680 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:07:49.339 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:07:49 smithi092 bash[28593]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:07:49] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T19:07:49.771 INFO:teuthology.orchestra.run.smithi016.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T19:07:49.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:49 smithi016 bash[27237]: cluster 2023-09-23T19:07:48.324063+0000 mon.a (mon.0) 681 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2023-09-23T19:07:49.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:49 smithi016 bash[27237]: cluster 2023-09-23T19:07:48.951357+0000 mgr.y (mgr.14454) 162 : cluster [DBG] pgmap v142: 89 pgs: 16 unknown, 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:49.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:49 smithi016 bash[27237]: cluster 2023-09-23T19:07:49.326942+0000 mon.a (mon.0) 682 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2023-09-23T19:07:49.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:49 smithi016 bash[33624]: cluster 2023-09-23T19:07:48.324063+0000 mon.a (mon.0) 681 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2023-09-23T19:07:49.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:49 smithi016 bash[33624]: cluster 2023-09-23T19:07:48.951357+0000 mgr.y (mgr.14454) 162 : cluster [DBG] pgmap v142: 89 pgs: 16 unknown, 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:49.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:49 smithi016 bash[33624]: cluster 2023-09-23T19:07:49.326942+0000 mon.a (mon.0) 682 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2023-09-23T19:07:49.786 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:49 smithi092 bash[27556]: cluster 2023-09-23T19:07:48.324063+0000 mon.a (mon.0) 681 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2023-09-23T19:07:49.786 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:49 smithi092 bash[27556]: cluster 2023-09-23T19:07:48.951357+0000 mgr.y (mgr.14454) 162 : cluster [DBG] pgmap v142: 89 pgs: 16 unknown, 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:49.786 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:49 smithi092 bash[27556]: cluster 2023-09-23T19:07:49.326942+0000 mon.a (mon.0) 682 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2023-09-23T19:07:50.285 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:07:50 smithi016 bash[27503]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:07:50] "GET /metrics HTTP/1.1" 200 191382 "" "Prometheus/2.33.4" 2023-09-23T19:07:50.557 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T19:07:51.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:50 smithi016 bash[27237]: audit 2023-09-23T19:07:49.758552+0000 mgr.y (mgr.14454) 163 : audit [DBG] from='client.14829 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:07:51.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:50 smithi016 bash[27237]: cephadm 2023-09-23T19:07:49.760124+0000 mgr.y (mgr.14454) 164 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T19:07:51.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:50 smithi016 bash[27237]: audit 2023-09-23T19:07:49.765743+0000 mon.a (mon.0) 683 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:07:51.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:50 smithi016 bash[27237]: audit 2023-09-23T19:07:49.809128+0000 mon.c (mon.2) 102 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:07:51.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:50 smithi016 bash[27237]: audit 2023-09-23T19:07:49.811617+0000 mon.c (mon.2) 103 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:07:51.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:50 smithi016 bash[27237]: audit 2023-09-23T19:07:49.812964+0000 mon.c (mon.2) 104 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:07:51.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:50 smithi016 bash[27237]: audit 2023-09-23T19:07:49.820301+0000 mon.a (mon.0) 684 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:07:51.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:50 smithi016 bash[27237]: cephadm 2023-09-23T19:07:49.829284+0000 mgr.y (mgr.14454) 165 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T19:07:51.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:50 smithi016 bash[33624]: audit 2023-09-23T19:07:49.758552+0000 mgr.y (mgr.14454) 163 : audit [DBG] from='client.14829 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:07:51.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:50 smithi016 bash[33624]: cephadm 2023-09-23T19:07:49.760124+0000 mgr.y (mgr.14454) 164 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T19:07:51.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:50 smithi016 bash[33624]: audit 2023-09-23T19:07:49.765743+0000 mon.a (mon.0) 683 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:07:51.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:50 smithi016 bash[33624]: audit 2023-09-23T19:07:49.809128+0000 mon.c (mon.2) 102 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:07:51.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:50 smithi016 bash[33624]: audit 2023-09-23T19:07:49.811617+0000 mon.c (mon.2) 103 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:07:51.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:50 smithi016 bash[33624]: audit 2023-09-23T19:07:49.812964+0000 mon.c (mon.2) 104 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:07:51.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:50 smithi016 bash[33624]: audit 2023-09-23T19:07:49.820301+0000 mon.a (mon.0) 684 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:07:51.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:50 smithi016 bash[33624]: cephadm 2023-09-23T19:07:49.829284+0000 mgr.y (mgr.14454) 165 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T19:07:51.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:50 smithi092 bash[27556]: audit 2023-09-23T19:07:49.758552+0000 mgr.y (mgr.14454) 163 : audit [DBG] from='client.14829 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:07:51.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:50 smithi092 bash[27556]: cephadm 2023-09-23T19:07:49.760124+0000 mgr.y (mgr.14454) 164 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T19:07:51.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:50 smithi092 bash[27556]: audit 2023-09-23T19:07:49.765743+0000 mon.a (mon.0) 683 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:07:51.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:50 smithi092 bash[27556]: audit 2023-09-23T19:07:49.809128+0000 mon.c (mon.2) 102 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:07:51.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:50 smithi092 bash[27556]: audit 2023-09-23T19:07:49.811617+0000 mon.c (mon.2) 103 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:07:51.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:50 smithi092 bash[27556]: audit 2023-09-23T19:07:49.812964+0000 mon.c (mon.2) 104 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:07:51.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:50 smithi092 bash[27556]: audit 2023-09-23T19:07:49.820301+0000 mon.a (mon.0) 684 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:07:51.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:50 smithi092 bash[27556]: cephadm 2023-09-23T19:07:49.829284+0000 mgr.y (mgr.14454) 165 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T19:07:52.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:51 smithi016 bash[27237]: cluster 2023-09-23T19:07:50.952056+0000 mgr.y (mgr.14454) 166 : cluster [DBG] pgmap v144: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:52.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:51 smithi016 bash[33624]: cluster 2023-09-23T19:07:50.952056+0000 mgr.y (mgr.14454) 166 : cluster [DBG] pgmap v144: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:52.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:51 smithi092 bash[27556]: cluster 2023-09-23T19:07:50.952056+0000 mgr.y (mgr.14454) 166 : cluster [DBG] pgmap v144: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:53.302 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:07:54.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:54 smithi016 bash[33624]: cluster 2023-09-23T19:07:52.952499+0000 mgr.y (mgr.14454) 167 : cluster [DBG] pgmap v145: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:54.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:54 smithi016 bash[27237]: cluster 2023-09-23T19:07:52.952499+0000 mgr.y (mgr.14454) 167 : cluster [DBG] pgmap v145: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:54.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:54 smithi092 bash[27556]: cluster 2023-09-23T19:07:52.952499+0000 mgr.y (mgr.14454) 167 : cluster [DBG] pgmap v145: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:54.974 INFO:teuthology.orchestra.run.smithi016.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T19:07:54.974 INFO:teuthology.orchestra.run.smithi016.stdout:alertmanager.a smithi016 *:9093,9094 running (2m) 7s ago 3m 14.1M - 0.23.0 ba2b418f427c ff6d7e7cd977 2023-09-23T19:07:54.974 INFO:teuthology.orchestra.run.smithi016.stdout:grafana.a smithi092 *:3000 running (3m) 8s ago 3m 42.8M - 8.3.5 dad864ee21e9 e781ecc0bb64 2023-09-23T19:07:54.974 INFO:teuthology.orchestra.run.smithi016.stdout:mds.foo.smithi016.uncssv smithi016 running (119s) 7s ago 119s 13.1M - 16.2.14-37-g2e1f1015 1e6a756174a8 95d1e187e3eb 2023-09-23T19:07:54.974 INFO:teuthology.orchestra.run.smithi016.stdout:mds.foo.smithi092.vbpqks smithi092 running (116s) 8s ago 116s 13.6M - 16.2.14-37-g2e1f1015 1e6a756174a8 1d619ec31bf1 2023-09-23T19:07:54.974 INFO:teuthology.orchestra.run.smithi016.stdout:mgr.x smithi092 *:8443 running (10m) 8s ago 10m 375M - 16.2.14-37-g2e1f1015 1e6a756174a8 26e3c1b9bb12 2023-09-23T19:07:54.974 INFO:teuthology.orchestra.run.smithi016.stdout:mgr.y smithi016 *:9283 running (12m) 7s ago 12m 428M - 16.2.14-37-g2e1f1015 1e6a756174a8 004fa9ebe371 2023-09-23T19:07:54.974 INFO:teuthology.orchestra.run.smithi016.stdout:mon.a smithi016 running (12m) 7s ago 12m 48.7M 2048M 16.2.14-37-g2e1f1015 1e6a756174a8 16af2f79be52 2023-09-23T19:07:54.975 INFO:teuthology.orchestra.run.smithi016.stdout:mon.b smithi092 running (10m) 8s ago 10m 42.1M 2048M 16.2.14-37-g2e1f1015 1e6a756174a8 7042e0aa9af7 2023-09-23T19:07:54.975 INFO:teuthology.orchestra.run.smithi016.stdout:mon.c smithi016 running (10m) 7s ago 10m 42.6M 2048M 16.2.14-37-g2e1f1015 1e6a756174a8 5c547dfcb630 2023-09-23T19:07:54.975 INFO:teuthology.orchestra.run.smithi016.stdout:node-exporter.a smithi016 *:9100 running (3m) 7s ago 3m 9111k - 1.3.1 1dbe0e931976 86c71fbee9dc 2023-09-23T19:07:54.975 INFO:teuthology.orchestra.run.smithi016.stdout:node-exporter.b smithi092 *:9100 running (3m) 8s ago 3m 9440k - 1.3.1 1dbe0e931976 55c937172f1b 2023-09-23T19:07:54.975 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0 smithi016 running (9m) 7s ago 9m 38.4M 4061M 16.2.14-37-g2e1f1015 1e6a756174a8 1458de442fb1 2023-09-23T19:07:54.975 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1 smithi016 running (9m) 7s ago 9m 38.7M 4061M 16.2.14-37-g2e1f1015 1e6a756174a8 7e0e07b2f6fd 2023-09-23T19:07:54.975 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2 smithi016 running (8m) 7s ago 8m 37.8M 4061M 16.2.14-37-g2e1f1015 1e6a756174a8 0c6fd3da0e21 2023-09-23T19:07:54.975 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3 smithi016 running (7m) 7s ago 7m 38.6M 4061M 16.2.14-37-g2e1f1015 1e6a756174a8 a34971c269fe 2023-09-23T19:07:54.975 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4 smithi092 running (7m) 8s ago 7m 38.7M 4316M 16.2.14-37-g2e1f1015 1e6a756174a8 253dc4b11010 2023-09-23T19:07:54.975 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5 smithi092 running (6m) 8s ago 6m 39.6M 4316M 16.2.14-37-g2e1f1015 1e6a756174a8 0a7306534382 2023-09-23T19:07:54.975 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6 smithi092 running (5m) 8s ago 5m 40.0M 4316M 16.2.14-37-g2e1f1015 1e6a756174a8 d6f3ee67cc5f 2023-09-23T19:07:54.975 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7 smithi092 running (4m) 8s ago 5m 38.4M 4316M 16.2.14-37-g2e1f1015 1e6a756174a8 794cf2340f17 2023-09-23T19:07:54.976 INFO:teuthology.orchestra.run.smithi016.stdout:prometheus.a smithi092 *:9095 running (2m) 8s ago 3m 52.2M - 2.33.4 514e6a882f6e 2e214c5a4c20 2023-09-23T19:07:56.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:56 smithi016 bash[27237]: audit 2023-09-23T19:07:54.948479+0000 mgr.y (mgr.14454) 168 : audit [DBG] from='client.24734 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:07:56.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:56 smithi016 bash[27237]: cluster 2023-09-23T19:07:54.952874+0000 mgr.y (mgr.14454) 169 : cluster [DBG] pgmap v146: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:56.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:56 smithi016 bash[33624]: audit 2023-09-23T19:07:54.948479+0000 mgr.y (mgr.14454) 168 : audit [DBG] from='client.24734 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:07:56.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:56 smithi016 bash[33624]: cluster 2023-09-23T19:07:54.952874+0000 mgr.y (mgr.14454) 169 : cluster [DBG] pgmap v146: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:56.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:56 smithi092 bash[27556]: audit 2023-09-23T19:07:54.948479+0000 mgr.y (mgr.14454) 168 : audit [DBG] from='client.24734 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:07:56.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:56 smithi092 bash[27556]: cluster 2023-09-23T19:07:54.952874+0000 mgr.y (mgr.14454) 169 : cluster [DBG] pgmap v146: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:56.321 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'echo wait for minority of mons to upgrade' 2023-09-23T19:07:58.193 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:58 smithi016 bash[27237]: cluster 2023-09-23T19:07:56.953539+0000 mgr.y (mgr.14454) 170 : cluster [DBG] pgmap v147: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:58.194 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:58 smithi016 bash[33624]: cluster 2023-09-23T19:07:56.953539+0000 mgr.y (mgr.14454) 170 : cluster [DBG] pgmap v147: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:58.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:58 smithi092 bash[27556]: cluster 2023-09-23T19:07:56.953539+0000 mgr.y (mgr.14454) 170 : cluster [DBG] pgmap v147: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:07:59.039 INFO:teuthology.orchestra.run.smithi016.stderr:enabled application 'rados' on pool 'unique_pool_0' 2023-09-23T19:07:59.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:07:59 smithi016 bash[33624]: audit 2023-09-23T19:07:58.401926+0000 mon.a (mon.0) 685 : audit [INF] from='client.? 172.21.15.16:0/3672341976' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:07:59.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:07:59 smithi016 bash[27237]: audit 2023-09-23T19:07:58.401926+0000 mon.a (mon.0) 685 : audit [INF] from='client.? 172.21.15.16:0/3672341976' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:07:59.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:07:59 smithi092 bash[27556]: audit 2023-09-23T19:07:58.401926+0000 mon.a (mon.0) 685 : audit [INF] from='client.? 172.21.15.16:0/3672341976' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:07:59.287 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:07:59 smithi092 bash[28593]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:07:59] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T19:08:00.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:00 smithi016 bash[27237]: cluster 2023-09-23T19:07:58.953988+0000 mgr.y (mgr.14454) 171 : cluster [DBG] pgmap v148: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:08:00.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:00 smithi016 bash[27237]: audit 2023-09-23T19:07:59.029248+0000 mon.a (mon.0) 686 : audit [INF] from='client.? 172.21.15.16:0/3672341976' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T19:08:00.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:00 smithi016 bash[27237]: cluster 2023-09-23T19:07:59.029342+0000 mon.a (mon.0) 687 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2023-09-23T19:08:00.286 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:08:00 smithi016 bash[27503]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:08:00] "GET /metrics HTTP/1.1" 200 193784 "" "Prometheus/2.33.4" 2023-09-23T19:08:00.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:00 smithi016 bash[33624]: cluster 2023-09-23T19:07:58.953988+0000 mgr.y (mgr.14454) 171 : cluster [DBG] pgmap v148: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:08:00.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:00 smithi016 bash[33624]: audit 2023-09-23T19:07:59.029248+0000 mon.a (mon.0) 686 : audit [INF] from='client.? 172.21.15.16:0/3672341976' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T19:08:00.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:00 smithi016 bash[33624]: cluster 2023-09-23T19:07:59.029342+0000 mon.a (mon.0) 687 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2023-09-23T19:08:00.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:00 smithi092 bash[27556]: cluster 2023-09-23T19:07:58.953988+0000 mgr.y (mgr.14454) 171 : cluster [DBG] pgmap v148: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:08:00.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:00 smithi092 bash[27556]: audit 2023-09-23T19:07:59.029248+0000 mon.a (mon.0) 686 : audit [INF] from='client.? 172.21.15.16:0/3672341976' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T19:08:00.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:00 smithi092 bash[27556]: cluster 2023-09-23T19:07:59.029342+0000 mon.a (mon.0) 687 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2023-09-23T19:08:00.971 INFO:teuthology.orchestra.run.smithi016.stdout:wait for minority of mons to upgrade 2023-09-23T19:08:01.758 DEBUG:teuthology.orchestra.run.smithi016:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_0 bench 90 write' 2023-09-23T19:08:01.761 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T19:08:01.761 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T19:08:01.809 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:hints = 1 2023-09-23T19:08:01.810 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T19:08:01.810 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Object prefix: benchmark_data_smithi016_66633 2023-09-23T19:08:01.810 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:08:01.810 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 0 0 0 0 0 0 - 0 2023-09-23T19:08:01.880 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done' 2023-09-23T19:08:02.205 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:02 smithi016 bash[27237]: cluster 2023-09-23T19:08:00.955019+0000 mgr.y (mgr.14454) 172 : cluster [DBG] pgmap v150: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:08:02.205 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:02 smithi016 bash[33624]: cluster 2023-09-23T19:08:00.955019+0000 mgr.y (mgr.14454) 172 : cluster [DBG] pgmap v150: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:08:02.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:02 smithi092 bash[27556]: cluster 2023-09-23T19:08:00.955019+0000 mgr.y (mgr.14454) 172 : cluster [DBG] pgmap v150: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:08:02.810 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 1 16 2625 2609 163.052 163.062 0.00482496 0.00611912 2023-09-23T19:08:03.810 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 2 16 5464 5448 170.236 177.438 0.00513183 0.00586066 2023-09-23T19:08:04.199 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:04 smithi016 bash[27237]: cluster 2023-09-23T19:08:02.955487+0000 mgr.y (mgr.14454) 173 : cluster [DBG] pgmap v151: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:08:04.199 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:04 smithi016 bash[33624]: cluster 2023-09-23T19:08:02.955487+0000 mgr.y (mgr.14454) 173 : cluster [DBG] pgmap v151: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:08:04.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:04 smithi092 bash[27556]: cluster 2023-09-23T19:08:02.955487+0000 mgr.y (mgr.14454) 173 : cluster [DBG] pgmap v151: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:08:04.810 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 3 15 8102 8087 168.465 164.938 0.0110241 0.00592448 2023-09-23T19:08:05.810 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 4 16 9837 9821 153.439 108.375 0.0122197 0.00650407 2023-09-23T19:08:06.443 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:06 smithi016 bash[27237]: cluster 2023-09-23T19:08:04.955892+0000 mgr.y (mgr.14454) 174 : cluster [DBG] pgmap v152: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:08:06.443 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:06 smithi016 bash[33624]: cluster 2023-09-23T19:08:04.955892+0000 mgr.y (mgr.14454) 174 : cluster [DBG] pgmap v152: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:08:06.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:06 smithi092 bash[27556]: cluster 2023-09-23T19:08:04.955892+0000 mgr.y (mgr.14454) 174 : cluster [DBG] pgmap v152: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:08:06.811 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 5 16 12198 12182 152.261 147.562 0.0113313 0.00655561 2023-09-23T19:08:07.811 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 6 16 14244 14228 148.194 127.875 0.00543927 0.00673433 2023-09-23T19:08:08.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:08 smithi016 bash[33624]: cluster 2023-09-23T19:08:06.956770+0000 mgr.y (mgr.14454) 175 : cluster [DBG] pgmap v153: 89 pgs: 89 active+clean; 633 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T19:08:08.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:08 smithi016 bash[27237]: cluster 2023-09-23T19:08:06.956770+0000 mgr.y (mgr.14454) 175 : cluster [DBG] pgmap v153: 89 pgs: 89 active+clean; 633 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T19:08:08.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:08 smithi092 bash[27556]: cluster 2023-09-23T19:08:06.956770+0000 mgr.y (mgr.14454) 175 : cluster [DBG] pgmap v153: 89 pgs: 89 active+clean; 633 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T19:08:08.811 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 7 16 16150 16134 144.04 119.125 0.0047023 0.00693394 2023-09-23T19:08:09.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:09 smithi016 bash[27237]: audit 2023-09-23T19:08:08.248995+0000 mon.b (mon.1) 28 : audit [DBG] from='client.? 172.21.15.16:0/3224674641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:09.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:09 smithi016 bash[33624]: audit 2023-09-23T19:08:08.248995+0000 mon.b (mon.1) 28 : audit [DBG] from='client.? 172.21.15.16:0/3224674641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:09.536 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:08:09 smithi092 bash[28593]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:08:09] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T19:08:09.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:09 smithi092 bash[27556]: audit 2023-09-23T19:08:08.248995+0000 mon.b (mon.1) 28 : audit [DBG] from='client.? 172.21.15.16:0/3224674641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:09.811 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 8 16 18038 18022 140.784 118 0.00472679 0.00709586 2023-09-23T19:08:10.535 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:08:10 smithi016 bash[27503]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:08:10] "GET /metrics HTTP/1.1" 200 193788 "" "Prometheus/2.33.4" 2023-09-23T19:08:10.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:10 smithi016 bash[27237]: cluster 2023-09-23T19:08:08.957264+0000 mgr.y (mgr.14454) 176 : cluster [DBG] pgmap v154: 89 pgs: 89 active+clean; 633 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T19:08:10.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:10 smithi016 bash[33624]: cluster 2023-09-23T19:08:08.957264+0000 mgr.y (mgr.14454) 176 : cluster [DBG] pgmap v154: 89 pgs: 89 active+clean; 633 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T19:08:10.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:10 smithi092 bash[27556]: cluster 2023-09-23T19:08:08.957264+0000 mgr.y (mgr.14454) 176 : cluster [DBG] pgmap v154: 89 pgs: 89 active+clean; 633 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T19:08:10.811 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 9 16 19774 19758 137.195 108.5 0.00236494 0.00727918 2023-09-23T19:08:11.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:11 smithi016 bash[27237]: audit 2023-09-23T19:08:10.295960+0000 mon.a (mon.0) 688 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:08:11.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:11 smithi016 bash[27237]: cluster 2023-09-23T19:08:10.958046+0000 mgr.y (mgr.14454) 177 : cluster [DBG] pgmap v155: 89 pgs: 89 active+clean; 1.2 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s 2023-09-23T19:08:11.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:11 smithi016 bash[27237]: audit 2023-09-23T19:08:10.999210+0000 mon.b (mon.1) 29 : audit [DBG] from='client.? 172.21.15.16:0/2424374168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:11.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:11 smithi016 bash[33624]: audit 2023-09-23T19:08:10.295960+0000 mon.a (mon.0) 688 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:08:11.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:11 smithi016 bash[33624]: cluster 2023-09-23T19:08:10.958046+0000 mgr.y (mgr.14454) 177 : cluster [DBG] pgmap v155: 89 pgs: 89 active+clean; 1.2 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s 2023-09-23T19:08:11.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:11 smithi016 bash[33624]: audit 2023-09-23T19:08:10.999210+0000 mon.b (mon.1) 29 : audit [DBG] from='client.? 172.21.15.16:0/2424374168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:11.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:11 smithi092 bash[27556]: audit 2023-09-23T19:08:10.295960+0000 mon.a (mon.0) 688 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:08:11.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:11 smithi092 bash[27556]: cluster 2023-09-23T19:08:10.958046+0000 mgr.y (mgr.14454) 177 : cluster [DBG] pgmap v155: 89 pgs: 89 active+clean; 1.2 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s 2023-09-23T19:08:11.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:11 smithi092 bash[27556]: audit 2023-09-23T19:08:10.999210+0000 mon.b (mon.1) 29 : audit [DBG] from='client.? 172.21.15.16:0/2424374168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:11.811 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 10 16 21515 21499 134.356 108.812 0.00271096 0.00743243 2023-09-23T19:08:12.811 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 11 16 23364 23348 132.647 115.562 0.00941485 0.00753164 2023-09-23T19:08:13.811 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 12 16 24923 24907 129.712 97.4375 0.00321684 0.00770003 2023-09-23T19:08:14.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:14 smithi016 bash[27237]: cluster 2023-09-23T19:08:12.958495+0000 mgr.y (mgr.14454) 178 : cluster [DBG] pgmap v156: 89 pgs: 89 active+clean; 1.2 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-09-23T19:08:14.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:14 smithi016 bash[27237]: audit 2023-09-23T19:08:13.776863+0000 mon.a (mon.0) 689 : audit [DBG] from='client.? 172.21.15.16:0/2994620331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:14.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:14 smithi016 bash[33624]: cluster 2023-09-23T19:08:12.958495+0000 mgr.y (mgr.14454) 178 : cluster [DBG] pgmap v156: 89 pgs: 89 active+clean; 1.2 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-09-23T19:08:14.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:14 smithi016 bash[33624]: audit 2023-09-23T19:08:13.776863+0000 mon.a (mon.0) 689 : audit [DBG] from='client.? 172.21.15.16:0/2994620331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:14.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:14 smithi092 bash[27556]: cluster 2023-09-23T19:08:12.958495+0000 mgr.y (mgr.14454) 178 : cluster [DBG] pgmap v156: 89 pgs: 89 active+clean; 1.2 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-09-23T19:08:14.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:14 smithi092 bash[27556]: audit 2023-09-23T19:08:13.776863+0000 mon.a (mon.0) 689 : audit [DBG] from='client.? 172.21.15.16:0/2994620331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:14.811 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 13 16 26568 26552 127.642 102.812 0.00519428 0.00782399 2023-09-23T19:08:15.812 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 14 16 28219 28203 125.894 103.188 0.00633758 0.00793497 2023-09-23T19:08:16.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:16 smithi016 bash[27237]: cluster 2023-09-23T19:08:14.959715+0000 mgr.y (mgr.14454) 179 : cluster [DBG] pgmap v157: 89 pgs: 89 active+clean; 1.2 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-09-23T19:08:16.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:16 smithi016 bash[33624]: cluster 2023-09-23T19:08:14.959715+0000 mgr.y (mgr.14454) 179 : cluster [DBG] pgmap v157: 89 pgs: 89 active+clean; 1.2 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-09-23T19:08:16.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:16 smithi092 bash[27556]: cluster 2023-09-23T19:08:14.959715+0000 mgr.y (mgr.14454) 179 : cluster [DBG] pgmap v157: 89 pgs: 89 active+clean; 1.2 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-09-23T19:08:16.811 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 15 15 29698 29683 123.668 92.5 0.00549677 0.00807562 2023-09-23T19:08:17.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:17 smithi016 bash[27237]: audit 2023-09-23T19:08:16.537884+0000 mon.c (mon.2) 105 : audit [DBG] from='client.? 172.21.15.16:0/318378096' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:17.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:17 smithi016 bash[33624]: audit 2023-09-23T19:08:16.537884+0000 mon.c (mon.2) 105 : audit [DBG] from='client.? 172.21.15.16:0/318378096' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:17.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:17 smithi092 bash[27556]: audit 2023-09-23T19:08:16.537884+0000 mon.c (mon.2) 105 : audit [DBG] from='client.? 172.21.15.16:0/318378096' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:17.812 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 16 16 31305 31289 122.211 100.375 0.00794945 0.00817552 2023-09-23T19:08:18.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:18 smithi016 bash[33624]: cluster 2023-09-23T19:08:16.960485+0000 mgr.y (mgr.14454) 180 : cluster [DBG] pgmap v158: 89 pgs: 89 active+clean; 1.7 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 148 MiB/s wr, 2.37k op/s 2023-09-23T19:08:18.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:18 smithi016 bash[27237]: cluster 2023-09-23T19:08:16.960485+0000 mgr.y (mgr.14454) 180 : cluster [DBG] pgmap v158: 89 pgs: 89 active+clean; 1.7 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 148 MiB/s wr, 2.37k op/s 2023-09-23T19:08:18.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:18 smithi092 bash[27556]: cluster 2023-09-23T19:08:16.960485+0000 mgr.y (mgr.14454) 180 : cluster [DBG] pgmap v158: 89 pgs: 89 active+clean; 1.7 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 148 MiB/s wr, 2.37k op/s 2023-09-23T19:08:18.812 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 17 15 32798 32783 120.514 93.375 0.0171343 0.00829114 2023-09-23T19:08:19.536 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:08:19 smithi092 bash[28593]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:08:19] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T19:08:19.812 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 18 16 34183 34167 118.624 86.5 0.0125694 0.00842013 2023-09-23T19:08:20.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:20 smithi016 bash[27237]: cluster 2023-09-23T19:08:18.960921+0000 mgr.y (mgr.14454) 181 : cluster [DBG] pgmap v159: 89 pgs: 89 active+clean; 1.7 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-09-23T19:08:20.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:20 smithi016 bash[27237]: audit 2023-09-23T19:08:19.300769+0000 mon.a (mon.0) 690 : audit [DBG] from='client.? 172.21.15.16:0/2202768260' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:20.536 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:08:20 smithi016 bash[27503]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:08:20] "GET /metrics HTTP/1.1" 200 193788 "" "Prometheus/2.33.4" 2023-09-23T19:08:20.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:20 smithi016 bash[33624]: cluster 2023-09-23T19:08:18.960921+0000 mgr.y (mgr.14454) 181 : cluster [DBG] pgmap v159: 89 pgs: 89 active+clean; 1.7 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-09-23T19:08:20.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:20 smithi016 bash[33624]: audit 2023-09-23T19:08:19.300769+0000 mon.a (mon.0) 690 : audit [DBG] from='client.? 172.21.15.16:0/2202768260' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:20.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:20 smithi092 bash[27556]: cluster 2023-09-23T19:08:18.960921+0000 mgr.y (mgr.14454) 181 : cluster [DBG] pgmap v159: 89 pgs: 89 active+clean; 1.7 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-09-23T19:08:20.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:20 smithi092 bash[27556]: audit 2023-09-23T19:08:19.300769+0000 mon.a (mon.0) 690 : audit [DBG] from='client.? 172.21.15.16:0/2202768260' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:20.812 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 19 16 35656 35640 117.226 92.0625 0.0188112 0.00852168 2023-09-23T19:08:21.812 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-09-23T19:08:21.809735+0000 min lat: 0.00145215 max lat: 0.276334 avg lat: 0.0086196 2023-09-23T19:08:21.812 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:08:21.812 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 20 16 37116 37100 115.927 91.25 0.0102171 0.0086196 2023-09-23T19:08:22.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:22 smithi016 bash[33624]: cluster 2023-09-23T19:08:20.961680+0000 mgr.y (mgr.14454) 182 : cluster [DBG] pgmap v160: 89 pgs: 89 active+clean; 2.1 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 129 MiB/s wr, 2.07k op/s 2023-09-23T19:08:22.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:22 smithi016 bash[27237]: cluster 2023-09-23T19:08:20.961680+0000 mgr.y (mgr.14454) 182 : cluster [DBG] pgmap v160: 89 pgs: 89 active+clean; 2.1 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 129 MiB/s wr, 2.07k op/s 2023-09-23T19:08:22.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:22 smithi092 bash[27556]: cluster 2023-09-23T19:08:20.961680+0000 mgr.y (mgr.14454) 182 : cluster [DBG] pgmap v160: 89 pgs: 89 active+clean; 2.1 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 129 MiB/s wr, 2.07k op/s 2023-09-23T19:08:22.812 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 21 16 38722 38706 115.185 100.375 0.00512009 0.00867511 2023-09-23T19:08:23.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:23 smithi016 bash[33624]: audit 2023-09-23T19:08:22.095895+0000 mon.b (mon.1) 30 : audit [DBG] from='client.? 172.21.15.16:0/3187720498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:23.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:23 smithi016 bash[27237]: audit 2023-09-23T19:08:22.095895+0000 mon.b (mon.1) 30 : audit [DBG] from='client.? 172.21.15.16:0/3187720498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:23.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:23 smithi092 bash[27556]: audit 2023-09-23T19:08:22.095895+0000 mon.b (mon.1) 30 : audit [DBG] from='client.? 172.21.15.16:0/3187720498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:23.812 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 22 16 40397 40381 114.708 104.688 0.0103219 0.00871157 2023-09-23T19:08:24.375 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:24 smithi016 bash[27237]: cluster 2023-09-23T19:08:22.962173+0000 mgr.y (mgr.14454) 183 : cluster [DBG] pgmap v161: 89 pgs: 89 active+clean; 2.2 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2023-09-23T19:08:24.376 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:24 smithi016 bash[33624]: cluster 2023-09-23T19:08:22.962173+0000 mgr.y (mgr.14454) 183 : cluster [DBG] pgmap v161: 89 pgs: 89 active+clean; 2.2 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2023-09-23T19:08:24.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:24 smithi092 bash[27556]: cluster 2023-09-23T19:08:22.962173+0000 mgr.y (mgr.14454) 183 : cluster [DBG] pgmap v161: 89 pgs: 89 active+clean; 2.2 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2023-09-23T19:08:24.812 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 23 16 42014 41998 114.114 101.062 0.00723533 0.00875758 2023-09-23T19:08:25.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:25 smithi016 bash[33624]: audit 2023-09-23T19:08:24.685847+0000 mon.c (mon.2) 106 : audit [DBG] from='client.? 172.21.15.16:0/1969120358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:25.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:25 smithi016 bash[27237]: audit 2023-09-23T19:08:24.685847+0000 mon.c (mon.2) 106 : audit [DBG] from='client.? 172.21.15.16:0/1969120358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:25.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:25 smithi092 bash[27556]: audit 2023-09-23T19:08:24.685847+0000 mon.c (mon.2) 106 : audit [DBG] from='client.? 172.21.15.16:0/1969120358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:25.812 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 24 16 43660 43644 113.645 102.875 0.00756509 0.00879358 2023-09-23T19:08:26.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:26 smithi016 bash[33624]: cluster 2023-09-23T19:08:24.962606+0000 mgr.y (mgr.14454) 184 : cluster [DBG] pgmap v162: 89 pgs: 89 active+clean; 2.2 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2023-09-23T19:08:26.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:26 smithi016 bash[27237]: cluster 2023-09-23T19:08:24.962606+0000 mgr.y (mgr.14454) 184 : cluster [DBG] pgmap v162: 89 pgs: 89 active+clean; 2.2 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2023-09-23T19:08:26.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:26 smithi092 bash[27556]: cluster 2023-09-23T19:08:24.962606+0000 mgr.y (mgr.14454) 184 : cluster [DBG] pgmap v162: 89 pgs: 89 active+clean; 2.2 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2023-09-23T19:08:26.812 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 25 16 45248 45232 113.069 99.25 0.00842097 0.00883843 2023-09-23T19:08:27.813 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 26 16 46773 46757 112.386 95.3125 0.00603151 0.00889178 2023-09-23T19:08:28.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:28 smithi016 bash[33624]: cluster 2023-09-23T19:08:26.963413+0000 mgr.y (mgr.14454) 185 : cluster [DBG] pgmap v163: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 125 MiB/s wr, 1.99k op/s 2023-09-23T19:08:28.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:28 smithi016 bash[33624]: audit 2023-09-23T19:08:27.363356+0000 mon.a (mon.0) 691 : audit [DBG] from='client.? 172.21.15.16:0/1391939999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:28.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:28 smithi016 bash[27237]: cluster 2023-09-23T19:08:26.963413+0000 mgr.y (mgr.14454) 185 : cluster [DBG] pgmap v163: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 125 MiB/s wr, 1.99k op/s 2023-09-23T19:08:28.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:28 smithi016 bash[27237]: audit 2023-09-23T19:08:27.363356+0000 mon.a (mon.0) 691 : audit [DBG] from='client.? 172.21.15.16:0/1391939999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:28.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:28 smithi092 bash[27556]: cluster 2023-09-23T19:08:26.963413+0000 mgr.y (mgr.14454) 185 : cluster [DBG] pgmap v163: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 125 MiB/s wr, 1.99k op/s 2023-09-23T19:08:28.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:28 smithi092 bash[27556]: audit 2023-09-23T19:08:27.363356+0000 mon.a (mon.0) 691 : audit [DBG] from='client.? 172.21.15.16:0/1391939999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:28.813 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 27 16 48326 48310 111.818 97.0625 0.00875152 0.00893703 2023-09-23T19:08:29.536 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:08:29 smithi092 bash[28593]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:08:29] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T19:08:29.813 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 28 16 49779 49763 111.068 90.8125 0.00849883 0.00899772 2023-09-23T19:08:30.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:30 smithi016 bash[27237]: cluster 2023-09-23T19:08:28.963963+0000 mgr.y (mgr.14454) 186 : cluster [DBG] pgmap v164: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-09-23T19:08:30.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:30 smithi016 bash[27237]: audit 2023-09-23T19:08:30.053235+0000 mon.a (mon.0) 692 : audit [DBG] from='client.? 172.21.15.16:0/1368291023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:30.536 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:08:30 smithi016 bash[27503]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:08:30] "GET /metrics HTTP/1.1" 200 194565 "" "Prometheus/2.33.4" 2023-09-23T19:08:30.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:30 smithi016 bash[33624]: cluster 2023-09-23T19:08:28.963963+0000 mgr.y (mgr.14454) 186 : cluster [DBG] pgmap v164: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-09-23T19:08:30.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:30 smithi016 bash[33624]: audit 2023-09-23T19:08:30.053235+0000 mon.a (mon.0) 692 : audit [DBG] from='client.? 172.21.15.16:0/1368291023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:30.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:30 smithi092 bash[27556]: cluster 2023-09-23T19:08:28.963963+0000 mgr.y (mgr.14454) 186 : cluster [DBG] pgmap v164: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-09-23T19:08:30.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:30 smithi092 bash[27556]: audit 2023-09-23T19:08:30.053235+0000 mon.a (mon.0) 692 : audit [DBG] from='client.? 172.21.15.16:0/1368291023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:30.813 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 29 16 51283 51267 110.479 94 0.00614694 0.00904556 2023-09-23T19:08:31.813 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 30 16 52784 52768 109.923 93.8125 0.00647085 0.00909162 2023-09-23T19:08:32.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:32 smithi016 bash[27237]: cluster 2023-09-23T19:08:30.964633+0000 mgr.y (mgr.14454) 187 : cluster [DBG] pgmap v165: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-09-23T19:08:32.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:32 smithi016 bash[33624]: cluster 2023-09-23T19:08:30.964633+0000 mgr.y (mgr.14454) 187 : cluster [DBG] pgmap v165: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-09-23T19:08:32.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:32 smithi092 bash[27556]: cluster 2023-09-23T19:08:30.964633+0000 mgr.y (mgr.14454) 187 : cluster [DBG] pgmap v165: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-09-23T19:08:32.813 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 31 16 54131 54115 109.093 84.1875 0.00670754 0.00916024 2023-09-23T19:08:33.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:33 smithi016 bash[27237]: audit 2023-09-23T19:08:32.737749+0000 mon.c (mon.2) 107 : audit [DBG] from='client.? 172.21.15.16:0/1131326316' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:33.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:33 smithi016 bash[33624]: audit 2023-09-23T19:08:32.737749+0000 mon.c (mon.2) 107 : audit [DBG] from='client.? 172.21.15.16:0/1131326316' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:33.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:33 smithi092 bash[27556]: audit 2023-09-23T19:08:32.737749+0000 mon.c (mon.2) 107 : audit [DBG] from='client.? 172.21.15.16:0/1131326316' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:33.813 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 32 16 55522 55506 108.4 86.9375 0.0196784 0.0092187 2023-09-23T19:08:34.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:34 smithi016 bash[27237]: cluster 2023-09-23T19:08:32.965114+0000 mgr.y (mgr.14454) 188 : cluster [DBG] pgmap v166: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-09-23T19:08:34.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:34 smithi016 bash[33624]: cluster 2023-09-23T19:08:32.965114+0000 mgr.y (mgr.14454) 188 : cluster [DBG] pgmap v166: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-09-23T19:08:34.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:34 smithi092 bash[27556]: cluster 2023-09-23T19:08:32.965114+0000 mgr.y (mgr.14454) 188 : cluster [DBG] pgmap v166: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-09-23T19:08:34.813 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 33 16 56918 56902 107.759 87.25 0.00967013 0.00927424 2023-09-23T19:08:35.813 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 34 16 58199 58183 106.944 80.0625 0.00762174 0.0093445 2023-09-23T19:08:36.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:35 smithi016 bash[27237]: cluster 2023-09-23T19:08:34.965548+0000 mgr.y (mgr.14454) 189 : cluster [DBG] pgmap v167: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-09-23T19:08:36.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:35 smithi016 bash[27237]: audit 2023-09-23T19:08:35.421593+0000 mon.b (mon.1) 31 : audit [DBG] from='client.? 172.21.15.16:0/3894418115' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:36.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:35 smithi016 bash[33624]: cluster 2023-09-23T19:08:34.965548+0000 mgr.y (mgr.14454) 189 : cluster [DBG] pgmap v167: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-09-23T19:08:36.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:35 smithi016 bash[33624]: audit 2023-09-23T19:08:35.421593+0000 mon.b (mon.1) 31 : audit [DBG] from='client.? 172.21.15.16:0/3894418115' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:36.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:35 smithi092 bash[27556]: cluster 2023-09-23T19:08:34.965548+0000 mgr.y (mgr.14454) 189 : cluster [DBG] pgmap v167: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-09-23T19:08:36.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:35 smithi092 bash[27556]: audit 2023-09-23T19:08:35.421593+0000 mon.b (mon.1) 31 : audit [DBG] from='client.? 172.21.15.16:0/3894418115' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:36.814 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 35 16 59550 59534 106.301 84.4375 0.00871812 0.00940182 2023-09-23T19:08:37.814 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 36 16 60818 60802 105.549 79.25 0.0116914 0.00946784 2023-09-23T19:08:38.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:37 smithi016 bash[27237]: cluster 2023-09-23T19:08:36.966301+0000 mgr.y (mgr.14454) 190 : cluster [DBG] pgmap v168: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 117 MiB/s wr, 1.88k op/s 2023-09-23T19:08:38.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:37 smithi016 bash[33624]: cluster 2023-09-23T19:08:36.966301+0000 mgr.y (mgr.14454) 190 : cluster [DBG] pgmap v168: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 117 MiB/s wr, 1.88k op/s 2023-09-23T19:08:38.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:37 smithi092 bash[27556]: cluster 2023-09-23T19:08:36.966301+0000 mgr.y (mgr.14454) 190 : cluster [DBG] pgmap v168: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 117 MiB/s wr, 1.88k op/s 2023-09-23T19:08:38.814 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 37 16 62096 62080 104.855 79.875 0.00834236 0.00953117 2023-09-23T19:08:39.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:38 smithi016 bash[27237]: audit 2023-09-23T19:08:38.112036+0000 mon.b (mon.1) 32 : audit [DBG] from='client.? 172.21.15.16:0/2626605901' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:39.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:38 smithi016 bash[27237]: audit 2023-09-23T19:08:38.856436+0000 mon.a (mon.0) 693 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:08:39.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:38 smithi016 bash[27237]: audit 2023-09-23T19:08:38.863952+0000 mon.c (mon.2) 108 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:08:39.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:38 smithi016 bash[27237]: audit 2023-09-23T19:08:38.872171+0000 mon.a (mon.0) 694 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:08:39.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:38 smithi016 bash[33624]: audit 2023-09-23T19:08:38.112036+0000 mon.b (mon.1) 32 : audit [DBG] from='client.? 172.21.15.16:0/2626605901' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:39.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:38 smithi016 bash[33624]: audit 2023-09-23T19:08:38.856436+0000 mon.a (mon.0) 693 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:08:39.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:38 smithi016 bash[33624]: audit 2023-09-23T19:08:38.863952+0000 mon.c (mon.2) 108 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:08:39.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:38 smithi016 bash[33624]: audit 2023-09-23T19:08:38.872171+0000 mon.a (mon.0) 694 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:08:39.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:39 smithi092 bash[27556]: audit 2023-09-23T19:08:38.112036+0000 mon.b (mon.1) 32 : audit [DBG] from='client.? 172.21.15.16:0/2626605901' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:39.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:39 smithi092 bash[27556]: audit 2023-09-23T19:08:38.856436+0000 mon.a (mon.0) 693 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:08:39.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:39 smithi092 bash[27556]: audit 2023-09-23T19:08:38.863952+0000 mon.c (mon.2) 108 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:08:39.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:39 smithi092 bash[27556]: audit 2023-09-23T19:08:38.872171+0000 mon.a (mon.0) 694 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:08:39.287 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:08:39 smithi092 bash[28593]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:08:39] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T19:08:39.814 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 38 16 63314 63298 104.099 76.125 0.0110886 0.00959996 2023-09-23T19:08:40.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:40 smithi016 bash[27237]: cephadm 2023-09-23T19:08:38.861720+0000 mgr.y (mgr.14454) 191 : cephadm [INF] Upgrade: Target is version 18.2.0-460-gbb0daeff (unknown) 2023-09-23T19:08:40.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:40 smithi016 bash[27237]: cephadm 2023-09-23T19:08:38.861828+0000 mgr.y (mgr.14454) 192 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d'] 2023-09-23T19:08:40.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:40 smithi016 bash[27237]: cephadm 2023-09-23T19:08:38.879508+0000 mgr.y (mgr.14454) 193 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-09-23T19:08:40.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:40 smithi016 bash[27237]: cluster 2023-09-23T19:08:38.966788+0000 mgr.y (mgr.14454) 194 : cluster [DBG] pgmap v169: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-09-23T19:08:40.286 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:08:40 smithi016 bash[27503]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:08:40] "GET /metrics HTTP/1.1" 200 194624 "" "Prometheus/2.33.4" 2023-09-23T19:08:40.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:40 smithi016 bash[33624]: cephadm 2023-09-23T19:08:38.861720+0000 mgr.y (mgr.14454) 191 : cephadm [INF] Upgrade: Target is version 18.2.0-460-gbb0daeff (unknown) 2023-09-23T19:08:40.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:40 smithi016 bash[33624]: cephadm 2023-09-23T19:08:38.861828+0000 mgr.y (mgr.14454) 192 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d'] 2023-09-23T19:08:40.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:40 smithi016 bash[33624]: cephadm 2023-09-23T19:08:38.879508+0000 mgr.y (mgr.14454) 193 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-09-23T19:08:40.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:40 smithi016 bash[33624]: cluster 2023-09-23T19:08:38.966788+0000 mgr.y (mgr.14454) 194 : cluster [DBG] pgmap v169: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-09-23T19:08:40.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:40 smithi092 bash[27556]: cephadm 2023-09-23T19:08:38.861720+0000 mgr.y (mgr.14454) 191 : cephadm [INF] Upgrade: Target is version 18.2.0-460-gbb0daeff (unknown) 2023-09-23T19:08:40.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:40 smithi092 bash[27556]: cephadm 2023-09-23T19:08:38.861828+0000 mgr.y (mgr.14454) 192 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d'] 2023-09-23T19:08:40.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:40 smithi092 bash[27556]: cephadm 2023-09-23T19:08:38.879508+0000 mgr.y (mgr.14454) 193 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-09-23T19:08:40.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:40 smithi092 bash[27556]: cluster 2023-09-23T19:08:38.966788+0000 mgr.y (mgr.14454) 194 : cluster [DBG] pgmap v169: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-09-23T19:08:40.815 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 39 16 64436 64420 103.227 70.125 0.00948875 0.00968137 2023-09-23T19:08:41.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:41 smithi016 bash[33624]: cephadm 2023-09-23T19:08:39.191108+0000 mgr.y (mgr.14454) 195 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d on smithi092 2023-09-23T19:08:41.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:41 smithi016 bash[33624]: audit 2023-09-23T19:08:40.734816+0000 mon.a (mon.0) 695 : audit [DBG] from='client.? 172.21.15.16:0/3255440326' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:41.289 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:41 smithi016 bash[27237]: cephadm 2023-09-23T19:08:39.191108+0000 mgr.y (mgr.14454) 195 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d on smithi092 2023-09-23T19:08:41.289 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:41 smithi016 bash[27237]: audit 2023-09-23T19:08:40.734816+0000 mon.a (mon.0) 695 : audit [DBG] from='client.? 172.21.15.16:0/3255440326' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:41.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:41 smithi092 bash[27556]: cephadm 2023-09-23T19:08:39.191108+0000 mgr.y (mgr.14454) 195 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d on smithi092 2023-09-23T19:08:41.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:41 smithi092 bash[27556]: audit 2023-09-23T19:08:40.734816+0000 mon.a (mon.0) 695 : audit [DBG] from='client.? 172.21.15.16:0/3255440326' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:41.814 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-09-23T19:08:41.811654+0000 min lat: 0.00145215 max lat: 0.276334 avg lat: 0.00975835 2023-09-23T19:08:41.815 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:08:41.815 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 40 16 65570 65554 102.418 70.875 0.0169932 0.00975835 2023-09-23T19:08:42.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:42 smithi016 bash[27237]: cluster 2023-09-23T19:08:40.967386+0000 mgr.y (mgr.14454) 196 : cluster [DBG] pgmap v170: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-09-23T19:08:42.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:42 smithi016 bash[33624]: cluster 2023-09-23T19:08:40.967386+0000 mgr.y (mgr.14454) 196 : cluster [DBG] pgmap v170: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-09-23T19:08:42.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:42 smithi092 bash[27556]: cluster 2023-09-23T19:08:40.967386+0000 mgr.y (mgr.14454) 196 : cluster [DBG] pgmap v170: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-09-23T19:08:42.814 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 41 16 66719 66703 101.672 71.8125 0.019275 0.00982808 2023-09-23T19:08:43.814 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 42 16 67828 67812 100.901 69.3125 0.0147179 0.00990483 2023-09-23T19:08:44.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:44 smithi016 bash[33624]: cluster 2023-09-23T19:08:42.968026+0000 mgr.y (mgr.14454) 197 : cluster [DBG] pgmap v171: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-09-23T19:08:44.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:44 smithi016 bash[33624]: audit 2023-09-23T19:08:43.414355+0000 mon.a (mon.0) 696 : audit [DBG] from='client.? 172.21.15.16:0/2135782625' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:44.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:44 smithi016 bash[27237]: cluster 2023-09-23T19:08:42.968026+0000 mgr.y (mgr.14454) 197 : cluster [DBG] pgmap v171: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-09-23T19:08:44.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:44 smithi016 bash[27237]: audit 2023-09-23T19:08:43.414355+0000 mon.a (mon.0) 696 : audit [DBG] from='client.? 172.21.15.16:0/2135782625' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:44.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:44 smithi092 bash[27556]: cluster 2023-09-23T19:08:42.968026+0000 mgr.y (mgr.14454) 197 : cluster [DBG] pgmap v171: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-09-23T19:08:44.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:44 smithi092 bash[27556]: audit 2023-09-23T19:08:43.414355+0000 mon.a (mon.0) 696 : audit [DBG] from='client.? 172.21.15.16:0/2135782625' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:44.814 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 43 15 68924 68909 100.149 68.5625 0.00726123 0.00997893 2023-09-23T19:08:45.814 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 44 16 70073 70057 99.5033 71.75 0.0178896 0.010044 2023-09-23T19:08:46.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:46 smithi016 bash[33624]: cluster 2023-09-23T19:08:44.968541+0000 mgr.y (mgr.14454) 198 : cluster [DBG] pgmap v172: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-23T19:08:46.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:46 smithi016 bash[27237]: cluster 2023-09-23T19:08:44.968541+0000 mgr.y (mgr.14454) 198 : cluster [DBG] pgmap v172: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-23T19:08:46.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:46 smithi092 bash[27556]: cluster 2023-09-23T19:08:44.968541+0000 mgr.y (mgr.14454) 198 : cluster [DBG] pgmap v172: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-23T19:08:46.815 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 45 16 71141 71125 98.7753 66.75 0.0113399 0.010118 2023-09-23T19:08:47.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:47 smithi016 bash[33624]: audit 2023-09-23T19:08:46.064221+0000 mon.c (mon.2) 109 : audit [DBG] from='client.? 172.21.15.16:0/1300771817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:47.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:47 smithi016 bash[27237]: audit 2023-09-23T19:08:46.064221+0000 mon.c (mon.2) 109 : audit [DBG] from='client.? 172.21.15.16:0/1300771817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:47.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:47 smithi092 bash[27556]: audit 2023-09-23T19:08:46.064221+0000 mon.c (mon.2) 109 : audit [DBG] from='client.? 172.21.15.16:0/1300771817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:47.815 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 46 16 72201 72185 98.0681 66.25 0.0113122 0.0101912 2023-09-23T19:08:48.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:48 smithi016 bash[33624]: cluster 2023-09-23T19:08:46.969275+0000 mgr.y (mgr.14454) 199 : cluster [DBG] pgmap v173: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2023-09-23T19:08:48.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:48 smithi016 bash[27237]: cluster 2023-09-23T19:08:46.969275+0000 mgr.y (mgr.14454) 199 : cluster [DBG] pgmap v173: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2023-09-23T19:08:48.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:48 smithi092 bash[27556]: cluster 2023-09-23T19:08:46.969275+0000 mgr.y (mgr.14454) 199 : cluster [DBG] pgmap v173: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2023-09-23T19:08:48.815 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 47 16 73238 73222 97.3604 64.8125 0.0139891 0.0102639 2023-09-23T19:08:49.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:49 smithi016 bash[27237]: audit 2023-09-23T19:08:48.709497+0000 mon.a (mon.0) 697 : audit [DBG] from='client.? 172.21.15.16:0/2503745093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:49.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:49 smithi016 bash[33624]: audit 2023-09-23T19:08:48.709497+0000 mon.a (mon.0) 697 : audit [DBG] from='client.? 172.21.15.16:0/2503745093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:49.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:49 smithi092 bash[27556]: audit 2023-09-23T19:08:48.709497+0000 mon.a (mon.0) 697 : audit [DBG] from='client.? 172.21.15.16:0/2503745093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:49.536 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:08:49 smithi092 bash[28593]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:08:49] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T19:08:49.815 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 48 16 74274 74258 96.6809 64.75 0.00923945 0.0103376 2023-09-23T19:08:50.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:50 smithi016 bash[33624]: cluster 2023-09-23T19:08:48.969714+0000 mgr.y (mgr.14454) 200 : cluster [DBG] pgmap v174: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 62 MiB/s wr, 985 op/s 2023-09-23T19:08:50.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:50 smithi016 bash[27237]: cluster 2023-09-23T19:08:48.969714+0000 mgr.y (mgr.14454) 200 : cluster [DBG] pgmap v174: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 62 MiB/s wr, 985 op/s 2023-09-23T19:08:50.536 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:08:50 smithi016 bash[27503]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:08:50] "GET /metrics HTTP/1.1" 200 194624 "" "Prometheus/2.33.4" 2023-09-23T19:08:50.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:50 smithi092 bash[27556]: cluster 2023-09-23T19:08:48.969714+0000 mgr.y (mgr.14454) 200 : cluster [DBG] pgmap v174: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 62 MiB/s wr, 985 op/s 2023-09-23T19:08:50.815 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 49 15 75304 75289 96.0227 64.4375 0.0200793 0.0104083 2023-09-23T19:08:51.815 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 50 16 76319 76303 95.3697 63.375 0.0132042 0.010479 2023-09-23T19:08:52.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:52 smithi016 bash[27237]: cluster 2023-09-23T19:08:50.970404+0000 mgr.y (mgr.14454) 201 : cluster [DBG] pgmap v175: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-09-23T19:08:52.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:52 smithi016 bash[27237]: audit 2023-09-23T19:08:51.325696+0000 mon.c (mon.2) 110 : audit [DBG] from='client.? 172.21.15.16:0/134472580' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:52.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:52 smithi016 bash[33624]: cluster 2023-09-23T19:08:50.970404+0000 mgr.y (mgr.14454) 201 : cluster [DBG] pgmap v175: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-09-23T19:08:52.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:52 smithi016 bash[33624]: audit 2023-09-23T19:08:51.325696+0000 mon.c (mon.2) 110 : audit [DBG] from='client.? 172.21.15.16:0/134472580' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:52.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:52 smithi092 bash[27556]: cluster 2023-09-23T19:08:50.970404+0000 mgr.y (mgr.14454) 201 : cluster [DBG] pgmap v175: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-09-23T19:08:52.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:52 smithi092 bash[27556]: audit 2023-09-23T19:08:51.325696+0000 mon.c (mon.2) 110 : audit [DBG] from='client.? 172.21.15.16:0/134472580' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:52.815 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 51 16 77335 77319 94.7447 63.5 0.00681714 0.0105485 2023-09-23T19:08:53.815 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 52 16 78324 78308 94.1112 61.8125 0.0167177 0.0106198 2023-09-23T19:08:54.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:54 smithi016 bash[27237]: cluster 2023-09-23T19:08:52.972705+0000 mgr.y (mgr.14454) 202 : cluster [DBG] pgmap v176: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-09-23T19:08:54.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:54 smithi016 bash[27237]: audit 2023-09-23T19:08:53.950460+0000 mon.c (mon.2) 111 : audit [DBG] from='client.? 172.21.15.16:0/596803151' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:54.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:54 smithi016 bash[33624]: cluster 2023-09-23T19:08:52.972705+0000 mgr.y (mgr.14454) 202 : cluster [DBG] pgmap v176: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-09-23T19:08:54.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:54 smithi016 bash[33624]: audit 2023-09-23T19:08:53.950460+0000 mon.c (mon.2) 111 : audit [DBG] from='client.? 172.21.15.16:0/596803151' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:54.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:54 smithi092 bash[27556]: cluster 2023-09-23T19:08:52.972705+0000 mgr.y (mgr.14454) 202 : cluster [DBG] pgmap v176: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-09-23T19:08:54.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:54 smithi092 bash[27556]: audit 2023-09-23T19:08:53.950460+0000 mon.c (mon.2) 111 : audit [DBG] from='client.? 172.21.15.16:0/596803151' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:54.815 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 53 16 79338 79322 93.5312 63.375 0.00918481 0.0106855 2023-09-23T19:08:55.817 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 54 16 80321 80305 92.9368 61.4375 0.0117084 0.0107538 2023-09-23T19:08:56.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:56 smithi016 bash[27237]: cluster 2023-09-23T19:08:54.973177+0000 mgr.y (mgr.14454) 203 : cluster [DBG] pgmap v177: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 57 MiB/s wr, 904 op/s 2023-09-23T19:08:56.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:56 smithi016 bash[33624]: cluster 2023-09-23T19:08:54.973177+0000 mgr.y (mgr.14454) 203 : cluster [DBG] pgmap v177: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 57 MiB/s wr, 904 op/s 2023-09-23T19:08:56.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:56 smithi092 bash[27556]: cluster 2023-09-23T19:08:54.973177+0000 mgr.y (mgr.14454) 203 : cluster [DBG] pgmap v177: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 57 MiB/s wr, 904 op/s 2023-09-23T19:08:56.815 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 55 16 81203 81187 92.2492 55.125 0.0443168 0.0108323 2023-09-23T19:08:57.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:57 smithi016 bash[27237]: audit 2023-09-23T19:08:56.546805+0000 mon.a (mon.0) 698 : audit [DBG] from='client.? 172.21.15.16:0/3571493318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:57.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:57 smithi016 bash[33624]: audit 2023-09-23T19:08:56.546805+0000 mon.a (mon.0) 698 : audit [DBG] from='client.? 172.21.15.16:0/3571493318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:57.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:57 smithi092 bash[27556]: audit 2023-09-23T19:08:56.546805+0000 mon.a (mon.0) 698 : audit [DBG] from='client.? 172.21.15.16:0/3571493318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:08:57.815 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 56 16 82085 82069 91.5862 55.125 0.0259807 0.0109129 2023-09-23T19:08:58.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:08:58 smithi016 bash[33624]: cluster 2023-09-23T19:08:56.973891+0000 mgr.y (mgr.14454) 204 : cluster [DBG] pgmap v178: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2023-09-23T19:08:58.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:08:58 smithi016 bash[27237]: cluster 2023-09-23T19:08:56.973891+0000 mgr.y (mgr.14454) 204 : cluster [DBG] pgmap v178: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2023-09-23T19:08:58.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:08:58 smithi092 bash[27556]: cluster 2023-09-23T19:08:56.973891+0000 mgr.y (mgr.14454) 204 : cluster [DBG] pgmap v178: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2023-09-23T19:08:58.815 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 57 16 82950 82934 90.9278 54.0625 0.0151613 0.010992 2023-09-23T19:08:59.536 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:08:59 smithi092 bash[28593]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:08:59] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T19:08:59.815 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 58 16 83823 83807 90.3007 54.5625 0.0162283 0.011068 2023-09-23T19:09:00.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:00 smithi016 bash[33624]: cluster 2023-09-23T19:08:58.977889+0000 mgr.y (mgr.14454) 205 : cluster [DBG] pgmap v179: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 53 MiB/s wr, 852 op/s 2023-09-23T19:09:00.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:00 smithi016 bash[33624]: audit 2023-09-23T19:08:59.167669+0000 mon.a (mon.0) 699 : audit [DBG] from='client.? 172.21.15.16:0/440404333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:00.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:00 smithi016 bash[27237]: cluster 2023-09-23T19:08:58.977889+0000 mgr.y (mgr.14454) 205 : cluster [DBG] pgmap v179: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 53 MiB/s wr, 852 op/s 2023-09-23T19:09:00.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:00 smithi016 bash[27237]: audit 2023-09-23T19:08:59.167669+0000 mon.a (mon.0) 699 : audit [DBG] from='client.? 172.21.15.16:0/440404333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:00.536 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:09:00 smithi016 bash[27503]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:09:00] "GET /metrics HTTP/1.1" 200 194656 "" "Prometheus/2.33.4" 2023-09-23T19:09:00.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:00 smithi092 bash[27556]: cluster 2023-09-23T19:08:58.977889+0000 mgr.y (mgr.14454) 205 : cluster [DBG] pgmap v179: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 53 MiB/s wr, 852 op/s 2023-09-23T19:09:00.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:00 smithi092 bash[27556]: audit 2023-09-23T19:08:59.167669+0000 mon.a (mon.0) 699 : audit [DBG] from='client.? 172.21.15.16:0/440404333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:00.816 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 59 16 84673 84657 89.6705 53.125 0.00740732 0.0111449 2023-09-23T19:09:01.816 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-09-23T19:09:01.813531+0000 min lat: 0.00145215 max lat: 0.276334 avg lat: 0.0112204 2023-09-23T19:09:01.816 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:09:01.816 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 60 16 85528 85512 89.0666 53.4375 0.0132609 0.0112204 2023-09-23T19:09:02.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:02 smithi016 bash[33624]: cluster 2023-09-23T19:09:00.978568+0000 mgr.y (mgr.14454) 206 : cluster [DBG] pgmap v180: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T19:09:02.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:02 smithi016 bash[33624]: audit 2023-09-23T19:09:01.767678+0000 mon.a (mon.0) 700 : audit [DBG] from='client.? 172.21.15.16:0/681543295' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:02.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:02 smithi016 bash[27237]: cluster 2023-09-23T19:09:00.978568+0000 mgr.y (mgr.14454) 206 : cluster [DBG] pgmap v180: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T19:09:02.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:02 smithi016 bash[27237]: audit 2023-09-23T19:09:01.767678+0000 mon.a (mon.0) 700 : audit [DBG] from='client.? 172.21.15.16:0/681543295' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:02.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:02 smithi092 bash[27556]: cluster 2023-09-23T19:09:00.978568+0000 mgr.y (mgr.14454) 206 : cluster [DBG] pgmap v180: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T19:09:02.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:02 smithi092 bash[27556]: audit 2023-09-23T19:09:01.767678+0000 mon.a (mon.0) 700 : audit [DBG] from='client.? 172.21.15.16:0/681543295' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:02.816 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 61 16 86401 86385 88.5008 54.5625 0.0287401 0.0112922 2023-09-23T19:09:03.816 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 62 16 87246 87230 87.9251 52.8125 0.00421523 0.0113667 2023-09-23T19:09:04.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:04 smithi016 bash[33624]: cluster 2023-09-23T19:09:02.979282+0000 mgr.y (mgr.14454) 207 : cluster [DBG] pgmap v181: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-09-23T19:09:04.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:04 smithi016 bash[27237]: cluster 2023-09-23T19:09:02.979282+0000 mgr.y (mgr.14454) 207 : cluster [DBG] pgmap v181: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-09-23T19:09:04.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:04 smithi092 bash[27556]: cluster 2023-09-23T19:09:02.979282+0000 mgr.y (mgr.14454) 207 : cluster [DBG] pgmap v181: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-09-23T19:09:04.816 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 63 16 88041 88025 87.3181 49.6875 0.0163389 0.011445 2023-09-23T19:09:05.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:05 smithi016 bash[27237]: audit 2023-09-23T19:09:04.358845+0000 mon.a (mon.0) 701 : audit [DBG] from='client.? 172.21.15.16:0/1233083323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:05.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:05 smithi016 bash[33624]: audit 2023-09-23T19:09:04.358845+0000 mon.a (mon.0) 701 : audit [DBG] from='client.? 172.21.15.16:0/1233083323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:05.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:05 smithi092 bash[27556]: audit 2023-09-23T19:09:04.358845+0000 mon.a (mon.0) 701 : audit [DBG] from='client.? 172.21.15.16:0/1233083323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:05.816 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 64 16 88860 88844 86.7534 51.1875 0.0183117 0.0115195 2023-09-23T19:09:06.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:06 smithi016 bash[27237]: cluster 2023-09-23T19:09:04.979738+0000 mgr.y (mgr.14454) 208 : cluster [DBG] pgmap v182: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 49 MiB/s wr, 778 op/s 2023-09-23T19:09:06.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:06 smithi016 bash[33624]: cluster 2023-09-23T19:09:04.979738+0000 mgr.y (mgr.14454) 208 : cluster [DBG] pgmap v182: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 49 MiB/s wr, 778 op/s 2023-09-23T19:09:06.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:06 smithi092 bash[27556]: cluster 2023-09-23T19:09:04.979738+0000 mgr.y (mgr.14454) 208 : cluster [DBG] pgmap v182: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 49 MiB/s wr, 778 op/s 2023-09-23T19:09:06.816 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 65 16 89685 89669 86.2119 51.5625 0.0460386 0.0115927 2023-09-23T19:09:07.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:07 smithi016 bash[27237]: audit 2023-09-23T19:09:06.945153+0000 mon.c (mon.2) 112 : audit [DBG] from='client.? 172.21.15.16:0/2944802906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:07.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:07 smithi016 bash[33624]: audit 2023-09-23T19:09:06.945153+0000 mon.c (mon.2) 112 : audit [DBG] from='client.? 172.21.15.16:0/2944802906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:07.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:07 smithi092 bash[27556]: audit 2023-09-23T19:09:06.945153+0000 mon.c (mon.2) 112 : audit [DBG] from='client.? 172.21.15.16:0/2944802906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:07.816 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 66 16 90512 90496 85.6888 51.6875 0.0153554 0.0116642 2023-09-23T19:09:08.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:08 smithi016 bash[27237]: cluster 2023-09-23T19:09:06.980551+0000 mgr.y (mgr.14454) 209 : cluster [DBG] pgmap v183: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 70 MiB/s wr, 1.13k op/s 2023-09-23T19:09:08.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:08 smithi016 bash[33624]: cluster 2023-09-23T19:09:06.980551+0000 mgr.y (mgr.14454) 209 : cluster [DBG] pgmap v183: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 70 MiB/s wr, 1.13k op/s 2023-09-23T19:09:08.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:08 smithi092 bash[27556]: cluster 2023-09-23T19:09:06.980551+0000 mgr.y (mgr.14454) 209 : cluster [DBG] pgmap v183: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 70 MiB/s wr, 1.13k op/s 2023-09-23T19:09:08.816 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 67 16 91311 91295 85.1551 49.9375 0.0364999 0.0117368 2023-09-23T19:09:09.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:09 smithi016 bash[27237]: cluster 2023-09-23T19:09:08.981038+0000 mgr.y (mgr.14454) 210 : cluster [DBG] pgmap v184: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 44 MiB/s wr, 710 op/s 2023-09-23T19:09:09.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:09 smithi016 bash[33624]: cluster 2023-09-23T19:09:08.981038+0000 mgr.y (mgr.14454) 210 : cluster [DBG] pgmap v184: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 44 MiB/s wr, 710 op/s 2023-09-23T19:09:09.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:09 smithi092 bash[27556]: cluster 2023-09-23T19:09:08.981038+0000 mgr.y (mgr.14454) 210 : cluster [DBG] pgmap v184: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 44 MiB/s wr, 710 op/s 2023-09-23T19:09:09.536 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:09 smithi092 bash[28593]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:09:09] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T19:09:09.816 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 68 16 92139 92123 84.6638 51.75 0.0106795 0.0118043 2023-09-23T19:09:10.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:10 smithi016 bash[33624]: audit 2023-09-23T19:09:09.567663+0000 mon.c (mon.2) 113 : audit [DBG] from='client.? 172.21.15.16:0/3824475892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:10.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:10 smithi016 bash[27237]: audit 2023-09-23T19:09:09.567663+0000 mon.c (mon.2) 113 : audit [DBG] from='client.? 172.21.15.16:0/3824475892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:10.535 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:09:10 smithi016 bash[27503]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:09:10] "GET /metrics HTTP/1.1" 200 194709 "" "Prometheus/2.33.4" 2023-09-23T19:09:10.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:10 smithi092 bash[27556]: audit 2023-09-23T19:09:09.567663+0000 mon.c (mon.2) 113 : audit [DBG] from='client.? 172.21.15.16:0/3824475892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:10.817 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 69 16 93068 93052 84.2781 58.0625 0.00890559 0.0118601 2023-09-23T19:09:11.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:11 smithi016 bash[33624]: cluster 2023-09-23T19:09:10.981723+0000 mgr.y (mgr.14454) 211 : cluster [DBG] pgmap v185: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 824 op/s 2023-09-23T19:09:11.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:11 smithi016 bash[27237]: cluster 2023-09-23T19:09:10.981723+0000 mgr.y (mgr.14454) 211 : cluster [DBG] pgmap v185: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 824 op/s 2023-09-23T19:09:11.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:11 smithi092 bash[27556]: cluster 2023-09-23T19:09:10.981723+0000 mgr.y (mgr.14454) 211 : cluster [DBG] pgmap v185: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 824 op/s 2023-09-23T19:09:11.817 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 70 16 94053 94037 83.9536 61.5625 0.012496 0.0119036 2023-09-23T19:09:12.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:12 smithi016 bash[27237]: audit 2023-09-23T19:09:12.234890+0000 mon.c (mon.2) 114 : audit [DBG] from='client.? 172.21.15.16:0/3813635442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:12.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:12 smithi016 bash[33624]: audit 2023-09-23T19:09:12.234890+0000 mon.c (mon.2) 114 : audit [DBG] from='client.? 172.21.15.16:0/3813635442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:12.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:12 smithi092 bash[27556]: audit 2023-09-23T19:09:12.234890+0000 mon.c (mon.2) 114 : audit [DBG] from='client.? 172.21.15.16:0/3813635442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:12.817 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 71 16 95031 95015 83.6319 61.125 0.0203958 0.0119515 2023-09-23T19:09:13.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:13 smithi016 bash[27237]: cluster 2023-09-23T19:09:12.982343+0000 mgr.y (mgr.14454) 212 : cluster [DBG] pgmap v186: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 57 MiB/s wr, 909 op/s 2023-09-23T19:09:13.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:13 smithi016 bash[33624]: cluster 2023-09-23T19:09:12.982343+0000 mgr.y (mgr.14454) 212 : cluster [DBG] pgmap v186: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 57 MiB/s wr, 909 op/s 2023-09-23T19:09:13.786 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:13 smithi092 bash[27556]: cluster 2023-09-23T19:09:12.982343+0000 mgr.y (mgr.14454) 212 : cluster [DBG] pgmap v186: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 57 MiB/s wr, 909 op/s 2023-09-23T19:09:13.817 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 72 16 96021 96005 83.3297 61.875 0.0166142 0.0119948 2023-09-23T19:09:14.817 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 73 16 96991 96975 83.0186 60.625 0.0157858 0.0120394 2023-09-23T19:09:15.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:14 smithi016 bash[33624]: audit 2023-09-23T19:09:14.909388+0000 mon.a (mon.0) 702 : audit [DBG] from='client.? 172.21.15.16:0/3595647453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:15.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:14 smithi016 bash[27237]: audit 2023-09-23T19:09:14.909388+0000 mon.a (mon.0) 702 : audit [DBG] from='client.? 172.21.15.16:0/3595647453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:15.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:14 smithi092 bash[27556]: audit 2023-09-23T19:09:14.909388+0000 mon.a (mon.0) 702 : audit [DBG] from='client.? 172.21.15.16:0/3595647453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:15.817 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 74 16 97943 97927 82.7007 59.5 0.0210591 0.0120859 2023-09-23T19:09:16.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:15 smithi016 bash[27237]: cluster 2023-09-23T19:09:14.982780+0000 mgr.y (mgr.14454) 213 : cluster [DBG] pgmap v187: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 44 MiB/s wr, 702 op/s 2023-09-23T19:09:16.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:15 smithi016 bash[33624]: cluster 2023-09-23T19:09:14.982780+0000 mgr.y (mgr.14454) 213 : cluster [DBG] pgmap v187: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 44 MiB/s wr, 702 op/s 2023-09-23T19:09:16.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:15 smithi092 bash[27556]: cluster 2023-09-23T19:09:14.982780+0000 mgr.y (mgr.14454) 213 : cluster [DBG] pgmap v187: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 44 MiB/s wr, 702 op/s 2023-09-23T19:09:16.817 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 75 16 98879 98863 82.3779 58.5 0.0241323 0.0121329 2023-09-23T19:09:17.817 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 76 16 99777 99761 82.0324 56.125 0.0197339 0.0121848 2023-09-23T19:09:18.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:17 smithi016 bash[33624]: cluster 2023-09-23T19:09:16.983571+0000 mgr.y (mgr.14454) 214 : cluster [DBG] pgmap v188: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-09-23T19:09:18.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:17 smithi016 bash[33624]: audit 2023-09-23T19:09:17.538081+0000 mon.b (mon.1) 33 : audit [DBG] from='client.? 172.21.15.16:0/2294254478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:18.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:17 smithi016 bash[27237]: cluster 2023-09-23T19:09:16.983571+0000 mgr.y (mgr.14454) 214 : cluster [DBG] pgmap v188: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-09-23T19:09:18.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:17 smithi016 bash[27237]: audit 2023-09-23T19:09:17.538081+0000 mon.b (mon.1) 33 : audit [DBG] from='client.? 172.21.15.16:0/2294254478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:18.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:17 smithi092 bash[27556]: cluster 2023-09-23T19:09:16.983571+0000 mgr.y (mgr.14454) 214 : cluster [DBG] pgmap v188: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-09-23T19:09:18.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:17 smithi092 bash[27556]: audit 2023-09-23T19:09:17.538081+0000 mon.b (mon.1) 33 : audit [DBG] from='client.? 172.21.15.16:0/2294254478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:18.817 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 77 16 100729 100713 81.7397 59.5 0.0106784 0.0122281 2023-09-23T19:09:19.536 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:19 smithi092 bash[28593]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:09:19] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T19:09:19.817 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 78 16 101667 101651 81.4433 58.625 0.00833712 0.0122718 2023-09-23T19:09:20.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:20 smithi016 bash[27237]: cluster 2023-09-23T19:09:18.983961+0000 mgr.y (mgr.14454) 215 : cluster [DBG] pgmap v189: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 47 MiB/s wr, 759 op/s 2023-09-23T19:09:20.286 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:09:20 smithi016 bash[27503]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:09:20] "GET /metrics HTTP/1.1" 200 194709 "" "Prometheus/2.33.4" 2023-09-23T19:09:20.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:20 smithi016 bash[33624]: cluster 2023-09-23T19:09:18.983961+0000 mgr.y (mgr.14454) 215 : cluster [DBG] pgmap v189: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 47 MiB/s wr, 759 op/s 2023-09-23T19:09:20.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:20 smithi092 bash[27556]: cluster 2023-09-23T19:09:18.983961+0000 mgr.y (mgr.14454) 215 : cluster [DBG] pgmap v189: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 47 MiB/s wr, 759 op/s 2023-09-23T19:09:20.817 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 79 16 102568 102552 81.1251 56.3125 0.0146121 0.0123204 2023-09-23T19:09:21.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:21 smithi016 bash[33624]: audit 2023-09-23T19:09:20.191421+0000 mon.b (mon.1) 34 : audit [DBG] from='client.? 172.21.15.16:0/2514068997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:21.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:21 smithi016 bash[27237]: audit 2023-09-23T19:09:20.191421+0000 mon.b (mon.1) 34 : audit [DBG] from='client.? 172.21.15.16:0/2514068997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:21.285 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:21 smithi092 bash[27556]: audit 2023-09-23T19:09:20.191421+0000 mon.b (mon.1) 34 : audit [DBG] from='client.? 172.21.15.16:0/2514068997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:21.817 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-09-23T19:09:21.815528+0000 min lat: 0.00145215 max lat: 0.276334 avg lat: 0.0123673 2023-09-23T19:09:21.818 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:09:21.818 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 80 16 103473 103457 80.818 56.5625 0.0137451 0.0123673 2023-09-23T19:09:22.018 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:22 smithi092 bash[27556]: cluster 2023-09-23T19:09:20.984523+0000 mgr.y (mgr.14454) 216 : cluster [DBG] pgmap v190: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 829 op/s 2023-09-23T19:09:22.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:22 smithi016 bash[27237]: cluster 2023-09-23T19:09:20.984523+0000 mgr.y (mgr.14454) 216 : cluster [DBG] pgmap v190: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 829 op/s 2023-09-23T19:09:22.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:22 smithi016 bash[33624]: cluster 2023-09-23T19:09:20.984523+0000 mgr.y (mgr.14454) 216 : cluster [DBG] pgmap v190: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 829 op/s 2023-09-23T19:09:22.818 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 81 15 104332 104317 80.4838 53.75 0.044083 0.0124189 2023-09-23T19:09:23.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:23 smithi016 bash[33624]: audit 2023-09-23T19:09:22.826653+0000 mon.c (mon.2) 115 : audit [DBG] from='client.? 172.21.15.16:0/479998087' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:23.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:23 smithi016 bash[27237]: audit 2023-09-23T19:09:22.826653+0000 mon.c (mon.2) 115 : audit [DBG] from='client.? 172.21.15.16:0/479998087' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:23.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:23 smithi092 bash[27556]: audit 2023-09-23T19:09:22.826653+0000 mon.c (mon.2) 115 : audit [DBG] from='client.? 172.21.15.16:0/479998087' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:23.818 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 82 16 105201 105185 80.1638 54.25 0.0144519 0.0124681 2023-09-23T19:09:24.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:24 smithi016 bash[33624]: cluster 2023-09-23T19:09:22.985083+0000 mgr.y (mgr.14454) 217 : cluster [DBG] pgmap v191: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-09-23T19:09:24.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:24 smithi016 bash[33624]: audit 2023-09-23T19:09:23.632668+0000 mon.a (mon.0) 703 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:09:24.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:24 smithi016 bash[33624]: audit 2023-09-23T19:09:23.635039+0000 mon.c (mon.2) 116 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T19:09:24.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:24 smithi016 bash[33624]: audit 2023-09-23T19:09:23.635604+0000 mon.a (mon.0) 704 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T19:09:24.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:24 smithi016 bash[33624]: audit 2023-09-23T19:09:23.641040+0000 mon.c (mon.2) 117 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T19:09:24.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:24 smithi016 bash[33624]: audit 2023-09-23T19:09:23.642619+0000 mon.c (mon.2) 118 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:09:24.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:24 smithi016 bash[27237]: cluster 2023-09-23T19:09:22.985083+0000 mgr.y (mgr.14454) 217 : cluster [DBG] pgmap v191: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-09-23T19:09:24.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:24 smithi016 bash[27237]: audit 2023-09-23T19:09:23.632668+0000 mon.a (mon.0) 703 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:09:24.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:24 smithi016 bash[27237]: audit 2023-09-23T19:09:23.635039+0000 mon.c (mon.2) 116 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T19:09:24.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:24 smithi016 bash[27237]: audit 2023-09-23T19:09:23.635604+0000 mon.a (mon.0) 704 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T19:09:24.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:24 smithi016 bash[27237]: audit 2023-09-23T19:09:23.641040+0000 mon.c (mon.2) 117 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T19:09:24.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:24 smithi016 bash[27237]: audit 2023-09-23T19:09:23.642619+0000 mon.c (mon.2) 118 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:09:24.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:24 smithi092 bash[27556]: cluster 2023-09-23T19:09:22.985083+0000 mgr.y (mgr.14454) 217 : cluster [DBG] pgmap v191: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-09-23T19:09:24.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:24 smithi092 bash[27556]: audit 2023-09-23T19:09:23.632668+0000 mon.a (mon.0) 703 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:09:24.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:24 smithi092 bash[27556]: audit 2023-09-23T19:09:23.635039+0000 mon.c (mon.2) 116 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T19:09:24.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:24 smithi092 bash[27556]: audit 2023-09-23T19:09:23.635604+0000 mon.a (mon.0) 704 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T19:09:24.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:24 smithi092 bash[27556]: audit 2023-09-23T19:09:23.641040+0000 mon.c (mon.2) 117 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T19:09:24.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:24 smithi092 bash[27556]: audit 2023-09-23T19:09:23.642619+0000 mon.c (mon.2) 118 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:09:24.818 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 83 16 106085 106069 79.8636 55.25 0.0132211 0.0125155 2023-09-23T19:09:25.178 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:25 smithi092 bash[27556]: cephadm 2023-09-23T19:09:23.626127+0000 mgr.y (mgr.14454) 218 : cephadm [INF] Upgrade: Updating mgr.x 2023-09-23T19:09:25.178 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:25 smithi092 bash[27556]: cephadm 2023-09-23T19:09:23.643768+0000 mgr.y (mgr.14454) 219 : cephadm [INF] Deploying daemon mgr.x on smithi092 2023-09-23T19:09:25.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:25 smithi016 bash[27237]: cephadm 2023-09-23T19:09:23.626127+0000 mgr.y (mgr.14454) 218 : cephadm [INF] Upgrade: Updating mgr.x 2023-09-23T19:09:25.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:25 smithi016 bash[27237]: cephadm 2023-09-23T19:09:23.643768+0000 mgr.y (mgr.14454) 219 : cephadm [INF] Deploying daemon mgr.x on smithi092 2023-09-23T19:09:25.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:25 smithi016 bash[33624]: cephadm 2023-09-23T19:09:23.626127+0000 mgr.y (mgr.14454) 218 : cephadm [INF] Upgrade: Updating mgr.x 2023-09-23T19:09:25.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:25 smithi016 bash[33624]: cephadm 2023-09-23T19:09:23.643768+0000 mgr.y (mgr.14454) 219 : cephadm [INF] Deploying daemon mgr.x on smithi092 2023-09-23T19:09:25.818 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 84 16 106930 106914 79.5414 52.8125 0.0127429 0.012565 2023-09-23T19:09:26.347 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:26 smithi092 bash[27556]: cluster 2023-09-23T19:09:24.985658+0000 mgr.y (mgr.14454) 220 : cluster [DBG] pgmap v192: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 49 MiB/s wr, 790 op/s 2023-09-23T19:09:26.347 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:26 smithi092 bash[27556]: audit 2023-09-23T19:09:25.472063+0000 mon.c (mon.2) 119 : audit [DBG] from='client.? 172.21.15.16:0/3334444745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:26.347 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:26 smithi092 systemd[1]: Stopping Ceph mgr.x for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7... 2023-09-23T19:09:26.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:26 smithi016 bash[27237]: cluster 2023-09-23T19:09:24.985658+0000 mgr.y (mgr.14454) 220 : cluster [DBG] pgmap v192: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 49 MiB/s wr, 790 op/s 2023-09-23T19:09:26.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:26 smithi016 bash[27237]: audit 2023-09-23T19:09:25.472063+0000 mon.c (mon.2) 119 : audit [DBG] from='client.? 172.21.15.16:0/3334444745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:26.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:26 smithi016 bash[33624]: cluster 2023-09-23T19:09:24.985658+0000 mgr.y (mgr.14454) 220 : cluster [DBG] pgmap v192: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 49 MiB/s wr, 790 op/s 2023-09-23T19:09:26.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:26 smithi016 bash[33624]: audit 2023-09-23T19:09:25.472063+0000 mon.c (mon.2) 119 : audit [DBG] from='client.? 172.21.15.16:0/3334444745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:26.741 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:26 smithi092 bash[50341]: Error response from daemon: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-mgr.x 2023-09-23T19:09:26.818 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 85 16 107762 107746 79.2174 52 0.0136421 0.0126177 2023-09-23T19:09:27.460 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:27 smithi092 bash[50370]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-mgr-x 2023-09-23T19:09:27.460 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:27 smithi092 bash[50441]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-mgr-x 2023-09-23T19:09:27.460 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:27 smithi092 systemd[1]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-09-23T19:09:27.460 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:27 smithi092 systemd[1]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@mgr.x.service: Failed with result 'exit-code'. 2023-09-23T19:09:27.460 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:27 smithi092 systemd[1]: Stopped Ceph mgr.x for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7. 2023-09-23T19:09:27.769 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:27 smithi092 systemd[1]: Started Ceph mgr.x for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7. 2023-09-23T19:09:27.818 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 86 16 108589 108573 78.8972 51.6875 0.0290316 0.0126688 2023-09-23T19:09:28.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:28 smithi016 bash[27237]: cluster 2023-09-23T19:09:26.986642+0000 mgr.y (mgr.14454) 221 : cluster [DBG] pgmap v193: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T19:09:28.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:28 smithi016 bash[27237]: audit 2023-09-23T19:09:27.813800+0000 mon.a (mon.0) 705 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:09:28.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:28 smithi016 bash[27237]: audit 2023-09-23T19:09:27.817218+0000 mon.c (mon.2) 120 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:09:28.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:28 smithi016 bash[33624]: cluster 2023-09-23T19:09:26.986642+0000 mgr.y (mgr.14454) 221 : cluster [DBG] pgmap v193: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T19:09:28.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:28 smithi016 bash[33624]: audit 2023-09-23T19:09:27.813800+0000 mon.a (mon.0) 705 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:09:28.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:28 smithi016 bash[33624]: audit 2023-09-23T19:09:27.817218+0000 mon.c (mon.2) 120 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:09:28.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:28 smithi092 bash[27556]: cluster 2023-09-23T19:09:26.986642+0000 mgr.y (mgr.14454) 221 : cluster [DBG] pgmap v193: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T19:09:28.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:28 smithi092 bash[27556]: audit 2023-09-23T19:09:27.813800+0000 mon.a (mon.0) 705 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:09:28.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:28 smithi092 bash[27556]: audit 2023-09-23T19:09:27.817218+0000 mon.c (mon.2) 120 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:09:28.818 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 87 16 109407 109391 78.5779 51.125 0.00894798 0.0127201 2023-09-23T19:09:29.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:29 smithi092 bash[27556]: audit 2023-09-23T19:09:28.182746+0000 mon.a (mon.0) 706 : audit [DBG] from='client.? 172.21.15.16:0/1944062468' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:29.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:29 smithi016 bash[27237]: audit 2023-09-23T19:09:28.182746+0000 mon.a (mon.0) 706 : audit [DBG] from='client.? 172.21.15.16:0/1944062468' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:29.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:29 smithi016 bash[33624]: audit 2023-09-23T19:09:28.182746+0000 mon.a (mon.0) 706 : audit [DBG] from='client.? 172.21.15.16:0/1944062468' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:29.786 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:29 smithi092 bash[50491]: debug 2023-09-23T19:09:29.320+0000 7fc204b3a200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T19:09:29.818 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 88 16 110201 110185 78.2488 49.625 0.0109802 0.0127742 2023-09-23T19:09:30.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:30 smithi016 bash[27237]: cluster 2023-09-23T19:09:28.987147+0000 mgr.y (mgr.14454) 222 : cluster [DBG] pgmap v194: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 47 MiB/s wr, 748 op/s 2023-09-23T19:09:30.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:30 smithi016 bash[33624]: cluster 2023-09-23T19:09:28.987147+0000 mgr.y (mgr.14454) 222 : cluster [DBG] pgmap v194: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 47 MiB/s wr, 748 op/s 2023-09-23T19:09:30.535 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:09:30 smithi016 bash[27503]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:09:30] "GET /metrics HTTP/1.1" 200 194739 "" "Prometheus/2.33.4" 2023-09-23T19:09:30.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:30 smithi092 bash[27556]: cluster 2023-09-23T19:09:28.987147+0000 mgr.y (mgr.14454) 222 : cluster [DBG] pgmap v194: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 47 MiB/s wr, 748 op/s 2023-09-23T19:09:30.818 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 89 16 110965 110949 77.9061 47.75 0.0185365 0.0128301 2023-09-23T19:09:30.977 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:30 smithi092 bash[50491]: debug 2023-09-23T19:09:30.684+0000 7fc204b3a200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T19:09:30.978 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:30 smithi092 bash[50491]: debug 2023-09-23T19:09:30.828+0000 7fc204b3a200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T19:09:31.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:31 smithi092 bash[27556]: audit 2023-09-23T19:09:30.798208+0000 mon.c (mon.2) 121 : audit [DBG] from='client.? 172.21.15.16:0/1126811031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:31.286 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:30 smithi092 bash[50491]: debug 2023-09-23T19:09:30.980+0000 7fc204b3a200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T19:09:31.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:31 smithi016 bash[33624]: audit 2023-09-23T19:09:30.798208+0000 mon.c (mon.2) 121 : audit [DBG] from='client.? 172.21.15.16:0/1126811031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:31.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:31 smithi016 bash[27237]: audit 2023-09-23T19:09:30.798208+0000 mon.c (mon.2) 121 : audit [DBG] from='client.? 172.21.15.16:0/1126811031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:31.786 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:31 smithi092 bash[50491]: debug 2023-09-23T19:09:31.364+0000 7fc204b3a200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T19:09:31.819 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 90 4 111768 111764 77.6064 50.9375 0.0158704 0.0128798 2023-09-23T19:09:32.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:32 smithi092 bash[27556]: cluster 2023-09-23T19:09:30.990174+0000 mgr.y (mgr.14454) 223 : cluster [DBG] pgmap v195: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 49 MiB/s wr, 788 op/s 2023-09-23T19:09:32.286 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:31 smithi092 bash[50491]: debug 2023-09-23T19:09:31.876+0000 7fc204b3a200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T19:09:32.286 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:32 smithi092 bash[50491]: debug 2023-09-23T19:09:32.052+0000 7fc204b3a200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T19:09:32.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:32 smithi016 bash[27237]: cluster 2023-09-23T19:09:30.990174+0000 mgr.y (mgr.14454) 223 : cluster [DBG] pgmap v195: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 49 MiB/s wr, 788 op/s 2023-09-23T19:09:32.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:32 smithi016 bash[33624]: cluster 2023-09-23T19:09:30.990174+0000 mgr.y (mgr.14454) 223 : cluster [DBG] pgmap v195: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 49 MiB/s wr, 788 op/s 2023-09-23T19:09:32.786 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:32 smithi092 bash[50491]: debug 2023-09-23T19:09:32.288+0000 7fc204b3a200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T19:09:32.819 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Total time run: 90.0104 2023-09-23T19:09:32.819 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Total writes made: 111768 2023-09-23T19:09:32.819 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Write size: 65536 2023-09-23T19:09:32.819 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Object size: 65536 2023-09-23T19:09:32.819 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Bandwidth (MB/sec): 77.6077 2023-09-23T19:09:32.819 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Stddev Bandwidth: 27.9298 2023-09-23T19:09:32.819 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Max bandwidth (MB/sec): 177.438 2023-09-23T19:09:32.819 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Min bandwidth (MB/sec): 47.75 2023-09-23T19:09:32.819 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Average IOPS: 1241 2023-09-23T19:09:32.820 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Stddev IOPS: 446.877 2023-09-23T19:09:32.820 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Max IOPS: 2839 2023-09-23T19:09:32.820 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Min IOPS: 764 2023-09-23T19:09:32.820 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Average Latency(s): 0.01288 2023-09-23T19:09:32.820 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Stddev Latency(s): 0.00870612 2023-09-23T19:09:32.820 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Max latency(s): 0.276334 2023-09-23T19:09:32.820 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Min latency(s): 0.00145215 2023-09-23T19:09:32.826 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T19:09:33.900 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:33 smithi092 bash[50491]: debug 2023-09-23T19:09:33.568+0000 7fc204b3a200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T19:09:33.901 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:33 smithi092 bash[50491]: debug 2023-09-23T19:09:33.728+0000 7fc204b3a200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T19:09:33.901 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:33 smithi092 bash[50491]: debug 2023-09-23T19:09:33.900+0000 7fc204b3a200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T19:09:34.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:34 smithi092 bash[27556]: cluster 2023-09-23T19:09:32.990895+0000 mgr.y (mgr.14454) 224 : cluster [DBG] pgmap v196: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T19:09:34.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:34 smithi092 bash[27556]: audit 2023-09-23T19:09:33.555362+0000 mon.a (mon.0) 707 : audit [DBG] from='client.? 172.21.15.16:0/3330073829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:34.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:34 smithi092 bash[27556]: audit 2023-09-23T19:09:33.817398+0000 mon.a (mon.0) 708 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:09:34.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:34 smithi016 bash[33624]: cluster 2023-09-23T19:09:32.990895+0000 mgr.y (mgr.14454) 224 : cluster [DBG] pgmap v196: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T19:09:34.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:34 smithi016 bash[33624]: audit 2023-09-23T19:09:33.555362+0000 mon.a (mon.0) 707 : audit [DBG] from='client.? 172.21.15.16:0/3330073829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:34.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:34 smithi016 bash[33624]: audit 2023-09-23T19:09:33.817398+0000 mon.a (mon.0) 708 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:09:34.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:34 smithi016 bash[27237]: cluster 2023-09-23T19:09:32.990895+0000 mgr.y (mgr.14454) 224 : cluster [DBG] pgmap v196: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T19:09:34.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:34 smithi016 bash[27237]: audit 2023-09-23T19:09:33.555362+0000 mon.a (mon.0) 707 : audit [DBG] from='client.? 172.21.15.16:0/3330073829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:34.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:34 smithi016 bash[27237]: audit 2023-09-23T19:09:33.817398+0000 mon.a (mon.0) 708 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:09:34.969 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:34 smithi092 bash[50491]: debug 2023-09-23T19:09:34.628+0000 7fc204b3a200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T19:09:34.969 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:34 smithi092 bash[50491]: debug 2023-09-23T19:09:34.968+0000 7fc204b3a200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T19:09:36.035 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:09:35 smithi016 bash[27503]: debug 2023-09-23T19:09:35.772+0000 7f0ead189700 -1 mgr handle_mgr_map I was active but no longer am 2023-09-23T19:09:36.035 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:09:35 smithi016 bash[27503]: ignoring --setuser ceph since I am not root 2023-09-23T19:09:36.035 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:09:35 smithi016 bash[27503]: ignoring --setgroup ceph since I am not root 2023-09-23T19:09:36.035 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:35 smithi092 bash[50491]: debug 2023-09-23T19:09:35.684+0000 7fc204b3a200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T19:09:36.364 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:36 smithi092 bash[27556]: cluster 2023-09-23T19:09:34.991356+0000 mgr.y (mgr.14454) 225 : cluster [DBG] pgmap v197: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 44 MiB/s wr, 698 op/s 2023-09-23T19:09:36.364 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:36 smithi092 bash[27556]: audit 2023-09-23T19:09:35.386122+0000 mon.a (mon.0) 709 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:09:36.364 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:36 smithi092 bash[27556]: audit 2023-09-23T19:09:35.726689+0000 mon.c (mon.2) 122 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:09:36.364 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:36 smithi092 bash[27556]: audit 2023-09-23T19:09:35.728070+0000 mon.c (mon.2) 123 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:09:36.364 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:36 smithi092 bash[27556]: audit 2023-09-23T19:09:35.735926+0000 mon.a (mon.0) 710 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:09:36.365 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:36 smithi092 bash[27556]: audit 2023-09-23T19:09:35.745866+0000 mon.c (mon.2) 124 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:09:36.365 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:36 smithi092 bash[27556]: audit 2023-09-23T19:09:35.764300+0000 mon.c (mon.2) 125 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-09-23T19:09:36.365 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:36 smithi092 bash[27556]: audit 2023-09-23T19:09:35.764887+0000 mon.a (mon.0) 711 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-09-23T19:09:36.365 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:36 smithi092 bash[27556]: cluster 2023-09-23T19:09:35.776516+0000 mon.a (mon.0) 712 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2023-09-23T19:09:36.365 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:36 smithi092 bash[27556]: audit 2023-09-23T19:09:35.777728+0000 mon.a (mon.0) 713 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-09-23T19:09:36.365 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:36 smithi092 bash[27556]: cluster 2023-09-23T19:09:35.778138+0000 mon.a (mon.0) 714 : cluster [DBG] mgrmap e20: x(active, starting, since 0.0122516s) 2023-09-23T19:09:36.365 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:36 smithi092 bash[50491]: debug 2023-09-23T19:09:36.364+0000 7fc204b3a200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T19:09:36.535 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:09:36 smithi016 bash[27503]: debug 2023-09-23T19:09:36.120+0000 7f4fd5574680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T19:09:36.536 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:09:36 smithi016 bash[27503]: debug 2023-09-23T19:09:36.284+0000 7f4fd5574680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T19:09:36.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:36 smithi016 bash[33624]: cluster 2023-09-23T19:09:34.991356+0000 mgr.y (mgr.14454) 225 : cluster [DBG] pgmap v197: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 44 MiB/s wr, 698 op/s 2023-09-23T19:09:36.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:36 smithi016 bash[33624]: audit 2023-09-23T19:09:35.386122+0000 mon.a (mon.0) 709 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:09:36.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:36 smithi016 bash[33624]: audit 2023-09-23T19:09:35.726689+0000 mon.c (mon.2) 122 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:09:36.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:36 smithi016 bash[33624]: audit 2023-09-23T19:09:35.728070+0000 mon.c (mon.2) 123 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:09:36.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:36 smithi016 bash[33624]: audit 2023-09-23T19:09:35.735926+0000 mon.a (mon.0) 710 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:09:36.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:36 smithi016 bash[33624]: audit 2023-09-23T19:09:35.745866+0000 mon.c (mon.2) 124 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:09:36.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:36 smithi016 bash[33624]: audit 2023-09-23T19:09:35.764300+0000 mon.c (mon.2) 125 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-09-23T19:09:36.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:36 smithi016 bash[33624]: audit 2023-09-23T19:09:35.764887+0000 mon.a (mon.0) 711 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-09-23T19:09:36.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:36 smithi016 bash[33624]: cluster 2023-09-23T19:09:35.776516+0000 mon.a (mon.0) 712 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2023-09-23T19:09:36.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:36 smithi016 bash[33624]: audit 2023-09-23T19:09:35.777728+0000 mon.a (mon.0) 713 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-09-23T19:09:36.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:36 smithi016 bash[33624]: cluster 2023-09-23T19:09:35.778138+0000 mon.a (mon.0) 714 : cluster [DBG] mgrmap e20: x(active, starting, since 0.0122516s) 2023-09-23T19:09:36.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:36 smithi016 bash[27237]: cluster 2023-09-23T19:09:34.991356+0000 mgr.y (mgr.14454) 225 : cluster [DBG] pgmap v197: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 44 MiB/s wr, 698 op/s 2023-09-23T19:09:36.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:36 smithi016 bash[27237]: audit 2023-09-23T19:09:35.386122+0000 mon.a (mon.0) 709 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:09:36.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:36 smithi016 bash[27237]: audit 2023-09-23T19:09:35.726689+0000 mon.c (mon.2) 122 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:09:36.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:36 smithi016 bash[27237]: audit 2023-09-23T19:09:35.728070+0000 mon.c (mon.2) 123 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:09:36.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:36 smithi016 bash[27237]: audit 2023-09-23T19:09:35.735926+0000 mon.a (mon.0) 710 : audit [INF] from='mgr.14454 ' entity='mgr.y' 2023-09-23T19:09:36.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:36 smithi016 bash[27237]: audit 2023-09-23T19:09:35.745866+0000 mon.c (mon.2) 124 : audit [DBG] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:09:36.538 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:36 smithi016 bash[27237]: audit 2023-09-23T19:09:35.764300+0000 mon.c (mon.2) 125 : audit [INF] from='mgr.14454 172.21.15.16:0/1484705891' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-09-23T19:09:36.538 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:36 smithi016 bash[27237]: audit 2023-09-23T19:09:35.764887+0000 mon.a (mon.0) 711 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-09-23T19:09:36.538 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:36 smithi016 bash[27237]: cluster 2023-09-23T19:09:35.776516+0000 mon.a (mon.0) 712 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2023-09-23T19:09:36.538 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:36 smithi016 bash[27237]: audit 2023-09-23T19:09:35.777728+0000 mon.a (mon.0) 713 : audit [INF] from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-09-23T19:09:36.538 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:36 smithi016 bash[27237]: cluster 2023-09-23T19:09:35.778138+0000 mon.a (mon.0) 714 : cluster [DBG] mgrmap e20: x(active, starting, since 0.0122516s) 2023-09-23T19:09:37.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:37 smithi016 bash[27237]: audit 2023-09-23T19:09:36.210002+0000 mon.c (mon.2) 126 : audit [DBG] from='client.? 172.21.15.16:0/2155611855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:37.285 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:09:36 smithi016 bash[27503]: debug 2023-09-23T19:09:36.980+0000 7f4fd5574680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T19:09:37.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:37 smithi016 bash[33624]: audit 2023-09-23T19:09:36.210002+0000 mon.c (mon.2) 126 : audit [DBG] from='client.? 172.21.15.16:0/2155611855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:37.536 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:37 smithi092 bash[50491]: debug 2023-09-23T19:09:37.136+0000 7fc204b3a200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T19:09:37.536 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:37 smithi092 bash[50491]: debug 2023-09-23T19:09:37.292+0000 7fc204b3a200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T19:09:37.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:37 smithi092 bash[27556]: audit 2023-09-23T19:09:36.210002+0000 mon.c (mon.2) 126 : audit [DBG] from='client.? 172.21.15.16:0/2155611855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:38.036 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:37 smithi092 bash[50491]: debug 2023-09-23T19:09:37.784+0000 7fc204b3a200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T19:09:38.036 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:37 smithi092 bash[50491]: debug 2023-09-23T19:09:37.944+0000 7fc204b3a200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T19:09:38.481 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:38 smithi092 bash[50491]: debug 2023-09-23T19:09:38.140+0000 7fc204b3a200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T19:09:38.481 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:38 smithi092 bash[50491]: debug 2023-09-23T19:09:38.296+0000 7fc204b3a200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T19:09:38.759 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:38 smithi092 bash[50491]: debug 2023-09-23T19:09:38.484+0000 7fc204b3a200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T19:09:38.785 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:09:38 smithi016 bash[27503]: debug 2023-09-23T19:09:38.380+0000 7f4fd5574680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T19:09:38.785 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:09:38 smithi016 bash[27503]: debug 2023-09-23T19:09:38.544+0000 7f4fd5574680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T19:09:39.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:38 smithi092 bash[27556]: audit 2023-09-23T19:09:38.891513+0000 mon.b (mon.1) 35 : audit [DBG] from='client.? 172.21.15.16:0/542846619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:39.036 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:38 smithi092 bash[50491]: debug 2023-09-23T19:09:38.760+0000 7fc204b3a200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T19:09:39.036 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:38 smithi092 bash[50491]: debug 2023-09-23T19:09:38.928+0000 7fc204b3a200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T19:09:39.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:38 smithi016 bash[27237]: audit 2023-09-23T19:09:38.891513+0000 mon.b (mon.1) 35 : audit [DBG] from='client.? 172.21.15.16:0/542846619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:39.286 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:09:38 smithi016 bash[27503]: debug 2023-09-23T19:09:38.948+0000 7f4fd5574680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T19:09:39.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:38 smithi016 bash[33624]: audit 2023-09-23T19:09:38.891513+0000 mon.b (mon.1) 35 : audit [DBG] from='client.? 172.21.15.16:0/542846619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:40.952 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:09:40 smithi016 bash[27503]: debug 2023-09-23T19:09:40.680+0000 7f4fd5574680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T19:09:40.952 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:09:40 smithi016 bash[27503]: debug 2023-09-23T19:09:40.944+0000 7f4fd5574680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T19:09:41.002 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:40 smithi092 bash[50491]: debug 2023-09-23T19:09:40.704+0000 7fc204b3a200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T19:09:41.002 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:40 smithi092 bash[50491]: [23/Sep/2023:19:09:40] ENGINE Bus STARTING 2023-09-23T19:09:41.002 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:40 smithi092 bash[50491]: CherryPy Checker: 2023-09-23T19:09:41.002 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:40 smithi092 bash[50491]: The Application mounted at '' has an empty config. 2023-09-23T19:09:41.002 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:40 smithi092 bash[50491]: [23/Sep/2023:19:09:40] ENGINE Serving on http://:::9283 2023-09-23T19:09:41.002 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:40 smithi092 bash[50491]: [23/Sep/2023:19:09:40] ENGINE Bus STARTED 2023-09-23T19:09:41.003 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:40 smithi092 bash[50491]: [23/Sep/2023:19:09:40] ENGINE Bus STOPPING 2023-09-23T19:09:41.003 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:40 smithi092 bash[50491]: [23/Sep/2023:19:09:40] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T19:09:41.003 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:40 smithi092 bash[50491]: [23/Sep/2023:19:09:40] ENGINE Bus STOPPED 2023-09-23T19:09:41.237 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:40 smithi016 bash[27237]: cluster 2023-09-23T19:09:40.725172+0000 mon.a (mon.0) 715 : cluster [INF] Active manager daemon x restarted 2023-09-23T19:09:41.237 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:40 smithi016 bash[27237]: cluster 2023-09-23T19:09:40.725771+0000 mon.a (mon.0) 716 : cluster [INF] Activating manager daemon x 2023-09-23T19:09:41.237 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:40 smithi016 bash[27237]: audit 2023-09-23T19:09:40.727058+0000 mon.a (mon.0) 717 : audit [DBG] from='mgr.? 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T19:09:41.237 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:40 smithi016 bash[27237]: cluster 2023-09-23T19:09:40.743354+0000 mon.a (mon.0) 718 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2023-09-23T19:09:41.237 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:41 smithi016 bash[27237]: audit 2023-09-23T19:09:40.747587+0000 mon.c (mon.2) 127 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T19:09:41.237 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:41 smithi016 bash[27237]: audit 2023-09-23T19:09:40.748464+0000 mon.c (mon.2) 128 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:09:41.237 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:41 smithi016 bash[27237]: cluster 2023-09-23T19:09:40.749702+0000 mon.a (mon.0) 719 : cluster [DBG] mgrmap e21: x(active, starting, since 0.0241826s) 2023-09-23T19:09:41.237 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:41 smithi016 bash[27237]: audit 2023-09-23T19:09:40.749863+0000 mon.c (mon.2) 129 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:09:41.237 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:41 smithi016 bash[27237]: audit 2023-09-23T19:09:40.750256+0000 mon.c (mon.2) 130 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:09:41.238 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:41 smithi016 bash[27237]: audit 2023-09-23T19:09:40.751789+0000 mon.c (mon.2) 131 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi016.uncssv"}]: dispatch 2023-09-23T19:09:41.238 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:41 smithi016 bash[27237]: audit 2023-09-23T19:09:40.752976+0000 mon.c (mon.2) 132 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi092.vbpqks"}]: dispatch 2023-09-23T19:09:41.238 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:41 smithi016 bash[27237]: audit 2023-09-23T19:09:40.754085+0000 mon.c (mon.2) 133 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T19:09:41.238 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:41 smithi016 bash[27237]: audit 2023-09-23T19:09:40.762317+0000 mon.c (mon.2) 134 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T19:09:41.238 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:41 smithi016 bash[27237]: audit 2023-09-23T19:09:40.763751+0000 mon.c (mon.2) 135 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T19:09:41.238 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:41 smithi016 bash[27237]: audit 2023-09-23T19:09:40.772570+0000 mon.c (mon.2) 136 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T19:09:41.238 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:41 smithi016 bash[27237]: audit 2023-09-23T19:09:40.773818+0000 mon.c (mon.2) 137 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T19:09:41.238 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:41 smithi016 bash[27237]: audit 2023-09-23T19:09:40.774879+0000 mon.c (mon.2) 138 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T19:09:41.238 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:41 smithi016 bash[27237]: audit 2023-09-23T19:09:40.775929+0000 mon.c (mon.2) 139 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T19:09:41.238 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:41 smithi016 bash[27237]: audit 2023-09-23T19:09:40.776966+0000 mon.c (mon.2) 140 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:09:41.238 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:41 smithi016 bash[27237]: audit 2023-09-23T19:09:40.778089+0000 mon.c (mon.2) 141 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T19:09:41.239 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:41 smithi016 bash[27237]: audit 2023-09-23T19:09:40.779174+0000 mon.c (mon.2) 142 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:09:41.239 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:41 smithi016 bash[27237]: audit 2023-09-23T19:09:40.780172+0000 mon.c (mon.2) 143 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:09:41.239 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:41 smithi016 bash[27237]: audit 2023-09-23T19:09:40.781434+0000 mon.c (mon.2) 144 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:09:41.239 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:41 smithi016 bash[27237]: audit 2023-09-23T19:09:40.782714+0000 mon.c (mon.2) 145 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T19:09:41.239 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:41 smithi016 bash[27237]: audit 2023-09-23T19:09:40.786784+0000 mon.c (mon.2) 146 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T19:09:41.239 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:41 smithi016 bash[27237]: cluster 2023-09-23T19:09:40.969914+0000 mon.a (mon.0) 720 : cluster [INF] Manager daemon x is now available 2023-09-23T19:09:41.239 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:09:41 smithi016 bash[27503]: debug 2023-09-23T19:09:41.096+0000 7f4fd5574680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T19:09:41.239 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:41 smithi016 bash[33624]: cluster 2023-09-23T19:09:40.725172+0000 mon.a (mon.0) 715 : cluster [INF] Active manager daemon x restarted 2023-09-23T19:09:41.240 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:41 smithi016 bash[33624]: cluster 2023-09-23T19:09:40.725771+0000 mon.a (mon.0) 716 : cluster [INF] Activating manager daemon x 2023-09-23T19:09:41.240 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:41 smithi016 bash[33624]: audit 2023-09-23T19:09:40.727058+0000 mon.a (mon.0) 717 : audit [DBG] from='mgr.? 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T19:09:41.240 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:41 smithi016 bash[33624]: cluster 2023-09-23T19:09:40.743354+0000 mon.a (mon.0) 718 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2023-09-23T19:09:41.240 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:41 smithi016 bash[33624]: audit 2023-09-23T19:09:40.747587+0000 mon.c (mon.2) 127 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T19:09:41.240 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:41 smithi016 bash[33624]: audit 2023-09-23T19:09:40.748464+0000 mon.c (mon.2) 128 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:09:41.240 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:41 smithi016 bash[33624]: cluster 2023-09-23T19:09:40.749702+0000 mon.a (mon.0) 719 : cluster [DBG] mgrmap e21: x(active, starting, since 0.0241826s) 2023-09-23T19:09:41.240 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:41 smithi016 bash[33624]: audit 2023-09-23T19:09:40.749863+0000 mon.c (mon.2) 129 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:09:41.240 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:41 smithi016 bash[33624]: audit 2023-09-23T19:09:40.750256+0000 mon.c (mon.2) 130 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:09:41.240 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:41 smithi016 bash[33624]: audit 2023-09-23T19:09:40.751789+0000 mon.c (mon.2) 131 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi016.uncssv"}]: dispatch 2023-09-23T19:09:41.240 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:41 smithi016 bash[33624]: audit 2023-09-23T19:09:40.752976+0000 mon.c (mon.2) 132 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi092.vbpqks"}]: dispatch 2023-09-23T19:09:41.240 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:41 smithi016 bash[33624]: audit 2023-09-23T19:09:40.754085+0000 mon.c (mon.2) 133 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T19:09:41.241 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:41 smithi016 bash[33624]: audit 2023-09-23T19:09:40.762317+0000 mon.c (mon.2) 134 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T19:09:41.241 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:41 smithi016 bash[33624]: audit 2023-09-23T19:09:40.763751+0000 mon.c (mon.2) 135 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T19:09:41.241 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:41 smithi016 bash[33624]: audit 2023-09-23T19:09:40.772570+0000 mon.c (mon.2) 136 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T19:09:41.241 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:41 smithi016 bash[33624]: audit 2023-09-23T19:09:40.773818+0000 mon.c (mon.2) 137 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T19:09:41.241 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:41 smithi016 bash[33624]: audit 2023-09-23T19:09:40.774879+0000 mon.c (mon.2) 138 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T19:09:41.241 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:41 smithi016 bash[33624]: audit 2023-09-23T19:09:40.775929+0000 mon.c (mon.2) 139 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T19:09:41.241 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:41 smithi016 bash[33624]: audit 2023-09-23T19:09:40.776966+0000 mon.c (mon.2) 140 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:09:41.241 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:41 smithi016 bash[33624]: audit 2023-09-23T19:09:40.778089+0000 mon.c (mon.2) 141 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T19:09:41.241 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:41 smithi016 bash[33624]: audit 2023-09-23T19:09:40.779174+0000 mon.c (mon.2) 142 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:09:41.241 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:41 smithi016 bash[33624]: audit 2023-09-23T19:09:40.780172+0000 mon.c (mon.2) 143 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:09:41.241 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:41 smithi016 bash[33624]: audit 2023-09-23T19:09:40.781434+0000 mon.c (mon.2) 144 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:09:41.241 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:41 smithi016 bash[33624]: audit 2023-09-23T19:09:40.782714+0000 mon.c (mon.2) 145 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T19:09:41.242 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:41 smithi016 bash[33624]: audit 2023-09-23T19:09:40.786784+0000 mon.c (mon.2) 146 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T19:09:41.242 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:41 smithi016 bash[33624]: cluster 2023-09-23T19:09:40.969914+0000 mon.a (mon.0) 720 : cluster [INF] Manager daemon x is now available 2023-09-23T19:09:41.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:41 smithi092 bash[27556]: cluster 2023-09-23T19:09:40.725172+0000 mon.a (mon.0) 715 : cluster [INF] Active manager daemon x restarted 2023-09-23T19:09:41.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:41 smithi092 bash[27556]: cluster 2023-09-23T19:09:40.725771+0000 mon.a (mon.0) 716 : cluster [INF] Activating manager daemon x 2023-09-23T19:09:41.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:41 smithi092 bash[27556]: audit 2023-09-23T19:09:40.727058+0000 mon.a (mon.0) 717 : audit [DBG] from='mgr.? 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T19:09:41.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:41 smithi092 bash[27556]: cluster 2023-09-23T19:09:40.743354+0000 mon.a (mon.0) 718 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2023-09-23T19:09:41.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:41 smithi092 bash[27556]: audit 2023-09-23T19:09:40.747587+0000 mon.c (mon.2) 127 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T19:09:41.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:41 smithi092 bash[27556]: audit 2023-09-23T19:09:40.748464+0000 mon.c (mon.2) 128 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:09:41.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:41 smithi092 bash[27556]: cluster 2023-09-23T19:09:40.749702+0000 mon.a (mon.0) 719 : cluster [DBG] mgrmap e21: x(active, starting, since 0.0241826s) 2023-09-23T19:09:41.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:41 smithi092 bash[27556]: audit 2023-09-23T19:09:40.749863+0000 mon.c (mon.2) 129 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:09:41.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:41 smithi092 bash[27556]: audit 2023-09-23T19:09:40.750256+0000 mon.c (mon.2) 130 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:09:41.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:41 smithi092 bash[27556]: audit 2023-09-23T19:09:40.751789+0000 mon.c (mon.2) 131 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi016.uncssv"}]: dispatch 2023-09-23T19:09:41.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:41 smithi092 bash[27556]: audit 2023-09-23T19:09:40.752976+0000 mon.c (mon.2) 132 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi092.vbpqks"}]: dispatch 2023-09-23T19:09:41.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:41 smithi092 bash[27556]: audit 2023-09-23T19:09:40.754085+0000 mon.c (mon.2) 133 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T19:09:41.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:41 smithi092 bash[27556]: audit 2023-09-23T19:09:40.762317+0000 mon.c (mon.2) 134 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T19:09:41.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:41 smithi092 bash[27556]: audit 2023-09-23T19:09:40.763751+0000 mon.c (mon.2) 135 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T19:09:41.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:41 smithi092 bash[27556]: audit 2023-09-23T19:09:40.772570+0000 mon.c (mon.2) 136 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T19:09:41.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:41 smithi092 bash[27556]: audit 2023-09-23T19:09:40.773818+0000 mon.c (mon.2) 137 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T19:09:41.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:41 smithi092 bash[27556]: audit 2023-09-23T19:09:40.774879+0000 mon.c (mon.2) 138 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T19:09:41.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:41 smithi092 bash[27556]: audit 2023-09-23T19:09:40.775929+0000 mon.c (mon.2) 139 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T19:09:41.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:41 smithi092 bash[27556]: audit 2023-09-23T19:09:40.776966+0000 mon.c (mon.2) 140 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:09:41.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:41 smithi092 bash[27556]: audit 2023-09-23T19:09:40.778089+0000 mon.c (mon.2) 141 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T19:09:41.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:41 smithi092 bash[27556]: audit 2023-09-23T19:09:40.779174+0000 mon.c (mon.2) 142 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:09:41.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:41 smithi092 bash[27556]: audit 2023-09-23T19:09:40.780172+0000 mon.c (mon.2) 143 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:09:41.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:41 smithi092 bash[27556]: audit 2023-09-23T19:09:40.781434+0000 mon.c (mon.2) 144 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:09:41.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:41 smithi092 bash[27556]: audit 2023-09-23T19:09:40.782714+0000 mon.c (mon.2) 145 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T19:09:41.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:41 smithi092 bash[27556]: audit 2023-09-23T19:09:40.786784+0000 mon.c (mon.2) 146 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T19:09:41.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:41 smithi092 bash[27556]: cluster 2023-09-23T19:09:40.969914+0000 mon.a (mon.0) 720 : cluster [INF] Manager daemon x is now available 2023-09-23T19:09:41.288 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:41 smithi092 bash[50491]: [23/Sep/2023:19:09:41] ENGINE Bus STARTING 2023-09-23T19:09:41.536 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:09:41 smithi016 bash[27503]: debug 2023-09-23T19:09:41.268+0000 7f4fd5574680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T19:09:41.786 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:41 smithi092 bash[50491]: debug 2023-09-23T19:09:41.332+0000 7fc161f98700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:09:41.786 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:41 smithi092 bash[50491]: debug 2023-09-23T19:09:41.332+0000 7fc161f98700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:09:41.786 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:41 smithi092 bash[50491]: debug 2023-09-23T19:09:41.332+0000 7fc161f98700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:09:41.787 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:41 smithi092 bash[50491]: debug 2023-09-23T19:09:41.332+0000 7fc161f98700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:09:41.787 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:41 smithi092 bash[50491]: debug 2023-09-23T19:09:41.332+0000 7fc161f98700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:09:41.787 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:41 smithi092 bash[50491]: debug 2023-09-23T19:09:41.336+0000 7fc168fa6700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:09:41.787 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:41 smithi092 bash[50491]: debug 2023-09-23T19:09:41.336+0000 7fc168fa6700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:09:41.787 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:41 smithi092 bash[50491]: debug 2023-09-23T19:09:41.336+0000 7fc168fa6700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:09:41.787 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:41 smithi092 bash[50491]: debug 2023-09-23T19:09:41.336+0000 7fc168fa6700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:09:41.787 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:41 smithi092 bash[50491]: debug 2023-09-23T19:09:41.336+0000 7fc168fa6700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:09:41.787 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:41 smithi092 bash[50491]: [23/Sep/2023:19:09:41] ENGINE Serving on http://:::9283 2023-09-23T19:09:41.787 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:41 smithi092 bash[50491]: [23/Sep/2023:19:09:41] ENGINE Bus STARTED 2023-09-23T19:09:42.035 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:09:41 smithi016 bash[27503]: debug 2023-09-23T19:09:41.604+0000 7f4fd5574680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T19:09:42.361 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:42 smithi016 bash[27237]: audit 2023-09-23T19:09:41.035117+0000 mon.a (mon.0) 721 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:09:42.361 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:42 smithi016 bash[27237]: audit 2023-09-23T19:09:41.052788+0000 mon.c (mon.2) 147 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:09:42.361 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:42 smithi016 bash[27237]: audit 2023-09-23T19:09:41.066294+0000 mon.c (mon.2) 148 : audit [INF] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:09:42.361 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:42 smithi016 bash[27237]: audit 2023-09-23T19:09:41.067642+0000 mon.a (mon.0) 722 : audit [INF] from='mgr.15012 ' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:09:42.361 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:42 smithi016 bash[27237]: audit 2023-09-23T19:09:41.080493+0000 mon.c (mon.2) 149 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:09:42.362 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:42 smithi016 bash[27237]: audit 2023-09-23T19:09:41.127374+0000 mon.c (mon.2) 150 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:09:42.362 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:42 smithi016 bash[27237]: audit 2023-09-23T19:09:41.146912+0000 mon.c (mon.2) 151 : audit [INF] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T19:09:42.362 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:42 smithi016 bash[27237]: audit 2023-09-23T19:09:41.147601+0000 mon.a (mon.0) 723 : audit [INF] from='mgr.15012 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T19:09:42.362 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:42 smithi016 bash[27237]: audit 2023-09-23T19:09:41.209700+0000 mon.a (mon.0) 724 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:09:42.362 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:42 smithi016 bash[27237]: audit 2023-09-23T19:09:41.296988+0000 mon.c (mon.2) 152 : audit [INF] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T19:09:42.362 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:42 smithi016 bash[27237]: audit 2023-09-23T19:09:41.297901+0000 mon.a (mon.0) 725 : audit [INF] from='mgr.15012 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T19:09:42.362 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:42 smithi016 bash[27237]: audit 2023-09-23T19:09:41.599895+0000 mon.a (mon.0) 726 : audit [DBG] from='client.? 172.21.15.16:0/344571885' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:42.362 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:42 smithi016 bash[27237]: cluster 2023-09-23T19:09:41.742022+0000 mon.a (mon.0) 727 : cluster [DBG] mgrmap e22: x(active, since 1.01653s) 2023-09-23T19:09:42.362 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:42 smithi016 bash[33624]: audit 2023-09-23T19:09:41.035117+0000 mon.a (mon.0) 721 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:09:42.363 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:42 smithi016 bash[33624]: audit 2023-09-23T19:09:41.052788+0000 mon.c (mon.2) 147 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:09:42.363 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:42 smithi016 bash[33624]: audit 2023-09-23T19:09:41.066294+0000 mon.c (mon.2) 148 : audit [INF] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:09:42.363 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:42 smithi016 bash[33624]: audit 2023-09-23T19:09:41.067642+0000 mon.a (mon.0) 722 : audit [INF] from='mgr.15012 ' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:09:42.363 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:42 smithi016 bash[33624]: audit 2023-09-23T19:09:41.080493+0000 mon.c (mon.2) 149 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:09:42.363 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:42 smithi016 bash[33624]: audit 2023-09-23T19:09:41.127374+0000 mon.c (mon.2) 150 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:09:42.363 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:42 smithi016 bash[33624]: audit 2023-09-23T19:09:41.146912+0000 mon.c (mon.2) 151 : audit [INF] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T19:09:42.363 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:42 smithi016 bash[33624]: audit 2023-09-23T19:09:41.147601+0000 mon.a (mon.0) 723 : audit [INF] from='mgr.15012 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T19:09:42.363 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:42 smithi016 bash[33624]: audit 2023-09-23T19:09:41.209700+0000 mon.a (mon.0) 724 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:09:42.363 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:42 smithi016 bash[33624]: audit 2023-09-23T19:09:41.296988+0000 mon.c (mon.2) 152 : audit [INF] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T19:09:42.363 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:42 smithi016 bash[33624]: audit 2023-09-23T19:09:41.297901+0000 mon.a (mon.0) 725 : audit [INF] from='mgr.15012 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T19:09:42.363 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:42 smithi016 bash[33624]: audit 2023-09-23T19:09:41.599895+0000 mon.a (mon.0) 726 : audit [DBG] from='client.? 172.21.15.16:0/344571885' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:42.364 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:42 smithi016 bash[33624]: cluster 2023-09-23T19:09:41.742022+0000 mon.a (mon.0) 727 : cluster [DBG] mgrmap e22: x(active, since 1.01653s) 2023-09-23T19:09:42.364 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:09:42 smithi016 bash[27503]: debug 2023-09-23T19:09:42.168+0000 7f4fd5574680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T19:09:42.364 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:09:42 smithi016 bash[27503]: debug 2023-09-23T19:09:42.352+0000 7f4fd5574680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T19:09:42.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:42 smithi092 bash[27556]: audit 2023-09-23T19:09:41.035117+0000 mon.a (mon.0) 721 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:09:42.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:42 smithi092 bash[27556]: audit 2023-09-23T19:09:41.052788+0000 mon.c (mon.2) 147 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:09:42.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:42 smithi092 bash[27556]: audit 2023-09-23T19:09:41.066294+0000 mon.c (mon.2) 148 : audit [INF] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:09:42.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:42 smithi092 bash[27556]: audit 2023-09-23T19:09:41.067642+0000 mon.a (mon.0) 722 : audit [INF] from='mgr.15012 ' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:09:42.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:42 smithi092 bash[27556]: audit 2023-09-23T19:09:41.080493+0000 mon.c (mon.2) 149 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:09:42.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:42 smithi092 bash[27556]: audit 2023-09-23T19:09:41.127374+0000 mon.c (mon.2) 150 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:09:42.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:42 smithi092 bash[27556]: audit 2023-09-23T19:09:41.146912+0000 mon.c (mon.2) 151 : audit [INF] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T19:09:42.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:42 smithi092 bash[27556]: audit 2023-09-23T19:09:41.147601+0000 mon.a (mon.0) 723 : audit [INF] from='mgr.15012 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T19:09:42.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:42 smithi092 bash[27556]: audit 2023-09-23T19:09:41.209700+0000 mon.a (mon.0) 724 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:09:42.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:42 smithi092 bash[27556]: audit 2023-09-23T19:09:41.296988+0000 mon.c (mon.2) 152 : audit [INF] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T19:09:42.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:42 smithi092 bash[27556]: audit 2023-09-23T19:09:41.297901+0000 mon.a (mon.0) 725 : audit [INF] from='mgr.15012 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T19:09:42.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:42 smithi092 bash[27556]: audit 2023-09-23T19:09:41.599895+0000 mon.a (mon.0) 726 : audit [DBG] from='client.? 172.21.15.16:0/344571885' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:42.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:42 smithi092 bash[27556]: cluster 2023-09-23T19:09:41.742022+0000 mon.a (mon.0) 727 : cluster [DBG] mgrmap e22: x(active, since 1.01653s) 2023-09-23T19:09:43.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:43 smithi016 bash[33624]: cephadm 2023-09-23T19:09:41.695511+0000 mgr.x (mgr.15012) 1 : cephadm [INF] Deploying cephadm binary to smithi092 2023-09-23T19:09:43.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:43 smithi016 bash[33624]: cluster 2023-09-23T19:09:41.813962+0000 mgr.x (mgr.15012) 2 : cluster [DBG] pgmap v3: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail 2023-09-23T19:09:43.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:43 smithi016 bash[33624]: cephadm 2023-09-23T19:09:41.872427+0000 mgr.x (mgr.15012) 3 : cephadm [INF] Deploying cephadm binary to smithi016 2023-09-23T19:09:43.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:43 smithi016 bash[33624]: audit 2023-09-23T19:09:42.036792+0000 mon.a (mon.0) 728 : audit [INF] from='mgr.15012 ' entity='mgr.x' cmd='[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]': finished 2023-09-23T19:09:43.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:43 smithi016 bash[33624]: cluster 2023-09-23T19:09:42.036864+0000 mon.a (mon.0) 729 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2023-09-23T19:09:43.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:43 smithi016 bash[33624]: audit 2023-09-23T19:09:42.076352+0000 mon.c (mon.2) 153 : audit [INF] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:09:43.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:43 smithi016 bash[33624]: audit 2023-09-23T19:09:42.077260+0000 mon.a (mon.0) 730 : audit [INF] from='mgr.15012 ' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:09:43.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:43 smithi016 bash[27237]: cephadm 2023-09-23T19:09:41.695511+0000 mgr.x (mgr.15012) 1 : cephadm [INF] Deploying cephadm binary to smithi092 2023-09-23T19:09:43.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:43 smithi016 bash[27237]: cluster 2023-09-23T19:09:41.813962+0000 mgr.x (mgr.15012) 2 : cluster [DBG] pgmap v3: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail 2023-09-23T19:09:43.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:43 smithi016 bash[27237]: cephadm 2023-09-23T19:09:41.872427+0000 mgr.x (mgr.15012) 3 : cephadm [INF] Deploying cephadm binary to smithi016 2023-09-23T19:09:43.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:43 smithi016 bash[27237]: audit 2023-09-23T19:09:42.036792+0000 mon.a (mon.0) 728 : audit [INF] from='mgr.15012 ' entity='mgr.x' cmd='[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]': finished 2023-09-23T19:09:43.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:43 smithi016 bash[27237]: cluster 2023-09-23T19:09:42.036864+0000 mon.a (mon.0) 729 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2023-09-23T19:09:43.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:43 smithi016 bash[27237]: audit 2023-09-23T19:09:42.076352+0000 mon.c (mon.2) 153 : audit [INF] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:09:43.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:43 smithi016 bash[27237]: audit 2023-09-23T19:09:42.077260+0000 mon.a (mon.0) 730 : audit [INF] from='mgr.15012 ' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:09:43.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:43 smithi092 bash[27556]: cephadm 2023-09-23T19:09:41.695511+0000 mgr.x (mgr.15012) 1 : cephadm [INF] Deploying cephadm binary to smithi092 2023-09-23T19:09:43.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:43 smithi092 bash[27556]: cluster 2023-09-23T19:09:41.813962+0000 mgr.x (mgr.15012) 2 : cluster [DBG] pgmap v3: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail 2023-09-23T19:09:43.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:43 smithi092 bash[27556]: cephadm 2023-09-23T19:09:41.872427+0000 mgr.x (mgr.15012) 3 : cephadm [INF] Deploying cephadm binary to smithi016 2023-09-23T19:09:43.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:43 smithi092 bash[27556]: audit 2023-09-23T19:09:42.036792+0000 mon.a (mon.0) 728 : audit [INF] from='mgr.15012 ' entity='mgr.x' cmd='[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]': finished 2023-09-23T19:09:43.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:43 smithi092 bash[27556]: cluster 2023-09-23T19:09:42.036864+0000 mon.a (mon.0) 729 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2023-09-23T19:09:43.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:43 smithi092 bash[27556]: audit 2023-09-23T19:09:42.076352+0000 mon.c (mon.2) 153 : audit [INF] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:09:43.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:43 smithi092 bash[27556]: audit 2023-09-23T19:09:42.077260+0000 mon.a (mon.0) 730 : audit [INF] from='mgr.15012 ' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:09:43.979 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:09:43 smithi016 bash[27503]: debug 2023-09-23T19:09:43.632+0000 7f4fd5574680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T19:09:43.980 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:09:43 smithi016 bash[27503]: debug 2023-09-23T19:09:43.780+0000 7f4fd5574680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T19:09:43.980 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:09:43 smithi016 bash[27503]: debug 2023-09-23T19:09:43.968+0000 7f4fd5574680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T19:09:44.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:44 smithi016 bash[27237]: cluster 2023-09-23T19:09:42.772016+0000 mgr.x (mgr.15012) 4 : cluster [DBG] pgmap v5: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail 2023-09-23T19:09:44.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:44 smithi016 bash[27237]: audit 2023-09-23T19:09:43.049124+0000 mon.a (mon.0) 731 : audit [INF] from='mgr.15012 ' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-09-23T19:09:44.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:44 smithi016 bash[27237]: cluster 2023-09-23T19:09:43.049375+0000 mon.a (mon.0) 732 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2023-09-23T19:09:44.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:44 smithi016 bash[27237]: cluster 2023-09-23T19:09:43.755764+0000 mon.a (mon.0) 733 : cluster [DBG] mgrmap e23: x(active, since 3s) 2023-09-23T19:09:44.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:44 smithi016 bash[33624]: cluster 2023-09-23T19:09:42.772016+0000 mgr.x (mgr.15012) 4 : cluster [DBG] pgmap v5: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail 2023-09-23T19:09:44.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:44 smithi016 bash[33624]: audit 2023-09-23T19:09:43.049124+0000 mon.a (mon.0) 731 : audit [INF] from='mgr.15012 ' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-09-23T19:09:44.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:44 smithi016 bash[33624]: cluster 2023-09-23T19:09:43.049375+0000 mon.a (mon.0) 732 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2023-09-23T19:09:44.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:44 smithi016 bash[33624]: cluster 2023-09-23T19:09:43.755764+0000 mon.a (mon.0) 733 : cluster [DBG] mgrmap e23: x(active, since 3s) 2023-09-23T19:09:44.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:44 smithi092 bash[27556]: cluster 2023-09-23T19:09:42.772016+0000 mgr.x (mgr.15012) 4 : cluster [DBG] pgmap v5: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail 2023-09-23T19:09:44.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:44 smithi092 bash[27556]: audit 2023-09-23T19:09:43.049124+0000 mon.a (mon.0) 731 : audit [INF] from='mgr.15012 ' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-09-23T19:09:44.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:44 smithi092 bash[27556]: cluster 2023-09-23T19:09:43.049375+0000 mon.a (mon.0) 732 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2023-09-23T19:09:44.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:44 smithi092 bash[27556]: cluster 2023-09-23T19:09:43.755764+0000 mon.a (mon.0) 733 : cluster [DBG] mgrmap e23: x(active, since 3s) 2023-09-23T19:09:44.707 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:09:44 smithi016 bash[27503]: debug 2023-09-23T19:09:44.292+0000 7f4fd5574680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T19:09:44.707 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:09:44 smithi016 bash[27503]: debug 2023-09-23T19:09:44.696+0000 7f4fd5574680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T19:09:44.966 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:09:44 smithi016 bash[27503]: debug 2023-09-23T19:09:44.956+0000 7f4fd5574680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T19:09:45.237 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:45 smithi016 bash[27237]: audit 2023-09-23T19:09:44.316404+0000 mon.a (mon.0) 734 : audit [DBG] from='client.? 172.21.15.16:0/3028289482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:45.237 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:45 smithi016 bash[27237]: audit 2023-09-23T19:09:44.648897+0000 mon.a (mon.0) 735 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:09:45.237 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:45 smithi016 bash[27237]: audit 2023-09-23T19:09:44.656326+0000 mon.a (mon.0) 736 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:09:45.237 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:09:45 smithi016 bash[27503]: debug 2023-09-23T19:09:45.228+0000 7f4fd5574680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T19:09:45.238 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:45 smithi016 bash[33624]: audit 2023-09-23T19:09:44.316404+0000 mon.a (mon.0) 734 : audit [DBG] from='client.? 172.21.15.16:0/3028289482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:45.238 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:45 smithi016 bash[33624]: audit 2023-09-23T19:09:44.648897+0000 mon.a (mon.0) 735 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:09:45.238 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:45 smithi016 bash[33624]: audit 2023-09-23T19:09:44.656326+0000 mon.a (mon.0) 736 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:09:45.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:45 smithi092 bash[27556]: audit 2023-09-23T19:09:44.316404+0000 mon.a (mon.0) 734 : audit [DBG] from='client.? 172.21.15.16:0/3028289482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:45.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:45 smithi092 bash[27556]: audit 2023-09-23T19:09:44.648897+0000 mon.a (mon.0) 735 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:09:45.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:45 smithi092 bash[27556]: audit 2023-09-23T19:09:44.656326+0000 mon.a (mon.0) 736 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:09:45.901 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:09:45 smithi016 bash[27503]: debug 2023-09-23T19:09:45.588+0000 7f4fd5574680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T19:09:45.901 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:09:45 smithi016 bash[27503]: debug 2023-09-23T19:09:45.752+0000 7f4fd5574680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T19:09:45.901 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:09:45 smithi016 bash[27503]: [23/Sep/2023:19:09:45] ENGINE Bus STARTING 2023-09-23T19:09:45.901 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:09:45 smithi016 bash[27503]: CherryPy Checker: 2023-09-23T19:09:45.901 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:09:45 smithi016 bash[27503]: The Application mounted at '' has an empty config. 2023-09-23T19:09:45.901 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:09:45 smithi016 bash[27503]: [23/Sep/2023:19:09:45] ENGINE Serving on http://:::9283 2023-09-23T19:09:46.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:46 smithi016 bash[27237]: cluster 2023-09-23T19:09:44.772436+0000 mgr.x (mgr.15012) 5 : cluster [DBG] pgmap v7: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail 2023-09-23T19:09:46.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:46 smithi016 bash[27237]: cephadm 2023-09-23T19:09:45.059722+0000 mgr.x (mgr.15012) 6 : cephadm [INF] [23/Sep/2023:19:09:45] ENGINE Bus STARTING 2023-09-23T19:09:46.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:46 smithi016 bash[27237]: audit 2023-09-23T19:09:45.504753+0000 mon.a (mon.0) 737 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:09:46.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:46 smithi016 bash[27237]: cluster 2023-09-23T19:09:45.670106+0000 mon.a (mon.0) 738 : cluster [DBG] mgrmap e24: x(active, since 4s) 2023-09-23T19:09:46.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:46 smithi016 bash[27237]: cluster 2023-09-23T19:09:45.779405+0000 mon.a (mon.0) 739 : cluster [DBG] Standby manager daemon y started 2023-09-23T19:09:46.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:46 smithi016 bash[27237]: audit 2023-09-23T19:09:45.794880+0000 mon.b (mon.1) 36 : audit [DBG] from='mgr.? 172.21.15.16:0/421602681' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T19:09:46.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:46 smithi016 bash[27237]: audit 2023-09-23T19:09:45.799281+0000 mon.b (mon.1) 37 : audit [DBG] from='mgr.? 172.21.15.16:0/421602681' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T19:09:46.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:46 smithi016 bash[27237]: audit 2023-09-23T19:09:45.807155+0000 mon.b (mon.1) 38 : audit [DBG] from='mgr.? 172.21.15.16:0/421602681' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T19:09:46.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:46 smithi016 bash[27237]: audit 2023-09-23T19:09:45.817018+0000 mon.b (mon.1) 39 : audit [DBG] from='mgr.? 172.21.15.16:0/421602681' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T19:09:46.287 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:09:45 smithi016 bash[27503]: [23/Sep/2023:19:09:45] ENGINE Bus STARTED 2023-09-23T19:09:46.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:46 smithi016 bash[33624]: cluster 2023-09-23T19:09:44.772436+0000 mgr.x (mgr.15012) 5 : cluster [DBG] pgmap v7: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail 2023-09-23T19:09:46.288 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:46 smithi016 bash[33624]: cephadm 2023-09-23T19:09:45.059722+0000 mgr.x (mgr.15012) 6 : cephadm [INF] [23/Sep/2023:19:09:45] ENGINE Bus STARTING 2023-09-23T19:09:46.288 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:46 smithi016 bash[33624]: audit 2023-09-23T19:09:45.504753+0000 mon.a (mon.0) 737 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:09:46.288 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:46 smithi016 bash[33624]: cluster 2023-09-23T19:09:45.670106+0000 mon.a (mon.0) 738 : cluster [DBG] mgrmap e24: x(active, since 4s) 2023-09-23T19:09:46.288 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:46 smithi016 bash[33624]: cluster 2023-09-23T19:09:45.779405+0000 mon.a (mon.0) 739 : cluster [DBG] Standby manager daemon y started 2023-09-23T19:09:46.288 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:46 smithi016 bash[33624]: audit 2023-09-23T19:09:45.794880+0000 mon.b (mon.1) 36 : audit [DBG] from='mgr.? 172.21.15.16:0/421602681' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T19:09:46.288 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:46 smithi016 bash[33624]: audit 2023-09-23T19:09:45.799281+0000 mon.b (mon.1) 37 : audit [DBG] from='mgr.? 172.21.15.16:0/421602681' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T19:09:46.289 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:46 smithi016 bash[33624]: audit 2023-09-23T19:09:45.807155+0000 mon.b (mon.1) 38 : audit [DBG] from='mgr.? 172.21.15.16:0/421602681' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T19:09:46.289 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:46 smithi016 bash[33624]: audit 2023-09-23T19:09:45.817018+0000 mon.b (mon.1) 39 : audit [DBG] from='mgr.? 172.21.15.16:0/421602681' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T19:09:46.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:46 smithi092 bash[27556]: cluster 2023-09-23T19:09:44.772436+0000 mgr.x (mgr.15012) 5 : cluster [DBG] pgmap v7: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail 2023-09-23T19:09:46.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:46 smithi092 bash[27556]: cephadm 2023-09-23T19:09:45.059722+0000 mgr.x (mgr.15012) 6 : cephadm [INF] [23/Sep/2023:19:09:45] ENGINE Bus STARTING 2023-09-23T19:09:46.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:46 smithi092 bash[27556]: audit 2023-09-23T19:09:45.504753+0000 mon.a (mon.0) 737 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:09:46.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:46 smithi092 bash[27556]: cluster 2023-09-23T19:09:45.670106+0000 mon.a (mon.0) 738 : cluster [DBG] mgrmap e24: x(active, since 4s) 2023-09-23T19:09:46.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:46 smithi092 bash[27556]: cluster 2023-09-23T19:09:45.779405+0000 mon.a (mon.0) 739 : cluster [DBG] Standby manager daemon y started 2023-09-23T19:09:46.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:46 smithi092 bash[27556]: audit 2023-09-23T19:09:45.794880+0000 mon.b (mon.1) 36 : audit [DBG] from='mgr.? 172.21.15.16:0/421602681' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T19:09:46.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:46 smithi092 bash[27556]: audit 2023-09-23T19:09:45.799281+0000 mon.b (mon.1) 37 : audit [DBG] from='mgr.? 172.21.15.16:0/421602681' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T19:09:46.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:46 smithi092 bash[27556]: audit 2023-09-23T19:09:45.807155+0000 mon.b (mon.1) 38 : audit [DBG] from='mgr.? 172.21.15.16:0/421602681' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T19:09:46.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:46 smithi092 bash[27556]: audit 2023-09-23T19:09:45.817018+0000 mon.b (mon.1) 39 : audit [DBG] from='mgr.? 172.21.15.16:0/421602681' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T19:09:47.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:47 smithi016 bash[27237]: cephadm 2023-09-23T19:09:45.180048+0000 mgr.x (mgr.15012) 7 : cephadm [INF] [23/Sep/2023:19:09:45] ENGINE Serving on http://172.21.15.92:8765 2023-09-23T19:09:47.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:47 smithi016 bash[27237]: cephadm 2023-09-23T19:09:45.378728+0000 mgr.x (mgr.15012) 8 : cephadm [INF] [23/Sep/2023:19:09:45] ENGINE Serving on https://172.21.15.92:7150 2023-09-23T19:09:47.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:47 smithi016 bash[27237]: cephadm 2023-09-23T19:09:45.378990+0000 mgr.x (mgr.15012) 9 : cephadm [INF] [23/Sep/2023:19:09:45] ENGINE Bus STARTED 2023-09-23T19:09:47.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:47 smithi016 bash[27237]: cluster 2023-09-23T19:09:46.671387+0000 mon.a (mon.0) 740 : cluster [DBG] mgrmap e25: x(active, since 5s), standbys: y 2023-09-23T19:09:47.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:47 smithi016 bash[27237]: audit 2023-09-23T19:09:46.679728+0000 mon.c (mon.2) 154 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T19:09:47.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:47 smithi016 bash[27237]: audit 2023-09-23T19:09:47.049126+0000 mon.c (mon.2) 155 : audit [DBG] from='client.? 172.21.15.16:0/1916142217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:47.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:47 smithi016 bash[33624]: cephadm 2023-09-23T19:09:45.180048+0000 mgr.x (mgr.15012) 7 : cephadm [INF] [23/Sep/2023:19:09:45] ENGINE Serving on http://172.21.15.92:8765 2023-09-23T19:09:47.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:47 smithi016 bash[33624]: cephadm 2023-09-23T19:09:45.378728+0000 mgr.x (mgr.15012) 8 : cephadm [INF] [23/Sep/2023:19:09:45] ENGINE Serving on https://172.21.15.92:7150 2023-09-23T19:09:47.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:47 smithi016 bash[33624]: cephadm 2023-09-23T19:09:45.378990+0000 mgr.x (mgr.15012) 9 : cephadm [INF] [23/Sep/2023:19:09:45] ENGINE Bus STARTED 2023-09-23T19:09:47.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:47 smithi016 bash[33624]: cluster 2023-09-23T19:09:46.671387+0000 mon.a (mon.0) 740 : cluster [DBG] mgrmap e25: x(active, since 5s), standbys: y 2023-09-23T19:09:47.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:47 smithi016 bash[33624]: audit 2023-09-23T19:09:46.679728+0000 mon.c (mon.2) 154 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T19:09:47.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:47 smithi016 bash[33624]: audit 2023-09-23T19:09:47.049126+0000 mon.c (mon.2) 155 : audit [DBG] from='client.? 172.21.15.16:0/1916142217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:47.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:47 smithi092 bash[27556]: cephadm 2023-09-23T19:09:45.180048+0000 mgr.x (mgr.15012) 7 : cephadm [INF] [23/Sep/2023:19:09:45] ENGINE Serving on http://172.21.15.92:8765 2023-09-23T19:09:47.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:47 smithi092 bash[27556]: cephadm 2023-09-23T19:09:45.378728+0000 mgr.x (mgr.15012) 8 : cephadm [INF] [23/Sep/2023:19:09:45] ENGINE Serving on https://172.21.15.92:7150 2023-09-23T19:09:47.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:47 smithi092 bash[27556]: cephadm 2023-09-23T19:09:45.378990+0000 mgr.x (mgr.15012) 9 : cephadm [INF] [23/Sep/2023:19:09:45] ENGINE Bus STARTED 2023-09-23T19:09:47.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:47 smithi092 bash[27556]: cluster 2023-09-23T19:09:46.671387+0000 mon.a (mon.0) 740 : cluster [DBG] mgrmap e25: x(active, since 5s), standbys: y 2023-09-23T19:09:47.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:47 smithi092 bash[27556]: audit 2023-09-23T19:09:46.679728+0000 mon.c (mon.2) 154 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T19:09:47.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:47 smithi092 bash[27556]: audit 2023-09-23T19:09:47.049126+0000 mon.c (mon.2) 155 : audit [DBG] from='client.? 172.21.15.16:0/1916142217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:48.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:48 smithi016 bash[33624]: cluster 2023-09-23T19:09:46.772971+0000 mgr.x (mgr.15012) 10 : cluster [DBG] pgmap v8: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 17 KiB/s rd, 329 KiB/s wr, 221 op/s 2023-09-23T19:09:48.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:48 smithi016 bash[27237]: cluster 2023-09-23T19:09:46.772971+0000 mgr.x (mgr.15012) 10 : cluster [DBG] pgmap v8: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 17 KiB/s rd, 329 KiB/s wr, 221 op/s 2023-09-23T19:09:48.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:48 smithi092 bash[27556]: cluster 2023-09-23T19:09:46.772971+0000 mgr.x (mgr.15012) 10 : cluster [DBG] pgmap v8: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 17 KiB/s rd, 329 KiB/s wr, 221 op/s 2023-09-23T19:09:49.536 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:49 smithi092 bash[50491]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:09:49] "GET /metrics HTTP/1.1" 200 33145 "" "Prometheus/2.33.4" 2023-09-23T19:09:50.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:50 smithi016 bash[27237]: cluster 2023-09-23T19:09:48.773561+0000 mgr.x (mgr.15012) 11 : cluster [DBG] pgmap v9: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 12 KiB/s rd, 234 KiB/s wr, 526 op/s 2023-09-23T19:09:50.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:50 smithi016 bash[27237]: audit 2023-09-23T19:09:49.775640+0000 mon.c (mon.2) 156 : audit [DBG] from='client.? 172.21.15.16:0/1827024134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:50.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:50 smithi016 bash[33624]: cluster 2023-09-23T19:09:48.773561+0000 mgr.x (mgr.15012) 11 : cluster [DBG] pgmap v9: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 12 KiB/s rd, 234 KiB/s wr, 526 op/s 2023-09-23T19:09:50.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:50 smithi016 bash[33624]: audit 2023-09-23T19:09:49.775640+0000 mon.c (mon.2) 156 : audit [DBG] from='client.? 172.21.15.16:0/1827024134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:50.536 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:09:50 smithi016 bash[27503]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:09:50] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T19:09:50.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:50 smithi092 bash[27556]: cluster 2023-09-23T19:09:48.773561+0000 mgr.x (mgr.15012) 11 : cluster [DBG] pgmap v9: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 12 KiB/s rd, 234 KiB/s wr, 526 op/s 2023-09-23T19:09:50.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:50 smithi092 bash[27556]: audit 2023-09-23T19:09:49.775640+0000 mon.c (mon.2) 156 : audit [DBG] from='client.? 172.21.15.16:0/1827024134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:52.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:52 smithi016 bash[27237]: cluster 2023-09-23T19:09:50.774097+0000 mgr.x (mgr.15012) 12 : cluster [DBG] pgmap v10: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 9.4 KiB/s rd, 187 KiB/s wr, 419 op/s 2023-09-23T19:09:52.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:52 smithi016 bash[27237]: audit 2023-09-23T19:09:51.284322+0000 mon.a (mon.0) 741 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:09:52.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:52 smithi016 bash[27237]: audit 2023-09-23T19:09:51.848413+0000 mon.a (mon.0) 742 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:09:52.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:52 smithi016 bash[27237]: audit 2023-09-23T19:09:51.864918+0000 mon.a (mon.0) 743 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:09:52.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:52 smithi016 bash[33624]: cluster 2023-09-23T19:09:50.774097+0000 mgr.x (mgr.15012) 12 : cluster [DBG] pgmap v10: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 9.4 KiB/s rd, 187 KiB/s wr, 419 op/s 2023-09-23T19:09:52.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:52 smithi016 bash[33624]: audit 2023-09-23T19:09:51.284322+0000 mon.a (mon.0) 741 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:09:52.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:52 smithi016 bash[33624]: audit 2023-09-23T19:09:51.848413+0000 mon.a (mon.0) 742 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:09:52.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:52 smithi016 bash[33624]: audit 2023-09-23T19:09:51.864918+0000 mon.a (mon.0) 743 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:09:52.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:52 smithi092 bash[27556]: cluster 2023-09-23T19:09:50.774097+0000 mgr.x (mgr.15012) 12 : cluster [DBG] pgmap v10: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 9.4 KiB/s rd, 187 KiB/s wr, 419 op/s 2023-09-23T19:09:52.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:52 smithi092 bash[27556]: audit 2023-09-23T19:09:51.284322+0000 mon.a (mon.0) 741 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:09:52.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:52 smithi092 bash[27556]: audit 2023-09-23T19:09:51.848413+0000 mon.a (mon.0) 742 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:09:52.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:52 smithi092 bash[27556]: audit 2023-09-23T19:09:51.864918+0000 mon.a (mon.0) 743 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:09:53.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:53 smithi016 bash[33624]: audit 2023-09-23T19:09:52.459366+0000 mon.a (mon.0) 744 : audit [DBG] from='client.? 172.21.15.16:0/1728323275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:53.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:53 smithi016 bash[33624]: audit 2023-09-23T19:09:52.770983+0000 mon.a (mon.0) 745 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:09:53.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:53 smithi016 bash[33624]: cluster 2023-09-23T19:09:52.775187+0000 mgr.x (mgr.15012) 13 : cluster [DBG] pgmap v11: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 9.1 KiB/s rd, 215 KiB/s wr, 737 op/s 2023-09-23T19:09:53.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:53 smithi016 bash[33624]: audit 2023-09-23T19:09:52.782031+0000 mon.a (mon.0) 746 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:09:53.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:53 smithi016 bash[27237]: audit 2023-09-23T19:09:52.459366+0000 mon.a (mon.0) 744 : audit [DBG] from='client.? 172.21.15.16:0/1728323275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:53.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:53 smithi016 bash[27237]: audit 2023-09-23T19:09:52.770983+0000 mon.a (mon.0) 745 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:09:53.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:53 smithi016 bash[27237]: cluster 2023-09-23T19:09:52.775187+0000 mgr.x (mgr.15012) 13 : cluster [DBG] pgmap v11: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 9.1 KiB/s rd, 215 KiB/s wr, 737 op/s 2023-09-23T19:09:53.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:53 smithi016 bash[27237]: audit 2023-09-23T19:09:52.782031+0000 mon.a (mon.0) 746 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:09:53.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:53 smithi092 bash[27556]: audit 2023-09-23T19:09:52.459366+0000 mon.a (mon.0) 744 : audit [DBG] from='client.? 172.21.15.16:0/1728323275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:53.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:53 smithi092 bash[27556]: audit 2023-09-23T19:09:52.770983+0000 mon.a (mon.0) 745 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:09:53.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:53 smithi092 bash[27556]: cluster 2023-09-23T19:09:52.775187+0000 mgr.x (mgr.15012) 13 : cluster [DBG] pgmap v11: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 9.1 KiB/s rd, 215 KiB/s wr, 737 op/s 2023-09-23T19:09:53.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:53 smithi092 bash[27556]: audit 2023-09-23T19:09:52.782031+0000 mon.a (mon.0) 746 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:09:54.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:54 smithi016 bash[27237]: audit 2023-09-23T19:09:53.207351+0000 mon.a (mon.0) 747 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:09:54.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:54 smithi016 bash[27237]: audit 2023-09-23T19:09:53.223949+0000 mon.a (mon.0) 748 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:09:54.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:54 smithi016 bash[27237]: audit 2023-09-23T19:09:53.837294+0000 mon.a (mon.0) 749 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:09:54.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:54 smithi016 bash[27237]: audit 2023-09-23T19:09:53.845906+0000 mon.a (mon.0) 750 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:09:54.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:54 smithi016 bash[33624]: audit 2023-09-23T19:09:53.207351+0000 mon.a (mon.0) 747 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:09:54.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:54 smithi016 bash[33624]: audit 2023-09-23T19:09:53.223949+0000 mon.a (mon.0) 748 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:09:54.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:54 smithi016 bash[33624]: audit 2023-09-23T19:09:53.837294+0000 mon.a (mon.0) 749 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:09:54.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:54 smithi016 bash[33624]: audit 2023-09-23T19:09:53.845906+0000 mon.a (mon.0) 750 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:09:54.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:54 smithi092 bash[27556]: audit 2023-09-23T19:09:53.207351+0000 mon.a (mon.0) 747 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:09:54.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:54 smithi092 bash[27556]: audit 2023-09-23T19:09:53.223949+0000 mon.a (mon.0) 748 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:09:54.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:54 smithi092 bash[27556]: audit 2023-09-23T19:09:53.837294+0000 mon.a (mon.0) 749 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:09:54.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:54 smithi092 bash[27556]: audit 2023-09-23T19:09:53.845906+0000 mon.a (mon.0) 750 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:09:57.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:56 smithi016 bash[27237]: cluster 2023-09-23T19:09:54.775553+0000 mgr.x (mgr.15012) 14 : cluster [DBG] pgmap v12: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 7.8 KiB/s rd, 184 KiB/s wr, 629 op/s 2023-09-23T19:09:57.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:56 smithi016 bash[27237]: audit 2023-09-23T19:09:55.104344+0000 mon.c (mon.2) 157 : audit [DBG] from='client.? 172.21.15.16:0/1688425093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:57.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:56 smithi016 bash[27237]: audit 2023-09-23T19:09:55.625566+0000 mon.a (mon.0) 751 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T19:09:57.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:56 smithi016 bash[33624]: cluster 2023-09-23T19:09:54.775553+0000 mgr.x (mgr.15012) 14 : cluster [DBG] pgmap v12: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 7.8 KiB/s rd, 184 KiB/s wr, 629 op/s 2023-09-23T19:09:57.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:56 smithi016 bash[33624]: audit 2023-09-23T19:09:55.104344+0000 mon.c (mon.2) 157 : audit [DBG] from='client.? 172.21.15.16:0/1688425093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:57.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:56 smithi016 bash[33624]: audit 2023-09-23T19:09:55.625566+0000 mon.a (mon.0) 751 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T19:09:58.222 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:57 smithi016 bash[27237]: audit 2023-09-23T19:09:55.625864+0000 mon.c (mon.2) 158 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:09:58.222 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:57 smithi016 bash[27237]: audit 2023-09-23T19:09:55.626346+0000 mon.c (mon.2) 159 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:09:58.222 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:57 smithi016 bash[27237]: audit 2023-09-23T19:09:55.626762+0000 mon.c (mon.2) 160 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:09:58.222 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:57 smithi016 bash[27237]: audit 2023-09-23T19:09:56.126059+0000 mon.c (mon.2) 161 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:09:58.222 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:57 smithi016 bash[27237]: audit 2023-09-23T19:09:56.760577+0000 mon.a (mon.0) 752 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T19:09:58.223 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:57 smithi016 bash[27237]: audit 2023-09-23T19:09:56.769487+0000 mon.c (mon.2) 162 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:09:58.223 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:57 smithi016 bash[27237]: audit 2023-09-23T19:09:56.769712+0000 mon.c (mon.2) 163 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:09:58.223 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:57 smithi016 bash[27237]: audit 2023-09-23T19:09:56.769931+0000 mon.c (mon.2) 164 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:09:58.223 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:57 smithi016 bash[27237]: cluster 2023-09-23T19:09:56.776063+0000 mgr.x (mgr.15012) 15 : cluster [DBG] pgmap v13: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 7.6 KiB/s rd, 179 KiB/s wr, 651 op/s 2023-09-23T19:09:58.223 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:57 smithi016 bash[33624]: audit 2023-09-23T19:09:55.625864+0000 mon.c (mon.2) 158 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:09:58.223 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:57 smithi016 bash[33624]: audit 2023-09-23T19:09:55.626346+0000 mon.c (mon.2) 159 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:09:58.223 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:57 smithi016 bash[33624]: audit 2023-09-23T19:09:55.626762+0000 mon.c (mon.2) 160 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:09:58.223 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:57 smithi016 bash[33624]: audit 2023-09-23T19:09:56.126059+0000 mon.c (mon.2) 161 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:09:58.223 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:57 smithi016 bash[33624]: audit 2023-09-23T19:09:56.760577+0000 mon.a (mon.0) 752 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T19:09:58.223 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:57 smithi016 bash[33624]: audit 2023-09-23T19:09:56.769487+0000 mon.c (mon.2) 162 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:09:58.224 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:57 smithi016 bash[33624]: audit 2023-09-23T19:09:56.769712+0000 mon.c (mon.2) 163 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:09:58.224 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:57 smithi016 bash[33624]: audit 2023-09-23T19:09:56.769931+0000 mon.c (mon.2) 164 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:09:58.224 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:57 smithi016 bash[33624]: cluster 2023-09-23T19:09:56.776063+0000 mgr.x (mgr.15012) 15 : cluster [DBG] pgmap v13: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 7.6 KiB/s rd, 179 KiB/s wr, 651 op/s 2023-09-23T19:09:58.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:57 smithi092 bash[27556]: cluster 2023-09-23T19:09:54.775553+0000 mgr.x (mgr.15012) 14 : cluster [DBG] pgmap v12: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 7.8 KiB/s rd, 184 KiB/s wr, 629 op/s 2023-09-23T19:09:58.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:57 smithi092 bash[27556]: audit 2023-09-23T19:09:55.104344+0000 mon.c (mon.2) 157 : audit [DBG] from='client.? 172.21.15.16:0/1688425093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:58.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:57 smithi092 bash[27556]: audit 2023-09-23T19:09:55.625566+0000 mon.a (mon.0) 751 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T19:09:58.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:57 smithi092 bash[27556]: audit 2023-09-23T19:09:55.625864+0000 mon.c (mon.2) 158 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:09:58.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:57 smithi092 bash[27556]: audit 2023-09-23T19:09:55.626346+0000 mon.c (mon.2) 159 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:09:58.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:57 smithi092 bash[27556]: audit 2023-09-23T19:09:55.626762+0000 mon.c (mon.2) 160 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:09:58.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:57 smithi092 bash[27556]: audit 2023-09-23T19:09:56.126059+0000 mon.c (mon.2) 161 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:09:58.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:57 smithi092 bash[27556]: audit 2023-09-23T19:09:56.760577+0000 mon.a (mon.0) 752 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T19:09:58.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:57 smithi092 bash[27556]: audit 2023-09-23T19:09:56.769487+0000 mon.c (mon.2) 162 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:09:58.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:57 smithi092 bash[27556]: audit 2023-09-23T19:09:56.769712+0000 mon.c (mon.2) 163 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:09:58.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:57 smithi092 bash[27556]: audit 2023-09-23T19:09:56.769931+0000 mon.c (mon.2) 164 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:09:58.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:57 smithi092 bash[27556]: cluster 2023-09-23T19:09:56.776063+0000 mgr.x (mgr.15012) 15 : cluster [DBG] pgmap v13: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 7.6 KiB/s rd, 179 KiB/s wr, 651 op/s 2023-09-23T19:09:59.368 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:09:59 smithi092 bash[50491]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:09:59] "GET /metrics HTTP/1.1" 200 36070 "" "Prometheus/2.33.4" 2023-09-23T19:09:59.368 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:59 smithi092 bash[27556]: audit 2023-09-23T19:09:56.772441+0000 mon.b (mon.1) 40 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T19:09:59.369 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:59 smithi092 bash[27556]: audit 2023-09-23T19:09:57.880539+0000 mon.b (mon.1) 41 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T19:09:59.369 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:09:59 smithi092 bash[27556]: audit 2023-09-23T19:09:58.442794+0000 mon.b (mon.1) 42 : audit [DBG] from='client.? 172.21.15.16:0/457777652' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:59.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:59 smithi016 bash[27237]: audit 2023-09-23T19:09:56.772441+0000 mon.b (mon.1) 40 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T19:09:59.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:59 smithi016 bash[27237]: audit 2023-09-23T19:09:57.880539+0000 mon.b (mon.1) 41 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T19:09:59.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:09:59 smithi016 bash[27237]: audit 2023-09-23T19:09:58.442794+0000 mon.b (mon.1) 42 : audit [DBG] from='client.? 172.21.15.16:0/457777652' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:09:59.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:59 smithi016 bash[33624]: audit 2023-09-23T19:09:56.772441+0000 mon.b (mon.1) 40 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T19:09:59.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:59 smithi016 bash[33624]: audit 2023-09-23T19:09:57.880539+0000 mon.b (mon.1) 41 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T19:09:59.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:09:59 smithi016 bash[33624]: audit 2023-09-23T19:09:58.442794+0000 mon.b (mon.1) 42 : audit [DBG] from='client.? 172.21.15.16:0/457777652' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:00.381 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:00 smithi092 bash[27556]: audit 2023-09-23T19:09:57.902649+0000 mon.c (mon.2) 165 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T19:10:00.381 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:00 smithi092 bash[27556]: cluster 2023-09-23T19:09:58.776760+0000 mgr.x (mgr.15012) 16 : cluster [DBG] pgmap v14: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 767 B/s rd, 43 KiB/s wr, 816 op/s 2023-09-23T19:10:00.381 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:00 smithi092 bash[27556]: audit 2023-09-23T19:09:59.055026+0000 mon.c (mon.2) 166 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T19:10:00.382 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:00 smithi092 bash[27556]: audit 2023-09-23T19:09:59.056620+0000 mon.c (mon.2) 167 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:10:00.382 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:00 smithi092 bash[27556]: audit 2023-09-23T19:09:59.056810+0000 mon.c (mon.2) 168 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:10:00.382 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:00 smithi092 bash[27556]: audit 2023-09-23T19:09:59.056949+0000 mon.c (mon.2) 169 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:10:00.382 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:00 smithi092 bash[27556]: cluster 2023-09-23T19:10:00.000089+0000 mon.a (mon.0) 753 : cluster [INF] overall HEALTH_OK 2023-09-23T19:10:00.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:00 smithi016 bash[27237]: audit 2023-09-23T19:09:57.902649+0000 mon.c (mon.2) 165 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T19:10:00.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:00 smithi016 bash[27237]: cluster 2023-09-23T19:09:58.776760+0000 mgr.x (mgr.15012) 16 : cluster [DBG] pgmap v14: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 767 B/s rd, 43 KiB/s wr, 816 op/s 2023-09-23T19:10:00.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:00 smithi016 bash[27237]: audit 2023-09-23T19:09:59.055026+0000 mon.c (mon.2) 166 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T19:10:00.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:00 smithi016 bash[27237]: audit 2023-09-23T19:09:59.056620+0000 mon.c (mon.2) 167 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:10:00.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:00 smithi016 bash[27237]: audit 2023-09-23T19:09:59.056810+0000 mon.c (mon.2) 168 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:10:00.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:00 smithi016 bash[27237]: audit 2023-09-23T19:09:59.056949+0000 mon.c (mon.2) 169 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:10:00.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:00 smithi016 bash[27237]: cluster 2023-09-23T19:10:00.000089+0000 mon.a (mon.0) 753 : cluster [INF] overall HEALTH_OK 2023-09-23T19:10:00.536 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:10:00 smithi016 bash[27503]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:10:00] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T19:10:00.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:00 smithi016 bash[33624]: audit 2023-09-23T19:09:57.902649+0000 mon.c (mon.2) 165 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T19:10:00.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:00 smithi016 bash[33624]: cluster 2023-09-23T19:09:58.776760+0000 mgr.x (mgr.15012) 16 : cluster [DBG] pgmap v14: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 767 B/s rd, 43 KiB/s wr, 816 op/s 2023-09-23T19:10:00.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:00 smithi016 bash[33624]: audit 2023-09-23T19:09:59.055026+0000 mon.c (mon.2) 166 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T19:10:00.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:00 smithi016 bash[33624]: audit 2023-09-23T19:09:59.056620+0000 mon.c (mon.2) 167 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:10:00.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:00 smithi016 bash[33624]: audit 2023-09-23T19:09:59.056810+0000 mon.c (mon.2) 168 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:10:00.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:00 smithi016 bash[33624]: audit 2023-09-23T19:09:59.056949+0000 mon.c (mon.2) 169 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:10:00.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:00 smithi016 bash[33624]: cluster 2023-09-23T19:10:00.000089+0000 mon.a (mon.0) 753 : cluster [INF] overall HEALTH_OK 2023-09-23T19:10:02.208 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:02 smithi016 bash[27237]: cluster 2023-09-23T19:10:00.777203+0000 mgr.x (mgr.15012) 17 : cluster [DBG] pgmap v15: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 767 B/s rd, 43 KiB/s wr, 602 op/s 2023-09-23T19:10:02.208 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:02 smithi016 bash[27237]: audit 2023-09-23T19:10:01.146950+0000 mon.a (mon.0) 754 : audit [DBG] from='client.? 172.21.15.16:0/4042146606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:02.208 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:02 smithi016 bash[33624]: cluster 2023-09-23T19:10:00.777203+0000 mgr.x (mgr.15012) 17 : cluster [DBG] pgmap v15: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 767 B/s rd, 43 KiB/s wr, 602 op/s 2023-09-23T19:10:02.208 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:02 smithi016 bash[33624]: audit 2023-09-23T19:10:01.146950+0000 mon.a (mon.0) 754 : audit [DBG] from='client.? 172.21.15.16:0/4042146606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:02.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:02 smithi092 bash[27556]: cluster 2023-09-23T19:10:00.777203+0000 mgr.x (mgr.15012) 17 : cluster [DBG] pgmap v15: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 767 B/s rd, 43 KiB/s wr, 602 op/s 2023-09-23T19:10:02.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:02 smithi092 bash[27556]: audit 2023-09-23T19:10:01.146950+0000 mon.a (mon.0) 754 : audit [DBG] from='client.? 172.21.15.16:0/4042146606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:02.536 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:10:02 smithi092 bash[48334]: ts=2023-09-23T19:10:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi092.front.sepia.ceph.com:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.16:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T19:10:04.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:04 smithi092 bash[27556]: cluster 2023-09-23T19:10:02.777963+0000 mgr.x (mgr.15012) 18 : cluster [DBG] pgmap v16: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 921 op/s 2023-09-23T19:10:04.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:04 smithi092 bash[27556]: audit 2023-09-23T19:10:03.871953+0000 mon.a (mon.0) 755 : audit [DBG] from='client.? 172.21.15.16:0/1604102759' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:04.538 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:04 smithi016 bash[27237]: cluster 2023-09-23T19:10:02.777963+0000 mgr.x (mgr.15012) 18 : cluster [DBG] pgmap v16: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 921 op/s 2023-09-23T19:10:04.538 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:04 smithi016 bash[27237]: audit 2023-09-23T19:10:03.871953+0000 mon.a (mon.0) 755 : audit [DBG] from='client.? 172.21.15.16:0/1604102759' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:04.538 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:04 smithi016 bash[33624]: cluster 2023-09-23T19:10:02.777963+0000 mgr.x (mgr.15012) 18 : cluster [DBG] pgmap v16: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 921 op/s 2023-09-23T19:10:04.538 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:04 smithi016 bash[33624]: audit 2023-09-23T19:10:03.871953+0000 mon.a (mon.0) 755 : audit [DBG] from='client.? 172.21.15.16:0/1604102759' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:05.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:05 smithi016 bash[33624]: audit 2023-09-23T19:10:04.319691+0000 mon.a (mon.0) 756 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:05.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:05 smithi016 bash[33624]: audit 2023-09-23T19:10:04.332569+0000 mon.a (mon.0) 757 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:05.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:05 smithi016 bash[33624]: audit 2023-09-23T19:10:04.338210+0000 mon.c (mon.2) 170 : audit [INF] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:10:05.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:05 smithi016 bash[33624]: audit 2023-09-23T19:10:04.344161+0000 mon.a (mon.0) 758 : audit [INF] from='mgr.15012 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:10:05.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:05 smithi016 bash[33624]: audit 2023-09-23T19:10:04.345866+0000 mon.c (mon.2) 171 : audit [INF] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:10:05.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:05 smithi016 bash[33624]: audit 2023-09-23T19:10:04.346530+0000 mon.a (mon.0) 759 : audit [INF] from='mgr.15012 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:10:05.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:05 smithi016 bash[33624]: audit 2023-09-23T19:10:04.347889+0000 mon.c (mon.2) 172 : audit [INF] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:10:05.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:05 smithi016 bash[33624]: audit 2023-09-23T19:10:04.348589+0000 mon.a (mon.0) 760 : audit [INF] from='mgr.15012 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:10:05.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:05 smithi016 bash[33624]: audit 2023-09-23T19:10:04.349931+0000 mon.c (mon.2) 173 : audit [INF] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:10:05.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:05 smithi016 bash[33624]: audit 2023-09-23T19:10:04.350563+0000 mon.a (mon.0) 761 : audit [INF] from='mgr.15012 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:10:05.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:05 smithi016 bash[33624]: cephadm 2023-09-23T19:10:04.358420+0000 mgr.x (mgr.15012) 19 : cephadm [INF] Adjusting osd_memory_target on smithi092 to 2524M 2023-09-23T19:10:05.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:05 smithi016 bash[33624]: audit 2023-09-23T19:10:04.369796+0000 mon.a (mon.0) 762 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:05.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:05 smithi016 bash[33624]: cluster 2023-09-23T19:10:04.778397+0000 mgr.x (mgr.15012) 20 : cluster [DBG] pgmap v17: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 612 op/s 2023-09-23T19:10:05.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:05 smithi016 bash[27237]: audit 2023-09-23T19:10:04.319691+0000 mon.a (mon.0) 756 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:05.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:05 smithi016 bash[27237]: audit 2023-09-23T19:10:04.332569+0000 mon.a (mon.0) 757 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:05.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:05 smithi016 bash[27237]: audit 2023-09-23T19:10:04.338210+0000 mon.c (mon.2) 170 : audit [INF] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:10:05.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:05 smithi016 bash[27237]: audit 2023-09-23T19:10:04.344161+0000 mon.a (mon.0) 758 : audit [INF] from='mgr.15012 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:10:05.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:05 smithi016 bash[27237]: audit 2023-09-23T19:10:04.345866+0000 mon.c (mon.2) 171 : audit [INF] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:10:05.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:05 smithi016 bash[27237]: audit 2023-09-23T19:10:04.346530+0000 mon.a (mon.0) 759 : audit [INF] from='mgr.15012 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:10:05.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:05 smithi016 bash[27237]: audit 2023-09-23T19:10:04.347889+0000 mon.c (mon.2) 172 : audit [INF] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:10:05.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:05 smithi016 bash[27237]: audit 2023-09-23T19:10:04.348589+0000 mon.a (mon.0) 760 : audit [INF] from='mgr.15012 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:10:05.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:05 smithi016 bash[27237]: audit 2023-09-23T19:10:04.349931+0000 mon.c (mon.2) 173 : audit [INF] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:10:05.788 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:05 smithi016 bash[27237]: audit 2023-09-23T19:10:04.350563+0000 mon.a (mon.0) 761 : audit [INF] from='mgr.15012 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:10:05.788 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:05 smithi016 bash[27237]: cephadm 2023-09-23T19:10:04.358420+0000 mgr.x (mgr.15012) 19 : cephadm [INF] Adjusting osd_memory_target on smithi092 to 2524M 2023-09-23T19:10:05.788 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:05 smithi016 bash[27237]: audit 2023-09-23T19:10:04.369796+0000 mon.a (mon.0) 762 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:05.788 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:05 smithi016 bash[27237]: cluster 2023-09-23T19:10:04.778397+0000 mgr.x (mgr.15012) 20 : cluster [DBG] pgmap v17: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 612 op/s 2023-09-23T19:10:05.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:05 smithi092 bash[27556]: audit 2023-09-23T19:10:04.319691+0000 mon.a (mon.0) 756 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:05.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:05 smithi092 bash[27556]: audit 2023-09-23T19:10:04.332569+0000 mon.a (mon.0) 757 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:05.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:05 smithi092 bash[27556]: audit 2023-09-23T19:10:04.338210+0000 mon.c (mon.2) 170 : audit [INF] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:10:05.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:05 smithi092 bash[27556]: audit 2023-09-23T19:10:04.344161+0000 mon.a (mon.0) 758 : audit [INF] from='mgr.15012 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:10:05.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:05 smithi092 bash[27556]: audit 2023-09-23T19:10:04.345866+0000 mon.c (mon.2) 171 : audit [INF] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:10:05.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:05 smithi092 bash[27556]: audit 2023-09-23T19:10:04.346530+0000 mon.a (mon.0) 759 : audit [INF] from='mgr.15012 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:10:05.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:05 smithi092 bash[27556]: audit 2023-09-23T19:10:04.347889+0000 mon.c (mon.2) 172 : audit [INF] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:10:05.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:05 smithi092 bash[27556]: audit 2023-09-23T19:10:04.348589+0000 mon.a (mon.0) 760 : audit [INF] from='mgr.15012 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:10:05.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:05 smithi092 bash[27556]: audit 2023-09-23T19:10:04.349931+0000 mon.c (mon.2) 173 : audit [INF] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:10:05.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:05 smithi092 bash[27556]: audit 2023-09-23T19:10:04.350563+0000 mon.a (mon.0) 761 : audit [INF] from='mgr.15012 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:10:05.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:05 smithi092 bash[27556]: cephadm 2023-09-23T19:10:04.358420+0000 mgr.x (mgr.15012) 19 : cephadm [INF] Adjusting osd_memory_target on smithi092 to 2524M 2023-09-23T19:10:05.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:05 smithi092 bash[27556]: audit 2023-09-23T19:10:04.369796+0000 mon.a (mon.0) 762 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:05.790 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:05 smithi092 bash[27556]: cluster 2023-09-23T19:10:04.778397+0000 mgr.x (mgr.15012) 20 : cluster [DBG] pgmap v17: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 612 op/s 2023-09-23T19:10:07.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:06 smithi016 bash[27237]: audit 2023-09-23T19:10:05.814385+0000 mon.a (mon.0) 763 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:07.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:06 smithi016 bash[27237]: audit 2023-09-23T19:10:05.823787+0000 mon.a (mon.0) 764 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:07.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:06 smithi016 bash[27237]: audit 2023-09-23T19:10:05.828643+0000 mon.c (mon.2) 174 : audit [INF] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:10:07.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:06 smithi016 bash[27237]: audit 2023-09-23T19:10:05.829183+0000 mon.a (mon.0) 765 : audit [INF] from='mgr.15012 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:10:07.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:06 smithi016 bash[27237]: audit 2023-09-23T19:10:05.835257+0000 mon.c (mon.2) 175 : audit [INF] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:10:07.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:06 smithi016 bash[27237]: audit 2023-09-23T19:10:05.835897+0000 mon.a (mon.0) 766 : audit [INF] from='mgr.15012 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:10:07.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:06 smithi016 bash[27237]: audit 2023-09-23T19:10:05.837749+0000 mon.c (mon.2) 176 : audit [INF] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:10:07.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:06 smithi016 bash[27237]: audit 2023-09-23T19:10:05.838159+0000 mon.a (mon.0) 767 : audit [INF] from='mgr.15012 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:10:07.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:06 smithi016 bash[27237]: audit 2023-09-23T19:10:05.840046+0000 mon.c (mon.2) 177 : audit [INF] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:10:07.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:06 smithi016 bash[27237]: audit 2023-09-23T19:10:05.840892+0000 mon.a (mon.0) 768 : audit [INF] from='mgr.15012 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:10:07.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:06 smithi016 bash[27237]: cephadm 2023-09-23T19:10:05.848669+0000 mgr.x (mgr.15012) 21 : cephadm [INF] Adjusting osd_memory_target on smithi016 to 2525M 2023-09-23T19:10:07.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:06 smithi016 bash[27237]: audit 2023-09-23T19:10:05.850408+0000 mon.a (mon.0) 769 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:07.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:06 smithi016 bash[27237]: audit 2023-09-23T19:10:05.858094+0000 mon.c (mon.2) 178 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:10:07.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:06 smithi016 bash[27237]: audit 2023-09-23T19:10:05.860058+0000 mon.c (mon.2) 179 : audit [INF] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:10:07.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:06 smithi016 bash[27237]: cephadm 2023-09-23T19:10:05.868843+0000 mgr.x (mgr.15012) 22 : cephadm [INF] Updating smithi016:/etc/ceph/ceph.conf 2023-09-23T19:10:07.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:06 smithi016 bash[27237]: cephadm 2023-09-23T19:10:05.869384+0000 mgr.x (mgr.15012) 23 : cephadm [INF] Updating smithi092:/etc/ceph/ceph.conf 2023-09-23T19:10:07.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:06 smithi016 bash[27237]: cephadm 2023-09-23T19:10:06.083631+0000 mgr.x (mgr.15012) 24 : cephadm [INF] Updating smithi016:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:10:07.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:06 smithi016 bash[27237]: cephadm 2023-09-23T19:10:06.083971+0000 mgr.x (mgr.15012) 25 : cephadm [INF] Updating smithi092:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:10:07.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:06 smithi016 bash[27237]: audit 2023-09-23T19:10:06.551391+0000 mon.c (mon.2) 180 : audit [DBG] from='client.? 172.21.15.16:0/3473297084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:07.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:06 smithi016 bash[27237]: audit 2023-09-23T19:10:06.803525+0000 mon.a (mon.0) 770 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:07.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:06 smithi016 bash[27237]: audit 2023-09-23T19:10:06.807888+0000 mon.a (mon.0) 771 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:07.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:06 smithi016 bash[33624]: audit 2023-09-23T19:10:05.814385+0000 mon.a (mon.0) 763 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:07.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:06 smithi016 bash[33624]: audit 2023-09-23T19:10:05.823787+0000 mon.a (mon.0) 764 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:07.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:06 smithi016 bash[33624]: audit 2023-09-23T19:10:05.828643+0000 mon.c (mon.2) 174 : audit [INF] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:10:07.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:06 smithi016 bash[33624]: audit 2023-09-23T19:10:05.829183+0000 mon.a (mon.0) 765 : audit [INF] from='mgr.15012 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:10:07.288 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:06 smithi016 bash[33624]: audit 2023-09-23T19:10:05.835257+0000 mon.c (mon.2) 175 : audit [INF] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:10:07.288 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:06 smithi016 bash[33624]: audit 2023-09-23T19:10:05.835897+0000 mon.a (mon.0) 766 : audit [INF] from='mgr.15012 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:10:07.288 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:06 smithi016 bash[33624]: audit 2023-09-23T19:10:05.837749+0000 mon.c (mon.2) 176 : audit [INF] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:10:07.288 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:06 smithi016 bash[33624]: audit 2023-09-23T19:10:05.838159+0000 mon.a (mon.0) 767 : audit [INF] from='mgr.15012 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:10:07.288 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:06 smithi016 bash[33624]: audit 2023-09-23T19:10:05.840046+0000 mon.c (mon.2) 177 : audit [INF] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:10:07.288 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:06 smithi016 bash[33624]: audit 2023-09-23T19:10:05.840892+0000 mon.a (mon.0) 768 : audit [INF] from='mgr.15012 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:10:07.288 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:06 smithi016 bash[33624]: cephadm 2023-09-23T19:10:05.848669+0000 mgr.x (mgr.15012) 21 : cephadm [INF] Adjusting osd_memory_target on smithi016 to 2525M 2023-09-23T19:10:07.288 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:06 smithi016 bash[33624]: audit 2023-09-23T19:10:05.850408+0000 mon.a (mon.0) 769 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:07.288 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:06 smithi016 bash[33624]: audit 2023-09-23T19:10:05.858094+0000 mon.c (mon.2) 178 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:10:07.288 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:06 smithi016 bash[33624]: audit 2023-09-23T19:10:05.860058+0000 mon.c (mon.2) 179 : audit [INF] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:10:07.288 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:06 smithi016 bash[33624]: cephadm 2023-09-23T19:10:05.868843+0000 mgr.x (mgr.15012) 22 : cephadm [INF] Updating smithi016:/etc/ceph/ceph.conf 2023-09-23T19:10:07.289 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:06 smithi016 bash[33624]: cephadm 2023-09-23T19:10:05.869384+0000 mgr.x (mgr.15012) 23 : cephadm [INF] Updating smithi092:/etc/ceph/ceph.conf 2023-09-23T19:10:07.289 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:06 smithi016 bash[33624]: cephadm 2023-09-23T19:10:06.083631+0000 mgr.x (mgr.15012) 24 : cephadm [INF] Updating smithi016:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:10:07.289 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:06 smithi016 bash[33624]: cephadm 2023-09-23T19:10:06.083971+0000 mgr.x (mgr.15012) 25 : cephadm [INF] Updating smithi092:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:10:07.289 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:06 smithi016 bash[33624]: audit 2023-09-23T19:10:06.551391+0000 mon.c (mon.2) 180 : audit [DBG] from='client.? 172.21.15.16:0/3473297084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:07.289 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:06 smithi016 bash[33624]: audit 2023-09-23T19:10:06.803525+0000 mon.a (mon.0) 770 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:07.289 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:06 smithi016 bash[33624]: audit 2023-09-23T19:10:06.807888+0000 mon.a (mon.0) 771 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:07.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:06 smithi092 bash[27556]: audit 2023-09-23T19:10:05.814385+0000 mon.a (mon.0) 763 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:07.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:06 smithi092 bash[27556]: audit 2023-09-23T19:10:05.823787+0000 mon.a (mon.0) 764 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:07.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:06 smithi092 bash[27556]: audit 2023-09-23T19:10:05.828643+0000 mon.c (mon.2) 174 : audit [INF] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:10:07.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:06 smithi092 bash[27556]: audit 2023-09-23T19:10:05.829183+0000 mon.a (mon.0) 765 : audit [INF] from='mgr.15012 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:10:07.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:06 smithi092 bash[27556]: audit 2023-09-23T19:10:05.835257+0000 mon.c (mon.2) 175 : audit [INF] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:10:07.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:06 smithi092 bash[27556]: audit 2023-09-23T19:10:05.835897+0000 mon.a (mon.0) 766 : audit [INF] from='mgr.15012 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:10:07.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:06 smithi092 bash[27556]: audit 2023-09-23T19:10:05.837749+0000 mon.c (mon.2) 176 : audit [INF] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:10:07.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:06 smithi092 bash[27556]: audit 2023-09-23T19:10:05.838159+0000 mon.a (mon.0) 767 : audit [INF] from='mgr.15012 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:10:07.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:06 smithi092 bash[27556]: audit 2023-09-23T19:10:05.840046+0000 mon.c (mon.2) 177 : audit [INF] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:10:07.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:06 smithi092 bash[27556]: audit 2023-09-23T19:10:05.840892+0000 mon.a (mon.0) 768 : audit [INF] from='mgr.15012 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:10:07.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:06 smithi092 bash[27556]: cephadm 2023-09-23T19:10:05.848669+0000 mgr.x (mgr.15012) 21 : cephadm [INF] Adjusting osd_memory_target on smithi016 to 2525M 2023-09-23T19:10:07.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:06 smithi092 bash[27556]: audit 2023-09-23T19:10:05.850408+0000 mon.a (mon.0) 769 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:07.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:06 smithi092 bash[27556]: audit 2023-09-23T19:10:05.858094+0000 mon.c (mon.2) 178 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:10:07.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:06 smithi092 bash[27556]: audit 2023-09-23T19:10:05.860058+0000 mon.c (mon.2) 179 : audit [INF] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:10:07.291 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:06 smithi092 bash[27556]: cephadm 2023-09-23T19:10:05.868843+0000 mgr.x (mgr.15012) 22 : cephadm [INF] Updating smithi016:/etc/ceph/ceph.conf 2023-09-23T19:10:07.291 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:06 smithi092 bash[27556]: cephadm 2023-09-23T19:10:05.869384+0000 mgr.x (mgr.15012) 23 : cephadm [INF] Updating smithi092:/etc/ceph/ceph.conf 2023-09-23T19:10:07.291 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:06 smithi092 bash[27556]: cephadm 2023-09-23T19:10:06.083631+0000 mgr.x (mgr.15012) 24 : cephadm [INF] Updating smithi016:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:10:07.291 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:06 smithi092 bash[27556]: cephadm 2023-09-23T19:10:06.083971+0000 mgr.x (mgr.15012) 25 : cephadm [INF] Updating smithi092:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:10:07.291 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:06 smithi092 bash[27556]: audit 2023-09-23T19:10:06.551391+0000 mon.c (mon.2) 180 : audit [DBG] from='client.? 172.21.15.16:0/3473297084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:07.291 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:06 smithi092 bash[27556]: audit 2023-09-23T19:10:06.803525+0000 mon.a (mon.0) 770 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:07.291 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:06 smithi092 bash[27556]: audit 2023-09-23T19:10:06.807888+0000 mon.a (mon.0) 771 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:07.730 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:07 smithi016 systemd[1]: Stopping Ceph node-exporter.a for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7... 2023-09-23T19:10:07.731 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:07 smithi016 bash[69521]: Error response from daemon: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-node-exporter.a 2023-09-23T19:10:08.069 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:07 smithi016 bash[27237]: cephadm 2023-09-23T19:10:06.307539+0000 mgr.x (mgr.15012) 26 : cephadm [INF] Updating smithi016:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:10:08.069 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:07 smithi016 bash[27237]: cephadm 2023-09-23T19:10:06.311175+0000 mgr.x (mgr.15012) 27 : cephadm [INF] Updating smithi092:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:10:08.069 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:07 smithi016 bash[27237]: cephadm 2023-09-23T19:10:06.583661+0000 mgr.x (mgr.15012) 28 : cephadm [INF] Updating smithi092:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:10:08.069 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:07 smithi016 bash[27237]: cephadm 2023-09-23T19:10:06.583976+0000 mgr.x (mgr.15012) 29 : cephadm [INF] Updating smithi016:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:10:08.069 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:07 smithi016 bash[27237]: cluster 2023-09-23T19:10:06.778992+0000 mgr.x (mgr.15012) 30 : cluster [DBG] pgmap v18: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 648 op/s 2023-09-23T19:10:08.069 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:07 smithi016 bash[27237]: audit 2023-09-23T19:10:06.816071+0000 mon.a (mon.0) 772 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:08.069 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:07 smithi016 bash[27237]: audit 2023-09-23T19:10:06.823793+0000 mon.a (mon.0) 773 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:08.069 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:07 smithi016 bash[27237]: audit 2023-09-23T19:10:06.848190+0000 mon.a (mon.0) 774 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:08.069 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:07 smithi016 bash[27237]: audit 2023-09-23T19:10:06.931082+0000 mon.a (mon.0) 775 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:08.069 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:07 smithi016 bash[27237]: audit 2023-09-23T19:10:06.944272+0000 mon.a (mon.0) 776 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:08.070 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:07 smithi016 bash[27237]: audit 2023-09-23T19:10:06.953101+0000 mon.a (mon.0) 777 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:08.070 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:07 smithi016 bash[27237]: audit 2023-09-23T19:10:06.959312+0000 mon.a (mon.0) 778 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:08.070 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:07 smithi016 bash[27237]: cephadm 2023-09-23T19:10:06.969974+0000 mgr.x (mgr.15012) 31 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2023-09-23T19:10:08.070 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:07 smithi016 bash[27237]: cephadm 2023-09-23T19:10:06.970844+0000 mgr.x (mgr.15012) 32 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi016 2023-09-23T19:10:08.070 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:07 smithi016 bash[33624]: cephadm 2023-09-23T19:10:06.307539+0000 mgr.x (mgr.15012) 26 : cephadm [INF] Updating smithi016:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:10:08.070 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:07 smithi016 bash[33624]: cephadm 2023-09-23T19:10:06.311175+0000 mgr.x (mgr.15012) 27 : cephadm [INF] Updating smithi092:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:10:08.070 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:07 smithi016 bash[33624]: cephadm 2023-09-23T19:10:06.583661+0000 mgr.x (mgr.15012) 28 : cephadm [INF] Updating smithi092:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:10:08.070 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:07 smithi016 bash[33624]: cephadm 2023-09-23T19:10:06.583976+0000 mgr.x (mgr.15012) 29 : cephadm [INF] Updating smithi016:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:10:08.071 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:07 smithi016 bash[33624]: cluster 2023-09-23T19:10:06.778992+0000 mgr.x (mgr.15012) 30 : cluster [DBG] pgmap v18: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 648 op/s 2023-09-23T19:10:08.071 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:07 smithi016 bash[33624]: audit 2023-09-23T19:10:06.816071+0000 mon.a (mon.0) 772 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:08.071 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:07 smithi016 bash[33624]: audit 2023-09-23T19:10:06.823793+0000 mon.a (mon.0) 773 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:08.071 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:07 smithi016 bash[33624]: audit 2023-09-23T19:10:06.848190+0000 mon.a (mon.0) 774 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:08.071 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:07 smithi016 bash[33624]: audit 2023-09-23T19:10:06.931082+0000 mon.a (mon.0) 775 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:08.071 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:07 smithi016 bash[33624]: audit 2023-09-23T19:10:06.944272+0000 mon.a (mon.0) 776 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:08.071 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:07 smithi016 bash[33624]: audit 2023-09-23T19:10:06.953101+0000 mon.a (mon.0) 777 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:08.071 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:07 smithi016 bash[33624]: audit 2023-09-23T19:10:06.959312+0000 mon.a (mon.0) 778 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:08.071 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:07 smithi016 bash[33624]: cephadm 2023-09-23T19:10:06.969974+0000 mgr.x (mgr.15012) 31 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2023-09-23T19:10:08.071 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:07 smithi016 bash[33624]: cephadm 2023-09-23T19:10:06.970844+0000 mgr.x (mgr.15012) 32 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi016 2023-09-23T19:10:08.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:07 smithi092 bash[27556]: cephadm 2023-09-23T19:10:06.307539+0000 mgr.x (mgr.15012) 26 : cephadm [INF] Updating smithi016:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:10:08.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:07 smithi092 bash[27556]: cephadm 2023-09-23T19:10:06.311175+0000 mgr.x (mgr.15012) 27 : cephadm [INF] Updating smithi092:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:10:08.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:07 smithi092 bash[27556]: cephadm 2023-09-23T19:10:06.583661+0000 mgr.x (mgr.15012) 28 : cephadm [INF] Updating smithi092:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:10:08.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:07 smithi092 bash[27556]: cephadm 2023-09-23T19:10:06.583976+0000 mgr.x (mgr.15012) 29 : cephadm [INF] Updating smithi016:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:10:08.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:07 smithi092 bash[27556]: cluster 2023-09-23T19:10:06.778992+0000 mgr.x (mgr.15012) 30 : cluster [DBG] pgmap v18: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 648 op/s 2023-09-23T19:10:08.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:07 smithi092 bash[27556]: audit 2023-09-23T19:10:06.816071+0000 mon.a (mon.0) 772 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:08.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:07 smithi092 bash[27556]: audit 2023-09-23T19:10:06.823793+0000 mon.a (mon.0) 773 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:08.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:07 smithi092 bash[27556]: audit 2023-09-23T19:10:06.848190+0000 mon.a (mon.0) 774 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:08.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:07 smithi092 bash[27556]: audit 2023-09-23T19:10:06.931082+0000 mon.a (mon.0) 775 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:08.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:07 smithi092 bash[27556]: audit 2023-09-23T19:10:06.944272+0000 mon.a (mon.0) 776 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:08.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:07 smithi092 bash[27556]: audit 2023-09-23T19:10:06.953101+0000 mon.a (mon.0) 777 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:08.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:07 smithi092 bash[27556]: audit 2023-09-23T19:10:06.959312+0000 mon.a (mon.0) 778 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:08.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:07 smithi092 bash[27556]: cephadm 2023-09-23T19:10:06.969974+0000 mgr.x (mgr.15012) 31 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2023-09-23T19:10:08.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:07 smithi092 bash[27556]: cephadm 2023-09-23T19:10:06.970844+0000 mgr.x (mgr.15012) 32 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi016 2023-09-23T19:10:08.728 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:08 smithi016 bash[69548]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-node-exporter-a 2023-09-23T19:10:08.728 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:08 smithi016 systemd[1]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-09-23T19:10:08.728 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:08 smithi016 bash[69609]: Error response from daemon: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-node-exporter-a 2023-09-23T19:10:08.729 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:08 smithi016 systemd[1]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@node-exporter.a.service: Failed with result 'exit-code'. 2023-09-23T19:10:08.729 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:08 smithi016 systemd[1]: Stopped Ceph node-exporter.a for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7. 2023-09-23T19:10:08.729 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:08 smithi016 systemd[1]: Started Ceph node-exporter.a for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7. 2023-09-23T19:10:09.535 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:10:09 smithi092 bash[50491]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:10:09] "GET /metrics HTTP/1.1" 200 36070 "" "Prometheus/2.33.4" 2023-09-23T19:10:09.944 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[27237]: audit 2023-09-23T19:10:08.681519+0000 mon.a (mon.0) 779 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:09.944 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[27237]: audit 2023-09-23T19:10:08.694657+0000 mon.a (mon.0) 780 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:09.944 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[27237]: cephadm 2023-09-23T19:10:08.752900+0000 mgr.x (mgr.15012) 33 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T19:10:09.944 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[27237]: cephadm 2023-09-23T19:10:08.770126+0000 mgr.x (mgr.15012) 34 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi016 2023-09-23T19:10:09.944 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[27237]: cluster 2023-09-23T19:10:08.779769+0000 mgr.x (mgr.15012) 35 : cluster [DBG] pgmap v19: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 885 op/s 2023-09-23T19:10:09.944 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[27237]: audit 2023-09-23T19:10:09.286412+0000 mon.c (mon.2) 181 : audit [DBG] from='client.? 172.21.15.16:0/2236367383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:09.944 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[33624]: audit 2023-09-23T19:10:08.681519+0000 mon.a (mon.0) 779 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:09.944 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[33624]: audit 2023-09-23T19:10:08.694657+0000 mon.a (mon.0) 780 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:09.945 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[33624]: cephadm 2023-09-23T19:10:08.752900+0000 mgr.x (mgr.15012) 33 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T19:10:09.945 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[33624]: cephadm 2023-09-23T19:10:08.770126+0000 mgr.x (mgr.15012) 34 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi016 2023-09-23T19:10:09.945 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[33624]: cluster 2023-09-23T19:10:08.779769+0000 mgr.x (mgr.15012) 35 : cluster [DBG] pgmap v19: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 885 op/s 2023-09-23T19:10:09.945 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[33624]: audit 2023-09-23T19:10:09.286412+0000 mon.c (mon.2) 181 : audit [DBG] from='client.? 172.21.15.16:0/2236367383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:10.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:09 smithi092 bash[27556]: audit 2023-09-23T19:10:08.681519+0000 mon.a (mon.0) 779 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:10.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:09 smithi092 bash[27556]: audit 2023-09-23T19:10:08.694657+0000 mon.a (mon.0) 780 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:10.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:09 smithi092 bash[27556]: cephadm 2023-09-23T19:10:08.752900+0000 mgr.x (mgr.15012) 33 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T19:10:10.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:09 smithi092 bash[27556]: cephadm 2023-09-23T19:10:08.770126+0000 mgr.x (mgr.15012) 34 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi016 2023-09-23T19:10:10.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:09 smithi092 bash[27556]: cluster 2023-09-23T19:10:08.779769+0000 mgr.x (mgr.15012) 35 : cluster [DBG] pgmap v19: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 885 op/s 2023-09-23T19:10:10.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:09 smithi092 bash[27556]: audit 2023-09-23T19:10:09.286412+0000 mon.c (mon.2) 181 : audit [DBG] from='client.? 172.21.15.16:0/2236367383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:10.286 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:10:10 smithi016 bash[27503]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:10:10] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T19:10:10.286 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[69632]: ts=2023-09-23T19:10:09.940Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-09-23T19:10:10.286 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[69632]: ts=2023-09-23T19:10:09.940Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-09-23T19:10:10.286 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[69632]: ts=2023-09-23T19:10:09.942Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-09-23T19:10:10.286 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[69632]: ts=2023-09-23T19:10:09.942Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-09-23T19:10:10.286 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[69632]: ts=2023-09-23T19:10:09.944Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-09-23T19:10:10.286 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[69632]: ts=2023-09-23T19:10:09.944Z caller=node_exporter.go:115 level=info collector=arp 2023-09-23T19:10:10.287 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[69632]: ts=2023-09-23T19:10:09.944Z caller=node_exporter.go:115 level=info collector=bcache 2023-09-23T19:10:10.287 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[69632]: ts=2023-09-23T19:10:09.944Z caller=node_exporter.go:115 level=info collector=bonding 2023-09-23T19:10:10.287 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[69632]: ts=2023-09-23T19:10:09.944Z caller=node_exporter.go:115 level=info collector=btrfs 2023-09-23T19:10:10.287 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[69632]: ts=2023-09-23T19:10:09.944Z caller=node_exporter.go:115 level=info collector=conntrack 2023-09-23T19:10:10.287 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[69632]: ts=2023-09-23T19:10:09.944Z caller=node_exporter.go:115 level=info collector=cpu 2023-09-23T19:10:10.287 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[69632]: ts=2023-09-23T19:10:09.944Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-09-23T19:10:10.287 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[69632]: ts=2023-09-23T19:10:09.944Z caller=node_exporter.go:115 level=info collector=diskstats 2023-09-23T19:10:10.287 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[69632]: ts=2023-09-23T19:10:09.944Z caller=node_exporter.go:115 level=info collector=dmi 2023-09-23T19:10:10.287 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[69632]: ts=2023-09-23T19:10:09.944Z caller=node_exporter.go:115 level=info collector=edac 2023-09-23T19:10:10.287 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[69632]: ts=2023-09-23T19:10:09.944Z caller=node_exporter.go:115 level=info collector=entropy 2023-09-23T19:10:10.287 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[69632]: ts=2023-09-23T19:10:09.944Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-09-23T19:10:10.287 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[69632]: ts=2023-09-23T19:10:09.944Z caller=node_exporter.go:115 level=info collector=filefd 2023-09-23T19:10:10.288 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[69632]: ts=2023-09-23T19:10:09.944Z caller=node_exporter.go:115 level=info collector=filesystem 2023-09-23T19:10:10.288 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[69632]: ts=2023-09-23T19:10:09.944Z caller=node_exporter.go:115 level=info collector=hwmon 2023-09-23T19:10:10.288 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[69632]: ts=2023-09-23T19:10:09.944Z caller=node_exporter.go:115 level=info collector=infiniband 2023-09-23T19:10:10.288 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[69632]: ts=2023-09-23T19:10:09.944Z caller=node_exporter.go:115 level=info collector=ipvs 2023-09-23T19:10:10.288 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[69632]: ts=2023-09-23T19:10:09.944Z caller=node_exporter.go:115 level=info collector=loadavg 2023-09-23T19:10:10.288 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[69632]: ts=2023-09-23T19:10:09.944Z caller=node_exporter.go:115 level=info collector=mdadm 2023-09-23T19:10:10.288 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[69632]: ts=2023-09-23T19:10:09.944Z caller=node_exporter.go:115 level=info collector=meminfo 2023-09-23T19:10:10.288 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[69632]: ts=2023-09-23T19:10:09.944Z caller=node_exporter.go:115 level=info collector=netclass 2023-09-23T19:10:10.288 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[69632]: ts=2023-09-23T19:10:09.944Z caller=node_exporter.go:115 level=info collector=netdev 2023-09-23T19:10:10.288 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[69632]: ts=2023-09-23T19:10:09.944Z caller=node_exporter.go:115 level=info collector=netstat 2023-09-23T19:10:10.288 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[69632]: ts=2023-09-23T19:10:09.944Z caller=node_exporter.go:115 level=info collector=nfs 2023-09-23T19:10:10.288 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[69632]: ts=2023-09-23T19:10:09.944Z caller=node_exporter.go:115 level=info collector=nfsd 2023-09-23T19:10:10.289 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[69632]: ts=2023-09-23T19:10:09.944Z caller=node_exporter.go:115 level=info collector=nvme 2023-09-23T19:10:10.289 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[69632]: ts=2023-09-23T19:10:09.944Z caller=node_exporter.go:115 level=info collector=os 2023-09-23T19:10:10.289 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[69632]: ts=2023-09-23T19:10:09.944Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-09-23T19:10:10.289 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[69632]: ts=2023-09-23T19:10:09.944Z caller=node_exporter.go:115 level=info collector=pressure 2023-09-23T19:10:10.289 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[69632]: ts=2023-09-23T19:10:09.944Z caller=node_exporter.go:115 level=info collector=rapl 2023-09-23T19:10:10.289 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[69632]: ts=2023-09-23T19:10:09.944Z caller=node_exporter.go:115 level=info collector=schedstat 2023-09-23T19:10:10.289 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[69632]: ts=2023-09-23T19:10:09.944Z caller=node_exporter.go:115 level=info collector=sockstat 2023-09-23T19:10:10.289 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[69632]: ts=2023-09-23T19:10:09.944Z caller=node_exporter.go:115 level=info collector=softnet 2023-09-23T19:10:10.289 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[69632]: ts=2023-09-23T19:10:09.944Z caller=node_exporter.go:115 level=info collector=stat 2023-09-23T19:10:10.289 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[69632]: ts=2023-09-23T19:10:09.944Z caller=node_exporter.go:115 level=info collector=tapestats 2023-09-23T19:10:10.289 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[69632]: ts=2023-09-23T19:10:09.944Z caller=node_exporter.go:115 level=info collector=textfile 2023-09-23T19:10:10.289 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[69632]: ts=2023-09-23T19:10:09.944Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-09-23T19:10:10.289 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[69632]: ts=2023-09-23T19:10:09.944Z caller=node_exporter.go:115 level=info collector=time 2023-09-23T19:10:10.290 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[69632]: ts=2023-09-23T19:10:09.944Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-09-23T19:10:10.290 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[69632]: ts=2023-09-23T19:10:09.944Z caller=node_exporter.go:115 level=info collector=uname 2023-09-23T19:10:10.290 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[69632]: ts=2023-09-23T19:10:09.944Z caller=node_exporter.go:115 level=info collector=vmstat 2023-09-23T19:10:10.290 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[69632]: ts=2023-09-23T19:10:09.944Z caller=node_exporter.go:115 level=info collector=xfs 2023-09-23T19:10:10.290 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[69632]: ts=2023-09-23T19:10:09.944Z caller=node_exporter.go:115 level=info collector=zfs 2023-09-23T19:10:10.290 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[69632]: ts=2023-09-23T19:10:09.944Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-09-23T19:10:10.290 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 19:10:09 smithi016 bash[69632]: ts=2023-09-23T19:10:09.945Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-09-23T19:10:12.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:12 smithi016 bash[27237]: cluster 2023-09-23T19:10:10.780196+0000 mgr.x (mgr.15012) 36 : cluster [DBG] pgmap v20: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 629 op/s 2023-09-23T19:10:12.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:12 smithi016 bash[27237]: audit 2023-09-23T19:10:11.126403+0000 mon.c (mon.2) 182 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:10:12.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:12 smithi016 bash[27237]: audit 2023-09-23T19:10:11.985787+0000 mon.a (mon.0) 781 : audit [DBG] from='client.? 172.21.15.16:0/1267251327' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:12.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:12 smithi016 bash[33624]: cluster 2023-09-23T19:10:10.780196+0000 mgr.x (mgr.15012) 36 : cluster [DBG] pgmap v20: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 629 op/s 2023-09-23T19:10:12.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:12 smithi016 bash[33624]: audit 2023-09-23T19:10:11.126403+0000 mon.c (mon.2) 182 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:10:12.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:12 smithi016 bash[33624]: audit 2023-09-23T19:10:11.985787+0000 mon.a (mon.0) 781 : audit [DBG] from='client.? 172.21.15.16:0/1267251327' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:12.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:12 smithi092 bash[27556]: cluster 2023-09-23T19:10:10.780196+0000 mgr.x (mgr.15012) 36 : cluster [DBG] pgmap v20: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 629 op/s 2023-09-23T19:10:12.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:12 smithi092 bash[27556]: audit 2023-09-23T19:10:11.126403+0000 mon.c (mon.2) 182 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:10:12.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:12 smithi092 bash[27556]: audit 2023-09-23T19:10:11.985787+0000 mon.a (mon.0) 781 : audit [DBG] from='client.? 172.21.15.16:0/1267251327' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:12.786 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:10:12 smithi092 bash[48334]: ts=2023-09-23T19:10:12.317Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi092.front.sepia.ceph.com:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.16:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T19:10:14.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:14 smithi016 bash[33624]: cluster 2023-09-23T19:10:12.780936+0000 mgr.x (mgr.15012) 37 : cluster [DBG] pgmap v21: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 928 op/s 2023-09-23T19:10:14.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:14 smithi016 bash[27237]: cluster 2023-09-23T19:10:12.780936+0000 mgr.x (mgr.15012) 37 : cluster [DBG] pgmap v21: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 928 op/s 2023-09-23T19:10:14.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:14 smithi092 bash[27556]: cluster 2023-09-23T19:10:12.780936+0000 mgr.x (mgr.15012) 37 : cluster [DBG] pgmap v21: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 928 op/s 2023-09-23T19:10:15.197 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:15 smithi016 bash[27237]: audit 2023-09-23T19:10:14.691691+0000 mon.a (mon.0) 782 : audit [DBG] from='client.? 172.21.15.16:0/4119093683' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:15.197 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:15 smithi016 bash[33624]: audit 2023-09-23T19:10:14.691691+0000 mon.a (mon.0) 782 : audit [DBG] from='client.? 172.21.15.16:0/4119093683' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:15.285 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:15 smithi092 bash[27556]: audit 2023-09-23T19:10:14.691691+0000 mon.a (mon.0) 782 : audit [DBG] from='client.? 172.21.15.16:0/4119093683' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:16.159 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:16 smithi016 bash[33624]: cluster 2023-09-23T19:10:14.782993+0000 mgr.x (mgr.15012) 38 : cluster [DBG] pgmap v22: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 609 op/s 2023-09-23T19:10:16.159 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:16 smithi016 bash[27237]: cluster 2023-09-23T19:10:14.782993+0000 mgr.x (mgr.15012) 38 : cluster [DBG] pgmap v22: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 609 op/s 2023-09-23T19:10:16.285 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:16 smithi092 bash[27556]: cluster 2023-09-23T19:10:14.782993+0000 mgr.x (mgr.15012) 38 : cluster [DBG] pgmap v22: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 609 op/s 2023-09-23T19:10:18.095 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Sep 23 19:10:17 smithi016 systemd[1]: Stopping Ceph alertmanager.a for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7... 2023-09-23T19:10:18.096 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Sep 23 19:10:17 smithi016 bash[69982]: Error response from daemon: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-alertmanager.a 2023-09-23T19:10:18.096 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Sep 23 19:10:18 smithi016 bash[56570]: level=info ts=2023-09-23T19:10:18.015Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2023-09-23T19:10:18.096 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:18 smithi016 bash[33624]: cluster 2023-09-23T19:10:16.783478+0000 mgr.x (mgr.15012) 39 : cluster [DBG] pgmap v23: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 649 op/s 2023-09-23T19:10:18.096 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:18 smithi016 bash[33624]: audit 2023-09-23T19:10:17.418696+0000 mon.c (mon.2) 183 : audit [DBG] from='client.? 172.21.15.16:0/2117756331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:18.096 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:18 smithi016 bash[27237]: cluster 2023-09-23T19:10:16.783478+0000 mgr.x (mgr.15012) 39 : cluster [DBG] pgmap v23: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 649 op/s 2023-09-23T19:10:18.096 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:18 smithi016 bash[27237]: audit 2023-09-23T19:10:17.418696+0000 mon.c (mon.2) 183 : audit [DBG] from='client.? 172.21.15.16:0/2117756331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:18.535 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:18 smithi092 bash[27556]: cluster 2023-09-23T19:10:16.783478+0000 mgr.x (mgr.15012) 39 : cluster [DBG] pgmap v23: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 649 op/s 2023-09-23T19:10:18.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:18 smithi092 bash[27556]: audit 2023-09-23T19:10:17.418696+0000 mon.c (mon.2) 183 : audit [DBG] from='client.? 172.21.15.16:0/2117756331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:18.906 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Sep 23 19:10:18 smithi016 bash[70012]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-alertmanager-a 2023-09-23T19:10:18.906 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Sep 23 19:10:18 smithi016 bash[70067]: Error response from daemon: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-alertmanager-a 2023-09-23T19:10:19.285 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Sep 23 19:10:18 smithi016 systemd[1]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-09-23T19:10:19.285 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Sep 23 19:10:18 smithi016 systemd[1]: Stopped Ceph alertmanager.a for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7. 2023-09-23T19:10:19.285 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Sep 23 19:10:18 smithi016 systemd[1]: Started Ceph alertmanager.a for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7. 2023-09-23T19:10:19.535 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:10:19 smithi092 bash[50491]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:10:19] "GET /metrics HTTP/1.1" 200 36067 "" "Prometheus/2.33.4" 2023-09-23T19:10:20.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:19 smithi016 bash[27237]: cluster 2023-09-23T19:10:18.784198+0000 mgr.x (mgr.15012) 40 : cluster [DBG] pgmap v24: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 886 op/s 2023-09-23T19:10:20.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:19 smithi016 bash[27237]: audit 2023-09-23T19:10:18.961045+0000 mon.a (mon.0) 783 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:20.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:19 smithi016 bash[27237]: audit 2023-09-23T19:10:18.975341+0000 mon.a (mon.0) 784 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:20.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:19 smithi016 bash[27237]: cephadm 2023-09-23T19:10:19.034216+0000 mgr.x (mgr.15012) 41 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T19:10:20.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:19 smithi016 bash[33624]: cluster 2023-09-23T19:10:18.784198+0000 mgr.x (mgr.15012) 40 : cluster [DBG] pgmap v24: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 886 op/s 2023-09-23T19:10:20.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:19 smithi016 bash[33624]: audit 2023-09-23T19:10:18.961045+0000 mon.a (mon.0) 783 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:20.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:19 smithi016 bash[33624]: audit 2023-09-23T19:10:18.975341+0000 mon.a (mon.0) 784 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:20.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:19 smithi016 bash[33624]: cephadm 2023-09-23T19:10:19.034216+0000 mgr.x (mgr.15012) 41 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T19:10:20.286 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Sep 23 19:10:20 smithi016 bash[70090]: level=info ts=2023-09-23T19:10:20.111Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2023-09-23T19:10:20.286 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Sep 23 19:10:20 smithi016 bash[70090]: level=info ts=2023-09-23T19:10:20.111Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2023-09-23T19:10:20.286 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Sep 23 19:10:20 smithi016 bash[70090]: level=info ts=2023-09-23T19:10:20.113Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.16 port=9094 2023-09-23T19:10:20.286 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Sep 23 19:10:20 smithi016 bash[70090]: level=info ts=2023-09-23T19:10:20.115Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-09-23T19:10:20.286 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Sep 23 19:10:20 smithi016 bash[70090]: level=info ts=2023-09-23T19:10:20.172Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T19:10:20.286 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Sep 23 19:10:20 smithi016 bash[70090]: level=info ts=2023-09-23T19:10:20.173Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T19:10:20.287 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Sep 23 19:10:20 smithi016 bash[70090]: level=info ts=2023-09-23T19:10:20.177Z caller=main.go:518 msg=Listening address=:9093 2023-09-23T19:10:20.287 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Sep 23 19:10:20 smithi016 bash[70090]: level=info ts=2023-09-23T19:10:20.177Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2023-09-23T19:10:20.287 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:10:20 smithi016 bash[27503]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:10:20] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T19:10:20.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:19 smithi092 bash[27556]: cluster 2023-09-23T19:10:18.784198+0000 mgr.x (mgr.15012) 40 : cluster [DBG] pgmap v24: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 886 op/s 2023-09-23T19:10:20.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:19 smithi092 bash[27556]: audit 2023-09-23T19:10:18.961045+0000 mon.a (mon.0) 783 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:20.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:19 smithi092 bash[27556]: audit 2023-09-23T19:10:18.975341+0000 mon.a (mon.0) 784 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:20.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:19 smithi092 bash[27556]: cephadm 2023-09-23T19:10:19.034216+0000 mgr.x (mgr.15012) 41 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T19:10:21.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:21 smithi016 bash[27237]: cephadm 2023-09-23T19:10:19.309350+0000 mgr.x (mgr.15012) 42 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi092 2023-09-23T19:10:21.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:21 smithi016 bash[27237]: audit 2023-09-23T19:10:20.111715+0000 mon.a (mon.0) 785 : audit [DBG] from='client.? 172.21.15.16:0/1552197471' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:21.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:21 smithi016 bash[33624]: cephadm 2023-09-23T19:10:19.309350+0000 mgr.x (mgr.15012) 42 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi092 2023-09-23T19:10:21.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:21 smithi016 bash[33624]: audit 2023-09-23T19:10:20.111715+0000 mon.a (mon.0) 785 : audit [DBG] from='client.? 172.21.15.16:0/1552197471' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:21.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:21 smithi092 bash[27556]: cephadm 2023-09-23T19:10:19.309350+0000 mgr.x (mgr.15012) 42 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi092 2023-09-23T19:10:21.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:21 smithi092 bash[27556]: audit 2023-09-23T19:10:20.111715+0000 mon.a (mon.0) 785 : audit [DBG] from='client.? 172.21.15.16:0/1552197471' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:22.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:22 smithi016 bash[33624]: cluster 2023-09-23T19:10:20.784648+0000 mgr.x (mgr.15012) 43 : cluster [DBG] pgmap v25: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 612 op/s 2023-09-23T19:10:22.285 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Sep 23 19:10:22 smithi016 bash[70090]: level=info ts=2023-09-23T19:10:22.116Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000794787s 2023-09-23T19:10:22.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:22 smithi016 bash[27237]: cluster 2023-09-23T19:10:20.784648+0000 mgr.x (mgr.15012) 43 : cluster [DBG] pgmap v25: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 612 op/s 2023-09-23T19:10:22.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:22 smithi092 bash[27556]: cluster 2023-09-23T19:10:20.784648+0000 mgr.x (mgr.15012) 43 : cluster [DBG] pgmap v25: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 612 op/s 2023-09-23T19:10:22.786 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:10:22 smithi092 bash[48334]: ts=2023-09-23T19:10:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi092.front.sepia.ceph.com:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.16:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T19:10:23.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:23 smithi016 bash[33624]: audit 2023-09-23T19:10:22.784735+0000 mon.c (mon.2) 184 : audit [DBG] from='client.? 172.21.15.16:0/2602348316' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:23.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:23 smithi016 bash[27237]: audit 2023-09-23T19:10:22.784735+0000 mon.c (mon.2) 184 : audit [DBG] from='client.? 172.21.15.16:0/2602348316' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:23.285 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:23 smithi092 bash[27556]: audit 2023-09-23T19:10:22.784735+0000 mon.c (mon.2) 184 : audit [DBG] from='client.? 172.21.15.16:0/2602348316' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:24.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:24 smithi016 bash[27237]: cluster 2023-09-23T19:10:22.785568+0000 mgr.x (mgr.15012) 44 : cluster [DBG] pgmap v26: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 905 op/s 2023-09-23T19:10:24.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:24 smithi016 bash[33624]: cluster 2023-09-23T19:10:22.785568+0000 mgr.x (mgr.15012) 44 : cluster [DBG] pgmap v26: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 905 op/s 2023-09-23T19:10:24.285 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:24 smithi092 bash[27556]: cluster 2023-09-23T19:10:22.785568+0000 mgr.x (mgr.15012) 44 : cluster [DBG] pgmap v26: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 905 op/s 2023-09-23T19:10:26.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:26 smithi016 bash[33624]: cluster 2023-09-23T19:10:24.786018+0000 mgr.x (mgr.15012) 45 : cluster [DBG] pgmap v27: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 606 op/s 2023-09-23T19:10:26.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:26 smithi016 bash[33624]: audit 2023-09-23T19:10:25.511223+0000 mon.c (mon.2) 185 : audit [DBG] from='client.? 172.21.15.16:0/3134114305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:26.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:26 smithi016 bash[27237]: cluster 2023-09-23T19:10:24.786018+0000 mgr.x (mgr.15012) 45 : cluster [DBG] pgmap v27: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 606 op/s 2023-09-23T19:10:26.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:26 smithi016 bash[27237]: audit 2023-09-23T19:10:25.511223+0000 mon.c (mon.2) 185 : audit [DBG] from='client.? 172.21.15.16:0/3134114305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:26.535 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:26 smithi092 bash[27556]: cluster 2023-09-23T19:10:24.786018+0000 mgr.x (mgr.15012) 45 : cluster [DBG] pgmap v27: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 606 op/s 2023-09-23T19:10:26.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:26 smithi092 bash[27556]: audit 2023-09-23T19:10:25.511223+0000 mon.c (mon.2) 185 : audit [DBG] from='client.? 172.21.15.16:0/3134114305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:27.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:27 smithi016 bash[33624]: audit 2023-09-23T19:10:26.126701+0000 mon.c (mon.2) 186 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:10:27.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:27 smithi016 bash[27237]: audit 2023-09-23T19:10:26.126701+0000 mon.c (mon.2) 186 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:10:27.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:27 smithi092 bash[27556]: audit 2023-09-23T19:10:26.126701+0000 mon.c (mon.2) 186 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:10:28.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:28 smithi016 bash[33624]: cluster 2023-09-23T19:10:26.786433+0000 mgr.x (mgr.15012) 46 : cluster [DBG] pgmap v28: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 606 op/s 2023-09-23T19:10:28.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:28 smithi016 bash[27237]: cluster 2023-09-23T19:10:26.786433+0000 mgr.x (mgr.15012) 46 : cluster [DBG] pgmap v28: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 606 op/s 2023-09-23T19:10:28.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:28 smithi092 bash[27556]: cluster 2023-09-23T19:10:26.786433+0000 mgr.x (mgr.15012) 46 : cluster [DBG] pgmap v28: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 606 op/s 2023-09-23T19:10:29.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:29 smithi016 bash[33624]: audit 2023-09-23T19:10:28.214315+0000 mon.c (mon.2) 187 : audit [DBG] from='client.? 172.21.15.16:0/2593226376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:29.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:29 smithi016 bash[27237]: audit 2023-09-23T19:10:28.214315+0000 mon.c (mon.2) 187 : audit [DBG] from='client.? 172.21.15.16:0/2593226376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:29.535 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:10:29 smithi092 bash[50491]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:10:29] "GET /metrics HTTP/1.1" 200 36070 "" "Prometheus/2.33.4" 2023-09-23T19:10:29.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:29 smithi092 bash[27556]: audit 2023-09-23T19:10:28.214315+0000 mon.c (mon.2) 187 : audit [DBG] from='client.? 172.21.15.16:0/2593226376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:30.447 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:30 smithi092 bash[27556]: cluster 2023-09-23T19:10:28.787111+0000 mgr.x (mgr.15012) 47 : cluster [DBG] pgmap v29: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 849 op/s 2023-09-23T19:10:30.534 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Sep 23 19:10:30 smithi016 bash[70090]: level=info ts=2023-09-23T19:10:30.120Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.004816864s 2023-09-23T19:10:30.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:30 smithi016 bash[33624]: cluster 2023-09-23T19:10:28.787111+0000 mgr.x (mgr.15012) 47 : cluster [DBG] pgmap v29: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 849 op/s 2023-09-23T19:10:30.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:30 smithi016 bash[27237]: cluster 2023-09-23T19:10:28.787111+0000 mgr.x (mgr.15012) 47 : cluster [DBG] pgmap v29: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 849 op/s 2023-09-23T19:10:30.535 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:10:30 smithi016 bash[27503]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:10:30] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T19:10:31.407 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:31 smithi092 bash[27556]: audit 2023-09-23T19:10:30.955617+0000 mon.a (mon.0) 786 : audit [DBG] from='client.? 172.21.15.16:0/3049126660' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:31.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:31 smithi016 bash[27237]: audit 2023-09-23T19:10:30.955617+0000 mon.a (mon.0) 786 : audit [DBG] from='client.? 172.21.15.16:0/3049126660' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:31.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:31 smithi016 bash[33624]: audit 2023-09-23T19:10:30.955617+0000 mon.a (mon.0) 786 : audit [DBG] from='client.? 172.21.15.16:0/3049126660' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:32.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:32 smithi016 bash[27237]: cluster 2023-09-23T19:10:30.787529+0000 mgr.x (mgr.15012) 48 : cluster [DBG] pgmap v30: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 576 op/s 2023-09-23T19:10:32.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:32 smithi016 bash[33624]: cluster 2023-09-23T19:10:30.787529+0000 mgr.x (mgr.15012) 48 : cluster [DBG] pgmap v30: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 576 op/s 2023-09-23T19:10:32.535 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:32 smithi092 bash[27556]: cluster 2023-09-23T19:10:30.787529+0000 mgr.x (mgr.15012) 48 : cluster [DBG] pgmap v30: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 576 op/s 2023-09-23T19:10:32.536 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:10:32 smithi092 bash[48334]: ts=2023-09-23T19:10:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi092.front.sepia.ceph.com:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.16:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T19:10:34.365 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:10:34 smithi092 systemd[1]: Stopping Ceph prometheus.a for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7... 2023-09-23T19:10:34.365 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:34 smithi092 bash[27556]: cluster 2023-09-23T19:10:32.788262+0000 mgr.x (mgr.15012) 49 : cluster [DBG] pgmap v31: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 883 op/s 2023-09-23T19:10:34.365 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:34 smithi092 bash[27556]: audit 2023-09-23T19:10:33.671922+0000 mon.c (mon.2) 188 : audit [DBG] from='client.? 172.21.15.16:0/2117333765' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:34.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:34 smithi016 bash[27237]: cluster 2023-09-23T19:10:32.788262+0000 mgr.x (mgr.15012) 49 : cluster [DBG] pgmap v31: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 883 op/s 2023-09-23T19:10:34.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:34 smithi016 bash[27237]: audit 2023-09-23T19:10:33.671922+0000 mon.c (mon.2) 188 : audit [DBG] from='client.? 172.21.15.16:0/2117333765' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:34.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:34 smithi016 bash[33624]: cluster 2023-09-23T19:10:32.788262+0000 mgr.x (mgr.15012) 49 : cluster [DBG] pgmap v31: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 883 op/s 2023-09-23T19:10:34.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:34 smithi016 bash[33624]: audit 2023-09-23T19:10:33.671922+0000 mon.c (mon.2) 188 : audit [DBG] from='client.? 172.21.15.16:0/2117333765' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:34.633 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:10:34 smithi092 bash[52736]: Error response from daemon: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-prometheus.a 2023-09-23T19:10:34.633 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:10:34 smithi092 bash[48334]: ts=2023-09-23T19:10:34.587Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-09-23T19:10:34.633 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:10:34 smithi092 bash[48334]: ts=2023-09-23T19:10:34.587Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-09-23T19:10:34.633 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:10:34 smithi092 bash[48334]: ts=2023-09-23T19:10:34.587Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-09-23T19:10:34.634 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:10:34 smithi092 bash[48334]: ts=2023-09-23T19:10:34.587Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-09-23T19:10:34.634 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:10:34 smithi092 bash[48334]: ts=2023-09-23T19:10:34.587Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-09-23T19:10:34.634 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:10:34 smithi092 bash[48334]: ts=2023-09-23T19:10:34.587Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-09-23T19:10:34.634 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:10:34 smithi092 bash[48334]: ts=2023-09-23T19:10:34.588Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-09-23T19:10:34.634 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:10:34 smithi092 bash[48334]: ts=2023-09-23T19:10:34.588Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-09-23T19:10:34.634 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:10:34 smithi092 bash[48334]: ts=2023-09-23T19:10:34.588Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-09-23T19:10:34.634 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:10:34 smithi092 bash[48334]: ts=2023-09-23T19:10:34.589Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-09-23T19:10:34.634 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:10:34 smithi092 bash[48334]: ts=2023-09-23T19:10:34.589Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-09-23T19:10:34.634 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:10:34 smithi092 bash[48334]: ts=2023-09-23T19:10:34.589Z caller=main.go:1066 level=info msg="See you next time!" 2023-09-23T19:10:35.219 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:10:35 smithi092 bash[52769]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-prometheus-a 2023-09-23T19:10:35.538 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:10:35 smithi092 bash[52825]: Error response from daemon: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-prometheus-a 2023-09-23T19:10:35.538 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:10:35 smithi092 systemd[1]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@prometheus.a.service: Succeeded. 2023-09-23T19:10:35.538 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:10:35 smithi092 systemd[1]: Stopped Ceph prometheus.a for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7. 2023-09-23T19:10:35.538 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:10:35 smithi092 systemd[1]: Started Ceph prometheus.a for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7. 2023-09-23T19:10:36.194 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:35 smithi092 systemd[1]: Stopping Ceph node-exporter.b for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7... 2023-09-23T19:10:36.194 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:35 smithi092 bash[52871]: Error response from daemon: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-node-exporter.b 2023-09-23T19:10:36.194 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:36 smithi092 bash[27556]: cluster 2023-09-23T19:10:34.788705+0000 mgr.x (mgr.15012) 50 : cluster [DBG] pgmap v32: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 590 op/s 2023-09-23T19:10:36.194 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:36 smithi092 bash[27556]: audit 2023-09-23T19:10:35.486564+0000 mon.a (mon.0) 787 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:36.194 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:36 smithi092 bash[27556]: audit 2023-09-23T19:10:35.500839+0000 mon.a (mon.0) 788 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:36.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:36 smithi016 bash[27237]: cluster 2023-09-23T19:10:34.788705+0000 mgr.x (mgr.15012) 50 : cluster [DBG] pgmap v32: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 590 op/s 2023-09-23T19:10:36.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:36 smithi016 bash[27237]: audit 2023-09-23T19:10:35.486564+0000 mon.a (mon.0) 787 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:36.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:36 smithi016 bash[27237]: audit 2023-09-23T19:10:35.500839+0000 mon.a (mon.0) 788 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:36.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:36 smithi016 bash[33624]: cluster 2023-09-23T19:10:34.788705+0000 mgr.x (mgr.15012) 50 : cluster [DBG] pgmap v32: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 590 op/s 2023-09-23T19:10:36.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:36 smithi016 bash[33624]: audit 2023-09-23T19:10:35.486564+0000 mon.a (mon.0) 787 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:36.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:36 smithi016 bash[33624]: audit 2023-09-23T19:10:35.500839+0000 mon.a (mon.0) 788 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:37.286 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:10:37 smithi092 bash[52849]: ts=2023-09-23T19:10:37.117Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2023-09-23T19:10:37.286 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:10:37 smithi092 bash[52849]: ts=2023-09-23T19:10:37.118Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2023-09-23T19:10:37.286 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:10:37 smithi092 bash[52849]: ts=2023-09-23T19:10:37.118Z caller=main.go:518 level=info host_details="(Linux 5.15.0-84-generic #93~20.04.1-Ubuntu SMP Wed Sep 6 16:15:40 UTC 2023 x86_64 smithi092 (none))" 2023-09-23T19:10:37.286 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:10:37 smithi092 bash[52849]: ts=2023-09-23T19:10:37.118Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-09-23T19:10:37.286 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:10:37 smithi092 bash[52849]: ts=2023-09-23T19:10:37.118Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-09-23T19:10:37.286 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:10:37 smithi092 bash[52849]: ts=2023-09-23T19:10:37.121Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2023-09-23T19:10:37.286 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:10:37 smithi092 bash[52849]: ts=2023-09-23T19:10:37.121Z caller=main.go:923 level=info msg="Starting TSDB ..." 2023-09-23T19:10:37.286 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:10:37 smithi092 bash[52849]: ts=2023-09-23T19:10:37.122Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2023-09-23T19:10:37.287 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:10:37 smithi092 bash[52849]: ts=2023-09-23T19:10:37.128Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-09-23T19:10:37.287 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:10:37 smithi092 bash[52849]: ts=2023-09-23T19:10:37.128Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=3.331µs 2023-09-23T19:10:37.287 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:10:37 smithi092 bash[52849]: ts=2023-09-23T19:10:37.128Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-09-23T19:10:37.287 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:37 smithi092 bash[52900]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-node-exporter-b 2023-09-23T19:10:37.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:37 smithi092 bash[27556]: cephadm 2023-09-23T19:10:35.535765+0000 mgr.x (mgr.15012) 51 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2023-09-23T19:10:37.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:37 smithi092 bash[27556]: cephadm 2023-09-23T19:10:35.536543+0000 mgr.x (mgr.15012) 52 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi092 2023-09-23T19:10:37.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:37 smithi092 bash[27556]: audit 2023-09-23T19:10:36.387572+0000 mon.b (mon.1) 43 : audit [DBG] from='client.? 172.21.15.16:0/431253749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:37.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:37 smithi016 bash[33624]: cephadm 2023-09-23T19:10:35.535765+0000 mgr.x (mgr.15012) 51 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2023-09-23T19:10:37.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:37 smithi016 bash[33624]: cephadm 2023-09-23T19:10:35.536543+0000 mgr.x (mgr.15012) 52 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi092 2023-09-23T19:10:37.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:37 smithi016 bash[33624]: audit 2023-09-23T19:10:36.387572+0000 mon.b (mon.1) 43 : audit [DBG] from='client.? 172.21.15.16:0/431253749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:37.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:37 smithi016 bash[27237]: cephadm 2023-09-23T19:10:35.535765+0000 mgr.x (mgr.15012) 51 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2023-09-23T19:10:37.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:37 smithi016 bash[27237]: cephadm 2023-09-23T19:10:35.536543+0000 mgr.x (mgr.15012) 52 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi092 2023-09-23T19:10:37.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:37 smithi016 bash[27237]: audit 2023-09-23T19:10:36.387572+0000 mon.b (mon.1) 43 : audit [DBG] from='client.? 172.21.15.16:0/431253749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:37.786 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:10:37 smithi092 bash[52849]: ts=2023-09-23T19:10:37.424Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2023-09-23T19:10:37.786 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:37 smithi092 systemd[1]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-09-23T19:10:37.786 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:37 smithi092 bash[53013]: Error response from daemon: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-node-exporter-b 2023-09-23T19:10:37.786 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:37 smithi092 systemd[1]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@node-exporter.b.service: Failed with result 'exit-code'. 2023-09-23T19:10:37.786 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:37 smithi092 systemd[1]: Stopped Ceph node-exporter.b for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7. 2023-09-23T19:10:37.786 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:37 smithi092 systemd[1]: Started Ceph node-exporter.b for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7. 2023-09-23T19:10:38.285 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:38 smithi092 bash[27556]: cluster 2023-09-23T19:10:36.789119+0000 mgr.x (mgr.15012) 53 : cluster [DBG] pgmap v33: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 590 op/s 2023-09-23T19:10:38.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:38 smithi092 bash[27556]: audit 2023-09-23T19:10:37.407277+0000 mon.a (mon.0) 789 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:38.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:38 smithi092 bash[27556]: audit 2023-09-23T19:10:37.418480+0000 mon.a (mon.0) 790 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:38.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:38 smithi092 bash[27556]: cephadm 2023-09-23T19:10:37.457529+0000 mgr.x (mgr.15012) 54 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2023-09-23T19:10:38.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:38 smithi092 bash[27556]: cephadm 2023-09-23T19:10:37.481421+0000 mgr.x (mgr.15012) 55 : cephadm [INF] Reconfiguring daemon grafana.a on smithi092 2023-09-23T19:10:38.286 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:10:37 smithi092 bash[52849]: ts=2023-09-23T19:10:37.901Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2023-09-23T19:10:38.286 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:10:37 smithi092 bash[52849]: ts=2023-09-23T19:10:37.901Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2023-09-23T19:10:38.286 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:10:37 smithi092 bash[52849]: ts=2023-09-23T19:10:37.901Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=131.407µs wal_replay_duration=773.247471ms total_replay_duration=773.467967ms 2023-09-23T19:10:38.287 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:10:37 smithi092 bash[52849]: ts=2023-09-23T19:10:37.906Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2023-09-23T19:10:38.287 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:10:37 smithi092 bash[52849]: ts=2023-09-23T19:10:37.907Z caller=main.go:947 level=info msg="TSDB started" 2023-09-23T19:10:38.287 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:10:37 smithi092 bash[52849]: ts=2023-09-23T19:10:37.907Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-09-23T19:10:38.287 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:10:37 smithi092 bash[52849]: ts=2023-09-23T19:10:37.937Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=29.410626ms db_storage=922ns remote_storage=1.793µs web_handler=488ns query_engine=970ns scrape=371.762µs scrape_sd=152.061µs notify=17.58µs notify_sd=14.366µs rules=28.157641ms 2023-09-23T19:10:38.287 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:10:37 smithi092 bash[52849]: ts=2023-09-23T19:10:37.937Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-09-23T19:10:38.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:38 smithi016 bash[27237]: cluster 2023-09-23T19:10:36.789119+0000 mgr.x (mgr.15012) 53 : cluster [DBG] pgmap v33: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 590 op/s 2023-09-23T19:10:38.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:38 smithi016 bash[27237]: audit 2023-09-23T19:10:37.407277+0000 mon.a (mon.0) 789 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:38.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:38 smithi016 bash[27237]: audit 2023-09-23T19:10:37.418480+0000 mon.a (mon.0) 790 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:38.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:38 smithi016 bash[27237]: cephadm 2023-09-23T19:10:37.457529+0000 mgr.x (mgr.15012) 54 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2023-09-23T19:10:38.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:38 smithi016 bash[27237]: cephadm 2023-09-23T19:10:37.481421+0000 mgr.x (mgr.15012) 55 : cephadm [INF] Reconfiguring daemon grafana.a on smithi092 2023-09-23T19:10:38.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:38 smithi016 bash[33624]: cluster 2023-09-23T19:10:36.789119+0000 mgr.x (mgr.15012) 53 : cluster [DBG] pgmap v33: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 590 op/s 2023-09-23T19:10:38.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:38 smithi016 bash[33624]: audit 2023-09-23T19:10:37.407277+0000 mon.a (mon.0) 789 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:38.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:38 smithi016 bash[33624]: audit 2023-09-23T19:10:37.418480+0000 mon.a (mon.0) 790 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:10:38.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:38 smithi016 bash[33624]: cephadm 2023-09-23T19:10:37.457529+0000 mgr.x (mgr.15012) 54 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2023-09-23T19:10:38.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:38 smithi016 bash[33624]: cephadm 2023-09-23T19:10:37.481421+0000 mgr.x (mgr.15012) 55 : cephadm [INF] Reconfiguring daemon grafana.a on smithi092 2023-09-23T19:10:38.786 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:38 smithi092 bash[53036]: ts=2023-09-23T19:10:38.544Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-09-23T19:10:38.786 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:38 smithi092 bash[53036]: ts=2023-09-23T19:10:38.545Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-09-23T19:10:38.786 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:38 smithi092 bash[53036]: ts=2023-09-23T19:10:38.546Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-09-23T19:10:38.786 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:38 smithi092 bash[53036]: ts=2023-09-23T19:10:38.546Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-09-23T19:10:38.787 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:38 smithi092 bash[53036]: ts=2023-09-23T19:10:38.546Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-09-23T19:10:38.787 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:38 smithi092 bash[53036]: ts=2023-09-23T19:10:38.546Z caller=node_exporter.go:115 level=info collector=arp 2023-09-23T19:10:38.787 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:38 smithi092 bash[53036]: ts=2023-09-23T19:10:38.546Z caller=node_exporter.go:115 level=info collector=bcache 2023-09-23T19:10:38.787 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:38 smithi092 bash[53036]: ts=2023-09-23T19:10:38.546Z caller=node_exporter.go:115 level=info collector=bonding 2023-09-23T19:10:38.787 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:38 smithi092 bash[53036]: ts=2023-09-23T19:10:38.546Z caller=node_exporter.go:115 level=info collector=btrfs 2023-09-23T19:10:38.787 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:38 smithi092 bash[53036]: ts=2023-09-23T19:10:38.546Z caller=node_exporter.go:115 level=info collector=conntrack 2023-09-23T19:10:38.787 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:38 smithi092 bash[53036]: ts=2023-09-23T19:10:38.546Z caller=node_exporter.go:115 level=info collector=cpu 2023-09-23T19:10:38.787 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:38 smithi092 bash[53036]: ts=2023-09-23T19:10:38.546Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-09-23T19:10:38.787 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:38 smithi092 bash[53036]: ts=2023-09-23T19:10:38.546Z caller=node_exporter.go:115 level=info collector=diskstats 2023-09-23T19:10:38.787 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:38 smithi092 bash[53036]: ts=2023-09-23T19:10:38.546Z caller=node_exporter.go:115 level=info collector=dmi 2023-09-23T19:10:38.787 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:38 smithi092 bash[53036]: ts=2023-09-23T19:10:38.546Z caller=node_exporter.go:115 level=info collector=edac 2023-09-23T19:10:38.788 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:38 smithi092 bash[53036]: ts=2023-09-23T19:10:38.546Z caller=node_exporter.go:115 level=info collector=entropy 2023-09-23T19:10:38.788 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:38 smithi092 bash[53036]: ts=2023-09-23T19:10:38.546Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-09-23T19:10:38.788 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:38 smithi092 bash[53036]: ts=2023-09-23T19:10:38.546Z caller=node_exporter.go:115 level=info collector=filefd 2023-09-23T19:10:38.788 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:38 smithi092 bash[53036]: ts=2023-09-23T19:10:38.546Z caller=node_exporter.go:115 level=info collector=filesystem 2023-09-23T19:10:38.788 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:38 smithi092 bash[53036]: ts=2023-09-23T19:10:38.546Z caller=node_exporter.go:115 level=info collector=hwmon 2023-09-23T19:10:38.788 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:38 smithi092 bash[53036]: ts=2023-09-23T19:10:38.546Z caller=node_exporter.go:115 level=info collector=infiniband 2023-09-23T19:10:38.788 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:38 smithi092 bash[53036]: ts=2023-09-23T19:10:38.546Z caller=node_exporter.go:115 level=info collector=ipvs 2023-09-23T19:10:38.788 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:38 smithi092 bash[53036]: ts=2023-09-23T19:10:38.546Z caller=node_exporter.go:115 level=info collector=loadavg 2023-09-23T19:10:38.788 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:38 smithi092 bash[53036]: ts=2023-09-23T19:10:38.546Z caller=node_exporter.go:115 level=info collector=mdadm 2023-09-23T19:10:38.788 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:38 smithi092 bash[53036]: ts=2023-09-23T19:10:38.546Z caller=node_exporter.go:115 level=info collector=meminfo 2023-09-23T19:10:38.788 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:38 smithi092 bash[53036]: ts=2023-09-23T19:10:38.546Z caller=node_exporter.go:115 level=info collector=netclass 2023-09-23T19:10:38.788 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:38 smithi092 bash[53036]: ts=2023-09-23T19:10:38.546Z caller=node_exporter.go:115 level=info collector=netdev 2023-09-23T19:10:38.789 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:38 smithi092 bash[53036]: ts=2023-09-23T19:10:38.546Z caller=node_exporter.go:115 level=info collector=netstat 2023-09-23T19:10:38.789 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:38 smithi092 bash[53036]: ts=2023-09-23T19:10:38.546Z caller=node_exporter.go:115 level=info collector=nfs 2023-09-23T19:10:38.789 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:38 smithi092 bash[53036]: ts=2023-09-23T19:10:38.546Z caller=node_exporter.go:115 level=info collector=nfsd 2023-09-23T19:10:38.789 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:38 smithi092 bash[53036]: ts=2023-09-23T19:10:38.546Z caller=node_exporter.go:115 level=info collector=nvme 2023-09-23T19:10:38.789 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:38 smithi092 bash[53036]: ts=2023-09-23T19:10:38.546Z caller=node_exporter.go:115 level=info collector=os 2023-09-23T19:10:38.789 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:38 smithi092 bash[53036]: ts=2023-09-23T19:10:38.546Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-09-23T19:10:38.789 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:38 smithi092 bash[53036]: ts=2023-09-23T19:10:38.546Z caller=node_exporter.go:115 level=info collector=pressure 2023-09-23T19:10:38.789 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:38 smithi092 bash[53036]: ts=2023-09-23T19:10:38.546Z caller=node_exporter.go:115 level=info collector=rapl 2023-09-23T19:10:38.789 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:38 smithi092 bash[53036]: ts=2023-09-23T19:10:38.546Z caller=node_exporter.go:115 level=info collector=schedstat 2023-09-23T19:10:38.789 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:38 smithi092 bash[53036]: ts=2023-09-23T19:10:38.546Z caller=node_exporter.go:115 level=info collector=sockstat 2023-09-23T19:10:38.789 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:38 smithi092 bash[53036]: ts=2023-09-23T19:10:38.546Z caller=node_exporter.go:115 level=info collector=softnet 2023-09-23T19:10:38.790 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:38 smithi092 bash[53036]: ts=2023-09-23T19:10:38.546Z caller=node_exporter.go:115 level=info collector=stat 2023-09-23T19:10:38.790 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:38 smithi092 bash[53036]: ts=2023-09-23T19:10:38.546Z caller=node_exporter.go:115 level=info collector=tapestats 2023-09-23T19:10:38.790 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:38 smithi092 bash[53036]: ts=2023-09-23T19:10:38.546Z caller=node_exporter.go:115 level=info collector=textfile 2023-09-23T19:10:38.790 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:38 smithi092 bash[53036]: ts=2023-09-23T19:10:38.546Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-09-23T19:10:38.790 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:38 smithi092 bash[53036]: ts=2023-09-23T19:10:38.546Z caller=node_exporter.go:115 level=info collector=time 2023-09-23T19:10:38.790 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:38 smithi092 bash[53036]: ts=2023-09-23T19:10:38.547Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-09-23T19:10:38.790 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:38 smithi092 bash[53036]: ts=2023-09-23T19:10:38.547Z caller=node_exporter.go:115 level=info collector=uname 2023-09-23T19:10:38.790 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:38 smithi092 bash[53036]: ts=2023-09-23T19:10:38.547Z caller=node_exporter.go:115 level=info collector=vmstat 2023-09-23T19:10:38.790 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:38 smithi092 bash[53036]: ts=2023-09-23T19:10:38.547Z caller=node_exporter.go:115 level=info collector=xfs 2023-09-23T19:10:38.790 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:38 smithi092 bash[53036]: ts=2023-09-23T19:10:38.547Z caller=node_exporter.go:115 level=info collector=zfs 2023-09-23T19:10:38.790 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:38 smithi092 bash[53036]: ts=2023-09-23T19:10:38.547Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-09-23T19:10:38.791 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 19:10:38 smithi092 bash[53036]: ts=2023-09-23T19:10:38.547Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-09-23T19:10:39.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:39 smithi016 bash[27237]: cluster 2023-09-23T19:10:38.789813+0000 mgr.x (mgr.15012) 56 : cluster [DBG] pgmap v34: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 885 op/s 2023-09-23T19:10:39.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:39 smithi016 bash[27237]: audit 2023-09-23T19:10:39.073645+0000 mon.a (mon.0) 791 : audit [DBG] from='client.? 172.21.15.16:0/1135174091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:39.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:39 smithi016 bash[33624]: cluster 2023-09-23T19:10:38.789813+0000 mgr.x (mgr.15012) 56 : cluster [DBG] pgmap v34: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 885 op/s 2023-09-23T19:10:39.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:39 smithi016 bash[33624]: audit 2023-09-23T19:10:39.073645+0000 mon.a (mon.0) 791 : audit [DBG] from='client.? 172.21.15.16:0/1135174091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:39.535 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:39 smithi092 bash[27556]: cluster 2023-09-23T19:10:38.789813+0000 mgr.x (mgr.15012) 56 : cluster [DBG] pgmap v34: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 885 op/s 2023-09-23T19:10:39.535 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:39 smithi092 bash[27556]: audit 2023-09-23T19:10:39.073645+0000 mon.a (mon.0) 791 : audit [DBG] from='client.? 172.21.15.16:0/1135174091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:42.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:42 smithi016 bash[33624]: cluster 2023-09-23T19:10:40.790218+0000 mgr.x (mgr.15012) 57 : cluster [DBG] pgmap v35: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 602 op/s 2023-09-23T19:10:42.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:42 smithi016 bash[33624]: audit 2023-09-23T19:10:41.127290+0000 mon.c (mon.2) 189 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:10:42.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:42 smithi016 bash[33624]: audit 2023-09-23T19:10:41.717243+0000 mon.c (mon.2) 190 : audit [DBG] from='client.? 172.21.15.16:0/1744591418' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:42.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:42 smithi016 bash[27237]: cluster 2023-09-23T19:10:40.790218+0000 mgr.x (mgr.15012) 57 : cluster [DBG] pgmap v35: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 602 op/s 2023-09-23T19:10:42.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:42 smithi016 bash[27237]: audit 2023-09-23T19:10:41.127290+0000 mon.c (mon.2) 189 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:10:42.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:42 smithi016 bash[27237]: audit 2023-09-23T19:10:41.717243+0000 mon.c (mon.2) 190 : audit [DBG] from='client.? 172.21.15.16:0/1744591418' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:42.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:42 smithi092 bash[27556]: cluster 2023-09-23T19:10:40.790218+0000 mgr.x (mgr.15012) 57 : cluster [DBG] pgmap v35: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 602 op/s 2023-09-23T19:10:42.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:42 smithi092 bash[27556]: audit 2023-09-23T19:10:41.127290+0000 mon.c (mon.2) 189 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:10:42.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:42 smithi092 bash[27556]: audit 2023-09-23T19:10:41.717243+0000 mon.c (mon.2) 190 : audit [DBG] from='client.? 172.21.15.16:0/1744591418' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:44.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:44 smithi016 bash[33624]: cluster 2023-09-23T19:10:42.791172+0000 mgr.x (mgr.15012) 58 : cluster [DBG] pgmap v36: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 871 op/s 2023-09-23T19:10:44.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:44 smithi016 bash[27237]: cluster 2023-09-23T19:10:42.791172+0000 mgr.x (mgr.15012) 58 : cluster [DBG] pgmap v36: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 871 op/s 2023-09-23T19:10:44.285 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:44 smithi092 bash[27556]: cluster 2023-09-23T19:10:42.791172+0000 mgr.x (mgr.15012) 58 : cluster [DBG] pgmap v36: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 871 op/s 2023-09-23T19:10:45.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:45 smithi016 bash[33624]: audit 2023-09-23T19:10:44.364260+0000 mon.b (mon.1) 44 : audit [DBG] from='client.? 172.21.15.16:0/3884073409' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:45.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:45 smithi016 bash[27237]: audit 2023-09-23T19:10:44.364260+0000 mon.b (mon.1) 44 : audit [DBG] from='client.? 172.21.15.16:0/3884073409' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:45.285 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:45 smithi092 bash[27556]: audit 2023-09-23T19:10:44.364260+0000 mon.b (mon.1) 44 : audit [DBG] from='client.? 172.21.15.16:0/3884073409' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:46.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:46 smithi016 bash[27237]: cluster 2023-09-23T19:10:44.791599+0000 mgr.x (mgr.15012) 59 : cluster [DBG] pgmap v37: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 563 op/s 2023-09-23T19:10:46.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:46 smithi016 bash[33624]: cluster 2023-09-23T19:10:44.791599+0000 mgr.x (mgr.15012) 59 : cluster [DBG] pgmap v37: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 563 op/s 2023-09-23T19:10:46.285 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:46 smithi092 bash[27556]: cluster 2023-09-23T19:10:44.791599+0000 mgr.x (mgr.15012) 59 : cluster [DBG] pgmap v37: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 563 op/s 2023-09-23T19:10:47.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:47 smithi016 bash[27237]: audit 2023-09-23T19:10:46.973416+0000 mon.c (mon.2) 191 : audit [DBG] from='client.? 172.21.15.16:0/196807679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:47.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:47 smithi016 bash[33624]: audit 2023-09-23T19:10:46.973416+0000 mon.c (mon.2) 191 : audit [DBG] from='client.? 172.21.15.16:0/196807679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:47.535 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:47 smithi092 bash[27556]: audit 2023-09-23T19:10:46.973416+0000 mon.c (mon.2) 191 : audit [DBG] from='client.? 172.21.15.16:0/196807679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:48.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:48 smithi016 bash[27237]: cluster 2023-09-23T19:10:46.792001+0000 mgr.x (mgr.15012) 60 : cluster [DBG] pgmap v38: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 563 op/s 2023-09-23T19:10:48.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:48 smithi016 bash[33624]: cluster 2023-09-23T19:10:46.792001+0000 mgr.x (mgr.15012) 60 : cluster [DBG] pgmap v38: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 563 op/s 2023-09-23T19:10:48.535 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:48 smithi092 bash[27556]: cluster 2023-09-23T19:10:46.792001+0000 mgr.x (mgr.15012) 60 : cluster [DBG] pgmap v38: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 563 op/s 2023-09-23T19:10:49.535 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:10:49 smithi092 bash[50491]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:10:49] "GET /metrics HTTP/1.1" 200 36066 "" "Prometheus/2.33.4" 2023-09-23T19:10:50.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:50 smithi016 bash[33624]: cluster 2023-09-23T19:10:48.792682+0000 mgr.x (mgr.15012) 61 : cluster [DBG] pgmap v39: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 813 op/s 2023-09-23T19:10:50.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:50 smithi016 bash[33624]: audit 2023-09-23T19:10:49.610672+0000 mon.a (mon.0) 792 : audit [DBG] from='client.? 172.21.15.16:0/3121918718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:50.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:50 smithi016 bash[27237]: cluster 2023-09-23T19:10:48.792682+0000 mgr.x (mgr.15012) 61 : cluster [DBG] pgmap v39: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 813 op/s 2023-09-23T19:10:50.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:50 smithi016 bash[27237]: audit 2023-09-23T19:10:49.610672+0000 mon.a (mon.0) 792 : audit [DBG] from='client.? 172.21.15.16:0/3121918718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:50.535 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:50 smithi092 bash[27556]: cluster 2023-09-23T19:10:48.792682+0000 mgr.x (mgr.15012) 61 : cluster [DBG] pgmap v39: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 813 op/s 2023-09-23T19:10:50.535 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:50 smithi092 bash[27556]: audit 2023-09-23T19:10:49.610672+0000 mon.a (mon.0) 792 : audit [DBG] from='client.? 172.21.15.16:0/3121918718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:52.285 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:52 smithi092 bash[27556]: cluster 2023-09-23T19:10:50.793187+0000 mgr.x (mgr.15012) 62 : cluster [DBG] pgmap v40: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 519 op/s 2023-09-23T19:10:52.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:52 smithi016 bash[27237]: cluster 2023-09-23T19:10:50.793187+0000 mgr.x (mgr.15012) 62 : cluster [DBG] pgmap v40: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 519 op/s 2023-09-23T19:10:52.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:52 smithi016 bash[33624]: cluster 2023-09-23T19:10:50.793187+0000 mgr.x (mgr.15012) 62 : cluster [DBG] pgmap v40: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 519 op/s 2023-09-23T19:10:53.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:53 smithi016 bash[27237]: audit 2023-09-23T19:10:52.282466+0000 mon.c (mon.2) 192 : audit [DBG] from='client.? 172.21.15.16:0/3799748683' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:53.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:53 smithi016 bash[33624]: audit 2023-09-23T19:10:52.282466+0000 mon.c (mon.2) 192 : audit [DBG] from='client.? 172.21.15.16:0/3799748683' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:53.535 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:53 smithi092 bash[27556]: audit 2023-09-23T19:10:52.282466+0000 mon.c (mon.2) 192 : audit [DBG] from='client.? 172.21.15.16:0/3799748683' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:54.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:54 smithi016 bash[27237]: cluster 2023-09-23T19:10:52.797418+0000 mgr.x (mgr.15012) 63 : cluster [DBG] pgmap v41: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 771 op/s 2023-09-23T19:10:54.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:54 smithi016 bash[33624]: cluster 2023-09-23T19:10:52.797418+0000 mgr.x (mgr.15012) 63 : cluster [DBG] pgmap v41: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 771 op/s 2023-09-23T19:10:54.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:54 smithi092 bash[27556]: cluster 2023-09-23T19:10:52.797418+0000 mgr.x (mgr.15012) 63 : cluster [DBG] pgmap v41: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 771 op/s 2023-09-23T19:10:55.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:55 smithi016 bash[27237]: audit 2023-09-23T19:10:54.949701+0000 mon.b (mon.1) 45 : audit [DBG] from='client.? 172.21.15.16:0/2914187596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:55.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:55 smithi016 bash[33624]: audit 2023-09-23T19:10:54.949701+0000 mon.b (mon.1) 45 : audit [DBG] from='client.? 172.21.15.16:0/2914187596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:55.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:55 smithi092 bash[27556]: audit 2023-09-23T19:10:54.949701+0000 mon.b (mon.1) 45 : audit [DBG] from='client.? 172.21.15.16:0/2914187596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:56.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:56 smithi016 bash[33624]: cluster 2023-09-23T19:10:54.797851+0000 mgr.x (mgr.15012) 64 : cluster [DBG] pgmap v42: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 502 op/s 2023-09-23T19:10:56.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:56 smithi016 bash[27237]: cluster 2023-09-23T19:10:54.797851+0000 mgr.x (mgr.15012) 64 : cluster [DBG] pgmap v42: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 502 op/s 2023-09-23T19:10:56.535 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:56 smithi092 bash[27556]: cluster 2023-09-23T19:10:54.797851+0000 mgr.x (mgr.15012) 64 : cluster [DBG] pgmap v42: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 502 op/s 2023-09-23T19:10:57.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:57 smithi016 bash[33624]: audit 2023-09-23T19:10:56.127633+0000 mon.c (mon.2) 193 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:10:57.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:57 smithi016 bash[27237]: audit 2023-09-23T19:10:56.127633+0000 mon.c (mon.2) 193 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:10:57.535 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:57 smithi092 bash[27556]: audit 2023-09-23T19:10:56.127633+0000 mon.c (mon.2) 193 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:10:58.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:58 smithi016 bash[33624]: cluster 2023-09-23T19:10:56.798286+0000 mgr.x (mgr.15012) 65 : cluster [DBG] pgmap v43: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 502 op/s 2023-09-23T19:10:58.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:10:58 smithi016 bash[33624]: audit 2023-09-23T19:10:57.660641+0000 mon.a (mon.0) 793 : audit [DBG] from='client.? 172.21.15.16:0/3251861841' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:58.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:58 smithi016 bash[27237]: cluster 2023-09-23T19:10:56.798286+0000 mgr.x (mgr.15012) 65 : cluster [DBG] pgmap v43: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 502 op/s 2023-09-23T19:10:58.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:10:58 smithi016 bash[27237]: audit 2023-09-23T19:10:57.660641+0000 mon.a (mon.0) 793 : audit [DBG] from='client.? 172.21.15.16:0/3251861841' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:58.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:58 smithi092 bash[27556]: cluster 2023-09-23T19:10:56.798286+0000 mgr.x (mgr.15012) 65 : cluster [DBG] pgmap v43: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 502 op/s 2023-09-23T19:10:58.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:10:58 smithi092 bash[27556]: audit 2023-09-23T19:10:57.660641+0000 mon.a (mon.0) 793 : audit [DBG] from='client.? 172.21.15.16:0/3251861841' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:10:59.036 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:10:58 smithi092 bash[52849]: ts=2023-09-23T19:10:58.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi092:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi092.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T19:10:59.535 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:10:59 smithi092 bash[50491]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:10:59] "GET /metrics HTTP/1.1" 200 36068 "" "Prometheus/2.33.4" 2023-09-23T19:11:00.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:00 smithi092 bash[27556]: cluster 2023-09-23T19:10:58.798983+0000 mgr.x (mgr.15012) 66 : cluster [DBG] pgmap v44: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 805 op/s 2023-09-23T19:11:00.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:00 smithi016 bash[33624]: cluster 2023-09-23T19:10:58.798983+0000 mgr.x (mgr.15012) 66 : cluster [DBG] pgmap v44: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 805 op/s 2023-09-23T19:11:00.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:00 smithi016 bash[27237]: cluster 2023-09-23T19:10:58.798983+0000 mgr.x (mgr.15012) 66 : cluster [DBG] pgmap v44: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 805 op/s 2023-09-23T19:11:01.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:01 smithi016 bash[27237]: audit 2023-09-23T19:11:00.361942+0000 mon.c (mon.2) 194 : audit [DBG] from='client.? 172.21.15.16:0/1583746472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:01.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:01 smithi016 bash[33624]: audit 2023-09-23T19:11:00.361942+0000 mon.c (mon.2) 194 : audit [DBG] from='client.? 172.21.15.16:0/1583746472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:01.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:01 smithi092 bash[27556]: audit 2023-09-23T19:11:00.361942+0000 mon.c (mon.2) 194 : audit [DBG] from='client.? 172.21.15.16:0/1583746472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:02.143 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:02 smithi092 bash[27556]: cluster 2023-09-23T19:11:00.799327+0000 mgr.x (mgr.15012) 67 : cluster [DBG] pgmap v45: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 555 op/s 2023-09-23T19:11:02.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:02 smithi016 bash[27237]: cluster 2023-09-23T19:11:00.799327+0000 mgr.x (mgr.15012) 67 : cluster [DBG] pgmap v45: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 555 op/s 2023-09-23T19:11:02.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:02 smithi016 bash[33624]: cluster 2023-09-23T19:11:00.799327+0000 mgr.x (mgr.15012) 67 : cluster [DBG] pgmap v45: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 555 op/s 2023-09-23T19:11:03.271 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:11:02 smithi092 systemd[1]: Stopping Ceph grafana.a for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7... 2023-09-23T19:11:03.272 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:11:02 smithi092 bash[53283]: Error response from daemon: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-grafana.a 2023-09-23T19:11:03.272 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:11:03 smithi092 bash[47618]: t=2023-09-23T19:11:03+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2023-09-23T19:11:03.272 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:03 smithi092 bash[27556]: audit 2023-09-23T19:11:03.042572+0000 mon.c (mon.2) 195 : audit [DBG] from='client.? 172.21.15.16:0/982727664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:03.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:03 smithi016 bash[33624]: audit 2023-09-23T19:11:03.042572+0000 mon.c (mon.2) 195 : audit [DBG] from='client.? 172.21.15.16:0/982727664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:03.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:03 smithi016 bash[27237]: audit 2023-09-23T19:11:03.042572+0000 mon.c (mon.2) 195 : audit [DBG] from='client.? 172.21.15.16:0/982727664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:04.220 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:11:03 smithi092 bash[53312]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-grafana-a 2023-09-23T19:11:04.221 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:11:04 smithi092 bash[53367]: Error response from daemon: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-grafana-a 2023-09-23T19:11:04.221 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:11:04 smithi092 systemd[1]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@grafana.a.service: Succeeded. 2023-09-23T19:11:04.221 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:11:04 smithi092 systemd[1]: Stopped Ceph grafana.a for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7. 2023-09-23T19:11:04.221 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:11:04 smithi092 systemd[1]: Started Ceph grafana.a for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7. 2023-09-23T19:11:04.222 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:04 smithi092 bash[27556]: cluster 2023-09-23T19:11:02.799960+0000 mgr.x (mgr.15012) 68 : cluster [DBG] pgmap v46: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 856 op/s 2023-09-23T19:11:04.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:04 smithi016 bash[27237]: cluster 2023-09-23T19:11:02.799960+0000 mgr.x (mgr.15012) 68 : cluster [DBG] pgmap v46: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 856 op/s 2023-09-23T19:11:04.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:04 smithi016 bash[33624]: cluster 2023-09-23T19:11:02.799960+0000 mgr.x (mgr.15012) 68 : cluster [DBG] pgmap v46: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 856 op/s 2023-09-23T19:11:05.366 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:05 smithi092 bash[27556]: audit 2023-09-23T19:11:04.174154+0000 mon.a (mon.0) 794 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:11:05.366 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:05 smithi092 bash[27556]: audit 2023-09-23T19:11:04.192471+0000 mon.a (mon.0) 795 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:11:05.366 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:05 smithi092 bash[27556]: audit 2023-09-23T19:11:04.248155+0000 mon.c (mon.2) 196 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T19:11:05.366 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:05 smithi092 bash[27556]: audit 2023-09-23T19:11:04.254646+0000 mon.c (mon.2) 197 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T19:11:05.366 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:05 smithi092 bash[27556]: audit 2023-09-23T19:11:04.255631+0000 mgr.x (mgr.15012) 69 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T19:11:05.366 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:05 smithi092 bash[27556]: audit 2023-09-23T19:11:04.262051+0000 mgr.x (mgr.15012) 70 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T19:11:05.367 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:05 smithi092 bash[27556]: audit 2023-09-23T19:11:04.264078+0000 mon.c (mon.2) 198 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T19:11:05.367 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:05 smithi092 bash[27556]: audit 2023-09-23T19:11:04.273988+0000 mgr.x (mgr.15012) 71 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T19:11:05.367 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:05 smithi092 bash[27556]: audit 2023-09-23T19:11:04.279885+0000 mon.c (mon.2) 199 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:11:05.367 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:05 smithi092 bash[27556]: cephadm 2023-09-23T19:11:04.293039+0000 mgr.x (mgr.15012) 72 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T19:11:05.367 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:05 smithi092 bash[27556]: cephadm 2023-09-23T19:11:04.293965+0000 mgr.x (mgr.15012) 73 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T19:11:05.367 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:05 smithi092 bash[27556]: cluster 2023-09-23T19:11:04.800400+0000 mgr.x (mgr.15012) 74 : cluster [DBG] pgmap v47: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 604 op/s 2023-09-23T19:11:05.367 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:11:05 smithi092 bash[53391]: t=2023-09-23T19:11:05+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2023-09-23T19:11:05.367 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:11:05 smithi092 bash[53391]: t=2023-09-23T19:11:05+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2023-09-23T19:11:05.367 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:11:05 smithi092 bash[53391]: t=2023-09-23T19:11:05+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2023-09-23T19:11:05.367 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:11:05 smithi092 bash[53391]: t=2023-09-23T19:11:05+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2023-09-23T19:11:05.368 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:11:05 smithi092 bash[53391]: t=2023-09-23T19:11:05+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2023-09-23T19:11:05.368 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:11:05 smithi092 bash[53391]: t=2023-09-23T19:11:05+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2023-09-23T19:11:05.368 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:11:05 smithi092 bash[53391]: t=2023-09-23T19:11:05+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-09-23T19:11:05.368 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:11:05 smithi092 bash[53391]: t=2023-09-23T19:11:05+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-09-23T19:11:05.368 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:11:05 smithi092 bash[53391]: t=2023-09-23T19:11:05+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2023-09-23T19:11:05.368 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:11:05 smithi092 bash[53391]: t=2023-09-23T19:11:05+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2023-09-23T19:11:05.368 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:11:05 smithi092 bash[53391]: t=2023-09-23T19:11:05+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2023-09-23T19:11:05.368 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:11:05 smithi092 bash[53391]: t=2023-09-23T19:11:05+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2023-09-23T19:11:05.368 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:11:05 smithi092 bash[53391]: t=2023-09-23T19:11:05+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2023-09-23T19:11:05.368 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:11:05 smithi092 bash[53391]: t=2023-09-23T19:11:05+0000 lvl=info msg="App mode production" logger=settings 2023-09-23T19:11:05.368 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:11:05 smithi092 bash[53391]: t=2023-09-23T19:11:05+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2023-09-23T19:11:05.368 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:11:05 smithi092 bash[53391]: t=2023-09-23T19:11:05+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-09-23T19:11:05.369 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:11:05 smithi092 bash[53391]: t=2023-09-23T19:11:05+0000 lvl=info msg="Starting DB migrations" logger=migrator 2023-09-23T19:11:05.369 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:11:05 smithi092 bash[53391]: t=2023-09-23T19:11:05+0000 lvl=info msg="migrations completed" logger=migrator performed=0 skipped=377 duration=1.195539ms 2023-09-23T19:11:05.369 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:11:05 smithi092 bash[53391]: t=2023-09-23T19:11:05+0000 lvl=info msg="Created default organization" logger=sqlstore 2023-09-23T19:11:05.369 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:11:05 smithi092 bash[53391]: t=2023-09-23T19:11:05+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2023-09-23T19:11:05.369 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:11:05 smithi092 bash[53391]: t=2023-09-23T19:11:05+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2023-09-23T19:11:05.414 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:11:05 smithi092 bash[53391]: t=2023-09-23T19:11:05+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2023-09-23T19:11:05.415 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:11:05 smithi092 bash[53391]: t=2023-09-23T19:11:05+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2023-09-23T19:11:05.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:05 smithi016 bash[27237]: audit 2023-09-23T19:11:04.174154+0000 mon.a (mon.0) 794 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:11:05.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:05 smithi016 bash[27237]: audit 2023-09-23T19:11:04.192471+0000 mon.a (mon.0) 795 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:11:05.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:05 smithi016 bash[27237]: audit 2023-09-23T19:11:04.248155+0000 mon.c (mon.2) 196 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T19:11:05.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:05 smithi016 bash[27237]: audit 2023-09-23T19:11:04.254646+0000 mon.c (mon.2) 197 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T19:11:05.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:05 smithi016 bash[27237]: audit 2023-09-23T19:11:04.255631+0000 mgr.x (mgr.15012) 69 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T19:11:05.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:05 smithi016 bash[27237]: audit 2023-09-23T19:11:04.262051+0000 mgr.x (mgr.15012) 70 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T19:11:05.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:05 smithi016 bash[27237]: audit 2023-09-23T19:11:04.264078+0000 mon.c (mon.2) 198 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T19:11:05.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:05 smithi016 bash[27237]: audit 2023-09-23T19:11:04.273988+0000 mgr.x (mgr.15012) 71 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T19:11:05.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:05 smithi016 bash[27237]: audit 2023-09-23T19:11:04.279885+0000 mon.c (mon.2) 199 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:11:05.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:05 smithi016 bash[27237]: cephadm 2023-09-23T19:11:04.293039+0000 mgr.x (mgr.15012) 72 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T19:11:05.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:05 smithi016 bash[27237]: cephadm 2023-09-23T19:11:04.293965+0000 mgr.x (mgr.15012) 73 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T19:11:05.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:05 smithi016 bash[27237]: cluster 2023-09-23T19:11:04.800400+0000 mgr.x (mgr.15012) 74 : cluster [DBG] pgmap v47: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 604 op/s 2023-09-23T19:11:05.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:05 smithi016 bash[33624]: audit 2023-09-23T19:11:04.174154+0000 mon.a (mon.0) 794 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:11:05.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:05 smithi016 bash[33624]: audit 2023-09-23T19:11:04.192471+0000 mon.a (mon.0) 795 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:11:05.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:05 smithi016 bash[33624]: audit 2023-09-23T19:11:04.248155+0000 mon.c (mon.2) 196 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T19:11:05.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:05 smithi016 bash[33624]: audit 2023-09-23T19:11:04.254646+0000 mon.c (mon.2) 197 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T19:11:05.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:05 smithi016 bash[33624]: audit 2023-09-23T19:11:04.255631+0000 mgr.x (mgr.15012) 69 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T19:11:05.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:05 smithi016 bash[33624]: audit 2023-09-23T19:11:04.262051+0000 mgr.x (mgr.15012) 70 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T19:11:05.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:05 smithi016 bash[33624]: audit 2023-09-23T19:11:04.264078+0000 mon.c (mon.2) 198 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T19:11:05.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:05 smithi016 bash[33624]: audit 2023-09-23T19:11:04.273988+0000 mgr.x (mgr.15012) 71 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T19:11:05.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:05 smithi016 bash[33624]: audit 2023-09-23T19:11:04.279885+0000 mon.c (mon.2) 199 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:11:05.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:05 smithi016 bash[33624]: cephadm 2023-09-23T19:11:04.293039+0000 mgr.x (mgr.15012) 72 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T19:11:05.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:05 smithi016 bash[33624]: cephadm 2023-09-23T19:11:04.293965+0000 mgr.x (mgr.15012) 73 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T19:11:05.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:05 smithi016 bash[33624]: cluster 2023-09-23T19:11:04.800400+0000 mgr.x (mgr.15012) 74 : cluster [DBG] pgmap v47: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 604 op/s 2023-09-23T19:11:05.625 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:11:05 smithi092 bash[53391]: t=2023-09-23T19:11:05+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2023-09-23T19:11:05.625 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:11:05 smithi092 bash[53391]: t=2023-09-23T19:11:05+0000 lvl=info msg="deleted datasource based on configuration" logger=provisioning.datasources name=Dashboard1 2023-09-23T19:11:05.625 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:11:05 smithi092 bash[53391]: t=2023-09-23T19:11:05+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2023-09-23T19:11:06.036 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:11:05 smithi092 bash[53391]: t=2023-09-23T19:11:05+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Loki uid=P8E80F9AEF21F6940 2023-09-23T19:11:06.036 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:11:05 smithi092 bash[53391]: t=2023-09-23T19:11:05+0000 lvl=info msg="warming cache for startup" logger=ngalert 2023-09-23T19:11:06.036 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:11:05 smithi092 bash[53391]: t=2023-09-23T19:11:05+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2023-09-23T19:11:06.036 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 19:11:05 smithi092 bash[53391]: t=2023-09-23T19:11:05+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2023-09-23T19:11:06.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:06 smithi016 bash[33624]: audit 2023-09-23T19:11:05.688878+0000 mon.c (mon.2) 200 : audit [DBG] from='client.? 172.21.15.16:0/726125796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:06.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:06 smithi016 bash[27237]: audit 2023-09-23T19:11:05.688878+0000 mon.c (mon.2) 200 : audit [DBG] from='client.? 172.21.15.16:0/726125796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:06.535 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:06 smithi092 bash[27556]: audit 2023-09-23T19:11:05.688878+0000 mon.c (mon.2) 200 : audit [DBG] from='client.? 172.21.15.16:0/726125796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:07.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:07 smithi016 bash[27237]: cluster 2023-09-23T19:11:06.800823+0000 mgr.x (mgr.15012) 75 : cluster [DBG] pgmap v48: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 604 op/s 2023-09-23T19:11:07.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:07 smithi016 bash[33624]: cluster 2023-09-23T19:11:06.800823+0000 mgr.x (mgr.15012) 75 : cluster [DBG] pgmap v48: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 604 op/s 2023-09-23T19:11:07.535 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:07 smithi092 bash[27556]: cluster 2023-09-23T19:11:06.800823+0000 mgr.x (mgr.15012) 75 : cluster [DBG] pgmap v48: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 604 op/s 2023-09-23T19:11:08.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:08 smithi016 bash[33624]: audit 2023-09-23T19:11:08.388553+0000 mon.c (mon.2) 201 : audit [DBG] from='client.? 172.21.15.16:0/303963734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:08.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:08 smithi016 bash[27237]: audit 2023-09-23T19:11:08.388553+0000 mon.c (mon.2) 201 : audit [DBG] from='client.? 172.21.15.16:0/303963734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:08.785 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:08 smithi092 bash[27556]: audit 2023-09-23T19:11:08.388553+0000 mon.c (mon.2) 201 : audit [DBG] from='client.? 172.21.15.16:0/303963734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:08.786 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:11:08 smithi092 bash[52849]: ts=2023-09-23T19:11:08.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi092:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi092.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T19:11:09.535 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:11:09 smithi092 bash[50491]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:11:09] "GET /metrics HTTP/1.1" 200 36068 "" "Prometheus/2.33.4" 2023-09-23T19:11:10.032 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:09 smithi016 bash[33624]: audit 2023-09-23T19:11:08.736274+0000 mon.a (mon.0) 796 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:11:10.032 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:09 smithi016 bash[33624]: cephadm 2023-09-23T19:11:08.736449+0000 mgr.x (mgr.15012) 76 : cephadm [INF] Upgrade: Updating mgr.y 2023-09-23T19:11:10.032 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:09 smithi016 bash[33624]: audit 2023-09-23T19:11:08.738472+0000 mon.c (mon.2) 202 : audit [INF] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T19:11:10.032 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:09 smithi016 bash[33624]: audit 2023-09-23T19:11:08.739372+0000 mon.a (mon.0) 797 : audit [INF] from='mgr.15012 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T19:11:10.032 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:09 smithi016 bash[33624]: audit 2023-09-23T19:11:08.745809+0000 mon.c (mon.2) 203 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T19:11:10.032 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:09 smithi016 bash[33624]: audit 2023-09-23T19:11:08.750233+0000 mon.c (mon.2) 204 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:11:10.032 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:09 smithi016 bash[33624]: cephadm 2023-09-23T19:11:08.759144+0000 mgr.x (mgr.15012) 77 : cephadm [INF] Deploying daemon mgr.y on smithi016 2023-09-23T19:11:10.032 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:09 smithi016 bash[33624]: cluster 2023-09-23T19:11:08.805406+0000 mgr.x (mgr.15012) 78 : cluster [DBG] pgmap v49: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 892 op/s 2023-09-23T19:11:10.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:09 smithi016 bash[27237]: audit 2023-09-23T19:11:08.736274+0000 mon.a (mon.0) 796 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:11:10.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:09 smithi016 bash[27237]: cephadm 2023-09-23T19:11:08.736449+0000 mgr.x (mgr.15012) 76 : cephadm [INF] Upgrade: Updating mgr.y 2023-09-23T19:11:10.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:09 smithi016 bash[27237]: audit 2023-09-23T19:11:08.738472+0000 mon.c (mon.2) 202 : audit [INF] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T19:11:10.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:09 smithi016 bash[27237]: audit 2023-09-23T19:11:08.739372+0000 mon.a (mon.0) 797 : audit [INF] from='mgr.15012 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T19:11:10.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:09 smithi016 bash[27237]: audit 2023-09-23T19:11:08.745809+0000 mon.c (mon.2) 203 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T19:11:10.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:09 smithi016 bash[27237]: audit 2023-09-23T19:11:08.750233+0000 mon.c (mon.2) 204 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:11:10.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:09 smithi016 bash[27237]: cephadm 2023-09-23T19:11:08.759144+0000 mgr.x (mgr.15012) 77 : cephadm [INF] Deploying daemon mgr.y on smithi016 2023-09-23T19:11:10.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:09 smithi016 bash[27237]: cluster 2023-09-23T19:11:08.805406+0000 mgr.x (mgr.15012) 78 : cluster [DBG] pgmap v49: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 892 op/s 2023-09-23T19:11:10.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:09 smithi092 bash[27556]: audit 2023-09-23T19:11:08.736274+0000 mon.a (mon.0) 796 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:11:10.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:09 smithi092 bash[27556]: cephadm 2023-09-23T19:11:08.736449+0000 mgr.x (mgr.15012) 76 : cephadm [INF] Upgrade: Updating mgr.y 2023-09-23T19:11:10.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:09 smithi092 bash[27556]: audit 2023-09-23T19:11:08.738472+0000 mon.c (mon.2) 202 : audit [INF] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T19:11:10.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:09 smithi092 bash[27556]: audit 2023-09-23T19:11:08.739372+0000 mon.a (mon.0) 797 : audit [INF] from='mgr.15012 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T19:11:10.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:09 smithi092 bash[27556]: audit 2023-09-23T19:11:08.745809+0000 mon.c (mon.2) 203 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T19:11:10.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:09 smithi092 bash[27556]: audit 2023-09-23T19:11:08.750233+0000 mon.c (mon.2) 204 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:11:10.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:09 smithi092 bash[27556]: cephadm 2023-09-23T19:11:08.759144+0000 mgr.x (mgr.15012) 77 : cephadm [INF] Deploying daemon mgr.y on smithi016 2023-09-23T19:11:10.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:09 smithi092 bash[27556]: cluster 2023-09-23T19:11:08.805406+0000 mgr.x (mgr.15012) 78 : cluster [DBG] pgmap v49: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 892 op/s 2023-09-23T19:11:11.363 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:11:11 smithi016 systemd[1]: Stopping Ceph mgr.y for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7... 2023-09-23T19:11:12.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:12 smithi016 bash[27237]: cluster 2023-09-23T19:11:10.805823+0000 mgr.x (mgr.15012) 79 : cluster [DBG] pgmap v50: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 588 op/s 2023-09-23T19:11:12.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:12 smithi016 bash[27237]: audit 2023-09-23T19:11:11.078955+0000 mon.c (mon.2) 205 : audit [DBG] from='client.? 172.21.15.16:0/2315846457' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:12.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:12 smithi016 bash[27237]: audit 2023-09-23T19:11:11.128112+0000 mon.c (mon.2) 206 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:11:12.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:12 smithi016 bash[33624]: cluster 2023-09-23T19:11:10.805823+0000 mgr.x (mgr.15012) 79 : cluster [DBG] pgmap v50: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 588 op/s 2023-09-23T19:11:12.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:12 smithi016 bash[33624]: audit 2023-09-23T19:11:11.078955+0000 mon.c (mon.2) 205 : audit [DBG] from='client.? 172.21.15.16:0/2315846457' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:12.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:12 smithi016 bash[33624]: audit 2023-09-23T19:11:11.128112+0000 mon.c (mon.2) 206 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:11:12.400 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:11:12 smithi016 bash[70884]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-mgr-y 2023-09-23T19:11:12.401 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:11:12 smithi016 bash[70940]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-mgr-y 2023-09-23T19:11:12.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:12 smithi092 bash[27556]: cluster 2023-09-23T19:11:10.805823+0000 mgr.x (mgr.15012) 79 : cluster [DBG] pgmap v50: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 588 op/s 2023-09-23T19:11:12.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:12 smithi092 bash[27556]: audit 2023-09-23T19:11:11.078955+0000 mon.c (mon.2) 205 : audit [DBG] from='client.? 172.21.15.16:0/2315846457' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:12.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:12 smithi092 bash[27556]: audit 2023-09-23T19:11:11.128112+0000 mon.c (mon.2) 206 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:11:12.724 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:11:12 smithi016 systemd[1]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@mgr.y.service: Main process exited, code=exited, status=143/n/a 2023-09-23T19:11:12.725 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:11:12 smithi016 systemd[1]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@mgr.y.service: Failed with result 'exit-code'. 2023-09-23T19:11:12.725 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:11:12 smithi016 systemd[1]: Stopped Ceph mgr.y for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7. 2023-09-23T19:11:12.725 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:11:12 smithi016 systemd[1]: Started Ceph mgr.y for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7. 2023-09-23T19:11:13.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:13 smithi016 bash[33624]: cluster 2023-09-23T19:11:12.031317+0000 mon.a (mon.0) 798 : cluster [DBG] mgrmap e26: x(active, since 91s), standbys: y 2023-09-23T19:11:13.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:13 smithi016 bash[33624]: audit 2023-09-23T19:11:12.767099+0000 mon.a (mon.0) 799 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:11:13.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:13 smithi016 bash[33624]: audit 2023-09-23T19:11:12.783202+0000 mon.a (mon.0) 800 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:11:13.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:13 smithi016 bash[33624]: audit 2023-09-23T19:11:12.799736+0000 mon.c (mon.2) 207 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:11:13.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:13 smithi016 bash[27237]: cluster 2023-09-23T19:11:12.031317+0000 mon.a (mon.0) 798 : cluster [DBG] mgrmap e26: x(active, since 91s), standbys: y 2023-09-23T19:11:13.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:13 smithi016 bash[27237]: audit 2023-09-23T19:11:12.767099+0000 mon.a (mon.0) 799 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:11:13.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:13 smithi016 bash[27237]: audit 2023-09-23T19:11:12.783202+0000 mon.a (mon.0) 800 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:11:13.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:13 smithi016 bash[27237]: audit 2023-09-23T19:11:12.799736+0000 mon.c (mon.2) 207 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:11:13.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:13 smithi092 bash[27556]: cluster 2023-09-23T19:11:12.031317+0000 mon.a (mon.0) 798 : cluster [DBG] mgrmap e26: x(active, since 91s), standbys: y 2023-09-23T19:11:13.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:13 smithi092 bash[27556]: audit 2023-09-23T19:11:12.767099+0000 mon.a (mon.0) 799 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:11:13.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:13 smithi092 bash[27556]: audit 2023-09-23T19:11:12.783202+0000 mon.a (mon.0) 800 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:11:13.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:13 smithi092 bash[27556]: audit 2023-09-23T19:11:12.799736+0000 mon.c (mon.2) 207 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:11:14.270 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:14 smithi016 bash[27237]: cluster 2023-09-23T19:11:12.806448+0000 mgr.x (mgr.15012) 80 : cluster [DBG] pgmap v51: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 255 B/s wr, 878 op/s 2023-09-23T19:11:14.270 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:14 smithi016 bash[27237]: audit 2023-09-23T19:11:13.804060+0000 mon.c (mon.2) 208 : audit [DBG] from='client.? 172.21.15.16:0/2000751318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:14.271 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:14 smithi016 bash[33624]: cluster 2023-09-23T19:11:12.806448+0000 mgr.x (mgr.15012) 80 : cluster [DBG] pgmap v51: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 255 B/s wr, 878 op/s 2023-09-23T19:11:14.271 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:14 smithi016 bash[33624]: audit 2023-09-23T19:11:13.804060+0000 mon.c (mon.2) 208 : audit [DBG] from='client.? 172.21.15.16:0/2000751318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:14.271 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:11:14 smithi016 bash[70992]: debug 2023-09-23T19:11:14.263+0000 7faa1c668200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T19:11:14.535 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:14 smithi092 bash[27556]: cluster 2023-09-23T19:11:12.806448+0000 mgr.x (mgr.15012) 80 : cluster [DBG] pgmap v51: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 255 B/s wr, 878 op/s 2023-09-23T19:11:14.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:14 smithi092 bash[27556]: audit 2023-09-23T19:11:13.804060+0000 mon.c (mon.2) 208 : audit [DBG] from='client.? 172.21.15.16:0/2000751318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:16.028 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:11:15 smithi016 bash[70992]: debug 2023-09-23T19:11:15.675+0000 7faa1c668200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T19:11:16.028 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:11:15 smithi016 bash[70992]: debug 2023-09-23T19:11:15.855+0000 7faa1c668200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T19:11:16.028 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:11:16 smithi016 bash[70992]: debug 2023-09-23T19:11:16.023+0000 7faa1c668200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T19:11:16.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:16 smithi016 bash[27237]: cluster 2023-09-23T19:11:14.806831+0000 mgr.x (mgr.15012) 81 : cluster [DBG] pgmap v52: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 255 B/s wr, 577 op/s 2023-09-23T19:11:16.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:16 smithi016 bash[33624]: cluster 2023-09-23T19:11:14.806831+0000 mgr.x (mgr.15012) 81 : cluster [DBG] pgmap v52: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 255 B/s wr, 577 op/s 2023-09-23T19:11:16.535 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:16 smithi092 bash[27556]: cluster 2023-09-23T19:11:14.806831+0000 mgr.x (mgr.15012) 81 : cluster [DBG] pgmap v52: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 255 B/s wr, 577 op/s 2023-09-23T19:11:16.784 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:11:16 smithi016 bash[70992]: debug 2023-09-23T19:11:16.443+0000 7faa1c668200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T19:11:17.275 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:17 smithi016 bash[33624]: audit 2023-09-23T19:11:16.594090+0000 mon.b (mon.1) 46 : audit [DBG] from='client.? 172.21.15.16:0/3553704180' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:17.276 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:17 smithi016 bash[27237]: audit 2023-09-23T19:11:16.594090+0000 mon.b (mon.1) 46 : audit [DBG] from='client.? 172.21.15.16:0/3553704180' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:17.276 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:11:17 smithi016 bash[70992]: debug 2023-09-23T19:11:17.011+0000 7faa1c668200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T19:11:17.276 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:11:17 smithi016 bash[70992]: debug 2023-09-23T19:11:17.271+0000 7faa1c668200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T19:11:17.535 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:17 smithi092 bash[27556]: audit 2023-09-23T19:11:16.594090+0000 mon.b (mon.1) 46 : audit [DBG] from='client.? 172.21.15.16:0/3553704180' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:18.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:11:17 smithi016 bash[70992]: debug 2023-09-23T19:11:17.603+0000 7faa1c668200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T19:11:18.328 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:18 smithi016 bash[27237]: cluster 2023-09-23T19:11:16.807211+0000 mgr.x (mgr.15012) 82 : cluster [DBG] pgmap v53: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 255 B/s wr, 577 op/s 2023-09-23T19:11:18.328 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:18 smithi016 bash[33624]: cluster 2023-09-23T19:11:16.807211+0000 mgr.x (mgr.15012) 82 : cluster [DBG] pgmap v53: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 255 B/s wr, 577 op/s 2023-09-23T19:11:18.535 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:18 smithi092 bash[27556]: cluster 2023-09-23T19:11:16.807211+0000 mgr.x (mgr.15012) 82 : cluster [DBG] pgmap v53: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 255 B/s wr, 577 op/s 2023-09-23T19:11:19.035 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:11:18 smithi092 bash[52849]: ts=2023-09-23T19:11:18.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi092:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi092.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T19:11:19.535 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:11:19 smithi092 bash[50491]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:11:19] "GET /metrics HTTP/1.1" 200 36065 "" "Prometheus/2.33.4" 2023-09-23T19:11:19.660 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:11:19 smithi016 bash[70992]: debug 2023-09-23T19:11:19.283+0000 7faa1c668200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T19:11:19.660 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:11:19 smithi016 bash[70992]: debug 2023-09-23T19:11:19.467+0000 7faa1c668200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T19:11:19.660 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:11:19 smithi016 bash[70992]: debug 2023-09-23T19:11:19.655+0000 7faa1c668200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T19:11:20.437 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:20 smithi016 bash[27237]: cluster 2023-09-23T19:11:18.807999+0000 mgr.x (mgr.15012) 83 : cluster [DBG] pgmap v54: 89 pgs: 89 active+clean; 2.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 255 B/s wr, 855 op/s 2023-09-23T19:11:20.437 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:20 smithi016 bash[27237]: audit 2023-09-23T19:11:19.321692+0000 mon.a (mon.0) 801 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:11:20.438 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:20 smithi016 bash[27237]: audit 2023-09-23T19:11:19.337399+0000 mon.a (mon.0) 802 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:11:20.438 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:20 smithi016 bash[27237]: audit 2023-09-23T19:11:19.413074+0000 mon.a (mon.0) 803 : audit [DBG] from='client.? 172.21.15.16:0/3792296425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:20.438 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:20 smithi016 bash[27237]: audit 2023-09-23T19:11:19.502546+0000 mon.a (mon.0) 804 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:11:20.438 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:20 smithi016 bash[27237]: audit 2023-09-23T19:11:19.518911+0000 mon.a (mon.0) 805 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:11:20.438 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:11:20 smithi016 bash[70992]: debug 2023-09-23T19:11:20.431+0000 7faa1c668200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T19:11:20.438 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:20 smithi016 bash[33624]: cluster 2023-09-23T19:11:18.807999+0000 mgr.x (mgr.15012) 83 : cluster [DBG] pgmap v54: 89 pgs: 89 active+clean; 2.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 255 B/s wr, 855 op/s 2023-09-23T19:11:20.438 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:20 smithi016 bash[33624]: audit 2023-09-23T19:11:19.321692+0000 mon.a (mon.0) 801 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:11:20.438 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:20 smithi016 bash[33624]: audit 2023-09-23T19:11:19.337399+0000 mon.a (mon.0) 802 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:11:20.439 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:20 smithi016 bash[33624]: audit 2023-09-23T19:11:19.413074+0000 mon.a (mon.0) 803 : audit [DBG] from='client.? 172.21.15.16:0/3792296425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:20.439 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:20 smithi016 bash[33624]: audit 2023-09-23T19:11:19.502546+0000 mon.a (mon.0) 804 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:11:20.439 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:20 smithi016 bash[33624]: audit 2023-09-23T19:11:19.518911+0000 mon.a (mon.0) 805 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:11:20.535 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:20 smithi092 bash[27556]: cluster 2023-09-23T19:11:18.807999+0000 mgr.x (mgr.15012) 83 : cluster [DBG] pgmap v54: 89 pgs: 89 active+clean; 2.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 255 B/s wr, 855 op/s 2023-09-23T19:11:20.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:20 smithi092 bash[27556]: audit 2023-09-23T19:11:19.321692+0000 mon.a (mon.0) 801 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:11:20.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:20 smithi092 bash[27556]: audit 2023-09-23T19:11:19.337399+0000 mon.a (mon.0) 802 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:11:20.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:20 smithi092 bash[27556]: audit 2023-09-23T19:11:19.413074+0000 mon.a (mon.0) 803 : audit [DBG] from='client.? 172.21.15.16:0/3792296425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:20.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:20 smithi092 bash[27556]: audit 2023-09-23T19:11:19.502546+0000 mon.a (mon.0) 804 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:11:20.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:20 smithi092 bash[27556]: audit 2023-09-23T19:11:19.518911+0000 mon.a (mon.0) 805 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:11:21.284 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:11:20 smithi016 bash[70992]: debug 2023-09-23T19:11:20.779+0000 7faa1c668200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T19:11:21.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:21 smithi016 bash[27237]: audit 2023-09-23T19:11:20.638669+0000 mon.a (mon.0) 806 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:11:21.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:21 smithi016 bash[27237]: audit 2023-09-23T19:11:20.650320+0000 mon.a (mon.0) 807 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:11:21.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:21 smithi016 bash[27237]: cluster 2023-09-23T19:11:20.808520+0000 mgr.x (mgr.15012) 84 : cluster [DBG] pgmap v55: 89 pgs: 89 active+clean; 2.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 255 B/s wr, 568 op/s 2023-09-23T19:11:21.785 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:11:21 smithi016 bash[70992]: debug 2023-09-23T19:11:21.431+0000 7faa1c668200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T19:11:21.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:21 smithi016 bash[33624]: audit 2023-09-23T19:11:20.638669+0000 mon.a (mon.0) 806 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:11:21.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:21 smithi016 bash[33624]: audit 2023-09-23T19:11:20.650320+0000 mon.a (mon.0) 807 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:11:21.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:21 smithi016 bash[33624]: cluster 2023-09-23T19:11:20.808520+0000 mgr.x (mgr.15012) 84 : cluster [DBG] pgmap v55: 89 pgs: 89 active+clean; 2.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 255 B/s wr, 568 op/s 2023-09-23T19:11:22.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:21 smithi092 bash[27556]: audit 2023-09-23T19:11:20.638669+0000 mon.a (mon.0) 806 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:11:22.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:21 smithi092 bash[27556]: audit 2023-09-23T19:11:20.650320+0000 mon.a (mon.0) 807 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:11:22.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:21 smithi092 bash[27556]: cluster 2023-09-23T19:11:20.808520+0000 mgr.x (mgr.15012) 84 : cluster [DBG] pgmap v55: 89 pgs: 89 active+clean; 2.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 255 B/s wr, 568 op/s 2023-09-23T19:11:22.534 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:11:22 smithi016 bash[70992]: debug 2023-09-23T19:11:22.147+0000 7faa1c668200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T19:11:22.995 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:22 smithi016 bash[27237]: audit 2023-09-23T19:11:22.226941+0000 mon.c (mon.2) 209 : audit [DBG] from='client.? 172.21.15.16:0/3050742649' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:22.995 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:11:22 smithi016 bash[70992]: debug 2023-09-23T19:11:22.991+0000 7faa1c668200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T19:11:22.995 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:22 smithi016 bash[33624]: audit 2023-09-23T19:11:22.226941+0000 mon.c (mon.2) 209 : audit [DBG] from='client.? 172.21.15.16:0/3050742649' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:23.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:22 smithi092 bash[27556]: audit 2023-09-23T19:11:22.226941+0000 mon.c (mon.2) 209 : audit [DBG] from='client.? 172.21.15.16:0/3050742649' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:23.284 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:11:23 smithi016 bash[70992]: debug 2023-09-23T19:11:23.151+0000 7faa1c668200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T19:11:24.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:23 smithi016 bash[33624]: cluster 2023-09-23T19:11:22.809355+0000 mgr.x (mgr.15012) 85 : cluster [DBG] pgmap v56: 89 pgs: 89 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 255 B/s wr, 819 op/s 2023-09-23T19:11:24.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:23 smithi016 bash[27237]: cluster 2023-09-23T19:11:22.809355+0000 mgr.x (mgr.15012) 85 : cluster [DBG] pgmap v56: 89 pgs: 89 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 255 B/s wr, 819 op/s 2023-09-23T19:11:24.035 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:11:23 smithi016 bash[70992]: debug 2023-09-23T19:11:23.687+0000 7faa1c668200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T19:11:24.035 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:11:23 smithi016 bash[70992]: debug 2023-09-23T19:11:23.859+0000 7faa1c668200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T19:11:24.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:23 smithi092 bash[27556]: cluster 2023-09-23T19:11:22.809355+0000 mgr.x (mgr.15012) 85 : cluster [DBG] pgmap v56: 89 pgs: 89 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 255 B/s wr, 819 op/s 2023-09-23T19:11:24.458 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:11:24 smithi016 bash[70992]: debug 2023-09-23T19:11:24.079+0000 7faa1c668200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T19:11:24.458 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:11:24 smithi016 bash[70992]: debug 2023-09-23T19:11:24.259+0000 7faa1c668200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T19:11:24.458 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:11:24 smithi016 bash[70992]: debug 2023-09-23T19:11:24.451+0000 7faa1c668200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T19:11:24.757 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:11:24 smithi016 bash[70992]: debug 2023-09-23T19:11:24.751+0000 7faa1c668200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T19:11:25.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:11:24 smithi016 bash[70992]: debug 2023-09-23T19:11:24.943+0000 7faa1c668200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T19:11:26.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:25 smithi016 bash[27237]: cluster 2023-09-23T19:11:24.809801+0000 mgr.x (mgr.15012) 86 : cluster [DBG] pgmap v57: 89 pgs: 89 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 529 op/s 2023-09-23T19:11:26.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:25 smithi016 bash[27237]: audit 2023-09-23T19:11:24.985311+0000 mon.c (mon.2) 210 : audit [DBG] from='client.? 172.21.15.16:0/3045006185' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:26.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:25 smithi016 bash[33624]: cluster 2023-09-23T19:11:24.809801+0000 mgr.x (mgr.15012) 86 : cluster [DBG] pgmap v57: 89 pgs: 89 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 529 op/s 2023-09-23T19:11:26.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:25 smithi016 bash[33624]: audit 2023-09-23T19:11:24.985311+0000 mon.c (mon.2) 210 : audit [DBG] from='client.? 172.21.15.16:0/3045006185' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:26.285 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:25 smithi092 bash[27556]: cluster 2023-09-23T19:11:24.809801+0000 mgr.x (mgr.15012) 86 : cluster [DBG] pgmap v57: 89 pgs: 89 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 529 op/s 2023-09-23T19:11:26.285 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:25 smithi092 bash[27556]: audit 2023-09-23T19:11:24.985311+0000 mon.c (mon.2) 210 : audit [DBG] from='client.? 172.21.15.16:0/3045006185' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:27.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:26 smithi016 bash[33624]: audit 2023-09-23T19:11:26.128100+0000 mon.c (mon.2) 211 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:11:27.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:26 smithi016 bash[33624]: cluster 2023-09-23T19:11:26.812762+0000 mon.a (mon.0) 808 : cluster [DBG] Standby manager daemon y restarted 2023-09-23T19:11:27.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:26 smithi016 bash[33624]: cluster 2023-09-23T19:11:26.812913+0000 mon.a (mon.0) 809 : cluster [DBG] Standby manager daemon y started 2023-09-23T19:11:27.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:26 smithi016 bash[33624]: audit 2023-09-23T19:11:26.815079+0000 mon.a (mon.0) 810 : audit [DBG] from='mgr.? 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T19:11:27.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:26 smithi016 bash[33624]: audit 2023-09-23T19:11:26.819832+0000 mon.a (mon.0) 811 : audit [DBG] from='mgr.? 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T19:11:27.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:26 smithi016 bash[33624]: audit 2023-09-23T19:11:26.827379+0000 mon.a (mon.0) 812 : audit [DBG] from='mgr.? 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T19:11:27.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:26 smithi016 bash[33624]: audit 2023-09-23T19:11:26.828461+0000 mon.a (mon.0) 813 : audit [DBG] from='mgr.? 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T19:11:27.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:26 smithi016 bash[27237]: audit 2023-09-23T19:11:26.128100+0000 mon.c (mon.2) 211 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:11:27.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:26 smithi016 bash[27237]: cluster 2023-09-23T19:11:26.812762+0000 mon.a (mon.0) 808 : cluster [DBG] Standby manager daemon y restarted 2023-09-23T19:11:27.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:26 smithi016 bash[27237]: cluster 2023-09-23T19:11:26.812913+0000 mon.a (mon.0) 809 : cluster [DBG] Standby manager daemon y started 2023-09-23T19:11:27.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:26 smithi016 bash[27237]: audit 2023-09-23T19:11:26.815079+0000 mon.a (mon.0) 810 : audit [DBG] from='mgr.? 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T19:11:27.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:26 smithi016 bash[27237]: audit 2023-09-23T19:11:26.819832+0000 mon.a (mon.0) 811 : audit [DBG] from='mgr.? 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T19:11:27.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:26 smithi016 bash[27237]: audit 2023-09-23T19:11:26.827379+0000 mon.a (mon.0) 812 : audit [DBG] from='mgr.? 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T19:11:27.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:26 smithi016 bash[27237]: audit 2023-09-23T19:11:26.828461+0000 mon.a (mon.0) 813 : audit [DBG] from='mgr.? 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T19:11:27.286 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:11:26 smithi016 bash[70992]: debug 2023-09-23T19:11:26.787+0000 7faa1c668200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T19:11:27.286 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:11:26 smithi016 bash[70992]: [23/Sep/2023:19:11:26] ENGINE Bus STARTING 2023-09-23T19:11:27.286 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:11:26 smithi016 bash[70992]: CherryPy Checker: 2023-09-23T19:11:27.287 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:11:26 smithi016 bash[70992]: The Application mounted at '' has an empty config. 2023-09-23T19:11:27.287 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:11:26 smithi016 bash[70992]: [23/Sep/2023:19:11:26] ENGINE Serving on http://:::9283 2023-09-23T19:11:27.287 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:11:26 smithi016 bash[70992]: [23/Sep/2023:19:11:26] ENGINE Bus STARTED 2023-09-23T19:11:27.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:26 smithi092 bash[27556]: audit 2023-09-23T19:11:26.128100+0000 mon.c (mon.2) 211 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:11:27.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:26 smithi092 bash[27556]: cluster 2023-09-23T19:11:26.812762+0000 mon.a (mon.0) 808 : cluster [DBG] Standby manager daemon y restarted 2023-09-23T19:11:27.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:26 smithi092 bash[27556]: cluster 2023-09-23T19:11:26.812913+0000 mon.a (mon.0) 809 : cluster [DBG] Standby manager daemon y started 2023-09-23T19:11:27.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:26 smithi092 bash[27556]: audit 2023-09-23T19:11:26.815079+0000 mon.a (mon.0) 810 : audit [DBG] from='mgr.? 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T19:11:27.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:26 smithi092 bash[27556]: audit 2023-09-23T19:11:26.819832+0000 mon.a (mon.0) 811 : audit [DBG] from='mgr.? 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T19:11:27.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:26 smithi092 bash[27556]: audit 2023-09-23T19:11:26.827379+0000 mon.a (mon.0) 812 : audit [DBG] from='mgr.? 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T19:11:27.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:26 smithi092 bash[27556]: audit 2023-09-23T19:11:26.828461+0000 mon.a (mon.0) 813 : audit [DBG] from='mgr.? 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T19:11:28.148 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:27 smithi016 bash[27237]: cluster 2023-09-23T19:11:26.810230+0000 mgr.x (mgr.15012) 87 : cluster [DBG] pgmap v58: 89 pgs: 89 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 529 op/s 2023-09-23T19:11:28.148 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:27 smithi016 bash[27237]: cluster 2023-09-23T19:11:26.889603+0000 mon.a (mon.0) 814 : cluster [DBG] mgrmap e27: x(active, since 106s), standbys: y 2023-09-23T19:11:28.148 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:27 smithi016 bash[27237]: audit 2023-09-23T19:11:27.657410+0000 mon.a (mon.0) 815 : audit [DBG] from='client.? 172.21.15.16:0/435920931' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:28.149 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:27 smithi016 bash[33624]: cluster 2023-09-23T19:11:26.810230+0000 mgr.x (mgr.15012) 87 : cluster [DBG] pgmap v58: 89 pgs: 89 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 529 op/s 2023-09-23T19:11:28.149 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:27 smithi016 bash[33624]: cluster 2023-09-23T19:11:26.889603+0000 mon.a (mon.0) 814 : cluster [DBG] mgrmap e27: x(active, since 106s), standbys: y 2023-09-23T19:11:28.149 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:27 smithi016 bash[33624]: audit 2023-09-23T19:11:27.657410+0000 mon.a (mon.0) 815 : audit [DBG] from='client.? 172.21.15.16:0/435920931' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:28.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:27 smithi092 bash[27556]: cluster 2023-09-23T19:11:26.810230+0000 mgr.x (mgr.15012) 87 : cluster [DBG] pgmap v58: 89 pgs: 89 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 529 op/s 2023-09-23T19:11:28.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:27 smithi092 bash[27556]: cluster 2023-09-23T19:11:26.889603+0000 mon.a (mon.0) 814 : cluster [DBG] mgrmap e27: x(active, since 106s), standbys: y 2023-09-23T19:11:28.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:27 smithi092 bash[27556]: audit 2023-09-23T19:11:27.657410+0000 mon.a (mon.0) 815 : audit [DBG] from='client.? 172.21.15.16:0/435920931' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:29.036 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:11:28 smithi092 bash[52849]: ts=2023-09-23T19:11:28.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi092:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi092.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T19:11:29.535 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:11:29 smithi092 bash[50491]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:11:29] "GET /metrics HTTP/1.1" 200 36059 "" "Prometheus/2.33.4" 2023-09-23T19:11:30.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:29 smithi016 bash[27237]: cluster 2023-09-23T19:11:28.810932+0000 mgr.x (mgr.15012) 88 : cluster [DBG] pgmap v59: 89 pgs: 89 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 808 op/s 2023-09-23T19:11:30.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:29 smithi016 bash[33624]: cluster 2023-09-23T19:11:28.810932+0000 mgr.x (mgr.15012) 88 : cluster [DBG] pgmap v59: 89 pgs: 89 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 808 op/s 2023-09-23T19:11:30.285 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:29 smithi092 bash[27556]: cluster 2023-09-23T19:11:28.810932+0000 mgr.x (mgr.15012) 88 : cluster [DBG] pgmap v59: 89 pgs: 89 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 808 op/s 2023-09-23T19:11:31.313 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:31 smithi016 bash[27237]: audit 2023-09-23T19:11:30.392674+0000 mon.a (mon.0) 816 : audit [DBG] from='client.? 172.21.15.16:0/1815193635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:31.314 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:31 smithi016 bash[33624]: audit 2023-09-23T19:11:30.392674+0000 mon.a (mon.0) 816 : audit [DBG] from='client.? 172.21.15.16:0/1815193635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:31.535 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:31 smithi092 bash[27556]: audit 2023-09-23T19:11:30.392674+0000 mon.a (mon.0) 816 : audit [DBG] from='client.? 172.21.15.16:0/1815193635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:32.303 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:32 smithi016 bash[27237]: cluster 2023-09-23T19:11:30.811362+0000 mgr.x (mgr.15012) 89 : cluster [DBG] pgmap v60: 89 pgs: 89 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 529 op/s 2023-09-23T19:11:32.303 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:32 smithi016 bash[33624]: cluster 2023-09-23T19:11:30.811362+0000 mgr.x (mgr.15012) 89 : cluster [DBG] pgmap v60: 89 pgs: 89 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 529 op/s 2023-09-23T19:11:32.304 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:11:32 smithi016 bash[70992]: [23/Sep/2023:19:11:32] ENGINE Bus STOPPING 2023-09-23T19:11:32.304 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:11:32 smithi016 bash[70992]: [23/Sep/2023:19:11:32] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T19:11:32.304 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:11:32 smithi016 bash[70992]: [23/Sep/2023:19:11:32] ENGINE Bus STOPPED 2023-09-23T19:11:32.304 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:32 smithi092 bash[27556]: cluster 2023-09-23T19:11:30.811362+0000 mgr.x (mgr.15012) 89 : cluster [DBG] pgmap v60: 89 pgs: 89 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 529 op/s 2023-09-23T19:11:32.572 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:11:32 smithi092 bash[50491]: ignoring --setuser ceph since I am not root 2023-09-23T19:11:32.572 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:11:32 smithi092 bash[50491]: ignoring --setgroup ceph since I am not root 2023-09-23T19:11:32.572 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:11:32 smithi092 bash[50491]: debug 2023-09-23T19:11:32.337+0000 7fcb3fac0700 1 -- 172.21.15.92:0/3937987974 <== mon.1 v2:172.21.15.92:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55f910b4b1e0 con 0x55f9118cac00 2023-09-23T19:11:32.671 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:11:32 smithi016 bash[70992]: [23/Sep/2023:19:11:32] ENGINE Bus STARTING 2023-09-23T19:11:32.957 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:11:32 smithi092 bash[50491]: debug 2023-09-23T19:11:32.569+0000 7fcb497ef200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T19:11:33.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:11:32 smithi016 bash[70992]: [23/Sep/2023:19:11:32] ENGINE Serving on http://:::9283 2023-09-23T19:11:33.035 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:11:32 smithi016 bash[70992]: [23/Sep/2023:19:11:32] ENGINE Bus STARTED 2023-09-23T19:11:33.035 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:11:32 smithi016 bash[70992]: debug 2023-09-23T19:11:32.871+0000 7fa97eacd700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:11:33.035 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:11:32 smithi016 bash[70992]: debug 2023-09-23T19:11:32.871+0000 7fa97eacd700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:11:33.035 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:11:32 smithi016 bash[70992]: debug 2023-09-23T19:11:32.871+0000 7fa97eacd700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:11:33.035 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:11:32 smithi016 bash[70992]: debug 2023-09-23T19:11:32.871+0000 7fa97eacd700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:11:33.035 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:11:32 smithi016 bash[70992]: debug 2023-09-23T19:11:32.875+0000 7fa97eacd700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:11:33.035 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:11:32 smithi016 bash[70992]: debug 2023-09-23T19:11:32.891+0000 7fa978ac1700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:11:33.035 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:11:32 smithi016 bash[70992]: debug 2023-09-23T19:11:32.891+0000 7fa978ac1700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:11:33.035 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:11:32 smithi016 bash[70992]: debug 2023-09-23T19:11:32.891+0000 7fa978ac1700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:11:33.035 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:11:32 smithi016 bash[70992]: debug 2023-09-23T19:11:32.891+0000 7fa978ac1700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:11:33.036 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:11:32 smithi016 bash[70992]: debug 2023-09-23T19:11:32.891+0000 7fa978ac1700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:11:33.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:33 smithi092 bash[27556]: audit 2023-09-23T19:11:32.134703+0000 mon.a (mon.0) 817 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:11:33.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:33 smithi092 bash[27556]: audit 2023-09-23T19:11:32.150136+0000 mon.a (mon.0) 818 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:11:33.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:33 smithi092 bash[27556]: audit 2023-09-23T19:11:32.154499+0000 mon.c (mon.2) 212 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:11:33.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:33 smithi092 bash[27556]: audit 2023-09-23T19:11:32.155775+0000 mon.c (mon.2) 213 : audit [INF] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:11:33.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:33 smithi092 bash[27556]: audit 2023-09-23T19:11:32.162804+0000 mon.a (mon.0) 819 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:11:33.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:33 smithi092 bash[27556]: audit 2023-09-23T19:11:32.236949+0000 mon.c (mon.2) 214 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:11:33.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:33 smithi092 bash[27556]: audit 2023-09-23T19:11:32.246918+0000 mon.c (mon.2) 215 : audit [INF] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T19:11:33.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:33 smithi092 bash[27556]: audit 2023-09-23T19:11:32.247418+0000 mon.a (mon.0) 820 : audit [INF] from='mgr.15012 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T19:11:33.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:33 smithi092 bash[27556]: cluster 2023-09-23T19:11:32.257229+0000 mon.a (mon.0) 821 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2023-09-23T19:11:33.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:33 smithi092 bash[27556]: audit 2023-09-23T19:11:32.257807+0000 mon.a (mon.0) 822 : audit [INF] from='mgr.15012 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-09-23T19:11:33.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:33 smithi092 bash[27556]: cluster 2023-09-23T19:11:32.257924+0000 mon.a (mon.0) 823 : cluster [DBG] mgrmap e28: y(active, starting, since 0.00994349s) 2023-09-23T19:11:33.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:33 smithi092 bash[27556]: audit 2023-09-23T19:11:32.262414+0000 mon.c (mon.2) 216 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:11:33.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:33 smithi092 bash[27556]: audit 2023-09-23T19:11:32.262750+0000 mon.c (mon.2) 217 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:11:33.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:33 smithi092 bash[27556]: audit 2023-09-23T19:11:32.262959+0000 mon.c (mon.2) 218 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:11:33.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:33 smithi092 bash[27556]: audit 2023-09-23T19:11:32.263101+0000 mon.c (mon.2) 219 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi016.uncssv"}]: dispatch 2023-09-23T19:11:33.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:33 smithi092 bash[27556]: audit 2023-09-23T19:11:32.263491+0000 mon.c (mon.2) 220 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi092.vbpqks"}]: dispatch 2023-09-23T19:11:33.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:33 smithi092 bash[27556]: audit 2023-09-23T19:11:32.264009+0000 mon.c (mon.2) 221 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T19:11:33.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:33 smithi092 bash[27556]: audit 2023-09-23T19:11:32.264532+0000 mon.c (mon.2) 222 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T19:11:33.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:33 smithi092 bash[27556]: audit 2023-09-23T19:11:32.264921+0000 mon.c (mon.2) 223 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T19:11:33.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:33 smithi092 bash[27556]: audit 2023-09-23T19:11:32.265226+0000 mon.c (mon.2) 224 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T19:11:33.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:33 smithi092 bash[27556]: audit 2023-09-23T19:11:32.265533+0000 mon.c (mon.2) 225 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:11:33.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:33 smithi092 bash[27556]: audit 2023-09-23T19:11:32.265883+0000 mon.c (mon.2) 226 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T19:11:33.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:33 smithi092 bash[27556]: audit 2023-09-23T19:11:32.266223+0000 mon.c (mon.2) 227 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:11:33.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:33 smithi092 bash[27556]: audit 2023-09-23T19:11:32.266473+0000 mon.c (mon.2) 228 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:11:33.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:33 smithi092 bash[27556]: audit 2023-09-23T19:11:32.266713+0000 mon.c (mon.2) 229 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:11:33.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:33 smithi092 bash[27556]: audit 2023-09-23T19:11:32.267224+0000 mon.c (mon.2) 230 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T19:11:33.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:33 smithi092 bash[27556]: audit 2023-09-23T19:11:32.267594+0000 mon.c (mon.2) 231 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T19:11:33.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:33 smithi092 bash[27556]: audit 2023-09-23T19:11:32.269590+0000 mon.c (mon.2) 232 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T19:11:33.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:33 smithi092 bash[27556]: cluster 2023-09-23T19:11:32.329971+0000 mon.a (mon.0) 824 : cluster [INF] Manager daemon y is now available 2023-09-23T19:11:33.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:33 smithi092 bash[27556]: audit 2023-09-23T19:11:32.444718+0000 mon.c (mon.2) 233 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:11:33.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:33 smithi092 bash[27556]: audit 2023-09-23T19:11:32.463735+0000 mon.c (mon.2) 234 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:11:33.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:33 smithi092 bash[27556]: audit 2023-09-23T19:11:32.475891+0000 mon.c (mon.2) 235 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:11:33.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:33 smithi092 bash[27556]: audit 2023-09-23T19:11:32.578089+0000 mon.c (mon.2) 236 : audit [INF] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T19:11:33.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:33 smithi092 bash[27556]: audit 2023-09-23T19:11:32.578933+0000 mon.a (mon.0) 825 : audit [INF] from='mgr.15228 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T19:11:33.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:33 smithi092 bash[27556]: audit 2023-09-23T19:11:32.715669+0000 mon.c (mon.2) 237 : audit [INF] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T19:11:33.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:33 smithi092 bash[27556]: audit 2023-09-23T19:11:32.716292+0000 mon.a (mon.0) 826 : audit [INF] from='mgr.15228 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T19:11:33.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[33624]: audit 2023-09-23T19:11:32.134703+0000 mon.a (mon.0) 817 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:11:33.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[33624]: audit 2023-09-23T19:11:32.150136+0000 mon.a (mon.0) 818 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:11:33.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[33624]: audit 2023-09-23T19:11:32.154499+0000 mon.c (mon.2) 212 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:11:33.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[33624]: audit 2023-09-23T19:11:32.155775+0000 mon.c (mon.2) 213 : audit [INF] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:11:33.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[33624]: audit 2023-09-23T19:11:32.162804+0000 mon.a (mon.0) 819 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:11:33.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[33624]: audit 2023-09-23T19:11:32.236949+0000 mon.c (mon.2) 214 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:11:33.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[33624]: audit 2023-09-23T19:11:32.246918+0000 mon.c (mon.2) 215 : audit [INF] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T19:11:33.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[33624]: audit 2023-09-23T19:11:32.247418+0000 mon.a (mon.0) 820 : audit [INF] from='mgr.15012 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T19:11:33.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[33624]: cluster 2023-09-23T19:11:32.257229+0000 mon.a (mon.0) 821 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2023-09-23T19:11:33.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[33624]: audit 2023-09-23T19:11:32.257807+0000 mon.a (mon.0) 822 : audit [INF] from='mgr.15012 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-09-23T19:11:33.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[33624]: cluster 2023-09-23T19:11:32.257924+0000 mon.a (mon.0) 823 : cluster [DBG] mgrmap e28: y(active, starting, since 0.00994349s) 2023-09-23T19:11:33.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[33624]: audit 2023-09-23T19:11:32.262414+0000 mon.c (mon.2) 216 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:11:33.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[33624]: audit 2023-09-23T19:11:32.262750+0000 mon.c (mon.2) 217 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:11:33.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[33624]: audit 2023-09-23T19:11:32.262959+0000 mon.c (mon.2) 218 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:11:33.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[33624]: audit 2023-09-23T19:11:32.263101+0000 mon.c (mon.2) 219 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi016.uncssv"}]: dispatch 2023-09-23T19:11:33.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[33624]: audit 2023-09-23T19:11:32.263491+0000 mon.c (mon.2) 220 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi092.vbpqks"}]: dispatch 2023-09-23T19:11:33.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[33624]: audit 2023-09-23T19:11:32.264009+0000 mon.c (mon.2) 221 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T19:11:33.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[33624]: audit 2023-09-23T19:11:32.264532+0000 mon.c (mon.2) 222 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T19:11:33.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[33624]: audit 2023-09-23T19:11:32.264921+0000 mon.c (mon.2) 223 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T19:11:33.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[33624]: audit 2023-09-23T19:11:32.265226+0000 mon.c (mon.2) 224 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T19:11:33.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[33624]: audit 2023-09-23T19:11:32.265533+0000 mon.c (mon.2) 225 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:11:33.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[33624]: audit 2023-09-23T19:11:32.265883+0000 mon.c (mon.2) 226 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T19:11:33.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[33624]: audit 2023-09-23T19:11:32.266223+0000 mon.c (mon.2) 227 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:11:33.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[33624]: audit 2023-09-23T19:11:32.266473+0000 mon.c (mon.2) 228 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:11:33.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[33624]: audit 2023-09-23T19:11:32.266713+0000 mon.c (mon.2) 229 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:11:33.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[33624]: audit 2023-09-23T19:11:32.267224+0000 mon.c (mon.2) 230 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T19:11:33.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[33624]: audit 2023-09-23T19:11:32.267594+0000 mon.c (mon.2) 231 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T19:11:33.538 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[33624]: audit 2023-09-23T19:11:32.269590+0000 mon.c (mon.2) 232 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T19:11:33.538 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[33624]: cluster 2023-09-23T19:11:32.329971+0000 mon.a (mon.0) 824 : cluster [INF] Manager daemon y is now available 2023-09-23T19:11:33.538 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[33624]: audit 2023-09-23T19:11:32.444718+0000 mon.c (mon.2) 233 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:11:33.538 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[33624]: audit 2023-09-23T19:11:32.463735+0000 mon.c (mon.2) 234 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:11:33.538 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[33624]: audit 2023-09-23T19:11:32.475891+0000 mon.c (mon.2) 235 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:11:33.538 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[33624]: audit 2023-09-23T19:11:32.578089+0000 mon.c (mon.2) 236 : audit [INF] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T19:11:33.538 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[33624]: audit 2023-09-23T19:11:32.578933+0000 mon.a (mon.0) 825 : audit [INF] from='mgr.15228 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T19:11:33.538 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[33624]: audit 2023-09-23T19:11:32.715669+0000 mon.c (mon.2) 237 : audit [INF] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T19:11:33.538 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[33624]: audit 2023-09-23T19:11:32.716292+0000 mon.a (mon.0) 826 : audit [INF] from='mgr.15228 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T19:11:33.539 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[27237]: audit 2023-09-23T19:11:32.134703+0000 mon.a (mon.0) 817 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:11:33.539 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[27237]: audit 2023-09-23T19:11:32.150136+0000 mon.a (mon.0) 818 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:11:33.539 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[27237]: audit 2023-09-23T19:11:32.154499+0000 mon.c (mon.2) 212 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:11:33.539 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[27237]: audit 2023-09-23T19:11:32.155775+0000 mon.c (mon.2) 213 : audit [INF] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:11:33.539 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[27237]: audit 2023-09-23T19:11:32.162804+0000 mon.a (mon.0) 819 : audit [INF] from='mgr.15012 ' entity='mgr.x' 2023-09-23T19:11:33.540 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[27237]: audit 2023-09-23T19:11:32.236949+0000 mon.c (mon.2) 214 : audit [DBG] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:11:33.540 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[27237]: audit 2023-09-23T19:11:32.246918+0000 mon.c (mon.2) 215 : audit [INF] from='mgr.15012 172.21.15.92:0/2518998347' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T19:11:33.540 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[27237]: audit 2023-09-23T19:11:32.247418+0000 mon.a (mon.0) 820 : audit [INF] from='mgr.15012 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T19:11:33.540 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[27237]: cluster 2023-09-23T19:11:32.257229+0000 mon.a (mon.0) 821 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2023-09-23T19:11:33.540 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[27237]: audit 2023-09-23T19:11:32.257807+0000 mon.a (mon.0) 822 : audit [INF] from='mgr.15012 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-09-23T19:11:33.540 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[27237]: cluster 2023-09-23T19:11:32.257924+0000 mon.a (mon.0) 823 : cluster [DBG] mgrmap e28: y(active, starting, since 0.00994349s) 2023-09-23T19:11:33.540 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[27237]: audit 2023-09-23T19:11:32.262414+0000 mon.c (mon.2) 216 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:11:33.540 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[27237]: audit 2023-09-23T19:11:32.262750+0000 mon.c (mon.2) 217 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:11:33.541 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[27237]: audit 2023-09-23T19:11:32.262959+0000 mon.c (mon.2) 218 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:11:33.541 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[27237]: audit 2023-09-23T19:11:32.263101+0000 mon.c (mon.2) 219 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi016.uncssv"}]: dispatch 2023-09-23T19:11:33.541 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[27237]: audit 2023-09-23T19:11:32.263491+0000 mon.c (mon.2) 220 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi092.vbpqks"}]: dispatch 2023-09-23T19:11:33.541 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[27237]: audit 2023-09-23T19:11:32.264009+0000 mon.c (mon.2) 221 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T19:11:33.541 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[27237]: audit 2023-09-23T19:11:32.264532+0000 mon.c (mon.2) 222 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T19:11:33.541 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[27237]: audit 2023-09-23T19:11:32.264921+0000 mon.c (mon.2) 223 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T19:11:33.541 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[27237]: audit 2023-09-23T19:11:32.265226+0000 mon.c (mon.2) 224 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T19:11:33.541 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[27237]: audit 2023-09-23T19:11:32.265533+0000 mon.c (mon.2) 225 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:11:33.541 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[27237]: audit 2023-09-23T19:11:32.265883+0000 mon.c (mon.2) 226 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T19:11:33.541 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[27237]: audit 2023-09-23T19:11:32.266223+0000 mon.c (mon.2) 227 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:11:33.541 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[27237]: audit 2023-09-23T19:11:32.266473+0000 mon.c (mon.2) 228 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:11:33.541 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[27237]: audit 2023-09-23T19:11:32.266713+0000 mon.c (mon.2) 229 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:11:33.542 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[27237]: audit 2023-09-23T19:11:32.267224+0000 mon.c (mon.2) 230 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T19:11:33.542 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[27237]: audit 2023-09-23T19:11:32.267594+0000 mon.c (mon.2) 231 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T19:11:33.542 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[27237]: audit 2023-09-23T19:11:32.269590+0000 mon.c (mon.2) 232 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T19:11:33.542 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[27237]: cluster 2023-09-23T19:11:32.329971+0000 mon.a (mon.0) 824 : cluster [INF] Manager daemon y is now available 2023-09-23T19:11:33.542 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[27237]: audit 2023-09-23T19:11:32.444718+0000 mon.c (mon.2) 233 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:11:33.542 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[27237]: audit 2023-09-23T19:11:32.463735+0000 mon.c (mon.2) 234 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:11:33.542 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[27237]: audit 2023-09-23T19:11:32.475891+0000 mon.c (mon.2) 235 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:11:33.542 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[27237]: audit 2023-09-23T19:11:32.578089+0000 mon.c (mon.2) 236 : audit [INF] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T19:11:33.542 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[27237]: audit 2023-09-23T19:11:32.578933+0000 mon.a (mon.0) 825 : audit [INF] from='mgr.15228 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T19:11:33.542 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[27237]: audit 2023-09-23T19:11:32.715669+0000 mon.c (mon.2) 237 : audit [INF] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T19:11:33.542 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:33 smithi016 bash[27237]: audit 2023-09-23T19:11:32.716292+0000 mon.a (mon.0) 826 : audit [INF] from='mgr.15228 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T19:11:34.267 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:11:33 smithi092 bash[50491]: debug 2023-09-23T19:11:33.941+0000 7fcb497ef200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T19:11:34.267 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:11:34 smithi092 bash[50491]: debug 2023-09-23T19:11:34.089+0000 7fcb497ef200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T19:11:34.268 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:34 smithi092 bash[27556]: audit 2023-09-23T19:11:33.168488+0000 mon.c (mon.2) 238 : audit [DBG] from='client.? 172.21.15.16:0/3982367528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:34.268 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:34 smithi092 bash[27556]: cluster 2023-09-23T19:11:33.272007+0000 mon.a (mon.0) 827 : cluster [DBG] mgrmap e29: y(active, since 1.02402s) 2023-09-23T19:11:34.268 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:34 smithi092 bash[27556]: cephadm 2023-09-23T19:11:33.777506+0000 mgr.y (mgr.15228) 2 : cephadm [INF] [23/Sep/2023:19:11:33] ENGINE Bus STARTING 2023-09-23T19:11:34.268 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:34 smithi092 bash[27556]: cephadm 2023-09-23T19:11:33.911195+0000 mgr.y (mgr.15228) 3 : cephadm [INF] [23/Sep/2023:19:11:33] ENGINE Serving on https://172.21.15.16:7150 2023-09-23T19:11:34.268 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:34 smithi092 bash[27556]: cephadm 2023-09-23T19:11:34.016436+0000 mgr.y (mgr.15228) 4 : cephadm [INF] [23/Sep/2023:19:11:34] ENGINE Serving on http://172.21.15.16:8765 2023-09-23T19:11:34.268 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:34 smithi092 bash[27556]: cephadm 2023-09-23T19:11:34.016591+0000 mgr.y (mgr.15228) 5 : cephadm [INF] [23/Sep/2023:19:11:34] ENGINE Bus STARTED 2023-09-23T19:11:34.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:34 smithi016 bash[33624]: audit 2023-09-23T19:11:33.168488+0000 mon.c (mon.2) 238 : audit [DBG] from='client.? 172.21.15.16:0/3982367528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:34.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:34 smithi016 bash[33624]: cluster 2023-09-23T19:11:33.272007+0000 mon.a (mon.0) 827 : cluster [DBG] mgrmap e29: y(active, since 1.02402s) 2023-09-23T19:11:34.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:34 smithi016 bash[33624]: cephadm 2023-09-23T19:11:33.777506+0000 mgr.y (mgr.15228) 2 : cephadm [INF] [23/Sep/2023:19:11:33] ENGINE Bus STARTING 2023-09-23T19:11:34.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:34 smithi016 bash[33624]: cephadm 2023-09-23T19:11:33.911195+0000 mgr.y (mgr.15228) 3 : cephadm [INF] [23/Sep/2023:19:11:33] ENGINE Serving on https://172.21.15.16:7150 2023-09-23T19:11:34.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:34 smithi016 bash[33624]: cephadm 2023-09-23T19:11:34.016436+0000 mgr.y (mgr.15228) 4 : cephadm [INF] [23/Sep/2023:19:11:34] ENGINE Serving on http://172.21.15.16:8765 2023-09-23T19:11:34.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:34 smithi016 bash[33624]: cephadm 2023-09-23T19:11:34.016591+0000 mgr.y (mgr.15228) 5 : cephadm [INF] [23/Sep/2023:19:11:34] ENGINE Bus STARTED 2023-09-23T19:11:34.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:34 smithi016 bash[27237]: audit 2023-09-23T19:11:33.168488+0000 mon.c (mon.2) 238 : audit [DBG] from='client.? 172.21.15.16:0/3982367528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:34.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:34 smithi016 bash[27237]: cluster 2023-09-23T19:11:33.272007+0000 mon.a (mon.0) 827 : cluster [DBG] mgrmap e29: y(active, since 1.02402s) 2023-09-23T19:11:34.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:34 smithi016 bash[27237]: cephadm 2023-09-23T19:11:33.777506+0000 mgr.y (mgr.15228) 2 : cephadm [INF] [23/Sep/2023:19:11:33] ENGINE Bus STARTING 2023-09-23T19:11:34.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:34 smithi016 bash[27237]: cephadm 2023-09-23T19:11:33.911195+0000 mgr.y (mgr.15228) 3 : cephadm [INF] [23/Sep/2023:19:11:33] ENGINE Serving on https://172.21.15.16:7150 2023-09-23T19:11:34.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:34 smithi016 bash[27237]: cephadm 2023-09-23T19:11:34.016436+0000 mgr.y (mgr.15228) 4 : cephadm [INF] [23/Sep/2023:19:11:34] ENGINE Serving on http://172.21.15.16:8765 2023-09-23T19:11:34.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:34 smithi016 bash[27237]: cephadm 2023-09-23T19:11:34.016591+0000 mgr.y (mgr.15228) 5 : cephadm [INF] [23/Sep/2023:19:11:34] ENGINE Bus STARTED 2023-09-23T19:11:34.536 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:11:34 smithi092 bash[50491]: debug 2023-09-23T19:11:34.245+0000 7fcb497ef200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T19:11:35.035 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:11:34 smithi092 bash[50491]: debug 2023-09-23T19:11:34.633+0000 7fcb497ef200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T19:11:35.535 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:11:35 smithi092 bash[50491]: debug 2023-09-23T19:11:35.185+0000 7fcb497ef200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T19:11:35.535 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:11:35 smithi092 bash[50491]: debug 2023-09-23T19:11:35.429+0000 7fcb497ef200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T19:11:36.035 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:11:35 smithi092 bash[50491]: debug 2023-09-23T19:11:35.753+0000 7fcb497ef200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T19:11:36.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:36 smithi016 bash[27237]: cluster 2023-09-23T19:11:34.266118+0000 mgr.y (mgr.15228) 6 : cluster [DBG] pgmap v4: 89 pgs: 89 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-09-23T19:11:36.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:36 smithi016 bash[27237]: cluster 2023-09-23T19:11:35.282710+0000 mon.a (mon.0) 828 : cluster [DBG] mgrmap e30: y(active, since 3s) 2023-09-23T19:11:36.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:36 smithi016 bash[27237]: audit 2023-09-23T19:11:35.858922+0000 mon.c (mon.2) 239 : audit [DBG] from='client.? 172.21.15.16:0/3845099119' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:36.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:36 smithi016 bash[33624]: cluster 2023-09-23T19:11:34.266118+0000 mgr.y (mgr.15228) 6 : cluster [DBG] pgmap v4: 89 pgs: 89 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-09-23T19:11:36.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:36 smithi016 bash[33624]: cluster 2023-09-23T19:11:35.282710+0000 mon.a (mon.0) 828 : cluster [DBG] mgrmap e30: y(active, since 3s) 2023-09-23T19:11:36.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:36 smithi016 bash[33624]: audit 2023-09-23T19:11:35.858922+0000 mon.c (mon.2) 239 : audit [DBG] from='client.? 172.21.15.16:0/3845099119' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:36.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:36 smithi092 bash[27556]: cluster 2023-09-23T19:11:34.266118+0000 mgr.y (mgr.15228) 6 : cluster [DBG] pgmap v4: 89 pgs: 89 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-09-23T19:11:36.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:36 smithi092 bash[27556]: cluster 2023-09-23T19:11:35.282710+0000 mon.a (mon.0) 828 : cluster [DBG] mgrmap e30: y(active, since 3s) 2023-09-23T19:11:36.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:36 smithi092 bash[27556]: audit 2023-09-23T19:11:35.858922+0000 mon.c (mon.2) 239 : audit [DBG] from='client.? 172.21.15.16:0/3845099119' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:37.535 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:11:37 smithi092 bash[50491]: debug 2023-09-23T19:11:37.233+0000 7fcb497ef200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T19:11:37.536 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:11:37 smithi092 bash[50491]: debug 2023-09-23T19:11:37.397+0000 7fcb497ef200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T19:11:37.910 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:11:37 smithi092 bash[52849]: ts=2023-09-23T19:11:37.914Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.92:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.92:8765: connect: connection refused" 2023-09-23T19:11:37.910 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:11:37 smithi092 bash[50491]: debug 2023-09-23T19:11:37.561+0000 7fcb497ef200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T19:11:38.245 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:38 smithi092 bash[27556]: cluster 2023-09-23T19:11:36.266598+0000 mgr.y (mgr.15228) 7 : cluster [DBG] pgmap v5: 89 pgs: 89 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-09-23T19:11:38.245 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:11:38 smithi092 bash[50491]: debug 2023-09-23T19:11:38.241+0000 7fcb497ef200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T19:11:38.246 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:11:37 smithi092 bash[52849]: ts=2023-09-23T19:11:37.916Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.92:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.92:8765: connect: connection refused" 2023-09-23T19:11:38.246 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:11:37 smithi092 bash[52849]: ts=2023-09-23T19:11:37.916Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.92:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.92:8765: connect: connection refused" 2023-09-23T19:11:38.246 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:11:37 smithi092 bash[52849]: ts=2023-09-23T19:11:37.919Z caller=refresh.go:98 level=error component="discovery manager notify" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.92:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.92:8765: connect: connection refused" 2023-09-23T19:11:38.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:38 smithi016 bash[33624]: cluster 2023-09-23T19:11:36.266598+0000 mgr.y (mgr.15228) 7 : cluster [DBG] pgmap v5: 89 pgs: 89 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-09-23T19:11:38.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:38 smithi016 bash[27237]: cluster 2023-09-23T19:11:36.266598+0000 mgr.y (mgr.15228) 7 : cluster [DBG] pgmap v5: 89 pgs: 89 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-09-23T19:11:39.035 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:11:38 smithi092 bash[50491]: debug 2023-09-23T19:11:38.577+0000 7fcb497ef200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T19:11:39.036 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:11:38 smithi092 bash[52849]: ts=2023-09-23T19:11:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi092:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi092.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T19:11:39.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:39 smithi016 bash[27237]: audit 2023-09-23T19:11:38.628788+0000 mon.a (mon.0) 829 : audit [DBG] from='client.? 172.21.15.16:0/1137211038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:39.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:39 smithi016 bash[33624]: audit 2023-09-23T19:11:38.628788+0000 mon.a (mon.0) 829 : audit [DBG] from='client.? 172.21.15.16:0/1137211038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:39.535 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:39 smithi092 bash[27556]: audit 2023-09-23T19:11:38.628788+0000 mon.a (mon.0) 829 : audit [DBG] from='client.? 172.21.15.16:0/1137211038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:39.535 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:11:39 smithi092 bash[50491]: debug 2023-09-23T19:11:39.277+0000 7fcb497ef200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T19:11:40.285 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:40 smithi092 bash[27556]: cluster 2023-09-23T19:11:38.267010+0000 mgr.y (mgr.15228) 8 : cluster [DBG] pgmap v6: 89 pgs: 89 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-09-23T19:11:40.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:40 smithi092 bash[27556]: audit 2023-09-23T19:11:39.542560+0000 mon.a (mon.0) 830 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:11:40.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:40 smithi092 bash[27556]: audit 2023-09-23T19:11:39.551388+0000 mon.a (mon.0) 831 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:11:40.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:40 smithi092 bash[27556]: audit 2023-09-23T19:11:39.914970+0000 mon.a (mon.0) 832 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:11:40.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:40 smithi092 bash[27556]: audit 2023-09-23T19:11:39.931203+0000 mon.a (mon.0) 833 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:11:40.286 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:11:40 smithi092 bash[50491]: debug 2023-09-23T19:11:40.013+0000 7fcb497ef200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T19:11:40.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:40 smithi016 bash[33624]: cluster 2023-09-23T19:11:38.267010+0000 mgr.y (mgr.15228) 8 : cluster [DBG] pgmap v6: 89 pgs: 89 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-09-23T19:11:40.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:40 smithi016 bash[33624]: audit 2023-09-23T19:11:39.542560+0000 mon.a (mon.0) 830 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:11:40.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:40 smithi016 bash[33624]: audit 2023-09-23T19:11:39.551388+0000 mon.a (mon.0) 831 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:11:40.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:40 smithi016 bash[33624]: audit 2023-09-23T19:11:39.914970+0000 mon.a (mon.0) 832 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:11:40.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:40 smithi016 bash[33624]: audit 2023-09-23T19:11:39.931203+0000 mon.a (mon.0) 833 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:11:40.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:40 smithi016 bash[27237]: cluster 2023-09-23T19:11:38.267010+0000 mgr.y (mgr.15228) 8 : cluster [DBG] pgmap v6: 89 pgs: 89 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-09-23T19:11:40.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:40 smithi016 bash[27237]: audit 2023-09-23T19:11:39.542560+0000 mon.a (mon.0) 830 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:11:40.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:40 smithi016 bash[27237]: audit 2023-09-23T19:11:39.551388+0000 mon.a (mon.0) 831 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:11:40.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:40 smithi016 bash[27237]: audit 2023-09-23T19:11:39.914970+0000 mon.a (mon.0) 832 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:11:40.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:40 smithi016 bash[27237]: audit 2023-09-23T19:11:39.931203+0000 mon.a (mon.0) 833 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:11:41.285 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:11:40 smithi092 bash[50491]: debug 2023-09-23T19:11:40.845+0000 7fcb497ef200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T19:11:41.285 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:11:41 smithi092 bash[50491]: debug 2023-09-23T19:11:41.001+0000 7fcb497ef200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T19:11:41.786 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:11:41 smithi092 bash[50491]: debug 2023-09-23T19:11:41.481+0000 7fcb497ef200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T19:11:41.786 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:11:41 smithi092 bash[50491]: debug 2023-09-23T19:11:41.641+0000 7fcb497ef200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T19:11:41.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:41 smithi016 bash[27237]: cluster 2023-09-23T19:11:40.267734+0000 mgr.y (mgr.15228) 9 : cluster [DBG] pgmap v7: 89 pgs: 89 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 463 op/s 2023-09-23T19:11:41.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:41 smithi016 bash[27237]: audit 2023-09-23T19:11:40.514444+0000 mon.a (mon.0) 834 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:11:41.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:41 smithi016 bash[27237]: audit 2023-09-23T19:11:40.523519+0000 mon.a (mon.0) 835 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:11:41.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:41 smithi016 bash[27237]: audit 2023-09-23T19:11:40.986215+0000 mon.a (mon.0) 836 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:11:41.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:41 smithi016 bash[27237]: audit 2023-09-23T19:11:40.996495+0000 mon.a (mon.0) 837 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:11:41.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:41 smithi016 bash[27237]: audit 2023-09-23T19:11:41.362106+0000 mon.a (mon.0) 838 : audit [DBG] from='client.? 172.21.15.16:0/520176140' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:41.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:41 smithi092 bash[27556]: cluster 2023-09-23T19:11:40.267734+0000 mgr.y (mgr.15228) 9 : cluster [DBG] pgmap v7: 89 pgs: 89 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 463 op/s 2023-09-23T19:11:41.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:41 smithi092 bash[27556]: audit 2023-09-23T19:11:40.514444+0000 mon.a (mon.0) 834 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:11:41.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:41 smithi092 bash[27556]: audit 2023-09-23T19:11:40.523519+0000 mon.a (mon.0) 835 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:11:41.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:41 smithi092 bash[27556]: audit 2023-09-23T19:11:40.986215+0000 mon.a (mon.0) 836 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:11:41.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:41 smithi092 bash[27556]: audit 2023-09-23T19:11:40.996495+0000 mon.a (mon.0) 837 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:11:41.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:41 smithi092 bash[27556]: audit 2023-09-23T19:11:41.362106+0000 mon.a (mon.0) 838 : audit [DBG] from='client.? 172.21.15.16:0/520176140' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:41.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:41 smithi016 bash[33624]: cluster 2023-09-23T19:11:40.267734+0000 mgr.y (mgr.15228) 9 : cluster [DBG] pgmap v7: 89 pgs: 89 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 463 op/s 2023-09-23T19:11:41.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:41 smithi016 bash[33624]: audit 2023-09-23T19:11:40.514444+0000 mon.a (mon.0) 834 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:11:41.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:41 smithi016 bash[33624]: audit 2023-09-23T19:11:40.523519+0000 mon.a (mon.0) 835 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:11:41.788 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:41 smithi016 bash[33624]: audit 2023-09-23T19:11:40.986215+0000 mon.a (mon.0) 836 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:11:41.788 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:41 smithi016 bash[33624]: audit 2023-09-23T19:11:40.996495+0000 mon.a (mon.0) 837 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:11:41.788 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:41 smithi016 bash[33624]: audit 2023-09-23T19:11:41.362106+0000 mon.a (mon.0) 838 : audit [DBG] from='client.? 172.21.15.16:0/520176140' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:42.190 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:11:41 smithi092 bash[50491]: debug 2023-09-23T19:11:41.853+0000 7fcb497ef200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T19:11:42.190 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:11:42 smithi092 bash[50491]: debug 2023-09-23T19:11:42.013+0000 7fcb497ef200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T19:11:42.190 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:11:42 smithi092 bash[50491]: debug 2023-09-23T19:11:42.189+0000 7fcb497ef200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T19:11:42.468 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:11:42 smithi092 bash[50491]: debug 2023-09-23T19:11:42.465+0000 7fcb497ef200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T19:11:42.785 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:11:42 smithi092 bash[50491]: debug 2023-09-23T19:11:42.649+0000 7fcb497ef200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T19:11:43.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:43 smithi016 bash[27237]: cluster 2023-09-23T19:11:42.268300+0000 mgr.y (mgr.15228) 10 : cluster [DBG] pgmap v8: 89 pgs: 89 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 359 op/s 2023-09-23T19:11:43.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:43 smithi016 bash[33624]: cluster 2023-09-23T19:11:42.268300+0000 mgr.y (mgr.15228) 10 : cluster [DBG] pgmap v8: 89 pgs: 89 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 359 op/s 2023-09-23T19:11:43.785 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:43 smithi092 bash[27556]: cluster 2023-09-23T19:11:42.268300+0000 mgr.y (mgr.15228) 10 : cluster [DBG] pgmap v8: 89 pgs: 89 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 359 op/s 2023-09-23T19:11:44.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:44 smithi016 bash[27237]: audit 2023-09-23T19:11:44.051920+0000 mon.a (mon.0) 839 : audit [DBG] from='client.? 172.21.15.16:0/3463907016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:44.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:44 smithi016 bash[27237]: cluster 2023-09-23T19:11:44.391553+0000 mon.a (mon.0) 840 : cluster [DBG] Standby manager daemon x started 2023-09-23T19:11:44.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:44 smithi016 bash[27237]: audit 2023-09-23T19:11:44.400957+0000 mon.a (mon.0) 841 : audit [DBG] from='mgr.? 172.21.15.92:0/1872637624' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T19:11:44.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:44 smithi016 bash[27237]: audit 2023-09-23T19:11:44.402357+0000 mon.a (mon.0) 842 : audit [DBG] from='mgr.? 172.21.15.92:0/1872637624' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T19:11:44.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:44 smithi016 bash[27237]: audit 2023-09-23T19:11:44.406743+0000 mon.a (mon.0) 843 : audit [DBG] from='mgr.? 172.21.15.92:0/1872637624' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T19:11:44.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:44 smithi016 bash[27237]: audit 2023-09-23T19:11:44.407355+0000 mon.a (mon.0) 844 : audit [DBG] from='mgr.? 172.21.15.92:0/1872637624' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T19:11:44.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:44 smithi016 bash[33624]: audit 2023-09-23T19:11:44.051920+0000 mon.a (mon.0) 839 : audit [DBG] from='client.? 172.21.15.16:0/3463907016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:44.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:44 smithi016 bash[33624]: cluster 2023-09-23T19:11:44.391553+0000 mon.a (mon.0) 840 : cluster [DBG] Standby manager daemon x started 2023-09-23T19:11:44.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:44 smithi016 bash[33624]: audit 2023-09-23T19:11:44.400957+0000 mon.a (mon.0) 841 : audit [DBG] from='mgr.? 172.21.15.92:0/1872637624' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T19:11:44.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:44 smithi016 bash[33624]: audit 2023-09-23T19:11:44.402357+0000 mon.a (mon.0) 842 : audit [DBG] from='mgr.? 172.21.15.92:0/1872637624' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T19:11:44.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:44 smithi016 bash[33624]: audit 2023-09-23T19:11:44.406743+0000 mon.a (mon.0) 843 : audit [DBG] from='mgr.? 172.21.15.92:0/1872637624' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T19:11:44.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:44 smithi016 bash[33624]: audit 2023-09-23T19:11:44.407355+0000 mon.a (mon.0) 844 : audit [DBG] from='mgr.? 172.21.15.92:0/1872637624' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T19:11:44.786 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:44 smithi092 bash[27556]: audit 2023-09-23T19:11:44.051920+0000 mon.a (mon.0) 839 : audit [DBG] from='client.? 172.21.15.16:0/3463907016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:44.786 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:44 smithi092 bash[27556]: cluster 2023-09-23T19:11:44.391553+0000 mon.a (mon.0) 840 : cluster [DBG] Standby manager daemon x started 2023-09-23T19:11:44.786 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:44 smithi092 bash[27556]: audit 2023-09-23T19:11:44.400957+0000 mon.a (mon.0) 841 : audit [DBG] from='mgr.? 172.21.15.92:0/1872637624' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T19:11:44.786 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:44 smithi092 bash[27556]: audit 2023-09-23T19:11:44.402357+0000 mon.a (mon.0) 842 : audit [DBG] from='mgr.? 172.21.15.92:0/1872637624' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T19:11:44.786 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:44 smithi092 bash[27556]: audit 2023-09-23T19:11:44.406743+0000 mon.a (mon.0) 843 : audit [DBG] from='mgr.? 172.21.15.92:0/1872637624' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T19:11:44.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:44 smithi092 bash[27556]: audit 2023-09-23T19:11:44.407355+0000 mon.a (mon.0) 844 : audit [DBG] from='mgr.? 172.21.15.92:0/1872637624' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T19:11:44.787 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:11:44 smithi092 bash[50491]: debug 2023-09-23T19:11:44.369+0000 7fcb497ef200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T19:11:44.787 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:11:44 smithi092 bash[50491]: [23/Sep/2023:19:11:44] ENGINE Bus STARTING 2023-09-23T19:11:44.787 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:11:44 smithi092 bash[50491]: CherryPy Checker: 2023-09-23T19:11:44.787 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:11:44 smithi092 bash[50491]: The Application mounted at '' has an empty config. 2023-09-23T19:11:44.787 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:11:44 smithi092 bash[50491]: [23/Sep/2023:19:11:44] ENGINE Serving on http://:::9283 2023-09-23T19:11:44.787 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:11:44 smithi092 bash[50491]: [23/Sep/2023:19:11:44] ENGINE Bus STARTED 2023-09-23T19:11:45.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:45 smithi016 bash[27237]: cluster 2023-09-23T19:11:44.269159+0000 mgr.y (mgr.15228) 11 : cluster [DBG] pgmap v9: 89 pgs: 89 active+clean; 1.3 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 573 op/s 2023-09-23T19:11:45.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:45 smithi016 bash[27237]: cluster 2023-09-23T19:11:44.534614+0000 mon.a (mon.0) 845 : cluster [DBG] mgrmap e31: y(active, since 12s), standbys: x 2023-09-23T19:11:45.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:45 smithi016 bash[27237]: audit 2023-09-23T19:11:44.536387+0000 mon.c (mon.2) 240 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T19:11:45.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:45 smithi016 bash[33624]: cluster 2023-09-23T19:11:44.269159+0000 mgr.y (mgr.15228) 11 : cluster [DBG] pgmap v9: 89 pgs: 89 active+clean; 1.3 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 573 op/s 2023-09-23T19:11:45.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:45 smithi016 bash[33624]: cluster 2023-09-23T19:11:44.534614+0000 mon.a (mon.0) 845 : cluster [DBG] mgrmap e31: y(active, since 12s), standbys: x 2023-09-23T19:11:45.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:45 smithi016 bash[33624]: audit 2023-09-23T19:11:44.536387+0000 mon.c (mon.2) 240 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T19:11:45.785 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:45 smithi092 bash[27556]: cluster 2023-09-23T19:11:44.269159+0000 mgr.y (mgr.15228) 11 : cluster [DBG] pgmap v9: 89 pgs: 89 active+clean; 1.3 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 573 op/s 2023-09-23T19:11:45.785 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:45 smithi092 bash[27556]: cluster 2023-09-23T19:11:44.534614+0000 mon.a (mon.0) 845 : cluster [DBG] mgrmap e31: y(active, since 12s), standbys: x 2023-09-23T19:11:45.786 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:45 smithi092 bash[27556]: audit 2023-09-23T19:11:44.536387+0000 mon.c (mon.2) 240 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T19:11:47.927 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:47 smithi016 bash[33624]: cluster 2023-09-23T19:11:46.271012+0000 mgr.y (mgr.15228) 12 : cluster [DBG] pgmap v10: 89 pgs: 89 active+clean; 1.3 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 522 op/s 2023-09-23T19:11:47.927 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:47 smithi016 bash[33624]: audit 2023-09-23T19:11:46.792451+0000 mon.b (mon.1) 47 : audit [DBG] from='client.? 172.21.15.16:0/2623872446' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:47.928 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:47 smithi016 bash[33624]: audit 2023-09-23T19:11:47.454157+0000 mon.c (mon.2) 241 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:11:47.928 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:47 smithi016 bash[27237]: cluster 2023-09-23T19:11:46.271012+0000 mgr.y (mgr.15228) 12 : cluster [DBG] pgmap v10: 89 pgs: 89 active+clean; 1.3 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 522 op/s 2023-09-23T19:11:47.928 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:47 smithi016 bash[27237]: audit 2023-09-23T19:11:46.792451+0000 mon.b (mon.1) 47 : audit [DBG] from='client.? 172.21.15.16:0/2623872446' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:47.928 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:47 smithi016 bash[27237]: audit 2023-09-23T19:11:47.454157+0000 mon.c (mon.2) 241 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:11:48.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:47 smithi092 bash[27556]: cluster 2023-09-23T19:11:46.271012+0000 mgr.y (mgr.15228) 12 : cluster [DBG] pgmap v10: 89 pgs: 89 active+clean; 1.3 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 522 op/s 2023-09-23T19:11:48.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:47 smithi092 bash[27556]: audit 2023-09-23T19:11:46.792451+0000 mon.b (mon.1) 47 : audit [DBG] from='client.? 172.21.15.16:0/2623872446' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:48.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:47 smithi092 bash[27556]: audit 2023-09-23T19:11:47.454157+0000 mon.c (mon.2) 241 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:11:49.535 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:11:49 smithi092 bash[50491]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:11:49] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T19:11:50.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:49 smithi016 bash[33624]: cluster 2023-09-23T19:11:48.271517+0000 mgr.y (mgr.15228) 13 : cluster [DBG] pgmap v11: 89 pgs: 89 active+clean; 1.3 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 522 op/s 2023-09-23T19:11:50.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:49 smithi016 bash[33624]: audit 2023-09-23T19:11:49.505495+0000 mon.c (mon.2) 242 : audit [DBG] from='client.? 172.21.15.16:0/1977676211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:50.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:49 smithi016 bash[27237]: cluster 2023-09-23T19:11:48.271517+0000 mgr.y (mgr.15228) 13 : cluster [DBG] pgmap v11: 89 pgs: 89 active+clean; 1.3 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 522 op/s 2023-09-23T19:11:50.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:49 smithi016 bash[27237]: audit 2023-09-23T19:11:49.505495+0000 mon.c (mon.2) 242 : audit [DBG] from='client.? 172.21.15.16:0/1977676211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:50.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:49 smithi092 bash[27556]: cluster 2023-09-23T19:11:48.271517+0000 mgr.y (mgr.15228) 13 : cluster [DBG] pgmap v11: 89 pgs: 89 active+clean; 1.3 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 522 op/s 2023-09-23T19:11:50.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:49 smithi092 bash[27556]: audit 2023-09-23T19:11:49.505495+0000 mon.c (mon.2) 242 : audit [DBG] from='client.? 172.21.15.16:0/1977676211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:52.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:52 smithi016 bash[33624]: cluster 2023-09-23T19:11:50.272211+0000 mgr.y (mgr.15228) 14 : cluster [DBG] pgmap v12: 89 pgs: 89 active+clean; 1.1 GiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 814 op/s 2023-09-23T19:11:52.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:52 smithi016 bash[33624]: audit 2023-09-23T19:11:51.703733+0000 mon.a (mon.0) 846 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:11:52.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:52 smithi016 bash[33624]: audit 2023-09-23T19:11:51.717057+0000 mon.a (mon.0) 847 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:11:52.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:52 smithi016 bash[33624]: audit 2023-09-23T19:11:51.719269+0000 mon.c (mon.2) 243 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:11:52.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:52 smithi016 bash[33624]: audit 2023-09-23T19:11:51.720516+0000 mon.c (mon.2) 244 : audit [INF] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:11:52.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:52 smithi016 bash[27237]: cluster 2023-09-23T19:11:50.272211+0000 mgr.y (mgr.15228) 14 : cluster [DBG] pgmap v12: 89 pgs: 89 active+clean; 1.1 GiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 814 op/s 2023-09-23T19:11:52.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:52 smithi016 bash[27237]: audit 2023-09-23T19:11:51.703733+0000 mon.a (mon.0) 846 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:11:52.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:52 smithi016 bash[27237]: audit 2023-09-23T19:11:51.717057+0000 mon.a (mon.0) 847 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:11:52.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:52 smithi016 bash[27237]: audit 2023-09-23T19:11:51.719269+0000 mon.c (mon.2) 243 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:11:52.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:52 smithi016 bash[27237]: audit 2023-09-23T19:11:51.720516+0000 mon.c (mon.2) 244 : audit [INF] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:11:52.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:52 smithi092 bash[27556]: cluster 2023-09-23T19:11:50.272211+0000 mgr.y (mgr.15228) 14 : cluster [DBG] pgmap v12: 89 pgs: 89 active+clean; 1.1 GiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 814 op/s 2023-09-23T19:11:52.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:52 smithi092 bash[27556]: audit 2023-09-23T19:11:51.703733+0000 mon.a (mon.0) 846 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:11:52.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:52 smithi092 bash[27556]: audit 2023-09-23T19:11:51.717057+0000 mon.a (mon.0) 847 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:11:52.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:52 smithi092 bash[27556]: audit 2023-09-23T19:11:51.719269+0000 mon.c (mon.2) 243 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:11:52.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:52 smithi092 bash[27556]: audit 2023-09-23T19:11:51.720516+0000 mon.c (mon.2) 244 : audit [INF] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:11:53.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:53 smithi016 bash[33624]: cephadm 2023-09-23T19:11:51.722802+0000 mgr.y (mgr.15228) 15 : cephadm [INF] Updating smithi016:/etc/ceph/ceph.conf 2023-09-23T19:11:53.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:53 smithi016 bash[33624]: cephadm 2023-09-23T19:11:51.723700+0000 mgr.y (mgr.15228) 16 : cephadm [INF] Updating smithi092:/etc/ceph/ceph.conf 2023-09-23T19:11:53.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:53 smithi016 bash[33624]: cephadm 2023-09-23T19:11:51.947767+0000 mgr.y (mgr.15228) 17 : cephadm [INF] Updating smithi092:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:11:53.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:53 smithi016 bash[33624]: cephadm 2023-09-23T19:11:51.948762+0000 mgr.y (mgr.15228) 18 : cephadm [INF] Updating smithi016:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:11:53.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:53 smithi016 bash[33624]: audit 2023-09-23T19:11:52.289882+0000 mon.b (mon.1) 48 : audit [DBG] from='client.? 172.21.15.16:0/3492268643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:53.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:53 smithi016 bash[33624]: audit 2023-09-23T19:11:52.654529+0000 mon.a (mon.0) 848 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:11:53.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:53 smithi016 bash[33624]: audit 2023-09-23T19:11:52.661688+0000 mon.a (mon.0) 849 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:11:53.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:53 smithi016 bash[33624]: audit 2023-09-23T19:11:52.669286+0000 mon.a (mon.0) 850 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:11:53.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:53 smithi016 bash[33624]: audit 2023-09-23T19:11:52.675747+0000 mon.a (mon.0) 851 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:11:53.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:53 smithi016 bash[33624]: audit 2023-09-23T19:11:52.699551+0000 mon.a (mon.0) 852 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:11:53.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:53 smithi016 bash[27237]: cephadm 2023-09-23T19:11:51.722802+0000 mgr.y (mgr.15228) 15 : cephadm [INF] Updating smithi016:/etc/ceph/ceph.conf 2023-09-23T19:11:53.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:53 smithi016 bash[27237]: cephadm 2023-09-23T19:11:51.723700+0000 mgr.y (mgr.15228) 16 : cephadm [INF] Updating smithi092:/etc/ceph/ceph.conf 2023-09-23T19:11:53.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:53 smithi016 bash[27237]: cephadm 2023-09-23T19:11:51.947767+0000 mgr.y (mgr.15228) 17 : cephadm [INF] Updating smithi092:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:11:53.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:53 smithi016 bash[27237]: cephadm 2023-09-23T19:11:51.948762+0000 mgr.y (mgr.15228) 18 : cephadm [INF] Updating smithi016:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:11:53.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:53 smithi016 bash[27237]: audit 2023-09-23T19:11:52.289882+0000 mon.b (mon.1) 48 : audit [DBG] from='client.? 172.21.15.16:0/3492268643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:53.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:53 smithi016 bash[27237]: audit 2023-09-23T19:11:52.654529+0000 mon.a (mon.0) 848 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:11:53.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:53 smithi016 bash[27237]: audit 2023-09-23T19:11:52.661688+0000 mon.a (mon.0) 849 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:11:53.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:53 smithi016 bash[27237]: audit 2023-09-23T19:11:52.669286+0000 mon.a (mon.0) 850 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:11:53.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:53 smithi016 bash[27237]: audit 2023-09-23T19:11:52.675747+0000 mon.a (mon.0) 851 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:11:53.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:53 smithi016 bash[27237]: audit 2023-09-23T19:11:52.699551+0000 mon.a (mon.0) 852 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:11:53.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:53 smithi092 bash[27556]: cephadm 2023-09-23T19:11:51.722802+0000 mgr.y (mgr.15228) 15 : cephadm [INF] Updating smithi016:/etc/ceph/ceph.conf 2023-09-23T19:11:53.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:53 smithi092 bash[27556]: cephadm 2023-09-23T19:11:51.723700+0000 mgr.y (mgr.15228) 16 : cephadm [INF] Updating smithi092:/etc/ceph/ceph.conf 2023-09-23T19:11:53.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:53 smithi092 bash[27556]: cephadm 2023-09-23T19:11:51.947767+0000 mgr.y (mgr.15228) 17 : cephadm [INF] Updating smithi092:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:11:53.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:53 smithi092 bash[27556]: cephadm 2023-09-23T19:11:51.948762+0000 mgr.y (mgr.15228) 18 : cephadm [INF] Updating smithi016:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:11:53.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:53 smithi092 bash[27556]: audit 2023-09-23T19:11:52.289882+0000 mon.b (mon.1) 48 : audit [DBG] from='client.? 172.21.15.16:0/3492268643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:53.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:53 smithi092 bash[27556]: audit 2023-09-23T19:11:52.654529+0000 mon.a (mon.0) 848 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:11:53.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:53 smithi092 bash[27556]: audit 2023-09-23T19:11:52.661688+0000 mon.a (mon.0) 849 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:11:53.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:53 smithi092 bash[27556]: audit 2023-09-23T19:11:52.669286+0000 mon.a (mon.0) 850 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:11:53.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:53 smithi092 bash[27556]: audit 2023-09-23T19:11:52.675747+0000 mon.a (mon.0) 851 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:11:53.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:53 smithi092 bash[27556]: audit 2023-09-23T19:11:52.699551+0000 mon.a (mon.0) 852 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:11:54.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:54 smithi016 bash[27237]: cephadm 2023-09-23T19:11:52.162991+0000 mgr.y (mgr.15228) 19 : cephadm [INF] Updating smithi092:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:11:54.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:54 smithi016 bash[27237]: cephadm 2023-09-23T19:11:52.163739+0000 mgr.y (mgr.15228) 20 : cephadm [INF] Updating smithi016:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:11:54.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:54 smithi016 bash[27237]: cluster 2023-09-23T19:11:52.272763+0000 mgr.y (mgr.15228) 21 : cluster [DBG] pgmap v13: 89 pgs: 89 active+clean; 1.1 GiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 547 op/s 2023-09-23T19:11:54.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:54 smithi016 bash[27237]: cephadm 2023-09-23T19:11:52.377394+0000 mgr.y (mgr.15228) 22 : cephadm [INF] Updating smithi092:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:11:54.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:54 smithi016 bash[27237]: cephadm 2023-09-23T19:11:52.377832+0000 mgr.y (mgr.15228) 23 : cephadm [INF] Updating smithi016:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:11:54.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:54 smithi016 bash[27237]: cephadm 2023-09-23T19:11:52.778114+0000 mgr.y (mgr.15228) 24 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T19:11:54.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:54 smithi016 bash[33624]: cephadm 2023-09-23T19:11:52.162991+0000 mgr.y (mgr.15228) 19 : cephadm [INF] Updating smithi092:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:11:54.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:54 smithi016 bash[33624]: cephadm 2023-09-23T19:11:52.163739+0000 mgr.y (mgr.15228) 20 : cephadm [INF] Updating smithi016:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:11:54.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:54 smithi016 bash[33624]: cluster 2023-09-23T19:11:52.272763+0000 mgr.y (mgr.15228) 21 : cluster [DBG] pgmap v13: 89 pgs: 89 active+clean; 1.1 GiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 547 op/s 2023-09-23T19:11:54.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:54 smithi016 bash[33624]: cephadm 2023-09-23T19:11:52.377394+0000 mgr.y (mgr.15228) 22 : cephadm [INF] Updating smithi092:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:11:54.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:54 smithi016 bash[33624]: cephadm 2023-09-23T19:11:52.377832+0000 mgr.y (mgr.15228) 23 : cephadm [INF] Updating smithi016:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:11:54.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:54 smithi016 bash[33624]: cephadm 2023-09-23T19:11:52.778114+0000 mgr.y (mgr.15228) 24 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T19:11:54.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:54 smithi092 bash[27556]: cephadm 2023-09-23T19:11:52.162991+0000 mgr.y (mgr.15228) 19 : cephadm [INF] Updating smithi092:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:11:54.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:54 smithi092 bash[27556]: cephadm 2023-09-23T19:11:52.163739+0000 mgr.y (mgr.15228) 20 : cephadm [INF] Updating smithi016:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:11:54.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:54 smithi092 bash[27556]: cluster 2023-09-23T19:11:52.272763+0000 mgr.y (mgr.15228) 21 : cluster [DBG] pgmap v13: 89 pgs: 89 active+clean; 1.1 GiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 547 op/s 2023-09-23T19:11:54.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:54 smithi092 bash[27556]: cephadm 2023-09-23T19:11:52.377394+0000 mgr.y (mgr.15228) 22 : cephadm [INF] Updating smithi092:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:11:54.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:54 smithi092 bash[27556]: cephadm 2023-09-23T19:11:52.377832+0000 mgr.y (mgr.15228) 23 : cephadm [INF] Updating smithi016:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:11:54.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:54 smithi092 bash[27556]: cephadm 2023-09-23T19:11:52.778114+0000 mgr.y (mgr.15228) 24 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T19:11:55.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:55 smithi016 bash[33624]: cephadm 2023-09-23T19:11:53.069383+0000 mgr.y (mgr.15228) 25 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi092 2023-09-23T19:11:55.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:55 smithi016 bash[33624]: audit 2023-09-23T19:11:54.987667+0000 mon.a (mon.0) 853 : audit [DBG] from='client.? 172.21.15.16:0/2148471603' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:55.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:55 smithi016 bash[27237]: cephadm 2023-09-23T19:11:53.069383+0000 mgr.y (mgr.15228) 25 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi092 2023-09-23T19:11:55.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:55 smithi016 bash[27237]: audit 2023-09-23T19:11:54.987667+0000 mon.a (mon.0) 853 : audit [DBG] from='client.? 172.21.15.16:0/2148471603' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:55.535 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:55 smithi092 bash[27556]: cephadm 2023-09-23T19:11:53.069383+0000 mgr.y (mgr.15228) 25 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi092 2023-09-23T19:11:55.535 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:55 smithi092 bash[27556]: audit 2023-09-23T19:11:54.987667+0000 mon.a (mon.0) 853 : audit [DBG] from='client.? 172.21.15.16:0/2148471603' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:56.285 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:56 smithi092 bash[27556]: cluster 2023-09-23T19:11:54.273503+0000 mgr.y (mgr.15228) 26 : cluster [DBG] pgmap v14: 89 pgs: 89 active+clean; 935 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 828 op/s 2023-09-23T19:11:56.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:56 smithi016 bash[27237]: cluster 2023-09-23T19:11:54.273503+0000 mgr.y (mgr.15228) 26 : cluster [DBG] pgmap v14: 89 pgs: 89 active+clean; 935 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 828 op/s 2023-09-23T19:11:56.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:56 smithi016 bash[33624]: cluster 2023-09-23T19:11:54.273503+0000 mgr.y (mgr.15228) 26 : cluster [DBG] pgmap v14: 89 pgs: 89 active+clean; 935 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 828 op/s 2023-09-23T19:11:57.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:57 smithi092 bash[27556]: cluster 2023-09-23T19:11:56.273990+0000 mgr.y (mgr.15228) 27 : cluster [DBG] pgmap v15: 89 pgs: 89 active+clean; 935 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 573 op/s 2023-09-23T19:11:57.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:57 smithi016 bash[33624]: cluster 2023-09-23T19:11:56.273990+0000 mgr.y (mgr.15228) 27 : cluster [DBG] pgmap v15: 89 pgs: 89 active+clean; 935 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 573 op/s 2023-09-23T19:11:57.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:57 smithi016 bash[27237]: cluster 2023-09-23T19:11:56.273990+0000 mgr.y (mgr.15228) 27 : cluster [DBG] pgmap v15: 89 pgs: 89 active+clean; 935 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 573 op/s 2023-09-23T19:11:58.285 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:58 smithi092 bash[27556]: audit 2023-09-23T19:11:57.708248+0000 mon.c (mon.2) 245 : audit [DBG] from='client.? 172.21.15.16:0/3938136105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:58.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:58 smithi016 bash[27237]: audit 2023-09-23T19:11:57.708248+0000 mon.c (mon.2) 245 : audit [DBG] from='client.? 172.21.15.16:0/3938136105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:58.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:58 smithi016 bash[33624]: audit 2023-09-23T19:11:57.708248+0000 mon.c (mon.2) 245 : audit [DBG] from='client.? 172.21.15.16:0/3938136105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:11:59.202 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:11:59 smithi092 bash[50491]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:11:59] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T19:11:59.202 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:11:59 smithi092 bash[27556]: cluster 2023-09-23T19:11:58.274371+0000 mgr.y (mgr.15228) 28 : cluster [DBG] pgmap v16: 89 pgs: 89 active+clean; 935 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 573 op/s 2023-09-23T19:11:59.495 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:11:59 smithi092 systemd[1]: Stopping Ceph prometheus.a for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7... 2023-09-23T19:11:59.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:11:59 smithi016 bash[33624]: cluster 2023-09-23T19:11:58.274371+0000 mgr.y (mgr.15228) 28 : cluster [DBG] pgmap v16: 89 pgs: 89 active+clean; 935 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 573 op/s 2023-09-23T19:11:59.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:11:59 smithi016 bash[27237]: cluster 2023-09-23T19:11:58.274371+0000 mgr.y (mgr.15228) 28 : cluster [DBG] pgmap v16: 89 pgs: 89 active+clean; 935 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 573 op/s 2023-09-23T19:11:59.785 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:11:59 smithi092 bash[54542]: Error response from daemon: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-prometheus.a 2023-09-23T19:11:59.786 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:11:59 smithi092 bash[52849]: ts=2023-09-23T19:11:59.691Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-09-23T19:11:59.786 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:11:59 smithi092 bash[52849]: ts=2023-09-23T19:11:59.691Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-09-23T19:11:59.786 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:11:59 smithi092 bash[52849]: ts=2023-09-23T19:11:59.691Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-09-23T19:11:59.786 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:11:59 smithi092 bash[52849]: ts=2023-09-23T19:11:59.691Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-09-23T19:11:59.786 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:11:59 smithi092 bash[52849]: ts=2023-09-23T19:11:59.691Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-09-23T19:11:59.786 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:11:59 smithi092 bash[52849]: ts=2023-09-23T19:11:59.691Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-09-23T19:11:59.786 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:11:59 smithi092 bash[52849]: ts=2023-09-23T19:11:59.691Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-09-23T19:11:59.786 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:11:59 smithi092 bash[52849]: ts=2023-09-23T19:11:59.691Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-09-23T19:11:59.786 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:11:59 smithi092 bash[52849]: ts=2023-09-23T19:11:59.691Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-09-23T19:11:59.786 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:11:59 smithi092 bash[52849]: ts=2023-09-23T19:11:59.696Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-09-23T19:11:59.786 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:11:59 smithi092 bash[52849]: ts=2023-09-23T19:11:59.697Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-09-23T19:11:59.787 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:11:59 smithi092 bash[52849]: ts=2023-09-23T19:11:59.697Z caller=main.go:1066 level=info msg="See you next time!" 2023-09-23T19:12:00.633 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:12:00 smithi092 bash[54571]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-prometheus-a 2023-09-23T19:12:00.634 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:12:00 smithi092 bash[54628]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-prometheus-a 2023-09-23T19:12:00.634 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:12:00 smithi092 systemd[1]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@prometheus.a.service: Succeeded. 2023-09-23T19:12:00.634 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:12:00 smithi092 systemd[1]: Stopped Ceph prometheus.a for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7. 2023-09-23T19:12:00.634 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:12:00 smithi092 systemd[1]: Started Ceph prometheus.a for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7. 2023-09-23T19:12:01.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:01 smithi016 bash[33624]: audit 2023-09-23T19:12:00.407531+0000 mon.a (mon.0) 854 : audit [DBG] from='client.? 172.21.15.16:0/352569054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:12:01.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:01 smithi016 bash[33624]: audit 2023-09-23T19:12:00.580329+0000 mon.a (mon.0) 855 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:12:01.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:01 smithi016 bash[33624]: audit 2023-09-23T19:12:00.589280+0000 mon.a (mon.0) 856 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:12:01.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:01 smithi016 bash[33624]: audit 2023-09-23T19:12:00.619258+0000 mon.c (mon.2) 246 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T19:12:01.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:01 smithi016 bash[33624]: audit 2023-09-23T19:12:00.630930+0000 mon.c (mon.2) 247 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:12:01.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:01 smithi016 bash[27237]: audit 2023-09-23T19:12:00.407531+0000 mon.a (mon.0) 854 : audit [DBG] from='client.? 172.21.15.16:0/352569054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:12:01.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:01 smithi016 bash[27237]: audit 2023-09-23T19:12:00.580329+0000 mon.a (mon.0) 855 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:12:01.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:01 smithi016 bash[27237]: audit 2023-09-23T19:12:00.589280+0000 mon.a (mon.0) 856 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:12:01.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:01 smithi016 bash[27237]: audit 2023-09-23T19:12:00.619258+0000 mon.c (mon.2) 246 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T19:12:01.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:01 smithi016 bash[27237]: audit 2023-09-23T19:12:00.630930+0000 mon.c (mon.2) 247 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:12:01.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:01 smithi092 bash[27556]: audit 2023-09-23T19:12:00.407531+0000 mon.a (mon.0) 854 : audit [DBG] from='client.? 172.21.15.16:0/352569054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:12:01.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:01 smithi092 bash[27556]: audit 2023-09-23T19:12:00.580329+0000 mon.a (mon.0) 855 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:12:01.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:01 smithi092 bash[27556]: audit 2023-09-23T19:12:00.589280+0000 mon.a (mon.0) 856 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:12:01.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:01 smithi092 bash[27556]: audit 2023-09-23T19:12:00.619258+0000 mon.c (mon.2) 246 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T19:12:01.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:01 smithi092 bash[27556]: audit 2023-09-23T19:12:00.630930+0000 mon.c (mon.2) 247 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:12:02.345 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:02 smithi092 bash[27556]: cluster 2023-09-23T19:12:00.275057+0000 mgr.y (mgr.15228) 29 : cluster [DBG] pgmap v17: 89 pgs: 89 active+clean; 713 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 868 op/s 2023-09-23T19:12:02.345 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:02 smithi092 bash[27556]: audit 2023-09-23T19:12:00.619940+0000 mgr.y (mgr.15228) 30 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T19:12:02.345 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:12:02 smithi092 bash[54653]: ts=2023-09-23T19:12:02.348Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2023-09-23T19:12:02.346 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:12:02 smithi092 bash[54653]: ts=2023-09-23T19:12:02.348Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2023-09-23T19:12:02.346 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:12:02 smithi092 bash[54653]: ts=2023-09-23T19:12:02.348Z caller=main.go:518 level=info host_details="(Linux 5.15.0-84-generic #93~20.04.1-Ubuntu SMP Wed Sep 6 16:15:40 UTC 2023 x86_64 smithi092 (none))" 2023-09-23T19:12:02.346 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:12:02 smithi092 bash[54653]: ts=2023-09-23T19:12:02.348Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-09-23T19:12:02.346 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:12:02 smithi092 bash[54653]: ts=2023-09-23T19:12:02.348Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-09-23T19:12:02.484 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:12:02 smithi092 bash[54653]: ts=2023-09-23T19:12:02.351Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2023-09-23T19:12:02.484 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:12:02 smithi092 bash[54653]: ts=2023-09-23T19:12:02.357Z caller=main.go:923 level=info msg="Starting TSDB ..." 2023-09-23T19:12:02.484 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:12:02 smithi092 bash[54653]: ts=2023-09-23T19:12:02.358Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2023-09-23T19:12:02.485 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:12:02 smithi092 bash[54653]: ts=2023-09-23T19:12:02.361Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-09-23T19:12:02.485 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:12:02 smithi092 bash[54653]: ts=2023-09-23T19:12:02.361Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=4.718µs 2023-09-23T19:12:02.485 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:12:02 smithi092 bash[54653]: ts=2023-09-23T19:12:02.361Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-09-23T19:12:02.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:02 smithi016 bash[27237]: cluster 2023-09-23T19:12:00.275057+0000 mgr.y (mgr.15228) 29 : cluster [DBG] pgmap v17: 89 pgs: 89 active+clean; 713 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 868 op/s 2023-09-23T19:12:02.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:02 smithi016 bash[27237]: audit 2023-09-23T19:12:00.619940+0000 mgr.y (mgr.15228) 30 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T19:12:02.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:02 smithi016 bash[33624]: cluster 2023-09-23T19:12:00.275057+0000 mgr.y (mgr.15228) 29 : cluster [DBG] pgmap v17: 89 pgs: 89 active+clean; 713 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 868 op/s 2023-09-23T19:12:02.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:02 smithi016 bash[33624]: audit 2023-09-23T19:12:00.619940+0000 mgr.y (mgr.15228) 30 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T19:12:03.035 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:12:02 smithi092 bash[54653]: ts=2023-09-23T19:12:02.659Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2023-09-23T19:12:03.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:03 smithi016 bash[27237]: audit 2023-09-23T19:12:02.457828+0000 mon.c (mon.2) 248 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:12:03.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:03 smithi016 bash[33624]: audit 2023-09-23T19:12:02.457828+0000 mon.c (mon.2) 248 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:12:03.535 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:03 smithi092 bash[27556]: audit 2023-09-23T19:12:02.457828+0000 mon.c (mon.2) 248 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:12:03.536 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:12:03 smithi092 bash[54653]: ts=2023-09-23T19:12:03.076Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2023-09-23T19:12:03.536 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:12:03 smithi092 bash[54653]: ts=2023-09-23T19:12:03.136Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2023-09-23T19:12:03.536 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:12:03 smithi092 bash[54653]: ts=2023-09-23T19:12:03.137Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2023-09-23T19:12:03.536 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:12:03 smithi092 bash[54653]: ts=2023-09-23T19:12:03.137Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=60.068µs wal_replay_duration=775.875066ms total_replay_duration=775.963669ms 2023-09-23T19:12:03.536 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:12:03 smithi092 bash[54653]: ts=2023-09-23T19:12:03.142Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2023-09-23T19:12:03.536 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:12:03 smithi092 bash[54653]: ts=2023-09-23T19:12:03.143Z caller=main.go:947 level=info msg="TSDB started" 2023-09-23T19:12:03.536 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:12:03 smithi092 bash[54653]: ts=2023-09-23T19:12:03.143Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-09-23T19:12:03.536 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:12:03 smithi092 bash[54653]: ts=2023-09-23T19:12:03.170Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=27.865237ms db_storage=1.138µs remote_storage=1.963µs web_handler=459ns query_engine=1.274µs scrape=332.737µs scrape_sd=137.059µs notify=17.338µs notify_sd=14.573µs rules=26.779079ms 2023-09-23T19:12:03.536 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:12:03 smithi092 bash[54653]: ts=2023-09-23T19:12:03.171Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-09-23T19:12:04.079 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:04 smithi092 bash[27556]: cluster 2023-09-23T19:12:02.275532+0000 mgr.y (mgr.15228) 31 : cluster [DBG] pgmap v18: 89 pgs: 89 active+clean; 713 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 576 op/s 2023-09-23T19:12:04.079 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:04 smithi092 bash[27556]: audit 2023-09-23T19:12:03.080268+0000 mon.a (mon.0) 857 : audit [DBG] from='client.? 172.21.15.16:0/204239463' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:12:04.079 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:04 smithi092 bash[27556]: cephadm 2023-09-23T19:12:03.396080+0000 mgr.y (mgr.15228) 32 : cephadm [INF] Upgrade: Updating mgr.x 2023-09-23T19:12:04.079 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:04 smithi092 bash[27556]: audit 2023-09-23T19:12:03.400689+0000 mon.c (mon.2) 249 : audit [INF] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T19:12:04.079 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:04 smithi092 bash[27556]: audit 2023-09-23T19:12:03.402480+0000 mon.a (mon.0) 858 : audit [INF] from='mgr.15228 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T19:12:04.079 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:04 smithi092 bash[27556]: audit 2023-09-23T19:12:03.405497+0000 mon.c (mon.2) 250 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T19:12:04.080 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:04 smithi092 bash[27556]: audit 2023-09-23T19:12:03.407154+0000 mon.c (mon.2) 251 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:12:04.080 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:04 smithi092 bash[27556]: cephadm 2023-09-23T19:12:03.409061+0000 mgr.y (mgr.15228) 33 : cephadm [INF] Deploying daemon mgr.x on smithi092 2023-09-23T19:12:04.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:04 smithi016 bash[33624]: cluster 2023-09-23T19:12:02.275532+0000 mgr.y (mgr.15228) 31 : cluster [DBG] pgmap v18: 89 pgs: 89 active+clean; 713 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 576 op/s 2023-09-23T19:12:04.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:04 smithi016 bash[33624]: audit 2023-09-23T19:12:03.080268+0000 mon.a (mon.0) 857 : audit [DBG] from='client.? 172.21.15.16:0/204239463' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:12:04.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:04 smithi016 bash[33624]: cephadm 2023-09-23T19:12:03.396080+0000 mgr.y (mgr.15228) 32 : cephadm [INF] Upgrade: Updating mgr.x 2023-09-23T19:12:04.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:04 smithi016 bash[33624]: audit 2023-09-23T19:12:03.400689+0000 mon.c (mon.2) 249 : audit [INF] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T19:12:04.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:04 smithi016 bash[33624]: audit 2023-09-23T19:12:03.402480+0000 mon.a (mon.0) 858 : audit [INF] from='mgr.15228 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T19:12:04.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:04 smithi016 bash[33624]: audit 2023-09-23T19:12:03.405497+0000 mon.c (mon.2) 250 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T19:12:04.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:04 smithi016 bash[33624]: audit 2023-09-23T19:12:03.407154+0000 mon.c (mon.2) 251 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:12:04.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:04 smithi016 bash[33624]: cephadm 2023-09-23T19:12:03.409061+0000 mgr.y (mgr.15228) 33 : cephadm [INF] Deploying daemon mgr.x on smithi092 2023-09-23T19:12:04.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:04 smithi016 bash[27237]: cluster 2023-09-23T19:12:02.275532+0000 mgr.y (mgr.15228) 31 : cluster [DBG] pgmap v18: 89 pgs: 89 active+clean; 713 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 576 op/s 2023-09-23T19:12:04.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:04 smithi016 bash[27237]: audit 2023-09-23T19:12:03.080268+0000 mon.a (mon.0) 857 : audit [DBG] from='client.? 172.21.15.16:0/204239463' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:12:04.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:04 smithi016 bash[27237]: cephadm 2023-09-23T19:12:03.396080+0000 mgr.y (mgr.15228) 32 : cephadm [INF] Upgrade: Updating mgr.x 2023-09-23T19:12:04.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:04 smithi016 bash[27237]: audit 2023-09-23T19:12:03.400689+0000 mon.c (mon.2) 249 : audit [INF] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T19:12:04.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:04 smithi016 bash[27237]: audit 2023-09-23T19:12:03.402480+0000 mon.a (mon.0) 858 : audit [INF] from='mgr.15228 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T19:12:04.538 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:04 smithi016 bash[27237]: audit 2023-09-23T19:12:03.405497+0000 mon.c (mon.2) 250 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T19:12:04.538 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:04 smithi016 bash[27237]: audit 2023-09-23T19:12:03.407154+0000 mon.c (mon.2) 251 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:12:04.538 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:04 smithi016 bash[27237]: cephadm 2023-09-23T19:12:03.409061+0000 mgr.y (mgr.15228) 33 : cephadm [INF] Deploying daemon mgr.x on smithi092 2023-09-23T19:12:06.230 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:06 smithi092 systemd[1]: Stopping Ceph mgr.x for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7... 2023-09-23T19:12:06.230 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:06 smithi092 bash[27556]: cluster 2023-09-23T19:12:04.276360+0000 mgr.y (mgr.15228) 34 : cluster [DBG] pgmap v19: 89 pgs: 89 active+clean; 502 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 858 op/s 2023-09-23T19:12:06.230 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:06 smithi092 bash[27556]: audit 2023-09-23T19:12:05.845061+0000 mon.b (mon.1) 49 : audit [DBG] from='client.? 172.21.15.16:0/1401833301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:12:06.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:06 smithi016 bash[33624]: cluster 2023-09-23T19:12:04.276360+0000 mgr.y (mgr.15228) 34 : cluster [DBG] pgmap v19: 89 pgs: 89 active+clean; 502 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 858 op/s 2023-09-23T19:12:06.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:06 smithi016 bash[33624]: audit 2023-09-23T19:12:05.845061+0000 mon.b (mon.1) 49 : audit [DBG] from='client.? 172.21.15.16:0/1401833301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:12:06.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:06 smithi016 bash[27237]: cluster 2023-09-23T19:12:04.276360+0000 mgr.y (mgr.15228) 34 : cluster [DBG] pgmap v19: 89 pgs: 89 active+clean; 502 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 858 op/s 2023-09-23T19:12:06.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:06 smithi016 bash[27237]: audit 2023-09-23T19:12:05.845061+0000 mon.b (mon.1) 49 : audit [DBG] from='client.? 172.21.15.16:0/1401833301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:12:07.403 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:07 smithi092 bash[54953]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-mgr-x 2023-09-23T19:12:07.751 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:07 smithi092 systemd[1]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-09-23T19:12:07.751 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:07 smithi092 bash[55007]: Error response from daemon: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-mgr-x 2023-09-23T19:12:07.751 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:07 smithi092 systemd[1]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@mgr.x.service: Failed with result 'exit-code'. 2023-09-23T19:12:07.751 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:07 smithi092 systemd[1]: Stopped Ceph mgr.x for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7. 2023-09-23T19:12:07.752 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:07 smithi092 systemd[1]: Started Ceph mgr.x for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7. 2023-09-23T19:12:08.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:08 smithi016 bash[27237]: cluster 2023-09-23T19:12:06.276760+0000 mgr.y (mgr.15228) 35 : cluster [DBG] pgmap v20: 89 pgs: 89 active+clean; 502 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 577 op/s 2023-09-23T19:12:08.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:08 smithi016 bash[27237]: audit 2023-09-23T19:12:07.798131+0000 mon.a (mon.0) 859 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:12:08.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:08 smithi016 bash[27237]: audit 2023-09-23T19:12:07.812456+0000 mon.a (mon.0) 860 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:12:08.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:08 smithi016 bash[27237]: audit 2023-09-23T19:12:07.837070+0000 mon.c (mon.2) 252 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:12:08.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:08 smithi016 bash[33624]: cluster 2023-09-23T19:12:06.276760+0000 mgr.y (mgr.15228) 35 : cluster [DBG] pgmap v20: 89 pgs: 89 active+clean; 502 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 577 op/s 2023-09-23T19:12:08.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:08 smithi016 bash[33624]: audit 2023-09-23T19:12:07.798131+0000 mon.a (mon.0) 859 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:12:08.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:08 smithi016 bash[33624]: audit 2023-09-23T19:12:07.812456+0000 mon.a (mon.0) 860 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:12:08.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:08 smithi016 bash[33624]: audit 2023-09-23T19:12:07.837070+0000 mon.c (mon.2) 252 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:12:08.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:08 smithi092 bash[27556]: cluster 2023-09-23T19:12:06.276760+0000 mgr.y (mgr.15228) 35 : cluster [DBG] pgmap v20: 89 pgs: 89 active+clean; 502 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 577 op/s 2023-09-23T19:12:08.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:08 smithi092 bash[27556]: audit 2023-09-23T19:12:07.798131+0000 mon.a (mon.0) 859 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:12:08.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:08 smithi092 bash[27556]: audit 2023-09-23T19:12:07.812456+0000 mon.a (mon.0) 860 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:12:08.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:08 smithi092 bash[27556]: audit 2023-09-23T19:12:07.837070+0000 mon.c (mon.2) 252 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:12:09.285 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:09 smithi092 bash[27556]: audit 2023-09-23T19:12:08.523377+0000 mon.c (mon.2) 253 : audit [DBG] from='client.? 172.21.15.16:0/207713925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:12:09.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:09 smithi016 bash[27237]: audit 2023-09-23T19:12:08.523377+0000 mon.c (mon.2) 253 : audit [DBG] from='client.? 172.21.15.16:0/207713925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:12:09.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:09 smithi016 bash[33624]: audit 2023-09-23T19:12:08.523377+0000 mon.c (mon.2) 253 : audit [DBG] from='client.? 172.21.15.16:0/207713925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:12:09.785 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:09 smithi092 bash[55056]: debug 2023-09-23T19:12:09.336+0000 7fad44b1a200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T19:12:10.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:10 smithi016 bash[33624]: cluster 2023-09-23T19:12:08.277142+0000 mgr.y (mgr.15228) 36 : cluster [DBG] pgmap v21: 89 pgs: 89 active+clean; 502 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 577 op/s 2023-09-23T19:12:10.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:10 smithi016 bash[27237]: cluster 2023-09-23T19:12:08.277142+0000 mgr.y (mgr.15228) 36 : cluster [DBG] pgmap v21: 89 pgs: 89 active+clean; 502 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 577 op/s 2023-09-23T19:12:10.535 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:10 smithi092 bash[27556]: cluster 2023-09-23T19:12:08.277142+0000 mgr.y (mgr.15228) 36 : cluster [DBG] pgmap v21: 89 pgs: 89 active+clean; 502 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 577 op/s 2023-09-23T19:12:10.983 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:10 smithi092 bash[55056]: debug 2023-09-23T19:12:10.684+0000 7fad44b1a200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T19:12:10.983 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:10 smithi092 bash[55056]: debug 2023-09-23T19:12:10.836+0000 7fad44b1a200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T19:12:10.983 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:10 smithi092 bash[55056]: debug 2023-09-23T19:12:10.984+0000 7fad44b1a200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T19:12:11.285 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:11 smithi092 bash[27556]: cluster 2023-09-23T19:12:10.277967+0000 mgr.y (mgr.15228) 37 : cluster [DBG] pgmap v22: 89 pgs: 89 active+clean; 289 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 861 op/s 2023-09-23T19:12:11.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:11 smithi016 bash[33624]: cluster 2023-09-23T19:12:10.277967+0000 mgr.y (mgr.15228) 37 : cluster [DBG] pgmap v22: 89 pgs: 89 active+clean; 289 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 861 op/s 2023-09-23T19:12:11.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:11 smithi016 bash[27237]: cluster 2023-09-23T19:12:10.277967+0000 mgr.y (mgr.15228) 37 : cluster [DBG] pgmap v22: 89 pgs: 89 active+clean; 289 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 861 op/s 2023-09-23T19:12:11.785 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:11 smithi092 bash[55056]: debug 2023-09-23T19:12:11.368+0000 7fad44b1a200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T19:12:12.285 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:12 smithi092 bash[27556]: audit 2023-09-23T19:12:11.219740+0000 mon.a (mon.0) 861 : audit [DBG] from='client.? 172.21.15.16:0/414177117' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:12:12.285 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:11 smithi092 bash[55056]: debug 2023-09-23T19:12:11.912+0000 7fad44b1a200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T19:12:12.286 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:12 smithi092 bash[55056]: debug 2023-09-23T19:12:12.140+0000 7fad44b1a200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T19:12:12.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:12 smithi016 bash[33624]: audit 2023-09-23T19:12:11.219740+0000 mon.a (mon.0) 861 : audit [DBG] from='client.? 172.21.15.16:0/414177117' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:12:12.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:12 smithi016 bash[27237]: audit 2023-09-23T19:12:11.219740+0000 mon.a (mon.0) 861 : audit [DBG] from='client.? 172.21.15.16:0/414177117' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:12:12.785 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:12 smithi092 bash[55056]: debug 2023-09-23T19:12:12.460+0000 7fad44b1a200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T19:12:13.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:13 smithi016 bash[27237]: cluster 2023-09-23T19:12:12.278447+0000 mgr.y (mgr.15228) 38 : cluster [DBG] pgmap v23: 89 pgs: 89 active+clean; 289 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 566 op/s 2023-09-23T19:12:13.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:13 smithi016 bash[33624]: cluster 2023-09-23T19:12:12.278447+0000 mgr.y (mgr.15228) 38 : cluster [DBG] pgmap v23: 89 pgs: 89 active+clean; 289 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 566 op/s 2023-09-23T19:12:13.535 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:13 smithi092 bash[27556]: cluster 2023-09-23T19:12:12.278447+0000 mgr.y (mgr.15228) 38 : cluster [DBG] pgmap v23: 89 pgs: 89 active+clean; 289 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 566 op/s 2023-09-23T19:12:14.285 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:14 smithi092 bash[27556]: audit 2023-09-23T19:12:13.846940+0000 mon.a (mon.0) 862 : audit [DBG] from='client.? 172.21.15.16:0/1185938341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:12:14.286 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:14 smithi092 bash[55056]: debug 2023-09-23T19:12:14.028+0000 7fad44b1a200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T19:12:14.286 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:14 smithi092 bash[55056]: debug 2023-09-23T19:12:14.212+0000 7fad44b1a200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T19:12:14.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:14 smithi016 bash[33624]: audit 2023-09-23T19:12:13.846940+0000 mon.a (mon.0) 862 : audit [DBG] from='client.? 172.21.15.16:0/1185938341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:12:14.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:14 smithi016 bash[27237]: audit 2023-09-23T19:12:13.846940+0000 mon.a (mon.0) 862 : audit [DBG] from='client.? 172.21.15.16:0/1185938341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:12:14.785 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:14 smithi092 bash[55056]: debug 2023-09-23T19:12:14.392+0000 7fad44b1a200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T19:12:15.500 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:15 smithi092 bash[55056]: debug 2023-09-23T19:12:15.144+0000 7fad44b1a200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T19:12:15.500 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:15 smithi092 bash[55056]: debug 2023-09-23T19:12:15.500+0000 7fad44b1a200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T19:12:15.710 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Removed 111768 objects 2023-09-23T19:12:15.711 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Clean up completed and total clean up time :162.885 2023-09-23T19:12:15.785 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:15 smithi092 bash[27556]: cluster 2023-09-23T19:12:14.279176+0000 mgr.y (mgr.15228) 39 : cluster [DBG] pgmap v24: 89 pgs: 89 active+clean; 86 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 836 op/s 2023-09-23T19:12:15.785 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:15 smithi092 bash[27556]: audit 2023-09-23T19:12:14.641739+0000 mon.a (mon.0) 863 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:12:15.785 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:15 smithi092 bash[27556]: audit 2023-09-23T19:12:14.652109+0000 mon.a (mon.0) 864 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:12:16.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:12:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:12:15] "GET /metrics HTTP/1.1" 200 36055 "" "Prometheus/2.33.4" 2023-09-23T19:12:16.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:15 smithi016 bash[33624]: cluster 2023-09-23T19:12:14.279176+0000 mgr.y (mgr.15228) 39 : cluster [DBG] pgmap v24: 89 pgs: 89 active+clean; 86 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 836 op/s 2023-09-23T19:12:16.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:15 smithi016 bash[33624]: audit 2023-09-23T19:12:14.641739+0000 mon.a (mon.0) 863 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:12:16.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:15 smithi016 bash[33624]: audit 2023-09-23T19:12:14.652109+0000 mon.a (mon.0) 864 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:12:16.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:15 smithi016 bash[27237]: cluster 2023-09-23T19:12:14.279176+0000 mgr.y (mgr.15228) 39 : cluster [DBG] pgmap v24: 89 pgs: 89 active+clean; 86 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 836 op/s 2023-09-23T19:12:16.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:15 smithi016 bash[27237]: audit 2023-09-23T19:12:14.641739+0000 mon.a (mon.0) 863 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:12:16.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:15 smithi016 bash[27237]: audit 2023-09-23T19:12:14.652109+0000 mon.a (mon.0) 864 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:12:16.535 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:16 smithi092 bash[55056]: debug 2023-09-23T19:12:16.056+0000 7fad44b1a200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T19:12:17.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:16 smithi016 bash[27237]: audit 2023-09-23T19:12:15.758932+0000 mon.a (mon.0) 865 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:12:17.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:16 smithi016 bash[27237]: audit 2023-09-23T19:12:15.768937+0000 mon.a (mon.0) 866 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:12:17.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:16 smithi016 bash[27237]: audit 2023-09-23T19:12:16.317680+0000 mon.c (mon.2) 254 : audit [DBG] from='client.? 172.21.15.16:0/1695040918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:12:17.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:16 smithi016 bash[33624]: audit 2023-09-23T19:12:15.758932+0000 mon.a (mon.0) 865 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:12:17.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:16 smithi016 bash[33624]: audit 2023-09-23T19:12:15.768937+0000 mon.a (mon.0) 866 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:12:17.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:16 smithi016 bash[33624]: audit 2023-09-23T19:12:16.317680+0000 mon.c (mon.2) 254 : audit [DBG] from='client.? 172.21.15.16:0/1695040918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:12:17.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:16 smithi092 bash[27556]: audit 2023-09-23T19:12:15.758932+0000 mon.a (mon.0) 865 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:12:17.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:16 smithi092 bash[27556]: audit 2023-09-23T19:12:15.768937+0000 mon.a (mon.0) 866 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:12:17.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:16 smithi092 bash[27556]: audit 2023-09-23T19:12:16.317680+0000 mon.c (mon.2) 254 : audit [DBG] from='client.? 172.21.15.16:0/1695040918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:12:17.036 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:16 smithi092 bash[55056]: debug 2023-09-23T19:12:16.556+0000 7fad44b1a200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T19:12:17.536 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:17 smithi092 bash[55056]: debug 2023-09-23T19:12:17.108+0000 7fad44b1a200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T19:12:17.536 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:17 smithi092 bash[55056]: debug 2023-09-23T19:12:17.220+0000 7fad44b1a200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T19:12:17.931 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:17 smithi092 bash[27556]: cluster 2023-09-23T19:12:16.279821+0000 mgr.y (mgr.15228) 40 : cluster [DBG] pgmap v25: 89 pgs: 89 active+clean; 86 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 554 op/s 2023-09-23T19:12:17.931 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:17 smithi092 bash[27556]: audit 2023-09-23T19:12:17.457747+0000 mon.c (mon.2) 255 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:12:17.932 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:17 smithi092 bash[55056]: debug 2023-09-23T19:12:17.568+0000 7fad44b1a200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T19:12:17.932 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:17 smithi092 bash[55056]: debug 2023-09-23T19:12:17.676+0000 7fad44b1a200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T19:12:17.932 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:17 smithi092 bash[55056]: debug 2023-09-23T19:12:17.816+0000 7fad44b1a200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T19:12:18.016 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2023-09-23T19:12:18.016 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2023-09-23T19:12:18.030 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:17 smithi016 bash[27237]: cluster 2023-09-23T19:12:16.279821+0000 mgr.y (mgr.15228) 40 : cluster [DBG] pgmap v25: 89 pgs: 89 active+clean; 86 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 554 op/s 2023-09-23T19:12:18.030 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:17 smithi016 bash[27237]: audit 2023-09-23T19:12:17.457747+0000 mon.c (mon.2) 255 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:12:18.030 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:17 smithi016 bash[33624]: cluster 2023-09-23T19:12:16.279821+0000 mgr.y (mgr.15228) 40 : cluster [DBG] pgmap v25: 89 pgs: 89 active+clean; 86 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 554 op/s 2023-09-23T19:12:18.030 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:17 smithi016 bash[33624]: audit 2023-09-23T19:12:17.457747+0000 mon.c (mon.2) 255 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:12:18.257 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:17 smithi092 bash[55056]: debug 2023-09-23T19:12:17.932+0000 7fad44b1a200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T19:12:18.257 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:18 smithi092 bash[55056]: debug 2023-09-23T19:12:18.060+0000 7fad44b1a200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T19:12:18.535 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:18 smithi092 bash[55056]: debug 2023-09-23T19:12:18.260+0000 7fad44b1a200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T19:12:18.535 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:18 smithi092 bash[55056]: debug 2023-09-23T19:12:18.388+0000 7fad44b1a200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T19:12:19.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:18 smithi016 bash[27237]: audit 2023-09-23T19:12:18.767246+0000 mon.a (mon.0) 867 : audit [DBG] from='client.? 172.21.15.16:0/3448737287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:12:19.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:18 smithi016 bash[33624]: audit 2023-09-23T19:12:18.767246+0000 mon.a (mon.0) 867 : audit [DBG] from='client.? 172.21.15.16:0/3448737287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:12:19.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:18 smithi092 bash[27556]: audit 2023-09-23T19:12:18.767246+0000 mon.a (mon.0) 867 : audit [DBG] from='client.? 172.21.15.16:0/3448737287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:12:19.035 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:12:18 smithi092 bash[54653]: ts=2023-09-23T19:12:18.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi092.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi016:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T19:12:20.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:19 smithi016 bash[27237]: cluster 2023-09-23T19:12:18.280537+0000 mgr.y (mgr.15228) 41 : cluster [DBG] pgmap v26: 89 pgs: 89 active+clean; 86 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 554 op/s 2023-09-23T19:12:20.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:19 smithi016 bash[27237]: cluster 2023-09-23T19:12:19.652643+0000 mon.a (mon.0) 868 : cluster [DBG] Standby manager daemon x restarted 2023-09-23T19:12:20.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:19 smithi016 bash[27237]: cluster 2023-09-23T19:12:19.652798+0000 mon.a (mon.0) 869 : cluster [DBG] Standby manager daemon x started 2023-09-23T19:12:20.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:19 smithi016 bash[27237]: audit 2023-09-23T19:12:19.657487+0000 mon.c (mon.2) 256 : audit [DBG] from='mgr.? 172.21.15.92:0/2846194057' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T19:12:20.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:19 smithi016 bash[27237]: audit 2023-09-23T19:12:19.658240+0000 mon.c (mon.2) 257 : audit [DBG] from='mgr.? 172.21.15.92:0/2846194057' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T19:12:20.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:19 smithi016 bash[27237]: audit 2023-09-23T19:12:19.663020+0000 mon.c (mon.2) 258 : audit [DBG] from='mgr.? 172.21.15.92:0/2846194057' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T19:12:20.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:19 smithi016 bash[27237]: audit 2023-09-23T19:12:19.665003+0000 mon.c (mon.2) 259 : audit [DBG] from='mgr.? 172.21.15.92:0/2846194057' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T19:12:20.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:19 smithi016 bash[33624]: cluster 2023-09-23T19:12:18.280537+0000 mgr.y (mgr.15228) 41 : cluster [DBG] pgmap v26: 89 pgs: 89 active+clean; 86 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 554 op/s 2023-09-23T19:12:20.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:19 smithi016 bash[33624]: cluster 2023-09-23T19:12:19.652643+0000 mon.a (mon.0) 868 : cluster [DBG] Standby manager daemon x restarted 2023-09-23T19:12:20.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:19 smithi016 bash[33624]: cluster 2023-09-23T19:12:19.652798+0000 mon.a (mon.0) 869 : cluster [DBG] Standby manager daemon x started 2023-09-23T19:12:20.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:19 smithi016 bash[33624]: audit 2023-09-23T19:12:19.657487+0000 mon.c (mon.2) 256 : audit [DBG] from='mgr.? 172.21.15.92:0/2846194057' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T19:12:20.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:19 smithi016 bash[33624]: audit 2023-09-23T19:12:19.658240+0000 mon.c (mon.2) 257 : audit [DBG] from='mgr.? 172.21.15.92:0/2846194057' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T19:12:20.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:19 smithi016 bash[33624]: audit 2023-09-23T19:12:19.663020+0000 mon.c (mon.2) 258 : audit [DBG] from='mgr.? 172.21.15.92:0/2846194057' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T19:12:20.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:19 smithi016 bash[33624]: audit 2023-09-23T19:12:19.665003+0000 mon.c (mon.2) 259 : audit [DBG] from='mgr.? 172.21.15.92:0/2846194057' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T19:12:20.036 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:19 smithi092 bash[55056]: debug 2023-09-23T19:12:19.636+0000 7fad44b1a200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T19:12:20.036 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:19 smithi092 bash[55056]: [23/Sep/2023:19:12:19] ENGINE Bus STARTING 2023-09-23T19:12:20.037 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:19 smithi092 bash[55056]: CherryPy Checker: 2023-09-23T19:12:20.037 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:19 smithi092 bash[55056]: The Application mounted at '' has an empty config. 2023-09-23T19:12:20.037 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:19 smithi092 bash[55056]: [23/Sep/2023:19:12:19] ENGINE Serving on http://:::9283 2023-09-23T19:12:20.037 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:19 smithi092 bash[55056]: [23/Sep/2023:19:12:19] ENGINE Bus STARTED 2023-09-23T19:12:20.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:19 smithi092 bash[27556]: cluster 2023-09-23T19:12:18.280537+0000 mgr.y (mgr.15228) 41 : cluster [DBG] pgmap v26: 89 pgs: 89 active+clean; 86 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 554 op/s 2023-09-23T19:12:20.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:19 smithi092 bash[27556]: cluster 2023-09-23T19:12:19.652643+0000 mon.a (mon.0) 868 : cluster [DBG] Standby manager daemon x restarted 2023-09-23T19:12:20.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:19 smithi092 bash[27556]: cluster 2023-09-23T19:12:19.652798+0000 mon.a (mon.0) 869 : cluster [DBG] Standby manager daemon x started 2023-09-23T19:12:20.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:19 smithi092 bash[27556]: audit 2023-09-23T19:12:19.657487+0000 mon.c (mon.2) 256 : audit [DBG] from='mgr.? 172.21.15.92:0/2846194057' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T19:12:20.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:19 smithi092 bash[27556]: audit 2023-09-23T19:12:19.658240+0000 mon.c (mon.2) 257 : audit [DBG] from='mgr.? 172.21.15.92:0/2846194057' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T19:12:20.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:19 smithi092 bash[27556]: audit 2023-09-23T19:12:19.663020+0000 mon.c (mon.2) 258 : audit [DBG] from='mgr.? 172.21.15.92:0/2846194057' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T19:12:20.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:19 smithi092 bash[27556]: audit 2023-09-23T19:12:19.665003+0000 mon.c (mon.2) 259 : audit [DBG] from='mgr.? 172.21.15.92:0/2846194057' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T19:12:21.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:20 smithi092 bash[27556]: cluster 2023-09-23T19:12:19.794581+0000 mon.a (mon.0) 870 : cluster [DBG] mgrmap e32: y(active, since 47s), standbys: x 2023-09-23T19:12:21.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:20 smithi016 bash[27237]: cluster 2023-09-23T19:12:19.794581+0000 mon.a (mon.0) 870 : cluster [DBG] mgrmap e32: y(active, since 47s), standbys: x 2023-09-23T19:12:21.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:20 smithi016 bash[33624]: cluster 2023-09-23T19:12:19.794581+0000 mon.a (mon.0) 870 : cluster [DBG] mgrmap e32: y(active, since 47s), standbys: x 2023-09-23T19:12:22.362 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:22 smithi092 bash[27556]: cluster 2023-09-23T19:12:20.281706+0000 mgr.y (mgr.15228) 42 : cluster [DBG] pgmap v27: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 667 op/s 2023-09-23T19:12:22.362 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:22 smithi092 bash[27556]: audit 2023-09-23T19:12:21.219334+0000 mon.a (mon.0) 871 : audit [DBG] from='client.? 172.21.15.16:0/725146029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:12:22.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:22 smithi016 bash[27237]: cluster 2023-09-23T19:12:20.281706+0000 mgr.y (mgr.15228) 42 : cluster [DBG] pgmap v27: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 667 op/s 2023-09-23T19:12:22.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:22 smithi016 bash[27237]: audit 2023-09-23T19:12:21.219334+0000 mon.a (mon.0) 871 : audit [DBG] from='client.? 172.21.15.16:0/725146029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:12:22.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:22 smithi016 bash[33624]: cluster 2023-09-23T19:12:20.281706+0000 mgr.y (mgr.15228) 42 : cluster [DBG] pgmap v27: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 667 op/s 2023-09-23T19:12:22.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:22 smithi016 bash[33624]: audit 2023-09-23T19:12:21.219334+0000 mon.a (mon.0) 871 : audit [DBG] from='client.? 172.21.15.16:0/725146029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:12:22.798 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:12:24.095 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:24 smithi016 bash[27237]: cluster 2023-09-23T19:12:22.282484+0000 mgr.y (mgr.15228) 43 : cluster [DBG] pgmap v28: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 384 op/s 2023-09-23T19:12:24.095 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:24 smithi016 bash[27237]: audit 2023-09-23T19:12:23.648757+0000 mon.a (mon.0) 872 : audit [DBG] from='client.? 172.21.15.16:0/4208203917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:12:24.285 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:24 smithi092 bash[27556]: cluster 2023-09-23T19:12:22.282484+0000 mgr.y (mgr.15228) 43 : cluster [DBG] pgmap v28: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 384 op/s 2023-09-23T19:12:24.285 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:24 smithi092 bash[27556]: audit 2023-09-23T19:12:23.648757+0000 mon.a (mon.0) 872 : audit [DBG] from='client.? 172.21.15.16:0/4208203917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:12:24.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:24 smithi016 bash[33624]: cluster 2023-09-23T19:12:22.282484+0000 mgr.y (mgr.15228) 43 : cluster [DBG] pgmap v28: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 384 op/s 2023-09-23T19:12:24.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:24 smithi016 bash[33624]: audit 2023-09-23T19:12:23.648757+0000 mon.a (mon.0) 872 : audit [DBG] from='client.? 172.21.15.16:0/4208203917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:12:25.784 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:12:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:12:25] "GET /metrics HTTP/1.1" 200 36052 "" "Prometheus/2.33.4" 2023-09-23T19:12:26.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:26 smithi016 bash[33624]: cluster 2023-09-23T19:12:24.283837+0000 mgr.y (mgr.15228) 44 : cluster [DBG] pgmap v29: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 384 op/s 2023-09-23T19:12:26.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:26 smithi016 bash[33624]: audit 2023-09-23T19:12:25.417313+0000 mon.a (mon.0) 873 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:12:26.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:26 smithi016 bash[33624]: audit 2023-09-23T19:12:25.427334+0000 mon.a (mon.0) 874 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:12:26.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:26 smithi016 bash[33624]: audit 2023-09-23T19:12:25.429604+0000 mon.c (mon.2) 260 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:12:26.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:26 smithi016 bash[33624]: audit 2023-09-23T19:12:25.431208+0000 mon.c (mon.2) 261 : audit [INF] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:12:26.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:26 smithi016 bash[33624]: audit 2023-09-23T19:12:25.439862+0000 mon.a (mon.0) 875 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:12:26.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:26 smithi016 bash[33624]: audit 2023-09-23T19:12:25.522709+0000 mon.c (mon.2) 262 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:12:26.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:26 smithi016 bash[33624]: audit 2023-09-23T19:12:25.528042+0000 mon.c (mon.2) 263 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:12:26.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:26 smithi016 bash[33624]: audit 2023-09-23T19:12:25.537564+0000 mon.a (mon.0) 876 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:12:26.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:26 smithi016 bash[33624]: audit 2023-09-23T19:12:25.539597+0000 mon.c (mon.2) 264 : audit [INF] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T19:12:26.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:26 smithi016 bash[33624]: audit 2023-09-23T19:12:25.540098+0000 mon.a (mon.0) 877 : audit [INF] from='mgr.15228 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T19:12:26.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:26 smithi016 bash[33624]: audit 2023-09-23T19:12:25.545313+0000 mon.a (mon.0) 878 : audit [INF] from='mgr.15228 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-09-23T19:12:26.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:26 smithi016 bash[33624]: audit 2023-09-23T19:12:25.546958+0000 mon.c (mon.2) 265 : audit [INF] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T19:12:26.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:26 smithi016 bash[33624]: audit 2023-09-23T19:12:25.547551+0000 mon.a (mon.0) 879 : audit [INF] from='mgr.15228 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T19:12:26.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:26 smithi016 bash[33624]: audit 2023-09-23T19:12:25.551930+0000 mon.a (mon.0) 880 : audit [INF] from='mgr.15228 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-09-23T19:12:26.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:26 smithi016 bash[33624]: audit 2023-09-23T19:12:25.555998+0000 mon.c (mon.2) 266 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T19:12:26.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:26 smithi016 bash[33624]: audit 2023-09-23T19:12:25.557402+0000 mon.c (mon.2) 267 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-09-23T19:12:26.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:26 smithi016 bash[33624]: audit 2023-09-23T19:12:25.977437+0000 mon.a (mon.0) 881 : audit [INF] from='client.? 172.21.15.16:0/2552494186' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T19:12:26.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:26 smithi016 bash[27237]: cluster 2023-09-23T19:12:24.283837+0000 mgr.y (mgr.15228) 44 : cluster [DBG] pgmap v29: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 384 op/s 2023-09-23T19:12:26.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:26 smithi016 bash[27237]: audit 2023-09-23T19:12:25.417313+0000 mon.a (mon.0) 873 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:12:26.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:26 smithi016 bash[27237]: audit 2023-09-23T19:12:25.427334+0000 mon.a (mon.0) 874 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:12:26.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:26 smithi016 bash[27237]: audit 2023-09-23T19:12:25.429604+0000 mon.c (mon.2) 260 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:12:26.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:26 smithi016 bash[27237]: audit 2023-09-23T19:12:25.431208+0000 mon.c (mon.2) 261 : audit [INF] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:12:26.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:26 smithi016 bash[27237]: audit 2023-09-23T19:12:25.439862+0000 mon.a (mon.0) 875 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:12:26.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:26 smithi016 bash[27237]: audit 2023-09-23T19:12:25.522709+0000 mon.c (mon.2) 262 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:12:26.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:26 smithi016 bash[27237]: audit 2023-09-23T19:12:25.528042+0000 mon.c (mon.2) 263 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:12:26.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:26 smithi016 bash[27237]: audit 2023-09-23T19:12:25.537564+0000 mon.a (mon.0) 876 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:12:26.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:26 smithi016 bash[27237]: audit 2023-09-23T19:12:25.539597+0000 mon.c (mon.2) 264 : audit [INF] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T19:12:26.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:26 smithi016 bash[27237]: audit 2023-09-23T19:12:25.540098+0000 mon.a (mon.0) 877 : audit [INF] from='mgr.15228 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T19:12:26.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:26 smithi016 bash[27237]: audit 2023-09-23T19:12:25.545313+0000 mon.a (mon.0) 878 : audit [INF] from='mgr.15228 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-09-23T19:12:26.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:26 smithi016 bash[27237]: audit 2023-09-23T19:12:25.546958+0000 mon.c (mon.2) 265 : audit [INF] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T19:12:26.288 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:26 smithi016 bash[27237]: audit 2023-09-23T19:12:25.547551+0000 mon.a (mon.0) 879 : audit [INF] from='mgr.15228 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T19:12:26.288 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:26 smithi016 bash[27237]: audit 2023-09-23T19:12:25.551930+0000 mon.a (mon.0) 880 : audit [INF] from='mgr.15228 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-09-23T19:12:26.288 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:26 smithi016 bash[27237]: audit 2023-09-23T19:12:25.555998+0000 mon.c (mon.2) 266 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T19:12:26.288 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:26 smithi016 bash[27237]: audit 2023-09-23T19:12:25.557402+0000 mon.c (mon.2) 267 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-09-23T19:12:26.288 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:26 smithi016 bash[27237]: audit 2023-09-23T19:12:25.977437+0000 mon.a (mon.0) 881 : audit [INF] from='client.? 172.21.15.16:0/2552494186' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T19:12:26.535 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:26 smithi092 bash[27556]: cluster 2023-09-23T19:12:24.283837+0000 mgr.y (mgr.15228) 44 : cluster [DBG] pgmap v29: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 384 op/s 2023-09-23T19:12:26.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:26 smithi092 bash[27556]: audit 2023-09-23T19:12:25.417313+0000 mon.a (mon.0) 873 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:12:26.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:26 smithi092 bash[27556]: audit 2023-09-23T19:12:25.427334+0000 mon.a (mon.0) 874 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:12:26.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:26 smithi092 bash[27556]: audit 2023-09-23T19:12:25.429604+0000 mon.c (mon.2) 260 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:12:26.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:26 smithi092 bash[27556]: audit 2023-09-23T19:12:25.431208+0000 mon.c (mon.2) 261 : audit [INF] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:12:26.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:26 smithi092 bash[27556]: audit 2023-09-23T19:12:25.439862+0000 mon.a (mon.0) 875 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:12:26.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:26 smithi092 bash[27556]: audit 2023-09-23T19:12:25.522709+0000 mon.c (mon.2) 262 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:12:26.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:26 smithi092 bash[27556]: audit 2023-09-23T19:12:25.528042+0000 mon.c (mon.2) 263 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:12:26.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:26 smithi092 bash[27556]: audit 2023-09-23T19:12:25.537564+0000 mon.a (mon.0) 876 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:12:26.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:26 smithi092 bash[27556]: audit 2023-09-23T19:12:25.539597+0000 mon.c (mon.2) 264 : audit [INF] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T19:12:26.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:26 smithi092 bash[27556]: audit 2023-09-23T19:12:25.540098+0000 mon.a (mon.0) 877 : audit [INF] from='mgr.15228 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T19:12:26.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:26 smithi092 bash[27556]: audit 2023-09-23T19:12:25.545313+0000 mon.a (mon.0) 878 : audit [INF] from='mgr.15228 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-09-23T19:12:26.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:26 smithi092 bash[27556]: audit 2023-09-23T19:12:25.546958+0000 mon.c (mon.2) 265 : audit [INF] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T19:12:26.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:26 smithi092 bash[27556]: audit 2023-09-23T19:12:25.547551+0000 mon.a (mon.0) 879 : audit [INF] from='mgr.15228 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T19:12:26.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:26 smithi092 bash[27556]: audit 2023-09-23T19:12:25.551930+0000 mon.a (mon.0) 880 : audit [INF] from='mgr.15228 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-09-23T19:12:26.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:26 smithi092 bash[27556]: audit 2023-09-23T19:12:25.555998+0000 mon.c (mon.2) 266 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T19:12:26.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:26 smithi092 bash[27556]: audit 2023-09-23T19:12:25.557402+0000 mon.c (mon.2) 267 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-09-23T19:12:26.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:26 smithi092 bash[27556]: audit 2023-09-23T19:12:25.977437+0000 mon.a (mon.0) 881 : audit [INF] from='client.? 172.21.15.16:0/2552494186' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T19:12:26.590 INFO:teuthology.orchestra.run.smithi016.stderr:pool 'unique_pool_0' removed 2023-09-23T19:12:27.215 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:27 smithi016 bash[33624]: cephadm 2023-09-23T19:12:25.529324+0000 mgr.y (mgr.15228) 45 : cephadm [INF] Upgrade: Setting container_image for all mgr 2023-09-23T19:12:27.215 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:27 smithi016 bash[33624]: cephadm 2023-09-23T19:12:25.558592+0000 mgr.y (mgr.15228) 46 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2023-09-23T19:12:27.215 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:27 smithi016 bash[33624]: audit 2023-09-23T19:12:26.110449+0000 mon.c (mon.2) 268 : audit [DBG] from='client.? 172.21.15.16:0/1706047482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:12:27.215 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:27 smithi016 bash[33624]: audit 2023-09-23T19:12:26.559107+0000 mon.a (mon.0) 882 : audit [INF] from='client.? 172.21.15.16:0/2552494186' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T19:12:27.215 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:27 smithi016 bash[33624]: cluster 2023-09-23T19:12:26.559224+0000 mon.a (mon.0) 883 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2023-09-23T19:12:27.215 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:27 smithi016 bash[27237]: cephadm 2023-09-23T19:12:25.529324+0000 mgr.y (mgr.15228) 45 : cephadm [INF] Upgrade: Setting container_image for all mgr 2023-09-23T19:12:27.215 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:27 smithi016 bash[27237]: cephadm 2023-09-23T19:12:25.558592+0000 mgr.y (mgr.15228) 46 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2023-09-23T19:12:27.215 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:27 smithi016 bash[27237]: audit 2023-09-23T19:12:26.110449+0000 mon.c (mon.2) 268 : audit [DBG] from='client.? 172.21.15.16:0/1706047482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:12:27.215 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:27 smithi016 bash[27237]: audit 2023-09-23T19:12:26.559107+0000 mon.a (mon.0) 882 : audit [INF] from='client.? 172.21.15.16:0/2552494186' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T19:12:27.216 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:27 smithi016 bash[27237]: cluster 2023-09-23T19:12:26.559224+0000 mon.a (mon.0) 883 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2023-09-23T19:12:27.535 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:27 smithi092 bash[27556]: cephadm 2023-09-23T19:12:25.529324+0000 mgr.y (mgr.15228) 45 : cephadm [INF] Upgrade: Setting container_image for all mgr 2023-09-23T19:12:27.535 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:27 smithi092 bash[27556]: cephadm 2023-09-23T19:12:25.558592+0000 mgr.y (mgr.15228) 46 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2023-09-23T19:12:27.535 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:27 smithi092 bash[27556]: audit 2023-09-23T19:12:26.110449+0000 mon.c (mon.2) 268 : audit [DBG] from='client.? 172.21.15.16:0/1706047482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:12:27.535 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:27 smithi092 bash[27556]: audit 2023-09-23T19:12:26.559107+0000 mon.a (mon.0) 882 : audit [INF] from='client.? 172.21.15.16:0/2552494186' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T19:12:27.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:27 smithi092 bash[27556]: cluster 2023-09-23T19:12:26.559224+0000 mon.a (mon.0) 883 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2023-09-23T19:12:27.830 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T19:12:27.830 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T19:12:27.831 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_1 2023-09-23T19:12:27.831 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_1 16 2023-09-23T19:12:28.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:28 smithi016 bash[33624]: cluster 2023-09-23T19:12:26.284338+0000 mgr.y (mgr.15228) 47 : cluster [DBG] pgmap v30: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 113 op/s 2023-09-23T19:12:28.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:28 smithi016 bash[27237]: cluster 2023-09-23T19:12:26.284338+0000 mgr.y (mgr.15228) 47 : cluster [DBG] pgmap v30: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 113 op/s 2023-09-23T19:12:28.535 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:28 smithi092 bash[27556]: cluster 2023-09-23T19:12:26.284338+0000 mgr.y (mgr.15228) 47 : cluster [DBG] pgmap v30: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 113 op/s 2023-09-23T19:12:29.035 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:12:28 smithi092 bash[54653]: ts=2023-09-23T19:12:28.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi092.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi016:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T19:12:29.737 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:29 smithi016 bash[33624]: cluster 2023-09-23T19:12:28.284894+0000 mgr.y (mgr.15228) 48 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:12:29.737 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:29 smithi016 bash[33624]: cephadm 2023-09-23T19:12:28.428916+0000 mgr.y (mgr.15228) 49 : cephadm [INF] Upgrade: Updating mon.a 2023-09-23T19:12:29.738 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:29 smithi016 bash[33624]: audit 2023-09-23T19:12:28.438281+0000 mon.a (mon.0) 884 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:12:29.738 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:29 smithi016 bash[33624]: audit 2023-09-23T19:12:28.439774+0000 mon.c (mon.2) 269 : audit [INF] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T19:12:29.738 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:29 smithi016 bash[33624]: audit 2023-09-23T19:12:28.440815+0000 mon.c (mon.2) 270 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T19:12:29.738 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:29 smithi016 bash[33624]: audit 2023-09-23T19:12:28.441698+0000 mon.c (mon.2) 271 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:12:29.738 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:29 smithi016 bash[33624]: cephadm 2023-09-23T19:12:28.442853+0000 mgr.y (mgr.15228) 50 : cephadm [INF] Deploying daemon mon.a on smithi016 2023-09-23T19:12:29.738 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:29 smithi016 bash[33624]: audit 2023-09-23T19:12:28.567831+0000 mon.c (mon.2) 272 : audit [DBG] from='client.? 172.21.15.16:0/581412619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:12:29.738 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:29 smithi016 bash[27237]: cluster 2023-09-23T19:12:28.284894+0000 mgr.y (mgr.15228) 48 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:12:29.738 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:29 smithi016 bash[27237]: cephadm 2023-09-23T19:12:28.428916+0000 mgr.y (mgr.15228) 49 : cephadm [INF] Upgrade: Updating mon.a 2023-09-23T19:12:29.739 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:29 smithi016 bash[27237]: audit 2023-09-23T19:12:28.438281+0000 mon.a (mon.0) 884 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:12:29.739 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:29 smithi016 bash[27237]: audit 2023-09-23T19:12:28.439774+0000 mon.c (mon.2) 269 : audit [INF] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T19:12:29.739 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:29 smithi016 bash[27237]: audit 2023-09-23T19:12:28.440815+0000 mon.c (mon.2) 270 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T19:12:29.739 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:29 smithi016 bash[27237]: audit 2023-09-23T19:12:28.441698+0000 mon.c (mon.2) 271 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:12:29.739 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:29 smithi016 bash[27237]: cephadm 2023-09-23T19:12:28.442853+0000 mgr.y (mgr.15228) 50 : cephadm [INF] Deploying daemon mon.a on smithi016 2023-09-23T19:12:29.739 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:29 smithi016 bash[27237]: audit 2023-09-23T19:12:28.567831+0000 mon.c (mon.2) 272 : audit [DBG] from='client.? 172.21.15.16:0/581412619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:12:29.785 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:29 smithi092 bash[27556]: cluster 2023-09-23T19:12:28.284894+0000 mgr.y (mgr.15228) 48 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:12:29.785 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:29 smithi092 bash[27556]: cephadm 2023-09-23T19:12:28.428916+0000 mgr.y (mgr.15228) 49 : cephadm [INF] Upgrade: Updating mon.a 2023-09-23T19:12:29.785 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:29 smithi092 bash[27556]: audit 2023-09-23T19:12:28.438281+0000 mon.a (mon.0) 884 : audit [INF] from='mgr.15228 ' entity='mgr.y' 2023-09-23T19:12:29.785 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:29 smithi092 bash[27556]: audit 2023-09-23T19:12:28.439774+0000 mon.c (mon.2) 269 : audit [INF] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T19:12:29.786 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:29 smithi092 bash[27556]: audit 2023-09-23T19:12:28.440815+0000 mon.c (mon.2) 270 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T19:12:29.786 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:29 smithi092 bash[27556]: audit 2023-09-23T19:12:28.441698+0000 mon.c (mon.2) 271 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:12:29.786 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:29 smithi092 bash[27556]: cephadm 2023-09-23T19:12:28.442853+0000 mgr.y (mgr.15228) 50 : cephadm [INF] Deploying daemon mon.a on smithi016 2023-09-23T19:12:29.786 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:29 smithi092 bash[27556]: audit 2023-09-23T19:12:28.567831+0000 mon.c (mon.2) 272 : audit [DBG] from='client.? 172.21.15.16:0/581412619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:12:31.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:30 smithi016 systemd[1]: Stopping Ceph mon.a for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7... 2023-09-23T19:12:31.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:30 smithi016 bash[27237]: debug 2023-09-23T19:12:30.930+0000 7f3882a91700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-09-23T19:12:31.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:30 smithi016 bash[27237]: debug 2023-09-23T19:12:30.930+0000 7f3882a91700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2023-09-23T19:12:32.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:31 smithi016 bash[74864]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-mon-a 2023-09-23T19:12:32.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:31 smithi016 bash[74947]: Error response from daemon: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-mon-a 2023-09-23T19:12:32.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:31 smithi016 systemd[1]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@mon.a.service: Succeeded. 2023-09-23T19:12:32.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:31 smithi016 systemd[1]: Stopped Ceph mon.a for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7. 2023-09-23T19:12:32.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:32 smithi016 systemd[1]: Started Ceph mon.a for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7. 2023-09-23T19:12:32.640 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:12:34.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.806+0000 7f091645fc80 0 set uid:gid to 167:167 (ceph:ceph) 2023-09-23T19:12:34.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.806+0000 7f091645fc80 0 ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable), process ceph-mon, pid 7 2023-09-23T19:12:34.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.806+0000 7f091645fc80 0 pidfile_write: ignore empty --pid-file 2023-09-23T19:12:34.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.810+0000 7f091645fc80 0 load: jerasure load: lrc 2023-09-23T19:12:34.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: RocksDB version: 7.9.2 2023-09-23T19:12:34.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Git sha 0 2023-09-23T19:12:34.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Compile date 2023-09-22 15:19:43 2023-09-23T19:12:34.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: DB SUMMARY 2023-09-23T19:12:34.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: DB Session ID: DH1UWJSROTV048HTYD6W 2023-09-23T19:12:34.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: CURRENT file: CURRENT 2023-09-23T19:12:34.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: IDENTITY file: IDENTITY 2023-09-23T19:12:34.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: MANIFEST file: MANIFEST-000008 size: 958 Bytes 2023-09-23T19:12:34.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000023.sst 2023-09-23T19:12:34.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000021.log size: 16155326 ; 2023-09-23T19:12:34.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.error_if_exists: 0 2023-09-23T19:12:34.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.create_if_missing: 0 2023-09-23T19:12:34.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.paranoid_checks: 1 2023-09-23T19:12:34.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.flush_verify_memtable_count: 1 2023-09-23T19:12:34.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-23T19:12:34.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-09-23T19:12:34.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.env: 0x559e0e7fc720 2023-09-23T19:12:34.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.fs: PosixFileSystem 2023-09-23T19:12:34.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.info_log: 0x559e1054d400 2023-09-23T19:12:34.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.max_file_opening_threads: 16 2023-09-23T19:12:34.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.statistics: (nil) 2023-09-23T19:12:34.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.use_fsync: 0 2023-09-23T19:12:34.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.max_log_file_size: 0 2023-09-23T19:12:34.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T19:12:34.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T19:12:34.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.keep_log_file_num: 1000 2023-09-23T19:12:34.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.recycle_log_file_num: 0 2023-09-23T19:12:34.039 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.allow_fallocate: 1 2023-09-23T19:12:34.039 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.allow_mmap_reads: 0 2023-09-23T19:12:34.039 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.allow_mmap_writes: 0 2023-09-23T19:12:34.039 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.use_direct_reads: 0 2023-09-23T19:12:34.039 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T19:12:34.039 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.create_missing_column_families: 0 2023-09-23T19:12:34.039 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.db_log_dir: 2023-09-23T19:12:34.039 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.wal_dir: 2023-09-23T19:12:34.039 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T19:12:34.039 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T19:12:34.039 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T19:12:34.040 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T19:12:34.040 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T19:12:34.040 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T19:12:34.040 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.advise_random_on_open: 1 2023-09-23T19:12:34.040 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.db_write_buffer_size: 0 2023-09-23T19:12:34.040 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.write_buffer_manager: 0x559e0f7da5a0 2023-09-23T19:12:34.040 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T19:12:34.040 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T19:12:34.040 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T19:12:34.040 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.rate_limiter: (nil) 2023-09-23T19:12:34.040 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T19:12:34.041 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.wal_recovery_mode: 2 2023-09-23T19:12:34.041 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.enable_thread_tracking: 0 2023-09-23T19:12:34.041 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.enable_pipelined_write: 0 2023-09-23T19:12:34.041 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.unordered_write: 0 2023-09-23T19:12:34.041 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T19:12:34.041 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T19:12:34.041 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T19:12:34.041 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T19:12:34.041 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.row_cache: None 2023-09-23T19:12:34.041 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.wal_filter: None 2023-09-23T19:12:34.041 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T19:12:34.041 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.allow_ingest_behind: 0 2023-09-23T19:12:34.041 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.two_write_queues: 0 2023-09-23T19:12:34.042 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.manual_wal_flush: 0 2023-09-23T19:12:34.042 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.wal_compression: 0 2023-09-23T19:12:34.042 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.atomic_flush: 0 2023-09-23T19:12:34.042 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T19:12:34.042 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T19:12:34.042 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T19:12:34.042 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.log_readahead_size: 0 2023-09-23T19:12:34.042 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-23T19:12:34.042 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.best_efforts_recovery: 0 2023-09-23T19:12:34.042 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-23T19:12:34.042 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-23T19:12:34.042 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.allow_data_in_errors: 0 2023-09-23T19:12:34.043 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.db_host_id: __hostname__ 2023-09-23T19:12:34.043 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.enforce_single_del_contracts: true 2023-09-23T19:12:34.043 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.max_background_jobs: 2 2023-09-23T19:12:34.043 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.max_background_compactions: -1 2023-09-23T19:12:34.043 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.max_subcompactions: 1 2023-09-23T19:12:34.043 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T19:12:34.043 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T19:12:34.043 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T19:12:34.043 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.max_total_wal_size: 0 2023-09-23T19:12:34.043 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T19:12:34.043 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T19:12:34.044 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T19:12:34.044 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T19:12:34.044 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.max_open_files: -1 2023-09-23T19:12:34.044 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.bytes_per_sync: 0 2023-09-23T19:12:34.044 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T19:12:34.044 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T19:12:34.044 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.compaction_readahead_size: 0 2023-09-23T19:12:34.044 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.max_background_flushes: -1 2023-09-23T19:12:34.044 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Compression algorithms supported: 2023-09-23T19:12:34.044 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T19:12:34.044 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: kZSTD supported: 0 2023-09-23T19:12:34.044 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: kXpressCompression supported: 0 2023-09-23T19:12:34.045 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: kLZ4HCCompression supported: 1 2023-09-23T19:12:34.045 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: kZlibCompression supported: 1 2023-09-23T19:12:34.045 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: kSnappyCompression supported: 1 2023-09-23T19:12:34.045 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: kLZ4Compression supported: 1 2023-09-23T19:12:34.045 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: kBZip2Compression supported: 0 2023-09-23T19:12:34.045 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T19:12:34.045 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: DMutex implementation: pthread_mutex_t 2023-09-23T19:12:34.045 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000008 2023-09-23T19:12:34.045 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-09-23T19:12:34.045 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T19:12:34.045 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.merge_operator: 2023-09-23T19:12:34.045 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.compaction_filter: None 2023-09-23T19:12:34.046 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.compaction_filter_factory: None 2023-09-23T19:12:34.046 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.sst_partitioner_factory: None 2023-09-23T19:12:34.046 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T19:12:34.046 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T19:12:34.046 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x559e1054d520) 2023-09-23T19:12:34.046 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: cache_index_and_filter_blocks: 1 2023-09-23T19:12:34.046 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T19:12:34.046 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T19:12:34.047 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: pin_top_level_index_and_filter: 1 2023-09-23T19:12:34.047 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: index_type: 0 2023-09-23T19:12:34.047 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: data_block_index_type: 0 2023-09-23T19:12:34.047 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: index_shortening: 1 2023-09-23T19:12:34.047 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: data_block_hash_table_util_ratio: 0.750000 2023-09-23T19:12:34.047 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: checksum: 4 2023-09-23T19:12:34.047 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: no_block_cache: 0 2023-09-23T19:12:34.047 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: block_cache: 0x559e0f85d1f0 2023-09-23T19:12:34.047 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: block_cache_name: BinnedLRUCache 2023-09-23T19:12:34.047 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: block_cache_options: 2023-09-23T19:12:34.047 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: capacity : 536870912 2023-09-23T19:12:34.048 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: num_shard_bits : 4 2023-09-23T19:12:34.048 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: strict_capacity_limit : 0 2023-09-23T19:12:34.048 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: high_pri_pool_ratio: 0.000 2023-09-23T19:12:34.048 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: block_cache_compressed: (nil) 2023-09-23T19:12:34.048 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: persistent_cache: (nil) 2023-09-23T19:12:34.048 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: block_size: 4096 2023-09-23T19:12:34.048 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: block_size_deviation: 10 2023-09-23T19:12:34.048 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: block_restart_interval: 16 2023-09-23T19:12:34.049 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: index_block_restart_interval: 1 2023-09-23T19:12:34.049 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: metadata_block_size: 4096 2023-09-23T19:12:34.049 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: partition_filters: 0 2023-09-23T19:12:34.049 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: use_delta_encoding: 1 2023-09-23T19:12:34.049 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: filter_policy: bloomfilter 2023-09-23T19:12:34.049 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: whole_key_filtering: 1 2023-09-23T19:12:34.049 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: verify_compression: 0 2023-09-23T19:12:34.049 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: read_amp_bytes_per_bit: 0 2023-09-23T19:12:34.049 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: format_version: 5 2023-09-23T19:12:34.049 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: enable_index_compression: 1 2023-09-23T19:12:34.049 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: block_align: 0 2023-09-23T19:12:34.049 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: max_auto_readahead_size: 262144 2023-09-23T19:12:34.049 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: prepopulate_block_cache: 0 2023-09-23T19:12:34.050 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: initial_auto_readahead_size: 8192 2023-09-23T19:12:34.050 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: num_file_reads_for_auto_readahead: 2 2023-09-23T19:12:34.050 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.write_buffer_size: 33554432 2023-09-23T19:12:34.050 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.max_write_buffer_number: 2 2023-09-23T19:12:34.050 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.compression: NoCompression 2023-09-23T19:12:34.050 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.bottommost_compression: Disabled 2023-09-23T19:12:34.050 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.prefix_extractor: nullptr 2023-09-23T19:12:34.050 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T19:12:34.051 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.num_levels: 7 2023-09-23T19:12:34.051 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T19:12:34.051 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T19:12:34.051 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T19:12:34.051 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T19:12:34.051 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T19:12:34.051 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T19:12:34.051 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T19:12:34.051 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T19:12:34.051 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-23T19:12:34.052 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T19:12:34.052 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-09-23T19:12:34.052 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-09-23T19:12:34.052 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T19:12:34.052 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.compression_opts.level: 32767 2023-09-23T19:12:34.052 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.compression_opts.strategy: 0 2023-09-23T19:12:34.052 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T19:12:34.052 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T19:12:34.052 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-09-23T19:12:34.052 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-23T19:12:34.052 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.compression_opts.enabled: false 2023-09-23T19:12:34.052 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-09-23T19:12:34.052 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T19:12:34.053 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T19:12:34.053 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T19:12:34.053 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.target_file_size_base: 67108864 2023-09-23T19:12:34.053 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T19:12:34.053 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T19:12:34.053 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T19:12:34.053 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T19:12:34.053 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T19:12:34.053 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T19:12:34.053 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T19:12:34.053 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T19:12:34.053 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T19:12:34.054 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T19:12:34.054 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T19:12:34.054 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T19:12:34.054 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T19:12:34.054 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-09-23T19:12:34.054 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.arena_block_size: 1048576 2023-09-23T19:12:34.054 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T19:12:34.054 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T19:12:34.054 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.disable_auto_compactions: 0 2023-09-23T19:12:34.054 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T19:12:34.054 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T19:12:34.055 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T19:12:34.055 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T19:12:34.055 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T19:12:34.056 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T19:12:34.056 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T19:12:34.056 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T19:12:34.056 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T19:12:34.057 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T19:12:34.057 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.table_properties_collectors: 2023-09-23T19:12:34.057 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.inplace_update_support: 0 2023-09-23T19:12:34.057 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T19:12:34.057 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T19:12:34.057 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T19:12:34.057 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T19:12:34.057 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.bloom_locality: 0 2023-09-23T19:12:34.057 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.max_successive_merges: 0 2023-09-23T19:12:34.057 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T19:12:34.058 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.paranoid_file_checks: 0 2023-09-23T19:12:34.058 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.force_consistency_checks: 1 2023-09-23T19:12:34.058 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.report_bg_io_stats: 0 2023-09-23T19:12:34.058 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.ttl: 2592000 2023-09-23T19:12:34.058 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T19:12:34.058 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2023-09-23T19:12:34.058 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.preserve_internal_time_seconds: 0 2023-09-23T19:12:34.058 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.enable_blob_files: false 2023-09-23T19:12:34.058 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.min_blob_size: 0 2023-09-23T19:12:34.059 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.blob_file_size: 268435456 2023-09-23T19:12:34.059 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.blob_compression_type: NoCompression 2023-09-23T19:12:34.059 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.enable_blob_garbage_collection: false 2023-09-23T19:12:34.059 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-23T19:12:34.059 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-09-23T19:12:34.059 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.blob_compaction_readahead_size: 0 2023-09-23T19:12:34.059 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.blob_file_starting_level: 0 2023-09-23T19:12:34.059 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-09-23T19:12:34.059 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f090bc7e700 3 rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 23.sst 2023-09-23T19:12:34.060 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T19:12:34.060 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000008 succeeded,manifest_file_number is 8, next_file_number is 25, last_sequence is 11342, log_number is 21,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T19:12:34.060 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 21 2023-09-23T19:12:34.060 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: b1a56064-2999-4cbd-a65a-0e2acd8b9b81 2023-09-23T19:12:34.060 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1695496353817933, "job": 1, "event": "recovery_started", "wal_files": [21]} 2023-09-23T19:12:34.060 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.814+0000 7f091645fc80 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #21 mode 2 2023-09-23T19:12:34.060 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.858+0000 7f091645fc80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1695496353861531, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 26, "file_size": 15656118, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 11343, "largest_seqno": 15091, "table_properties": {"data_size": 15634522, "index_size": 17182, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 3397, "raw_key_size": 25927, "raw_average_key_size": 19, "raw_value_size": 15598492, "raw_average_value_size": 11563, "num_data_blocks": 748, "num_entries": 1349, "num_filter_entries": 1349, "num_deletions": 253, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1695496353, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "b1a56064-2999-4cbd-a65a-0e2acd8b9b81", "db_session_id": "DH1UWJSROTV048HTYD6W", "orig_file_number": 26, "seqno_to_time_mapping": "N/A"}} 2023-09-23T19:12:34.060 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.858+0000 7f091645fc80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1695496353861657, "job": 1, "event": "recovery_finished"} 2023-09-23T19:12:34.060 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.858+0000 7f091645fc80 4 rocksdb: [db/version_set.cc:5047] Creating manifest 28 2023-09-23T19:12:34.061 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.858+0000 7f091645fc80 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T19:12:34.061 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.862+0000 7f091645fc80 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000021.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-23T19:12:34.061 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.862+0000 7f091645fc80 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x559e0f8fe000 2023-09-23T19:12:34.061 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.862+0000 7f091645fc80 4 rocksdb: DB pointer 0x559e0f8ec000 2023-09-23T19:12:34.061 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.862+0000 7f0904c70700 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-09-23T19:12:34.061 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.862+0000 7f0904c70700 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2023-09-23T19:12:34.061 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: ** DB Stats ** 2023-09-23T19:12:34.061 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T19:12:34.061 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T19:12:34.062 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T19:12:34.062 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T19:12:34.062 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T19:12:34.062 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T19:12:34.062 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T19:12:34.062 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: ** Compaction Stats [default] ** 2023-09-23T19:12:34.062 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T19:12:34.062 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-09-23T19:12:34.062 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: L0 1/0 14.93 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 734.9 0.02 0.00 1 0.020 0 0 0.0 0.0 2023-09-23T19:12:34.062 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: L6 1/0 22.52 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-09-23T19:12:34.062 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: Sum 2/0 37.45 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 734.9 0.02 0.00 1 0.020 0 0 0.0 0.0 2023-09-23T19:12:34.063 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 734.9 0.02 0.00 1 0.020 0 0 0.0 0.0 2023-09-23T19:12:34.063 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: ** Compaction Stats [default] ** 2023-09-23T19:12:34.063 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T19:12:34.063 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T19:12:34.063 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 734.9 0.02 0.00 1 0.020 0 0 0.0 0.0 2023-09-23T19:12:34.063 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-09-23T19:12:34.063 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T19:12:34.063 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: Flush(GB): cumulative 0.015, interval 0.015 2023-09-23T19:12:34.063 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T19:12:34.063 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T19:12:34.064 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T19:12:34.064 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: AddFile(Keys): cumulative 0, interval 0 2023-09-23T19:12:34.064 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: Cumulative compaction: 0.01 GB write, 310.10 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T19:12:34.064 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: Interval compaction: 0.01 GB write, 310.10 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T19:12:34.064 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T19:12:34.064 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: Block cache BinnedLRUCache@0x559e0f85d1f0#7 capacity: 512.00 MB usage: 24.11 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 2.5e-05 secs_since: 0 2023-09-23T19:12:34.064 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: Block cache entry stats(count,size,portion): FilterBlock(1,4.03 KB,0.0007689%) IndexBlock(1,20.08 KB,0.0038296%) Misc(1,0.00 KB,0%) 2023-09-23T19:12:34.064 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: ** File Read Latency Histogram By Level [default] ** 2023-09-23T19:12:34.064 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.862+0000 7f091645fc80 0 starting mon.a rank 0 at public addrs [v2:172.21.15.16:3300/0,v1:172.21.15.16:6789/0] at bind addrs [v2:172.21.15.16:3300/0,v1:172.21.15.16:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 2023-09-23T19:12:34.064 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.862+0000 7f091645fc80 1 mon.a@-1(???) e3 preinit fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 2023-09-23T19:12:34.065 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.862+0000 7f091645fc80 0 mon.a@-1(???).mds e6 new map 2023-09-23T19:12:34.065 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.862+0000 7f091645fc80 0 mon.a@-1(???).mds e6 print_map 2023-09-23T19:12:34.065 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: e6 2023-09-23T19:12:34.065 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T19:12:34.065 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T19:12:34.065 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: legacy client fscid: 1 2023-09-23T19:12:34.065 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: 2023-09-23T19:12:34.065 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: Filesystem 'foo' (1) 2023-09-23T19:12:34.065 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: fs_name foo 2023-09-23T19:12:34.065 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: epoch 5 2023-09-23T19:12:34.065 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: flags 12 joinable allow_snaps allow_multimds_snaps 2023-09-23T19:12:34.065 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: created 2023-09-23T19:05:52.117022+0000 2023-09-23T19:12:34.065 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: modified 2023-09-23T19:05:58.952970+0000 2023-09-23T19:12:34.066 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: tableserver 0 2023-09-23T19:12:34.066 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: root 0 2023-09-23T19:12:34.066 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: session_timeout 60 2023-09-23T19:12:34.066 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: session_autoclose 300 2023-09-23T19:12:34.066 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: max_file_size 1099511627776 2023-09-23T19:12:34.066 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: required_client_features {} 2023-09-23T19:12:34.066 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: last_failure 0 2023-09-23T19:12:34.066 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: last_failure_osd_epoch 0 2023-09-23T19:12:34.066 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T19:12:34.066 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: max_mds 1 2023-09-23T19:12:34.066 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: in 0 2023-09-23T19:12:34.066 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: up {0=14655} 2023-09-23T19:12:34.067 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: failed 2023-09-23T19:12:34.067 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: damaged 2023-09-23T19:12:34.067 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: stopped 2023-09-23T19:12:34.067 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: data_pools [4] 2023-09-23T19:12:34.067 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: metadata_pool 3 2023-09-23T19:12:34.067 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: inline_data disabled 2023-09-23T19:12:34.067 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: balancer 2023-09-23T19:12:34.067 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: bal_rank_mask -1 2023-09-23T19:12:34.067 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: standby_count_wanted 1 2023-09-23T19:12:34.067 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: [mds.foo.smithi016.uncssv{0:14655} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.16:6834/161572387,v1:172.21.15.16:6835/161572387] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T19:12:34.067 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: 2023-09-23T19:12:34.067 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: 2023-09-23T19:12:34.067 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: Standby daemons: 2023-09-23T19:12:34.068 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: 2023-09-23T19:12:34.068 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: [mds.foo.smithi092.vbpqks{-1:24520} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.92:6832/1118427195,v1:172.21.15.92:6833/1118427195] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T19:12:34.068 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.862+0000 7f091645fc80 0 mon.a@-1(???).osd e71 crush map has features 3314933000854323200, adjusting msgr requires 2023-09-23T19:12:34.068 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.862+0000 7f091645fc80 0 mon.a@-1(???).osd e71 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T19:12:34.068 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.862+0000 7f091645fc80 0 mon.a@-1(???).osd e71 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T19:12:34.068 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.862+0000 7f091645fc80 0 mon.a@-1(???).osd e71 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T19:12:34.068 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:12:33 smithi016 bash[75004]: debug 2023-09-23T19:12:33.862+0000 7f091645fc80 1 mon.a@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2023-09-23T19:12:35.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:12:34 smithi016 bash[70992]: ignoring --setuser ceph since I am not root 2023-09-23T19:12:35.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:12:34 smithi016 bash[70992]: ignoring --setgroup ceph since I am not root 2023-09-23T19:12:35.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:12:34 smithi016 bash[70992]: debug 2023-09-23T19:12:34.590+0000 7fb2e3d91700 1 -- 172.21.15.16:0/3194101812 <== mon.2 v2:172.21.15.16:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x559f0782f1e0 con 0x559f085aec00 2023-09-23T19:12:35.035 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:12:34 smithi016 bash[70992]: debug 2023-09-23T19:12:34.754+0000 7fb2edac0200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T19:12:35.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:34 smithi016 bash[33624]: cluster 2023-09-23T19:12:30.286048+0000 mgr.y (mgr.15228) 51 : cluster [DBG] pgmap v33: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:12:35.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:34 smithi016 bash[33624]: audit 2023-09-23T19:12:31.056556+0000 mon.b (mon.1) 50 : audit [DBG] from='client.15429 172.21.15.16:0/2902047969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:12:35.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:34 smithi016 bash[33624]: cluster 2023-09-23T19:12:32.286676+0000 mgr.y (mgr.15228) 52 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:12:35.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:34 smithi016 bash[33624]: audit 2023-09-23T19:12:32.458802+0000 mon.c (mon.2) 273 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:12:35.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:34 smithi016 bash[33624]: audit 2023-09-23T19:12:33.506398+0000 mon.c (mon.2) 274 : audit [DBG] from='client.? 172.21.15.16:0/3242850589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:12:35.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:34 smithi016 bash[33624]: cluster 2023-09-23T19:12:34.078140+0000 mon.a (mon.0) 1 : cluster [INF] mon.a calling monitor election 2023-09-23T19:12:35.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:34 smithi016 bash[33624]: cluster 2023-09-23T19:12:34.483735+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T19:12:35.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:34 smithi016 bash[33624]: cluster 2023-09-23T19:12:34.496437+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.16:3300/0,v1:172.21.15.16:6789/0],b=[v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0],c=[v2:172.21.15.16:3301/0,v1:172.21.15.16:6790/0]} removed_ranks: {} 2023-09-23T19:12:35.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:34 smithi016 bash[33624]: cluster 2023-09-23T19:12:34.497060+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap foo:1 {0=foo.smithi016.uncssv=up:active} 1 up:standby 2023-09-23T19:12:35.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:34 smithi016 bash[33624]: cluster 2023-09-23T19:12:34.497092+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2023-09-23T19:12:35.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:34 smithi016 bash[33624]: cluster 2023-09-23T19:12:34.498102+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e32: y(active, since 62s), standbys: x 2023-09-23T19:12:35.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:34 smithi016 bash[33624]: cluster 2023-09-23T19:12:34.525947+0000 mon.a (mon.0) 7 : cluster [DBG] mgrmap e33: y(active, since 62s), standbys: x 2023-09-23T19:12:35.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:34 smithi016 bash[33624]: cluster 2023-09-23T19:12:34.526011+0000 mon.a (mon.0) 8 : cluster [INF] overall HEALTH_OK 2023-09-23T19:12:35.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:34 smithi016 bash[33624]: audit 2023-09-23T19:12:34.526146+0000 mon.a (mon.0) 9 : audit [INF] from='mgr.15228 ' entity='' 2023-09-23T19:12:35.037 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:34 smithi092 bash[55056]: ignoring --setuser ceph since I am not root 2023-09-23T19:12:35.037 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:34 smithi092 bash[55056]: ignoring --setgroup ceph since I am not root 2023-09-23T19:12:35.037 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:34 smithi092 bash[55056]: debug 2023-09-23T19:12:34.596+0000 7f1456f1d700 1 -- 172.21.15.92:0/3095046319 <== mon.1 v2:172.21.15.92:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55b085e211e0 con 0x55b086ba0800 2023-09-23T19:12:35.037 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:34 smithi092 bash[55056]: debug 2023-09-23T19:12:34.756+0000 7f1460c4c200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T19:12:35.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:34 smithi092 bash[27556]: cluster 2023-09-23T19:12:30.286048+0000 mgr.y (mgr.15228) 51 : cluster [DBG] pgmap v33: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:12:35.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:34 smithi092 bash[27556]: audit 2023-09-23T19:12:31.056556+0000 mon.b (mon.1) 50 : audit [DBG] from='client.15429 172.21.15.16:0/2902047969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:12:35.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:34 smithi092 bash[27556]: cluster 2023-09-23T19:12:32.286676+0000 mgr.y (mgr.15228) 52 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:12:35.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:34 smithi092 bash[27556]: audit 2023-09-23T19:12:32.458802+0000 mon.c (mon.2) 273 : audit [DBG] from='mgr.15228 172.21.15.16:0/122523777' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:12:35.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:34 smithi092 bash[27556]: audit 2023-09-23T19:12:33.506398+0000 mon.c (mon.2) 274 : audit [DBG] from='client.? 172.21.15.16:0/3242850589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:12:35.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:34 smithi092 bash[27556]: cluster 2023-09-23T19:12:34.078140+0000 mon.a (mon.0) 1 : cluster [INF] mon.a calling monitor election 2023-09-23T19:12:35.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:34 smithi092 bash[27556]: cluster 2023-09-23T19:12:34.483735+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T19:12:35.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:34 smithi092 bash[27556]: cluster 2023-09-23T19:12:34.496437+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.16:3300/0,v1:172.21.15.16:6789/0],b=[v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0],c=[v2:172.21.15.16:3301/0,v1:172.21.15.16:6790/0]} removed_ranks: {} 2023-09-23T19:12:35.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:34 smithi092 bash[27556]: cluster 2023-09-23T19:12:34.497060+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap foo:1 {0=foo.smithi016.uncssv=up:active} 1 up:standby 2023-09-23T19:12:35.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:34 smithi092 bash[27556]: cluster 2023-09-23T19:12:34.497092+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2023-09-23T19:12:35.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:34 smithi092 bash[27556]: cluster 2023-09-23T19:12:34.498102+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e32: y(active, since 62s), standbys: x 2023-09-23T19:12:35.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:34 smithi092 bash[27556]: cluster 2023-09-23T19:12:34.525947+0000 mon.a (mon.0) 7 : cluster [DBG] mgrmap e33: y(active, since 62s), standbys: x 2023-09-23T19:12:35.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:34 smithi092 bash[27556]: cluster 2023-09-23T19:12:34.526011+0000 mon.a (mon.0) 8 : cluster [INF] overall HEALTH_OK 2023-09-23T19:12:35.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:34 smithi092 bash[27556]: audit 2023-09-23T19:12:34.526146+0000 mon.a (mon.0) 9 : audit [INF] from='mgr.15228 ' entity='' 2023-09-23T19:12:35.951 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:12:35 smithi016 bash[70992]: debug 2023-09-23T19:12:35.718+0000 7fb2edac0200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T19:12:35.951 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:12:35 smithi016 bash[70992]: debug 2023-09-23T19:12:35.826+0000 7fb2edac0200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T19:12:35.986 INFO:teuthology.orchestra.run.smithi016.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 1 2023-09-23T19:12:36.025 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:35 smithi092 bash[55056]: debug 2023-09-23T19:12:35.676+0000 7f1460c4c200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T19:12:36.025 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:35 smithi092 bash[55056]: debug 2023-09-23T19:12:35.780+0000 7f1460c4c200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T19:12:36.026 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:35 smithi092 bash[55056]: debug 2023-09-23T19:12:35.880+0000 7f1460c4c200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T19:12:36.211 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:36 smithi016 bash[33624]: audit 2023-09-23T19:12:35.969709+0000 mon.c (mon.2) 275 : audit [DBG] from='client.? 172.21.15.16:0/1418665878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:12:36.212 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:12:35 smithi016 bash[70992]: debug 2023-09-23T19:12:35.946+0000 7fb2edac0200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T19:12:36.285 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:36 smithi092 bash[55056]: debug 2023-09-23T19:12:36.148+0000 7f1460c4c200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T19:12:36.285 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:36 smithi092 bash[27556]: audit 2023-09-23T19:12:35.969709+0000 mon.c (mon.2) 275 : audit [DBG] from='client.? 172.21.15.16:0/1418665878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:12:36.534 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:12:36 smithi016 bash[70992]: debug 2023-09-23T19:12:36.230+0000 7fb2edac0200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T19:12:36.744 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T19:12:36.785 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:36 smithi092 bash[55056]: debug 2023-09-23T19:12:36.508+0000 7f1460c4c200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T19:12:36.785 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:36 smithi092 bash[55056]: debug 2023-09-23T19:12:36.664+0000 7f1460c4c200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T19:12:36.930 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:12:36 smithi016 bash[70992]: debug 2023-09-23T19:12:36.598+0000 7fb2edac0200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T19:12:36.931 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:12:36 smithi016 bash[70992]: debug 2023-09-23T19:12:36.770+0000 7fb2edac0200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T19:12:37.049 INFO:teuthology.orchestra.run.smithi016.stderr:pool 'unique_pool_1' created 2023-09-23T19:12:37.284 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:12:37 smithi016 bash[70992]: debug 2023-09-23T19:12:37.006+0000 7fb2edac0200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T19:12:37.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:37 smithi016 bash[33624]: audit 2023-09-23T19:12:36.096667+0000 mon.a (mon.0) 10 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-09-23T19:12:37.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:37 smithi016 bash[33624]: debug 2023-09-23T19:12:37.026+0000 7f954298b700 -1 mon.c@2(peon).osd e72 update_from_paxos full map CRC mismatch, resetting to canonical 2023-09-23T19:12:37.285 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:36 smithi092 bash[55056]: debug 2023-09-23T19:12:36.880+0000 7f1460c4c200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T19:12:37.285 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:37 smithi092 bash[27556]: audit 2023-09-23T19:12:36.096667+0000 mon.a (mon.0) 10 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-09-23T19:12:37.285 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:37 smithi092 bash[27556]: debug 2023-09-23T19:12:37.040+0000 7f3f62042700 -1 mon.b@1(peon).osd e72 update_from_paxos full map CRC mismatch, resetting to canonical 2023-09-23T19:12:38.136 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:37 smithi092 bash[55056]: debug 2023-09-23T19:12:37.888+0000 7f1460c4c200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T19:12:38.136 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:38 smithi092 bash[55056]: debug 2023-09-23T19:12:38.008+0000 7f1460c4c200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T19:12:38.136 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:38 smithi092 bash[27556]: audit 2023-09-23T19:12:37.036017+0000 mon.a (mon.0) 11 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-09-23T19:12:38.136 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:38 smithi092 bash[27556]: cluster 2023-09-23T19:12:37.036075+0000 osd.1 (osd.1) 3 : cluster [WRN] failed to encode map e72 with expected crc 2023-09-23T19:12:38.137 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:38 smithi092 bash[27556]: cluster 2023-09-23T19:12:37.036181+0000 mon.a (mon.0) 12 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2023-09-23T19:12:38.137 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:38 smithi092 bash[27556]: debug 2023-09-23T19:12:38.044+0000 7f3f62042700 -1 mon.b@1(peon).osd e73 update_from_paxos full map CRC mismatch, resetting to canonical 2023-09-23T19:12:38.383 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:38 smithi016 bash[33624]: audit 2023-09-23T19:12:37.036017+0000 mon.a (mon.0) 11 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-09-23T19:12:38.383 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:38 smithi016 bash[33624]: cluster 2023-09-23T19:12:37.036075+0000 osd.1 (osd.1) 3 : cluster [WRN] failed to encode map e72 with expected crc 2023-09-23T19:12:38.383 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:38 smithi016 bash[33624]: cluster 2023-09-23T19:12:37.036181+0000 mon.a (mon.0) 12 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2023-09-23T19:12:38.383 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:38 smithi016 bash[33624]: debug 2023-09-23T19:12:38.038+0000 7f954298b700 -1 mon.c@2(peon).osd e73 update_from_paxos full map CRC mismatch, resetting to canonical 2023-09-23T19:12:38.384 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:12:38 smithi016 bash[70992]: debug 2023-09-23T19:12:38.130+0000 7fb2edac0200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T19:12:38.384 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:12:38 smithi016 bash[70992]: debug 2023-09-23T19:12:38.246+0000 7fb2edac0200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T19:12:38.384 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:12:38 smithi016 bash[70992]: debug 2023-09-23T19:12:38.378+0000 7fb2edac0200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T19:12:38.416 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_1 rados --yes-i-really-mean-it || true 2023-09-23T19:12:38.535 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:38 smithi092 bash[55056]: debug 2023-09-23T19:12:38.140+0000 7f1460c4c200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T19:12:39.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:12:38 smithi016 bash[70992]: debug 2023-09-23T19:12:38.918+0000 7fb2edac0200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T19:12:39.035 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:38 smithi092 bash[55056]: debug 2023-09-23T19:12:38.628+0000 7f1460c4c200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T19:12:39.035 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:38 smithi092 bash[55056]: debug 2023-09-23T19:12:38.860+0000 7f1460c4c200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T19:12:39.158 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:39 smithi016 bash[33624]: cluster 2023-09-23T19:12:38.042286+0000 mon.a (mon.0) 13 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2023-09-23T19:12:39.158 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:39 smithi016 bash[33624]: cluster 2023-09-23T19:12:38.046537+0000 osd.2 (osd.2) 3 : cluster [WRN] failed to encode map e73 with expected crc 2023-09-23T19:12:39.158 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:39 smithi016 bash[33624]: cluster 2023-09-23T19:12:38.047974+0000 osd.3 (osd.3) 3 : cluster [WRN] failed to encode map e73 with expected crc 2023-09-23T19:12:39.158 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:39 smithi016 bash[33624]: cluster 2023-09-23T19:12:38.050614+0000 osd.0 (osd.0) 3 : cluster [WRN] failed to encode map e73 with expected crc 2023-09-23T19:12:39.158 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:39 smithi016 bash[33624]: cluster 2023-09-23T19:12:38.050668+0000 osd.1 (osd.1) 4 : cluster [WRN] failed to encode map e73 with expected crc 2023-09-23T19:12:39.159 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:39 smithi016 bash[33624]: cluster 2023-09-23T19:12:38.050785+0000 osd.3 (osd.3) 4 : cluster [WRN] failed to encode map e73 with expected crc 2023-09-23T19:12:39.159 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:39 smithi016 bash[33624]: cluster 2023-09-23T19:12:38.051341+0000 osd.6 (osd.6) 3 : cluster [WRN] failed to encode map e73 with expected crc 2023-09-23T19:12:39.159 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:39 smithi016 bash[33624]: cluster 2023-09-23T19:12:38.053491+0000 osd.5 (osd.5) 3 : cluster [WRN] failed to encode map e73 with expected crc 2023-09-23T19:12:39.159 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:39 smithi016 bash[33624]: cluster 2023-09-23T19:12:38.053524+0000 osd.7 (osd.7) 3 : cluster [WRN] failed to encode map e73 with expected crc 2023-09-23T19:12:39.159 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:39 smithi016 bash[33624]: cluster 2023-09-23T19:12:38.053614+0000 osd.4 (osd.4) 3 : cluster [WRN] failed to encode map e73 with expected crc 2023-09-23T19:12:39.159 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:39 smithi016 bash[33624]: debug 2023-09-23T19:12:39.046+0000 7f954298b700 -1 mon.c@2(peon).osd e74 update_from_paxos full map CRC mismatch, resetting to canonical 2023-09-23T19:12:39.314 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:39 smithi092 bash[27556]: cluster 2023-09-23T19:12:38.042286+0000 mon.a (mon.0) 13 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2023-09-23T19:12:39.314 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:39 smithi092 bash[27556]: cluster 2023-09-23T19:12:38.046537+0000 osd.2 (osd.2) 3 : cluster [WRN] failed to encode map e73 with expected crc 2023-09-23T19:12:39.314 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:39 smithi092 bash[27556]: cluster 2023-09-23T19:12:38.047974+0000 osd.3 (osd.3) 3 : cluster [WRN] failed to encode map e73 with expected crc 2023-09-23T19:12:39.315 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:39 smithi092 bash[27556]: cluster 2023-09-23T19:12:38.050614+0000 osd.0 (osd.0) 3 : cluster [WRN] failed to encode map e73 with expected crc 2023-09-23T19:12:39.315 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:39 smithi092 bash[27556]: cluster 2023-09-23T19:12:38.050668+0000 osd.1 (osd.1) 4 : cluster [WRN] failed to encode map e73 with expected crc 2023-09-23T19:12:39.315 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:39 smithi092 bash[27556]: cluster 2023-09-23T19:12:38.050785+0000 osd.3 (osd.3) 4 : cluster [WRN] failed to encode map e73 with expected crc 2023-09-23T19:12:39.315 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:39 smithi092 bash[27556]: cluster 2023-09-23T19:12:38.051341+0000 osd.6 (osd.6) 3 : cluster [WRN] failed to encode map e73 with expected crc 2023-09-23T19:12:39.315 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:39 smithi092 bash[27556]: cluster 2023-09-23T19:12:38.053491+0000 osd.5 (osd.5) 3 : cluster [WRN] failed to encode map e73 with expected crc 2023-09-23T19:12:39.315 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:39 smithi092 bash[27556]: cluster 2023-09-23T19:12:38.053524+0000 osd.7 (osd.7) 3 : cluster [WRN] failed to encode map e73 with expected crc 2023-09-23T19:12:39.315 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:39 smithi092 bash[27556]: cluster 2023-09-23T19:12:38.053614+0000 osd.4 (osd.4) 3 : cluster [WRN] failed to encode map e73 with expected crc 2023-09-23T19:12:39.315 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:39 smithi092 bash[27556]: debug 2023-09-23T19:12:39.056+0000 7f3f62042700 -1 mon.b@1(peon).osd e74 update_from_paxos full map CRC mismatch, resetting to canonical 2023-09-23T19:12:39.437 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:12:39 smithi016 bash[70992]: debug 2023-09-23T19:12:39.154+0000 7fb2edac0200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T19:12:39.784 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:12:39 smithi016 bash[70992]: debug 2023-09-23T19:12:39.642+0000 7fb2edac0200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T19:12:39.785 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:39 smithi092 bash[55056]: debug 2023-09-23T19:12:39.316+0000 7f1460c4c200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T19:12:40.285 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:39 smithi092 bash[55056]: debug 2023-09-23T19:12:39.820+0000 7f1460c4c200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T19:12:40.285 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:40 smithi092 bash[27556]: cluster 2023-09-23T19:12:39.051842+0000 mon.a (mon.0) 14 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2023-09-23T19:12:40.285 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:40 smithi092 bash[27556]: cluster 2023-09-23T19:12:39.054910+0000 osd.1 (osd.1) 5 : cluster [WRN] failed to encode map e74 with expected crc 2023-09-23T19:12:40.285 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:40 smithi092 bash[27556]: cluster 2023-09-23T19:12:39.063748+0000 osd.4 (osd.4) 4 : cluster [WRN] failed to encode map e74 with expected crc 2023-09-23T19:12:40.340 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:40 smithi016 bash[33624]: cluster 2023-09-23T19:12:39.051842+0000 mon.a (mon.0) 14 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2023-09-23T19:12:40.340 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:40 smithi016 bash[33624]: cluster 2023-09-23T19:12:39.054910+0000 osd.1 (osd.1) 5 : cluster [WRN] failed to encode map e74 with expected crc 2023-09-23T19:12:40.340 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:40 smithi016 bash[33624]: cluster 2023-09-23T19:12:39.063748+0000 osd.4 (osd.4) 4 : cluster [WRN] failed to encode map e74 with expected crc 2023-09-23T19:12:40.340 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:12:40 smithi016 bash[70992]: debug 2023-09-23T19:12:40.166+0000 7fb2edac0200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T19:12:40.785 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:40 smithi092 bash[55056]: debug 2023-09-23T19:12:40.388+0000 7f1460c4c200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T19:12:40.785 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:40 smithi092 bash[55056]: debug 2023-09-23T19:12:40.500+0000 7f1460c4c200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T19:12:41.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:12:40 smithi016 bash[70992]: debug 2023-09-23T19:12:40.766+0000 7fb2edac0200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T19:12:41.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:12:40 smithi016 bash[70992]: debug 2023-09-23T19:12:40.890+0000 7fb2edac0200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T19:12:41.109 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:40 smithi092 bash[55056]: debug 2023-09-23T19:12:40.848+0000 7f1460c4c200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T19:12:41.110 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:40 smithi092 bash[55056]: debug 2023-09-23T19:12:40.968+0000 7f1460c4c200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T19:12:41.529 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:12:41 smithi016 bash[70992]: debug 2023-09-23T19:12:41.246+0000 7fb2edac0200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T19:12:41.529 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:12:41 smithi016 bash[70992]: debug 2023-09-23T19:12:41.370+0000 7fb2edac0200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T19:12:41.535 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:41 smithi092 bash[55056]: debug 2023-09-23T19:12:41.112+0000 7f1460c4c200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T19:12:41.535 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:41 smithi092 bash[55056]: debug 2023-09-23T19:12:41.232+0000 7f1460c4c200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T19:12:41.535 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:41 smithi092 bash[55056]: debug 2023-09-23T19:12:41.360+0000 7f1460c4c200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T19:12:41.784 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:12:41 smithi016 bash[70992]: debug 2023-09-23T19:12:41.522+0000 7fb2edac0200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T19:12:41.784 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:12:41 smithi016 bash[70992]: debug 2023-09-23T19:12:41.650+0000 7fb2edac0200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T19:12:42.035 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:41 smithi092 bash[55056]: debug 2023-09-23T19:12:41.552+0000 7f1460c4c200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T19:12:42.035 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:41 smithi092 bash[55056]: debug 2023-09-23T19:12:41.676+0000 7f1460c4c200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T19:12:42.128 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:12:41 smithi016 bash[70992]: debug 2023-09-23T19:12:41.790+0000 7fb2edac0200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T19:12:42.129 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:12:41 smithi016 bash[70992]: debug 2023-09-23T19:12:41.990+0000 7fb2edac0200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T19:12:42.129 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:12:42 smithi016 bash[70992]: debug 2023-09-23T19:12:42.122+0000 7fb2edac0200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T19:12:43.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:43 smithi016 bash[33624]: cluster 2023-09-23T19:12:42.933883+0000 mon.a (mon.0) 15 : cluster [DBG] Standby manager daemon x restarted 2023-09-23T19:12:43.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:43 smithi016 bash[33624]: cluster 2023-09-23T19:12:42.934195+0000 mon.a (mon.0) 16 : cluster [DBG] Standby manager daemon x started 2023-09-23T19:12:43.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:43 smithi016 bash[33624]: audit 2023-09-23T19:12:42.944890+0000 mon.b (mon.1) 51 : audit [DBG] from='mgr.? 172.21.15.92:0/3838217579' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T19:12:43.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:43 smithi016 bash[33624]: audit 2023-09-23T19:12:42.947227+0000 mon.b (mon.1) 52 : audit [DBG] from='mgr.? 172.21.15.92:0/3838217579' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T19:12:43.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:43 smithi016 bash[33624]: audit 2023-09-23T19:12:42.951164+0000 mon.b (mon.1) 53 : audit [DBG] from='mgr.? 172.21.15.92:0/3838217579' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T19:12:43.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:43 smithi016 bash[33624]: audit 2023-09-23T19:12:42.952671+0000 mon.b (mon.1) 54 : audit [DBG] from='mgr.? 172.21.15.92:0/3838217579' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T19:12:43.285 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:42 smithi092 bash[55056]: debug 2023-09-23T19:12:42.916+0000 7f1460c4c200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T19:12:43.285 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:42 smithi092 bash[55056]: [23/Sep/2023:19:12:42] ENGINE Bus STARTING 2023-09-23T19:12:43.286 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:43 smithi092 bash[55056]: CherryPy Checker: 2023-09-23T19:12:43.286 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:43 smithi092 bash[55056]: The Application mounted at '' has an empty config. 2023-09-23T19:12:43.286 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:43 smithi092 bash[55056]: [23/Sep/2023:19:12:43] ENGINE Serving on http://:::9283 2023-09-23T19:12:43.286 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 19:12:43 smithi092 bash[55056]: [23/Sep/2023:19:12:43] ENGINE Bus STARTED 2023-09-23T19:12:43.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:43 smithi092 bash[27556]: cluster 2023-09-23T19:12:42.933883+0000 mon.a (mon.0) 15 : cluster [DBG] Standby manager daemon x restarted 2023-09-23T19:12:43.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:43 smithi092 bash[27556]: cluster 2023-09-23T19:12:42.934195+0000 mon.a (mon.0) 16 : cluster [DBG] Standby manager daemon x started 2023-09-23T19:12:43.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:43 smithi092 bash[27556]: audit 2023-09-23T19:12:42.944890+0000 mon.b (mon.1) 51 : audit [DBG] from='mgr.? 172.21.15.92:0/3838217579' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T19:12:43.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:43 smithi092 bash[27556]: audit 2023-09-23T19:12:42.947227+0000 mon.b (mon.1) 52 : audit [DBG] from='mgr.? 172.21.15.92:0/3838217579' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T19:12:43.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:43 smithi092 bash[27556]: audit 2023-09-23T19:12:42.951164+0000 mon.b (mon.1) 53 : audit [DBG] from='mgr.? 172.21.15.92:0/3838217579' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T19:12:43.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:43 smithi092 bash[27556]: audit 2023-09-23T19:12:42.952671+0000 mon.b (mon.1) 54 : audit [DBG] from='mgr.? 172.21.15.92:0/3838217579' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T19:12:43.721 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:43 smithi092 bash[27556]: debug 2023-09-23T19:12:43.464+0000 7f3f62042700 -1 mon.b@1(peon).osd e75 update_from_paxos full map CRC mismatch, resetting to canonical 2023-09-23T19:12:43.722 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:12:43 smithi016 bash[70992]: debug 2023-09-23T19:12:43.426+0000 7fb2edac0200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T19:12:43.722 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:12:43 smithi016 bash[70992]: [23/Sep/2023:19:12:43] ENGINE Bus STARTING 2023-09-23T19:12:43.723 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:43 smithi016 bash[33624]: debug 2023-09-23T19:12:43.454+0000 7f954298b700 -1 mon.c@2(peon).osd e75 update_from_paxos full map CRC mismatch, resetting to canonical 2023-09-23T19:12:44.006 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:44 smithi016 bash[33624]: cluster 2023-09-23T19:12:42.993884+0000 mon.a (mon.0) 17 : cluster [DBG] mgrmap e34: y(active, since 70s), standbys: x 2023-09-23T19:12:44.006 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:44 smithi016 bash[33624]: cluster 2023-09-23T19:12:43.452271+0000 mon.a (mon.0) 18 : cluster [INF] Active manager daemon y restarted 2023-09-23T19:12:44.006 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:44 smithi016 bash[33624]: cluster 2023-09-23T19:12:43.452963+0000 mon.a (mon.0) 19 : cluster [INF] Activating manager daemon y 2023-09-23T19:12:44.006 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:44 smithi016 bash[33624]: cluster 2023-09-23T19:12:43.460985+0000 mon.a (mon.0) 20 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2023-09-23T19:12:44.006 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:44 smithi016 bash[33624]: cluster 2023-09-23T19:12:43.461340+0000 mon.a (mon.0) 21 : cluster [DBG] mgrmap e35: y(active, starting, since 0.00855648s), standbys: x 2023-09-23T19:12:44.007 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:44 smithi016 bash[33624]: cluster 2023-09-23T19:12:43.465910+0000 osd.1 (osd.1) 6 : cluster [WRN] failed to encode map e75 with expected crc 2023-09-23T19:12:44.007 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:44 smithi016 bash[33624]: cluster 2023-09-23T19:12:43.474075+0000 osd.4 (osd.4) 5 : cluster [WRN] failed to encode map e75 with expected crc 2023-09-23T19:12:44.007 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:44 smithi016 bash[33624]: audit 2023-09-23T19:12:43.474509+0000 mon.a (mon.0) 22 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:12:44.007 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:12:43 smithi016 bash[70992]: debug 2023-09-23T19:12:43.718+0000 7fb24f627700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:12:44.007 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:12:43 smithi016 bash[70992]: debug 2023-09-23T19:12:43.718+0000 7fb24f627700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:12:44.007 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:12:43 smithi016 bash[70992]: debug 2023-09-23T19:12:43.718+0000 7fb24f627700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:12:44.007 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:12:43 smithi016 bash[70992]: debug 2023-09-23T19:12:43.718+0000 7fb24f627700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:12:44.007 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:12:43 smithi016 bash[70992]: debug 2023-09-23T19:12:43.718+0000 7fb24f627700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:12:44.007 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:12:43 smithi016 bash[70992]: debug 2023-09-23T19:12:43.722+0000 7fb248619700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:12:44.008 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:12:43 smithi016 bash[70992]: debug 2023-09-23T19:12:43.722+0000 7fb248619700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:12:44.008 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:12:43 smithi016 bash[70992]: debug 2023-09-23T19:12:43.722+0000 7fb248619700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:12:44.008 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:12:43 smithi016 bash[70992]: debug 2023-09-23T19:12:43.722+0000 7fb248619700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:12:44.008 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:12:43 smithi016 bash[70992]: debug 2023-09-23T19:12:43.722+0000 7fb248619700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:12:44.008 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:12:43 smithi016 bash[70992]: CherryPy Checker: 2023-09-23T19:12:44.008 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:12:43 smithi016 bash[70992]: The Application mounted at '' has an empty config. 2023-09-23T19:12:44.008 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:12:43 smithi016 bash[70992]: [23/Sep/2023:19:12:43] ENGINE Serving on http://:::9283 2023-09-23T19:12:44.008 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:12:43 smithi016 bash[70992]: [23/Sep/2023:19:12:43] ENGINE Bus STARTED 2023-09-23T19:12:44.181 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:12:44.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:44 smithi092 bash[27556]: cluster 2023-09-23T19:12:42.993884+0000 mon.a (mon.0) 17 : cluster [DBG] mgrmap e34: y(active, since 70s), standbys: x 2023-09-23T19:12:44.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:44 smithi092 bash[27556]: cluster 2023-09-23T19:12:43.452271+0000 mon.a (mon.0) 18 : cluster [INF] Active manager daemon y restarted 2023-09-23T19:12:44.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:44 smithi092 bash[27556]: cluster 2023-09-23T19:12:43.452963+0000 mon.a (mon.0) 19 : cluster [INF] Activating manager daemon y 2023-09-23T19:12:44.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:44 smithi092 bash[27556]: cluster 2023-09-23T19:12:43.460985+0000 mon.a (mon.0) 20 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2023-09-23T19:12:44.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:44 smithi092 bash[27556]: cluster 2023-09-23T19:12:43.461340+0000 mon.a (mon.0) 21 : cluster [DBG] mgrmap e35: y(active, starting, since 0.00855648s), standbys: x 2023-09-23T19:12:44.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:44 smithi092 bash[27556]: cluster 2023-09-23T19:12:43.465910+0000 osd.1 (osd.1) 6 : cluster [WRN] failed to encode map e75 with expected crc 2023-09-23T19:12:44.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:44 smithi092 bash[27556]: cluster 2023-09-23T19:12:43.474075+0000 osd.4 (osd.4) 5 : cluster [WRN] failed to encode map e75 with expected crc 2023-09-23T19:12:44.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:44 smithi092 bash[27556]: audit 2023-09-23T19:12:43.474509+0000 mon.a (mon.0) 22 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:12:44.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:44 smithi092 bash[27556]: audit 2023-09-23T19:12:43.474693+0000 mon.a (mon.0) 23 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:12:44.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:44 smithi092 bash[27556]: audit 2023-09-23T19:12:43.474845+0000 mon.a (mon.0) 24 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:12:44.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:44 smithi092 bash[27556]: audit 2023-09-23T19:12:43.476004+0000 mon.a (mon.0) 25 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi016.uncssv"}]: dispatch 2023-09-23T19:12:44.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:44 smithi092 bash[27556]: audit 2023-09-23T19:12:43.476282+0000 mon.a (mon.0) 26 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi092.vbpqks"}]: dispatch 2023-09-23T19:12:44.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:44 smithi092 bash[27556]: audit 2023-09-23T19:12:43.476987+0000 mon.a (mon.0) 27 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T19:12:44.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:44 smithi092 bash[27556]: audit 2023-09-23T19:12:43.477206+0000 mon.a (mon.0) 28 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T19:12:44.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:44 smithi092 bash[27556]: audit 2023-09-23T19:12:43.477392+0000 mon.a (mon.0) 29 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T19:12:44.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:44 smithi092 bash[27556]: audit 2023-09-23T19:12:43.477653+0000 mon.a (mon.0) 30 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T19:12:44.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:44 smithi092 bash[27556]: audit 2023-09-23T19:12:43.477877+0000 mon.a (mon.0) 31 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T19:12:44.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:44 smithi092 bash[27556]: audit 2023-09-23T19:12:43.478098+0000 mon.a (mon.0) 32 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:12:44.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:44 smithi092 bash[27556]: audit 2023-09-23T19:12:43.478308+0000 mon.a (mon.0) 33 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T19:12:44.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:44 smithi092 bash[27556]: audit 2023-09-23T19:12:43.478583+0000 mon.a (mon.0) 34 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:12:44.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:44 smithi092 bash[27556]: audit 2023-09-23T19:12:43.479014+0000 mon.a (mon.0) 35 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:12:44.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:44 smithi092 bash[27556]: audit 2023-09-23T19:12:43.479287+0000 mon.a (mon.0) 36 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:12:44.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:44 smithi092 bash[27556]: audit 2023-09-23T19:12:43.479603+0000 mon.a (mon.0) 37 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T19:12:44.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:44 smithi092 bash[27556]: audit 2023-09-23T19:12:43.479857+0000 mon.a (mon.0) 38 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T19:12:44.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:44 smithi092 bash[27556]: audit 2023-09-23T19:12:43.480452+0000 mon.a (mon.0) 39 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T19:12:44.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:44 smithi092 bash[27556]: cluster 2023-09-23T19:12:43.483950+0000 osd.4 (osd.4) 6 : cluster [WRN] failed to encode map e75 with expected crc 2023-09-23T19:12:44.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:44 smithi092 bash[27556]: cluster 2023-09-23T19:12:43.513062+0000 mon.a (mon.0) 40 : cluster [INF] Manager daemon y is now available 2023-09-23T19:12:44.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:44 smithi092 bash[27556]: audit 2023-09-23T19:12:43.553761+0000 mon.a (mon.0) 41 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:12:44.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:44 smithi092 bash[27556]: audit 2023-09-23T19:12:43.557041+0000 mon.a (mon.0) 42 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:12:44.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:44 smithi092 bash[27556]: audit 2023-09-23T19:12:43.598143+0000 mon.a (mon.0) 43 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:12:44.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:44 smithi092 bash[27556]: audit 2023-09-23T19:12:43.609390+0000 mon.a (mon.0) 44 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T19:12:44.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:44 smithi092 bash[27556]: cluster 2023-09-23T19:12:43.625185+0000 osd.5 (osd.5) 4 : cluster [WRN] failed to encode map e75 with expected crc 2023-09-23T19:12:44.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:44 smithi092 bash[27556]: audit 2023-09-23T19:12:43.658710+0000 mon.a (mon.0) 45 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T19:12:44.290 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:44 smithi016 bash[33624]: audit 2023-09-23T19:12:43.474693+0000 mon.a (mon.0) 23 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:12:44.290 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:44 smithi016 bash[33624]: audit 2023-09-23T19:12:43.474845+0000 mon.a (mon.0) 24 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:12:44.290 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:44 smithi016 bash[33624]: audit 2023-09-23T19:12:43.476004+0000 mon.a (mon.0) 25 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi016.uncssv"}]: dispatch 2023-09-23T19:12:44.290 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:44 smithi016 bash[33624]: audit 2023-09-23T19:12:43.476282+0000 mon.a (mon.0) 26 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi092.vbpqks"}]: dispatch 2023-09-23T19:12:44.290 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:44 smithi016 bash[33624]: audit 2023-09-23T19:12:43.476987+0000 mon.a (mon.0) 27 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T19:12:44.291 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:44 smithi016 bash[33624]: audit 2023-09-23T19:12:43.477206+0000 mon.a (mon.0) 28 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T19:12:44.291 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:44 smithi016 bash[33624]: audit 2023-09-23T19:12:43.477392+0000 mon.a (mon.0) 29 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T19:12:44.291 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:44 smithi016 bash[33624]: audit 2023-09-23T19:12:43.477653+0000 mon.a (mon.0) 30 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T19:12:44.291 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:44 smithi016 bash[33624]: audit 2023-09-23T19:12:43.477877+0000 mon.a (mon.0) 31 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T19:12:44.291 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:44 smithi016 bash[33624]: audit 2023-09-23T19:12:43.478098+0000 mon.a (mon.0) 32 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:12:44.291 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:44 smithi016 bash[33624]: audit 2023-09-23T19:12:43.478308+0000 mon.a (mon.0) 33 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T19:12:44.291 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:44 smithi016 bash[33624]: audit 2023-09-23T19:12:43.478583+0000 mon.a (mon.0) 34 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:12:44.291 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:44 smithi016 bash[33624]: audit 2023-09-23T19:12:43.479014+0000 mon.a (mon.0) 35 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:12:44.291 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:44 smithi016 bash[33624]: audit 2023-09-23T19:12:43.479287+0000 mon.a (mon.0) 36 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:12:44.291 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:44 smithi016 bash[33624]: audit 2023-09-23T19:12:43.479603+0000 mon.a (mon.0) 37 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T19:12:44.291 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:44 smithi016 bash[33624]: audit 2023-09-23T19:12:43.479857+0000 mon.a (mon.0) 38 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T19:12:44.292 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:44 smithi016 bash[33624]: audit 2023-09-23T19:12:43.480452+0000 mon.a (mon.0) 39 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T19:12:44.292 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:44 smithi016 bash[33624]: cluster 2023-09-23T19:12:43.483950+0000 osd.4 (osd.4) 6 : cluster [WRN] failed to encode map e75 with expected crc 2023-09-23T19:12:44.292 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:44 smithi016 bash[33624]: cluster 2023-09-23T19:12:43.513062+0000 mon.a (mon.0) 40 : cluster [INF] Manager daemon y is now available 2023-09-23T19:12:44.292 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:44 smithi016 bash[33624]: audit 2023-09-23T19:12:43.553761+0000 mon.a (mon.0) 41 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:12:44.292 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:44 smithi016 bash[33624]: audit 2023-09-23T19:12:43.557041+0000 mon.a (mon.0) 42 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:12:44.292 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:44 smithi016 bash[33624]: audit 2023-09-23T19:12:43.598143+0000 mon.a (mon.0) 43 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:12:44.292 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:44 smithi016 bash[33624]: audit 2023-09-23T19:12:43.609390+0000 mon.a (mon.0) 44 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T19:12:44.292 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:44 smithi016 bash[33624]: cluster 2023-09-23T19:12:43.625185+0000 osd.5 (osd.5) 4 : cluster [WRN] failed to encode map e75 with expected crc 2023-09-23T19:12:44.292 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:44 smithi016 bash[33624]: audit 2023-09-23T19:12:43.658710+0000 mon.a (mon.0) 45 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T19:12:44.489 INFO:teuthology.orchestra.run.smithi016.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T19:12:44.489 INFO:teuthology.orchestra.run.smithi016.stdout:alertmanager.a smithi016 *:9093,9094 running (2m) - 8m 13.6M - 0.23.0 ba2b418f427c b362c3d9bae2 2023-09-23T19:12:44.489 INFO:teuthology.orchestra.run.smithi016.stdout:grafana.a smithi092 *:3000 running (100s) - 8m 42.4M - 8.3.5 dad864ee21e9 891a61cf5fff 2023-09-23T19:12:44.489 INFO:teuthology.orchestra.run.smithi016.stdout:mds.foo.smithi016.uncssv smithi016 running (6m) - 6m 15.0M - 16.2.14-37-g2e1f1015 1e6a756174a8 95d1e187e3eb 2023-09-23T19:12:44.489 INFO:teuthology.orchestra.run.smithi016.stdout:mds.foo.smithi092.vbpqks smithi092 running (6m) - 6m 15.1M - 16.2.14-37-g2e1f1015 1e6a756174a8 1d619ec31bf1 2023-09-23T19:12:44.489 INFO:teuthology.orchestra.run.smithi016.stdout:mgr.x smithi092 *:8443,9283,8765 running (36s) - 15m 56.0M - 18.2.0-460-gbb0daeff e32c2a64c126 d54b4354eade 2023-09-23T19:12:44.489 INFO:teuthology.orchestra.run.smithi016.stdout:mgr.y smithi016 *:8443,9283,8765 running (91s) - 17m 457M - 18.2.0-460-gbb0daeff e32c2a64c126 19706820b434 2023-09-23T19:12:44.489 INFO:teuthology.orchestra.run.smithi016.stdout:mon.a smithi016 running (17m) - 17m 61.1M 2048M 16.2.14-37-g2e1f1015 1e6a756174a8 16af2f79be52 2023-09-23T19:12:44.490 INFO:teuthology.orchestra.run.smithi016.stdout:mon.b smithi092 running (15m) - 15m 60.2M 2048M 16.2.14-37-g2e1f1015 1e6a756174a8 7042e0aa9af7 2023-09-23T19:12:44.490 INFO:teuthology.orchestra.run.smithi016.stdout:mon.c smithi016 running (15m) - 15m 57.1M 2048M 16.2.14-37-g2e1f1015 1e6a756174a8 5c547dfcb630 2023-09-23T19:12:44.490 INFO:teuthology.orchestra.run.smithi016.stdout:node-exporter.a smithi016 *:9100 running (2m) - 8m 8840k - 1.3.1 1dbe0e931976 7f13d7a8905d 2023-09-23T19:12:44.490 INFO:teuthology.orchestra.run.smithi016.stdout:node-exporter.b smithi092 *:9100 running (2m) - 8m 8796k - 1.3.1 1dbe0e931976 4823347a98b5 2023-09-23T19:12:44.490 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0 smithi016 running (14m) - 14m 366M 2525M 16.2.14-37-g2e1f1015 1e6a756174a8 1458de442fb1 2023-09-23T19:12:44.490 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1 smithi016 running (14m) - 14m 238M 2525M 16.2.14-37-g2e1f1015 1e6a756174a8 7e0e07b2f6fd 2023-09-23T19:12:44.490 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2 smithi016 running (13m) - 13m 331M 2525M 16.2.14-37-g2e1f1015 1e6a756174a8 0c6fd3da0e21 2023-09-23T19:12:44.490 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3 smithi016 running (12m) - 12m 164M 2525M 16.2.14-37-g2e1f1015 1e6a756174a8 a34971c269fe 2023-09-23T19:12:44.490 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4 smithi092 running (11m) - 11m 368M 2524M 16.2.14-37-g2e1f1015 1e6a756174a8 253dc4b11010 2023-09-23T19:12:44.490 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5 smithi092 running (11m) - 11m 404M 2524M 16.2.14-37-g2e1f1015 1e6a756174a8 0a7306534382 2023-09-23T19:12:44.490 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6 smithi092 running (10m) - 10m 208M 2524M 16.2.14-37-g2e1f1015 1e6a756174a8 d6f3ee67cc5f 2023-09-23T19:12:44.490 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7 smithi092 running (9m) - 9m 260M 2524M 16.2.14-37-g2e1f1015 1e6a756174a8 794cf2340f17 2023-09-23T19:12:44.491 INFO:teuthology.orchestra.run.smithi016.stdout:prometheus.a smithi092 *:9095 running (43s) - 8m 46.1M - 2.33.4 514e6a882f6e 2666eba9f715 2023-09-23T19:12:44.784 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:12:44 smithi016 bash[70992]: debug 2023-09-23T19:12:44.466+0000 7fb2a5a7c700 -1 mgr.server handle_report got status from non-daemon mon.a 2023-09-23T19:12:45.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:45 smithi016 bash[33624]: cluster 2023-09-23T19:12:43.644414+0000 osd.2 (osd.2) 4 : cluster [WRN] failed to encode map e75 with expected crc 2023-09-23T19:12:45.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:45 smithi016 bash[33624]: cluster 2023-09-23T19:12:44.467101+0000 mon.a (mon.0) 46 : cluster [DBG] mgrmap e36: y(active, since 1.01432s), standbys: x 2023-09-23T19:12:45.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:45 smithi016 bash[33624]: audit 2023-09-23T19:12:44.470475+0000 mgr.y (mgr.25325) 1 : audit [DBG] from='client.34125 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:12:45.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:45 smithi016 bash[33624]: cluster 2023-09-23T19:12:44.479362+0000 mgr.y (mgr.25325) 2 : cluster [DBG] pgmap v3: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:12:45.285 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:45 smithi092 bash[27556]: cluster 2023-09-23T19:12:43.644414+0000 osd.2 (osd.2) 4 : cluster [WRN] failed to encode map e75 with expected crc 2023-09-23T19:12:45.285 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:45 smithi092 bash[27556]: cluster 2023-09-23T19:12:44.467101+0000 mon.a (mon.0) 46 : cluster [DBG] mgrmap e36: y(active, since 1.01432s), standbys: x 2023-09-23T19:12:45.285 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:45 smithi092 bash[27556]: audit 2023-09-23T19:12:44.470475+0000 mgr.y (mgr.25325) 1 : audit [DBG] from='client.34125 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:12:45.285 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:45 smithi092 bash[27556]: cluster 2023-09-23T19:12:44.479362+0000 mgr.y (mgr.25325) 2 : cluster [DBG] pgmap v3: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:12:45.961 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:12:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:12:45] "GET /metrics HTTP/1.1" 200 33341 "" "Prometheus/2.33.4" 2023-09-23T19:12:45.962 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade pause' 2023-09-23T19:12:46.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:46 smithi016 bash[33624]: cephadm 2023-09-23T19:12:45.208811+0000 mgr.y (mgr.25325) 3 : cephadm [INF] [23/Sep/2023:19:12:45] ENGINE Bus STARTING 2023-09-23T19:12:46.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:46 smithi016 bash[33624]: cephadm 2023-09-23T19:12:45.311158+0000 mgr.y (mgr.25325) 4 : cephadm [INF] [23/Sep/2023:19:12:45] ENGINE Serving on http://172.21.15.16:8765 2023-09-23T19:12:46.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:46 smithi016 bash[33624]: cephadm 2023-09-23T19:12:45.435516+0000 mgr.y (mgr.25325) 5 : cephadm [INF] [23/Sep/2023:19:12:45] ENGINE Serving on https://172.21.15.16:7150 2023-09-23T19:12:46.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:46 smithi016 bash[33624]: cephadm 2023-09-23T19:12:45.435832+0000 mgr.y (mgr.25325) 6 : cephadm [INF] [23/Sep/2023:19:12:45] ENGINE Bus STARTED 2023-09-23T19:12:46.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:46 smithi016 bash[33624]: cluster 2023-09-23T19:12:45.478887+0000 mgr.y (mgr.25325) 7 : cluster [DBG] pgmap v4: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:12:46.785 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:46 smithi092 bash[27556]: cephadm 2023-09-23T19:12:45.208811+0000 mgr.y (mgr.25325) 3 : cephadm [INF] [23/Sep/2023:19:12:45] ENGINE Bus STARTING 2023-09-23T19:12:46.785 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:46 smithi092 bash[27556]: cephadm 2023-09-23T19:12:45.311158+0000 mgr.y (mgr.25325) 4 : cephadm [INF] [23/Sep/2023:19:12:45] ENGINE Serving on http://172.21.15.16:8765 2023-09-23T19:12:46.785 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:46 smithi092 bash[27556]: cephadm 2023-09-23T19:12:45.435516+0000 mgr.y (mgr.25325) 5 : cephadm [INF] [23/Sep/2023:19:12:45] ENGINE Serving on https://172.21.15.16:7150 2023-09-23T19:12:46.785 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:46 smithi092 bash[27556]: cephadm 2023-09-23T19:12:45.435832+0000 mgr.y (mgr.25325) 6 : cephadm [INF] [23/Sep/2023:19:12:45] ENGINE Bus STARTED 2023-09-23T19:12:46.785 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:46 smithi092 bash[27556]: cluster 2023-09-23T19:12:45.478887+0000 mgr.y (mgr.25325) 7 : cluster [DBG] pgmap v4: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:12:47.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:47 smithi016 bash[33624]: cluster 2023-09-23T19:12:46.485594+0000 mon.a (mon.0) 47 : cluster [DBG] mgrmap e37: y(active, since 3s), standbys: x 2023-09-23T19:12:47.785 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:47 smithi092 bash[27556]: cluster 2023-09-23T19:12:46.485594+0000 mon.a (mon.0) 47 : cluster [DBG] mgrmap e37: y(active, since 3s), standbys: x 2023-09-23T19:12:48.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:48 smithi016 bash[33624]: cluster 2023-09-23T19:12:47.479701+0000 mgr.y (mgr.25325) 8 : cluster [DBG] pgmap v5: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:12:48.785 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:48 smithi092 bash[27556]: cluster 2023-09-23T19:12:47.479701+0000 mgr.y (mgr.25325) 8 : cluster [DBG] pgmap v5: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:12:48.785 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:12:48 smithi092 bash[54653]: ts=2023-09-23T19:12:48.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi092.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi016:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T19:12:49.898 INFO:teuthology.orchestra.run.smithi016.stderr:enabled application 'rados' on pool 'unique_pool_1' 2023-09-23T19:12:50.201 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:49 smithi016 bash[33624]: debug 2023-09-23T19:12:49.882+0000 7f954298b700 -1 mon.c@2(peon).osd e76 update_from_paxos full map CRC mismatch, resetting to canonical 2023-09-23T19:12:50.201 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:49 smithi016 bash[33624]: cluster 2023-09-23T19:12:49.480745+0000 mgr.y (mgr.25325) 9 : cluster [DBG] pgmap v6: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:12:50.201 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:49 smithi016 bash[33624]: audit 2023-09-23T19:12:49.617081+0000 mon.a (mon.0) 48 : audit [INF] from='client.? 172.21.15.16:0/1719600252' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:12:50.285 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:49 smithi092 bash[27556]: debug 2023-09-23T19:12:49.891+0000 7f3f62042700 -1 mon.b@1(peon).osd e76 update_from_paxos full map CRC mismatch, resetting to canonical 2023-09-23T19:12:50.285 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:49 smithi092 bash[27556]: cluster 2023-09-23T19:12:49.480745+0000 mgr.y (mgr.25325) 9 : cluster [DBG] pgmap v6: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:12:50.285 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:49 smithi092 bash[27556]: audit 2023-09-23T19:12:49.617081+0000 mon.a (mon.0) 48 : audit [INF] from='client.? 172.21.15.16:0/1719600252' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:12:50.828 INFO:teuthology.orchestra.run.smithi016.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:12:51.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:50 smithi016 bash[33624]: audit 2023-09-23T19:12:49.890392+0000 mon.a (mon.0) 49 : audit [INF] from='client.? 172.21.15.16:0/1719600252' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T19:12:51.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:50 smithi016 bash[33624]: cluster 2023-09-23T19:12:49.890520+0000 mon.a (mon.0) 50 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2023-09-23T19:12:51.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:50 smithi016 bash[33624]: cluster 2023-09-23T19:12:49.892887+0000 osd.1 (osd.1) 7 : cluster [WRN] failed to encode map e76 with expected crc 2023-09-23T19:12:51.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:50 smithi016 bash[33624]: cluster 2023-09-23T19:12:49.892889+0000 osd.2 (osd.2) 5 : cluster [WRN] failed to encode map e76 with expected crc 2023-09-23T19:12:51.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:50 smithi016 bash[33624]: audit 2023-09-23T19:12:50.296301+0000 mon.a (mon.0) 51 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:12:51.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:50 smithi016 bash[33624]: audit 2023-09-23T19:12:50.303835+0000 mon.a (mon.0) 52 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:12:51.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:50 smithi016 bash[33624]: cluster 2023-09-23T19:12:50.636597+0000 osd.6 (osd.6) 4 : cluster [WRN] failed to encode map e76 with expected crc 2023-09-23T19:12:51.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:50 smithi016 bash[33624]: audit 2023-09-23T19:12:50.826131+0000 mon.a (mon.0) 53 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:12:51.285 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:50 smithi092 bash[27556]: audit 2023-09-23T19:12:49.890392+0000 mon.a (mon.0) 49 : audit [INF] from='client.? 172.21.15.16:0/1719600252' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T19:12:51.285 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:50 smithi092 bash[27556]: cluster 2023-09-23T19:12:49.890520+0000 mon.a (mon.0) 50 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2023-09-23T19:12:51.285 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:50 smithi092 bash[27556]: cluster 2023-09-23T19:12:49.892887+0000 osd.1 (osd.1) 7 : cluster [WRN] failed to encode map e76 with expected crc 2023-09-23T19:12:51.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:50 smithi092 bash[27556]: cluster 2023-09-23T19:12:49.892889+0000 osd.2 (osd.2) 5 : cluster [WRN] failed to encode map e76 with expected crc 2023-09-23T19:12:51.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:50 smithi092 bash[27556]: audit 2023-09-23T19:12:50.296301+0000 mon.a (mon.0) 51 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:12:51.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:50 smithi092 bash[27556]: audit 2023-09-23T19:12:50.303835+0000 mon.a (mon.0) 52 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:12:51.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:50 smithi092 bash[27556]: cluster 2023-09-23T19:12:50.636597+0000 osd.6 (osd.6) 4 : cluster [WRN] failed to encode map e76 with expected crc 2023-09-23T19:12:51.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:50 smithi092 bash[27556]: audit 2023-09-23T19:12:50.826131+0000 mon.a (mon.0) 53 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:12:51.594 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'sleep 60' 2023-09-23T19:12:51.870 DEBUG:teuthology.orchestra.run.smithi016:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_1 bench 90 write' 2023-09-23T19:12:51.872 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T19:12:51.872 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T19:12:51.913 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:hints = 1 2023-09-23T19:12:51.913 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T19:12:51.913 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Object prefix: benchmark_data_smithi016_76297 2023-09-23T19:12:51.914 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:12:51.914 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 0 0 0 0 0 0 - 0 2023-09-23T19:12:52.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:52 smithi016 bash[33624]: audit 2023-09-23T19:12:50.820962+0000 mgr.y (mgr.25325) 10 : audit [DBG] from='client.34173 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:12:52.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:52 smithi016 bash[33624]: cephadm 2023-09-23T19:12:50.821433+0000 mgr.y (mgr.25325) 11 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:12:52.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:52 smithi016 bash[33624]: audit 2023-09-23T19:12:51.096489+0000 mon.a (mon.0) 54 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:12:52.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:52 smithi016 bash[33624]: audit 2023-09-23T19:12:51.104902+0000 mon.a (mon.0) 55 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:12:52.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:52 smithi016 bash[33624]: cluster 2023-09-23T19:12:51.139038+0000 osd.7 (osd.7) 4 : cluster [WRN] failed to encode map e76 with expected crc 2023-09-23T19:12:52.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:52 smithi016 bash[33624]: cluster 2023-09-23T19:12:51.206479+0000 osd.5 (osd.5) 5 : cluster [WRN] failed to encode map e76 with expected crc 2023-09-23T19:12:52.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:52 smithi016 bash[33624]: cluster 2023-09-23T19:12:51.481464+0000 mgr.y (mgr.25325) 12 : cluster [DBG] pgmap v8: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:12:52.535 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:52 smithi092 bash[27556]: audit 2023-09-23T19:12:50.820962+0000 mgr.y (mgr.25325) 10 : audit [DBG] from='client.34173 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:12:52.535 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:52 smithi092 bash[27556]: cephadm 2023-09-23T19:12:50.821433+0000 mgr.y (mgr.25325) 11 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:12:52.535 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:52 smithi092 bash[27556]: audit 2023-09-23T19:12:51.096489+0000 mon.a (mon.0) 54 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:12:52.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:52 smithi092 bash[27556]: audit 2023-09-23T19:12:51.104902+0000 mon.a (mon.0) 55 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:12:52.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:52 smithi092 bash[27556]: cluster 2023-09-23T19:12:51.139038+0000 osd.7 (osd.7) 4 : cluster [WRN] failed to encode map e76 with expected crc 2023-09-23T19:12:52.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:52 smithi092 bash[27556]: cluster 2023-09-23T19:12:51.206479+0000 osd.5 (osd.5) 5 : cluster [WRN] failed to encode map e76 with expected crc 2023-09-23T19:12:52.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:52 smithi092 bash[27556]: cluster 2023-09-23T19:12:51.481464+0000 mgr.y (mgr.25325) 12 : cluster [DBG] pgmap v8: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:12:52.915 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 1 16 2507 2491 155.614 155.688 0.00954293 0.00638203 2023-09-23T19:12:53.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:53 smithi016 bash[33624]: cluster 2023-09-23T19:12:51.247330+0000 osd.3 (osd.3) 5 : cluster [WRN] failed to encode map e76 with expected crc 2023-09-23T19:12:53.535 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:53 smithi092 bash[27556]: cluster 2023-09-23T19:12:51.247330+0000 osd.3 (osd.3) 5 : cluster [WRN] failed to encode map e76 with expected crc 2023-09-23T19:12:53.914 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 2 16 4815 4799 149.927 144.25 0.00544527 0.00665229 2023-09-23T19:12:54.741 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:54 smithi016 bash[33624]: audit 2023-09-23T19:12:53.366794+0000 mon.a (mon.0) 56 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:12:54.741 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:54 smithi016 bash[33624]: audit 2023-09-23T19:12:53.375043+0000 mon.a (mon.0) 57 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:12:54.741 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:54 smithi016 bash[33624]: cluster 2023-09-23T19:12:53.481904+0000 mgr.y (mgr.25325) 13 : cluster [DBG] pgmap v9: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:12:54.785 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:54 smithi092 bash[27556]: audit 2023-09-23T19:12:53.366794+0000 mon.a (mon.0) 56 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:12:54.785 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:54 smithi092 bash[27556]: audit 2023-09-23T19:12:53.375043+0000 mon.a (mon.0) 57 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:12:54.785 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:54 smithi092 bash[27556]: cluster 2023-09-23T19:12:53.481904+0000 mgr.y (mgr.25325) 13 : cluster [DBG] pgmap v9: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:12:54.914 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 3 16 7198 7182 149.593 148.938 0.0129442 0.0066716 2023-09-23T19:12:55.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:55 smithi016 bash[33624]: audit 2023-09-23T19:12:54.440509+0000 mon.a (mon.0) 58 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:12:55.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:55 smithi016 bash[33624]: audit 2023-09-23T19:12:54.452419+0000 mon.a (mon.0) 59 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:12:55.785 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:12:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:12:55] "GET /metrics HTTP/1.1" 200 33341 "" "Prometheus/2.33.4" 2023-09-23T19:12:55.785 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:55 smithi092 bash[27556]: audit 2023-09-23T19:12:54.440509+0000 mon.a (mon.0) 58 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:12:55.785 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:55 smithi092 bash[27556]: audit 2023-09-23T19:12:54.452419+0000 mon.a (mon.0) 59 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:12:55.915 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 4 16 9838 9822 153.44 165 0.00499658 0.00650845 2023-09-23T19:12:56.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:56 smithi016 bash[33624]: cluster 2023-09-23T19:12:55.484958+0000 mgr.y (mgr.25325) 14 : cluster [DBG] pgmap v10: 89 pgs: 89 active+clean; 383 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s wr, 610 op/s 2023-09-23T19:12:56.785 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:56 smithi092 bash[27556]: cluster 2023-09-23T19:12:55.484958+0000 mgr.y (mgr.25325) 14 : cluster [DBG] pgmap v10: 89 pgs: 89 active+clean; 383 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s wr, 610 op/s 2023-09-23T19:12:56.915 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 5 15 12266 12251 153.112 151.812 0.00901536 0.0065206 2023-09-23T19:12:57.915 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 6 16 13920 13904 144.811 103.312 0.0104534 0.00689539 2023-09-23T19:12:58.785 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:12:58 smithi092 bash[54653]: ts=2023-09-23T19:12:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi092.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi016:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T19:12:58.785 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:58 smithi092 bash[27556]: cluster 2023-09-23T19:12:57.485618+0000 mgr.y (mgr.25325) 15 : cluster [DBG] pgmap v11: 89 pgs: 89 active+clean; 383 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s wr, 610 op/s 2023-09-23T19:12:58.915 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 7 16 16216 16200 144.622 143.5 0.00817322 0.00690747 2023-09-23T19:12:59.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:58 smithi016 bash[33624]: cluster 2023-09-23T19:12:57.485618+0000 mgr.y (mgr.25325) 15 : cluster [DBG] pgmap v11: 89 pgs: 89 active+clean; 383 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s wr, 610 op/s 2023-09-23T19:12:59.915 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 8 15 18379 18364 143.45 135.25 0.00377764 0.00696323 2023-09-23T19:13:00.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:12:59 smithi016 bash[33624]: audit 2023-09-23T19:12:58.599323+0000 mon.a (mon.0) 60 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:13:00.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:12:59 smithi092 bash[27556]: audit 2023-09-23T19:12:58.599323+0000 mon.a (mon.0) 60 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:13:00.915 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 9 16 20258 20242 140.551 117.375 0.00389816 0.0071055 2023-09-23T19:13:00.972 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:13:00 smithi092 bash[27556]: cluster 2023-09-23T19:12:59.486393+0000 mgr.y (mgr.25325) 16 : cluster [DBG] pgmap v12: 89 pgs: 89 active+clean; 1.1 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s 2023-09-23T19:13:01.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:13:00 smithi016 bash[33624]: cluster 2023-09-23T19:12:59.486393+0000 mgr.y (mgr.25325) 16 : cluster [DBG] pgmap v12: 89 pgs: 89 active+clean; 1.1 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s 2023-09-23T19:13:01.915 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 10 16 22281 22265 139.138 126.438 0.00766806 0.00717913 2023-09-23T19:13:02.915 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 11 16 24256 24240 137.71 123.438 0.00979843 0.00725456 2023-09-23T19:13:03.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:13:02 smithi016 bash[33624]: cluster 2023-09-23T19:13:01.486896+0000 mgr.y (mgr.25325) 17 : cluster [DBG] pgmap v13: 89 pgs: 89 active+clean; 1.1 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2023-09-23T19:13:03.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:13:02 smithi092 bash[27556]: cluster 2023-09-23T19:13:01.486896+0000 mgr.y (mgr.25325) 17 : cluster [DBG] pgmap v13: 89 pgs: 89 active+clean; 1.1 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2023-09-23T19:13:03.915 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 12 16 25919 25903 134.895 103.938 0.00985108 0.00740582 2023-09-23T19:13:04.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:13:04 smithi016 bash[33624]: audit 2023-09-23T19:13:03.278699+0000 mon.a (mon.0) 61 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:13:04.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:13:04 smithi016 bash[33624]: audit 2023-09-23T19:13:03.288560+0000 mon.a (mon.0) 62 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:13:04.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:13:04 smithi016 bash[33624]: audit 2023-09-23T19:13:03.290965+0000 mon.a (mon.0) 63 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:13:04.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:13:04 smithi016 bash[33624]: audit 2023-09-23T19:13:03.292581+0000 mon.a (mon.0) 64 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:13:04.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:13:04 smithi016 bash[33624]: cephadm 2023-09-23T19:13:03.295263+0000 mgr.y (mgr.25325) 18 : cephadm [INF] Updating smithi016:/etc/ceph/ceph.conf 2023-09-23T19:13:04.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:13:04 smithi016 bash[33624]: cephadm 2023-09-23T19:13:03.295553+0000 mgr.y (mgr.25325) 19 : cephadm [INF] Updating smithi092:/etc/ceph/ceph.conf 2023-09-23T19:13:04.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:13:04 smithi016 bash[33624]: cluster 2023-09-23T19:13:03.488817+0000 mgr.y (mgr.25325) 20 : cluster [DBG] pgmap v14: 89 pgs: 89 active+clean; 1.1 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2023-09-23T19:13:04.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:13:04 smithi016 bash[33624]: cephadm 2023-09-23T19:13:03.562963+0000 mgr.y (mgr.25325) 21 : cephadm [INF] Updating smithi092:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:13:04.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:13:04 smithi016 bash[33624]: cephadm 2023-09-23T19:13:03.563191+0000 mgr.y (mgr.25325) 22 : cephadm [INF] Updating smithi016:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:13:04.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:13:04 smithi092 bash[27556]: audit 2023-09-23T19:13:03.278699+0000 mon.a (mon.0) 61 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:13:04.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:13:04 smithi092 bash[27556]: audit 2023-09-23T19:13:03.288560+0000 mon.a (mon.0) 62 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:13:04.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:13:04 smithi092 bash[27556]: audit 2023-09-23T19:13:03.290965+0000 mon.a (mon.0) 63 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:13:04.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:13:04 smithi092 bash[27556]: audit 2023-09-23T19:13:03.292581+0000 mon.a (mon.0) 64 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:13:04.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:13:04 smithi092 bash[27556]: cephadm 2023-09-23T19:13:03.295263+0000 mgr.y (mgr.25325) 18 : cephadm [INF] Updating smithi016:/etc/ceph/ceph.conf 2023-09-23T19:13:04.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:13:04 smithi092 bash[27556]: cephadm 2023-09-23T19:13:03.295553+0000 mgr.y (mgr.25325) 19 : cephadm [INF] Updating smithi092:/etc/ceph/ceph.conf 2023-09-23T19:13:04.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:13:04 smithi092 bash[27556]: cluster 2023-09-23T19:13:03.488817+0000 mgr.y (mgr.25325) 20 : cluster [DBG] pgmap v14: 89 pgs: 89 active+clean; 1.1 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2023-09-23T19:13:04.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:13:04 smithi092 bash[27556]: cephadm 2023-09-23T19:13:03.562963+0000 mgr.y (mgr.25325) 21 : cephadm [INF] Updating smithi092:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:13:04.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:13:04 smithi092 bash[27556]: cephadm 2023-09-23T19:13:03.563191+0000 mgr.y (mgr.25325) 22 : cephadm [INF] Updating smithi016:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:13:04.915 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 13 16 27474 27458 131.994 97.1875 0.00873091 0.00756891 2023-09-23T19:13:05.639 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:13:05 smithi016 bash[33624]: cephadm 2023-09-23T19:13:03.827935+0000 mgr.y (mgr.25325) 23 : cephadm [INF] Updating smithi092:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:13:05.639 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:13:05 smithi016 bash[33624]: cephadm 2023-09-23T19:13:03.828145+0000 mgr.y (mgr.25325) 24 : cephadm [INF] Updating smithi016:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:13:05.639 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:13:05 smithi016 bash[33624]: cephadm 2023-09-23T19:13:04.094271+0000 mgr.y (mgr.25325) 25 : cephadm [INF] Updating smithi092:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:13:05.639 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:13:05 smithi016 bash[33624]: cephadm 2023-09-23T19:13:04.097737+0000 mgr.y (mgr.25325) 26 : cephadm [INF] Updating smithi016:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:13:05.640 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:13:05 smithi016 bash[33624]: audit 2023-09-23T19:13:04.377351+0000 mon.a (mon.0) 65 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:13:05.640 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:13:05 smithi016 bash[33624]: audit 2023-09-23T19:13:04.408038+0000 mon.a (mon.0) 66 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:13:05.640 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:13:05 smithi016 bash[33624]: audit 2023-09-23T19:13:04.425529+0000 mon.a (mon.0) 67 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:13:05.640 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:13:05 smithi016 bash[33624]: audit 2023-09-23T19:13:04.436689+0000 mon.a (mon.0) 68 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:13:05.640 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:13:05 smithi016 bash[33624]: audit 2023-09-23T19:13:04.445698+0000 mon.a (mon.0) 69 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:13:05.640 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:13:05 smithi016 bash[33624]: audit 2023-09-23T19:13:04.532126+0000 mon.a (mon.0) 70 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:13:05.640 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:13:05 smithi016 bash[33624]: audit 2023-09-23T19:13:04.534992+0000 mon.a (mon.0) 71 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:13:05.640 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:13:05 smithi016 bash[33624]: audit 2023-09-23T19:13:04.536382+0000 mon.a (mon.0) 72 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:13:05.640 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:13:05 smithi016 bash[33624]: audit 2023-09-23T19:13:04.547622+0000 mon.a (mon.0) 73 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:13:05.640 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:13:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:13:05] "GET /metrics HTTP/1.1" 200 36055 "" "Prometheus/2.33.4" 2023-09-23T19:13:05.785 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:13:05 smithi092 bash[27556]: cephadm 2023-09-23T19:13:03.827935+0000 mgr.y (mgr.25325) 23 : cephadm [INF] Updating smithi092:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:13:05.785 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:13:05 smithi092 bash[27556]: cephadm 2023-09-23T19:13:03.828145+0000 mgr.y (mgr.25325) 24 : cephadm [INF] Updating smithi016:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:13:05.785 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:13:05 smithi092 bash[27556]: cephadm 2023-09-23T19:13:04.094271+0000 mgr.y (mgr.25325) 25 : cephadm [INF] Updating smithi092:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:13:05.786 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:13:05 smithi092 bash[27556]: cephadm 2023-09-23T19:13:04.097737+0000 mgr.y (mgr.25325) 26 : cephadm [INF] Updating smithi016:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:13:05.786 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:13:05 smithi092 bash[27556]: audit 2023-09-23T19:13:04.377351+0000 mon.a (mon.0) 65 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:13:05.786 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:13:05 smithi092 bash[27556]: audit 2023-09-23T19:13:04.408038+0000 mon.a (mon.0) 66 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:13:05.786 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:13:05 smithi092 bash[27556]: audit 2023-09-23T19:13:04.425529+0000 mon.a (mon.0) 67 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:13:05.786 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:13:05 smithi092 bash[27556]: audit 2023-09-23T19:13:04.436689+0000 mon.a (mon.0) 68 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:13:05.786 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:13:05 smithi092 bash[27556]: audit 2023-09-23T19:13:04.445698+0000 mon.a (mon.0) 69 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:13:05.786 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:13:05 smithi092 bash[27556]: audit 2023-09-23T19:13:04.532126+0000 mon.a (mon.0) 70 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:13:05.786 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:13:05 smithi092 bash[27556]: audit 2023-09-23T19:13:04.534992+0000 mon.a (mon.0) 71 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:13:05.786 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:13:05 smithi092 bash[27556]: audit 2023-09-23T19:13:04.536382+0000 mon.a (mon.0) 72 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:13:05.786 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:13:05 smithi092 bash[27556]: audit 2023-09-23T19:13:04.547622+0000 mon.a (mon.0) 73 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:13:05.916 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 14 16 29321 29305 130.81 115.438 0.00489473 0.00763574 2023-09-23T19:13:06.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:13:06 smithi016 bash[33624]: cluster 2023-09-23T19:13:05.489660+0000 mgr.y (mgr.25325) 27 : cluster [DBG] pgmap v15: 89 pgs: 89 active+clean; 1.6 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s 2023-09-23T19:13:06.785 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:13:06 smithi092 bash[27556]: cluster 2023-09-23T19:13:05.489660+0000 mgr.y (mgr.25325) 27 : cluster [DBG] pgmap v15: 89 pgs: 89 active+clean; 1.6 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s 2023-09-23T19:13:06.916 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 15 16 31084 31068 129.435 110.188 0.00340019 0.00771876 2023-09-23T19:13:07.916 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 16 16 32810 32794 128.087 107.875 0.0102919 0.00780024 2023-09-23T19:13:08.916 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 17 16 34506 34490 126.787 106 0.0111044 0.00788126 2023-09-23T19:13:09.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:13:08 smithi016 bash[33624]: cluster 2023-09-23T19:13:07.490272+0000 mgr.y (mgr.25325) 28 : cluster [DBG] pgmap v16: 89 pgs: 89 active+clean; 1.6 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s 2023-09-23T19:13:09.034 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:13:08 smithi092 bash[27556]: cluster 2023-09-23T19:13:07.490272+0000 mgr.y (mgr.25325) 28 : cluster [DBG] pgmap v16: 89 pgs: 89 active+clean; 1.6 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s 2023-09-23T19:13:09.035 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:13:08 smithi092 bash[54653]: ts=2023-09-23T19:13:08.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi092.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi016:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T19:13:09.916 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 18 16 36148 36132 125.444 102.625 0.0112324 0.00796515 2023-09-23T19:13:10.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:13:09 smithi016 bash[33624]: cluster 2023-09-23T19:13:09.491113+0000 mgr.y (mgr.25325) 29 : cluster [DBG] pgmap v17: 89 pgs: 89 active+clean; 2.1 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 144 MiB/s wr, 2.31k op/s 2023-09-23T19:13:10.285 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:13:09 smithi092 bash[27556]: cluster 2023-09-23T19:13:09.491113+0000 mgr.y (mgr.25325) 29 : cluster [DBG] pgmap v17: 89 pgs: 89 active+clean; 2.1 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 144 MiB/s wr, 2.31k op/s 2023-09-23T19:13:10.916 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 19 16 37785 37769 124.226 102.312 0.00515794 0.0080432 2023-09-23T19:13:11.916 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-09-23T19:13:11.914996+0000 min lat: 0.00132766 max lat: 0.345747 avg lat: 0.0081216 2023-09-23T19:13:11.916 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:13:11.916 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 20 16 39394 39378 123.042 100.562 0.0107841 0.0081216 2023-09-23T19:13:12.916 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 21 16 41031 41015 122.055 102.312 0.00916526 0.00818604 2023-09-23T19:13:13.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:13:12 smithi016 bash[33624]: cluster 2023-09-23T19:13:11.491725+0000 mgr.y (mgr.25325) 30 : cluster [DBG] pgmap v18: 89 pgs: 89 active+clean; 2.2 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2023-09-23T19:13:13.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:13:12 smithi092 bash[27556]: cluster 2023-09-23T19:13:11.491725+0000 mgr.y (mgr.25325) 30 : cluster [DBG] pgmap v18: 89 pgs: 89 active+clean; 2.2 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2023-09-23T19:13:13.916 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 22 16 42580 42564 120.907 96.8125 0.0102711 0.00826406 2023-09-23T19:13:14.916 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 23 16 44130 44114 119.862 96.875 0.00571616 0.00833636 2023-09-23T19:13:15.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:13:14 smithi016 bash[33624]: cluster 2023-09-23T19:13:13.492275+0000 mgr.y (mgr.25325) 31 : cluster [DBG] pgmap v19: 89 pgs: 89 active+clean; 2.2 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2023-09-23T19:13:15.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:13:14 smithi016 bash[33624]: audit 2023-09-23T19:13:13.599313+0000 mon.a (mon.0) 74 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:13:15.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:13:14 smithi092 bash[27556]: cluster 2023-09-23T19:13:13.492275+0000 mgr.y (mgr.25325) 31 : cluster [DBG] pgmap v19: 89 pgs: 89 active+clean; 2.2 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2023-09-23T19:13:15.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:13:14 smithi092 bash[27556]: audit 2023-09-23T19:13:13.599313+0000 mon.a (mon.0) 74 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:13:15.917 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 24 16 45672 45656 118.883 96.375 0.00630875 0.00840584 2023-09-23T19:13:16.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:13:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:13:15] "GET /metrics HTTP/1.1" 200 36053 "" "Prometheus/2.33.4" 2023-09-23T19:13:16.917 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 25 16 47121 47105 117.749 90.5625 0.00591254 0.00848684 2023-09-23T19:13:17.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:13:16 smithi016 bash[33624]: cluster 2023-09-23T19:13:15.493060+0000 mgr.y (mgr.25325) 32 : cluster [DBG] pgmap v20: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 135 MiB/s wr, 2.16k op/s 2023-09-23T19:13:17.034 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:13:16 smithi092 bash[27556]: cluster 2023-09-23T19:13:15.493060+0000 mgr.y (mgr.25325) 32 : cluster [DBG] pgmap v20: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 135 MiB/s wr, 2.16k op/s 2023-09-23T19:13:17.917 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 26 16 48497 48481 116.528 86 0.00621914 0.00857511 2023-09-23T19:13:18.917 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 27 16 49893 49877 115.443 87.25 0.00682558 0.00865553 2023-09-23T19:13:19.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:13:18 smithi016 bash[33624]: cluster 2023-09-23T19:13:17.493525+0000 mgr.y (mgr.25325) 33 : cluster [DBG] pgmap v21: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-09-23T19:13:19.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:13:18 smithi092 bash[27556]: cluster 2023-09-23T19:13:17.493525+0000 mgr.y (mgr.25325) 33 : cluster [DBG] pgmap v21: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-09-23T19:13:19.035 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:13:18 smithi092 bash[54653]: ts=2023-09-23T19:13:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi092.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi016:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T19:13:19.917 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 28 16 51228 51212 114.3 83.4375 0.00995542 0.00874242 2023-09-23T19:13:20.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:13:19 smithi016 bash[33624]: cluster 2023-09-23T19:13:19.494289+0000 mgr.y (mgr.25325) 34 : cluster [DBG] pgmap v22: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-09-23T19:13:20.285 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:13:19 smithi092 bash[27556]: cluster 2023-09-23T19:13:19.494289+0000 mgr.y (mgr.25325) 34 : cluster [DBG] pgmap v22: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-09-23T19:13:20.917 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 29 16 52575 52559 113.261 84.1875 0.00724532 0.00882304 2023-09-23T19:13:21.917 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 30 16 53883 53867 112.211 81.75 0.0108874 0.00890601 2023-09-23T19:13:22.917 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 31 16 55129 55113 111.103 77.875 0.0230545 0.00899407 2023-09-23T19:13:23.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:13:22 smithi016 bash[33624]: cluster 2023-09-23T19:13:21.494891+0000 mgr.y (mgr.25325) 35 : cluster [DBG] pgmap v23: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s 2023-09-23T19:13:23.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:13:22 smithi092 bash[27556]: cluster 2023-09-23T19:13:21.494891+0000 mgr.y (mgr.25325) 35 : cluster [DBG] pgmap v23: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s 2023-09-23T19:13:23.917 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 32 16 56360 56344 110.035 76.9375 0.0116345 0.00908072 2023-09-23T19:13:24.917 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 33 15 57651 57636 109.147 80.75 0.00951744 0.00915572 2023-09-23T19:13:25.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:13:24 smithi016 bash[33624]: cluster 2023-09-23T19:13:23.495347+0000 mgr.y (mgr.25325) 36 : cluster [DBG] pgmap v24: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-09-23T19:13:25.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:13:24 smithi092 bash[27556]: cluster 2023-09-23T19:13:23.495347+0000 mgr.y (mgr.25325) 36 : cluster [DBG] pgmap v24: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-09-23T19:13:25.917 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 34 16 58894 58878 108.22 77.625 0.00672343 0.00923451 2023-09-23T19:13:26.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:13:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:13:25] "GET /metrics HTTP/1.1" 200 36053 "" "Prometheus/2.33.4" 2023-09-23T19:13:26.918 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 35 16 60125 60109 107.326 76.9375 0.0130246 0.00931093 2023-09-23T19:13:27.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:13:26 smithi016 bash[33624]: cluster 2023-09-23T19:13:25.496112+0000 mgr.y (mgr.25325) 37 : cluster [DBG] pgmap v25: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-09-23T19:13:27.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:13:26 smithi092 bash[27556]: cluster 2023-09-23T19:13:25.496112+0000 mgr.y (mgr.25325) 37 : cluster [DBG] pgmap v25: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-09-23T19:13:27.918 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 36 16 61327 61311 106.431 75.125 0.0103828 0.00939025 2023-09-23T19:13:28.918 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 37 16 62520 62504 105.57 74.5625 0.00894698 0.00946556 2023-09-23T19:13:29.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:13:28 smithi016 bash[33624]: cluster 2023-09-23T19:13:27.496566+0000 mgr.y (mgr.25325) 38 : cluster [DBG] pgmap v26: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-09-23T19:13:29.035 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:13:28 smithi092 bash[54653]: ts=2023-09-23T19:13:28.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi092.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi016:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T19:13:29.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:13:28 smithi092 bash[27556]: cluster 2023-09-23T19:13:27.496566+0000 mgr.y (mgr.25325) 38 : cluster [DBG] pgmap v26: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-09-23T19:13:29.918 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 38 16 63663 63647 104.672 71.4375 0.00517747 0.00954643 2023-09-23T19:13:30.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:13:29 smithi016 bash[33624]: audit 2023-09-23T19:13:28.599317+0000 mon.a (mon.0) 75 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:13:30.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:13:29 smithi092 bash[27556]: audit 2023-09-23T19:13:28.599317+0000 mon.a (mon.0) 75 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:13:30.918 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 39 16 64824 64808 103.848 72.5625 0.00929855 0.00962355 2023-09-23T19:13:31.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:13:30 smithi016 bash[33624]: cluster 2023-09-23T19:13:29.497288+0000 mgr.y (mgr.25325) 39 : cluster [DBG] pgmap v27: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2023-09-23T19:13:31.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:13:30 smithi092 bash[27556]: cluster 2023-09-23T19:13:29.497288+0000 mgr.y (mgr.25325) 39 : cluster [DBG] pgmap v27: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2023-09-23T19:13:31.918 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-09-23T19:13:31.916923+0000 min lat: 0.00132766 max lat: 0.345747 avg lat: 0.00969633 2023-09-23T19:13:31.918 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:13:31.918 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 40 16 65987 65971 103.069 72.6875 0.00617857 0.00969633 2023-09-23T19:13:32.918 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 41 16 67135 67119 102.305 71.75 0.0282833 0.00976801 2023-09-23T19:13:33.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:13:32 smithi016 bash[33624]: cluster 2023-09-23T19:13:31.497942+0000 mgr.y (mgr.25325) 40 : cluster [DBG] pgmap v28: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-09-23T19:13:33.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:13:32 smithi092 bash[27556]: cluster 2023-09-23T19:13:31.497942+0000 mgr.y (mgr.25325) 40 : cluster [DBG] pgmap v28: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-09-23T19:13:33.918 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 42 16 68258 68242 101.54 70.1875 0.00842718 0.00984151 2023-09-23T19:13:34.918 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 43 16 69357 69341 100.776 68.6875 0.00833807 0.00991623 2023-09-23T19:13:35.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:13:34 smithi016 bash[33624]: cluster 2023-09-23T19:13:33.498414+0000 mgr.y (mgr.25325) 41 : cluster [DBG] pgmap v29: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T19:13:35.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:13:34 smithi092 bash[27556]: cluster 2023-09-23T19:13:33.498414+0000 mgr.y (mgr.25325) 41 : cluster [DBG] pgmap v29: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T19:13:35.918 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 44 16 70484 70468 100.086 70.4375 0.00508009 0.00998526 2023-09-23T19:13:36.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:13:35 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:13:35] "GET /metrics HTTP/1.1" 200 36056 "" "Prometheus/2.33.4" 2023-09-23T19:13:36.918 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 45 16 71582 71566 99.3868 68.625 0.00875705 0.0100554 2023-09-23T19:13:37.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:13:36 smithi016 bash[33624]: cluster 2023-09-23T19:13:35.499401+0000 mgr.y (mgr.25325) 42 : cluster [DBG] pgmap v30: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-09-23T19:13:37.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:13:36 smithi092 bash[27556]: cluster 2023-09-23T19:13:35.499401+0000 mgr.y (mgr.25325) 42 : cluster [DBG] pgmap v30: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-09-23T19:13:37.918 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 46 16 72658 72642 98.6881 67.25 0.0136955 0.0101262 2023-09-23T19:13:38.919 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 47 16 73735 73719 98.0204 67.3125 0.0081883 0.0101943 2023-09-23T19:13:39.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:13:38 smithi016 bash[33624]: cluster 2023-09-23T19:13:37.499798+0000 mgr.y (mgr.25325) 43 : cluster [DBG] pgmap v31: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 61 MiB/s wr, 983 op/s 2023-09-23T19:13:39.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:13:38 smithi092 bash[27556]: cluster 2023-09-23T19:13:37.499798+0000 mgr.y (mgr.25325) 43 : cluster [DBG] pgmap v31: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 61 MiB/s wr, 983 op/s 2023-09-23T19:13:39.035 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:13:38 smithi092 bash[54653]: ts=2023-09-23T19:13:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi092.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi016:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T19:13:39.919 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 48 16 74787 74771 97.3479 65.75 0.0254805 0.0102661 2023-09-23T19:13:40.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:13:39 smithi016 bash[33624]: cluster 2023-09-23T19:13:39.500503+0000 mgr.y (mgr.25325) 44 : cluster [DBG] pgmap v32: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-09-23T19:13:40.285 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:13:39 smithi092 bash[27556]: cluster 2023-09-23T19:13:39.500503+0000 mgr.y (mgr.25325) 44 : cluster [DBG] pgmap v32: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-09-23T19:13:40.919 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 49 16 75841 75825 96.7055 65.875 0.0174147 0.0103352 2023-09-23T19:13:41.919 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 50 16 76893 76877 96.0863 65.75 0.0179066 0.0104013 2023-09-23T19:13:42.919 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 51 16 77917 77901 95.457 64 0.0120677 0.0104699 2023-09-23T19:13:43.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:13:42 smithi016 bash[33624]: cluster 2023-09-23T19:13:41.501084+0000 mgr.y (mgr.25325) 45 : cluster [DBG] pgmap v33: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-09-23T19:13:43.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:13:42 smithi092 bash[27556]: cluster 2023-09-23T19:13:41.501084+0000 mgr.y (mgr.25325) 45 : cluster [DBG] pgmap v33: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-09-23T19:13:43.919 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 52 16 78988 78972 94.9085 66.9375 0.00886445 0.0105305 2023-09-23T19:13:44.919 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 53 16 80010 79994 94.3228 63.875 0.00993674 0.0105961 2023-09-23T19:13:45.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:13:44 smithi016 bash[33624]: cluster 2023-09-23T19:13:43.501517+0000 mgr.y (mgr.25325) 46 : cluster [DBG] pgmap v34: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 58 MiB/s wr, 931 op/s 2023-09-23T19:13:45.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:13:44 smithi016 bash[33624]: audit 2023-09-23T19:13:43.605248+0000 mon.a (mon.0) 76 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:13:45.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:13:44 smithi092 bash[27556]: cluster 2023-09-23T19:13:43.501517+0000 mgr.y (mgr.25325) 46 : cluster [DBG] pgmap v34: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 58 MiB/s wr, 931 op/s 2023-09-23T19:13:45.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:13:44 smithi092 bash[27556]: audit 2023-09-23T19:13:43.605248+0000 mon.a (mon.0) 76 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:13:45.919 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 54 16 81012 80996 93.7357 62.625 0.00457872 0.010662 2023-09-23T19:13:46.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:13:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:13:45] "GET /metrics HTTP/1.1" 200 36059 "" "Prometheus/2.33.4" 2023-09-23T19:13:46.919 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 55 16 82045 82029 93.2052 64.5625 0.0071257 0.0107226 2023-09-23T19:13:47.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:13:46 smithi016 bash[33624]: cluster 2023-09-23T19:13:45.502113+0000 mgr.y (mgr.25325) 47 : cluster [DBG] pgmap v35: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2023-09-23T19:13:47.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:13:46 smithi092 bash[27556]: cluster 2023-09-23T19:13:45.502113+0000 mgr.y (mgr.25325) 47 : cluster [DBG] pgmap v35: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2023-09-23T19:13:47.919 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 56 16 83028 83012 92.6378 61.4375 0.0149054 0.0107885 2023-09-23T19:13:48.920 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 57 16 84015 83999 92.0947 61.6875 0.014938 0.010852 2023-09-23T19:13:49.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:13:48 smithi016 bash[33624]: cluster 2023-09-23T19:13:47.502510+0000 mgr.y (mgr.25325) 48 : cluster [DBG] pgmap v36: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 56 MiB/s wr, 893 op/s 2023-09-23T19:13:49.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:13:48 smithi092 bash[27556]: cluster 2023-09-23T19:13:47.502510+0000 mgr.y (mgr.25325) 48 : cluster [DBG] pgmap v36: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 56 MiB/s wr, 893 op/s 2023-09-23T19:13:49.035 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:13:48 smithi092 bash[54653]: ts=2023-09-23T19:13:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi092.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi016:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T19:13:49.920 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 58 16 84979 84963 91.5456 60.25 0.0151868 0.0109162 2023-09-23T19:13:50.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:13:49 smithi016 bash[33624]: cluster 2023-09-23T19:13:49.503164+0000 mgr.y (mgr.25325) 49 : cluster [DBG] pgmap v37: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-23T19:13:50.285 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:13:49 smithi092 bash[27556]: cluster 2023-09-23T19:13:49.503164+0000 mgr.y (mgr.25325) 49 : cluster [DBG] pgmap v37: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-23T19:13:50.920 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 59 16 85929 85913 91.0002 59.375 0.0234534 0.0109826 2023-09-23T19:13:51.920 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-09-23T19:13:51.918869+0000 min lat: 0.00132766 max lat: 0.345747 avg lat: 0.0110449 2023-09-23T19:13:51.920 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:13:51.920 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 60 16 86888 86872 90.4824 59.9375 0.0295395 0.0110449 2023-09-23T19:13:52.920 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 61 16 87808 87792 89.9416 57.5 0.0147947 0.0111121 2023-09-23T19:13:53.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:13:52 smithi016 bash[33624]: cluster 2023-09-23T19:13:51.503751+0000 mgr.y (mgr.25325) 50 : cluster [DBG] pgmap v38: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-09-23T19:13:53.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:13:52 smithi092 bash[27556]: cluster 2023-09-23T19:13:51.503751+0000 mgr.y (mgr.25325) 50 : cluster [DBG] pgmap v38: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-09-23T19:13:53.920 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 62 16 88752 88736 89.4424 59 0.00991896 0.0111735 2023-09-23T19:13:54.920 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 63 16 89696 89680 88.9591 59 0.00956029 0.0112349 2023-09-23T19:13:55.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:13:54 smithi016 bash[33624]: cluster 2023-09-23T19:13:53.504175+0000 mgr.y (mgr.25325) 51 : cluster [DBG] pgmap v39: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 53 MiB/s wr, 853 op/s 2023-09-23T19:13:55.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:13:54 smithi092 bash[27556]: cluster 2023-09-23T19:13:53.504175+0000 mgr.y (mgr.25325) 51 : cluster [DBG] pgmap v39: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 53 MiB/s wr, 853 op/s 2023-09-23T19:13:55.910 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:13:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:13:55] "GET /metrics HTTP/1.1" 200 36059 "" "Prometheus/2.33.4" 2023-09-23T19:13:55.920 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 64 16 90622 90606 88.4733 57.875 0.0118178 0.0112966 2023-09-23T19:13:56.920 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 65 16 91561 91545 88.015 58.6875 0.0183903 0.0113553 2023-09-23T19:13:57.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:13:56 smithi016 bash[33624]: cluster 2023-09-23T19:13:55.504957+0000 mgr.y (mgr.25325) 52 : cluster [DBG] pgmap v40: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-09-23T19:13:57.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:13:56 smithi092 bash[27556]: cluster 2023-09-23T19:13:55.504957+0000 mgr.y (mgr.25325) 52 : cluster [DBG] pgmap v40: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-09-23T19:13:57.920 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 66 16 92515 92499 87.5848 59.625 0.00518026 0.0114105 2023-09-23T19:13:58.067 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade resume' 2023-09-23T19:13:58.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:13:58 smithi016 bash[33624]: cluster 2023-09-23T19:13:57.505358+0000 mgr.y (mgr.25325) 53 : cluster [DBG] pgmap v41: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 50 MiB/s wr, 807 op/s 2023-09-23T19:13:58.920 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 67 16 93364 93348 87.0695 53.0625 0.0178501 0.0114783 2023-09-23T19:13:59.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:13:58 smithi092 bash[27556]: cluster 2023-09-23T19:13:57.505358+0000 mgr.y (mgr.25325) 53 : cluster [DBG] pgmap v41: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 50 MiB/s wr, 807 op/s 2023-09-23T19:13:59.036 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:13:58 smithi092 bash[54653]: ts=2023-09-23T19:13:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi092.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi016:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T19:13:59.758 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:13:59 smithi016 bash[33624]: audit 2023-09-23T19:13:58.605968+0000 mon.a (mon.0) 77 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:13:59.921 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 68 16 94199 94183 86.5564 52.1875 0.0126638 0.011546 2023-09-23T19:14:00.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:13:59 smithi092 bash[27556]: audit 2023-09-23T19:13:58.605968+0000 mon.a (mon.0) 77 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:14:00.851 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:00 smithi016 bash[33624]: cluster 2023-09-23T19:13:59.505877+0000 mgr.y (mgr.25325) 54 : cluster [DBG] pgmap v42: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 925 op/s 2023-09-23T19:14:00.921 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 69 16 95094 95078 86.1126 55.9375 0.0172161 0.0116058 2023-09-23T19:14:01.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:00 smithi092 bash[27556]: cluster 2023-09-23T19:13:59.505877+0000 mgr.y (mgr.25325) 54 : cluster [DBG] pgmap v42: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 925 op/s 2023-09-23T19:14:01.921 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 70 16 95954 95938 85.6501 53.75 0.00889591 0.0116691 2023-09-23T19:14:02.263 INFO:teuthology.orchestra.run.smithi016.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:14:02.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:02 smithi016 bash[33624]: cluster 2023-09-23T19:14:01.506443+0000 mgr.y (mgr.25325) 55 : cluster [DBG] pgmap v43: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 62 MiB/s wr, 986 op/s 2023-09-23T19:14:02.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:02 smithi016 bash[33624]: audit 2023-09-23T19:14:02.254714+0000 mon.a (mon.0) 78 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:14:02.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:02 smithi016 bash[33624]: audit 2023-09-23T19:14:02.285179+0000 mon.a (mon.0) 79 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:14:02.921 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 71 16 96754 96738 85.1479 50 0.0274685 0.0117375 2023-09-23T19:14:03.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:02 smithi092 bash[27556]: cluster 2023-09-23T19:14:01.506443+0000 mgr.y (mgr.25325) 55 : cluster [DBG] pgmap v43: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 62 MiB/s wr, 986 op/s 2023-09-23T19:14:03.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:02 smithi092 bash[27556]: audit 2023-09-23T19:14:02.254714+0000 mon.a (mon.0) 78 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:14:03.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:02 smithi092 bash[27556]: audit 2023-09-23T19:14:02.285179+0000 mon.a (mon.0) 79 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:14:03.238 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'echo wait for majority of mons to upgrade' 2023-09-23T19:14:03.921 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 72 16 97628 97612 84.724 54.625 0.0247066 0.0117971 2023-09-23T19:14:04.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:03 smithi016 bash[33624]: audit 2023-09-23T19:14:02.240391+0000 mgr.y (mgr.25325) 56 : audit [DBG] from='client.25210 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:14:04.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:03 smithi016 bash[33624]: cephadm 2023-09-23T19:14:02.241916+0000 mgr.y (mgr.25325) 57 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:14:04.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:03 smithi092 bash[27556]: audit 2023-09-23T19:14:02.240391+0000 mgr.y (mgr.25325) 56 : audit [DBG] from='client.25210 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:14:04.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:03 smithi092 bash[27556]: cephadm 2023-09-23T19:14:02.241916+0000 mgr.y (mgr.25325) 57 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:14:04.921 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 73 16 98535 98519 84.3398 56.6875 0.00676805 0.0118506 2023-09-23T19:14:04.982 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:04 smithi016 bash[33624]: cluster 2023-09-23T19:14:03.506831+0000 mgr.y (mgr.25325) 58 : cluster [DBG] pgmap v44: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 48 MiB/s wr, 773 op/s 2023-09-23T19:14:05.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:04 smithi092 bash[27556]: cluster 2023-09-23T19:14:03.506831+0000 mgr.y (mgr.25325) 58 : cluster [DBG] pgmap v44: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 48 MiB/s wr, 773 op/s 2023-09-23T19:14:05.921 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 74 16 99403 99387 83.9332 54.25 0.0105821 0.0119084 2023-09-23T19:14:06.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:14:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:14:05] "GET /metrics HTTP/1.1" 200 36056 "" "Prometheus/2.33.4" 2023-09-23T19:14:06.851 INFO:teuthology.orchestra.run.smithi016.stdout:wait for majority of mons to upgrade 2023-09-23T19:14:06.921 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 75 16 100239 100223 83.5107 52.25 0.0109622 0.0119671 2023-09-23T19:14:06.922 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:06 smithi016 bash[33624]: cluster 2023-09-23T19:14:05.507597+0000 mgr.y (mgr.25325) 59 : cluster [DBG] pgmap v45: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T19:14:07.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:06 smithi092 bash[27556]: cluster 2023-09-23T19:14:05.507597+0000 mgr.y (mgr.25325) 59 : cluster [DBG] pgmap v45: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T19:14:07.733 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'while ! ceph mon versions | grep $sha1 | egrep '"'"': [23]'"'"' ; do sleep 2 ; done' 2023-09-23T19:14:07.922 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 76 16 100993 100977 83.0308 47.125 0.0121591 0.0120365 2023-09-23T19:14:08.922 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 77 16 101678 101662 82.5085 42.8125 0.0202394 0.0121119 2023-09-23T19:14:09.018 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:08 smithi016 bash[33624]: cluster 2023-09-23T19:14:07.508088+0000 mgr.y (mgr.25325) 60 : cluster [DBG] pgmap v46: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 46 MiB/s wr, 740 op/s 2023-09-23T19:14:09.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:08 smithi092 bash[27556]: cluster 2023-09-23T19:14:07.508088+0000 mgr.y (mgr.25325) 60 : cluster [DBG] pgmap v46: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 46 MiB/s wr, 740 op/s 2023-09-23T19:14:09.036 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:14:08 smithi092 bash[54653]: ts=2023-09-23T19:14:08.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi092.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi016:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T19:14:09.722 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:09 smithi016 bash[33624]: audit 2023-09-23T19:14:08.717561+0000 mon.a (mon.0) 80 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:14:09.723 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:09 smithi016 bash[33624]: audit 2023-09-23T19:14:08.730619+0000 mon.a (mon.0) 81 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:14:09.723 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:09 smithi016 bash[33624]: audit 2023-09-23T19:14:08.846293+0000 mon.a (mon.0) 82 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:14:09.723 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:09 smithi016 bash[33624]: audit 2023-09-23T19:14:08.862204+0000 mon.a (mon.0) 83 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:14:09.723 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:09 smithi016 bash[33624]: cluster 2023-09-23T19:14:09.508634+0000 mgr.y (mgr.25325) 61 : cluster [DBG] pgmap v47: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 49 MiB/s wr, 779 op/s 2023-09-23T19:14:09.723 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:09 smithi016 bash[33624]: audit 2023-09-23T19:14:09.516264+0000 mon.a (mon.0) 84 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:14:09.923 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 78 16 102375 102359 82.0091 43.5625 0.0261822 0.0121863 2023-09-23T19:14:10.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:09 smithi016 bash[33624]: audit 2023-09-23T19:14:09.517650+0000 mon.a (mon.0) 85 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:14:10.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:09 smithi016 bash[33624]: audit 2023-09-23T19:14:09.526850+0000 mon.a (mon.0) 86 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:14:10.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:09 smithi016 bash[33624]: audit 2023-09-23T19:14:09.608508+0000 mon.a (mon.0) 87 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:14:10.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:09 smithi016 bash[33624]: audit 2023-09-23T19:14:09.613732+0000 mon.a (mon.0) 88 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:14:10.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:09 smithi016 bash[33624]: audit 2023-09-23T19:14:09.616541+0000 mon.a (mon.0) 89 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T19:14:10.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:09 smithi016 bash[33624]: audit 2023-09-23T19:14:09.617804+0000 mon.a (mon.0) 90 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-09-23T19:14:10.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:09 smithi016 bash[33624]: cephadm 2023-09-23T19:14:09.619086+0000 mgr.y (mgr.25325) 62 : cephadm [INF] Upgrade: It appears safe to stop mon.c 2023-09-23T19:14:10.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:09 smithi092 bash[27556]: audit 2023-09-23T19:14:08.717561+0000 mon.a (mon.0) 80 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:14:10.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:09 smithi092 bash[27556]: audit 2023-09-23T19:14:08.730619+0000 mon.a (mon.0) 81 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:14:10.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:09 smithi092 bash[27556]: audit 2023-09-23T19:14:08.846293+0000 mon.a (mon.0) 82 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:14:10.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:09 smithi092 bash[27556]: audit 2023-09-23T19:14:08.862204+0000 mon.a (mon.0) 83 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:14:10.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:09 smithi092 bash[27556]: cluster 2023-09-23T19:14:09.508634+0000 mgr.y (mgr.25325) 61 : cluster [DBG] pgmap v47: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 49 MiB/s wr, 779 op/s 2023-09-23T19:14:10.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:09 smithi092 bash[27556]: audit 2023-09-23T19:14:09.516264+0000 mon.a (mon.0) 84 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:14:10.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:09 smithi092 bash[27556]: audit 2023-09-23T19:14:09.517650+0000 mon.a (mon.0) 85 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:14:10.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:09 smithi092 bash[27556]: audit 2023-09-23T19:14:09.526850+0000 mon.a (mon.0) 86 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:14:10.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:09 smithi092 bash[27556]: audit 2023-09-23T19:14:09.608508+0000 mon.a (mon.0) 87 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:14:10.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:09 smithi092 bash[27556]: audit 2023-09-23T19:14:09.613732+0000 mon.a (mon.0) 88 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:14:10.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:09 smithi092 bash[27556]: audit 2023-09-23T19:14:09.616541+0000 mon.a (mon.0) 89 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T19:14:10.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:09 smithi092 bash[27556]: audit 2023-09-23T19:14:09.617804+0000 mon.a (mon.0) 90 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-09-23T19:14:10.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:09 smithi092 bash[27556]: cephadm 2023-09-23T19:14:09.619086+0000 mgr.y (mgr.25325) 62 : cephadm [INF] Upgrade: It appears safe to stop mon.c 2023-09-23T19:14:10.923 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 79 16 103204 103188 81.6268 51.8125 0.00603392 0.0122445 2023-09-23T19:14:11.923 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-09-23T19:14:11.921775+0000 min lat: 0.00132766 max lat: 0.345747 avg lat: 0.0122943 2023-09-23T19:14:11.923 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:14:11.923 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 80 16 104065 104049 81.2791 53.8125 0.0214786 0.0122943 2023-09-23T19:14:12.923 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 81 16 104815 104799 80.8543 46.875 0.0258686 0.0123616 2023-09-23T19:14:13.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:12 smithi016 bash[33624]: cluster 2023-09-23T19:14:11.509366+0000 mgr.y (mgr.25325) 63 : cluster [DBG] pgmap v48: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 59 MiB/s wr, 947 op/s 2023-09-23T19:14:13.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:12 smithi092 bash[27556]: cluster 2023-09-23T19:14:11.509366+0000 mgr.y (mgr.25325) 63 : cluster [DBG] pgmap v48: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 59 MiB/s wr, 947 op/s 2023-09-23T19:14:13.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:13 smithi016 bash[33624]: audit 2023-09-23T19:14:12.664093+0000 mon.a (mon.0) 91 : audit [DBG] from='client.? 172.21.15.16:0/4024211665' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:14:13.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:13 smithi016 bash[33624]: audit 2023-09-23T19:14:13.066564+0000 mon.a (mon.0) 92 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:14:13.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:13 smithi016 bash[33624]: audit 2023-09-23T19:14:13.068124+0000 mon.a (mon.0) 93 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T19:14:13.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:13 smithi016 bash[33624]: audit 2023-09-23T19:14:13.073670+0000 mon.a (mon.0) 94 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T19:14:13.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:13 smithi016 bash[33624]: audit 2023-09-23T19:14:13.075842+0000 mon.a (mon.0) 95 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:14:13.923 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 82 16 105597 105581 80.4642 48.875 0.00760472 0.0124207 2023-09-23T19:14:14.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:13 smithi092 bash[27556]: audit 2023-09-23T19:14:12.664093+0000 mon.a (mon.0) 91 : audit [DBG] from='client.? 172.21.15.16:0/4024211665' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:14:14.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:13 smithi092 bash[27556]: audit 2023-09-23T19:14:13.066564+0000 mon.a (mon.0) 92 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:14:14.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:13 smithi092 bash[27556]: audit 2023-09-23T19:14:13.068124+0000 mon.a (mon.0) 93 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T19:14:14.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:13 smithi092 bash[27556]: audit 2023-09-23T19:14:13.073670+0000 mon.a (mon.0) 94 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T19:14:14.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:13 smithi092 bash[27556]: audit 2023-09-23T19:14:13.075842+0000 mon.a (mon.0) 95 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:14:14.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:14 smithi016 bash[33624]: cephadm 2023-09-23T19:14:13.056347+0000 mgr.y (mgr.25325) 64 : cephadm [INF] Upgrade: Updating mon.c 2023-09-23T19:14:14.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:14 smithi016 bash[33624]: cephadm 2023-09-23T19:14:13.077863+0000 mgr.y (mgr.25325) 65 : cephadm [INF] Deploying daemon mon.c on smithi016 2023-09-23T19:14:14.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:14 smithi016 bash[33624]: cluster 2023-09-23T19:14:13.509962+0000 mgr.y (mgr.25325) 66 : cluster [DBG] pgmap v49: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 43 MiB/s wr, 684 op/s 2023-09-23T19:14:14.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:14 smithi016 bash[33624]: audit 2023-09-23T19:14:13.606604+0000 mon.a (mon.0) 96 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:14:14.923 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 83 16 106370 106354 80.0768 48.3125 0.019701 0.0124817 2023-09-23T19:14:15.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:14 smithi092 bash[27556]: cephadm 2023-09-23T19:14:13.056347+0000 mgr.y (mgr.25325) 64 : cephadm [INF] Upgrade: Updating mon.c 2023-09-23T19:14:15.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:14 smithi092 bash[27556]: cephadm 2023-09-23T19:14:13.077863+0000 mgr.y (mgr.25325) 65 : cephadm [INF] Deploying daemon mon.c on smithi016 2023-09-23T19:14:15.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:14 smithi092 bash[27556]: cluster 2023-09-23T19:14:13.509962+0000 mgr.y (mgr.25325) 66 : cluster [DBG] pgmap v49: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 43 MiB/s wr, 684 op/s 2023-09-23T19:14:15.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:14 smithi092 bash[27556]: audit 2023-09-23T19:14:13.606604+0000 mon.a (mon.0) 96 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:14:15.893 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:14:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:14:15] "GET /metrics HTTP/1.1" 200 36062 "" "Prometheus/2.33.4" 2023-09-23T19:14:15.893 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:15 smithi016 systemd[1]: Stopping Ceph mon.c for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7... 2023-09-23T19:14:15.923 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 84 16 107127 107111 79.6867 47.3125 0.00762063 0.0125426 2023-09-23T19:14:16.256 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:16 smithi016 bash[33624]: cluster 2023-09-23T19:14:15.078528+0000 mon.a (mon.0) 97 : cluster [DBG] mgrmap e38: y(active, since 91s), standbys: x 2023-09-23T19:14:16.256 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:16 smithi016 bash[33624]: audit 2023-09-23T19:14:15.426946+0000 mon.a (mon.0) 98 : audit [DBG] from='client.? 172.21.15.16:0/183737010' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:14:16.256 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:16 smithi016 bash[33624]: cluster 2023-09-23T19:14:15.510744+0000 mgr.y (mgr.25325) 67 : cluster [DBG] pgmap v50: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-09-23T19:14:16.256 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:16 smithi016 bash[33624]: debug 2023-09-23T19:14:16.089+0000 7f9549198700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-09-23T19:14:16.256 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:16 smithi016 bash[33624]: debug 2023-09-23T19:14:16.089+0000 7f9549198700 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2023-09-23T19:14:16.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:16 smithi092 bash[27556]: cluster 2023-09-23T19:14:15.078528+0000 mon.a (mon.0) 97 : cluster [DBG] mgrmap e38: y(active, since 91s), standbys: x 2023-09-23T19:14:16.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:16 smithi092 bash[27556]: audit 2023-09-23T19:14:15.426946+0000 mon.a (mon.0) 98 : audit [DBG] from='client.? 172.21.15.16:0/183737010' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:14:16.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:16 smithi092 bash[27556]: cluster 2023-09-23T19:14:15.510744+0000 mgr.y (mgr.25325) 67 : cluster [DBG] pgmap v50: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-09-23T19:14:16.923 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 85 16 107919 107903 79.3315 49.5 0.0146802 0.0125988 2023-09-23T19:14:17.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:16 smithi016 bash[77889]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-mon-c 2023-09-23T19:14:17.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:17 smithi016 bash[77941]: Error response from daemon: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-mon-c 2023-09-23T19:14:17.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:17 smithi016 systemd[1]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@mon.c.service: Succeeded. 2023-09-23T19:14:17.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:17 smithi016 systemd[1]: Stopped Ceph mon.c for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7. 2023-09-23T19:14:17.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:17 smithi016 systemd[1]: Started Ceph mon.c for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7. 2023-09-23T19:14:17.923 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 86 16 108635 108619 78.9294 44.75 0.0200555 0.0126631 2023-09-23T19:14:18.923 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 87 16 109395 109379 78.5681 47.5 0.0152439 0.0127215 2023-09-23T19:14:19.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.737+0000 7f3959d1ac80 0 set uid:gid to 167:167 (ceph:ceph) 2023-09-23T19:14:19.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.737+0000 7f3959d1ac80 0 ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable), process ceph-mon, pid 7 2023-09-23T19:14:19.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.737+0000 7f3959d1ac80 0 pidfile_write: ignore empty --pid-file 2023-09-23T19:14:19.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.741+0000 7f3959d1ac80 0 load: jerasure load: lrc 2023-09-23T19:14:19.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: RocksDB version: 7.9.2 2023-09-23T19:14:19.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Git sha 0 2023-09-23T19:14:19.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Compile date 2023-09-22 15:19:43 2023-09-23T19:14:19.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: DB SUMMARY 2023-09-23T19:14:19.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: DB Session ID: HC54CMMNV61PBYNG9WWF 2023-09-23T19:14:19.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: CURRENT file: CURRENT 2023-09-23T19:14:19.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: IDENTITY file: IDENTITY 2023-09-23T19:14:19.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: MANIFEST file: MANIFEST-000005 size: 1269 Bytes 2023-09-23T19:14:19.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000026.sst 2023-09-23T19:14:19.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000024.log size: 1732753 ; 2023-09-23T19:14:19.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.error_if_exists: 0 2023-09-23T19:14:19.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.create_if_missing: 0 2023-09-23T19:14:19.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.paranoid_checks: 1 2023-09-23T19:14:19.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.flush_verify_memtable_count: 1 2023-09-23T19:14:19.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-23T19:14:19.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-09-23T19:14:19.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.env: 0x555f4a2a4720 2023-09-23T19:14:19.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.fs: PosixFileSystem 2023-09-23T19:14:19.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.info_log: 0x555f4d56b400 2023-09-23T19:14:19.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.max_file_opening_threads: 16 2023-09-23T19:14:19.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.statistics: (nil) 2023-09-23T19:14:19.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.use_fsync: 0 2023-09-23T19:14:19.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.max_log_file_size: 0 2023-09-23T19:14:19.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T19:14:19.040 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T19:14:19.040 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.keep_log_file_num: 1000 2023-09-23T19:14:19.040 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.recycle_log_file_num: 0 2023-09-23T19:14:19.040 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.allow_fallocate: 1 2023-09-23T19:14:19.040 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.allow_mmap_reads: 0 2023-09-23T19:14:19.040 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.allow_mmap_writes: 0 2023-09-23T19:14:19.040 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.use_direct_reads: 0 2023-09-23T19:14:19.040 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T19:14:19.040 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.create_missing_column_families: 0 2023-09-23T19:14:19.040 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.db_log_dir: 2023-09-23T19:14:19.040 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.wal_dir: 2023-09-23T19:14:19.040 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T19:14:19.040 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T19:14:19.041 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T19:14:19.041 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T19:14:19.041 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T19:14:19.041 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T19:14:19.041 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.advise_random_on_open: 1 2023-09-23T19:14:19.041 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.db_write_buffer_size: 0 2023-09-23T19:14:19.041 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.write_buffer_manager: 0x555f4c7f85a0 2023-09-23T19:14:19.041 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T19:14:19.041 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T19:14:19.041 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T19:14:19.041 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.rate_limiter: (nil) 2023-09-23T19:14:19.041 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T19:14:19.041 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.wal_recovery_mode: 2 2023-09-23T19:14:19.042 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.enable_thread_tracking: 0 2023-09-23T19:14:19.042 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.enable_pipelined_write: 0 2023-09-23T19:14:19.042 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.unordered_write: 0 2023-09-23T19:14:19.042 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T19:14:19.042 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T19:14:19.042 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T19:14:19.042 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T19:14:19.042 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.row_cache: None 2023-09-23T19:14:19.042 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.wal_filter: None 2023-09-23T19:14:19.042 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T19:14:19.042 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.allow_ingest_behind: 0 2023-09-23T19:14:19.042 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.two_write_queues: 0 2023-09-23T19:14:19.043 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.manual_wal_flush: 0 2023-09-23T19:14:19.043 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.wal_compression: 0 2023-09-23T19:14:19.043 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.atomic_flush: 0 2023-09-23T19:14:19.043 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T19:14:19.043 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T19:14:19.043 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T19:14:19.043 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.log_readahead_size: 0 2023-09-23T19:14:19.043 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-23T19:14:19.043 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.best_efforts_recovery: 0 2023-09-23T19:14:19.043 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-23T19:14:19.043 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-23T19:14:19.043 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.allow_data_in_errors: 0 2023-09-23T19:14:19.043 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.db_host_id: __hostname__ 2023-09-23T19:14:19.044 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.enforce_single_del_contracts: true 2023-09-23T19:14:19.044 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.max_background_jobs: 2 2023-09-23T19:14:19.044 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.max_background_compactions: -1 2023-09-23T19:14:19.044 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:14:18 smithi092 bash[54653]: ts=2023-09-23T19:14:18.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi092.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi016:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T19:14:19.045 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.max_subcompactions: 1 2023-09-23T19:14:19.045 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T19:14:19.045 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T19:14:19.046 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T19:14:19.046 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.max_total_wal_size: 0 2023-09-23T19:14:19.046 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T19:14:19.046 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T19:14:19.046 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T19:14:19.046 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T19:14:19.046 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.max_open_files: -1 2023-09-23T19:14:19.046 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.bytes_per_sync: 0 2023-09-23T19:14:19.046 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T19:14:19.046 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T19:14:19.046 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.compaction_readahead_size: 0 2023-09-23T19:14:19.046 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.max_background_flushes: -1 2023-09-23T19:14:19.046 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Compression algorithms supported: 2023-09-23T19:14:19.047 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T19:14:19.047 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: kZSTD supported: 0 2023-09-23T19:14:19.047 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: kXpressCompression supported: 0 2023-09-23T19:14:19.047 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: kLZ4HCCompression supported: 1 2023-09-23T19:14:19.047 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: kZlibCompression supported: 1 2023-09-23T19:14:19.047 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: kSnappyCompression supported: 1 2023-09-23T19:14:19.047 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: kLZ4Compression supported: 1 2023-09-23T19:14:19.047 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: kBZip2Compression supported: 0 2023-09-23T19:14:19.047 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T19:14:19.047 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: DMutex implementation: pthread_mutex_t 2023-09-23T19:14:19.047 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 2023-09-23T19:14:19.047 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-09-23T19:14:19.047 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T19:14:19.047 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.merge_operator: 2023-09-23T19:14:19.048 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.compaction_filter: None 2023-09-23T19:14:19.048 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.compaction_filter_factory: None 2023-09-23T19:14:19.048 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.sst_partitioner_factory: None 2023-09-23T19:14:19.048 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T19:14:19.048 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T19:14:19.048 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x555f4d56b520) 2023-09-23T19:14:19.048 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: cache_index_and_filter_blocks: 1 2023-09-23T19:14:19.048 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T19:14:19.048 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T19:14:19.048 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: pin_top_level_index_and_filter: 1 2023-09-23T19:14:19.048 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: index_type: 0 2023-09-23T19:14:19.048 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: data_block_index_type: 0 2023-09-23T19:14:19.048 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: index_shortening: 1 2023-09-23T19:14:19.048 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: data_block_hash_table_util_ratio: 0.750000 2023-09-23T19:14:19.049 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: checksum: 4 2023-09-23T19:14:19.049 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: no_block_cache: 0 2023-09-23T19:14:19.049 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: block_cache: 0x555f4c87b350 2023-09-23T19:14:19.049 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: block_cache_name: BinnedLRUCache 2023-09-23T19:14:19.049 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: block_cache_options: 2023-09-23T19:14:19.049 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: capacity : 536870912 2023-09-23T19:14:19.049 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: num_shard_bits : 4 2023-09-23T19:14:19.049 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: strict_capacity_limit : 0 2023-09-23T19:14:19.049 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: high_pri_pool_ratio: 0.000 2023-09-23T19:14:19.049 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: block_cache_compressed: (nil) 2023-09-23T19:14:19.049 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: persistent_cache: (nil) 2023-09-23T19:14:19.049 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: block_size: 4096 2023-09-23T19:14:19.049 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: block_size_deviation: 10 2023-09-23T19:14:19.049 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: block_restart_interval: 16 2023-09-23T19:14:19.050 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: index_block_restart_interval: 1 2023-09-23T19:14:19.050 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: metadata_block_size: 4096 2023-09-23T19:14:19.050 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: partition_filters: 0 2023-09-23T19:14:19.050 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: use_delta_encoding: 1 2023-09-23T19:14:19.050 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: filter_policy: bloomfilter 2023-09-23T19:14:19.050 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: whole_key_filtering: 1 2023-09-23T19:14:19.050 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: verify_compression: 0 2023-09-23T19:14:19.050 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: read_amp_bytes_per_bit: 0 2023-09-23T19:14:19.050 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: format_version: 5 2023-09-23T19:14:19.050 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: enable_index_compression: 1 2023-09-23T19:14:19.050 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: block_align: 0 2023-09-23T19:14:19.050 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: max_auto_readahead_size: 262144 2023-09-23T19:14:19.050 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: prepopulate_block_cache: 0 2023-09-23T19:14:19.051 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: initial_auto_readahead_size: 8192 2023-09-23T19:14:19.051 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: num_file_reads_for_auto_readahead: 2 2023-09-23T19:14:19.051 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.write_buffer_size: 33554432 2023-09-23T19:14:19.051 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.max_write_buffer_number: 2 2023-09-23T19:14:19.051 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.compression: NoCompression 2023-09-23T19:14:19.051 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.bottommost_compression: Disabled 2023-09-23T19:14:19.051 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.prefix_extractor: nullptr 2023-09-23T19:14:19.051 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T19:14:19.051 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.num_levels: 7 2023-09-23T19:14:19.051 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T19:14:19.051 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T19:14:19.051 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T19:14:19.051 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T19:14:19.052 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T19:14:19.052 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T19:14:19.052 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T19:14:19.052 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T19:14:19.052 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-23T19:14:19.052 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T19:14:19.052 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-09-23T19:14:19.052 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-09-23T19:14:19.052 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T19:14:19.052 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.compression_opts.level: 32767 2023-09-23T19:14:19.052 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.compression_opts.strategy: 0 2023-09-23T19:14:19.052 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T19:14:19.052 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T19:14:19.052 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-09-23T19:14:19.053 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-23T19:14:19.053 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.compression_opts.enabled: false 2023-09-23T19:14:19.053 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-09-23T19:14:19.053 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T19:14:19.053 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T19:14:19.053 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T19:14:19.053 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.target_file_size_base: 67108864 2023-09-23T19:14:19.053 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T19:14:19.053 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T19:14:19.053 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T19:14:19.053 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T19:14:19.053 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T19:14:19.053 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T19:14:19.054 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T19:14:19.054 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T19:14:19.054 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T19:14:19.054 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T19:14:19.054 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T19:14:19.054 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T19:14:19.054 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T19:14:19.054 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-09-23T19:14:19.054 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.arena_block_size: 1048576 2023-09-23T19:14:19.054 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T19:14:19.054 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T19:14:19.054 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.disable_auto_compactions: 0 2023-09-23T19:14:19.054 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T19:14:19.054 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T19:14:19.055 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T19:14:19.055 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T19:14:19.055 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T19:14:19.055 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T19:14:19.055 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T19:14:19.055 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T19:14:19.055 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T19:14:19.055 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T19:14:19.055 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.table_properties_collectors: 2023-09-23T19:14:19.055 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.inplace_update_support: 0 2023-09-23T19:14:19.056 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T19:14:19.056 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T19:14:19.056 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T19:14:19.056 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T19:14:19.056 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.bloom_locality: 0 2023-09-23T19:14:19.056 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.max_successive_merges: 0 2023-09-23T19:14:19.056 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T19:14:19.056 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.paranoid_file_checks: 0 2023-09-23T19:14:19.056 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.force_consistency_checks: 1 2023-09-23T19:14:19.056 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.report_bg_io_stats: 0 2023-09-23T19:14:19.056 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.ttl: 2592000 2023-09-23T19:14:19.056 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T19:14:19.056 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2023-09-23T19:14:19.057 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.preserve_internal_time_seconds: 0 2023-09-23T19:14:19.057 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.enable_blob_files: false 2023-09-23T19:14:19.057 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.min_blob_size: 0 2023-09-23T19:14:19.057 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.blob_file_size: 268435456 2023-09-23T19:14:19.057 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.blob_compression_type: NoCompression 2023-09-23T19:14:19.057 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.enable_blob_garbage_collection: false 2023-09-23T19:14:19.057 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-23T19:14:19.057 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-09-23T19:14:19.057 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.blob_compaction_readahead_size: 0 2023-09-23T19:14:19.057 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.blob_file_starting_level: 0 2023-09-23T19:14:19.057 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-09-23T19:14:19.057 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f394f539700 3 rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 26.sst 2023-09-23T19:14:19.057 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 28, last_sequence is 17780, log_number is 24,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T19:14:19.058 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 24 2023-09-23T19:14:19.058 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: d0ce1bad-73fb-4d57-a8f1-6492dd1c4fca 2023-09-23T19:14:19.058 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1695496458750768, "job": 1, "event": "recovery_started", "wal_files": [24]} 2023-09-23T19:14:19.058 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.745+0000 7f3959d1ac80 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #24 mode 2 2023-09-23T19:14:19.058 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.757+0000 7f3959d1ac80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1695496458761773, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 29, "file_size": 1152994, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 17775, "largest_seqno": 18060, "table_properties": {"data_size": 1150474, "index_size": 1252, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 261, "raw_key_size": 2245, "raw_average_key_size": 22, "raw_value_size": 1147521, "raw_average_value_size": 11361, "num_data_blocks": 54, "num_entries": 101, "num_filter_entries": 101, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1695496458, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "d0ce1bad-73fb-4d57-a8f1-6492dd1c4fca", "db_session_id": "HC54CMMNV61PBYNG9WWF", "orig_file_number": 29, "seqno_to_time_mapping": "N/A"}} 2023-09-23T19:14:19.058 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.757+0000 7f3959d1ac80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1695496458762160, "job": 1, "event": "recovery_finished"} 2023-09-23T19:14:19.058 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.757+0000 7f3959d1ac80 4 rocksdb: [db/version_set.cc:5047] Creating manifest 31 2023-09-23T19:14:19.058 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.761+0000 7f3959d1ac80 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000024.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-23T19:14:19.058 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.761+0000 7f3959d1ac80 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x555f4c91c000 2023-09-23T19:14:19.058 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.761+0000 7f3959d1ac80 4 rocksdb: DB pointer 0x555f4c90a000 2023-09-23T19:14:19.058 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.761+0000 7f3959d1ac80 0 starting mon.c rank 2 at public addrs [v2:172.21.15.16:3301/0,v1:172.21.15.16:6790/0] at bind addrs [v2:172.21.15.16:3301/0,v1:172.21.15.16:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 2023-09-23T19:14:19.058 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.761+0000 7f3959d1ac80 1 mon.c@-1(???) e3 preinit fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 2023-09-23T19:14:19.059 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.761+0000 7f3959d1ac80 0 mon.c@-1(???).mds e6 new map 2023-09-23T19:14:19.059 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.765+0000 7f3959d1ac80 0 mon.c@-1(???).mds e6 print_map 2023-09-23T19:14:19.059 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: e6 2023-09-23T19:14:19.059 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T19:14:19.059 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T19:14:19.059 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: legacy client fscid: 1 2023-09-23T19:14:19.059 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: 2023-09-23T19:14:19.059 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: Filesystem 'foo' (1) 2023-09-23T19:14:19.059 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: fs_name foo 2023-09-23T19:14:19.059 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: epoch 5 2023-09-23T19:14:19.059 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: flags 12 joinable allow_snaps allow_multimds_snaps 2023-09-23T19:14:19.059 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: created 2023-09-23T19:05:52.117022+0000 2023-09-23T19:14:19.059 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: modified 2023-09-23T19:05:58.952970+0000 2023-09-23T19:14:19.060 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: tableserver 0 2023-09-23T19:14:19.060 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: root 0 2023-09-23T19:14:19.060 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: session_timeout 60 2023-09-23T19:14:19.060 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: session_autoclose 300 2023-09-23T19:14:19.060 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: max_file_size 1099511627776 2023-09-23T19:14:19.060 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: required_client_features {} 2023-09-23T19:14:19.060 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: last_failure 0 2023-09-23T19:14:19.060 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: last_failure_osd_epoch 0 2023-09-23T19:14:19.060 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T19:14:19.060 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: max_mds 1 2023-09-23T19:14:19.060 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: in 0 2023-09-23T19:14:19.060 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: up {0=14655} 2023-09-23T19:14:19.060 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: failed 2023-09-23T19:14:19.060 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: damaged 2023-09-23T19:14:19.061 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: stopped 2023-09-23T19:14:19.061 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: data_pools [4] 2023-09-23T19:14:19.061 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: metadata_pool 3 2023-09-23T19:14:19.061 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: inline_data disabled 2023-09-23T19:14:19.061 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: balancer 2023-09-23T19:14:19.061 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: bal_rank_mask -1 2023-09-23T19:14:19.061 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: standby_count_wanted 1 2023-09-23T19:14:19.061 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: [mds.foo.smithi016.uncssv{0:14655} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.16:6834/161572387,v1:172.21.15.16:6835/161572387] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T19:14:19.061 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: 2023-09-23T19:14:19.061 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: 2023-09-23T19:14:19.061 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: Standby daemons: 2023-09-23T19:14:19.061 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: 2023-09-23T19:14:19.061 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: [mds.foo.smithi092.vbpqks{-1:24520} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.92:6832/1118427195,v1:172.21.15.92:6833/1118427195] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T19:14:19.062 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.769+0000 7f3959d1ac80 0 mon.c@-1(???).osd e76 crush map has features 3314933000854323200, adjusting msgr requires 2023-09-23T19:14:19.062 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.769+0000 7f3959d1ac80 0 mon.c@-1(???).osd e76 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T19:14:19.062 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.769+0000 7f3959d1ac80 0 mon.c@-1(???).osd e76 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T19:14:19.062 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.769+0000 7f3959d1ac80 0 mon.c@-1(???).osd e76 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T19:14:19.062 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.769+0000 7f3959d1ac80 1 mon.c@-1(???).paxosservice(auth 1..21) refresh upgraded, format 0 -> 3 2023-09-23T19:14:19.062 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.773+0000 7f394852b700 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-09-23T19:14:19.062 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.773+0000 7f394852b700 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2023-09-23T19:14:19.062 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: ** DB Stats ** 2023-09-23T19:14:19.062 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T19:14:19.062 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T19:14:19.062 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T19:14:19.062 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T19:14:19.063 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T19:14:19.063 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T19:14:19.063 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T19:14:19.063 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: ** Compaction Stats [default] ** 2023-09-23T19:14:19.063 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T19:14:19.063 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-09-23T19:14:19.063 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: L0 1/0 1.10 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 263.8 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-09-23T19:14:19.063 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: L6 1/0 33.24 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-09-23T19:14:19.063 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: Sum 2/0 34.34 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 263.8 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-09-23T19:14:19.063 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 263.8 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-09-23T19:14:19.063 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: ** Compaction Stats [default] ** 2023-09-23T19:14:19.063 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T19:14:19.063 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T19:14:19.063 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 263.8 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-09-23T19:14:19.064 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-09-23T19:14:19.064 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T19:14:19.064 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: Flush(GB): cumulative 0.001, interval 0.001 2023-09-23T19:14:19.064 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T19:14:19.064 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T19:14:19.064 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T19:14:19.064 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: AddFile(Keys): cumulative 0, interval 0 2023-09-23T19:14:19.064 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: Cumulative compaction: 0.00 GB write, 39.69 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T19:14:19.064 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: Interval compaction: 0.00 GB write, 39.69 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T19:14:19.064 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T19:14:19.064 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: Block cache BinnedLRUCache@0x555f4c87b350#7 capacity: 512.00 MB usage: 497.67 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 4.5e-05 secs_since: 0 2023-09-23T19:14:19.064 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: Block cache entry stats(count,size,portion): DataBlock(21,431.92 KB,0.0823826%) FilterBlock(2,8.34 KB,0.00159144%) IndexBlock(2,57.41 KB,0.0109494%) Misc(1,0.00 KB,0%) 2023-09-23T19:14:19.065 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: ** File Read Latency Histogram By Level [default] ** 2023-09-23T19:14:19.065 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.777+0000 7f3959d1ac80 20 expand_channel_meta expand map: {default=false} 2023-09-23T19:14:19.065 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.777+0000 7f3959d1ac80 20 expand_channel_meta from 'false' to 'false' 2023-09-23T19:14:19.065 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.777+0000 7f3959d1ac80 20 expand_channel_meta expanded map: {default=false} 2023-09-23T19:14:19.065 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.777+0000 7f3959d1ac80 20 expand_channel_meta expand map: {default=info} 2023-09-23T19:14:19.065 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.777+0000 7f3959d1ac80 20 expand_channel_meta from 'info' to 'info' 2023-09-23T19:14:19.065 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.777+0000 7f3959d1ac80 20 expand_channel_meta expanded map: {default=info} 2023-09-23T19:14:19.065 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.777+0000 7f3959d1ac80 20 expand_channel_meta expand map: {default=daemon} 2023-09-23T19:14:19.065 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.777+0000 7f3959d1ac80 20 expand_channel_meta from 'daemon' to 'daemon' 2023-09-23T19:14:19.065 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.777+0000 7f3959d1ac80 20 expand_channel_meta expanded map: {default=daemon} 2023-09-23T19:14:19.065 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.777+0000 7f3959d1ac80 20 expand_channel_meta expand map: {default=debug} 2023-09-23T19:14:19.065 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.777+0000 7f3959d1ac80 20 expand_channel_meta from 'debug' to 'debug' 2023-09-23T19:14:19.065 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:14:18 smithi016 bash[77995]: debug 2023-09-23T19:14:18.777+0000 7f3959d1ac80 20 expand_channel_meta expanded map: {default=debug} 2023-09-23T19:14:19.923 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 88 16 110197 110181 78.2448 50.125 0.0316679 0.0127739 2023-09-23T19:14:20.923 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 89 16 111006 110990 77.9337 50.5625 0.0117748 0.0128258 2023-09-23T19:14:21.924 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 90 10 111814 111804 77.633 50.875 0.0204492 0.0128749 2023-09-23T19:14:22.924 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Total time run: 90.0158 2023-09-23T19:14:22.924 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Total writes made: 111814 2023-09-23T19:14:22.924 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Write size: 65536 2023-09-23T19:14:22.924 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Object size: 65536 2023-09-23T19:14:22.925 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Bandwidth (MB/sec): 77.635 2023-09-23T19:14:22.925 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Stddev Bandwidth: 29.2528 2023-09-23T19:14:22.925 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Max bandwidth (MB/sec): 165 2023-09-23T19:14:22.925 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Min bandwidth (MB/sec): 42.8125 2023-09-23T19:14:22.925 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Average IOPS: 1242 2023-09-23T19:14:22.925 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Stddev IOPS: 468.044 2023-09-23T19:14:22.925 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Max IOPS: 2640 2023-09-23T19:14:22.925 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Min IOPS: 685 2023-09-23T19:14:22.925 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Average Latency(s): 0.0128754 2023-09-23T19:14:22.925 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Stddev Latency(s): 0.00905916 2023-09-23T19:14:22.925 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Max latency(s): 0.345747 2023-09-23T19:14:22.925 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Min latency(s): 0.00132766 2023-09-23T19:14:22.934 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T19:14:26.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:14:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:14:25] "GET /metrics HTTP/1.1" 200 36062 "" "Prometheus/2.33.4" 2023-09-23T19:14:26.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:25 smithi092 bash[27556]: cluster 2023-09-23T19:14:18.792136+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-09-23T19:14:26.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:25 smithi092 bash[27556]: cluster 2023-09-23T19:14:24.804758+0000 mon.a (mon.0) 113 : cluster [INF] mon.a calling monitor election 2023-09-23T19:14:26.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:25 smithi092 bash[27556]: cluster 2023-09-23T19:14:24.812813+0000 mon.a (mon.0) 114 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T19:14:26.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:25 smithi092 bash[27556]: cluster 2023-09-23T19:14:24.812988+0000 mon.b (mon.1) 57 : cluster [INF] mon.b calling monitor election 2023-09-23T19:14:26.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:25 smithi092 bash[27556]: cluster 2023-09-23T19:14:24.830265+0000 mon.a (mon.0) 115 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.16:3300/0,v1:172.21.15.16:6789/0],b=[v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0],c=[v2:172.21.15.16:3301/0,v1:172.21.15.16:6790/0]} removed_ranks: {} 2023-09-23T19:14:26.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:25 smithi092 bash[27556]: cluster 2023-09-23T19:14:24.831105+0000 mon.a (mon.0) 116 : cluster [DBG] fsmap foo:1 {0=foo.smithi016.uncssv=up:active} 1 up:standby 2023-09-23T19:14:26.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:25 smithi092 bash[27556]: cluster 2023-09-23T19:14:24.831235+0000 mon.a (mon.0) 117 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2023-09-23T19:14:26.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:25 smithi092 bash[27556]: cluster 2023-09-23T19:14:24.838664+0000 mon.a (mon.0) 118 : cluster [DBG] mgrmap e38: y(active, since 101s), standbys: x 2023-09-23T19:14:26.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:25 smithi092 bash[27556]: cluster 2023-09-23T19:14:24.839137+0000 mon.a (mon.0) 119 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-09-23T19:14:26.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:25 smithi092 bash[27556]: cluster 2023-09-23T19:14:24.839207+0000 mon.a (mon.0) 120 : cluster [INF] Cluster is now healthy 2023-09-23T19:14:26.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:25 smithi092 bash[27556]: cluster 2023-09-23T19:14:24.850740+0000 mon.a (mon.0) 121 : cluster [INF] overall HEALTH_OK 2023-09-23T19:14:26.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:25 smithi092 bash[27556]: cluster 2023-09-23T19:14:25.513772+0000 mgr.y (mgr.25325) 72 : cluster [DBG] pgmap v55: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 51 MiB/s wr, 915 op/s 2023-09-23T19:14:27.382 INFO:teuthology.orchestra.run.smithi016.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 2 2023-09-23T19:14:27.786 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:27 smithi092 bash[27556]: audit 2023-09-23T19:14:27.359368+0000 mon.a (mon.0) 122 : audit [DBG] from='client.? 172.21.15.16:0/1182051871' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:14:28.786 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:28 smithi092 bash[27556]: cluster 2023-09-23T19:14:27.514221+0000 mgr.y (mgr.25325) 73 : cluster [DBG] pgmap v56: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 30 MiB/s wr, 582 op/s 2023-09-23T19:14:28.786 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:14:28 smithi092 bash[54653]: ts=2023-09-23T19:14:28.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi092.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi016:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T19:14:29.261 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:14:28 smithi016 bash[70992]: debug 2023-09-23T19:14:28.785+0000 7fb2a5a7c700 -1 mgr.server handle_report got status from non-daemon mon.c 2023-09-23T19:14:29.262 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T19:14:29.786 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:29 smithi092 bash[27556]: audit 2023-09-23T19:14:28.608814+0000 mon.a (mon.0) 123 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:14:30.786 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:30 smithi092 bash[27556]: cluster 2023-09-23T19:14:29.514833+0000 mgr.y (mgr.25325) 74 : cluster [DBG] pgmap v57: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 30 MiB/s wr, 624 op/s 2023-09-23T19:14:32.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:32 smithi092 bash[27556]: audit 2023-09-23T19:14:31.197105+0000 mon.a (mon.0) 124 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:14:32.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:32 smithi092 bash[27556]: audit 2023-09-23T19:14:31.211590+0000 mon.a (mon.0) 125 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:14:32.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:32 smithi092 bash[27556]: cluster 2023-09-23T19:14:31.515478+0000 mgr.y (mgr.25325) 75 : cluster [DBG] pgmap v58: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 27 MiB/s wr, 873 op/s 2023-09-23T19:14:33.270 INFO:teuthology.orchestra.run.smithi016.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T19:14:33.270 INFO:teuthology.orchestra.run.smithi016.stdout:alertmanager.a smithi016 *:9093,9094 running (4m) - 10m 13.3M - 0.23.0 ba2b418f427c b362c3d9bae2 2023-09-23T19:14:33.270 INFO:teuthology.orchestra.run.smithi016.stdout:grafana.a smithi092 *:3000 running (3m) - 9m 40.4M - 8.3.5 dad864ee21e9 891a61cf5fff 2023-09-23T19:14:33.270 INFO:teuthology.orchestra.run.smithi016.stdout:mds.foo.smithi016.uncssv smithi016 running (8m) - 8m 16.4M - 16.2.14-37-g2e1f1015 1e6a756174a8 95d1e187e3eb 2023-09-23T19:14:33.271 INFO:teuthology.orchestra.run.smithi016.stdout:mds.foo.smithi092.vbpqks smithi092 running (8m) - 8m 15.7M - 16.2.14-37-g2e1f1015 1e6a756174a8 1d619ec31bf1 2023-09-23T19:14:33.271 INFO:teuthology.orchestra.run.smithi016.stdout:mgr.x smithi092 *:8443,9283,8765 running (2m) - 17m 426M - 18.2.0-460-gbb0daeff e32c2a64c126 d54b4354eade 2023-09-23T19:14:33.271 INFO:teuthology.orchestra.run.smithi016.stdout:mgr.y smithi016 *:8443,9283,8765 running (3m) - 19m 474M - 18.2.0-460-gbb0daeff e32c2a64c126 19706820b434 2023-09-23T19:14:33.271 INFO:teuthology.orchestra.run.smithi016.stdout:mon.a smithi016 running (2m) - 19m 50.0M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 e5aad364678c 2023-09-23T19:14:33.271 INFO:teuthology.orchestra.run.smithi016.stdout:mon.b smithi092 running (17m) - 17m 50.0M 2048M 16.2.14-37-g2e1f1015 1e6a756174a8 7042e0aa9af7 2023-09-23T19:14:33.271 INFO:teuthology.orchestra.run.smithi016.stdout:mon.c smithi016 running (15s) - 17m 18.2M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 61a05c2c22a7 2023-09-23T19:14:33.271 INFO:teuthology.orchestra.run.smithi016.stdout:node-exporter.a smithi016 *:9100 running (4m) - 10m 10.7M - 1.3.1 1dbe0e931976 7f13d7a8905d 2023-09-23T19:14:33.271 INFO:teuthology.orchestra.run.smithi016.stdout:node-exporter.b smithi092 *:9100 running (3m) - 10m 11.7M - 1.3.1 1dbe0e931976 4823347a98b5 2023-09-23T19:14:33.271 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0 smithi016 running (16m) - 16m 390M 2525M 16.2.14-37-g2e1f1015 1e6a756174a8 1458de442fb1 2023-09-23T19:14:33.271 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1 smithi016 running (15m) - 15m 325M 2525M 16.2.14-37-g2e1f1015 1e6a756174a8 7e0e07b2f6fd 2023-09-23T19:14:33.271 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2 smithi016 running (15m) - 15m 473M 2525M 16.2.14-37-g2e1f1015 1e6a756174a8 0c6fd3da0e21 2023-09-23T19:14:33.271 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3 smithi016 running (14m) - 14m 455M 2525M 16.2.14-37-g2e1f1015 1e6a756174a8 a34971c269fe 2023-09-23T19:14:33.272 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4 smithi092 running (13m) - 13m 395M 2524M 16.2.14-37-g2e1f1015 1e6a756174a8 253dc4b11010 2023-09-23T19:14:33.272 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5 smithi092 running (13m) - 13m 430M 2524M 16.2.14-37-g2e1f1015 1e6a756174a8 0a7306534382 2023-09-23T19:14:33.272 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6 smithi092 running (12m) - 12m 269M 2524M 16.2.14-37-g2e1f1015 1e6a756174a8 d6f3ee67cc5f 2023-09-23T19:14:33.272 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7 smithi092 running (11m) - 11m 362M 2524M 16.2.14-37-g2e1f1015 1e6a756174a8 794cf2340f17 2023-09-23T19:14:33.272 INFO:teuthology.orchestra.run.smithi016.stdout:prometheus.a smithi092 *:9095 running (2m) - 10m 53.6M - 2.33.4 514e6a882f6e 2666eba9f715 2023-09-23T19:14:33.786 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:33 smithi092 bash[27556]: audit 2023-09-23T19:14:32.294031+0000 mon.a (mon.0) 126 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:14:33.786 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:33 smithi092 bash[27556]: audit 2023-09-23T19:14:32.303948+0000 mon.a (mon.0) 127 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:14:34.237 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade pause' 2023-09-23T19:14:34.786 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:34 smithi092 bash[27556]: audit 2023-09-23T19:14:33.249135+0000 mgr.y (mgr.25325) 76 : audit [DBG] from='client.25231 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:14:34.786 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:34 smithi092 bash[27556]: cluster 2023-09-23T19:14:33.516061+0000 mgr.y (mgr.25325) 77 : cluster [DBG] pgmap v59: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 9.9 MiB/s wr, 596 op/s 2023-09-23T19:14:35.784 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:14:35 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:14:35] "GET /metrics HTTP/1.1" 200 36056 "" "Prometheus/2.33.4" 2023-09-23T19:14:37.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:36 smithi092 bash[27556]: cluster 2023-09-23T19:14:35.516805+0000 mgr.y (mgr.25325) 78 : cluster [DBG] pgmap v60: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 9.9 MiB/s wr, 891 op/s 2023-09-23T19:14:38.302 INFO:teuthology.orchestra.run.smithi016.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:14:39.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:38 smithi092 bash[27556]: cluster 2023-09-23T19:14:37.517432+0000 mgr.y (mgr.25325) 79 : cluster [DBG] pgmap v61: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 626 op/s 2023-09-23T19:14:39.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:38 smithi092 bash[27556]: audit 2023-09-23T19:14:38.299749+0000 mon.a (mon.0) 128 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:14:39.036 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:14:38 smithi092 bash[54653]: ts=2023-09-23T19:14:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi092.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi016:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T19:14:39.574 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'sleep 60' 2023-09-23T19:14:40.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:39 smithi092 bash[27556]: audit 2023-09-23T19:14:38.292783+0000 mgr.y (mgr.25325) 80 : audit [DBG] from='client.25234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:14:40.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:39 smithi092 bash[27556]: cephadm 2023-09-23T19:14:38.293361+0000 mgr.y (mgr.25325) 81 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:14:41.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:40 smithi092 bash[27556]: cluster 2023-09-23T19:14:39.517874+0000 mgr.y (mgr.25325) 82 : cluster [DBG] pgmap v62: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 666 op/s 2023-09-23T19:14:43.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:42 smithi092 bash[27556]: cluster 2023-09-23T19:14:41.518490+0000 mgr.y (mgr.25325) 83 : cluster [DBG] pgmap v63: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 905 op/s 2023-09-23T19:14:45.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:44 smithi092 bash[27556]: cluster 2023-09-23T19:14:43.518861+0000 mgr.y (mgr.25325) 84 : cluster [DBG] pgmap v64: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2023-09-23T19:14:45.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:44 smithi092 bash[27556]: audit 2023-09-23T19:14:43.609499+0000 mon.a (mon.0) 129 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:14:46.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:14:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:14:45] "GET /metrics HTTP/1.1" 200 36056 "" "Prometheus/2.33.4" 2023-09-23T19:14:46.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:46 smithi092 bash[27556]: audit 2023-09-23T19:14:45.253213+0000 mon.a (mon.0) 130 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:14:46.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:46 smithi092 bash[27556]: audit 2023-09-23T19:14:45.268607+0000 mon.a (mon.0) 131 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:14:46.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:46 smithi092 bash[27556]: audit 2023-09-23T19:14:45.269883+0000 mon.a (mon.0) 132 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:14:46.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:46 smithi092 bash[27556]: audit 2023-09-23T19:14:45.270914+0000 mon.a (mon.0) 133 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:14:46.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:46 smithi092 bash[27556]: audit 2023-09-23T19:14:45.277385+0000 mon.a (mon.0) 134 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:14:46.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:46 smithi092 bash[27556]: audit 2023-09-23T19:14:45.353643+0000 mon.a (mon.0) 135 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:14:46.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:46 smithi092 bash[27556]: audit 2023-09-23T19:14:45.356202+0000 mon.a (mon.0) 136 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:14:46.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:46 smithi092 bash[27556]: audit 2023-09-23T19:14:45.360648+0000 mon.a (mon.0) 137 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:14:46.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:46 smithi092 bash[27556]: audit 2023-09-23T19:14:45.370410+0000 mon.a (mon.0) 138 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:14:46.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:46 smithi092 bash[27556]: cluster 2023-09-23T19:14:45.519558+0000 mgr.y (mgr.25325) 85 : cluster [DBG] pgmap v65: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 937 op/s 2023-09-23T19:14:49.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:48 smithi092 bash[27556]: cluster 2023-09-23T19:14:47.520013+0000 mgr.y (mgr.25325) 86 : cluster [DBG] pgmap v66: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 642 op/s 2023-09-23T19:14:49.036 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:14:48 smithi092 bash[54653]: ts=2023-09-23T19:14:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi092.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi016:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T19:14:50.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:49 smithi092 bash[27556]: cluster 2023-09-23T19:14:49.520605+0000 mgr.y (mgr.25325) 87 : cluster [DBG] pgmap v67: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2023-09-23T19:14:53.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:52 smithi092 bash[27556]: cluster 2023-09-23T19:14:51.521294+0000 mgr.y (mgr.25325) 88 : cluster [DBG] pgmap v68: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 918 op/s 2023-09-23T19:14:55.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:54 smithi092 bash[27556]: cluster 2023-09-23T19:14:53.521740+0000 mgr.y (mgr.25325) 89 : cluster [DBG] pgmap v69: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 637 op/s 2023-09-23T19:14:56.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:14:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:14:55] "GET /metrics HTTP/1.1" 200 36056 "" "Prometheus/2.33.4" 2023-09-23T19:14:57.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:56 smithi092 bash[27556]: cluster 2023-09-23T19:14:55.522324+0000 mgr.y (mgr.25325) 90 : cluster [DBG] pgmap v70: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 955 op/s 2023-09-23T19:14:59.036 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:14:58 smithi092 bash[54653]: ts=2023-09-23T19:14:58.658Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi092.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi016:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T19:14:59.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:58 smithi092 bash[27556]: cluster 2023-09-23T19:14:57.522720+0000 mgr.y (mgr.25325) 91 : cluster [DBG] pgmap v71: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 633 op/s 2023-09-23T19:15:00.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:14:59 smithi092 bash[27556]: audit 2023-09-23T19:14:58.609959+0000 mon.a (mon.0) 139 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:15:01.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:15:00 smithi092 bash[27556]: cluster 2023-09-23T19:14:59.523242+0000 mgr.y (mgr.25325) 92 : cluster [DBG] pgmap v72: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 674 op/s 2023-09-23T19:15:03.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:15:02 smithi092 bash[27556]: cluster 2023-09-23T19:15:01.523817+0000 mgr.y (mgr.25325) 93 : cluster [DBG] pgmap v73: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 903 op/s 2023-09-23T19:15:05.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:15:04 smithi092 bash[27556]: cluster 2023-09-23T19:15:03.524348+0000 mgr.y (mgr.25325) 94 : cluster [DBG] pgmap v74: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 626 op/s 2023-09-23T19:15:06.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:15:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:15:05] "GET /metrics HTTP/1.1" 200 36064 "" "Prometheus/2.33.4" 2023-09-23T19:15:07.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:15:06 smithi092 bash[27556]: cluster 2023-09-23T19:15:05.525073+0000 mgr.y (mgr.25325) 95 : cluster [DBG] pgmap v75: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 937 op/s 2023-09-23T19:15:09.037 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:15:08 smithi092 bash[54653]: ts=2023-09-23T19:15:08.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi092.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi016:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T19:15:09.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:15:08 smithi092 bash[27556]: cluster 2023-09-23T19:15:07.525526+0000 mgr.y (mgr.25325) 96 : cluster [DBG] pgmap v76: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 619 op/s 2023-09-23T19:15:10.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:15:09 smithi092 bash[27556]: cluster 2023-09-23T19:15:09.526028+0000 mgr.y (mgr.25325) 97 : cluster [DBG] pgmap v77: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 657 op/s 2023-09-23T19:15:13.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:15:12 smithi092 bash[27556]: cluster 2023-09-23T19:15:11.526752+0000 mgr.y (mgr.25325) 98 : cluster [DBG] pgmap v78: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 889 op/s 2023-09-23T19:15:15.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:15:14 smithi092 bash[27556]: cluster 2023-09-23T19:15:13.527188+0000 mgr.y (mgr.25325) 99 : cluster [DBG] pgmap v79: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 621 op/s 2023-09-23T19:15:15.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:15:14 smithi092 bash[27556]: audit 2023-09-23T19:15:13.610106+0000 mon.a (mon.0) 140 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:15:16.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:15:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:15:15] "GET /metrics HTTP/1.1" 200 36070 "" "Prometheus/2.33.4" 2023-09-23T19:15:17.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:15:16 smithi092 bash[27556]: cluster 2023-09-23T19:15:15.527904+0000 mgr.y (mgr.25325) 100 : cluster [DBG] pgmap v80: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 926 op/s 2023-09-23T19:15:19.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:15:18 smithi092 bash[27556]: cluster 2023-09-23T19:15:17.528520+0000 mgr.y (mgr.25325) 101 : cluster [DBG] pgmap v81: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 616 op/s 2023-09-23T19:15:19.037 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:15:18 smithi092 bash[54653]: ts=2023-09-23T19:15:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi092.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi016:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T19:15:20.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:15:19 smithi092 bash[27556]: cluster 2023-09-23T19:15:19.529035+0000 mgr.y (mgr.25325) 102 : cluster [DBG] pgmap v82: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 616 op/s 2023-09-23T19:15:23.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:15:22 smithi092 bash[27556]: cluster 2023-09-23T19:15:21.529810+0000 mgr.y (mgr.25325) 103 : cluster [DBG] pgmap v83: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 883 op/s 2023-09-23T19:15:25.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:15:24 smithi092 bash[27556]: cluster 2023-09-23T19:15:23.530250+0000 mgr.y (mgr.25325) 104 : cluster [DBG] pgmap v84: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 610 op/s 2023-09-23T19:15:26.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:15:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:15:25] "GET /metrics HTTP/1.1" 200 36070 "" "Prometheus/2.33.4" 2023-09-23T19:15:27.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:15:26 smithi092 bash[27556]: cluster 2023-09-23T19:15:25.530969+0000 mgr.y (mgr.25325) 105 : cluster [DBG] pgmap v85: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 907 op/s 2023-09-23T19:15:29.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:15:28 smithi092 bash[27556]: cluster 2023-09-23T19:15:27.531357+0000 mgr.y (mgr.25325) 106 : cluster [DBG] pgmap v86: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 602 op/s 2023-09-23T19:15:29.037 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 19:15:28 smithi092 bash[54653]: ts=2023-09-23T19:15:28.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi092.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.16\", device_class=\"ssd\", hostname=\"smithi016\", instance=\"smithi016:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.16\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T19:15:30.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:15:29 smithi092 bash[27556]: audit 2023-09-23T19:15:28.619806+0000 mon.a (mon.0) 141 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:15:31.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:15:30 smithi092 bash[27556]: cluster 2023-09-23T19:15:29.531840+0000 mgr.y (mgr.25325) 107 : cluster [DBG] pgmap v87: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 602 op/s 2023-09-23T19:15:33.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:15:32 smithi092 bash[27556]: cluster 2023-09-23T19:15:31.532626+0000 mgr.y (mgr.25325) 108 : cluster [DBG] pgmap v88: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 898 op/s 2023-09-23T19:15:35.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:15:34 smithi092 bash[27556]: cluster 2023-09-23T19:15:33.533106+0000 mgr.y (mgr.25325) 109 : cluster [DBG] pgmap v89: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 592 op/s 2023-09-23T19:15:36.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:15:35 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:15:35] "GET /metrics HTTP/1.1" 200 36067 "" "Prometheus/2.33.4" 2023-09-23T19:15:37.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:15:36 smithi092 bash[27556]: cluster 2023-09-23T19:15:35.533806+0000 mgr.y (mgr.25325) 110 : cluster [DBG] pgmap v90: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 889 op/s 2023-09-23T19:15:39.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:15:38 smithi092 bash[27556]: cluster 2023-09-23T19:15:37.534248+0000 mgr.y (mgr.25325) 111 : cluster [DBG] pgmap v91: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-09-23T19:15:40.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:15:39 smithi092 bash[27556]: cluster 2023-09-23T19:15:39.534690+0000 mgr.y (mgr.25325) 112 : cluster [DBG] pgmap v92: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-09-23T19:15:43.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:15:42 smithi092 bash[27556]: cluster 2023-09-23T19:15:41.535489+0000 mgr.y (mgr.25325) 113 : cluster [DBG] pgmap v93: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2023-09-23T19:15:45.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:15:44 smithi092 bash[27556]: cluster 2023-09-23T19:15:43.535884+0000 mgr.y (mgr.25325) 114 : cluster [DBG] pgmap v94: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 597 op/s 2023-09-23T19:15:45.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:15:44 smithi092 bash[27556]: audit 2023-09-23T19:15:43.611046+0000 mon.a (mon.0) 142 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:15:46.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:15:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:15:45] "GET /metrics HTTP/1.1" 200 36069 "" "Prometheus/2.33.4" 2023-09-23T19:15:46.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:15:45 smithi092 bash[27556]: audit 2023-09-23T19:15:45.460341+0000 mon.a (mon.0) 143 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:15:46.565 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade resume' 2023-09-23T19:15:47.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:15:46 smithi092 bash[27556]: cluster 2023-09-23T19:15:45.536540+0000 mgr.y (mgr.25325) 115 : cluster [DBG] pgmap v95: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2023-09-23T19:15:49.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:15:48 smithi092 bash[27556]: cluster 2023-09-23T19:15:47.536998+0000 mgr.y (mgr.25325) 116 : cluster [DBG] pgmap v96: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 596 op/s 2023-09-23T19:15:50.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:15:49 smithi092 bash[27556]: cluster 2023-09-23T19:15:49.537424+0000 mgr.y (mgr.25325) 117 : cluster [DBG] pgmap v97: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 596 op/s 2023-09-23T19:15:50.426 INFO:teuthology.orchestra.run.smithi016.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:15:51.398 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'echo wait for all mons to upgrade' 2023-09-23T19:15:51.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:15:51 smithi092 bash[27556]: audit 2023-09-23T19:15:50.413481+0000 mgr.y (mgr.25325) 118 : audit [DBG] from='client.34224 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:15:51.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:15:51 smithi092 bash[27556]: cephadm 2023-09-23T19:15:50.413999+0000 mgr.y (mgr.25325) 119 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:15:51.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:15:51 smithi092 bash[27556]: audit 2023-09-23T19:15:50.423925+0000 mon.a (mon.0) 144 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:15:52.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:15:52 smithi092 bash[27556]: cluster 2023-09-23T19:15:51.540867+0000 mgr.y (mgr.25325) 120 : cluster [DBG] pgmap v98: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 883 op/s 2023-09-23T19:15:52.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:15:52 smithi092 bash[27556]: audit 2023-09-23T19:15:51.925924+0000 mon.a (mon.0) 145 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:15:52.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:15:52 smithi092 bash[27556]: audit 2023-09-23T19:15:51.938936+0000 mon.a (mon.0) 146 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:15:54.769 INFO:teuthology.orchestra.run.smithi016.stdout:wait for all mons to upgrade 2023-09-23T19:15:54.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:15:54 smithi092 bash[27556]: audit 2023-09-23T19:15:53.351475+0000 mon.a (mon.0) 147 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:15:54.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:15:54 smithi092 bash[27556]: audit 2023-09-23T19:15:53.372664+0000 mon.a (mon.0) 148 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:15:54.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:15:54 smithi092 bash[27556]: cluster 2023-09-23T19:15:53.541320+0000 mgr.y (mgr.25325) 121 : cluster [DBG] pgmap v99: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-09-23T19:15:54.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:15:54 smithi092 bash[27556]: audit 2023-09-23T19:15:54.013219+0000 mon.a (mon.0) 149 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:15:54.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:15:54 smithi092 bash[27556]: audit 2023-09-23T19:15:54.015276+0000 mon.a (mon.0) 150 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:15:54.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:15:54 smithi092 bash[27556]: audit 2023-09-23T19:15:54.025438+0000 mon.a (mon.0) 151 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:15:54.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:15:54 smithi092 bash[27556]: audit 2023-09-23T19:15:54.106844+0000 mon.a (mon.0) 152 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:15:54.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:15:54 smithi092 bash[27556]: audit 2023-09-23T19:15:54.111405+0000 mon.a (mon.0) 153 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:15:54.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:15:54 smithi092 bash[27556]: audit 2023-09-23T19:15:54.113517+0000 mon.a (mon.0) 154 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T19:15:54.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:15:54 smithi092 bash[27556]: audit 2023-09-23T19:15:54.117249+0000 mon.a (mon.0) 155 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-09-23T19:15:55.663 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'while ! ceph mon versions | grep $sha1 | grep '"'"': 3'"'"' ; do sleep 2 ; done' 2023-09-23T19:15:55.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:15:55 smithi092 bash[27556]: cephadm 2023-09-23T19:15:54.118478+0000 mgr.y (mgr.25325) 122 : cephadm [INF] Upgrade: It appears safe to stop mon.b 2023-09-23T19:15:55.937 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:15:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:15:55] "GET /metrics HTTP/1.1" 200 36069 "" "Prometheus/2.33.4" 2023-09-23T19:15:56.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:15:56 smithi092 bash[27556]: cluster 2023-09-23T19:15:55.542051+0000 mgr.y (mgr.25325) 123 : cluster [DBG] pgmap v100: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 843 op/s 2023-09-23T19:15:59.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:15:58 smithi092 bash[27556]: cluster 2023-09-23T19:15:57.542541+0000 mgr.y (mgr.25325) 124 : cluster [DBG] pgmap v101: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 547 op/s 2023-09-23T19:16:00.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:15:59 smithi092 bash[27556]: audit 2023-09-23T19:15:58.611300+0000 mon.a (mon.0) 156 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:16:00.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:15:59 smithi092 bash[27556]: audit 2023-09-23T19:15:58.865673+0000 mon.a (mon.0) 157 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:00.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:15:59 smithi092 bash[27556]: audit 2023-09-23T19:15:58.867749+0000 mon.a (mon.0) 158 : audit [INF] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T19:16:00.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:15:59 smithi092 bash[27556]: audit 2023-09-23T19:15:58.868653+0000 mon.a (mon.0) 159 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T19:16:00.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:15:59 smithi092 bash[27556]: audit 2023-09-23T19:15:58.869657+0000 mon.a (mon.0) 160 : audit [DBG] from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:01.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:00 smithi092 bash[27556]: cephadm 2023-09-23T19:15:58.858256+0000 mgr.y (mgr.25325) 125 : cephadm [INF] Upgrade: Updating mon.b 2023-09-23T19:16:01.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:00 smithi092 bash[27556]: cephadm 2023-09-23T19:15:58.870880+0000 mgr.y (mgr.25325) 126 : cephadm [INF] Deploying daemon mon.b on smithi092 2023-09-23T19:16:01.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:00 smithi092 bash[27556]: cluster 2023-09-23T19:15:59.542968+0000 mgr.y (mgr.25325) 127 : cluster [DBG] pgmap v102: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 547 op/s 2023-09-23T19:16:01.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:00 smithi092 bash[27556]: audit 2023-09-23T19:15:59.985784+0000 mon.c (mon.2) 2 : audit [DBG] from='client.? 172.21.15.16:0/310864528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:16:01.845 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:01 smithi092 systemd[1]: Stopping Ceph mon.b for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7... 2023-09-23T19:16:01.845 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:01 smithi092 bash[27556]: debug 2023-09-23T19:16:01.732+0000 7f3f6884f700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-09-23T19:16:01.845 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:01 smithi092 bash[27556]: debug 2023-09-23T19:16:01.732+0000 7f3f6884f700 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2023-09-23T19:16:02.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:02 smithi092 bash[58748]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-mon-b 2023-09-23T19:16:02.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:02 smithi092 bash[58806]: Error response from daemon: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-mon-b 2023-09-23T19:16:02.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:02 smithi092 systemd[1]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@mon.b.service: Succeeded. 2023-09-23T19:16:02.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:02 smithi092 systemd[1]: Stopped Ceph mon.b for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7. 2023-09-23T19:16:03.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:03 smithi092 systemd[1]: Started Ceph mon.b for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7. 2023-09-23T19:16:04.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.188+0000 7ff677dcdc80 0 set uid:gid to 167:167 (ceph:ceph) 2023-09-23T19:16:04.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.192+0000 7ff677dcdc80 0 ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable), process ceph-mon, pid 7 2023-09-23T19:16:04.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.192+0000 7ff677dcdc80 0 pidfile_write: ignore empty --pid-file 2023-09-23T19:16:04.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.196+0000 7ff677dcdc80 0 load: jerasure load: lrc 2023-09-23T19:16:04.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.196+0000 7ff677dcdc80 4 rocksdb: RocksDB version: 7.9.2 2023-09-23T19:16:04.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.196+0000 7ff677dcdc80 4 rocksdb: Git sha 0 2023-09-23T19:16:04.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.196+0000 7ff677dcdc80 4 rocksdb: Compile date 2023-09-22 15:19:43 2023-09-23T19:16:04.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.196+0000 7ff677dcdc80 4 rocksdb: DB SUMMARY 2023-09-23T19:16:04.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.196+0000 7ff677dcdc80 4 rocksdb: DB Session ID: T2PGRW7MO8QUJULKJMCC 2023-09-23T19:16:04.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.196+0000 7ff677dcdc80 4 rocksdb: CURRENT file: CURRENT 2023-09-23T19:16:04.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.196+0000 7ff677dcdc80 4 rocksdb: IDENTITY file: IDENTITY 2023-09-23T19:16:04.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.196+0000 7ff677dcdc80 4 rocksdb: MANIFEST file: MANIFEST-000005 size: 1847 Bytes 2023-09-23T19:16:04.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.196+0000 7ff677dcdc80 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000035.sst 2023-09-23T19:16:04.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.196+0000 7ff677dcdc80 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000033.log size: 462256 ; 2023-09-23T19:16:04.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.196+0000 7ff677dcdc80 4 rocksdb: Options.error_if_exists: 0 2023-09-23T19:16:04.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.200+0000 7ff677dcdc80 4 rocksdb: Options.create_if_missing: 0 2023-09-23T19:16:04.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.200+0000 7ff677dcdc80 4 rocksdb: Options.paranoid_checks: 1 2023-09-23T19:16:04.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.200+0000 7ff677dcdc80 4 rocksdb: Options.flush_verify_memtable_count: 1 2023-09-23T19:16:04.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.200+0000 7ff677dcdc80 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-23T19:16:04.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.200+0000 7ff677dcdc80 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-09-23T19:16:04.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.200+0000 7ff677dcdc80 4 rocksdb: Options.env: 0x55d021a23720 2023-09-23T19:16:04.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.200+0000 7ff677dcdc80 4 rocksdb: Options.fs: PosixFileSystem 2023-09-23T19:16:04.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.200+0000 7ff677dcdc80 4 rocksdb: Options.info_log: 0x55d024e4d400 2023-09-23T19:16:04.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.200+0000 7ff677dcdc80 4 rocksdb: Options.max_file_opening_threads: 16 2023-09-23T19:16:04.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.200+0000 7ff677dcdc80 4 rocksdb: Options.statistics: (nil) 2023-09-23T19:16:04.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.200+0000 7ff677dcdc80 4 rocksdb: Options.use_fsync: 0 2023-09-23T19:16:04.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.200+0000 7ff677dcdc80 4 rocksdb: Options.max_log_file_size: 0 2023-09-23T19:16:04.541 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.200+0000 7ff677dcdc80 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T19:16:04.541 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.200+0000 7ff677dcdc80 4 rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T19:16:04.541 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.200+0000 7ff677dcdc80 4 rocksdb: Options.keep_log_file_num: 1000 2023-09-23T19:16:04.541 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.200+0000 7ff677dcdc80 4 rocksdb: Options.recycle_log_file_num: 0 2023-09-23T19:16:04.541 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.200+0000 7ff677dcdc80 4 rocksdb: Options.allow_fallocate: 1 2023-09-23T19:16:04.541 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.200+0000 7ff677dcdc80 4 rocksdb: Options.allow_mmap_reads: 0 2023-09-23T19:16:04.541 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.200+0000 7ff677dcdc80 4 rocksdb: Options.allow_mmap_writes: 0 2023-09-23T19:16:04.541 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.200+0000 7ff677dcdc80 4 rocksdb: Options.use_direct_reads: 0 2023-09-23T19:16:04.541 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T19:16:04.541 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.create_missing_column_families: 0 2023-09-23T19:16:04.541 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.db_log_dir: 2023-09-23T19:16:04.542 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.wal_dir: 2023-09-23T19:16:04.542 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T19:16:04.542 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T19:16:04.542 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T19:16:04.542 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T19:16:04.542 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T19:16:04.542 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T19:16:04.542 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.advise_random_on_open: 1 2023-09-23T19:16:04.542 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.db_write_buffer_size: 0 2023-09-23T19:16:04.542 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.write_buffer_manager: 0x55d0240da5a0 2023-09-23T19:16:04.542 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T19:16:04.542 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T19:16:04.542 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T19:16:04.543 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.rate_limiter: (nil) 2023-09-23T19:16:04.543 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T19:16:04.543 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.wal_recovery_mode: 2 2023-09-23T19:16:04.543 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.enable_thread_tracking: 0 2023-09-23T19:16:04.543 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.enable_pipelined_write: 0 2023-09-23T19:16:04.543 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.unordered_write: 0 2023-09-23T19:16:04.543 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T19:16:04.543 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T19:16:04.543 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T19:16:04.543 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T19:16:04.543 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.row_cache: None 2023-09-23T19:16:04.543 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.wal_filter: None 2023-09-23T19:16:04.543 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T19:16:04.544 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.allow_ingest_behind: 0 2023-09-23T19:16:04.544 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.two_write_queues: 0 2023-09-23T19:16:04.544 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.manual_wal_flush: 0 2023-09-23T19:16:04.544 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.wal_compression: 0 2023-09-23T19:16:04.544 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.atomic_flush: 0 2023-09-23T19:16:04.544 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T19:16:04.544 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T19:16:04.544 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T19:16:04.544 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.log_readahead_size: 0 2023-09-23T19:16:04.544 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-23T19:16:04.544 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.best_efforts_recovery: 0 2023-09-23T19:16:04.544 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-23T19:16:04.544 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-23T19:16:04.545 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.allow_data_in_errors: 0 2023-09-23T19:16:04.545 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.db_host_id: __hostname__ 2023-09-23T19:16:04.545 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.enforce_single_del_contracts: true 2023-09-23T19:16:04.545 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.max_background_jobs: 2 2023-09-23T19:16:04.545 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.max_background_compactions: -1 2023-09-23T19:16:04.545 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.max_subcompactions: 1 2023-09-23T19:16:04.545 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T19:16:04.545 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T19:16:04.545 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T19:16:04.545 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.max_total_wal_size: 0 2023-09-23T19:16:04.545 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T19:16:04.545 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T19:16:04.545 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T19:16:04.546 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T19:16:04.546 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.max_open_files: -1 2023-09-23T19:16:04.546 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.bytes_per_sync: 0 2023-09-23T19:16:04.546 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T19:16:04.546 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T19:16:04.546 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.compaction_readahead_size: 0 2023-09-23T19:16:04.546 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.max_background_flushes: -1 2023-09-23T19:16:04.546 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Compression algorithms supported: 2023-09-23T19:16:04.546 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T19:16:04.546 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: kZSTD supported: 0 2023-09-23T19:16:04.546 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: kXpressCompression supported: 0 2023-09-23T19:16:04.546 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: kLZ4HCCompression supported: 1 2023-09-23T19:16:04.547 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: kZlibCompression supported: 1 2023-09-23T19:16:04.547 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: kSnappyCompression supported: 1 2023-09-23T19:16:04.547 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: kLZ4Compression supported: 1 2023-09-23T19:16:04.547 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: kBZip2Compression supported: 0 2023-09-23T19:16:04.547 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T19:16:04.547 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: DMutex implementation: pthread_mutex_t 2023-09-23T19:16:04.547 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000005 2023-09-23T19:16:04.547 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-09-23T19:16:04.547 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T19:16:04.547 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.merge_operator: 2023-09-23T19:16:04.547 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.compaction_filter: None 2023-09-23T19:16:04.547 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.compaction_filter_factory: None 2023-09-23T19:16:04.547 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.sst_partitioner_factory: None 2023-09-23T19:16:04.548 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T19:16:04.548 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T19:16:04.548 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55d024e4d520) 2023-09-23T19:16:04.548 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: cache_index_and_filter_blocks: 1 2023-09-23T19:16:04.548 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T19:16:04.548 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T19:16:04.548 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: pin_top_level_index_and_filter: 1 2023-09-23T19:16:04.548 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: index_type: 0 2023-09-23T19:16:04.548 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: data_block_index_type: 0 2023-09-23T19:16:04.548 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: index_shortening: 1 2023-09-23T19:16:04.548 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: data_block_hash_table_util_ratio: 0.750000 2023-09-23T19:16:04.548 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: checksum: 4 2023-09-23T19:16:04.548 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: no_block_cache: 0 2023-09-23T19:16:04.548 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: block_cache: 0x55d02415d1f0 2023-09-23T19:16:04.549 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: block_cache_name: BinnedLRUCache 2023-09-23T19:16:04.549 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: block_cache_options: 2023-09-23T19:16:04.549 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: capacity : 536870912 2023-09-23T19:16:04.549 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: num_shard_bits : 4 2023-09-23T19:16:04.549 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: strict_capacity_limit : 0 2023-09-23T19:16:04.549 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: high_pri_pool_ratio: 0.000 2023-09-23T19:16:04.549 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: block_cache_compressed: (nil) 2023-09-23T19:16:04.549 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: persistent_cache: (nil) 2023-09-23T19:16:04.549 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: block_size: 4096 2023-09-23T19:16:04.549 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: block_size_deviation: 10 2023-09-23T19:16:04.549 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: block_restart_interval: 16 2023-09-23T19:16:04.549 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: index_block_restart_interval: 1 2023-09-23T19:16:04.549 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: metadata_block_size: 4096 2023-09-23T19:16:04.550 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: partition_filters: 0 2023-09-23T19:16:04.550 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: use_delta_encoding: 1 2023-09-23T19:16:04.550 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: filter_policy: bloomfilter 2023-09-23T19:16:04.550 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: whole_key_filtering: 1 2023-09-23T19:16:04.550 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: verify_compression: 0 2023-09-23T19:16:04.550 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: read_amp_bytes_per_bit: 0 2023-09-23T19:16:04.550 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: format_version: 5 2023-09-23T19:16:04.550 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: enable_index_compression: 1 2023-09-23T19:16:04.550 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: block_align: 0 2023-09-23T19:16:04.550 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: max_auto_readahead_size: 262144 2023-09-23T19:16:04.550 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: prepopulate_block_cache: 0 2023-09-23T19:16:04.550 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: initial_auto_readahead_size: 8192 2023-09-23T19:16:04.551 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: num_file_reads_for_auto_readahead: 2 2023-09-23T19:16:04.551 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.write_buffer_size: 33554432 2023-09-23T19:16:04.551 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.max_write_buffer_number: 2 2023-09-23T19:16:04.551 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.compression: NoCompression 2023-09-23T19:16:04.551 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.bottommost_compression: Disabled 2023-09-23T19:16:04.551 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.prefix_extractor: nullptr 2023-09-23T19:16:04.551 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T19:16:04.551 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.num_levels: 7 2023-09-23T19:16:04.551 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T19:16:04.551 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T19:16:04.551 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T19:16:04.551 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T19:16:04.551 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T19:16:04.552 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T19:16:04.552 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T19:16:04.552 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T19:16:04.552 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-23T19:16:04.552 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T19:16:04.552 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-09-23T19:16:04.552 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-09-23T19:16:04.552 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T19:16:04.552 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.compression_opts.level: 32767 2023-09-23T19:16:04.552 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.compression_opts.strategy: 0 2023-09-23T19:16:04.552 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T19:16:04.552 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T19:16:04.552 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-09-23T19:16:04.552 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-23T19:16:04.553 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.compression_opts.enabled: false 2023-09-23T19:16:04.553 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-09-23T19:16:04.553 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T19:16:04.553 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T19:16:04.553 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T19:16:04.553 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.target_file_size_base: 67108864 2023-09-23T19:16:04.553 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T19:16:04.553 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T19:16:04.553 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T19:16:04.553 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T19:16:04.553 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T19:16:04.553 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T19:16:04.553 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T19:16:04.553 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T19:16:04.554 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T19:16:04.554 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T19:16:04.554 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T19:16:04.554 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T19:16:04.554 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T19:16:04.554 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-09-23T19:16:04.554 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.arena_block_size: 1048576 2023-09-23T19:16:04.554 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T19:16:04.554 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T19:16:04.554 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.disable_auto_compactions: 0 2023-09-23T19:16:04.554 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T19:16:04.554 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T19:16:04.554 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T19:16:04.555 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T19:16:04.555 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T19:16:04.555 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T19:16:04.555 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T19:16:04.555 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T19:16:04.555 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T19:16:04.555 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T19:16:04.555 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.table_properties_collectors: 2023-09-23T19:16:04.555 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.inplace_update_support: 0 2023-09-23T19:16:04.555 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T19:16:04.555 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T19:16:04.555 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T19:16:04.555 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T19:16:04.556 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.bloom_locality: 0 2023-09-23T19:16:04.556 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.max_successive_merges: 0 2023-09-23T19:16:04.556 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T19:16:04.556 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.paranoid_file_checks: 0 2023-09-23T19:16:04.556 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.force_consistency_checks: 1 2023-09-23T19:16:04.556 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.report_bg_io_stats: 0 2023-09-23T19:16:04.556 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.ttl: 2592000 2023-09-23T19:16:04.556 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T19:16:04.556 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2023-09-23T19:16:04.556 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.preserve_internal_time_seconds: 0 2023-09-23T19:16:04.556 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.enable_blob_files: false 2023-09-23T19:16:04.556 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.min_blob_size: 0 2023-09-23T19:16:04.556 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.blob_file_size: 268435456 2023-09-23T19:16:04.557 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.blob_compression_type: NoCompression 2023-09-23T19:16:04.557 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.enable_blob_garbage_collection: false 2023-09-23T19:16:04.557 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-23T19:16:04.557 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-09-23T19:16:04.557 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.blob_compaction_readahead_size: 0 2023-09-23T19:16:04.557 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.blob_file_starting_level: 0 2023-09-23T19:16:04.557 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff677dcdc80 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-09-23T19:16:04.557 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.204+0000 7ff66b5e8700 3 rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 35.sst 2023-09-23T19:16:04.557 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.216+0000 7ff677dcdc80 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 37, last_sequence is 20433, log_number is 33,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T19:16:04.557 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.216+0000 7ff677dcdc80 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 33 2023-09-23T19:16:04.557 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.216+0000 7ff677dcdc80 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: b629fb32-d392-4120-9cee-c3a4669d91c8 2023-09-23T19:16:04.557 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.216+0000 7ff677dcdc80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1695496564217231, "job": 1, "event": "recovery_started", "wal_files": [33]} 2023-09-23T19:16:04.557 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.216+0000 7ff677dcdc80 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #33 mode 2 2023-09-23T19:16:04.558 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.216+0000 7ff677dcdc80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1695496564220408, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 38, "file_size": 308632, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 20438, "largest_seqno": 20523, "table_properties": {"data_size": 307108, "index_size": 384, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 133, "raw_key_size": 952, "raw_average_key_size": 23, "raw_value_size": 305947, "raw_average_value_size": 7462, "num_data_blocks": 17, "num_entries": 41, "num_filter_entries": 41, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1695496564, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "b629fb32-d392-4120-9cee-c3a4669d91c8", "db_session_id": "T2PGRW7MO8QUJULKJMCC", "orig_file_number": 38, "seqno_to_time_mapping": "N/A"}} 2023-09-23T19:16:04.558 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.216+0000 7ff677dcdc80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1695496564220626, "job": 1, "event": "recovery_finished"} 2023-09-23T19:16:04.558 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.216+0000 7ff677dcdc80 4 rocksdb: [db/version_set.cc:5047] Creating manifest 40 2023-09-23T19:16:04.558 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.224+0000 7ff677dcdc80 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000033.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-23T19:16:04.558 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.224+0000 7ff677dcdc80 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55d0241fe000 2023-09-23T19:16:04.558 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.224+0000 7ff677dcdc80 4 rocksdb: DB pointer 0x55d0241ec000 2023-09-23T19:16:04.559 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.224+0000 7ff6665de700 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-09-23T19:16:04.559 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.224+0000 7ff6665de700 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2023-09-23T19:16:04.559 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: ** DB Stats ** 2023-09-23T19:16:04.559 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T19:16:04.559 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T19:16:04.559 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T19:16:04.559 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T19:16:04.559 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T19:16:04.559 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T19:16:04.559 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T19:16:04.559 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: ** Compaction Stats [default] ** 2023-09-23T19:16:04.559 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T19:16:04.559 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-09-23T19:16:04.559 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: L0 1/0 301.40 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 167.5 0.00 0.00 1 0.002 0 0 0.0 0.0 2023-09-23T19:16:04.560 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: L6 1/0 27.39 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-09-23T19:16:04.560 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: Sum 2/0 27.68 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 167.5 0.00 0.00 1 0.002 0 0 0.0 0.0 2023-09-23T19:16:04.560 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 167.5 0.00 0.00 1 0.002 0 0 0.0 0.0 2023-09-23T19:16:04.560 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: ** Compaction Stats [default] ** 2023-09-23T19:16:04.560 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T19:16:04.560 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T19:16:04.560 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 167.5 0.00 0.00 1 0.002 0 0 0.0 0.0 2023-09-23T19:16:04.560 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-09-23T19:16:04.560 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T19:16:04.560 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T19:16:04.560 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T19:16:04.560 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T19:16:04.560 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T19:16:04.561 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: AddFile(Keys): cumulative 0, interval 0 2023-09-23T19:16:04.561 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: Cumulative compaction: 0.00 GB write, 13.75 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T19:16:04.561 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: Interval compaction: 0.00 GB write, 13.75 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T19:16:04.561 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T19:16:04.561 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: Block cache BinnedLRUCache@0x55d02415d1f0#7 capacity: 512.00 MB usage: 0.62 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.6e-05 secs_since: 0 2023-09-23T19:16:04.561 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: Block cache entry stats(count,size,portion): FilterBlock(1,0.17 KB,3.27826e-05%) IndexBlock(1,0.45 KB,8.64267e-05%) Misc(1,0.00 KB,0%) 2023-09-23T19:16:04.561 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: ** File Read Latency Histogram By Level [default] ** 2023-09-23T19:16:04.561 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.224+0000 7ff677dcdc80 0 starting mon.b rank 1 at public addrs [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] at bind addrs [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 2023-09-23T19:16:04.561 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.228+0000 7ff677dcdc80 1 mon.b@-1(???) e3 preinit fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 2023-09-23T19:16:04.561 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.228+0000 7ff677dcdc80 0 mon.b@-1(???).mds e6 new map 2023-09-23T19:16:04.561 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.228+0000 7ff677dcdc80 0 mon.b@-1(???).mds e6 print_map 2023-09-23T19:16:04.561 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: e6 2023-09-23T19:16:04.561 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T19:16:04.562 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T19:16:04.562 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: legacy client fscid: 1 2023-09-23T19:16:04.562 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: 2023-09-23T19:16:04.562 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: Filesystem 'foo' (1) 2023-09-23T19:16:04.562 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: fs_name foo 2023-09-23T19:16:04.562 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: epoch 5 2023-09-23T19:16:04.562 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: flags 12 joinable allow_snaps allow_multimds_snaps 2023-09-23T19:16:04.562 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: created 2023-09-23T19:05:52.117022+0000 2023-09-23T19:16:04.562 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: modified 2023-09-23T19:05:58.952970+0000 2023-09-23T19:16:04.562 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: tableserver 0 2023-09-23T19:16:04.562 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: root 0 2023-09-23T19:16:04.562 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: session_timeout 60 2023-09-23T19:16:04.562 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: session_autoclose 300 2023-09-23T19:16:04.563 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: max_file_size 1099511627776 2023-09-23T19:16:04.563 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: required_client_features {} 2023-09-23T19:16:04.563 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: last_failure 0 2023-09-23T19:16:04.563 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: last_failure_osd_epoch 0 2023-09-23T19:16:04.563 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T19:16:04.563 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: max_mds 1 2023-09-23T19:16:04.563 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: in 0 2023-09-23T19:16:04.563 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: up {0=14655} 2023-09-23T19:16:04.563 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: failed 2023-09-23T19:16:04.563 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: damaged 2023-09-23T19:16:04.563 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: stopped 2023-09-23T19:16:04.563 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: data_pools [4] 2023-09-23T19:16:04.563 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: metadata_pool 3 2023-09-23T19:16:04.564 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: inline_data disabled 2023-09-23T19:16:04.564 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: balancer 2023-09-23T19:16:04.564 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: bal_rank_mask -1 2023-09-23T19:16:04.564 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: standby_count_wanted 1 2023-09-23T19:16:04.564 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: [mds.foo.smithi016.uncssv{0:14655} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.16:6834/161572387,v1:172.21.15.16:6835/161572387] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T19:16:04.564 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: 2023-09-23T19:16:04.564 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: 2023-09-23T19:16:04.564 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: Standby daemons: 2023-09-23T19:16:04.564 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: 2023-09-23T19:16:04.564 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: [mds.foo.smithi092.vbpqks{-1:24520} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.92:6832/1118427195,v1:172.21.15.92:6833/1118427195] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T19:16:04.564 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.228+0000 7ff677dcdc80 0 mon.b@-1(???).osd e76 crush map has features 3314933000854323200, adjusting msgr requires 2023-09-23T19:16:04.564 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.228+0000 7ff677dcdc80 0 mon.b@-1(???).osd e76 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T19:16:04.564 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.228+0000 7ff677dcdc80 0 mon.b@-1(???).osd e76 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T19:16:04.564 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.228+0000 7ff677dcdc80 0 mon.b@-1(???).osd e76 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T19:16:04.565 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.232+0000 7ff677dcdc80 1 mon.b@-1(???).paxosservice(auth 1..23) refresh upgraded, format 0 -> 3 2023-09-23T19:16:04.565 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.236+0000 7ff677dcdc80 20 expand_channel_meta expand map: {default=false} 2023-09-23T19:16:04.565 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.236+0000 7ff677dcdc80 20 expand_channel_meta from 'false' to 'false' 2023-09-23T19:16:04.565 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.236+0000 7ff677dcdc80 20 expand_channel_meta expanded map: {default=false} 2023-09-23T19:16:04.565 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.236+0000 7ff677dcdc80 20 expand_channel_meta expand map: {default=info} 2023-09-23T19:16:04.565 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.236+0000 7ff677dcdc80 20 expand_channel_meta from 'info' to 'info' 2023-09-23T19:16:04.565 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.236+0000 7ff677dcdc80 20 expand_channel_meta expanded map: {default=info} 2023-09-23T19:16:04.565 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.236+0000 7ff677dcdc80 20 expand_channel_meta expand map: {default=daemon} 2023-09-23T19:16:04.565 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.236+0000 7ff677dcdc80 20 expand_channel_meta from 'daemon' to 'daemon' 2023-09-23T19:16:04.565 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.236+0000 7ff677dcdc80 20 expand_channel_meta expanded map: {default=daemon} 2023-09-23T19:16:04.565 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.236+0000 7ff677dcdc80 20 expand_channel_meta expand map: {default=debug} 2023-09-23T19:16:04.565 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.236+0000 7ff677dcdc80 20 expand_channel_meta from 'debug' to 'debug' 2023-09-23T19:16:04.565 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:04 smithi092 bash[58855]: debug 2023-09-23T19:16:04.236+0000 7ff677dcdc80 20 expand_channel_meta expanded map: {default=debug} 2023-09-23T19:16:05.548 INFO:teuthology.orchestra.run.smithi016.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 3 2023-09-23T19:16:06.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:16:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:16:05] "GET /metrics HTTP/1.1" 200 36066 "" "Prometheus/2.33.4" 2023-09-23T19:16:06.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T18:55:23.803111+0000 mon.a (mon.0) 0 : 1 mkfs 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 2023-09-23T19:16:06.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cephadm 2023-09-23T19:10:35.535765+0000 mgr.x (mgr.15012) 51 : cephadm 1 Reconfiguring node-exporter.b (dependencies changed)... 2023-09-23T19:16:06.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cephadm 2023-09-23T19:10:35.536543+0000 mgr.x (mgr.15012) 52 : cephadm 1 Reconfiguring daemon node-exporter.b on smithi092 2023-09-23T19:16:06.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cephadm 2023-09-23T19:10:37.457529+0000 mgr.x (mgr.15012) 54 : cephadm 1 Reconfiguring grafana.a (dependencies changed)... 2023-09-23T19:16:06.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cephadm 2023-09-23T19:10:37.481421+0000 mgr.x (mgr.15012) 55 : cephadm 1 Reconfiguring daemon grafana.a on smithi092 2023-09-23T19:16:06.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cephadm 2023-09-23T19:11:04.293039+0000 mgr.x (mgr.15012) 72 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T19:16:06.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cephadm 2023-09-23T19:11:04.293965+0000 mgr.x (mgr.15012) 73 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T19:16:06.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cephadm 2023-09-23T19:11:08.736449+0000 mgr.x (mgr.15012) 76 : cephadm 1 Upgrade: Updating mgr.y 2023-09-23T19:16:06.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cephadm 2023-09-23T19:11:08.759144+0000 mgr.x (mgr.15012) 77 : cephadm 1 Deploying daemon mgr.y on smithi016 2023-09-23T19:16:06.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cephadm 2023-09-23T19:11:33.777506+0000 mgr.y (mgr.15228) 2 : cephadm 1 [23/Sep/2023:19:11:33] ENGINE Bus STARTING 2023-09-23T19:16:06.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cephadm 2023-09-23T19:11:33.911195+0000 mgr.y (mgr.15228) 3 : cephadm 1 [23/Sep/2023:19:11:33] ENGINE Serving on https://172.21.15.16:7150 2023-09-23T19:16:06.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cephadm 2023-09-23T19:11:34.016436+0000 mgr.y (mgr.15228) 4 : cephadm 1 [23/Sep/2023:19:11:34] ENGINE Serving on http://172.21.15.16:8765 2023-09-23T19:16:06.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cephadm 2023-09-23T19:11:34.016591+0000 mgr.y (mgr.15228) 5 : cephadm 1 [23/Sep/2023:19:11:34] ENGINE Bus STARTED 2023-09-23T19:16:06.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cephadm 2023-09-23T19:11:51.722802+0000 mgr.y (mgr.15228) 15 : cephadm 1 Updating smithi016:/etc/ceph/ceph.conf 2023-09-23T19:16:06.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cephadm 2023-09-23T19:11:51.723700+0000 mgr.y (mgr.15228) 16 : cephadm 1 Updating smithi092:/etc/ceph/ceph.conf 2023-09-23T19:16:06.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cephadm 2023-09-23T19:11:51.947767+0000 mgr.y (mgr.15228) 17 : cephadm 1 Updating smithi092:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:16:06.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cephadm 2023-09-23T19:11:51.948762+0000 mgr.y (mgr.15228) 18 : cephadm 1 Updating smithi016:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:16:06.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cephadm 2023-09-23T19:11:52.162991+0000 mgr.y (mgr.15228) 19 : cephadm 1 Updating smithi092:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:16:06.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cephadm 2023-09-23T19:11:52.163739+0000 mgr.y (mgr.15228) 20 : cephadm 1 Updating smithi016:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:16:06.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cephadm 2023-09-23T19:11:52.377394+0000 mgr.y (mgr.15228) 22 : cephadm 1 Updating smithi092:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:16:06.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cephadm 2023-09-23T19:11:52.377832+0000 mgr.y (mgr.15228) 23 : cephadm 1 Updating smithi016:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:16:06.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cephadm 2023-09-23T19:11:52.778114+0000 mgr.y (mgr.15228) 24 : cephadm 1 Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T19:16:06.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cephadm 2023-09-23T19:11:53.069383+0000 mgr.y (mgr.15228) 25 : cephadm 1 Reconfiguring daemon prometheus.a on smithi092 2023-09-23T19:16:06.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cephadm 2023-09-23T19:12:03.396080+0000 mgr.y (mgr.15228) 32 : cephadm 1 Upgrade: Updating mgr.x 2023-09-23T19:16:06.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cephadm 2023-09-23T19:12:03.409061+0000 mgr.y (mgr.15228) 33 : cephadm 1 Deploying daemon mgr.x on smithi092 2023-09-23T19:16:06.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cephadm 2023-09-23T19:12:25.529324+0000 mgr.y (mgr.15228) 45 : cephadm 1 Upgrade: Setting container_image for all mgr 2023-09-23T19:16:06.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cephadm 2023-09-23T19:12:25.558592+0000 mgr.y (mgr.15228) 46 : cephadm 1 Upgrade: It appears safe to stop mon.a 2023-09-23T19:16:06.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cephadm 2023-09-23T19:12:28.428916+0000 mgr.y (mgr.15228) 49 : cephadm 1 Upgrade: Updating mon.a 2023-09-23T19:16:06.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cephadm 2023-09-23T19:12:28.442853+0000 mgr.y (mgr.15228) 50 : cephadm 1 Deploying daemon mon.a on smithi016 2023-09-23T19:16:06.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cephadm 2023-09-23T19:12:45.208811+0000 mgr.y (mgr.25325) 3 : cephadm 1 [23/Sep/2023:19:12:45] ENGINE Bus STARTING 2023-09-23T19:16:06.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cephadm 2023-09-23T19:12:45.311158+0000 mgr.y (mgr.25325) 4 : cephadm 1 [23/Sep/2023:19:12:45] ENGINE Serving on http://172.21.15.16:8765 2023-09-23T19:16:06.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cephadm 2023-09-23T19:12:45.435516+0000 mgr.y (mgr.25325) 5 : cephadm 1 [23/Sep/2023:19:12:45] ENGINE Serving on https://172.21.15.16:7150 2023-09-23T19:16:06.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cephadm 2023-09-23T19:12:45.435832+0000 mgr.y (mgr.25325) 6 : cephadm 1 [23/Sep/2023:19:12:45] ENGINE Bus STARTED 2023-09-23T19:16:06.541 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cephadm 2023-09-23T19:12:50.821433+0000 mgr.y (mgr.25325) 11 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:16:06.541 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cephadm 2023-09-23T19:13:03.295263+0000 mgr.y (mgr.25325) 18 : cephadm 1 Updating smithi016:/etc/ceph/ceph.conf 2023-09-23T19:16:06.541 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cephadm 2023-09-23T19:13:03.295553+0000 mgr.y (mgr.25325) 19 : cephadm 1 Updating smithi092:/etc/ceph/ceph.conf 2023-09-23T19:16:06.541 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cephadm 2023-09-23T19:13:03.562963+0000 mgr.y (mgr.25325) 21 : cephadm 1 Updating smithi092:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:16:06.541 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cephadm 2023-09-23T19:13:03.563191+0000 mgr.y (mgr.25325) 22 : cephadm 1 Updating smithi016:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:16:06.541 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cephadm 2023-09-23T19:13:03.827935+0000 mgr.y (mgr.25325) 23 : cephadm 1 Updating smithi092:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:16:06.541 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cephadm 2023-09-23T19:13:03.828145+0000 mgr.y (mgr.25325) 24 : cephadm 1 Updating smithi016:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:16:06.541 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cephadm 2023-09-23T19:13:04.094271+0000 mgr.y (mgr.25325) 25 : cephadm 1 Updating smithi092:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:16:06.541 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cephadm 2023-09-23T19:13:04.097737+0000 mgr.y (mgr.25325) 26 : cephadm 1 Updating smithi016:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:16:06.541 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cephadm 2023-09-23T19:14:02.241916+0000 mgr.y (mgr.25325) 57 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:16:06.541 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cephadm 2023-09-23T19:14:09.619086+0000 mgr.y (mgr.25325) 62 : cephadm 1 Upgrade: It appears safe to stop mon.c 2023-09-23T19:16:06.541 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:14:12.664093+0000 mon.a (mon.0) 91 : audit 0 from='client.? 172.21.15.16:0/4024211665' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:16:06.542 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cephadm 2023-09-23T19:14:13.056347+0000 mgr.y (mgr.25325) 64 : cephadm 1 Upgrade: Updating mon.c 2023-09-23T19:16:06.542 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:14:13.066564+0000 mon.a (mon.0) 92 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.542 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:14:13.068124+0000 mon.a (mon.0) 93 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T19:16:06.542 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:14:13.073670+0000 mon.a (mon.0) 94 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T19:16:06.542 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:14:13.075842+0000 mon.a (mon.0) 95 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:06.542 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cephadm 2023-09-23T19:14:13.077863+0000 mgr.y (mgr.25325) 65 : cephadm 1 Deploying daemon mon.c on smithi016 2023-09-23T19:16:06.542 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:14:13.606604+0000 mon.a (mon.0) 96 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:16:06.542 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:14:15.426946+0000 mon.a (mon.0) 98 : audit 0 from='client.? 172.21.15.16:0/183737010' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:16:06.542 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:14:24.850740+0000 mon.a (mon.0) 121 : cluster 1 overall HEALTH_OK 2023-09-23T19:16:06.542 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:14:25.513772+0000 mgr.y (mgr.25325) 72 : cluster 0 pgmap v55: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 51 MiB/s wr, 915 op/s 2023-09-23T19:16:06.542 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:14:27.359368+0000 mon.a (mon.0) 122 : audit 0 from='client.? 172.21.15.16:0/1182051871' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:16:06.542 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:14:27.514221+0000 mgr.y (mgr.25325) 73 : cluster 0 pgmap v56: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 30 MiB/s wr, 582 op/s 2023-09-23T19:16:06.543 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:14:28.608814+0000 mon.a (mon.0) 123 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:16:06.543 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:14:29.514833+0000 mgr.y (mgr.25325) 74 : cluster 0 pgmap v57: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 30 MiB/s wr, 624 op/s 2023-09-23T19:16:06.543 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:14:31.197105+0000 mon.a (mon.0) 124 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.543 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:14:31.211590+0000 mon.a (mon.0) 125 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.543 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:14:31.515478+0000 mgr.y (mgr.25325) 75 : cluster 0 pgmap v58: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 27 MiB/s wr, 873 op/s 2023-09-23T19:16:06.543 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:14:32.294031+0000 mon.a (mon.0) 126 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.543 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:14:32.303948+0000 mon.a (mon.0) 127 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.543 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:14:33.249135+0000 mgr.y (mgr.25325) 76 : audit 0 from='client.25231 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:16:06.543 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:14:33.516061+0000 mgr.y (mgr.25325) 77 : cluster 0 pgmap v59: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 9.9 MiB/s wr, 596 op/s 2023-09-23T19:16:06.543 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:14:35.516805+0000 mgr.y (mgr.25325) 78 : cluster 0 pgmap v60: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 9.9 MiB/s wr, 891 op/s 2023-09-23T19:16:06.543 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:14:37.517432+0000 mgr.y (mgr.25325) 79 : cluster 0 pgmap v61: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 626 op/s 2023-09-23T19:16:06.543 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:14:38.292783+0000 mgr.y (mgr.25325) 80 : audit 0 from='client.25234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:16:06.543 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cephadm 2023-09-23T19:14:38.293361+0000 mgr.y (mgr.25325) 81 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:16:06.544 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:14:38.299749+0000 mon.a (mon.0) 128 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.544 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:14:39.517874+0000 mgr.y (mgr.25325) 82 : cluster 0 pgmap v62: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 666 op/s 2023-09-23T19:16:06.544 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:14:41.518490+0000 mgr.y (mgr.25325) 83 : cluster 0 pgmap v63: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 905 op/s 2023-09-23T19:16:06.544 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:14:43.518861+0000 mgr.y (mgr.25325) 84 : cluster 0 pgmap v64: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2023-09-23T19:16:06.544 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:14:43.609499+0000 mon.a (mon.0) 129 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:16:06.544 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:14:45.253213+0000 mon.a (mon.0) 130 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.544 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:14:45.268607+0000 mon.a (mon.0) 131 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.544 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:14:45.269883+0000 mon.a (mon.0) 132 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:06.544 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:14:45.270914+0000 mon.a (mon.0) 133 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:16:06.544 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:14:45.277385+0000 mon.a (mon.0) 134 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.544 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:14:45.353643+0000 mon.a (mon.0) 135 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:16:06.544 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:14:45.356202+0000 mon.a (mon.0) 136 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:06.545 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:14:45.360648+0000 mon.a (mon.0) 137 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:16:06.545 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:14:45.370410+0000 mon.a (mon.0) 138 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.545 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:14:45.519558+0000 mgr.y (mgr.25325) 85 : cluster 0 pgmap v65: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 937 op/s 2023-09-23T19:16:06.545 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:14:47.520013+0000 mgr.y (mgr.25325) 86 : cluster 0 pgmap v66: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 642 op/s 2023-09-23T19:16:06.545 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:14:49.520605+0000 mgr.y (mgr.25325) 87 : cluster 0 pgmap v67: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2023-09-23T19:16:06.545 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:14:51.521294+0000 mgr.y (mgr.25325) 88 : cluster 0 pgmap v68: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 918 op/s 2023-09-23T19:16:06.545 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:14:53.521740+0000 mgr.y (mgr.25325) 89 : cluster 0 pgmap v69: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 637 op/s 2023-09-23T19:16:06.545 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:14:55.522324+0000 mgr.y (mgr.25325) 90 : cluster 0 pgmap v70: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 955 op/s 2023-09-23T19:16:06.545 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:14:57.522720+0000 mgr.y (mgr.25325) 91 : cluster 0 pgmap v71: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 633 op/s 2023-09-23T19:16:06.545 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:14:58.609959+0000 mon.a (mon.0) 139 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:16:06.545 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:14:59.523242+0000 mgr.y (mgr.25325) 92 : cluster 0 pgmap v72: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 674 op/s 2023-09-23T19:16:06.545 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:15:01.523817+0000 mgr.y (mgr.25325) 93 : cluster 0 pgmap v73: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 903 op/s 2023-09-23T19:16:06.546 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:15:03.524348+0000 mgr.y (mgr.25325) 94 : cluster 0 pgmap v74: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 626 op/s 2023-09-23T19:16:06.546 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:15:05.525073+0000 mgr.y (mgr.25325) 95 : cluster 0 pgmap v75: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 937 op/s 2023-09-23T19:16:06.546 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:15:07.525526+0000 mgr.y (mgr.25325) 96 : cluster 0 pgmap v76: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 619 op/s 2023-09-23T19:16:06.546 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:15:09.526028+0000 mgr.y (mgr.25325) 97 : cluster 0 pgmap v77: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 657 op/s 2023-09-23T19:16:06.546 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:15:11.526752+0000 mgr.y (mgr.25325) 98 : cluster 0 pgmap v78: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 889 op/s 2023-09-23T19:16:06.546 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:15:13.527188+0000 mgr.y (mgr.25325) 99 : cluster 0 pgmap v79: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 621 op/s 2023-09-23T19:16:06.546 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:15:13.610106+0000 mon.a (mon.0) 140 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:16:06.546 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:15:15.527904+0000 mgr.y (mgr.25325) 100 : cluster 0 pgmap v80: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 926 op/s 2023-09-23T19:16:06.546 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:15:17.528520+0000 mgr.y (mgr.25325) 101 : cluster 0 pgmap v81: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 616 op/s 2023-09-23T19:16:06.546 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:15:19.529035+0000 mgr.y (mgr.25325) 102 : cluster 0 pgmap v82: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 616 op/s 2023-09-23T19:16:06.546 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:15:21.529810+0000 mgr.y (mgr.25325) 103 : cluster 0 pgmap v83: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 883 op/s 2023-09-23T19:16:06.546 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:15:23.530250+0000 mgr.y (mgr.25325) 104 : cluster 0 pgmap v84: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 610 op/s 2023-09-23T19:16:06.547 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:15:25.530969+0000 mgr.y (mgr.25325) 105 : cluster 0 pgmap v85: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 907 op/s 2023-09-23T19:16:06.547 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:15:27.531357+0000 mgr.y (mgr.25325) 106 : cluster 0 pgmap v86: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 602 op/s 2023-09-23T19:16:06.547 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:15:28.619806+0000 mon.a (mon.0) 141 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:16:06.547 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:15:29.531840+0000 mgr.y (mgr.25325) 107 : cluster 0 pgmap v87: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 602 op/s 2023-09-23T19:16:06.547 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:15:31.532626+0000 mgr.y (mgr.25325) 108 : cluster 0 pgmap v88: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 898 op/s 2023-09-23T19:16:06.547 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:15:33.533106+0000 mgr.y (mgr.25325) 109 : cluster 0 pgmap v89: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 592 op/s 2023-09-23T19:16:06.547 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:15:35.533806+0000 mgr.y (mgr.25325) 110 : cluster 0 pgmap v90: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 889 op/s 2023-09-23T19:16:06.547 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:15:37.534248+0000 mgr.y (mgr.25325) 111 : cluster 0 pgmap v91: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-09-23T19:16:06.547 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:15:39.534690+0000 mgr.y (mgr.25325) 112 : cluster 0 pgmap v92: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-09-23T19:16:06.547 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:15:41.535489+0000 mgr.y (mgr.25325) 113 : cluster 0 pgmap v93: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2023-09-23T19:16:06.547 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:15:43.535884+0000 mgr.y (mgr.25325) 114 : cluster 0 pgmap v94: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 597 op/s 2023-09-23T19:16:06.547 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:15:43.611046+0000 mon.a (mon.0) 142 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:16:06.548 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:15:45.460341+0000 mon.a (mon.0) 143 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:16:06.548 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:15:45.536540+0000 mgr.y (mgr.25325) 115 : cluster 0 pgmap v95: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2023-09-23T19:16:06.548 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:15:47.536998+0000 mgr.y (mgr.25325) 116 : cluster 0 pgmap v96: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 596 op/s 2023-09-23T19:16:06.548 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:15:49.537424+0000 mgr.y (mgr.25325) 117 : cluster 0 pgmap v97: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 596 op/s 2023-09-23T19:16:06.548 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:15:50.413481+0000 mgr.y (mgr.25325) 118 : audit 0 from='client.34224 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:16:06.548 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cephadm 2023-09-23T19:15:50.413999+0000 mgr.y (mgr.25325) 119 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:16:06.548 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:15:50.423925+0000 mon.a (mon.0) 144 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.548 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:15:51.540867+0000 mgr.y (mgr.25325) 120 : cluster 0 pgmap v98: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 883 op/s 2023-09-23T19:16:06.548 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:15:51.925924+0000 mon.a (mon.0) 145 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.548 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:15:51.938936+0000 mon.a (mon.0) 146 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.548 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:15:53.351475+0000 mon.a (mon.0) 147 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.548 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:15:53.372664+0000 mon.a (mon.0) 148 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.548 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:15:53.541320+0000 mgr.y (mgr.25325) 121 : cluster 0 pgmap v99: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-09-23T19:16:06.549 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:15:54.013219+0000 mon.a (mon.0) 149 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:06.549 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:15:54.015276+0000 mon.a (mon.0) 150 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:16:06.549 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:15:54.025438+0000 mon.a (mon.0) 151 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.549 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:15:54.106844+0000 mon.a (mon.0) 152 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:16:06.549 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:15:54.111405+0000 mon.a (mon.0) 153 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:16:06.549 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:15:54.113517+0000 mon.a (mon.0) 154 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T19:16:06.549 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:15:54.117249+0000 mon.a (mon.0) 155 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-09-23T19:16:06.549 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cephadm 2023-09-23T19:15:54.118478+0000 mgr.y (mgr.25325) 122 : cephadm 1 Upgrade: It appears safe to stop mon.b 2023-09-23T19:16:06.549 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:15:55.542051+0000 mgr.y (mgr.25325) 123 : cluster 0 pgmap v100: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 843 op/s 2023-09-23T19:16:06.549 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:15:57.542541+0000 mgr.y (mgr.25325) 124 : cluster 0 pgmap v101: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 547 op/s 2023-09-23T19:16:06.549 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:15:58.611300+0000 mon.a (mon.0) 156 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:16:06.549 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cephadm 2023-09-23T19:15:58.858256+0000 mgr.y (mgr.25325) 125 : cephadm 1 Upgrade: Updating mon.b 2023-09-23T19:16:06.550 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:15:58.865673+0000 mon.a (mon.0) 157 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.550 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:15:58.867749+0000 mon.a (mon.0) 158 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T19:16:06.550 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:15:58.868653+0000 mon.a (mon.0) 159 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T19:16:06.550 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:15:58.869657+0000 mon.a (mon.0) 160 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:06.550 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cephadm 2023-09-23T19:15:58.870880+0000 mgr.y (mgr.25325) 126 : cephadm 1 Deploying daemon mon.b on smithi092 2023-09-23T19:16:06.550 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:15:59.542968+0000 mgr.y (mgr.25325) 127 : cluster 0 pgmap v102: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 547 op/s 2023-09-23T19:16:06.550 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:15:59.985784+0000 mon.c (mon.2) 2 : audit 0 from='client.? 172.21.15.16:0/310864528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:16:06.550 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:16:01.544472+0000 mgr.y (mgr.25325) 128 : cluster 0 pgmap v103: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 829 op/s 2023-09-23T19:16:06.550 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:16:05.210863+0000 mon.c (mon.2) 3 : cluster 1 mon.c calling monitor election 2023-09-23T19:16:06.550 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:16:05.214911+0000 mon.b (mon.1) 2 : cluster 1 mon.b calling monitor election 2023-09-23T19:16:06.550 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:16:05.217672+0000 mon.a (mon.0) 168 : cluster 1 overall HEALTH_OK 2023-09-23T19:16:06.550 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:16:05.218777+0000 mon.a (mon.0) 169 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:16:06.551 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:16:05.218969+0000 mon.a (mon.0) 170 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:16:06.551 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:16:05.219155+0000 mon.a (mon.0) 171 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:16:06.551 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:16:05.220979+0000 mon.a (mon.0) 172 : cluster 1 mon.a calling monitor election 2023-09-23T19:16:06.551 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:16:05.226877+0000 mon.a (mon.0) 173 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T19:16:06.551 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:16:05.242224+0000 mon.a (mon.0) 174 : cluster 0 monmap e4: 3 mons at {a=[v2:172.21.15.16:3300/0,v1:172.21.15.16:6789/0],b=[v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0],c=[v2:172.21.15.16:3301/0,v1:172.21.15.16:6790/0]} removed_ranks: {} 2023-09-23T19:16:06.551 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:16:05.242907+0000 mon.a (mon.0) 175 : cluster 0 fsmap foo:1 {0=foo.smithi016.uncssv=up:active} 1 up:standby 2023-09-23T19:16:06.551 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:16:05.242940+0000 mon.a (mon.0) 176 : cluster 0 osdmap e76: 8 total, 8 up, 8 in 2023-09-23T19:16:06.551 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:16:05.245518+0000 mon.a (mon.0) 177 : cluster 0 mgrmap e38: y(active, since 3m), standbys: x 2023-09-23T19:16:06.551 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:16:05.254458+0000 mon.a (mon.0) 178 : cluster 1 overall HEALTH_OK 2023-09-23T19:16:06.551 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:16:05.260731+0000 mon.a (mon.0) 179 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.553 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:16:05.268175+0000 mon.a (mon.0) 180 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.553 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:16:05.317749+0000 mon.a (mon.0) 181 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:16:06.553 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: audit 2023-09-23T19:16:05.523426+0000 mon.a (mon.0) 182 : audit 0 from='client.? 172.21.15.16:0/1642019130' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:16:06.553 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:06 smithi092 bash[58855]: cluster 2023-09-23T19:16:05.545696+0000 mgr.y (mgr.25325) 130 : cluster 0 pgmap v105: 89 pgs: 89 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 827 op/s 2023-09-23T19:16:06.554 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T18:55:23.803111+0000 mon.a (mon.0) 0 : 1 mkfs 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 2023-09-23T19:16:06.554 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cephadm 2023-09-23T19:10:35.535765+0000 mgr.x (mgr.15012) 51 : cephadm 1 Reconfiguring node-exporter.b (dependencies changed)... 2023-09-23T19:16:06.554 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cephadm 2023-09-23T19:10:35.536543+0000 mgr.x (mgr.15012) 52 : cephadm 1 Reconfiguring daemon node-exporter.b on smithi092 2023-09-23T19:16:06.554 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cephadm 2023-09-23T19:10:37.457529+0000 mgr.x (mgr.15012) 54 : cephadm 1 Reconfiguring grafana.a (dependencies changed)... 2023-09-23T19:16:06.554 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cephadm 2023-09-23T19:10:37.481421+0000 mgr.x (mgr.15012) 55 : cephadm 1 Reconfiguring daemon grafana.a on smithi092 2023-09-23T19:16:06.554 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cephadm 2023-09-23T19:11:04.293039+0000 mgr.x (mgr.15012) 72 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T19:16:06.554 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cephadm 2023-09-23T19:11:04.293965+0000 mgr.x (mgr.15012) 73 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T19:16:06.554 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cephadm 2023-09-23T19:11:08.736449+0000 mgr.x (mgr.15012) 76 : cephadm 1 Upgrade: Updating mgr.y 2023-09-23T19:16:06.554 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cephadm 2023-09-23T19:11:08.759144+0000 mgr.x (mgr.15012) 77 : cephadm 1 Deploying daemon mgr.y on smithi016 2023-09-23T19:16:06.554 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cephadm 2023-09-23T19:11:33.777506+0000 mgr.y (mgr.15228) 2 : cephadm 1 [23/Sep/2023:19:11:33] ENGINE Bus STARTING 2023-09-23T19:16:06.555 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cephadm 2023-09-23T19:11:33.911195+0000 mgr.y (mgr.15228) 3 : cephadm 1 [23/Sep/2023:19:11:33] ENGINE Serving on https://172.21.15.16:7150 2023-09-23T19:16:06.555 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cephadm 2023-09-23T19:11:34.016436+0000 mgr.y (mgr.15228) 4 : cephadm 1 [23/Sep/2023:19:11:34] ENGINE Serving on http://172.21.15.16:8765 2023-09-23T19:16:06.555 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cephadm 2023-09-23T19:11:34.016591+0000 mgr.y (mgr.15228) 5 : cephadm 1 [23/Sep/2023:19:11:34] ENGINE Bus STARTED 2023-09-23T19:16:06.555 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cephadm 2023-09-23T19:11:51.722802+0000 mgr.y (mgr.15228) 15 : cephadm 1 Updating smithi016:/etc/ceph/ceph.conf 2023-09-23T19:16:06.555 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cephadm 2023-09-23T19:11:51.723700+0000 mgr.y (mgr.15228) 16 : cephadm 1 Updating smithi092:/etc/ceph/ceph.conf 2023-09-23T19:16:06.555 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cephadm 2023-09-23T19:11:51.947767+0000 mgr.y (mgr.15228) 17 : cephadm 1 Updating smithi092:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:16:06.555 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cephadm 2023-09-23T19:11:51.948762+0000 mgr.y (mgr.15228) 18 : cephadm 1 Updating smithi016:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:16:06.555 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cephadm 2023-09-23T19:11:52.162991+0000 mgr.y (mgr.15228) 19 : cephadm 1 Updating smithi092:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:16:06.555 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cephadm 2023-09-23T19:11:52.163739+0000 mgr.y (mgr.15228) 20 : cephadm 1 Updating smithi016:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:16:06.555 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cephadm 2023-09-23T19:11:52.377394+0000 mgr.y (mgr.15228) 22 : cephadm 1 Updating smithi092:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:16:06.555 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cephadm 2023-09-23T19:11:52.377832+0000 mgr.y (mgr.15228) 23 : cephadm 1 Updating smithi016:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:16:06.555 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cephadm 2023-09-23T19:11:52.778114+0000 mgr.y (mgr.15228) 24 : cephadm 1 Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T19:16:06.555 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cephadm 2023-09-23T19:11:53.069383+0000 mgr.y (mgr.15228) 25 : cephadm 1 Reconfiguring daemon prometheus.a on smithi092 2023-09-23T19:16:06.555 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cephadm 2023-09-23T19:12:03.396080+0000 mgr.y (mgr.15228) 32 : cephadm 1 Upgrade: Updating mgr.x 2023-09-23T19:16:06.556 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cephadm 2023-09-23T19:12:03.409061+0000 mgr.y (mgr.15228) 33 : cephadm 1 Deploying daemon mgr.x on smithi092 2023-09-23T19:16:06.556 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cephadm 2023-09-23T19:12:25.529324+0000 mgr.y (mgr.15228) 45 : cephadm 1 Upgrade: Setting container_image for all mgr 2023-09-23T19:16:06.556 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cephadm 2023-09-23T19:12:25.558592+0000 mgr.y (mgr.15228) 46 : cephadm 1 Upgrade: It appears safe to stop mon.a 2023-09-23T19:16:06.556 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cephadm 2023-09-23T19:12:28.428916+0000 mgr.y (mgr.15228) 49 : cephadm 1 Upgrade: Updating mon.a 2023-09-23T19:16:06.556 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cephadm 2023-09-23T19:12:28.442853+0000 mgr.y (mgr.15228) 50 : cephadm 1 Deploying daemon mon.a on smithi016 2023-09-23T19:16:06.556 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cephadm 2023-09-23T19:12:45.208811+0000 mgr.y (mgr.25325) 3 : cephadm 1 [23/Sep/2023:19:12:45] ENGINE Bus STARTING 2023-09-23T19:16:06.556 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cephadm 2023-09-23T19:12:45.311158+0000 mgr.y (mgr.25325) 4 : cephadm 1 [23/Sep/2023:19:12:45] ENGINE Serving on http://172.21.15.16:8765 2023-09-23T19:16:06.556 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cephadm 2023-09-23T19:12:45.435516+0000 mgr.y (mgr.25325) 5 : cephadm 1 [23/Sep/2023:19:12:45] ENGINE Serving on https://172.21.15.16:7150 2023-09-23T19:16:06.556 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cephadm 2023-09-23T19:12:45.435832+0000 mgr.y (mgr.25325) 6 : cephadm 1 [23/Sep/2023:19:12:45] ENGINE Bus STARTED 2023-09-23T19:16:06.556 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cephadm 2023-09-23T19:12:50.821433+0000 mgr.y (mgr.25325) 11 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:16:06.556 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cephadm 2023-09-23T19:13:03.295263+0000 mgr.y (mgr.25325) 18 : cephadm 1 Updating smithi016:/etc/ceph/ceph.conf 2023-09-23T19:16:06.556 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cephadm 2023-09-23T19:13:03.295553+0000 mgr.y (mgr.25325) 19 : cephadm 1 Updating smithi092:/etc/ceph/ceph.conf 2023-09-23T19:16:06.557 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cephadm 2023-09-23T19:13:03.562963+0000 mgr.y (mgr.25325) 21 : cephadm 1 Updating smithi092:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:16:06.557 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cephadm 2023-09-23T19:13:03.563191+0000 mgr.y (mgr.25325) 22 : cephadm 1 Updating smithi016:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:16:06.557 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cephadm 2023-09-23T19:13:03.827935+0000 mgr.y (mgr.25325) 23 : cephadm 1 Updating smithi092:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:16:06.557 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cephadm 2023-09-23T19:13:03.828145+0000 mgr.y (mgr.25325) 24 : cephadm 1 Updating smithi016:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:16:06.557 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cephadm 2023-09-23T19:13:04.094271+0000 mgr.y (mgr.25325) 25 : cephadm 1 Updating smithi092:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:16:06.557 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cephadm 2023-09-23T19:13:04.097737+0000 mgr.y (mgr.25325) 26 : cephadm 1 Updating smithi016:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:16:06.557 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cephadm 2023-09-23T19:14:02.241916+0000 mgr.y (mgr.25325) 57 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:16:06.557 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cephadm 2023-09-23T19:14:09.619086+0000 mgr.y (mgr.25325) 62 : cephadm 1 Upgrade: It appears safe to stop mon.c 2023-09-23T19:16:06.557 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:14:12.664093+0000 mon.a (mon.0) 91 : audit 0 from='client.? 172.21.15.16:0/4024211665' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:16:06.557 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cephadm 2023-09-23T19:14:13.056347+0000 mgr.y (mgr.25325) 64 : cephadm 1 Upgrade: Updating mon.c 2023-09-23T19:16:06.557 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:14:13.066564+0000 mon.a (mon.0) 92 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.557 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:14:13.068124+0000 mon.a (mon.0) 93 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T19:16:06.558 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:14:13.073670+0000 mon.a (mon.0) 94 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T19:16:06.558 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:14:13.075842+0000 mon.a (mon.0) 95 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:06.558 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cephadm 2023-09-23T19:14:13.077863+0000 mgr.y (mgr.25325) 65 : cephadm 1 Deploying daemon mon.c on smithi016 2023-09-23T19:16:06.558 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:14:13.606604+0000 mon.a (mon.0) 96 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:16:06.558 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:14:15.426946+0000 mon.a (mon.0) 98 : audit 0 from='client.? 172.21.15.16:0/183737010' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:16:06.558 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:14:24.850740+0000 mon.a (mon.0) 121 : cluster 1 overall HEALTH_OK 2023-09-23T19:16:06.558 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:14:25.513772+0000 mgr.y (mgr.25325) 72 : cluster 0 pgmap v55: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 51 MiB/s wr, 915 op/s 2023-09-23T19:16:06.558 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:14:27.359368+0000 mon.a (mon.0) 122 : audit 0 from='client.? 172.21.15.16:0/1182051871' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:16:06.558 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:14:27.514221+0000 mgr.y (mgr.25325) 73 : cluster 0 pgmap v56: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 30 MiB/s wr, 582 op/s 2023-09-23T19:16:06.558 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:14:28.608814+0000 mon.a (mon.0) 123 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:16:06.558 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:14:29.514833+0000 mgr.y (mgr.25325) 74 : cluster 0 pgmap v57: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 30 MiB/s wr, 624 op/s 2023-09-23T19:16:06.558 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:14:31.197105+0000 mon.a (mon.0) 124 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.559 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:14:31.211590+0000 mon.a (mon.0) 125 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.559 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:14:31.515478+0000 mgr.y (mgr.25325) 75 : cluster 0 pgmap v58: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 27 MiB/s wr, 873 op/s 2023-09-23T19:16:06.559 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:14:32.294031+0000 mon.a (mon.0) 126 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.559 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:14:32.303948+0000 mon.a (mon.0) 127 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.559 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:14:33.249135+0000 mgr.y (mgr.25325) 76 : audit 0 from='client.25231 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:16:06.559 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:14:33.516061+0000 mgr.y (mgr.25325) 77 : cluster 0 pgmap v59: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 9.9 MiB/s wr, 596 op/s 2023-09-23T19:16:06.559 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:14:35.516805+0000 mgr.y (mgr.25325) 78 : cluster 0 pgmap v60: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 9.9 MiB/s wr, 891 op/s 2023-09-23T19:16:06.559 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:14:37.517432+0000 mgr.y (mgr.25325) 79 : cluster 0 pgmap v61: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 626 op/s 2023-09-23T19:16:06.559 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:14:38.292783+0000 mgr.y (mgr.25325) 80 : audit 0 from='client.25234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:16:06.559 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cephadm 2023-09-23T19:14:38.293361+0000 mgr.y (mgr.25325) 81 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:16:06.559 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:14:38.299749+0000 mon.a (mon.0) 128 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.559 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:14:39.517874+0000 mgr.y (mgr.25325) 82 : cluster 0 pgmap v62: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 666 op/s 2023-09-23T19:16:06.559 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:14:41.518490+0000 mgr.y (mgr.25325) 83 : cluster 0 pgmap v63: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 905 op/s 2023-09-23T19:16:06.560 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:14:43.518861+0000 mgr.y (mgr.25325) 84 : cluster 0 pgmap v64: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2023-09-23T19:16:06.560 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:14:43.609499+0000 mon.a (mon.0) 129 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:16:06.560 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:14:45.253213+0000 mon.a (mon.0) 130 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.560 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:14:45.268607+0000 mon.a (mon.0) 131 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.560 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:14:45.269883+0000 mon.a (mon.0) 132 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:06.560 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:14:45.270914+0000 mon.a (mon.0) 133 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:16:06.560 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:14:45.277385+0000 mon.a (mon.0) 134 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.560 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:14:45.353643+0000 mon.a (mon.0) 135 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:16:06.560 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:14:45.356202+0000 mon.a (mon.0) 136 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:06.560 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:14:45.360648+0000 mon.a (mon.0) 137 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:16:06.560 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:14:45.370410+0000 mon.a (mon.0) 138 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.560 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:14:45.519558+0000 mgr.y (mgr.25325) 85 : cluster 0 pgmap v65: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 937 op/s 2023-09-23T19:16:06.561 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:14:47.520013+0000 mgr.y (mgr.25325) 86 : cluster 0 pgmap v66: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 642 op/s 2023-09-23T19:16:06.561 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:14:49.520605+0000 mgr.y (mgr.25325) 87 : cluster 0 pgmap v67: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2023-09-23T19:16:06.561 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T18:55:23.803111+0000 mon.a (mon.0) 0 : 1 mkfs 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 2023-09-23T19:16:06.561 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cephadm 2023-09-23T19:10:35.535765+0000 mgr.x (mgr.15012) 51 : cephadm 1 Reconfiguring node-exporter.b (dependencies changed)... 2023-09-23T19:16:06.561 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cephadm 2023-09-23T19:10:35.536543+0000 mgr.x (mgr.15012) 52 : cephadm 1 Reconfiguring daemon node-exporter.b on smithi092 2023-09-23T19:16:06.561 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cephadm 2023-09-23T19:10:37.457529+0000 mgr.x (mgr.15012) 54 : cephadm 1 Reconfiguring grafana.a (dependencies changed)... 2023-09-23T19:16:06.561 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cephadm 2023-09-23T19:10:37.481421+0000 mgr.x (mgr.15012) 55 : cephadm 1 Reconfiguring daemon grafana.a on smithi092 2023-09-23T19:16:06.561 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cephadm 2023-09-23T19:11:04.293039+0000 mgr.x (mgr.15012) 72 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T19:16:06.561 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cephadm 2023-09-23T19:11:04.293965+0000 mgr.x (mgr.15012) 73 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T19:16:06.562 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cephadm 2023-09-23T19:11:08.736449+0000 mgr.x (mgr.15012) 76 : cephadm 1 Upgrade: Updating mgr.y 2023-09-23T19:16:06.562 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cephadm 2023-09-23T19:11:08.759144+0000 mgr.x (mgr.15012) 77 : cephadm 1 Deploying daemon mgr.y on smithi016 2023-09-23T19:16:06.562 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cephadm 2023-09-23T19:11:33.777506+0000 mgr.y (mgr.15228) 2 : cephadm 1 [23/Sep/2023:19:11:33] ENGINE Bus STARTING 2023-09-23T19:16:06.562 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cephadm 2023-09-23T19:11:33.911195+0000 mgr.y (mgr.15228) 3 : cephadm 1 [23/Sep/2023:19:11:33] ENGINE Serving on https://172.21.15.16:7150 2023-09-23T19:16:06.562 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cephadm 2023-09-23T19:11:34.016436+0000 mgr.y (mgr.15228) 4 : cephadm 1 [23/Sep/2023:19:11:34] ENGINE Serving on http://172.21.15.16:8765 2023-09-23T19:16:06.562 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cephadm 2023-09-23T19:11:34.016591+0000 mgr.y (mgr.15228) 5 : cephadm 1 [23/Sep/2023:19:11:34] ENGINE Bus STARTED 2023-09-23T19:16:06.562 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cephadm 2023-09-23T19:11:51.722802+0000 mgr.y (mgr.15228) 15 : cephadm 1 Updating smithi016:/etc/ceph/ceph.conf 2023-09-23T19:16:06.562 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cephadm 2023-09-23T19:11:51.723700+0000 mgr.y (mgr.15228) 16 : cephadm 1 Updating smithi092:/etc/ceph/ceph.conf 2023-09-23T19:16:06.562 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cephadm 2023-09-23T19:11:51.947767+0000 mgr.y (mgr.15228) 17 : cephadm 1 Updating smithi092:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:16:06.562 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cephadm 2023-09-23T19:11:51.948762+0000 mgr.y (mgr.15228) 18 : cephadm 1 Updating smithi016:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:16:06.562 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cephadm 2023-09-23T19:11:52.162991+0000 mgr.y (mgr.15228) 19 : cephadm 1 Updating smithi092:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:16:06.562 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cephadm 2023-09-23T19:11:52.163739+0000 mgr.y (mgr.15228) 20 : cephadm 1 Updating smithi016:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:16:06.563 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cephadm 2023-09-23T19:11:52.377394+0000 mgr.y (mgr.15228) 22 : cephadm 1 Updating smithi092:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:16:06.563 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cephadm 2023-09-23T19:11:52.377832+0000 mgr.y (mgr.15228) 23 : cephadm 1 Updating smithi016:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:16:06.563 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cephadm 2023-09-23T19:11:52.778114+0000 mgr.y (mgr.15228) 24 : cephadm 1 Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T19:16:06.563 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cephadm 2023-09-23T19:11:53.069383+0000 mgr.y (mgr.15228) 25 : cephadm 1 Reconfiguring daemon prometheus.a on smithi092 2023-09-23T19:16:06.563 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cephadm 2023-09-23T19:12:03.396080+0000 mgr.y (mgr.15228) 32 : cephadm 1 Upgrade: Updating mgr.x 2023-09-23T19:16:06.563 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cephadm 2023-09-23T19:12:03.409061+0000 mgr.y (mgr.15228) 33 : cephadm 1 Deploying daemon mgr.x on smithi092 2023-09-23T19:16:06.563 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cephadm 2023-09-23T19:12:25.529324+0000 mgr.y (mgr.15228) 45 : cephadm 1 Upgrade: Setting container_image for all mgr 2023-09-23T19:16:06.563 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cephadm 2023-09-23T19:12:25.558592+0000 mgr.y (mgr.15228) 46 : cephadm 1 Upgrade: It appears safe to stop mon.a 2023-09-23T19:16:06.563 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cephadm 2023-09-23T19:12:28.428916+0000 mgr.y (mgr.15228) 49 : cephadm 1 Upgrade: Updating mon.a 2023-09-23T19:16:06.563 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cephadm 2023-09-23T19:12:28.442853+0000 mgr.y (mgr.15228) 50 : cephadm 1 Deploying daemon mon.a on smithi016 2023-09-23T19:16:06.563 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cephadm 2023-09-23T19:12:45.208811+0000 mgr.y (mgr.25325) 3 : cephadm 1 [23/Sep/2023:19:12:45] ENGINE Bus STARTING 2023-09-23T19:16:06.563 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cephadm 2023-09-23T19:12:45.311158+0000 mgr.y (mgr.25325) 4 : cephadm 1 [23/Sep/2023:19:12:45] ENGINE Serving on http://172.21.15.16:8765 2023-09-23T19:16:06.564 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cephadm 2023-09-23T19:12:45.435516+0000 mgr.y (mgr.25325) 5 : cephadm 1 [23/Sep/2023:19:12:45] ENGINE Serving on https://172.21.15.16:7150 2023-09-23T19:16:06.564 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cephadm 2023-09-23T19:12:45.435832+0000 mgr.y (mgr.25325) 6 : cephadm 1 [23/Sep/2023:19:12:45] ENGINE Bus STARTED 2023-09-23T19:16:06.564 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cephadm 2023-09-23T19:12:50.821433+0000 mgr.y (mgr.25325) 11 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:16:06.564 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cephadm 2023-09-23T19:13:03.295263+0000 mgr.y (mgr.25325) 18 : cephadm 1 Updating smithi016:/etc/ceph/ceph.conf 2023-09-23T19:16:06.564 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cephadm 2023-09-23T19:13:03.295553+0000 mgr.y (mgr.25325) 19 : cephadm 1 Updating smithi092:/etc/ceph/ceph.conf 2023-09-23T19:16:06.564 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cephadm 2023-09-23T19:13:03.562963+0000 mgr.y (mgr.25325) 21 : cephadm 1 Updating smithi092:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:16:06.564 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cephadm 2023-09-23T19:13:03.563191+0000 mgr.y (mgr.25325) 22 : cephadm 1 Updating smithi016:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:16:06.564 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cephadm 2023-09-23T19:13:03.827935+0000 mgr.y (mgr.25325) 23 : cephadm 1 Updating smithi092:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:16:06.564 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cephadm 2023-09-23T19:13:03.828145+0000 mgr.y (mgr.25325) 24 : cephadm 1 Updating smithi016:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:16:06.564 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cephadm 2023-09-23T19:13:04.094271+0000 mgr.y (mgr.25325) 25 : cephadm 1 Updating smithi092:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:16:06.564 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cephadm 2023-09-23T19:13:04.097737+0000 mgr.y (mgr.25325) 26 : cephadm 1 Updating smithi016:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:16:06.564 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cephadm 2023-09-23T19:14:02.241916+0000 mgr.y (mgr.25325) 57 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:16:06.564 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cephadm 2023-09-23T19:14:09.619086+0000 mgr.y (mgr.25325) 62 : cephadm 1 Upgrade: It appears safe to stop mon.c 2023-09-23T19:16:06.565 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:14:12.664093+0000 mon.a (mon.0) 91 : audit 0 from='client.? 172.21.15.16:0/4024211665' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:16:06.565 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cephadm 2023-09-23T19:14:13.056347+0000 mgr.y (mgr.25325) 64 : cephadm 1 Upgrade: Updating mon.c 2023-09-23T19:16:06.565 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:14:13.066564+0000 mon.a (mon.0) 92 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.565 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:14:13.068124+0000 mon.a (mon.0) 93 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T19:16:06.565 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:14:13.073670+0000 mon.a (mon.0) 94 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T19:16:06.565 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:14:13.075842+0000 mon.a (mon.0) 95 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:06.565 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cephadm 2023-09-23T19:14:13.077863+0000 mgr.y (mgr.25325) 65 : cephadm 1 Deploying daemon mon.c on smithi016 2023-09-23T19:16:06.565 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:14:13.606604+0000 mon.a (mon.0) 96 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:16:06.565 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:14:15.426946+0000 mon.a (mon.0) 98 : audit 0 from='client.? 172.21.15.16:0/183737010' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:16:06.565 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:14:24.850740+0000 mon.a (mon.0) 121 : cluster 1 overall HEALTH_OK 2023-09-23T19:16:06.565 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:14:25.513772+0000 mgr.y (mgr.25325) 72 : cluster 0 pgmap v55: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 51 MiB/s wr, 915 op/s 2023-09-23T19:16:06.565 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:14:27.359368+0000 mon.a (mon.0) 122 : audit 0 from='client.? 172.21.15.16:0/1182051871' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:16:06.566 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:14:27.514221+0000 mgr.y (mgr.25325) 73 : cluster 0 pgmap v56: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 30 MiB/s wr, 582 op/s 2023-09-23T19:16:06.566 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:14:28.608814+0000 mon.a (mon.0) 123 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:16:06.566 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:14:29.514833+0000 mgr.y (mgr.25325) 74 : cluster 0 pgmap v57: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 30 MiB/s wr, 624 op/s 2023-09-23T19:16:06.566 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:14:31.197105+0000 mon.a (mon.0) 124 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.566 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:14:31.211590+0000 mon.a (mon.0) 125 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.566 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:14:31.515478+0000 mgr.y (mgr.25325) 75 : cluster 0 pgmap v58: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 27 MiB/s wr, 873 op/s 2023-09-23T19:16:06.566 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:14:32.294031+0000 mon.a (mon.0) 126 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.566 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:14:32.303948+0000 mon.a (mon.0) 127 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.566 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:14:33.249135+0000 mgr.y (mgr.25325) 76 : audit 0 from='client.25231 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:16:06.566 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:14:33.516061+0000 mgr.y (mgr.25325) 77 : cluster 0 pgmap v59: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 9.9 MiB/s wr, 596 op/s 2023-09-23T19:16:06.566 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:14:35.516805+0000 mgr.y (mgr.25325) 78 : cluster 0 pgmap v60: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 9.9 MiB/s wr, 891 op/s 2023-09-23T19:16:06.567 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:14:37.517432+0000 mgr.y (mgr.25325) 79 : cluster 0 pgmap v61: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 626 op/s 2023-09-23T19:16:06.567 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:14:38.292783+0000 mgr.y (mgr.25325) 80 : audit 0 from='client.25234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:16:06.567 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cephadm 2023-09-23T19:14:38.293361+0000 mgr.y (mgr.25325) 81 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:16:06.567 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:14:38.299749+0000 mon.a (mon.0) 128 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.567 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:14:39.517874+0000 mgr.y (mgr.25325) 82 : cluster 0 pgmap v62: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 666 op/s 2023-09-23T19:16:06.567 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:14:41.518490+0000 mgr.y (mgr.25325) 83 : cluster 0 pgmap v63: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 905 op/s 2023-09-23T19:16:06.567 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:14:43.518861+0000 mgr.y (mgr.25325) 84 : cluster 0 pgmap v64: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2023-09-23T19:16:06.567 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:14:43.609499+0000 mon.a (mon.0) 129 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:16:06.567 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:14:45.253213+0000 mon.a (mon.0) 130 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.567 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:14:45.268607+0000 mon.a (mon.0) 131 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.567 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:14:45.269883+0000 mon.a (mon.0) 132 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:06.567 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:14:45.270914+0000 mon.a (mon.0) 133 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:16:06.568 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:14:45.277385+0000 mon.a (mon.0) 134 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.568 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:14:45.353643+0000 mon.a (mon.0) 135 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:16:06.568 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:14:45.356202+0000 mon.a (mon.0) 136 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:06.568 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:14:45.360648+0000 mon.a (mon.0) 137 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:16:06.568 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:14:45.370410+0000 mon.a (mon.0) 138 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.568 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:14:45.519558+0000 mgr.y (mgr.25325) 85 : cluster 0 pgmap v65: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 937 op/s 2023-09-23T19:16:06.568 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:14:47.520013+0000 mgr.y (mgr.25325) 86 : cluster 0 pgmap v66: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 642 op/s 2023-09-23T19:16:06.568 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:14:49.520605+0000 mgr.y (mgr.25325) 87 : cluster 0 pgmap v67: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2023-09-23T19:16:06.568 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:14:51.521294+0000 mgr.y (mgr.25325) 88 : cluster 0 pgmap v68: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 918 op/s 2023-09-23T19:16:06.568 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:14:53.521740+0000 mgr.y (mgr.25325) 89 : cluster 0 pgmap v69: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 637 op/s 2023-09-23T19:16:06.569 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:14:55.522324+0000 mgr.y (mgr.25325) 90 : cluster 0 pgmap v70: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 955 op/s 2023-09-23T19:16:06.569 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:14:57.522720+0000 mgr.y (mgr.25325) 91 : cluster 0 pgmap v71: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 633 op/s 2023-09-23T19:16:06.569 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:14:58.609959+0000 mon.a (mon.0) 139 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:16:06.569 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:14:59.523242+0000 mgr.y (mgr.25325) 92 : cluster 0 pgmap v72: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 674 op/s 2023-09-23T19:16:06.569 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:15:01.523817+0000 mgr.y (mgr.25325) 93 : cluster 0 pgmap v73: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 903 op/s 2023-09-23T19:16:06.569 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:15:03.524348+0000 mgr.y (mgr.25325) 94 : cluster 0 pgmap v74: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 626 op/s 2023-09-23T19:16:06.569 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:15:05.525073+0000 mgr.y (mgr.25325) 95 : cluster 0 pgmap v75: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 937 op/s 2023-09-23T19:16:06.569 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:15:07.525526+0000 mgr.y (mgr.25325) 96 : cluster 0 pgmap v76: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 619 op/s 2023-09-23T19:16:06.569 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:15:09.526028+0000 mgr.y (mgr.25325) 97 : cluster 0 pgmap v77: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 657 op/s 2023-09-23T19:16:06.569 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:15:11.526752+0000 mgr.y (mgr.25325) 98 : cluster 0 pgmap v78: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 889 op/s 2023-09-23T19:16:06.569 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:15:13.527188+0000 mgr.y (mgr.25325) 99 : cluster 0 pgmap v79: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 621 op/s 2023-09-23T19:16:06.569 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:15:13.610106+0000 mon.a (mon.0) 140 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:16:06.570 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:15:15.527904+0000 mgr.y (mgr.25325) 100 : cluster 0 pgmap v80: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 926 op/s 2023-09-23T19:16:06.570 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:15:17.528520+0000 mgr.y (mgr.25325) 101 : cluster 0 pgmap v81: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 616 op/s 2023-09-23T19:16:06.570 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:15:19.529035+0000 mgr.y (mgr.25325) 102 : cluster 0 pgmap v82: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 616 op/s 2023-09-23T19:16:06.570 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:15:21.529810+0000 mgr.y (mgr.25325) 103 : cluster 0 pgmap v83: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 883 op/s 2023-09-23T19:16:06.570 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:15:23.530250+0000 mgr.y (mgr.25325) 104 : cluster 0 pgmap v84: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 610 op/s 2023-09-23T19:16:06.570 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:15:25.530969+0000 mgr.y (mgr.25325) 105 : cluster 0 pgmap v85: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 907 op/s 2023-09-23T19:16:06.570 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:15:27.531357+0000 mgr.y (mgr.25325) 106 : cluster 0 pgmap v86: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 602 op/s 2023-09-23T19:16:06.570 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:15:28.619806+0000 mon.a (mon.0) 141 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:16:06.570 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:15:29.531840+0000 mgr.y (mgr.25325) 107 : cluster 0 pgmap v87: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 602 op/s 2023-09-23T19:16:06.570 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:15:31.532626+0000 mgr.y (mgr.25325) 108 : cluster 0 pgmap v88: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 898 op/s 2023-09-23T19:16:06.570 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:15:33.533106+0000 mgr.y (mgr.25325) 109 : cluster 0 pgmap v89: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 592 op/s 2023-09-23T19:16:06.570 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:15:35.533806+0000 mgr.y (mgr.25325) 110 : cluster 0 pgmap v90: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 889 op/s 2023-09-23T19:16:06.571 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:15:37.534248+0000 mgr.y (mgr.25325) 111 : cluster 0 pgmap v91: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-09-23T19:16:06.571 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:15:39.534690+0000 mgr.y (mgr.25325) 112 : cluster 0 pgmap v92: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-09-23T19:16:06.571 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:15:41.535489+0000 mgr.y (mgr.25325) 113 : cluster 0 pgmap v93: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2023-09-23T19:16:06.571 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:15:43.535884+0000 mgr.y (mgr.25325) 114 : cluster 0 pgmap v94: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 597 op/s 2023-09-23T19:16:06.571 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:15:43.611046+0000 mon.a (mon.0) 142 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:16:06.571 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:15:45.460341+0000 mon.a (mon.0) 143 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:16:06.571 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:15:45.536540+0000 mgr.y (mgr.25325) 115 : cluster 0 pgmap v95: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2023-09-23T19:16:06.571 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:15:47.536998+0000 mgr.y (mgr.25325) 116 : cluster 0 pgmap v96: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 596 op/s 2023-09-23T19:16:06.571 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:15:49.537424+0000 mgr.y (mgr.25325) 117 : cluster 0 pgmap v97: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 596 op/s 2023-09-23T19:16:06.571 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:15:50.413481+0000 mgr.y (mgr.25325) 118 : audit 0 from='client.34224 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:16:06.571 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cephadm 2023-09-23T19:15:50.413999+0000 mgr.y (mgr.25325) 119 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:16:06.571 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:15:50.423925+0000 mon.a (mon.0) 144 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.571 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:15:51.540867+0000 mgr.y (mgr.25325) 120 : cluster 0 pgmap v98: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 883 op/s 2023-09-23T19:16:06.572 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:15:51.925924+0000 mon.a (mon.0) 145 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.572 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:15:51.938936+0000 mon.a (mon.0) 146 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.572 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:15:53.351475+0000 mon.a (mon.0) 147 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.572 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:15:53.372664+0000 mon.a (mon.0) 148 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.572 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:15:53.541320+0000 mgr.y (mgr.25325) 121 : cluster 0 pgmap v99: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-09-23T19:16:06.572 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:15:54.013219+0000 mon.a (mon.0) 149 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:06.572 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:15:54.015276+0000 mon.a (mon.0) 150 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:16:06.572 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:15:54.025438+0000 mon.a (mon.0) 151 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.572 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:15:54.106844+0000 mon.a (mon.0) 152 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:16:06.572 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:15:54.111405+0000 mon.a (mon.0) 153 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:16:06.572 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:15:54.113517+0000 mon.a (mon.0) 154 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T19:16:06.572 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:15:54.117249+0000 mon.a (mon.0) 155 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-09-23T19:16:06.572 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cephadm 2023-09-23T19:15:54.118478+0000 mgr.y (mgr.25325) 122 : cephadm 1 Upgrade: It appears safe to stop mon.b 2023-09-23T19:16:06.573 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:15:55.542051+0000 mgr.y (mgr.25325) 123 : cluster 0 pgmap v100: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 843 op/s 2023-09-23T19:16:06.573 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:15:57.542541+0000 mgr.y (mgr.25325) 124 : cluster 0 pgmap v101: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 547 op/s 2023-09-23T19:16:06.573 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:15:58.611300+0000 mon.a (mon.0) 156 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:16:06.573 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cephadm 2023-09-23T19:15:58.858256+0000 mgr.y (mgr.25325) 125 : cephadm 1 Upgrade: Updating mon.b 2023-09-23T19:16:06.573 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:15:58.865673+0000 mon.a (mon.0) 157 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.573 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:15:58.867749+0000 mon.a (mon.0) 158 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T19:16:06.573 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:15:58.868653+0000 mon.a (mon.0) 159 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T19:16:06.573 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:15:58.869657+0000 mon.a (mon.0) 160 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:06.573 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cephadm 2023-09-23T19:15:58.870880+0000 mgr.y (mgr.25325) 126 : cephadm 1 Deploying daemon mon.b on smithi092 2023-09-23T19:16:06.573 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:15:59.542968+0000 mgr.y (mgr.25325) 127 : cluster 0 pgmap v102: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 547 op/s 2023-09-23T19:16:06.573 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:15:59.985784+0000 mon.c (mon.2) 2 : audit 0 from='client.? 172.21.15.16:0/310864528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:16:06.573 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:16:01.544472+0000 mgr.y (mgr.25325) 128 : cluster 0 pgmap v103: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 829 op/s 2023-09-23T19:16:06.573 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:16:05.210863+0000 mon.c (mon.2) 3 : cluster 1 mon.c calling monitor election 2023-09-23T19:16:06.573 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:16:05.214911+0000 mon.b (mon.1) 2 : cluster 1 mon.b calling monitor election 2023-09-23T19:16:06.574 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:16:05.217672+0000 mon.a (mon.0) 168 : cluster 1 overall HEALTH_OK 2023-09-23T19:16:06.574 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:16:05.218777+0000 mon.a (mon.0) 169 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:16:06.574 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:16:05.218969+0000 mon.a (mon.0) 170 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:16:06.574 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:16:05.219155+0000 mon.a (mon.0) 171 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:16:06.574 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:16:05.220979+0000 mon.a (mon.0) 172 : cluster 1 mon.a calling monitor election 2023-09-23T19:16:06.574 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:16:05.226877+0000 mon.a (mon.0) 173 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T19:16:06.574 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:16:05.242224+0000 mon.a (mon.0) 174 : cluster 0 monmap e4: 3 mons at {a=[v2:172.21.15.16:3300/0,v1:172.21.15.16:6789/0],b=[v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0],c=[v2:172.21.15.16:3301/0,v1:172.21.15.16:6790/0]} removed_ranks: {} 2023-09-23T19:16:06.574 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:16:05.242907+0000 mon.a (mon.0) 175 : cluster 0 fsmap foo:1 {0=foo.smithi016.uncssv=up:active} 1 up:standby 2023-09-23T19:16:06.574 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:16:05.242940+0000 mon.a (mon.0) 176 : cluster 0 osdmap e76: 8 total, 8 up, 8 in 2023-09-23T19:16:06.574 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:16:05.245518+0000 mon.a (mon.0) 177 : cluster 0 mgrmap e38: y(active, since 3m), standbys: x 2023-09-23T19:16:06.574 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:16:05.254458+0000 mon.a (mon.0) 178 : cluster 1 overall HEALTH_OK 2023-09-23T19:16:06.574 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:16:05.260731+0000 mon.a (mon.0) 179 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.575 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:16:05.268175+0000 mon.a (mon.0) 180 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.575 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:16:05.317749+0000 mon.a (mon.0) 181 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:16:06.575 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: audit 2023-09-23T19:16:05.523426+0000 mon.a (mon.0) 182 : audit 0 from='client.? 172.21.15.16:0/1642019130' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:16:06.575 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:14:51.521294+0000 mgr.y (mgr.25325) 88 : cluster 0 pgmap v68: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 918 op/s 2023-09-23T19:16:06.575 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:14:53.521740+0000 mgr.y (mgr.25325) 89 : cluster 0 pgmap v69: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 637 op/s 2023-09-23T19:16:06.575 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:14:55.522324+0000 mgr.y (mgr.25325) 90 : cluster 0 pgmap v70: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 955 op/s 2023-09-23T19:16:06.575 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:14:57.522720+0000 mgr.y (mgr.25325) 91 : cluster 0 pgmap v71: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 633 op/s 2023-09-23T19:16:06.575 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:14:58.609959+0000 mon.a (mon.0) 139 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:16:06.575 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:14:59.523242+0000 mgr.y (mgr.25325) 92 : cluster 0 pgmap v72: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 674 op/s 2023-09-23T19:16:06.575 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:15:01.523817+0000 mgr.y (mgr.25325) 93 : cluster 0 pgmap v73: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 903 op/s 2023-09-23T19:16:06.576 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:15:03.524348+0000 mgr.y (mgr.25325) 94 : cluster 0 pgmap v74: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 626 op/s 2023-09-23T19:16:06.576 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:15:05.525073+0000 mgr.y (mgr.25325) 95 : cluster 0 pgmap v75: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 937 op/s 2023-09-23T19:16:06.576 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:15:07.525526+0000 mgr.y (mgr.25325) 96 : cluster 0 pgmap v76: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 619 op/s 2023-09-23T19:16:06.576 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:15:09.526028+0000 mgr.y (mgr.25325) 97 : cluster 0 pgmap v77: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 657 op/s 2023-09-23T19:16:06.576 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:15:11.526752+0000 mgr.y (mgr.25325) 98 : cluster 0 pgmap v78: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 889 op/s 2023-09-23T19:16:06.576 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:15:13.527188+0000 mgr.y (mgr.25325) 99 : cluster 0 pgmap v79: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 621 op/s 2023-09-23T19:16:06.576 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:15:13.610106+0000 mon.a (mon.0) 140 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:16:06.576 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:15:15.527904+0000 mgr.y (mgr.25325) 100 : cluster 0 pgmap v80: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 926 op/s 2023-09-23T19:16:06.576 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:15:17.528520+0000 mgr.y (mgr.25325) 101 : cluster 0 pgmap v81: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 616 op/s 2023-09-23T19:16:06.576 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:15:19.529035+0000 mgr.y (mgr.25325) 102 : cluster 0 pgmap v82: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 616 op/s 2023-09-23T19:16:06.576 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:15:21.529810+0000 mgr.y (mgr.25325) 103 : cluster 0 pgmap v83: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 883 op/s 2023-09-23T19:16:06.576 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:15:23.530250+0000 mgr.y (mgr.25325) 104 : cluster 0 pgmap v84: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 610 op/s 2023-09-23T19:16:06.577 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:15:25.530969+0000 mgr.y (mgr.25325) 105 : cluster 0 pgmap v85: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 907 op/s 2023-09-23T19:16:06.577 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:15:27.531357+0000 mgr.y (mgr.25325) 106 : cluster 0 pgmap v86: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 602 op/s 2023-09-23T19:16:06.577 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:15:28.619806+0000 mon.a (mon.0) 141 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:16:06.577 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:15:29.531840+0000 mgr.y (mgr.25325) 107 : cluster 0 pgmap v87: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 602 op/s 2023-09-23T19:16:06.577 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:15:31.532626+0000 mgr.y (mgr.25325) 108 : cluster 0 pgmap v88: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 898 op/s 2023-09-23T19:16:06.577 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:15:33.533106+0000 mgr.y (mgr.25325) 109 : cluster 0 pgmap v89: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 592 op/s 2023-09-23T19:16:06.577 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:15:35.533806+0000 mgr.y (mgr.25325) 110 : cluster 0 pgmap v90: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 889 op/s 2023-09-23T19:16:06.577 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:15:37.534248+0000 mgr.y (mgr.25325) 111 : cluster 0 pgmap v91: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-09-23T19:16:06.577 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:15:39.534690+0000 mgr.y (mgr.25325) 112 : cluster 0 pgmap v92: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-09-23T19:16:06.577 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:15:41.535489+0000 mgr.y (mgr.25325) 113 : cluster 0 pgmap v93: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2023-09-23T19:16:06.577 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:15:43.535884+0000 mgr.y (mgr.25325) 114 : cluster 0 pgmap v94: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 597 op/s 2023-09-23T19:16:06.577 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:15:43.611046+0000 mon.a (mon.0) 142 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:16:06.578 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:15:45.460341+0000 mon.a (mon.0) 143 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:16:06.578 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:15:45.536540+0000 mgr.y (mgr.25325) 115 : cluster 0 pgmap v95: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2023-09-23T19:16:06.578 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:15:47.536998+0000 mgr.y (mgr.25325) 116 : cluster 0 pgmap v96: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 596 op/s 2023-09-23T19:16:06.578 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:15:49.537424+0000 mgr.y (mgr.25325) 117 : cluster 0 pgmap v97: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 596 op/s 2023-09-23T19:16:06.578 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:15:50.413481+0000 mgr.y (mgr.25325) 118 : audit 0 from='client.34224 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:16:06.578 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cephadm 2023-09-23T19:15:50.413999+0000 mgr.y (mgr.25325) 119 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:16:06.578 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:15:50.423925+0000 mon.a (mon.0) 144 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.578 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:15:51.540867+0000 mgr.y (mgr.25325) 120 : cluster 0 pgmap v98: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 883 op/s 2023-09-23T19:16:06.578 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:15:51.925924+0000 mon.a (mon.0) 145 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.578 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:15:51.938936+0000 mon.a (mon.0) 146 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.578 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:15:53.351475+0000 mon.a (mon.0) 147 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.578 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:15:53.372664+0000 mon.a (mon.0) 148 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.579 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:15:53.541320+0000 mgr.y (mgr.25325) 121 : cluster 0 pgmap v99: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-09-23T19:16:06.579 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:15:54.013219+0000 mon.a (mon.0) 149 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:06.579 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:15:54.015276+0000 mon.a (mon.0) 150 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:16:06.579 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:15:54.025438+0000 mon.a (mon.0) 151 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.579 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:15:54.106844+0000 mon.a (mon.0) 152 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:16:06.579 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:15:54.111405+0000 mon.a (mon.0) 153 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:16:06.579 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:15:54.113517+0000 mon.a (mon.0) 154 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T19:16:06.579 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:15:54.117249+0000 mon.a (mon.0) 155 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-09-23T19:16:06.579 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cephadm 2023-09-23T19:15:54.118478+0000 mgr.y (mgr.25325) 122 : cephadm 1 Upgrade: It appears safe to stop mon.b 2023-09-23T19:16:06.579 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:15:55.542051+0000 mgr.y (mgr.25325) 123 : cluster 0 pgmap v100: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 843 op/s 2023-09-23T19:16:06.579 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:15:57.542541+0000 mgr.y (mgr.25325) 124 : cluster 0 pgmap v101: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 547 op/s 2023-09-23T19:16:06.579 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:15:58.611300+0000 mon.a (mon.0) 156 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:16:06.579 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cephadm 2023-09-23T19:15:58.858256+0000 mgr.y (mgr.25325) 125 : cephadm 1 Upgrade: Updating mon.b 2023-09-23T19:16:06.580 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:15:58.865673+0000 mon.a (mon.0) 157 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.580 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:15:58.867749+0000 mon.a (mon.0) 158 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T19:16:06.580 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:15:58.868653+0000 mon.a (mon.0) 159 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T19:16:06.580 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:15:58.869657+0000 mon.a (mon.0) 160 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:06.580 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cephadm 2023-09-23T19:15:58.870880+0000 mgr.y (mgr.25325) 126 : cephadm 1 Deploying daemon mon.b on smithi092 2023-09-23T19:16:06.580 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:15:59.542968+0000 mgr.y (mgr.25325) 127 : cluster 0 pgmap v102: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 547 op/s 2023-09-23T19:16:06.580 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:15:59.985784+0000 mon.c (mon.2) 2 : audit 0 from='client.? 172.21.15.16:0/310864528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:16:06.580 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:16:01.544472+0000 mgr.y (mgr.25325) 128 : cluster 0 pgmap v103: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 829 op/s 2023-09-23T19:16:06.580 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:16:05.210863+0000 mon.c (mon.2) 3 : cluster 1 mon.c calling monitor election 2023-09-23T19:16:06.580 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:16:05.214911+0000 mon.b (mon.1) 2 : cluster 1 mon.b calling monitor election 2023-09-23T19:16:06.580 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:16:05.217672+0000 mon.a (mon.0) 168 : cluster 1 overall HEALTH_OK 2023-09-23T19:16:06.580 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:16:05.218777+0000 mon.a (mon.0) 169 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:16:06.581 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:16:05.218969+0000 mon.a (mon.0) 170 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:16:06.581 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:16:05.219155+0000 mon.a (mon.0) 171 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:16:06.581 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:16:05.220979+0000 mon.a (mon.0) 172 : cluster 1 mon.a calling monitor election 2023-09-23T19:16:06.581 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:16:05.226877+0000 mon.a (mon.0) 173 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T19:16:06.581 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:16:05.242224+0000 mon.a (mon.0) 174 : cluster 0 monmap e4: 3 mons at {a=[v2:172.21.15.16:3300/0,v1:172.21.15.16:6789/0],b=[v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0],c=[v2:172.21.15.16:3301/0,v1:172.21.15.16:6790/0]} removed_ranks: {} 2023-09-23T19:16:06.581 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:16:05.242907+0000 mon.a (mon.0) 175 : cluster 0 fsmap foo:1 {0=foo.smithi016.uncssv=up:active} 1 up:standby 2023-09-23T19:16:06.581 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:16:05.242940+0000 mon.a (mon.0) 176 : cluster 0 osdmap e76: 8 total, 8 up, 8 in 2023-09-23T19:16:06.581 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:16:05.245518+0000 mon.a (mon.0) 177 : cluster 0 mgrmap e38: y(active, since 3m), standbys: x 2023-09-23T19:16:06.581 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:16:05.254458+0000 mon.a (mon.0) 178 : cluster 1 overall HEALTH_OK 2023-09-23T19:16:06.581 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:16:05.260731+0000 mon.a (mon.0) 179 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.581 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:16:05.268175+0000 mon.a (mon.0) 180 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:06.581 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:16:05.317749+0000 mon.a (mon.0) 181 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:16:06.581 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: audit 2023-09-23T19:16:05.523426+0000 mon.a (mon.0) 182 : audit 0 from='client.? 172.21.15.16:0/1642019130' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:16:06.582 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[75004]: cluster 2023-09-23T19:16:05.545696+0000 mgr.y (mgr.25325) 130 : cluster 0 pgmap v105: 89 pgs: 89 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 827 op/s 2023-09-23T19:16:06.582 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:06 smithi016 bash[77995]: cluster 2023-09-23T19:16:05.545696+0000 mgr.y (mgr.25325) 130 : cluster 0 pgmap v105: 89 pgs: 89 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 827 op/s 2023-09-23T19:16:07.287 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T19:16:08.917 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:08 smithi016 bash[77995]: cluster 2023-09-23T19:16:07.546044+0000 mgr.y (mgr.25325) 131 : cluster 0 pgmap v106: 89 pgs: 89 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 566 op/s 2023-09-23T19:16:08.917 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:08 smithi016 bash[75004]: cluster 2023-09-23T19:16:07.546044+0000 mgr.y (mgr.25325) 131 : cluster 0 pgmap v106: 89 pgs: 89 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 566 op/s 2023-09-23T19:16:09.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:08 smithi092 bash[58855]: cluster 2023-09-23T19:16:07.546044+0000 mgr.y (mgr.25325) 131 : cluster 0 pgmap v106: 89 pgs: 89 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 566 op/s 2023-09-23T19:16:09.258 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:16:09 smithi016 bash[70992]: debug 2023-09-23T19:16:09.255+0000 7fb2a5a7c700 -1 mgr.server handle_report got status from non-daemon mon.b 2023-09-23T19:16:10.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:10 smithi016 bash[75004]: cluster 2023-09-23T19:16:09.546469+0000 mgr.y (mgr.25325) 132 : cluster 0 pgmap v107: 89 pgs: 89 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 566 op/s 2023-09-23T19:16:10.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:10 smithi016 bash[77995]: cluster 2023-09-23T19:16:09.546469+0000 mgr.y (mgr.25325) 132 : cluster 0 pgmap v107: 89 pgs: 89 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 566 op/s 2023-09-23T19:16:10.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:10 smithi092 bash[58855]: cluster 2023-09-23T19:16:09.546469+0000 mgr.y (mgr.25325) 132 : cluster 0 pgmap v107: 89 pgs: 89 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 566 op/s 2023-09-23T19:16:11.256 INFO:teuthology.orchestra.run.smithi016.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T19:16:11.256 INFO:teuthology.orchestra.run.smithi016.stdout:alertmanager.a smithi016 *:9093,9094 running (5m) - 12m 13.0M - 0.23.0 ba2b418f427c b362c3d9bae2 2023-09-23T19:16:11.256 INFO:teuthology.orchestra.run.smithi016.stdout:grafana.a smithi092 *:3000 running (5m) - 11m 41.6M - 8.3.5 dad864ee21e9 891a61cf5fff 2023-09-23T19:16:11.256 INFO:teuthology.orchestra.run.smithi016.stdout:mds.foo.smithi016.uncssv smithi016 running (10m) - 10m 16.8M - 16.2.14-37-g2e1f1015 1e6a756174a8 95d1e187e3eb 2023-09-23T19:16:11.257 INFO:teuthology.orchestra.run.smithi016.stdout:mds.foo.smithi092.vbpqks smithi092 running (10m) - 10m 16.0M - 16.2.14-37-g2e1f1015 1e6a756174a8 1d619ec31bf1 2023-09-23T19:16:11.257 INFO:teuthology.orchestra.run.smithi016.stdout:mgr.x smithi092 *:8443,9283,8765 running (4m) - 18m 427M - 18.2.0-460-gbb0daeff e32c2a64c126 d54b4354eade 2023-09-23T19:16:11.257 INFO:teuthology.orchestra.run.smithi016.stdout:mgr.y smithi016 *:8443,9283,8765 running (4m) - 20m 476M - 18.2.0-460-gbb0daeff e32c2a64c126 19706820b434 2023-09-23T19:16:11.257 INFO:teuthology.orchestra.run.smithi016.stdout:mon.a smithi016 running (3m) - 20m 60.9M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 e5aad364678c 2023-09-23T19:16:11.257 INFO:teuthology.orchestra.run.smithi016.stdout:mon.b smithi092 starting - - - 2048M 2023-09-23T19:16:11.257 INFO:teuthology.orchestra.run.smithi016.stdout:mon.c smithi016 running (113s) - 19m 35.1M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 61a05c2c22a7 2023-09-23T19:16:11.257 INFO:teuthology.orchestra.run.smithi016.stdout:node-exporter.a smithi016 *:9100 running (6m) - 12m 10.8M - 1.3.1 1dbe0e931976 7f13d7a8905d 2023-09-23T19:16:11.257 INFO:teuthology.orchestra.run.smithi016.stdout:node-exporter.b smithi092 *:9100 running (5m) - 12m 11.7M - 1.3.1 1dbe0e931976 4823347a98b5 2023-09-23T19:16:11.257 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0 smithi016 running (18m) - 18m 414M 2525M 16.2.14-37-g2e1f1015 1e6a756174a8 1458de442fb1 2023-09-23T19:16:11.257 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1 smithi016 running (17m) - 17m 350M 2525M 16.2.14-37-g2e1f1015 1e6a756174a8 7e0e07b2f6fd 2023-09-23T19:16:11.257 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2 smithi016 running (16m) - 16m 509M 2525M 16.2.14-37-g2e1f1015 1e6a756174a8 0c6fd3da0e21 2023-09-23T19:16:11.258 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3 smithi016 running (16m) - 16m 496M 2525M 16.2.14-37-g2e1f1015 1e6a756174a8 a34971c269fe 2023-09-23T19:16:11.258 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4 smithi092 running (15m) - 15m 435M 2524M 16.2.14-37-g2e1f1015 1e6a756174a8 253dc4b11010 2023-09-23T19:16:11.258 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5 smithi092 running (14m) - 14m 473M 2524M 16.2.14-37-g2e1f1015 1e6a756174a8 0a7306534382 2023-09-23T19:16:11.258 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6 smithi092 running (13m) - 14m 309M 2524M 16.2.14-37-g2e1f1015 1e6a756174a8 d6f3ee67cc5f 2023-09-23T19:16:11.258 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7 smithi092 running (13m) - 13m 418M 2524M 16.2.14-37-g2e1f1015 1e6a756174a8 794cf2340f17 2023-09-23T19:16:11.258 INFO:teuthology.orchestra.run.smithi016.stdout:prometheus.a smithi092 *:9095 running (4m) - 12m 58.0M - 2.33.4 514e6a882f6e 2666eba9f715 2023-09-23T19:16:12.207 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade pause' 2023-09-23T19:16:12.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:12 smithi016 bash[75004]: audit 2023-09-23T19:16:11.237614+0000 mgr.y (mgr.25325) 133 : audit 0 from='client.34245 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:16:12.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:12 smithi016 bash[75004]: cluster 2023-09-23T19:16:11.547072+0000 mgr.y (mgr.25325) 134 : cluster 0 pgmap v108: 89 pgs: 89 active+clean; 2.1 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 848 op/s 2023-09-23T19:16:12.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:12 smithi016 bash[75004]: audit 2023-09-23T19:16:11.767470+0000 mon.a (mon.0) 183 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:12.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:12 smithi016 bash[75004]: audit 2023-09-23T19:16:11.777054+0000 mon.a (mon.0) 184 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:12.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:12 smithi016 bash[77995]: audit 2023-09-23T19:16:11.237614+0000 mgr.y (mgr.25325) 133 : audit 0 from='client.34245 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:16:12.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:12 smithi016 bash[77995]: cluster 2023-09-23T19:16:11.547072+0000 mgr.y (mgr.25325) 134 : cluster 0 pgmap v108: 89 pgs: 89 active+clean; 2.1 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 848 op/s 2023-09-23T19:16:12.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:12 smithi016 bash[77995]: audit 2023-09-23T19:16:11.767470+0000 mon.a (mon.0) 183 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:12.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:12 smithi016 bash[77995]: audit 2023-09-23T19:16:11.777054+0000 mon.a (mon.0) 184 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:13.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:12 smithi092 bash[58855]: audit 2023-09-23T19:16:11.237614+0000 mgr.y (mgr.25325) 133 : audit 0 from='client.34245 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:16:13.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:12 smithi092 bash[58855]: cluster 2023-09-23T19:16:11.547072+0000 mgr.y (mgr.25325) 134 : cluster 0 pgmap v108: 89 pgs: 89 active+clean; 2.1 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 848 op/s 2023-09-23T19:16:13.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:12 smithi092 bash[58855]: audit 2023-09-23T19:16:11.767470+0000 mon.a (mon.0) 183 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:13.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:12 smithi092 bash[58855]: audit 2023-09-23T19:16:11.777054+0000 mon.a (mon.0) 184 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:13.941 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:13 smithi016 bash[75004]: audit 2023-09-23T19:16:12.729802+0000 mon.a (mon.0) 185 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:13.941 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:13 smithi016 bash[75004]: audit 2023-09-23T19:16:12.736955+0000 mon.a (mon.0) 186 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:13.942 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:13 smithi016 bash[75004]: cluster 2023-09-23T19:16:13.547521+0000 mgr.y (mgr.25325) 135 : cluster 0 pgmap v109: 89 pgs: 89 active+clean; 2.1 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 567 op/s 2023-09-23T19:16:13.942 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:13 smithi016 bash[75004]: audit 2023-09-23T19:16:13.611652+0000 mon.a (mon.0) 187 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:16:13.942 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:13 smithi016 bash[77995]: audit 2023-09-23T19:16:12.729802+0000 mon.a (mon.0) 185 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:13.942 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:13 smithi016 bash[77995]: audit 2023-09-23T19:16:12.736955+0000 mon.a (mon.0) 186 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:13.942 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:13 smithi016 bash[77995]: cluster 2023-09-23T19:16:13.547521+0000 mgr.y (mgr.25325) 135 : cluster 0 pgmap v109: 89 pgs: 89 active+clean; 2.1 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 567 op/s 2023-09-23T19:16:13.942 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:13 smithi016 bash[77995]: audit 2023-09-23T19:16:13.611652+0000 mon.a (mon.0) 187 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:16:14.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:13 smithi092 bash[58855]: audit 2023-09-23T19:16:12.729802+0000 mon.a (mon.0) 185 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:14.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:13 smithi092 bash[58855]: audit 2023-09-23T19:16:12.736955+0000 mon.a (mon.0) 186 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:14.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:13 smithi092 bash[58855]: cluster 2023-09-23T19:16:13.547521+0000 mgr.y (mgr.25325) 135 : cluster 0 pgmap v109: 89 pgs: 89 active+clean; 2.1 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 567 op/s 2023-09-23T19:16:14.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:13 smithi092 bash[58855]: audit 2023-09-23T19:16:13.611652+0000 mon.a (mon.0) 187 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:16:16.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:16:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:16:15] "GET /metrics HTTP/1.1" 200 36053 "" "Prometheus/2.33.4" 2023-09-23T19:16:16.289 INFO:teuthology.orchestra.run.smithi016.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:16:16.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:16 smithi016 bash[77995]: cluster 2023-09-23T19:16:15.548191+0000 mgr.y (mgr.25325) 136 : cluster 0 pgmap v110: 89 pgs: 89 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 845 op/s 2023-09-23T19:16:16.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:16 smithi016 bash[77995]: audit 2023-09-23T19:16:16.286538+0000 mon.a (mon.0) 188 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:16.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:16 smithi016 bash[75004]: cluster 2023-09-23T19:16:15.548191+0000 mgr.y (mgr.25325) 136 : cluster 0 pgmap v110: 89 pgs: 89 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 845 op/s 2023-09-23T19:16:16.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:16 smithi016 bash[75004]: audit 2023-09-23T19:16:16.286538+0000 mon.a (mon.0) 188 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:17.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:16 smithi092 bash[58855]: cluster 2023-09-23T19:16:15.548191+0000 mgr.y (mgr.25325) 136 : cluster 0 pgmap v110: 89 pgs: 89 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 845 op/s 2023-09-23T19:16:17.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:16 smithi092 bash[58855]: audit 2023-09-23T19:16:16.286538+0000 mon.a (mon.0) 188 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:17.273 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'sleep 60' 2023-09-23T19:16:18.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:17 smithi016 bash[77995]: audit 2023-09-23T19:16:16.278526+0000 mgr.y (mgr.25325) 137 : audit 0 from='client.44129 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:16:18.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:17 smithi016 bash[77995]: cephadm 2023-09-23T19:16:16.279021+0000 mgr.y (mgr.25325) 138 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:16:18.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:17 smithi016 bash[75004]: audit 2023-09-23T19:16:16.278526+0000 mgr.y (mgr.25325) 137 : audit 0 from='client.44129 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:16:18.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:17 smithi016 bash[75004]: cephadm 2023-09-23T19:16:16.279021+0000 mgr.y (mgr.25325) 138 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:16:18.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:17 smithi092 bash[58855]: audit 2023-09-23T19:16:16.278526+0000 mgr.y (mgr.25325) 137 : audit 0 from='client.44129 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:16:18.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:17 smithi092 bash[58855]: cephadm 2023-09-23T19:16:16.279021+0000 mgr.y (mgr.25325) 138 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:16:18.903 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:18 smithi016 bash[77995]: cluster 2023-09-23T19:16:17.548625+0000 mgr.y (mgr.25325) 139 : cluster 0 pgmap v111: 89 pgs: 89 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 560 op/s 2023-09-23T19:16:18.903 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:18 smithi016 bash[75004]: cluster 2023-09-23T19:16:17.548625+0000 mgr.y (mgr.25325) 139 : cluster 0 pgmap v111: 89 pgs: 89 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 560 op/s 2023-09-23T19:16:19.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:18 smithi092 bash[58855]: cluster 2023-09-23T19:16:17.548625+0000 mgr.y (mgr.25325) 139 : cluster 0 pgmap v111: 89 pgs: 89 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 560 op/s 2023-09-23T19:16:20.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:19 smithi016 bash[77995]: cluster 2023-09-23T19:16:19.549120+0000 mgr.y (mgr.25325) 140 : cluster 0 pgmap v112: 89 pgs: 89 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 560 op/s 2023-09-23T19:16:20.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:19 smithi016 bash[75004]: cluster 2023-09-23T19:16:19.549120+0000 mgr.y (mgr.25325) 140 : cluster 0 pgmap v112: 89 pgs: 89 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 560 op/s 2023-09-23T19:16:20.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:19 smithi092 bash[58855]: cluster 2023-09-23T19:16:19.549120+0000 mgr.y (mgr.25325) 140 : cluster 0 pgmap v112: 89 pgs: 89 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 560 op/s 2023-09-23T19:16:22.981 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:22 smithi092 bash[58855]: cluster 2023-09-23T19:16:21.549859+0000 mgr.y (mgr.25325) 141 : cluster 0 pgmap v113: 89 pgs: 89 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 845 op/s 2023-09-23T19:16:23.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:22 smithi016 bash[75004]: cluster 2023-09-23T19:16:21.549859+0000 mgr.y (mgr.25325) 141 : cluster 0 pgmap v113: 89 pgs: 89 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 845 op/s 2023-09-23T19:16:23.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:22 smithi016 bash[77995]: cluster 2023-09-23T19:16:21.549859+0000 mgr.y (mgr.25325) 141 : cluster 0 pgmap v113: 89 pgs: 89 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 845 op/s 2023-09-23T19:16:25.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:24 smithi016 bash[77995]: cluster 2023-09-23T19:16:23.550307+0000 mgr.y (mgr.25325) 142 : cluster 0 pgmap v114: 89 pgs: 89 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 562 op/s 2023-09-23T19:16:25.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:24 smithi016 bash[77995]: audit 2023-09-23T19:16:23.704858+0000 mon.a (mon.0) 189 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:25.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:24 smithi016 bash[77995]: audit 2023-09-23T19:16:23.722444+0000 mon.a (mon.0) 190 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:25.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:24 smithi016 bash[77995]: audit 2023-09-23T19:16:23.728170+0000 mon.a (mon.0) 191 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:25.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:24 smithi016 bash[77995]: audit 2023-09-23T19:16:23.730298+0000 mon.a (mon.0) 192 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:16:25.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:24 smithi016 bash[77995]: audit 2023-09-23T19:16:23.739594+0000 mon.a (mon.0) 193 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:25.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:24 smithi016 bash[77995]: audit 2023-09-23T19:16:23.809789+0000 mon.a (mon.0) 194 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T19:16:25.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:24 smithi016 bash[77995]: audit 2023-09-23T19:16:23.811126+0000 mon.a (mon.0) 195 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T19:16:25.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:24 smithi016 bash[77995]: audit 2023-09-23T19:16:23.812220+0000 mon.a (mon.0) 196 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:25.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:24 smithi016 bash[75004]: cluster 2023-09-23T19:16:23.550307+0000 mgr.y (mgr.25325) 142 : cluster 0 pgmap v114: 89 pgs: 89 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 562 op/s 2023-09-23T19:16:25.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:24 smithi016 bash[75004]: audit 2023-09-23T19:16:23.704858+0000 mon.a (mon.0) 189 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:25.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:24 smithi016 bash[75004]: audit 2023-09-23T19:16:23.722444+0000 mon.a (mon.0) 190 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:25.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:24 smithi016 bash[75004]: audit 2023-09-23T19:16:23.728170+0000 mon.a (mon.0) 191 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:25.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:24 smithi016 bash[75004]: audit 2023-09-23T19:16:23.730298+0000 mon.a (mon.0) 192 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:16:25.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:24 smithi016 bash[75004]: audit 2023-09-23T19:16:23.739594+0000 mon.a (mon.0) 193 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:25.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:24 smithi016 bash[75004]: audit 2023-09-23T19:16:23.809789+0000 mon.a (mon.0) 194 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T19:16:25.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:24 smithi016 bash[75004]: audit 2023-09-23T19:16:23.811126+0000 mon.a (mon.0) 195 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T19:16:25.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:24 smithi016 bash[75004]: audit 2023-09-23T19:16:23.812220+0000 mon.a (mon.0) 196 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:25.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:24 smithi092 bash[58855]: cluster 2023-09-23T19:16:23.550307+0000 mgr.y (mgr.25325) 142 : cluster 0 pgmap v114: 89 pgs: 89 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 562 op/s 2023-09-23T19:16:25.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:24 smithi092 bash[58855]: audit 2023-09-23T19:16:23.704858+0000 mon.a (mon.0) 189 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:25.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:24 smithi092 bash[58855]: audit 2023-09-23T19:16:23.722444+0000 mon.a (mon.0) 190 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:25.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:24 smithi092 bash[58855]: audit 2023-09-23T19:16:23.728170+0000 mon.a (mon.0) 191 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:25.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:24 smithi092 bash[58855]: audit 2023-09-23T19:16:23.730298+0000 mon.a (mon.0) 192 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:16:25.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:24 smithi092 bash[58855]: audit 2023-09-23T19:16:23.739594+0000 mon.a (mon.0) 193 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:25.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:24 smithi092 bash[58855]: audit 2023-09-23T19:16:23.809789+0000 mon.a (mon.0) 194 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T19:16:25.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:24 smithi092 bash[58855]: audit 2023-09-23T19:16:23.811126+0000 mon.a (mon.0) 195 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T19:16:25.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:24 smithi092 bash[58855]: audit 2023-09-23T19:16:23.812220+0000 mon.a (mon.0) 196 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:25.947 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:25 smithi016 bash[77995]: cephadm 2023-09-23T19:16:23.809311+0000 mgr.y (mgr.25325) 143 : cephadm 1 Reconfiguring mon.a (monmap changed)... 2023-09-23T19:16:25.947 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:25 smithi016 bash[77995]: cephadm 2023-09-23T19:16:23.813727+0000 mgr.y (mgr.25325) 144 : cephadm 1 Reconfiguring daemon mon.a on smithi016 2023-09-23T19:16:25.947 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:25 smithi016 bash[75004]: cephadm 2023-09-23T19:16:23.809311+0000 mgr.y (mgr.25325) 143 : cephadm 1 Reconfiguring mon.a (monmap changed)... 2023-09-23T19:16:25.947 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:25 smithi016 bash[75004]: cephadm 2023-09-23T19:16:23.813727+0000 mgr.y (mgr.25325) 144 : cephadm 1 Reconfiguring daemon mon.a on smithi016 2023-09-23T19:16:25.948 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:16:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:16:25] "GET /metrics HTTP/1.1" 200 36053 "" "Prometheus/2.33.4" 2023-09-23T19:16:26.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:25 smithi092 bash[58855]: cephadm 2023-09-23T19:16:23.809311+0000 mgr.y (mgr.25325) 143 : cephadm 1 Reconfiguring mon.a (monmap changed)... 2023-09-23T19:16:26.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:25 smithi092 bash[58855]: cephadm 2023-09-23T19:16:23.813727+0000 mgr.y (mgr.25325) 144 : cephadm 1 Reconfiguring daemon mon.a on smithi016 2023-09-23T19:16:27.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:26 smithi016 bash[77995]: cluster 2023-09-23T19:16:25.550920+0000 mgr.y (mgr.25325) 145 : cluster 0 pgmap v115: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 848 op/s 2023-09-23T19:16:27.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:26 smithi016 bash[75004]: cluster 2023-09-23T19:16:25.550920+0000 mgr.y (mgr.25325) 145 : cluster 0 pgmap v115: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 848 op/s 2023-09-23T19:16:27.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:26 smithi092 bash[58855]: cluster 2023-09-23T19:16:25.550920+0000 mgr.y (mgr.25325) 145 : cluster 0 pgmap v115: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 848 op/s 2023-09-23T19:16:28.205 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:27 smithi016 bash[77995]: audit 2023-09-23T19:16:26.813711+0000 mon.a (mon.0) 197 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:28.205 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:27 smithi016 bash[77995]: audit 2023-09-23T19:16:26.832196+0000 mon.a (mon.0) 198 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:28.205 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:27 smithi016 bash[77995]: cephadm 2023-09-23T19:16:26.838908+0000 mgr.y (mgr.25325) 146 : cephadm 1 Reconfiguring mgr.y (monmap changed)... 2023-09-23T19:16:28.205 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:27 smithi016 bash[77995]: audit 2023-09-23T19:16:26.839750+0000 mon.a (mon.0) 199 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T19:16:28.205 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:27 smithi016 bash[77995]: audit 2023-09-23T19:16:26.842373+0000 mon.a (mon.0) 200 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T19:16:28.206 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:27 smithi016 bash[77995]: audit 2023-09-23T19:16:26.844695+0000 mon.a (mon.0) 201 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:28.206 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:27 smithi016 bash[77995]: cephadm 2023-09-23T19:16:26.846612+0000 mgr.y (mgr.25325) 147 : cephadm 1 Reconfiguring daemon mgr.y on smithi016 2023-09-23T19:16:28.206 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:27 smithi016 bash[77995]: cluster 2023-09-23T19:16:27.551287+0000 mgr.y (mgr.25325) 148 : cluster 0 pgmap v116: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 570 op/s 2023-09-23T19:16:28.206 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:27 smithi016 bash[75004]: audit 2023-09-23T19:16:26.813711+0000 mon.a (mon.0) 197 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:28.206 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:27 smithi016 bash[75004]: audit 2023-09-23T19:16:26.832196+0000 mon.a (mon.0) 198 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:28.206 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:27 smithi016 bash[75004]: cephadm 2023-09-23T19:16:26.838908+0000 mgr.y (mgr.25325) 146 : cephadm 1 Reconfiguring mgr.y (monmap changed)... 2023-09-23T19:16:28.206 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:27 smithi016 bash[75004]: audit 2023-09-23T19:16:26.839750+0000 mon.a (mon.0) 199 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T19:16:28.206 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:27 smithi016 bash[75004]: audit 2023-09-23T19:16:26.842373+0000 mon.a (mon.0) 200 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T19:16:28.206 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:27 smithi016 bash[75004]: audit 2023-09-23T19:16:26.844695+0000 mon.a (mon.0) 201 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:28.207 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:27 smithi016 bash[75004]: cephadm 2023-09-23T19:16:26.846612+0000 mgr.y (mgr.25325) 147 : cephadm 1 Reconfiguring daemon mgr.y on smithi016 2023-09-23T19:16:28.207 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:27 smithi016 bash[75004]: cluster 2023-09-23T19:16:27.551287+0000 mgr.y (mgr.25325) 148 : cluster 0 pgmap v116: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 570 op/s 2023-09-23T19:16:28.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:27 smithi092 bash[58855]: audit 2023-09-23T19:16:26.813711+0000 mon.a (mon.0) 197 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:28.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:27 smithi092 bash[58855]: audit 2023-09-23T19:16:26.832196+0000 mon.a (mon.0) 198 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:28.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:27 smithi092 bash[58855]: cephadm 2023-09-23T19:16:26.838908+0000 mgr.y (mgr.25325) 146 : cephadm 1 Reconfiguring mgr.y (monmap changed)... 2023-09-23T19:16:28.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:27 smithi092 bash[58855]: audit 2023-09-23T19:16:26.839750+0000 mon.a (mon.0) 199 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T19:16:28.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:27 smithi092 bash[58855]: audit 2023-09-23T19:16:26.842373+0000 mon.a (mon.0) 200 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T19:16:28.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:27 smithi092 bash[58855]: audit 2023-09-23T19:16:26.844695+0000 mon.a (mon.0) 201 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:28.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:27 smithi092 bash[58855]: cephadm 2023-09-23T19:16:26.846612+0000 mgr.y (mgr.25325) 147 : cephadm 1 Reconfiguring daemon mgr.y on smithi016 2023-09-23T19:16:28.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:27 smithi092 bash[58855]: cluster 2023-09-23T19:16:27.551287+0000 mgr.y (mgr.25325) 148 : cluster 0 pgmap v116: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 570 op/s 2023-09-23T19:16:29.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:28 smithi016 bash[75004]: audit 2023-09-23T19:16:28.612036+0000 mon.a (mon.0) 202 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:16:29.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:28 smithi016 bash[77995]: audit 2023-09-23T19:16:28.612036+0000 mon.a (mon.0) 202 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:16:29.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:28 smithi092 bash[58855]: audit 2023-09-23T19:16:28.612036+0000 mon.a (mon.0) 202 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:16:30.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:30 smithi016 bash[75004]: audit 2023-09-23T19:16:29.281069+0000 mon.a (mon.0) 203 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:30.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:30 smithi016 bash[75004]: audit 2023-09-23T19:16:29.293461+0000 mon.a (mon.0) 204 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:30.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:30 smithi016 bash[75004]: cephadm 2023-09-23T19:16:29.334903+0000 mgr.y (mgr.25325) 149 : cephadm 1 Reconfiguring mon.c (monmap changed)... 2023-09-23T19:16:30.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:30 smithi016 bash[75004]: audit 2023-09-23T19:16:29.335363+0000 mon.a (mon.0) 205 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T19:16:30.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:30 smithi016 bash[75004]: audit 2023-09-23T19:16:29.340844+0000 mon.a (mon.0) 206 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T19:16:30.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:30 smithi016 bash[75004]: audit 2023-09-23T19:16:29.341986+0000 mon.a (mon.0) 207 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:30.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:30 smithi016 bash[75004]: cephadm 2023-09-23T19:16:29.345008+0000 mgr.y (mgr.25325) 150 : cephadm 1 Reconfiguring daemon mon.c on smithi016 2023-09-23T19:16:30.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:30 smithi016 bash[75004]: cluster 2023-09-23T19:16:29.551701+0000 mgr.y (mgr.25325) 151 : cluster 0 pgmap v117: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 570 op/s 2023-09-23T19:16:30.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:30 smithi016 bash[77995]: audit 2023-09-23T19:16:29.281069+0000 mon.a (mon.0) 203 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:30.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:30 smithi016 bash[77995]: audit 2023-09-23T19:16:29.293461+0000 mon.a (mon.0) 204 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:30.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:30 smithi016 bash[77995]: cephadm 2023-09-23T19:16:29.334903+0000 mgr.y (mgr.25325) 149 : cephadm 1 Reconfiguring mon.c (monmap changed)... 2023-09-23T19:16:30.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:30 smithi016 bash[77995]: audit 2023-09-23T19:16:29.335363+0000 mon.a (mon.0) 205 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T19:16:30.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:30 smithi016 bash[77995]: audit 2023-09-23T19:16:29.340844+0000 mon.a (mon.0) 206 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T19:16:30.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:30 smithi016 bash[77995]: audit 2023-09-23T19:16:29.341986+0000 mon.a (mon.0) 207 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:30.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:30 smithi016 bash[77995]: cephadm 2023-09-23T19:16:29.345008+0000 mgr.y (mgr.25325) 150 : cephadm 1 Reconfiguring daemon mon.c on smithi016 2023-09-23T19:16:30.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:30 smithi016 bash[77995]: cluster 2023-09-23T19:16:29.551701+0000 mgr.y (mgr.25325) 151 : cluster 0 pgmap v117: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 570 op/s 2023-09-23T19:16:30.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:30 smithi092 bash[58855]: audit 2023-09-23T19:16:29.281069+0000 mon.a (mon.0) 203 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:30.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:30 smithi092 bash[58855]: audit 2023-09-23T19:16:29.293461+0000 mon.a (mon.0) 204 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:30.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:30 smithi092 bash[58855]: cephadm 2023-09-23T19:16:29.334903+0000 mgr.y (mgr.25325) 149 : cephadm 1 Reconfiguring mon.c (monmap changed)... 2023-09-23T19:16:30.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:30 smithi092 bash[58855]: audit 2023-09-23T19:16:29.335363+0000 mon.a (mon.0) 205 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T19:16:30.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:30 smithi092 bash[58855]: audit 2023-09-23T19:16:29.340844+0000 mon.a (mon.0) 206 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T19:16:30.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:30 smithi092 bash[58855]: audit 2023-09-23T19:16:29.341986+0000 mon.a (mon.0) 207 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:30.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:30 smithi092 bash[58855]: cephadm 2023-09-23T19:16:29.345008+0000 mgr.y (mgr.25325) 150 : cephadm 1 Reconfiguring daemon mon.c on smithi016 2023-09-23T19:16:30.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:30 smithi092 bash[58855]: cluster 2023-09-23T19:16:29.551701+0000 mgr.y (mgr.25325) 151 : cluster 0 pgmap v117: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 570 op/s 2023-09-23T19:16:33.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:32 smithi016 bash[75004]: cluster 2023-09-23T19:16:31.552490+0000 mgr.y (mgr.25325) 152 : cluster 0 pgmap v118: 89 pgs: 89 active+clean; 1.3 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 854 op/s 2023-09-23T19:16:33.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:32 smithi016 bash[75004]: audit 2023-09-23T19:16:32.135204+0000 mon.a (mon.0) 208 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:33.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:32 smithi016 bash[75004]: audit 2023-09-23T19:16:32.147955+0000 mon.a (mon.0) 209 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:33.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:32 smithi016 bash[75004]: audit 2023-09-23T19:16:32.176087+0000 mon.a (mon.0) 210 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T19:16:33.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:32 smithi016 bash[75004]: audit 2023-09-23T19:16:32.177296+0000 mon.a (mon.0) 211 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:33.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:32 smithi016 bash[77995]: cluster 2023-09-23T19:16:31.552490+0000 mgr.y (mgr.25325) 152 : cluster 0 pgmap v118: 89 pgs: 89 active+clean; 1.3 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 854 op/s 2023-09-23T19:16:33.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:32 smithi016 bash[77995]: audit 2023-09-23T19:16:32.135204+0000 mon.a (mon.0) 208 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:33.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:32 smithi016 bash[77995]: audit 2023-09-23T19:16:32.147955+0000 mon.a (mon.0) 209 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:33.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:32 smithi016 bash[77995]: audit 2023-09-23T19:16:32.176087+0000 mon.a (mon.0) 210 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T19:16:33.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:32 smithi016 bash[77995]: audit 2023-09-23T19:16:32.177296+0000 mon.a (mon.0) 211 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:33.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:32 smithi092 bash[58855]: cluster 2023-09-23T19:16:31.552490+0000 mgr.y (mgr.25325) 152 : cluster 0 pgmap v118: 89 pgs: 89 active+clean; 1.3 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 854 op/s 2023-09-23T19:16:33.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:32 smithi092 bash[58855]: audit 2023-09-23T19:16:32.135204+0000 mon.a (mon.0) 208 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:33.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:32 smithi092 bash[58855]: audit 2023-09-23T19:16:32.147955+0000 mon.a (mon.0) 209 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:33.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:32 smithi092 bash[58855]: audit 2023-09-23T19:16:32.176087+0000 mon.a (mon.0) 210 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T19:16:33.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:32 smithi092 bash[58855]: audit 2023-09-23T19:16:32.177296+0000 mon.a (mon.0) 211 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:33.903 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:33 smithi016 bash[75004]: cephadm 2023-09-23T19:16:32.175671+0000 mgr.y (mgr.25325) 153 : cephadm 1 Reconfiguring osd.0 (monmap changed)... 2023-09-23T19:16:33.904 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:33 smithi016 bash[75004]: cephadm 2023-09-23T19:16:32.181385+0000 mgr.y (mgr.25325) 154 : cephadm 1 Reconfiguring daemon osd.0 on smithi016 2023-09-23T19:16:33.904 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:33 smithi016 bash[77995]: cephadm 2023-09-23T19:16:32.175671+0000 mgr.y (mgr.25325) 153 : cephadm 1 Reconfiguring osd.0 (monmap changed)... 2023-09-23T19:16:33.904 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:33 smithi016 bash[77995]: cephadm 2023-09-23T19:16:32.181385+0000 mgr.y (mgr.25325) 154 : cephadm 1 Reconfiguring daemon osd.0 on smithi016 2023-09-23T19:16:34.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:33 smithi092 bash[58855]: cephadm 2023-09-23T19:16:32.175671+0000 mgr.y (mgr.25325) 153 : cephadm 1 Reconfiguring osd.0 (monmap changed)... 2023-09-23T19:16:34.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:33 smithi092 bash[58855]: cephadm 2023-09-23T19:16:32.181385+0000 mgr.y (mgr.25325) 154 : cephadm 1 Reconfiguring daemon osd.0 on smithi016 2023-09-23T19:16:35.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:34 smithi016 bash[75004]: cluster 2023-09-23T19:16:33.553024+0000 mgr.y (mgr.25325) 155 : cluster 0 pgmap v119: 89 pgs: 89 active+clean; 1.3 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 570 op/s 2023-09-23T19:16:35.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:34 smithi016 bash[77995]: cluster 2023-09-23T19:16:33.553024+0000 mgr.y (mgr.25325) 155 : cluster 0 pgmap v119: 89 pgs: 89 active+clean; 1.3 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 570 op/s 2023-09-23T19:16:35.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:34 smithi092 bash[58855]: cluster 2023-09-23T19:16:33.553024+0000 mgr.y (mgr.25325) 155 : cluster 0 pgmap v119: 89 pgs: 89 active+clean; 1.3 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 570 op/s 2023-09-23T19:16:35.639 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:16:35 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:16:35] "GET /metrics HTTP/1.1" 200 36055 "" "Prometheus/2.33.4" 2023-09-23T19:16:36.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:35 smithi016 bash[75004]: audit 2023-09-23T19:16:34.884548+0000 mon.a (mon.0) 212 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:36.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:35 smithi016 bash[75004]: audit 2023-09-23T19:16:34.895047+0000 mon.a (mon.0) 213 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:36.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:35 smithi016 bash[75004]: cephadm 2023-09-23T19:16:34.909354+0000 mgr.y (mgr.25325) 156 : cephadm 1 Reconfiguring osd.1 (monmap changed)... 2023-09-23T19:16:36.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:35 smithi016 bash[75004]: audit 2023-09-23T19:16:34.909893+0000 mon.a (mon.0) 214 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T19:16:36.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:35 smithi016 bash[75004]: audit 2023-09-23T19:16:34.910987+0000 mon.a (mon.0) 215 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:36.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:35 smithi016 bash[75004]: cephadm 2023-09-23T19:16:34.914036+0000 mgr.y (mgr.25325) 157 : cephadm 1 Reconfiguring daemon osd.1 on smithi016 2023-09-23T19:16:36.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:35 smithi016 bash[75004]: cluster 2023-09-23T19:16:35.553868+0000 mgr.y (mgr.25325) 158 : cluster 0 pgmap v120: 89 pgs: 89 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 853 op/s 2023-09-23T19:16:36.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:35 smithi016 bash[77995]: audit 2023-09-23T19:16:34.884548+0000 mon.a (mon.0) 212 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:36.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:35 smithi016 bash[77995]: audit 2023-09-23T19:16:34.895047+0000 mon.a (mon.0) 213 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:36.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:35 smithi016 bash[77995]: cephadm 2023-09-23T19:16:34.909354+0000 mgr.y (mgr.25325) 156 : cephadm 1 Reconfiguring osd.1 (monmap changed)... 2023-09-23T19:16:36.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:35 smithi016 bash[77995]: audit 2023-09-23T19:16:34.909893+0000 mon.a (mon.0) 214 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T19:16:36.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:35 smithi016 bash[77995]: audit 2023-09-23T19:16:34.910987+0000 mon.a (mon.0) 215 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:36.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:35 smithi016 bash[77995]: cephadm 2023-09-23T19:16:34.914036+0000 mgr.y (mgr.25325) 157 : cephadm 1 Reconfiguring daemon osd.1 on smithi016 2023-09-23T19:16:36.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:35 smithi016 bash[77995]: cluster 2023-09-23T19:16:35.553868+0000 mgr.y (mgr.25325) 158 : cluster 0 pgmap v120: 89 pgs: 89 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 853 op/s 2023-09-23T19:16:36.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:35 smithi092 bash[58855]: audit 2023-09-23T19:16:34.884548+0000 mon.a (mon.0) 212 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:36.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:35 smithi092 bash[58855]: audit 2023-09-23T19:16:34.895047+0000 mon.a (mon.0) 213 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:36.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:35 smithi092 bash[58855]: cephadm 2023-09-23T19:16:34.909354+0000 mgr.y (mgr.25325) 156 : cephadm 1 Reconfiguring osd.1 (monmap changed)... 2023-09-23T19:16:36.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:35 smithi092 bash[58855]: audit 2023-09-23T19:16:34.909893+0000 mon.a (mon.0) 214 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T19:16:36.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:35 smithi092 bash[58855]: audit 2023-09-23T19:16:34.910987+0000 mon.a (mon.0) 215 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:36.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:35 smithi092 bash[58855]: cephadm 2023-09-23T19:16:34.914036+0000 mgr.y (mgr.25325) 157 : cephadm 1 Reconfiguring daemon osd.1 on smithi016 2023-09-23T19:16:36.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:35 smithi092 bash[58855]: cluster 2023-09-23T19:16:35.553868+0000 mgr.y (mgr.25325) 158 : cluster 0 pgmap v120: 89 pgs: 89 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 853 op/s 2023-09-23T19:16:39.021 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:38 smithi016 bash[77995]: audit 2023-09-23T19:16:37.539679+0000 mon.a (mon.0) 216 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:39.021 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:38 smithi016 bash[77995]: audit 2023-09-23T19:16:37.552428+0000 mon.a (mon.0) 217 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:39.021 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:38 smithi016 bash[77995]: cluster 2023-09-23T19:16:37.555592+0000 mgr.y (mgr.25325) 159 : cluster 0 pgmap v121: 89 pgs: 89 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 567 op/s 2023-09-23T19:16:39.021 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:38 smithi016 bash[77995]: cephadm 2023-09-23T19:16:37.588365+0000 mgr.y (mgr.25325) 160 : cephadm 1 Reconfiguring osd.2 (monmap changed)... 2023-09-23T19:16:39.022 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:38 smithi016 bash[77995]: audit 2023-09-23T19:16:37.588792+0000 mon.a (mon.0) 218 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T19:16:39.022 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:38 smithi016 bash[77995]: audit 2023-09-23T19:16:37.589703+0000 mon.a (mon.0) 219 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:39.022 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:38 smithi016 bash[77995]: cephadm 2023-09-23T19:16:37.592881+0000 mgr.y (mgr.25325) 161 : cephadm 1 Reconfiguring daemon osd.2 on smithi016 2023-09-23T19:16:39.022 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:38 smithi016 bash[75004]: audit 2023-09-23T19:16:37.539679+0000 mon.a (mon.0) 216 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:39.022 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:38 smithi016 bash[75004]: audit 2023-09-23T19:16:37.552428+0000 mon.a (mon.0) 217 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:39.022 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:38 smithi016 bash[75004]: cluster 2023-09-23T19:16:37.555592+0000 mgr.y (mgr.25325) 159 : cluster 0 pgmap v121: 89 pgs: 89 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 567 op/s 2023-09-23T19:16:39.022 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:38 smithi016 bash[75004]: cephadm 2023-09-23T19:16:37.588365+0000 mgr.y (mgr.25325) 160 : cephadm 1 Reconfiguring osd.2 (monmap changed)... 2023-09-23T19:16:39.022 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:38 smithi016 bash[75004]: audit 2023-09-23T19:16:37.588792+0000 mon.a (mon.0) 218 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T19:16:39.022 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:38 smithi016 bash[75004]: audit 2023-09-23T19:16:37.589703+0000 mon.a (mon.0) 219 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:39.022 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:38 smithi016 bash[75004]: cephadm 2023-09-23T19:16:37.592881+0000 mgr.y (mgr.25325) 161 : cephadm 1 Reconfiguring daemon osd.2 on smithi016 2023-09-23T19:16:39.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:38 smithi092 bash[58855]: audit 2023-09-23T19:16:37.539679+0000 mon.a (mon.0) 216 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:39.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:38 smithi092 bash[58855]: audit 2023-09-23T19:16:37.552428+0000 mon.a (mon.0) 217 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:39.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:38 smithi092 bash[58855]: cluster 2023-09-23T19:16:37.555592+0000 mgr.y (mgr.25325) 159 : cluster 0 pgmap v121: 89 pgs: 89 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 567 op/s 2023-09-23T19:16:39.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:38 smithi092 bash[58855]: cephadm 2023-09-23T19:16:37.588365+0000 mgr.y (mgr.25325) 160 : cephadm 1 Reconfiguring osd.2 (monmap changed)... 2023-09-23T19:16:39.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:38 smithi092 bash[58855]: audit 2023-09-23T19:16:37.588792+0000 mon.a (mon.0) 218 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T19:16:39.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:38 smithi092 bash[58855]: audit 2023-09-23T19:16:37.589703+0000 mon.a (mon.0) 219 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:39.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:38 smithi092 bash[58855]: cephadm 2023-09-23T19:16:37.592881+0000 mgr.y (mgr.25325) 161 : cephadm 1 Reconfiguring daemon osd.2 on smithi016 2023-09-23T19:16:40.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:39 smithi016 bash[77995]: cluster 2023-09-23T19:16:39.556055+0000 mgr.y (mgr.25325) 162 : cluster 0 pgmap v122: 89 pgs: 89 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 567 op/s 2023-09-23T19:16:40.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:39 smithi016 bash[75004]: cluster 2023-09-23T19:16:39.556055+0000 mgr.y (mgr.25325) 162 : cluster 0 pgmap v122: 89 pgs: 89 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 567 op/s 2023-09-23T19:16:40.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:39 smithi092 bash[58855]: cluster 2023-09-23T19:16:39.556055+0000 mgr.y (mgr.25325) 162 : cluster 0 pgmap v122: 89 pgs: 89 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 567 op/s 2023-09-23T19:16:41.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:41 smithi016 bash[77995]: audit 2023-09-23T19:16:40.214200+0000 mon.a (mon.0) 220 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:41.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:41 smithi016 bash[77995]: audit 2023-09-23T19:16:40.233991+0000 mon.a (mon.0) 221 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:41.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:41 smithi016 bash[77995]: cephadm 2023-09-23T19:16:40.262108+0000 mgr.y (mgr.25325) 163 : cephadm 1 Reconfiguring osd.3 (monmap changed)... 2023-09-23T19:16:41.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:41 smithi016 bash[77995]: audit 2023-09-23T19:16:40.262658+0000 mon.a (mon.0) 222 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T19:16:41.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:41 smithi016 bash[77995]: audit 2023-09-23T19:16:40.263707+0000 mon.a (mon.0) 223 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:41.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:41 smithi016 bash[77995]: cephadm 2023-09-23T19:16:40.267736+0000 mgr.y (mgr.25325) 164 : cephadm 1 Reconfiguring daemon osd.3 on smithi016 2023-09-23T19:16:41.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:41 smithi016 bash[75004]: audit 2023-09-23T19:16:40.214200+0000 mon.a (mon.0) 220 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:41.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:41 smithi016 bash[75004]: audit 2023-09-23T19:16:40.233991+0000 mon.a (mon.0) 221 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:41.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:41 smithi016 bash[75004]: cephadm 2023-09-23T19:16:40.262108+0000 mgr.y (mgr.25325) 163 : cephadm 1 Reconfiguring osd.3 (monmap changed)... 2023-09-23T19:16:41.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:41 smithi016 bash[75004]: audit 2023-09-23T19:16:40.262658+0000 mon.a (mon.0) 222 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T19:16:41.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:41 smithi016 bash[75004]: audit 2023-09-23T19:16:40.263707+0000 mon.a (mon.0) 223 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:41.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:41 smithi016 bash[75004]: cephadm 2023-09-23T19:16:40.267736+0000 mgr.y (mgr.25325) 164 : cephadm 1 Reconfiguring daemon osd.3 on smithi016 2023-09-23T19:16:41.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:41 smithi092 bash[58855]: audit 2023-09-23T19:16:40.214200+0000 mon.a (mon.0) 220 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:41.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:41 smithi092 bash[58855]: audit 2023-09-23T19:16:40.233991+0000 mon.a (mon.0) 221 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:41.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:41 smithi092 bash[58855]: cephadm 2023-09-23T19:16:40.262108+0000 mgr.y (mgr.25325) 163 : cephadm 1 Reconfiguring osd.3 (monmap changed)... 2023-09-23T19:16:41.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:41 smithi092 bash[58855]: audit 2023-09-23T19:16:40.262658+0000 mon.a (mon.0) 222 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T19:16:41.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:41 smithi092 bash[58855]: audit 2023-09-23T19:16:40.263707+0000 mon.a (mon.0) 223 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:41.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:41 smithi092 bash[58855]: cephadm 2023-09-23T19:16:40.267736+0000 mgr.y (mgr.25325) 164 : cephadm 1 Reconfiguring daemon osd.3 on smithi016 2023-09-23T19:16:42.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:42 smithi016 bash[77995]: cluster 2023-09-23T19:16:41.556982+0000 mgr.y (mgr.25325) 165 : cluster 0 pgmap v123: 89 pgs: 89 active+clean; 889 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 849 op/s 2023-09-23T19:16:42.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:42 smithi016 bash[75004]: cluster 2023-09-23T19:16:41.556982+0000 mgr.y (mgr.25325) 165 : cluster 0 pgmap v123: 89 pgs: 89 active+clean; 889 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 849 op/s 2023-09-23T19:16:42.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:42 smithi092 bash[58855]: cluster 2023-09-23T19:16:41.556982+0000 mgr.y (mgr.25325) 165 : cluster 0 pgmap v123: 89 pgs: 89 active+clean; 889 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 849 op/s 2023-09-23T19:16:44.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:43 smithi016 bash[77995]: audit 2023-09-23T19:16:42.948188+0000 mon.a (mon.0) 224 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:44.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:43 smithi016 bash[77995]: audit 2023-09-23T19:16:42.959705+0000 mon.a (mon.0) 225 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:44.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:43 smithi016 bash[77995]: cephadm 2023-09-23T19:16:42.993902+0000 mgr.y (mgr.25325) 166 : cephadm 1 Reconfiguring mds.foo.smithi016.uncssv (monmap changed)... 2023-09-23T19:16:44.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:43 smithi016 bash[77995]: audit 2023-09-23T19:16:42.994549+0000 mon.a (mon.0) 226 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi016.uncssv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T19:16:44.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:43 smithi016 bash[77995]: audit 2023-09-23T19:16:42.995930+0000 mon.a (mon.0) 227 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:44.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:43 smithi016 bash[77995]: cephadm 2023-09-23T19:16:42.997330+0000 mgr.y (mgr.25325) 167 : cephadm 1 Reconfiguring daemon mds.foo.smithi016.uncssv on smithi016 2023-09-23T19:16:44.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:43 smithi016 bash[77995]: cluster 2023-09-23T19:16:43.557470+0000 mgr.y (mgr.25325) 168 : cluster 0 pgmap v124: 89 pgs: 89 active+clean; 889 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 564 op/s 2023-09-23T19:16:44.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:43 smithi016 bash[77995]: audit 2023-09-23T19:16:43.612322+0000 mon.a (mon.0) 228 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:16:44.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:43 smithi016 bash[75004]: audit 2023-09-23T19:16:42.948188+0000 mon.a (mon.0) 224 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:44.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:43 smithi016 bash[75004]: audit 2023-09-23T19:16:42.959705+0000 mon.a (mon.0) 225 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:44.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:43 smithi016 bash[75004]: cephadm 2023-09-23T19:16:42.993902+0000 mgr.y (mgr.25325) 166 : cephadm 1 Reconfiguring mds.foo.smithi016.uncssv (monmap changed)... 2023-09-23T19:16:44.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:43 smithi016 bash[75004]: audit 2023-09-23T19:16:42.994549+0000 mon.a (mon.0) 226 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi016.uncssv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T19:16:44.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:43 smithi016 bash[75004]: audit 2023-09-23T19:16:42.995930+0000 mon.a (mon.0) 227 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:44.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:43 smithi016 bash[75004]: cephadm 2023-09-23T19:16:42.997330+0000 mgr.y (mgr.25325) 167 : cephadm 1 Reconfiguring daemon mds.foo.smithi016.uncssv on smithi016 2023-09-23T19:16:44.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:43 smithi016 bash[75004]: cluster 2023-09-23T19:16:43.557470+0000 mgr.y (mgr.25325) 168 : cluster 0 pgmap v124: 89 pgs: 89 active+clean; 889 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 564 op/s 2023-09-23T19:16:44.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:43 smithi016 bash[75004]: audit 2023-09-23T19:16:43.612322+0000 mon.a (mon.0) 228 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:16:44.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:43 smithi092 bash[58855]: audit 2023-09-23T19:16:42.948188+0000 mon.a (mon.0) 224 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:44.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:43 smithi092 bash[58855]: audit 2023-09-23T19:16:42.959705+0000 mon.a (mon.0) 225 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:44.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:43 smithi092 bash[58855]: cephadm 2023-09-23T19:16:42.993902+0000 mgr.y (mgr.25325) 166 : cephadm 1 Reconfiguring mds.foo.smithi016.uncssv (monmap changed)... 2023-09-23T19:16:44.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:43 smithi092 bash[58855]: audit 2023-09-23T19:16:42.994549+0000 mon.a (mon.0) 226 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi016.uncssv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T19:16:44.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:43 smithi092 bash[58855]: audit 2023-09-23T19:16:42.995930+0000 mon.a (mon.0) 227 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:44.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:43 smithi092 bash[58855]: cephadm 2023-09-23T19:16:42.997330+0000 mgr.y (mgr.25325) 167 : cephadm 1 Reconfiguring daemon mds.foo.smithi016.uncssv on smithi016 2023-09-23T19:16:44.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:43 smithi092 bash[58855]: cluster 2023-09-23T19:16:43.557470+0000 mgr.y (mgr.25325) 168 : cluster 0 pgmap v124: 89 pgs: 89 active+clean; 889 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 564 op/s 2023-09-23T19:16:44.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:43 smithi092 bash[58855]: audit 2023-09-23T19:16:43.612322+0000 mon.a (mon.0) 228 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:16:46.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:16:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:16:45] "GET /metrics HTTP/1.1" 200 36055 "" "Prometheus/2.33.4" 2023-09-23T19:16:46.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:46 smithi016 bash[77995]: audit 2023-09-23T19:16:45.536087+0000 mon.a (mon.0) 229 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:46.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:46 smithi016 bash[77995]: audit 2023-09-23T19:16:45.549288+0000 mon.a (mon.0) 230 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:46.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:46 smithi016 bash[77995]: cluster 2023-09-23T19:16:45.558176+0000 mgr.y (mgr.25325) 169 : cluster 0 pgmap v125: 89 pgs: 89 active+clean; 676 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 848 op/s 2023-09-23T19:16:46.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:46 smithi016 bash[77995]: cephadm 2023-09-23T19:16:45.569836+0000 mgr.y (mgr.25325) 170 : cephadm 1 Reconfiguring mon.b (monmap changed)... 2023-09-23T19:16:46.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:46 smithi016 bash[77995]: audit 2023-09-23T19:16:45.571300+0000 mon.a (mon.0) 231 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T19:16:46.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:46 smithi016 bash[77995]: audit 2023-09-23T19:16:45.576484+0000 mon.a (mon.0) 232 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T19:16:46.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:46 smithi016 bash[77995]: audit 2023-09-23T19:16:45.577794+0000 mon.a (mon.0) 233 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:46.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:46 smithi016 bash[77995]: cephadm 2023-09-23T19:16:45.579161+0000 mgr.y (mgr.25325) 171 : cephadm 1 Reconfiguring daemon mon.b on smithi092 2023-09-23T19:16:46.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:46 smithi016 bash[75004]: audit 2023-09-23T19:16:45.536087+0000 mon.a (mon.0) 229 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:46.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:46 smithi016 bash[75004]: audit 2023-09-23T19:16:45.549288+0000 mon.a (mon.0) 230 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:46.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:46 smithi016 bash[75004]: cluster 2023-09-23T19:16:45.558176+0000 mgr.y (mgr.25325) 169 : cluster 0 pgmap v125: 89 pgs: 89 active+clean; 676 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 848 op/s 2023-09-23T19:16:46.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:46 smithi016 bash[75004]: cephadm 2023-09-23T19:16:45.569836+0000 mgr.y (mgr.25325) 170 : cephadm 1 Reconfiguring mon.b (monmap changed)... 2023-09-23T19:16:46.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:46 smithi016 bash[75004]: audit 2023-09-23T19:16:45.571300+0000 mon.a (mon.0) 231 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T19:16:46.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:46 smithi016 bash[75004]: audit 2023-09-23T19:16:45.576484+0000 mon.a (mon.0) 232 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T19:16:46.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:46 smithi016 bash[75004]: audit 2023-09-23T19:16:45.577794+0000 mon.a (mon.0) 233 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:46.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:46 smithi016 bash[75004]: cephadm 2023-09-23T19:16:45.579161+0000 mgr.y (mgr.25325) 171 : cephadm 1 Reconfiguring daemon mon.b on smithi092 2023-09-23T19:16:46.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:46 smithi092 bash[58855]: audit 2023-09-23T19:16:45.536087+0000 mon.a (mon.0) 229 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:46.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:46 smithi092 bash[58855]: audit 2023-09-23T19:16:45.549288+0000 mon.a (mon.0) 230 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:46.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:46 smithi092 bash[58855]: cluster 2023-09-23T19:16:45.558176+0000 mgr.y (mgr.25325) 169 : cluster 0 pgmap v125: 89 pgs: 89 active+clean; 676 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 848 op/s 2023-09-23T19:16:46.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:46 smithi092 bash[58855]: cephadm 2023-09-23T19:16:45.569836+0000 mgr.y (mgr.25325) 170 : cephadm 1 Reconfiguring mon.b (monmap changed)... 2023-09-23T19:16:46.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:46 smithi092 bash[58855]: audit 2023-09-23T19:16:45.571300+0000 mon.a (mon.0) 231 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T19:16:46.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:46 smithi092 bash[58855]: audit 2023-09-23T19:16:45.576484+0000 mon.a (mon.0) 232 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T19:16:46.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:46 smithi092 bash[58855]: audit 2023-09-23T19:16:45.577794+0000 mon.a (mon.0) 233 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:46.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:46 smithi092 bash[58855]: cephadm 2023-09-23T19:16:45.579161+0000 mgr.y (mgr.25325) 171 : cephadm 1 Reconfiguring daemon mon.b on smithi092 2023-09-23T19:16:48.942 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:48 smithi092 bash[58855]: cluster 2023-09-23T19:16:47.558622+0000 mgr.y (mgr.25325) 172 : cluster 0 pgmap v126: 89 pgs: 89 active+clean; 676 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 565 op/s 2023-09-23T19:16:49.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:48 smithi016 bash[77995]: cluster 2023-09-23T19:16:47.558622+0000 mgr.y (mgr.25325) 172 : cluster 0 pgmap v126: 89 pgs: 89 active+clean; 676 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 565 op/s 2023-09-23T19:16:49.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:48 smithi016 bash[75004]: cluster 2023-09-23T19:16:47.558622+0000 mgr.y (mgr.25325) 172 : cluster 0 pgmap v126: 89 pgs: 89 active+clean; 676 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 565 op/s 2023-09-23T19:16:51.009 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:50 smithi092 bash[58855]: cluster 2023-09-23T19:16:49.559192+0000 mgr.y (mgr.25325) 173 : cluster 0 pgmap v127: 89 pgs: 89 active+clean; 676 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 565 op/s 2023-09-23T19:16:51.009 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:50 smithi092 bash[58855]: audit 2023-09-23T19:16:49.692919+0000 mon.a (mon.0) 234 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:51.009 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:50 smithi092 bash[58855]: audit 2023-09-23T19:16:49.702226+0000 mon.a (mon.0) 235 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:51.010 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:50 smithi092 bash[58855]: audit 2023-09-23T19:16:49.725902+0000 mon.a (mon.0) 236 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T19:16:51.010 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:50 smithi092 bash[58855]: audit 2023-09-23T19:16:49.727136+0000 mon.a (mon.0) 237 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T19:16:51.010 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:50 smithi092 bash[58855]: audit 2023-09-23T19:16:49.728259+0000 mon.a (mon.0) 238 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:51.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:50 smithi016 bash[77995]: cluster 2023-09-23T19:16:49.559192+0000 mgr.y (mgr.25325) 173 : cluster 0 pgmap v127: 89 pgs: 89 active+clean; 676 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 565 op/s 2023-09-23T19:16:51.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:50 smithi016 bash[77995]: audit 2023-09-23T19:16:49.692919+0000 mon.a (mon.0) 234 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:51.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:50 smithi016 bash[77995]: audit 2023-09-23T19:16:49.702226+0000 mon.a (mon.0) 235 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:51.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:50 smithi016 bash[77995]: audit 2023-09-23T19:16:49.725902+0000 mon.a (mon.0) 236 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T19:16:51.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:50 smithi016 bash[77995]: audit 2023-09-23T19:16:49.727136+0000 mon.a (mon.0) 237 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T19:16:51.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:50 smithi016 bash[77995]: audit 2023-09-23T19:16:49.728259+0000 mon.a (mon.0) 238 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:51.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:50 smithi016 bash[75004]: cluster 2023-09-23T19:16:49.559192+0000 mgr.y (mgr.25325) 173 : cluster 0 pgmap v127: 89 pgs: 89 active+clean; 676 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 565 op/s 2023-09-23T19:16:51.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:50 smithi016 bash[75004]: audit 2023-09-23T19:16:49.692919+0000 mon.a (mon.0) 234 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:51.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:50 smithi016 bash[75004]: audit 2023-09-23T19:16:49.702226+0000 mon.a (mon.0) 235 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:51.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:50 smithi016 bash[75004]: audit 2023-09-23T19:16:49.725902+0000 mon.a (mon.0) 236 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T19:16:51.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:50 smithi016 bash[75004]: audit 2023-09-23T19:16:49.727136+0000 mon.a (mon.0) 237 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T19:16:51.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:50 smithi016 bash[75004]: audit 2023-09-23T19:16:49.728259+0000 mon.a (mon.0) 238 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:51.703 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:51 smithi092 bash[58855]: cephadm 2023-09-23T19:16:49.725376+0000 mgr.y (mgr.25325) 174 : cephadm 1 Reconfiguring mgr.x (monmap changed)... 2023-09-23T19:16:51.704 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:51 smithi092 bash[58855]: cephadm 2023-09-23T19:16:49.729565+0000 mgr.y (mgr.25325) 175 : cephadm 1 Reconfiguring daemon mgr.x on smithi092 2023-09-23T19:16:51.704 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:51 smithi092 bash[58855]: cluster 2023-09-23T19:16:51.559961+0000 mgr.y (mgr.25325) 176 : cluster 0 pgmap v128: 89 pgs: 89 active+clean; 458 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 856 op/s 2023-09-23T19:16:52.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:51 smithi016 bash[75004]: cephadm 2023-09-23T19:16:49.725376+0000 mgr.y (mgr.25325) 174 : cephadm 1 Reconfiguring mgr.x (monmap changed)... 2023-09-23T19:16:52.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:51 smithi016 bash[75004]: cephadm 2023-09-23T19:16:49.729565+0000 mgr.y (mgr.25325) 175 : cephadm 1 Reconfiguring daemon mgr.x on smithi092 2023-09-23T19:16:52.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:51 smithi016 bash[75004]: cluster 2023-09-23T19:16:51.559961+0000 mgr.y (mgr.25325) 176 : cluster 0 pgmap v128: 89 pgs: 89 active+clean; 458 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 856 op/s 2023-09-23T19:16:52.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:51 smithi016 bash[77995]: cephadm 2023-09-23T19:16:49.725376+0000 mgr.y (mgr.25325) 174 : cephadm 1 Reconfiguring mgr.x (monmap changed)... 2023-09-23T19:16:52.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:51 smithi016 bash[77995]: cephadm 2023-09-23T19:16:49.729565+0000 mgr.y (mgr.25325) 175 : cephadm 1 Reconfiguring daemon mgr.x on smithi092 2023-09-23T19:16:52.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:51 smithi016 bash[77995]: cluster 2023-09-23T19:16:51.559961+0000 mgr.y (mgr.25325) 176 : cluster 0 pgmap v128: 89 pgs: 89 active+clean; 458 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 856 op/s 2023-09-23T19:16:53.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:53 smithi016 bash[77995]: audit 2023-09-23T19:16:52.004844+0000 mon.a (mon.0) 239 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:53.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:53 smithi016 bash[77995]: audit 2023-09-23T19:16:52.022687+0000 mon.a (mon.0) 240 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:53.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:53 smithi016 bash[77995]: cephadm 2023-09-23T19:16:52.055380+0000 mgr.y (mgr.25325) 177 : cephadm 1 Reconfiguring osd.4 (monmap changed)... 2023-09-23T19:16:53.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:53 smithi016 bash[77995]: audit 2023-09-23T19:16:52.055955+0000 mon.a (mon.0) 241 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T19:16:53.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:53 smithi016 bash[77995]: audit 2023-09-23T19:16:52.057217+0000 mon.a (mon.0) 242 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:53.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:53 smithi016 bash[77995]: cephadm 2023-09-23T19:16:52.059773+0000 mgr.y (mgr.25325) 178 : cephadm 1 Reconfiguring daemon osd.4 on smithi092 2023-09-23T19:16:53.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:53 smithi016 bash[75004]: audit 2023-09-23T19:16:52.004844+0000 mon.a (mon.0) 239 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:53.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:53 smithi016 bash[75004]: audit 2023-09-23T19:16:52.022687+0000 mon.a (mon.0) 240 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:53.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:53 smithi016 bash[75004]: cephadm 2023-09-23T19:16:52.055380+0000 mgr.y (mgr.25325) 177 : cephadm 1 Reconfiguring osd.4 (monmap changed)... 2023-09-23T19:16:53.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:53 smithi016 bash[75004]: audit 2023-09-23T19:16:52.055955+0000 mon.a (mon.0) 241 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T19:16:53.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:53 smithi016 bash[75004]: audit 2023-09-23T19:16:52.057217+0000 mon.a (mon.0) 242 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:53.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:53 smithi016 bash[75004]: cephadm 2023-09-23T19:16:52.059773+0000 mgr.y (mgr.25325) 178 : cephadm 1 Reconfiguring daemon osd.4 on smithi092 2023-09-23T19:16:53.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:53 smithi092 bash[58855]: audit 2023-09-23T19:16:52.004844+0000 mon.a (mon.0) 239 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:53.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:53 smithi092 bash[58855]: audit 2023-09-23T19:16:52.022687+0000 mon.a (mon.0) 240 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:53.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:53 smithi092 bash[58855]: cephadm 2023-09-23T19:16:52.055380+0000 mgr.y (mgr.25325) 177 : cephadm 1 Reconfiguring osd.4 (monmap changed)... 2023-09-23T19:16:53.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:53 smithi092 bash[58855]: audit 2023-09-23T19:16:52.055955+0000 mon.a (mon.0) 241 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T19:16:53.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:53 smithi092 bash[58855]: audit 2023-09-23T19:16:52.057217+0000 mon.a (mon.0) 242 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:53.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:53 smithi092 bash[58855]: cephadm 2023-09-23T19:16:52.059773+0000 mgr.y (mgr.25325) 178 : cephadm 1 Reconfiguring daemon osd.4 on smithi092 2023-09-23T19:16:54.015 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:54 smithi092 bash[58855]: cluster 2023-09-23T19:16:53.560406+0000 mgr.y (mgr.25325) 179 : cluster 0 pgmap v129: 89 pgs: 89 active+clean; 458 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 574 op/s 2023-09-23T19:16:54.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:54 smithi016 bash[77995]: cluster 2023-09-23T19:16:53.560406+0000 mgr.y (mgr.25325) 179 : cluster 0 pgmap v129: 89 pgs: 89 active+clean; 458 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 574 op/s 2023-09-23T19:16:54.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:54 smithi016 bash[75004]: cluster 2023-09-23T19:16:53.560406+0000 mgr.y (mgr.25325) 179 : cluster 0 pgmap v129: 89 pgs: 89 active+clean; 458 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 574 op/s 2023-09-23T19:16:55.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:55 smithi016 bash[75004]: audit 2023-09-23T19:16:54.426522+0000 mon.a (mon.0) 243 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:55.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:55 smithi016 bash[75004]: audit 2023-09-23T19:16:54.436094+0000 mon.a (mon.0) 244 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:55.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:55 smithi016 bash[75004]: cephadm 2023-09-23T19:16:54.485277+0000 mgr.y (mgr.25325) 180 : cephadm 1 Reconfiguring osd.5 (monmap changed)... 2023-09-23T19:16:55.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:55 smithi016 bash[75004]: audit 2023-09-23T19:16:54.485825+0000 mon.a (mon.0) 245 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T19:16:55.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:55 smithi016 bash[75004]: audit 2023-09-23T19:16:54.486975+0000 mon.a (mon.0) 246 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:55.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:55 smithi016 bash[75004]: cephadm 2023-09-23T19:16:54.490290+0000 mgr.y (mgr.25325) 181 : cephadm 1 Reconfiguring daemon osd.5 on smithi092 2023-09-23T19:16:55.785 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:16:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:16:55] "GET /metrics HTTP/1.1" 200 36055 "" "Prometheus/2.33.4" 2023-09-23T19:16:55.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:55 smithi016 bash[77995]: audit 2023-09-23T19:16:54.426522+0000 mon.a (mon.0) 243 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:55.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:55 smithi016 bash[77995]: audit 2023-09-23T19:16:54.436094+0000 mon.a (mon.0) 244 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:55.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:55 smithi016 bash[77995]: cephadm 2023-09-23T19:16:54.485277+0000 mgr.y (mgr.25325) 180 : cephadm 1 Reconfiguring osd.5 (monmap changed)... 2023-09-23T19:16:55.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:55 smithi016 bash[77995]: audit 2023-09-23T19:16:54.485825+0000 mon.a (mon.0) 245 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T19:16:55.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:55 smithi016 bash[77995]: audit 2023-09-23T19:16:54.486975+0000 mon.a (mon.0) 246 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:55.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:55 smithi016 bash[77995]: cephadm 2023-09-23T19:16:54.490290+0000 mgr.y (mgr.25325) 181 : cephadm 1 Reconfiguring daemon osd.5 on smithi092 2023-09-23T19:16:55.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:55 smithi092 bash[58855]: audit 2023-09-23T19:16:54.426522+0000 mon.a (mon.0) 243 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:55.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:55 smithi092 bash[58855]: audit 2023-09-23T19:16:54.436094+0000 mon.a (mon.0) 244 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:55.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:55 smithi092 bash[58855]: cephadm 2023-09-23T19:16:54.485277+0000 mgr.y (mgr.25325) 180 : cephadm 1 Reconfiguring osd.5 (monmap changed)... 2023-09-23T19:16:55.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:55 smithi092 bash[58855]: audit 2023-09-23T19:16:54.485825+0000 mon.a (mon.0) 245 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T19:16:55.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:55 smithi092 bash[58855]: audit 2023-09-23T19:16:54.486975+0000 mon.a (mon.0) 246 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:55.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:55 smithi092 bash[58855]: cephadm 2023-09-23T19:16:54.490290+0000 mgr.y (mgr.25325) 181 : cephadm 1 Reconfiguring daemon osd.5 on smithi092 2023-09-23T19:16:56.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:56 smithi016 bash[75004]: cluster 2023-09-23T19:16:55.561120+0000 mgr.y (mgr.25325) 182 : cluster 0 pgmap v130: 89 pgs: 89 active+clean; 241 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 863 op/s 2023-09-23T19:16:56.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:56 smithi016 bash[77995]: cluster 2023-09-23T19:16:55.561120+0000 mgr.y (mgr.25325) 182 : cluster 0 pgmap v130: 89 pgs: 89 active+clean; 241 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 863 op/s 2023-09-23T19:16:56.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:56 smithi092 bash[58855]: cluster 2023-09-23T19:16:55.561120+0000 mgr.y (mgr.25325) 182 : cluster 0 pgmap v130: 89 pgs: 89 active+clean; 241 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 863 op/s 2023-09-23T19:16:58.226 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:57 smithi092 bash[58855]: audit 2023-09-23T19:16:56.911668+0000 mon.a (mon.0) 247 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:58.226 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:57 smithi092 bash[58855]: audit 2023-09-23T19:16:56.919190+0000 mon.a (mon.0) 248 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:58.226 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:57 smithi092 bash[58855]: cephadm 2023-09-23T19:16:56.963417+0000 mgr.y (mgr.25325) 183 : cephadm 1 Reconfiguring osd.6 (monmap changed)... 2023-09-23T19:16:58.226 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:57 smithi092 bash[58855]: audit 2023-09-23T19:16:56.963880+0000 mon.a (mon.0) 249 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T19:16:58.226 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:57 smithi092 bash[58855]: audit 2023-09-23T19:16:56.964961+0000 mon.a (mon.0) 250 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:58.226 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:57 smithi092 bash[58855]: cephadm 2023-09-23T19:16:56.967355+0000 mgr.y (mgr.25325) 184 : cephadm 1 Reconfiguring daemon osd.6 on smithi092 2023-09-23T19:16:58.226 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:57 smithi092 bash[58855]: cluster 2023-09-23T19:16:57.561561+0000 mgr.y (mgr.25325) 185 : cluster 0 pgmap v131: 89 pgs: 89 active+clean; 241 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 580 op/s 2023-09-23T19:16:58.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:57 smithi016 bash[75004]: audit 2023-09-23T19:16:56.911668+0000 mon.a (mon.0) 247 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:58.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:57 smithi016 bash[75004]: audit 2023-09-23T19:16:56.919190+0000 mon.a (mon.0) 248 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:58.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:57 smithi016 bash[75004]: cephadm 2023-09-23T19:16:56.963417+0000 mgr.y (mgr.25325) 183 : cephadm 1 Reconfiguring osd.6 (monmap changed)... 2023-09-23T19:16:58.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:57 smithi016 bash[75004]: audit 2023-09-23T19:16:56.963880+0000 mon.a (mon.0) 249 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T19:16:58.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:57 smithi016 bash[75004]: audit 2023-09-23T19:16:56.964961+0000 mon.a (mon.0) 250 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:58.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:57 smithi016 bash[75004]: cephadm 2023-09-23T19:16:56.967355+0000 mgr.y (mgr.25325) 184 : cephadm 1 Reconfiguring daemon osd.6 on smithi092 2023-09-23T19:16:58.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:57 smithi016 bash[75004]: cluster 2023-09-23T19:16:57.561561+0000 mgr.y (mgr.25325) 185 : cluster 0 pgmap v131: 89 pgs: 89 active+clean; 241 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 580 op/s 2023-09-23T19:16:58.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:57 smithi016 bash[77995]: audit 2023-09-23T19:16:56.911668+0000 mon.a (mon.0) 247 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:58.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:57 smithi016 bash[77995]: audit 2023-09-23T19:16:56.919190+0000 mon.a (mon.0) 248 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:16:58.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:57 smithi016 bash[77995]: cephadm 2023-09-23T19:16:56.963417+0000 mgr.y (mgr.25325) 183 : cephadm 1 Reconfiguring osd.6 (monmap changed)... 2023-09-23T19:16:58.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:57 smithi016 bash[77995]: audit 2023-09-23T19:16:56.963880+0000 mon.a (mon.0) 249 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T19:16:58.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:57 smithi016 bash[77995]: audit 2023-09-23T19:16:56.964961+0000 mon.a (mon.0) 250 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:16:58.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:57 smithi016 bash[77995]: cephadm 2023-09-23T19:16:56.967355+0000 mgr.y (mgr.25325) 184 : cephadm 1 Reconfiguring daemon osd.6 on smithi092 2023-09-23T19:16:58.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:57 smithi016 bash[77995]: cluster 2023-09-23T19:16:57.561561+0000 mgr.y (mgr.25325) 185 : cluster 0 pgmap v131: 89 pgs: 89 active+clean; 241 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 580 op/s 2023-09-23T19:16:59.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:16:58 smithi016 bash[75004]: audit 2023-09-23T19:16:58.612766+0000 mon.a (mon.0) 251 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:16:59.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:16:58 smithi016 bash[77995]: audit 2023-09-23T19:16:58.612766+0000 mon.a (mon.0) 251 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:16:59.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:16:58 smithi092 bash[58855]: audit 2023-09-23T19:16:58.612766+0000 mon.a (mon.0) 251 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:17:00.411 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Removed 111814 objects 2023-09-23T19:17:00.411 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Clean up completed and total clean up time :157.477 2023-09-23T19:17:00.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:00 smithi016 bash[75004]: audit 2023-09-23T19:16:59.236867+0000 mon.a (mon.0) 252 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:00.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:00 smithi016 bash[75004]: audit 2023-09-23T19:16:59.250498+0000 mon.a (mon.0) 253 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:00.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:00 smithi016 bash[75004]: cephadm 2023-09-23T19:16:59.252361+0000 mgr.y (mgr.25325) 186 : cephadm 1 Reconfiguring osd.7 (monmap changed)... 2023-09-23T19:17:00.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:00 smithi016 bash[75004]: audit 2023-09-23T19:16:59.252776+0000 mon.a (mon.0) 254 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T19:17:00.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:00 smithi016 bash[75004]: audit 2023-09-23T19:16:59.255618+0000 mon.a (mon.0) 255 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:17:00.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:00 smithi016 bash[75004]: cephadm 2023-09-23T19:16:59.258843+0000 mgr.y (mgr.25325) 187 : cephadm 1 Reconfiguring daemon osd.7 on smithi092 2023-09-23T19:17:00.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:00 smithi016 bash[75004]: cluster 2023-09-23T19:16:59.564600+0000 mgr.y (mgr.25325) 188 : cluster 0 pgmap v132: 89 pgs: 89 active+clean; 241 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 580 op/s 2023-09-23T19:17:00.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:00 smithi016 bash[77995]: audit 2023-09-23T19:16:59.236867+0000 mon.a (mon.0) 252 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:00.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:00 smithi016 bash[77995]: audit 2023-09-23T19:16:59.250498+0000 mon.a (mon.0) 253 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:00.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:00 smithi016 bash[77995]: cephadm 2023-09-23T19:16:59.252361+0000 mgr.y (mgr.25325) 186 : cephadm 1 Reconfiguring osd.7 (monmap changed)... 2023-09-23T19:17:00.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:00 smithi016 bash[77995]: audit 2023-09-23T19:16:59.252776+0000 mon.a (mon.0) 254 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T19:17:00.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:00 smithi016 bash[77995]: audit 2023-09-23T19:16:59.255618+0000 mon.a (mon.0) 255 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:17:00.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:00 smithi016 bash[77995]: cephadm 2023-09-23T19:16:59.258843+0000 mgr.y (mgr.25325) 187 : cephadm 1 Reconfiguring daemon osd.7 on smithi092 2023-09-23T19:17:00.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:00 smithi016 bash[77995]: cluster 2023-09-23T19:16:59.564600+0000 mgr.y (mgr.25325) 188 : cluster 0 pgmap v132: 89 pgs: 89 active+clean; 241 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 580 op/s 2023-09-23T19:17:00.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:00 smithi092 bash[58855]: audit 2023-09-23T19:16:59.236867+0000 mon.a (mon.0) 252 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:00.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:00 smithi092 bash[58855]: audit 2023-09-23T19:16:59.250498+0000 mon.a (mon.0) 253 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:00.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:00 smithi092 bash[58855]: cephadm 2023-09-23T19:16:59.252361+0000 mgr.y (mgr.25325) 186 : cephadm 1 Reconfiguring osd.7 (monmap changed)... 2023-09-23T19:17:00.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:00 smithi092 bash[58855]: audit 2023-09-23T19:16:59.252776+0000 mon.a (mon.0) 254 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T19:17:00.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:00 smithi092 bash[58855]: audit 2023-09-23T19:16:59.255618+0000 mon.a (mon.0) 255 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:17:00.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:00 smithi092 bash[58855]: cephadm 2023-09-23T19:16:59.258843+0000 mgr.y (mgr.25325) 187 : cephadm 1 Reconfiguring daemon osd.7 on smithi092 2023-09-23T19:17:00.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:00 smithi092 bash[58855]: cluster 2023-09-23T19:16:59.564600+0000 mgr.y (mgr.25325) 188 : cluster 0 pgmap v132: 89 pgs: 89 active+clean; 241 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 580 op/s 2023-09-23T19:17:02.025 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_1 2023-09-23T19:17:02.025 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_1 unique_pool_1 --yes-i-really-really-mean-it 2023-09-23T19:17:02.882 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:02 smithi092 bash[58855]: cluster 2023-09-23T19:17:01.565786+0000 mgr.y (mgr.25325) 189 : cluster 0 pgmap v133: 89 pgs: 89 active+clean; 25 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 868 op/s 2023-09-23T19:17:02.882 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:02 smithi092 bash[58855]: audit 2023-09-23T19:17:01.726046+0000 mon.a (mon.0) 256 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:02.882 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:02 smithi092 bash[58855]: audit 2023-09-23T19:17:01.734185+0000 mon.a (mon.0) 257 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:02.882 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:02 smithi092 bash[58855]: audit 2023-09-23T19:17:01.739887+0000 mon.a (mon.0) 258 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi092.vbpqks", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T19:17:02.882 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:02 smithi092 bash[58855]: audit 2023-09-23T19:17:01.741496+0000 mon.a (mon.0) 259 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:17:03.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:02 smithi016 bash[77995]: cluster 2023-09-23T19:17:01.565786+0000 mgr.y (mgr.25325) 189 : cluster 0 pgmap v133: 89 pgs: 89 active+clean; 25 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 868 op/s 2023-09-23T19:17:03.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:02 smithi016 bash[77995]: audit 2023-09-23T19:17:01.726046+0000 mon.a (mon.0) 256 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:03.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:02 smithi016 bash[77995]: audit 2023-09-23T19:17:01.734185+0000 mon.a (mon.0) 257 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:03.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:02 smithi016 bash[77995]: audit 2023-09-23T19:17:01.739887+0000 mon.a (mon.0) 258 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi092.vbpqks", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T19:17:03.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:02 smithi016 bash[77995]: audit 2023-09-23T19:17:01.741496+0000 mon.a (mon.0) 259 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:17:03.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:02 smithi016 bash[75004]: cluster 2023-09-23T19:17:01.565786+0000 mgr.y (mgr.25325) 189 : cluster 0 pgmap v133: 89 pgs: 89 active+clean; 25 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 868 op/s 2023-09-23T19:17:03.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:02 smithi016 bash[75004]: audit 2023-09-23T19:17:01.726046+0000 mon.a (mon.0) 256 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:03.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:02 smithi016 bash[75004]: audit 2023-09-23T19:17:01.734185+0000 mon.a (mon.0) 257 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:03.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:02 smithi016 bash[75004]: audit 2023-09-23T19:17:01.739887+0000 mon.a (mon.0) 258 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi092.vbpqks", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T19:17:03.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:02 smithi016 bash[75004]: audit 2023-09-23T19:17:01.741496+0000 mon.a (mon.0) 259 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:17:04.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:03 smithi016 bash[75004]: cephadm 2023-09-23T19:17:01.739310+0000 mgr.y (mgr.25325) 190 : cephadm 1 Reconfiguring mds.foo.smithi092.vbpqks (monmap changed)... 2023-09-23T19:17:04.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:03 smithi016 bash[75004]: cephadm 2023-09-23T19:17:01.743178+0000 mgr.y (mgr.25325) 191 : cephadm 1 Reconfiguring daemon mds.foo.smithi092.vbpqks on smithi092 2023-09-23T19:17:04.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:03 smithi016 bash[77995]: cephadm 2023-09-23T19:17:01.739310+0000 mgr.y (mgr.25325) 190 : cephadm 1 Reconfiguring mds.foo.smithi092.vbpqks (monmap changed)... 2023-09-23T19:17:04.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:03 smithi016 bash[77995]: cephadm 2023-09-23T19:17:01.743178+0000 mgr.y (mgr.25325) 191 : cephadm 1 Reconfiguring daemon mds.foo.smithi092.vbpqks on smithi092 2023-09-23T19:17:04.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:03 smithi092 bash[58855]: cephadm 2023-09-23T19:17:01.739310+0000 mgr.y (mgr.25325) 190 : cephadm 1 Reconfiguring mds.foo.smithi092.vbpqks (monmap changed)... 2023-09-23T19:17:04.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:03 smithi092 bash[58855]: cephadm 2023-09-23T19:17:01.743178+0000 mgr.y (mgr.25325) 191 : cephadm 1 Reconfiguring daemon mds.foo.smithi092.vbpqks on smithi092 2023-09-23T19:17:05.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:04 smithi016 bash[75004]: cluster 2023-09-23T19:17:03.566592+0000 mgr.y (mgr.25325) 192 : cluster 0 pgmap v134: 89 pgs: 89 active+clean; 25 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 576 op/s 2023-09-23T19:17:05.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:04 smithi016 bash[75004]: audit 2023-09-23T19:17:03.764274+0000 mon.a (mon.0) 260 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:05.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:04 smithi016 bash[75004]: audit 2023-09-23T19:17:03.771865+0000 mon.a (mon.0) 261 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:05.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:04 smithi016 bash[75004]: audit 2023-09-23T19:17:03.824797+0000 mon.a (mon.0) 262 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:17:05.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:04 smithi016 bash[77995]: cluster 2023-09-23T19:17:03.566592+0000 mgr.y (mgr.25325) 192 : cluster 0 pgmap v134: 89 pgs: 89 active+clean; 25 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 576 op/s 2023-09-23T19:17:05.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:04 smithi016 bash[77995]: audit 2023-09-23T19:17:03.764274+0000 mon.a (mon.0) 260 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:05.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:04 smithi016 bash[77995]: audit 2023-09-23T19:17:03.771865+0000 mon.a (mon.0) 261 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:05.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:04 smithi016 bash[77995]: audit 2023-09-23T19:17:03.824797+0000 mon.a (mon.0) 262 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:17:05.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:04 smithi092 bash[58855]: cluster 2023-09-23T19:17:03.566592+0000 mgr.y (mgr.25325) 192 : cluster 0 pgmap v134: 89 pgs: 89 active+clean; 25 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 576 op/s 2023-09-23T19:17:05.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:04 smithi092 bash[58855]: audit 2023-09-23T19:17:03.764274+0000 mon.a (mon.0) 260 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:05.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:04 smithi092 bash[58855]: audit 2023-09-23T19:17:03.771865+0000 mon.a (mon.0) 261 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:05.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:04 smithi092 bash[58855]: audit 2023-09-23T19:17:03.824797+0000 mon.a (mon.0) 262 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:17:06.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:17:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:17:05] "GET /metrics HTTP/1.1" 200 36053 "" "Prometheus/2.33.4" 2023-09-23T19:17:06.821 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:17:07.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:06 smithi016 bash[75004]: cluster 2023-09-23T19:17:05.567867+0000 mgr.y (mgr.25325) 193 : cluster 0 pgmap v135: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 609 op/s 2023-09-23T19:17:07.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:06 smithi016 bash[77995]: cluster 2023-09-23T19:17:05.567867+0000 mgr.y (mgr.25325) 193 : cluster 0 pgmap v135: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 609 op/s 2023-09-23T19:17:07.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:06 smithi092 bash[58855]: cluster 2023-09-23T19:17:05.567867+0000 mgr.y (mgr.25325) 193 : cluster 0 pgmap v135: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 609 op/s 2023-09-23T19:17:09.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:08 smithi016 bash[75004]: cluster 2023-09-23T19:17:07.568565+0000 mgr.y (mgr.25325) 194 : cluster 0 pgmap v136: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 320 op/s 2023-09-23T19:17:09.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:08 smithi016 bash[77995]: cluster 2023-09-23T19:17:07.568565+0000 mgr.y (mgr.25325) 194 : cluster 0 pgmap v136: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 320 op/s 2023-09-23T19:17:09.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:08 smithi092 bash[58855]: cluster 2023-09-23T19:17:07.568565+0000 mgr.y (mgr.25325) 194 : cluster 0 pgmap v136: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 320 op/s 2023-09-23T19:17:10.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:09 smithi016 bash[75004]: audit 2023-09-23T19:17:08.972677+0000 mon.a (mon.0) 263 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:10.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:09 smithi016 bash[75004]: audit 2023-09-23T19:17:08.978379+0000 mon.a (mon.0) 264 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:10.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:09 smithi016 bash[75004]: audit 2023-09-23T19:17:09.500042+0000 mon.a (mon.0) 265 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:17:10.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:09 smithi016 bash[75004]: audit 2023-09-23T19:17:09.501615+0000 mon.a (mon.0) 266 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:17:10.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:09 smithi016 bash[75004]: audit 2023-09-23T19:17:09.510776+0000 mon.a (mon.0) 267 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:10.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:09 smithi016 bash[75004]: cluster 2023-09-23T19:17:09.569237+0000 mgr.y (mgr.25325) 195 : cluster 0 pgmap v137: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 320 op/s 2023-09-23T19:17:10.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:09 smithi016 bash[77995]: audit 2023-09-23T19:17:08.972677+0000 mon.a (mon.0) 263 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:10.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:09 smithi016 bash[77995]: audit 2023-09-23T19:17:08.978379+0000 mon.a (mon.0) 264 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:10.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:09 smithi016 bash[77995]: audit 2023-09-23T19:17:09.500042+0000 mon.a (mon.0) 265 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:17:10.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:09 smithi016 bash[77995]: audit 2023-09-23T19:17:09.501615+0000 mon.a (mon.0) 266 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:17:10.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:09 smithi016 bash[77995]: audit 2023-09-23T19:17:09.510776+0000 mon.a (mon.0) 267 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:10.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:09 smithi016 bash[77995]: cluster 2023-09-23T19:17:09.569237+0000 mgr.y (mgr.25325) 195 : cluster 0 pgmap v137: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 320 op/s 2023-09-23T19:17:10.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:09 smithi092 bash[58855]: audit 2023-09-23T19:17:08.972677+0000 mon.a (mon.0) 263 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:10.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:09 smithi092 bash[58855]: audit 2023-09-23T19:17:08.978379+0000 mon.a (mon.0) 264 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:10.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:09 smithi092 bash[58855]: audit 2023-09-23T19:17:09.500042+0000 mon.a (mon.0) 265 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:17:10.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:09 smithi092 bash[58855]: audit 2023-09-23T19:17:09.501615+0000 mon.a (mon.0) 266 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:17:10.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:09 smithi092 bash[58855]: audit 2023-09-23T19:17:09.510776+0000 mon.a (mon.0) 267 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:10.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:09 smithi092 bash[58855]: cluster 2023-09-23T19:17:09.569237+0000 mgr.y (mgr.25325) 195 : cluster 0 pgmap v137: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 320 op/s 2023-09-23T19:17:12.639 INFO:teuthology.orchestra.run.smithi016.stderr:pool 'unique_pool_1' removed 2023-09-23T19:17:13.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:12 smithi016 bash[75004]: cluster 2023-09-23T19:17:11.570381+0000 mgr.y (mgr.25325) 196 : cluster 0 pgmap v138: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 320 op/s 2023-09-23T19:17:13.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:12 smithi016 bash[75004]: audit 2023-09-23T19:17:11.715104+0000 mon.c (mon.2) 4 : audit 1 from='client.? 172.21.15.16:0/3305335960' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T19:17:13.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:12 smithi016 bash[75004]: audit 2023-09-23T19:17:11.715895+0000 mon.a (mon.0) 268 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T19:17:13.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:12 smithi016 bash[77995]: cluster 2023-09-23T19:17:11.570381+0000 mgr.y (mgr.25325) 196 : cluster 0 pgmap v138: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 320 op/s 2023-09-23T19:17:13.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:12 smithi016 bash[77995]: audit 2023-09-23T19:17:11.715104+0000 mon.c (mon.2) 4 : audit 1 from='client.? 172.21.15.16:0/3305335960' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T19:17:13.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:12 smithi016 bash[77995]: audit 2023-09-23T19:17:11.715895+0000 mon.a (mon.0) 268 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T19:17:13.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:12 smithi092 bash[58855]: cluster 2023-09-23T19:17:11.570381+0000 mgr.y (mgr.25325) 196 : cluster 0 pgmap v138: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 320 op/s 2023-09-23T19:17:13.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:12 smithi092 bash[58855]: audit 2023-09-23T19:17:11.715104+0000 mon.c (mon.2) 4 : audit 1 from='client.? 172.21.15.16:0/3305335960' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T19:17:13.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:12 smithi092 bash[58855]: audit 2023-09-23T19:17:11.715895+0000 mon.a (mon.0) 268 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T19:17:13.447 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T19:17:13.448 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T19:17:13.448 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_2 2023-09-23T19:17:13.448 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_2 16 2023-09-23T19:17:13.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:13 smithi016 bash[75004]: cluster 2023-09-23T19:17:12.629479+0000 osd.0 (osd.0) 4 : cluster 3 failed to encode map e77 with expected crc 2023-09-23T19:17:13.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:13 smithi016 bash[75004]: audit 2023-09-23T19:17:12.631670+0000 mon.a (mon.0) 269 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T19:17:13.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:13 smithi016 bash[75004]: cluster 2023-09-23T19:17:12.631798+0000 mon.a (mon.0) 270 : cluster 0 osdmap e77: 8 total, 8 up, 8 in 2023-09-23T19:17:13.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:13 smithi016 bash[75004]: cluster 2023-09-23T19:17:12.632003+0000 osd.7 (osd.7) 5 : cluster 3 failed to encode map e77 with expected crc 2023-09-23T19:17:13.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:13 smithi016 bash[75004]: cluster 2023-09-23T19:17:13.230757+0000 osd.3 (osd.3) 6 : cluster 3 failed to encode map e77 with expected crc 2023-09-23T19:17:13.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:13 smithi016 bash[75004]: audit 2023-09-23T19:17:13.613143+0000 mon.a (mon.0) 271 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:17:13.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:13 smithi016 bash[77995]: cluster 2023-09-23T19:17:12.629479+0000 osd.0 (osd.0) 4 : cluster 3 failed to encode map e77 with expected crc 2023-09-23T19:17:13.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:13 smithi016 bash[77995]: audit 2023-09-23T19:17:12.631670+0000 mon.a (mon.0) 269 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T19:17:13.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:13 smithi016 bash[77995]: cluster 2023-09-23T19:17:12.631798+0000 mon.a (mon.0) 270 : cluster 0 osdmap e77: 8 total, 8 up, 8 in 2023-09-23T19:17:13.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:13 smithi016 bash[77995]: cluster 2023-09-23T19:17:12.632003+0000 osd.7 (osd.7) 5 : cluster 3 failed to encode map e77 with expected crc 2023-09-23T19:17:13.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:13 smithi016 bash[77995]: cluster 2023-09-23T19:17:13.230757+0000 osd.3 (osd.3) 6 : cluster 3 failed to encode map e77 with expected crc 2023-09-23T19:17:13.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:13 smithi016 bash[77995]: audit 2023-09-23T19:17:13.613143+0000 mon.a (mon.0) 271 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:17:14.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:13 smithi092 bash[58855]: cluster 2023-09-23T19:17:12.629479+0000 osd.0 (osd.0) 4 : cluster 3 failed to encode map e77 with expected crc 2023-09-23T19:17:14.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:13 smithi092 bash[58855]: audit 2023-09-23T19:17:12.631670+0000 mon.a (mon.0) 269 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T19:17:14.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:13 smithi092 bash[58855]: cluster 2023-09-23T19:17:12.631798+0000 mon.a (mon.0) 270 : cluster 0 osdmap e77: 8 total, 8 up, 8 in 2023-09-23T19:17:14.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:13 smithi092 bash[58855]: cluster 2023-09-23T19:17:12.632003+0000 osd.7 (osd.7) 5 : cluster 3 failed to encode map e77 with expected crc 2023-09-23T19:17:14.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:13 smithi092 bash[58855]: cluster 2023-09-23T19:17:13.230757+0000 osd.3 (osd.3) 6 : cluster 3 failed to encode map e77 with expected crc 2023-09-23T19:17:14.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:13 smithi092 bash[58855]: audit 2023-09-23T19:17:13.613143+0000 mon.a (mon.0) 271 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:17:15.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:14 smithi016 bash[75004]: cluster 2023-09-23T19:17:13.105009+0000 osd.5 (osd.5) 6 : cluster 3 failed to encode map e77 with expected crc 2023-09-23T19:17:15.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:14 smithi016 bash[75004]: cluster 2023-09-23T19:17:13.232959+0000 osd.6 (osd.6) 5 : cluster 3 failed to encode map e77 with expected crc 2023-09-23T19:17:15.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:14 smithi016 bash[75004]: cluster 2023-09-23T19:17:13.571019+0000 mgr.y (mgr.25325) 197 : cluster 0 pgmap v140: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:17:15.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:14 smithi016 bash[77995]: cluster 2023-09-23T19:17:13.105009+0000 osd.5 (osd.5) 6 : cluster 3 failed to encode map e77 with expected crc 2023-09-23T19:17:15.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:14 smithi016 bash[77995]: cluster 2023-09-23T19:17:13.232959+0000 osd.6 (osd.6) 5 : cluster 3 failed to encode map e77 with expected crc 2023-09-23T19:17:15.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:14 smithi016 bash[77995]: cluster 2023-09-23T19:17:13.571019+0000 mgr.y (mgr.25325) 197 : cluster 0 pgmap v140: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:17:15.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:14 smithi092 bash[58855]: cluster 2023-09-23T19:17:13.105009+0000 osd.5 (osd.5) 6 : cluster 3 failed to encode map e77 with expected crc 2023-09-23T19:17:15.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:14 smithi092 bash[58855]: cluster 2023-09-23T19:17:13.232959+0000 osd.6 (osd.6) 5 : cluster 3 failed to encode map e77 with expected crc 2023-09-23T19:17:15.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:14 smithi092 bash[58855]: cluster 2023-09-23T19:17:13.571019+0000 mgr.y (mgr.25325) 197 : cluster 0 pgmap v140: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:17:16.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:17:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:17:15] "GET /metrics HTTP/1.1" 200 33702 "" "Prometheus/2.33.4" 2023-09-23T19:17:17.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:16 smithi016 bash[75004]: cluster 2023-09-23T19:17:15.572048+0000 mgr.y (mgr.25325) 198 : cluster 0 pgmap v141: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:17:17.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:16 smithi016 bash[77995]: cluster 2023-09-23T19:17:15.572048+0000 mgr.y (mgr.25325) 198 : cluster 0 pgmap v141: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:17:17.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:16 smithi092 bash[58855]: cluster 2023-09-23T19:17:15.572048+0000 mgr.y (mgr.25325) 198 : cluster 0 pgmap v141: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:17:18.268 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:17:19.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:18 smithi016 bash[77995]: cluster 2023-09-23T19:17:17.572659+0000 mgr.y (mgr.25325) 199 : cluster 0 pgmap v142: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:17:19.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:18 smithi016 bash[75004]: cluster 2023-09-23T19:17:17.572659+0000 mgr.y (mgr.25325) 199 : cluster 0 pgmap v142: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:17:19.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:18 smithi092 bash[58855]: cluster 2023-09-23T19:17:17.572659+0000 mgr.y (mgr.25325) 199 : cluster 0 pgmap v142: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:17:20.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:19 smithi016 bash[75004]: cluster 2023-09-23T19:17:19.573076+0000 mgr.y (mgr.25325) 200 : cluster 0 pgmap v143: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:17:20.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:19 smithi016 bash[77995]: cluster 2023-09-23T19:17:19.573076+0000 mgr.y (mgr.25325) 200 : cluster 0 pgmap v143: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:17:20.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:19 smithi092 bash[58855]: cluster 2023-09-23T19:17:19.573076+0000 mgr.y (mgr.25325) 200 : cluster 0 pgmap v143: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:17:21.708 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade resume' 2023-09-23T19:17:22.113 INFO:teuthology.orchestra.run.smithi016.stderr:pool 'unique_pool_2' created 2023-09-23T19:17:22.163 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:22 smithi016 bash[77995]: cluster 2023-09-23T19:17:21.574105+0000 mgr.y (mgr.25325) 201 : cluster 0 pgmap v144: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:17:22.163 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:22 smithi016 bash[77995]: audit 2023-09-23T19:17:21.586669+0000 mon.c (mon.2) 5 : audit 1 from='client.? 172.21.15.16:0/3259797871' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-09-23T19:17:22.163 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:22 smithi016 bash[77995]: audit 2023-09-23T19:17:21.587316+0000 mon.a (mon.0) 272 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-09-23T19:17:22.163 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:22 smithi016 bash[75004]: cluster 2023-09-23T19:17:21.574105+0000 mgr.y (mgr.25325) 201 : cluster 0 pgmap v144: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:17:22.163 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:22 smithi016 bash[75004]: audit 2023-09-23T19:17:21.586669+0000 mon.c (mon.2) 5 : audit 1 from='client.? 172.21.15.16:0/3259797871' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-09-23T19:17:22.164 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:22 smithi016 bash[75004]: audit 2023-09-23T19:17:21.587316+0000 mon.a (mon.0) 272 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-09-23T19:17:22.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:22 smithi092 bash[58855]: cluster 2023-09-23T19:17:21.574105+0000 mgr.y (mgr.25325) 201 : cluster 0 pgmap v144: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:17:22.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:22 smithi092 bash[58855]: audit 2023-09-23T19:17:21.586669+0000 mon.c (mon.2) 5 : audit 1 from='client.? 172.21.15.16:0/3259797871' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-09-23T19:17:22.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:22 smithi092 bash[58855]: audit 2023-09-23T19:17:21.587316+0000 mon.a (mon.0) 272 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-09-23T19:17:23.117 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:23 smithi016 bash[77995]: cluster 2023-09-23T19:17:22.096461+0000 osd.0 (osd.0) 5 : cluster 3 failed to encode map e78 with expected crc 2023-09-23T19:17:23.117 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:23 smithi016 bash[77995]: cluster 2023-09-23T19:17:22.100114+0000 osd.6 (osd.6) 6 : cluster 3 failed to encode map e78 with expected crc 2023-09-23T19:17:23.117 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:23 smithi016 bash[77995]: audit 2023-09-23T19:17:22.100217+0000 mon.a (mon.0) 273 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-09-23T19:17:23.117 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:23 smithi016 bash[77995]: cluster 2023-09-23T19:17:22.100405+0000 mon.a (mon.0) 274 : cluster 0 osdmap e78: 8 total, 8 up, 8 in 2023-09-23T19:17:23.117 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:23 smithi016 bash[77995]: cluster 2023-09-23T19:17:23.101092+0000 mon.a (mon.0) 275 : cluster 0 osdmap e79: 8 total, 8 up, 8 in 2023-09-23T19:17:23.118 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:23 smithi016 bash[75004]: cluster 2023-09-23T19:17:22.096461+0000 osd.0 (osd.0) 5 : cluster 3 failed to encode map e78 with expected crc 2023-09-23T19:17:23.118 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:23 smithi016 bash[75004]: cluster 2023-09-23T19:17:22.100114+0000 osd.6 (osd.6) 6 : cluster 3 failed to encode map e78 with expected crc 2023-09-23T19:17:23.118 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:23 smithi016 bash[75004]: audit 2023-09-23T19:17:22.100217+0000 mon.a (mon.0) 273 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-09-23T19:17:23.118 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:23 smithi016 bash[75004]: cluster 2023-09-23T19:17:22.100405+0000 mon.a (mon.0) 274 : cluster 0 osdmap e78: 8 total, 8 up, 8 in 2023-09-23T19:17:23.118 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:23 smithi016 bash[75004]: cluster 2023-09-23T19:17:23.101092+0000 mon.a (mon.0) 275 : cluster 0 osdmap e79: 8 total, 8 up, 8 in 2023-09-23T19:17:23.254 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_2 rados --yes-i-really-mean-it || true 2023-09-23T19:17:23.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:23 smithi092 bash[58855]: cluster 2023-09-23T19:17:22.096461+0000 osd.0 (osd.0) 5 : cluster 3 failed to encode map e78 with expected crc 2023-09-23T19:17:23.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:23 smithi092 bash[58855]: cluster 2023-09-23T19:17:22.100114+0000 osd.6 (osd.6) 6 : cluster 3 failed to encode map e78 with expected crc 2023-09-23T19:17:23.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:23 smithi092 bash[58855]: audit 2023-09-23T19:17:22.100217+0000 mon.a (mon.0) 273 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-09-23T19:17:23.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:23 smithi092 bash[58855]: cluster 2023-09-23T19:17:22.100405+0000 mon.a (mon.0) 274 : cluster 0 osdmap e78: 8 total, 8 up, 8 in 2023-09-23T19:17:23.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:23 smithi092 bash[58855]: cluster 2023-09-23T19:17:23.101092+0000 mon.a (mon.0) 275 : cluster 0 osdmap e79: 8 total, 8 up, 8 in 2023-09-23T19:17:24.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:24 smithi016 bash[75004]: cluster 2023-09-23T19:17:23.098310+0000 osd.1 (osd.1) 8 : cluster 3 failed to encode map e79 with expected crc 2023-09-23T19:17:24.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:24 smithi016 bash[75004]: cluster 2023-09-23T19:17:23.099619+0000 osd.0 (osd.0) 6 : cluster 3 failed to encode map e79 with expected crc 2023-09-23T19:17:24.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:24 smithi016 bash[75004]: cluster 2023-09-23T19:17:23.101350+0000 osd.1 (osd.1) 9 : cluster 3 failed to encode map e79 with expected crc 2023-09-23T19:17:24.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:24 smithi016 bash[75004]: cluster 2023-09-23T19:17:23.101443+0000 osd.2 (osd.2) 6 : cluster 3 failed to encode map e79 with expected crc 2023-09-23T19:17:24.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:24 smithi016 bash[75004]: cluster 2023-09-23T19:17:23.101691+0000 osd.3 (osd.3) 7 : cluster 3 failed to encode map e79 with expected crc 2023-09-23T19:17:24.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:24 smithi016 bash[75004]: cluster 2023-09-23T19:17:23.105315+0000 osd.0 (osd.0) 7 : cluster 3 failed to encode map e79 with expected crc 2023-09-23T19:17:24.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:24 smithi016 bash[75004]: cluster 2023-09-23T19:17:23.105632+0000 osd.4 (osd.4) 7 : cluster 3 failed to encode map e79 with expected crc 2023-09-23T19:17:24.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:24 smithi016 bash[75004]: cluster 2023-09-23T19:17:23.105666+0000 osd.6 (osd.6) 7 : cluster 3 failed to encode map e79 with expected crc 2023-09-23T19:17:24.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:24 smithi016 bash[75004]: cluster 2023-09-23T19:17:23.105727+0000 osd.5 (osd.5) 7 : cluster 3 failed to encode map e79 with expected crc 2023-09-23T19:17:24.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:24 smithi016 bash[75004]: cluster 2023-09-23T19:17:23.105944+0000 osd.7 (osd.7) 6 : cluster 3 failed to encode map e79 with expected crc 2023-09-23T19:17:24.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:24 smithi016 bash[75004]: cluster 2023-09-23T19:17:23.574749+0000 mgr.y (mgr.25325) 202 : cluster 0 pgmap v147: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:17:24.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:24 smithi016 bash[77995]: cluster 2023-09-23T19:17:23.098310+0000 osd.1 (osd.1) 8 : cluster 3 failed to encode map e79 with expected crc 2023-09-23T19:17:24.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:24 smithi016 bash[77995]: cluster 2023-09-23T19:17:23.099619+0000 osd.0 (osd.0) 6 : cluster 3 failed to encode map e79 with expected crc 2023-09-23T19:17:24.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:24 smithi016 bash[77995]: cluster 2023-09-23T19:17:23.101350+0000 osd.1 (osd.1) 9 : cluster 3 failed to encode map e79 with expected crc 2023-09-23T19:17:24.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:24 smithi016 bash[77995]: cluster 2023-09-23T19:17:23.101443+0000 osd.2 (osd.2) 6 : cluster 3 failed to encode map e79 with expected crc 2023-09-23T19:17:24.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:24 smithi016 bash[77995]: cluster 2023-09-23T19:17:23.101691+0000 osd.3 (osd.3) 7 : cluster 3 failed to encode map e79 with expected crc 2023-09-23T19:17:24.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:24 smithi016 bash[77995]: cluster 2023-09-23T19:17:23.105315+0000 osd.0 (osd.0) 7 : cluster 3 failed to encode map e79 with expected crc 2023-09-23T19:17:24.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:24 smithi016 bash[77995]: cluster 2023-09-23T19:17:23.105632+0000 osd.4 (osd.4) 7 : cluster 3 failed to encode map e79 with expected crc 2023-09-23T19:17:24.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:24 smithi016 bash[77995]: cluster 2023-09-23T19:17:23.105666+0000 osd.6 (osd.6) 7 : cluster 3 failed to encode map e79 with expected crc 2023-09-23T19:17:24.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:24 smithi016 bash[77995]: cluster 2023-09-23T19:17:23.105727+0000 osd.5 (osd.5) 7 : cluster 3 failed to encode map e79 with expected crc 2023-09-23T19:17:24.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:24 smithi016 bash[77995]: cluster 2023-09-23T19:17:23.105944+0000 osd.7 (osd.7) 6 : cluster 3 failed to encode map e79 with expected crc 2023-09-23T19:17:24.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:24 smithi016 bash[77995]: cluster 2023-09-23T19:17:23.574749+0000 mgr.y (mgr.25325) 202 : cluster 0 pgmap v147: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:17:24.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:24 smithi092 bash[58855]: cluster 2023-09-23T19:17:23.098310+0000 osd.1 (osd.1) 8 : cluster 3 failed to encode map e79 with expected crc 2023-09-23T19:17:24.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:24 smithi092 bash[58855]: cluster 2023-09-23T19:17:23.099619+0000 osd.0 (osd.0) 6 : cluster 3 failed to encode map e79 with expected crc 2023-09-23T19:17:24.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:24 smithi092 bash[58855]: cluster 2023-09-23T19:17:23.101350+0000 osd.1 (osd.1) 9 : cluster 3 failed to encode map e79 with expected crc 2023-09-23T19:17:24.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:24 smithi092 bash[58855]: cluster 2023-09-23T19:17:23.101443+0000 osd.2 (osd.2) 6 : cluster 3 failed to encode map e79 with expected crc 2023-09-23T19:17:24.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:24 smithi092 bash[58855]: cluster 2023-09-23T19:17:23.101691+0000 osd.3 (osd.3) 7 : cluster 3 failed to encode map e79 with expected crc 2023-09-23T19:17:24.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:24 smithi092 bash[58855]: cluster 2023-09-23T19:17:23.105315+0000 osd.0 (osd.0) 7 : cluster 3 failed to encode map e79 with expected crc 2023-09-23T19:17:24.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:24 smithi092 bash[58855]: cluster 2023-09-23T19:17:23.105632+0000 osd.4 (osd.4) 7 : cluster 3 failed to encode map e79 with expected crc 2023-09-23T19:17:24.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:24 smithi092 bash[58855]: cluster 2023-09-23T19:17:23.105666+0000 osd.6 (osd.6) 7 : cluster 3 failed to encode map e79 with expected crc 2023-09-23T19:17:24.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:24 smithi092 bash[58855]: cluster 2023-09-23T19:17:23.105727+0000 osd.5 (osd.5) 7 : cluster 3 failed to encode map e79 with expected crc 2023-09-23T19:17:24.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:24 smithi092 bash[58855]: cluster 2023-09-23T19:17:23.105944+0000 osd.7 (osd.7) 6 : cluster 3 failed to encode map e79 with expected crc 2023-09-23T19:17:24.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:24 smithi092 bash[58855]: cluster 2023-09-23T19:17:23.574749+0000 mgr.y (mgr.25325) 202 : cluster 0 pgmap v147: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:17:25.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:25 smithi016 bash[75004]: cluster 2023-09-23T19:17:24.123385+0000 osd.3 (osd.3) 8 : cluster 3 failed to encode map e80 with expected crc 2023-09-23T19:17:25.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:25 smithi016 bash[75004]: cluster 2023-09-23T19:17:24.123899+0000 osd.0 (osd.0) 8 : cluster 3 failed to encode map e80 with expected crc 2023-09-23T19:17:25.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:25 smithi016 bash[75004]: cluster 2023-09-23T19:17:24.125920+0000 mon.a (mon.0) 276 : cluster 0 osdmap e80: 8 total, 8 up, 8 in 2023-09-23T19:17:25.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:25 smithi016 bash[77995]: cluster 2023-09-23T19:17:24.123385+0000 osd.3 (osd.3) 8 : cluster 3 failed to encode map e80 with expected crc 2023-09-23T19:17:25.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:25 smithi016 bash[77995]: cluster 2023-09-23T19:17:24.123899+0000 osd.0 (osd.0) 8 : cluster 3 failed to encode map e80 with expected crc 2023-09-23T19:17:25.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:25 smithi016 bash[77995]: cluster 2023-09-23T19:17:24.125920+0000 mon.a (mon.0) 276 : cluster 0 osdmap e80: 8 total, 8 up, 8 in 2023-09-23T19:17:25.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:25 smithi092 bash[58855]: cluster 2023-09-23T19:17:24.123385+0000 osd.3 (osd.3) 8 : cluster 3 failed to encode map e80 with expected crc 2023-09-23T19:17:25.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:25 smithi092 bash[58855]: cluster 2023-09-23T19:17:24.123899+0000 osd.0 (osd.0) 8 : cluster 3 failed to encode map e80 with expected crc 2023-09-23T19:17:25.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:25 smithi092 bash[58855]: cluster 2023-09-23T19:17:24.125920+0000 mon.a (mon.0) 276 : cluster 0 osdmap e80: 8 total, 8 up, 8 in 2023-09-23T19:17:25.742 INFO:teuthology.orchestra.run.smithi016.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:17:25.981 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:17:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:17:25] "GET /metrics HTTP/1.1" 200 33702 "" "Prometheus/2.33.4" 2023-09-23T19:17:26.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:26 smithi016 bash[77995]: cluster 2023-09-23T19:17:25.575998+0000 mgr.y (mgr.25325) 203 : cluster 0 pgmap v149: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:17:26.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:26 smithi016 bash[77995]: audit 2023-09-23T19:17:25.740468+0000 mon.a (mon.0) 277 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:26.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:26 smithi016 bash[77995]: audit 2023-09-23T19:17:25.744922+0000 mon.a (mon.0) 278 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:17:26.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:26 smithi016 bash[77995]: audit 2023-09-23T19:17:26.121354+0000 mon.a (mon.0) 279 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:26.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:26 smithi016 bash[75004]: cluster 2023-09-23T19:17:25.575998+0000 mgr.y (mgr.25325) 203 : cluster 0 pgmap v149: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:17:26.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:26 smithi016 bash[75004]: audit 2023-09-23T19:17:25.740468+0000 mon.a (mon.0) 277 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:26.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:26 smithi016 bash[75004]: audit 2023-09-23T19:17:25.744922+0000 mon.a (mon.0) 278 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:17:26.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:26 smithi016 bash[75004]: audit 2023-09-23T19:17:26.121354+0000 mon.a (mon.0) 279 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:26.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:26 smithi092 bash[58855]: cluster 2023-09-23T19:17:25.575998+0000 mgr.y (mgr.25325) 203 : cluster 0 pgmap v149: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:17:26.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:26 smithi092 bash[58855]: audit 2023-09-23T19:17:25.740468+0000 mon.a (mon.0) 277 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:26.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:26 smithi092 bash[58855]: audit 2023-09-23T19:17:25.744922+0000 mon.a (mon.0) 278 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:17:26.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:26 smithi092 bash[58855]: audit 2023-09-23T19:17:26.121354+0000 mon.a (mon.0) 279 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:26.619 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'echo wait for half of osds to upgrade' 2023-09-23T19:17:27.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:27 smithi016 bash[77995]: audit 2023-09-23T19:17:25.735333+0000 mgr.y (mgr.25325) 204 : audit 0 from='client.44147 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:17:27.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:27 smithi016 bash[77995]: cephadm 2023-09-23T19:17:25.735748+0000 mgr.y (mgr.25325) 205 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:17:27.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:27 smithi016 bash[77995]: audit 2023-09-23T19:17:26.125587+0000 mon.a (mon.0) 280 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:27.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:27 smithi016 bash[77995]: audit 2023-09-23T19:17:26.130257+0000 mon.a (mon.0) 281 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:27.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:27 smithi016 bash[77995]: audit 2023-09-23T19:17:26.135768+0000 mon.a (mon.0) 282 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:27.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:27 smithi016 bash[75004]: audit 2023-09-23T19:17:25.735333+0000 mgr.y (mgr.25325) 204 : audit 0 from='client.44147 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:17:27.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:27 smithi016 bash[75004]: cephadm 2023-09-23T19:17:25.735748+0000 mgr.y (mgr.25325) 205 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:17:27.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:27 smithi016 bash[75004]: audit 2023-09-23T19:17:26.125587+0000 mon.a (mon.0) 280 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:27.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:27 smithi016 bash[75004]: audit 2023-09-23T19:17:26.130257+0000 mon.a (mon.0) 281 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:27.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:27 smithi016 bash[75004]: audit 2023-09-23T19:17:26.135768+0000 mon.a (mon.0) 282 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:27.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:27 smithi092 bash[58855]: audit 2023-09-23T19:17:25.735333+0000 mgr.y (mgr.25325) 204 : audit 0 from='client.44147 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:17:27.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:27 smithi092 bash[58855]: cephadm 2023-09-23T19:17:25.735748+0000 mgr.y (mgr.25325) 205 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:17:27.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:27 smithi092 bash[58855]: audit 2023-09-23T19:17:26.125587+0000 mon.a (mon.0) 280 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:27.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:27 smithi092 bash[58855]: audit 2023-09-23T19:17:26.130257+0000 mon.a (mon.0) 281 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:27.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:27 smithi092 bash[58855]: audit 2023-09-23T19:17:26.135768+0000 mon.a (mon.0) 282 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:28.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:28 smithi016 bash[75004]: cluster 2023-09-23T19:17:27.576694+0000 mgr.y (mgr.25325) 206 : cluster 0 pgmap v150: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:17:28.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:28 smithi016 bash[77995]: cluster 2023-09-23T19:17:27.576694+0000 mgr.y (mgr.25325) 206 : cluster 0 pgmap v150: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:17:28.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:28 smithi092 bash[58855]: cluster 2023-09-23T19:17:27.576694+0000 mgr.y (mgr.25325) 206 : cluster 0 pgmap v150: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:17:29.174 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:17:29.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:29 smithi016 bash[77995]: audit 2023-09-23T19:17:28.613731+0000 mon.a (mon.0) 283 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:17:29.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:29 smithi016 bash[75004]: audit 2023-09-23T19:17:28.613731+0000 mon.a (mon.0) 283 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:17:29.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:29 smithi092 bash[58855]: audit 2023-09-23T19:17:28.613731+0000 mon.a (mon.0) 283 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:17:30.312 INFO:teuthology.orchestra.run.smithi016.stdout:wait for half of osds to upgrade 2023-09-23T19:17:30.420 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:30 smithi016 bash[75004]: cluster 2023-09-23T19:17:29.577326+0000 mgr.y (mgr.25325) 207 : cluster 0 pgmap v151: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:17:30.420 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:30 smithi016 bash[77995]: cluster 2023-09-23T19:17:29.577326+0000 mgr.y (mgr.25325) 207 : cluster 0 pgmap v151: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:17:30.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:30 smithi092 bash[58855]: cluster 2023-09-23T19:17:29.577326+0000 mgr.y (mgr.25325) 207 : cluster 0 pgmap v151: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:17:31.598 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'while ! ceph osd versions | grep $sha1 | egrep '"'"': [45678]'"'"'; do sleep 2 ; done' 2023-09-23T19:17:33.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:32 smithi016 bash[75004]: cluster 2023-09-23T19:17:31.578271+0000 mgr.y (mgr.25325) 208 : cluster 0 pgmap v152: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:17:33.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:32 smithi016 bash[75004]: audit 2023-09-23T19:17:32.374388+0000 mon.a (mon.0) 284 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:33.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:32 smithi016 bash[75004]: audit 2023-09-23T19:17:32.381168+0000 mon.a (mon.0) 285 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:33.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:32 smithi016 bash[77995]: cluster 2023-09-23T19:17:31.578271+0000 mgr.y (mgr.25325) 208 : cluster 0 pgmap v152: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:17:33.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:32 smithi016 bash[77995]: audit 2023-09-23T19:17:32.374388+0000 mon.a (mon.0) 284 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:33.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:32 smithi016 bash[77995]: audit 2023-09-23T19:17:32.381168+0000 mon.a (mon.0) 285 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:33.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:32 smithi092 bash[58855]: cluster 2023-09-23T19:17:31.578271+0000 mgr.y (mgr.25325) 208 : cluster 0 pgmap v152: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:17:33.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:32 smithi092 bash[58855]: audit 2023-09-23T19:17:32.374388+0000 mon.a (mon.0) 284 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:33.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:32 smithi092 bash[58855]: audit 2023-09-23T19:17:32.381168+0000 mon.a (mon.0) 285 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:33.947 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:33 smithi016 bash[75004]: audit 2023-09-23T19:17:32.849503+0000 mon.a (mon.0) 286 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:17:33.947 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:33 smithi016 bash[75004]: audit 2023-09-23T19:17:32.851438+0000 mon.a (mon.0) 287 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:17:33.947 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:33 smithi016 bash[75004]: audit 2023-09-23T19:17:32.861359+0000 mon.a (mon.0) 288 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:33.947 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:33 smithi016 bash[75004]: audit 2023-09-23T19:17:32.944880+0000 mon.a (mon.0) 289 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:17:33.947 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:33 smithi016 bash[75004]: audit 2023-09-23T19:17:32.949071+0000 mon.a (mon.0) 290 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:17:33.947 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:33 smithi016 bash[75004]: audit 2023-09-23T19:17:32.951387+0000 mon.a (mon.0) 291 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:17:33.948 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:33 smithi016 bash[75004]: audit 2023-09-23T19:17:32.957784+0000 mon.a (mon.0) 292 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:33.948 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:33 smithi016 bash[75004]: audit 2023-09-23T19:17:32.959296+0000 mon.a (mon.0) 293 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-09-23T19:17:33.948 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:33 smithi016 bash[75004]: audit 2023-09-23T19:17:32.964497+0000 mon.a (mon.0) 294 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-09-23T19:17:33.948 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:33 smithi016 bash[75004]: audit 2023-09-23T19:17:32.965838+0000 mon.a (mon.0) 295 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-09-23T19:17:33.948 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:33 smithi016 bash[75004]: audit 2023-09-23T19:17:32.969924+0000 mon.a (mon.0) 296 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-09-23T19:17:33.948 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:33 smithi016 bash[75004]: audit 2023-09-23T19:17:32.971201+0000 mon.a (mon.0) 297 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-09-23T19:17:33.948 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:33 smithi016 bash[75004]: audit 2023-09-23T19:17:32.975241+0000 mon.a (mon.0) 298 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-09-23T19:17:33.948 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:33 smithi016 bash[75004]: audit 2023-09-23T19:17:32.977158+0000 mon.a (mon.0) 299 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:17:33.948 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:33 smithi016 bash[75004]: audit 2023-09-23T19:17:32.983276+0000 mon.a (mon.0) 300 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:33.948 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:33 smithi016 bash[75004]: audit 2023-09-23T19:17:32.985504+0000 mon.a (mon.0) 301 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T19:17:33.948 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:33 smithi016 bash[77995]: audit 2023-09-23T19:17:32.849503+0000 mon.a (mon.0) 286 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:17:33.949 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:33 smithi016 bash[77995]: audit 2023-09-23T19:17:32.851438+0000 mon.a (mon.0) 287 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:17:33.949 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:33 smithi016 bash[77995]: audit 2023-09-23T19:17:32.861359+0000 mon.a (mon.0) 288 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:33.949 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:33 smithi016 bash[77995]: audit 2023-09-23T19:17:32.944880+0000 mon.a (mon.0) 289 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:17:33.949 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:33 smithi016 bash[77995]: audit 2023-09-23T19:17:32.949071+0000 mon.a (mon.0) 290 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:17:33.949 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:33 smithi016 bash[77995]: audit 2023-09-23T19:17:32.951387+0000 mon.a (mon.0) 291 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:17:33.949 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:33 smithi016 bash[77995]: audit 2023-09-23T19:17:32.957784+0000 mon.a (mon.0) 292 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:33.949 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:33 smithi016 bash[77995]: audit 2023-09-23T19:17:32.959296+0000 mon.a (mon.0) 293 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-09-23T19:17:33.949 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:33 smithi016 bash[77995]: audit 2023-09-23T19:17:32.964497+0000 mon.a (mon.0) 294 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-09-23T19:17:33.949 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:33 smithi016 bash[77995]: audit 2023-09-23T19:17:32.965838+0000 mon.a (mon.0) 295 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-09-23T19:17:33.949 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:33 smithi016 bash[77995]: audit 2023-09-23T19:17:32.969924+0000 mon.a (mon.0) 296 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-09-23T19:17:33.949 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:33 smithi016 bash[77995]: audit 2023-09-23T19:17:32.971201+0000 mon.a (mon.0) 297 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-09-23T19:17:33.949 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:33 smithi016 bash[77995]: audit 2023-09-23T19:17:32.975241+0000 mon.a (mon.0) 298 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-09-23T19:17:33.950 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:33 smithi016 bash[77995]: audit 2023-09-23T19:17:32.977158+0000 mon.a (mon.0) 299 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:17:33.950 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:33 smithi016 bash[77995]: audit 2023-09-23T19:17:32.983276+0000 mon.a (mon.0) 300 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:33.950 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:33 smithi016 bash[77995]: audit 2023-09-23T19:17:32.985504+0000 mon.a (mon.0) 301 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T19:17:34.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:33 smithi092 bash[58855]: audit 2023-09-23T19:17:32.849503+0000 mon.a (mon.0) 286 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:17:34.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:33 smithi092 bash[58855]: audit 2023-09-23T19:17:32.851438+0000 mon.a (mon.0) 287 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:17:34.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:33 smithi092 bash[58855]: audit 2023-09-23T19:17:32.861359+0000 mon.a (mon.0) 288 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:34.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:33 smithi092 bash[58855]: audit 2023-09-23T19:17:32.944880+0000 mon.a (mon.0) 289 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:17:34.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:33 smithi092 bash[58855]: audit 2023-09-23T19:17:32.949071+0000 mon.a (mon.0) 290 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:17:34.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:33 smithi092 bash[58855]: audit 2023-09-23T19:17:32.951387+0000 mon.a (mon.0) 291 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:17:34.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:33 smithi092 bash[58855]: audit 2023-09-23T19:17:32.957784+0000 mon.a (mon.0) 292 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:34.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:33 smithi092 bash[58855]: audit 2023-09-23T19:17:32.959296+0000 mon.a (mon.0) 293 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-09-23T19:17:34.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:33 smithi092 bash[58855]: audit 2023-09-23T19:17:32.964497+0000 mon.a (mon.0) 294 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-09-23T19:17:34.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:33 smithi092 bash[58855]: audit 2023-09-23T19:17:32.965838+0000 mon.a (mon.0) 295 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-09-23T19:17:34.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:33 smithi092 bash[58855]: audit 2023-09-23T19:17:32.969924+0000 mon.a (mon.0) 296 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-09-23T19:17:34.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:33 smithi092 bash[58855]: audit 2023-09-23T19:17:32.971201+0000 mon.a (mon.0) 297 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-09-23T19:17:34.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:33 smithi092 bash[58855]: audit 2023-09-23T19:17:32.975241+0000 mon.a (mon.0) 298 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-09-23T19:17:34.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:33 smithi092 bash[58855]: audit 2023-09-23T19:17:32.977158+0000 mon.a (mon.0) 299 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:17:34.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:33 smithi092 bash[58855]: audit 2023-09-23T19:17:32.983276+0000 mon.a (mon.0) 300 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:34.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:33 smithi092 bash[58855]: audit 2023-09-23T19:17:32.985504+0000 mon.a (mon.0) 301 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T19:17:34.675 INFO:teuthology.orchestra.run.smithi016.stderr:enabled application 'rados' on pool 'unique_pool_2' 2023-09-23T19:17:34.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:34 smithi016 bash[75004]: cephadm 2023-09-23T19:17:32.952350+0000 mgr.y (mgr.25325) 209 : cephadm 1 Upgrade: Setting container_image for all mon 2023-09-23T19:17:34.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:34 smithi016 bash[75004]: cephadm 2023-09-23T19:17:32.978354+0000 mgr.y (mgr.25325) 210 : cephadm 1 Upgrade: Setting container_image for all crash 2023-09-23T19:17:34.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:34 smithi016 bash[75004]: audit 2023-09-23T19:17:32.985781+0000 mgr.y (mgr.25325) 211 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T19:17:34.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:34 smithi016 bash[75004]: cephadm 2023-09-23T19:17:32.987272+0000 mgr.y (mgr.25325) 212 : cephadm 1 Upgrade: osd.0 is safe to restart 2023-09-23T19:17:34.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:34 smithi016 bash[75004]: cluster 2023-09-23T19:17:33.578833+0000 mgr.y (mgr.25325) 213 : cluster 0 pgmap v153: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:17:34.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:34 smithi016 bash[75004]: audit 2023-09-23T19:17:33.660273+0000 mon.a (mon.0) 302 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:34.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:34 smithi016 bash[75004]: audit 2023-09-23T19:17:33.719126+0000 mon.a (mon.0) 303 : audit 1 from='client.? 172.21.15.16:0/1043465352' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:17:34.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:34 smithi016 bash[77995]: cephadm 2023-09-23T19:17:32.952350+0000 mgr.y (mgr.25325) 209 : cephadm 1 Upgrade: Setting container_image for all mon 2023-09-23T19:17:34.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:34 smithi016 bash[77995]: cephadm 2023-09-23T19:17:32.978354+0000 mgr.y (mgr.25325) 210 : cephadm 1 Upgrade: Setting container_image for all crash 2023-09-23T19:17:34.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:34 smithi016 bash[77995]: audit 2023-09-23T19:17:32.985781+0000 mgr.y (mgr.25325) 211 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T19:17:34.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:34 smithi016 bash[77995]: cephadm 2023-09-23T19:17:32.987272+0000 mgr.y (mgr.25325) 212 : cephadm 1 Upgrade: osd.0 is safe to restart 2023-09-23T19:17:34.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:34 smithi016 bash[77995]: cluster 2023-09-23T19:17:33.578833+0000 mgr.y (mgr.25325) 213 : cluster 0 pgmap v153: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:17:34.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:34 smithi016 bash[77995]: audit 2023-09-23T19:17:33.660273+0000 mon.a (mon.0) 302 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:34.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:34 smithi016 bash[77995]: audit 2023-09-23T19:17:33.719126+0000 mon.a (mon.0) 303 : audit 1 from='client.? 172.21.15.16:0/1043465352' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:17:35.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:34 smithi092 bash[58855]: cephadm 2023-09-23T19:17:32.952350+0000 mgr.y (mgr.25325) 209 : cephadm 1 Upgrade: Setting container_image for all mon 2023-09-23T19:17:35.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:34 smithi092 bash[58855]: cephadm 2023-09-23T19:17:32.978354+0000 mgr.y (mgr.25325) 210 : cephadm 1 Upgrade: Setting container_image for all crash 2023-09-23T19:17:35.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:34 smithi092 bash[58855]: audit 2023-09-23T19:17:32.985781+0000 mgr.y (mgr.25325) 211 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T19:17:35.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:34 smithi092 bash[58855]: cephadm 2023-09-23T19:17:32.987272+0000 mgr.y (mgr.25325) 212 : cephadm 1 Upgrade: osd.0 is safe to restart 2023-09-23T19:17:35.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:34 smithi092 bash[58855]: cluster 2023-09-23T19:17:33.578833+0000 mgr.y (mgr.25325) 213 : cluster 0 pgmap v153: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:17:35.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:34 smithi092 bash[58855]: audit 2023-09-23T19:17:33.660273+0000 mon.a (mon.0) 302 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:35.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:34 smithi092 bash[58855]: audit 2023-09-23T19:17:33.719126+0000 mon.a (mon.0) 303 : audit 1 from='client.? 172.21.15.16:0/1043465352' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:17:36.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:35 smithi016 bash[75004]: cluster 2023-09-23T19:17:34.666089+0000 osd.1 (osd.1) 10 : cluster 3 failed to encode map e81 with expected crc 2023-09-23T19:17:36.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:35 smithi016 bash[75004]: cluster 2023-09-23T19:17:34.666480+0000 osd.0 (osd.0) 9 : cluster 3 failed to encode map e81 with expected crc 2023-09-23T19:17:36.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:35 smithi016 bash[75004]: audit 2023-09-23T19:17:34.668362+0000 mon.a (mon.0) 304 : audit 1 from='client.? 172.21.15.16:0/1043465352' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T19:17:36.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:35 smithi016 bash[75004]: cluster 2023-09-23T19:17:34.668433+0000 mon.a (mon.0) 305 : cluster 0 osdmap e81: 8 total, 8 up, 8 in 2023-09-23T19:17:36.035 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:17:35 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:17:35] "GET /metrics HTTP/1.1" 200 35990 "" "Prometheus/2.33.4" 2023-09-23T19:17:36.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:35 smithi016 bash[77995]: cluster 2023-09-23T19:17:34.666089+0000 osd.1 (osd.1) 10 : cluster 3 failed to encode map e81 with expected crc 2023-09-23T19:17:36.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:35 smithi016 bash[77995]: cluster 2023-09-23T19:17:34.666480+0000 osd.0 (osd.0) 9 : cluster 3 failed to encode map e81 with expected crc 2023-09-23T19:17:36.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:35 smithi016 bash[77995]: audit 2023-09-23T19:17:34.668362+0000 mon.a (mon.0) 304 : audit 1 from='client.? 172.21.15.16:0/1043465352' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T19:17:36.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:35 smithi016 bash[77995]: cluster 2023-09-23T19:17:34.668433+0000 mon.a (mon.0) 305 : cluster 0 osdmap e81: 8 total, 8 up, 8 in 2023-09-23T19:17:36.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:35 smithi092 bash[58855]: cluster 2023-09-23T19:17:34.666089+0000 osd.1 (osd.1) 10 : cluster 3 failed to encode map e81 with expected crc 2023-09-23T19:17:36.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:35 smithi092 bash[58855]: cluster 2023-09-23T19:17:34.666480+0000 osd.0 (osd.0) 9 : cluster 3 failed to encode map e81 with expected crc 2023-09-23T19:17:36.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:35 smithi092 bash[58855]: audit 2023-09-23T19:17:34.668362+0000 mon.a (mon.0) 304 : audit 1 from='client.? 172.21.15.16:0/1043465352' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T19:17:36.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:35 smithi092 bash[58855]: cluster 2023-09-23T19:17:34.668433+0000 mon.a (mon.0) 305 : cluster 0 osdmap e81: 8 total, 8 up, 8 in 2023-09-23T19:17:36.895 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:36 smithi016 bash[75004]: cluster 2023-09-23T19:17:35.579876+0000 mgr.y (mgr.25325) 214 : cluster 0 pgmap v155: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:17:36.895 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:36 smithi016 bash[75004]: cluster 2023-09-23T19:17:36.298638+0000 osd.2 (osd.2) 7 : cluster 3 failed to encode map e81 with expected crc 2023-09-23T19:17:36.895 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:36 smithi016 bash[75004]: cluster 2023-09-23T19:17:36.333497+0000 osd.3 (osd.3) 9 : cluster 3 failed to encode map e81 with expected crc 2023-09-23T19:17:36.896 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:36 smithi016 bash[77995]: cluster 2023-09-23T19:17:35.579876+0000 mgr.y (mgr.25325) 214 : cluster 0 pgmap v155: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:17:36.896 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:36 smithi016 bash[77995]: cluster 2023-09-23T19:17:36.298638+0000 osd.2 (osd.2) 7 : cluster 3 failed to encode map e81 with expected crc 2023-09-23T19:17:36.896 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:36 smithi016 bash[77995]: cluster 2023-09-23T19:17:36.333497+0000 osd.3 (osd.3) 9 : cluster 3 failed to encode map e81 with expected crc 2023-09-23T19:17:37.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:36 smithi092 bash[58855]: cluster 2023-09-23T19:17:35.579876+0000 mgr.y (mgr.25325) 214 : cluster 0 pgmap v155: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:17:37.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:36 smithi092 bash[58855]: cluster 2023-09-23T19:17:36.298638+0000 osd.2 (osd.2) 7 : cluster 3 failed to encode map e81 with expected crc 2023-09-23T19:17:37.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:36 smithi092 bash[58855]: cluster 2023-09-23T19:17:36.333497+0000 osd.3 (osd.3) 9 : cluster 3 failed to encode map e81 with expected crc 2023-09-23T19:17:37.898 DEBUG:teuthology.orchestra.run.smithi016:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_2 bench 90 write' 2023-09-23T19:17:37.901 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T19:17:37.901 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T19:17:37.946 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:hints = 1 2023-09-23T19:17:37.946 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T19:17:37.946 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Object prefix: benchmark_data_smithi016_83549 2023-09-23T19:17:37.947 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:17:37.947 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 0 0 0 0 0 0 - 0 2023-09-23T19:17:37.965 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:37 smithi016 bash[75004]: cluster 2023-09-23T19:17:36.609966+0000 osd.7 (osd.7) 7 : cluster 3 failed to encode map e81 with expected crc 2023-09-23T19:17:37.965 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:37 smithi016 bash[77995]: cluster 2023-09-23T19:17:36.609966+0000 osd.7 (osd.7) 7 : cluster 3 failed to encode map e81 with expected crc 2023-09-23T19:17:38.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:37 smithi092 bash[58855]: cluster 2023-09-23T19:17:36.609966+0000 osd.7 (osd.7) 7 : cluster 3 failed to encode map e81 with expected crc 2023-09-23T19:17:38.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:38 smithi016 bash[75004]: cluster 2023-09-23T19:17:37.001169+0000 osd.5 (osd.5) 8 : cluster 3 failed to encode map e81 with expected crc 2023-09-23T19:17:38.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:38 smithi016 bash[75004]: cluster 2023-09-23T19:17:37.580355+0000 mgr.y (mgr.25325) 215 : cluster 0 pgmap v156: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:17:38.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:38 smithi016 bash[75004]: audit 2023-09-23T19:17:37.977507+0000 mon.a (mon.0) 306 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:38.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:38 smithi016 bash[75004]: audit 2023-09-23T19:17:37.979015+0000 mon.a (mon.0) 307 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T19:17:38.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:38 smithi016 bash[75004]: audit 2023-09-23T19:17:37.980353+0000 mon.a (mon.0) 308 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:17:38.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:38 smithi016 bash[75004]: audit 2023-09-23T19:17:38.619132+0000 mon.c (mon.2) 6 : audit 0 from='client.? 172.21.15.16:0/2556962802' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:17:38.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:38 smithi016 bash[77995]: cluster 2023-09-23T19:17:37.001169+0000 osd.5 (osd.5) 8 : cluster 3 failed to encode map e81 with expected crc 2023-09-23T19:17:38.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:38 smithi016 bash[77995]: cluster 2023-09-23T19:17:37.580355+0000 mgr.y (mgr.25325) 215 : cluster 0 pgmap v156: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:17:38.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:38 smithi016 bash[77995]: audit 2023-09-23T19:17:37.977507+0000 mon.a (mon.0) 306 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:38.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:38 smithi016 bash[77995]: audit 2023-09-23T19:17:37.979015+0000 mon.a (mon.0) 307 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T19:17:38.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:38 smithi016 bash[77995]: audit 2023-09-23T19:17:37.980353+0000 mon.a (mon.0) 308 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:17:38.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:38 smithi016 bash[77995]: audit 2023-09-23T19:17:38.619132+0000 mon.c (mon.2) 6 : audit 0 from='client.? 172.21.15.16:0/2556962802' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:17:38.947 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 1 16 2309 2293 143.311 143.312 0.00270414 0.00691291 2023-09-23T19:17:39.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:38 smithi092 bash[58855]: cluster 2023-09-23T19:17:37.001169+0000 osd.5 (osd.5) 8 : cluster 3 failed to encode map e81 with expected crc 2023-09-23T19:17:39.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:38 smithi092 bash[58855]: cluster 2023-09-23T19:17:37.580355+0000 mgr.y (mgr.25325) 215 : cluster 0 pgmap v156: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:17:39.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:38 smithi092 bash[58855]: audit 2023-09-23T19:17:37.977507+0000 mon.a (mon.0) 306 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:39.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:38 smithi092 bash[58855]: audit 2023-09-23T19:17:37.979015+0000 mon.a (mon.0) 307 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T19:17:39.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:38 smithi092 bash[58855]: audit 2023-09-23T19:17:37.980353+0000 mon.a (mon.0) 308 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:17:39.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:38 smithi092 bash[58855]: audit 2023-09-23T19:17:38.619132+0000 mon.c (mon.2) 6 : audit 0 from='client.? 172.21.15.16:0/2556962802' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:17:39.947 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 2 16 4890 4874 152.304 161.312 0.00927426 0.00653715 2023-09-23T19:17:40.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:39 smithi016 bash[75004]: cephadm 2023-09-23T19:17:37.963656+0000 mgr.y (mgr.25325) 216 : cephadm 1 Upgrade: Updating osd.0 2023-09-23T19:17:40.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:39 smithi016 bash[75004]: cephadm 2023-09-23T19:17:37.983532+0000 mgr.y (mgr.25325) 217 : cephadm 1 Deploying daemon osd.0 on smithi016 2023-09-23T19:17:40.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:39 smithi016 bash[77995]: cephadm 2023-09-23T19:17:37.963656+0000 mgr.y (mgr.25325) 216 : cephadm 1 Upgrade: Updating osd.0 2023-09-23T19:17:40.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:39 smithi016 bash[77995]: cephadm 2023-09-23T19:17:37.983532+0000 mgr.y (mgr.25325) 217 : cephadm 1 Deploying daemon osd.0 on smithi016 2023-09-23T19:17:40.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:39 smithi092 bash[58855]: cephadm 2023-09-23T19:17:37.963656+0000 mgr.y (mgr.25325) 216 : cephadm 1 Upgrade: Updating osd.0 2023-09-23T19:17:40.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:39 smithi092 bash[58855]: cephadm 2023-09-23T19:17:37.983532+0000 mgr.y (mgr.25325) 217 : cephadm 1 Deploying daemon osd.0 on smithi016 2023-09-23T19:17:40.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:40 smithi016 bash[77995]: cluster 2023-09-23T19:17:39.580823+0000 mgr.y (mgr.25325) 218 : cluster 0 pgmap v157: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:17:40.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:40 smithi016 bash[75004]: cluster 2023-09-23T19:17:39.580823+0000 mgr.y (mgr.25325) 218 : cluster 0 pgmap v157: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:17:40.947 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 3 16 7187 7171 149.386 143.562 0.00257996 0.00667269 2023-09-23T19:17:41.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:40 smithi092 bash[58855]: cluster 2023-09-23T19:17:39.580823+0000 mgr.y (mgr.25325) 218 : cluster 0 pgmap v157: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:17:41.947 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 4 16 9502 9486 148.208 144.688 0.00619261 0.00673156 2023-09-23T19:17:42.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:41 smithi016 bash[77995]: audit 2023-09-23T19:17:41.328888+0000 mon.c (mon.2) 7 : audit 0 from='client.? 172.21.15.16:0/2786732907' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:17:42.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:41 smithi016 bash[75004]: audit 2023-09-23T19:17:41.328888+0000 mon.c (mon.2) 7 : audit 0 from='client.? 172.21.15.16:0/2786732907' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:17:42.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:41 smithi092 bash[58855]: audit 2023-09-23T19:17:41.328888+0000 mon.c (mon.2) 7 : audit 0 from='client.? 172.21.15.16:0/2786732907' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:17:42.947 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 5 16 11701 11685 146.051 137.438 0.00504408 0.00683331 2023-09-23T19:17:43.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:42 smithi016 bash[77995]: cluster 2023-09-23T19:17:41.581598+0000 mgr.y (mgr.25325) 219 : cluster 0 pgmap v158: 89 pgs: 89 active+clean; 296 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 30 MiB/s wr, 473 op/s 2023-09-23T19:17:43.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:42 smithi016 bash[75004]: cluster 2023-09-23T19:17:41.581598+0000 mgr.y (mgr.25325) 219 : cluster 0 pgmap v158: 89 pgs: 89 active+clean; 296 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 30 MiB/s wr, 473 op/s 2023-09-23T19:17:43.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:42 smithi092 bash[58855]: cluster 2023-09-23T19:17:41.581598+0000 mgr.y (mgr.25325) 219 : cluster 0 pgmap v158: 89 pgs: 89 active+clean; 296 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 30 MiB/s wr, 473 op/s 2023-09-23T19:17:43.947 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 6 16 13682 13666 142.343 123.812 0.011566 0.00700807 2023-09-23T19:17:44.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:43 smithi016 bash[75004]: audit 2023-09-23T19:17:43.613817+0000 mon.a (mon.0) 309 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:17:44.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:43 smithi016 bash[77995]: audit 2023-09-23T19:17:43.613817+0000 mon.a (mon.0) 309 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:17:44.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:43 smithi092 bash[58855]: audit 2023-09-23T19:17:43.613817+0000 mon.a (mon.0) 309 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:17:44.710 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:44 smithi016 bash[75004]: cluster 2023-09-23T19:17:43.582080+0000 mgr.y (mgr.25325) 220 : cluster 0 pgmap v159: 89 pgs: 89 active+clean; 296 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 30 MiB/s wr, 473 op/s 2023-09-23T19:17:44.710 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:44 smithi016 bash[75004]: audit 2023-09-23T19:17:44.067098+0000 mon.c (mon.2) 8 : audit 0 from='client.? 172.21.15.16:0/1436159703' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:17:44.947 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 7 16 15701 15685 140.033 126.188 0.00605674 0.00712999 2023-09-23T19:17:45.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:44 smithi016 bash[77995]: cluster 2023-09-23T19:17:43.582080+0000 mgr.y (mgr.25325) 220 : cluster 0 pgmap v159: 89 pgs: 89 active+clean; 296 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 30 MiB/s wr, 473 op/s 2023-09-23T19:17:45.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:44 smithi016 bash[77995]: audit 2023-09-23T19:17:44.067098+0000 mon.c (mon.2) 8 : audit 0 from='client.? 172.21.15.16:0/1436159703' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:17:45.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:44 smithi092 bash[58855]: cluster 2023-09-23T19:17:43.582080+0000 mgr.y (mgr.25325) 220 : cluster 0 pgmap v159: 89 pgs: 89 active+clean; 296 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 30 MiB/s wr, 473 op/s 2023-09-23T19:17:45.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:44 smithi092 bash[58855]: audit 2023-09-23T19:17:44.067098+0000 mon.c (mon.2) 8 : audit 0 from='client.? 172.21.15.16:0/1436159703' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:17:45.947 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 8 16 17759 17743 138.605 128.625 0.00925906 0.00720339 2023-09-23T19:17:45.974 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:45 smithi016 bash[75004]: cluster 2023-09-23T19:17:45.582765+0000 mgr.y (mgr.25325) 221 : cluster 0 pgmap v160: 89 pgs: 89 active+clean; 976 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-09-23T19:17:45.974 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:17:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:17:45] "GET /metrics HTTP/1.1" 200 36045 "" "Prometheus/2.33.4" 2023-09-23T19:17:45.974 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:45 smithi016 bash[77995]: cluster 2023-09-23T19:17:45.582765+0000 mgr.y (mgr.25325) 221 : cluster 0 pgmap v160: 89 pgs: 89 active+clean; 976 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-09-23T19:17:46.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:45 smithi092 bash[58855]: cluster 2023-09-23T19:17:45.582765+0000 mgr.y (mgr.25325) 221 : cluster 0 pgmap v160: 89 pgs: 89 active+clean; 976 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-09-23T19:17:46.277 INFO:journalctl@ceph.osd.0.smithi016.stdout:Sep 23 19:17:46 smithi016 systemd[1]: Stopping Ceph osd.0 for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7... 2023-09-23T19:17:46.533 INFO:journalctl@ceph.osd.0.smithi016.stdout:Sep 23 19:17:46 smithi016 bash[36773]: debug 2023-09-23T19:17:46.454+0000 7ff860d70700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-09-23T19:17:46.534 INFO:journalctl@ceph.osd.0.smithi016.stdout:Sep 23 19:17:46 smithi016 bash[36773]: debug 2023-09-23T19:17:46.454+0000 7ff860d70700 -1 osd.0 81 *** Got signal Terminated *** 2023-09-23T19:17:46.534 INFO:journalctl@ceph.osd.0.smithi016.stdout:Sep 23 19:17:46 smithi016 bash[36773]: debug 2023-09-23T19:17:46.454+0000 7ff860d70700 -1 osd.0 81 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T19:17:46.948 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 9 16 19021 19005 131.968 78.875 0.00177686 0.00739585 2023-09-23T19:17:47.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:46 smithi016 bash[77995]: cluster 2023-09-23T19:17:46.465218+0000 mon.a (mon.0) 310 : cluster 1 osd.0 marked itself down and dead 2023-09-23T19:17:47.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:46 smithi016 bash[77995]: audit 2023-09-23T19:17:46.686090+0000 mon.c (mon.2) 9 : audit 0 from='client.? 172.21.15.16:0/3662862458' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:17:47.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:46 smithi016 bash[75004]: cluster 2023-09-23T19:17:46.465218+0000 mon.a (mon.0) 310 : cluster 1 osd.0 marked itself down and dead 2023-09-23T19:17:47.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:46 smithi016 bash[75004]: audit 2023-09-23T19:17:46.686090+0000 mon.c (mon.2) 9 : audit 0 from='client.? 172.21.15.16:0/3662862458' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:17:47.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:46 smithi092 bash[58855]: cluster 2023-09-23T19:17:46.465218+0000 mon.a (mon.0) 310 : cluster 1 osd.0 marked itself down and dead 2023-09-23T19:17:47.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:46 smithi092 bash[58855]: audit 2023-09-23T19:17:46.686090+0000 mon.c (mon.2) 9 : audit 0 from='client.? 172.21.15.16:0/3662862458' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:17:47.948 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 10 16 19460 19444 121.514 27.4375 0.00253413 0.00821892 2023-09-23T19:17:48.034 INFO:journalctl@ceph.osd.0.smithi016.stdout:Sep 23 19:17:47 smithi016 bash[83930]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd-0 2023-09-23T19:17:48.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:47 smithi016 bash[75004]: cluster 2023-09-23T19:17:46.718903+0000 mon.a (mon.0) 311 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T19:17:48.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:47 smithi016 bash[75004]: cluster 2023-09-23T19:17:46.731873+0000 mon.a (mon.0) 312 : cluster 0 osdmap e82: 8 total, 7 up, 8 in 2023-09-23T19:17:48.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:47 smithi016 bash[75004]: cluster 2023-09-23T19:17:46.733215+0000 osd.7 (osd.7) 8 : cluster 3 failed to encode map e82 with expected crc 2023-09-23T19:17:48.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:47 smithi016 bash[75004]: cluster 2023-09-23T19:17:47.583374+0000 mgr.y (mgr.25325) 222 : cluster 0 pgmap v162: 89 pgs: 15 stale+active+clean, 74 active+clean; 976 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2023-09-23T19:17:48.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:47 smithi016 bash[77995]: cluster 2023-09-23T19:17:46.718903+0000 mon.a (mon.0) 311 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T19:17:48.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:47 smithi016 bash[77995]: cluster 2023-09-23T19:17:46.731873+0000 mon.a (mon.0) 312 : cluster 0 osdmap e82: 8 total, 7 up, 8 in 2023-09-23T19:17:48.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:47 smithi016 bash[77995]: cluster 2023-09-23T19:17:46.733215+0000 osd.7 (osd.7) 8 : cluster 3 failed to encode map e82 with expected crc 2023-09-23T19:17:48.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:47 smithi016 bash[77995]: cluster 2023-09-23T19:17:47.583374+0000 mgr.y (mgr.25325) 222 : cluster 0 pgmap v162: 89 pgs: 15 stale+active+clean, 74 active+clean; 976 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2023-09-23T19:17:48.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:47 smithi092 bash[58855]: cluster 2023-09-23T19:17:46.718903+0000 mon.a (mon.0) 311 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T19:17:48.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:47 smithi092 bash[58855]: cluster 2023-09-23T19:17:46.731873+0000 mon.a (mon.0) 312 : cluster 0 osdmap e82: 8 total, 7 up, 8 in 2023-09-23T19:17:48.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:47 smithi092 bash[58855]: cluster 2023-09-23T19:17:46.733215+0000 osd.7 (osd.7) 8 : cluster 3 failed to encode map e82 with expected crc 2023-09-23T19:17:48.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:47 smithi092 bash[58855]: cluster 2023-09-23T19:17:47.583374+0000 mgr.y (mgr.25325) 222 : cluster 0 pgmap v162: 89 pgs: 15 stale+active+clean, 74 active+clean; 976 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2023-09-23T19:17:48.534 INFO:journalctl@ceph.osd.0.smithi016.stdout:Sep 23 19:17:48 smithi016 bash[83992]: Error response from daemon: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd-0 2023-09-23T19:17:48.948 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 11 16 21701 21685 123.199 140.062 0.0223269 0.00810868 2023-09-23T19:17:49.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:48 smithi016 bash[75004]: cluster 2023-09-23T19:17:47.735688+0000 osd.2 (osd.2) 8 : cluster 3 failed to encode map e83 with expected crc 2023-09-23T19:17:49.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:48 smithi016 bash[75004]: cluster 2023-09-23T19:17:47.737825+0000 mon.a (mon.0) 313 : cluster 0 osdmap e83: 8 total, 7 up, 8 in 2023-09-23T19:17:49.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:48 smithi016 bash[75004]: cluster 2023-09-23T19:17:47.738226+0000 osd.1 (osd.1) 11 : cluster 3 failed to encode map e83 with expected crc 2023-09-23T19:17:49.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:48 smithi016 bash[75004]: cluster 2023-09-23T19:17:47.738405+0000 osd.2 (osd.2) 9 : cluster 3 failed to encode map e83 with expected crc 2023-09-23T19:17:49.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:48 smithi016 bash[75004]: cluster 2023-09-23T19:17:47.739474+0000 osd.3 (osd.3) 10 : cluster 3 failed to encode map e83 with expected crc 2023-09-23T19:17:49.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:48 smithi016 bash[75004]: cluster 2023-09-23T19:17:47.742194+0000 osd.7 (osd.7) 9 : cluster 3 failed to encode map e83 with expected crc 2023-09-23T19:17:49.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:48 smithi016 bash[75004]: cluster 2023-09-23T19:17:47.742233+0000 osd.4 (osd.4) 8 : cluster 3 failed to encode map e83 with expected crc 2023-09-23T19:17:49.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:48 smithi016 bash[75004]: cluster 2023-09-23T19:17:47.742367+0000 osd.6 (osd.6) 8 : cluster 3 failed to encode map e83 with expected crc 2023-09-23T19:17:49.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:48 smithi016 bash[75004]: cluster 2023-09-23T19:17:47.742367+0000 osd.5 (osd.5) 9 : cluster 3 failed to encode map e83 with expected crc 2023-09-23T19:17:49.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:48 smithi016 bash[77995]: cluster 2023-09-23T19:17:47.735688+0000 osd.2 (osd.2) 8 : cluster 3 failed to encode map e83 with expected crc 2023-09-23T19:17:49.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:48 smithi016 bash[77995]: cluster 2023-09-23T19:17:47.737825+0000 mon.a (mon.0) 313 : cluster 0 osdmap e83: 8 total, 7 up, 8 in 2023-09-23T19:17:49.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:48 smithi016 bash[77995]: cluster 2023-09-23T19:17:47.738226+0000 osd.1 (osd.1) 11 : cluster 3 failed to encode map e83 with expected crc 2023-09-23T19:17:49.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:48 smithi016 bash[77995]: cluster 2023-09-23T19:17:47.738405+0000 osd.2 (osd.2) 9 : cluster 3 failed to encode map e83 with expected crc 2023-09-23T19:17:49.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:48 smithi016 bash[77995]: cluster 2023-09-23T19:17:47.739474+0000 osd.3 (osd.3) 10 : cluster 3 failed to encode map e83 with expected crc 2023-09-23T19:17:49.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:48 smithi016 bash[77995]: cluster 2023-09-23T19:17:47.742194+0000 osd.7 (osd.7) 9 : cluster 3 failed to encode map e83 with expected crc 2023-09-23T19:17:49.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:48 smithi016 bash[77995]: cluster 2023-09-23T19:17:47.742233+0000 osd.4 (osd.4) 8 : cluster 3 failed to encode map e83 with expected crc 2023-09-23T19:17:49.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:48 smithi016 bash[77995]: cluster 2023-09-23T19:17:47.742367+0000 osd.6 (osd.6) 8 : cluster 3 failed to encode map e83 with expected crc 2023-09-23T19:17:49.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:48 smithi016 bash[77995]: cluster 2023-09-23T19:17:47.742367+0000 osd.5 (osd.5) 9 : cluster 3 failed to encode map e83 with expected crc 2023-09-23T19:17:49.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:48 smithi092 bash[58855]: cluster 2023-09-23T19:17:47.735688+0000 osd.2 (osd.2) 8 : cluster 3 failed to encode map e83 with expected crc 2023-09-23T19:17:49.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:48 smithi092 bash[58855]: cluster 2023-09-23T19:17:47.737825+0000 mon.a (mon.0) 313 : cluster 0 osdmap e83: 8 total, 7 up, 8 in 2023-09-23T19:17:49.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:48 smithi092 bash[58855]: cluster 2023-09-23T19:17:47.738226+0000 osd.1 (osd.1) 11 : cluster 3 failed to encode map e83 with expected crc 2023-09-23T19:17:49.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:48 smithi092 bash[58855]: cluster 2023-09-23T19:17:47.738405+0000 osd.2 (osd.2) 9 : cluster 3 failed to encode map e83 with expected crc 2023-09-23T19:17:49.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:48 smithi092 bash[58855]: cluster 2023-09-23T19:17:47.739474+0000 osd.3 (osd.3) 10 : cluster 3 failed to encode map e83 with expected crc 2023-09-23T19:17:49.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:48 smithi092 bash[58855]: cluster 2023-09-23T19:17:47.742194+0000 osd.7 (osd.7) 9 : cluster 3 failed to encode map e83 with expected crc 2023-09-23T19:17:49.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:48 smithi092 bash[58855]: cluster 2023-09-23T19:17:47.742233+0000 osd.4 (osd.4) 8 : cluster 3 failed to encode map e83 with expected crc 2023-09-23T19:17:49.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:48 smithi092 bash[58855]: cluster 2023-09-23T19:17:47.742367+0000 osd.6 (osd.6) 8 : cluster 3 failed to encode map e83 with expected crc 2023-09-23T19:17:49.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:48 smithi092 bash[58855]: cluster 2023-09-23T19:17:47.742367+0000 osd.5 (osd.5) 9 : cluster 3 failed to encode map e83 with expected crc 2023-09-23T19:17:49.948 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 12 16 23880 23864 124.28 136.188 0.00864929 0.00803544 2023-09-23T19:17:50.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:49 smithi016 bash[75004]: audit 2023-09-23T19:17:49.280856+0000 mon.b (mon.1) 3 : audit 0 from='client.? 172.21.15.16:0/174238095' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:17:50.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:49 smithi016 bash[75004]: cluster 2023-09-23T19:17:49.583848+0000 mgr.y (mgr.25325) 223 : cluster 0 pgmap v164: 89 pgs: 15 stale+active+clean, 74 active+clean; 976 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-09-23T19:17:50.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:49 smithi016 bash[77995]: audit 2023-09-23T19:17:49.280856+0000 mon.b (mon.1) 3 : audit 0 from='client.? 172.21.15.16:0/174238095' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:17:50.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:49 smithi016 bash[77995]: cluster 2023-09-23T19:17:49.583848+0000 mgr.y (mgr.25325) 223 : cluster 0 pgmap v164: 89 pgs: 15 stale+active+clean, 74 active+clean; 976 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-09-23T19:17:50.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:49 smithi092 bash[58855]: audit 2023-09-23T19:17:49.280856+0000 mon.b (mon.1) 3 : audit 0 from='client.? 172.21.15.16:0/174238095' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:17:50.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:49 smithi092 bash[58855]: cluster 2023-09-23T19:17:49.583848+0000 mgr.y (mgr.25325) 223 : cluster 0 pgmap v164: 89 pgs: 15 stale+active+clean, 74 active+clean; 976 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-09-23T19:17:50.948 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 13 16 25969 25953 124.763 130.562 0.00973732 0.00800547 2023-09-23T19:17:51.948 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 14 16 27926 27910 124.587 122.312 0.0107852 0.00801562 2023-09-23T19:17:52.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:51 smithi016 bash[75004]: cluster 2023-09-23T19:17:51.636251+0000 mon.a (mon.0) 314 : cluster 3 Health check failed: Degraded data redundancy: 11825/71337 objects degraded (16.576%), 15 pgs degraded (PG_DEGRADED) 2023-09-23T19:17:52.034 INFO:journalctl@ceph.osd.0.smithi016.stdout:Sep 23 19:17:51 smithi016 systemd[1]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@osd.0.service: Succeeded. 2023-09-23T19:17:52.034 INFO:journalctl@ceph.osd.0.smithi016.stdout:Sep 23 19:17:51 smithi016 systemd[1]: Stopped Ceph osd.0 for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7. 2023-09-23T19:17:52.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:51 smithi016 bash[77995]: cluster 2023-09-23T19:17:51.636251+0000 mon.a (mon.0) 314 : cluster 3 Health check failed: Degraded data redundancy: 11825/71337 objects degraded (16.576%), 15 pgs degraded (PG_DEGRADED) 2023-09-23T19:17:52.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:51 smithi092 bash[58855]: cluster 2023-09-23T19:17:51.636251+0000 mon.a (mon.0) 314 : cluster 3 Health check failed: Degraded data redundancy: 11825/71337 objects degraded (16.576%), 15 pgs degraded (PG_DEGRADED) 2023-09-23T19:17:52.534 INFO:journalctl@ceph.osd.0.smithi016.stdout:Sep 23 19:17:52 smithi016 systemd[1]: Started Ceph osd.0 for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7. 2023-09-23T19:17:52.948 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 15 16 29791 29775 124.051 116.562 0.0062013 0.00805212 2023-09-23T19:17:53.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:52 smithi016 bash[75004]: cluster 2023-09-23T19:17:51.584885+0000 mgr.y (mgr.25325) 224 : cluster 0 pgmap v165: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 1.5 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 149 MiB/s wr, 2.38k op/s; 11825/71337 objects degraded (16.576%); 8.0 KiB/s, 0 objects/s recovering 2023-09-23T19:17:53.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:52 smithi016 bash[75004]: audit 2023-09-23T19:17:51.866086+0000 mon.a (mon.0) 315 : audit 0 from='client.? 172.21.15.16:0/254857485' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:17:53.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:52 smithi016 bash[75004]: audit 2023-09-23T19:17:52.128794+0000 mon.a (mon.0) 316 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:53.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:52 smithi016 bash[75004]: audit 2023-09-23T19:17:52.141307+0000 mon.a (mon.0) 317 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:53.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:52 smithi016 bash[75004]: audit 2023-09-23T19:17:52.163779+0000 mon.a (mon.0) 318 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:17:53.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:52 smithi016 bash[77995]: cluster 2023-09-23T19:17:51.584885+0000 mgr.y (mgr.25325) 224 : cluster 0 pgmap v165: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 1.5 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 149 MiB/s wr, 2.38k op/s; 11825/71337 objects degraded (16.576%); 8.0 KiB/s, 0 objects/s recovering 2023-09-23T19:17:53.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:52 smithi016 bash[77995]: audit 2023-09-23T19:17:51.866086+0000 mon.a (mon.0) 315 : audit 0 from='client.? 172.21.15.16:0/254857485' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:17:53.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:52 smithi016 bash[77995]: audit 2023-09-23T19:17:52.128794+0000 mon.a (mon.0) 316 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:53.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:52 smithi016 bash[77995]: audit 2023-09-23T19:17:52.141307+0000 mon.a (mon.0) 317 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:53.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:52 smithi016 bash[77995]: audit 2023-09-23T19:17:52.163779+0000 mon.a (mon.0) 318 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:17:53.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:52 smithi092 bash[58855]: cluster 2023-09-23T19:17:51.584885+0000 mgr.y (mgr.25325) 224 : cluster 0 pgmap v165: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 1.5 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 149 MiB/s wr, 2.38k op/s; 11825/71337 objects degraded (16.576%); 8.0 KiB/s, 0 objects/s recovering 2023-09-23T19:17:53.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:52 smithi092 bash[58855]: audit 2023-09-23T19:17:51.866086+0000 mon.a (mon.0) 315 : audit 0 from='client.? 172.21.15.16:0/254857485' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:17:53.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:52 smithi092 bash[58855]: audit 2023-09-23T19:17:52.128794+0000 mon.a (mon.0) 316 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:53.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:52 smithi092 bash[58855]: audit 2023-09-23T19:17:52.141307+0000 mon.a (mon.0) 317 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:53.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:52 smithi092 bash[58855]: audit 2023-09-23T19:17:52.163779+0000 mon.a (mon.0) 318 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:17:53.948 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 16 16 31812 31796 124.192 126.312 0.00526625 0.00804238 2023-09-23T19:17:54.802 INFO:journalctl@ceph.osd.0.smithi016.stdout:Sep 23 19:17:54 smithi016 bash[84192]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T19:17:54.803 INFO:journalctl@ceph.osd.0.smithi016.stdout:Sep 23 19:17:54 smithi016 bash[84192]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-09-23T19:17:54.803 INFO:journalctl@ceph.osd.0.smithi016.stdout:Sep 23 19:17:54 smithi016 bash[84192]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2023-09-23T19:17:54.803 INFO:journalctl@ceph.osd.0.smithi016.stdout:Sep 23 19:17:54 smithi016 bash[84192]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2023-09-23T19:17:54.803 INFO:journalctl@ceph.osd.0.smithi016.stdout:Sep 23 19:17:54 smithi016 bash[84192]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2023-09-23T19:17:54.803 INFO:journalctl@ceph.osd.0.smithi016.stdout:Sep 23 19:17:54 smithi016 bash[84192]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2023-09-23T19:17:54.803 INFO:journalctl@ceph.osd.0.smithi016.stdout:Sep 23 19:17:54 smithi016 bash[84192]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-09-23T19:17:54.803 INFO:journalctl@ceph.osd.0.smithi016.stdout:Sep 23 19:17:54 smithi016 bash[84192]: --> ceph-volume lvm activate successful for osd ID: 0 2023-09-23T19:17:54.804 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:54 smithi016 bash[75004]: cluster 2023-09-23T19:17:53.585469+0000 mgr.y (mgr.25325) 225 : cluster 0 pgmap v166: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 1.5 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 11825/71337 objects degraded (16.576%); 8.0 KiB/s, 0 objects/s recovering 2023-09-23T19:17:54.804 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:54 smithi016 bash[75004]: audit 2023-09-23T19:17:54.455472+0000 mon.b (mon.1) 4 : audit 0 from='client.? 172.21.15.16:0/1470532052' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:17:54.804 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:54 smithi016 bash[77995]: cluster 2023-09-23T19:17:53.585469+0000 mgr.y (mgr.25325) 225 : cluster 0 pgmap v166: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 1.5 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 11825/71337 objects degraded (16.576%); 8.0 KiB/s, 0 objects/s recovering 2023-09-23T19:17:54.804 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:54 smithi016 bash[77995]: audit 2023-09-23T19:17:54.455472+0000 mon.b (mon.1) 4 : audit 0 from='client.? 172.21.15.16:0/1470532052' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:17:54.948 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 17 16 33593 33577 123.434 111.312 0.00654754 0.008093 2023-09-23T19:17:55.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:54 smithi092 bash[58855]: cluster 2023-09-23T19:17:53.585469+0000 mgr.y (mgr.25325) 225 : cluster 0 pgmap v166: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 1.5 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 11825/71337 objects degraded (16.576%); 8.0 KiB/s, 0 objects/s recovering 2023-09-23T19:17:55.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:54 smithi092 bash[58855]: audit 2023-09-23T19:17:54.455472+0000 mon.b (mon.1) 4 : audit 0 from='client.? 172.21.15.16:0/1470532052' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:17:55.948 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 18 16 35570 35554 123.44 123.562 0.00605581 0.00809226 2023-09-23T19:17:55.991 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:17:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:17:55] "GET /metrics HTTP/1.1" 200 36045 "" "Prometheus/2.33.4" 2023-09-23T19:17:56.948 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 19 16 37275 37259 122.551 106.562 0.0115009 0.00815181 2023-09-23T19:17:57.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:56 smithi016 bash[75004]: cluster 2023-09-23T19:17:55.586462+0000 mgr.y (mgr.25325) 226 : cluster 0 pgmap v167: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.0 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 126 MiB/s wr, 2.02k op/s; 16677/100494 objects degraded (16.595%); 7.2 KiB/s, 0 objects/s recovering 2023-09-23T19:17:57.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:56 smithi016 bash[77995]: cluster 2023-09-23T19:17:55.586462+0000 mgr.y (mgr.25325) 226 : cluster 0 pgmap v167: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.0 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 126 MiB/s wr, 2.02k op/s; 16677/100494 objects degraded (16.595%); 7.2 KiB/s, 0 objects/s recovering 2023-09-23T19:17:57.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:56 smithi092 bash[58855]: cluster 2023-09-23T19:17:55.586462+0000 mgr.y (mgr.25325) 226 : cluster 0 pgmap v167: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.0 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 126 MiB/s wr, 2.02k op/s; 16677/100494 objects degraded (16.595%); 7.2 KiB/s, 0 objects/s recovering 2023-09-23T19:17:57.949 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-09-23T19:17:57.947910+0000 min lat: 0.00151424 max lat: 1.03538 avg lat: 0.00818746 2023-09-23T19:17:57.949 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:17:57.949 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 20 16 39058 39042 121.994 111.438 0.0161863 0.00818746 2023-09-23T19:17:58.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:57 smithi016 bash[75004]: audit 2023-09-23T19:17:57.038255+0000 mon.c (mon.2) 10 : audit 0 from='client.? 172.21.15.16:0/2435156143' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:17:58.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:57 smithi016 bash[77995]: audit 2023-09-23T19:17:57.038255+0000 mon.c (mon.2) 10 : audit 0 from='client.? 172.21.15.16:0/2435156143' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:17:58.035 INFO:journalctl@ceph.osd.0.smithi016.stdout:Sep 23 19:17:57 smithi016 bash[84502]: debug 2023-09-23T19:17:57.897+0000 7f72951fc640 -1 Falling back to public interface 2023-09-23T19:17:58.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:57 smithi092 bash[58855]: audit 2023-09-23T19:17:57.038255+0000 mon.c (mon.2) 10 : audit 0 from='client.? 172.21.15.16:0/2435156143' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:17:58.949 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 21 16 40815 40799 121.414 109.812 0.00763228 0.00822832 2023-09-23T19:17:59.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:58 smithi016 bash[75004]: cluster 2023-09-23T19:17:57.587166+0000 mgr.y (mgr.25325) 227 : cluster 0 pgmap v168: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.0 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s; 16677/100494 objects degraded (16.595%); 6.4 KiB/s, 0 objects/s recovering 2023-09-23T19:17:59.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:58 smithi016 bash[75004]: audit 2023-09-23T19:17:58.617983+0000 mon.a (mon.0) 319 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:59.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:58 smithi016 bash[75004]: audit 2023-09-23T19:17:58.621194+0000 mon.a (mon.0) 320 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:17:59.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:58 smithi016 bash[77995]: cluster 2023-09-23T19:17:57.587166+0000 mgr.y (mgr.25325) 227 : cluster 0 pgmap v168: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.0 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s; 16677/100494 objects degraded (16.595%); 6.4 KiB/s, 0 objects/s recovering 2023-09-23T19:17:59.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:58 smithi016 bash[77995]: audit 2023-09-23T19:17:58.617983+0000 mon.a (mon.0) 319 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:59.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:58 smithi016 bash[77995]: audit 2023-09-23T19:17:58.621194+0000 mon.a (mon.0) 320 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:17:59.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:58 smithi092 bash[58855]: cluster 2023-09-23T19:17:57.587166+0000 mgr.y (mgr.25325) 227 : cluster 0 pgmap v168: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.0 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s; 16677/100494 objects degraded (16.595%); 6.4 KiB/s, 0 objects/s recovering 2023-09-23T19:17:59.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:58 smithi092 bash[58855]: audit 2023-09-23T19:17:58.617983+0000 mon.a (mon.0) 319 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:17:59.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:58 smithi092 bash[58855]: audit 2023-09-23T19:17:58.621194+0000 mon.a (mon.0) 320 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:17:59.949 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 22 16 42312 42296 120.147 93.5625 0.00543679 0.00831199 2023-09-23T19:18:00.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:59 smithi016 bash[75004]: cluster 2023-09-23T19:17:58.978448+0000 mon.a (mon.0) 321 : cluster 3 Health check update: Degraded data redundancy: 16677/100494 objects degraded (16.595%), 15 pgs degraded (PG_DEGRADED) 2023-09-23T19:18:00.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:59 smithi016 bash[75004]: cluster 2023-09-23T19:17:59.588881+0000 mgr.y (mgr.25325) 228 : cluster 0 pgmap v169: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.1 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s; 17792/105537 objects degraded (16.859%); 5.4 KiB/s, 0 objects/s recovering 2023-09-23T19:18:00.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:17:59 smithi016 bash[75004]: audit 2023-09-23T19:17:59.652588+0000 mon.b (mon.1) 5 : audit 0 from='client.? 172.21.15.16:0/97624746' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:00.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:59 smithi016 bash[77995]: cluster 2023-09-23T19:17:58.978448+0000 mon.a (mon.0) 321 : cluster 3 Health check update: Degraded data redundancy: 16677/100494 objects degraded (16.595%), 15 pgs degraded (PG_DEGRADED) 2023-09-23T19:18:00.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:59 smithi016 bash[77995]: cluster 2023-09-23T19:17:59.588881+0000 mgr.y (mgr.25325) 228 : cluster 0 pgmap v169: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.1 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s; 17792/105537 objects degraded (16.859%); 5.4 KiB/s, 0 objects/s recovering 2023-09-23T19:18:00.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:17:59 smithi016 bash[77995]: audit 2023-09-23T19:17:59.652588+0000 mon.b (mon.1) 5 : audit 0 from='client.? 172.21.15.16:0/97624746' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:00.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:59 smithi092 bash[58855]: cluster 2023-09-23T19:17:58.978448+0000 mon.a (mon.0) 321 : cluster 3 Health check update: Degraded data redundancy: 16677/100494 objects degraded (16.595%), 15 pgs degraded (PG_DEGRADED) 2023-09-23T19:18:00.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:59 smithi092 bash[58855]: cluster 2023-09-23T19:17:59.588881+0000 mgr.y (mgr.25325) 228 : cluster 0 pgmap v169: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.1 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s; 17792/105537 objects degraded (16.859%); 5.4 KiB/s, 0 objects/s recovering 2023-09-23T19:18:00.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:17:59 smithi092 bash[58855]: audit 2023-09-23T19:17:59.652588+0000 mon.b (mon.1) 5 : audit 0 from='client.? 172.21.15.16:0/97624746' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:00.949 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 23 16 43690 43674 118.668 86.125 0.00747136 0.00841834 2023-09-23T19:18:01.951 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 24 16 44979 44963 117.069 80.5625 0.00596494 0.00853298 2023-09-23T19:18:02.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:01 smithi016 bash[75004]: audit 2023-09-23T19:18:00.835742+0000 mon.a (mon.0) 322 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:18:02.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:01 smithi016 bash[75004]: audit 2023-09-23T19:18:00.848732+0000 mon.a (mon.0) 323 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:18:02.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:01 smithi016 bash[75004]: cluster 2023-09-23T19:18:01.589782+0000 mgr.y (mgr.25325) 229 : cluster 0 pgmap v170: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.6 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s; 21072/126828 objects degraded (16.615%); 5.3 KiB/s, 0 objects/s recovering 2023-09-23T19:18:02.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:01 smithi016 bash[77995]: audit 2023-09-23T19:18:00.835742+0000 mon.a (mon.0) 322 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:18:02.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:01 smithi016 bash[77995]: audit 2023-09-23T19:18:00.848732+0000 mon.a (mon.0) 323 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:18:02.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:01 smithi016 bash[77995]: cluster 2023-09-23T19:18:01.589782+0000 mgr.y (mgr.25325) 229 : cluster 0 pgmap v170: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.6 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s; 21072/126828 objects degraded (16.615%); 5.3 KiB/s, 0 objects/s recovering 2023-09-23T19:18:02.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:01 smithi092 bash[58855]: audit 2023-09-23T19:18:00.835742+0000 mon.a (mon.0) 322 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:18:02.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:01 smithi092 bash[58855]: audit 2023-09-23T19:18:00.848732+0000 mon.a (mon.0) 323 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:18:02.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:01 smithi092 bash[58855]: cluster 2023-09-23T19:18:01.589782+0000 mgr.y (mgr.25325) 229 : cluster 0 pgmap v170: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.6 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s; 21072/126828 objects degraded (16.615%); 5.3 KiB/s, 0 objects/s recovering 2023-09-23T19:18:02.951 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 25 16 46366 46350 115.853 86.6875 0.00536329 0.00862355 2023-09-23T19:18:03.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:03 smithi016 bash[77995]: audit 2023-09-23T19:18:02.050677+0000 mon.a (mon.0) 324 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:18:03.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:03 smithi016 bash[77995]: audit 2023-09-23T19:18:02.068347+0000 mon.a (mon.0) 325 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:18:03.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:03 smithi016 bash[77995]: audit 2023-09-23T19:18:02.305413+0000 mon.a (mon.0) 326 : audit 0 from='client.? 172.21.15.16:0/2386201854' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:03.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:03 smithi016 bash[75004]: audit 2023-09-23T19:18:02.050677+0000 mon.a (mon.0) 324 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:18:03.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:03 smithi016 bash[75004]: audit 2023-09-23T19:18:02.068347+0000 mon.a (mon.0) 325 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:18:03.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:03 smithi016 bash[75004]: audit 2023-09-23T19:18:02.305413+0000 mon.a (mon.0) 326 : audit 0 from='client.? 172.21.15.16:0/2386201854' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:03.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:03 smithi092 bash[58855]: audit 2023-09-23T19:18:02.050677+0000 mon.a (mon.0) 324 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:18:03.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:03 smithi092 bash[58855]: audit 2023-09-23T19:18:02.068347+0000 mon.a (mon.0) 325 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:18:03.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:03 smithi092 bash[58855]: audit 2023-09-23T19:18:02.305413+0000 mon.a (mon.0) 326 : audit 0 from='client.? 172.21.15.16:0/2386201854' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:03.952 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 26 16 47854 47838 114.974 93 0.00876718 0.00869037 2023-09-23T19:18:04.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:04 smithi016 bash[75004]: cluster 2023-09-23T19:18:03.590408+0000 mgr.y (mgr.25325) 230 : cluster 0 pgmap v171: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.6 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s; 21072/126828 objects degraded (16.615%) 2023-09-23T19:18:04.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:04 smithi016 bash[75004]: cluster 2023-09-23T19:18:03.979772+0000 mon.a (mon.0) 327 : cluster 3 Health check update: Degraded data redundancy: 21072/126828 objects degraded (16.615%), 15 pgs degraded (PG_DEGRADED) 2023-09-23T19:18:04.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:04 smithi016 bash[77995]: cluster 2023-09-23T19:18:03.590408+0000 mgr.y (mgr.25325) 230 : cluster 0 pgmap v171: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.6 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s; 21072/126828 objects degraded (16.615%) 2023-09-23T19:18:04.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:04 smithi016 bash[77995]: cluster 2023-09-23T19:18:03.979772+0000 mon.a (mon.0) 327 : cluster 3 Health check update: Degraded data redundancy: 21072/126828 objects degraded (16.615%), 15 pgs degraded (PG_DEGRADED) 2023-09-23T19:18:04.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:04 smithi092 bash[58855]: cluster 2023-09-23T19:18:03.590408+0000 mgr.y (mgr.25325) 230 : cluster 0 pgmap v171: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.6 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s; 21072/126828 objects degraded (16.615%) 2023-09-23T19:18:04.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:04 smithi092 bash[58855]: cluster 2023-09-23T19:18:03.979772+0000 mon.a (mon.0) 327 : cluster 3 Health check update: Degraded data redundancy: 21072/126828 objects degraded (16.615%), 15 pgs degraded (PG_DEGRADED) 2023-09-23T19:18:04.952 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 27 16 49213 49197 113.861 84.9375 0.0143009 0.00877445 2023-09-23T19:18:05.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:05 smithi016 bash[75004]: audit 2023-09-23T19:18:04.948096+0000 mon.b (mon.1) 6 : audit 0 from='client.? 172.21.15.16:0/658166145' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:05.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:05 smithi016 bash[77995]: audit 2023-09-23T19:18:04.948096+0000 mon.b (mon.1) 6 : audit 0 from='client.? 172.21.15.16:0/658166145' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:05.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:05 smithi092 bash[58855]: audit 2023-09-23T19:18:04.948096+0000 mon.b (mon.1) 6 : audit 0 from='client.? 172.21.15.16:0/658166145' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:05.952 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 28 16 50614 50598 112.922 87.5625 0.0063743 0.00884747 2023-09-23T19:18:06.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:18:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:18:05] "GET /metrics HTTP/1.1" 200 36192 "" "Prometheus/2.33.4" 2023-09-23T19:18:06.534 INFO:journalctl@ceph.osd.0.smithi016.stdout:Sep 23 19:18:06 smithi016 bash[84502]: debug 2023-09-23T19:18:06.169+0000 7f72951fc640 -1 osd.0 81 log_to_monitors true 2023-09-23T19:18:06.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:06 smithi016 bash[77995]: cluster 2023-09-23T19:18:05.591335+0000 mgr.y (mgr.25325) 231 : cluster 0 pgmap v172: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 132 MiB/s wr, 2.12k op/s; 24545/147546 objects degraded (16.635%) 2023-09-23T19:18:06.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:06 smithi016 bash[75004]: cluster 2023-09-23T19:18:05.591335+0000 mgr.y (mgr.25325) 231 : cluster 0 pgmap v172: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 132 MiB/s wr, 2.12k op/s; 24545/147546 objects degraded (16.635%) 2023-09-23T19:18:06.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:06 smithi092 bash[58855]: cluster 2023-09-23T19:18:05.591335+0000 mgr.y (mgr.25325) 231 : cluster 0 pgmap v172: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 132 MiB/s wr, 2.12k op/s; 24545/147546 objects degraded (16.635%) 2023-09-23T19:18:06.952 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 29 16 52146 52130 112.33 95.75 0.00793482 0.00889449 2023-09-23T19:18:07.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:07 smithi016 bash[75004]: audit 2023-09-23T19:18:06.189228+0000 mon.a (mon.0) 328 : audit 1 from='osd.0 [v2:172.21.15.16:6802/3034120466,v1:172.21.15.16:6803/3034120466]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T19:18:07.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:07 smithi016 bash[77995]: audit 2023-09-23T19:18:06.189228+0000 mon.a (mon.0) 328 : audit 1 from='osd.0 [v2:172.21.15.16:6802/3034120466,v1:172.21.15.16:6803/3034120466]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T19:18:07.535 INFO:journalctl@ceph.osd.0.smithi016.stdout:Sep 23 19:18:07 smithi016 bash[84502]: debug 2023-09-23T19:18:07.133+0000 7f7286669700 -1 osd.0 81 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T19:18:07.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:07 smithi092 bash[58855]: audit 2023-09-23T19:18:06.189228+0000 mon.a (mon.0) 328 : audit 1 from='osd.0 [v2:172.21.15.16:6802/3034120466,v1:172.21.15.16:6803/3034120466]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T19:18:07.952 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 30 16 53571 53555 111.554 89.0625 0.00718127 0.00895735 2023-09-23T19:18:08.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:08 smithi016 bash[75004]: audit 2023-09-23T19:18:07.079733+0000 mon.a (mon.0) 329 : audit 1 from='osd.0 [v2:172.21.15.16:6802/3034120466,v1:172.21.15.16:6803/3034120466]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T19:18:08.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:08 smithi016 bash[75004]: cluster 2023-09-23T19:18:07.079891+0000 mon.a (mon.0) 330 : cluster 0 osdmap e84: 8 total, 7 up, 8 in 2023-09-23T19:18:08.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:08 smithi016 bash[75004]: cluster 2023-09-23T19:18:07.080590+0000 osd.4 (osd.4) 9 : cluster 3 failed to encode map e84 with expected crc 2023-09-23T19:18:08.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:08 smithi016 bash[75004]: audit 2023-09-23T19:18:07.082332+0000 mon.a (mon.0) 331 : audit 1 from='osd.0 [v2:172.21.15.16:6802/3034120466,v1:172.21.15.16:6803/3034120466]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-09-23T19:18:08.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:08 smithi016 bash[75004]: cluster 2023-09-23T19:18:07.101870+0000 osd.2 (osd.2) 10 : cluster 3 failed to encode map e84 with expected crc 2023-09-23T19:18:08.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:08 smithi016 bash[75004]: cluster 2023-09-23T19:18:07.108520+0000 osd.7 (osd.7) 10 : cluster 3 failed to encode map e84 with expected crc 2023-09-23T19:18:08.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:08 smithi016 bash[75004]: audit 2023-09-23T19:18:07.552147+0000 mon.b (mon.1) 7 : audit 0 from='client.? 172.21.15.16:0/2991747771' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:08.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:08 smithi016 bash[75004]: cluster 2023-09-23T19:18:07.591972+0000 mgr.y (mgr.25325) 232 : cluster 0 pgmap v174: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s; 24545/147546 objects degraded (16.635%) 2023-09-23T19:18:08.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:08 smithi016 bash[77995]: audit 2023-09-23T19:18:07.079733+0000 mon.a (mon.0) 329 : audit 1 from='osd.0 [v2:172.21.15.16:6802/3034120466,v1:172.21.15.16:6803/3034120466]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T19:18:08.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:08 smithi016 bash[77995]: cluster 2023-09-23T19:18:07.079891+0000 mon.a (mon.0) 330 : cluster 0 osdmap e84: 8 total, 7 up, 8 in 2023-09-23T19:18:08.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:08 smithi016 bash[77995]: cluster 2023-09-23T19:18:07.080590+0000 osd.4 (osd.4) 9 : cluster 3 failed to encode map e84 with expected crc 2023-09-23T19:18:08.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:08 smithi016 bash[77995]: audit 2023-09-23T19:18:07.082332+0000 mon.a (mon.0) 331 : audit 1 from='osd.0 [v2:172.21.15.16:6802/3034120466,v1:172.21.15.16:6803/3034120466]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-09-23T19:18:08.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:08 smithi016 bash[77995]: cluster 2023-09-23T19:18:07.101870+0000 osd.2 (osd.2) 10 : cluster 3 failed to encode map e84 with expected crc 2023-09-23T19:18:08.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:08 smithi016 bash[77995]: cluster 2023-09-23T19:18:07.108520+0000 osd.7 (osd.7) 10 : cluster 3 failed to encode map e84 with expected crc 2023-09-23T19:18:08.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:08 smithi016 bash[77995]: audit 2023-09-23T19:18:07.552147+0000 mon.b (mon.1) 7 : audit 0 from='client.? 172.21.15.16:0/2991747771' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:08.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:08 smithi016 bash[77995]: cluster 2023-09-23T19:18:07.591972+0000 mgr.y (mgr.25325) 232 : cluster 0 pgmap v174: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s; 24545/147546 objects degraded (16.635%) 2023-09-23T19:18:08.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:08 smithi092 bash[58855]: audit 2023-09-23T19:18:07.079733+0000 mon.a (mon.0) 329 : audit 1 from='osd.0 [v2:172.21.15.16:6802/3034120466,v1:172.21.15.16:6803/3034120466]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T19:18:08.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:08 smithi092 bash[58855]: cluster 2023-09-23T19:18:07.079891+0000 mon.a (mon.0) 330 : cluster 0 osdmap e84: 8 total, 7 up, 8 in 2023-09-23T19:18:08.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:08 smithi092 bash[58855]: cluster 2023-09-23T19:18:07.080590+0000 osd.4 (osd.4) 9 : cluster 3 failed to encode map e84 with expected crc 2023-09-23T19:18:08.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:08 smithi092 bash[58855]: audit 2023-09-23T19:18:07.082332+0000 mon.a (mon.0) 331 : audit 1 from='osd.0 [v2:172.21.15.16:6802/3034120466,v1:172.21.15.16:6803/3034120466]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-09-23T19:18:08.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:08 smithi092 bash[58855]: cluster 2023-09-23T19:18:07.101870+0000 osd.2 (osd.2) 10 : cluster 3 failed to encode map e84 with expected crc 2023-09-23T19:18:08.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:08 smithi092 bash[58855]: cluster 2023-09-23T19:18:07.108520+0000 osd.7 (osd.7) 10 : cluster 3 failed to encode map e84 with expected crc 2023-09-23T19:18:08.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:08 smithi092 bash[58855]: audit 2023-09-23T19:18:07.552147+0000 mon.b (mon.1) 7 : audit 0 from='client.? 172.21.15.16:0/2991747771' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:08.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:08 smithi092 bash[58855]: cluster 2023-09-23T19:18:07.591972+0000 mgr.y (mgr.25325) 232 : cluster 0 pgmap v174: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s; 24545/147546 objects degraded (16.635%) 2023-09-23T19:18:08.952 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 31 16 53825 53809 108.467 15.875 0.0157934 0.00896673 2023-09-23T19:18:09.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:09 smithi016 bash[75004]: cluster 2023-09-23T19:18:08.077066+0000 mon.a (mon.0) 332 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T19:18:09.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:09 smithi016 bash[75004]: cluster 2023-09-23T19:18:08.100117+0000 mon.a (mon.0) 333 : cluster 1 osd.0 [v2:172.21.15.16:6802/3034120466,v1:172.21.15.16:6803/3034120466] boot 2023-09-23T19:18:09.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:09 smithi016 bash[75004]: cluster 2023-09-23T19:18:08.100293+0000 mon.a (mon.0) 334 : cluster 0 osdmap e85: 8 total, 8 up, 8 in 2023-09-23T19:18:09.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:09 smithi016 bash[75004]: cluster 2023-09-23T19:18:08.101287+0000 osd.1 (osd.1) 12 : cluster 3 failed to encode map e85 with expected crc 2023-09-23T19:18:09.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:09 smithi016 bash[75004]: audit 2023-09-23T19:18:08.101503+0000 mon.a (mon.0) 335 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T19:18:09.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:09 smithi016 bash[75004]: cluster 2023-09-23T19:18:08.981675+0000 mon.a (mon.0) 336 : cluster 3 Health check update: Degraded data redundancy: 24545/147546 objects degraded (16.635%), 15 pgs degraded (PG_DEGRADED) 2023-09-23T19:18:09.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:09 smithi016 bash[75004]: cluster 2023-09-23T19:18:08.989995+0000 mon.a (mon.0) 337 : cluster 0 osdmap e86: 8 total, 8 up, 8 in 2023-09-23T19:18:09.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:09 smithi016 bash[77995]: cluster 2023-09-23T19:18:08.077066+0000 mon.a (mon.0) 332 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T19:18:09.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:09 smithi016 bash[77995]: cluster 2023-09-23T19:18:08.100117+0000 mon.a (mon.0) 333 : cluster 1 osd.0 [v2:172.21.15.16:6802/3034120466,v1:172.21.15.16:6803/3034120466] boot 2023-09-23T19:18:09.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:09 smithi016 bash[77995]: cluster 2023-09-23T19:18:08.100293+0000 mon.a (mon.0) 334 : cluster 0 osdmap e85: 8 total, 8 up, 8 in 2023-09-23T19:18:09.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:09 smithi016 bash[77995]: cluster 2023-09-23T19:18:08.101287+0000 osd.1 (osd.1) 12 : cluster 3 failed to encode map e85 with expected crc 2023-09-23T19:18:09.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:09 smithi016 bash[77995]: audit 2023-09-23T19:18:08.101503+0000 mon.a (mon.0) 335 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T19:18:09.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:09 smithi016 bash[77995]: cluster 2023-09-23T19:18:08.981675+0000 mon.a (mon.0) 336 : cluster 3 Health check update: Degraded data redundancy: 24545/147546 objects degraded (16.635%), 15 pgs degraded (PG_DEGRADED) 2023-09-23T19:18:09.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:09 smithi016 bash[77995]: cluster 2023-09-23T19:18:08.989995+0000 mon.a (mon.0) 337 : cluster 0 osdmap e86: 8 total, 8 up, 8 in 2023-09-23T19:18:09.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:09 smithi092 bash[58855]: cluster 2023-09-23T19:18:08.077066+0000 mon.a (mon.0) 332 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T19:18:09.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:09 smithi092 bash[58855]: cluster 2023-09-23T19:18:08.100117+0000 mon.a (mon.0) 333 : cluster 1 osd.0 [v2:172.21.15.16:6802/3034120466,v1:172.21.15.16:6803/3034120466] boot 2023-09-23T19:18:09.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:09 smithi092 bash[58855]: cluster 2023-09-23T19:18:08.100293+0000 mon.a (mon.0) 334 : cluster 0 osdmap e85: 8 total, 8 up, 8 in 2023-09-23T19:18:09.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:09 smithi092 bash[58855]: cluster 2023-09-23T19:18:08.101287+0000 osd.1 (osd.1) 12 : cluster 3 failed to encode map e85 with expected crc 2023-09-23T19:18:09.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:09 smithi092 bash[58855]: audit 2023-09-23T19:18:08.101503+0000 mon.a (mon.0) 335 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T19:18:09.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:09 smithi092 bash[58855]: cluster 2023-09-23T19:18:08.981675+0000 mon.a (mon.0) 336 : cluster 3 Health check update: Degraded data redundancy: 24545/147546 objects degraded (16.635%), 15 pgs degraded (PG_DEGRADED) 2023-09-23T19:18:09.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:09 smithi092 bash[58855]: cluster 2023-09-23T19:18:08.989995+0000 mon.a (mon.0) 337 : cluster 0 osdmap e86: 8 total, 8 up, 8 in 2023-09-23T19:18:09.952 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 32 16 53825 53809 105.078 0 - 0.00896673 2023-09-23T19:18:10.361 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:10 smithi016 bash[77995]: cluster 2023-09-23T19:18:08.989769+0000 osd.5 (osd.5) 10 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T19:18:10.362 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:10 smithi016 bash[77995]: cluster 2023-09-23T19:18:08.990414+0000 osd.2 (osd.2) 11 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T19:18:10.362 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:10 smithi016 bash[77995]: cluster 2023-09-23T19:18:08.990416+0000 osd.3 (osd.3) 11 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T19:18:10.362 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:10 smithi016 bash[77995]: cluster 2023-09-23T19:18:08.990490+0000 osd.1 (osd.1) 13 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T19:18:10.362 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:10 smithi016 bash[77995]: cluster 2023-09-23T19:18:08.993162+0000 osd.2 (osd.2) 12 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T19:18:10.362 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:10 smithi016 bash[77995]: cluster 2023-09-23T19:18:08.994128+0000 osd.7 (osd.7) 11 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T19:18:10.362 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:10 smithi016 bash[77995]: cluster 2023-09-23T19:18:08.994332+0000 osd.6 (osd.6) 9 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T19:18:10.362 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:10 smithi016 bash[77995]: cluster 2023-09-23T19:18:08.994355+0000 osd.4 (osd.4) 10 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T19:18:10.362 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:10 smithi016 bash[77995]: cluster 2023-09-23T19:18:08.995341+0000 osd.3 (osd.3) 12 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T19:18:10.362 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:10 smithi016 bash[77995]: cluster 2023-09-23T19:18:08.996327+0000 osd.1 (osd.1) 14 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T19:18:10.362 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:10 smithi016 bash[77995]: cluster 2023-09-23T19:18:08.997582+0000 osd.6 (osd.6) 10 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T19:18:10.362 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:10 smithi016 bash[77995]: cluster 2023-09-23T19:18:08.997957+0000 osd.4 (osd.4) 11 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T19:18:10.363 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:10 smithi016 bash[77995]: cluster 2023-09-23T19:18:08.999182+0000 osd.7 (osd.7) 12 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T19:18:10.363 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:10 smithi016 bash[77995]: cluster 2023-09-23T19:18:09.000901+0000 osd.4 (osd.4) 12 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T19:18:10.363 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:10 smithi016 bash[77995]: cluster 2023-09-23T19:18:09.592927+0000 mgr.y (mgr.25325) 233 : cluster 0 pgmap v177: 89 pgs: 2 remapped+peering, 12 active+undersized+degraded, 19 active+undersized, 56 active+clean; 3.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 82 MiB/s wr, 1.30k op/s; 18423/150336 objects degraded (12.255%) 2023-09-23T19:18:10.363 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:10 smithi016 bash[77995]: audit 2023-09-23T19:18:09.933482+0000 mon.c (mon.2) 11 : audit 0 from='client.? 172.21.15.16:0/3912185143' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:10.363 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:10 smithi016 bash[77995]: cluster 2023-09-23T19:18:09.999939+0000 mon.a (mon.0) 338 : cluster 0 osdmap e87: 8 total, 8 up, 8 in 2023-09-23T19:18:10.363 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:10 smithi016 bash[75004]: cluster 2023-09-23T19:18:08.989769+0000 osd.5 (osd.5) 10 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T19:18:10.363 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:10 smithi016 bash[75004]: cluster 2023-09-23T19:18:08.990414+0000 osd.2 (osd.2) 11 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T19:18:10.363 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:10 smithi016 bash[75004]: cluster 2023-09-23T19:18:08.990416+0000 osd.3 (osd.3) 11 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T19:18:10.363 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:10 smithi016 bash[75004]: cluster 2023-09-23T19:18:08.990490+0000 osd.1 (osd.1) 13 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T19:18:10.364 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:10 smithi016 bash[75004]: cluster 2023-09-23T19:18:08.993162+0000 osd.2 (osd.2) 12 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T19:18:10.364 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:10 smithi016 bash[75004]: cluster 2023-09-23T19:18:08.994128+0000 osd.7 (osd.7) 11 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T19:18:10.364 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:10 smithi016 bash[75004]: cluster 2023-09-23T19:18:08.994332+0000 osd.6 (osd.6) 9 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T19:18:10.364 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:10 smithi016 bash[75004]: cluster 2023-09-23T19:18:08.994355+0000 osd.4 (osd.4) 10 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T19:18:10.364 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:10 smithi016 bash[75004]: cluster 2023-09-23T19:18:08.995341+0000 osd.3 (osd.3) 12 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T19:18:10.364 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:10 smithi016 bash[75004]: cluster 2023-09-23T19:18:08.996327+0000 osd.1 (osd.1) 14 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T19:18:10.364 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:10 smithi016 bash[75004]: cluster 2023-09-23T19:18:08.997582+0000 osd.6 (osd.6) 10 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T19:18:10.364 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:10 smithi016 bash[75004]: cluster 2023-09-23T19:18:08.997957+0000 osd.4 (osd.4) 11 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T19:18:10.364 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:10 smithi016 bash[75004]: cluster 2023-09-23T19:18:08.999182+0000 osd.7 (osd.7) 12 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T19:18:10.364 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:10 smithi016 bash[75004]: cluster 2023-09-23T19:18:09.000901+0000 osd.4 (osd.4) 12 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T19:18:10.364 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:10 smithi016 bash[75004]: cluster 2023-09-23T19:18:09.592927+0000 mgr.y (mgr.25325) 233 : cluster 0 pgmap v177: 89 pgs: 2 remapped+peering, 12 active+undersized+degraded, 19 active+undersized, 56 active+clean; 3.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 82 MiB/s wr, 1.30k op/s; 18423/150336 objects degraded (12.255%) 2023-09-23T19:18:10.364 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:10 smithi016 bash[75004]: audit 2023-09-23T19:18:09.933482+0000 mon.c (mon.2) 11 : audit 0 from='client.? 172.21.15.16:0/3912185143' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:10.365 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:10 smithi016 bash[75004]: cluster 2023-09-23T19:18:09.999939+0000 mon.a (mon.0) 338 : cluster 0 osdmap e87: 8 total, 8 up, 8 in 2023-09-23T19:18:10.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:10 smithi092 bash[58855]: cluster 2023-09-23T19:18:08.989769+0000 osd.5 (osd.5) 10 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T19:18:10.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:10 smithi092 bash[58855]: cluster 2023-09-23T19:18:08.990414+0000 osd.2 (osd.2) 11 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T19:18:10.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:10 smithi092 bash[58855]: cluster 2023-09-23T19:18:08.990416+0000 osd.3 (osd.3) 11 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T19:18:10.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:10 smithi092 bash[58855]: cluster 2023-09-23T19:18:08.990490+0000 osd.1 (osd.1) 13 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T19:18:10.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:10 smithi092 bash[58855]: cluster 2023-09-23T19:18:08.993162+0000 osd.2 (osd.2) 12 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T19:18:10.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:10 smithi092 bash[58855]: cluster 2023-09-23T19:18:08.994128+0000 osd.7 (osd.7) 11 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T19:18:10.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:10 smithi092 bash[58855]: cluster 2023-09-23T19:18:08.994332+0000 osd.6 (osd.6) 9 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T19:18:10.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:10 smithi092 bash[58855]: cluster 2023-09-23T19:18:08.994355+0000 osd.4 (osd.4) 10 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T19:18:10.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:10 smithi092 bash[58855]: cluster 2023-09-23T19:18:08.995341+0000 osd.3 (osd.3) 12 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T19:18:10.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:10 smithi092 bash[58855]: cluster 2023-09-23T19:18:08.996327+0000 osd.1 (osd.1) 14 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T19:18:10.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:10 smithi092 bash[58855]: cluster 2023-09-23T19:18:08.997582+0000 osd.6 (osd.6) 10 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T19:18:10.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:10 smithi092 bash[58855]: cluster 2023-09-23T19:18:08.997957+0000 osd.4 (osd.4) 11 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T19:18:10.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:10 smithi092 bash[58855]: cluster 2023-09-23T19:18:08.999182+0000 osd.7 (osd.7) 12 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T19:18:10.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:10 smithi092 bash[58855]: cluster 2023-09-23T19:18:09.000901+0000 osd.4 (osd.4) 12 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T19:18:10.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:10 smithi092 bash[58855]: cluster 2023-09-23T19:18:09.592927+0000 mgr.y (mgr.25325) 233 : cluster 0 pgmap v177: 89 pgs: 2 remapped+peering, 12 active+undersized+degraded, 19 active+undersized, 56 active+clean; 3.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 82 MiB/s wr, 1.30k op/s; 18423/150336 objects degraded (12.255%) 2023-09-23T19:18:10.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:10 smithi092 bash[58855]: audit 2023-09-23T19:18:09.933482+0000 mon.c (mon.2) 11 : audit 0 from='client.? 172.21.15.16:0/3912185143' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:10.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:10 smithi092 bash[58855]: cluster 2023-09-23T19:18:09.999939+0000 mon.a (mon.0) 338 : cluster 0 osdmap e87: 8 total, 8 up, 8 in 2023-09-23T19:18:10.952 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 33 16 54274 54258 102.744 14.0312 0.00347972 0.00972157 2023-09-23T19:18:11.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:11 smithi016 bash[75004]: cluster 2023-09-23T19:18:09.995644+0000 osd.2 (osd.2) 13 : cluster 3 failed to encode map e87 with expected crc 2023-09-23T19:18:11.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:11 smithi016 bash[75004]: cluster 2023-09-23T19:18:09.999979+0000 osd.2 (osd.2) 14 : cluster 3 failed to encode map e87 with expected crc 2023-09-23T19:18:11.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:11 smithi016 bash[75004]: cluster 2023-09-23T19:18:10.000517+0000 osd.3 (osd.3) 13 : cluster 3 failed to encode map e87 with expected crc 2023-09-23T19:18:11.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:11 smithi016 bash[75004]: cluster 2023-09-23T19:18:10.000555+0000 osd.1 (osd.1) 15 : cluster 3 failed to encode map e87 with expected crc 2023-09-23T19:18:11.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:11 smithi016 bash[75004]: cluster 2023-09-23T19:18:10.004319+0000 osd.6 (osd.6) 11 : cluster 3 failed to encode map e87 with expected crc 2023-09-23T19:18:11.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:11 smithi016 bash[75004]: cluster 2023-09-23T19:18:10.004509+0000 osd.7 (osd.7) 13 : cluster 3 failed to encode map e87 with expected crc 2023-09-23T19:18:11.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:11 smithi016 bash[77995]: cluster 2023-09-23T19:18:09.995644+0000 osd.2 (osd.2) 13 : cluster 3 failed to encode map e87 with expected crc 2023-09-23T19:18:11.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:11 smithi016 bash[77995]: cluster 2023-09-23T19:18:09.999979+0000 osd.2 (osd.2) 14 : cluster 3 failed to encode map e87 with expected crc 2023-09-23T19:18:11.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:11 smithi016 bash[77995]: cluster 2023-09-23T19:18:10.000517+0000 osd.3 (osd.3) 13 : cluster 3 failed to encode map e87 with expected crc 2023-09-23T19:18:11.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:11 smithi016 bash[77995]: cluster 2023-09-23T19:18:10.000555+0000 osd.1 (osd.1) 15 : cluster 3 failed to encode map e87 with expected crc 2023-09-23T19:18:11.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:11 smithi016 bash[77995]: cluster 2023-09-23T19:18:10.004319+0000 osd.6 (osd.6) 11 : cluster 3 failed to encode map e87 with expected crc 2023-09-23T19:18:11.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:11 smithi016 bash[77995]: cluster 2023-09-23T19:18:10.004509+0000 osd.7 (osd.7) 13 : cluster 3 failed to encode map e87 with expected crc 2023-09-23T19:18:11.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:11 smithi092 bash[58855]: cluster 2023-09-23T19:18:09.995644+0000 osd.2 (osd.2) 13 : cluster 3 failed to encode map e87 with expected crc 2023-09-23T19:18:11.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:11 smithi092 bash[58855]: cluster 2023-09-23T19:18:09.999979+0000 osd.2 (osd.2) 14 : cluster 3 failed to encode map e87 with expected crc 2023-09-23T19:18:11.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:11 smithi092 bash[58855]: cluster 2023-09-23T19:18:10.000517+0000 osd.3 (osd.3) 13 : cluster 3 failed to encode map e87 with expected crc 2023-09-23T19:18:11.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:11 smithi092 bash[58855]: cluster 2023-09-23T19:18:10.000555+0000 osd.1 (osd.1) 15 : cluster 3 failed to encode map e87 with expected crc 2023-09-23T19:18:11.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:11 smithi092 bash[58855]: cluster 2023-09-23T19:18:10.004319+0000 osd.6 (osd.6) 11 : cluster 3 failed to encode map e87 with expected crc 2023-09-23T19:18:11.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:11 smithi092 bash[58855]: cluster 2023-09-23T19:18:10.004509+0000 osd.7 (osd.7) 13 : cluster 3 failed to encode map e87 with expected crc 2023-09-23T19:18:11.952 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 34 15 55107 55092 101.255 52.125 0.0105449 0.00986696 2023-09-23T19:18:12.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:12 smithi016 bash[75004]: cluster 2023-09-23T19:18:11.593565+0000 mgr.y (mgr.25325) 234 : cluster 0 pgmap v179: 89 pgs: 1 active+undersized+degraded+remapped, 5 remapped+peering, 2 activating+undersized+degraded+remapped, 2 active+undersized+degraded, 9 active+undersized, 70 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s; 6572/161514 objects degraded (4.069%) 2023-09-23T19:18:12.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:12 smithi016 bash[77995]: cluster 2023-09-23T19:18:11.593565+0000 mgr.y (mgr.25325) 234 : cluster 0 pgmap v179: 89 pgs: 1 active+undersized+degraded+remapped, 5 remapped+peering, 2 activating+undersized+degraded+remapped, 2 active+undersized+degraded, 9 active+undersized, 70 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s; 6572/161514 objects degraded (4.069%) 2023-09-23T19:18:12.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:12 smithi092 bash[58855]: cluster 2023-09-23T19:18:11.593565+0000 mgr.y (mgr.25325) 234 : cluster 0 pgmap v179: 89 pgs: 1 active+undersized+degraded+remapped, 5 remapped+peering, 2 activating+undersized+degraded+remapped, 2 active+undersized+degraded, 9 active+undersized, 70 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s; 6572/161514 objects degraded (4.069%) 2023-09-23T19:18:12.952 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 35 16 56000 55984 99.9552 55.75 0.0170881 0.00999463 2023-09-23T19:18:13.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:13 smithi016 bash[75004]: audit 2023-09-23T19:18:12.533252+0000 mon.a (mon.0) 339 : audit 0 from='client.? 172.21.15.16:0/1044083652' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:13.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:13 smithi016 bash[77995]: audit 2023-09-23T19:18:12.533252+0000 mon.a (mon.0) 339 : audit 0 from='client.? 172.21.15.16:0/1044083652' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:13.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:13 smithi092 bash[58855]: audit 2023-09-23T19:18:12.533252+0000 mon.a (mon.0) 339 : audit 0 from='client.? 172.21.15.16:0/1044083652' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:13.953 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 36 16 56857 56841 98.6664 53.5625 0.0101226 0.0101251 2023-09-23T19:18:14.953 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 37 16 57742 57726 97.4946 55.3125 0.0132512 0.0102476 2023-09-23T19:18:15.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:14 smithi016 bash[75004]: cluster 2023-09-23T19:18:13.594447+0000 mgr.y (mgr.25325) 235 : cluster 0 pgmap v180: 89 pgs: 1 active+undersized+degraded+remapped, 5 remapped+peering, 2 activating+undersized+degraded+remapped, 81 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 48 MiB/s wr, 775 op/s; 6569/161514 objects degraded (4.067%) 2023-09-23T19:18:15.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:14 smithi016 bash[75004]: audit 2023-09-23T19:18:13.624139+0000 mon.a (mon.0) 340 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:18:15.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:14 smithi016 bash[75004]: audit 2023-09-23T19:18:13.625671+0000 mon.a (mon.0) 341 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:18:15.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:14 smithi016 bash[75004]: cluster 2023-09-23T19:18:13.983366+0000 mon.a (mon.0) 342 : cluster 3 Health check update: Degraded data redundancy: 6572/161514 objects degraded (4.069%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T19:18:15.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:14 smithi016 bash[77995]: cluster 2023-09-23T19:18:13.594447+0000 mgr.y (mgr.25325) 235 : cluster 0 pgmap v180: 89 pgs: 1 active+undersized+degraded+remapped, 5 remapped+peering, 2 activating+undersized+degraded+remapped, 81 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 48 MiB/s wr, 775 op/s; 6569/161514 objects degraded (4.067%) 2023-09-23T19:18:15.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:14 smithi016 bash[77995]: audit 2023-09-23T19:18:13.624139+0000 mon.a (mon.0) 340 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:18:15.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:14 smithi016 bash[77995]: audit 2023-09-23T19:18:13.625671+0000 mon.a (mon.0) 341 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:18:15.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:14 smithi016 bash[77995]: cluster 2023-09-23T19:18:13.983366+0000 mon.a (mon.0) 342 : cluster 3 Health check update: Degraded data redundancy: 6572/161514 objects degraded (4.069%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T19:18:15.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:14 smithi092 bash[58855]: cluster 2023-09-23T19:18:13.594447+0000 mgr.y (mgr.25325) 235 : cluster 0 pgmap v180: 89 pgs: 1 active+undersized+degraded+remapped, 5 remapped+peering, 2 activating+undersized+degraded+remapped, 81 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 48 MiB/s wr, 775 op/s; 6569/161514 objects degraded (4.067%) 2023-09-23T19:18:15.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:14 smithi092 bash[58855]: audit 2023-09-23T19:18:13.624139+0000 mon.a (mon.0) 340 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:18:15.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:14 smithi092 bash[58855]: audit 2023-09-23T19:18:13.625671+0000 mon.a (mon.0) 341 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:18:15.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:14 smithi092 bash[58855]: cluster 2023-09-23T19:18:13.983366+0000 mon.a (mon.0) 342 : cluster 3 Health check update: Degraded data redundancy: 6572/161514 objects degraded (4.069%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T19:18:15.953 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 38 16 58634 58618 96.3959 55.75 0.0424889 0.010366 2023-09-23T19:18:16.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:15 smithi016 bash[77995]: audit 2023-09-23T19:18:15.168327+0000 mon.c (mon.2) 12 : audit 0 from='client.? 172.21.15.16:0/2404438479' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:16.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:15 smithi016 bash[77995]: audit 2023-09-23T19:18:15.449881+0000 mon.a (mon.0) 343 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:18:16.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:15 smithi016 bash[77995]: audit 2023-09-23T19:18:15.459061+0000 mon.a (mon.0) 344 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:18:16.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:15 smithi016 bash[77995]: audit 2023-09-23T19:18:15.460756+0000 mon.a (mon.0) 345 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:18:16.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:15 smithi016 bash[77995]: audit 2023-09-23T19:18:15.462563+0000 mon.a (mon.0) 346 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:18:16.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:15 smithi016 bash[77995]: audit 2023-09-23T19:18:15.471481+0000 mon.a (mon.0) 347 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:18:16.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:15 smithi016 bash[77995]: audit 2023-09-23T19:18:15.561345+0000 mon.a (mon.0) 348 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:18:16.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:15 smithi016 bash[77995]: audit 2023-09-23T19:18:15.565446+0000 mon.a (mon.0) 349 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:18:16.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:15 smithi016 bash[77995]: audit 2023-09-23T19:18:15.568112+0000 mon.a (mon.0) 350 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:18:16.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:15 smithi016 bash[77995]: audit 2023-09-23T19:18:15.571085+0000 mon.a (mon.0) 351 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:18:16.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:15 smithi016 bash[77995]: audit 2023-09-23T19:18:15.573960+0000 mon.a (mon.0) 352 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:18:16.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:15 smithi016 bash[75004]: audit 2023-09-23T19:18:15.168327+0000 mon.c (mon.2) 12 : audit 0 from='client.? 172.21.15.16:0/2404438479' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:16.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:15 smithi016 bash[75004]: audit 2023-09-23T19:18:15.449881+0000 mon.a (mon.0) 343 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:18:16.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:15 smithi016 bash[75004]: audit 2023-09-23T19:18:15.459061+0000 mon.a (mon.0) 344 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:18:16.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:15 smithi016 bash[75004]: audit 2023-09-23T19:18:15.460756+0000 mon.a (mon.0) 345 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:18:16.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:15 smithi016 bash[75004]: audit 2023-09-23T19:18:15.462563+0000 mon.a (mon.0) 346 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:18:16.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:15 smithi016 bash[75004]: audit 2023-09-23T19:18:15.471481+0000 mon.a (mon.0) 347 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:18:16.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:15 smithi016 bash[75004]: audit 2023-09-23T19:18:15.561345+0000 mon.a (mon.0) 348 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:18:16.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:15 smithi016 bash[75004]: audit 2023-09-23T19:18:15.565446+0000 mon.a (mon.0) 349 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:18:16.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:15 smithi016 bash[75004]: audit 2023-09-23T19:18:15.568112+0000 mon.a (mon.0) 350 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:18:16.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:15 smithi016 bash[75004]: audit 2023-09-23T19:18:15.571085+0000 mon.a (mon.0) 351 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:18:16.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:15 smithi016 bash[75004]: audit 2023-09-23T19:18:15.573960+0000 mon.a (mon.0) 352 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:18:16.037 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:18:15 smithi016 bash[70992]: debug 2023-09-23T19:18:15.569+0000 7fb2a5a7c700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:18:16.037 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:18:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:18:15] "GET /metrics HTTP/1.1" 200 36181 "" "Prometheus/2.33.4" 2023-09-23T19:18:16.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:15 smithi092 bash[58855]: audit 2023-09-23T19:18:15.168327+0000 mon.c (mon.2) 12 : audit 0 from='client.? 172.21.15.16:0/2404438479' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:16.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:15 smithi092 bash[58855]: audit 2023-09-23T19:18:15.449881+0000 mon.a (mon.0) 343 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:18:16.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:15 smithi092 bash[58855]: audit 2023-09-23T19:18:15.459061+0000 mon.a (mon.0) 344 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:18:16.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:15 smithi092 bash[58855]: audit 2023-09-23T19:18:15.460756+0000 mon.a (mon.0) 345 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:18:16.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:15 smithi092 bash[58855]: audit 2023-09-23T19:18:15.462563+0000 mon.a (mon.0) 346 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:18:16.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:15 smithi092 bash[58855]: audit 2023-09-23T19:18:15.471481+0000 mon.a (mon.0) 347 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:18:16.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:15 smithi092 bash[58855]: audit 2023-09-23T19:18:15.561345+0000 mon.a (mon.0) 348 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:18:16.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:15 smithi092 bash[58855]: audit 2023-09-23T19:18:15.565446+0000 mon.a (mon.0) 349 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:18:16.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:15 smithi092 bash[58855]: audit 2023-09-23T19:18:15.568112+0000 mon.a (mon.0) 350 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:18:16.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:15 smithi092 bash[58855]: audit 2023-09-23T19:18:15.571085+0000 mon.a (mon.0) 351 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:18:16.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:15 smithi092 bash[58855]: audit 2023-09-23T19:18:15.573960+0000 mon.a (mon.0) 352 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:18:16.953 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 39 16 59536 59520 95.3697 56.375 0.00600098 0.0104773 2023-09-23T19:18:17.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:16 smithi016 bash[75004]: audit 2023-09-23T19:18:15.574289+0000 mgr.y (mgr.25325) 236 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:18:17.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:16 smithi016 bash[75004]: cephadm 2023-09-23T19:18:15.575624+0000 mgr.y (mgr.25325) 237 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:18:17.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:16 smithi016 bash[75004]: cluster 2023-09-23T19:18:15.595907+0000 mgr.y (mgr.25325) 238 : cluster 0 pgmap v181: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 17273/173388 objects degraded (9.962%); 606 KiB/s, 9 objects/s recovering 2023-09-23T19:18:17.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:16 smithi016 bash[77995]: audit 2023-09-23T19:18:15.574289+0000 mgr.y (mgr.25325) 236 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:18:17.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:16 smithi016 bash[77995]: cephadm 2023-09-23T19:18:15.575624+0000 mgr.y (mgr.25325) 237 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:18:17.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:16 smithi016 bash[77995]: cluster 2023-09-23T19:18:15.595907+0000 mgr.y (mgr.25325) 238 : cluster 0 pgmap v181: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 17273/173388 objects degraded (9.962%); 606 KiB/s, 9 objects/s recovering 2023-09-23T19:18:17.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:16 smithi092 bash[58855]: audit 2023-09-23T19:18:15.574289+0000 mgr.y (mgr.25325) 236 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:18:17.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:16 smithi092 bash[58855]: cephadm 2023-09-23T19:18:15.575624+0000 mgr.y (mgr.25325) 237 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:18:17.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:16 smithi092 bash[58855]: cluster 2023-09-23T19:18:15.595907+0000 mgr.y (mgr.25325) 238 : cluster 0 pgmap v181: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 17273/173388 objects degraded (9.962%); 606 KiB/s, 9 objects/s recovering 2023-09-23T19:18:17.953 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-09-23T19:18:17.952183+0000 min lat: 0.00151424 max lat: 2.34811 avg lat: 0.0106012 2023-09-23T19:18:17.953 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:18:17.953 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 40 16 60352 60336 94.2604 51 0.00972604 0.0106012 2023-09-23T19:18:18.953 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 41 16 61249 61233 93.3285 56.0625 0.0327152 0.0107057 2023-09-23T19:18:19.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:18 smithi016 bash[75004]: cluster 2023-09-23T19:18:17.596525+0000 mgr.y (mgr.25325) 239 : cluster 0 pgmap v182: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 63 MiB/s wr, 1k op/s; 17273/173388 objects degraded (9.962%); 528 KiB/s, 8 objects/s recovering 2023-09-23T19:18:19.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:18 smithi016 bash[75004]: audit 2023-09-23T19:18:17.804417+0000 mon.c (mon.2) 13 : audit 0 from='client.? 172.21.15.16:0/2257087132' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:19.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:18 smithi016 bash[77995]: cluster 2023-09-23T19:18:17.596525+0000 mgr.y (mgr.25325) 239 : cluster 0 pgmap v182: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 63 MiB/s wr, 1k op/s; 17273/173388 objects degraded (9.962%); 528 KiB/s, 8 objects/s recovering 2023-09-23T19:18:19.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:18 smithi016 bash[77995]: audit 2023-09-23T19:18:17.804417+0000 mon.c (mon.2) 13 : audit 0 from='client.? 172.21.15.16:0/2257087132' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:19.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:18 smithi092 bash[58855]: cluster 2023-09-23T19:18:17.596525+0000 mgr.y (mgr.25325) 239 : cluster 0 pgmap v182: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 63 MiB/s wr, 1k op/s; 17273/173388 objects degraded (9.962%); 528 KiB/s, 8 objects/s recovering 2023-09-23T19:18:19.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:18 smithi092 bash[58855]: audit 2023-09-23T19:18:17.804417+0000 mon.c (mon.2) 13 : audit 0 from='client.? 172.21.15.16:0/2257087132' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:19.953 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 42 16 62108 62092 92.3846 53.6875 0.0203528 0.0108168 2023-09-23T19:18:20.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:19 smithi016 bash[75004]: cluster 2023-09-23T19:18:18.985023+0000 mon.a (mon.0) 353 : cluster 3 Health check update: Degraded data redundancy: 17273/173388 objects degraded (9.962%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T19:18:20.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:19 smithi016 bash[77995]: cluster 2023-09-23T19:18:18.985023+0000 mon.a (mon.0) 353 : cluster 3 Health check update: Degraded data redundancy: 17273/173388 objects degraded (9.962%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T19:18:20.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:19 smithi092 bash[58855]: cluster 2023-09-23T19:18:18.985023+0000 mon.a (mon.0) 353 : cluster 3 Health check update: Degraded data redundancy: 17273/173388 objects degraded (9.962%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T19:18:20.953 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 43 16 62947 62931 91.4555 52.4375 0.0147921 0.010925 2023-09-23T19:18:21.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:20 smithi016 bash[77995]: cluster 2023-09-23T19:18:19.597247+0000 mgr.y (mgr.25325) 240 : cluster 0 pgmap v183: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 53 MiB/s wr, 847 op/s; 17273/175758 objects degraded (9.828%); 454 KiB/s, 7 objects/s recovering 2023-09-23T19:18:21.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:20 smithi016 bash[77995]: audit 2023-09-23T19:18:20.497651+0000 mon.b (mon.1) 8 : audit 0 from='client.? 172.21.15.16:0/1043154344' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:21.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:20 smithi016 bash[75004]: cluster 2023-09-23T19:18:19.597247+0000 mgr.y (mgr.25325) 240 : cluster 0 pgmap v183: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 53 MiB/s wr, 847 op/s; 17273/175758 objects degraded (9.828%); 454 KiB/s, 7 objects/s recovering 2023-09-23T19:18:21.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:20 smithi016 bash[75004]: audit 2023-09-23T19:18:20.497651+0000 mon.b (mon.1) 8 : audit 0 from='client.? 172.21.15.16:0/1043154344' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:21.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:20 smithi092 bash[58855]: cluster 2023-09-23T19:18:19.597247+0000 mgr.y (mgr.25325) 240 : cluster 0 pgmap v183: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 53 MiB/s wr, 847 op/s; 17273/175758 objects degraded (9.828%); 454 KiB/s, 7 objects/s recovering 2023-09-23T19:18:21.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:20 smithi092 bash[58855]: audit 2023-09-23T19:18:20.497651+0000 mon.b (mon.1) 8 : audit 0 from='client.? 172.21.15.16:0/1043154344' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:21.953 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 44 16 63833 63817 90.6354 55.375 0.0111514 0.0110252 2023-09-23T19:18:22.953 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 45 16 64661 64645 89.7713 51.75 0.0164077 0.0111303 2023-09-23T19:18:23.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:22 smithi016 bash[75004]: cluster 2023-09-23T19:18:21.598034+0000 mgr.y (mgr.25325) 241 : cluster 0 pgmap v184: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 55 MiB/s wr, 879 op/s; 17207/186417 objects degraded (9.230%); 756 KiB/s, 11 objects/s recovering 2023-09-23T19:18:23.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:22 smithi016 bash[77995]: cluster 2023-09-23T19:18:21.598034+0000 mgr.y (mgr.25325) 241 : cluster 0 pgmap v184: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 55 MiB/s wr, 879 op/s; 17207/186417 objects degraded (9.230%); 756 KiB/s, 11 objects/s recovering 2023-09-23T19:18:23.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:22 smithi092 bash[58855]: cluster 2023-09-23T19:18:21.598034+0000 mgr.y (mgr.25325) 241 : cluster 0 pgmap v184: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 55 MiB/s wr, 879 op/s; 17207/186417 objects degraded (9.230%); 756 KiB/s, 11 objects/s recovering 2023-09-23T19:18:23.954 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 46 15 65503 65488 88.9651 52.6875 0.00902533 0.011232 2023-09-23T19:18:24.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:23 smithi016 bash[75004]: audit 2023-09-23T19:18:23.111520+0000 mon.a (mon.0) 354 : audit 0 from='client.? 172.21.15.16:0/1571001136' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:24.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:23 smithi016 bash[77995]: audit 2023-09-23T19:18:23.111520+0000 mon.a (mon.0) 354 : audit 0 from='client.? 172.21.15.16:0/1571001136' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:24.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:23 smithi092 bash[58855]: audit 2023-09-23T19:18:23.111520+0000 mon.a (mon.0) 354 : audit 0 from='client.? 172.21.15.16:0/1571001136' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:24.954 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 47 16 66348 66332 88.1945 52.75 0.0242642 0.0113309 2023-09-23T19:18:25.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:24 smithi016 bash[75004]: cluster 2023-09-23T19:18:23.598579+0000 mgr.y (mgr.25325) 242 : cluster 0 pgmap v185: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 43 MiB/s wr, 691 op/s; 17207/186417 objects degraded (9.230%); 730 KiB/s, 11 objects/s recovering 2023-09-23T19:18:25.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:24 smithi016 bash[75004]: cluster 2023-09-23T19:18:23.986802+0000 mon.a (mon.0) 355 : cluster 3 Health check update: Degraded data redundancy: 17207/186417 objects degraded (9.230%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T19:18:25.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:24 smithi016 bash[77995]: cluster 2023-09-23T19:18:23.598579+0000 mgr.y (mgr.25325) 242 : cluster 0 pgmap v185: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 43 MiB/s wr, 691 op/s; 17207/186417 objects degraded (9.230%); 730 KiB/s, 11 objects/s recovering 2023-09-23T19:18:25.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:24 smithi016 bash[77995]: cluster 2023-09-23T19:18:23.986802+0000 mon.a (mon.0) 355 : cluster 3 Health check update: Degraded data redundancy: 17207/186417 objects degraded (9.230%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T19:18:25.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:24 smithi092 bash[58855]: cluster 2023-09-23T19:18:23.598579+0000 mgr.y (mgr.25325) 242 : cluster 0 pgmap v185: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 43 MiB/s wr, 691 op/s; 17207/186417 objects degraded (9.230%); 730 KiB/s, 11 objects/s recovering 2023-09-23T19:18:25.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:24 smithi092 bash[58855]: cluster 2023-09-23T19:18:23.986802+0000 mon.a (mon.0) 355 : cluster 3 Health check update: Degraded data redundancy: 17207/186417 objects degraded (9.230%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T19:18:25.954 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 48 16 67132 67116 87.3779 49 0.0278249 0.0114368 2023-09-23T19:18:26.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:18:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:18:25] "GET /metrics HTTP/1.1" 200 36181 "" "Prometheus/2.33.4" 2023-09-23T19:18:26.954 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 49 16 67998 67982 86.6991 54.125 0.0148379 0.0115263 2023-09-23T19:18:27.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:26 smithi016 bash[75004]: cluster 2023-09-23T19:18:25.599473+0000 mgr.y (mgr.25325) 243 : cluster 0 pgmap v186: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 17137/199164 objects degraded (8.604%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:18:27.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:26 smithi016 bash[75004]: audit 2023-09-23T19:18:25.753369+0000 mon.c (mon.2) 14 : audit 0 from='client.? 172.21.15.16:0/2730627511' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:27.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:26 smithi016 bash[77995]: cluster 2023-09-23T19:18:25.599473+0000 mgr.y (mgr.25325) 243 : cluster 0 pgmap v186: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 17137/199164 objects degraded (8.604%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:18:27.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:26 smithi016 bash[77995]: audit 2023-09-23T19:18:25.753369+0000 mon.c (mon.2) 14 : audit 0 from='client.? 172.21.15.16:0/2730627511' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:27.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:26 smithi092 bash[58855]: cluster 2023-09-23T19:18:25.599473+0000 mgr.y (mgr.25325) 243 : cluster 0 pgmap v186: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 17137/199164 objects degraded (8.604%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:18:27.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:26 smithi092 bash[58855]: audit 2023-09-23T19:18:25.753369+0000 mon.c (mon.2) 14 : audit 0 from='client.? 172.21.15.16:0/2730627511' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:27.954 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 50 16 68813 68797 85.9838 50.9375 0.0465038 0.0116211 2023-09-23T19:18:28.954 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 51 16 69598 69582 85.2598 49.0625 0.0365749 0.0117209 2023-09-23T19:18:29.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:28 smithi016 bash[77995]: cluster 2023-09-23T19:18:27.600102+0000 mgr.y (mgr.25325) 244 : cluster 0 pgmap v187: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 45 MiB/s wr, 715 op/s; 17137/199164 objects degraded (8.604%); 725 KiB/s, 11 objects/s recovering 2023-09-23T19:18:29.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:28 smithi016 bash[77995]: audit 2023-09-23T19:18:28.370952+0000 mon.c (mon.2) 15 : audit 0 from='client.? 172.21.15.16:0/529546934' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:29.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:28 smithi016 bash[77995]: audit 2023-09-23T19:18:28.617912+0000 mon.a (mon.0) 356 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:18:29.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:28 smithi016 bash[75004]: cluster 2023-09-23T19:18:27.600102+0000 mgr.y (mgr.25325) 244 : cluster 0 pgmap v187: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 45 MiB/s wr, 715 op/s; 17137/199164 objects degraded (8.604%); 725 KiB/s, 11 objects/s recovering 2023-09-23T19:18:29.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:28 smithi016 bash[75004]: audit 2023-09-23T19:18:28.370952+0000 mon.c (mon.2) 15 : audit 0 from='client.? 172.21.15.16:0/529546934' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:29.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:28 smithi016 bash[75004]: audit 2023-09-23T19:18:28.617912+0000 mon.a (mon.0) 356 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:18:29.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:28 smithi092 bash[58855]: cluster 2023-09-23T19:18:27.600102+0000 mgr.y (mgr.25325) 244 : cluster 0 pgmap v187: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 45 MiB/s wr, 715 op/s; 17137/199164 objects degraded (8.604%); 725 KiB/s, 11 objects/s recovering 2023-09-23T19:18:29.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:28 smithi092 bash[58855]: audit 2023-09-23T19:18:28.370952+0000 mon.c (mon.2) 15 : audit 0 from='client.? 172.21.15.16:0/529546934' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:29.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:28 smithi092 bash[58855]: audit 2023-09-23T19:18:28.617912+0000 mon.a (mon.0) 356 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:18:29.954 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 52 16 70426 70410 84.6154 51.75 0.0119758 0.0118105 2023-09-23T19:18:30.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:29 smithi016 bash[77995]: cluster 2023-09-23T19:18:28.987968+0000 mon.a (mon.0) 357 : cluster 3 Health check update: Degraded data redundancy: 17137/199164 objects degraded (8.604%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T19:18:30.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:29 smithi016 bash[75004]: cluster 2023-09-23T19:18:28.987968+0000 mon.a (mon.0) 357 : cluster 3 Health check update: Degraded data redundancy: 17137/199164 objects degraded (8.604%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T19:18:30.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:29 smithi092 bash[58855]: cluster 2023-09-23T19:18:28.987968+0000 mon.a (mon.0) 357 : cluster 3 Health check update: Degraded data redundancy: 17137/199164 objects degraded (8.604%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T19:18:30.954 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 53 16 71201 71185 83.9327 48.4375 0.00516144 0.0119048 2023-09-23T19:18:31.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:30 smithi016 bash[77995]: cluster 2023-09-23T19:18:29.600785+0000 mgr.y (mgr.25325) 245 : cluster 0 pgmap v188: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 49 MiB/s wr, 782 op/s; 17137/201564 objects degraded (8.502%); 725 KiB/s, 11 objects/s recovering 2023-09-23T19:18:31.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:30 smithi016 bash[77995]: audit 2023-09-23T19:18:30.590591+0000 mon.a (mon.0) 358 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:18:31.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:30 smithi016 bash[75004]: cluster 2023-09-23T19:18:29.600785+0000 mgr.y (mgr.25325) 245 : cluster 0 pgmap v188: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 49 MiB/s wr, 782 op/s; 17137/201564 objects degraded (8.502%); 725 KiB/s, 11 objects/s recovering 2023-09-23T19:18:31.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:30 smithi016 bash[75004]: audit 2023-09-23T19:18:30.590591+0000 mon.a (mon.0) 358 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:18:31.035 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:18:30 smithi016 bash[70992]: debug 2023-09-23T19:18:30.585+0000 7fb2a5a7c700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:18:31.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:30 smithi092 bash[58855]: cluster 2023-09-23T19:18:29.600785+0000 mgr.y (mgr.25325) 245 : cluster 0 pgmap v188: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 49 MiB/s wr, 782 op/s; 17137/201564 objects degraded (8.502%); 725 KiB/s, 11 objects/s recovering 2023-09-23T19:18:31.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:30 smithi092 bash[58855]: audit 2023-09-23T19:18:30.590591+0000 mon.a (mon.0) 358 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:18:31.954 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 54 16 72051 72035 83.3621 53.125 0.0203521 0.0119872 2023-09-23T19:18:32.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:31 smithi016 bash[75004]: audit 2023-09-23T19:18:30.590900+0000 mgr.y (mgr.25325) 246 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:18:32.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:31 smithi016 bash[75004]: cephadm 2023-09-23T19:18:30.592255+0000 mgr.y (mgr.25325) 247 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:18:32.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:31 smithi016 bash[75004]: audit 2023-09-23T19:18:30.994229+0000 mon.a (mon.0) 359 : audit 0 from='client.? 172.21.15.16:0/1736400332' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:32.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:31 smithi016 bash[77995]: audit 2023-09-23T19:18:30.590900+0000 mgr.y (mgr.25325) 246 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:18:32.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:31 smithi016 bash[77995]: cephadm 2023-09-23T19:18:30.592255+0000 mgr.y (mgr.25325) 247 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:18:32.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:31 smithi016 bash[77995]: audit 2023-09-23T19:18:30.994229+0000 mon.a (mon.0) 359 : audit 0 from='client.? 172.21.15.16:0/1736400332' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:32.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:31 smithi092 bash[58855]: audit 2023-09-23T19:18:30.590900+0000 mgr.y (mgr.25325) 246 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:18:32.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:31 smithi092 bash[58855]: cephadm 2023-09-23T19:18:30.592255+0000 mgr.y (mgr.25325) 247 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:18:32.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:31 smithi092 bash[58855]: audit 2023-09-23T19:18:30.994229+0000 mon.a (mon.0) 359 : audit 0 from='client.? 172.21.15.16:0/1736400332' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:32.954 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 55 16 72878 72862 82.7862 51.6875 0.0282015 0.0120719 2023-09-23T19:18:33.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:32 smithi016 bash[75004]: cluster 2023-09-23T19:18:31.601565+0000 mgr.y (mgr.25325) 248 : cluster 0 pgmap v189: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 62 MiB/s wr, 989 op/s; 17068/211395 objects degraded (8.074%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:18:33.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:32 smithi016 bash[77995]: cluster 2023-09-23T19:18:31.601565+0000 mgr.y (mgr.25325) 248 : cluster 0 pgmap v189: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 62 MiB/s wr, 989 op/s; 17068/211395 objects degraded (8.074%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:18:33.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:32 smithi092 bash[58855]: cluster 2023-09-23T19:18:31.601565+0000 mgr.y (mgr.25325) 248 : cluster 0 pgmap v189: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 62 MiB/s wr, 989 op/s; 17068/211395 objects degraded (8.074%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:18:33.954 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 56 16 73636 73620 82.1538 47.375 0.013834 0.0121647 2023-09-23T19:18:34.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:33 smithi016 bash[75004]: audit 2023-09-23T19:18:33.656791+0000 mon.a (mon.0) 360 : audit 0 from='client.? 172.21.15.16:0/2320220971' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:34.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:33 smithi016 bash[77995]: audit 2023-09-23T19:18:33.656791+0000 mon.a (mon.0) 360 : audit 0 from='client.? 172.21.15.16:0/2320220971' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:34.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:33 smithi092 bash[58855]: audit 2023-09-23T19:18:33.656791+0000 mon.a (mon.0) 360 : audit 0 from='client.? 172.21.15.16:0/2320220971' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:34.955 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 57 16 74435 74419 81.5885 49.9375 0.0213264 0.0122482 2023-09-23T19:18:35.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:34 smithi016 bash[77995]: cluster 2023-09-23T19:18:33.602148+0000 mgr.y (mgr.25325) 249 : cluster 0 pgmap v190: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 43 MiB/s wr, 693 op/s; 17068/211395 objects degraded (8.074%); 741 KiB/s, 11 objects/s recovering 2023-09-23T19:18:35.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:34 smithi016 bash[77995]: cluster 2023-09-23T19:18:33.989180+0000 mon.a (mon.0) 361 : cluster 3 Health check update: Degraded data redundancy: 17068/211395 objects degraded (8.074%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T19:18:35.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:34 smithi016 bash[75004]: cluster 2023-09-23T19:18:33.602148+0000 mgr.y (mgr.25325) 249 : cluster 0 pgmap v190: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 43 MiB/s wr, 693 op/s; 17068/211395 objects degraded (8.074%); 741 KiB/s, 11 objects/s recovering 2023-09-23T19:18:35.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:34 smithi016 bash[75004]: cluster 2023-09-23T19:18:33.989180+0000 mon.a (mon.0) 361 : cluster 3 Health check update: Degraded data redundancy: 17068/211395 objects degraded (8.074%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T19:18:35.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:34 smithi092 bash[58855]: cluster 2023-09-23T19:18:33.602148+0000 mgr.y (mgr.25325) 249 : cluster 0 pgmap v190: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 43 MiB/s wr, 693 op/s; 17068/211395 objects degraded (8.074%); 741 KiB/s, 11 objects/s recovering 2023-09-23T19:18:35.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:34 smithi092 bash[58855]: cluster 2023-09-23T19:18:33.989180+0000 mon.a (mon.0) 361 : cluster 3 Health check update: Degraded data redundancy: 17068/211395 objects degraded (8.074%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T19:18:35.955 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 58 16 75214 75198 81.0212 48.6875 0.0192817 0.0123353 2023-09-23T19:18:36.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:18:35 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:18:35] "GET /metrics HTTP/1.1" 200 36202 "" "Prometheus/2.33.4" 2023-09-23T19:18:36.955 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 59 16 75978 75962 80.4572 47.75 0.0152091 0.0124198 2023-09-23T19:18:37.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:36 smithi016 bash[75004]: cluster 2023-09-23T19:18:35.603034+0000 mgr.y (mgr.25325) 250 : cluster 0 pgmap v191: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s; 17001/223398 objects degraded (7.610%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:18:37.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:36 smithi016 bash[75004]: audit 2023-09-23T19:18:36.303358+0000 mon.c (mon.2) 16 : audit 0 from='client.? 172.21.15.16:0/3927061419' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:37.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:36 smithi016 bash[77995]: cluster 2023-09-23T19:18:35.603034+0000 mgr.y (mgr.25325) 250 : cluster 0 pgmap v191: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s; 17001/223398 objects degraded (7.610%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:18:37.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:36 smithi016 bash[77995]: audit 2023-09-23T19:18:36.303358+0000 mon.c (mon.2) 16 : audit 0 from='client.? 172.21.15.16:0/3927061419' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:37.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:36 smithi092 bash[58855]: cluster 2023-09-23T19:18:35.603034+0000 mgr.y (mgr.25325) 250 : cluster 0 pgmap v191: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s; 17001/223398 objects degraded (7.610%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:18:37.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:36 smithi092 bash[58855]: audit 2023-09-23T19:18:36.303358+0000 mon.c (mon.2) 16 : audit 0 from='client.? 172.21.15.16:0/3927061419' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:37.955 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-09-23T19:18:37.954155+0000 min lat: 0.00151424 max lat: 2.34811 avg lat: 0.0124932 2023-09-23T19:18:37.955 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:18:37.955 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 60 16 76819 76803 79.9922 52.5625 0.0277267 0.0124932 2023-09-23T19:18:38.955 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 61 16 77581 77565 79.4615 47.625 0.00484886 0.012576 2023-09-23T19:18:39.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:38 smithi016 bash[77995]: cluster 2023-09-23T19:18:37.603821+0000 mgr.y (mgr.25325) 251 : cluster 0 pgmap v192: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 42 MiB/s wr, 672 op/s; 17001/223398 objects degraded (7.610%); 725 KiB/s, 11 objects/s recovering 2023-09-23T19:18:39.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:38 smithi016 bash[75004]: cluster 2023-09-23T19:18:37.603821+0000 mgr.y (mgr.25325) 251 : cluster 0 pgmap v192: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 42 MiB/s wr, 672 op/s; 17001/223398 objects degraded (7.610%); 725 KiB/s, 11 objects/s recovering 2023-09-23T19:18:39.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:38 smithi092 bash[58855]: cluster 2023-09-23T19:18:37.603821+0000 mgr.y (mgr.25325) 251 : cluster 0 pgmap v192: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 42 MiB/s wr, 672 op/s; 17001/223398 objects degraded (7.610%); 725 KiB/s, 11 objects/s recovering 2023-09-23T19:18:39.955 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 62 16 78398 78382 79.0034 51.0625 0.0321081 0.0126496 2023-09-23T19:18:40.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:39 smithi016 bash[75004]: audit 2023-09-23T19:18:38.910262+0000 mon.a (mon.0) 362 : audit 0 from='client.? 172.21.15.16:0/1830652567' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:40.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:39 smithi016 bash[75004]: cluster 2023-09-23T19:18:38.990397+0000 mon.a (mon.0) 363 : cluster 3 Health check update: Degraded data redundancy: 17001/223398 objects degraded (7.610%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T19:18:40.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:39 smithi016 bash[77995]: audit 2023-09-23T19:18:38.910262+0000 mon.a (mon.0) 362 : audit 0 from='client.? 172.21.15.16:0/1830652567' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:40.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:39 smithi016 bash[77995]: cluster 2023-09-23T19:18:38.990397+0000 mon.a (mon.0) 363 : cluster 3 Health check update: Degraded data redundancy: 17001/223398 objects degraded (7.610%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T19:18:40.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:39 smithi092 bash[58855]: audit 2023-09-23T19:18:38.910262+0000 mon.a (mon.0) 362 : audit 0 from='client.? 172.21.15.16:0/1830652567' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:40.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:39 smithi092 bash[58855]: cluster 2023-09-23T19:18:38.990397+0000 mon.a (mon.0) 363 : cluster 3 Health check update: Degraded data redundancy: 17001/223398 objects degraded (7.610%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T19:18:40.955 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 63 16 79187 79171 78.5321 49.3125 0.037291 0.0127259 2023-09-23T19:18:41.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:40 smithi016 bash[77995]: cluster 2023-09-23T19:18:39.604558+0000 mgr.y (mgr.25325) 252 : cluster 0 pgmap v193: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 46 MiB/s wr, 739 op/s; 17001/225792 objects degraded (7.529%); 725 KiB/s, 11 objects/s recovering 2023-09-23T19:18:41.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:40 smithi016 bash[75004]: cluster 2023-09-23T19:18:39.604558+0000 mgr.y (mgr.25325) 252 : cluster 0 pgmap v193: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 46 MiB/s wr, 739 op/s; 17001/225792 objects degraded (7.529%); 725 KiB/s, 11 objects/s recovering 2023-09-23T19:18:41.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:40 smithi092 bash[58855]: cluster 2023-09-23T19:18:39.604558+0000 mgr.y (mgr.25325) 252 : cluster 0 pgmap v193: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 46 MiB/s wr, 739 op/s; 17001/225792 objects degraded (7.529%); 725 KiB/s, 11 objects/s recovering 2023-09-23T19:18:41.955 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 64 16 79901 79885 78.0022 44.625 0.0554859 0.0128122 2023-09-23T19:18:42.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:41 smithi016 bash[77995]: audit 2023-09-23T19:18:41.545127+0000 mon.a (mon.0) 364 : audit 0 from='client.? 172.21.15.16:0/2992607799' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:42.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:41 smithi016 bash[77995]: cluster 2023-09-23T19:18:41.605373+0000 mgr.y (mgr.25325) 253 : cluster 0 pgmap v194: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 59 MiB/s wr, 939 op/s; 16930/235386 objects degraded (7.192%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:18:42.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:41 smithi016 bash[75004]: audit 2023-09-23T19:18:41.545127+0000 mon.a (mon.0) 364 : audit 0 from='client.? 172.21.15.16:0/2992607799' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:42.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:41 smithi016 bash[75004]: cluster 2023-09-23T19:18:41.605373+0000 mgr.y (mgr.25325) 253 : cluster 0 pgmap v194: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 59 MiB/s wr, 939 op/s; 16930/235386 objects degraded (7.192%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:18:42.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:41 smithi092 bash[58855]: audit 2023-09-23T19:18:41.545127+0000 mon.a (mon.0) 364 : audit 0 from='client.? 172.21.15.16:0/2992607799' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:42.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:41 smithi092 bash[58855]: cluster 2023-09-23T19:18:41.605373+0000 mgr.y (mgr.25325) 253 : cluster 0 pgmap v194: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 59 MiB/s wr, 939 op/s; 16930/235386 objects degraded (7.192%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:18:42.955 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 65 16 80679 80663 77.5502 48.625 0.00691225 0.0128876 2023-09-23T19:18:43.955 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 66 16 81410 81394 77.0674 45.6875 0.0358459 0.0129671 2023-09-23T19:18:44.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:43 smithi016 bash[77995]: audit 2023-09-23T19:18:43.620732+0000 mon.a (mon.0) 365 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:18:44.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:43 smithi016 bash[75004]: audit 2023-09-23T19:18:43.620732+0000 mon.a (mon.0) 365 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:18:44.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:43 smithi092 bash[58855]: audit 2023-09-23T19:18:43.620732+0000 mon.a (mon.0) 365 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:18:44.956 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 67 16 82150 82134 76.6074 46.25 0.00674376 0.0130452 2023-09-23T19:18:45.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:44 smithi016 bash[75004]: cluster 2023-09-23T19:18:43.606049+0000 mgr.y (mgr.25325) 254 : cluster 0 pgmap v195: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 42 MiB/s wr, 666 op/s; 16930/235386 objects degraded (7.192%); 736 KiB/s, 11 objects/s recovering 2023-09-23T19:18:45.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:44 smithi016 bash[75004]: cluster 2023-09-23T19:18:43.991536+0000 mon.a (mon.0) 366 : cluster 3 Health check update: Degraded data redundancy: 16930/235386 objects degraded (7.192%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T19:18:45.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:44 smithi016 bash[75004]: audit 2023-09-23T19:18:44.199254+0000 mon.b (mon.1) 9 : audit 0 from='client.? 172.21.15.16:0/4041089090' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:45.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:44 smithi016 bash[77995]: cluster 2023-09-23T19:18:43.606049+0000 mgr.y (mgr.25325) 254 : cluster 0 pgmap v195: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 42 MiB/s wr, 666 op/s; 16930/235386 objects degraded (7.192%); 736 KiB/s, 11 objects/s recovering 2023-09-23T19:18:45.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:44 smithi016 bash[77995]: cluster 2023-09-23T19:18:43.991536+0000 mon.a (mon.0) 366 : cluster 3 Health check update: Degraded data redundancy: 16930/235386 objects degraded (7.192%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T19:18:45.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:44 smithi016 bash[77995]: audit 2023-09-23T19:18:44.199254+0000 mon.b (mon.1) 9 : audit 0 from='client.? 172.21.15.16:0/4041089090' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:45.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:44 smithi092 bash[58855]: cluster 2023-09-23T19:18:43.606049+0000 mgr.y (mgr.25325) 254 : cluster 0 pgmap v195: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 42 MiB/s wr, 666 op/s; 16930/235386 objects degraded (7.192%); 736 KiB/s, 11 objects/s recovering 2023-09-23T19:18:45.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:44 smithi092 bash[58855]: cluster 2023-09-23T19:18:43.991536+0000 mon.a (mon.0) 366 : cluster 3 Health check update: Degraded data redundancy: 16930/235386 objects degraded (7.192%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T19:18:45.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:44 smithi092 bash[58855]: audit 2023-09-23T19:18:44.199254+0000 mon.b (mon.1) 9 : audit 0 from='client.? 172.21.15.16:0/4041089090' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:45.956 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 68 16 82904 82888 76.1738 47.125 0.0144236 0.0131196 2023-09-23T19:18:46.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:45 smithi016 bash[75004]: audit 2023-09-23T19:18:45.605566+0000 mon.a (mon.0) 367 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:18:46.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:45 smithi016 bash[77995]: audit 2023-09-23T19:18:45.605566+0000 mon.a (mon.0) 367 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:18:46.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:18:45 smithi016 bash[70992]: debug 2023-09-23T19:18:45.601+0000 7fb2a5a7c700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:18:46.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:18:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:18:45] "GET /metrics HTTP/1.1" 200 36201 "" "Prometheus/2.33.4" 2023-09-23T19:18:46.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:45 smithi092 bash[58855]: audit 2023-09-23T19:18:45.605566+0000 mon.a (mon.0) 367 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:18:46.956 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 69 16 83594 83578 75.6947 43.125 0.0238517 0.0132033 2023-09-23T19:18:47.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:46 smithi016 bash[75004]: cluster 2023-09-23T19:18:45.606876+0000 mgr.y (mgr.25325) 255 : cluster 0 pgmap v196: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 61 MiB/s wr, 977 op/s; 16861/246591 objects degraded (6.838%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:18:47.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:46 smithi016 bash[75004]: audit 2023-09-23T19:18:45.607109+0000 mgr.y (mgr.25325) 256 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:18:47.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:46 smithi016 bash[75004]: cephadm 2023-09-23T19:18:45.610420+0000 mgr.y (mgr.25325) 257 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:18:47.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:46 smithi016 bash[77995]: cluster 2023-09-23T19:18:45.606876+0000 mgr.y (mgr.25325) 255 : cluster 0 pgmap v196: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 61 MiB/s wr, 977 op/s; 16861/246591 objects degraded (6.838%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:18:47.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:46 smithi016 bash[77995]: audit 2023-09-23T19:18:45.607109+0000 mgr.y (mgr.25325) 256 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:18:47.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:46 smithi016 bash[77995]: cephadm 2023-09-23T19:18:45.610420+0000 mgr.y (mgr.25325) 257 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:18:47.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:46 smithi092 bash[58855]: cluster 2023-09-23T19:18:45.606876+0000 mgr.y (mgr.25325) 255 : cluster 0 pgmap v196: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 61 MiB/s wr, 977 op/s; 16861/246591 objects degraded (6.838%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:18:47.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:46 smithi092 bash[58855]: audit 2023-09-23T19:18:45.607109+0000 mgr.y (mgr.25325) 256 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:18:47.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:46 smithi092 bash[58855]: cephadm 2023-09-23T19:18:45.610420+0000 mgr.y (mgr.25325) 257 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:18:47.956 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 70 16 84351 84335 75.2892 47.3125 0.0180937 0.0132741 2023-09-23T19:18:48.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:47 smithi016 bash[75004]: audit 2023-09-23T19:18:46.841919+0000 mon.c (mon.2) 17 : audit 0 from='client.? 172.21.15.16:0/322100780' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:48.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:47 smithi016 bash[77995]: audit 2023-09-23T19:18:46.841919+0000 mon.c (mon.2) 17 : audit 0 from='client.? 172.21.15.16:0/322100780' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:48.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:47 smithi092 bash[58855]: audit 2023-09-23T19:18:46.841919+0000 mon.c (mon.2) 17 : audit 0 from='client.? 172.21.15.16:0/322100780' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:48.956 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 71 16 85088 85072 74.8775 46.0625 0.0174919 0.0133472 2023-09-23T19:18:49.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:48 smithi016 bash[75004]: cluster 2023-09-23T19:18:47.607414+0000 mgr.y (mgr.25325) 258 : cluster 0 pgmap v197: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 40 MiB/s wr, 643 op/s; 16861/246591 objects degraded (6.838%); 746 KiB/s, 11 objects/s recovering 2023-09-23T19:18:49.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:48 smithi016 bash[77995]: cluster 2023-09-23T19:18:47.607414+0000 mgr.y (mgr.25325) 258 : cluster 0 pgmap v197: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 40 MiB/s wr, 643 op/s; 16861/246591 objects degraded (6.838%); 746 KiB/s, 11 objects/s recovering 2023-09-23T19:18:49.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:48 smithi092 bash[58855]: cluster 2023-09-23T19:18:47.607414+0000 mgr.y (mgr.25325) 258 : cluster 0 pgmap v197: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 40 MiB/s wr, 643 op/s; 16861/246591 objects degraded (6.838%); 746 KiB/s, 11 objects/s recovering 2023-09-23T19:18:49.956 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 72 16 85802 85786 74.4573 44.625 0.0124849 0.0134231 2023-09-23T19:18:50.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:49 smithi016 bash[77995]: cluster 2023-09-23T19:18:48.995162+0000 mon.a (mon.0) 368 : cluster 3 Health check update: Degraded data redundancy: 16861/246591 objects degraded (6.838%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T19:18:50.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:49 smithi016 bash[77995]: audit 2023-09-23T19:18:49.518306+0000 mon.a (mon.0) 369 : audit 0 from='client.? 172.21.15.16:0/4284575752' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:50.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:49 smithi016 bash[75004]: cluster 2023-09-23T19:18:48.995162+0000 mon.a (mon.0) 368 : cluster 3 Health check update: Degraded data redundancy: 16861/246591 objects degraded (6.838%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T19:18:50.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:49 smithi016 bash[75004]: audit 2023-09-23T19:18:49.518306+0000 mon.a (mon.0) 369 : audit 0 from='client.? 172.21.15.16:0/4284575752' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:50.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:49 smithi092 bash[58855]: cluster 2023-09-23T19:18:48.995162+0000 mon.a (mon.0) 368 : cluster 3 Health check update: Degraded data redundancy: 16861/246591 objects degraded (6.838%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T19:18:50.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:49 smithi092 bash[58855]: audit 2023-09-23T19:18:49.518306+0000 mon.a (mon.0) 369 : audit 0 from='client.? 172.21.15.16:0/4284575752' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:50.956 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 73 16 86580 86564 74.1034 48.625 0.0206442 0.0134868 2023-09-23T19:18:51.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:50 smithi016 bash[75004]: cluster 2023-09-23T19:18:49.607929+0000 mgr.y (mgr.25325) 259 : cluster 0 pgmap v198: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 44 MiB/s wr, 701 op/s; 16861/248667 objects degraded (6.781%); 746 KiB/s, 11 objects/s recovering 2023-09-23T19:18:51.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:50 smithi016 bash[77995]: cluster 2023-09-23T19:18:49.607929+0000 mgr.y (mgr.25325) 259 : cluster 0 pgmap v198: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 44 MiB/s wr, 701 op/s; 16861/248667 objects degraded (6.781%); 746 KiB/s, 11 objects/s recovering 2023-09-23T19:18:51.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:50 smithi092 bash[58855]: cluster 2023-09-23T19:18:49.607929+0000 mgr.y (mgr.25325) 259 : cluster 0 pgmap v198: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 44 MiB/s wr, 701 op/s; 16861/248667 objects degraded (6.781%); 746 KiB/s, 11 objects/s recovering 2023-09-23T19:18:51.956 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 74 16 87267 87251 73.6822 42.9375 0.0149463 0.0135635 2023-09-23T19:18:52.956 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 75 16 87957 87941 73.2747 43.125 0.0497433 0.0136392 2023-09-23T19:18:53.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:52 smithi016 bash[75004]: cluster 2023-09-23T19:18:51.608751+0000 mgr.y (mgr.25325) 260 : cluster 0 pgmap v199: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 55 MiB/s wr, 880 op/s; 16794/257508 objects degraded (6.522%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:18:53.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:52 smithi016 bash[75004]: audit 2023-09-23T19:18:52.150149+0000 mon.b (mon.1) 10 : audit 0 from='client.? 172.21.15.16:0/3890543721' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:53.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:52 smithi016 bash[77995]: cluster 2023-09-23T19:18:51.608751+0000 mgr.y (mgr.25325) 260 : cluster 0 pgmap v199: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 55 MiB/s wr, 880 op/s; 16794/257508 objects degraded (6.522%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:18:53.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:52 smithi016 bash[77995]: audit 2023-09-23T19:18:52.150149+0000 mon.b (mon.1) 10 : audit 0 from='client.? 172.21.15.16:0/3890543721' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:53.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:52 smithi092 bash[58855]: cluster 2023-09-23T19:18:51.608751+0000 mgr.y (mgr.25325) 260 : cluster 0 pgmap v199: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 55 MiB/s wr, 880 op/s; 16794/257508 objects degraded (6.522%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:18:53.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:52 smithi092 bash[58855]: audit 2023-09-23T19:18:52.150149+0000 mon.b (mon.1) 10 : audit 0 from='client.? 172.21.15.16:0/3890543721' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:53.956 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 76 16 88690 88674 72.9133 45.8125 0.0157056 0.0137067 2023-09-23T19:18:54.957 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 77 16 89368 89352 72.5167 42.375 0.0175544 0.0137824 2023-09-23T19:18:55.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:54 smithi016 bash[75004]: cluster 2023-09-23T19:18:53.609552+0000 mgr.y (mgr.25325) 261 : cluster 0 pgmap v200: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 38 MiB/s wr, 614 op/s; 16794/257508 objects degraded (6.522%); 725 KiB/s, 11 objects/s recovering 2023-09-23T19:18:55.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:54 smithi016 bash[75004]: cluster 2023-09-23T19:18:53.996499+0000 mon.a (mon.0) 370 : cluster 3 Health check update: Degraded data redundancy: 16794/257508 objects degraded (6.522%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T19:18:55.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:54 smithi016 bash[77995]: cluster 2023-09-23T19:18:53.609552+0000 mgr.y (mgr.25325) 261 : cluster 0 pgmap v200: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 38 MiB/s wr, 614 op/s; 16794/257508 objects degraded (6.522%); 725 KiB/s, 11 objects/s recovering 2023-09-23T19:18:55.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:54 smithi016 bash[77995]: cluster 2023-09-23T19:18:53.996499+0000 mon.a (mon.0) 370 : cluster 3 Health check update: Degraded data redundancy: 16794/257508 objects degraded (6.522%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T19:18:55.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:54 smithi092 bash[58855]: cluster 2023-09-23T19:18:53.609552+0000 mgr.y (mgr.25325) 261 : cluster 0 pgmap v200: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 38 MiB/s wr, 614 op/s; 16794/257508 objects degraded (6.522%); 725 KiB/s, 11 objects/s recovering 2023-09-23T19:18:55.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:54 smithi092 bash[58855]: cluster 2023-09-23T19:18:53.996499+0000 mon.a (mon.0) 370 : cluster 3 Health check update: Degraded data redundancy: 16794/257508 objects degraded (6.522%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T19:18:55.957 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 78 16 90104 90088 72.1767 46 0.00691622 0.0138476 2023-09-23T19:18:56.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:55 smithi016 bash[75004]: audit 2023-09-23T19:18:54.808641+0000 mon.c (mon.2) 18 : audit 0 from='client.? 172.21.15.16:0/2956113923' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:56.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:18:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:18:55] "GET /metrics HTTP/1.1" 200 36201 "" "Prometheus/2.33.4" 2023-09-23T19:18:56.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:55 smithi016 bash[77995]: audit 2023-09-23T19:18:54.808641+0000 mon.c (mon.2) 18 : audit 0 from='client.? 172.21.15.16:0/2956113923' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:56.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:55 smithi092 bash[58855]: audit 2023-09-23T19:18:54.808641+0000 mon.c (mon.2) 18 : audit 0 from='client.? 172.21.15.16:0/2956113923' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:56.957 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 79 16 90817 90801 71.8271 44.5625 0.0148713 0.0139136 2023-09-23T19:18:57.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:56 smithi016 bash[75004]: cluster 2023-09-23T19:18:55.610475+0000 mgr.y (mgr.25325) 262 : cluster 0 pgmap v201: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 57 MiB/s wr, 914 op/s; 16726/268329 objects degraded (6.233%); 1.1 MiB/s, 16 objects/s recovering 2023-09-23T19:18:57.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:56 smithi016 bash[77995]: cluster 2023-09-23T19:18:55.610475+0000 mgr.y (mgr.25325) 262 : cluster 0 pgmap v201: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 57 MiB/s wr, 914 op/s; 16726/268329 objects degraded (6.233%); 1.1 MiB/s, 16 objects/s recovering 2023-09-23T19:18:57.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:56 smithi092 bash[58855]: cluster 2023-09-23T19:18:55.610475+0000 mgr.y (mgr.25325) 262 : cluster 0 pgmap v201: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 57 MiB/s wr, 914 op/s; 16726/268329 objects degraded (6.233%); 1.1 MiB/s, 16 objects/s recovering 2023-09-23T19:18:57.957 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-09-23T19:18:57.956152+0000 min lat: 0.00151424 max lat: 2.34811 avg lat: 0.0139855 2023-09-23T19:18:57.958 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:18:57.958 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 80 16 91496 91480 71.4597 42.4375 0.0206306 0.0139855 2023-09-23T19:18:58.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:57 smithi016 bash[75004]: audit 2023-09-23T19:18:57.446499+0000 mon.c (mon.2) 19 : audit 0 from='client.? 172.21.15.16:0/1606267297' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:58.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:57 smithi016 bash[75004]: cluster 2023-09-23T19:18:57.611123+0000 mgr.y (mgr.25325) 263 : cluster 0 pgmap v202: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 38 MiB/s wr, 603 op/s; 16726/268329 objects degraded (6.233%); 720 KiB/s, 11 objects/s recovering 2023-09-23T19:18:58.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:57 smithi016 bash[77995]: audit 2023-09-23T19:18:57.446499+0000 mon.c (mon.2) 19 : audit 0 from='client.? 172.21.15.16:0/1606267297' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:58.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:57 smithi016 bash[77995]: cluster 2023-09-23T19:18:57.611123+0000 mgr.y (mgr.25325) 263 : cluster 0 pgmap v202: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 38 MiB/s wr, 603 op/s; 16726/268329 objects degraded (6.233%); 720 KiB/s, 11 objects/s recovering 2023-09-23T19:18:58.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:57 smithi092 bash[58855]: audit 2023-09-23T19:18:57.446499+0000 mon.c (mon.2) 19 : audit 0 from='client.? 172.21.15.16:0/1606267297' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:18:58.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:57 smithi092 bash[58855]: cluster 2023-09-23T19:18:57.611123+0000 mgr.y (mgr.25325) 263 : cluster 0 pgmap v202: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 38 MiB/s wr, 603 op/s; 16726/268329 objects degraded (6.233%); 720 KiB/s, 11 objects/s recovering 2023-09-23T19:18:58.958 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 81 16 92222 92206 71.1376 45.375 0.0110544 0.0140491 2023-09-23T19:18:59.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:58 smithi016 bash[75004]: audit 2023-09-23T19:18:58.618182+0000 mon.a (mon.0) 371 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:18:59.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:58 smithi016 bash[77995]: audit 2023-09-23T19:18:58.618182+0000 mon.a (mon.0) 371 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:18:59.042 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:58 smithi092 bash[58855]: audit 2023-09-23T19:18:58.618182+0000 mon.a (mon.0) 371 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:18:59.957 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 82 16 92913 92897 70.7967 43.1875 0.0375723 0.0141175 2023-09-23T19:19:00.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:59 smithi016 bash[77995]: cluster 2023-09-23T19:18:58.997736+0000 mon.a (mon.0) 372 : cluster 3 Health check update: Degraded data redundancy: 16726/268329 objects degraded (6.233%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T19:19:00.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:18:59 smithi016 bash[77995]: cluster 2023-09-23T19:18:59.611709+0000 mgr.y (mgr.25325) 264 : cluster 0 pgmap v203: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 41 MiB/s wr, 656 op/s; 16726/270237 objects degraded (6.189%); 720 KiB/s, 11 objects/s recovering 2023-09-23T19:19:00.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:59 smithi016 bash[75004]: cluster 2023-09-23T19:18:58.997736+0000 mon.a (mon.0) 372 : cluster 3 Health check update: Degraded data redundancy: 16726/268329 objects degraded (6.233%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T19:19:00.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:18:59 smithi016 bash[75004]: cluster 2023-09-23T19:18:59.611709+0000 mgr.y (mgr.25325) 264 : cluster 0 pgmap v203: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 41 MiB/s wr, 656 op/s; 16726/270237 objects degraded (6.189%); 720 KiB/s, 11 objects/s recovering 2023-09-23T19:19:00.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:59 smithi092 bash[58855]: cluster 2023-09-23T19:18:58.997736+0000 mon.a (mon.0) 372 : cluster 3 Health check update: Degraded data redundancy: 16726/268329 objects degraded (6.233%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T19:19:00.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:18:59 smithi092 bash[58855]: cluster 2023-09-23T19:18:59.611709+0000 mgr.y (mgr.25325) 264 : cluster 0 pgmap v203: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 41 MiB/s wr, 656 op/s; 16726/270237 objects degraded (6.189%); 720 KiB/s, 11 objects/s recovering 2023-09-23T19:19:00.957 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 83 16 93591 93575 70.4542 42.375 0.0529861 0.0141856 2023-09-23T19:19:01.007 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:19:00 smithi016 bash[70992]: debug 2023-09-23T19:19:00.621+0000 7fb2a5a7c700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:19:01.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:01 smithi016 bash[75004]: audit 2023-09-23T19:19:00.141637+0000 mon.b (mon.1) 11 : audit 0 from='client.? 172.21.15.16:0/989411565' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:01.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:01 smithi016 bash[75004]: audit 2023-09-23T19:19:00.624956+0000 mon.a (mon.0) 373 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:19:01.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:01 smithi016 bash[75004]: audit 2023-09-23T19:19:00.625389+0000 mgr.y (mgr.25325) 265 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:19:01.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:01 smithi016 bash[75004]: cephadm 2023-09-23T19:19:00.628106+0000 mgr.y (mgr.25325) 266 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:19:01.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:01 smithi016 bash[77995]: audit 2023-09-23T19:19:00.141637+0000 mon.b (mon.1) 11 : audit 0 from='client.? 172.21.15.16:0/989411565' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:01.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:01 smithi016 bash[77995]: audit 2023-09-23T19:19:00.624956+0000 mon.a (mon.0) 373 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:19:01.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:01 smithi016 bash[77995]: audit 2023-09-23T19:19:00.625389+0000 mgr.y (mgr.25325) 265 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:19:01.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:01 smithi016 bash[77995]: cephadm 2023-09-23T19:19:00.628106+0000 mgr.y (mgr.25325) 266 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:19:01.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:01 smithi092 bash[58855]: audit 2023-09-23T19:19:00.141637+0000 mon.b (mon.1) 11 : audit 0 from='client.? 172.21.15.16:0/989411565' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:01.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:01 smithi092 bash[58855]: audit 2023-09-23T19:19:00.624956+0000 mon.a (mon.0) 373 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:19:01.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:01 smithi092 bash[58855]: audit 2023-09-23T19:19:00.625389+0000 mgr.y (mgr.25325) 265 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:19:01.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:01 smithi092 bash[58855]: cephadm 2023-09-23T19:19:00.628106+0000 mgr.y (mgr.25325) 266 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:19:01.957 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 84 16 94314 94298 70.1534 45.1875 0.0308641 0.0142466 2023-09-23T19:19:02.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:02 smithi016 bash[77995]: cluster 2023-09-23T19:19:01.612586+0000 mgr.y (mgr.25325) 267 : cluster 0 pgmap v204: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 52 MiB/s wr, 839 op/s; 16657/278904 objects degraded (5.972%); 1.1 MiB/s, 16 objects/s recovering 2023-09-23T19:19:02.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:02 smithi016 bash[75004]: cluster 2023-09-23T19:19:01.612586+0000 mgr.y (mgr.25325) 267 : cluster 0 pgmap v204: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 52 MiB/s wr, 839 op/s; 16657/278904 objects degraded (5.972%); 1.1 MiB/s, 16 objects/s recovering 2023-09-23T19:19:02.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:02 smithi092 bash[58855]: cluster 2023-09-23T19:19:01.612586+0000 mgr.y (mgr.25325) 267 : cluster 0 pgmap v204: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 52 MiB/s wr, 839 op/s; 16657/278904 objects degraded (5.972%); 1.1 MiB/s, 16 objects/s recovering 2023-09-23T19:19:02.957 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 85 16 94967 94951 69.8081 40.8125 0.0157263 0.0143162 2023-09-23T19:19:03.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:03 smithi016 bash[77995]: audit 2023-09-23T19:19:02.787591+0000 mon.a (mon.0) 374 : audit 0 from='client.? 172.21.15.16:0/131842407' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:03.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:03 smithi016 bash[75004]: audit 2023-09-23T19:19:02.787591+0000 mon.a (mon.0) 374 : audit 0 from='client.? 172.21.15.16:0/131842407' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:03.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:03 smithi092 bash[58855]: audit 2023-09-23T19:19:02.787591+0000 mon.a (mon.0) 374 : audit 0 from='client.? 172.21.15.16:0/131842407' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:03.957 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 86 16 95666 95650 69.5044 43.6875 0.0245248 0.0143787 2023-09-23T19:19:04.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:04 smithi016 bash[75004]: cluster 2023-09-23T19:19:03.613208+0000 mgr.y (mgr.25325) 268 : cluster 0 pgmap v205: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 37 MiB/s wr, 594 op/s; 16657/278904 objects degraded (5.972%); 730 KiB/s, 11 objects/s recovering 2023-09-23T19:19:04.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:04 smithi016 bash[75004]: cluster 2023-09-23T19:19:03.998801+0000 mon.a (mon.0) 375 : cluster 3 Health check update: Degraded data redundancy: 16657/278904 objects degraded (5.972%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T19:19:04.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:04 smithi016 bash[77995]: cluster 2023-09-23T19:19:03.613208+0000 mgr.y (mgr.25325) 268 : cluster 0 pgmap v205: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 37 MiB/s wr, 594 op/s; 16657/278904 objects degraded (5.972%); 730 KiB/s, 11 objects/s recovering 2023-09-23T19:19:04.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:04 smithi016 bash[77995]: cluster 2023-09-23T19:19:03.998801+0000 mon.a (mon.0) 375 : cluster 3 Health check update: Degraded data redundancy: 16657/278904 objects degraded (5.972%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T19:19:04.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:04 smithi092 bash[58855]: cluster 2023-09-23T19:19:03.613208+0000 mgr.y (mgr.25325) 268 : cluster 0 pgmap v205: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 37 MiB/s wr, 594 op/s; 16657/278904 objects degraded (5.972%); 730 KiB/s, 11 objects/s recovering 2023-09-23T19:19:04.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:04 smithi092 bash[58855]: cluster 2023-09-23T19:19:03.998801+0000 mon.a (mon.0) 375 : cluster 3 Health check update: Degraded data redundancy: 16657/278904 objects degraded (5.972%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T19:19:04.958 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 87 16 96364 96348 69.2068 43.625 0.0115121 0.0144416 2023-09-23T19:19:05.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:05 smithi016 bash[77995]: audit 2023-09-23T19:19:05.420738+0000 mon.a (mon.0) 376 : audit 0 from='client.? 172.21.15.16:0/971816913' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:05.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:05 smithi016 bash[75004]: audit 2023-09-23T19:19:05.420738+0000 mon.a (mon.0) 376 : audit 0 from='client.? 172.21.15.16:0/971816913' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:05.784 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:19:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:19:05] "GET /metrics HTTP/1.1" 200 36200 "" "Prometheus/2.33.4" 2023-09-23T19:19:05.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:05 smithi092 bash[58855]: audit 2023-09-23T19:19:05.420738+0000 mon.a (mon.0) 376 : audit 0 from='client.? 172.21.15.16:0/971816913' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:05.958 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 88 16 97013 96997 68.8813 40.5625 0.00945876 0.0145096 2023-09-23T19:19:06.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:06 smithi016 bash[75004]: cluster 2023-09-23T19:19:05.614070+0000 mgr.y (mgr.25325) 269 : cluster 0 pgmap v206: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 881 op/s; 16590/289263 objects degraded (5.735%); 1.1 MiB/s, 16 objects/s recovering 2023-09-23T19:19:06.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:06 smithi016 bash[77995]: cluster 2023-09-23T19:19:05.614070+0000 mgr.y (mgr.25325) 269 : cluster 0 pgmap v206: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 881 op/s; 16590/289263 objects degraded (5.735%); 1.1 MiB/s, 16 objects/s recovering 2023-09-23T19:19:06.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:06 smithi092 bash[58855]: cluster 2023-09-23T19:19:05.614070+0000 mgr.y (mgr.25325) 269 : cluster 0 pgmap v206: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 881 op/s; 16590/289263 objects degraded (5.735%); 1.1 MiB/s, 16 objects/s recovering 2023-09-23T19:19:06.958 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 89 16 97743 97727 68.62 45.625 0.0203105 0.0145656 2023-09-23T19:19:07.958 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 90 5 98409 98404 68.3276 42.3125 0.0266397 0.0146284 2023-09-23T19:19:08.958 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Total time run: 90.0141 2023-09-23T19:19:08.959 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Total writes made: 98409 2023-09-23T19:19:08.959 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Write size: 65536 2023-09-23T19:19:08.959 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Object size: 65536 2023-09-23T19:19:08.959 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Bandwidth (MB/sec): 68.3289 2023-09-23T19:19:08.959 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Stddev Bandwidth: 35.5566 2023-09-23T19:19:08.959 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Max bandwidth (MB/sec): 161.312 2023-09-23T19:19:08.959 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Min bandwidth (MB/sec): 0 2023-09-23T19:19:08.959 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Average IOPS: 1093 2023-09-23T19:19:08.959 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Stddev IOPS: 568.914 2023-09-23T19:19:08.959 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Max IOPS: 2581 2023-09-23T19:19:08.960 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Min IOPS: 0 2023-09-23T19:19:08.960 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Average Latency(s): 0.0146287 2023-09-23T19:19:08.960 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Stddev Latency(s): 0.0340354 2023-09-23T19:19:08.960 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Max latency(s): 2.34811 2023-09-23T19:19:08.960 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Min latency(s): 0.00151424 2023-09-23T19:19:08.972 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T19:19:09.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:08 smithi016 bash[75004]: cluster 2023-09-23T19:19:07.614684+0000 mgr.y (mgr.25325) 270 : cluster 0 pgmap v207: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 36 MiB/s wr, 581 op/s; 16590/289263 objects degraded (5.735%); 725 KiB/s, 11 objects/s recovering 2023-09-23T19:19:09.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:08 smithi016 bash[75004]: audit 2023-09-23T19:19:07.977281+0000 mon.c (mon.2) 20 : audit 0 from='client.? 172.21.15.16:0/2275481511' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:09.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:08 smithi016 bash[77995]: cluster 2023-09-23T19:19:07.614684+0000 mgr.y (mgr.25325) 270 : cluster 0 pgmap v207: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 36 MiB/s wr, 581 op/s; 16590/289263 objects degraded (5.735%); 725 KiB/s, 11 objects/s recovering 2023-09-23T19:19:09.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:08 smithi016 bash[77995]: audit 2023-09-23T19:19:07.977281+0000 mon.c (mon.2) 20 : audit 0 from='client.? 172.21.15.16:0/2275481511' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:09.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:08 smithi092 bash[58855]: cluster 2023-09-23T19:19:07.614684+0000 mgr.y (mgr.25325) 270 : cluster 0 pgmap v207: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 36 MiB/s wr, 581 op/s; 16590/289263 objects degraded (5.735%); 725 KiB/s, 11 objects/s recovering 2023-09-23T19:19:09.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:08 smithi092 bash[58855]: audit 2023-09-23T19:19:07.977281+0000 mon.c (mon.2) 20 : audit 0 from='client.? 172.21.15.16:0/2275481511' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:10.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:09 smithi016 bash[75004]: cluster 2023-09-23T19:19:09.000314+0000 mon.a (mon.0) 377 : cluster 3 Health check update: Degraded data redundancy: 16590/289263 objects degraded (5.735%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T19:19:10.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:09 smithi016 bash[77995]: cluster 2023-09-23T19:19:09.000314+0000 mon.a (mon.0) 377 : cluster 3 Health check update: Degraded data redundancy: 16590/289263 objects degraded (5.735%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T19:19:10.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:09 smithi092 bash[58855]: cluster 2023-09-23T19:19:09.000314+0000 mon.a (mon.0) 377 : cluster 3 Health check update: Degraded data redundancy: 16590/289263 objects degraded (5.735%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T19:19:11.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:10 smithi016 bash[75004]: cluster 2023-09-23T19:19:09.615254+0000 mgr.y (mgr.25325) 271 : cluster 0 pgmap v208: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 36 MiB/s wr, 581 op/s; 16590/289263 objects degraded (5.735%); 725 KiB/s, 11 objects/s recovering 2023-09-23T19:19:11.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:10 smithi016 bash[75004]: audit 2023-09-23T19:19:10.568226+0000 mon.c (mon.2) 21 : audit 0 from='client.? 172.21.15.16:0/218238536' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:11.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:10 smithi016 bash[77995]: cluster 2023-09-23T19:19:09.615254+0000 mgr.y (mgr.25325) 271 : cluster 0 pgmap v208: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 36 MiB/s wr, 581 op/s; 16590/289263 objects degraded (5.735%); 725 KiB/s, 11 objects/s recovering 2023-09-23T19:19:11.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:10 smithi016 bash[77995]: audit 2023-09-23T19:19:10.568226+0000 mon.c (mon.2) 21 : audit 0 from='client.? 172.21.15.16:0/218238536' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:11.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:10 smithi092 bash[58855]: cluster 2023-09-23T19:19:09.615254+0000 mgr.y (mgr.25325) 271 : cluster 0 pgmap v208: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 36 MiB/s wr, 581 op/s; 16590/289263 objects degraded (5.735%); 725 KiB/s, 11 objects/s recovering 2023-09-23T19:19:11.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:10 smithi092 bash[58855]: audit 2023-09-23T19:19:10.568226+0000 mon.c (mon.2) 21 : audit 0 from='client.? 172.21.15.16:0/218238536' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:13.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:12 smithi016 bash[75004]: cluster 2023-09-23T19:19:11.616213+0000 mgr.y (mgr.25325) 272 : cluster 0 pgmap v209: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 85 B/s rd, 43 MiB/s wr, 760 op/s; 16516/292983 objects degraded (5.637%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:19:13.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:12 smithi016 bash[77995]: cluster 2023-09-23T19:19:11.616213+0000 mgr.y (mgr.25325) 272 : cluster 0 pgmap v209: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 85 B/s rd, 43 MiB/s wr, 760 op/s; 16516/292983 objects degraded (5.637%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:19:13.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:12 smithi092 bash[58855]: cluster 2023-09-23T19:19:11.616213+0000 mgr.y (mgr.25325) 272 : cluster 0 pgmap v209: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 85 B/s rd, 43 MiB/s wr, 760 op/s; 16516/292983 objects degraded (5.637%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:19:14.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:13 smithi016 bash[75004]: audit 2023-09-23T19:19:13.236933+0000 mon.b (mon.1) 12 : audit 0 from='client.? 172.21.15.16:0/98908163' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:14.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:13 smithi016 bash[75004]: audit 2023-09-23T19:19:13.618562+0000 mon.a (mon.0) 378 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:19:14.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:13 smithi016 bash[77995]: audit 2023-09-23T19:19:13.236933+0000 mon.b (mon.1) 12 : audit 0 from='client.? 172.21.15.16:0/98908163' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:14.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:13 smithi016 bash[77995]: audit 2023-09-23T19:19:13.618562+0000 mon.a (mon.0) 378 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:19:14.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:13 smithi092 bash[58855]: audit 2023-09-23T19:19:13.236933+0000 mon.b (mon.1) 12 : audit 0 from='client.? 172.21.15.16:0/98908163' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:14.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:13 smithi092 bash[58855]: audit 2023-09-23T19:19:13.618562+0000 mon.a (mon.0) 378 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:19:15.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:14 smithi016 bash[75004]: cluster 2023-09-23T19:19:13.616887+0000 mgr.y (mgr.25325) 273 : cluster 0 pgmap v210: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 85 B/s rd, 28 MiB/s wr, 519 op/s; 16516/292983 objects degraded (5.637%); 752 KiB/s, 11 objects/s recovering 2023-09-23T19:19:15.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:14 smithi016 bash[75004]: cluster 2023-09-23T19:19:14.003197+0000 mon.a (mon.0) 379 : cluster 3 Health check update: Degraded data redundancy: 16516/292983 objects degraded (5.637%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:19:15.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:14 smithi016 bash[77995]: cluster 2023-09-23T19:19:13.616887+0000 mgr.y (mgr.25325) 273 : cluster 0 pgmap v210: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 85 B/s rd, 28 MiB/s wr, 519 op/s; 16516/292983 objects degraded (5.637%); 752 KiB/s, 11 objects/s recovering 2023-09-23T19:19:15.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:14 smithi016 bash[77995]: cluster 2023-09-23T19:19:14.003197+0000 mon.a (mon.0) 379 : cluster 3 Health check update: Degraded data redundancy: 16516/292983 objects degraded (5.637%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:19:15.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:14 smithi092 bash[58855]: cluster 2023-09-23T19:19:13.616887+0000 mgr.y (mgr.25325) 273 : cluster 0 pgmap v210: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 85 B/s rd, 28 MiB/s wr, 519 op/s; 16516/292983 objects degraded (5.637%); 752 KiB/s, 11 objects/s recovering 2023-09-23T19:19:15.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:14 smithi092 bash[58855]: cluster 2023-09-23T19:19:14.003197+0000 mon.a (mon.0) 379 : cluster 3 Health check update: Degraded data redundancy: 16516/292983 objects degraded (5.637%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:19:16.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:15 smithi016 bash[75004]: audit 2023-09-23T19:19:15.643306+0000 mon.a (mon.0) 380 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:19:16.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:19:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:19:15] "GET /metrics HTTP/1.1" 200 36192 "" "Prometheus/2.33.4" 2023-09-23T19:19:16.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:15 smithi016 bash[77995]: audit 2023-09-23T19:19:15.643306+0000 mon.a (mon.0) 380 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:19:16.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:15 smithi092 bash[58855]: audit 2023-09-23T19:19:15.643306+0000 mon.a (mon.0) 380 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:19:17.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:16 smithi016 bash[75004]: cluster 2023-09-23T19:19:15.617698+0000 mgr.y (mgr.25325) 274 : cluster 0 pgmap v211: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 85 B/s rd, 28 MiB/s wr, 718 op/s; 16516/285810 objects degraded (5.779%); 752 KiB/s, 11 objects/s recovering 2023-09-23T19:19:17.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:16 smithi016 bash[75004]: audit 2023-09-23T19:19:15.932983+0000 mon.a (mon.0) 381 : audit 0 from='client.? 172.21.15.16:0/306317276' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:17.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:16 smithi016 bash[77995]: cluster 2023-09-23T19:19:15.617698+0000 mgr.y (mgr.25325) 274 : cluster 0 pgmap v211: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 85 B/s rd, 28 MiB/s wr, 718 op/s; 16516/285810 objects degraded (5.779%); 752 KiB/s, 11 objects/s recovering 2023-09-23T19:19:17.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:16 smithi016 bash[77995]: audit 2023-09-23T19:19:15.932983+0000 mon.a (mon.0) 381 : audit 0 from='client.? 172.21.15.16:0/306317276' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:17.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:16 smithi092 bash[58855]: cluster 2023-09-23T19:19:15.617698+0000 mgr.y (mgr.25325) 274 : cluster 0 pgmap v211: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 85 B/s rd, 28 MiB/s wr, 718 op/s; 16516/285810 objects degraded (5.779%); 752 KiB/s, 11 objects/s recovering 2023-09-23T19:19:17.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:16 smithi092 bash[58855]: audit 2023-09-23T19:19:15.932983+0000 mon.a (mon.0) 381 : audit 0 from='client.? 172.21.15.16:0/306317276' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:19.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:18 smithi016 bash[75004]: cluster 2023-09-23T19:19:17.618452+0000 mgr.y (mgr.25325) 275 : cluster 0 pgmap v212: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 85 B/s rd, 10 MiB/s wr, 525 op/s; 16451/282420 objects degraded (5.825%); 741 KiB/s, 11 objects/s recovering 2023-09-23T19:19:19.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:18 smithi016 bash[75004]: audit 2023-09-23T19:19:18.555276+0000 mon.a (mon.0) 382 : audit 0 from='client.? 172.21.15.16:0/2024762991' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:19.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:18 smithi016 bash[77995]: cluster 2023-09-23T19:19:17.618452+0000 mgr.y (mgr.25325) 275 : cluster 0 pgmap v212: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 85 B/s rd, 10 MiB/s wr, 525 op/s; 16451/282420 objects degraded (5.825%); 741 KiB/s, 11 objects/s recovering 2023-09-23T19:19:19.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:18 smithi016 bash[77995]: audit 2023-09-23T19:19:18.555276+0000 mon.a (mon.0) 382 : audit 0 from='client.? 172.21.15.16:0/2024762991' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:19.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:18 smithi092 bash[58855]: cluster 2023-09-23T19:19:17.618452+0000 mgr.y (mgr.25325) 275 : cluster 0 pgmap v212: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 85 B/s rd, 10 MiB/s wr, 525 op/s; 16451/282420 objects degraded (5.825%); 741 KiB/s, 11 objects/s recovering 2023-09-23T19:19:19.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:18 smithi092 bash[58855]: audit 2023-09-23T19:19:18.555276+0000 mon.a (mon.0) 382 : audit 0 from='client.? 172.21.15.16:0/2024762991' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:20.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:19 smithi016 bash[75004]: cluster 2023-09-23T19:19:19.005928+0000 mon.a (mon.0) 383 : cluster 3 Health check update: Degraded data redundancy: 16451/282420 objects degraded (5.825%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:19:20.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:19 smithi016 bash[77995]: cluster 2023-09-23T19:19:19.005928+0000 mon.a (mon.0) 383 : cluster 3 Health check update: Degraded data redundancy: 16451/282420 objects degraded (5.825%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:19:20.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:19 smithi092 bash[58855]: cluster 2023-09-23T19:19:19.005928+0000 mon.a (mon.0) 383 : cluster 3 Health check update: Degraded data redundancy: 16451/282420 objects degraded (5.825%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:19:21.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:20 smithi016 bash[75004]: cluster 2023-09-23T19:19:19.619050+0000 mgr.y (mgr.25325) 276 : cluster 0 pgmap v213: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 85 B/s rd, 10 MiB/s wr, 525 op/s; 16451/282420 objects degraded (5.825%); 741 KiB/s, 11 objects/s recovering 2023-09-23T19:19:21.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:20 smithi016 bash[77995]: cluster 2023-09-23T19:19:19.619050+0000 mgr.y (mgr.25325) 276 : cluster 0 pgmap v213: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 85 B/s rd, 10 MiB/s wr, 525 op/s; 16451/282420 objects degraded (5.825%); 741 KiB/s, 11 objects/s recovering 2023-09-23T19:19:21.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:20 smithi092 bash[58855]: cluster 2023-09-23T19:19:19.619050+0000 mgr.y (mgr.25325) 276 : cluster 0 pgmap v213: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 85 B/s rd, 10 MiB/s wr, 525 op/s; 16451/282420 objects degraded (5.825%); 741 KiB/s, 11 objects/s recovering 2023-09-23T19:19:22.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:21 smithi016 bash[75004]: audit 2023-09-23T19:19:21.203692+0000 mon.b (mon.1) 13 : audit 0 from='client.? 172.21.15.16:0/2752803896' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:22.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:21 smithi016 bash[77995]: audit 2023-09-23T19:19:21.203692+0000 mon.b (mon.1) 13 : audit 0 from='client.? 172.21.15.16:0/2752803896' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:22.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:21 smithi092 bash[58855]: audit 2023-09-23T19:19:21.203692+0000 mon.b (mon.1) 13 : audit 0 from='client.? 172.21.15.16:0/2752803896' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:23.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:22 smithi016 bash[75004]: cluster 2023-09-23T19:19:21.619973+0000 mgr.y (mgr.25325) 277 : cluster 0 pgmap v214: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 85 B/s rd, 10 MiB/s wr, 807 op/s; 16388/272271 objects degraded (6.019%); 1.1 MiB/s, 16 objects/s recovering 2023-09-23T19:19:23.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:22 smithi016 bash[75004]: audit 2023-09-23T19:19:21.912982+0000 mon.a (mon.0) 384 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:19:23.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:22 smithi016 bash[75004]: audit 2023-09-23T19:19:21.924721+0000 mon.a (mon.0) 385 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:19:23.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:22 smithi016 bash[75004]: audit 2023-09-23T19:19:22.300788+0000 mon.a (mon.0) 386 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:19:23.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:22 smithi016 bash[75004]: audit 2023-09-23T19:19:22.316863+0000 mon.a (mon.0) 387 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:19:23.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:22 smithi016 bash[77995]: cluster 2023-09-23T19:19:21.619973+0000 mgr.y (mgr.25325) 277 : cluster 0 pgmap v214: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 85 B/s rd, 10 MiB/s wr, 807 op/s; 16388/272271 objects degraded (6.019%); 1.1 MiB/s, 16 objects/s recovering 2023-09-23T19:19:23.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:22 smithi016 bash[77995]: audit 2023-09-23T19:19:21.912982+0000 mon.a (mon.0) 384 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:19:23.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:22 smithi016 bash[77995]: audit 2023-09-23T19:19:21.924721+0000 mon.a (mon.0) 385 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:19:23.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:22 smithi016 bash[77995]: audit 2023-09-23T19:19:22.300788+0000 mon.a (mon.0) 386 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:19:23.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:22 smithi016 bash[77995]: audit 2023-09-23T19:19:22.316863+0000 mon.a (mon.0) 387 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:19:23.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:22 smithi092 bash[58855]: cluster 2023-09-23T19:19:21.619973+0000 mgr.y (mgr.25325) 277 : cluster 0 pgmap v214: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 85 B/s rd, 10 MiB/s wr, 807 op/s; 16388/272271 objects degraded (6.019%); 1.1 MiB/s, 16 objects/s recovering 2023-09-23T19:19:23.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:22 smithi092 bash[58855]: audit 2023-09-23T19:19:21.912982+0000 mon.a (mon.0) 384 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:19:23.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:22 smithi092 bash[58855]: audit 2023-09-23T19:19:21.924721+0000 mon.a (mon.0) 385 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:19:23.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:22 smithi092 bash[58855]: audit 2023-09-23T19:19:22.300788+0000 mon.a (mon.0) 386 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:19:23.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:22 smithi092 bash[58855]: audit 2023-09-23T19:19:22.316863+0000 mon.a (mon.0) 387 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:19:23.534 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:19:23 smithi016 bash[70992]: debug 2023-09-23T19:19:23.144+0000 7fb2a5a7c700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:19:24.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:23 smithi016 bash[75004]: audit 2023-09-23T19:19:23.050873+0000 mon.a (mon.0) 388 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:19:24.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:23 smithi016 bash[75004]: audit 2023-09-23T19:19:23.052048+0000 mon.a (mon.0) 389 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:19:24.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:23 smithi016 bash[75004]: audit 2023-09-23T19:19:23.060294+0000 mon.a (mon.0) 390 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:19:24.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:23 smithi016 bash[75004]: audit 2023-09-23T19:19:23.132068+0000 mon.a (mon.0) 391 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:19:24.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:23 smithi016 bash[75004]: audit 2023-09-23T19:19:23.138897+0000 mon.a (mon.0) 392 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:19:24.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:23 smithi016 bash[75004]: audit 2023-09-23T19:19:23.142717+0000 mon.a (mon.0) 393 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:19:24.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:23 smithi016 bash[75004]: audit 2023-09-23T19:19:23.145345+0000 mon.a (mon.0) 394 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:19:24.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:23 smithi016 bash[75004]: audit 2023-09-23T19:19:23.148793+0000 mon.a (mon.0) 395 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:19:24.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:23 smithi016 bash[77995]: audit 2023-09-23T19:19:23.050873+0000 mon.a (mon.0) 388 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:19:24.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:23 smithi016 bash[77995]: audit 2023-09-23T19:19:23.052048+0000 mon.a (mon.0) 389 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:19:24.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:23 smithi016 bash[77995]: audit 2023-09-23T19:19:23.060294+0000 mon.a (mon.0) 390 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:19:24.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:23 smithi016 bash[77995]: audit 2023-09-23T19:19:23.132068+0000 mon.a (mon.0) 391 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:19:24.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:23 smithi016 bash[77995]: audit 2023-09-23T19:19:23.138897+0000 mon.a (mon.0) 392 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:19:24.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:23 smithi016 bash[77995]: audit 2023-09-23T19:19:23.142717+0000 mon.a (mon.0) 393 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:19:24.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:23 smithi016 bash[77995]: audit 2023-09-23T19:19:23.145345+0000 mon.a (mon.0) 394 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:19:24.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:23 smithi016 bash[77995]: audit 2023-09-23T19:19:23.148793+0000 mon.a (mon.0) 395 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:19:24.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:23 smithi092 bash[58855]: audit 2023-09-23T19:19:23.050873+0000 mon.a (mon.0) 388 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:19:24.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:23 smithi092 bash[58855]: audit 2023-09-23T19:19:23.052048+0000 mon.a (mon.0) 389 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:19:24.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:23 smithi092 bash[58855]: audit 2023-09-23T19:19:23.060294+0000 mon.a (mon.0) 390 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:19:24.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:23 smithi092 bash[58855]: audit 2023-09-23T19:19:23.132068+0000 mon.a (mon.0) 391 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:19:24.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:23 smithi092 bash[58855]: audit 2023-09-23T19:19:23.138897+0000 mon.a (mon.0) 392 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:19:24.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:23 smithi092 bash[58855]: audit 2023-09-23T19:19:23.142717+0000 mon.a (mon.0) 393 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:19:24.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:23 smithi092 bash[58855]: audit 2023-09-23T19:19:23.145345+0000 mon.a (mon.0) 394 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:19:24.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:23 smithi092 bash[58855]: audit 2023-09-23T19:19:23.148793+0000 mon.a (mon.0) 395 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:19:25.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:24 smithi016 bash[75004]: audit 2023-09-23T19:19:23.149355+0000 mgr.y (mgr.25325) 278 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:19:25.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:24 smithi016 bash[75004]: cephadm 2023-09-23T19:19:23.150787+0000 mgr.y (mgr.25325) 279 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:19:25.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:24 smithi016 bash[75004]: cluster 2023-09-23T19:19:23.620968+0000 mgr.y (mgr.25325) 280 : cluster 0 pgmap v215: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 16388/272271 objects degraded (6.019%); 682 KiB/s, 10 objects/s recovering 2023-09-23T19:19:25.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:24 smithi016 bash[75004]: audit 2023-09-23T19:19:23.795856+0000 mon.a (mon.0) 396 : audit 0 from='client.? 172.21.15.16:0/4121500856' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:25.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:24 smithi016 bash[75004]: cluster 2023-09-23T19:19:24.007142+0000 mon.a (mon.0) 397 : cluster 3 Health check update: Degraded data redundancy: 16388/272271 objects degraded (6.019%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:19:25.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:24 smithi016 bash[77995]: audit 2023-09-23T19:19:23.149355+0000 mgr.y (mgr.25325) 278 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:19:25.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:24 smithi016 bash[77995]: cephadm 2023-09-23T19:19:23.150787+0000 mgr.y (mgr.25325) 279 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:19:25.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:24 smithi016 bash[77995]: cluster 2023-09-23T19:19:23.620968+0000 mgr.y (mgr.25325) 280 : cluster 0 pgmap v215: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 16388/272271 objects degraded (6.019%); 682 KiB/s, 10 objects/s recovering 2023-09-23T19:19:25.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:24 smithi016 bash[77995]: audit 2023-09-23T19:19:23.795856+0000 mon.a (mon.0) 396 : audit 0 from='client.? 172.21.15.16:0/4121500856' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:25.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:24 smithi016 bash[77995]: cluster 2023-09-23T19:19:24.007142+0000 mon.a (mon.0) 397 : cluster 3 Health check update: Degraded data redundancy: 16388/272271 objects degraded (6.019%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:19:25.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:24 smithi092 bash[58855]: audit 2023-09-23T19:19:23.149355+0000 mgr.y (mgr.25325) 278 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:19:25.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:24 smithi092 bash[58855]: cephadm 2023-09-23T19:19:23.150787+0000 mgr.y (mgr.25325) 279 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:19:25.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:24 smithi092 bash[58855]: cluster 2023-09-23T19:19:23.620968+0000 mgr.y (mgr.25325) 280 : cluster 0 pgmap v215: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 16388/272271 objects degraded (6.019%); 682 KiB/s, 10 objects/s recovering 2023-09-23T19:19:25.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:24 smithi092 bash[58855]: audit 2023-09-23T19:19:23.795856+0000 mon.a (mon.0) 396 : audit 0 from='client.? 172.21.15.16:0/4121500856' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:25.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:24 smithi092 bash[58855]: cluster 2023-09-23T19:19:24.007142+0000 mon.a (mon.0) 397 : cluster 3 Health check update: Degraded data redundancy: 16388/272271 objects degraded (6.019%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:19:26.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:19:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:19:25] "GET /metrics HTTP/1.1" 200 36192 "" "Prometheus/2.33.4" 2023-09-23T19:19:27.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:26 smithi016 bash[77995]: cluster 2023-09-23T19:19:25.621982+0000 mgr.y (mgr.25325) 281 : cluster 0 pgmap v216: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 16388/265635 objects degraded (6.169%); 682 KiB/s, 10 objects/s recovering 2023-09-23T19:19:27.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:26 smithi016 bash[77995]: audit 2023-09-23T19:19:26.417971+0000 mon.c (mon.2) 22 : audit 0 from='client.? 172.21.15.16:0/1594552011' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:27.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:26 smithi016 bash[75004]: cluster 2023-09-23T19:19:25.621982+0000 mgr.y (mgr.25325) 281 : cluster 0 pgmap v216: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 16388/265635 objects degraded (6.169%); 682 KiB/s, 10 objects/s recovering 2023-09-23T19:19:27.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:26 smithi016 bash[75004]: audit 2023-09-23T19:19:26.417971+0000 mon.c (mon.2) 22 : audit 0 from='client.? 172.21.15.16:0/1594552011' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:27.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:26 smithi092 bash[58855]: cluster 2023-09-23T19:19:25.621982+0000 mgr.y (mgr.25325) 281 : cluster 0 pgmap v216: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 16388/265635 objects degraded (6.169%); 682 KiB/s, 10 objects/s recovering 2023-09-23T19:19:27.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:26 smithi092 bash[58855]: audit 2023-09-23T19:19:26.417971+0000 mon.c (mon.2) 22 : audit 0 from='client.? 172.21.15.16:0/1594552011' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:28.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:27 smithi016 bash[75004]: cluster 2023-09-23T19:19:27.625434+0000 mgr.y (mgr.25325) 282 : cluster 0 pgmap v217: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 16325/262656 objects degraded (6.215%); 1018 KiB/s, 15 objects/s recovering 2023-09-23T19:19:28.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:27 smithi016 bash[77995]: cluster 2023-09-23T19:19:27.625434+0000 mgr.y (mgr.25325) 282 : cluster 0 pgmap v217: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 16325/262656 objects degraded (6.215%); 1018 KiB/s, 15 objects/s recovering 2023-09-23T19:19:28.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:27 smithi092 bash[58855]: cluster 2023-09-23T19:19:27.625434+0000 mgr.y (mgr.25325) 282 : cluster 0 pgmap v217: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 16325/262656 objects degraded (6.215%); 1018 KiB/s, 15 objects/s recovering 2023-09-23T19:19:29.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:28 smithi016 bash[75004]: audit 2023-09-23T19:19:28.618849+0000 mon.a (mon.0) 398 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:19:29.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:28 smithi016 bash[77995]: audit 2023-09-23T19:19:28.618849+0000 mon.a (mon.0) 398 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:19:29.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:28 smithi092 bash[58855]: audit 2023-09-23T19:19:28.618849+0000 mon.a (mon.0) 398 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:19:30.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:29 smithi016 bash[75004]: cluster 2023-09-23T19:19:29.011342+0000 mon.a (mon.0) 399 : cluster 3 Health check update: Degraded data redundancy: 16325/262656 objects degraded (6.215%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:19:30.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:29 smithi016 bash[75004]: audit 2023-09-23T19:19:29.047243+0000 mon.c (mon.2) 23 : audit 0 from='client.? 172.21.15.16:0/3509777400' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:30.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:29 smithi016 bash[75004]: cluster 2023-09-23T19:19:29.626197+0000 mgr.y (mgr.25325) 283 : cluster 0 pgmap v218: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 548 op/s; 16325/262656 objects degraded (6.215%); 672 KiB/s, 10 objects/s recovering 2023-09-23T19:19:30.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:29 smithi016 bash[77995]: cluster 2023-09-23T19:19:29.011342+0000 mon.a (mon.0) 399 : cluster 3 Health check update: Degraded data redundancy: 16325/262656 objects degraded (6.215%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:19:30.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:29 smithi016 bash[77995]: audit 2023-09-23T19:19:29.047243+0000 mon.c (mon.2) 23 : audit 0 from='client.? 172.21.15.16:0/3509777400' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:30.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:29 smithi016 bash[77995]: cluster 2023-09-23T19:19:29.626197+0000 mgr.y (mgr.25325) 283 : cluster 0 pgmap v218: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 548 op/s; 16325/262656 objects degraded (6.215%); 672 KiB/s, 10 objects/s recovering 2023-09-23T19:19:30.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:29 smithi092 bash[58855]: cluster 2023-09-23T19:19:29.011342+0000 mon.a (mon.0) 399 : cluster 3 Health check update: Degraded data redundancy: 16325/262656 objects degraded (6.215%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:19:30.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:29 smithi092 bash[58855]: audit 2023-09-23T19:19:29.047243+0000 mon.c (mon.2) 23 : audit 0 from='client.? 172.21.15.16:0/3509777400' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:30.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:29 smithi092 bash[58855]: cluster 2023-09-23T19:19:29.626197+0000 mgr.y (mgr.25325) 283 : cluster 0 pgmap v218: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 548 op/s; 16325/262656 objects degraded (6.215%); 672 KiB/s, 10 objects/s recovering 2023-09-23T19:19:32.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:31 smithi016 bash[75004]: audit 2023-09-23T19:19:31.661859+0000 mon.c (mon.2) 24 : audit 0 from='client.? 172.21.15.16:0/237590748' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:32.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:31 smithi016 bash[77995]: audit 2023-09-23T19:19:31.661859+0000 mon.c (mon.2) 24 : audit 0 from='client.? 172.21.15.16:0/237590748' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:32.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:31 smithi092 bash[58855]: audit 2023-09-23T19:19:31.661859+0000 mon.c (mon.2) 24 : audit 0 from='client.? 172.21.15.16:0/237590748' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:33.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:32 smithi016 bash[75004]: cluster 2023-09-23T19:19:31.627076+0000 mgr.y (mgr.25325) 284 : cluster 0 pgmap v219: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 825 op/s; 16259/252699 objects degraded (6.434%); 1023 KiB/s, 15 objects/s recovering 2023-09-23T19:19:33.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:32 smithi016 bash[77995]: cluster 2023-09-23T19:19:31.627076+0000 mgr.y (mgr.25325) 284 : cluster 0 pgmap v219: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 825 op/s; 16259/252699 objects degraded (6.434%); 1023 KiB/s, 15 objects/s recovering 2023-09-23T19:19:33.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:32 smithi092 bash[58855]: cluster 2023-09-23T19:19:31.627076+0000 mgr.y (mgr.25325) 284 : cluster 0 pgmap v219: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 825 op/s; 16259/252699 objects degraded (6.434%); 1023 KiB/s, 15 objects/s recovering 2023-09-23T19:19:35.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:34 smithi016 bash[75004]: cluster 2023-09-23T19:19:33.627701+0000 mgr.y (mgr.25325) 285 : cluster 0 pgmap v220: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 543 op/s; 16259/252699 objects degraded (6.434%); 688 KiB/s, 10 objects/s recovering 2023-09-23T19:19:35.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:34 smithi016 bash[75004]: cluster 2023-09-23T19:19:34.014255+0000 mon.a (mon.0) 400 : cluster 3 Health check update: Degraded data redundancy: 16259/252699 objects degraded (6.434%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:19:35.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:34 smithi016 bash[75004]: audit 2023-09-23T19:19:34.275518+0000 mon.c (mon.2) 25 : audit 0 from='client.? 172.21.15.16:0/4117328424' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:35.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:34 smithi016 bash[77995]: cluster 2023-09-23T19:19:33.627701+0000 mgr.y (mgr.25325) 285 : cluster 0 pgmap v220: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 543 op/s; 16259/252699 objects degraded (6.434%); 688 KiB/s, 10 objects/s recovering 2023-09-23T19:19:35.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:34 smithi016 bash[77995]: cluster 2023-09-23T19:19:34.014255+0000 mon.a (mon.0) 400 : cluster 3 Health check update: Degraded data redundancy: 16259/252699 objects degraded (6.434%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:19:35.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:34 smithi016 bash[77995]: audit 2023-09-23T19:19:34.275518+0000 mon.c (mon.2) 25 : audit 0 from='client.? 172.21.15.16:0/4117328424' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:35.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:34 smithi092 bash[58855]: cluster 2023-09-23T19:19:33.627701+0000 mgr.y (mgr.25325) 285 : cluster 0 pgmap v220: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 543 op/s; 16259/252699 objects degraded (6.434%); 688 KiB/s, 10 objects/s recovering 2023-09-23T19:19:35.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:34 smithi092 bash[58855]: cluster 2023-09-23T19:19:34.014255+0000 mon.a (mon.0) 400 : cluster 3 Health check update: Degraded data redundancy: 16259/252699 objects degraded (6.434%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:19:35.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:34 smithi092 bash[58855]: audit 2023-09-23T19:19:34.275518+0000 mon.c (mon.2) 25 : audit 0 from='client.? 172.21.15.16:0/4117328424' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:36.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:19:35 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:19:35] "GET /metrics HTTP/1.1" 200 36205 "" "Prometheus/2.33.4" 2023-09-23T19:19:37.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:36 smithi016 bash[75004]: cluster 2023-09-23T19:19:35.628343+0000 mgr.y (mgr.25325) 286 : cluster 0 pgmap v221: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 16259/245982 objects degraded (6.610%); 688 KiB/s, 10 objects/s recovering 2023-09-23T19:19:37.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:36 smithi016 bash[77995]: cluster 2023-09-23T19:19:35.628343+0000 mgr.y (mgr.25325) 286 : cluster 0 pgmap v221: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 16259/245982 objects degraded (6.610%); 688 KiB/s, 10 objects/s recovering 2023-09-23T19:19:37.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:36 smithi092 bash[58855]: cluster 2023-09-23T19:19:35.628343+0000 mgr.y (mgr.25325) 286 : cluster 0 pgmap v221: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 16259/245982 objects degraded (6.610%); 688 KiB/s, 10 objects/s recovering 2023-09-23T19:19:38.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:37 smithi016 bash[75004]: audit 2023-09-23T19:19:36.875361+0000 mon.a (mon.0) 401 : audit 0 from='client.? 172.21.15.16:0/3407340977' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:38.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:37 smithi016 bash[77995]: audit 2023-09-23T19:19:36.875361+0000 mon.a (mon.0) 401 : audit 0 from='client.? 172.21.15.16:0/3407340977' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:38.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:37 smithi092 bash[58855]: audit 2023-09-23T19:19:36.875361+0000 mon.a (mon.0) 401 : audit 0 from='client.? 172.21.15.16:0/3407340977' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:38.534 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:19:38 smithi016 bash[70992]: debug 2023-09-23T19:19:38.160+0000 7fb2a5a7c700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:19:39.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:38 smithi016 bash[75004]: cluster 2023-09-23T19:19:37.629027+0000 mgr.y (mgr.25325) 287 : cluster 0 pgmap v222: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 16190/243000 objects degraded (6.663%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T19:19:39.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:38 smithi016 bash[75004]: audit 2023-09-23T19:19:38.164962+0000 mon.a (mon.0) 402 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:19:39.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:38 smithi016 bash[77995]: cluster 2023-09-23T19:19:37.629027+0000 mgr.y (mgr.25325) 287 : cluster 0 pgmap v222: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 16190/243000 objects degraded (6.663%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T19:19:39.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:38 smithi016 bash[77995]: audit 2023-09-23T19:19:38.164962+0000 mon.a (mon.0) 402 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:19:39.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:38 smithi092 bash[58855]: cluster 2023-09-23T19:19:37.629027+0000 mgr.y (mgr.25325) 287 : cluster 0 pgmap v222: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 16190/243000 objects degraded (6.663%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T19:19:39.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:38 smithi092 bash[58855]: audit 2023-09-23T19:19:38.164962+0000 mon.a (mon.0) 402 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:19:40.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:39 smithi016 bash[75004]: audit 2023-09-23T19:19:38.165515+0000 mgr.y (mgr.25325) 288 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:19:40.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:39 smithi016 bash[75004]: cephadm 2023-09-23T19:19:38.166706+0000 mgr.y (mgr.25325) 289 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:19:40.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:39 smithi016 bash[75004]: cluster 2023-09-23T19:19:39.015471+0000 mon.a (mon.0) 403 : cluster 3 Health check update: Degraded data redundancy: 16190/243000 objects degraded (6.663%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:19:40.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:39 smithi016 bash[75004]: audit 2023-09-23T19:19:39.493343+0000 mon.a (mon.0) 404 : audit 0 from='client.? 172.21.15.16:0/656815497' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:40.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:39 smithi016 bash[77995]: audit 2023-09-23T19:19:38.165515+0000 mgr.y (mgr.25325) 288 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:19:40.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:39 smithi016 bash[77995]: cephadm 2023-09-23T19:19:38.166706+0000 mgr.y (mgr.25325) 289 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:19:40.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:39 smithi016 bash[77995]: cluster 2023-09-23T19:19:39.015471+0000 mon.a (mon.0) 403 : cluster 3 Health check update: Degraded data redundancy: 16190/243000 objects degraded (6.663%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:19:40.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:39 smithi016 bash[77995]: audit 2023-09-23T19:19:39.493343+0000 mon.a (mon.0) 404 : audit 0 from='client.? 172.21.15.16:0/656815497' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:40.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:39 smithi092 bash[58855]: audit 2023-09-23T19:19:38.165515+0000 mgr.y (mgr.25325) 288 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:19:40.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:39 smithi092 bash[58855]: cephadm 2023-09-23T19:19:38.166706+0000 mgr.y (mgr.25325) 289 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:19:40.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:39 smithi092 bash[58855]: cluster 2023-09-23T19:19:39.015471+0000 mon.a (mon.0) 403 : cluster 3 Health check update: Degraded data redundancy: 16190/243000 objects degraded (6.663%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:19:40.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:39 smithi092 bash[58855]: audit 2023-09-23T19:19:39.493343+0000 mon.a (mon.0) 404 : audit 0 from='client.? 172.21.15.16:0/656815497' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:41.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:40 smithi016 bash[75004]: cluster 2023-09-23T19:19:39.629774+0000 mgr.y (mgr.25325) 290 : cluster 0 pgmap v223: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 545 op/s; 16190/243000 objects degraded (6.663%); 720 KiB/s, 11 objects/s recovering 2023-09-23T19:19:41.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:40 smithi016 bash[77995]: cluster 2023-09-23T19:19:39.629774+0000 mgr.y (mgr.25325) 290 : cluster 0 pgmap v223: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 545 op/s; 16190/243000 objects degraded (6.663%); 720 KiB/s, 11 objects/s recovering 2023-09-23T19:19:41.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:40 smithi092 bash[58855]: cluster 2023-09-23T19:19:39.629774+0000 mgr.y (mgr.25325) 290 : cluster 0 pgmap v223: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 545 op/s; 16190/243000 objects degraded (6.663%); 720 KiB/s, 11 objects/s recovering 2023-09-23T19:19:42.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:41 smithi016 bash[75004]: cluster 2023-09-23T19:19:41.630664+0000 mgr.y (mgr.25325) 291 : cluster 0 pgmap v224: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 16121/233178 objects degraded (6.914%); 1.1 MiB/s, 16 objects/s recovering 2023-09-23T19:19:42.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:41 smithi016 bash[77995]: cluster 2023-09-23T19:19:41.630664+0000 mgr.y (mgr.25325) 291 : cluster 0 pgmap v224: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 16121/233178 objects degraded (6.914%); 1.1 MiB/s, 16 objects/s recovering 2023-09-23T19:19:42.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:41 smithi092 bash[58855]: cluster 2023-09-23T19:19:41.630664+0000 mgr.y (mgr.25325) 291 : cluster 0 pgmap v224: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 16121/233178 objects degraded (6.914%); 1.1 MiB/s, 16 objects/s recovering 2023-09-23T19:19:43.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:42 smithi016 bash[75004]: audit 2023-09-23T19:19:42.127878+0000 mon.b (mon.1) 14 : audit 0 from='client.? 172.21.15.16:0/466486039' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:43.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:42 smithi016 bash[77995]: audit 2023-09-23T19:19:42.127878+0000 mon.b (mon.1) 14 : audit 0 from='client.? 172.21.15.16:0/466486039' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:43.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:42 smithi092 bash[58855]: audit 2023-09-23T19:19:42.127878+0000 mon.b (mon.1) 14 : audit 0 from='client.? 172.21.15.16:0/466486039' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:44.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:43 smithi016 bash[75004]: audit 2023-09-23T19:19:43.618986+0000 mon.a (mon.0) 405 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:19:44.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:43 smithi016 bash[75004]: cluster 2023-09-23T19:19:43.631226+0000 mgr.y (mgr.25325) 292 : cluster 0 pgmap v225: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 16121/233178 objects degraded (6.914%); 736 KiB/s, 11 objects/s recovering 2023-09-23T19:19:44.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:43 smithi016 bash[77995]: audit 2023-09-23T19:19:43.618986+0000 mon.a (mon.0) 405 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:19:44.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:43 smithi016 bash[77995]: cluster 2023-09-23T19:19:43.631226+0000 mgr.y (mgr.25325) 292 : cluster 0 pgmap v225: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 16121/233178 objects degraded (6.914%); 736 KiB/s, 11 objects/s recovering 2023-09-23T19:19:44.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:43 smithi092 bash[58855]: audit 2023-09-23T19:19:43.618986+0000 mon.a (mon.0) 405 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:19:44.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:43 smithi092 bash[58855]: cluster 2023-09-23T19:19:43.631226+0000 mgr.y (mgr.25325) 292 : cluster 0 pgmap v225: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 16121/233178 objects degraded (6.914%); 736 KiB/s, 11 objects/s recovering 2023-09-23T19:19:45.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:44 smithi016 bash[75004]: cluster 2023-09-23T19:19:44.016687+0000 mon.a (mon.0) 406 : cluster 3 Health check update: Degraded data redundancy: 16121/233178 objects degraded (6.914%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:19:45.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:44 smithi016 bash[75004]: audit 2023-09-23T19:19:44.740579+0000 mon.b (mon.1) 15 : audit 0 from='client.? 172.21.15.16:0/3604482943' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:45.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:44 smithi016 bash[77995]: cluster 2023-09-23T19:19:44.016687+0000 mon.a (mon.0) 406 : cluster 3 Health check update: Degraded data redundancy: 16121/233178 objects degraded (6.914%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:19:45.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:44 smithi016 bash[77995]: audit 2023-09-23T19:19:44.740579+0000 mon.b (mon.1) 15 : audit 0 from='client.? 172.21.15.16:0/3604482943' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:45.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:44 smithi092 bash[58855]: cluster 2023-09-23T19:19:44.016687+0000 mon.a (mon.0) 406 : cluster 3 Health check update: Degraded data redundancy: 16121/233178 objects degraded (6.914%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:19:45.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:44 smithi092 bash[58855]: audit 2023-09-23T19:19:44.740579+0000 mon.b (mon.1) 15 : audit 0 from='client.? 172.21.15.16:0/3604482943' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:46.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:45 smithi016 bash[75004]: cluster 2023-09-23T19:19:45.632049+0000 mgr.y (mgr.25325) 293 : cluster 0 pgmap v226: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 16121/226686 objects degraded (7.112%); 736 KiB/s, 11 objects/s recovering 2023-09-23T19:19:46.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:19:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:19:45] "GET /metrics HTTP/1.1" 200 36207 "" "Prometheus/2.33.4" 2023-09-23T19:19:46.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:45 smithi016 bash[77995]: cluster 2023-09-23T19:19:45.632049+0000 mgr.y (mgr.25325) 293 : cluster 0 pgmap v226: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 16121/226686 objects degraded (7.112%); 736 KiB/s, 11 objects/s recovering 2023-09-23T19:19:46.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:45 smithi092 bash[58855]: cluster 2023-09-23T19:19:45.632049+0000 mgr.y (mgr.25325) 293 : cluster 0 pgmap v226: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 16121/226686 objects degraded (7.112%); 736 KiB/s, 11 objects/s recovering 2023-09-23T19:19:47.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:47 smithi016 bash[75004]: audit 2023-09-23T19:19:47.371091+0000 mon.c (mon.2) 26 : audit 0 from='client.? 172.21.15.16:0/725791935' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:47.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:47 smithi016 bash[77995]: audit 2023-09-23T19:19:47.371091+0000 mon.c (mon.2) 26 : audit 0 from='client.? 172.21.15.16:0/725791935' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:47.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:47 smithi092 bash[58855]: audit 2023-09-23T19:19:47.371091+0000 mon.c (mon.2) 26 : audit 0 from='client.? 172.21.15.16:0/725791935' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:48.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:48 smithi016 bash[75004]: cluster 2023-09-23T19:19:47.634356+0000 mgr.y (mgr.25325) 294 : cluster 0 pgmap v227: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 16059/223653 objects degraded (7.180%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T19:19:48.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:48 smithi016 bash[77995]: cluster 2023-09-23T19:19:47.634356+0000 mgr.y (mgr.25325) 294 : cluster 0 pgmap v227: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 16059/223653 objects degraded (7.180%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T19:19:48.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:48 smithi092 bash[58855]: cluster 2023-09-23T19:19:47.634356+0000 mgr.y (mgr.25325) 294 : cluster 0 pgmap v227: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 16059/223653 objects degraded (7.180%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T19:19:49.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:49 smithi016 bash[75004]: cluster 2023-09-23T19:19:49.017965+0000 mon.a (mon.0) 407 : cluster 3 Health check update: Degraded data redundancy: 16059/223653 objects degraded (7.180%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:19:49.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:49 smithi016 bash[77995]: cluster 2023-09-23T19:19:49.017965+0000 mon.a (mon.0) 407 : cluster 3 Health check update: Degraded data redundancy: 16059/223653 objects degraded (7.180%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:19:49.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:49 smithi092 bash[58855]: cluster 2023-09-23T19:19:49.017965+0000 mon.a (mon.0) 407 : cluster 3 Health check update: Degraded data redundancy: 16059/223653 objects degraded (7.180%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:19:50.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:50 smithi016 bash[75004]: cluster 2023-09-23T19:19:49.634952+0000 mgr.y (mgr.25325) 295 : cluster 0 pgmap v228: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 537 op/s; 16059/223653 objects degraded (7.180%); 698 KiB/s, 10 objects/s recovering 2023-09-23T19:19:50.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:50 smithi016 bash[75004]: audit 2023-09-23T19:19:50.003371+0000 mon.c (mon.2) 27 : audit 0 from='client.? 172.21.15.16:0/46123192' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:50.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:50 smithi016 bash[77995]: cluster 2023-09-23T19:19:49.634952+0000 mgr.y (mgr.25325) 295 : cluster 0 pgmap v228: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 537 op/s; 16059/223653 objects degraded (7.180%); 698 KiB/s, 10 objects/s recovering 2023-09-23T19:19:50.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:50 smithi016 bash[77995]: audit 2023-09-23T19:19:50.003371+0000 mon.c (mon.2) 27 : audit 0 from='client.? 172.21.15.16:0/46123192' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:50.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:50 smithi092 bash[58855]: cluster 2023-09-23T19:19:49.634952+0000 mgr.y (mgr.25325) 295 : cluster 0 pgmap v228: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 537 op/s; 16059/223653 objects degraded (7.180%); 698 KiB/s, 10 objects/s recovering 2023-09-23T19:19:50.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:50 smithi092 bash[58855]: audit 2023-09-23T19:19:50.003371+0000 mon.c (mon.2) 27 : audit 0 from='client.? 172.21.15.16:0/46123192' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:53.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:52 smithi016 bash[75004]: cluster 2023-09-23T19:19:51.635811+0000 mgr.y (mgr.25325) 296 : cluster 0 pgmap v229: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 797 op/s; 15993/214284 objects degraded (7.463%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T19:19:53.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:52 smithi016 bash[75004]: audit 2023-09-23T19:19:52.609944+0000 mon.c (mon.2) 28 : audit 0 from='client.? 172.21.15.16:0/668395296' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:53.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:52 smithi016 bash[77995]: cluster 2023-09-23T19:19:51.635811+0000 mgr.y (mgr.25325) 296 : cluster 0 pgmap v229: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 797 op/s; 15993/214284 objects degraded (7.463%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T19:19:53.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:52 smithi016 bash[77995]: audit 2023-09-23T19:19:52.609944+0000 mon.c (mon.2) 28 : audit 0 from='client.? 172.21.15.16:0/668395296' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:53.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:52 smithi092 bash[58855]: cluster 2023-09-23T19:19:51.635811+0000 mgr.y (mgr.25325) 296 : cluster 0 pgmap v229: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 797 op/s; 15993/214284 objects degraded (7.463%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T19:19:53.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:52 smithi092 bash[58855]: audit 2023-09-23T19:19:52.609944+0000 mon.c (mon.2) 28 : audit 0 from='client.? 172.21.15.16:0/668395296' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:53.534 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:19:53 smithi016 bash[70992]: debug 2023-09-23T19:19:53.180+0000 7fb2a5a7c700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:19:54.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:53 smithi016 bash[75004]: audit 2023-09-23T19:19:53.184628+0000 mon.a (mon.0) 408 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:19:54.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:53 smithi016 bash[77995]: audit 2023-09-23T19:19:53.184628+0000 mon.a (mon.0) 408 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:19:54.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:53 smithi092 bash[58855]: audit 2023-09-23T19:19:53.184628+0000 mon.a (mon.0) 408 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:19:55.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:54 smithi016 bash[75004]: audit 2023-09-23T19:19:53.184894+0000 mgr.y (mgr.25325) 297 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:19:55.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:54 smithi016 bash[75004]: cephadm 2023-09-23T19:19:53.186421+0000 mgr.y (mgr.25325) 298 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:19:55.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:54 smithi016 bash[75004]: cluster 2023-09-23T19:19:53.636470+0000 mgr.y (mgr.25325) 299 : cluster 0 pgmap v230: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 524 op/s; 15993/214284 objects degraded (7.463%); 682 KiB/s, 10 objects/s recovering 2023-09-23T19:19:55.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:54 smithi016 bash[75004]: cluster 2023-09-23T19:19:54.021845+0000 mon.a (mon.0) 409 : cluster 3 Health check update: Degraded data redundancy: 15993/214284 objects degraded (7.463%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:19:55.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:54 smithi016 bash[77995]: audit 2023-09-23T19:19:53.184894+0000 mgr.y (mgr.25325) 297 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:19:55.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:54 smithi016 bash[77995]: cephadm 2023-09-23T19:19:53.186421+0000 mgr.y (mgr.25325) 298 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:19:55.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:54 smithi016 bash[77995]: cluster 2023-09-23T19:19:53.636470+0000 mgr.y (mgr.25325) 299 : cluster 0 pgmap v230: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 524 op/s; 15993/214284 objects degraded (7.463%); 682 KiB/s, 10 objects/s recovering 2023-09-23T19:19:55.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:54 smithi016 bash[77995]: cluster 2023-09-23T19:19:54.021845+0000 mon.a (mon.0) 409 : cluster 3 Health check update: Degraded data redundancy: 15993/214284 objects degraded (7.463%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:19:55.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:54 smithi092 bash[58855]: audit 2023-09-23T19:19:53.184894+0000 mgr.y (mgr.25325) 297 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:19:55.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:54 smithi092 bash[58855]: cephadm 2023-09-23T19:19:53.186421+0000 mgr.y (mgr.25325) 298 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:19:55.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:54 smithi092 bash[58855]: cluster 2023-09-23T19:19:53.636470+0000 mgr.y (mgr.25325) 299 : cluster 0 pgmap v230: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 524 op/s; 15993/214284 objects degraded (7.463%); 682 KiB/s, 10 objects/s recovering 2023-09-23T19:19:55.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:54 smithi092 bash[58855]: cluster 2023-09-23T19:19:54.021845+0000 mon.a (mon.0) 409 : cluster 3 Health check update: Degraded data redundancy: 15993/214284 objects degraded (7.463%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:19:56.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:55 smithi016 bash[75004]: audit 2023-09-23T19:19:55.250690+0000 mon.c (mon.2) 29 : audit 0 from='client.? 172.21.15.16:0/1362111939' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:56.035 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:19:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:19:55] "GET /metrics HTTP/1.1" 200 36207 "" "Prometheus/2.33.4" 2023-09-23T19:19:56.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:55 smithi016 bash[77995]: audit 2023-09-23T19:19:55.250690+0000 mon.c (mon.2) 29 : audit 0 from='client.? 172.21.15.16:0/1362111939' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:56.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:55 smithi092 bash[58855]: audit 2023-09-23T19:19:55.250690+0000 mon.c (mon.2) 29 : audit 0 from='client.? 172.21.15.16:0/1362111939' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:57.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:56 smithi016 bash[75004]: cluster 2023-09-23T19:19:55.637280+0000 mgr.y (mgr.25325) 300 : cluster 0 pgmap v231: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 15993/207864 objects degraded (7.694%); 682 KiB/s, 10 objects/s recovering 2023-09-23T19:19:57.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:56 smithi016 bash[77995]: cluster 2023-09-23T19:19:55.637280+0000 mgr.y (mgr.25325) 300 : cluster 0 pgmap v231: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 15993/207864 objects degraded (7.694%); 682 KiB/s, 10 objects/s recovering 2023-09-23T19:19:57.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:56 smithi092 bash[58855]: cluster 2023-09-23T19:19:55.637280+0000 mgr.y (mgr.25325) 300 : cluster 0 pgmap v231: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 15993/207864 objects degraded (7.694%); 682 KiB/s, 10 objects/s recovering 2023-09-23T19:19:59.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:58 smithi016 bash[75004]: cluster 2023-09-23T19:19:57.637988+0000 mgr.y (mgr.25325) 301 : cluster 0 pgmap v232: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 15927/204990 objects degraded (7.770%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T19:19:59.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:58 smithi016 bash[75004]: audit 2023-09-23T19:19:57.876986+0000 mon.b (mon.1) 16 : audit 0 from='client.? 172.21.15.16:0/220569485' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:59.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:58 smithi016 bash[75004]: audit 2023-09-23T19:19:58.619427+0000 mon.a (mon.0) 410 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:19:59.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:58 smithi016 bash[77995]: cluster 2023-09-23T19:19:57.637988+0000 mgr.y (mgr.25325) 301 : cluster 0 pgmap v232: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 15927/204990 objects degraded (7.770%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T19:19:59.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:58 smithi016 bash[77995]: audit 2023-09-23T19:19:57.876986+0000 mon.b (mon.1) 16 : audit 0 from='client.? 172.21.15.16:0/220569485' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:59.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:58 smithi016 bash[77995]: audit 2023-09-23T19:19:58.619427+0000 mon.a (mon.0) 410 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:19:59.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:58 smithi092 bash[58855]: cluster 2023-09-23T19:19:57.637988+0000 mgr.y (mgr.25325) 301 : cluster 0 pgmap v232: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 15927/204990 objects degraded (7.770%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T19:19:59.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:58 smithi092 bash[58855]: audit 2023-09-23T19:19:57.876986+0000 mon.b (mon.1) 16 : audit 0 from='client.? 172.21.15.16:0/220569485' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:19:59.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:58 smithi092 bash[58855]: audit 2023-09-23T19:19:58.619427+0000 mon.a (mon.0) 410 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:20:00.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:19:59 smithi016 bash[75004]: cluster 2023-09-23T19:19:59.023144+0000 mon.a (mon.0) 411 : cluster 3 Health check update: Degraded data redundancy: 15927/204990 objects degraded (7.770%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:20:00.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:19:59 smithi016 bash[77995]: cluster 2023-09-23T19:19:59.023144+0000 mon.a (mon.0) 411 : cluster 3 Health check update: Degraded data redundancy: 15927/204990 objects degraded (7.770%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:20:00.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:19:59 smithi092 bash[58855]: cluster 2023-09-23T19:19:59.023144+0000 mon.a (mon.0) 411 : cluster 3 Health check update: Degraded data redundancy: 15927/204990 objects degraded (7.770%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:20:01.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:00 smithi016 bash[75004]: cluster 2023-09-23T19:19:59.638563+0000 mgr.y (mgr.25325) 302 : cluster 0 pgmap v233: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 518 op/s; 15927/204990 objects degraded (7.770%); 704 KiB/s, 10 objects/s recovering 2023-09-23T19:20:01.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:00 smithi016 bash[75004]: cluster 2023-09-23T19:20:00.001322+0000 mon.a (mon.0) 412 : cluster 3 Health detail: HEALTH_WARN Degraded data redundancy: 15927/204990 objects degraded (7.770%), 8 pgs degraded, 8 pgs undersized 2023-09-23T19:20:01.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:00 smithi016 bash[75004]: cluster 2023-09-23T19:20:00.001366+0000 mon.a (mon.0) 413 : cluster 3 [WRN] PG_DEGRADED: Degraded data redundancy: 15927/204990 objects degraded (7.770%), 8 pgs degraded, 8 pgs undersized 2023-09-23T19:20:01.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:00 smithi016 bash[75004]: cluster 2023-09-23T19:20:00.001391+0000 mon.a (mon.0) 414 : cluster 3 pg 7.4 is stuck undersized for 107s, current state active+recovery_wait+undersized+degraded+remapped, last acting [1,4] 2023-09-23T19:20:01.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:00 smithi016 bash[75004]: cluster 2023-09-23T19:20:00.001411+0000 mon.a (mon.0) 415 : cluster 3 pg 7.5 is stuck undersized for 107s, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,2] 2023-09-23T19:20:01.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:00 smithi016 bash[75004]: cluster 2023-09-23T19:20:00.001430+0000 mon.a (mon.0) 416 : cluster 3 pg 7.6 is stuck undersized for 107s, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,7] 2023-09-23T19:20:01.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:00 smithi016 bash[75004]: cluster 2023-09-23T19:20:00.001450+0000 mon.a (mon.0) 417 : cluster 3 pg 7.7 is stuck undersized for 107s, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,3] 2023-09-23T19:20:01.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:00 smithi016 bash[75004]: cluster 2023-09-23T19:20:00.001472+0000 mon.a (mon.0) 418 : cluster 3 pg 7.9 is stuck undersized for 107s, current state active+recovery_wait+undersized+degraded+remapped, last acting [1,4] 2023-09-23T19:20:01.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:00 smithi016 bash[75004]: cluster 2023-09-23T19:20:00.001516+0000 mon.a (mon.0) 419 : cluster 3 pg 7.d is stuck undersized for 107s, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,1] 2023-09-23T19:20:01.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:00 smithi016 bash[75004]: cluster 2023-09-23T19:20:00.001547+0000 mon.a (mon.0) 420 : cluster 3 pg 7.e is stuck undersized for 107s, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,5] 2023-09-23T19:20:01.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:00 smithi016 bash[75004]: cluster 2023-09-23T19:20:00.001568+0000 mon.a (mon.0) 421 : cluster 3 pg 7.f is stuck undersized for 107s, current state active+recovering+undersized+degraded+remapped, last acting [3,7] 2023-09-23T19:20:01.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:00 smithi016 bash[75004]: audit 2023-09-23T19:20:00.512720+0000 mon.a (mon.0) 422 : audit 0 from='client.? 172.21.15.16:0/2869600970' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:01.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:00 smithi016 bash[77995]: cluster 2023-09-23T19:19:59.638563+0000 mgr.y (mgr.25325) 302 : cluster 0 pgmap v233: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 518 op/s; 15927/204990 objects degraded (7.770%); 704 KiB/s, 10 objects/s recovering 2023-09-23T19:20:01.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:00 smithi016 bash[77995]: cluster 2023-09-23T19:20:00.001322+0000 mon.a (mon.0) 412 : cluster 3 Health detail: HEALTH_WARN Degraded data redundancy: 15927/204990 objects degraded (7.770%), 8 pgs degraded, 8 pgs undersized 2023-09-23T19:20:01.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:00 smithi016 bash[77995]: cluster 2023-09-23T19:20:00.001366+0000 mon.a (mon.0) 413 : cluster 3 [WRN] PG_DEGRADED: Degraded data redundancy: 15927/204990 objects degraded (7.770%), 8 pgs degraded, 8 pgs undersized 2023-09-23T19:20:01.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:00 smithi016 bash[77995]: cluster 2023-09-23T19:20:00.001391+0000 mon.a (mon.0) 414 : cluster 3 pg 7.4 is stuck undersized for 107s, current state active+recovery_wait+undersized+degraded+remapped, last acting [1,4] 2023-09-23T19:20:01.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:00 smithi016 bash[77995]: cluster 2023-09-23T19:20:00.001411+0000 mon.a (mon.0) 415 : cluster 3 pg 7.5 is stuck undersized for 107s, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,2] 2023-09-23T19:20:01.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:00 smithi016 bash[77995]: cluster 2023-09-23T19:20:00.001430+0000 mon.a (mon.0) 416 : cluster 3 pg 7.6 is stuck undersized for 107s, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,7] 2023-09-23T19:20:01.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:00 smithi016 bash[77995]: cluster 2023-09-23T19:20:00.001450+0000 mon.a (mon.0) 417 : cluster 3 pg 7.7 is stuck undersized for 107s, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,3] 2023-09-23T19:20:01.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:00 smithi016 bash[77995]: cluster 2023-09-23T19:20:00.001472+0000 mon.a (mon.0) 418 : cluster 3 pg 7.9 is stuck undersized for 107s, current state active+recovery_wait+undersized+degraded+remapped, last acting [1,4] 2023-09-23T19:20:01.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:00 smithi016 bash[77995]: cluster 2023-09-23T19:20:00.001516+0000 mon.a (mon.0) 419 : cluster 3 pg 7.d is stuck undersized for 107s, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,1] 2023-09-23T19:20:01.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:00 smithi016 bash[77995]: cluster 2023-09-23T19:20:00.001547+0000 mon.a (mon.0) 420 : cluster 3 pg 7.e is stuck undersized for 107s, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,5] 2023-09-23T19:20:01.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:00 smithi016 bash[77995]: cluster 2023-09-23T19:20:00.001568+0000 mon.a (mon.0) 421 : cluster 3 pg 7.f is stuck undersized for 107s, current state active+recovering+undersized+degraded+remapped, last acting [3,7] 2023-09-23T19:20:01.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:00 smithi016 bash[77995]: audit 2023-09-23T19:20:00.512720+0000 mon.a (mon.0) 422 : audit 0 from='client.? 172.21.15.16:0/2869600970' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:01.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:00 smithi092 bash[58855]: cluster 2023-09-23T19:19:59.638563+0000 mgr.y (mgr.25325) 302 : cluster 0 pgmap v233: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 518 op/s; 15927/204990 objects degraded (7.770%); 704 KiB/s, 10 objects/s recovering 2023-09-23T19:20:01.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:00 smithi092 bash[58855]: cluster 2023-09-23T19:20:00.001322+0000 mon.a (mon.0) 412 : cluster 3 Health detail: HEALTH_WARN Degraded data redundancy: 15927/204990 objects degraded (7.770%), 8 pgs degraded, 8 pgs undersized 2023-09-23T19:20:01.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:00 smithi092 bash[58855]: cluster 2023-09-23T19:20:00.001366+0000 mon.a (mon.0) 413 : cluster 3 [WRN] PG_DEGRADED: Degraded data redundancy: 15927/204990 objects degraded (7.770%), 8 pgs degraded, 8 pgs undersized 2023-09-23T19:20:01.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:00 smithi092 bash[58855]: cluster 2023-09-23T19:20:00.001391+0000 mon.a (mon.0) 414 : cluster 3 pg 7.4 is stuck undersized for 107s, current state active+recovery_wait+undersized+degraded+remapped, last acting [1,4] 2023-09-23T19:20:01.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:00 smithi092 bash[58855]: cluster 2023-09-23T19:20:00.001411+0000 mon.a (mon.0) 415 : cluster 3 pg 7.5 is stuck undersized for 107s, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,2] 2023-09-23T19:20:01.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:00 smithi092 bash[58855]: cluster 2023-09-23T19:20:00.001430+0000 mon.a (mon.0) 416 : cluster 3 pg 7.6 is stuck undersized for 107s, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,7] 2023-09-23T19:20:01.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:00 smithi092 bash[58855]: cluster 2023-09-23T19:20:00.001450+0000 mon.a (mon.0) 417 : cluster 3 pg 7.7 is stuck undersized for 107s, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,3] 2023-09-23T19:20:01.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:00 smithi092 bash[58855]: cluster 2023-09-23T19:20:00.001472+0000 mon.a (mon.0) 418 : cluster 3 pg 7.9 is stuck undersized for 107s, current state active+recovery_wait+undersized+degraded+remapped, last acting [1,4] 2023-09-23T19:20:01.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:00 smithi092 bash[58855]: cluster 2023-09-23T19:20:00.001516+0000 mon.a (mon.0) 419 : cluster 3 pg 7.d is stuck undersized for 107s, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,1] 2023-09-23T19:20:01.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:00 smithi092 bash[58855]: cluster 2023-09-23T19:20:00.001547+0000 mon.a (mon.0) 420 : cluster 3 pg 7.e is stuck undersized for 107s, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,5] 2023-09-23T19:20:01.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:00 smithi092 bash[58855]: cluster 2023-09-23T19:20:00.001568+0000 mon.a (mon.0) 421 : cluster 3 pg 7.f is stuck undersized for 107s, current state active+recovering+undersized+degraded+remapped, last acting [3,7] 2023-09-23T19:20:01.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:00 smithi092 bash[58855]: audit 2023-09-23T19:20:00.512720+0000 mon.a (mon.0) 422 : audit 0 from='client.? 172.21.15.16:0/2869600970' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:03.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:02 smithi016 bash[75004]: cluster 2023-09-23T19:20:01.639316+0000 mgr.y (mgr.25325) 303 : cluster 0 pgmap v234: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.0 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 776 op/s; 15863/195675 objects degraded (8.107%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T19:20:03.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:02 smithi016 bash[77995]: cluster 2023-09-23T19:20:01.639316+0000 mgr.y (mgr.25325) 303 : cluster 0 pgmap v234: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.0 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 776 op/s; 15863/195675 objects degraded (8.107%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T19:20:03.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:02 smithi092 bash[58855]: cluster 2023-09-23T19:20:01.639316+0000 mgr.y (mgr.25325) 303 : cluster 0 pgmap v234: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.0 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 776 op/s; 15863/195675 objects degraded (8.107%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T19:20:04.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:03 smithi016 bash[75004]: audit 2023-09-23T19:20:03.145912+0000 mon.c (mon.2) 30 : audit 0 from='client.? 172.21.15.16:0/815166618' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:04.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:03 smithi016 bash[77995]: audit 2023-09-23T19:20:03.145912+0000 mon.c (mon.2) 30 : audit 0 from='client.? 172.21.15.16:0/815166618' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:04.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:03 smithi092 bash[58855]: audit 2023-09-23T19:20:03.145912+0000 mon.c (mon.2) 30 : audit 0 from='client.? 172.21.15.16:0/815166618' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:05.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:04 smithi016 bash[75004]: cluster 2023-09-23T19:20:03.641039+0000 mgr.y (mgr.25325) 304 : cluster 0 pgmap v235: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.0 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 516 op/s; 15863/195675 objects degraded (8.107%); 693 KiB/s, 10 objects/s recovering 2023-09-23T19:20:05.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:04 smithi016 bash[75004]: cluster 2023-09-23T19:20:04.024261+0000 mon.a (mon.0) 423 : cluster 3 Health check update: Degraded data redundancy: 15863/195675 objects degraded (8.107%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:20:05.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:04 smithi016 bash[77995]: cluster 2023-09-23T19:20:03.641039+0000 mgr.y (mgr.25325) 304 : cluster 0 pgmap v235: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.0 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 516 op/s; 15863/195675 objects degraded (8.107%); 693 KiB/s, 10 objects/s recovering 2023-09-23T19:20:05.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:04 smithi016 bash[77995]: cluster 2023-09-23T19:20:04.024261+0000 mon.a (mon.0) 423 : cluster 3 Health check update: Degraded data redundancy: 15863/195675 objects degraded (8.107%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:20:05.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:04 smithi092 bash[58855]: cluster 2023-09-23T19:20:03.641039+0000 mgr.y (mgr.25325) 304 : cluster 0 pgmap v235: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.0 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 516 op/s; 15863/195675 objects degraded (8.107%); 693 KiB/s, 10 objects/s recovering 2023-09-23T19:20:05.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:04 smithi092 bash[58855]: cluster 2023-09-23T19:20:04.024261+0000 mon.a (mon.0) 423 : cluster 3 Health check update: Degraded data redundancy: 15863/195675 objects degraded (8.107%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:20:06.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:20:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:20:05] "GET /metrics HTTP/1.1" 200 36209 "" "Prometheus/2.33.4" 2023-09-23T19:20:07.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:06 smithi016 bash[77995]: cluster 2023-09-23T19:20:05.649368+0000 mgr.y (mgr.25325) 305 : cluster 0 pgmap v236: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 15863/189261 objects degraded (8.382%); 693 KiB/s, 10 objects/s recovering 2023-09-23T19:20:07.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:06 smithi016 bash[77995]: audit 2023-09-23T19:20:05.750612+0000 mon.c (mon.2) 31 : audit 0 from='client.? 172.21.15.16:0/363711419' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:07.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:06 smithi016 bash[75004]: cluster 2023-09-23T19:20:05.649368+0000 mgr.y (mgr.25325) 305 : cluster 0 pgmap v236: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 15863/189261 objects degraded (8.382%); 693 KiB/s, 10 objects/s recovering 2023-09-23T19:20:07.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:06 smithi016 bash[75004]: audit 2023-09-23T19:20:05.750612+0000 mon.c (mon.2) 31 : audit 0 from='client.? 172.21.15.16:0/363711419' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:07.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:06 smithi092 bash[58855]: cluster 2023-09-23T19:20:05.649368+0000 mgr.y (mgr.25325) 305 : cluster 0 pgmap v236: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 15863/189261 objects degraded (8.382%); 693 KiB/s, 10 objects/s recovering 2023-09-23T19:20:07.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:06 smithi092 bash[58855]: audit 2023-09-23T19:20:05.750612+0000 mon.c (mon.2) 31 : audit 0 from='client.? 172.21.15.16:0/363711419' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:08.534 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:20:08 smithi016 bash[70992]: debug 2023-09-23T19:20:08.200+0000 7fb2a5a7c700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:20:09.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:08 smithi016 bash[75004]: cluster 2023-09-23T19:20:07.650215+0000 mgr.y (mgr.25325) 306 : cluster 0 pgmap v237: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 595 op/s; 15796/186387 objects degraded (8.475%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T19:20:09.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:08 smithi016 bash[75004]: audit 2023-09-23T19:20:08.201765+0000 mon.a (mon.0) 424 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:20:09.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:08 smithi016 bash[75004]: audit 2023-09-23T19:20:08.374038+0000 mon.a (mon.0) 425 : audit 0 from='client.? 172.21.15.16:0/3015286424' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:09.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:08 smithi016 bash[77995]: cluster 2023-09-23T19:20:07.650215+0000 mgr.y (mgr.25325) 306 : cluster 0 pgmap v237: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 595 op/s; 15796/186387 objects degraded (8.475%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T19:20:09.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:08 smithi016 bash[77995]: audit 2023-09-23T19:20:08.201765+0000 mon.a (mon.0) 424 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:20:09.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:08 smithi016 bash[77995]: audit 2023-09-23T19:20:08.374038+0000 mon.a (mon.0) 425 : audit 0 from='client.? 172.21.15.16:0/3015286424' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:09.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:08 smithi092 bash[58855]: cluster 2023-09-23T19:20:07.650215+0000 mgr.y (mgr.25325) 306 : cluster 0 pgmap v237: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 595 op/s; 15796/186387 objects degraded (8.475%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T19:20:09.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:08 smithi092 bash[58855]: audit 2023-09-23T19:20:08.201765+0000 mon.a (mon.0) 424 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:20:09.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:08 smithi092 bash[58855]: audit 2023-09-23T19:20:08.374038+0000 mon.a (mon.0) 425 : audit 0 from='client.? 172.21.15.16:0/3015286424' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:10.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:09 smithi016 bash[75004]: audit 2023-09-23T19:20:08.202066+0000 mgr.y (mgr.25325) 307 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:20:10.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:09 smithi016 bash[75004]: cephadm 2023-09-23T19:20:08.204073+0000 mgr.y (mgr.25325) 308 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:20:10.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:09 smithi016 bash[75004]: cluster 2023-09-23T19:20:09.028577+0000 mon.a (mon.0) 426 : cluster 3 Health check update: Degraded data redundancy: 15796/186387 objects degraded (8.475%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:20:10.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:09 smithi016 bash[75004]: cluster 2023-09-23T19:20:09.650820+0000 mgr.y (mgr.25325) 309 : cluster 0 pgmap v238: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 516 op/s; 15796/186387 objects degraded (8.475%); 698 KiB/s, 10 objects/s recovering 2023-09-23T19:20:10.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:09 smithi016 bash[77995]: audit 2023-09-23T19:20:08.202066+0000 mgr.y (mgr.25325) 307 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:20:10.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:09 smithi016 bash[77995]: cephadm 2023-09-23T19:20:08.204073+0000 mgr.y (mgr.25325) 308 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:20:10.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:09 smithi016 bash[77995]: cluster 2023-09-23T19:20:09.028577+0000 mon.a (mon.0) 426 : cluster 3 Health check update: Degraded data redundancy: 15796/186387 objects degraded (8.475%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:20:10.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:09 smithi016 bash[77995]: cluster 2023-09-23T19:20:09.650820+0000 mgr.y (mgr.25325) 309 : cluster 0 pgmap v238: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 516 op/s; 15796/186387 objects degraded (8.475%); 698 KiB/s, 10 objects/s recovering 2023-09-23T19:20:10.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:09 smithi092 bash[58855]: audit 2023-09-23T19:20:08.202066+0000 mgr.y (mgr.25325) 307 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:20:10.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:09 smithi092 bash[58855]: cephadm 2023-09-23T19:20:08.204073+0000 mgr.y (mgr.25325) 308 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:20:10.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:09 smithi092 bash[58855]: cluster 2023-09-23T19:20:09.028577+0000 mon.a (mon.0) 426 : cluster 3 Health check update: Degraded data redundancy: 15796/186387 objects degraded (8.475%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:20:10.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:09 smithi092 bash[58855]: cluster 2023-09-23T19:20:09.650820+0000 mgr.y (mgr.25325) 309 : cluster 0 pgmap v238: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 516 op/s; 15796/186387 objects degraded (8.475%); 698 KiB/s, 10 objects/s recovering 2023-09-23T19:20:11.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:11 smithi092 bash[58855]: audit 2023-09-23T19:20:10.996957+0000 mon.a (mon.0) 427 : audit 0 from='client.? 172.21.15.16:0/582597156' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:11.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:11 smithi016 bash[77995]: audit 2023-09-23T19:20:10.996957+0000 mon.a (mon.0) 427 : audit 0 from='client.? 172.21.15.16:0/582597156' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:11.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:11 smithi016 bash[75004]: audit 2023-09-23T19:20:10.996957+0000 mon.a (mon.0) 427 : audit 0 from='client.? 172.21.15.16:0/582597156' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:12.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:12 smithi016 bash[75004]: cluster 2023-09-23T19:20:11.651697+0000 mgr.y (mgr.25325) 310 : cluster 0 pgmap v239: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 15724/176718 objects degraded (8.898%); 1.1 MiB/s, 16 objects/s recovering 2023-09-23T19:20:12.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:12 smithi016 bash[77995]: cluster 2023-09-23T19:20:11.651697+0000 mgr.y (mgr.25325) 310 : cluster 0 pgmap v239: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 15724/176718 objects degraded (8.898%); 1.1 MiB/s, 16 objects/s recovering 2023-09-23T19:20:12.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:12 smithi092 bash[58855]: cluster 2023-09-23T19:20:11.651697+0000 mgr.y (mgr.25325) 310 : cluster 0 pgmap v239: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 15724/176718 objects degraded (8.898%); 1.1 MiB/s, 16 objects/s recovering 2023-09-23T19:20:14.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:13 smithi016 bash[75004]: audit 2023-09-23T19:20:13.600297+0000 mon.a (mon.0) 428 : audit 0 from='client.? 172.21.15.16:0/798387752' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:14.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:13 smithi016 bash[75004]: audit 2023-09-23T19:20:13.620677+0000 mon.a (mon.0) 429 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:20:14.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:13 smithi016 bash[77995]: audit 2023-09-23T19:20:13.600297+0000 mon.a (mon.0) 428 : audit 0 from='client.? 172.21.15.16:0/798387752' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:14.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:13 smithi016 bash[77995]: audit 2023-09-23T19:20:13.620677+0000 mon.a (mon.0) 429 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:20:14.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:13 smithi092 bash[58855]: audit 2023-09-23T19:20:13.600297+0000 mon.a (mon.0) 428 : audit 0 from='client.? 172.21.15.16:0/798387752' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:14.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:13 smithi092 bash[58855]: audit 2023-09-23T19:20:13.620677+0000 mon.a (mon.0) 429 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:20:15.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:14 smithi016 bash[75004]: cluster 2023-09-23T19:20:13.652241+0000 mgr.y (mgr.25325) 311 : cluster 0 pgmap v240: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 526 op/s; 15724/176718 objects degraded (8.898%); 741 KiB/s, 11 objects/s recovering 2023-09-23T19:20:15.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:14 smithi016 bash[75004]: cluster 2023-09-23T19:20:14.029683+0000 mon.a (mon.0) 430 : cluster 3 Health check update: Degraded data redundancy: 15724/176718 objects degraded (8.898%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:20:15.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:14 smithi016 bash[77995]: cluster 2023-09-23T19:20:13.652241+0000 mgr.y (mgr.25325) 311 : cluster 0 pgmap v240: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 526 op/s; 15724/176718 objects degraded (8.898%); 741 KiB/s, 11 objects/s recovering 2023-09-23T19:20:15.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:14 smithi016 bash[77995]: cluster 2023-09-23T19:20:14.029683+0000 mon.a (mon.0) 430 : cluster 3 Health check update: Degraded data redundancy: 15724/176718 objects degraded (8.898%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:20:15.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:14 smithi092 bash[58855]: cluster 2023-09-23T19:20:13.652241+0000 mgr.y (mgr.25325) 311 : cluster 0 pgmap v240: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 526 op/s; 15724/176718 objects degraded (8.898%); 741 KiB/s, 11 objects/s recovering 2023-09-23T19:20:15.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:14 smithi092 bash[58855]: cluster 2023-09-23T19:20:14.029683+0000 mon.a (mon.0) 430 : cluster 3 Health check update: Degraded data redundancy: 15724/176718 objects degraded (8.898%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:20:16.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:20:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:20:15] "GET /metrics HTTP/1.1" 200 36204 "" "Prometheus/2.33.4" 2023-09-23T19:20:17.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:16 smithi016 bash[75004]: cluster 2023-09-23T19:20:15.653057+0000 mgr.y (mgr.25325) 312 : cluster 0 pgmap v241: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 703 op/s; 15724/170334 objects degraded (9.231%); 741 KiB/s, 11 objects/s recovering 2023-09-23T19:20:17.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:16 smithi016 bash[75004]: audit 2023-09-23T19:20:16.246010+0000 mon.c (mon.2) 32 : audit 0 from='client.? 172.21.15.16:0/2622074266' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:17.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:16 smithi016 bash[77995]: cluster 2023-09-23T19:20:15.653057+0000 mgr.y (mgr.25325) 312 : cluster 0 pgmap v241: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 703 op/s; 15724/170334 objects degraded (9.231%); 741 KiB/s, 11 objects/s recovering 2023-09-23T19:20:17.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:16 smithi016 bash[77995]: audit 2023-09-23T19:20:16.246010+0000 mon.c (mon.2) 32 : audit 0 from='client.? 172.21.15.16:0/2622074266' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:17.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:16 smithi092 bash[58855]: cluster 2023-09-23T19:20:15.653057+0000 mgr.y (mgr.25325) 312 : cluster 0 pgmap v241: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 703 op/s; 15724/170334 objects degraded (9.231%); 741 KiB/s, 11 objects/s recovering 2023-09-23T19:20:17.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:16 smithi092 bash[58855]: audit 2023-09-23T19:20:16.246010+0000 mon.c (mon.2) 32 : audit 0 from='client.? 172.21.15.16:0/2622074266' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:19.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:18 smithi016 bash[77995]: cluster 2023-09-23T19:20:17.656781+0000 mgr.y (mgr.25325) 313 : cluster 0 pgmap v242: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 15656/167448 objects degraded (9.350%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:20:19.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:18 smithi016 bash[75004]: cluster 2023-09-23T19:20:17.656781+0000 mgr.y (mgr.25325) 313 : cluster 0 pgmap v242: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 15656/167448 objects degraded (9.350%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:20:19.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:18 smithi092 bash[58855]: cluster 2023-09-23T19:20:17.656781+0000 mgr.y (mgr.25325) 313 : cluster 0 pgmap v242: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 15656/167448 objects degraded (9.350%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:20:20.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:19 smithi016 bash[77995]: audit 2023-09-23T19:20:18.916365+0000 mon.c (mon.2) 33 : audit 0 from='client.? 172.21.15.16:0/2191601612' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:20.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:19 smithi016 bash[77995]: cluster 2023-09-23T19:20:19.030981+0000 mon.a (mon.0) 431 : cluster 3 Health check update: Degraded data redundancy: 15656/167448 objects degraded (9.350%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:20:20.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:19 smithi016 bash[75004]: audit 2023-09-23T19:20:18.916365+0000 mon.c (mon.2) 33 : audit 0 from='client.? 172.21.15.16:0/2191601612' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:20.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:19 smithi016 bash[75004]: cluster 2023-09-23T19:20:19.030981+0000 mon.a (mon.0) 431 : cluster 3 Health check update: Degraded data redundancy: 15656/167448 objects degraded (9.350%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:20:20.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:19 smithi092 bash[58855]: audit 2023-09-23T19:20:18.916365+0000 mon.c (mon.2) 33 : audit 0 from='client.? 172.21.15.16:0/2191601612' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:20.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:19 smithi092 bash[58855]: cluster 2023-09-23T19:20:19.030981+0000 mon.a (mon.0) 431 : cluster 3 Health check update: Degraded data redundancy: 15656/167448 objects degraded (9.350%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:20:21.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:20 smithi016 bash[75004]: cluster 2023-09-23T19:20:19.660194+0000 mgr.y (mgr.25325) 314 : cluster 0 pgmap v243: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 525 op/s; 15656/167448 objects degraded (9.350%); 746 KiB/s, 11 objects/s recovering 2023-09-23T19:20:21.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:20 smithi016 bash[77995]: cluster 2023-09-23T19:20:19.660194+0000 mgr.y (mgr.25325) 314 : cluster 0 pgmap v243: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 525 op/s; 15656/167448 objects degraded (9.350%); 746 KiB/s, 11 objects/s recovering 2023-09-23T19:20:21.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:20 smithi092 bash[58855]: cluster 2023-09-23T19:20:19.660194+0000 mgr.y (mgr.25325) 314 : cluster 0 pgmap v243: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 525 op/s; 15656/167448 objects degraded (9.350%); 746 KiB/s, 11 objects/s recovering 2023-09-23T19:20:22.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:21 smithi016 bash[75004]: audit 2023-09-23T19:20:21.535922+0000 mon.a (mon.0) 432 : audit 0 from='client.? 172.21.15.16:0/2289781656' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:22.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:21 smithi016 bash[77995]: audit 2023-09-23T19:20:21.535922+0000 mon.a (mon.0) 432 : audit 0 from='client.? 172.21.15.16:0/2289781656' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:22.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:21 smithi092 bash[58855]: audit 2023-09-23T19:20:21.535922+0000 mon.a (mon.0) 432 : audit 0 from='client.? 172.21.15.16:0/2289781656' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:23.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:22 smithi016 bash[77995]: cluster 2023-09-23T19:20:21.661158+0000 mgr.y (mgr.25325) 315 : cluster 0 pgmap v244: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 15584/158292 objects degraded (9.845%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:20:23.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:22 smithi016 bash[75004]: cluster 2023-09-23T19:20:21.661158+0000 mgr.y (mgr.25325) 315 : cluster 0 pgmap v244: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 15584/158292 objects degraded (9.845%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:20:23.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:22 smithi092 bash[58855]: cluster 2023-09-23T19:20:21.661158+0000 mgr.y (mgr.25325) 315 : cluster 0 pgmap v244: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 15584/158292 objects degraded (9.845%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:20:24.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:23 smithi016 bash[75004]: audit 2023-09-23T19:20:23.221191+0000 mon.a (mon.0) 433 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:20:24.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:23 smithi016 bash[77995]: audit 2023-09-23T19:20:23.221191+0000 mon.a (mon.0) 433 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:20:24.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:23 smithi092 bash[58855]: audit 2023-09-23T19:20:23.221191+0000 mon.a (mon.0) 433 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:20:25.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:24 smithi016 bash[75004]: cluster 2023-09-23T19:20:23.661802+0000 mgr.y (mgr.25325) 316 : cluster 0 pgmap v245: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 511 op/s; 15584/158292 objects degraded (9.845%); 746 KiB/s, 11 objects/s recovering 2023-09-23T19:20:25.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:24 smithi016 bash[75004]: cluster 2023-09-23T19:20:24.032195+0000 mon.a (mon.0) 434 : cluster 3 Health check update: Degraded data redundancy: 15584/158292 objects degraded (9.845%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:20:25.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:24 smithi016 bash[75004]: audit 2023-09-23T19:20:24.146138+0000 mon.c (mon.2) 34 : audit 0 from='client.? 172.21.15.16:0/1675548674' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:25.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:24 smithi016 bash[77995]: cluster 2023-09-23T19:20:23.661802+0000 mgr.y (mgr.25325) 316 : cluster 0 pgmap v245: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 511 op/s; 15584/158292 objects degraded (9.845%); 746 KiB/s, 11 objects/s recovering 2023-09-23T19:20:25.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:24 smithi016 bash[77995]: cluster 2023-09-23T19:20:24.032195+0000 mon.a (mon.0) 434 : cluster 3 Health check update: Degraded data redundancy: 15584/158292 objects degraded (9.845%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:20:25.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:24 smithi016 bash[77995]: audit 2023-09-23T19:20:24.146138+0000 mon.c (mon.2) 34 : audit 0 from='client.? 172.21.15.16:0/1675548674' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:25.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:24 smithi092 bash[58855]: cluster 2023-09-23T19:20:23.661802+0000 mgr.y (mgr.25325) 316 : cluster 0 pgmap v245: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 511 op/s; 15584/158292 objects degraded (9.845%); 746 KiB/s, 11 objects/s recovering 2023-09-23T19:20:25.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:24 smithi092 bash[58855]: cluster 2023-09-23T19:20:24.032195+0000 mon.a (mon.0) 434 : cluster 3 Health check update: Degraded data redundancy: 15584/158292 objects degraded (9.845%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:20:25.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:24 smithi092 bash[58855]: audit 2023-09-23T19:20:24.146138+0000 mon.c (mon.2) 34 : audit 0 from='client.? 172.21.15.16:0/1675548674' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:26.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:20:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:20:25] "GET /metrics HTTP/1.1" 200 36204 "" "Prometheus/2.33.4" 2023-09-23T19:20:27.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:26 smithi016 bash[77995]: cluster 2023-09-23T19:20:25.662681+0000 mgr.y (mgr.25325) 317 : cluster 0 pgmap v246: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 15584/152259 objects degraded (10.235%); 746 KiB/s, 11 objects/s recovering 2023-09-23T19:20:27.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:26 smithi016 bash[75004]: cluster 2023-09-23T19:20:25.662681+0000 mgr.y (mgr.25325) 317 : cluster 0 pgmap v246: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 15584/152259 objects degraded (10.235%); 746 KiB/s, 11 objects/s recovering 2023-09-23T19:20:27.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:26 smithi092 bash[58855]: cluster 2023-09-23T19:20:25.662681+0000 mgr.y (mgr.25325) 317 : cluster 0 pgmap v246: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 15584/152259 objects degraded (10.235%); 746 KiB/s, 11 objects/s recovering 2023-09-23T19:20:28.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:27 smithi016 bash[75004]: audit 2023-09-23T19:20:26.838626+0000 mon.b (mon.1) 17 : audit 0 from='client.? 172.21.15.16:0/3064633509' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:28.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:27 smithi016 bash[77995]: audit 2023-09-23T19:20:26.838626+0000 mon.b (mon.1) 17 : audit 0 from='client.? 172.21.15.16:0/3064633509' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:28.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:27 smithi092 bash[58855]: audit 2023-09-23T19:20:26.838626+0000 mon.b (mon.1) 17 : audit 0 from='client.? 172.21.15.16:0/3064633509' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:29.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:28 smithi016 bash[75004]: cluster 2023-09-23T19:20:27.663422+0000 mgr.y (mgr.25325) 318 : cluster 0 pgmap v247: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 15520/149397 objects degraded (10.388%); 1.1 MiB/s, 16 objects/s recovering 2023-09-23T19:20:29.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:28 smithi016 bash[75004]: audit 2023-09-23T19:20:28.620426+0000 mon.a (mon.0) 435 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:20:29.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:28 smithi016 bash[77995]: cluster 2023-09-23T19:20:27.663422+0000 mgr.y (mgr.25325) 318 : cluster 0 pgmap v247: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 15520/149397 objects degraded (10.388%); 1.1 MiB/s, 16 objects/s recovering 2023-09-23T19:20:29.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:28 smithi016 bash[77995]: audit 2023-09-23T19:20:28.620426+0000 mon.a (mon.0) 435 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:20:29.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:28 smithi092 bash[58855]: cluster 2023-09-23T19:20:27.663422+0000 mgr.y (mgr.25325) 318 : cluster 0 pgmap v247: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 15520/149397 objects degraded (10.388%); 1.1 MiB/s, 16 objects/s recovering 2023-09-23T19:20:29.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:28 smithi092 bash[58855]: audit 2023-09-23T19:20:28.620426+0000 mon.a (mon.0) 435 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:20:30.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:29 smithi016 bash[77995]: cluster 2023-09-23T19:20:29.036204+0000 mon.a (mon.0) 436 : cluster 3 Health check update: Degraded data redundancy: 15520/149397 objects degraded (10.388%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:20:30.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:29 smithi016 bash[77995]: audit 2023-09-23T19:20:29.435768+0000 mon.a (mon.0) 437 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:20:30.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:29 smithi016 bash[77995]: audit 2023-09-23T19:20:29.453226+0000 mon.a (mon.0) 438 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:20:30.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:29 smithi016 bash[77995]: audit 2023-09-23T19:20:29.469410+0000 mon.c (mon.2) 35 : audit 0 from='client.? 172.21.15.16:0/651280419' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:30.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:29 smithi016 bash[77995]: cluster 2023-09-23T19:20:29.663994+0000 mgr.y (mgr.25325) 319 : cluster 0 pgmap v248: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 501 op/s; 15520/149397 objects degraded (10.388%); 725 KiB/s, 11 objects/s recovering 2023-09-23T19:20:30.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:29 smithi016 bash[75004]: cluster 2023-09-23T19:20:29.036204+0000 mon.a (mon.0) 436 : cluster 3 Health check update: Degraded data redundancy: 15520/149397 objects degraded (10.388%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:20:30.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:29 smithi016 bash[75004]: audit 2023-09-23T19:20:29.435768+0000 mon.a (mon.0) 437 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:20:30.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:29 smithi016 bash[75004]: audit 2023-09-23T19:20:29.453226+0000 mon.a (mon.0) 438 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:20:30.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:29 smithi016 bash[75004]: audit 2023-09-23T19:20:29.469410+0000 mon.c (mon.2) 35 : audit 0 from='client.? 172.21.15.16:0/651280419' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:30.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:29 smithi016 bash[75004]: cluster 2023-09-23T19:20:29.663994+0000 mgr.y (mgr.25325) 319 : cluster 0 pgmap v248: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 501 op/s; 15520/149397 objects degraded (10.388%); 725 KiB/s, 11 objects/s recovering 2023-09-23T19:20:30.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:29 smithi092 bash[58855]: cluster 2023-09-23T19:20:29.036204+0000 mon.a (mon.0) 436 : cluster 3 Health check update: Degraded data redundancy: 15520/149397 objects degraded (10.388%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:20:30.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:29 smithi092 bash[58855]: audit 2023-09-23T19:20:29.435768+0000 mon.a (mon.0) 437 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:20:30.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:29 smithi092 bash[58855]: audit 2023-09-23T19:20:29.453226+0000 mon.a (mon.0) 438 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:20:30.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:29 smithi092 bash[58855]: audit 2023-09-23T19:20:29.469410+0000 mon.c (mon.2) 35 : audit 0 from='client.? 172.21.15.16:0/651280419' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:30.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:29 smithi092 bash[58855]: cluster 2023-09-23T19:20:29.663994+0000 mgr.y (mgr.25325) 319 : cluster 0 pgmap v248: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 501 op/s; 15520/149397 objects degraded (10.388%); 725 KiB/s, 11 objects/s recovering 2023-09-23T19:20:31.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:30 smithi016 bash[75004]: audit 2023-09-23T19:20:29.884248+0000 mon.a (mon.0) 439 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:20:31.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:30 smithi016 bash[75004]: audit 2023-09-23T19:20:29.897280+0000 mon.a (mon.0) 440 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:20:31.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:30 smithi016 bash[75004]: audit 2023-09-23T19:20:30.600492+0000 mon.a (mon.0) 441 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:20:31.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:30 smithi016 bash[75004]: audit 2023-09-23T19:20:30.601941+0000 mon.a (mon.0) 442 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:20:31.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:30 smithi016 bash[75004]: audit 2023-09-23T19:20:30.607858+0000 mon.a (mon.0) 443 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:20:31.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:30 smithi016 bash[75004]: audit 2023-09-23T19:20:30.693454+0000 mon.a (mon.0) 444 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:20:31.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:30 smithi016 bash[75004]: audit 2023-09-23T19:20:30.697434+0000 mon.a (mon.0) 445 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:20:31.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:30 smithi016 bash[75004]: audit 2023-09-23T19:20:30.699967+0000 mon.a (mon.0) 446 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:20:31.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:30 smithi016 bash[75004]: audit 2023-09-23T19:20:30.705705+0000 mon.a (mon.0) 447 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:20:31.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:30 smithi016 bash[75004]: audit 2023-09-23T19:20:30.707758+0000 mon.a (mon.0) 448 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:20:31.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:30 smithi016 bash[75004]: audit 2023-09-23T19:20:30.708009+0000 mgr.y (mgr.25325) 320 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:20:31.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:30 smithi016 bash[75004]: cephadm 2023-09-23T19:20:30.709197+0000 mgr.y (mgr.25325) 321 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:20:31.035 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:20:30 smithi016 bash[70992]: debug 2023-09-23T19:20:30.703+0000 7fb2a5a7c700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:20:31.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:30 smithi016 bash[77995]: audit 2023-09-23T19:20:29.884248+0000 mon.a (mon.0) 439 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:20:31.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:30 smithi016 bash[77995]: audit 2023-09-23T19:20:29.897280+0000 mon.a (mon.0) 440 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:20:31.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:30 smithi016 bash[77995]: audit 2023-09-23T19:20:30.600492+0000 mon.a (mon.0) 441 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:20:31.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:30 smithi016 bash[77995]: audit 2023-09-23T19:20:30.601941+0000 mon.a (mon.0) 442 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:20:31.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:30 smithi016 bash[77995]: audit 2023-09-23T19:20:30.607858+0000 mon.a (mon.0) 443 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:20:31.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:30 smithi016 bash[77995]: audit 2023-09-23T19:20:30.693454+0000 mon.a (mon.0) 444 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:20:31.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:30 smithi016 bash[77995]: audit 2023-09-23T19:20:30.697434+0000 mon.a (mon.0) 445 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:20:31.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:30 smithi016 bash[77995]: audit 2023-09-23T19:20:30.699967+0000 mon.a (mon.0) 446 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:20:31.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:30 smithi016 bash[77995]: audit 2023-09-23T19:20:30.705705+0000 mon.a (mon.0) 447 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:20:31.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:30 smithi016 bash[77995]: audit 2023-09-23T19:20:30.707758+0000 mon.a (mon.0) 448 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:20:31.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:30 smithi016 bash[77995]: audit 2023-09-23T19:20:30.708009+0000 mgr.y (mgr.25325) 320 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:20:31.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:30 smithi016 bash[77995]: cephadm 2023-09-23T19:20:30.709197+0000 mgr.y (mgr.25325) 321 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:20:31.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:30 smithi092 bash[58855]: audit 2023-09-23T19:20:29.884248+0000 mon.a (mon.0) 439 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:20:31.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:30 smithi092 bash[58855]: audit 2023-09-23T19:20:29.897280+0000 mon.a (mon.0) 440 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:20:31.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:30 smithi092 bash[58855]: audit 2023-09-23T19:20:30.600492+0000 mon.a (mon.0) 441 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:20:31.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:30 smithi092 bash[58855]: audit 2023-09-23T19:20:30.601941+0000 mon.a (mon.0) 442 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:20:31.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:30 smithi092 bash[58855]: audit 2023-09-23T19:20:30.607858+0000 mon.a (mon.0) 443 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:20:31.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:30 smithi092 bash[58855]: audit 2023-09-23T19:20:30.693454+0000 mon.a (mon.0) 444 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:20:31.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:30 smithi092 bash[58855]: audit 2023-09-23T19:20:30.697434+0000 mon.a (mon.0) 445 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:20:31.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:30 smithi092 bash[58855]: audit 2023-09-23T19:20:30.699967+0000 mon.a (mon.0) 446 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:20:31.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:30 smithi092 bash[58855]: audit 2023-09-23T19:20:30.705705+0000 mon.a (mon.0) 447 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:20:31.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:30 smithi092 bash[58855]: audit 2023-09-23T19:20:30.707758+0000 mon.a (mon.0) 448 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:20:31.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:30 smithi092 bash[58855]: audit 2023-09-23T19:20:30.708009+0000 mgr.y (mgr.25325) 320 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:20:31.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:30 smithi092 bash[58855]: cephadm 2023-09-23T19:20:30.709197+0000 mgr.y (mgr.25325) 321 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:20:32.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:31 smithi016 bash[75004]: cluster 2023-09-23T19:20:31.664878+0000 mgr.y (mgr.25325) 322 : cluster 0 pgmap v249: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 15451/140970 objects degraded (10.960%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:20:32.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:31 smithi016 bash[77995]: cluster 2023-09-23T19:20:31.664878+0000 mgr.y (mgr.25325) 322 : cluster 0 pgmap v249: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 15451/140970 objects degraded (10.960%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:20:32.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:31 smithi092 bash[58855]: cluster 2023-09-23T19:20:31.664878+0000 mgr.y (mgr.25325) 322 : cluster 0 pgmap v249: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 15451/140970 objects degraded (10.960%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:20:33.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:32 smithi016 bash[75004]: audit 2023-09-23T19:20:32.077482+0000 mon.c (mon.2) 36 : audit 0 from='client.? 172.21.15.16:0/3999613740' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:33.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:32 smithi016 bash[77995]: audit 2023-09-23T19:20:32.077482+0000 mon.c (mon.2) 36 : audit 0 from='client.? 172.21.15.16:0/3999613740' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:33.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:32 smithi092 bash[58855]: audit 2023-09-23T19:20:32.077482+0000 mon.c (mon.2) 36 : audit 0 from='client.? 172.21.15.16:0/3999613740' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:34.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:33 smithi016 bash[75004]: cluster 2023-09-23T19:20:33.665512+0000 mgr.y (mgr.25325) 323 : cluster 0 pgmap v250: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 480 op/s; 15451/140970 objects degraded (10.960%); 709 KiB/s, 11 objects/s recovering 2023-09-23T19:20:34.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:33 smithi016 bash[77995]: cluster 2023-09-23T19:20:33.665512+0000 mgr.y (mgr.25325) 323 : cluster 0 pgmap v250: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 480 op/s; 15451/140970 objects degraded (10.960%); 709 KiB/s, 11 objects/s recovering 2023-09-23T19:20:34.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:33 smithi092 bash[58855]: cluster 2023-09-23T19:20:33.665512+0000 mgr.y (mgr.25325) 323 : cluster 0 pgmap v250: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 480 op/s; 15451/140970 objects degraded (10.960%); 709 KiB/s, 11 objects/s recovering 2023-09-23T19:20:35.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:34 smithi016 bash[75004]: cluster 2023-09-23T19:20:34.037468+0000 mon.a (mon.0) 449 : cluster 3 Health check update: Degraded data redundancy: 15451/140970 objects degraded (10.960%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:20:35.335 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:34 smithi016 bash[75004]: audit 2023-09-23T19:20:34.667829+0000 mon.a (mon.0) 450 : audit 0 from='client.? 172.21.15.16:0/70284226' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:35.336 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:34 smithi016 bash[77995]: cluster 2023-09-23T19:20:34.037468+0000 mon.a (mon.0) 449 : cluster 3 Health check update: Degraded data redundancy: 15451/140970 objects degraded (10.960%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:20:35.369 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:34 smithi016 bash[77995]: audit 2023-09-23T19:20:34.667829+0000 mon.a (mon.0) 450 : audit 0 from='client.? 172.21.15.16:0/70284226' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:35.370 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:34 smithi092 bash[58855]: cluster 2023-09-23T19:20:34.037468+0000 mon.a (mon.0) 449 : cluster 3 Health check update: Degraded data redundancy: 15451/140970 objects degraded (10.960%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:20:35.370 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:34 smithi092 bash[58855]: audit 2023-09-23T19:20:34.667829+0000 mon.a (mon.0) 450 : audit 0 from='client.? 172.21.15.16:0/70284226' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:35.931 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:20:35 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:20:35] "GET /metrics HTTP/1.1" 200 36204 "" "Prometheus/2.33.4" 2023-09-23T19:20:35.932 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:35 smithi016 bash[77995]: cluster 2023-09-23T19:20:35.666317+0000 mgr.y (mgr.25325) 324 : cluster 0 pgmap v251: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 15451/134685 objects degraded (11.472%); 709 KiB/s, 11 objects/s recovering 2023-09-23T19:20:36.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:35 smithi016 bash[75004]: cluster 2023-09-23T19:20:35.666317+0000 mgr.y (mgr.25325) 324 : cluster 0 pgmap v251: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 15451/134685 objects degraded (11.472%); 709 KiB/s, 11 objects/s recovering 2023-09-23T19:20:36.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:35 smithi092 bash[58855]: cluster 2023-09-23T19:20:35.666317+0000 mgr.y (mgr.25325) 324 : cluster 0 pgmap v251: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 15451/134685 objects degraded (11.472%); 709 KiB/s, 11 objects/s recovering 2023-09-23T19:20:37.783 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:37 smithi016 bash[75004]: audit 2023-09-23T19:20:37.248674+0000 mon.a (mon.0) 451 : audit 0 from='client.? 172.21.15.16:0/2140507666' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:37.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:37 smithi016 bash[77995]: audit 2023-09-23T19:20:37.248674+0000 mon.a (mon.0) 451 : audit 0 from='client.? 172.21.15.16:0/2140507666' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:37.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:37 smithi092 bash[58855]: audit 2023-09-23T19:20:37.248674+0000 mon.a (mon.0) 451 : audit 0 from='client.? 172.21.15.16:0/2140507666' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:38.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:38 smithi016 bash[75004]: cluster 2023-09-23T19:20:37.668826+0000 mgr.y (mgr.25325) 325 : cluster 0 pgmap v252: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 567 op/s; 15369/131826 objects degraded (11.659%); 944 KiB/s, 17 objects/s recovering 2023-09-23T19:20:38.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:38 smithi016 bash[77995]: cluster 2023-09-23T19:20:37.668826+0000 mgr.y (mgr.25325) 325 : cluster 0 pgmap v252: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 567 op/s; 15369/131826 objects degraded (11.659%); 944 KiB/s, 17 objects/s recovering 2023-09-23T19:20:38.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:38 smithi092 bash[58855]: cluster 2023-09-23T19:20:37.668826+0000 mgr.y (mgr.25325) 325 : cluster 0 pgmap v252: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 567 op/s; 15369/131826 objects degraded (11.659%); 944 KiB/s, 17 objects/s recovering 2023-09-23T19:20:39.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:39 smithi016 bash[75004]: cluster 2023-09-23T19:20:39.041136+0000 mon.a (mon.0) 452 : cluster 3 Health check update: Degraded data redundancy: 15369/131826 objects degraded (11.659%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:20:39.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:39 smithi016 bash[77995]: cluster 2023-09-23T19:20:39.041136+0000 mon.a (mon.0) 452 : cluster 3 Health check update: Degraded data redundancy: 15369/131826 objects degraded (11.659%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:20:39.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:39 smithi092 bash[58855]: cluster 2023-09-23T19:20:39.041136+0000 mon.a (mon.0) 452 : cluster 3 Health check update: Degraded data redundancy: 15369/131826 objects degraded (11.659%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:20:40.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:40 smithi016 bash[75004]: cluster 2023-09-23T19:20:39.669543+0000 mgr.y (mgr.25325) 326 : cluster 0 pgmap v253: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 487 op/s; 15369/131826 objects degraded (11.659%); 602 KiB/s, 12 objects/s recovering 2023-09-23T19:20:40.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:40 smithi016 bash[75004]: audit 2023-09-23T19:20:39.905426+0000 mon.a (mon.0) 453 : audit 0 from='client.? 172.21.15.16:0/3008696304' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:40.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:40 smithi016 bash[77995]: cluster 2023-09-23T19:20:39.669543+0000 mgr.y (mgr.25325) 326 : cluster 0 pgmap v253: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 487 op/s; 15369/131826 objects degraded (11.659%); 602 KiB/s, 12 objects/s recovering 2023-09-23T19:20:40.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:40 smithi016 bash[77995]: audit 2023-09-23T19:20:39.905426+0000 mon.a (mon.0) 453 : audit 0 from='client.? 172.21.15.16:0/3008696304' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:40.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:40 smithi092 bash[58855]: cluster 2023-09-23T19:20:39.669543+0000 mgr.y (mgr.25325) 326 : cluster 0 pgmap v253: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 487 op/s; 15369/131826 objects degraded (11.659%); 602 KiB/s, 12 objects/s recovering 2023-09-23T19:20:40.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:40 smithi092 bash[58855]: audit 2023-09-23T19:20:39.905426+0000 mon.a (mon.0) 453 : audit 0 from='client.? 172.21.15.16:0/3008696304' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:43.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:42 smithi016 bash[75004]: cluster 2023-09-23T19:20:41.670568+0000 mgr.y (mgr.25325) 327 : cluster 0 pgmap v254: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 15265/122376 objects degraded (12.474%); 602 KiB/s, 21 objects/s recovering 2023-09-23T19:20:43.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:42 smithi016 bash[75004]: audit 2023-09-23T19:20:42.511380+0000 mon.c (mon.2) 37 : audit 0 from='client.? 172.21.15.16:0/2360428178' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:43.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:42 smithi016 bash[77995]: cluster 2023-09-23T19:20:41.670568+0000 mgr.y (mgr.25325) 327 : cluster 0 pgmap v254: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 15265/122376 objects degraded (12.474%); 602 KiB/s, 21 objects/s recovering 2023-09-23T19:20:43.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:42 smithi016 bash[77995]: audit 2023-09-23T19:20:42.511380+0000 mon.c (mon.2) 37 : audit 0 from='client.? 172.21.15.16:0/2360428178' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:43.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:42 smithi092 bash[58855]: cluster 2023-09-23T19:20:41.670568+0000 mgr.y (mgr.25325) 327 : cluster 0 pgmap v254: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 15265/122376 objects degraded (12.474%); 602 KiB/s, 21 objects/s recovering 2023-09-23T19:20:43.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:42 smithi092 bash[58855]: audit 2023-09-23T19:20:42.511380+0000 mon.c (mon.2) 37 : audit 0 from='client.? 172.21.15.16:0/2360428178' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:44.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:43 smithi016 bash[75004]: audit 2023-09-23T19:20:43.620819+0000 mon.a (mon.0) 454 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:20:44.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:43 smithi016 bash[77995]: audit 2023-09-23T19:20:43.620819+0000 mon.a (mon.0) 454 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:20:44.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:43 smithi092 bash[58855]: audit 2023-09-23T19:20:43.620819+0000 mon.a (mon.0) 454 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:20:45.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:44 smithi016 bash[75004]: cluster 2023-09-23T19:20:43.671221+0000 mgr.y (mgr.25325) 328 : cluster 0 pgmap v255: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.5 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 516 op/s; 15265/122376 objects degraded (12.474%); 235 KiB/s, 15 objects/s recovering 2023-09-23T19:20:45.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:44 smithi016 bash[75004]: cluster 2023-09-23T19:20:44.042480+0000 mon.a (mon.0) 455 : cluster 3 Health check update: Degraded data redundancy: 15265/122376 objects degraded (12.474%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:20:45.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:44 smithi016 bash[77995]: cluster 2023-09-23T19:20:43.671221+0000 mgr.y (mgr.25325) 328 : cluster 0 pgmap v255: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.5 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 516 op/s; 15265/122376 objects degraded (12.474%); 235 KiB/s, 15 objects/s recovering 2023-09-23T19:20:45.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:44 smithi016 bash[77995]: cluster 2023-09-23T19:20:44.042480+0000 mon.a (mon.0) 455 : cluster 3 Health check update: Degraded data redundancy: 15265/122376 objects degraded (12.474%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:20:45.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:44 smithi092 bash[58855]: cluster 2023-09-23T19:20:43.671221+0000 mgr.y (mgr.25325) 328 : cluster 0 pgmap v255: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.5 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 516 op/s; 15265/122376 objects degraded (12.474%); 235 KiB/s, 15 objects/s recovering 2023-09-23T19:20:45.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:44 smithi092 bash[58855]: cluster 2023-09-23T19:20:44.042480+0000 mon.a (mon.0) 455 : cluster 3 Health check update: Degraded data redundancy: 15265/122376 objects degraded (12.474%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:20:46.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:45 smithi016 bash[75004]: audit 2023-09-23T19:20:45.150000+0000 mon.b (mon.1) 18 : audit 0 from='client.? 172.21.15.16:0/68234779' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:46.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:45 smithi016 bash[75004]: audit 2023-09-23T19:20:45.720830+0000 mon.a (mon.0) 456 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:20:46.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:20:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:20:45] "GET /metrics HTTP/1.1" 200 36200 "" "Prometheus/2.33.4" 2023-09-23T19:20:46.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:20:45 smithi016 bash[70992]: debug 2023-09-23T19:20:45.719+0000 7fb2a5a7c700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:20:46.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:45 smithi016 bash[77995]: audit 2023-09-23T19:20:45.150000+0000 mon.b (mon.1) 18 : audit 0 from='client.? 172.21.15.16:0/68234779' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:46.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:45 smithi016 bash[77995]: audit 2023-09-23T19:20:45.720830+0000 mon.a (mon.0) 456 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:20:46.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:45 smithi092 bash[58855]: audit 2023-09-23T19:20:45.150000+0000 mon.b (mon.1) 18 : audit 0 from='client.? 172.21.15.16:0/68234779' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:46.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:45 smithi092 bash[58855]: audit 2023-09-23T19:20:45.720830+0000 mon.a (mon.0) 456 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:20:47.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:46 smithi016 bash[75004]: cluster 2023-09-23T19:20:45.672059+0000 mgr.y (mgr.25325) 329 : cluster 0 pgmap v256: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.4 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 15265/116112 objects degraded (13.147%); 235 KiB/s, 15 objects/s recovering 2023-09-23T19:20:47.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:46 smithi016 bash[75004]: audit 2023-09-23T19:20:45.721236+0000 mgr.y (mgr.25325) 330 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:20:47.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:46 smithi016 bash[75004]: cephadm 2023-09-23T19:20:45.722617+0000 mgr.y (mgr.25325) 331 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:20:47.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:46 smithi016 bash[77995]: cluster 2023-09-23T19:20:45.672059+0000 mgr.y (mgr.25325) 329 : cluster 0 pgmap v256: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.4 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 15265/116112 objects degraded (13.147%); 235 KiB/s, 15 objects/s recovering 2023-09-23T19:20:47.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:46 smithi016 bash[77995]: audit 2023-09-23T19:20:45.721236+0000 mgr.y (mgr.25325) 330 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:20:47.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:46 smithi016 bash[77995]: cephadm 2023-09-23T19:20:45.722617+0000 mgr.y (mgr.25325) 331 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:20:47.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:46 smithi092 bash[58855]: cluster 2023-09-23T19:20:45.672059+0000 mgr.y (mgr.25325) 329 : cluster 0 pgmap v256: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.4 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 15265/116112 objects degraded (13.147%); 235 KiB/s, 15 objects/s recovering 2023-09-23T19:20:47.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:46 smithi092 bash[58855]: audit 2023-09-23T19:20:45.721236+0000 mgr.y (mgr.25325) 330 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:20:47.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:46 smithi092 bash[58855]: cephadm 2023-09-23T19:20:45.722617+0000 mgr.y (mgr.25325) 331 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:20:49.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:48 smithi016 bash[75004]: cluster 2023-09-23T19:20:47.672777+0000 mgr.y (mgr.25325) 332 : cluster 0 pgmap v257: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 15165/113205 objects degraded (13.396%); 235 KiB/s, 23 objects/s recovering 2023-09-23T19:20:49.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:48 smithi016 bash[75004]: cluster 2023-09-23T19:20:47.761985+0000 mon.a (mon.0) 457 : cluster 0 osdmap e88: 8 total, 8 up, 8 in 2023-09-23T19:20:49.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:48 smithi016 bash[75004]: cluster 2023-09-23T19:20:47.764797+0000 osd.3 (osd.3) 14 : cluster 3 failed to encode map e88 with expected crc 2023-09-23T19:20:49.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:48 smithi016 bash[75004]: audit 2023-09-23T19:20:47.765331+0000 mon.c (mon.2) 38 : audit 0 from='client.? 172.21.15.16:0/2775374183' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:49.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:48 smithi016 bash[75004]: cluster 2023-09-23T19:20:47.766991+0000 osd.3 (osd.3) 15 : cluster 3 failed to encode map e88 with expected crc 2023-09-23T19:20:49.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:48 smithi016 bash[75004]: cluster 2023-09-23T19:20:47.783326+0000 osd.1 (osd.1) 16 : cluster 3 failed to encode map e88 with expected crc 2023-09-23T19:20:49.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:48 smithi016 bash[77995]: cluster 2023-09-23T19:20:47.672777+0000 mgr.y (mgr.25325) 332 : cluster 0 pgmap v257: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 15165/113205 objects degraded (13.396%); 235 KiB/s, 23 objects/s recovering 2023-09-23T19:20:49.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:48 smithi016 bash[77995]: cluster 2023-09-23T19:20:47.761985+0000 mon.a (mon.0) 457 : cluster 0 osdmap e88: 8 total, 8 up, 8 in 2023-09-23T19:20:49.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:48 smithi016 bash[77995]: cluster 2023-09-23T19:20:47.764797+0000 osd.3 (osd.3) 14 : cluster 3 failed to encode map e88 with expected crc 2023-09-23T19:20:49.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:48 smithi016 bash[77995]: audit 2023-09-23T19:20:47.765331+0000 mon.c (mon.2) 38 : audit 0 from='client.? 172.21.15.16:0/2775374183' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:49.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:48 smithi016 bash[77995]: cluster 2023-09-23T19:20:47.766991+0000 osd.3 (osd.3) 15 : cluster 3 failed to encode map e88 with expected crc 2023-09-23T19:20:49.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:48 smithi016 bash[77995]: cluster 2023-09-23T19:20:47.783326+0000 osd.1 (osd.1) 16 : cluster 3 failed to encode map e88 with expected crc 2023-09-23T19:20:49.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:48 smithi092 bash[58855]: cluster 2023-09-23T19:20:47.672777+0000 mgr.y (mgr.25325) 332 : cluster 0 pgmap v257: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 15165/113205 objects degraded (13.396%); 235 KiB/s, 23 objects/s recovering 2023-09-23T19:20:49.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:48 smithi092 bash[58855]: cluster 2023-09-23T19:20:47.761985+0000 mon.a (mon.0) 457 : cluster 0 osdmap e88: 8 total, 8 up, 8 in 2023-09-23T19:20:49.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:48 smithi092 bash[58855]: cluster 2023-09-23T19:20:47.764797+0000 osd.3 (osd.3) 14 : cluster 3 failed to encode map e88 with expected crc 2023-09-23T19:20:49.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:48 smithi092 bash[58855]: audit 2023-09-23T19:20:47.765331+0000 mon.c (mon.2) 38 : audit 0 from='client.? 172.21.15.16:0/2775374183' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:49.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:48 smithi092 bash[58855]: cluster 2023-09-23T19:20:47.766991+0000 osd.3 (osd.3) 15 : cluster 3 failed to encode map e88 with expected crc 2023-09-23T19:20:49.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:48 smithi092 bash[58855]: cluster 2023-09-23T19:20:47.783326+0000 osd.1 (osd.1) 16 : cluster 3 failed to encode map e88 with expected crc 2023-09-23T19:20:50.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:49 smithi016 bash[75004]: cluster 2023-09-23T19:20:48.761290+0000 osd.2 (osd.2) 15 : cluster 3 failed to encode map e89 with expected crc 2023-09-23T19:20:50.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:49 smithi016 bash[75004]: cluster 2023-09-23T19:20:48.765210+0000 mon.a (mon.0) 458 : cluster 0 osdmap e89: 8 total, 8 up, 8 in 2023-09-23T19:20:50.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:49 smithi016 bash[75004]: cluster 2023-09-23T19:20:48.768938+0000 osd.3 (osd.3) 16 : cluster 3 failed to encode map e89 with expected crc 2023-09-23T19:20:50.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:49 smithi016 bash[75004]: cluster 2023-09-23T19:20:48.772175+0000 osd.7 (osd.7) 14 : cluster 3 failed to encode map e89 with expected crc 2023-09-23T19:20:50.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:49 smithi016 bash[75004]: cluster 2023-09-23T19:20:48.825804+0000 osd.6 (osd.6) 12 : cluster 3 failed to encode map e89 with expected crc 2023-09-23T19:20:50.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:49 smithi016 bash[75004]: cluster 2023-09-23T19:20:48.841311+0000 osd.1 (osd.1) 17 : cluster 3 failed to encode map e89 with expected crc 2023-09-23T19:20:50.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:49 smithi016 bash[75004]: cluster 2023-09-23T19:20:49.056762+0000 mon.a (mon.0) 459 : cluster 3 Health check update: Degraded data redundancy: 15165/113205 objects degraded (13.396%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:20:50.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:49 smithi016 bash[77995]: cluster 2023-09-23T19:20:48.761290+0000 osd.2 (osd.2) 15 : cluster 3 failed to encode map e89 with expected crc 2023-09-23T19:20:50.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:49 smithi016 bash[77995]: cluster 2023-09-23T19:20:48.765210+0000 mon.a (mon.0) 458 : cluster 0 osdmap e89: 8 total, 8 up, 8 in 2023-09-23T19:20:50.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:49 smithi016 bash[77995]: cluster 2023-09-23T19:20:48.768938+0000 osd.3 (osd.3) 16 : cluster 3 failed to encode map e89 with expected crc 2023-09-23T19:20:50.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:49 smithi016 bash[77995]: cluster 2023-09-23T19:20:48.772175+0000 osd.7 (osd.7) 14 : cluster 3 failed to encode map e89 with expected crc 2023-09-23T19:20:50.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:49 smithi016 bash[77995]: cluster 2023-09-23T19:20:48.825804+0000 osd.6 (osd.6) 12 : cluster 3 failed to encode map e89 with expected crc 2023-09-23T19:20:50.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:49 smithi016 bash[77995]: cluster 2023-09-23T19:20:48.841311+0000 osd.1 (osd.1) 17 : cluster 3 failed to encode map e89 with expected crc 2023-09-23T19:20:50.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:49 smithi016 bash[77995]: cluster 2023-09-23T19:20:49.056762+0000 mon.a (mon.0) 459 : cluster 3 Health check update: Degraded data redundancy: 15165/113205 objects degraded (13.396%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:20:50.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:49 smithi092 bash[58855]: cluster 2023-09-23T19:20:48.761290+0000 osd.2 (osd.2) 15 : cluster 3 failed to encode map e89 with expected crc 2023-09-23T19:20:50.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:49 smithi092 bash[58855]: cluster 2023-09-23T19:20:48.765210+0000 mon.a (mon.0) 458 : cluster 0 osdmap e89: 8 total, 8 up, 8 in 2023-09-23T19:20:50.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:49 smithi092 bash[58855]: cluster 2023-09-23T19:20:48.768938+0000 osd.3 (osd.3) 16 : cluster 3 failed to encode map e89 with expected crc 2023-09-23T19:20:50.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:49 smithi092 bash[58855]: cluster 2023-09-23T19:20:48.772175+0000 osd.7 (osd.7) 14 : cluster 3 failed to encode map e89 with expected crc 2023-09-23T19:20:50.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:49 smithi092 bash[58855]: cluster 2023-09-23T19:20:48.825804+0000 osd.6 (osd.6) 12 : cluster 3 failed to encode map e89 with expected crc 2023-09-23T19:20:50.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:49 smithi092 bash[58855]: cluster 2023-09-23T19:20:48.841311+0000 osd.1 (osd.1) 17 : cluster 3 failed to encode map e89 with expected crc 2023-09-23T19:20:50.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:49 smithi092 bash[58855]: cluster 2023-09-23T19:20:49.056762+0000 mon.a (mon.0) 459 : cluster 3 Health check update: Degraded data redundancy: 15165/113205 objects degraded (13.396%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T19:20:51.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:50 smithi016 bash[75004]: cluster 2023-09-23T19:20:49.673331+0000 mgr.y (mgr.25325) 333 : cluster 0 pgmap v260: 89 pgs: 1 peering, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 390 op/s; 15137/112998 objects degraded (13.396%); 0 B/s, 15 objects/s recovering 2023-09-23T19:20:51.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:50 smithi016 bash[75004]: audit 2023-09-23T19:20:50.373658+0000 mon.a (mon.0) 460 : audit 0 from='client.? 172.21.15.16:0/2524783039' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:51.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:50 smithi016 bash[77995]: cluster 2023-09-23T19:20:49.673331+0000 mgr.y (mgr.25325) 333 : cluster 0 pgmap v260: 89 pgs: 1 peering, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 390 op/s; 15137/112998 objects degraded (13.396%); 0 B/s, 15 objects/s recovering 2023-09-23T19:20:51.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:50 smithi016 bash[77995]: audit 2023-09-23T19:20:50.373658+0000 mon.a (mon.0) 460 : audit 0 from='client.? 172.21.15.16:0/2524783039' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:51.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:50 smithi092 bash[58855]: cluster 2023-09-23T19:20:49.673331+0000 mgr.y (mgr.25325) 333 : cluster 0 pgmap v260: 89 pgs: 1 peering, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 390 op/s; 15137/112998 objects degraded (13.396%); 0 B/s, 15 objects/s recovering 2023-09-23T19:20:51.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:50 smithi092 bash[58855]: audit 2023-09-23T19:20:50.373658+0000 mon.a (mon.0) 460 : audit 0 from='client.? 172.21.15.16:0/2524783039' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:52.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:51 smithi016 bash[75004]: cluster 2023-09-23T19:20:51.675059+0000 mgr.y (mgr.25325) 334 : cluster 0 pgmap v261: 89 pgs: 1 peering, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 15083/105744 objects degraded (14.264%); 0 B/s, 22 objects/s recovering 2023-09-23T19:20:52.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:51 smithi016 bash[77995]: cluster 2023-09-23T19:20:51.675059+0000 mgr.y (mgr.25325) 334 : cluster 0 pgmap v261: 89 pgs: 1 peering, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 15083/105744 objects degraded (14.264%); 0 B/s, 22 objects/s recovering 2023-09-23T19:20:52.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:51 smithi092 bash[58855]: cluster 2023-09-23T19:20:51.675059+0000 mgr.y (mgr.25325) 334 : cluster 0 pgmap v261: 89 pgs: 1 peering, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 15083/105744 objects degraded (14.264%); 0 B/s, 22 objects/s recovering 2023-09-23T19:20:53.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:53 smithi016 bash[75004]: audit 2023-09-23T19:20:53.006663+0000 mon.a (mon.0) 461 : audit 0 from='client.? 172.21.15.16:0/2340756574' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:53.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:53 smithi016 bash[77995]: audit 2023-09-23T19:20:53.006663+0000 mon.a (mon.0) 461 : audit 0 from='client.? 172.21.15.16:0/2340756574' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:53.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:53 smithi092 bash[58855]: audit 2023-09-23T19:20:53.006663+0000 mon.a (mon.0) 461 : audit 0 from='client.? 172.21.15.16:0/2340756574' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:54.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:54 smithi016 bash[75004]: cluster 2023-09-23T19:20:53.675673+0000 mgr.y (mgr.25325) 335 : cluster 0 pgmap v262: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.1 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 452 op/s; 15083/105240 objects degraded (14.332%); 0 B/s, 22 objects/s recovering 2023-09-23T19:20:54.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:54 smithi016 bash[77995]: cluster 2023-09-23T19:20:53.675673+0000 mgr.y (mgr.25325) 335 : cluster 0 pgmap v262: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.1 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 452 op/s; 15083/105240 objects degraded (14.332%); 0 B/s, 22 objects/s recovering 2023-09-23T19:20:54.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:54 smithi092 bash[58855]: cluster 2023-09-23T19:20:53.675673+0000 mgr.y (mgr.25325) 335 : cluster 0 pgmap v262: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.1 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 452 op/s; 15083/105240 objects degraded (14.332%); 0 B/s, 22 objects/s recovering 2023-09-23T19:20:55.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:55 smithi016 bash[75004]: cluster 2023-09-23T19:20:54.058191+0000 mon.a (mon.0) 462 : cluster 3 Health check update: Degraded data redundancy: 15083/105744 objects degraded (14.264%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:20:55.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:55 smithi016 bash[77995]: cluster 2023-09-23T19:20:54.058191+0000 mon.a (mon.0) 462 : cluster 3 Health check update: Degraded data redundancy: 15083/105744 objects degraded (14.264%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:20:55.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:55 smithi092 bash[58855]: cluster 2023-09-23T19:20:54.058191+0000 mon.a (mon.0) 462 : cluster 3 Health check update: Degraded data redundancy: 15083/105744 objects degraded (14.264%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:20:56.033 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:20:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:20:55] "GET /metrics HTTP/1.1" 200 36200 "" "Prometheus/2.33.4" 2023-09-23T19:20:56.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:56 smithi016 bash[75004]: audit 2023-09-23T19:20:55.648747+0000 mon.a (mon.0) 463 : audit 0 from='client.? 172.21.15.16:0/3831304708' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:56.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:56 smithi016 bash[75004]: cluster 2023-09-23T19:20:55.676640+0000 mgr.y (mgr.25325) 336 : cluster 0 pgmap v263: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 14976/98940 objects degraded (15.136%); 0 B/s, 23 objects/s recovering 2023-09-23T19:20:56.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:56 smithi016 bash[77995]: audit 2023-09-23T19:20:55.648747+0000 mon.a (mon.0) 463 : audit 0 from='client.? 172.21.15.16:0/3831304708' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:56.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:56 smithi016 bash[77995]: cluster 2023-09-23T19:20:55.676640+0000 mgr.y (mgr.25325) 336 : cluster 0 pgmap v263: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 14976/98940 objects degraded (15.136%); 0 B/s, 23 objects/s recovering 2023-09-23T19:20:56.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:56 smithi092 bash[58855]: audit 2023-09-23T19:20:55.648747+0000 mon.a (mon.0) 463 : audit 0 from='client.? 172.21.15.16:0/3831304708' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:56.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:56 smithi092 bash[58855]: cluster 2023-09-23T19:20:55.676640+0000 mgr.y (mgr.25325) 336 : cluster 0 pgmap v263: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 14976/98940 objects degraded (15.136%); 0 B/s, 23 objects/s recovering 2023-09-23T19:20:59.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:58 smithi016 bash[75004]: cluster 2023-09-23T19:20:57.677290+0000 mgr.y (mgr.25325) 337 : cluster 0 pgmap v264: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.0 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 554 op/s; 14976/96714 objects degraded (15.485%); 0 B/s, 19 objects/s recovering 2023-09-23T19:20:59.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:58 smithi016 bash[75004]: audit 2023-09-23T19:20:58.284546+0000 mon.c (mon.2) 39 : audit 0 from='client.? 172.21.15.16:0/2171425233' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:59.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:58 smithi016 bash[75004]: audit 2023-09-23T19:20:58.621058+0000 mon.a (mon.0) 464 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:20:59.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:58 smithi016 bash[77995]: cluster 2023-09-23T19:20:57.677290+0000 mgr.y (mgr.25325) 337 : cluster 0 pgmap v264: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.0 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 554 op/s; 14976/96714 objects degraded (15.485%); 0 B/s, 19 objects/s recovering 2023-09-23T19:20:59.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:58 smithi016 bash[77995]: audit 2023-09-23T19:20:58.284546+0000 mon.c (mon.2) 39 : audit 0 from='client.? 172.21.15.16:0/2171425233' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:59.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:58 smithi016 bash[77995]: audit 2023-09-23T19:20:58.621058+0000 mon.a (mon.0) 464 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:20:59.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:58 smithi092 bash[58855]: cluster 2023-09-23T19:20:57.677290+0000 mgr.y (mgr.25325) 337 : cluster 0 pgmap v264: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.0 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 554 op/s; 14976/96714 objects degraded (15.485%); 0 B/s, 19 objects/s recovering 2023-09-23T19:20:59.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:58 smithi092 bash[58855]: audit 2023-09-23T19:20:58.284546+0000 mon.c (mon.2) 39 : audit 0 from='client.? 172.21.15.16:0/2171425233' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:20:59.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:58 smithi092 bash[58855]: audit 2023-09-23T19:20:58.621058+0000 mon.a (mon.0) 464 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:21:00.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:20:59 smithi016 bash[75004]: cluster 2023-09-23T19:20:59.059475+0000 mon.a (mon.0) 465 : cluster 3 Health check update: Degraded data redundancy: 14976/96714 objects degraded (15.485%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:21:00.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:20:59 smithi016 bash[77995]: cluster 2023-09-23T19:20:59.059475+0000 mon.a (mon.0) 465 : cluster 3 Health check update: Degraded data redundancy: 14976/96714 objects degraded (15.485%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:21:00.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:20:59 smithi092 bash[58855]: cluster 2023-09-23T19:20:59.059475+0000 mon.a (mon.0) 465 : cluster 3 Health check update: Degraded data redundancy: 14976/96714 objects degraded (15.485%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:21:01.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:00 smithi016 bash[75004]: cluster 2023-09-23T19:20:59.677926+0000 mgr.y (mgr.25325) 338 : cluster 0 pgmap v265: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.0 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 518 op/s; 14976/96024 objects degraded (15.596%); 0 B/s, 14 objects/s recovering 2023-09-23T19:21:01.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:21:00 smithi016 bash[70992]: debug 2023-09-23T19:21:00.743+0000 7fb2a5a7c700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:21:01.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:00 smithi016 bash[77995]: cluster 2023-09-23T19:20:59.677926+0000 mgr.y (mgr.25325) 338 : cluster 0 pgmap v265: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.0 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 518 op/s; 14976/96024 objects degraded (15.596%); 0 B/s, 14 objects/s recovering 2023-09-23T19:21:01.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:00 smithi092 bash[58855]: cluster 2023-09-23T19:20:59.677926+0000 mgr.y (mgr.25325) 338 : cluster 0 pgmap v265: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.0 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 518 op/s; 14976/96024 objects degraded (15.596%); 0 B/s, 14 objects/s recovering 2023-09-23T19:21:02.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:01 smithi016 bash[75004]: audit 2023-09-23T19:21:00.745526+0000 mon.a (mon.0) 466 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:21:02.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:01 smithi016 bash[75004]: audit 2023-09-23T19:21:00.745871+0000 mgr.y (mgr.25325) 339 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:21:02.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:01 smithi016 bash[75004]: cephadm 2023-09-23T19:21:00.749364+0000 mgr.y (mgr.25325) 340 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:21:02.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:01 smithi016 bash[75004]: audit 2023-09-23T19:21:00.911090+0000 mon.a (mon.0) 467 : audit 0 from='client.? 172.21.15.16:0/911633971' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:02.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:01 smithi016 bash[77995]: audit 2023-09-23T19:21:00.745526+0000 mon.a (mon.0) 466 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:21:02.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:01 smithi016 bash[77995]: audit 2023-09-23T19:21:00.745871+0000 mgr.y (mgr.25325) 339 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:21:02.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:01 smithi016 bash[77995]: cephadm 2023-09-23T19:21:00.749364+0000 mgr.y (mgr.25325) 340 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:21:02.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:01 smithi016 bash[77995]: audit 2023-09-23T19:21:00.911090+0000 mon.a (mon.0) 467 : audit 0 from='client.? 172.21.15.16:0/911633971' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:02.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:01 smithi092 bash[58855]: audit 2023-09-23T19:21:00.745526+0000 mon.a (mon.0) 466 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:21:02.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:01 smithi092 bash[58855]: audit 2023-09-23T19:21:00.745871+0000 mgr.y (mgr.25325) 339 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:21:02.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:01 smithi092 bash[58855]: cephadm 2023-09-23T19:21:00.749364+0000 mgr.y (mgr.25325) 340 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:21:02.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:01 smithi092 bash[58855]: audit 2023-09-23T19:21:00.911090+0000 mon.a (mon.0) 467 : audit 0 from='client.? 172.21.15.16:0/911633971' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:03.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:02 smithi016 bash[75004]: cluster 2023-09-23T19:21:01.678852+0000 mgr.y (mgr.25325) 341 : cluster 0 pgmap v266: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.8 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 700 op/s; 14875/87780 objects degraded (16.946%); 0 B/s, 21 objects/s recovering 2023-09-23T19:21:03.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:02 smithi016 bash[77995]: cluster 2023-09-23T19:21:01.678852+0000 mgr.y (mgr.25325) 341 : cluster 0 pgmap v266: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.8 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 700 op/s; 14875/87780 objects degraded (16.946%); 0 B/s, 21 objects/s recovering 2023-09-23T19:21:03.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:02 smithi092 bash[58855]: cluster 2023-09-23T19:21:01.678852+0000 mgr.y (mgr.25325) 341 : cluster 0 pgmap v266: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.8 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 700 op/s; 14875/87780 objects degraded (16.946%); 0 B/s, 21 objects/s recovering 2023-09-23T19:21:04.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:03 smithi016 bash[75004]: audit 2023-09-23T19:21:03.537838+0000 mon.c (mon.2) 40 : audit 0 from='client.? 172.21.15.16:0/3673794236' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:04.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:03 smithi016 bash[77995]: audit 2023-09-23T19:21:03.537838+0000 mon.c (mon.2) 40 : audit 0 from='client.? 172.21.15.16:0/3673794236' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:04.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:03 smithi092 bash[58855]: audit 2023-09-23T19:21:03.537838+0000 mon.c (mon.2) 40 : audit 0 from='client.? 172.21.15.16:0/3673794236' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:05.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:04 smithi016 bash[75004]: cluster 2023-09-23T19:21:03.679463+0000 mgr.y (mgr.25325) 342 : cluster 0 pgmap v267: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 514 op/s; 14875/87213 objects degraded (17.056%); 0 B/s, 17 objects/s recovering 2023-09-23T19:21:05.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:04 smithi016 bash[75004]: cluster 2023-09-23T19:21:04.060684+0000 mon.a (mon.0) 468 : cluster 3 Health check update: Degraded data redundancy: 14875/87213 objects degraded (17.056%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:21:05.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:04 smithi016 bash[77995]: cluster 2023-09-23T19:21:03.679463+0000 mgr.y (mgr.25325) 342 : cluster 0 pgmap v267: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 514 op/s; 14875/87213 objects degraded (17.056%); 0 B/s, 17 objects/s recovering 2023-09-23T19:21:05.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:04 smithi016 bash[77995]: cluster 2023-09-23T19:21:04.060684+0000 mon.a (mon.0) 468 : cluster 3 Health check update: Degraded data redundancy: 14875/87213 objects degraded (17.056%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:21:05.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:04 smithi092 bash[58855]: cluster 2023-09-23T19:21:03.679463+0000 mgr.y (mgr.25325) 342 : cluster 0 pgmap v267: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 514 op/s; 14875/87213 objects degraded (17.056%); 0 B/s, 17 objects/s recovering 2023-09-23T19:21:05.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:04 smithi092 bash[58855]: cluster 2023-09-23T19:21:04.060684+0000 mon.a (mon.0) 468 : cluster 3 Health check update: Degraded data redundancy: 14875/87213 objects degraded (17.056%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:21:06.033 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:21:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:21:05] "GET /metrics HTTP/1.1" 200 36184 "" "Prometheus/2.33.4" 2023-09-23T19:21:07.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:06 smithi016 bash[75004]: cluster 2023-09-23T19:21:05.680279+0000 mgr.y (mgr.25325) 343 : cluster 0 pgmap v268: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.6 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 14770/81117 objects degraded (18.208%); 0 B/s, 26 objects/s recovering 2023-09-23T19:21:07.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:06 smithi016 bash[75004]: audit 2023-09-23T19:21:06.132367+0000 mon.c (mon.2) 41 : audit 0 from='client.? 172.21.15.16:0/2948007165' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:07.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:06 smithi016 bash[77995]: cluster 2023-09-23T19:21:05.680279+0000 mgr.y (mgr.25325) 343 : cluster 0 pgmap v268: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.6 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 14770/81117 objects degraded (18.208%); 0 B/s, 26 objects/s recovering 2023-09-23T19:21:07.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:06 smithi016 bash[77995]: audit 2023-09-23T19:21:06.132367+0000 mon.c (mon.2) 41 : audit 0 from='client.? 172.21.15.16:0/2948007165' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:07.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:06 smithi092 bash[58855]: cluster 2023-09-23T19:21:05.680279+0000 mgr.y (mgr.25325) 343 : cluster 0 pgmap v268: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.6 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 14770/81117 objects degraded (18.208%); 0 B/s, 26 objects/s recovering 2023-09-23T19:21:07.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:06 smithi092 bash[58855]: audit 2023-09-23T19:21:06.132367+0000 mon.c (mon.2) 41 : audit 0 from='client.? 172.21.15.16:0/2948007165' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:09.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:08 smithi016 bash[75004]: cluster 2023-09-23T19:21:07.680923+0000 mgr.y (mgr.25325) 344 : cluster 0 pgmap v269: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.6 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 14770/78738 objects degraded (18.758%); 0 B/s, 17 objects/s recovering 2023-09-23T19:21:09.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:08 smithi016 bash[75004]: audit 2023-09-23T19:21:08.768788+0000 mon.a (mon.0) 469 : audit 0 from='client.? 172.21.15.16:0/2719724739' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:09.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:08 smithi016 bash[77995]: cluster 2023-09-23T19:21:07.680923+0000 mgr.y (mgr.25325) 344 : cluster 0 pgmap v269: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.6 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 14770/78738 objects degraded (18.758%); 0 B/s, 17 objects/s recovering 2023-09-23T19:21:09.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:08 smithi016 bash[77995]: audit 2023-09-23T19:21:08.768788+0000 mon.a (mon.0) 469 : audit 0 from='client.? 172.21.15.16:0/2719724739' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:09.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:08 smithi092 bash[58855]: cluster 2023-09-23T19:21:07.680923+0000 mgr.y (mgr.25325) 344 : cluster 0 pgmap v269: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.6 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 14770/78738 objects degraded (18.758%); 0 B/s, 17 objects/s recovering 2023-09-23T19:21:09.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:08 smithi092 bash[58855]: audit 2023-09-23T19:21:08.768788+0000 mon.a (mon.0) 469 : audit 0 from='client.? 172.21.15.16:0/2719724739' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:10.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:09 smithi016 bash[75004]: cluster 2023-09-23T19:21:09.061931+0000 mon.a (mon.0) 470 : cluster 3 Health check update: Degraded data redundancy: 14770/78738 objects degraded (18.758%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:21:10.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:09 smithi016 bash[75004]: cluster 2023-09-23T19:21:09.681438+0000 mgr.y (mgr.25325) 345 : cluster 0 pgmap v270: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.6 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 514 op/s; 14770/78174 objects degraded (18.894%); 0 B/s, 17 objects/s recovering 2023-09-23T19:21:10.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:09 smithi016 bash[77995]: cluster 2023-09-23T19:21:09.061931+0000 mon.a (mon.0) 470 : cluster 3 Health check update: Degraded data redundancy: 14770/78738 objects degraded (18.758%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:21:10.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:09 smithi016 bash[77995]: cluster 2023-09-23T19:21:09.681438+0000 mgr.y (mgr.25325) 345 : cluster 0 pgmap v270: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.6 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 514 op/s; 14770/78174 objects degraded (18.894%); 0 B/s, 17 objects/s recovering 2023-09-23T19:21:10.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:09 smithi092 bash[58855]: cluster 2023-09-23T19:21:09.061931+0000 mon.a (mon.0) 470 : cluster 3 Health check update: Degraded data redundancy: 14770/78738 objects degraded (18.758%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:21:10.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:09 smithi092 bash[58855]: cluster 2023-09-23T19:21:09.681438+0000 mgr.y (mgr.25325) 345 : cluster 0 pgmap v270: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.6 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 514 op/s; 14770/78174 objects degraded (18.894%); 0 B/s, 17 objects/s recovering 2023-09-23T19:21:11.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:11 smithi016 bash[75004]: audit 2023-09-23T19:21:11.382551+0000 mon.a (mon.0) 471 : audit 0 from='client.? 172.21.15.16:0/837991132' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:11.803 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:11 smithi016 bash[77995]: audit 2023-09-23T19:21:11.382551+0000 mon.a (mon.0) 471 : audit 0 from='client.? 172.21.15.16:0/837991132' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:11.803 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:11 smithi092 bash[58855]: audit 2023-09-23T19:21:11.382551+0000 mon.a (mon.0) 471 : audit 0 from='client.? 172.21.15.16:0/837991132' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:12.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:12 smithi016 bash[75004]: cluster 2023-09-23T19:21:11.682300+0000 mgr.y (mgr.25325) 346 : cluster 0 pgmap v271: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.4 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 14668/69774 objects degraded (21.022%); 0 B/s, 25 objects/s recovering 2023-09-23T19:21:12.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:12 smithi016 bash[77995]: cluster 2023-09-23T19:21:11.682300+0000 mgr.y (mgr.25325) 346 : cluster 0 pgmap v271: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.4 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 14668/69774 objects degraded (21.022%); 0 B/s, 25 objects/s recovering 2023-09-23T19:21:12.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:12 smithi092 bash[58855]: cluster 2023-09-23T19:21:11.682300+0000 mgr.y (mgr.25325) 346 : cluster 0 pgmap v271: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.4 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 14668/69774 objects degraded (21.022%); 0 B/s, 25 objects/s recovering 2023-09-23T19:21:14.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:13 smithi016 bash[75004]: audit 2023-09-23T19:21:13.622562+0000 mon.a (mon.0) 472 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:21:14.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:13 smithi016 bash[77995]: audit 2023-09-23T19:21:13.622562+0000 mon.a (mon.0) 472 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:21:14.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:13 smithi092 bash[58855]: audit 2023-09-23T19:21:13.622562+0000 mon.a (mon.0) 472 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:21:15.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:14 smithi016 bash[75004]: cluster 2023-09-23T19:21:13.682919+0000 mgr.y (mgr.25325) 347 : cluster 0 pgmap v272: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.4 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 516 op/s; 14668/69180 objects degraded (21.203%); 0 B/s, 17 objects/s recovering 2023-09-23T19:21:15.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:14 smithi016 bash[75004]: audit 2023-09-23T19:21:13.971750+0000 mon.a (mon.0) 473 : audit 0 from='client.? 172.21.15.16:0/2330656664' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:15.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:14 smithi016 bash[75004]: cluster 2023-09-23T19:21:14.063076+0000 mon.a (mon.0) 474 : cluster 3 Health check update: Degraded data redundancy: 14668/69774 objects degraded (21.022%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:21:15.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:14 smithi016 bash[77995]: cluster 2023-09-23T19:21:13.682919+0000 mgr.y (mgr.25325) 347 : cluster 0 pgmap v272: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.4 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 516 op/s; 14668/69180 objects degraded (21.203%); 0 B/s, 17 objects/s recovering 2023-09-23T19:21:15.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:14 smithi016 bash[77995]: audit 2023-09-23T19:21:13.971750+0000 mon.a (mon.0) 473 : audit 0 from='client.? 172.21.15.16:0/2330656664' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:15.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:14 smithi016 bash[77995]: cluster 2023-09-23T19:21:14.063076+0000 mon.a (mon.0) 474 : cluster 3 Health check update: Degraded data redundancy: 14668/69774 objects degraded (21.022%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:21:15.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:14 smithi092 bash[58855]: cluster 2023-09-23T19:21:13.682919+0000 mgr.y (mgr.25325) 347 : cluster 0 pgmap v272: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.4 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 516 op/s; 14668/69180 objects degraded (21.203%); 0 B/s, 17 objects/s recovering 2023-09-23T19:21:15.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:14 smithi092 bash[58855]: audit 2023-09-23T19:21:13.971750+0000 mon.a (mon.0) 473 : audit 0 from='client.? 172.21.15.16:0/2330656664' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:15.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:14 smithi092 bash[58855]: cluster 2023-09-23T19:21:14.063076+0000 mon.a (mon.0) 474 : cluster 3 Health check update: Degraded data redundancy: 14668/69774 objects degraded (21.022%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:21:16.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:21:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:21:15] "GET /metrics HTTP/1.1" 200 36186 "" "Prometheus/2.33.4" 2023-09-23T19:21:16.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:21:15 smithi016 bash[70992]: debug 2023-09-23T19:21:15.763+0000 7fb2a5a7c700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:21:17.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:16 smithi016 bash[75004]: cluster 2023-09-23T19:21:15.683549+0000 mgr.y (mgr.25325) 348 : cluster 0 pgmap v273: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.3 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 14567/63093 objects degraded (23.088%); 0 B/s, 25 objects/s recovering 2023-09-23T19:21:17.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:16 smithi016 bash[75004]: audit 2023-09-23T19:21:15.764973+0000 mon.a (mon.0) 475 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:21:17.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:16 smithi016 bash[75004]: audit 2023-09-23T19:21:15.765329+0000 mgr.y (mgr.25325) 349 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:21:17.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:16 smithi016 bash[75004]: cephadm 2023-09-23T19:21:15.766691+0000 mgr.y (mgr.25325) 350 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:21:17.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:16 smithi016 bash[75004]: audit 2023-09-23T19:21:16.609652+0000 mon.b (mon.1) 19 : audit 0 from='client.? 172.21.15.16:0/2091052938' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:17.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:16 smithi016 bash[77995]: cluster 2023-09-23T19:21:15.683549+0000 mgr.y (mgr.25325) 348 : cluster 0 pgmap v273: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.3 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 14567/63093 objects degraded (23.088%); 0 B/s, 25 objects/s recovering 2023-09-23T19:21:17.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:16 smithi016 bash[77995]: audit 2023-09-23T19:21:15.764973+0000 mon.a (mon.0) 475 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:21:17.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:16 smithi016 bash[77995]: audit 2023-09-23T19:21:15.765329+0000 mgr.y (mgr.25325) 349 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:21:17.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:16 smithi016 bash[77995]: cephadm 2023-09-23T19:21:15.766691+0000 mgr.y (mgr.25325) 350 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:21:17.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:16 smithi016 bash[77995]: audit 2023-09-23T19:21:16.609652+0000 mon.b (mon.1) 19 : audit 0 from='client.? 172.21.15.16:0/2091052938' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:17.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:16 smithi092 bash[58855]: cluster 2023-09-23T19:21:15.683549+0000 mgr.y (mgr.25325) 348 : cluster 0 pgmap v273: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.3 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 14567/63093 objects degraded (23.088%); 0 B/s, 25 objects/s recovering 2023-09-23T19:21:17.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:16 smithi092 bash[58855]: audit 2023-09-23T19:21:15.764973+0000 mon.a (mon.0) 475 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:21:17.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:16 smithi092 bash[58855]: audit 2023-09-23T19:21:15.765329+0000 mgr.y (mgr.25325) 349 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:21:17.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:16 smithi092 bash[58855]: cephadm 2023-09-23T19:21:15.766691+0000 mgr.y (mgr.25325) 350 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:21:17.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:16 smithi092 bash[58855]: audit 2023-09-23T19:21:16.609652+0000 mon.b (mon.1) 19 : audit 0 from='client.? 172.21.15.16:0/2091052938' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:19.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:18 smithi016 bash[75004]: cluster 2023-09-23T19:21:17.684313+0000 mgr.y (mgr.25325) 351 : cluster 0 pgmap v274: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 565 op/s; 14567/60750 objects degraded (23.979%); 0 B/s, 16 objects/s recovering 2023-09-23T19:21:19.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:18 smithi016 bash[77995]: cluster 2023-09-23T19:21:17.684313+0000 mgr.y (mgr.25325) 351 : cluster 0 pgmap v274: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 565 op/s; 14567/60750 objects degraded (23.979%); 0 B/s, 16 objects/s recovering 2023-09-23T19:21:19.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:18 smithi092 bash[58855]: cluster 2023-09-23T19:21:17.684313+0000 mgr.y (mgr.25325) 351 : cluster 0 pgmap v274: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 565 op/s; 14567/60750 objects degraded (23.979%); 0 B/s, 16 objects/s recovering 2023-09-23T19:21:20.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:19 smithi016 bash[77995]: cluster 2023-09-23T19:21:19.064312+0000 mon.a (mon.0) 476 : cluster 3 Health check update: Degraded data redundancy: 14567/60750 objects degraded (23.979%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:21:20.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:19 smithi016 bash[77995]: audit 2023-09-23T19:21:19.225986+0000 mon.c (mon.2) 42 : audit 0 from='client.? 172.21.15.16:0/1321114456' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:20.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:19 smithi016 bash[75004]: cluster 2023-09-23T19:21:19.064312+0000 mon.a (mon.0) 476 : cluster 3 Health check update: Degraded data redundancy: 14567/60750 objects degraded (23.979%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:21:20.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:19 smithi016 bash[75004]: audit 2023-09-23T19:21:19.225986+0000 mon.c (mon.2) 42 : audit 0 from='client.? 172.21.15.16:0/1321114456' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:20.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:19 smithi092 bash[58855]: cluster 2023-09-23T19:21:19.064312+0000 mon.a (mon.0) 476 : cluster 3 Health check update: Degraded data redundancy: 14567/60750 objects degraded (23.979%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:21:20.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:19 smithi092 bash[58855]: audit 2023-09-23T19:21:19.225986+0000 mon.c (mon.2) 42 : audit 0 from='client.? 172.21.15.16:0/1321114456' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:21.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:20 smithi016 bash[75004]: cluster 2023-09-23T19:21:19.684900+0000 mgr.y (mgr.25325) 352 : cluster 0 pgmap v275: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 516 op/s; 14567/60147 objects degraded (24.219%); 0 B/s, 16 objects/s recovering 2023-09-23T19:21:21.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:20 smithi016 bash[77995]: cluster 2023-09-23T19:21:19.684900+0000 mgr.y (mgr.25325) 352 : cluster 0 pgmap v275: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 516 op/s; 14567/60147 objects degraded (24.219%); 0 B/s, 16 objects/s recovering 2023-09-23T19:21:21.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:20 smithi092 bash[58855]: cluster 2023-09-23T19:21:19.684900+0000 mgr.y (mgr.25325) 352 : cluster 0 pgmap v275: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 516 op/s; 14567/60147 objects degraded (24.219%); 0 B/s, 16 objects/s recovering 2023-09-23T19:21:23.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:22 smithi016 bash[77995]: cluster 2023-09-23T19:21:21.685778+0000 mgr.y (mgr.25325) 353 : cluster 0 pgmap v276: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.1 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 14462/51828 objects degraded (27.904%); 0 B/s, 25 objects/s recovering 2023-09-23T19:21:23.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:22 smithi016 bash[77995]: audit 2023-09-23T19:21:21.833087+0000 mon.c (mon.2) 43 : audit 0 from='client.? 172.21.15.16:0/2953597523' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:23.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:22 smithi016 bash[75004]: cluster 2023-09-23T19:21:21.685778+0000 mgr.y (mgr.25325) 353 : cluster 0 pgmap v276: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.1 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 14462/51828 objects degraded (27.904%); 0 B/s, 25 objects/s recovering 2023-09-23T19:21:23.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:22 smithi016 bash[75004]: audit 2023-09-23T19:21:21.833087+0000 mon.c (mon.2) 43 : audit 0 from='client.? 172.21.15.16:0/2953597523' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:23.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:22 smithi092 bash[58855]: cluster 2023-09-23T19:21:21.685778+0000 mgr.y (mgr.25325) 353 : cluster 0 pgmap v276: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.1 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 14462/51828 objects degraded (27.904%); 0 B/s, 25 objects/s recovering 2023-09-23T19:21:23.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:22 smithi092 bash[58855]: audit 2023-09-23T19:21:21.833087+0000 mon.c (mon.2) 43 : audit 0 from='client.? 172.21.15.16:0/2953597523' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:25.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:24 smithi016 bash[75004]: cluster 2023-09-23T19:21:23.686439+0000 mgr.y (mgr.25325) 354 : cluster 0 pgmap v277: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.0 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 514 op/s; 14462/51258 objects degraded (28.214%); 0 B/s, 17 objects/s recovering 2023-09-23T19:21:25.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:24 smithi016 bash[75004]: cluster 2023-09-23T19:21:24.065492+0000 mon.a (mon.0) 477 : cluster 3 Health check update: Degraded data redundancy: 14462/51258 objects degraded (28.214%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:21:25.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:24 smithi016 bash[75004]: audit 2023-09-23T19:21:24.471904+0000 mon.c (mon.2) 44 : audit 0 from='client.? 172.21.15.16:0/3071455073' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:25.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:24 smithi016 bash[77995]: cluster 2023-09-23T19:21:23.686439+0000 mgr.y (mgr.25325) 354 : cluster 0 pgmap v277: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.0 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 514 op/s; 14462/51258 objects degraded (28.214%); 0 B/s, 17 objects/s recovering 2023-09-23T19:21:25.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:24 smithi016 bash[77995]: cluster 2023-09-23T19:21:24.065492+0000 mon.a (mon.0) 477 : cluster 3 Health check update: Degraded data redundancy: 14462/51258 objects degraded (28.214%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:21:25.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:24 smithi016 bash[77995]: audit 2023-09-23T19:21:24.471904+0000 mon.c (mon.2) 44 : audit 0 from='client.? 172.21.15.16:0/3071455073' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:25.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:24 smithi092 bash[58855]: cluster 2023-09-23T19:21:23.686439+0000 mgr.y (mgr.25325) 354 : cluster 0 pgmap v277: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.0 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 514 op/s; 14462/51258 objects degraded (28.214%); 0 B/s, 17 objects/s recovering 2023-09-23T19:21:25.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:24 smithi092 bash[58855]: cluster 2023-09-23T19:21:24.065492+0000 mon.a (mon.0) 477 : cluster 3 Health check update: Degraded data redundancy: 14462/51258 objects degraded (28.214%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:21:25.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:24 smithi092 bash[58855]: audit 2023-09-23T19:21:24.471904+0000 mon.c (mon.2) 44 : audit 0 from='client.? 172.21.15.16:0/3071455073' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:26.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:21:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:21:25] "GET /metrics HTTP/1.1" 200 36186 "" "Prometheus/2.33.4" 2023-09-23T19:21:27.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:26 smithi016 bash[75004]: cluster 2023-09-23T19:21:25.687305+0000 mgr.y (mgr.25325) 355 : cluster 0 pgmap v278: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 941 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 665 op/s; 14360/45207 objects degraded (31.765%); 0 B/s, 25 objects/s recovering 2023-09-23T19:21:27.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:26 smithi016 bash[77995]: cluster 2023-09-23T19:21:25.687305+0000 mgr.y (mgr.25325) 355 : cluster 0 pgmap v278: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 941 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 665 op/s; 14360/45207 objects degraded (31.765%); 0 B/s, 25 objects/s recovering 2023-09-23T19:21:27.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:26 smithi092 bash[58855]: cluster 2023-09-23T19:21:25.687305+0000 mgr.y (mgr.25325) 355 : cluster 0 pgmap v278: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 941 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 665 op/s; 14360/45207 objects degraded (31.765%); 0 B/s, 25 objects/s recovering 2023-09-23T19:21:28.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:27 smithi016 bash[75004]: audit 2023-09-23T19:21:27.138990+0000 mon.b (mon.1) 20 : audit 0 from='client.? 172.21.15.16:0/1600348831' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:28.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:27 smithi016 bash[77995]: audit 2023-09-23T19:21:27.138990+0000 mon.b (mon.1) 20 : audit 0 from='client.? 172.21.15.16:0/1600348831' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:28.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:27 smithi092 bash[58855]: audit 2023-09-23T19:21:27.138990+0000 mon.b (mon.1) 20 : audit 0 from='client.? 172.21.15.16:0/1600348831' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:29.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:28 smithi016 bash[75004]: cluster 2023-09-23T19:21:27.688081+0000 mgr.y (mgr.25325) 356 : cluster 0 pgmap v279: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 895 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 558 op/s; 14360/42990 objects degraded (33.403%); 0 B/s, 17 objects/s recovering 2023-09-23T19:21:29.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:28 smithi016 bash[75004]: audit 2023-09-23T19:21:28.621909+0000 mon.a (mon.0) 478 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:21:29.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:28 smithi016 bash[77995]: cluster 2023-09-23T19:21:27.688081+0000 mgr.y (mgr.25325) 356 : cluster 0 pgmap v279: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 895 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 558 op/s; 14360/42990 objects degraded (33.403%); 0 B/s, 17 objects/s recovering 2023-09-23T19:21:29.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:28 smithi016 bash[77995]: audit 2023-09-23T19:21:28.621909+0000 mon.a (mon.0) 478 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:21:29.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:28 smithi092 bash[58855]: cluster 2023-09-23T19:21:27.688081+0000 mgr.y (mgr.25325) 356 : cluster 0 pgmap v279: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 895 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 558 op/s; 14360/42990 objects degraded (33.403%); 0 B/s, 17 objects/s recovering 2023-09-23T19:21:29.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:28 smithi092 bash[58855]: audit 2023-09-23T19:21:28.621909+0000 mon.a (mon.0) 478 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:21:30.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:29 smithi016 bash[75004]: cluster 2023-09-23T19:21:29.066646+0000 mon.a (mon.0) 479 : cluster 3 Health check update: Degraded data redundancy: 14360/42990 objects degraded (33.403%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:21:30.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:29 smithi016 bash[75004]: audit 2023-09-23T19:21:29.757706+0000 mon.c (mon.2) 45 : audit 0 from='client.? 172.21.15.16:0/2172973796' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:30.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:29 smithi016 bash[77995]: cluster 2023-09-23T19:21:29.066646+0000 mon.a (mon.0) 479 : cluster 3 Health check update: Degraded data redundancy: 14360/42990 objects degraded (33.403%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:21:30.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:29 smithi016 bash[77995]: audit 2023-09-23T19:21:29.757706+0000 mon.c (mon.2) 45 : audit 0 from='client.? 172.21.15.16:0/2172973796' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:30.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:29 smithi092 bash[58855]: cluster 2023-09-23T19:21:29.066646+0000 mon.a (mon.0) 479 : cluster 3 Health check update: Degraded data redundancy: 14360/42990 objects degraded (33.403%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:21:30.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:29 smithi092 bash[58855]: audit 2023-09-23T19:21:29.757706+0000 mon.c (mon.2) 45 : audit 0 from='client.? 172.21.15.16:0/2172973796' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:31.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:30 smithi016 bash[75004]: cluster 2023-09-23T19:21:29.688731+0000 mgr.y (mgr.25325) 357 : cluster 0 pgmap v280: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 883 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 508 op/s; 14360/42420 objects degraded (33.852%); 0 B/s, 17 objects/s recovering 2023-09-23T19:21:31.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:30 smithi016 bash[77995]: cluster 2023-09-23T19:21:29.688731+0000 mgr.y (mgr.25325) 357 : cluster 0 pgmap v280: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 883 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 508 op/s; 14360/42420 objects degraded (33.852%); 0 B/s, 17 objects/s recovering 2023-09-23T19:21:31.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:30 smithi092 bash[58855]: cluster 2023-09-23T19:21:29.688731+0000 mgr.y (mgr.25325) 357 : cluster 0 pgmap v280: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 883 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 508 op/s; 14360/42420 objects degraded (33.852%); 0 B/s, 17 objects/s recovering 2023-09-23T19:21:32.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:31 smithi016 bash[75004]: audit 2023-09-23T19:21:30.780939+0000 mon.a (mon.0) 480 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:21:32.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:31 smithi016 bash[77995]: audit 2023-09-23T19:21:30.780939+0000 mon.a (mon.0) 480 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:21:32.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:31 smithi092 bash[58855]: audit 2023-09-23T19:21:30.780939+0000 mon.a (mon.0) 480 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:21:33.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:32 smithi016 bash[77995]: cluster 2023-09-23T19:21:31.689594+0000 mgr.y (mgr.25325) 358 : cluster 0 pgmap v281: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 708 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 724 op/s; 14252/34050 objects degraded (41.856%); 0 B/s, 26 objects/s recovering 2023-09-23T19:21:33.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:32 smithi016 bash[77995]: audit 2023-09-23T19:21:32.406129+0000 mon.c (mon.2) 46 : audit 0 from='client.? 172.21.15.16:0/4214658131' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:33.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:32 smithi016 bash[75004]: cluster 2023-09-23T19:21:31.689594+0000 mgr.y (mgr.25325) 358 : cluster 0 pgmap v281: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 708 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 724 op/s; 14252/34050 objects degraded (41.856%); 0 B/s, 26 objects/s recovering 2023-09-23T19:21:33.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:32 smithi016 bash[75004]: audit 2023-09-23T19:21:32.406129+0000 mon.c (mon.2) 46 : audit 0 from='client.? 172.21.15.16:0/4214658131' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:33.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:32 smithi092 bash[58855]: cluster 2023-09-23T19:21:31.689594+0000 mgr.y (mgr.25325) 358 : cluster 0 pgmap v281: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 708 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 724 op/s; 14252/34050 objects degraded (41.856%); 0 B/s, 26 objects/s recovering 2023-09-23T19:21:33.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:32 smithi092 bash[58855]: audit 2023-09-23T19:21:32.406129+0000 mon.c (mon.2) 46 : audit 0 from='client.? 172.21.15.16:0/4214658131' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:34.283 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:33 smithi016 bash[77995]: cluster 2023-09-23T19:21:33.690207+0000 mgr.y (mgr.25325) 359 : cluster 0 pgmap v282: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 697 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 508 op/s; 14252/33504 objects degraded (42.538%); 0 B/s, 17 objects/s recovering 2023-09-23T19:21:34.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:33 smithi016 bash[75004]: cluster 2023-09-23T19:21:33.690207+0000 mgr.y (mgr.25325) 359 : cluster 0 pgmap v282: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 697 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 508 op/s; 14252/33504 objects degraded (42.538%); 0 B/s, 17 objects/s recovering 2023-09-23T19:21:34.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:33 smithi092 bash[58855]: cluster 2023-09-23T19:21:33.690207+0000 mgr.y (mgr.25325) 359 : cluster 0 pgmap v282: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 697 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 508 op/s; 14252/33504 objects degraded (42.538%); 0 B/s, 17 objects/s recovering 2023-09-23T19:21:35.283 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:34 smithi016 bash[77995]: cluster 2023-09-23T19:21:34.067898+0000 mon.a (mon.0) 481 : cluster 3 Health check update: Degraded data redundancy: 14252/33504 objects degraded (42.538%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:21:35.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:34 smithi016 bash[75004]: cluster 2023-09-23T19:21:34.067898+0000 mon.a (mon.0) 481 : cluster 3 Health check update: Degraded data redundancy: 14252/33504 objects degraded (42.538%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:21:35.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:34 smithi092 bash[58855]: cluster 2023-09-23T19:21:34.067898+0000 mon.a (mon.0) 481 : cluster 3 Health check update: Degraded data redundancy: 14252/33504 objects degraded (42.538%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:21:36.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:35 smithi016 bash[75004]: audit 2023-09-23T19:21:35.062910+0000 mon.b (mon.1) 21 : audit 0 from='client.? 172.21.15.16:0/3212218017' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:36.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:35 smithi016 bash[75004]: cluster 2023-09-23T19:21:35.690999+0000 mgr.y (mgr.25325) 360 : cluster 0 pgmap v283: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 574 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 14158/27585 objects degraded (51.325%); 0 B/s, 25 objects/s recovering 2023-09-23T19:21:36.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:21:35 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:21:35] "GET /metrics HTTP/1.1" 200 36183 "" "Prometheus/2.33.4" 2023-09-23T19:21:36.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:35 smithi016 bash[77995]: audit 2023-09-23T19:21:35.062910+0000 mon.b (mon.1) 21 : audit 0 from='client.? 172.21.15.16:0/3212218017' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:36.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:35 smithi016 bash[77995]: cluster 2023-09-23T19:21:35.690999+0000 mgr.y (mgr.25325) 360 : cluster 0 pgmap v283: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 574 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 14158/27585 objects degraded (51.325%); 0 B/s, 25 objects/s recovering 2023-09-23T19:21:36.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:35 smithi092 bash[58855]: audit 2023-09-23T19:21:35.062910+0000 mon.b (mon.1) 21 : audit 0 from='client.? 172.21.15.16:0/3212218017' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:36.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:35 smithi092 bash[58855]: cluster 2023-09-23T19:21:35.690999+0000 mgr.y (mgr.25325) 360 : cluster 0 pgmap v283: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 574 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 14158/27585 objects degraded (51.325%); 0 B/s, 25 objects/s recovering 2023-09-23T19:21:38.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:38 smithi016 bash[75004]: audit 2023-09-23T19:21:37.224727+0000 mon.a (mon.0) 482 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:21:38.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:38 smithi016 bash[75004]: audit 2023-09-23T19:21:37.250267+0000 mon.a (mon.0) 483 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:21:38.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:38 smithi016 bash[75004]: audit 2023-09-23T19:21:37.340705+0000 mon.a (mon.0) 484 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:21:38.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:38 smithi016 bash[75004]: audit 2023-09-23T19:21:37.354290+0000 mon.a (mon.0) 485 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:21:38.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:38 smithi016 bash[75004]: cluster 2023-09-23T19:21:37.691644+0000 mgr.y (mgr.25325) 361 : cluster 0 pgmap v284: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 528 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 549 op/s; 14158/25407 objects degraded (55.725%); 0 B/s, 16 objects/s recovering 2023-09-23T19:21:38.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:38 smithi016 bash[75004]: audit 2023-09-23T19:21:37.712893+0000 mon.c (mon.2) 47 : audit 0 from='client.? 172.21.15.16:0/2364737046' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:38.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:38 smithi016 bash[75004]: audit 2023-09-23T19:21:37.995865+0000 mon.a (mon.0) 486 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:21:38.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:38 smithi016 bash[75004]: audit 2023-09-23T19:21:37.997391+0000 mon.a (mon.0) 487 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:21:38.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:38 smithi016 bash[75004]: audit 2023-09-23T19:21:38.015531+0000 mon.a (mon.0) 488 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:21:38.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:38 smithi016 bash[75004]: audit 2023-09-23T19:21:38.095319+0000 mon.a (mon.0) 489 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:21:38.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:38 smithi016 bash[75004]: audit 2023-09-23T19:21:38.099828+0000 mon.a (mon.0) 490 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:21:38.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:38 smithi016 bash[75004]: audit 2023-09-23T19:21:38.102395+0000 mon.a (mon.0) 491 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:21:38.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:38 smithi016 bash[75004]: audit 2023-09-23T19:21:38.104117+0000 mon.a (mon.0) 492 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:21:38.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:38 smithi016 bash[75004]: audit 2023-09-23T19:21:38.107125+0000 mon.a (mon.0) 493 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:21:38.536 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:21:38 smithi016 bash[70992]: debug 2023-09-23T19:21:38.102+0000 7fb2a5a7c700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:21:38.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:38 smithi016 bash[77995]: audit 2023-09-23T19:21:37.224727+0000 mon.a (mon.0) 482 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:21:38.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:38 smithi016 bash[77995]: audit 2023-09-23T19:21:37.250267+0000 mon.a (mon.0) 483 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:21:38.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:38 smithi016 bash[77995]: audit 2023-09-23T19:21:37.340705+0000 mon.a (mon.0) 484 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:21:38.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:38 smithi016 bash[77995]: audit 2023-09-23T19:21:37.354290+0000 mon.a (mon.0) 485 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:21:38.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:38 smithi016 bash[77995]: cluster 2023-09-23T19:21:37.691644+0000 mgr.y (mgr.25325) 361 : cluster 0 pgmap v284: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 528 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 549 op/s; 14158/25407 objects degraded (55.725%); 0 B/s, 16 objects/s recovering 2023-09-23T19:21:38.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:38 smithi016 bash[77995]: audit 2023-09-23T19:21:37.712893+0000 mon.c (mon.2) 47 : audit 0 from='client.? 172.21.15.16:0/2364737046' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:38.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:38 smithi016 bash[77995]: audit 2023-09-23T19:21:37.995865+0000 mon.a (mon.0) 486 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:21:38.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:38 smithi016 bash[77995]: audit 2023-09-23T19:21:37.997391+0000 mon.a (mon.0) 487 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:21:38.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:38 smithi016 bash[77995]: audit 2023-09-23T19:21:38.015531+0000 mon.a (mon.0) 488 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:21:38.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:38 smithi016 bash[77995]: audit 2023-09-23T19:21:38.095319+0000 mon.a (mon.0) 489 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:21:38.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:38 smithi016 bash[77995]: audit 2023-09-23T19:21:38.099828+0000 mon.a (mon.0) 490 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:21:38.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:38 smithi016 bash[77995]: audit 2023-09-23T19:21:38.102395+0000 mon.a (mon.0) 491 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:21:38.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:38 smithi016 bash[77995]: audit 2023-09-23T19:21:38.104117+0000 mon.a (mon.0) 492 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:21:38.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:38 smithi016 bash[77995]: audit 2023-09-23T19:21:38.107125+0000 mon.a (mon.0) 493 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:21:38.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:38 smithi092 bash[58855]: audit 2023-09-23T19:21:37.224727+0000 mon.a (mon.0) 482 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:21:38.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:38 smithi092 bash[58855]: audit 2023-09-23T19:21:37.250267+0000 mon.a (mon.0) 483 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:21:38.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:38 smithi092 bash[58855]: audit 2023-09-23T19:21:37.340705+0000 mon.a (mon.0) 484 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:21:38.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:38 smithi092 bash[58855]: audit 2023-09-23T19:21:37.354290+0000 mon.a (mon.0) 485 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:21:38.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:38 smithi092 bash[58855]: cluster 2023-09-23T19:21:37.691644+0000 mgr.y (mgr.25325) 361 : cluster 0 pgmap v284: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 528 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 549 op/s; 14158/25407 objects degraded (55.725%); 0 B/s, 16 objects/s recovering 2023-09-23T19:21:38.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:38 smithi092 bash[58855]: audit 2023-09-23T19:21:37.712893+0000 mon.c (mon.2) 47 : audit 0 from='client.? 172.21.15.16:0/2364737046' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:38.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:38 smithi092 bash[58855]: audit 2023-09-23T19:21:37.995865+0000 mon.a (mon.0) 486 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:21:38.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:38 smithi092 bash[58855]: audit 2023-09-23T19:21:37.997391+0000 mon.a (mon.0) 487 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:21:38.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:38 smithi092 bash[58855]: audit 2023-09-23T19:21:38.015531+0000 mon.a (mon.0) 488 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:21:38.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:38 smithi092 bash[58855]: audit 2023-09-23T19:21:38.095319+0000 mon.a (mon.0) 489 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:21:38.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:38 smithi092 bash[58855]: audit 2023-09-23T19:21:38.099828+0000 mon.a (mon.0) 490 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:21:38.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:38 smithi092 bash[58855]: audit 2023-09-23T19:21:38.102395+0000 mon.a (mon.0) 491 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:21:38.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:38 smithi092 bash[58855]: audit 2023-09-23T19:21:38.104117+0000 mon.a (mon.0) 492 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:21:38.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:38 smithi092 bash[58855]: audit 2023-09-23T19:21:38.107125+0000 mon.a (mon.0) 493 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:21:39.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:39 smithi016 bash[75004]: audit 2023-09-23T19:21:38.107421+0000 mgr.y (mgr.25325) 362 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:21:39.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:39 smithi016 bash[75004]: cephadm 2023-09-23T19:21:38.108688+0000 mgr.y (mgr.25325) 363 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:21:39.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:39 smithi016 bash[75004]: cluster 2023-09-23T19:21:39.069143+0000 mon.a (mon.0) 494 : cluster 3 Health check update: Degraded data redundancy: 14158/25407 objects degraded (55.725%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:21:39.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:39 smithi016 bash[77995]: audit 2023-09-23T19:21:38.107421+0000 mgr.y (mgr.25325) 362 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:21:39.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:39 smithi016 bash[77995]: cephadm 2023-09-23T19:21:38.108688+0000 mgr.y (mgr.25325) 363 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:21:39.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:39 smithi016 bash[77995]: cluster 2023-09-23T19:21:39.069143+0000 mon.a (mon.0) 494 : cluster 3 Health check update: Degraded data redundancy: 14158/25407 objects degraded (55.725%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:21:39.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:39 smithi092 bash[58855]: audit 2023-09-23T19:21:38.107421+0000 mgr.y (mgr.25325) 362 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:21:39.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:39 smithi092 bash[58855]: cephadm 2023-09-23T19:21:38.108688+0000 mgr.y (mgr.25325) 363 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:21:39.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:39 smithi092 bash[58855]: cluster 2023-09-23T19:21:39.069143+0000 mon.a (mon.0) 494 : cluster 3 Health check update: Degraded data redundancy: 14158/25407 objects degraded (55.725%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:21:40.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:40 smithi016 bash[75004]: cluster 2023-09-23T19:21:39.692124+0000 mgr.y (mgr.25325) 364 : cluster 0 pgmap v285: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 517 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 503 op/s; 14158/24867 objects degraded (56.935%); 0 B/s, 16 objects/s recovering 2023-09-23T19:21:40.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:40 smithi016 bash[75004]: audit 2023-09-23T19:21:40.303701+0000 mon.a (mon.0) 495 : audit 0 from='client.? 172.21.15.16:0/1138003026' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:40.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:40 smithi016 bash[77995]: cluster 2023-09-23T19:21:39.692124+0000 mgr.y (mgr.25325) 364 : cluster 0 pgmap v285: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 517 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 503 op/s; 14158/24867 objects degraded (56.935%); 0 B/s, 16 objects/s recovering 2023-09-23T19:21:40.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:40 smithi016 bash[77995]: audit 2023-09-23T19:21:40.303701+0000 mon.a (mon.0) 495 : audit 0 from='client.? 172.21.15.16:0/1138003026' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:40.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:40 smithi092 bash[58855]: cluster 2023-09-23T19:21:39.692124+0000 mgr.y (mgr.25325) 364 : cluster 0 pgmap v285: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 517 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 503 op/s; 14158/24867 objects degraded (56.935%); 0 B/s, 16 objects/s recovering 2023-09-23T19:21:40.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:40 smithi092 bash[58855]: audit 2023-09-23T19:21:40.303701+0000 mon.a (mon.0) 495 : audit 0 from='client.? 172.21.15.16:0/1138003026' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:43.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:42 smithi016 bash[75004]: cluster 2023-09-23T19:21:41.693055+0000 mgr.y (mgr.25325) 365 : cluster 0 pgmap v286: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 14054/16734 objects degraded (83.985%); 0 B/s, 25 objects/s recovering 2023-09-23T19:21:43.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:42 smithi016 bash[77995]: cluster 2023-09-23T19:21:41.693055+0000 mgr.y (mgr.25325) 365 : cluster 0 pgmap v286: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 14054/16734 objects degraded (83.985%); 0 B/s, 25 objects/s recovering 2023-09-23T19:21:43.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:42 smithi092 bash[58855]: cluster 2023-09-23T19:21:41.693055+0000 mgr.y (mgr.25325) 365 : cluster 0 pgmap v286: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 14054/16734 objects degraded (83.985%); 0 B/s, 25 objects/s recovering 2023-09-23T19:21:44.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:43 smithi016 bash[75004]: audit 2023-09-23T19:21:42.954885+0000 mon.b (mon.1) 22 : audit 0 from='client.? 172.21.15.16:0/4024521321' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:44.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:43 smithi016 bash[75004]: audit 2023-09-23T19:21:43.622041+0000 mon.a (mon.0) 496 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:21:44.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:43 smithi016 bash[77995]: audit 2023-09-23T19:21:42.954885+0000 mon.b (mon.1) 22 : audit 0 from='client.? 172.21.15.16:0/4024521321' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:44.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:43 smithi016 bash[77995]: audit 2023-09-23T19:21:43.622041+0000 mon.a (mon.0) 496 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:21:44.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:43 smithi092 bash[58855]: audit 2023-09-23T19:21:42.954885+0000 mon.b (mon.1) 22 : audit 0 from='client.? 172.21.15.16:0/4024521321' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:44.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:43 smithi092 bash[58855]: audit 2023-09-23T19:21:43.622041+0000 mon.a (mon.0) 496 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:21:45.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:44 smithi016 bash[75004]: cluster 2023-09-23T19:21:43.693673+0000 mgr.y (mgr.25325) 366 : cluster 0 pgmap v287: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 497 op/s; 14054/16146 objects degraded (87.043%); 0 B/s, 16 objects/s recovering 2023-09-23T19:21:45.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:44 smithi016 bash[75004]: cluster 2023-09-23T19:21:44.074767+0000 mon.a (mon.0) 497 : cluster 3 Health check update: Degraded data redundancy: 14054/16146 objects degraded (87.043%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:21:45.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:44 smithi016 bash[77995]: cluster 2023-09-23T19:21:43.693673+0000 mgr.y (mgr.25325) 366 : cluster 0 pgmap v287: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 497 op/s; 14054/16146 objects degraded (87.043%); 0 B/s, 16 objects/s recovering 2023-09-23T19:21:45.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:44 smithi016 bash[77995]: cluster 2023-09-23T19:21:44.074767+0000 mon.a (mon.0) 497 : cluster 3 Health check update: Degraded data redundancy: 14054/16146 objects degraded (87.043%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:21:45.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:44 smithi092 bash[58855]: cluster 2023-09-23T19:21:43.693673+0000 mgr.y (mgr.25325) 366 : cluster 0 pgmap v287: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 497 op/s; 14054/16146 objects degraded (87.043%); 0 B/s, 16 objects/s recovering 2023-09-23T19:21:45.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:44 smithi092 bash[58855]: cluster 2023-09-23T19:21:44.074767+0000 mon.a (mon.0) 497 : cluster 3 Health check update: Degraded data redundancy: 14054/16146 objects degraded (87.043%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:21:46.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:45 smithi016 bash[75004]: audit 2023-09-23T19:21:45.521922+0000 mon.a (mon.0) 498 : audit 0 from='client.? 172.21.15.16:0/2779941763' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:46.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:21:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:21:45] "GET /metrics HTTP/1.1" 200 36188 "" "Prometheus/2.33.4" 2023-09-23T19:21:46.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:45 smithi016 bash[77995]: audit 2023-09-23T19:21:45.521922+0000 mon.a (mon.0) 498 : audit 0 from='client.? 172.21.15.16:0/2779941763' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:46.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:45 smithi092 bash[58855]: audit 2023-09-23T19:21:45.521922+0000 mon.a (mon.0) 498 : audit 0 from='client.? 172.21.15.16:0/2779941763' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:47.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:46 smithi016 bash[75004]: cluster 2023-09-23T19:21:45.694703+0000 mgr.y (mgr.25325) 367 : cluster 0 pgmap v288: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 206 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 13952/9912 objects degraded (140.759%); 0 B/s, 24 objects/s recovering 2023-09-23T19:21:47.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:46 smithi016 bash[77995]: cluster 2023-09-23T19:21:45.694703+0000 mgr.y (mgr.25325) 367 : cluster 0 pgmap v288: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 206 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 13952/9912 objects degraded (140.759%); 0 B/s, 24 objects/s recovering 2023-09-23T19:21:47.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:46 smithi092 bash[58855]: cluster 2023-09-23T19:21:45.694703+0000 mgr.y (mgr.25325) 367 : cluster 0 pgmap v288: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 206 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 13952/9912 objects degraded (140.759%); 0 B/s, 24 objects/s recovering 2023-09-23T19:21:49.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:48 smithi092 bash[58855]: cluster 2023-09-23T19:21:47.695397+0000 mgr.y (mgr.25325) 368 : cluster 0 pgmap v289: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 157 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 555 op/s; 13952/7590 objects degraded (183.821%); 0 B/s, 17 objects/s recovering 2023-09-23T19:21:49.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:48 smithi092 bash[58855]: audit 2023-09-23T19:21:48.164717+0000 mon.a (mon.0) 499 : audit 0 from='client.? 172.21.15.16:0/1187943788' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:49.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:48 smithi016 bash[75004]: cluster 2023-09-23T19:21:47.695397+0000 mgr.y (mgr.25325) 368 : cluster 0 pgmap v289: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 157 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 555 op/s; 13952/7590 objects degraded (183.821%); 0 B/s, 17 objects/s recovering 2023-09-23T19:21:49.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:48 smithi016 bash[75004]: audit 2023-09-23T19:21:48.164717+0000 mon.a (mon.0) 499 : audit 0 from='client.? 172.21.15.16:0/1187943788' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:49.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:48 smithi016 bash[77995]: cluster 2023-09-23T19:21:47.695397+0000 mgr.y (mgr.25325) 368 : cluster 0 pgmap v289: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 157 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 555 op/s; 13952/7590 objects degraded (183.821%); 0 B/s, 17 objects/s recovering 2023-09-23T19:21:49.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:48 smithi016 bash[77995]: audit 2023-09-23T19:21:48.164717+0000 mon.a (mon.0) 499 : audit 0 from='client.? 172.21.15.16:0/1187943788' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:49.355 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Removed 98409 objects 2023-09-23T19:21:49.355 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Clean up completed and total clean up time :160.383 2023-09-23T19:21:50.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:49 smithi016 bash[75004]: cluster 2023-09-23T19:21:49.076282+0000 mon.a (mon.0) 500 : cluster 3 Health check update: Degraded data redundancy: 13952/7590 objects degraded (183.821%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:21:50.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:49 smithi016 bash[77995]: cluster 2023-09-23T19:21:49.076282+0000 mon.a (mon.0) 500 : cluster 3 Health check update: Degraded data redundancy: 13952/7590 objects degraded (183.821%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:21:50.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:49 smithi092 bash[58855]: cluster 2023-09-23T19:21:49.076282+0000 mon.a (mon.0) 500 : cluster 3 Health check update: Degraded data redundancy: 13952/7590 objects degraded (183.821%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:21:51.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:50 smithi016 bash[75004]: cluster 2023-09-23T19:21:49.696537+0000 mgr.y (mgr.25325) 369 : cluster 0 pgmap v290: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 145 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 510 op/s; 13952/7005 objects degraded (199.172%); 0 B/s, 17 objects/s recovering 2023-09-23T19:21:51.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:50 smithi016 bash[75004]: audit 2023-09-23T19:21:50.555115+0000 mon.a (mon.0) 501 : audit 0 from='client.? 172.21.15.16:0/1527682332' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:51.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:50 smithi016 bash[77995]: cluster 2023-09-23T19:21:49.696537+0000 mgr.y (mgr.25325) 369 : cluster 0 pgmap v290: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 145 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 510 op/s; 13952/7005 objects degraded (199.172%); 0 B/s, 17 objects/s recovering 2023-09-23T19:21:51.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:50 smithi016 bash[77995]: audit 2023-09-23T19:21:50.555115+0000 mon.a (mon.0) 501 : audit 0 from='client.? 172.21.15.16:0/1527682332' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:51.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:50 smithi092 bash[58855]: cluster 2023-09-23T19:21:49.696537+0000 mgr.y (mgr.25325) 369 : cluster 0 pgmap v290: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 145 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 510 op/s; 13952/7005 objects degraded (199.172%); 0 B/s, 17 objects/s recovering 2023-09-23T19:21:51.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:50 smithi092 bash[58855]: audit 2023-09-23T19:21:50.555115+0000 mon.a (mon.0) 501 : audit 0 from='client.? 172.21.15.16:0/1527682332' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:52.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:51 smithi016 bash[75004]: cluster 2023-09-23T19:21:51.697845+0000 mgr.y (mgr.25325) 370 : cluster 0 pgmap v291: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.4 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 13847/255 objects degraded (5430.196%); 0 B/s, 25 objects/s recovering 2023-09-23T19:21:52.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:51 smithi016 bash[77995]: cluster 2023-09-23T19:21:51.697845+0000 mgr.y (mgr.25325) 370 : cluster 0 pgmap v291: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.4 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 13847/255 objects degraded (5430.196%); 0 B/s, 25 objects/s recovering 2023-09-23T19:21:52.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:51 smithi092 bash[58855]: cluster 2023-09-23T19:21:51.697845+0000 mgr.y (mgr.25325) 370 : cluster 0 pgmap v291: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.4 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 13847/255 objects degraded (5430.196%); 0 B/s, 25 objects/s recovering 2023-09-23T19:21:53.284 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:21:53 smithi016 bash[70992]: debug 2023-09-23T19:21:53.118+0000 7fb2a5a7c700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:21:53.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:52 smithi016 bash[75004]: audit 2023-09-23T19:21:52.942666+0000 mon.c (mon.2) 48 : audit 0 from='client.? 172.21.15.16:0/754177635' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:53.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:52 smithi016 bash[77995]: audit 2023-09-23T19:21:52.942666+0000 mon.c (mon.2) 48 : audit 0 from='client.? 172.21.15.16:0/754177635' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:53.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:52 smithi092 bash[58855]: audit 2023-09-23T19:21:52.942666+0000 mon.c (mon.2) 48 : audit 0 from='client.? 172.21.15.16:0/754177635' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:54.159 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_2 2023-09-23T19:21:54.160 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_2 unique_pool_2 --yes-i-really-really-mean-it 2023-09-23T19:21:54.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:54 smithi016 bash[75004]: audit 2023-09-23T19:21:53.123443+0000 mon.a (mon.0) 502 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:21:54.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:54 smithi016 bash[75004]: audit 2023-09-23T19:21:53.123842+0000 mgr.y (mgr.25325) 371 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:21:54.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:54 smithi016 bash[75004]: cephadm 2023-09-23T19:21:53.125595+0000 mgr.y (mgr.25325) 372 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:21:54.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:54 smithi016 bash[75004]: cluster 2023-09-23T19:21:53.698867+0000 mgr.y (mgr.25325) 373 : cluster 0 pgmap v292: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 462 op/s; 13847/72 objects degraded (19231.944%); 0 B/s, 17 objects/s recovering 2023-09-23T19:21:54.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:54 smithi016 bash[77995]: audit 2023-09-23T19:21:53.123443+0000 mon.a (mon.0) 502 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:21:54.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:54 smithi016 bash[77995]: audit 2023-09-23T19:21:53.123842+0000 mgr.y (mgr.25325) 371 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:21:54.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:54 smithi016 bash[77995]: cephadm 2023-09-23T19:21:53.125595+0000 mgr.y (mgr.25325) 372 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:21:54.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:54 smithi016 bash[77995]: cluster 2023-09-23T19:21:53.698867+0000 mgr.y (mgr.25325) 373 : cluster 0 pgmap v292: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 462 op/s; 13847/72 objects degraded (19231.944%); 0 B/s, 17 objects/s recovering 2023-09-23T19:21:54.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:54 smithi092 bash[58855]: audit 2023-09-23T19:21:53.123443+0000 mon.a (mon.0) 502 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:21:54.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:54 smithi092 bash[58855]: audit 2023-09-23T19:21:53.123842+0000 mgr.y (mgr.25325) 371 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:21:54.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:54 smithi092 bash[58855]: cephadm 2023-09-23T19:21:53.125595+0000 mgr.y (mgr.25325) 372 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:21:54.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:54 smithi092 bash[58855]: cluster 2023-09-23T19:21:53.698867+0000 mgr.y (mgr.25325) 373 : cluster 0 pgmap v292: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 462 op/s; 13847/72 objects degraded (19231.944%); 0 B/s, 17 objects/s recovering 2023-09-23T19:21:55.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:55 smithi016 bash[75004]: cluster 2023-09-23T19:21:54.080781+0000 mon.a (mon.0) 503 : cluster 3 Health check update: Degraded data redundancy: 13847/72 objects degraded (19231.944%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:21:55.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:55 smithi016 bash[77995]: cluster 2023-09-23T19:21:54.080781+0000 mon.a (mon.0) 503 : cluster 3 Health check update: Degraded data redundancy: 13847/72 objects degraded (19231.944%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:21:55.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:55 smithi092 bash[58855]: cluster 2023-09-23T19:21:54.080781+0000 mon.a (mon.0) 503 : cluster 3 Health check update: Degraded data redundancy: 13847/72 objects degraded (19231.944%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:21:56.019 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:21:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:21:55] "GET /metrics HTTP/1.1" 200 36188 "" "Prometheus/2.33.4" 2023-09-23T19:21:56.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:56 smithi016 bash[75004]: audit 2023-09-23T19:21:55.316444+0000 mon.a (mon.0) 504 : audit 0 from='client.? 172.21.15.16:0/1458226160' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:56.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:56 smithi016 bash[75004]: cluster 2023-09-23T19:21:55.700202+0000 mgr.y (mgr.25325) 374 : cluster 0 pgmap v293: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 446 op/s; 13730/72 objects degraded (19069.444%); 0 B/s, 26 objects/s recovering 2023-09-23T19:21:56.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:56 smithi016 bash[77995]: audit 2023-09-23T19:21:55.316444+0000 mon.a (mon.0) 504 : audit 0 from='client.? 172.21.15.16:0/1458226160' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:56.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:56 smithi016 bash[77995]: cluster 2023-09-23T19:21:55.700202+0000 mgr.y (mgr.25325) 374 : cluster 0 pgmap v293: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 446 op/s; 13730/72 objects degraded (19069.444%); 0 B/s, 26 objects/s recovering 2023-09-23T19:21:56.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:56 smithi092 bash[58855]: audit 2023-09-23T19:21:55.316444+0000 mon.a (mon.0) 504 : audit 0 from='client.? 172.21.15.16:0/1458226160' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:56.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:56 smithi092 bash[58855]: cluster 2023-09-23T19:21:55.700202+0000 mgr.y (mgr.25325) 374 : cluster 0 pgmap v293: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 446 op/s; 13730/72 objects degraded (19069.444%); 0 B/s, 26 objects/s recovering 2023-09-23T19:21:58.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:57 smithi016 bash[75004]: audit 2023-09-23T19:21:57.696992+0000 mon.c (mon.2) 49 : audit 0 from='client.? 172.21.15.16:0/3020619125' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:58.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:57 smithi016 bash[77995]: audit 2023-09-23T19:21:57.696992+0000 mon.c (mon.2) 49 : audit 0 from='client.? 172.21.15.16:0/3020619125' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:58.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:57 smithi092 bash[58855]: audit 2023-09-23T19:21:57.696992+0000 mon.c (mon.2) 49 : audit 0 from='client.? 172.21.15.16:0/3020619125' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:21:58.943 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:21:59.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:58 smithi016 bash[75004]: cluster 2023-09-23T19:21:57.701222+0000 mgr.y (mgr.25325) 375 : cluster 0 pgmap v294: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 273 op/s; 13730/72 objects degraded (19069.444%); 0 B/s, 18 objects/s recovering 2023-09-23T19:21:59.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:58 smithi016 bash[75004]: audit 2023-09-23T19:21:58.623214+0000 mon.a (mon.0) 505 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:21:59.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:58 smithi016 bash[77995]: cluster 2023-09-23T19:21:57.701222+0000 mgr.y (mgr.25325) 375 : cluster 0 pgmap v294: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 273 op/s; 13730/72 objects degraded (19069.444%); 0 B/s, 18 objects/s recovering 2023-09-23T19:21:59.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:58 smithi016 bash[77995]: audit 2023-09-23T19:21:58.623214+0000 mon.a (mon.0) 505 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:21:59.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:58 smithi092 bash[58855]: cluster 2023-09-23T19:21:57.701222+0000 mgr.y (mgr.25325) 375 : cluster 0 pgmap v294: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 273 op/s; 13730/72 objects degraded (19069.444%); 0 B/s, 18 objects/s recovering 2023-09-23T19:21:59.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:58 smithi092 bash[58855]: audit 2023-09-23T19:21:58.623214+0000 mon.a (mon.0) 505 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:22:00.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:21:59 smithi016 bash[75004]: cluster 2023-09-23T19:21:59.082590+0000 mon.a (mon.0) 506 : cluster 3 Health check update: Degraded data redundancy: 13730/72 objects degraded (19069.444%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:22:00.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:21:59 smithi016 bash[77995]: cluster 2023-09-23T19:21:59.082590+0000 mon.a (mon.0) 506 : cluster 3 Health check update: Degraded data redundancy: 13730/72 objects degraded (19069.444%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:22:00.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:21:59 smithi092 bash[58855]: cluster 2023-09-23T19:21:59.082590+0000 mon.a (mon.0) 506 : cluster 3 Health check update: Degraded data redundancy: 13730/72 objects degraded (19069.444%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:22:01.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:00 smithi016 bash[77995]: cluster 2023-09-23T19:21:59.701911+0000 mgr.y (mgr.25325) 376 : cluster 0 pgmap v295: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 208 op/s; 13730/72 objects degraded (19069.444%); 0 B/s, 18 objects/s recovering 2023-09-23T19:22:01.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:00 smithi016 bash[77995]: audit 2023-09-23T19:22:00.070028+0000 mon.a (mon.0) 507 : audit 0 from='client.? 172.21.15.16:0/1158076902' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:01.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:00 smithi016 bash[75004]: cluster 2023-09-23T19:21:59.701911+0000 mgr.y (mgr.25325) 376 : cluster 0 pgmap v295: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 208 op/s; 13730/72 objects degraded (19069.444%); 0 B/s, 18 objects/s recovering 2023-09-23T19:22:01.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:00 smithi016 bash[75004]: audit 2023-09-23T19:22:00.070028+0000 mon.a (mon.0) 507 : audit 0 from='client.? 172.21.15.16:0/1158076902' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:01.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:00 smithi092 bash[58855]: cluster 2023-09-23T19:21:59.701911+0000 mgr.y (mgr.25325) 376 : cluster 0 pgmap v295: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 208 op/s; 13730/72 objects degraded (19069.444%); 0 B/s, 18 objects/s recovering 2023-09-23T19:22:01.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:00 smithi092 bash[58855]: audit 2023-09-23T19:22:00.070028+0000 mon.a (mon.0) 507 : audit 0 from='client.? 172.21.15.16:0/1158076902' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:03.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:02 smithi016 bash[75004]: cluster 2023-09-23T19:22:01.703195+0000 mgr.y (mgr.25325) 377 : cluster 0 pgmap v296: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 192 op/s; 13614/72 objects degraded (18908.333%); 0 B/s, 28 objects/s recovering 2023-09-23T19:22:03.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:02 smithi016 bash[75004]: audit 2023-09-23T19:22:02.436057+0000 mon.c (mon.2) 50 : audit 0 from='client.? 172.21.15.16:0/3581021658' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:03.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:02 smithi016 bash[77995]: cluster 2023-09-23T19:22:01.703195+0000 mgr.y (mgr.25325) 377 : cluster 0 pgmap v296: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 192 op/s; 13614/72 objects degraded (18908.333%); 0 B/s, 28 objects/s recovering 2023-09-23T19:22:03.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:02 smithi016 bash[77995]: audit 2023-09-23T19:22:02.436057+0000 mon.c (mon.2) 50 : audit 0 from='client.? 172.21.15.16:0/3581021658' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:03.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:02 smithi092 bash[58855]: cluster 2023-09-23T19:22:01.703195+0000 mgr.y (mgr.25325) 377 : cluster 0 pgmap v296: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 192 op/s; 13614/72 objects degraded (18908.333%); 0 B/s, 28 objects/s recovering 2023-09-23T19:22:03.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:02 smithi092 bash[58855]: audit 2023-09-23T19:22:02.436057+0000 mon.c (mon.2) 50 : audit 0 from='client.? 172.21.15.16:0/3581021658' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:03.787 INFO:teuthology.orchestra.run.smithi016.stderr:pool 'unique_pool_2' removed 2023-09-23T19:22:04.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:03 smithi016 bash[75004]: audit 2023-09-23T19:22:03.298356+0000 mon.a (mon.0) 508 : audit 1 from='client.? 172.21.15.16:0/879120032' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T19:22:04.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:03 smithi016 bash[77995]: audit 2023-09-23T19:22:03.298356+0000 mon.a (mon.0) 508 : audit 1 from='client.? 172.21.15.16:0/879120032' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T19:22:04.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:03 smithi092 bash[58855]: audit 2023-09-23T19:22:03.298356+0000 mon.a (mon.0) 508 : audit 1 from='client.? 172.21.15.16:0/879120032' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T19:22:04.566 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T19:22:04.567 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T19:22:04.567 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_3 2023-09-23T19:22:04.567 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_3 16 2023-09-23T19:22:05.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:04 smithi016 bash[75004]: cluster 2023-09-23T19:22:03.704190+0000 mgr.y (mgr.25325) 378 : cluster 0 pgmap v297: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 5 op/s; 13614/72 objects degraded (18908.333%); 0 B/s, 19 objects/s recovering 2023-09-23T19:22:05.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:04 smithi016 bash[75004]: cluster 2023-09-23T19:22:03.775920+0000 osd.3 (osd.3) 17 : cluster 3 failed to encode map e90 with expected crc 2023-09-23T19:22:05.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:04 smithi016 bash[75004]: audit 2023-09-23T19:22:03.778095+0000 mon.a (mon.0) 509 : audit 1 from='client.? 172.21.15.16:0/879120032' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T19:22:05.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:04 smithi016 bash[75004]: cluster 2023-09-23T19:22:03.778230+0000 mon.a (mon.0) 510 : cluster 0 osdmap e90: 8 total, 8 up, 8 in 2023-09-23T19:22:05.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:04 smithi016 bash[75004]: cluster 2023-09-23T19:22:04.084154+0000 mon.a (mon.0) 511 : cluster 3 Health check update: Degraded data redundancy: 13614/72 objects degraded (18908.333%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:22:05.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:04 smithi016 bash[77995]: cluster 2023-09-23T19:22:03.704190+0000 mgr.y (mgr.25325) 378 : cluster 0 pgmap v297: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 5 op/s; 13614/72 objects degraded (18908.333%); 0 B/s, 19 objects/s recovering 2023-09-23T19:22:05.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:04 smithi016 bash[77995]: cluster 2023-09-23T19:22:03.775920+0000 osd.3 (osd.3) 17 : cluster 3 failed to encode map e90 with expected crc 2023-09-23T19:22:05.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:04 smithi016 bash[77995]: audit 2023-09-23T19:22:03.778095+0000 mon.a (mon.0) 509 : audit 1 from='client.? 172.21.15.16:0/879120032' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T19:22:05.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:04 smithi016 bash[77995]: cluster 2023-09-23T19:22:03.778230+0000 mon.a (mon.0) 510 : cluster 0 osdmap e90: 8 total, 8 up, 8 in 2023-09-23T19:22:05.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:04 smithi016 bash[77995]: cluster 2023-09-23T19:22:04.084154+0000 mon.a (mon.0) 511 : cluster 3 Health check update: Degraded data redundancy: 13614/72 objects degraded (18908.333%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:22:05.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:04 smithi092 bash[58855]: cluster 2023-09-23T19:22:03.704190+0000 mgr.y (mgr.25325) 378 : cluster 0 pgmap v297: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 5 op/s; 13614/72 objects degraded (18908.333%); 0 B/s, 19 objects/s recovering 2023-09-23T19:22:05.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:04 smithi092 bash[58855]: cluster 2023-09-23T19:22:03.775920+0000 osd.3 (osd.3) 17 : cluster 3 failed to encode map e90 with expected crc 2023-09-23T19:22:05.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:04 smithi092 bash[58855]: audit 2023-09-23T19:22:03.778095+0000 mon.a (mon.0) 509 : audit 1 from='client.? 172.21.15.16:0/879120032' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T19:22:05.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:04 smithi092 bash[58855]: cluster 2023-09-23T19:22:03.778230+0000 mon.a (mon.0) 510 : cluster 0 osdmap e90: 8 total, 8 up, 8 in 2023-09-23T19:22:05.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:04 smithi092 bash[58855]: cluster 2023-09-23T19:22:04.084154+0000 mon.a (mon.0) 511 : cluster 3 Health check update: Degraded data redundancy: 13614/72 objects degraded (18908.333%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:22:06.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:05 smithi016 bash[75004]: cluster 2023-09-23T19:22:04.764122+0000 osd.2 (osd.2) 16 : cluster 3 failed to encode map e90 with expected crc 2023-09-23T19:22:06.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:05 smithi016 bash[75004]: audit 2023-09-23T19:22:04.827614+0000 mon.a (mon.0) 512 : audit 0 from='client.? 172.21.15.16:0/1120648469' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:06.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:05 smithi016 bash[77995]: cluster 2023-09-23T19:22:04.764122+0000 osd.2 (osd.2) 16 : cluster 3 failed to encode map e90 with expected crc 2023-09-23T19:22:06.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:05 smithi016 bash[77995]: audit 2023-09-23T19:22:04.827614+0000 mon.a (mon.0) 512 : audit 0 from='client.? 172.21.15.16:0/1120648469' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:06.035 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:22:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:22:05] "GET /metrics HTTP/1.1" 200 36138 "" "Prometheus/2.33.4" 2023-09-23T19:22:06.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:05 smithi092 bash[58855]: cluster 2023-09-23T19:22:04.764122+0000 osd.2 (osd.2) 16 : cluster 3 failed to encode map e90 with expected crc 2023-09-23T19:22:06.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:05 smithi092 bash[58855]: audit 2023-09-23T19:22:04.827614+0000 mon.a (mon.0) 512 : audit 0 from='client.? 172.21.15.16:0/1120648469' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:07.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:06 smithi016 bash[75004]: cluster 2023-09-23T19:22:05.705179+0000 mgr.y (mgr.25325) 379 : cluster 0 pgmap v299: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:22:07.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:06 smithi016 bash[75004]: cluster 2023-09-23T19:22:05.776646+0000 mon.a (mon.0) 513 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 13614/72 objects degraded (18908.333%), 7 pgs degraded, 7 pgs undersized) 2023-09-23T19:22:07.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:06 smithi016 bash[75004]: cluster 2023-09-23T19:22:05.776719+0000 mon.a (mon.0) 514 : cluster 1 Cluster is now healthy 2023-09-23T19:22:07.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:06 smithi016 bash[77995]: cluster 2023-09-23T19:22:05.705179+0000 mgr.y (mgr.25325) 379 : cluster 0 pgmap v299: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:22:07.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:06 smithi016 bash[77995]: cluster 2023-09-23T19:22:05.776646+0000 mon.a (mon.0) 513 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 13614/72 objects degraded (18908.333%), 7 pgs degraded, 7 pgs undersized) 2023-09-23T19:22:07.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:06 smithi016 bash[77995]: cluster 2023-09-23T19:22:05.776719+0000 mon.a (mon.0) 514 : cluster 1 Cluster is now healthy 2023-09-23T19:22:07.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:06 smithi092 bash[58855]: cluster 2023-09-23T19:22:05.705179+0000 mgr.y (mgr.25325) 379 : cluster 0 pgmap v299: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:22:07.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:06 smithi092 bash[58855]: cluster 2023-09-23T19:22:05.776646+0000 mon.a (mon.0) 513 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 13614/72 objects degraded (18908.333%), 7 pgs degraded, 7 pgs undersized) 2023-09-23T19:22:07.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:06 smithi092 bash[58855]: cluster 2023-09-23T19:22:05.776719+0000 mon.a (mon.0) 514 : cluster 1 Cluster is now healthy 2023-09-23T19:22:08.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:07 smithi016 bash[75004]: audit 2023-09-23T19:22:07.203082+0000 mon.c (mon.2) 51 : audit 0 from='client.? 172.21.15.16:0/4043901416' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:08.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:07 smithi016 bash[77995]: audit 2023-09-23T19:22:07.203082+0000 mon.c (mon.2) 51 : audit 0 from='client.? 172.21.15.16:0/4043901416' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:08.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:07 smithi092 bash[58855]: audit 2023-09-23T19:22:07.203082+0000 mon.c (mon.2) 51 : audit 0 from='client.? 172.21.15.16:0/4043901416' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:09.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:08 smithi016 bash[75004]: cluster 2023-09-23T19:22:07.705947+0000 mgr.y (mgr.25325) 380 : cluster 0 pgmap v300: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:22:09.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:08 smithi016 bash[75004]: audit 2023-09-23T19:22:08.141124+0000 mon.a (mon.0) 515 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:22:09.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:08 smithi016 bash[77995]: cluster 2023-09-23T19:22:07.705947+0000 mgr.y (mgr.25325) 380 : cluster 0 pgmap v300: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:22:09.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:08 smithi016 bash[77995]: audit 2023-09-23T19:22:08.141124+0000 mon.a (mon.0) 515 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:22:09.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:08 smithi092 bash[58855]: cluster 2023-09-23T19:22:07.705947+0000 mgr.y (mgr.25325) 380 : cluster 0 pgmap v300: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:22:09.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:08 smithi092 bash[58855]: audit 2023-09-23T19:22:08.141124+0000 mon.a (mon.0) 515 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:22:09.361 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:22:10.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:09 smithi016 bash[77995]: audit 2023-09-23T19:22:08.141571+0000 mgr.y (mgr.25325) 381 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:22:10.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:09 smithi016 bash[77995]: cephadm 2023-09-23T19:22:08.143713+0000 mgr.y (mgr.25325) 382 : cephadm 1 Upgrade: osd.1 is safe to restart 2023-09-23T19:22:10.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:09 smithi016 bash[77995]: audit 2023-09-23T19:22:09.573800+0000 mon.c (mon.2) 52 : audit 0 from='client.? 172.21.15.16:0/2446163399' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:10.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:09 smithi016 bash[75004]: audit 2023-09-23T19:22:08.141571+0000 mgr.y (mgr.25325) 381 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:22:10.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:09 smithi016 bash[75004]: cephadm 2023-09-23T19:22:08.143713+0000 mgr.y (mgr.25325) 382 : cephadm 1 Upgrade: osd.1 is safe to restart 2023-09-23T19:22:10.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:09 smithi016 bash[75004]: audit 2023-09-23T19:22:09.573800+0000 mon.c (mon.2) 52 : audit 0 from='client.? 172.21.15.16:0/2446163399' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:10.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:09 smithi092 bash[58855]: audit 2023-09-23T19:22:08.141571+0000 mgr.y (mgr.25325) 381 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:22:10.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:09 smithi092 bash[58855]: cephadm 2023-09-23T19:22:08.143713+0000 mgr.y (mgr.25325) 382 : cephadm 1 Upgrade: osd.1 is safe to restart 2023-09-23T19:22:10.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:09 smithi092 bash[58855]: audit 2023-09-23T19:22:09.573800+0000 mon.c (mon.2) 52 : audit 0 from='client.? 172.21.15.16:0/2446163399' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:11.224 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:10 smithi016 bash[77995]: cluster 2023-09-23T19:22:09.706424+0000 mgr.y (mgr.25325) 383 : cluster 0 pgmap v301: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:22:11.224 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:10 smithi016 bash[75004]: cluster 2023-09-23T19:22:09.706424+0000 mgr.y (mgr.25325) 383 : cluster 0 pgmap v301: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:22:11.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:10 smithi092 bash[58855]: cluster 2023-09-23T19:22:09.706424+0000 mgr.y (mgr.25325) 383 : cluster 0 pgmap v301: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:22:12.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:12 smithi016 bash[75004]: cephadm 2023-09-23T19:22:11.215375+0000 mgr.y (mgr.25325) 384 : cephadm 1 Upgrade: Updating osd.1 2023-09-23T19:22:12.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:12 smithi016 bash[75004]: audit 2023-09-23T19:22:11.223841+0000 mon.a (mon.0) 516 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:22:12.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:12 smithi016 bash[75004]: audit 2023-09-23T19:22:11.225411+0000 mon.a (mon.0) 517 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T19:22:12.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:12 smithi016 bash[75004]: audit 2023-09-23T19:22:11.226830+0000 mon.a (mon.0) 518 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:22:12.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:12 smithi016 bash[75004]: cephadm 2023-09-23T19:22:11.229812+0000 mgr.y (mgr.25325) 385 : cephadm 1 Deploying daemon osd.1 on smithi016 2023-09-23T19:22:12.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:12 smithi016 bash[75004]: cluster 2023-09-23T19:22:11.706993+0000 mgr.y (mgr.25325) 386 : cluster 0 pgmap v302: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:22:12.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:12 smithi016 bash[75004]: audit 2023-09-23T19:22:11.935771+0000 mon.c (mon.2) 53 : audit 0 from='client.? 172.21.15.16:0/3039410029' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:12.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:12 smithi016 bash[77995]: cephadm 2023-09-23T19:22:11.215375+0000 mgr.y (mgr.25325) 384 : cephadm 1 Upgrade: Updating osd.1 2023-09-23T19:22:12.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:12 smithi016 bash[77995]: audit 2023-09-23T19:22:11.223841+0000 mon.a (mon.0) 516 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:22:12.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:12 smithi016 bash[77995]: audit 2023-09-23T19:22:11.225411+0000 mon.a (mon.0) 517 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T19:22:12.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:12 smithi016 bash[77995]: audit 2023-09-23T19:22:11.226830+0000 mon.a (mon.0) 518 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:22:12.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:12 smithi016 bash[77995]: cephadm 2023-09-23T19:22:11.229812+0000 mgr.y (mgr.25325) 385 : cephadm 1 Deploying daemon osd.1 on smithi016 2023-09-23T19:22:12.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:12 smithi016 bash[77995]: cluster 2023-09-23T19:22:11.706993+0000 mgr.y (mgr.25325) 386 : cluster 0 pgmap v302: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:22:12.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:12 smithi016 bash[77995]: audit 2023-09-23T19:22:11.935771+0000 mon.c (mon.2) 53 : audit 0 from='client.? 172.21.15.16:0/3039410029' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:12.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:12 smithi092 bash[58855]: cephadm 2023-09-23T19:22:11.215375+0000 mgr.y (mgr.25325) 384 : cephadm 1 Upgrade: Updating osd.1 2023-09-23T19:22:12.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:12 smithi092 bash[58855]: audit 2023-09-23T19:22:11.223841+0000 mon.a (mon.0) 516 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:22:12.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:12 smithi092 bash[58855]: audit 2023-09-23T19:22:11.225411+0000 mon.a (mon.0) 517 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T19:22:12.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:12 smithi092 bash[58855]: audit 2023-09-23T19:22:11.226830+0000 mon.a (mon.0) 518 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:22:12.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:12 smithi092 bash[58855]: cephadm 2023-09-23T19:22:11.229812+0000 mgr.y (mgr.25325) 385 : cephadm 1 Deploying daemon osd.1 on smithi016 2023-09-23T19:22:12.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:12 smithi092 bash[58855]: cluster 2023-09-23T19:22:11.706993+0000 mgr.y (mgr.25325) 386 : cluster 0 pgmap v302: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:22:12.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:12 smithi092 bash[58855]: audit 2023-09-23T19:22:11.935771+0000 mon.c (mon.2) 53 : audit 0 from='client.? 172.21.15.16:0/3039410029' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:15.110 INFO:teuthology.orchestra.run.smithi016.stderr:pool 'unique_pool_3' created 2023-09-23T19:22:15.184 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:14 smithi016 bash[75004]: audit 2023-09-23T19:22:13.627226+0000 mon.a (mon.0) 519 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:22:15.184 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:14 smithi016 bash[75004]: audit 2023-09-23T19:22:13.628715+0000 mon.a (mon.0) 520 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:22:15.184 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:14 smithi016 bash[75004]: cluster 2023-09-23T19:22:13.707685+0000 mgr.y (mgr.25325) 387 : cluster 0 pgmap v303: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:22:15.184 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:14 smithi016 bash[75004]: audit 2023-09-23T19:22:13.728185+0000 mon.a (mon.0) 521 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:22:15.184 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:14 smithi016 bash[75004]: audit 2023-09-23T19:22:14.308470+0000 mon.a (mon.0) 522 : audit 0 from='client.? 172.21.15.16:0/3547670435' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:15.184 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:14 smithi016 bash[77995]: audit 2023-09-23T19:22:13.627226+0000 mon.a (mon.0) 519 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:22:15.184 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:14 smithi016 bash[77995]: audit 2023-09-23T19:22:13.628715+0000 mon.a (mon.0) 520 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:22:15.185 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:14 smithi016 bash[77995]: cluster 2023-09-23T19:22:13.707685+0000 mgr.y (mgr.25325) 387 : cluster 0 pgmap v303: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:22:15.185 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:14 smithi016 bash[77995]: audit 2023-09-23T19:22:13.728185+0000 mon.a (mon.0) 521 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:22:15.185 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:14 smithi016 bash[77995]: audit 2023-09-23T19:22:14.308470+0000 mon.a (mon.0) 522 : audit 0 from='client.? 172.21.15.16:0/3547670435' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:15.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:14 smithi092 bash[58855]: audit 2023-09-23T19:22:13.627226+0000 mon.a (mon.0) 519 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:22:15.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:14 smithi092 bash[58855]: audit 2023-09-23T19:22:13.628715+0000 mon.a (mon.0) 520 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:22:15.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:14 smithi092 bash[58855]: cluster 2023-09-23T19:22:13.707685+0000 mgr.y (mgr.25325) 387 : cluster 0 pgmap v303: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:22:15.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:14 smithi092 bash[58855]: audit 2023-09-23T19:22:13.728185+0000 mon.a (mon.0) 521 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:22:15.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:14 smithi092 bash[58855]: audit 2023-09-23T19:22:14.308470+0000 mon.a (mon.0) 522 : audit 0 from='client.? 172.21.15.16:0/3547670435' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:15.788 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:22:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:22:15] "GET /metrics HTTP/1.1" 200 33829 "" "Prometheus/2.33.4" 2023-09-23T19:22:16.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:15 smithi016 bash[77995]: audit 2023-09-23T19:22:14.448397+0000 mon.c (mon.2) 54 : audit 1 from='client.? 172.21.15.16:0/3017743629' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-09-23T19:22:16.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:15 smithi016 bash[77995]: audit 2023-09-23T19:22:14.448880+0000 mon.a (mon.0) 523 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-09-23T19:22:16.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:15 smithi016 bash[77995]: cluster 2023-09-23T19:22:15.093637+0000 osd.3 (osd.3) 18 : cluster 3 failed to encode map e91 with expected crc 2023-09-23T19:22:16.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:15 smithi016 bash[77995]: audit 2023-09-23T19:22:15.095739+0000 mon.a (mon.0) 524 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-09-23T19:22:16.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:15 smithi016 bash[77995]: cluster 2023-09-23T19:22:15.095838+0000 mon.a (mon.0) 525 : cluster 0 osdmap e91: 8 total, 8 up, 8 in 2023-09-23T19:22:16.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:15 smithi016 bash[77995]: cluster 2023-09-23T19:22:15.708513+0000 mgr.y (mgr.25325) 388 : cluster 0 pgmap v305: 89 pgs: 4 creating+peering, 12 unknown, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:22:16.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:15 smithi016 bash[75004]: audit 2023-09-23T19:22:14.448397+0000 mon.c (mon.2) 54 : audit 1 from='client.? 172.21.15.16:0/3017743629' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-09-23T19:22:16.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:15 smithi016 bash[75004]: audit 2023-09-23T19:22:14.448880+0000 mon.a (mon.0) 523 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-09-23T19:22:16.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:15 smithi016 bash[75004]: cluster 2023-09-23T19:22:15.093637+0000 osd.3 (osd.3) 18 : cluster 3 failed to encode map e91 with expected crc 2023-09-23T19:22:16.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:15 smithi016 bash[75004]: audit 2023-09-23T19:22:15.095739+0000 mon.a (mon.0) 524 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-09-23T19:22:16.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:15 smithi016 bash[75004]: cluster 2023-09-23T19:22:15.095838+0000 mon.a (mon.0) 525 : cluster 0 osdmap e91: 8 total, 8 up, 8 in 2023-09-23T19:22:16.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:15 smithi016 bash[75004]: cluster 2023-09-23T19:22:15.708513+0000 mgr.y (mgr.25325) 388 : cluster 0 pgmap v305: 89 pgs: 4 creating+peering, 12 unknown, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:22:16.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:15 smithi092 bash[58855]: audit 2023-09-23T19:22:14.448397+0000 mon.c (mon.2) 54 : audit 1 from='client.? 172.21.15.16:0/3017743629' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-09-23T19:22:16.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:15 smithi092 bash[58855]: audit 2023-09-23T19:22:14.448880+0000 mon.a (mon.0) 523 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-09-23T19:22:16.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:15 smithi092 bash[58855]: cluster 2023-09-23T19:22:15.093637+0000 osd.3 (osd.3) 18 : cluster 3 failed to encode map e91 with expected crc 2023-09-23T19:22:16.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:15 smithi092 bash[58855]: audit 2023-09-23T19:22:15.095739+0000 mon.a (mon.0) 524 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-09-23T19:22:16.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:15 smithi092 bash[58855]: cluster 2023-09-23T19:22:15.095838+0000 mon.a (mon.0) 525 : cluster 0 osdmap e91: 8 total, 8 up, 8 in 2023-09-23T19:22:16.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:15 smithi092 bash[58855]: cluster 2023-09-23T19:22:15.708513+0000 mgr.y (mgr.25325) 388 : cluster 0 pgmap v305: 89 pgs: 4 creating+peering, 12 unknown, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:22:16.397 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_3 rados --yes-i-really-mean-it || true 2023-09-23T19:22:17.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:17 smithi016 bash[75004]: cluster 2023-09-23T19:22:16.106016+0000 osd.6 (osd.6) 13 : cluster 3 failed to encode map e92 with expected crc 2023-09-23T19:22:17.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:17 smithi016 bash[75004]: cluster 2023-09-23T19:22:16.106104+0000 mon.a (mon.0) 526 : cluster 0 osdmap e92: 8 total, 8 up, 8 in 2023-09-23T19:22:17.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:17 smithi016 bash[75004]: cluster 2023-09-23T19:22:16.106578+0000 osd.3 (osd.3) 19 : cluster 3 failed to encode map e92 with expected crc 2023-09-23T19:22:17.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:17 smithi016 bash[75004]: cluster 2023-09-23T19:22:16.106595+0000 osd.1 (osd.1) 18 : cluster 3 failed to encode map e92 with expected crc 2023-09-23T19:22:17.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:17 smithi016 bash[75004]: cluster 2023-09-23T19:22:16.107322+0000 osd.6 (osd.6) 14 : cluster 3 failed to encode map e92 with expected crc 2023-09-23T19:22:17.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:17 smithi016 bash[75004]: cluster 2023-09-23T19:22:16.109702+0000 osd.4 (osd.4) 13 : cluster 3 failed to encode map e92 with expected crc 2023-09-23T19:22:17.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:17 smithi016 bash[75004]: cluster 2023-09-23T19:22:16.109708+0000 osd.5 (osd.5) 11 : cluster 3 failed to encode map e92 with expected crc 2023-09-23T19:22:17.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:17 smithi016 bash[75004]: audit 2023-09-23T19:22:16.684861+0000 mon.c (mon.2) 55 : audit 0 from='client.? 172.21.15.16:0/3153059375' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:17.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:17 smithi016 bash[77995]: cluster 2023-09-23T19:22:16.106016+0000 osd.6 (osd.6) 13 : cluster 3 failed to encode map e92 with expected crc 2023-09-23T19:22:17.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:17 smithi016 bash[77995]: cluster 2023-09-23T19:22:16.106104+0000 mon.a (mon.0) 526 : cluster 0 osdmap e92: 8 total, 8 up, 8 in 2023-09-23T19:22:17.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:17 smithi016 bash[77995]: cluster 2023-09-23T19:22:16.106578+0000 osd.3 (osd.3) 19 : cluster 3 failed to encode map e92 with expected crc 2023-09-23T19:22:17.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:17 smithi016 bash[77995]: cluster 2023-09-23T19:22:16.106595+0000 osd.1 (osd.1) 18 : cluster 3 failed to encode map e92 with expected crc 2023-09-23T19:22:17.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:17 smithi016 bash[77995]: cluster 2023-09-23T19:22:16.107322+0000 osd.6 (osd.6) 14 : cluster 3 failed to encode map e92 with expected crc 2023-09-23T19:22:17.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:17 smithi016 bash[77995]: cluster 2023-09-23T19:22:16.109702+0000 osd.4 (osd.4) 13 : cluster 3 failed to encode map e92 with expected crc 2023-09-23T19:22:17.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:17 smithi016 bash[77995]: cluster 2023-09-23T19:22:16.109708+0000 osd.5 (osd.5) 11 : cluster 3 failed to encode map e92 with expected crc 2023-09-23T19:22:17.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:17 smithi016 bash[77995]: audit 2023-09-23T19:22:16.684861+0000 mon.c (mon.2) 55 : audit 0 from='client.? 172.21.15.16:0/3153059375' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:17.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:17 smithi092 bash[58855]: cluster 2023-09-23T19:22:16.106016+0000 osd.6 (osd.6) 13 : cluster 3 failed to encode map e92 with expected crc 2023-09-23T19:22:17.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:17 smithi092 bash[58855]: cluster 2023-09-23T19:22:16.106104+0000 mon.a (mon.0) 526 : cluster 0 osdmap e92: 8 total, 8 up, 8 in 2023-09-23T19:22:17.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:17 smithi092 bash[58855]: cluster 2023-09-23T19:22:16.106578+0000 osd.3 (osd.3) 19 : cluster 3 failed to encode map e92 with expected crc 2023-09-23T19:22:17.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:17 smithi092 bash[58855]: cluster 2023-09-23T19:22:16.106595+0000 osd.1 (osd.1) 18 : cluster 3 failed to encode map e92 with expected crc 2023-09-23T19:22:17.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:17 smithi092 bash[58855]: cluster 2023-09-23T19:22:16.107322+0000 osd.6 (osd.6) 14 : cluster 3 failed to encode map e92 with expected crc 2023-09-23T19:22:17.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:17 smithi092 bash[58855]: cluster 2023-09-23T19:22:16.109702+0000 osd.4 (osd.4) 13 : cluster 3 failed to encode map e92 with expected crc 2023-09-23T19:22:17.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:17 smithi092 bash[58855]: cluster 2023-09-23T19:22:16.109708+0000 osd.5 (osd.5) 11 : cluster 3 failed to encode map e92 with expected crc 2023-09-23T19:22:17.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:17 smithi092 bash[58855]: audit 2023-09-23T19:22:16.684861+0000 mon.c (mon.2) 55 : audit 0 from='client.? 172.21.15.16:0/3153059375' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:18.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:18 smithi016 bash[75004]: cluster 2023-09-23T19:22:16.828433+0000 osd.2 (osd.2) 17 : cluster 3 failed to encode map e92 with expected crc 2023-09-23T19:22:18.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:18 smithi016 bash[75004]: cluster 2023-09-23T19:22:16.926827+0000 osd.7 (osd.7) 15 : cluster 3 failed to encode map e92 with expected crc 2023-09-23T19:22:18.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:18 smithi016 bash[75004]: cluster 2023-09-23T19:22:17.107165+0000 mon.a (mon.0) 527 : cluster 0 osdmap e93: 8 total, 8 up, 8 in 2023-09-23T19:22:18.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:18 smithi016 bash[75004]: cluster 2023-09-23T19:22:17.108539+0000 osd.4 (osd.4) 14 : cluster 3 failed to encode map e93 with expected crc 2023-09-23T19:22:18.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:18 smithi016 bash[75004]: cluster 2023-09-23T19:22:17.709169+0000 mgr.y (mgr.25325) 389 : cluster 0 pgmap v308: 89 pgs: 1 creating+activating, 4 creating+peering, 9 unknown, 75 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:22:18.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:18 smithi016 bash[75004]: cluster 2023-09-23T19:22:17.776980+0000 osd.3 (osd.3) 20 : cluster 3 failed to encode map e93 with expected crc 2023-09-23T19:22:18.284 INFO:journalctl@ceph.osd.1.smithi016.stdout:Sep 23 19:22:18 smithi016 systemd[1]: Stopping Ceph osd.1 for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7... 2023-09-23T19:22:18.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:18 smithi016 bash[77995]: cluster 2023-09-23T19:22:16.828433+0000 osd.2 (osd.2) 17 : cluster 3 failed to encode map e92 with expected crc 2023-09-23T19:22:18.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:18 smithi016 bash[77995]: cluster 2023-09-23T19:22:16.926827+0000 osd.7 (osd.7) 15 : cluster 3 failed to encode map e92 with expected crc 2023-09-23T19:22:18.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:18 smithi016 bash[77995]: cluster 2023-09-23T19:22:17.107165+0000 mon.a (mon.0) 527 : cluster 0 osdmap e93: 8 total, 8 up, 8 in 2023-09-23T19:22:18.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:18 smithi016 bash[77995]: cluster 2023-09-23T19:22:17.108539+0000 osd.4 (osd.4) 14 : cluster 3 failed to encode map e93 with expected crc 2023-09-23T19:22:18.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:18 smithi016 bash[77995]: cluster 2023-09-23T19:22:17.709169+0000 mgr.y (mgr.25325) 389 : cluster 0 pgmap v308: 89 pgs: 1 creating+activating, 4 creating+peering, 9 unknown, 75 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:22:18.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:18 smithi016 bash[77995]: cluster 2023-09-23T19:22:17.776980+0000 osd.3 (osd.3) 20 : cluster 3 failed to encode map e93 with expected crc 2023-09-23T19:22:18.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:18 smithi092 bash[58855]: cluster 2023-09-23T19:22:16.828433+0000 osd.2 (osd.2) 17 : cluster 3 failed to encode map e92 with expected crc 2023-09-23T19:22:18.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:18 smithi092 bash[58855]: cluster 2023-09-23T19:22:16.926827+0000 osd.7 (osd.7) 15 : cluster 3 failed to encode map e92 with expected crc 2023-09-23T19:22:18.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:18 smithi092 bash[58855]: cluster 2023-09-23T19:22:17.107165+0000 mon.a (mon.0) 527 : cluster 0 osdmap e93: 8 total, 8 up, 8 in 2023-09-23T19:22:18.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:18 smithi092 bash[58855]: cluster 2023-09-23T19:22:17.108539+0000 osd.4 (osd.4) 14 : cluster 3 failed to encode map e93 with expected crc 2023-09-23T19:22:18.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:18 smithi092 bash[58855]: cluster 2023-09-23T19:22:17.709169+0000 mgr.y (mgr.25325) 389 : cluster 0 pgmap v308: 89 pgs: 1 creating+activating, 4 creating+peering, 9 unknown, 75 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:22:18.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:18 smithi092 bash[58855]: cluster 2023-09-23T19:22:17.776980+0000 osd.3 (osd.3) 20 : cluster 3 failed to encode map e93 with expected crc 2023-09-23T19:22:18.784 INFO:journalctl@ceph.osd.1.smithi016.stdout:Sep 23 19:22:18 smithi016 bash[40902]: debug 2023-09-23T19:22:18.305+0000 7f55060df700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-09-23T19:22:18.784 INFO:journalctl@ceph.osd.1.smithi016.stdout:Sep 23 19:22:18 smithi016 bash[40902]: debug 2023-09-23T19:22:18.305+0000 7f55060df700 -1 osd.1 93 *** Got signal Terminated *** 2023-09-23T19:22:18.784 INFO:journalctl@ceph.osd.1.smithi016.stdout:Sep 23 19:22:18 smithi016 bash[40902]: debug 2023-09-23T19:22:18.305+0000 7f55060df700 -1 osd.1 93 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T19:22:19.510 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:19 smithi016 bash[75004]: cluster 2023-09-23T19:22:18.311337+0000 mon.a (mon.0) 528 : cluster 1 osd.1 marked itself down and dead 2023-09-23T19:22:19.510 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:19 smithi016 bash[75004]: audit 2023-09-23T19:22:19.060318+0000 mon.c (mon.2) 56 : audit 0 from='client.? 172.21.15.16:0/4276040696' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:19.510 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:19 smithi016 bash[77995]: cluster 2023-09-23T19:22:18.311337+0000 mon.a (mon.0) 528 : cluster 1 osd.1 marked itself down and dead 2023-09-23T19:22:19.511 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:19 smithi016 bash[77995]: audit 2023-09-23T19:22:19.060318+0000 mon.c (mon.2) 56 : audit 0 from='client.? 172.21.15.16:0/4276040696' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:19.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:19 smithi092 bash[58855]: cluster 2023-09-23T19:22:18.311337+0000 mon.a (mon.0) 528 : cluster 1 osd.1 marked itself down and dead 2023-09-23T19:22:19.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:19 smithi092 bash[58855]: audit 2023-09-23T19:22:19.060318+0000 mon.c (mon.2) 56 : audit 0 from='client.? 172.21.15.16:0/4276040696' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:20.175 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:20 smithi016 bash[77995]: cluster 2023-09-23T19:22:19.122413+0000 mon.a (mon.0) 529 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T19:22:20.175 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:20 smithi016 bash[77995]: cluster 2023-09-23T19:22:19.130531+0000 osd.2 (osd.2) 18 : cluster 3 failed to encode map e94 with expected crc 2023-09-23T19:22:20.175 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:20 smithi016 bash[77995]: cluster 2023-09-23T19:22:19.132880+0000 mon.a (mon.0) 530 : cluster 0 osdmap e94: 8 total, 7 up, 8 in 2023-09-23T19:22:20.175 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:20 smithi016 bash[77995]: cluster 2023-09-23T19:22:19.709764+0000 mgr.y (mgr.25325) 390 : cluster 0 pgmap v310: 89 pgs: 9 stale+active+clean, 1 creating+activating, 6 unknown, 4 stale+creating+peering, 69 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:22:20.176 INFO:journalctl@ceph.osd.1.smithi016.stdout:Sep 23 19:22:19 smithi016 bash[90518]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd-1 2023-09-23T19:22:20.176 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:20 smithi016 bash[75004]: cluster 2023-09-23T19:22:19.122413+0000 mon.a (mon.0) 529 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T19:22:20.176 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:20 smithi016 bash[75004]: cluster 2023-09-23T19:22:19.130531+0000 osd.2 (osd.2) 18 : cluster 3 failed to encode map e94 with expected crc 2023-09-23T19:22:20.176 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:20 smithi016 bash[75004]: cluster 2023-09-23T19:22:19.132880+0000 mon.a (mon.0) 530 : cluster 0 osdmap e94: 8 total, 7 up, 8 in 2023-09-23T19:22:20.176 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:20 smithi016 bash[75004]: cluster 2023-09-23T19:22:19.709764+0000 mgr.y (mgr.25325) 390 : cluster 0 pgmap v310: 89 pgs: 9 stale+active+clean, 1 creating+activating, 6 unknown, 4 stale+creating+peering, 69 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:22:20.533 INFO:journalctl@ceph.osd.1.smithi016.stdout:Sep 23 19:22:20 smithi016 bash[90626]: Error response from daemon: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd-1 2023-09-23T19:22:20.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:20 smithi092 bash[58855]: cluster 2023-09-23T19:22:19.122413+0000 mon.a (mon.0) 529 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T19:22:20.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:20 smithi092 bash[58855]: cluster 2023-09-23T19:22:19.130531+0000 osd.2 (osd.2) 18 : cluster 3 failed to encode map e94 with expected crc 2023-09-23T19:22:20.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:20 smithi092 bash[58855]: cluster 2023-09-23T19:22:19.132880+0000 mon.a (mon.0) 530 : cluster 0 osdmap e94: 8 total, 7 up, 8 in 2023-09-23T19:22:20.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:20 smithi092 bash[58855]: cluster 2023-09-23T19:22:19.709764+0000 mgr.y (mgr.25325) 390 : cluster 0 pgmap v310: 89 pgs: 9 stale+active+clean, 1 creating+activating, 6 unknown, 4 stale+creating+peering, 69 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:22:21.289 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:22:21.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:21 smithi016 bash[77995]: cluster 2023-09-23T19:22:20.142519+0000 mon.a (mon.0) 531 : cluster 0 osdmap e95: 8 total, 7 up, 8 in 2023-09-23T19:22:21.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:21 smithi016 bash[77995]: cluster 2023-09-23T19:22:20.142887+0000 osd.2 (osd.2) 19 : cluster 3 failed to encode map e95 with expected crc 2023-09-23T19:22:21.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:21 smithi016 bash[77995]: cluster 2023-09-23T19:22:20.143179+0000 osd.3 (osd.3) 21 : cluster 3 failed to encode map e95 with expected crc 2023-09-23T19:22:21.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:21 smithi016 bash[77995]: cluster 2023-09-23T19:22:20.143395+0000 osd.4 (osd.4) 15 : cluster 3 failed to encode map e95 with expected crc 2023-09-23T19:22:21.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:21 smithi016 bash[77995]: cluster 2023-09-23T19:22:20.146082+0000 osd.6 (osd.6) 15 : cluster 3 failed to encode map e95 with expected crc 2023-09-23T19:22:21.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:21 smithi016 bash[77995]: cluster 2023-09-23T19:22:20.146124+0000 osd.4 (osd.4) 16 : cluster 3 failed to encode map e95 with expected crc 2023-09-23T19:22:21.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:21 smithi016 bash[77995]: cluster 2023-09-23T19:22:20.146402+0000 osd.7 (osd.7) 16 : cluster 3 failed to encode map e95 with expected crc 2023-09-23T19:22:21.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:21 smithi016 bash[77995]: cluster 2023-09-23T19:22:20.146406+0000 osd.5 (osd.5) 12 : cluster 3 failed to encode map e95 with expected crc 2023-09-23T19:22:21.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:21 smithi016 bash[77995]: cluster 2023-09-23T19:22:20.146650+0000 osd.3 (osd.3) 22 : cluster 3 failed to encode map e95 with expected crc 2023-09-23T19:22:21.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:21 smithi016 bash[77995]: cluster 2023-09-23T19:22:20.151327+0000 osd.7 (osd.7) 17 : cluster 3 failed to encode map e95 with expected crc 2023-09-23T19:22:21.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:21 smithi016 bash[77995]: cluster 2023-09-23T19:22:20.151607+0000 osd.5 (osd.5) 13 : cluster 3 failed to encode map e95 with expected crc 2023-09-23T19:22:21.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:21 smithi016 bash[75004]: cluster 2023-09-23T19:22:20.142519+0000 mon.a (mon.0) 531 : cluster 0 osdmap e95: 8 total, 7 up, 8 in 2023-09-23T19:22:21.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:21 smithi016 bash[75004]: cluster 2023-09-23T19:22:20.142887+0000 osd.2 (osd.2) 19 : cluster 3 failed to encode map e95 with expected crc 2023-09-23T19:22:21.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:21 smithi016 bash[75004]: cluster 2023-09-23T19:22:20.143179+0000 osd.3 (osd.3) 21 : cluster 3 failed to encode map e95 with expected crc 2023-09-23T19:22:21.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:21 smithi016 bash[75004]: cluster 2023-09-23T19:22:20.143395+0000 osd.4 (osd.4) 15 : cluster 3 failed to encode map e95 with expected crc 2023-09-23T19:22:21.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:21 smithi016 bash[75004]: cluster 2023-09-23T19:22:20.146082+0000 osd.6 (osd.6) 15 : cluster 3 failed to encode map e95 with expected crc 2023-09-23T19:22:21.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:21 smithi016 bash[75004]: cluster 2023-09-23T19:22:20.146124+0000 osd.4 (osd.4) 16 : cluster 3 failed to encode map e95 with expected crc 2023-09-23T19:22:21.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:21 smithi016 bash[75004]: cluster 2023-09-23T19:22:20.146402+0000 osd.7 (osd.7) 16 : cluster 3 failed to encode map e95 with expected crc 2023-09-23T19:22:21.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:21 smithi016 bash[75004]: cluster 2023-09-23T19:22:20.146406+0000 osd.5 (osd.5) 12 : cluster 3 failed to encode map e95 with expected crc 2023-09-23T19:22:21.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:21 smithi016 bash[75004]: cluster 2023-09-23T19:22:20.146650+0000 osd.3 (osd.3) 22 : cluster 3 failed to encode map e95 with expected crc 2023-09-23T19:22:21.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:21 smithi016 bash[75004]: cluster 2023-09-23T19:22:20.151327+0000 osd.7 (osd.7) 17 : cluster 3 failed to encode map e95 with expected crc 2023-09-23T19:22:21.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:21 smithi016 bash[75004]: cluster 2023-09-23T19:22:20.151607+0000 osd.5 (osd.5) 13 : cluster 3 failed to encode map e95 with expected crc 2023-09-23T19:22:21.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:21 smithi092 bash[58855]: cluster 2023-09-23T19:22:20.142519+0000 mon.a (mon.0) 531 : cluster 0 osdmap e95: 8 total, 7 up, 8 in 2023-09-23T19:22:21.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:21 smithi092 bash[58855]: cluster 2023-09-23T19:22:20.142887+0000 osd.2 (osd.2) 19 : cluster 3 failed to encode map e95 with expected crc 2023-09-23T19:22:21.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:21 smithi092 bash[58855]: cluster 2023-09-23T19:22:20.143179+0000 osd.3 (osd.3) 21 : cluster 3 failed to encode map e95 with expected crc 2023-09-23T19:22:21.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:21 smithi092 bash[58855]: cluster 2023-09-23T19:22:20.143395+0000 osd.4 (osd.4) 15 : cluster 3 failed to encode map e95 with expected crc 2023-09-23T19:22:21.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:21 smithi092 bash[58855]: cluster 2023-09-23T19:22:20.146082+0000 osd.6 (osd.6) 15 : cluster 3 failed to encode map e95 with expected crc 2023-09-23T19:22:21.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:21 smithi092 bash[58855]: cluster 2023-09-23T19:22:20.146124+0000 osd.4 (osd.4) 16 : cluster 3 failed to encode map e95 with expected crc 2023-09-23T19:22:21.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:21 smithi092 bash[58855]: cluster 2023-09-23T19:22:20.146402+0000 osd.7 (osd.7) 16 : cluster 3 failed to encode map e95 with expected crc 2023-09-23T19:22:21.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:21 smithi092 bash[58855]: cluster 2023-09-23T19:22:20.146406+0000 osd.5 (osd.5) 12 : cluster 3 failed to encode map e95 with expected crc 2023-09-23T19:22:21.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:21 smithi092 bash[58855]: cluster 2023-09-23T19:22:20.146650+0000 osd.3 (osd.3) 22 : cluster 3 failed to encode map e95 with expected crc 2023-09-23T19:22:21.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:21 smithi092 bash[58855]: cluster 2023-09-23T19:22:20.151327+0000 osd.7 (osd.7) 17 : cluster 3 failed to encode map e95 with expected crc 2023-09-23T19:22:21.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:21 smithi092 bash[58855]: cluster 2023-09-23T19:22:20.151607+0000 osd.5 (osd.5) 13 : cluster 3 failed to encode map e95 with expected crc 2023-09-23T19:22:22.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:22 smithi016 bash[77995]: audit 2023-09-23T19:22:21.445856+0000 mon.c (mon.2) 57 : audit 0 from='client.? 172.21.15.16:0/2353631046' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:22.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:22 smithi016 bash[77995]: cluster 2023-09-23T19:22:21.710543+0000 mgr.y (mgr.25325) 391 : cluster 0 pgmap v312: 89 pgs: 17 peering, 1 stale+active+clean, 17 active+undersized, 1 stale+creating+peering, 53 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:22:22.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:22 smithi016 bash[75004]: audit 2023-09-23T19:22:21.445856+0000 mon.c (mon.2) 57 : audit 0 from='client.? 172.21.15.16:0/2353631046' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:22.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:22 smithi016 bash[75004]: cluster 2023-09-23T19:22:21.710543+0000 mgr.y (mgr.25325) 391 : cluster 0 pgmap v312: 89 pgs: 17 peering, 1 stale+active+clean, 17 active+undersized, 1 stale+creating+peering, 53 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:22:22.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:22 smithi092 bash[58855]: audit 2023-09-23T19:22:21.445856+0000 mon.c (mon.2) 57 : audit 0 from='client.? 172.21.15.16:0/2353631046' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:22.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:22 smithi092 bash[58855]: cluster 2023-09-23T19:22:21.710543+0000 mgr.y (mgr.25325) 391 : cluster 0 pgmap v312: 89 pgs: 17 peering, 1 stale+active+clean, 17 active+undersized, 1 stale+creating+peering, 53 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:22:23.193 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:23 smithi016 bash[77995]: cluster 2023-09-23T19:22:22.141466+0000 mon.a (mon.0) 532 : cluster 3 Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T19:22:23.193 INFO:journalctl@ceph.osd.1.smithi016.stdout:Sep 23 19:22:22 smithi016 systemd[1]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@osd.1.service: Succeeded. 2023-09-23T19:22:23.193 INFO:journalctl@ceph.osd.1.smithi016.stdout:Sep 23 19:22:22 smithi016 systemd[1]: Stopped Ceph osd.1 for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7. 2023-09-23T19:22:23.194 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:23 smithi016 bash[75004]: cluster 2023-09-23T19:22:22.141466+0000 mon.a (mon.0) 532 : cluster 3 Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T19:22:23.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:23 smithi092 bash[58855]: cluster 2023-09-23T19:22:22.141466+0000 mon.a (mon.0) 532 : cluster 3 Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T19:22:23.588 INFO:journalctl@ceph.osd.1.smithi016.stdout:Sep 23 19:22:23 smithi016 systemd[1]: Started Ceph osd.1 for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7. 2023-09-23T19:22:24.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:24 smithi016 bash[77995]: audit 2023-09-23T19:22:23.236038+0000 mon.a (mon.0) 533 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:22:24.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:24 smithi016 bash[77995]: audit 2023-09-23T19:22:23.244163+0000 mon.a (mon.0) 534 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:22:24.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:24 smithi016 bash[77995]: audit 2023-09-23T19:22:23.273401+0000 mon.a (mon.0) 535 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:22:24.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:24 smithi016 bash[77995]: cluster 2023-09-23T19:22:23.711305+0000 mgr.y (mgr.25325) 392 : cluster 0 pgmap v313: 89 pgs: 1 active+undersized+degraded, 17 peering, 21 active+undersized, 50 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2023-09-23T19:22:24.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:24 smithi016 bash[77995]: audit 2023-09-23T19:22:23.791285+0000 mon.a (mon.0) 536 : audit 0 from='client.? 172.21.15.16:0/870854100' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:24.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:24 smithi016 bash[75004]: audit 2023-09-23T19:22:23.236038+0000 mon.a (mon.0) 533 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:22:24.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:24 smithi016 bash[75004]: audit 2023-09-23T19:22:23.244163+0000 mon.a (mon.0) 534 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:22:24.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:24 smithi016 bash[75004]: audit 2023-09-23T19:22:23.273401+0000 mon.a (mon.0) 535 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:22:24.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:24 smithi016 bash[75004]: cluster 2023-09-23T19:22:23.711305+0000 mgr.y (mgr.25325) 392 : cluster 0 pgmap v313: 89 pgs: 1 active+undersized+degraded, 17 peering, 21 active+undersized, 50 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2023-09-23T19:22:24.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:24 smithi016 bash[75004]: audit 2023-09-23T19:22:23.791285+0000 mon.a (mon.0) 536 : audit 0 from='client.? 172.21.15.16:0/870854100' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:24.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:24 smithi092 bash[58855]: audit 2023-09-23T19:22:23.236038+0000 mon.a (mon.0) 533 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:22:24.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:24 smithi092 bash[58855]: audit 2023-09-23T19:22:23.244163+0000 mon.a (mon.0) 534 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:22:24.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:24 smithi092 bash[58855]: audit 2023-09-23T19:22:23.273401+0000 mon.a (mon.0) 535 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:22:24.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:24 smithi092 bash[58855]: cluster 2023-09-23T19:22:23.711305+0000 mgr.y (mgr.25325) 392 : cluster 0 pgmap v313: 89 pgs: 1 active+undersized+degraded, 17 peering, 21 active+undersized, 50 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2023-09-23T19:22:24.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:24 smithi092 bash[58855]: audit 2023-09-23T19:22:23.791285+0000 mon.a (mon.0) 536 : audit 0 from='client.? 172.21.15.16:0/870854100' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:25.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:25 smithi016 bash[75004]: cluster 2023-09-23T19:22:24.241747+0000 mon.a (mon.0) 537 : cluster 3 Health check failed: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded (PG_DEGRADED) 2023-09-23T19:22:25.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:25 smithi016 bash[77995]: cluster 2023-09-23T19:22:24.241747+0000 mon.a (mon.0) 537 : cluster 3 Health check failed: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded (PG_DEGRADED) 2023-09-23T19:22:25.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:25 smithi092 bash[58855]: cluster 2023-09-23T19:22:24.241747+0000 mon.a (mon.0) 537 : cluster 3 Health check failed: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded (PG_DEGRADED) 2023-09-23T19:22:25.966 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:22:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:22:25] "GET /metrics HTTP/1.1" 200 33829 "" "Prometheus/2.33.4" 2023-09-23T19:22:26.248 INFO:journalctl@ceph.osd.1.smithi016.stdout:Sep 23 19:22:25 smithi016 bash[90891]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T19:22:26.248 INFO:journalctl@ceph.osd.1.smithi016.stdout:Sep 23 19:22:25 smithi016 bash[90891]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-09-23T19:22:26.248 INFO:journalctl@ceph.osd.1.smithi016.stdout:Sep 23 19:22:25 smithi016 bash[90891]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2023-09-23T19:22:26.248 INFO:journalctl@ceph.osd.1.smithi016.stdout:Sep 23 19:22:25 smithi016 bash[90891]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2023-09-23T19:22:26.248 INFO:journalctl@ceph.osd.1.smithi016.stdout:Sep 23 19:22:25 smithi016 bash[90891]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2023-09-23T19:22:26.249 INFO:journalctl@ceph.osd.1.smithi016.stdout:Sep 23 19:22:25 smithi016 bash[90891]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2023-09-23T19:22:26.249 INFO:journalctl@ceph.osd.1.smithi016.stdout:Sep 23 19:22:25 smithi016 bash[90891]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-09-23T19:22:26.249 INFO:journalctl@ceph.osd.1.smithi016.stdout:Sep 23 19:22:25 smithi016 bash[90891]: --> ceph-volume lvm activate successful for osd ID: 1 2023-09-23T19:22:26.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:26 smithi016 bash[75004]: cluster 2023-09-23T19:22:25.712342+0000 mgr.y (mgr.25325) 393 : cluster 0 pgmap v314: 89 pgs: 3 active+undersized+degraded, 36 active+undersized, 50 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-09-23T19:22:26.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:26 smithi016 bash[75004]: audit 2023-09-23T19:22:26.157159+0000 mon.c (mon.2) 58 : audit 0 from='client.? 172.21.15.16:0/2647374753' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:26.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:26 smithi016 bash[77995]: cluster 2023-09-23T19:22:25.712342+0000 mgr.y (mgr.25325) 393 : cluster 0 pgmap v314: 89 pgs: 3 active+undersized+degraded, 36 active+undersized, 50 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-09-23T19:22:26.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:26 smithi016 bash[77995]: audit 2023-09-23T19:22:26.157159+0000 mon.c (mon.2) 58 : audit 0 from='client.? 172.21.15.16:0/2647374753' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:26.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:26 smithi092 bash[58855]: cluster 2023-09-23T19:22:25.712342+0000 mgr.y (mgr.25325) 393 : cluster 0 pgmap v314: 89 pgs: 3 active+undersized+degraded, 36 active+undersized, 50 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-09-23T19:22:26.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:26 smithi092 bash[58855]: audit 2023-09-23T19:22:26.157159+0000 mon.c (mon.2) 58 : audit 0 from='client.? 172.21.15.16:0/2647374753' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:27.266 INFO:teuthology.orchestra.run.smithi016.stderr:enabled application 'rados' on pool 'unique_pool_3' 2023-09-23T19:22:27.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:27 smithi016 bash[75004]: cluster 2023-09-23T19:22:26.237500+0000 mon.a (mon.0) 538 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-09-23T19:22:27.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:27 smithi016 bash[75004]: audit 2023-09-23T19:22:26.246368+0000 mon.c (mon.2) 59 : audit 1 from='client.? 172.21.15.16:0/3326671807' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:22:27.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:27 smithi016 bash[75004]: audit 2023-09-23T19:22:26.251223+0000 mon.a (mon.0) 539 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:22:27.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:27 smithi016 bash[77995]: cluster 2023-09-23T19:22:26.237500+0000 mon.a (mon.0) 538 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-09-23T19:22:27.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:27 smithi016 bash[77995]: audit 2023-09-23T19:22:26.246368+0000 mon.c (mon.2) 59 : audit 1 from='client.? 172.21.15.16:0/3326671807' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:22:27.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:27 smithi016 bash[77995]: audit 2023-09-23T19:22:26.251223+0000 mon.a (mon.0) 539 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:22:27.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:27 smithi092 bash[58855]: cluster 2023-09-23T19:22:26.237500+0000 mon.a (mon.0) 538 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-09-23T19:22:27.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:27 smithi092 bash[58855]: audit 2023-09-23T19:22:26.246368+0000 mon.c (mon.2) 59 : audit 1 from='client.? 172.21.15.16:0/3326671807' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:22:27.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:27 smithi092 bash[58855]: audit 2023-09-23T19:22:26.251223+0000 mon.a (mon.0) 539 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:22:28.520 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:28 smithi016 bash[75004]: audit 2023-09-23T19:22:27.260211+0000 mon.a (mon.0) 540 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T19:22:28.520 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:28 smithi016 bash[75004]: cluster 2023-09-23T19:22:27.260348+0000 mon.a (mon.0) 541 : cluster 0 osdmap e96: 8 total, 7 up, 8 in 2023-09-23T19:22:28.520 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:28 smithi016 bash[75004]: cluster 2023-09-23T19:22:27.713059+0000 mgr.y (mgr.25325) 394 : cluster 0 pgmap v316: 89 pgs: 3 active+undersized+degraded, 36 active+undersized, 50 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-09-23T19:22:28.521 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:28 smithi016 bash[77995]: audit 2023-09-23T19:22:27.260211+0000 mon.a (mon.0) 540 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T19:22:28.521 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:28 smithi016 bash[77995]: cluster 2023-09-23T19:22:27.260348+0000 mon.a (mon.0) 541 : cluster 0 osdmap e96: 8 total, 7 up, 8 in 2023-09-23T19:22:28.521 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:28 smithi016 bash[77995]: cluster 2023-09-23T19:22:27.713059+0000 mgr.y (mgr.25325) 394 : cluster 0 pgmap v316: 89 pgs: 3 active+undersized+degraded, 36 active+undersized, 50 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-09-23T19:22:28.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:28 smithi092 bash[58855]: audit 2023-09-23T19:22:27.260211+0000 mon.a (mon.0) 540 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T19:22:28.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:28 smithi092 bash[58855]: cluster 2023-09-23T19:22:27.260348+0000 mon.a (mon.0) 541 : cluster 0 osdmap e96: 8 total, 7 up, 8 in 2023-09-23T19:22:28.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:28 smithi092 bash[58855]: cluster 2023-09-23T19:22:27.713059+0000 mgr.y (mgr.25325) 394 : cluster 0 pgmap v316: 89 pgs: 3 active+undersized+degraded, 36 active+undersized, 50 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-09-23T19:22:28.784 INFO:journalctl@ceph.osd.1.smithi016.stdout:Sep 23 19:22:28 smithi016 bash[91347]: debug 2023-09-23T19:22:28.513+0000 7f8eea03b700 1 -- 172.21.15.16:0/3433057621 <== mon.0 v2:172.21.15.16:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5583c8e064e0 con 0x5583c8e60800 2023-09-23T19:22:29.462 DEBUG:teuthology.orchestra.run.smithi016:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_3 bench 90 write' 2023-09-23T19:22:29.464 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T19:22:29.465 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T19:22:29.511 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:hints = 1 2023-09-23T19:22:29.512 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T19:22:29.512 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Object prefix: benchmark_data_smithi016_91497 2023-09-23T19:22:29.512 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:22:29.512 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 0 0 0 0 0 0 - 0 2023-09-23T19:22:29.533 INFO:journalctl@ceph.osd.1.smithi016.stdout:Sep 23 19:22:29 smithi016 bash[91347]: debug 2023-09-23T19:22:29.053+0000 7f8ef31c0640 -1 Falling back to public interface 2023-09-23T19:22:29.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:29 smithi016 bash[77995]: cluster 2023-09-23T19:22:28.340656+0000 osd.4 (osd.4) 17 : cluster 3 failed to encode map e96 with expected crc 2023-09-23T19:22:29.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:29 smithi016 bash[77995]: audit 2023-09-23T19:22:28.535307+0000 mon.a (mon.0) 542 : audit 0 from='client.? 172.21.15.16:0/3228592232' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:29.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:29 smithi016 bash[77995]: audit 2023-09-23T19:22:28.629749+0000 mon.a (mon.0) 543 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:22:29.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:29 smithi016 bash[77995]: audit 2023-09-23T19:22:28.631489+0000 mon.a (mon.0) 544 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:22:29.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:29 smithi016 bash[77995]: cluster 2023-09-23T19:22:28.666364+0000 osd.7 (osd.7) 18 : cluster 3 failed to encode map e96 with expected crc 2023-09-23T19:22:29.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:29 smithi016 bash[77995]: cluster 2023-09-23T19:22:28.743583+0000 osd.5 (osd.5) 14 : cluster 3 failed to encode map e96 with expected crc 2023-09-23T19:22:29.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:29 smithi016 bash[75004]: cluster 2023-09-23T19:22:28.340656+0000 osd.4 (osd.4) 17 : cluster 3 failed to encode map e96 with expected crc 2023-09-23T19:22:29.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:29 smithi016 bash[75004]: audit 2023-09-23T19:22:28.535307+0000 mon.a (mon.0) 542 : audit 0 from='client.? 172.21.15.16:0/3228592232' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:29.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:29 smithi016 bash[75004]: audit 2023-09-23T19:22:28.629749+0000 mon.a (mon.0) 543 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:22:29.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:29 smithi016 bash[75004]: audit 2023-09-23T19:22:28.631489+0000 mon.a (mon.0) 544 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:22:29.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:29 smithi016 bash[75004]: cluster 2023-09-23T19:22:28.666364+0000 osd.7 (osd.7) 18 : cluster 3 failed to encode map e96 with expected crc 2023-09-23T19:22:29.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:29 smithi016 bash[75004]: cluster 2023-09-23T19:22:28.743583+0000 osd.5 (osd.5) 14 : cluster 3 failed to encode map e96 with expected crc 2023-09-23T19:22:29.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:29 smithi092 bash[58855]: cluster 2023-09-23T19:22:28.340656+0000 osd.4 (osd.4) 17 : cluster 3 failed to encode map e96 with expected crc 2023-09-23T19:22:29.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:29 smithi092 bash[58855]: audit 2023-09-23T19:22:28.535307+0000 mon.a (mon.0) 542 : audit 0 from='client.? 172.21.15.16:0/3228592232' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:29.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:29 smithi092 bash[58855]: audit 2023-09-23T19:22:28.629749+0000 mon.a (mon.0) 543 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:22:29.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:29 smithi092 bash[58855]: audit 2023-09-23T19:22:28.631489+0000 mon.a (mon.0) 544 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:22:29.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:29 smithi092 bash[58855]: cluster 2023-09-23T19:22:28.666364+0000 osd.7 (osd.7) 18 : cluster 3 failed to encode map e96 with expected crc 2023-09-23T19:22:29.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:29 smithi092 bash[58855]: cluster 2023-09-23T19:22:28.743583+0000 osd.5 (osd.5) 14 : cluster 3 failed to encode map e96 with expected crc 2023-09-23T19:22:30.512 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 1 16 3286 3270 204.364 204.375 0.00750385 0.00487447 2023-09-23T19:22:30.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:30 smithi016 bash[75004]: cluster 2023-09-23T19:22:28.898431+0000 osd.3 (osd.3) 23 : cluster 3 failed to encode map e96 with expected crc 2023-09-23T19:22:30.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:30 smithi016 bash[75004]: cluster 2023-09-23T19:22:28.931709+0000 osd.2 (osd.2) 20 : cluster 3 failed to encode map e96 with expected crc 2023-09-23T19:22:30.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:30 smithi016 bash[75004]: cluster 2023-09-23T19:22:29.132155+0000 osd.6 (osd.6) 16 : cluster 3 failed to encode map e96 with expected crc 2023-09-23T19:22:30.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:30 smithi016 bash[75004]: cluster 2023-09-23T19:22:29.713756+0000 mgr.y (mgr.25325) 395 : cluster 0 pgmap v317: 89 pgs: 3 active+undersized+degraded, 36 active+undersized, 50 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-09-23T19:22:30.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:30 smithi016 bash[77995]: cluster 2023-09-23T19:22:28.898431+0000 osd.3 (osd.3) 23 : cluster 3 failed to encode map e96 with expected crc 2023-09-23T19:22:30.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:30 smithi016 bash[77995]: cluster 2023-09-23T19:22:28.931709+0000 osd.2 (osd.2) 20 : cluster 3 failed to encode map e96 with expected crc 2023-09-23T19:22:30.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:30 smithi016 bash[77995]: cluster 2023-09-23T19:22:29.132155+0000 osd.6 (osd.6) 16 : cluster 3 failed to encode map e96 with expected crc 2023-09-23T19:22:30.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:30 smithi016 bash[77995]: cluster 2023-09-23T19:22:29.713756+0000 mgr.y (mgr.25325) 395 : cluster 0 pgmap v317: 89 pgs: 3 active+undersized+degraded, 36 active+undersized, 50 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-09-23T19:22:30.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:30 smithi092 bash[58855]: cluster 2023-09-23T19:22:28.898431+0000 osd.3 (osd.3) 23 : cluster 3 failed to encode map e96 with expected crc 2023-09-23T19:22:30.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:30 smithi092 bash[58855]: cluster 2023-09-23T19:22:28.931709+0000 osd.2 (osd.2) 20 : cluster 3 failed to encode map e96 with expected crc 2023-09-23T19:22:30.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:30 smithi092 bash[58855]: cluster 2023-09-23T19:22:29.132155+0000 osd.6 (osd.6) 16 : cluster 3 failed to encode map e96 with expected crc 2023-09-23T19:22:30.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:30 smithi092 bash[58855]: cluster 2023-09-23T19:22:29.713756+0000 mgr.y (mgr.25325) 395 : cluster 0 pgmap v317: 89 pgs: 3 active+undersized+degraded, 36 active+undersized, 50 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-09-23T19:22:31.513 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 2 16 6071 6055 189.206 174.062 0.00580917 0.00527609 2023-09-23T19:22:31.533 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:31 smithi016 bash[77995]: audit 2023-09-23T19:22:31.157927+0000 mon.c (mon.2) 60 : audit 0 from='client.? 172.21.15.16:0/2334949770' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:31.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:31 smithi016 bash[75004]: audit 2023-09-23T19:22:31.157927+0000 mon.c (mon.2) 60 : audit 0 from='client.? 172.21.15.16:0/2334949770' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:31.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:31 smithi092 bash[58855]: audit 2023-09-23T19:22:31.157927+0000 mon.c (mon.2) 60 : audit 0 from='client.? 172.21.15.16:0/2334949770' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:32.513 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 3 16 8475 8459 176.216 150.25 0.00707374 0.00566425 2023-09-23T19:22:32.533 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:32 smithi016 bash[77995]: cluster 2023-09-23T19:22:31.714504+0000 mgr.y (mgr.25325) 396 : cluster 0 pgmap v318: 89 pgs: 9 active+undersized+degraded, 30 active+undersized, 50 active+clean; 117 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s wr, 186 op/s; 1598/5679 objects degraded (28.139%) 2023-09-23T19:22:32.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:32 smithi016 bash[75004]: cluster 2023-09-23T19:22:31.714504+0000 mgr.y (mgr.25325) 396 : cluster 0 pgmap v318: 89 pgs: 9 active+undersized+degraded, 30 active+undersized, 50 active+clean; 117 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s wr, 186 op/s; 1598/5679 objects degraded (28.139%) 2023-09-23T19:22:32.790 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:32 smithi092 bash[58855]: cluster 2023-09-23T19:22:31.714504+0000 mgr.y (mgr.25325) 396 : cluster 0 pgmap v318: 89 pgs: 9 active+undersized+degraded, 30 active+undersized, 50 active+clean; 117 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s wr, 186 op/s; 1598/5679 objects degraded (28.139%) 2023-09-23T19:22:33.513 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 4 16 10832 10816 168.987 147.312 0.00216189 0.00590408 2023-09-23T19:22:33.783 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:33 smithi016 bash[77995]: cluster 2023-09-23T19:22:32.279011+0000 mon.a (mon.0) 545 : cluster 3 Health check update: Degraded data redundancy: 1598/5679 objects degraded (28.139%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T19:22:33.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:33 smithi016 bash[77995]: audit 2023-09-23T19:22:32.327737+0000 mon.a (mon.0) 546 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:22:33.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:33 smithi016 bash[77995]: audit 2023-09-23T19:22:32.337824+0000 mon.a (mon.0) 547 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:22:33.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:33 smithi016 bash[75004]: cluster 2023-09-23T19:22:32.279011+0000 mon.a (mon.0) 545 : cluster 3 Health check update: Degraded data redundancy: 1598/5679 objects degraded (28.139%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T19:22:33.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:33 smithi016 bash[75004]: audit 2023-09-23T19:22:32.327737+0000 mon.a (mon.0) 546 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:22:33.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:33 smithi016 bash[75004]: audit 2023-09-23T19:22:32.337824+0000 mon.a (mon.0) 547 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:22:33.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:33 smithi092 bash[58855]: cluster 2023-09-23T19:22:32.279011+0000 mon.a (mon.0) 545 : cluster 3 Health check update: Degraded data redundancy: 1598/5679 objects degraded (28.139%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T19:22:33.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:33 smithi092 bash[58855]: audit 2023-09-23T19:22:32.327737+0000 mon.a (mon.0) 546 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:22:33.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:33 smithi092 bash[58855]: audit 2023-09-23T19:22:32.337824+0000 mon.a (mon.0) 547 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:22:34.513 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 5 16 13197 13181 164.748 147.812 0.00322595 0.00605751 2023-09-23T19:22:35.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:34 smithi016 bash[75004]: audit 2023-09-23T19:22:33.592930+0000 mon.a (mon.0) 548 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:22:35.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:34 smithi016 bash[75004]: audit 2023-09-23T19:22:33.602843+0000 mon.a (mon.0) 549 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:22:35.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:34 smithi016 bash[75004]: cluster 2023-09-23T19:22:33.715214+0000 mgr.y (mgr.25325) 397 : cluster 0 pgmap v319: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 277 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s wr, 441 op/s; 2244/13335 objects degraded (16.828%) 2023-09-23T19:22:35.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:34 smithi016 bash[75004]: audit 2023-09-23T19:22:33.846763+0000 mon.a (mon.0) 550 : audit 0 from='client.? 172.21.15.16:0/2414771463' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:35.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:34 smithi016 bash[77995]: audit 2023-09-23T19:22:33.592930+0000 mon.a (mon.0) 548 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:22:35.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:34 smithi016 bash[77995]: audit 2023-09-23T19:22:33.602843+0000 mon.a (mon.0) 549 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:22:35.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:34 smithi016 bash[77995]: cluster 2023-09-23T19:22:33.715214+0000 mgr.y (mgr.25325) 397 : cluster 0 pgmap v319: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 277 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s wr, 441 op/s; 2244/13335 objects degraded (16.828%) 2023-09-23T19:22:35.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:34 smithi016 bash[77995]: audit 2023-09-23T19:22:33.846763+0000 mon.a (mon.0) 550 : audit 0 from='client.? 172.21.15.16:0/2414771463' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:35.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:34 smithi092 bash[58855]: audit 2023-09-23T19:22:33.592930+0000 mon.a (mon.0) 548 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:22:35.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:34 smithi092 bash[58855]: audit 2023-09-23T19:22:33.602843+0000 mon.a (mon.0) 549 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:22:35.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:34 smithi092 bash[58855]: cluster 2023-09-23T19:22:33.715214+0000 mgr.y (mgr.25325) 397 : cluster 0 pgmap v319: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 277 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s wr, 441 op/s; 2244/13335 objects degraded (16.828%) 2023-09-23T19:22:35.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:34 smithi092 bash[58855]: audit 2023-09-23T19:22:33.846763+0000 mon.a (mon.0) 550 : audit 0 from='client.? 172.21.15.16:0/2414771463' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:35.513 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 6 16 15729 15713 163.662 158.25 0.00953042 0.00610386 2023-09-23T19:22:36.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:22:35 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:22:35] "GET /metrics HTTP/1.1" 200 36118 "" "Prometheus/2.33.4" 2023-09-23T19:22:36.513 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 7 16 18003 17987 160.584 142.125 0.00675819 0.00621961 2023-09-23T19:22:37.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:36 smithi016 bash[75004]: cluster 2023-09-23T19:22:35.716291+0000 mgr.y (mgr.25325) 398 : cluster 0 pgmap v320: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 675 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s; 3771/32466 objects degraded (11.615%) 2023-09-23T19:22:37.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:36 smithi016 bash[75004]: audit 2023-09-23T19:22:36.452484+0000 mon.c (mon.2) 61 : audit 0 from='client.? 172.21.15.16:0/71299853' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:37.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:36 smithi016 bash[77995]: cluster 2023-09-23T19:22:35.716291+0000 mgr.y (mgr.25325) 398 : cluster 0 pgmap v320: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 675 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s; 3771/32466 objects degraded (11.615%) 2023-09-23T19:22:37.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:36 smithi016 bash[77995]: audit 2023-09-23T19:22:36.452484+0000 mon.c (mon.2) 61 : audit 0 from='client.? 172.21.15.16:0/71299853' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:37.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:36 smithi092 bash[58855]: cluster 2023-09-23T19:22:35.716291+0000 mgr.y (mgr.25325) 398 : cluster 0 pgmap v320: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 675 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s; 3771/32466 objects degraded (11.615%) 2023-09-23T19:22:37.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:36 smithi092 bash[58855]: audit 2023-09-23T19:22:36.452484+0000 mon.c (mon.2) 61 : audit 0 from='client.? 172.21.15.16:0/71299853' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:37.513 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 8 16 20245 20229 158.024 140.125 0.00772046 0.00632139 2023-09-23T19:22:38.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:37 smithi016 bash[75004]: cluster 2023-09-23T19:22:37.717112+0000 mgr.y (mgr.25325) 399 : cluster 0 pgmap v321: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 867 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s; 6837/41664 objects degraded (16.410%) 2023-09-23T19:22:38.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:37 smithi016 bash[75004]: cluster 2023-09-23T19:22:37.771289+0000 mon.a (mon.0) 551 : cluster 3 Health check update: Degraded data redundancy: 6837/41664 objects degraded (16.410%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T19:22:38.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:37 smithi016 bash[77995]: cluster 2023-09-23T19:22:37.717112+0000 mgr.y (mgr.25325) 399 : cluster 0 pgmap v321: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 867 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s; 6837/41664 objects degraded (16.410%) 2023-09-23T19:22:38.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:37 smithi016 bash[77995]: cluster 2023-09-23T19:22:37.771289+0000 mon.a (mon.0) 551 : cluster 3 Health check update: Degraded data redundancy: 6837/41664 objects degraded (16.410%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T19:22:38.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:37 smithi092 bash[58855]: cluster 2023-09-23T19:22:37.717112+0000 mgr.y (mgr.25325) 399 : cluster 0 pgmap v321: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 867 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s; 6837/41664 objects degraded (16.410%) 2023-09-23T19:22:38.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:37 smithi092 bash[58855]: cluster 2023-09-23T19:22:37.771289+0000 mon.a (mon.0) 551 : cluster 3 Health check update: Degraded data redundancy: 6837/41664 objects degraded (16.410%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T19:22:38.513 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 9 16 22457 22441 155.826 138.25 0.0102886 0.00640909 2023-09-23T19:22:39.514 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 10 16 24373 24357 152.217 119.75 0.00586051 0.00656287 2023-09-23T19:22:39.533 INFO:journalctl@ceph.osd.1.smithi016.stdout:Sep 23 19:22:39 smithi016 bash[91347]: debug 2023-09-23T19:22:39.245+0000 7f8ef31c0640 -1 osd.1 93 log_to_monitors true 2023-09-23T19:22:40.513 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 11 15 26538 26523 150.685 135.375 0.00425101 0.00663034 2023-09-23T19:22:40.533 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:40 smithi016 bash[77995]: audit 2023-09-23T19:22:39.140599+0000 mon.c (mon.2) 62 : audit 0 from='client.? 172.21.15.16:0/3951419624' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:40.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:40 smithi016 bash[77995]: audit 2023-09-23T19:22:39.268954+0000 mon.c (mon.2) 63 : audit 1 from='osd.1 [v2:172.21.15.16:6810/3881357617,v1:172.21.15.16:6811/3881357617]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T19:22:40.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:40 smithi016 bash[77995]: audit 2023-09-23T19:22:39.271013+0000 mon.a (mon.0) 552 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T19:22:40.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:40 smithi016 bash[77995]: cluster 2023-09-23T19:22:39.717686+0000 mgr.y (mgr.25325) 400 : cluster 0 pgmap v322: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 1.0 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s; 7537/50301 objects degraded (14.984%) 2023-09-23T19:22:40.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:40 smithi016 bash[75004]: audit 2023-09-23T19:22:39.140599+0000 mon.c (mon.2) 62 : audit 0 from='client.? 172.21.15.16:0/3951419624' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:40.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:40 smithi016 bash[75004]: audit 2023-09-23T19:22:39.268954+0000 mon.c (mon.2) 63 : audit 1 from='osd.1 [v2:172.21.15.16:6810/3881357617,v1:172.21.15.16:6811/3881357617]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T19:22:40.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:40 smithi016 bash[75004]: audit 2023-09-23T19:22:39.271013+0000 mon.a (mon.0) 552 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T19:22:40.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:40 smithi016 bash[75004]: cluster 2023-09-23T19:22:39.717686+0000 mgr.y (mgr.25325) 400 : cluster 0 pgmap v322: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 1.0 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s; 7537/50301 objects degraded (14.984%) 2023-09-23T19:22:40.535 INFO:journalctl@ceph.osd.1.smithi016.stdout:Sep 23 19:22:40 smithi016 bash[91347]: debug 2023-09-23T19:22:40.165+0000 7f8ee1627700 -1 osd.1 93 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T19:22:40.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:40 smithi092 bash[58855]: audit 2023-09-23T19:22:39.140599+0000 mon.c (mon.2) 62 : audit 0 from='client.? 172.21.15.16:0/3951419624' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:40.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:40 smithi092 bash[58855]: audit 2023-09-23T19:22:39.268954+0000 mon.c (mon.2) 63 : audit 1 from='osd.1 [v2:172.21.15.16:6810/3881357617,v1:172.21.15.16:6811/3881357617]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T19:22:40.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:40 smithi092 bash[58855]: audit 2023-09-23T19:22:39.271013+0000 mon.a (mon.0) 552 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T19:22:40.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:40 smithi092 bash[58855]: cluster 2023-09-23T19:22:39.717686+0000 mgr.y (mgr.25325) 400 : cluster 0 pgmap v322: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 1.0 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s; 7537/50301 objects degraded (14.984%) 2023-09-23T19:22:41.514 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 12 16 27924 27908 145.34 86.5625 0.00242465 0.00666454 2023-09-23T19:22:41.533 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:41 smithi016 bash[77995]: cluster 2023-09-23T19:22:40.116087+0000 osd.3 (osd.3) 24 : cluster 3 failed to encode map e97 with expected crc 2023-09-23T19:22:41.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:41 smithi016 bash[77995]: audit 2023-09-23T19:22:40.119849+0000 mon.a (mon.0) 553 : audit 1 from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T19:22:41.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:41 smithi016 bash[77995]: cluster 2023-09-23T19:22:40.119988+0000 mon.a (mon.0) 554 : cluster 0 osdmap e97: 8 total, 7 up, 8 in 2023-09-23T19:22:41.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:41 smithi016 bash[77995]: audit 2023-09-23T19:22:40.122731+0000 mon.c (mon.2) 64 : audit 1 from='osd.1 [v2:172.21.15.16:6810/3881357617,v1:172.21.15.16:6811/3881357617]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-09-23T19:22:41.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:41 smithi016 bash[77995]: audit 2023-09-23T19:22:40.123279+0000 mon.a (mon.0) 555 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-09-23T19:22:41.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:41 smithi016 bash[77995]: cluster 2023-09-23T19:22:40.139101+0000 osd.4 (osd.4) 18 : cluster 3 failed to encode map e97 with expected crc 2023-09-23T19:22:41.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:41 smithi016 bash[75004]: cluster 2023-09-23T19:22:40.116087+0000 osd.3 (osd.3) 24 : cluster 3 failed to encode map e97 with expected crc 2023-09-23T19:22:41.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:41 smithi016 bash[75004]: audit 2023-09-23T19:22:40.119849+0000 mon.a (mon.0) 553 : audit 1 from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T19:22:41.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:41 smithi016 bash[75004]: cluster 2023-09-23T19:22:40.119988+0000 mon.a (mon.0) 554 : cluster 0 osdmap e97: 8 total, 7 up, 8 in 2023-09-23T19:22:41.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:41 smithi016 bash[75004]: audit 2023-09-23T19:22:40.122731+0000 mon.c (mon.2) 64 : audit 1 from='osd.1 [v2:172.21.15.16:6810/3881357617,v1:172.21.15.16:6811/3881357617]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-09-23T19:22:41.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:41 smithi016 bash[75004]: audit 2023-09-23T19:22:40.123279+0000 mon.a (mon.0) 555 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-09-23T19:22:41.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:41 smithi016 bash[75004]: cluster 2023-09-23T19:22:40.139101+0000 osd.4 (osd.4) 18 : cluster 3 failed to encode map e97 with expected crc 2023-09-23T19:22:41.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:41 smithi092 bash[58855]: cluster 2023-09-23T19:22:40.116087+0000 osd.3 (osd.3) 24 : cluster 3 failed to encode map e97 with expected crc 2023-09-23T19:22:41.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:41 smithi092 bash[58855]: audit 2023-09-23T19:22:40.119849+0000 mon.a (mon.0) 553 : audit 1 from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T19:22:41.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:41 smithi092 bash[58855]: cluster 2023-09-23T19:22:40.119988+0000 mon.a (mon.0) 554 : cluster 0 osdmap e97: 8 total, 7 up, 8 in 2023-09-23T19:22:41.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:41 smithi092 bash[58855]: audit 2023-09-23T19:22:40.122731+0000 mon.c (mon.2) 64 : audit 1 from='osd.1 [v2:172.21.15.16:6810/3881357617,v1:172.21.15.16:6811/3881357617]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-09-23T19:22:41.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:41 smithi092 bash[58855]: audit 2023-09-23T19:22:40.123279+0000 mon.a (mon.0) 555 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-09-23T19:22:41.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:41 smithi092 bash[58855]: cluster 2023-09-23T19:22:40.139101+0000 osd.4 (osd.4) 18 : cluster 3 failed to encode map e97 with expected crc 2023-09-23T19:22:42.514 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 13 16 27924 27908 134.16 0 - 0.00666454 2023-09-23T19:22:42.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:42 smithi016 bash[77995]: cluster 2023-09-23T19:22:41.115171+0000 mon.a (mon.0) 556 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T19:22:42.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:42 smithi016 bash[77995]: cluster 2023-09-23T19:22:41.123130+0000 osd.2 (osd.2) 21 : cluster 3 failed to encode map e98 with expected crc 2023-09-23T19:22:42.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:42 smithi016 bash[77995]: cluster 2023-09-23T19:22:41.124732+0000 mon.a (mon.0) 557 : cluster 1 osd.1 [v2:172.21.15.16:6810/3881357617,v1:172.21.15.16:6811/3881357617] boot 2023-09-23T19:22:42.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:42 smithi016 bash[77995]: cluster 2023-09-23T19:22:41.124859+0000 mon.a (mon.0) 558 : cluster 0 osdmap e98: 8 total, 8 up, 8 in 2023-09-23T19:22:42.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:42 smithi016 bash[77995]: audit 2023-09-23T19:22:41.125422+0000 mon.a (mon.0) 559 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T19:22:42.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:42 smithi016 bash[77995]: cluster 2023-09-23T19:22:41.126339+0000 osd.4 (osd.4) 19 : cluster 3 failed to encode map e98 with expected crc 2023-09-23T19:22:42.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:42 smithi016 bash[77995]: audit 2023-09-23T19:22:41.495591+0000 mon.c (mon.2) 65 : audit 0 from='client.? 172.21.15.16:0/3639127694' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:42.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:42 smithi016 bash[77995]: cluster 2023-09-23T19:22:41.718619+0000 mgr.y (mgr.25325) 401 : cluster 0 pgmap v325: 89 pgs: 9 peering, 7 active+undersized+degraded, 23 active+undersized, 50 active+clean; 1.5 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 156 MiB/s wr, 2.49k op/s; 5656/73128 objects degraded (7.734%) 2023-09-23T19:22:42.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:42 smithi016 bash[77995]: cluster 2023-09-23T19:22:42.122020+0000 mon.a (mon.0) 560 : cluster 0 osdmap e99: 8 total, 8 up, 8 in 2023-09-23T19:22:42.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:42 smithi016 bash[75004]: cluster 2023-09-23T19:22:41.115171+0000 mon.a (mon.0) 556 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T19:22:42.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:42 smithi016 bash[75004]: cluster 2023-09-23T19:22:41.123130+0000 osd.2 (osd.2) 21 : cluster 3 failed to encode map e98 with expected crc 2023-09-23T19:22:42.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:42 smithi016 bash[75004]: cluster 2023-09-23T19:22:41.124732+0000 mon.a (mon.0) 557 : cluster 1 osd.1 [v2:172.21.15.16:6810/3881357617,v1:172.21.15.16:6811/3881357617] boot 2023-09-23T19:22:42.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:42 smithi016 bash[75004]: cluster 2023-09-23T19:22:41.124859+0000 mon.a (mon.0) 558 : cluster 0 osdmap e98: 8 total, 8 up, 8 in 2023-09-23T19:22:42.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:42 smithi016 bash[75004]: audit 2023-09-23T19:22:41.125422+0000 mon.a (mon.0) 559 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T19:22:42.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:42 smithi016 bash[75004]: cluster 2023-09-23T19:22:41.126339+0000 osd.4 (osd.4) 19 : cluster 3 failed to encode map e98 with expected crc 2023-09-23T19:22:42.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:42 smithi016 bash[75004]: audit 2023-09-23T19:22:41.495591+0000 mon.c (mon.2) 65 : audit 0 from='client.? 172.21.15.16:0/3639127694' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:42.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:42 smithi016 bash[75004]: cluster 2023-09-23T19:22:41.718619+0000 mgr.y (mgr.25325) 401 : cluster 0 pgmap v325: 89 pgs: 9 peering, 7 active+undersized+degraded, 23 active+undersized, 50 active+clean; 1.5 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 156 MiB/s wr, 2.49k op/s; 5656/73128 objects degraded (7.734%) 2023-09-23T19:22:42.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:42 smithi016 bash[75004]: cluster 2023-09-23T19:22:42.122020+0000 mon.a (mon.0) 560 : cluster 0 osdmap e99: 8 total, 8 up, 8 in 2023-09-23T19:22:42.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:42 smithi092 bash[58855]: cluster 2023-09-23T19:22:41.115171+0000 mon.a (mon.0) 556 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T19:22:42.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:42 smithi092 bash[58855]: cluster 2023-09-23T19:22:41.123130+0000 osd.2 (osd.2) 21 : cluster 3 failed to encode map e98 with expected crc 2023-09-23T19:22:42.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:42 smithi092 bash[58855]: cluster 2023-09-23T19:22:41.124732+0000 mon.a (mon.0) 557 : cluster 1 osd.1 [v2:172.21.15.16:6810/3881357617,v1:172.21.15.16:6811/3881357617] boot 2023-09-23T19:22:42.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:42 smithi092 bash[58855]: cluster 2023-09-23T19:22:41.124859+0000 mon.a (mon.0) 558 : cluster 0 osdmap e98: 8 total, 8 up, 8 in 2023-09-23T19:22:42.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:42 smithi092 bash[58855]: audit 2023-09-23T19:22:41.125422+0000 mon.a (mon.0) 559 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T19:22:42.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:42 smithi092 bash[58855]: cluster 2023-09-23T19:22:41.126339+0000 osd.4 (osd.4) 19 : cluster 3 failed to encode map e98 with expected crc 2023-09-23T19:22:42.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:42 smithi092 bash[58855]: audit 2023-09-23T19:22:41.495591+0000 mon.c (mon.2) 65 : audit 0 from='client.? 172.21.15.16:0/3639127694' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:42.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:42 smithi092 bash[58855]: cluster 2023-09-23T19:22:41.718619+0000 mgr.y (mgr.25325) 401 : cluster 0 pgmap v325: 89 pgs: 9 peering, 7 active+undersized+degraded, 23 active+undersized, 50 active+clean; 1.5 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 156 MiB/s wr, 2.49k op/s; 5656/73128 objects degraded (7.734%) 2023-09-23T19:22:42.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:42 smithi092 bash[58855]: cluster 2023-09-23T19:22:42.122020+0000 mon.a (mon.0) 560 : cluster 0 osdmap e99: 8 total, 8 up, 8 in 2023-09-23T19:22:43.466 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:43 smithi016 bash[77995]: cluster 2023-09-23T19:22:42.122627+0000 osd.3 (osd.3) 25 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T19:22:43.466 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:43 smithi016 bash[77995]: cluster 2023-09-23T19:22:42.122948+0000 osd.7 (osd.7) 19 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T19:22:43.466 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:43 smithi016 bash[77995]: cluster 2023-09-23T19:22:42.123538+0000 osd.2 (osd.2) 22 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T19:22:43.466 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:43 smithi016 bash[77995]: cluster 2023-09-23T19:22:42.125188+0000 osd.3 (osd.3) 26 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T19:22:43.466 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:43 smithi016 bash[77995]: cluster 2023-09-23T19:22:42.125452+0000 osd.4 (osd.4) 20 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T19:22:43.466 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:43 smithi016 bash[77995]: cluster 2023-09-23T19:22:42.125676+0000 osd.6 (osd.6) 17 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T19:22:43.466 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:43 smithi016 bash[77995]: cluster 2023-09-23T19:22:42.125882+0000 osd.5 (osd.5) 15 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T19:22:43.466 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:43 smithi016 bash[77995]: cluster 2023-09-23T19:22:42.126795+0000 osd.4 (osd.4) 21 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T19:22:43.466 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:43 smithi016 bash[77995]: cluster 2023-09-23T19:22:42.126978+0000 osd.7 (osd.7) 20 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T19:22:43.467 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:43 smithi016 bash[77995]: cluster 2023-09-23T19:22:42.128093+0000 osd.3 (osd.3) 27 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T19:22:43.467 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:43 smithi016 bash[77995]: cluster 2023-09-23T19:22:42.130531+0000 osd.6 (osd.6) 18 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T19:22:43.467 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:43 smithi016 bash[77995]: cluster 2023-09-23T19:22:42.131500+0000 osd.5 (osd.5) 16 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T19:22:43.467 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:43 smithi016 bash[77995]: cluster 2023-09-23T19:22:43.128723+0000 mon.a (mon.0) 561 : cluster 0 osdmap e100: 8 total, 8 up, 8 in 2023-09-23T19:22:43.467 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:43 smithi016 bash[75004]: cluster 2023-09-23T19:22:42.122627+0000 osd.3 (osd.3) 25 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T19:22:43.467 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:43 smithi016 bash[75004]: cluster 2023-09-23T19:22:42.122948+0000 osd.7 (osd.7) 19 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T19:22:43.467 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:43 smithi016 bash[75004]: cluster 2023-09-23T19:22:42.123538+0000 osd.2 (osd.2) 22 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T19:22:43.467 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:43 smithi016 bash[75004]: cluster 2023-09-23T19:22:42.125188+0000 osd.3 (osd.3) 26 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T19:22:43.467 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:43 smithi016 bash[75004]: cluster 2023-09-23T19:22:42.125452+0000 osd.4 (osd.4) 20 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T19:22:43.467 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:43 smithi016 bash[75004]: cluster 2023-09-23T19:22:42.125676+0000 osd.6 (osd.6) 17 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T19:22:43.468 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:43 smithi016 bash[75004]: cluster 2023-09-23T19:22:42.125882+0000 osd.5 (osd.5) 15 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T19:22:43.468 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:43 smithi016 bash[75004]: cluster 2023-09-23T19:22:42.126795+0000 osd.4 (osd.4) 21 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T19:22:43.468 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:43 smithi016 bash[75004]: cluster 2023-09-23T19:22:42.126978+0000 osd.7 (osd.7) 20 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T19:22:43.468 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:43 smithi016 bash[75004]: cluster 2023-09-23T19:22:42.128093+0000 osd.3 (osd.3) 27 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T19:22:43.468 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:43 smithi016 bash[75004]: cluster 2023-09-23T19:22:42.130531+0000 osd.6 (osd.6) 18 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T19:22:43.468 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:43 smithi016 bash[75004]: cluster 2023-09-23T19:22:42.131500+0000 osd.5 (osd.5) 16 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T19:22:43.468 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:43 smithi016 bash[75004]: cluster 2023-09-23T19:22:43.128723+0000 mon.a (mon.0) 561 : cluster 0 osdmap e100: 8 total, 8 up, 8 in 2023-09-23T19:22:43.514 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 14 16 28237 28221 125.974 9.78125 0.00575206 0.00792928 2023-09-23T19:22:43.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:43 smithi092 bash[58855]: cluster 2023-09-23T19:22:42.122627+0000 osd.3 (osd.3) 25 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T19:22:43.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:43 smithi092 bash[58855]: cluster 2023-09-23T19:22:42.122948+0000 osd.7 (osd.7) 19 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T19:22:43.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:43 smithi092 bash[58855]: cluster 2023-09-23T19:22:42.123538+0000 osd.2 (osd.2) 22 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T19:22:43.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:43 smithi092 bash[58855]: cluster 2023-09-23T19:22:42.125188+0000 osd.3 (osd.3) 26 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T19:22:43.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:43 smithi092 bash[58855]: cluster 2023-09-23T19:22:42.125452+0000 osd.4 (osd.4) 20 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T19:22:43.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:43 smithi092 bash[58855]: cluster 2023-09-23T19:22:42.125676+0000 osd.6 (osd.6) 17 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T19:22:43.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:43 smithi092 bash[58855]: cluster 2023-09-23T19:22:42.125882+0000 osd.5 (osd.5) 15 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T19:22:43.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:43 smithi092 bash[58855]: cluster 2023-09-23T19:22:42.126795+0000 osd.4 (osd.4) 21 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T19:22:43.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:43 smithi092 bash[58855]: cluster 2023-09-23T19:22:42.126978+0000 osd.7 (osd.7) 20 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T19:22:43.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:43 smithi092 bash[58855]: cluster 2023-09-23T19:22:42.128093+0000 osd.3 (osd.3) 27 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T19:22:43.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:43 smithi092 bash[58855]: cluster 2023-09-23T19:22:42.130531+0000 osd.6 (osd.6) 18 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T19:22:43.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:43 smithi092 bash[58855]: cluster 2023-09-23T19:22:42.131500+0000 osd.5 (osd.5) 16 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T19:22:43.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:43 smithi092 bash[58855]: cluster 2023-09-23T19:22:43.128723+0000 mon.a (mon.0) 561 : cluster 0 osdmap e100: 8 total, 8 up, 8 in 2023-09-23T19:22:44.514 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 15 16 29450 29434 122.629 75.8125 0.0156051 0.00814305 2023-09-23T19:22:44.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:44 smithi016 bash[75004]: cluster 2023-09-23T19:22:43.125849+0000 osd.5 (osd.5) 17 : cluster 3 failed to encode map e100 with expected crc 2023-09-23T19:22:44.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:44 smithi016 bash[75004]: cluster 2023-09-23T19:22:43.126239+0000 osd.3 (osd.3) 28 : cluster 3 failed to encode map e100 with expected crc 2023-09-23T19:22:44.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:44 smithi016 bash[75004]: cluster 2023-09-23T19:22:43.128331+0000 osd.2 (osd.2) 23 : cluster 3 failed to encode map e100 with expected crc 2023-09-23T19:22:44.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:44 smithi016 bash[75004]: cluster 2023-09-23T19:22:43.130280+0000 osd.5 (osd.5) 18 : cluster 3 failed to encode map e100 with expected crc 2023-09-23T19:22:44.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:44 smithi016 bash[75004]: cluster 2023-09-23T19:22:43.130325+0000 osd.3 (osd.3) 29 : cluster 3 failed to encode map e100 with expected crc 2023-09-23T19:22:44.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:44 smithi016 bash[75004]: cluster 2023-09-23T19:22:43.130575+0000 osd.2 (osd.2) 24 : cluster 3 failed to encode map e100 with expected crc 2023-09-23T19:22:44.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:44 smithi016 bash[75004]: cluster 2023-09-23T19:22:43.132636+0000 osd.5 (osd.5) 19 : cluster 3 failed to encode map e100 with expected crc 2023-09-23T19:22:44.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:44 smithi016 bash[75004]: cluster 2023-09-23T19:22:43.168961+0000 osd.7 (osd.7) 21 : cluster 3 failed to encode map e100 with expected crc 2023-09-23T19:22:44.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:44 smithi016 bash[75004]: audit 2023-09-23T19:22:43.632010+0000 mon.a (mon.0) 562 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:22:44.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:44 smithi016 bash[75004]: audit 2023-09-23T19:22:43.633216+0000 mon.a (mon.0) 563 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:22:44.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:44 smithi016 bash[75004]: cluster 2023-09-23T19:22:43.719187+0000 mgr.y (mgr.25325) 402 : cluster 0 pgmap v328: 89 pgs: 1 activating+undersized+degraded+remapped, 9 peering, 5 active+undersized+degraded, 21 active+undersized, 53 active+clean; 1.6 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 144 MiB/s wr, 2.30k op/s; 6041/77940 objects degraded (7.751%) 2023-09-23T19:22:44.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:44 smithi016 bash[75004]: cluster 2023-09-23T19:22:44.100275+0000 mon.a (mon.0) 564 : cluster 3 Health check update: Degraded data redundancy: 5656/73128 objects degraded (7.734%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T19:22:44.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:44 smithi016 bash[75004]: audit 2023-09-23T19:22:44.115026+0000 mon.a (mon.0) 565 : audit 0 from='client.? 172.21.15.16:0/1347754233' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:44.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:44 smithi016 bash[77995]: cluster 2023-09-23T19:22:43.125849+0000 osd.5 (osd.5) 17 : cluster 3 failed to encode map e100 with expected crc 2023-09-23T19:22:44.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:44 smithi016 bash[77995]: cluster 2023-09-23T19:22:43.126239+0000 osd.3 (osd.3) 28 : cluster 3 failed to encode map e100 with expected crc 2023-09-23T19:22:44.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:44 smithi016 bash[77995]: cluster 2023-09-23T19:22:43.128331+0000 osd.2 (osd.2) 23 : cluster 3 failed to encode map e100 with expected crc 2023-09-23T19:22:44.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:44 smithi016 bash[77995]: cluster 2023-09-23T19:22:43.130280+0000 osd.5 (osd.5) 18 : cluster 3 failed to encode map e100 with expected crc 2023-09-23T19:22:44.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:44 smithi016 bash[77995]: cluster 2023-09-23T19:22:43.130325+0000 osd.3 (osd.3) 29 : cluster 3 failed to encode map e100 with expected crc 2023-09-23T19:22:44.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:44 smithi016 bash[77995]: cluster 2023-09-23T19:22:43.130575+0000 osd.2 (osd.2) 24 : cluster 3 failed to encode map e100 with expected crc 2023-09-23T19:22:44.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:44 smithi016 bash[77995]: cluster 2023-09-23T19:22:43.132636+0000 osd.5 (osd.5) 19 : cluster 3 failed to encode map e100 with expected crc 2023-09-23T19:22:44.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:44 smithi016 bash[77995]: cluster 2023-09-23T19:22:43.168961+0000 osd.7 (osd.7) 21 : cluster 3 failed to encode map e100 with expected crc 2023-09-23T19:22:44.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:44 smithi016 bash[77995]: audit 2023-09-23T19:22:43.632010+0000 mon.a (mon.0) 562 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:22:44.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:44 smithi016 bash[77995]: audit 2023-09-23T19:22:43.633216+0000 mon.a (mon.0) 563 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:22:44.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:44 smithi016 bash[77995]: cluster 2023-09-23T19:22:43.719187+0000 mgr.y (mgr.25325) 402 : cluster 0 pgmap v328: 89 pgs: 1 activating+undersized+degraded+remapped, 9 peering, 5 active+undersized+degraded, 21 active+undersized, 53 active+clean; 1.6 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 144 MiB/s wr, 2.30k op/s; 6041/77940 objects degraded (7.751%) 2023-09-23T19:22:44.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:44 smithi016 bash[77995]: cluster 2023-09-23T19:22:44.100275+0000 mon.a (mon.0) 564 : cluster 3 Health check update: Degraded data redundancy: 5656/73128 objects degraded (7.734%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T19:22:44.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:44 smithi016 bash[77995]: audit 2023-09-23T19:22:44.115026+0000 mon.a (mon.0) 565 : audit 0 from='client.? 172.21.15.16:0/1347754233' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:44.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:44 smithi092 bash[58855]: cluster 2023-09-23T19:22:43.125849+0000 osd.5 (osd.5) 17 : cluster 3 failed to encode map e100 with expected crc 2023-09-23T19:22:44.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:44 smithi092 bash[58855]: cluster 2023-09-23T19:22:43.126239+0000 osd.3 (osd.3) 28 : cluster 3 failed to encode map e100 with expected crc 2023-09-23T19:22:44.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:44 smithi092 bash[58855]: cluster 2023-09-23T19:22:43.128331+0000 osd.2 (osd.2) 23 : cluster 3 failed to encode map e100 with expected crc 2023-09-23T19:22:44.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:44 smithi092 bash[58855]: cluster 2023-09-23T19:22:43.130280+0000 osd.5 (osd.5) 18 : cluster 3 failed to encode map e100 with expected crc 2023-09-23T19:22:44.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:44 smithi092 bash[58855]: cluster 2023-09-23T19:22:43.130325+0000 osd.3 (osd.3) 29 : cluster 3 failed to encode map e100 with expected crc 2023-09-23T19:22:44.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:44 smithi092 bash[58855]: cluster 2023-09-23T19:22:43.130575+0000 osd.2 (osd.2) 24 : cluster 3 failed to encode map e100 with expected crc 2023-09-23T19:22:44.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:44 smithi092 bash[58855]: cluster 2023-09-23T19:22:43.132636+0000 osd.5 (osd.5) 19 : cluster 3 failed to encode map e100 with expected crc 2023-09-23T19:22:44.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:44 smithi092 bash[58855]: cluster 2023-09-23T19:22:43.168961+0000 osd.7 (osd.7) 21 : cluster 3 failed to encode map e100 with expected crc 2023-09-23T19:22:44.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:44 smithi092 bash[58855]: audit 2023-09-23T19:22:43.632010+0000 mon.a (mon.0) 562 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:22:44.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:44 smithi092 bash[58855]: audit 2023-09-23T19:22:43.633216+0000 mon.a (mon.0) 563 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:22:44.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:44 smithi092 bash[58855]: cluster 2023-09-23T19:22:43.719187+0000 mgr.y (mgr.25325) 402 : cluster 0 pgmap v328: 89 pgs: 1 activating+undersized+degraded+remapped, 9 peering, 5 active+undersized+degraded, 21 active+undersized, 53 active+clean; 1.6 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 144 MiB/s wr, 2.30k op/s; 6041/77940 objects degraded (7.751%) 2023-09-23T19:22:44.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:44 smithi092 bash[58855]: cluster 2023-09-23T19:22:44.100275+0000 mon.a (mon.0) 564 : cluster 3 Health check update: Degraded data redundancy: 5656/73128 objects degraded (7.734%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T19:22:44.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:44 smithi092 bash[58855]: audit 2023-09-23T19:22:44.115026+0000 mon.a (mon.0) 565 : audit 0 from='client.? 172.21.15.16:0/1347754233' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:45.514 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 16 16 30776 30760 120.144 82.875 0.0159916 0.00830818 2023-09-23T19:22:46.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:22:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:22:45] "GET /metrics HTTP/1.1" 200 36026 "" "Prometheus/2.33.4" 2023-09-23T19:22:46.514 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 17 16 31925 31909 117.301 71.8125 0.00617426 0.00851286 2023-09-23T19:22:47.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:46 smithi016 bash[75004]: cluster 2023-09-23T19:22:45.719956+0000 mgr.y (mgr.25325) 403 : cluster 0 pgmap v329: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 9 peering, 1 active+undersized+degraded, 76 active+clean; 1.7 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s; 6303/84969 objects degraded (7.418%); 34 KiB/s, 0 objects/s recovering 2023-09-23T19:22:47.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:46 smithi016 bash[75004]: audit 2023-09-23T19:22:46.750796+0000 mon.b (mon.1) 23 : audit 0 from='client.? 172.21.15.16:0/379039900' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:47.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:46 smithi016 bash[77995]: cluster 2023-09-23T19:22:45.719956+0000 mgr.y (mgr.25325) 403 : cluster 0 pgmap v329: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 9 peering, 1 active+undersized+degraded, 76 active+clean; 1.7 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s; 6303/84969 objects degraded (7.418%); 34 KiB/s, 0 objects/s recovering 2023-09-23T19:22:47.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:46 smithi016 bash[77995]: audit 2023-09-23T19:22:46.750796+0000 mon.b (mon.1) 23 : audit 0 from='client.? 172.21.15.16:0/379039900' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:47.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:46 smithi092 bash[58855]: cluster 2023-09-23T19:22:45.719956+0000 mgr.y (mgr.25325) 403 : cluster 0 pgmap v329: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 9 peering, 1 active+undersized+degraded, 76 active+clean; 1.7 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s; 6303/84969 objects degraded (7.418%); 34 KiB/s, 0 objects/s recovering 2023-09-23T19:22:47.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:46 smithi092 bash[58855]: audit 2023-09-23T19:22:46.750796+0000 mon.b (mon.1) 23 : audit 0 from='client.? 172.21.15.16:0/379039900' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:47.514 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 18 16 33111 33095 114.902 74.125 0.0148035 0.00869658 2023-09-23T19:22:48.514 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 19 16 34378 34362 113.022 79.1875 0.0208985 0.00884064 2023-09-23T19:22:48.784 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:22:48 smithi016 bash[70992]: debug 2023-09-23T19:22:48.501+0000 7fb2a5a7c700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:22:49.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:48 smithi016 bash[75004]: cluster 2023-09-23T19:22:47.720686+0000 mgr.y (mgr.25325) 404 : cluster 0 pgmap v330: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.8 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s; 12163/89622 objects degraded (13.571%); 533 KiB/s, 8 objects/s recovering 2023-09-23T19:22:49.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:48 smithi016 bash[75004]: audit 2023-09-23T19:22:48.384720+0000 mon.a (mon.0) 566 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:22:49.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:48 smithi016 bash[75004]: audit 2023-09-23T19:22:48.393986+0000 mon.a (mon.0) 567 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:22:49.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:48 smithi016 bash[75004]: audit 2023-09-23T19:22:48.395467+0000 mon.a (mon.0) 568 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:22:49.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:48 smithi016 bash[75004]: audit 2023-09-23T19:22:48.397364+0000 mon.a (mon.0) 569 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:22:49.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:48 smithi016 bash[75004]: audit 2023-09-23T19:22:48.405464+0000 mon.a (mon.0) 570 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:22:49.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:48 smithi016 bash[75004]: audit 2023-09-23T19:22:48.488486+0000 mon.a (mon.0) 571 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:22:49.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:48 smithi016 bash[75004]: audit 2023-09-23T19:22:48.493839+0000 mon.a (mon.0) 572 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:22:49.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:48 smithi016 bash[75004]: audit 2023-09-23T19:22:48.497707+0000 mon.a (mon.0) 573 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:22:49.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:48 smithi016 bash[75004]: audit 2023-09-23T19:22:48.500170+0000 mon.a (mon.0) 574 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:22:49.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:48 smithi016 bash[75004]: audit 2023-09-23T19:22:48.504436+0000 mon.a (mon.0) 575 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:22:49.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:48 smithi016 bash[77995]: cluster 2023-09-23T19:22:47.720686+0000 mgr.y (mgr.25325) 404 : cluster 0 pgmap v330: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.8 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s; 12163/89622 objects degraded (13.571%); 533 KiB/s, 8 objects/s recovering 2023-09-23T19:22:49.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:48 smithi016 bash[77995]: audit 2023-09-23T19:22:48.384720+0000 mon.a (mon.0) 566 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:22:49.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:48 smithi016 bash[77995]: audit 2023-09-23T19:22:48.393986+0000 mon.a (mon.0) 567 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:22:49.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:48 smithi016 bash[77995]: audit 2023-09-23T19:22:48.395467+0000 mon.a (mon.0) 568 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:22:49.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:48 smithi016 bash[77995]: audit 2023-09-23T19:22:48.397364+0000 mon.a (mon.0) 569 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:22:49.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:48 smithi016 bash[77995]: audit 2023-09-23T19:22:48.405464+0000 mon.a (mon.0) 570 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:22:49.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:48 smithi016 bash[77995]: audit 2023-09-23T19:22:48.488486+0000 mon.a (mon.0) 571 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:22:49.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:48 smithi016 bash[77995]: audit 2023-09-23T19:22:48.493839+0000 mon.a (mon.0) 572 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:22:49.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:48 smithi016 bash[77995]: audit 2023-09-23T19:22:48.497707+0000 mon.a (mon.0) 573 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:22:49.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:48 smithi016 bash[77995]: audit 2023-09-23T19:22:48.500170+0000 mon.a (mon.0) 574 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:22:49.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:48 smithi016 bash[77995]: audit 2023-09-23T19:22:48.504436+0000 mon.a (mon.0) 575 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:22:49.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:48 smithi092 bash[58855]: cluster 2023-09-23T19:22:47.720686+0000 mgr.y (mgr.25325) 404 : cluster 0 pgmap v330: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.8 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s; 12163/89622 objects degraded (13.571%); 533 KiB/s, 8 objects/s recovering 2023-09-23T19:22:49.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:48 smithi092 bash[58855]: audit 2023-09-23T19:22:48.384720+0000 mon.a (mon.0) 566 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:22:49.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:48 smithi092 bash[58855]: audit 2023-09-23T19:22:48.393986+0000 mon.a (mon.0) 567 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:22:49.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:48 smithi092 bash[58855]: audit 2023-09-23T19:22:48.395467+0000 mon.a (mon.0) 568 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:22:49.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:48 smithi092 bash[58855]: audit 2023-09-23T19:22:48.397364+0000 mon.a (mon.0) 569 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:22:49.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:48 smithi092 bash[58855]: audit 2023-09-23T19:22:48.405464+0000 mon.a (mon.0) 570 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:22:49.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:48 smithi092 bash[58855]: audit 2023-09-23T19:22:48.488486+0000 mon.a (mon.0) 571 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:22:49.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:48 smithi092 bash[58855]: audit 2023-09-23T19:22:48.493839+0000 mon.a (mon.0) 572 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:22:49.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:48 smithi092 bash[58855]: audit 2023-09-23T19:22:48.497707+0000 mon.a (mon.0) 573 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:22:49.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:48 smithi092 bash[58855]: audit 2023-09-23T19:22:48.500170+0000 mon.a (mon.0) 574 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:22:49.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:48 smithi092 bash[58855]: audit 2023-09-23T19:22:48.504436+0000 mon.a (mon.0) 575 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:22:49.514 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-09-23T19:22:49.513893+0000 min lat: 0.00119935 max lat: 2.18958 avg lat: 0.00899236 2023-09-23T19:22:49.515 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:22:49.515 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 20 16 35576 35560 111.114 74.875 0.0116552 0.00899236 2023-09-23T19:22:50.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:49 smithi016 bash[77995]: audit 2023-09-23T19:22:48.504696+0000 mgr.y (mgr.25325) 405 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:22:50.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:49 smithi016 bash[77995]: cephadm 2023-09-23T19:22:48.506367+0000 mgr.y (mgr.25325) 406 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:22:50.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:49 smithi016 bash[77995]: cluster 2023-09-23T19:22:49.101568+0000 mon.a (mon.0) 576 : cluster 3 Health check update: Degraded data redundancy: 12163/89622 objects degraded (13.571%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T19:22:50.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:49 smithi016 bash[77995]: audit 2023-09-23T19:22:49.348127+0000 mon.a (mon.0) 577 : audit 0 from='client.? 172.21.15.16:0/3765141125' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:50.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:49 smithi016 bash[75004]: audit 2023-09-23T19:22:48.504696+0000 mgr.y (mgr.25325) 405 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:22:50.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:49 smithi016 bash[75004]: cephadm 2023-09-23T19:22:48.506367+0000 mgr.y (mgr.25325) 406 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:22:50.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:49 smithi016 bash[75004]: cluster 2023-09-23T19:22:49.101568+0000 mon.a (mon.0) 576 : cluster 3 Health check update: Degraded data redundancy: 12163/89622 objects degraded (13.571%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T19:22:50.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:49 smithi016 bash[75004]: audit 2023-09-23T19:22:49.348127+0000 mon.a (mon.0) 577 : audit 0 from='client.? 172.21.15.16:0/3765141125' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:50.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:49 smithi092 bash[58855]: audit 2023-09-23T19:22:48.504696+0000 mgr.y (mgr.25325) 405 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:22:50.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:49 smithi092 bash[58855]: cephadm 2023-09-23T19:22:48.506367+0000 mgr.y (mgr.25325) 406 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:22:50.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:49 smithi092 bash[58855]: cluster 2023-09-23T19:22:49.101568+0000 mon.a (mon.0) 576 : cluster 3 Health check update: Degraded data redundancy: 12163/89622 objects degraded (13.571%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T19:22:50.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:49 smithi092 bash[58855]: audit 2023-09-23T19:22:49.348127+0000 mon.a (mon.0) 577 : audit 0 from='client.? 172.21.15.16:0/3765141125' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:50.515 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 21 15 36733 36718 109.269 72.375 0.0210865 0.00914407 2023-09-23T19:22:51.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:50 smithi016 bash[75004]: cluster 2023-09-23T19:22:49.721232+0000 mgr.y (mgr.25325) 407 : cluster 0 pgmap v331: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.9 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 54 MiB/s wr, 871 op/s; 12163/94041 objects degraded (12.934%); 440 KiB/s, 6 objects/s recovering 2023-09-23T19:22:51.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:50 smithi016 bash[77995]: cluster 2023-09-23T19:22:49.721232+0000 mgr.y (mgr.25325) 407 : cluster 0 pgmap v331: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.9 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 54 MiB/s wr, 871 op/s; 12163/94041 objects degraded (12.934%); 440 KiB/s, 6 objects/s recovering 2023-09-23T19:22:51.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:50 smithi092 bash[58855]: cluster 2023-09-23T19:22:49.721232+0000 mgr.y (mgr.25325) 407 : cluster 0 pgmap v331: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.9 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 54 MiB/s wr, 871 op/s; 12163/94041 objects degraded (12.934%); 440 KiB/s, 6 objects/s recovering 2023-09-23T19:22:51.515 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 22 16 37949 37933 107.753 75.9375 0.00636836 0.00927072 2023-09-23T19:22:52.515 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 23 16 39132 39116 106.283 73.9375 0.00975963 0.0094015 2023-09-23T19:22:53.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:52 smithi016 bash[75004]: cluster 2023-09-23T19:22:51.722132+0000 mgr.y (mgr.25325) 408 : cluster 0 pgmap v332: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s; 12068/107763 objects degraded (11.199%); 1000 KiB/s, 15 objects/s recovering 2023-09-23T19:22:53.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:52 smithi016 bash[75004]: audit 2023-09-23T19:22:51.984047+0000 mon.b (mon.1) 24 : audit 0 from='client.? 172.21.15.16:0/3576775308' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:53.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:52 smithi016 bash[77995]: cluster 2023-09-23T19:22:51.722132+0000 mgr.y (mgr.25325) 408 : cluster 0 pgmap v332: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s; 12068/107763 objects degraded (11.199%); 1000 KiB/s, 15 objects/s recovering 2023-09-23T19:22:53.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:52 smithi016 bash[77995]: audit 2023-09-23T19:22:51.984047+0000 mon.b (mon.1) 24 : audit 0 from='client.? 172.21.15.16:0/3576775308' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:53.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:52 smithi092 bash[58855]: cluster 2023-09-23T19:22:51.722132+0000 mgr.y (mgr.25325) 408 : cluster 0 pgmap v332: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s; 12068/107763 objects degraded (11.199%); 1000 KiB/s, 15 objects/s recovering 2023-09-23T19:22:53.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:52 smithi092 bash[58855]: audit 2023-09-23T19:22:51.984047+0000 mon.b (mon.1) 24 : audit 0 from='client.? 172.21.15.16:0/3576775308' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:53.515 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 24 16 40345 40329 105.013 75.8125 0.00576813 0.00951566 2023-09-23T19:22:54.515 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 25 16 41476 41460 103.64 70.6875 0.0156601 0.00964005 2023-09-23T19:22:55.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:54 smithi016 bash[77995]: cluster 2023-09-23T19:22:53.722871+0000 mgr.y (mgr.25325) 409 : cluster 0 pgmap v333: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.3 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s; 12068/112101 objects degraded (10.765%); 906 KiB/s, 14 objects/s recovering 2023-09-23T19:22:55.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:54 smithi016 bash[77995]: cluster 2023-09-23T19:22:54.102596+0000 mon.a (mon.0) 578 : cluster 3 Health check update: Degraded data redundancy: 12068/112101 objects degraded (10.765%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T19:22:55.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:54 smithi016 bash[77995]: audit 2023-09-23T19:22:54.642364+0000 mon.c (mon.2) 66 : audit 0 from='client.? 172.21.15.16:0/2287756495' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:55.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:54 smithi016 bash[75004]: cluster 2023-09-23T19:22:53.722871+0000 mgr.y (mgr.25325) 409 : cluster 0 pgmap v333: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.3 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s; 12068/112101 objects degraded (10.765%); 906 KiB/s, 14 objects/s recovering 2023-09-23T19:22:55.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:54 smithi016 bash[75004]: cluster 2023-09-23T19:22:54.102596+0000 mon.a (mon.0) 578 : cluster 3 Health check update: Degraded data redundancy: 12068/112101 objects degraded (10.765%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T19:22:55.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:54 smithi016 bash[75004]: audit 2023-09-23T19:22:54.642364+0000 mon.c (mon.2) 66 : audit 0 from='client.? 172.21.15.16:0/2287756495' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:55.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:54 smithi092 bash[58855]: cluster 2023-09-23T19:22:53.722871+0000 mgr.y (mgr.25325) 409 : cluster 0 pgmap v333: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.3 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s; 12068/112101 objects degraded (10.765%); 906 KiB/s, 14 objects/s recovering 2023-09-23T19:22:55.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:54 smithi092 bash[58855]: cluster 2023-09-23T19:22:54.102596+0000 mon.a (mon.0) 578 : cluster 3 Health check update: Degraded data redundancy: 12068/112101 objects degraded (10.765%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T19:22:55.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:54 smithi092 bash[58855]: audit 2023-09-23T19:22:54.642364+0000 mon.c (mon.2) 66 : audit 0 from='client.? 172.21.15.16:0/2287756495' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:55.516 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 26 16 42625 42609 102.415 71.8125 0.0301565 0.00975575 2023-09-23T19:22:56.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:22:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:22:55] "GET /metrics HTTP/1.1" 200 36026 "" "Prometheus/2.33.4" 2023-09-23T19:22:56.515 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 27 16 43801 43785 101.344 73.5 0.0306561 0.00986012 2023-09-23T19:22:57.283 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:56 smithi016 bash[77995]: cluster 2023-09-23T19:22:55.723911+0000 mgr.y (mgr.25325) 410 : cluster 0 pgmap v334: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.5 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s; 12068/120993 objects degraded (9.974%); 800 KiB/s, 12 objects/s recovering 2023-09-23T19:22:57.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:56 smithi016 bash[75004]: cluster 2023-09-23T19:22:55.723911+0000 mgr.y (mgr.25325) 410 : cluster 0 pgmap v334: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.5 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s; 12068/120993 objects degraded (9.974%); 800 KiB/s, 12 objects/s recovering 2023-09-23T19:22:57.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:56 smithi092 bash[58855]: cluster 2023-09-23T19:22:55.723911+0000 mgr.y (mgr.25325) 410 : cluster 0 pgmap v334: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.5 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s; 12068/120993 objects degraded (9.974%); 800 KiB/s, 12 objects/s recovering 2023-09-23T19:22:57.515 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 28 16 44872 44856 100.115 66.9375 0.00960539 0.0099795 2023-09-23T19:22:58.283 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:57 smithi016 bash[77995]: audit 2023-09-23T19:22:57.222709+0000 mon.a (mon.0) 579 : audit 0 from='client.? 172.21.15.16:0/495782497' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:58.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:57 smithi016 bash[75004]: audit 2023-09-23T19:22:57.222709+0000 mon.a (mon.0) 579 : audit 0 from='client.? 172.21.15.16:0/495782497' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:58.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:57 smithi092 bash[58855]: audit 2023-09-23T19:22:57.222709+0000 mon.a (mon.0) 579 : audit 0 from='client.? 172.21.15.16:0/495782497' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:22:58.515 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 29 16 45999 45983 99.0913 70.4375 0.0118101 0.0100835 2023-09-23T19:22:59.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:58 smithi016 bash[77995]: cluster 2023-09-23T19:22:57.724651+0000 mgr.y (mgr.25325) 411 : cluster 0 pgmap v335: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.5 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 11967/125340 objects degraded (9.548%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T19:22:59.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:58 smithi016 bash[77995]: audit 2023-09-23T19:22:58.623855+0000 mon.a (mon.0) 580 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:22:59.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:58 smithi016 bash[75004]: cluster 2023-09-23T19:22:57.724651+0000 mgr.y (mgr.25325) 411 : cluster 0 pgmap v335: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.5 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 11967/125340 objects degraded (9.548%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T19:22:59.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:58 smithi016 bash[75004]: audit 2023-09-23T19:22:58.623855+0000 mon.a (mon.0) 580 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:22:59.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:58 smithi092 bash[58855]: cluster 2023-09-23T19:22:57.724651+0000 mgr.y (mgr.25325) 411 : cluster 0 pgmap v335: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.5 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 11967/125340 objects degraded (9.548%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T19:22:59.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:58 smithi092 bash[58855]: audit 2023-09-23T19:22:58.623855+0000 mon.a (mon.0) 580 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:22:59.516 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 30 16 47101 47085 98.0838 68.875 0.0125135 0.0101878 2023-09-23T19:23:00.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:59 smithi016 bash[75004]: cluster 2023-09-23T19:22:59.103777+0000 mon.a (mon.0) 581 : cluster 3 Health check update: Degraded data redundancy: 11967/125340 objects degraded (9.548%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T19:23:00.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:22:59 smithi016 bash[75004]: cluster 2023-09-23T19:22:59.725425+0000 mgr.y (mgr.25325) 412 : cluster 0 pgmap v336: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.6 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s; 11967/129558 objects degraded (9.237%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T19:23:00.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:59 smithi016 bash[77995]: cluster 2023-09-23T19:22:59.103777+0000 mon.a (mon.0) 581 : cluster 3 Health check update: Degraded data redundancy: 11967/125340 objects degraded (9.548%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T19:23:00.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:22:59 smithi016 bash[77995]: cluster 2023-09-23T19:22:59.725425+0000 mgr.y (mgr.25325) 412 : cluster 0 pgmap v336: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.6 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s; 11967/129558 objects degraded (9.237%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T19:23:00.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:59 smithi092 bash[58855]: cluster 2023-09-23T19:22:59.103777+0000 mon.a (mon.0) 581 : cluster 3 Health check update: Degraded data redundancy: 11967/125340 objects degraded (9.548%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T19:23:00.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:22:59 smithi092 bash[58855]: cluster 2023-09-23T19:22:59.725425+0000 mgr.y (mgr.25325) 412 : cluster 0 pgmap v336: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.6 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s; 11967/129558 objects degraded (9.237%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T19:23:00.516 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 31 16 48202 48186 97.1394 68.8125 0.00580547 0.0102853 2023-09-23T19:23:01.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:00 smithi016 bash[77995]: audit 2023-09-23T19:22:59.848783+0000 mon.a (mon.0) 582 : audit 0 from='client.? 172.21.15.16:0/626957839' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:01.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:00 smithi016 bash[75004]: audit 2023-09-23T19:22:59.848783+0000 mon.a (mon.0) 582 : audit 0 from='client.? 172.21.15.16:0/626957839' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:01.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:00 smithi092 bash[58855]: audit 2023-09-23T19:22:59.848783+0000 mon.a (mon.0) 582 : audit 0 from='client.? 172.21.15.16:0/626957839' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:01.516 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 32 16 49322 49306 96.2911 70 0.00932708 0.0103789 2023-09-23T19:23:02.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:01 smithi016 bash[77995]: cluster 2023-09-23T19:23:01.726314+0000 mgr.y (mgr.25325) 413 : cluster 0 pgmap v337: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s; 11864/142113 objects degraded (8.348%); 1.6 MiB/s, 24 objects/s recovering 2023-09-23T19:23:02.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:01 smithi016 bash[75004]: cluster 2023-09-23T19:23:01.726314+0000 mgr.y (mgr.25325) 413 : cluster 0 pgmap v337: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s; 11864/142113 objects degraded (8.348%); 1.6 MiB/s, 24 objects/s recovering 2023-09-23T19:23:02.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:01 smithi092 bash[58855]: cluster 2023-09-23T19:23:01.726314+0000 mgr.y (mgr.25325) 413 : cluster 0 pgmap v337: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s; 11864/142113 objects degraded (8.348%); 1.6 MiB/s, 24 objects/s recovering 2023-09-23T19:23:02.517 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 33 16 50357 50341 95.3332 64.6875 0.00675386 0.0104828 2023-09-23T19:23:03.283 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:02 smithi016 bash[77995]: audit 2023-09-23T19:23:02.473476+0000 mon.c (mon.2) 67 : audit 0 from='client.? 172.21.15.16:0/489864816' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:03.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:02 smithi016 bash[75004]: audit 2023-09-23T19:23:02.473476+0000 mon.c (mon.2) 67 : audit 0 from='client.? 172.21.15.16:0/489864816' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:03.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:02 smithi092 bash[58855]: audit 2023-09-23T19:23:02.473476+0000 mon.c (mon.2) 67 : audit 0 from='client.? 172.21.15.16:0/489864816' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:03.516 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 34 16 51426 51410 94.4942 66.8125 0.00643751 0.0105743 2023-09-23T19:23:03.784 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:23:03 smithi016 bash[70992]: debug 2023-09-23T19:23:03.517+0000 7fb2a5a7c700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:23:04.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:03 smithi016 bash[75004]: audit 2023-09-23T19:23:03.521141+0000 mon.a (mon.0) 583 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:23:04.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:03 smithi016 bash[75004]: audit 2023-09-23T19:23:03.521491+0000 mgr.y (mgr.25325) 414 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:23:04.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:03 smithi016 bash[75004]: cephadm 2023-09-23T19:23:03.522859+0000 mgr.y (mgr.25325) 415 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:23:04.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:03 smithi016 bash[75004]: cluster 2023-09-23T19:23:03.727154+0000 mgr.y (mgr.25325) 416 : cluster 0 pgmap v338: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 11864/146241 objects degraded (8.113%); 1.1 MiB/s, 16 objects/s recovering 2023-09-23T19:23:04.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:03 smithi016 bash[77995]: audit 2023-09-23T19:23:03.521141+0000 mon.a (mon.0) 583 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:23:04.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:03 smithi016 bash[77995]: audit 2023-09-23T19:23:03.521491+0000 mgr.y (mgr.25325) 414 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:23:04.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:03 smithi016 bash[77995]: cephadm 2023-09-23T19:23:03.522859+0000 mgr.y (mgr.25325) 415 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:23:04.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:03 smithi016 bash[77995]: cluster 2023-09-23T19:23:03.727154+0000 mgr.y (mgr.25325) 416 : cluster 0 pgmap v338: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 11864/146241 objects degraded (8.113%); 1.1 MiB/s, 16 objects/s recovering 2023-09-23T19:23:04.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:03 smithi092 bash[58855]: audit 2023-09-23T19:23:03.521141+0000 mon.a (mon.0) 583 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:23:04.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:03 smithi092 bash[58855]: audit 2023-09-23T19:23:03.521491+0000 mgr.y (mgr.25325) 414 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:23:04.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:03 smithi092 bash[58855]: cephadm 2023-09-23T19:23:03.522859+0000 mgr.y (mgr.25325) 415 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:23:04.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:03 smithi092 bash[58855]: cluster 2023-09-23T19:23:03.727154+0000 mgr.y (mgr.25325) 416 : cluster 0 pgmap v338: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 11864/146241 objects degraded (8.113%); 1.1 MiB/s, 16 objects/s recovering 2023-09-23T19:23:04.516 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 35 16 52470 52454 93.6585 65.25 0.00781372 0.010666 2023-09-23T19:23:05.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:04 smithi016 bash[75004]: cluster 2023-09-23T19:23:04.104876+0000 mon.a (mon.0) 584 : cluster 3 Health check update: Degraded data redundancy: 11864/146241 objects degraded (8.113%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T19:23:05.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:04 smithi016 bash[77995]: cluster 2023-09-23T19:23:04.104876+0000 mon.a (mon.0) 584 : cluster 3 Health check update: Degraded data redundancy: 11864/146241 objects degraded (8.113%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T19:23:05.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:04 smithi092 bash[58855]: cluster 2023-09-23T19:23:04.104876+0000 mon.a (mon.0) 584 : cluster 3 Health check update: Degraded data redundancy: 11864/146241 objects degraded (8.113%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T19:23:05.516 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 36 16 53504 53488 92.8518 64.625 0.00736572 0.0107626 2023-09-23T19:23:06.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:05 smithi016 bash[75004]: audit 2023-09-23T19:23:05.079642+0000 mon.b (mon.1) 25 : audit 0 from='client.? 172.21.15.16:0/890602537' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:06.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:05 smithi016 bash[75004]: cluster 2023-09-23T19:23:05.727957+0000 mgr.y (mgr.25325) 417 : cluster 0 pgmap v339: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 11864/154350 objects degraded (7.686%); 1.1 MiB/s, 16 objects/s recovering 2023-09-23T19:23:06.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:23:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:23:05] "GET /metrics HTTP/1.1" 200 36185 "" "Prometheus/2.33.4" 2023-09-23T19:23:06.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:05 smithi016 bash[77995]: audit 2023-09-23T19:23:05.079642+0000 mon.b (mon.1) 25 : audit 0 from='client.? 172.21.15.16:0/890602537' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:06.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:05 smithi016 bash[77995]: cluster 2023-09-23T19:23:05.727957+0000 mgr.y (mgr.25325) 417 : cluster 0 pgmap v339: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 11864/154350 objects degraded (7.686%); 1.1 MiB/s, 16 objects/s recovering 2023-09-23T19:23:06.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:05 smithi092 bash[58855]: audit 2023-09-23T19:23:05.079642+0000 mon.b (mon.1) 25 : audit 0 from='client.? 172.21.15.16:0/890602537' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:06.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:05 smithi092 bash[58855]: cluster 2023-09-23T19:23:05.727957+0000 mgr.y (mgr.25325) 417 : cluster 0 pgmap v339: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 11864/154350 objects degraded (7.686%); 1.1 MiB/s, 16 objects/s recovering 2023-09-23T19:23:06.516 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 37 16 54594 54578 92.1833 68.125 0.00877027 0.0108403 2023-09-23T19:23:07.516 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 38 16 55609 55593 91.4267 63.4375 0.0129054 0.0109295 2023-09-23T19:23:08.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:07 smithi016 bash[75004]: audit 2023-09-23T19:23:07.697234+0000 mon.a (mon.0) 585 : audit 0 from='client.? 172.21.15.16:0/3216145463' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:08.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:07 smithi016 bash[77995]: audit 2023-09-23T19:23:07.697234+0000 mon.a (mon.0) 585 : audit 0 from='client.? 172.21.15.16:0/3216145463' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:08.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:07 smithi092 bash[58855]: audit 2023-09-23T19:23:07.697234+0000 mon.a (mon.0) 585 : audit 0 from='client.? 172.21.15.16:0/3216145463' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:08.516 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 39 16 56607 56591 90.6817 62.375 0.00387341 0.0110193 2023-09-23T19:23:09.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:08 smithi092 bash[58855]: cluster 2023-09-23T19:23:07.728665+0000 mgr.y (mgr.25325) 418 : cluster 0 pgmap v340: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s; 11767/158292 objects degraded (7.434%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T19:23:09.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:08 smithi016 bash[77995]: cluster 2023-09-23T19:23:07.728665+0000 mgr.y (mgr.25325) 418 : cluster 0 pgmap v340: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s; 11767/158292 objects degraded (7.434%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T19:23:09.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:08 smithi016 bash[75004]: cluster 2023-09-23T19:23:07.728665+0000 mgr.y (mgr.25325) 418 : cluster 0 pgmap v340: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s; 11767/158292 objects degraded (7.434%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T19:23:09.516 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-09-23T19:23:09.515860+0000 min lat: 0.00119935 max lat: 2.18958 avg lat: 0.0111053 2023-09-23T19:23:09.517 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:23:09.517 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 40 16 57612 57596 89.9848 62.8125 0.0102652 0.0111053 2023-09-23T19:23:10.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:09 smithi016 bash[75004]: cluster 2023-09-23T19:23:09.105922+0000 mon.a (mon.0) 586 : cluster 3 Health check update: Degraded data redundancy: 11767/158292 objects degraded (7.434%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T19:23:10.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:09 smithi016 bash[77995]: cluster 2023-09-23T19:23:09.105922+0000 mon.a (mon.0) 586 : cluster 3 Health check update: Degraded data redundancy: 11767/158292 objects degraded (7.434%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T19:23:10.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:09 smithi092 bash[58855]: cluster 2023-09-23T19:23:09.105922+0000 mon.a (mon.0) 586 : cluster 3 Health check update: Degraded data redundancy: 11767/158292 objects degraded (7.434%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T19:23:10.517 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 41 16 58585 58569 89.2732 60.8125 0.00656459 0.0111928 2023-09-23T19:23:11.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:10 smithi016 bash[75004]: cluster 2023-09-23T19:23:09.729792+0000 mgr.y (mgr.25325) 419 : cluster 0 pgmap v341: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 11767/162261 objects degraded (7.252%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T19:23:11.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:10 smithi016 bash[75004]: audit 2023-09-23T19:23:10.281427+0000 mon.c (mon.2) 68 : audit 0 from='client.? 172.21.15.16:0/139454625' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:11.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:10 smithi016 bash[77995]: cluster 2023-09-23T19:23:09.729792+0000 mgr.y (mgr.25325) 419 : cluster 0 pgmap v341: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 11767/162261 objects degraded (7.252%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T19:23:11.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:10 smithi016 bash[77995]: audit 2023-09-23T19:23:10.281427+0000 mon.c (mon.2) 68 : audit 0 from='client.? 172.21.15.16:0/139454625' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:11.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:10 smithi092 bash[58855]: cluster 2023-09-23T19:23:09.729792+0000 mgr.y (mgr.25325) 419 : cluster 0 pgmap v341: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 11767/162261 objects degraded (7.252%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T19:23:11.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:10 smithi092 bash[58855]: audit 2023-09-23T19:23:10.281427+0000 mon.c (mon.2) 68 : audit 0 from='client.? 172.21.15.16:0/139454625' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:11.517 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 42 16 59665 59649 88.7546 67.5 0.0267965 0.011259 2023-09-23T19:23:12.517 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 43 16 60737 60721 88.2485 67 0.0130348 0.0113245 2023-09-23T19:23:13.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:12 smithi016 bash[75004]: cluster 2023-09-23T19:23:11.730439+0000 mgr.y (mgr.25325) 420 : cluster 0 pgmap v342: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s; 11661/173703 objects degraded (6.713%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T19:23:13.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:12 smithi016 bash[77995]: cluster 2023-09-23T19:23:11.730439+0000 mgr.y (mgr.25325) 420 : cluster 0 pgmap v342: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s; 11661/173703 objects degraded (6.713%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T19:23:13.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:12 smithi092 bash[58855]: cluster 2023-09-23T19:23:11.730439+0000 mgr.y (mgr.25325) 420 : cluster 0 pgmap v342: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s; 11661/173703 objects degraded (6.713%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T19:23:13.517 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 44 16 61765 61749 87.7029 64.25 0.0160523 0.0113951 2023-09-23T19:23:14.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:13 smithi016 bash[75004]: audit 2023-09-23T19:23:12.910531+0000 mon.a (mon.0) 587 : audit 0 from='client.? 172.21.15.16:0/3599656370' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:14.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:13 smithi016 bash[75004]: audit 2023-09-23T19:23:13.624112+0000 mon.a (mon.0) 588 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:23:14.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:13 smithi016 bash[77995]: audit 2023-09-23T19:23:12.910531+0000 mon.a (mon.0) 587 : audit 0 from='client.? 172.21.15.16:0/3599656370' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:14.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:13 smithi016 bash[77995]: audit 2023-09-23T19:23:13.624112+0000 mon.a (mon.0) 588 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:23:14.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:13 smithi092 bash[58855]: audit 2023-09-23T19:23:12.910531+0000 mon.a (mon.0) 587 : audit 0 from='client.? 172.21.15.16:0/3599656370' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:14.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:13 smithi092 bash[58855]: audit 2023-09-23T19:23:13.624112+0000 mon.a (mon.0) 588 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:23:14.517 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 45 16 62790 62774 87.1775 64.0625 0.00710917 0.0114641 2023-09-23T19:23:15.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:14 smithi016 bash[75004]: cluster 2023-09-23T19:23:13.731010+0000 mgr.y (mgr.25325) 421 : cluster 0 pgmap v343: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 62 MiB/s wr, 984 op/s; 11661/177561 objects degraded (6.567%); 1.1 MiB/s, 16 objects/s recovering 2023-09-23T19:23:15.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:14 smithi016 bash[75004]: cluster 2023-09-23T19:23:14.107167+0000 mon.a (mon.0) 589 : cluster 3 Health check update: Degraded data redundancy: 11661/177561 objects degraded (6.567%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T19:23:15.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:14 smithi016 bash[77995]: cluster 2023-09-23T19:23:13.731010+0000 mgr.y (mgr.25325) 421 : cluster 0 pgmap v343: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 62 MiB/s wr, 984 op/s; 11661/177561 objects degraded (6.567%); 1.1 MiB/s, 16 objects/s recovering 2023-09-23T19:23:15.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:14 smithi016 bash[77995]: cluster 2023-09-23T19:23:14.107167+0000 mon.a (mon.0) 589 : cluster 3 Health check update: Degraded data redundancy: 11661/177561 objects degraded (6.567%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T19:23:15.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:14 smithi092 bash[58855]: cluster 2023-09-23T19:23:13.731010+0000 mgr.y (mgr.25325) 421 : cluster 0 pgmap v343: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 62 MiB/s wr, 984 op/s; 11661/177561 objects degraded (6.567%); 1.1 MiB/s, 16 objects/s recovering 2023-09-23T19:23:15.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:14 smithi092 bash[58855]: cluster 2023-09-23T19:23:14.107167+0000 mon.a (mon.0) 589 : cluster 3 Health check update: Degraded data redundancy: 11661/177561 objects degraded (6.567%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T19:23:15.517 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 46 16 63710 63694 86.5322 57.5 0.0161459 0.011549 2023-09-23T19:23:16.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:15 smithi016 bash[75004]: audit 2023-09-23T19:23:15.528631+0000 mon.c (mon.2) 69 : audit 0 from='client.? 172.21.15.16:0/1901436752' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:16.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:23:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:23:15] "GET /metrics HTTP/1.1" 200 36200 "" "Prometheus/2.33.4" 2023-09-23T19:23:16.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:15 smithi016 bash[77995]: audit 2023-09-23T19:23:15.528631+0000 mon.c (mon.2) 69 : audit 0 from='client.? 172.21.15.16:0/1901436752' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:16.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:15 smithi092 bash[58855]: audit 2023-09-23T19:23:15.528631+0000 mon.c (mon.2) 69 : audit 0 from='client.? 172.21.15.16:0/1901436752' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:16.517 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 47 16 64682 64666 85.9835 60.75 0.0245287 0.0116231 2023-09-23T19:23:17.283 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:16 smithi016 bash[77995]: cluster 2023-09-23T19:23:15.732183+0000 mgr.y (mgr.25325) 422 : cluster 0 pgmap v344: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s; 11661/185226 objects degraded (6.296%); 1.1 MiB/s, 16 objects/s recovering 2023-09-23T19:23:17.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:16 smithi016 bash[75004]: cluster 2023-09-23T19:23:15.732183+0000 mgr.y (mgr.25325) 422 : cluster 0 pgmap v344: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s; 11661/185226 objects degraded (6.296%); 1.1 MiB/s, 16 objects/s recovering 2023-09-23T19:23:17.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:16 smithi092 bash[58855]: cluster 2023-09-23T19:23:15.732183+0000 mgr.y (mgr.25325) 422 : cluster 0 pgmap v344: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s; 11661/185226 objects degraded (6.296%); 1.1 MiB/s, 16 objects/s recovering 2023-09-23T19:23:17.517 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 48 16 65654 65638 85.4577 60.75 0.0104619 0.0116948 2023-09-23T19:23:18.517 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 49 16 66549 66533 84.8551 55.9375 0.011984 0.011778 2023-09-23T19:23:18.822 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:18 smithi016 bash[75004]: cluster 2023-09-23T19:23:17.733056+0000 mgr.y (mgr.25325) 423 : cluster 0 pgmap v345: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 60 MiB/s wr, 961 op/s; 11551/188979 objects degraded (6.112%); 1.6 MiB/s, 26 objects/s recovering 2023-09-23T19:23:18.823 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:18 smithi016 bash[75004]: audit 2023-09-23T19:23:18.122744+0000 mon.a (mon.0) 590 : audit 0 from='client.? 172.21.15.16:0/2300635642' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:18.823 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:18 smithi016 bash[75004]: audit 2023-09-23T19:23:18.537603+0000 mon.a (mon.0) 591 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:23:18.823 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:23:18 smithi016 bash[70992]: debug 2023-09-23T19:23:18.532+0000 7fb2a5a7c700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:23:18.823 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:18 smithi016 bash[77995]: cluster 2023-09-23T19:23:17.733056+0000 mgr.y (mgr.25325) 423 : cluster 0 pgmap v345: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 60 MiB/s wr, 961 op/s; 11551/188979 objects degraded (6.112%); 1.6 MiB/s, 26 objects/s recovering 2023-09-23T19:23:18.823 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:18 smithi016 bash[77995]: audit 2023-09-23T19:23:18.122744+0000 mon.a (mon.0) 590 : audit 0 from='client.? 172.21.15.16:0/2300635642' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:18.823 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:18 smithi016 bash[77995]: audit 2023-09-23T19:23:18.537603+0000 mon.a (mon.0) 591 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:23:19.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:18 smithi092 bash[58855]: cluster 2023-09-23T19:23:17.733056+0000 mgr.y (mgr.25325) 423 : cluster 0 pgmap v345: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 60 MiB/s wr, 961 op/s; 11551/188979 objects degraded (6.112%); 1.6 MiB/s, 26 objects/s recovering 2023-09-23T19:23:19.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:18 smithi092 bash[58855]: audit 2023-09-23T19:23:18.122744+0000 mon.a (mon.0) 590 : audit 0 from='client.? 172.21.15.16:0/2300635642' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:19.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:18 smithi092 bash[58855]: audit 2023-09-23T19:23:18.537603+0000 mon.a (mon.0) 591 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:23:19.517 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 50 16 67501 67485 84.3479 59.5 0.00487166 0.0118485 2023-09-23T19:23:20.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:19 smithi016 bash[75004]: audit 2023-09-23T19:23:18.537999+0000 mgr.y (mgr.25325) 424 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:23:20.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:19 smithi016 bash[75004]: cephadm 2023-09-23T19:23:18.541940+0000 mgr.y (mgr.25325) 425 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:23:20.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:19 smithi016 bash[75004]: cluster 2023-09-23T19:23:19.108103+0000 mon.a (mon.0) 592 : cluster 3 Health check update: Degraded data redundancy: 11551/188979 objects degraded (6.112%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T19:23:20.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:19 smithi016 bash[77995]: audit 2023-09-23T19:23:18.537999+0000 mgr.y (mgr.25325) 424 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:23:20.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:19 smithi016 bash[77995]: cephadm 2023-09-23T19:23:18.541940+0000 mgr.y (mgr.25325) 425 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:23:20.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:19 smithi016 bash[77995]: cluster 2023-09-23T19:23:19.108103+0000 mon.a (mon.0) 592 : cluster 3 Health check update: Degraded data redundancy: 11551/188979 objects degraded (6.112%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T19:23:20.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:19 smithi092 bash[58855]: audit 2023-09-23T19:23:18.537999+0000 mgr.y (mgr.25325) 424 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:23:20.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:19 smithi092 bash[58855]: cephadm 2023-09-23T19:23:18.541940+0000 mgr.y (mgr.25325) 425 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:23:20.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:19 smithi092 bash[58855]: cluster 2023-09-23T19:23:19.108103+0000 mon.a (mon.0) 592 : cluster 3 Health check update: Degraded data redundancy: 11551/188979 objects degraded (6.112%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T19:23:20.518 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 51 16 68408 68392 83.8054 56.6875 0.0195507 0.0119248 2023-09-23T19:23:21.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:20 smithi016 bash[75004]: cluster 2023-09-23T19:23:19.733667+0000 mgr.y (mgr.25325) 426 : cluster 0 pgmap v346: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 60 MiB/s wr, 957 op/s; 11551/192768 objects degraded (5.992%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:23:21.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:20 smithi016 bash[75004]: audit 2023-09-23T19:23:20.741702+0000 mon.a (mon.0) 593 : audit 0 from='client.? 172.21.15.16:0/3697143185' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:21.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:20 smithi016 bash[77995]: cluster 2023-09-23T19:23:19.733667+0000 mgr.y (mgr.25325) 426 : cluster 0 pgmap v346: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 60 MiB/s wr, 957 op/s; 11551/192768 objects degraded (5.992%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:23:21.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:20 smithi016 bash[77995]: audit 2023-09-23T19:23:20.741702+0000 mon.a (mon.0) 593 : audit 0 from='client.? 172.21.15.16:0/3697143185' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:21.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:20 smithi092 bash[58855]: cluster 2023-09-23T19:23:19.733667+0000 mgr.y (mgr.25325) 426 : cluster 0 pgmap v346: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 60 MiB/s wr, 957 op/s; 11551/192768 objects degraded (5.992%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:23:21.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:20 smithi092 bash[58855]: audit 2023-09-23T19:23:20.741702+0000 mon.a (mon.0) 593 : audit 0 from='client.? 172.21.15.16:0/3697143185' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:21.518 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 52 16 69312 69296 83.2802 56.5 0.0105638 0.0119976 2023-09-23T19:23:22.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:21 smithi016 bash[75004]: cluster 2023-09-23T19:23:21.734527+0000 mgr.y (mgr.25325) 427 : cluster 0 pgmap v347: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s; 11447/203376 objects degraded (5.628%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T19:23:22.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:21 smithi016 bash[77995]: cluster 2023-09-23T19:23:21.734527+0000 mgr.y (mgr.25325) 427 : cluster 0 pgmap v347: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s; 11447/203376 objects degraded (5.628%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T19:23:22.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:21 smithi092 bash[58855]: cluster 2023-09-23T19:23:21.734527+0000 mgr.y (mgr.25325) 427 : cluster 0 pgmap v347: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s; 11447/203376 objects degraded (5.628%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T19:23:22.518 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 53 16 70247 70231 82.8114 58.4375 0.0285509 0.0120687 2023-09-23T19:23:23.518 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 54 16 71099 71083 82.2639 53.25 0.0085335 0.0121493 2023-09-23T19:23:23.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:23 smithi016 bash[75004]: audit 2023-09-23T19:23:23.381772+0000 mon.b (mon.1) 26 : audit 0 from='client.? 172.21.15.16:0/3968660781' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:23.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:23 smithi016 bash[77995]: audit 2023-09-23T19:23:23.381772+0000 mon.b (mon.1) 26 : audit 0 from='client.? 172.21.15.16:0/3968660781' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:23.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:23 smithi092 bash[58855]: audit 2023-09-23T19:23:23.381772+0000 mon.b (mon.1) 26 : audit 0 from='client.? 172.21.15.16:0/3968660781' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:24.518 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 55 16 72029 72013 81.8249 58.125 0.0367631 0.0122135 2023-09-23T19:23:24.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:24 smithi016 bash[77995]: cluster 2023-09-23T19:23:23.735194+0000 mgr.y (mgr.25325) 428 : cluster 0 pgmap v348: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 57 MiB/s wr, 919 op/s; 11447/206826 objects degraded (5.535%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:23:24.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:24 smithi016 bash[77995]: cluster 2023-09-23T19:23:24.109365+0000 mon.a (mon.0) 594 : cluster 3 Health check update: Degraded data redundancy: 11447/203376 objects degraded (5.628%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T19:23:24.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:24 smithi016 bash[75004]: cluster 2023-09-23T19:23:23.735194+0000 mgr.y (mgr.25325) 428 : cluster 0 pgmap v348: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 57 MiB/s wr, 919 op/s; 11447/206826 objects degraded (5.535%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:23:24.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:24 smithi016 bash[75004]: cluster 2023-09-23T19:23:24.109365+0000 mon.a (mon.0) 594 : cluster 3 Health check update: Degraded data redundancy: 11447/203376 objects degraded (5.628%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T19:23:24.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:24 smithi092 bash[58855]: cluster 2023-09-23T19:23:23.735194+0000 mgr.y (mgr.25325) 428 : cluster 0 pgmap v348: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 57 MiB/s wr, 919 op/s; 11447/206826 objects degraded (5.535%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:23:24.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:24 smithi092 bash[58855]: cluster 2023-09-23T19:23:24.109365+0000 mon.a (mon.0) 594 : cluster 3 Health check update: Degraded data redundancy: 11447/203376 objects degraded (5.628%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T19:23:25.518 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 56 16 72941 72925 81.3815 57 0.00784551 0.0122805 2023-09-23T19:23:26.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:23:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:23:25] "GET /metrics HTTP/1.1" 200 36200 "" "Prometheus/2.33.4" 2023-09-23T19:23:26.519 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 57 16 73791 73775 80.8857 53.125 0.0187065 0.0123557 2023-09-23T19:23:27.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:26 smithi016 bash[77995]: cluster 2023-09-23T19:23:25.735947+0000 mgr.y (mgr.25325) 429 : cluster 0 pgmap v349: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 62 MiB/s wr, 998 op/s; 11447/213531 objects degraded (5.361%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:23:27.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:26 smithi016 bash[77995]: audit 2023-09-23T19:23:25.970314+0000 mon.b (mon.1) 27 : audit 0 from='client.? 172.21.15.16:0/3312967897' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:27.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:26 smithi016 bash[75004]: cluster 2023-09-23T19:23:25.735947+0000 mgr.y (mgr.25325) 429 : cluster 0 pgmap v349: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 62 MiB/s wr, 998 op/s; 11447/213531 objects degraded (5.361%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:23:27.288 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:26 smithi016 bash[75004]: audit 2023-09-23T19:23:25.970314+0000 mon.b (mon.1) 27 : audit 0 from='client.? 172.21.15.16:0/3312967897' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:27.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:26 smithi092 bash[58855]: cluster 2023-09-23T19:23:25.735947+0000 mgr.y (mgr.25325) 429 : cluster 0 pgmap v349: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 62 MiB/s wr, 998 op/s; 11447/213531 objects degraded (5.361%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:23:27.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:26 smithi092 bash[58855]: audit 2023-09-23T19:23:25.970314+0000 mon.b (mon.1) 27 : audit 0 from='client.? 172.21.15.16:0/3312967897' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:27.518 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 58 16 74701 74685 80.4716 56.875 0.0195534 0.0124196 2023-09-23T19:23:28.519 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 59 16 75550 75534 80.0069 53.0625 0.0111611 0.0124915 2023-09-23T19:23:29.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:28 smithi016 bash[77995]: cluster 2023-09-23T19:23:27.736612+0000 mgr.y (mgr.25325) 430 : cluster 0 pgmap v350: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 55 MiB/s wr, 878 op/s; 11343/216852 objects degraded (5.231%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T19:23:29.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:28 smithi016 bash[77995]: audit 2023-09-23T19:23:28.590769+0000 mon.a (mon.0) 595 : audit 0 from='client.? 172.21.15.16:0/270408634' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:29.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:28 smithi016 bash[77995]: audit 2023-09-23T19:23:28.624568+0000 mon.a (mon.0) 596 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:23:29.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:28 smithi016 bash[75004]: cluster 2023-09-23T19:23:27.736612+0000 mgr.y (mgr.25325) 430 : cluster 0 pgmap v350: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 55 MiB/s wr, 878 op/s; 11343/216852 objects degraded (5.231%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T19:23:29.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:28 smithi016 bash[75004]: audit 2023-09-23T19:23:28.590769+0000 mon.a (mon.0) 595 : audit 0 from='client.? 172.21.15.16:0/270408634' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:29.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:28 smithi016 bash[75004]: audit 2023-09-23T19:23:28.624568+0000 mon.a (mon.0) 596 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:23:29.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:28 smithi092 bash[58855]: cluster 2023-09-23T19:23:27.736612+0000 mgr.y (mgr.25325) 430 : cluster 0 pgmap v350: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 55 MiB/s wr, 878 op/s; 11343/216852 objects degraded (5.231%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T19:23:29.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:28 smithi092 bash[58855]: audit 2023-09-23T19:23:28.590769+0000 mon.a (mon.0) 595 : audit 0 from='client.? 172.21.15.16:0/270408634' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:29.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:28 smithi092 bash[58855]: audit 2023-09-23T19:23:28.624568+0000 mon.a (mon.0) 596 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:23:29.518 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-09-23T19:23:29.517822+0000 min lat: 0.00119935 max lat: 2.18958 avg lat: 0.0125559 2023-09-23T19:23:29.519 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:23:29.519 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 60 16 76432 76416 79.5921 55.125 0.0606207 0.0125559 2023-09-23T19:23:30.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:29 smithi016 bash[77995]: cluster 2023-09-23T19:23:29.111112+0000 mon.a (mon.0) 597 : cluster 3 Health check update: Degraded data redundancy: 11343/216852 objects degraded (5.231%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T19:23:30.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:29 smithi016 bash[75004]: cluster 2023-09-23T19:23:29.111112+0000 mon.a (mon.0) 597 : cluster 3 Health check update: Degraded data redundancy: 11343/216852 objects degraded (5.231%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T19:23:30.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:29 smithi092 bash[58855]: cluster 2023-09-23T19:23:29.111112+0000 mon.a (mon.0) 597 : cluster 3 Health check update: Degraded data redundancy: 11343/216852 objects degraded (5.231%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T19:23:30.518 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 61 16 77329 77313 79.2063 56.0625 0.0227314 0.0126186 2023-09-23T19:23:31.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:30 smithi016 bash[75004]: cluster 2023-09-23T19:23:29.737282+0000 mgr.y (mgr.25325) 431 : cluster 0 pgmap v351: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 54 MiB/s wr, 867 op/s; 11343/220212 objects degraded (5.151%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:23:31.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:30 smithi016 bash[77995]: cluster 2023-09-23T19:23:29.737282+0000 mgr.y (mgr.25325) 431 : cluster 0 pgmap v351: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 54 MiB/s wr, 867 op/s; 11343/220212 objects degraded (5.151%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:23:31.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:30 smithi092 bash[58855]: cluster 2023-09-23T19:23:29.737282+0000 mgr.y (mgr.25325) 431 : cluster 0 pgmap v351: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 54 MiB/s wr, 867 op/s; 11343/220212 objects degraded (5.151%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:23:31.519 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 62 16 78149 78133 78.7553 51.25 0.00846126 0.0126904 2023-09-23T19:23:32.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:31 smithi016 bash[77995]: audit 2023-09-23T19:23:31.171971+0000 mon.b (mon.1) 28 : audit 0 from='client.? 172.21.15.16:0/2670636242' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:32.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:31 smithi016 bash[75004]: audit 2023-09-23T19:23:31.171971+0000 mon.b (mon.1) 28 : audit 0 from='client.? 172.21.15.16:0/2670636242' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:32.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:31 smithi092 bash[58855]: audit 2023-09-23T19:23:31.171971+0000 mon.b (mon.1) 28 : audit 0 from='client.? 172.21.15.16:0/2670636242' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:32.519 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 63 16 79052 79036 78.401 56.4375 0.0323925 0.0127486 2023-09-23T19:23:33.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:32 smithi016 bash[75004]: cluster 2023-09-23T19:23:31.738021+0000 mgr.y (mgr.25325) 432 : cluster 0 pgmap v352: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 11240/230208 objects degraded (4.883%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T19:23:33.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:32 smithi016 bash[77995]: cluster 2023-09-23T19:23:31.738021+0000 mgr.y (mgr.25325) 432 : cluster 0 pgmap v352: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 11240/230208 objects degraded (4.883%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T19:23:33.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:32 smithi092 bash[58855]: cluster 2023-09-23T19:23:31.738021+0000 mgr.y (mgr.25325) 432 : cluster 0 pgmap v352: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 11240/230208 objects degraded (4.883%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T19:23:33.519 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 64 16 79892 79876 77.9962 52.5 0.00902153 0.0128141 2023-09-23T19:23:33.830 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:33 smithi016 bash[75004]: audit 2023-09-23T19:23:33.557157+0000 mon.a (mon.0) 598 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:23:33.830 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:33 smithi016 bash[75004]: audit 2023-09-23T19:23:33.776447+0000 mon.c (mon.2) 70 : audit 0 from='client.? 172.21.15.16:0/405290390' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:33.831 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:23:33 smithi016 bash[70992]: debug 2023-09-23T19:23:33.552+0000 7fb2a5a7c700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:23:33.831 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:33 smithi016 bash[77995]: audit 2023-09-23T19:23:33.557157+0000 mon.a (mon.0) 598 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:23:33.831 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:33 smithi016 bash[77995]: audit 2023-09-23T19:23:33.776447+0000 mon.c (mon.2) 70 : audit 0 from='client.? 172.21.15.16:0/405290390' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:34.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:33 smithi092 bash[58855]: audit 2023-09-23T19:23:33.557157+0000 mon.a (mon.0) 598 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:23:34.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:33 smithi092 bash[58855]: audit 2023-09-23T19:23:33.776447+0000 mon.c (mon.2) 70 : audit 0 from='client.? 172.21.15.16:0/405290390' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:34.519 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 65 16 80731 80715 77.6029 52.4375 0.0201467 0.0128785 2023-09-23T19:23:35.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:34 smithi016 bash[77995]: audit 2023-09-23T19:23:33.557505+0000 mgr.y (mgr.25325) 433 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:23:35.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:34 smithi016 bash[77995]: cephadm 2023-09-23T19:23:33.559285+0000 mgr.y (mgr.25325) 434 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:23:35.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:34 smithi016 bash[77995]: cluster 2023-09-23T19:23:33.738642+0000 mgr.y (mgr.25325) 435 : cluster 0 pgmap v353: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 52 MiB/s wr, 832 op/s; 11240/233367 objects degraded (4.816%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:23:35.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:34 smithi016 bash[77995]: cluster 2023-09-23T19:23:34.113055+0000 mon.a (mon.0) 599 : cluster 3 Health check update: Degraded data redundancy: 11240/233367 objects degraded (4.816%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T19:23:35.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:34 smithi016 bash[75004]: audit 2023-09-23T19:23:33.557505+0000 mgr.y (mgr.25325) 433 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:23:35.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:34 smithi016 bash[75004]: cephadm 2023-09-23T19:23:33.559285+0000 mgr.y (mgr.25325) 434 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:23:35.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:34 smithi016 bash[75004]: cluster 2023-09-23T19:23:33.738642+0000 mgr.y (mgr.25325) 435 : cluster 0 pgmap v353: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 52 MiB/s wr, 832 op/s; 11240/233367 objects degraded (4.816%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:23:35.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:34 smithi016 bash[75004]: cluster 2023-09-23T19:23:34.113055+0000 mon.a (mon.0) 599 : cluster 3 Health check update: Degraded data redundancy: 11240/233367 objects degraded (4.816%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T19:23:35.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:34 smithi092 bash[58855]: audit 2023-09-23T19:23:33.557505+0000 mgr.y (mgr.25325) 433 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:23:35.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:34 smithi092 bash[58855]: cephadm 2023-09-23T19:23:33.559285+0000 mgr.y (mgr.25325) 434 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:23:35.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:34 smithi092 bash[58855]: cluster 2023-09-23T19:23:33.738642+0000 mgr.y (mgr.25325) 435 : cluster 0 pgmap v353: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 52 MiB/s wr, 832 op/s; 11240/233367 objects degraded (4.816%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:23:35.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:34 smithi092 bash[58855]: cluster 2023-09-23T19:23:34.113055+0000 mon.a (mon.0) 599 : cluster 3 Health check update: Degraded data redundancy: 11240/233367 objects degraded (4.816%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T19:23:35.519 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 66 16 81594 81578 77.2443 53.9375 0.0257984 0.0129391 2023-09-23T19:23:36.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:35 smithi016 bash[75004]: cluster 2023-09-23T19:23:35.739479+0000 mgr.y (mgr.25325) 436 : cluster 0 pgmap v354: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s; 11240/239883 objects degraded (4.686%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:23:36.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:23:35 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:23:35] "GET /metrics HTTP/1.1" 200 36197 "" "Prometheus/2.33.4" 2023-09-23T19:23:36.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:35 smithi016 bash[77995]: cluster 2023-09-23T19:23:35.739479+0000 mgr.y (mgr.25325) 436 : cluster 0 pgmap v354: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s; 11240/239883 objects degraded (4.686%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:23:36.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:35 smithi092 bash[58855]: cluster 2023-09-23T19:23:35.739479+0000 mgr.y (mgr.25325) 436 : cluster 0 pgmap v354: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s; 11240/239883 objects degraded (4.686%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:23:36.519 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 67 16 82383 82367 76.8273 49.3125 0.00963649 0.0130089 2023-09-23T19:23:37.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:36 smithi016 bash[75004]: audit 2023-09-23T19:23:36.381297+0000 mon.c (mon.2) 71 : audit 0 from='client.? 172.21.15.16:0/68823736' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:37.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:36 smithi016 bash[77995]: audit 2023-09-23T19:23:36.381297+0000 mon.c (mon.2) 71 : audit 0 from='client.? 172.21.15.16:0/68823736' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:37.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:36 smithi092 bash[58855]: audit 2023-09-23T19:23:36.381297+0000 mon.c (mon.2) 71 : audit 0 from='client.? 172.21.15.16:0/68823736' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:37.519 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 68 16 83255 83239 76.4989 54.5 0.00576422 0.0130645 2023-09-23T19:23:38.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:37 smithi016 bash[75004]: cluster 2023-09-23T19:23:37.740152+0000 mgr.y (mgr.25325) 437 : cluster 0 pgmap v355: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 51 MiB/s wr, 822 op/s; 11142/243138 objects degraded (4.583%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T19:23:38.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:37 smithi016 bash[77995]: cluster 2023-09-23T19:23:37.740152+0000 mgr.y (mgr.25325) 437 : cluster 0 pgmap v355: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 51 MiB/s wr, 822 op/s; 11142/243138 objects degraded (4.583%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T19:23:38.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:37 smithi092 bash[58855]: cluster 2023-09-23T19:23:37.740152+0000 mgr.y (mgr.25325) 437 : cluster 0 pgmap v355: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 51 MiB/s wr, 822 op/s; 11142/243138 objects degraded (4.583%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T19:23:38.519 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 69 16 84131 84115 76.1836 54.75 0.0143162 0.0131189 2023-09-23T19:23:39.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:39 smithi016 bash[75004]: audit 2023-09-23T19:23:38.954595+0000 mon.c (mon.2) 72 : audit 0 from='client.? 172.21.15.16:0/2515198063' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:39.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:39 smithi016 bash[77995]: audit 2023-09-23T19:23:38.954595+0000 mon.c (mon.2) 72 : audit 0 from='client.? 172.21.15.16:0/2515198063' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:39.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:39 smithi092 bash[58855]: audit 2023-09-23T19:23:38.954595+0000 mon.c (mon.2) 72 : audit 0 from='client.? 172.21.15.16:0/2515198063' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:39.519 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 70 16 84933 84917 75.8113 50.125 0.00587213 0.0131833 2023-09-23T19:23:40.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:40 smithi016 bash[75004]: cluster 2023-09-23T19:23:39.114180+0000 mon.a (mon.0) 600 : cluster 3 Health check update: Degraded data redundancy: 11142/243138 objects degraded (4.583%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T19:23:40.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:40 smithi016 bash[75004]: cluster 2023-09-23T19:23:39.740830+0000 mgr.y (mgr.25325) 438 : cluster 0 pgmap v356: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 51 MiB/s wr, 815 op/s; 11142/246231 objects degraded (4.525%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T19:23:40.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:40 smithi016 bash[77995]: cluster 2023-09-23T19:23:39.114180+0000 mon.a (mon.0) 600 : cluster 3 Health check update: Degraded data redundancy: 11142/243138 objects degraded (4.583%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T19:23:40.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:40 smithi016 bash[77995]: cluster 2023-09-23T19:23:39.740830+0000 mgr.y (mgr.25325) 438 : cluster 0 pgmap v356: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 51 MiB/s wr, 815 op/s; 11142/246231 objects degraded (4.525%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T19:23:40.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:40 smithi092 bash[58855]: cluster 2023-09-23T19:23:39.114180+0000 mon.a (mon.0) 600 : cluster 3 Health check update: Degraded data redundancy: 11142/243138 objects degraded (4.583%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T19:23:40.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:40 smithi092 bash[58855]: cluster 2023-09-23T19:23:39.740830+0000 mgr.y (mgr.25325) 438 : cluster 0 pgmap v356: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 51 MiB/s wr, 815 op/s; 11142/246231 objects degraded (4.525%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T19:23:40.520 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 71 16 85766 85750 75.4767 52.0625 0.0242433 0.0132425 2023-09-23T19:23:41.520 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 72 16 86582 86566 75.1367 51 0.0444313 0.0133017 2023-09-23T19:23:42.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:41 smithi016 bash[75004]: audit 2023-09-23T19:23:41.560985+0000 mon.c (mon.2) 73 : audit 0 from='client.? 172.21.15.16:0/2629593102' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:42.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:41 smithi016 bash[77995]: audit 2023-09-23T19:23:41.560985+0000 mon.c (mon.2) 73 : audit 0 from='client.? 172.21.15.16:0/2629593102' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:42.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:41 smithi092 bash[58855]: audit 2023-09-23T19:23:41.560985+0000 mon.c (mon.2) 73 : audit 0 from='client.? 172.21.15.16:0/2629593102' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:42.520 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 73 16 87465 87449 74.8633 55.1875 0.0262248 0.0133514 2023-09-23T19:23:43.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:42 smithi016 bash[75004]: cluster 2023-09-23T19:23:41.741679+0000 mgr.y (mgr.25325) 439 : cluster 0 pgmap v357: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 983 op/s; 11034/255636 objects degraded (4.316%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T19:23:43.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:42 smithi016 bash[77995]: cluster 2023-09-23T19:23:41.741679+0000 mgr.y (mgr.25325) 439 : cluster 0 pgmap v357: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 983 op/s; 11034/255636 objects degraded (4.316%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T19:23:43.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:42 smithi092 bash[58855]: cluster 2023-09-23T19:23:41.741679+0000 mgr.y (mgr.25325) 439 : cluster 0 pgmap v357: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 983 op/s; 11034/255636 objects degraded (4.316%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T19:23:43.520 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 74 16 88350 88334 74.5991 55.3125 0.00684751 0.0133976 2023-09-23T19:23:44.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:43 smithi016 bash[75004]: audit 2023-09-23T19:23:43.624928+0000 mon.a (mon.0) 601 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:23:44.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:43 smithi016 bash[77995]: audit 2023-09-23T19:23:43.624928+0000 mon.a (mon.0) 601 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:23:44.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:43 smithi092 bash[58855]: audit 2023-09-23T19:23:43.624928+0000 mon.a (mon.0) 601 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:23:44.520 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 75 16 89125 89109 74.2502 48.4375 0.0181252 0.0134618 2023-09-23T19:23:45.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:44 smithi016 bash[77995]: cluster 2023-09-23T19:23:43.742332+0000 mgr.y (mgr.25325) 440 : cluster 0 pgmap v358: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 49 MiB/s wr, 791 op/s; 11034/258711 objects degraded (4.265%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:23:45.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:44 smithi016 bash[77995]: cluster 2023-09-23T19:23:44.115383+0000 mon.a (mon.0) 602 : cluster 3 Health check update: Degraded data redundancy: 11034/255636 objects degraded (4.316%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T19:23:45.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:44 smithi016 bash[77995]: audit 2023-09-23T19:23:44.143854+0000 mon.a (mon.0) 603 : audit 0 from='client.? 172.21.15.16:0/3466676186' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:45.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:44 smithi016 bash[75004]: cluster 2023-09-23T19:23:43.742332+0000 mgr.y (mgr.25325) 440 : cluster 0 pgmap v358: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 49 MiB/s wr, 791 op/s; 11034/258711 objects degraded (4.265%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:23:45.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:44 smithi016 bash[75004]: cluster 2023-09-23T19:23:44.115383+0000 mon.a (mon.0) 602 : cluster 3 Health check update: Degraded data redundancy: 11034/255636 objects degraded (4.316%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T19:23:45.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:44 smithi016 bash[75004]: audit 2023-09-23T19:23:44.143854+0000 mon.a (mon.0) 603 : audit 0 from='client.? 172.21.15.16:0/3466676186' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:45.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:44 smithi092 bash[58855]: cluster 2023-09-23T19:23:43.742332+0000 mgr.y (mgr.25325) 440 : cluster 0 pgmap v358: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 49 MiB/s wr, 791 op/s; 11034/258711 objects degraded (4.265%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:23:45.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:44 smithi092 bash[58855]: cluster 2023-09-23T19:23:44.115383+0000 mon.a (mon.0) 602 : cluster 3 Health check update: Degraded data redundancy: 11034/255636 objects degraded (4.316%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T19:23:45.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:44 smithi092 bash[58855]: audit 2023-09-23T19:23:44.143854+0000 mon.a (mon.0) 603 : audit 0 from='client.? 172.21.15.16:0/3466676186' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:45.520 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 76 16 89956 89940 73.9565 51.9375 0.0283501 0.0135148 2023-09-23T19:23:46.033 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:23:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:23:45] "GET /metrics HTTP/1.1" 200 36196 "" "Prometheus/2.33.4" 2023-09-23T19:23:46.520 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 77 16 90749 90733 73.6397 49.5625 0.0217148 0.0135709 2023-09-23T19:23:47.283 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:46 smithi016 bash[77995]: cluster 2023-09-23T19:23:45.743261+0000 mgr.y (mgr.25325) 441 : cluster 0 pgmap v359: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 881 op/s; 11034/265110 objects degraded (4.162%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:23:47.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:46 smithi016 bash[77995]: audit 2023-09-23T19:23:46.732908+0000 mon.c (mon.2) 74 : audit 0 from='client.? 172.21.15.16:0/4234446557' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:47.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:46 smithi016 bash[75004]: cluster 2023-09-23T19:23:45.743261+0000 mgr.y (mgr.25325) 441 : cluster 0 pgmap v359: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 881 op/s; 11034/265110 objects degraded (4.162%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:23:47.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:46 smithi016 bash[75004]: audit 2023-09-23T19:23:46.732908+0000 mon.c (mon.2) 74 : audit 0 from='client.? 172.21.15.16:0/4234446557' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:47.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:46 smithi092 bash[58855]: cluster 2023-09-23T19:23:45.743261+0000 mgr.y (mgr.25325) 441 : cluster 0 pgmap v359: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 881 op/s; 11034/265110 objects degraded (4.162%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:23:47.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:46 smithi092 bash[58855]: audit 2023-09-23T19:23:46.732908+0000 mon.c (mon.2) 74 : audit 0 from='client.? 172.21.15.16:0/4234446557' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:47.520 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 78 16 91539 91523 73.3285 49.375 0.0164167 0.0136306 2023-09-23T19:23:48.520 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 79 16 92344 92328 73.0371 50.3125 0.0145654 0.0136846 2023-09-23T19:23:49.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:48 smithi016 bash[75004]: cluster 2023-09-23T19:23:47.743858+0000 mgr.y (mgr.25325) 442 : cluster 0 pgmap v360: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 49 MiB/s wr, 784 op/s; 10924/268152 objects degraded (4.074%); 1.6 MiB/s, 26 objects/s recovering 2023-09-23T19:23:49.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:48 smithi016 bash[75004]: audit 2023-09-23T19:23:48.573265+0000 mon.a (mon.0) 604 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:23:49.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:48 smithi016 bash[77995]: cluster 2023-09-23T19:23:47.743858+0000 mgr.y (mgr.25325) 442 : cluster 0 pgmap v360: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 49 MiB/s wr, 784 op/s; 10924/268152 objects degraded (4.074%); 1.6 MiB/s, 26 objects/s recovering 2023-09-23T19:23:49.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:48 smithi016 bash[77995]: audit 2023-09-23T19:23:48.573265+0000 mon.a (mon.0) 604 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:23:49.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:48 smithi092 bash[58855]: cluster 2023-09-23T19:23:47.743858+0000 mgr.y (mgr.25325) 442 : cluster 0 pgmap v360: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 49 MiB/s wr, 784 op/s; 10924/268152 objects degraded (4.074%); 1.6 MiB/s, 26 objects/s recovering 2023-09-23T19:23:49.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:48 smithi092 bash[58855]: audit 2023-09-23T19:23:48.573265+0000 mon.a (mon.0) 604 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:23:49.520 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-09-23T19:23:49.519784+0000 min lat: 0.00119935 max lat: 2.18958 avg lat: 0.0137519 2023-09-23T19:23:49.521 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:23:49.521 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 80 16 93057 93041 72.6811 44.5625 0.0156753 0.0137519 2023-09-23T19:23:50.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:49 smithi016 bash[77995]: cluster 2023-09-23T19:23:49.117369+0000 mon.a (mon.0) 605 : cluster 3 Health check update: Degraded data redundancy: 10924/268152 objects degraded (4.074%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:23:50.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:49 smithi016 bash[77995]: audit 2023-09-23T19:23:49.364862+0000 mon.a (mon.0) 606 : audit 0 from='client.? 172.21.15.16:0/1804413039' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:50.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:49 smithi016 bash[75004]: cluster 2023-09-23T19:23:49.117369+0000 mon.a (mon.0) 605 : cluster 3 Health check update: Degraded data redundancy: 10924/268152 objects degraded (4.074%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:23:50.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:49 smithi016 bash[75004]: audit 2023-09-23T19:23:49.364862+0000 mon.a (mon.0) 606 : audit 0 from='client.? 172.21.15.16:0/1804413039' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:50.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:49 smithi092 bash[58855]: cluster 2023-09-23T19:23:49.117369+0000 mon.a (mon.0) 605 : cluster 3 Health check update: Degraded data redundancy: 10924/268152 objects degraded (4.074%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:23:50.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:49 smithi092 bash[58855]: audit 2023-09-23T19:23:49.364862+0000 mon.a (mon.0) 606 : audit 0 from='client.? 172.21.15.16:0/1804413039' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:50.521 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 81 16 93815 93799 72.3686 47.375 0.021953 0.0138116 2023-09-23T19:23:51.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:50 smithi016 bash[75004]: cluster 2023-09-23T19:23:49.744505+0000 mgr.y (mgr.25325) 443 : cluster 0 pgmap v361: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 49 MiB/s wr, 778 op/s; 10924/271182 objects degraded (4.028%); 1.1 MiB/s, 18 objects/s recovering 2023-09-23T19:23:51.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:50 smithi016 bash[77995]: cluster 2023-09-23T19:23:49.744505+0000 mgr.y (mgr.25325) 443 : cluster 0 pgmap v361: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 49 MiB/s wr, 778 op/s; 10924/271182 objects degraded (4.028%); 1.1 MiB/s, 18 objects/s recovering 2023-09-23T19:23:51.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:50 smithi092 bash[58855]: cluster 2023-09-23T19:23:49.744505+0000 mgr.y (mgr.25325) 443 : cluster 0 pgmap v361: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 49 MiB/s wr, 778 op/s; 10924/271182 objects degraded (4.028%); 1.1 MiB/s, 18 objects/s recovering 2023-09-23T19:23:51.521 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 82 16 94593 94577 72.079 48.625 0.0270541 0.013866 2023-09-23T19:23:52.521 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 83 16 95328 95312 71.764 45.9375 0.00904099 0.0139276 2023-09-23T19:23:53.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:52 smithi016 bash[75004]: cluster 2023-09-23T19:23:51.745504+0000 mgr.y (mgr.25325) 444 : cluster 0 pgmap v362: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 59 MiB/s wr, 939 op/s; 10819/280074 objects degraded (3.863%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T19:23:53.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:52 smithi016 bash[75004]: audit 2023-09-23T19:23:51.973681+0000 mon.a (mon.0) 607 : audit 0 from='client.? 172.21.15.16:0/2340249767' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:53.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:52 smithi016 bash[77995]: cluster 2023-09-23T19:23:51.745504+0000 mgr.y (mgr.25325) 444 : cluster 0 pgmap v362: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 59 MiB/s wr, 939 op/s; 10819/280074 objects degraded (3.863%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T19:23:53.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:52 smithi016 bash[77995]: audit 2023-09-23T19:23:51.973681+0000 mon.a (mon.0) 607 : audit 0 from='client.? 172.21.15.16:0/2340249767' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:53.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:52 smithi092 bash[58855]: cluster 2023-09-23T19:23:51.745504+0000 mgr.y (mgr.25325) 444 : cluster 0 pgmap v362: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 59 MiB/s wr, 939 op/s; 10819/280074 objects degraded (3.863%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T19:23:53.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:52 smithi092 bash[58855]: audit 2023-09-23T19:23:51.973681+0000 mon.a (mon.0) 607 : audit 0 from='client.? 172.21.15.16:0/2340249767' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:53.521 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 84 16 96111 96095 71.4922 48.9375 0.0428795 0.01398 2023-09-23T19:23:54.521 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 85 16 96771 96755 71.1364 41.25 0.00645633 0.0140497 2023-09-23T19:23:55.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:54 smithi016 bash[75004]: cluster 2023-09-23T19:23:53.746441+0000 mgr.y (mgr.25325) 445 : cluster 0 pgmap v363: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 47 MiB/s wr, 755 op/s; 10819/282840 objects degraded (3.825%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:23:55.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:54 smithi016 bash[75004]: cluster 2023-09-23T19:23:54.122032+0000 mon.a (mon.0) 608 : cluster 3 Health check update: Degraded data redundancy: 10819/282840 objects degraded (3.825%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:23:55.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:54 smithi016 bash[75004]: audit 2023-09-23T19:23:54.634922+0000 mon.a (mon.0) 609 : audit 0 from='client.? 172.21.15.16:0/206062756' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:55.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:54 smithi016 bash[77995]: cluster 2023-09-23T19:23:53.746441+0000 mgr.y (mgr.25325) 445 : cluster 0 pgmap v363: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 47 MiB/s wr, 755 op/s; 10819/282840 objects degraded (3.825%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:23:55.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:54 smithi016 bash[77995]: cluster 2023-09-23T19:23:54.122032+0000 mon.a (mon.0) 608 : cluster 3 Health check update: Degraded data redundancy: 10819/282840 objects degraded (3.825%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:23:55.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:54 smithi016 bash[77995]: audit 2023-09-23T19:23:54.634922+0000 mon.a (mon.0) 609 : audit 0 from='client.? 172.21.15.16:0/206062756' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:55.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:54 smithi092 bash[58855]: cluster 2023-09-23T19:23:53.746441+0000 mgr.y (mgr.25325) 445 : cluster 0 pgmap v363: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 47 MiB/s wr, 755 op/s; 10819/282840 objects degraded (3.825%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:23:55.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:54 smithi092 bash[58855]: cluster 2023-09-23T19:23:54.122032+0000 mon.a (mon.0) 608 : cluster 3 Health check update: Degraded data redundancy: 10819/282840 objects degraded (3.825%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:23:55.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:54 smithi092 bash[58855]: audit 2023-09-23T19:23:54.634922+0000 mon.a (mon.0) 609 : audit 0 from='client.? 172.21.15.16:0/206062756' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:55.521 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 86 16 97408 97392 70.7721 39.8125 0.00775665 0.0141217 2023-09-23T19:23:55.916 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:23:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:23:55] "GET /metrics HTTP/1.1" 200 36196 "" "Prometheus/2.33.4" 2023-09-23T19:23:55.916 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:23:55 smithi016 bash[70992]: debug 2023-09-23T19:23:55.908+0000 7fb2a5a7c700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:23:56.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:55 smithi016 bash[77995]: audit 2023-09-23T19:23:54.962729+0000 mon.a (mon.0) 610 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:23:56.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:55 smithi016 bash[77995]: audit 2023-09-23T19:23:54.974083+0000 mon.a (mon.0) 611 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:23:56.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:55 smithi016 bash[77995]: audit 2023-09-23T19:23:55.141587+0000 mon.a (mon.0) 612 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:23:56.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:55 smithi016 bash[77995]: audit 2023-09-23T19:23:55.149443+0000 mon.a (mon.0) 613 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:23:56.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:55 smithi016 bash[77995]: cluster 2023-09-23T19:23:55.747277+0000 mgr.y (mgr.25325) 446 : cluster 0 pgmap v364: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 822 op/s; 10819/288327 objects degraded (3.752%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:23:56.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:55 smithi016 bash[77995]: audit 2023-09-23T19:23:55.786808+0000 mon.a (mon.0) 614 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:23:56.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:55 smithi016 bash[77995]: audit 2023-09-23T19:23:55.788043+0000 mon.a (mon.0) 615 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:23:56.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:55 smithi016 bash[77995]: audit 2023-09-23T19:23:55.800738+0000 mon.a (mon.0) 616 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:23:56.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:55 smithi016 bash[77995]: audit 2023-09-23T19:23:55.884454+0000 mon.a (mon.0) 617 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:23:56.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:55 smithi016 bash[77995]: audit 2023-09-23T19:23:55.897741+0000 mon.a (mon.0) 618 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:23:56.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:55 smithi016 bash[77995]: audit 2023-09-23T19:23:55.901252+0000 mon.a (mon.0) 619 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:23:56.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:55 smithi016 bash[77995]: audit 2023-09-23T19:23:55.903928+0000 mon.a (mon.0) 620 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:23:56.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:55 smithi016 bash[77995]: audit 2023-09-23T19:23:55.912425+0000 mon.a (mon.0) 621 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:23:56.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:55 smithi016 bash[75004]: audit 2023-09-23T19:23:54.962729+0000 mon.a (mon.0) 610 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:23:56.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:55 smithi016 bash[75004]: audit 2023-09-23T19:23:54.974083+0000 mon.a (mon.0) 611 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:23:56.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:55 smithi016 bash[75004]: audit 2023-09-23T19:23:55.141587+0000 mon.a (mon.0) 612 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:23:56.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:55 smithi016 bash[75004]: audit 2023-09-23T19:23:55.149443+0000 mon.a (mon.0) 613 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:23:56.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:55 smithi016 bash[75004]: cluster 2023-09-23T19:23:55.747277+0000 mgr.y (mgr.25325) 446 : cluster 0 pgmap v364: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 822 op/s; 10819/288327 objects degraded (3.752%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:23:56.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:55 smithi016 bash[75004]: audit 2023-09-23T19:23:55.786808+0000 mon.a (mon.0) 614 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:23:56.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:55 smithi016 bash[75004]: audit 2023-09-23T19:23:55.788043+0000 mon.a (mon.0) 615 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:23:56.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:55 smithi016 bash[75004]: audit 2023-09-23T19:23:55.800738+0000 mon.a (mon.0) 616 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:23:56.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:55 smithi016 bash[75004]: audit 2023-09-23T19:23:55.884454+0000 mon.a (mon.0) 617 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:23:56.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:55 smithi016 bash[75004]: audit 2023-09-23T19:23:55.897741+0000 mon.a (mon.0) 618 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:23:56.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:55 smithi016 bash[75004]: audit 2023-09-23T19:23:55.901252+0000 mon.a (mon.0) 619 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:23:56.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:55 smithi016 bash[75004]: audit 2023-09-23T19:23:55.903928+0000 mon.a (mon.0) 620 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:23:56.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:55 smithi016 bash[75004]: audit 2023-09-23T19:23:55.912425+0000 mon.a (mon.0) 621 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:23:56.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:55 smithi092 bash[58855]: audit 2023-09-23T19:23:54.962729+0000 mon.a (mon.0) 610 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:23:56.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:55 smithi092 bash[58855]: audit 2023-09-23T19:23:54.974083+0000 mon.a (mon.0) 611 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:23:56.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:55 smithi092 bash[58855]: audit 2023-09-23T19:23:55.141587+0000 mon.a (mon.0) 612 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:23:56.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:55 smithi092 bash[58855]: audit 2023-09-23T19:23:55.149443+0000 mon.a (mon.0) 613 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:23:56.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:55 smithi092 bash[58855]: cluster 2023-09-23T19:23:55.747277+0000 mgr.y (mgr.25325) 446 : cluster 0 pgmap v364: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 822 op/s; 10819/288327 objects degraded (3.752%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:23:56.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:55 smithi092 bash[58855]: audit 2023-09-23T19:23:55.786808+0000 mon.a (mon.0) 614 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:23:56.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:55 smithi092 bash[58855]: audit 2023-09-23T19:23:55.788043+0000 mon.a (mon.0) 615 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:23:56.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:55 smithi092 bash[58855]: audit 2023-09-23T19:23:55.800738+0000 mon.a (mon.0) 616 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:23:56.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:55 smithi092 bash[58855]: audit 2023-09-23T19:23:55.884454+0000 mon.a (mon.0) 617 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:23:56.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:55 smithi092 bash[58855]: audit 2023-09-23T19:23:55.897741+0000 mon.a (mon.0) 618 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:23:56.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:55 smithi092 bash[58855]: audit 2023-09-23T19:23:55.901252+0000 mon.a (mon.0) 619 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:23:56.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:55 smithi092 bash[58855]: audit 2023-09-23T19:23:55.903928+0000 mon.a (mon.0) 620 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:23:56.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:55 smithi092 bash[58855]: audit 2023-09-23T19:23:55.912425+0000 mon.a (mon.0) 621 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:23:56.521 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 87 16 98167 98151 70.5038 47.4375 0.00748134 0.0141762 2023-09-23T19:23:57.522 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 88 16 98886 98870 70.2133 44.9375 0.0191168 0.0142358 2023-09-23T19:23:57.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:57 smithi016 bash[75004]: audit 2023-09-23T19:23:55.912668+0000 mgr.y (mgr.25325) 447 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:23:57.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:57 smithi016 bash[75004]: cephadm 2023-09-23T19:23:55.913876+0000 mgr.y (mgr.25325) 448 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:23:57.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:57 smithi016 bash[77995]: audit 2023-09-23T19:23:55.912668+0000 mgr.y (mgr.25325) 447 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:23:57.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:57 smithi016 bash[77995]: cephadm 2023-09-23T19:23:55.913876+0000 mgr.y (mgr.25325) 448 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:23:57.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:57 smithi092 bash[58855]: audit 2023-09-23T19:23:55.912668+0000 mgr.y (mgr.25325) 447 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:23:57.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:57 smithi092 bash[58855]: cephadm 2023-09-23T19:23:55.913876+0000 mgr.y (mgr.25325) 448 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:23:58.522 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 89 16 99654 99638 69.9636 48 0.00785468 0.0142857 2023-09-23T19:23:58.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:58 smithi016 bash[75004]: audit 2023-09-23T19:23:57.273597+0000 mon.c (mon.2) 75 : audit 0 from='client.? 172.21.15.16:0/2315726333' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:58.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:58 smithi016 bash[75004]: cluster 2023-09-23T19:23:57.747955+0000 mgr.y (mgr.25325) 449 : cluster 0 pgmap v365: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 45 MiB/s wr, 718 op/s; 10718/290982 objects degraded (3.683%); 1.6 MiB/s, 26 objects/s recovering 2023-09-23T19:23:58.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:58 smithi016 bash[77995]: audit 2023-09-23T19:23:57.273597+0000 mon.c (mon.2) 75 : audit 0 from='client.? 172.21.15.16:0/2315726333' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:58.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:58 smithi016 bash[77995]: cluster 2023-09-23T19:23:57.747955+0000 mgr.y (mgr.25325) 449 : cluster 0 pgmap v365: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 45 MiB/s wr, 718 op/s; 10718/290982 objects degraded (3.683%); 1.6 MiB/s, 26 objects/s recovering 2023-09-23T19:23:58.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:58 smithi092 bash[58855]: audit 2023-09-23T19:23:57.273597+0000 mon.c (mon.2) 75 : audit 0 from='client.? 172.21.15.16:0/2315726333' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:23:58.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:58 smithi092 bash[58855]: cluster 2023-09-23T19:23:57.747955+0000 mgr.y (mgr.25325) 449 : cluster 0 pgmap v365: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 45 MiB/s wr, 718 op/s; 10718/290982 objects degraded (3.683%); 1.6 MiB/s, 26 objects/s recovering 2023-09-23T19:23:59.522 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 90 6 100425 100419 69.7286 48.8125 0.0223863 0.0143349 2023-09-23T19:23:59.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:59 smithi016 bash[75004]: audit 2023-09-23T19:23:58.625095+0000 mon.a (mon.0) 622 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:23:59.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:23:59 smithi016 bash[75004]: cluster 2023-09-23T19:23:59.123165+0000 mon.a (mon.0) 623 : cluster 3 Health check update: Degraded data redundancy: 10718/290982 objects degraded (3.683%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:23:59.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:59 smithi016 bash[77995]: audit 2023-09-23T19:23:58.625095+0000 mon.a (mon.0) 622 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:23:59.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:23:59 smithi016 bash[77995]: cluster 2023-09-23T19:23:59.123165+0000 mon.a (mon.0) 623 : cluster 3 Health check update: Degraded data redundancy: 10718/290982 objects degraded (3.683%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:23:59.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:59 smithi092 bash[58855]: audit 2023-09-23T19:23:58.625095+0000 mon.a (mon.0) 622 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:23:59.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:23:59 smithi092 bash[58855]: cluster 2023-09-23T19:23:59.123165+0000 mon.a (mon.0) 623 : cluster 3 Health check update: Degraded data redundancy: 10718/290982 objects degraded (3.683%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:24:00.522 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Total time run: 90.0148 2023-09-23T19:24:00.522 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Total writes made: 100425 2023-09-23T19:24:00.522 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Write size: 65536 2023-09-23T19:24:00.522 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Object size: 65536 2023-09-23T19:24:00.522 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Bandwidth (MB/sec): 69.7281 2023-09-23T19:24:00.522 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Stddev Bandwidth: 33.7688 2023-09-23T19:24:00.522 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Max bandwidth (MB/sec): 204.375 2023-09-23T19:24:00.522 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Min bandwidth (MB/sec): 0 2023-09-23T19:24:00.523 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Average IOPS: 1115 2023-09-23T19:24:00.523 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Stddev IOPS: 540.311 2023-09-23T19:24:00.523 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Max IOPS: 3270 2023-09-23T19:24:00.523 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Min IOPS: 0 2023-09-23T19:24:00.523 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Average Latency(s): 0.0143356 2023-09-23T19:24:00.523 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Stddev Latency(s): 0.0288939 2023-09-23T19:24:00.523 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Max latency(s): 2.18958 2023-09-23T19:24:00.523 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Min latency(s): 0.00119935 2023-09-23T19:24:00.531 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T19:24:00.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:00 smithi016 bash[75004]: audit 2023-09-23T19:23:59.732944+0000 mon.c (mon.2) 76 : audit 0 from='client.? 172.21.15.16:0/4214054493' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:00.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:00 smithi016 bash[75004]: cluster 2023-09-23T19:23:59.748825+0000 mgr.y (mgr.25325) 450 : cluster 0 pgmap v366: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 44 MiB/s wr, 708 op/s; 10718/293664 objects degraded (3.650%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:24:00.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:00 smithi016 bash[77995]: audit 2023-09-23T19:23:59.732944+0000 mon.c (mon.2) 76 : audit 0 from='client.? 172.21.15.16:0/4214054493' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:00.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:00 smithi016 bash[77995]: cluster 2023-09-23T19:23:59.748825+0000 mgr.y (mgr.25325) 450 : cluster 0 pgmap v366: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 44 MiB/s wr, 708 op/s; 10718/293664 objects degraded (3.650%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:24:00.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:00 smithi092 bash[58855]: audit 2023-09-23T19:23:59.732944+0000 mon.c (mon.2) 76 : audit 0 from='client.? 172.21.15.16:0/4214054493' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:00.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:00 smithi092 bash[58855]: cluster 2023-09-23T19:23:59.748825+0000 mgr.y (mgr.25325) 450 : cluster 0 pgmap v366: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 44 MiB/s wr, 708 op/s; 10718/293664 objects degraded (3.650%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:24:03.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:02 smithi016 bash[75004]: cluster 2023-09-23T19:24:01.749789+0000 mgr.y (mgr.25325) 451 : cluster 0 pgmap v367: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 833 op/s; 10610/300258 objects degraded (3.534%); 1.6 MiB/s, 26 objects/s recovering 2023-09-23T19:24:03.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:02 smithi016 bash[75004]: audit 2023-09-23T19:24:02.339129+0000 mon.a (mon.0) 624 : audit 0 from='client.? 172.21.15.16:0/1755332413' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:03.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:02 smithi016 bash[77995]: cluster 2023-09-23T19:24:01.749789+0000 mgr.y (mgr.25325) 451 : cluster 0 pgmap v367: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 833 op/s; 10610/300258 objects degraded (3.534%); 1.6 MiB/s, 26 objects/s recovering 2023-09-23T19:24:03.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:02 smithi016 bash[77995]: audit 2023-09-23T19:24:02.339129+0000 mon.a (mon.0) 624 : audit 0 from='client.? 172.21.15.16:0/1755332413' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:03.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:02 smithi092 bash[58855]: cluster 2023-09-23T19:24:01.749789+0000 mgr.y (mgr.25325) 451 : cluster 0 pgmap v367: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 833 op/s; 10610/300258 objects degraded (3.534%); 1.6 MiB/s, 26 objects/s recovering 2023-09-23T19:24:03.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:02 smithi092 bash[58855]: audit 2023-09-23T19:24:02.339129+0000 mon.a (mon.0) 624 : audit 0 from='client.? 172.21.15.16:0/1755332413' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:05.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:04 smithi016 bash[75004]: cluster 2023-09-23T19:24:03.750457+0000 mgr.y (mgr.25325) 452 : cluster 0 pgmap v368: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 37 MiB/s wr, 649 op/s; 10610/299214 objects degraded (3.546%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:24:05.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:04 smithi016 bash[75004]: cluster 2023-09-23T19:24:04.124482+0000 mon.a (mon.0) 625 : cluster 3 Health check update: Degraded data redundancy: 10610/299214 objects degraded (3.546%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:24:05.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:04 smithi016 bash[77995]: cluster 2023-09-23T19:24:03.750457+0000 mgr.y (mgr.25325) 452 : cluster 0 pgmap v368: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 37 MiB/s wr, 649 op/s; 10610/299214 objects degraded (3.546%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:24:05.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:04 smithi016 bash[77995]: cluster 2023-09-23T19:24:04.124482+0000 mon.a (mon.0) 625 : cluster 3 Health check update: Degraded data redundancy: 10610/299214 objects degraded (3.546%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:24:05.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:04 smithi092 bash[58855]: cluster 2023-09-23T19:24:03.750457+0000 mgr.y (mgr.25325) 452 : cluster 0 pgmap v368: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 37 MiB/s wr, 649 op/s; 10610/299214 objects degraded (3.546%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:24:05.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:04 smithi092 bash[58855]: cluster 2023-09-23T19:24:04.124482+0000 mon.a (mon.0) 625 : cluster 3 Health check update: Degraded data redundancy: 10610/299214 objects degraded (3.546%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:24:06.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:05 smithi016 bash[75004]: audit 2023-09-23T19:24:04.947874+0000 mon.c (mon.2) 77 : audit 0 from='client.? 172.21.15.16:0/3410519106' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:06.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:24:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:24:05] "GET /metrics HTTP/1.1" 200 36200 "" "Prometheus/2.33.4" 2023-09-23T19:24:06.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:05 smithi016 bash[77995]: audit 2023-09-23T19:24:04.947874+0000 mon.c (mon.2) 77 : audit 0 from='client.? 172.21.15.16:0/3410519106' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:06.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:05 smithi092 bash[58855]: audit 2023-09-23T19:24:04.947874+0000 mon.c (mon.2) 77 : audit 0 from='client.? 172.21.15.16:0/3410519106' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:07.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:06 smithi016 bash[75004]: cluster 2023-09-23T19:24:05.751094+0000 mgr.y (mgr.25325) 453 : cluster 0 pgmap v369: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 32 MiB/s wr, 711 op/s; 10610/294231 objects degraded (3.606%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:24:07.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:06 smithi016 bash[77995]: cluster 2023-09-23T19:24:05.751094+0000 mgr.y (mgr.25325) 453 : cluster 0 pgmap v369: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 32 MiB/s wr, 711 op/s; 10610/294231 objects degraded (3.606%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:24:07.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:06 smithi092 bash[58855]: cluster 2023-09-23T19:24:05.751094+0000 mgr.y (mgr.25325) 453 : cluster 0 pgmap v369: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 32 MiB/s wr, 711 op/s; 10610/294231 objects degraded (3.606%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:24:08.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:07 smithi016 bash[75004]: audit 2023-09-23T19:24:07.526713+0000 mon.a (mon.0) 626 : audit 0 from='client.? 172.21.15.16:0/739589669' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:08.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:07 smithi016 bash[77995]: audit 2023-09-23T19:24:07.526713+0000 mon.a (mon.0) 626 : audit 0 from='client.? 172.21.15.16:0/739589669' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:08.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:07 smithi092 bash[58855]: audit 2023-09-23T19:24:07.526713+0000 mon.a (mon.0) 626 : audit 0 from='client.? 172.21.15.16:0/739589669' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:09.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:08 smithi016 bash[75004]: cluster 2023-09-23T19:24:07.751745+0000 mgr.y (mgr.25325) 454 : cluster 0 pgmap v370: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 23 MiB/s wr, 642 op/s; 10505/291249 objects degraded (3.607%); 1.6 MiB/s, 26 objects/s recovering 2023-09-23T19:24:09.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:08 smithi016 bash[77995]: cluster 2023-09-23T19:24:07.751745+0000 mgr.y (mgr.25325) 454 : cluster 0 pgmap v370: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 23 MiB/s wr, 642 op/s; 10505/291249 objects degraded (3.607%); 1.6 MiB/s, 26 objects/s recovering 2023-09-23T19:24:09.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:08 smithi092 bash[58855]: cluster 2023-09-23T19:24:07.751745+0000 mgr.y (mgr.25325) 454 : cluster 0 pgmap v370: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 23 MiB/s wr, 642 op/s; 10505/291249 objects degraded (3.607%); 1.6 MiB/s, 26 objects/s recovering 2023-09-23T19:24:10.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:09 smithi016 bash[75004]: cluster 2023-09-23T19:24:08.854469+0000 mon.a (mon.0) 627 : cluster 0 osdmap e101: 8 total, 8 up, 8 in 2023-09-23T19:24:10.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:09 smithi016 bash[75004]: cluster 2023-09-23T19:24:08.855065+0000 osd.3 (osd.3) 30 : cluster 3 failed to encode map e101 with expected crc 2023-09-23T19:24:10.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:09 smithi016 bash[75004]: cluster 2023-09-23T19:24:08.855166+0000 osd.6 (osd.6) 19 : cluster 3 failed to encode map e101 with expected crc 2023-09-23T19:24:10.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:09 smithi016 bash[75004]: cluster 2023-09-23T19:24:08.863430+0000 osd.5 (osd.5) 20 : cluster 3 failed to encode map e101 with expected crc 2023-09-23T19:24:10.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:09 smithi016 bash[75004]: cluster 2023-09-23T19:24:09.125695+0000 mon.a (mon.0) 628 : cluster 3 Health check update: Degraded data redundancy: 10505/291249 objects degraded (3.607%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:24:10.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:09 smithi016 bash[75004]: cluster 2023-09-23T19:24:09.752312+0000 mgr.y (mgr.25325) 455 : cluster 0 pgmap v372: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 16 MiB/s wr, 689 op/s; 10495/288357 objects degraded (3.640%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T19:24:10.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:09 smithi016 bash[77995]: cluster 2023-09-23T19:24:08.854469+0000 mon.a (mon.0) 627 : cluster 0 osdmap e101: 8 total, 8 up, 8 in 2023-09-23T19:24:10.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:09 smithi016 bash[77995]: cluster 2023-09-23T19:24:08.855065+0000 osd.3 (osd.3) 30 : cluster 3 failed to encode map e101 with expected crc 2023-09-23T19:24:10.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:09 smithi016 bash[77995]: cluster 2023-09-23T19:24:08.855166+0000 osd.6 (osd.6) 19 : cluster 3 failed to encode map e101 with expected crc 2023-09-23T19:24:10.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:09 smithi016 bash[77995]: cluster 2023-09-23T19:24:08.863430+0000 osd.5 (osd.5) 20 : cluster 3 failed to encode map e101 with expected crc 2023-09-23T19:24:10.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:09 smithi016 bash[77995]: cluster 2023-09-23T19:24:09.125695+0000 mon.a (mon.0) 628 : cluster 3 Health check update: Degraded data redundancy: 10505/291249 objects degraded (3.607%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:24:10.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:09 smithi016 bash[77995]: cluster 2023-09-23T19:24:09.752312+0000 mgr.y (mgr.25325) 455 : cluster 0 pgmap v372: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 16 MiB/s wr, 689 op/s; 10495/288357 objects degraded (3.640%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T19:24:10.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:09 smithi092 bash[58855]: cluster 2023-09-23T19:24:08.854469+0000 mon.a (mon.0) 627 : cluster 0 osdmap e101: 8 total, 8 up, 8 in 2023-09-23T19:24:10.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:09 smithi092 bash[58855]: cluster 2023-09-23T19:24:08.855065+0000 osd.3 (osd.3) 30 : cluster 3 failed to encode map e101 with expected crc 2023-09-23T19:24:10.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:09 smithi092 bash[58855]: cluster 2023-09-23T19:24:08.855166+0000 osd.6 (osd.6) 19 : cluster 3 failed to encode map e101 with expected crc 2023-09-23T19:24:10.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:09 smithi092 bash[58855]: cluster 2023-09-23T19:24:08.863430+0000 osd.5 (osd.5) 20 : cluster 3 failed to encode map e101 with expected crc 2023-09-23T19:24:10.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:09 smithi092 bash[58855]: cluster 2023-09-23T19:24:09.125695+0000 mon.a (mon.0) 628 : cluster 3 Health check update: Degraded data redundancy: 10505/291249 objects degraded (3.607%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:24:10.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:09 smithi092 bash[58855]: cluster 2023-09-23T19:24:09.752312+0000 mgr.y (mgr.25325) 455 : cluster 0 pgmap v372: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 16 MiB/s wr, 689 op/s; 10495/288357 objects degraded (3.640%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T19:24:11.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:10 smithi016 bash[75004]: cluster 2023-09-23T19:24:09.859448+0000 osd.2 (osd.2) 25 : cluster 3 failed to encode map e102 with expected crc 2023-09-23T19:24:11.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:10 smithi016 bash[75004]: cluster 2023-09-23T19:24:09.862135+0000 mon.a (mon.0) 629 : cluster 0 osdmap e102: 8 total, 8 up, 8 in 2023-09-23T19:24:11.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:10 smithi016 bash[75004]: cluster 2023-09-23T19:24:09.867925+0000 osd.3 (osd.3) 31 : cluster 3 failed to encode map e102 with expected crc 2023-09-23T19:24:11.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:10 smithi016 bash[75004]: cluster 2023-09-23T19:24:09.871353+0000 osd.6 (osd.6) 20 : cluster 3 failed to encode map e102 with expected crc 2023-09-23T19:24:11.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:10 smithi016 bash[75004]: cluster 2023-09-23T19:24:09.904515+0000 osd.5 (osd.5) 21 : cluster 3 failed to encode map e102 with expected crc 2023-09-23T19:24:11.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:10 smithi016 bash[75004]: audit 2023-09-23T19:24:09.970805+0000 mon.a (mon.0) 630 : audit 0 from='client.? 172.21.15.16:0/1012965298' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:11.285 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:24:10 smithi016 bash[70992]: debug 2023-09-23T19:24:10.928+0000 7fb2a5a7c700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:24:11.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:10 smithi016 bash[77995]: cluster 2023-09-23T19:24:09.859448+0000 osd.2 (osd.2) 25 : cluster 3 failed to encode map e102 with expected crc 2023-09-23T19:24:11.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:10 smithi016 bash[77995]: cluster 2023-09-23T19:24:09.862135+0000 mon.a (mon.0) 629 : cluster 0 osdmap e102: 8 total, 8 up, 8 in 2023-09-23T19:24:11.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:10 smithi016 bash[77995]: cluster 2023-09-23T19:24:09.867925+0000 osd.3 (osd.3) 31 : cluster 3 failed to encode map e102 with expected crc 2023-09-23T19:24:11.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:10 smithi016 bash[77995]: cluster 2023-09-23T19:24:09.871353+0000 osd.6 (osd.6) 20 : cluster 3 failed to encode map e102 with expected crc 2023-09-23T19:24:11.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:10 smithi016 bash[77995]: cluster 2023-09-23T19:24:09.904515+0000 osd.5 (osd.5) 21 : cluster 3 failed to encode map e102 with expected crc 2023-09-23T19:24:11.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:10 smithi016 bash[77995]: audit 2023-09-23T19:24:09.970805+0000 mon.a (mon.0) 630 : audit 0 from='client.? 172.21.15.16:0/1012965298' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:11.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:10 smithi092 bash[58855]: cluster 2023-09-23T19:24:09.859448+0000 osd.2 (osd.2) 25 : cluster 3 failed to encode map e102 with expected crc 2023-09-23T19:24:11.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:10 smithi092 bash[58855]: cluster 2023-09-23T19:24:09.862135+0000 mon.a (mon.0) 629 : cluster 0 osdmap e102: 8 total, 8 up, 8 in 2023-09-23T19:24:11.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:10 smithi092 bash[58855]: cluster 2023-09-23T19:24:09.867925+0000 osd.3 (osd.3) 31 : cluster 3 failed to encode map e102 with expected crc 2023-09-23T19:24:11.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:10 smithi092 bash[58855]: cluster 2023-09-23T19:24:09.871353+0000 osd.6 (osd.6) 20 : cluster 3 failed to encode map e102 with expected crc 2023-09-23T19:24:11.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:10 smithi092 bash[58855]: cluster 2023-09-23T19:24:09.904515+0000 osd.5 (osd.5) 21 : cluster 3 failed to encode map e102 with expected crc 2023-09-23T19:24:11.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:10 smithi092 bash[58855]: audit 2023-09-23T19:24:09.970805+0000 mon.a (mon.0) 630 : audit 0 from='client.? 172.21.15.16:0/1012965298' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:12.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:11 smithi016 bash[75004]: audit 2023-09-23T19:24:10.928830+0000 mon.a (mon.0) 631 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:24:12.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:11 smithi016 bash[75004]: audit 2023-09-23T19:24:10.929096+0000 mgr.y (mgr.25325) 456 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:24:12.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:11 smithi016 bash[75004]: cephadm 2023-09-23T19:24:10.930528+0000 mgr.y (mgr.25325) 457 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:24:12.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:11 smithi016 bash[75004]: cluster 2023-09-23T19:24:11.753058+0000 mgr.y (mgr.25325) 458 : cluster 0 pgmap v374: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 127 B/s rd, 127 B/s wr, 751 op/s; 10463/281193 objects degraded (3.721%); 1.1 MiB/s, 18 objects/s recovering 2023-09-23T19:24:12.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:11 smithi016 bash[77995]: audit 2023-09-23T19:24:10.928830+0000 mon.a (mon.0) 631 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:24:12.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:11 smithi016 bash[77995]: audit 2023-09-23T19:24:10.929096+0000 mgr.y (mgr.25325) 456 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:24:12.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:11 smithi016 bash[77995]: cephadm 2023-09-23T19:24:10.930528+0000 mgr.y (mgr.25325) 457 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:24:12.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:11 smithi016 bash[77995]: cluster 2023-09-23T19:24:11.753058+0000 mgr.y (mgr.25325) 458 : cluster 0 pgmap v374: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 127 B/s rd, 127 B/s wr, 751 op/s; 10463/281193 objects degraded (3.721%); 1.1 MiB/s, 18 objects/s recovering 2023-09-23T19:24:12.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:11 smithi092 bash[58855]: audit 2023-09-23T19:24:10.928830+0000 mon.a (mon.0) 631 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:24:12.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:11 smithi092 bash[58855]: audit 2023-09-23T19:24:10.929096+0000 mgr.y (mgr.25325) 456 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:24:12.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:11 smithi092 bash[58855]: cephadm 2023-09-23T19:24:10.930528+0000 mgr.y (mgr.25325) 457 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:24:12.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:11 smithi092 bash[58855]: cluster 2023-09-23T19:24:11.753058+0000 mgr.y (mgr.25325) 458 : cluster 0 pgmap v374: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 127 B/s rd, 127 B/s wr, 751 op/s; 10463/281193 objects degraded (3.721%); 1.1 MiB/s, 18 objects/s recovering 2023-09-23T19:24:13.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:12 smithi016 bash[75004]: audit 2023-09-23T19:24:12.620575+0000 mon.a (mon.0) 632 : audit 0 from='client.? 172.21.15.16:0/2724164243' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:13.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:12 smithi016 bash[77995]: audit 2023-09-23T19:24:12.620575+0000 mon.a (mon.0) 632 : audit 0 from='client.? 172.21.15.16:0/2724164243' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:13.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:12 smithi092 bash[58855]: audit 2023-09-23T19:24:12.620575+0000 mon.a (mon.0) 632 : audit 0 from='client.? 172.21.15.16:0/2724164243' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:14.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:13 smithi016 bash[77995]: audit 2023-09-23T19:24:13.625349+0000 mon.a (mon.0) 633 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:24:14.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:13 smithi016 bash[77995]: cluster 2023-09-23T19:24:13.753760+0000 mgr.y (mgr.25325) 459 : cluster 0 pgmap v375: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 10212/278829 objects degraded (3.662%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T19:24:14.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:13 smithi016 bash[75004]: audit 2023-09-23T19:24:13.625349+0000 mon.a (mon.0) 633 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:24:14.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:13 smithi016 bash[75004]: cluster 2023-09-23T19:24:13.753760+0000 mgr.y (mgr.25325) 459 : cluster 0 pgmap v375: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 10212/278829 objects degraded (3.662%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T19:24:14.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:13 smithi092 bash[58855]: audit 2023-09-23T19:24:13.625349+0000 mon.a (mon.0) 633 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:24:14.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:13 smithi092 bash[58855]: cluster 2023-09-23T19:24:13.753760+0000 mgr.y (mgr.25325) 459 : cluster 0 pgmap v375: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 10212/278829 objects degraded (3.662%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T19:24:15.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:14 smithi016 bash[75004]: cluster 2023-09-23T19:24:14.126723+0000 mon.a (mon.0) 634 : cluster 3 Health check update: Degraded data redundancy: 10212/278829 objects degraded (3.662%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:24:15.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:14 smithi016 bash[77995]: cluster 2023-09-23T19:24:14.126723+0000 mon.a (mon.0) 634 : cluster 3 Health check update: Degraded data redundancy: 10212/278829 objects degraded (3.662%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:24:15.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:14 smithi092 bash[58855]: cluster 2023-09-23T19:24:14.126723+0000 mon.a (mon.0) 634 : cluster 3 Health check update: Degraded data redundancy: 10212/278829 objects degraded (3.662%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:24:16.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:15 smithi016 bash[75004]: audit 2023-09-23T19:24:15.222254+0000 mon.c (mon.2) 78 : audit 0 from='client.? 172.21.15.16:0/4197509891' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:16.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:15 smithi016 bash[75004]: cluster 2023-09-23T19:24:15.756865+0000 mgr.y (mgr.25325) 460 : cluster 0 pgmap v376: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 768 op/s; 10212/272787 objects degraded (3.744%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T19:24:16.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:24:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:24:15] "GET /metrics HTTP/1.1" 200 36201 "" "Prometheus/2.33.4" 2023-09-23T19:24:16.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:15 smithi016 bash[77995]: audit 2023-09-23T19:24:15.222254+0000 mon.c (mon.2) 78 : audit 0 from='client.? 172.21.15.16:0/4197509891' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:16.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:15 smithi016 bash[77995]: cluster 2023-09-23T19:24:15.756865+0000 mgr.y (mgr.25325) 460 : cluster 0 pgmap v376: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 768 op/s; 10212/272787 objects degraded (3.744%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T19:24:16.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:15 smithi092 bash[58855]: audit 2023-09-23T19:24:15.222254+0000 mon.c (mon.2) 78 : audit 0 from='client.? 172.21.15.16:0/4197509891' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:16.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:15 smithi092 bash[58855]: cluster 2023-09-23T19:24:15.756865+0000 mgr.y (mgr.25325) 460 : cluster 0 pgmap v376: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 768 op/s; 10212/272787 objects degraded (3.744%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T19:24:19.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:18 smithi016 bash[75004]: cluster 2023-09-23T19:24:17.757426+0000 mgr.y (mgr.25325) 461 : cluster 0 pgmap v377: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 10212/270621 objects degraded (3.774%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T19:24:19.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:18 smithi016 bash[75004]: audit 2023-09-23T19:24:17.823921+0000 mon.c (mon.2) 79 : audit 0 from='client.? 172.21.15.16:0/1889225787' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:19.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:18 smithi016 bash[77995]: cluster 2023-09-23T19:24:17.757426+0000 mgr.y (mgr.25325) 461 : cluster 0 pgmap v377: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 10212/270621 objects degraded (3.774%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T19:24:19.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:18 smithi016 bash[77995]: audit 2023-09-23T19:24:17.823921+0000 mon.c (mon.2) 79 : audit 0 from='client.? 172.21.15.16:0/1889225787' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:19.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:18 smithi092 bash[58855]: cluster 2023-09-23T19:24:17.757426+0000 mgr.y (mgr.25325) 461 : cluster 0 pgmap v377: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 10212/270621 objects degraded (3.774%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T19:24:19.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:18 smithi092 bash[58855]: audit 2023-09-23T19:24:17.823921+0000 mon.c (mon.2) 79 : audit 0 from='client.? 172.21.15.16:0/1889225787' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:20.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:19 smithi016 bash[75004]: cluster 2023-09-23T19:24:19.128048+0000 mon.a (mon.0) 635 : cluster 3 Health check update: Degraded data redundancy: 10212/270621 objects degraded (3.774%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:24:20.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:19 smithi016 bash[77995]: cluster 2023-09-23T19:24:19.128048+0000 mon.a (mon.0) 635 : cluster 3 Health check update: Degraded data redundancy: 10212/270621 objects degraded (3.774%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:24:20.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:19 smithi092 bash[58855]: cluster 2023-09-23T19:24:19.128048+0000 mon.a (mon.0) 635 : cluster 3 Health check update: Degraded data redundancy: 10212/270621 objects degraded (3.774%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:24:21.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:20 smithi016 bash[75004]: cluster 2023-09-23T19:24:19.757909+0000 mgr.y (mgr.25325) 462 : cluster 0 pgmap v378: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 10027/267840 objects degraded (3.744%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T19:24:21.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:20 smithi016 bash[75004]: audit 2023-09-23T19:24:20.452383+0000 mon.c (mon.2) 80 : audit 0 from='client.? 172.21.15.16:0/3947880004' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:21.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:20 smithi016 bash[77995]: cluster 2023-09-23T19:24:19.757909+0000 mgr.y (mgr.25325) 462 : cluster 0 pgmap v378: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 10027/267840 objects degraded (3.744%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T19:24:21.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:20 smithi016 bash[77995]: audit 2023-09-23T19:24:20.452383+0000 mon.c (mon.2) 80 : audit 0 from='client.? 172.21.15.16:0/3947880004' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:21.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:20 smithi092 bash[58855]: cluster 2023-09-23T19:24:19.757909+0000 mgr.y (mgr.25325) 462 : cluster 0 pgmap v378: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 10027/267840 objects degraded (3.744%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T19:24:21.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:20 smithi092 bash[58855]: audit 2023-09-23T19:24:20.452383+0000 mon.c (mon.2) 80 : audit 0 from='client.? 172.21.15.16:0/3947880004' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:23.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:22 smithi016 bash[75004]: cluster 2023-09-23T19:24:21.758767+0000 mgr.y (mgr.25325) 463 : cluster 0 pgmap v379: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 10027/259368 objects degraded (3.866%); 2.5 MiB/s, 39 objects/s recovering 2023-09-23T19:24:23.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:22 smithi016 bash[77995]: cluster 2023-09-23T19:24:21.758767+0000 mgr.y (mgr.25325) 463 : cluster 0 pgmap v379: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 10027/259368 objects degraded (3.866%); 2.5 MiB/s, 39 objects/s recovering 2023-09-23T19:24:23.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:22 smithi092 bash[58855]: cluster 2023-09-23T19:24:21.758767+0000 mgr.y (mgr.25325) 463 : cluster 0 pgmap v379: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 10027/259368 objects degraded (3.866%); 2.5 MiB/s, 39 objects/s recovering 2023-09-23T19:24:24.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:23 smithi016 bash[75004]: audit 2023-09-23T19:24:23.095817+0000 mon.b (mon.1) 29 : audit 0 from='client.? 172.21.15.16:0/4018161947' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:24.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:23 smithi016 bash[77995]: audit 2023-09-23T19:24:23.095817+0000 mon.b (mon.1) 29 : audit 0 from='client.? 172.21.15.16:0/4018161947' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:24.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:23 smithi092 bash[58855]: audit 2023-09-23T19:24:23.095817+0000 mon.b (mon.1) 29 : audit 0 from='client.? 172.21.15.16:0/4018161947' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:25.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:24 smithi016 bash[75004]: cluster 2023-09-23T19:24:23.759417+0000 mgr.y (mgr.25325) 464 : cluster 0 pgmap v380: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 682 op/s; 9834/256617 objects degraded (3.832%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T19:24:25.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:24 smithi016 bash[75004]: cluster 2023-09-23T19:24:24.129101+0000 mon.a (mon.0) 636 : cluster 3 Health check update: Degraded data redundancy: 9834/256617 objects degraded (3.832%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:24:25.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:24 smithi016 bash[77995]: cluster 2023-09-23T19:24:23.759417+0000 mgr.y (mgr.25325) 464 : cluster 0 pgmap v380: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 682 op/s; 9834/256617 objects degraded (3.832%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T19:24:25.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:24 smithi016 bash[77995]: cluster 2023-09-23T19:24:24.129101+0000 mon.a (mon.0) 636 : cluster 3 Health check update: Degraded data redundancy: 9834/256617 objects degraded (3.832%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:24:25.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:24 smithi092 bash[58855]: cluster 2023-09-23T19:24:23.759417+0000 mgr.y (mgr.25325) 464 : cluster 0 pgmap v380: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 682 op/s; 9834/256617 objects degraded (3.832%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T19:24:25.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:24 smithi092 bash[58855]: cluster 2023-09-23T19:24:24.129101+0000 mon.a (mon.0) 636 : cluster 3 Health check update: Degraded data redundancy: 9834/256617 objects degraded (3.832%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:24:25.844 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:25 smithi016 bash[75004]: audit 2023-09-23T19:24:25.661714+0000 mon.c (mon.2) 81 : audit 0 from='client.? 172.21.15.16:0/2630374218' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:25.844 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:24:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:24:25] "GET /metrics HTTP/1.1" 200 36201 "" "Prometheus/2.33.4" 2023-09-23T19:24:25.844 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:25 smithi016 bash[77995]: audit 2023-09-23T19:24:25.661714+0000 mon.c (mon.2) 81 : audit 0 from='client.? 172.21.15.16:0/2630374218' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:25.947 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:24:25 smithi016 bash[70992]: debug 2023-09-23T19:24:25.943+0000 7fb2a5a7c700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:24:26.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:25 smithi092 bash[58855]: audit 2023-09-23T19:24:25.661714+0000 mon.c (mon.2) 81 : audit 0 from='client.? 172.21.15.16:0/2630374218' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:27.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:26 smithi016 bash[75004]: cluster 2023-09-23T19:24:25.760140+0000 mgr.y (mgr.25325) 465 : cluster 0 pgmap v381: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 791 op/s; 9834/250329 objects degraded (3.928%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T19:24:27.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:26 smithi016 bash[75004]: audit 2023-09-23T19:24:25.944791+0000 mon.a (mon.0) 637 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:24:27.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:26 smithi016 bash[75004]: audit 2023-09-23T19:24:25.945061+0000 mgr.y (mgr.25325) 466 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:24:27.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:26 smithi016 bash[75004]: cephadm 2023-09-23T19:24:25.946329+0000 mgr.y (mgr.25325) 467 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:24:27.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:26 smithi016 bash[77995]: cluster 2023-09-23T19:24:25.760140+0000 mgr.y (mgr.25325) 465 : cluster 0 pgmap v381: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 791 op/s; 9834/250329 objects degraded (3.928%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T19:24:27.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:26 smithi016 bash[77995]: audit 2023-09-23T19:24:25.944791+0000 mon.a (mon.0) 637 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:24:27.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:26 smithi016 bash[77995]: audit 2023-09-23T19:24:25.945061+0000 mgr.y (mgr.25325) 466 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:24:27.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:26 smithi016 bash[77995]: cephadm 2023-09-23T19:24:25.946329+0000 mgr.y (mgr.25325) 467 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:24:27.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:26 smithi092 bash[58855]: cluster 2023-09-23T19:24:25.760140+0000 mgr.y (mgr.25325) 465 : cluster 0 pgmap v381: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 791 op/s; 9834/250329 objects degraded (3.928%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T19:24:27.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:26 smithi092 bash[58855]: audit 2023-09-23T19:24:25.944791+0000 mon.a (mon.0) 637 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:24:27.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:26 smithi092 bash[58855]: audit 2023-09-23T19:24:25.945061+0000 mgr.y (mgr.25325) 466 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:24:27.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:26 smithi092 bash[58855]: cephadm 2023-09-23T19:24:25.946329+0000 mgr.y (mgr.25325) 467 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:24:28.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:27 smithi016 bash[75004]: cluster 2023-09-23T19:24:27.760789+0000 mgr.y (mgr.25325) 468 : cluster 0 pgmap v382: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 9834/248319 objects degraded (3.960%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T19:24:28.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:27 smithi016 bash[77995]: cluster 2023-09-23T19:24:27.760789+0000 mgr.y (mgr.25325) 468 : cluster 0 pgmap v382: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 9834/248319 objects degraded (3.960%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T19:24:28.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:27 smithi092 bash[58855]: cluster 2023-09-23T19:24:27.760789+0000 mgr.y (mgr.25325) 468 : cluster 0 pgmap v382: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 9834/248319 objects degraded (3.960%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T19:24:29.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:28 smithi016 bash[77995]: audit 2023-09-23T19:24:28.264572+0000 mon.c (mon.2) 82 : audit 0 from='client.? 172.21.15.16:0/2915400776' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:29.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:28 smithi016 bash[77995]: audit 2023-09-23T19:24:28.625897+0000 mon.a (mon.0) 638 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:24:29.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:28 smithi016 bash[75004]: audit 2023-09-23T19:24:28.264572+0000 mon.c (mon.2) 82 : audit 0 from='client.? 172.21.15.16:0/2915400776' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:29.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:28 smithi016 bash[75004]: audit 2023-09-23T19:24:28.625897+0000 mon.a (mon.0) 638 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:24:29.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:28 smithi092 bash[58855]: audit 2023-09-23T19:24:28.264572+0000 mon.c (mon.2) 82 : audit 0 from='client.? 172.21.15.16:0/2915400776' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:29.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:28 smithi092 bash[58855]: audit 2023-09-23T19:24:28.625897+0000 mon.a (mon.0) 638 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:24:30.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:29 smithi016 bash[75004]: cluster 2023-09-23T19:24:29.131276+0000 mon.a (mon.0) 639 : cluster 3 Health check update: Degraded data redundancy: 9834/248319 objects degraded (3.960%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:24:30.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:29 smithi016 bash[75004]: cluster 2023-09-23T19:24:29.761653+0000 mgr.y (mgr.25325) 469 : cluster 0 pgmap v383: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 695 op/s; 9624/245580 objects degraded (3.919%); 3.1 MiB/s, 48 objects/s recovering 2023-09-23T19:24:30.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:29 smithi016 bash[77995]: cluster 2023-09-23T19:24:29.131276+0000 mon.a (mon.0) 639 : cluster 3 Health check update: Degraded data redundancy: 9834/248319 objects degraded (3.960%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:24:30.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:29 smithi016 bash[77995]: cluster 2023-09-23T19:24:29.761653+0000 mgr.y (mgr.25325) 469 : cluster 0 pgmap v383: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 695 op/s; 9624/245580 objects degraded (3.919%); 3.1 MiB/s, 48 objects/s recovering 2023-09-23T19:24:30.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:29 smithi092 bash[58855]: cluster 2023-09-23T19:24:29.131276+0000 mon.a (mon.0) 639 : cluster 3 Health check update: Degraded data redundancy: 9834/248319 objects degraded (3.960%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:24:30.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:29 smithi092 bash[58855]: cluster 2023-09-23T19:24:29.761653+0000 mgr.y (mgr.25325) 469 : cluster 0 pgmap v383: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 695 op/s; 9624/245580 objects degraded (3.919%); 3.1 MiB/s, 48 objects/s recovering 2023-09-23T19:24:31.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:31 smithi016 bash[75004]: audit 2023-09-23T19:24:30.884410+0000 mon.b (mon.1) 30 : audit 0 from='client.? 172.21.15.16:0/136403172' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:31.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:31 smithi016 bash[77995]: audit 2023-09-23T19:24:30.884410+0000 mon.b (mon.1) 30 : audit 0 from='client.? 172.21.15.16:0/136403172' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:31.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:31 smithi092 bash[58855]: audit 2023-09-23T19:24:30.884410+0000 mon.b (mon.1) 30 : audit 0 from='client.? 172.21.15.16:0/136403172' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:32.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:32 smithi016 bash[75004]: cluster 2023-09-23T19:24:31.762530+0000 mgr.y (mgr.25325) 470 : cluster 0 pgmap v384: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 840 op/s; 9624/237576 objects degraded (4.051%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T19:24:32.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:32 smithi016 bash[77995]: cluster 2023-09-23T19:24:31.762530+0000 mgr.y (mgr.25325) 470 : cluster 0 pgmap v384: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 840 op/s; 9624/237576 objects degraded (4.051%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T19:24:32.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:32 smithi092 bash[58855]: cluster 2023-09-23T19:24:31.762530+0000 mgr.y (mgr.25325) 470 : cluster 0 pgmap v384: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 840 op/s; 9624/237576 objects degraded (4.051%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T19:24:34.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:33 smithi016 bash[75004]: audit 2023-09-23T19:24:33.536193+0000 mon.a (mon.0) 640 : audit 0 from='client.? 172.21.15.16:0/1866755418' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:34.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:33 smithi016 bash[77995]: audit 2023-09-23T19:24:33.536193+0000 mon.a (mon.0) 640 : audit 0 from='client.? 172.21.15.16:0/1866755418' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:34.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:33 smithi092 bash[58855]: audit 2023-09-23T19:24:33.536193+0000 mon.a (mon.0) 640 : audit 0 from='client.? 172.21.15.16:0/1866755418' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:35.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:34 smithi016 bash[75004]: cluster 2023-09-23T19:24:33.763196+0000 mgr.y (mgr.25325) 471 : cluster 0 pgmap v385: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 9398/234906 objects degraded (4.001%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T19:24:35.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:34 smithi016 bash[75004]: cluster 2023-09-23T19:24:34.132558+0000 mon.a (mon.0) 641 : cluster 3 Health check update: Degraded data redundancy: 9624/237576 objects degraded (4.051%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:24:35.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:34 smithi016 bash[77995]: cluster 2023-09-23T19:24:33.763196+0000 mgr.y (mgr.25325) 471 : cluster 0 pgmap v385: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 9398/234906 objects degraded (4.001%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T19:24:35.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:34 smithi016 bash[77995]: cluster 2023-09-23T19:24:34.132558+0000 mon.a (mon.0) 641 : cluster 3 Health check update: Degraded data redundancy: 9624/237576 objects degraded (4.051%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:24:35.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:34 smithi092 bash[58855]: cluster 2023-09-23T19:24:33.763196+0000 mgr.y (mgr.25325) 471 : cluster 0 pgmap v385: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 9398/234906 objects degraded (4.001%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T19:24:35.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:34 smithi092 bash[58855]: cluster 2023-09-23T19:24:34.132558+0000 mon.a (mon.0) 641 : cluster 3 Health check update: Degraded data redundancy: 9624/237576 objects degraded (4.051%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:24:36.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:24:35 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:24:35] "GET /metrics HTTP/1.1" 200 36200 "" "Prometheus/2.33.4" 2023-09-23T19:24:37.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:36 smithi016 bash[75004]: cluster 2023-09-23T19:24:35.764243+0000 mgr.y (mgr.25325) 472 : cluster 0 pgmap v386: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 9398/229158 objects degraded (4.101%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T19:24:37.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:36 smithi016 bash[75004]: audit 2023-09-23T19:24:36.136597+0000 mon.a (mon.0) 642 : audit 0 from='client.? 172.21.15.16:0/1050713671' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:37.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:36 smithi016 bash[77995]: cluster 2023-09-23T19:24:35.764243+0000 mgr.y (mgr.25325) 472 : cluster 0 pgmap v386: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 9398/229158 objects degraded (4.101%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T19:24:37.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:36 smithi016 bash[77995]: audit 2023-09-23T19:24:36.136597+0000 mon.a (mon.0) 642 : audit 0 from='client.? 172.21.15.16:0/1050713671' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:37.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:36 smithi092 bash[58855]: cluster 2023-09-23T19:24:35.764243+0000 mgr.y (mgr.25325) 472 : cluster 0 pgmap v386: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 9398/229158 objects degraded (4.101%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T19:24:37.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:36 smithi092 bash[58855]: audit 2023-09-23T19:24:36.136597+0000 mon.a (mon.0) 642 : audit 0 from='client.? 172.21.15.16:0/1050713671' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:39.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:38 smithi016 bash[75004]: cluster 2023-09-23T19:24:37.764926+0000 mgr.y (mgr.25325) 473 : cluster 0 pgmap v387: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 9398/227145 objects degraded (4.137%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T19:24:39.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:38 smithi016 bash[75004]: audit 2023-09-23T19:24:38.760688+0000 mon.c (mon.2) 83 : audit 0 from='client.? 172.21.15.16:0/3498727405' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:39.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:38 smithi016 bash[77995]: cluster 2023-09-23T19:24:37.764926+0000 mgr.y (mgr.25325) 473 : cluster 0 pgmap v387: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 9398/227145 objects degraded (4.137%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T19:24:39.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:38 smithi016 bash[77995]: audit 2023-09-23T19:24:38.760688+0000 mon.c (mon.2) 83 : audit 0 from='client.? 172.21.15.16:0/3498727405' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:39.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:38 smithi092 bash[58855]: cluster 2023-09-23T19:24:37.764926+0000 mgr.y (mgr.25325) 473 : cluster 0 pgmap v387: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 9398/227145 objects degraded (4.137%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T19:24:39.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:38 smithi092 bash[58855]: audit 2023-09-23T19:24:38.760688+0000 mon.c (mon.2) 83 : audit 0 from='client.? 172.21.15.16:0/3498727405' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:40.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:39 smithi016 bash[75004]: cluster 2023-09-23T19:24:39.133996+0000 mon.a (mon.0) 643 : cluster 3 Health check update: Degraded data redundancy: 9398/227145 objects degraded (4.137%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:24:40.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:39 smithi016 bash[77995]: cluster 2023-09-23T19:24:39.133996+0000 mon.a (mon.0) 643 : cluster 3 Health check update: Degraded data redundancy: 9398/227145 objects degraded (4.137%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:24:40.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:39 smithi092 bash[58855]: cluster 2023-09-23T19:24:39.133996+0000 mon.a (mon.0) 643 : cluster 3 Health check update: Degraded data redundancy: 9398/227145 objects degraded (4.137%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:24:41.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:40 smithi016 bash[75004]: cluster 2023-09-23T19:24:39.765551+0000 mgr.y (mgr.25325) 474 : cluster 0 pgmap v388: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 9165/224484 objects degraded (4.083%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T19:24:41.284 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:24:40 smithi016 bash[70992]: debug 2023-09-23T19:24:40.959+0000 7fb2a5a7c700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:24:41.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:40 smithi016 bash[77995]: cluster 2023-09-23T19:24:39.765551+0000 mgr.y (mgr.25325) 474 : cluster 0 pgmap v388: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 9165/224484 objects degraded (4.083%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T19:24:41.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:40 smithi092 bash[58855]: cluster 2023-09-23T19:24:39.765551+0000 mgr.y (mgr.25325) 474 : cluster 0 pgmap v388: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 9165/224484 objects degraded (4.083%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T19:24:42.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:41 smithi016 bash[75004]: audit 2023-09-23T19:24:40.961267+0000 mon.a (mon.0) 644 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:24:42.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:41 smithi016 bash[75004]: audit 2023-09-23T19:24:41.367038+0000 mon.c (mon.2) 84 : audit 0 from='client.? 172.21.15.16:0/2838343895' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:42.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:41 smithi016 bash[77995]: audit 2023-09-23T19:24:40.961267+0000 mon.a (mon.0) 644 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:24:42.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:41 smithi016 bash[77995]: audit 2023-09-23T19:24:41.367038+0000 mon.c (mon.2) 84 : audit 0 from='client.? 172.21.15.16:0/2838343895' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:42.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:41 smithi092 bash[58855]: audit 2023-09-23T19:24:40.961267+0000 mon.a (mon.0) 644 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:24:42.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:41 smithi092 bash[58855]: audit 2023-09-23T19:24:41.367038+0000 mon.c (mon.2) 84 : audit 0 from='client.? 172.21.15.16:0/2838343895' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:43.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:42 smithi016 bash[77995]: audit 2023-09-23T19:24:40.961497+0000 mgr.y (mgr.25325) 475 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:24:43.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:42 smithi016 bash[77995]: cephadm 2023-09-23T19:24:40.962893+0000 mgr.y (mgr.25325) 476 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:24:43.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:42 smithi016 bash[77995]: cluster 2023-09-23T19:24:41.766637+0000 mgr.y (mgr.25325) 477 : cluster 0 pgmap v389: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 795 op/s; 9165/216921 objects degraded (4.225%); 2.4 MiB/s, 38 objects/s recovering 2023-09-23T19:24:43.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:42 smithi016 bash[75004]: audit 2023-09-23T19:24:40.961497+0000 mgr.y (mgr.25325) 475 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:24:43.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:42 smithi016 bash[75004]: cephadm 2023-09-23T19:24:40.962893+0000 mgr.y (mgr.25325) 476 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:24:43.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:42 smithi016 bash[75004]: cluster 2023-09-23T19:24:41.766637+0000 mgr.y (mgr.25325) 477 : cluster 0 pgmap v389: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 795 op/s; 9165/216921 objects degraded (4.225%); 2.4 MiB/s, 38 objects/s recovering 2023-09-23T19:24:43.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:42 smithi092 bash[58855]: audit 2023-09-23T19:24:40.961497+0000 mgr.y (mgr.25325) 475 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:24:43.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:42 smithi092 bash[58855]: cephadm 2023-09-23T19:24:40.962893+0000 mgr.y (mgr.25325) 476 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T19:24:43.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:42 smithi092 bash[58855]: cluster 2023-09-23T19:24:41.766637+0000 mgr.y (mgr.25325) 477 : cluster 0 pgmap v389: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 795 op/s; 9165/216921 objects degraded (4.225%); 2.4 MiB/s, 38 objects/s recovering 2023-09-23T19:24:44.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:43 smithi016 bash[75004]: audit 2023-09-23T19:24:43.626415+0000 mon.a (mon.0) 645 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:24:44.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:43 smithi016 bash[75004]: cluster 2023-09-23T19:24:43.767319+0000 mgr.y (mgr.25325) 478 : cluster 0 pgmap v390: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 644 op/s; 8936/214350 objects degraded (4.169%); 2.4 MiB/s, 57 objects/s recovering 2023-09-23T19:24:44.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:43 smithi016 bash[77995]: audit 2023-09-23T19:24:43.626415+0000 mon.a (mon.0) 645 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:24:44.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:43 smithi016 bash[77995]: cluster 2023-09-23T19:24:43.767319+0000 mgr.y (mgr.25325) 478 : cluster 0 pgmap v390: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 644 op/s; 8936/214350 objects degraded (4.169%); 2.4 MiB/s, 57 objects/s recovering 2023-09-23T19:24:44.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:43 smithi092 bash[58855]: audit 2023-09-23T19:24:43.626415+0000 mon.a (mon.0) 645 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:24:44.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:43 smithi092 bash[58855]: cluster 2023-09-23T19:24:43.767319+0000 mgr.y (mgr.25325) 478 : cluster 0 pgmap v390: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 644 op/s; 8936/214350 objects degraded (4.169%); 2.4 MiB/s, 57 objects/s recovering 2023-09-23T19:24:45.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:44 smithi016 bash[77995]: audit 2023-09-23T19:24:43.979505+0000 mon.a (mon.0) 646 : audit 0 from='client.? 172.21.15.16:0/2334122240' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:45.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:44 smithi016 bash[77995]: cluster 2023-09-23T19:24:44.135549+0000 mon.a (mon.0) 647 : cluster 3 Health check update: Degraded data redundancy: 8936/214350 objects degraded (4.169%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:24:45.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:44 smithi016 bash[75004]: audit 2023-09-23T19:24:43.979505+0000 mon.a (mon.0) 646 : audit 0 from='client.? 172.21.15.16:0/2334122240' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:45.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:44 smithi016 bash[75004]: cluster 2023-09-23T19:24:44.135549+0000 mon.a (mon.0) 647 : cluster 3 Health check update: Degraded data redundancy: 8936/214350 objects degraded (4.169%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:24:45.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:44 smithi092 bash[58855]: audit 2023-09-23T19:24:43.979505+0000 mon.a (mon.0) 646 : audit 0 from='client.? 172.21.15.16:0/2334122240' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:45.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:44 smithi092 bash[58855]: cluster 2023-09-23T19:24:44.135549+0000 mon.a (mon.0) 647 : cluster 3 Health check update: Degraded data redundancy: 8936/214350 objects degraded (4.169%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:24:46.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:45 smithi016 bash[75004]: cluster 2023-09-23T19:24:45.768117+0000 mgr.y (mgr.25325) 479 : cluster 0 pgmap v391: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 732 op/s; 8936/208521 objects degraded (4.285%); 1.3 MiB/s, 38 objects/s recovering 2023-09-23T19:24:46.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:24:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:24:45] "GET /metrics HTTP/1.1" 200 36197 "" "Prometheus/2.33.4" 2023-09-23T19:24:46.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:45 smithi016 bash[77995]: cluster 2023-09-23T19:24:45.768117+0000 mgr.y (mgr.25325) 479 : cluster 0 pgmap v391: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 732 op/s; 8936/208521 objects degraded (4.285%); 1.3 MiB/s, 38 objects/s recovering 2023-09-23T19:24:46.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:45 smithi092 bash[58855]: cluster 2023-09-23T19:24:45.768117+0000 mgr.y (mgr.25325) 479 : cluster 0 pgmap v391: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 732 op/s; 8936/208521 objects degraded (4.285%); 1.3 MiB/s, 38 objects/s recovering 2023-09-23T19:24:47.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:46 smithi016 bash[75004]: audit 2023-09-23T19:24:46.563034+0000 mon.a (mon.0) 648 : audit 0 from='client.? 172.21.15.16:0/611101644' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:47.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:46 smithi016 bash[77995]: audit 2023-09-23T19:24:46.563034+0000 mon.a (mon.0) 648 : audit 0 from='client.? 172.21.15.16:0/611101644' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:47.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:46 smithi092 bash[58855]: audit 2023-09-23T19:24:46.563034+0000 mon.a (mon.0) 648 : audit 0 from='client.? 172.21.15.16:0/611101644' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:48.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:47 smithi016 bash[75004]: cluster 2023-09-23T19:24:47.768792+0000 mgr.y (mgr.25325) 480 : cluster 0 pgmap v392: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 8936/206457 objects degraded (4.328%); 1.3 MiB/s, 38 objects/s recovering 2023-09-23T19:24:48.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:47 smithi016 bash[77995]: cluster 2023-09-23T19:24:47.768792+0000 mgr.y (mgr.25325) 480 : cluster 0 pgmap v392: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 8936/206457 objects degraded (4.328%); 1.3 MiB/s, 38 objects/s recovering 2023-09-23T19:24:48.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:47 smithi092 bash[58855]: cluster 2023-09-23T19:24:47.768792+0000 mgr.y (mgr.25325) 480 : cluster 0 pgmap v392: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 8936/206457 objects degraded (4.328%); 1.3 MiB/s, 38 objects/s recovering 2023-09-23T19:24:49.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:48 smithi016 bash[75004]: cluster 2023-09-23T19:24:47.877211+0000 mon.a (mon.0) 649 : cluster 0 osdmap e103: 8 total, 8 up, 8 in 2023-09-23T19:24:49.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:48 smithi016 bash[75004]: cluster 2023-09-23T19:24:47.877272+0000 osd.6 (osd.6) 21 : cluster 3 failed to encode map e103 with expected crc 2023-09-23T19:24:49.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:48 smithi016 bash[75004]: cluster 2023-09-23T19:24:47.882740+0000 osd.2 (osd.2) 26 : cluster 3 failed to encode map e103 with expected crc 2023-09-23T19:24:49.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:48 smithi016 bash[75004]: cluster 2023-09-23T19:24:47.882873+0000 osd.3 (osd.3) 32 : cluster 3 failed to encode map e103 with expected crc 2023-09-23T19:24:49.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:48 smithi016 bash[75004]: cluster 2023-09-23T19:24:47.884991+0000 osd.2 (osd.2) 27 : cluster 3 failed to encode map e103 with expected crc 2023-09-23T19:24:49.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:48 smithi016 bash[75004]: cluster 2023-09-23T19:24:47.885149+0000 osd.3 (osd.3) 33 : cluster 3 failed to encode map e103 with expected crc 2023-09-23T19:24:49.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:48 smithi016 bash[75004]: cluster 2023-09-23T19:24:47.885634+0000 osd.4 (osd.4) 22 : cluster 3 failed to encode map e103 with expected crc 2023-09-23T19:24:49.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:48 smithi016 bash[75004]: cluster 2023-09-23T19:24:47.889820+0000 osd.7 (osd.7) 22 : cluster 3 failed to encode map e103 with expected crc 2023-09-23T19:24:49.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:48 smithi016 bash[77995]: cluster 2023-09-23T19:24:47.877211+0000 mon.a (mon.0) 649 : cluster 0 osdmap e103: 8 total, 8 up, 8 in 2023-09-23T19:24:49.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:48 smithi016 bash[77995]: cluster 2023-09-23T19:24:47.877272+0000 osd.6 (osd.6) 21 : cluster 3 failed to encode map e103 with expected crc 2023-09-23T19:24:49.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:48 smithi016 bash[77995]: cluster 2023-09-23T19:24:47.882740+0000 osd.2 (osd.2) 26 : cluster 3 failed to encode map e103 with expected crc 2023-09-23T19:24:49.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:48 smithi016 bash[77995]: cluster 2023-09-23T19:24:47.882873+0000 osd.3 (osd.3) 32 : cluster 3 failed to encode map e103 with expected crc 2023-09-23T19:24:49.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:48 smithi016 bash[77995]: cluster 2023-09-23T19:24:47.884991+0000 osd.2 (osd.2) 27 : cluster 3 failed to encode map e103 with expected crc 2023-09-23T19:24:49.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:48 smithi016 bash[77995]: cluster 2023-09-23T19:24:47.885149+0000 osd.3 (osd.3) 33 : cluster 3 failed to encode map e103 with expected crc 2023-09-23T19:24:49.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:48 smithi016 bash[77995]: cluster 2023-09-23T19:24:47.885634+0000 osd.4 (osd.4) 22 : cluster 3 failed to encode map e103 with expected crc 2023-09-23T19:24:49.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:48 smithi016 bash[77995]: cluster 2023-09-23T19:24:47.889820+0000 osd.7 (osd.7) 22 : cluster 3 failed to encode map e103 with expected crc 2023-09-23T19:24:49.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:48 smithi092 bash[58855]: cluster 2023-09-23T19:24:47.877211+0000 mon.a (mon.0) 649 : cluster 0 osdmap e103: 8 total, 8 up, 8 in 2023-09-23T19:24:49.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:48 smithi092 bash[58855]: cluster 2023-09-23T19:24:47.877272+0000 osd.6 (osd.6) 21 : cluster 3 failed to encode map e103 with expected crc 2023-09-23T19:24:49.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:48 smithi092 bash[58855]: cluster 2023-09-23T19:24:47.882740+0000 osd.2 (osd.2) 26 : cluster 3 failed to encode map e103 with expected crc 2023-09-23T19:24:49.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:48 smithi092 bash[58855]: cluster 2023-09-23T19:24:47.882873+0000 osd.3 (osd.3) 32 : cluster 3 failed to encode map e103 with expected crc 2023-09-23T19:24:49.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:48 smithi092 bash[58855]: cluster 2023-09-23T19:24:47.884991+0000 osd.2 (osd.2) 27 : cluster 3 failed to encode map e103 with expected crc 2023-09-23T19:24:49.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:48 smithi092 bash[58855]: cluster 2023-09-23T19:24:47.885149+0000 osd.3 (osd.3) 33 : cluster 3 failed to encode map e103 with expected crc 2023-09-23T19:24:49.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:48 smithi092 bash[58855]: cluster 2023-09-23T19:24:47.885634+0000 osd.4 (osd.4) 22 : cluster 3 failed to encode map e103 with expected crc 2023-09-23T19:24:49.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:48 smithi092 bash[58855]: cluster 2023-09-23T19:24:47.889820+0000 osd.7 (osd.7) 22 : cluster 3 failed to encode map e103 with expected crc 2023-09-23T19:24:50.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:49 smithi016 bash[75004]: cluster 2023-09-23T19:24:48.894489+0000 osd.3 (osd.3) 34 : cluster 3 failed to encode map e104 with expected crc 2023-09-23T19:24:50.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:49 smithi016 bash[75004]: cluster 2023-09-23T19:24:48.896944+0000 mon.a (mon.0) 650 : cluster 0 osdmap e104: 8 total, 8 up, 8 in 2023-09-23T19:24:50.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:49 smithi016 bash[75004]: cluster 2023-09-23T19:24:48.897404+0000 osd.2 (osd.2) 28 : cluster 3 failed to encode map e104 with expected crc 2023-09-23T19:24:50.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:49 smithi016 bash[75004]: cluster 2023-09-23T19:24:48.908395+0000 osd.5 (osd.5) 22 : cluster 3 failed to encode map e104 with expected crc 2023-09-23T19:24:50.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:49 smithi016 bash[75004]: cluster 2023-09-23T19:24:48.955732+0000 osd.6 (osd.6) 22 : cluster 3 failed to encode map e104 with expected crc 2023-09-23T19:24:50.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:49 smithi016 bash[75004]: audit 2023-09-23T19:24:48.970406+0000 mon.c (mon.2) 85 : audit 0 from='client.? 172.21.15.16:0/1542022414' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:50.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:49 smithi016 bash[75004]: cluster 2023-09-23T19:24:49.136719+0000 mon.a (mon.0) 651 : cluster 3 Health check update: Degraded data redundancy: 8936/206457 objects degraded (4.328%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:24:50.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:49 smithi016 bash[75004]: cluster 2023-09-23T19:24:49.769383+0000 mgr.y (mgr.25325) 481 : cluster 0 pgmap v395: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 508 op/s; 8936/204705 objects degraded (4.365%); 80 KiB/s, 28 objects/s recovering 2023-09-23T19:24:50.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:49 smithi016 bash[77995]: cluster 2023-09-23T19:24:48.894489+0000 osd.3 (osd.3) 34 : cluster 3 failed to encode map e104 with expected crc 2023-09-23T19:24:50.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:49 smithi016 bash[77995]: cluster 2023-09-23T19:24:48.896944+0000 mon.a (mon.0) 650 : cluster 0 osdmap e104: 8 total, 8 up, 8 in 2023-09-23T19:24:50.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:49 smithi016 bash[77995]: cluster 2023-09-23T19:24:48.897404+0000 osd.2 (osd.2) 28 : cluster 3 failed to encode map e104 with expected crc 2023-09-23T19:24:50.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:49 smithi016 bash[77995]: cluster 2023-09-23T19:24:48.908395+0000 osd.5 (osd.5) 22 : cluster 3 failed to encode map e104 with expected crc 2023-09-23T19:24:50.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:49 smithi016 bash[77995]: cluster 2023-09-23T19:24:48.955732+0000 osd.6 (osd.6) 22 : cluster 3 failed to encode map e104 with expected crc 2023-09-23T19:24:50.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:49 smithi016 bash[77995]: audit 2023-09-23T19:24:48.970406+0000 mon.c (mon.2) 85 : audit 0 from='client.? 172.21.15.16:0/1542022414' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:50.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:49 smithi016 bash[77995]: cluster 2023-09-23T19:24:49.136719+0000 mon.a (mon.0) 651 : cluster 3 Health check update: Degraded data redundancy: 8936/206457 objects degraded (4.328%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:24:50.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:49 smithi016 bash[77995]: cluster 2023-09-23T19:24:49.769383+0000 mgr.y (mgr.25325) 481 : cluster 0 pgmap v395: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 508 op/s; 8936/204705 objects degraded (4.365%); 80 KiB/s, 28 objects/s recovering 2023-09-23T19:24:50.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:49 smithi092 bash[58855]: cluster 2023-09-23T19:24:48.894489+0000 osd.3 (osd.3) 34 : cluster 3 failed to encode map e104 with expected crc 2023-09-23T19:24:50.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:49 smithi092 bash[58855]: cluster 2023-09-23T19:24:48.896944+0000 mon.a (mon.0) 650 : cluster 0 osdmap e104: 8 total, 8 up, 8 in 2023-09-23T19:24:50.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:49 smithi092 bash[58855]: cluster 2023-09-23T19:24:48.897404+0000 osd.2 (osd.2) 28 : cluster 3 failed to encode map e104 with expected crc 2023-09-23T19:24:50.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:49 smithi092 bash[58855]: cluster 2023-09-23T19:24:48.908395+0000 osd.5 (osd.5) 22 : cluster 3 failed to encode map e104 with expected crc 2023-09-23T19:24:50.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:49 smithi092 bash[58855]: cluster 2023-09-23T19:24:48.955732+0000 osd.6 (osd.6) 22 : cluster 3 failed to encode map e104 with expected crc 2023-09-23T19:24:50.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:49 smithi092 bash[58855]: audit 2023-09-23T19:24:48.970406+0000 mon.c (mon.2) 85 : audit 0 from='client.? 172.21.15.16:0/1542022414' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:50.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:49 smithi092 bash[58855]: cluster 2023-09-23T19:24:49.136719+0000 mon.a (mon.0) 651 : cluster 3 Health check update: Degraded data redundancy: 8936/206457 objects degraded (4.328%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:24:50.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:49 smithi092 bash[58855]: cluster 2023-09-23T19:24:49.769383+0000 mgr.y (mgr.25325) 481 : cluster 0 pgmap v395: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 508 op/s; 8936/204705 objects degraded (4.365%); 80 KiB/s, 28 objects/s recovering 2023-09-23T19:24:52.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:51 smithi016 bash[77995]: audit 2023-09-23T19:24:51.595741+0000 mon.a (mon.0) 652 : audit 0 from='client.? 172.21.15.16:0/1294422557' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:52.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:51 smithi016 bash[75004]: audit 2023-09-23T19:24:51.595741+0000 mon.a (mon.0) 652 : audit 0 from='client.? 172.21.15.16:0/1294422557' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:52.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:51 smithi092 bash[58855]: audit 2023-09-23T19:24:51.595741+0000 mon.a (mon.0) 652 : audit 0 from='client.? 172.21.15.16:0/1294422557' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:53.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:52 smithi016 bash[75004]: cluster 2023-09-23T19:24:51.770256+0000 mgr.y (mgr.25325) 482 : cluster 0 pgmap v396: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 714 op/s; 8585/197196 objects degraded (4.354%); 0 B/s, 43 objects/s recovering 2023-09-23T19:24:53.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:52 smithi016 bash[77995]: cluster 2023-09-23T19:24:51.770256+0000 mgr.y (mgr.25325) 482 : cluster 0 pgmap v396: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 714 op/s; 8585/197196 objects degraded (4.354%); 0 B/s, 43 objects/s recovering 2023-09-23T19:24:53.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:52 smithi092 bash[58855]: cluster 2023-09-23T19:24:51.770256+0000 mgr.y (mgr.25325) 482 : cluster 0 pgmap v396: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 714 op/s; 8585/197196 objects degraded (4.354%); 0 B/s, 43 objects/s recovering 2023-09-23T19:24:55.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:54 smithi016 bash[75004]: cluster 2023-09-23T19:24:53.770905+0000 mgr.y (mgr.25325) 483 : cluster 0 pgmap v397: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 8585/195519 objects degraded (4.391%); 0 B/s, 43 objects/s recovering 2023-09-23T19:24:55.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:54 smithi016 bash[75004]: cluster 2023-09-23T19:24:54.138006+0000 mon.a (mon.0) 653 : cluster 3 Health check update: Degraded data redundancy: 8585/195519 objects degraded (4.391%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:24:55.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:54 smithi016 bash[75004]: audit 2023-09-23T19:24:54.212347+0000 mon.c (mon.2) 86 : audit 0 from='client.? 172.21.15.16:0/3929967296' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:55.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:54 smithi016 bash[77995]: cluster 2023-09-23T19:24:53.770905+0000 mgr.y (mgr.25325) 483 : cluster 0 pgmap v397: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 8585/195519 objects degraded (4.391%); 0 B/s, 43 objects/s recovering 2023-09-23T19:24:55.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:54 smithi016 bash[77995]: cluster 2023-09-23T19:24:54.138006+0000 mon.a (mon.0) 653 : cluster 3 Health check update: Degraded data redundancy: 8585/195519 objects degraded (4.391%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:24:55.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:54 smithi016 bash[77995]: audit 2023-09-23T19:24:54.212347+0000 mon.c (mon.2) 86 : audit 0 from='client.? 172.21.15.16:0/3929967296' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:55.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:54 smithi092 bash[58855]: cluster 2023-09-23T19:24:53.770905+0000 mgr.y (mgr.25325) 483 : cluster 0 pgmap v397: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 8585/195519 objects degraded (4.391%); 0 B/s, 43 objects/s recovering 2023-09-23T19:24:55.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:54 smithi092 bash[58855]: cluster 2023-09-23T19:24:54.138006+0000 mon.a (mon.0) 653 : cluster 3 Health check update: Degraded data redundancy: 8585/195519 objects degraded (4.391%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:24:55.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:54 smithi092 bash[58855]: audit 2023-09-23T19:24:54.212347+0000 mon.c (mon.2) 86 : audit 0 from='client.? 172.21.15.16:0/3929967296' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:56.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:24:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:24:55] "GET /metrics HTTP/1.1" 200 36197 "" "Prometheus/2.33.4" 2023-09-23T19:24:57.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:56 smithi016 bash[75004]: cluster 2023-09-23T19:24:55.771680+0000 mgr.y (mgr.25325) 484 : cluster 0 pgmap v398: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 8411/189039 objects degraded (4.449%); 0 B/s, 65 objects/s recovering 2023-09-23T19:24:57.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:56 smithi016 bash[75004]: audit 2023-09-23T19:24:55.979814+0000 mon.a (mon.0) 654 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:24:57.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:56 smithi016 bash[77995]: cluster 2023-09-23T19:24:55.771680+0000 mgr.y (mgr.25325) 484 : cluster 0 pgmap v398: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 8411/189039 objects degraded (4.449%); 0 B/s, 65 objects/s recovering 2023-09-23T19:24:57.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:56 smithi016 bash[77995]: audit 2023-09-23T19:24:55.979814+0000 mon.a (mon.0) 654 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:24:57.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:56 smithi092 bash[58855]: cluster 2023-09-23T19:24:55.771680+0000 mgr.y (mgr.25325) 484 : cluster 0 pgmap v398: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 8411/189039 objects degraded (4.449%); 0 B/s, 65 objects/s recovering 2023-09-23T19:24:57.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:56 smithi092 bash[58855]: audit 2023-09-23T19:24:55.979814+0000 mon.a (mon.0) 654 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:24:58.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:57 smithi016 bash[75004]: audit 2023-09-23T19:24:56.852747+0000 mon.c (mon.2) 87 : audit 0 from='client.? 172.21.15.16:0/3805560949' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:58.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:57 smithi016 bash[77995]: audit 2023-09-23T19:24:56.852747+0000 mon.c (mon.2) 87 : audit 0 from='client.? 172.21.15.16:0/3805560949' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:58.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:57 smithi092 bash[58855]: audit 2023-09-23T19:24:56.852747+0000 mon.c (mon.2) 87 : audit 0 from='client.? 172.21.15.16:0/3805560949' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:24:59.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:58 smithi016 bash[75004]: cluster 2023-09-23T19:24:57.772362+0000 mgr.y (mgr.25325) 485 : cluster 0 pgmap v399: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 8411/187074 objects degraded (4.496%); 0 B/s, 53 objects/s recovering 2023-09-23T19:24:59.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:58 smithi016 bash[75004]: audit 2023-09-23T19:24:58.626596+0000 mon.a (mon.0) 655 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:24:59.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:58 smithi016 bash[77995]: cluster 2023-09-23T19:24:57.772362+0000 mgr.y (mgr.25325) 485 : cluster 0 pgmap v399: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 8411/187074 objects degraded (4.496%); 0 B/s, 53 objects/s recovering 2023-09-23T19:24:59.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:58 smithi016 bash[77995]: audit 2023-09-23T19:24:58.626596+0000 mon.a (mon.0) 655 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:24:59.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:58 smithi092 bash[58855]: cluster 2023-09-23T19:24:57.772362+0000 mgr.y (mgr.25325) 485 : cluster 0 pgmap v399: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 8411/187074 objects degraded (4.496%); 0 B/s, 53 objects/s recovering 2023-09-23T19:24:59.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:58 smithi092 bash[58855]: audit 2023-09-23T19:24:58.626596+0000 mon.a (mon.0) 655 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:25:00.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:59 smithi016 bash[75004]: cluster 2023-09-23T19:24:59.138946+0000 mon.a (mon.0) 656 : cluster 3 Health check update: Degraded data redundancy: 8411/187074 objects degraded (4.496%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:25:00.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:59 smithi016 bash[75004]: audit 2023-09-23T19:24:59.478929+0000 mon.b (mon.1) 31 : audit 0 from='client.? 172.21.15.16:0/3590948621' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:00.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:24:59 smithi016 bash[75004]: cluster 2023-09-23T19:24:59.772984+0000 mgr.y (mgr.25325) 486 : cluster 0 pgmap v400: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 8411/185211 objects degraded (4.541%); 0 B/s, 48 objects/s recovering 2023-09-23T19:25:00.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:59 smithi016 bash[77995]: cluster 2023-09-23T19:24:59.138946+0000 mon.a (mon.0) 656 : cluster 3 Health check update: Degraded data redundancy: 8411/187074 objects degraded (4.496%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:25:00.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:59 smithi016 bash[77995]: audit 2023-09-23T19:24:59.478929+0000 mon.b (mon.1) 31 : audit 0 from='client.? 172.21.15.16:0/3590948621' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:00.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:24:59 smithi016 bash[77995]: cluster 2023-09-23T19:24:59.772984+0000 mgr.y (mgr.25325) 486 : cluster 0 pgmap v400: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 8411/185211 objects degraded (4.541%); 0 B/s, 48 objects/s recovering 2023-09-23T19:25:00.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:59 smithi092 bash[58855]: cluster 2023-09-23T19:24:59.138946+0000 mon.a (mon.0) 656 : cluster 3 Health check update: Degraded data redundancy: 8411/187074 objects degraded (4.496%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:25:00.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:59 smithi092 bash[58855]: audit 2023-09-23T19:24:59.478929+0000 mon.b (mon.1) 31 : audit 0 from='client.? 172.21.15.16:0/3590948621' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:00.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:24:59 smithi092 bash[58855]: cluster 2023-09-23T19:24:59.772984+0000 mgr.y (mgr.25325) 486 : cluster 0 pgmap v400: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 8411/185211 objects degraded (4.541%); 0 B/s, 48 objects/s recovering 2023-09-23T19:25:03.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:02 smithi016 bash[77995]: cluster 2023-09-23T19:25:01.773831+0000 mgr.y (mgr.25325) 487 : cluster 0 pgmap v401: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 767 op/s; 8235/177063 objects degraded (4.651%); 0 B/s, 58 objects/s recovering 2023-09-23T19:25:03.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:02 smithi016 bash[77995]: audit 2023-09-23T19:25:02.134211+0000 mon.c (mon.2) 88 : audit 0 from='client.? 172.21.15.16:0/3800649920' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:03.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:02 smithi016 bash[77995]: audit 2023-09-23T19:25:02.448664+0000 mon.a (mon.0) 657 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:25:03.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:02 smithi016 bash[77995]: audit 2023-09-23T19:25:02.458128+0000 mon.a (mon.0) 658 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:25:03.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:02 smithi016 bash[77995]: audit 2023-09-23T19:25:02.587708+0000 mon.a (mon.0) 659 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:25:03.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:02 smithi016 bash[77995]: audit 2023-09-23T19:25:02.596081+0000 mon.a (mon.0) 660 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:25:03.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:02 smithi016 bash[75004]: cluster 2023-09-23T19:25:01.773831+0000 mgr.y (mgr.25325) 487 : cluster 0 pgmap v401: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 767 op/s; 8235/177063 objects degraded (4.651%); 0 B/s, 58 objects/s recovering 2023-09-23T19:25:03.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:02 smithi016 bash[75004]: audit 2023-09-23T19:25:02.134211+0000 mon.c (mon.2) 88 : audit 0 from='client.? 172.21.15.16:0/3800649920' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:03.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:02 smithi016 bash[75004]: audit 2023-09-23T19:25:02.448664+0000 mon.a (mon.0) 657 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:25:03.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:02 smithi016 bash[75004]: audit 2023-09-23T19:25:02.458128+0000 mon.a (mon.0) 658 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:25:03.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:02 smithi016 bash[75004]: audit 2023-09-23T19:25:02.587708+0000 mon.a (mon.0) 659 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:25:03.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:02 smithi016 bash[75004]: audit 2023-09-23T19:25:02.596081+0000 mon.a (mon.0) 660 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:25:03.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:02 smithi092 bash[58855]: cluster 2023-09-23T19:25:01.773831+0000 mgr.y (mgr.25325) 487 : cluster 0 pgmap v401: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 767 op/s; 8235/177063 objects degraded (4.651%); 0 B/s, 58 objects/s recovering 2023-09-23T19:25:03.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:02 smithi092 bash[58855]: audit 2023-09-23T19:25:02.134211+0000 mon.c (mon.2) 88 : audit 0 from='client.? 172.21.15.16:0/3800649920' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:03.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:02 smithi092 bash[58855]: audit 2023-09-23T19:25:02.448664+0000 mon.a (mon.0) 657 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:25:03.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:02 smithi092 bash[58855]: audit 2023-09-23T19:25:02.458128+0000 mon.a (mon.0) 658 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:25:03.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:02 smithi092 bash[58855]: audit 2023-09-23T19:25:02.587708+0000 mon.a (mon.0) 659 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:25:03.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:02 smithi092 bash[58855]: audit 2023-09-23T19:25:02.596081+0000 mon.a (mon.0) 660 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:25:03.784 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:25:03 smithi016 bash[70992]: debug 2023-09-23T19:25:03.359+0000 7fb2a5a7c700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T19:25:04.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:03 smithi016 bash[77995]: audit 2023-09-23T19:25:03.237738+0000 mon.a (mon.0) 661 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:25:04.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:03 smithi016 bash[77995]: audit 2023-09-23T19:25:03.238855+0000 mon.a (mon.0) 662 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:25:04.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:03 smithi016 bash[77995]: audit 2023-09-23T19:25:03.249628+0000 mon.a (mon.0) 663 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:25:04.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:03 smithi016 bash[77995]: audit 2023-09-23T19:25:03.343841+0000 mon.a (mon.0) 664 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:25:04.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:03 smithi016 bash[77995]: audit 2023-09-23T19:25:03.352951+0000 mon.a (mon.0) 665 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:25:04.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:03 smithi016 bash[77995]: audit 2023-09-23T19:25:03.355785+0000 mon.a (mon.0) 666 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:25:04.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:03 smithi016 bash[77995]: audit 2023-09-23T19:25:03.358007+0000 mon.a (mon.0) 667 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:25:04.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:03 smithi016 bash[77995]: audit 2023-09-23T19:25:03.360415+0000 mon.a (mon.0) 668 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:25:04.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:03 smithi016 bash[75004]: audit 2023-09-23T19:25:03.237738+0000 mon.a (mon.0) 661 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:25:04.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:03 smithi016 bash[75004]: audit 2023-09-23T19:25:03.238855+0000 mon.a (mon.0) 662 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:25:04.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:03 smithi016 bash[75004]: audit 2023-09-23T19:25:03.249628+0000 mon.a (mon.0) 663 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:25:04.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:03 smithi016 bash[75004]: audit 2023-09-23T19:25:03.343841+0000 mon.a (mon.0) 664 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:25:04.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:03 smithi016 bash[75004]: audit 2023-09-23T19:25:03.352951+0000 mon.a (mon.0) 665 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:25:04.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:03 smithi016 bash[75004]: audit 2023-09-23T19:25:03.355785+0000 mon.a (mon.0) 666 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:25:04.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:03 smithi016 bash[75004]: audit 2023-09-23T19:25:03.358007+0000 mon.a (mon.0) 667 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:25:04.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:03 smithi016 bash[75004]: audit 2023-09-23T19:25:03.360415+0000 mon.a (mon.0) 668 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:25:04.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:03 smithi092 bash[58855]: audit 2023-09-23T19:25:03.237738+0000 mon.a (mon.0) 661 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:25:04.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:03 smithi092 bash[58855]: audit 2023-09-23T19:25:03.238855+0000 mon.a (mon.0) 662 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:25:04.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:03 smithi092 bash[58855]: audit 2023-09-23T19:25:03.249628+0000 mon.a (mon.0) 663 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:25:04.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:03 smithi092 bash[58855]: audit 2023-09-23T19:25:03.343841+0000 mon.a (mon.0) 664 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:25:04.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:03 smithi092 bash[58855]: audit 2023-09-23T19:25:03.352951+0000 mon.a (mon.0) 665 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:25:04.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:03 smithi092 bash[58855]: audit 2023-09-23T19:25:03.355785+0000 mon.a (mon.0) 666 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:25:04.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:03 smithi092 bash[58855]: audit 2023-09-23T19:25:03.358007+0000 mon.a (mon.0) 667 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:25:04.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:03 smithi092 bash[58855]: audit 2023-09-23T19:25:03.360415+0000 mon.a (mon.0) 668 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:25:05.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:04 smithi016 bash[77995]: audit 2023-09-23T19:25:03.360683+0000 mgr.y (mgr.25325) 488 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:25:05.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:04 smithi016 bash[77995]: cephadm 2023-09-23T19:25:03.361908+0000 mgr.y (mgr.25325) 489 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T19:25:05.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:04 smithi016 bash[77995]: cluster 2023-09-23T19:25:03.776840+0000 mgr.y (mgr.25325) 490 : cluster 0 pgmap v402: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 8235/175314 objects degraded (4.697%); 0 B/s, 29 objects/s recovering 2023-09-23T19:25:05.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:04 smithi016 bash[77995]: cluster 2023-09-23T19:25:04.140163+0000 mon.a (mon.0) 669 : cluster 3 Health check update: Degraded data redundancy: 8235/177063 objects degraded (4.651%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:25:05.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:04 smithi016 bash[77995]: audit 2023-09-23T19:25:04.757164+0000 mon.c (mon.2) 89 : audit 0 from='client.? 172.21.15.16:0/3677249570' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:05.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:04 smithi016 bash[75004]: audit 2023-09-23T19:25:03.360683+0000 mgr.y (mgr.25325) 488 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:25:05.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:04 smithi016 bash[75004]: cephadm 2023-09-23T19:25:03.361908+0000 mgr.y (mgr.25325) 489 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T19:25:05.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:04 smithi016 bash[75004]: cluster 2023-09-23T19:25:03.776840+0000 mgr.y (mgr.25325) 490 : cluster 0 pgmap v402: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 8235/175314 objects degraded (4.697%); 0 B/s, 29 objects/s recovering 2023-09-23T19:25:05.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:04 smithi016 bash[75004]: cluster 2023-09-23T19:25:04.140163+0000 mon.a (mon.0) 669 : cluster 3 Health check update: Degraded data redundancy: 8235/177063 objects degraded (4.651%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:25:05.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:04 smithi016 bash[75004]: audit 2023-09-23T19:25:04.757164+0000 mon.c (mon.2) 89 : audit 0 from='client.? 172.21.15.16:0/3677249570' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:05.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:04 smithi092 bash[58855]: audit 2023-09-23T19:25:03.360683+0000 mgr.y (mgr.25325) 488 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:25:05.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:04 smithi092 bash[58855]: cephadm 2023-09-23T19:25:03.361908+0000 mgr.y (mgr.25325) 489 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T19:25:05.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:04 smithi092 bash[58855]: cluster 2023-09-23T19:25:03.776840+0000 mgr.y (mgr.25325) 490 : cluster 0 pgmap v402: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 8235/175314 objects degraded (4.697%); 0 B/s, 29 objects/s recovering 2023-09-23T19:25:05.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:04 smithi092 bash[58855]: cluster 2023-09-23T19:25:04.140163+0000 mon.a (mon.0) 669 : cluster 3 Health check update: Degraded data redundancy: 8235/177063 objects degraded (4.651%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:25:05.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:04 smithi092 bash[58855]: audit 2023-09-23T19:25:04.757164+0000 mon.c (mon.2) 89 : audit 0 from='client.? 172.21.15.16:0/3677249570' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:06.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:25:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:25:05] "GET /metrics HTTP/1.1" 200 36185 "" "Prometheus/2.33.4" 2023-09-23T19:25:07.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:06 smithi016 bash[77995]: cluster 2023-09-23T19:25:05.777593+0000 mgr.y (mgr.25325) 491 : cluster 0 pgmap v403: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 8079/170133 objects degraded (4.749%); 0 B/s, 42 objects/s recovering 2023-09-23T19:25:07.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:06 smithi016 bash[75004]: cluster 2023-09-23T19:25:05.777593+0000 mgr.y (mgr.25325) 491 : cluster 0 pgmap v403: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 8079/170133 objects degraded (4.749%); 0 B/s, 42 objects/s recovering 2023-09-23T19:25:07.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:06 smithi092 bash[58855]: cluster 2023-09-23T19:25:05.777593+0000 mgr.y (mgr.25325) 491 : cluster 0 pgmap v403: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 8079/170133 objects degraded (4.749%); 0 B/s, 42 objects/s recovering 2023-09-23T19:25:08.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:07 smithi016 bash[77995]: audit 2023-09-23T19:25:07.358286+0000 mon.a (mon.0) 670 : audit 0 from='client.? 172.21.15.16:0/2651442910' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:08.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:07 smithi016 bash[77995]: cluster 2023-09-23T19:25:07.778212+0000 mgr.y (mgr.25325) 492 : cluster 0 pgmap v404: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 8079/167613 objects degraded (4.820%); 0 B/s, 27 objects/s recovering 2023-09-23T19:25:08.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:07 smithi016 bash[75004]: audit 2023-09-23T19:25:07.358286+0000 mon.a (mon.0) 670 : audit 0 from='client.? 172.21.15.16:0/2651442910' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:08.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:07 smithi016 bash[75004]: cluster 2023-09-23T19:25:07.778212+0000 mgr.y (mgr.25325) 492 : cluster 0 pgmap v404: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 8079/167613 objects degraded (4.820%); 0 B/s, 27 objects/s recovering 2023-09-23T19:25:08.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:07 smithi092 bash[58855]: audit 2023-09-23T19:25:07.358286+0000 mon.a (mon.0) 670 : audit 0 from='client.? 172.21.15.16:0/2651442910' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:08.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:07 smithi092 bash[58855]: cluster 2023-09-23T19:25:07.778212+0000 mgr.y (mgr.25325) 492 : cluster 0 pgmap v404: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 8079/167613 objects degraded (4.820%); 0 B/s, 27 objects/s recovering 2023-09-23T19:25:10.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:10 smithi016 bash[77995]: cluster 2023-09-23T19:25:09.141351+0000 mon.a (mon.0) 671 : cluster 3 Health check update: Degraded data redundancy: 8079/167613 objects degraded (4.820%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:25:10.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:10 smithi016 bash[77995]: cluster 2023-09-23T19:25:09.778851+0000 mgr.y (mgr.25325) 493 : cluster 0 pgmap v405: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 8079/165792 objects degraded (4.873%); 0 B/s, 27 objects/s recovering 2023-09-23T19:25:10.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:10 smithi016 bash[77995]: audit 2023-09-23T19:25:09.948205+0000 mon.c (mon.2) 90 : audit 0 from='client.? 172.21.15.16:0/148938857' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:10.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:10 smithi016 bash[75004]: cluster 2023-09-23T19:25:09.141351+0000 mon.a (mon.0) 671 : cluster 3 Health check update: Degraded data redundancy: 8079/167613 objects degraded (4.820%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:25:10.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:10 smithi016 bash[75004]: cluster 2023-09-23T19:25:09.778851+0000 mgr.y (mgr.25325) 493 : cluster 0 pgmap v405: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 8079/165792 objects degraded (4.873%); 0 B/s, 27 objects/s recovering 2023-09-23T19:25:10.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:10 smithi016 bash[75004]: audit 2023-09-23T19:25:09.948205+0000 mon.c (mon.2) 90 : audit 0 from='client.? 172.21.15.16:0/148938857' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:10.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:10 smithi092 bash[58855]: cluster 2023-09-23T19:25:09.141351+0000 mon.a (mon.0) 671 : cluster 3 Health check update: Degraded data redundancy: 8079/167613 objects degraded (4.820%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:25:10.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:10 smithi092 bash[58855]: cluster 2023-09-23T19:25:09.778851+0000 mgr.y (mgr.25325) 493 : cluster 0 pgmap v405: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 8079/165792 objects degraded (4.873%); 0 B/s, 27 objects/s recovering 2023-09-23T19:25:10.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:10 smithi092 bash[58855]: audit 2023-09-23T19:25:09.948205+0000 mon.c (mon.2) 90 : audit 0 from='client.? 172.21.15.16:0/148938857' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:13.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:12 smithi016 bash[75004]: cluster 2023-09-23T19:25:11.779826+0000 mgr.y (mgr.25325) 494 : cluster 0 pgmap v406: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 7896/157743 objects degraded (5.006%); 0 B/s, 42 objects/s recovering 2023-09-23T19:25:13.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:12 smithi016 bash[75004]: audit 2023-09-23T19:25:12.584640+0000 mon.c (mon.2) 91 : audit 0 from='client.? 172.21.15.16:0/2242364466' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:13.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:12 smithi016 bash[77995]: cluster 2023-09-23T19:25:11.779826+0000 mgr.y (mgr.25325) 494 : cluster 0 pgmap v406: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 7896/157743 objects degraded (5.006%); 0 B/s, 42 objects/s recovering 2023-09-23T19:25:13.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:12 smithi016 bash[77995]: audit 2023-09-23T19:25:12.584640+0000 mon.c (mon.2) 91 : audit 0 from='client.? 172.21.15.16:0/2242364466' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:13.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:12 smithi092 bash[58855]: cluster 2023-09-23T19:25:11.779826+0000 mgr.y (mgr.25325) 494 : cluster 0 pgmap v406: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 7896/157743 objects degraded (5.006%); 0 B/s, 42 objects/s recovering 2023-09-23T19:25:13.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:12 smithi092 bash[58855]: audit 2023-09-23T19:25:12.584640+0000 mon.c (mon.2) 91 : audit 0 from='client.? 172.21.15.16:0/2242364466' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:14.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:13 smithi016 bash[77995]: audit 2023-09-23T19:25:13.627029+0000 mon.a (mon.0) 672 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:25:14.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:13 smithi016 bash[75004]: audit 2023-09-23T19:25:13.627029+0000 mon.a (mon.0) 672 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:25:14.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:13 smithi092 bash[58855]: audit 2023-09-23T19:25:13.627029+0000 mon.a (mon.0) 672 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:25:15.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:14 smithi016 bash[75004]: cluster 2023-09-23T19:25:13.780457+0000 mgr.y (mgr.25325) 495 : cluster 0 pgmap v407: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 7896/155805 objects degraded (5.068%); 0 B/s, 28 objects/s recovering 2023-09-23T19:25:15.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:14 smithi016 bash[75004]: cluster 2023-09-23T19:25:14.142640+0000 mon.a (mon.0) 673 : cluster 3 Health check update: Degraded data redundancy: 7896/155805 objects degraded (5.068%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:25:15.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:14 smithi016 bash[77995]: cluster 2023-09-23T19:25:13.780457+0000 mgr.y (mgr.25325) 495 : cluster 0 pgmap v407: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 7896/155805 objects degraded (5.068%); 0 B/s, 28 objects/s recovering 2023-09-23T19:25:15.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:14 smithi016 bash[77995]: cluster 2023-09-23T19:25:14.142640+0000 mon.a (mon.0) 673 : cluster 3 Health check update: Degraded data redundancy: 7896/155805 objects degraded (5.068%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:25:15.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:14 smithi092 bash[58855]: cluster 2023-09-23T19:25:13.780457+0000 mgr.y (mgr.25325) 495 : cluster 0 pgmap v407: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 7896/155805 objects degraded (5.068%); 0 B/s, 28 objects/s recovering 2023-09-23T19:25:15.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:14 smithi092 bash[58855]: cluster 2023-09-23T19:25:14.142640+0000 mon.a (mon.0) 673 : cluster 3 Health check update: Degraded data redundancy: 7896/155805 objects degraded (5.068%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:25:16.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:15 smithi016 bash[75004]: audit 2023-09-23T19:25:15.166458+0000 mon.c (mon.2) 92 : audit 0 from='client.? 172.21.15.16:0/3206829525' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:16.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:25:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:25:15] "GET /metrics HTTP/1.1" 200 36184 "" "Prometheus/2.33.4" 2023-09-23T19:25:16.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:15 smithi016 bash[77995]: audit 2023-09-23T19:25:15.166458+0000 mon.c (mon.2) 92 : audit 0 from='client.? 172.21.15.16:0/3206829525' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:16.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:15 smithi092 bash[58855]: audit 2023-09-23T19:25:15.166458+0000 mon.c (mon.2) 92 : audit 0 from='client.? 172.21.15.16:0/3206829525' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:17.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:16 smithi016 bash[75004]: cluster 2023-09-23T19:25:15.781243+0000 mgr.y (mgr.25325) 496 : cluster 0 pgmap v408: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 7714/150117 objects degraded (5.139%); 0 B/s, 43 objects/s recovering 2023-09-23T19:25:17.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:16 smithi016 bash[77995]: cluster 2023-09-23T19:25:15.781243+0000 mgr.y (mgr.25325) 496 : cluster 0 pgmap v408: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 7714/150117 objects degraded (5.139%); 0 B/s, 43 objects/s recovering 2023-09-23T19:25:17.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:16 smithi092 bash[58855]: cluster 2023-09-23T19:25:15.781243+0000 mgr.y (mgr.25325) 496 : cluster 0 pgmap v408: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 7714/150117 objects degraded (5.139%); 0 B/s, 43 objects/s recovering 2023-09-23T19:25:18.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:17 smithi016 bash[75004]: cluster 2023-09-23T19:25:17.781905+0000 mgr.y (mgr.25325) 497 : cluster 0 pgmap v409: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 7714/147738 objects degraded (5.221%); 0 B/s, 30 objects/s recovering 2023-09-23T19:25:18.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:17 smithi016 bash[75004]: audit 2023-09-23T19:25:17.784184+0000 mon.c (mon.2) 93 : audit 0 from='client.? 172.21.15.16:0/1742130678' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:18.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:17 smithi016 bash[77995]: cluster 2023-09-23T19:25:17.781905+0000 mgr.y (mgr.25325) 497 : cluster 0 pgmap v409: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 7714/147738 objects degraded (5.221%); 0 B/s, 30 objects/s recovering 2023-09-23T19:25:18.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:17 smithi016 bash[77995]: audit 2023-09-23T19:25:17.784184+0000 mon.c (mon.2) 93 : audit 0 from='client.? 172.21.15.16:0/1742130678' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:18.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:17 smithi092 bash[58855]: cluster 2023-09-23T19:25:17.781905+0000 mgr.y (mgr.25325) 497 : cluster 0 pgmap v409: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 7714/147738 objects degraded (5.221%); 0 B/s, 30 objects/s recovering 2023-09-23T19:25:18.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:17 smithi092 bash[58855]: audit 2023-09-23T19:25:17.784184+0000 mon.c (mon.2) 93 : audit 0 from='client.? 172.21.15.16:0/1742130678' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:18.784 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:25:18 smithi016 bash[70992]: debug 2023-09-23T19:25:18.374+0000 7fb2a5a7c700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T19:25:19.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:18 smithi016 bash[77995]: audit 2023-09-23T19:25:18.376798+0000 mon.a (mon.0) 674 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:25:19.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:18 smithi016 bash[77995]: audit 2023-09-23T19:25:18.377030+0000 mgr.y (mgr.25325) 498 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:25:19.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:18 smithi016 bash[77995]: cephadm 2023-09-23T19:25:18.381435+0000 mgr.y (mgr.25325) 499 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T19:25:19.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:18 smithi016 bash[75004]: audit 2023-09-23T19:25:18.376798+0000 mon.a (mon.0) 674 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:25:19.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:18 smithi016 bash[75004]: audit 2023-09-23T19:25:18.377030+0000 mgr.y (mgr.25325) 498 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:25:19.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:18 smithi016 bash[75004]: cephadm 2023-09-23T19:25:18.381435+0000 mgr.y (mgr.25325) 499 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T19:25:19.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:18 smithi092 bash[58855]: audit 2023-09-23T19:25:18.376798+0000 mon.a (mon.0) 674 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:25:19.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:18 smithi092 bash[58855]: audit 2023-09-23T19:25:18.377030+0000 mgr.y (mgr.25325) 498 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:25:19.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:18 smithi092 bash[58855]: cephadm 2023-09-23T19:25:18.381435+0000 mgr.y (mgr.25325) 499 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T19:25:20.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:19 smithi016 bash[75004]: cluster 2023-09-23T19:25:19.143797+0000 mon.a (mon.0) 675 : cluster 3 Health check update: Degraded data redundancy: 7714/147738 objects degraded (5.221%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:25:20.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:19 smithi016 bash[75004]: cluster 2023-09-23T19:25:19.782517+0000 mgr.y (mgr.25325) 500 : cluster 0 pgmap v410: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 7714/145971 objects degraded (5.285%); 0 B/s, 30 objects/s recovering 2023-09-23T19:25:20.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:19 smithi016 bash[77995]: cluster 2023-09-23T19:25:19.143797+0000 mon.a (mon.0) 675 : cluster 3 Health check update: Degraded data redundancy: 7714/147738 objects degraded (5.221%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:25:20.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:19 smithi016 bash[77995]: cluster 2023-09-23T19:25:19.782517+0000 mgr.y (mgr.25325) 500 : cluster 0 pgmap v410: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 7714/145971 objects degraded (5.285%); 0 B/s, 30 objects/s recovering 2023-09-23T19:25:20.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:19 smithi092 bash[58855]: cluster 2023-09-23T19:25:19.143797+0000 mon.a (mon.0) 675 : cluster 3 Health check update: Degraded data redundancy: 7714/147738 objects degraded (5.221%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:25:20.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:19 smithi092 bash[58855]: cluster 2023-09-23T19:25:19.782517+0000 mgr.y (mgr.25325) 500 : cluster 0 pgmap v410: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 7714/145971 objects degraded (5.285%); 0 B/s, 30 objects/s recovering 2023-09-23T19:25:21.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:21 smithi016 bash[77995]: audit 2023-09-23T19:25:20.411829+0000 mon.c (mon.2) 94 : audit 0 from='client.? 172.21.15.16:0/871357266' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:21.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:21 smithi016 bash[75004]: audit 2023-09-23T19:25:20.411829+0000 mon.c (mon.2) 94 : audit 0 from='client.? 172.21.15.16:0/871357266' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:21.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:21 smithi092 bash[58855]: audit 2023-09-23T19:25:20.411829+0000 mon.c (mon.2) 94 : audit 0 from='client.? 172.21.15.16:0/871357266' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:22.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:22 smithi016 bash[75004]: cluster 2023-09-23T19:25:21.783433+0000 mgr.y (mgr.25325) 501 : cluster 0 pgmap v411: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 771 op/s; 7524/138015 objects degraded (5.452%); 0 B/s, 46 objects/s recovering 2023-09-23T19:25:22.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:22 smithi016 bash[77995]: cluster 2023-09-23T19:25:21.783433+0000 mgr.y (mgr.25325) 501 : cluster 0 pgmap v411: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 771 op/s; 7524/138015 objects degraded (5.452%); 0 B/s, 46 objects/s recovering 2023-09-23T19:25:22.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:22 smithi092 bash[58855]: cluster 2023-09-23T19:25:21.783433+0000 mgr.y (mgr.25325) 501 : cluster 0 pgmap v411: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 771 op/s; 7524/138015 objects degraded (5.452%); 0 B/s, 46 objects/s recovering 2023-09-23T19:25:23.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:23 smithi016 bash[77995]: audit 2023-09-23T19:25:23.003497+0000 mon.a (mon.0) 676 : audit 0 from='client.? 172.21.15.16:0/2593276048' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:23.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:23 smithi016 bash[75004]: audit 2023-09-23T19:25:23.003497+0000 mon.a (mon.0) 676 : audit 0 from='client.? 172.21.15.16:0/2593276048' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:23.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:23 smithi092 bash[58855]: audit 2023-09-23T19:25:23.003497+0000 mon.a (mon.0) 676 : audit 0 from='client.? 172.21.15.16:0/2593276048' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:24.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:24 smithi016 bash[75004]: cluster 2023-09-23T19:25:23.784116+0000 mgr.y (mgr.25325) 502 : cluster 0 pgmap v412: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 7524/136047 objects degraded (5.530%); 0 B/s, 30 objects/s recovering 2023-09-23T19:25:24.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:24 smithi016 bash[75004]: cluster 2023-09-23T19:25:24.148619+0000 mon.a (mon.0) 677 : cluster 3 Health check update: Degraded data redundancy: 7524/138015 objects degraded (5.452%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:25:24.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:24 smithi016 bash[77995]: cluster 2023-09-23T19:25:23.784116+0000 mgr.y (mgr.25325) 502 : cluster 0 pgmap v412: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 7524/136047 objects degraded (5.530%); 0 B/s, 30 objects/s recovering 2023-09-23T19:25:24.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:24 smithi016 bash[77995]: cluster 2023-09-23T19:25:24.148619+0000 mon.a (mon.0) 677 : cluster 3 Health check update: Degraded data redundancy: 7524/138015 objects degraded (5.452%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:25:24.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:24 smithi092 bash[58855]: cluster 2023-09-23T19:25:23.784116+0000 mgr.y (mgr.25325) 502 : cluster 0 pgmap v412: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 7524/136047 objects degraded (5.530%); 0 B/s, 30 objects/s recovering 2023-09-23T19:25:24.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:24 smithi092 bash[58855]: cluster 2023-09-23T19:25:24.148619+0000 mon.a (mon.0) 677 : cluster 3 Health check update: Degraded data redundancy: 7524/138015 objects degraded (5.452%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:25:26.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:25 smithi016 bash[77995]: audit 2023-09-23T19:25:25.588589+0000 mon.a (mon.0) 678 : audit 0 from='client.? 172.21.15.16:0/2849644438' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:26.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:25 smithi016 bash[75004]: audit 2023-09-23T19:25:25.588589+0000 mon.a (mon.0) 678 : audit 0 from='client.? 172.21.15.16:0/2849644438' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:26.035 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:25:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:25:25] "GET /metrics HTTP/1.1" 200 36184 "" "Prometheus/2.33.4" 2023-09-23T19:25:26.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:25 smithi092 bash[58855]: audit 2023-09-23T19:25:25.588589+0000 mon.a (mon.0) 678 : audit 0 from='client.? 172.21.15.16:0/2849644438' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:27.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:26 smithi016 bash[75004]: cluster 2023-09-23T19:25:25.784890+0000 mgr.y (mgr.25325) 503 : cluster 0 pgmap v413: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 700 op/s; 7341/130593 objects degraded (5.621%); 0 B/s, 46 objects/s recovering 2023-09-23T19:25:27.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:26 smithi016 bash[77995]: cluster 2023-09-23T19:25:25.784890+0000 mgr.y (mgr.25325) 503 : cluster 0 pgmap v413: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 700 op/s; 7341/130593 objects degraded (5.621%); 0 B/s, 46 objects/s recovering 2023-09-23T19:25:27.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:26 smithi092 bash[58855]: cluster 2023-09-23T19:25:25.784890+0000 mgr.y (mgr.25325) 503 : cluster 0 pgmap v413: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 700 op/s; 7341/130593 objects degraded (5.621%); 0 B/s, 46 objects/s recovering 2023-09-23T19:25:29.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:28 smithi016 bash[77995]: cluster 2023-09-23T19:25:27.785513+0000 mgr.y (mgr.25325) 504 : cluster 0 pgmap v414: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 7341/128196 objects degraded (5.726%); 0 B/s, 31 objects/s recovering 2023-09-23T19:25:29.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:28 smithi016 bash[77995]: audit 2023-09-23T19:25:28.198292+0000 mon.c (mon.2) 95 : audit 0 from='client.? 172.21.15.16:0/31455406' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:29.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:28 smithi016 bash[77995]: audit 2023-09-23T19:25:28.627093+0000 mon.a (mon.0) 679 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:25:29.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:28 smithi016 bash[75004]: cluster 2023-09-23T19:25:27.785513+0000 mgr.y (mgr.25325) 504 : cluster 0 pgmap v414: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 7341/128196 objects degraded (5.726%); 0 B/s, 31 objects/s recovering 2023-09-23T19:25:29.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:28 smithi016 bash[75004]: audit 2023-09-23T19:25:28.198292+0000 mon.c (mon.2) 95 : audit 0 from='client.? 172.21.15.16:0/31455406' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:29.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:28 smithi016 bash[75004]: audit 2023-09-23T19:25:28.627093+0000 mon.a (mon.0) 679 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:25:29.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:28 smithi092 bash[58855]: cluster 2023-09-23T19:25:27.785513+0000 mgr.y (mgr.25325) 504 : cluster 0 pgmap v414: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 7341/128196 objects degraded (5.726%); 0 B/s, 31 objects/s recovering 2023-09-23T19:25:29.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:28 smithi092 bash[58855]: audit 2023-09-23T19:25:28.198292+0000 mon.c (mon.2) 95 : audit 0 from='client.? 172.21.15.16:0/31455406' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:29.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:28 smithi092 bash[58855]: audit 2023-09-23T19:25:28.627093+0000 mon.a (mon.0) 679 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:25:30.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:29 smithi016 bash[77995]: cluster 2023-09-23T19:25:29.149755+0000 mon.a (mon.0) 680 : cluster 3 Health check update: Degraded data redundancy: 7341/128196 objects degraded (5.726%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:25:30.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:29 smithi016 bash[75004]: cluster 2023-09-23T19:25:29.149755+0000 mon.a (mon.0) 680 : cluster 3 Health check update: Degraded data redundancy: 7341/128196 objects degraded (5.726%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:25:30.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:29 smithi092 bash[58855]: cluster 2023-09-23T19:25:29.149755+0000 mon.a (mon.0) 680 : cluster 3 Health check update: Degraded data redundancy: 7341/128196 objects degraded (5.726%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:25:31.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:30 smithi016 bash[77995]: cluster 2023-09-23T19:25:29.786136+0000 mgr.y (mgr.25325) 505 : cluster 0 pgmap v415: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 592 op/s; 7341/126396 objects degraded (5.808%); 0 B/s, 31 objects/s recovering 2023-09-23T19:25:31.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:30 smithi016 bash[77995]: audit 2023-09-23T19:25:30.761665+0000 mon.c (mon.2) 96 : audit 0 from='client.? 172.21.15.16:0/430373309' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:31.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:30 smithi016 bash[75004]: cluster 2023-09-23T19:25:29.786136+0000 mgr.y (mgr.25325) 505 : cluster 0 pgmap v415: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 592 op/s; 7341/126396 objects degraded (5.808%); 0 B/s, 31 objects/s recovering 2023-09-23T19:25:31.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:30 smithi016 bash[75004]: audit 2023-09-23T19:25:30.761665+0000 mon.c (mon.2) 96 : audit 0 from='client.? 172.21.15.16:0/430373309' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:31.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:30 smithi092 bash[58855]: cluster 2023-09-23T19:25:29.786136+0000 mgr.y (mgr.25325) 505 : cluster 0 pgmap v415: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 592 op/s; 7341/126396 objects degraded (5.808%); 0 B/s, 31 objects/s recovering 2023-09-23T19:25:31.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:30 smithi092 bash[58855]: audit 2023-09-23T19:25:30.761665+0000 mon.c (mon.2) 96 : audit 0 from='client.? 172.21.15.16:0/430373309' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:32.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:31 smithi016 bash[75004]: cluster 2023-09-23T19:25:31.787045+0000 mgr.y (mgr.25325) 506 : cluster 0 pgmap v416: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 765 op/s; 7151/118389 objects degraded (6.040%); 0 B/s, 46 objects/s recovering 2023-09-23T19:25:32.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:31 smithi016 bash[77995]: cluster 2023-09-23T19:25:31.787045+0000 mgr.y (mgr.25325) 506 : cluster 0 pgmap v416: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 765 op/s; 7151/118389 objects degraded (6.040%); 0 B/s, 46 objects/s recovering 2023-09-23T19:25:32.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:31 smithi092 bash[58855]: cluster 2023-09-23T19:25:31.787045+0000 mgr.y (mgr.25325) 506 : cluster 0 pgmap v416: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 765 op/s; 7151/118389 objects degraded (6.040%); 0 B/s, 46 objects/s recovering 2023-09-23T19:25:33.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:33 smithi016 bash[75004]: audit 2023-09-23T19:25:33.396866+0000 mon.b (mon.1) 32 : audit 0 from='client.? 172.21.15.16:0/4021461642' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:33.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:33 smithi016 bash[75004]: audit 2023-09-23T19:25:33.397081+0000 mon.a (mon.0) 681 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:25:33.784 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:25:33 smithi016 bash[70992]: debug 2023-09-23T19:25:33.394+0000 7fb2a5a7c700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T19:25:33.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:33 smithi016 bash[77995]: audit 2023-09-23T19:25:33.396866+0000 mon.b (mon.1) 32 : audit 0 from='client.? 172.21.15.16:0/4021461642' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:33.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:33 smithi016 bash[77995]: audit 2023-09-23T19:25:33.397081+0000 mon.a (mon.0) 681 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:25:33.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:33 smithi092 bash[58855]: audit 2023-09-23T19:25:33.396866+0000 mon.b (mon.1) 32 : audit 0 from='client.? 172.21.15.16:0/4021461642' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:33.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:33 smithi092 bash[58855]: audit 2023-09-23T19:25:33.397081+0000 mon.a (mon.0) 681 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:25:34.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:34 smithi016 bash[75004]: audit 2023-09-23T19:25:33.399129+0000 mgr.y (mgr.25325) 507 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:25:34.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:34 smithi016 bash[75004]: cephadm 2023-09-23T19:25:33.401064+0000 mgr.y (mgr.25325) 508 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T19:25:34.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:34 smithi016 bash[75004]: cluster 2023-09-23T19:25:33.787728+0000 mgr.y (mgr.25325) 509 : cluster 0 pgmap v417: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 7151/116436 objects degraded (6.142%); 0 B/s, 31 objects/s recovering 2023-09-23T19:25:34.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:34 smithi016 bash[75004]: cluster 2023-09-23T19:25:34.153424+0000 mon.a (mon.0) 682 : cluster 3 Health check update: Degraded data redundancy: 7151/118389 objects degraded (6.040%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:25:34.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:34 smithi016 bash[75004]: cluster 2023-09-23T19:25:34.163316+0000 mon.a (mon.0) 683 : cluster 0 osdmap e105: 8 total, 8 up, 8 in 2023-09-23T19:25:34.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:34 smithi016 bash[77995]: audit 2023-09-23T19:25:33.399129+0000 mgr.y (mgr.25325) 507 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:25:34.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:34 smithi016 bash[77995]: cephadm 2023-09-23T19:25:33.401064+0000 mgr.y (mgr.25325) 508 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T19:25:34.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:34 smithi016 bash[77995]: cluster 2023-09-23T19:25:33.787728+0000 mgr.y (mgr.25325) 509 : cluster 0 pgmap v417: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 7151/116436 objects degraded (6.142%); 0 B/s, 31 objects/s recovering 2023-09-23T19:25:34.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:34 smithi016 bash[77995]: cluster 2023-09-23T19:25:34.153424+0000 mon.a (mon.0) 682 : cluster 3 Health check update: Degraded data redundancy: 7151/118389 objects degraded (6.040%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:25:34.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:34 smithi016 bash[77995]: cluster 2023-09-23T19:25:34.163316+0000 mon.a (mon.0) 683 : cluster 0 osdmap e105: 8 total, 8 up, 8 in 2023-09-23T19:25:34.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:34 smithi092 bash[58855]: audit 2023-09-23T19:25:33.399129+0000 mgr.y (mgr.25325) 507 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:25:34.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:34 smithi092 bash[58855]: cephadm 2023-09-23T19:25:33.401064+0000 mgr.y (mgr.25325) 508 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T19:25:34.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:34 smithi092 bash[58855]: cluster 2023-09-23T19:25:33.787728+0000 mgr.y (mgr.25325) 509 : cluster 0 pgmap v417: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 7151/116436 objects degraded (6.142%); 0 B/s, 31 objects/s recovering 2023-09-23T19:25:34.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:34 smithi092 bash[58855]: cluster 2023-09-23T19:25:34.153424+0000 mon.a (mon.0) 682 : cluster 3 Health check update: Degraded data redundancy: 7151/118389 objects degraded (6.040%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:25:34.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:34 smithi092 bash[58855]: cluster 2023-09-23T19:25:34.163316+0000 mon.a (mon.0) 683 : cluster 0 osdmap e105: 8 total, 8 up, 8 in 2023-09-23T19:25:35.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:35 smithi016 bash[75004]: cluster 2023-09-23T19:25:34.160471+0000 osd.3 (osd.3) 35 : cluster 3 failed to encode map e105 with expected crc 2023-09-23T19:25:35.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:35 smithi016 bash[75004]: cluster 2023-09-23T19:25:34.170267+0000 osd.5 (osd.5) 23 : cluster 3 failed to encode map e105 with expected crc 2023-09-23T19:25:35.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:35 smithi016 bash[75004]: cluster 2023-09-23T19:25:34.177133+0000 osd.6 (osd.6) 23 : cluster 3 failed to encode map e105 with expected crc 2023-09-23T19:25:35.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:35 smithi016 bash[75004]: cluster 2023-09-23T19:25:35.171736+0000 mon.a (mon.0) 684 : cluster 0 osdmap e106: 8 total, 8 up, 8 in 2023-09-23T19:25:35.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:35 smithi016 bash[75004]: cluster 2023-09-23T19:25:35.172612+0000 osd.7 (osd.7) 23 : cluster 3 failed to encode map e106 with expected crc 2023-09-23T19:25:35.785 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:25:35 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:25:35] "GET /metrics HTTP/1.1" 200 36185 "" "Prometheus/2.33.4" 2023-09-23T19:25:35.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:35 smithi016 bash[77995]: cluster 2023-09-23T19:25:34.160471+0000 osd.3 (osd.3) 35 : cluster 3 failed to encode map e105 with expected crc 2023-09-23T19:25:35.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:35 smithi016 bash[77995]: cluster 2023-09-23T19:25:34.170267+0000 osd.5 (osd.5) 23 : cluster 3 failed to encode map e105 with expected crc 2023-09-23T19:25:35.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:35 smithi016 bash[77995]: cluster 2023-09-23T19:25:34.177133+0000 osd.6 (osd.6) 23 : cluster 3 failed to encode map e105 with expected crc 2023-09-23T19:25:35.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:35 smithi016 bash[77995]: cluster 2023-09-23T19:25:35.171736+0000 mon.a (mon.0) 684 : cluster 0 osdmap e106: 8 total, 8 up, 8 in 2023-09-23T19:25:35.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:35 smithi016 bash[77995]: cluster 2023-09-23T19:25:35.172612+0000 osd.7 (osd.7) 23 : cluster 3 failed to encode map e106 with expected crc 2023-09-23T19:25:35.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:35 smithi092 bash[58855]: cluster 2023-09-23T19:25:34.160471+0000 osd.3 (osd.3) 35 : cluster 3 failed to encode map e105 with expected crc 2023-09-23T19:25:35.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:35 smithi092 bash[58855]: cluster 2023-09-23T19:25:34.170267+0000 osd.5 (osd.5) 23 : cluster 3 failed to encode map e105 with expected crc 2023-09-23T19:25:35.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:35 smithi092 bash[58855]: cluster 2023-09-23T19:25:34.177133+0000 osd.6 (osd.6) 23 : cluster 3 failed to encode map e105 with expected crc 2023-09-23T19:25:35.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:35 smithi092 bash[58855]: cluster 2023-09-23T19:25:35.171736+0000 mon.a (mon.0) 684 : cluster 0 osdmap e106: 8 total, 8 up, 8 in 2023-09-23T19:25:35.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:35 smithi092 bash[58855]: cluster 2023-09-23T19:25:35.172612+0000 osd.7 (osd.7) 23 : cluster 3 failed to encode map e106 with expected crc 2023-09-23T19:25:36.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:36 smithi016 bash[77995]: cluster 2023-09-23T19:25:35.178478+0000 osd.6 (osd.6) 24 : cluster 3 failed to encode map e106 with expected crc 2023-09-23T19:25:36.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:36 smithi016 bash[77995]: cluster 2023-09-23T19:25:35.179542+0000 osd.5 (osd.5) 24 : cluster 3 failed to encode map e106 with expected crc 2023-09-23T19:25:36.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:36 smithi016 bash[77995]: cluster 2023-09-23T19:25:35.182148+0000 osd.3 (osd.3) 36 : cluster 3 failed to encode map e106 with expected crc 2023-09-23T19:25:36.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:36 smithi016 bash[77995]: cluster 2023-09-23T19:25:35.242943+0000 osd.2 (osd.2) 29 : cluster 3 failed to encode map e106 with expected crc 2023-09-23T19:25:36.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:36 smithi016 bash[77995]: cluster 2023-09-23T19:25:35.252622+0000 osd.4 (osd.4) 23 : cluster 3 failed to encode map e106 with expected crc 2023-09-23T19:25:36.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:36 smithi016 bash[77995]: cluster 2023-09-23T19:25:35.788335+0000 mgr.y (mgr.25325) 510 : cluster 0 pgmap v420: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 6992/111522 objects degraded (6.270%); 0 B/s, 43 objects/s recovering 2023-09-23T19:25:36.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:36 smithi016 bash[77995]: audit 2023-09-23T19:25:35.963071+0000 mon.c (mon.2) 97 : audit 0 from='client.? 172.21.15.16:0/862958034' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:36.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:36 smithi016 bash[75004]: cluster 2023-09-23T19:25:35.178478+0000 osd.6 (osd.6) 24 : cluster 3 failed to encode map e106 with expected crc 2023-09-23T19:25:36.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:36 smithi016 bash[75004]: cluster 2023-09-23T19:25:35.179542+0000 osd.5 (osd.5) 24 : cluster 3 failed to encode map e106 with expected crc 2023-09-23T19:25:36.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:36 smithi016 bash[75004]: cluster 2023-09-23T19:25:35.182148+0000 osd.3 (osd.3) 36 : cluster 3 failed to encode map e106 with expected crc 2023-09-23T19:25:36.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:36 smithi016 bash[75004]: cluster 2023-09-23T19:25:35.242943+0000 osd.2 (osd.2) 29 : cluster 3 failed to encode map e106 with expected crc 2023-09-23T19:25:36.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:36 smithi016 bash[75004]: cluster 2023-09-23T19:25:35.252622+0000 osd.4 (osd.4) 23 : cluster 3 failed to encode map e106 with expected crc 2023-09-23T19:25:36.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:36 smithi016 bash[75004]: cluster 2023-09-23T19:25:35.788335+0000 mgr.y (mgr.25325) 510 : cluster 0 pgmap v420: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 6992/111522 objects degraded (6.270%); 0 B/s, 43 objects/s recovering 2023-09-23T19:25:36.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:36 smithi016 bash[75004]: audit 2023-09-23T19:25:35.963071+0000 mon.c (mon.2) 97 : audit 0 from='client.? 172.21.15.16:0/862958034' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:36.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:36 smithi092 bash[58855]: cluster 2023-09-23T19:25:35.178478+0000 osd.6 (osd.6) 24 : cluster 3 failed to encode map e106 with expected crc 2023-09-23T19:25:36.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:36 smithi092 bash[58855]: cluster 2023-09-23T19:25:35.179542+0000 osd.5 (osd.5) 24 : cluster 3 failed to encode map e106 with expected crc 2023-09-23T19:25:36.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:36 smithi092 bash[58855]: cluster 2023-09-23T19:25:35.182148+0000 osd.3 (osd.3) 36 : cluster 3 failed to encode map e106 with expected crc 2023-09-23T19:25:36.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:36 smithi092 bash[58855]: cluster 2023-09-23T19:25:35.242943+0000 osd.2 (osd.2) 29 : cluster 3 failed to encode map e106 with expected crc 2023-09-23T19:25:36.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:36 smithi092 bash[58855]: cluster 2023-09-23T19:25:35.252622+0000 osd.4 (osd.4) 23 : cluster 3 failed to encode map e106 with expected crc 2023-09-23T19:25:36.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:36 smithi092 bash[58855]: cluster 2023-09-23T19:25:35.788335+0000 mgr.y (mgr.25325) 510 : cluster 0 pgmap v420: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 6992/111522 objects degraded (6.270%); 0 B/s, 43 objects/s recovering 2023-09-23T19:25:36.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:36 smithi092 bash[58855]: audit 2023-09-23T19:25:35.963071+0000 mon.c (mon.2) 97 : audit 0 from='client.? 172.21.15.16:0/862958034' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:39.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:38 smithi016 bash[77995]: cluster 2023-09-23T19:25:37.788972+0000 mgr.y (mgr.25325) 511 : cluster 0 pgmap v421: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 6939/109557 objects degraded (6.334%); 0 B/s, 50 objects/s recovering 2023-09-23T19:25:39.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:38 smithi016 bash[77995]: audit 2023-09-23T19:25:38.577874+0000 mon.c (mon.2) 98 : audit 0 from='client.? 172.21.15.16:0/3852601925' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:39.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:38 smithi016 bash[75004]: cluster 2023-09-23T19:25:37.788972+0000 mgr.y (mgr.25325) 511 : cluster 0 pgmap v421: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 6939/109557 objects degraded (6.334%); 0 B/s, 50 objects/s recovering 2023-09-23T19:25:39.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:38 smithi016 bash[75004]: audit 2023-09-23T19:25:38.577874+0000 mon.c (mon.2) 98 : audit 0 from='client.? 172.21.15.16:0/3852601925' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:39.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:38 smithi092 bash[58855]: cluster 2023-09-23T19:25:37.788972+0000 mgr.y (mgr.25325) 511 : cluster 0 pgmap v421: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 6939/109557 objects degraded (6.334%); 0 B/s, 50 objects/s recovering 2023-09-23T19:25:39.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:38 smithi092 bash[58855]: audit 2023-09-23T19:25:38.577874+0000 mon.c (mon.2) 98 : audit 0 from='client.? 172.21.15.16:0/3852601925' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:40.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:39 smithi016 bash[77995]: cluster 2023-09-23T19:25:39.155021+0000 mon.a (mon.0) 685 : cluster 3 Health check update: Degraded data redundancy: 6939/109557 objects degraded (6.334%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:25:40.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:39 smithi016 bash[75004]: cluster 2023-09-23T19:25:39.155021+0000 mon.a (mon.0) 685 : cluster 3 Health check update: Degraded data redundancy: 6939/109557 objects degraded (6.334%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:25:40.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:39 smithi092 bash[58855]: cluster 2023-09-23T19:25:39.155021+0000 mon.a (mon.0) 685 : cluster 3 Health check update: Degraded data redundancy: 6939/109557 objects degraded (6.334%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:25:41.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:40 smithi016 bash[75004]: cluster 2023-09-23T19:25:39.790397+0000 mgr.y (mgr.25325) 512 : cluster 0 pgmap v422: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 433 op/s; 6939/107976 objects degraded (6.426%); 0 B/s, 26 objects/s recovering 2023-09-23T19:25:41.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:40 smithi016 bash[77995]: cluster 2023-09-23T19:25:39.790397+0000 mgr.y (mgr.25325) 512 : cluster 0 pgmap v422: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 433 op/s; 6939/107976 objects degraded (6.426%); 0 B/s, 26 objects/s recovering 2023-09-23T19:25:41.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:40 smithi092 bash[58855]: cluster 2023-09-23T19:25:39.790397+0000 mgr.y (mgr.25325) 512 : cluster 0 pgmap v422: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 433 op/s; 6939/107976 objects degraded (6.426%); 0 B/s, 26 objects/s recovering 2023-09-23T19:25:42.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:41 smithi016 bash[75004]: audit 2023-09-23T19:25:41.209598+0000 mon.a (mon.0) 686 : audit 0 from='client.? 172.21.15.16:0/907658202' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:42.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:41 smithi016 bash[77995]: audit 2023-09-23T19:25:41.209598+0000 mon.a (mon.0) 686 : audit 0 from='client.? 172.21.15.16:0/907658202' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:42.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:41 smithi092 bash[58855]: audit 2023-09-23T19:25:41.209598+0000 mon.a (mon.0) 686 : audit 0 from='client.? 172.21.15.16:0/907658202' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:43.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:42 smithi016 bash[75004]: cluster 2023-09-23T19:25:41.791077+0000 mgr.y (mgr.25325) 513 : cluster 0 pgmap v423: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 6832/100512 objects degraded (6.797%); 0 B/s, 39 objects/s recovering 2023-09-23T19:25:43.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:42 smithi016 bash[77995]: cluster 2023-09-23T19:25:41.791077+0000 mgr.y (mgr.25325) 513 : cluster 0 pgmap v423: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 6832/100512 objects degraded (6.797%); 0 B/s, 39 objects/s recovering 2023-09-23T19:25:43.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:42 smithi092 bash[58855]: cluster 2023-09-23T19:25:41.791077+0000 mgr.y (mgr.25325) 513 : cluster 0 pgmap v423: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 6832/100512 objects degraded (6.797%); 0 B/s, 39 objects/s recovering 2023-09-23T19:25:44.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:43 smithi016 bash[75004]: audit 2023-09-23T19:25:43.627822+0000 mon.a (mon.0) 687 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:25:44.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:43 smithi016 bash[75004]: cluster 2023-09-23T19:25:43.791681+0000 mgr.y (mgr.25325) 514 : cluster 0 pgmap v424: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 6832/98655 objects degraded (6.925%); 0 B/s, 33 objects/s recovering 2023-09-23T19:25:44.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:43 smithi016 bash[75004]: audit 2023-09-23T19:25:43.862358+0000 mon.a (mon.0) 688 : audit 0 from='client.? 172.21.15.16:0/930721356' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:44.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:43 smithi016 bash[77995]: audit 2023-09-23T19:25:43.627822+0000 mon.a (mon.0) 687 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:25:44.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:43 smithi016 bash[77995]: cluster 2023-09-23T19:25:43.791681+0000 mgr.y (mgr.25325) 514 : cluster 0 pgmap v424: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 6832/98655 objects degraded (6.925%); 0 B/s, 33 objects/s recovering 2023-09-23T19:25:44.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:43 smithi016 bash[77995]: audit 2023-09-23T19:25:43.862358+0000 mon.a (mon.0) 688 : audit 0 from='client.? 172.21.15.16:0/930721356' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:44.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:43 smithi092 bash[58855]: audit 2023-09-23T19:25:43.627822+0000 mon.a (mon.0) 687 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:25:44.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:43 smithi092 bash[58855]: cluster 2023-09-23T19:25:43.791681+0000 mgr.y (mgr.25325) 514 : cluster 0 pgmap v424: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 6832/98655 objects degraded (6.925%); 0 B/s, 33 objects/s recovering 2023-09-23T19:25:44.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:43 smithi092 bash[58855]: audit 2023-09-23T19:25:43.862358+0000 mon.a (mon.0) 688 : audit 0 from='client.? 172.21.15.16:0/930721356' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:45.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:44 smithi016 bash[77995]: cluster 2023-09-23T19:25:44.156077+0000 mon.a (mon.0) 689 : cluster 3 Health check update: Degraded data redundancy: 6832/98655 objects degraded (6.925%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:25:45.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:44 smithi016 bash[75004]: cluster 2023-09-23T19:25:44.156077+0000 mon.a (mon.0) 689 : cluster 3 Health check update: Degraded data redundancy: 6832/98655 objects degraded (6.925%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:25:45.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:44 smithi092 bash[58855]: cluster 2023-09-23T19:25:44.156077+0000 mon.a (mon.0) 689 : cluster 3 Health check update: Degraded data redundancy: 6832/98655 objects degraded (6.925%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:25:46.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:45 smithi016 bash[75004]: cluster 2023-09-23T19:25:45.792359+0000 mgr.y (mgr.25325) 515 : cluster 0 pgmap v425: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 6832/95151 objects degraded (7.180%); 0 B/s, 30 objects/s recovering 2023-09-23T19:25:46.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:25:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:25:45] "GET /metrics HTTP/1.1" 200 36180 "" "Prometheus/2.33.4" 2023-09-23T19:25:46.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:45 smithi016 bash[77995]: cluster 2023-09-23T19:25:45.792359+0000 mgr.y (mgr.25325) 515 : cluster 0 pgmap v425: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 6832/95151 objects degraded (7.180%); 0 B/s, 30 objects/s recovering 2023-09-23T19:25:46.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:45 smithi092 bash[58855]: cluster 2023-09-23T19:25:45.792359+0000 mgr.y (mgr.25325) 515 : cluster 0 pgmap v425: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 6832/95151 objects degraded (7.180%); 0 B/s, 30 objects/s recovering 2023-09-23T19:25:47.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:46 smithi016 bash[75004]: audit 2023-09-23T19:25:46.483321+0000 mon.a (mon.0) 690 : audit 0 from='client.? 172.21.15.16:0/651086610' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:47.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:46 smithi016 bash[77995]: audit 2023-09-23T19:25:46.483321+0000 mon.a (mon.0) 690 : audit 0 from='client.? 172.21.15.16:0/651086610' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:47.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:46 smithi092 bash[58855]: audit 2023-09-23T19:25:46.483321+0000 mon.a (mon.0) 690 : audit 0 from='client.? 172.21.15.16:0/651086610' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:48.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:47 smithi016 bash[75004]: cluster 2023-09-23T19:25:47.793281+0000 mgr.y (mgr.25325) 516 : cluster 0 pgmap v426: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.8 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 6726/90870 objects degraded (7.402%); 0 B/s, 22 objects/s recovering 2023-09-23T19:25:48.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:47 smithi016 bash[77995]: cluster 2023-09-23T19:25:47.793281+0000 mgr.y (mgr.25325) 516 : cluster 0 pgmap v426: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.8 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 6726/90870 objects degraded (7.402%); 0 B/s, 22 objects/s recovering 2023-09-23T19:25:48.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:47 smithi092 bash[58855]: cluster 2023-09-23T19:25:47.793281+0000 mgr.y (mgr.25325) 516 : cluster 0 pgmap v426: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.8 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 6726/90870 objects degraded (7.402%); 0 B/s, 22 objects/s recovering 2023-09-23T19:25:48.784 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:25:48 smithi016 bash[70992]: debug 2023-09-23T19:25:48.410+0000 7fb2a5a7c700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T19:25:49.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:48 smithi016 bash[75004]: audit 2023-09-23T19:25:48.412984+0000 mon.a (mon.0) 691 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:25:49.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:48 smithi016 bash[75004]: audit 2023-09-23T19:25:48.413293+0000 mgr.y (mgr.25325) 517 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:25:49.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:48 smithi016 bash[75004]: cephadm 2023-09-23T19:25:48.416844+0000 mgr.y (mgr.25325) 518 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T19:25:49.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:48 smithi016 bash[77995]: audit 2023-09-23T19:25:48.412984+0000 mon.a (mon.0) 691 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:25:49.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:48 smithi016 bash[77995]: audit 2023-09-23T19:25:48.413293+0000 mgr.y (mgr.25325) 517 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:25:49.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:48 smithi016 bash[77995]: cephadm 2023-09-23T19:25:48.416844+0000 mgr.y (mgr.25325) 518 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T19:25:49.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:48 smithi092 bash[58855]: audit 2023-09-23T19:25:48.412984+0000 mon.a (mon.0) 691 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:25:49.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:48 smithi092 bash[58855]: audit 2023-09-23T19:25:48.413293+0000 mgr.y (mgr.25325) 517 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:25:49.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:48 smithi092 bash[58855]: cephadm 2023-09-23T19:25:48.416844+0000 mgr.y (mgr.25325) 518 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T19:25:50.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:49 smithi016 bash[77995]: audit 2023-09-23T19:25:49.131363+0000 mon.b (mon.1) 33 : audit 0 from='client.? 172.21.15.16:0/1971857179' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:50.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:49 smithi016 bash[77995]: cluster 2023-09-23T19:25:49.160523+0000 mon.a (mon.0) 692 : cluster 3 Health check update: Degraded data redundancy: 6726/90870 objects degraded (7.402%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:25:50.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:49 smithi016 bash[77995]: cluster 2023-09-23T19:25:49.793912+0000 mgr.y (mgr.25325) 519 : cluster 0 pgmap v427: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.8 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 570 op/s; 6726/89028 objects degraded (7.555%); 0 B/s, 17 objects/s recovering 2023-09-23T19:25:50.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:49 smithi016 bash[75004]: audit 2023-09-23T19:25:49.131363+0000 mon.b (mon.1) 33 : audit 0 from='client.? 172.21.15.16:0/1971857179' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:50.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:49 smithi016 bash[75004]: cluster 2023-09-23T19:25:49.160523+0000 mon.a (mon.0) 692 : cluster 3 Health check update: Degraded data redundancy: 6726/90870 objects degraded (7.402%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:25:50.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:49 smithi016 bash[75004]: cluster 2023-09-23T19:25:49.793912+0000 mgr.y (mgr.25325) 519 : cluster 0 pgmap v427: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.8 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 570 op/s; 6726/89028 objects degraded (7.555%); 0 B/s, 17 objects/s recovering 2023-09-23T19:25:50.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:49 smithi092 bash[58855]: audit 2023-09-23T19:25:49.131363+0000 mon.b (mon.1) 33 : audit 0 from='client.? 172.21.15.16:0/1971857179' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:50.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:49 smithi092 bash[58855]: cluster 2023-09-23T19:25:49.160523+0000 mon.a (mon.0) 692 : cluster 3 Health check update: Degraded data redundancy: 6726/90870 objects degraded (7.402%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:25:50.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:49 smithi092 bash[58855]: cluster 2023-09-23T19:25:49.793912+0000 mgr.y (mgr.25325) 519 : cluster 0 pgmap v427: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.8 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 570 op/s; 6726/89028 objects degraded (7.555%); 0 B/s, 17 objects/s recovering 2023-09-23T19:25:52.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:51 smithi016 bash[77995]: audit 2023-09-23T19:25:51.749028+0000 mon.c (mon.2) 99 : audit 0 from='client.? 172.21.15.16:0/827534386' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:52.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:51 smithi016 bash[75004]: audit 2023-09-23T19:25:51.749028+0000 mon.c (mon.2) 99 : audit 0 from='client.? 172.21.15.16:0/827534386' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:52.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:51 smithi092 bash[58855]: audit 2023-09-23T19:25:51.749028+0000 mon.c (mon.2) 99 : audit 0 from='client.? 172.21.15.16:0/827534386' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:53.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:52 smithi016 bash[75004]: cluster 2023-09-23T19:25:51.794726+0000 mgr.y (mgr.25325) 520 : cluster 0 pgmap v428: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 6621/81213 objects degraded (8.153%); 0 B/s, 26 objects/s recovering 2023-09-23T19:25:53.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:52 smithi016 bash[77995]: cluster 2023-09-23T19:25:51.794726+0000 mgr.y (mgr.25325) 520 : cluster 0 pgmap v428: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 6621/81213 objects degraded (8.153%); 0 B/s, 26 objects/s recovering 2023-09-23T19:25:53.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:52 smithi092 bash[58855]: cluster 2023-09-23T19:25:51.794726+0000 mgr.y (mgr.25325) 520 : cluster 0 pgmap v428: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 6621/81213 objects degraded (8.153%); 0 B/s, 26 objects/s recovering 2023-09-23T19:25:54.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:53 smithi016 bash[75004]: cluster 2023-09-23T19:25:53.795351+0000 mgr.y (mgr.25325) 521 : cluster 0 pgmap v429: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 6621/79428 objects degraded (8.336%); 0 B/s, 17 objects/s recovering 2023-09-23T19:25:54.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:53 smithi016 bash[77995]: cluster 2023-09-23T19:25:53.795351+0000 mgr.y (mgr.25325) 521 : cluster 0 pgmap v429: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 6621/79428 objects degraded (8.336%); 0 B/s, 17 objects/s recovering 2023-09-23T19:25:54.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:53 smithi092 bash[58855]: cluster 2023-09-23T19:25:53.795351+0000 mgr.y (mgr.25325) 521 : cluster 0 pgmap v429: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 6621/79428 objects degraded (8.336%); 0 B/s, 17 objects/s recovering 2023-09-23T19:25:55.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:54 smithi016 bash[75004]: cluster 2023-09-23T19:25:54.161931+0000 mon.a (mon.0) 693 : cluster 3 Health check update: Degraded data redundancy: 6621/79428 objects degraded (8.336%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:25:55.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:54 smithi016 bash[75004]: audit 2023-09-23T19:25:54.392893+0000 mon.b (mon.1) 34 : audit 0 from='client.? 172.21.15.16:0/887075758' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:55.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:54 smithi016 bash[77995]: cluster 2023-09-23T19:25:54.161931+0000 mon.a (mon.0) 693 : cluster 3 Health check update: Degraded data redundancy: 6621/79428 objects degraded (8.336%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:25:55.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:54 smithi016 bash[77995]: audit 2023-09-23T19:25:54.392893+0000 mon.b (mon.1) 34 : audit 0 from='client.? 172.21.15.16:0/887075758' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:55.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:54 smithi092 bash[58855]: cluster 2023-09-23T19:25:54.161931+0000 mon.a (mon.0) 693 : cluster 3 Health check update: Degraded data redundancy: 6621/79428 objects degraded (8.336%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:25:55.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:54 smithi092 bash[58855]: audit 2023-09-23T19:25:54.392893+0000 mon.b (mon.1) 34 : audit 0 from='client.? 172.21.15.16:0/887075758' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:55.921 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:55 smithi016 bash[75004]: cluster 2023-09-23T19:25:55.796111+0000 mgr.y (mgr.25325) 522 : cluster 0 pgmap v430: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 6621/75843 objects degraded (8.730%); 0 B/s, 17 objects/s recovering 2023-09-23T19:25:55.921 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:55 smithi016 bash[77995]: cluster 2023-09-23T19:25:55.796111+0000 mgr.y (mgr.25325) 522 : cluster 0 pgmap v430: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 6621/75843 objects degraded (8.730%); 0 B/s, 17 objects/s recovering 2023-09-23T19:25:55.922 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:25:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:25:55] "GET /metrics HTTP/1.1" 200 36180 "" "Prometheus/2.33.4" 2023-09-23T19:25:56.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:55 smithi092 bash[58855]: cluster 2023-09-23T19:25:55.796111+0000 mgr.y (mgr.25325) 522 : cluster 0 pgmap v430: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 6621/75843 objects degraded (8.730%); 0 B/s, 17 objects/s recovering 2023-09-23T19:25:57.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:57 smithi016 bash[75004]: audit 2023-09-23T19:25:57.022028+0000 mon.c (mon.2) 100 : audit 0 from='client.? 172.21.15.16:0/3343069800' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:57.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:57 smithi016 bash[77995]: audit 2023-09-23T19:25:57.022028+0000 mon.c (mon.2) 100 : audit 0 from='client.? 172.21.15.16:0/3343069800' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:57.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:57 smithi092 bash[58855]: audit 2023-09-23T19:25:57.022028+0000 mon.c (mon.2) 100 : audit 0 from='client.? 172.21.15.16:0/3343069800' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:25:58.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:58 smithi016 bash[77995]: cluster 2023-09-23T19:25:57.797633+0000 mgr.y (mgr.25325) 523 : cluster 0 pgmap v431: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 6511/71667 objects degraded (9.085%); 0 B/s, 26 objects/s recovering 2023-09-23T19:25:58.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:58 smithi016 bash[75004]: cluster 2023-09-23T19:25:57.797633+0000 mgr.y (mgr.25325) 523 : cluster 0 pgmap v431: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 6511/71667 objects degraded (9.085%); 0 B/s, 26 objects/s recovering 2023-09-23T19:25:58.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:58 smithi092 bash[58855]: cluster 2023-09-23T19:25:57.797633+0000 mgr.y (mgr.25325) 523 : cluster 0 pgmap v431: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 6511/71667 objects degraded (9.085%); 0 B/s, 26 objects/s recovering 2023-09-23T19:25:59.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:25:59 smithi016 bash[75004]: audit 2023-09-23T19:25:58.627621+0000 mon.a (mon.0) 694 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:25:59.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:25:59 smithi016 bash[77995]: audit 2023-09-23T19:25:58.627621+0000 mon.a (mon.0) 694 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:25:59.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:25:59 smithi092 bash[58855]: audit 2023-09-23T19:25:58.627621+0000 mon.a (mon.0) 694 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:26:00.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:00 smithi016 bash[75004]: cluster 2023-09-23T19:25:59.163029+0000 mon.a (mon.0) 695 : cluster 3 Health check update: Degraded data redundancy: 6511/71667 objects degraded (9.085%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:26:00.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:00 smithi016 bash[75004]: audit 2023-09-23T19:25:59.602563+0000 mon.c (mon.2) 101 : audit 0 from='client.? 172.21.15.16:0/2464499605' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:00.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:00 smithi016 bash[75004]: cluster 2023-09-23T19:25:59.798222+0000 mgr.y (mgr.25325) 524 : cluster 0 pgmap v432: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.4 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 6511/69846 objects degraded (9.322%); 0 B/s, 17 objects/s recovering 2023-09-23T19:26:00.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:00 smithi016 bash[77995]: cluster 2023-09-23T19:25:59.163029+0000 mon.a (mon.0) 695 : cluster 3 Health check update: Degraded data redundancy: 6511/71667 objects degraded (9.085%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:26:00.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:00 smithi016 bash[77995]: audit 2023-09-23T19:25:59.602563+0000 mon.c (mon.2) 101 : audit 0 from='client.? 172.21.15.16:0/2464499605' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:00.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:00 smithi016 bash[77995]: cluster 2023-09-23T19:25:59.798222+0000 mgr.y (mgr.25325) 524 : cluster 0 pgmap v432: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.4 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 6511/69846 objects degraded (9.322%); 0 B/s, 17 objects/s recovering 2023-09-23T19:26:00.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:00 smithi092 bash[58855]: cluster 2023-09-23T19:25:59.163029+0000 mon.a (mon.0) 695 : cluster 3 Health check update: Degraded data redundancy: 6511/71667 objects degraded (9.085%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:26:00.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:00 smithi092 bash[58855]: audit 2023-09-23T19:25:59.602563+0000 mon.c (mon.2) 101 : audit 0 from='client.? 172.21.15.16:0/2464499605' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:00.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:00 smithi092 bash[58855]: cluster 2023-09-23T19:25:59.798222+0000 mgr.y (mgr.25325) 524 : cluster 0 pgmap v432: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.4 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 6511/69846 objects degraded (9.322%); 0 B/s, 17 objects/s recovering 2023-09-23T19:26:03.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:02 smithi016 bash[77995]: cluster 2023-09-23T19:26:01.799062+0000 mgr.y (mgr.25325) 525 : cluster 0 pgmap v433: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 6403/61821 objects degraded (10.357%); 0 B/s, 26 objects/s recovering 2023-09-23T19:26:03.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:02 smithi016 bash[77995]: audit 2023-09-23T19:26:02.195108+0000 mon.a (mon.0) 696 : audit 0 from='client.? 172.21.15.16:0/1512305549' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:03.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:02 smithi016 bash[75004]: cluster 2023-09-23T19:26:01.799062+0000 mgr.y (mgr.25325) 525 : cluster 0 pgmap v433: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 6403/61821 objects degraded (10.357%); 0 B/s, 26 objects/s recovering 2023-09-23T19:26:03.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:02 smithi016 bash[75004]: audit 2023-09-23T19:26:02.195108+0000 mon.a (mon.0) 696 : audit 0 from='client.? 172.21.15.16:0/1512305549' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:03.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:02 smithi092 bash[58855]: cluster 2023-09-23T19:26:01.799062+0000 mgr.y (mgr.25325) 525 : cluster 0 pgmap v433: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 6403/61821 objects degraded (10.357%); 0 B/s, 26 objects/s recovering 2023-09-23T19:26:03.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:02 smithi092 bash[58855]: audit 2023-09-23T19:26:02.195108+0000 mon.a (mon.0) 696 : audit 0 from='client.? 172.21.15.16:0/1512305549' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:04.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:03 smithi016 bash[77995]: audit 2023-09-23T19:26:03.428953+0000 mon.a (mon.0) 697 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:26:04.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:03 smithi016 bash[75004]: audit 2023-09-23T19:26:03.428953+0000 mon.a (mon.0) 697 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:26:04.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:03 smithi092 bash[58855]: audit 2023-09-23T19:26:03.428953+0000 mon.a (mon.0) 697 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:26:05.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:04 smithi016 bash[75004]: cluster 2023-09-23T19:26:03.801608+0000 mgr.y (mgr.25325) 526 : cluster 0 pgmap v434: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 6403/60093 objects degraded (10.655%); 0 B/s, 18 objects/s recovering 2023-09-23T19:26:05.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:04 smithi016 bash[75004]: cluster 2023-09-23T19:26:04.180168+0000 mon.a (mon.0) 698 : cluster 3 Health check update: Degraded data redundancy: 6403/60093 objects degraded (10.655%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:26:05.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:04 smithi016 bash[75004]: audit 2023-09-23T19:26:04.812073+0000 mon.c (mon.2) 102 : audit 0 from='client.? 172.21.15.16:0/4221909329' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:05.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:04 smithi016 bash[77995]: cluster 2023-09-23T19:26:03.801608+0000 mgr.y (mgr.25325) 526 : cluster 0 pgmap v434: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 6403/60093 objects degraded (10.655%); 0 B/s, 18 objects/s recovering 2023-09-23T19:26:05.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:04 smithi016 bash[77995]: cluster 2023-09-23T19:26:04.180168+0000 mon.a (mon.0) 698 : cluster 3 Health check update: Degraded data redundancy: 6403/60093 objects degraded (10.655%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:26:05.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:04 smithi016 bash[77995]: audit 2023-09-23T19:26:04.812073+0000 mon.c (mon.2) 102 : audit 0 from='client.? 172.21.15.16:0/4221909329' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:05.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:04 smithi092 bash[58855]: cluster 2023-09-23T19:26:03.801608+0000 mgr.y (mgr.25325) 526 : cluster 0 pgmap v434: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 6403/60093 objects degraded (10.655%); 0 B/s, 18 objects/s recovering 2023-09-23T19:26:05.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:04 smithi092 bash[58855]: cluster 2023-09-23T19:26:04.180168+0000 mon.a (mon.0) 698 : cluster 3 Health check update: Degraded data redundancy: 6403/60093 objects degraded (10.655%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:26:05.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:04 smithi092 bash[58855]: audit 2023-09-23T19:26:04.812073+0000 mon.c (mon.2) 102 : audit 0 from='client.? 172.21.15.16:0/4221909329' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:06.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:05 smithi016 bash[75004]: cluster 2023-09-23T19:26:05.802309+0000 mgr.y (mgr.25325) 527 : cluster 0 pgmap v435: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.1 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 6403/56559 objects degraded (11.321%); 0 B/s, 18 objects/s recovering 2023-09-23T19:26:06.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:26:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:26:05] "GET /metrics HTTP/1.1" 200 36182 "" "Prometheus/2.33.4" 2023-09-23T19:26:06.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:05 smithi016 bash[77995]: cluster 2023-09-23T19:26:05.802309+0000 mgr.y (mgr.25325) 527 : cluster 0 pgmap v435: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.1 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 6403/56559 objects degraded (11.321%); 0 B/s, 18 objects/s recovering 2023-09-23T19:26:06.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:05 smithi092 bash[58855]: cluster 2023-09-23T19:26:05.802309+0000 mgr.y (mgr.25325) 527 : cluster 0 pgmap v435: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.1 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 6403/56559 objects degraded (11.321%); 0 B/s, 18 objects/s recovering 2023-09-23T19:26:07.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:07 smithi016 bash[75004]: audit 2023-09-23T19:26:07.427620+0000 mon.b (mon.1) 35 : audit 0 from='client.? 172.21.15.16:0/1602690164' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:07.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:07 smithi016 bash[77995]: audit 2023-09-23T19:26:07.427620+0000 mon.b (mon.1) 35 : audit 0 from='client.? 172.21.15.16:0/1602690164' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:07.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:07 smithi092 bash[58855]: audit 2023-09-23T19:26:07.427620+0000 mon.b (mon.1) 35 : audit 0 from='client.? 172.21.15.16:0/1602690164' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:08.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:08 smithi016 bash[75004]: cluster 2023-09-23T19:26:07.802977+0000 mgr.y (mgr.25325) 528 : cluster 0 pgmap v436: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.1 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 6292/52443 objects degraded (11.998%); 0 B/s, 27 objects/s recovering 2023-09-23T19:26:08.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:08 smithi016 bash[77995]: cluster 2023-09-23T19:26:07.802977+0000 mgr.y (mgr.25325) 528 : cluster 0 pgmap v436: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.1 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 6292/52443 objects degraded (11.998%); 0 B/s, 27 objects/s recovering 2023-09-23T19:26:08.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:08 smithi092 bash[58855]: cluster 2023-09-23T19:26:07.802977+0000 mgr.y (mgr.25325) 528 : cluster 0 pgmap v436: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.1 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 6292/52443 objects degraded (11.998%); 0 B/s, 27 objects/s recovering 2023-09-23T19:26:09.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:09 smithi092 bash[58855]: cluster 2023-09-23T19:26:09.181420+0000 mon.a (mon.0) 699 : cluster 3 Health check update: Degraded data redundancy: 6292/52443 objects degraded (11.998%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:26:09.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:09 smithi016 bash[77995]: cluster 2023-09-23T19:26:09.181420+0000 mon.a (mon.0) 699 : cluster 3 Health check update: Degraded data redundancy: 6292/52443 objects degraded (11.998%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:26:09.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:09 smithi016 bash[75004]: cluster 2023-09-23T19:26:09.181420+0000 mon.a (mon.0) 699 : cluster 3 Health check update: Degraded data redundancy: 6292/52443 objects degraded (11.998%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:26:10.782 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:10 smithi016 bash[75004]: cluster 2023-09-23T19:26:09.803632+0000 mgr.y (mgr.25325) 529 : cluster 0 pgmap v437: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.0 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 6292/50634 objects degraded (12.426%); 0 B/s, 18 objects/s recovering 2023-09-23T19:26:10.782 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:10 smithi016 bash[75004]: audit 2023-09-23T19:26:09.951583+0000 mon.a (mon.0) 700 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:26:10.782 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:10 smithi016 bash[75004]: audit 2023-09-23T19:26:09.976529+0000 mon.a (mon.0) 701 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:26:10.782 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:10 smithi016 bash[75004]: audit 2023-09-23T19:26:10.054558+0000 mon.a (mon.0) 702 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:26:10.782 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:10 smithi016 bash[75004]: audit 2023-09-23T19:26:10.064200+0000 mon.a (mon.0) 703 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:26:10.782 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:10 smithi016 bash[75004]: audit 2023-09-23T19:26:10.109038+0000 mon.c (mon.2) 103 : audit 0 from='client.? 172.21.15.16:0/777775567' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:10.782 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:10 smithi016 bash[77995]: cluster 2023-09-23T19:26:09.803632+0000 mgr.y (mgr.25325) 529 : cluster 0 pgmap v437: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.0 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 6292/50634 objects degraded (12.426%); 0 B/s, 18 objects/s recovering 2023-09-23T19:26:10.783 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:10 smithi016 bash[77995]: audit 2023-09-23T19:26:09.951583+0000 mon.a (mon.0) 700 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:26:10.783 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:10 smithi016 bash[77995]: audit 2023-09-23T19:26:09.976529+0000 mon.a (mon.0) 701 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:26:10.783 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:10 smithi016 bash[77995]: audit 2023-09-23T19:26:10.054558+0000 mon.a (mon.0) 702 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:26:10.783 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:10 smithi016 bash[77995]: audit 2023-09-23T19:26:10.064200+0000 mon.a (mon.0) 703 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:26:10.783 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:10 smithi016 bash[77995]: audit 2023-09-23T19:26:10.109038+0000 mon.c (mon.2) 103 : audit 0 from='client.? 172.21.15.16:0/777775567' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:10.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:10 smithi092 bash[58855]: cluster 2023-09-23T19:26:09.803632+0000 mgr.y (mgr.25325) 529 : cluster 0 pgmap v437: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.0 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 6292/50634 objects degraded (12.426%); 0 B/s, 18 objects/s recovering 2023-09-23T19:26:10.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:10 smithi092 bash[58855]: audit 2023-09-23T19:26:09.951583+0000 mon.a (mon.0) 700 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:26:10.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:10 smithi092 bash[58855]: audit 2023-09-23T19:26:09.976529+0000 mon.a (mon.0) 701 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:26:10.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:10 smithi092 bash[58855]: audit 2023-09-23T19:26:10.054558+0000 mon.a (mon.0) 702 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:26:10.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:10 smithi092 bash[58855]: audit 2023-09-23T19:26:10.064200+0000 mon.a (mon.0) 703 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:26:10.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:10 smithi092 bash[58855]: audit 2023-09-23T19:26:10.109038+0000 mon.c (mon.2) 103 : audit 0 from='client.? 172.21.15.16:0/777775567' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:11.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:26:10 smithi016 bash[70992]: debug 2023-09-23T19:26:10.773+0000 7fb2a5a7c700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T19:26:11.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:11 smithi016 bash[77995]: audit 2023-09-23T19:26:10.662695+0000 mon.a (mon.0) 704 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:26:11.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:11 smithi016 bash[77995]: audit 2023-09-23T19:26:10.667244+0000 mon.a (mon.0) 705 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:26:11.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:11 smithi016 bash[77995]: audit 2023-09-23T19:26:10.676520+0000 mon.a (mon.0) 706 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:26:11.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:11 smithi016 bash[77995]: audit 2023-09-23T19:26:10.763194+0000 mon.a (mon.0) 707 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:26:11.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:11 smithi016 bash[77995]: audit 2023-09-23T19:26:10.768039+0000 mon.a (mon.0) 708 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:26:11.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:11 smithi016 bash[77995]: audit 2023-09-23T19:26:10.774705+0000 mon.a (mon.0) 709 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:26:11.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:11 smithi016 bash[77995]: audit 2023-09-23T19:26:10.776572+0000 mon.a (mon.0) 710 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:26:11.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:11 smithi016 bash[77995]: audit 2023-09-23T19:26:10.779384+0000 mon.a (mon.0) 711 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:26:11.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:11 smithi016 bash[77995]: audit 2023-09-23T19:26:10.779613+0000 mgr.y (mgr.25325) 530 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:26:11.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:11 smithi016 bash[77995]: cephadm 2023-09-23T19:26:10.782116+0000 mgr.y (mgr.25325) 531 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T19:26:11.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:11 smithi016 bash[75004]: audit 2023-09-23T19:26:10.662695+0000 mon.a (mon.0) 704 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:26:11.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:11 smithi016 bash[75004]: audit 2023-09-23T19:26:10.667244+0000 mon.a (mon.0) 705 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:26:11.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:11 smithi016 bash[75004]: audit 2023-09-23T19:26:10.676520+0000 mon.a (mon.0) 706 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:26:11.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:11 smithi016 bash[75004]: audit 2023-09-23T19:26:10.763194+0000 mon.a (mon.0) 707 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:26:11.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:11 smithi016 bash[75004]: audit 2023-09-23T19:26:10.768039+0000 mon.a (mon.0) 708 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:26:11.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:11 smithi016 bash[75004]: audit 2023-09-23T19:26:10.774705+0000 mon.a (mon.0) 709 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:26:11.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:11 smithi016 bash[75004]: audit 2023-09-23T19:26:10.776572+0000 mon.a (mon.0) 710 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:26:11.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:11 smithi016 bash[75004]: audit 2023-09-23T19:26:10.779384+0000 mon.a (mon.0) 711 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:26:11.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:11 smithi016 bash[75004]: audit 2023-09-23T19:26:10.779613+0000 mgr.y (mgr.25325) 530 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:26:11.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:11 smithi016 bash[75004]: cephadm 2023-09-23T19:26:10.782116+0000 mgr.y (mgr.25325) 531 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T19:26:11.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:11 smithi092 bash[58855]: audit 2023-09-23T19:26:10.662695+0000 mon.a (mon.0) 704 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:26:11.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:11 smithi092 bash[58855]: audit 2023-09-23T19:26:10.667244+0000 mon.a (mon.0) 705 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:26:11.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:11 smithi092 bash[58855]: audit 2023-09-23T19:26:10.676520+0000 mon.a (mon.0) 706 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:26:11.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:11 smithi092 bash[58855]: audit 2023-09-23T19:26:10.763194+0000 mon.a (mon.0) 707 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:26:11.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:11 smithi092 bash[58855]: audit 2023-09-23T19:26:10.768039+0000 mon.a (mon.0) 708 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:26:11.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:11 smithi092 bash[58855]: audit 2023-09-23T19:26:10.774705+0000 mon.a (mon.0) 709 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:26:11.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:11 smithi092 bash[58855]: audit 2023-09-23T19:26:10.776572+0000 mon.a (mon.0) 710 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:26:11.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:11 smithi092 bash[58855]: audit 2023-09-23T19:26:10.779384+0000 mon.a (mon.0) 711 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:26:11.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:11 smithi092 bash[58855]: audit 2023-09-23T19:26:10.779613+0000 mgr.y (mgr.25325) 530 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:26:11.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:11 smithi092 bash[58855]: cephadm 2023-09-23T19:26:10.782116+0000 mgr.y (mgr.25325) 531 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T19:26:12.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:12 smithi016 bash[77995]: cluster 2023-09-23T19:26:11.804631+0000 mgr.y (mgr.25325) 532 : cluster 0 pgmap v438: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 899 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 740 op/s; 6186/43191 objects degraded (14.322%); 0 B/s, 27 objects/s recovering 2023-09-23T19:26:12.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:12 smithi016 bash[75004]: cluster 2023-09-23T19:26:11.804631+0000 mgr.y (mgr.25325) 532 : cluster 0 pgmap v438: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 899 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 740 op/s; 6186/43191 objects degraded (14.322%); 0 B/s, 27 objects/s recovering 2023-09-23T19:26:12.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:12 smithi092 bash[58855]: cluster 2023-09-23T19:26:11.804631+0000 mgr.y (mgr.25325) 532 : cluster 0 pgmap v438: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 899 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 740 op/s; 6186/43191 objects degraded (14.322%); 0 B/s, 27 objects/s recovering 2023-09-23T19:26:13.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:13 smithi016 bash[75004]: audit 2023-09-23T19:26:12.735035+0000 mon.a (mon.0) 712 : audit 0 from='client.? 172.21.15.16:0/3044206457' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:13.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:13 smithi016 bash[77995]: audit 2023-09-23T19:26:12.735035+0000 mon.a (mon.0) 712 : audit 0 from='client.? 172.21.15.16:0/3044206457' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:13.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:13 smithi092 bash[58855]: audit 2023-09-23T19:26:12.735035+0000 mon.a (mon.0) 712 : audit 0 from='client.? 172.21.15.16:0/3044206457' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:14.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:14 smithi016 bash[75004]: audit 2023-09-23T19:26:13.628421+0000 mon.a (mon.0) 713 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:26:14.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:14 smithi016 bash[75004]: cluster 2023-09-23T19:26:13.805292+0000 mgr.y (mgr.25325) 533 : cluster 0 pgmap v439: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 865 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 562 op/s; 6186/41544 objects degraded (14.890%); 0 B/s, 18 objects/s recovering 2023-09-23T19:26:14.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:14 smithi016 bash[75004]: cluster 2023-09-23T19:26:14.182973+0000 mon.a (mon.0) 714 : cluster 3 Health check update: Degraded data redundancy: 6186/43191 objects degraded (14.322%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:26:14.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:14 smithi016 bash[77995]: audit 2023-09-23T19:26:13.628421+0000 mon.a (mon.0) 713 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:26:14.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:14 smithi016 bash[77995]: cluster 2023-09-23T19:26:13.805292+0000 mgr.y (mgr.25325) 533 : cluster 0 pgmap v439: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 865 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 562 op/s; 6186/41544 objects degraded (14.890%); 0 B/s, 18 objects/s recovering 2023-09-23T19:26:14.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:14 smithi016 bash[77995]: cluster 2023-09-23T19:26:14.182973+0000 mon.a (mon.0) 714 : cluster 3 Health check update: Degraded data redundancy: 6186/43191 objects degraded (14.322%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:26:14.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:14 smithi092 bash[58855]: audit 2023-09-23T19:26:13.628421+0000 mon.a (mon.0) 713 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:26:14.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:14 smithi092 bash[58855]: cluster 2023-09-23T19:26:13.805292+0000 mgr.y (mgr.25325) 533 : cluster 0 pgmap v439: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 865 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 562 op/s; 6186/41544 objects degraded (14.890%); 0 B/s, 18 objects/s recovering 2023-09-23T19:26:14.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:14 smithi092 bash[58855]: cluster 2023-09-23T19:26:14.182973+0000 mon.a (mon.0) 714 : cluster 3 Health check update: Degraded data redundancy: 6186/43191 objects degraded (14.322%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:26:15.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:15 smithi016 bash[75004]: audit 2023-09-23T19:26:15.383235+0000 mon.b (mon.1) 36 : audit 0 from='client.? 172.21.15.16:0/517292201' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:15.784 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:26:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:26:15] "GET /metrics HTTP/1.1" 200 36180 "" "Prometheus/2.33.4" 2023-09-23T19:26:15.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:15 smithi016 bash[77995]: audit 2023-09-23T19:26:15.383235+0000 mon.b (mon.1) 36 : audit 0 from='client.? 172.21.15.16:0/517292201' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:15.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:15 smithi092 bash[58855]: audit 2023-09-23T19:26:15.383235+0000 mon.b (mon.1) 36 : audit 0 from='client.? 172.21.15.16:0/517292201' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:17.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:16 smithi016 bash[75004]: cluster 2023-09-23T19:26:15.806015+0000 mgr.y (mgr.25325) 534 : cluster 0 pgmap v440: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 793 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 6186/38088 objects degraded (16.241%); 0 B/s, 18 objects/s recovering 2023-09-23T19:26:17.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:16 smithi016 bash[77995]: cluster 2023-09-23T19:26:15.806015+0000 mgr.y (mgr.25325) 534 : cluster 0 pgmap v440: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 793 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 6186/38088 objects degraded (16.241%); 0 B/s, 18 objects/s recovering 2023-09-23T19:26:17.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:16 smithi092 bash[58855]: cluster 2023-09-23T19:26:15.806015+0000 mgr.y (mgr.25325) 534 : cluster 0 pgmap v440: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 793 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 6186/38088 objects degraded (16.241%); 0 B/s, 18 objects/s recovering 2023-09-23T19:26:19.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:18 smithi016 bash[75004]: cluster 2023-09-23T19:26:17.806986+0000 mgr.y (mgr.25325) 535 : cluster 0 pgmap v441: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 704 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 6077/33843 objects degraded (17.956%); 0 B/s, 27 objects/s recovering 2023-09-23T19:26:19.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:18 smithi016 bash[75004]: audit 2023-09-23T19:26:18.006825+0000 mon.a (mon.0) 715 : audit 0 from='client.? 172.21.15.16:0/95336196' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:19.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:18 smithi016 bash[77995]: cluster 2023-09-23T19:26:17.806986+0000 mgr.y (mgr.25325) 535 : cluster 0 pgmap v441: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 704 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 6077/33843 objects degraded (17.956%); 0 B/s, 27 objects/s recovering 2023-09-23T19:26:19.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:18 smithi016 bash[77995]: audit 2023-09-23T19:26:18.006825+0000 mon.a (mon.0) 715 : audit 0 from='client.? 172.21.15.16:0/95336196' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:19.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:18 smithi092 bash[58855]: cluster 2023-09-23T19:26:17.806986+0000 mgr.y (mgr.25325) 535 : cluster 0 pgmap v441: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 704 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 6077/33843 objects degraded (17.956%); 0 B/s, 27 objects/s recovering 2023-09-23T19:26:19.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:18 smithi092 bash[58855]: audit 2023-09-23T19:26:18.006825+0000 mon.a (mon.0) 715 : audit 0 from='client.? 172.21.15.16:0/95336196' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:20.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:19 smithi016 bash[75004]: cluster 2023-09-23T19:26:19.184158+0000 mon.a (mon.0) 716 : cluster 3 Health check update: Degraded data redundancy: 6077/33843 objects degraded (17.956%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:26:20.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:19 smithi016 bash[77995]: cluster 2023-09-23T19:26:19.184158+0000 mon.a (mon.0) 716 : cluster 3 Health check update: Degraded data redundancy: 6077/33843 objects degraded (17.956%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:26:20.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:19 smithi092 bash[58855]: cluster 2023-09-23T19:26:19.184158+0000 mon.a (mon.0) 716 : cluster 3 Health check update: Degraded data redundancy: 6077/33843 objects degraded (17.956%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:26:21.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:20 smithi016 bash[75004]: cluster 2023-09-23T19:26:19.807656+0000 mgr.y (mgr.25325) 536 : cluster 0 pgmap v442: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 665 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 568 op/s; 6077/31977 objects degraded (19.004%); 0 B/s, 17 objects/s recovering 2023-09-23T19:26:21.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:20 smithi016 bash[75004]: audit 2023-09-23T19:26:20.640738+0000 mon.a (mon.0) 717 : audit 0 from='client.? 172.21.15.16:0/2992969201' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:21.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:20 smithi016 bash[77995]: cluster 2023-09-23T19:26:19.807656+0000 mgr.y (mgr.25325) 536 : cluster 0 pgmap v442: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 665 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 568 op/s; 6077/31977 objects degraded (19.004%); 0 B/s, 17 objects/s recovering 2023-09-23T19:26:21.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:20 smithi016 bash[77995]: audit 2023-09-23T19:26:20.640738+0000 mon.a (mon.0) 717 : audit 0 from='client.? 172.21.15.16:0/2992969201' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:21.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:20 smithi092 bash[58855]: cluster 2023-09-23T19:26:19.807656+0000 mgr.y (mgr.25325) 536 : cluster 0 pgmap v442: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 665 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 568 op/s; 6077/31977 objects degraded (19.004%); 0 B/s, 17 objects/s recovering 2023-09-23T19:26:21.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:20 smithi092 bash[58855]: audit 2023-09-23T19:26:20.640738+0000 mon.a (mon.0) 717 : audit 0 from='client.? 172.21.15.16:0/2992969201' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:22.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:21 smithi016 bash[75004]: cluster 2023-09-23T19:26:21.808794+0000 mgr.y (mgr.25325) 537 : cluster 0 pgmap v443: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 501 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 5970/24081 objects degraded (24.791%); 0 B/s, 26 objects/s recovering 2023-09-23T19:26:22.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:21 smithi016 bash[77995]: cluster 2023-09-23T19:26:21.808794+0000 mgr.y (mgr.25325) 537 : cluster 0 pgmap v443: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 501 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 5970/24081 objects degraded (24.791%); 0 B/s, 26 objects/s recovering 2023-09-23T19:26:22.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:21 smithi092 bash[58855]: cluster 2023-09-23T19:26:21.808794+0000 mgr.y (mgr.25325) 537 : cluster 0 pgmap v443: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 501 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 5970/24081 objects degraded (24.791%); 0 B/s, 26 objects/s recovering 2023-09-23T19:26:23.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:23 smithi016 bash[75004]: audit 2023-09-23T19:26:23.226707+0000 mon.a (mon.0) 718 : audit 0 from='client.? 172.21.15.16:0/2282242825' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:23.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:23 smithi016 bash[77995]: audit 2023-09-23T19:26:23.226707+0000 mon.a (mon.0) 718 : audit 0 from='client.? 172.21.15.16:0/2282242825' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:23.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:23 smithi092 bash[58855]: audit 2023-09-23T19:26:23.226707+0000 mon.a (mon.0) 718 : audit 0 from='client.? 172.21.15.16:0/2282242825' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:24.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:24 smithi016 bash[75004]: cluster 2023-09-23T19:26:23.809398+0000 mgr.y (mgr.25325) 538 : cluster 0 pgmap v444: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 464 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 5970/22320 objects degraded (26.747%); 0 B/s, 17 objects/s recovering 2023-09-23T19:26:24.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:24 smithi016 bash[75004]: cluster 2023-09-23T19:26:24.185479+0000 mon.a (mon.0) 719 : cluster 3 Health check update: Degraded data redundancy: 5970/24081 objects degraded (24.791%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:26:24.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:24 smithi016 bash[77995]: cluster 2023-09-23T19:26:23.809398+0000 mgr.y (mgr.25325) 538 : cluster 0 pgmap v444: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 464 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 5970/22320 objects degraded (26.747%); 0 B/s, 17 objects/s recovering 2023-09-23T19:26:24.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:24 smithi016 bash[77995]: cluster 2023-09-23T19:26:24.185479+0000 mon.a (mon.0) 719 : cluster 3 Health check update: Degraded data redundancy: 5970/24081 objects degraded (24.791%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:26:24.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:24 smithi092 bash[58855]: cluster 2023-09-23T19:26:23.809398+0000 mgr.y (mgr.25325) 538 : cluster 0 pgmap v444: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 464 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 5970/22320 objects degraded (26.747%); 0 B/s, 17 objects/s recovering 2023-09-23T19:26:24.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:24 smithi092 bash[58855]: cluster 2023-09-23T19:26:24.185479+0000 mon.a (mon.0) 719 : cluster 3 Health check update: Degraded data redundancy: 5970/24081 objects degraded (24.791%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:26:26.060 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:25 smithi016 bash[75004]: audit 2023-09-23T19:26:25.796993+0000 mon.a (mon.0) 720 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:26:26.060 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:26:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:26:25] "GET /metrics HTTP/1.1" 200 36180 "" "Prometheus/2.33.4" 2023-09-23T19:26:26.061 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:26:25 smithi016 bash[70992]: debug 2023-09-23T19:26:25.793+0000 7fb2a5a7c700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T19:26:26.061 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:25 smithi016 bash[77995]: audit 2023-09-23T19:26:25.796993+0000 mon.a (mon.0) 720 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:26:26.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:25 smithi092 bash[58855]: audit 2023-09-23T19:26:25.796993+0000 mon.a (mon.0) 720 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:26:27.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:26 smithi016 bash[75004]: audit 2023-09-23T19:26:25.797336+0000 mgr.y (mgr.25325) 539 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:26:27.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:26 smithi016 bash[75004]: cephadm 2023-09-23T19:26:25.798628+0000 mgr.y (mgr.25325) 540 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T19:26:27.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:26 smithi016 bash[75004]: cluster 2023-09-23T19:26:25.810159+0000 mgr.y (mgr.25325) 541 : cluster 0 pgmap v445: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 392 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 5970/18840 objects degraded (31.688%); 0 B/s, 17 objects/s recovering 2023-09-23T19:26:27.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:26 smithi016 bash[75004]: audit 2023-09-23T19:26:25.874374+0000 mon.c (mon.2) 104 : audit 0 from='client.? 172.21.15.16:0/3827253457' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:27.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:26 smithi016 bash[77995]: audit 2023-09-23T19:26:25.797336+0000 mgr.y (mgr.25325) 539 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:26:27.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:26 smithi016 bash[77995]: cephadm 2023-09-23T19:26:25.798628+0000 mgr.y (mgr.25325) 540 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T19:26:27.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:26 smithi016 bash[77995]: cluster 2023-09-23T19:26:25.810159+0000 mgr.y (mgr.25325) 541 : cluster 0 pgmap v445: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 392 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 5970/18840 objects degraded (31.688%); 0 B/s, 17 objects/s recovering 2023-09-23T19:26:27.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:26 smithi016 bash[77995]: audit 2023-09-23T19:26:25.874374+0000 mon.c (mon.2) 104 : audit 0 from='client.? 172.21.15.16:0/3827253457' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:27.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:26 smithi092 bash[58855]: audit 2023-09-23T19:26:25.797336+0000 mgr.y (mgr.25325) 539 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:26:27.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:26 smithi092 bash[58855]: cephadm 2023-09-23T19:26:25.798628+0000 mgr.y (mgr.25325) 540 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T19:26:27.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:26 smithi092 bash[58855]: cluster 2023-09-23T19:26:25.810159+0000 mgr.y (mgr.25325) 541 : cluster 0 pgmap v445: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 392 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 5970/18840 objects degraded (31.688%); 0 B/s, 17 objects/s recovering 2023-09-23T19:26:27.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:26 smithi092 bash[58855]: audit 2023-09-23T19:26:25.874374+0000 mon.c (mon.2) 104 : audit 0 from='client.? 172.21.15.16:0/3827253457' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:28.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:27 smithi016 bash[75004]: cluster 2023-09-23T19:26:27.815230+0000 mgr.y (mgr.25325) 542 : cluster 0 pgmap v446: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 301 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 5862/14511 objects degraded (40.397%); 0 B/s, 26 objects/s recovering 2023-09-23T19:26:28.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:27 smithi016 bash[77995]: cluster 2023-09-23T19:26:27.815230+0000 mgr.y (mgr.25325) 542 : cluster 0 pgmap v446: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 301 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 5862/14511 objects degraded (40.397%); 0 B/s, 26 objects/s recovering 2023-09-23T19:26:28.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:27 smithi092 bash[58855]: cluster 2023-09-23T19:26:27.815230+0000 mgr.y (mgr.25325) 542 : cluster 0 pgmap v446: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 301 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 5862/14511 objects degraded (40.397%); 0 B/s, 26 objects/s recovering 2023-09-23T19:26:29.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:28 smithi016 bash[77995]: audit 2023-09-23T19:26:28.549062+0000 mon.b (mon.1) 37 : audit 0 from='client.? 172.21.15.16:0/2694366942' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:29.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:28 smithi016 bash[77995]: audit 2023-09-23T19:26:28.628627+0000 mon.a (mon.0) 721 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:26:29.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:28 smithi016 bash[75004]: audit 2023-09-23T19:26:28.549062+0000 mon.b (mon.1) 37 : audit 0 from='client.? 172.21.15.16:0/2694366942' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:29.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:28 smithi016 bash[75004]: audit 2023-09-23T19:26:28.628627+0000 mon.a (mon.0) 721 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:26:29.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:28 smithi092 bash[58855]: audit 2023-09-23T19:26:28.549062+0000 mon.b (mon.1) 37 : audit 0 from='client.? 172.21.15.16:0/2694366942' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:29.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:28 smithi092 bash[58855]: audit 2023-09-23T19:26:28.628627+0000 mon.a (mon.0) 721 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:26:30.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:29 smithi016 bash[77995]: cluster 2023-09-23T19:26:29.186688+0000 mon.a (mon.0) 722 : cluster 3 Health check update: Degraded data redundancy: 5862/14511 objects degraded (40.397%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:26:30.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:29 smithi016 bash[77995]: cluster 2023-09-23T19:26:29.816626+0000 mgr.y (mgr.25325) 543 : cluster 0 pgmap v447: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 263 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 587 op/s; 5862/12675 objects degraded (46.249%); 0 B/s, 17 objects/s recovering 2023-09-23T19:26:30.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:29 smithi016 bash[75004]: cluster 2023-09-23T19:26:29.186688+0000 mon.a (mon.0) 722 : cluster 3 Health check update: Degraded data redundancy: 5862/14511 objects degraded (40.397%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:26:30.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:29 smithi016 bash[75004]: cluster 2023-09-23T19:26:29.816626+0000 mgr.y (mgr.25325) 543 : cluster 0 pgmap v447: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 263 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 587 op/s; 5862/12675 objects degraded (46.249%); 0 B/s, 17 objects/s recovering 2023-09-23T19:26:30.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:29 smithi092 bash[58855]: cluster 2023-09-23T19:26:29.186688+0000 mon.a (mon.0) 722 : cluster 3 Health check update: Degraded data redundancy: 5862/14511 objects degraded (40.397%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:26:30.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:29 smithi092 bash[58855]: cluster 2023-09-23T19:26:29.816626+0000 mgr.y (mgr.25325) 543 : cluster 0 pgmap v447: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 263 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 587 op/s; 5862/12675 objects degraded (46.249%); 0 B/s, 17 objects/s recovering 2023-09-23T19:26:31.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:31 smithi016 bash[75004]: audit 2023-09-23T19:26:31.155583+0000 mon.c (mon.2) 105 : audit 0 from='client.? 172.21.15.16:0/2907006109' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:31.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:31 smithi016 bash[77995]: audit 2023-09-23T19:26:31.155583+0000 mon.c (mon.2) 105 : audit 0 from='client.? 172.21.15.16:0/2907006109' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:31.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:31 smithi092 bash[58855]: audit 2023-09-23T19:26:31.155583+0000 mon.c (mon.2) 105 : audit 0 from='client.? 172.21.15.16:0/2907006109' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:32.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:32 smithi016 bash[75004]: cluster 2023-09-23T19:26:31.817504+0000 mgr.y (mgr.25325) 544 : cluster 0 pgmap v448: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 97 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 5748/4695 objects degraded (122.428%); 0 B/s, 27 objects/s recovering 2023-09-23T19:26:32.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:32 smithi016 bash[77995]: cluster 2023-09-23T19:26:31.817504+0000 mgr.y (mgr.25325) 544 : cluster 0 pgmap v448: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 97 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 5748/4695 objects degraded (122.428%); 0 B/s, 27 objects/s recovering 2023-09-23T19:26:32.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:32 smithi092 bash[58855]: cluster 2023-09-23T19:26:31.817504+0000 mgr.y (mgr.25325) 544 : cluster 0 pgmap v448: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 97 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 5748/4695 objects degraded (122.428%); 0 B/s, 27 objects/s recovering 2023-09-23T19:26:32.700 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Removed 100425 objects 2023-09-23T19:26:32.700 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Clean up completed and total clean up time :152.168 2023-09-23T19:26:33.518 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_3 2023-09-23T19:26:33.519 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_3 unique_pool_3 --yes-i-really-really-mean-it 2023-09-23T19:26:34.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:33 smithi016 bash[75004]: audit 2023-09-23T19:26:33.541879+0000 mon.c (mon.2) 106 : audit 0 from='client.? 172.21.15.16:0/1721002393' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:34.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:33 smithi016 bash[77995]: audit 2023-09-23T19:26:33.541879+0000 mon.c (mon.2) 106 : audit 0 from='client.? 172.21.15.16:0/1721002393' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:34.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:33 smithi092 bash[58855]: audit 2023-09-23T19:26:33.541879+0000 mon.c (mon.2) 106 : audit 0 from='client.? 172.21.15.16:0/1721002393' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:35.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:34 smithi016 bash[75004]: cluster 2023-09-23T19:26:33.818367+0000 mgr.y (mgr.25325) 545 : cluster 0 pgmap v449: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 69 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 5748/3348 objects degraded (171.685%); 0 B/s, 18 objects/s recovering 2023-09-23T19:26:35.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:34 smithi016 bash[75004]: cluster 2023-09-23T19:26:34.187980+0000 mon.a (mon.0) 723 : cluster 3 Health check update: Degraded data redundancy: 5748/4695 objects degraded (122.428%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:26:35.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:34 smithi016 bash[77995]: cluster 2023-09-23T19:26:33.818367+0000 mgr.y (mgr.25325) 545 : cluster 0 pgmap v449: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 69 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 5748/3348 objects degraded (171.685%); 0 B/s, 18 objects/s recovering 2023-09-23T19:26:35.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:34 smithi016 bash[77995]: cluster 2023-09-23T19:26:34.187980+0000 mon.a (mon.0) 723 : cluster 3 Health check update: Degraded data redundancy: 5748/4695 objects degraded (122.428%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:26:35.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:34 smithi092 bash[58855]: cluster 2023-09-23T19:26:33.818367+0000 mgr.y (mgr.25325) 545 : cluster 0 pgmap v449: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 69 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 5748/3348 objects degraded (171.685%); 0 B/s, 18 objects/s recovering 2023-09-23T19:26:35.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:34 smithi092 bash[58855]: cluster 2023-09-23T19:26:34.187980+0000 mon.a (mon.0) 723 : cluster 3 Health check update: Degraded data redundancy: 5748/4695 objects degraded (122.428%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:26:35.924 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:26:35 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:26:35] "GET /metrics HTTP/1.1" 200 36175 "" "Prometheus/2.33.4" 2023-09-23T19:26:36.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:35 smithi016 bash[75004]: cluster 2023-09-23T19:26:35.819325+0000 mgr.y (mgr.25325) 546 : cluster 0 pgmap v450: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 24 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 5748/1197 objects degraded (480.201%); 0 B/s, 18 objects/s recovering 2023-09-23T19:26:36.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:35 smithi016 bash[75004]: audit 2023-09-23T19:26:35.918648+0000 mon.c (mon.2) 107 : audit 0 from='client.? 172.21.15.16:0/93479201' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:36.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:35 smithi016 bash[77995]: cluster 2023-09-23T19:26:35.819325+0000 mgr.y (mgr.25325) 546 : cluster 0 pgmap v450: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 24 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 5748/1197 objects degraded (480.201%); 0 B/s, 18 objects/s recovering 2023-09-23T19:26:36.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:35 smithi016 bash[77995]: audit 2023-09-23T19:26:35.918648+0000 mon.c (mon.2) 107 : audit 0 from='client.? 172.21.15.16:0/93479201' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:36.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:35 smithi092 bash[58855]: cluster 2023-09-23T19:26:35.819325+0000 mgr.y (mgr.25325) 546 : cluster 0 pgmap v450: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 24 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 5748/1197 objects degraded (480.201%); 0 B/s, 18 objects/s recovering 2023-09-23T19:26:36.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:35 smithi092 bash[58855]: audit 2023-09-23T19:26:35.918648+0000 mon.c (mon.2) 107 : audit 0 from='client.? 172.21.15.16:0/93479201' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:38.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:37 smithi016 bash[75004]: cluster 2023-09-23T19:26:37.819995+0000 mgr.y (mgr.25325) 547 : cluster 0 pgmap v451: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 5632/72 objects degraded (7822.222%); 0 B/s, 28 objects/s recovering 2023-09-23T19:26:38.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:37 smithi016 bash[77995]: cluster 2023-09-23T19:26:37.819995+0000 mgr.y (mgr.25325) 547 : cluster 0 pgmap v451: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 5632/72 objects degraded (7822.222%); 0 B/s, 28 objects/s recovering 2023-09-23T19:26:38.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:37 smithi092 bash[58855]: cluster 2023-09-23T19:26:37.819995+0000 mgr.y (mgr.25325) 547 : cluster 0 pgmap v451: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 5632/72 objects degraded (7822.222%); 0 B/s, 28 objects/s recovering 2023-09-23T19:26:38.581 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:26:39.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:38 smithi016 bash[75004]: audit 2023-09-23T19:26:38.368460+0000 mon.c (mon.2) 108 : audit 0 from='client.? 172.21.15.16:0/4254824913' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:39.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:38 smithi016 bash[77995]: audit 2023-09-23T19:26:38.368460+0000 mon.c (mon.2) 108 : audit 0 from='client.? 172.21.15.16:0/4254824913' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:39.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:38 smithi092 bash[58855]: audit 2023-09-23T19:26:38.368460+0000 mon.c (mon.2) 108 : audit 0 from='client.? 172.21.15.16:0/4254824913' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:40.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:39 smithi016 bash[77995]: cluster 2023-09-23T19:26:39.189249+0000 mon.a (mon.0) 724 : cluster 3 Health check update: Degraded data redundancy: 5632/72 objects degraded (7822.222%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:26:40.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:39 smithi016 bash[77995]: cluster 2023-09-23T19:26:39.820615+0000 mgr.y (mgr.25325) 548 : cluster 0 pgmap v452: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 400 op/s; 5632/72 objects degraded (7822.222%); 0 B/s, 19 objects/s recovering 2023-09-23T19:26:40.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:39 smithi016 bash[75004]: cluster 2023-09-23T19:26:39.189249+0000 mon.a (mon.0) 724 : cluster 3 Health check update: Degraded data redundancy: 5632/72 objects degraded (7822.222%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:26:40.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:39 smithi016 bash[75004]: cluster 2023-09-23T19:26:39.820615+0000 mgr.y (mgr.25325) 548 : cluster 0 pgmap v452: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 400 op/s; 5632/72 objects degraded (7822.222%); 0 B/s, 19 objects/s recovering 2023-09-23T19:26:40.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:39 smithi092 bash[58855]: cluster 2023-09-23T19:26:39.189249+0000 mon.a (mon.0) 724 : cluster 3 Health check update: Degraded data redundancy: 5632/72 objects degraded (7822.222%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:26:40.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:39 smithi092 bash[58855]: cluster 2023-09-23T19:26:39.820615+0000 mgr.y (mgr.25325) 548 : cluster 0 pgmap v452: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 400 op/s; 5632/72 objects degraded (7822.222%); 0 B/s, 19 objects/s recovering 2023-09-23T19:26:41.199 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:41 smithi016 bash[75004]: audit 2023-09-23T19:26:40.721779+0000 mon.a (mon.0) 725 : audit 0 from='client.? 172.21.15.16:0/1138100468' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:41.201 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:41 smithi016 bash[75004]: audit 2023-09-23T19:26:40.812946+0000 mon.a (mon.0) 726 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:26:41.201 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:41 smithi016 bash[75004]: audit 2023-09-23T19:26:40.813193+0000 mgr.y (mgr.25325) 549 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:26:41.202 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:41 smithi016 bash[75004]: cephadm 2023-09-23T19:26:40.814802+0000 mgr.y (mgr.25325) 550 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T19:26:41.202 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:26:40 smithi016 bash[70992]: debug 2023-09-23T19:26:40.809+0000 7fb2a5a7c700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T19:26:41.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:41 smithi016 bash[77995]: audit 2023-09-23T19:26:40.721779+0000 mon.a (mon.0) 725 : audit 0 from='client.? 172.21.15.16:0/1138100468' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:41.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:41 smithi016 bash[77995]: audit 2023-09-23T19:26:40.812946+0000 mon.a (mon.0) 726 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:26:41.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:41 smithi016 bash[77995]: audit 2023-09-23T19:26:40.813193+0000 mgr.y (mgr.25325) 549 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:26:41.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:41 smithi016 bash[77995]: cephadm 2023-09-23T19:26:40.814802+0000 mgr.y (mgr.25325) 550 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T19:26:41.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:41 smithi092 bash[58855]: audit 2023-09-23T19:26:40.721779+0000 mon.a (mon.0) 725 : audit 0 from='client.? 172.21.15.16:0/1138100468' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:41.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:41 smithi092 bash[58855]: audit 2023-09-23T19:26:40.812946+0000 mon.a (mon.0) 726 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:26:41.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:41 smithi092 bash[58855]: audit 2023-09-23T19:26:40.813193+0000 mgr.y (mgr.25325) 549 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:26:41.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:41 smithi092 bash[58855]: cephadm 2023-09-23T19:26:40.814802+0000 mgr.y (mgr.25325) 550 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T19:26:42.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:42 smithi016 bash[75004]: cluster 2023-09-23T19:26:41.821483+0000 mgr.y (mgr.25325) 551 : cluster 0 pgmap v453: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 349 op/s; 5516/72 objects degraded (7661.111%); 0 B/s, 28 objects/s recovering 2023-09-23T19:26:42.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:42 smithi016 bash[77995]: cluster 2023-09-23T19:26:41.821483+0000 mgr.y (mgr.25325) 551 : cluster 0 pgmap v453: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 349 op/s; 5516/72 objects degraded (7661.111%); 0 B/s, 28 objects/s recovering 2023-09-23T19:26:42.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:42 smithi092 bash[58855]: cluster 2023-09-23T19:26:41.821483+0000 mgr.y (mgr.25325) 551 : cluster 0 pgmap v453: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 349 op/s; 5516/72 objects degraded (7661.111%); 0 B/s, 28 objects/s recovering 2023-09-23T19:26:43.239 INFO:teuthology.orchestra.run.smithi016.stderr:pool 'unique_pool_3' removed 2023-09-23T19:26:43.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:43 smithi016 bash[75004]: audit 2023-09-23T19:26:42.284815+0000 mon.c (mon.2) 109 : audit 1 from='client.? 172.21.15.16:0/551759875' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T19:26:43.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:43 smithi016 bash[75004]: audit 2023-09-23T19:26:42.285485+0000 mon.a (mon.0) 727 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T19:26:43.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:43 smithi016 bash[75004]: audit 2023-09-23T19:26:43.079931+0000 mon.a (mon.0) 728 : audit 0 from='client.? 172.21.15.16:0/695091982' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:43.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:43 smithi016 bash[77995]: audit 2023-09-23T19:26:42.284815+0000 mon.c (mon.2) 109 : audit 1 from='client.? 172.21.15.16:0/551759875' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T19:26:43.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:43 smithi016 bash[77995]: audit 2023-09-23T19:26:42.285485+0000 mon.a (mon.0) 727 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T19:26:43.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:43 smithi016 bash[77995]: audit 2023-09-23T19:26:43.079931+0000 mon.a (mon.0) 728 : audit 0 from='client.? 172.21.15.16:0/695091982' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:43.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:43 smithi092 bash[58855]: audit 2023-09-23T19:26:42.284815+0000 mon.c (mon.2) 109 : audit 1 from='client.? 172.21.15.16:0/551759875' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T19:26:43.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:43 smithi092 bash[58855]: audit 2023-09-23T19:26:42.285485+0000 mon.a (mon.0) 727 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T19:26:43.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:43 smithi092 bash[58855]: audit 2023-09-23T19:26:43.079931+0000 mon.a (mon.0) 728 : audit 0 from='client.? 172.21.15.16:0/695091982' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:44.099 INFO:teuthology.task.sequential:In sequential, running task print... 2023-09-23T19:26:44.099 INFO:teuthology.task.print:**** done end radosbench.yaml 2023-09-23T19:26:44.099 DEBUG:teuthology.parallel:result is None 2023-09-23T19:26:44.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:44 smithi016 bash[77995]: cluster 2023-09-23T19:26:43.212028+0000 osd.2 (osd.2) 30 : cluster 3 failed to encode map e107 with expected crc 2023-09-23T19:26:44.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:44 smithi016 bash[77995]: audit 2023-09-23T19:26:43.214621+0000 mon.a (mon.0) 729 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T19:26:44.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:44 smithi016 bash[77995]: cluster 2023-09-23T19:26:43.214828+0000 mon.a (mon.0) 730 : cluster 0 osdmap e107: 8 total, 8 up, 8 in 2023-09-23T19:26:44.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:44 smithi016 bash[77995]: cluster 2023-09-23T19:26:43.242455+0000 osd.3 (osd.3) 37 : cluster 3 failed to encode map e107 with expected crc 2023-09-23T19:26:44.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:44 smithi016 bash[77995]: audit 2023-09-23T19:26:43.629674+0000 mon.a (mon.0) 731 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:26:44.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:44 smithi016 bash[77995]: cluster 2023-09-23T19:26:43.822151+0000 mgr.y (mgr.25325) 552 : cluster 0 pgmap v455: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:26:44.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:44 smithi016 bash[77995]: cluster 2023-09-23T19:26:44.190718+0000 mon.a (mon.0) 732 : cluster 3 Health check update: Degraded data redundancy: 5516/72 objects degraded (7661.111%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:26:44.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:44 smithi016 bash[75004]: cluster 2023-09-23T19:26:43.212028+0000 osd.2 (osd.2) 30 : cluster 3 failed to encode map e107 with expected crc 2023-09-23T19:26:44.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:44 smithi016 bash[75004]: audit 2023-09-23T19:26:43.214621+0000 mon.a (mon.0) 729 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T19:26:44.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:44 smithi016 bash[75004]: cluster 2023-09-23T19:26:43.214828+0000 mon.a (mon.0) 730 : cluster 0 osdmap e107: 8 total, 8 up, 8 in 2023-09-23T19:26:44.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:44 smithi016 bash[75004]: cluster 2023-09-23T19:26:43.242455+0000 osd.3 (osd.3) 37 : cluster 3 failed to encode map e107 with expected crc 2023-09-23T19:26:44.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:44 smithi016 bash[75004]: audit 2023-09-23T19:26:43.629674+0000 mon.a (mon.0) 731 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:26:44.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:44 smithi016 bash[75004]: cluster 2023-09-23T19:26:43.822151+0000 mgr.y (mgr.25325) 552 : cluster 0 pgmap v455: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:26:44.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:44 smithi016 bash[75004]: cluster 2023-09-23T19:26:44.190718+0000 mon.a (mon.0) 732 : cluster 3 Health check update: Degraded data redundancy: 5516/72 objects degraded (7661.111%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:26:44.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:44 smithi092 bash[58855]: cluster 2023-09-23T19:26:43.212028+0000 osd.2 (osd.2) 30 : cluster 3 failed to encode map e107 with expected crc 2023-09-23T19:26:44.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:44 smithi092 bash[58855]: audit 2023-09-23T19:26:43.214621+0000 mon.a (mon.0) 729 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T19:26:44.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:44 smithi092 bash[58855]: cluster 2023-09-23T19:26:43.214828+0000 mon.a (mon.0) 730 : cluster 0 osdmap e107: 8 total, 8 up, 8 in 2023-09-23T19:26:44.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:44 smithi092 bash[58855]: cluster 2023-09-23T19:26:43.242455+0000 osd.3 (osd.3) 37 : cluster 3 failed to encode map e107 with expected crc 2023-09-23T19:26:44.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:44 smithi092 bash[58855]: audit 2023-09-23T19:26:43.629674+0000 mon.a (mon.0) 731 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:26:44.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:44 smithi092 bash[58855]: cluster 2023-09-23T19:26:43.822151+0000 mgr.y (mgr.25325) 552 : cluster 0 pgmap v455: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:26:44.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:44 smithi092 bash[58855]: cluster 2023-09-23T19:26:44.190718+0000 mon.a (mon.0) 732 : cluster 3 Health check update: Degraded data redundancy: 5516/72 objects degraded (7661.111%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:26:45.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:45 smithi016 bash[77995]: cluster 2023-09-23T19:26:44.211005+0000 mon.a (mon.0) 733 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5516/72 objects degraded (7661.111%), 4 pgs degraded, 4 pgs undersized) 2023-09-23T19:26:45.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:45 smithi016 bash[77995]: cluster 2023-09-23T19:26:44.211043+0000 mon.a (mon.0) 734 : cluster 1 Cluster is now healthy 2023-09-23T19:26:45.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:45 smithi016 bash[77995]: cluster 2023-09-23T19:26:44.422004+0000 osd.5 (osd.5) 25 : cluster 3 failed to encode map e107 with expected crc 2023-09-23T19:26:45.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:45 smithi016 bash[77995]: cluster 2023-09-23T19:26:44.424878+0000 osd.5 (osd.5) 26 : cluster 3 failed to encode map e107 with expected crc 2023-09-23T19:26:45.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:45 smithi016 bash[77995]: cluster 2023-09-23T19:26:44.582989+0000 osd.4 (osd.4) 24 : cluster 3 failed to encode map e107 with expected crc 2023-09-23T19:26:45.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:45 smithi016 bash[75004]: cluster 2023-09-23T19:26:44.211005+0000 mon.a (mon.0) 733 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5516/72 objects degraded (7661.111%), 4 pgs degraded, 4 pgs undersized) 2023-09-23T19:26:45.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:45 smithi016 bash[75004]: cluster 2023-09-23T19:26:44.211043+0000 mon.a (mon.0) 734 : cluster 1 Cluster is now healthy 2023-09-23T19:26:45.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:45 smithi016 bash[75004]: cluster 2023-09-23T19:26:44.422004+0000 osd.5 (osd.5) 25 : cluster 3 failed to encode map e107 with expected crc 2023-09-23T19:26:45.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:45 smithi016 bash[75004]: cluster 2023-09-23T19:26:44.424878+0000 osd.5 (osd.5) 26 : cluster 3 failed to encode map e107 with expected crc 2023-09-23T19:26:45.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:45 smithi016 bash[75004]: cluster 2023-09-23T19:26:44.582989+0000 osd.4 (osd.4) 24 : cluster 3 failed to encode map e107 with expected crc 2023-09-23T19:26:45.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:45 smithi092 bash[58855]: cluster 2023-09-23T19:26:44.211005+0000 mon.a (mon.0) 733 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5516/72 objects degraded (7661.111%), 4 pgs degraded, 4 pgs undersized) 2023-09-23T19:26:45.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:45 smithi092 bash[58855]: cluster 2023-09-23T19:26:44.211043+0000 mon.a (mon.0) 734 : cluster 1 Cluster is now healthy 2023-09-23T19:26:45.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:45 smithi092 bash[58855]: cluster 2023-09-23T19:26:44.422004+0000 osd.5 (osd.5) 25 : cluster 3 failed to encode map e107 with expected crc 2023-09-23T19:26:45.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:45 smithi092 bash[58855]: cluster 2023-09-23T19:26:44.424878+0000 osd.5 (osd.5) 26 : cluster 3 failed to encode map e107 with expected crc 2023-09-23T19:26:45.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:45 smithi092 bash[58855]: cluster 2023-09-23T19:26:44.582989+0000 osd.4 (osd.4) 24 : cluster 3 failed to encode map e107 with expected crc 2023-09-23T19:26:46.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:26:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:26:45] "GET /metrics HTTP/1.1" 200 33819 "" "Prometheus/2.33.4" 2023-09-23T19:26:46.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:46 smithi016 bash[75004]: audit 2023-09-23T19:26:45.460002+0000 mon.a (mon.0) 735 : audit 0 from='client.? 172.21.15.16:0/800161858' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:46.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:46 smithi016 bash[75004]: cluster 2023-09-23T19:26:45.822994+0000 mgr.y (mgr.25325) 553 : cluster 0 pgmap v456: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:26:46.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:46 smithi016 bash[77995]: audit 2023-09-23T19:26:45.460002+0000 mon.a (mon.0) 735 : audit 0 from='client.? 172.21.15.16:0/800161858' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:46.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:46 smithi016 bash[77995]: cluster 2023-09-23T19:26:45.822994+0000 mgr.y (mgr.25325) 553 : cluster 0 pgmap v456: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:26:46.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:46 smithi092 bash[58855]: audit 2023-09-23T19:26:45.460002+0000 mon.a (mon.0) 735 : audit 0 from='client.? 172.21.15.16:0/800161858' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:46.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:46 smithi092 bash[58855]: cluster 2023-09-23T19:26:45.822994+0000 mgr.y (mgr.25325) 553 : cluster 0 pgmap v456: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:26:48.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:47 smithi016 bash[75004]: cluster 2023-09-23T19:26:47.823804+0000 mgr.y (mgr.25325) 554 : cluster 0 pgmap v457: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:26:48.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:47 smithi016 bash[75004]: audit 2023-09-23T19:26:47.832957+0000 mon.a (mon.0) 736 : audit 0 from='client.? 172.21.15.16:0/1464978611' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:48.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:47 smithi016 bash[77995]: cluster 2023-09-23T19:26:47.823804+0000 mgr.y (mgr.25325) 554 : cluster 0 pgmap v457: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:26:48.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:47 smithi016 bash[77995]: audit 2023-09-23T19:26:47.832957+0000 mon.a (mon.0) 736 : audit 0 from='client.? 172.21.15.16:0/1464978611' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:48.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:47 smithi092 bash[58855]: cluster 2023-09-23T19:26:47.823804+0000 mgr.y (mgr.25325) 554 : cluster 0 pgmap v457: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:26:48.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:47 smithi092 bash[58855]: audit 2023-09-23T19:26:47.832957+0000 mon.a (mon.0) 736 : audit 0 from='client.? 172.21.15.16:0/1464978611' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:50.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:49 smithi016 bash[75004]: audit 2023-09-23T19:26:48.805440+0000 mon.a (mon.0) 737 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:26:50.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:49 smithi016 bash[77995]: audit 2023-09-23T19:26:48.805440+0000 mon.a (mon.0) 737 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:26:50.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:49 smithi092 bash[58855]: audit 2023-09-23T19:26:48.805440+0000 mon.a (mon.0) 737 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:26:51.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:50 smithi016 bash[75004]: cluster 2023-09-23T19:26:49.824631+0000 mgr.y (mgr.25325) 555 : cluster 0 pgmap v458: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:26:51.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:50 smithi016 bash[75004]: audit 2023-09-23T19:26:50.211642+0000 mon.c (mon.2) 110 : audit 0 from='client.? 172.21.15.16:0/1558330031' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:51.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:50 smithi016 bash[77995]: cluster 2023-09-23T19:26:49.824631+0000 mgr.y (mgr.25325) 555 : cluster 0 pgmap v458: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:26:51.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:50 smithi016 bash[77995]: audit 2023-09-23T19:26:50.211642+0000 mon.c (mon.2) 110 : audit 0 from='client.? 172.21.15.16:0/1558330031' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:51.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:50 smithi092 bash[58855]: cluster 2023-09-23T19:26:49.824631+0000 mgr.y (mgr.25325) 555 : cluster 0 pgmap v458: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:26:51.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:50 smithi092 bash[58855]: audit 2023-09-23T19:26:50.211642+0000 mon.c (mon.2) 110 : audit 0 from='client.? 172.21.15.16:0/1558330031' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:52.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:51 smithi016 bash[77995]: cluster 2023-09-23T19:26:51.825657+0000 mgr.y (mgr.25325) 556 : cluster 0 pgmap v459: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:26:52.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:51 smithi016 bash[75004]: cluster 2023-09-23T19:26:51.825657+0000 mgr.y (mgr.25325) 556 : cluster 0 pgmap v459: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:26:52.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:51 smithi092 bash[58855]: cluster 2023-09-23T19:26:51.825657+0000 mgr.y (mgr.25325) 556 : cluster 0 pgmap v459: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:26:53.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:52 smithi016 bash[75004]: audit 2023-09-23T19:26:52.578181+0000 mon.a (mon.0) 738 : audit 0 from='client.? 172.21.15.16:0/410927757' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:53.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:52 smithi016 bash[77995]: audit 2023-09-23T19:26:52.578181+0000 mon.a (mon.0) 738 : audit 0 from='client.? 172.21.15.16:0/410927757' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:53.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:52 smithi092 bash[58855]: audit 2023-09-23T19:26:52.578181+0000 mon.a (mon.0) 738 : audit 0 from='client.? 172.21.15.16:0/410927757' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:54.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:53 smithi016 bash[75004]: cluster 2023-09-23T19:26:53.826363+0000 mgr.y (mgr.25325) 557 : cluster 0 pgmap v460: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:26:54.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:53 smithi016 bash[77995]: cluster 2023-09-23T19:26:53.826363+0000 mgr.y (mgr.25325) 557 : cluster 0 pgmap v460: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:26:54.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:53 smithi092 bash[58855]: cluster 2023-09-23T19:26:53.826363+0000 mgr.y (mgr.25325) 557 : cluster 0 pgmap v460: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:26:55.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:55 smithi016 bash[75004]: audit 2023-09-23T19:26:54.949574+0000 mon.c (mon.2) 111 : audit 0 from='client.? 172.21.15.16:0/2532452548' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:55.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:55 smithi016 bash[77995]: audit 2023-09-23T19:26:54.949574+0000 mon.c (mon.2) 111 : audit 0 from='client.? 172.21.15.16:0/2532452548' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:55.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:55 smithi092 bash[58855]: audit 2023-09-23T19:26:54.949574+0000 mon.c (mon.2) 111 : audit 0 from='client.? 172.21.15.16:0/2532452548' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:56.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:26:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:26:55] "GET /metrics HTTP/1.1" 200 33819 "" "Prometheus/2.33.4" 2023-09-23T19:26:56.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:56 smithi016 bash[75004]: cluster 2023-09-23T19:26:55.827257+0000 mgr.y (mgr.25325) 558 : cluster 0 pgmap v461: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:26:56.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:56 smithi016 bash[75004]: audit 2023-09-23T19:26:55.830577+0000 mon.a (mon.0) 739 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:26:56.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:56 smithi016 bash[75004]: audit 2023-09-23T19:26:55.830970+0000 mgr.y (mgr.25325) 559 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:26:56.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:56 smithi016 bash[75004]: cephadm 2023-09-23T19:26:55.833350+0000 mgr.y (mgr.25325) 560 : cephadm 1 Upgrade: osd.2 is safe to restart 2023-09-23T19:26:56.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:56 smithi016 bash[77995]: cluster 2023-09-23T19:26:55.827257+0000 mgr.y (mgr.25325) 558 : cluster 0 pgmap v461: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:26:56.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:56 smithi016 bash[77995]: audit 2023-09-23T19:26:55.830577+0000 mon.a (mon.0) 739 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:26:56.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:56 smithi016 bash[77995]: audit 2023-09-23T19:26:55.830970+0000 mgr.y (mgr.25325) 559 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:26:56.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:56 smithi016 bash[77995]: cephadm 2023-09-23T19:26:55.833350+0000 mgr.y (mgr.25325) 560 : cephadm 1 Upgrade: osd.2 is safe to restart 2023-09-23T19:26:56.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:56 smithi092 bash[58855]: cluster 2023-09-23T19:26:55.827257+0000 mgr.y (mgr.25325) 558 : cluster 0 pgmap v461: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:26:56.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:56 smithi092 bash[58855]: audit 2023-09-23T19:26:55.830577+0000 mon.a (mon.0) 739 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:26:56.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:56 smithi092 bash[58855]: audit 2023-09-23T19:26:55.830970+0000 mgr.y (mgr.25325) 559 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:26:56.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:56 smithi092 bash[58855]: cephadm 2023-09-23T19:26:55.833350+0000 mgr.y (mgr.25325) 560 : cephadm 1 Upgrade: osd.2 is safe to restart 2023-09-23T19:26:57.649 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:57 smithi016 bash[75004]: audit 2023-09-23T19:26:57.317212+0000 mon.c (mon.2) 112 : audit 0 from='client.? 172.21.15.16:0/3030751829' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:57.649 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:57 smithi016 bash[77995]: audit 2023-09-23T19:26:57.317212+0000 mon.c (mon.2) 112 : audit 0 from='client.? 172.21.15.16:0/3030751829' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:57.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:57 smithi092 bash[58855]: audit 2023-09-23T19:26:57.317212+0000 mon.c (mon.2) 112 : audit 0 from='client.? 172.21.15.16:0/3030751829' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:26:58.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:58 smithi016 bash[75004]: cluster 2023-09-23T19:26:57.828059+0000 mgr.y (mgr.25325) 561 : cluster 0 pgmap v462: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:26:58.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:58 smithi016 bash[77995]: cluster 2023-09-23T19:26:57.828059+0000 mgr.y (mgr.25325) 561 : cluster 0 pgmap v462: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:26:58.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:58 smithi092 bash[58855]: cluster 2023-09-23T19:26:57.828059+0000 mgr.y (mgr.25325) 561 : cluster 0 pgmap v462: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:26:59.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:59 smithi016 bash[75004]: cephadm 2023-09-23T19:26:58.503465+0000 mgr.y (mgr.25325) 562 : cephadm 1 Upgrade: Updating osd.2 2023-09-23T19:26:59.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:59 smithi016 bash[75004]: audit 2023-09-23T19:26:58.514673+0000 mon.a (mon.0) 740 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:26:59.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:59 smithi016 bash[75004]: audit 2023-09-23T19:26:58.516485+0000 mon.a (mon.0) 741 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T19:26:59.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:59 smithi016 bash[75004]: audit 2023-09-23T19:26:58.517813+0000 mon.a (mon.0) 742 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:26:59.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:59 smithi016 bash[75004]: cephadm 2023-09-23T19:26:58.521717+0000 mgr.y (mgr.25325) 563 : cephadm 1 Deploying daemon osd.2 on smithi016 2023-09-23T19:26:59.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:59 smithi016 bash[75004]: audit 2023-09-23T19:26:58.635708+0000 mon.a (mon.0) 743 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:26:59.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:26:59 smithi016 bash[75004]: audit 2023-09-23T19:26:58.637114+0000 mon.a (mon.0) 744 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:26:59.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:59 smithi016 bash[77995]: cephadm 2023-09-23T19:26:58.503465+0000 mgr.y (mgr.25325) 562 : cephadm 1 Upgrade: Updating osd.2 2023-09-23T19:26:59.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:59 smithi016 bash[77995]: audit 2023-09-23T19:26:58.514673+0000 mon.a (mon.0) 740 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:26:59.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:59 smithi016 bash[77995]: audit 2023-09-23T19:26:58.516485+0000 mon.a (mon.0) 741 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T19:26:59.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:59 smithi016 bash[77995]: audit 2023-09-23T19:26:58.517813+0000 mon.a (mon.0) 742 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:26:59.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:59 smithi016 bash[77995]: cephadm 2023-09-23T19:26:58.521717+0000 mgr.y (mgr.25325) 563 : cephadm 1 Deploying daemon osd.2 on smithi016 2023-09-23T19:26:59.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:59 smithi016 bash[77995]: audit 2023-09-23T19:26:58.635708+0000 mon.a (mon.0) 743 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:26:59.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:26:59 smithi016 bash[77995]: audit 2023-09-23T19:26:58.637114+0000 mon.a (mon.0) 744 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:26:59.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:59 smithi092 bash[58855]: cephadm 2023-09-23T19:26:58.503465+0000 mgr.y (mgr.25325) 562 : cephadm 1 Upgrade: Updating osd.2 2023-09-23T19:26:59.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:59 smithi092 bash[58855]: audit 2023-09-23T19:26:58.514673+0000 mon.a (mon.0) 740 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:26:59.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:59 smithi092 bash[58855]: audit 2023-09-23T19:26:58.516485+0000 mon.a (mon.0) 741 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T19:26:59.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:59 smithi092 bash[58855]: audit 2023-09-23T19:26:58.517813+0000 mon.a (mon.0) 742 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:26:59.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:59 smithi092 bash[58855]: cephadm 2023-09-23T19:26:58.521717+0000 mgr.y (mgr.25325) 563 : cephadm 1 Deploying daemon osd.2 on smithi016 2023-09-23T19:26:59.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:59 smithi092 bash[58855]: audit 2023-09-23T19:26:58.635708+0000 mon.a (mon.0) 743 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:26:59.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:26:59 smithi092 bash[58855]: audit 2023-09-23T19:26:58.637114+0000 mon.a (mon.0) 744 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:27:00.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:00 smithi016 bash[77995]: audit 2023-09-23T19:26:59.685209+0000 mon.a (mon.0) 745 : audit 0 from='client.? 172.21.15.16:0/2864455133' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:00.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:00 smithi016 bash[77995]: cluster 2023-09-23T19:26:59.828709+0000 mgr.y (mgr.25325) 564 : cluster 0 pgmap v463: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:27:00.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:00 smithi016 bash[75004]: audit 2023-09-23T19:26:59.685209+0000 mon.a (mon.0) 745 : audit 0 from='client.? 172.21.15.16:0/2864455133' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:00.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:00 smithi016 bash[75004]: cluster 2023-09-23T19:26:59.828709+0000 mgr.y (mgr.25325) 564 : cluster 0 pgmap v463: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:27:00.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:00 smithi092 bash[58855]: audit 2023-09-23T19:26:59.685209+0000 mon.a (mon.0) 745 : audit 0 from='client.? 172.21.15.16:0/2864455133' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:00.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:00 smithi092 bash[58855]: cluster 2023-09-23T19:26:59.828709+0000 mgr.y (mgr.25325) 564 : cluster 0 pgmap v463: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:27:02.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:01 smithi016 bash[75004]: cluster 2023-09-23T19:27:01.829435+0000 mgr.y (mgr.25325) 565 : cluster 0 pgmap v464: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:27:02.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:01 smithi016 bash[77995]: cluster 2023-09-23T19:27:01.829435+0000 mgr.y (mgr.25325) 565 : cluster 0 pgmap v464: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:27:02.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:01 smithi092 bash[58855]: cluster 2023-09-23T19:27:01.829435+0000 mgr.y (mgr.25325) 565 : cluster 0 pgmap v464: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:27:03.207 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:02 smithi016 bash[75004]: audit 2023-09-23T19:27:02.066846+0000 mon.a (mon.0) 746 : audit 0 from='client.? 172.21.15.16:0/817019209' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:03.207 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:02 smithi016 bash[77995]: audit 2023-09-23T19:27:02.066846+0000 mon.a (mon.0) 746 : audit 0 from='client.? 172.21.15.16:0/817019209' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:03.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:02 smithi092 bash[58855]: audit 2023-09-23T19:27:02.066846+0000 mon.a (mon.0) 746 : audit 0 from='client.? 172.21.15.16:0/817019209' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:03.785 INFO:journalctl@ceph.osd.2.smithi016.stdout:Sep 23 19:27:03 smithi016 systemd[1]: Stopping Ceph osd.2 for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7... 2023-09-23T19:27:03.785 INFO:journalctl@ceph.osd.2.smithi016.stdout:Sep 23 19:27:03 smithi016 bash[44998]: debug 2023-09-23T19:27:03.629+0000 7f37467c8700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-09-23T19:27:03.785 INFO:journalctl@ceph.osd.2.smithi016.stdout:Sep 23 19:27:03 smithi016 bash[44998]: debug 2023-09-23T19:27:03.629+0000 7f37467c8700 -1 osd.2 107 *** Got signal Terminated *** 2023-09-23T19:27:03.785 INFO:journalctl@ceph.osd.2.smithi016.stdout:Sep 23 19:27:03 smithi016 bash[44998]: debug 2023-09-23T19:27:03.629+0000 7f37467c8700 -1 osd.2 107 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T19:27:04.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:03 smithi016 bash[75004]: cluster 2023-09-23T19:27:03.634230+0000 mon.a (mon.0) 747 : cluster 1 osd.2 marked itself down and dead 2023-09-23T19:27:04.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:03 smithi016 bash[75004]: cluster 2023-09-23T19:27:03.830175+0000 mgr.y (mgr.25325) 566 : cluster 0 pgmap v465: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:27:04.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:03 smithi016 bash[77995]: cluster 2023-09-23T19:27:03.634230+0000 mon.a (mon.0) 747 : cluster 1 osd.2 marked itself down and dead 2023-09-23T19:27:04.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:03 smithi016 bash[77995]: cluster 2023-09-23T19:27:03.830175+0000 mgr.y (mgr.25325) 566 : cluster 0 pgmap v465: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:27:04.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:03 smithi092 bash[58855]: cluster 2023-09-23T19:27:03.634230+0000 mon.a (mon.0) 747 : cluster 1 osd.2 marked itself down and dead 2023-09-23T19:27:04.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:03 smithi092 bash[58855]: cluster 2023-09-23T19:27:03.830175+0000 mgr.y (mgr.25325) 566 : cluster 0 pgmap v465: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:27:05.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:04 smithi016 bash[77995]: cluster 2023-09-23T19:27:03.930473+0000 mon.a (mon.0) 748 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T19:27:05.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:04 smithi016 bash[77995]: cluster 2023-09-23T19:27:03.942168+0000 osd.3 (osd.3) 38 : cluster 3 failed to encode map e108 with expected crc 2023-09-23T19:27:05.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:04 smithi016 bash[77995]: cluster 2023-09-23T19:27:03.944571+0000 mon.a (mon.0) 749 : cluster 0 osdmap e108: 8 total, 7 up, 8 in 2023-09-23T19:27:05.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:04 smithi016 bash[77995]: cluster 2023-09-23T19:27:03.946171+0000 osd.6 (osd.6) 25 : cluster 3 failed to encode map e108 with expected crc 2023-09-23T19:27:05.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:04 smithi016 bash[77995]: cluster 2023-09-23T19:27:03.947857+0000 osd.4 (osd.4) 25 : cluster 3 failed to encode map e108 with expected crc 2023-09-23T19:27:05.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:04 smithi016 bash[77995]: cluster 2023-09-23T19:27:03.948934+0000 osd.5 (osd.5) 27 : cluster 3 failed to encode map e108 with expected crc 2023-09-23T19:27:05.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:04 smithi016 bash[77995]: cluster 2023-09-23T19:27:03.950191+0000 osd.5 (osd.5) 28 : cluster 3 failed to encode map e108 with expected crc 2023-09-23T19:27:05.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:04 smithi016 bash[77995]: audit 2023-09-23T19:27:04.432810+0000 mon.c (mon.2) 113 : audit 0 from='client.? 172.21.15.16:0/2440127014' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:05.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:04 smithi016 bash[75004]: cluster 2023-09-23T19:27:03.930473+0000 mon.a (mon.0) 748 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T19:27:05.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:04 smithi016 bash[75004]: cluster 2023-09-23T19:27:03.942168+0000 osd.3 (osd.3) 38 : cluster 3 failed to encode map e108 with expected crc 2023-09-23T19:27:05.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:04 smithi016 bash[75004]: cluster 2023-09-23T19:27:03.944571+0000 mon.a (mon.0) 749 : cluster 0 osdmap e108: 8 total, 7 up, 8 in 2023-09-23T19:27:05.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:04 smithi016 bash[75004]: cluster 2023-09-23T19:27:03.946171+0000 osd.6 (osd.6) 25 : cluster 3 failed to encode map e108 with expected crc 2023-09-23T19:27:05.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:04 smithi016 bash[75004]: cluster 2023-09-23T19:27:03.947857+0000 osd.4 (osd.4) 25 : cluster 3 failed to encode map e108 with expected crc 2023-09-23T19:27:05.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:04 smithi016 bash[75004]: cluster 2023-09-23T19:27:03.948934+0000 osd.5 (osd.5) 27 : cluster 3 failed to encode map e108 with expected crc 2023-09-23T19:27:05.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:04 smithi016 bash[75004]: cluster 2023-09-23T19:27:03.950191+0000 osd.5 (osd.5) 28 : cluster 3 failed to encode map e108 with expected crc 2023-09-23T19:27:05.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:04 smithi016 bash[75004]: audit 2023-09-23T19:27:04.432810+0000 mon.c (mon.2) 113 : audit 0 from='client.? 172.21.15.16:0/2440127014' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:05.037 INFO:journalctl@ceph.osd.2.smithi016.stdout:Sep 23 19:27:04 smithi016 bash[97319]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd-2 2023-09-23T19:27:05.037 INFO:journalctl@ceph.osd.2.smithi016.stdout:Sep 23 19:27:05 smithi016 bash[97421]: Error response from daemon: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd-2 2023-09-23T19:27:05.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:04 smithi092 bash[58855]: cluster 2023-09-23T19:27:03.930473+0000 mon.a (mon.0) 748 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T19:27:05.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:04 smithi092 bash[58855]: cluster 2023-09-23T19:27:03.942168+0000 osd.3 (osd.3) 38 : cluster 3 failed to encode map e108 with expected crc 2023-09-23T19:27:05.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:04 smithi092 bash[58855]: cluster 2023-09-23T19:27:03.944571+0000 mon.a (mon.0) 749 : cluster 0 osdmap e108: 8 total, 7 up, 8 in 2023-09-23T19:27:05.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:04 smithi092 bash[58855]: cluster 2023-09-23T19:27:03.946171+0000 osd.6 (osd.6) 25 : cluster 3 failed to encode map e108 with expected crc 2023-09-23T19:27:05.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:04 smithi092 bash[58855]: cluster 2023-09-23T19:27:03.947857+0000 osd.4 (osd.4) 25 : cluster 3 failed to encode map e108 with expected crc 2023-09-23T19:27:05.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:04 smithi092 bash[58855]: cluster 2023-09-23T19:27:03.948934+0000 osd.5 (osd.5) 27 : cluster 3 failed to encode map e108 with expected crc 2023-09-23T19:27:05.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:04 smithi092 bash[58855]: cluster 2023-09-23T19:27:03.950191+0000 osd.5 (osd.5) 28 : cluster 3 failed to encode map e108 with expected crc 2023-09-23T19:27:05.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:04 smithi092 bash[58855]: audit 2023-09-23T19:27:04.432810+0000 mon.c (mon.2) 113 : audit 0 from='client.? 172.21.15.16:0/2440127014' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:05.959 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:27:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:27:05] "GET /metrics HTTP/1.1" 200 33816 "" "Prometheus/2.33.4" 2023-09-23T19:27:06.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:05 smithi016 bash[77995]: cluster 2023-09-23T19:27:04.944630+0000 osd.6 (osd.6) 26 : cluster 3 failed to encode map e109 with expected crc 2023-09-23T19:27:06.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:05 smithi016 bash[77995]: cluster 2023-09-23T19:27:04.946119+0000 mon.a (mon.0) 750 : cluster 0 osdmap e109: 8 total, 7 up, 8 in 2023-09-23T19:27:06.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:05 smithi016 bash[77995]: cluster 2023-09-23T19:27:04.947252+0000 osd.3 (osd.3) 39 : cluster 3 failed to encode map e109 with expected crc 2023-09-23T19:27:06.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:05 smithi016 bash[77995]: cluster 2023-09-23T19:27:04.947269+0000 osd.4 (osd.4) 26 : cluster 3 failed to encode map e109 with expected crc 2023-09-23T19:27:06.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:05 smithi016 bash[77995]: cluster 2023-09-23T19:27:04.950914+0000 osd.7 (osd.7) 24 : cluster 3 failed to encode map e109 with expected crc 2023-09-23T19:27:06.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:05 smithi016 bash[77995]: cluster 2023-09-23T19:27:04.950925+0000 osd.3 (osd.3) 40 : cluster 3 failed to encode map e109 with expected crc 2023-09-23T19:27:06.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:05 smithi016 bash[77995]: cluster 2023-09-23T19:27:04.951274+0000 osd.5 (osd.5) 29 : cluster 3 failed to encode map e109 with expected crc 2023-09-23T19:27:06.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:05 smithi016 bash[77995]: cluster 2023-09-23T19:27:04.952007+0000 osd.4 (osd.4) 27 : cluster 3 failed to encode map e109 with expected crc 2023-09-23T19:27:06.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:05 smithi016 bash[77995]: cluster 2023-09-23T19:27:04.952114+0000 osd.6 (osd.6) 27 : cluster 3 failed to encode map e109 with expected crc 2023-09-23T19:27:06.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:05 smithi016 bash[77995]: cluster 2023-09-23T19:27:04.952381+0000 osd.3 (osd.3) 41 : cluster 3 failed to encode map e109 with expected crc 2023-09-23T19:27:06.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:05 smithi016 bash[77995]: cluster 2023-09-23T19:27:04.953615+0000 osd.5 (osd.5) 30 : cluster 3 failed to encode map e109 with expected crc 2023-09-23T19:27:06.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:05 smithi016 bash[77995]: cluster 2023-09-23T19:27:05.830876+0000 mgr.y (mgr.25325) 567 : cluster 0 pgmap v468: 73 pgs: 1 activating+undersized, 6 peering, 2 stale+active+clean, 2 active+undersized, 62 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:27:06.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:05 smithi016 bash[75004]: cluster 2023-09-23T19:27:04.944630+0000 osd.6 (osd.6) 26 : cluster 3 failed to encode map e109 with expected crc 2023-09-23T19:27:06.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:05 smithi016 bash[75004]: cluster 2023-09-23T19:27:04.946119+0000 mon.a (mon.0) 750 : cluster 0 osdmap e109: 8 total, 7 up, 8 in 2023-09-23T19:27:06.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:05 smithi016 bash[75004]: cluster 2023-09-23T19:27:04.947252+0000 osd.3 (osd.3) 39 : cluster 3 failed to encode map e109 with expected crc 2023-09-23T19:27:06.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:05 smithi016 bash[75004]: cluster 2023-09-23T19:27:04.947269+0000 osd.4 (osd.4) 26 : cluster 3 failed to encode map e109 with expected crc 2023-09-23T19:27:06.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:05 smithi016 bash[75004]: cluster 2023-09-23T19:27:04.950914+0000 osd.7 (osd.7) 24 : cluster 3 failed to encode map e109 with expected crc 2023-09-23T19:27:06.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:05 smithi016 bash[75004]: cluster 2023-09-23T19:27:04.950925+0000 osd.3 (osd.3) 40 : cluster 3 failed to encode map e109 with expected crc 2023-09-23T19:27:06.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:05 smithi016 bash[75004]: cluster 2023-09-23T19:27:04.951274+0000 osd.5 (osd.5) 29 : cluster 3 failed to encode map e109 with expected crc 2023-09-23T19:27:06.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:05 smithi016 bash[75004]: cluster 2023-09-23T19:27:04.952007+0000 osd.4 (osd.4) 27 : cluster 3 failed to encode map e109 with expected crc 2023-09-23T19:27:06.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:05 smithi016 bash[75004]: cluster 2023-09-23T19:27:04.952114+0000 osd.6 (osd.6) 27 : cluster 3 failed to encode map e109 with expected crc 2023-09-23T19:27:06.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:05 smithi016 bash[75004]: cluster 2023-09-23T19:27:04.952381+0000 osd.3 (osd.3) 41 : cluster 3 failed to encode map e109 with expected crc 2023-09-23T19:27:06.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:05 smithi016 bash[75004]: cluster 2023-09-23T19:27:04.953615+0000 osd.5 (osd.5) 30 : cluster 3 failed to encode map e109 with expected crc 2023-09-23T19:27:06.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:05 smithi016 bash[75004]: cluster 2023-09-23T19:27:05.830876+0000 mgr.y (mgr.25325) 567 : cluster 0 pgmap v468: 73 pgs: 1 activating+undersized, 6 peering, 2 stale+active+clean, 2 active+undersized, 62 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:27:06.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:05 smithi092 bash[58855]: cluster 2023-09-23T19:27:04.944630+0000 osd.6 (osd.6) 26 : cluster 3 failed to encode map e109 with expected crc 2023-09-23T19:27:06.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:05 smithi092 bash[58855]: cluster 2023-09-23T19:27:04.946119+0000 mon.a (mon.0) 750 : cluster 0 osdmap e109: 8 total, 7 up, 8 in 2023-09-23T19:27:06.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:05 smithi092 bash[58855]: cluster 2023-09-23T19:27:04.947252+0000 osd.3 (osd.3) 39 : cluster 3 failed to encode map e109 with expected crc 2023-09-23T19:27:06.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:05 smithi092 bash[58855]: cluster 2023-09-23T19:27:04.947269+0000 osd.4 (osd.4) 26 : cluster 3 failed to encode map e109 with expected crc 2023-09-23T19:27:06.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:05 smithi092 bash[58855]: cluster 2023-09-23T19:27:04.950914+0000 osd.7 (osd.7) 24 : cluster 3 failed to encode map e109 with expected crc 2023-09-23T19:27:06.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:05 smithi092 bash[58855]: cluster 2023-09-23T19:27:04.950925+0000 osd.3 (osd.3) 40 : cluster 3 failed to encode map e109 with expected crc 2023-09-23T19:27:06.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:05 smithi092 bash[58855]: cluster 2023-09-23T19:27:04.951274+0000 osd.5 (osd.5) 29 : cluster 3 failed to encode map e109 with expected crc 2023-09-23T19:27:06.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:05 smithi092 bash[58855]: cluster 2023-09-23T19:27:04.952007+0000 osd.4 (osd.4) 27 : cluster 3 failed to encode map e109 with expected crc 2023-09-23T19:27:06.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:05 smithi092 bash[58855]: cluster 2023-09-23T19:27:04.952114+0000 osd.6 (osd.6) 27 : cluster 3 failed to encode map e109 with expected crc 2023-09-23T19:27:06.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:05 smithi092 bash[58855]: cluster 2023-09-23T19:27:04.952381+0000 osd.3 (osd.3) 41 : cluster 3 failed to encode map e109 with expected crc 2023-09-23T19:27:06.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:05 smithi092 bash[58855]: cluster 2023-09-23T19:27:04.953615+0000 osd.5 (osd.5) 30 : cluster 3 failed to encode map e109 with expected crc 2023-09-23T19:27:06.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:05 smithi092 bash[58855]: cluster 2023-09-23T19:27:05.830876+0000 mgr.y (mgr.25325) 567 : cluster 0 pgmap v468: 73 pgs: 1 activating+undersized, 6 peering, 2 stale+active+clean, 2 active+undersized, 62 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:27:07.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:06 smithi016 bash[75004]: audit 2023-09-23T19:27:06.804521+0000 mon.a (mon.0) 751 : audit 0 from='client.? 172.21.15.16:0/2282789637' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:07.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:06 smithi016 bash[77995]: audit 2023-09-23T19:27:06.804521+0000 mon.a (mon.0) 751 : audit 0 from='client.? 172.21.15.16:0/2282789637' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:07.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:06 smithi092 bash[58855]: audit 2023-09-23T19:27:06.804521+0000 mon.a (mon.0) 751 : audit 0 from='client.? 172.21.15.16:0/2282789637' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:07.646 INFO:journalctl@ceph.osd.2.smithi016.stdout:Sep 23 19:27:07 smithi016 systemd[1]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@osd.2.service: Succeeded. 2023-09-23T19:27:07.646 INFO:journalctl@ceph.osd.2.smithi016.stdout:Sep 23 19:27:07 smithi016 systemd[1]: Stopped Ceph osd.2 for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7. 2023-09-23T19:27:08.034 INFO:journalctl@ceph.osd.2.smithi016.stdout:Sep 23 19:27:07 smithi016 systemd[1]: Started Ceph osd.2 for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7. 2023-09-23T19:27:09.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:08 smithi016 bash[75004]: audit 2023-09-23T19:27:07.684040+0000 mon.a (mon.0) 752 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:09.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:08 smithi016 bash[75004]: audit 2023-09-23T19:27:07.690978+0000 mon.a (mon.0) 753 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:09.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:08 smithi016 bash[75004]: audit 2023-09-23T19:27:07.708084+0000 mon.a (mon.0) 754 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:27:09.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:08 smithi016 bash[75004]: cluster 2023-09-23T19:27:07.831483+0000 mgr.y (mgr.25325) 568 : cluster 0 pgmap v469: 73 pgs: 1 activating+undersized, 6 peering, 10 active+undersized, 56 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:27:09.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:08 smithi016 bash[77995]: audit 2023-09-23T19:27:07.684040+0000 mon.a (mon.0) 752 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:09.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:08 smithi016 bash[77995]: audit 2023-09-23T19:27:07.690978+0000 mon.a (mon.0) 753 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:09.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:08 smithi016 bash[77995]: audit 2023-09-23T19:27:07.708084+0000 mon.a (mon.0) 754 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:27:09.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:08 smithi016 bash[77995]: cluster 2023-09-23T19:27:07.831483+0000 mgr.y (mgr.25325) 568 : cluster 0 pgmap v469: 73 pgs: 1 activating+undersized, 6 peering, 10 active+undersized, 56 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:27:09.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:08 smithi092 bash[58855]: audit 2023-09-23T19:27:07.684040+0000 mon.a (mon.0) 752 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:09.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:08 smithi092 bash[58855]: audit 2023-09-23T19:27:07.690978+0000 mon.a (mon.0) 753 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:09.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:08 smithi092 bash[58855]: audit 2023-09-23T19:27:07.708084+0000 mon.a (mon.0) 754 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:27:09.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:08 smithi092 bash[58855]: cluster 2023-09-23T19:27:07.831483+0000 mgr.y (mgr.25325) 568 : cluster 0 pgmap v469: 73 pgs: 1 activating+undersized, 6 peering, 10 active+undersized, 56 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:27:09.530 INFO:journalctl@ceph.osd.2.smithi016.stdout:Sep 23 19:27:09 smithi016 bash[97603]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T19:27:09.530 INFO:journalctl@ceph.osd.2.smithi016.stdout:Sep 23 19:27:09 smithi016 bash[97603]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-09-23T19:27:09.530 INFO:journalctl@ceph.osd.2.smithi016.stdout:Sep 23 19:27:09 smithi016 bash[97603]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2023-09-23T19:27:09.530 INFO:journalctl@ceph.osd.2.smithi016.stdout:Sep 23 19:27:09 smithi016 bash[97603]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2023-09-23T19:27:09.530 INFO:journalctl@ceph.osd.2.smithi016.stdout:Sep 23 19:27:09 smithi016 bash[97603]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2023-09-23T19:27:09.530 INFO:journalctl@ceph.osd.2.smithi016.stdout:Sep 23 19:27:09 smithi016 bash[97603]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2023-09-23T19:27:09.530 INFO:journalctl@ceph.osd.2.smithi016.stdout:Sep 23 19:27:09 smithi016 bash[97603]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-09-23T19:27:09.531 INFO:journalctl@ceph.osd.2.smithi016.stdout:Sep 23 19:27:09 smithi016 bash[97603]: --> ceph-volume lvm activate successful for osd ID: 2 2023-09-23T19:27:09.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:09 smithi016 bash[77995]: audit 2023-09-23T19:27:09.166666+0000 mon.a (mon.0) 755 : audit 0 from='client.? 172.21.15.16:0/2784953575' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:09.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:09 smithi016 bash[75004]: audit 2023-09-23T19:27:09.166666+0000 mon.a (mon.0) 755 : audit 0 from='client.? 172.21.15.16:0/2784953575' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:10.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:09 smithi092 bash[58855]: audit 2023-09-23T19:27:09.166666+0000 mon.a (mon.0) 755 : audit 0 from='client.? 172.21.15.16:0/2784953575' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:11.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:10 smithi016 bash[75004]: cluster 2023-09-23T19:27:09.832255+0000 mgr.y (mgr.25325) 569 : cluster 0 pgmap v470: 73 pgs: 1 activating+undersized, 1 active+undersized+degraded, 6 peering, 11 active+undersized, 54 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-09-23T19:27:11.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:10 smithi016 bash[77995]: cluster 2023-09-23T19:27:09.832255+0000 mgr.y (mgr.25325) 569 : cluster 0 pgmap v470: 73 pgs: 1 activating+undersized, 1 active+undersized+degraded, 6 peering, 11 active+undersized, 54 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-09-23T19:27:11.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:10 smithi092 bash[58855]: cluster 2023-09-23T19:27:09.832255+0000 mgr.y (mgr.25325) 569 : cluster 0 pgmap v470: 73 pgs: 1 activating+undersized, 1 active+undersized+degraded, 6 peering, 11 active+undersized, 54 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-09-23T19:27:11.534 INFO:journalctl@ceph.osd.2.smithi016.stdout:Sep 23 19:27:11 smithi016 bash[97918]: debug 2023-09-23T19:27:11.192+0000 7f6c0a76b700 1 -- 172.21.15.16:0/4028170635 <== mon.2 v2:172.21.15.16:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5577f4d1e4e0 con 0x5577f4d79000 2023-09-23T19:27:12.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:11 smithi016 bash[75004]: cluster 2023-09-23T19:27:10.691470+0000 mon.a (mon.0) 756 : cluster 3 Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-09-23T19:27:12.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:11 smithi016 bash[75004]: audit 2023-09-23T19:27:11.535563+0000 mon.a (mon.0) 757 : audit 0 from='client.? 172.21.15.16:0/719215060' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:12.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:11 smithi016 bash[77995]: cluster 2023-09-23T19:27:10.691470+0000 mon.a (mon.0) 756 : cluster 3 Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-09-23T19:27:12.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:11 smithi016 bash[77995]: audit 2023-09-23T19:27:11.535563+0000 mon.a (mon.0) 757 : audit 0 from='client.? 172.21.15.16:0/719215060' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:12.035 INFO:journalctl@ceph.osd.2.smithi016.stdout:Sep 23 19:27:11 smithi016 bash[97918]: debug 2023-09-23T19:27:11.744+0000 7f6c138f0640 -1 Falling back to public interface 2023-09-23T19:27:12.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:11 smithi092 bash[58855]: cluster 2023-09-23T19:27:10.691470+0000 mon.a (mon.0) 756 : cluster 3 Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-09-23T19:27:12.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:11 smithi092 bash[58855]: audit 2023-09-23T19:27:11.535563+0000 mon.a (mon.0) 757 : audit 0 from='client.? 172.21.15.16:0/719215060' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:13.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:12 smithi016 bash[75004]: cluster 2023-09-23T19:27:11.833403+0000 mgr.y (mgr.25325) 570 : cluster 0 pgmap v471: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T19:27:13.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:12 smithi016 bash[77995]: cluster 2023-09-23T19:27:11.833403+0000 mgr.y (mgr.25325) 570 : cluster 0 pgmap v471: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T19:27:13.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:12 smithi092 bash[58855]: cluster 2023-09-23T19:27:11.833403+0000 mgr.y (mgr.25325) 570 : cluster 0 pgmap v471: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T19:27:15.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:14 smithi016 bash[77995]: audit 2023-09-23T19:27:13.636184+0000 mon.a (mon.0) 758 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:15.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:14 smithi016 bash[77995]: audit 2023-09-23T19:27:13.637802+0000 mon.a (mon.0) 759 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:27:15.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:14 smithi016 bash[77995]: cluster 2023-09-23T19:27:13.834211+0000 mgr.y (mgr.25325) 571 : cluster 0 pgmap v472: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T19:27:15.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:14 smithi016 bash[77995]: audit 2023-09-23T19:27:13.929764+0000 mon.a (mon.0) 760 : audit 0 from='client.? 172.21.15.16:0/4150492357' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:15.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:14 smithi016 bash[75004]: audit 2023-09-23T19:27:13.636184+0000 mon.a (mon.0) 758 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:15.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:14 smithi016 bash[75004]: audit 2023-09-23T19:27:13.637802+0000 mon.a (mon.0) 759 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:27:15.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:14 smithi016 bash[75004]: cluster 2023-09-23T19:27:13.834211+0000 mgr.y (mgr.25325) 571 : cluster 0 pgmap v472: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T19:27:15.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:14 smithi016 bash[75004]: audit 2023-09-23T19:27:13.929764+0000 mon.a (mon.0) 760 : audit 0 from='client.? 172.21.15.16:0/4150492357' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:15.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:14 smithi092 bash[58855]: audit 2023-09-23T19:27:13.636184+0000 mon.a (mon.0) 758 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:15.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:14 smithi092 bash[58855]: audit 2023-09-23T19:27:13.637802+0000 mon.a (mon.0) 759 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:27:15.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:14 smithi092 bash[58855]: cluster 2023-09-23T19:27:13.834211+0000 mgr.y (mgr.25325) 571 : cluster 0 pgmap v472: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T19:27:15.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:14 smithi092 bash[58855]: audit 2023-09-23T19:27:13.929764+0000 mon.a (mon.0) 760 : audit 0 from='client.? 172.21.15.16:0/4150492357' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:15.907 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:27:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:27:15] "GET /metrics HTTP/1.1" 200 33821 "" "Prometheus/2.33.4" 2023-09-23T19:27:16.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:15 smithi016 bash[75004]: audit 2023-09-23T19:27:14.908201+0000 mon.a (mon.0) 761 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:16.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:15 smithi016 bash[75004]: audit 2023-09-23T19:27:14.915476+0000 mon.a (mon.0) 762 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:16.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:15 smithi016 bash[75004]: audit 2023-09-23T19:27:15.760777+0000 mon.a (mon.0) 763 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:16.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:15 smithi016 bash[75004]: audit 2023-09-23T19:27:15.768854+0000 mon.a (mon.0) 764 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:16.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:15 smithi016 bash[75004]: cluster 2023-09-23T19:27:15.835264+0000 mgr.y (mgr.25325) 572 : cluster 0 pgmap v473: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T19:27:16.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:15 smithi016 bash[77995]: audit 2023-09-23T19:27:14.908201+0000 mon.a (mon.0) 761 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:16.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:15 smithi016 bash[77995]: audit 2023-09-23T19:27:14.915476+0000 mon.a (mon.0) 762 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:16.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:15 smithi016 bash[77995]: audit 2023-09-23T19:27:15.760777+0000 mon.a (mon.0) 763 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:16.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:15 smithi016 bash[77995]: audit 2023-09-23T19:27:15.768854+0000 mon.a (mon.0) 764 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:16.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:15 smithi016 bash[77995]: cluster 2023-09-23T19:27:15.835264+0000 mgr.y (mgr.25325) 572 : cluster 0 pgmap v473: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T19:27:16.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:15 smithi092 bash[58855]: audit 2023-09-23T19:27:14.908201+0000 mon.a (mon.0) 761 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:16.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:15 smithi092 bash[58855]: audit 2023-09-23T19:27:14.915476+0000 mon.a (mon.0) 762 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:16.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:15 smithi092 bash[58855]: audit 2023-09-23T19:27:15.760777+0000 mon.a (mon.0) 763 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:16.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:15 smithi092 bash[58855]: audit 2023-09-23T19:27:15.768854+0000 mon.a (mon.0) 764 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:16.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:15 smithi092 bash[58855]: cluster 2023-09-23T19:27:15.835264+0000 mgr.y (mgr.25325) 572 : cluster 0 pgmap v473: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T19:27:17.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:16 smithi016 bash[75004]: audit 2023-09-23T19:27:16.294870+0000 mon.c (mon.2) 114 : audit 0 from='client.? 172.21.15.16:0/3434523925' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:17.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:16 smithi016 bash[77995]: audit 2023-09-23T19:27:16.294870+0000 mon.c (mon.2) 114 : audit 0 from='client.? 172.21.15.16:0/3434523925' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:17.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:16 smithi092 bash[58855]: audit 2023-09-23T19:27:16.294870+0000 mon.c (mon.2) 114 : audit 0 from='client.? 172.21.15.16:0/3434523925' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:18.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:17 smithi016 bash[75004]: cluster 2023-09-23T19:27:17.836108+0000 mgr.y (mgr.25325) 573 : cluster 0 pgmap v474: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T19:27:18.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:17 smithi016 bash[77995]: cluster 2023-09-23T19:27:17.836108+0000 mgr.y (mgr.25325) 573 : cluster 0 pgmap v474: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T19:27:18.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:17 smithi092 bash[58855]: cluster 2023-09-23T19:27:17.836108+0000 mgr.y (mgr.25325) 573 : cluster 0 pgmap v474: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T19:27:19.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:18 smithi016 bash[75004]: audit 2023-09-23T19:27:18.646590+0000 mon.c (mon.2) 115 : audit 0 from='client.? 172.21.15.16:0/3221494416' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:19.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:18 smithi016 bash[75004]: audit 2023-09-23T19:27:18.825194+0000 mon.a (mon.0) 765 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:19.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:18 smithi016 bash[77995]: audit 2023-09-23T19:27:18.646590+0000 mon.c (mon.2) 115 : audit 0 from='client.? 172.21.15.16:0/3221494416' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:19.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:18 smithi016 bash[77995]: audit 2023-09-23T19:27:18.825194+0000 mon.a (mon.0) 765 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:19.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:18 smithi092 bash[58855]: audit 2023-09-23T19:27:18.646590+0000 mon.c (mon.2) 115 : audit 0 from='client.? 172.21.15.16:0/3221494416' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:19.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:18 smithi092 bash[58855]: audit 2023-09-23T19:27:18.825194+0000 mon.a (mon.0) 765 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:20.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:19 smithi016 bash[77995]: cluster 2023-09-23T19:27:19.201325+0000 mon.a (mon.0) 766 : cluster 3 Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T19:27:20.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:19 smithi016 bash[77995]: cluster 2023-09-23T19:27:19.836816+0000 mgr.y (mgr.25325) 574 : cluster 0 pgmap v475: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T19:27:20.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:19 smithi016 bash[75004]: cluster 2023-09-23T19:27:19.201325+0000 mon.a (mon.0) 766 : cluster 3 Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T19:27:20.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:19 smithi016 bash[75004]: cluster 2023-09-23T19:27:19.836816+0000 mgr.y (mgr.25325) 574 : cluster 0 pgmap v475: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T19:27:20.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:19 smithi092 bash[58855]: cluster 2023-09-23T19:27:19.201325+0000 mon.a (mon.0) 766 : cluster 3 Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T19:27:20.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:19 smithi092 bash[58855]: cluster 2023-09-23T19:27:19.836816+0000 mgr.y (mgr.25325) 574 : cluster 0 pgmap v475: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T19:27:21.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:21 smithi016 bash[75004]: audit 2023-09-23T19:27:20.998823+0000 mon.a (mon.0) 767 : audit 0 from='client.? 172.21.15.16:0/2771023048' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:21.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:21 smithi016 bash[77995]: audit 2023-09-23T19:27:20.998823+0000 mon.a (mon.0) 767 : audit 0 from='client.? 172.21.15.16:0/2771023048' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:21.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:21 smithi092 bash[58855]: audit 2023-09-23T19:27:20.998823+0000 mon.a (mon.0) 767 : audit 0 from='client.? 172.21.15.16:0/2771023048' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:22.030 INFO:journalctl@ceph.osd.2.smithi016.stdout:Sep 23 19:27:21 smithi016 bash[97918]: debug 2023-09-23T19:27:21.548+0000 7f6c138f0640 -1 osd.2 107 log_to_monitors true 2023-09-23T19:27:22.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:22 smithi016 bash[77995]: audit 2023-09-23T19:27:21.557557+0000 mon.c (mon.2) 116 : audit 1 from='osd.2 [v2:172.21.15.16:6818/486425090,v1:172.21.15.16:6819/486425090]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T19:27:22.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:22 smithi016 bash[77995]: audit 2023-09-23T19:27:21.558177+0000 mon.a (mon.0) 768 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T19:27:22.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:22 smithi016 bash[77995]: cluster 2023-09-23T19:27:21.837856+0000 mgr.y (mgr.25325) 575 : cluster 0 pgmap v476: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T19:27:22.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:22 smithi016 bash[75004]: audit 2023-09-23T19:27:21.557557+0000 mon.c (mon.2) 116 : audit 1 from='osd.2 [v2:172.21.15.16:6818/486425090,v1:172.21.15.16:6819/486425090]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T19:27:22.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:22 smithi016 bash[75004]: audit 2023-09-23T19:27:21.558177+0000 mon.a (mon.0) 768 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T19:27:22.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:22 smithi016 bash[75004]: cluster 2023-09-23T19:27:21.837856+0000 mgr.y (mgr.25325) 575 : cluster 0 pgmap v476: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T19:27:22.285 INFO:journalctl@ceph.osd.2.smithi016.stdout:Sep 23 19:27:22 smithi016 bash[97918]: debug 2023-09-23T19:27:22.256+0000 7f6c04d5d700 -1 osd.2 107 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T19:27:22.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:22 smithi092 bash[58855]: audit 2023-09-23T19:27:21.557557+0000 mon.c (mon.2) 116 : audit 1 from='osd.2 [v2:172.21.15.16:6818/486425090,v1:172.21.15.16:6819/486425090]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T19:27:22.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:22 smithi092 bash[58855]: audit 2023-09-23T19:27:21.558177+0000 mon.a (mon.0) 768 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T19:27:22.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:22 smithi092 bash[58855]: cluster 2023-09-23T19:27:21.837856+0000 mgr.y (mgr.25325) 575 : cluster 0 pgmap v476: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T19:27:23.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:23 smithi016 bash[75004]: cluster 2023-09-23T19:27:22.218819+0000 osd.3 (osd.3) 42 : cluster 3 failed to encode map e110 with expected crc 2023-09-23T19:27:23.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:23 smithi016 bash[75004]: audit 2023-09-23T19:27:22.222481+0000 mon.a (mon.0) 769 : audit 1 from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T19:27:23.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:23 smithi016 bash[75004]: cluster 2023-09-23T19:27:22.222625+0000 mon.a (mon.0) 770 : cluster 0 osdmap e110: 8 total, 7 up, 8 in 2023-09-23T19:27:23.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:23 smithi016 bash[75004]: audit 2023-09-23T19:27:22.223498+0000 mon.c (mon.2) 117 : audit 1 from='osd.2 [v2:172.21.15.16:6818/486425090,v1:172.21.15.16:6819/486425090]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-09-23T19:27:23.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:23 smithi016 bash[75004]: audit 2023-09-23T19:27:22.224175+0000 mon.a (mon.0) 771 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-09-23T19:27:23.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:23 smithi016 bash[75004]: cluster 2023-09-23T19:27:22.588804+0000 osd.4 (osd.4) 28 : cluster 3 failed to encode map e110 with expected crc 2023-09-23T19:27:23.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:23 smithi016 bash[75004]: cluster 2023-09-23T19:27:22.599616+0000 osd.6 (osd.6) 28 : cluster 3 failed to encode map e110 with expected crc 2023-09-23T19:27:23.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:23 smithi016 bash[75004]: cluster 2023-09-23T19:27:22.599625+0000 osd.5 (osd.5) 31 : cluster 3 failed to encode map e110 with expected crc 2023-09-23T19:27:23.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:23 smithi016 bash[77995]: cluster 2023-09-23T19:27:22.218819+0000 osd.3 (osd.3) 42 : cluster 3 failed to encode map e110 with expected crc 2023-09-23T19:27:23.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:23 smithi016 bash[77995]: audit 2023-09-23T19:27:22.222481+0000 mon.a (mon.0) 769 : audit 1 from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T19:27:23.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:23 smithi016 bash[77995]: cluster 2023-09-23T19:27:22.222625+0000 mon.a (mon.0) 770 : cluster 0 osdmap e110: 8 total, 7 up, 8 in 2023-09-23T19:27:23.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:23 smithi016 bash[77995]: audit 2023-09-23T19:27:22.223498+0000 mon.c (mon.2) 117 : audit 1 from='osd.2 [v2:172.21.15.16:6818/486425090,v1:172.21.15.16:6819/486425090]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-09-23T19:27:23.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:23 smithi016 bash[77995]: audit 2023-09-23T19:27:22.224175+0000 mon.a (mon.0) 771 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-09-23T19:27:23.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:23 smithi016 bash[77995]: cluster 2023-09-23T19:27:22.588804+0000 osd.4 (osd.4) 28 : cluster 3 failed to encode map e110 with expected crc 2023-09-23T19:27:23.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:23 smithi016 bash[77995]: cluster 2023-09-23T19:27:22.599616+0000 osd.6 (osd.6) 28 : cluster 3 failed to encode map e110 with expected crc 2023-09-23T19:27:23.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:23 smithi016 bash[77995]: cluster 2023-09-23T19:27:22.599625+0000 osd.5 (osd.5) 31 : cluster 3 failed to encode map e110 with expected crc 2023-09-23T19:27:23.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:23 smithi092 bash[58855]: cluster 2023-09-23T19:27:22.218819+0000 osd.3 (osd.3) 42 : cluster 3 failed to encode map e110 with expected crc 2023-09-23T19:27:23.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:23 smithi092 bash[58855]: audit 2023-09-23T19:27:22.222481+0000 mon.a (mon.0) 769 : audit 1 from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T19:27:23.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:23 smithi092 bash[58855]: cluster 2023-09-23T19:27:22.222625+0000 mon.a (mon.0) 770 : cluster 0 osdmap e110: 8 total, 7 up, 8 in 2023-09-23T19:27:23.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:23 smithi092 bash[58855]: audit 2023-09-23T19:27:22.223498+0000 mon.c (mon.2) 117 : audit 1 from='osd.2 [v2:172.21.15.16:6818/486425090,v1:172.21.15.16:6819/486425090]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-09-23T19:27:23.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:23 smithi092 bash[58855]: audit 2023-09-23T19:27:22.224175+0000 mon.a (mon.0) 771 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-09-23T19:27:23.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:23 smithi092 bash[58855]: cluster 2023-09-23T19:27:22.588804+0000 osd.4 (osd.4) 28 : cluster 3 failed to encode map e110 with expected crc 2023-09-23T19:27:23.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:23 smithi092 bash[58855]: cluster 2023-09-23T19:27:22.599616+0000 osd.6 (osd.6) 28 : cluster 3 failed to encode map e110 with expected crc 2023-09-23T19:27:23.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:23 smithi092 bash[58855]: cluster 2023-09-23T19:27:22.599625+0000 osd.5 (osd.5) 31 : cluster 3 failed to encode map e110 with expected crc 2023-09-23T19:27:24.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:24 smithi016 bash[75004]: cluster 2023-09-23T19:27:22.355847+0000 osd.7 (osd.7) 25 : cluster 3 failed to encode map e110 with expected crc 2023-09-23T19:27:24.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:24 smithi016 bash[75004]: cluster 2023-09-23T19:27:23.218537+0000 mon.a (mon.0) 772 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T19:27:24.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:24 smithi016 bash[75004]: cluster 2023-09-23T19:27:23.225302+0000 osd.5 (osd.5) 32 : cluster 3 failed to encode map e111 with expected crc 2023-09-23T19:27:24.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:24 smithi016 bash[75004]: cluster 2023-09-23T19:27:23.228021+0000 mon.a (mon.0) 773 : cluster 1 osd.2 [v2:172.21.15.16:6818/486425090,v1:172.21.15.16:6819/486425090] boot 2023-09-23T19:27:24.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:24 smithi016 bash[75004]: cluster 2023-09-23T19:27:23.228177+0000 mon.a (mon.0) 774 : cluster 0 osdmap e111: 8 total, 8 up, 8 in 2023-09-23T19:27:24.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:24 smithi016 bash[75004]: audit 2023-09-23T19:27:23.229031+0000 mon.a (mon.0) 775 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T19:27:24.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:24 smithi016 bash[75004]: audit 2023-09-23T19:27:23.377496+0000 mon.a (mon.0) 776 : audit 0 from='client.? 172.21.15.16:0/820922856' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:24.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:24 smithi016 bash[75004]: cluster 2023-09-23T19:27:23.838643+0000 mgr.y (mgr.25325) 576 : cluster 0 pgmap v479: 73 pgs: 2 peering, 1 active+undersized+degraded, 16 active+undersized, 54 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-09-23T19:27:24.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:24 smithi016 bash[77995]: cluster 2023-09-23T19:27:22.355847+0000 osd.7 (osd.7) 25 : cluster 3 failed to encode map e110 with expected crc 2023-09-23T19:27:24.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:24 smithi016 bash[77995]: cluster 2023-09-23T19:27:23.218537+0000 mon.a (mon.0) 772 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T19:27:24.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:24 smithi016 bash[77995]: cluster 2023-09-23T19:27:23.225302+0000 osd.5 (osd.5) 32 : cluster 3 failed to encode map e111 with expected crc 2023-09-23T19:27:24.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:24 smithi016 bash[77995]: cluster 2023-09-23T19:27:23.228021+0000 mon.a (mon.0) 773 : cluster 1 osd.2 [v2:172.21.15.16:6818/486425090,v1:172.21.15.16:6819/486425090] boot 2023-09-23T19:27:24.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:24 smithi016 bash[77995]: cluster 2023-09-23T19:27:23.228177+0000 mon.a (mon.0) 774 : cluster 0 osdmap e111: 8 total, 8 up, 8 in 2023-09-23T19:27:24.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:24 smithi016 bash[77995]: audit 2023-09-23T19:27:23.229031+0000 mon.a (mon.0) 775 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T19:27:24.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:24 smithi016 bash[77995]: audit 2023-09-23T19:27:23.377496+0000 mon.a (mon.0) 776 : audit 0 from='client.? 172.21.15.16:0/820922856' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:24.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:24 smithi016 bash[77995]: cluster 2023-09-23T19:27:23.838643+0000 mgr.y (mgr.25325) 576 : cluster 0 pgmap v479: 73 pgs: 2 peering, 1 active+undersized+degraded, 16 active+undersized, 54 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-09-23T19:27:24.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:24 smithi092 bash[58855]: cluster 2023-09-23T19:27:22.355847+0000 osd.7 (osd.7) 25 : cluster 3 failed to encode map e110 with expected crc 2023-09-23T19:27:24.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:24 smithi092 bash[58855]: cluster 2023-09-23T19:27:23.218537+0000 mon.a (mon.0) 772 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T19:27:24.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:24 smithi092 bash[58855]: cluster 2023-09-23T19:27:23.225302+0000 osd.5 (osd.5) 32 : cluster 3 failed to encode map e111 with expected crc 2023-09-23T19:27:24.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:24 smithi092 bash[58855]: cluster 2023-09-23T19:27:23.228021+0000 mon.a (mon.0) 773 : cluster 1 osd.2 [v2:172.21.15.16:6818/486425090,v1:172.21.15.16:6819/486425090] boot 2023-09-23T19:27:24.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:24 smithi092 bash[58855]: cluster 2023-09-23T19:27:23.228177+0000 mon.a (mon.0) 774 : cluster 0 osdmap e111: 8 total, 8 up, 8 in 2023-09-23T19:27:24.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:24 smithi092 bash[58855]: audit 2023-09-23T19:27:23.229031+0000 mon.a (mon.0) 775 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T19:27:24.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:24 smithi092 bash[58855]: audit 2023-09-23T19:27:23.377496+0000 mon.a (mon.0) 776 : audit 0 from='client.? 172.21.15.16:0/820922856' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:24.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:24 smithi092 bash[58855]: cluster 2023-09-23T19:27:23.838643+0000 mgr.y (mgr.25325) 576 : cluster 0 pgmap v479: 73 pgs: 2 peering, 1 active+undersized+degraded, 16 active+undersized, 54 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-09-23T19:27:25.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:25 smithi016 bash[77995]: cluster 2023-09-23T19:27:24.232912+0000 mon.a (mon.0) 777 : cluster 3 Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-09-23T19:27:25.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:25 smithi016 bash[77995]: cluster 2023-09-23T19:27:24.247172+0000 osd.3 (osd.3) 43 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T19:27:25.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:25 smithi016 bash[77995]: cluster 2023-09-23T19:27:24.247479+0000 osd.7 (osd.7) 26 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T19:27:25.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:25 smithi016 bash[77995]: cluster 2023-09-23T19:27:24.250419+0000 osd.4 (osd.4) 29 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T19:27:25.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:25 smithi016 bash[77995]: cluster 2023-09-23T19:27:24.251139+0000 osd.6 (osd.6) 29 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T19:27:25.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:25 smithi016 bash[77995]: cluster 2023-09-23T19:27:24.251177+0000 osd.5 (osd.5) 33 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T19:27:25.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:25 smithi016 bash[77995]: cluster 2023-09-23T19:27:24.253226+0000 osd.4 (osd.4) 30 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T19:27:25.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:25 smithi016 bash[77995]: cluster 2023-09-23T19:27:24.253524+0000 osd.3 (osd.3) 44 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T19:27:25.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:25 smithi016 bash[77995]: cluster 2023-09-23T19:27:24.255733+0000 osd.3 (osd.3) 45 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T19:27:25.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:25 smithi016 bash[77995]: cluster 2023-09-23T19:27:24.257103+0000 osd.5 (osd.5) 34 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T19:27:25.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:25 smithi016 bash[77995]: cluster 2023-09-23T19:27:24.257841+0000 mon.a (mon.0) 778 : cluster 0 osdmap e112: 8 total, 8 up, 8 in 2023-09-23T19:27:25.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:25 smithi016 bash[77995]: cluster 2023-09-23T19:27:24.258368+0000 osd.3 (osd.3) 46 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T19:27:25.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:25 smithi016 bash[77995]: cluster 2023-09-23T19:27:24.261402+0000 osd.5 (osd.5) 35 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T19:27:25.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:25 smithi016 bash[77995]: cluster 2023-09-23T19:27:24.261545+0000 osd.6 (osd.6) 30 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T19:27:25.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:25 smithi016 bash[77995]: cluster 2023-09-23T19:27:24.261568+0000 osd.4 (osd.4) 31 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T19:27:25.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:25 smithi016 bash[75004]: cluster 2023-09-23T19:27:24.232912+0000 mon.a (mon.0) 777 : cluster 3 Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-09-23T19:27:25.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:25 smithi016 bash[75004]: cluster 2023-09-23T19:27:24.247172+0000 osd.3 (osd.3) 43 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T19:27:25.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:25 smithi016 bash[75004]: cluster 2023-09-23T19:27:24.247479+0000 osd.7 (osd.7) 26 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T19:27:25.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:25 smithi016 bash[75004]: cluster 2023-09-23T19:27:24.250419+0000 osd.4 (osd.4) 29 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T19:27:25.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:25 smithi016 bash[75004]: cluster 2023-09-23T19:27:24.251139+0000 osd.6 (osd.6) 29 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T19:27:25.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:25 smithi016 bash[75004]: cluster 2023-09-23T19:27:24.251177+0000 osd.5 (osd.5) 33 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T19:27:25.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:25 smithi016 bash[75004]: cluster 2023-09-23T19:27:24.253226+0000 osd.4 (osd.4) 30 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T19:27:25.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:25 smithi016 bash[75004]: cluster 2023-09-23T19:27:24.253524+0000 osd.3 (osd.3) 44 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T19:27:25.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:25 smithi016 bash[75004]: cluster 2023-09-23T19:27:24.255733+0000 osd.3 (osd.3) 45 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T19:27:25.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:25 smithi016 bash[75004]: cluster 2023-09-23T19:27:24.257103+0000 osd.5 (osd.5) 34 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T19:27:25.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:25 smithi016 bash[75004]: cluster 2023-09-23T19:27:24.257841+0000 mon.a (mon.0) 778 : cluster 0 osdmap e112: 8 total, 8 up, 8 in 2023-09-23T19:27:25.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:25 smithi016 bash[75004]: cluster 2023-09-23T19:27:24.258368+0000 osd.3 (osd.3) 46 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T19:27:25.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:25 smithi016 bash[75004]: cluster 2023-09-23T19:27:24.261402+0000 osd.5 (osd.5) 35 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T19:27:25.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:25 smithi016 bash[75004]: cluster 2023-09-23T19:27:24.261545+0000 osd.6 (osd.6) 30 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T19:27:25.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:25 smithi016 bash[75004]: cluster 2023-09-23T19:27:24.261568+0000 osd.4 (osd.4) 31 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T19:27:25.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:25 smithi092 bash[58855]: cluster 2023-09-23T19:27:24.232912+0000 mon.a (mon.0) 777 : cluster 3 Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-09-23T19:27:25.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:25 smithi092 bash[58855]: cluster 2023-09-23T19:27:24.247172+0000 osd.3 (osd.3) 43 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T19:27:25.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:25 smithi092 bash[58855]: cluster 2023-09-23T19:27:24.247479+0000 osd.7 (osd.7) 26 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T19:27:25.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:25 smithi092 bash[58855]: cluster 2023-09-23T19:27:24.250419+0000 osd.4 (osd.4) 29 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T19:27:25.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:25 smithi092 bash[58855]: cluster 2023-09-23T19:27:24.251139+0000 osd.6 (osd.6) 29 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T19:27:25.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:25 smithi092 bash[58855]: cluster 2023-09-23T19:27:24.251177+0000 osd.5 (osd.5) 33 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T19:27:25.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:25 smithi092 bash[58855]: cluster 2023-09-23T19:27:24.253226+0000 osd.4 (osd.4) 30 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T19:27:25.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:25 smithi092 bash[58855]: cluster 2023-09-23T19:27:24.253524+0000 osd.3 (osd.3) 44 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T19:27:25.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:25 smithi092 bash[58855]: cluster 2023-09-23T19:27:24.255733+0000 osd.3 (osd.3) 45 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T19:27:25.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:25 smithi092 bash[58855]: cluster 2023-09-23T19:27:24.257103+0000 osd.5 (osd.5) 34 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T19:27:25.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:25 smithi092 bash[58855]: cluster 2023-09-23T19:27:24.257841+0000 mon.a (mon.0) 778 : cluster 0 osdmap e112: 8 total, 8 up, 8 in 2023-09-23T19:27:25.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:25 smithi092 bash[58855]: cluster 2023-09-23T19:27:24.258368+0000 osd.3 (osd.3) 46 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T19:27:25.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:25 smithi092 bash[58855]: cluster 2023-09-23T19:27:24.261402+0000 osd.5 (osd.5) 35 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T19:27:25.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:25 smithi092 bash[58855]: cluster 2023-09-23T19:27:24.261545+0000 osd.6 (osd.6) 30 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T19:27:25.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:25 smithi092 bash[58855]: cluster 2023-09-23T19:27:24.261568+0000 osd.4 (osd.4) 31 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T19:27:26.021 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:27:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:27:25] "GET /metrics HTTP/1.1" 200 33821 "" "Prometheus/2.33.4" 2023-09-23T19:27:26.284 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:27:26 smithi016 bash[70992]: debug 2023-09-23T19:27:26.012+0000 7fb2a5a7c700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2023-09-23T19:27:26.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:26 smithi016 bash[75004]: cluster 2023-09-23T19:27:24.259011+0000 osd.7 (osd.7) 27 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T19:27:26.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:26 smithi016 bash[75004]: cluster 2023-09-23T19:27:24.261629+0000 osd.7 (osd.7) 28 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T19:27:26.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:26 smithi016 bash[75004]: audit 2023-09-23T19:27:25.743194+0000 mon.a (mon.0) 779 : audit 0 from='client.? 172.21.15.16:0/2248461823' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:26.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:26 smithi016 bash[75004]: cluster 2023-09-23T19:27:25.839561+0000 mgr.y (mgr.25325) 577 : cluster 0 pgmap v481: 73 pgs: 2 peering, 9 active+undersized, 62 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:27:26.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:26 smithi016 bash[75004]: audit 2023-09-23T19:27:25.868293+0000 mon.a (mon.0) 780 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:26.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:26 smithi016 bash[75004]: audit 2023-09-23T19:27:25.875619+0000 mon.a (mon.0) 781 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:26.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:26 smithi016 bash[75004]: audit 2023-09-23T19:27:25.877319+0000 mon.a (mon.0) 782 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:27:26.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:26 smithi016 bash[75004]: audit 2023-09-23T19:27:25.878710+0000 mon.a (mon.0) 783 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:27:26.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:26 smithi016 bash[75004]: audit 2023-09-23T19:27:25.886037+0000 mon.a (mon.0) 784 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:26.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:26 smithi016 bash[75004]: audit 2023-09-23T19:27:26.005133+0000 mon.a (mon.0) 785 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:27:26.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:26 smithi016 bash[75004]: audit 2023-09-23T19:27:26.010845+0000 mon.a (mon.0) 786 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:27:26.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:26 smithi016 bash[75004]: audit 2023-09-23T19:27:26.013911+0000 mon.a (mon.0) 787 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:27:26.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:26 smithi016 bash[75004]: audit 2023-09-23T19:27:26.016393+0000 mon.a (mon.0) 788 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:27:26.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:26 smithi016 bash[75004]: audit 2023-09-23T19:27:26.019252+0000 mon.a (mon.0) 789 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T19:27:26.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:26 smithi016 bash[77995]: cluster 2023-09-23T19:27:24.259011+0000 osd.7 (osd.7) 27 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T19:27:26.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:26 smithi016 bash[77995]: cluster 2023-09-23T19:27:24.261629+0000 osd.7 (osd.7) 28 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T19:27:26.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:26 smithi016 bash[77995]: audit 2023-09-23T19:27:25.743194+0000 mon.a (mon.0) 779 : audit 0 from='client.? 172.21.15.16:0/2248461823' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:26.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:26 smithi016 bash[77995]: cluster 2023-09-23T19:27:25.839561+0000 mgr.y (mgr.25325) 577 : cluster 0 pgmap v481: 73 pgs: 2 peering, 9 active+undersized, 62 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:27:26.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:26 smithi016 bash[77995]: audit 2023-09-23T19:27:25.868293+0000 mon.a (mon.0) 780 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:26.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:26 smithi016 bash[77995]: audit 2023-09-23T19:27:25.875619+0000 mon.a (mon.0) 781 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:26.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:26 smithi016 bash[77995]: audit 2023-09-23T19:27:25.877319+0000 mon.a (mon.0) 782 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:27:26.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:26 smithi016 bash[77995]: audit 2023-09-23T19:27:25.878710+0000 mon.a (mon.0) 783 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:27:26.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:26 smithi016 bash[77995]: audit 2023-09-23T19:27:25.886037+0000 mon.a (mon.0) 784 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:26.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:26 smithi016 bash[77995]: audit 2023-09-23T19:27:26.005133+0000 mon.a (mon.0) 785 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:27:26.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:26 smithi016 bash[77995]: audit 2023-09-23T19:27:26.010845+0000 mon.a (mon.0) 786 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:27:26.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:26 smithi016 bash[77995]: audit 2023-09-23T19:27:26.013911+0000 mon.a (mon.0) 787 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:27:26.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:26 smithi016 bash[77995]: audit 2023-09-23T19:27:26.016393+0000 mon.a (mon.0) 788 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:27:26.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:26 smithi016 bash[77995]: audit 2023-09-23T19:27:26.019252+0000 mon.a (mon.0) 789 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T19:27:26.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:26 smithi092 bash[58855]: cluster 2023-09-23T19:27:24.259011+0000 osd.7 (osd.7) 27 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T19:27:26.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:26 smithi092 bash[58855]: cluster 2023-09-23T19:27:24.261629+0000 osd.7 (osd.7) 28 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T19:27:26.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:26 smithi092 bash[58855]: audit 2023-09-23T19:27:25.743194+0000 mon.a (mon.0) 779 : audit 0 from='client.? 172.21.15.16:0/2248461823' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:26.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:26 smithi092 bash[58855]: cluster 2023-09-23T19:27:25.839561+0000 mgr.y (mgr.25325) 577 : cluster 0 pgmap v481: 73 pgs: 2 peering, 9 active+undersized, 62 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:27:26.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:26 smithi092 bash[58855]: audit 2023-09-23T19:27:25.868293+0000 mon.a (mon.0) 780 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:26.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:26 smithi092 bash[58855]: audit 2023-09-23T19:27:25.875619+0000 mon.a (mon.0) 781 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:26.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:26 smithi092 bash[58855]: audit 2023-09-23T19:27:25.877319+0000 mon.a (mon.0) 782 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:27:26.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:26 smithi092 bash[58855]: audit 2023-09-23T19:27:25.878710+0000 mon.a (mon.0) 783 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:27:26.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:26 smithi092 bash[58855]: audit 2023-09-23T19:27:25.886037+0000 mon.a (mon.0) 784 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:26.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:26 smithi092 bash[58855]: audit 2023-09-23T19:27:26.005133+0000 mon.a (mon.0) 785 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:27:26.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:26 smithi092 bash[58855]: audit 2023-09-23T19:27:26.010845+0000 mon.a (mon.0) 786 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:27:26.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:26 smithi092 bash[58855]: audit 2023-09-23T19:27:26.013911+0000 mon.a (mon.0) 787 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:27:26.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:26 smithi092 bash[58855]: audit 2023-09-23T19:27:26.016393+0000 mon.a (mon.0) 788 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:27:26.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:26 smithi092 bash[58855]: audit 2023-09-23T19:27:26.019252+0000 mon.a (mon.0) 789 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T19:27:27.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:27 smithi016 bash[75004]: audit 2023-09-23T19:27:26.019597+0000 mgr.y (mgr.25325) 578 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T19:27:27.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:27 smithi016 bash[75004]: cephadm 2023-09-23T19:27:26.021253+0000 mgr.y (mgr.25325) 579 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2023-09-23T19:27:27.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:27 smithi016 bash[75004]: cluster 2023-09-23T19:27:26.236764+0000 mon.a (mon.0) 790 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2023-09-23T19:27:27.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:27 smithi016 bash[75004]: cluster 2023-09-23T19:27:26.236834+0000 mon.a (mon.0) 791 : cluster 1 Cluster is now healthy 2023-09-23T19:27:27.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:27 smithi016 bash[77995]: audit 2023-09-23T19:27:26.019597+0000 mgr.y (mgr.25325) 578 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T19:27:27.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:27 smithi016 bash[77995]: cephadm 2023-09-23T19:27:26.021253+0000 mgr.y (mgr.25325) 579 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2023-09-23T19:27:27.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:27 smithi016 bash[77995]: cluster 2023-09-23T19:27:26.236764+0000 mon.a (mon.0) 790 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2023-09-23T19:27:27.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:27 smithi016 bash[77995]: cluster 2023-09-23T19:27:26.236834+0000 mon.a (mon.0) 791 : cluster 1 Cluster is now healthy 2023-09-23T19:27:27.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:27 smithi092 bash[58855]: audit 2023-09-23T19:27:26.019597+0000 mgr.y (mgr.25325) 578 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T19:27:27.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:27 smithi092 bash[58855]: cephadm 2023-09-23T19:27:26.021253+0000 mgr.y (mgr.25325) 579 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2023-09-23T19:27:27.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:27 smithi092 bash[58855]: cluster 2023-09-23T19:27:26.236764+0000 mon.a (mon.0) 790 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2023-09-23T19:27:27.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:27 smithi092 bash[58855]: cluster 2023-09-23T19:27:26.236834+0000 mon.a (mon.0) 791 : cluster 1 Cluster is now healthy 2023-09-23T19:27:28.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:28 smithi016 bash[75004]: cluster 2023-09-23T19:27:27.840419+0000 mgr.y (mgr.25325) 580 : cluster 0 pgmap v482: 73 pgs: 2 peering, 71 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:27:28.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:28 smithi016 bash[75004]: audit 2023-09-23T19:27:28.113025+0000 mon.a (mon.0) 792 : audit 0 from='client.? 172.21.15.16:0/3567387482' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:28.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:28 smithi016 bash[77995]: cluster 2023-09-23T19:27:27.840419+0000 mgr.y (mgr.25325) 580 : cluster 0 pgmap v482: 73 pgs: 2 peering, 71 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:27:28.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:28 smithi016 bash[77995]: audit 2023-09-23T19:27:28.113025+0000 mon.a (mon.0) 792 : audit 0 from='client.? 172.21.15.16:0/3567387482' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:28.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:28 smithi092 bash[58855]: cluster 2023-09-23T19:27:27.840419+0000 mgr.y (mgr.25325) 580 : cluster 0 pgmap v482: 73 pgs: 2 peering, 71 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:27:28.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:28 smithi092 bash[58855]: audit 2023-09-23T19:27:28.113025+0000 mon.a (mon.0) 792 : audit 0 from='client.? 172.21.15.16:0/3567387482' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:30.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:29 smithi016 bash[75004]: audit 2023-09-23T19:27:28.636688+0000 mon.a (mon.0) 793 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:30.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:29 smithi016 bash[75004]: audit 2023-09-23T19:27:28.638267+0000 mon.a (mon.0) 794 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:27:30.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:29 smithi016 bash[77995]: audit 2023-09-23T19:27:28.636688+0000 mon.a (mon.0) 793 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:30.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:29 smithi016 bash[77995]: audit 2023-09-23T19:27:28.638267+0000 mon.a (mon.0) 794 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:27:30.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:29 smithi092 bash[58855]: audit 2023-09-23T19:27:28.636688+0000 mon.a (mon.0) 793 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:30.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:29 smithi092 bash[58855]: audit 2023-09-23T19:27:28.638267+0000 mon.a (mon.0) 794 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:27:31.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:30 smithi016 bash[75004]: cluster 2023-09-23T19:27:29.841132+0000 mgr.y (mgr.25325) 581 : cluster 0 pgmap v483: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:27:31.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:30 smithi016 bash[75004]: audit 2023-09-23T19:27:30.485533+0000 mon.c (mon.2) 118 : audit 0 from='client.? 172.21.15.16:0/4110287984' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:31.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:30 smithi016 bash[77995]: cluster 2023-09-23T19:27:29.841132+0000 mgr.y (mgr.25325) 581 : cluster 0 pgmap v483: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:27:31.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:30 smithi016 bash[77995]: audit 2023-09-23T19:27:30.485533+0000 mon.c (mon.2) 118 : audit 0 from='client.? 172.21.15.16:0/4110287984' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:31.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:30 smithi092 bash[58855]: cluster 2023-09-23T19:27:29.841132+0000 mgr.y (mgr.25325) 581 : cluster 0 pgmap v483: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:27:31.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:30 smithi092 bash[58855]: audit 2023-09-23T19:27:30.485533+0000 mon.c (mon.2) 118 : audit 0 from='client.? 172.21.15.16:0/4110287984' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:32.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:31 smithi016 bash[75004]: cluster 2023-09-23T19:27:31.842149+0000 mgr.y (mgr.25325) 582 : cluster 0 pgmap v484: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:27:32.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:31 smithi016 bash[77995]: cluster 2023-09-23T19:27:31.842149+0000 mgr.y (mgr.25325) 582 : cluster 0 pgmap v484: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:27:32.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:31 smithi092 bash[58855]: cluster 2023-09-23T19:27:31.842149+0000 mgr.y (mgr.25325) 582 : cluster 0 pgmap v484: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:27:33.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:32 smithi016 bash[75004]: audit 2023-09-23T19:27:32.856673+0000 mon.c (mon.2) 119 : audit 0 from='client.? 172.21.15.16:0/2495113544' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:33.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:32 smithi016 bash[77995]: audit 2023-09-23T19:27:32.856673+0000 mon.c (mon.2) 119 : audit 0 from='client.? 172.21.15.16:0/2495113544' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:33.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:32 smithi092 bash[58855]: audit 2023-09-23T19:27:32.856673+0000 mon.c (mon.2) 119 : audit 0 from='client.? 172.21.15.16:0/2495113544' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:34.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:33 smithi016 bash[75004]: cluster 2023-09-23T19:27:33.842783+0000 mgr.y (mgr.25325) 583 : cluster 0 pgmap v485: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:27:34.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:33 smithi016 bash[77995]: cluster 2023-09-23T19:27:33.842783+0000 mgr.y (mgr.25325) 583 : cluster 0 pgmap v485: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:27:34.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:33 smithi092 bash[58855]: cluster 2023-09-23T19:27:33.842783+0000 mgr.y (mgr.25325) 583 : cluster 0 pgmap v485: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:27:35.642 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:35 smithi016 bash[77995]: audit 2023-09-23T19:27:35.233638+0000 mon.c (mon.2) 120 : audit 0 from='client.? 172.21.15.16:0/3300014196' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:35.643 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:35 smithi016 bash[75004]: audit 2023-09-23T19:27:35.233638+0000 mon.c (mon.2) 120 : audit 0 from='client.? 172.21.15.16:0/3300014196' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:35.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:35 smithi092 bash[58855]: audit 2023-09-23T19:27:35.233638+0000 mon.c (mon.2) 120 : audit 0 from='client.? 172.21.15.16:0/3300014196' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:36.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:27:35 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:27:35] "GET /metrics HTTP/1.1" 200 33814 "" "Prometheus/2.33.4" 2023-09-23T19:27:36.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:36 smithi016 bash[75004]: cluster 2023-09-23T19:27:35.843720+0000 mgr.y (mgr.25325) 584 : cluster 0 pgmap v486: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:27:36.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:36 smithi016 bash[77995]: cluster 2023-09-23T19:27:35.843720+0000 mgr.y (mgr.25325) 584 : cluster 0 pgmap v486: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:27:36.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:36 smithi092 bash[58855]: cluster 2023-09-23T19:27:35.843720+0000 mgr.y (mgr.25325) 584 : cluster 0 pgmap v486: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:27:38.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:37 smithi016 bash[77995]: audit 2023-09-23T19:27:37.614237+0000 mon.c (mon.2) 121 : audit 0 from='client.? 172.21.15.16:0/3027851111' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:38.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:37 smithi016 bash[75004]: audit 2023-09-23T19:27:37.614237+0000 mon.c (mon.2) 121 : audit 0 from='client.? 172.21.15.16:0/3027851111' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:38.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:37 smithi092 bash[58855]: audit 2023-09-23T19:27:37.614237+0000 mon.c (mon.2) 121 : audit 0 from='client.? 172.21.15.16:0/3027851111' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:39.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:38 smithi016 bash[77995]: cluster 2023-09-23T19:27:37.844698+0000 mgr.y (mgr.25325) 585 : cluster 0 pgmap v487: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:27:39.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:38 smithi016 bash[75004]: cluster 2023-09-23T19:27:37.844698+0000 mgr.y (mgr.25325) 585 : cluster 0 pgmap v487: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:27:39.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:38 smithi092 bash[58855]: cluster 2023-09-23T19:27:37.844698+0000 mgr.y (mgr.25325) 585 : cluster 0 pgmap v487: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:27:40.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:39 smithi016 bash[75004]: audit 2023-09-23T19:27:38.846380+0000 mon.a (mon.0) 795 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:40.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:39 smithi016 bash[77995]: audit 2023-09-23T19:27:38.846380+0000 mon.a (mon.0) 795 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:40.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:39 smithi092 bash[58855]: audit 2023-09-23T19:27:38.846380+0000 mon.a (mon.0) 795 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:41.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:40 smithi016 bash[77995]: cluster 2023-09-23T19:27:39.845382+0000 mgr.y (mgr.25325) 586 : cluster 0 pgmap v488: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:27:41.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:40 smithi016 bash[77995]: audit 2023-09-23T19:27:39.994887+0000 mon.a (mon.0) 796 : audit 0 from='client.? 172.21.15.16:0/3083866360' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:41.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:40 smithi016 bash[75004]: cluster 2023-09-23T19:27:39.845382+0000 mgr.y (mgr.25325) 586 : cluster 0 pgmap v488: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:27:41.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:40 smithi016 bash[75004]: audit 2023-09-23T19:27:39.994887+0000 mon.a (mon.0) 796 : audit 0 from='client.? 172.21.15.16:0/3083866360' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:41.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:40 smithi092 bash[58855]: cluster 2023-09-23T19:27:39.845382+0000 mgr.y (mgr.25325) 586 : cluster 0 pgmap v488: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:27:41.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:40 smithi092 bash[58855]: audit 2023-09-23T19:27:39.994887+0000 mon.a (mon.0) 796 : audit 0 from='client.? 172.21.15.16:0/3083866360' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:42.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:41 smithi016 bash[77995]: audit 2023-09-23T19:27:41.037112+0000 mon.a (mon.0) 797 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T19:27:42.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:41 smithi016 bash[75004]: audit 2023-09-23T19:27:41.037112+0000 mon.a (mon.0) 797 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T19:27:42.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:41 smithi092 bash[58855]: audit 2023-09-23T19:27:41.037112+0000 mon.a (mon.0) 797 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T19:27:43.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:42 smithi016 bash[75004]: audit 2023-09-23T19:27:41.037536+0000 mgr.y (mgr.25325) 587 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T19:27:43.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:42 smithi016 bash[75004]: cephadm 2023-09-23T19:27:41.039838+0000 mgr.y (mgr.25325) 588 : cephadm 1 Upgrade: osd.3 is safe to restart 2023-09-23T19:27:43.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:42 smithi016 bash[75004]: cluster 2023-09-23T19:27:41.845992+0000 mgr.y (mgr.25325) 589 : cluster 0 pgmap v489: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:27:43.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:42 smithi016 bash[75004]: audit 2023-09-23T19:27:42.363855+0000 mon.c (mon.2) 122 : audit 0 from='client.? 172.21.15.16:0/3497513737' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:43.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:42 smithi016 bash[77995]: audit 2023-09-23T19:27:41.037536+0000 mgr.y (mgr.25325) 587 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T19:27:43.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:42 smithi016 bash[77995]: cephadm 2023-09-23T19:27:41.039838+0000 mgr.y (mgr.25325) 588 : cephadm 1 Upgrade: osd.3 is safe to restart 2023-09-23T19:27:43.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:42 smithi016 bash[77995]: cluster 2023-09-23T19:27:41.845992+0000 mgr.y (mgr.25325) 589 : cluster 0 pgmap v489: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:27:43.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:42 smithi016 bash[77995]: audit 2023-09-23T19:27:42.363855+0000 mon.c (mon.2) 122 : audit 0 from='client.? 172.21.15.16:0/3497513737' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:43.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:42 smithi092 bash[58855]: audit 2023-09-23T19:27:41.037536+0000 mgr.y (mgr.25325) 587 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T19:27:43.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:42 smithi092 bash[58855]: cephadm 2023-09-23T19:27:41.039838+0000 mgr.y (mgr.25325) 588 : cephadm 1 Upgrade: osd.3 is safe to restart 2023-09-23T19:27:43.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:42 smithi092 bash[58855]: cluster 2023-09-23T19:27:41.845992+0000 mgr.y (mgr.25325) 589 : cluster 0 pgmap v489: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:27:43.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:42 smithi092 bash[58855]: audit 2023-09-23T19:27:42.363855+0000 mon.c (mon.2) 122 : audit 0 from='client.? 172.21.15.16:0/3497513737' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:44.881 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:44 smithi016 bash[75004]: cephadm 2023-09-23T19:27:43.561193+0000 mgr.y (mgr.25325) 590 : cephadm 1 Upgrade: Updating osd.3 2023-09-23T19:27:44.881 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:44 smithi016 bash[75004]: audit 2023-09-23T19:27:43.572803+0000 mon.a (mon.0) 798 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:44.881 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:44 smithi016 bash[75004]: audit 2023-09-23T19:27:43.574915+0000 mon.a (mon.0) 799 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T19:27:44.881 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:44 smithi016 bash[75004]: audit 2023-09-23T19:27:43.576088+0000 mon.a (mon.0) 800 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:27:44.881 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:44 smithi016 bash[75004]: cephadm 2023-09-23T19:27:43.579060+0000 mgr.y (mgr.25325) 591 : cephadm 1 Deploying daemon osd.3 on smithi016 2023-09-23T19:27:44.881 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:44 smithi016 bash[75004]: audit 2023-09-23T19:27:43.630601+0000 mon.a (mon.0) 801 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:27:44.881 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:44 smithi016 bash[75004]: cluster 2023-09-23T19:27:43.846383+0000 mgr.y (mgr.25325) 592 : cluster 0 pgmap v490: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:27:44.882 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:44 smithi016 bash[77995]: cephadm 2023-09-23T19:27:43.561193+0000 mgr.y (mgr.25325) 590 : cephadm 1 Upgrade: Updating osd.3 2023-09-23T19:27:44.882 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:44 smithi016 bash[77995]: audit 2023-09-23T19:27:43.572803+0000 mon.a (mon.0) 798 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:44.882 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:44 smithi016 bash[77995]: audit 2023-09-23T19:27:43.574915+0000 mon.a (mon.0) 799 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T19:27:44.882 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:44 smithi016 bash[77995]: audit 2023-09-23T19:27:43.576088+0000 mon.a (mon.0) 800 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:27:44.882 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:44 smithi016 bash[77995]: cephadm 2023-09-23T19:27:43.579060+0000 mgr.y (mgr.25325) 591 : cephadm 1 Deploying daemon osd.3 on smithi016 2023-09-23T19:27:44.882 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:44 smithi016 bash[77995]: audit 2023-09-23T19:27:43.630601+0000 mon.a (mon.0) 801 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:27:44.882 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:44 smithi016 bash[77995]: cluster 2023-09-23T19:27:43.846383+0000 mgr.y (mgr.25325) 592 : cluster 0 pgmap v490: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:27:45.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:44 smithi092 bash[58855]: cephadm 2023-09-23T19:27:43.561193+0000 mgr.y (mgr.25325) 590 : cephadm 1 Upgrade: Updating osd.3 2023-09-23T19:27:45.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:44 smithi092 bash[58855]: audit 2023-09-23T19:27:43.572803+0000 mon.a (mon.0) 798 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:45.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:44 smithi092 bash[58855]: audit 2023-09-23T19:27:43.574915+0000 mon.a (mon.0) 799 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T19:27:45.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:44 smithi092 bash[58855]: audit 2023-09-23T19:27:43.576088+0000 mon.a (mon.0) 800 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:27:45.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:44 smithi092 bash[58855]: cephadm 2023-09-23T19:27:43.579060+0000 mgr.y (mgr.25325) 591 : cephadm 1 Deploying daemon osd.3 on smithi016 2023-09-23T19:27:45.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:44 smithi092 bash[58855]: audit 2023-09-23T19:27:43.630601+0000 mon.a (mon.0) 801 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:27:45.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:44 smithi092 bash[58855]: cluster 2023-09-23T19:27:43.846383+0000 mgr.y (mgr.25325) 592 : cluster 0 pgmap v490: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:27:45.859 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:45 smithi016 bash[77995]: audit 2023-09-23T19:27:44.739701+0000 mon.a (mon.0) 802 : audit 0 from='client.? 172.21.15.16:0/1093718925' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:45.859 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:45 smithi016 bash[75004]: audit 2023-09-23T19:27:44.739701+0000 mon.a (mon.0) 802 : audit 0 from='client.? 172.21.15.16:0/1093718925' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:45.860 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:27:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:27:45] "GET /metrics HTTP/1.1" 200 33817 "" "Prometheus/2.33.4" 2023-09-23T19:27:46.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:45 smithi092 bash[58855]: audit 2023-09-23T19:27:44.739701+0000 mon.a (mon.0) 802 : audit 0 from='client.? 172.21.15.16:0/1093718925' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:46.847 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:46 smithi016 bash[75004]: cluster 2023-09-23T19:27:45.847026+0000 mgr.y (mgr.25325) 593 : cluster 0 pgmap v491: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:27:46.847 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:46 smithi016 bash[77995]: cluster 2023-09-23T19:27:45.847026+0000 mgr.y (mgr.25325) 593 : cluster 0 pgmap v491: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:27:47.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:46 smithi092 bash[58855]: cluster 2023-09-23T19:27:45.847026+0000 mgr.y (mgr.25325) 593 : cluster 0 pgmap v491: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:27:47.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:47 smithi016 bash[75004]: audit 2023-09-23T19:27:47.114141+0000 mon.a (mon.0) 803 : audit 0 from='client.? 172.21.15.16:0/1200453764' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:47.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:47 smithi016 bash[77995]: audit 2023-09-23T19:27:47.114141+0000 mon.a (mon.0) 803 : audit 0 from='client.? 172.21.15.16:0/1200453764' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:48.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:47 smithi092 bash[58855]: audit 2023-09-23T19:27:47.114141+0000 mon.a (mon.0) 803 : audit 0 from='client.? 172.21.15.16:0/1200453764' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:48.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:48 smithi016 bash[77995]: cluster 2023-09-23T19:27:47.847892+0000 mgr.y (mgr.25325) 594 : cluster 0 pgmap v492: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:27:48.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:48 smithi016 bash[77995]: cluster 2023-09-23T19:27:48.584315+0000 mon.a (mon.0) 804 : cluster 1 osd.3 marked itself down and dead 2023-09-23T19:27:48.785 INFO:journalctl@ceph.osd.3.smithi016.stdout:Sep 23 19:27:48 smithi016 systemd[1]: Stopping Ceph osd.3 for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7... 2023-09-23T19:27:48.785 INFO:journalctl@ceph.osd.3.smithi016.stdout:Sep 23 19:27:48 smithi016 bash[49243]: debug 2023-09-23T19:27:48.580+0000 7f67962a2700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-09-23T19:27:48.785 INFO:journalctl@ceph.osd.3.smithi016.stdout:Sep 23 19:27:48 smithi016 bash[49243]: debug 2023-09-23T19:27:48.580+0000 7f67962a2700 -1 osd.3 112 *** Got signal Terminated *** 2023-09-23T19:27:48.785 INFO:journalctl@ceph.osd.3.smithi016.stdout:Sep 23 19:27:48 smithi016 bash[49243]: debug 2023-09-23T19:27:48.580+0000 7f67962a2700 -1 osd.3 112 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T19:27:48.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:48 smithi016 bash[75004]: cluster 2023-09-23T19:27:47.847892+0000 mgr.y (mgr.25325) 594 : cluster 0 pgmap v492: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:27:48.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:48 smithi016 bash[75004]: cluster 2023-09-23T19:27:48.584315+0000 mon.a (mon.0) 804 : cluster 1 osd.3 marked itself down and dead 2023-09-23T19:27:49.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:48 smithi092 bash[58855]: cluster 2023-09-23T19:27:47.847892+0000 mgr.y (mgr.25325) 594 : cluster 0 pgmap v492: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:27:49.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:48 smithi092 bash[58855]: cluster 2023-09-23T19:27:48.584315+0000 mon.a (mon.0) 804 : cluster 1 osd.3 marked itself down and dead 2023-09-23T19:27:49.708 INFO:journalctl@ceph.osd.3.smithi016.stdout:Sep 23 19:27:49 smithi016 bash[100280]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd-3 2023-09-23T19:27:49.708 INFO:journalctl@ceph.osd.3.smithi016.stdout:Sep 23 19:27:49 smithi016 bash[100372]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd-3 2023-09-23T19:27:49.709 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:49 smithi016 bash[75004]: cluster 2023-09-23T19:27:48.586278+0000 mon.a (mon.0) 805 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T19:27:49.709 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:49 smithi016 bash[75004]: cluster 2023-09-23T19:27:48.597635+0000 mon.a (mon.0) 806 : cluster 0 osdmap e113: 8 total, 7 up, 8 in 2023-09-23T19:27:49.709 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:49 smithi016 bash[75004]: cluster 2023-09-23T19:27:48.605188+0000 osd.7 (osd.7) 29 : cluster 3 failed to encode map e113 with expected crc 2023-09-23T19:27:49.709 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:49 smithi016 bash[75004]: cluster 2023-09-23T19:27:48.605321+0000 osd.4 (osd.4) 32 : cluster 3 failed to encode map e113 with expected crc 2023-09-23T19:27:49.709 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:49 smithi016 bash[75004]: cluster 2023-09-23T19:27:48.605394+0000 osd.5 (osd.5) 36 : cluster 3 failed to encode map e113 with expected crc 2023-09-23T19:27:49.709 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:49 smithi016 bash[75004]: cluster 2023-09-23T19:27:48.607357+0000 osd.6 (osd.6) 31 : cluster 3 failed to encode map e113 with expected crc 2023-09-23T19:27:49.709 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:49 smithi016 bash[75004]: cluster 2023-09-23T19:27:48.607707+0000 osd.4 (osd.4) 33 : cluster 3 failed to encode map e113 with expected crc 2023-09-23T19:27:49.709 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:49 smithi016 bash[75004]: cluster 2023-09-23T19:27:48.607865+0000 osd.7 (osd.7) 30 : cluster 3 failed to encode map e113 with expected crc 2023-09-23T19:27:49.709 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:49 smithi016 bash[75004]: cluster 2023-09-23T19:27:48.610328+0000 osd.6 (osd.6) 32 : cluster 3 failed to encode map e113 with expected crc 2023-09-23T19:27:49.710 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:49 smithi016 bash[75004]: cluster 2023-09-23T19:27:48.610623+0000 osd.4 (osd.4) 34 : cluster 3 failed to encode map e113 with expected crc 2023-09-23T19:27:49.710 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:49 smithi016 bash[75004]: audit 2023-09-23T19:27:49.475203+0000 mon.a (mon.0) 807 : audit 0 from='client.? 172.21.15.16:0/855821163' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:49.710 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:49 smithi016 bash[77995]: cluster 2023-09-23T19:27:48.586278+0000 mon.a (mon.0) 805 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T19:27:49.710 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:49 smithi016 bash[77995]: cluster 2023-09-23T19:27:48.597635+0000 mon.a (mon.0) 806 : cluster 0 osdmap e113: 8 total, 7 up, 8 in 2023-09-23T19:27:49.710 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:49 smithi016 bash[77995]: cluster 2023-09-23T19:27:48.605188+0000 osd.7 (osd.7) 29 : cluster 3 failed to encode map e113 with expected crc 2023-09-23T19:27:49.710 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:49 smithi016 bash[77995]: cluster 2023-09-23T19:27:48.605321+0000 osd.4 (osd.4) 32 : cluster 3 failed to encode map e113 with expected crc 2023-09-23T19:27:49.710 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:49 smithi016 bash[77995]: cluster 2023-09-23T19:27:48.605394+0000 osd.5 (osd.5) 36 : cluster 3 failed to encode map e113 with expected crc 2023-09-23T19:27:49.710 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:49 smithi016 bash[77995]: cluster 2023-09-23T19:27:48.607357+0000 osd.6 (osd.6) 31 : cluster 3 failed to encode map e113 with expected crc 2023-09-23T19:27:49.710 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:49 smithi016 bash[77995]: cluster 2023-09-23T19:27:48.607707+0000 osd.4 (osd.4) 33 : cluster 3 failed to encode map e113 with expected crc 2023-09-23T19:27:49.710 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:49 smithi016 bash[77995]: cluster 2023-09-23T19:27:48.607865+0000 osd.7 (osd.7) 30 : cluster 3 failed to encode map e113 with expected crc 2023-09-23T19:27:49.711 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:49 smithi016 bash[77995]: cluster 2023-09-23T19:27:48.610328+0000 osd.6 (osd.6) 32 : cluster 3 failed to encode map e113 with expected crc 2023-09-23T19:27:49.711 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:49 smithi016 bash[77995]: cluster 2023-09-23T19:27:48.610623+0000 osd.4 (osd.4) 34 : cluster 3 failed to encode map e113 with expected crc 2023-09-23T19:27:49.711 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:49 smithi016 bash[77995]: audit 2023-09-23T19:27:49.475203+0000 mon.a (mon.0) 807 : audit 0 from='client.? 172.21.15.16:0/855821163' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:50.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:49 smithi092 bash[58855]: cluster 2023-09-23T19:27:48.586278+0000 mon.a (mon.0) 805 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T19:27:50.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:49 smithi092 bash[58855]: cluster 2023-09-23T19:27:48.597635+0000 mon.a (mon.0) 806 : cluster 0 osdmap e113: 8 total, 7 up, 8 in 2023-09-23T19:27:50.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:49 smithi092 bash[58855]: cluster 2023-09-23T19:27:48.605188+0000 osd.7 (osd.7) 29 : cluster 3 failed to encode map e113 with expected crc 2023-09-23T19:27:50.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:49 smithi092 bash[58855]: cluster 2023-09-23T19:27:48.605321+0000 osd.4 (osd.4) 32 : cluster 3 failed to encode map e113 with expected crc 2023-09-23T19:27:50.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:49 smithi092 bash[58855]: cluster 2023-09-23T19:27:48.605394+0000 osd.5 (osd.5) 36 : cluster 3 failed to encode map e113 with expected crc 2023-09-23T19:27:50.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:49 smithi092 bash[58855]: cluster 2023-09-23T19:27:48.607357+0000 osd.6 (osd.6) 31 : cluster 3 failed to encode map e113 with expected crc 2023-09-23T19:27:50.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:49 smithi092 bash[58855]: cluster 2023-09-23T19:27:48.607707+0000 osd.4 (osd.4) 33 : cluster 3 failed to encode map e113 with expected crc 2023-09-23T19:27:50.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:49 smithi092 bash[58855]: cluster 2023-09-23T19:27:48.607865+0000 osd.7 (osd.7) 30 : cluster 3 failed to encode map e113 with expected crc 2023-09-23T19:27:50.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:49 smithi092 bash[58855]: cluster 2023-09-23T19:27:48.610328+0000 osd.6 (osd.6) 32 : cluster 3 failed to encode map e113 with expected crc 2023-09-23T19:27:50.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:49 smithi092 bash[58855]: cluster 2023-09-23T19:27:48.610623+0000 osd.4 (osd.4) 34 : cluster 3 failed to encode map e113 with expected crc 2023-09-23T19:27:50.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:49 smithi092 bash[58855]: audit 2023-09-23T19:27:49.475203+0000 mon.a (mon.0) 807 : audit 0 from='client.? 172.21.15.16:0/855821163' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:51.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:50 smithi016 bash[77995]: cluster 2023-09-23T19:27:49.601629+0000 mon.a (mon.0) 808 : cluster 0 osdmap e114: 8 total, 7 up, 8 in 2023-09-23T19:27:51.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:50 smithi016 bash[77995]: cluster 2023-09-23T19:27:49.604520+0000 osd.7 (osd.7) 31 : cluster 3 failed to encode map e114 with expected crc 2023-09-23T19:27:51.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:50 smithi016 bash[77995]: cluster 2023-09-23T19:27:49.604639+0000 osd.4 (osd.4) 35 : cluster 3 failed to encode map e114 with expected crc 2023-09-23T19:27:51.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:50 smithi016 bash[77995]: cluster 2023-09-23T19:27:49.605454+0000 osd.5 (osd.5) 37 : cluster 3 failed to encode map e114 with expected crc 2023-09-23T19:27:51.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:50 smithi016 bash[77995]: cluster 2023-09-23T19:27:49.605654+0000 osd.6 (osd.6) 33 : cluster 3 failed to encode map e114 with expected crc 2023-09-23T19:27:51.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:50 smithi016 bash[77995]: cluster 2023-09-23T19:27:49.606604+0000 osd.7 (osd.7) 32 : cluster 3 failed to encode map e114 with expected crc 2023-09-23T19:27:51.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:50 smithi016 bash[77995]: cluster 2023-09-23T19:27:49.608391+0000 osd.4 (osd.4) 36 : cluster 3 failed to encode map e114 with expected crc 2023-09-23T19:27:51.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:50 smithi016 bash[77995]: cluster 2023-09-23T19:27:49.610234+0000 osd.5 (osd.5) 38 : cluster 3 failed to encode map e114 with expected crc 2023-09-23T19:27:51.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:50 smithi016 bash[77995]: cluster 2023-09-23T19:27:49.610526+0000 osd.4 (osd.4) 37 : cluster 3 failed to encode map e114 with expected crc 2023-09-23T19:27:51.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:50 smithi016 bash[77995]: cluster 2023-09-23T19:27:49.610635+0000 osd.6 (osd.6) 34 : cluster 3 failed to encode map e114 with expected crc 2023-09-23T19:27:51.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:50 smithi016 bash[77995]: cluster 2023-09-23T19:27:49.612709+0000 osd.4 (osd.4) 38 : cluster 3 failed to encode map e114 with expected crc 2023-09-23T19:27:51.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:50 smithi016 bash[77995]: cluster 2023-09-23T19:27:49.848649+0000 mgr.y (mgr.25325) 595 : cluster 0 pgmap v495: 73 pgs: 8 peering, 10 stale+active+clean, 55 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:27:51.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:50 smithi016 bash[75004]: cluster 2023-09-23T19:27:49.601629+0000 mon.a (mon.0) 808 : cluster 0 osdmap e114: 8 total, 7 up, 8 in 2023-09-23T19:27:51.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:50 smithi016 bash[75004]: cluster 2023-09-23T19:27:49.604520+0000 osd.7 (osd.7) 31 : cluster 3 failed to encode map e114 with expected crc 2023-09-23T19:27:51.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:50 smithi016 bash[75004]: cluster 2023-09-23T19:27:49.604639+0000 osd.4 (osd.4) 35 : cluster 3 failed to encode map e114 with expected crc 2023-09-23T19:27:51.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:50 smithi016 bash[75004]: cluster 2023-09-23T19:27:49.605454+0000 osd.5 (osd.5) 37 : cluster 3 failed to encode map e114 with expected crc 2023-09-23T19:27:51.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:50 smithi016 bash[75004]: cluster 2023-09-23T19:27:49.605654+0000 osd.6 (osd.6) 33 : cluster 3 failed to encode map e114 with expected crc 2023-09-23T19:27:51.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:50 smithi016 bash[75004]: cluster 2023-09-23T19:27:49.606604+0000 osd.7 (osd.7) 32 : cluster 3 failed to encode map e114 with expected crc 2023-09-23T19:27:51.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:50 smithi016 bash[75004]: cluster 2023-09-23T19:27:49.608391+0000 osd.4 (osd.4) 36 : cluster 3 failed to encode map e114 with expected crc 2023-09-23T19:27:51.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:50 smithi016 bash[75004]: cluster 2023-09-23T19:27:49.610234+0000 osd.5 (osd.5) 38 : cluster 3 failed to encode map e114 with expected crc 2023-09-23T19:27:51.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:50 smithi016 bash[75004]: cluster 2023-09-23T19:27:49.610526+0000 osd.4 (osd.4) 37 : cluster 3 failed to encode map e114 with expected crc 2023-09-23T19:27:51.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:50 smithi016 bash[75004]: cluster 2023-09-23T19:27:49.610635+0000 osd.6 (osd.6) 34 : cluster 3 failed to encode map e114 with expected crc 2023-09-23T19:27:51.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:50 smithi016 bash[75004]: cluster 2023-09-23T19:27:49.612709+0000 osd.4 (osd.4) 38 : cluster 3 failed to encode map e114 with expected crc 2023-09-23T19:27:51.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:50 smithi016 bash[75004]: cluster 2023-09-23T19:27:49.848649+0000 mgr.y (mgr.25325) 595 : cluster 0 pgmap v495: 73 pgs: 8 peering, 10 stale+active+clean, 55 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:27:51.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:50 smithi092 bash[58855]: cluster 2023-09-23T19:27:49.601629+0000 mon.a (mon.0) 808 : cluster 0 osdmap e114: 8 total, 7 up, 8 in 2023-09-23T19:27:51.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:50 smithi092 bash[58855]: cluster 2023-09-23T19:27:49.604520+0000 osd.7 (osd.7) 31 : cluster 3 failed to encode map e114 with expected crc 2023-09-23T19:27:51.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:50 smithi092 bash[58855]: cluster 2023-09-23T19:27:49.604639+0000 osd.4 (osd.4) 35 : cluster 3 failed to encode map e114 with expected crc 2023-09-23T19:27:51.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:50 smithi092 bash[58855]: cluster 2023-09-23T19:27:49.605454+0000 osd.5 (osd.5) 37 : cluster 3 failed to encode map e114 with expected crc 2023-09-23T19:27:51.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:50 smithi092 bash[58855]: cluster 2023-09-23T19:27:49.605654+0000 osd.6 (osd.6) 33 : cluster 3 failed to encode map e114 with expected crc 2023-09-23T19:27:51.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:50 smithi092 bash[58855]: cluster 2023-09-23T19:27:49.606604+0000 osd.7 (osd.7) 32 : cluster 3 failed to encode map e114 with expected crc 2023-09-23T19:27:51.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:50 smithi092 bash[58855]: cluster 2023-09-23T19:27:49.608391+0000 osd.4 (osd.4) 36 : cluster 3 failed to encode map e114 with expected crc 2023-09-23T19:27:51.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:50 smithi092 bash[58855]: cluster 2023-09-23T19:27:49.610234+0000 osd.5 (osd.5) 38 : cluster 3 failed to encode map e114 with expected crc 2023-09-23T19:27:51.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:50 smithi092 bash[58855]: cluster 2023-09-23T19:27:49.610526+0000 osd.4 (osd.4) 37 : cluster 3 failed to encode map e114 with expected crc 2023-09-23T19:27:51.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:50 smithi092 bash[58855]: cluster 2023-09-23T19:27:49.610635+0000 osd.6 (osd.6) 34 : cluster 3 failed to encode map e114 with expected crc 2023-09-23T19:27:51.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:50 smithi092 bash[58855]: cluster 2023-09-23T19:27:49.612709+0000 osd.4 (osd.4) 38 : cluster 3 failed to encode map e114 with expected crc 2023-09-23T19:27:51.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:50 smithi092 bash[58855]: cluster 2023-09-23T19:27:49.848649+0000 mgr.y (mgr.25325) 595 : cluster 0 pgmap v495: 73 pgs: 8 peering, 10 stale+active+clean, 55 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:27:51.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:51 smithi016 bash[77995]: cluster 2023-09-23T19:27:50.604136+0000 mon.a (mon.0) 809 : cluster 3 Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T19:27:51.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:51 smithi016 bash[75004]: cluster 2023-09-23T19:27:50.604136+0000 mon.a (mon.0) 809 : cluster 3 Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T19:27:52.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:51 smithi092 bash[58855]: cluster 2023-09-23T19:27:50.604136+0000 mon.a (mon.0) 809 : cluster 3 Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T19:27:52.231 INFO:journalctl@ceph.osd.3.smithi016.stdout:Sep 23 19:27:51 smithi016 systemd[1]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@osd.3.service: Succeeded. 2023-09-23T19:27:52.231 INFO:journalctl@ceph.osd.3.smithi016.stdout:Sep 23 19:27:51 smithi016 systemd[1]: Stopped Ceph osd.3 for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7. 2023-09-23T19:27:52.534 INFO:journalctl@ceph.osd.3.smithi016.stdout:Sep 23 19:27:52 smithi016 systemd[1]: Started Ceph osd.3 for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7. 2023-09-23T19:27:53.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:52 smithi016 bash[75004]: audit 2023-09-23T19:27:51.846934+0000 mon.c (mon.2) 123 : audit 0 from='client.? 172.21.15.16:0/4259921704' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:53.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:52 smithi016 bash[75004]: cluster 2023-09-23T19:27:51.849270+0000 mgr.y (mgr.25325) 596 : cluster 0 pgmap v496: 73 pgs: 2 active+undersized+degraded, 8 peering, 1 stale+active+clean, 18 active+undersized, 44 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-09-23T19:27:53.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:52 smithi016 bash[75004]: audit 2023-09-23T19:27:52.268030+0000 mon.a (mon.0) 810 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:53.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:52 smithi016 bash[75004]: audit 2023-09-23T19:27:52.276293+0000 mon.a (mon.0) 811 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:53.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:52 smithi016 bash[75004]: audit 2023-09-23T19:27:52.318036+0000 mon.a (mon.0) 812 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:27:53.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:52 smithi016 bash[77995]: audit 2023-09-23T19:27:51.846934+0000 mon.c (mon.2) 123 : audit 0 from='client.? 172.21.15.16:0/4259921704' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:53.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:52 smithi016 bash[77995]: cluster 2023-09-23T19:27:51.849270+0000 mgr.y (mgr.25325) 596 : cluster 0 pgmap v496: 73 pgs: 2 active+undersized+degraded, 8 peering, 1 stale+active+clean, 18 active+undersized, 44 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-09-23T19:27:53.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:52 smithi016 bash[77995]: audit 2023-09-23T19:27:52.268030+0000 mon.a (mon.0) 810 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:53.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:52 smithi016 bash[77995]: audit 2023-09-23T19:27:52.276293+0000 mon.a (mon.0) 811 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:53.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:52 smithi016 bash[77995]: audit 2023-09-23T19:27:52.318036+0000 mon.a (mon.0) 812 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:27:53.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:52 smithi092 bash[58855]: audit 2023-09-23T19:27:51.846934+0000 mon.c (mon.2) 123 : audit 0 from='client.? 172.21.15.16:0/4259921704' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:53.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:52 smithi092 bash[58855]: cluster 2023-09-23T19:27:51.849270+0000 mgr.y (mgr.25325) 596 : cluster 0 pgmap v496: 73 pgs: 2 active+undersized+degraded, 8 peering, 1 stale+active+clean, 18 active+undersized, 44 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-09-23T19:27:53.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:52 smithi092 bash[58855]: audit 2023-09-23T19:27:52.268030+0000 mon.a (mon.0) 810 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:53.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:52 smithi092 bash[58855]: audit 2023-09-23T19:27:52.276293+0000 mon.a (mon.0) 811 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:53.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:52 smithi092 bash[58855]: audit 2023-09-23T19:27:52.318036+0000 mon.a (mon.0) 812 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:27:53.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:53 smithi016 bash[75004]: cluster 2023-09-23T19:27:52.613956+0000 mon.a (mon.0) 813 : cluster 3 Health check failed: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T19:27:53.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:53 smithi016 bash[75004]: audit 2023-09-23T19:27:52.696363+0000 mon.a (mon.0) 814 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:53.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:53 smithi016 bash[75004]: audit 2023-09-23T19:27:52.702076+0000 mon.a (mon.0) 815 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:53.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:53 smithi016 bash[75004]: audit 2023-09-23T19:27:52.708112+0000 mon.a (mon.0) 816 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:53.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:53 smithi016 bash[75004]: audit 2023-09-23T19:27:52.713111+0000 mon.a (mon.0) 817 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:53.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:53 smithi016 bash[77995]: cluster 2023-09-23T19:27:52.613956+0000 mon.a (mon.0) 813 : cluster 3 Health check failed: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T19:27:53.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:53 smithi016 bash[77995]: audit 2023-09-23T19:27:52.696363+0000 mon.a (mon.0) 814 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:53.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:53 smithi016 bash[77995]: audit 2023-09-23T19:27:52.702076+0000 mon.a (mon.0) 815 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:53.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:53 smithi016 bash[77995]: audit 2023-09-23T19:27:52.708112+0000 mon.a (mon.0) 816 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:53.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:53 smithi016 bash[77995]: audit 2023-09-23T19:27:52.713111+0000 mon.a (mon.0) 817 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:54.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:53 smithi092 bash[58855]: cluster 2023-09-23T19:27:52.613956+0000 mon.a (mon.0) 813 : cluster 3 Health check failed: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T19:27:54.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:53 smithi092 bash[58855]: audit 2023-09-23T19:27:52.696363+0000 mon.a (mon.0) 814 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:54.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:53 smithi092 bash[58855]: audit 2023-09-23T19:27:52.702076+0000 mon.a (mon.0) 815 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:54.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:53 smithi092 bash[58855]: audit 2023-09-23T19:27:52.708112+0000 mon.a (mon.0) 816 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:54.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:53 smithi092 bash[58855]: audit 2023-09-23T19:27:52.713111+0000 mon.a (mon.0) 817 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:27:54.284 INFO:journalctl@ceph.osd.3.smithi016.stdout:Sep 23 19:27:54 smithi016 bash[100553]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T19:27:54.284 INFO:journalctl@ceph.osd.3.smithi016.stdout:Sep 23 19:27:54 smithi016 bash[100553]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-09-23T19:27:54.284 INFO:journalctl@ceph.osd.3.smithi016.stdout:Sep 23 19:27:54 smithi016 bash[100553]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2023-09-23T19:27:54.285 INFO:journalctl@ceph.osd.3.smithi016.stdout:Sep 23 19:27:54 smithi016 bash[100553]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2023-09-23T19:27:54.285 INFO:journalctl@ceph.osd.3.smithi016.stdout:Sep 23 19:27:54 smithi016 bash[100553]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2023-09-23T19:27:54.285 INFO:journalctl@ceph.osd.3.smithi016.stdout:Sep 23 19:27:54 smithi016 bash[100553]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2023-09-23T19:27:54.285 INFO:journalctl@ceph.osd.3.smithi016.stdout:Sep 23 19:27:54 smithi016 bash[100553]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-09-23T19:27:54.285 INFO:journalctl@ceph.osd.3.smithi016.stdout:Sep 23 19:27:54 smithi016 bash[100553]: --> ceph-volume lvm activate successful for osd ID: 3 2023-09-23T19:27:55.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:54 smithi016 bash[75004]: cluster 2023-09-23T19:27:53.849992+0000 mgr.y (mgr.25325) 597 : cluster 0 pgmap v497: 73 pgs: 3 active+undersized+degraded, 1 peering, 27 active+undersized, 42 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T19:27:55.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:54 smithi016 bash[75004]: audit 2023-09-23T19:27:54.219736+0000 mon.c (mon.2) 124 : audit 0 from='client.? 172.21.15.16:0/1117166730' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:55.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:54 smithi016 bash[77995]: cluster 2023-09-23T19:27:53.849992+0000 mgr.y (mgr.25325) 597 : cluster 0 pgmap v497: 73 pgs: 3 active+undersized+degraded, 1 peering, 27 active+undersized, 42 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T19:27:55.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:54 smithi016 bash[77995]: audit 2023-09-23T19:27:54.219736+0000 mon.c (mon.2) 124 : audit 0 from='client.? 172.21.15.16:0/1117166730' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:55.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:54 smithi092 bash[58855]: cluster 2023-09-23T19:27:53.849992+0000 mgr.y (mgr.25325) 597 : cluster 0 pgmap v497: 73 pgs: 3 active+undersized+degraded, 1 peering, 27 active+undersized, 42 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T19:27:55.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:54 smithi092 bash[58855]: audit 2023-09-23T19:27:54.219736+0000 mon.c (mon.2) 124 : audit 0 from='client.? 172.21.15.16:0/1117166730' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:55.900 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:27:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:27:55] "GET /metrics HTTP/1.1" 200 33817 "" "Prometheus/2.33.4" 2023-09-23T19:27:56.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:55 smithi016 bash[77995]: cluster 2023-09-23T19:27:55.851262+0000 mgr.y (mgr.25325) 598 : cluster 0 pgmap v498: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T19:27:56.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:55 smithi016 bash[77995]: cluster 2023-09-23T19:27:55.902828+0000 mon.a (mon.0) 818 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T19:27:56.285 INFO:journalctl@ceph.osd.3.smithi016.stdout:Sep 23 19:27:55 smithi016 bash[100899]: debug 2023-09-23T19:27:55.968+0000 7f3b96005700 1 -- 172.21.15.16:0/2048612931 <== mon.2 v2:172.21.15.16:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55f3b60f24e0 con 0x55f3b614c800 2023-09-23T19:27:56.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:55 smithi016 bash[75004]: cluster 2023-09-23T19:27:55.851262+0000 mgr.y (mgr.25325) 598 : cluster 0 pgmap v498: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T19:27:56.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:55 smithi016 bash[75004]: cluster 2023-09-23T19:27:55.902828+0000 mon.a (mon.0) 818 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T19:27:56.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:55 smithi092 bash[58855]: cluster 2023-09-23T19:27:55.851262+0000 mgr.y (mgr.25325) 598 : cluster 0 pgmap v498: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T19:27:56.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:55 smithi092 bash[58855]: cluster 2023-09-23T19:27:55.902828+0000 mon.a (mon.0) 818 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T19:27:56.784 INFO:journalctl@ceph.osd.3.smithi016.stdout:Sep 23 19:27:56 smithi016 bash[100899]: debug 2023-09-23T19:27:56.508+0000 7f3b9f18a640 -1 Falling back to public interface 2023-09-23T19:27:57.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:56 smithi016 bash[75004]: audit 2023-09-23T19:27:56.593746+0000 mon.c (mon.2) 125 : audit 0 from='client.? 172.21.15.16:0/1804249930' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:57.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:56 smithi016 bash[77995]: audit 2023-09-23T19:27:56.593746+0000 mon.c (mon.2) 125 : audit 0 from='client.? 172.21.15.16:0/1804249930' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:57.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:56 smithi092 bash[58855]: audit 2023-09-23T19:27:56.593746+0000 mon.c (mon.2) 125 : audit 0 from='client.? 172.21.15.16:0/1804249930' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:27:58.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:57 smithi016 bash[75004]: cluster 2023-09-23T19:27:57.852120+0000 mgr.y (mgr.25325) 599 : cluster 0 pgmap v499: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T19:27:58.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:57 smithi016 bash[77995]: cluster 2023-09-23T19:27:57.852120+0000 mgr.y (mgr.25325) 599 : cluster 0 pgmap v499: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T19:27:58.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:57 smithi092 bash[58855]: cluster 2023-09-23T19:27:57.852120+0000 mgr.y (mgr.25325) 599 : cluster 0 pgmap v499: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T19:28:00.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:59 smithi016 bash[75004]: audit 2023-09-23T19:27:58.636410+0000 mon.a (mon.0) 819 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:28:00.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:59 smithi016 bash[75004]: audit 2023-09-23T19:27:58.638213+0000 mon.a (mon.0) 820 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:28:00.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:59 smithi016 bash[75004]: audit 2023-09-23T19:27:58.911544+0000 mon.a (mon.0) 821 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:28:00.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:59 smithi016 bash[75004]: audit 2023-09-23T19:27:58.919497+0000 mon.a (mon.0) 822 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:28:00.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:59 smithi016 bash[75004]: audit 2023-09-23T19:27:58.946277+0000 mon.a (mon.0) 823 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:28:00.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:59 smithi016 bash[75004]: audit 2023-09-23T19:27:58.954283+0000 mon.a (mon.0) 824 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:28:00.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:59 smithi016 bash[75004]: audit 2023-09-23T19:27:59.014488+0000 mon.a (mon.0) 825 : audit 0 from='client.? 172.21.15.16:0/1519429999' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:28:00.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:27:59 smithi016 bash[75004]: cluster 2023-09-23T19:27:59.214478+0000 mon.a (mon.0) 826 : cluster 3 Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T19:28:00.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:59 smithi016 bash[77995]: audit 2023-09-23T19:27:58.636410+0000 mon.a (mon.0) 819 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:28:00.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:59 smithi016 bash[77995]: audit 2023-09-23T19:27:58.638213+0000 mon.a (mon.0) 820 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:28:00.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:59 smithi016 bash[77995]: audit 2023-09-23T19:27:58.911544+0000 mon.a (mon.0) 821 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:28:00.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:59 smithi016 bash[77995]: audit 2023-09-23T19:27:58.919497+0000 mon.a (mon.0) 822 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:28:00.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:59 smithi016 bash[77995]: audit 2023-09-23T19:27:58.946277+0000 mon.a (mon.0) 823 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:28:00.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:59 smithi016 bash[77995]: audit 2023-09-23T19:27:58.954283+0000 mon.a (mon.0) 824 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:28:00.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:59 smithi016 bash[77995]: audit 2023-09-23T19:27:59.014488+0000 mon.a (mon.0) 825 : audit 0 from='client.? 172.21.15.16:0/1519429999' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:28:00.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:27:59 smithi016 bash[77995]: cluster 2023-09-23T19:27:59.214478+0000 mon.a (mon.0) 826 : cluster 3 Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T19:28:00.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:59 smithi092 bash[58855]: audit 2023-09-23T19:27:58.636410+0000 mon.a (mon.0) 819 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:28:00.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:59 smithi092 bash[58855]: audit 2023-09-23T19:27:58.638213+0000 mon.a (mon.0) 820 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:28:00.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:59 smithi092 bash[58855]: audit 2023-09-23T19:27:58.911544+0000 mon.a (mon.0) 821 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:28:00.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:59 smithi092 bash[58855]: audit 2023-09-23T19:27:58.919497+0000 mon.a (mon.0) 822 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:28:00.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:59 smithi092 bash[58855]: audit 2023-09-23T19:27:58.946277+0000 mon.a (mon.0) 823 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:28:00.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:59 smithi092 bash[58855]: audit 2023-09-23T19:27:58.954283+0000 mon.a (mon.0) 824 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:28:00.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:59 smithi092 bash[58855]: audit 2023-09-23T19:27:59.014488+0000 mon.a (mon.0) 825 : audit 0 from='client.? 172.21.15.16:0/1519429999' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:28:00.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:27:59 smithi092 bash[58855]: cluster 2023-09-23T19:27:59.214478+0000 mon.a (mon.0) 826 : cluster 3 Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T19:28:01.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:00 smithi016 bash[75004]: audit 2023-09-23T19:27:59.824506+0000 mon.a (mon.0) 827 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:28:01.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:00 smithi016 bash[75004]: audit 2023-09-23T19:27:59.831295+0000 mon.a (mon.0) 828 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:28:01.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:00 smithi016 bash[75004]: cluster 2023-09-23T19:27:59.852915+0000 mgr.y (mgr.25325) 600 : cluster 0 pgmap v500: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T19:28:01.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:00 smithi016 bash[77995]: audit 2023-09-23T19:27:59.824506+0000 mon.a (mon.0) 827 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:28:01.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:00 smithi016 bash[77995]: audit 2023-09-23T19:27:59.831295+0000 mon.a (mon.0) 828 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:28:01.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:00 smithi016 bash[77995]: cluster 2023-09-23T19:27:59.852915+0000 mgr.y (mgr.25325) 600 : cluster 0 pgmap v500: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T19:28:01.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:00 smithi092 bash[58855]: audit 2023-09-23T19:27:59.824506+0000 mon.a (mon.0) 827 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:28:01.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:00 smithi092 bash[58855]: audit 2023-09-23T19:27:59.831295+0000 mon.a (mon.0) 828 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:28:01.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:00 smithi092 bash[58855]: cluster 2023-09-23T19:27:59.852915+0000 mgr.y (mgr.25325) 600 : cluster 0 pgmap v500: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T19:28:02.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:01 smithi016 bash[75004]: audit 2023-09-23T19:28:01.380772+0000 mon.a (mon.0) 829 : audit 0 from='client.? 172.21.15.16:0/2928594639' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:28:02.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:01 smithi016 bash[77995]: audit 2023-09-23T19:28:01.380772+0000 mon.a (mon.0) 829 : audit 0 from='client.? 172.21.15.16:0/2928594639' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:28:02.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:01 smithi092 bash[58855]: audit 2023-09-23T19:28:01.380772+0000 mon.a (mon.0) 829 : audit 0 from='client.? 172.21.15.16:0/2928594639' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:28:03.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:02 smithi016 bash[75004]: cluster 2023-09-23T19:28:01.853884+0000 mgr.y (mgr.25325) 601 : cluster 0 pgmap v501: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T19:28:03.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:02 smithi016 bash[77995]: cluster 2023-09-23T19:28:01.853884+0000 mgr.y (mgr.25325) 601 : cluster 0 pgmap v501: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T19:28:03.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:02 smithi092 bash[58855]: cluster 2023-09-23T19:28:01.853884+0000 mgr.y (mgr.25325) 601 : cluster 0 pgmap v501: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T19:28:04.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:03 smithi016 bash[75004]: audit 2023-09-23T19:28:03.743324+0000 mon.a (mon.0) 830 : audit 0 from='client.? 172.21.15.16:0/1934199756' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:28:04.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:03 smithi016 bash[77995]: audit 2023-09-23T19:28:03.743324+0000 mon.a (mon.0) 830 : audit 0 from='client.? 172.21.15.16:0/1934199756' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:28:04.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:03 smithi092 bash[58855]: audit 2023-09-23T19:28:03.743324+0000 mon.a (mon.0) 830 : audit 0 from='client.? 172.21.15.16:0/1934199756' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:28:05.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:04 smithi016 bash[75004]: cluster 2023-09-23T19:28:03.854460+0000 mgr.y (mgr.25325) 602 : cluster 0 pgmap v502: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T19:28:05.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:04 smithi016 bash[77995]: cluster 2023-09-23T19:28:03.854460+0000 mgr.y (mgr.25325) 602 : cluster 0 pgmap v502: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T19:28:05.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:04 smithi092 bash[58855]: cluster 2023-09-23T19:28:03.854460+0000 mgr.y (mgr.25325) 602 : cluster 0 pgmap v502: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T19:28:05.852 INFO:journalctl@ceph.osd.3.smithi016.stdout:Sep 23 19:28:05 smithi016 bash[100899]: debug 2023-09-23T19:28:05.535+0000 7f3b9f18a640 -1 osd.3 112 log_to_monitors true 2023-09-23T19:28:05.853 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:28:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:28:05] "GET /metrics HTTP/1.1" 200 33821 "" "Prometheus/2.33.4" 2023-09-23T19:28:06.189 INFO:journalctl@ceph.osd.3.smithi016.stdout:Sep 23 19:28:05 smithi016 bash[100899]: debug 2023-09-23T19:28:05.903+0000 7f3b905f7700 -1 osd.3 112 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T19:28:06.190 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:05 smithi016 bash[75004]: audit 2023-09-23T19:28:05.541127+0000 mon.a (mon.0) 831 : audit 1 from='osd.3 [v2:172.21.15.16:6826/3155744043,v1:172.21.15.16:6827/3155744043]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T19:28:06.190 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:05 smithi016 bash[77995]: audit 2023-09-23T19:28:05.541127+0000 mon.a (mon.0) 831 : audit 1 from='osd.3 [v2:172.21.15.16:6826/3155744043,v1:172.21.15.16:6827/3155744043]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T19:28:06.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:05 smithi092 bash[58855]: audit 2023-09-23T19:28:05.541127+0000 mon.a (mon.0) 831 : audit 1 from='osd.3 [v2:172.21.15.16:6826/3155744043,v1:172.21.15.16:6827/3155744043]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T19:28:07.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:06 smithi016 bash[75004]: cluster 2023-09-23T19:28:05.855636+0000 mgr.y (mgr.25325) 603 : cluster 0 pgmap v503: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T19:28:07.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:06 smithi016 bash[75004]: audit 2023-09-23T19:28:05.860877+0000 mon.a (mon.0) 832 : audit 1 from='osd.3 [v2:172.21.15.16:6826/3155744043,v1:172.21.15.16:6827/3155744043]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T19:28:07.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:06 smithi016 bash[75004]: cluster 2023-09-23T19:28:05.860948+0000 mon.a (mon.0) 833 : cluster 0 osdmap e115: 8 total, 7 up, 8 in 2023-09-23T19:28:07.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:06 smithi016 bash[75004]: audit 2023-09-23T19:28:05.861835+0000 mon.a (mon.0) 834 : audit 1 from='osd.3 [v2:172.21.15.16:6826/3155744043,v1:172.21.15.16:6827/3155744043]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-09-23T19:28:07.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:06 smithi016 bash[75004]: audit 2023-09-23T19:28:06.104822+0000 mon.c (mon.2) 126 : audit 0 from='client.? 172.21.15.16:0/1775561850' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:28:07.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:06 smithi016 bash[77995]: cluster 2023-09-23T19:28:05.855636+0000 mgr.y (mgr.25325) 603 : cluster 0 pgmap v503: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T19:28:07.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:06 smithi016 bash[77995]: audit 2023-09-23T19:28:05.860877+0000 mon.a (mon.0) 832 : audit 1 from='osd.3 [v2:172.21.15.16:6826/3155744043,v1:172.21.15.16:6827/3155744043]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T19:28:07.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:06 smithi016 bash[77995]: cluster 2023-09-23T19:28:05.860948+0000 mon.a (mon.0) 833 : cluster 0 osdmap e115: 8 total, 7 up, 8 in 2023-09-23T19:28:07.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:06 smithi016 bash[77995]: audit 2023-09-23T19:28:05.861835+0000 mon.a (mon.0) 834 : audit 1 from='osd.3 [v2:172.21.15.16:6826/3155744043,v1:172.21.15.16:6827/3155744043]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-09-23T19:28:07.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:06 smithi016 bash[77995]: audit 2023-09-23T19:28:06.104822+0000 mon.c (mon.2) 126 : audit 0 from='client.? 172.21.15.16:0/1775561850' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:28:07.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:06 smithi092 bash[58855]: cluster 2023-09-23T19:28:05.855636+0000 mgr.y (mgr.25325) 603 : cluster 0 pgmap v503: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T19:28:07.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:06 smithi092 bash[58855]: audit 2023-09-23T19:28:05.860877+0000 mon.a (mon.0) 832 : audit 1 from='osd.3 [v2:172.21.15.16:6826/3155744043,v1:172.21.15.16:6827/3155744043]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T19:28:07.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:06 smithi092 bash[58855]: cluster 2023-09-23T19:28:05.860948+0000 mon.a (mon.0) 833 : cluster 0 osdmap e115: 8 total, 7 up, 8 in 2023-09-23T19:28:07.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:06 smithi092 bash[58855]: audit 2023-09-23T19:28:05.861835+0000 mon.a (mon.0) 834 : audit 1 from='osd.3 [v2:172.21.15.16:6826/3155744043,v1:172.21.15.16:6827/3155744043]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-09-23T19:28:07.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:06 smithi092 bash[58855]: audit 2023-09-23T19:28:06.104822+0000 mon.c (mon.2) 126 : audit 0 from='client.? 172.21.15.16:0/1775561850' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:28:08.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:07 smithi016 bash[77995]: cluster 2023-09-23T19:28:06.196927+0000 osd.6 (osd.6) 35 : cluster 3 failed to encode map e115 with expected crc 2023-09-23T19:28:08.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:07 smithi016 bash[77995]: cluster 2023-09-23T19:28:06.199702+0000 osd.6 (osd.6) 36 : cluster 3 failed to encode map e115 with expected crc 2023-09-23T19:28:08.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:07 smithi016 bash[77995]: cluster 2023-09-23T19:28:06.276604+0000 osd.4 (osd.4) 39 : cluster 3 failed to encode map e115 with expected crc 2023-09-23T19:28:08.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:07 smithi016 bash[77995]: cluster 2023-09-23T19:28:06.278922+0000 osd.4 (osd.4) 40 : cluster 3 failed to encode map e115 with expected crc 2023-09-23T19:28:08.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:07 smithi016 bash[77995]: cluster 2023-09-23T19:28:06.858478+0000 mon.a (mon.0) 835 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T19:28:08.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:07 smithi016 bash[77995]: cluster 2023-09-23T19:28:06.865474+0000 mon.a (mon.0) 836 : cluster 1 osd.3 [v2:172.21.15.16:6826/3155744043,v1:172.21.15.16:6827/3155744043] boot 2023-09-23T19:28:08.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:07 smithi016 bash[77995]: cluster 2023-09-23T19:28:06.865602+0000 mon.a (mon.0) 837 : cluster 0 osdmap e116: 8 total, 8 up, 8 in 2023-09-23T19:28:08.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:07 smithi016 bash[77995]: cluster 2023-09-23T19:28:06.874363+0000 osd.5 (osd.5) 39 : cluster 3 failed to encode map e116 with expected crc 2023-09-23T19:28:08.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:07 smithi016 bash[77995]: audit 2023-09-23T19:28:06.874768+0000 mon.a (mon.0) 838 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:28:08.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:07 smithi016 bash[77995]: cluster 2023-09-23T19:28:06.876081+0000 osd.7 (osd.7) 33 : cluster 3 failed to encode map e116 with expected crc 2023-09-23T19:28:08.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:07 smithi016 bash[77995]: cluster 2023-09-23T19:28:07.868648+0000 mon.a (mon.0) 839 : cluster 0 osdmap e117: 8 total, 8 up, 8 in 2023-09-23T19:28:08.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:07 smithi016 bash[75004]: cluster 2023-09-23T19:28:06.196927+0000 osd.6 (osd.6) 35 : cluster 3 failed to encode map e115 with expected crc 2023-09-23T19:28:08.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:07 smithi016 bash[75004]: cluster 2023-09-23T19:28:06.199702+0000 osd.6 (osd.6) 36 : cluster 3 failed to encode map e115 with expected crc 2023-09-23T19:28:08.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:07 smithi016 bash[75004]: cluster 2023-09-23T19:28:06.276604+0000 osd.4 (osd.4) 39 : cluster 3 failed to encode map e115 with expected crc 2023-09-23T19:28:08.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:07 smithi016 bash[75004]: cluster 2023-09-23T19:28:06.278922+0000 osd.4 (osd.4) 40 : cluster 3 failed to encode map e115 with expected crc 2023-09-23T19:28:08.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:07 smithi016 bash[75004]: cluster 2023-09-23T19:28:06.858478+0000 mon.a (mon.0) 835 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T19:28:08.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:07 smithi016 bash[75004]: cluster 2023-09-23T19:28:06.865474+0000 mon.a (mon.0) 836 : cluster 1 osd.3 [v2:172.21.15.16:6826/3155744043,v1:172.21.15.16:6827/3155744043] boot 2023-09-23T19:28:08.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:07 smithi016 bash[75004]: cluster 2023-09-23T19:28:06.865602+0000 mon.a (mon.0) 837 : cluster 0 osdmap e116: 8 total, 8 up, 8 in 2023-09-23T19:28:08.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:07 smithi016 bash[75004]: cluster 2023-09-23T19:28:06.874363+0000 osd.5 (osd.5) 39 : cluster 3 failed to encode map e116 with expected crc 2023-09-23T19:28:08.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:07 smithi016 bash[75004]: audit 2023-09-23T19:28:06.874768+0000 mon.a (mon.0) 838 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:28:08.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:07 smithi016 bash[75004]: cluster 2023-09-23T19:28:06.876081+0000 osd.7 (osd.7) 33 : cluster 3 failed to encode map e116 with expected crc 2023-09-23T19:28:08.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:07 smithi016 bash[75004]: cluster 2023-09-23T19:28:07.868648+0000 mon.a (mon.0) 839 : cluster 0 osdmap e117: 8 total, 8 up, 8 in 2023-09-23T19:28:08.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:07 smithi092 bash[58855]: cluster 2023-09-23T19:28:06.196927+0000 osd.6 (osd.6) 35 : cluster 3 failed to encode map e115 with expected crc 2023-09-23T19:28:08.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:07 smithi092 bash[58855]: cluster 2023-09-23T19:28:06.199702+0000 osd.6 (osd.6) 36 : cluster 3 failed to encode map e115 with expected crc 2023-09-23T19:28:08.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:07 smithi092 bash[58855]: cluster 2023-09-23T19:28:06.276604+0000 osd.4 (osd.4) 39 : cluster 3 failed to encode map e115 with expected crc 2023-09-23T19:28:08.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:07 smithi092 bash[58855]: cluster 2023-09-23T19:28:06.278922+0000 osd.4 (osd.4) 40 : cluster 3 failed to encode map e115 with expected crc 2023-09-23T19:28:08.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:07 smithi092 bash[58855]: cluster 2023-09-23T19:28:06.858478+0000 mon.a (mon.0) 835 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T19:28:08.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:07 smithi092 bash[58855]: cluster 2023-09-23T19:28:06.865474+0000 mon.a (mon.0) 836 : cluster 1 osd.3 [v2:172.21.15.16:6826/3155744043,v1:172.21.15.16:6827/3155744043] boot 2023-09-23T19:28:08.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:07 smithi092 bash[58855]: cluster 2023-09-23T19:28:06.865602+0000 mon.a (mon.0) 837 : cluster 0 osdmap e116: 8 total, 8 up, 8 in 2023-09-23T19:28:08.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:07 smithi092 bash[58855]: cluster 2023-09-23T19:28:06.874363+0000 osd.5 (osd.5) 39 : cluster 3 failed to encode map e116 with expected crc 2023-09-23T19:28:08.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:07 smithi092 bash[58855]: audit 2023-09-23T19:28:06.874768+0000 mon.a (mon.0) 838 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:28:08.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:07 smithi092 bash[58855]: cluster 2023-09-23T19:28:06.876081+0000 osd.7 (osd.7) 33 : cluster 3 failed to encode map e116 with expected crc 2023-09-23T19:28:08.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:07 smithi092 bash[58855]: cluster 2023-09-23T19:28:07.868648+0000 mon.a (mon.0) 839 : cluster 0 osdmap e117: 8 total, 8 up, 8 in 2023-09-23T19:28:08.525 INFO:teuthology.orchestra.run.smithi016.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 4 2023-09-23T19:28:09.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:08 smithi016 bash[75004]: cluster 2023-09-23T19:28:07.856515+0000 mgr.y (mgr.25325) 604 : cluster 0 pgmap v506: 73 pgs: 2 peering, 3 active+undersized+degraded, 26 active+undersized, 42 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T19:28:09.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:08 smithi016 bash[75004]: cluster 2023-09-23T19:28:07.867068+0000 osd.5 (osd.5) 40 : cluster 3 failed to encode map e117 with expected crc 2023-09-23T19:28:09.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:08 smithi016 bash[75004]: cluster 2023-09-23T19:28:07.873323+0000 osd.4 (osd.4) 41 : cluster 3 failed to encode map e117 with expected crc 2023-09-23T19:28:09.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:08 smithi016 bash[75004]: cluster 2023-09-23T19:28:07.873748+0000 osd.5 (osd.5) 41 : cluster 3 failed to encode map e117 with expected crc 2023-09-23T19:28:09.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:08 smithi016 bash[75004]: cluster 2023-09-23T19:28:07.875189+0000 osd.7 (osd.7) 34 : cluster 3 failed to encode map e117 with expected crc 2023-09-23T19:28:09.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:08 smithi016 bash[75004]: cluster 2023-09-23T19:28:07.875330+0000 osd.4 (osd.4) 42 : cluster 3 failed to encode map e117 with expected crc 2023-09-23T19:28:09.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:08 smithi016 bash[75004]: cluster 2023-09-23T19:28:07.876037+0000 osd.5 (osd.5) 42 : cluster 3 failed to encode map e117 with expected crc 2023-09-23T19:28:09.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:08 smithi016 bash[75004]: cluster 2023-09-23T19:28:07.877384+0000 osd.4 (osd.4) 43 : cluster 3 failed to encode map e117 with expected crc 2023-09-23T19:28:09.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:08 smithi016 bash[75004]: cluster 2023-09-23T19:28:07.878968+0000 osd.5 (osd.5) 43 : cluster 3 failed to encode map e117 with expected crc 2023-09-23T19:28:09.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:08 smithi016 bash[75004]: cluster 2023-09-23T19:28:07.880266+0000 osd.4 (osd.4) 44 : cluster 3 failed to encode map e117 with expected crc 2023-09-23T19:28:09.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:08 smithi016 bash[75004]: cluster 2023-09-23T19:28:07.882414+0000 osd.4 (osd.4) 45 : cluster 3 failed to encode map e117 with expected crc 2023-09-23T19:28:09.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:08 smithi016 bash[75004]: cluster 2023-09-23T19:28:07.883740+0000 osd.6 (osd.6) 37 : cluster 3 failed to encode map e117 with expected crc 2023-09-23T19:28:09.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:08 smithi016 bash[75004]: audit 2023-09-23T19:28:08.506538+0000 mon.a (mon.0) 840 : audit 0 from='client.? 172.21.15.16:0/824321434' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:28:09.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:08 smithi016 bash[77995]: cluster 2023-09-23T19:28:07.856515+0000 mgr.y (mgr.25325) 604 : cluster 0 pgmap v506: 73 pgs: 2 peering, 3 active+undersized+degraded, 26 active+undersized, 42 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T19:28:09.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:08 smithi016 bash[77995]: cluster 2023-09-23T19:28:07.867068+0000 osd.5 (osd.5) 40 : cluster 3 failed to encode map e117 with expected crc 2023-09-23T19:28:09.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:08 smithi016 bash[77995]: cluster 2023-09-23T19:28:07.873323+0000 osd.4 (osd.4) 41 : cluster 3 failed to encode map e117 with expected crc 2023-09-23T19:28:09.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:08 smithi016 bash[77995]: cluster 2023-09-23T19:28:07.873748+0000 osd.5 (osd.5) 41 : cluster 3 failed to encode map e117 with expected crc 2023-09-23T19:28:09.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:08 smithi016 bash[77995]: cluster 2023-09-23T19:28:07.875189+0000 osd.7 (osd.7) 34 : cluster 3 failed to encode map e117 with expected crc 2023-09-23T19:28:09.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:08 smithi016 bash[77995]: cluster 2023-09-23T19:28:07.875330+0000 osd.4 (osd.4) 42 : cluster 3 failed to encode map e117 with expected crc 2023-09-23T19:28:09.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:08 smithi016 bash[77995]: cluster 2023-09-23T19:28:07.876037+0000 osd.5 (osd.5) 42 : cluster 3 failed to encode map e117 with expected crc 2023-09-23T19:28:09.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:08 smithi016 bash[77995]: cluster 2023-09-23T19:28:07.877384+0000 osd.4 (osd.4) 43 : cluster 3 failed to encode map e117 with expected crc 2023-09-23T19:28:09.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:08 smithi016 bash[77995]: cluster 2023-09-23T19:28:07.878968+0000 osd.5 (osd.5) 43 : cluster 3 failed to encode map e117 with expected crc 2023-09-23T19:28:09.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:08 smithi016 bash[77995]: cluster 2023-09-23T19:28:07.880266+0000 osd.4 (osd.4) 44 : cluster 3 failed to encode map e117 with expected crc 2023-09-23T19:28:09.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:08 smithi016 bash[77995]: cluster 2023-09-23T19:28:07.882414+0000 osd.4 (osd.4) 45 : cluster 3 failed to encode map e117 with expected crc 2023-09-23T19:28:09.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:08 smithi016 bash[77995]: cluster 2023-09-23T19:28:07.883740+0000 osd.6 (osd.6) 37 : cluster 3 failed to encode map e117 with expected crc 2023-09-23T19:28:09.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:08 smithi016 bash[77995]: audit 2023-09-23T19:28:08.506538+0000 mon.a (mon.0) 840 : audit 0 from='client.? 172.21.15.16:0/824321434' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:28:09.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:08 smithi092 bash[58855]: cluster 2023-09-23T19:28:07.856515+0000 mgr.y (mgr.25325) 604 : cluster 0 pgmap v506: 73 pgs: 2 peering, 3 active+undersized+degraded, 26 active+undersized, 42 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T19:28:09.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:08 smithi092 bash[58855]: cluster 2023-09-23T19:28:07.867068+0000 osd.5 (osd.5) 40 : cluster 3 failed to encode map e117 with expected crc 2023-09-23T19:28:09.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:08 smithi092 bash[58855]: cluster 2023-09-23T19:28:07.873323+0000 osd.4 (osd.4) 41 : cluster 3 failed to encode map e117 with expected crc 2023-09-23T19:28:09.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:08 smithi092 bash[58855]: cluster 2023-09-23T19:28:07.873748+0000 osd.5 (osd.5) 41 : cluster 3 failed to encode map e117 with expected crc 2023-09-23T19:28:09.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:08 smithi092 bash[58855]: cluster 2023-09-23T19:28:07.875189+0000 osd.7 (osd.7) 34 : cluster 3 failed to encode map e117 with expected crc 2023-09-23T19:28:09.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:08 smithi092 bash[58855]: cluster 2023-09-23T19:28:07.875330+0000 osd.4 (osd.4) 42 : cluster 3 failed to encode map e117 with expected crc 2023-09-23T19:28:09.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:08 smithi092 bash[58855]: cluster 2023-09-23T19:28:07.876037+0000 osd.5 (osd.5) 42 : cluster 3 failed to encode map e117 with expected crc 2023-09-23T19:28:09.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:08 smithi092 bash[58855]: cluster 2023-09-23T19:28:07.877384+0000 osd.4 (osd.4) 43 : cluster 3 failed to encode map e117 with expected crc 2023-09-23T19:28:09.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:08 smithi092 bash[58855]: cluster 2023-09-23T19:28:07.878968+0000 osd.5 (osd.5) 43 : cluster 3 failed to encode map e117 with expected crc 2023-09-23T19:28:09.291 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:08 smithi092 bash[58855]: cluster 2023-09-23T19:28:07.880266+0000 osd.4 (osd.4) 44 : cluster 3 failed to encode map e117 with expected crc 2023-09-23T19:28:09.291 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:08 smithi092 bash[58855]: cluster 2023-09-23T19:28:07.882414+0000 osd.4 (osd.4) 45 : cluster 3 failed to encode map e117 with expected crc 2023-09-23T19:28:09.291 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:08 smithi092 bash[58855]: cluster 2023-09-23T19:28:07.883740+0000 osd.6 (osd.6) 37 : cluster 3 failed to encode map e117 with expected crc 2023-09-23T19:28:09.291 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:08 smithi092 bash[58855]: audit 2023-09-23T19:28:08.506538+0000 mon.a (mon.0) 840 : audit 0 from='client.? 172.21.15.16:0/824321434' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:28:10.016 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade pause' 2023-09-23T19:28:10.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:10 smithi016 bash[77995]: cluster 2023-09-23T19:28:09.857420+0000 mgr.y (mgr.25325) 605 : cluster 0 pgmap v508: 73 pgs: 2 peering, 3 active+undersized+degraded, 21 active+undersized, 47 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T19:28:10.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:10 smithi016 bash[75004]: cluster 2023-09-23T19:28:09.857420+0000 mgr.y (mgr.25325) 605 : cluster 0 pgmap v508: 73 pgs: 2 peering, 3 active+undersized+degraded, 21 active+undersized, 47 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T19:28:10.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:10 smithi092 bash[58855]: cluster 2023-09-23T19:28:09.857420+0000 mgr.y (mgr.25325) 605 : cluster 0 pgmap v508: 73 pgs: 2 peering, 3 active+undersized+degraded, 21 active+undersized, 47 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T19:28:10.784 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:28:10 smithi016 bash[70992]: debug 2023-09-23T19:28:10.391+0000 7fb2a5a7c700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2023-09-23T19:28:11.533 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:11 smithi016 bash[77995]: audit 2023-09-23T19:28:10.281257+0000 mon.a (mon.0) 841 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:28:11.533 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:11 smithi016 bash[77995]: audit 2023-09-23T19:28:10.288263+0000 mon.a (mon.0) 842 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:28:11.533 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:11 smithi016 bash[77995]: audit 2023-09-23T19:28:10.289619+0000 mon.a (mon.0) 843 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:28:11.533 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:11 smithi016 bash[77995]: audit 2023-09-23T19:28:10.290765+0000 mon.a (mon.0) 844 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:28:11.533 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:11 smithi016 bash[77995]: audit 2023-09-23T19:28:10.297916+0000 mon.a (mon.0) 845 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:28:11.533 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:11 smithi016 bash[77995]: audit 2023-09-23T19:28:10.378679+0000 mon.a (mon.0) 846 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:28:11.533 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:11 smithi016 bash[77995]: audit 2023-09-23T19:28:10.384802+0000 mon.a (mon.0) 847 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:28:11.533 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:11 smithi016 bash[77995]: audit 2023-09-23T19:28:10.388408+0000 mon.a (mon.0) 848 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:28:11.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:11 smithi016 bash[77995]: audit 2023-09-23T19:28:10.390322+0000 mon.a (mon.0) 849 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:28:11.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:11 smithi016 bash[77995]: audit 2023-09-23T19:28:10.392519+0000 mon.a (mon.0) 850 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T19:28:11.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:11 smithi016 bash[77995]: audit 2023-09-23T19:28:10.392793+0000 mgr.y (mgr.25325) 606 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T19:28:11.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:11 smithi016 bash[77995]: cephadm 2023-09-23T19:28:10.393802+0000 mgr.y (mgr.25325) 607 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2023-09-23T19:28:11.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:11 smithi016 bash[75004]: audit 2023-09-23T19:28:10.281257+0000 mon.a (mon.0) 841 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:28:11.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:11 smithi016 bash[75004]: audit 2023-09-23T19:28:10.288263+0000 mon.a (mon.0) 842 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:28:11.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:11 smithi016 bash[75004]: audit 2023-09-23T19:28:10.289619+0000 mon.a (mon.0) 843 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:28:11.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:11 smithi016 bash[75004]: audit 2023-09-23T19:28:10.290765+0000 mon.a (mon.0) 844 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:28:11.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:11 smithi016 bash[75004]: audit 2023-09-23T19:28:10.297916+0000 mon.a (mon.0) 845 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:28:11.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:11 smithi016 bash[75004]: audit 2023-09-23T19:28:10.378679+0000 mon.a (mon.0) 846 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:28:11.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:11 smithi016 bash[75004]: audit 2023-09-23T19:28:10.384802+0000 mon.a (mon.0) 847 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:28:11.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:11 smithi016 bash[75004]: audit 2023-09-23T19:28:10.388408+0000 mon.a (mon.0) 848 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:28:11.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:11 smithi016 bash[75004]: audit 2023-09-23T19:28:10.390322+0000 mon.a (mon.0) 849 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:28:11.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:11 smithi016 bash[75004]: audit 2023-09-23T19:28:10.392519+0000 mon.a (mon.0) 850 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T19:28:11.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:11 smithi016 bash[75004]: audit 2023-09-23T19:28:10.392793+0000 mgr.y (mgr.25325) 606 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T19:28:11.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:11 smithi016 bash[75004]: cephadm 2023-09-23T19:28:10.393802+0000 mgr.y (mgr.25325) 607 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2023-09-23T19:28:11.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:11 smithi092 bash[58855]: audit 2023-09-23T19:28:10.281257+0000 mon.a (mon.0) 841 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:28:11.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:11 smithi092 bash[58855]: audit 2023-09-23T19:28:10.288263+0000 mon.a (mon.0) 842 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:28:11.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:11 smithi092 bash[58855]: audit 2023-09-23T19:28:10.289619+0000 mon.a (mon.0) 843 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:28:11.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:11 smithi092 bash[58855]: audit 2023-09-23T19:28:10.290765+0000 mon.a (mon.0) 844 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:28:11.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:11 smithi092 bash[58855]: audit 2023-09-23T19:28:10.297916+0000 mon.a (mon.0) 845 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:28:11.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:11 smithi092 bash[58855]: audit 2023-09-23T19:28:10.378679+0000 mon.a (mon.0) 846 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:28:11.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:11 smithi092 bash[58855]: audit 2023-09-23T19:28:10.384802+0000 mon.a (mon.0) 847 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:28:11.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:11 smithi092 bash[58855]: audit 2023-09-23T19:28:10.388408+0000 mon.a (mon.0) 848 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:28:11.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:11 smithi092 bash[58855]: audit 2023-09-23T19:28:10.390322+0000 mon.a (mon.0) 849 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:28:11.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:11 smithi092 bash[58855]: audit 2023-09-23T19:28:10.392519+0000 mon.a (mon.0) 850 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T19:28:11.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:11 smithi092 bash[58855]: audit 2023-09-23T19:28:10.392793+0000 mgr.y (mgr.25325) 606 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T19:28:11.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:11 smithi092 bash[58855]: cephadm 2023-09-23T19:28:10.393802+0000 mgr.y (mgr.25325) 607 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2023-09-23T19:28:12.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:12 smithi016 bash[77995]: cluster 2023-09-23T19:28:11.858471+0000 mgr.y (mgr.25325) 608 : cluster 0 pgmap v509: 73 pgs: 2 peering, 71 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:12.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:12 smithi016 bash[75004]: cluster 2023-09-23T19:28:11.858471+0000 mgr.y (mgr.25325) 608 : cluster 0 pgmap v509: 73 pgs: 2 peering, 71 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:12.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:12 smithi092 bash[58855]: cluster 2023-09-23T19:28:11.858471+0000 mgr.y (mgr.25325) 608 : cluster 0 pgmap v509: 73 pgs: 2 peering, 71 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:13.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:13 smithi016 bash[75004]: cluster 2023-09-23T19:28:12.279268+0000 mon.a (mon.0) 851 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded) 2023-09-23T19:28:13.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:13 smithi016 bash[75004]: cluster 2023-09-23T19:28:12.279328+0000 mon.a (mon.0) 852 : cluster 1 Cluster is now healthy 2023-09-23T19:28:13.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:13 smithi016 bash[77995]: cluster 2023-09-23T19:28:12.279268+0000 mon.a (mon.0) 851 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded) 2023-09-23T19:28:13.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:13 smithi016 bash[77995]: cluster 2023-09-23T19:28:12.279328+0000 mon.a (mon.0) 852 : cluster 1 Cluster is now healthy 2023-09-23T19:28:13.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:13 smithi092 bash[58855]: cluster 2023-09-23T19:28:12.279268+0000 mon.a (mon.0) 851 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded) 2023-09-23T19:28:13.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:13 smithi092 bash[58855]: cluster 2023-09-23T19:28:12.279328+0000 mon.a (mon.0) 852 : cluster 1 Cluster is now healthy 2023-09-23T19:28:13.849 INFO:teuthology.orchestra.run.smithi016.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:28:14.627 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T19:28:15.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:14 smithi016 bash[77995]: audit 2023-09-23T19:28:13.637145+0000 mon.a (mon.0) 853 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:28:15.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:14 smithi016 bash[77995]: audit 2023-09-23T19:28:13.638714+0000 mon.a (mon.0) 854 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:28:15.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:14 smithi016 bash[77995]: audit 2023-09-23T19:28:13.839840+0000 mgr.y (mgr.25325) 609 : audit 0 from='client.35535 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:28:15.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:14 smithi016 bash[77995]: cephadm 2023-09-23T19:28:13.840517+0000 mgr.y (mgr.25325) 610 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:28:15.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:14 smithi016 bash[77995]: audit 2023-09-23T19:28:13.847247+0000 mon.a (mon.0) 855 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:28:15.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:14 smithi016 bash[77995]: cluster 2023-09-23T19:28:13.859249+0000 mgr.y (mgr.25325) 611 : cluster 0 pgmap v510: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:15.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:14 smithi016 bash[75004]: audit 2023-09-23T19:28:13.637145+0000 mon.a (mon.0) 853 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:28:15.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:14 smithi016 bash[75004]: audit 2023-09-23T19:28:13.638714+0000 mon.a (mon.0) 854 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:28:15.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:14 smithi016 bash[75004]: audit 2023-09-23T19:28:13.839840+0000 mgr.y (mgr.25325) 609 : audit 0 from='client.35535 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:28:15.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:14 smithi016 bash[75004]: cephadm 2023-09-23T19:28:13.840517+0000 mgr.y (mgr.25325) 610 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:28:15.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:14 smithi016 bash[75004]: audit 2023-09-23T19:28:13.847247+0000 mon.a (mon.0) 855 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:28:15.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:14 smithi016 bash[75004]: cluster 2023-09-23T19:28:13.859249+0000 mgr.y (mgr.25325) 611 : cluster 0 pgmap v510: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:15.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:14 smithi092 bash[58855]: audit 2023-09-23T19:28:13.637145+0000 mon.a (mon.0) 853 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:28:15.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:14 smithi092 bash[58855]: audit 2023-09-23T19:28:13.638714+0000 mon.a (mon.0) 854 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:28:15.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:14 smithi092 bash[58855]: audit 2023-09-23T19:28:13.839840+0000 mgr.y (mgr.25325) 609 : audit 0 from='client.35535 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:28:15.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:14 smithi092 bash[58855]: cephadm 2023-09-23T19:28:13.840517+0000 mgr.y (mgr.25325) 610 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:28:15.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:14 smithi092 bash[58855]: audit 2023-09-23T19:28:13.847247+0000 mon.a (mon.0) 855 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:28:15.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:14 smithi092 bash[58855]: cluster 2023-09-23T19:28:13.859249+0000 mgr.y (mgr.25325) 611 : cluster 0 pgmap v510: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:15.936 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:28:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:28:15] "GET /metrics HTTP/1.1" 200 33812 "" "Prometheus/2.33.4" 2023-09-23T19:28:16.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:15 smithi016 bash[75004]: cluster 2023-09-23T19:28:15.859996+0000 mgr.y (mgr.25325) 612 : cluster 0 pgmap v511: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:16.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:15 smithi016 bash[77995]: cluster 2023-09-23T19:28:15.859996+0000 mgr.y (mgr.25325) 612 : cluster 0 pgmap v511: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:16.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:15 smithi092 bash[58855]: cluster 2023-09-23T19:28:15.859996+0000 mgr.y (mgr.25325) 612 : cluster 0 pgmap v511: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:18.272 INFO:teuthology.orchestra.run.smithi016.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T19:28:18.272 INFO:teuthology.orchestra.run.smithi016.stdout:alertmanager.a smithi016 *:9093,9094 running (17m) - 24m 13.7M - 0.23.0 ba2b418f427c b362c3d9bae2 2023-09-23T19:28:18.273 INFO:teuthology.orchestra.run.smithi016.stdout:grafana.a smithi092 *:3000 running (17m) - 23m 42.2M - 8.3.5 dad864ee21e9 891a61cf5fff 2023-09-23T19:28:18.273 INFO:teuthology.orchestra.run.smithi016.stdout:mds.foo.smithi016.uncssv smithi016 running (22m) - 22m 20.1M - 16.2.14-37-g2e1f1015 1e6a756174a8 95d1e187e3eb 2023-09-23T19:28:18.273 INFO:teuthology.orchestra.run.smithi016.stdout:mds.foo.smithi092.vbpqks smithi092 running (22m) - 22m 18.2M - 16.2.14-37-g2e1f1015 1e6a756174a8 1d619ec31bf1 2023-09-23T19:28:18.273 INFO:teuthology.orchestra.run.smithi016.stdout:mgr.x smithi092 *:8443,9283,8765 running (16m) - 30m 433M - 18.2.0-460-gbb0daeff e32c2a64c126 d54b4354eade 2023-09-23T19:28:18.273 INFO:teuthology.orchestra.run.smithi016.stdout:mgr.y smithi016 *:8443,9283,8765 running (17m) - 32m 488M - 18.2.0-460-gbb0daeff e32c2a64c126 19706820b434 2023-09-23T19:28:18.274 INFO:teuthology.orchestra.run.smithi016.stdout:mon.a smithi016 running (15m) - 32m 108M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 e5aad364678c 2023-09-23T19:28:18.274 INFO:teuthology.orchestra.run.smithi016.stdout:mon.b smithi092 running (12m) - 31m 51.0M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 09d45d4f5248 2023-09-23T19:28:18.274 INFO:teuthology.orchestra.run.smithi016.stdout:mon.c smithi016 running (14m) - 31m 59.3M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 61a05c2c22a7 2023-09-23T19:28:18.274 INFO:teuthology.orchestra.run.smithi016.stdout:node-exporter.a smithi016 *:9100 running (18m) - 24m 10.1M - 1.3.1 1dbe0e931976 7f13d7a8905d 2023-09-23T19:28:18.274 INFO:teuthology.orchestra.run.smithi016.stdout:node-exporter.b smithi092 *:9100 running (17m) - 24m 9364k - 1.3.1 1dbe0e931976 4823347a98b5 2023-09-23T19:28:18.274 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0 smithi016 running (10m) - 30m 728M 2525M 18.2.0-460-gbb0daeff e32c2a64c126 aa95286475c5 2023-09-23T19:28:18.274 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1 smithi016 running (5m) - 29m 520M 2525M 18.2.0-460-gbb0daeff e32c2a64c126 f4575761955d 2023-09-23T19:28:18.275 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2 smithi016 running (68s) - 28m 403M 2525M 18.2.0-460-gbb0daeff e32c2a64c126 25d0dac6c5cf 2023-09-23T19:28:18.275 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3 smithi016 running (23s) - 28m - 2525M 18.2.0-460-gbb0daeff e32c2a64c126 e78a562df218 2023-09-23T19:28:18.275 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4 smithi092 running (27m) - 27m 623M 2524M 16.2.14-37-g2e1f1015 1e6a756174a8 253dc4b11010 2023-09-23T19:28:18.275 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5 smithi092 running (26m) - 26m 945M 2524M 16.2.14-37-g2e1f1015 1e6a756174a8 0a7306534382 2023-09-23T19:28:18.275 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6 smithi092 running (26m) - 26m 827M 2524M 16.2.14-37-g2e1f1015 1e6a756174a8 d6f3ee67cc5f 2023-09-23T19:28:18.275 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7 smithi092 running (25m) - 25m 587M 2524M 16.2.14-37-g2e1f1015 1e6a756174a8 794cf2340f17 2023-09-23T19:28:18.275 INFO:teuthology.orchestra.run.smithi016.stdout:prometheus.a smithi092 *:9095 running (16m) - 24m 62.5M - 2.33.4 514e6a882f6e 2666eba9f715 2023-09-23T19:28:18.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:17 smithi016 bash[75004]: cluster 2023-09-23T19:28:17.860804+0000 mgr.y (mgr.25325) 613 : cluster 0 pgmap v512: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:18.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:17 smithi016 bash[77995]: cluster 2023-09-23T19:28:17.860804+0000 mgr.y (mgr.25325) 613 : cluster 0 pgmap v512: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:18.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:17 smithi092 bash[58855]: cluster 2023-09-23T19:28:17.860804+0000 mgr.y (mgr.25325) 613 : cluster 0 pgmap v512: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:19.133 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T19:28:20.230 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:19 smithi016 bash[77995]: audit 2023-09-23T19:28:18.251801+0000 mgr.y (mgr.25325) 614 : audit 0 from='client.45452 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:28:20.230 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:19 smithi016 bash[77995]: audit 2023-09-23T19:28:18.878971+0000 mon.a (mon.0) 856 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:28:20.230 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:19 smithi016 bash[75004]: audit 2023-09-23T19:28:18.251801+0000 mgr.y (mgr.25325) 614 : audit 0 from='client.45452 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:28:20.230 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:19 smithi016 bash[75004]: audit 2023-09-23T19:28:18.878971+0000 mon.a (mon.0) 856 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:28:20.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:19 smithi092 bash[58855]: audit 2023-09-23T19:28:18.251801+0000 mgr.y (mgr.25325) 614 : audit 0 from='client.45452 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:28:20.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:19 smithi092 bash[58855]: audit 2023-09-23T19:28:18.878971+0000 mon.a (mon.0) 856 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:28:21.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:20 smithi016 bash[75004]: cluster 2023-09-23T19:28:19.861573+0000 mgr.y (mgr.25325) 615 : cluster 0 pgmap v513: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:21.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:20 smithi016 bash[77995]: cluster 2023-09-23T19:28:19.861573+0000 mgr.y (mgr.25325) 615 : cluster 0 pgmap v513: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:21.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:20 smithi092 bash[58855]: cluster 2023-09-23T19:28:19.861573+0000 mgr.y (mgr.25325) 615 : cluster 0 pgmap v513: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:22.504 INFO:teuthology.orchestra.run.smithi016.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T19:28:22.504 INFO:teuthology.orchestra.run.smithi016.stdout:alertmanager.a smithi016 *:9093,9094 running (18m) - 24m 13.7M - 0.23.0 ba2b418f427c b362c3d9bae2 2023-09-23T19:28:22.504 INFO:teuthology.orchestra.run.smithi016.stdout:grafana.a smithi092 *:3000 running (17m) - 23m 42.2M - 8.3.5 dad864ee21e9 891a61cf5fff 2023-09-23T19:28:22.504 INFO:teuthology.orchestra.run.smithi016.stdout:mds.foo.smithi016.uncssv smithi016 running (22m) - 22m 20.1M - 16.2.14-37-g2e1f1015 1e6a756174a8 95d1e187e3eb 2023-09-23T19:28:22.504 INFO:teuthology.orchestra.run.smithi016.stdout:mds.foo.smithi092.vbpqks smithi092 running (22m) - 22m 18.2M - 16.2.14-37-g2e1f1015 1e6a756174a8 1d619ec31bf1 2023-09-23T19:28:22.504 INFO:teuthology.orchestra.run.smithi016.stdout:mgr.x smithi092 *:8443,9283,8765 running (16m) - 30m 433M - 18.2.0-460-gbb0daeff e32c2a64c126 d54b4354eade 2023-09-23T19:28:22.504 INFO:teuthology.orchestra.run.smithi016.stdout:mgr.y smithi016 *:8443,9283,8765 running (17m) - 32m 488M - 18.2.0-460-gbb0daeff e32c2a64c126 19706820b434 2023-09-23T19:28:22.505 INFO:teuthology.orchestra.run.smithi016.stdout:mon.a smithi016 running (15m) - 32m 108M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 e5aad364678c 2023-09-23T19:28:22.505 INFO:teuthology.orchestra.run.smithi016.stdout:mon.b smithi092 running (12m) - 31m 51.0M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 09d45d4f5248 2023-09-23T19:28:22.505 INFO:teuthology.orchestra.run.smithi016.stdout:mon.c smithi016 running (14m) - 31m 59.3M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 61a05c2c22a7 2023-09-23T19:28:22.505 INFO:teuthology.orchestra.run.smithi016.stdout:node-exporter.a smithi016 *:9100 running (18m) - 24m 10.1M - 1.3.1 1dbe0e931976 7f13d7a8905d 2023-09-23T19:28:22.505 INFO:teuthology.orchestra.run.smithi016.stdout:node-exporter.b smithi092 *:9100 running (17m) - 24m 9364k - 1.3.1 1dbe0e931976 4823347a98b5 2023-09-23T19:28:22.505 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0 smithi016 running (10m) - 30m 728M 2525M 18.2.0-460-gbb0daeff e32c2a64c126 aa95286475c5 2023-09-23T19:28:22.505 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1 smithi016 running (5m) - 29m 520M 2525M 18.2.0-460-gbb0daeff e32c2a64c126 f4575761955d 2023-09-23T19:28:22.505 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2 smithi016 running (72s) - 28m 403M 2525M 18.2.0-460-gbb0daeff e32c2a64c126 25d0dac6c5cf 2023-09-23T19:28:22.505 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3 smithi016 running (27s) - 28m - 2525M 18.2.0-460-gbb0daeff e32c2a64c126 e78a562df218 2023-09-23T19:28:22.505 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4 smithi092 running (27m) - 27m 623M 2524M 16.2.14-37-g2e1f1015 1e6a756174a8 253dc4b11010 2023-09-23T19:28:22.506 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5 smithi092 running (26m) - 26m 945M 2524M 16.2.14-37-g2e1f1015 1e6a756174a8 0a7306534382 2023-09-23T19:28:22.506 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6 smithi092 running (26m) - 26m 827M 2524M 16.2.14-37-g2e1f1015 1e6a756174a8 d6f3ee67cc5f 2023-09-23T19:28:22.506 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7 smithi092 running (25m) - 25m 587M 2524M 16.2.14-37-g2e1f1015 1e6a756174a8 794cf2340f17 2023-09-23T19:28:22.506 INFO:teuthology.orchestra.run.smithi016.stdout:prometheus.a smithi092 *:9095 running (16m) - 24m 62.5M - 2.33.4 514e6a882f6e 2666eba9f715 2023-09-23T19:28:23.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:22 smithi016 bash[75004]: cluster 2023-09-23T19:28:21.862618+0000 mgr.y (mgr.25325) 616 : cluster 0 pgmap v514: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:23.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:22 smithi016 bash[77995]: cluster 2023-09-23T19:28:21.862618+0000 mgr.y (mgr.25325) 616 : cluster 0 pgmap v514: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:23.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:22 smithi092 bash[58855]: cluster 2023-09-23T19:28:21.862618+0000 mgr.y (mgr.25325) 616 : cluster 0 pgmap v514: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:23.350 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph versions' 2023-09-23T19:28:24.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:23 smithi016 bash[75004]: audit 2023-09-23T19:28:22.482337+0000 mgr.y (mgr.25325) 617 : audit 0 from='client.35547 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:28:24.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:23 smithi016 bash[77995]: audit 2023-09-23T19:28:22.482337+0000 mgr.y (mgr.25325) 617 : audit 0 from='client.35547 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:28:24.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:23 smithi092 bash[58855]: audit 2023-09-23T19:28:22.482337+0000 mgr.y (mgr.25325) 617 : audit 0 from='client.35547 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:28:25.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:24 smithi016 bash[75004]: cluster 2023-09-23T19:28:23.863410+0000 mgr.y (mgr.25325) 618 : cluster 0 pgmap v515: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:25.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:24 smithi016 bash[77995]: cluster 2023-09-23T19:28:23.863410+0000 mgr.y (mgr.25325) 618 : cluster 0 pgmap v515: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:25.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:24 smithi092 bash[58855]: cluster 2023-09-23T19:28:23.863410+0000 mgr.y (mgr.25325) 618 : cluster 0 pgmap v515: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:25.785 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:28:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:28:25] "GET /metrics HTTP/1.1" 200 33812 "" "Prometheus/2.33.4" 2023-09-23T19:28:26.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:25 smithi016 bash[75004]: audit 2023-09-23T19:28:25.409163+0000 mon.a (mon.0) 857 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:28:26.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:25 smithi016 bash[75004]: audit 2023-09-23T19:28:25.412517+0000 mon.a (mon.0) 858 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:28:26.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:25 smithi016 bash[75004]: audit 2023-09-23T19:28:25.414030+0000 mon.a (mon.0) 859 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:28:26.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:25 smithi016 bash[75004]: audit 2023-09-23T19:28:25.421712+0000 mon.a (mon.0) 860 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:28:26.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:25 smithi016 bash[75004]: audit 2023-09-23T19:28:25.501064+0000 mon.a (mon.0) 861 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:28:26.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:25 smithi016 bash[75004]: audit 2023-09-23T19:28:25.502976+0000 mon.a (mon.0) 862 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:28:26.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:25 smithi016 bash[75004]: audit 2023-09-23T19:28:25.503907+0000 mon.a (mon.0) 863 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:28:26.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:25 smithi016 bash[75004]: audit 2023-09-23T19:28:25.511196+0000 mon.a (mon.0) 864 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:28:26.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:25 smithi016 bash[77995]: audit 2023-09-23T19:28:25.409163+0000 mon.a (mon.0) 857 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:28:26.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:25 smithi016 bash[77995]: audit 2023-09-23T19:28:25.412517+0000 mon.a (mon.0) 858 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:28:26.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:25 smithi016 bash[77995]: audit 2023-09-23T19:28:25.414030+0000 mon.a (mon.0) 859 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:28:26.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:25 smithi016 bash[77995]: audit 2023-09-23T19:28:25.421712+0000 mon.a (mon.0) 860 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:28:26.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:25 smithi016 bash[77995]: audit 2023-09-23T19:28:25.501064+0000 mon.a (mon.0) 861 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:28:26.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:25 smithi016 bash[77995]: audit 2023-09-23T19:28:25.502976+0000 mon.a (mon.0) 862 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:28:26.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:25 smithi016 bash[77995]: audit 2023-09-23T19:28:25.503907+0000 mon.a (mon.0) 863 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:28:26.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:25 smithi016 bash[77995]: audit 2023-09-23T19:28:25.511196+0000 mon.a (mon.0) 864 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:28:26.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:25 smithi092 bash[58855]: audit 2023-09-23T19:28:25.409163+0000 mon.a (mon.0) 857 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:28:26.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:25 smithi092 bash[58855]: audit 2023-09-23T19:28:25.412517+0000 mon.a (mon.0) 858 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:28:26.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:25 smithi092 bash[58855]: audit 2023-09-23T19:28:25.414030+0000 mon.a (mon.0) 859 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:28:26.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:25 smithi092 bash[58855]: audit 2023-09-23T19:28:25.421712+0000 mon.a (mon.0) 860 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:28:26.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:25 smithi092 bash[58855]: audit 2023-09-23T19:28:25.501064+0000 mon.a (mon.0) 861 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:28:26.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:25 smithi092 bash[58855]: audit 2023-09-23T19:28:25.502976+0000 mon.a (mon.0) 862 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:28:26.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:25 smithi092 bash[58855]: audit 2023-09-23T19:28:25.503907+0000 mon.a (mon.0) 863 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:28:26.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:25 smithi092 bash[58855]: audit 2023-09-23T19:28:25.511196+0000 mon.a (mon.0) 864 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:28:27.043 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-09-23T19:28:27.043 INFO:teuthology.orchestra.run.smithi016.stdout: "mon": { 2023-09-23T19:28:27.044 INFO:teuthology.orchestra.run.smithi016.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 3 2023-09-23T19:28:27.044 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:28:27.044 INFO:teuthology.orchestra.run.smithi016.stdout: "mgr": { 2023-09-23T19:28:27.044 INFO:teuthology.orchestra.run.smithi016.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 2 2023-09-23T19:28:27.044 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:28:27.044 INFO:teuthology.orchestra.run.smithi016.stdout: "osd": { 2023-09-23T19:28:27.044 INFO:teuthology.orchestra.run.smithi016.stdout: "ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)": 4, 2023-09-23T19:28:27.044 INFO:teuthology.orchestra.run.smithi016.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 4 2023-09-23T19:28:27.044 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:28:27.044 INFO:teuthology.orchestra.run.smithi016.stdout: "mds": { 2023-09-23T19:28:27.044 INFO:teuthology.orchestra.run.smithi016.stdout: "ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)": 2 2023-09-23T19:28:27.045 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:28:27.045 INFO:teuthology.orchestra.run.smithi016.stdout: "overall": { 2023-09-23T19:28:27.045 INFO:teuthology.orchestra.run.smithi016.stdout: "ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)": 6, 2023-09-23T19:28:27.045 INFO:teuthology.orchestra.run.smithi016.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 9 2023-09-23T19:28:27.045 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:28:27.045 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-09-23T19:28:27.278 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:26 smithi016 bash[75004]: cluster 2023-09-23T19:28:25.864384+0000 mgr.y (mgr.25325) 619 : cluster 0 pgmap v516: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:27.279 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:26 smithi016 bash[77995]: cluster 2023-09-23T19:28:25.864384+0000 mgr.y (mgr.25325) 619 : cluster 0 pgmap v516: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:27.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:26 smithi092 bash[58855]: cluster 2023-09-23T19:28:25.864384+0000 mgr.y (mgr.25325) 619 : cluster 0 pgmap v516: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:27.973 DEBUG:teuthology.parallel:result is None 2023-09-23T19:28:27.974 INFO:teuthology.run_tasks:Running task print... 2023-09-23T19:28:27.983 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2023-09-23T19:28:27.983 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-09-23T19:28:27.991 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-09-23T19:28:27.992 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-09-23T19:28:27.992 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph mgr dump --format=json 2023-09-23T19:28:28.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:27 smithi016 bash[75004]: audit 2023-09-23T19:28:27.040865+0000 mon.c (mon.2) 127 : audit 0 from='client.? 172.21.15.16:0/1810246629' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:28:28.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:27 smithi016 bash[75004]: cluster 2023-09-23T19:28:27.864890+0000 mgr.y (mgr.25325) 620 : cluster 0 pgmap v517: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:28.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:27 smithi016 bash[77995]: audit 2023-09-23T19:28:27.040865+0000 mon.c (mon.2) 127 : audit 0 from='client.? 172.21.15.16:0/1810246629' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:28:28.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:27 smithi016 bash[77995]: cluster 2023-09-23T19:28:27.864890+0000 mgr.y (mgr.25325) 620 : cluster 0 pgmap v517: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:28.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:27 smithi092 bash[58855]: audit 2023-09-23T19:28:27.040865+0000 mon.c (mon.2) 127 : audit 0 from='client.? 172.21.15.16:0/1810246629' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:28:28.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:27 smithi092 bash[58855]: cluster 2023-09-23T19:28:27.864890+0000 mgr.y (mgr.25325) 620 : cluster 0 pgmap v517: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:29.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:28 smithi016 bash[75004]: audit 2023-09-23T19:28:28.632198+0000 mon.a (mon.0) 865 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:28:29.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:28 smithi016 bash[77995]: audit 2023-09-23T19:28:28.632198+0000 mon.a (mon.0) 865 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:28:29.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:28 smithi092 bash[58855]: audit 2023-09-23T19:28:28.632198+0000 mon.a (mon.0) 865 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:28:30.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:30 smithi016 bash[75004]: cluster 2023-09-23T19:28:29.865634+0000 mgr.y (mgr.25325) 621 : cluster 0 pgmap v518: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:30.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:30 smithi016 bash[77995]: cluster 2023-09-23T19:28:29.865634+0000 mgr.y (mgr.25325) 621 : cluster 0 pgmap v518: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:30.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:30 smithi092 bash[58855]: cluster 2023-09-23T19:28:29.865634+0000 mgr.y (mgr.25325) 621 : cluster 0 pgmap v518: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:32.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:31 smithi016 bash[75004]: cluster 2023-09-23T19:28:31.866664+0000 mgr.y (mgr.25325) 622 : cluster 0 pgmap v519: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:32.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:31 smithi016 bash[77995]: cluster 2023-09-23T19:28:31.866664+0000 mgr.y (mgr.25325) 622 : cluster 0 pgmap v519: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:32.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:31 smithi092 bash[58855]: cluster 2023-09-23T19:28:31.866664+0000 mgr.y (mgr.25325) 622 : cluster 0 pgmap v519: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:32.770 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:28:34.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:33 smithi016 bash[75004]: cluster 2023-09-23T19:28:33.867370+0000 mgr.y (mgr.25325) 623 : cluster 0 pgmap v520: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:34.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:33 smithi016 bash[77995]: cluster 2023-09-23T19:28:33.867370+0000 mgr.y (mgr.25325) 623 : cluster 0 pgmap v520: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:34.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:33 smithi092 bash[58855]: cluster 2023-09-23T19:28:33.867370+0000 mgr.y (mgr.25325) 623 : cluster 0 pgmap v520: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:35.948 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:28:35 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:28:35] "GET /metrics HTTP/1.1" 200 33813 "" "Prometheus/2.33.4" 2023-09-23T19:28:36.093 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-09-23T19:28:36.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:35 smithi016 bash[77995]: cluster 2023-09-23T19:28:35.868296+0000 mgr.y (mgr.25325) 624 : cluster 0 pgmap v521: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:36.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:35 smithi016 bash[75004]: cluster 2023-09-23T19:28:35.868296+0000 mgr.y (mgr.25325) 624 : cluster 0 pgmap v521: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:36.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:35 smithi092 bash[58855]: cluster 2023-09-23T19:28:35.868296+0000 mgr.y (mgr.25325) 624 : cluster 0 pgmap v521: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:37.041 INFO:teuthology.orchestra.run.smithi016.stdout:{"epoch":38,"active_gid":25325,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6800","nonce":1315373977},{"type":"v1","addr":"172.21.15.16:6801","nonce":1315373977}]},"active_addr":"172.21.15.16:6801/1315373977","active_change":"2023-09-23T19:12:43.452772+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":25141,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph:v18","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph:v18","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.16:8443/","prometheus":"http://172.21.15.16:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":75,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.16:0","nonce":2693463821}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.16:0","nonce":777984439}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.16:0","nonce":2591242976}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.16:0","nonce":1657316894}]}]} 2023-09-23T19:28:37.046 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-09-23T19:28:37.046 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-09-23T19:28:37.046 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T19:28:37.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:36 smithi016 bash[75004]: audit 2023-09-23T19:28:36.084620+0000 mon.c (mon.2) 128 : audit 0 from='client.? 172.21.15.16:0/890096050' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T19:28:37.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:36 smithi016 bash[77995]: audit 2023-09-23T19:28:36.084620+0000 mon.c (mon.2) 128 : audit 0 from='client.? 172.21.15.16:0/890096050' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T19:28:37.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:36 smithi092 bash[58855]: audit 2023-09-23T19:28:36.084620+0000 mon.c (mon.2) 128 : audit 0 from='client.? 172.21.15.16:0/890096050' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T19:28:38.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:37 smithi016 bash[75004]: cluster 2023-09-23T19:28:37.869131+0000 mgr.y (mgr.25325) 625 : cluster 0 pgmap v522: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:38.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:37 smithi016 bash[77995]: cluster 2023-09-23T19:28:37.869131+0000 mgr.y (mgr.25325) 625 : cluster 0 pgmap v522: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:38.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:37 smithi092 bash[58855]: cluster 2023-09-23T19:28:37.869131+0000 mgr.y (mgr.25325) 625 : cluster 0 pgmap v522: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:40.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:40 smithi016 bash[75004]: cluster 2023-09-23T19:28:39.869942+0000 mgr.y (mgr.25325) 626 : cluster 0 pgmap v523: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:40.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:40 smithi016 bash[77995]: cluster 2023-09-23T19:28:39.869942+0000 mgr.y (mgr.25325) 626 : cluster 0 pgmap v523: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:40.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:40 smithi092 bash[58855]: cluster 2023-09-23T19:28:39.869942+0000 mgr.y (mgr.25325) 626 : cluster 0 pgmap v523: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:41.835 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:28:42.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:41 smithi016 bash[77995]: cluster 2023-09-23T19:28:41.870629+0000 mgr.y (mgr.25325) 627 : cluster 0 pgmap v524: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:42.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:41 smithi016 bash[75004]: cluster 2023-09-23T19:28:41.870629+0000 mgr.y (mgr.25325) 627 : cluster 0 pgmap v524: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:42.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:41 smithi092 bash[58855]: cluster 2023-09-23T19:28:41.870629+0000 mgr.y (mgr.25325) 627 : cluster 0 pgmap v524: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:44.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:43 smithi016 bash[75004]: audit 2023-09-23T19:28:43.632756+0000 mon.a (mon.0) 866 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:28:44.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:43 smithi016 bash[77995]: audit 2023-09-23T19:28:43.632756+0000 mon.a (mon.0) 866 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:28:44.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:43 smithi092 bash[58855]: audit 2023-09-23T19:28:43.632756+0000 mon.a (mon.0) 866 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:28:44.984 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:44 smithi016 bash[75004]: cluster 2023-09-23T19:28:43.871313+0000 mgr.y (mgr.25325) 628 : cluster 0 pgmap v525: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:44.985 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:44 smithi016 bash[77995]: cluster 2023-09-23T19:28:43.871313+0000 mgr.y (mgr.25325) 628 : cluster 0 pgmap v525: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:45.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:44 smithi092 bash[58855]: cluster 2023-09-23T19:28:43.871313+0000 mgr.y (mgr.25325) 628 : cluster 0 pgmap v525: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:45.357 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-09-23T19:28:45.358 INFO:teuthology.orchestra.run.smithi016.stdout:{"epoch":117,"fsid":"95ce9eb6-5a42-11ee-8db4-212e2dc638e7","created":"2023-09-23T18:55:23.802934+0000","modified":"2023-09-23T19:28:07.860023+0000","last_up_change":"2023-09-23T19:28:06.857617+0000","last_in_change":"2023-09-23T19:02:36.618072+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":8,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T18:59:33.315416+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{},"mgr_devicehealth":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T19:04:01.440427+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T19:05:50.982524+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T19:05:51.538373+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"a4e8ed09-1195-4878-928a-13881bc04dc7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":81,"up_from":85,"up_thru":116,"down_at":82,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6802","nonce":3034120466},{"type":"v1","addr":"172.21.15.16:6803","nonce":3034120466}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6804","nonce":3034120466},{"type":"v1","addr":"172.21.15.16:6805","nonce":3034120466}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6808","nonce":3034120466},{"type":"v1","addr":"172.21.15.16:6809","nonce":3034120466}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6806","nonce":3034120466},{"type":"v1","addr":"172.21.15.16:6807","nonce":3034120466}]},"public_addr":"172.21.15.16:6803/3034120466","cluster_addr":"172.21.15.16:6805/3034120466","heartbeat_back_addr":"172.21.15.16:6809/3034120466","heartbeat_front_addr":"172.21.15.16:6807/3034120466","state":["exists","up"]},{"osd":1,"uuid":"c5ff2879-0d1c-481a-b847-248433c12aa4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":93,"up_from":98,"up_thru":116,"down_at":94,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6810","nonce":3881357617},{"type":"v1","addr":"172.21.15.16:6811","nonce":3881357617}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6812","nonce":3881357617},{"type":"v1","addr":"172.21.15.16:6813","nonce":3881357617}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6816","nonce":3881357617},{"type":"v1","addr":"172.21.15.16:6817","nonce":3881357617}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6814","nonce":3881357617},{"type":"v1","addr":"172.21.15.16:6815","nonce":3881357617}]},"public_addr":"172.21.15.16:6811/3881357617","cluster_addr":"172.21.15.16:6813/3881357617","heartbeat_back_addr":"172.21.15.16:6817/3881357617","heartbeat_front_addr":"172.21.15.16:6815/3881357617","state":["exists","up"]},{"osd":2,"uuid":"7974ce4d-9ed8-4a72-b44b-d500b34da8cf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":107,"up_from":111,"up_thru":116,"down_at":108,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6818","nonce":486425090},{"type":"v1","addr":"172.21.15.16:6819","nonce":486425090}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6820","nonce":486425090},{"type":"v1","addr":"172.21.15.16:6821","nonce":486425090}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6824","nonce":486425090},{"type":"v1","addr":"172.21.15.16:6825","nonce":486425090}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6822","nonce":486425090},{"type":"v1","addr":"172.21.15.16:6823","nonce":486425090}]},"public_addr":"172.21.15.16:6819/486425090","cluster_addr":"172.21.15.16:6821/486425090","heartbeat_back_addr":"172.21.15.16:6825/486425090","heartbeat_front_addr":"172.21.15.16:6823/486425090","state":["exists","up"]},{"osd":3,"uuid":"f5e0d57b-d629-4453-a527-ad77e6c210ad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":112,"up_from":116,"up_thru":116,"down_at":113,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6826","nonce":3155744043},{"type":"v1","addr":"172.21.15.16:6827","nonce":3155744043}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6828","nonce":3155744043},{"type":"v1","addr":"172.21.15.16:6829","nonce":3155744043}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6832","nonce":3155744043},{"type":"v1","addr":"172.21.15.16:6833","nonce":3155744043}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6830","nonce":3155744043},{"type":"v1","addr":"172.21.15.16:6831","nonce":3155744043}]},"public_addr":"172.21.15.16:6827/3155744043","cluster_addr":"172.21.15.16:6829/3155744043","heartbeat_back_addr":"172.21.15.16:6833/3155744043","heartbeat_front_addr":"172.21.15.16:6831/3155744043","state":["exists","up"]},{"osd":4,"uuid":"55a8dab2-6393-44ab-943e-cb7ab2cb7c31","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":116,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6800","nonce":99745912},{"type":"v1","addr":"172.21.15.92:6801","nonce":99745912}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6802","nonce":99745912},{"type":"v1","addr":"172.21.15.92:6803","nonce":99745912}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6806","nonce":99745912},{"type":"v1","addr":"172.21.15.92:6807","nonce":99745912}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6804","nonce":99745912},{"type":"v1","addr":"172.21.15.92:6805","nonce":99745912}]},"public_addr":"172.21.15.92:6801/99745912","cluster_addr":"172.21.15.92:6803/99745912","heartbeat_back_addr":"172.21.15.92:6807/99745912","heartbeat_front_addr":"172.21.15.92:6805/99745912","state":["exists","up"]},{"osd":5,"uuid":"b29b73b1-51d7-4484-bd79-2334c650cb62","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":116,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6808","nonce":1987214502},{"type":"v1","addr":"172.21.15.92:6809","nonce":1987214502}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6810","nonce":1987214502},{"type":"v1","addr":"172.21.15.92:6811","nonce":1987214502}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6814","nonce":1987214502},{"type":"v1","addr":"172.21.15.92:6815","nonce":1987214502}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6812","nonce":1987214502},{"type":"v1","addr":"172.21.15.92:6813","nonce":1987214502}]},"public_addr":"172.21.15.92:6809/1987214502","cluster_addr":"172.21.15.92:6811/1987214502","heartbeat_back_addr":"172.21.15.92:6815/1987214502","heartbeat_front_addr":"172.21.15.92:6813/1987214502","state":["exists","up"]},{"osd":6,"uuid":"9fe16cc9-f0d3-4b79-bc68-38305c55b551","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":113,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6816","nonce":2820798178},{"type":"v1","addr":"172.21.15.92:6817","nonce":2820798178}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6818","nonce":2820798178},{"type":"v1","addr":"172.21.15.92:6819","nonce":2820798178}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6822","nonce":2820798178},{"type":"v1","addr":"172.21.15.92:6823","nonce":2820798178}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6820","nonce":2820798178},{"type":"v1","addr":"172.21.15.92:6821","nonce":2820798178}]},"public_addr":"172.21.15.92:6817/2820798178","cluster_addr":"172.21.15.92:6819/2820798178","heartbeat_back_addr":"172.21.15.92:6823/2820798178","heartbeat_front_addr":"172.21.15.92:6821/2820798178","state":["exists","up"]},{"osd":7,"uuid":"d086b076-127f-42de-bee9-157bf579d92d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":116,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6824","nonce":98624761},{"type":"v1","addr":"172.21.15.92:6825","nonce":98624761}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6826","nonce":98624761},{"type":"v1","addr":"172.21.15.92:6827","nonce":98624761}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6830","nonce":98624761},{"type":"v1","addr":"172.21.15.92:6831","nonce":98624761}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6828","nonce":98624761},{"type":"v1","addr":"172.21.15.92:6829","nonce":98624761}]},"public_addr":"172.21.15.92:6825/98624761","cluster_addr":"172.21.15.92:6827/98624761","heartbeat_back_addr":"172.21.15.92:6831/98624761","heartbeat_front_addr":"172.21.15.92:6829/98624761","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:58:07.335980+0000","dead_epoch":81},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:58:49.342213+0000","dead_epoch":93},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:59:30.939986+0000","dead_epoch":107},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:00:12.926364+0000","dead_epoch":112},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:00:53.466692+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:01:36.332957+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:02:18.593854+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:03:00.817448+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.2","mappings":[{"from":1,"to":2}]},{"pgid":"4.18","mappings":[{"from":1,"to":5}]},{"pgid":"4.1b","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.16:6800/2756103674":"2023-09-24T18:56:20.574625+0000","172.21.15.16:0/2843459289":"2023-09-24T18:56:20.574625+0000","172.21.15.16:6801/2756103674":"2023-09-24T18:56:20.574625+0000","172.21.15.92:0/1841502917":"2023-09-24T19:11:32.247480+0000","172.21.15.16:6801/777573315":"2023-09-24T19:12:43.452315+0000","172.21.15.16:6800/777573315":"2023-09-24T19:12:43.452315+0000","172.21.15.16:6801/2615282850":"2023-09-24T18:55:50.550546+0000","172.21.15.16:6801/1259630240":"2023-09-24T19:03:34.846451+0000","172.21.15.16:6800/1259630240":"2023-09-24T19:03:34.846451+0000","172.21.15.16:0/79169563":"2023-09-24T18:55:50.550546+0000","172.21.15.16:0/3595835147":"2023-09-24T19:12:43.452315+0000","172.21.15.92:0/3173267635":"2023-09-24T19:11:32.247480+0000","172.21.15.16:0/3394254122":"2023-09-24T19:12:43.452315+0000","172.21.15.16:0/272242693":"2023-09-24T18:56:20.574625+0000","172.21.15.16:0/736761601":"2023-09-24T19:12:43.452315+0000","172.21.15.92:0/3338210669":"2023-09-24T19:11:32.247480+0000","172.21.15.92:6835/2057654371":"2023-09-24T19:11:32.247480+0000","172.21.15.16:0/3767844203":"2023-09-24T19:03:34.846451+0000","172.21.15.16:0/2334334390":"2023-09-24T19:12:43.452315+0000","172.21.15.16:0/4196794739":"2023-09-24T19:12:43.452315+0000","172.21.15.16:6800/1501185908":"2023-09-24T19:09:35.764967+0000","172.21.15.16:0/3765291224":"2023-09-24T19:09:35.764967+0000","172.21.15.16:6801/1501185908":"2023-09-24T19:09:35.764967+0000","172.21.15.16:0/647003663":"2023-09-24T19:12:43.452315+0000","172.21.15.16:0/2669353350":"2023-09-24T19:03:34.846451+0000","172.21.15.92:6834/2057654371":"2023-09-24T19:11:32.247480+0000","172.21.15.92:0/561988499":"2023-09-24T19:11:32.247480+0000","172.21.15.16:0/4152413938":"2023-09-24T19:09:35.764967+0000","172.21.15.16:6800/2615282850":"2023-09-24T18:55:50.550546+0000","172.21.15.16:0/349446544":"2023-09-24T18:55:50.550546+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T19:28:45.697 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:28:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:28:45] "GET /metrics HTTP/1.1" 200 33809 "" "Prometheus/2.33.4" 2023-09-23T19:28:46.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:45 smithi016 bash[75004]: audit 2023-09-23T19:28:45.352180+0000 mon.c (mon.2) 129 : audit 0 from='client.? 172.21.15.16:0/4124042071' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T19:28:46.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:45 smithi016 bash[77995]: audit 2023-09-23T19:28:45.352180+0000 mon.c (mon.2) 129 : audit 0 from='client.? 172.21.15.16:0/4124042071' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T19:28:46.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:45 smithi092 bash[58855]: audit 2023-09-23T19:28:45.352180+0000 mon.c (mon.2) 129 : audit 0 from='client.? 172.21.15.16:0/4124042071' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T19:28:46.222 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-09-23T19:28:46.223 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T19:28:47.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:46 smithi016 bash[75004]: cluster 2023-09-23T19:28:45.872448+0000 mgr.y (mgr.25325) 629 : cluster 0 pgmap v526: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:47.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:46 smithi016 bash[77995]: cluster 2023-09-23T19:28:45.872448+0000 mgr.y (mgr.25325) 629 : cluster 0 pgmap v526: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:47.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:46 smithi092 bash[58855]: cluster 2023-09-23T19:28:45.872448+0000 mgr.y (mgr.25325) 629 : cluster 0 pgmap v526: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:48.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:47 smithi016 bash[77995]: cluster 2023-09-23T19:28:47.873302+0000 mgr.y (mgr.25325) 630 : cluster 0 pgmap v527: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:48.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:47 smithi016 bash[75004]: cluster 2023-09-23T19:28:47.873302+0000 mgr.y (mgr.25325) 630 : cluster 0 pgmap v527: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:48.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:47 smithi092 bash[58855]: cluster 2023-09-23T19:28:47.873302+0000 mgr.y (mgr.25325) 630 : cluster 0 pgmap v527: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:50.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:50 smithi016 bash[75004]: cluster 2023-09-23T19:28:49.874111+0000 mgr.y (mgr.25325) 631 : cluster 0 pgmap v528: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:50.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:50 smithi016 bash[77995]: cluster 2023-09-23T19:28:49.874111+0000 mgr.y (mgr.25325) 631 : cluster 0 pgmap v528: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:50.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:50 smithi092 bash[58855]: cluster 2023-09-23T19:28:49.874111+0000 mgr.y (mgr.25325) 631 : cluster 0 pgmap v528: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:51.012 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:28:52.201 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:51 smithi016 bash[75004]: cluster 2023-09-23T19:28:51.875117+0000 mgr.y (mgr.25325) 632 : cluster 0 pgmap v529: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:52.201 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:51 smithi016 bash[77995]: cluster 2023-09-23T19:28:51.875117+0000 mgr.y (mgr.25325) 632 : cluster 0 pgmap v529: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:52.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:51 smithi092 bash[58855]: cluster 2023-09-23T19:28:51.875117+0000 mgr.y (mgr.25325) 632 : cluster 0 pgmap v529: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:54.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:53 smithi016 bash[75004]: cluster 2023-09-23T19:28:53.875864+0000 mgr.y (mgr.25325) 633 : cluster 0 pgmap v530: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:54.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:53 smithi016 bash[77995]: cluster 2023-09-23T19:28:53.875864+0000 mgr.y (mgr.25325) 633 : cluster 0 pgmap v530: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:54.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:53 smithi092 bash[58855]: cluster 2023-09-23T19:28:53.875864+0000 mgr.y (mgr.25325) 633 : cluster 0 pgmap v530: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:54.477 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-09-23T19:28:54.477 INFO:teuthology.orchestra.run.smithi016.stdout:{"epoch":117,"fsid":"95ce9eb6-5a42-11ee-8db4-212e2dc638e7","created":"2023-09-23T18:55:23.802934+0000","modified":"2023-09-23T19:28:07.860023+0000","last_up_change":"2023-09-23T19:28:06.857617+0000","last_in_change":"2023-09-23T19:02:36.618072+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":8,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T18:59:33.315416+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{},"mgr_devicehealth":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T19:04:01.440427+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T19:05:50.982524+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T19:05:51.538373+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"a4e8ed09-1195-4878-928a-13881bc04dc7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":81,"up_from":85,"up_thru":116,"down_at":82,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6802","nonce":3034120466},{"type":"v1","addr":"172.21.15.16:6803","nonce":3034120466}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6804","nonce":3034120466},{"type":"v1","addr":"172.21.15.16:6805","nonce":3034120466}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6808","nonce":3034120466},{"type":"v1","addr":"172.21.15.16:6809","nonce":3034120466}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6806","nonce":3034120466},{"type":"v1","addr":"172.21.15.16:6807","nonce":3034120466}]},"public_addr":"172.21.15.16:6803/3034120466","cluster_addr":"172.21.15.16:6805/3034120466","heartbeat_back_addr":"172.21.15.16:6809/3034120466","heartbeat_front_addr":"172.21.15.16:6807/3034120466","state":["exists","up"]},{"osd":1,"uuid":"c5ff2879-0d1c-481a-b847-248433c12aa4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":93,"up_from":98,"up_thru":116,"down_at":94,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6810","nonce":3881357617},{"type":"v1","addr":"172.21.15.16:6811","nonce":3881357617}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6812","nonce":3881357617},{"type":"v1","addr":"172.21.15.16:6813","nonce":3881357617}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6816","nonce":3881357617},{"type":"v1","addr":"172.21.15.16:6817","nonce":3881357617}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6814","nonce":3881357617},{"type":"v1","addr":"172.21.15.16:6815","nonce":3881357617}]},"public_addr":"172.21.15.16:6811/3881357617","cluster_addr":"172.21.15.16:6813/3881357617","heartbeat_back_addr":"172.21.15.16:6817/3881357617","heartbeat_front_addr":"172.21.15.16:6815/3881357617","state":["exists","up"]},{"osd":2,"uuid":"7974ce4d-9ed8-4a72-b44b-d500b34da8cf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":107,"up_from":111,"up_thru":116,"down_at":108,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6818","nonce":486425090},{"type":"v1","addr":"172.21.15.16:6819","nonce":486425090}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6820","nonce":486425090},{"type":"v1","addr":"172.21.15.16:6821","nonce":486425090}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6824","nonce":486425090},{"type":"v1","addr":"172.21.15.16:6825","nonce":486425090}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6822","nonce":486425090},{"type":"v1","addr":"172.21.15.16:6823","nonce":486425090}]},"public_addr":"172.21.15.16:6819/486425090","cluster_addr":"172.21.15.16:6821/486425090","heartbeat_back_addr":"172.21.15.16:6825/486425090","heartbeat_front_addr":"172.21.15.16:6823/486425090","state":["exists","up"]},{"osd":3,"uuid":"f5e0d57b-d629-4453-a527-ad77e6c210ad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":112,"up_from":116,"up_thru":116,"down_at":113,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6826","nonce":3155744043},{"type":"v1","addr":"172.21.15.16:6827","nonce":3155744043}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6828","nonce":3155744043},{"type":"v1","addr":"172.21.15.16:6829","nonce":3155744043}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6832","nonce":3155744043},{"type":"v1","addr":"172.21.15.16:6833","nonce":3155744043}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6830","nonce":3155744043},{"type":"v1","addr":"172.21.15.16:6831","nonce":3155744043}]},"public_addr":"172.21.15.16:6827/3155744043","cluster_addr":"172.21.15.16:6829/3155744043","heartbeat_back_addr":"172.21.15.16:6833/3155744043","heartbeat_front_addr":"172.21.15.16:6831/3155744043","state":["exists","up"]},{"osd":4,"uuid":"55a8dab2-6393-44ab-943e-cb7ab2cb7c31","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":116,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6800","nonce":99745912},{"type":"v1","addr":"172.21.15.92:6801","nonce":99745912}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6802","nonce":99745912},{"type":"v1","addr":"172.21.15.92:6803","nonce":99745912}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6806","nonce":99745912},{"type":"v1","addr":"172.21.15.92:6807","nonce":99745912}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6804","nonce":99745912},{"type":"v1","addr":"172.21.15.92:6805","nonce":99745912}]},"public_addr":"172.21.15.92:6801/99745912","cluster_addr":"172.21.15.92:6803/99745912","heartbeat_back_addr":"172.21.15.92:6807/99745912","heartbeat_front_addr":"172.21.15.92:6805/99745912","state":["exists","up"]},{"osd":5,"uuid":"b29b73b1-51d7-4484-bd79-2334c650cb62","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":116,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6808","nonce":1987214502},{"type":"v1","addr":"172.21.15.92:6809","nonce":1987214502}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6810","nonce":1987214502},{"type":"v1","addr":"172.21.15.92:6811","nonce":1987214502}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6814","nonce":1987214502},{"type":"v1","addr":"172.21.15.92:6815","nonce":1987214502}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6812","nonce":1987214502},{"type":"v1","addr":"172.21.15.92:6813","nonce":1987214502}]},"public_addr":"172.21.15.92:6809/1987214502","cluster_addr":"172.21.15.92:6811/1987214502","heartbeat_back_addr":"172.21.15.92:6815/1987214502","heartbeat_front_addr":"172.21.15.92:6813/1987214502","state":["exists","up"]},{"osd":6,"uuid":"9fe16cc9-f0d3-4b79-bc68-38305c55b551","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":113,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6816","nonce":2820798178},{"type":"v1","addr":"172.21.15.92:6817","nonce":2820798178}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6818","nonce":2820798178},{"type":"v1","addr":"172.21.15.92:6819","nonce":2820798178}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6822","nonce":2820798178},{"type":"v1","addr":"172.21.15.92:6823","nonce":2820798178}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6820","nonce":2820798178},{"type":"v1","addr":"172.21.15.92:6821","nonce":2820798178}]},"public_addr":"172.21.15.92:6817/2820798178","cluster_addr":"172.21.15.92:6819/2820798178","heartbeat_back_addr":"172.21.15.92:6823/2820798178","heartbeat_front_addr":"172.21.15.92:6821/2820798178","state":["exists","up"]},{"osd":7,"uuid":"d086b076-127f-42de-bee9-157bf579d92d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":116,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6824","nonce":98624761},{"type":"v1","addr":"172.21.15.92:6825","nonce":98624761}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6826","nonce":98624761},{"type":"v1","addr":"172.21.15.92:6827","nonce":98624761}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6830","nonce":98624761},{"type":"v1","addr":"172.21.15.92:6831","nonce":98624761}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6828","nonce":98624761},{"type":"v1","addr":"172.21.15.92:6829","nonce":98624761}]},"public_addr":"172.21.15.92:6825/98624761","cluster_addr":"172.21.15.92:6827/98624761","heartbeat_back_addr":"172.21.15.92:6831/98624761","heartbeat_front_addr":"172.21.15.92:6829/98624761","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:58:07.335980+0000","dead_epoch":81},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:58:49.342213+0000","dead_epoch":93},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:59:30.939986+0000","dead_epoch":107},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:00:12.926364+0000","dead_epoch":112},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:00:53.466692+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:01:36.332957+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:02:18.593854+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:03:00.817448+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.2","mappings":[{"from":1,"to":2}]},{"pgid":"4.18","mappings":[{"from":1,"to":5}]},{"pgid":"4.1b","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.16:6800/2756103674":"2023-09-24T18:56:20.574625+0000","172.21.15.16:0/2843459289":"2023-09-24T18:56:20.574625+0000","172.21.15.16:6801/2756103674":"2023-09-24T18:56:20.574625+0000","172.21.15.92:0/1841502917":"2023-09-24T19:11:32.247480+0000","172.21.15.16:6801/777573315":"2023-09-24T19:12:43.452315+0000","172.21.15.16:6800/777573315":"2023-09-24T19:12:43.452315+0000","172.21.15.16:6801/2615282850":"2023-09-24T18:55:50.550546+0000","172.21.15.16:6801/1259630240":"2023-09-24T19:03:34.846451+0000","172.21.15.16:6800/1259630240":"2023-09-24T19:03:34.846451+0000","172.21.15.16:0/79169563":"2023-09-24T18:55:50.550546+0000","172.21.15.16:0/3595835147":"2023-09-24T19:12:43.452315+0000","172.21.15.92:0/3173267635":"2023-09-24T19:11:32.247480+0000","172.21.15.16:0/3394254122":"2023-09-24T19:12:43.452315+0000","172.21.15.16:0/272242693":"2023-09-24T18:56:20.574625+0000","172.21.15.16:0/736761601":"2023-09-24T19:12:43.452315+0000","172.21.15.92:0/3338210669":"2023-09-24T19:11:32.247480+0000","172.21.15.92:6835/2057654371":"2023-09-24T19:11:32.247480+0000","172.21.15.16:0/3767844203":"2023-09-24T19:03:34.846451+0000","172.21.15.16:0/2334334390":"2023-09-24T19:12:43.452315+0000","172.21.15.16:0/4196794739":"2023-09-24T19:12:43.452315+0000","172.21.15.16:6800/1501185908":"2023-09-24T19:09:35.764967+0000","172.21.15.16:0/3765291224":"2023-09-24T19:09:35.764967+0000","172.21.15.16:6801/1501185908":"2023-09-24T19:09:35.764967+0000","172.21.15.16:0/647003663":"2023-09-24T19:12:43.452315+0000","172.21.15.16:0/2669353350":"2023-09-24T19:03:34.846451+0000","172.21.15.92:6834/2057654371":"2023-09-24T19:11:32.247480+0000","172.21.15.92:0/561988499":"2023-09-24T19:11:32.247480+0000","172.21.15.16:0/4152413938":"2023-09-24T19:09:35.764967+0000","172.21.15.16:6800/2615282850":"2023-09-24T18:55:50.550546+0000","172.21.15.16:0/349446544":"2023-09-24T18:55:50.550546+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T19:28:55.274 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-09-23T19:28:55.274 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-09-23T19:28:55.275 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-09-23T19:28:55.275 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-09-23T19:28:55.276 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-09-23T19:28:55.276 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-09-23T19:28:55.276 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-09-23T19:28:55.277 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-09-23T19:28:55.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:55 smithi016 bash[75004]: audit 2023-09-23T19:28:54.471930+0000 mon.c (mon.2) 130 : audit 0 from='client.? 172.21.15.16:0/4124721811' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T19:28:55.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:55 smithi016 bash[77995]: audit 2023-09-23T19:28:54.471930+0000 mon.c (mon.2) 130 : audit 0 from='client.? 172.21.15.16:0/4124721811' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T19:28:55.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:55 smithi092 bash[58855]: audit 2023-09-23T19:28:54.471930+0000 mon.c (mon.2) 130 : audit 0 from='client.? 172.21.15.16:0/4124721811' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T19:28:56.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:28:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:28:55] "GET /metrics HTTP/1.1" 200 33809 "" "Prometheus/2.33.4" 2023-09-23T19:28:56.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:56 smithi016 bash[75004]: cluster 2023-09-23T19:28:55.876787+0000 mgr.y (mgr.25325) 634 : cluster 0 pgmap v531: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:56.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:56 smithi016 bash[77995]: cluster 2023-09-23T19:28:55.876787+0000 mgr.y (mgr.25325) 634 : cluster 0 pgmap v531: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:56.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:56 smithi092 bash[58855]: cluster 2023-09-23T19:28:55.876787+0000 mgr.y (mgr.25325) 634 : cluster 0 pgmap v531: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:58.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:57 smithi016 bash[77995]: cluster 2023-09-23T19:28:57.877627+0000 mgr.y (mgr.25325) 635 : cluster 0 pgmap v532: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:58.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:57 smithi016 bash[75004]: cluster 2023-09-23T19:28:57.877627+0000 mgr.y (mgr.25325) 635 : cluster 0 pgmap v532: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:58.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:57 smithi092 bash[58855]: cluster 2023-09-23T19:28:57.877627+0000 mgr.y (mgr.25325) 635 : cluster 0 pgmap v532: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:28:59.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:28:58 smithi016 bash[75004]: audit 2023-09-23T19:28:58.632991+0000 mon.a (mon.0) 867 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:28:59.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:28:58 smithi016 bash[77995]: audit 2023-09-23T19:28:58.632991+0000 mon.a (mon.0) 867 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:28:59.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:28:58 smithi092 bash[58855]: audit 2023-09-23T19:28:58.632991+0000 mon.a (mon.0) 867 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:29:00.175 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:29:00.177 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:29:00.177 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:29:00.178 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:29:00.178 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:29:00.179 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:29:00.184 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:29:00.184 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:29:00.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:29:00 smithi016 bash[75004]: cluster 2023-09-23T19:28:59.878479+0000 mgr.y (mgr.25325) 636 : cluster 0 pgmap v533: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:00.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:29:00 smithi016 bash[77995]: cluster 2023-09-23T19:28:59.878479+0000 mgr.y (mgr.25325) 636 : cluster 0 pgmap v533: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:00.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:29:00 smithi092 bash[58855]: cluster 2023-09-23T19:28:59.878479+0000 mgr.y (mgr.25325) 636 : cluster 0 pgmap v533: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:02.141 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:29:01 smithi016 bash[75004]: cluster 2023-09-23T19:29:01.879546+0000 mgr.y (mgr.25325) 637 : cluster 0 pgmap v534: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:02.142 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:29:01 smithi016 bash[77995]: cluster 2023-09-23T19:29:01.879546+0000 mgr.y (mgr.25325) 637 : cluster 0 pgmap v534: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:02.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:29:01 smithi092 bash[58855]: cluster 2023-09-23T19:29:01.879546+0000 mgr.y (mgr.25325) 637 : cluster 0 pgmap v534: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:04.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:29:03 smithi016 bash[75004]: cluster 2023-09-23T19:29:03.880085+0000 mgr.y (mgr.25325) 638 : cluster 0 pgmap v535: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:04.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:29:03 smithi016 bash[77995]: cluster 2023-09-23T19:29:03.880085+0000 mgr.y (mgr.25325) 638 : cluster 0 pgmap v535: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:04.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:29:03 smithi092 bash[58855]: cluster 2023-09-23T19:29:03.880085+0000 mgr.y (mgr.25325) 638 : cluster 0 pgmap v535: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:05.953 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:29:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:29:05] "GET /metrics HTTP/1.1" 200 33814 "" "Prometheus/2.33.4" 2023-09-23T19:29:06.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:29:05 smithi016 bash[75004]: cluster 2023-09-23T19:29:05.881063+0000 mgr.y (mgr.25325) 639 : cluster 0 pgmap v536: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:06.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:29:05 smithi016 bash[77995]: cluster 2023-09-23T19:29:05.881063+0000 mgr.y (mgr.25325) 639 : cluster 0 pgmap v536: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:06.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:29:05 smithi092 bash[58855]: cluster 2023-09-23T19:29:05.881063+0000 mgr.y (mgr.25325) 639 : cluster 0 pgmap v536: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:08.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:29:07 smithi016 bash[77995]: cluster 2023-09-23T19:29:07.881871+0000 mgr.y (mgr.25325) 640 : cluster 0 pgmap v537: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:08.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:29:07 smithi016 bash[75004]: cluster 2023-09-23T19:29:07.881871+0000 mgr.y (mgr.25325) 640 : cluster 0 pgmap v537: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:08.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:29:07 smithi092 bash[58855]: cluster 2023-09-23T19:29:07.881871+0000 mgr.y (mgr.25325) 640 : cluster 0 pgmap v537: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:10.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:29:10 smithi016 bash[75004]: cluster 2023-09-23T19:29:09.882650+0000 mgr.y (mgr.25325) 641 : cluster 0 pgmap v538: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:10.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:29:10 smithi016 bash[77995]: cluster 2023-09-23T19:29:09.882650+0000 mgr.y (mgr.25325) 641 : cluster 0 pgmap v538: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:10.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:29:10 smithi092 bash[58855]: cluster 2023-09-23T19:29:09.882650+0000 mgr.y (mgr.25325) 641 : cluster 0 pgmap v538: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:12.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:29:11 smithi016 bash[77995]: cluster 2023-09-23T19:29:11.883591+0000 mgr.y (mgr.25325) 642 : cluster 0 pgmap v539: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:12.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:29:11 smithi016 bash[75004]: cluster 2023-09-23T19:29:11.883591+0000 mgr.y (mgr.25325) 642 : cluster 0 pgmap v539: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:12.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:29:11 smithi092 bash[58855]: cluster 2023-09-23T19:29:11.883591+0000 mgr.y (mgr.25325) 642 : cluster 0 pgmap v539: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:14.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:29:13 smithi092 bash[58855]: audit 2023-09-23T19:29:13.632859+0000 mon.a (mon.0) 868 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:29:14.050 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:29:13 smithi016 bash[77995]: audit 2023-09-23T19:29:13.632859+0000 mon.a (mon.0) 868 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:29:14.051 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:29:13 smithi016 bash[75004]: audit 2023-09-23T19:29:13.632859+0000 mon.a (mon.0) 868 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:29:14.117 INFO:teuthology.orchestra.run.smithi016.stdout:206158430524 2023-09-23T19:29:14.118 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-09-23T19:29:14.846 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:29:14 smithi016 bash[75004]: cluster 2023-09-23T19:29:13.884407+0000 mgr.y (mgr.25325) 643 : cluster 0 pgmap v540: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:14.846 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:29:14 smithi016 bash[77995]: cluster 2023-09-23T19:29:13.884407+0000 mgr.y (mgr.25325) 643 : cluster 0 pgmap v540: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:15.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:29:14 smithi092 bash[58855]: cluster 2023-09-23T19:29:13.884407+0000 mgr.y (mgr.25325) 643 : cluster 0 pgmap v540: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:15.956 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:29:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:29:15] "GET /metrics HTTP/1.1" 200 33812 "" "Prometheus/2.33.4" 2023-09-23T19:29:16.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:29:15 smithi016 bash[75004]: cluster 2023-09-23T19:29:15.885502+0000 mgr.y (mgr.25325) 644 : cluster 0 pgmap v541: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:16.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:29:15 smithi016 bash[77995]: cluster 2023-09-23T19:29:15.885502+0000 mgr.y (mgr.25325) 644 : cluster 0 pgmap v541: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:16.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:29:15 smithi092 bash[58855]: cluster 2023-09-23T19:29:15.885502+0000 mgr.y (mgr.25325) 644 : cluster 0 pgmap v541: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:16.726 INFO:teuthology.orchestra.run.smithi016.stdout:365072220294 2023-09-23T19:29:16.726 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-09-23T19:29:17.524 INFO:teuthology.orchestra.run.smithi016.stdout:180388626757 2023-09-23T19:29:17.524 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-09-23T19:29:18.221 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:29:17 smithi016 bash[77995]: cluster 2023-09-23T19:29:17.886323+0000 mgr.y (mgr.25325) 645 : cluster 0 pgmap v542: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:18.221 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:29:17 smithi016 bash[75004]: cluster 2023-09-23T19:29:17.886323+0000 mgr.y (mgr.25325) 645 : cluster 0 pgmap v542: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:18.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:29:17 smithi092 bash[58855]: cluster 2023-09-23T19:29:17.886323+0000 mgr.y (mgr.25325) 645 : cluster 0 pgmap v542: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:18.868 INFO:teuthology.orchestra.run.smithi016.stdout:128849019222 2023-09-23T19:29:18.869 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-09-23T19:29:18.906 INFO:teuthology.orchestra.run.smithi016.stdout:420906795088 2023-09-23T19:29:18.907 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-09-23T19:29:18.908 INFO:teuthology.orchestra.run.smithi016.stdout:154618822989 2023-09-23T19:29:18.908 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-09-23T19:29:19.843 INFO:teuthology.orchestra.run.smithi016.stdout:476741369881 2023-09-23T19:29:19.844 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-09-23T19:29:19.855 INFO:teuthology.orchestra.run.smithi016.stdout:498216206352 2023-09-23T19:29:19.855 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-09-23T19:29:20.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:29:20 smithi016 bash[77995]: cluster 2023-09-23T19:29:19.887157+0000 mgr.y (mgr.25325) 646 : cluster 0 pgmap v543: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:20.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:29:20 smithi016 bash[75004]: cluster 2023-09-23T19:29:19.887157+0000 mgr.y (mgr.25325) 646 : cluster 0 pgmap v543: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:20.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:29:20 smithi092 bash[58855]: cluster 2023-09-23T19:29:19.887157+0000 mgr.y (mgr.25325) 646 : cluster 0 pgmap v543: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:21.930 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:29:21.930 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:29:21.931 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:29:22.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:29:21 smithi016 bash[75004]: cluster 2023-09-23T19:29:21.888345+0000 mgr.y (mgr.25325) 647 : cluster 0 pgmap v544: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:22.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:29:21 smithi016 bash[77995]: cluster 2023-09-23T19:29:21.888345+0000 mgr.y (mgr.25325) 647 : cluster 0 pgmap v544: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:22.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:29:21 smithi092 bash[58855]: cluster 2023-09-23T19:29:21.888345+0000 mgr.y (mgr.25325) 647 : cluster 0 pgmap v544: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:22.946 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:29:22.946 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:29:22.947 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:29:23.959 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:29:23.961 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:29:24.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:29:23 smithi092 bash[58855]: cluster 2023-09-23T19:29:23.888906+0000 mgr.y (mgr.25325) 648 : cluster 0 pgmap v545: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:24.298 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:29:23 smithi016 bash[75004]: cluster 2023-09-23T19:29:23.888906+0000 mgr.y (mgr.25325) 648 : cluster 0 pgmap v545: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:24.298 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:29:23 smithi016 bash[77995]: cluster 2023-09-23T19:29:23.888906+0000 mgr.y (mgr.25325) 648 : cluster 0 pgmap v545: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:25.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:29:25 smithi016 bash[75004]: audit 2023-09-23T19:29:25.589838+0000 mon.a (mon.0) 869 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:29:25.785 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:29:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:29:25] "GET /metrics HTTP/1.1" 200 33812 "" "Prometheus/2.33.4" 2023-09-23T19:29:25.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:29:25 smithi016 bash[77995]: audit 2023-09-23T19:29:25.589838+0000 mon.a (mon.0) 869 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:29:26.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:29:25 smithi092 bash[58855]: audit 2023-09-23T19:29:25.589838+0000 mon.a (mon.0) 869 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:29:26.819 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:29:26 smithi016 bash[75004]: cluster 2023-09-23T19:29:25.889881+0000 mgr.y (mgr.25325) 649 : cluster 0 pgmap v546: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:26.820 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:29:26 smithi016 bash[77995]: cluster 2023-09-23T19:29:25.889881+0000 mgr.y (mgr.25325) 649 : cluster 0 pgmap v546: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:27.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:29:26 smithi092 bash[58855]: cluster 2023-09-23T19:29:25.889881+0000 mgr.y (mgr.25325) 649 : cluster 0 pgmap v546: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:28.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:29:27 smithi016 bash[75004]: cluster 2023-09-23T19:29:27.890471+0000 mgr.y (mgr.25325) 650 : cluster 0 pgmap v547: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:28.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:29:27 smithi016 bash[77995]: cluster 2023-09-23T19:29:27.890471+0000 mgr.y (mgr.25325) 650 : cluster 0 pgmap v547: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:28.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:29:27 smithi092 bash[58855]: cluster 2023-09-23T19:29:27.890471+0000 mgr.y (mgr.25325) 650 : cluster 0 pgmap v547: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:29.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:29:28 smithi016 bash[75004]: audit 2023-09-23T19:29:28.633546+0000 mon.a (mon.0) 870 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:29:29.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:29:28 smithi016 bash[77995]: audit 2023-09-23T19:29:28.633546+0000 mon.a (mon.0) 870 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:29:29.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:29:28 smithi092 bash[58855]: audit 2023-09-23T19:29:28.633546+0000 mon.a (mon.0) 870 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:29:30.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:29:30 smithi092 bash[58855]: cluster 2023-09-23T19:29:29.891012+0000 mgr.y (mgr.25325) 651 : cluster 0 pgmap v548: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:30.587 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:29:30 smithi016 bash[75004]: cluster 2023-09-23T19:29:29.891012+0000 mgr.y (mgr.25325) 651 : cluster 0 pgmap v548: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:30.587 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:29:30 smithi016 bash[77995]: cluster 2023-09-23T19:29:29.891012+0000 mgr.y (mgr.25325) 651 : cluster 0 pgmap v548: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:33.064 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:29:32 smithi016 bash[77995]: audit 2023-09-23T19:29:31.830936+0000 mon.a (mon.0) 871 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:29:33.064 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:29:32 smithi016 bash[77995]: audit 2023-09-23T19:29:31.838468+0000 mon.a (mon.0) 872 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:29:33.064 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:29:32 smithi016 bash[77995]: cluster 2023-09-23T19:29:31.892009+0000 mgr.y (mgr.25325) 652 : cluster 0 pgmap v549: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:33.065 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:29:32 smithi016 bash[75004]: audit 2023-09-23T19:29:31.830936+0000 mon.a (mon.0) 871 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:29:33.065 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:29:32 smithi016 bash[75004]: audit 2023-09-23T19:29:31.838468+0000 mon.a (mon.0) 872 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:29:33.065 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:29:32 smithi016 bash[75004]: cluster 2023-09-23T19:29:31.892009+0000 mgr.y (mgr.25325) 652 : cluster 0 pgmap v549: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:33.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:29:32 smithi092 bash[58855]: audit 2023-09-23T19:29:31.830936+0000 mon.a (mon.0) 871 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:29:33.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:29:32 smithi092 bash[58855]: audit 2023-09-23T19:29:31.838468+0000 mon.a (mon.0) 872 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:29:33.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:29:32 smithi092 bash[58855]: cluster 2023-09-23T19:29:31.892009+0000 mgr.y (mgr.25325) 652 : cluster 0 pgmap v549: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:34.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:29:33 smithi092 bash[58855]: cluster 2023-09-23T19:29:33.893015+0000 mgr.y (mgr.25325) 653 : cluster 0 pgmap v550: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:34.291 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:29:33 smithi016 bash[75004]: cluster 2023-09-23T19:29:33.893015+0000 mgr.y (mgr.25325) 653 : cluster 0 pgmap v550: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:34.291 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:29:33 smithi016 bash[77995]: cluster 2023-09-23T19:29:33.893015+0000 mgr.y (mgr.25325) 653 : cluster 0 pgmap v550: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:35.957 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:29:35 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:29:35] "GET /metrics HTTP/1.1" 200 33813 "" "Prometheus/2.33.4" 2023-09-23T19:29:35.958 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:29:35 smithi016 bash[75004]: cluster 2023-09-23T19:29:35.893692+0000 mgr.y (mgr.25325) 654 : cluster 0 pgmap v551: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:35.994 INFO:teuthology.orchestra.run.smithi016.stdout:206158430529 2023-09-23T19:29:36.114 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:29:35 smithi016 bash[77995]: cluster 2023-09-23T19:29:35.893692+0000 mgr.y (mgr.25325) 654 : cluster 0 pgmap v551: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:36.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:29:35 smithi092 bash[58855]: cluster 2023-09-23T19:29:35.893692+0000 mgr.y (mgr.25325) 654 : cluster 0 pgmap v551: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:37.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:29:36 smithi016 bash[75004]: audit 2023-09-23T19:29:35.992423+0000 mon.c (mon.2) 131 : audit 0 from='client.? 172.21.15.16:0/3814262840' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T19:29:37.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:29:36 smithi016 bash[77995]: audit 2023-09-23T19:29:35.992423+0000 mon.c (mon.2) 131 : audit 0 from='client.? 172.21.15.16:0/3814262840' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T19:29:37.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:29:36 smithi092 bash[58855]: audit 2023-09-23T19:29:35.992423+0000 mon.c (mon.2) 131 : audit 0 from='client.? 172.21.15.16:0/3814262840' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T19:29:37.821 INFO:teuthology.orchestra.run.smithi016.stdout:365072220299 2023-09-23T19:29:38.055 INFO:teuthology.orchestra.run.smithi016.stdout:180388626761 2023-09-23T19:29:38.174 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:29:37 smithi016 bash[75004]: audit 2023-09-23T19:29:37.819091+0000 mon.c (mon.2) 132 : audit 0 from='client.? 172.21.15.16:0/4100108421' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T19:29:38.174 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:29:37 smithi016 bash[75004]: cluster 2023-09-23T19:29:37.894209+0000 mgr.y (mgr.25325) 655 : cluster 0 pgmap v552: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:38.175 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:29:37 smithi016 bash[77995]: audit 2023-09-23T19:29:37.819091+0000 mon.c (mon.2) 132 : audit 0 from='client.? 172.21.15.16:0/4100108421' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T19:29:38.175 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:29:37 smithi016 bash[77995]: cluster 2023-09-23T19:29:37.894209+0000 mgr.y (mgr.25325) 655 : cluster 0 pgmap v552: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:38.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:29:37 smithi092 bash[58855]: audit 2023-09-23T19:29:37.819091+0000 mon.c (mon.2) 132 : audit 0 from='client.? 172.21.15.16:0/4100108421' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T19:29:38.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:29:37 smithi092 bash[58855]: cluster 2023-09-23T19:29:37.894209+0000 mgr.y (mgr.25325) 655 : cluster 0 pgmap v552: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:39.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:29:38 smithi016 bash[75004]: audit 2023-09-23T19:29:38.053557+0000 mon.c (mon.2) 133 : audit 0 from='client.? 172.21.15.16:0/535615520' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T19:29:39.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:29:38 smithi016 bash[77995]: audit 2023-09-23T19:29:38.053557+0000 mon.c (mon.2) 133 : audit 0 from='client.? 172.21.15.16:0/535615520' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T19:29:39.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:29:38 smithi092 bash[58855]: audit 2023-09-23T19:29:38.053557+0000 mon.c (mon.2) 133 : audit 0 from='client.? 172.21.15.16:0/535615520' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T19:29:39.443 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430524 got 206158430529 for osd.7 2023-09-23T19:29:39.444 DEBUG:teuthology.parallel:result is None 2023-09-23T19:29:39.755 INFO:teuthology.orchestra.run.smithi016.stdout:154618822993 2023-09-23T19:29:39.926 INFO:teuthology.orchestra.run.smithi016.stdout:128849019226 2023-09-23T19:29:40.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:29:40 smithi016 bash[75004]: audit 2023-09-23T19:29:39.753070+0000 mon.c (mon.2) 134 : audit 0 from='client.? 172.21.15.16:0/4150192534' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T19:29:40.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:29:40 smithi016 bash[75004]: cluster 2023-09-23T19:29:39.894723+0000 mgr.y (mgr.25325) 656 : cluster 0 pgmap v553: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:40.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:29:40 smithi016 bash[75004]: audit 2023-09-23T19:29:39.924350+0000 mon.a (mon.0) 873 : audit 0 from='client.? 172.21.15.16:0/123991587' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T19:29:40.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:29:40 smithi016 bash[77995]: audit 2023-09-23T19:29:39.753070+0000 mon.c (mon.2) 134 : audit 0 from='client.? 172.21.15.16:0/4150192534' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T19:29:40.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:29:40 smithi016 bash[77995]: cluster 2023-09-23T19:29:39.894723+0000 mgr.y (mgr.25325) 656 : cluster 0 pgmap v553: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:40.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:29:40 smithi016 bash[77995]: audit 2023-09-23T19:29:39.924350+0000 mon.a (mon.0) 873 : audit 0 from='client.? 172.21.15.16:0/123991587' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T19:29:40.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:29:40 smithi092 bash[58855]: audit 2023-09-23T19:29:39.753070+0000 mon.c (mon.2) 134 : audit 0 from='client.? 172.21.15.16:0/4150192534' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T19:29:40.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:29:40 smithi092 bash[58855]: cluster 2023-09-23T19:29:39.894723+0000 mgr.y (mgr.25325) 656 : cluster 0 pgmap v553: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:40.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:29:40 smithi092 bash[58855]: audit 2023-09-23T19:29:39.924350+0000 mon.a (mon.0) 873 : audit 0 from='client.? 172.21.15.16:0/123991587' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T19:29:41.256 INFO:teuthology.orchestra.run.smithi016.stdout:498216206356 2023-09-23T19:29:41.296 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388626757 got 180388626761 for osd.6 2023-09-23T19:29:41.296 DEBUG:teuthology.parallel:result is None 2023-09-23T19:29:41.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:29:41 smithi016 bash[75004]: audit 2023-09-23T19:29:41.254272+0000 mon.a (mon.0) 874 : audit 0 from='client.? 172.21.15.16:0/1197710113' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T19:29:41.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:29:41 smithi016 bash[77995]: audit 2023-09-23T19:29:41.254272+0000 mon.a (mon.0) 874 : audit 0 from='client.? 172.21.15.16:0/1197710113' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T19:29:41.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:29:41 smithi092 bash[58855]: audit 2023-09-23T19:29:41.254272+0000 mon.a (mon.0) 874 : audit 0 from='client.? 172.21.15.16:0/1197710113' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T19:29:41.591 INFO:tasks.cephadm.ceph_manager.ceph:need seq 365072220294 got 365072220299 for osd.0 2023-09-23T19:29:41.592 DEBUG:teuthology.parallel:result is None 2023-09-23T19:29:41.800 INFO:teuthology.orchestra.run.smithi016.stdout:420906795093 2023-09-23T19:29:42.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:29:42 smithi016 bash[75004]: audit 2023-09-23T19:29:41.797939+0000 mon.c (mon.2) 135 : audit 0 from='client.? 172.21.15.16:0/2296470698' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T19:29:42.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:29:42 smithi016 bash[75004]: cluster 2023-09-23T19:29:41.895270+0000 mgr.y (mgr.25325) 657 : cluster 0 pgmap v554: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:42.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:29:42 smithi016 bash[77995]: audit 2023-09-23T19:29:41.797939+0000 mon.c (mon.2) 135 : audit 0 from='client.? 172.21.15.16:0/2296470698' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T19:29:42.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:29:42 smithi016 bash[77995]: cluster 2023-09-23T19:29:41.895270+0000 mgr.y (mgr.25325) 657 : cluster 0 pgmap v554: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:42.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:29:42 smithi092 bash[58855]: audit 2023-09-23T19:29:41.797939+0000 mon.c (mon.2) 135 : audit 0 from='client.? 172.21.15.16:0/2296470698' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T19:29:42.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:29:42 smithi092 bash[58855]: cluster 2023-09-23T19:29:41.895270+0000 mgr.y (mgr.25325) 657 : cluster 0 pgmap v554: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:43.303 INFO:teuthology.orchestra.run.smithi016.stdout:476741369885 2023-09-23T19:29:43.491 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849019222 got 128849019226 for osd.4 2023-09-23T19:29:43.491 DEBUG:teuthology.parallel:result is None 2023-09-23T19:29:43.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:29:43 smithi016 bash[75004]: audit 2023-09-23T19:29:43.300932+0000 mon.a (mon.0) 875 : audit 0 from='client.? 172.21.15.16:0/2512247696' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T19:29:43.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:29:43 smithi016 bash[77995]: audit 2023-09-23T19:29:43.300932+0000 mon.a (mon.0) 875 : audit 0 from='client.? 172.21.15.16:0/2512247696' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T19:29:43.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:29:43 smithi092 bash[58855]: audit 2023-09-23T19:29:43.300932+0000 mon.a (mon.0) 875 : audit 0 from='client.? 172.21.15.16:0/2512247696' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T19:29:43.829 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618822989 got 154618822993 for osd.5 2023-09-23T19:29:43.829 DEBUG:teuthology.parallel:result is None 2023-09-23T19:29:44.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:29:44 smithi016 bash[75004]: audit 2023-09-23T19:29:43.633307+0000 mon.a (mon.0) 876 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:29:44.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:29:44 smithi016 bash[75004]: cluster 2023-09-23T19:29:43.895702+0000 mgr.y (mgr.25325) 658 : cluster 0 pgmap v555: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:44.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:29:44 smithi016 bash[77995]: audit 2023-09-23T19:29:43.633307+0000 mon.a (mon.0) 876 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:29:44.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:29:44 smithi016 bash[77995]: cluster 2023-09-23T19:29:43.895702+0000 mgr.y (mgr.25325) 658 : cluster 0 pgmap v555: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:44.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:29:44 smithi092 bash[58855]: audit 2023-09-23T19:29:43.633307+0000 mon.a (mon.0) 876 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:29:44.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:29:44 smithi092 bash[58855]: cluster 2023-09-23T19:29:43.895702+0000 mgr.y (mgr.25325) 658 : cluster 0 pgmap v555: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:44.908 INFO:tasks.cephadm.ceph_manager.ceph:need seq 498216206352 got 498216206356 for osd.3 2023-09-23T19:29:44.909 DEBUG:teuthology.parallel:result is None 2023-09-23T19:29:44.917 INFO:tasks.cephadm.ceph_manager.ceph:need seq 420906795088 got 420906795093 for osd.1 2023-09-23T19:29:44.917 DEBUG:teuthology.parallel:result is None 2023-09-23T19:29:45.119 INFO:tasks.cephadm.ceph_manager.ceph:need seq 476741369881 got 476741369885 for osd.2 2023-09-23T19:29:45.119 DEBUG:teuthology.parallel:result is None 2023-09-23T19:29:45.120 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-09-23T19:29:45.120 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T19:29:45.959 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:29:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:29:45] "GET /metrics HTTP/1.1" 200 33814 "" "Prometheus/2.33.4" 2023-09-23T19:29:46.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:29:45 smithi016 bash[75004]: cluster 2023-09-23T19:29:45.896493+0000 mgr.y (mgr.25325) 659 : cluster 0 pgmap v556: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:46.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:29:45 smithi016 bash[77995]: cluster 2023-09-23T19:29:45.896493+0000 mgr.y (mgr.25325) 659 : cluster 0 pgmap v556: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:46.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:29:45 smithi092 bash[58855]: cluster 2023-09-23T19:29:45.896493+0000 mgr.y (mgr.25325) 659 : cluster 0 pgmap v556: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:48.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:29:47 smithi016 bash[75004]: cluster 2023-09-23T19:29:47.896971+0000 mgr.y (mgr.25325) 660 : cluster 0 pgmap v557: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:48.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:29:47 smithi016 bash[77995]: cluster 2023-09-23T19:29:47.896971+0000 mgr.y (mgr.25325) 660 : cluster 0 pgmap v557: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:48.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:29:47 smithi092 bash[58855]: cluster 2023-09-23T19:29:47.896971+0000 mgr.y (mgr.25325) 660 : cluster 0 pgmap v557: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:49.457 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:29:49.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:29:49 smithi016 bash[75004]: audit 2023-09-23T19:29:48.627291+0000 mon.a (mon.0) 877 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:29:49.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:29:49 smithi016 bash[75004]: audit 2023-09-23T19:29:48.635245+0000 mon.a (mon.0) 878 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:29:49.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:29:49 smithi016 bash[75004]: audit 2023-09-23T19:29:49.158524+0000 mon.a (mon.0) 879 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:29:49.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:29:49 smithi016 bash[75004]: audit 2023-09-23T19:29:49.160409+0000 mon.a (mon.0) 880 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:29:49.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:29:49 smithi016 bash[75004]: audit 2023-09-23T19:29:49.170306+0000 mon.a (mon.0) 881 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:29:49.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:29:49 smithi016 bash[77995]: audit 2023-09-23T19:29:48.627291+0000 mon.a (mon.0) 877 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:29:49.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:29:49 smithi016 bash[77995]: audit 2023-09-23T19:29:48.635245+0000 mon.a (mon.0) 878 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:29:49.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:29:49 smithi016 bash[77995]: audit 2023-09-23T19:29:49.158524+0000 mon.a (mon.0) 879 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:29:49.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:29:49 smithi016 bash[77995]: audit 2023-09-23T19:29:49.160409+0000 mon.a (mon.0) 880 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:29:49.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:29:49 smithi016 bash[77995]: audit 2023-09-23T19:29:49.170306+0000 mon.a (mon.0) 881 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:29:50.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:29:49 smithi092 bash[58855]: audit 2023-09-23T19:29:48.627291+0000 mon.a (mon.0) 877 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:29:50.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:29:49 smithi092 bash[58855]: audit 2023-09-23T19:29:48.635245+0000 mon.a (mon.0) 878 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:29:50.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:29:49 smithi092 bash[58855]: audit 2023-09-23T19:29:49.158524+0000 mon.a (mon.0) 879 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:29:50.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:29:49 smithi092 bash[58855]: audit 2023-09-23T19:29:49.160409+0000 mon.a (mon.0) 880 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:29:50.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:29:49 smithi092 bash[58855]: audit 2023-09-23T19:29:49.170306+0000 mon.a (mon.0) 881 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:29:50.959 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:29:50 smithi016 bash[77995]: cluster 2023-09-23T19:29:49.897769+0000 mgr.y (mgr.25325) 661 : cluster 0 pgmap v558: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:50.960 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:29:50 smithi016 bash[75004]: cluster 2023-09-23T19:29:49.897769+0000 mgr.y (mgr.25325) 661 : cluster 0 pgmap v558: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:51.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:29:50 smithi092 bash[58855]: cluster 2023-09-23T19:29:49.897769+0000 mgr.y (mgr.25325) 661 : cluster 0 pgmap v558: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:52.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:29:51 smithi016 bash[75004]: cluster 2023-09-23T19:29:51.898427+0000 mgr.y (mgr.25325) 662 : cluster 0 pgmap v559: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:52.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:29:51 smithi016 bash[77995]: cluster 2023-09-23T19:29:51.898427+0000 mgr.y (mgr.25325) 662 : cluster 0 pgmap v559: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:52.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:29:51 smithi092 bash[58855]: cluster 2023-09-23T19:29:51.898427+0000 mgr.y (mgr.25325) 662 : cluster 0 pgmap v559: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:52.861 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-09-23T19:29:52.864 INFO:teuthology.orchestra.run.smithi016.stderr:dumped all 2023-09-23T19:29:53.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:29:52 smithi016 bash[75004]: audit 2023-09-23T19:29:52.840516+0000 mgr.y (mgr.25325) 663 : audit 0 from='client.45584 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:29:53.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:29:52 smithi016 bash[77995]: audit 2023-09-23T19:29:52.840516+0000 mgr.y (mgr.25325) 663 : audit 0 from='client.45584 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:29:53.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:29:52 smithi092 bash[58855]: audit 2023-09-23T19:29:52.840516+0000 mgr.y (mgr.25325) 663 : audit 0 from='client.45584 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:29:53.702 INFO:teuthology.orchestra.run.smithi016.stdout:{"pg_ready":true,"pg_map":{"version":559,"stamp":"2023-09-23T19:29:51.897981+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":261,"num_write_kb":2446,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":195,"ondisk_log_size":195,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2683944,"kb_used_data":10920,"kb_used_omap":20,"kb_used_meta":2672939,"kb_avail":747277272,"statfs":{"total":767960285184,"available":765211926528,"internally_reserved":0,"allocated":11182080,"data_stored":7375395,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":2737090190},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003589"},"pg_stats":[{"pgid":"3.1a","version":"59'1","reported_seq":100,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.878040+0000","last_change":"2023-09-23T19:27:24.274696+0000","last_active":"2023-09-23T19:28:06.878040+0000","last_peered":"2023-09-23T19:28:06.878040+0000","last_clean":"2023-09-23T19:28:06.878040+0000","last_became_active":"2023-09-23T19:27:24.273561+0000","last_became_peered":"2023-09-23T19:27:24.273561+0000","last_unstale":"2023-09-23T19:28:06.878040+0000","last_undegraded":"2023-09-23T19:28:06.878040+0000","last_fullsized":"2023-09-23T19:28:06.878040+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":78,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.894651+0000","last_change":"2023-09-23T19:28:07.894651+0000","last_active":"2023-09-23T19:28:07.894651+0000","last_peered":"2023-09-23T19:28:07.894651+0000","last_clean":"2023-09-23T19:28:07.894651+0000","last_became_active":"2023-09-23T19:28:07.892919+0000","last_became_peered":"2023-09-23T19:28:07.892919+0000","last_unstale":"2023-09-23T19:28:07.894651+0000","last_undegraded":"2023-09-23T19:28:07.894651+0000","last_fullsized":"2023-09-23T19:28:07.894651+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:56.579997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":57,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.890462+0000","last_change":"2023-09-23T19:28:07.890462+0000","last_active":"2023-09-23T19:28:07.890462+0000","last_peered":"2023-09-23T19:28:07.890462+0000","last_clean":"2023-09-23T19:28:07.890462+0000","last_became_active":"2023-09-23T19:28:07.890084+0000","last_became_peered":"2023-09-23T19:28:07.890084+0000","last_unstale":"2023-09-23T19:28:07.890462+0000","last_undegraded":"2023-09-23T19:28:07.890462+0000","last_fullsized":"2023-09-23T19:28:07.890462+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:50.458218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":73,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.885162+0000","last_change":"2023-09-23T19:28:07.884498+0000","last_active":"2023-09-23T19:28:07.885162+0000","last_peered":"2023-09-23T19:28:07.885162+0000","last_clean":"2023-09-23T19:28:07.885162+0000","last_became_active":"2023-09-23T19:28:07.884102+0000","last_became_peered":"2023-09-23T19:28:07.884102+0000","last_unstale":"2023-09-23T19:28:07.885162+0000","last_undegraded":"2023-09-23T19:28:07.885162+0000","last_fullsized":"2023-09-23T19:28:07.885162+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:05:26.348041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":94,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.887122+0000","last_change":"2023-09-23T19:28:07.887122+0000","last_active":"2023-09-23T19:28:07.887122+0000","last_peered":"2023-09-23T19:28:07.887122+0000","last_clean":"2023-09-23T19:28:07.887122+0000","last_became_active":"2023-09-23T19:28:07.886743+0000","last_became_peered":"2023-09-23T19:28:07.886743+0000","last_unstale":"2023-09-23T19:28:07.887122+0000","last_undegraded":"2023-09-23T19:28:07.887122+0000","last_fullsized":"2023-09-23T19:28:07.887122+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:59:45.973619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":83,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.876761+0000","last_change":"2023-09-23T19:22:42.150582+0000","last_active":"2023-09-23T19:28:06.876761+0000","last_peered":"2023-09-23T19:28:06.876761+0000","last_clean":"2023-09-23T19:28:06.876761+0000","last_became_active":"2023-09-23T19:22:42.150229+0000","last_became_peered":"2023-09-23T19:22:42.150229+0000","last_unstale":"2023-09-23T19:28:06.876761+0000","last_undegraded":"2023-09-23T19:28:06.876761+0000","last_fullsized":"2023-09-23T19:28:06.876761+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":72,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.887141+0000","last_change":"2023-09-23T19:28:07.887141+0000","last_active":"2023-09-23T19:28:07.887141+0000","last_peered":"2023-09-23T19:28:07.887141+0000","last_clean":"2023-09-23T19:28:07.887141+0000","last_became_active":"2023-09-23T19:28:07.886494+0000","last_became_peered":"2023-09-23T19:28:07.886494+0000","last_unstale":"2023-09-23T19:28:07.887141+0000","last_undegraded":"2023-09-23T19:28:07.887141+0000","last_fullsized":"2023-09-23T19:28:07.887141+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:24:07.615051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":78,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.890970+0000","last_change":"2023-09-23T19:28:07.890970+0000","last_active":"2023-09-23T19:28:07.890970+0000","last_peered":"2023-09-23T19:28:07.890970+0000","last_clean":"2023-09-23T19:28:07.890970+0000","last_became_active":"2023-09-23T19:28:07.890218+0000","last_became_peered":"2023-09-23T19:28:07.890218+0000","last_unstale":"2023-09-23T19:28:07.890970+0000","last_undegraded":"2023-09-23T19:28:07.890970+0000","last_fullsized":"2023-09-23T19:28:07.890970+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:02:28.214414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":84,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.880173+0000","last_change":"2023-09-23T19:22:42.151617+0000","last_active":"2023-09-23T19:28:06.880173+0000","last_peered":"2023-09-23T19:28:06.880173+0000","last_clean":"2023-09-23T19:28:06.880173+0000","last_became_active":"2023-09-23T19:22:42.151217+0000","last_became_peered":"2023-09-23T19:22:42.151217+0000","last_unstale":"2023-09-23T19:28:06.880173+0000","last_undegraded":"2023-09-23T19:28:06.880173+0000","last_fullsized":"2023-09-23T19:28:06.880173+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":64,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.889639+0000","last_change":"2023-09-23T19:28:07.889639+0000","last_active":"2023-09-23T19:28:07.889639+0000","last_peered":"2023-09-23T19:28:07.889639+0000","last_clean":"2023-09-23T19:28:07.889639+0000","last_became_active":"2023-09-23T19:28:07.889313+0000","last_became_peered":"2023-09-23T19:28:07.889313+0000","last_unstale":"2023-09-23T19:28:07.889639+0000","last_undegraded":"2023-09-23T19:28:07.889639+0000","last_fullsized":"2023-09-23T19:28:07.889639+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:54:03.826337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":87,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.892835+0000","last_change":"2023-09-23T19:28:07.892835+0000","last_active":"2023-09-23T19:28:07.892835+0000","last_peered":"2023-09-23T19:28:07.892835+0000","last_clean":"2023-09-23T19:28:07.892835+0000","last_became_active":"2023-09-23T19:28:07.892338+0000","last_became_peered":"2023-09-23T19:28:07.892338+0000","last_unstale":"2023-09-23T19:28:07.892835+0000","last_undegraded":"2023-09-23T19:28:07.892835+0000","last_fullsized":"2023-09-23T19:28:07.892835+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":58,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.869474+0000","last_change":"2023-09-23T19:27:24.268005+0000","last_active":"2023-09-23T19:28:07.869474+0000","last_peered":"2023-09-23T19:28:07.869474+0000","last_clean":"2023-09-23T19:28:07.869474+0000","last_became_active":"2023-09-23T19:27:24.267513+0000","last_became_peered":"2023-09-23T19:27:24.267513+0000","last_unstale":"2023-09-23T19:28:07.869474+0000","last_undegraded":"2023-09-23T19:28:07.869474+0000","last_fullsized":"2023-09-23T19:28:07.869474+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.961399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'23","reported_seq":92,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.878175+0000","last_change":"2023-09-23T19:05:52.107727+0000","last_active":"2023-09-23T19:28:06.878175+0000","last_peered":"2023-09-23T19:28:06.878175+0000","last_clean":"2023-09-23T19:28:06.878175+0000","last_became_active":"2023-09-23T19:05:52.107378+0000","last_became_peered":"2023-09-23T19:05:52.107378+0000","last_unstale":"2023-09-23T19:28:06.878175+0000","last_undegraded":"2023-09-23T19:28:06.878175+0000","last_fullsized":"2023-09-23T19:28:06.878175+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":83,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.882441+0000","last_change":"2023-09-23T19:22:42.148497+0000","last_active":"2023-09-23T19:28:06.882441+0000","last_peered":"2023-09-23T19:28:06.882441+0000","last_clean":"2023-09-23T19:28:06.882441+0000","last_became_active":"2023-09-23T19:22:42.148147+0000","last_became_peered":"2023-09-23T19:22:42.148147+0000","last_unstale":"2023-09-23T19:28:06.882441+0000","last_undegraded":"2023-09-23T19:28:06.882441+0000","last_fullsized":"2023-09-23T19:28:06.882441+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":86,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.885743+0000","last_change":"2023-09-23T19:27:24.273667+0000","last_active":"2023-09-23T19:28:06.885743+0000","last_peered":"2023-09-23T19:28:06.885743+0000","last_clean":"2023-09-23T19:28:06.885743+0000","last_became_active":"2023-09-23T19:27:24.273234+0000","last_became_peered":"2023-09-23T19:27:24.273234+0000","last_unstale":"2023-09-23T19:28:06.885743+0000","last_undegraded":"2023-09-23T19:28:06.885743+0000","last_fullsized":"2023-09-23T19:28:06.885743+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":78,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.886657+0000","last_change":"2023-09-23T19:28:07.886657+0000","last_active":"2023-09-23T19:28:07.886657+0000","last_peered":"2023-09-23T19:28:07.886657+0000","last_clean":"2023-09-23T19:28:07.886657+0000","last_became_active":"2023-09-23T19:28:07.886210+0000","last_became_peered":"2023-09-23T19:28:07.886210+0000","last_unstale":"2023-09-23T19:28:07.886657+0000","last_undegraded":"2023-09-23T19:28:07.886657+0000","last_fullsized":"2023-09-23T19:28:07.886657+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:21.131194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":73,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.884766+0000","last_change":"2023-09-23T19:28:07.884394+0000","last_active":"2023-09-23T19:28:07.884766+0000","last_peered":"2023-09-23T19:28:07.884766+0000","last_clean":"2023-09-23T19:28:07.884766+0000","last_became_active":"2023-09-23T19:28:07.883944+0000","last_became_peered":"2023-09-23T19:28:07.883944+0000","last_unstale":"2023-09-23T19:28:07.884766+0000","last_undegraded":"2023-09-23T19:28:07.884766+0000","last_fullsized":"2023-09-23T19:28:07.884766+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:08:17.608926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":83,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.891056+0000","last_change":"2023-09-23T19:28:07.891056+0000","last_active":"2023-09-23T19:28:07.891056+0000","last_peered":"2023-09-23T19:28:07.891056+0000","last_clean":"2023-09-23T19:28:07.891056+0000","last_became_active":"2023-09-23T19:28:07.890625+0000","last_became_peered":"2023-09-23T19:28:07.890625+0000","last_unstale":"2023-09-23T19:28:07.891056+0000","last_undegraded":"2023-09-23T19:28:07.891056+0000","last_fullsized":"2023-09-23T19:28:07.891056+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":69,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.885427+0000","last_change":"2023-09-23T19:05:52.114186+0000","last_active":"2023-09-23T19:28:06.885427+0000","last_peered":"2023-09-23T19:28:06.885427+0000","last_clean":"2023-09-23T19:28:06.885427+0000","last_became_active":"2023-09-23T19:05:52.113635+0000","last_became_peered":"2023-09-23T19:05:52.113635+0000","last_unstale":"2023-09-23T19:28:06.885427+0000","last_undegraded":"2023-09-23T19:28:06.885427+0000","last_fullsized":"2023-09-23T19:28:06.885427+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":72,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.884512+0000","last_change":"2023-09-23T19:28:07.884512+0000","last_active":"2023-09-23T19:28:07.884512+0000","last_peered":"2023-09-23T19:28:07.884512+0000","last_clean":"2023-09-23T19:28:07.884512+0000","last_became_active":"2023-09-23T19:28:07.884155+0000","last_became_peered":"2023-09-23T19:28:07.884155+0000","last_unstale":"2023-09-23T19:28:07.884512+0000","last_undegraded":"2023-09-23T19:28:07.884512+0000","last_fullsized":"2023-09-23T19:28:07.884512+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:10.654958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":92,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.876318+0000","last_change":"2023-09-23T19:18:09.010195+0000","last_active":"2023-09-23T19:28:06.876318+0000","last_peered":"2023-09-23T19:28:06.876318+0000","last_clean":"2023-09-23T19:28:06.876318+0000","last_became_active":"2023-09-23T19:18:09.009687+0000","last_became_peered":"2023-09-23T19:18:09.009687+0000","last_unstale":"2023-09-23T19:28:06.876318+0000","last_undegraded":"2023-09-23T19:28:06.876318+0000","last_fullsized":"2023-09-23T19:28:06.876318+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":78,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.885265+0000","last_change":"2023-09-23T19:28:07.885265+0000","last_active":"2023-09-23T19:28:07.885265+0000","last_peered":"2023-09-23T19:28:07.885265+0000","last_clean":"2023-09-23T19:28:07.885265+0000","last_became_active":"2023-09-23T19:28:07.884353+0000","last_became_peered":"2023-09-23T19:28:07.884353+0000","last_unstale":"2023-09-23T19:28:07.885265+0000","last_undegraded":"2023-09-23T19:28:07.885265+0000","last_fullsized":"2023-09-23T19:28:07.885265+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:56.325352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":85,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.886566+0000","last_change":"2023-09-23T19:18:09.035116+0000","last_active":"2023-09-23T19:28:06.886566+0000","last_peered":"2023-09-23T19:28:06.886566+0000","last_clean":"2023-09-23T19:28:06.886566+0000","last_became_active":"2023-09-23T19:18:09.034416+0000","last_became_peered":"2023-09-23T19:18:09.034416+0000","last_unstale":"2023-09-23T19:28:06.886566+0000","last_undegraded":"2023-09-23T19:28:06.886566+0000","last_fullsized":"2023-09-23T19:28:06.886566+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":68,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.879702+0000","last_change":"2023-09-23T19:05:52.789323+0000","last_active":"2023-09-23T19:28:06.879702+0000","last_peered":"2023-09-23T19:28:06.879702+0000","last_clean":"2023-09-23T19:28:06.879702+0000","last_became_active":"2023-09-23T19:05:52.788820+0000","last_became_peered":"2023-09-23T19:05:52.788820+0000","last_unstale":"2023-09-23T19:28:06.879702+0000","last_undegraded":"2023-09-23T19:28:06.879702+0000","last_fullsized":"2023-09-23T19:28:06.879702+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":86,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.885780+0000","last_change":"2023-09-23T19:22:42.151134+0000","last_active":"2023-09-23T19:28:06.885780+0000","last_peered":"2023-09-23T19:28:06.885780+0000","last_clean":"2023-09-23T19:28:06.885780+0000","last_became_active":"2023-09-23T19:22:42.150553+0000","last_became_peered":"2023-09-23T19:22:42.150553+0000","last_unstale":"2023-09-23T19:28:06.885780+0000","last_undegraded":"2023-09-23T19:28:06.885780+0000","last_fullsized":"2023-09-23T19:28:06.885780+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":98,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.899262+0000","last_change":"2023-09-23T19:28:07.899262+0000","last_active":"2023-09-23T19:28:07.899262+0000","last_peered":"2023-09-23T19:28:07.899262+0000","last_clean":"2023-09-23T19:28:07.899262+0000","last_became_active":"2023-09-23T19:28:07.898848+0000","last_became_peered":"2023-09-23T19:28:07.898848+0000","last_unstale":"2023-09-23T19:28:07.899262+0000","last_undegraded":"2023-09-23T19:28:07.899262+0000","last_fullsized":"2023-09-23T19:28:07.899262+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":70,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.885984+0000","last_change":"2023-09-23T19:05:52.114108+0000","last_active":"2023-09-23T19:28:06.885984+0000","last_peered":"2023-09-23T19:28:06.885984+0000","last_clean":"2023-09-23T19:28:06.885984+0000","last_became_active":"2023-09-23T19:05:52.113609+0000","last_became_peered":"2023-09-23T19:05:52.113609+0000","last_unstale":"2023-09-23T19:28:06.885984+0000","last_undegraded":"2023-09-23T19:28:06.885984+0000","last_fullsized":"2023-09-23T19:28:06.885984+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":83,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.875937+0000","last_change":"2023-09-23T19:22:42.156825+0000","last_active":"2023-09-23T19:28:06.875937+0000","last_peered":"2023-09-23T19:28:06.875937+0000","last_clean":"2023-09-23T19:28:06.875937+0000","last_became_active":"2023-09-23T19:22:42.156474+0000","last_became_peered":"2023-09-23T19:22:42.156474+0000","last_unstale":"2023-09-23T19:28:06.875937+0000","last_undegraded":"2023-09-23T19:28:06.875937+0000","last_fullsized":"2023-09-23T19:28:06.875937+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":84,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.891536+0000","last_change":"2023-09-23T19:28:07.891536+0000","last_active":"2023-09-23T19:28:07.891536+0000","last_peered":"2023-09-23T19:28:07.891536+0000","last_clean":"2023-09-23T19:28:07.891536+0000","last_became_active":"2023-09-23T19:28:07.891153+0000","last_became_peered":"2023-09-23T19:28:07.891153+0000","last_unstale":"2023-09-23T19:28:07.891536+0000","last_undegraded":"2023-09-23T19:28:07.891536+0000","last_fullsized":"2023-09-23T19:28:07.891536+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":73,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.874482+0000","last_change":"2023-09-23T19:22:42.135572+0000","last_active":"2023-09-23T19:28:07.874482+0000","last_peered":"2023-09-23T19:28:07.874482+0000","last_clean":"2023-09-23T19:28:07.874482+0000","last_became_active":"2023-09-23T19:22:42.135237+0000","last_became_peered":"2023-09-23T19:22:42.135237+0000","last_unstale":"2023-09-23T19:28:07.874482+0000","last_undegraded":"2023-09-23T19:28:07.874482+0000","last_fullsized":"2023-09-23T19:28:07.874482+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:35.838718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":68,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.893901+0000","last_change":"2023-09-23T19:28:07.893901+0000","last_active":"2023-09-23T19:28:07.893901+0000","last_peered":"2023-09-23T19:28:07.893901+0000","last_clean":"2023-09-23T19:28:07.893901+0000","last_became_active":"2023-09-23T19:28:07.892973+0000","last_became_peered":"2023-09-23T19:28:07.892973+0000","last_unstale":"2023-09-23T19:28:07.893901+0000","last_undegraded":"2023-09-23T19:28:07.893901+0000","last_fullsized":"2023-09-23T19:28:07.893901+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:27.445855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":83,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.896803+0000","last_change":"2023-09-23T19:28:07.896803+0000","last_active":"2023-09-23T19:28:07.896803+0000","last_peered":"2023-09-23T19:28:07.896803+0000","last_clean":"2023-09-23T19:28:07.896803+0000","last_became_active":"2023-09-23T19:28:07.896480+0000","last_became_peered":"2023-09-23T19:28:07.896480+0000","last_unstale":"2023-09-23T19:28:07.896803+0000","last_undegraded":"2023-09-23T19:28:07.896803+0000","last_fullsized":"2023-09-23T19:28:07.896803+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":84,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.876231+0000","last_change":"2023-09-23T19:22:42.156582+0000","last_active":"2023-09-23T19:28:06.876231+0000","last_peered":"2023-09-23T19:28:06.876231+0000","last_clean":"2023-09-23T19:28:06.876231+0000","last_became_active":"2023-09-23T19:22:42.156251+0000","last_became_peered":"2023-09-23T19:22:42.156251+0000","last_unstale":"2023-09-23T19:28:06.876231+0000","last_undegraded":"2023-09-23T19:28:06.876231+0000","last_fullsized":"2023-09-23T19:28:06.876231+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":98,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.879979+0000","last_change":"2023-09-23T19:27:24.273971+0000","last_active":"2023-09-23T19:28:06.879979+0000","last_peered":"2023-09-23T19:28:06.879979+0000","last_clean":"2023-09-23T19:28:06.879979+0000","last_became_active":"2023-09-23T19:27:24.273491+0000","last_became_peered":"2023-09-23T19:27:24.273491+0000","last_unstale":"2023-09-23T19:28:06.879979+0000","last_undegraded":"2023-09-23T19:28:06.879979+0000","last_fullsized":"2023-09-23T19:28:06.879979+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":84,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.878781+0000","last_change":"2023-09-23T19:27:24.274698+0000","last_active":"2023-09-23T19:28:06.878781+0000","last_peered":"2023-09-23T19:28:06.878781+0000","last_clean":"2023-09-23T19:28:06.878781+0000","last_became_active":"2023-09-23T19:27:24.273264+0000","last_became_peered":"2023-09-23T19:27:24.273264+0000","last_unstale":"2023-09-23T19:28:06.878781+0000","last_undegraded":"2023-09-23T19:28:06.878781+0000","last_fullsized":"2023-09-23T19:28:06.878781+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":83,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.878941+0000","last_change":"2023-09-23T19:22:42.149670+0000","last_active":"2023-09-23T19:28:06.878941+0000","last_peered":"2023-09-23T19:28:06.878941+0000","last_clean":"2023-09-23T19:28:06.878941+0000","last_became_active":"2023-09-23T19:22:42.149022+0000","last_became_peered":"2023-09-23T19:22:42.149022+0000","last_unstale":"2023-09-23T19:28:06.878941+0000","last_undegraded":"2023-09-23T19:28:06.878941+0000","last_fullsized":"2023-09-23T19:28:06.878941+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":79,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.886933+0000","last_change":"2023-09-23T19:28:07.886933+0000","last_active":"2023-09-23T19:28:07.886933+0000","last_peered":"2023-09-23T19:28:07.886933+0000","last_clean":"2023-09-23T19:28:07.886933+0000","last_became_active":"2023-09-23T19:28:07.886572+0000","last_became_peered":"2023-09-23T19:28:07.886572+0000","last_unstale":"2023-09-23T19:28:07.886933+0000","last_undegraded":"2023-09-23T19:28:07.886933+0000","last_fullsized":"2023-09-23T19:28:07.886933+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:41.332875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":58,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.889464+0000","last_change":"2023-09-23T19:28:07.889095+0000","last_active":"2023-09-23T19:28:07.889464+0000","last_peered":"2023-09-23T19:28:07.889464+0000","last_clean":"2023-09-23T19:28:07.889464+0000","last_became_active":"2023-09-23T19:28:07.888742+0000","last_became_peered":"2023-09-23T19:28:07.888742+0000","last_unstale":"2023-09-23T19:28:07.889464+0000","last_undegraded":"2023-09-23T19:28:07.889464+0000","last_fullsized":"2023-09-23T19:28:07.889464+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:43.929488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":91,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.870233+0000","last_change":"2023-09-23T19:22:42.143289+0000","last_active":"2023-09-23T19:28:07.870233+0000","last_peered":"2023-09-23T19:28:07.870233+0000","last_clean":"2023-09-23T19:28:07.870233+0000","last_became_active":"2023-09-23T19:22:42.142656+0000","last_became_peered":"2023-09-23T19:22:42.142656+0000","last_unstale":"2023-09-23T19:28:07.870233+0000","last_undegraded":"2023-09-23T19:28:07.870233+0000","last_fullsized":"2023-09-23T19:28:07.870233+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:38.467441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":99,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.890623+0000","last_change":"2023-09-23T19:28:07.890623+0000","last_active":"2023-09-23T19:28:07.890623+0000","last_peered":"2023-09-23T19:28:07.890623+0000","last_clean":"2023-09-23T19:28:07.890623+0000","last_became_active":"2023-09-23T19:28:07.890148+0000","last_became_peered":"2023-09-23T19:28:07.890148+0000","last_unstale":"2023-09-23T19:28:07.890623+0000","last_undegraded":"2023-09-23T19:28:07.890623+0000","last_fullsized":"2023-09-23T19:28:07.890623+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":71,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.865316+0000","last_change":"2023-09-23T19:27:24.269080+0000","last_active":"2023-09-23T19:28:07.865316+0000","last_peered":"2023-09-23T19:28:07.865316+0000","last_clean":"2023-09-23T19:28:07.865316+0000","last_became_active":"2023-09-23T19:27:24.268748+0000","last_became_peered":"2023-09-23T19:27:24.268748+0000","last_unstale":"2023-09-23T19:28:07.865316+0000","last_undegraded":"2023-09-23T19:28:07.865316+0000","last_fullsized":"2023-09-23T19:28:07.865316+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:49.014902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":88,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.880479+0000","last_change":"2023-09-23T19:22:42.153647+0000","last_active":"2023-09-23T19:28:06.880479+0000","last_peered":"2023-09-23T19:28:06.880479+0000","last_clean":"2023-09-23T19:28:06.880479+0000","last_became_active":"2023-09-23T19:22:42.153317+0000","last_became_peered":"2023-09-23T19:22:42.153317+0000","last_unstale":"2023-09-23T19:28:06.880479+0000","last_undegraded":"2023-09-23T19:28:06.880479+0000","last_fullsized":"2023-09-23T19:28:06.880479+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":84,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.882331+0000","last_change":"2023-09-23T19:18:09.017525+0000","last_active":"2023-09-23T19:28:06.882331+0000","last_peered":"2023-09-23T19:28:06.882331+0000","last_clean":"2023-09-23T19:28:06.882331+0000","last_became_active":"2023-09-23T19:18:09.017193+0000","last_became_peered":"2023-09-23T19:18:09.017193+0000","last_unstale":"2023-09-23T19:28:06.882331+0000","last_undegraded":"2023-09-23T19:28:06.882331+0000","last_fullsized":"2023-09-23T19:28:06.882331+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":73,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.879432+0000","last_change":"2023-09-23T19:28:07.875796+0000","last_active":"2023-09-23T19:28:07.879432+0000","last_peered":"2023-09-23T19:28:07.879432+0000","last_clean":"2023-09-23T19:28:07.879432+0000","last_became_active":"2023-09-23T19:28:07.875310+0000","last_became_peered":"2023-09-23T19:28:07.875310+0000","last_unstale":"2023-09-23T19:28:07.879432+0000","last_undegraded":"2023-09-23T19:28:07.879432+0000","last_fullsized":"2023-09-23T19:28:07.879432+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:44:13.379633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":91,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.879332+0000","last_change":"2023-09-23T19:28:07.878688+0000","last_active":"2023-09-23T19:28:07.879332+0000","last_peered":"2023-09-23T19:28:07.879332+0000","last_clean":"2023-09-23T19:28:07.879332+0000","last_became_active":"2023-09-23T19:28:07.878309+0000","last_became_peered":"2023-09-23T19:28:07.878309+0000","last_unstale":"2023-09-23T19:28:07.879332+0000","last_undegraded":"2023-09-23T19:28:07.879332+0000","last_fullsized":"2023-09-23T19:28:07.879332+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:49:06.202657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":58,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.867585+0000","last_change":"2023-09-23T19:27:24.267897+0000","last_active":"2023-09-23T19:28:07.867585+0000","last_peered":"2023-09-23T19:28:07.867585+0000","last_clean":"2023-09-23T19:28:07.867585+0000","last_became_active":"2023-09-23T19:27:24.267535+0000","last_became_peered":"2023-09-23T19:27:24.267535+0000","last_unstale":"2023-09-23T19:28:07.867585+0000","last_undegraded":"2023-09-23T19:28:07.867585+0000","last_fullsized":"2023-09-23T19:28:07.867585+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:13:57.447808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":86,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.874771+0000","last_change":"2023-09-23T19:22:42.143733+0000","last_active":"2023-09-23T19:28:07.874771+0000","last_peered":"2023-09-23T19:28:07.874771+0000","last_clean":"2023-09-23T19:28:07.874771+0000","last_became_active":"2023-09-23T19:22:42.143077+0000","last_became_peered":"2023-09-23T19:22:42.143077+0000","last_unstale":"2023-09-23T19:28:07.874771+0000","last_undegraded":"2023-09-23T19:28:07.874771+0000","last_fullsized":"2023-09-23T19:28:07.874771+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:47:33.086994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":88,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.879925+0000","last_change":"2023-09-23T19:27:24.271520+0000","last_active":"2023-09-23T19:28:06.879925+0000","last_peered":"2023-09-23T19:28:06.879925+0000","last_clean":"2023-09-23T19:28:06.879925+0000","last_became_active":"2023-09-23T19:27:24.271165+0000","last_became_peered":"2023-09-23T19:27:24.271165+0000","last_unstale":"2023-09-23T19:28:06.879925+0000","last_undegraded":"2023-09-23T19:28:06.879925+0000","last_fullsized":"2023-09-23T19:28:06.879925+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":320,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.885321+0000","last_change":"2023-09-23T19:18:09.016297+0000","last_active":"2023-09-23T19:28:06.885321+0000","last_peered":"2023-09-23T19:28:06.885321+0000","last_clean":"2023-09-23T19:28:06.885321+0000","last_became_active":"2023-09-23T19:18:09.015436+0000","last_became_peered":"2023-09-23T19:18:09.015436+0000","last_unstale":"2023-09-23T19:28:06.885321+0000","last_undegraded":"2023-09-23T19:28:06.885321+0000","last_fullsized":"2023-09-23T19:28:06.885321+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:59:33.840751+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:59:33.840751+0000","last_clean_scrub_stamp":"2023-09-23T18:59:33.840751+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":64,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.889756+0000","last_change":"2023-09-23T19:28:07.889756+0000","last_active":"2023-09-23T19:28:07.889756+0000","last_peered":"2023-09-23T19:28:07.889756+0000","last_clean":"2023-09-23T19:28:07.889756+0000","last_became_active":"2023-09-23T19:28:07.889050+0000","last_became_peered":"2023-09-23T19:28:07.889050+0000","last_unstale":"2023-09-23T19:28:07.889756+0000","last_undegraded":"2023-09-23T19:28:07.889756+0000","last_fullsized":"2023-09-23T19:28:07.889756+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:04:55.924553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":68,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.876575+0000","last_change":"2023-09-23T19:05:52.786771+0000","last_active":"2023-09-23T19:28:06.876575+0000","last_peered":"2023-09-23T19:28:06.876575+0000","last_clean":"2023-09-23T19:28:06.876575+0000","last_became_active":"2023-09-23T19:05:52.785797+0000","last_became_peered":"2023-09-23T19:05:52.785797+0000","last_unstale":"2023-09-23T19:28:06.876575+0000","last_undegraded":"2023-09-23T19:28:06.876575+0000","last_fullsized":"2023-09-23T19:28:06.876575+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":103,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.885258+0000","last_change":"2023-09-23T19:22:42.150636+0000","last_active":"2023-09-23T19:28:06.885258+0000","last_peered":"2023-09-23T19:28:06.885258+0000","last_clean":"2023-09-23T19:28:06.885258+0000","last_became_active":"2023-09-23T19:22:42.150045+0000","last_became_peered":"2023-09-23T19:22:42.150045+0000","last_unstale":"2023-09-23T19:28:06.885258+0000","last_undegraded":"2023-09-23T19:28:06.885258+0000","last_fullsized":"2023-09-23T19:28:06.885258+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":73,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.889776+0000","last_change":"2023-09-23T19:28:07.889194+0000","last_active":"2023-09-23T19:28:07.889776+0000","last_peered":"2023-09-23T19:28:07.889776+0000","last_clean":"2023-09-23T19:28:07.889776+0000","last_became_active":"2023-09-23T19:28:07.888823+0000","last_became_peered":"2023-09-23T19:28:07.888823+0000","last_unstale":"2023-09-23T19:28:07.889776+0000","last_undegraded":"2023-09-23T19:28:07.889776+0000","last_fullsized":"2023-09-23T19:28:07.889776+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:23.464688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":88,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.874696+0000","last_change":"2023-09-23T19:27:24.256410+0000","last_active":"2023-09-23T19:28:07.874696+0000","last_peered":"2023-09-23T19:28:07.874696+0000","last_clean":"2023-09-23T19:28:07.874696+0000","last_became_active":"2023-09-23T19:27:24.256052+0000","last_became_peered":"2023-09-23T19:27:24.256052+0000","last_unstale":"2023-09-23T19:28:07.874696+0000","last_undegraded":"2023-09-23T19:28:07.874696+0000","last_fullsized":"2023-09-23T19:28:07.874696+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:11.951400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":88,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.886287+0000","last_change":"2023-09-23T19:18:09.015853+0000","last_active":"2023-09-23T19:28:06.886287+0000","last_peered":"2023-09-23T19:28:06.886287+0000","last_clean":"2023-09-23T19:28:06.886287+0000","last_became_active":"2023-09-23T19:18:09.015338+0000","last_became_peered":"2023-09-23T19:18:09.015338+0000","last_unstale":"2023-09-23T19:28:06.886287+0000","last_undegraded":"2023-09-23T19:28:06.886287+0000","last_fullsized":"2023-09-23T19:28:06.886287+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":58,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.874558+0000","last_change":"2023-09-23T19:27:24.266241+0000","last_active":"2023-09-23T19:28:07.874558+0000","last_peered":"2023-09-23T19:28:07.874558+0000","last_clean":"2023-09-23T19:28:07.874558+0000","last_became_active":"2023-09-23T19:27:24.265764+0000","last_became_peered":"2023-09-23T19:27:24.265764+0000","last_unstale":"2023-09-23T19:28:07.874558+0000","last_undegraded":"2023-09-23T19:28:07.874558+0000","last_fullsized":"2023-09-23T19:28:07.874558+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:07.694732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":58,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.873806+0000","last_change":"2023-09-23T19:18:09.005823+0000","last_active":"2023-09-23T19:28:07.873806+0000","last_peered":"2023-09-23T19:28:07.873806+0000","last_clean":"2023-09-23T19:28:07.873806+0000","last_became_active":"2023-09-23T19:18:09.005207+0000","last_became_peered":"2023-09-23T19:18:09.005207+0000","last_unstale":"2023-09-23T19:28:07.873806+0000","last_undegraded":"2023-09-23T19:28:07.873806+0000","last_fullsized":"2023-09-23T19:28:07.873806+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:05.520188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":68,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.884711+0000","last_change":"2023-09-23T19:28:07.884711+0000","last_active":"2023-09-23T19:28:07.884711+0000","last_peered":"2023-09-23T19:28:07.884711+0000","last_clean":"2023-09-23T19:28:07.884711+0000","last_became_active":"2023-09-23T19:28:07.883878+0000","last_became_peered":"2023-09-23T19:28:07.883878+0000","last_unstale":"2023-09-23T19:28:07.884711+0000","last_undegraded":"2023-09-23T19:28:07.884711+0000","last_fullsized":"2023-09-23T19:28:07.884711+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:55.538826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":43,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.874598+0000","last_change":"2023-09-23T19:22:42.145506+0000","last_active":"2023-09-23T19:28:07.874598+0000","last_peered":"2023-09-23T19:28:07.874598+0000","last_clean":"2023-09-23T19:28:07.874598+0000","last_became_active":"2023-09-23T19:22:42.143991+0000","last_became_peered":"2023-09-23T19:22:42.143991+0000","last_unstale":"2023-09-23T19:28:07.874598+0000","last_undegraded":"2023-09-23T19:28:07.874598+0000","last_fullsized":"2023-09-23T19:28:07.874598+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:56.875175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":65,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.886079+0000","last_change":"2023-09-23T19:28:07.886079+0000","last_active":"2023-09-23T19:28:07.886079+0000","last_peered":"2023-09-23T19:28:07.886079+0000","last_clean":"2023-09-23T19:28:07.886079+0000","last_became_active":"2023-09-23T19:28:07.885209+0000","last_became_peered":"2023-09-23T19:28:07.885209+0000","last_unstale":"2023-09-23T19:28:07.886079+0000","last_undegraded":"2023-09-23T19:28:07.886079+0000","last_fullsized":"2023-09-23T19:28:07.886079+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:19.991162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":88,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.876444+0000","last_change":"2023-09-23T19:27:24.271827+0000","last_active":"2023-09-23T19:28:06.876444+0000","last_peered":"2023-09-23T19:28:06.876444+0000","last_clean":"2023-09-23T19:28:06.876444+0000","last_became_active":"2023-09-23T19:27:24.271328+0000","last_became_peered":"2023-09-23T19:27:24.271328+0000","last_unstale":"2023-09-23T19:28:06.876444+0000","last_undegraded":"2023-09-23T19:28:06.876444+0000","last_fullsized":"2023-09-23T19:28:06.876444+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":76,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.874297+0000","last_change":"2023-09-23T19:22:42.149694+0000","last_active":"2023-09-23T19:28:07.874297+0000","last_peered":"2023-09-23T19:28:07.874297+0000","last_clean":"2023-09-23T19:28:07.874297+0000","last_became_active":"2023-09-23T19:22:42.148194+0000","last_became_peered":"2023-09-23T19:22:42.148194+0000","last_unstale":"2023-09-23T19:28:07.874297+0000","last_undegraded":"2023-09-23T19:28:07.874297+0000","last_fullsized":"2023-09-23T19:28:07.874297+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:47.842383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":68,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.878925+0000","last_change":"2023-09-23T19:05:52.138620+0000","last_active":"2023-09-23T19:28:06.878925+0000","last_peered":"2023-09-23T19:28:06.878925+0000","last_clean":"2023-09-23T19:28:06.878925+0000","last_became_active":"2023-09-23T19:05:52.138243+0000","last_became_peered":"2023-09-23T19:05:52.138243+0000","last_unstale":"2023-09-23T19:28:06.878925+0000","last_undegraded":"2023-09-23T19:28:06.878925+0000","last_fullsized":"2023-09-23T19:28:06.878925+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":60,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.874590+0000","last_change":"2023-09-23T19:18:09.008811+0000","last_active":"2023-09-23T19:28:07.874590+0000","last_peered":"2023-09-23T19:28:07.874590+0000","last_clean":"2023-09-23T19:28:07.874590+0000","last_became_active":"2023-09-23T19:18:09.007255+0000","last_became_peered":"2023-09-23T19:18:09.007255+0000","last_unstale":"2023-09-23T19:28:07.874590+0000","last_undegraded":"2023-09-23T19:28:07.874590+0000","last_fullsized":"2023-09-23T19:28:07.874590+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:25.874038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":98,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.876141+0000","last_change":"2023-09-23T19:28:07.876141+0000","last_active":"2023-09-23T19:28:07.876141+0000","last_peered":"2023-09-23T19:28:07.876141+0000","last_clean":"2023-09-23T19:28:07.876141+0000","last_became_active":"2023-09-23T19:28:07.875802+0000","last_became_peered":"2023-09-23T19:28:07.875802+0000","last_unstale":"2023-09-23T19:28:07.876141+0000","last_undegraded":"2023-09-23T19:28:07.876141+0000","last_fullsized":"2023-09-23T19:28:07.876141+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:38.947881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":86,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.881984+0000","last_change":"2023-09-23T19:22:42.153825+0000","last_active":"2023-09-23T19:28:06.881984+0000","last_peered":"2023-09-23T19:28:06.881984+0000","last_clean":"2023-09-23T19:28:06.881984+0000","last_became_active":"2023-09-23T19:22:42.153416+0000","last_became_peered":"2023-09-23T19:22:42.153416+0000","last_unstale":"2023-09-23T19:28:06.881984+0000","last_undegraded":"2023-09-23T19:28:06.881984+0000","last_fullsized":"2023-09-23T19:28:06.881984+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":107,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.899164+0000","last_change":"2023-09-23T19:28:07.899164+0000","last_active":"2023-09-23T19:28:07.899164+0000","last_peered":"2023-09-23T19:28:07.899164+0000","last_clean":"2023-09-23T19:28:07.899164+0000","last_became_active":"2023-09-23T19:28:07.898812+0000","last_became_peered":"2023-09-23T19:28:07.898812+0000","last_unstale":"2023-09-23T19:28:07.899164+0000","last_undegraded":"2023-09-23T19:28:07.899164+0000","last_fullsized":"2023-09-23T19:28:07.899164+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":71,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.884673+0000","last_change":"2023-09-23T19:05:52.107570+0000","last_active":"2023-09-23T19:28:06.884673+0000","last_peered":"2023-09-23T19:28:06.884673+0000","last_clean":"2023-09-23T19:28:06.884673+0000","last_became_active":"2023-09-23T19:05:52.107215+0000","last_became_peered":"2023-09-23T19:05:52.107215+0000","last_unstale":"2023-09-23T19:28:06.884673+0000","last_undegraded":"2023-09-23T19:28:06.884673+0000","last_fullsized":"2023-09-23T19:28:06.884673+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":98,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.896983+0000","last_change":"2023-09-23T19:28:07.896983+0000","last_active":"2023-09-23T19:28:07.896983+0000","last_peered":"2023-09-23T19:28:07.896983+0000","last_clean":"2023-09-23T19:28:07.896983+0000","last_became_active":"2023-09-23T19:28:07.896577+0000","last_became_peered":"2023-09-23T19:28:07.896577+0000","last_unstale":"2023-09-23T19:28:07.896983+0000","last_undegraded":"2023-09-23T19:28:07.896983+0000","last_fullsized":"2023-09-23T19:28:07.896983+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":92,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.891196+0000","last_change":"2023-09-23T19:28:07.891196+0000","last_active":"2023-09-23T19:28:07.891196+0000","last_peered":"2023-09-23T19:28:07.891196+0000","last_clean":"2023-09-23T19:28:07.891196+0000","last_became_active":"2023-09-23T19:28:07.890484+0000","last_became_peered":"2023-09-23T19:28:07.890484+0000","last_unstale":"2023-09-23T19:28:07.891196+0000","last_undegraded":"2023-09-23T19:28:07.891196+0000","last_fullsized":"2023-09-23T19:28:07.891196+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:24.574716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":105,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.883754+0000","last_change":"2023-09-23T19:28:07.883754+0000","last_active":"2023-09-23T19:28:07.883754+0000","last_peered":"2023-09-23T19:28:07.883754+0000","last_clean":"2023-09-23T19:28:07.883754+0000","last_became_active":"2023-09-23T19:28:07.883363+0000","last_became_peered":"2023-09-23T19:28:07.883363+0000","last_unstale":"2023-09-23T19:28:07.883754+0000","last_undegraded":"2023-09-23T19:28:07.883754+0000","last_fullsized":"2023-09-23T19:28:07.883754+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:56:35.805451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":76,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.874954+0000","last_change":"2023-09-23T19:27:24.267931+0000","last_active":"2023-09-23T19:28:07.874954+0000","last_peered":"2023-09-23T19:28:07.874954+0000","last_clean":"2023-09-23T19:28:07.874954+0000","last_became_active":"2023-09-23T19:27:24.267552+0000","last_became_peered":"2023-09-23T19:27:24.267552+0000","last_unstale":"2023-09-23T19:28:07.874954+0000","last_undegraded":"2023-09-23T19:28:07.874954+0000","last_fullsized":"2023-09-23T19:28:07.874954+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:12.425257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":77,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.878171+0000","last_change":"2023-09-23T19:06:36.673002+0000","last_active":"2023-09-23T19:28:06.878171+0000","last_peered":"2023-09-23T19:28:06.878171+0000","last_clean":"2023-09-23T19:28:06.878171+0000","last_became_active":"2023-09-23T19:06:36.671911+0000","last_became_peered":"2023-09-23T19:06:36.671911+0000","last_unstale":"2023-09-23T19:28:06.878171+0000","last_undegraded":"2023-09-23T19:28:06.878171+0000","last_fullsized":"2023-09-23T19:28:06.878171+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430532,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307128,"kb_used_data":1712,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":93438024,"statfs":{"total":95995035648,"available":95680536576,"internally_reserved":0,"allocated":1753088,"data_stored":1287929,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:29:24 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":1.2130000000000001,"15min":1.2929999999999999},"min":{"1min":0.60199999999999998,"5min":0.27700000000000002,"15min":0.26400000000000001},"max":{"1min":1.3799999999999999,"5min":9.6760000000000002,"15min":9.6760000000000002},"last":0.78500000000000003},{"interface":"front","average":{"1min":0.85099999999999998,"5min":1.109,"15min":1.4219999999999999},"min":{"1min":0.70899999999999996,"5min":0.30299999999999999,"15min":0.28999999999999998},"max":{"1min":1.1040000000000001,"5min":7.8230000000000004,"15min":8.9670000000000005},"last":0.90500000000000003}]},{"osd":1,"last update":"Sat Sep 23 19:28:51 2023","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":1.319,"15min":1.4570000000000001},"min":{"1min":0.66100000000000003,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":1.298,"5min":7.6180000000000003,"15min":7.6180000000000003},"last":1.107},{"interface":"front","average":{"1min":0.996,"5min":1.3640000000000001,"15min":1.45},"min":{"1min":0.77000000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":1.1859999999999999,"5min":7.3739999999999997,"15min":7.3739999999999997},"last":1.0349999999999999}]},{"osd":2,"last update":"Sat Sep 23 19:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.83899999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"min":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"max":{"1min":1.0429999999999999,"5min":1.0449999999999999,"15min":1.0449999999999999},"last":0.99099999999999999},{"interface":"front","average":{"1min":0.998,"5min":0.97699999999999998,"15min":0.97299999999999998},"min":{"1min":0.76300000000000001,"5min":0.622,"15min":0.622},"max":{"1min":1.2170000000000001,"5min":1.294,"15min":1.294},"last":0.86599999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:29:07 2023","interfaces":[{"interface":"back","average":{"1min":1.0580000000000001,"5min":1.0580000000000001,"15min":1.0580000000000001},"min":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"max":{"1min":1.296,"5min":1.296,"15min":1.296},"last":1.2010000000000001},{"interface":"front","average":{"1min":1.1279999999999999,"5min":1.1279999999999999,"15min":1.1279999999999999},"min":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"max":{"1min":1.298,"5min":1.298,"15min":1.298},"last":1.2210000000000001}]},{"osd":4,"last update":"Sat Sep 23 19:28:51 2023","interfaces":[{"interface":"back","average":{"1min":0.89400000000000002,"5min":1.0960000000000001,"15min":1.1899999999999999},"min":{"1min":0.69099999999999995,"5min":0.35399999999999998,"15min":0.28699999999999998},"max":{"1min":1.333,"5min":9.4190000000000005,"15min":9.4190000000000005},"last":0.73799999999999999},{"interface":"front","average":{"1min":1.02,"5min":1.03,"15min":1.1160000000000001},"min":{"1min":0.59799999999999998,"5min":0.36099999999999999,"15min":0.32600000000000001},"max":{"1min":1.26,"5min":4.6769999999999996,"15min":7.9790000000000001},"last":0.92900000000000005}]},{"osd":5,"last update":"Sat Sep 23 19:28:51 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":1.3069999999999999,"15min":1.329},"min":{"1min":0.72599999999999998,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.198,"5min":6.7130000000000001,"15min":9.8580000000000005},"last":1.0549999999999999},{"interface":"front","average":{"1min":1.073,"5min":1.3380000000000001,"15min":1.4419999999999999},"min":{"1min":0.83299999999999996,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":1.2430000000000001,"5min":9.7829999999999995,"15min":10.013999999999999},"last":0.82099999999999995}]},{"osd":6,"last update":"Sat Sep 23 19:28:51 2023","interfaces":[{"interface":"back","average":{"1min":1.0780000000000001,"5min":1.254,"15min":1.3779999999999999},"min":{"1min":0.83799999999999997,"5min":0.41399999999999998,"15min":0.34599999999999997},"max":{"1min":1.292,"5min":5.2320000000000002,"15min":13.462},"last":0.96199999999999997},{"interface":"front","average":{"1min":1.137,"5min":1.0700000000000001,"15min":1.1839999999999999},"min":{"1min":0.84899999999999998,"5min":0.42299999999999999,"15min":0.36799999999999999},"max":{"1min":1.3009999999999999,"5min":3.4649999999999999,"15min":7.9109999999999996},"last":1.145}]}]},{"osd":6,"up_from":42,"seq":180388626764,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307112,"kb_used_data":1696,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":93438040,"statfs":{"total":95995035648,"available":95680552960,"internally_reserved":0,"allocated":1736704,"data_stored":1287468,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:29:36 2023","interfaces":[{"interface":"back","average":{"1min":0.82499999999999996,"5min":1.216,"15min":1.405},"min":{"1min":0.45900000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":1.141,"5min":8.7110000000000003,"15min":8.8989999999999991},"last":1.0640000000000001},{"interface":"front","average":{"1min":0.88700000000000001,"5min":1.256,"15min":1.2310000000000001},"min":{"1min":0.505,"5min":0.36599999999999999,"15min":0.25900000000000001},"max":{"1min":1.2190000000000001,"5min":7.5110000000000001,"15min":7.5110000000000001},"last":0.91200000000000003}]},{"osd":1,"last update":"Sat Sep 23 19:28:49 2023","interfaces":[{"interface":"back","average":{"1min":0.93200000000000005,"5min":1.631,"15min":1.6000000000000001},"min":{"1min":0.69899999999999995,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.1870000000000001,"5min":11.351000000000001,"15min":11.351000000000001},"last":0.71999999999999997},{"interface":"front","average":{"1min":1.038,"5min":1.8029999999999999,"15min":1.7250000000000001},"min":{"1min":0.56100000000000005,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":1.351,"5min":12.728999999999999,"15min":12.728999999999999},"last":1.1020000000000001}]},{"osd":2,"last update":"Sat Sep 23 19:29:26 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":0.93500000000000005,"15min":0.93000000000000005},"min":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"max":{"1min":1.3120000000000001,"5min":1.3120000000000001,"15min":1.3120000000000001},"last":1.032},{"interface":"front","average":{"1min":0.97699999999999998,"5min":0.91200000000000003,"15min":0.90100000000000002},"min":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"max":{"1min":1.1919999999999999,"5min":1.1919999999999999,"15min":1.1919999999999999},"last":0.81299999999999994}]},{"osd":3,"last update":"Sat Sep 23 19:29:09 2023","interfaces":[{"interface":"back","average":{"1min":0.99399999999999999,"5min":0.99399999999999999,"15min":0.99399999999999999},"min":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"max":{"1min":1.212,"5min":1.212,"15min":1.212},"last":1.175},{"interface":"front","average":{"1min":0.96799999999999997,"5min":0.96799999999999997,"15min":0.96799999999999997},"min":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"max":{"1min":1.177,"5min":1.177,"15min":1.177},"last":0.97499999999999998}]},{"osd":4,"last update":"Sat Sep 23 19:29:13 2023","interfaces":[{"interface":"back","average":{"1min":0.90600000000000003,"5min":0.996,"15min":0.999},"min":{"1min":0.42999999999999999,"5min":0.33500000000000002,"15min":0.29299999999999998},"max":{"1min":1.298,"5min":4.6799999999999997,"15min":6.5869999999999997},"last":1.282},{"interface":"front","average":{"1min":1.0249999999999999,"5min":1.1950000000000001,"15min":1.1559999999999999},"min":{"1min":0.53200000000000003,"5min":0.36599999999999999,"15min":0.30499999999999999},"max":{"1min":1.411,"5min":8.5540000000000003,"15min":8.5540000000000003},"last":0.94299999999999995}]},{"osd":5,"last update":"Sat Sep 23 19:29:13 2023","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":1.21,"15min":1.159},"min":{"1min":0.63700000000000001,"5min":0.371,"15min":0.316},"max":{"1min":1.2809999999999999,"5min":4.8869999999999996,"15min":10.398},"last":1.0149999999999999},{"interface":"front","average":{"1min":1.0409999999999999,"5min":1.448,"15min":1.3520000000000001},"min":{"1min":0.66800000000000004,"5min":0.39500000000000002,"15min":0.36499999999999999},"max":{"1min":1.3200000000000001,"5min":8.5820000000000007,"15min":8.6699999999999999},"last":1.089}]},{"osd":7,"last update":"Sat Sep 23 19:29:40 2023","interfaces":[{"interface":"back","average":{"1min":1.0669999999999999,"5min":1.1539999999999999,"15min":1.1419999999999999},"min":{"1min":0.55600000000000005,"5min":0.41799999999999998,"15min":0.38300000000000001},"max":{"1min":1.3740000000000001,"5min":8.7949999999999999,"15min":8.7949999999999999},"last":1.2030000000000001},{"interface":"front","average":{"1min":1.1639999999999999,"5min":1.2290000000000001,"15min":1.3109999999999999},"min":{"1min":0.85099999999999998,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.516,"5min":3.9820000000000002,"15min":6.7400000000000002},"last":1.139}]}]},{"osd":1,"up_from":98,"seq":420906795096,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":252144,"kb_used_data":1192,"kb_used_omap":4,"kb_used_meta":250939,"kb_avail":93493008,"statfs":{"total":95995035648,"available":95736840192,"internally_reserved":0,"allocated":1220608,"data_stored":707168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":256962446},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:28:55 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":1.5209999999999999,"15min":2.2850000000000001},"min":{"1min":0.53500000000000003,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.91200000000000003,"5min":14.99,"15min":14.99},"last":0.86299999999999999},{"interface":"front","average":{"1min":0.78600000000000003,"5min":1.603,"15min":2.2400000000000002},"min":{"1min":0.47699999999999998,"5min":0.26700000000000002,"15min":0.23499999999999999},"max":{"1min":1.1140000000000001,"5min":12.215,"15min":12.215},"last":0.77500000000000002}]},{"osd":2,"last update":"Sat Sep 23 19:29:28 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":0.72799999999999998,"15min":0.73399999999999999},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":1.3200000000000001,"5min":1.3200000000000001,"15min":1.3200000000000001},"last":0.96599999999999997},{"interface":"front","average":{"1min":0.74399999999999999,"5min":0.68799999999999994,"15min":0.67800000000000005},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":1.2350000000000001,"5min":1.2350000000000001,"15min":1.2350000000000001},"last":1.181}]},{"osd":3,"last update":"Sat Sep 23 19:29:07 2023","interfaces":[{"interface":"back","average":{"1min":0.753,"5min":0.753,"15min":0.753},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":1.02,"5min":1.02,"15min":1.02},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"min":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"max":{"1min":1.131,"5min":1.131,"15min":1.131},"last":0.72599999999999998}]},{"osd":4,"last update":"Sat Sep 23 19:28:55 2023","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":1.4790000000000001,"15min":1.696},"min":{"1min":0.753,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":1.323,"5min":7.6130000000000004,"15min":7.6130000000000004},"last":1.103},{"interface":"front","average":{"1min":0.95699999999999996,"5min":1.881,"15min":1.397},"min":{"1min":0.65200000000000002,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.1819999999999999,"5min":10.69,"15min":10.69},"last":0.996}]},{"osd":5,"last update":"Sat Sep 23 19:28:55 2023","interfaces":[{"interface":"back","average":{"1min":0.97699999999999998,"5min":1.9650000000000001,"15min":1.484},"min":{"1min":0.67400000000000004,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":1.1899999999999999,"5min":12.640000000000001,"15min":12.640000000000001},"last":0.83299999999999996},{"interface":"front","average":{"1min":1.0169999999999999,"5min":1.5669999999999999,"15min":1.5960000000000001},"min":{"1min":0.71199999999999997,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":1.4350000000000001,"5min":9.218,"15min":9.218},"last":1.1319999999999999}]},{"osd":6,"last update":"Sat Sep 23 19:28:55 2023","interfaces":[{"interface":"back","average":{"1min":1.0760000000000001,"5min":1.837,"15min":1.6259999999999999},"min":{"1min":0.77400000000000002,"5min":0.45000000000000001,"15min":0.40100000000000002},"max":{"1min":1.5009999999999999,"5min":9.3010000000000002,"15min":9.3010000000000002},"last":1.0489999999999999},{"interface":"front","average":{"1min":1.0529999999999999,"5min":1.8109999999999999,"15min":1.863},"min":{"1min":0.83799999999999997,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.355,"5min":9.1699999999999999,"15min":9.1699999999999999},"last":1.335}]},{"osd":7,"last update":"Sat Sep 23 19:28:55 2023","interfaces":[{"interface":"back","average":{"1min":1.111,"5min":1.649,"15min":2.081},"min":{"1min":0.82599999999999996,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":1.4199999999999999,"5min":9.2159999999999993,"15min":9.2159999999999993},"last":1.274},{"interface":"front","average":{"1min":1.0669999999999999,"5min":1.7210000000000001,"15min":1.5900000000000001},"min":{"1min":0.82599999999999996,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":1.302,"5min":10.742000000000001,"15min":10.742000000000001},"last":1.081}]}]},{"osd":0,"up_from":85,"seq":365072220302,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":310232,"kb_used_data":1808,"kb_used_omap":5,"kb_used_meta":308410,"kb_avail":93434920,"statfs":{"total":95995035648,"available":95677358080,"internally_reserved":0,"allocated":1851392,"data_stored":1304001,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":315812474},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 19:28:52 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":1.8149999999999999,"15min":1.5920000000000001},"min":{"1min":0.30199999999999999,"5min":0.214,"15min":0.214},"max":{"1min":0.97299999999999998,"5min":13.557,"15min":13.557},"last":0.86499999999999999},{"interface":"front","average":{"1min":0.73799999999999999,"5min":1.528,"15min":2.0649999999999999},"min":{"1min":0.42599999999999999,"5min":0.253,"15min":0.253},"max":{"1min":1.079,"5min":11.507999999999999,"15min":11.507999999999999},"last":0.65500000000000003}]},{"osd":2,"last update":"Sat Sep 23 19:29:25 2023","interfaces":[{"interface":"back","average":{"1min":0.70299999999999996,"5min":0.75800000000000001,"15min":0.76700000000000002},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":1.2410000000000001,"5min":1.2410000000000001,"15min":1.2410000000000001},"last":0.748},{"interface":"front","average":{"1min":0.72699999999999998,"5min":0.77500000000000002,"15min":0.78300000000000003},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":1.2330000000000001,"5min":1.2330000000000001,"15min":1.2330000000000001},"last":0.90000000000000002}]},{"osd":3,"last update":"Sat Sep 23 19:29:07 2023","interfaces":[{"interface":"back","average":{"1min":0.754,"5min":0.754,"15min":0.754},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":1.1259999999999999,"5min":1.1259999999999999,"15min":1.1259999999999999},"last":1.087},{"interface":"front","average":{"1min":0.97999999999999998,"5min":0.97999999999999998,"15min":0.97999999999999998},"min":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"max":{"1min":1.5149999999999999,"5min":1.5149999999999999,"15min":1.5149999999999999},"last":1.0149999999999999}]},{"osd":4,"last update":"Sat Sep 23 19:29:33 2023","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":1.2110000000000001,"15min":1.6619999999999999},"min":{"1min":0.54400000000000004,"5min":0.33000000000000002,"15min":0.32100000000000001},"max":{"1min":1.5309999999999999,"5min":6.5659999999999998,"15min":11.156000000000001},"last":0.96999999999999997},{"interface":"front","average":{"1min":1.0900000000000001,"5min":1.4690000000000001,"15min":1.784},"min":{"1min":0.52600000000000002,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.536,"5min":5.8979999999999997,"15min":13.566000000000001},"last":1.1539999999999999}]},{"osd":5,"last update":"Sat Sep 23 19:29:33 2023","interfaces":[{"interface":"back","average":{"1min":1.1220000000000001,"5min":1.6639999999999999,"15min":1.819},"min":{"1min":0.505,"5min":0.371,"15min":0.33600000000000002},"max":{"1min":1.51,"5min":11.553000000000001,"15min":13.509},"last":1.298},{"interface":"front","average":{"1min":1.083,"5min":1.472,"15min":1.804},"min":{"1min":0.628,"5min":0.38400000000000001,"15min":0.33400000000000002},"max":{"1min":1.6160000000000001,"5min":7.0229999999999997,"15min":11.048999999999999},"last":1.2390000000000001}]},{"osd":6,"last update":"Sat Sep 23 19:29:33 2023","interfaces":[{"interface":"back","average":{"1min":1.0620000000000001,"5min":1.3109999999999999,"15min":1.8560000000000001},"min":{"1min":0.64100000000000001,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.669,"5min":7.069,"15min":11.467000000000001},"last":1.198},{"interface":"front","average":{"1min":1.177,"5min":1.601,"15min":1.9119999999999999},"min":{"1min":0.81100000000000005,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":1.5860000000000001,"5min":9.6679999999999993,"15min":13.603999999999999},"last":1.399}]},{"osd":7,"last update":"Sat Sep 23 19:29:33 2023","interfaces":[{"interface":"back","average":{"1min":1.1379999999999999,"5min":1.5329999999999999,"15min":1.9199999999999999},"min":{"1min":0.72199999999999998,"5min":0.44,"15min":0.434},"max":{"1min":1.645,"5min":13.218,"15min":16.271999999999998},"last":1.3340000000000001},{"interface":"front","average":{"1min":1.133,"5min":1.417,"15min":1.861},"min":{"1min":0.67500000000000004,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":1.5620000000000001,"5min":7.101,"15min":11.098000000000001},"last":1.274}]}]},{"osd":2,"up_from":111,"seq":476741369887,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":161480,"kb_used_data":1152,"kb_used_omap":7,"kb_used_meta":160312,"kb_avail":93583672,"statfs":{"total":95995035648,"available":95829680128,"internally_reserved":0,"allocated":1179648,"data_stored":701589,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":164159569},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:29:26 2023","interfaces":[{"interface":"back","average":{"1min":0.71199999999999997,"5min":0.70799999999999996,"15min":0.70699999999999996},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":1.1839999999999999,"5min":1.1839999999999999,"15min":1.1839999999999999},"last":0.94999999999999996},{"interface":"front","average":{"1min":0.73599999999999999,"5min":0.76100000000000001,"15min":0.76500000000000001},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.038,"5min":1.1639999999999999,"15min":1.1639999999999999},"last":0.98499999999999999}]},{"osd":1,"last update":"Sat Sep 23 19:29:26 2023","interfaces":[{"interface":"back","average":{"1min":0.67800000000000005,"5min":0.74199999999999999,"15min":0.753},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":1.0700000000000001,"5min":1.3740000000000001,"15min":1.3740000000000001},"last":0.89800000000000002},{"interface":"front","average":{"1min":0.80100000000000005,"5min":0.84099999999999997,"15min":0.84799999999999998},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":1.097,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":1.2609999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:29:07 2023","interfaces":[{"interface":"back","average":{"1min":0.746,"5min":0.746,"15min":0.746},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":1.3280000000000001,"5min":1.3280000000000001,"15min":1.3280000000000001},"last":0.75},{"interface":"front","average":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":1.5389999999999999,"5min":1.5389999999999999,"15min":1.5389999999999999},"last":0.78100000000000003}]},{"osd":4,"last update":"Sat Sep 23 19:29:26 2023","interfaces":[{"interface":"back","average":{"1min":0.996,"5min":1.0169999999999999,"15min":1.0209999999999999},"min":{"1min":0.60499999999999998,"5min":0.58699999999999997,"15min":0.58699999999999997},"max":{"1min":1.353,"5min":1.353,"15min":1.353},"last":1.1220000000000001},{"interface":"front","average":{"1min":1.0129999999999999,"5min":1.0089999999999999,"15min":1.0089999999999999},"min":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"max":{"1min":1.5,"5min":1.5,"15min":1.5},"last":1.1459999999999999}]},{"osd":5,"last update":"Sat Sep 23 19:29:26 2023","interfaces":[{"interface":"back","average":{"1min":1.0860000000000001,"5min":1.1299999999999999,"15min":1.137},"min":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"max":{"1min":1.371,"5min":1.554,"15min":1.554},"last":1.085},{"interface":"front","average":{"1min":1.0620000000000001,"5min":1.127,"15min":1.1379999999999999},"min":{"1min":0.68200000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"max":{"1min":1.4690000000000001,"5min":1.8340000000000001,"15min":1.8340000000000001},"last":0.85799999999999998}]},{"osd":6,"last update":"Sat Sep 23 19:29:26 2023","interfaces":[{"interface":"back","average":{"1min":0.997,"5min":1.089,"15min":1.105},"min":{"1min":0.64000000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"max":{"1min":1.409,"5min":1.738,"15min":1.738},"last":1.052},{"interface":"front","average":{"1min":1.1539999999999999,"5min":1.167,"15min":1.169},"min":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"max":{"1min":1.6140000000000001,"5min":1.7030000000000001,"15min":1.7030000000000001},"last":1.1970000000000001}]},{"osd":7,"last update":"Sat Sep 23 19:29:26 2023","interfaces":[{"interface":"back","average":{"1min":1.117,"5min":1.1970000000000001,"15min":1.2110000000000001},"min":{"1min":0.79300000000000004,"5min":0.74399999999999999,"15min":0.74399999999999999},"max":{"1min":1.5840000000000001,"5min":1.77,"15min":1.77},"last":1.179},{"interface":"front","average":{"1min":1.131,"5min":1.173,"15min":1.1799999999999999},"min":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"max":{"1min":1.5669999999999999,"5min":1.5669999999999999,"15min":1.5669999999999999},"last":1.23}]}]},{"osd":3,"up_from":116,"seq":498216206359,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":160112,"kb_used_data":1128,"kb_used_omap":2,"kb_used_meta":158973,"kb_avail":93585040,"statfs":{"total":95995035648,"available":95831080960,"internally_reserved":0,"allocated":1155072,"data_stored":697144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":162788405},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:29:08 2023","interfaces":[{"interface":"back","average":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"min":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.71499999999999997},{"interface":"front","average":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"min":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":1.3160000000000001,"5min":1.3160000000000001,"15min":1.3160000000000001},"last":0.996}]},{"osd":1,"last update":"Sat Sep 23 19:29:08 2023","interfaces":[{"interface":"back","average":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"min":{"1min":0.502,"5min":0.502,"15min":0.502},"max":{"1min":1.0629999999999999,"5min":1.0629999999999999,"15min":1.0629999999999999},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"min":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":1.1240000000000001,"5min":1.1240000000000001,"15min":1.1240000000000001},"last":0.86599999999999999}]},{"osd":2,"last update":"Sat Sep 23 19:29:08 2023","interfaces":[{"interface":"back","average":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"min":{"1min":0.495,"5min":0.495,"15min":0.495},"max":{"1min":1.137,"5min":1.137,"15min":1.137},"last":0.77900000000000003},{"interface":"front","average":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"min":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"max":{"1min":1.2849999999999999,"5min":1.2849999999999999,"15min":1.2849999999999999},"last":1.091}]},{"osd":4,"last update":"Sat Sep 23 19:29:08 2023","interfaces":[{"interface":"back","average":{"1min":1.077,"5min":1.077,"15min":1.077},"min":{"1min":0.753,"5min":0.753,"15min":0.753},"max":{"1min":1.454,"5min":1.454,"15min":1.454},"last":1.252},{"interface":"front","average":{"1min":1.149,"5min":1.149,"15min":1.149},"min":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"max":{"1min":1.506,"5min":1.506,"15min":1.506},"last":1.1870000000000001}]},{"osd":5,"last update":"Sat Sep 23 19:29:08 2023","interfaces":[{"interface":"back","average":{"1min":1.1259999999999999,"5min":1.1259999999999999,"15min":1.1259999999999999},"min":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"max":{"1min":1.4059999999999999,"5min":1.4059999999999999,"15min":1.4059999999999999},"last":1.294},{"interface":"front","average":{"1min":1.081,"5min":1.081,"15min":1.081},"min":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"max":{"1min":1.3520000000000001,"5min":1.3520000000000001,"15min":1.3520000000000001},"last":1.0429999999999999}]},{"osd":6,"last update":"Sat Sep 23 19:29:08 2023","interfaces":[{"interface":"back","average":{"1min":1.1459999999999999,"5min":1.1459999999999999,"15min":1.1459999999999999},"min":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"max":{"1min":1.431,"5min":1.431,"15min":1.431},"last":1.3799999999999999},{"interface":"front","average":{"1min":1.1479999999999999,"5min":1.1479999999999999,"15min":1.1479999999999999},"min":{"1min":0.748,"5min":0.748,"15min":0.748},"max":{"1min":1.5109999999999999,"5min":1.5109999999999999,"15min":1.5109999999999999},"last":1.3220000000000001}]},{"osd":7,"last update":"Sat Sep 23 19:29:08 2023","interfaces":[{"interface":"back","average":{"1min":1.1599999999999999,"5min":1.1599999999999999,"15min":1.1599999999999999},"min":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"max":{"1min":1.597,"5min":1.597,"15min":1.597},"last":1.218},{"interface":"front","average":{"1min":1.1559999999999999,"5min":1.1559999999999999,"15min":1.1559999999999999},"min":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"max":{"1min":1.5489999999999999,"5min":1.5489999999999999,"15min":1.5489999999999999},"last":1.345}]}]},{"osd":4,"up_from":30,"seq":128849019229,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":590832,"kb_used_data":1128,"kb_used_omap":0,"kb_used_meta":589696,"kb_avail":93154320,"statfs":{"total":95995035648,"available":95390023680,"internally_reserved":0,"allocated":1155072,"data_stored":697491,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603848704},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:29:30 2023","interfaces":[{"interface":"back","average":{"1min":0.88100000000000001,"5min":1.3260000000000001,"15min":3.306},"min":{"1min":0.61499999999999999,"5min":0.27400000000000002,"15min":0.26200000000000001},"max":{"1min":1.1719999999999999,"5min":8.7680000000000007,"15min":129.16200000000001},"last":0.91700000000000004},{"interface":"front","average":{"1min":0.90400000000000003,"5min":1.244,"15min":3.5640000000000001},"min":{"1min":0.53700000000000003,"5min":0.32200000000000001,"15min":0.27700000000000002},"max":{"1min":1.23,"5min":6.2450000000000001,"15min":130.40600000000001},"last":1.25}]},{"osd":1,"last update":"Sat Sep 23 19:28:55 2023","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":1.6659999999999999,"15min":1.669},"min":{"1min":0.65800000000000003,"5min":0.30099999999999999,"15min":0.29499999999999998},"max":{"1min":1.179,"5min":12.372,"15min":12.372},"last":1.0629999999999999},{"interface":"front","average":{"1min":0.97199999999999998,"5min":1.663,"15min":1.5880000000000001},"min":{"1min":0.54900000000000004,"5min":0.31,"15min":0.31},"max":{"1min":1.274,"5min":7.3360000000000003,"15min":7.3360000000000003},"last":0.996}]},{"osd":2,"last update":"Sat Sep 23 19:29:26 2023","interfaces":[{"interface":"back","average":{"1min":0.92500000000000004,"5min":0.89300000000000002,"15min":0.88800000000000001},"min":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"max":{"1min":1.2290000000000001,"5min":1.327,"15min":1.327},"last":1.171},{"interface":"front","average":{"1min":0.90400000000000003,"5min":0.89900000000000002,"15min":0.89800000000000002},"min":{"1min":0.69699999999999995,"5min":0.51700000000000002,"15min":0.51700000000000002},"max":{"1min":1.1599999999999999,"5min":1.4379999999999999,"15min":1.4379999999999999},"last":1.1080000000000001}]},{"osd":3,"last update":"Sat Sep 23 19:29:10 2023","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":0.97499999999999998,"15min":0.97499999999999998},"min":{"1min":0.622,"5min":0.622,"15min":0.622},"max":{"1min":1.2909999999999999,"5min":1.2909999999999999,"15min":1.2909999999999999},"last":1.151},{"interface":"front","average":{"1min":0.96199999999999997,"5min":0.96199999999999997,"15min":0.96199999999999997},"min":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"max":{"1min":1.552,"5min":1.552,"15min":1.552},"last":1.202}]},{"osd":5,"last update":"Sat Sep 23 19:29:42 2023","interfaces":[{"interface":"back","average":{"1min":1.018,"5min":1.4299999999999999,"15min":1.1279999999999999},"min":{"1min":0.57699999999999996,"5min":0.32500000000000001,"15min":0.29199999999999998},"max":{"1min":2.1709999999999998,"5min":12.459,"15min":12.459},"last":0.82199999999999995},{"interface":"front","average":{"1min":1.1220000000000001,"5min":1.4730000000000001,"15min":1.234},"min":{"1min":0.877,"5min":0.34699999999999998,"15min":0.26700000000000002},"max":{"1min":1.8680000000000001,"5min":12.478,"15min":12.478},"last":1.272}]},{"osd":6,"last update":"Sat Sep 23 19:28:59 2023","interfaces":[{"interface":"back","average":{"1min":0.98699999999999999,"5min":1.6220000000000001,"15min":1.2609999999999999},"min":{"1min":0.46600000000000003,"5min":0.377,"15min":0.29999999999999999},"max":{"1min":1.397,"5min":7.0810000000000004,"15min":7.0810000000000004},"last":1.2370000000000001},{"interface":"front","average":{"1min":1.1120000000000001,"5min":1.4390000000000001,"15min":1.248},"min":{"1min":0.626,"5min":0.40600000000000003,"15min":0.34300000000000003},"max":{"1min":1.4950000000000001,"5min":4.3440000000000003,"15min":6.9050000000000002},"last":1.3300000000000001}]},{"osd":7,"last update":"Sat Sep 23 19:28:55 2023","interfaces":[{"interface":"back","average":{"1min":0.98199999999999998,"5min":1.4059999999999999,"15min":1.244},"min":{"1min":0.55400000000000005,"5min":0.41599999999999998,"15min":0.312},"max":{"1min":1.4430000000000001,"5min":7.8479999999999999,"15min":7.8479999999999999},"last":1.3540000000000001},{"interface":"front","average":{"1min":1.0569999999999999,"5min":1.3080000000000001,"15min":1.147},"min":{"1min":0.58499999999999996,"5min":0.40899999999999997,"15min":0.34999999999999998},"max":{"1min":1.538,"5min":5.8570000000000002,"15min":8.3460000000000001},"last":1.3100000000000001}]}]},{"osd":5,"up_from":36,"seq":154618822996,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":594904,"kb_used_data":1104,"kb_used_omap":0,"kb_used_meta":593792,"kb_avail":93150248,"statfs":{"total":95995035648,"available":95385853952,"internally_reserved":0,"allocated":1130496,"data_stored":692605,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":608043008},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.88100000000000001,"5min":1.0649999999999999,"15min":1.3360000000000001},"min":{"1min":0.49299999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.2360000000000002,"5min":7.6669999999999998,"15min":11.058},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.98899999999999999,"5min":1.478,"15min":1.7050000000000001},"min":{"1min":0.54000000000000004,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":3.7890000000000001,"5min":10.802,"15min":10.802},"last":0.78800000000000003}]},{"osd":1,"last update":"Sat Sep 23 19:28:55 2023","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":1.385,"15min":1.28},"min":{"1min":0.57299999999999995,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":1.869,"5min":9.5389999999999997,"15min":9.5389999999999997},"last":0.94199999999999995},{"interface":"front","average":{"1min":0.95399999999999996,"5min":1.579,"15min":2.1920000000000002},"min":{"1min":0.67400000000000004,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":1.7829999999999999,"5min":11.228999999999999,"15min":11.228999999999999},"last":0.96599999999999997}]},{"osd":2,"last update":"Sat Sep 23 19:29:29 2023","interfaces":[{"interface":"back","average":{"1min":1.0700000000000001,"5min":0.98099999999999998,"15min":0.96599999999999997},"min":{"1min":0.65100000000000002,"5min":0.56899999999999995,"15min":0.56899999999999995},"max":{"1min":2.3069999999999999,"5min":2.3069999999999999,"15min":2.3069999999999999},"last":0.85699999999999998},{"interface":"front","average":{"1min":1.151,"5min":1.0680000000000001,"15min":1.054},"min":{"1min":0.78800000000000003,"5min":0.753,"15min":0.753},"max":{"1min":2.335,"5min":2.335,"15min":2.335},"last":0.72499999999999998}]},{"osd":3,"last update":"Sat Sep 23 19:29:09 2023","interfaces":[{"interface":"back","average":{"1min":1.0620000000000001,"5min":1.0620000000000001,"15min":1.0620000000000001},"min":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"max":{"1min":1.702,"5min":1.702,"15min":1.702},"last":1.048},{"interface":"front","average":{"1min":1.1739999999999999,"5min":1.1739999999999999,"15min":1.1739999999999999},"min":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"max":{"1min":2.5179999999999998,"5min":2.5179999999999998,"15min":2.5179999999999998},"last":0.81200000000000006}]},{"osd":4,"last update":"Sat Sep 23 19:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":1.244,"15min":1.1519999999999999},"min":{"1min":0.63600000000000001,"5min":0.33200000000000002,"15min":0.29199999999999998},"max":{"1min":1.2669999999999999,"5min":8.1790000000000003,"15min":13.753},"last":0.68400000000000005},{"interface":"front","average":{"1min":0.96899999999999997,"5min":1.3029999999999999,"15min":1.137},"min":{"1min":0.72199999999999998,"5min":0.34399999999999997,"15min":0.27000000000000002},"max":{"1min":1.444,"5min":13.714,"15min":13.714},"last":1.097}]},{"osd":6,"last update":"Sat Sep 23 19:29:14 2023","interfaces":[{"interface":"back","average":{"1min":1.0860000000000001,"5min":1.2829999999999999,"15min":1.246},"min":{"1min":0.68899999999999995,"5min":0.36899999999999999,"15min":0.28599999999999998},"max":{"1min":1.4730000000000001,"5min":7.3739999999999997,"15min":26.448},"last":1.083},{"interface":"front","average":{"1min":1.149,"5min":1.3959999999999999,"15min":1.298},"min":{"1min":0.76900000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.6990000000000001,"5min":4.8399999999999999,"15min":26.841999999999999},"last":1.1419999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:29:47 2023","interfaces":[{"interface":"back","average":{"1min":1.161,"5min":1.1379999999999999,"15min":1.361},"min":{"1min":0.77000000000000002,"5min":0.41399999999999998,"15min":0.36599999999999999},"max":{"1min":1.794,"5min":4.7990000000000004,"15min":32.585999999999999},"last":1.002},{"interface":"front","average":{"1min":1.1850000000000001,"5min":1.145,"15min":1.25},"min":{"1min":0.80400000000000005,"5min":0.39800000000000002,"15min":0.35199999999999998},"max":{"1min":1.542,"5min":7.7050000000000001,"15min":33.014000000000003},"last":1.0229999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T19:29:53.707 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T19:29:54.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:29:53 smithi016 bash[75004]: cluster 2023-09-23T19:29:53.899107+0000 mgr.y (mgr.25325) 664 : cluster 0 pgmap v560: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:54.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:29:53 smithi016 bash[77995]: cluster 2023-09-23T19:29:53.899107+0000 mgr.y (mgr.25325) 664 : cluster 0 pgmap v560: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:54.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:29:53 smithi092 bash[58855]: cluster 2023-09-23T19:29:53.899107+0000 mgr.y (mgr.25325) 664 : cluster 0 pgmap v560: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:55.965 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:29:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:29:55] "GET /metrics HTTP/1.1" 200 33814 "" "Prometheus/2.33.4" 2023-09-23T19:29:56.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:29:55 smithi016 bash[77995]: cluster 2023-09-23T19:29:55.900062+0000 mgr.y (mgr.25325) 665 : cluster 0 pgmap v561: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:56.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:29:55 smithi016 bash[75004]: cluster 2023-09-23T19:29:55.900062+0000 mgr.y (mgr.25325) 665 : cluster 0 pgmap v561: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:56.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:29:55 smithi092 bash[58855]: cluster 2023-09-23T19:29:55.900062+0000 mgr.y (mgr.25325) 665 : cluster 0 pgmap v561: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:58.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:29:57 smithi016 bash[75004]: cluster 2023-09-23T19:29:57.900849+0000 mgr.y (mgr.25325) 666 : cluster 0 pgmap v562: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:58.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:29:57 smithi016 bash[77995]: cluster 2023-09-23T19:29:57.900849+0000 mgr.y (mgr.25325) 666 : cluster 0 pgmap v562: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:58.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:29:57 smithi092 bash[58855]: cluster 2023-09-23T19:29:57.900849+0000 mgr.y (mgr.25325) 666 : cluster 0 pgmap v562: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:29:58.524 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:29:59.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:29:58 smithi016 bash[75004]: audit 2023-09-23T19:29:58.634614+0000 mon.a (mon.0) 882 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:29:59.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:29:58 smithi016 bash[77995]: audit 2023-09-23T19:29:58.634614+0000 mon.a (mon.0) 882 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:29:59.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:29:58 smithi092 bash[58855]: audit 2023-09-23T19:29:58.634614+0000 mon.a (mon.0) 882 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:30:00.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:30:00 smithi016 bash[75004]: cluster 2023-09-23T19:29:59.901410+0000 mgr.y (mgr.25325) 667 : cluster 0 pgmap v563: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:00.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:30:00 smithi016 bash[75004]: cluster 2023-09-23T19:30:00.000134+0000 mon.a (mon.0) 883 : cluster 1 overall HEALTH_OK 2023-09-23T19:30:00.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:30:00 smithi016 bash[77995]: cluster 2023-09-23T19:29:59.901410+0000 mgr.y (mgr.25325) 667 : cluster 0 pgmap v563: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:00.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:30:00 smithi016 bash[77995]: cluster 2023-09-23T19:30:00.000134+0000 mon.a (mon.0) 883 : cluster 1 overall HEALTH_OK 2023-09-23T19:30:00.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:30:00 smithi092 bash[58855]: cluster 2023-09-23T19:29:59.901410+0000 mgr.y (mgr.25325) 667 : cluster 0 pgmap v563: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:00.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:30:00 smithi092 bash[58855]: cluster 2023-09-23T19:30:00.000134+0000 mon.a (mon.0) 883 : cluster 1 overall HEALTH_OK 2023-09-23T19:30:01.846 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-09-23T19:30:01.850 INFO:teuthology.orchestra.run.smithi016.stderr:dumped all 2023-09-23T19:30:02.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:30:01 smithi016 bash[77995]: audit 2023-09-23T19:30:01.826069+0000 mgr.y (mgr.25325) 668 : audit 0 from='client.35679 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:30:02.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:30:01 smithi016 bash[77995]: cluster 2023-09-23T19:30:01.902224+0000 mgr.y (mgr.25325) 669 : cluster 0 pgmap v564: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:02.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:30:01 smithi016 bash[75004]: audit 2023-09-23T19:30:01.826069+0000 mgr.y (mgr.25325) 668 : audit 0 from='client.35679 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:30:02.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:30:01 smithi016 bash[75004]: cluster 2023-09-23T19:30:01.902224+0000 mgr.y (mgr.25325) 669 : cluster 0 pgmap v564: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:02.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:30:01 smithi092 bash[58855]: audit 2023-09-23T19:30:01.826069+0000 mgr.y (mgr.25325) 668 : audit 0 from='client.35679 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:30:02.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:30:01 smithi092 bash[58855]: cluster 2023-09-23T19:30:01.902224+0000 mgr.y (mgr.25325) 669 : cluster 0 pgmap v564: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:02.675 INFO:teuthology.orchestra.run.smithi016.stdout:{"pg_ready":true,"pg_map":{"version":563,"stamp":"2023-09-23T19:29:59.901084+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":261,"num_write_kb":2446,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":195,"ondisk_log_size":195,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2683944,"kb_used_data":10920,"kb_used_omap":20,"kb_used_meta":2672939,"kb_avail":747277272,"statfs":{"total":767960285184,"available":765211926528,"internally_reserved":0,"allocated":11182080,"data_stored":7375395,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":2737090190},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004390"},"pg_stats":[{"pgid":"3.1a","version":"59'1","reported_seq":100,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.878040+0000","last_change":"2023-09-23T19:27:24.274696+0000","last_active":"2023-09-23T19:28:06.878040+0000","last_peered":"2023-09-23T19:28:06.878040+0000","last_clean":"2023-09-23T19:28:06.878040+0000","last_became_active":"2023-09-23T19:27:24.273561+0000","last_became_peered":"2023-09-23T19:27:24.273561+0000","last_unstale":"2023-09-23T19:28:06.878040+0000","last_undegraded":"2023-09-23T19:28:06.878040+0000","last_fullsized":"2023-09-23T19:28:06.878040+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":78,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.894651+0000","last_change":"2023-09-23T19:28:07.894651+0000","last_active":"2023-09-23T19:28:07.894651+0000","last_peered":"2023-09-23T19:28:07.894651+0000","last_clean":"2023-09-23T19:28:07.894651+0000","last_became_active":"2023-09-23T19:28:07.892919+0000","last_became_peered":"2023-09-23T19:28:07.892919+0000","last_unstale":"2023-09-23T19:28:07.894651+0000","last_undegraded":"2023-09-23T19:28:07.894651+0000","last_fullsized":"2023-09-23T19:28:07.894651+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:56.579997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":58,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.890735+0000","last_change":"2023-09-23T19:28:07.890462+0000","last_active":"2023-09-23T19:28:07.890735+0000","last_peered":"2023-09-23T19:28:07.890735+0000","last_clean":"2023-09-23T19:28:07.890735+0000","last_became_active":"2023-09-23T19:28:07.890084+0000","last_became_peered":"2023-09-23T19:28:07.890084+0000","last_unstale":"2023-09-23T19:28:07.890735+0000","last_undegraded":"2023-09-23T19:28:07.890735+0000","last_fullsized":"2023-09-23T19:28:07.890735+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:50.458218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":73,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.885162+0000","last_change":"2023-09-23T19:28:07.884498+0000","last_active":"2023-09-23T19:28:07.885162+0000","last_peered":"2023-09-23T19:28:07.885162+0000","last_clean":"2023-09-23T19:28:07.885162+0000","last_became_active":"2023-09-23T19:28:07.884102+0000","last_became_peered":"2023-09-23T19:28:07.884102+0000","last_unstale":"2023-09-23T19:28:07.885162+0000","last_undegraded":"2023-09-23T19:28:07.885162+0000","last_fullsized":"2023-09-23T19:28:07.885162+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:05:26.348041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":94,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.887122+0000","last_change":"2023-09-23T19:28:07.887122+0000","last_active":"2023-09-23T19:28:07.887122+0000","last_peered":"2023-09-23T19:28:07.887122+0000","last_clean":"2023-09-23T19:28:07.887122+0000","last_became_active":"2023-09-23T19:28:07.886743+0000","last_became_peered":"2023-09-23T19:28:07.886743+0000","last_unstale":"2023-09-23T19:28:07.887122+0000","last_undegraded":"2023-09-23T19:28:07.887122+0000","last_fullsized":"2023-09-23T19:28:07.887122+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:59:45.973619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":83,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.876761+0000","last_change":"2023-09-23T19:22:42.150582+0000","last_active":"2023-09-23T19:28:06.876761+0000","last_peered":"2023-09-23T19:28:06.876761+0000","last_clean":"2023-09-23T19:28:06.876761+0000","last_became_active":"2023-09-23T19:22:42.150229+0000","last_became_peered":"2023-09-23T19:22:42.150229+0000","last_unstale":"2023-09-23T19:28:06.876761+0000","last_undegraded":"2023-09-23T19:28:06.876761+0000","last_fullsized":"2023-09-23T19:28:06.876761+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":72,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.887141+0000","last_change":"2023-09-23T19:28:07.887141+0000","last_active":"2023-09-23T19:28:07.887141+0000","last_peered":"2023-09-23T19:28:07.887141+0000","last_clean":"2023-09-23T19:28:07.887141+0000","last_became_active":"2023-09-23T19:28:07.886494+0000","last_became_peered":"2023-09-23T19:28:07.886494+0000","last_unstale":"2023-09-23T19:28:07.887141+0000","last_undegraded":"2023-09-23T19:28:07.887141+0000","last_fullsized":"2023-09-23T19:28:07.887141+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:24:07.615051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":78,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.890970+0000","last_change":"2023-09-23T19:28:07.890970+0000","last_active":"2023-09-23T19:28:07.890970+0000","last_peered":"2023-09-23T19:28:07.890970+0000","last_clean":"2023-09-23T19:28:07.890970+0000","last_became_active":"2023-09-23T19:28:07.890218+0000","last_became_peered":"2023-09-23T19:28:07.890218+0000","last_unstale":"2023-09-23T19:28:07.890970+0000","last_undegraded":"2023-09-23T19:28:07.890970+0000","last_fullsized":"2023-09-23T19:28:07.890970+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:02:28.214414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":84,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.880173+0000","last_change":"2023-09-23T19:22:42.151617+0000","last_active":"2023-09-23T19:28:06.880173+0000","last_peered":"2023-09-23T19:28:06.880173+0000","last_clean":"2023-09-23T19:28:06.880173+0000","last_became_active":"2023-09-23T19:22:42.151217+0000","last_became_peered":"2023-09-23T19:22:42.151217+0000","last_unstale":"2023-09-23T19:28:06.880173+0000","last_undegraded":"2023-09-23T19:28:06.880173+0000","last_fullsized":"2023-09-23T19:28:06.880173+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":64,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.889639+0000","last_change":"2023-09-23T19:28:07.889639+0000","last_active":"2023-09-23T19:28:07.889639+0000","last_peered":"2023-09-23T19:28:07.889639+0000","last_clean":"2023-09-23T19:28:07.889639+0000","last_became_active":"2023-09-23T19:28:07.889313+0000","last_became_peered":"2023-09-23T19:28:07.889313+0000","last_unstale":"2023-09-23T19:28:07.889639+0000","last_undegraded":"2023-09-23T19:28:07.889639+0000","last_fullsized":"2023-09-23T19:28:07.889639+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:54:03.826337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":87,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.892835+0000","last_change":"2023-09-23T19:28:07.892835+0000","last_active":"2023-09-23T19:28:07.892835+0000","last_peered":"2023-09-23T19:28:07.892835+0000","last_clean":"2023-09-23T19:28:07.892835+0000","last_became_active":"2023-09-23T19:28:07.892338+0000","last_became_peered":"2023-09-23T19:28:07.892338+0000","last_unstale":"2023-09-23T19:28:07.892835+0000","last_undegraded":"2023-09-23T19:28:07.892835+0000","last_fullsized":"2023-09-23T19:28:07.892835+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":58,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.869474+0000","last_change":"2023-09-23T19:27:24.268005+0000","last_active":"2023-09-23T19:28:07.869474+0000","last_peered":"2023-09-23T19:28:07.869474+0000","last_clean":"2023-09-23T19:28:07.869474+0000","last_became_active":"2023-09-23T19:27:24.267513+0000","last_became_peered":"2023-09-23T19:27:24.267513+0000","last_unstale":"2023-09-23T19:28:07.869474+0000","last_undegraded":"2023-09-23T19:28:07.869474+0000","last_fullsized":"2023-09-23T19:28:07.869474+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.961399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'23","reported_seq":92,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.878175+0000","last_change":"2023-09-23T19:05:52.107727+0000","last_active":"2023-09-23T19:28:06.878175+0000","last_peered":"2023-09-23T19:28:06.878175+0000","last_clean":"2023-09-23T19:28:06.878175+0000","last_became_active":"2023-09-23T19:05:52.107378+0000","last_became_peered":"2023-09-23T19:05:52.107378+0000","last_unstale":"2023-09-23T19:28:06.878175+0000","last_undegraded":"2023-09-23T19:28:06.878175+0000","last_fullsized":"2023-09-23T19:28:06.878175+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":83,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.882441+0000","last_change":"2023-09-23T19:22:42.148497+0000","last_active":"2023-09-23T19:28:06.882441+0000","last_peered":"2023-09-23T19:28:06.882441+0000","last_clean":"2023-09-23T19:28:06.882441+0000","last_became_active":"2023-09-23T19:22:42.148147+0000","last_became_peered":"2023-09-23T19:22:42.148147+0000","last_unstale":"2023-09-23T19:28:06.882441+0000","last_undegraded":"2023-09-23T19:28:06.882441+0000","last_fullsized":"2023-09-23T19:28:06.882441+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":86,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.885743+0000","last_change":"2023-09-23T19:27:24.273667+0000","last_active":"2023-09-23T19:28:06.885743+0000","last_peered":"2023-09-23T19:28:06.885743+0000","last_clean":"2023-09-23T19:28:06.885743+0000","last_became_active":"2023-09-23T19:27:24.273234+0000","last_became_peered":"2023-09-23T19:27:24.273234+0000","last_unstale":"2023-09-23T19:28:06.885743+0000","last_undegraded":"2023-09-23T19:28:06.885743+0000","last_fullsized":"2023-09-23T19:28:06.885743+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":78,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.886657+0000","last_change":"2023-09-23T19:28:07.886657+0000","last_active":"2023-09-23T19:28:07.886657+0000","last_peered":"2023-09-23T19:28:07.886657+0000","last_clean":"2023-09-23T19:28:07.886657+0000","last_became_active":"2023-09-23T19:28:07.886210+0000","last_became_peered":"2023-09-23T19:28:07.886210+0000","last_unstale":"2023-09-23T19:28:07.886657+0000","last_undegraded":"2023-09-23T19:28:07.886657+0000","last_fullsized":"2023-09-23T19:28:07.886657+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:21.131194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":73,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.884766+0000","last_change":"2023-09-23T19:28:07.884394+0000","last_active":"2023-09-23T19:28:07.884766+0000","last_peered":"2023-09-23T19:28:07.884766+0000","last_clean":"2023-09-23T19:28:07.884766+0000","last_became_active":"2023-09-23T19:28:07.883944+0000","last_became_peered":"2023-09-23T19:28:07.883944+0000","last_unstale":"2023-09-23T19:28:07.884766+0000","last_undegraded":"2023-09-23T19:28:07.884766+0000","last_fullsized":"2023-09-23T19:28:07.884766+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:08:17.608926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":83,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.891056+0000","last_change":"2023-09-23T19:28:07.891056+0000","last_active":"2023-09-23T19:28:07.891056+0000","last_peered":"2023-09-23T19:28:07.891056+0000","last_clean":"2023-09-23T19:28:07.891056+0000","last_became_active":"2023-09-23T19:28:07.890625+0000","last_became_peered":"2023-09-23T19:28:07.890625+0000","last_unstale":"2023-09-23T19:28:07.891056+0000","last_undegraded":"2023-09-23T19:28:07.891056+0000","last_fullsized":"2023-09-23T19:28:07.891056+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":69,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.885427+0000","last_change":"2023-09-23T19:05:52.114186+0000","last_active":"2023-09-23T19:28:06.885427+0000","last_peered":"2023-09-23T19:28:06.885427+0000","last_clean":"2023-09-23T19:28:06.885427+0000","last_became_active":"2023-09-23T19:05:52.113635+0000","last_became_peered":"2023-09-23T19:05:52.113635+0000","last_unstale":"2023-09-23T19:28:06.885427+0000","last_undegraded":"2023-09-23T19:28:06.885427+0000","last_fullsized":"2023-09-23T19:28:06.885427+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":72,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.884512+0000","last_change":"2023-09-23T19:28:07.884512+0000","last_active":"2023-09-23T19:28:07.884512+0000","last_peered":"2023-09-23T19:28:07.884512+0000","last_clean":"2023-09-23T19:28:07.884512+0000","last_became_active":"2023-09-23T19:28:07.884155+0000","last_became_peered":"2023-09-23T19:28:07.884155+0000","last_unstale":"2023-09-23T19:28:07.884512+0000","last_undegraded":"2023-09-23T19:28:07.884512+0000","last_fullsized":"2023-09-23T19:28:07.884512+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:10.654958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":92,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.876318+0000","last_change":"2023-09-23T19:18:09.010195+0000","last_active":"2023-09-23T19:28:06.876318+0000","last_peered":"2023-09-23T19:28:06.876318+0000","last_clean":"2023-09-23T19:28:06.876318+0000","last_became_active":"2023-09-23T19:18:09.009687+0000","last_became_peered":"2023-09-23T19:18:09.009687+0000","last_unstale":"2023-09-23T19:28:06.876318+0000","last_undegraded":"2023-09-23T19:28:06.876318+0000","last_fullsized":"2023-09-23T19:28:06.876318+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":78,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.885265+0000","last_change":"2023-09-23T19:28:07.885265+0000","last_active":"2023-09-23T19:28:07.885265+0000","last_peered":"2023-09-23T19:28:07.885265+0000","last_clean":"2023-09-23T19:28:07.885265+0000","last_became_active":"2023-09-23T19:28:07.884353+0000","last_became_peered":"2023-09-23T19:28:07.884353+0000","last_unstale":"2023-09-23T19:28:07.885265+0000","last_undegraded":"2023-09-23T19:28:07.885265+0000","last_fullsized":"2023-09-23T19:28:07.885265+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:56.325352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":85,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.886566+0000","last_change":"2023-09-23T19:18:09.035116+0000","last_active":"2023-09-23T19:28:06.886566+0000","last_peered":"2023-09-23T19:28:06.886566+0000","last_clean":"2023-09-23T19:28:06.886566+0000","last_became_active":"2023-09-23T19:18:09.034416+0000","last_became_peered":"2023-09-23T19:18:09.034416+0000","last_unstale":"2023-09-23T19:28:06.886566+0000","last_undegraded":"2023-09-23T19:28:06.886566+0000","last_fullsized":"2023-09-23T19:28:06.886566+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":68,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.879702+0000","last_change":"2023-09-23T19:05:52.789323+0000","last_active":"2023-09-23T19:28:06.879702+0000","last_peered":"2023-09-23T19:28:06.879702+0000","last_clean":"2023-09-23T19:28:06.879702+0000","last_became_active":"2023-09-23T19:05:52.788820+0000","last_became_peered":"2023-09-23T19:05:52.788820+0000","last_unstale":"2023-09-23T19:28:06.879702+0000","last_undegraded":"2023-09-23T19:28:06.879702+0000","last_fullsized":"2023-09-23T19:28:06.879702+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":86,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.885780+0000","last_change":"2023-09-23T19:22:42.151134+0000","last_active":"2023-09-23T19:28:06.885780+0000","last_peered":"2023-09-23T19:28:06.885780+0000","last_clean":"2023-09-23T19:28:06.885780+0000","last_became_active":"2023-09-23T19:22:42.150553+0000","last_became_peered":"2023-09-23T19:22:42.150553+0000","last_unstale":"2023-09-23T19:28:06.885780+0000","last_undegraded":"2023-09-23T19:28:06.885780+0000","last_fullsized":"2023-09-23T19:28:06.885780+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":98,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.899262+0000","last_change":"2023-09-23T19:28:07.899262+0000","last_active":"2023-09-23T19:28:07.899262+0000","last_peered":"2023-09-23T19:28:07.899262+0000","last_clean":"2023-09-23T19:28:07.899262+0000","last_became_active":"2023-09-23T19:28:07.898848+0000","last_became_peered":"2023-09-23T19:28:07.898848+0000","last_unstale":"2023-09-23T19:28:07.899262+0000","last_undegraded":"2023-09-23T19:28:07.899262+0000","last_fullsized":"2023-09-23T19:28:07.899262+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":70,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.885984+0000","last_change":"2023-09-23T19:05:52.114108+0000","last_active":"2023-09-23T19:28:06.885984+0000","last_peered":"2023-09-23T19:28:06.885984+0000","last_clean":"2023-09-23T19:28:06.885984+0000","last_became_active":"2023-09-23T19:05:52.113609+0000","last_became_peered":"2023-09-23T19:05:52.113609+0000","last_unstale":"2023-09-23T19:28:06.885984+0000","last_undegraded":"2023-09-23T19:28:06.885984+0000","last_fullsized":"2023-09-23T19:28:06.885984+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":83,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.875937+0000","last_change":"2023-09-23T19:22:42.156825+0000","last_active":"2023-09-23T19:28:06.875937+0000","last_peered":"2023-09-23T19:28:06.875937+0000","last_clean":"2023-09-23T19:28:06.875937+0000","last_became_active":"2023-09-23T19:22:42.156474+0000","last_became_peered":"2023-09-23T19:22:42.156474+0000","last_unstale":"2023-09-23T19:28:06.875937+0000","last_undegraded":"2023-09-23T19:28:06.875937+0000","last_fullsized":"2023-09-23T19:28:06.875937+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":84,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.891536+0000","last_change":"2023-09-23T19:28:07.891536+0000","last_active":"2023-09-23T19:28:07.891536+0000","last_peered":"2023-09-23T19:28:07.891536+0000","last_clean":"2023-09-23T19:28:07.891536+0000","last_became_active":"2023-09-23T19:28:07.891153+0000","last_became_peered":"2023-09-23T19:28:07.891153+0000","last_unstale":"2023-09-23T19:28:07.891536+0000","last_undegraded":"2023-09-23T19:28:07.891536+0000","last_fullsized":"2023-09-23T19:28:07.891536+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":73,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.874482+0000","last_change":"2023-09-23T19:22:42.135572+0000","last_active":"2023-09-23T19:28:07.874482+0000","last_peered":"2023-09-23T19:28:07.874482+0000","last_clean":"2023-09-23T19:28:07.874482+0000","last_became_active":"2023-09-23T19:22:42.135237+0000","last_became_peered":"2023-09-23T19:22:42.135237+0000","last_unstale":"2023-09-23T19:28:07.874482+0000","last_undegraded":"2023-09-23T19:28:07.874482+0000","last_fullsized":"2023-09-23T19:28:07.874482+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:35.838718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":68,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.893901+0000","last_change":"2023-09-23T19:28:07.893901+0000","last_active":"2023-09-23T19:28:07.893901+0000","last_peered":"2023-09-23T19:28:07.893901+0000","last_clean":"2023-09-23T19:28:07.893901+0000","last_became_active":"2023-09-23T19:28:07.892973+0000","last_became_peered":"2023-09-23T19:28:07.892973+0000","last_unstale":"2023-09-23T19:28:07.893901+0000","last_undegraded":"2023-09-23T19:28:07.893901+0000","last_fullsized":"2023-09-23T19:28:07.893901+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:27.445855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":83,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.896803+0000","last_change":"2023-09-23T19:28:07.896803+0000","last_active":"2023-09-23T19:28:07.896803+0000","last_peered":"2023-09-23T19:28:07.896803+0000","last_clean":"2023-09-23T19:28:07.896803+0000","last_became_active":"2023-09-23T19:28:07.896480+0000","last_became_peered":"2023-09-23T19:28:07.896480+0000","last_unstale":"2023-09-23T19:28:07.896803+0000","last_undegraded":"2023-09-23T19:28:07.896803+0000","last_fullsized":"2023-09-23T19:28:07.896803+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":84,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.876231+0000","last_change":"2023-09-23T19:22:42.156582+0000","last_active":"2023-09-23T19:28:06.876231+0000","last_peered":"2023-09-23T19:28:06.876231+0000","last_clean":"2023-09-23T19:28:06.876231+0000","last_became_active":"2023-09-23T19:22:42.156251+0000","last_became_peered":"2023-09-23T19:22:42.156251+0000","last_unstale":"2023-09-23T19:28:06.876231+0000","last_undegraded":"2023-09-23T19:28:06.876231+0000","last_fullsized":"2023-09-23T19:28:06.876231+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":98,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.879979+0000","last_change":"2023-09-23T19:27:24.273971+0000","last_active":"2023-09-23T19:28:06.879979+0000","last_peered":"2023-09-23T19:28:06.879979+0000","last_clean":"2023-09-23T19:28:06.879979+0000","last_became_active":"2023-09-23T19:27:24.273491+0000","last_became_peered":"2023-09-23T19:27:24.273491+0000","last_unstale":"2023-09-23T19:28:06.879979+0000","last_undegraded":"2023-09-23T19:28:06.879979+0000","last_fullsized":"2023-09-23T19:28:06.879979+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":84,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.878781+0000","last_change":"2023-09-23T19:27:24.274698+0000","last_active":"2023-09-23T19:28:06.878781+0000","last_peered":"2023-09-23T19:28:06.878781+0000","last_clean":"2023-09-23T19:28:06.878781+0000","last_became_active":"2023-09-23T19:27:24.273264+0000","last_became_peered":"2023-09-23T19:27:24.273264+0000","last_unstale":"2023-09-23T19:28:06.878781+0000","last_undegraded":"2023-09-23T19:28:06.878781+0000","last_fullsized":"2023-09-23T19:28:06.878781+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":83,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.878941+0000","last_change":"2023-09-23T19:22:42.149670+0000","last_active":"2023-09-23T19:28:06.878941+0000","last_peered":"2023-09-23T19:28:06.878941+0000","last_clean":"2023-09-23T19:28:06.878941+0000","last_became_active":"2023-09-23T19:22:42.149022+0000","last_became_peered":"2023-09-23T19:22:42.149022+0000","last_unstale":"2023-09-23T19:28:06.878941+0000","last_undegraded":"2023-09-23T19:28:06.878941+0000","last_fullsized":"2023-09-23T19:28:06.878941+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":79,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.886933+0000","last_change":"2023-09-23T19:28:07.886933+0000","last_active":"2023-09-23T19:28:07.886933+0000","last_peered":"2023-09-23T19:28:07.886933+0000","last_clean":"2023-09-23T19:28:07.886933+0000","last_became_active":"2023-09-23T19:28:07.886572+0000","last_became_peered":"2023-09-23T19:28:07.886572+0000","last_unstale":"2023-09-23T19:28:07.886933+0000","last_undegraded":"2023-09-23T19:28:07.886933+0000","last_fullsized":"2023-09-23T19:28:07.886933+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:41.332875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":58,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.889464+0000","last_change":"2023-09-23T19:28:07.889095+0000","last_active":"2023-09-23T19:28:07.889464+0000","last_peered":"2023-09-23T19:28:07.889464+0000","last_clean":"2023-09-23T19:28:07.889464+0000","last_became_active":"2023-09-23T19:28:07.888742+0000","last_became_peered":"2023-09-23T19:28:07.888742+0000","last_unstale":"2023-09-23T19:28:07.889464+0000","last_undegraded":"2023-09-23T19:28:07.889464+0000","last_fullsized":"2023-09-23T19:28:07.889464+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:43.929488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":91,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.870233+0000","last_change":"2023-09-23T19:22:42.143289+0000","last_active":"2023-09-23T19:28:07.870233+0000","last_peered":"2023-09-23T19:28:07.870233+0000","last_clean":"2023-09-23T19:28:07.870233+0000","last_became_active":"2023-09-23T19:22:42.142656+0000","last_became_peered":"2023-09-23T19:22:42.142656+0000","last_unstale":"2023-09-23T19:28:07.870233+0000","last_undegraded":"2023-09-23T19:28:07.870233+0000","last_fullsized":"2023-09-23T19:28:07.870233+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:38.467441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":99,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.890623+0000","last_change":"2023-09-23T19:28:07.890623+0000","last_active":"2023-09-23T19:28:07.890623+0000","last_peered":"2023-09-23T19:28:07.890623+0000","last_clean":"2023-09-23T19:28:07.890623+0000","last_became_active":"2023-09-23T19:28:07.890148+0000","last_became_peered":"2023-09-23T19:28:07.890148+0000","last_unstale":"2023-09-23T19:28:07.890623+0000","last_undegraded":"2023-09-23T19:28:07.890623+0000","last_fullsized":"2023-09-23T19:28:07.890623+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":71,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.865316+0000","last_change":"2023-09-23T19:27:24.269080+0000","last_active":"2023-09-23T19:28:07.865316+0000","last_peered":"2023-09-23T19:28:07.865316+0000","last_clean":"2023-09-23T19:28:07.865316+0000","last_became_active":"2023-09-23T19:27:24.268748+0000","last_became_peered":"2023-09-23T19:27:24.268748+0000","last_unstale":"2023-09-23T19:28:07.865316+0000","last_undegraded":"2023-09-23T19:28:07.865316+0000","last_fullsized":"2023-09-23T19:28:07.865316+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:49.014902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":88,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.880479+0000","last_change":"2023-09-23T19:22:42.153647+0000","last_active":"2023-09-23T19:28:06.880479+0000","last_peered":"2023-09-23T19:28:06.880479+0000","last_clean":"2023-09-23T19:28:06.880479+0000","last_became_active":"2023-09-23T19:22:42.153317+0000","last_became_peered":"2023-09-23T19:22:42.153317+0000","last_unstale":"2023-09-23T19:28:06.880479+0000","last_undegraded":"2023-09-23T19:28:06.880479+0000","last_fullsized":"2023-09-23T19:28:06.880479+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":84,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.882331+0000","last_change":"2023-09-23T19:18:09.017525+0000","last_active":"2023-09-23T19:28:06.882331+0000","last_peered":"2023-09-23T19:28:06.882331+0000","last_clean":"2023-09-23T19:28:06.882331+0000","last_became_active":"2023-09-23T19:18:09.017193+0000","last_became_peered":"2023-09-23T19:18:09.017193+0000","last_unstale":"2023-09-23T19:28:06.882331+0000","last_undegraded":"2023-09-23T19:28:06.882331+0000","last_fullsized":"2023-09-23T19:28:06.882331+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":73,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.879432+0000","last_change":"2023-09-23T19:28:07.875796+0000","last_active":"2023-09-23T19:28:07.879432+0000","last_peered":"2023-09-23T19:28:07.879432+0000","last_clean":"2023-09-23T19:28:07.879432+0000","last_became_active":"2023-09-23T19:28:07.875310+0000","last_became_peered":"2023-09-23T19:28:07.875310+0000","last_unstale":"2023-09-23T19:28:07.879432+0000","last_undegraded":"2023-09-23T19:28:07.879432+0000","last_fullsized":"2023-09-23T19:28:07.879432+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:44:13.379633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":91,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.879332+0000","last_change":"2023-09-23T19:28:07.878688+0000","last_active":"2023-09-23T19:28:07.879332+0000","last_peered":"2023-09-23T19:28:07.879332+0000","last_clean":"2023-09-23T19:28:07.879332+0000","last_became_active":"2023-09-23T19:28:07.878309+0000","last_became_peered":"2023-09-23T19:28:07.878309+0000","last_unstale":"2023-09-23T19:28:07.879332+0000","last_undegraded":"2023-09-23T19:28:07.879332+0000","last_fullsized":"2023-09-23T19:28:07.879332+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:49:06.202657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":58,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.867585+0000","last_change":"2023-09-23T19:27:24.267897+0000","last_active":"2023-09-23T19:28:07.867585+0000","last_peered":"2023-09-23T19:28:07.867585+0000","last_clean":"2023-09-23T19:28:07.867585+0000","last_became_active":"2023-09-23T19:27:24.267535+0000","last_became_peered":"2023-09-23T19:27:24.267535+0000","last_unstale":"2023-09-23T19:28:07.867585+0000","last_undegraded":"2023-09-23T19:28:07.867585+0000","last_fullsized":"2023-09-23T19:28:07.867585+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:13:57.447808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":86,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.874771+0000","last_change":"2023-09-23T19:22:42.143733+0000","last_active":"2023-09-23T19:28:07.874771+0000","last_peered":"2023-09-23T19:28:07.874771+0000","last_clean":"2023-09-23T19:28:07.874771+0000","last_became_active":"2023-09-23T19:22:42.143077+0000","last_became_peered":"2023-09-23T19:22:42.143077+0000","last_unstale":"2023-09-23T19:28:07.874771+0000","last_undegraded":"2023-09-23T19:28:07.874771+0000","last_fullsized":"2023-09-23T19:28:07.874771+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:47:33.086994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":88,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.879925+0000","last_change":"2023-09-23T19:27:24.271520+0000","last_active":"2023-09-23T19:28:06.879925+0000","last_peered":"2023-09-23T19:28:06.879925+0000","last_clean":"2023-09-23T19:28:06.879925+0000","last_became_active":"2023-09-23T19:27:24.271165+0000","last_became_peered":"2023-09-23T19:27:24.271165+0000","last_unstale":"2023-09-23T19:28:06.879925+0000","last_undegraded":"2023-09-23T19:28:06.879925+0000","last_fullsized":"2023-09-23T19:28:06.879925+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":320,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.885321+0000","last_change":"2023-09-23T19:18:09.016297+0000","last_active":"2023-09-23T19:28:06.885321+0000","last_peered":"2023-09-23T19:28:06.885321+0000","last_clean":"2023-09-23T19:28:06.885321+0000","last_became_active":"2023-09-23T19:18:09.015436+0000","last_became_peered":"2023-09-23T19:18:09.015436+0000","last_unstale":"2023-09-23T19:28:06.885321+0000","last_undegraded":"2023-09-23T19:28:06.885321+0000","last_fullsized":"2023-09-23T19:28:06.885321+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:59:33.840751+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:59:33.840751+0000","last_clean_scrub_stamp":"2023-09-23T18:59:33.840751+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":64,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.889756+0000","last_change":"2023-09-23T19:28:07.889756+0000","last_active":"2023-09-23T19:28:07.889756+0000","last_peered":"2023-09-23T19:28:07.889756+0000","last_clean":"2023-09-23T19:28:07.889756+0000","last_became_active":"2023-09-23T19:28:07.889050+0000","last_became_peered":"2023-09-23T19:28:07.889050+0000","last_unstale":"2023-09-23T19:28:07.889756+0000","last_undegraded":"2023-09-23T19:28:07.889756+0000","last_fullsized":"2023-09-23T19:28:07.889756+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:04:55.924553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":68,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.876575+0000","last_change":"2023-09-23T19:05:52.786771+0000","last_active":"2023-09-23T19:28:06.876575+0000","last_peered":"2023-09-23T19:28:06.876575+0000","last_clean":"2023-09-23T19:28:06.876575+0000","last_became_active":"2023-09-23T19:05:52.785797+0000","last_became_peered":"2023-09-23T19:05:52.785797+0000","last_unstale":"2023-09-23T19:28:06.876575+0000","last_undegraded":"2023-09-23T19:28:06.876575+0000","last_fullsized":"2023-09-23T19:28:06.876575+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":103,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.885258+0000","last_change":"2023-09-23T19:22:42.150636+0000","last_active":"2023-09-23T19:28:06.885258+0000","last_peered":"2023-09-23T19:28:06.885258+0000","last_clean":"2023-09-23T19:28:06.885258+0000","last_became_active":"2023-09-23T19:22:42.150045+0000","last_became_peered":"2023-09-23T19:22:42.150045+0000","last_unstale":"2023-09-23T19:28:06.885258+0000","last_undegraded":"2023-09-23T19:28:06.885258+0000","last_fullsized":"2023-09-23T19:28:06.885258+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":73,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.889776+0000","last_change":"2023-09-23T19:28:07.889194+0000","last_active":"2023-09-23T19:28:07.889776+0000","last_peered":"2023-09-23T19:28:07.889776+0000","last_clean":"2023-09-23T19:28:07.889776+0000","last_became_active":"2023-09-23T19:28:07.888823+0000","last_became_peered":"2023-09-23T19:28:07.888823+0000","last_unstale":"2023-09-23T19:28:07.889776+0000","last_undegraded":"2023-09-23T19:28:07.889776+0000","last_fullsized":"2023-09-23T19:28:07.889776+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:23.464688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":88,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.874696+0000","last_change":"2023-09-23T19:27:24.256410+0000","last_active":"2023-09-23T19:28:07.874696+0000","last_peered":"2023-09-23T19:28:07.874696+0000","last_clean":"2023-09-23T19:28:07.874696+0000","last_became_active":"2023-09-23T19:27:24.256052+0000","last_became_peered":"2023-09-23T19:27:24.256052+0000","last_unstale":"2023-09-23T19:28:07.874696+0000","last_undegraded":"2023-09-23T19:28:07.874696+0000","last_fullsized":"2023-09-23T19:28:07.874696+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:11.951400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":88,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.886287+0000","last_change":"2023-09-23T19:18:09.015853+0000","last_active":"2023-09-23T19:28:06.886287+0000","last_peered":"2023-09-23T19:28:06.886287+0000","last_clean":"2023-09-23T19:28:06.886287+0000","last_became_active":"2023-09-23T19:18:09.015338+0000","last_became_peered":"2023-09-23T19:18:09.015338+0000","last_unstale":"2023-09-23T19:28:06.886287+0000","last_undegraded":"2023-09-23T19:28:06.886287+0000","last_fullsized":"2023-09-23T19:28:06.886287+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":58,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.874558+0000","last_change":"2023-09-23T19:27:24.266241+0000","last_active":"2023-09-23T19:28:07.874558+0000","last_peered":"2023-09-23T19:28:07.874558+0000","last_clean":"2023-09-23T19:28:07.874558+0000","last_became_active":"2023-09-23T19:27:24.265764+0000","last_became_peered":"2023-09-23T19:27:24.265764+0000","last_unstale":"2023-09-23T19:28:07.874558+0000","last_undegraded":"2023-09-23T19:28:07.874558+0000","last_fullsized":"2023-09-23T19:28:07.874558+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:07.694732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":58,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.873806+0000","last_change":"2023-09-23T19:18:09.005823+0000","last_active":"2023-09-23T19:28:07.873806+0000","last_peered":"2023-09-23T19:28:07.873806+0000","last_clean":"2023-09-23T19:28:07.873806+0000","last_became_active":"2023-09-23T19:18:09.005207+0000","last_became_peered":"2023-09-23T19:18:09.005207+0000","last_unstale":"2023-09-23T19:28:07.873806+0000","last_undegraded":"2023-09-23T19:28:07.873806+0000","last_fullsized":"2023-09-23T19:28:07.873806+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:05.520188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":68,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.884711+0000","last_change":"2023-09-23T19:28:07.884711+0000","last_active":"2023-09-23T19:28:07.884711+0000","last_peered":"2023-09-23T19:28:07.884711+0000","last_clean":"2023-09-23T19:28:07.884711+0000","last_became_active":"2023-09-23T19:28:07.883878+0000","last_became_peered":"2023-09-23T19:28:07.883878+0000","last_unstale":"2023-09-23T19:28:07.884711+0000","last_undegraded":"2023-09-23T19:28:07.884711+0000","last_fullsized":"2023-09-23T19:28:07.884711+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:55.538826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":43,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.874598+0000","last_change":"2023-09-23T19:22:42.145506+0000","last_active":"2023-09-23T19:28:07.874598+0000","last_peered":"2023-09-23T19:28:07.874598+0000","last_clean":"2023-09-23T19:28:07.874598+0000","last_became_active":"2023-09-23T19:22:42.143991+0000","last_became_peered":"2023-09-23T19:22:42.143991+0000","last_unstale":"2023-09-23T19:28:07.874598+0000","last_undegraded":"2023-09-23T19:28:07.874598+0000","last_fullsized":"2023-09-23T19:28:07.874598+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:56.875175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":65,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.886079+0000","last_change":"2023-09-23T19:28:07.886079+0000","last_active":"2023-09-23T19:28:07.886079+0000","last_peered":"2023-09-23T19:28:07.886079+0000","last_clean":"2023-09-23T19:28:07.886079+0000","last_became_active":"2023-09-23T19:28:07.885209+0000","last_became_peered":"2023-09-23T19:28:07.885209+0000","last_unstale":"2023-09-23T19:28:07.886079+0000","last_undegraded":"2023-09-23T19:28:07.886079+0000","last_fullsized":"2023-09-23T19:28:07.886079+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:19.991162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":88,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.876444+0000","last_change":"2023-09-23T19:27:24.271827+0000","last_active":"2023-09-23T19:28:06.876444+0000","last_peered":"2023-09-23T19:28:06.876444+0000","last_clean":"2023-09-23T19:28:06.876444+0000","last_became_active":"2023-09-23T19:27:24.271328+0000","last_became_peered":"2023-09-23T19:27:24.271328+0000","last_unstale":"2023-09-23T19:28:06.876444+0000","last_undegraded":"2023-09-23T19:28:06.876444+0000","last_fullsized":"2023-09-23T19:28:06.876444+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":76,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.874297+0000","last_change":"2023-09-23T19:22:42.149694+0000","last_active":"2023-09-23T19:28:07.874297+0000","last_peered":"2023-09-23T19:28:07.874297+0000","last_clean":"2023-09-23T19:28:07.874297+0000","last_became_active":"2023-09-23T19:22:42.148194+0000","last_became_peered":"2023-09-23T19:22:42.148194+0000","last_unstale":"2023-09-23T19:28:07.874297+0000","last_undegraded":"2023-09-23T19:28:07.874297+0000","last_fullsized":"2023-09-23T19:28:07.874297+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:47.842383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":68,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.878925+0000","last_change":"2023-09-23T19:05:52.138620+0000","last_active":"2023-09-23T19:28:06.878925+0000","last_peered":"2023-09-23T19:28:06.878925+0000","last_clean":"2023-09-23T19:28:06.878925+0000","last_became_active":"2023-09-23T19:05:52.138243+0000","last_became_peered":"2023-09-23T19:05:52.138243+0000","last_unstale":"2023-09-23T19:28:06.878925+0000","last_undegraded":"2023-09-23T19:28:06.878925+0000","last_fullsized":"2023-09-23T19:28:06.878925+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":60,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.874590+0000","last_change":"2023-09-23T19:18:09.008811+0000","last_active":"2023-09-23T19:28:07.874590+0000","last_peered":"2023-09-23T19:28:07.874590+0000","last_clean":"2023-09-23T19:28:07.874590+0000","last_became_active":"2023-09-23T19:18:09.007255+0000","last_became_peered":"2023-09-23T19:18:09.007255+0000","last_unstale":"2023-09-23T19:28:07.874590+0000","last_undegraded":"2023-09-23T19:28:07.874590+0000","last_fullsized":"2023-09-23T19:28:07.874590+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:25.874038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":99,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.876494+0000","last_change":"2023-09-23T19:28:07.876141+0000","last_active":"2023-09-23T19:28:07.876494+0000","last_peered":"2023-09-23T19:28:07.876494+0000","last_clean":"2023-09-23T19:28:07.876494+0000","last_became_active":"2023-09-23T19:28:07.875802+0000","last_became_peered":"2023-09-23T19:28:07.875802+0000","last_unstale":"2023-09-23T19:28:07.876494+0000","last_undegraded":"2023-09-23T19:28:07.876494+0000","last_fullsized":"2023-09-23T19:28:07.876494+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:38.947881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":86,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.881984+0000","last_change":"2023-09-23T19:22:42.153825+0000","last_active":"2023-09-23T19:28:06.881984+0000","last_peered":"2023-09-23T19:28:06.881984+0000","last_clean":"2023-09-23T19:28:06.881984+0000","last_became_active":"2023-09-23T19:22:42.153416+0000","last_became_peered":"2023-09-23T19:22:42.153416+0000","last_unstale":"2023-09-23T19:28:06.881984+0000","last_undegraded":"2023-09-23T19:28:06.881984+0000","last_fullsized":"2023-09-23T19:28:06.881984+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":107,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.899164+0000","last_change":"2023-09-23T19:28:07.899164+0000","last_active":"2023-09-23T19:28:07.899164+0000","last_peered":"2023-09-23T19:28:07.899164+0000","last_clean":"2023-09-23T19:28:07.899164+0000","last_became_active":"2023-09-23T19:28:07.898812+0000","last_became_peered":"2023-09-23T19:28:07.898812+0000","last_unstale":"2023-09-23T19:28:07.899164+0000","last_undegraded":"2023-09-23T19:28:07.899164+0000","last_fullsized":"2023-09-23T19:28:07.899164+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":71,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.884673+0000","last_change":"2023-09-23T19:05:52.107570+0000","last_active":"2023-09-23T19:28:06.884673+0000","last_peered":"2023-09-23T19:28:06.884673+0000","last_clean":"2023-09-23T19:28:06.884673+0000","last_became_active":"2023-09-23T19:05:52.107215+0000","last_became_peered":"2023-09-23T19:05:52.107215+0000","last_unstale":"2023-09-23T19:28:06.884673+0000","last_undegraded":"2023-09-23T19:28:06.884673+0000","last_fullsized":"2023-09-23T19:28:06.884673+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":98,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.896983+0000","last_change":"2023-09-23T19:28:07.896983+0000","last_active":"2023-09-23T19:28:07.896983+0000","last_peered":"2023-09-23T19:28:07.896983+0000","last_clean":"2023-09-23T19:28:07.896983+0000","last_became_active":"2023-09-23T19:28:07.896577+0000","last_became_peered":"2023-09-23T19:28:07.896577+0000","last_unstale":"2023-09-23T19:28:07.896983+0000","last_undegraded":"2023-09-23T19:28:07.896983+0000","last_fullsized":"2023-09-23T19:28:07.896983+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":92,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.891196+0000","last_change":"2023-09-23T19:28:07.891196+0000","last_active":"2023-09-23T19:28:07.891196+0000","last_peered":"2023-09-23T19:28:07.891196+0000","last_clean":"2023-09-23T19:28:07.891196+0000","last_became_active":"2023-09-23T19:28:07.890484+0000","last_became_peered":"2023-09-23T19:28:07.890484+0000","last_unstale":"2023-09-23T19:28:07.891196+0000","last_undegraded":"2023-09-23T19:28:07.891196+0000","last_fullsized":"2023-09-23T19:28:07.891196+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:24.574716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":105,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.883754+0000","last_change":"2023-09-23T19:28:07.883754+0000","last_active":"2023-09-23T19:28:07.883754+0000","last_peered":"2023-09-23T19:28:07.883754+0000","last_clean":"2023-09-23T19:28:07.883754+0000","last_became_active":"2023-09-23T19:28:07.883363+0000","last_became_peered":"2023-09-23T19:28:07.883363+0000","last_unstale":"2023-09-23T19:28:07.883754+0000","last_undegraded":"2023-09-23T19:28:07.883754+0000","last_fullsized":"2023-09-23T19:28:07.883754+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:56:35.805451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":76,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.874954+0000","last_change":"2023-09-23T19:27:24.267931+0000","last_active":"2023-09-23T19:28:07.874954+0000","last_peered":"2023-09-23T19:28:07.874954+0000","last_clean":"2023-09-23T19:28:07.874954+0000","last_became_active":"2023-09-23T19:27:24.267552+0000","last_became_peered":"2023-09-23T19:27:24.267552+0000","last_unstale":"2023-09-23T19:28:07.874954+0000","last_undegraded":"2023-09-23T19:28:07.874954+0000","last_fullsized":"2023-09-23T19:28:07.874954+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:12.425257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":77,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.878171+0000","last_change":"2023-09-23T19:06:36.673002+0000","last_active":"2023-09-23T19:28:06.878171+0000","last_peered":"2023-09-23T19:28:06.878171+0000","last_clean":"2023-09-23T19:28:06.878171+0000","last_became_active":"2023-09-23T19:06:36.671911+0000","last_became_peered":"2023-09-23T19:06:36.671911+0000","last_unstale":"2023-09-23T19:28:06.878171+0000","last_undegraded":"2023-09-23T19:28:06.878171+0000","last_fullsized":"2023-09-23T19:28:06.878171+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430534,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307128,"kb_used_data":1712,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":93438024,"statfs":{"total":95995035648,"available":95680536576,"internally_reserved":0,"allocated":1753088,"data_stored":1287929,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:29:24 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":1.2130000000000001,"15min":1.2929999999999999},"min":{"1min":0.60199999999999998,"5min":0.27700000000000002,"15min":0.26400000000000001},"max":{"1min":1.3799999999999999,"5min":9.6760000000000002,"15min":9.6760000000000002},"last":0.97499999999999998},{"interface":"front","average":{"1min":0.85099999999999998,"5min":1.109,"15min":1.4219999999999999},"min":{"1min":0.70899999999999996,"5min":0.30299999999999999,"15min":0.28999999999999998},"max":{"1min":1.1040000000000001,"5min":7.8230000000000004,"15min":8.9670000000000005},"last":1.1299999999999999}]},{"osd":1,"last update":"Sat Sep 23 19:29:51 2023","interfaces":[{"interface":"back","average":{"1min":0.87,"5min":1.2050000000000001,"15min":1.413},"min":{"1min":0.56000000000000005,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":1.218,"5min":7.6180000000000003,"15min":7.6180000000000003},"last":1.01},{"interface":"front","average":{"1min":0.93799999999999994,"5min":1.179,"15min":1.413},"min":{"1min":0.61199999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":1.1619999999999999,"5min":7.0010000000000003,"15min":7.3739999999999997},"last":1.208}]},{"osd":2,"last update":"Sat Sep 23 19:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.83899999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"min":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"max":{"1min":1.0429999999999999,"5min":1.0449999999999999,"15min":1.0449999999999999},"last":0.753},{"interface":"front","average":{"1min":0.998,"5min":0.97699999999999998,"15min":0.97299999999999998},"min":{"1min":0.76300000000000001,"5min":0.622,"15min":0.622},"max":{"1min":1.2170000000000001,"5min":1.294,"15min":1.294},"last":1.079}]},{"osd":3,"last update":"Sat Sep 23 19:29:07 2023","interfaces":[{"interface":"back","average":{"1min":1.0580000000000001,"5min":1.0580000000000001,"15min":1.0580000000000001},"min":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"max":{"1min":1.296,"5min":1.296,"15min":1.296},"last":1.1100000000000001},{"interface":"front","average":{"1min":1.1279999999999999,"5min":1.1279999999999999,"15min":1.1279999999999999},"min":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"max":{"1min":1.298,"5min":1.298,"15min":1.298},"last":1.1659999999999999}]},{"osd":4,"last update":"Sat Sep 23 19:29:51 2023","interfaces":[{"interface":"back","average":{"1min":0.96099999999999997,"5min":1.0840000000000001,"15min":1.1699999999999999},"min":{"1min":0.73399999999999999,"5min":0.35399999999999998,"15min":0.28699999999999998},"max":{"1min":1.1839999999999999,"5min":6.1870000000000003,"15min":9.4190000000000005},"last":1.1830000000000001},{"interface":"front","average":{"1min":1.1259999999999999,"5min":1.099,"15min":1.101},"min":{"1min":0.88600000000000001,"5min":0.36099999999999999,"15min":0.33700000000000002},"max":{"1min":1.3640000000000001,"5min":4.6769999999999996,"15min":7.9790000000000001},"last":1.038}]},{"osd":5,"last update":"Sat Sep 23 19:29:51 2023","interfaces":[{"interface":"back","average":{"1min":1.034,"5min":1.258,"15min":1.2769999999999999},"min":{"1min":0.68600000000000005,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.3480000000000001,"5min":6.7130000000000001,"15min":9.8580000000000005},"last":1.0569999999999999},{"interface":"front","average":{"1min":1.0720000000000001,"5min":1.371,"15min":1.4279999999999999},"min":{"1min":0.77100000000000002,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":1.321,"5min":9.7829999999999995,"15min":10.013999999999999},"last":1.238}]},{"osd":6,"last update":"Sat Sep 23 19:29:51 2023","interfaces":[{"interface":"back","average":{"1min":1.125,"5min":1.298,"15min":1.377},"min":{"1min":0.87,"5min":0.41799999999999998,"15min":0.34599999999999997},"max":{"1min":1.3939999999999999,"5min":5.2320000000000002,"15min":13.462},"last":1.282},{"interface":"front","average":{"1min":1.179,"5min":1.0649999999999999,"15min":1.1739999999999999},"min":{"1min":0.80800000000000005,"5min":0.42299999999999999,"15min":0.36799999999999999},"max":{"1min":1.47,"5min":2.532,"15min":7.9109999999999996},"last":1.2549999999999999}]}]},{"osd":6,"up_from":42,"seq":180388626766,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307112,"kb_used_data":1696,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":93438040,"statfs":{"total":95995035648,"available":95680552960,"internally_reserved":0,"allocated":1736704,"data_stored":1287468,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:29:36 2023","interfaces":[{"interface":"back","average":{"1min":0.82499999999999996,"5min":1.216,"15min":1.405},"min":{"1min":0.45900000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":1.141,"5min":8.7110000000000003,"15min":8.8989999999999991},"last":0.69699999999999995},{"interface":"front","average":{"1min":0.88700000000000001,"5min":1.256,"15min":1.2310000000000001},"min":{"1min":0.505,"5min":0.36599999999999999,"15min":0.25900000000000001},"max":{"1min":1.2190000000000001,"5min":7.5110000000000001,"15min":7.5110000000000001},"last":0.81799999999999995}]},{"osd":1,"last update":"Sat Sep 23 19:29:51 2023","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":1.349,"15min":1.5589999999999999},"min":{"1min":0.71999999999999997,"5min":0.33100000000000002,"15min":0.29999999999999999},"max":{"1min":1.274,"5min":8.8130000000000006,"15min":11.351000000000001},"last":1.014},{"interface":"front","average":{"1min":1.0580000000000001,"5min":1.492,"15min":1.6830000000000001},"min":{"1min":0.754,"5min":0.41399999999999998,"15min":0.34200000000000003},"max":{"1min":1.3620000000000001,"5min":12.728999999999999,"15min":12.728999999999999},"last":0.98999999999999999}]},{"osd":2,"last update":"Sat Sep 23 19:29:26 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":0.93500000000000005,"15min":0.93000000000000005},"min":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"max":{"1min":1.3120000000000001,"5min":1.3120000000000001,"15min":1.3120000000000001},"last":0.96499999999999997},{"interface":"front","average":{"1min":0.97699999999999998,"5min":0.91200000000000003,"15min":0.90100000000000002},"min":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"max":{"1min":1.1919999999999999,"5min":1.1919999999999999,"15min":1.1919999999999999},"last":0.78300000000000003}]},{"osd":3,"last update":"Sat Sep 23 19:29:09 2023","interfaces":[{"interface":"back","average":{"1min":0.99399999999999999,"5min":0.99399999999999999,"15min":0.99399999999999999},"min":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"max":{"1min":1.212,"5min":1.212,"15min":1.212},"last":1.1160000000000001},{"interface":"front","average":{"1min":0.96799999999999997,"5min":0.96799999999999997,"15min":0.96799999999999997},"min":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"max":{"1min":1.177,"5min":1.177,"15min":1.177},"last":0.93799999999999994}]},{"osd":4,"last update":"Sat Sep 23 19:29:13 2023","interfaces":[{"interface":"back","average":{"1min":0.90600000000000003,"5min":0.996,"15min":0.999},"min":{"1min":0.42999999999999999,"5min":0.33500000000000002,"15min":0.29299999999999998},"max":{"1min":1.298,"5min":4.6799999999999997,"15min":6.5869999999999997},"last":0.65100000000000002},{"interface":"front","average":{"1min":1.0249999999999999,"5min":1.1950000000000001,"15min":1.1559999999999999},"min":{"1min":0.53200000000000003,"5min":0.36599999999999999,"15min":0.30499999999999999},"max":{"1min":1.411,"5min":8.5540000000000003,"15min":8.5540000000000003},"last":1.165}]},{"osd":5,"last update":"Sat Sep 23 19:29:13 2023","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":1.21,"15min":1.159},"min":{"1min":0.63700000000000001,"5min":0.371,"15min":0.316},"max":{"1min":1.2809999999999999,"5min":4.8869999999999996,"15min":10.398},"last":0.74299999999999999},{"interface":"front","average":{"1min":1.0409999999999999,"5min":1.448,"15min":1.3520000000000001},"min":{"1min":0.66800000000000004,"5min":0.39500000000000002,"15min":0.36499999999999999},"max":{"1min":1.3200000000000001,"5min":8.5820000000000007,"15min":8.6699999999999999},"last":0.86299999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:29:40 2023","interfaces":[{"interface":"back","average":{"1min":1.0669999999999999,"5min":1.1539999999999999,"15min":1.1419999999999999},"min":{"1min":0.55600000000000005,"5min":0.41799999999999998,"15min":0.38300000000000001},"max":{"1min":1.3740000000000001,"5min":8.7949999999999999,"15min":8.7949999999999999},"last":0.90300000000000002},{"interface":"front","average":{"1min":1.1639999999999999,"5min":1.2290000000000001,"15min":1.3109999999999999},"min":{"1min":0.85099999999999998,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.516,"5min":3.9820000000000002,"15min":6.7400000000000002},"last":1.034}]}]},{"osd":1,"up_from":98,"seq":420906795097,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":252144,"kb_used_data":1192,"kb_used_omap":4,"kb_used_meta":250939,"kb_avail":93493008,"statfs":{"total":95995035648,"available":95736840192,"internally_reserved":0,"allocated":1220608,"data_stored":707168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":256962446},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:29:55 2023","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":1.359,"15min":2.1509999999999998},"min":{"1min":0.34300000000000003,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":1.036,"5min":14.99,"15min":14.99},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.63100000000000001,"5min":1.2929999999999999,"15min":2.1110000000000002},"min":{"1min":0.255,"5min":0.255,"15min":0.23499999999999999},"max":{"1min":0.92400000000000004,"5min":12.215,"15min":12.215},"last":0.83599999999999997}]},{"osd":2,"last update":"Sat Sep 23 19:29:28 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":0.72799999999999998,"15min":0.73399999999999999},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":1.3200000000000001,"5min":1.3200000000000001,"15min":1.3200000000000001},"last":1.069},{"interface":"front","average":{"1min":0.74399999999999999,"5min":0.68799999999999994,"15min":0.67800000000000005},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":1.2350000000000001,"5min":1.2350000000000001,"15min":1.2350000000000001},"last":0.68999999999999995}]},{"osd":3,"last update":"Sat Sep 23 19:29:07 2023","interfaces":[{"interface":"back","average":{"1min":0.753,"5min":0.753,"15min":0.753},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":1.02,"5min":1.02,"15min":1.02},"last":1.0349999999999999},{"interface":"front","average":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"min":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"max":{"1min":1.131,"5min":1.131,"15min":1.131},"last":1.008}]},{"osd":4,"last update":"Sat Sep 23 19:29:55 2023","interfaces":[{"interface":"back","average":{"1min":0.92900000000000005,"5min":1.3140000000000001,"15min":1.6379999999999999},"min":{"1min":0.52400000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":1.2889999999999999,"5min":6.2830000000000004,"15min":7.6130000000000004},"last":1.2889999999999999},{"interface":"front","average":{"1min":0.95799999999999996,"5min":1.512,"15min":1.383},"min":{"1min":0.58899999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.3600000000000001,"5min":10.69,"15min":10.69},"last":1.3600000000000001}]},{"osd":5,"last update":"Sat Sep 23 19:29:55 2023","interfaces":[{"interface":"back","average":{"1min":0.94699999999999995,"5min":1.5009999999999999,"15min":1.464},"min":{"1min":0.626,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":1.2669999999999999,"5min":10.714,"15min":12.640000000000001},"last":1.141},{"interface":"front","average":{"1min":0.999,"5min":1.4830000000000001,"15min":1.5549999999999999},"min":{"1min":0.61099999999999999,"5min":0.38700000000000001,"15min":0.36299999999999999},"max":{"1min":1.4730000000000001,"5min":9.218,"15min":9.218},"last":0.96599999999999997}]},{"osd":6,"last update":"Sat Sep 23 19:29:55 2023","interfaces":[{"interface":"back","average":{"1min":1.0629999999999999,"5min":1.5800000000000001,"15min":1.595},"min":{"1min":0.63800000000000001,"5min":0.45100000000000001,"15min":0.40100000000000002},"max":{"1min":1.508,"5min":9.3010000000000002,"15min":9.3010000000000002},"last":1.117},{"interface":"front","average":{"1min":1.0720000000000001,"5min":1.5880000000000001,"15min":1.8089999999999999},"min":{"1min":0.86699999999999999,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.528,"5min":9.1699999999999999,"15min":9.1699999999999999},"last":1.528}]},{"osd":7,"last update":"Sat Sep 23 19:29:55 2023","interfaces":[{"interface":"back","average":{"1min":1.0589999999999999,"5min":1.4299999999999999,"15min":1.9990000000000001},"min":{"1min":0.72699999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":1.617,"5min":9.2159999999999993,"15min":9.2159999999999993},"last":1.617},{"interface":"front","average":{"1min":1.0700000000000001,"5min":1.454,"15min":1.5600000000000001},"min":{"1min":0.65200000000000002,"5min":0.42899999999999999,"15min":0.42699999999999999},"max":{"1min":1.349,"5min":10.742000000000001,"15min":10.742000000000001},"last":1.262}]}]},{"osd":0,"up_from":85,"seq":365072220304,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":310232,"kb_used_data":1808,"kb_used_omap":5,"kb_used_meta":308410,"kb_avail":93434920,"statfs":{"total":95995035648,"available":95677358080,"internally_reserved":0,"allocated":1851392,"data_stored":1304001,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":315812474},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 19:29:54 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":1.3640000000000001,"15min":1.534},"min":{"1min":0.23899999999999999,"5min":0.22500000000000001,"15min":0.214},"max":{"1min":0.86499999999999999,"5min":9.2690000000000001,"15min":13.557},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.66800000000000004,"5min":1.167,"15min":1.954},"min":{"1min":0.29099999999999998,"5min":0.253,"15min":0.253},"max":{"1min":0.84599999999999997,"5min":7.6920000000000002,"15min":11.507999999999999},"last":0.72499999999999998}]},{"osd":2,"last update":"Sat Sep 23 19:29:25 2023","interfaces":[{"interface":"back","average":{"1min":0.70299999999999996,"5min":0.75800000000000001,"15min":0.76700000000000002},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":1.2410000000000001,"5min":1.2410000000000001,"15min":1.2410000000000001},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.72699999999999998,"5min":0.77500000000000002,"15min":0.78300000000000003},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":1.2330000000000001,"5min":1.2330000000000001,"15min":1.2330000000000001},"last":0.52200000000000002}]},{"osd":3,"last update":"Sat Sep 23 19:29:07 2023","interfaces":[{"interface":"back","average":{"1min":0.754,"5min":0.754,"15min":0.754},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":1.1259999999999999,"5min":1.1259999999999999,"15min":1.1259999999999999},"last":0.78900000000000003},{"interface":"front","average":{"1min":0.97999999999999998,"5min":0.97999999999999998,"15min":0.97999999999999998},"min":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"max":{"1min":1.5149999999999999,"5min":1.5149999999999999,"15min":1.5149999999999999},"last":0.89500000000000002}]},{"osd":4,"last update":"Sat Sep 23 19:29:33 2023","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":1.2110000000000001,"15min":1.6619999999999999},"min":{"1min":0.54400000000000004,"5min":0.33000000000000002,"15min":0.32100000000000001},"max":{"1min":1.5309999999999999,"5min":6.5659999999999998,"15min":11.156000000000001},"last":0.95399999999999996},{"interface":"front","average":{"1min":1.0900000000000001,"5min":1.4690000000000001,"15min":1.784},"min":{"1min":0.52600000000000002,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.536,"5min":5.8979999999999997,"15min":13.566000000000001},"last":1.0980000000000001}]},{"osd":5,"last update":"Sat Sep 23 19:29:33 2023","interfaces":[{"interface":"back","average":{"1min":1.1220000000000001,"5min":1.6639999999999999,"15min":1.819},"min":{"1min":0.505,"5min":0.371,"15min":0.33600000000000002},"max":{"1min":1.51,"5min":11.553000000000001,"15min":13.509},"last":0.98899999999999999},{"interface":"front","average":{"1min":1.083,"5min":1.472,"15min":1.804},"min":{"1min":0.628,"5min":0.38400000000000001,"15min":0.33400000000000002},"max":{"1min":1.6160000000000001,"5min":7.0229999999999997,"15min":11.048999999999999},"last":1.145}]},{"osd":6,"last update":"Sat Sep 23 19:29:33 2023","interfaces":[{"interface":"back","average":{"1min":1.0620000000000001,"5min":1.3109999999999999,"15min":1.8560000000000001},"min":{"1min":0.64100000000000001,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.669,"5min":7.069,"15min":11.467000000000001},"last":1.0580000000000001},{"interface":"front","average":{"1min":1.177,"5min":1.601,"15min":1.9119999999999999},"min":{"1min":0.81100000000000005,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":1.5860000000000001,"5min":9.6679999999999993,"15min":13.603999999999999},"last":1.206}]},{"osd":7,"last update":"Sat Sep 23 19:29:33 2023","interfaces":[{"interface":"back","average":{"1min":1.1379999999999999,"5min":1.5329999999999999,"15min":1.9199999999999999},"min":{"1min":0.72199999999999998,"5min":0.44,"15min":0.434},"max":{"1min":1.645,"5min":13.218,"15min":16.271999999999998},"last":0.92900000000000005},{"interface":"front","average":{"1min":1.133,"5min":1.417,"15min":1.861},"min":{"1min":0.67500000000000004,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":1.5620000000000001,"5min":7.101,"15min":11.098000000000001},"last":1.25}]}]},{"osd":2,"up_from":111,"seq":476741369889,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":161480,"kb_used_data":1152,"kb_used_omap":7,"kb_used_meta":160312,"kb_avail":93583672,"statfs":{"total":95995035648,"available":95829680128,"internally_reserved":0,"allocated":1179648,"data_stored":701589,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":164159569},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:29:26 2023","interfaces":[{"interface":"back","average":{"1min":0.71199999999999997,"5min":0.70799999999999996,"15min":0.70699999999999996},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":1.1839999999999999,"5min":1.1839999999999999,"15min":1.1839999999999999},"last":0.91100000000000003},{"interface":"front","average":{"1min":0.73599999999999999,"5min":0.76100000000000001,"15min":0.76500000000000001},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.038,"5min":1.1639999999999999,"15min":1.1639999999999999},"last":0.86099999999999999}]},{"osd":1,"last update":"Sat Sep 23 19:29:26 2023","interfaces":[{"interface":"back","average":{"1min":0.67800000000000005,"5min":0.74199999999999999,"15min":0.753},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":1.0700000000000001,"5min":1.3740000000000001,"15min":1.3740000000000001},"last":0.81299999999999994},{"interface":"front","average":{"1min":0.80100000000000005,"5min":0.84099999999999997,"15min":0.84799999999999998},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":1.097,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":1.2310000000000001}]},{"osd":3,"last update":"Sat Sep 23 19:29:07 2023","interfaces":[{"interface":"back","average":{"1min":0.746,"5min":0.746,"15min":0.746},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":1.3280000000000001,"5min":1.3280000000000001,"15min":1.3280000000000001},"last":0.71099999999999997},{"interface":"front","average":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":1.5389999999999999,"5min":1.5389999999999999,"15min":1.5389999999999999},"last":1.0960000000000001}]},{"osd":4,"last update":"Sat Sep 23 19:29:26 2023","interfaces":[{"interface":"back","average":{"1min":0.996,"5min":1.0169999999999999,"15min":1.0209999999999999},"min":{"1min":0.60499999999999998,"5min":0.58699999999999997,"15min":0.58699999999999997},"max":{"1min":1.353,"5min":1.353,"15min":1.353},"last":0.98799999999999999},{"interface":"front","average":{"1min":1.0129999999999999,"5min":1.0089999999999999,"15min":1.0089999999999999},"min":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"max":{"1min":1.5,"5min":1.5,"15min":1.5},"last":1.3169999999999999}]},{"osd":5,"last update":"Sat Sep 23 19:29:26 2023","interfaces":[{"interface":"back","average":{"1min":1.0860000000000001,"5min":1.1299999999999999,"15min":1.137},"min":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"max":{"1min":1.371,"5min":1.554,"15min":1.554},"last":1.3009999999999999},{"interface":"front","average":{"1min":1.0620000000000001,"5min":1.127,"15min":1.1379999999999999},"min":{"1min":0.68200000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"max":{"1min":1.4690000000000001,"5min":1.8340000000000001,"15min":1.8340000000000001},"last":1.1379999999999999}]},{"osd":6,"last update":"Sat Sep 23 19:29:26 2023","interfaces":[{"interface":"back","average":{"1min":0.997,"5min":1.089,"15min":1.105},"min":{"1min":0.64000000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"max":{"1min":1.409,"5min":1.738,"15min":1.738},"last":1.1679999999999999},{"interface":"front","average":{"1min":1.1539999999999999,"5min":1.167,"15min":1.169},"min":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"max":{"1min":1.6140000000000001,"5min":1.7030000000000001,"15min":1.7030000000000001},"last":1.347}]},{"osd":7,"last update":"Sat Sep 23 19:29:26 2023","interfaces":[{"interface":"back","average":{"1min":1.117,"5min":1.1970000000000001,"15min":1.2110000000000001},"min":{"1min":0.79300000000000004,"5min":0.74399999999999999,"15min":0.74399999999999999},"max":{"1min":1.5840000000000001,"5min":1.77,"15min":1.77},"last":1.206},{"interface":"front","average":{"1min":1.131,"5min":1.173,"15min":1.1799999999999999},"min":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"max":{"1min":1.5669999999999999,"5min":1.5669999999999999,"15min":1.5669999999999999},"last":1.383}]}]},{"osd":3,"up_from":116,"seq":498216206360,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":160112,"kb_used_data":1128,"kb_used_omap":2,"kb_used_meta":158973,"kb_avail":93585040,"statfs":{"total":95995035648,"available":95831080960,"internally_reserved":0,"allocated":1155072,"data_stored":697144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":162788405},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:29:08 2023","interfaces":[{"interface":"back","average":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"min":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"min":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":1.3160000000000001,"5min":1.3160000000000001,"15min":1.3160000000000001},"last":0.89600000000000002}]},{"osd":1,"last update":"Sat Sep 23 19:29:08 2023","interfaces":[{"interface":"back","average":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"min":{"1min":0.502,"5min":0.502,"15min":0.502},"max":{"1min":1.0629999999999999,"5min":1.0629999999999999,"15min":1.0629999999999999},"last":1.0700000000000001},{"interface":"front","average":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"min":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":1.1240000000000001,"5min":1.1240000000000001,"15min":1.1240000000000001},"last":0.85399999999999998}]},{"osd":2,"last update":"Sat Sep 23 19:29:08 2023","interfaces":[{"interface":"back","average":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"min":{"1min":0.495,"5min":0.495,"15min":0.495},"max":{"1min":1.137,"5min":1.137,"15min":1.137},"last":0.70799999999999996},{"interface":"front","average":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"min":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"max":{"1min":1.2849999999999999,"5min":1.2849999999999999,"15min":1.2849999999999999},"last":1.103}]},{"osd":4,"last update":"Sat Sep 23 19:29:08 2023","interfaces":[{"interface":"back","average":{"1min":1.077,"5min":1.077,"15min":1.077},"min":{"1min":0.753,"5min":0.753,"15min":0.753},"max":{"1min":1.454,"5min":1.454,"15min":1.454},"last":0.97599999999999998},{"interface":"front","average":{"1min":1.149,"5min":1.149,"15min":1.149},"min":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"max":{"1min":1.506,"5min":1.506,"15min":1.506},"last":1.329}]},{"osd":5,"last update":"Sat Sep 23 19:29:08 2023","interfaces":[{"interface":"back","average":{"1min":1.1259999999999999,"5min":1.1259999999999999,"15min":1.1259999999999999},"min":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"max":{"1min":1.4059999999999999,"5min":1.4059999999999999,"15min":1.4059999999999999},"last":1.27},{"interface":"front","average":{"1min":1.081,"5min":1.081,"15min":1.081},"min":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"max":{"1min":1.3520000000000001,"5min":1.3520000000000001,"15min":1.3520000000000001},"last":1.2250000000000001}]},{"osd":6,"last update":"Sat Sep 23 19:29:08 2023","interfaces":[{"interface":"back","average":{"1min":1.1459999999999999,"5min":1.1459999999999999,"15min":1.1459999999999999},"min":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"max":{"1min":1.431,"5min":1.431,"15min":1.431},"last":1.1699999999999999},{"interface":"front","average":{"1min":1.1479999999999999,"5min":1.1479999999999999,"15min":1.1479999999999999},"min":{"1min":0.748,"5min":0.748,"15min":0.748},"max":{"1min":1.5109999999999999,"5min":1.5109999999999999,"15min":1.5109999999999999},"last":1.349}]},{"osd":7,"last update":"Sat Sep 23 19:29:08 2023","interfaces":[{"interface":"back","average":{"1min":1.1599999999999999,"5min":1.1599999999999999,"15min":1.1599999999999999},"min":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"max":{"1min":1.597,"5min":1.597,"15min":1.597},"last":1.294},{"interface":"front","average":{"1min":1.1559999999999999,"5min":1.1559999999999999,"15min":1.1559999999999999},"min":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"max":{"1min":1.5489999999999999,"5min":1.5489999999999999,"15min":1.5489999999999999},"last":1.1899999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019230,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":590832,"kb_used_data":1128,"kb_used_omap":0,"kb_used_meta":589696,"kb_avail":93154320,"statfs":{"total":95995035648,"available":95390023680,"internally_reserved":0,"allocated":1155072,"data_stored":697491,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603848704},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:29:30 2023","interfaces":[{"interface":"back","average":{"1min":0.88100000000000001,"5min":1.3260000000000001,"15min":3.306},"min":{"1min":0.61499999999999999,"5min":0.27400000000000002,"15min":0.26200000000000001},"max":{"1min":1.1719999999999999,"5min":8.7680000000000007,"15min":129.16200000000001},"last":0.91700000000000004},{"interface":"front","average":{"1min":0.90400000000000003,"5min":1.244,"15min":3.5640000000000001},"min":{"1min":0.53700000000000003,"5min":0.32200000000000001,"15min":0.27700000000000002},"max":{"1min":1.23,"5min":6.2450000000000001,"15min":130.40600000000001},"last":1.25}]},{"osd":1,"last update":"Sat Sep 23 19:28:55 2023","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":1.6659999999999999,"15min":1.669},"min":{"1min":0.65800000000000003,"5min":0.30099999999999999,"15min":0.29499999999999998},"max":{"1min":1.179,"5min":12.372,"15min":12.372},"last":1.0629999999999999},{"interface":"front","average":{"1min":0.97199999999999998,"5min":1.663,"15min":1.5880000000000001},"min":{"1min":0.54900000000000004,"5min":0.31,"15min":0.31},"max":{"1min":1.274,"5min":7.3360000000000003,"15min":7.3360000000000003},"last":0.996}]},{"osd":2,"last update":"Sat Sep 23 19:29:26 2023","interfaces":[{"interface":"back","average":{"1min":0.92500000000000004,"5min":0.89300000000000002,"15min":0.88800000000000001},"min":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"max":{"1min":1.2290000000000001,"5min":1.327,"15min":1.327},"last":1.171},{"interface":"front","average":{"1min":0.90400000000000003,"5min":0.89900000000000002,"15min":0.89800000000000002},"min":{"1min":0.69699999999999995,"5min":0.51700000000000002,"15min":0.51700000000000002},"max":{"1min":1.1599999999999999,"5min":1.4379999999999999,"15min":1.4379999999999999},"last":1.1080000000000001}]},{"osd":3,"last update":"Sat Sep 23 19:29:10 2023","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":0.97499999999999998,"15min":0.97499999999999998},"min":{"1min":0.622,"5min":0.622,"15min":0.622},"max":{"1min":1.2909999999999999,"5min":1.2909999999999999,"15min":1.2909999999999999},"last":1.151},{"interface":"front","average":{"1min":0.96199999999999997,"5min":0.96199999999999997,"15min":0.96199999999999997},"min":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"max":{"1min":1.552,"5min":1.552,"15min":1.552},"last":1.202}]},{"osd":5,"last update":"Sat Sep 23 19:29:42 2023","interfaces":[{"interface":"back","average":{"1min":1.018,"5min":1.4299999999999999,"15min":1.1279999999999999},"min":{"1min":0.57699999999999996,"5min":0.32500000000000001,"15min":0.29199999999999998},"max":{"1min":2.1709999999999998,"5min":12.459,"15min":12.459},"last":0.82199999999999995},{"interface":"front","average":{"1min":1.1220000000000001,"5min":1.4730000000000001,"15min":1.234},"min":{"1min":0.877,"5min":0.34699999999999998,"15min":0.26700000000000002},"max":{"1min":1.8680000000000001,"5min":12.478,"15min":12.478},"last":1.272}]},{"osd":6,"last update":"Sat Sep 23 19:28:59 2023","interfaces":[{"interface":"back","average":{"1min":0.98699999999999999,"5min":1.6220000000000001,"15min":1.2609999999999999},"min":{"1min":0.46600000000000003,"5min":0.377,"15min":0.29999999999999999},"max":{"1min":1.397,"5min":7.0810000000000004,"15min":7.0810000000000004},"last":1.2370000000000001},{"interface":"front","average":{"1min":1.1120000000000001,"5min":1.4390000000000001,"15min":1.248},"min":{"1min":0.626,"5min":0.40600000000000003,"15min":0.34300000000000003},"max":{"1min":1.4950000000000001,"5min":4.3440000000000003,"15min":6.9050000000000002},"last":1.3300000000000001}]},{"osd":7,"last update":"Sat Sep 23 19:28:55 2023","interfaces":[{"interface":"back","average":{"1min":0.98199999999999998,"5min":1.4059999999999999,"15min":1.244},"min":{"1min":0.55400000000000005,"5min":0.41599999999999998,"15min":0.312},"max":{"1min":1.4430000000000001,"5min":7.8479999999999999,"15min":7.8479999999999999},"last":1.3540000000000001},{"interface":"front","average":{"1min":1.0569999999999999,"5min":1.3080000000000001,"15min":1.147},"min":{"1min":0.58499999999999996,"5min":0.40899999999999997,"15min":0.34999999999999998},"max":{"1min":1.538,"5min":5.8570000000000002,"15min":8.3460000000000001},"last":1.3100000000000001}]}]},{"osd":5,"up_from":36,"seq":154618822997,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":594904,"kb_used_data":1104,"kb_used_omap":0,"kb_used_meta":593792,"kb_avail":93150248,"statfs":{"total":95995035648,"available":95385853952,"internally_reserved":0,"allocated":1130496,"data_stored":692605,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":608043008},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.88100000000000001,"5min":1.0649999999999999,"15min":1.3360000000000001},"min":{"1min":0.49299999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.2360000000000002,"5min":7.6669999999999998,"15min":11.058},"last":0.82299999999999995},{"interface":"front","average":{"1min":0.98899999999999999,"5min":1.478,"15min":1.7050000000000001},"min":{"1min":0.54000000000000004,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":3.7890000000000001,"5min":10.802,"15min":10.802},"last":1.0640000000000001}]},{"osd":1,"last update":"Sat Sep 23 19:28:55 2023","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":1.385,"15min":1.28},"min":{"1min":0.57299999999999995,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":1.869,"5min":9.5389999999999997,"15min":9.5389999999999997},"last":0.73299999999999998},{"interface":"front","average":{"1min":0.95399999999999996,"5min":1.579,"15min":2.1920000000000002},"min":{"1min":0.67400000000000004,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":1.7829999999999999,"5min":11.228999999999999,"15min":11.228999999999999},"last":0.91100000000000003}]},{"osd":2,"last update":"Sat Sep 23 19:29:29 2023","interfaces":[{"interface":"back","average":{"1min":1.0700000000000001,"5min":0.98099999999999998,"15min":0.96599999999999997},"min":{"1min":0.65100000000000002,"5min":0.56899999999999995,"15min":0.56899999999999995},"max":{"1min":2.3069999999999999,"5min":2.3069999999999999,"15min":2.3069999999999999},"last":1.04},{"interface":"front","average":{"1min":1.151,"5min":1.0680000000000001,"15min":1.054},"min":{"1min":0.78800000000000003,"5min":0.753,"15min":0.753},"max":{"1min":2.335,"5min":2.335,"15min":2.335},"last":1.0860000000000001}]},{"osd":3,"last update":"Sat Sep 23 19:29:09 2023","interfaces":[{"interface":"back","average":{"1min":1.0620000000000001,"5min":1.0620000000000001,"15min":1.0620000000000001},"min":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"max":{"1min":1.702,"5min":1.702,"15min":1.702},"last":1.268},{"interface":"front","average":{"1min":1.1739999999999999,"5min":1.1739999999999999,"15min":1.1739999999999999},"min":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"max":{"1min":2.5179999999999998,"5min":2.5179999999999998,"15min":2.5179999999999998},"last":1.196}]},{"osd":4,"last update":"Sat Sep 23 19:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":1.244,"15min":1.1519999999999999},"min":{"1min":0.63600000000000001,"5min":0.33200000000000002,"15min":0.29199999999999998},"max":{"1min":1.2669999999999999,"5min":8.1790000000000003,"15min":13.753},"last":1},{"interface":"front","average":{"1min":0.96899999999999997,"5min":1.3029999999999999,"15min":1.137},"min":{"1min":0.72199999999999998,"5min":0.34399999999999997,"15min":0.27000000000000002},"max":{"1min":1.444,"5min":13.714,"15min":13.714},"last":0.96599999999999997}]},{"osd":6,"last update":"Sat Sep 23 19:29:14 2023","interfaces":[{"interface":"back","average":{"1min":1.0860000000000001,"5min":1.2829999999999999,"15min":1.246},"min":{"1min":0.68899999999999995,"5min":0.36899999999999999,"15min":0.28599999999999998},"max":{"1min":1.4730000000000001,"5min":7.3739999999999997,"15min":26.448},"last":1.296},{"interface":"front","average":{"1min":1.149,"5min":1.3959999999999999,"15min":1.298},"min":{"1min":0.76900000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.6990000000000001,"5min":4.8399999999999999,"15min":26.841999999999999},"last":1.22}]},{"osd":7,"last update":"Sat Sep 23 19:29:47 2023","interfaces":[{"interface":"back","average":{"1min":1.161,"5min":1.1379999999999999,"15min":1.361},"min":{"1min":0.77000000000000002,"5min":0.41399999999999998,"15min":0.36599999999999999},"max":{"1min":1.794,"5min":4.7990000000000004,"15min":32.585999999999999},"last":1.1399999999999999},{"interface":"front","average":{"1min":1.1850000000000001,"5min":1.145,"15min":1.25},"min":{"1min":0.80400000000000005,"5min":0.39800000000000002,"15min":0.35199999999999998},"max":{"1min":1.542,"5min":7.7050000000000001,"15min":33.014000000000003},"last":1.3799999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T19:30:02.681 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-09-23T19:30:02.681 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-09-23T19:30:02.681 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-09-23T19:30:02.681 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph health --format=json 2023-09-23T19:30:04.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:30:03 smithi016 bash[75004]: cluster 2023-09-23T19:30:03.902934+0000 mgr.y (mgr.25325) 670 : cluster 0 pgmap v565: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:04.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:30:03 smithi016 bash[77995]: cluster 2023-09-23T19:30:03.902934+0000 mgr.y (mgr.25325) 670 : cluster 0 pgmap v565: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:04.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:30:03 smithi092 bash[58855]: cluster 2023-09-23T19:30:03.902934+0000 mgr.y (mgr.25325) 670 : cluster 0 pgmap v565: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:05.967 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:30:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:30:05] "GET /metrics HTTP/1.1" 200 33814 "" "Prometheus/2.33.4" 2023-09-23T19:30:06.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:30:05 smithi016 bash[75004]: cluster 2023-09-23T19:30:05.903897+0000 mgr.y (mgr.25325) 671 : cluster 0 pgmap v566: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:06.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:30:05 smithi016 bash[77995]: cluster 2023-09-23T19:30:05.903897+0000 mgr.y (mgr.25325) 671 : cluster 0 pgmap v566: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:06.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:30:05 smithi092 bash[58855]: cluster 2023-09-23T19:30:05.903897+0000 mgr.y (mgr.25325) 671 : cluster 0 pgmap v566: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:07.499 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:30:08.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:30:07 smithi016 bash[75004]: cluster 2023-09-23T19:30:07.904592+0000 mgr.y (mgr.25325) 672 : cluster 0 pgmap v567: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:08.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:30:07 smithi016 bash[77995]: cluster 2023-09-23T19:30:07.904592+0000 mgr.y (mgr.25325) 672 : cluster 0 pgmap v567: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:08.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:30:07 smithi092 bash[58855]: cluster 2023-09-23T19:30:07.904592+0000 mgr.y (mgr.25325) 672 : cluster 0 pgmap v567: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:10.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:30:10 smithi016 bash[75004]: cluster 2023-09-23T19:30:09.905290+0000 mgr.y (mgr.25325) 673 : cluster 0 pgmap v568: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:10.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:30:10 smithi016 bash[77995]: cluster 2023-09-23T19:30:09.905290+0000 mgr.y (mgr.25325) 673 : cluster 0 pgmap v568: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:10.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:30:10 smithi092 bash[58855]: cluster 2023-09-23T19:30:09.905290+0000 mgr.y (mgr.25325) 673 : cluster 0 pgmap v568: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:10.798 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-09-23T19:30:10.799 INFO:teuthology.orchestra.run.smithi016.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-09-23T19:30:11.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:30:11 smithi016 bash[75004]: audit 2023-09-23T19:30:10.796226+0000 mon.a (mon.0) 884 : audit 0 from='client.? 172.21.15.16:0/616131079' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T19:30:11.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:30:11 smithi016 bash[77995]: audit 2023-09-23T19:30:10.796226+0000 mon.a (mon.0) 884 : audit 0 from='client.? 172.21.15.16:0/616131079' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T19:30:11.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:30:11 smithi092 bash[58855]: audit 2023-09-23T19:30:10.796226+0000 mon.a (mon.0) 884 : audit 0 from='client.? 172.21.15.16:0/616131079' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T19:30:11.649 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-09-23T19:30:11.649 INFO:teuthology.run_tasks:Running task print... 2023-09-23T19:30:11.658 INFO:teuthology.task.print:**** applying stress + thrashing to mixed-version cluster **** 2023-09-23T19:30:11.658 INFO:teuthology.run_tasks:Running task parallel... 2023-09-23T19:30:11.669 INFO:teuthology.task.parallel:starting parallel... 2023-09-23T19:30:11.669 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-09-23T19:30:11.669 INFO:teuthology.task.sequential:In sequential, running task thrashosds... 2023-09-23T19:30:11.671 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-09-23T19:30:11.671 INFO:tasks.thrashosds:overrides is {'admin_socket': {'branch': 'reef'}, 'ceph': {'conf': {'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'}}, 'kernel': {'hwe': True}, 'workunit': {'branch': 'reef', 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'}} 2023-09-23T19:30:11.671 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-09-23T19:30:11.671 INFO:tasks.thrashosds:Beginning thrashosds... 2023-09-23T19:30:11.671 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-09-23T19:30:11.672 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T19:30:12.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:30:12 smithi016 bash[75004]: cluster 2023-09-23T19:30:11.906056+0000 mgr.y (mgr.25325) 674 : cluster 0 pgmap v569: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:12.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:30:12 smithi016 bash[77995]: cluster 2023-09-23T19:30:11.906056+0000 mgr.y (mgr.25325) 674 : cluster 0 pgmap v569: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:12.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:30:12 smithi092 bash[58855]: cluster 2023-09-23T19:30:11.906056+0000 mgr.y (mgr.25325) 674 : cluster 0 pgmap v569: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:14.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:30:13 smithi016 bash[75004]: audit 2023-09-23T19:30:13.634613+0000 mon.a (mon.0) 885 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:30:14.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:30:13 smithi016 bash[77995]: audit 2023-09-23T19:30:13.634613+0000 mon.a (mon.0) 885 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:30:14.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:30:13 smithi092 bash[58855]: audit 2023-09-23T19:30:13.634613+0000 mon.a (mon.0) 885 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:30:15.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:30:14 smithi016 bash[75004]: cluster 2023-09-23T19:30:13.906737+0000 mgr.y (mgr.25325) 675 : cluster 0 pgmap v570: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:15.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:30:14 smithi016 bash[77995]: cluster 2023-09-23T19:30:13.906737+0000 mgr.y (mgr.25325) 675 : cluster 0 pgmap v570: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:15.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:30:14 smithi092 bash[58855]: cluster 2023-09-23T19:30:13.906737+0000 mgr.y (mgr.25325) 675 : cluster 0 pgmap v570: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:15.973 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:30:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:30:15] "GET /metrics HTTP/1.1" 200 33811 "" "Prometheus/2.33.4" 2023-09-23T19:30:16.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:30:15 smithi016 bash[75004]: cluster 2023-09-23T19:30:15.907819+0000 mgr.y (mgr.25325) 676 : cluster 0 pgmap v571: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:16.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:30:15 smithi016 bash[77995]: cluster 2023-09-23T19:30:15.907819+0000 mgr.y (mgr.25325) 676 : cluster 0 pgmap v571: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:16.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:30:15 smithi092 bash[58855]: cluster 2023-09-23T19:30:15.907819+0000 mgr.y (mgr.25325) 676 : cluster 0 pgmap v571: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:16.458 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:30:18.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:30:17 smithi016 bash[75004]: cluster 2023-09-23T19:30:17.908725+0000 mgr.y (mgr.25325) 677 : cluster 0 pgmap v572: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:18.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:30:17 smithi016 bash[77995]: cluster 2023-09-23T19:30:17.908725+0000 mgr.y (mgr.25325) 677 : cluster 0 pgmap v572: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:18.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:30:17 smithi092 bash[58855]: cluster 2023-09-23T19:30:17.908725+0000 mgr.y (mgr.25325) 677 : cluster 0 pgmap v572: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:19.793 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-09-23T19:30:19.797 INFO:teuthology.orchestra.run.smithi016.stderr:dumped all 2023-09-23T19:30:20.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:30:20 smithi016 bash[75004]: audit 2023-09-23T19:30:19.778092+0000 mgr.y (mgr.25325) 678 : audit 0 from='client.45602 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:30:20.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:30:20 smithi016 bash[75004]: cluster 2023-09-23T19:30:19.909533+0000 mgr.y (mgr.25325) 679 : cluster 0 pgmap v573: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:20.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:30:20 smithi016 bash[77995]: audit 2023-09-23T19:30:19.778092+0000 mgr.y (mgr.25325) 678 : audit 0 from='client.45602 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:30:20.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:30:20 smithi016 bash[77995]: cluster 2023-09-23T19:30:19.909533+0000 mgr.y (mgr.25325) 679 : cluster 0 pgmap v573: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:20.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:30:20 smithi092 bash[58855]: audit 2023-09-23T19:30:19.778092+0000 mgr.y (mgr.25325) 678 : audit 0 from='client.45602 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:30:20.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:30:20 smithi092 bash[58855]: cluster 2023-09-23T19:30:19.909533+0000 mgr.y (mgr.25325) 679 : cluster 0 pgmap v573: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:20.573 INFO:teuthology.orchestra.run.smithi016.stdout:{"pg_ready":true,"pg_map":{"version":572,"stamp":"2023-09-23T19:30:17.908168+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":261,"num_write_kb":2446,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":195,"ondisk_log_size":195,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2683944,"kb_used_data":10920,"kb_used_omap":20,"kb_used_meta":2672939,"kb_avail":747277272,"statfs":{"total":767960285184,"available":765211926528,"internally_reserved":0,"allocated":11182080,"data_stored":7375395,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":2737090190},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004925"},"pg_stats":[{"pgid":"3.1a","version":"59'1","reported_seq":100,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.878040+0000","last_change":"2023-09-23T19:27:24.274696+0000","last_active":"2023-09-23T19:28:06.878040+0000","last_peered":"2023-09-23T19:28:06.878040+0000","last_clean":"2023-09-23T19:28:06.878040+0000","last_became_active":"2023-09-23T19:27:24.273561+0000","last_became_peered":"2023-09-23T19:27:24.273561+0000","last_unstale":"2023-09-23T19:28:06.878040+0000","last_undegraded":"2023-09-23T19:28:06.878040+0000","last_fullsized":"2023-09-23T19:28:06.878040+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":78,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.894651+0000","last_change":"2023-09-23T19:28:07.894651+0000","last_active":"2023-09-23T19:28:07.894651+0000","last_peered":"2023-09-23T19:28:07.894651+0000","last_clean":"2023-09-23T19:28:07.894651+0000","last_became_active":"2023-09-23T19:28:07.892919+0000","last_became_peered":"2023-09-23T19:28:07.892919+0000","last_unstale":"2023-09-23T19:28:07.894651+0000","last_undegraded":"2023-09-23T19:28:07.894651+0000","last_fullsized":"2023-09-23T19:28:07.894651+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:56.579997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":58,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.890735+0000","last_change":"2023-09-23T19:28:07.890462+0000","last_active":"2023-09-23T19:28:07.890735+0000","last_peered":"2023-09-23T19:28:07.890735+0000","last_clean":"2023-09-23T19:28:07.890735+0000","last_became_active":"2023-09-23T19:28:07.890084+0000","last_became_peered":"2023-09-23T19:28:07.890084+0000","last_unstale":"2023-09-23T19:28:07.890735+0000","last_undegraded":"2023-09-23T19:28:07.890735+0000","last_fullsized":"2023-09-23T19:28:07.890735+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:50.458218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":74,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:08.830333+0000","last_change":"2023-09-23T19:28:07.884498+0000","last_active":"2023-09-23T19:28:08.830333+0000","last_peered":"2023-09-23T19:28:08.830333+0000","last_clean":"2023-09-23T19:28:08.830333+0000","last_became_active":"2023-09-23T19:28:07.884102+0000","last_became_peered":"2023-09-23T19:28:07.884102+0000","last_unstale":"2023-09-23T19:28:08.830333+0000","last_undegraded":"2023-09-23T19:28:08.830333+0000","last_fullsized":"2023-09-23T19:28:08.830333+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:05:26.348041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":94,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.887122+0000","last_change":"2023-09-23T19:28:07.887122+0000","last_active":"2023-09-23T19:28:07.887122+0000","last_peered":"2023-09-23T19:28:07.887122+0000","last_clean":"2023-09-23T19:28:07.887122+0000","last_became_active":"2023-09-23T19:28:07.886743+0000","last_became_peered":"2023-09-23T19:28:07.886743+0000","last_unstale":"2023-09-23T19:28:07.887122+0000","last_undegraded":"2023-09-23T19:28:07.887122+0000","last_fullsized":"2023-09-23T19:28:07.887122+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:59:45.973619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":83,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.876761+0000","last_change":"2023-09-23T19:22:42.150582+0000","last_active":"2023-09-23T19:28:06.876761+0000","last_peered":"2023-09-23T19:28:06.876761+0000","last_clean":"2023-09-23T19:28:06.876761+0000","last_became_active":"2023-09-23T19:22:42.150229+0000","last_became_peered":"2023-09-23T19:22:42.150229+0000","last_unstale":"2023-09-23T19:28:06.876761+0000","last_undegraded":"2023-09-23T19:28:06.876761+0000","last_fullsized":"2023-09-23T19:28:06.876761+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":73,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.887587+0000","last_change":"2023-09-23T19:28:07.887141+0000","last_active":"2023-09-23T19:28:07.887587+0000","last_peered":"2023-09-23T19:28:07.887587+0000","last_clean":"2023-09-23T19:28:07.887587+0000","last_became_active":"2023-09-23T19:28:07.886494+0000","last_became_peered":"2023-09-23T19:28:07.886494+0000","last_unstale":"2023-09-23T19:28:07.887587+0000","last_undegraded":"2023-09-23T19:28:07.887587+0000","last_fullsized":"2023-09-23T19:28:07.887587+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:24:07.615051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":78,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.890970+0000","last_change":"2023-09-23T19:28:07.890970+0000","last_active":"2023-09-23T19:28:07.890970+0000","last_peered":"2023-09-23T19:28:07.890970+0000","last_clean":"2023-09-23T19:28:07.890970+0000","last_became_active":"2023-09-23T19:28:07.890218+0000","last_became_peered":"2023-09-23T19:28:07.890218+0000","last_unstale":"2023-09-23T19:28:07.890970+0000","last_undegraded":"2023-09-23T19:28:07.890970+0000","last_fullsized":"2023-09-23T19:28:07.890970+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:02:28.214414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":84,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.880173+0000","last_change":"2023-09-23T19:22:42.151617+0000","last_active":"2023-09-23T19:28:06.880173+0000","last_peered":"2023-09-23T19:28:06.880173+0000","last_clean":"2023-09-23T19:28:06.880173+0000","last_became_active":"2023-09-23T19:22:42.151217+0000","last_became_peered":"2023-09-23T19:22:42.151217+0000","last_unstale":"2023-09-23T19:28:06.880173+0000","last_undegraded":"2023-09-23T19:28:06.880173+0000","last_fullsized":"2023-09-23T19:28:06.880173+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":64,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.889639+0000","last_change":"2023-09-23T19:28:07.889639+0000","last_active":"2023-09-23T19:28:07.889639+0000","last_peered":"2023-09-23T19:28:07.889639+0000","last_clean":"2023-09-23T19:28:07.889639+0000","last_became_active":"2023-09-23T19:28:07.889313+0000","last_became_peered":"2023-09-23T19:28:07.889313+0000","last_unstale":"2023-09-23T19:28:07.889639+0000","last_undegraded":"2023-09-23T19:28:07.889639+0000","last_fullsized":"2023-09-23T19:28:07.889639+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:54:03.826337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":87,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.892835+0000","last_change":"2023-09-23T19:28:07.892835+0000","last_active":"2023-09-23T19:28:07.892835+0000","last_peered":"2023-09-23T19:28:07.892835+0000","last_clean":"2023-09-23T19:28:07.892835+0000","last_became_active":"2023-09-23T19:28:07.892338+0000","last_became_peered":"2023-09-23T19:28:07.892338+0000","last_unstale":"2023-09-23T19:28:07.892835+0000","last_undegraded":"2023-09-23T19:28:07.892835+0000","last_fullsized":"2023-09-23T19:28:07.892835+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":58,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.869474+0000","last_change":"2023-09-23T19:27:24.268005+0000","last_active":"2023-09-23T19:28:07.869474+0000","last_peered":"2023-09-23T19:28:07.869474+0000","last_clean":"2023-09-23T19:28:07.869474+0000","last_became_active":"2023-09-23T19:27:24.267513+0000","last_became_peered":"2023-09-23T19:27:24.267513+0000","last_unstale":"2023-09-23T19:28:07.869474+0000","last_undegraded":"2023-09-23T19:28:07.869474+0000","last_fullsized":"2023-09-23T19:28:07.869474+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.961399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'23","reported_seq":92,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.878175+0000","last_change":"2023-09-23T19:05:52.107727+0000","last_active":"2023-09-23T19:28:06.878175+0000","last_peered":"2023-09-23T19:28:06.878175+0000","last_clean":"2023-09-23T19:28:06.878175+0000","last_became_active":"2023-09-23T19:05:52.107378+0000","last_became_peered":"2023-09-23T19:05:52.107378+0000","last_unstale":"2023-09-23T19:28:06.878175+0000","last_undegraded":"2023-09-23T19:28:06.878175+0000","last_fullsized":"2023-09-23T19:28:06.878175+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":83,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.882441+0000","last_change":"2023-09-23T19:22:42.148497+0000","last_active":"2023-09-23T19:28:06.882441+0000","last_peered":"2023-09-23T19:28:06.882441+0000","last_clean":"2023-09-23T19:28:06.882441+0000","last_became_active":"2023-09-23T19:22:42.148147+0000","last_became_peered":"2023-09-23T19:22:42.148147+0000","last_unstale":"2023-09-23T19:28:06.882441+0000","last_undegraded":"2023-09-23T19:28:06.882441+0000","last_fullsized":"2023-09-23T19:28:06.882441+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":86,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.885743+0000","last_change":"2023-09-23T19:27:24.273667+0000","last_active":"2023-09-23T19:28:06.885743+0000","last_peered":"2023-09-23T19:28:06.885743+0000","last_clean":"2023-09-23T19:28:06.885743+0000","last_became_active":"2023-09-23T19:27:24.273234+0000","last_became_peered":"2023-09-23T19:27:24.273234+0000","last_unstale":"2023-09-23T19:28:06.885743+0000","last_undegraded":"2023-09-23T19:28:06.885743+0000","last_fullsized":"2023-09-23T19:28:06.885743+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":78,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.886657+0000","last_change":"2023-09-23T19:28:07.886657+0000","last_active":"2023-09-23T19:28:07.886657+0000","last_peered":"2023-09-23T19:28:07.886657+0000","last_clean":"2023-09-23T19:28:07.886657+0000","last_became_active":"2023-09-23T19:28:07.886210+0000","last_became_peered":"2023-09-23T19:28:07.886210+0000","last_unstale":"2023-09-23T19:28:07.886657+0000","last_undegraded":"2023-09-23T19:28:07.886657+0000","last_fullsized":"2023-09-23T19:28:07.886657+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:21.131194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":73,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.884766+0000","last_change":"2023-09-23T19:28:07.884394+0000","last_active":"2023-09-23T19:28:07.884766+0000","last_peered":"2023-09-23T19:28:07.884766+0000","last_clean":"2023-09-23T19:28:07.884766+0000","last_became_active":"2023-09-23T19:28:07.883944+0000","last_became_peered":"2023-09-23T19:28:07.883944+0000","last_unstale":"2023-09-23T19:28:07.884766+0000","last_undegraded":"2023-09-23T19:28:07.884766+0000","last_fullsized":"2023-09-23T19:28:07.884766+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:08:17.608926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":83,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.891056+0000","last_change":"2023-09-23T19:28:07.891056+0000","last_active":"2023-09-23T19:28:07.891056+0000","last_peered":"2023-09-23T19:28:07.891056+0000","last_clean":"2023-09-23T19:28:07.891056+0000","last_became_active":"2023-09-23T19:28:07.890625+0000","last_became_peered":"2023-09-23T19:28:07.890625+0000","last_unstale":"2023-09-23T19:28:07.891056+0000","last_undegraded":"2023-09-23T19:28:07.891056+0000","last_fullsized":"2023-09-23T19:28:07.891056+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":69,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.885427+0000","last_change":"2023-09-23T19:05:52.114186+0000","last_active":"2023-09-23T19:28:06.885427+0000","last_peered":"2023-09-23T19:28:06.885427+0000","last_clean":"2023-09-23T19:28:06.885427+0000","last_became_active":"2023-09-23T19:05:52.113635+0000","last_became_peered":"2023-09-23T19:05:52.113635+0000","last_unstale":"2023-09-23T19:28:06.885427+0000","last_undegraded":"2023-09-23T19:28:06.885427+0000","last_fullsized":"2023-09-23T19:28:06.885427+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":73,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.885148+0000","last_change":"2023-09-23T19:28:07.884512+0000","last_active":"2023-09-23T19:28:07.885148+0000","last_peered":"2023-09-23T19:28:07.885148+0000","last_clean":"2023-09-23T19:28:07.885148+0000","last_became_active":"2023-09-23T19:28:07.884155+0000","last_became_peered":"2023-09-23T19:28:07.884155+0000","last_unstale":"2023-09-23T19:28:07.885148+0000","last_undegraded":"2023-09-23T19:28:07.885148+0000","last_fullsized":"2023-09-23T19:28:07.885148+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:10.654958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":92,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.876318+0000","last_change":"2023-09-23T19:18:09.010195+0000","last_active":"2023-09-23T19:28:06.876318+0000","last_peered":"2023-09-23T19:28:06.876318+0000","last_clean":"2023-09-23T19:28:06.876318+0000","last_became_active":"2023-09-23T19:18:09.009687+0000","last_became_peered":"2023-09-23T19:18:09.009687+0000","last_unstale":"2023-09-23T19:28:06.876318+0000","last_undegraded":"2023-09-23T19:28:06.876318+0000","last_fullsized":"2023-09-23T19:28:06.876318+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":78,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.885265+0000","last_change":"2023-09-23T19:28:07.885265+0000","last_active":"2023-09-23T19:28:07.885265+0000","last_peered":"2023-09-23T19:28:07.885265+0000","last_clean":"2023-09-23T19:28:07.885265+0000","last_became_active":"2023-09-23T19:28:07.884353+0000","last_became_peered":"2023-09-23T19:28:07.884353+0000","last_unstale":"2023-09-23T19:28:07.885265+0000","last_undegraded":"2023-09-23T19:28:07.885265+0000","last_fullsized":"2023-09-23T19:28:07.885265+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:56.325352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":85,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.886566+0000","last_change":"2023-09-23T19:18:09.035116+0000","last_active":"2023-09-23T19:28:06.886566+0000","last_peered":"2023-09-23T19:28:06.886566+0000","last_clean":"2023-09-23T19:28:06.886566+0000","last_became_active":"2023-09-23T19:18:09.034416+0000","last_became_peered":"2023-09-23T19:18:09.034416+0000","last_unstale":"2023-09-23T19:28:06.886566+0000","last_undegraded":"2023-09-23T19:28:06.886566+0000","last_fullsized":"2023-09-23T19:28:06.886566+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":68,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.879702+0000","last_change":"2023-09-23T19:05:52.789323+0000","last_active":"2023-09-23T19:28:06.879702+0000","last_peered":"2023-09-23T19:28:06.879702+0000","last_clean":"2023-09-23T19:28:06.879702+0000","last_became_active":"2023-09-23T19:05:52.788820+0000","last_became_peered":"2023-09-23T19:05:52.788820+0000","last_unstale":"2023-09-23T19:28:06.879702+0000","last_undegraded":"2023-09-23T19:28:06.879702+0000","last_fullsized":"2023-09-23T19:28:06.879702+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":86,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.885780+0000","last_change":"2023-09-23T19:22:42.151134+0000","last_active":"2023-09-23T19:28:06.885780+0000","last_peered":"2023-09-23T19:28:06.885780+0000","last_clean":"2023-09-23T19:28:06.885780+0000","last_became_active":"2023-09-23T19:22:42.150553+0000","last_became_peered":"2023-09-23T19:22:42.150553+0000","last_unstale":"2023-09-23T19:28:06.885780+0000","last_undegraded":"2023-09-23T19:28:06.885780+0000","last_fullsized":"2023-09-23T19:28:06.885780+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":98,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.899262+0000","last_change":"2023-09-23T19:28:07.899262+0000","last_active":"2023-09-23T19:28:07.899262+0000","last_peered":"2023-09-23T19:28:07.899262+0000","last_clean":"2023-09-23T19:28:07.899262+0000","last_became_active":"2023-09-23T19:28:07.898848+0000","last_became_peered":"2023-09-23T19:28:07.898848+0000","last_unstale":"2023-09-23T19:28:07.899262+0000","last_undegraded":"2023-09-23T19:28:07.899262+0000","last_fullsized":"2023-09-23T19:28:07.899262+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":70,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.885984+0000","last_change":"2023-09-23T19:05:52.114108+0000","last_active":"2023-09-23T19:28:06.885984+0000","last_peered":"2023-09-23T19:28:06.885984+0000","last_clean":"2023-09-23T19:28:06.885984+0000","last_became_active":"2023-09-23T19:05:52.113609+0000","last_became_peered":"2023-09-23T19:05:52.113609+0000","last_unstale":"2023-09-23T19:28:06.885984+0000","last_undegraded":"2023-09-23T19:28:06.885984+0000","last_fullsized":"2023-09-23T19:28:06.885984+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":83,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.875937+0000","last_change":"2023-09-23T19:22:42.156825+0000","last_active":"2023-09-23T19:28:06.875937+0000","last_peered":"2023-09-23T19:28:06.875937+0000","last_clean":"2023-09-23T19:28:06.875937+0000","last_became_active":"2023-09-23T19:22:42.156474+0000","last_became_peered":"2023-09-23T19:22:42.156474+0000","last_unstale":"2023-09-23T19:28:06.875937+0000","last_undegraded":"2023-09-23T19:28:06.875937+0000","last_fullsized":"2023-09-23T19:28:06.875937+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":84,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.891536+0000","last_change":"2023-09-23T19:28:07.891536+0000","last_active":"2023-09-23T19:28:07.891536+0000","last_peered":"2023-09-23T19:28:07.891536+0000","last_clean":"2023-09-23T19:28:07.891536+0000","last_became_active":"2023-09-23T19:28:07.891153+0000","last_became_peered":"2023-09-23T19:28:07.891153+0000","last_unstale":"2023-09-23T19:28:07.891536+0000","last_undegraded":"2023-09-23T19:28:07.891536+0000","last_fullsized":"2023-09-23T19:28:07.891536+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":73,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.874482+0000","last_change":"2023-09-23T19:22:42.135572+0000","last_active":"2023-09-23T19:28:07.874482+0000","last_peered":"2023-09-23T19:28:07.874482+0000","last_clean":"2023-09-23T19:28:07.874482+0000","last_became_active":"2023-09-23T19:22:42.135237+0000","last_became_peered":"2023-09-23T19:22:42.135237+0000","last_unstale":"2023-09-23T19:28:07.874482+0000","last_undegraded":"2023-09-23T19:28:07.874482+0000","last_fullsized":"2023-09-23T19:28:07.874482+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:35.838718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":68,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.893901+0000","last_change":"2023-09-23T19:28:07.893901+0000","last_active":"2023-09-23T19:28:07.893901+0000","last_peered":"2023-09-23T19:28:07.893901+0000","last_clean":"2023-09-23T19:28:07.893901+0000","last_became_active":"2023-09-23T19:28:07.892973+0000","last_became_peered":"2023-09-23T19:28:07.892973+0000","last_unstale":"2023-09-23T19:28:07.893901+0000","last_undegraded":"2023-09-23T19:28:07.893901+0000","last_fullsized":"2023-09-23T19:28:07.893901+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:27.445855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":83,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.896803+0000","last_change":"2023-09-23T19:28:07.896803+0000","last_active":"2023-09-23T19:28:07.896803+0000","last_peered":"2023-09-23T19:28:07.896803+0000","last_clean":"2023-09-23T19:28:07.896803+0000","last_became_active":"2023-09-23T19:28:07.896480+0000","last_became_peered":"2023-09-23T19:28:07.896480+0000","last_unstale":"2023-09-23T19:28:07.896803+0000","last_undegraded":"2023-09-23T19:28:07.896803+0000","last_fullsized":"2023-09-23T19:28:07.896803+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":84,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.876231+0000","last_change":"2023-09-23T19:22:42.156582+0000","last_active":"2023-09-23T19:28:06.876231+0000","last_peered":"2023-09-23T19:28:06.876231+0000","last_clean":"2023-09-23T19:28:06.876231+0000","last_became_active":"2023-09-23T19:22:42.156251+0000","last_became_peered":"2023-09-23T19:22:42.156251+0000","last_unstale":"2023-09-23T19:28:06.876231+0000","last_undegraded":"2023-09-23T19:28:06.876231+0000","last_fullsized":"2023-09-23T19:28:06.876231+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":98,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.879979+0000","last_change":"2023-09-23T19:27:24.273971+0000","last_active":"2023-09-23T19:28:06.879979+0000","last_peered":"2023-09-23T19:28:06.879979+0000","last_clean":"2023-09-23T19:28:06.879979+0000","last_became_active":"2023-09-23T19:27:24.273491+0000","last_became_peered":"2023-09-23T19:27:24.273491+0000","last_unstale":"2023-09-23T19:28:06.879979+0000","last_undegraded":"2023-09-23T19:28:06.879979+0000","last_fullsized":"2023-09-23T19:28:06.879979+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":84,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.878781+0000","last_change":"2023-09-23T19:27:24.274698+0000","last_active":"2023-09-23T19:28:06.878781+0000","last_peered":"2023-09-23T19:28:06.878781+0000","last_clean":"2023-09-23T19:28:06.878781+0000","last_became_active":"2023-09-23T19:27:24.273264+0000","last_became_peered":"2023-09-23T19:27:24.273264+0000","last_unstale":"2023-09-23T19:28:06.878781+0000","last_undegraded":"2023-09-23T19:28:06.878781+0000","last_fullsized":"2023-09-23T19:28:06.878781+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":83,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.878941+0000","last_change":"2023-09-23T19:22:42.149670+0000","last_active":"2023-09-23T19:28:06.878941+0000","last_peered":"2023-09-23T19:28:06.878941+0000","last_clean":"2023-09-23T19:28:06.878941+0000","last_became_active":"2023-09-23T19:22:42.149022+0000","last_became_peered":"2023-09-23T19:22:42.149022+0000","last_unstale":"2023-09-23T19:28:06.878941+0000","last_undegraded":"2023-09-23T19:28:06.878941+0000","last_fullsized":"2023-09-23T19:28:06.878941+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":80,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.888099+0000","last_change":"2023-09-23T19:28:07.886933+0000","last_active":"2023-09-23T19:28:07.888099+0000","last_peered":"2023-09-23T19:28:07.888099+0000","last_clean":"2023-09-23T19:28:07.888099+0000","last_became_active":"2023-09-23T19:28:07.886572+0000","last_became_peered":"2023-09-23T19:28:07.886572+0000","last_unstale":"2023-09-23T19:28:07.888099+0000","last_undegraded":"2023-09-23T19:28:07.888099+0000","last_fullsized":"2023-09-23T19:28:07.888099+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:41.332875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":58,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.889464+0000","last_change":"2023-09-23T19:28:07.889095+0000","last_active":"2023-09-23T19:28:07.889464+0000","last_peered":"2023-09-23T19:28:07.889464+0000","last_clean":"2023-09-23T19:28:07.889464+0000","last_became_active":"2023-09-23T19:28:07.888742+0000","last_became_peered":"2023-09-23T19:28:07.888742+0000","last_unstale":"2023-09-23T19:28:07.889464+0000","last_undegraded":"2023-09-23T19:28:07.889464+0000","last_fullsized":"2023-09-23T19:28:07.889464+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:43.929488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":91,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.870233+0000","last_change":"2023-09-23T19:22:42.143289+0000","last_active":"2023-09-23T19:28:07.870233+0000","last_peered":"2023-09-23T19:28:07.870233+0000","last_clean":"2023-09-23T19:28:07.870233+0000","last_became_active":"2023-09-23T19:22:42.142656+0000","last_became_peered":"2023-09-23T19:22:42.142656+0000","last_unstale":"2023-09-23T19:28:07.870233+0000","last_undegraded":"2023-09-23T19:28:07.870233+0000","last_fullsized":"2023-09-23T19:28:07.870233+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:38.467441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":99,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.890623+0000","last_change":"2023-09-23T19:28:07.890623+0000","last_active":"2023-09-23T19:28:07.890623+0000","last_peered":"2023-09-23T19:28:07.890623+0000","last_clean":"2023-09-23T19:28:07.890623+0000","last_became_active":"2023-09-23T19:28:07.890148+0000","last_became_peered":"2023-09-23T19:28:07.890148+0000","last_unstale":"2023-09-23T19:28:07.890623+0000","last_undegraded":"2023-09-23T19:28:07.890623+0000","last_fullsized":"2023-09-23T19:28:07.890623+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":71,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.865316+0000","last_change":"2023-09-23T19:27:24.269080+0000","last_active":"2023-09-23T19:28:07.865316+0000","last_peered":"2023-09-23T19:28:07.865316+0000","last_clean":"2023-09-23T19:28:07.865316+0000","last_became_active":"2023-09-23T19:27:24.268748+0000","last_became_peered":"2023-09-23T19:27:24.268748+0000","last_unstale":"2023-09-23T19:28:07.865316+0000","last_undegraded":"2023-09-23T19:28:07.865316+0000","last_fullsized":"2023-09-23T19:28:07.865316+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:49.014902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":88,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.880479+0000","last_change":"2023-09-23T19:22:42.153647+0000","last_active":"2023-09-23T19:28:06.880479+0000","last_peered":"2023-09-23T19:28:06.880479+0000","last_clean":"2023-09-23T19:28:06.880479+0000","last_became_active":"2023-09-23T19:22:42.153317+0000","last_became_peered":"2023-09-23T19:22:42.153317+0000","last_unstale":"2023-09-23T19:28:06.880479+0000","last_undegraded":"2023-09-23T19:28:06.880479+0000","last_fullsized":"2023-09-23T19:28:06.880479+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":84,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.882331+0000","last_change":"2023-09-23T19:18:09.017525+0000","last_active":"2023-09-23T19:28:06.882331+0000","last_peered":"2023-09-23T19:28:06.882331+0000","last_clean":"2023-09-23T19:28:06.882331+0000","last_became_active":"2023-09-23T19:18:09.017193+0000","last_became_peered":"2023-09-23T19:18:09.017193+0000","last_unstale":"2023-09-23T19:28:06.882331+0000","last_undegraded":"2023-09-23T19:28:06.882331+0000","last_fullsized":"2023-09-23T19:28:06.882331+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":73,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.879432+0000","last_change":"2023-09-23T19:28:07.875796+0000","last_active":"2023-09-23T19:28:07.879432+0000","last_peered":"2023-09-23T19:28:07.879432+0000","last_clean":"2023-09-23T19:28:07.879432+0000","last_became_active":"2023-09-23T19:28:07.875310+0000","last_became_peered":"2023-09-23T19:28:07.875310+0000","last_unstale":"2023-09-23T19:28:07.879432+0000","last_undegraded":"2023-09-23T19:28:07.879432+0000","last_fullsized":"2023-09-23T19:28:07.879432+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:44:13.379633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":91,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.879332+0000","last_change":"2023-09-23T19:28:07.878688+0000","last_active":"2023-09-23T19:28:07.879332+0000","last_peered":"2023-09-23T19:28:07.879332+0000","last_clean":"2023-09-23T19:28:07.879332+0000","last_became_active":"2023-09-23T19:28:07.878309+0000","last_became_peered":"2023-09-23T19:28:07.878309+0000","last_unstale":"2023-09-23T19:28:07.879332+0000","last_undegraded":"2023-09-23T19:28:07.879332+0000","last_fullsized":"2023-09-23T19:28:07.879332+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:49:06.202657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":58,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.867585+0000","last_change":"2023-09-23T19:27:24.267897+0000","last_active":"2023-09-23T19:28:07.867585+0000","last_peered":"2023-09-23T19:28:07.867585+0000","last_clean":"2023-09-23T19:28:07.867585+0000","last_became_active":"2023-09-23T19:27:24.267535+0000","last_became_peered":"2023-09-23T19:27:24.267535+0000","last_unstale":"2023-09-23T19:28:07.867585+0000","last_undegraded":"2023-09-23T19:28:07.867585+0000","last_fullsized":"2023-09-23T19:28:07.867585+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:13:57.447808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":86,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.874771+0000","last_change":"2023-09-23T19:22:42.143733+0000","last_active":"2023-09-23T19:28:07.874771+0000","last_peered":"2023-09-23T19:28:07.874771+0000","last_clean":"2023-09-23T19:28:07.874771+0000","last_became_active":"2023-09-23T19:22:42.143077+0000","last_became_peered":"2023-09-23T19:22:42.143077+0000","last_unstale":"2023-09-23T19:28:07.874771+0000","last_undegraded":"2023-09-23T19:28:07.874771+0000","last_fullsized":"2023-09-23T19:28:07.874771+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:47:33.086994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":88,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.879925+0000","last_change":"2023-09-23T19:27:24.271520+0000","last_active":"2023-09-23T19:28:06.879925+0000","last_peered":"2023-09-23T19:28:06.879925+0000","last_clean":"2023-09-23T19:28:06.879925+0000","last_became_active":"2023-09-23T19:27:24.271165+0000","last_became_peered":"2023-09-23T19:27:24.271165+0000","last_unstale":"2023-09-23T19:28:06.879925+0000","last_undegraded":"2023-09-23T19:28:06.879925+0000","last_fullsized":"2023-09-23T19:28:06.879925+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":320,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.885321+0000","last_change":"2023-09-23T19:18:09.016297+0000","last_active":"2023-09-23T19:28:06.885321+0000","last_peered":"2023-09-23T19:28:06.885321+0000","last_clean":"2023-09-23T19:28:06.885321+0000","last_became_active":"2023-09-23T19:18:09.015436+0000","last_became_peered":"2023-09-23T19:18:09.015436+0000","last_unstale":"2023-09-23T19:28:06.885321+0000","last_undegraded":"2023-09-23T19:28:06.885321+0000","last_fullsized":"2023-09-23T19:28:06.885321+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:59:33.840751+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:59:33.840751+0000","last_clean_scrub_stamp":"2023-09-23T18:59:33.840751+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":65,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.892254+0000","last_change":"2023-09-23T19:28:07.889756+0000","last_active":"2023-09-23T19:28:07.892254+0000","last_peered":"2023-09-23T19:28:07.892254+0000","last_clean":"2023-09-23T19:28:07.892254+0000","last_became_active":"2023-09-23T19:28:07.889050+0000","last_became_peered":"2023-09-23T19:28:07.889050+0000","last_unstale":"2023-09-23T19:28:07.892254+0000","last_undegraded":"2023-09-23T19:28:07.892254+0000","last_fullsized":"2023-09-23T19:28:07.892254+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:04:55.924553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":68,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.876575+0000","last_change":"2023-09-23T19:05:52.786771+0000","last_active":"2023-09-23T19:28:06.876575+0000","last_peered":"2023-09-23T19:28:06.876575+0000","last_clean":"2023-09-23T19:28:06.876575+0000","last_became_active":"2023-09-23T19:05:52.785797+0000","last_became_peered":"2023-09-23T19:05:52.785797+0000","last_unstale":"2023-09-23T19:28:06.876575+0000","last_undegraded":"2023-09-23T19:28:06.876575+0000","last_fullsized":"2023-09-23T19:28:06.876575+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":103,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.885258+0000","last_change":"2023-09-23T19:22:42.150636+0000","last_active":"2023-09-23T19:28:06.885258+0000","last_peered":"2023-09-23T19:28:06.885258+0000","last_clean":"2023-09-23T19:28:06.885258+0000","last_became_active":"2023-09-23T19:22:42.150045+0000","last_became_peered":"2023-09-23T19:22:42.150045+0000","last_unstale":"2023-09-23T19:28:06.885258+0000","last_undegraded":"2023-09-23T19:28:06.885258+0000","last_fullsized":"2023-09-23T19:28:06.885258+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":73,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.889776+0000","last_change":"2023-09-23T19:28:07.889194+0000","last_active":"2023-09-23T19:28:07.889776+0000","last_peered":"2023-09-23T19:28:07.889776+0000","last_clean":"2023-09-23T19:28:07.889776+0000","last_became_active":"2023-09-23T19:28:07.888823+0000","last_became_peered":"2023-09-23T19:28:07.888823+0000","last_unstale":"2023-09-23T19:28:07.889776+0000","last_undegraded":"2023-09-23T19:28:07.889776+0000","last_fullsized":"2023-09-23T19:28:07.889776+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:23.464688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":88,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.874696+0000","last_change":"2023-09-23T19:27:24.256410+0000","last_active":"2023-09-23T19:28:07.874696+0000","last_peered":"2023-09-23T19:28:07.874696+0000","last_clean":"2023-09-23T19:28:07.874696+0000","last_became_active":"2023-09-23T19:27:24.256052+0000","last_became_peered":"2023-09-23T19:27:24.256052+0000","last_unstale":"2023-09-23T19:28:07.874696+0000","last_undegraded":"2023-09-23T19:28:07.874696+0000","last_fullsized":"2023-09-23T19:28:07.874696+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:11.951400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":88,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.886287+0000","last_change":"2023-09-23T19:18:09.015853+0000","last_active":"2023-09-23T19:28:06.886287+0000","last_peered":"2023-09-23T19:28:06.886287+0000","last_clean":"2023-09-23T19:28:06.886287+0000","last_became_active":"2023-09-23T19:18:09.015338+0000","last_became_peered":"2023-09-23T19:18:09.015338+0000","last_unstale":"2023-09-23T19:28:06.886287+0000","last_undegraded":"2023-09-23T19:28:06.886287+0000","last_fullsized":"2023-09-23T19:28:06.886287+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":58,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.874558+0000","last_change":"2023-09-23T19:27:24.266241+0000","last_active":"2023-09-23T19:28:07.874558+0000","last_peered":"2023-09-23T19:28:07.874558+0000","last_clean":"2023-09-23T19:28:07.874558+0000","last_became_active":"2023-09-23T19:27:24.265764+0000","last_became_peered":"2023-09-23T19:27:24.265764+0000","last_unstale":"2023-09-23T19:28:07.874558+0000","last_undegraded":"2023-09-23T19:28:07.874558+0000","last_fullsized":"2023-09-23T19:28:07.874558+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:07.694732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":58,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.873806+0000","last_change":"2023-09-23T19:18:09.005823+0000","last_active":"2023-09-23T19:28:07.873806+0000","last_peered":"2023-09-23T19:28:07.873806+0000","last_clean":"2023-09-23T19:28:07.873806+0000","last_became_active":"2023-09-23T19:18:09.005207+0000","last_became_peered":"2023-09-23T19:18:09.005207+0000","last_unstale":"2023-09-23T19:28:07.873806+0000","last_undegraded":"2023-09-23T19:28:07.873806+0000","last_fullsized":"2023-09-23T19:28:07.873806+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:05.520188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":69,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.885518+0000","last_change":"2023-09-23T19:28:07.884711+0000","last_active":"2023-09-23T19:28:07.885518+0000","last_peered":"2023-09-23T19:28:07.885518+0000","last_clean":"2023-09-23T19:28:07.885518+0000","last_became_active":"2023-09-23T19:28:07.883878+0000","last_became_peered":"2023-09-23T19:28:07.883878+0000","last_unstale":"2023-09-23T19:28:07.885518+0000","last_undegraded":"2023-09-23T19:28:07.885518+0000","last_fullsized":"2023-09-23T19:28:07.885518+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:55.538826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":43,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.874598+0000","last_change":"2023-09-23T19:22:42.145506+0000","last_active":"2023-09-23T19:28:07.874598+0000","last_peered":"2023-09-23T19:28:07.874598+0000","last_clean":"2023-09-23T19:28:07.874598+0000","last_became_active":"2023-09-23T19:22:42.143991+0000","last_became_peered":"2023-09-23T19:22:42.143991+0000","last_unstale":"2023-09-23T19:28:07.874598+0000","last_undegraded":"2023-09-23T19:28:07.874598+0000","last_fullsized":"2023-09-23T19:28:07.874598+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:56.875175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":66,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.887425+0000","last_change":"2023-09-23T19:28:07.886079+0000","last_active":"2023-09-23T19:28:07.887425+0000","last_peered":"2023-09-23T19:28:07.887425+0000","last_clean":"2023-09-23T19:28:07.887425+0000","last_became_active":"2023-09-23T19:28:07.885209+0000","last_became_peered":"2023-09-23T19:28:07.885209+0000","last_unstale":"2023-09-23T19:28:07.887425+0000","last_undegraded":"2023-09-23T19:28:07.887425+0000","last_fullsized":"2023-09-23T19:28:07.887425+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:19.991162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":88,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.876444+0000","last_change":"2023-09-23T19:27:24.271827+0000","last_active":"2023-09-23T19:28:06.876444+0000","last_peered":"2023-09-23T19:28:06.876444+0000","last_clean":"2023-09-23T19:28:06.876444+0000","last_became_active":"2023-09-23T19:27:24.271328+0000","last_became_peered":"2023-09-23T19:27:24.271328+0000","last_unstale":"2023-09-23T19:28:06.876444+0000","last_undegraded":"2023-09-23T19:28:06.876444+0000","last_fullsized":"2023-09-23T19:28:06.876444+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":76,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.874297+0000","last_change":"2023-09-23T19:22:42.149694+0000","last_active":"2023-09-23T19:28:07.874297+0000","last_peered":"2023-09-23T19:28:07.874297+0000","last_clean":"2023-09-23T19:28:07.874297+0000","last_became_active":"2023-09-23T19:22:42.148194+0000","last_became_peered":"2023-09-23T19:22:42.148194+0000","last_unstale":"2023-09-23T19:28:07.874297+0000","last_undegraded":"2023-09-23T19:28:07.874297+0000","last_fullsized":"2023-09-23T19:28:07.874297+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:47.842383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":68,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.878925+0000","last_change":"2023-09-23T19:05:52.138620+0000","last_active":"2023-09-23T19:28:06.878925+0000","last_peered":"2023-09-23T19:28:06.878925+0000","last_clean":"2023-09-23T19:28:06.878925+0000","last_became_active":"2023-09-23T19:05:52.138243+0000","last_became_peered":"2023-09-23T19:05:52.138243+0000","last_unstale":"2023-09-23T19:28:06.878925+0000","last_undegraded":"2023-09-23T19:28:06.878925+0000","last_fullsized":"2023-09-23T19:28:06.878925+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":60,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.874590+0000","last_change":"2023-09-23T19:18:09.008811+0000","last_active":"2023-09-23T19:28:07.874590+0000","last_peered":"2023-09-23T19:28:07.874590+0000","last_clean":"2023-09-23T19:28:07.874590+0000","last_became_active":"2023-09-23T19:18:09.007255+0000","last_became_peered":"2023-09-23T19:18:09.007255+0000","last_unstale":"2023-09-23T19:28:07.874590+0000","last_undegraded":"2023-09-23T19:28:07.874590+0000","last_fullsized":"2023-09-23T19:28:07.874590+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:25.874038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":99,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.876494+0000","last_change":"2023-09-23T19:28:07.876141+0000","last_active":"2023-09-23T19:28:07.876494+0000","last_peered":"2023-09-23T19:28:07.876494+0000","last_clean":"2023-09-23T19:28:07.876494+0000","last_became_active":"2023-09-23T19:28:07.875802+0000","last_became_peered":"2023-09-23T19:28:07.875802+0000","last_unstale":"2023-09-23T19:28:07.876494+0000","last_undegraded":"2023-09-23T19:28:07.876494+0000","last_fullsized":"2023-09-23T19:28:07.876494+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:38.947881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":86,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.881984+0000","last_change":"2023-09-23T19:22:42.153825+0000","last_active":"2023-09-23T19:28:06.881984+0000","last_peered":"2023-09-23T19:28:06.881984+0000","last_clean":"2023-09-23T19:28:06.881984+0000","last_became_active":"2023-09-23T19:22:42.153416+0000","last_became_peered":"2023-09-23T19:22:42.153416+0000","last_unstale":"2023-09-23T19:28:06.881984+0000","last_undegraded":"2023-09-23T19:28:06.881984+0000","last_fullsized":"2023-09-23T19:28:06.881984+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":107,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.899164+0000","last_change":"2023-09-23T19:28:07.899164+0000","last_active":"2023-09-23T19:28:07.899164+0000","last_peered":"2023-09-23T19:28:07.899164+0000","last_clean":"2023-09-23T19:28:07.899164+0000","last_became_active":"2023-09-23T19:28:07.898812+0000","last_became_peered":"2023-09-23T19:28:07.898812+0000","last_unstale":"2023-09-23T19:28:07.899164+0000","last_undegraded":"2023-09-23T19:28:07.899164+0000","last_fullsized":"2023-09-23T19:28:07.899164+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":71,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.884673+0000","last_change":"2023-09-23T19:05:52.107570+0000","last_active":"2023-09-23T19:28:06.884673+0000","last_peered":"2023-09-23T19:28:06.884673+0000","last_clean":"2023-09-23T19:28:06.884673+0000","last_became_active":"2023-09-23T19:05:52.107215+0000","last_became_peered":"2023-09-23T19:05:52.107215+0000","last_unstale":"2023-09-23T19:28:06.884673+0000","last_undegraded":"2023-09-23T19:28:06.884673+0000","last_fullsized":"2023-09-23T19:28:06.884673+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":98,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.896983+0000","last_change":"2023-09-23T19:28:07.896983+0000","last_active":"2023-09-23T19:28:07.896983+0000","last_peered":"2023-09-23T19:28:07.896983+0000","last_clean":"2023-09-23T19:28:07.896983+0000","last_became_active":"2023-09-23T19:28:07.896577+0000","last_became_peered":"2023-09-23T19:28:07.896577+0000","last_unstale":"2023-09-23T19:28:07.896983+0000","last_undegraded":"2023-09-23T19:28:07.896983+0000","last_fullsized":"2023-09-23T19:28:07.896983+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":93,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.892409+0000","last_change":"2023-09-23T19:28:07.891196+0000","last_active":"2023-09-23T19:28:07.892409+0000","last_peered":"2023-09-23T19:28:07.892409+0000","last_clean":"2023-09-23T19:28:07.892409+0000","last_became_active":"2023-09-23T19:28:07.890484+0000","last_became_peered":"2023-09-23T19:28:07.890484+0000","last_unstale":"2023-09-23T19:28:07.892409+0000","last_undegraded":"2023-09-23T19:28:07.892409+0000","last_fullsized":"2023-09-23T19:28:07.892409+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:24.574716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":105,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.883754+0000","last_change":"2023-09-23T19:28:07.883754+0000","last_active":"2023-09-23T19:28:07.883754+0000","last_peered":"2023-09-23T19:28:07.883754+0000","last_clean":"2023-09-23T19:28:07.883754+0000","last_became_active":"2023-09-23T19:28:07.883363+0000","last_became_peered":"2023-09-23T19:28:07.883363+0000","last_unstale":"2023-09-23T19:28:07.883754+0000","last_undegraded":"2023-09-23T19:28:07.883754+0000","last_fullsized":"2023-09-23T19:28:07.883754+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:56:35.805451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":76,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.874954+0000","last_change":"2023-09-23T19:27:24.267931+0000","last_active":"2023-09-23T19:28:07.874954+0000","last_peered":"2023-09-23T19:28:07.874954+0000","last_clean":"2023-09-23T19:28:07.874954+0000","last_became_active":"2023-09-23T19:27:24.267552+0000","last_became_peered":"2023-09-23T19:27:24.267552+0000","last_unstale":"2023-09-23T19:28:07.874954+0000","last_undegraded":"2023-09-23T19:28:07.874954+0000","last_fullsized":"2023-09-23T19:28:07.874954+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:12.425257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":77,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.878171+0000","last_change":"2023-09-23T19:06:36.673002+0000","last_active":"2023-09-23T19:28:06.878171+0000","last_peered":"2023-09-23T19:28:06.878171+0000","last_clean":"2023-09-23T19:28:06.878171+0000","last_became_active":"2023-09-23T19:06:36.671911+0000","last_became_peered":"2023-09-23T19:06:36.671911+0000","last_unstale":"2023-09-23T19:28:06.878171+0000","last_undegraded":"2023-09-23T19:28:06.878171+0000","last_fullsized":"2023-09-23T19:28:06.878171+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430537,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307128,"kb_used_data":1712,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":93438024,"statfs":{"total":95995035648,"available":95680536576,"internally_reserved":0,"allocated":1753088,"data_stored":1287929,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:29:24 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":1.2130000000000001,"15min":1.2929999999999999},"min":{"1min":0.60199999999999998,"5min":0.27700000000000002,"15min":0.26400000000000001},"max":{"1min":1.3799999999999999,"5min":9.6760000000000002,"15min":9.6760000000000002},"last":0.77200000000000002},{"interface":"front","average":{"1min":0.85099999999999998,"5min":1.109,"15min":1.4219999999999999},"min":{"1min":0.70899999999999996,"5min":0.30299999999999999,"15min":0.28999999999999998},"max":{"1min":1.1040000000000001,"5min":7.8230000000000004,"15min":8.9670000000000005},"last":0.89800000000000002}]},{"osd":1,"last update":"Sat Sep 23 19:29:51 2023","interfaces":[{"interface":"back","average":{"1min":0.87,"5min":1.2050000000000001,"15min":1.413},"min":{"1min":0.56000000000000005,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":1.218,"5min":7.6180000000000003,"15min":7.6180000000000003},"last":1.044},{"interface":"front","average":{"1min":0.93799999999999994,"5min":1.179,"15min":1.413},"min":{"1min":0.61199999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":1.1619999999999999,"5min":7.0010000000000003,"15min":7.3739999999999997},"last":1.0860000000000001}]},{"osd":2,"last update":"Sat Sep 23 19:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.83899999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"min":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"max":{"1min":1.0429999999999999,"5min":1.0449999999999999,"15min":1.0449999999999999},"last":0.74099999999999999},{"interface":"front","average":{"1min":0.998,"5min":0.97699999999999998,"15min":0.97299999999999998},"min":{"1min":0.76300000000000001,"5min":0.622,"15min":0.622},"max":{"1min":1.2170000000000001,"5min":1.294,"15min":1.294},"last":1.1799999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:30:07 2023","interfaces":[{"interface":"back","average":{"1min":1.1619999999999999,"5min":1.0780000000000001,"15min":1.0640000000000001},"min":{"1min":0.755,"5min":0.755,"15min":0.755},"max":{"1min":2.8660000000000001,"5min":2.8660000000000001,"15min":2.8660000000000001},"last":1.1279999999999999},{"interface":"front","average":{"1min":1.23,"5min":1.1479999999999999,"15min":1.1339999999999999},"min":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"max":{"1min":2.7869999999999999,"5min":2.7869999999999999,"15min":2.7869999999999999},"last":1.2689999999999999}]},{"osd":4,"last update":"Sat Sep 23 19:29:51 2023","interfaces":[{"interface":"back","average":{"1min":0.96099999999999997,"5min":1.0840000000000001,"15min":1.1699999999999999},"min":{"1min":0.73399999999999999,"5min":0.35399999999999998,"15min":0.28699999999999998},"max":{"1min":1.1839999999999999,"5min":6.1870000000000003,"15min":9.4190000000000005},"last":1.2909999999999999},{"interface":"front","average":{"1min":1.1259999999999999,"5min":1.099,"15min":1.101},"min":{"1min":0.88600000000000001,"5min":0.36099999999999999,"15min":0.33700000000000002},"max":{"1min":1.3640000000000001,"5min":4.6769999999999996,"15min":7.9790000000000001},"last":0.97899999999999998}]},{"osd":5,"last update":"Sat Sep 23 19:29:51 2023","interfaces":[{"interface":"back","average":{"1min":1.034,"5min":1.258,"15min":1.2769999999999999},"min":{"1min":0.68600000000000005,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.3480000000000001,"5min":6.7130000000000001,"15min":9.8580000000000005},"last":0.998},{"interface":"front","average":{"1min":1.0720000000000001,"5min":1.371,"15min":1.4279999999999999},"min":{"1min":0.77100000000000002,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":1.321,"5min":9.7829999999999995,"15min":10.013999999999999},"last":1.212}]},{"osd":6,"last update":"Sat Sep 23 19:29:51 2023","interfaces":[{"interface":"back","average":{"1min":1.125,"5min":1.298,"15min":1.377},"min":{"1min":0.87,"5min":0.41799999999999998,"15min":0.34599999999999997},"max":{"1min":1.3939999999999999,"5min":5.2320000000000002,"15min":13.462},"last":1.351},{"interface":"front","average":{"1min":1.179,"5min":1.0649999999999999,"15min":1.1739999999999999},"min":{"1min":0.80800000000000005,"5min":0.42299999999999999,"15min":0.36799999999999999},"max":{"1min":1.47,"5min":2.532,"15min":7.9109999999999996},"last":1.3160000000000001}]}]},{"osd":6,"up_from":42,"seq":180388626769,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307112,"kb_used_data":1696,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":93438040,"statfs":{"total":95995035648,"available":95680552960,"internally_reserved":0,"allocated":1736704,"data_stored":1287468,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:29:36 2023","interfaces":[{"interface":"back","average":{"1min":0.82499999999999996,"5min":1.216,"15min":1.405},"min":{"1min":0.45900000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":1.141,"5min":8.7110000000000003,"15min":8.8989999999999991},"last":1.038},{"interface":"front","average":{"1min":0.88700000000000001,"5min":1.256,"15min":1.2310000000000001},"min":{"1min":0.505,"5min":0.36599999999999999,"15min":0.25900000000000001},"max":{"1min":1.2190000000000001,"5min":7.5110000000000001,"15min":7.5110000000000001},"last":0.77000000000000002}]},{"osd":1,"last update":"Sat Sep 23 19:29:51 2023","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":1.349,"15min":1.5589999999999999},"min":{"1min":0.71999999999999997,"5min":0.33100000000000002,"15min":0.29999999999999999},"max":{"1min":1.274,"5min":8.8130000000000006,"15min":11.351000000000001},"last":1.2010000000000001},{"interface":"front","average":{"1min":1.0580000000000001,"5min":1.492,"15min":1.6830000000000001},"min":{"1min":0.754,"5min":0.41399999999999998,"15min":0.34200000000000003},"max":{"1min":1.3620000000000001,"5min":12.728999999999999,"15min":12.728999999999999},"last":1.0760000000000001}]},{"osd":2,"last update":"Sat Sep 23 19:29:26 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":0.93500000000000005,"15min":0.93000000000000005},"min":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"max":{"1min":1.3120000000000001,"5min":1.3120000000000001,"15min":1.3120000000000001},"last":0.86899999999999999},{"interface":"front","average":{"1min":0.97699999999999998,"5min":0.91200000000000003,"15min":0.90100000000000002},"min":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"max":{"1min":1.1919999999999999,"5min":1.1919999999999999,"15min":1.1919999999999999},"last":1.131}]},{"osd":3,"last update":"Sat Sep 23 19:30:12 2023","interfaces":[{"interface":"back","average":{"1min":0.998,"5min":0.99399999999999999,"15min":0.99399999999999999},"min":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"max":{"1min":1.306,"5min":1.306,"15min":1.306},"last":0.93999999999999995},{"interface":"front","average":{"1min":0.97799999999999998,"5min":0.96999999999999997,"15min":0.96799999999999997},"min":{"1min":0.72699999999999998,"5min":0.65000000000000002,"15min":0.65000000000000002},"max":{"1min":1.1519999999999999,"5min":1.177,"15min":1.177},"last":0.91600000000000004}]},{"osd":4,"last update":"Sat Sep 23 19:30:14 2023","interfaces":[{"interface":"back","average":{"1min":0.95799999999999996,"5min":0.98599999999999999,"15min":1.002},"min":{"1min":0.63,"5min":0.33700000000000002,"15min":0.29299999999999998},"max":{"1min":1.282,"5min":4.6799999999999997,"15min":6.5869999999999997},"last":0.83599999999999997},{"interface":"front","average":{"1min":1.0169999999999999,"5min":1.054,"15min":1.167},"min":{"1min":0.60499999999999998,"5min":0.36899999999999999,"15min":0.30499999999999999},"max":{"1min":1.2749999999999999,"5min":4.1040000000000001,"15min":8.5540000000000003},"last":0.72399999999999998}]},{"osd":5,"last update":"Sat Sep 23 19:30:14 2023","interfaces":[{"interface":"back","average":{"1min":1,"5min":1.171,"15min":1.1619999999999999},"min":{"1min":0.65200000000000002,"5min":0.371,"15min":0.316},"max":{"1min":1.484,"5min":4.8869999999999996,"15min":10.398},"last":0.97699999999999998},{"interface":"front","average":{"1min":1.052,"5min":1.28,"15min":1.355},"min":{"1min":0.76200000000000001,"5min":0.39500000000000002,"15min":0.36499999999999999},"max":{"1min":1.542,"5min":5.7140000000000004,"15min":8.6699999999999999},"last":0.78700000000000003}]},{"osd":7,"last update":"Sat Sep 23 19:29:40 2023","interfaces":[{"interface":"back","average":{"1min":1.0669999999999999,"5min":1.1539999999999999,"15min":1.1419999999999999},"min":{"1min":0.55600000000000005,"5min":0.41799999999999998,"15min":0.38300000000000001},"max":{"1min":1.3740000000000001,"5min":8.7949999999999999,"15min":8.7949999999999999},"last":0.90000000000000002},{"interface":"front","average":{"1min":1.1639999999999999,"5min":1.2290000000000001,"15min":1.3109999999999999},"min":{"1min":0.85099999999999998,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.516,"5min":3.9820000000000002,"15min":6.7400000000000002},"last":1.0129999999999999}]}]},{"osd":1,"up_from":98,"seq":420906795101,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":252144,"kb_used_data":1192,"kb_used_omap":4,"kb_used_meta":250939,"kb_avail":93493008,"statfs":{"total":95995035648,"available":95736840192,"internally_reserved":0,"allocated":1220608,"data_stored":707168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":256962446},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:29:55 2023","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":1.359,"15min":2.1509999999999998},"min":{"1min":0.34300000000000003,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":1.036,"5min":14.99,"15min":14.99},"last":0.71299999999999997},{"interface":"front","average":{"1min":0.63100000000000001,"5min":1.2929999999999999,"15min":2.1110000000000002},"min":{"1min":0.255,"5min":0.255,"15min":0.23499999999999999},"max":{"1min":0.92400000000000004,"5min":12.215,"15min":12.215},"last":0.90400000000000003}]},{"osd":2,"last update":"Sat Sep 23 19:29:28 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":0.72799999999999998,"15min":0.73399999999999999},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":1.3200000000000001,"5min":1.3200000000000001,"15min":1.3200000000000001},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.74399999999999999,"5min":0.68799999999999994,"15min":0.67800000000000005},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":1.2350000000000001,"5min":1.2350000000000001,"15min":1.2350000000000001},"last":0.82499999999999996}]},{"osd":3,"last update":"Sat Sep 23 19:30:09 2023","interfaces":[{"interface":"back","average":{"1min":0.66100000000000003,"5min":0.73399999999999999,"15min":0.746},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":1.099,"5min":1.099,"15min":1.099},"last":0.99299999999999999},{"interface":"front","average":{"1min":0.72499999999999998,"5min":0.80300000000000005,"15min":0.81599999999999995},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":1.0580000000000001,"5min":1.131,"15min":1.131},"last":0.96099999999999997}]},{"osd":4,"last update":"Sat Sep 23 19:29:55 2023","interfaces":[{"interface":"back","average":{"1min":0.92900000000000005,"5min":1.3140000000000001,"15min":1.6379999999999999},"min":{"1min":0.52400000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":1.2889999999999999,"5min":6.2830000000000004,"15min":7.6130000000000004},"last":1.1160000000000001},{"interface":"front","average":{"1min":0.95799999999999996,"5min":1.512,"15min":1.383},"min":{"1min":0.58899999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.3600000000000001,"5min":10.69,"15min":10.69},"last":1.0900000000000001}]},{"osd":5,"last update":"Sat Sep 23 19:29:55 2023","interfaces":[{"interface":"back","average":{"1min":0.94699999999999995,"5min":1.5009999999999999,"15min":1.464},"min":{"1min":0.626,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":1.2669999999999999,"5min":10.714,"15min":12.640000000000001},"last":1.0329999999999999},{"interface":"front","average":{"1min":0.999,"5min":1.4830000000000001,"15min":1.5549999999999999},"min":{"1min":0.61099999999999999,"5min":0.38700000000000001,"15min":0.36299999999999999},"max":{"1min":1.4730000000000001,"5min":9.218,"15min":9.218},"last":1.157}]},{"osd":6,"last update":"Sat Sep 23 19:29:55 2023","interfaces":[{"interface":"back","average":{"1min":1.0629999999999999,"5min":1.5800000000000001,"15min":1.595},"min":{"1min":0.63800000000000001,"5min":0.45100000000000001,"15min":0.40100000000000002},"max":{"1min":1.508,"5min":9.3010000000000002,"15min":9.3010000000000002},"last":1.23},{"interface":"front","average":{"1min":1.0720000000000001,"5min":1.5880000000000001,"15min":1.8089999999999999},"min":{"1min":0.86699999999999999,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.528,"5min":9.1699999999999999,"15min":9.1699999999999999},"last":1.2410000000000001}]},{"osd":7,"last update":"Sat Sep 23 19:29:55 2023","interfaces":[{"interface":"back","average":{"1min":1.0589999999999999,"5min":1.4299999999999999,"15min":1.9990000000000001},"min":{"1min":0.72699999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":1.617,"5min":9.2159999999999993,"15min":9.2159999999999993},"last":1.3360000000000001},{"interface":"front","average":{"1min":1.0700000000000001,"5min":1.454,"15min":1.5600000000000001},"min":{"1min":0.65200000000000002,"5min":0.42899999999999999,"15min":0.42699999999999999},"max":{"1min":1.349,"5min":10.742000000000001,"15min":10.742000000000001},"last":1.2190000000000001}]}]},{"osd":0,"up_from":85,"seq":365072220307,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":310232,"kb_used_data":1808,"kb_used_omap":5,"kb_used_meta":308410,"kb_avail":93434920,"statfs":{"total":95995035648,"available":95677358080,"internally_reserved":0,"allocated":1851392,"data_stored":1304001,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":315812474},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 19:29:54 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":1.3640000000000001,"15min":1.534},"min":{"1min":0.23899999999999999,"5min":0.22500000000000001,"15min":0.214},"max":{"1min":0.86499999999999999,"5min":9.2690000000000001,"15min":13.557},"last":0.89000000000000001},{"interface":"front","average":{"1min":0.66800000000000004,"5min":1.167,"15min":1.954},"min":{"1min":0.29099999999999998,"5min":0.253,"15min":0.253},"max":{"1min":0.84599999999999997,"5min":7.6920000000000002,"15min":11.507999999999999},"last":0.69699999999999995}]},{"osd":2,"last update":"Sat Sep 23 19:29:25 2023","interfaces":[{"interface":"back","average":{"1min":0.70299999999999996,"5min":0.75800000000000001,"15min":0.76700000000000002},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":1.2410000000000001,"5min":1.2410000000000001,"15min":1.2410000000000001},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.72699999999999998,"5min":0.77500000000000002,"15min":0.78300000000000003},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":1.2330000000000001,"5min":1.2330000000000001,"15min":1.2330000000000001},"last":0.82699999999999996}]},{"osd":3,"last update":"Sat Sep 23 19:30:09 2023","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":0.77200000000000002,"15min":0.76000000000000001},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":1.254,"5min":1.254,"15min":1.254},"last":1.3440000000000001},{"interface":"front","average":{"1min":0.95399999999999996,"5min":0.97399999999999998,"15min":0.97799999999999998},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":1.3660000000000001,"5min":1.5149999999999999,"15min":1.5149999999999999},"last":1.0389999999999999}]},{"osd":4,"last update":"Sat Sep 23 19:29:33 2023","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":1.2110000000000001,"15min":1.6619999999999999},"min":{"1min":0.54400000000000004,"5min":0.33000000000000002,"15min":0.32100000000000001},"max":{"1min":1.5309999999999999,"5min":6.5659999999999998,"15min":11.156000000000001},"last":1.45},{"interface":"front","average":{"1min":1.0900000000000001,"5min":1.4690000000000001,"15min":1.784},"min":{"1min":0.52600000000000002,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.536,"5min":5.8979999999999997,"15min":13.566000000000001},"last":1.135}]},{"osd":5,"last update":"Sat Sep 23 19:29:33 2023","interfaces":[{"interface":"back","average":{"1min":1.1220000000000001,"5min":1.6639999999999999,"15min":1.819},"min":{"1min":0.505,"5min":0.371,"15min":0.33600000000000002},"max":{"1min":1.51,"5min":11.553000000000001,"15min":13.509},"last":1.2869999999999999},{"interface":"front","average":{"1min":1.083,"5min":1.472,"15min":1.804},"min":{"1min":0.628,"5min":0.38400000000000001,"15min":0.33400000000000002},"max":{"1min":1.6160000000000001,"5min":7.0229999999999997,"15min":11.048999999999999},"last":1.401}]},{"osd":6,"last update":"Sat Sep 23 19:29:33 2023","interfaces":[{"interface":"back","average":{"1min":1.0620000000000001,"5min":1.3109999999999999,"15min":1.8560000000000001},"min":{"1min":0.64100000000000001,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.669,"5min":7.069,"15min":11.467000000000001},"last":1.1020000000000001},{"interface":"front","average":{"1min":1.177,"5min":1.601,"15min":1.9119999999999999},"min":{"1min":0.81100000000000005,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":1.5860000000000001,"5min":9.6679999999999993,"15min":13.603999999999999},"last":1.2090000000000001}]},{"osd":7,"last update":"Sat Sep 23 19:29:33 2023","interfaces":[{"interface":"back","average":{"1min":1.1379999999999999,"5min":1.5329999999999999,"15min":1.9199999999999999},"min":{"1min":0.72199999999999998,"5min":0.44,"15min":0.434},"max":{"1min":1.645,"5min":13.218,"15min":16.271999999999998},"last":1.1839999999999999},{"interface":"front","average":{"1min":1.133,"5min":1.417,"15min":1.861},"min":{"1min":0.67500000000000004,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":1.5620000000000001,"5min":7.101,"15min":11.098000000000001},"last":1.244}]}]},{"osd":2,"up_from":111,"seq":476741369893,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":161480,"kb_used_data":1152,"kb_used_omap":7,"kb_used_meta":160312,"kb_avail":93583672,"statfs":{"total":95995035648,"available":95829680128,"internally_reserved":0,"allocated":1179648,"data_stored":701589,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":164159569},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:29:26 2023","interfaces":[{"interface":"back","average":{"1min":0.71199999999999997,"5min":0.70799999999999996,"15min":0.70699999999999996},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":1.1839999999999999,"5min":1.1839999999999999,"15min":1.1839999999999999},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.73599999999999999,"5min":0.76100000000000001,"15min":0.76500000000000001},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.038,"5min":1.1639999999999999,"15min":1.1639999999999999},"last":0.81499999999999995}]},{"osd":1,"last update":"Sat Sep 23 19:29:26 2023","interfaces":[{"interface":"back","average":{"1min":0.67800000000000005,"5min":0.74199999999999999,"15min":0.753},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":1.0700000000000001,"5min":1.3740000000000001,"15min":1.3740000000000001},"last":0.90200000000000002},{"interface":"front","average":{"1min":0.80100000000000005,"5min":0.84099999999999997,"15min":0.84799999999999998},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":1.097,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":0.97599999999999998}]},{"osd":3,"last update":"Sat Sep 23 19:30:10 2023","interfaces":[{"interface":"back","average":{"1min":0.752,"5min":0.747,"15min":0.746},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":1.4219999999999999,"5min":1.4219999999999999,"15min":1.4219999999999999},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.78300000000000003,"5min":0.84799999999999998,"15min":0.85899999999999999},"min":{"1min":0.442,"5min":0.434,"15min":0.434},"max":{"1min":1.1910000000000001,"5min":1.5389999999999999,"15min":1.5389999999999999},"last":1.1080000000000001}]},{"osd":4,"last update":"Sat Sep 23 19:29:26 2023","interfaces":[{"interface":"back","average":{"1min":0.996,"5min":1.0169999999999999,"15min":1.0209999999999999},"min":{"1min":0.60499999999999998,"5min":0.58699999999999997,"15min":0.58699999999999997},"max":{"1min":1.353,"5min":1.353,"15min":1.353},"last":0.94099999999999995},{"interface":"front","average":{"1min":1.0129999999999999,"5min":1.0089999999999999,"15min":1.0089999999999999},"min":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"max":{"1min":1.5,"5min":1.5,"15min":1.5},"last":1.0329999999999999}]},{"osd":5,"last update":"Sat Sep 23 19:29:26 2023","interfaces":[{"interface":"back","average":{"1min":1.0860000000000001,"5min":1.1299999999999999,"15min":1.137},"min":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"max":{"1min":1.371,"5min":1.554,"15min":1.554},"last":1.204},{"interface":"front","average":{"1min":1.0620000000000001,"5min":1.127,"15min":1.1379999999999999},"min":{"1min":0.68200000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"max":{"1min":1.4690000000000001,"5min":1.8340000000000001,"15min":1.8340000000000001},"last":1.014}]},{"osd":6,"last update":"Sat Sep 23 19:29:26 2023","interfaces":[{"interface":"back","average":{"1min":0.997,"5min":1.089,"15min":1.105},"min":{"1min":0.64000000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"max":{"1min":1.409,"5min":1.738,"15min":1.738},"last":1.149},{"interface":"front","average":{"1min":1.1539999999999999,"5min":1.167,"15min":1.169},"min":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"max":{"1min":1.6140000000000001,"5min":1.7030000000000001,"15min":1.7030000000000001},"last":1.369}]},{"osd":7,"last update":"Sat Sep 23 19:29:26 2023","interfaces":[{"interface":"back","average":{"1min":1.117,"5min":1.1970000000000001,"15min":1.2110000000000001},"min":{"1min":0.79300000000000004,"5min":0.74399999999999999,"15min":0.74399999999999999},"max":{"1min":1.5840000000000001,"5min":1.77,"15min":1.77},"last":1.1699999999999999},{"interface":"front","average":{"1min":1.131,"5min":1.173,"15min":1.1799999999999999},"min":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"max":{"1min":1.5669999999999999,"5min":1.5669999999999999,"15min":1.5669999999999999},"last":1.3109999999999999}]}]},{"osd":3,"up_from":116,"seq":498216206364,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":160112,"kb_used_data":1128,"kb_used_omap":2,"kb_used_meta":158973,"kb_avail":93585040,"statfs":{"total":95995035648,"available":95831080960,"internally_reserved":0,"allocated":1155072,"data_stored":697144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":162788405},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:30:09 2023","interfaces":[{"interface":"back","average":{"1min":0.70899999999999996,"5min":0.66800000000000004,"15min":0.66100000000000003},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":1.1000000000000001,"5min":1.1000000000000001,"15min":1.1000000000000001},"last":1.0489999999999999},{"interface":"front","average":{"1min":0.69499999999999995,"5min":0.78400000000000003,"15min":0.79900000000000004},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.996,"5min":1.3160000000000001,"15min":1.3160000000000001},"last":0.59699999999999998}]},{"osd":1,"last update":"Sat Sep 23 19:30:09 2023","interfaces":[{"interface":"back","average":{"1min":0.76000000000000001,"5min":0.746,"15min":0.74399999999999999},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.0960000000000001,"5min":1.0960000000000001,"15min":1.0960000000000001},"last":0.72199999999999998},{"interface":"front","average":{"1min":0.77500000000000002,"5min":0.79500000000000004,"15min":0.79800000000000004},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":1.2250000000000001,"5min":1.2250000000000001,"15min":1.2250000000000001},"last":1.1499999999999999}]},{"osd":2,"last update":"Sat Sep 23 19:30:09 2023","interfaces":[{"interface":"back","average":{"1min":0.67300000000000004,"5min":0.755,"15min":0.76900000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.86699999999999999,"5min":1.137,"15min":1.137},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.84199999999999997,"5min":0.84599999999999997,"15min":0.84699999999999998},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":1.3320000000000001,"5min":1.3320000000000001,"15min":1.3320000000000001},"last":1.2030000000000001}]},{"osd":4,"last update":"Sat Sep 23 19:30:09 2023","interfaces":[{"interface":"back","average":{"1min":1.073,"5min":1.0760000000000001,"15min":1.0760000000000001},"min":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"max":{"1min":1.448,"5min":1.454,"15min":1.454},"last":1.3009999999999999},{"interface":"front","average":{"1min":1.1040000000000001,"5min":1.1399999999999999,"15min":1.1459999999999999},"min":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"max":{"1min":1.6080000000000001,"5min":1.6080000000000001,"15min":1.6080000000000001},"last":1.1160000000000001}]},{"osd":5,"last update":"Sat Sep 23 19:30:09 2023","interfaces":[{"interface":"back","average":{"1min":1.0760000000000001,"5min":1.1160000000000001,"15min":1.1220000000000001},"min":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"max":{"1min":1.5169999999999999,"5min":1.5169999999999999,"15min":1.5169999999999999},"last":0.83699999999999997},{"interface":"front","average":{"1min":1.0680000000000001,"5min":1.0780000000000001,"15min":1.0800000000000001},"min":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":1.361,"5min":1.361,"15min":1.361},"last":1.181}]},{"osd":6,"last update":"Sat Sep 23 19:30:09 2023","interfaces":[{"interface":"back","average":{"1min":1.1599999999999999,"5min":1.1479999999999999,"15min":1.1459999999999999},"min":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"max":{"1min":1.6830000000000001,"5min":1.6830000000000001,"15min":1.6830000000000001},"last":0.88900000000000001},{"interface":"front","average":{"1min":1.117,"5min":1.141,"15min":1.145},"min":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"max":{"1min":1.4750000000000001,"5min":1.5109999999999999,"15min":1.5109999999999999},"last":0.997}]},{"osd":7,"last update":"Sat Sep 23 19:30:09 2023","interfaces":[{"interface":"back","average":{"1min":1.127,"5min":1.153,"15min":1.157},"min":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"max":{"1min":1.4810000000000001,"5min":1.597,"15min":1.597},"last":1.097},{"interface":"front","average":{"1min":1.1899999999999999,"5min":1.1619999999999999,"15min":1.1579999999999999},"min":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"max":{"1min":1.7210000000000001,"5min":1.7210000000000001,"15min":1.7210000000000001},"last":1.625}]}]},{"osd":4,"up_from":30,"seq":128849019234,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":590832,"kb_used_data":1128,"kb_used_omap":0,"kb_used_meta":589696,"kb_avail":93154320,"statfs":{"total":95995035648,"available":95390023680,"internally_reserved":0,"allocated":1155072,"data_stored":697491,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603848704},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:29:30 2023","interfaces":[{"interface":"back","average":{"1min":0.88100000000000001,"5min":1.3260000000000001,"15min":3.306},"min":{"1min":0.61499999999999999,"5min":0.27400000000000002,"15min":0.26200000000000001},"max":{"1min":1.1719999999999999,"5min":8.7680000000000007,"15min":129.16200000000001},"last":0.84799999999999998},{"interface":"front","average":{"1min":0.90400000000000003,"5min":1.244,"15min":3.5640000000000001},"min":{"1min":0.53700000000000003,"5min":0.32200000000000001,"15min":0.27700000000000002},"max":{"1min":1.23,"5min":6.2450000000000001,"15min":130.40600000000001},"last":1.081}]},{"osd":1,"last update":"Sat Sep 23 19:29:58 2023","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":1.46,"15min":1.619},"min":{"1min":0.69899999999999995,"5min":0.30099999999999999,"15min":0.29499999999999998},"max":{"1min":1.3260000000000001,"5min":12.372,"15min":12.372},"last":1.1919999999999999},{"interface":"front","average":{"1min":0.90800000000000003,"5min":1.4390000000000001,"15min":1.5449999999999999},"min":{"1min":0.67500000000000004,"5min":0.33600000000000002,"15min":0.31},"max":{"1min":1.2250000000000001,"5min":7.3360000000000003,"15min":7.3360000000000003},"last":1.29}]},{"osd":2,"last update":"Sat Sep 23 19:29:26 2023","interfaces":[{"interface":"back","average":{"1min":0.92500000000000004,"5min":0.89300000000000002,"15min":0.88800000000000001},"min":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"max":{"1min":1.2290000000000001,"5min":1.327,"15min":1.327},"last":1.264},{"interface":"front","average":{"1min":0.90400000000000003,"5min":0.89900000000000002,"15min":0.89800000000000002},"min":{"1min":0.69699999999999995,"5min":0.51700000000000002,"15min":0.51700000000000002},"max":{"1min":1.1599999999999999,"5min":1.4379999999999999,"15min":1.4379999999999999},"last":1.1319999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:30:13 2023","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":0.96799999999999997,"15min":0.97199999999999998},"min":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"max":{"1min":1.6899999999999999,"5min":1.6899999999999999,"15min":1.6899999999999999},"last":1.1599999999999999},{"interface":"front","average":{"1min":0.96699999999999997,"5min":0.96299999999999997,"15min":0.96199999999999997},"min":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":1.413,"5min":1.552,"15min":1.552},"last":1.2350000000000001}]},{"osd":5,"last update":"Sat Sep 23 19:29:42 2023","interfaces":[{"interface":"back","average":{"1min":1.018,"5min":1.4299999999999999,"15min":1.1279999999999999},"min":{"1min":0.57699999999999996,"5min":0.32500000000000001,"15min":0.29199999999999998},"max":{"1min":2.1709999999999998,"5min":12.459,"15min":12.459},"last":1.0009999999999999},{"interface":"front","average":{"1min":1.1220000000000001,"5min":1.4730000000000001,"15min":1.234},"min":{"1min":0.877,"5min":0.34699999999999998,"15min":0.26700000000000002},"max":{"1min":1.8680000000000001,"5min":12.478,"15min":12.478},"last":1.3220000000000001}]},{"osd":6,"last update":"Sat Sep 23 19:30:00 2023","interfaces":[{"interface":"back","average":{"1min":1.0940000000000001,"5min":1.367,"15min":1.274},"min":{"1min":0.495,"5min":0.39600000000000002,"15min":0.29999999999999999},"max":{"1min":2.5739999999999998,"5min":7.0810000000000004,"15min":7.0810000000000004},"last":1.3759999999999999},{"interface":"front","average":{"1min":1.1919999999999999,"5min":1.3260000000000001,"15min":1.2609999999999999},"min":{"1min":0.61599999999999999,"5min":0.40600000000000003,"15min":0.34300000000000003},"max":{"1min":2.5419999999999998,"5min":4.25,"15min":6.9050000000000002},"last":1.399}]},{"osd":7,"last update":"Sat Sep 23 19:29:58 2023","interfaces":[{"interface":"back","average":{"1min":1.133,"5min":1.377,"15min":1.2110000000000001},"min":{"1min":0.67000000000000004,"5min":0.41599999999999998,"15min":0.312},"max":{"1min":1.9350000000000001,"5min":7.8479999999999999,"15min":7.8479999999999999},"last":1.3520000000000001},{"interface":"front","average":{"1min":1.181,"5min":1.296,"15min":1.1220000000000001},"min":{"1min":0.76800000000000002,"5min":0.42499999999999999,"15min":0.36699999999999999},"max":{"1min":2.2629999999999999,"5min":5.8570000000000002,"15min":5.8570000000000002},"last":1.4419999999999999}]}]},{"osd":5,"up_from":36,"seq":154618823001,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":594904,"kb_used_data":1104,"kb_used_omap":0,"kb_used_meta":593792,"kb_avail":93150248,"statfs":{"total":95995035648,"available":95385853952,"internally_reserved":0,"allocated":1130496,"data_stored":692605,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":608043008},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.88100000000000001,"5min":1.0649999999999999,"15min":1.3360000000000001},"min":{"1min":0.49299999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.2360000000000002,"5min":7.6669999999999998,"15min":11.058},"last":1.002},{"interface":"front","average":{"1min":0.98899999999999999,"5min":1.478,"15min":1.7050000000000001},"min":{"1min":0.54000000000000004,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":3.7890000000000001,"5min":10.802,"15min":10.802},"last":0.81799999999999995}]},{"osd":1,"last update":"Sat Sep 23 19:29:57 2023","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":1.0660000000000001,"15min":1.2589999999999999},"min":{"1min":0.625,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":1.2310000000000001,"5min":4.9720000000000004,"15min":9.5389999999999997},"last":0.86199999999999999},{"interface":"front","average":{"1min":0.878,"5min":1.3859999999999999,"15min":2.0840000000000001},"min":{"1min":0.65300000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":1.161,"5min":11.228999999999999,"15min":11.228999999999999},"last":0.95299999999999996}]},{"osd":2,"last update":"Sat Sep 23 19:29:29 2023","interfaces":[{"interface":"back","average":{"1min":1.0700000000000001,"5min":0.98099999999999998,"15min":0.96599999999999997},"min":{"1min":0.65100000000000002,"5min":0.56899999999999995,"15min":0.56899999999999995},"max":{"1min":2.3069999999999999,"5min":2.3069999999999999,"15min":2.3069999999999999},"last":0.72999999999999998},{"interface":"front","average":{"1min":1.151,"5min":1.0680000000000001,"15min":1.054},"min":{"1min":0.78800000000000003,"5min":0.753,"15min":0.753},"max":{"1min":2.335,"5min":2.335,"15min":2.335},"last":1.079}]},{"osd":3,"last update":"Sat Sep 23 19:30:10 2023","interfaces":[{"interface":"back","average":{"1min":0.90600000000000003,"5min":1.03,"15min":1.0509999999999999},"min":{"1min":0.70499999999999996,"5min":0.59399999999999997,"15min":0.59399999999999997},"max":{"1min":1.268,"5min":1.702,"15min":1.702},"last":0.622},{"interface":"front","average":{"1min":0.998,"5min":1.1379999999999999,"15min":1.1619999999999999},"min":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"max":{"1min":1.288,"5min":2.5179999999999998,"15min":2.5179999999999998},"last":1.236}]},{"osd":4,"last update":"Sat Sep 23 19:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":1.244,"15min":1.1519999999999999},"min":{"1min":0.63600000000000001,"5min":0.33200000000000002,"15min":0.29199999999999998},"max":{"1min":1.2669999999999999,"5min":8.1790000000000003,"15min":13.753},"last":1.0569999999999999},{"interface":"front","average":{"1min":0.96899999999999997,"5min":1.3029999999999999,"15min":1.137},"min":{"1min":0.72199999999999998,"5min":0.34399999999999997,"15min":0.27000000000000002},"max":{"1min":1.444,"5min":13.714,"15min":13.714},"last":0.99199999999999999}]},{"osd":6,"last update":"Sat Sep 23 19:29:14 2023","interfaces":[{"interface":"back","average":{"1min":1.0860000000000001,"5min":1.2829999999999999,"15min":1.246},"min":{"1min":0.68899999999999995,"5min":0.36899999999999999,"15min":0.28599999999999998},"max":{"1min":1.4730000000000001,"5min":7.3739999999999997,"15min":26.448},"last":1.196},{"interface":"front","average":{"1min":1.149,"5min":1.3959999999999999,"15min":1.298},"min":{"1min":0.76900000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.6990000000000001,"5min":4.8399999999999999,"15min":26.841999999999999},"last":0.91300000000000003}]},{"osd":7,"last update":"Sat Sep 23 19:29:47 2023","interfaces":[{"interface":"back","average":{"1min":1.161,"5min":1.1379999999999999,"15min":1.361},"min":{"1min":0.77000000000000002,"5min":0.41399999999999998,"15min":0.36599999999999999},"max":{"1min":1.794,"5min":4.7990000000000004,"15min":32.585999999999999},"last":1.0329999999999999},{"interface":"front","average":{"1min":1.1850000000000001,"5min":1.145,"15min":1.25},"min":{"1min":0.80400000000000005,"5min":0.39800000000000002,"15min":0.35199999999999998},"max":{"1min":1.542,"5min":7.7050000000000001,"15min":33.014000000000003},"last":1.278}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T19:30:20.579 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T19:30:22.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:30:21 smithi016 bash[75004]: cluster 2023-09-23T19:30:21.910401+0000 mgr.y (mgr.25325) 680 : cluster 0 pgmap v574: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:22.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:30:21 smithi016 bash[77995]: cluster 2023-09-23T19:30:21.910401+0000 mgr.y (mgr.25325) 680 : cluster 0 pgmap v574: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:22.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:30:21 smithi092 bash[58855]: cluster 2023-09-23T19:30:21.910401+0000 mgr.y (mgr.25325) 680 : cluster 0 pgmap v574: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:24.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:30:23 smithi016 bash[75004]: cluster 2023-09-23T19:30:23.911014+0000 mgr.y (mgr.25325) 681 : cluster 0 pgmap v575: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:24.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:30:23 smithi016 bash[77995]: cluster 2023-09-23T19:30:23.911014+0000 mgr.y (mgr.25325) 681 : cluster 0 pgmap v575: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:24.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:30:23 smithi092 bash[58855]: cluster 2023-09-23T19:30:23.911014+0000 mgr.y (mgr.25325) 681 : cluster 0 pgmap v575: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:25.408 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:30:25.785 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:30:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:30:25] "GET /metrics HTTP/1.1" 200 33811 "" "Prometheus/2.33.4" 2023-09-23T19:30:26.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:30:25 smithi016 bash[75004]: cluster 2023-09-23T19:30:25.911887+0000 mgr.y (mgr.25325) 682 : cluster 0 pgmap v576: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:26.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:30:25 smithi016 bash[77995]: cluster 2023-09-23T19:30:25.911887+0000 mgr.y (mgr.25325) 682 : cluster 0 pgmap v576: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:26.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:30:25 smithi092 bash[58855]: cluster 2023-09-23T19:30:25.911887+0000 mgr.y (mgr.25325) 682 : cluster 0 pgmap v576: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:28.698 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-09-23T19:30:28.702 INFO:teuthology.orchestra.run.smithi016.stderr:dumped all 2023-09-23T19:30:29.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:30:28 smithi016 bash[75004]: cluster 2023-09-23T19:30:27.912663+0000 mgr.y (mgr.25325) 683 : cluster 0 pgmap v577: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:29.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:30:28 smithi016 bash[75004]: audit 2023-09-23T19:30:28.635141+0000 mon.a (mon.0) 886 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:30:29.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:30:28 smithi016 bash[75004]: audit 2023-09-23T19:30:28.677491+0000 mgr.y (mgr.25325) 684 : audit 0 from='client.45608 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:30:29.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:30:28 smithi016 bash[77995]: cluster 2023-09-23T19:30:27.912663+0000 mgr.y (mgr.25325) 683 : cluster 0 pgmap v577: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:29.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:30:28 smithi016 bash[77995]: audit 2023-09-23T19:30:28.635141+0000 mon.a (mon.0) 886 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:30:29.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:30:28 smithi016 bash[77995]: audit 2023-09-23T19:30:28.677491+0000 mgr.y (mgr.25325) 684 : audit 0 from='client.45608 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:30:29.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:30:28 smithi092 bash[58855]: cluster 2023-09-23T19:30:27.912663+0000 mgr.y (mgr.25325) 683 : cluster 0 pgmap v577: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:29.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:30:28 smithi092 bash[58855]: audit 2023-09-23T19:30:28.635141+0000 mon.a (mon.0) 886 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:30:29.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:30:28 smithi092 bash[58855]: audit 2023-09-23T19:30:28.677491+0000 mgr.y (mgr.25325) 684 : audit 0 from='client.45608 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:30:29.519 INFO:teuthology.orchestra.run.smithi016.stdout:{"pg_ready":true,"pg_map":{"version":577,"stamp":"2023-09-23T19:30:27.912187+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":261,"num_write_kb":2446,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":195,"ondisk_log_size":195,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2683944,"kb_used_data":10920,"kb_used_omap":20,"kb_used_meta":2672939,"kb_avail":747277272,"statfs":{"total":767960285184,"available":765211926528,"internally_reserved":0,"allocated":11182080,"data_stored":7375395,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":2737090190},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.005105"},"pg_stats":[{"pgid":"3.1a","version":"59'1","reported_seq":100,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.878040+0000","last_change":"2023-09-23T19:27:24.274696+0000","last_active":"2023-09-23T19:28:06.878040+0000","last_peered":"2023-09-23T19:28:06.878040+0000","last_clean":"2023-09-23T19:28:06.878040+0000","last_became_active":"2023-09-23T19:27:24.273561+0000","last_became_peered":"2023-09-23T19:27:24.273561+0000","last_unstale":"2023-09-23T19:28:06.878040+0000","last_undegraded":"2023-09-23T19:28:06.878040+0000","last_fullsized":"2023-09-23T19:28:06.878040+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":78,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.894651+0000","last_change":"2023-09-23T19:28:07.894651+0000","last_active":"2023-09-23T19:28:07.894651+0000","last_peered":"2023-09-23T19:28:07.894651+0000","last_clean":"2023-09-23T19:28:07.894651+0000","last_became_active":"2023-09-23T19:28:07.892919+0000","last_became_peered":"2023-09-23T19:28:07.892919+0000","last_unstale":"2023-09-23T19:28:07.894651+0000","last_undegraded":"2023-09-23T19:28:07.894651+0000","last_fullsized":"2023-09-23T19:28:07.894651+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:56.579997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":58,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.890735+0000","last_change":"2023-09-23T19:28:07.890462+0000","last_active":"2023-09-23T19:28:07.890735+0000","last_peered":"2023-09-23T19:28:07.890735+0000","last_clean":"2023-09-23T19:28:07.890735+0000","last_became_active":"2023-09-23T19:28:07.890084+0000","last_became_peered":"2023-09-23T19:28:07.890084+0000","last_unstale":"2023-09-23T19:28:07.890735+0000","last_undegraded":"2023-09-23T19:28:07.890735+0000","last_fullsized":"2023-09-23T19:28:07.890735+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:50.458218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":74,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:08.830333+0000","last_change":"2023-09-23T19:28:07.884498+0000","last_active":"2023-09-23T19:28:08.830333+0000","last_peered":"2023-09-23T19:28:08.830333+0000","last_clean":"2023-09-23T19:28:08.830333+0000","last_became_active":"2023-09-23T19:28:07.884102+0000","last_became_peered":"2023-09-23T19:28:07.884102+0000","last_unstale":"2023-09-23T19:28:08.830333+0000","last_undegraded":"2023-09-23T19:28:08.830333+0000","last_fullsized":"2023-09-23T19:28:08.830333+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:05:26.348041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":94,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.887122+0000","last_change":"2023-09-23T19:28:07.887122+0000","last_active":"2023-09-23T19:28:07.887122+0000","last_peered":"2023-09-23T19:28:07.887122+0000","last_clean":"2023-09-23T19:28:07.887122+0000","last_became_active":"2023-09-23T19:28:07.886743+0000","last_became_peered":"2023-09-23T19:28:07.886743+0000","last_unstale":"2023-09-23T19:28:07.887122+0000","last_undegraded":"2023-09-23T19:28:07.887122+0000","last_fullsized":"2023-09-23T19:28:07.887122+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:59:45.973619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":83,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.876761+0000","last_change":"2023-09-23T19:22:42.150582+0000","last_active":"2023-09-23T19:28:06.876761+0000","last_peered":"2023-09-23T19:28:06.876761+0000","last_clean":"2023-09-23T19:28:06.876761+0000","last_became_active":"2023-09-23T19:22:42.150229+0000","last_became_peered":"2023-09-23T19:22:42.150229+0000","last_unstale":"2023-09-23T19:28:06.876761+0000","last_undegraded":"2023-09-23T19:28:06.876761+0000","last_fullsized":"2023-09-23T19:28:06.876761+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":73,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.887587+0000","last_change":"2023-09-23T19:28:07.887141+0000","last_active":"2023-09-23T19:28:07.887587+0000","last_peered":"2023-09-23T19:28:07.887587+0000","last_clean":"2023-09-23T19:28:07.887587+0000","last_became_active":"2023-09-23T19:28:07.886494+0000","last_became_peered":"2023-09-23T19:28:07.886494+0000","last_unstale":"2023-09-23T19:28:07.887587+0000","last_undegraded":"2023-09-23T19:28:07.887587+0000","last_fullsized":"2023-09-23T19:28:07.887587+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:24:07.615051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":79,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.892484+0000","last_change":"2023-09-23T19:28:07.890970+0000","last_active":"2023-09-23T19:28:07.892484+0000","last_peered":"2023-09-23T19:28:07.892484+0000","last_clean":"2023-09-23T19:28:07.892484+0000","last_became_active":"2023-09-23T19:28:07.890218+0000","last_became_peered":"2023-09-23T19:28:07.890218+0000","last_unstale":"2023-09-23T19:28:07.892484+0000","last_undegraded":"2023-09-23T19:28:07.892484+0000","last_fullsized":"2023-09-23T19:28:07.892484+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:02:28.214414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":84,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.880173+0000","last_change":"2023-09-23T19:22:42.151617+0000","last_active":"2023-09-23T19:28:06.880173+0000","last_peered":"2023-09-23T19:28:06.880173+0000","last_clean":"2023-09-23T19:28:06.880173+0000","last_became_active":"2023-09-23T19:22:42.151217+0000","last_became_peered":"2023-09-23T19:22:42.151217+0000","last_unstale":"2023-09-23T19:28:06.880173+0000","last_undegraded":"2023-09-23T19:28:06.880173+0000","last_fullsized":"2023-09-23T19:28:06.880173+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":65,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.892444+0000","last_change":"2023-09-23T19:28:07.889639+0000","last_active":"2023-09-23T19:28:07.892444+0000","last_peered":"2023-09-23T19:28:07.892444+0000","last_clean":"2023-09-23T19:28:07.892444+0000","last_became_active":"2023-09-23T19:28:07.889313+0000","last_became_peered":"2023-09-23T19:28:07.889313+0000","last_unstale":"2023-09-23T19:28:07.892444+0000","last_undegraded":"2023-09-23T19:28:07.892444+0000","last_fullsized":"2023-09-23T19:28:07.892444+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:54:03.826337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":87,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.892835+0000","last_change":"2023-09-23T19:28:07.892835+0000","last_active":"2023-09-23T19:28:07.892835+0000","last_peered":"2023-09-23T19:28:07.892835+0000","last_clean":"2023-09-23T19:28:07.892835+0000","last_became_active":"2023-09-23T19:28:07.892338+0000","last_became_peered":"2023-09-23T19:28:07.892338+0000","last_unstale":"2023-09-23T19:28:07.892835+0000","last_undegraded":"2023-09-23T19:28:07.892835+0000","last_fullsized":"2023-09-23T19:28:07.892835+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":58,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.869474+0000","last_change":"2023-09-23T19:27:24.268005+0000","last_active":"2023-09-23T19:28:07.869474+0000","last_peered":"2023-09-23T19:28:07.869474+0000","last_clean":"2023-09-23T19:28:07.869474+0000","last_became_active":"2023-09-23T19:27:24.267513+0000","last_became_peered":"2023-09-23T19:27:24.267513+0000","last_unstale":"2023-09-23T19:28:07.869474+0000","last_undegraded":"2023-09-23T19:28:07.869474+0000","last_fullsized":"2023-09-23T19:28:07.869474+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.961399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'23","reported_seq":92,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.878175+0000","last_change":"2023-09-23T19:05:52.107727+0000","last_active":"2023-09-23T19:28:06.878175+0000","last_peered":"2023-09-23T19:28:06.878175+0000","last_clean":"2023-09-23T19:28:06.878175+0000","last_became_active":"2023-09-23T19:05:52.107378+0000","last_became_peered":"2023-09-23T19:05:52.107378+0000","last_unstale":"2023-09-23T19:28:06.878175+0000","last_undegraded":"2023-09-23T19:28:06.878175+0000","last_fullsized":"2023-09-23T19:28:06.878175+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":83,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.882441+0000","last_change":"2023-09-23T19:22:42.148497+0000","last_active":"2023-09-23T19:28:06.882441+0000","last_peered":"2023-09-23T19:28:06.882441+0000","last_clean":"2023-09-23T19:28:06.882441+0000","last_became_active":"2023-09-23T19:22:42.148147+0000","last_became_peered":"2023-09-23T19:22:42.148147+0000","last_unstale":"2023-09-23T19:28:06.882441+0000","last_undegraded":"2023-09-23T19:28:06.882441+0000","last_fullsized":"2023-09-23T19:28:06.882441+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":86,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.885743+0000","last_change":"2023-09-23T19:27:24.273667+0000","last_active":"2023-09-23T19:28:06.885743+0000","last_peered":"2023-09-23T19:28:06.885743+0000","last_clean":"2023-09-23T19:28:06.885743+0000","last_became_active":"2023-09-23T19:27:24.273234+0000","last_became_peered":"2023-09-23T19:27:24.273234+0000","last_unstale":"2023-09-23T19:28:06.885743+0000","last_undegraded":"2023-09-23T19:28:06.885743+0000","last_fullsized":"2023-09-23T19:28:06.885743+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":79,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.887361+0000","last_change":"2023-09-23T19:28:07.886657+0000","last_active":"2023-09-23T19:28:07.887361+0000","last_peered":"2023-09-23T19:28:07.887361+0000","last_clean":"2023-09-23T19:28:07.887361+0000","last_became_active":"2023-09-23T19:28:07.886210+0000","last_became_peered":"2023-09-23T19:28:07.886210+0000","last_unstale":"2023-09-23T19:28:07.887361+0000","last_undegraded":"2023-09-23T19:28:07.887361+0000","last_fullsized":"2023-09-23T19:28:07.887361+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:21.131194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":73,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.884766+0000","last_change":"2023-09-23T19:28:07.884394+0000","last_active":"2023-09-23T19:28:07.884766+0000","last_peered":"2023-09-23T19:28:07.884766+0000","last_clean":"2023-09-23T19:28:07.884766+0000","last_became_active":"2023-09-23T19:28:07.883944+0000","last_became_peered":"2023-09-23T19:28:07.883944+0000","last_unstale":"2023-09-23T19:28:07.884766+0000","last_undegraded":"2023-09-23T19:28:07.884766+0000","last_fullsized":"2023-09-23T19:28:07.884766+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:08:17.608926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":83,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.891056+0000","last_change":"2023-09-23T19:28:07.891056+0000","last_active":"2023-09-23T19:28:07.891056+0000","last_peered":"2023-09-23T19:28:07.891056+0000","last_clean":"2023-09-23T19:28:07.891056+0000","last_became_active":"2023-09-23T19:28:07.890625+0000","last_became_peered":"2023-09-23T19:28:07.890625+0000","last_unstale":"2023-09-23T19:28:07.891056+0000","last_undegraded":"2023-09-23T19:28:07.891056+0000","last_fullsized":"2023-09-23T19:28:07.891056+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":69,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.885427+0000","last_change":"2023-09-23T19:05:52.114186+0000","last_active":"2023-09-23T19:28:06.885427+0000","last_peered":"2023-09-23T19:28:06.885427+0000","last_clean":"2023-09-23T19:28:06.885427+0000","last_became_active":"2023-09-23T19:05:52.113635+0000","last_became_peered":"2023-09-23T19:05:52.113635+0000","last_unstale":"2023-09-23T19:28:06.885427+0000","last_undegraded":"2023-09-23T19:28:06.885427+0000","last_fullsized":"2023-09-23T19:28:06.885427+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":73,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.885148+0000","last_change":"2023-09-23T19:28:07.884512+0000","last_active":"2023-09-23T19:28:07.885148+0000","last_peered":"2023-09-23T19:28:07.885148+0000","last_clean":"2023-09-23T19:28:07.885148+0000","last_became_active":"2023-09-23T19:28:07.884155+0000","last_became_peered":"2023-09-23T19:28:07.884155+0000","last_unstale":"2023-09-23T19:28:07.885148+0000","last_undegraded":"2023-09-23T19:28:07.885148+0000","last_fullsized":"2023-09-23T19:28:07.885148+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:10.654958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":92,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.876318+0000","last_change":"2023-09-23T19:18:09.010195+0000","last_active":"2023-09-23T19:28:06.876318+0000","last_peered":"2023-09-23T19:28:06.876318+0000","last_clean":"2023-09-23T19:28:06.876318+0000","last_became_active":"2023-09-23T19:18:09.009687+0000","last_became_peered":"2023-09-23T19:18:09.009687+0000","last_unstale":"2023-09-23T19:28:06.876318+0000","last_undegraded":"2023-09-23T19:28:06.876318+0000","last_fullsized":"2023-09-23T19:28:06.876318+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":78,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.885265+0000","last_change":"2023-09-23T19:28:07.885265+0000","last_active":"2023-09-23T19:28:07.885265+0000","last_peered":"2023-09-23T19:28:07.885265+0000","last_clean":"2023-09-23T19:28:07.885265+0000","last_became_active":"2023-09-23T19:28:07.884353+0000","last_became_peered":"2023-09-23T19:28:07.884353+0000","last_unstale":"2023-09-23T19:28:07.885265+0000","last_undegraded":"2023-09-23T19:28:07.885265+0000","last_fullsized":"2023-09-23T19:28:07.885265+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:56.325352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":85,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.886566+0000","last_change":"2023-09-23T19:18:09.035116+0000","last_active":"2023-09-23T19:28:06.886566+0000","last_peered":"2023-09-23T19:28:06.886566+0000","last_clean":"2023-09-23T19:28:06.886566+0000","last_became_active":"2023-09-23T19:18:09.034416+0000","last_became_peered":"2023-09-23T19:18:09.034416+0000","last_unstale":"2023-09-23T19:28:06.886566+0000","last_undegraded":"2023-09-23T19:28:06.886566+0000","last_fullsized":"2023-09-23T19:28:06.886566+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":68,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.879702+0000","last_change":"2023-09-23T19:05:52.789323+0000","last_active":"2023-09-23T19:28:06.879702+0000","last_peered":"2023-09-23T19:28:06.879702+0000","last_clean":"2023-09-23T19:28:06.879702+0000","last_became_active":"2023-09-23T19:05:52.788820+0000","last_became_peered":"2023-09-23T19:05:52.788820+0000","last_unstale":"2023-09-23T19:28:06.879702+0000","last_undegraded":"2023-09-23T19:28:06.879702+0000","last_fullsized":"2023-09-23T19:28:06.879702+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":86,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.885780+0000","last_change":"2023-09-23T19:22:42.151134+0000","last_active":"2023-09-23T19:28:06.885780+0000","last_peered":"2023-09-23T19:28:06.885780+0000","last_clean":"2023-09-23T19:28:06.885780+0000","last_became_active":"2023-09-23T19:22:42.150553+0000","last_became_peered":"2023-09-23T19:22:42.150553+0000","last_unstale":"2023-09-23T19:28:06.885780+0000","last_undegraded":"2023-09-23T19:28:06.885780+0000","last_fullsized":"2023-09-23T19:28:06.885780+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":98,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.899262+0000","last_change":"2023-09-23T19:28:07.899262+0000","last_active":"2023-09-23T19:28:07.899262+0000","last_peered":"2023-09-23T19:28:07.899262+0000","last_clean":"2023-09-23T19:28:07.899262+0000","last_became_active":"2023-09-23T19:28:07.898848+0000","last_became_peered":"2023-09-23T19:28:07.898848+0000","last_unstale":"2023-09-23T19:28:07.899262+0000","last_undegraded":"2023-09-23T19:28:07.899262+0000","last_fullsized":"2023-09-23T19:28:07.899262+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":70,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.885984+0000","last_change":"2023-09-23T19:05:52.114108+0000","last_active":"2023-09-23T19:28:06.885984+0000","last_peered":"2023-09-23T19:28:06.885984+0000","last_clean":"2023-09-23T19:28:06.885984+0000","last_became_active":"2023-09-23T19:05:52.113609+0000","last_became_peered":"2023-09-23T19:05:52.113609+0000","last_unstale":"2023-09-23T19:28:06.885984+0000","last_undegraded":"2023-09-23T19:28:06.885984+0000","last_fullsized":"2023-09-23T19:28:06.885984+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":83,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.875937+0000","last_change":"2023-09-23T19:22:42.156825+0000","last_active":"2023-09-23T19:28:06.875937+0000","last_peered":"2023-09-23T19:28:06.875937+0000","last_clean":"2023-09-23T19:28:06.875937+0000","last_became_active":"2023-09-23T19:22:42.156474+0000","last_became_peered":"2023-09-23T19:22:42.156474+0000","last_unstale":"2023-09-23T19:28:06.875937+0000","last_undegraded":"2023-09-23T19:28:06.875937+0000","last_fullsized":"2023-09-23T19:28:06.875937+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":84,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.891536+0000","last_change":"2023-09-23T19:28:07.891536+0000","last_active":"2023-09-23T19:28:07.891536+0000","last_peered":"2023-09-23T19:28:07.891536+0000","last_clean":"2023-09-23T19:28:07.891536+0000","last_became_active":"2023-09-23T19:28:07.891153+0000","last_became_peered":"2023-09-23T19:28:07.891153+0000","last_unstale":"2023-09-23T19:28:07.891536+0000","last_undegraded":"2023-09-23T19:28:07.891536+0000","last_fullsized":"2023-09-23T19:28:07.891536+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":73,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.874482+0000","last_change":"2023-09-23T19:22:42.135572+0000","last_active":"2023-09-23T19:28:07.874482+0000","last_peered":"2023-09-23T19:28:07.874482+0000","last_clean":"2023-09-23T19:28:07.874482+0000","last_became_active":"2023-09-23T19:22:42.135237+0000","last_became_peered":"2023-09-23T19:22:42.135237+0000","last_unstale":"2023-09-23T19:28:07.874482+0000","last_undegraded":"2023-09-23T19:28:07.874482+0000","last_fullsized":"2023-09-23T19:28:07.874482+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:35.838718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":69,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.894528+0000","last_change":"2023-09-23T19:28:07.893901+0000","last_active":"2023-09-23T19:28:07.894528+0000","last_peered":"2023-09-23T19:28:07.894528+0000","last_clean":"2023-09-23T19:28:07.894528+0000","last_became_active":"2023-09-23T19:28:07.892973+0000","last_became_peered":"2023-09-23T19:28:07.892973+0000","last_unstale":"2023-09-23T19:28:07.894528+0000","last_undegraded":"2023-09-23T19:28:07.894528+0000","last_fullsized":"2023-09-23T19:28:07.894528+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:27.445855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":83,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.896803+0000","last_change":"2023-09-23T19:28:07.896803+0000","last_active":"2023-09-23T19:28:07.896803+0000","last_peered":"2023-09-23T19:28:07.896803+0000","last_clean":"2023-09-23T19:28:07.896803+0000","last_became_active":"2023-09-23T19:28:07.896480+0000","last_became_peered":"2023-09-23T19:28:07.896480+0000","last_unstale":"2023-09-23T19:28:07.896803+0000","last_undegraded":"2023-09-23T19:28:07.896803+0000","last_fullsized":"2023-09-23T19:28:07.896803+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":84,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.876231+0000","last_change":"2023-09-23T19:22:42.156582+0000","last_active":"2023-09-23T19:28:06.876231+0000","last_peered":"2023-09-23T19:28:06.876231+0000","last_clean":"2023-09-23T19:28:06.876231+0000","last_became_active":"2023-09-23T19:22:42.156251+0000","last_became_peered":"2023-09-23T19:22:42.156251+0000","last_unstale":"2023-09-23T19:28:06.876231+0000","last_undegraded":"2023-09-23T19:28:06.876231+0000","last_fullsized":"2023-09-23T19:28:06.876231+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":98,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.879979+0000","last_change":"2023-09-23T19:27:24.273971+0000","last_active":"2023-09-23T19:28:06.879979+0000","last_peered":"2023-09-23T19:28:06.879979+0000","last_clean":"2023-09-23T19:28:06.879979+0000","last_became_active":"2023-09-23T19:27:24.273491+0000","last_became_peered":"2023-09-23T19:27:24.273491+0000","last_unstale":"2023-09-23T19:28:06.879979+0000","last_undegraded":"2023-09-23T19:28:06.879979+0000","last_fullsized":"2023-09-23T19:28:06.879979+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":84,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.878781+0000","last_change":"2023-09-23T19:27:24.274698+0000","last_active":"2023-09-23T19:28:06.878781+0000","last_peered":"2023-09-23T19:28:06.878781+0000","last_clean":"2023-09-23T19:28:06.878781+0000","last_became_active":"2023-09-23T19:27:24.273264+0000","last_became_peered":"2023-09-23T19:27:24.273264+0000","last_unstale":"2023-09-23T19:28:06.878781+0000","last_undegraded":"2023-09-23T19:28:06.878781+0000","last_fullsized":"2023-09-23T19:28:06.878781+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":83,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.878941+0000","last_change":"2023-09-23T19:22:42.149670+0000","last_active":"2023-09-23T19:28:06.878941+0000","last_peered":"2023-09-23T19:28:06.878941+0000","last_clean":"2023-09-23T19:28:06.878941+0000","last_became_active":"2023-09-23T19:22:42.149022+0000","last_became_peered":"2023-09-23T19:22:42.149022+0000","last_unstale":"2023-09-23T19:28:06.878941+0000","last_undegraded":"2023-09-23T19:28:06.878941+0000","last_fullsized":"2023-09-23T19:28:06.878941+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":80,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.888099+0000","last_change":"2023-09-23T19:28:07.886933+0000","last_active":"2023-09-23T19:28:07.888099+0000","last_peered":"2023-09-23T19:28:07.888099+0000","last_clean":"2023-09-23T19:28:07.888099+0000","last_became_active":"2023-09-23T19:28:07.886572+0000","last_became_peered":"2023-09-23T19:28:07.886572+0000","last_unstale":"2023-09-23T19:28:07.888099+0000","last_undegraded":"2023-09-23T19:28:07.888099+0000","last_fullsized":"2023-09-23T19:28:07.888099+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:41.332875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":58,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.889464+0000","last_change":"2023-09-23T19:28:07.889095+0000","last_active":"2023-09-23T19:28:07.889464+0000","last_peered":"2023-09-23T19:28:07.889464+0000","last_clean":"2023-09-23T19:28:07.889464+0000","last_became_active":"2023-09-23T19:28:07.888742+0000","last_became_peered":"2023-09-23T19:28:07.888742+0000","last_unstale":"2023-09-23T19:28:07.889464+0000","last_undegraded":"2023-09-23T19:28:07.889464+0000","last_fullsized":"2023-09-23T19:28:07.889464+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:43.929488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":91,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.870233+0000","last_change":"2023-09-23T19:22:42.143289+0000","last_active":"2023-09-23T19:28:07.870233+0000","last_peered":"2023-09-23T19:28:07.870233+0000","last_clean":"2023-09-23T19:28:07.870233+0000","last_became_active":"2023-09-23T19:22:42.142656+0000","last_became_peered":"2023-09-23T19:22:42.142656+0000","last_unstale":"2023-09-23T19:28:07.870233+0000","last_undegraded":"2023-09-23T19:28:07.870233+0000","last_fullsized":"2023-09-23T19:28:07.870233+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:38.467441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":99,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.890623+0000","last_change":"2023-09-23T19:28:07.890623+0000","last_active":"2023-09-23T19:28:07.890623+0000","last_peered":"2023-09-23T19:28:07.890623+0000","last_clean":"2023-09-23T19:28:07.890623+0000","last_became_active":"2023-09-23T19:28:07.890148+0000","last_became_peered":"2023-09-23T19:28:07.890148+0000","last_unstale":"2023-09-23T19:28:07.890623+0000","last_undegraded":"2023-09-23T19:28:07.890623+0000","last_fullsized":"2023-09-23T19:28:07.890623+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":71,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.865316+0000","last_change":"2023-09-23T19:27:24.269080+0000","last_active":"2023-09-23T19:28:07.865316+0000","last_peered":"2023-09-23T19:28:07.865316+0000","last_clean":"2023-09-23T19:28:07.865316+0000","last_became_active":"2023-09-23T19:27:24.268748+0000","last_became_peered":"2023-09-23T19:27:24.268748+0000","last_unstale":"2023-09-23T19:28:07.865316+0000","last_undegraded":"2023-09-23T19:28:07.865316+0000","last_fullsized":"2023-09-23T19:28:07.865316+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:49.014902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":88,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.880479+0000","last_change":"2023-09-23T19:22:42.153647+0000","last_active":"2023-09-23T19:28:06.880479+0000","last_peered":"2023-09-23T19:28:06.880479+0000","last_clean":"2023-09-23T19:28:06.880479+0000","last_became_active":"2023-09-23T19:22:42.153317+0000","last_became_peered":"2023-09-23T19:22:42.153317+0000","last_unstale":"2023-09-23T19:28:06.880479+0000","last_undegraded":"2023-09-23T19:28:06.880479+0000","last_fullsized":"2023-09-23T19:28:06.880479+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":84,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.882331+0000","last_change":"2023-09-23T19:18:09.017525+0000","last_active":"2023-09-23T19:28:06.882331+0000","last_peered":"2023-09-23T19:28:06.882331+0000","last_clean":"2023-09-23T19:28:06.882331+0000","last_became_active":"2023-09-23T19:18:09.017193+0000","last_became_peered":"2023-09-23T19:18:09.017193+0000","last_unstale":"2023-09-23T19:28:06.882331+0000","last_undegraded":"2023-09-23T19:28:06.882331+0000","last_fullsized":"2023-09-23T19:28:06.882331+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":73,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.879432+0000","last_change":"2023-09-23T19:28:07.875796+0000","last_active":"2023-09-23T19:28:07.879432+0000","last_peered":"2023-09-23T19:28:07.879432+0000","last_clean":"2023-09-23T19:28:07.879432+0000","last_became_active":"2023-09-23T19:28:07.875310+0000","last_became_peered":"2023-09-23T19:28:07.875310+0000","last_unstale":"2023-09-23T19:28:07.879432+0000","last_undegraded":"2023-09-23T19:28:07.879432+0000","last_fullsized":"2023-09-23T19:28:07.879432+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:44:13.379633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":91,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.879332+0000","last_change":"2023-09-23T19:28:07.878688+0000","last_active":"2023-09-23T19:28:07.879332+0000","last_peered":"2023-09-23T19:28:07.879332+0000","last_clean":"2023-09-23T19:28:07.879332+0000","last_became_active":"2023-09-23T19:28:07.878309+0000","last_became_peered":"2023-09-23T19:28:07.878309+0000","last_unstale":"2023-09-23T19:28:07.879332+0000","last_undegraded":"2023-09-23T19:28:07.879332+0000","last_fullsized":"2023-09-23T19:28:07.879332+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:49:06.202657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":58,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.867585+0000","last_change":"2023-09-23T19:27:24.267897+0000","last_active":"2023-09-23T19:28:07.867585+0000","last_peered":"2023-09-23T19:28:07.867585+0000","last_clean":"2023-09-23T19:28:07.867585+0000","last_became_active":"2023-09-23T19:27:24.267535+0000","last_became_peered":"2023-09-23T19:27:24.267535+0000","last_unstale":"2023-09-23T19:28:07.867585+0000","last_undegraded":"2023-09-23T19:28:07.867585+0000","last_fullsized":"2023-09-23T19:28:07.867585+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:13:57.447808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":86,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.874771+0000","last_change":"2023-09-23T19:22:42.143733+0000","last_active":"2023-09-23T19:28:07.874771+0000","last_peered":"2023-09-23T19:28:07.874771+0000","last_clean":"2023-09-23T19:28:07.874771+0000","last_became_active":"2023-09-23T19:22:42.143077+0000","last_became_peered":"2023-09-23T19:22:42.143077+0000","last_unstale":"2023-09-23T19:28:07.874771+0000","last_undegraded":"2023-09-23T19:28:07.874771+0000","last_fullsized":"2023-09-23T19:28:07.874771+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:47:33.086994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":88,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.879925+0000","last_change":"2023-09-23T19:27:24.271520+0000","last_active":"2023-09-23T19:28:06.879925+0000","last_peered":"2023-09-23T19:28:06.879925+0000","last_clean":"2023-09-23T19:28:06.879925+0000","last_became_active":"2023-09-23T19:27:24.271165+0000","last_became_peered":"2023-09-23T19:27:24.271165+0000","last_unstale":"2023-09-23T19:28:06.879925+0000","last_undegraded":"2023-09-23T19:28:06.879925+0000","last_fullsized":"2023-09-23T19:28:06.879925+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":320,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.885321+0000","last_change":"2023-09-23T19:18:09.016297+0000","last_active":"2023-09-23T19:28:06.885321+0000","last_peered":"2023-09-23T19:28:06.885321+0000","last_clean":"2023-09-23T19:28:06.885321+0000","last_became_active":"2023-09-23T19:18:09.015436+0000","last_became_peered":"2023-09-23T19:18:09.015436+0000","last_unstale":"2023-09-23T19:28:06.885321+0000","last_undegraded":"2023-09-23T19:28:06.885321+0000","last_fullsized":"2023-09-23T19:28:06.885321+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:59:33.840751+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:59:33.840751+0000","last_clean_scrub_stamp":"2023-09-23T18:59:33.840751+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":65,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.892254+0000","last_change":"2023-09-23T19:28:07.889756+0000","last_active":"2023-09-23T19:28:07.892254+0000","last_peered":"2023-09-23T19:28:07.892254+0000","last_clean":"2023-09-23T19:28:07.892254+0000","last_became_active":"2023-09-23T19:28:07.889050+0000","last_became_peered":"2023-09-23T19:28:07.889050+0000","last_unstale":"2023-09-23T19:28:07.892254+0000","last_undegraded":"2023-09-23T19:28:07.892254+0000","last_fullsized":"2023-09-23T19:28:07.892254+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:04:55.924553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":68,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.876575+0000","last_change":"2023-09-23T19:05:52.786771+0000","last_active":"2023-09-23T19:28:06.876575+0000","last_peered":"2023-09-23T19:28:06.876575+0000","last_clean":"2023-09-23T19:28:06.876575+0000","last_became_active":"2023-09-23T19:05:52.785797+0000","last_became_peered":"2023-09-23T19:05:52.785797+0000","last_unstale":"2023-09-23T19:28:06.876575+0000","last_undegraded":"2023-09-23T19:28:06.876575+0000","last_fullsized":"2023-09-23T19:28:06.876575+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":103,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.885258+0000","last_change":"2023-09-23T19:22:42.150636+0000","last_active":"2023-09-23T19:28:06.885258+0000","last_peered":"2023-09-23T19:28:06.885258+0000","last_clean":"2023-09-23T19:28:06.885258+0000","last_became_active":"2023-09-23T19:22:42.150045+0000","last_became_peered":"2023-09-23T19:22:42.150045+0000","last_unstale":"2023-09-23T19:28:06.885258+0000","last_undegraded":"2023-09-23T19:28:06.885258+0000","last_fullsized":"2023-09-23T19:28:06.885258+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":73,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.889776+0000","last_change":"2023-09-23T19:28:07.889194+0000","last_active":"2023-09-23T19:28:07.889776+0000","last_peered":"2023-09-23T19:28:07.889776+0000","last_clean":"2023-09-23T19:28:07.889776+0000","last_became_active":"2023-09-23T19:28:07.888823+0000","last_became_peered":"2023-09-23T19:28:07.888823+0000","last_unstale":"2023-09-23T19:28:07.889776+0000","last_undegraded":"2023-09-23T19:28:07.889776+0000","last_fullsized":"2023-09-23T19:28:07.889776+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:23.464688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":88,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.874696+0000","last_change":"2023-09-23T19:27:24.256410+0000","last_active":"2023-09-23T19:28:07.874696+0000","last_peered":"2023-09-23T19:28:07.874696+0000","last_clean":"2023-09-23T19:28:07.874696+0000","last_became_active":"2023-09-23T19:27:24.256052+0000","last_became_peered":"2023-09-23T19:27:24.256052+0000","last_unstale":"2023-09-23T19:28:07.874696+0000","last_undegraded":"2023-09-23T19:28:07.874696+0000","last_fullsized":"2023-09-23T19:28:07.874696+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:11.951400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":88,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.886287+0000","last_change":"2023-09-23T19:18:09.015853+0000","last_active":"2023-09-23T19:28:06.886287+0000","last_peered":"2023-09-23T19:28:06.886287+0000","last_clean":"2023-09-23T19:28:06.886287+0000","last_became_active":"2023-09-23T19:18:09.015338+0000","last_became_peered":"2023-09-23T19:18:09.015338+0000","last_unstale":"2023-09-23T19:28:06.886287+0000","last_undegraded":"2023-09-23T19:28:06.886287+0000","last_fullsized":"2023-09-23T19:28:06.886287+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":58,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.874558+0000","last_change":"2023-09-23T19:27:24.266241+0000","last_active":"2023-09-23T19:28:07.874558+0000","last_peered":"2023-09-23T19:28:07.874558+0000","last_clean":"2023-09-23T19:28:07.874558+0000","last_became_active":"2023-09-23T19:27:24.265764+0000","last_became_peered":"2023-09-23T19:27:24.265764+0000","last_unstale":"2023-09-23T19:28:07.874558+0000","last_undegraded":"2023-09-23T19:28:07.874558+0000","last_fullsized":"2023-09-23T19:28:07.874558+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:07.694732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":58,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.873806+0000","last_change":"2023-09-23T19:18:09.005823+0000","last_active":"2023-09-23T19:28:07.873806+0000","last_peered":"2023-09-23T19:28:07.873806+0000","last_clean":"2023-09-23T19:28:07.873806+0000","last_became_active":"2023-09-23T19:18:09.005207+0000","last_became_peered":"2023-09-23T19:18:09.005207+0000","last_unstale":"2023-09-23T19:28:07.873806+0000","last_undegraded":"2023-09-23T19:28:07.873806+0000","last_fullsized":"2023-09-23T19:28:07.873806+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:05.520188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":69,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.885518+0000","last_change":"2023-09-23T19:28:07.884711+0000","last_active":"2023-09-23T19:28:07.885518+0000","last_peered":"2023-09-23T19:28:07.885518+0000","last_clean":"2023-09-23T19:28:07.885518+0000","last_became_active":"2023-09-23T19:28:07.883878+0000","last_became_peered":"2023-09-23T19:28:07.883878+0000","last_unstale":"2023-09-23T19:28:07.885518+0000","last_undegraded":"2023-09-23T19:28:07.885518+0000","last_fullsized":"2023-09-23T19:28:07.885518+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:55.538826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":43,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.874598+0000","last_change":"2023-09-23T19:22:42.145506+0000","last_active":"2023-09-23T19:28:07.874598+0000","last_peered":"2023-09-23T19:28:07.874598+0000","last_clean":"2023-09-23T19:28:07.874598+0000","last_became_active":"2023-09-23T19:22:42.143991+0000","last_became_peered":"2023-09-23T19:22:42.143991+0000","last_unstale":"2023-09-23T19:28:07.874598+0000","last_undegraded":"2023-09-23T19:28:07.874598+0000","last_fullsized":"2023-09-23T19:28:07.874598+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:56.875175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":66,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.887425+0000","last_change":"2023-09-23T19:28:07.886079+0000","last_active":"2023-09-23T19:28:07.887425+0000","last_peered":"2023-09-23T19:28:07.887425+0000","last_clean":"2023-09-23T19:28:07.887425+0000","last_became_active":"2023-09-23T19:28:07.885209+0000","last_became_peered":"2023-09-23T19:28:07.885209+0000","last_unstale":"2023-09-23T19:28:07.887425+0000","last_undegraded":"2023-09-23T19:28:07.887425+0000","last_fullsized":"2023-09-23T19:28:07.887425+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:19.991162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":88,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.876444+0000","last_change":"2023-09-23T19:27:24.271827+0000","last_active":"2023-09-23T19:28:06.876444+0000","last_peered":"2023-09-23T19:28:06.876444+0000","last_clean":"2023-09-23T19:28:06.876444+0000","last_became_active":"2023-09-23T19:27:24.271328+0000","last_became_peered":"2023-09-23T19:27:24.271328+0000","last_unstale":"2023-09-23T19:28:06.876444+0000","last_undegraded":"2023-09-23T19:28:06.876444+0000","last_fullsized":"2023-09-23T19:28:06.876444+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":76,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.874297+0000","last_change":"2023-09-23T19:22:42.149694+0000","last_active":"2023-09-23T19:28:07.874297+0000","last_peered":"2023-09-23T19:28:07.874297+0000","last_clean":"2023-09-23T19:28:07.874297+0000","last_became_active":"2023-09-23T19:22:42.148194+0000","last_became_peered":"2023-09-23T19:22:42.148194+0000","last_unstale":"2023-09-23T19:28:07.874297+0000","last_undegraded":"2023-09-23T19:28:07.874297+0000","last_fullsized":"2023-09-23T19:28:07.874297+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:47.842383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":68,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.878925+0000","last_change":"2023-09-23T19:05:52.138620+0000","last_active":"2023-09-23T19:28:06.878925+0000","last_peered":"2023-09-23T19:28:06.878925+0000","last_clean":"2023-09-23T19:28:06.878925+0000","last_became_active":"2023-09-23T19:05:52.138243+0000","last_became_peered":"2023-09-23T19:05:52.138243+0000","last_unstale":"2023-09-23T19:28:06.878925+0000","last_undegraded":"2023-09-23T19:28:06.878925+0000","last_fullsized":"2023-09-23T19:28:06.878925+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":60,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.874590+0000","last_change":"2023-09-23T19:18:09.008811+0000","last_active":"2023-09-23T19:28:07.874590+0000","last_peered":"2023-09-23T19:28:07.874590+0000","last_clean":"2023-09-23T19:28:07.874590+0000","last_became_active":"2023-09-23T19:18:09.007255+0000","last_became_peered":"2023-09-23T19:18:09.007255+0000","last_unstale":"2023-09-23T19:28:07.874590+0000","last_undegraded":"2023-09-23T19:28:07.874590+0000","last_fullsized":"2023-09-23T19:28:07.874590+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:25.874038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":99,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.876494+0000","last_change":"2023-09-23T19:28:07.876141+0000","last_active":"2023-09-23T19:28:07.876494+0000","last_peered":"2023-09-23T19:28:07.876494+0000","last_clean":"2023-09-23T19:28:07.876494+0000","last_became_active":"2023-09-23T19:28:07.875802+0000","last_became_peered":"2023-09-23T19:28:07.875802+0000","last_unstale":"2023-09-23T19:28:07.876494+0000","last_undegraded":"2023-09-23T19:28:07.876494+0000","last_fullsized":"2023-09-23T19:28:07.876494+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:38.947881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":86,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.881984+0000","last_change":"2023-09-23T19:22:42.153825+0000","last_active":"2023-09-23T19:28:06.881984+0000","last_peered":"2023-09-23T19:28:06.881984+0000","last_clean":"2023-09-23T19:28:06.881984+0000","last_became_active":"2023-09-23T19:22:42.153416+0000","last_became_peered":"2023-09-23T19:22:42.153416+0000","last_unstale":"2023-09-23T19:28:06.881984+0000","last_undegraded":"2023-09-23T19:28:06.881984+0000","last_fullsized":"2023-09-23T19:28:06.881984+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":107,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.899164+0000","last_change":"2023-09-23T19:28:07.899164+0000","last_active":"2023-09-23T19:28:07.899164+0000","last_peered":"2023-09-23T19:28:07.899164+0000","last_clean":"2023-09-23T19:28:07.899164+0000","last_became_active":"2023-09-23T19:28:07.898812+0000","last_became_peered":"2023-09-23T19:28:07.898812+0000","last_unstale":"2023-09-23T19:28:07.899164+0000","last_undegraded":"2023-09-23T19:28:07.899164+0000","last_fullsized":"2023-09-23T19:28:07.899164+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":71,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.884673+0000","last_change":"2023-09-23T19:05:52.107570+0000","last_active":"2023-09-23T19:28:06.884673+0000","last_peered":"2023-09-23T19:28:06.884673+0000","last_clean":"2023-09-23T19:28:06.884673+0000","last_became_active":"2023-09-23T19:05:52.107215+0000","last_became_peered":"2023-09-23T19:05:52.107215+0000","last_unstale":"2023-09-23T19:28:06.884673+0000","last_undegraded":"2023-09-23T19:28:06.884673+0000","last_fullsized":"2023-09-23T19:28:06.884673+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":98,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.896983+0000","last_change":"2023-09-23T19:28:07.896983+0000","last_active":"2023-09-23T19:28:07.896983+0000","last_peered":"2023-09-23T19:28:07.896983+0000","last_clean":"2023-09-23T19:28:07.896983+0000","last_became_active":"2023-09-23T19:28:07.896577+0000","last_became_peered":"2023-09-23T19:28:07.896577+0000","last_unstale":"2023-09-23T19:28:07.896983+0000","last_undegraded":"2023-09-23T19:28:07.896983+0000","last_fullsized":"2023-09-23T19:28:07.896983+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":93,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.892409+0000","last_change":"2023-09-23T19:28:07.891196+0000","last_active":"2023-09-23T19:28:07.892409+0000","last_peered":"2023-09-23T19:28:07.892409+0000","last_clean":"2023-09-23T19:28:07.892409+0000","last_became_active":"2023-09-23T19:28:07.890484+0000","last_became_peered":"2023-09-23T19:28:07.890484+0000","last_unstale":"2023-09-23T19:28:07.892409+0000","last_undegraded":"2023-09-23T19:28:07.892409+0000","last_fullsized":"2023-09-23T19:28:07.892409+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:24.574716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":105,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.883754+0000","last_change":"2023-09-23T19:28:07.883754+0000","last_active":"2023-09-23T19:28:07.883754+0000","last_peered":"2023-09-23T19:28:07.883754+0000","last_clean":"2023-09-23T19:28:07.883754+0000","last_became_active":"2023-09-23T19:28:07.883363+0000","last_became_peered":"2023-09-23T19:28:07.883363+0000","last_unstale":"2023-09-23T19:28:07.883754+0000","last_undegraded":"2023-09-23T19:28:07.883754+0000","last_fullsized":"2023-09-23T19:28:07.883754+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:56:35.805451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":76,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T19:28:07.874954+0000","last_change":"2023-09-23T19:27:24.267931+0000","last_active":"2023-09-23T19:28:07.874954+0000","last_peered":"2023-09-23T19:28:07.874954+0000","last_clean":"2023-09-23T19:28:07.874954+0000","last_became_active":"2023-09-23T19:27:24.267552+0000","last_became_peered":"2023-09-23T19:27:24.267552+0000","last_unstale":"2023-09-23T19:28:07.874954+0000","last_undegraded":"2023-09-23T19:28:07.874954+0000","last_fullsized":"2023-09-23T19:28:07.874954+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:12.425257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":77,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:28:06.878171+0000","last_change":"2023-09-23T19:06:36.673002+0000","last_active":"2023-09-23T19:28:06.878171+0000","last_peered":"2023-09-23T19:28:06.878171+0000","last_clean":"2023-09-23T19:28:06.878171+0000","last_became_active":"2023-09-23T19:06:36.671911+0000","last_became_peered":"2023-09-23T19:06:36.671911+0000","last_unstale":"2023-09-23T19:28:06.878171+0000","last_undegraded":"2023-09-23T19:28:06.878171+0000","last_fullsized":"2023-09-23T19:28:06.878171+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430539,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307128,"kb_used_data":1712,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":93438024,"statfs":{"total":95995035648,"available":95680536576,"internally_reserved":0,"allocated":1753088,"data_stored":1287929,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:29:24 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":1.2130000000000001,"15min":1.2929999999999999},"min":{"1min":0.60199999999999998,"5min":0.27700000000000002,"15min":0.26400000000000001},"max":{"1min":1.3799999999999999,"5min":9.6760000000000002,"15min":9.6760000000000002},"last":0.95799999999999996},{"interface":"front","average":{"1min":0.85099999999999998,"5min":1.109,"15min":1.4219999999999999},"min":{"1min":0.70899999999999996,"5min":0.30299999999999999,"15min":0.28999999999999998},"max":{"1min":1.1040000000000001,"5min":7.8230000000000004,"15min":8.9670000000000005},"last":1.2789999999999999}]},{"osd":1,"last update":"Sat Sep 23 19:29:51 2023","interfaces":[{"interface":"back","average":{"1min":0.87,"5min":1.2050000000000001,"15min":1.413},"min":{"1min":0.56000000000000005,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":1.218,"5min":7.6180000000000003,"15min":7.6180000000000003},"last":1.002},{"interface":"front","average":{"1min":0.93799999999999994,"5min":1.179,"15min":1.413},"min":{"1min":0.61199999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":1.1619999999999999,"5min":7.0010000000000003,"15min":7.3739999999999997},"last":0.78000000000000003}]},{"osd":2,"last update":"Sat Sep 23 19:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.83899999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"min":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"max":{"1min":1.0429999999999999,"5min":1.0449999999999999,"15min":1.0449999999999999},"last":1.0820000000000001},{"interface":"front","average":{"1min":0.998,"5min":0.97699999999999998,"15min":0.97299999999999998},"min":{"1min":0.76300000000000001,"5min":0.622,"15min":0.622},"max":{"1min":1.2170000000000001,"5min":1.294,"15min":1.294},"last":1.0449999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:30:07 2023","interfaces":[{"interface":"back","average":{"1min":1.1619999999999999,"5min":1.0780000000000001,"15min":1.0640000000000001},"min":{"1min":0.755,"5min":0.755,"15min":0.755},"max":{"1min":2.8660000000000001,"5min":2.8660000000000001,"15min":2.8660000000000001},"last":1.22},{"interface":"front","average":{"1min":1.23,"5min":1.1479999999999999,"15min":1.1339999999999999},"min":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"max":{"1min":2.7869999999999999,"5min":2.7869999999999999,"15min":2.7869999999999999},"last":1.248}]},{"osd":4,"last update":"Sat Sep 23 19:29:51 2023","interfaces":[{"interface":"back","average":{"1min":0.96099999999999997,"5min":1.0840000000000001,"15min":1.1699999999999999},"min":{"1min":0.73399999999999999,"5min":0.35399999999999998,"15min":0.28699999999999998},"max":{"1min":1.1839999999999999,"5min":6.1870000000000003,"15min":9.4190000000000005},"last":0.84499999999999997},{"interface":"front","average":{"1min":1.1259999999999999,"5min":1.099,"15min":1.101},"min":{"1min":0.88600000000000001,"5min":0.36099999999999999,"15min":0.33700000000000002},"max":{"1min":1.3640000000000001,"5min":4.6769999999999996,"15min":7.9790000000000001},"last":1.1439999999999999}]},{"osd":5,"last update":"Sat Sep 23 19:29:51 2023","interfaces":[{"interface":"back","average":{"1min":1.034,"5min":1.258,"15min":1.2769999999999999},"min":{"1min":0.68600000000000005,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.3480000000000001,"5min":6.7130000000000001,"15min":9.8580000000000005},"last":1.1830000000000001},{"interface":"front","average":{"1min":1.0720000000000001,"5min":1.371,"15min":1.4279999999999999},"min":{"1min":0.77100000000000002,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":1.321,"5min":9.7829999999999995,"15min":10.013999999999999},"last":1.1160000000000001}]},{"osd":6,"last update":"Sat Sep 23 19:29:51 2023","interfaces":[{"interface":"back","average":{"1min":1.125,"5min":1.298,"15min":1.377},"min":{"1min":0.87,"5min":0.41799999999999998,"15min":0.34599999999999997},"max":{"1min":1.3939999999999999,"5min":5.2320000000000002,"15min":13.462},"last":1.3049999999999999},{"interface":"front","average":{"1min":1.179,"5min":1.0649999999999999,"15min":1.1739999999999999},"min":{"1min":0.80800000000000005,"5min":0.42299999999999999,"15min":0.36799999999999999},"max":{"1min":1.47,"5min":2.532,"15min":7.9109999999999996},"last":1.371}]}]},{"osd":6,"up_from":42,"seq":180388626771,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307112,"kb_used_data":1696,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":93438040,"statfs":{"total":95995035648,"available":95680552960,"internally_reserved":0,"allocated":1736704,"data_stored":1287468,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:29:36 2023","interfaces":[{"interface":"back","average":{"1min":0.82499999999999996,"5min":1.216,"15min":1.405},"min":{"1min":0.45900000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":1.141,"5min":8.7110000000000003,"15min":8.8989999999999991},"last":0.98699999999999999},{"interface":"front","average":{"1min":0.88700000000000001,"5min":1.256,"15min":1.2310000000000001},"min":{"1min":0.505,"5min":0.36599999999999999,"15min":0.25900000000000001},"max":{"1min":1.2190000000000001,"5min":7.5110000000000001,"15min":7.5110000000000001},"last":1.1679999999999999}]},{"osd":1,"last update":"Sat Sep 23 19:29:51 2023","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":1.349,"15min":1.5589999999999999},"min":{"1min":0.71999999999999997,"5min":0.33100000000000002,"15min":0.29999999999999999},"max":{"1min":1.274,"5min":8.8130000000000006,"15min":11.351000000000001},"last":1.1459999999999999},{"interface":"front","average":{"1min":1.0580000000000001,"5min":1.492,"15min":1.6830000000000001},"min":{"1min":0.754,"5min":0.41399999999999998,"15min":0.34200000000000003},"max":{"1min":1.3620000000000001,"5min":12.728999999999999,"15min":12.728999999999999},"last":1.2829999999999999}]},{"osd":2,"last update":"Sat Sep 23 19:29:26 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":0.93500000000000005,"15min":0.93000000000000005},"min":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"max":{"1min":1.3120000000000001,"5min":1.3120000000000001,"15min":1.3120000000000001},"last":1.21},{"interface":"front","average":{"1min":0.97699999999999998,"5min":0.91200000000000003,"15min":0.90100000000000002},"min":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"max":{"1min":1.1919999999999999,"5min":1.1919999999999999,"15min":1.1919999999999999},"last":0.82199999999999995}]},{"osd":3,"last update":"Sat Sep 23 19:30:12 2023","interfaces":[{"interface":"back","average":{"1min":0.998,"5min":0.99399999999999999,"15min":0.99399999999999999},"min":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"max":{"1min":1.306,"5min":1.306,"15min":1.306},"last":1.101},{"interface":"front","average":{"1min":0.97799999999999998,"5min":0.96999999999999997,"15min":0.96799999999999997},"min":{"1min":0.72699999999999998,"5min":0.65000000000000002,"15min":0.65000000000000002},"max":{"1min":1.1519999999999999,"5min":1.177,"15min":1.177},"last":1.0820000000000001}]},{"osd":4,"last update":"Sat Sep 23 19:30:14 2023","interfaces":[{"interface":"back","average":{"1min":0.95799999999999996,"5min":0.98599999999999999,"15min":1.002},"min":{"1min":0.63,"5min":0.33700000000000002,"15min":0.29299999999999998},"max":{"1min":1.282,"5min":4.6799999999999997,"15min":6.5869999999999997},"last":0.74399999999999999},{"interface":"front","average":{"1min":1.0169999999999999,"5min":1.054,"15min":1.167},"min":{"1min":0.60499999999999998,"5min":0.36899999999999999,"15min":0.30499999999999999},"max":{"1min":1.2749999999999999,"5min":4.1040000000000001,"15min":8.5540000000000003},"last":0.91000000000000003}]},{"osd":5,"last update":"Sat Sep 23 19:30:14 2023","interfaces":[{"interface":"back","average":{"1min":1,"5min":1.171,"15min":1.1619999999999999},"min":{"1min":0.65200000000000002,"5min":0.371,"15min":0.316},"max":{"1min":1.484,"5min":4.8869999999999996,"15min":10.398},"last":1.0429999999999999},{"interface":"front","average":{"1min":1.052,"5min":1.28,"15min":1.355},"min":{"1min":0.76200000000000001,"5min":0.39500000000000002,"15min":0.36499999999999999},"max":{"1min":1.542,"5min":5.7140000000000004,"15min":8.6699999999999999},"last":1.01}]},{"osd":7,"last update":"Sat Sep 23 19:29:40 2023","interfaces":[{"interface":"back","average":{"1min":1.0669999999999999,"5min":1.1539999999999999,"15min":1.1419999999999999},"min":{"1min":0.55600000000000005,"5min":0.41799999999999998,"15min":0.38300000000000001},"max":{"1min":1.3740000000000001,"5min":8.7949999999999999,"15min":8.7949999999999999},"last":0.95299999999999996},{"interface":"front","average":{"1min":1.1639999999999999,"5min":1.2290000000000001,"15min":1.3109999999999999},"min":{"1min":0.85099999999999998,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.516,"5min":3.9820000000000002,"15min":6.7400000000000002},"last":1.266}]}]},{"osd":1,"up_from":98,"seq":420906795103,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":252144,"kb_used_data":1192,"kb_used_omap":4,"kb_used_meta":250939,"kb_avail":93493008,"statfs":{"total":95995035648,"available":95736840192,"internally_reserved":0,"allocated":1220608,"data_stored":707168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":256962446},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:29:55 2023","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":1.359,"15min":2.1509999999999998},"min":{"1min":0.34300000000000003,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":1.036,"5min":14.99,"15min":14.99},"last":0.98799999999999999},{"interface":"front","average":{"1min":0.63100000000000001,"5min":1.2929999999999999,"15min":2.1110000000000002},"min":{"1min":0.255,"5min":0.255,"15min":0.23499999999999999},"max":{"1min":0.92400000000000004,"5min":12.215,"15min":12.215},"last":0.86299999999999999}]},{"osd":2,"last update":"Sat Sep 23 19:29:28 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":0.72799999999999998,"15min":0.73399999999999999},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":1.3200000000000001,"5min":1.3200000000000001,"15min":1.3200000000000001},"last":0.94799999999999995},{"interface":"front","average":{"1min":0.74399999999999999,"5min":0.68799999999999994,"15min":0.67800000000000005},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":1.2350000000000001,"5min":1.2350000000000001,"15min":1.2350000000000001},"last":0.60799999999999998}]},{"osd":3,"last update":"Sat Sep 23 19:30:09 2023","interfaces":[{"interface":"back","average":{"1min":0.66100000000000003,"5min":0.73399999999999999,"15min":0.746},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":1.099,"5min":1.099,"15min":1.099},"last":0.79400000000000004},{"interface":"front","average":{"1min":0.72499999999999998,"5min":0.80300000000000005,"15min":0.81599999999999995},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":1.0580000000000001,"5min":1.131,"15min":1.131},"last":0.71599999999999997}]},{"osd":4,"last update":"Sat Sep 23 19:29:55 2023","interfaces":[{"interface":"back","average":{"1min":0.92900000000000005,"5min":1.3140000000000001,"15min":1.6379999999999999},"min":{"1min":0.52400000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":1.2889999999999999,"5min":6.2830000000000004,"15min":7.6130000000000004},"last":1.044},{"interface":"front","average":{"1min":0.95799999999999996,"5min":1.512,"15min":1.383},"min":{"1min":0.58899999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.3600000000000001,"5min":10.69,"15min":10.69},"last":1.1379999999999999}]},{"osd":5,"last update":"Sat Sep 23 19:29:55 2023","interfaces":[{"interface":"back","average":{"1min":0.94699999999999995,"5min":1.5009999999999999,"15min":1.464},"min":{"1min":0.626,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":1.2669999999999999,"5min":10.714,"15min":12.640000000000001},"last":1.0209999999999999},{"interface":"front","average":{"1min":0.999,"5min":1.4830000000000001,"15min":1.5549999999999999},"min":{"1min":0.61099999999999999,"5min":0.38700000000000001,"15min":0.36299999999999999},"max":{"1min":1.4730000000000001,"5min":9.218,"15min":9.218},"last":0.90700000000000003}]},{"osd":6,"last update":"Sat Sep 23 19:29:55 2023","interfaces":[{"interface":"back","average":{"1min":1.0629999999999999,"5min":1.5800000000000001,"15min":1.595},"min":{"1min":0.63800000000000001,"5min":0.45100000000000001,"15min":0.40100000000000002},"max":{"1min":1.508,"5min":9.3010000000000002,"15min":9.3010000000000002},"last":1.167},{"interface":"front","average":{"1min":1.0720000000000001,"5min":1.5880000000000001,"15min":1.8089999999999999},"min":{"1min":0.86699999999999999,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.528,"5min":9.1699999999999999,"15min":9.1699999999999999},"last":1.2050000000000001}]},{"osd":7,"last update":"Sat Sep 23 19:29:55 2023","interfaces":[{"interface":"back","average":{"1min":1.0589999999999999,"5min":1.4299999999999999,"15min":1.9990000000000001},"min":{"1min":0.72699999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":1.617,"5min":9.2159999999999993,"15min":9.2159999999999993},"last":1.1100000000000001},{"interface":"front","average":{"1min":1.0700000000000001,"5min":1.454,"15min":1.5600000000000001},"min":{"1min":0.65200000000000002,"5min":0.42899999999999999,"15min":0.42699999999999999},"max":{"1min":1.349,"5min":10.742000000000001,"15min":10.742000000000001},"last":1.0840000000000001}]}]},{"osd":0,"up_from":85,"seq":365072220309,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":310232,"kb_used_data":1808,"kb_used_omap":5,"kb_used_meta":308410,"kb_avail":93434920,"statfs":{"total":95995035648,"available":95677358080,"internally_reserved":0,"allocated":1851392,"data_stored":1304001,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":315812474},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 19:29:54 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":1.3640000000000001,"15min":1.534},"min":{"1min":0.23899999999999999,"5min":0.22500000000000001,"15min":0.214},"max":{"1min":0.86499999999999999,"5min":9.2690000000000001,"15min":13.557},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.66800000000000004,"5min":1.167,"15min":1.954},"min":{"1min":0.29099999999999998,"5min":0.253,"15min":0.253},"max":{"1min":0.84599999999999997,"5min":7.6920000000000002,"15min":11.507999999999999},"last":0.64900000000000002}]},{"osd":2,"last update":"Sat Sep 23 19:29:25 2023","interfaces":[{"interface":"back","average":{"1min":0.70299999999999996,"5min":0.75800000000000001,"15min":0.76700000000000002},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":1.2410000000000001,"5min":1.2410000000000001,"15min":1.2410000000000001},"last":0.876},{"interface":"front","average":{"1min":0.72699999999999998,"5min":0.77500000000000002,"15min":0.78300000000000003},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":1.2330000000000001,"5min":1.2330000000000001,"15min":1.2330000000000001},"last":0.93799999999999994}]},{"osd":3,"last update":"Sat Sep 23 19:30:09 2023","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":0.77200000000000002,"15min":0.76000000000000001},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":1.254,"5min":1.254,"15min":1.254},"last":1.0509999999999999},{"interface":"front","average":{"1min":0.95399999999999996,"5min":0.97399999999999998,"15min":0.97799999999999998},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":1.3660000000000001,"5min":1.5149999999999999,"15min":1.5149999999999999},"last":0.82799999999999996}]},{"osd":4,"last update":"Sat Sep 23 19:29:33 2023","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":1.2110000000000001,"15min":1.6619999999999999},"min":{"1min":0.54400000000000004,"5min":0.33000000000000002,"15min":0.32100000000000001},"max":{"1min":1.5309999999999999,"5min":6.5659999999999998,"15min":11.156000000000001},"last":1.3580000000000001},{"interface":"front","average":{"1min":1.0900000000000001,"5min":1.4690000000000001,"15min":1.784},"min":{"1min":0.52600000000000002,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.536,"5min":5.8979999999999997,"15min":13.566000000000001},"last":1.0960000000000001}]},{"osd":5,"last update":"Sat Sep 23 19:29:33 2023","interfaces":[{"interface":"back","average":{"1min":1.1220000000000001,"5min":1.6639999999999999,"15min":1.819},"min":{"1min":0.505,"5min":0.371,"15min":0.33600000000000002},"max":{"1min":1.51,"5min":11.553000000000001,"15min":13.509},"last":1.012},{"interface":"front","average":{"1min":1.083,"5min":1.472,"15min":1.804},"min":{"1min":0.628,"5min":0.38400000000000001,"15min":0.33400000000000002},"max":{"1min":1.6160000000000001,"5min":7.0229999999999997,"15min":11.048999999999999},"last":1.1990000000000001}]},{"osd":6,"last update":"Sat Sep 23 19:29:33 2023","interfaces":[{"interface":"back","average":{"1min":1.0620000000000001,"5min":1.3109999999999999,"15min":1.8560000000000001},"min":{"1min":0.64100000000000001,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.669,"5min":7.069,"15min":11.467000000000001},"last":1.3},{"interface":"front","average":{"1min":1.177,"5min":1.601,"15min":1.9119999999999999},"min":{"1min":0.81100000000000005,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":1.5860000000000001,"5min":9.6679999999999993,"15min":13.603999999999999},"last":1.26}]},{"osd":7,"last update":"Sat Sep 23 19:29:33 2023","interfaces":[{"interface":"back","average":{"1min":1.1379999999999999,"5min":1.5329999999999999,"15min":1.9199999999999999},"min":{"1min":0.72199999999999998,"5min":0.44,"15min":0.434},"max":{"1min":1.645,"5min":13.218,"15min":16.271999999999998},"last":1.23},{"interface":"front","average":{"1min":1.133,"5min":1.417,"15min":1.861},"min":{"1min":0.67500000000000004,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":1.5620000000000001,"5min":7.101,"15min":11.098000000000001},"last":1.4119999999999999}]}]},{"osd":2,"up_from":111,"seq":476741369895,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":161480,"kb_used_data":1152,"kb_used_omap":7,"kb_used_meta":160312,"kb_avail":93583672,"statfs":{"total":95995035648,"available":95829680128,"internally_reserved":0,"allocated":1179648,"data_stored":701589,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":164159569},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:30:26 2023","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":0.70499999999999996,"15min":0.70599999999999996},"min":{"1min":0.39500000000000002,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":1.1719999999999999,"5min":1.1839999999999999,"15min":1.1839999999999999},"last":1.069},{"interface":"front","average":{"1min":0.83599999999999997,"5min":0.77500000000000002,"15min":0.77000000000000002},"min":{"1min":0.50700000000000001,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.514,"5min":1.514,"15min":1.514},"last":0.95799999999999996}]},{"osd":1,"last update":"Sat Sep 23 19:30:26 2023","interfaces":[{"interface":"back","average":{"1min":0.754,"5min":0.74099999999999999,"15min":0.753},"min":{"1min":0.47599999999999998,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":1.1910000000000001,"5min":1.3740000000000001,"15min":1.3740000000000001},"last":0.83199999999999996},{"interface":"front","average":{"1min":0.88900000000000001,"5min":0.84899999999999998,"15min":0.85099999999999998},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":1.2989999999999999,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":1.2509999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:30:10 2023","interfaces":[{"interface":"back","average":{"1min":0.752,"5min":0.747,"15min":0.746},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":1.4219999999999999,"5min":1.4219999999999999,"15min":1.4219999999999999},"last":0.875},{"interface":"front","average":{"1min":0.78300000000000003,"5min":0.84799999999999998,"15min":0.85899999999999999},"min":{"1min":0.442,"5min":0.434,"15min":0.434},"max":{"1min":1.1910000000000001,"5min":1.5389999999999999,"15min":1.5389999999999999},"last":1.329}]},{"osd":4,"last update":"Sat Sep 23 19:30:26 2023","interfaces":[{"interface":"back","average":{"1min":1.018,"5min":1.016,"15min":1.02},"min":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.589,"5min":1.589,"15min":1.589},"last":1.139},{"interface":"front","average":{"1min":0.96399999999999997,"5min":1,"15min":1.006},"min":{"1min":0.73299999999999998,"5min":0.69899999999999995,"15min":0.69899999999999995},"max":{"1min":1.3169999999999999,"5min":1.5,"15min":1.5},"last":1.1120000000000001}]},{"osd":5,"last update":"Sat Sep 23 19:30:26 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":1.1140000000000001,"15min":1.1319999999999999},"min":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"max":{"1min":1.3899999999999999,"5min":1.554,"15min":1.554},"last":1.375},{"interface":"front","average":{"1min":1.095,"5min":1.117,"15min":1.135},"min":{"1min":0.68600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"max":{"1min":1.5569999999999999,"5min":1.8340000000000001,"15min":1.8340000000000001},"last":1.04}]},{"osd":6,"last update":"Sat Sep 23 19:30:26 2023","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":1.0620000000000001,"15min":1.0960000000000001},"min":{"1min":0.67400000000000004,"5min":0.63100000000000001,"15min":0.63100000000000001},"max":{"1min":1.266,"5min":1.738,"15min":1.738},"last":0.97899999999999998},{"interface":"front","average":{"1min":1.157,"5min":1.1639999999999999,"15min":1.1679999999999999},"min":{"1min":0.82899999999999996,"5min":0.76200000000000001,"15min":0.76200000000000001},"max":{"1min":1.4830000000000001,"5min":1.7030000000000001,"15min":1.7030000000000001},"last":1.419}]},{"osd":7,"last update":"Sat Sep 23 19:30:26 2023","interfaces":[{"interface":"back","average":{"1min":1.1990000000000001,"5min":1.194,"15min":1.21},"min":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"max":{"1min":1.8220000000000001,"5min":1.8220000000000001,"15min":1.8220000000000001},"last":1.216},{"interface":"front","average":{"1min":1.1160000000000001,"5min":1.159,"15min":1.175},"min":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"max":{"1min":1.4790000000000001,"5min":1.5669999999999999,"15min":1.5669999999999999},"last":1.1950000000000001}]}]},{"osd":3,"up_from":116,"seq":498216206366,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":160112,"kb_used_data":1128,"kb_used_omap":2,"kb_used_meta":158973,"kb_avail":93585040,"statfs":{"total":95995035648,"available":95831080960,"internally_reserved":0,"allocated":1155072,"data_stored":697144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":162788405},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:30:09 2023","interfaces":[{"interface":"back","average":{"1min":0.70899999999999996,"5min":0.66800000000000004,"15min":0.66100000000000003},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":1.1000000000000001,"5min":1.1000000000000001,"15min":1.1000000000000001},"last":0.79800000000000004},{"interface":"front","average":{"1min":0.69499999999999995,"5min":0.78400000000000003,"15min":0.79900000000000004},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.996,"5min":1.3160000000000001,"15min":1.3160000000000001},"last":0.73999999999999999}]},{"osd":1,"last update":"Sat Sep 23 19:30:09 2023","interfaces":[{"interface":"back","average":{"1min":0.76000000000000001,"5min":0.746,"15min":0.74399999999999999},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.0960000000000001,"5min":1.0960000000000001,"15min":1.0960000000000001},"last":0.85899999999999999},{"interface":"front","average":{"1min":0.77500000000000002,"5min":0.79500000000000004,"15min":0.79800000000000004},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":1.2250000000000001,"5min":1.2250000000000001,"15min":1.2250000000000001},"last":1.022}]},{"osd":2,"last update":"Sat Sep 23 19:30:09 2023","interfaces":[{"interface":"back","average":{"1min":0.67300000000000004,"5min":0.755,"15min":0.76900000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.86699999999999999,"5min":1.137,"15min":1.137},"last":0.90600000000000003},{"interface":"front","average":{"1min":0.84199999999999997,"5min":0.84599999999999997,"15min":0.84699999999999998},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":1.3320000000000001,"5min":1.3320000000000001,"15min":1.3320000000000001},"last":1.286}]},{"osd":4,"last update":"Sat Sep 23 19:30:09 2023","interfaces":[{"interface":"back","average":{"1min":1.073,"5min":1.0760000000000001,"15min":1.0760000000000001},"min":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"max":{"1min":1.448,"5min":1.454,"15min":1.454},"last":1.397},{"interface":"front","average":{"1min":1.1040000000000001,"5min":1.1399999999999999,"15min":1.1459999999999999},"min":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"max":{"1min":1.6080000000000001,"5min":1.6080000000000001,"15min":1.6080000000000001},"last":1.2490000000000001}]},{"osd":5,"last update":"Sat Sep 23 19:30:09 2023","interfaces":[{"interface":"back","average":{"1min":1.0760000000000001,"5min":1.1160000000000001,"15min":1.1220000000000001},"min":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"max":{"1min":1.5169999999999999,"5min":1.5169999999999999,"15min":1.5169999999999999},"last":1.1080000000000001},{"interface":"front","average":{"1min":1.0680000000000001,"5min":1.0780000000000001,"15min":1.0800000000000001},"min":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":1.361,"5min":1.361,"15min":1.361},"last":1.0740000000000001}]},{"osd":6,"last update":"Sat Sep 23 19:30:09 2023","interfaces":[{"interface":"back","average":{"1min":1.1599999999999999,"5min":1.1479999999999999,"15min":1.1459999999999999},"min":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"max":{"1min":1.6830000000000001,"5min":1.6830000000000001,"15min":1.6830000000000001},"last":1.3340000000000001},{"interface":"front","average":{"1min":1.117,"5min":1.141,"15min":1.145},"min":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"max":{"1min":1.4750000000000001,"5min":1.5109999999999999,"15min":1.5109999999999999},"last":1.2110000000000001}]},{"osd":7,"last update":"Sat Sep 23 19:30:09 2023","interfaces":[{"interface":"back","average":{"1min":1.127,"5min":1.153,"15min":1.157},"min":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"max":{"1min":1.4810000000000001,"5min":1.597,"15min":1.597},"last":1.3720000000000001},{"interface":"front","average":{"1min":1.1899999999999999,"5min":1.1619999999999999,"15min":1.1579999999999999},"min":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"max":{"1min":1.7210000000000001,"5min":1.7210000000000001,"15min":1.7210000000000001},"last":1.4990000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019236,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":590832,"kb_used_data":1128,"kb_used_omap":0,"kb_used_meta":589696,"kb_avail":93154320,"statfs":{"total":95995035648,"available":95390023680,"internally_reserved":0,"allocated":1155072,"data_stored":697491,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603848704},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:29:30 2023","interfaces":[{"interface":"back","average":{"1min":0.88100000000000001,"5min":1.3260000000000001,"15min":3.306},"min":{"1min":0.61499999999999999,"5min":0.27400000000000002,"15min":0.26200000000000001},"max":{"1min":1.1719999999999999,"5min":8.7680000000000007,"15min":129.16200000000001},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.90400000000000003,"5min":1.244,"15min":3.5640000000000001},"min":{"1min":0.53700000000000003,"5min":0.32200000000000001,"15min":0.27700000000000002},"max":{"1min":1.23,"5min":6.2450000000000001,"15min":130.40600000000001},"last":1.0840000000000001}]},{"osd":1,"last update":"Sat Sep 23 19:29:58 2023","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":1.46,"15min":1.619},"min":{"1min":0.69899999999999995,"5min":0.30099999999999999,"15min":0.29499999999999998},"max":{"1min":1.3260000000000001,"5min":12.372,"15min":12.372},"last":0.79300000000000004},{"interface":"front","average":{"1min":0.90800000000000003,"5min":1.4390000000000001,"15min":1.5449999999999999},"min":{"1min":0.67500000000000004,"5min":0.33600000000000002,"15min":0.31},"max":{"1min":1.2250000000000001,"5min":7.3360000000000003,"15min":7.3360000000000003},"last":1.115}]},{"osd":2,"last update":"Sat Sep 23 19:29:26 2023","interfaces":[{"interface":"back","average":{"1min":0.92500000000000004,"5min":0.89300000000000002,"15min":0.88800000000000001},"min":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"max":{"1min":1.2290000000000001,"5min":1.327,"15min":1.327},"last":0.88},{"interface":"front","average":{"1min":0.90400000000000003,"5min":0.89900000000000002,"15min":0.89800000000000002},"min":{"1min":0.69699999999999995,"5min":0.51700000000000002,"15min":0.51700000000000002},"max":{"1min":1.1599999999999999,"5min":1.4379999999999999,"15min":1.4379999999999999},"last":0.90800000000000003}]},{"osd":3,"last update":"Sat Sep 23 19:30:13 2023","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":0.96799999999999997,"15min":0.97199999999999998},"min":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"max":{"1min":1.6899999999999999,"5min":1.6899999999999999,"15min":1.6899999999999999},"last":1.0589999999999999},{"interface":"front","average":{"1min":0.96699999999999997,"5min":0.96299999999999997,"15min":0.96199999999999997},"min":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":1.413,"5min":1.552,"15min":1.552},"last":1.1419999999999999}]},{"osd":5,"last update":"Sat Sep 23 19:29:42 2023","interfaces":[{"interface":"back","average":{"1min":1.018,"5min":1.4299999999999999,"15min":1.1279999999999999},"min":{"1min":0.57699999999999996,"5min":0.32500000000000001,"15min":0.29199999999999998},"max":{"1min":2.1709999999999998,"5min":12.459,"15min":12.459},"last":1.177},{"interface":"front","average":{"1min":1.1220000000000001,"5min":1.4730000000000001,"15min":1.234},"min":{"1min":0.877,"5min":0.34699999999999998,"15min":0.26700000000000002},"max":{"1min":1.8680000000000001,"5min":12.478,"15min":12.478},"last":1.03}]},{"osd":6,"last update":"Sat Sep 23 19:30:00 2023","interfaces":[{"interface":"back","average":{"1min":1.0940000000000001,"5min":1.367,"15min":1.274},"min":{"1min":0.495,"5min":0.39600000000000002,"15min":0.29999999999999999},"max":{"1min":2.5739999999999998,"5min":7.0810000000000004,"15min":7.0810000000000004},"last":0.95899999999999996},{"interface":"front","average":{"1min":1.1919999999999999,"5min":1.3260000000000001,"15min":1.2609999999999999},"min":{"1min":0.61599999999999999,"5min":0.40600000000000003,"15min":0.34300000000000003},"max":{"1min":2.5419999999999998,"5min":4.25,"15min":6.9050000000000002},"last":1.2090000000000001}]},{"osd":7,"last update":"Sat Sep 23 19:29:58 2023","interfaces":[{"interface":"back","average":{"1min":1.133,"5min":1.377,"15min":1.2110000000000001},"min":{"1min":0.67000000000000004,"5min":0.41599999999999998,"15min":0.312},"max":{"1min":1.9350000000000001,"5min":7.8479999999999999,"15min":7.8479999999999999},"last":1.004},{"interface":"front","average":{"1min":1.181,"5min":1.296,"15min":1.1220000000000001},"min":{"1min":0.76800000000000002,"5min":0.42499999999999999,"15min":0.36699999999999999},"max":{"1min":2.2629999999999999,"5min":5.8570000000000002,"15min":5.8570000000000002},"last":1.2290000000000001}]}]},{"osd":5,"up_from":36,"seq":154618823003,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":594904,"kb_used_data":1104,"kb_used_omap":0,"kb_used_meta":593792,"kb_avail":93150248,"statfs":{"total":95995035648,"available":95385853952,"internally_reserved":0,"allocated":1130496,"data_stored":692605,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":608043008},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.88100000000000001,"5min":1.0649999999999999,"15min":1.3360000000000001},"min":{"1min":0.49299999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.2360000000000002,"5min":7.6669999999999998,"15min":11.058},"last":0.95999999999999996},{"interface":"front","average":{"1min":0.98899999999999999,"5min":1.478,"15min":1.7050000000000001},"min":{"1min":0.54000000000000004,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":3.7890000000000001,"5min":10.802,"15min":10.802},"last":0.88100000000000001}]},{"osd":1,"last update":"Sat Sep 23 19:29:57 2023","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":1.0660000000000001,"15min":1.2589999999999999},"min":{"1min":0.625,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":1.2310000000000001,"5min":4.9720000000000004,"15min":9.5389999999999997},"last":1.23},{"interface":"front","average":{"1min":0.878,"5min":1.3859999999999999,"15min":2.0840000000000001},"min":{"1min":0.65300000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":1.161,"5min":11.228999999999999,"15min":11.228999999999999},"last":0.629}]},{"osd":2,"last update":"Sat Sep 23 19:29:29 2023","interfaces":[{"interface":"back","average":{"1min":1.0700000000000001,"5min":0.98099999999999998,"15min":0.96599999999999997},"min":{"1min":0.65100000000000002,"5min":0.56899999999999995,"15min":0.56899999999999995},"max":{"1min":2.3069999999999999,"5min":2.3069999999999999,"15min":2.3069999999999999},"last":1.1040000000000001},{"interface":"front","average":{"1min":1.151,"5min":1.0680000000000001,"15min":1.054},"min":{"1min":0.78800000000000003,"5min":0.753,"15min":0.753},"max":{"1min":2.335,"5min":2.335,"15min":2.335},"last":1.2769999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:30:10 2023","interfaces":[{"interface":"back","average":{"1min":0.90600000000000003,"5min":1.03,"15min":1.0509999999999999},"min":{"1min":0.70499999999999996,"5min":0.59399999999999997,"15min":0.59399999999999997},"max":{"1min":1.268,"5min":1.702,"15min":1.702},"last":1.302},{"interface":"front","average":{"1min":0.998,"5min":1.1379999999999999,"15min":1.1619999999999999},"min":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"max":{"1min":1.288,"5min":2.5179999999999998,"15min":2.5179999999999998},"last":1.131}]},{"osd":4,"last update":"Sat Sep 23 19:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":1.244,"15min":1.1519999999999999},"min":{"1min":0.63600000000000001,"5min":0.33200000000000002,"15min":0.29199999999999998},"max":{"1min":1.2669999999999999,"5min":8.1790000000000003,"15min":13.753},"last":0.997},{"interface":"front","average":{"1min":0.96899999999999997,"5min":1.3029999999999999,"15min":1.137},"min":{"1min":0.72199999999999998,"5min":0.34399999999999997,"15min":0.27000000000000002},"max":{"1min":1.444,"5min":13.714,"15min":13.714},"last":1.0229999999999999}]},{"osd":6,"last update":"Sat Sep 23 19:30:16 2023","interfaces":[{"interface":"back","average":{"1min":1.149,"5min":1.052,"15min":1.23},"min":{"1min":0.89900000000000002,"5min":0.36899999999999999,"15min":0.307},"max":{"1min":1.3600000000000001,"5min":4.2919999999999998,"15min":26.448},"last":1.0600000000000001},{"interface":"front","average":{"1min":1.1699999999999999,"5min":1.218,"15min":1.302},"min":{"1min":0.72999999999999998,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.395,"5min":4.7210000000000001,"15min":26.841999999999999},"last":1.258}]},{"osd":7,"last update":"Sat Sep 23 19:29:47 2023","interfaces":[{"interface":"back","average":{"1min":1.161,"5min":1.1379999999999999,"15min":1.361},"min":{"1min":0.77000000000000002,"5min":0.41399999999999998,"15min":0.36599999999999999},"max":{"1min":1.794,"5min":4.7990000000000004,"15min":32.585999999999999},"last":1.1990000000000001},{"interface":"front","average":{"1min":1.1850000000000001,"5min":1.145,"15min":1.25},"min":{"1min":0.80400000000000005,"5min":0.39800000000000002,"15min":0.35199999999999998},"max":{"1min":1.542,"5min":7.7050000000000001,"15min":33.014000000000003},"last":1.1839999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T19:30:29.524 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-09-23T19:30:29.525 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd dump 2023-09-23T19:30:31.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:30:31 smithi016 bash[75004]: cluster 2023-09-23T19:30:29.913520+0000 mgr.y (mgr.25325) 685 : cluster 0 pgmap v578: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:31.538 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:30:31 smithi016 bash[77995]: cluster 2023-09-23T19:30:29.913520+0000 mgr.y (mgr.25325) 685 : cluster 0 pgmap v578: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:31.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:30:31 smithi092 bash[58855]: cluster 2023-09-23T19:30:29.913520+0000 mgr.y (mgr.25325) 685 : cluster 0 pgmap v578: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:33.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:30:33 smithi016 bash[75004]: cluster 2023-09-23T19:30:31.914485+0000 mgr.y (mgr.25325) 686 : cluster 0 pgmap v579: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:33.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:30:33 smithi016 bash[77995]: cluster 2023-09-23T19:30:31.914485+0000 mgr.y (mgr.25325) 686 : cluster 0 pgmap v579: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:33.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:30:33 smithi092 bash[58855]: cluster 2023-09-23T19:30:31.914485+0000 mgr.y (mgr.25325) 686 : cluster 0 pgmap v579: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:34.351 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:30:35.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:30:35 smithi016 bash[75004]: cluster 2023-09-23T19:30:33.915224+0000 mgr.y (mgr.25325) 687 : cluster 0 pgmap v580: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:35.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:30:35 smithi016 bash[77995]: cluster 2023-09-23T19:30:33.915224+0000 mgr.y (mgr.25325) 687 : cluster 0 pgmap v580: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:35.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:30:35 smithi092 bash[58855]: cluster 2023-09-23T19:30:33.915224+0000 mgr.y (mgr.25325) 687 : cluster 0 pgmap v580: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:36.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:30:35 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:30:35] "GET /metrics HTTP/1.1" 200 33814 "" "Prometheus/2.33.4" 2023-09-23T19:30:37.534 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:30:37 smithi016 bash[75004]: cluster 2023-09-23T19:30:35.916167+0000 mgr.y (mgr.25325) 688 : cluster 0 pgmap v581: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:37.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:30:37 smithi016 bash[77995]: cluster 2023-09-23T19:30:35.916167+0000 mgr.y (mgr.25325) 688 : cluster 0 pgmap v581: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:37.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:30:37 smithi092 bash[58855]: cluster 2023-09-23T19:30:35.916167+0000 mgr.y (mgr.25325) 688 : cluster 0 pgmap v581: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:37.699 INFO:teuthology.orchestra.run.smithi016.stdout:epoch 117 2023-09-23T19:30:37.699 INFO:teuthology.orchestra.run.smithi016.stdout:fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 2023-09-23T19:30:37.700 INFO:teuthology.orchestra.run.smithi016.stdout:created 2023-09-23T18:55:23.802934+0000 2023-09-23T19:30:37.700 INFO:teuthology.orchestra.run.smithi016.stdout:modified 2023-09-23T19:28:07.860023+0000 2023-09-23T19:30:37.700 INFO:teuthology.orchestra.run.smithi016.stdout:flags sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit 2023-09-23T19:30:37.700 INFO:teuthology.orchestra.run.smithi016.stdout:crush_version 22 2023-09-23T19:30:37.700 INFO:teuthology.orchestra.run.smithi016.stdout:full_ratio 0.95 2023-09-23T19:30:37.700 INFO:teuthology.orchestra.run.smithi016.stdout:backfillfull_ratio 0.9 2023-09-23T19:30:37.700 INFO:teuthology.orchestra.run.smithi016.stdout:nearfull_ratio 0.85 2023-09-23T19:30:37.700 INFO:teuthology.orchestra.run.smithi016.stdout:require_min_compat_client luminous 2023-09-23T19:30:37.700 INFO:teuthology.orchestra.run.smithi016.stdout:min_compat_client luminous 2023-09-23T19:30:37.700 INFO:teuthology.orchestra.run.smithi016.stdout:require_osd_release pacific 2023-09-23T19:30:37.700 INFO:teuthology.orchestra.run.smithi016.stdout:stretch_mode_enabled false 2023-09-23T19:30:37.700 INFO:teuthology.orchestra.run.smithi016.stdout:pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode off last_change 69 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr,mgr_devicehealth read_balance_score 7.89 2023-09-23T19:30:37.701 INFO:teuthology.orchestra.run.smithi016.stdout:pool 2 'rbd' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode off last_change 55 flags hashpspool stripe_width 0 application rbd read_balance_score 2.00 2023-09-23T19:30:37.701 INFO:teuthology.orchestra.run.smithi016.stdout:pool 3 'cephfs.foo.meta' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 59 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 16 recovery_priority 5 application cephfs read_balance_score 1.50 2023-09-23T19:30:37.701 INFO:teuthology.orchestra.run.smithi016.stdout:pool 4 'cephfs.foo.data' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 59 flags hashpspool,bulk stripe_width 0 application cephfs read_balance_score 2.25 2023-09-23T19:30:37.701 INFO:teuthology.orchestra.run.smithi016.stdout:max_osd 8 2023-09-23T19:30:37.701 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0 up in weight 1 up_from 85 up_thru 116 down_at 82 last_clean_interval [8,81) [v2:172.21.15.16:6802/3034120466,v1:172.21.15.16:6803/3034120466] [v2:172.21.15.16:6804/3034120466,v1:172.21.15.16:6805/3034120466] exists,up a4e8ed09-1195-4878-928a-13881bc04dc7 2023-09-23T19:30:37.701 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1 up in weight 1 up_from 98 up_thru 116 down_at 94 last_clean_interval [13,93) [v2:172.21.15.16:6810/3881357617,v1:172.21.15.16:6811/3881357617] [v2:172.21.15.16:6812/3881357617,v1:172.21.15.16:6813/3881357617] exists,up c5ff2879-0d1c-481a-b847-248433c12aa4 2023-09-23T19:30:37.701 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2 up in weight 1 up_from 111 up_thru 116 down_at 108 last_clean_interval [18,107) [v2:172.21.15.16:6818/486425090,v1:172.21.15.16:6819/486425090] [v2:172.21.15.16:6820/486425090,v1:172.21.15.16:6821/486425090] exists,up 7974ce4d-9ed8-4a72-b44b-d500b34da8cf 2023-09-23T19:30:37.701 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3 up in weight 1 up_from 116 up_thru 116 down_at 113 last_clean_interval [25,112) [v2:172.21.15.16:6826/3155744043,v1:172.21.15.16:6827/3155744043] [v2:172.21.15.16:6828/3155744043,v1:172.21.15.16:6829/3155744043] exists,up f5e0d57b-d629-4453-a527-ad77e6c210ad 2023-09-23T19:30:37.701 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4 up in weight 1 up_from 30 up_thru 116 down_at 0 last_clean_interval [0,0) [v2:172.21.15.92:6800/99745912,v1:172.21.15.92:6801/99745912] [v2:172.21.15.92:6802/99745912,v1:172.21.15.92:6803/99745912] exists,up 55a8dab2-6393-44ab-943e-cb7ab2cb7c31 2023-09-23T19:30:37.701 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5 up in weight 1 up_from 36 up_thru 116 down_at 0 last_clean_interval [0,0) [v2:172.21.15.92:6808/1987214502,v1:172.21.15.92:6809/1987214502] [v2:172.21.15.92:6810/1987214502,v1:172.21.15.92:6811/1987214502] exists,up b29b73b1-51d7-4484-bd79-2334c650cb62 2023-09-23T19:30:37.701 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6 up in weight 1 up_from 42 up_thru 113 down_at 0 last_clean_interval [0,0) [v2:172.21.15.92:6816/2820798178,v1:172.21.15.92:6817/2820798178] [v2:172.21.15.92:6818/2820798178,v1:172.21.15.92:6819/2820798178] exists,up 9fe16cc9-f0d3-4b79-bc68-38305c55b551 2023-09-23T19:30:37.701 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7 up in weight 1 up_from 48 up_thru 116 down_at 0 last_clean_interval [0,0) [v2:172.21.15.92:6824/98624761,v1:172.21.15.92:6825/98624761] [v2:172.21.15.92:6826/98624761,v1:172.21.15.92:6827/98624761] exists,up d086b076-127f-42de-bee9-157bf579d92d 2023-09-23T19:30:37.702 INFO:teuthology.orchestra.run.smithi016.stdout:pg_upmap_items 4.2 [1,2] 2023-09-23T19:30:37.702 INFO:teuthology.orchestra.run.smithi016.stdout:pg_upmap_items 4.18 [1,5] 2023-09-23T19:30:37.702 INFO:teuthology.orchestra.run.smithi016.stdout:pg_upmap_items 4.1b [1,2] 2023-09-23T19:30:37.702 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.16:6800/2756103674 expires 2023-09-24T18:56:20.574625+0000 2023-09-23T19:30:37.702 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.16:0/2843459289 expires 2023-09-24T18:56:20.574625+0000 2023-09-23T19:30:37.702 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.16:6801/2756103674 expires 2023-09-24T18:56:20.574625+0000 2023-09-23T19:30:37.702 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.92:0/1841502917 expires 2023-09-24T19:11:32.247480+0000 2023-09-23T19:30:37.702 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.16:6801/777573315 expires 2023-09-24T19:12:43.452315+0000 2023-09-23T19:30:37.702 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.16:6800/777573315 expires 2023-09-24T19:12:43.452315+0000 2023-09-23T19:30:37.702 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.16:6801/2615282850 expires 2023-09-24T18:55:50.550546+0000 2023-09-23T19:30:37.702 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.16:6801/1259630240 expires 2023-09-24T19:03:34.846451+0000 2023-09-23T19:30:37.703 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.16:6800/1259630240 expires 2023-09-24T19:03:34.846451+0000 2023-09-23T19:30:37.703 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.16:0/79169563 expires 2023-09-24T18:55:50.550546+0000 2023-09-23T19:30:37.703 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.16:0/3595835147 expires 2023-09-24T19:12:43.452315+0000 2023-09-23T19:30:37.703 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.92:0/3173267635 expires 2023-09-24T19:11:32.247480+0000 2023-09-23T19:30:37.703 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.16:0/3394254122 expires 2023-09-24T19:12:43.452315+0000 2023-09-23T19:30:37.703 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.16:0/272242693 expires 2023-09-24T18:56:20.574625+0000 2023-09-23T19:30:37.703 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.16:0/736761601 expires 2023-09-24T19:12:43.452315+0000 2023-09-23T19:30:37.703 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.92:0/3338210669 expires 2023-09-24T19:11:32.247480+0000 2023-09-23T19:30:37.703 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.92:6835/2057654371 expires 2023-09-24T19:11:32.247480+0000 2023-09-23T19:30:37.703 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.16:0/3767844203 expires 2023-09-24T19:03:34.846451+0000 2023-09-23T19:30:37.703 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.16:0/2334334390 expires 2023-09-24T19:12:43.452315+0000 2023-09-23T19:30:37.703 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.16:0/4196794739 expires 2023-09-24T19:12:43.452315+0000 2023-09-23T19:30:37.704 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.16:6800/1501185908 expires 2023-09-24T19:09:35.764967+0000 2023-09-23T19:30:37.704 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.16:0/3765291224 expires 2023-09-24T19:09:35.764967+0000 2023-09-23T19:30:37.704 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.16:6801/1501185908 expires 2023-09-24T19:09:35.764967+0000 2023-09-23T19:30:37.704 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.16:0/647003663 expires 2023-09-24T19:12:43.452315+0000 2023-09-23T19:30:37.704 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.16:0/2669353350 expires 2023-09-24T19:03:34.846451+0000 2023-09-23T19:30:37.704 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.92:6834/2057654371 expires 2023-09-24T19:11:32.247480+0000 2023-09-23T19:30:37.704 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.92:0/561988499 expires 2023-09-24T19:11:32.247480+0000 2023-09-23T19:30:37.704 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.16:0/4152413938 expires 2023-09-24T19:09:35.764967+0000 2023-09-23T19:30:37.704 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.16:6800/2615282850 expires 2023-09-24T18:55:50.550546+0000 2023-09-23T19:30:37.704 INFO:teuthology.orchestra.run.smithi016.stdout:blocklist 172.21.15.16:0/349446544 expires 2023-09-24T18:55:50.550546+0000 2023-09-23T19:30:38.583 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:30:38 smithi016 bash[77995]: audit 2023-09-23T19:30:37.695373+0000 mon.c (mon.2) 136 : audit 0 from='client.? 172.21.15.16:0/3213793045' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T19:30:38.584 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:30:38 smithi016 bash[75004]: audit 2023-09-23T19:30:37.695373+0000 mon.c (mon.2) 136 : audit 0 from='client.? 172.21.15.16:0/3213793045' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T19:30:38.584 INFO:tasks.cephadm.ceph_manager.ceph:['osd.0 up in weight 1 up_from 85 up_thru 116 down_at 82 last_clean_interval [8,81) [v2:172.21.15.16:6802/3034120466,v1:172.21.15.16:6803/3034120466] [v2:172.21.15.16:6804/3034120466,v1:172.21.15.16:6805/3034120466] exists,up a4e8ed09-1195-4878-928a-13881bc04dc7', 'osd.1 up in weight 1 up_from 98 up_thru 116 down_at 94 last_clean_interval [13,93) [v2:172.21.15.16:6810/3881357617,v1:172.21.15.16:6811/3881357617] [v2:172.21.15.16:6812/3881357617,v1:172.21.15.16:6813/3881357617] exists,up c5ff2879-0d1c-481a-b847-248433c12aa4', 'osd.2 up in weight 1 up_from 111 up_thru 116 down_at 108 last_clean_interval [18,107) [v2:172.21.15.16:6818/486425090,v1:172.21.15.16:6819/486425090] [v2:172.21.15.16:6820/486425090,v1:172.21.15.16:6821/486425090] exists,up 7974ce4d-9ed8-4a72-b44b-d500b34da8cf', 'osd.3 up in weight 1 up_from 116 up_thru 116 down_at 113 last_clean_interval [25,112) [v2:172.21.15.16:6826/3155744043,v1:172.21.15.16:6827/3155744043] [v2:172.21.15.16:6828/3155744043,v1:172.21.15.16:6829/3155744043] exists,up f5e0d57b-d629-4453-a527-ad77e6c210ad', 'osd.4 up in weight 1 up_from 30 up_thru 116 down_at 0 last_clean_interval [0,0) [v2:172.21.15.92:6800/99745912,v1:172.21.15.92:6801/99745912] [v2:172.21.15.92:6802/99745912,v1:172.21.15.92:6803/99745912] exists,up 55a8dab2-6393-44ab-943e-cb7ab2cb7c31', 'osd.5 up in weight 1 up_from 36 up_thru 116 down_at 0 last_clean_interval [0,0) [v2:172.21.15.92:6808/1987214502,v1:172.21.15.92:6809/1987214502] [v2:172.21.15.92:6810/1987214502,v1:172.21.15.92:6811/1987214502] exists,up b29b73b1-51d7-4484-bd79-2334c650cb62', 'osd.6 up in weight 1 up_from 42 up_thru 113 down_at 0 last_clean_interval [0,0) [v2:172.21.15.92:6816/2820798178,v1:172.21.15.92:6817/2820798178] [v2:172.21.15.92:6818/2820798178,v1:172.21.15.92:6819/2820798178] exists,up 9fe16cc9-f0d3-4b79-bc68-38305c55b551', 'osd.7 up in weight 1 up_from 48 up_thru 116 down_at 0 last_clean_interval [0,0) [v2:172.21.15.92:6824/98624761,v1:172.21.15.92:6825/98624761] [v2:172.21.15.92:6826/98624761,v1:172.21.15.92:6827/98624761] exists,up d086b076-127f-42de-bee9-157bf579d92d'] 2023-09-23T19:30:38.585 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon mon.a config show 2023-09-23T19:30:38.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:30:38 smithi092 bash[58855]: audit 2023-09-23T19:30:37.695373+0000 mon.c (mon.2) 136 : audit 0 from='client.? 172.21.15.16:0/3213793045' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T19:30:39.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:30:39 smithi016 bash[75004]: cluster 2023-09-23T19:30:37.916888+0000 mgr.y (mgr.25325) 689 : cluster 0 pgmap v582: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:39.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:30:39 smithi016 bash[77995]: cluster 2023-09-23T19:30:37.916888+0000 mgr.y (mgr.25325) 689 : cluster 0 pgmap v582: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:39.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:30:39 smithi092 bash[58855]: cluster 2023-09-23T19:30:37.916888+0000 mgr.y (mgr.25325) 689 : cluster 0 pgmap v582: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:41.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:30:41 smithi016 bash[75004]: cluster 2023-09-23T19:30:39.917860+0000 mgr.y (mgr.25325) 690 : cluster 0 pgmap v583: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:41.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:30:41 smithi016 bash[77995]: cluster 2023-09-23T19:30:39.917860+0000 mgr.y (mgr.25325) 690 : cluster 0 pgmap v583: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:41.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:30:41 smithi092 bash[58855]: cluster 2023-09-23T19:30:39.917860+0000 mgr.y (mgr.25325) 690 : cluster 0 pgmap v583: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:43.371 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:30:43.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:30:43 smithi016 bash[75004]: cluster 2023-09-23T19:30:41.918756+0000 mgr.y (mgr.25325) 691 : cluster 0 pgmap v584: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:43.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:30:43 smithi016 bash[77995]: cluster 2023-09-23T19:30:41.918756+0000 mgr.y (mgr.25325) 691 : cluster 0 pgmap v584: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:43.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:30:43 smithi092 bash[58855]: cluster 2023-09-23T19:30:41.918756+0000 mgr.y (mgr.25325) 691 : cluster 0 pgmap v584: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:44.638 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:30:44 smithi016 bash[75004]: audit 2023-09-23T19:30:43.635475+0000 mon.a (mon.0) 887 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:30:44.638 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:30:44 smithi016 bash[77995]: audit 2023-09-23T19:30:43.635475+0000 mon.a (mon.0) 887 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:30:44.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:30:44 smithi092 bash[58855]: audit 2023-09-23T19:30:43.635475+0000 mon.a (mon.0) 887 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:30:45.641 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:30:45 smithi016 bash[75004]: cluster 2023-09-23T19:30:43.919398+0000 mgr.y (mgr.25325) 692 : cluster 0 pgmap v585: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:45.642 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:30:45 smithi016 bash[77995]: cluster 2023-09-23T19:30:43.919398+0000 mgr.y (mgr.25325) 692 : cluster 0 pgmap v585: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:45.784 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:30:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:30:45] "GET /metrics HTTP/1.1" 200 33814 "" "Prometheus/2.33.4" 2023-09-23T19:30:45.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:30:45 smithi092 bash[58855]: cluster 2023-09-23T19:30:43.919398+0000 mgr.y (mgr.25325) 692 : cluster 0 pgmap v585: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:46.706 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-09-23T19:30:46.706 INFO:teuthology.orchestra.run.smithi016.stdout: "name": "mon.a", 2023-09-23T19:30:46.706 INFO:teuthology.orchestra.run.smithi016.stdout: "cluster": "ceph", 2023-09-23T19:30:46.706 INFO:teuthology.orchestra.run.smithi016.stdout: "admin_socket": "/var/run/ceph/ceph-mon.a.asok", 2023-09-23T19:30:46.706 INFO:teuthology.orchestra.run.smithi016.stdout: "admin_socket_mode": "", 2023-09-23T19:30:46.707 INFO:teuthology.orchestra.run.smithi016.stdout: "auth_allow_insecure_global_id_reclaim": "false", 2023-09-23T19:30:46.707 INFO:teuthology.orchestra.run.smithi016.stdout: "auth_client_required": "cephx, none", 2023-09-23T19:30:46.707 INFO:teuthology.orchestra.run.smithi016.stdout: "auth_cluster_required": "cephx", 2023-09-23T19:30:46.707 INFO:teuthology.orchestra.run.smithi016.stdout: "auth_debug": "true", 2023-09-23T19:30:46.707 INFO:teuthology.orchestra.run.smithi016.stdout: "auth_expose_insecure_global_id_reclaim": "true", 2023-09-23T19:30:46.707 INFO:teuthology.orchestra.run.smithi016.stdout: "auth_mon_ticket_ttl": "660.000000", 2023-09-23T19:30:46.707 INFO:teuthology.orchestra.run.smithi016.stdout: "auth_service_required": "cephx", 2023-09-23T19:30:46.707 INFO:teuthology.orchestra.run.smithi016.stdout: "auth_service_ticket_ttl": "240.000000", 2023-09-23T19:30:46.707 INFO:teuthology.orchestra.run.smithi016.stdout: "auth_supported": "", 2023-09-23T19:30:46.708 INFO:teuthology.orchestra.run.smithi016.stdout: "bdev_aio": "true", 2023-09-23T19:30:46.708 INFO:teuthology.orchestra.run.smithi016.stdout: "bdev_aio_max_queue_depth": "1024", 2023-09-23T19:30:46.708 INFO:teuthology.orchestra.run.smithi016.stdout: "bdev_aio_poll_ms": "250", 2023-09-23T19:30:46.708 INFO:teuthology.orchestra.run.smithi016.stdout: "bdev_aio_reap_max": "16", 2023-09-23T19:30:46.708 INFO:teuthology.orchestra.run.smithi016.stdout: "bdev_async_discard": "false", 2023-09-23T19:30:46.708 INFO:teuthology.orchestra.run.smithi016.stdout: "bdev_block_size": "4096", 2023-09-23T19:30:46.708 INFO:teuthology.orchestra.run.smithi016.stdout: "bdev_debug_aio": "false", 2023-09-23T19:30:46.708 INFO:teuthology.orchestra.run.smithi016.stdout: "bdev_debug_aio_log_age": "5.000000", 2023-09-23T19:30:46.708 INFO:teuthology.orchestra.run.smithi016.stdout: "bdev_debug_aio_suicide_timeout": "60.000000", 2023-09-23T19:30:46.708 INFO:teuthology.orchestra.run.smithi016.stdout: "bdev_debug_inflight_ios": "false", 2023-09-23T19:30:46.708 INFO:teuthology.orchestra.run.smithi016.stdout: "bdev_enable_discard": "false", 2023-09-23T19:30:46.709 INFO:teuthology.orchestra.run.smithi016.stdout: "bdev_flock_retry": "3", 2023-09-23T19:30:46.709 INFO:teuthology.orchestra.run.smithi016.stdout: "bdev_flock_retry_interval": "0.100000", 2023-09-23T19:30:46.709 INFO:teuthology.orchestra.run.smithi016.stdout: "bdev_inject_crash": "0", 2023-09-23T19:30:46.709 INFO:teuthology.orchestra.run.smithi016.stdout: "bdev_inject_crash_flush_delay": "2", 2023-09-23T19:30:46.709 INFO:teuthology.orchestra.run.smithi016.stdout: "bdev_ioring": "false", 2023-09-23T19:30:46.709 INFO:teuthology.orchestra.run.smithi016.stdout: "bdev_ioring_hipri": "false", 2023-09-23T19:30:46.709 INFO:teuthology.orchestra.run.smithi016.stdout: "bdev_ioring_sqthread_poll": "false", 2023-09-23T19:30:46.709 INFO:teuthology.orchestra.run.smithi016.stdout: "bdev_nvme_unbind_from_kernel": "false", 2023-09-23T19:30:46.709 INFO:teuthology.orchestra.run.smithi016.stdout: "bdev_read_buffer_alignment": "4096", 2023-09-23T19:30:46.709 INFO:teuthology.orchestra.run.smithi016.stdout: "bdev_read_preallocated_huge_buffers": "", 2023-09-23T19:30:46.709 INFO:teuthology.orchestra.run.smithi016.stdout: "bdev_type": "", 2023-09-23T19:30:46.709 INFO:teuthology.orchestra.run.smithi016.stdout: "bluefs_alloc_size": "1048576", 2023-09-23T19:30:46.710 INFO:teuthology.orchestra.run.smithi016.stdout: "bluefs_allocator": "hybrid", 2023-09-23T19:30:46.710 INFO:teuthology.orchestra.run.smithi016.stdout: "bluefs_buffered_io": "true", 2023-09-23T19:30:46.710 INFO:teuthology.orchestra.run.smithi016.stdout: "bluefs_check_for_zeros": "false", 2023-09-23T19:30:46.710 INFO:teuthology.orchestra.run.smithi016.stdout: "bluefs_check_volume_selector_often": "false", 2023-09-23T19:30:46.710 INFO:teuthology.orchestra.run.smithi016.stdout: "bluefs_check_volume_selector_on_umount": "false", 2023-09-23T19:30:46.710 INFO:teuthology.orchestra.run.smithi016.stdout: "bluefs_compact_log_sync": "false", 2023-09-23T19:30:46.710 INFO:teuthology.orchestra.run.smithi016.stdout: "bluefs_failed_shared_alloc_cooldown": "600.000000", 2023-09-23T19:30:46.710 INFO:teuthology.orchestra.run.smithi016.stdout: "bluefs_log_compact_min_ratio": "5.000000", 2023-09-23T19:30:46.710 INFO:teuthology.orchestra.run.smithi016.stdout: "bluefs_log_compact_min_size": "16777216", 2023-09-23T19:30:46.710 INFO:teuthology.orchestra.run.smithi016.stdout: "bluefs_log_replay_check_allocations": "true", 2023-09-23T19:30:46.710 INFO:teuthology.orchestra.run.smithi016.stdout: "bluefs_max_log_runway": "4194304", 2023-09-23T19:30:46.710 INFO:teuthology.orchestra.run.smithi016.stdout: "bluefs_max_prefetch": "1048576", 2023-09-23T19:30:46.711 INFO:teuthology.orchestra.run.smithi016.stdout: "bluefs_min_flush_size": "524288", 2023-09-23T19:30:46.711 INFO:teuthology.orchestra.run.smithi016.stdout: "bluefs_min_log_runway": "1048576", 2023-09-23T19:30:46.711 INFO:teuthology.orchestra.run.smithi016.stdout: "bluefs_replay_recovery": "false", 2023-09-23T19:30:46.711 INFO:teuthology.orchestra.run.smithi016.stdout: "bluefs_replay_recovery_disable_compact": "false", 2023-09-23T19:30:46.711 INFO:teuthology.orchestra.run.smithi016.stdout: "bluefs_shared_alloc_size": "65536", 2023-09-23T19:30:46.711 INFO:teuthology.orchestra.run.smithi016.stdout: "bluefs_sync_write": "false", 2023-09-23T19:30:46.711 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_2q_cache_kin_ratio": "0.500000", 2023-09-23T19:30:46.711 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_2q_cache_kout_ratio": "0.500000", 2023-09-23T19:30:46.711 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_alloc_stats_dump_interval": "86400.000000", 2023-09-23T19:30:46.712 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_allocation_from_file": "true", 2023-09-23T19:30:46.712 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_allocator": "hybrid", 2023-09-23T19:30:46.712 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_avl_alloc_bf_free_pct": "4", 2023-09-23T19:30:46.712 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_avl_alloc_bf_threshold": "131072", 2023-09-23T19:30:46.712 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_avl_alloc_ff_max_search_bytes": "16777216", 2023-09-23T19:30:46.712 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_avl_alloc_ff_max_search_count": "100", 2023-09-23T19:30:46.712 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_bitmapallocator_blocks_per_zone": "1024", 2023-09-23T19:30:46.712 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_bitmapallocator_span_size": "1024", 2023-09-23T19:30:46.713 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_blobid_prealloc": "10240", 2023-09-23T19:30:46.713 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_block_create": "true", 2023-09-23T19:30:46.713 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_block_db_create": "false", 2023-09-23T19:30:46.713 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_block_db_path": "", 2023-09-23T19:30:46.713 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_block_db_size": "0", 2023-09-23T19:30:46.713 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_block_path": "", 2023-09-23T19:30:46.713 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_block_preallocate_file": "false", 2023-09-23T19:30:46.713 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_block_size": "107374182400", 2023-09-23T19:30:46.713 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_block_wal_create": "false", 2023-09-23T19:30:46.714 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_block_wal_path": "", 2023-09-23T19:30:46.714 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_block_wal_size": "100663296", 2023-09-23T19:30:46.714 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_bluefs": "true", 2023-09-23T19:30:46.714 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_bluefs_alloc_failure_dump_interval": "0.000000", 2023-09-23T19:30:46.714 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_bluefs_env_mirror": "false", 2023-09-23T19:30:46.714 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_bluefs_max_free": "10737418240", 2023-09-23T19:30:46.714 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_cache_age_bin_interval": "1.000000", 2023-09-23T19:30:46.714 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_cache_age_bins_data": "1 2 6 24 120 720 0 0 0 0", 2023-09-23T19:30:46.715 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_cache_age_bins_kv": "1 2 6 24 120 720 0 0 0 0", 2023-09-23T19:30:46.715 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_cache_age_bins_kv_onode": "0 0 0 0 0 0 0 0 0 720", 2023-09-23T19:30:46.715 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_cache_age_bins_meta": "1 2 6 24 120 720 0 0 0 0", 2023-09-23T19:30:46.715 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_cache_autotune": "true", 2023-09-23T19:30:46.715 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_cache_autotune_interval": "5.000000", 2023-09-23T19:30:46.715 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_cache_kv_onode_ratio": "0.040000", 2023-09-23T19:30:46.715 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_cache_kv_ratio": "0.450000", 2023-09-23T19:30:46.715 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_cache_meta_ratio": "0.450000", 2023-09-23T19:30:46.716 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_cache_size": "0", 2023-09-23T19:30:46.716 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_cache_size_hdd": "1073741824", 2023-09-23T19:30:46.716 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_cache_size_ssd": "3221225472", 2023-09-23T19:30:46.716 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_cache_trim_interval": "0.050000", 2023-09-23T19:30:46.716 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_cache_trim_max_skip_pinned": "1000", 2023-09-23T19:30:46.716 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_cache_type": "2q", 2023-09-23T19:30:46.716 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_cleaner_sleep_interval": "5.000000", 2023-09-23T19:30:46.716 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_clone_cow": "true", 2023-09-23T19:30:46.716 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_compression_algorithm": "snappy", 2023-09-23T19:30:46.717 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_compression_max_blob_size": "0", 2023-09-23T19:30:46.717 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_compression_max_blob_size_hdd": "65536", 2023-09-23T19:30:46.717 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_compression_max_blob_size_ssd": "65536", 2023-09-23T19:30:46.717 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_compression_min_blob_size": "0", 2023-09-23T19:30:46.717 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_compression_min_blob_size_hdd": "8192", 2023-09-23T19:30:46.717 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_compression_min_blob_size_ssd": "65536", 2023-09-23T19:30:46.717 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_compression_mode": "none", 2023-09-23T19:30:46.717 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_compression_required_ratio": "0.875000", 2023-09-23T19:30:46.718 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_csum_type": "crc32c", 2023-09-23T19:30:46.718 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_debug_enforce_settings": "default", 2023-09-23T19:30:46.718 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_debug_freelist": "false", 2023-09-23T19:30:46.718 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_debug_fsck_abort": "false", 2023-09-23T19:30:46.718 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_debug_inject_allocation_from_file_failure": "0.000000", 2023-09-23T19:30:46.718 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_debug_inject_bug21040": "false", 2023-09-23T19:30:46.718 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_debug_inject_csum_err_probability": "0.000000", 2023-09-23T19:30:46.718 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_debug_inject_read_err": "false", 2023-09-23T19:30:46.718 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_debug_legacy_omap": "false", 2023-09-23T19:30:46.719 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_debug_no_reuse_blocks": "false", 2023-09-23T19:30:46.719 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_debug_omit_block_device_write": "false", 2023-09-23T19:30:46.719 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_debug_omit_kv_commit": "false", 2023-09-23T19:30:46.719 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_debug_permit_any_bdev_label": "false", 2023-09-23T19:30:46.719 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_debug_prefill": "0.000000", 2023-09-23T19:30:46.719 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_debug_prefragment_max": "1048576", 2023-09-23T19:30:46.719 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_debug_random_read_err": "0.000000", 2023-09-23T19:30:46.719 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_debug_randomize_serial_transaction": "0", 2023-09-23T19:30:46.720 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_debug_small_allocations": "0", 2023-09-23T19:30:46.720 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_debug_too_many_blobs_threshold": "24576", 2023-09-23T19:30:46.720 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_default_buffered_read": "true", 2023-09-23T19:30:46.720 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_default_buffered_write": "false", 2023-09-23T19:30:46.720 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_deferred_batch_ops": "0", 2023-09-23T19:30:46.720 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_deferred_batch_ops_hdd": "64", 2023-09-23T19:30:46.720 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_deferred_batch_ops_ssd": "16", 2023-09-23T19:30:46.720 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_extent_map_inline_shard_prealloc_size": "256", 2023-09-23T19:30:46.720 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_extent_map_shard_max_size": "1200", 2023-09-23T19:30:46.721 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_extent_map_shard_min_size": "150", 2023-09-23T19:30:46.721 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_extent_map_shard_target_size": "500", 2023-09-23T19:30:46.721 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_extent_map_shard_target_size_slop": "0.200000", 2023-09-23T19:30:46.721 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_fail_eio": "false", 2023-09-23T19:30:46.721 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_freelist_blocks_per_key": "128", 2023-09-23T19:30:46.721 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_fsck_error_on_no_per_pg_omap": "false", 2023-09-23T19:30:46.721 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_fsck_error_on_no_per_pool_omap": "false", 2023-09-23T19:30:46.721 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_fsck_error_on_no_per_pool_stats": "false", 2023-09-23T19:30:46.721 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_fsck_on_mkfs": "true", 2023-09-23T19:30:46.722 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_fsck_on_mkfs_deep": "false", 2023-09-23T19:30:46.722 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_fsck_on_mount": "false", 2023-09-23T19:30:46.722 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_fsck_on_mount_deep": "false", 2023-09-23T19:30:46.722 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_fsck_on_umount": "false", 2023-09-23T19:30:46.722 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_fsck_on_umount_deep": "false", 2023-09-23T19:30:46.722 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_fsck_quick_fix_on_mount": "false", 2023-09-23T19:30:46.722 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_fsck_quick_fix_threads": "2", 2023-09-23T19:30:46.722 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_fsck_read_bytes_cap": "67108864", 2023-09-23T19:30:46.722 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_fsck_shared_blob_tracker_size": "0.031250", 2023-09-23T19:30:46.722 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_gc_enable_blob_threshold": "0", 2023-09-23T19:30:46.723 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_gc_enable_total_threshold": "0", 2023-09-23T19:30:46.723 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_hybrid_alloc_mem_cap": "67108864", 2023-09-23T19:30:46.723 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_ignore_data_csum": "false", 2023-09-23T19:30:46.723 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_kv_sync_util_logging_s": "10.000000", 2023-09-23T19:30:46.723 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_kvbackend": "rocksdb", 2023-09-23T19:30:46.723 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_log_collection_list_age": "60.000000", 2023-09-23T19:30:46.723 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_log_omap_iterator_age": "5.000000", 2023-09-23T19:30:46.723 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_log_op_age": "5.000000", 2023-09-23T19:30:46.723 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_max_alloc_size": "0", 2023-09-23T19:30:46.724 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_max_blob_size": "0", 2023-09-23T19:30:46.724 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_max_blob_size_hdd": "65536", 2023-09-23T19:30:46.724 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_max_blob_size_ssd": "65536", 2023-09-23T19:30:46.724 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_max_defer_interval": "3.000000", 2023-09-23T19:30:46.724 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_max_deferred_txc": "32", 2023-09-23T19:30:46.724 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_min_alloc_size": "0", 2023-09-23T19:30:46.724 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_min_alloc_size_hdd": "4096", 2023-09-23T19:30:46.724 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_min_alloc_size_ssd": "4096", 2023-09-23T19:30:46.724 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_nid_prealloc": "1024", 2023-09-23T19:30:46.724 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_prefer_deferred_size": "0", 2023-09-23T19:30:46.724 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_prefer_deferred_size_hdd": "65536", 2023-09-23T19:30:46.725 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_prefer_deferred_size_ssd": "0", 2023-09-23T19:30:46.725 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_qfsck_on_mount": "true", 2023-09-23T19:30:46.725 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_retry_disk_reads": "3", 2023-09-23T19:30:46.725 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_rocksdb_cf": "true", 2023-09-23T19:30:46.725 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_rocksdb_cfs": "m(3) p(3,0-12) O(3,0-13)=block_cache={type=binned_lru} L=min_write_buffer_number_to_merge=32 P=min_write_buffer_number_to_merge=32", 2023-09-23T19:30:46.725 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_rocksdb_options": "compression=kNoCompression,max_write_buffer_number=64,min_write_buffer_number_to_merge=6,compaction_style=kCompactionStyleLevel,write_buffer_size=16777216,max_background_jobs=4,level0_file_num_compaction_trigger=8,max_bytes_for_level_base=1073741824,max_bytes_for_level_multiplier=8,compaction_readahead_size=2MB,max_total_wal_size=1073741824,writable_file_max_buffer_size=0", 2023-09-23T19:30:46.725 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_rocksdb_options_annex": "", 2023-09-23T19:30:46.725 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_spdk_coremask": "0x1", 2023-09-23T19:30:46.726 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_spdk_io_sleep": "5", 2023-09-23T19:30:46.726 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_spdk_max_io_completion": "0", 2023-09-23T19:30:46.726 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_spdk_mem": "512", 2023-09-23T19:30:46.726 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_sync_submit_transaction": "false", 2023-09-23T19:30:46.726 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_throttle_bytes": "67108864", 2023-09-23T19:30:46.726 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_throttle_cost_per_io": "0", 2023-09-23T19:30:46.726 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_throttle_cost_per_io_hdd": "670000", 2023-09-23T19:30:46.726 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_throttle_cost_per_io_ssd": "4000", 2023-09-23T19:30:46.727 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_throttle_deferred_bytes": "134217728", 2023-09-23T19:30:46.727 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_throttle_trace_rate": "0.000000", 2023-09-23T19:30:46.727 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_tracing": "false", 2023-09-23T19:30:46.727 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_use_optimal_io_size_for_min_alloc_size": "false", 2023-09-23T19:30:46.727 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_volume_selection_policy": "use_some_extra", 2023-09-23T19:30:46.727 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_volume_selection_reserved": "0", 2023-09-23T19:30:46.727 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_volume_selection_reserved_factor": "2.000000", 2023-09-23T19:30:46.727 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_warn_on_bluefs_spillover": "true", 2023-09-23T19:30:46.727 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_warn_on_legacy_statfs": "true", 2023-09-23T19:30:46.728 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_warn_on_no_per_pg_omap": "false", 2023-09-23T19:30:46.728 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_warn_on_no_per_pool_omap": "true", 2023-09-23T19:30:46.728 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_warn_on_spurious_read_errors": "true", 2023-09-23T19:30:46.728 INFO:teuthology.orchestra.run.smithi016.stdout: "bluestore_zero_block_detection": "false", 2023-09-23T19:30:46.728 INFO:teuthology.orchestra.run.smithi016.stdout: "cephadm_path": "/usr/sbin/cephadm", 2023-09-23T19:30:46.728 INFO:teuthology.orchestra.run.smithi016.stdout: "cephfs_mirror_action_update_interval": "2", 2023-09-23T19:30:46.728 INFO:teuthology.orchestra.run.smithi016.stdout: "cephfs_mirror_directory_scan_interval": "10", 2023-09-23T19:30:46.728 INFO:teuthology.orchestra.run.smithi016.stdout: "cephfs_mirror_max_concurrent_directory_syncs": "3", 2023-09-23T19:30:46.729 INFO:teuthology.orchestra.run.smithi016.stdout: "cephfs_mirror_max_consecutive_failures_per_directory": "10", 2023-09-23T19:30:46.729 INFO:teuthology.orchestra.run.smithi016.stdout: "cephfs_mirror_max_snapshot_sync_per_cycle": "3", 2023-09-23T19:30:46.729 INFO:teuthology.orchestra.run.smithi016.stdout: "cephfs_mirror_mount_timeout": "10", 2023-09-23T19:30:46.729 INFO:teuthology.orchestra.run.smithi016.stdout: "cephfs_mirror_restart_mirror_on_blocklist_interval": "30", 2023-09-23T19:30:46.729 INFO:teuthology.orchestra.run.smithi016.stdout: "cephfs_mirror_restart_mirror_on_failure_interval": "20", 2023-09-23T19:30:46.729 INFO:teuthology.orchestra.run.smithi016.stdout: "cephfs_mirror_retry_failed_directories_interval": "60", 2023-09-23T19:30:46.729 INFO:teuthology.orchestra.run.smithi016.stdout: "cephsqlite_blocklist_dead_locker": "true", 2023-09-23T19:30:46.729 INFO:teuthology.orchestra.run.smithi016.stdout: "cephsqlite_lock_renewal_interval": "2000", 2023-09-23T19:30:46.729 INFO:teuthology.orchestra.run.smithi016.stdout: "cephsqlite_lock_renewal_timeout": "30000", 2023-09-23T19:30:46.729 INFO:teuthology.orchestra.run.smithi016.stdout: "cephx_cluster_require_signatures": "false", 2023-09-23T19:30:46.729 INFO:teuthology.orchestra.run.smithi016.stdout: "cephx_cluster_require_version": "2", 2023-09-23T19:30:46.730 INFO:teuthology.orchestra.run.smithi016.stdout: "cephx_require_signatures": "false", 2023-09-23T19:30:46.730 INFO:teuthology.orchestra.run.smithi016.stdout: "cephx_require_version": "2", 2023-09-23T19:30:46.730 INFO:teuthology.orchestra.run.smithi016.stdout: "cephx_service_require_signatures": "false", 2023-09-23T19:30:46.730 INFO:teuthology.orchestra.run.smithi016.stdout: "cephx_service_require_version": "2", 2023-09-23T19:30:46.730 INFO:teuthology.orchestra.run.smithi016.stdout: "cephx_sign_messages": "true", 2023-09-23T19:30:46.730 INFO:teuthology.orchestra.run.smithi016.stdout: "chdir": "", 2023-09-23T19:30:46.730 INFO:teuthology.orchestra.run.smithi016.stdout: "client_acl_type": "", 2023-09-23T19:30:46.730 INFO:teuthology.orchestra.run.smithi016.stdout: "client_asio_thread_count": "2", 2023-09-23T19:30:46.730 INFO:teuthology.orchestra.run.smithi016.stdout: "client_cache_mid": "0.750000", 2023-09-23T19:30:46.730 INFO:teuthology.orchestra.run.smithi016.stdout: "client_cache_size": "16384", 2023-09-23T19:30:46.730 INFO:teuthology.orchestra.run.smithi016.stdout: "client_caps_release_delay": "5", 2023-09-23T19:30:46.730 INFO:teuthology.orchestra.run.smithi016.stdout: "client_check_pool_perm": "true", 2023-09-23T19:30:46.731 INFO:teuthology.orchestra.run.smithi016.stdout: "client_collect_and_send_global_metrics": "false", 2023-09-23T19:30:46.731 INFO:teuthology.orchestra.run.smithi016.stdout: "client_debug_force_sync_read": "false", 2023-09-23T19:30:46.731 INFO:teuthology.orchestra.run.smithi016.stdout: "client_debug_getattr_caps": "false", 2023-09-23T19:30:46.731 INFO:teuthology.orchestra.run.smithi016.stdout: "client_debug_inject_tick_delay": "0", 2023-09-23T19:30:46.731 INFO:teuthology.orchestra.run.smithi016.stdout: "client_die_on_failed_dentry_invalidate": "true", 2023-09-23T19:30:46.731 INFO:teuthology.orchestra.run.smithi016.stdout: "client_die_on_failed_remount": "false", 2023-09-23T19:30:46.731 INFO:teuthology.orchestra.run.smithi016.stdout: "client_dirsize_rbytes": "true", 2023-09-23T19:30:46.731 INFO:teuthology.orchestra.run.smithi016.stdout: "client_force_lazyio": "false", 2023-09-23T19:30:46.731 INFO:teuthology.orchestra.run.smithi016.stdout: "client_fs": "", 2023-09-23T19:30:46.731 INFO:teuthology.orchestra.run.smithi016.stdout: "client_inject_fixed_oldest_tid": "false", 2023-09-23T19:30:46.731 INFO:teuthology.orchestra.run.smithi016.stdout: "client_inject_release_failure": "false", 2023-09-23T19:30:46.731 INFO:teuthology.orchestra.run.smithi016.stdout: "client_max_inline_size": "4096", 2023-09-23T19:30:46.732 INFO:teuthology.orchestra.run.smithi016.stdout: "client_max_retries_on_remount_failure": "5", 2023-09-23T19:30:46.732 INFO:teuthology.orchestra.run.smithi016.stdout: "client_mds_namespace": "", 2023-09-23T19:30:46.732 INFO:teuthology.orchestra.run.smithi016.stdout: "client_metadata": "", 2023-09-23T19:30:46.732 INFO:teuthology.orchestra.run.smithi016.stdout: "client_mount_gid": "-1", 2023-09-23T19:30:46.732 INFO:teuthology.orchestra.run.smithi016.stdout: "client_mount_timeout": "300", 2023-09-23T19:30:46.732 INFO:teuthology.orchestra.run.smithi016.stdout: "client_mount_uid": "-1", 2023-09-23T19:30:46.732 INFO:teuthology.orchestra.run.smithi016.stdout: "client_mountpoint": "/", 2023-09-23T19:30:46.732 INFO:teuthology.orchestra.run.smithi016.stdout: "client_notify_timeout": "10", 2023-09-23T19:30:46.732 INFO:teuthology.orchestra.run.smithi016.stdout: "client_oc": "true", 2023-09-23T19:30:46.732 INFO:teuthology.orchestra.run.smithi016.stdout: "client_oc_max_dirty": "104857600", 2023-09-23T19:30:46.732 INFO:teuthology.orchestra.run.smithi016.stdout: "client_oc_max_dirty_age": "5.000000", 2023-09-23T19:30:46.732 INFO:teuthology.orchestra.run.smithi016.stdout: "client_oc_max_objects": "1000", 2023-09-23T19:30:46.733 INFO:teuthology.orchestra.run.smithi016.stdout: "client_oc_size": "209715200", 2023-09-23T19:30:46.733 INFO:teuthology.orchestra.run.smithi016.stdout: "client_oc_target_dirty": "8388608", 2023-09-23T19:30:46.733 INFO:teuthology.orchestra.run.smithi016.stdout: "client_permissions": "true", 2023-09-23T19:30:46.733 INFO:teuthology.orchestra.run.smithi016.stdout: "client_quota": "true", 2023-09-23T19:30:46.733 INFO:teuthology.orchestra.run.smithi016.stdout: "client_quota_df": "true", 2023-09-23T19:30:46.733 INFO:teuthology.orchestra.run.smithi016.stdout: "client_readahead_max_bytes": "0", 2023-09-23T19:30:46.733 INFO:teuthology.orchestra.run.smithi016.stdout: "client_readahead_max_periods": "4", 2023-09-23T19:30:46.733 INFO:teuthology.orchestra.run.smithi016.stdout: "client_readahead_min": "131072", 2023-09-23T19:30:46.733 INFO:teuthology.orchestra.run.smithi016.stdout: "client_reconnect_stale": "false", 2023-09-23T19:30:46.733 INFO:teuthology.orchestra.run.smithi016.stdout: "client_shutdown_timeout": "30", 2023-09-23T19:30:46.733 INFO:teuthology.orchestra.run.smithi016.stdout: "client_snapdir": ".snap", 2023-09-23T19:30:46.733 INFO:teuthology.orchestra.run.smithi016.stdout: "client_tick_interval": "1", 2023-09-23T19:30:46.734 INFO:teuthology.orchestra.run.smithi016.stdout: "client_trace": "", 2023-09-23T19:30:46.734 INFO:teuthology.orchestra.run.smithi016.stdout: "client_try_dentry_invalidate": "false", 2023-09-23T19:30:46.734 INFO:teuthology.orchestra.run.smithi016.stdout: "client_use_faked_inos": "false", 2023-09-23T19:30:46.734 INFO:teuthology.orchestra.run.smithi016.stdout: "client_use_random_mds": "false", 2023-09-23T19:30:46.734 INFO:teuthology.orchestra.run.smithi016.stdout: "clog_to_graylog": "false", 2023-09-23T19:30:46.734 INFO:teuthology.orchestra.run.smithi016.stdout: "clog_to_graylog_host": "127.0.0.1", 2023-09-23T19:30:46.734 INFO:teuthology.orchestra.run.smithi016.stdout: "clog_to_graylog_port": "12201", 2023-09-23T19:30:46.734 INFO:teuthology.orchestra.run.smithi016.stdout: "clog_to_monitors": "default=true", 2023-09-23T19:30:46.734 INFO:teuthology.orchestra.run.smithi016.stdout: "clog_to_syslog": "false", 2023-09-23T19:30:46.734 INFO:teuthology.orchestra.run.smithi016.stdout: "clog_to_syslog_facility": "default=daemon audit=local0", 2023-09-23T19:30:46.734 INFO:teuthology.orchestra.run.smithi016.stdout: "clog_to_syslog_level": "info", 2023-09-23T19:30:46.734 INFO:teuthology.orchestra.run.smithi016.stdout: "cluster_addr": "-", 2023-09-23T19:30:46.735 INFO:teuthology.orchestra.run.smithi016.stdout: "cluster_network": "", 2023-09-23T19:30:46.735 INFO:teuthology.orchestra.run.smithi016.stdout: "cluster_network_interface": "", 2023-09-23T19:30:46.735 INFO:teuthology.orchestra.run.smithi016.stdout: "compressor_zlib_isal": "false", 2023-09-23T19:30:46.735 INFO:teuthology.orchestra.run.smithi016.stdout: "compressor_zlib_level": "5", 2023-09-23T19:30:46.735 INFO:teuthology.orchestra.run.smithi016.stdout: "compressor_zlib_winsize": "-15", 2023-09-23T19:30:46.735 INFO:teuthology.orchestra.run.smithi016.stdout: "compressor_zstd_level": "1", 2023-09-23T19:30:46.735 INFO:teuthology.orchestra.run.smithi016.stdout: "container_image": "quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d", 2023-09-23T19:30:46.735 INFO:teuthology.orchestra.run.smithi016.stdout: "crash_dir": "/var/lib/ceph/crash", 2023-09-23T19:30:46.735 INFO:teuthology.orchestra.run.smithi016.stdout: "crimson_alien_op_num_threads": "6", 2023-09-23T19:30:46.735 INFO:teuthology.orchestra.run.smithi016.stdout: "crimson_alien_thread_cpu_cores": "", 2023-09-23T19:30:46.735 INFO:teuthology.orchestra.run.smithi016.stdout: "crimson_osd_obc_lru_size": "10", 2023-09-23T19:30:46.735 INFO:teuthology.orchestra.run.smithi016.stdout: "crimson_osd_scheduler_concurrency": "0", 2023-09-23T19:30:46.736 INFO:teuthology.orchestra.run.smithi016.stdout: "crimson_seastar_smp": "1", 2023-09-23T19:30:46.736 INFO:teuthology.orchestra.run.smithi016.stdout: "crush_location": "", 2023-09-23T19:30:46.736 INFO:teuthology.orchestra.run.smithi016.stdout: "crush_location_hook": "", 2023-09-23T19:30:46.736 INFO:teuthology.orchestra.run.smithi016.stdout: "crush_location_hook_timeout": "10", 2023-09-23T19:30:46.736 INFO:teuthology.orchestra.run.smithi016.stdout: "daemonize": "false", 2023-09-23T19:30:46.736 INFO:teuthology.orchestra.run.smithi016.stdout: "daos_pool": "tank", 2023-09-23T19:30:46.736 INFO:teuthology.orchestra.run.smithi016.stdout: "dbstore_config_uri": "file:/var/lib/ceph/radosgw/dbstore-config.db", 2023-09-23T19:30:46.736 INFO:teuthology.orchestra.run.smithi016.stdout: "dbstore_db_dir": "/var/lib/ceph/radosgw", 2023-09-23T19:30:46.736 INFO:teuthology.orchestra.run.smithi016.stdout: "dbstore_db_name_prefix": "dbstore", 2023-09-23T19:30:46.736 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_alienstore": "0/5", 2023-09-23T19:30:46.736 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_allow_any_pool_priority": "false", 2023-09-23T19:30:46.736 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_asok": "1/5", 2023-09-23T19:30:46.737 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_asok_assert_abort": "false", 2023-09-23T19:30:46.737 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_asserts_on_shutdown": "true", 2023-09-23T19:30:46.737 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_auth": "1/5", 2023-09-23T19:30:46.737 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_bdev": "1/3", 2023-09-23T19:30:46.737 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_bluefs": "1/5", 2023-09-23T19:30:46.737 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_bluestore": "1/5", 2023-09-23T19:30:46.737 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_buffer": "0/1", 2023-09-23T19:30:46.737 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_ceph_exporter": "1/5", 2023-09-23T19:30:46.737 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_cephfs_mirror": "0/5", 2023-09-23T19:30:46.737 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_cephsqlite": "0/5", 2023-09-23T19:30:46.737 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_client": "0/5", 2023-09-23T19:30:46.738 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_compressor": "1/5", 2023-09-23T19:30:46.738 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_context": "0/1", 2023-09-23T19:30:46.738 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_crush": "1/1", 2023-09-23T19:30:46.738 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_crypto": "1/5", 2023-09-23T19:30:46.738 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_cyanstore": "0/5", 2023-09-23T19:30:46.738 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_deliberately_leak_memory": "false", 2023-09-23T19:30:46.738 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_disable_randomized_ping": "false", 2023-09-23T19:30:46.738 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_dpdk": "1/5", 2023-09-23T19:30:46.738 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_eventtrace": "1/5", 2023-09-23T19:30:46.738 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_filer": "0/1", 2023-09-23T19:30:46.738 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_filestore": "1/3", 2023-09-23T19:30:46.738 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_finisher": "1/1", 2023-09-23T19:30:46.739 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_fuse": "1/5", 2023-09-23T19:30:46.739 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_heartbeat_testing_span": "0", 2023-09-23T19:30:46.739 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_heartbeatmap": "1/5", 2023-09-23T19:30:46.739 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_immutable_obj_cache": "0/5", 2023-09-23T19:30:46.739 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_javaclient": "1/5", 2023-09-23T19:30:46.739 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_journal": "1/3", 2023-09-23T19:30:46.739 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_journaler": "0/5", 2023-09-23T19:30:46.739 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_kstore": "1/5", 2023-09-23T19:30:46.739 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_leveldb": "4/5", 2023-09-23T19:30:46.739 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_lockdep": "0/1", 2023-09-23T19:30:46.739 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_mclock": "1/5", 2023-09-23T19:30:46.740 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_mds": "1/5", 2023-09-23T19:30:46.740 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_mds_balancer": "1/5", 2023-09-23T19:30:46.740 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_mds_locker": "1/5", 2023-09-23T19:30:46.740 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_mds_log": "1/5", 2023-09-23T19:30:46.740 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_mds_log_expire": "1/5", 2023-09-23T19:30:46.740 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_mds_migrator": "1/5", 2023-09-23T19:30:46.740 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_memstore": "1/5", 2023-09-23T19:30:46.740 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_mgr": "2/5", 2023-09-23T19:30:46.740 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_mgrc": "1/5", 2023-09-23T19:30:46.741 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_mon": "20/20", 2023-09-23T19:30:46.741 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_monc": "0/10", 2023-09-23T19:30:46.741 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_ms": "1/1", 2023-09-23T19:30:46.741 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_none": "0/5", 2023-09-23T19:30:46.741 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_objclass": "0/5", 2023-09-23T19:30:46.741 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_objectcacher": "0/5", 2023-09-23T19:30:46.741 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_objecter": "0/1", 2023-09-23T19:30:46.741 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_optracker": "0/5", 2023-09-23T19:30:46.741 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_osd": "1/5", 2023-09-23T19:30:46.742 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_paxos": "20/20", 2023-09-23T19:30:46.742 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_perfcounter": "1/5", 2023-09-23T19:30:46.742 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_prioritycache": "1/5", 2023-09-23T19:30:46.742 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_rados": "0/5", 2023-09-23T19:30:46.742 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_rbd": "0/5", 2023-09-23T19:30:46.742 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_rbd_mirror": "0/5", 2023-09-23T19:30:46.742 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_rbd_pwl": "0/5", 2023-09-23T19:30:46.742 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_rbd_replay": "0/5", 2023-09-23T19:30:46.742 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_refs": "0/0", 2023-09-23T19:30:46.743 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_reserver": "1/1", 2023-09-23T19:30:46.743 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_rgw": "1/5", 2023-09-23T19:30:46.743 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_rgw_access": "1/5", 2023-09-23T19:30:46.743 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_rgw_datacache": "1/5", 2023-09-23T19:30:46.743 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_rgw_dbstore": "1/5", 2023-09-23T19:30:46.743 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_rgw_flight": "1/5", 2023-09-23T19:30:46.743 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_rgw_sync": "1/5", 2023-09-23T19:30:46.743 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_rocksdb": "4/5", 2023-09-23T19:30:46.743 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_seastore": "0/5", 2023-09-23T19:30:46.743 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_seastore_backref": "0/5", 2023-09-23T19:30:46.744 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_seastore_cache": "0/5", 2023-09-23T19:30:46.744 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_seastore_cleaner": "0/5", 2023-09-23T19:30:46.744 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_seastore_device": "0/5", 2023-09-23T19:30:46.744 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_seastore_epm": "0/5", 2023-09-23T19:30:46.744 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_seastore_fixedkv_tree": "0/5", 2023-09-23T19:30:46.744 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_seastore_journal": "0/5", 2023-09-23T19:30:46.744 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_seastore_lba": "0/5", 2023-09-23T19:30:46.744 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_seastore_odata": "0/5", 2023-09-23T19:30:46.744 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_seastore_omap": "0/5", 2023-09-23T19:30:46.744 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_seastore_onode": "0/5", 2023-09-23T19:30:46.745 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_seastore_t": "0/5", 2023-09-23T19:30:46.745 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_seastore_tm": "0/5", 2023-09-23T19:30:46.745 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_striper": "0/1", 2023-09-23T19:30:46.745 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_test": "0/5", 2023-09-23T19:30:46.745 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_throttle": "1/1", 2023-09-23T19:30:46.745 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_timer": "0/1", 2023-09-23T19:30:46.745 INFO:teuthology.orchestra.run.smithi016.stdout: "debug_tp": "0/5", 2023-09-23T19:30:46.745 INFO:teuthology.orchestra.run.smithi016.stdout: "device_failure_prediction_mode": "none", 2023-09-23T19:30:46.745 INFO:teuthology.orchestra.run.smithi016.stdout: "enable_experimental_unrecoverable_data_corrupting_features": "", 2023-09-23T19:30:46.746 INFO:teuthology.orchestra.run.smithi016.stdout: "erasure_code_dir": "/usr/lib64/ceph/erasure-code", 2023-09-23T19:30:46.746 INFO:teuthology.orchestra.run.smithi016.stdout: "err_to_graylog": "false", 2023-09-23T19:30:46.746 INFO:teuthology.orchestra.run.smithi016.stdout: "err_to_journald": "false", 2023-09-23T19:30:46.746 INFO:teuthology.orchestra.run.smithi016.stdout: "err_to_stderr": "true", 2023-09-23T19:30:46.746 INFO:teuthology.orchestra.run.smithi016.stdout: "err_to_syslog": "false", 2023-09-23T19:30:46.746 INFO:teuthology.orchestra.run.smithi016.stdout: "event_tracing": "false", 2023-09-23T19:30:46.746 INFO:teuthology.orchestra.run.smithi016.stdout: "exporter_addr": "0.0.0.0", 2023-09-23T19:30:46.746 INFO:teuthology.orchestra.run.smithi016.stdout: "exporter_http_port": "9926", 2023-09-23T19:30:46.746 INFO:teuthology.orchestra.run.smithi016.stdout: "exporter_prio_limit": "5", 2023-09-23T19:30:46.747 INFO:teuthology.orchestra.run.smithi016.stdout: "exporter_sock_dir": "/var/run/ceph/", 2023-09-23T19:30:46.747 INFO:teuthology.orchestra.run.smithi016.stdout: "exporter_sort_metrics": "true", 2023-09-23T19:30:46.747 INFO:teuthology.orchestra.run.smithi016.stdout: "exporter_stats_period": "5", 2023-09-23T19:30:46.747 INFO:teuthology.orchestra.run.smithi016.stdout: "fake_statfs_for_testing": "0", 2023-09-23T19:30:46.747 INFO:teuthology.orchestra.run.smithi016.stdout: "fatal_signal_handlers": "true", 2023-09-23T19:30:46.747 INFO:teuthology.orchestra.run.smithi016.stdout: "filer_max_purge_ops": "10", 2023-09-23T19:30:46.747 INFO:teuthology.orchestra.run.smithi016.stdout: "filer_max_truncate_ops": "128", 2023-09-23T19:30:46.747 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_apply_finisher_threads": "1", 2023-09-23T19:30:46.747 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_blackhole": "false", 2023-09-23T19:30:46.747 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_btrfs_clone_range": "true", 2023-09-23T19:30:46.748 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_btrfs_snap": "true", 2023-09-23T19:30:46.748 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_caller_concurrency": "10", 2023-09-23T19:30:46.748 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_collect_device_partition_information": "true", 2023-09-23T19:30:46.748 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_commit_timeout": "600.000000", 2023-09-23T19:30:46.748 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_debug_inject_read_err": "false", 2023-09-23T19:30:46.748 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_debug_omap_check": "false", 2023-09-23T19:30:46.748 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_debug_random_read_err": "0.000000", 2023-09-23T19:30:46.748 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_debug_verify_split": "false", 2023-09-23T19:30:46.749 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_dump_file": "", 2023-09-23T19:30:46.749 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_expected_throughput_bytes": "209715200.000000", 2023-09-23T19:30:46.749 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_expected_throughput_ops": "200.000000", 2023-09-23T19:30:46.749 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_fadvise": "true", 2023-09-23T19:30:46.749 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_fail_eio": "true", 2023-09-23T19:30:46.749 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_fd_cache_shards": "16", 2023-09-23T19:30:46.749 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_fd_cache_size": "128", 2023-09-23T19:30:46.749 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_fiemap": "false", 2023-09-23T19:30:46.750 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_fiemap_threshold": "4096", 2023-09-23T19:30:46.750 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_fsync_flushes_journal_data": "false", 2023-09-23T19:30:46.750 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_index_retry_probability": "0.000000", 2023-09-23T19:30:46.750 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_inject_stall": "0", 2023-09-23T19:30:46.750 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_journal_parallel": "false", 2023-09-23T19:30:46.750 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_journal_trailing": "false", 2023-09-23T19:30:46.750 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_journal_writeahead": "false", 2023-09-23T19:30:46.750 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_kill_at": "0", 2023-09-23T19:30:46.751 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_max_alloc_hint_size": "1048576", 2023-09-23T19:30:46.751 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_max_inline_xattr_size": "0", 2023-09-23T19:30:46.751 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_max_inline_xattr_size_btrfs": "2048", 2023-09-23T19:30:46.751 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_max_inline_xattr_size_other": "512", 2023-09-23T19:30:46.751 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_max_inline_xattr_size_xfs": "65536", 2023-09-23T19:30:46.751 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_max_inline_xattrs": "0", 2023-09-23T19:30:46.751 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_max_inline_xattrs_btrfs": "10", 2023-09-23T19:30:46.751 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_max_inline_xattrs_other": "2", 2023-09-23T19:30:46.752 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_max_inline_xattrs_xfs": "10", 2023-09-23T19:30:46.752 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_max_sync_interval": "5.000000", 2023-09-23T19:30:46.752 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_max_xattr_value_size": "0", 2023-09-23T19:30:46.752 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_max_xattr_value_size_btrfs": "65536", 2023-09-23T19:30:46.752 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_max_xattr_value_size_other": "1024", 2023-09-23T19:30:46.752 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_max_xattr_value_size_xfs": "65536", 2023-09-23T19:30:46.752 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_merge_threshold": "-10", 2023-09-23T19:30:46.752 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_min_sync_interval": "0.010000", 2023-09-23T19:30:46.752 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_odsync_write": "false", 2023-09-23T19:30:46.752 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_omap_backend": "rocksdb", 2023-09-23T19:30:46.753 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_omap_backend_path": "", 2023-09-23T19:30:46.753 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_omap_header_cache_size": "1024", 2023-09-23T19:30:46.753 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_ondisk_finisher_threads": "1", 2023-09-23T19:30:46.753 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_op_thread_suicide_timeout": "180", 2023-09-23T19:30:46.753 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_op_thread_timeout": "60", 2023-09-23T19:30:46.753 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_op_threads": "2", 2023-09-23T19:30:46.753 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_punch_hole": "false", 2023-09-23T19:30:46.753 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_queue_high_delay_multiple": "0.000000", 2023-09-23T19:30:46.753 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_queue_high_delay_multiple_bytes": "0.000000", 2023-09-23T19:30:46.753 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_queue_high_delay_multiple_ops": "0.000000", 2023-09-23T19:30:46.754 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_queue_high_threshhold": "0.900000", 2023-09-23T19:30:46.754 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_queue_low_threshhold": "0.300000", 2023-09-23T19:30:46.754 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_queue_max_bytes": "104857600", 2023-09-23T19:30:46.754 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_queue_max_delay_multiple": "0.000000", 2023-09-23T19:30:46.754 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_queue_max_delay_multiple_bytes": "0.000000", 2023-09-23T19:30:46.754 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_queue_max_delay_multiple_ops": "0.000000", 2023-09-23T19:30:46.754 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_queue_max_ops": "50", 2023-09-23T19:30:46.754 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_rocksdb_options": "max_background_jobs=10,compaction_readahead_size=2097152,compression=kNoCompression", 2023-09-23T19:30:46.754 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_seek_data_hole": "false", 2023-09-23T19:30:46.754 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_sloppy_crc": "false", 2023-09-23T19:30:46.755 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_sloppy_crc_block_size": "65536", 2023-09-23T19:30:46.755 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_splice": "false", 2023-09-23T19:30:46.755 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_split_multiple": "2", 2023-09-23T19:30:46.755 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_split_rand_factor": "20", 2023-09-23T19:30:46.755 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_update_to": "1000", 2023-09-23T19:30:46.755 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_wbthrottle_btrfs_bytes_hard_limit": "419430400", 2023-09-23T19:30:46.755 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_wbthrottle_btrfs_bytes_start_flusher": "41943040", 2023-09-23T19:30:46.755 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_wbthrottle_btrfs_inodes_hard_limit": "5000", 2023-09-23T19:30:46.755 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_wbthrottle_btrfs_inodes_start_flusher": "500", 2023-09-23T19:30:46.755 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_wbthrottle_btrfs_ios_hard_limit": "5000", 2023-09-23T19:30:46.756 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_wbthrottle_btrfs_ios_start_flusher": "500", 2023-09-23T19:30:46.756 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_wbthrottle_enable": "true", 2023-09-23T19:30:46.756 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_wbthrottle_xfs_bytes_hard_limit": "419430400", 2023-09-23T19:30:46.756 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_wbthrottle_xfs_bytes_start_flusher": "41943040", 2023-09-23T19:30:46.756 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_wbthrottle_xfs_inodes_hard_limit": "5000", 2023-09-23T19:30:46.756 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_wbthrottle_xfs_inodes_start_flusher": "500", 2023-09-23T19:30:46.756 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_wbthrottle_xfs_ios_hard_limit": "5000", 2023-09-23T19:30:46.756 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_wbthrottle_xfs_ios_start_flusher": "500", 2023-09-23T19:30:46.756 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_xfs_extsize": "false", 2023-09-23T19:30:46.756 INFO:teuthology.orchestra.run.smithi016.stdout: "filestore_zfs_snap": "false", 2023-09-23T19:30:46.757 INFO:teuthology.orchestra.run.smithi016.stdout: "fio_dir": "/tmp/fio", 2023-09-23T19:30:46.757 INFO:teuthology.orchestra.run.smithi016.stdout: "fsid": "95ce9eb6-5a42-11ee-8db4-212e2dc638e7", 2023-09-23T19:30:46.757 INFO:teuthology.orchestra.run.smithi016.stdout: "fuse_allow_other": "true", 2023-09-23T19:30:46.757 INFO:teuthology.orchestra.run.smithi016.stdout: "fuse_atomic_o_trunc": "true", 2023-09-23T19:30:46.757 INFO:teuthology.orchestra.run.smithi016.stdout: "fuse_big_writes": "true", 2023-09-23T19:30:46.757 INFO:teuthology.orchestra.run.smithi016.stdout: "fuse_debug": "false", 2023-09-23T19:30:46.757 INFO:teuthology.orchestra.run.smithi016.stdout: "fuse_default_permissions": "false", 2023-09-23T19:30:46.757 INFO:teuthology.orchestra.run.smithi016.stdout: "fuse_disable_pagecache": "false", 2023-09-23T19:30:46.757 INFO:teuthology.orchestra.run.smithi016.stdout: "fuse_max_write": "0", 2023-09-23T19:30:46.758 INFO:teuthology.orchestra.run.smithi016.stdout: "fuse_multithreaded": "true", 2023-09-23T19:30:46.758 INFO:teuthology.orchestra.run.smithi016.stdout: "fuse_require_active_mds": "true", 2023-09-23T19:30:46.758 INFO:teuthology.orchestra.run.smithi016.stdout: "fuse_set_user_groups": "true", 2023-09-23T19:30:46.758 INFO:teuthology.orchestra.run.smithi016.stdout: "fuse_splice_move": "true", 2023-09-23T19:30:46.758 INFO:teuthology.orchestra.run.smithi016.stdout: "fuse_splice_read": "true", 2023-09-23T19:30:46.758 INFO:teuthology.orchestra.run.smithi016.stdout: "fuse_splice_write": "true", 2023-09-23T19:30:46.758 INFO:teuthology.orchestra.run.smithi016.stdout: "fuse_syncfs_on_mksnap": "true", 2023-09-23T19:30:46.758 INFO:teuthology.orchestra.run.smithi016.stdout: "fuse_use_invalidate_cb": "true", 2023-09-23T19:30:46.758 INFO:teuthology.orchestra.run.smithi016.stdout: "gss_ktab_client_file": "/var/lib/ceph/mon.a/gss_client_mon.a.ktab", 2023-09-23T19:30:46.759 INFO:teuthology.orchestra.run.smithi016.stdout: "gss_target_name": "ceph", 2023-09-23T19:30:46.759 INFO:teuthology.orchestra.run.smithi016.stdout: "heartbeat_file": "", 2023-09-23T19:30:46.759 INFO:teuthology.orchestra.run.smithi016.stdout: "heartbeat_inject_failure": "0", 2023-09-23T19:30:46.759 INFO:teuthology.orchestra.run.smithi016.stdout: "heartbeat_interval": "5", 2023-09-23T19:30:46.759 INFO:teuthology.orchestra.run.smithi016.stdout: "host": "smithi016", 2023-09-23T19:30:46.759 INFO:teuthology.orchestra.run.smithi016.stdout: "immutable_object_cache_client_dedicated_thread_num": "2", 2023-09-23T19:30:46.759 INFO:teuthology.orchestra.run.smithi016.stdout: "immutable_object_cache_max_inflight_ops": "128", 2023-09-23T19:30:46.759 INFO:teuthology.orchestra.run.smithi016.stdout: "immutable_object_cache_max_size": "1073741824", 2023-09-23T19:30:46.759 INFO:teuthology.orchestra.run.smithi016.stdout: "immutable_object_cache_path": "/tmp/ceph_immutable_object_cache", 2023-09-23T19:30:46.760 INFO:teuthology.orchestra.run.smithi016.stdout: "immutable_object_cache_qos_bps_burst": "0", 2023-09-23T19:30:46.760 INFO:teuthology.orchestra.run.smithi016.stdout: "immutable_object_cache_qos_bps_burst_seconds": "1", 2023-09-23T19:30:46.760 INFO:teuthology.orchestra.run.smithi016.stdout: "immutable_object_cache_qos_bps_limit": "0", 2023-09-23T19:30:46.760 INFO:teuthology.orchestra.run.smithi016.stdout: "immutable_object_cache_qos_iops_burst": "0", 2023-09-23T19:30:46.760 INFO:teuthology.orchestra.run.smithi016.stdout: "immutable_object_cache_qos_iops_burst_seconds": "1", 2023-09-23T19:30:46.760 INFO:teuthology.orchestra.run.smithi016.stdout: "immutable_object_cache_qos_iops_limit": "0", 2023-09-23T19:30:46.760 INFO:teuthology.orchestra.run.smithi016.stdout: "immutable_object_cache_qos_schedule_tick_min": "50", 2023-09-23T19:30:46.760 INFO:teuthology.orchestra.run.smithi016.stdout: "immutable_object_cache_sock": "/var/run/ceph/immutable_object_cache_sock", 2023-09-23T19:30:46.760 INFO:teuthology.orchestra.run.smithi016.stdout: "immutable_object_cache_watermark": "0.900000", 2023-09-23T19:30:46.760 INFO:teuthology.orchestra.run.smithi016.stdout: "inject_early_sigterm": "false", 2023-09-23T19:30:46.761 INFO:teuthology.orchestra.run.smithi016.stdout: "jaeger_agent_port": "6799", 2023-09-23T19:30:46.761 INFO:teuthology.orchestra.run.smithi016.stdout: "jaeger_tracing_enable": "false", 2023-09-23T19:30:46.761 INFO:teuthology.orchestra.run.smithi016.stdout: "journal_aio": "true", 2023-09-23T19:30:46.761 INFO:teuthology.orchestra.run.smithi016.stdout: "journal_align_min_size": "65536", 2023-09-23T19:30:46.761 INFO:teuthology.orchestra.run.smithi016.stdout: "journal_block_align": "true", 2023-09-23T19:30:46.761 INFO:teuthology.orchestra.run.smithi016.stdout: "journal_block_size": "4096", 2023-09-23T19:30:46.761 INFO:teuthology.orchestra.run.smithi016.stdout: "journal_dio": "true", 2023-09-23T19:30:46.761 INFO:teuthology.orchestra.run.smithi016.stdout: "journal_discard": "false", 2023-09-23T19:30:46.762 INFO:teuthology.orchestra.run.smithi016.stdout: "journal_force_aio": "false", 2023-09-23T19:30:46.762 INFO:teuthology.orchestra.run.smithi016.stdout: "journal_ignore_corruption": "false", 2023-09-23T19:30:46.762 INFO:teuthology.orchestra.run.smithi016.stdout: "journal_max_write_bytes": "10485760", 2023-09-23T19:30:46.762 INFO:teuthology.orchestra.run.smithi016.stdout: "journal_max_write_entries": "100", 2023-09-23T19:30:46.762 INFO:teuthology.orchestra.run.smithi016.stdout: "journal_replay_from": "0", 2023-09-23T19:30:46.762 INFO:teuthology.orchestra.run.smithi016.stdout: "journal_throttle_high_multiple": "0.000000", 2023-09-23T19:30:46.762 INFO:teuthology.orchestra.run.smithi016.stdout: "journal_throttle_high_threshhold": "0.900000", 2023-09-23T19:30:46.762 INFO:teuthology.orchestra.run.smithi016.stdout: "journal_throttle_low_threshhold": "0.600000", 2023-09-23T19:30:46.763 INFO:teuthology.orchestra.run.smithi016.stdout: "journal_throttle_max_multiple": "0.000000", 2023-09-23T19:30:46.763 INFO:teuthology.orchestra.run.smithi016.stdout: "journal_write_header_frequency": "0", 2023-09-23T19:30:46.763 INFO:teuthology.orchestra.run.smithi016.stdout: "journal_zero_on_create": "false", 2023-09-23T19:30:46.763 INFO:teuthology.orchestra.run.smithi016.stdout: "journaler_prefetch_periods": "10", 2023-09-23T19:30:46.763 INFO:teuthology.orchestra.run.smithi016.stdout: "journaler_prezero_periods": "5", 2023-09-23T19:30:46.763 INFO:teuthology.orchestra.run.smithi016.stdout: "journaler_write_head_interval": "15", 2023-09-23T19:30:46.763 INFO:teuthology.orchestra.run.smithi016.stdout: "key": "", 2023-09-23T19:30:46.763 INFO:teuthology.orchestra.run.smithi016.stdout: "keyfile": "", 2023-09-23T19:30:46.763 INFO:teuthology.orchestra.run.smithi016.stdout: "keyring": "/var/lib/ceph/mon/ceph-a/keyring", 2023-09-23T19:30:46.763 INFO:teuthology.orchestra.run.smithi016.stdout: "kstore_backend": "rocksdb", 2023-09-23T19:30:46.763 INFO:teuthology.orchestra.run.smithi016.stdout: "kstore_default_stripe_size": "65536", 2023-09-23T19:30:46.763 INFO:teuthology.orchestra.run.smithi016.stdout: "kstore_fsck_on_mount": "false", 2023-09-23T19:30:46.764 INFO:teuthology.orchestra.run.smithi016.stdout: "kstore_fsck_on_mount_deep": "true", 2023-09-23T19:30:46.764 INFO:teuthology.orchestra.run.smithi016.stdout: "kstore_max_bytes": "67108864", 2023-09-23T19:30:46.764 INFO:teuthology.orchestra.run.smithi016.stdout: "kstore_max_ops": "512", 2023-09-23T19:30:46.764 INFO:teuthology.orchestra.run.smithi016.stdout: "kstore_nid_prealloc": "1024", 2023-09-23T19:30:46.764 INFO:teuthology.orchestra.run.smithi016.stdout: "kstore_onode_map_size": "1024", 2023-09-23T19:30:46.764 INFO:teuthology.orchestra.run.smithi016.stdout: "kstore_rocksdb_options": "compression=kNoCompression", 2023-09-23T19:30:46.764 INFO:teuthology.orchestra.run.smithi016.stdout: "kstore_sync_submit_transaction": "false", 2023-09-23T19:30:46.764 INFO:teuthology.orchestra.run.smithi016.stdout: "kstore_sync_transaction": "false", 2023-09-23T19:30:46.764 INFO:teuthology.orchestra.run.smithi016.stdout: "leveldb_block_size": "65536", 2023-09-23T19:30:46.764 INFO:teuthology.orchestra.run.smithi016.stdout: "leveldb_bloom_size": "0", 2023-09-23T19:30:46.764 INFO:teuthology.orchestra.run.smithi016.stdout: "leveldb_cache_size": "536870912", 2023-09-23T19:30:46.764 INFO:teuthology.orchestra.run.smithi016.stdout: "leveldb_compact_on_mount": "false", 2023-09-23T19:30:46.765 INFO:teuthology.orchestra.run.smithi016.stdout: "leveldb_compression": "false", 2023-09-23T19:30:46.765 INFO:teuthology.orchestra.run.smithi016.stdout: "leveldb_log": "", 2023-09-23T19:30:46.765 INFO:teuthology.orchestra.run.smithi016.stdout: "leveldb_log_to_ceph_log": "true", 2023-09-23T19:30:46.765 INFO:teuthology.orchestra.run.smithi016.stdout: "leveldb_max_open_files": "0", 2023-09-23T19:30:46.765 INFO:teuthology.orchestra.run.smithi016.stdout: "leveldb_paranoid": "false", 2023-09-23T19:30:46.765 INFO:teuthology.orchestra.run.smithi016.stdout: "leveldb_write_buffer_size": "33554432", 2023-09-23T19:30:46.765 INFO:teuthology.orchestra.run.smithi016.stdout: "librados_thread_count": "2", 2023-09-23T19:30:46.765 INFO:teuthology.orchestra.run.smithi016.stdout: "lockdep": "false", 2023-09-23T19:30:46.765 INFO:teuthology.orchestra.run.smithi016.stdout: "lockdep_force_backtrace": "false", 2023-09-23T19:30:46.765 INFO:teuthology.orchestra.run.smithi016.stdout: "log_coarse_timestamps": "true", 2023-09-23T19:30:46.765 INFO:teuthology.orchestra.run.smithi016.stdout: "log_file": "/var/log/ceph/ceph-mon.a.log", 2023-09-23T19:30:46.765 INFO:teuthology.orchestra.run.smithi016.stdout: "log_flush_on_exit": "false", 2023-09-23T19:30:46.766 INFO:teuthology.orchestra.run.smithi016.stdout: "log_graylog_host": "127.0.0.1", 2023-09-23T19:30:46.766 INFO:teuthology.orchestra.run.smithi016.stdout: "log_graylog_port": "12201", 2023-09-23T19:30:46.766 INFO:teuthology.orchestra.run.smithi016.stdout: "log_max_new": "1000", 2023-09-23T19:30:46.766 INFO:teuthology.orchestra.run.smithi016.stdout: "log_max_recent": "10000", 2023-09-23T19:30:46.766 INFO:teuthology.orchestra.run.smithi016.stdout: "log_stderr_prefix": "debug ", 2023-09-23T19:30:46.766 INFO:teuthology.orchestra.run.smithi016.stdout: "log_stop_at_utilization": "0.970000", 2023-09-23T19:30:46.766 INFO:teuthology.orchestra.run.smithi016.stdout: "log_to_file": "true", 2023-09-23T19:30:46.766 INFO:teuthology.orchestra.run.smithi016.stdout: "log_to_graylog": "false", 2023-09-23T19:30:46.766 INFO:teuthology.orchestra.run.smithi016.stdout: "log_to_journald": "false", 2023-09-23T19:30:46.766 INFO:teuthology.orchestra.run.smithi016.stdout: "log_to_stderr": "false", 2023-09-23T19:30:46.767 INFO:teuthology.orchestra.run.smithi016.stdout: "log_to_syslog": "false", 2023-09-23T19:30:46.767 INFO:teuthology.orchestra.run.smithi016.stdout: "max_rotating_auth_attempts": "10", 2023-09-23T19:30:46.767 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_abort_on_newly_corrupt_dentry": "true", 2023-09-23T19:30:46.767 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_action_on_write_error": "1", 2023-09-23T19:30:46.767 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_alternate_name_max": "8192", 2023-09-23T19:30:46.767 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_asio_thread_count": "2", 2023-09-23T19:30:46.767 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_bal_export_pin": "true", 2023-09-23T19:30:46.767 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_bal_fragment_dirs": "true", 2023-09-23T19:30:46.767 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_bal_fragment_fast_factor": "1.500000", 2023-09-23T19:30:46.767 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_bal_fragment_interval": "5", 2023-09-23T19:30:46.767 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_bal_fragment_size_max": "100000", 2023-09-23T19:30:46.767 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_bal_idle_threshold": "0.000000", 2023-09-23T19:30:46.768 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_bal_interval": "10", 2023-09-23T19:30:46.768 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_bal_max": "-1", 2023-09-23T19:30:46.768 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_bal_max_until": "-1", 2023-09-23T19:30:46.768 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_bal_merge_size": "50", 2023-09-23T19:30:46.768 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_bal_midchunk": "0.300000", 2023-09-23T19:30:46.768 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_bal_min_rebalance": "0.100000", 2023-09-23T19:30:46.768 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_bal_min_start": "0.200000", 2023-09-23T19:30:46.768 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_bal_minchunk": "0.001000", 2023-09-23T19:30:46.768 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_bal_mode": "0", 2023-09-23T19:30:46.768 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_bal_need_max": "1.200000", 2023-09-23T19:30:46.768 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_bal_need_min": "0.800000", 2023-09-23T19:30:46.768 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_bal_replicate_threshold": "8000.000000", 2023-09-23T19:30:46.769 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_bal_sample_interval": "3.000000", 2023-09-23T19:30:46.769 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_bal_split_bits": "3", 2023-09-23T19:30:46.769 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_bal_split_rd": "25000.000000", 2023-09-23T19:30:46.769 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_bal_split_size": "10000", 2023-09-23T19:30:46.769 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_bal_split_wr": "10000.000000", 2023-09-23T19:30:46.769 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_bal_target_decay": "10.000000", 2023-09-23T19:30:46.769 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_bal_unreplicate_threshold": "0.000000", 2023-09-23T19:30:46.769 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_beacon_grace": "15.000000", 2023-09-23T19:30:46.769 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_beacon_interval": "4.000000", 2023-09-23T19:30:46.769 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_beacon_mon_down_grace": "60", 2023-09-23T19:30:46.769 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_cache_memory_limit": "4294967296", 2023-09-23T19:30:46.770 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_cache_mid": "0.700000", 2023-09-23T19:30:46.770 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_cache_release_free_interval": "10", 2023-09-23T19:30:46.770 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_cache_reservation": "0.050000", 2023-09-23T19:30:46.770 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_cache_trim_decay_rate": "1.000000", 2023-09-23T19:30:46.770 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_cache_trim_interval": "1", 2023-09-23T19:30:46.770 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_cache_trim_threshold": "262144", 2023-09-23T19:30:46.770 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_cap_acquisition_throttle_retry_request_timeout": "0.500000", 2023-09-23T19:30:46.770 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_cap_revoke_eviction_timeout": "0.000000", 2023-09-23T19:30:46.770 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_client_delegate_inos_pct": "50", 2023-09-23T19:30:46.770 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_client_prealloc_inos": "1000", 2023-09-23T19:30:46.770 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_client_writeable_range_max_inc_objs": "1024", 2023-09-23T19:30:46.770 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_connect_bootstrapping": "false", 2023-09-23T19:30:46.771 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_damage_table_max_entries": "10000", 2023-09-23T19:30:46.771 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_data": "/var/lib/ceph/mds/ceph-a", 2023-09-23T19:30:46.771 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_debug_auth_pins": "false", 2023-09-23T19:30:46.771 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_debug_frag": "false", 2023-09-23T19:30:46.771 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_debug_scatterstat": "false", 2023-09-23T19:30:46.771 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_debug_subtrees": "false", 2023-09-23T19:30:46.771 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_decay_halflife": "5.000000", 2023-09-23T19:30:46.771 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_default_dir_hash": "2", 2023-09-23T19:30:46.771 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_defer_session_stale": "true", 2023-09-23T19:30:46.771 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_deny_all_reconnect": "false", 2023-09-23T19:30:46.771 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_dir_keys_per_op": "16384", 2023-09-23T19:30:46.771 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_dir_max_commit_size": "10", 2023-09-23T19:30:46.772 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_dir_max_entries": "0", 2023-09-23T19:30:46.772 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_dir_prefetch": "true", 2023-09-23T19:30:46.772 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_dirstat_min_interval": "1.000000", 2023-09-23T19:30:46.772 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_dump_cache_after_rejoin": "false", 2023-09-23T19:30:46.772 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_dump_cache_on_map": "false", 2023-09-23T19:30:46.772 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_dump_cache_threshold_file": "0", 2023-09-23T19:30:46.772 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_dump_cache_threshold_formatter": "1073741824", 2023-09-23T19:30:46.772 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_early_reply": "true", 2023-09-23T19:30:46.772 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_enable_op_tracker": "true", 2023-09-23T19:30:46.772 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_enforce_unique_name": "true", 2023-09-23T19:30:46.772 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_export_ephemeral_distributed": "true", 2023-09-23T19:30:46.772 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_export_ephemeral_distributed_factor": "2.000000", 2023-09-23T19:30:46.773 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_export_ephemeral_random": "true", 2023-09-23T19:30:46.773 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_export_ephemeral_random_max": "0.010000", 2023-09-23T19:30:46.773 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_extraordinary_events_dump_interval": "0", 2023-09-23T19:30:46.773 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_forward_all_requests_to_auth": "false", 2023-09-23T19:30:46.773 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_freeze_tree_timeout": "30.000000", 2023-09-23T19:30:46.773 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_fscrypt_last_block_max_size": "4096", 2023-09-23T19:30:46.773 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_go_bad_corrupt_dentry": "true", 2023-09-23T19:30:46.773 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_hack_allow_loading_invalid_metadata": "false", 2023-09-23T19:30:46.774 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_health_cache_threshold": "1.500000", 2023-09-23T19:30:46.774 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_health_summarize_threshold": "10", 2023-09-23T19:30:46.774 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_heartbeat_grace": "15.000000", 2023-09-23T19:30:46.774 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_heartbeat_reset_grace": "1000", 2023-09-23T19:30:46.774 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_inject_health_dummy": "false", 2023-09-23T19:30:46.774 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_inject_journal_corrupt_dentry_first": "0.000000", 2023-09-23T19:30:46.774 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_inject_migrator_session_race": "false", 2023-09-23T19:30:46.774 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_inject_rename_corrupt_dentry_first": "0.000000", 2023-09-23T19:30:46.774 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_inject_skip_replaying_inotable": "false", 2023-09-23T19:30:46.774 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_inject_traceless_reply_probability": "0.000000", 2023-09-23T19:30:46.774 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_join_fs": "", 2023-09-23T19:30:46.775 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_journal_format": "1", 2023-09-23T19:30:46.775 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_kill_create_at": "0", 2023-09-23T19:30:46.775 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_kill_export_at": "0", 2023-09-23T19:30:46.775 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_kill_import_at": "0", 2023-09-23T19:30:46.775 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_kill_journal_at": "0", 2023-09-23T19:30:46.775 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_kill_journal_expire_at": "0", 2023-09-23T19:30:46.775 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_kill_journal_replay_at": "0", 2023-09-23T19:30:46.775 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_kill_link_at": "0", 2023-09-23T19:30:46.775 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_kill_mdstable_at": "0", 2023-09-23T19:30:46.775 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_kill_openc_at": "0", 2023-09-23T19:30:46.776 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_kill_rename_at": "0", 2023-09-23T19:30:46.776 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_kill_skip_replaying_inotable": "false", 2023-09-23T19:30:46.776 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_log_events_per_segment": "1024", 2023-09-23T19:30:46.776 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_log_max_events": "-1", 2023-09-23T19:30:46.776 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_log_max_segments": "128", 2023-09-23T19:30:46.776 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_log_pause": "false", 2023-09-23T19:30:46.776 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_log_segment_size": "0", 2023-09-23T19:30:46.776 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_log_skip_corrupt_events": "false", 2023-09-23T19:30:46.776 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_log_warn_factor": "2.000000", 2023-09-23T19:30:46.776 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_max_caps_per_client": "1048576", 2023-09-23T19:30:46.777 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_max_completed_flushes": "100000", 2023-09-23T19:30:46.777 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_max_completed_requests": "100000", 2023-09-23T19:30:46.777 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_max_export_size": "20971520", 2023-09-23T19:30:46.777 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_max_file_recover": "32", 2023-09-23T19:30:46.777 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_max_purge_files": "64", 2023-09-23T19:30:46.777 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_max_purge_ops": "8192", 2023-09-23T19:30:46.777 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_max_purge_ops_per_pg": "0.500000", 2023-09-23T19:30:46.777 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_max_scrub_ops_in_progress": "5", 2023-09-23T19:30:46.777 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_max_snaps_per_dir": "100", 2023-09-23T19:30:46.777 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_max_xattr_pairs_size": "65536", 2023-09-23T19:30:46.777 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_metrics_update_interval": "2", 2023-09-23T19:30:46.777 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_min_caps_per_client": "100", 2023-09-23T19:30:46.778 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_min_caps_working_set": "10000", 2023-09-23T19:30:46.778 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_mon_shutdown_timeout": "5.000000", 2023-09-23T19:30:46.778 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_numa_node": "-1", 2023-09-23T19:30:46.778 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_oft_prefetch_dirfrags": "false", 2023-09-23T19:30:46.778 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_op_complaint_time": "30.000000", 2023-09-23T19:30:46.778 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_op_history_duration": "600", 2023-09-23T19:30:46.778 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_op_history_size": "20", 2023-09-23T19:30:46.778 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_op_log_threshold": "5", 2023-09-23T19:30:46.778 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_ping_grace": "15", 2023-09-23T19:30:46.778 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_ping_interval": "5", 2023-09-23T19:30:46.779 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_purge_queue_busy_flush_period": "1.000000", 2023-09-23T19:30:46.779 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_recall_global_max_decay_threshold": "131072", 2023-09-23T19:30:46.779 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_recall_max_caps": "30000", 2023-09-23T19:30:46.779 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_recall_max_decay_rate": "1.500000", 2023-09-23T19:30:46.779 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_recall_max_decay_threshold": "131072", 2023-09-23T19:30:46.779 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_recall_warning_decay_rate": "60.000000", 2023-09-23T19:30:46.779 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_recall_warning_threshold": "262144", 2023-09-23T19:30:46.779 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_reconnect_timeout": "45.000000", 2023-09-23T19:30:46.779 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_replay_interval": "1.000000", 2023-09-23T19:30:46.779 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_replay_unsafe_with_closed_session": "false", 2023-09-23T19:30:46.780 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_request_load_average_decay_rate": "60.000000", 2023-09-23T19:30:46.780 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_root_ino_gid": "0", 2023-09-23T19:30:46.780 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_root_ino_uid": "0", 2023-09-23T19:30:46.780 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_scatter_nudge_interval": "5.000000", 2023-09-23T19:30:46.780 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_session_blocklist_on_evict": "true", 2023-09-23T19:30:46.780 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_session_blocklist_on_timeout": "true", 2023-09-23T19:30:46.780 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_session_cache_liveness_decay_rate": "300.000000", 2023-09-23T19:30:46.780 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_session_cache_liveness_magnitude": "10", 2023-09-23T19:30:46.780 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_session_cap_acquisition_decay_rate": "30.000000", 2023-09-23T19:30:46.781 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_session_cap_acquisition_throttle": "100000", 2023-09-23T19:30:46.781 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_session_max_caps_throttle_ratio": "1.100000", 2023-09-23T19:30:46.781 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_sessionmap_keys_per_op": "1024", 2023-09-23T19:30:46.781 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_shutdown_check": "0", 2023-09-23T19:30:46.781 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_skip_ino": "0", 2023-09-23T19:30:46.781 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_sleep_rank_change": "0.000000", 2023-09-23T19:30:46.781 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_snap_max_uid": "4294967294", 2023-09-23T19:30:46.781 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_snap_min_uid": "0", 2023-09-23T19:30:46.781 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_snap_rstat": "false", 2023-09-23T19:30:46.781 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_standby_replay_damaged": "false", 2023-09-23T19:30:46.782 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_symlink_recovery": "true", 2023-09-23T19:30:46.782 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_task_status_update_interval": "2.000000", 2023-09-23T19:30:46.782 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_thrash_exports": "0", 2023-09-23T19:30:46.782 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_thrash_fragments": "0", 2023-09-23T19:30:46.782 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_tick_interval": "5.000000", 2023-09-23T19:30:46.782 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_valgrind_exit": "false", 2023-09-23T19:30:46.782 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_verify_backtrace": "1", 2023-09-23T19:30:46.782 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_verify_scatter": "false", 2023-09-23T19:30:46.782 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_wipe_ino_prealloc": "false", 2023-09-23T19:30:46.782 INFO:teuthology.orchestra.run.smithi016.stdout: "mds_wipe_sessions": "false", 2023-09-23T19:30:46.783 INFO:teuthology.orchestra.run.smithi016.stdout: "mempool_debug": "false", 2023-09-23T19:30:46.783 INFO:teuthology.orchestra.run.smithi016.stdout: "memstore_debug_omit_block_device_write": "false", 2023-09-23T19:30:46.783 INFO:teuthology.orchestra.run.smithi016.stdout: "memstore_device_bytes": "1073741824", 2023-09-23T19:30:46.783 INFO:teuthology.orchestra.run.smithi016.stdout: "memstore_page_set": "false", 2023-09-23T19:30:46.783 INFO:teuthology.orchestra.run.smithi016.stdout: "memstore_page_size": "65536", 2023-09-23T19:30:46.783 INFO:teuthology.orchestra.run.smithi016.stdout: "mgr_client_bytes": "134217728", 2023-09-23T19:30:46.783 INFO:teuthology.orchestra.run.smithi016.stdout: "mgr_client_messages": "512", 2023-09-23T19:30:46.783 INFO:teuthology.orchestra.run.smithi016.stdout: "mgr_client_service_daemon_unregister_timeout": "1.000000", 2023-09-23T19:30:46.783 INFO:teuthology.orchestra.run.smithi016.stdout: "mgr_connect_retry_interval": "1.000000", 2023-09-23T19:30:46.783 INFO:teuthology.orchestra.run.smithi016.stdout: "mgr_data": "/var/lib/ceph/mgr/ceph-a", 2023-09-23T19:30:46.783 INFO:teuthology.orchestra.run.smithi016.stdout: "mgr_debug_aggressive_pg_num_changes": "false", 2023-09-23T19:30:46.784 INFO:teuthology.orchestra.run.smithi016.stdout: "mgr_disabled_modules": "", 2023-09-23T19:30:46.784 INFO:teuthology.orchestra.run.smithi016.stdout: "mgr_initial_modules": "restful iostat nfs", 2023-09-23T19:30:46.784 INFO:teuthology.orchestra.run.smithi016.stdout: "mgr_max_pg_num_change": "128", 2023-09-23T19:30:46.784 INFO:teuthology.orchestra.run.smithi016.stdout: "mgr_mds_bytes": "134217728", 2023-09-23T19:30:46.784 INFO:teuthology.orchestra.run.smithi016.stdout: "mgr_mds_messages": "128", 2023-09-23T19:30:46.784 INFO:teuthology.orchestra.run.smithi016.stdout: "mgr_module_path": "/usr/share/ceph/mgr", 2023-09-23T19:30:46.784 INFO:teuthology.orchestra.run.smithi016.stdout: "mgr_mon_bytes": "134217728", 2023-09-23T19:30:46.784 INFO:teuthology.orchestra.run.smithi016.stdout: "mgr_mon_messages": "128", 2023-09-23T19:30:46.784 INFO:teuthology.orchestra.run.smithi016.stdout: "mgr_osd_bytes": "536870912", 2023-09-23T19:30:46.784 INFO:teuthology.orchestra.run.smithi016.stdout: "mgr_osd_messages": "8192", 2023-09-23T19:30:46.784 INFO:teuthology.orchestra.run.smithi016.stdout: "mgr_pool": "true", 2023-09-23T19:30:46.785 INFO:teuthology.orchestra.run.smithi016.stdout: "mgr_service_beacon_grace": "60.000000", 2023-09-23T19:30:46.785 INFO:teuthology.orchestra.run.smithi016.stdout: "mgr_standby_modules": "true", 2023-09-23T19:30:46.785 INFO:teuthology.orchestra.run.smithi016.stdout: "mgr_stats_period": "5", 2023-09-23T19:30:46.785 INFO:teuthology.orchestra.run.smithi016.stdout: "mgr_stats_threshold": "5", 2023-09-23T19:30:46.785 INFO:teuthology.orchestra.run.smithi016.stdout: "mgr_tick_period": "2", 2023-09-23T19:30:46.785 INFO:teuthology.orchestra.run.smithi016.stdout: "mgr_ttl_cache_expire_seconds": "0", 2023-09-23T19:30:46.785 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_accept_timeout_factor": "2.000000", 2023-09-23T19:30:46.785 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_allow_pool_delete": "true", 2023-09-23T19:30:46.785 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_allow_pool_size_one": "false", 2023-09-23T19:30:46.786 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_auth_validate_all_caps": "true", 2023-09-23T19:30:46.786 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_cache_target_full_warn_ratio": "0.660000", 2023-09-23T19:30:46.786 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_clean_pg_upmaps_per_chunk": "256", 2023-09-23T19:30:46.786 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_client_bytes": "104857600", 2023-09-23T19:30:46.786 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_client_directed_command_retry": "2", 2023-09-23T19:30:46.786 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_client_hunt_interval": "3.000000", 2023-09-23T19:30:46.786 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_client_hunt_interval_backoff": "1.500000", 2023-09-23T19:30:46.786 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_client_hunt_interval_max_multiple": "10.000000", 2023-09-23T19:30:46.786 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_client_hunt_interval_min_multiple": "1.000000", 2023-09-23T19:30:46.787 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_client_hunt_parallel": "3", 2023-09-23T19:30:46.787 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_client_log_interval": "1.000000", 2023-09-23T19:30:46.787 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_client_max_log_entries_per_message": "1000", 2023-09-23T19:30:46.787 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_client_ping_interval": "10.000000", 2023-09-23T19:30:46.787 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_client_ping_timeout": "30.000000", 2023-09-23T19:30:46.787 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_clock_drift_allowed": "1.000000", 2023-09-23T19:30:46.787 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_clock_drift_warn_backoff": "5.000000", 2023-09-23T19:30:46.787 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_cluster_log_file": "default=/var/log/ceph/ceph.$channel.log cluster=/var/log/ceph/ceph.log", 2023-09-23T19:30:46.787 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_cluster_log_file_level": "debug", 2023-09-23T19:30:46.787 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_cluster_log_to_file": "false", 2023-09-23T19:30:46.787 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_cluster_log_to_graylog": "false", 2023-09-23T19:30:46.788 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_cluster_log_to_graylog_host": "127.0.0.1", 2023-09-23T19:30:46.788 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_cluster_log_to_graylog_port": "12201", 2023-09-23T19:30:46.788 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_cluster_log_to_journald": "false", 2023-09-23T19:30:46.788 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_cluster_log_to_stderr": "true", 2023-09-23T19:30:46.788 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_cluster_log_to_syslog": "default=false", 2023-09-23T19:30:46.788 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_cluster_log_to_syslog_facility": "daemon", 2023-09-23T19:30:46.788 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_cluster_log_to_syslog_level": "info", 2023-09-23T19:30:46.788 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_compact_on_bootstrap": "false", 2023-09-23T19:30:46.789 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_compact_on_start": "false", 2023-09-23T19:30:46.789 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_compact_on_trim": "true", 2023-09-23T19:30:46.789 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_con_tracker_persist_interval": "10", 2023-09-23T19:30:46.789 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_con_tracker_score_halflife": "43200", 2023-09-23T19:30:46.789 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_config_key_max_entry_size": "65536", 2023-09-23T19:30:46.789 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_cpu_threads": "4", 2023-09-23T19:30:46.789 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_crush_min_required_version": "hammer", 2023-09-23T19:30:46.789 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_daemon_bytes": "419430400", 2023-09-23T19:30:46.790 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_data": "/var/lib/ceph/mon/ceph-a", 2023-09-23T19:30:46.790 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_data_avail_crit": "5", 2023-09-23T19:30:46.790 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_data_avail_warn": "5", 2023-09-23T19:30:46.790 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_data_size_warn": "16106127360", 2023-09-23T19:30:46.790 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_debug_block_osdmap_trim": "false", 2023-09-23T19:30:46.790 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_debug_deprecated_as_obsolete": "false", 2023-09-23T19:30:46.790 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_debug_dump_json": "false", 2023-09-23T19:30:46.790 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_debug_dump_location": "/var/log/ceph/ceph-mon.a.tdump", 2023-09-23T19:30:46.791 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_debug_dump_transactions": "false", 2023-09-23T19:30:46.791 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_debug_extra_checks": "false", 2023-09-23T19:30:46.791 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_debug_no_initial_persistent_features": "false", 2023-09-23T19:30:46.791 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_debug_no_require_bluestore_for_ec_overwrites": "false", 2023-09-23T19:30:46.791 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_debug_no_require_quincy": "false", 2023-09-23T19:30:46.791 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_debug_no_require_reef": "false", 2023-09-23T19:30:46.791 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_debug_unsafe_allow_tier_with_nonempty_snaps": "false", 2023-09-23T19:30:46.791 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_delta_reset_interval": "10.000000", 2023-09-23T19:30:46.791 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_dns_srv_name": "ceph-mon", 2023-09-23T19:30:46.791 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_down_mkfs_grace": "60", 2023-09-23T19:30:46.791 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_election_default_strategy": "3", 2023-09-23T19:30:46.792 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_election_timeout": "5.000000", 2023-09-23T19:30:46.792 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_elector_ignore_propose_margin": "0.000500", 2023-09-23T19:30:46.792 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_elector_ping_divisor": "2", 2023-09-23T19:30:46.792 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_elector_ping_timeout": "2.000000", 2023-09-23T19:30:46.792 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_enable_op_tracker": "true", 2023-09-23T19:30:46.792 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_fake_pool_delete": "false", 2023-09-23T19:30:46.792 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_force_quorum_join": "false", 2023-09-23T19:30:46.792 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_globalid_prealloc": "10000", 2023-09-23T19:30:46.792 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_health_detail_to_clog": "true", 2023-09-23T19:30:46.792 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_health_log_update_period": "5", 2023-09-23T19:30:46.793 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_health_max_detail": "50", 2023-09-23T19:30:46.793 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_health_to_clog": "true", 2023-09-23T19:30:46.793 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_health_to_clog_interval": "600", 2023-09-23T19:30:46.793 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_health_to_clog_tick_interval": "60.000000", 2023-09-23T19:30:46.793 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_host": "[v2:172.21.15.16:3300/0,v1:172.21.15.16:6789/0] [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] [v2:172.21.15.16:3301/0,v1:172.21.15.16:6790/0]", 2023-09-23T19:30:46.793 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_host_override": "", 2023-09-23T19:30:46.793 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_initial_members": "", 2023-09-23T19:30:46.793 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_inject_pg_merge_bounce_probability": "0.000000", 2023-09-23T19:30:46.793 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_inject_sync_get_chunk_delay": "0.000000", 2023-09-23T19:30:46.793 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_inject_transaction_delay_max": "10.000000", 2023-09-23T19:30:46.794 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_inject_transaction_delay_probability": "0.000000", 2023-09-23T19:30:46.794 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_keyvaluedb": "rocksdb", 2023-09-23T19:30:46.794 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_lease": "5.000000", 2023-09-23T19:30:46.794 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_lease_ack_timeout_factor": "2.000000", 2023-09-23T19:30:46.794 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_lease_renew_interval_factor": "0.600000", 2023-09-23T19:30:46.794 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_log_full_interval": "50", 2023-09-23T19:30:46.794 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_log_max": "10000", 2023-09-23T19:30:46.794 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_log_max_summary": "50", 2023-09-23T19:30:46.794 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_max_log_entries_per_event": "4096", 2023-09-23T19:30:46.795 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_max_log_epochs": "500", 2023-09-23T19:30:46.795 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_max_mdsmap_epochs": "500", 2023-09-23T19:30:46.795 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_max_mgrmap_epochs": "500", 2023-09-23T19:30:46.795 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_max_osd": "10000", 2023-09-23T19:30:46.795 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_max_pg_per_osd": "10000", 2023-09-23T19:30:46.795 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_max_pool_pg_num": "65536", 2023-09-23T19:30:46.795 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_max_snap_prune_per_epoch": "100", 2023-09-23T19:30:46.795 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_mds_blocklist_interval": "86400.000000", 2023-09-23T19:30:46.795 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_mds_force_trim_to": "0", 2023-09-23T19:30:46.796 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_mds_skip_sanity": "true", 2023-09-23T19:30:46.796 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_memory_autotune": "true", 2023-09-23T19:30:46.796 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_memory_target": "2147483648", 2023-09-23T19:30:46.796 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_mgr_beacon_grace": "30", 2023-09-23T19:30:46.796 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_mgr_blocklist_interval": "86400.000000", 2023-09-23T19:30:46.796 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_mgr_digest_period": "5", 2023-09-23T19:30:46.796 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_mgr_inactive_grace": "60", 2023-09-23T19:30:46.796 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_mgr_mkfs_grace": "240", 2023-09-23T19:30:46.796 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_mgr_proxy_client_bytes_ratio": "0.300000", 2023-09-23T19:30:46.797 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_min_osdmap_epochs": "500", 2023-09-23T19:30:46.797 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_op_complaint_time": "30", 2023-09-23T19:30:46.797 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_op_history_duration": "600", 2023-09-23T19:30:46.797 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_op_history_size": "20", 2023-09-23T19:30:46.797 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_op_history_slow_op_size": "20", 2023-09-23T19:30:46.797 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_op_history_slow_op_threshold": "10", 2023-09-23T19:30:46.797 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_op_log_threshold": "5", 2023-09-23T19:30:46.797 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_adjust_down_out_interval": "true", 2023-09-23T19:30:46.797 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_adjust_heartbeat_grace": "true", 2023-09-23T19:30:46.797 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_auto_mark_auto_out_in": "true", 2023-09-23T19:30:46.798 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_auto_mark_in": "false", 2023-09-23T19:30:46.798 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_auto_mark_new_in": "true", 2023-09-23T19:30:46.798 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_backfillfull_ratio": "0.900000", 2023-09-23T19:30:46.798 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_blocklist_default_expire": "3600.000000", 2023-09-23T19:30:46.798 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_cache_size": "500", 2023-09-23T19:30:46.798 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_cache_size_min": "134217728", 2023-09-23T19:30:46.798 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_crush_smoke_test": "true", 2023-09-23T19:30:46.798 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_destroyed_out_interval": "600", 2023-09-23T19:30:46.799 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_down_out_interval": "600", 2023-09-23T19:30:46.799 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_down_out_subtree_limit": "rack", 2023-09-23T19:30:46.799 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_err_op_age_ratio": "128.000000", 2023-09-23T19:30:46.799 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_force_trim_to": "0", 2023-09-23T19:30:46.799 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_full_ratio": "0.950000", 2023-09-23T19:30:46.799 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_initial_require_min_compat_client": "luminous", 2023-09-23T19:30:46.799 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_laggy_halflife": "3600", 2023-09-23T19:30:46.799 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_laggy_max_interval": "300", 2023-09-23T19:30:46.799 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_laggy_weight": "0.300000", 2023-09-23T19:30:46.800 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_mapping_pgs_per_chunk": "4096", 2023-09-23T19:30:46.800 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_max_creating_pgs": "1024", 2023-09-23T19:30:46.800 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_max_initial_pgs": "1024", 2023-09-23T19:30:46.800 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_min_down_reporters": "2", 2023-09-23T19:30:46.800 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_min_in_ratio": "0.750000", 2023-09-23T19:30:46.800 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_min_up_ratio": "0.300000", 2023-09-23T19:30:46.800 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_nearfull_ratio": "0.850000", 2023-09-23T19:30:46.800 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_prime_pg_temp": "true", 2023-09-23T19:30:46.800 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_prime_pg_temp_max_estimate": "0.250000", 2023-09-23T19:30:46.801 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_prime_pg_temp_max_time": "0.500000", 2023-09-23T19:30:46.801 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_report_timeout": "900", 2023-09-23T19:30:46.801 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_reporter_subtree_level": "osd", 2023-09-23T19:30:46.801 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_snap_trim_queue_warn_on": "32768", 2023-09-23T19:30:46.801 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_warn_num_repaired": "10", 2023-09-23T19:30:46.801 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osd_warn_op_age": "32.000000", 2023-09-23T19:30:46.801 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osdmap_full_prune_enabled": "true", 2023-09-23T19:30:46.801 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osdmap_full_prune_interval": "10", 2023-09-23T19:30:46.801 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osdmap_full_prune_min": "10000", 2023-09-23T19:30:46.801 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_osdmap_full_prune_txsize": "100", 2023-09-23T19:30:46.801 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_pg_check_down_all_threshold": "0.500000", 2023-09-23T19:30:46.802 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_pg_stuck_threshold": "60", 2023-09-23T19:30:46.802 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_pg_warn_max_object_skew": "0.000000", 2023-09-23T19:30:46.802 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_pg_warn_min_objects": "10000", 2023-09-23T19:30:46.802 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_pg_warn_min_per_osd": "0", 2023-09-23T19:30:46.802 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_pg_warn_min_pool_objects": "1000", 2023-09-23T19:30:46.802 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_pool_quota_crit_threshold": "0", 2023-09-23T19:30:46.802 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_pool_quota_warn_threshold": "0", 2023-09-23T19:30:46.802 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_probe_timeout": "2.000000", 2023-09-23T19:30:46.802 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_reweight_max_change": "0.050000", 2023-09-23T19:30:46.802 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_reweight_max_osds": "4", 2023-09-23T19:30:46.802 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_reweight_min_bytes_per_osd": "10", 2023-09-23T19:30:46.803 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_reweight_min_pgs_per_osd": "4", 2023-09-23T19:30:46.803 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_rocksdb_options": "write_buffer_size=33554432,compression=kNoCompression,level_compaction_dynamic_level_bytes=true", 2023-09-23T19:30:46.803 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_scrub_inject_crc_mismatch": "0.000000", 2023-09-23T19:30:46.803 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_scrub_inject_missing_keys": "0.000000", 2023-09-23T19:30:46.803 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_scrub_interval": "86400", 2023-09-23T19:30:46.803 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_scrub_max_keys": "100", 2023-09-23T19:30:46.803 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_scrub_timeout": "300", 2023-09-23T19:30:46.803 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_session_timeout": "300", 2023-09-23T19:30:46.803 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_smart_report_timeout": "5", 2023-09-23T19:30:46.803 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_stat_smooth_intervals": "6", 2023-09-23T19:30:46.803 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_stretch_cluster_recovery_ratio": "0.600000", 2023-09-23T19:30:46.803 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_stretch_pool_min_size": "2", 2023-09-23T19:30:46.804 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_stretch_pool_size": "4", 2023-09-23T19:30:46.804 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_stretch_recovery_min_wait": "15.000000", 2023-09-23T19:30:46.804 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_subscribe_interval": "86400.000000", 2023-09-23T19:30:46.804 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_sync_debug": "false", 2023-09-23T19:30:46.804 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_sync_max_payload_keys": "2000", 2023-09-23T19:30:46.804 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_sync_max_payload_size": "1048576", 2023-09-23T19:30:46.804 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_sync_provider_kill_at": "0", 2023-09-23T19:30:46.804 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_sync_requester_kill_at": "0", 2023-09-23T19:30:46.804 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_sync_timeout": "60.000000", 2023-09-23T19:30:46.804 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_target_pg_per_osd": "100", 2023-09-23T19:30:46.804 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_tick_interval": "5", 2023-09-23T19:30:46.804 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_timecheck_interval": "300.000000", 2023-09-23T19:30:46.805 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_timecheck_skew_interval": "30.000000", 2023-09-23T19:30:46.805 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_use_min_delay_socket": "false", 2023-09-23T19:30:46.805 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_warn_older_version_delay": "604800", 2023-09-23T19:30:46.805 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_warn_on_cache_pools_without_hit_sets": "true", 2023-09-23T19:30:46.805 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_warn_on_crush_straw_calc_version_zero": "false", 2023-09-23T19:30:46.805 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_warn_on_degraded_stretch_mode": "true", 2023-09-23T19:30:46.805 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_warn_on_filestore_osds": "true", 2023-09-23T19:30:46.805 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_warn_on_insecure_global_id_reclaim": "false", 2023-09-23T19:30:46.805 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed": "false", 2023-09-23T19:30:46.805 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_warn_on_legacy_crush_tunables": "false", 2023-09-23T19:30:46.805 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_warn_on_misplaced": "false", 2023-09-23T19:30:46.805 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_warn_on_msgr2_not_enabled": "true", 2023-09-23T19:30:46.806 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_warn_on_older_version": "true", 2023-09-23T19:30:46.806 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_warn_on_osd_down_out_interval_zero": "false", 2023-09-23T19:30:46.806 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_warn_on_pool_no_app": "true", 2023-09-23T19:30:46.806 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_warn_on_pool_no_redundancy": "true", 2023-09-23T19:30:46.806 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_warn_on_pool_pg_num_not_power_of_two": "false", 2023-09-23T19:30:46.806 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_warn_on_slow_ping_ratio": "0.050000", 2023-09-23T19:30:46.806 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_warn_on_slow_ping_time": "0.000000", 2023-09-23T19:30:46.806 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_warn_on_too_few_osds": "false", 2023-09-23T19:30:46.806 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_warn_pg_not_deep_scrubbed_ratio": "0.750000", 2023-09-23T19:30:46.806 INFO:teuthology.orchestra.run.smithi016.stdout: "mon_warn_pg_not_scrubbed_ratio": "0.500000", 2023-09-23T19:30:46.806 INFO:teuthology.orchestra.run.smithi016.stdout: "monmap": "", 2023-09-23T19:30:46.807 INFO:teuthology.orchestra.run.smithi016.stdout: "motr_admin_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-09-23T19:30:46.807 INFO:teuthology.orchestra.run.smithi016.stdout: "motr_admin_fid": "0x7200000000000001:0x0", 2023-09-23T19:30:46.807 INFO:teuthology.orchestra.run.smithi016.stdout: "motr_ha_endpoint": "192.168.180.182@tcp:12345:1:1", 2023-09-23T19:30:46.807 INFO:teuthology.orchestra.run.smithi016.stdout: "motr_my_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-09-23T19:30:46.807 INFO:teuthology.orchestra.run.smithi016.stdout: "motr_my_fid": "0x7200000000000001:0x0", 2023-09-23T19:30:46.807 INFO:teuthology.orchestra.run.smithi016.stdout: "motr_profile_fid": "0x7000000000000001:0x0", 2023-09-23T19:30:46.807 INFO:teuthology.orchestra.run.smithi016.stdout: "motr_tracing_enabled": "false", 2023-09-23T19:30:46.807 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_async_op_threads": "3", 2023-09-23T19:30:46.807 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_async_rdma_buffer_size": "131072", 2023-09-23T19:30:46.807 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_async_rdma_cm": "false", 2023-09-23T19:30:46.807 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_async_rdma_device_name": "", 2023-09-23T19:30:46.807 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_async_rdma_dscp": "96", 2023-09-23T19:30:46.808 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_async_rdma_enable_hugepage": "false", 2023-09-23T19:30:46.808 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_async_rdma_gid_idx": "0", 2023-09-23T19:30:46.808 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_async_rdma_local_gid": "", 2023-09-23T19:30:46.808 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_async_rdma_polling_us": "1000", 2023-09-23T19:30:46.808 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_async_rdma_port_num": "1", 2023-09-23T19:30:46.808 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_async_rdma_receive_buffers": "32768", 2023-09-23T19:30:46.808 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_async_rdma_receive_queue_len": "4096", 2023-09-23T19:30:46.808 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_async_rdma_roce_ver": "1", 2023-09-23T19:30:46.808 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_async_rdma_send_buffers": "1024", 2023-09-23T19:30:46.808 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_async_rdma_sl": "3", 2023-09-23T19:30:46.808 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_async_rdma_support_srq": "true", 2023-09-23T19:30:46.808 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_async_rdma_type": "ib", 2023-09-23T19:30:46.809 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_async_reap_threshold": "5", 2023-09-23T19:30:46.809 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_bind_before_connect": "false", 2023-09-23T19:30:46.809 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_bind_ipv4": "true", 2023-09-23T19:30:46.809 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_bind_ipv6": "false", 2023-09-23T19:30:46.809 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_bind_msgr1": "true", 2023-09-23T19:30:46.809 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_bind_msgr2": "true", 2023-09-23T19:30:46.809 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_bind_port_max": "7568", 2023-09-23T19:30:46.809 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_bind_port_min": "6800", 2023-09-23T19:30:46.809 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_bind_prefer_ipv4": "false", 2023-09-23T19:30:46.809 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_bind_retry_count": "3", 2023-09-23T19:30:46.809 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_bind_retry_delay": "5", 2023-09-23T19:30:46.809 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_blackhole_client": "false", 2023-09-23T19:30:46.810 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_blackhole_mds": "false", 2023-09-23T19:30:46.810 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_blackhole_mgr": "false", 2023-09-23T19:30:46.810 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_blackhole_mon": "false", 2023-09-23T19:30:46.810 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_blackhole_osd": "false", 2023-09-23T19:30:46.810 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_client_mode": "crc secure", 2023-09-23T19:30:46.810 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_cluster_mode": "crc secure", 2023-09-23T19:30:46.810 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_cluster_type": "", 2023-09-23T19:30:46.810 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_compress_secure": "false", 2023-09-23T19:30:46.810 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_connection_idle_timeout": "900", 2023-09-23T19:30:46.810 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_connection_ready_timeout": "10", 2023-09-23T19:30:46.810 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_crc_data": "true", 2023-09-23T19:30:46.810 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_crc_header": "true", 2023-09-23T19:30:46.811 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_die_on_bad_msg": "false", 2023-09-23T19:30:46.811 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_die_on_bug": "true", 2023-09-23T19:30:46.811 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_die_on_old_message": "true", 2023-09-23T19:30:46.811 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_die_on_skipped_message": "false", 2023-09-23T19:30:46.811 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_die_on_unhandled_msg": "false", 2023-09-23T19:30:46.811 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_dispatch_throttle_bytes": "104857600", 2023-09-23T19:30:46.811 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_dpdk_coremask": "0xF", 2023-09-23T19:30:46.811 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_dpdk_debug_allow_loopback": "false", 2023-09-23T19:30:46.811 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_dpdk_devs_allowlist": "", 2023-09-23T19:30:46.811 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_dpdk_enable_tso": "true", 2023-09-23T19:30:46.811 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_dpdk_gateway_ipv4_addr": "", 2023-09-23T19:30:46.812 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_dpdk_host_ipv4_addr": "", 2023-09-23T19:30:46.812 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_dpdk_hugepages": "", 2023-09-23T19:30:46.812 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_dpdk_hw_flow_control": "true", 2023-09-23T19:30:46.812 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_dpdk_hw_queue_weight": "1.000000", 2023-09-23T19:30:46.812 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_dpdk_lro": "true", 2023-09-23T19:30:46.812 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_dpdk_memory_channel": "4", 2023-09-23T19:30:46.812 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_dpdk_netmask_ipv4_addr": "", 2023-09-23T19:30:46.812 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_dpdk_pmd": "", 2023-09-23T19:30:46.812 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_dpdk_port_id": "0", 2023-09-23T19:30:46.812 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_dpdk_rx_buffer_count_per_core": "8192", 2023-09-23T19:30:46.813 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_dump_corrupt_message_level": "1", 2023-09-23T19:30:46.813 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_dump_on_send": "false", 2023-09-23T19:30:46.813 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_initial_backoff": "0.200000", 2023-09-23T19:30:46.813 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_inject_delay_max": "1.000000", 2023-09-23T19:30:46.813 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_inject_delay_probability": "0.000000", 2023-09-23T19:30:46.813 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_inject_delay_type": "", 2023-09-23T19:30:46.813 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_inject_internal_delays": "0.000000", 2023-09-23T19:30:46.813 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_inject_network_congestion": "0", 2023-09-23T19:30:46.813 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_inject_socket_failures": "0", 2023-09-23T19:30:46.814 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_learn_addr_from_peer": "true", 2023-09-23T19:30:46.814 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_max_accept_failures": "4", 2023-09-23T19:30:46.814 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_max_backoff": "15.000000", 2023-09-23T19:30:46.814 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_mon_client_mode": "secure crc", 2023-09-23T19:30:46.814 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_mon_cluster_mode": "secure crc", 2023-09-23T19:30:46.814 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_mon_service_mode": "secure crc", 2023-09-23T19:30:46.814 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_osd_compress_min_size": "1024", 2023-09-23T19:30:46.814 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_osd_compress_mode": "none", 2023-09-23T19:30:46.814 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_osd_compression_algorithm": "snappy", 2023-09-23T19:30:46.814 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_pq_max_tokens_per_priority": "16777216", 2023-09-23T19:30:46.815 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_pq_min_cost": "65536", 2023-09-23T19:30:46.815 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_public_type": "", 2023-09-23T19:30:46.815 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_service_mode": "crc secure", 2023-09-23T19:30:46.815 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_tcp_listen_backlog": "512", 2023-09-23T19:30:46.815 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_tcp_nodelay": "true", 2023-09-23T19:30:46.815 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_tcp_prefetch_max_size": "4096", 2023-09-23T19:30:46.815 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_tcp_rcvbuf": "0", 2023-09-23T19:30:46.815 INFO:teuthology.orchestra.run.smithi016.stdout: "ms_type": "async+posix", 2023-09-23T19:30:46.815 INFO:teuthology.orchestra.run.smithi016.stdout: "no_config_file": "false", 2023-09-23T19:30:46.815 INFO:teuthology.orchestra.run.smithi016.stdout: "objecter_completion_locks_per_session": "32", 2023-09-23T19:30:46.816 INFO:teuthology.orchestra.run.smithi016.stdout: "objecter_debug_inject_relock_delay": "false", 2023-09-23T19:30:46.816 INFO:teuthology.orchestra.run.smithi016.stdout: "objecter_inflight_op_bytes": "104857600", 2023-09-23T19:30:46.816 INFO:teuthology.orchestra.run.smithi016.stdout: "objecter_inflight_ops": "1024", 2023-09-23T19:30:46.816 INFO:teuthology.orchestra.run.smithi016.stdout: "objecter_inject_no_watch_ping": "false", 2023-09-23T19:30:46.816 INFO:teuthology.orchestra.run.smithi016.stdout: "objecter_retry_writes_after_first_reply": "false", 2023-09-23T19:30:46.816 INFO:teuthology.orchestra.run.smithi016.stdout: "objecter_tick_interval": "5.000000", 2023-09-23T19:30:46.816 INFO:teuthology.orchestra.run.smithi016.stdout: "objecter_timeout": "10.000000", 2023-09-23T19:30:46.816 INFO:teuthology.orchestra.run.smithi016.stdout: "objectstore_blackhole": "false", 2023-09-23T19:30:46.816 INFO:teuthology.orchestra.run.smithi016.stdout: "openssl_engine_opts": "", 2023-09-23T19:30:46.816 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_agent_delay_time": "5.000000", 2023-09-23T19:30:46.817 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_agent_hist_halflife": "1000", 2023-09-23T19:30:46.817 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_agent_max_low_ops": "2", 2023-09-23T19:30:46.817 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_agent_max_ops": "4", 2023-09-23T19:30:46.817 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_agent_min_evict_effort": "0.100000", 2023-09-23T19:30:46.817 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_agent_quantize_effort": "0.100000", 2023-09-23T19:30:46.817 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_agent_slop": "0.020000", 2023-09-23T19:30:46.817 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_aggregated_slow_ops_logging": "true", 2023-09-23T19:30:46.817 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_allow_recovery_below_min_size": "true", 2023-09-23T19:30:46.817 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_asio_thread_count": "2", 2023-09-23T19:30:46.818 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_async_recovery_min_cost": "100", 2023-09-23T19:30:46.818 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_auto_mark_unfound_lost": "false", 2023-09-23T19:30:46.818 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_backfill_retry_interval": "30.000000", 2023-09-23T19:30:46.818 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_backfill_scan_max": "512", 2023-09-23T19:30:46.818 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_backfill_scan_min": "64", 2023-09-23T19:30:46.818 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_backoff_on_degraded": "false", 2023-09-23T19:30:46.818 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_backoff_on_peering": "false", 2023-09-23T19:30:46.818 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_backoff_on_unfound": "true", 2023-09-23T19:30:46.818 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_beacon_report_interval": "300", 2023-09-23T19:30:46.818 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_bench_duration": "30", 2023-09-23T19:30:46.819 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_bench_large_size_max_throughput": "104857600", 2023-09-23T19:30:46.819 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_bench_max_block_size": "67108864", 2023-09-23T19:30:46.819 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_bench_small_size_max_iops": "100", 2023-09-23T19:30:46.819 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_blkin_trace_all": "false", 2023-09-23T19:30:46.819 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_blocked_scrub_grace_period": "120", 2023-09-23T19:30:46.819 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_calc_pg_upmaps_aggressively": "true", 2023-09-23T19:30:46.819 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_calc_pg_upmaps_aggressively_fast": "true", 2023-09-23T19:30:46.819 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_calc_pg_upmaps_local_fallback_retries": "100", 2023-09-23T19:30:46.819 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_check_for_log_corruption": "false", 2023-09-23T19:30:46.819 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_check_max_object_name_len_on_startup": "true", 2023-09-23T19:30:46.820 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_class_default_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-09-23T19:30:46.820 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_class_dir": "/usr/lib64/rados-classes", 2023-09-23T19:30:46.820 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_class_load_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-09-23T19:30:46.820 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_class_update_on_start": "true", 2023-09-23T19:30:46.820 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_client_message_cap": "256", 2023-09-23T19:30:46.820 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_client_message_size_cap": "524288000", 2023-09-23T19:30:46.820 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_client_op_priority": "63", 2023-09-23T19:30:46.820 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_client_watch_timeout": "30", 2023-09-23T19:30:46.821 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_command_max_records": "256", 2023-09-23T19:30:46.821 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_compact_on_start": "false", 2023-09-23T19:30:46.821 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_copyfrom_max_chunk": "8388608", 2023-09-23T19:30:46.821 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_crush_chooseleaf_type": "1", 2023-09-23T19:30:46.821 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_crush_initial_weight": "-1.000000", 2023-09-23T19:30:46.821 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_crush_update_on_start": "true", 2023-09-23T19:30:46.821 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_crush_update_weight_set": "true", 2023-09-23T19:30:46.821 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_data": "/var/lib/ceph/osd/ceph-a", 2023-09-23T19:30:46.822 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_debug_crash_on_ignored_backoff": "false", 2023-09-23T19:30:46.822 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_debug_deep_scrub_sleep": "0.000000", 2023-09-23T19:30:46.822 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_debug_drop_ping_duration": "0", 2023-09-23T19:30:46.822 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_debug_drop_ping_probability": "0.000000", 2023-09-23T19:30:46.822 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_debug_feed_pullee": "-1", 2023-09-23T19:30:46.822 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_debug_inject_copyfrom_error": "false", 2023-09-23T19:30:46.822 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_debug_inject_dispatch_delay_duration": "0.100000", 2023-09-23T19:30:46.822 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_debug_inject_dispatch_delay_probability": "0.000000", 2023-09-23T19:30:46.823 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_debug_misdirected_ops": "false", 2023-09-23T19:30:46.823 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_debug_no_acting_change": "false", 2023-09-23T19:30:46.823 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_debug_no_purge_strays": "false", 2023-09-23T19:30:46.823 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_debug_op_order": "false", 2023-09-23T19:30:46.823 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_debug_pg_log_writeout": "false", 2023-09-23T19:30:46.823 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_debug_pretend_recovery_active": "false", 2023-09-23T19:30:46.823 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_debug_random_push_read_error": "0.000000", 2023-09-23T19:30:46.823 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_debug_reject_backfill_probability": "0.000000", 2023-09-23T19:30:46.823 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_debug_shutdown": "false", 2023-09-23T19:30:46.824 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_debug_skip_full_check_in_backfill_reservation": "false", 2023-09-23T19:30:46.824 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_debug_skip_full_check_in_recovery": "false", 2023-09-23T19:30:46.824 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_debug_verify_cached_snaps": "false", 2023-09-23T19:30:46.824 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_debug_verify_missing_on_start": "false", 2023-09-23T19:30:46.824 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_debug_verify_snaps": "false", 2023-09-23T19:30:46.824 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_debug_verify_stray_on_activate": "false", 2023-09-23T19:30:46.824 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_deep_scrub_interval": "604800.000000", 2023-09-23T19:30:46.825 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_deep_scrub_keys": "1024", 2023-09-23T19:30:46.825 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_deep_scrub_large_omap_object_key_threshold": "200000", 2023-09-23T19:30:46.825 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_deep_scrub_large_omap_object_value_sum_threshold": "1073741824", 2023-09-23T19:30:46.825 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_deep_scrub_randomize_ratio": "0.150000", 2023-09-23T19:30:46.825 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_deep_scrub_stride": "524288", 2023-09-23T19:30:46.825 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_deep_scrub_update_digest_min_age": "7200", 2023-09-23T19:30:46.825 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_default_data_pool_replay_window": "45", 2023-09-23T19:30:46.826 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_default_notify_timeout": "30", 2023-09-23T19:30:46.826 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_delete_sleep": "0.000000", 2023-09-23T19:30:46.826 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_delete_sleep_hdd": "5.000000", 2023-09-23T19:30:46.826 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_delete_sleep_hybrid": "1.000000", 2023-09-23T19:30:46.826 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_delete_sleep_ssd": "1.000000", 2023-09-23T19:30:46.826 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_discard_disconnected_ops": "true", 2023-09-23T19:30:46.826 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_enable_op_tracker": "true", 2023-09-23T19:30:46.827 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_erasure_code_plugins": "jerasure lrc", 2023-09-23T19:30:46.827 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_extblkdev_plugins": "vdo", 2023-09-23T19:30:46.827 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_failsafe_full_ratio": "0.970000", 2023-09-23T19:30:46.827 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_fast_fail_on_connection_refused": "true", 2023-09-23T19:30:46.827 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_fast_info": "true", 2023-09-23T19:30:46.827 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_fast_shutdown": "true", 2023-09-23T19:30:46.827 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_fast_shutdown_notify_mon": "true", 2023-09-23T19:30:46.827 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_fast_shutdown_timeout": "15", 2023-09-23T19:30:46.828 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_find_best_info_ignore_history_les": "false", 2023-09-23T19:30:46.828 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_force_auth_primary_missing_objects": "100", 2023-09-23T19:30:46.828 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_force_recovery_pg_log_entries_factor": "1.300000", 2023-09-23T19:30:46.828 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_function_tracing": "false", 2023-09-23T19:30:46.828 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_heartbeat_grace": "20", 2023-09-23T19:30:46.828 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_heartbeat_interval": "6", 2023-09-23T19:30:46.828 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_heartbeat_min_healthy_ratio": "0.330000", 2023-09-23T19:30:46.828 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_heartbeat_min_peers": "10", 2023-09-23T19:30:46.828 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_heartbeat_min_size": "2000", 2023-09-23T19:30:46.828 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_heartbeat_stale": "600", 2023-09-23T19:30:46.829 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_heartbeat_use_min_delay_socket": "false", 2023-09-23T19:30:46.829 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_hit_set_max_size": "100000", 2023-09-23T19:30:46.829 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_hit_set_min_size": "1000", 2023-09-23T19:30:46.829 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_hit_set_namespace": ".ceph-internal", 2023-09-23T19:30:46.829 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_ignore_stale_divergent_priors": "false", 2023-09-23T19:30:46.829 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_inject_bad_map_crc_probability": "0.000000", 2023-09-23T19:30:46.829 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_inject_failure_on_pg_removal": "false", 2023-09-23T19:30:46.829 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_journal": "/var/lib/ceph/osd/ceph-a/journal", 2023-09-23T19:30:46.830 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_journal_flush_on_shutdown": "true", 2023-09-23T19:30:46.830 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_journal_size": "5120", 2023-09-23T19:30:46.830 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_kill_backfill_at": "0", 2023-09-23T19:30:46.830 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_loop_before_reset_tphandle": "64", 2023-09-23T19:30:46.830 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_map_cache_size": "50", 2023-09-23T19:30:46.830 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_map_dedup": "true", 2023-09-23T19:30:46.830 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_map_message_max": "40", 2023-09-23T19:30:46.830 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_map_message_max_bytes": "10485760", 2023-09-23T19:30:46.831 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_map_share_max_epochs": "40", 2023-09-23T19:30:46.831 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_max_attr_name_len": "100", 2023-09-23T19:30:46.831 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_max_attr_size": "0", 2023-09-23T19:30:46.831 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_max_backfills": "1", 2023-09-23T19:30:46.831 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_max_markdown_count": "5", 2023-09-23T19:30:46.831 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_max_markdown_period": "600", 2023-09-23T19:30:46.831 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_max_object_name_len": "2048", 2023-09-23T19:30:46.831 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_max_object_namespace_len": "256", 2023-09-23T19:30:46.832 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_max_object_size": "134217728", 2023-09-23T19:30:46.832 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_max_omap_bytes_per_request": "1073741824", 2023-09-23T19:30:46.832 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_max_omap_entries_per_request": "1024", 2023-09-23T19:30:46.832 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_max_pg_blocked_by": "16", 2023-09-23T19:30:46.832 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_max_pg_log_entries": "10000", 2023-09-23T19:30:46.832 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_max_pg_per_osd_hard_ratio": "3.000000", 2023-09-23T19:30:46.832 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_max_pgls": "1024", 2023-09-23T19:30:46.832 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_max_push_cost": "8388608", 2023-09-23T19:30:46.832 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_max_push_objects": "10", 2023-09-23T19:30:46.833 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_max_scrubs": "1", 2023-09-23T19:30:46.833 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_max_snap_prune_intervals_per_epoch": "512", 2023-09-23T19:30:46.833 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_max_trimming_pgs": "2", 2023-09-23T19:30:46.833 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_max_write_op_reply_len": "64", 2023-09-23T19:30:46.833 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_max_write_size": "90", 2023-09-23T19:30:46.833 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_mclock_force_run_benchmark_on_init": "false", 2023-09-23T19:30:46.833 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_mclock_iops_capacity_threshold_hdd": "500.000000", 2023-09-23T19:30:46.833 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_mclock_iops_capacity_threshold_ssd": "80000.000000", 2023-09-23T19:30:46.833 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_mclock_max_capacity_iops_hdd": "315.000000", 2023-09-23T19:30:46.834 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_mclock_max_capacity_iops_ssd": "21500.000000", 2023-09-23T19:30:46.834 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_mclock_max_sequential_bandwidth_hdd": "157286400", 2023-09-23T19:30:46.834 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_mclock_max_sequential_bandwidth_ssd": "1258291200", 2023-09-23T19:30:46.834 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_mclock_override_recovery_settings": "false", 2023-09-23T19:30:46.834 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_mclock_profile": "balanced", 2023-09-23T19:30:46.834 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_mclock_scheduler_anticipation_timeout": "0.000000", 2023-09-23T19:30:46.834 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_mclock_scheduler_background_best_effort_lim": "0.000000", 2023-09-23T19:30:46.834 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_mclock_scheduler_background_best_effort_res": "0.000000", 2023-09-23T19:30:46.834 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_mclock_scheduler_background_best_effort_wgt": "1", 2023-09-23T19:30:46.834 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_mclock_scheduler_background_recovery_lim": "0.000000", 2023-09-23T19:30:46.834 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_mclock_scheduler_background_recovery_res": "0.000000", 2023-09-23T19:30:46.835 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_mclock_scheduler_background_recovery_wgt": "1", 2023-09-23T19:30:46.835 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_mclock_scheduler_client_lim": "0.000000", 2023-09-23T19:30:46.835 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_mclock_scheduler_client_res": "0.000000", 2023-09-23T19:30:46.835 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_mclock_scheduler_client_wgt": "1", 2023-09-23T19:30:46.835 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_mclock_skip_benchmark": "false", 2023-09-23T19:30:46.835 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_memory_base": "805306368", 2023-09-23T19:30:46.835 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_memory_cache_min": "134217728", 2023-09-23T19:30:46.835 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_memory_cache_resize_interval": "1.000000", 2023-09-23T19:30:46.835 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_memory_expected_fragmentation": "0.150000", 2023-09-23T19:30:46.835 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_memory_target": "4294967296", 2023-09-23T19:30:46.835 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_memory_target_autotune": "false", 2023-09-23T19:30:46.836 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_memory_target_cgroup_limit_ratio": "0.800000", 2023-09-23T19:30:46.836 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_min_pg_log_entries": "250", 2023-09-23T19:30:46.836 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_min_recovery_priority": "0", 2023-09-23T19:30:46.836 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_mon_heartbeat_interval": "30", 2023-09-23T19:30:46.836 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_mon_heartbeat_stat_stale": "3600", 2023-09-23T19:30:46.836 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_mon_report_interval": "5", 2023-09-23T19:30:46.836 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_mon_report_max_in_flight": "2", 2023-09-23T19:30:46.836 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_mon_shutdown_timeout": "5.000000", 2023-09-23T19:30:46.836 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_num_cache_shards": "32", 2023-09-23T19:30:46.836 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_num_op_tracker_shard": "32", 2023-09-23T19:30:46.836 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_numa_auto_affinity": "true", 2023-09-23T19:30:46.836 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_numa_node": "-1", 2023-09-23T19:30:46.837 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_numa_prefer_iface": "true", 2023-09-23T19:30:46.837 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_object_clean_region_max_num_intervals": "10", 2023-09-23T19:30:46.837 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_objecter_finishers": "1", 2023-09-23T19:30:46.837 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_objectstore": "bluestore", 2023-09-23T19:30:46.837 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_objectstore_fuse": "false", 2023-09-23T19:30:46.837 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_objectstore_tracing": "false", 2023-09-23T19:30:46.837 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_op_complaint_time": "30.000000", 2023-09-23T19:30:46.837 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_op_history_duration": "600", 2023-09-23T19:30:46.837 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_op_history_size": "20", 2023-09-23T19:30:46.837 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_op_history_slow_op_size": "20", 2023-09-23T19:30:46.837 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_op_history_slow_op_threshold": "10.000000", 2023-09-23T19:30:46.837 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_op_log_threshold": "5", 2023-09-23T19:30:46.838 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_op_num_shards": "0", 2023-09-23T19:30:46.838 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_op_num_shards_hdd": "5", 2023-09-23T19:30:46.838 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_op_num_shards_ssd": "8", 2023-09-23T19:30:46.838 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_op_num_threads_per_shard": "0", 2023-09-23T19:30:46.838 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_op_num_threads_per_shard_hdd": "1", 2023-09-23T19:30:46.838 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_op_num_threads_per_shard_ssd": "2", 2023-09-23T19:30:46.838 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_op_pq_max_tokens_per_priority": "4194304", 2023-09-23T19:30:46.838 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_op_pq_min_cost": "65536", 2023-09-23T19:30:46.838 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_op_queue": "mclock_scheduler", 2023-09-23T19:30:46.838 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_op_queue_cut_off": "high", 2023-09-23T19:30:46.838 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_op_thread_suicide_timeout": "150", 2023-09-23T19:30:46.838 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_op_thread_timeout": "15", 2023-09-23T19:30:46.839 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_open_classes_on_start": "true", 2023-09-23T19:30:46.839 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_os_flags": "0", 2023-09-23T19:30:46.839 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_peering_op_priority": "255", 2023-09-23T19:30:46.839 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pg_delete_cost": "1048576", 2023-09-23T19:30:46.839 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pg_delete_priority": "5", 2023-09-23T19:30:46.839 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pg_epoch_max_lag_factor": "2.000000", 2023-09-23T19:30:46.839 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pg_epoch_persisted_max_stale": "40", 2023-09-23T19:30:46.839 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pg_log_dups_tracked": "3000", 2023-09-23T19:30:46.839 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pg_log_trim_max": "10000", 2023-09-23T19:30:46.839 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pg_log_trim_min": "100", 2023-09-23T19:30:46.839 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pg_max_concurrent_snap_trims": "2", 2023-09-23T19:30:46.839 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pg_object_context_cache_count": "64", 2023-09-23T19:30:46.840 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pg_stat_report_interval_max": "500", 2023-09-23T19:30:46.840 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pool_default_cache_max_evict_check_size": "10", 2023-09-23T19:30:46.840 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pool_default_cache_min_evict_age": "0", 2023-09-23T19:30:46.840 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pool_default_cache_min_flush_age": "0", 2023-09-23T19:30:46.840 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pool_default_cache_target_dirty_high_ratio": "0.600000", 2023-09-23T19:30:46.840 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pool_default_cache_target_dirty_ratio": "0.400000", 2023-09-23T19:30:46.840 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pool_default_cache_target_full_ratio": "0.800000", 2023-09-23T19:30:46.840 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pool_default_crimson": "false", 2023-09-23T19:30:46.840 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pool_default_crush_rule": "-1", 2023-09-23T19:30:46.840 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pool_default_ec_fast_read": "false", 2023-09-23T19:30:46.840 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pool_default_erasure_code_profile": "plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd", 2023-09-23T19:30:46.841 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pool_default_flag_bulk": "false", 2023-09-23T19:30:46.841 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pool_default_flag_hashpspool": "true", 2023-09-23T19:30:46.841 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pool_default_flag_nodelete": "false", 2023-09-23T19:30:46.841 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pool_default_flag_nopgchange": "false", 2023-09-23T19:30:46.841 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pool_default_flag_nosizechange": "false", 2023-09-23T19:30:46.841 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pool_default_flags": "0", 2023-09-23T19:30:46.841 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pool_default_hit_set_bloom_fpp": "0.050000", 2023-09-23T19:30:46.841 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pool_default_min_size": "0", 2023-09-23T19:30:46.841 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pool_default_pg_autoscale_mode": "off", 2023-09-23T19:30:46.841 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pool_default_pg_num": "32", 2023-09-23T19:30:46.841 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pool_default_pgp_num": "0", 2023-09-23T19:30:46.841 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pool_default_read_lease_ratio": "0.800000", 2023-09-23T19:30:46.842 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pool_default_size": "3", 2023-09-23T19:30:46.842 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pool_default_type": "replicated", 2023-09-23T19:30:46.842 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pool_erasure_code_stripe_unit": "4096", 2023-09-23T19:30:46.842 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_pool_use_gmt_hitset": "true", 2023-09-23T19:30:46.842 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_push_per_object_cost": "1000", 2023-09-23T19:30:46.842 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_read_ec_check_for_errors": "false", 2023-09-23T19:30:46.842 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_recover_clone_overlap": "true", 2023-09-23T19:30:46.842 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_recover_clone_overlap_limit": "10", 2023-09-23T19:30:46.842 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_recovery_cost": "20971520", 2023-09-23T19:30:46.842 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_recovery_delay_start": "0.000000", 2023-09-23T19:30:46.842 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_recovery_max_active": "0", 2023-09-23T19:30:46.842 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_recovery_max_active_hdd": "3", 2023-09-23T19:30:46.843 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_recovery_max_active_ssd": "10", 2023-09-23T19:30:46.843 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_recovery_max_chunk": "8388608", 2023-09-23T19:30:46.843 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_recovery_max_omap_entries_per_chunk": "8096", 2023-09-23T19:30:46.843 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_recovery_max_single_start": "1", 2023-09-23T19:30:46.843 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_recovery_op_priority": "3", 2023-09-23T19:30:46.843 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_recovery_op_warn_multiple": "16", 2023-09-23T19:30:46.843 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_recovery_priority": "5", 2023-09-23T19:30:46.843 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_recovery_retry_interval": "30.000000", 2023-09-23T19:30:46.843 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_recovery_sleep": "0.000000", 2023-09-23T19:30:46.843 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_recovery_sleep_hdd": "0.100000", 2023-09-23T19:30:46.843 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_recovery_sleep_hybrid": "0.025000", 2023-09-23T19:30:46.843 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_recovery_sleep_ssd": "0.000000", 2023-09-23T19:30:46.844 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_repair_during_recovery": "false", 2023-09-23T19:30:46.844 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_requested_scrub_priority": "120", 2023-09-23T19:30:46.844 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_rocksdb_iterator_bounds_enabled": "true", 2023-09-23T19:30:46.844 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_rollback_to_cluster_snap": "", 2023-09-23T19:30:46.844 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_scrub_auto_repair": "false", 2023-09-23T19:30:46.844 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_scrub_auto_repair_num_errors": "5", 2023-09-23T19:30:46.844 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_scrub_backoff_ratio": "0.660000", 2023-09-23T19:30:46.844 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_scrub_begin_hour": "0", 2023-09-23T19:30:46.844 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_scrub_begin_week_day": "0", 2023-09-23T19:30:46.844 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_scrub_chunk_max": "25", 2023-09-23T19:30:46.844 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_scrub_chunk_min": "5", 2023-09-23T19:30:46.844 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_scrub_cost": "52428800", 2023-09-23T19:30:46.845 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_scrub_during_recovery": "false", 2023-09-23T19:30:46.845 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_scrub_end_hour": "0", 2023-09-23T19:30:46.845 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_scrub_end_week_day": "0", 2023-09-23T19:30:46.845 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_scrub_event_cost": "4096", 2023-09-23T19:30:46.845 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_scrub_extended_sleep": "0.000000", 2023-09-23T19:30:46.845 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_scrub_interval_randomize_ratio": "0.500000", 2023-09-23T19:30:46.845 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_scrub_invalid_stats": "true", 2023-09-23T19:30:46.845 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_scrub_load_threshold": "0.500000", 2023-09-23T19:30:46.845 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_scrub_max_interval": "604800.000000", 2023-09-23T19:30:46.845 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_scrub_max_preemptions": "5", 2023-09-23T19:30:46.845 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_scrub_min_interval": "86400.000000", 2023-09-23T19:30:46.845 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_scrub_priority": "5", 2023-09-23T19:30:46.846 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_scrub_reservation_timeout": "5000", 2023-09-23T19:30:46.846 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_scrub_sleep": "0.000000", 2023-09-23T19:30:46.846 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_scrub_slow_reservation_response": "2200", 2023-09-23T19:30:46.846 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_shallow_scrub_chunk_max": "100", 2023-09-23T19:30:46.846 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_shallow_scrub_chunk_min": "50", 2023-09-23T19:30:46.846 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_shutdown_pgref_assert": "false", 2023-09-23T19:30:46.846 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_skip_data_digest": "false", 2023-09-23T19:30:46.846 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_smart_report_timeout": "5", 2023-09-23T19:30:46.846 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_snap_trim_cost": "1048576", 2023-09-23T19:30:46.847 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_snap_trim_priority": "5", 2023-09-23T19:30:46.847 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_snap_trim_sleep": "0.000000", 2023-09-23T19:30:46.847 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_snap_trim_sleep_hdd": "5.000000", 2023-09-23T19:30:46.847 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_snap_trim_sleep_hybrid": "2.000000", 2023-09-23T19:30:46.847 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_snap_trim_sleep_ssd": "0.000000", 2023-09-23T19:30:46.847 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_stats_update_period_not_scrubbing": "120", 2023-09-23T19:30:46.847 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_stats_update_period_scrubbing": "15", 2023-09-23T19:30:46.847 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_target_pg_log_entries_per_osd": "300000", 2023-09-23T19:30:46.847 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_target_transaction_size": "30", 2023-09-23T19:30:46.848 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_tier_default_cache_hit_set_count": "4", 2023-09-23T19:30:46.848 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_tier_default_cache_hit_set_grade_decay_rate": "20", 2023-09-23T19:30:46.848 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_tier_default_cache_hit_set_period": "1200", 2023-09-23T19:30:46.848 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_tier_default_cache_hit_set_search_last_n": "1", 2023-09-23T19:30:46.848 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_tier_default_cache_hit_set_type": "bloom", 2023-09-23T19:30:46.848 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_tier_default_cache_min_read_recency_for_promote": "1", 2023-09-23T19:30:46.848 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_tier_default_cache_min_write_recency_for_promote": "1", 2023-09-23T19:30:46.848 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_tier_default_cache_mode": "writeback", 2023-09-23T19:30:46.848 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_tier_promote_max_bytes_sec": "5242880", 2023-09-23T19:30:46.849 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_tier_promote_max_objects_sec": "25", 2023-09-23T19:30:46.849 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_tracing": "false", 2023-09-23T19:30:46.849 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_use_stale_snap": "false", 2023-09-23T19:30:46.849 INFO:teuthology.orchestra.run.smithi016.stdout: "osd_uuid": "00000000-0000-0000-0000-000000000000", 2023-09-23T19:30:46.849 INFO:teuthology.orchestra.run.smithi016.stdout: "osdc_blkin_trace_all": "false", 2023-09-23T19:30:46.849 INFO:teuthology.orchestra.run.smithi016.stdout: "paxos_kill_at": "0", 2023-09-23T19:30:46.849 INFO:teuthology.orchestra.run.smithi016.stdout: "paxos_max_join_drift": "10", 2023-09-23T19:30:46.849 INFO:teuthology.orchestra.run.smithi016.stdout: "paxos_min": "500", 2023-09-23T19:30:46.849 INFO:teuthology.orchestra.run.smithi016.stdout: "paxos_min_wait": "0.050000", 2023-09-23T19:30:46.850 INFO:teuthology.orchestra.run.smithi016.stdout: "paxos_propose_interval": "1.000000", 2023-09-23T19:30:46.850 INFO:teuthology.orchestra.run.smithi016.stdout: "paxos_service_trim_max": "500", 2023-09-23T19:30:46.850 INFO:teuthology.orchestra.run.smithi016.stdout: "paxos_service_trim_max_multiplier": "20", 2023-09-23T19:30:46.850 INFO:teuthology.orchestra.run.smithi016.stdout: "paxos_service_trim_min": "250", 2023-09-23T19:30:46.850 INFO:teuthology.orchestra.run.smithi016.stdout: "paxos_stash_full_interval": "25", 2023-09-23T19:30:46.850 INFO:teuthology.orchestra.run.smithi016.stdout: "paxos_trim_max": "500", 2023-09-23T19:30:46.850 INFO:teuthology.orchestra.run.smithi016.stdout: "paxos_trim_min": "250", 2023-09-23T19:30:46.850 INFO:teuthology.orchestra.run.smithi016.stdout: "perf": "true", 2023-09-23T19:30:46.851 INFO:teuthology.orchestra.run.smithi016.stdout: "pid_file": "", 2023-09-23T19:30:46.851 INFO:teuthology.orchestra.run.smithi016.stdout: "plugin_crypto_accelerator": "crypto_isal", 2023-09-23T19:30:46.851 INFO:teuthology.orchestra.run.smithi016.stdout: "plugin_dir": "/usr/lib64/ceph", 2023-09-23T19:30:46.851 INFO:teuthology.orchestra.run.smithi016.stdout: "public_addr": "-", 2023-09-23T19:30:46.851 INFO:teuthology.orchestra.run.smithi016.stdout: "public_addrv": "", 2023-09-23T19:30:46.851 INFO:teuthology.orchestra.run.smithi016.stdout: "public_bind_addr": "-", 2023-09-23T19:30:46.851 INFO:teuthology.orchestra.run.smithi016.stdout: "public_network": "172.21.0.0/20", 2023-09-23T19:30:46.851 INFO:teuthology.orchestra.run.smithi016.stdout: "public_network_interface": "", 2023-09-23T19:30:46.851 INFO:teuthology.orchestra.run.smithi016.stdout: "qat_compressor_enabled": "false", 2023-09-23T19:30:46.852 INFO:teuthology.orchestra.run.smithi016.stdout: "qat_compressor_session_max_number": "256", 2023-09-23T19:30:46.852 INFO:teuthology.orchestra.run.smithi016.stdout: "rados_mon_op_timeout": "0", 2023-09-23T19:30:46.852 INFO:teuthology.orchestra.run.smithi016.stdout: "rados_osd_op_timeout": "0", 2023-09-23T19:30:46.852 INFO:teuthology.orchestra.run.smithi016.stdout: "rados_tracing": "false", 2023-09-23T19:30:46.852 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_atime_update_interval": "60", 2023-09-23T19:30:46.852 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_auto_exclusive_lock_until_manual_request": "true", 2023-09-23T19:30:46.852 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_balance_parent_reads": "false", 2023-09-23T19:30:46.852 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_balance_snap_reads": "false", 2023-09-23T19:30:46.852 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_blkin_trace_all": "false", 2023-09-23T19:30:46.852 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_blocklist_expire_seconds": "0", 2023-09-23T19:30:46.853 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_blocklist_on_break_lock": "true", 2023-09-23T19:30:46.853 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_cache": "true", 2023-09-23T19:30:46.853 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_cache_block_writes_upfront": "false", 2023-09-23T19:30:46.853 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_cache_max_dirty": "25165824", 2023-09-23T19:30:46.853 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_cache_max_dirty_age": "1.000000", 2023-09-23T19:30:46.853 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_cache_max_dirty_object": "0", 2023-09-23T19:30:46.853 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_cache_policy": "writearound", 2023-09-23T19:30:46.853 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_cache_size": "33554432", 2023-09-23T19:30:46.853 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_cache_target_dirty": "16777216", 2023-09-23T19:30:46.854 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_cache_writethrough_until_flush": "true", 2023-09-23T19:30:46.854 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_clone_copy_on_read": "false", 2023-09-23T19:30:46.854 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_compression_hint": "none", 2023-09-23T19:30:46.854 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_concurrent_management_ops": "10", 2023-09-23T19:30:46.854 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_config_pool_override_update_timestamp": "0", 2023-09-23T19:30:46.854 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_default_clone_format": "auto", 2023-09-23T19:30:46.854 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_default_data_pool": "", 2023-09-23T19:30:46.855 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_default_features": "61", 2023-09-23T19:30:46.855 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_default_format": "2", 2023-09-23T19:30:46.855 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_default_map_options": "", 2023-09-23T19:30:46.855 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_default_order": "22", 2023-09-23T19:30:46.855 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_default_pool": "rbd", 2023-09-23T19:30:46.855 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_default_snapshot_quiesce_mode": "required", 2023-09-23T19:30:46.855 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_default_stripe_count": "0", 2023-09-23T19:30:46.856 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_default_stripe_unit": "0", 2023-09-23T19:30:46.856 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_disable_zero_copy_writes": "true", 2023-09-23T19:30:46.856 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_discard_granularity_bytes": "65536", 2023-09-23T19:30:46.856 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_discard_on_zeroed_write_same": "true", 2023-09-23T19:30:46.856 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_enable_alloc_hint": "true", 2023-09-23T19:30:46.856 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_invalidate_object_map_on_timeout": "true", 2023-09-23T19:30:46.856 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_io_scheduler": "simple", 2023-09-23T19:30:46.856 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_io_scheduler_simple_max_delay": "0", 2023-09-23T19:30:46.857 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_journal_commit_age": "5.000000", 2023-09-23T19:30:46.857 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_journal_max_concurrent_object_sets": "0", 2023-09-23T19:30:46.857 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_journal_max_payload_bytes": "16384", 2023-09-23T19:30:46.857 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_journal_object_flush_age": "0.000000", 2023-09-23T19:30:46.857 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_journal_object_flush_bytes": "1048576", 2023-09-23T19:30:46.857 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_journal_object_flush_interval": "0", 2023-09-23T19:30:46.857 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_journal_object_max_in_flight_appends": "0", 2023-09-23T19:30:46.857 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_journal_object_writethrough_until_flush": "true", 2023-09-23T19:30:46.857 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_journal_order": "24", 2023-09-23T19:30:46.858 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_journal_pool": "", 2023-09-23T19:30:46.858 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_journal_splay_width": "4", 2023-09-23T19:30:46.858 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_localize_parent_reads": "false", 2023-09-23T19:30:46.858 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_localize_snap_reads": "false", 2023-09-23T19:30:46.858 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirror_concurrent_image_deletions": "1", 2023-09-23T19:30:46.858 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirror_concurrent_image_syncs": "5", 2023-09-23T19:30:46.858 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirror_delete_retry_interval": "30.000000", 2023-09-23T19:30:46.858 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirror_image_perf_stats_prio": "5", 2023-09-23T19:30:46.858 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirror_image_policy_migration_throttle": "300", 2023-09-23T19:30:46.859 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirror_image_policy_rebalance_timeout": "0.000000", 2023-09-23T19:30:46.859 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirror_image_policy_type": "simple", 2023-09-23T19:30:46.859 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirror_image_policy_update_throttle_interval": "1.000000", 2023-09-23T19:30:46.859 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirror_image_state_check_interval": "30", 2023-09-23T19:30:46.859 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirror_journal_commit_age": "5.000000", 2023-09-23T19:30:46.859 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirror_journal_poll_age": "5.000000", 2023-09-23T19:30:46.859 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirror_leader_heartbeat_interval": "5", 2023-09-23T19:30:46.860 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirror_leader_max_acquire_attempts_before_break": "3", 2023-09-23T19:30:46.860 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirror_leader_max_missed_heartbeats": "2", 2023-09-23T19:30:46.860 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirror_memory_autotune": "true", 2023-09-23T19:30:46.860 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirror_memory_base": "805306368", 2023-09-23T19:30:46.860 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirror_memory_cache_autotune_interval": "30.000000", 2023-09-23T19:30:46.860 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirror_memory_cache_min": "134217728", 2023-09-23T19:30:46.860 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirror_memory_cache_resize_interval": "5.000000", 2023-09-23T19:30:46.860 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirror_memory_expected_fragmentation": "0.150000", 2023-09-23T19:30:46.861 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirror_memory_target": "4294967296", 2023-09-23T19:30:46.861 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirror_perf_stats_prio": "5", 2023-09-23T19:30:46.861 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirror_pool_replayers_refresh_interval": "30", 2023-09-23T19:30:46.861 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirror_sync_point_update_age": "30.000000", 2023-09-23T19:30:46.861 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirroring_delete_delay": "0", 2023-09-23T19:30:46.861 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirroring_max_mirroring_snapshots": "5", 2023-09-23T19:30:46.861 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirroring_replay_delay": "0", 2023-09-23T19:30:46.861 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mirroring_resync_after_disconnect": "false", 2023-09-23T19:30:46.861 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_move_parent_to_trash_on_remove": "false", 2023-09-23T19:30:46.862 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_move_to_trash_on_remove": "false", 2023-09-23T19:30:46.862 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_move_to_trash_on_remove_expire_seconds": "0", 2023-09-23T19:30:46.862 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_mtime_update_interval": "60", 2023-09-23T19:30:46.862 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_non_blocking_aio": "true", 2023-09-23T19:30:46.862 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_op_thread_timeout": "60", 2023-09-23T19:30:46.862 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_op_threads": "1", 2023-09-23T19:30:46.862 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_parent_cache_enabled": "false", 2023-09-23T19:30:46.862 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_persistent_cache_mode": "disabled", 2023-09-23T19:30:46.862 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_persistent_cache_path": "/tmp", 2023-09-23T19:30:46.862 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_persistent_cache_size": "1073741824", 2023-09-23T19:30:46.862 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_plugins": "", 2023-09-23T19:30:46.863 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_qos_bps_burst": "0", 2023-09-23T19:30:46.863 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_qos_bps_burst_seconds": "1", 2023-09-23T19:30:46.863 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_qos_bps_limit": "0", 2023-09-23T19:30:46.863 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_qos_exclude_ops": "0", 2023-09-23T19:30:46.863 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_qos_iops_burst": "0", 2023-09-23T19:30:46.863 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_qos_iops_burst_seconds": "1", 2023-09-23T19:30:46.863 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_qos_iops_limit": "0", 2023-09-23T19:30:46.863 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_qos_read_bps_burst": "0", 2023-09-23T19:30:46.863 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_qos_read_bps_burst_seconds": "1", 2023-09-23T19:30:46.863 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_qos_read_bps_limit": "0", 2023-09-23T19:30:46.864 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_qos_read_iops_burst": "0", 2023-09-23T19:30:46.864 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_qos_read_iops_burst_seconds": "1", 2023-09-23T19:30:46.864 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_qos_read_iops_limit": "0", 2023-09-23T19:30:46.864 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_qos_schedule_tick_min": "50", 2023-09-23T19:30:46.864 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_qos_write_bps_burst": "0", 2023-09-23T19:30:46.864 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_qos_write_bps_burst_seconds": "1", 2023-09-23T19:30:46.864 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_qos_write_bps_limit": "0", 2023-09-23T19:30:46.864 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_qos_write_iops_burst": "0", 2023-09-23T19:30:46.865 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_qos_write_iops_burst_seconds": "1", 2023-09-23T19:30:46.865 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_qos_write_iops_limit": "0", 2023-09-23T19:30:46.865 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_quiesce_notification_attempts": "10", 2023-09-23T19:30:46.865 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_read_from_replica_policy": "default", 2023-09-23T19:30:46.865 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_readahead_disable_after_bytes": "52428800", 2023-09-23T19:30:46.865 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_readahead_max_bytes": "524288", 2023-09-23T19:30:46.865 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_readahead_trigger_requests": "10", 2023-09-23T19:30:46.866 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_request_timed_out_seconds": "30", 2023-09-23T19:30:46.866 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_skip_partial_discard": "true", 2023-09-23T19:30:46.866 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_sparse_read_threshold_bytes": "65536", 2023-09-23T19:30:46.866 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_tracing": "false", 2023-09-23T19:30:46.866 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_validate_names": "true", 2023-09-23T19:30:46.866 INFO:teuthology.orchestra.run.smithi016.stdout: "rbd_validate_pool": "true", 2023-09-23T19:30:46.866 INFO:teuthology.orchestra.run.smithi016.stdout: "restapi_base_url": "", 2023-09-23T19:30:46.866 INFO:teuthology.orchestra.run.smithi016.stdout: "restapi_log_level": "", 2023-09-23T19:30:46.867 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_acl_grants_max_num": "100", 2023-09-23T19:30:46.867 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_admin_entry": "admin", 2023-09-23T19:30:46.867 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_allow_notification_secrets_in_cleartext": "false", 2023-09-23T19:30:46.867 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_backend_store": "rados", 2023-09-23T19:30:46.867 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_barbican_url": "", 2023-09-23T19:30:46.867 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_beast_enable_async": "true", 2023-09-23T19:30:46.867 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_bucket_default_quota_max_objects": "-1", 2023-09-23T19:30:46.868 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_bucket_default_quota_max_size": "-1", 2023-09-23T19:30:46.868 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_bucket_index_max_aio": "128", 2023-09-23T19:30:46.868 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_bucket_index_transaction_instrumentation": "false", 2023-09-23T19:30:46.868 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_bucket_quota_cache_size": "10000", 2023-09-23T19:30:46.868 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_bucket_quota_ttl": "600", 2023-09-23T19:30:46.868 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_bucket_sync_spawn_window": "20", 2023-09-23T19:30:46.868 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_cache_enabled": "true", 2023-09-23T19:30:46.868 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_cache_expiry_interval": "900", 2023-09-23T19:30:46.869 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_cache_lru_size": "10000", 2023-09-23T19:30:46.869 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_config_store": "rados", 2023-09-23T19:30:46.869 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_content_length_compat": "false", 2023-09-23T19:30:46.869 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_copy_obj_progress": "true", 2023-09-23T19:30:46.869 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_copy_obj_progress_every_bytes": "1048576", 2023-09-23T19:30:46.869 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_cors_rules_max_num": "100", 2023-09-23T19:30:46.869 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_cross_domain_policy": "", 2023-09-23T19:30:46.869 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_default_encryption_key": "", 2023-09-23T19:30:46.870 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_kmip_addr": "", 2023-09-23T19:30:46.870 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_kmip_ca_path": "", 2023-09-23T19:30:46.870 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_kmip_client_cert": "", 2023-09-23T19:30:46.870 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_kmip_client_key": "", 2023-09-23T19:30:46.870 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_kmip_kms_key_template": "", 2023-09-23T19:30:46.870 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_kmip_password": "", 2023-09-23T19:30:46.870 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_kmip_s3_key_template": "$keyid", 2023-09-23T19:30:46.870 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_kmip_username": "", 2023-09-23T19:30:46.870 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_require_ssl": "true", 2023-09-23T19:30:46.871 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_s3_kms_backend": "barbican", 2023-09-23T19:30:46.871 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_s3_kms_encryption_keys": "", 2023-09-23T19:30:46.871 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_sse_s3_backend": "vault", 2023-09-23T19:30:46.871 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_sse_s3_key_template": "%bucket_id", 2023-09-23T19:30:46.871 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_sse_s3_vault_addr": "", 2023-09-23T19:30:46.871 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_sse_s3_vault_auth": "token", 2023-09-23T19:30:46.871 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_sse_s3_vault_namespace": "", 2023-09-23T19:30:46.871 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_sse_s3_vault_prefix": "", 2023-09-23T19:30:46.871 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_sse_s3_vault_secret_engine": "transit", 2023-09-23T19:30:46.871 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_sse_s3_vault_ssl_cacert": "", 2023-09-23T19:30:46.871 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_sse_s3_vault_ssl_clientcert": "", 2023-09-23T19:30:46.871 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_sse_s3_vault_ssl_clientkey": "", 2023-09-23T19:30:46.872 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_sse_s3_vault_token_file": "", 2023-09-23T19:30:46.872 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_sse_s3_vault_verify_ssl": "true", 2023-09-23T19:30:46.872 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_suppress_logs": "true", 2023-09-23T19:30:46.872 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_vault_addr": "", 2023-09-23T19:30:46.872 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_vault_auth": "token", 2023-09-23T19:30:46.872 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_vault_namespace": "", 2023-09-23T19:30:46.872 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_vault_prefix": "", 2023-09-23T19:30:46.872 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_vault_secret_engine": "transit", 2023-09-23T19:30:46.872 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_vault_ssl_cacert": "", 2023-09-23T19:30:46.872 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_vault_ssl_clientcert": "", 2023-09-23T19:30:46.872 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_vault_ssl_clientkey": "", 2023-09-23T19:30:46.872 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_vault_token_file": "", 2023-09-23T19:30:46.873 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_crypt_vault_verify_ssl": "true", 2023-09-23T19:30:46.873 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_curl_buffersize": "524288", 2023-09-23T19:30:46.873 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_curl_low_speed_limit": "1024", 2023-09-23T19:30:46.873 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_curl_low_speed_time": "300", 2023-09-23T19:30:46.873 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_curl_tcp_keepalive": "0", 2023-09-23T19:30:46.873 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_curl_wait_timeout_ms": "1000", 2023-09-23T19:30:46.873 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_d3n_l1_datacache_persistent_path": "/tmp/rgw_datacache/", 2023-09-23T19:30:46.873 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_d3n_l1_datacache_size": "1073741824", 2023-09-23T19:30:46.873 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_d3n_l1_evict_cache_on_start": "true", 2023-09-23T19:30:46.873 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_d3n_l1_eviction_policy": "lru", 2023-09-23T19:30:46.873 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_d3n_l1_fadvise": "4", 2023-09-23T19:30:46.873 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_d3n_l1_local_datacache_enabled": "false", 2023-09-23T19:30:46.874 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_d3n_libaio_aio_num": "64", 2023-09-23T19:30:46.874 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_d3n_libaio_aio_threads": "20", 2023-09-23T19:30:46.874 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_data": "/var/lib/ceph/radosgw/ceph-a", 2023-09-23T19:30:46.874 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_data_log_changes_size": "1000", 2023-09-23T19:30:46.874 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_data_log_num_shards": "128", 2023-09-23T19:30:46.874 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_data_log_obj_prefix": "data_log", 2023-09-23T19:30:46.874 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_data_log_window": "30", 2023-09-23T19:30:46.874 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_data_notify_interval_msec": "0", 2023-09-23T19:30:46.874 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_data_sync_poll_interval": "20", 2023-09-23T19:30:46.874 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_data_sync_spawn_window": "20", 2023-09-23T19:30:46.874 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_debug_inject_olh_cancel_modification_err": "false", 2023-09-23T19:30:46.875 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_debug_inject_set_olh_err": "0", 2023-09-23T19:30:46.875 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_default_data_log_backing": "fifo", 2023-09-23T19:30:46.875 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_default_realm_info_oid": "default.realm", 2023-09-23T19:30:46.875 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_default_region_info_oid": "default.region", 2023-09-23T19:30:46.875 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_default_zone_info_oid": "default.zone", 2023-09-23T19:30:46.875 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_default_zonegroup_info_oid": "default.zonegroup", 2023-09-23T19:30:46.875 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_defer_to_bucket_acls": "", 2023-09-23T19:30:46.875 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_delete_multi_obj_max_num": "1000", 2023-09-23T19:30:46.875 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_dmclock_admin_lim": "0.000000", 2023-09-23T19:30:46.875 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_dmclock_admin_res": "100.000000", 2023-09-23T19:30:46.877 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_dmclock_admin_wgt": "100.000000", 2023-09-23T19:30:46.877 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_dmclock_auth_lim": "0.000000", 2023-09-23T19:30:46.877 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_dmclock_auth_res": "200.000000", 2023-09-23T19:30:46.877 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_dmclock_auth_wgt": "100.000000", 2023-09-23T19:30:46.877 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_dmclock_data_lim": "0.000000", 2023-09-23T19:30:46.877 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_dmclock_data_res": "500.000000", 2023-09-23T19:30:46.877 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_dmclock_data_wgt": "500.000000", 2023-09-23T19:30:46.877 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_dmclock_metadata_lim": "0.000000", 2023-09-23T19:30:46.877 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_dmclock_metadata_res": "500.000000", 2023-09-23T19:30:46.877 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_dmclock_metadata_wgt": "500.000000", 2023-09-23T19:30:46.878 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_dns_name": "", 2023-09-23T19:30:46.878 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_dns_s3website_name": "", 2023-09-23T19:30:46.878 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_dynamic_resharding": "true", 2023-09-23T19:30:46.878 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_enable_apis": "s3, s3website, swift, swift_auth, admin, sts, iam, notifications", 2023-09-23T19:30:46.878 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_enable_gc_threads": "true", 2023-09-23T19:30:46.878 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_enable_lc_threads": "true", 2023-09-23T19:30:46.878 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_enable_ops_log": "false", 2023-09-23T19:30:46.878 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_enable_quota_threads": "true", 2023-09-23T19:30:46.878 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_enable_static_website": "false", 2023-09-23T19:30:46.878 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_enable_usage_log": "false", 2023-09-23T19:30:46.878 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_enforce_swift_acls": "true", 2023-09-23T19:30:46.878 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_exit_timeout_secs": "120", 2023-09-23T19:30:46.879 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_expose_bucket": "false", 2023-09-23T19:30:46.879 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_extended_http_attrs": "", 2023-09-23T19:30:46.879 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_filter": "none", 2023-09-23T19:30:46.879 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_frontend_defaults": "beast ssl_certificate=config://rgw/cert/$realm/$zone.crt ssl_private_key=config://rgw/cert/$realm/$zone.key", 2023-09-23T19:30:46.879 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_frontends": "beast port=7480", 2023-09-23T19:30:46.879 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_gc_max_concurrent_io": "10", 2023-09-23T19:30:46.879 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_gc_max_deferred": "50", 2023-09-23T19:30:46.879 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_gc_max_deferred_entries_size": "3072", 2023-09-23T19:30:46.879 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_gc_max_objs": "32", 2023-09-23T19:30:46.879 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_gc_max_queue_size": "134213632", 2023-09-23T19:30:46.879 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_gc_max_trim_chunk": "16", 2023-09-23T19:30:46.879 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_gc_obj_min_wait": "7200", 2023-09-23T19:30:46.880 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_gc_processor_max_time": "3600", 2023-09-23T19:30:46.880 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_gc_processor_period": "3600", 2023-09-23T19:30:46.880 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_get_obj_max_req_size": "4194304", 2023-09-23T19:30:46.880 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_get_obj_window_size": "16777216", 2023-09-23T19:30:46.880 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_healthcheck_disabling_path": "", 2023-09-23T19:30:46.880 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_ignore_get_invalid_range": "false", 2023-09-23T19:30:46.880 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_init_timeout": "300", 2023-09-23T19:30:46.880 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_inject_notify_timeout_probability": "0.000000", 2023-09-23T19:30:46.880 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_json_config": "/var/lib/ceph/radosgw/config.json", 2023-09-23T19:30:46.880 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_keystone_accepted_admin_roles": "", 2023-09-23T19:30:46.880 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_keystone_accepted_roles": "Member, admin", 2023-09-23T19:30:46.881 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_keystone_admin_domain": "", 2023-09-23T19:30:46.881 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_keystone_admin_password": "", 2023-09-23T19:30:46.881 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_keystone_admin_password_path": "", 2023-09-23T19:30:46.881 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_keystone_admin_project": "", 2023-09-23T19:30:46.881 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_keystone_admin_tenant": "", 2023-09-23T19:30:46.881 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_keystone_admin_token": "", 2023-09-23T19:30:46.881 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_keystone_admin_token_path": "", 2023-09-23T19:30:46.881 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_keystone_admin_user": "", 2023-09-23T19:30:46.881 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_keystone_api_version": "2", 2023-09-23T19:30:46.882 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_keystone_barbican_domain": "", 2023-09-23T19:30:46.882 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_keystone_barbican_password": "", 2023-09-23T19:30:46.882 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_keystone_barbican_project": "", 2023-09-23T19:30:46.882 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_keystone_barbican_tenant": "", 2023-09-23T19:30:46.882 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_keystone_barbican_user": "", 2023-09-23T19:30:46.882 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_keystone_expired_token_cache_expiration": "3600", 2023-09-23T19:30:46.882 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_keystone_implicit_tenants": "false", 2023-09-23T19:30:46.882 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_keystone_service_token_accepted_roles": "admin", 2023-09-23T19:30:46.882 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_keystone_service_token_enabled": "false", 2023-09-23T19:30:46.882 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_keystone_token_cache_size": "10000", 2023-09-23T19:30:46.882 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_keystone_url": "", 2023-09-23T19:30:46.883 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_keystone_verify_ssl": "true", 2023-09-23T19:30:46.883 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_lc_debug_interval": "-1", 2023-09-23T19:30:46.883 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_lc_lock_max_time": "90", 2023-09-23T19:30:46.883 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_lc_max_objs": "32", 2023-09-23T19:30:46.883 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_lc_max_rules": "1000", 2023-09-23T19:30:46.883 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_lc_max_worker": "3", 2023-09-23T19:30:46.883 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_lc_max_wp_worker": "3", 2023-09-23T19:30:46.883 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_lc_thread_delay": "0", 2023-09-23T19:30:46.884 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_ldap_binddn": "uid=admin,cn=users,dc=example,dc=com", 2023-09-23T19:30:46.884 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_ldap_dnattr": "uid", 2023-09-23T19:30:46.884 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_ldap_searchdn": "cn=users,cn=accounts,dc=example,dc=com", 2023-09-23T19:30:46.884 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_ldap_searchfilter": "", 2023-09-23T19:30:46.884 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_ldap_secret": "/etc/openldap/secret", 2023-09-23T19:30:46.884 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_ldap_uri": "ldaps://", 2023-09-23T19:30:46.884 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_lifecycle_work_time": "00:00-06:00", 2023-09-23T19:30:46.884 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_list_bucket_min_readahead": "1000", 2023-09-23T19:30:46.884 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_list_buckets_max_chunk": "1000", 2023-09-23T19:30:46.884 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_log_http_headers": "", 2023-09-23T19:30:46.884 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_log_nonexistent_bucket": "false", 2023-09-23T19:30:46.885 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_log_object_name": "%Y-%m-%d-%H-%i-%n", 2023-09-23T19:30:46.885 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_log_object_name_utc": "false", 2023-09-23T19:30:46.885 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_luarocks_location": "/tmp/luarocks", 2023-09-23T19:30:46.885 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_max_attr_name_len": "0", 2023-09-23T19:30:46.885 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_max_attr_size": "0", 2023-09-23T19:30:46.885 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_max_attrs_num_in_req": "0", 2023-09-23T19:30:46.885 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_max_chunk_size": "4194304", 2023-09-23T19:30:46.885 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_max_concurrent_requests": "1024", 2023-09-23T19:30:46.886 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_max_copy_obj_concurrent_io": "10", 2023-09-23T19:30:46.886 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_max_dynamic_shards": "1999", 2023-09-23T19:30:46.886 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_max_listing_results": "1000", 2023-09-23T19:30:46.886 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_max_notify_retries": "10", 2023-09-23T19:30:46.886 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_max_objs_per_shard": "100000", 2023-09-23T19:30:46.886 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_max_put_param_size": "1048576", 2023-09-23T19:30:46.886 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_max_put_size": "5368709120", 2023-09-23T19:30:46.886 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_max_slo_entries": "1000", 2023-09-23T19:30:46.886 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_md_log_max_shards": "64", 2023-09-23T19:30:46.886 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_md_notify_interval_msec": "200", 2023-09-23T19:30:46.886 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_meta_sync_poll_interval": "20", 2023-09-23T19:30:46.887 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_meta_sync_spawn_window": "20", 2023-09-23T19:30:46.887 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_mime_types_file": "/etc/mime.types", 2023-09-23T19:30:46.887 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_mp_lock_max_time": "600", 2023-09-23T19:30:46.887 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_multi_obj_del_max_aio": "16", 2023-09-23T19:30:46.887 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_multipart_min_part_size": "5242880", 2023-09-23T19:30:46.887 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_multipart_part_upload_limit": "10000", 2023-09-23T19:30:46.887 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_nfs_fhcache_partitions": "3", 2023-09-23T19:30:46.887 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_nfs_fhcache_size": "2017", 2023-09-23T19:30:46.888 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_nfs_frontends": "rgw-nfs", 2023-09-23T19:30:46.888 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_nfs_lru_lane_hiwat": "911", 2023-09-23T19:30:46.888 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_nfs_lru_lanes": "5", 2023-09-23T19:30:46.888 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_nfs_max_gc": "300", 2023-09-23T19:30:46.888 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_nfs_namespace_expire_secs": "300", 2023-09-23T19:30:46.888 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_nfs_run_gc_threads": "false", 2023-09-23T19:30:46.888 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_nfs_run_lc_threads": "false", 2023-09-23T19:30:46.888 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_nfs_run_quota_threads": "false", 2023-09-23T19:30:46.889 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_nfs_run_sync_thread": "false", 2023-09-23T19:30:46.889 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_nfs_s3_fast_attrs": "false", 2023-09-23T19:30:46.889 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_nfs_write_completion_interval_s": "10", 2023-09-23T19:30:46.889 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_num_async_rados_threads": "32", 2023-09-23T19:30:46.889 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_num_control_oids": "8", 2023-09-23T19:30:46.889 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_numa_node": "-1", 2023-09-23T19:30:46.889 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_obj_stripe_size": "4194304", 2023-09-23T19:30:46.889 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_obj_tombstone_cache_size": "1000", 2023-09-23T19:30:46.889 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_objexp_chunk_size": "100", 2023-09-23T19:30:46.889 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_objexp_gc_interval": "600", 2023-09-23T19:30:46.890 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_objexp_hints_num_shards": "127", 2023-09-23T19:30:46.890 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_olh_pending_timeout_sec": "3600", 2023-09-23T19:30:46.890 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_op_thread_suicide_timeout": "0", 2023-09-23T19:30:46.890 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_op_thread_timeout": "600", 2023-09-23T19:30:46.890 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_op_tracing": "false", 2023-09-23T19:30:46.890 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_opa_token": "", 2023-09-23T19:30:46.890 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_opa_url": "", 2023-09-23T19:30:46.890 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_opa_verify_ssl": "true", 2023-09-23T19:30:46.891 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_ops_log_data_backlog": "5242880", 2023-09-23T19:30:46.891 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_ops_log_file_path": "/var/log/ceph/ops-log-ceph-mon.a.log", 2023-09-23T19:30:46.891 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_ops_log_rados": "false", 2023-09-23T19:30:46.891 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_ops_log_socket_path": "", 2023-09-23T19:30:46.891 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_override_bucket_index_max_shards": "0", 2023-09-23T19:30:46.891 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_pending_bucket_index_op_expiration": "120", 2023-09-23T19:30:46.891 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_period_latest_epoch_info_oid": ".latest_epoch", 2023-09-23T19:30:46.891 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_period_push_interval": "2.000000", 2023-09-23T19:30:46.891 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_period_push_interval_max": "30.000000", 2023-09-23T19:30:46.892 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_period_root_pool": ".rgw.root", 2023-09-23T19:30:46.892 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_policy_reject_invalid_principals": "true", 2023-09-23T19:30:46.892 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_print_continue": "true", 2023-09-23T19:30:46.892 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_print_prohibited_content_length": "false", 2023-09-23T19:30:46.892 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_put_obj_max_window_size": "67108864", 2023-09-23T19:30:46.892 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_put_obj_min_window_size": "16777216", 2023-09-23T19:30:46.892 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_rados_pool_autoscale_bias": "4.000000", 2023-09-23T19:30:46.892 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_rados_pool_recovery_priority": "5", 2023-09-23T19:30:46.893 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_rados_tracing": "false", 2023-09-23T19:30:46.893 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_realm": "", 2023-09-23T19:30:46.893 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_realm_id": "", 2023-09-23T19:30:46.893 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_realm_root_pool": ".rgw.root", 2023-09-23T19:30:46.893 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_region": "", 2023-09-23T19:30:46.893 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_region_root_pool": ".rgw.root", 2023-09-23T19:30:46.893 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_relaxed_region_enforcement": "false", 2023-09-23T19:30:46.894 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_relaxed_s3_bucket_names": "false", 2023-09-23T19:30:46.894 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_remote_addr_param": "REMOTE_ADDR", 2023-09-23T19:30:46.894 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_request_uri": "", 2023-09-23T19:30:46.894 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_reshard_batch_size": "64", 2023-09-23T19:30:46.894 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_reshard_bucket_lock_duration": "360", 2023-09-23T19:30:46.894 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_reshard_max_aio": "128", 2023-09-23T19:30:46.894 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_reshard_num_logs": "16", 2023-09-23T19:30:46.894 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_reshard_thread_interval": "600", 2023-09-23T19:30:46.894 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_resolve_cname": "false", 2023-09-23T19:30:46.894 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_rest_getusage_op_compat": "false", 2023-09-23T19:30:46.895 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_run_sync_thread": "true", 2023-09-23T19:30:46.895 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_s3_auth_order": "sts, external, local", 2023-09-23T19:30:46.895 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_s3_auth_use_keystone": "false", 2023-09-23T19:30:46.895 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_s3_auth_use_ldap": "false", 2023-09-23T19:30:46.895 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_s3_auth_use_rados": "true", 2023-09-23T19:30:46.895 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_s3_auth_use_sts": "false", 2023-09-23T19:30:46.895 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_s3_client_max_sig_ver": "-1", 2023-09-23T19:30:46.895 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_s3_success_create_obj_status": "0", 2023-09-23T19:30:46.896 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_safe_max_objects_per_shard": "102400", 2023-09-23T19:30:46.896 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_scheduler_type": "throttler", 2023-09-23T19:30:46.896 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_script_uri": "", 2023-09-23T19:30:46.896 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_service_provider_name": "", 2023-09-23T19:30:46.896 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_shard_warning_threshold": "90.000000", 2023-09-23T19:30:46.896 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_sts_client_id": "", 2023-09-23T19:30:46.896 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_sts_client_secret": "", 2023-09-23T19:30:46.896 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_sts_entry": "sts", 2023-09-23T19:30:46.896 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_sts_key": "sts", 2023-09-23T19:30:46.896 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_sts_max_session_duration": "43200", 2023-09-23T19:30:46.896 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_sts_min_session_duration": "900", 2023-09-23T19:30:46.897 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_sts_token_introspection_url": "", 2023-09-23T19:30:46.897 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_swift_account_in_url": "false", 2023-09-23T19:30:46.897 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_swift_auth_entry": "auth", 2023-09-23T19:30:46.897 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_swift_auth_url": "", 2023-09-23T19:30:46.897 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_swift_custom_header": "", 2023-09-23T19:30:46.897 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_swift_enforce_content_length": "false", 2023-09-23T19:30:46.897 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_swift_need_stats": "true", 2023-09-23T19:30:46.897 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_swift_tenant_name": "", 2023-09-23T19:30:46.897 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_swift_token_expiration": "86400", 2023-09-23T19:30:46.898 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_swift_url": "", 2023-09-23T19:30:46.898 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_swift_url_prefix": "swift", 2023-09-23T19:30:46.898 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_swift_versioning_enabled": "false", 2023-09-23T19:30:46.898 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_sync_data_full_inject_err_probability": "0.000000", 2023-09-23T19:30:46.898 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_sync_data_inject_err_probability": "0.000000", 2023-09-23T19:30:46.898 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_sync_lease_period": "120", 2023-09-23T19:30:46.898 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_sync_log_trim_concurrent_buckets": "4", 2023-09-23T19:30:46.898 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_sync_log_trim_interval": "1200", 2023-09-23T19:30:46.898 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_sync_log_trim_max_buckets": "16", 2023-09-23T19:30:46.898 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_sync_log_trim_min_cold_buckets": "4", 2023-09-23T19:30:46.899 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_sync_meta_inject_err_probability": "0.000000", 2023-09-23T19:30:46.899 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_sync_obj_etag_verify": "false", 2023-09-23T19:30:46.899 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_sync_trace_history_size": "4096", 2023-09-23T19:30:46.899 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_sync_trace_per_node_log_size": "32", 2023-09-23T19:30:46.899 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_sync_trace_servicemap_update_interval": "10", 2023-09-23T19:30:46.899 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_thread_pool_size": "512", 2023-09-23T19:30:46.899 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_torrent_comment": "", 2023-09-23T19:30:46.899 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_torrent_createby": "", 2023-09-23T19:30:46.900 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_torrent_encoding": "", 2023-09-23T19:30:46.900 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_torrent_flag": "false", 2023-09-23T19:30:46.900 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_torrent_origin": "", 2023-09-23T19:30:46.900 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_torrent_sha_unit": "524288", 2023-09-23T19:30:46.900 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_torrent_tracker": "", 2023-09-23T19:30:46.900 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_trust_forwarded_https": "false", 2023-09-23T19:30:46.900 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_usage_log_flush_threshold": "1024", 2023-09-23T19:30:46.900 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_usage_log_tick_interval": "30", 2023-09-23T19:30:46.900 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_usage_max_shards": "32", 2023-09-23T19:30:46.901 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_usage_max_user_shards": "1", 2023-09-23T19:30:46.901 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_use_opa_authz": "false", 2023-09-23T19:30:46.901 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_user_default_quota_max_objects": "-1", 2023-09-23T19:30:46.901 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_user_default_quota_max_size": "-1", 2023-09-23T19:30:46.901 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_user_max_buckets": "1000", 2023-09-23T19:30:46.901 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_user_policies_max_num": "100", 2023-09-23T19:30:46.901 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_user_quota_bucket_sync_interval": "180", 2023-09-23T19:30:46.901 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_user_quota_sync_idle_users": "false", 2023-09-23T19:30:46.902 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_user_quota_sync_interval": "86400", 2023-09-23T19:30:46.902 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_user_quota_sync_wait_time": "86400", 2023-09-23T19:30:46.902 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_user_unique_email": "true", 2023-09-23T19:30:46.902 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_verify_ssl": "true", 2023-09-23T19:30:46.902 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_website_routing_rules_max_num": "50", 2023-09-23T19:30:46.902 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_zone": "", 2023-09-23T19:30:46.902 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_zone_id": "", 2023-09-23T19:30:46.902 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_zone_root_pool": ".rgw.root", 2023-09-23T19:30:46.903 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_zonegroup": "", 2023-09-23T19:30:46.903 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_zonegroup_id": "", 2023-09-23T19:30:46.903 INFO:teuthology.orchestra.run.smithi016.stdout: "rgw_zonegroup_root_pool": ".rgw.root", 2023-09-23T19:30:46.903 INFO:teuthology.orchestra.run.smithi016.stdout: "rgwlc_auto_session_clear": "true", 2023-09-23T19:30:46.903 INFO:teuthology.orchestra.run.smithi016.stdout: "rgwlc_skip_bucket_step": "false", 2023-09-23T19:30:46.903 INFO:teuthology.orchestra.run.smithi016.stdout: "rocksdb_block_size": "4096", 2023-09-23T19:30:46.903 INFO:teuthology.orchestra.run.smithi016.stdout: "rocksdb_bloom_bits_per_key": "20", 2023-09-23T19:30:46.903 INFO:teuthology.orchestra.run.smithi016.stdout: "rocksdb_cache_index_and_filter_blocks": "true", 2023-09-23T19:30:46.904 INFO:teuthology.orchestra.run.smithi016.stdout: "rocksdb_cache_index_and_filter_blocks_with_high_priority": "false", 2023-09-23T19:30:46.904 INFO:teuthology.orchestra.run.smithi016.stdout: "rocksdb_cache_row_ratio": "0.000000", 2023-09-23T19:30:46.904 INFO:teuthology.orchestra.run.smithi016.stdout: "rocksdb_cache_shard_bits": "4", 2023-09-23T19:30:46.904 INFO:teuthology.orchestra.run.smithi016.stdout: "rocksdb_cache_size": "536870912", 2023-09-23T19:30:46.904 INFO:teuthology.orchestra.run.smithi016.stdout: "rocksdb_cache_type": "binned_lru", 2023-09-23T19:30:46.904 INFO:teuthology.orchestra.run.smithi016.stdout: "rocksdb_cf_compact_on_deletion": "true", 2023-09-23T19:30:46.904 INFO:teuthology.orchestra.run.smithi016.stdout: "rocksdb_cf_compact_on_deletion_sliding_window": "32768", 2023-09-23T19:30:46.904 INFO:teuthology.orchestra.run.smithi016.stdout: "rocksdb_cf_compact_on_deletion_trigger": "16384", 2023-09-23T19:30:46.905 INFO:teuthology.orchestra.run.smithi016.stdout: "rocksdb_collect_compaction_stats": "false", 2023-09-23T19:30:46.905 INFO:teuthology.orchestra.run.smithi016.stdout: "rocksdb_collect_extended_stats": "false", 2023-09-23T19:30:46.905 INFO:teuthology.orchestra.run.smithi016.stdout: "rocksdb_collect_memory_stats": "false", 2023-09-23T19:30:46.905 INFO:teuthology.orchestra.run.smithi016.stdout: "rocksdb_delete_range_threshold": "1048576", 2023-09-23T19:30:46.905 INFO:teuthology.orchestra.run.smithi016.stdout: "rocksdb_index_type": "binary_search", 2023-09-23T19:30:46.905 INFO:teuthology.orchestra.run.smithi016.stdout: "rocksdb_log_to_ceph_log": "true", 2023-09-23T19:30:46.905 INFO:teuthology.orchestra.run.smithi016.stdout: "rocksdb_metadata_block_size": "4096", 2023-09-23T19:30:46.905 INFO:teuthology.orchestra.run.smithi016.stdout: "rocksdb_partition_filters": "false", 2023-09-23T19:30:46.906 INFO:teuthology.orchestra.run.smithi016.stdout: "rocksdb_perf": "false", 2023-09-23T19:30:46.906 INFO:teuthology.orchestra.run.smithi016.stdout: "rocksdb_pin_l0_filter_and_index_blocks_in_cache": "false", 2023-09-23T19:30:46.906 INFO:teuthology.orchestra.run.smithi016.stdout: "rotating_keys_bootstrap_timeout": "30", 2023-09-23T19:30:46.906 INFO:teuthology.orchestra.run.smithi016.stdout: "rotating_keys_renewal_timeout": "10", 2023-09-23T19:30:46.906 INFO:teuthology.orchestra.run.smithi016.stdout: "run_dir": "/var/run/ceph", 2023-09-23T19:30:46.906 INFO:teuthology.orchestra.run.smithi016.stdout: "seastore_block_create": "true", 2023-09-23T19:30:46.906 INFO:teuthology.orchestra.run.smithi016.stdout: "seastore_cache_lru_size": "67108864", 2023-09-23T19:30:46.906 INFO:teuthology.orchestra.run.smithi016.stdout: "seastore_cbjournal_size": "5368709120", 2023-09-23T19:30:46.906 INFO:teuthology.orchestra.run.smithi016.stdout: "seastore_default_max_object_size": "16777216", 2023-09-23T19:30:46.907 INFO:teuthology.orchestra.run.smithi016.stdout: "seastore_default_object_metadata_reservation": "16777216", 2023-09-23T19:30:46.907 INFO:teuthology.orchestra.run.smithi016.stdout: "seastore_device_size": "53687091200", 2023-09-23T19:30:46.907 INFO:teuthology.orchestra.run.smithi016.stdout: "seastore_journal_batch_capacity": "16", 2023-09-23T19:30:46.907 INFO:teuthology.orchestra.run.smithi016.stdout: "seastore_journal_batch_flush_size": "16777216", 2023-09-23T19:30:46.907 INFO:teuthology.orchestra.run.smithi016.stdout: "seastore_journal_batch_preferred_fullness": "0.950000", 2023-09-23T19:30:46.907 INFO:teuthology.orchestra.run.smithi016.stdout: "seastore_journal_iodepth_limit": "5", 2023-09-23T19:30:46.907 INFO:teuthology.orchestra.run.smithi016.stdout: "seastore_main_device_type": "SSD", 2023-09-23T19:30:46.907 INFO:teuthology.orchestra.run.smithi016.stdout: "seastore_max_concurrent_transactions": "8", 2023-09-23T19:30:46.907 INFO:teuthology.orchestra.run.smithi016.stdout: "seastore_multiple_tiers_default_evict_ratio": "0.600000", 2023-09-23T19:30:46.907 INFO:teuthology.orchestra.run.smithi016.stdout: "seastore_multiple_tiers_fast_evict_ratio": "0.700000", 2023-09-23T19:30:46.907 INFO:teuthology.orchestra.run.smithi016.stdout: "seastore_multiple_tiers_stop_evict_ratio": "0.500000", 2023-09-23T19:30:46.908 INFO:teuthology.orchestra.run.smithi016.stdout: "seastore_obj_data_write_amplification": "1.250000", 2023-09-23T19:30:46.908 INFO:teuthology.orchestra.run.smithi016.stdout: "seastore_segment_size": "67108864", 2023-09-23T19:30:46.908 INFO:teuthology.orchestra.run.smithi016.stdout: "set_keepcaps": "false", 2023-09-23T19:30:46.908 INFO:teuthology.orchestra.run.smithi016.stdout: "setgroup": "ceph", 2023-09-23T19:30:46.908 INFO:teuthology.orchestra.run.smithi016.stdout: "setuser": "ceph", 2023-09-23T19:30:46.908 INFO:teuthology.orchestra.run.smithi016.stdout: "setuser_match_path": "", 2023-09-23T19:30:46.908 INFO:teuthology.orchestra.run.smithi016.stdout: "target_max_misplaced_ratio": "0.050000", 2023-09-23T19:30:46.908 INFO:teuthology.orchestra.run.smithi016.stdout: "thp": "false", 2023-09-23T19:30:46.908 INFO:teuthology.orchestra.run.smithi016.stdout: "threadpool_default_timeout": "60", 2023-09-23T19:30:46.908 INFO:teuthology.orchestra.run.smithi016.stdout: "threadpool_empty_queue_max_wait": "2", 2023-09-23T19:30:46.908 INFO:teuthology.orchestra.run.smithi016.stdout: "throttler_perf_counter": "true" 2023-09-23T19:30:46.908 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-09-23T19:30:47.494 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=0 2023-09-23T19:30:47.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:30:47 smithi016 bash[75004]: cluster 2023-09-23T19:30:45.920368+0000 mgr.y (mgr.25325) 693 : cluster 0 pgmap v586: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:47.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:30:47 smithi016 bash[77995]: cluster 2023-09-23T19:30:45.920368+0000 mgr.y (mgr.25325) 693 : cluster 0 pgmap v586: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:47.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:30:47 smithi092 bash[58855]: cluster 2023-09-23T19:30:45.920368+0000 mgr.y (mgr.25325) 693 : cluster 0 pgmap v586: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:49.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:30:49 smithi016 bash[75004]: cluster 2023-09-23T19:30:47.921052+0000 mgr.y (mgr.25325) 694 : cluster 0 pgmap v587: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:49.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:30:49 smithi016 bash[75004]: audit 2023-09-23T19:30:49.255536+0000 mon.a (mon.0) 888 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:30:49.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:30:49 smithi016 bash[77995]: cluster 2023-09-23T19:30:47.921052+0000 mgr.y (mgr.25325) 694 : cluster 0 pgmap v587: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:49.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:30:49 smithi016 bash[77995]: audit 2023-09-23T19:30:49.255536+0000 mon.a (mon.0) 888 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:30:49.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:30:49 smithi092 bash[58855]: cluster 2023-09-23T19:30:47.921052+0000 mgr.y (mgr.25325) 694 : cluster 0 pgmap v587: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:49.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:30:49 smithi092 bash[58855]: audit 2023-09-23T19:30:49.255536+0000 mon.a (mon.0) 888 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:30:51.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:30:51 smithi016 bash[75004]: cluster 2023-09-23T19:30:49.921922+0000 mgr.y (mgr.25325) 695 : cluster 0 pgmap v588: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:51.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:30:51 smithi016 bash[77995]: cluster 2023-09-23T19:30:49.921922+0000 mgr.y (mgr.25325) 695 : cluster 0 pgmap v588: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:51.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:30:51 smithi092 bash[58855]: cluster 2023-09-23T19:30:49.921922+0000 mgr.y (mgr.25325) 695 : cluster 0 pgmap v588: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:52.280 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:30:53.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:30:53 smithi016 bash[77995]: cluster 2023-09-23T19:30:51.922708+0000 mgr.y (mgr.25325) 696 : cluster 0 pgmap v589: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:53.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:30:53 smithi016 bash[75004]: cluster 2023-09-23T19:30:51.922708+0000 mgr.y (mgr.25325) 696 : cluster 0 pgmap v589: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:53.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:30:53 smithi092 bash[58855]: cluster 2023-09-23T19:30:51.922708+0000 mgr.y (mgr.25325) 696 : cluster 0 pgmap v589: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:55.195 INFO:teuthology.orchestra.run.smithi016.stdout:mon.a: {} 2023-09-23T19:30:55.212 INFO:teuthology.orchestra.run.smithi016.stdout:mon.b: {} 2023-09-23T19:30:55.228 INFO:teuthology.orchestra.run.smithi016.stdout:mon.c: {} 2023-09-23T19:30:55.229 INFO:teuthology.orchestra.run.smithi016.stderr:mon.a: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-09-23T19:30:55.230 INFO:teuthology.orchestra.run.smithi016.stderr:mon.b: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-09-23T19:30:55.230 INFO:teuthology.orchestra.run.smithi016.stderr:mon.c: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-09-23T19:30:55.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:30:55 smithi016 bash[75004]: cluster 2023-09-23T19:30:53.923409+0000 mgr.y (mgr.25325) 697 : cluster 0 pgmap v590: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:55.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:30:55 smithi016 bash[75004]: audit 2023-09-23T19:30:54.443689+0000 mon.a (mon.0) 889 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:30:55.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:30:55 smithi016 bash[75004]: audit 2023-09-23T19:30:54.450778+0000 mon.a (mon.0) 890 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:30:55.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:30:55 smithi016 bash[75004]: audit 2023-09-23T19:30:55.175351+0000 mon.c (mon.2) 137 : audit 0 from='client.? 172.21.15.16:0/4089463685' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T19:30:55.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:30:55 smithi016 bash[77995]: cluster 2023-09-23T19:30:53.923409+0000 mgr.y (mgr.25325) 697 : cluster 0 pgmap v590: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:55.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:30:55 smithi016 bash[77995]: audit 2023-09-23T19:30:54.443689+0000 mon.a (mon.0) 889 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:30:55.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:30:55 smithi016 bash[77995]: audit 2023-09-23T19:30:54.450778+0000 mon.a (mon.0) 890 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:30:55.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:30:55 smithi016 bash[77995]: audit 2023-09-23T19:30:55.175351+0000 mon.c (mon.2) 137 : audit 0 from='client.? 172.21.15.16:0/4089463685' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T19:30:55.785 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:30:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:30:55] "GET /metrics HTTP/1.1" 200 33814 "" "Prometheus/2.33.4" 2023-09-23T19:30:55.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:30:55 smithi092 bash[58855]: cluster 2023-09-23T19:30:53.923409+0000 mgr.y (mgr.25325) 697 : cluster 0 pgmap v590: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:55.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:30:55 smithi092 bash[58855]: audit 2023-09-23T19:30:54.443689+0000 mon.a (mon.0) 889 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:30:55.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:30:55 smithi092 bash[58855]: audit 2023-09-23T19:30:54.450778+0000 mon.a (mon.0) 890 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:30:55.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:30:55 smithi092 bash[58855]: audit 2023-09-23T19:30:55.175351+0000 mon.c (mon.2) 137 : audit 0 from='client.? 172.21.15.16:0/4089463685' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T19:30:56.086 INFO:tasks.thrashosds.thrasher:Unable to test ceph-objectstore-tool, not available on all OSD nodes 2023-09-23T19:30:56.087 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2023-09-23T19:30:56.087 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T19:30:56.088 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T19:30:56.088 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_4 2023-09-23T19:30:56.088 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_4 16 2023-09-23T19:30:56.089 INFO:tasks.thrashosds.thrasher:starting do_thrash 2023-09-23T19:30:56.089 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T19:30:56.089 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T19:30:56.090 INFO:tasks.thrashosds.thrasher:starting do_sighup with a delay of 0.1 2023-09-23T19:30:56.090 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:30:56.091 INFO:tasks.thrashosds.thrasher:starting do_optrack_toggle with a delay of 2.0 2023-09-23T19:30:56.091 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:30:56.091 INFO:tasks.thrashosds.thrasher:starting do_dump_ops 2023-09-23T19:30:56.092 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T19:30:56.092 INFO:tasks.thrashosds.thrasher:starting do_noscrub_toggle with a delay of 2.0 2023-09-23T19:30:56.092 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T19:30:56.124 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:30:56.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:30:56.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:30:56.228 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:30:56.257 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:30:56.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:30:56.259 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:30:56.360 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:30:56.390 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:30:56.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:30:56.393 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:30:56.494 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:30:56.524 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:30:56.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:30:56.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:30:56.628 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:30:56.665 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:30:56.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:30:56.668 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:30:56.768 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:30:56.782 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:30:56 smithi016 bash[77995]: audit 2023-09-23T19:30:55.502889+0000 mon.a (mon.0) 891 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:30:56.783 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:30:56 smithi016 bash[77995]: audit 2023-09-23T19:30:55.510210+0000 mon.a (mon.0) 892 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:30:56.783 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:30:56 smithi016 bash[77995]: audit 2023-09-23T19:30:55.977756+0000 mon.a (mon.0) 893 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:30:56.783 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:30:56 smithi016 bash[77995]: audit 2023-09-23T19:30:55.979105+0000 mon.a (mon.0) 894 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:30:56.783 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:30:56 smithi016 bash[77995]: audit 2023-09-23T19:30:55.987292+0000 mon.a (mon.0) 895 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:30:56.783 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:30:56 smithi016 bash[75004]: audit 2023-09-23T19:30:55.502889+0000 mon.a (mon.0) 891 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:30:56.783 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:30:56 smithi016 bash[75004]: audit 2023-09-23T19:30:55.510210+0000 mon.a (mon.0) 892 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:30:56.783 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:30:56 smithi016 bash[75004]: audit 2023-09-23T19:30:55.977756+0000 mon.a (mon.0) 893 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:30:56.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:30:56 smithi016 bash[75004]: audit 2023-09-23T19:30:55.979105+0000 mon.a (mon.0) 894 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:30:56.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:30:56 smithi016 bash[75004]: audit 2023-09-23T19:30:55.987292+0000 mon.a (mon.0) 895 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:30:56.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:30:56 smithi092 bash[58855]: audit 2023-09-23T19:30:55.502889+0000 mon.a (mon.0) 891 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:30:56.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:30:56 smithi092 bash[58855]: audit 2023-09-23T19:30:55.510210+0000 mon.a (mon.0) 892 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:30:56.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:30:56 smithi092 bash[58855]: audit 2023-09-23T19:30:55.977756+0000 mon.a (mon.0) 893 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:30:56.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:30:56 smithi092 bash[58855]: audit 2023-09-23T19:30:55.979105+0000 mon.a (mon.0) 894 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:30:56.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:30:56 smithi092 bash[58855]: audit 2023-09-23T19:30:55.987292+0000 mon.a (mon.0) 895 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:30:56.801 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:30:56.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:30:56.805 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:30:56.905 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:30:56.936 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:30:56.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:30:56.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:30:57.039 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:30:57.072 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:30:57.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:30:57.074 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:30:57.175 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:30:57.206 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:30:57.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:30:57.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:30:57.309 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:30:57.339 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:30:57.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:30:57.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:30:57.443 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:30:57.475 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:30:57.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:30:57.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:30:57.579 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:30:57.592 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:30:57 smithi092 bash[58855]: cluster 2023-09-23T19:30:55.923940+0000 mgr.y (mgr.25325) 698 : cluster 0 pgmap v591: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:57.610 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:30:57.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:30:57.613 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:30:57.714 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:30:57.726 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:30:57 smithi016 bash[75004]: cluster 2023-09-23T19:30:55.923940+0000 mgr.y (mgr.25325) 698 : cluster 0 pgmap v591: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:57.726 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:30:57 smithi016 bash[77995]: cluster 2023-09-23T19:30:55.923940+0000 mgr.y (mgr.25325) 698 : cluster 0 pgmap v591: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:57.745 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:30:57.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:30:57.748 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:30:57.849 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:30:57.881 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:30:57.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:30:57.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:30:57.984 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:30:58.017 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:30:58.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:30:58.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:30:58.123 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:30:58.160 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:30:58.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:30:58.163 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:30:58.264 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:30:58.294 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:30:58.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:30:58.297 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:30:58.397 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:30:58.430 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:30:58.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:30:58.432 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:30:58.533 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:30:58.565 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:30:58.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:30:58.569 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:30:58.670 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:30:58.702 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:30:58.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:30:58.705 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:30:58.806 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:30:58.840 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:30:58.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:30:58.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:30:58.948 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:30:58.979 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:30:58.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:30:58.982 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:30:59.082 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:30:59.116 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:30:59.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:30:59.119 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:30:59.219 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:30:59.252 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:30:59.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:30:59.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:30:59.356 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:30:59.388 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:30:59.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:30:59.391 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:30:59.492 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:30:59.528 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:30:59.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:30:59.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:30:59.631 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:30:59.658 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:30:59.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:30:59.662 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:30:59.762 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:30:59.778 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:30:59 smithi092 bash[58855]: cluster 2023-09-23T19:30:57.924653+0000 mgr.y (mgr.25325) 699 : cluster 0 pgmap v592: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:59.778 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:30:59 smithi092 bash[58855]: audit 2023-09-23T19:30:58.636131+0000 mon.a (mon.0) 896 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:30:59.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:30:59 smithi016 bash[75004]: cluster 2023-09-23T19:30:57.924653+0000 mgr.y (mgr.25325) 699 : cluster 0 pgmap v592: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:59.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:30:59 smithi016 bash[75004]: audit 2023-09-23T19:30:58.636131+0000 mon.a (mon.0) 896 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:30:59.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:30:59 smithi016 bash[77995]: cluster 2023-09-23T19:30:57.924653+0000 mgr.y (mgr.25325) 699 : cluster 0 pgmap v592: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:30:59.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:30:59 smithi016 bash[77995]: audit 2023-09-23T19:30:58.636131+0000 mon.a (mon.0) 896 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:30:59.799 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:30:59.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:30:59.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:30:59.903 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:30:59.936 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:30:59.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:30:59.939 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:00.040 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:00.072 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:00.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:00.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:00.176 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:00.203 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:00.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:00.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:00.307 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:00.338 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:00.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:00.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:00.442 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:00.476 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:00.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:00.479 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:31:00.580 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:00.612 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:00.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:00.616 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:00.717 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:00.747 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:00.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:00.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:00.852 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:00.883 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:00.884 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:31:00.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:00.885 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:31:00.886 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:31:00.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:00.887 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:31:00.987 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:01.022 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:01.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:01.025 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:31:01.126 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:01.156 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:01.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:01.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:01.260 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:01.290 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:01.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:01.292 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:01.393 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:01.422 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:01.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:01.425 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:01.526 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:01.549 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:01.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:01.552 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:31:01.653 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:01.665 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:01 smithi092 bash[58855]: cluster 2023-09-23T19:30:59.925288+0000 mgr.y (mgr.25325) 700 : cluster 0 pgmap v593: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:31:01.683 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:01.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:01.687 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:01.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:01 smithi016 bash[75004]: cluster 2023-09-23T19:30:59.925288+0000 mgr.y (mgr.25325) 700 : cluster 0 pgmap v593: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:31:01.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:01 smithi016 bash[77995]: cluster 2023-09-23T19:30:59.925288+0000 mgr.y (mgr.25325) 700 : cluster 0 pgmap v593: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:31:01.787 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:01.814 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:01.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:01.817 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:01.918 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:01.940 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:01.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:01.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:02.043 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:02.075 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:02.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:02.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:31:02.179 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:02.207 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:02.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:02.209 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:31:02.310 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:02.339 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:02.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:02.341 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:02.442 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:02.470 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:02.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:02.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:31:02.574 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:02.607 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:02.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:02.611 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:02.711 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:02.741 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:02.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:02.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:02.844 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:02.871 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:02.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:02.874 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:02.974 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:03.000 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:03.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:03.003 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:31:03.105 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:03.132 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:03.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:03.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:03.235 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:03.267 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:03.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:03.270 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:31:03.371 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:03.400 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:03.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:03.403 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:03.504 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:03.531 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:03.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:03.534 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:03.635 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:03.662 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:03.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:03.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:03.700 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:03 smithi016 bash[75004]: cluster 2023-09-23T19:31:01.925810+0000 mgr.y (mgr.25325) 701 : cluster 0 pgmap v594: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:31:03.701 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:03 smithi016 bash[77995]: cluster 2023-09-23T19:31:01.925810+0000 mgr.y (mgr.25325) 701 : cluster 0 pgmap v594: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:31:03.765 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:03.773 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:03 smithi092 bash[58855]: cluster 2023-09-23T19:31:01.925810+0000 mgr.y (mgr.25325) 701 : cluster 0 pgmap v594: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:31:03.791 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:03.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:03.795 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:03.895 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:03.923 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:03.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:03.926 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:31:04.027 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:04.054 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:04.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:04.057 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:04.158 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:04.193 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:04.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:04.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:04.297 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:04.325 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:04.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:04.328 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:04.428 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:04.463 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:04.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:04.465 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:04.566 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:04.587 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:04.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:04.590 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:04.691 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:04.718 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:04.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:04.720 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:04.821 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:04.843 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:04.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:04.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:31:04.947 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:04.979 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:04.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:04.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:05.083 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:05.108 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:05.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:05.111 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:05.212 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:05.287 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:05.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:05.291 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:05.393 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:05.427 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:05.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:05.430 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:31:05.531 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:05.560 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:05.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:05.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:31:05.664 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:05.687 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:05.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:05.689 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:05.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:05 smithi016 bash[77995]: cluster 2023-09-23T19:31:03.926531+0000 mgr.y (mgr.25325) 702 : cluster 0 pgmap v595: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:31:05.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:05 smithi016 bash[75004]: cluster 2023-09-23T19:31:03.926531+0000 mgr.y (mgr.25325) 702 : cluster 0 pgmap v595: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:31:05.786 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:31:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:31:05] "GET /metrics HTTP/1.1" 200 33813 "" "Prometheus/2.33.4" 2023-09-23T19:31:05.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:05 smithi092 bash[58855]: cluster 2023-09-23T19:31:03.926531+0000 mgr.y (mgr.25325) 702 : cluster 0 pgmap v595: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:31:05.791 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:05.820 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:05.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:05.824 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:05.925 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:05.952 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:05.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:05.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:31:06.056 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:06.089 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:06.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:06.091 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:06.192 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:06.223 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:06.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:06.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:06.327 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:06.361 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:06.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:06.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:31:06.465 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:06.495 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:06.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:06.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:06.599 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:06.631 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:06.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:06.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:06.734 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:06.767 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:06.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:06.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:06.871 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:06.902 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:06.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:06.906 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:07.006 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:07.033 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:07.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:07.036 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:07.137 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:07.164 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:07.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:07.167 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:07.268 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:07.294 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:07.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:07.297 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:31:07.397 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:07.423 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:07.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:07.425 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:07.526 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:07.551 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:07.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:07.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:31:07.655 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:07.663 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:07 smithi016 bash[75004]: cluster 2023-09-23T19:31:05.927304+0000 mgr.y (mgr.25325) 703 : cluster 0 pgmap v596: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:31:07.663 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:07 smithi016 bash[77995]: cluster 2023-09-23T19:31:05.927304+0000 mgr.y (mgr.25325) 703 : cluster 0 pgmap v596: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:31:07.677 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:07.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:07.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:07.780 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:07.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:07 smithi092 bash[58855]: cluster 2023-09-23T19:31:05.927304+0000 mgr.y (mgr.25325) 703 : cluster 0 pgmap v596: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:31:07.812 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:07.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:07.816 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:07.917 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:07.947 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:07.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:07.950 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:08.051 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:08.082 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:08.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:08.085 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:08.186 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:08.211 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:08.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:08.214 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:31:08.315 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:08.338 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:08.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:08.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:31:08.441 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:08.472 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:08.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:08.476 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:08.577 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:08.608 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:08.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:08.611 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:08.712 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:08.746 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:08.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:08.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:08.850 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:08.879 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:08.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:08.882 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:31:08.983 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:09.012 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:09.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:09.014 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:09.115 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:09.149 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:09.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:09.152 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:31:09.253 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:09.280 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:09.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:09.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:09.384 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:09.420 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:09.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:09.424 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:09.525 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:09.556 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:09.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:09.558 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:09.659 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:09.688 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:09.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:09.691 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:31:09.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:09 smithi092 bash[58855]: cluster 2023-09-23T19:31:07.928013+0000 mgr.y (mgr.25325) 704 : cluster 0 pgmap v597: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:31:09.792 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:09.828 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:09.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:09.831 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:09.852 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-09-23T19:31:09.852 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [], 2023-09-23T19:31:09.852 INFO:teuthology.orchestra.run.smithi016.stdout: "num_ops": 0 2023-09-23T19:31:09.852 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-09-23T19:31:09.897 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:09 smithi016 bash[75004]: cluster 2023-09-23T19:31:07.928013+0000 mgr.y (mgr.25325) 704 : cluster 0 pgmap v597: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:31:09.897 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:09 smithi016 bash[77995]: cluster 2023-09-23T19:31:07.928013+0000 mgr.y (mgr.25325) 704 : cluster 0 pgmap v597: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:31:09.933 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:09.961 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:09.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:09.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:31:10.065 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:10.100 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:10.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:10.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:10.204 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:10.237 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:10.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:10.240 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:10.340 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:10.363 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:10.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:10.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:10.405 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:31:10.429 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:31:10.451 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:31:10.466 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:10.477 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:31:10.496 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:10.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:10.498 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:10.505 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:31:10.532 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:31:10.555 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is set 2023-09-23T19:31:10.557 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:31:10.579 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:31:10.581 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:31:10.581 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:31:10.581 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:31:10.581 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:31:10.581 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:31:10.581 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:31:10.582 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:31:10.582 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:31:10.599 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:10.620 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:10.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:10.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:10.723 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:10.736 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:10 smithi016 bash[75004]: audit 2023-09-23T19:31:10.015385+0000 mon.c (mon.2) 138 : audit 1 from='client.? 172.21.15.16:0/2047227605' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:31:10.736 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:10 smithi016 bash[75004]: audit 2023-09-23T19:31:10.015837+0000 mon.a (mon.0) 897 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:31:10.736 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:10 smithi016 bash[75004]: audit 2023-09-23T19:31:10.377949+0000 mon.c (mon.2) 139 : audit 0 from='client.? 172.21.15.16:0/2682226701' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:31:10.737 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:10 smithi016 bash[77995]: audit 2023-09-23T19:31:10.015385+0000 mon.c (mon.2) 138 : audit 1 from='client.? 172.21.15.16:0/2047227605' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:31:10.737 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:10 smithi016 bash[77995]: audit 2023-09-23T19:31:10.015837+0000 mon.a (mon.0) 897 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:31:10.737 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:10 smithi016 bash[77995]: audit 2023-09-23T19:31:10.377949+0000 mon.c (mon.2) 139 : audit 0 from='client.? 172.21.15.16:0/2682226701' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:31:10.748 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:10.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:10.750 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:10.851 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:10.865 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:10 smithi092 bash[58855]: audit 2023-09-23T19:31:10.015385+0000 mon.c (mon.2) 138 : audit 1 from='client.? 172.21.15.16:0/2047227605' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:31:10.865 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:10 smithi092 bash[58855]: audit 2023-09-23T19:31:10.015837+0000 mon.a (mon.0) 897 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:31:10.865 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:10 smithi092 bash[58855]: audit 2023-09-23T19:31:10.377949+0000 mon.c (mon.2) 139 : audit 0 from='client.? 172.21.15.16:0/2682226701' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:31:10.883 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:10.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:10.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:31:10.987 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:11.020 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:11.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:11.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:11.124 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:11.155 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:11.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:11.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:11.259 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:11.292 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:11.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:11.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:11.396 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:11.425 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:11.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:11.428 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:11.529 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:11.561 INFO:teuthology.orchestra.run.smithi016.stderr:pool 'unique_pool_4' created 2023-09-23T19:31:11.564 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:11.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:11.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:11.667 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:11.679 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:11 smithi092 bash[58855]: cluster 2023-09-23T19:31:09.928541+0000 mgr.y (mgr.25325) 705 : cluster 0 pgmap v598: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:31:11.679 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:11 smithi092 bash[58855]: cluster 2023-09-23T19:31:10.535466+0000 mon.a (mon.0) 898 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:31:11.679 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:11 smithi092 bash[58855]: audit 2023-09-23T19:31:10.549264+0000 mon.a (mon.0) 899 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:31:11.679 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:11 smithi092 bash[58855]: cluster 2023-09-23T19:31:10.549341+0000 mon.a (mon.0) 900 : cluster 0 osdmap e118: 8 total, 8 up, 8 in 2023-09-23T19:31:11.679 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:11 smithi092 bash[58855]: cluster 2023-09-23T19:31:10.550410+0000 osd.5 (osd.5) 44 : cluster 3 failed to encode map e118 with expected crc 2023-09-23T19:31:11.679 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:11 smithi092 bash[58855]: audit 2023-09-23T19:31:10.833585+0000 mon.c (mon.2) 140 : audit 1 from='client.? 172.21.15.16:0/2773903450' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T19:31:11.679 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:11 smithi092 bash[58855]: audit 2023-09-23T19:31:10.834113+0000 mon.a (mon.0) 901 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T19:31:11.695 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:11.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:11.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:11.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:11 smithi016 bash[75004]: cluster 2023-09-23T19:31:09.928541+0000 mgr.y (mgr.25325) 705 : cluster 0 pgmap v598: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:31:11.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:11 smithi016 bash[75004]: cluster 2023-09-23T19:31:10.535466+0000 mon.a (mon.0) 898 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:31:11.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:11 smithi016 bash[75004]: audit 2023-09-23T19:31:10.549264+0000 mon.a (mon.0) 899 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:31:11.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:11 smithi016 bash[75004]: cluster 2023-09-23T19:31:10.549341+0000 mon.a (mon.0) 900 : cluster 0 osdmap e118: 8 total, 8 up, 8 in 2023-09-23T19:31:11.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:11 smithi016 bash[75004]: cluster 2023-09-23T19:31:10.550410+0000 osd.5 (osd.5) 44 : cluster 3 failed to encode map e118 with expected crc 2023-09-23T19:31:11.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:11 smithi016 bash[75004]: audit 2023-09-23T19:31:10.833585+0000 mon.c (mon.2) 140 : audit 1 from='client.? 172.21.15.16:0/2773903450' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T19:31:11.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:11 smithi016 bash[75004]: audit 2023-09-23T19:31:10.834113+0000 mon.a (mon.0) 901 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T19:31:11.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:11 smithi016 bash[77995]: cluster 2023-09-23T19:31:09.928541+0000 mgr.y (mgr.25325) 705 : cluster 0 pgmap v598: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:31:11.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:11 smithi016 bash[77995]: cluster 2023-09-23T19:31:10.535466+0000 mon.a (mon.0) 898 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:31:11.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:11 smithi016 bash[77995]: audit 2023-09-23T19:31:10.549264+0000 mon.a (mon.0) 899 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:31:11.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:11 smithi016 bash[77995]: cluster 2023-09-23T19:31:10.549341+0000 mon.a (mon.0) 900 : cluster 0 osdmap e118: 8 total, 8 up, 8 in 2023-09-23T19:31:11.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:11 smithi016 bash[77995]: cluster 2023-09-23T19:31:10.550410+0000 osd.5 (osd.5) 44 : cluster 3 failed to encode map e118 with expected crc 2023-09-23T19:31:11.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:11 smithi016 bash[77995]: audit 2023-09-23T19:31:10.833585+0000 mon.c (mon.2) 140 : audit 1 from='client.? 172.21.15.16:0/2773903450' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T19:31:11.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:11 smithi016 bash[77995]: audit 2023-09-23T19:31:10.834113+0000 mon.a (mon.0) 901 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T19:31:11.799 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:11.830 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:11.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:11.833 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:31:11.872 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T19:31:11.934 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:11.965 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:11.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:11.968 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:12.068 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:12.101 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:12.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:12.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:31:12.205 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:12.236 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:12.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:12.238 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:12.339 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:12.362 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:12.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:12.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:12.465 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:12.491 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:12.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:12.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:12.594 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:12.601 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:12 smithi016 bash[77995]: audit 2023-09-23T19:31:11.550907+0000 mon.a (mon.0) 902 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-09-23T19:31:12.601 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:12 smithi016 bash[77995]: cluster 2023-09-23T19:31:11.551049+0000 mon.a (mon.0) 903 : cluster 0 osdmap e119: 8 total, 8 up, 8 in 2023-09-23T19:31:12.601 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:12 smithi016 bash[77995]: cluster 2023-09-23T19:31:11.552327+0000 osd.5 (osd.5) 45 : cluster 3 failed to encode map e119 with expected crc 2023-09-23T19:31:12.601 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:12 smithi016 bash[77995]: cluster 2023-09-23T19:31:11.554364+0000 osd.6 (osd.6) 38 : cluster 3 failed to encode map e118 with expected crc 2023-09-23T19:31:12.602 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:12 smithi016 bash[77995]: cluster 2023-09-23T19:31:11.554579+0000 osd.4 (osd.4) 46 : cluster 3 failed to encode map e118 with expected crc 2023-09-23T19:31:12.602 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:12 smithi016 bash[77995]: cluster 2023-09-23T19:31:11.558014+0000 osd.7 (osd.7) 35 : cluster 3 failed to encode map e118 with expected crc 2023-09-23T19:31:12.602 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:12 smithi016 bash[75004]: audit 2023-09-23T19:31:11.550907+0000 mon.a (mon.0) 902 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-09-23T19:31:12.602 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:12 smithi016 bash[75004]: cluster 2023-09-23T19:31:11.551049+0000 mon.a (mon.0) 903 : cluster 0 osdmap e119: 8 total, 8 up, 8 in 2023-09-23T19:31:12.602 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:12 smithi016 bash[75004]: cluster 2023-09-23T19:31:11.552327+0000 osd.5 (osd.5) 45 : cluster 3 failed to encode map e119 with expected crc 2023-09-23T19:31:12.602 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:12 smithi016 bash[75004]: cluster 2023-09-23T19:31:11.554364+0000 osd.6 (osd.6) 38 : cluster 3 failed to encode map e118 with expected crc 2023-09-23T19:31:12.602 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:12 smithi016 bash[75004]: cluster 2023-09-23T19:31:11.554579+0000 osd.4 (osd.4) 46 : cluster 3 failed to encode map e118 with expected crc 2023-09-23T19:31:12.602 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:12 smithi016 bash[75004]: cluster 2023-09-23T19:31:11.558014+0000 osd.7 (osd.7) 35 : cluster 3 failed to encode map e118 with expected crc 2023-09-23T19:31:12.617 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:12.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:12.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:12.720 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:12.751 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:12.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:12.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:12.854 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:12.866 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:12 smithi092 bash[58855]: audit 2023-09-23T19:31:11.550907+0000 mon.a (mon.0) 902 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-09-23T19:31:12.866 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:12 smithi092 bash[58855]: cluster 2023-09-23T19:31:11.551049+0000 mon.a (mon.0) 903 : cluster 0 osdmap e119: 8 total, 8 up, 8 in 2023-09-23T19:31:12.866 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:12 smithi092 bash[58855]: cluster 2023-09-23T19:31:11.552327+0000 osd.5 (osd.5) 45 : cluster 3 failed to encode map e119 with expected crc 2023-09-23T19:31:12.866 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:12 smithi092 bash[58855]: cluster 2023-09-23T19:31:11.554364+0000 osd.6 (osd.6) 38 : cluster 3 failed to encode map e118 with expected crc 2023-09-23T19:31:12.866 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:12 smithi092 bash[58855]: cluster 2023-09-23T19:31:11.554579+0000 osd.4 (osd.4) 46 : cluster 3 failed to encode map e118 with expected crc 2023-09-23T19:31:12.866 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:12 smithi092 bash[58855]: cluster 2023-09-23T19:31:11.558014+0000 osd.7 (osd.7) 35 : cluster 3 failed to encode map e118 with expected crc 2023-09-23T19:31:12.883 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:12.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:12.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:31:12.987 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:13.021 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:13.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:13.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:13.124 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:13.154 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:13.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:13.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:13.258 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:13.290 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:13.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:13.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:13.393 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:13.422 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:13.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:13.425 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:13.526 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:13.556 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_4 rados --yes-i-really-mean-it || true 2023-09-23T19:31:13.557 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:13.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:13.561 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:13.661 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:13.675 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:13 smithi016 bash[77995]: cluster 2023-09-23T19:31:11.929126+0000 mgr.y (mgr.25325) 706 : cluster 0 pgmap v601: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:31:13.675 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:13 smithi016 bash[77995]: cluster 2023-09-23T19:31:12.560997+0000 osd.7 (osd.7) 36 : cluster 3 failed to encode map e120 with expected crc 2023-09-23T19:31:13.675 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:13 smithi016 bash[77995]: cluster 2023-09-23T19:31:12.562191+0000 mon.a (mon.0) 904 : cluster 0 osdmap e120: 8 total, 8 up, 8 in 2023-09-23T19:31:13.675 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:13 smithi016 bash[77995]: cluster 2023-09-23T19:31:12.566343+0000 osd.7 (osd.7) 37 : cluster 3 failed to encode map e120 with expected crc 2023-09-23T19:31:13.675 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:13 smithi016 bash[77995]: cluster 2023-09-23T19:31:12.566374+0000 osd.5 (osd.5) 46 : cluster 3 failed to encode map e120 with expected crc 2023-09-23T19:31:13.675 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:13 smithi016 bash[77995]: cluster 2023-09-23T19:31:12.586654+0000 osd.4 (osd.4) 47 : cluster 3 failed to encode map e120 with expected crc 2023-09-23T19:31:13.675 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:13 smithi016 bash[77995]: cluster 2023-09-23T19:31:12.586703+0000 osd.6 (osd.6) 39 : cluster 3 failed to encode map e120 with expected crc 2023-09-23T19:31:13.675 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:13 smithi016 bash[75004]: cluster 2023-09-23T19:31:11.929126+0000 mgr.y (mgr.25325) 706 : cluster 0 pgmap v601: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:31:13.675 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:13 smithi016 bash[75004]: cluster 2023-09-23T19:31:12.560997+0000 osd.7 (osd.7) 36 : cluster 3 failed to encode map e120 with expected crc 2023-09-23T19:31:13.676 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:13 smithi016 bash[75004]: cluster 2023-09-23T19:31:12.562191+0000 mon.a (mon.0) 904 : cluster 0 osdmap e120: 8 total, 8 up, 8 in 2023-09-23T19:31:13.676 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:13 smithi016 bash[75004]: cluster 2023-09-23T19:31:12.566343+0000 osd.7 (osd.7) 37 : cluster 3 failed to encode map e120 with expected crc 2023-09-23T19:31:13.676 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:13 smithi016 bash[75004]: cluster 2023-09-23T19:31:12.566374+0000 osd.5 (osd.5) 46 : cluster 3 failed to encode map e120 with expected crc 2023-09-23T19:31:13.676 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:13 smithi016 bash[75004]: cluster 2023-09-23T19:31:12.586654+0000 osd.4 (osd.4) 47 : cluster 3 failed to encode map e120 with expected crc 2023-09-23T19:31:13.676 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:13 smithi016 bash[75004]: cluster 2023-09-23T19:31:12.586703+0000 osd.6 (osd.6) 39 : cluster 3 failed to encode map e120 with expected crc 2023-09-23T19:31:13.693 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:13.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:13.697 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:13.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:13 smithi092 bash[58855]: cluster 2023-09-23T19:31:11.929126+0000 mgr.y (mgr.25325) 706 : cluster 0 pgmap v601: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:31:13.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:13 smithi092 bash[58855]: cluster 2023-09-23T19:31:12.560997+0000 osd.7 (osd.7) 36 : cluster 3 failed to encode map e120 with expected crc 2023-09-23T19:31:13.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:13 smithi092 bash[58855]: cluster 2023-09-23T19:31:12.562191+0000 mon.a (mon.0) 904 : cluster 0 osdmap e120: 8 total, 8 up, 8 in 2023-09-23T19:31:13.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:13 smithi092 bash[58855]: cluster 2023-09-23T19:31:12.566343+0000 osd.7 (osd.7) 37 : cluster 3 failed to encode map e120 with expected crc 2023-09-23T19:31:13.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:13 smithi092 bash[58855]: cluster 2023-09-23T19:31:12.566374+0000 osd.5 (osd.5) 46 : cluster 3 failed to encode map e120 with expected crc 2023-09-23T19:31:13.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:13 smithi092 bash[58855]: cluster 2023-09-23T19:31:12.586654+0000 osd.4 (osd.4) 47 : cluster 3 failed to encode map e120 with expected crc 2023-09-23T19:31:13.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:13 smithi092 bash[58855]: cluster 2023-09-23T19:31:12.586703+0000 osd.6 (osd.6) 39 : cluster 3 failed to encode map e120 with expected crc 2023-09-23T19:31:13.798 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:13.834 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:13.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:13.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:13.938 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:13.963 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:13.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:13.966 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:31:14.067 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:14.097 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:14.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:14.099 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:14.200 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:14.235 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:14.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:14.238 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:14.339 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:14.370 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:14.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:14.373 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:14.449 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T19:31:14.474 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:14.527 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:14.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:14.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:14.631 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:14.664 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:14.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:14.668 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:31:14.768 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:14.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:14 smithi016 bash[75004]: cluster 2023-09-23T19:31:13.569711+0000 mon.a (mon.0) 905 : cluster 0 osdmap e121: 8 total, 8 up, 8 in 2023-09-23T19:31:14.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:14 smithi016 bash[75004]: cluster 2023-09-23T19:31:13.573398+0000 osd.7 (osd.7) 38 : cluster 3 failed to encode map e121 with expected crc 2023-09-23T19:31:14.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:14 smithi016 bash[75004]: cluster 2023-09-23T19:31:13.576038+0000 osd.7 (osd.7) 39 : cluster 3 failed to encode map e121 with expected crc 2023-09-23T19:31:14.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:14 smithi016 bash[75004]: audit 2023-09-23T19:31:13.641252+0000 mon.a (mon.0) 906 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:31:14.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:14 smithi016 bash[75004]: audit 2023-09-23T19:31:13.642723+0000 mon.a (mon.0) 907 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:31:14.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:14 smithi016 bash[75004]: cluster 2023-09-23T19:31:13.722737+0000 osd.4 (osd.4) 48 : cluster 3 failed to encode map e121 with expected crc 2023-09-23T19:31:14.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:14 smithi016 bash[75004]: cluster 2023-09-23T19:31:13.722764+0000 osd.6 (osd.6) 40 : cluster 3 failed to encode map e121 with expected crc 2023-09-23T19:31:14.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:14 smithi016 bash[75004]: cluster 2023-09-23T19:31:13.722851+0000 osd.5 (osd.5) 47 : cluster 3 failed to encode map e121 with expected crc 2023-09-23T19:31:14.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:14 smithi016 bash[77995]: cluster 2023-09-23T19:31:13.569711+0000 mon.a (mon.0) 905 : cluster 0 osdmap e121: 8 total, 8 up, 8 in 2023-09-23T19:31:14.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:14 smithi016 bash[77995]: cluster 2023-09-23T19:31:13.573398+0000 osd.7 (osd.7) 38 : cluster 3 failed to encode map e121 with expected crc 2023-09-23T19:31:14.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:14 smithi016 bash[77995]: cluster 2023-09-23T19:31:13.576038+0000 osd.7 (osd.7) 39 : cluster 3 failed to encode map e121 with expected crc 2023-09-23T19:31:14.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:14 smithi016 bash[77995]: audit 2023-09-23T19:31:13.641252+0000 mon.a (mon.0) 906 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:31:14.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:14 smithi016 bash[77995]: audit 2023-09-23T19:31:13.642723+0000 mon.a (mon.0) 907 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:31:14.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:14 smithi016 bash[77995]: cluster 2023-09-23T19:31:13.722737+0000 osd.4 (osd.4) 48 : cluster 3 failed to encode map e121 with expected crc 2023-09-23T19:31:14.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:14 smithi016 bash[77995]: cluster 2023-09-23T19:31:13.722764+0000 osd.6 (osd.6) 40 : cluster 3 failed to encode map e121 with expected crc 2023-09-23T19:31:14.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:14 smithi016 bash[77995]: cluster 2023-09-23T19:31:13.722851+0000 osd.5 (osd.5) 47 : cluster 3 failed to encode map e121 with expected crc 2023-09-23T19:31:14.797 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:14.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:14.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:14.901 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:14.915 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:14 smithi092 bash[58855]: cluster 2023-09-23T19:31:13.569711+0000 mon.a (mon.0) 905 : cluster 0 osdmap e121: 8 total, 8 up, 8 in 2023-09-23T19:31:14.915 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:14 smithi092 bash[58855]: cluster 2023-09-23T19:31:13.573398+0000 osd.7 (osd.7) 38 : cluster 3 failed to encode map e121 with expected crc 2023-09-23T19:31:14.915 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:14 smithi092 bash[58855]: cluster 2023-09-23T19:31:13.576038+0000 osd.7 (osd.7) 39 : cluster 3 failed to encode map e121 with expected crc 2023-09-23T19:31:14.915 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:14 smithi092 bash[58855]: audit 2023-09-23T19:31:13.641252+0000 mon.a (mon.0) 906 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:31:14.915 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:14 smithi092 bash[58855]: audit 2023-09-23T19:31:13.642723+0000 mon.a (mon.0) 907 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:31:14.915 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:14 smithi092 bash[58855]: cluster 2023-09-23T19:31:13.722737+0000 osd.4 (osd.4) 48 : cluster 3 failed to encode map e121 with expected crc 2023-09-23T19:31:14.915 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:14 smithi092 bash[58855]: cluster 2023-09-23T19:31:13.722764+0000 osd.6 (osd.6) 40 : cluster 3 failed to encode map e121 with expected crc 2023-09-23T19:31:14.915 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:14 smithi092 bash[58855]: cluster 2023-09-23T19:31:13.722851+0000 osd.5 (osd.5) 47 : cluster 3 failed to encode map e121 with expected crc 2023-09-23T19:31:14.933 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:14.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:14.936 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:31:15.037 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:15.070 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:15.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:15.074 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:15.175 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:15.212 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:15.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:15.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:15.316 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:15.348 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:15.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:15.351 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:15.452 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:15.476 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:31:15.480 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:15.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:15.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:15.584 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:15.617 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:15.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:15.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:15.721 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:15.732 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:15 smithi092 bash[58855]: cluster 2023-09-23T19:31:13.946087+0000 mgr.y (mgr.25325) 707 : cluster 0 pgmap v604: 89 pgs: 13 unknown, 76 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:31:15.754 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:15.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:15.758 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:31:15.859 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:15.870 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:15 smithi016 bash[75004]: cluster 2023-09-23T19:31:13.946087+0000 mgr.y (mgr.25325) 707 : cluster 0 pgmap v604: 89 pgs: 13 unknown, 76 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:31:15.870 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:15 smithi016 bash[77995]: cluster 2023-09-23T19:31:13.946087+0000 mgr.y (mgr.25325) 707 : cluster 0 pgmap v604: 89 pgs: 13 unknown, 76 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:31:15.871 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:31:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:31:15] "GET /metrics HTTP/1.1" 200 36168 "" "Prometheus/2.33.4" 2023-09-23T19:31:15.888 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:15.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:15.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:31:15.992 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:16.024 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:16.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:16.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:31:16.128 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:16.159 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:16.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:16.163 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:16.264 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:16.296 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:16.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:16.299 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:16.400 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:16.436 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:16.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:16.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:16.540 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:16.574 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:16.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:16.577 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:31:16.633 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:31:16.677 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:16.712 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:16.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:16.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:16.815 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:16.842 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:16.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:16.844 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:16.945 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:16.974 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:16.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:16.978 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:17.079 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:17.106 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:17.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:17.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:31:17.209 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:17.244 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:17.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:17.247 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:17.347 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:17.377 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:17.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:17.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:17.482 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:17.513 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:17.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:17.517 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:17.617 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:17.646 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:17.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:17.651 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:31:17.655 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:31:17.751 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:17.780 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:17.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:17.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:17.883 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:17.896 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:17 smithi016 bash[75004]: cluster 2023-09-23T19:31:15.949061+0000 mgr.y (mgr.25325) 708 : cluster 0 pgmap v605: 89 pgs: 2 unknown, 87 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:31:17.896 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:17 smithi016 bash[77995]: cluster 2023-09-23T19:31:15.949061+0000 mgr.y (mgr.25325) 708 : cluster 0 pgmap v605: 89 pgs: 2 unknown, 87 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:31:17.915 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:17.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:17.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:18.019 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:18.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:17 smithi092 bash[58855]: cluster 2023-09-23T19:31:15.949061+0000 mgr.y (mgr.25325) 708 : cluster 0 pgmap v605: 89 pgs: 2 unknown, 87 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:31:18.052 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:18.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:18.054 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:18.155 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:18.179 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:18.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:18.183 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:18.283 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:18.311 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:18.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:18.313 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:18.414 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:18.443 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:18.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:18.446 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:31:18.547 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:18.572 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:18.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:18.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:18.668 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:31:18.675 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:18.699 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:18.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:18.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:18.802 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:18.839 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:18.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:18.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:18.943 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:18.972 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:18.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:18.975 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:19.075 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:19.096 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:19.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:19.098 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:19.199 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:19.235 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:19.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:19.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:19.338 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:19.375 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:19.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:19.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:19.478 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:19.505 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:19.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:19.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:19.608 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:19.640 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:19.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:19.644 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:19.686 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:31:19.745 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:19.778 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:19.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:19.782 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:31:19.882 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:19.894 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:19 smithi016 bash[77995]: cluster 2023-09-23T19:31:17.949872+0000 mgr.y (mgr.25325) 709 : cluster 0 pgmap v606: 89 pgs: 89 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:31:19.895 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:19 smithi016 bash[75004]: cluster 2023-09-23T19:31:17.949872+0000 mgr.y (mgr.25325) 709 : cluster 0 pgmap v606: 89 pgs: 89 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:31:19.911 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:19.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:19.914 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:20.015 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:20.028 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:19 smithi092 bash[58855]: cluster 2023-09-23T19:31:17.949872+0000 mgr.y (mgr.25325) 709 : cluster 0 pgmap v606: 89 pgs: 89 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:31:20.047 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:20.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:20.050 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:20.151 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:20.180 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:20.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:20.183 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:20.283 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:20.315 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:20.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:20.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:20.418 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:20.454 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:20.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:20.456 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:20.557 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:20.590 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:20.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:20.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:20.695 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:20.727 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:20.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:20.730 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:20.831 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:20.855 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:20.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:20.858 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:20.959 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:20.988 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:20.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:20.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:21.091 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:21.117 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:21.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:21.120 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:21.220 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:21.242 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:21.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:21.245 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:21.345 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:21.371 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:21.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:21.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:31:21.476 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:21.502 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:21.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:21.505 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:21.606 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:21.629 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:21.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:21.631 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:31:21.732 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:21.763 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:21.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:21.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:21.866 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:21.901 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:21.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:21.904 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:22.005 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:22.012 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:21 smithi016 bash[77995]: cluster 2023-09-23T19:31:19.950817+0000 mgr.y (mgr.25325) 710 : cluster 0 pgmap v607: 89 pgs: 89 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:31:22.013 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:21 smithi016 bash[75004]: cluster 2023-09-23T19:31:19.950817+0000 mgr.y (mgr.25325) 710 : cluster 0 pgmap v607: 89 pgs: 89 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:31:22.031 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:22.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:22.033 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:22.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:21 smithi092 bash[58855]: cluster 2023-09-23T19:31:19.950817+0000 mgr.y (mgr.25325) 710 : cluster 0 pgmap v607: 89 pgs: 89 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:31:22.134 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:22.167 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:22.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:22.171 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:31:22.271 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:22.302 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:22.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:22.305 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:31:22.406 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:22.428 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:22.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:22.430 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:22.531 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:22.554 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:22.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:22.558 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:22.658 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:22.685 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:22.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:22.688 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:22.789 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:22.811 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:22.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:22.813 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:22.914 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:22.940 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:22.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:22.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:23.044 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:23.076 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:23.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:23.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:23.180 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:23.203 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:23.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:23.205 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:23.306 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:23.331 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:23.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:23.334 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:23.434 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:23.465 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:23.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:23.468 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:23.569 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:23.598 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:23.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:23.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:23.706 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:23.724 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:23 smithi092 bash[58855]: cluster 2023-09-23T19:31:21.951649+0000 mgr.y (mgr.25325) 711 : cluster 0 pgmap v608: 89 pgs: 89 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:31:23.744 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:23.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:23.748 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:31:23.848 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:23.864 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:23 smithi016 bash[75004]: cluster 2023-09-23T19:31:21.951649+0000 mgr.y (mgr.25325) 711 : cluster 0 pgmap v608: 89 pgs: 89 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:31:23.864 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:23 smithi016 bash[77995]: cluster 2023-09-23T19:31:21.951649+0000 mgr.y (mgr.25325) 711 : cluster 0 pgmap v608: 89 pgs: 89 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:31:23.878 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:23.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:23.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:31:23.981 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:24.014 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:24.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:24.017 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:24.118 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:24.142 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:24.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:24.145 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:24.246 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:24.282 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:24.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:24.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:24.385 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:24.411 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:24.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:24.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:24.514 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:24.535 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:24.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:24.538 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:24.639 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:24.669 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:24.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:24.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:31:24.773 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:24.804 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:24.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:24.806 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:24.907 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:24.932 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:24.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:24.935 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:25.035 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:25.066 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:25.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:25.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:25.170 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:25.189 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:24 smithi016 bash[75004]: cluster 2023-09-23T19:31:23.952174+0000 mgr.y (mgr.25325) 712 : cluster 0 pgmap v609: 89 pgs: 89 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:31:25.189 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:24 smithi016 bash[75004]: audit 2023-09-23T19:31:23.966440+0000 mon.a (mon.0) 908 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:31:25.189 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:24 smithi016 bash[77995]: cluster 2023-09-23T19:31:23.952174+0000 mgr.y (mgr.25325) 712 : cluster 0 pgmap v609: 89 pgs: 89 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:31:25.190 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:24 smithi016 bash[77995]: audit 2023-09-23T19:31:23.966440+0000 mon.a (mon.0) 908 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:31:25.200 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:25.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:25.203 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:25.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:24 smithi092 bash[58855]: cluster 2023-09-23T19:31:23.952174+0000 mgr.y (mgr.25325) 712 : cluster 0 pgmap v609: 89 pgs: 89 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:31:25.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:24 smithi092 bash[58855]: audit 2023-09-23T19:31:23.966440+0000 mon.a (mon.0) 908 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:31:25.304 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:25.333 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:25.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:25.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:25.436 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:25.469 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:25.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:25.472 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:25.573 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:25.603 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:25.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:25.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:25.706 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:25.741 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:25.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:25.744 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:31:25.845 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:25.866 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:25.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:25.869 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:25.881 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:31:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:31:25] "GET /metrics HTTP/1.1" 200 36168 "" "Prometheus/2.33.4" 2023-09-23T19:31:25.971 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:25.992 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:25.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:25.994 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:26.094 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:26.125 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:26.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:26.128 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:31:26.228 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:26.259 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:26.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:26.262 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:31:26.363 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:26.384 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:26.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:26.386 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:26.487 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:26.520 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:26.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:26.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:26.625 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:26.659 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:26.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:26.662 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:26.763 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:26.795 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:26.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:26.797 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:26.898 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:26.930 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:26.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:26.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:27.034 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:27.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:26 smithi016 bash[75004]: cluster 2023-09-23T19:31:25.953143+0000 mgr.y (mgr.25325) 713 : cluster 0 pgmap v610: 89 pgs: 89 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:31:27.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:26 smithi016 bash[77995]: cluster 2023-09-23T19:31:25.953143+0000 mgr.y (mgr.25325) 713 : cluster 0 pgmap v610: 89 pgs: 89 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:31:27.063 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:27.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:27.065 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:27.165 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:27.206 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:27.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:27.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:27.300 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:26 smithi092 bash[58855]: cluster 2023-09-23T19:31:25.953143+0000 mgr.y (mgr.25325) 713 : cluster 0 pgmap v610: 89 pgs: 89 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:31:27.309 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:27.342 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:27.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:27.344 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:27.445 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:27.479 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:27.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:27.483 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:27.584 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:27.618 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:27.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:27.621 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:31:27.722 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:27.748 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:27.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:27.751 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:27.838 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-09-23T19:31:27.838 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:31:27.852 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:27.882 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:27.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:27.886 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:27.987 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:28.016 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:28.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:28.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:28.119 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:28.147 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:28.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:28.149 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:31:28.250 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:28.278 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:28.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:28.280 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:28.381 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:28.413 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:28.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:28.417 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:31:28.517 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:28.549 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:28.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:28.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:28.654 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:28.677 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:28.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:28.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:28.780 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:28.810 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:28.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:28.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:28.914 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:28.940 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:28.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:28.943 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:29.044 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:29.070 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:29.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:29.073 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:29.173 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:29.206 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:29.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:29.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:29.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:28 smithi016 bash[75004]: cluster 2023-09-23T19:31:27.953616+0000 mgr.y (mgr.25325) 714 : cluster 0 pgmap v611: 89 pgs: 89 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:31:29.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:28 smithi016 bash[75004]: audit 2023-09-23T19:31:28.636952+0000 mon.a (mon.0) 909 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:31:29.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:28 smithi016 bash[77995]: cluster 2023-09-23T19:31:27.953616+0000 mgr.y (mgr.25325) 714 : cluster 0 pgmap v611: 89 pgs: 89 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:31:29.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:28 smithi016 bash[77995]: audit 2023-09-23T19:31:28.636952+0000 mon.a (mon.0) 909 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:31:29.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:28 smithi092 bash[58855]: cluster 2023-09-23T19:31:27.953616+0000 mgr.y (mgr.25325) 714 : cluster 0 pgmap v611: 89 pgs: 89 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:31:29.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:28 smithi092 bash[58855]: audit 2023-09-23T19:31:28.636952+0000 mon.a (mon.0) 909 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:31:29.310 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:29.336 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:29.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:29.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:29.440 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:29.471 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:29.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:29.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:29.576 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:29.609 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:29.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:29.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:29.712 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:29.744 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:29.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:29.747 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:29.847 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:29.880 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:29.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:29.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:29.983 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:30.006 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:30.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:30.009 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:30.110 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:30.139 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:30.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:30.143 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:30.243 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:30.270 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T19:31:30.273 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:30.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:30.276 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:31:30.377 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:30.408 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:30.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:30.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:30.512 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:30.541 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:30.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:30.544 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:30.644 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:30.677 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:30.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:30.680 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:31:30.781 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:30.794 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:31:30.804 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:30.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:30.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:30.836 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:31:30.863 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:31:30.888 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:31:30.908 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:30.914 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:31:30.935 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:30.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:30.938 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:30.954 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:31:30.982 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:31:31.011 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:31:31.013 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:31:31.013 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:31:31.014 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:31:31.014 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:31:31.014 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:31:31.014 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:31:31.014 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:31:31.014 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:31:31.040 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:31.069 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:31.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:31.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:31.173 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:31.203 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:31.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:31.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:31.307 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:31.315 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:31 smithi016 bash[75004]: cluster 2023-09-23T19:31:29.954510+0000 mgr.y (mgr.25325) 715 : cluster 0 pgmap v612: 89 pgs: 89 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:31:31.315 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:31 smithi016 bash[75004]: audit 2023-09-23T19:31:30.764849+0000 mon.c (mon.2) 141 : audit 0 from='client.? 172.21.15.16:0/1990546605' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:31:31.315 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:31 smithi016 bash[75004]: audit 2023-09-23T19:31:31.032635+0000 mon.c (mon.2) 142 : audit 1 from='client.? 172.21.15.16:0/3257319599' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:31:31.315 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:31 smithi016 bash[75004]: audit 2023-09-23T19:31:31.033053+0000 mon.a (mon.0) 910 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:31:31.315 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:31 smithi016 bash[75004]: audit 2023-09-23T19:31:31.125973+0000 mon.a (mon.0) 911 : audit 1 from='client.? 172.21.15.16:0/3045823289' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:31:31.316 INFO:teuthology.orchestra.run.smithi016.stderr:enabled application 'rados' on pool 'unique_pool_4' 2023-09-23T19:31:31.317 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is set 2023-09-23T19:31:31.336 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:31.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:31.338 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:31.439 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:31.467 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:31.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:31.470 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:31:31.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:31 smithi016 bash[77995]: cluster 2023-09-23T19:31:29.954510+0000 mgr.y (mgr.25325) 715 : cluster 0 pgmap v612: 89 pgs: 89 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:31:31.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:31 smithi016 bash[77995]: audit 2023-09-23T19:31:30.764849+0000 mon.c (mon.2) 141 : audit 0 from='client.? 172.21.15.16:0/1990546605' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:31:31.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:31 smithi016 bash[77995]: audit 2023-09-23T19:31:31.032635+0000 mon.c (mon.2) 142 : audit 1 from='client.? 172.21.15.16:0/3257319599' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:31:31.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:31 smithi016 bash[77995]: audit 2023-09-23T19:31:31.033053+0000 mon.a (mon.0) 910 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:31:31.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:31 smithi016 bash[77995]: audit 2023-09-23T19:31:31.125973+0000 mon.a (mon.0) 911 : audit 1 from='client.? 172.21.15.16:0/3045823289' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:31:31.571 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:31.601 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:31.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:31.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:31.704 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:31.731 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:31.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:31.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:31.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:31 smithi092 bash[58855]: cluster 2023-09-23T19:31:29.954510+0000 mgr.y (mgr.25325) 715 : cluster 0 pgmap v612: 89 pgs: 89 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:31:31.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:31 smithi092 bash[58855]: audit 2023-09-23T19:31:30.764849+0000 mon.c (mon.2) 141 : audit 0 from='client.? 172.21.15.16:0/1990546605' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:31:31.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:31 smithi092 bash[58855]: audit 2023-09-23T19:31:31.032635+0000 mon.c (mon.2) 142 : audit 1 from='client.? 172.21.15.16:0/3257319599' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:31:31.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:31 smithi092 bash[58855]: audit 2023-09-23T19:31:31.033053+0000 mon.a (mon.0) 910 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:31:31.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:31 smithi092 bash[58855]: audit 2023-09-23T19:31:31.125973+0000 mon.a (mon.0) 911 : audit 1 from='client.? 172.21.15.16:0/3045823289' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:31:31.834 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:31.866 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:31.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:31.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:31:31.972 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:32.005 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:32.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:32.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:32.109 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:32.146 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:32.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:32.149 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:31:32.250 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:32.283 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:32.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:32.286 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:32.387 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:32.419 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:32.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:32.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:31:32.522 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:32.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:32 smithi092 bash[58855]: cluster 2023-09-23T19:31:31.300256+0000 mon.a (mon.0) 912 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:31:32.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:32 smithi092 bash[58855]: audit 2023-09-23T19:31:31.306615+0000 mon.a (mon.0) 913 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:31:32.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:32 smithi092 bash[58855]: audit 2023-09-23T19:31:31.306742+0000 mon.a (mon.0) 914 : audit 1 from='client.? 172.21.15.16:0/3045823289' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T19:31:32.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:32 smithi092 bash[58855]: cluster 2023-09-23T19:31:31.306799+0000 mon.a (mon.0) 915 : cluster 0 osdmap e122: 8 total, 8 up, 8 in 2023-09-23T19:31:32.555 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:32.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:32.558 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:32.659 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:32.692 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:32.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:32.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:31:32.795 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:32.827 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:32.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:32.830 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:32.931 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:32.943 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:32 smithi016 bash[75004]: cluster 2023-09-23T19:31:31.300256+0000 mon.a (mon.0) 912 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:31:32.943 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:32 smithi016 bash[75004]: audit 2023-09-23T19:31:31.306615+0000 mon.a (mon.0) 913 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:31:32.943 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:32 smithi016 bash[75004]: audit 2023-09-23T19:31:31.306742+0000 mon.a (mon.0) 914 : audit 1 from='client.? 172.21.15.16:0/3045823289' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T19:31:32.943 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:32 smithi016 bash[75004]: cluster 2023-09-23T19:31:31.306799+0000 mon.a (mon.0) 915 : cluster 0 osdmap e122: 8 total, 8 up, 8 in 2023-09-23T19:31:32.944 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:32 smithi016 bash[77995]: cluster 2023-09-23T19:31:31.300256+0000 mon.a (mon.0) 912 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:31:32.944 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:32 smithi016 bash[77995]: audit 2023-09-23T19:31:31.306615+0000 mon.a (mon.0) 913 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:31:32.944 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:32 smithi016 bash[77995]: audit 2023-09-23T19:31:31.306742+0000 mon.a (mon.0) 914 : audit 1 from='client.? 172.21.15.16:0/3045823289' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T19:31:32.944 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:32 smithi016 bash[77995]: cluster 2023-09-23T19:31:31.306799+0000 mon.a (mon.0) 915 : cluster 0 osdmap e122: 8 total, 8 up, 8 in 2023-09-23T19:31:32.966 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:32.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:32.969 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:31:33.070 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:33.100 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:33.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:33.104 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:33.205 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:33.238 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:33.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:33.240 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:33.341 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:33.374 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:33.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:33.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:33.479 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:33.490 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:33 smithi016 bash[77995]: cluster 2023-09-23T19:31:31.955066+0000 mgr.y (mgr.25325) 716 : cluster 0 pgmap v614: 89 pgs: 89 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:31:33.490 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:33 smithi016 bash[77995]: cluster 2023-09-23T19:31:32.575850+0000 osd.7 (osd.7) 40 : cluster 3 failed to encode map e122 with expected crc 2023-09-23T19:31:33.490 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:33 smithi016 bash[77995]: cluster 2023-09-23T19:31:32.619184+0000 osd.6 (osd.6) 41 : cluster 3 failed to encode map e122 with expected crc 2023-09-23T19:31:33.490 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:33 smithi016 bash[77995]: cluster 2023-09-23T19:31:32.619242+0000 osd.5 (osd.5) 48 : cluster 3 failed to encode map e122 with expected crc 2023-09-23T19:31:33.490 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:33 smithi016 bash[77995]: cluster 2023-09-23T19:31:32.619368+0000 osd.4 (osd.4) 49 : cluster 3 failed to encode map e122 with expected crc 2023-09-23T19:31:33.491 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:33 smithi016 bash[75004]: cluster 2023-09-23T19:31:31.955066+0000 mgr.y (mgr.25325) 716 : cluster 0 pgmap v614: 89 pgs: 89 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:31:33.491 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:33 smithi016 bash[75004]: cluster 2023-09-23T19:31:32.575850+0000 osd.7 (osd.7) 40 : cluster 3 failed to encode map e122 with expected crc 2023-09-23T19:31:33.491 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:33 smithi016 bash[75004]: cluster 2023-09-23T19:31:32.619184+0000 osd.6 (osd.6) 41 : cluster 3 failed to encode map e122 with expected crc 2023-09-23T19:31:33.491 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:33 smithi016 bash[75004]: cluster 2023-09-23T19:31:32.619242+0000 osd.5 (osd.5) 48 : cluster 3 failed to encode map e122 with expected crc 2023-09-23T19:31:33.491 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:33 smithi016 bash[75004]: cluster 2023-09-23T19:31:32.619368+0000 osd.4 (osd.4) 49 : cluster 3 failed to encode map e122 with expected crc 2023-09-23T19:31:33.507 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:33.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:33.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:33.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:33 smithi092 bash[58855]: cluster 2023-09-23T19:31:31.955066+0000 mgr.y (mgr.25325) 716 : cluster 0 pgmap v614: 89 pgs: 89 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:31:33.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:33 smithi092 bash[58855]: cluster 2023-09-23T19:31:32.575850+0000 osd.7 (osd.7) 40 : cluster 3 failed to encode map e122 with expected crc 2023-09-23T19:31:33.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:33 smithi092 bash[58855]: cluster 2023-09-23T19:31:32.619184+0000 osd.6 (osd.6) 41 : cluster 3 failed to encode map e122 with expected crc 2023-09-23T19:31:33.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:33 smithi092 bash[58855]: cluster 2023-09-23T19:31:32.619242+0000 osd.5 (osd.5) 48 : cluster 3 failed to encode map e122 with expected crc 2023-09-23T19:31:33.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:33 smithi092 bash[58855]: cluster 2023-09-23T19:31:32.619368+0000 osd.4 (osd.4) 49 : cluster 3 failed to encode map e122 with expected crc 2023-09-23T19:31:33.610 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:33.640 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:33.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:33.642 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:31:33.743 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:33.774 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:33.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:33.777 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:33.877 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:33.909 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:33.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:33.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:31:33.978 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:31:34.013 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:34.060 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:34.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:34.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:34.163 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:34.196 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:34.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:34.198 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:31:34.207 DEBUG:teuthology.orchestra.run.smithi016:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_4 bench 90 write' 2023-09-23T19:31:34.208 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_4 2023-09-23T19:31:34.208 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T19:31:34.244 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T19:31:34.244 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T19:31:34.289 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:hints = 1 2023-09-23T19:31:34.289 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T19:31:34.289 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Object prefix: benchmark_data_smithi016_113233 2023-09-23T19:31:34.290 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:31:34.290 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 0 0 0 0 0 0 - 0 2023-09-23T19:31:34.299 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:34.332 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:34.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:34.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:34.436 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:34.465 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:34.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:34.469 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:34.570 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:34.595 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:34.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:34.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:34.698 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:34.727 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:34.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:34.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:34.831 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:34.856 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:34.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:34.858 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:34.959 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:34.986 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:34.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:34.990 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:35.091 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:35.119 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:35.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:35.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:35.195 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T19:31:35.225 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:35.268 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:35.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:35.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:35.289 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 1 16 2701 2685 167.797 167.812 0.00835812 0.00593783 2023-09-23T19:31:35.374 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:35.405 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:35.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:35.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:35.510 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:35.525 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:35 smithi016 bash[75004]: cluster 2023-09-23T19:31:33.955879+0000 mgr.y (mgr.25325) 717 : cluster 0 pgmap v615: 89 pgs: 89 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:31:35.526 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:35 smithi016 bash[77995]: cluster 2023-09-23T19:31:33.955879+0000 mgr.y (mgr.25325) 717 : cluster 0 pgmap v615: 89 pgs: 89 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:31:35.540 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:35.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:35.544 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:35.645 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:35.655 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:35 smithi092 bash[58855]: cluster 2023-09-23T19:31:33.955879+0000 mgr.y (mgr.25325) 717 : cluster 0 pgmap v615: 89 pgs: 89 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T19:31:35.671 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:35.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:35.674 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:35.774 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:35.785 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:31:35 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:31:35] "GET /metrics HTTP/1.1" 200 36172 "" "Prometheus/2.33.4" 2023-09-23T19:31:35.804 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:35.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:35.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:35.908 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:35.933 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:35.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:35.936 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:36.037 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:36.071 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:36.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:36.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:36.175 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:36.206 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:36.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:36.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:31:36.289 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 2 16 5448 5432 169.733 171.688 0.00619491 0.00588221 2023-09-23T19:31:36.311 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:36.339 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:36.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:36.341 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:36.442 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:36.469 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:36.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:36.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:36.572 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:36.602 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:36.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:36.605 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:36.705 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:36.732 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:36.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:36.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:36.745 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:31:36.835 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:36.864 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:36.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:36.868 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:31:36.969 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:36.994 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:36.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:36.997 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:31:37.097 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:37.122 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:37.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:37.125 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:31:37.226 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:37.257 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:37.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:37.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:31:37.290 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 3 16 8024 8008 166.817 161 0.00467657 0.00598187 2023-09-23T19:31:37.359 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:37.368 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:37 smithi092 bash[58855]: cluster 2023-09-23T19:31:35.956656+0000 mgr.y (mgr.25325) 718 : cluster 0 pgmap v616: 89 pgs: 89 active+clean; 121 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s wr, 192 op/s 2023-09-23T19:31:37.395 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:37.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:37.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:37.497 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:37.506 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:37 smithi016 bash[75004]: cluster 2023-09-23T19:31:35.956656+0000 mgr.y (mgr.25325) 718 : cluster 0 pgmap v616: 89 pgs: 89 active+clean; 121 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s wr, 192 op/s 2023-09-23T19:31:37.507 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:37 smithi016 bash[77995]: cluster 2023-09-23T19:31:35.956656+0000 mgr.y (mgr.25325) 718 : cluster 0 pgmap v616: 89 pgs: 89 active+clean; 121 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s wr, 192 op/s 2023-09-23T19:31:37.524 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:37.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:37.528 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:37.629 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:37.656 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:37.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:37.658 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:37.759 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:37.795 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:37.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:37.800 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:37.901 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:37.928 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:37.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:37.933 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:38.034 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:38.060 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:38.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:38.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:38.164 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:38.194 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:38.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:38.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:38.290 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 4 16 10554 10538 164.64 158.125 0.00854536 0.00606494 2023-09-23T19:31:38.297 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:38.326 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:38.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:38.328 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:38.429 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:38.458 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:38.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:38.463 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:38.563 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:38.589 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:38.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:38.592 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:38.692 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:38.720 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:38.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:38.724 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:38.787 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:31:38.788 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:31:38.824 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:38.854 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:38.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:38.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:31:38.958 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:38.991 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:38.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:38.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:39.094 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:39.121 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:39.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:39.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:39.224 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:39.249 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:39.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:39.252 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:39.290 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 5 16 13016 13000 162.484 153.875 0.00364386 0.00614382 2023-09-23T19:31:39.353 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:39.380 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:39.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:39.384 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:31:39.485 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:39.495 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:39 smithi016 bash[77995]: cluster 2023-09-23T19:31:37.957139+0000 mgr.y (mgr.25325) 719 : cluster 0 pgmap v617: 89 pgs: 89 active+clean; 160 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 16 MiB/s wr, 255 op/s 2023-09-23T19:31:39.495 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:39 smithi016 bash[75004]: cluster 2023-09-23T19:31:37.957139+0000 mgr.y (mgr.25325) 719 : cluster 0 pgmap v617: 89 pgs: 89 active+clean; 160 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 16 MiB/s wr, 255 op/s 2023-09-23T19:31:39.514 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:39.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:39.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:31:39.618 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:39.661 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:39.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:39.663 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:39.764 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:39.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:39 smithi092 bash[58855]: cluster 2023-09-23T19:31:37.957139+0000 mgr.y (mgr.25325) 719 : cluster 0 pgmap v617: 89 pgs: 89 active+clean; 160 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 16 MiB/s wr, 255 op/s 2023-09-23T19:31:39.791 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:39.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:39.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:39.810 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:31:39.895 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:39.923 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:39.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:39.928 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:31:40.028 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:40.050 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:40.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:40.052 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:40.153 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:40.185 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:40.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:40.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:40.289 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:40.290 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 6 16 15175 15159 157.889 134.938 0.00709719 0.0063257 2023-09-23T19:31:40.316 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:40.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:40.319 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:40.420 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:40.448 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:40.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:40.451 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:40.552 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:40.582 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:40.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:40.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:40.684 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:40.709 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:40.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:40.712 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:40.813 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:40.844 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:40.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:40.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:40.950 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:40.976 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:40.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:40.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:41.081 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:41.106 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:41.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:41.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:41.213 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:41.244 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:41.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:41.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:41.290 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 7 16 17465 17449 155.778 143.125 0.0072643 0.00641144 2023-09-23T19:31:41.347 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:41.371 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:41.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:41.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:41.476 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:41.486 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:41 smithi016 bash[77995]: cluster 2023-09-23T19:31:39.957763+0000 mgr.y (mgr.25325) 720 : cluster 0 pgmap v618: 89 pgs: 89 active+clean; 450 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 45 MiB/s wr, 719 op/s 2023-09-23T19:31:41.486 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:41 smithi016 bash[75004]: cluster 2023-09-23T19:31:39.957763+0000 mgr.y (mgr.25325) 720 : cluster 0 pgmap v618: 89 pgs: 89 active+clean; 450 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 45 MiB/s wr, 719 op/s 2023-09-23T19:31:41.503 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:41.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:41.506 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:41.607 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:41.616 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:41 smithi092 bash[58855]: cluster 2023-09-23T19:31:39.957763+0000 mgr.y (mgr.25325) 720 : cluster 0 pgmap v618: 89 pgs: 89 active+clean; 450 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 45 MiB/s wr, 719 op/s 2023-09-23T19:31:41.634 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:41.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:41.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:41.738 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:41.766 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:41.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:41.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:31:41.871 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:41.901 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:41.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:41.903 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:31:42.004 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:42.033 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:42.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:42.036 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:42.137 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:42.157 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:42.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:42.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:31:42.260 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:42.288 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:42.290 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 8 16 19371 19355 151.195 119.125 0.0144333 0.00660531 2023-09-23T19:31:42.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:42.292 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:31:42.393 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:42.415 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:42.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:42.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:31:42.520 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:42.543 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:42.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:42.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:42.647 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:42.680 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:42.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:42.684 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:42.785 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:42.821 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:42.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:42.825 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:31:42.926 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:42.951 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:42.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:42.953 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:43.054 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:43.081 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:43.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:43.084 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:43.185 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:43.208 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:43.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:43.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:43.290 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 9 16 21098 21082 146.388 107.938 0.00535937 0.00682372 2023-09-23T19:31:43.312 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:43.341 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:43.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:43.344 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:43.444 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:43.453 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:43 smithi016 bash[77995]: cluster 2023-09-23T19:31:41.958363+0000 mgr.y (mgr.25325) 721 : cluster 0 pgmap v619: 89 pgs: 89 active+clean; 931 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-09-23T19:31:43.454 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:43 smithi016 bash[75004]: cluster 2023-09-23T19:31:41.958363+0000 mgr.y (mgr.25325) 721 : cluster 0 pgmap v619: 89 pgs: 89 active+clean; 931 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-09-23T19:31:43.471 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:43.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:43.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:43.576 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:43.605 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:43.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:43.608 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:43.709 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:43.718 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:43 smithi092 bash[58855]: cluster 2023-09-23T19:31:41.958363+0000 mgr.y (mgr.25325) 721 : cluster 0 pgmap v619: 89 pgs: 89 active+clean; 931 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-09-23T19:31:43.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:43.739 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:43.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:43.840 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:43.862 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:43.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:43.864 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:43.965 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:43.989 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:43.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:43.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:44.092 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:44.115 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:44.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:44.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:44.220 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:44.244 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:44.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:44.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:44.290 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 10 16 22890 22874 142.948 112 0.00977868 0.0069903 2023-09-23T19:31:44.347 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:44.375 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:44.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:44.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:44.478 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:44.514 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:44.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:44.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:44.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:44 smithi092 bash[58855]: audit 2023-09-23T19:31:43.637037+0000 mon.a (mon.0) 916 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:31:44.617 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:44.639 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:44.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:44.641 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:44.675 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:44 smithi016 bash[77995]: audit 2023-09-23T19:31:43.637037+0000 mon.a (mon.0) 916 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:31:44.675 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:44 smithi016 bash[75004]: audit 2023-09-23T19:31:43.637037+0000 mon.a (mon.0) 916 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:31:44.742 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:44.769 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:44.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:44.772 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:44.873 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:44.904 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:44.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:44.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:45.007 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:45.030 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:45.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:45.033 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:45.133 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:45.159 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:45.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:45.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:45.263 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:45.288 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:45.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:45.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:45.291 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 11 16 24454 24438 138.838 97.75 0.0135442 0.00719474 2023-09-23T19:31:45.391 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:45.426 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:45.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:45.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:31:45.530 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:45.534 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:45 smithi016 bash[77995]: cluster 2023-09-23T19:31:43.958935+0000 mgr.y (mgr.25325) 722 : cluster 0 pgmap v620: 89 pgs: 89 active+clean; 1.0 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-09-23T19:31:45.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:45 smithi016 bash[75004]: cluster 2023-09-23T19:31:43.958935+0000 mgr.y (mgr.25325) 722 : cluster 0 pgmap v620: 89 pgs: 89 active+clean; 1.0 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-09-23T19:31:45.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:45 smithi092 bash[58855]: cluster 2023-09-23T19:31:43.958935+0000 mgr.y (mgr.25325) 722 : cluster 0 pgmap v620: 89 pgs: 89 active+clean; 1.0 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-09-23T19:31:45.568 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:45.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:45.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:31:45.674 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:45.698 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:45.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:45.701 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:45.773 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:31:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:31:45] "GET /metrics HTTP/1.1" 200 36227 "" "Prometheus/2.33.4" 2023-09-23T19:31:45.802 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:45.834 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:45.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:45.837 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:45.937 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:45.967 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:45.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:45.971 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:46.072 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:46.101 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:46.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:46.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:31:46.205 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:46.231 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:46.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:46.233 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:46.290 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 12 16 26095 26079 135.814 102.562 0.00561804 0.00735687 2023-09-23T19:31:46.334 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:46.361 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:46.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:46.363 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:46.464 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:46.487 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:46.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:46.490 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:46.591 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:46.620 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:46.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:46.623 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:46.724 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:46.762 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:46.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:46.766 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:31:46.867 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:46.890 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:46.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:46.893 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:46.993 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:47.021 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:47.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:47.024 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:47.124 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:47.153 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:47.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:47.156 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:47.257 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:47.283 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:47.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:47.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:47.290 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 13 16 27490 27474 132.073 87.1875 0.00199286 0.0075318 2023-09-23T19:31:47.387 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:47.420 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:47.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:47.423 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:31:47.523 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:47.527 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:47 smithi016 bash[77995]: cluster 2023-09-23T19:31:45.959700+0000 mgr.y (mgr.25325) 723 : cluster 0 pgmap v621: 89 pgs: 89 active+clean; 1.4 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 120 MiB/s wr, 1.92k op/s 2023-09-23T19:31:47.528 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:47 smithi016 bash[75004]: cluster 2023-09-23T19:31:45.959700+0000 mgr.y (mgr.25325) 723 : cluster 0 pgmap v621: 89 pgs: 89 active+clean; 1.4 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 120 MiB/s wr, 1.92k op/s 2023-09-23T19:31:47.532 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:47 smithi092 bash[58855]: cluster 2023-09-23T19:31:45.959700+0000 mgr.y (mgr.25325) 723 : cluster 0 pgmap v621: 89 pgs: 89 active+clean; 1.4 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 120 MiB/s wr, 1.92k op/s 2023-09-23T19:31:47.550 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:47.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:47.554 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:31:47.655 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:47.676 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:47.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:47.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:47.779 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:47.805 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:47.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:47.808 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:31:47.909 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:47.940 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:47.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:47.944 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:31:48.045 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:48.069 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:48.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:48.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:48.172 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:48.202 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:48.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:48.209 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:31:48.291 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 14 16 29065 29049 129.67 98.4375 0.00639183 0.0077054 2023-09-23T19:31:48.310 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:48.335 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:48.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:48.339 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:48.439 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:48.463 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:48.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:48.465 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:48.566 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:48.598 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:48.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:48.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:48.703 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:48.742 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:48.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:48.747 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:31:48.848 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:48.879 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:48.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:48.881 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:48.982 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:49.011 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:49.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:49.015 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:49.115 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:49.140 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:49.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:49.143 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:49.243 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:49.273 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:49.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:49.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:49.291 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 15 16 30716 30700 127.904 103.188 0.00451792 0.00781195 2023-09-23T19:31:49.376 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:49.398 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:49.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:49.402 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:49.503 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:49.528 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:49.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:49.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:49.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:49 smithi092 bash[58855]: cluster 2023-09-23T19:31:47.960285+0000 mgr.y (mgr.25325) 724 : cluster 0 pgmap v622: 89 pgs: 89 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 116 MiB/s wr, 1.85k op/s 2023-09-23T19:31:49.632 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:49.641 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:49 smithi016 bash[77995]: cluster 2023-09-23T19:31:47.960285+0000 mgr.y (mgr.25325) 724 : cluster 0 pgmap v622: 89 pgs: 89 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 116 MiB/s wr, 1.85k op/s 2023-09-23T19:31:49.642 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:49 smithi016 bash[75004]: cluster 2023-09-23T19:31:47.960285+0000 mgr.y (mgr.25325) 724 : cluster 0 pgmap v622: 89 pgs: 89 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 116 MiB/s wr, 1.85k op/s 2023-09-23T19:31:49.661 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:49.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:49.664 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:49.765 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:49.793 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:49.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:49.798 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:49.898 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:49.931 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:49.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:49.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:50.036 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:50.074 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:50.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:50.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:50.177 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:50.203 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:50.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:50.209 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:31:50.294 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 16 16 32420 32404 126.566 106.5 0.00451194 0.00789387 2023-09-23T19:31:50.310 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:50.333 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:50.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:50.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:50.436 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:50.462 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:50.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:50.465 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:50.566 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:50.594 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:50.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:50.597 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:31:50.698 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:50.726 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:50.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:50.732 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:50.833 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:50.857 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:50.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:50.860 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:50.960 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:50.995 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:50.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:50.999 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:51.100 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:51.125 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:51.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:51.128 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:31:51.229 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:51.262 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:51.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:51.266 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:51.291 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 17 16 34029 34013 125.035 100.562 0.00986293 0.00799089 2023-09-23T19:31:51.367 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:51.394 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:51.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:51.400 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:51.501 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:51.510 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:51 smithi016 bash[75004]: cluster 2023-09-23T19:31:49.962993+0000 mgr.y (mgr.25325) 725 : cluster 0 pgmap v623: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2023-09-23T19:31:51.510 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:51 smithi016 bash[77995]: cluster 2023-09-23T19:31:49.962993+0000 mgr.y (mgr.25325) 725 : cluster 0 pgmap v623: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2023-09-23T19:31:51.527 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:51.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:51.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:51.631 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:51.639 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:51 smithi092 bash[58855]: cluster 2023-09-23T19:31:49.962993+0000 mgr.y (mgr.25325) 725 : cluster 0 pgmap v623: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2023-09-23T19:31:51.656 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:51.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:51.662 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:51.763 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:51.788 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:51.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:51.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:51.891 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:51.921 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:51.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:51.927 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:31:52.028 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:52.051 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:52.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:52.056 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:52.157 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:52.190 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:52.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:52.192 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:52.291 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 18 16 35670 35654 123.786 102.562 0.00730098 0.00807177 2023-09-23T19:31:52.326 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:52.362 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:52.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:52.366 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:52.467 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:52.496 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:52.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:52.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:52.600 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:52.629 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:52.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:52.631 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:31:52.732 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:52.767 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:52.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:52.771 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:52.871 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:52.899 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:52.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:52.902 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:31:53.002 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:53.031 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:53.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:53.034 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:53.135 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:53.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:53.166 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:53.167 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:53.267 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:53.291 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 19 16 37114 37098 122.021 90.25 0.0130388 0.0081885 2023-09-23T19:31:53.298 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:53.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:53.302 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:53.403 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:53.426 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:53.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:53.431 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:53.532 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:53.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:53 smithi092 bash[58855]: cluster 2023-09-23T19:31:51.963642+0000 mgr.y (mgr.25325) 726 : cluster 0 pgmap v624: 89 pgs: 89 active+clean; 2.0 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 130 MiB/s wr, 2.08k op/s 2023-09-23T19:31:53.559 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:53.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:53.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:53.664 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:53.675 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:53 smithi016 bash[75004]: cluster 2023-09-23T19:31:51.963642+0000 mgr.y (mgr.25325) 726 : cluster 0 pgmap v624: 89 pgs: 89 active+clean; 2.0 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 130 MiB/s wr, 2.08k op/s 2023-09-23T19:31:53.675 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:53 smithi016 bash[77995]: cluster 2023-09-23T19:31:51.963642+0000 mgr.y (mgr.25325) 726 : cluster 0 pgmap v624: 89 pgs: 89 active+clean; 2.0 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 130 MiB/s wr, 2.08k op/s 2023-09-23T19:31:53.702 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:53.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:53.705 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:53.806 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:53.832 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:53.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:53.835 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:53.935 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:53.968 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:53.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:53.971 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:54.071 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:54.099 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:54.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:54.102 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:54.202 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:54.233 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:54.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:54.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:54.250 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-09-23T19:31:54.250 INFO:teuthology.orchestra.run.smithi016.stdout: "size": 20, 2023-09-23T19:31:54.250 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 600, 2023-09-23T19:31:54.250 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [ 2023-09-23T19:31:54.250 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.250 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.34836.0:27883 8.1 8:88969814:::benchmark_data_smithi016_91497_object27882:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e99)", 2023-09-23T19:31:54.250 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:22:42.130036+0000", 2023-09-23T19:31:54.250 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 552.11245642799997, 2023-09-23T19:31:54.251 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.128187067, 2023-09-23T19:31:54.251 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:31:54.251 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "reached pg", 2023-09-23T19:31:54.251 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:31:54.251 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.34836", 2023-09-23T19:31:54.251 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1127631747", 2023-09-23T19:31:54.251 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 27883 2023-09-23T19:31:54.251 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.251 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:31:54.251 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.252 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:31:54.252 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:42.130036+0000", 2023-09-23T19:31:54.252 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:31:54.252 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.252 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.252 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:31:54.252 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:42.130036+0000", 2023-09-23T19:31:54.252 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:31:54.252 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.252 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.253 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:31:54.253 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:42.130033+0000", 2023-09-23T19:31:54.253 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999976 2023-09-23T19:31:54.253 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.253 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.253 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:31:54.253 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:42.130087+0000", 2023-09-23T19:31:54.253 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.3838000000000002e-05 2023-09-23T19:31:54.253 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.253 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.253 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:31:54.254 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:42.130103+0000", 2023-09-23T19:31:54.254 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.5650000000000001e-05 2023-09-23T19:31:54.254 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.254 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.254 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:31:54.254 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:42.130116+0000", 2023-09-23T19:31:54.254 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3434999999999999e-05 2023-09-23T19:31:54.254 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.254 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.254 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:31:54.255 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:42.130245+0000", 2023-09-23T19:31:54.255 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000128687 2023-09-23T19:31:54.255 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.255 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.255 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-09-23T19:31:54.255 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:42.130260+0000", 2023-09-23T19:31:54.255 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.5139e-05 2023-09-23T19:31:54.255 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.255 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.255 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:31:54.255 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:43.257922+0000", 2023-09-23T19:31:54.256 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1276620099999999 2023-09-23T19:31:54.256 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.256 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.256 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:31:54.256 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:43.258223+0000", 2023-09-23T19:31:54.256 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00030047299999999999 2023-09-23T19:31:54.256 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:31:54.256 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:31:54.256 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:31:54.256 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.256 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.256 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.34836.0:27898 8.1 8:8284f325:::benchmark_data_smithi016_91497_object27897:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e99)", 2023-09-23T19:31:54.257 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:22:42.130133+0000", 2023-09-23T19:31:54.257 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 552.11235945700003, 2023-09-23T19:31:54.257 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.157612731, 2023-09-23T19:31:54.257 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:31:54.257 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:31:54.257 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:31:54.257 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.34836", 2023-09-23T19:31:54.257 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1127631747", 2023-09-23T19:31:54.257 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 27898 2023-09-23T19:31:54.257 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.258 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:31:54.258 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.258 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:31:54.258 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:42.130133+0000", 2023-09-23T19:31:54.258 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:31:54.258 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.258 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.258 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:31:54.258 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:42.130133+0000", 2023-09-23T19:31:54.258 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:31:54.258 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.258 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.258 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:31:54.259 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:42.130132+0000", 2023-09-23T19:31:54.259 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:31:54.259 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.259 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.259 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:31:54.259 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:42.130170+0000", 2023-09-23T19:31:54.259 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.8665999999999999e-05 2023-09-23T19:31:54.259 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.259 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.259 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:31:54.259 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:42.130182+0000", 2023-09-23T19:31:54.259 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1576999999999999e-05 2023-09-23T19:31:54.260 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.260 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.260 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:31:54.260 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:42.130191+0000", 2023-09-23T19:31:54.260 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.2180000000000005e-06 2023-09-23T19:31:54.260 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.260 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.260 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:31:54.260 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:42.130303+0000", 2023-09-23T19:31:54.260 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000111673 2023-09-23T19:31:54.260 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.260 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.261 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-09-23T19:31:54.261 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:42.130311+0000", 2023-09-23T19:31:54.261 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.4470000000000006e-06 2023-09-23T19:31:54.261 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.261 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.261 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:31:54.261 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:43.260301+0000", 2023-09-23T19:31:54.261 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1299895449999999 2023-09-23T19:31:54.261 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.261 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.261 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:31:54.261 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:43.260430+0000", 2023-09-23T19:31:54.262 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00012896399999999999 2023-09-23T19:31:54.262 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.262 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.262 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 1,2", 2023-09-23T19:31:54.262 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:43.260551+0000", 2023-09-23T19:31:54.262 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000121468 2023-09-23T19:31:54.262 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.262 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.262 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:31:54.262 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:43.263190+0000", 2023-09-23T19:31:54.262 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0026390070000000001 2023-09-23T19:31:54.262 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.263 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.263 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:31:54.263 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:43.263230+0000", 2023-09-23T19:31:54.263 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.9356000000000001e-05 2023-09-23T19:31:54.263 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.263 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.263 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:31:54.263 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:43.286277+0000", 2023-09-23T19:31:54.263 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.023047339999999999 2023-09-23T19:31:54.263 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.263 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.264 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:31:54.264 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:43.286542+0000", 2023-09-23T19:31:54.264 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00026542500000000001 2023-09-23T19:31:54.264 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.264 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.264 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:31:54.264 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:43.287745+0000", 2023-09-23T19:31:54.264 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0012029219999999999 2023-09-23T19:31:54.264 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:31:54.264 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:31:54.264 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:31:54.265 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.265 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.265 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.34836.0:27907 8.1 8:86ac35ab:::benchmark_data_smithi016_91497_object27906:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e99)", 2023-09-23T19:31:54.265 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:22:42.130199+0000", 2023-09-23T19:31:54.265 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 552.11229332100004, 2023-09-23T19:31:54.265 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.159096994, 2023-09-23T19:31:54.265 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:31:54.265 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:31:54.265 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:31:54.265 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.34836", 2023-09-23T19:31:54.265 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1127631747", 2023-09-23T19:31:54.265 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 27907 2023-09-23T19:31:54.266 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.266 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:31:54.266 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.266 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:31:54.266 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:42.130199+0000", 2023-09-23T19:31:54.266 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:31:54.266 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.266 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.266 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:31:54.266 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:42.130199+0000", 2023-09-23T19:31:54.266 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:31:54.267 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.267 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.267 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:31:54.267 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:42.130198+0000", 2023-09-23T19:31:54.267 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:31:54.267 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.267 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.267 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:31:54.267 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:42.130234+0000", 2023-09-23T19:31:54.267 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.6161999999999997e-05 2023-09-23T19:31:54.267 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.267 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.268 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:31:54.268 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:42.130243+0000", 2023-09-23T19:31:54.268 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.5189999999999998e-06 2023-09-23T19:31:54.268 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.268 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.268 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:31:54.268 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:42.130251+0000", 2023-09-23T19:31:54.268 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.1160000000000001e-06 2023-09-23T19:31:54.268 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.268 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.268 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:31:54.268 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:42.130339+0000", 2023-09-23T19:31:54.269 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.8843000000000003e-05 2023-09-23T19:31:54.269 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.269 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.269 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-09-23T19:31:54.269 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:42.130347+0000", 2023-09-23T19:31:54.269 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.729e-06 2023-09-23T19:31:54.269 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.269 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.269 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:31:54.269 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:43.264839+0000", 2023-09-23T19:31:54.269 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1344920380000001 2023-09-23T19:31:54.270 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.270 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.270 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:31:54.270 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:43.265165+0000", 2023-09-23T19:31:54.270 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00032538199999999998 2023-09-23T19:31:54.270 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.270 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.270 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 1,2", 2023-09-23T19:31:54.270 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:43.265553+0000", 2023-09-23T19:31:54.270 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00038889300000000002 2023-09-23T19:31:54.270 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.271 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.271 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:31:54.271 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:43.272403+0000", 2023-09-23T19:31:54.271 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0068496290000000003 2023-09-23T19:31:54.271 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.271 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.271 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:31:54.271 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:43.277977+0000", 2023-09-23T19:31:54.271 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0055737579999999998 2023-09-23T19:31:54.271 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.271 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.271 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:31:54.272 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:43.287872+0000", 2023-09-23T19:31:54.272 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0098953940000000001 2023-09-23T19:31:54.272 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.272 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.272 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:31:54.272 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:43.287989+0000", 2023-09-23T19:31:54.272 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000117007 2023-09-23T19:31:54.272 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.272 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.272 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:31:54.272 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:43.289296+0000", 2023-09-23T19:31:54.272 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001306389 2023-09-23T19:31:54.272 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:31:54.273 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:31:54.273 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:31:54.273 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.273 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.273 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.34836.0:27909 8.1 8:83ffecad:::benchmark_data_smithi016_91497_object27908:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e99)", 2023-09-23T19:31:54.273 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:22:42.130259+0000", 2023-09-23T19:31:54.273 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 552.11223308700005, 2023-09-23T19:31:54.273 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1598347920000001, 2023-09-23T19:31:54.273 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:31:54.273 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:31:54.273 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:31:54.273 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.34836", 2023-09-23T19:31:54.274 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1127631747", 2023-09-23T19:31:54.274 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 27909 2023-09-23T19:31:54.274 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.274 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:31:54.274 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.274 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:31:54.274 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:42.130259+0000", 2023-09-23T19:31:54.274 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:31:54.274 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.274 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.274 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:31:54.275 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:42.130259+0000", 2023-09-23T19:31:54.275 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:31:54.275 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.275 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.275 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:31:54.275 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:42.130257+0000", 2023-09-23T19:31:54.275 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999976 2023-09-23T19:31:54.275 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.275 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.275 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:31:54.275 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:42.130304+0000", 2023-09-23T19:31:54.275 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.7778999999999998e-05 2023-09-23T19:31:54.276 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.276 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.276 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:31:54.276 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:42.130319+0000", 2023-09-23T19:31:54.276 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.4422e-05 2023-09-23T19:31:54.276 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.276 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.276 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:31:54.276 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:42.130330+0000", 2023-09-23T19:31:54.276 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0978999999999999e-05 2023-09-23T19:31:54.276 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.277 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.277 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:31:54.277 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:42.130382+0000", 2023-09-23T19:31:54.277 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.1674e-05 2023-09-23T19:31:54.277 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.277 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.277 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-09-23T19:31:54.277 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:42.130390+0000", 2023-09-23T19:31:54.277 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.8869999999999998e-06 2023-09-23T19:31:54.277 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.277 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.278 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:31:54.278 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:43.267356+0000", 2023-09-23T19:31:54.278 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1369651890000001 2023-09-23T19:31:54.278 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.278 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.278 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:31:54.278 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:43.267648+0000", 2023-09-23T19:31:54.278 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00029235200000000001 2023-09-23T19:31:54.278 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.278 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.278 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 1,2", 2023-09-23T19:31:54.278 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:43.268073+0000", 2023-09-23T19:31:54.279 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00042524899999999999 2023-09-23T19:31:54.279 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.279 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.279 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:31:54.279 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:43.276000+0000", 2023-09-23T19:31:54.279 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0079268710000000003 2023-09-23T19:31:54.279 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.279 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.279 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:31:54.279 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:43.278123+0000", 2023-09-23T19:31:54.279 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0021226700000000001 2023-09-23T19:31:54.279 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.280 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.280 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:31:54.280 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:43.289364+0000", 2023-09-23T19:31:54.280 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.011241434999999999 2023-09-23T19:31:54.280 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.280 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.280 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:31:54.280 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:43.289431+0000", 2023-09-23T19:31:54.280 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.6742999999999994e-05 2023-09-23T19:31:54.280 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.280 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.280 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:31:54.281 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:43.290094+0000", 2023-09-23T19:31:54.281 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00066274699999999999 2023-09-23T19:31:54.281 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:31:54.281 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:31:54.281 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:31:54.281 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.281 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.281 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.34836.0:27921 8.1 8:89f617d0:::benchmark_data_smithi016_91497_object27920:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e99)", 2023-09-23T19:31:54.281 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:22:42.130342+0000", 2023-09-23T19:31:54.281 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 552.11214981299997, 2023-09-23T19:31:54.281 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1610621919999999, 2023-09-23T19:31:54.282 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:31:54.282 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:31:54.282 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:31:54.282 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.34836", 2023-09-23T19:31:54.282 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1127631747", 2023-09-23T19:31:54.282 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 27921 2023-09-23T19:31:54.282 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.282 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:31:54.282 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.282 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:31:54.282 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:42.130342+0000", 2023-09-23T19:31:54.282 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:31:54.283 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.283 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.283 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:31:54.283 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:42.130342+0000", 2023-09-23T19:31:54.283 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:31:54.283 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.283 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.283 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:31:54.283 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:42.130341+0000", 2023-09-23T19:31:54.283 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:31:54.283 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.283 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.283 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:31:54.284 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:42.130377+0000", 2023-09-23T19:31:54.284 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.5871999999999998e-05 2023-09-23T19:31:54.284 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.284 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.284 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:31:54.284 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:42.130389+0000", 2023-09-23T19:31:54.284 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.135e-05 2023-09-23T19:31:54.284 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.284 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.284 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:31:54.284 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:42.130399+0000", 2023-09-23T19:31:54.284 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0034000000000001e-05 2023-09-23T19:31:54.284 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.285 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.285 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:31:54.285 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:42.130448+0000", 2023-09-23T19:31:54.285 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.9116000000000002e-05 2023-09-23T19:31:54.285 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.285 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.285 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-09-23T19:31:54.285 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:42.130457+0000", 2023-09-23T19:31:54.285 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.3030000000000005e-06 2023-09-23T19:31:54.285 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.285 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.285 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:31:54.286 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:43.269843+0000", 2023-09-23T19:31:54.286 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1393857729999999 2023-09-23T19:31:54.286 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.286 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.286 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:31:54.286 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:43.270039+0000", 2023-09-23T19:31:54.286 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000196617 2023-09-23T19:31:54.286 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.286 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.286 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 1,2", 2023-09-23T19:31:54.286 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:43.270361+0000", 2023-09-23T19:31:54.287 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00032172499999999997 2023-09-23T19:31:54.287 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.287 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.287 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:31:54.287 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:43.276019+0000", 2023-09-23T19:31:54.287 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0056575169999999999 2023-09-23T19:31:54.287 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.287 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.287 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:31:54.287 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:43.278333+0000", 2023-09-23T19:31:54.287 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0023139789999999999 2023-09-23T19:31:54.287 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.287 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.288 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:31:54.288 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:43.290161+0000", 2023-09-23T19:31:54.288 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.01182827 2023-09-23T19:31:54.288 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.288 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.288 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:31:54.288 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:43.290224+0000", 2023-09-23T19:31:54.288 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.3235e-05 2023-09-23T19:31:54.288 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.288 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.288 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:31:54.288 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:43.291404+0000", 2023-09-23T19:31:54.288 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001180223 2023-09-23T19:31:54.289 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:31:54.289 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:31:54.289 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:31:54.289 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.289 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.289 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.34836.0:27922 8.1 8:8a60e720:::benchmark_data_smithi016_91497_object27921:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e99)", 2023-09-23T19:31:54.289 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:22:42.130416+0000", 2023-09-23T19:31:54.289 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 552.11207627299996, 2023-09-23T19:31:54.289 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1666472029999999, 2023-09-23T19:31:54.289 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:31:54.289 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:31:54.289 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:31:54.290 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.34836", 2023-09-23T19:31:54.290 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1127631747", 2023-09-23T19:31:54.290 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 27922 2023-09-23T19:31:54.290 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.290 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:31:54.290 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.290 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:31:54.290 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:42.130416+0000", 2023-09-23T19:31:54.290 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:31:54.290 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.290 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.291 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:31:54.291 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:42.130416+0000", 2023-09-23T19:31:54.291 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:31:54.291 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.291 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.291 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:31:54.291 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:42.130415+0000", 2023-09-23T19:31:54.291 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:31:54.291 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.291 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.291 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:31:54.291 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:42.130449+0000", 2023-09-23T19:31:54.292 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.4273000000000002e-05 2023-09-23T19:31:54.292 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.292 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.292 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:31:54.292 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:42.130459+0000", 2023-09-23T19:31:54.292 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.7710000000000001e-06 2023-09-23T19:31:54.292 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.292 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.292 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:31:54.292 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:42.130466+0000", 2023-09-23T19:31:54.292 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.311e-06 2023-09-23T19:31:54.293 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.293 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.293 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:31:54.293 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:42.130519+0000", 2023-09-23T19:31:54.293 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.2460999999999998e-05 2023-09-23T19:31:54.293 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.293 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.293 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-09-23T19:31:54.293 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:42.130528+0000", 2023-09-23T19:31:54.293 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.9889999999999998e-06 2023-09-23T19:31:54.293 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.294 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.294 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:31:54.294 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:43.272529+0000", 2023-09-23T19:31:54.294 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1420009419999999 2023-09-23T19:31:54.294 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.294 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.294 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:31:54.294 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:43.273015+0000", 2023-09-23T19:31:54.294 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00048667799999999999 2023-09-23T19:31:54.295 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.295 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.295 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 1,2", 2023-09-23T19:31:54.295 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:43.273638+0000", 2023-09-23T19:31:54.296 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00062293099999999996 2023-09-23T19:31:54.296 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.296 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.296 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:31:54.296 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:43.276040+0000", 2023-09-23T19:31:54.296 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0024017660000000001 2023-09-23T19:31:54.296 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.296 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.296 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:31:54.296 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:43.291596+0000", 2023-09-23T19:31:54.296 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.015556057999999999 2023-09-23T19:31:54.297 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.297 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.297 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:31:54.297 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:43.292458+0000", 2023-09-23T19:31:54.297 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00086221400000000004 2023-09-23T19:31:54.297 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.297 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.297 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:31:54.297 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:43.292784+0000", 2023-09-23T19:31:54.297 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00032612299999999999 2023-09-23T19:31:54.297 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.297 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.298 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:31:54.298 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:22:43.297063+0000", 2023-09-23T19:31:54.298 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0042784709999999998 2023-09-23T19:31:54.298 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:31:54.298 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:31:54.298 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:31:54.298 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.298 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.298 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45671.0:19142 9.b 9:d3b64c3d:::benchmark_data_smithi016_113233_object19141:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e122)", 2023-09-23T19:31:54.298 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:31:42.042244+0000", 2023-09-23T19:31:54.299 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 12.200247693, 2023-09-23T19:31:54.299 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.15077631799999999, 2023-09-23T19:31:54.299 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:31:54.299 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:31:54.299 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:31:54.299 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45671", 2023-09-23T19:31:54.299 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:31:54.299 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 19142 2023-09-23T19:31:54.299 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.299 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:31:54.299 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.300 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:31:54.300 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.042244+0000", 2023-09-23T19:31:54.300 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:31:54.300 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.300 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.300 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:31:54.300 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.042244+0000", 2023-09-23T19:31:54.300 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:31:54.300 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.300 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.300 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:31:54.300 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.042244+0000", 2023-09-23T19:31:54.301 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T19:31:54.301 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.301 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.301 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:31:54.301 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.042261+0000", 2023-09-23T19:31:54.301 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.6874000000000001e-05 2023-09-23T19:31:54.301 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.301 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.301 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:31:54.301 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.042268+0000", 2023-09-23T19:31:54.301 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.8630000000000002e-06 2023-09-23T19:31:54.301 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.302 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.302 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:31:54.302 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.042274+0000", 2023-09-23T19:31:54.302 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.3380000000000001e-06 2023-09-23T19:31:54.302 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.302 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.302 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:31:54.302 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.043858+0000", 2023-09-23T19:31:54.302 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0015839669999999999 2023-09-23T19:31:54.302 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.302 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.303 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:31:54.303 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.043926+0000", 2023-09-23T19:31:54.303 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.8548999999999997e-05 2023-09-23T19:31:54.303 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.303 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.303 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 5,6", 2023-09-23T19:31:54.303 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.044006+0000", 2023-09-23T19:31:54.303 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.9386000000000002e-05 2023-09-23T19:31:54.303 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.303 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.303 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:31:54.303 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.045454+0000", 2023-09-23T19:31:54.304 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00144794 2023-09-23T19:31:54.304 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.304 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.304 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:31:54.304 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.046460+0000", 2023-09-23T19:31:54.304 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0010058529999999999 2023-09-23T19:31:54.304 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.304 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.304 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:31:54.304 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.046898+0000", 2023-09-23T19:31:54.304 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00043816100000000001 2023-09-23T19:31:54.304 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.304 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.305 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:31:54.305 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.192884+0000", 2023-09-23T19:31:54.305 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.14598634999999999 2023-09-23T19:31:54.305 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.305 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.305 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:31:54.305 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.193021+0000", 2023-09-23T19:31:54.305 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000136475 2023-09-23T19:31:54.305 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:31:54.305 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:31:54.305 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:31:54.305 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.305 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.306 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45671.0:19143 9.b 9:d463260b:::benchmark_data_smithi016_113233_object19142:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e122)", 2023-09-23T19:31:54.306 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:31:42.043806+0000", 2023-09-23T19:31:54.306 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 12.198685527, 2023-09-23T19:31:54.306 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.14957359000000001, 2023-09-23T19:31:54.306 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:31:54.306 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:31:54.306 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:31:54.306 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45671", 2023-09-23T19:31:54.306 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:31:54.306 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 19143 2023-09-23T19:31:54.306 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.306 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:31:54.307 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.307 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:31:54.307 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.043806+0000", 2023-09-23T19:31:54.307 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:31:54.307 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.307 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.307 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:31:54.307 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.043806+0000", 2023-09-23T19:31:54.307 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:31:54.307 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.307 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.307 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:31:54.307 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.043805+0000", 2023-09-23T19:31:54.308 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:31:54.308 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.308 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.308 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:31:54.308 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.043830+0000", 2023-09-23T19:31:54.308 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.4510000000000001e-05 2023-09-23T19:31:54.308 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.308 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.308 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:31:54.308 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.043843+0000", 2023-09-23T19:31:54.308 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2972e-05 2023-09-23T19:31:54.308 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.308 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.309 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:31:54.309 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.043849+0000", 2023-09-23T19:31:54.309 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.049e-06 2023-09-23T19:31:54.309 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.309 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.309 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:31:54.309 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.044543+0000", 2023-09-23T19:31:54.309 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00069373899999999997 2023-09-23T19:31:54.309 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.309 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.309 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:31:54.310 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.044667+0000", 2023-09-23T19:31:54.310 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00012435500000000001 2023-09-23T19:31:54.310 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.310 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.310 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 5,6", 2023-09-23T19:31:54.310 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.044790+0000", 2023-09-23T19:31:54.310 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0001227 2023-09-23T19:31:54.310 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.310 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.310 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:31:54.310 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.045861+0000", 2023-09-23T19:31:54.310 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0010713420000000001 2023-09-23T19:31:54.311 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.311 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.311 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:31:54.311 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.046629+0000", 2023-09-23T19:31:54.311 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00076834099999999997 2023-09-23T19:31:54.311 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.311 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.311 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:31:54.311 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.193294+0000", 2023-09-23T19:31:54.311 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.14666451899999999 2023-09-23T19:31:54.311 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.311 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.311 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:31:54.312 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.193342+0000", 2023-09-23T19:31:54.312 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.7833000000000001e-05 2023-09-23T19:31:54.312 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.312 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.312 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:31:54.312 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.193380+0000", 2023-09-23T19:31:54.312 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.8285999999999999e-05 2023-09-23T19:31:54.312 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:31:54.312 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:31:54.312 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:31:54.312 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.313 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.313 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop_reply(client.45671.0:19142 9.b e122/119 ondisk, result = 0)", 2023-09-23T19:31:54.313 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:31:42.046651+0000", 2023-09-23T19:31:54.313 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 12.195840465, 2023-09-23T19:31:54.313 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.14639395199999999, 2023-09-23T19:31:54.313 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:31:54.313 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "started", 2023-09-23T19:31:54.313 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:31:54.313 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.313 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:31:54.313 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.046651+0000", 2023-09-23T19:31:54.314 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:31:54.314 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.314 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.314 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:31:54.314 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.046651+0000", 2023-09-23T19:31:54.314 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:31:54.314 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.314 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.314 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:31:54.314 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.046651+0000", 2023-09-23T19:31:54.314 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967296 2023-09-23T19:31:54.315 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.315 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.315 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:31:54.315 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.046653+0000", 2023-09-23T19:31:54.315 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.096e-06 2023-09-23T19:31:54.315 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.315 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.315 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:31:54.315 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.046655+0000", 2023-09-23T19:31:54.315 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.649e-06 2023-09-23T19:31:54.315 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.315 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.315 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:31:54.316 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.046657+0000", 2023-09-23T19:31:54.316 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.125e-06 2023-09-23T19:31:54.316 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.316 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.316 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:31:54.316 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.046881+0000", 2023-09-23T19:31:54.316 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000223828 2023-09-23T19:31:54.316 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.316 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.316 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:31:54.316 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.046888+0000", 2023-09-23T19:31:54.316 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.3440000000000003e-06 2023-09-23T19:31:54.317 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.317 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.317 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:31:54.317 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.193045+0000", 2023-09-23T19:31:54.317 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.14615699600000001 2023-09-23T19:31:54.317 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:31:54.317 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:31:54.317 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:31:54.317 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.317 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.317 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45671.0:19154 9.9 9:9fca7707:::benchmark_data_smithi016_113233_object19153:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e122)", 2023-09-23T19:31:54.318 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:31:42.048781+0000", 2023-09-23T19:31:54.318 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 12.193710968, 2023-09-23T19:31:54.318 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.164320364, 2023-09-23T19:31:54.318 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:31:54.318 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:31:54.318 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:31:54.318 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45671", 2023-09-23T19:31:54.318 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:31:54.318 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 19154 2023-09-23T19:31:54.318 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.319 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:31:54.319 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.319 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:31:54.319 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.048781+0000", 2023-09-23T19:31:54.319 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:31:54.319 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.319 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.319 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:31:54.319 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.048781+0000", 2023-09-23T19:31:54.319 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:31:54.319 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.320 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.320 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:31:54.320 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.048779+0000", 2023-09-23T19:31:54.320 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-09-23T19:31:54.320 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.320 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.320 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:31:54.320 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.048804+0000", 2023-09-23T19:31:54.320 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.4301e-05 2023-09-23T19:31:54.320 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.320 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.321 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:31:54.321 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.193070+0000", 2023-09-23T19:31:54.321 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.144265904 2023-09-23T19:31:54.321 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.321 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.321 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:31:54.321 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.193094+0000", 2023-09-23T19:31:54.321 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.4773000000000002e-05 2023-09-23T19:31:54.321 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.321 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.321 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:31:54.322 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.193427+0000", 2023-09-23T19:31:54.322 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00033257799999999999 2023-09-23T19:31:54.322 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.322 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.322 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:31:54.322 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.193586+0000", 2023-09-23T19:31:54.322 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00015939900000000001 2023-09-23T19:31:54.322 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.322 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.322 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 5,7", 2023-09-23T19:31:54.322 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.195215+0000", 2023-09-23T19:31:54.322 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001628359 2023-09-23T19:31:54.323 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.323 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.323 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:31:54.323 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.198925+0000", 2023-09-23T19:31:54.323 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0037104009999999999 2023-09-23T19:31:54.323 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.323 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.323 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:31:54.323 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.204277+0000", 2023-09-23T19:31:54.323 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0053514280000000001 2023-09-23T19:31:54.323 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.323 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.324 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:31:54.324 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.212966+0000", 2023-09-23T19:31:54.324 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0086890839999999997 2023-09-23T19:31:54.324 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.324 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.324 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:31:54.324 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.213045+0000", 2023-09-23T19:31:54.324 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.9141999999999995e-05 2023-09-23T19:31:54.324 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.324 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.324 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:31:54.324 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.213101+0000", 2023-09-23T19:31:54.325 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.6511999999999997e-05 2023-09-23T19:31:54.325 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:31:54.325 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:31:54.325 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:31:54.325 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.325 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.325 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop_reply(client.45671.0:19143 9.b e122/119 ondisk, result = 0)", 2023-09-23T19:31:54.325 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:31:42.049081+0000", 2023-09-23T19:31:54.325 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 12.193411351, 2023-09-23T19:31:54.325 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.14431285899999999, 2023-09-23T19:31:54.326 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:31:54.326 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "started", 2023-09-23T19:31:54.326 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:31:54.326 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.326 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:31:54.326 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.049081+0000", 2023-09-23T19:31:54.326 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:31:54.326 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.326 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.326 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:31:54.326 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.049081+0000", 2023-09-23T19:31:54.327 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:31:54.327 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.327 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.327 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:31:54.327 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.049080+0000", 2023-09-23T19:31:54.327 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:31:54.327 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.327 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.327 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:31:54.327 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.049086+0000", 2023-09-23T19:31:54.327 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.3380000000000001e-06 2023-09-23T19:31:54.327 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.328 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.328 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:31:54.328 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.193214+0000", 2023-09-23T19:31:54.328 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.14412796999999999 2023-09-23T19:31:54.328 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.328 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.328 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:31:54.328 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.193227+0000", 2023-09-23T19:31:54.328 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.277e-05 2023-09-23T19:31:54.328 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.328 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.328 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:31:54.329 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.193276+0000", 2023-09-23T19:31:54.329 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.9400999999999997e-05 2023-09-23T19:31:54.329 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.329 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.329 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:31:54.329 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.193286+0000", 2023-09-23T19:31:54.329 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0084e-05 2023-09-23T19:31:54.329 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.329 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.329 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:31:54.329 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.193393+0000", 2023-09-23T19:31:54.329 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00010712700000000001 2023-09-23T19:31:54.330 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:31:54.330 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:31:54.330 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:31:54.330 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.330 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.330 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45671.0:25080 9.5 9:af076f85:::benchmark_data_smithi016_113233_object25079:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e122)", 2023-09-23T19:31:54.330 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:31:45.636690+0000", 2023-09-23T19:31:54.330 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 8.6058020919999993, 2023-09-23T19:31:54.330 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.136468279, 2023-09-23T19:31:54.330 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:31:54.330 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:31:54.330 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:31:54.331 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45671", 2023-09-23T19:31:54.331 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:31:54.331 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 25080 2023-09-23T19:31:54.331 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.331 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:31:54.331 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.331 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:31:54.331 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.636690+0000", 2023-09-23T19:31:54.331 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:31:54.331 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.331 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.332 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:31:54.332 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.636690+0000", 2023-09-23T19:31:54.332 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:31:54.332 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.332 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.332 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:31:54.332 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.636689+0000", 2023-09-23T19:31:54.332 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:31:54.332 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.332 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.332 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:31:54.332 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.636711+0000", 2023-09-23T19:31:54.333 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.1885000000000001e-05 2023-09-23T19:31:54.333 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.333 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.333 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:31:54.333 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.636720+0000", 2023-09-23T19:31:54.333 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.397e-06 2023-09-23T19:31:54.333 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.333 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.333 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:31:54.333 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.636726+0000", 2023-09-23T19:31:54.333 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.6890000000000001e-06 2023-09-23T19:31:54.334 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.334 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.334 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:31:54.334 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.636801+0000", 2023-09-23T19:31:54.334 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.5346999999999995e-05 2023-09-23T19:31:54.334 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.334 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.334 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:31:54.334 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.637084+0000", 2023-09-23T19:31:54.334 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00028237100000000001 2023-09-23T19:31:54.334 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.335 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.335 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 5,7", 2023-09-23T19:31:54.335 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.637291+0000", 2023-09-23T19:31:54.335 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000207152 2023-09-23T19:31:54.335 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.335 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.335 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:31:54.335 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.639127+0000", 2023-09-23T19:31:54.335 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0018358649999999999 2023-09-23T19:31:54.335 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.335 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.335 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:31:54.336 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.772939+0000", 2023-09-23T19:31:54.336 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.13381230899999999 2023-09-23T19:31:54.336 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.336 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.336 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:31:54.336 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.773005+0000", 2023-09-23T19:31:54.336 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.5860999999999998e-05 2023-09-23T19:31:54.336 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.336 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.336 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:31:54.336 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.773106+0000", 2023-09-23T19:31:54.337 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00010149000000000001 2023-09-23T19:31:54.337 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.337 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.337 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:31:54.337 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.773158+0000", 2023-09-23T19:31:54.337 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.1678000000000002e-05 2023-09-23T19:31:54.337 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:31:54.337 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:31:54.337 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:31:54.337 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.337 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.337 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45671.0:25081 9.9 9:9760ab71:::benchmark_data_smithi016_113233_object25080:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e122)", 2023-09-23T19:31:54.338 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:31:45.636739+0000", 2023-09-23T19:31:54.338 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 8.6057533460000002, 2023-09-23T19:31:54.338 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.13268706499999999, 2023-09-23T19:31:54.338 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:31:54.338 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:31:54.338 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:31:54.338 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45671", 2023-09-23T19:31:54.338 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:31:54.338 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 25081 2023-09-23T19:31:54.338 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.338 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:31:54.338 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.339 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:31:54.339 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.636739+0000", 2023-09-23T19:31:54.339 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:31:54.339 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.339 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.339 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:31:54.339 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.636739+0000", 2023-09-23T19:31:54.339 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:31:54.339 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.339 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.339 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:31:54.339 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.636738+0000", 2023-09-23T19:31:54.340 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T19:31:54.340 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.340 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.340 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:31:54.340 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.636789+0000", 2023-09-23T19:31:54.340 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.1215999999999999e-05 2023-09-23T19:31:54.340 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.340 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.340 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:31:54.340 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.636795+0000", 2023-09-23T19:31:54.340 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.7660000000000004e-06 2023-09-23T19:31:54.340 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.341 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.341 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:31:54.341 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.636799+0000", 2023-09-23T19:31:54.341 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.9190000000000001e-06 2023-09-23T19:31:54.341 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.341 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.341 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:31:54.341 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.637996+0000", 2023-09-23T19:31:54.341 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0011972 2023-09-23T19:31:54.341 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.341 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.342 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:31:54.342 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.638069+0000", 2023-09-23T19:31:54.342 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.224e-05 2023-09-23T19:31:54.342 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.342 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.342 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 5,7", 2023-09-23T19:31:54.342 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.638157+0000", 2023-09-23T19:31:54.342 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.8325999999999994e-05 2023-09-23T19:31:54.342 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.342 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.342 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:31:54.342 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.639404+0000", 2023-09-23T19:31:54.343 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001246989 2023-09-23T19:31:54.343 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.343 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.343 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:31:54.343 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.768541+0000", 2023-09-23T19:31:54.343 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.12913717499999999 2023-09-23T19:31:54.343 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.343 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.343 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:31:54.343 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.768670+0000", 2023-09-23T19:31:54.343 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00012930999999999999 2023-09-23T19:31:54.344 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.344 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.344 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:31:54.344 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.769311+0000", 2023-09-23T19:31:54.344 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00064020899999999998 2023-09-23T19:31:54.344 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.344 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.344 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:31:54.344 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.769426+0000", 2023-09-23T19:31:54.344 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00011514399999999999 2023-09-23T19:31:54.344 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:31:54.344 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:31:54.345 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:31:54.345 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.345 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.345 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45671.0:25083 9.b 9:d0955703:::benchmark_data_smithi016_113233_object25082:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e122)", 2023-09-23T19:31:54.345 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:31:45.636810+0000", 2023-09-23T19:31:54.345 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 8.6056817540000008, 2023-09-23T19:31:54.345 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.13163836000000001, 2023-09-23T19:31:54.345 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:31:54.345 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:31:54.345 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:31:54.345 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45671", 2023-09-23T19:31:54.346 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:31:54.346 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 25083 2023-09-23T19:31:54.346 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.346 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:31:54.346 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.346 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:31:54.346 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.636810+0000", 2023-09-23T19:31:54.346 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:31:54.346 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.346 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.346 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:31:54.347 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.636810+0000", 2023-09-23T19:31:54.347 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:31:54.347 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.347 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.347 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:31:54.347 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.636810+0000", 2023-09-23T19:31:54.347 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T19:31:54.347 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.347 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.347 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:31:54.347 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.636829+0000", 2023-09-23T19:31:54.348 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.9595000000000001e-05 2023-09-23T19:31:54.348 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.348 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.348 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:31:54.348 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.636834+0000", 2023-09-23T19:31:54.348 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.3830000000000002e-06 2023-09-23T19:31:54.348 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.348 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.348 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:31:54.348 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.636837+0000", 2023-09-23T19:31:54.348 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.1250000000000001e-06 2023-09-23T19:31:54.348 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.348 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.349 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:31:54.349 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.636865+0000", 2023-09-23T19:31:54.349 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.8048e-05 2023-09-23T19:31:54.349 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.349 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.349 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:31:54.349 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.636945+0000", 2023-09-23T19:31:54.349 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.9660999999999995e-05 2023-09-23T19:31:54.349 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.349 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.349 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 5,6", 2023-09-23T19:31:54.350 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.637028+0000", 2023-09-23T19:31:54.350 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.3227999999999997e-05 2023-09-23T19:31:54.350 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.350 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.350 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:31:54.350 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.638119+0000", 2023-09-23T19:31:54.350 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0010910830000000001 2023-09-23T19:31:54.350 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.350 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.350 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:31:54.350 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.768287+0000", 2023-09-23T19:31:54.350 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.13016824099999999 2023-09-23T19:31:54.351 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.351 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.351 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:31:54.351 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.768326+0000", 2023-09-23T19:31:54.351 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.8776999999999998e-05 2023-09-23T19:31:54.351 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.351 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.351 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:31:54.351 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.768395+0000", 2023-09-23T19:31:54.351 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.9416000000000001e-05 2023-09-23T19:31:54.351 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.352 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.352 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:31:54.352 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.768449+0000", 2023-09-23T19:31:54.352 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.3226000000000002e-05 2023-09-23T19:31:54.352 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:31:54.352 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:31:54.352 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:31:54.352 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.352 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.352 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop_reply(client.45671.0:25081 9.9 e122/119 ondisk, result = 0)", 2023-09-23T19:31:54.352 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:31:45.639744+0000", 2023-09-23T19:31:54.352 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 8.6027478849999994, 2023-09-23T19:31:54.353 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.128812961, 2023-09-23T19:31:54.353 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:31:54.353 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "started", 2023-09-23T19:31:54.353 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:31:54.353 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.353 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:31:54.353 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.639744+0000", 2023-09-23T19:31:54.353 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:31:54.353 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.353 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.353 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:31:54.353 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.639744+0000", 2023-09-23T19:31:54.354 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:31:54.354 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.354 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.354 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:31:54.354 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.639743+0000", 2023-09-23T19:31:54.354 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T19:31:54.354 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.354 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.354 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:31:54.354 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.639748+0000", 2023-09-23T19:31:54.354 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.7469999999999996e-06 2023-09-23T19:31:54.354 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.355 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.355 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:31:54.355 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.639756+0000", 2023-09-23T19:31:54.355 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.7309999999999993e-06 2023-09-23T19:31:54.355 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.355 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.355 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:31:54.355 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.639761+0000", 2023-09-23T19:31:54.355 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.2839999999999997e-06 2023-09-23T19:31:54.355 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.355 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.355 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:31:54.356 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.639782+0000", 2023-09-23T19:31:54.356 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.0763000000000001e-05 2023-09-23T19:31:54.356 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.356 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.356 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:31:54.356 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.639786+0000", 2023-09-23T19:31:54.356 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.3830000000000002e-06 2023-09-23T19:31:54.356 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.356 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.356 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:31:54.356 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.768557+0000", 2023-09-23T19:31:54.356 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.128770679 2023-09-23T19:31:54.357 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:31:54.357 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:31:54.357 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:31:54.357 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.357 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.357 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop_reply(client.45671.0:25083 9.b e122/119 ondisk, result = 0)", 2023-09-23T19:31:54.357 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:31:45.639800+0000", 2023-09-23T19:31:54.357 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 8.6026920489999998, 2023-09-23T19:31:54.357 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.12849273999999999, 2023-09-23T19:31:54.357 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:31:54.357 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "started", 2023-09-23T19:31:54.357 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:31:54.358 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.358 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:31:54.358 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.639800+0000", 2023-09-23T19:31:54.358 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:31:54.358 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.358 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.358 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:31:54.358 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.639800+0000", 2023-09-23T19:31:54.358 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:31:54.358 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.358 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.359 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:31:54.359 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.639799+0000", 2023-09-23T19:31:54.359 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:31:54.359 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.359 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.359 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:31:54.359 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.639804+0000", 2023-09-23T19:31:54.359 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.2059999999999998e-06 2023-09-23T19:31:54.359 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.359 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.359 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:31:54.359 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.768023+0000", 2023-09-23T19:31:54.360 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.12821864599999999 2023-09-23T19:31:54.360 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.360 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.360 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:31:54.360 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.768033+0000", 2023-09-23T19:31:54.360 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.9350000000000006e-06 2023-09-23T19:31:54.360 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.360 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.360 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:31:54.360 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.768266+0000", 2023-09-23T19:31:54.360 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000233074 2023-09-23T19:31:54.360 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.361 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.361 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:31:54.361 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.768279+0000", 2023-09-23T19:31:54.361 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.27e-05 2023-09-23T19:31:54.361 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.361 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.361 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:31:54.361 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.768293+0000", 2023-09-23T19:31:54.361 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3976e-05 2023-09-23T19:31:54.361 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:31:54.361 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:31:54.361 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:31:54.362 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.362 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.362 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop_reply(client.45671.0:25080 9.5 e122/119 ondisk, result = 0)", 2023-09-23T19:31:54.362 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:31:45.640116+0000", 2023-09-23T19:31:54.362 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 8.6023760360000008, 2023-09-23T19:31:54.362 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.13282833199999999, 2023-09-23T19:31:54.362 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:31:54.362 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "started", 2023-09-23T19:31:54.362 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:31:54.362 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.362 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:31:54.362 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.640116+0000", 2023-09-23T19:31:54.363 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:31:54.363 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.363 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.363 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:31:54.363 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.640116+0000", 2023-09-23T19:31:54.363 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:31:54.363 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.363 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.363 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:31:54.363 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.640115+0000", 2023-09-23T19:31:54.363 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:31:54.364 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.364 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.364 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:31:54.364 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.640121+0000", 2023-09-23T19:31:54.364 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.8080000000000001e-06 2023-09-23T19:31:54.364 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.364 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.364 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:31:54.364 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.767910+0000", 2023-09-23T19:31:54.364 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.12778891000000001 2023-09-23T19:31:54.364 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.364 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.365 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:31:54.365 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.767934+0000", 2023-09-23T19:31:54.365 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.4312e-05 2023-09-23T19:31:54.365 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.365 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.365 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:31:54.365 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.772913+0000", 2023-09-23T19:31:54.365 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0049787249999999998 2023-09-23T19:31:54.365 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.365 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.365 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:31:54.366 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.772928+0000", 2023-09-23T19:31:54.366 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.5781999999999999e-05 2023-09-23T19:31:54.366 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.366 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.366 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:31:54.366 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:45.772944+0000", 2023-09-23T19:31:54.366 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.5784e-05 2023-09-23T19:31:54.366 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:31:54.366 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:31:54.366 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:31:54.366 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.366 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.367 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45671.0:28039 9.b 9:d1d0567e:::benchmark_data_smithi016_113233_object28038:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e122)", 2023-09-23T19:31:54.367 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:31:47.622821+0000", 2023-09-23T19:31:54.367 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 6.6196713259999997, 2023-09-23T19:31:54.367 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.114018856, 2023-09-23T19:31:54.367 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:31:54.367 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:31:54.367 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:31:54.367 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45671", 2023-09-23T19:31:54.367 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:31:54.367 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 28039 2023-09-23T19:31:54.367 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.368 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:31:54.368 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.368 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:31:54.368 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:47.622821+0000", 2023-09-23T19:31:54.368 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:31:54.368 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.368 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.368 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:31:54.368 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:47.622821+0000", 2023-09-23T19:31:54.368 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:31:54.368 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.369 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.369 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:31:54.369 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:47.622819+0000", 2023-09-23T19:31:54.369 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-09-23T19:31:54.369 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.369 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.369 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:31:54.369 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:47.622843+0000", 2023-09-23T19:31:54.369 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.4051000000000001e-05 2023-09-23T19:31:54.369 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.369 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.369 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:31:54.370 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:47.622856+0000", 2023-09-23T19:31:54.370 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2891000000000001e-05 2023-09-23T19:31:54.370 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.370 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.370 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:31:54.370 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:47.622864+0000", 2023-09-23T19:31:54.370 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.9559999999999997e-06 2023-09-23T19:31:54.370 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.370 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.370 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:31:54.370 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:47.622912+0000", 2023-09-23T19:31:54.371 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.7599999999999998e-05 2023-09-23T19:31:54.371 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.371 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.371 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:31:54.371 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:47.623026+0000", 2023-09-23T19:31:54.371 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00011402 2023-09-23T19:31:54.371 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.373 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.373 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 5,6", 2023-09-23T19:31:54.373 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:47.727489+0000", 2023-09-23T19:31:54.373 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.104463191 2023-09-23T19:31:54.373 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.373 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.373 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:31:54.373 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:47.734711+0000", 2023-09-23T19:31:54.373 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0072223080000000002 2023-09-23T19:31:54.374 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.374 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.374 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:31:54.374 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:47.735523+0000", 2023-09-23T19:31:54.374 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00081181500000000002 2023-09-23T19:31:54.374 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.374 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.374 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:31:54.374 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:47.736749+0000", 2023-09-23T19:31:54.374 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00122576 2023-09-23T19:31:54.374 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.374 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.375 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:31:54.375 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:47.736800+0000", 2023-09-23T19:31:54.375 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.1399e-05 2023-09-23T19:31:54.375 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.375 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.375 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:31:54.375 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:47.736839+0000", 2023-09-23T19:31:54.375 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.9087e-05 2023-09-23T19:31:54.375 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:31:54.375 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:31:54.375 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:31:54.375 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.375 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.376 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45671.0:28040 9.b 9:d15510fa:::benchmark_data_smithi016_113233_object28039:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e122)", 2023-09-23T19:31:54.376 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:31:47.623887+0000", 2023-09-23T19:31:54.376 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 6.618604575, 2023-09-23T19:31:54.376 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.115488206, 2023-09-23T19:31:54.376 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:31:54.376 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:31:54.376 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:31:54.376 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45671", 2023-09-23T19:31:54.376 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:31:54.376 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 28040 2023-09-23T19:31:54.376 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.376 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:31:54.376 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.377 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:31:54.377 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:47.623887+0000", 2023-09-23T19:31:54.377 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:31:54.377 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.377 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.377 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:31:54.377 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:47.623887+0000", 2023-09-23T19:31:54.377 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:31:54.377 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.377 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.377 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:31:54.377 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:47.623886+0000", 2023-09-23T19:31:54.377 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:31:54.377 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.378 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.378 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:31:54.378 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:47.623913+0000", 2023-09-23T19:31:54.378 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.7178999999999998e-05 2023-09-23T19:31:54.378 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.378 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.378 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:31:54.378 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:47.727431+0000", 2023-09-23T19:31:54.378 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.103517462 2023-09-23T19:31:54.378 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.378 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.378 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:31:54.379 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:47.727466+0000", 2023-09-23T19:31:54.379 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.5534000000000002e-05 2023-09-23T19:31:54.379 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.379 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.379 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:31:54.379 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:47.729007+0000", 2023-09-23T19:31:54.379 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0015410619999999999 2023-09-23T19:31:54.379 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.379 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.379 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:31:54.379 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:47.729223+0000", 2023-09-23T19:31:54.379 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00021548199999999999 2023-09-23T19:31:54.380 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.380 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.380 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 5,6", 2023-09-23T19:31:54.380 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:47.729572+0000", 2023-09-23T19:31:54.380 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00034896599999999998 2023-09-23T19:31:54.380 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.380 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.380 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:31:54.380 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:47.735528+0000", 2023-09-23T19:31:54.380 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0059562290000000004 2023-09-23T19:31:54.380 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.381 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.381 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:31:54.381 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:47.736924+0000", 2023-09-23T19:31:54.381 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001396223 2023-09-23T19:31:54.381 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.381 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.381 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:31:54.381 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:47.739233+0000", 2023-09-23T19:31:54.381 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0023083779999999998 2023-09-23T19:31:54.381 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.381 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.381 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:31:54.382 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:47.739323+0000", 2023-09-23T19:31:54.382 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.0165000000000002e-05 2023-09-23T19:31:54.382 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.382 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.382 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:31:54.382 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:47.739376+0000", 2023-09-23T19:31:54.382 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.2686999999999999e-05 2023-09-23T19:31:54.382 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:31:54.382 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:31:54.382 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:31:54.382 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.382 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.383 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45671.0:29343 9.b 9:dea82875:::benchmark_data_smithi016_113233_object29342:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e122)", 2023-09-23T19:31:54.383 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:31:48.442222+0000", 2023-09-23T19:31:54.383 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 5.8002698949999996, 2023-09-23T19:31:54.383 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.11365974199999999, 2023-09-23T19:31:54.383 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:31:54.383 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:31:54.383 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:31:54.383 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45671", 2023-09-23T19:31:54.383 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:31:54.383 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 29343 2023-09-23T19:31:54.383 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.383 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:31:54.384 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.384 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:31:54.384 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:48.442222+0000", 2023-09-23T19:31:54.384 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:31:54.384 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.384 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.384 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:31:54.384 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:48.442222+0000", 2023-09-23T19:31:54.384 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:31:54.384 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.384 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.385 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:31:54.385 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:48.442220+0000", 2023-09-23T19:31:54.385 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-09-23T19:31:54.385 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.385 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.385 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:31:54.385 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:48.442281+0000", 2023-09-23T19:31:54.385 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.0556000000000001e-05 2023-09-23T19:31:54.385 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.385 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.385 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:31:54.385 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:48.543052+0000", 2023-09-23T19:31:54.385 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.100771028 2023-09-23T19:31:54.386 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.386 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.386 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:31:54.386 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:48.543070+0000", 2023-09-23T19:31:54.386 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.8559999999999998e-05 2023-09-23T19:31:54.386 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.386 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.386 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:31:54.386 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:48.543178+0000", 2023-09-23T19:31:54.386 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000107648 2023-09-23T19:31:54.386 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.387 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.387 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:31:54.387 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:48.543316+0000", 2023-09-23T19:31:54.387 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00013819900000000001 2023-09-23T19:31:54.387 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.387 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.387 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 5,6", 2023-09-23T19:31:54.387 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:48.543425+0000", 2023-09-23T19:31:54.387 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000108806 2023-09-23T19:31:54.387 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.387 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.387 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:31:54.388 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:48.548504+0000", 2023-09-23T19:31:54.388 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0050792199999999997 2023-09-23T19:31:54.388 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.388 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.388 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:31:54.388 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:48.553066+0000", 2023-09-23T19:31:54.388 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.004561379 2023-09-23T19:31:54.388 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.388 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.388 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:31:54.388 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:48.555604+0000", 2023-09-23T19:31:54.388 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0025378699999999998 2023-09-23T19:31:54.389 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.389 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.389 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:31:54.389 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:48.555835+0000", 2023-09-23T19:31:54.389 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000231034 2023-09-23T19:31:54.389 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:31:54.389 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:31:54.389 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:31:54.389 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:48.555882+0000", 2023-09-23T19:31:54.389 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.7256000000000003e-05 2023-09-23T19:31:54.389 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:31:54.389 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:31:54.390 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:31:54.390 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:31:54.390 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:31:54.390 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-09-23T19:31:54.390 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:54.391 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-09-23T19:31:54.289625+0000 min lat: 0.00139623 max lat: 0.199525 avg lat: 0.00827573 2023-09-23T19:31:54.391 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:31:54.391 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 20 16 38651 38635 120.722 96.0625 0.0114336 0.00827573 2023-09-23T19:31:54.418 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:54.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:54.422 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:54.523 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:54.550 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:54.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:54.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:54.592 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:31:54.652 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:31:54.656 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:54.683 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:54.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:54.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:54.722 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:31:54.762 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:31:54.788 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:54.808 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:31:54.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:54.819 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:54.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:54.844 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:31:54.879 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:31:54.901 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:31:54.904 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:31:54.904 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:31:54.904 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:31:54.905 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:31:54.906 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:31:54.906 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:31:54.906 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:31:54.906 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:31:54.921 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:54.952 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:54.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:54.954 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:55.055 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:55.086 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:55.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:55.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:55.193 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:55.226 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:55.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:55.228 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:55.291 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 21 16 39934 39918 118.792 80.1875 0.0108092 0.00841172 2023-09-23T19:31:55.328 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:55.352 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:55.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:55.354 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:55.384 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is unset 2023-09-23T19:31:55.388 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:55 smithi016 bash[75004]: cluster 2023-09-23T19:31:53.964082+0000 mgr.y (mgr.25325) 727 : cluster 0 pgmap v625: 89 pgs: 89 active+clean; 2.1 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2023-09-23T19:31:55.454 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:55.478 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:55.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:55.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:55.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:55 smithi092 bash[58855]: cluster 2023-09-23T19:31:53.964082+0000 mgr.y (mgr.25325) 727 : cluster 0 pgmap v625: 89 pgs: 89 active+clean; 2.1 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2023-09-23T19:31:55.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:55 smithi092 bash[58855]: audit 2023-09-23T19:31:54.537135+0000 mon.b (mon.1) 38 : audit 0 from='client.? 172.21.15.16:0/2175799111' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:31:55.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:55 smithi092 bash[58855]: audit 2023-09-23T19:31:55.334771+0000 mon.a (mon.0) 917 : audit 1 from='client.? 172.21.15.16:0/3207394809' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:31:55.580 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:55.615 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:55.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:55.621 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:31:55.641 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:55 smithi016 bash[77995]: cluster 2023-09-23T19:31:53.964082+0000 mgr.y (mgr.25325) 727 : cluster 0 pgmap v625: 89 pgs: 89 active+clean; 2.1 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2023-09-23T19:31:55.641 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:55 smithi016 bash[77995]: audit 2023-09-23T19:31:54.537135+0000 mon.b (mon.1) 38 : audit 0 from='client.? 172.21.15.16:0/2175799111' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:31:55.641 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:55 smithi016 bash[77995]: audit 2023-09-23T19:31:55.334771+0000 mon.a (mon.0) 917 : audit 1 from='client.? 172.21.15.16:0/3207394809' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:31:55.642 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:55 smithi016 bash[75004]: audit 2023-09-23T19:31:54.537135+0000 mon.b (mon.1) 38 : audit 0 from='client.? 172.21.15.16:0/2175799111' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:31:55.642 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:55 smithi016 bash[75004]: audit 2023-09-23T19:31:55.334771+0000 mon.a (mon.0) 917 : audit 1 from='client.? 172.21.15.16:0/3207394809' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:31:55.642 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:31:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:31:55] "GET /metrics HTTP/1.1" 200 36227 "" "Prometheus/2.33.4" 2023-09-23T19:31:55.722 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:55.730 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-09-23T19:31:55.734 INFO:teuthology.orchestra.run.smithi016.stderr:dumped all 2023-09-23T19:31:55.748 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:55.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:55.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:55.854 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:55.884 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:55.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:55.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:55.987 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:56.012 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:56.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:56.015 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:56.115 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:56.144 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:56.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:56.148 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:31:56.248 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:56.278 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:56.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:56.282 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:56.291 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 22 16 41397 41381 117.548 91.4375 0.0129022 0.00849983 2023-09-23T19:31:56.383 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:56.412 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:56.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:56.416 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:56.516 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:56.545 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:56.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:56.548 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:56.649 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:56.659 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:56 smithi092 bash[58855]: cluster 2023-09-23T19:31:55.370542+0000 mon.a (mon.0) 918 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:31:56.659 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:56 smithi092 bash[58855]: audit 2023-09-23T19:31:55.376734+0000 mon.a (mon.0) 919 : audit 1 from='client.? 172.21.15.16:0/3207394809' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:31:56.659 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:56 smithi092 bash[58855]: cluster 2023-09-23T19:31:55.377096+0000 mon.a (mon.0) 920 : cluster 0 osdmap e123: 8 total, 8 up, 8 in 2023-09-23T19:31:56.659 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:56 smithi092 bash[58855]: cluster 2023-09-23T19:31:55.378048+0000 osd.6 (osd.6) 42 : cluster 3 failed to encode map e123 with expected crc 2023-09-23T19:31:56.659 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:56 smithi092 bash[58855]: cluster 2023-09-23T19:31:55.381218+0000 osd.6 (osd.6) 43 : cluster 3 failed to encode map e123 with expected crc 2023-09-23T19:31:56.659 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:56 smithi092 bash[58855]: cluster 2023-09-23T19:31:55.382032+0000 osd.7 (osd.7) 41 : cluster 3 failed to encode map e123 with expected crc 2023-09-23T19:31:56.660 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:56 smithi092 bash[58855]: cluster 2023-09-23T19:31:55.386018+0000 osd.5 (osd.5) 49 : cluster 3 failed to encode map e123 with expected crc 2023-09-23T19:31:56.660 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:56 smithi092 bash[58855]: cluster 2023-09-23T19:31:55.386394+0000 osd.4 (osd.4) 50 : cluster 3 failed to encode map e123 with expected crc 2023-09-23T19:31:56.660 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:56 smithi092 bash[58855]: cluster 2023-09-23T19:31:55.389049+0000 osd.5 (osd.5) 50 : cluster 3 failed to encode map e123 with expected crc 2023-09-23T19:31:56.660 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:56 smithi092 bash[58855]: audit 2023-09-23T19:31:55.711385+0000 mgr.y (mgr.25325) 728 : audit 0 from='client.35775 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:31:56.660 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:56 smithi092 bash[58855]: audit 2023-09-23T19:31:56.066502+0000 mon.a (mon.0) 921 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:31:56.674 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:31:56.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:56.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:31:56.778 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:56.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:56 smithi016 bash[77995]: cluster 2023-09-23T19:31:55.370542+0000 mon.a (mon.0) 918 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:31:56.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:56 smithi016 bash[77995]: audit 2023-09-23T19:31:55.376734+0000 mon.a (mon.0) 919 : audit 1 from='client.? 172.21.15.16:0/3207394809' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:31:56.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:56 smithi016 bash[77995]: cluster 2023-09-23T19:31:55.377096+0000 mon.a (mon.0) 920 : cluster 0 osdmap e123: 8 total, 8 up, 8 in 2023-09-23T19:31:56.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:56 smithi016 bash[77995]: cluster 2023-09-23T19:31:55.378048+0000 osd.6 (osd.6) 42 : cluster 3 failed to encode map e123 with expected crc 2023-09-23T19:31:56.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:56 smithi016 bash[77995]: cluster 2023-09-23T19:31:55.381218+0000 osd.6 (osd.6) 43 : cluster 3 failed to encode map e123 with expected crc 2023-09-23T19:31:56.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:56 smithi016 bash[77995]: cluster 2023-09-23T19:31:55.382032+0000 osd.7 (osd.7) 41 : cluster 3 failed to encode map e123 with expected crc 2023-09-23T19:31:56.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:56 smithi016 bash[77995]: cluster 2023-09-23T19:31:55.386018+0000 osd.5 (osd.5) 49 : cluster 3 failed to encode map e123 with expected crc 2023-09-23T19:31:56.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:56 smithi016 bash[77995]: cluster 2023-09-23T19:31:55.386394+0000 osd.4 (osd.4) 50 : cluster 3 failed to encode map e123 with expected crc 2023-09-23T19:31:56.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:56 smithi016 bash[77995]: cluster 2023-09-23T19:31:55.389049+0000 osd.5 (osd.5) 50 : cluster 3 failed to encode map e123 with expected crc 2023-09-23T19:31:56.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:56 smithi016 bash[77995]: audit 2023-09-23T19:31:55.711385+0000 mgr.y (mgr.25325) 728 : audit 0 from='client.35775 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:31:56.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:56 smithi016 bash[77995]: audit 2023-09-23T19:31:56.066502+0000 mon.a (mon.0) 921 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:31:56.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:56 smithi016 bash[75004]: cluster 2023-09-23T19:31:55.370542+0000 mon.a (mon.0) 918 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:31:56.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:56 smithi016 bash[75004]: audit 2023-09-23T19:31:55.376734+0000 mon.a (mon.0) 919 : audit 1 from='client.? 172.21.15.16:0/3207394809' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:31:56.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:56 smithi016 bash[75004]: cluster 2023-09-23T19:31:55.377096+0000 mon.a (mon.0) 920 : cluster 0 osdmap e123: 8 total, 8 up, 8 in 2023-09-23T19:31:56.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:56 smithi016 bash[75004]: cluster 2023-09-23T19:31:55.378048+0000 osd.6 (osd.6) 42 : cluster 3 failed to encode map e123 with expected crc 2023-09-23T19:31:56.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:56 smithi016 bash[75004]: cluster 2023-09-23T19:31:55.381218+0000 osd.6 (osd.6) 43 : cluster 3 failed to encode map e123 with expected crc 2023-09-23T19:31:56.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:56 smithi016 bash[75004]: cluster 2023-09-23T19:31:55.382032+0000 osd.7 (osd.7) 41 : cluster 3 failed to encode map e123 with expected crc 2023-09-23T19:31:56.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:56 smithi016 bash[75004]: cluster 2023-09-23T19:31:55.386018+0000 osd.5 (osd.5) 49 : cluster 3 failed to encode map e123 with expected crc 2023-09-23T19:31:56.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:56 smithi016 bash[75004]: cluster 2023-09-23T19:31:55.386394+0000 osd.4 (osd.4) 50 : cluster 3 failed to encode map e123 with expected crc 2023-09-23T19:31:56.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:56 smithi016 bash[75004]: cluster 2023-09-23T19:31:55.389049+0000 osd.5 (osd.5) 50 : cluster 3 failed to encode map e123 with expected crc 2023-09-23T19:31:56.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:56 smithi016 bash[75004]: audit 2023-09-23T19:31:55.711385+0000 mgr.y (mgr.25325) 728 : audit 0 from='client.35775 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:31:56.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:56 smithi016 bash[75004]: audit 2023-09-23T19:31:56.066502+0000 mon.a (mon.0) 921 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:31:56.824 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:56.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:56.827 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:56.928 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:56.952 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:56.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:56.955 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:57.055 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:57.082 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:57.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:57.086 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:57.186 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:57.213 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:57.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:57.217 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:57.292 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 23 16 42755 42739 116.127 84.875 0.00310813 0.00858795 2023-09-23T19:31:57.317 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:57.340 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:57.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:57.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:57.442 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:57.452 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:57 smithi016 bash[75004]: cluster 2023-09-23T19:31:55.964616+0000 mgr.y (mgr.25325) 729 : cluster 0 pgmap v627: 89 pgs: 89 active+clean; 2.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-09-23T19:31:57.452 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:57 smithi016 bash[77995]: cluster 2023-09-23T19:31:55.964616+0000 mgr.y (mgr.25325) 729 : cluster 0 pgmap v627: 89 pgs: 89 active+clean; 2.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-09-23T19:31:57.475 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:57.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:57.480 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:57.581 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:57.600 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:57.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:57.602 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:57.703 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:57.713 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:57 smithi092 bash[58855]: cluster 2023-09-23T19:31:55.964616+0000 mgr.y (mgr.25325) 729 : cluster 0 pgmap v627: 89 pgs: 89 active+clean; 2.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-09-23T19:31:57.730 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:57.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:57.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:57.810 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T19:31:57.837 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:57.878 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:57.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:57.880 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:57.980 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:58.014 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:58.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:58.020 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:58.121 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:58.157 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:58.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:58.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:58.261 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:58.289 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:58.292 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 24 16 43766 43750 113.921 63.1875 0.00879759 0.00877115 2023-09-23T19:31:58.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:58.293 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:58.394 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:58.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:58.429 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:31:58.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:31:58.530 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:58.563 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:31:58.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:58.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:31:58.668 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:58.703 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:58.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:58.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:58.807 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:58.835 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:31:58.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:58.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:31:58.940 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:58.972 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:58.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:58.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:59.076 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:59.104 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:31:59.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:59.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:31:59.211 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:59.245 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:31:59.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:59.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:31:59.292 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 25 16 45208 45192 112.968 90.125 0.00802129 0.00884537 2023-09-23T19:31:59.351 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:59.381 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:59.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:59.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:59.404 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:59 smithi016 bash[77995]: cluster 2023-09-23T19:31:57.965228+0000 mgr.y (mgr.25325) 730 : cluster 0 pgmap v628: 89 pgs: 89 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2023-09-23T19:31:59.404 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:31:59 smithi016 bash[77995]: audit 2023-09-23T19:31:58.636810+0000 mon.a (mon.0) 922 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:31:59.486 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:59.520 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:31:59.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:59.522 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:31:59.623 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:59.661 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:59.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:59.665 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:59.665 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:59 smithi016 bash[75004]: cluster 2023-09-23T19:31:57.965228+0000 mgr.y (mgr.25325) 730 : cluster 0 pgmap v628: 89 pgs: 89 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2023-09-23T19:31:59.665 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:31:59 smithi016 bash[75004]: audit 2023-09-23T19:31:58.636810+0000 mon.a (mon.0) 922 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:31:59.766 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:59.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:59 smithi092 bash[58855]: cluster 2023-09-23T19:31:57.965228+0000 mgr.y (mgr.25325) 730 : cluster 0 pgmap v628: 89 pgs: 89 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2023-09-23T19:31:59.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:31:59 smithi092 bash[58855]: audit 2023-09-23T19:31:58.636810+0000 mon.a (mon.0) 922 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:31:59.883 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:31:59.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:59.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:31:59.911 INFO:teuthology.orchestra.run.smithi016.stdout:{"pg_ready":true,"pg_map":{"version":626,"stamp":"2023-09-23T19:31:55.372635+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2425952160,"num_objects":37032,"num_object_clones":0,"num_object_copies":111096,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":37032,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":37269,"num_write_kb":2370958,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":37203,"ondisk_log_size":37203,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":9898616,"kb_used_data":6966456,"kb_used_omap":20,"kb_used_meta":2932011,"kb_avail":740062600,"statfs":{"total":767960285184,"available":757824102400,"internally_reserved":0,"allocated":7133650944,"data_stored":7129752443,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":3002379918},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,5,4,8,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1324810240,"num_objects":20215,"num_object_clones":0,"num_object_copies":60645,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":20215,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":20215,"num_write_kb":1293760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"11.414080"},"pg_stats":[{"pgid":"3.1a","version":"59'1","reported_seq":104,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.728782+0000","last_change":"2023-09-23T19:27:24.274696+0000","last_active":"2023-09-23T19:31:13.728782+0000","last_peered":"2023-09-23T19:31:13.728782+0000","last_clean":"2023-09-23T19:31:13.728782+0000","last_became_active":"2023-09-23T19:27:24.273561+0000","last_became_peered":"2023-09-23T19:27:24.273561+0000","last_unstale":"2023-09-23T19:31:13.728782+0000","last_undegraded":"2023-09-23T19:31:13.728782+0000","last_fullsized":"2023-09-23T19:31:13.728782+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":83,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.723405+0000","last_change":"2023-09-23T19:28:07.894651+0000","last_active":"2023-09-23T19:31:13.723405+0000","last_peered":"2023-09-23T19:31:13.723405+0000","last_clean":"2023-09-23T19:31:13.723405+0000","last_became_active":"2023-09-23T19:28:07.892919+0000","last_became_peered":"2023-09-23T19:28:07.892919+0000","last_unstale":"2023-09-23T19:31:13.723405+0000","last_undegraded":"2023-09-23T19:31:13.723405+0000","last_fullsized":"2023-09-23T19:31:13.723405+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:56.579997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":63,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.720116+0000","last_change":"2023-09-23T19:28:07.890462+0000","last_active":"2023-09-23T19:31:13.720116+0000","last_peered":"2023-09-23T19:31:13.720116+0000","last_clean":"2023-09-23T19:31:13.720116+0000","last_became_active":"2023-09-23T19:28:07.890084+0000","last_became_peered":"2023-09-23T19:28:07.890084+0000","last_unstale":"2023-09-23T19:31:13.720116+0000","last_undegraded":"2023-09-23T19:31:13.720116+0000","last_fullsized":"2023-09-23T19:31:13.720116+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:50.458218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":79,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.567867+0000","last_change":"2023-09-23T19:28:07.884498+0000","last_active":"2023-09-23T19:31:13.567867+0000","last_peered":"2023-09-23T19:31:13.567867+0000","last_clean":"2023-09-23T19:31:13.567867+0000","last_became_active":"2023-09-23T19:28:07.884102+0000","last_became_peered":"2023-09-23T19:28:07.884102+0000","last_unstale":"2023-09-23T19:31:13.567867+0000","last_undegraded":"2023-09-23T19:31:13.567867+0000","last_fullsized":"2023-09-23T19:31:13.567867+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:05:26.348041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":99,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.721399+0000","last_change":"2023-09-23T19:28:07.887122+0000","last_active":"2023-09-23T19:31:13.721399+0000","last_peered":"2023-09-23T19:31:13.721399+0000","last_clean":"2023-09-23T19:31:13.721399+0000","last_became_active":"2023-09-23T19:28:07.886743+0000","last_became_peered":"2023-09-23T19:28:07.886743+0000","last_unstale":"2023-09-23T19:31:13.721399+0000","last_undegraded":"2023-09-23T19:31:13.721399+0000","last_fullsized":"2023-09-23T19:31:13.721399+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:59:45.973619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":87,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.729241+0000","last_change":"2023-09-23T19:22:42.150582+0000","last_active":"2023-09-23T19:31:13.729241+0000","last_peered":"2023-09-23T19:31:13.729241+0000","last_clean":"2023-09-23T19:31:13.729241+0000","last_became_active":"2023-09-23T19:22:42.150229+0000","last_became_peered":"2023-09-23T19:22:42.150229+0000","last_unstale":"2023-09-23T19:31:13.729241+0000","last_undegraded":"2023-09-23T19:31:13.729241+0000","last_fullsized":"2023-09-23T19:31:13.729241+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":78,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.568897+0000","last_change":"2023-09-23T19:28:07.887141+0000","last_active":"2023-09-23T19:31:13.568897+0000","last_peered":"2023-09-23T19:31:13.568897+0000","last_clean":"2023-09-23T19:31:13.568897+0000","last_became_active":"2023-09-23T19:28:07.886494+0000","last_became_peered":"2023-09-23T19:28:07.886494+0000","last_unstale":"2023-09-23T19:31:13.568897+0000","last_undegraded":"2023-09-23T19:31:13.568897+0000","last_fullsized":"2023-09-23T19:31:13.568897+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:24:07.615051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":83,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.719815+0000","last_change":"2023-09-23T19:28:07.890970+0000","last_active":"2023-09-23T19:31:13.719815+0000","last_peered":"2023-09-23T19:31:13.719815+0000","last_clean":"2023-09-23T19:31:13.719815+0000","last_became_active":"2023-09-23T19:28:07.890218+0000","last_became_peered":"2023-09-23T19:28:07.890218+0000","last_unstale":"2023-09-23T19:31:13.719815+0000","last_undegraded":"2023-09-23T19:31:13.719815+0000","last_fullsized":"2023-09-23T19:31:13.719815+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:02:28.214414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":89,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.728177+0000","last_change":"2023-09-23T19:22:42.151617+0000","last_active":"2023-09-23T19:31:13.728177+0000","last_peered":"2023-09-23T19:31:13.728177+0000","last_clean":"2023-09-23T19:31:13.728177+0000","last_became_active":"2023-09-23T19:22:42.151217+0000","last_became_peered":"2023-09-23T19:22:42.151217+0000","last_unstale":"2023-09-23T19:31:13.728177+0000","last_undegraded":"2023-09-23T19:31:13.728177+0000","last_fullsized":"2023-09-23T19:31:13.728177+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":69,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.722923+0000","last_change":"2023-09-23T19:28:07.889639+0000","last_active":"2023-09-23T19:31:13.722923+0000","last_peered":"2023-09-23T19:31:13.722923+0000","last_clean":"2023-09-23T19:31:13.722923+0000","last_became_active":"2023-09-23T19:28:07.889313+0000","last_became_peered":"2023-09-23T19:28:07.889313+0000","last_unstale":"2023-09-23T19:31:13.722923+0000","last_undegraded":"2023-09-23T19:31:13.722923+0000","last_fullsized":"2023-09-23T19:31:13.722923+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:54:03.826337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":91,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.580511+0000","last_change":"2023-09-23T19:28:07.892835+0000","last_active":"2023-09-23T19:31:13.580511+0000","last_peered":"2023-09-23T19:31:13.580511+0000","last_clean":"2023-09-23T19:31:13.580511+0000","last_became_active":"2023-09-23T19:28:07.892338+0000","last_became_peered":"2023-09-23T19:28:07.892338+0000","last_unstale":"2023-09-23T19:31:13.580511+0000","last_undegraded":"2023-09-23T19:31:13.580511+0000","last_fullsized":"2023-09-23T19:31:13.580511+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":63,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.721041+0000","last_change":"2023-09-23T19:27:24.268005+0000","last_active":"2023-09-23T19:31:13.721041+0000","last_peered":"2023-09-23T19:31:13.721041+0000","last_clean":"2023-09-23T19:31:13.721041+0000","last_became_active":"2023-09-23T19:27:24.267513+0000","last_became_peered":"2023-09-23T19:27:24.267513+0000","last_unstale":"2023-09-23T19:31:13.721041+0000","last_undegraded":"2023-09-23T19:31:13.721041+0000","last_fullsized":"2023-09-23T19:31:13.721041+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.961399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'23","reported_seq":96,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.729931+0000","last_change":"2023-09-23T19:05:52.107727+0000","last_active":"2023-09-23T19:31:13.729931+0000","last_peered":"2023-09-23T19:31:13.729931+0000","last_clean":"2023-09-23T19:31:13.729931+0000","last_became_active":"2023-09-23T19:05:52.107378+0000","last_became_peered":"2023-09-23T19:05:52.107378+0000","last_unstale":"2023-09-23T19:31:13.729931+0000","last_undegraded":"2023-09-23T19:31:13.729931+0000","last_fullsized":"2023-09-23T19:31:13.729931+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":87,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.728896+0000","last_change":"2023-09-23T19:22:42.148497+0000","last_active":"2023-09-23T19:31:13.728896+0000","last_peered":"2023-09-23T19:31:13.728896+0000","last_clean":"2023-09-23T19:31:13.728896+0000","last_became_active":"2023-09-23T19:22:42.148147+0000","last_became_peered":"2023-09-23T19:22:42.148147+0000","last_unstale":"2023-09-23T19:31:13.728896+0000","last_undegraded":"2023-09-23T19:31:13.728896+0000","last_fullsized":"2023-09-23T19:31:13.728896+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":90,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.580091+0000","last_change":"2023-09-23T19:27:24.273667+0000","last_active":"2023-09-23T19:31:13.580091+0000","last_peered":"2023-09-23T19:31:13.580091+0000","last_clean":"2023-09-23T19:31:13.580091+0000","last_became_active":"2023-09-23T19:27:24.273234+0000","last_became_peered":"2023-09-23T19:27:24.273234+0000","last_unstale":"2023-09-23T19:31:13.580091+0000","last_undegraded":"2023-09-23T19:31:13.580091+0000","last_fullsized":"2023-09-23T19:31:13.580091+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":83,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.721924+0000","last_change":"2023-09-23T19:28:07.886657+0000","last_active":"2023-09-23T19:31:13.721924+0000","last_peered":"2023-09-23T19:31:13.721924+0000","last_clean":"2023-09-23T19:31:13.721924+0000","last_became_active":"2023-09-23T19:28:07.886210+0000","last_became_peered":"2023-09-23T19:28:07.886210+0000","last_unstale":"2023-09-23T19:31:13.721924+0000","last_undegraded":"2023-09-23T19:31:13.721924+0000","last_fullsized":"2023-09-23T19:31:13.721924+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:21.131194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":78,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.567600+0000","last_change":"2023-09-23T19:28:07.884394+0000","last_active":"2023-09-23T19:31:13.567600+0000","last_peered":"2023-09-23T19:31:13.567600+0000","last_clean":"2023-09-23T19:31:13.567600+0000","last_became_active":"2023-09-23T19:28:07.883944+0000","last_became_peered":"2023-09-23T19:28:07.883944+0000","last_unstale":"2023-09-23T19:31:13.567600+0000","last_undegraded":"2023-09-23T19:31:13.567600+0000","last_fullsized":"2023-09-23T19:31:13.567600+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:08:17.608926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":88,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.728359+0000","last_change":"2023-09-23T19:28:07.891056+0000","last_active":"2023-09-23T19:31:13.728359+0000","last_peered":"2023-09-23T19:31:13.728359+0000","last_clean":"2023-09-23T19:31:13.728359+0000","last_became_active":"2023-09-23T19:28:07.890625+0000","last_became_peered":"2023-09-23T19:28:07.890625+0000","last_unstale":"2023-09-23T19:31:13.728359+0000","last_undegraded":"2023-09-23T19:31:13.728359+0000","last_fullsized":"2023-09-23T19:31:13.728359+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":73,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.580095+0000","last_change":"2023-09-23T19:05:52.114186+0000","last_active":"2023-09-23T19:31:13.580095+0000","last_peered":"2023-09-23T19:31:13.580095+0000","last_clean":"2023-09-23T19:31:13.580095+0000","last_became_active":"2023-09-23T19:05:52.113635+0000","last_became_peered":"2023-09-23T19:05:52.113635+0000","last_unstale":"2023-09-23T19:31:13.580095+0000","last_undegraded":"2023-09-23T19:31:13.580095+0000","last_fullsized":"2023-09-23T19:31:13.580095+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":78,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.568083+0000","last_change":"2023-09-23T19:28:07.884512+0000","last_active":"2023-09-23T19:31:13.568083+0000","last_peered":"2023-09-23T19:31:13.568083+0000","last_clean":"2023-09-23T19:31:13.568083+0000","last_became_active":"2023-09-23T19:28:07.884155+0000","last_became_peered":"2023-09-23T19:28:07.884155+0000","last_unstale":"2023-09-23T19:31:13.568083+0000","last_undegraded":"2023-09-23T19:31:13.568083+0000","last_fullsized":"2023-09-23T19:31:13.568083+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:10.654958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":96,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.727503+0000","last_change":"2023-09-23T19:18:09.010195+0000","last_active":"2023-09-23T19:31:13.727503+0000","last_peered":"2023-09-23T19:31:13.727503+0000","last_clean":"2023-09-23T19:31:13.727503+0000","last_became_active":"2023-09-23T19:18:09.009687+0000","last_became_peered":"2023-09-23T19:18:09.009687+0000","last_unstale":"2023-09-23T19:31:13.727503+0000","last_undegraded":"2023-09-23T19:31:13.727503+0000","last_fullsized":"2023-09-23T19:31:13.727503+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":83,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.723621+0000","last_change":"2023-09-23T19:28:07.885265+0000","last_active":"2023-09-23T19:31:13.723621+0000","last_peered":"2023-09-23T19:31:13.723621+0000","last_clean":"2023-09-23T19:31:13.723621+0000","last_became_active":"2023-09-23T19:28:07.884353+0000","last_became_peered":"2023-09-23T19:28:07.884353+0000","last_unstale":"2023-09-23T19:31:13.723621+0000","last_undegraded":"2023-09-23T19:31:13.723621+0000","last_fullsized":"2023-09-23T19:31:13.723621+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:56.325352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"122'2458","reported_seq":2470,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-09-23T19:31:53.864821+0000","last_change":"2023-09-23T19:31:12.572210+0000","last_active":"2023-09-23T19:31:53.864821+0000","last_peered":"2023-09-23T19:31:53.864821+0000","last_clean":"2023-09-23T19:31:53.864821+0000","last_became_active":"2023-09-23T19:31:12.571806+0000","last_became_peered":"2023-09-23T19:31:12.571806+0000","last_unstale":"2023-09-23T19:31:53.864821+0000","last_undegraded":"2023-09-23T19:31:53.864821+0000","last_fullsized":"2023-09-23T19:31:53.864821+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_clean_scrub_stamp":"2023-09-23T19:31:11.546138+0000","objects_scrubbed":0,"log_size":2458,"log_dups_size":0,"ondisk_log_size":2458,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:41:17.174555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161087488,"num_objects":2458,"num_object_clones":0,"num_object_copies":7374,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2458,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2458,"num_write_kb":157312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":89,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.580641+0000","last_change":"2023-09-23T19:18:09.035116+0000","last_active":"2023-09-23T19:31:13.580641+0000","last_peered":"2023-09-23T19:31:13.580641+0000","last_clean":"2023-09-23T19:31:13.580641+0000","last_became_active":"2023-09-23T19:18:09.034416+0000","last_became_peered":"2023-09-23T19:18:09.034416+0000","last_unstale":"2023-09-23T19:31:13.580641+0000","last_undegraded":"2023-09-23T19:31:13.580641+0000","last_fullsized":"2023-09-23T19:31:13.580641+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":73,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.727228+0000","last_change":"2023-09-23T19:05:52.789323+0000","last_active":"2023-09-23T19:31:13.727228+0000","last_peered":"2023-09-23T19:31:13.727228+0000","last_clean":"2023-09-23T19:31:13.727228+0000","last_became_active":"2023-09-23T19:05:52.788820+0000","last_became_peered":"2023-09-23T19:05:52.788820+0000","last_unstale":"2023-09-23T19:31:13.727228+0000","last_undegraded":"2023-09-23T19:31:13.727228+0000","last_fullsized":"2023-09-23T19:31:13.727228+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"122'2176","reported_seq":2188,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-09-23T19:31:51.088194+0000","last_change":"2023-09-23T19:31:12.589012+0000","last_active":"2023-09-23T19:31:51.088194+0000","last_peered":"2023-09-23T19:31:51.088194+0000","last_clean":"2023-09-23T19:31:51.088194+0000","last_became_active":"2023-09-23T19:31:12.588473+0000","last_became_peered":"2023-09-23T19:31:12.588473+0000","last_unstale":"2023-09-23T19:31:51.088194+0000","last_undegraded":"2023-09-23T19:31:51.088194+0000","last_fullsized":"2023-09-23T19:31:51.088194+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_clean_scrub_stamp":"2023-09-23T19:31:11.546138+0000","objects_scrubbed":0,"log_size":2176,"log_dups_size":0,"ondisk_log_size":2176,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:47:07.601624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142606336,"num_objects":2176,"num_object_clones":0,"num_object_copies":6528,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2176,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2176,"num_write_kb":139264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":90,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.580504+0000","last_change":"2023-09-23T19:22:42.151134+0000","last_active":"2023-09-23T19:31:13.580504+0000","last_peered":"2023-09-23T19:31:13.580504+0000","last_clean":"2023-09-23T19:31:13.580504+0000","last_became_active":"2023-09-23T19:22:42.150553+0000","last_became_peered":"2023-09-23T19:22:42.150553+0000","last_unstale":"2023-09-23T19:31:13.580504+0000","last_undegraded":"2023-09-23T19:31:13.580504+0000","last_fullsized":"2023-09-23T19:31:13.580504+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":102,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.729464+0000","last_change":"2023-09-23T19:28:07.899262+0000","last_active":"2023-09-23T19:31:13.729464+0000","last_peered":"2023-09-23T19:31:13.729464+0000","last_clean":"2023-09-23T19:31:13.729464+0000","last_became_active":"2023-09-23T19:28:07.898848+0000","last_became_peered":"2023-09-23T19:28:07.898848+0000","last_unstale":"2023-09-23T19:31:13.729464+0000","last_undegraded":"2023-09-23T19:31:13.729464+0000","last_fullsized":"2023-09-23T19:31:13.729464+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"122'2428","reported_seq":2439,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-09-23T19:31:54.994326+0000","last_change":"2023-09-23T19:31:12.576224+0000","last_active":"2023-09-23T19:31:54.994326+0000","last_peered":"2023-09-23T19:31:54.994326+0000","last_clean":"2023-09-23T19:31:54.994326+0000","last_became_active":"2023-09-23T19:31:12.575807+0000","last_became_peered":"2023-09-23T19:31:12.575807+0000","last_unstale":"2023-09-23T19:31:54.994326+0000","last_undegraded":"2023-09-23T19:31:54.994326+0000","last_fullsized":"2023-09-23T19:31:54.994326+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_clean_scrub_stamp":"2023-09-23T19:31:11.546138+0000","objects_scrubbed":0,"log_size":2428,"log_dups_size":0,"ondisk_log_size":2428,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159121408,"num_objects":2428,"num_object_clones":0,"num_object_copies":7284,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2428,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2428,"num_write_kb":155392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":74,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.580269+0000","last_change":"2023-09-23T19:05:52.114108+0000","last_active":"2023-09-23T19:31:13.580269+0000","last_peered":"2023-09-23T19:31:13.580269+0000","last_clean":"2023-09-23T19:31:13.580269+0000","last_became_active":"2023-09-23T19:05:52.113609+0000","last_became_peered":"2023-09-23T19:05:52.113609+0000","last_unstale":"2023-09-23T19:31:13.580269+0000","last_undegraded":"2023-09-23T19:31:13.580269+0000","last_fullsized":"2023-09-23T19:31:13.580269+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":87,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.729053+0000","last_change":"2023-09-23T19:22:42.156825+0000","last_active":"2023-09-23T19:31:13.729053+0000","last_peered":"2023-09-23T19:31:13.729053+0000","last_clean":"2023-09-23T19:31:13.729053+0000","last_became_active":"2023-09-23T19:22:42.156474+0000","last_became_peered":"2023-09-23T19:22:42.156474+0000","last_unstale":"2023-09-23T19:31:13.729053+0000","last_undegraded":"2023-09-23T19:31:13.729053+0000","last_fullsized":"2023-09-23T19:31:13.729053+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"122'2357","reported_seq":2368,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-09-23T19:31:54.989856+0000","last_change":"2023-09-23T19:31:12.589020+0000","last_active":"2023-09-23T19:31:54.989856+0000","last_peered":"2023-09-23T19:31:54.989856+0000","last_clean":"2023-09-23T19:31:54.989856+0000","last_became_active":"2023-09-23T19:31:12.588673+0000","last_became_peered":"2023-09-23T19:31:12.588673+0000","last_unstale":"2023-09-23T19:31:54.989856+0000","last_undegraded":"2023-09-23T19:31:54.989856+0000","last_fullsized":"2023-09-23T19:31:54.989856+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_clean_scrub_stamp":"2023-09-23T19:31:11.546138+0000","objects_scrubbed":0,"log_size":2357,"log_dups_size":0,"ondisk_log_size":2357,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154468352,"num_objects":2357,"num_object_clones":0,"num_object_copies":7071,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2357,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2357,"num_write_kb":150848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":89,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.727952+0000","last_change":"2023-09-23T19:28:07.891536+0000","last_active":"2023-09-23T19:31:13.727952+0000","last_peered":"2023-09-23T19:31:13.727952+0000","last_clean":"2023-09-23T19:31:13.727952+0000","last_became_active":"2023-09-23T19:28:07.891153+0000","last_became_peered":"2023-09-23T19:28:07.891153+0000","last_unstale":"2023-09-23T19:31:13.727952+0000","last_undegraded":"2023-09-23T19:31:13.727952+0000","last_fullsized":"2023-09-23T19:31:13.727952+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":79,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-09-23T19:31:32.617577+0000","last_change":"2023-09-23T19:22:42.135572+0000","last_active":"2023-09-23T19:31:32.617577+0000","last_peered":"2023-09-23T19:31:32.617577+0000","last_clean":"2023-09-23T19:31:32.617577+0000","last_became_active":"2023-09-23T19:22:42.135237+0000","last_became_peered":"2023-09-23T19:22:42.135237+0000","last_unstale":"2023-09-23T19:31:32.617577+0000","last_undegraded":"2023-09-23T19:31:32.617577+0000","last_fullsized":"2023-09-23T19:31:32.617577+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:35.838718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"122'2110","reported_seq":2122,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-09-23T19:31:50.905354+0000","last_change":"2023-09-23T19:31:12.590766+0000","last_active":"2023-09-23T19:31:50.905354+0000","last_peered":"2023-09-23T19:31:50.905354+0000","last_clean":"2023-09-23T19:31:50.905354+0000","last_became_active":"2023-09-23T19:31:12.590458+0000","last_became_peered":"2023-09-23T19:31:12.590458+0000","last_unstale":"2023-09-23T19:31:50.905354+0000","last_undegraded":"2023-09-23T19:31:50.905354+0000","last_fullsized":"2023-09-23T19:31:50.905354+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_clean_scrub_stamp":"2023-09-23T19:31:11.546138+0000","objects_scrubbed":0,"log_size":2110,"log_dups_size":0,"ondisk_log_size":2110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:09:10.802905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138280960,"num_objects":2110,"num_object_clones":0,"num_object_copies":6330,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2110,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2110,"num_write_kb":135040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":73,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.720809+0000","last_change":"2023-09-23T19:28:07.893901+0000","last_active":"2023-09-23T19:31:13.720809+0000","last_peered":"2023-09-23T19:31:13.720809+0000","last_clean":"2023-09-23T19:31:13.720809+0000","last_became_active":"2023-09-23T19:28:07.892973+0000","last_became_peered":"2023-09-23T19:28:07.892973+0000","last_unstale":"2023-09-23T19:31:13.720809+0000","last_undegraded":"2023-09-23T19:31:13.720809+0000","last_fullsized":"2023-09-23T19:31:13.720809+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:27.445855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":87,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.730031+0000","last_change":"2023-09-23T19:28:07.896803+0000","last_active":"2023-09-23T19:31:13.730031+0000","last_peered":"2023-09-23T19:31:13.730031+0000","last_clean":"2023-09-23T19:31:13.730031+0000","last_became_active":"2023-09-23T19:28:07.896480+0000","last_became_peered":"2023-09-23T19:28:07.896480+0000","last_unstale":"2023-09-23T19:31:13.730031+0000","last_undegraded":"2023-09-23T19:31:13.730031+0000","last_fullsized":"2023-09-23T19:31:13.730031+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"122'2089","reported_seq":2101,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-09-23T19:31:51.101605+0000","last_change":"2023-09-23T19:31:12.591859+0000","last_active":"2023-09-23T19:31:51.101605+0000","last_peered":"2023-09-23T19:31:51.101605+0000","last_clean":"2023-09-23T19:31:51.101605+0000","last_became_active":"2023-09-23T19:31:12.591505+0000","last_became_peered":"2023-09-23T19:31:12.591505+0000","last_unstale":"2023-09-23T19:31:51.101605+0000","last_undegraded":"2023-09-23T19:31:51.101605+0000","last_fullsized":"2023-09-23T19:31:51.101605+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_clean_scrub_stamp":"2023-09-23T19:31:11.546138+0000","objects_scrubbed":0,"log_size":2089,"log_dups_size":0,"ondisk_log_size":2089,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:33:29.412181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136904704,"num_objects":2089,"num_object_clones":0,"num_object_copies":6267,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2089,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2089,"num_write_kb":133696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":88,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.729409+0000","last_change":"2023-09-23T19:22:42.156582+0000","last_active":"2023-09-23T19:31:13.729409+0000","last_peered":"2023-09-23T19:31:13.729409+0000","last_clean":"2023-09-23T19:31:13.729409+0000","last_became_active":"2023-09-23T19:22:42.156251+0000","last_became_peered":"2023-09-23T19:22:42.156251+0000","last_unstale":"2023-09-23T19:31:13.729409+0000","last_undegraded":"2023-09-23T19:31:13.729409+0000","last_fullsized":"2023-09-23T19:31:13.729409+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":102,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.729933+0000","last_change":"2023-09-23T19:27:24.273971+0000","last_active":"2023-09-23T19:31:13.729933+0000","last_peered":"2023-09-23T19:31:13.729933+0000","last_clean":"2023-09-23T19:31:13.729933+0000","last_became_active":"2023-09-23T19:27:24.273491+0000","last_became_peered":"2023-09-23T19:27:24.273491+0000","last_unstale":"2023-09-23T19:31:13.729933+0000","last_undegraded":"2023-09-23T19:31:13.729933+0000","last_fullsized":"2023-09-23T19:31:13.729933+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"122'2447","reported_seq":2458,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-09-23T19:31:54.858288+0000","last_change":"2023-09-23T19:31:12.573917+0000","last_active":"2023-09-23T19:31:54.858288+0000","last_peered":"2023-09-23T19:31:54.858288+0000","last_clean":"2023-09-23T19:31:54.858288+0000","last_became_active":"2023-09-23T19:31:12.573499+0000","last_became_peered":"2023-09-23T19:31:12.573499+0000","last_unstale":"2023-09-23T19:31:54.858288+0000","last_undegraded":"2023-09-23T19:31:54.858288+0000","last_fullsized":"2023-09-23T19:31:54.858288+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_clean_scrub_stamp":"2023-09-23T19:31:11.546138+0000","objects_scrubbed":0,"log_size":2447,"log_dups_size":0,"ondisk_log_size":2447,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160366592,"num_objects":2447,"num_object_clones":0,"num_object_copies":7341,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2447,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2447,"num_write_kb":156608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":88,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.729065+0000","last_change":"2023-09-23T19:27:24.274698+0000","last_active":"2023-09-23T19:31:13.729065+0000","last_peered":"2023-09-23T19:31:13.729065+0000","last_clean":"2023-09-23T19:31:13.729065+0000","last_became_active":"2023-09-23T19:27:24.273264+0000","last_became_peered":"2023-09-23T19:27:24.273264+0000","last_unstale":"2023-09-23T19:31:13.729065+0000","last_undegraded":"2023-09-23T19:31:13.729065+0000","last_fullsized":"2023-09-23T19:31:13.729065+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":87,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.730444+0000","last_change":"2023-09-23T19:22:42.149670+0000","last_active":"2023-09-23T19:31:13.730444+0000","last_peered":"2023-09-23T19:31:13.730444+0000","last_clean":"2023-09-23T19:31:13.730444+0000","last_became_active":"2023-09-23T19:22:42.149022+0000","last_became_peered":"2023-09-23T19:22:42.149022+0000","last_unstale":"2023-09-23T19:31:13.730444+0000","last_undegraded":"2023-09-23T19:31:13.730444+0000","last_fullsized":"2023-09-23T19:31:13.730444+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"122'2060","reported_seq":2072,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-09-23T19:31:50.893975+0000","last_change":"2023-09-23T19:31:12.585061+0000","last_active":"2023-09-23T19:31:50.893975+0000","last_peered":"2023-09-23T19:31:50.893975+0000","last_clean":"2023-09-23T19:31:50.893975+0000","last_became_active":"2023-09-23T19:31:12.584706+0000","last_became_peered":"2023-09-23T19:31:12.584706+0000","last_unstale":"2023-09-23T19:31:50.893975+0000","last_undegraded":"2023-09-23T19:31:50.893975+0000","last_fullsized":"2023-09-23T19:31:50.893975+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_clean_scrub_stamp":"2023-09-23T19:31:11.546138+0000","objects_scrubbed":0,"log_size":2060,"log_dups_size":0,"ondisk_log_size":2060,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:10:17.200327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135004160,"num_objects":2060,"num_object_clones":0,"num_object_copies":6180,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2060,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2060,"num_write_kb":131840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":84,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.721862+0000","last_change":"2023-09-23T19:28:07.886933+0000","last_active":"2023-09-23T19:31:13.721862+0000","last_peered":"2023-09-23T19:31:13.721862+0000","last_clean":"2023-09-23T19:31:13.721862+0000","last_became_active":"2023-09-23T19:28:07.886572+0000","last_became_peered":"2023-09-23T19:28:07.886572+0000","last_unstale":"2023-09-23T19:31:13.721862+0000","last_undegraded":"2023-09-23T19:31:13.721862+0000","last_fullsized":"2023-09-23T19:31:13.721862+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:41.332875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":64,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-09-23T19:31:31.308397+0000","last_change":"2023-09-23T19:28:07.889095+0000","last_active":"2023-09-23T19:31:31.308397+0000","last_peered":"2023-09-23T19:31:31.308397+0000","last_clean":"2023-09-23T19:31:31.308397+0000","last_became_active":"2023-09-23T19:28:07.888742+0000","last_became_peered":"2023-09-23T19:28:07.888742+0000","last_unstale":"2023-09-23T19:31:31.308397+0000","last_undegraded":"2023-09-23T19:31:31.308397+0000","last_fullsized":"2023-09-23T19:31:31.308397+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:43.929488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"122'2471","reported_seq":2481,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-09-23T19:31:54.993633+0000","last_change":"2023-09-23T19:31:12.577456+0000","last_active":"2023-09-23T19:31:54.993633+0000","last_peered":"2023-09-23T19:31:54.993633+0000","last_clean":"2023-09-23T19:31:54.993633+0000","last_became_active":"2023-09-23T19:31:12.576998+0000","last_became_peered":"2023-09-23T19:31:12.576998+0000","last_unstale":"2023-09-23T19:31:54.993633+0000","last_undegraded":"2023-09-23T19:31:54.993633+0000","last_fullsized":"2023-09-23T19:31:54.993633+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_clean_scrub_stamp":"2023-09-23T19:31:11.546138+0000","objects_scrubbed":0,"log_size":2471,"log_dups_size":0,"ondisk_log_size":2471,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161939456,"num_objects":2471,"num_object_clones":0,"num_object_copies":7413,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2471,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2471,"num_write_kb":158144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":97,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.720999+0000","last_change":"2023-09-23T19:22:42.143289+0000","last_active":"2023-09-23T19:31:13.720999+0000","last_peered":"2023-09-23T19:31:13.720999+0000","last_clean":"2023-09-23T19:31:13.720999+0000","last_became_active":"2023-09-23T19:22:42.142656+0000","last_became_peered":"2023-09-23T19:22:42.142656+0000","last_unstale":"2023-09-23T19:31:13.720999+0000","last_undegraded":"2023-09-23T19:31:13.720999+0000","last_fullsized":"2023-09-23T19:31:13.720999+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:38.467441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":104,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.728417+0000","last_change":"2023-09-23T19:28:07.890623+0000","last_active":"2023-09-23T19:31:13.728417+0000","last_peered":"2023-09-23T19:31:13.728417+0000","last_clean":"2023-09-23T19:31:13.728417+0000","last_became_active":"2023-09-23T19:28:07.890148+0000","last_became_peered":"2023-09-23T19:28:07.890148+0000","last_unstale":"2023-09-23T19:31:13.728417+0000","last_undegraded":"2023-09-23T19:31:13.728417+0000","last_fullsized":"2023-09-23T19:31:13.728417+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":76,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-09-23T19:31:31.515171+0000","last_change":"2023-09-23T19:27:24.269080+0000","last_active":"2023-09-23T19:31:31.515171+0000","last_peered":"2023-09-23T19:31:31.515171+0000","last_clean":"2023-09-23T19:31:31.515171+0000","last_became_active":"2023-09-23T19:27:24.268748+0000","last_became_peered":"2023-09-23T19:27:24.268748+0000","last_unstale":"2023-09-23T19:31:31.515171+0000","last_undegraded":"2023-09-23T19:31:31.515171+0000","last_fullsized":"2023-09-23T19:31:31.515171+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:49.014902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.9","version":"122'2382","reported_seq":2394,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-09-23T19:31:53.869595+0000","last_change":"2023-09-23T19:31:12.572956+0000","last_active":"2023-09-23T19:31:53.869595+0000","last_peered":"2023-09-23T19:31:53.869595+0000","last_clean":"2023-09-23T19:31:53.869595+0000","last_became_active":"2023-09-23T19:31:12.572413+0000","last_became_peered":"2023-09-23T19:31:12.572413+0000","last_unstale":"2023-09-23T19:31:53.869595+0000","last_undegraded":"2023-09-23T19:31:53.869595+0000","last_fullsized":"2023-09-23T19:31:53.869595+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_clean_scrub_stamp":"2023-09-23T19:31:11.546138+0000","objects_scrubbed":0,"log_size":2382,"log_dups_size":0,"ondisk_log_size":2382,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:46:33.130893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156106752,"num_objects":2382,"num_object_clones":0,"num_object_copies":7146,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2382,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2382,"num_write_kb":152448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":93,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.728172+0000","last_change":"2023-09-23T19:22:42.153647+0000","last_active":"2023-09-23T19:31:13.728172+0000","last_peered":"2023-09-23T19:31:13.728172+0000","last_clean":"2023-09-23T19:31:13.728172+0000","last_became_active":"2023-09-23T19:22:42.153317+0000","last_became_peered":"2023-09-23T19:22:42.153317+0000","last_unstale":"2023-09-23T19:31:13.728172+0000","last_undegraded":"2023-09-23T19:31:13.728172+0000","last_fullsized":"2023-09-23T19:31:13.728172+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":88,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.728738+0000","last_change":"2023-09-23T19:18:09.017525+0000","last_active":"2023-09-23T19:31:13.728738+0000","last_peered":"2023-09-23T19:31:13.728738+0000","last_clean":"2023-09-23T19:31:13.728738+0000","last_became_active":"2023-09-23T19:18:09.017193+0000","last_became_peered":"2023-09-23T19:18:09.017193+0000","last_unstale":"2023-09-23T19:31:13.728738+0000","last_undegraded":"2023-09-23T19:31:13.728738+0000","last_fullsized":"2023-09-23T19:31:13.728738+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":79,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.720820+0000","last_change":"2023-09-23T19:28:07.875796+0000","last_active":"2023-09-23T19:31:13.720820+0000","last_peered":"2023-09-23T19:31:13.720820+0000","last_clean":"2023-09-23T19:31:13.720820+0000","last_became_active":"2023-09-23T19:28:07.875310+0000","last_became_peered":"2023-09-23T19:28:07.875310+0000","last_unstale":"2023-09-23T19:31:13.720820+0000","last_undegraded":"2023-09-23T19:31:13.720820+0000","last_fullsized":"2023-09-23T19:31:13.720820+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:44:13.379633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"122'2542","reported_seq":2552,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-09-23T19:31:54.994808+0000","last_change":"2023-09-23T19:31:12.576188+0000","last_active":"2023-09-23T19:31:54.994808+0000","last_peered":"2023-09-23T19:31:54.994808+0000","last_clean":"2023-09-23T19:31:54.994808+0000","last_became_active":"2023-09-23T19:31:12.575832+0000","last_became_peered":"2023-09-23T19:31:12.575832+0000","last_unstale":"2023-09-23T19:31:54.994808+0000","last_undegraded":"2023-09-23T19:31:54.994808+0000","last_fullsized":"2023-09-23T19:31:54.994808+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_clean_scrub_stamp":"2023-09-23T19:31:11.546138+0000","objects_scrubbed":0,"log_size":2542,"log_dups_size":0,"ondisk_log_size":2542,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166592512,"num_objects":2542,"num_object_clones":0,"num_object_copies":7626,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2542,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2542,"num_write_kb":162688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":96,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-09-23T19:31:31.515174+0000","last_change":"2023-09-23T19:28:07.878688+0000","last_active":"2023-09-23T19:31:31.515174+0000","last_peered":"2023-09-23T19:31:31.515174+0000","last_clean":"2023-09-23T19:31:31.515174+0000","last_became_active":"2023-09-23T19:28:07.878309+0000","last_became_peered":"2023-09-23T19:28:07.878309+0000","last_unstale":"2023-09-23T19:31:31.515174+0000","last_undegraded":"2023-09-23T19:31:31.515174+0000","last_fullsized":"2023-09-23T19:31:31.515174+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:49:06.202657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":64,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.719750+0000","last_change":"2023-09-23T19:27:24.267897+0000","last_active":"2023-09-23T19:31:13.719750+0000","last_peered":"2023-09-23T19:31:13.719750+0000","last_clean":"2023-09-23T19:31:13.719750+0000","last_became_active":"2023-09-23T19:27:24.267535+0000","last_became_peered":"2023-09-23T19:27:24.267535+0000","last_unstale":"2023-09-23T19:31:13.719750+0000","last_undegraded":"2023-09-23T19:31:13.719750+0000","last_fullsized":"2023-09-23T19:31:13.719750+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:13:57.447808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":92,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:21.094142+0000","last_change":"2023-09-23T19:22:42.143733+0000","last_active":"2023-09-23T19:31:21.094142+0000","last_peered":"2023-09-23T19:31:21.094142+0000","last_clean":"2023-09-23T19:31:21.094142+0000","last_became_active":"2023-09-23T19:22:42.143077+0000","last_became_peered":"2023-09-23T19:22:42.143077+0000","last_unstale":"2023-09-23T19:31:21.094142+0000","last_undegraded":"2023-09-23T19:31:21.094142+0000","last_fullsized":"2023-09-23T19:31:21.094142+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:47:33.086994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"122'2060","reported_seq":2072,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-09-23T19:31:50.855915+0000","last_change":"2023-09-23T19:31:12.570337+0000","last_active":"2023-09-23T19:31:50.855915+0000","last_peered":"2023-09-23T19:31:50.855915+0000","last_clean":"2023-09-23T19:31:50.855915+0000","last_became_active":"2023-09-23T19:31:12.569856+0000","last_became_peered":"2023-09-23T19:31:12.569856+0000","last_unstale":"2023-09-23T19:31:50.855915+0000","last_undegraded":"2023-09-23T19:31:50.855915+0000","last_fullsized":"2023-09-23T19:31:50.855915+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_clean_scrub_stamp":"2023-09-23T19:31:11.546138+0000","objects_scrubbed":0,"log_size":2060,"log_dups_size":0,"ondisk_log_size":2060,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:14:25.250630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135004160,"num_objects":2060,"num_object_clones":0,"num_object_copies":6180,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2060,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2060,"num_write_kb":131840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":93,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.727762+0000","last_change":"2023-09-23T19:27:24.271520+0000","last_active":"2023-09-23T19:31:13.727762+0000","last_peered":"2023-09-23T19:31:13.727762+0000","last_clean":"2023-09-23T19:31:13.727762+0000","last_became_active":"2023-09-23T19:27:24.271165+0000","last_became_peered":"2023-09-23T19:27:24.271165+0000","last_unstale":"2023-09-23T19:31:13.727762+0000","last_undegraded":"2023-09-23T19:31:13.727762+0000","last_fullsized":"2023-09-23T19:31:13.727762+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":324,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.579727+0000","last_change":"2023-09-23T19:18:09.016297+0000","last_active":"2023-09-23T19:31:13.579727+0000","last_peered":"2023-09-23T19:31:13.579727+0000","last_clean":"2023-09-23T19:31:13.579727+0000","last_became_active":"2023-09-23T19:18:09.015436+0000","last_became_peered":"2023-09-23T19:18:09.015436+0000","last_unstale":"2023-09-23T19:31:13.579727+0000","last_undegraded":"2023-09-23T19:31:13.579727+0000","last_fullsized":"2023-09-23T19:31:13.579727+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:59:33.840751+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:59:33.840751+0000","last_clean_scrub_stamp":"2023-09-23T18:59:33.840751+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":69,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.721332+0000","last_change":"2023-09-23T19:28:07.889756+0000","last_active":"2023-09-23T19:31:13.721332+0000","last_peered":"2023-09-23T19:31:13.721332+0000","last_clean":"2023-09-23T19:31:13.721332+0000","last_became_active":"2023-09-23T19:28:07.889050+0000","last_became_peered":"2023-09-23T19:28:07.889050+0000","last_unstale":"2023-09-23T19:31:13.721332+0000","last_undegraded":"2023-09-23T19:31:13.721332+0000","last_fullsized":"2023-09-23T19:31:13.721332+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:04:55.924553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":72,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.728938+0000","last_change":"2023-09-23T19:05:52.786771+0000","last_active":"2023-09-23T19:31:13.728938+0000","last_peered":"2023-09-23T19:31:13.728938+0000","last_clean":"2023-09-23T19:31:13.728938+0000","last_became_active":"2023-09-23T19:05:52.785797+0000","last_became_peered":"2023-09-23T19:05:52.785797+0000","last_unstale":"2023-09-23T19:31:13.728938+0000","last_undegraded":"2023-09-23T19:31:13.728938+0000","last_fullsized":"2023-09-23T19:31:13.728938+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"122'2326","reported_seq":2338,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-09-23T19:31:53.877060+0000","last_change":"2023-09-23T19:31:12.591240+0000","last_active":"2023-09-23T19:31:53.877060+0000","last_peered":"2023-09-23T19:31:53.877060+0000","last_clean":"2023-09-23T19:31:53.877060+0000","last_became_active":"2023-09-23T19:31:12.590654+0000","last_became_peered":"2023-09-23T19:31:12.590654+0000","last_unstale":"2023-09-23T19:31:53.877060+0000","last_undegraded":"2023-09-23T19:31:53.877060+0000","last_fullsized":"2023-09-23T19:31:53.877060+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_clean_scrub_stamp":"2023-09-23T19:31:11.546138+0000","objects_scrubbed":0,"log_size":2326,"log_dups_size":0,"ondisk_log_size":2326,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:03.681405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152436736,"num_objects":2326,"num_object_clones":0,"num_object_copies":6978,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2326,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2326,"num_write_kb":148864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":107,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.579858+0000","last_change":"2023-09-23T19:22:42.150636+0000","last_active":"2023-09-23T19:31:13.579858+0000","last_peered":"2023-09-23T19:31:13.579858+0000","last_clean":"2023-09-23T19:31:13.579858+0000","last_became_active":"2023-09-23T19:22:42.150045+0000","last_became_peered":"2023-09-23T19:22:42.150045+0000","last_unstale":"2023-09-23T19:31:13.579858+0000","last_undegraded":"2023-09-23T19:31:13.579858+0000","last_fullsized":"2023-09-23T19:31:13.579858+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":79,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.567275+0000","last_change":"2023-09-23T19:28:07.889194+0000","last_active":"2023-09-23T19:31:13.567275+0000","last_peered":"2023-09-23T19:31:13.567275+0000","last_clean":"2023-09-23T19:31:13.567275+0000","last_became_active":"2023-09-23T19:28:07.888823+0000","last_became_peered":"2023-09-23T19:28:07.888823+0000","last_unstale":"2023-09-23T19:31:13.567275+0000","last_undegraded":"2023-09-23T19:31:13.567275+0000","last_fullsized":"2023-09-23T19:31:13.567275+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:23.464688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":94,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.567983+0000","last_change":"2023-09-23T19:27:24.256410+0000","last_active":"2023-09-23T19:31:13.567983+0000","last_peered":"2023-09-23T19:31:13.567983+0000","last_clean":"2023-09-23T19:31:13.567983+0000","last_became_active":"2023-09-23T19:27:24.256052+0000","last_became_peered":"2023-09-23T19:27:24.256052+0000","last_unstale":"2023-09-23T19:31:13.567983+0000","last_undegraded":"2023-09-23T19:31:13.567983+0000","last_fullsized":"2023-09-23T19:31:13.567983+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:11.951400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"122'2481","reported_seq":2492,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-09-23T19:31:54.876805+0000","last_change":"2023-09-23T19:31:12.593664+0000","last_active":"2023-09-23T19:31:54.876805+0000","last_peered":"2023-09-23T19:31:54.876805+0000","last_clean":"2023-09-23T19:31:54.876805+0000","last_became_active":"2023-09-23T19:31:12.592662+0000","last_became_peered":"2023-09-23T19:31:12.592662+0000","last_unstale":"2023-09-23T19:31:54.876805+0000","last_undegraded":"2023-09-23T19:31:54.876805+0000","last_fullsized":"2023-09-23T19:31:54.876805+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_clean_scrub_stamp":"2023-09-23T19:31:11.546138+0000","objects_scrubbed":0,"log_size":2481,"log_dups_size":0,"ondisk_log_size":2481,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162594816,"num_objects":2481,"num_object_clones":0,"num_object_copies":7443,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2481,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2481,"num_write_kb":158784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":92,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.580334+0000","last_change":"2023-09-23T19:18:09.015853+0000","last_active":"2023-09-23T19:31:13.580334+0000","last_peered":"2023-09-23T19:31:13.580334+0000","last_clean":"2023-09-23T19:31:13.580334+0000","last_became_active":"2023-09-23T19:18:09.015338+0000","last_became_peered":"2023-09-23T19:18:09.015338+0000","last_unstale":"2023-09-23T19:31:13.580334+0000","last_undegraded":"2023-09-23T19:31:13.580334+0000","last_fullsized":"2023-09-23T19:31:13.580334+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":64,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.720630+0000","last_change":"2023-09-23T19:27:24.266241+0000","last_active":"2023-09-23T19:31:13.720630+0000","last_peered":"2023-09-23T19:31:13.720630+0000","last_clean":"2023-09-23T19:31:13.720630+0000","last_became_active":"2023-09-23T19:27:24.265764+0000","last_became_peered":"2023-09-23T19:27:24.265764+0000","last_unstale":"2023-09-23T19:31:13.720630+0000","last_undegraded":"2023-09-23T19:31:13.720630+0000","last_fullsized":"2023-09-23T19:31:13.720630+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:07.694732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":64,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.567808+0000","last_change":"2023-09-23T19:18:09.005823+0000","last_active":"2023-09-23T19:31:13.567808+0000","last_peered":"2023-09-23T19:31:13.567808+0000","last_clean":"2023-09-23T19:31:13.567808+0000","last_became_active":"2023-09-23T19:18:09.005207+0000","last_became_peered":"2023-09-23T19:18:09.005207+0000","last_unstale":"2023-09-23T19:31:13.567808+0000","last_undegraded":"2023-09-23T19:31:13.567808+0000","last_fullsized":"2023-09-23T19:31:13.567808+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:05.520188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"122'2485","reported_seq":2496,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-09-23T19:31:54.813557+0000","last_change":"2023-09-23T19:31:12.576193+0000","last_active":"2023-09-23T19:31:54.813557+0000","last_peered":"2023-09-23T19:31:54.813557+0000","last_clean":"2023-09-23T19:31:54.813557+0000","last_became_active":"2023-09-23T19:31:12.575804+0000","last_became_peered":"2023-09-23T19:31:12.575804+0000","last_unstale":"2023-09-23T19:31:54.813557+0000","last_undegraded":"2023-09-23T19:31:54.813557+0000","last_fullsized":"2023-09-23T19:31:54.813557+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_clean_scrub_stamp":"2023-09-23T19:31:11.546138+0000","objects_scrubbed":0,"log_size":2485,"log_dups_size":0,"ondisk_log_size":2485,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162856960,"num_objects":2485,"num_object_clones":0,"num_object_copies":7455,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2485,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2485,"num_write_kb":159040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":73,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.723758+0000","last_change":"2023-09-23T19:28:07.884711+0000","last_active":"2023-09-23T19:31:13.723758+0000","last_peered":"2023-09-23T19:31:13.723758+0000","last_clean":"2023-09-23T19:31:13.723758+0000","last_became_active":"2023-09-23T19:28:07.883878+0000","last_became_peered":"2023-09-23T19:28:07.883878+0000","last_unstale":"2023-09-23T19:31:13.723758+0000","last_undegraded":"2023-09-23T19:31:13.723758+0000","last_fullsized":"2023-09-23T19:31:13.723758+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:55.538826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":49,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:21.094120+0000","last_change":"2023-09-23T19:22:42.145506+0000","last_active":"2023-09-23T19:31:21.094120+0000","last_peered":"2023-09-23T19:31:21.094120+0000","last_clean":"2023-09-23T19:31:21.094120+0000","last_became_active":"2023-09-23T19:22:42.143991+0000","last_became_peered":"2023-09-23T19:22:42.143991+0000","last_unstale":"2023-09-23T19:31:21.094120+0000","last_undegraded":"2023-09-23T19:31:21.094120+0000","last_fullsized":"2023-09-23T19:31:21.094120+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:56.875175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":70,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.721605+0000","last_change":"2023-09-23T19:28:07.886079+0000","last_active":"2023-09-23T19:31:13.721605+0000","last_peered":"2023-09-23T19:31:13.721605+0000","last_clean":"2023-09-23T19:31:13.721605+0000","last_became_active":"2023-09-23T19:28:07.885209+0000","last_became_peered":"2023-09-23T19:28:07.885209+0000","last_unstale":"2023-09-23T19:31:13.721605+0000","last_undegraded":"2023-09-23T19:31:13.721605+0000","last_fullsized":"2023-09-23T19:31:13.721605+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:19.991162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"122'2136","reported_seq":2148,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-09-23T19:31:50.900490+0000","last_change":"2023-09-23T19:31:12.591003+0000","last_active":"2023-09-23T19:31:50.900490+0000","last_peered":"2023-09-23T19:31:50.900490+0000","last_clean":"2023-09-23T19:31:50.900490+0000","last_became_active":"2023-09-23T19:31:12.590526+0000","last_became_peered":"2023-09-23T19:31:12.590526+0000","last_unstale":"2023-09-23T19:31:50.900490+0000","last_undegraded":"2023-09-23T19:31:50.900490+0000","last_fullsized":"2023-09-23T19:31:50.900490+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_clean_scrub_stamp":"2023-09-23T19:31:11.546138+0000","objects_scrubbed":0,"log_size":2136,"log_dups_size":0,"ondisk_log_size":2136,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:49.655472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139984896,"num_objects":2136,"num_object_clones":0,"num_object_copies":6408,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2136,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2136,"num_write_kb":136704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":92,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.729062+0000","last_change":"2023-09-23T19:27:24.271827+0000","last_active":"2023-09-23T19:31:13.729062+0000","last_peered":"2023-09-23T19:31:13.729062+0000","last_clean":"2023-09-23T19:31:13.729062+0000","last_became_active":"2023-09-23T19:27:24.271328+0000","last_became_peered":"2023-09-23T19:27:24.271328+0000","last_unstale":"2023-09-23T19:31:13.729062+0000","last_undegraded":"2023-09-23T19:31:13.729062+0000","last_fullsized":"2023-09-23T19:31:13.729062+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":82,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.568206+0000","last_change":"2023-09-23T19:22:42.149694+0000","last_active":"2023-09-23T19:31:13.568206+0000","last_peered":"2023-09-23T19:31:13.568206+0000","last_clean":"2023-09-23T19:31:13.568206+0000","last_became_active":"2023-09-23T19:22:42.148194+0000","last_became_peered":"2023-09-23T19:22:42.148194+0000","last_unstale":"2023-09-23T19:31:13.568206+0000","last_undegraded":"2023-09-23T19:31:13.568206+0000","last_fullsized":"2023-09-23T19:31:13.568206+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:47.842383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":72,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.729259+0000","last_change":"2023-09-23T19:05:52.138620+0000","last_active":"2023-09-23T19:31:13.729259+0000","last_peered":"2023-09-23T19:31:13.729259+0000","last_clean":"2023-09-23T19:31:13.729259+0000","last_became_active":"2023-09-23T19:05:52.138243+0000","last_became_peered":"2023-09-23T19:05:52.138243+0000","last_unstale":"2023-09-23T19:31:13.729259+0000","last_undegraded":"2023-09-23T19:31:13.729259+0000","last_fullsized":"2023-09-23T19:31:13.729259+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":66,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.568489+0000","last_change":"2023-09-23T19:18:09.008811+0000","last_active":"2023-09-23T19:31:13.568489+0000","last_peered":"2023-09-23T19:31:13.568489+0000","last_clean":"2023-09-23T19:31:13.568489+0000","last_became_active":"2023-09-23T19:18:09.007255+0000","last_became_peered":"2023-09-23T19:18:09.007255+0000","last_unstale":"2023-09-23T19:31:13.568489+0000","last_undegraded":"2023-09-23T19:31:13.568489+0000","last_fullsized":"2023-09-23T19:31:13.568489+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:25.874038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":103,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.568472+0000","last_change":"2023-09-23T19:28:07.876141+0000","last_active":"2023-09-23T19:31:13.568472+0000","last_peered":"2023-09-23T19:31:13.568472+0000","last_clean":"2023-09-23T19:31:13.568472+0000","last_became_active":"2023-09-23T19:28:07.875802+0000","last_became_peered":"2023-09-23T19:28:07.875802+0000","last_unstale":"2023-09-23T19:31:13.568472+0000","last_undegraded":"2023-09-23T19:31:13.568472+0000","last_fullsized":"2023-09-23T19:31:13.568472+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:38.947881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":91,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.728027+0000","last_change":"2023-09-23T19:22:42.153825+0000","last_active":"2023-09-23T19:31:13.728027+0000","last_peered":"2023-09-23T19:31:13.728027+0000","last_clean":"2023-09-23T19:31:13.728027+0000","last_became_active":"2023-09-23T19:22:42.153416+0000","last_became_peered":"2023-09-23T19:22:42.153416+0000","last_unstale":"2023-09-23T19:31:13.728027+0000","last_undegraded":"2023-09-23T19:31:13.728027+0000","last_fullsized":"2023-09-23T19:31:13.728027+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":111,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.729138+0000","last_change":"2023-09-23T19:28:07.899164+0000","last_active":"2023-09-23T19:31:13.729138+0000","last_peered":"2023-09-23T19:31:13.729138+0000","last_clean":"2023-09-23T19:31:13.729138+0000","last_became_active":"2023-09-23T19:28:07.898812+0000","last_became_peered":"2023-09-23T19:28:07.898812+0000","last_unstale":"2023-09-23T19:31:13.729138+0000","last_undegraded":"2023-09-23T19:31:13.729138+0000","last_fullsized":"2023-09-23T19:31:13.729138+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":76,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.728537+0000","last_change":"2023-09-23T19:05:52.107570+0000","last_active":"2023-09-23T19:31:13.728537+0000","last_peered":"2023-09-23T19:31:13.728537+0000","last_clean":"2023-09-23T19:31:13.728537+0000","last_became_active":"2023-09-23T19:05:52.107215+0000","last_became_peered":"2023-09-23T19:05:52.107215+0000","last_unstale":"2023-09-23T19:31:13.728537+0000","last_undegraded":"2023-09-23T19:31:13.728537+0000","last_fullsized":"2023-09-23T19:31:13.728537+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":102,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.729150+0000","last_change":"2023-09-23T19:28:07.896983+0000","last_active":"2023-09-23T19:31:13.729150+0000","last_peered":"2023-09-23T19:31:13.729150+0000","last_clean":"2023-09-23T19:31:13.729150+0000","last_became_active":"2023-09-23T19:28:07.896577+0000","last_became_peered":"2023-09-23T19:28:07.896577+0000","last_unstale":"2023-09-23T19:31:13.729150+0000","last_undegraded":"2023-09-23T19:31:13.729150+0000","last_fullsized":"2023-09-23T19:31:13.729150+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":97,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.723400+0000","last_change":"2023-09-23T19:28:07.891196+0000","last_active":"2023-09-23T19:31:13.723400+0000","last_peered":"2023-09-23T19:31:13.723400+0000","last_clean":"2023-09-23T19:31:13.723400+0000","last_became_active":"2023-09-23T19:28:07.890484+0000","last_became_peered":"2023-09-23T19:28:07.890484+0000","last_unstale":"2023-09-23T19:31:13.723400+0000","last_undegraded":"2023-09-23T19:31:13.723400+0000","last_fullsized":"2023-09-23T19:31:13.723400+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:24.574716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":110,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.722803+0000","last_change":"2023-09-23T19:28:07.883754+0000","last_active":"2023-09-23T19:31:13.722803+0000","last_peered":"2023-09-23T19:31:13.722803+0000","last_clean":"2023-09-23T19:31:13.722803+0000","last_became_active":"2023-09-23T19:28:07.883363+0000","last_became_peered":"2023-09-23T19:28:07.883363+0000","last_unstale":"2023-09-23T19:31:13.722803+0000","last_undegraded":"2023-09-23T19:31:13.722803+0000","last_fullsized":"2023-09-23T19:31:13.722803+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:56:35.805451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":82,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.568683+0000","last_change":"2023-09-23T19:27:24.267931+0000","last_active":"2023-09-23T19:31:13.568683+0000","last_peered":"2023-09-23T19:31:13.568683+0000","last_clean":"2023-09-23T19:31:13.568683+0000","last_became_active":"2023-09-23T19:27:24.267552+0000","last_became_peered":"2023-09-23T19:27:24.267552+0000","last_unstale":"2023-09-23T19:31:13.568683+0000","last_undegraded":"2023-09-23T19:31:13.568683+0000","last_fullsized":"2023-09-23T19:31:13.568683+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:12.425257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":81,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-09-23T19:31:13.727521+0000","last_change":"2023-09-23T19:06:36.673002+0000","last_active":"2023-09-23T19:31:13.727521+0000","last_peered":"2023-09-23T19:31:13.727521+0000","last_clean":"2023-09-23T19:31:13.727521+0000","last_became_active":"2023-09-23T19:06:36.671911+0000","last_became_peered":"2023-09-23T19:06:36.671911+0000","last_unstale":"2023-09-23T19:31:13.727521+0000","last_undegraded":"2023-09-23T19:31:13.727521+0000","last_fullsized":"2023-09-23T19:31:13.727521+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":2425356288,"num_objects":37008,"num_object_clones":0,"num_object_copies":111024,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":37008,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":37008,"num_write_kb":2368512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7277641728,"data_stored":7277641728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":37008,"ondisk_log_size":37008,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430557,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2019360,"kb_used_data":1417368,"kb_used_omap":0,"kb_used_meta":601984,"kb_avail":91725792,"statfs":{"total":95995035648,"available":93927211008,"internally_reserved":0,"allocated":1451384832,"data_stored":1450917079,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":616431616},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:31:27 2023","interfaces":[{"interface":"back","average":{"1min":0.96799999999999997,"5min":1.0069999999999999,"15min":1.23},"min":{"1min":0.66300000000000003,"5min":0.46000000000000002,"15min":0.26400000000000001},"max":{"1min":1.3169999999999999,"5min":9.6760000000000002,"15min":9.6760000000000002},"last":0.76500000000000001},{"interface":"front","average":{"1min":0.84799999999999998,"5min":0.85499999999999998,"15min":1.3160000000000001},"min":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.28999999999999998},"max":{"1min":1.3420000000000001,"5min":1.498,"15min":8.9670000000000005},"last":0.81299999999999994}]},{"osd":1,"last update":"Sat Sep 23 19:30:55 2023","interfaces":[{"interface":"back","average":{"1min":0.995,"5min":1.0029999999999999,"15min":1.3779999999999999},"min":{"1min":0.58299999999999996,"5min":0.29099999999999998,"15min":0.28499999999999998},"max":{"1min":1.206,"5min":5.1459999999999999,"15min":7.6180000000000003},"last":0.68500000000000005},{"interface":"front","average":{"1min":1.026,"5min":1.085,"15min":1.3819999999999999},"min":{"1min":0.71699999999999997,"5min":0.33800000000000002,"15min":0.29599999999999999},"max":{"1min":1.4179999999999999,"5min":5.1180000000000003,"15min":7.3739999999999997},"last":0.623}]},{"osd":2,"last update":"Sat Sep 23 19:31:33 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":0.86499999999999999,"15min":0.84499999999999997},"min":{"1min":0.67000000000000004,"5min":0.61499999999999999,"15min":0.61499999999999999},"max":{"1min":1.3720000000000001,"5min":1.3720000000000001,"15min":1.3720000000000001},"last":0.70199999999999996},{"interface":"front","average":{"1min":0.97099999999999997,"5min":0.97499999999999998,"15min":0.97299999999999998},"min":{"1min":0.64600000000000002,"5min":0.622,"15min":0.622},"max":{"1min":1.1659999999999999,"5min":1.294,"15min":1.294},"last":0.54200000000000004}]},{"osd":3,"last update":"Sat Sep 23 19:31:09 2023","interfaces":[{"interface":"back","average":{"1min":1.0409999999999999,"5min":1.075,"15min":1.0629999999999999},"min":{"1min":0.78400000000000003,"5min":0.755,"15min":0.755},"max":{"1min":1.22,"5min":2.8660000000000001,"15min":2.8660000000000001},"last":1.494},{"interface":"front","average":{"1min":1.1599999999999999,"5min":1.1539999999999999,"15min":1.1359999999999999},"min":{"1min":0.92600000000000005,"5min":0.61899999999999999,"15min":0.61899999999999999},"max":{"1min":1.454,"5min":2.7869999999999999,"15min":2.7869999999999999},"last":1.4670000000000001}]},{"osd":4,"last update":"Sat Sep 23 19:30:55 2023","interfaces":[{"interface":"back","average":{"1min":0.97999999999999998,"5min":0.94699999999999995,"15min":1.1659999999999999},"min":{"1min":0.67100000000000004,"5min":0.36899999999999999,"15min":0.28699999999999998},"max":{"1min":1.2909999999999999,"5min":4.1740000000000004,"15min":9.4190000000000005},"last":0.73899999999999999},{"interface":"front","average":{"1min":1.0269999999999999,"5min":1.1240000000000001,"15min":1.0760000000000001},"min":{"1min":0.76200000000000001,"5min":0.36099999999999999,"15min":0.33700000000000002},"max":{"1min":1.3129999999999999,"5min":4.6769999999999996,"15min":7.9790000000000001},"last":0.67300000000000004}]},{"osd":5,"last update":"Sat Sep 23 19:30:55 2023","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":1.0289999999999999,"15min":1.2749999999999999},"min":{"1min":0.745,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.212,"5min":3.8599999999999999,"15min":9.8580000000000005},"last":0.752},{"interface":"front","average":{"1min":1.151,"5min":1.236,"15min":1.4059999999999999},"min":{"1min":0.88200000000000001,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":1.341,"5min":9.7829999999999995,"15min":10.013999999999999},"last":1.0189999999999999}]},{"osd":6,"last update":"Sat Sep 23 19:30:55 2023","interfaces":[{"interface":"back","average":{"1min":1.1890000000000001,"5min":1.1499999999999999,"15min":1.3600000000000001},"min":{"1min":0.97999999999999998,"5min":0.41799999999999998,"15min":0.34599999999999997},"max":{"1min":1.395,"5min":5.2320000000000002,"15min":13.462},"last":0.80600000000000005},{"interface":"front","average":{"1min":1.2589999999999999,"5min":1.153,"15min":1.171},"min":{"1min":0.84199999999999997,"5min":0.442,"15min":0.36799999999999999},"max":{"1min":1.4630000000000001,"5min":2.532,"15min":7.9109999999999996},"last":0.83499999999999996}]}]},{"osd":6,"up_from":42,"seq":180388626789,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":750544,"kb_used_data":449160,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92994608,"statfs":{"total":95995035648,"available":95226478592,"internally_reserved":0,"allocated":459939840,"data_stored":459488010,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:31:39 2023","interfaces":[{"interface":"back","average":{"1min":0.86699999999999999,"5min":0.86599999999999999,"15min":1.323},"min":{"1min":0.61499999999999999,"5min":0.42799999999999999,"15min":0.24099999999999999},"max":{"1min":1.1879999999999999,"5min":1.2170000000000001,"15min":8.8989999999999991},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.998,"5min":0.94999999999999996,"15min":1.2430000000000001},"min":{"1min":0.53800000000000003,"5min":0.505,"15min":0.25900000000000001},"max":{"1min":1.885,"5min":1.885,"15min":7.5110000000000001},"last":0.60999999999999999}]},{"osd":1,"last update":"Sat Sep 23 19:30:53 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":1.1839999999999999,"15min":1.5229999999999999},"min":{"1min":0.72599999999999998,"5min":0.53100000000000003,"15min":0.29999999999999999},"max":{"1min":1.2010000000000001,"5min":5.4880000000000004,"15min":11.351000000000001},"last":1.319},{"interface":"front","average":{"1min":1.109,"5min":1.409,"15min":1.6439999999999999},"min":{"1min":0.82399999999999995,"5min":0.56100000000000005,"15min":0.34200000000000003},"max":{"1min":1.292,"5min":12.728999999999999,"15min":12.728999999999999},"last":1.3320000000000001}]},{"osd":2,"last update":"Sat Sep 23 19:31:33 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":0.94399999999999995,"15min":0.93300000000000005},"min":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"max":{"1min":1.2629999999999999,"5min":1.3120000000000001,"15min":1.3120000000000001},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.95999999999999996,"5min":0.92500000000000004,"15min":0.90500000000000003},"min":{"1min":0.61799999999999999,"5min":0.58499999999999996,"15min":0.58499999999999996},"max":{"1min":1.73,"5min":1.73,"15min":1.73},"last":0.58999999999999997}]},{"osd":3,"last update":"Sat Sep 23 19:31:15 2023","interfaces":[{"interface":"back","average":{"1min":1.077,"5min":1.0109999999999999,"15min":0.999},"min":{"1min":0.79800000000000004,"5min":0.68400000000000005,"15min":0.68400000000000005},"max":{"1min":2.0329999999999999,"5min":2.0329999999999999,"15min":2.0329999999999999},"last":0.621},{"interface":"front","average":{"1min":1.0289999999999999,"5min":0.98199999999999998,"15min":0.97199999999999998},"min":{"1min":0.67800000000000005,"5min":0.65000000000000002,"15min":0.65000000000000002},"max":{"1min":1.258,"5min":1.258,"15min":1.258},"last":0.51700000000000002}]},{"osd":4,"last update":"Sat Sep 23 19:31:15 2023","interfaces":[{"interface":"back","average":{"1min":0.879,"5min":0.91300000000000003,"15min":0.97299999999999998},"min":{"1min":0.66800000000000004,"5min":0.42999999999999999,"15min":0.29299999999999998},"max":{"1min":1.224,"5min":1.3169999999999999,"15min":6.5869999999999997},"last":0.55100000000000005},{"interface":"front","average":{"1min":1.028,"5min":1.0529999999999999,"15min":1.123},"min":{"1min":0.82799999999999996,"5min":0.40899999999999997,"15min":0.30499999999999999},"max":{"1min":1.3540000000000001,"5min":4.1040000000000001,"15min":8.5540000000000003},"last":1.972}]},{"osd":5,"last update":"Sat Sep 23 19:31:15 2023","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":1.0680000000000001,"15min":1.115},"min":{"1min":0.77500000000000002,"5min":0.48299999999999998,"15min":0.316},"max":{"1min":1.3169999999999999,"5min":4.1310000000000002,"15min":6.6660000000000004},"last":0.67300000000000004},{"interface":"front","average":{"1min":1.091,"5min":1.0700000000000001,"15min":1.3049999999999999},"min":{"1min":0.89700000000000002,"5min":0.44,"15min":0.36499999999999999},"max":{"1min":1.3979999999999999,"5min":2.774,"15min":8.5820000000000007},"last":0.97299999999999998}]},{"osd":7,"last update":"Sat Sep 23 19:31:47 2023","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":1.0840000000000001,"15min":1.1379999999999999},"min":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.38300000000000001},"max":{"1min":1.357,"5min":1.383,"15min":8.7949999999999999},"last":0.64600000000000002},{"interface":"front","average":{"1min":1.099,"5min":1.1120000000000001,"15min":1.2470000000000001},"min":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.39300000000000002},"max":{"1min":1.3859999999999999,"5min":1.516,"15min":6.7400000000000002},"last":0.56499999999999995}]}]},{"osd":1,"up_from":98,"seq":420906795120,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1407212,"kb_used_data":1197860,"kb_used_omap":4,"kb_used_meta":209339,"kb_avail":92337940,"statfs":{"total":95995035648,"available":94554050560,"internally_reserved":0,"allocated":1226608640,"data_stored":1226074840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":214364046},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:30:56 2023","interfaces":[{"interface":"back","average":{"1min":0.85099999999999998,"5min":1.002,"15min":2.0299999999999998},"min":{"1min":0.50800000000000001,"5min":0.27100000000000002,"15min":0.20799999999999999},"max":{"1min":1.321,"5min":14.99,"15min":14.99},"last":0.70899999999999996},{"interface":"front","average":{"1min":0.81799999999999995,"5min":0.96099999999999997,"15min":1.9950000000000001},"min":{"1min":0.56999999999999995,"5min":0.255,"15min":0.23499999999999999},"max":{"1min":1.3169999999999999,"5min":12.215,"15min":12.215},"last":0.33500000000000002}]},{"osd":2,"last update":"Sat Sep 23 19:31:32 2023","interfaces":[{"interface":"back","average":{"1min":0.72399999999999998,"5min":0.72899999999999998,"15min":0.73399999999999999},"min":{"1min":0.48799999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":1.306,"5min":1.3200000000000001,"15min":1.3200000000000001},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.73699999999999999,"5min":0.70399999999999996,"15min":0.68400000000000005},"min":{"1min":0.45800000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":1.2450000000000001,"5min":1.2450000000000001,"15min":1.2450000000000001},"last":2.5419999999999998}]},{"osd":3,"last update":"Sat Sep 23 19:31:11 2023","interfaces":[{"interface":"back","average":{"1min":0.78800000000000003,"5min":0.74099999999999999,"15min":0.749},"min":{"1min":0.61199999999999999,"5min":0.312,"15min":0.312},"max":{"1min":1.097,"5min":1.099,"15min":1.099},"last":0.80200000000000005},{"interface":"front","average":{"1min":0.81000000000000005,"5min":0.80000000000000004,"15min":0.81499999999999995},"min":{"1min":0.48799999999999999,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":1.302,"5min":1.302,"15min":1.302},"last":5.5049999999999999}]},{"osd":4,"last update":"Sat Sep 23 19:30:56 2023","interfaces":[{"interface":"back","average":{"1min":1.0669999999999999,"5min":1.133,"15min":1.5880000000000001},"min":{"1min":0.90300000000000002,"5min":0.36399999999999999,"15min":0.35899999999999999},"max":{"1min":1.2390000000000001,"5min":5.8920000000000003,"15min":7.6130000000000004},"last":0.50600000000000001},{"interface":"front","average":{"1min":1.05,"5min":1.179,"15min":1.3759999999999999},"min":{"1min":0.752,"5min":0.503,"15min":0.34399999999999997},"max":{"1min":1.278,"5min":7.6820000000000004,"15min":10.69},"last":2.6150000000000002}]},{"osd":5,"last update":"Sat Sep 23 19:30:56 2023","interfaces":[{"interface":"back","average":{"1min":1.014,"5min":1.218,"15min":1.4490000000000001},"min":{"1min":0.86499999999999999,"5min":0.53400000000000003,"15min":0.36699999999999999},"max":{"1min":1.341,"5min":7.4770000000000003,"15min":12.640000000000001},"last":2.6520000000000001},{"interface":"front","average":{"1min":1.1319999999999999,"5min":1.1950000000000001,"15min":1.5229999999999999},"min":{"1min":0.76000000000000001,"5min":0.45500000000000002,"15min":0.36299999999999999},"max":{"1min":1.47,"5min":7.5279999999999996,"15min":9.218},"last":0.75900000000000001}]},{"osd":6,"last update":"Sat Sep 23 19:30:56 2023","interfaces":[{"interface":"back","average":{"1min":1.169,"5min":1.252,"15min":1.5720000000000001},"min":{"1min":0.81200000000000006,"5min":0.45100000000000001,"15min":0.40100000000000002},"max":{"1min":1.5069999999999999,"5min":8.1549999999999994,"15min":9.3010000000000002},"last":0.78200000000000003},{"interface":"front","average":{"1min":1.1779999999999999,"5min":1.3440000000000001,"15min":1.7609999999999999},"min":{"1min":0.89200000000000002,"5min":0.42999999999999999,"15min":0.38800000000000001},"max":{"1min":1.4650000000000001,"5min":9.1699999999999999,"15min":9.1699999999999999},"last":0.83399999999999996}]},{"osd":7,"last update":"Sat Sep 23 19:30:56 2023","interfaces":[{"interface":"back","average":{"1min":1.2150000000000001,"5min":1.2849999999999999,"15min":1.927},"min":{"1min":0.80800000000000005,"5min":0.46899999999999997,"15min":0.40999999999999998},"max":{"1min":1.385,"5min":9.2159999999999993,"15min":9.2159999999999993},"last":0.872},{"interface":"front","average":{"1min":1.1599999999999999,"5min":1.212,"15min":1.5349999999999999},"min":{"1min":0.86299999999999999,"5min":0.57399999999999995,"15min":0.42699999999999999},"max":{"1min":1.4370000000000001,"5min":7.4390000000000001,"15min":10.742000000000001},"last":3.7869999999999999}]}]},{"osd":0,"up_from":85,"seq":365072220327,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":902036,"kb_used_data":593612,"kb_used_omap":5,"kb_used_meta":308410,"kb_avail":92843116,"statfs":{"total":95995035648,"available":95071350784,"internally_reserved":0,"allocated":607858688,"data_stored":607290937,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":315812474},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 19:30:56 2023","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":0.95699999999999996,"15min":1.4810000000000001},"min":{"1min":0.48399999999999999,"5min":0.23899999999999999,"15min":0.214},"max":{"1min":1.0209999999999999,"5min":9.2690000000000001,"15min":13.557},"last":2.3570000000000002},{"interface":"front","average":{"1min":0.78600000000000003,"5min":0.84599999999999997,"15min":1.851},"min":{"1min":0.52600000000000002,"5min":0.253,"15min":0.253},"max":{"1min":1.1000000000000001,"5min":4.1059999999999999,"15min":11.507999999999999},"last":0.86199999999999999}]},{"osd":2,"last update":"Sat Sep 23 19:31:30 2023","interfaces":[{"interface":"back","average":{"1min":0.70699999999999996,"5min":0.73899999999999999,"15min":0.76100000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":1.038,"5min":1.2609999999999999,"15min":1.2609999999999999},"last":2.399},{"interface":"front","average":{"1min":0.82599999999999996,"5min":0.79300000000000004,"15min":0.78900000000000003},"min":{"1min":0.41399999999999998,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":1.629,"5min":1.629,"15min":1.629},"last":0.71899999999999997}]},{"osd":3,"last update":"Sat Sep 23 19:31:09 2023","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":0.80400000000000005,"15min":0.77000000000000002},"min":{"1min":0.49199999999999999,"5min":0.251,"15min":0.251},"max":{"1min":1.3440000000000001,"5min":1.3440000000000001,"15min":1.3440000000000001},"last":0.35099999999999998},{"interface":"front","average":{"1min":1.0109999999999999,"5min":0.98099999999999998,"15min":0.97999999999999998},"min":{"1min":0.77600000000000002,"5min":0.372,"15min":0.372},"max":{"1min":1.2769999999999999,"5min":1.5149999999999999,"15min":1.5149999999999999},"last":0.68999999999999995}]},{"osd":4,"last update":"Sat Sep 23 19:31:38 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":0.95899999999999996,"15min":1.54},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.32100000000000001},"max":{"1min":1.1950000000000001,"5min":1.5309999999999999,"15min":11.156000000000001},"last":0.55600000000000005},{"interface":"front","average":{"1min":1.1399999999999999,"5min":1.115,"15min":1.6499999999999999},"min":{"1min":0.71499999999999997,"5min":0.48699999999999999,"15min":0.35599999999999998},"max":{"1min":2.4460000000000002,"5min":3.3180000000000001,"15min":13.566000000000001},"last":0.73999999999999999}]},{"osd":5,"last update":"Sat Sep 23 19:31:38 2023","interfaces":[{"interface":"back","average":{"1min":1.1859999999999999,"5min":1.2090000000000001,"15min":1.7250000000000001},"min":{"1min":0.68300000000000005,"5min":0.505,"15min":0.33600000000000002},"max":{"1min":2.3690000000000002,"5min":7.0780000000000003,"15min":13.509},"last":1.323},{"interface":"front","average":{"1min":1.0349999999999999,"5min":1.079,"15min":1.6879999999999999},"min":{"1min":0.80500000000000005,"5min":0.53700000000000003,"15min":0.33400000000000002},"max":{"1min":1.333,"5min":3.645,"15min":11.048999999999999},"last":1.2589999999999999}]},{"osd":6,"last update":"Sat Sep 23 19:31:38 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":1.077,"15min":1.7210000000000001},"min":{"1min":0.80800000000000005,"5min":0.59899999999999998,"15min":0.36499999999999999},"max":{"1min":1.3129999999999999,"5min":3.3359999999999999,"15min":11.467000000000001},"last":0.41799999999999998},{"interface":"front","average":{"1min":1.208,"5min":1.2090000000000001,"15min":1.8029999999999999},"min":{"1min":0.82699999999999996,"5min":0.51900000000000002,"15min":0.36399999999999999},"max":{"1min":2.4100000000000001,"5min":3.2829999999999999,"15min":13.603999999999999},"last":0.75800000000000001}]},{"osd":7,"last update":"Sat Sep 23 19:31:38 2023","interfaces":[{"interface":"back","average":{"1min":0.97399999999999998,"5min":1.1140000000000001,"15min":1.7709999999999999},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.434},"max":{"1min":1.2829999999999999,"5min":3.2400000000000002,"15min":16.271999999999998},"last":2.4289999999999998},{"interface":"front","average":{"1min":1.089,"5min":1.125,"15min":1.736},"min":{"1min":0.746,"5min":0.67500000000000004,"15min":0.38300000000000001},"max":{"1min":1.4079999999999999,"5min":1.6379999999999999,"15min":11.098000000000001},"last":3.5950000000000002}]}]},{"osd":2,"up_from":111,"seq":476741369912,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":570500,"kb_used_data":401980,"kb_used_omap":7,"kb_used_meta":168504,"kb_avail":93174652,"statfs":{"total":95995035648,"available":95410843648,"internally_reserved":0,"allocated":411627520,"data_stored":411129101,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":172548177},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:31:27 2023","interfaces":[{"interface":"back","average":{"1min":0.74099999999999999,"5min":0.71099999999999997,"15min":0.70799999999999996},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.056,"5min":1.1839999999999999,"15min":1.1839999999999999},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.78700000000000003,"5min":0.77900000000000003,"15min":0.77100000000000002},"min":{"1min":0.41699999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.6299999999999999,"5min":1.6299999999999999,"15min":1.6299999999999999},"last":0.50900000000000001}]},{"osd":1,"last update":"Sat Sep 23 19:31:27 2023","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":0.72799999999999998,"15min":0.748},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.98199999999999998,"5min":1.3740000000000001,"15min":1.3740000000000001},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.78400000000000003,"5min":0.83499999999999996,"15min":0.84599999999999997},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":1.1719999999999999,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":0.65700000000000003}]},{"osd":3,"last update":"Sat Sep 23 19:31:13 2023","interfaces":[{"interface":"back","average":{"1min":0.76800000000000002,"5min":0.751,"15min":0.747},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":1.6839999999999999,"5min":1.6839999999999999,"15min":1.6839999999999999},"last":1.835},{"interface":"front","average":{"1min":0.88200000000000001,"5min":0.85199999999999998,"15min":0.85999999999999999},"min":{"1min":0.46000000000000002,"5min":0.434,"15min":0.434},"max":{"1min":1.329,"5min":1.5389999999999999,"15min":1.5389999999999999},"last":1.4339999999999999}]},{"osd":4,"last update":"Sat Sep 23 19:31:27 2023","interfaces":[{"interface":"back","average":{"1min":1.0600000000000001,"5min":1.024,"15min":1.0229999999999999},"min":{"1min":0.69199999999999995,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.75,"5min":1.75,"15min":1.75},"last":0.85099999999999998},{"interface":"front","average":{"1min":1.0269999999999999,"5min":1.004,"15min":1.0069999999999999},"min":{"1min":0.745,"5min":0.69899999999999995,"15min":0.69899999999999995},"max":{"1min":1.3089999999999999,"5min":1.5,"15min":1.5},"last":0.871}]},{"osd":5,"last update":"Sat Sep 23 19:31:27 2023","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":1.087,"15min":1.123},"min":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"max":{"1min":1.4319999999999999,"5min":1.554,"15min":1.554},"last":0.82799999999999996},{"interface":"front","average":{"1min":1.0549999999999999,"5min":1.1000000000000001,"15min":1.129},"min":{"1min":0.69899999999999995,"5min":0.67600000000000005,"15min":0.67600000000000005},"max":{"1min":1.786,"5min":1.8340000000000001,"15min":1.8340000000000001},"last":0.89200000000000002}]},{"osd":6,"last update":"Sat Sep 23 19:31:27 2023","interfaces":[{"interface":"back","average":{"1min":1.0009999999999999,"5min":1.04,"15min":1.0880000000000001},"min":{"1min":0.77800000000000002,"5min":0.63100000000000001,"15min":0.63100000000000001},"max":{"1min":1.2649999999999999,"5min":1.738,"15min":1.738},"last":0.91200000000000003},{"interface":"front","average":{"1min":1.093,"5min":1.149,"15min":1.163},"min":{"1min":0.79500000000000004,"5min":0.76200000000000001,"15min":0.76200000000000001},"max":{"1min":1.5329999999999999,"5min":1.7030000000000001,"15min":1.7030000000000001},"last":0.86099999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:31:27 2023","interfaces":[{"interface":"back","average":{"1min":1.1439999999999999,"5min":1.179,"15min":1.2050000000000001},"min":{"1min":0.83999999999999997,"5min":0.70799999999999996,"15min":0.70799999999999996},"max":{"1min":1.8140000000000001,"5min":1.8220000000000001,"15min":1.8220000000000001},"last":0.99199999999999999},{"interface":"front","average":{"1min":1.0960000000000001,"5min":1.1419999999999999,"15min":1.1699999999999999},"min":{"1min":0.71899999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"max":{"1min":1.468,"5min":1.5669999999999999,"15min":1.5669999999999999},"last":1.004}]}]},{"osd":3,"up_from":116,"seq":498216206383,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1209388,"kb_used_data":1050404,"kb_used_omap":2,"kb_used_meta":158973,"kb_avail":92535764,"statfs":{"total":95995035648,"available":94756622336,"internally_reserved":0,"allocated":1075613696,"data_stored":1075135408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":162788405},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3,0,4,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:31:12 2023","interfaces":[{"interface":"back","average":{"1min":0.751,"5min":0.68600000000000005,"15min":0.66700000000000004},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.085,"5min":1.1000000000000001,"15min":1.1000000000000001},"last":1.647},{"interface":"front","average":{"1min":0.70699999999999996,"5min":0.76400000000000001,"15min":0.79200000000000004},"min":{"1min":0.42899999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.91400000000000003,"5min":1.3160000000000001,"15min":1.3160000000000001},"last":4.9640000000000004}]},{"osd":1,"last update":"Sat Sep 23 19:31:12 2023","interfaces":[{"interface":"back","average":{"1min":0.77700000000000002,"5min":0.753,"15min":0.746},"min":{"1min":0.40699999999999997,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.173,"5min":1.173,"15min":1.173},"last":4.8330000000000002},{"interface":"front","average":{"1min":0.88900000000000001,"5min":0.81200000000000006,"15min":0.80400000000000005},"min":{"1min":0.51800000000000002,"5min":0.373,"15min":0.373},"max":{"1min":1.2669999999999999,"5min":1.2669999999999999,"15min":1.2669999999999999},"last":1.6930000000000001}]},{"osd":2,"last update":"Sat Sep 23 19:31:12 2023","interfaces":[{"interface":"back","average":{"1min":0.74199999999999999,"5min":0.748,"15min":0.76600000000000001},"min":{"1min":0.38700000000000001,"5min":0.311,"15min":0.311},"max":{"1min":0.97999999999999998,"5min":1.137,"15min":1.137},"last":2.2770000000000001},{"interface":"front","average":{"1min":0.94199999999999995,"5min":0.86499999999999999,"15min":0.85299999999999998},"min":{"1min":0.45400000000000001,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":1.286,"5min":1.3320000000000001,"15min":1.3320000000000001},"last":2.456}]},{"osd":4,"last update":"Sat Sep 23 19:31:12 2023","interfaces":[{"interface":"back","average":{"1min":1.0589999999999999,"5min":1.0720000000000001,"15min":1.075},"min":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"max":{"1min":1.397,"5min":1.454,"15min":1.454},"last":0.753},{"interface":"front","average":{"1min":1.101,"5min":1.1299999999999999,"15min":1.1419999999999999},"min":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"max":{"1min":1.419,"5min":1.6080000000000001,"15min":1.6080000000000001},"last":5.0049999999999999}]},{"osd":5,"last update":"Sat Sep 23 19:31:12 2023","interfaces":[{"interface":"back","average":{"1min":1.014,"5min":1.093,"15min":1.115},"min":{"1min":0.67900000000000005,"5min":0.57699999999999996,"15min":0.57699999999999996},"max":{"1min":1.413,"5min":1.5169999999999999,"15min":1.5169999999999999},"last":4.923},{"interface":"front","average":{"1min":1.091,"5min":1.0800000000000001,"15min":1.0800000000000001},"min":{"1min":0.70599999999999996,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":1.4079999999999999,"5min":1.4079999999999999,"15min":1.4079999999999999},"last":0.47899999999999998}]},{"osd":6,"last update":"Sat Sep 23 19:31:12 2023","interfaces":[{"interface":"back","average":{"1min":1.105,"5min":1.1399999999999999,"15min":1.1439999999999999},"min":{"1min":0.63700000000000001,"5min":0.60899999999999999,"15min":0.60899999999999999},"max":{"1min":1.4790000000000001,"5min":1.6830000000000001,"15min":1.6830000000000001},"last":5.0469999999999997},{"interface":"front","average":{"1min":1.1379999999999999,"5min":1.139,"15min":1.145},"min":{"1min":0.80300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"max":{"1min":1.286,"5min":1.5109999999999999,"15min":1.5109999999999999},"last":0.43099999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:31:12 2023","interfaces":[{"interface":"back","average":{"1min":1.1839999999999999,"5min":1.1579999999999999,"15min":1.159},"min":{"1min":0.77200000000000002,"5min":0.71199999999999997,"15min":0.71199999999999997},"max":{"1min":1.4530000000000001,"5min":1.597,"15min":1.597},"last":0.53400000000000003},{"interface":"front","average":{"1min":1.2549999999999999,"5min":1.1819999999999999,"15min":1.1639999999999999},"min":{"1min":0.65600000000000003,"5min":0.58899999999999997,"15min":0.58899999999999997},"max":{"1min":1.625,"5min":1.7210000000000001,"15min":1.7210000000000001},"last":0.80500000000000005}]}]},{"osd":4,"up_from":30,"seq":128849019254,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1065624,"kb_used_data":480016,"kb_used_omap":0,"kb_used_meta":585600,"kb_avail":92679528,"statfs":{"total":95995035648,"available":94903836672,"internally_reserved":0,"allocated":491536384,"data_stored":491076209,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":599654400},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.85899999999999999,"5min":0.83599999999999997,"15min":2.472},"min":{"1min":0.57199999999999995,"5min":0.33200000000000002,"15min":0.26200000000000001},"max":{"1min":1.1479999999999999,"5min":1.3100000000000001,"15min":129.16200000000001},"last":0.309},{"interface":"front","average":{"1min":0.84799999999999998,"5min":0.85799999999999998,"15min":2.6339999999999999},"min":{"1min":0.57199999999999995,"5min":0.48099999999999998,"15min":0.27700000000000002},"max":{"1min":1.2030000000000001,"5min":1.2729999999999999,"15min":130.40600000000001},"last":0.48099999999999998}]},{"osd":1,"last update":"Sat Sep 23 19:31:01 2023","interfaces":[{"interface":"back","average":{"1min":0.95599999999999996,"5min":1.2729999999999999,"15min":1.571},"min":{"1min":0.629,"5min":0.30099999999999999,"15min":0.29499999999999998},"max":{"1min":1.1919999999999999,"5min":12.372,"15min":12.372},"last":0.57399999999999995},{"interface":"front","average":{"1min":1.0720000000000001,"5min":1.2889999999999999,"15min":1.5129999999999999},"min":{"1min":0.60099999999999998,"5min":0.34699999999999998,"15min":0.31},"max":{"1min":1.339,"5min":7.3360000000000003,"15min":7.3360000000000003},"last":0.54000000000000004}]},{"osd":2,"last update":"Sat Sep 23 19:31:30 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":0.91900000000000004,"15min":0.89700000000000002},"min":{"1min":0.65500000000000003,"5min":0.59999999999999998,"15min":0.59999999999999998},"max":{"1min":1.2649999999999999,"5min":1.327,"15min":1.327},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.91100000000000003,"5min":0.90500000000000003,"15min":0.90000000000000002},"min":{"1min":0.55200000000000005,"5min":0.499,"15min":0.499},"max":{"1min":1.1919999999999999,"5min":1.4379999999999999,"15min":1.4379999999999999},"last":0.46400000000000002}]},{"osd":3,"last update":"Sat Sep 23 19:31:15 2023","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":0.96099999999999997,"15min":0.96999999999999997},"min":{"1min":0.63500000000000001,"5min":0.60099999999999998,"15min":0.60099999999999998},"max":{"1min":1.216,"5min":1.6899999999999999,"15min":1.6899999999999999},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.99399999999999999,"5min":0.96899999999999997,"15min":0.96399999999999997},"min":{"1min":0.73499999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":1.2589999999999999,"5min":1.552,"15min":1.552},"last":1.0169999999999999}]},{"osd":5,"last update":"Sat Sep 23 19:31:44 2023","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":1.01,"15min":1.1579999999999999},"min":{"1min":0.65000000000000002,"5min":0.504,"15min":0.29199999999999998},"max":{"1min":1.528,"5min":2.1709999999999998,"15min":12.459},"last":0.40200000000000002},{"interface":"front","average":{"1min":1.036,"5min":1.0660000000000001,"15min":1.2090000000000001},"min":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.26700000000000002},"max":{"1min":1.4319999999999999,"5min":1.8680000000000001,"15min":12.478},"last":0.55400000000000005}]},{"osd":6,"last update":"Sat Sep 23 19:31:01 2023","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":1.129,"15min":1.2649999999999999},"min":{"1min":0.93200000000000005,"5min":0.39600000000000002,"15min":0.29999999999999999},"max":{"1min":1.387,"5min":3.8530000000000002,"15min":7.0810000000000004},"last":0.432},{"interface":"front","average":{"1min":1.1619999999999999,"5min":1.198,"15min":1.2789999999999999},"min":{"1min":0.70099999999999996,"5min":0.40600000000000003,"15min":0.34300000000000003},"max":{"1min":1.399,"5min":4.2110000000000003,"15min":6.9050000000000002},"last":0.51000000000000001}]},{"osd":7,"last update":"Sat Sep 23 19:31:01 2023","interfaces":[{"interface":"back","average":{"1min":1.0780000000000001,"5min":1.204,"15min":1.2250000000000001},"min":{"1min":0.71399999999999997,"5min":0.41599999999999998,"15min":0.312},"max":{"1min":1.3520000000000001,"5min":6.6340000000000003,"15min":7.8479999999999999},"last":0.66200000000000003},{"interface":"front","average":{"1min":1.165,"5min":1.2210000000000001,"15min":1.1339999999999999},"min":{"1min":0.83199999999999996,"5min":0.42499999999999999,"15min":0.36699999999999999},"max":{"1min":1.4419999999999999,"5min":5.8570000000000002,"15min":5.8570000000000002},"last":0.85399999999999998}]}]},{"osd":5,"up_from":36,"seq":154618823021,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1973952,"kb_used_data":1376056,"kb_used_omap":0,"kb_used_meta":597888,"kb_avail":91771200,"statfs":{"total":95995035648,"available":93973708800,"internally_reserved":0,"allocated":1409081344,"data_stored":1408640859,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":612237312},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2,3,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:31:33 2023","interfaces":[{"interface":"back","average":{"1min":0.85899999999999999,"5min":0.90600000000000003,"15min":1.288},"min":{"1min":0.59899999999999998,"5min":0.49299999999999999,"15min":0.27600000000000002},"max":{"1min":1.22,"5min":2.2360000000000002,"15min":11.058},"last":0.497},{"interface":"front","average":{"1min":0.87,"5min":0.92100000000000004,"15min":1.5640000000000001},"min":{"1min":0.63900000000000001,"5min":0.54000000000000004,"15min":0.29699999999999999},"max":{"1min":1.1100000000000001,"5min":3.7890000000000001,"15min":10.802},"last":2.1120000000000001}]},{"osd":1,"last update":"Sat Sep 23 19:30:58 2023","interfaces":[{"interface":"back","average":{"1min":0.95399999999999996,"5min":0.99399999999999999,"15min":1.24},"min":{"1min":0.65600000000000003,"5min":0.29999999999999999,"15min":0.27200000000000002},"max":{"1min":1.24,"5min":4.8899999999999997,"15min":9.5389999999999997},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.96599999999999997,"5min":1.2609999999999999,"15min":1.982},"min":{"1min":0.54200000000000004,"5min":0.30499999999999999,"15min":0.26700000000000002},"max":{"1min":1.2010000000000001,"5min":11.228999999999999,"15min":11.228999999999999},"last":2.1669999999999998}]},{"osd":2,"last update":"Sat Sep 23 19:31:33 2023","interfaces":[{"interface":"back","average":{"1min":1.0169999999999999,"5min":1.0009999999999999,"15min":0.97299999999999998},"min":{"1min":0.63500000000000001,"5min":0.56899999999999995,"15min":0.56899999999999995},"max":{"1min":1.206,"5min":2.3069999999999999,"15min":2.3069999999999999},"last":0.50600000000000001},{"interface":"front","average":{"1min":1.0269999999999999,"5min":1.0760000000000001,"15min":1.0569999999999999},"min":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"max":{"1min":1.274,"5min":2.335,"15min":2.335},"last":0.58299999999999996}]},{"osd":3,"last update":"Sat Sep 23 19:31:10 2023","interfaces":[{"interface":"back","average":{"1min":0.97599999999999998,"5min":1.0129999999999999,"15min":1.0449999999999999},"min":{"1min":0.622,"5min":0.59399999999999997,"15min":0.59399999999999997},"max":{"1min":1.349,"5min":1.702,"15min":1.702},"last":0.47199999999999998},{"interface":"front","average":{"1min":1.1120000000000001,"5min":1.1259999999999999,"15min":1.1579999999999999},"min":{"1min":0.81699999999999995,"5min":0.68600000000000005,"15min":0.68600000000000005},"max":{"1min":1.3859999999999999,"5min":2.5179999999999998,"15min":2.5179999999999998},"last":0.55100000000000005}]},{"osd":4,"last update":"Sat Sep 23 19:31:33 2023","interfaces":[{"interface":"back","average":{"1min":0.95099999999999996,"5min":0.92200000000000004,"15min":1.0600000000000001},"min":{"1min":0.58899999999999997,"5min":0.48099999999999998,"15min":0.29199999999999998},"max":{"1min":1.4730000000000001,"5min":2.8250000000000002,"15min":8.1790000000000003},"last":2.1400000000000001},{"interface":"front","average":{"1min":0.98999999999999999,"5min":1.002,"15min":1.0780000000000001},"min":{"1min":0.63400000000000001,"5min":0.56999999999999995,"15min":0.27000000000000002},"max":{"1min":1.2929999999999999,"5min":2.5920000000000001,"15min":13.714},"last":0.54000000000000004}]},{"osd":6,"last update":"Sat Sep 23 19:31:18 2023","interfaces":[{"interface":"back","average":{"1min":1.1000000000000001,"5min":1.0800000000000001,"15min":1.21},"min":{"1min":0.75800000000000001,"5min":0.42599999999999999,"15min":0.307},"max":{"1min":1.321,"5min":2.1920000000000002,"15min":26.448},"last":2.149},{"interface":"front","average":{"1min":1.167,"5min":1.131,"15min":1.2769999999999999},"min":{"1min":0.92800000000000005,"5min":0.40500000000000003,"15min":0.39300000000000002},"max":{"1min":1.395,"5min":2.6230000000000002,"15min":26.841999999999999},"last":2.0449999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:31:49 2023","interfaces":[{"interface":"back","average":{"1min":1.282,"5min":1.123,"15min":1.2190000000000001},"min":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.36599999999999999},"max":{"1min":3.5950000000000002,"5min":3.5950000000000002,"15min":7.9320000000000004},"last":0.57299999999999995},{"interface":"front","average":{"1min":1.0640000000000001,"5min":1.1539999999999999,"15min":1.101},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.35799999999999998},"max":{"1min":1.3680000000000001,"5min":1.633,"15min":7.7050000000000001},"last":1.524}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":621346816,"data_stored":621346816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1244135424,"data_stored":1244135424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":428474368,"data_stored":428474368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1099235328,"data_stored":1099235328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":493289472,"data_stored":493289472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1452212224,"data_stored":1452212224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":473366528,"data_stored":473366528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1465581568,"data_stored":1465581568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T19:31:59.917 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_4 pgp_num 16 2023-09-23T19:31:59.987 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:00.011 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:00.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:00.015 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:32:00.116 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:00.143 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:00.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:00.149 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:32:00.250 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:00.275 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:00.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:00.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:00.294 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 26 16 46571 46555 111.893 85.1875 0.00986969 0.00892941 2023-09-23T19:32:00.379 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:00.410 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:00.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:00.414 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:00.515 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:00.545 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:00.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:00.550 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:00.629 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:32:00.650 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:00.706 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:00.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:00.709 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:32:00.809 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:00.852 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:00.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:00.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:32:00.957 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:00.994 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:00.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:00.998 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:32:01.099 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:01.158 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:01.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:01.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:32:01.262 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:01.290 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:01.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:01.294 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:01.295 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 27 16 47831 47815 110.665 78.75 0.0125964 0.00902904 2023-09-23T19:32:01.395 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:01.425 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:01.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:01.428 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:32:01.529 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:01.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:01 smithi092 bash[58855]: cluster 2023-09-23T19:31:59.965919+0000 mgr.y (mgr.25325) 731 : cluster 0 pgmap v629: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2023-09-23T19:32:01.584 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:01.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:01.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:01.690 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:01.700 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:01 smithi016 bash[77995]: cluster 2023-09-23T19:31:59.965919+0000 mgr.y (mgr.25325) 731 : cluster 0 pgmap v629: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2023-09-23T19:32:01.700 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:01 smithi016 bash[75004]: cluster 2023-09-23T19:31:59.965919+0000 mgr.y (mgr.25325) 731 : cluster 0 pgmap v629: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2023-09-23T19:32:01.717 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:01.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:01.720 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:32:01.820 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:01.849 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:01.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:01.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:01.918 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T19:32:01.953 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:02.016 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:02.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:02.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:32:02.028 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:32:02.122 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:02.159 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:02.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:02.165 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:02.265 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:02.294 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 28 16 49026 49010 109.38 74.6875 0.0176343 0.00913623 2023-09-23T19:32:02.442 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:02.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:02.446 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:32:02.546 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:02.577 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:02.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:02.579 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:32:02.680 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:02.705 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:02.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:02.709 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:02.810 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:02.838 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:02.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:02.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:02.942 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:02.986 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:02.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:02.993 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:32:03.093 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:03.206 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:03.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:03.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:32:03.295 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 29 16 50174 50158 108.079 71.75 0.0149303 0.00924592 2023-09-23T19:32:03.310 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:03.338 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:03.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:03.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:03.441 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:03.467 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:03.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:03.469 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:32:03.570 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:03.592 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:03.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:03.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:03.696 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:03.727 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:03.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:03.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:03.831 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:03.840 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:03 smithi092 bash[58855]: cluster 2023-09-23T19:32:01.966720+0000 mgr.y (mgr.25325) 732 : cluster 0 pgmap v630: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-09-23T19:32:03.840 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:03 smithi092 bash[58855]: audit 2023-09-23T19:32:02.627321+0000 mon.a (mon.0) 923 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:32:03.840 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:03 smithi092 bash[58855]: audit 2023-09-23T19:32:02.639535+0000 mon.a (mon.0) 924 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:32:03.857 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:03.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:03.859 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:03.960 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:03.994 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:03.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:03.997 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:04.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:03 smithi016 bash[75004]: cluster 2023-09-23T19:32:01.966720+0000 mgr.y (mgr.25325) 732 : cluster 0 pgmap v630: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-09-23T19:32:04.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:03 smithi016 bash[75004]: audit 2023-09-23T19:32:02.627321+0000 mon.a (mon.0) 923 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:32:04.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:03 smithi016 bash[75004]: audit 2023-09-23T19:32:02.639535+0000 mon.a (mon.0) 924 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:32:04.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:03 smithi016 bash[77995]: cluster 2023-09-23T19:32:01.966720+0000 mgr.y (mgr.25325) 732 : cluster 0 pgmap v630: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-09-23T19:32:04.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:03 smithi016 bash[77995]: audit 2023-09-23T19:32:02.627321+0000 mon.a (mon.0) 923 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:32:04.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:03 smithi016 bash[77995]: audit 2023-09-23T19:32:02.639535+0000 mon.a (mon.0) 924 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:32:04.069 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:32:04.098 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:04.122 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:04.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:04.129 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:04.230 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:04.255 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:04.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:04.259 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:04.295 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 30 16 51437 51421 107.108 78.9375 0.0117924 0.00933065 2023-09-23T19:32:04.359 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:04.754 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:04.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:04.757 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:32:04.858 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:04.886 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:04.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:04.888 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:32:04.989 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:05.012 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:05.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:05.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:32:05.090 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:32:05.117 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:05.142 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:05 smithi092 bash[58855]: cluster 2023-09-23T19:32:03.967178+0000 mgr.y (mgr.25325) 733 : cluster 0 pgmap v631: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-09-23T19:32:05.143 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:05 smithi092 bash[58855]: audit 2023-09-23T19:32:04.139826+0000 mon.a (mon.0) 925 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:32:05.143 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:05 smithi092 bash[58855]: audit 2023-09-23T19:32:04.147382+0000 mon.a (mon.0) 926 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:32:05.143 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:05 smithi092 bash[58855]: audit 2023-09-23T19:32:04.839258+0000 mon.a (mon.0) 927 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:32:05.143 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:05 smithi092 bash[58855]: audit 2023-09-23T19:32:04.840625+0000 mon.a (mon.0) 928 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:32:05.143 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:05 smithi092 bash[58855]: audit 2023-09-23T19:32:04.851432+0000 mon.a (mon.0) 929 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:32:05.148 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:05.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:05.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:32:05.251 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:05.278 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:05.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:05.281 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:32:05.295 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 31 16 52585 52569 105.967 71.75 0.00779752 0.00941105 2023-09-23T19:32:05.344 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:05 smithi016 bash[75004]: cluster 2023-09-23T19:32:03.967178+0000 mgr.y (mgr.25325) 733 : cluster 0 pgmap v631: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-09-23T19:32:05.344 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:05 smithi016 bash[75004]: audit 2023-09-23T19:32:04.139826+0000 mon.a (mon.0) 925 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:32:05.344 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:05 smithi016 bash[75004]: audit 2023-09-23T19:32:04.147382+0000 mon.a (mon.0) 926 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:32:05.344 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:05 smithi016 bash[75004]: audit 2023-09-23T19:32:04.839258+0000 mon.a (mon.0) 927 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:32:05.344 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:05 smithi016 bash[75004]: audit 2023-09-23T19:32:04.840625+0000 mon.a (mon.0) 928 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:32:05.344 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:05 smithi016 bash[75004]: audit 2023-09-23T19:32:04.851432+0000 mon.a (mon.0) 929 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:32:05.345 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:05 smithi016 bash[77995]: cluster 2023-09-23T19:32:03.967178+0000 mgr.y (mgr.25325) 733 : cluster 0 pgmap v631: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-09-23T19:32:05.345 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:05 smithi016 bash[77995]: audit 2023-09-23T19:32:04.139826+0000 mon.a (mon.0) 925 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:32:05.345 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:05 smithi016 bash[77995]: audit 2023-09-23T19:32:04.147382+0000 mon.a (mon.0) 926 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:32:05.345 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:05 smithi016 bash[77995]: audit 2023-09-23T19:32:04.839258+0000 mon.a (mon.0) 927 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:32:05.345 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:05 smithi016 bash[77995]: audit 2023-09-23T19:32:04.840625+0000 mon.a (mon.0) 928 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:32:05.345 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:05 smithi016 bash[77995]: audit 2023-09-23T19:32:04.851432+0000 mon.a (mon.0) 929 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:32:05.382 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:05.412 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:05.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:05.418 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:05.518 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:05.543 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:05.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:05.545 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:05.645 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:05.670 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:05.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:05.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:05.777 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:05.805 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:05.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:05.809 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:05.909 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:05.935 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:05.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:05.938 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:32:06.035 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:32:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:32:05] "GET /metrics HTTP/1.1" 200 36235 "" "Prometheus/2.33.4" 2023-09-23T19:32:06.039 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:06.068 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:06.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:06.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:32:06.173 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:06.293 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:06.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:06.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:32:06.295 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 32 16 53857 53841 105.139 79.5 0.0134897 0.00950575 2023-09-23T19:32:06.396 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:06.426 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:06.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:06.429 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:32:06.529 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:06.554 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:06.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:06.557 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:06.657 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:06.886 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:06.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:06.888 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:32:06.989 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:07.013 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:07.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:07.018 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:32:07.119 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:07.155 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:07.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:07.161 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:07.261 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:07.294 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:07.295 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 33 16 55056 55040 104.224 74.9375 0.0111901 0.00958896 2023-09-23T19:32:07.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:07.297 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:32:07.398 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:07.431 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:07 smithi016 bash[77995]: cluster 2023-09-23T19:32:05.967848+0000 mgr.y (mgr.25325) 734 : cluster 0 pgmap v632: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-09-23T19:32:07.431 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:07 smithi016 bash[75004]: cluster 2023-09-23T19:32:05.967848+0000 mgr.y (mgr.25325) 734 : cluster 0 pgmap v632: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-09-23T19:32:07.451 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:07.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:07.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:32:07.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:07 smithi092 bash[58855]: cluster 2023-09-23T19:32:05.967848+0000 mgr.y (mgr.25325) 734 : cluster 0 pgmap v632: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-09-23T19:32:07.556 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:07.587 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:07.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:07.592 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:32:07.693 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:07.722 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:07.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:07.724 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:07.825 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:07.853 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:07.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:07.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:32:07.957 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:07.993 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:07.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:07.997 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:08.022 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:32:08.099 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:08.129 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:08.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:08.133 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:08.233 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:08.260 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:08.295 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 34 16 56253 56237 103.359 74.8125 0.0120012 0.00966893 2023-09-23T19:32:08.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:08.316 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:32:08.416 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:08.451 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:08.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:08.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:32:08.555 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:08.579 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:08.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:08.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:32:08.684 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:08.710 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:08.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:08.713 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:32:08.813 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:08.847 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:08.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:08.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:08.951 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:09.029 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:09.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:09.032 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:32:09.133 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:09.159 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:09.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:09.163 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:32:09.264 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:09.289 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:09.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:09.292 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:32:09.295 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 35 16 57486 57470 102.607 77.0625 0.00702921 0.00974012 2023-09-23T19:32:09.393 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:09.420 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:09.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:09.424 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:32:09.424 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:09 smithi092 bash[58855]: cluster 2023-09-23T19:32:07.968442+0000 mgr.y (mgr.25325) 735 : cluster 0 pgmap v633: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-09-23T19:32:09.525 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:09.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:09 smithi016 bash[75004]: cluster 2023-09-23T19:32:07.968442+0000 mgr.y (mgr.25325) 735 : cluster 0 pgmap v633: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-09-23T19:32:09.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:09 smithi016 bash[77995]: cluster 2023-09-23T19:32:07.968442+0000 mgr.y (mgr.25325) 735 : cluster 0 pgmap v633: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-09-23T19:32:09.565 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:09.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:09.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:32:09.667 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:09.702 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:09.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:09.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:09.809 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:09.835 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:09.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:09.837 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:09.938 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:10.007 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:10.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:10.009 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:32:10.110 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:10.141 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:10.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:10.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:32:10.245 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:10.275 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:10.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:10.281 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:10.295 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 36 16 58556 58540 101.615 66.875 0.0185994 0.00983493 2023-09-23T19:32:10.381 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:10.450 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:10.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:10.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:32:10.556 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:10.593 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:10.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:10.601 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:10.702 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:10.729 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:10.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:10.734 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:10.835 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:10.862 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:10.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:10.865 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:32:10.966 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:10.992 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:10.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:10.997 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:32:11.098 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:11.134 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:11.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:11.137 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:32:11.238 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:11.261 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:11.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:11.265 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:11.296 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 37 16 59771 59755 100.921 75.9375 0.00606287 0.00989626 2023-09-23T19:32:11.366 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:11.393 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:11 smithi016 bash[75004]: cluster 2023-09-23T19:32:09.969051+0000 mgr.y (mgr.25325) 736 : cluster 0 pgmap v634: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-09-23T19:32:11.393 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:11 smithi016 bash[77995]: cluster 2023-09-23T19:32:09.969051+0000 mgr.y (mgr.25325) 736 : cluster 0 pgmap v634: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-09-23T19:32:11.399 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:11.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:11.405 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:32:11.505 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:11.537 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:11.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:11 smithi092 bash[58855]: cluster 2023-09-23T19:32:09.969051+0000 mgr.y (mgr.25325) 736 : cluster 0 pgmap v634: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-09-23T19:32:11.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:11.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:32:11.642 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:11.670 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:11.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:11.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:32:11.776 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:11.801 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:11.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:11.806 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:11.907 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:11.941 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:11.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:11.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:32:12.044 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:12.107 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:12.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:12.110 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:32:12.210 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:12.237 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:12.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:12.241 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:32:12.296 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 38 16 60928 60912 100.168 72.3125 0.00755578 0.00997693 2023-09-23T19:32:12.342 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:12.370 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:12.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:12.373 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:32:12.474 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:12.501 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:12.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:12.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:32:12.604 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:12.633 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:12.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:12.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:12.736 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:12.764 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:12.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:12.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:32:12.872 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:12.897 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:12.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:12.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:13.000 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:13.030 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:13.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:13.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:32:13.136 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:13.172 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:13.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:13.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:32:13.279 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:13.296 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 39 16 62122 62106 99.5125 74.625 0.0180591 0.0100422 2023-09-23T19:32:13.305 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:13.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:13.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:32:13.409 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:13.440 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:13.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:13.444 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:13.544 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:13.572 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:13.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:13.577 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:13 smithi092 bash[58855]: cluster 2023-09-23T19:32:11.969634+0000 mgr.y (mgr.25325) 737 : cluster 0 pgmap v635: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-09-23T19:32:13.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:32:13.678 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:13.686 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:13 smithi016 bash[75004]: cluster 2023-09-23T19:32:11.969634+0000 mgr.y (mgr.25325) 737 : cluster 0 pgmap v635: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-09-23T19:32:13.687 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:13 smithi016 bash[77995]: cluster 2023-09-23T19:32:11.969634+0000 mgr.y (mgr.25325) 737 : cluster 0 pgmap v635: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-09-23T19:32:13.715 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:13.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:13.720 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:32:13.820 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:13.854 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:13.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:13.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:32:13.960 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:13.988 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:13.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:13.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:32:14.092 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:14.116 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:14.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:14.118 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:32:14.218 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:14.245 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:14.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:14.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:32:14.296 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-09-23T19:32:14.294169+0000 min lat: 0.00139623 max lat: 0.199525 avg lat: 0.0101063 2023-09-23T19:32:14.296 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:32:14.296 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 40 16 63316 63300 98.8901 74.625 0.0124281 0.0101063 2023-09-23T19:32:14.349 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:14.357 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:14 smithi092 bash[58855]: audit 2023-09-23T19:32:13.636952+0000 mon.a (mon.0) 930 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:32:14.375 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:14.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:14.380 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:32:14.480 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:14.512 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:14.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:14.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:32:14.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:14 smithi016 bash[77995]: audit 2023-09-23T19:32:13.636952+0000 mon.a (mon.0) 930 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:32:14.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:14 smithi016 bash[75004]: audit 2023-09-23T19:32:13.636952+0000 mon.a (mon.0) 930 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:32:14.615 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:14.652 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:14.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:14.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:32:14.756 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:14.780 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:14.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:14.784 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:32:14.885 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:14.910 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:14.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:14.912 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:32:15.013 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:15.032 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:15.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:15.033 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:32:15.134 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:15.166 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:15.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:15.169 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:15.269 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:15.296 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:15.296 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 41 16 64346 64330 98.0481 64.375 0.0086409 0.010193 2023-09-23T19:32:15.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:15.301 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:32:15.401 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:15.434 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:15.436 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:15 smithi092 bash[58855]: cluster 2023-09-23T19:32:13.972045+0000 mgr.y (mgr.25325) 738 : cluster 0 pgmap v636: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-09-23T19:32:15.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:15.437 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:32:15.491 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:15 smithi016 bash[77995]: cluster 2023-09-23T19:32:13.972045+0000 mgr.y (mgr.25325) 738 : cluster 0 pgmap v636: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-09-23T19:32:15.492 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:15 smithi016 bash[75004]: cluster 2023-09-23T19:32:13.972045+0000 mgr.y (mgr.25325) 738 : cluster 0 pgmap v636: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-09-23T19:32:15.537 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:15.564 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:15.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:15.572 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:32:15.672 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:15.699 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:15.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:15.701 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:32:15.793 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:32:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:32:15] "GET /metrics HTTP/1.1" 200 36239 "" "Prometheus/2.33.4" 2023-09-23T19:32:15.802 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:15.835 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:15.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:15.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:15.938 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:15.964 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:15.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:15.967 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:32:16.068 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:16.098 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:16.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:16.101 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:32:16.202 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:16.235 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:16.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:16.238 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:32:16.296 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 42 16 65309 65293 97.1466 60.1875 0.00681714 0.0102866 2023-09-23T19:32:16.339 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:16.354 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-09-23T19:32:16.354 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:32:16.370 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:16.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:16.373 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:16.473 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:16.501 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:16.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:16.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:32:16.605 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:16.643 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:16.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:16.648 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:32:16.749 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:16.782 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:16.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:16.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:16.885 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:16.936 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:16.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:16.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:32:17.039 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:17.071 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:17.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:17.077 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:17.177 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:17.208 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:17.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:17.211 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:17.296 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 43 16 66370 66354 96.4294 66.3125 0.0212048 0.0103637 2023-09-23T19:32:17.312 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:17.338 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:17.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:17.343 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:32:17.445 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:17.469 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:17.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:17.472 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:32:17.573 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:17.600 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:17.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:17.604 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:32:17.652 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:17 smithi016 bash[75004]: cluster 2023-09-23T19:32:15.972734+0000 mgr.y (mgr.25325) 739 : cluster 0 pgmap v637: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-09-23T19:32:17.653 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:17 smithi016 bash[77995]: cluster 2023-09-23T19:32:15.972734+0000 mgr.y (mgr.25325) 739 : cluster 0 pgmap v637: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-09-23T19:32:17.705 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:17.714 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:17 smithi092 bash[58855]: cluster 2023-09-23T19:32:15.972734+0000 mgr.y (mgr.25325) 739 : cluster 0 pgmap v637: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-09-23T19:32:17.732 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:17.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:17.735 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:32:17.836 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:17.877 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:17.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:17.880 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:32:17.980 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:18.014 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:18.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:18.017 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:32:18.118 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:18.146 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:18.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:18.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:32:18.252 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:18.281 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:18.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:18.285 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:32:18.296 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 44 16 67252 67236 95.4907 55.125 0.0131412 0.0104656 2023-09-23T19:32:18.386 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:18.417 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:18.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:18.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:32:18.520 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:18.546 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:18.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:18.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:18.649 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:18.683 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:18.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:18.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:32:18.787 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:18.810 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:18.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:18.813 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:32:18.914 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:18.941 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:18.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:18.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:18.947 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:32:18.989 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:32:19.027 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:32:19.046 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:19.067 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:32:19.073 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:19.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:19.075 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:32:19.098 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:32:19.135 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:32:19.177 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:19.182 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:32:19.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:19.219 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:19.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:32:19.230 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:32:19.238 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:32:19.239 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:32:19.239 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:32:19.239 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:32:19.239 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:32:19.239 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:32:19.239 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:32:19.239 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:32:19.296 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 45 16 68236 68220 94.7353 61.5 0.0134023 0.0105503 2023-09-23T19:32:19.320 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:19.345 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:19.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:19.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:32:19.451 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:19.478 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:19.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:19.480 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:32:19.581 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:19.589 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:19 smithi016 bash[77995]: cluster 2023-09-23T19:32:17.973427+0000 mgr.y (mgr.25325) 740 : cluster 0 pgmap v638: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T19:32:19.590 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:19 smithi016 bash[77995]: audit 2023-09-23T19:32:18.858664+0000 mon.a (mon.0) 931 : audit 0 from='client.? 172.21.15.16:0/3878619084' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:32:19.590 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:19 smithi016 bash[75004]: cluster 2023-09-23T19:32:17.973427+0000 mgr.y (mgr.25325) 740 : cluster 0 pgmap v638: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T19:32:19.590 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:19 smithi016 bash[75004]: audit 2023-09-23T19:32:18.858664+0000 mon.a (mon.0) 931 : audit 0 from='client.? 172.21.15.16:0/3878619084' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:32:19.609 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:19.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:19.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:32:19.715 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:19.748 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:19.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:19.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:32:19.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:19 smithi092 bash[58855]: cluster 2023-09-23T19:32:17.973427+0000 mgr.y (mgr.25325) 740 : cluster 0 pgmap v638: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T19:32:19.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:19 smithi092 bash[58855]: audit 2023-09-23T19:32:18.858664+0000 mon.a (mon.0) 931 : audit 0 from='client.? 172.21.15.16:0/3878619084' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:32:19.852 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:19.871 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:19.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:19.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:32:19.973 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:20.005 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:20.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:20.009 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:20.110 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:20.135 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:20.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:20.140 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:20.241 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:20.262 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:20.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:20.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:32:20.296 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 46 16 69281 69265 94.0956 65.3125 0.0131512 0.0106189 2023-09-23T19:32:20.362 INFO:teuthology.orchestra.run.smithi016.stderr:set pool 9 pgp_num to 16 2023-09-23T19:32:20.366 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:20.402 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:20.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:20.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:20.507 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:20.534 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:20.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:20 smithi016 bash[77995]: audit 2023-09-23T19:32:19.371921+0000 mon.c (mon.2) 143 : audit 1 from='client.? 172.21.15.16:0/1451087303' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T19:32:20.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:20 smithi016 bash[77995]: audit 2023-09-23T19:32:19.372595+0000 mon.a (mon.0) 932 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T19:32:20.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:20 smithi016 bash[75004]: audit 2023-09-23T19:32:19.371921+0000 mon.c (mon.2) 143 : audit 1 from='client.? 172.21.15.16:0/1451087303' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T19:32:20.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:20 smithi016 bash[75004]: audit 2023-09-23T19:32:19.372595+0000 mon.a (mon.0) 932 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T19:32:20.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:20.536 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:32:20.637 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:20.672 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:20.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:20.675 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:32:20.776 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:20.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:20 smithi092 bash[58855]: audit 2023-09-23T19:32:19.371921+0000 mon.c (mon.2) 143 : audit 1 from='client.? 172.21.15.16:0/1451087303' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T19:32:20.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:20 smithi092 bash[58855]: audit 2023-09-23T19:32:19.372595+0000 mon.a (mon.0) 932 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T19:32:20.812 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:20.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:20.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:20.916 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:20.939 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:20.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:20.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:21.045 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:21.078 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:21.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:21.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:32:21.156 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T19:32:21.182 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:21.249 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:21.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:21.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:32:21.296 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 47 16 70280 70264 93.4219 62.4375 0.00356592 0.0106965 2023-09-23T19:32:21.353 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:21.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:21.403 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:21.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:32:21.504 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:21.513 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:21 smithi016 bash[75004]: cluster 2023-09-23T19:32:19.974013+0000 mgr.y (mgr.25325) 741 : cluster 0 pgmap v639: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T19:32:21.513 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:21 smithi016 bash[75004]: cluster 2023-09-23T19:32:20.353220+0000 osd.4 (osd.4) 51 : cluster 3 failed to encode map e124 with expected crc 2023-09-23T19:32:21.513 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:21 smithi016 bash[75004]: audit 2023-09-23T19:32:20.354750+0000 mon.a (mon.0) 933 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T19:32:21.513 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:21 smithi016 bash[75004]: cluster 2023-09-23T19:32:20.355331+0000 mon.a (mon.0) 934 : cluster 0 osdmap e124: 8 total, 8 up, 8 in 2023-09-23T19:32:21.513 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:21 smithi016 bash[75004]: cluster 2023-09-23T19:32:20.358860+0000 osd.6 (osd.6) 44 : cluster 3 failed to encode map e124 with expected crc 2023-09-23T19:32:21.513 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:21 smithi016 bash[75004]: cluster 2023-09-23T19:32:20.359647+0000 osd.5 (osd.5) 51 : cluster 3 failed to encode map e124 with expected crc 2023-09-23T19:32:21.513 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:21 smithi016 bash[75004]: cluster 2023-09-23T19:32:20.362263+0000 osd.5 (osd.5) 52 : cluster 3 failed to encode map e124 with expected crc 2023-09-23T19:32:21.513 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:21 smithi016 bash[75004]: cluster 2023-09-23T19:32:20.368735+0000 osd.7 (osd.7) 42 : cluster 3 failed to encode map e124 with expected crc 2023-09-23T19:32:21.514 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:21 smithi016 bash[77995]: cluster 2023-09-23T19:32:19.974013+0000 mgr.y (mgr.25325) 741 : cluster 0 pgmap v639: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T19:32:21.514 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:21 smithi016 bash[77995]: cluster 2023-09-23T19:32:20.353220+0000 osd.4 (osd.4) 51 : cluster 3 failed to encode map e124 with expected crc 2023-09-23T19:32:21.514 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:21 smithi016 bash[77995]: audit 2023-09-23T19:32:20.354750+0000 mon.a (mon.0) 933 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T19:32:21.514 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:21 smithi016 bash[77995]: cluster 2023-09-23T19:32:20.355331+0000 mon.a (mon.0) 934 : cluster 0 osdmap e124: 8 total, 8 up, 8 in 2023-09-23T19:32:21.514 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:21 smithi016 bash[77995]: cluster 2023-09-23T19:32:20.358860+0000 osd.6 (osd.6) 44 : cluster 3 failed to encode map e124 with expected crc 2023-09-23T19:32:21.514 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:21 smithi016 bash[77995]: cluster 2023-09-23T19:32:20.359647+0000 osd.5 (osd.5) 51 : cluster 3 failed to encode map e124 with expected crc 2023-09-23T19:32:21.514 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:21 smithi016 bash[77995]: cluster 2023-09-23T19:32:20.362263+0000 osd.5 (osd.5) 52 : cluster 3 failed to encode map e124 with expected crc 2023-09-23T19:32:21.514 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:21 smithi016 bash[77995]: cluster 2023-09-23T19:32:20.368735+0000 osd.7 (osd.7) 42 : cluster 3 failed to encode map e124 with expected crc 2023-09-23T19:32:21.535 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:21.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:21.539 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:32:21.639 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:21.650 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:21 smithi092 bash[58855]: cluster 2023-09-23T19:32:19.974013+0000 mgr.y (mgr.25325) 741 : cluster 0 pgmap v639: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T19:32:21.650 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:21 smithi092 bash[58855]: cluster 2023-09-23T19:32:20.353220+0000 osd.4 (osd.4) 51 : cluster 3 failed to encode map e124 with expected crc 2023-09-23T19:32:21.650 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:21 smithi092 bash[58855]: audit 2023-09-23T19:32:20.354750+0000 mon.a (mon.0) 933 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T19:32:21.650 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:21 smithi092 bash[58855]: cluster 2023-09-23T19:32:20.355331+0000 mon.a (mon.0) 934 : cluster 0 osdmap e124: 8 total, 8 up, 8 in 2023-09-23T19:32:21.650 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:21 smithi092 bash[58855]: cluster 2023-09-23T19:32:20.358860+0000 osd.6 (osd.6) 44 : cluster 3 failed to encode map e124 with expected crc 2023-09-23T19:32:21.650 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:21 smithi092 bash[58855]: cluster 2023-09-23T19:32:20.359647+0000 osd.5 (osd.5) 51 : cluster 3 failed to encode map e124 with expected crc 2023-09-23T19:32:21.651 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:21 smithi092 bash[58855]: cluster 2023-09-23T19:32:20.362263+0000 osd.5 (osd.5) 52 : cluster 3 failed to encode map e124 with expected crc 2023-09-23T19:32:21.651 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:21 smithi092 bash[58855]: cluster 2023-09-23T19:32:20.368735+0000 osd.7 (osd.7) 42 : cluster 3 failed to encode map e124 with expected crc 2023-09-23T19:32:21.663 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:21.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:21.666 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:32:21.767 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:21.799 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:21.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:21.803 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:32:21.903 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:21.930 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:21.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:21.932 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:32:22.033 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:22.056 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:22.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:22.059 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:32:22.159 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:22.185 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:22.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:22.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:32:22.287 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:22.297 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 48 16 71198 71182 92.6709 57.375 0.0112713 0.0107847 2023-09-23T19:32:22.317 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:22.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:22.320 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:22.421 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:22.449 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:22.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:22.453 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:32:22.553 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:22.584 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:22.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:22.586 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:32:22.687 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:22.710 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:22.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:22.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:22.814 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:22.850 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:22.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:22.853 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:32:22.954 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:22.980 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:22.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:22.985 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:23.086 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:23.120 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:23.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:23.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:32:23.224 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:23.249 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:23.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:23.252 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:32:23.297 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 49 16 72273 72257 92.1507 67.1875 0.0221342 0.010846 2023-09-23T19:32:23.369 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:23.407 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:23.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:23.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:32:23.511 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:23.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:23 smithi092 bash[58855]: cluster 2023-09-23T19:32:21.974549+0000 mgr.y (mgr.25325) 742 : cluster 0 pgmap v641: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-09-23T19:32:23.556 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:23.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:23.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:32:23.660 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:23.669 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:23 smithi016 bash[75004]: cluster 2023-09-23T19:32:21.974549+0000 mgr.y (mgr.25325) 742 : cluster 0 pgmap v641: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-09-23T19:32:23.669 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:23 smithi016 bash[77995]: cluster 2023-09-23T19:32:21.974549+0000 mgr.y (mgr.25325) 742 : cluster 0 pgmap v641: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-09-23T19:32:23.688 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:23.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:23.692 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:32:23.793 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:23.817 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:23.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:23.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:32:23.921 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:23.946 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:23.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:23.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:24.049 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:24.075 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:24.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:24.077 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:32:24.177 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:24.208 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:24.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:24.213 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:32:24.297 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 50 16 73332 73316 91.6314 66.1875 0.00919064 0.0109074 2023-09-23T19:32:24.314 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:24.340 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:24.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:24.345 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:24.381 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is unset 2023-09-23T19:32:24.446 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:24.473 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:24.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:24.481 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:32:24.582 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:24.589 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:24 smithi092 bash[58855]: audit 2023-09-23T19:32:23.659202+0000 mon.a (mon.0) 935 : audit 1 from='client.? 172.21.15.16:0/3950713190' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:32:24.615 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:24.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:24.623 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:24.647 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:24 smithi016 bash[77995]: audit 2023-09-23T19:32:23.659202+0000 mon.a (mon.0) 935 : audit 1 from='client.? 172.21.15.16:0/3950713190' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:32:24.648 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:24 smithi016 bash[75004]: audit 2023-09-23T19:32:23.659202+0000 mon.a (mon.0) 935 : audit 1 from='client.? 172.21.15.16:0/3950713190' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:32:24.696 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:32:24.724 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:24.776 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:24.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:24.780 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:32:24.881 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:24.908 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:24.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:24.911 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:32:25.011 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:25.038 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:25.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:25.041 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:25.141 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:25.183 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:25.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:25.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:32:25.287 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:25.299 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 51 16 74361 74345 91.0954 64.3125 0.0123592 0.0109715 2023-09-23T19:32:25.312 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:25.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:25.317 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:32:25.417 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:25.449 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:25.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:25.453 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:25.554 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:25.580 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:25.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:25.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:32:25.641 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:25 smithi016 bash[77995]: cluster 2023-09-23T19:32:23.975342+0000 mgr.y (mgr.25325) 743 : cluster 0 pgmap v642: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T19:32:25.642 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:25 smithi016 bash[77995]: cluster 2023-09-23T19:32:24.354654+0000 mon.a (mon.0) 936 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:32:25.642 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:25 smithi016 bash[77995]: cluster 2023-09-23T19:32:24.354685+0000 mon.a (mon.0) 937 : cluster 1 Cluster is now healthy 2023-09-23T19:32:25.642 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:25 smithi016 bash[77995]: audit 2023-09-23T19:32:24.368736+0000 mon.a (mon.0) 938 : audit 1 from='client.? 172.21.15.16:0/3950713190' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:32:25.642 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:25 smithi016 bash[77995]: cluster 2023-09-23T19:32:24.368995+0000 mon.a (mon.0) 939 : cluster 0 osdmap e125: 8 total, 8 up, 8 in 2023-09-23T19:32:25.642 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:25 smithi016 bash[77995]: cluster 2023-09-23T19:32:24.373832+0000 osd.5 (osd.5) 53 : cluster 3 failed to encode map e125 with expected crc 2023-09-23T19:32:25.642 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:25 smithi016 bash[77995]: cluster 2023-09-23T19:32:24.381868+0000 osd.7 (osd.7) 43 : cluster 3 failed to encode map e125 with expected crc 2023-09-23T19:32:25.642 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:25 smithi016 bash[77995]: cluster 2023-09-23T19:32:24.382386+0000 osd.6 (osd.6) 45 : cluster 3 failed to encode map e125 with expected crc 2023-09-23T19:32:25.642 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:25 smithi016 bash[77995]: cluster 2023-09-23T19:32:24.389178+0000 osd.4 (osd.4) 52 : cluster 3 failed to encode map e125 with expected crc 2023-09-23T19:32:25.643 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:25 smithi016 bash[75004]: cluster 2023-09-23T19:32:23.975342+0000 mgr.y (mgr.25325) 743 : cluster 0 pgmap v642: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T19:32:25.643 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:25 smithi016 bash[75004]: cluster 2023-09-23T19:32:24.354654+0000 mon.a (mon.0) 936 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:32:25.643 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:25 smithi016 bash[75004]: cluster 2023-09-23T19:32:24.354685+0000 mon.a (mon.0) 937 : cluster 1 Cluster is now healthy 2023-09-23T19:32:25.643 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:25 smithi016 bash[75004]: audit 2023-09-23T19:32:24.368736+0000 mon.a (mon.0) 938 : audit 1 from='client.? 172.21.15.16:0/3950713190' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:32:25.643 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:25 smithi016 bash[75004]: cluster 2023-09-23T19:32:24.368995+0000 mon.a (mon.0) 939 : cluster 0 osdmap e125: 8 total, 8 up, 8 in 2023-09-23T19:32:25.643 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:25 smithi016 bash[75004]: cluster 2023-09-23T19:32:24.373832+0000 osd.5 (osd.5) 53 : cluster 3 failed to encode map e125 with expected crc 2023-09-23T19:32:25.643 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:25 smithi016 bash[75004]: cluster 2023-09-23T19:32:24.381868+0000 osd.7 (osd.7) 43 : cluster 3 failed to encode map e125 with expected crc 2023-09-23T19:32:25.643 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:25 smithi016 bash[75004]: cluster 2023-09-23T19:32:24.382386+0000 osd.6 (osd.6) 45 : cluster 3 failed to encode map e125 with expected crc 2023-09-23T19:32:25.643 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:25 smithi016 bash[75004]: cluster 2023-09-23T19:32:24.389178+0000 osd.4 (osd.4) 52 : cluster 3 failed to encode map e125 with expected crc 2023-09-23T19:32:25.643 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:32:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:32:25] "GET /metrics HTTP/1.1" 200 36239 "" "Prometheus/2.33.4" 2023-09-23T19:32:25.685 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:25.692 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:25 smithi092 bash[58855]: cluster 2023-09-23T19:32:23.975342+0000 mgr.y (mgr.25325) 743 : cluster 0 pgmap v642: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T19:32:25.693 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:25 smithi092 bash[58855]: cluster 2023-09-23T19:32:24.354654+0000 mon.a (mon.0) 936 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:32:25.693 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:25 smithi092 bash[58855]: cluster 2023-09-23T19:32:24.354685+0000 mon.a (mon.0) 937 : cluster 1 Cluster is now healthy 2023-09-23T19:32:25.693 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:25 smithi092 bash[58855]: audit 2023-09-23T19:32:24.368736+0000 mon.a (mon.0) 938 : audit 1 from='client.? 172.21.15.16:0/3950713190' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:32:25.693 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:25 smithi092 bash[58855]: cluster 2023-09-23T19:32:24.368995+0000 mon.a (mon.0) 939 : cluster 0 osdmap e125: 8 total, 8 up, 8 in 2023-09-23T19:32:25.693 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:25 smithi092 bash[58855]: cluster 2023-09-23T19:32:24.373832+0000 osd.5 (osd.5) 53 : cluster 3 failed to encode map e125 with expected crc 2023-09-23T19:32:25.693 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:25 smithi092 bash[58855]: cluster 2023-09-23T19:32:24.381868+0000 osd.7 (osd.7) 43 : cluster 3 failed to encode map e125 with expected crc 2023-09-23T19:32:25.693 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:25 smithi092 bash[58855]: cluster 2023-09-23T19:32:24.382386+0000 osd.6 (osd.6) 45 : cluster 3 failed to encode map e125 with expected crc 2023-09-23T19:32:25.693 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:25 smithi092 bash[58855]: cluster 2023-09-23T19:32:24.389178+0000 osd.4 (osd.4) 52 : cluster 3 failed to encode map e125 with expected crc 2023-09-23T19:32:25.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:25.722 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:25.722 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:32:25.823 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:25.848 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:25.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:25.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:25.954 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:25.987 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:25.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:25.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:32:26.092 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:26.123 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:26.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:26.128 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:26.228 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:26.261 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:26.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:26.265 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:32:26.297 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 52 16 75434 75418 90.6331 67.0625 0.0110557 0.0110276 2023-09-23T19:32:26.365 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:26.391 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:26.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:26.396 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:32:26.496 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:26.528 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:26.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:26.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:32:26.633 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:26.664 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:26.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:26.672 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:32:26.773 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:26.801 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:26.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:26.807 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:32:26.908 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:26.941 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:26.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:26.945 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:27.045 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:27.077 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:27.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:27.081 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:32:27.182 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:27.206 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:27.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:27.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:32:27.297 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 53 16 76486 76470 90.1635 65.75 0.00826368 0.0110856 2023-09-23T19:32:27.312 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:27.336 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:27.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:27.341 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:32:27.372 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:27 smithi016 bash[75004]: cluster 2023-09-23T19:32:25.976642+0000 mgr.y (mgr.25325) 744 : cluster 0 pgmap v644: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-09-23T19:32:27.372 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:27 smithi016 bash[77995]: cluster 2023-09-23T19:32:25.976642+0000 mgr.y (mgr.25325) 744 : cluster 0 pgmap v644: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-09-23T19:32:27.442 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:27.468 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:27.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:27.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:27.572 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:27.598 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:27.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:27.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:32:27.704 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:27.714 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:27 smithi092 bash[58855]: cluster 2023-09-23T19:32:25.976642+0000 mgr.y (mgr.25325) 744 : cluster 0 pgmap v644: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-09-23T19:32:27.740 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T19:32:27.741 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T19:32:27.741 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T19:32:27.741 INFO:tasks.thrashosds.thrasher:Setting osd 3 primary_affinity to 0.097720 2023-09-23T19:32:27.741 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 3 0.09772045968511078 2023-09-23T19:32:27.742 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:27.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:27.746 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:32:27.847 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:27.893 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:27.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:27.899 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:32:27.999 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:28.024 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:28.027 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T19:32:28.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:28.029 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:32:28.129 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:28.165 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:28.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:28.170 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:32:28.271 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:28.297 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 54 16 77452 77436 89.6118 60.375 0.0148831 0.0111523 2023-09-23T19:32:28.309 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:28.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:28.313 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:32:28.414 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:28.443 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:28.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:28.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:32:28.540 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:32:28.548 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:28.578 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:28.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:28.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:28.683 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:28.709 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:28.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:28.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:32:28.814 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:28.841 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:28.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:28.843 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:32:28.944 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:28.972 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:28.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:28.976 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:32:29.077 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:29.102 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:29.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:29.107 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:32:29.208 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:29.237 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:29.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:29.245 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:32:29.298 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 55 16 78463 78447 89.1312 63.1875 0.00994874 0.0112136 2023-09-23T19:32:29.346 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:29.377 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:29.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:29.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:32:29.484 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:29.512 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:29.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:29.514 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:32:29.615 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:29.644 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:29.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:29.648 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:32:29.749 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:29.758 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:29 smithi016 bash[75004]: cluster 2023-09-23T19:32:27.977198+0000 mgr.y (mgr.25325) 745 : cluster 0 pgmap v645: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T19:32:29.758 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:29 smithi016 bash[75004]: audit 2023-09-23T19:32:28.642153+0000 mon.a (mon.0) 940 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:32:29.758 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:29 smithi016 bash[75004]: audit 2023-09-23T19:32:28.643615+0000 mon.a (mon.0) 941 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:32:29.758 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:29 smithi016 bash[77995]: cluster 2023-09-23T19:32:27.977198+0000 mgr.y (mgr.25325) 745 : cluster 0 pgmap v645: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T19:32:29.758 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:29 smithi016 bash[77995]: audit 2023-09-23T19:32:28.642153+0000 mon.a (mon.0) 940 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:32:29.758 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:29 smithi016 bash[77995]: audit 2023-09-23T19:32:28.643615+0000 mon.a (mon.0) 941 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:32:29.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:29.784 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:29.784 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:32:29.884 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:29.909 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:29.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:29.913 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:32:30.014 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:30.023 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:29 smithi092 bash[58855]: cluster 2023-09-23T19:32:27.977198+0000 mgr.y (mgr.25325) 745 : cluster 0 pgmap v645: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T19:32:30.023 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:29 smithi092 bash[58855]: audit 2023-09-23T19:32:28.642153+0000 mon.a (mon.0) 940 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:32:30.023 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:29 smithi092 bash[58855]: audit 2023-09-23T19:32:28.643615+0000 mon.a (mon.0) 941 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:32:30.049 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:30.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:30.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:32:30.153 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:30.188 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:30.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:30.194 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:32:30.295 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:30.297 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 56 16 79474 79458 88.6679 63.1875 0.0117434 0.0112712 2023-09-23T19:32:30.330 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:30.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:30.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:32:30.436 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:30.465 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:30.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:30.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:30.568 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:30.582 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:32:30.596 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:30.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:30.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:30.699 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:30.733 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:30.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:30.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:32:30.836 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:30.863 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:30.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:30.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:30.966 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:31.000 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:31.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:31.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:32:31.102 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:31.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:31.134 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:31.134 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:32:31.235 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:31.266 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:31.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:31.269 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:32:31.298 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 57 16 80478 80462 88.2131 62.75 0.0122053 0.01133 2023-09-23T19:32:31.369 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:31.395 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:31.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:31.399 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:32:31.500 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:31.528 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:31.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:31.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:32:31.632 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:31.655 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:31.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:31.658 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:32:31.759 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:31.800 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:31.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:31.803 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:32:31.904 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:31.912 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:31 smithi016 bash[77995]: cluster 2023-09-23T19:32:29.977868+0000 mgr.y (mgr.25325) 746 : cluster 0 pgmap v646: 89 pgs: 89 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s 2023-09-23T19:32:31.912 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:31 smithi016 bash[75004]: cluster 2023-09-23T19:32:29.977868+0000 mgr.y (mgr.25325) 746 : cluster 0 pgmap v646: 89 pgs: 89 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s 2023-09-23T19:32:31.932 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:31.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:31.937 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:32:32.037 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:32.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:31 smithi092 bash[58855]: cluster 2023-09-23T19:32:29.977868+0000 mgr.y (mgr.25325) 746 : cluster 0 pgmap v646: 89 pgs: 89 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s 2023-09-23T19:32:32.072 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:32.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:32.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:32.179 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:32.208 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:32.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:32.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:32:32.298 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 58 16 81399 81383 87.6846 57.5625 0.0134649 0.0113989 2023-09-23T19:32:32.315 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:32.347 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:32.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:32.352 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:32:32.453 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:32.481 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:32.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:32.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:32:32.585 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:32.607 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:32.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:32.609 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:32.710 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:32.735 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:32.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:32.737 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:32:32.838 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:32.857 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:32.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:32.859 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:32:32.960 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:32.983 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:32.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:32.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:32:33.089 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:33.120 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:33.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:33.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:32:33.223 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:33.255 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:33.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:33.258 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:32:33.298 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 59 16 82089 82073 86.9293 43.125 0.0137174 0.0114979 2023-09-23T19:32:33.359 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:33.387 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:33.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:33.390 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:32:33.490 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:33.529 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:33.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:33.532 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:33.633 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:33.674 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:33.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:33.684 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:33.785 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:33.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:33 smithi092 bash[58855]: cluster 2023-09-23T19:32:31.978484+0000 mgr.y (mgr.25325) 747 : cluster 0 pgmap v647: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-09-23T19:32:33.820 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:33.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:33.824 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:32:33.914 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:33 smithi016 bash[77995]: cluster 2023-09-23T19:32:31.978484+0000 mgr.y (mgr.25325) 747 : cluster 0 pgmap v647: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-09-23T19:32:33.914 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:33 smithi016 bash[75004]: cluster 2023-09-23T19:32:31.978484+0000 mgr.y (mgr.25325) 747 : cluster 0 pgmap v647: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-09-23T19:32:33.926 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:33.952 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:33.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:33.954 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:32:34.055 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:34.085 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:34.086 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:32:34.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:34.088 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:32:34.090 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:32:34.188 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:34.218 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:34.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:34.221 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:34.298 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-09-23T19:32:34.296176+0000 min lat: 0.00139623 max lat: 0.221111 avg lat: 0.0115555 2023-09-23T19:32:34.298 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:32:34.298 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 60 16 83062 83046 86.494 60.8125 0.0300402 0.0115555 2023-09-23T19:32:34.321 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:34.356 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:34.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:34.361 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:34.461 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:34.492 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:34.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:34.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:34.595 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:34.622 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:34.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:34.624 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:32:34.725 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:34.747 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:34.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:34.750 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:32:34.850 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:34.881 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:34.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:34.885 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:34.985 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:35.010 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:35.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:35.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:32:35.113 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:35.143 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:35.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:35.145 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:32:35.246 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:35.278 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:35.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:35.281 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:35.298 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 61 16 84018 84002 86.0555 59.75 0.0145721 0.0116143 2023-09-23T19:32:35.382 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:35.409 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:35.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:35.416 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:32:35.517 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:35.558 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:35.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:35.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:32:35.660 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:35.693 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:35.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:35.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:32:35.798 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:35.827 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:35.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:35.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:32:35.931 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:35.961 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:35.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:35.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:32:36.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:35 smithi092 bash[58855]: cluster 2023-09-23T19:32:33.979023+0000 mgr.y (mgr.25325) 748 : cluster 0 pgmap v648: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-09-23T19:32:36.055 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:35 smithi016 bash[75004]: cluster 2023-09-23T19:32:33.979023+0000 mgr.y (mgr.25325) 748 : cluster 0 pgmap v648: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-09-23T19:32:36.056 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:32:35 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:32:35] "GET /metrics HTTP/1.1" 200 36235 "" "Prometheus/2.33.4" 2023-09-23T19:32:36.056 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:35 smithi016 bash[77995]: cluster 2023-09-23T19:32:33.979023+0000 mgr.y (mgr.25325) 748 : cluster 0 pgmap v648: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-09-23T19:32:36.065 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:36.102 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:36.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:36.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:36.207 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:36.235 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:36.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:36.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:32:36.298 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 62 16 85004 84988 85.6614 61.625 0.0101452 0.0116689 2023-09-23T19:32:36.338 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:36.367 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:36.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:36.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:32:36.475 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:36.508 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:36.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:36.511 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:32:36.611 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:36.638 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:36.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:36.643 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:32:36.744 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:36.775 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:36.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:36.778 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:32:36.879 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:36.914 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:36.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:36.918 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:37.019 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:37.059 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:37.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:37.064 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:32:37.165 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:37.191 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:37.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:37.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:32:37.294 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:37.298 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 63 16 86019 86003 85.3085 63.4375 0.0203729 0.0117158 2023-09-23T19:32:37.317 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:37.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:37.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:32:37.420 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:37.447 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:37.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:37.452 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:32:37.553 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:37.577 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:37.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:37.581 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:37.682 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:37.737 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:37.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:37.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:32:37.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:37 smithi016 bash[75004]: cluster 2023-09-23T19:32:35.979729+0000 mgr.y (mgr.25325) 749 : cluster 0 pgmap v649: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-09-23T19:32:37.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:37 smithi016 bash[77995]: cluster 2023-09-23T19:32:35.979729+0000 mgr.y (mgr.25325) 749 : cluster 0 pgmap v649: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-09-23T19:32:37.843 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:37.870 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:37.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:37.873 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:32:37.973 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:37.983 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:37 smithi092 bash[58855]: cluster 2023-09-23T19:32:35.979729+0000 mgr.y (mgr.25325) 749 : cluster 0 pgmap v649: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-09-23T19:32:38.004 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:38.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:38.008 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:38.109 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:38.144 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:38.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:38.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:32:38.248 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:38.284 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:38.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:38.289 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:38.298 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 64 16 86939 86923 84.874 57.5 0.0157963 0.0117759 2023-09-23T19:32:38.390 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:38.422 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:38.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:38.427 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:32:38.528 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:38.559 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:38.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:38.562 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:38.662 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:38.694 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:38.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:38.696 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:38.797 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:38.834 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:38.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:38.840 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:38.941 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:38.964 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:38.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:38.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:32:39.068 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:39.094 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:39.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:39.098 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:32:39.198 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:39.231 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:39.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:39.233 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:39.299 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 65 16 87704 87688 84.3038 47.8125 0.00751039 0.0118559 2023-09-23T19:32:39.334 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:39.368 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:39.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:39.370 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:32:39.471 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:39.509 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:39.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:39.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:32:39.611 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:39.646 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:39.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:39.652 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:32:39.752 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:39.773 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:39.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:39.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:39.811 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:39 smithi016 bash[75004]: cluster 2023-09-23T19:32:37.980336+0000 mgr.y (mgr.25325) 750 : cluster 0 pgmap v650: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 864 op/s 2023-09-23T19:32:39.811 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:39 smithi016 bash[77995]: cluster 2023-09-23T19:32:37.980336+0000 mgr.y (mgr.25325) 750 : cluster 0 pgmap v650: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 864 op/s 2023-09-23T19:32:39.877 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:39.905 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:39.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:39.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:40.012 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:40.020 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:39 smithi092 bash[58855]: cluster 2023-09-23T19:32:37.980336+0000 mgr.y (mgr.25325) 750 : cluster 0 pgmap v650: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 864 op/s 2023-09-23T19:32:40.038 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:40.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:40.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:32:40.143 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:40.181 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:40.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:40.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:32:40.288 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:40.298 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 66 16 88521 88505 83.8 51.0625 0.0235741 0.0119266 2023-09-23T19:32:40.317 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:40.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:40.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:32:40.424 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:40.451 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:40.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:40.453 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:32:40.554 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:40.590 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:40.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:40.592 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:32:40.693 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:40.724 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:40.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:40.728 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:32:40.829 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:40.853 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:40.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:40.858 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:40.959 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:40.992 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:40.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:40.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:32:41.098 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:41.127 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:41.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:41.130 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:32:41.231 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:41.258 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:41.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:41.261 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:32:41.299 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 67 16 89439 89423 83.4056 57.375 0.0119748 0.0119837 2023-09-23T19:32:41.362 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:41.394 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:41.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:41.396 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:32:41.497 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:41.535 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:41.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:41.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:41.578 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-09-23T19:32:41.578 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [], 2023-09-23T19:32:41.579 INFO:teuthology.orchestra.run.smithi016.stdout: "complaint_time": 30, 2023-09-23T19:32:41.579 INFO:teuthology.orchestra.run.smithi016.stdout: "num_blocked_ops": 0 2023-09-23T19:32:41.579 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-09-23T19:32:41.640 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:41.676 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:41.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:41.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:32:41.730 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:32:41.750 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:32:41.782 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:41.784 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:32:41.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:41 smithi092 bash[58855]: cluster 2023-09-23T19:32:39.980959+0000 mgr.y (mgr.25325) 751 : cluster 0 pgmap v651: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 955 op/s 2023-09-23T19:32:41.813 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:41.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:41.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:41.822 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:32:41.858 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:32:41.906 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:32:41.917 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:41.926 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:41 smithi016 bash[75004]: cluster 2023-09-23T19:32:39.980959+0000 mgr.y (mgr.25325) 751 : cluster 0 pgmap v651: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 955 op/s 2023-09-23T19:32:41.926 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:41 smithi016 bash[77995]: cluster 2023-09-23T19:32:39.980959+0000 mgr.y (mgr.25325) 751 : cluster 0 pgmap v651: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 955 op/s 2023-09-23T19:32:41.944 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:41.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:41.946 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:32:41.948 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:32:41.986 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:32:41.990 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:32:41.990 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:32:41.991 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:32:41.991 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:32:41.991 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:32:41.991 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:32:41.991 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:32:41.991 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:32:42.048 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:42.167 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:42.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:42.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:32:42.270 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:42.299 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 68 16 90346 90330 83.0126 56.6875 0.0195981 0.01204 2023-09-23T19:32:42.305 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:42.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:42.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:32:42.411 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:42.433 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:42.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:42.437 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:42.538 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:42.563 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:42.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:42.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:32:42.667 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:42.691 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:42 smithi092 bash[58855]: audit 2023-09-23T19:32:41.682624+0000 mon.a (mon.0) 942 : audit 0 from='client.? 172.21.15.16:0/1151932622' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:32:42.696 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:42.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:42.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:32:42.800 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:42.822 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:42.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:42.826 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:32:42.872 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:42 smithi016 bash[75004]: audit 2023-09-23T19:32:41.682624+0000 mon.a (mon.0) 942 : audit 0 from='client.? 172.21.15.16:0/1151932622' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:32:42.872 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:42 smithi016 bash[77995]: audit 2023-09-23T19:32:41.682624+0000 mon.a (mon.0) 942 : audit 0 from='client.? 172.21.15.16:0/1151932622' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:32:42.927 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:42.966 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:42.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:42.969 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:32:43.069 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:43.095 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:43.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:43.100 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:43.201 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:43.240 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:43.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:43.244 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:43.299 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 69 16 91259 91243 82.6364 57.0625 0.0162819 0.0120955 2023-09-23T19:32:43.344 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:43.369 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:43.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:43.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:43.477 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:43.513 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:43.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:43.518 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:32:43.618 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:43.651 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:43.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:43.657 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:43.758 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:43.766 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:43 smithi016 bash[77995]: cluster 2023-09-23T19:32:41.981600+0000 mgr.y (mgr.25325) 752 : cluster 0 pgmap v652: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-09-23T19:32:43.767 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:43 smithi016 bash[77995]: audit 2023-09-23T19:32:43.637973+0000 mon.a (mon.0) 943 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:32:43.767 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:43 smithi016 bash[75004]: cluster 2023-09-23T19:32:41.981600+0000 mgr.y (mgr.25325) 752 : cluster 0 pgmap v652: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-09-23T19:32:43.767 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:43 smithi016 bash[75004]: audit 2023-09-23T19:32:43.637973+0000 mon.a (mon.0) 943 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:32:43.779 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:43.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:43.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:32:43.882 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:43.902 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:43 smithi092 bash[58855]: cluster 2023-09-23T19:32:41.981600+0000 mgr.y (mgr.25325) 752 : cluster 0 pgmap v652: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-09-23T19:32:43.902 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:43 smithi092 bash[58855]: audit 2023-09-23T19:32:43.637973+0000 mon.a (mon.0) 943 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:32:43.922 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:43.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:43.924 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:32:44.025 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:44.066 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:44.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:44.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:44.170 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:44.221 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:44.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:44.223 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:32:44.299 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 70 16 92147 92131 82.2487 55.5 0.0113813 0.0121528 2023-09-23T19:32:44.324 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:44.368 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:44.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:44.376 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:32:44.400 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T19:32:44.477 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:44.557 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:44.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:44.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:32:44.662 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:44.696 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:44.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:44.698 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:44.799 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:44.823 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:44.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:44.828 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:44.928 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:44.952 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:44.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:44.955 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:45.055 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:45.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:45.085 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:45.086 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:32:45.186 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:45.210 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:45.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:45.213 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:32:45.299 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 71 16 93032 93016 81.8693 55.3125 0.0201255 0.0122084 2023-09-23T19:32:45.313 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:45.437 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:45.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:45.441 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:32:45.541 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:45.574 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:45.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:45.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:45.678 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:45.702 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:45 smithi016 bash[75004]: cluster 2023-09-23T19:32:43.982107+0000 mgr.y (mgr.25325) 753 : cluster 0 pgmap v653: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 825 op/s 2023-09-23T19:32:45.703 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:32:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:32:45] "GET /metrics HTTP/1.1" 200 36237 "" "Prometheus/2.33.4" 2023-09-23T19:32:45.703 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:45 smithi016 bash[77995]: cluster 2023-09-23T19:32:43.982107+0000 mgr.y (mgr.25325) 753 : cluster 0 pgmap v653: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 825 op/s 2023-09-23T19:32:45.703 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:45.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:45.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:32:45.809 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:45.817 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:45 smithi092 bash[58855]: cluster 2023-09-23T19:32:43.982107+0000 mgr.y (mgr.25325) 753 : cluster 0 pgmap v653: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 825 op/s 2023-09-23T19:32:45.845 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:45.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:45.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:32:45.954 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:45.981 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:45.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:45.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:32:46.087 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:46.212 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:46.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:46.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:32:46.299 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 72 16 94006 93990 81.5776 60.875 0.0112574 0.0122524 2023-09-23T19:32:46.318 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:46.355 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:46.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:46.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:46.417 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:32:46.459 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:46.488 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:46.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:46.490 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:32:46.591 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:46.622 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:46.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:46.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:46.725 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:46.756 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:46.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:46.761 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:46.862 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:46.884 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:46.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:46.887 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:32:46.987 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:47.012 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:47.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:47.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:32:47.116 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:47.149 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:47.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:47.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:32:47.252 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:47.276 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:47.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:47.278 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:32:47.299 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 73 16 94926 94910 81.2477 57.5 0.0170157 0.0123022 2023-09-23T19:32:47.382 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:47.405 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:47.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:47.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:32:47.510 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:47.544 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:47.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:47.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:32:47.648 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:47.674 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:47.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:47.676 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:47.777 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:47.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:47 smithi016 bash[75004]: cluster 2023-09-23T19:32:45.982679+0000 mgr.y (mgr.25325) 754 : cluster 0 pgmap v654: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 57 MiB/s wr, 919 op/s 2023-09-23T19:32:47.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:47 smithi016 bash[77995]: cluster 2023-09-23T19:32:45.982679+0000 mgr.y (mgr.25325) 754 : cluster 0 pgmap v654: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 57 MiB/s wr, 919 op/s 2023-09-23T19:32:47.824 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:47.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:47.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:32:47.930 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:47.937 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:47 smithi092 bash[58855]: cluster 2023-09-23T19:32:45.982679+0000 mgr.y (mgr.25325) 754 : cluster 0 pgmap v654: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 57 MiB/s wr, 919 op/s 2023-09-23T19:32:47.958 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:47.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:47.961 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:48.062 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:48.087 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:48.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:48.091 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:48.192 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:48.223 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:48.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:48.228 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:48.299 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 74 16 95759 95743 80.8533 52.0625 0.00996422 0.0123618 2023-09-23T19:32:48.329 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:48.354 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:48.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:48.361 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:32:48.462 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:48.490 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:48.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:48.493 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:32:48.593 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:48.620 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:48.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:48.629 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:32:48.725 INFO:teuthology.orchestra.run.smithi016.stderr:set osd.3 primary-affinity to 0.0977205 (864042) 2023-09-23T19:32:48.725 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is set 2023-09-23T19:32:48.729 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:48.757 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:48.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:48.760 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:32:48.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:48 smithi092 bash[58855]: audit 2023-09-23T19:32:47.978894+0000 mon.a (mon.0) 944 : audit 1 from='client.? 172.21.15.16:0/2931296256' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.09772045968511078}]: dispatch 2023-09-23T19:32:48.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:48 smithi092 bash[58855]: audit 2023-09-23T19:32:48.008678+0000 mon.a (mon.0) 945 : audit 1 from='client.? 172.21.15.16:0/3372162570' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:32:48.861 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:48.908 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:48.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:48.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:32:49.011 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:49.024 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:48 smithi016 bash[75004]: audit 2023-09-23T19:32:47.978894+0000 mon.a (mon.0) 944 : audit 1 from='client.? 172.21.15.16:0/2931296256' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.09772045968511078}]: dispatch 2023-09-23T19:32:49.024 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:48 smithi016 bash[75004]: audit 2023-09-23T19:32:48.008678+0000 mon.a (mon.0) 945 : audit 1 from='client.? 172.21.15.16:0/3372162570' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:32:49.024 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:48 smithi016 bash[77995]: audit 2023-09-23T19:32:47.978894+0000 mon.a (mon.0) 944 : audit 1 from='client.? 172.21.15.16:0/2931296256' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.09772045968511078}]: dispatch 2023-09-23T19:32:49.025 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:48 smithi016 bash[77995]: audit 2023-09-23T19:32:48.008678+0000 mon.a (mon.0) 945 : audit 1 from='client.? 172.21.15.16:0/3372162570' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:32:49.040 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:49.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:49.042 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:32:49.143 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:49.176 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:49.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:49.179 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:49.280 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:49.299 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 75 16 96201 96185 80.1435 27.625 0.00530418 0.0123853 2023-09-23T19:32:49.315 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:49.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:49.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:32:49.418 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:49.448 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:49.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:49.451 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:32:49.552 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:49.583 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:49.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:49.586 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:49.687 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:49.721 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:49.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:49.726 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:32:49.827 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:49.836 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:49 smithi092 bash[58855]: cluster 2023-09-23T19:32:47.983221+0000 mgr.y (mgr.25325) 755 : cluster 0 pgmap v655: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 890 op/s 2023-09-23T19:32:49.836 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:49 smithi092 bash[58855]: cluster 2023-09-23T19:32:48.702230+0000 mon.a (mon.0) 946 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:32:49.836 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:49 smithi092 bash[58855]: audit 2023-09-23T19:32:48.714617+0000 mon.a (mon.0) 947 : audit 1 from='client.? 172.21.15.16:0/2931296256' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.09772045968511078}]': finished 2023-09-23T19:32:49.837 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:49 smithi092 bash[58855]: audit 2023-09-23T19:32:48.714837+0000 mon.a (mon.0) 948 : audit 1 from='client.? 172.21.15.16:0/3372162570' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:32:49.837 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:49 smithi092 bash[58855]: cluster 2023-09-23T19:32:48.715399+0000 mon.a (mon.0) 949 : cluster 0 osdmap e126: 8 total, 8 up, 8 in 2023-09-23T19:32:49.837 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:49 smithi092 bash[58855]: cluster 2023-09-23T19:32:48.719183+0000 osd.5 (osd.5) 54 : cluster 3 failed to encode map e126 with expected crc 2023-09-23T19:32:49.837 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:49 smithi092 bash[58855]: cluster 2023-09-23T19:32:48.723532+0000 osd.7 (osd.7) 44 : cluster 3 failed to encode map e126 with expected crc 2023-09-23T19:32:49.837 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:49 smithi092 bash[58855]: cluster 2023-09-23T19:32:48.728127+0000 osd.4 (osd.4) 53 : cluster 3 failed to encode map e126 with expected crc 2023-09-23T19:32:49.867 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:49.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:49.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:49.973 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:49.982 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:49 smithi016 bash[75004]: cluster 2023-09-23T19:32:47.983221+0000 mgr.y (mgr.25325) 755 : cluster 0 pgmap v655: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 890 op/s 2023-09-23T19:32:49.983 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:49 smithi016 bash[75004]: cluster 2023-09-23T19:32:48.702230+0000 mon.a (mon.0) 946 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:32:49.983 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:49 smithi016 bash[75004]: audit 2023-09-23T19:32:48.714617+0000 mon.a (mon.0) 947 : audit 1 from='client.? 172.21.15.16:0/2931296256' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.09772045968511078}]': finished 2023-09-23T19:32:49.983 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:49 smithi016 bash[75004]: audit 2023-09-23T19:32:48.714837+0000 mon.a (mon.0) 948 : audit 1 from='client.? 172.21.15.16:0/3372162570' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:32:49.983 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:49 smithi016 bash[75004]: cluster 2023-09-23T19:32:48.715399+0000 mon.a (mon.0) 949 : cluster 0 osdmap e126: 8 total, 8 up, 8 in 2023-09-23T19:32:49.983 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:49 smithi016 bash[75004]: cluster 2023-09-23T19:32:48.719183+0000 osd.5 (osd.5) 54 : cluster 3 failed to encode map e126 with expected crc 2023-09-23T19:32:49.983 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:49 smithi016 bash[75004]: cluster 2023-09-23T19:32:48.723532+0000 osd.7 (osd.7) 44 : cluster 3 failed to encode map e126 with expected crc 2023-09-23T19:32:49.983 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:49 smithi016 bash[75004]: cluster 2023-09-23T19:32:48.728127+0000 osd.4 (osd.4) 53 : cluster 3 failed to encode map e126 with expected crc 2023-09-23T19:32:49.983 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:49 smithi016 bash[77995]: cluster 2023-09-23T19:32:47.983221+0000 mgr.y (mgr.25325) 755 : cluster 0 pgmap v655: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 890 op/s 2023-09-23T19:32:49.983 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:49 smithi016 bash[77995]: cluster 2023-09-23T19:32:48.702230+0000 mon.a (mon.0) 946 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:32:49.984 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:49 smithi016 bash[77995]: audit 2023-09-23T19:32:48.714617+0000 mon.a (mon.0) 947 : audit 1 from='client.? 172.21.15.16:0/2931296256' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.09772045968511078}]': finished 2023-09-23T19:32:49.984 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:49 smithi016 bash[77995]: audit 2023-09-23T19:32:48.714837+0000 mon.a (mon.0) 948 : audit 1 from='client.? 172.21.15.16:0/3372162570' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:32:49.984 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:49 smithi016 bash[77995]: cluster 2023-09-23T19:32:48.715399+0000 mon.a (mon.0) 949 : cluster 0 osdmap e126: 8 total, 8 up, 8 in 2023-09-23T19:32:49.984 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:49 smithi016 bash[77995]: cluster 2023-09-23T19:32:48.719183+0000 osd.5 (osd.5) 54 : cluster 3 failed to encode map e126 with expected crc 2023-09-23T19:32:49.984 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:49 smithi016 bash[77995]: cluster 2023-09-23T19:32:48.723532+0000 osd.7 (osd.7) 44 : cluster 3 failed to encode map e126 with expected crc 2023-09-23T19:32:49.984 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:49 smithi016 bash[77995]: cluster 2023-09-23T19:32:48.728127+0000 osd.4 (osd.4) 53 : cluster 3 failed to encode map e126 with expected crc 2023-09-23T19:32:50.013 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:50.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:50.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:32:50.120 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:50.159 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:50.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:50.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:32:50.266 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:50.299 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:50.300 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 76 16 96662 96646 79.468 28.8125 0.0272373 0.0125776 2023-09-23T19:32:50.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:50.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:50.403 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:50.431 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:50.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:50.434 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:32:50.535 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:50.563 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:50.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:50.570 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:32:50.671 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:50.700 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:50.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:50.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:32:50.738 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:32:50.807 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:50.818 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:50 smithi016 bash[77995]: cluster 2023-09-23T19:32:49.715947+0000 mon.a (mon.0) 950 : cluster 0 osdmap e127: 8 total, 8 up, 8 in 2023-09-23T19:32:50.818 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:50 smithi016 bash[77995]: cluster 2023-09-23T19:32:49.718289+0000 osd.5 (osd.5) 55 : cluster 3 failed to encode map e127 with expected crc 2023-09-23T19:32:50.818 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:50 smithi016 bash[77995]: cluster 2023-09-23T19:32:49.720556+0000 osd.6 (osd.6) 46 : cluster 3 failed to encode map e127 with expected crc 2023-09-23T19:32:50.818 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:50 smithi016 bash[77995]: cluster 2023-09-23T19:32:49.720577+0000 osd.7 (osd.7) 45 : cluster 3 failed to encode map e127 with expected crc 2023-09-23T19:32:50.818 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:50 smithi016 bash[77995]: cluster 2023-09-23T19:32:49.720702+0000 osd.4 (osd.4) 54 : cluster 3 failed to encode map e127 with expected crc 2023-09-23T19:32:50.818 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:50 smithi016 bash[77995]: cluster 2023-09-23T19:32:49.721263+0000 osd.5 (osd.5) 56 : cluster 3 failed to encode map e127 with expected crc 2023-09-23T19:32:50.818 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:50 smithi016 bash[77995]: cluster 2023-09-23T19:32:49.723298+0000 osd.5 (osd.5) 57 : cluster 3 failed to encode map e127 with expected crc 2023-09-23T19:32:50.818 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:50 smithi016 bash[77995]: cluster 2023-09-23T19:32:49.723447+0000 osd.7 (osd.7) 46 : cluster 3 failed to encode map e127 with expected crc 2023-09-23T19:32:50.818 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:50 smithi016 bash[77995]: cluster 2023-09-23T19:32:49.724129+0000 osd.4 (osd.4) 55 : cluster 3 failed to encode map e127 with expected crc 2023-09-23T19:32:50.819 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:50 smithi016 bash[77995]: cluster 2023-09-23T19:32:49.724131+0000 osd.6 (osd.6) 47 : cluster 3 failed to encode map e127 with expected crc 2023-09-23T19:32:50.819 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:50 smithi016 bash[75004]: cluster 2023-09-23T19:32:49.715947+0000 mon.a (mon.0) 950 : cluster 0 osdmap e127: 8 total, 8 up, 8 in 2023-09-23T19:32:50.819 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:50 smithi016 bash[75004]: cluster 2023-09-23T19:32:49.718289+0000 osd.5 (osd.5) 55 : cluster 3 failed to encode map e127 with expected crc 2023-09-23T19:32:50.819 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:50 smithi016 bash[75004]: cluster 2023-09-23T19:32:49.720556+0000 osd.6 (osd.6) 46 : cluster 3 failed to encode map e127 with expected crc 2023-09-23T19:32:50.819 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:50 smithi016 bash[75004]: cluster 2023-09-23T19:32:49.720577+0000 osd.7 (osd.7) 45 : cluster 3 failed to encode map e127 with expected crc 2023-09-23T19:32:50.819 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:50 smithi016 bash[75004]: cluster 2023-09-23T19:32:49.720702+0000 osd.4 (osd.4) 54 : cluster 3 failed to encode map e127 with expected crc 2023-09-23T19:32:50.819 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:50 smithi016 bash[75004]: cluster 2023-09-23T19:32:49.721263+0000 osd.5 (osd.5) 56 : cluster 3 failed to encode map e127 with expected crc 2023-09-23T19:32:50.819 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:50 smithi016 bash[75004]: cluster 2023-09-23T19:32:49.723298+0000 osd.5 (osd.5) 57 : cluster 3 failed to encode map e127 with expected crc 2023-09-23T19:32:50.819 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:50 smithi016 bash[75004]: cluster 2023-09-23T19:32:49.723447+0000 osd.7 (osd.7) 46 : cluster 3 failed to encode map e127 with expected crc 2023-09-23T19:32:50.819 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:50 smithi016 bash[75004]: cluster 2023-09-23T19:32:49.724129+0000 osd.4 (osd.4) 55 : cluster 3 failed to encode map e127 with expected crc 2023-09-23T19:32:50.820 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:50 smithi016 bash[75004]: cluster 2023-09-23T19:32:49.724131+0000 osd.6 (osd.6) 47 : cluster 3 failed to encode map e127 with expected crc 2023-09-23T19:32:50.842 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:50.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:50.844 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:32:50.945 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:50.972 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:50.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:50.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:32:51.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:50 smithi092 bash[58855]: cluster 2023-09-23T19:32:49.715947+0000 mon.a (mon.0) 950 : cluster 0 osdmap e127: 8 total, 8 up, 8 in 2023-09-23T19:32:51.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:50 smithi092 bash[58855]: cluster 2023-09-23T19:32:49.718289+0000 osd.5 (osd.5) 55 : cluster 3 failed to encode map e127 with expected crc 2023-09-23T19:32:51.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:50 smithi092 bash[58855]: cluster 2023-09-23T19:32:49.720556+0000 osd.6 (osd.6) 46 : cluster 3 failed to encode map e127 with expected crc 2023-09-23T19:32:51.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:50 smithi092 bash[58855]: cluster 2023-09-23T19:32:49.720577+0000 osd.7 (osd.7) 45 : cluster 3 failed to encode map e127 with expected crc 2023-09-23T19:32:51.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:50 smithi092 bash[58855]: cluster 2023-09-23T19:32:49.720702+0000 osd.4 (osd.4) 54 : cluster 3 failed to encode map e127 with expected crc 2023-09-23T19:32:51.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:50 smithi092 bash[58855]: cluster 2023-09-23T19:32:49.721263+0000 osd.5 (osd.5) 56 : cluster 3 failed to encode map e127 with expected crc 2023-09-23T19:32:51.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:50 smithi092 bash[58855]: cluster 2023-09-23T19:32:49.723298+0000 osd.5 (osd.5) 57 : cluster 3 failed to encode map e127 with expected crc 2023-09-23T19:32:51.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:50 smithi092 bash[58855]: cluster 2023-09-23T19:32:49.723447+0000 osd.7 (osd.7) 46 : cluster 3 failed to encode map e127 with expected crc 2023-09-23T19:32:51.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:50 smithi092 bash[58855]: cluster 2023-09-23T19:32:49.724129+0000 osd.4 (osd.4) 55 : cluster 3 failed to encode map e127 with expected crc 2023-09-23T19:32:51.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:50 smithi092 bash[58855]: cluster 2023-09-23T19:32:49.724131+0000 osd.6 (osd.6) 47 : cluster 3 failed to encode map e127 with expected crc 2023-09-23T19:32:51.074 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:51.102 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:51.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:51.105 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:32:51.205 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:51.233 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:51.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:51.237 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:32:51.300 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 77 16 97560 97544 79.1648 56.125 0.0161485 0.0126267 2023-09-23T19:32:51.338 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:51.361 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:51.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:51.364 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:32:51.465 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:51.498 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:51.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:51.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:51.605 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:51.634 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:51.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:51.639 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:32:51.739 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:51.769 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:51.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:51.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:32:51.876 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:51.884 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:51 smithi016 bash[77995]: cluster 2023-09-23T19:32:49.985431+0000 mgr.y (mgr.25325) 756 : cluster 0 pgmap v658: 89 pgs: 5 peering, 84 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 808 op/s 2023-09-23T19:32:51.884 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:51 smithi016 bash[77995]: cluster 2023-09-23T19:32:50.713293+0000 mon.a (mon.0) 951 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T19:32:51.884 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:51 smithi016 bash[75004]: cluster 2023-09-23T19:32:49.985431+0000 mgr.y (mgr.25325) 756 : cluster 0 pgmap v658: 89 pgs: 5 peering, 84 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 808 op/s 2023-09-23T19:32:51.884 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:51 smithi016 bash[75004]: cluster 2023-09-23T19:32:50.713293+0000 mon.a (mon.0) 951 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T19:32:51.900 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:51.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:51.903 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:32:52.004 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:52.013 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:51 smithi092 bash[58855]: cluster 2023-09-23T19:32:49.985431+0000 mgr.y (mgr.25325) 756 : cluster 0 pgmap v658: 89 pgs: 5 peering, 84 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 808 op/s 2023-09-23T19:32:52.013 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:51 smithi092 bash[58855]: cluster 2023-09-23T19:32:50.713293+0000 mon.a (mon.0) 951 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T19:32:52.030 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:52.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:52.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:52.133 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:52.171 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:52.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:52.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:32:52.263 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T19:32:52.278 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:52.300 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 78 16 98448 98432 78.8614 55.5 0.0195302 0.0126745 2023-09-23T19:32:52.334 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:52.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:52.345 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:32:52.445 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:52.485 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:52.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:52.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:32:52.590 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:52.617 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:52.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:52.622 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:52.723 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:52.747 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:52.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:52.751 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:32:52.786 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:32:52.852 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:52.882 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:52.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:52.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:52.989 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:53.066 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:53.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:53.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:32:53.169 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:53.195 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:53.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:53.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:32:53.299 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:53.300 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 79 16 99273 99257 78.5158 51.5625 0.0085272 0.0127298 2023-09-23T19:32:53.327 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:53.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:53.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:32:53.433 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:53.461 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:53.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:53.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:32:53.568 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:53.598 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:53.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:53.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:32:53.703 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:53.731 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:53 smithi092 bash[58855]: cluster 2023-09-23T19:32:51.985993+0000 mgr.y (mgr.25325) 757 : cluster 0 pgmap v659: 89 pgs: 5 peering, 84 active+clean; 5.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 902 op/s 2023-09-23T19:32:53.738 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:53.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:53.741 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:32:53.841 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:53.867 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:53.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:53.869 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:53.970 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:53.999 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:54.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:54.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:54.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:53 smithi016 bash[77995]: cluster 2023-09-23T19:32:51.985993+0000 mgr.y (mgr.25325) 757 : cluster 0 pgmap v659: 89 pgs: 5 peering, 84 active+clean; 5.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 902 op/s 2023-09-23T19:32:54.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:53 smithi016 bash[75004]: cluster 2023-09-23T19:32:51.985993+0000 mgr.y (mgr.25325) 757 : cluster 0 pgmap v659: 89 pgs: 5 peering, 84 active+clean; 5.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 902 op/s 2023-09-23T19:32:54.102 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:54.125 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:54.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:54.128 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:32:54.228 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:54.256 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:54.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:54.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:32:54.300 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-09-23T19:32:54.298165+0000 min lat: 0.00139623 max lat: 1.04089 avg lat: 0.0127773 2023-09-23T19:32:54.300 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:32:54.300 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 80 16 100133 100117 78.2061 53.75 0.00505216 0.0127773 2023-09-23T19:32:54.362 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:54.397 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:54.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:54.401 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:32:54.502 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:54.538 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:54.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:54.541 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:32:54.641 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:54.671 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:54.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:54.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:32:54.776 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:54.809 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:54.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:54.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:32:54.915 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:54.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:54.949 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:54.949 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:32:55.049 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:55.073 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:55.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:55.077 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:32:55.178 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:55.211 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:55.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:55.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:32:55.245 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T19:32:55.245 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T19:32:55.246 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-09-23T19:32:55.246 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T19:32:55.246 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T19:32:55.246 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-09-23T19:32:55.300 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 81 16 100932 100916 77.8571 49.9375 0.0196003 0.0128369 2023-09-23T19:32:55.315 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:55.340 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:55.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:55.345 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:55.445 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:55.469 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:55.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:55.473 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:55.573 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:55.612 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:55.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:55.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:55.717 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:55.749 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:55.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:55.754 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:32:55.854 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:55.862 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:55 smithi092 bash[58855]: cluster 2023-09-23T19:32:53.986557+0000 mgr.y (mgr.25325) 758 : cluster 0 pgmap v660: 89 pgs: 2 peering, 87 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 48 MiB/s wr, 773 op/s 2023-09-23T19:32:55.886 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:55.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:55.889 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:32:55.893 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:55 smithi016 bash[77995]: cluster 2023-09-23T19:32:53.986557+0000 mgr.y (mgr.25325) 758 : cluster 0 pgmap v660: 89 pgs: 2 peering, 87 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 48 MiB/s wr, 773 op/s 2023-09-23T19:32:55.893 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:55 smithi016 bash[75004]: cluster 2023-09-23T19:32:53.986557+0000 mgr.y (mgr.25325) 758 : cluster 0 pgmap v660: 89 pgs: 2 peering, 87 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 48 MiB/s wr, 773 op/s 2023-09-23T19:32:55.893 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:32:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:32:55] "GET /metrics HTTP/1.1" 200 36237 "" "Prometheus/2.33.4" 2023-09-23T19:32:55.990 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:56.028 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:56.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:56.035 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:56.136 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:56.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:56.170 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:56.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:32:56.271 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:56.299 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:56.300 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 82 16 101753 101737 77.5333 51.3125 0.00915461 0.0128903 2023-09-23T19:32:56.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:56.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:32:56.403 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:56.428 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:56.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:56.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:56.537 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:56.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:56.577 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:56.577 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:32:56.678 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:56.738 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:56.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:56.743 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:32:56.843 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:56.879 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:56.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:56.885 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:56.986 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:57.073 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:57 smithi016 bash[77995]: cluster 2023-09-23T19:32:56.730242+0000 mon.a (mon.0) 952 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T19:32:57.073 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:57 smithi016 bash[75004]: cluster 2023-09-23T19:32:56.730242+0000 mon.a (mon.0) 952 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T19:32:57.089 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:57.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:57.092 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:32:57.193 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:57.218 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:57.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:57.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:32:57.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:56 smithi092 bash[58855]: cluster 2023-09-23T19:32:56.730242+0000 mon.a (mon.0) 952 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T19:32:57.300 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 83 16 102617 102601 77.2497 54 0.013838 0.0129388 2023-09-23T19:32:57.321 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:57.346 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:57.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:57.353 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:32:57.453 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:57.490 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:57.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:57.494 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:32:57.594 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:57.631 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:57.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:57.633 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:57.734 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:57.761 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:57.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:57.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:32:57.864 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:57.875 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:57 smithi092 bash[58855]: cluster 2023-09-23T19:32:55.987189+0000 mgr.y (mgr.25325) 759 : cluster 0 pgmap v661: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 54 MiB/s wr, 863 op/s 2023-09-23T19:32:57.902 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:57.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:57.908 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:58.009 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:58.019 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:57 smithi016 bash[77995]: cluster 2023-09-23T19:32:55.987189+0000 mgr.y (mgr.25325) 759 : cluster 0 pgmap v661: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 54 MiB/s wr, 863 op/s 2023-09-23T19:32:58.019 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:57 smithi016 bash[75004]: cluster 2023-09-23T19:32:55.987189+0000 mgr.y (mgr.25325) 759 : cluster 0 pgmap v661: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 54 MiB/s wr, 863 op/s 2023-09-23T19:32:58.036 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:58.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:58.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:32:58.086 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-09-23T19:32:58.086 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:32:58.148 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:58.193 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:58.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:58.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:32:58.299 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:58.301 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 84 16 103467 103451 76.9625 53.125 0.0238253 0.0129882 2023-09-23T19:32:58.325 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:32:58.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:58.330 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:32:58.430 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:58.467 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:58.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:58.469 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:58.569 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:58.596 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:58.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:58.602 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:32:58.702 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:58.733 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:32:58.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:58.736 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:32:58.836 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:58.869 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:32:58.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:58.874 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:32:58.974 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:59.004 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:59.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:59.009 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:32:59.110 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:59.145 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:32:59.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:59.150 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:32:59.251 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:59.278 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:32:59.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:59.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:32:59.301 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 85 16 104311 104295 76.6776 52.75 0.0232378 0.0130367 2023-09-23T19:32:59.381 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:59.418 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:32:59.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:59.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:32:59.521 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:59.546 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:59.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:59.553 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:32:59.654 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:59.682 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:32:59.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:59.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:32:59.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:59 smithi092 bash[58855]: cluster 2023-09-23T19:32:57.987732+0000 mgr.y (mgr.25325) 760 : cluster 0 pgmap v662: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 52 MiB/s wr, 831 op/s 2023-09-23T19:32:59.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:59 smithi092 bash[58855]: audit 2023-09-23T19:32:58.644862+0000 mon.a (mon.0) 953 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:32:59.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:32:59 smithi092 bash[58855]: audit 2023-09-23T19:32:58.646130+0000 mon.a (mon.0) 954 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:32:59.789 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:59.812 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:59.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:59.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:32:59.920 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:59.949 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:32:59.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:32:59.954 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:33:00.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:59 smithi016 bash[75004]: cluster 2023-09-23T19:32:57.987732+0000 mgr.y (mgr.25325) 760 : cluster 0 pgmap v662: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 52 MiB/s wr, 831 op/s 2023-09-23T19:33:00.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:59 smithi016 bash[75004]: audit 2023-09-23T19:32:58.644862+0000 mon.a (mon.0) 953 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:33:00.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:32:59 smithi016 bash[75004]: audit 2023-09-23T19:32:58.646130+0000 mon.a (mon.0) 954 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:33:00.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:59 smithi016 bash[77995]: cluster 2023-09-23T19:32:57.987732+0000 mgr.y (mgr.25325) 760 : cluster 0 pgmap v662: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 52 MiB/s wr, 831 op/s 2023-09-23T19:33:00.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:59 smithi016 bash[77995]: audit 2023-09-23T19:32:58.644862+0000 mon.a (mon.0) 953 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:33:00.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:32:59 smithi016 bash[77995]: audit 2023-09-23T19:32:58.646130+0000 mon.a (mon.0) 954 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:33:00.055 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:00.083 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:00.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:00.088 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:00.187 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T19:33:00.189 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:00.232 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:33:00.268 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:00.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:00.278 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:00.292 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:33:00.301 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 86 16 105129 105113 76.3804 51.125 0.0214168 0.0130864 2023-09-23T19:33:00.334 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:33:00.379 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:00.380 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:33:00.412 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:00.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:00.416 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:33:00.427 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:33:00.461 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:33:00.501 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:33:00.517 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:00.536 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:33:00.539 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:33:00.539 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:33:00.539 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:33:00.539 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:33:00.539 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:33:00.539 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:33:00.539 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:33:00.539 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:33:00.550 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:00.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:00.553 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:33:00.653 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:00.656 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:00 smithi092 bash[58855]: audit 2023-09-23T19:33:00.170536+0000 mon.c (mon.2) 144 : audit 0 from='client.? 172.21.15.16:0/2777317745' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:33:00.689 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:00.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:00.692 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:33:00.793 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:00.818 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:00.819 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:00 smithi016 bash[77995]: audit 2023-09-23T19:33:00.170536+0000 mon.c (mon.2) 144 : audit 0 from='client.? 172.21.15.16:0/2777317745' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:33:00.819 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:00 smithi016 bash[75004]: audit 2023-09-23T19:33:00.170536+0000 mon.c (mon.2) 144 : audit 0 from='client.? 172.21.15.16:0/2777317745' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:33:00.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:00.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:00.921 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:00.948 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:00.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:00.950 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:33:01.051 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:01.079 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:01.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:01.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:33:01.185 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:01.248 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:01.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:01.252 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:33:01.301 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 87 16 105962 105946 76.1008 52.0625 0.0255115 0.013135 2023-09-23T19:33:01.353 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:01.380 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:01.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:01.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:33:01.483 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:01.520 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:01.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:01.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:33:01.626 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:01.653 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:01.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:01.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:33:01.757 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:01.785 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:01.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:01.788 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:33:01.889 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:01.915 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:01.920 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:01 smithi092 bash[58855]: cluster 2023-09-23T19:32:59.988434+0000 mgr.y (mgr.25325) 761 : cluster 0 pgmap v663: 89 pgs: 89 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 867 op/s 2023-09-23T19:33:01.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:01.921 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:33:01.986 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:33:02.021 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:01 smithi016 bash[77995]: cluster 2023-09-23T19:32:59.988434+0000 mgr.y (mgr.25325) 761 : cluster 0 pgmap v663: 89 pgs: 89 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 867 op/s 2023-09-23T19:33:02.021 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:01 smithi016 bash[75004]: cluster 2023-09-23T19:32:59.988434+0000 mgr.y (mgr.25325) 761 : cluster 0 pgmap v663: 89 pgs: 89 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 867 op/s 2023-09-23T19:33:02.022 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:02.050 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:02.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:02.053 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:02.154 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:02.176 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:02.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:02.179 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:33:02.280 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:02.301 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 88 16 106797 106781 75.829 52.1875 0.0316443 0.013181 2023-09-23T19:33:02.315 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:02.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:02.321 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:02.422 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:02.455 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:02.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:02.461 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:33:02.562 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:02.594 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:02.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:02.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:02.698 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:02.727 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:02.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:02.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:33:02.832 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:02.862 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:02.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:02.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:33:02.967 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:02.993 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:02.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:02.997 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:33:03.098 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:03.140 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:03.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:03.145 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:33:03.246 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:03.278 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:03.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:03.283 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:03.301 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 89 16 107611 107595 75.5486 50.875 0.0272699 0.0132299 2023-09-23T19:33:03.384 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:03.408 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:03.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:03.412 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:03.513 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:03.543 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:03.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:03.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:33:03.562 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:33:03.649 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:03.683 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:03.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:03.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:03.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:03 smithi016 bash[75004]: cluster 2023-09-23T19:33:01.989055+0000 mgr.y (mgr.25325) 762 : cluster 0 pgmap v664: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 51 MiB/s wr, 822 op/s 2023-09-23T19:33:03.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:03 smithi016 bash[77995]: cluster 2023-09-23T19:33:01.989055+0000 mgr.y (mgr.25325) 762 : cluster 0 pgmap v664: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 51 MiB/s wr, 822 op/s 2023-09-23T19:33:03.788 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:03.825 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:03.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:03.830 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:33:03.930 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:03.939 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:03 smithi092 bash[58855]: cluster 2023-09-23T19:33:01.989055+0000 mgr.y (mgr.25325) 762 : cluster 0 pgmap v664: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 51 MiB/s wr, 822 op/s 2023-09-23T19:33:03.974 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:03.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:03.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:33:04.081 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:04.107 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:04.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:04.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:33:04.210 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:04.238 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:04.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:04.243 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:33:04.301 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Total time run: 90.0074 2023-09-23T19:33:04.301 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Total writes made: 108379 2023-09-23T19:33:04.302 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Write size: 65536 2023-09-23T19:33:04.302 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Object size: 65536 2023-09-23T19:33:04.302 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Bandwidth (MB/sec): 75.257 2023-09-23T19:33:04.302 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Stddev Bandwidth: 29.7577 2023-09-23T19:33:04.302 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Max bandwidth (MB/sec): 171.688 2023-09-23T19:33:04.302 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Min bandwidth (MB/sec): 27.625 2023-09-23T19:33:04.302 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Average IOPS: 1204 2023-09-23T19:33:04.302 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Stddev IOPS: 476.123 2023-09-23T19:33:04.302 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Max IOPS: 2747 2023-09-23T19:33:04.302 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Min IOPS: 442 2023-09-23T19:33:04.302 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Average Latency(s): 0.0132831 2023-09-23T19:33:04.303 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Stddev Latency(s): 0.0163359 2023-09-23T19:33:04.303 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Max latency(s): 1.04089 2023-09-23T19:33:04.303 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Min latency(s): 0.00139623 2023-09-23T19:33:04.307 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T19:33:04.344 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:04.376 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:04.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:04.384 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:04.485 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:04.526 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:04.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:04.530 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:33:04.631 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:04.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:04.666 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:04.666 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:04.767 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:04.814 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:04.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:04.817 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:33:04.917 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:04.949 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:04.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:04.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:33:04.982 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:33:05.057 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:05.089 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:05.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:05.091 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:05.098 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:33:05.192 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:05.205 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:04 smithi092 bash[58855]: cluster 2023-09-23T19:33:03.989564+0000 mgr.y (mgr.25325) 763 : cluster 0 pgmap v665: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 48 MiB/s wr, 774 op/s 2023-09-23T19:33:05.205 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:04 smithi092 bash[58855]: audit 2023-09-23T19:33:03.995567+0000 mon.a (mon.0) 955 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:33:05.227 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:05.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:05.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:05.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:04 smithi016 bash[75004]: cluster 2023-09-23T19:33:03.989564+0000 mgr.y (mgr.25325) 763 : cluster 0 pgmap v665: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 48 MiB/s wr, 774 op/s 2023-09-23T19:33:05.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:04 smithi016 bash[75004]: audit 2023-09-23T19:33:03.995567+0000 mon.a (mon.0) 955 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:33:05.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:04 smithi016 bash[77995]: cluster 2023-09-23T19:33:03.989564+0000 mgr.y (mgr.25325) 763 : cluster 0 pgmap v665: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 48 MiB/s wr, 774 op/s 2023-09-23T19:33:05.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:04 smithi016 bash[77995]: audit 2023-09-23T19:33:03.995567+0000 mon.a (mon.0) 955 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:33:05.338 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:05.370 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:05.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:05.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:05.477 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:05.507 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:05.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:05.511 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:05.612 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:05.641 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:05.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:05.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:05.748 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:05.771 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:05.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:05.778 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:33:05.878 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:05.887 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:33:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:33:05] "GET /metrics HTTP/1.1" 200 36239 "" "Prometheus/2.33.4" 2023-09-23T19:33:05.901 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:05.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:05.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:33:06.004 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:06.008 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:06 smithi092 bash[58855]: audit 2023-09-23T19:33:05.019361+0000 mon.a (mon.0) 956 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:33:06.032 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:06.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:06.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:06.137 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:06.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:06.167 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:06.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:33:06.268 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:06.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:06 smithi016 bash[77995]: audit 2023-09-23T19:33:05.019361+0000 mon.a (mon.0) 956 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:33:06.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:06 smithi016 bash[75004]: audit 2023-09-23T19:33:05.019361+0000 mon.a (mon.0) 956 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:33:06.300 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:06.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:06.304 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:06.405 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:06.457 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:06.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:06.460 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:33:06.561 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:06.598 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:06.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:06.601 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:33:06.702 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:06.734 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:06.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:06.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:33:06.841 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:06.880 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:06.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:06.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:06.983 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:07.008 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:07.010 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:07 smithi016 bash[77995]: cluster 2023-09-23T19:33:05.990053+0000 mgr.y (mgr.25325) 764 : cluster 0 pgmap v666: 89 pgs: 89 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 54 MiB/s wr, 888 op/s 2023-09-23T19:33:07.010 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:07 smithi016 bash[75004]: cluster 2023-09-23T19:33:05.990053+0000 mgr.y (mgr.25325) 764 : cluster 0 pgmap v666: 89 pgs: 89 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 54 MiB/s wr, 888 op/s 2023-09-23T19:33:07.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:07.013 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:33:07.114 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:07.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:07.143 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:07.143 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:33:07.244 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:07.253 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:07 smithi092 bash[58855]: cluster 2023-09-23T19:33:05.990053+0000 mgr.y (mgr.25325) 764 : cluster 0 pgmap v666: 89 pgs: 89 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 54 MiB/s wr, 888 op/s 2023-09-23T19:33:07.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:07.275 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:07.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:33:07.376 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:07.400 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:07.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:07.402 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:07.503 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:07.523 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:07.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:07.525 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:07.626 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:07.667 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:07.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:07.669 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:33:07.769 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:07.811 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:07.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:07.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:07.916 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:07.941 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:07.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:07.950 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:33:08.051 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:08.091 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:08.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:08.097 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:33:08.150 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:33:08.197 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:08.225 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:08.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:08.228 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:33:08.328 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:08.362 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:08.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:08.365 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:33:08.466 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:08.490 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:08.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:08.493 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:33:08.593 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:08.618 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:08.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:08.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:08.728 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:08.758 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:08.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:08.761 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:08.861 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:08.894 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:08.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:08.899 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:08.999 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:09.031 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:09.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:09.037 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:09.138 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:09.146 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:09 smithi016 bash[77995]: cluster 2023-09-23T19:33:07.990723+0000 mgr.y (mgr.25325) 765 : cluster 0 pgmap v667: 89 pgs: 89 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 44 MiB/s wr, 812 op/s 2023-09-23T19:33:09.147 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:09 smithi016 bash[75004]: cluster 2023-09-23T19:33:07.990723+0000 mgr.y (mgr.25325) 765 : cluster 0 pgmap v667: 89 pgs: 89 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 44 MiB/s wr, 812 op/s 2023-09-23T19:33:09.163 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:09.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:09.172 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:09.272 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:09.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:09 smithi092 bash[58855]: cluster 2023-09-23T19:33:07.990723+0000 mgr.y (mgr.25325) 765 : cluster 0 pgmap v667: 89 pgs: 89 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 44 MiB/s wr, 812 op/s 2023-09-23T19:33:09.297 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:09.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:09.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:33:09.401 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:09.424 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:09.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:09.426 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:09.527 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:09.559 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:09.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:09.565 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:33:09.665 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:09.696 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:09.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:09.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:09.799 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:09.821 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:09.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:09.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:33:09.925 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:09.953 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:09.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:09.961 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:33:10.061 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:10.094 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:10.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:10.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:10.197 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:10.239 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:10.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:10.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:10.346 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:10.384 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:10.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:10.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:10.489 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:10.512 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:10.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:10.515 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:33:10.615 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:10.640 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:10.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:10.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:33:10.747 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:10.781 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:10.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:10.786 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:33:10.887 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:10.928 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:10.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:10.938 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:11.039 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:11.215 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:11.548 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is set 2023-09-23T19:33:11.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:11.549 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:33:11.550 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:11 smithi092 bash[58855]: cluster 2023-09-23T19:33:09.991295+0000 mgr.y (mgr.25325) 766 : cluster 0 pgmap v668: 89 pgs: 89 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 40 MiB/s wr, 854 op/s 2023-09-23T19:33:11.550 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:11 smithi092 bash[58855]: audit 2023-09-23T19:33:10.469176+0000 mon.a (mon.0) 957 : audit 1 from='client.? 172.21.15.16:0/4075827669' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:33:11.596 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:11 smithi016 bash[75004]: cluster 2023-09-23T19:33:09.991295+0000 mgr.y (mgr.25325) 766 : cluster 0 pgmap v668: 89 pgs: 89 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 40 MiB/s wr, 854 op/s 2023-09-23T19:33:11.597 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:11 smithi016 bash[75004]: audit 2023-09-23T19:33:10.469176+0000 mon.a (mon.0) 957 : audit 1 from='client.? 172.21.15.16:0/4075827669' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:33:11.601 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:11 smithi016 bash[77995]: cluster 2023-09-23T19:33:09.991295+0000 mgr.y (mgr.25325) 766 : cluster 0 pgmap v668: 89 pgs: 89 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 40 MiB/s wr, 854 op/s 2023-09-23T19:33:11.601 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:11 smithi016 bash[77995]: audit 2023-09-23T19:33:10.469176+0000 mon.a (mon.0) 957 : audit 1 from='client.? 172.21.15.16:0/4075827669' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:33:11.650 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:11.758 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:11.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:11.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:33:11.864 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:11.912 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:11.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:11.913 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:33:12.014 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:12.035 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:12.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:12.040 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:12.141 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:12.172 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:12.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:12.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:12.280 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:12.314 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:12.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:12.317 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:12.417 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:12.446 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:12.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:12.452 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:33:12.552 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:12.585 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:12.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:12.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:33:12.690 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:12.717 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:12.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:12.720 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:12.720 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:12 smithi016 bash[75004]: cluster 2023-09-23T19:33:11.322496+0000 mon.a (mon.0) 958 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:33:12.720 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:12 smithi016 bash[75004]: audit 2023-09-23T19:33:11.337554+0000 mon.a (mon.0) 959 : audit 1 from='client.? 172.21.15.16:0/4075827669' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:33:12.720 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:12 smithi016 bash[75004]: cluster 2023-09-23T19:33:11.337622+0000 mon.a (mon.0) 960 : cluster 0 osdmap e128: 8 total, 8 up, 8 in 2023-09-23T19:33:12.720 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:12 smithi016 bash[75004]: cluster 2023-09-23T19:33:11.338094+0000 osd.6 (osd.6) 48 : cluster 3 failed to encode map e128 with expected crc 2023-09-23T19:33:12.720 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:12 smithi016 bash[75004]: cluster 2023-09-23T19:33:11.343159+0000 osd.6 (osd.6) 49 : cluster 3 failed to encode map e128 with expected crc 2023-09-23T19:33:12.720 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:12 smithi016 bash[75004]: cluster 2023-09-23T19:33:11.344727+0000 osd.7 (osd.7) 47 : cluster 3 failed to encode map e128 with expected crc 2023-09-23T19:33:12.720 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:12 smithi016 bash[75004]: cluster 2023-09-23T19:33:11.350951+0000 osd.4 (osd.4) 56 : cluster 3 failed to encode map e128 with expected crc 2023-09-23T19:33:12.720 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:12 smithi016 bash[75004]: audit 2023-09-23T19:33:11.548329+0000 mon.a (mon.0) 961 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:33:12.721 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:12 smithi016 bash[75004]: audit 2023-09-23T19:33:11.556876+0000 mon.a (mon.0) 962 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:33:12.721 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:12 smithi016 bash[77995]: cluster 2023-09-23T19:33:11.322496+0000 mon.a (mon.0) 958 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:33:12.721 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:12 smithi016 bash[77995]: audit 2023-09-23T19:33:11.337554+0000 mon.a (mon.0) 959 : audit 1 from='client.? 172.21.15.16:0/4075827669' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:33:12.721 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:12 smithi016 bash[77995]: cluster 2023-09-23T19:33:11.337622+0000 mon.a (mon.0) 960 : cluster 0 osdmap e128: 8 total, 8 up, 8 in 2023-09-23T19:33:12.721 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:12 smithi016 bash[77995]: cluster 2023-09-23T19:33:11.338094+0000 osd.6 (osd.6) 48 : cluster 3 failed to encode map e128 with expected crc 2023-09-23T19:33:12.721 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:12 smithi016 bash[77995]: cluster 2023-09-23T19:33:11.343159+0000 osd.6 (osd.6) 49 : cluster 3 failed to encode map e128 with expected crc 2023-09-23T19:33:12.721 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:12 smithi016 bash[77995]: cluster 2023-09-23T19:33:11.344727+0000 osd.7 (osd.7) 47 : cluster 3 failed to encode map e128 with expected crc 2023-09-23T19:33:12.721 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:12 smithi016 bash[77995]: cluster 2023-09-23T19:33:11.350951+0000 osd.4 (osd.4) 56 : cluster 3 failed to encode map e128 with expected crc 2023-09-23T19:33:12.721 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:12 smithi016 bash[77995]: audit 2023-09-23T19:33:11.548329+0000 mon.a (mon.0) 961 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:33:12.721 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:12 smithi016 bash[77995]: audit 2023-09-23T19:33:11.556876+0000 mon.a (mon.0) 962 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:33:12.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:12 smithi092 bash[58855]: cluster 2023-09-23T19:33:11.322496+0000 mon.a (mon.0) 958 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:33:12.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:12 smithi092 bash[58855]: audit 2023-09-23T19:33:11.337554+0000 mon.a (mon.0) 959 : audit 1 from='client.? 172.21.15.16:0/4075827669' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:33:12.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:12 smithi092 bash[58855]: cluster 2023-09-23T19:33:11.337622+0000 mon.a (mon.0) 960 : cluster 0 osdmap e128: 8 total, 8 up, 8 in 2023-09-23T19:33:12.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:12 smithi092 bash[58855]: cluster 2023-09-23T19:33:11.338094+0000 osd.6 (osd.6) 48 : cluster 3 failed to encode map e128 with expected crc 2023-09-23T19:33:12.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:12 smithi092 bash[58855]: cluster 2023-09-23T19:33:11.343159+0000 osd.6 (osd.6) 49 : cluster 3 failed to encode map e128 with expected crc 2023-09-23T19:33:12.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:12 smithi092 bash[58855]: cluster 2023-09-23T19:33:11.344727+0000 osd.7 (osd.7) 47 : cluster 3 failed to encode map e128 with expected crc 2023-09-23T19:33:12.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:12 smithi092 bash[58855]: cluster 2023-09-23T19:33:11.350951+0000 osd.4 (osd.4) 56 : cluster 3 failed to encode map e128 with expected crc 2023-09-23T19:33:12.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:12 smithi092 bash[58855]: audit 2023-09-23T19:33:11.548329+0000 mon.a (mon.0) 961 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:33:12.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:12 smithi092 bash[58855]: audit 2023-09-23T19:33:11.556876+0000 mon.a (mon.0) 962 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:33:12.820 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:12.853 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:12.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:12.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:33:12.956 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:12.994 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:12.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:12.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:33:13.096 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:13.124 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:13.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:13.127 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:13.228 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:13.253 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:13.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:13.256 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:13.357 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:13.387 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:13.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:13.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:13.493 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:13.519 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:13.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:13.525 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:13.626 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:13.658 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:13.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:13.660 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:33:13.761 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:13.770 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:13 smithi092 bash[58855]: cluster 2023-09-23T19:33:11.991740+0000 mgr.y (mgr.25325) 767 : cluster 0 pgmap v670: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 22 MiB/s wr, 786 op/s 2023-09-23T19:33:13.790 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:13.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:13.795 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:13.896 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:13.931 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:13.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:13.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:33:13.988 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:13 smithi016 bash[77995]: cluster 2023-09-23T19:33:11.991740+0000 mgr.y (mgr.25325) 767 : cluster 0 pgmap v670: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 22 MiB/s wr, 786 op/s 2023-09-23T19:33:13.988 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:13 smithi016 bash[75004]: cluster 2023-09-23T19:33:11.991740+0000 mgr.y (mgr.25325) 767 : cluster 0 pgmap v670: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 22 MiB/s wr, 786 op/s 2023-09-23T19:33:14.039 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:14.068 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:14.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:14.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:33:14.171 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:14.223 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:14.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:14.242 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:14.343 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:14.379 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:14.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:14.382 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:33:14.482 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:14.518 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:14.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:14.524 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:33:14.624 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:14.645 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:14.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:14.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:14.749 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:14.758 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:14 smithi092 bash[58855]: audit 2023-09-23T19:33:13.639687+0000 mon.a (mon.0) 963 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:33:14.772 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:14.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:14.775 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:14.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:14 smithi016 bash[77995]: audit 2023-09-23T19:33:13.639687+0000 mon.a (mon.0) 963 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:33:14.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:14 smithi016 bash[75004]: audit 2023-09-23T19:33:13.639687+0000 mon.a (mon.0) 963 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:33:14.875 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:14.903 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:14.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:14.906 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:15.006 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:15.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:15.037 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:15.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:15.138 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:15.168 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:15.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:15.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:33:15.271 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:15.296 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:15.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:15.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:33:15.400 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:15.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:15.427 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:15.427 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:33:15.528 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:15.560 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:15.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:15.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:15.663 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:15.685 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:15.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:15.687 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:33:15.787 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:15.809 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:15.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:15.811 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:33:15.837 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:33:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:33:15] "GET /metrics HTTP/1.1" 200 36243 "" "Prometheus/2.33.4" 2023-09-23T19:33:15.838 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:15 smithi016 bash[77995]: cluster 2023-09-23T19:33:13.992524+0000 mgr.y (mgr.25325) 768 : cluster 0 pgmap v671: 89 pgs: 89 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 16 MiB/s wr, 775 op/s 2023-09-23T19:33:15.838 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:15 smithi016 bash[75004]: cluster 2023-09-23T19:33:13.992524+0000 mgr.y (mgr.25325) 768 : cluster 0 pgmap v671: 89 pgs: 89 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 16 MiB/s wr, 775 op/s 2023-09-23T19:33:15.912 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:15.922 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:15 smithi092 bash[58855]: cluster 2023-09-23T19:33:13.992524+0000 mgr.y (mgr.25325) 768 : cluster 0 pgmap v671: 89 pgs: 89 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 16 MiB/s wr, 775 op/s 2023-09-23T19:33:15.957 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:15.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:15.961 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:33:16.061 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:16.083 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:16.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:16.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:16.186 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:16.209 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:16.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:16.211 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:16.311 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:16.343 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:16.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:16.349 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:33:16.449 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:16.476 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:16.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:16.479 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:16.580 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:16.619 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:16.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:16.625 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:33:16.725 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:16.762 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:16.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:16.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:33:16.865 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:16.910 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:16.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:16.917 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:33:16.974 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-09-23T19:33:16.975 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [], 2023-09-23T19:33:16.975 INFO:teuthology.orchestra.run.smithi016.stdout: "num_ops": 0 2023-09-23T19:33:16.975 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-09-23T19:33:17.017 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:17.051 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:17.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:17.055 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:33:17.155 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:17.187 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:17.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:17.192 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:17.193 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-09-23T19:33:17.193 INFO:teuthology.orchestra.run.smithi016.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T19:33:17.193 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-09-23T19:33:17.292 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:17.321 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:17.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:17.326 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:17.405 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T19:33:17.427 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:17.494 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:17.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:17.499 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:33:17.600 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:17.625 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:17.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:17.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:17.728 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:17.736 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:17 smithi016 bash[77995]: cluster 2023-09-23T19:33:15.993137+0000 mgr.y (mgr.25325) 769 : cluster 0 pgmap v672: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 3.4 MiB/s wr, 764 op/s 2023-09-23T19:33:17.737 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:17 smithi016 bash[77995]: audit 2023-09-23T19:33:16.483430+0000 mon.a (mon.0) 964 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:33:17.737 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:17 smithi016 bash[77995]: audit 2023-09-23T19:33:16.495039+0000 mon.a (mon.0) 965 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:33:17.737 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:17 smithi016 bash[77995]: audit 2023-09-23T19:33:17.082215+0000 mon.a (mon.0) 966 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:33:17.737 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:17 smithi016 bash[77995]: audit 2023-09-23T19:33:17.083267+0000 mon.a (mon.0) 967 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:33:17.737 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:17 smithi016 bash[77995]: audit 2023-09-23T19:33:17.094057+0000 mon.a (mon.0) 968 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:33:17.737 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:17 smithi016 bash[75004]: cluster 2023-09-23T19:33:15.993137+0000 mgr.y (mgr.25325) 769 : cluster 0 pgmap v672: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 3.4 MiB/s wr, 764 op/s 2023-09-23T19:33:17.738 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:17 smithi016 bash[75004]: audit 2023-09-23T19:33:16.483430+0000 mon.a (mon.0) 964 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:33:17.738 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:17 smithi016 bash[75004]: audit 2023-09-23T19:33:16.495039+0000 mon.a (mon.0) 965 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:33:17.738 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:17 smithi016 bash[75004]: audit 2023-09-23T19:33:17.082215+0000 mon.a (mon.0) 966 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:33:17.738 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:17 smithi016 bash[75004]: audit 2023-09-23T19:33:17.083267+0000 mon.a (mon.0) 967 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:33:17.738 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:17 smithi016 bash[75004]: audit 2023-09-23T19:33:17.094057+0000 mon.a (mon.0) 968 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:33:17.763 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:17.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:17.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:17.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:17 smithi092 bash[58855]: cluster 2023-09-23T19:33:15.993137+0000 mgr.y (mgr.25325) 769 : cluster 0 pgmap v672: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 3.4 MiB/s wr, 764 op/s 2023-09-23T19:33:17.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:17 smithi092 bash[58855]: audit 2023-09-23T19:33:16.483430+0000 mon.a (mon.0) 964 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:33:17.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:17 smithi092 bash[58855]: audit 2023-09-23T19:33:16.495039+0000 mon.a (mon.0) 965 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:33:17.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:17 smithi092 bash[58855]: audit 2023-09-23T19:33:17.082215+0000 mon.a (mon.0) 966 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:33:17.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:17 smithi092 bash[58855]: audit 2023-09-23T19:33:17.083267+0000 mon.a (mon.0) 967 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:33:17.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:17 smithi092 bash[58855]: audit 2023-09-23T19:33:17.094057+0000 mon.a (mon.0) 968 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:33:17.867 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:17.892 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:17.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:17.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:33:17.998 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:18.034 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:18.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:18.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:18.139 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:18.172 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:18.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:18.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:33:18.279 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:18.309 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:18.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:18.313 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:33:18.413 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:18.436 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:18.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:18.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:33:18.539 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:18.584 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:18.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:18.590 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:33:18.691 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:18.726 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:18.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:18.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:33:18.829 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:18.852 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:18.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:18.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:33:18.955 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:18.983 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:18.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:18.986 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:33:19.087 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:19.122 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:19.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:19.126 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:19.227 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:19.258 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:19.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:19.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:33:19.362 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:19.397 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:19.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:19.401 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:19.487 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:19 smithi092 bash[58855]: cluster 2023-09-23T19:33:17.996476+0000 mgr.y (mgr.25325) 770 : cluster 0 pgmap v673: 89 pgs: 89 active+clean; 6.1 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 787 op/s 2023-09-23T19:33:19.501 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:19.536 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:19.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:19.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:19.642 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:19.672 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:19.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:19.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:19.775 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:19.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:19 smithi016 bash[77995]: cluster 2023-09-23T19:33:17.996476+0000 mgr.y (mgr.25325) 770 : cluster 0 pgmap v673: 89 pgs: 89 active+clean; 6.1 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 787 op/s 2023-09-23T19:33:19.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:19 smithi016 bash[75004]: cluster 2023-09-23T19:33:17.996476+0000 mgr.y (mgr.25325) 770 : cluster 0 pgmap v673: 89 pgs: 89 active+clean; 6.1 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 787 op/s 2023-09-23T19:33:19.809 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:19.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:19.815 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:33:19.915 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:19.944 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:19.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:19.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:20.049 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:20.094 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:20.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:20.098 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:20.142 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T19:33:20.199 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:20.264 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:20.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:20.267 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:20.368 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:20.394 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:20.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:20.397 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:33:20.497 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:20.531 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:20.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:20.536 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:33:20.575 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:33:20.616 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:33:20.636 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:20.650 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:33:20.666 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:20.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:20.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:20.690 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:33:20.725 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:33:20.761 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:33:20.771 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:20.800 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:20.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:20.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:20.817 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:33:20.858 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:33:20.866 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:33:20.866 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:33:20.867 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:33:20.867 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:33:20.867 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:33:20.867 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:33:20.867 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:33:20.867 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:33:20.906 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:20.937 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:20.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:20.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:33:21.041 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:21.066 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:21.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:21.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:21.171 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:21.197 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:21.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:21.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:21.303 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:21.332 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:21.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:21.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:33:21.440 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:21.470 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:21.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:21.472 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:33:21.573 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:21.584 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:21 smithi016 bash[77995]: cluster 2023-09-23T19:33:19.997111+0000 mgr.y (mgr.25325) 771 : cluster 0 pgmap v674: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 786 op/s 2023-09-23T19:33:21.584 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:21 smithi016 bash[77995]: audit 2023-09-23T19:33:20.541834+0000 mon.b (mon.1) 39 : audit 0 from='client.? 172.21.15.16:0/3693738343' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:33:21.584 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:21 smithi016 bash[75004]: cluster 2023-09-23T19:33:19.997111+0000 mgr.y (mgr.25325) 771 : cluster 0 pgmap v674: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 786 op/s 2023-09-23T19:33:21.585 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:21 smithi016 bash[75004]: audit 2023-09-23T19:33:20.541834+0000 mon.b (mon.1) 39 : audit 0 from='client.? 172.21.15.16:0/3693738343' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:33:21.603 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:21.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:21.608 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:33:21.709 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:21.737 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:21.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:21.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:21.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:21 smithi092 bash[58855]: cluster 2023-09-23T19:33:19.997111+0000 mgr.y (mgr.25325) 771 : cluster 0 pgmap v674: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 786 op/s 2023-09-23T19:33:21.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:21 smithi092 bash[58855]: audit 2023-09-23T19:33:20.541834+0000 mon.b (mon.1) 39 : audit 0 from='client.? 172.21.15.16:0/3693738343' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:33:21.841 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:21.870 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:21.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:21.876 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:33:21.977 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:22.001 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:22.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:22.005 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:33:22.106 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:22.136 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:22.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:22.142 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:22.242 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:22.293 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:22.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:22.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:33:22.398 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:22.423 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:22.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:22.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:22.531 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:22.545 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:33:22.569 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:22.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:22.578 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:33:22.679 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:22.721 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:22.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:22.728 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:22.828 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:22.850 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:22.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:22.856 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:33:22.956 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:22.997 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:23.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:23.001 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:33:23.102 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:23.142 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:23.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:23.145 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:33:23.246 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:23.274 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:23.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:23.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:33:23.379 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:23.411 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:23.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:23.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:23.514 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:23.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:23 smithi016 bash[75004]: cluster 2023-09-23T19:33:21.997719+0000 mgr.y (mgr.25325) 772 : cluster 0 pgmap v675: 89 pgs: 89 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 741 op/s 2023-09-23T19:33:23.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:23 smithi016 bash[77995]: cluster 2023-09-23T19:33:21.997719+0000 mgr.y (mgr.25325) 772 : cluster 0 pgmap v675: 89 pgs: 89 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 741 op/s 2023-09-23T19:33:23.549 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:23.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:23.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:23.653 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:23.662 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:23 smithi092 bash[58855]: cluster 2023-09-23T19:33:21.997719+0000 mgr.y (mgr.25325) 772 : cluster 0 pgmap v675: 89 pgs: 89 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 741 op/s 2023-09-23T19:33:23.699 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:23.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:23.702 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:33:23.802 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:23.827 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:23.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:23.829 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:33:23.930 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:23.963 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:23.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:23.964 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:33:24.057 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:33:24.065 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:24.091 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:24.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:24.096 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:33:24.197 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:24.237 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:24.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:24.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:33:24.345 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:24.370 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:24.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:24.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:24.478 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:24.506 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:24.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:24.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:33:24.609 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:24.636 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:24.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:24.641 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:33:24.742 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:24.767 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:24.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:24.773 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:33:24.873 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:24.902 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:24.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:24.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:25.008 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:25.037 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:25.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:25.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:33:25.143 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:25.168 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:25.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:25.171 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:33:25.180 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T19:33:25.180 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T19:33:25.180 INFO:tasks.thrashosds.thrasher:Growing pool device_health_metrics 2023-09-23T19:33:25.180 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T19:33:25.272 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:25.298 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:25.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:25.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:33:25.400 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:25.442 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:25.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:25.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:33:25.482 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:33:25.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:25 smithi092 bash[58855]: cluster 2023-09-23T19:33:24.000083+0000 mgr.y (mgr.25325) 773 : cluster 0 pgmap v676: 89 pgs: 89 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 738 op/s 2023-09-23T19:33:25.545 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:25.575 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:25.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:25.579 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:25.681 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:25.690 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:25 smithi016 bash[75004]: cluster 2023-09-23T19:33:24.000083+0000 mgr.y (mgr.25325) 773 : cluster 0 pgmap v676: 89 pgs: 89 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 738 op/s 2023-09-23T19:33:25.690 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:25 smithi016 bash[77995]: cluster 2023-09-23T19:33:24.000083+0000 mgr.y (mgr.25325) 773 : cluster 0 pgmap v676: 89 pgs: 89 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 738 op/s 2023-09-23T19:33:25.690 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:33:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:33:25] "GET /metrics HTTP/1.1" 200 36243 "" "Prometheus/2.33.4" 2023-09-23T19:33:25.729 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:25.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:25.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:25.833 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:25.865 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:25.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:25.873 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:25.973 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:26.000 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:26.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:26.003 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:33:26.104 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:26.146 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:26.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:26.149 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:33:26.250 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:26.280 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:26.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:26.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:33:26.384 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:26.408 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:26.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:26.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:26.512 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:26.555 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:26.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:26.558 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:33:26.659 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:26.714 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:26.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:26.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:26.818 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:26.849 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:26.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:26.856 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:33:26.957 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:26.982 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:26.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:26.985 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:33:27.086 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:27.111 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:27.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:27.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:33:27.218 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:27.249 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:27.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:27.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:33:27.353 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:27.376 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:27.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:27.381 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:27.481 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:27.526 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:27.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:27.533 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:33:27.634 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:27.672 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:27.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:27.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:27.780 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:27.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:27 smithi016 bash[75004]: cluster 2023-09-23T19:33:26.000633+0000 mgr.y (mgr.25325) 774 : cluster 0 pgmap v677: 89 pgs: 89 active+clean; 5.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 842 op/s 2023-09-23T19:33:27.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:27 smithi016 bash[77995]: cluster 2023-09-23T19:33:26.000633+0000 mgr.y (mgr.25325) 774 : cluster 0 pgmap v677: 89 pgs: 89 active+clean; 5.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 842 op/s 2023-09-23T19:33:27.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:27 smithi092 bash[58855]: cluster 2023-09-23T19:33:26.000633+0000 mgr.y (mgr.25325) 774 : cluster 0 pgmap v677: 89 pgs: 89 active+clean; 5.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 842 op/s 2023-09-23T19:33:27.803 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:27.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:27.809 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:33:27.909 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:27.937 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:27.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:27.945 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:28.046 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:28.082 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:28.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:28.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:33:28.187 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:28.214 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:28.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:28.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:28.317 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:28.343 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:28.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:28.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:28.450 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:28.484 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:28.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:28.489 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:33:28.509 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:28 smithi016 bash[75004]: audit 2023-09-23T19:33:28.156521+0000 mon.c (mon.2) 145 : audit 1 from='client.? 172.21.15.16:0/3134359522' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:33:28.510 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:28 smithi016 bash[75004]: audit 2023-09-23T19:33:28.157030+0000 mon.a (mon.0) 969 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:33:28.531 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is unset 2023-09-23T19:33:28.589 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:28.598 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:28 smithi092 bash[58855]: audit 2023-09-23T19:33:28.156521+0000 mon.c (mon.2) 145 : audit 1 from='client.? 172.21.15.16:0/3134359522' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:33:28.598 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:28 smithi092 bash[58855]: audit 2023-09-23T19:33:28.157030+0000 mon.a (mon.0) 969 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:33:28.619 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:28.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:28.625 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:33:28.726 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:28.786 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:28.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:28.790 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:28 smithi016 bash[77995]: audit 2023-09-23T19:33:28.156521+0000 mon.c (mon.2) 145 : audit 1 from='client.? 172.21.15.16:0/3134359522' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:33:28.791 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:28 smithi016 bash[77995]: audit 2023-09-23T19:33:28.157030+0000 mon.a (mon.0) 969 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:33:28.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:33:28.892 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:28.915 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:28.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:28.921 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:33:29.021 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:29.054 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:29.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:29.060 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:29.160 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:29.190 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:29.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:29.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:33:29.294 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:29.319 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:29.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:29.325 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:33:29.426 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:29.458 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:29.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:29.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:29.561 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:29.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:29.595 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:29.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:29.696 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:29.703 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:29 smithi016 bash[77995]: cluster 2023-09-23T19:33:28.001267+0000 mgr.y (mgr.25325) 775 : cluster 0 pgmap v678: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 804 op/s 2023-09-23T19:33:29.704 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:29 smithi016 bash[77995]: cluster 2023-09-23T19:33:28.504888+0000 mon.a (mon.0) 970 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:33:29.704 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:29 smithi016 bash[77995]: audit 2023-09-23T19:33:28.520193+0000 mon.a (mon.0) 971 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:33:29.704 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:29 smithi016 bash[77995]: cluster 2023-09-23T19:33:28.520364+0000 mon.a (mon.0) 972 : cluster 0 osdmap e129: 8 total, 8 up, 8 in 2023-09-23T19:33:29.704 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:29 smithi016 bash[77995]: cluster 2023-09-23T19:33:28.538132+0000 osd.6 (osd.6) 50 : cluster 3 failed to encode map e129 with expected crc 2023-09-23T19:33:29.704 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:29 smithi016 bash[77995]: cluster 2023-09-23T19:33:28.539203+0000 osd.5 (osd.5) 58 : cluster 3 failed to encode map e129 with expected crc 2023-09-23T19:33:29.704 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:29 smithi016 bash[77995]: cluster 2023-09-23T19:33:28.541084+0000 osd.5 (osd.5) 59 : cluster 3 failed to encode map e129 with expected crc 2023-09-23T19:33:29.704 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:29 smithi016 bash[77995]: cluster 2023-09-23T19:33:28.544015+0000 osd.7 (osd.7) 48 : cluster 3 failed to encode map e129 with expected crc 2023-09-23T19:33:29.704 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:29 smithi016 bash[77995]: cluster 2023-09-23T19:33:28.561232+0000 osd.4 (osd.4) 57 : cluster 3 failed to encode map e129 with expected crc 2023-09-23T19:33:29.704 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:29 smithi016 bash[77995]: audit 2023-09-23T19:33:28.639610+0000 mon.a (mon.0) 973 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:33:29.704 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:29 smithi016 bash[75004]: cluster 2023-09-23T19:33:28.001267+0000 mgr.y (mgr.25325) 775 : cluster 0 pgmap v678: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 804 op/s 2023-09-23T19:33:29.705 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:29 smithi016 bash[75004]: cluster 2023-09-23T19:33:28.504888+0000 mon.a (mon.0) 970 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:33:29.705 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:29 smithi016 bash[75004]: audit 2023-09-23T19:33:28.520193+0000 mon.a (mon.0) 971 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:33:29.705 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:29 smithi016 bash[75004]: cluster 2023-09-23T19:33:28.520364+0000 mon.a (mon.0) 972 : cluster 0 osdmap e129: 8 total, 8 up, 8 in 2023-09-23T19:33:29.705 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:29 smithi016 bash[75004]: cluster 2023-09-23T19:33:28.538132+0000 osd.6 (osd.6) 50 : cluster 3 failed to encode map e129 with expected crc 2023-09-23T19:33:29.705 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:29 smithi016 bash[75004]: cluster 2023-09-23T19:33:28.539203+0000 osd.5 (osd.5) 58 : cluster 3 failed to encode map e129 with expected crc 2023-09-23T19:33:29.705 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:29 smithi016 bash[75004]: cluster 2023-09-23T19:33:28.541084+0000 osd.5 (osd.5) 59 : cluster 3 failed to encode map e129 with expected crc 2023-09-23T19:33:29.705 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:29 smithi016 bash[75004]: cluster 2023-09-23T19:33:28.544015+0000 osd.7 (osd.7) 48 : cluster 3 failed to encode map e129 with expected crc 2023-09-23T19:33:29.705 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:29 smithi016 bash[75004]: cluster 2023-09-23T19:33:28.561232+0000 osd.4 (osd.4) 57 : cluster 3 failed to encode map e129 with expected crc 2023-09-23T19:33:29.705 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:29 smithi016 bash[75004]: audit 2023-09-23T19:33:28.639610+0000 mon.a (mon.0) 973 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:33:29.718 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:29.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:29.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:33:29.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:29 smithi092 bash[58855]: cluster 2023-09-23T19:33:28.001267+0000 mgr.y (mgr.25325) 775 : cluster 0 pgmap v678: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 804 op/s 2023-09-23T19:33:29.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:29 smithi092 bash[58855]: cluster 2023-09-23T19:33:28.504888+0000 mon.a (mon.0) 970 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:33:29.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:29 smithi092 bash[58855]: audit 2023-09-23T19:33:28.520193+0000 mon.a (mon.0) 971 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:33:29.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:29 smithi092 bash[58855]: cluster 2023-09-23T19:33:28.520364+0000 mon.a (mon.0) 972 : cluster 0 osdmap e129: 8 total, 8 up, 8 in 2023-09-23T19:33:29.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:29 smithi092 bash[58855]: cluster 2023-09-23T19:33:28.538132+0000 osd.6 (osd.6) 50 : cluster 3 failed to encode map e129 with expected crc 2023-09-23T19:33:29.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:29 smithi092 bash[58855]: cluster 2023-09-23T19:33:28.539203+0000 osd.5 (osd.5) 58 : cluster 3 failed to encode map e129 with expected crc 2023-09-23T19:33:29.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:29 smithi092 bash[58855]: cluster 2023-09-23T19:33:28.541084+0000 osd.5 (osd.5) 59 : cluster 3 failed to encode map e129 with expected crc 2023-09-23T19:33:29.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:29 smithi092 bash[58855]: cluster 2023-09-23T19:33:28.544015+0000 osd.7 (osd.7) 48 : cluster 3 failed to encode map e129 with expected crc 2023-09-23T19:33:29.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:29 smithi092 bash[58855]: cluster 2023-09-23T19:33:28.561232+0000 osd.4 (osd.4) 57 : cluster 3 failed to encode map e129 with expected crc 2023-09-23T19:33:29.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:29 smithi092 bash[58855]: audit 2023-09-23T19:33:28.639610+0000 mon.a (mon.0) 973 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:33:29.824 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:29.857 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:29.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:29.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:33:29.959 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:29.992 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:29.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:29.995 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:30.096 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:30.127 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:30.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:30.132 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:30.232 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:30.255 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:30.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:30.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:33:30.365 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:30.394 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:30.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:30.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:33:30.500 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:30.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:30.530 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:30.530 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:33:30.630 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:30.653 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:30.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:30.660 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:30.761 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:30.792 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:30.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:30.797 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:30.847 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:33:30.898 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:30.934 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:30.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:30.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:31.036 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:33:31.042 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:31.077 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:31.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:31.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:33:31.149 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-09-23T19:33:31.149 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:33:31.180 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:31.210 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:31.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:31.217 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:31.318 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:31.347 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:31.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:31.352 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:31.453 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:31.490 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:31.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:31.493 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:33:31.515 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:31 smithi016 bash[75004]: cluster 2023-09-23T19:33:30.001795+0000 mgr.y (mgr.25325) 776 : cluster 0 pgmap v680: 89 pgs: 89 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 785 op/s 2023-09-23T19:33:31.594 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:31.619 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:31.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:31.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:31.725 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:31.754 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:31.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:31.757 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:33:31.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:31 smithi016 bash[77995]: cluster 2023-09-23T19:33:30.001795+0000 mgr.y (mgr.25325) 776 : cluster 0 pgmap v680: 89 pgs: 89 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 785 op/s 2023-09-23T19:33:31.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:31 smithi092 bash[58855]: cluster 2023-09-23T19:33:30.001795+0000 mgr.y (mgr.25325) 776 : cluster 0 pgmap v680: 89 pgs: 89 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 785 op/s 2023-09-23T19:33:31.857 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:32.255 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:32.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:32.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:32.360 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:32.399 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:32.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:32.404 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:32.505 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:32.535 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:32.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:32.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:32.566 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T19:33:32.642 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:32.680 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:32.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:32.684 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:32.785 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:32.820 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:32.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:32.829 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:32.930 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:32.966 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:32.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:32.970 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:33.071 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:33.102 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:33.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:33.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:33.205 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:33.232 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:33.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:33.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:33:33.338 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:33.366 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:33.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:33.369 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:33:33.470 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:33.509 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:33.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:33.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:33.616 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:33.643 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:33.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:33.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:33:33.746 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:33.768 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T19:33:33.776 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:33.780 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:33 smithi016 bash[75004]: cluster 2023-09-23T19:33:32.002315+0000 mgr.y (mgr.25325) 777 : cluster 0 pgmap v681: 89 pgs: 89 active+clean; 5.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 771 op/s 2023-09-23T19:33:33.780 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:33 smithi016 bash[77995]: cluster 2023-09-23T19:33:32.002315+0000 mgr.y (mgr.25325) 777 : cluster 0 pgmap v681: 89 pgs: 89 active+clean; 5.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 771 op/s 2023-09-23T19:33:33.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:33.781 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:33.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:33 smithi092 bash[58855]: cluster 2023-09-23T19:33:32.002315+0000 mgr.y (mgr.25325) 777 : cluster 0 pgmap v681: 89 pgs: 89 active+clean; 5.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 771 op/s 2023-09-23T19:33:33.882 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:33.917 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:33.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:33.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:33:34.024 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:34.058 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:34.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:34.061 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:33:34.162 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:34.189 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:34.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:34.193 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:34.293 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:34.326 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:34.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:34.330 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:34.430 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:34.463 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:34.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:34.467 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:33:34.568 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:34.600 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:34.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:34.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:34.705 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:34.731 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:34.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:34.737 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:33:34.837 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:34.863 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:34.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:34.865 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:34.966 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:34.999 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:35.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:35.003 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:35.104 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:35.128 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:35.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:35.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:33:35.233 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:35.266 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:35.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:35.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:33:35.374 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:35.415 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:35.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:35.419 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:33:35.520 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:35.523 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:35 smithi092 bash[58855]: cluster 2023-09-23T19:33:34.002872+0000 mgr.y (mgr.25325) 778 : cluster 0 pgmap v682: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 768 op/s 2023-09-23T19:33:35.554 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:35.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:35.561 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:33:35.641 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:35 smithi016 bash[77995]: cluster 2023-09-23T19:33:34.002872+0000 mgr.y (mgr.25325) 778 : cluster 0 pgmap v682: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 768 op/s 2023-09-23T19:33:35.642 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:35 smithi016 bash[75004]: cluster 2023-09-23T19:33:34.002872+0000 mgr.y (mgr.25325) 778 : cluster 0 pgmap v682: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 768 op/s 2023-09-23T19:33:35.661 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:35.697 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:35.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:35.701 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:33:35.802 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:35.829 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:35.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:35.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:35.935 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:35.960 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:35.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:35.966 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:36.035 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:33:35 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:33:35] "GET /metrics HTTP/1.1" 200 36240 "" "Prometheus/2.33.4" 2023-09-23T19:33:36.066 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:36.091 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:36.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:36.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:36.195 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:36.238 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:36.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:36.242 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:33:36.343 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:36.364 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:36.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:36.372 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:33:36.473 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:36.505 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:36.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:36.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:36.610 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:36.636 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:36.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:36.642 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:36.743 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:36.778 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:36.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:36.784 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:33:36.885 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:36.914 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:36.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:36.917 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:33:37.018 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:37.046 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:37.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:37.049 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:33:37.149 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:37.193 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:37.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:37.200 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:33:37.300 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:37.329 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:37.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:37.332 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:37.433 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:37.461 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:37.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:37.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:37.566 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:37.574 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:37 smithi016 bash[75004]: cluster 2023-09-23T19:33:36.003538+0000 mgr.y (mgr.25325) 779 : cluster 0 pgmap v683: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 764 op/s 2023-09-23T19:33:37.574 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:37 smithi016 bash[77995]: cluster 2023-09-23T19:33:36.003538+0000 mgr.y (mgr.25325) 779 : cluster 0 pgmap v683: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 764 op/s 2023-09-23T19:33:37.591 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:37.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:37.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:37.695 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:37.741 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:37.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:37.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:33:37.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:37 smithi092 bash[58855]: cluster 2023-09-23T19:33:36.003538+0000 mgr.y (mgr.25325) 779 : cluster 0 pgmap v683: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 764 op/s 2023-09-23T19:33:37.845 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:37.881 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:37.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:37.884 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:33:37.953 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:33:37.985 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:38.000 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:33:38.017 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:38.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:38.025 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:33:38.042 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:33:38.087 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:33:38.126 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:38.135 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:33:38.158 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:38.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:38.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:38.182 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:33:38.220 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:33:38.264 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:38.271 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:33:38.277 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:33:38.277 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:33:38.277 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:33:38.277 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:33:38.277 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:33:38.277 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:33:38.277 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:33:38.277 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:33:38.297 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:38.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:38.300 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:33:38.308 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-09-23T19:33:38.310 INFO:teuthology.orchestra.run.smithi016.stderr:dumped all 2023-09-23T19:33:38.401 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:38.432 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:38.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:38.436 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:38.537 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:38.565 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:38.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:38.569 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:33:38.669 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:38.694 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:38.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:38.698 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:33:38.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:38 smithi016 bash[75004]: audit 2023-09-23T19:33:37.886447+0000 mon.c (mon.2) 146 : audit 0 from='client.? 172.21.15.16:0/676700517' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:33:38.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:38 smithi016 bash[77995]: audit 2023-09-23T19:33:37.886447+0000 mon.c (mon.2) 146 : audit 0 from='client.? 172.21.15.16:0/676700517' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:33:38.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:38 smithi092 bash[58855]: audit 2023-09-23T19:33:37.886447+0000 mon.c (mon.2) 146 : audit 0 from='client.? 172.21.15.16:0/676700517' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:33:38.799 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:38.838 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:38.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:38.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:38.943 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:38.968 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:38.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:38.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:39.074 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:39.115 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:39.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:39.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:33:39.218 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:39.246 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:39.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:39.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:39.352 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:39.432 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:39.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:39.434 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:39.535 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:39.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:39 smithi016 bash[75004]: cluster 2023-09-23T19:33:38.004130+0000 mgr.y (mgr.25325) 780 : cluster 0 pgmap v684: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 757 op/s 2023-09-23T19:33:39.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:39 smithi016 bash[75004]: audit 2023-09-23T19:33:38.284536+0000 mgr.y (mgr.25325) 781 : audit 0 from='client.35835 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:33:39.569 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:39.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:39.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:33:39.677 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:39.686 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:39 smithi092 bash[58855]: cluster 2023-09-23T19:33:38.004130+0000 mgr.y (mgr.25325) 780 : cluster 0 pgmap v684: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 757 op/s 2023-09-23T19:33:39.687 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:39 smithi092 bash[58855]: audit 2023-09-23T19:33:38.284536+0000 mgr.y (mgr.25325) 781 : audit 0 from='client.35835 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:33:39.708 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:39.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:39.709 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:33:39.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:39 smithi016 bash[77995]: cluster 2023-09-23T19:33:38.004130+0000 mgr.y (mgr.25325) 780 : cluster 0 pgmap v684: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 757 op/s 2023-09-23T19:33:39.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:39 smithi016 bash[77995]: audit 2023-09-23T19:33:38.284536+0000 mgr.y (mgr.25325) 781 : audit 0 from='client.35835 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:33:39.810 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:39.843 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:39.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:39.850 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:39.950 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:39.992 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:39.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:39.996 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:33:40.097 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:40.128 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:40.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:40.129 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:40.195 INFO:teuthology.orchestra.run.smithi016.stdout:{"pg_ready":true,"pg_map":{"version":684,"stamp":"2023-09-23T19:33:38.003733+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5488580536,"num_objects":83765,"num_object_clones":0,"num_object_copies":251295,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":83765,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":133280,"num_write_kb":6938703,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128614,"ondisk_log_size":128614,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":19707220,"kb_used_data":16146964,"kb_used_omap":20,"kb_used_meta":3560107,"kb_avail":730253996,"statfs":{"total":767960285184,"available":747780091904,"internally_reserved":0,"allocated":16534491136,"data_stored":16530482319,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":3645550222},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-496697344,"num_objects":-7579,"num_object_clones":0,"num_object_copies":-22737,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-7579,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7579,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002926"},"pg_stats":[{"pgid":"3.1a","version":"59'1","reported_seq":111,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.355690+0000","last_change":"2023-09-23T19:27:24.274696+0000","last_active":"2023-09-23T19:33:11.355690+0000","last_peered":"2023-09-23T19:33:11.355690+0000","last_clean":"2023-09-23T19:33:11.355690+0000","last_became_active":"2023-09-23T19:27:24.273561+0000","last_became_peered":"2023-09-23T19:27:24.273561+0000","last_unstale":"2023-09-23T19:33:11.355690+0000","last_undegraded":"2023-09-23T19:33:11.355690+0000","last_fullsized":"2023-09-23T19:33:11.355690+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":83,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.338186+0000","last_change":"2023-09-23T19:32:49.731128+0000","last_active":"2023-09-23T19:33:11.338186+0000","last_peered":"2023-09-23T19:33:11.338186+0000","last_clean":"2023-09-23T19:33:11.338186+0000","last_became_active":"2023-09-23T19:32:49.730757+0000","last_became_peered":"2023-09-23T19:32:49.730757+0000","last_unstale":"2023-09-23T19:33:11.338186+0000","last_undegraded":"2023-09-23T19:33:11.338186+0000","last_fullsized":"2023-09-23T19:33:11.338186+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:40.183605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":71,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.340628+0000","last_change":"2023-09-23T19:28:07.890462+0000","last_active":"2023-09-23T19:33:11.340628+0000","last_peered":"2023-09-23T19:33:11.340628+0000","last_clean":"2023-09-23T19:33:11.340628+0000","last_became_active":"2023-09-23T19:28:07.890084+0000","last_became_peered":"2023-09-23T19:28:07.890084+0000","last_unstale":"2023-09-23T19:33:11.340628+0000","last_undegraded":"2023-09-23T19:33:11.340628+0000","last_fullsized":"2023-09-23T19:33:11.340628+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:50.458218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":87,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.343776+0000","last_change":"2023-09-23T19:28:07.884498+0000","last_active":"2023-09-23T19:33:11.343776+0000","last_peered":"2023-09-23T19:33:11.343776+0000","last_clean":"2023-09-23T19:33:11.343776+0000","last_became_active":"2023-09-23T19:28:07.884102+0000","last_became_peered":"2023-09-23T19:28:07.884102+0000","last_unstale":"2023-09-23T19:33:11.343776+0000","last_undegraded":"2023-09-23T19:33:11.343776+0000","last_fullsized":"2023-09-23T19:33:11.343776+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:05:26.348041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":99,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.343442+0000","last_change":"2023-09-23T19:32:49.729961+0000","last_active":"2023-09-23T19:33:11.343442+0000","last_peered":"2023-09-23T19:33:11.343442+0000","last_clean":"2023-09-23T19:33:11.343442+0000","last_became_active":"2023-09-23T19:32:49.729251+0000","last_became_peered":"2023-09-23T19:32:49.729251+0000","last_unstale":"2023-09-23T19:33:11.343442+0000","last_undegraded":"2023-09-23T19:33:11.343442+0000","last_fullsized":"2023-09-23T19:33:11.343442+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:20:56.595732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":94,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.346714+0000","last_change":"2023-09-23T19:22:42.150582+0000","last_active":"2023-09-23T19:33:11.346714+0000","last_peered":"2023-09-23T19:33:11.346714+0000","last_clean":"2023-09-23T19:33:11.346714+0000","last_became_active":"2023-09-23T19:22:42.150229+0000","last_became_peered":"2023-09-23T19:22:42.150229+0000","last_unstale":"2023-09-23T19:33:11.346714+0000","last_undegraded":"2023-09-23T19:33:11.346714+0000","last_fullsized":"2023-09-23T19:33:11.346714+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":86,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.344075+0000","last_change":"2023-09-23T19:28:07.887141+0000","last_active":"2023-09-23T19:33:11.344075+0000","last_peered":"2023-09-23T19:33:11.344075+0000","last_clean":"2023-09-23T19:33:11.344075+0000","last_became_active":"2023-09-23T19:28:07.886494+0000","last_became_peered":"2023-09-23T19:28:07.886494+0000","last_unstale":"2023-09-23T19:33:11.344075+0000","last_undegraded":"2023-09-23T19:33:11.344075+0000","last_fullsized":"2023-09-23T19:33:11.344075+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:24:07.615051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":83,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.337488+0000","last_change":"2023-09-23T19:32:49.729765+0000","last_active":"2023-09-23T19:33:11.337488+0000","last_peered":"2023-09-23T19:33:11.337488+0000","last_clean":"2023-09-23T19:33:11.337488+0000","last_became_active":"2023-09-23T19:32:49.729007+0000","last_became_peered":"2023-09-23T19:32:49.729007+0000","last_unstale":"2023-09-23T19:33:11.337488+0000","last_undegraded":"2023-09-23T19:33:11.337488+0000","last_fullsized":"2023-09-23T19:33:11.337488+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:59:31.183753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":96,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.355339+0000","last_change":"2023-09-23T19:22:42.151617+0000","last_active":"2023-09-23T19:33:11.355339+0000","last_peered":"2023-09-23T19:33:11.355339+0000","last_clean":"2023-09-23T19:33:11.355339+0000","last_became_active":"2023-09-23T19:22:42.151217+0000","last_became_peered":"2023-09-23T19:22:42.151217+0000","last_unstale":"2023-09-23T19:33:11.355339+0000","last_undegraded":"2023-09-23T19:33:11.355339+0000","last_fullsized":"2023-09-23T19:33:11.355339+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":69,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.354682+0000","last_change":"2023-09-23T19:32:49.736864+0000","last_active":"2023-09-23T19:33:11.354682+0000","last_peered":"2023-09-23T19:33:11.354682+0000","last_clean":"2023-09-23T19:33:11.354682+0000","last_became_active":"2023-09-23T19:32:49.736523+0000","last_became_peered":"2023-09-23T19:32:49.736523+0000","last_unstale":"2023-09-23T19:33:11.354682+0000","last_undegraded":"2023-09-23T19:33:11.354682+0000","last_fullsized":"2023-09-23T19:33:11.354682+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":98,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.360367+0000","last_change":"2023-09-23T19:28:07.892835+0000","last_active":"2023-09-23T19:33:11.360367+0000","last_peered":"2023-09-23T19:33:11.360367+0000","last_clean":"2023-09-23T19:33:11.360367+0000","last_became_active":"2023-09-23T19:28:07.892338+0000","last_became_peered":"2023-09-23T19:28:07.892338+0000","last_unstale":"2023-09-23T19:33:11.360367+0000","last_undegraded":"2023-09-23T19:33:11.360367+0000","last_fullsized":"2023-09-23T19:33:11.360367+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":71,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.346213+0000","last_change":"2023-09-23T19:27:24.268005+0000","last_active":"2023-09-23T19:33:11.346213+0000","last_peered":"2023-09-23T19:33:11.346213+0000","last_clean":"2023-09-23T19:33:11.346213+0000","last_became_active":"2023-09-23T19:27:24.267513+0000","last_became_peered":"2023-09-23T19:27:24.267513+0000","last_unstale":"2023-09-23T19:33:11.346213+0000","last_undegraded":"2023-09-23T19:33:11.346213+0000","last_fullsized":"2023-09-23T19:33:11.346213+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.961399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'23","reported_seq":103,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.357110+0000","last_change":"2023-09-23T19:05:52.107727+0000","last_active":"2023-09-23T19:33:11.357110+0000","last_peered":"2023-09-23T19:33:11.357110+0000","last_clean":"2023-09-23T19:33:11.357110+0000","last_became_active":"2023-09-23T19:05:52.107378+0000","last_became_peered":"2023-09-23T19:05:52.107378+0000","last_unstale":"2023-09-23T19:33:11.357110+0000","last_undegraded":"2023-09-23T19:33:11.357110+0000","last_fullsized":"2023-09-23T19:33:11.357110+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":94,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.356022+0000","last_change":"2023-09-23T19:22:42.148497+0000","last_active":"2023-09-23T19:33:11.356022+0000","last_peered":"2023-09-23T19:33:11.356022+0000","last_clean":"2023-09-23T19:33:11.356022+0000","last_became_active":"2023-09-23T19:22:42.148147+0000","last_became_peered":"2023-09-23T19:22:42.148147+0000","last_unstale":"2023-09-23T19:33:11.356022+0000","last_undegraded":"2023-09-23T19:33:11.356022+0000","last_fullsized":"2023-09-23T19:33:11.356022+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":97,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.354538+0000","last_change":"2023-09-23T19:27:24.273667+0000","last_active":"2023-09-23T19:33:11.354538+0000","last_peered":"2023-09-23T19:33:11.354538+0000","last_clean":"2023-09-23T19:33:11.354538+0000","last_became_active":"2023-09-23T19:27:24.273234+0000","last_became_peered":"2023-09-23T19:27:24.273234+0000","last_unstale":"2023-09-23T19:33:11.354538+0000","last_undegraded":"2023-09-23T19:33:11.354538+0000","last_fullsized":"2023-09-23T19:33:11.354538+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":83,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.336379+0000","last_change":"2023-09-23T19:32:49.730549+0000","last_active":"2023-09-23T19:33:11.336379+0000","last_peered":"2023-09-23T19:33:11.336379+0000","last_clean":"2023-09-23T19:33:11.336379+0000","last_became_active":"2023-09-23T19:32:49.730105+0000","last_became_peered":"2023-09-23T19:32:49.730105+0000","last_unstale":"2023-09-23T19:33:11.336379+0000","last_undegraded":"2023-09-23T19:33:11.336379+0000","last_fullsized":"2023-09-23T19:33:11.336379+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:49.751893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":86,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.343593+0000","last_change":"2023-09-23T19:28:07.884394+0000","last_active":"2023-09-23T19:33:11.343593+0000","last_peered":"2023-09-23T19:33:11.343593+0000","last_clean":"2023-09-23T19:33:11.343593+0000","last_became_active":"2023-09-23T19:28:07.883944+0000","last_became_peered":"2023-09-23T19:28:07.883944+0000","last_unstale":"2023-09-23T19:33:11.343593+0000","last_undegraded":"2023-09-23T19:33:11.343593+0000","last_fullsized":"2023-09-23T19:33:11.343593+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:08:17.608926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":95,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.356438+0000","last_change":"2023-09-23T19:28:07.891056+0000","last_active":"2023-09-23T19:33:11.356438+0000","last_peered":"2023-09-23T19:33:11.356438+0000","last_clean":"2023-09-23T19:33:11.356438+0000","last_became_active":"2023-09-23T19:28:07.890625+0000","last_became_peered":"2023-09-23T19:28:07.890625+0000","last_unstale":"2023-09-23T19:33:11.356438+0000","last_undegraded":"2023-09-23T19:33:11.356438+0000","last_fullsized":"2023-09-23T19:33:11.356438+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":80,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.354773+0000","last_change":"2023-09-23T19:05:52.114186+0000","last_active":"2023-09-23T19:33:11.354773+0000","last_peered":"2023-09-23T19:33:11.354773+0000","last_clean":"2023-09-23T19:33:11.354773+0000","last_became_active":"2023-09-23T19:05:52.113635+0000","last_became_peered":"2023-09-23T19:05:52.113635+0000","last_unstale":"2023-09-23T19:33:11.354773+0000","last_undegraded":"2023-09-23T19:33:11.354773+0000","last_fullsized":"2023-09-23T19:33:11.354773+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":86,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.343861+0000","last_change":"2023-09-23T19:28:07.884512+0000","last_active":"2023-09-23T19:33:11.343861+0000","last_peered":"2023-09-23T19:33:11.343861+0000","last_clean":"2023-09-23T19:33:11.343861+0000","last_became_active":"2023-09-23T19:28:07.884155+0000","last_became_peered":"2023-09-23T19:28:07.884155+0000","last_unstale":"2023-09-23T19:33:11.343861+0000","last_undegraded":"2023-09-23T19:33:11.343861+0000","last_fullsized":"2023-09-23T19:33:11.343861+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:10.654958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":103,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.350998+0000","last_change":"2023-09-23T19:18:09.010195+0000","last_active":"2023-09-23T19:33:11.350998+0000","last_peered":"2023-09-23T19:33:11.350998+0000","last_clean":"2023-09-23T19:33:11.350998+0000","last_became_active":"2023-09-23T19:18:09.009687+0000","last_became_peered":"2023-09-23T19:18:09.009687+0000","last_unstale":"2023-09-23T19:33:11.350998+0000","last_undegraded":"2023-09-23T19:33:11.350998+0000","last_fullsized":"2023-09-23T19:33:11.350998+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":91,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.352107+0000","last_change":"2023-09-23T19:28:07.885265+0000","last_active":"2023-09-23T19:33:11.352107+0000","last_peered":"2023-09-23T19:33:11.352107+0000","last_clean":"2023-09-23T19:33:11.352107+0000","last_became_active":"2023-09-23T19:28:07.884353+0000","last_became_peered":"2023-09-23T19:28:07.884353+0000","last_unstale":"2023-09-23T19:33:11.352107+0000","last_undegraded":"2023-09-23T19:33:11.352107+0000","last_fullsized":"2023-09-23T19:33:11.352107+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:56.325352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"129'8426","reported_seq":8446,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T19:33:37.175465+0000","last_change":"2023-09-23T19:31:12.572210+0000","last_active":"2023-09-23T19:33:37.175465+0000","last_peered":"2023-09-23T19:33:37.175465+0000","last_clean":"2023-09-23T19:33:37.175465+0000","last_became_active":"2023-09-23T19:31:12.571806+0000","last_became_peered":"2023-09-23T19:31:12.571806+0000","last_unstale":"2023-09-23T19:33:37.175465+0000","last_undegraded":"2023-09-23T19:33:37.175465+0000","last_fullsized":"2023-09-23T19:33:37.175465+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_clean_scrub_stamp":"2023-09-23T19:31:11.546138+0000","objects_scrubbed":0,"log_size":8426,"log_dups_size":0,"ondisk_log_size":8426,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:41:17.174555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336068608,"num_objects":5128,"num_object_clones":0,"num_object_copies":15384,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5128,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8426,"num_write_kb":433728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":96,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.363613+0000","last_change":"2023-09-23T19:18:09.035116+0000","last_active":"2023-09-23T19:33:11.363613+0000","last_peered":"2023-09-23T19:33:11.363613+0000","last_clean":"2023-09-23T19:33:11.363613+0000","last_became_active":"2023-09-23T19:18:09.034416+0000","last_became_peered":"2023-09-23T19:18:09.034416+0000","last_unstale":"2023-09-23T19:33:11.363613+0000","last_undegraded":"2023-09-23T19:33:11.363613+0000","last_fullsized":"2023-09-23T19:33:11.363613+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":80,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.356229+0000","last_change":"2023-09-23T19:05:52.789323+0000","last_active":"2023-09-23T19:33:11.356229+0000","last_peered":"2023-09-23T19:33:11.356229+0000","last_clean":"2023-09-23T19:33:11.356229+0000","last_became_active":"2023-09-23T19:05:52.788820+0000","last_became_peered":"2023-09-23T19:05:52.788820+0000","last_unstale":"2023-09-23T19:33:11.356229+0000","last_undegraded":"2023-09-23T19:33:11.356229+0000","last_fullsized":"2023-09-23T19:33:11.356229+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"129'8414","reported_seq":8434,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T19:33:36.348016+0000","last_change":"2023-09-23T19:31:12.589012+0000","last_active":"2023-09-23T19:33:36.348016+0000","last_peered":"2023-09-23T19:33:36.348016+0000","last_clean":"2023-09-23T19:33:36.348016+0000","last_became_active":"2023-09-23T19:31:12.588473+0000","last_became_peered":"2023-09-23T19:31:12.588473+0000","last_unstale":"2023-09-23T19:33:36.348016+0000","last_undegraded":"2023-09-23T19:33:36.348016+0000","last_fullsized":"2023-09-23T19:33:36.348016+0000","mapping_epoch":119,"log_start":"122'1000","ondisk_log_start":"122'1000","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_clean_scrub_stamp":"2023-09-23T19:31:11.546138+0000","objects_scrubbed":0,"log_size":7414,"log_dups_size":0,"ondisk_log_size":7414,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:47:07.601624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":343408640,"num_objects":5240,"num_object_clones":0,"num_object_copies":15720,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5240,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8414,"num_write_kb":436928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":97,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.358061+0000","last_change":"2023-09-23T19:22:42.151134+0000","last_active":"2023-09-23T19:33:11.358061+0000","last_peered":"2023-09-23T19:33:11.358061+0000","last_clean":"2023-09-23T19:33:11.358061+0000","last_became_active":"2023-09-23T19:22:42.150553+0000","last_became_peered":"2023-09-23T19:22:42.150553+0000","last_unstale":"2023-09-23T19:33:11.358061+0000","last_undegraded":"2023-09-23T19:33:11.358061+0000","last_fullsized":"2023-09-23T19:33:11.358061+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":109,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.355511+0000","last_change":"2023-09-23T19:28:07.899262+0000","last_active":"2023-09-23T19:33:11.355511+0000","last_peered":"2023-09-23T19:33:11.355511+0000","last_clean":"2023-09-23T19:33:11.355511+0000","last_became_active":"2023-09-23T19:28:07.898848+0000","last_became_peered":"2023-09-23T19:28:07.898848+0000","last_unstale":"2023-09-23T19:33:11.355511+0000","last_undegraded":"2023-09-23T19:33:11.355511+0000","last_fullsized":"2023-09-23T19:33:11.355511+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"129'8310","reported_seq":8328,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T19:33:35.154464+0000","last_change":"2023-09-23T19:31:12.576224+0000","last_active":"2023-09-23T19:33:35.154464+0000","last_peered":"2023-09-23T19:33:35.154464+0000","last_clean":"2023-09-23T19:33:35.154464+0000","last_became_active":"2023-09-23T19:31:12.575807+0000","last_became_peered":"2023-09-23T19:31:12.575807+0000","last_unstale":"2023-09-23T19:33:35.154464+0000","last_undegraded":"2023-09-23T19:33:35.154464+0000","last_fullsized":"2023-09-23T19:33:35.154464+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_clean_scrub_stamp":"2023-09-23T19:31:11.546138+0000","objects_scrubbed":0,"log_size":8310,"log_dups_size":0,"ondisk_log_size":8310,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":348389376,"num_objects":5316,"num_object_clones":0,"num_object_copies":15948,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5316,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8310,"num_write_kb":436032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":81,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.360268+0000","last_change":"2023-09-23T19:05:52.114108+0000","last_active":"2023-09-23T19:33:11.360268+0000","last_peered":"2023-09-23T19:33:11.360268+0000","last_clean":"2023-09-23T19:33:11.360268+0000","last_became_active":"2023-09-23T19:05:52.113609+0000","last_became_peered":"2023-09-23T19:05:52.113609+0000","last_unstale":"2023-09-23T19:33:11.360268+0000","last_undegraded":"2023-09-23T19:33:11.360268+0000","last_fullsized":"2023-09-23T19:33:11.360268+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":94,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.346035+0000","last_change":"2023-09-23T19:22:42.156825+0000","last_active":"2023-09-23T19:33:11.346035+0000","last_peered":"2023-09-23T19:33:11.346035+0000","last_clean":"2023-09-23T19:33:11.346035+0000","last_became_active":"2023-09-23T19:22:42.156474+0000","last_became_peered":"2023-09-23T19:22:42.156474+0000","last_unstale":"2023-09-23T19:33:11.346035+0000","last_undegraded":"2023-09-23T19:33:11.346035+0000","last_fullsized":"2023-09-23T19:33:11.346035+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"129'8083","reported_seq":8100,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T19:33:35.121003+0000","last_change":"2023-09-23T19:31:12.589020+0000","last_active":"2023-09-23T19:33:35.121003+0000","last_peered":"2023-09-23T19:33:35.121003+0000","last_clean":"2023-09-23T19:33:35.121003+0000","last_became_active":"2023-09-23T19:31:12.588673+0000","last_became_peered":"2023-09-23T19:31:12.588673+0000","last_unstale":"2023-09-23T19:33:35.121003+0000","last_undegraded":"2023-09-23T19:33:35.121003+0000","last_fullsized":"2023-09-23T19:33:35.121003+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_clean_scrub_stamp":"2023-09-23T19:31:11.546138+0000","objects_scrubbed":0,"log_size":8083,"log_dups_size":0,"ondisk_log_size":8083,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":343605248,"num_objects":5243,"num_object_clones":0,"num_object_copies":15729,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5243,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8083,"num_write_kb":426432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":96,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.357039+0000","last_change":"2023-09-23T19:28:07.891536+0000","last_active":"2023-09-23T19:33:11.357039+0000","last_peered":"2023-09-23T19:33:11.357039+0000","last_clean":"2023-09-23T19:33:11.357039+0000","last_became_active":"2023-09-23T19:28:07.891153+0000","last_became_peered":"2023-09-23T19:28:07.891153+0000","last_unstale":"2023-09-23T19:33:11.357039+0000","last_undegraded":"2023-09-23T19:33:11.357039+0000","last_fullsized":"2023-09-23T19:33:11.357039+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":86,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.345930+0000","last_change":"2023-09-23T19:22:42.135572+0000","last_active":"2023-09-23T19:33:11.345930+0000","last_peered":"2023-09-23T19:33:11.345930+0000","last_clean":"2023-09-23T19:33:11.345930+0000","last_became_active":"2023-09-23T19:22:42.135237+0000","last_became_peered":"2023-09-23T19:22:42.135237+0000","last_unstale":"2023-09-23T19:33:11.345930+0000","last_undegraded":"2023-09-23T19:33:11.345930+0000","last_fullsized":"2023-09-23T19:33:11.345930+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:35.838718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"129'8139","reported_seq":8166,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T19:33:34.446403+0000","last_change":"2023-09-23T19:32:49.767399+0000","last_active":"2023-09-23T19:33:34.446403+0000","last_peered":"2023-09-23T19:33:34.446403+0000","last_clean":"2023-09-23T19:33:34.446403+0000","last_became_active":"2023-09-23T19:32:49.735026+0000","last_became_peered":"2023-09-23T19:32:49.735026+0000","last_unstale":"2023-09-23T19:33:34.446403+0000","last_undegraded":"2023-09-23T19:33:34.446403+0000","last_fullsized":"2023-09-23T19:33:34.446403+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_clean_scrub_stamp":"2023-09-23T19:31:11.546138+0000","objects_scrubbed":0,"log_size":8139,"log_dups_size":0,"ondisk_log_size":8139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":340066304,"num_objects":5189,"num_object_clones":0,"num_object_copies":15567,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5189,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8139,"num_write_kb":426496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":33,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.345848+0000","last_change":"2023-09-23T19:32:49.728589+0000","last_active":"2023-09-23T19:33:11.345848+0000","last_peered":"2023-09-23T19:33:11.345848+0000","last_clean":"2023-09-23T19:33:11.345848+0000","last_became_active":"2023-09-23T19:32:49.728037+0000","last_became_peered":"2023-09-23T19:32:49.728037+0000","last_unstale":"2023-09-23T19:33:11.345848+0000","last_undegraded":"2023-09-23T19:33:11.345848+0000","last_fullsized":"2023-09-23T19:33:11.345848+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:26:00.957969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":94,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.357183+0000","last_change":"2023-09-23T19:28:07.896803+0000","last_active":"2023-09-23T19:33:11.357183+0000","last_peered":"2023-09-23T19:33:11.357183+0000","last_clean":"2023-09-23T19:33:11.357183+0000","last_became_active":"2023-09-23T19:28:07.896480+0000","last_became_peered":"2023-09-23T19:28:07.896480+0000","last_unstale":"2023-09-23T19:33:11.357183+0000","last_undegraded":"2023-09-23T19:33:11.357183+0000","last_fullsized":"2023-09-23T19:33:11.357183+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"129'8342","reported_seq":8362,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T19:33:36.343495+0000","last_change":"2023-09-23T19:31:12.591859+0000","last_active":"2023-09-23T19:33:36.343495+0000","last_peered":"2023-09-23T19:33:36.343495+0000","last_clean":"2023-09-23T19:33:36.343495+0000","last_became_active":"2023-09-23T19:31:12.591505+0000","last_became_peered":"2023-09-23T19:31:12.591505+0000","last_unstale":"2023-09-23T19:33:36.343495+0000","last_undegraded":"2023-09-23T19:33:36.343495+0000","last_fullsized":"2023-09-23T19:33:36.343495+0000","mapping_epoch":119,"log_start":"122'1000","ondisk_log_start":"122'1000","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_clean_scrub_stamp":"2023-09-23T19:31:11.546138+0000","objects_scrubbed":0,"log_size":7342,"log_dups_size":0,"ondisk_log_size":7342,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:33:29.412181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":342622208,"num_objects":5228,"num_object_clones":0,"num_object_copies":15684,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5228,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8342,"num_write_kb":434240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":95,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.346237+0000","last_change":"2023-09-23T19:22:42.156582+0000","last_active":"2023-09-23T19:33:11.346237+0000","last_peered":"2023-09-23T19:33:11.346237+0000","last_clean":"2023-09-23T19:33:11.346237+0000","last_became_active":"2023-09-23T19:22:42.156251+0000","last_became_peered":"2023-09-23T19:22:42.156251+0000","last_unstale":"2023-09-23T19:33:11.346237+0000","last_undegraded":"2023-09-23T19:33:11.346237+0000","last_fullsized":"2023-09-23T19:33:11.346237+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":109,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.356261+0000","last_change":"2023-09-23T19:27:24.273971+0000","last_active":"2023-09-23T19:33:11.356261+0000","last_peered":"2023-09-23T19:33:11.356261+0000","last_clean":"2023-09-23T19:33:11.356261+0000","last_became_active":"2023-09-23T19:27:24.273491+0000","last_became_peered":"2023-09-23T19:27:24.273491+0000","last_unstale":"2023-09-23T19:33:11.356261+0000","last_undegraded":"2023-09-23T19:33:11.356261+0000","last_fullsized":"2023-09-23T19:33:11.356261+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"129'8260","reported_seq":8278,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T19:33:34.875409+0000","last_change":"2023-09-23T19:31:12.573917+0000","last_active":"2023-09-23T19:33:34.875409+0000","last_peered":"2023-09-23T19:33:34.875409+0000","last_clean":"2023-09-23T19:33:34.875409+0000","last_became_active":"2023-09-23T19:31:12.573499+0000","last_became_peered":"2023-09-23T19:31:12.573499+0000","last_unstale":"2023-09-23T19:33:34.875409+0000","last_undegraded":"2023-09-23T19:33:34.875409+0000","last_fullsized":"2023-09-23T19:33:34.875409+0000","mapping_epoch":119,"log_start":"122'500","ondisk_log_start":"122'500","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_clean_scrub_stamp":"2023-09-23T19:31:11.546138+0000","objects_scrubbed":0,"log_size":7760,"log_dups_size":0,"ondisk_log_size":7760,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":345899008,"num_objects":5278,"num_object_clones":0,"num_object_copies":15834,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5278,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8260,"num_write_kb":433216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":95,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.355294+0000","last_change":"2023-09-23T19:27:24.274698+0000","last_active":"2023-09-23T19:33:11.355294+0000","last_peered":"2023-09-23T19:33:11.355294+0000","last_clean":"2023-09-23T19:33:11.355294+0000","last_became_active":"2023-09-23T19:27:24.273264+0000","last_became_peered":"2023-09-23T19:27:24.273264+0000","last_unstale":"2023-09-23T19:33:11.355294+0000","last_undegraded":"2023-09-23T19:33:11.355294+0000","last_fullsized":"2023-09-23T19:33:11.355294+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":94,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.359754+0000","last_change":"2023-09-23T19:22:42.149670+0000","last_active":"2023-09-23T19:33:11.359754+0000","last_peered":"2023-09-23T19:33:11.359754+0000","last_clean":"2023-09-23T19:33:11.359754+0000","last_became_active":"2023-09-23T19:22:42.149022+0000","last_became_peered":"2023-09-23T19:22:42.149022+0000","last_unstale":"2023-09-23T19:33:11.359754+0000","last_undegraded":"2023-09-23T19:33:11.359754+0000","last_fullsized":"2023-09-23T19:33:11.359754+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"129'8245","reported_seq":8271,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T19:33:36.253651+0000","last_change":"2023-09-23T19:32:49.742544+0000","last_active":"2023-09-23T19:33:36.253651+0000","last_peered":"2023-09-23T19:33:36.253651+0000","last_clean":"2023-09-23T19:33:36.253651+0000","last_became_active":"2023-09-23T19:32:49.728515+0000","last_became_peered":"2023-09-23T19:32:49.728515+0000","last_unstale":"2023-09-23T19:33:36.253651+0000","last_undegraded":"2023-09-23T19:33:36.253651+0000","last_fullsized":"2023-09-23T19:33:36.253651+0000","mapping_epoch":126,"log_start":"122'900","ondisk_log_start":"122'900","created":119,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_clean_scrub_stamp":"2023-09-23T19:31:11.546138+0000","objects_scrubbed":0,"log_size":7345,"log_dups_size":0,"ondisk_log_size":7345,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:01.357892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335085568,"num_objects":5113,"num_object_clones":0,"num_object_copies":15339,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5113,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8245,"num_write_kb":427456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":84,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.337353+0000","last_change":"2023-09-23T19:32:49.729244+0000","last_active":"2023-09-23T19:33:11.337353+0000","last_peered":"2023-09-23T19:33:11.337353+0000","last_clean":"2023-09-23T19:33:11.337353+0000","last_became_active":"2023-09-23T19:32:49.728810+0000","last_became_peered":"2023-09-23T19:32:49.728810+0000","last_unstale":"2023-09-23T19:33:11.337353+0000","last_undegraded":"2023-09-23T19:33:11.337353+0000","last_fullsized":"2023-09-23T19:33:11.337353+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:57.540826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":71,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.335634+0000","last_change":"2023-09-23T19:28:07.889095+0000","last_active":"2023-09-23T19:33:11.335634+0000","last_peered":"2023-09-23T19:33:11.335634+0000","last_clean":"2023-09-23T19:33:11.335634+0000","last_became_active":"2023-09-23T19:28:07.888742+0000","last_became_peered":"2023-09-23T19:28:07.888742+0000","last_unstale":"2023-09-23T19:33:11.335634+0000","last_undegraded":"2023-09-23T19:33:11.335634+0000","last_fullsized":"2023-09-23T19:33:11.335634+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:43.929488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"129'8241","reported_seq":8258,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T19:33:35.148742+0000","last_change":"2023-09-23T19:31:12.577456+0000","last_active":"2023-09-23T19:33:35.148742+0000","last_peered":"2023-09-23T19:33:35.148742+0000","last_clean":"2023-09-23T19:33:35.148742+0000","last_became_active":"2023-09-23T19:31:12.576998+0000","last_became_peered":"2023-09-23T19:31:12.576998+0000","last_unstale":"2023-09-23T19:33:35.148742+0000","last_undegraded":"2023-09-23T19:33:35.148742+0000","last_fullsized":"2023-09-23T19:33:35.148742+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_clean_scrub_stamp":"2023-09-23T19:31:11.546138+0000","objects_scrubbed":0,"log_size":8241,"log_dups_size":0,"ondisk_log_size":8241,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":343736320,"num_objects":5245,"num_object_clones":0,"num_object_copies":15735,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5245,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8241,"num_write_kb":431552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":105,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.336313+0000","last_change":"2023-09-23T19:22:42.143289+0000","last_active":"2023-09-23T19:33:11.336313+0000","last_peered":"2023-09-23T19:33:11.336313+0000","last_clean":"2023-09-23T19:33:11.336313+0000","last_became_active":"2023-09-23T19:22:42.142656+0000","last_became_peered":"2023-09-23T19:22:42.142656+0000","last_unstale":"2023-09-23T19:33:11.336313+0000","last_undegraded":"2023-09-23T19:33:11.336313+0000","last_fullsized":"2023-09-23T19:33:11.336313+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:38.467441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":111,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.356512+0000","last_change":"2023-09-23T19:28:07.890623+0000","last_active":"2023-09-23T19:33:11.356512+0000","last_peered":"2023-09-23T19:33:11.356512+0000","last_clean":"2023-09-23T19:33:11.356512+0000","last_became_active":"2023-09-23T19:28:07.890148+0000","last_became_peered":"2023-09-23T19:28:07.890148+0000","last_unstale":"2023-09-23T19:33:11.356512+0000","last_undegraded":"2023-09-23T19:33:11.356512+0000","last_fullsized":"2023-09-23T19:33:11.356512+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":84,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T19:33:28.540376+0000","last_change":"2023-09-23T19:27:24.269080+0000","last_active":"2023-09-23T19:33:28.540376+0000","last_peered":"2023-09-23T19:33:28.540376+0000","last_clean":"2023-09-23T19:33:28.540376+0000","last_became_active":"2023-09-23T19:27:24.268748+0000","last_became_peered":"2023-09-23T19:27:24.268748+0000","last_unstale":"2023-09-23T19:33:28.540376+0000","last_undegraded":"2023-09-23T19:33:28.540376+0000","last_fullsized":"2023-09-23T19:33:28.540376+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:49.014902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.9","version":"129'8454","reported_seq":8474,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T19:33:37.178280+0000","last_change":"2023-09-23T19:31:12.572956+0000","last_active":"2023-09-23T19:33:37.178280+0000","last_peered":"2023-09-23T19:33:37.178280+0000","last_clean":"2023-09-23T19:33:37.178280+0000","last_became_active":"2023-09-23T19:31:12.572413+0000","last_became_peered":"2023-09-23T19:31:12.572413+0000","last_unstale":"2023-09-23T19:33:37.178280+0000","last_undegraded":"2023-09-23T19:33:37.178280+0000","last_fullsized":"2023-09-23T19:33:37.178280+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_clean_scrub_stamp":"2023-09-23T19:31:11.546138+0000","objects_scrubbed":0,"log_size":8454,"log_dups_size":0,"ondisk_log_size":8454,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:46:33.130893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":344850432,"num_objects":5262,"num_object_clones":0,"num_object_copies":15786,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5262,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8454,"num_write_kb":438912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":100,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.368025+0000","last_change":"2023-09-23T19:22:42.153647+0000","last_active":"2023-09-23T19:33:11.368025+0000","last_peered":"2023-09-23T19:33:11.368025+0000","last_clean":"2023-09-23T19:33:11.368025+0000","last_became_active":"2023-09-23T19:22:42.153317+0000","last_became_peered":"2023-09-23T19:22:42.153317+0000","last_unstale":"2023-09-23T19:33:11.368025+0000","last_undegraded":"2023-09-23T19:33:11.368025+0000","last_fullsized":"2023-09-23T19:33:11.368025+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":95,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.355973+0000","last_change":"2023-09-23T19:18:09.017525+0000","last_active":"2023-09-23T19:33:11.355973+0000","last_peered":"2023-09-23T19:33:11.355973+0000","last_clean":"2023-09-23T19:33:11.355973+0000","last_became_active":"2023-09-23T19:18:09.017193+0000","last_became_peered":"2023-09-23T19:18:09.017193+0000","last_unstale":"2023-09-23T19:33:11.355973+0000","last_undegraded":"2023-09-23T19:33:11.355973+0000","last_fullsized":"2023-09-23T19:33:11.355973+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":87,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.336248+0000","last_change":"2023-09-23T19:28:07.875796+0000","last_active":"2023-09-23T19:33:11.336248+0000","last_peered":"2023-09-23T19:33:11.336248+0000","last_clean":"2023-09-23T19:33:11.336248+0000","last_became_active":"2023-09-23T19:28:07.875310+0000","last_became_peered":"2023-09-23T19:28:07.875310+0000","last_unstale":"2023-09-23T19:33:11.336248+0000","last_undegraded":"2023-09-23T19:33:11.336248+0000","last_fullsized":"2023-09-23T19:33:11.336248+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:44:13.379633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"129'8561","reported_seq":8579,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T19:33:35.091041+0000","last_change":"2023-09-23T19:31:12.576188+0000","last_active":"2023-09-23T19:33:35.091041+0000","last_peered":"2023-09-23T19:33:35.091041+0000","last_clean":"2023-09-23T19:33:35.091041+0000","last_became_active":"2023-09-23T19:31:12.575832+0000","last_became_peered":"2023-09-23T19:31:12.575832+0000","last_unstale":"2023-09-23T19:33:35.091041+0000","last_undegraded":"2023-09-23T19:33:35.091041+0000","last_fullsized":"2023-09-23T19:33:35.091041+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_clean_scrub_stamp":"2023-09-23T19:31:11.546138+0000","objects_scrubbed":0,"log_size":8561,"log_dups_size":0,"ondisk_log_size":8561,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":356319232,"num_objects":5437,"num_object_clones":0,"num_object_copies":16311,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5437,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8561,"num_write_kb":447936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":104,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T19:33:28.540432+0000","last_change":"2023-09-23T19:28:07.878688+0000","last_active":"2023-09-23T19:33:28.540432+0000","last_peered":"2023-09-23T19:33:28.540432+0000","last_clean":"2023-09-23T19:33:28.540432+0000","last_became_active":"2023-09-23T19:28:07.878309+0000","last_became_peered":"2023-09-23T19:28:07.878309+0000","last_unstale":"2023-09-23T19:33:28.540432+0000","last_undegraded":"2023-09-23T19:33:28.540432+0000","last_fullsized":"2023-09-23T19:33:28.540432+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:49:06.202657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":72,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.337256+0000","last_change":"2023-09-23T19:27:24.267897+0000","last_active":"2023-09-23T19:33:11.337256+0000","last_peered":"2023-09-23T19:33:11.337256+0000","last_clean":"2023-09-23T19:33:11.337256+0000","last_became_active":"2023-09-23T19:27:24.267535+0000","last_became_peered":"2023-09-23T19:27:24.267535+0000","last_unstale":"2023-09-23T19:33:11.337256+0000","last_undegraded":"2023-09-23T19:33:11.337256+0000","last_fullsized":"2023-09-23T19:33:11.337256+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:13:57.447808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":100,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T19:33:28.534151+0000","last_change":"2023-09-23T19:22:42.143733+0000","last_active":"2023-09-23T19:33:28.534151+0000","last_peered":"2023-09-23T19:33:28.534151+0000","last_clean":"2023-09-23T19:33:28.534151+0000","last_became_active":"2023-09-23T19:22:42.143077+0000","last_became_peered":"2023-09-23T19:22:42.143077+0000","last_unstale":"2023-09-23T19:33:28.534151+0000","last_undegraded":"2023-09-23T19:33:28.534151+0000","last_fullsized":"2023-09-23T19:33:28.534151+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:47:33.086994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"129'8249","reported_seq":8276,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T19:33:37.164479+0000","last_change":"2023-09-23T19:32:49.743645+0000","last_active":"2023-09-23T19:33:37.164479+0000","last_peered":"2023-09-23T19:33:37.164479+0000","last_clean":"2023-09-23T19:33:37.164479+0000","last_became_active":"2023-09-23T19:32:49.729828+0000","last_became_peered":"2023-09-23T19:32:49.729828+0000","last_unstale":"2023-09-23T19:33:37.164479+0000","last_undegraded":"2023-09-23T19:33:37.164479+0000","last_fullsized":"2023-09-23T19:33:37.164479+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_clean_scrub_stamp":"2023-09-23T19:31:11.546138+0000","objects_scrubbed":0,"log_size":8249,"log_dups_size":0,"ondisk_log_size":8249,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:44:38.489342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":329449472,"num_objects":5027,"num_object_clones":0,"num_object_copies":15081,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5027,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8249,"num_write_kb":424832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":100,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.355258+0000","last_change":"2023-09-23T19:27:24.271520+0000","last_active":"2023-09-23T19:33:11.355258+0000","last_peered":"2023-09-23T19:33:11.355258+0000","last_clean":"2023-09-23T19:33:11.355258+0000","last_became_active":"2023-09-23T19:27:24.271165+0000","last_became_peered":"2023-09-23T19:27:24.271165+0000","last_unstale":"2023-09-23T19:33:11.355258+0000","last_undegraded":"2023-09-23T19:33:11.355258+0000","last_fullsized":"2023-09-23T19:33:11.355258+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":331,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.353801+0000","last_change":"2023-09-23T19:18:09.016297+0000","last_active":"2023-09-23T19:33:11.353801+0000","last_peered":"2023-09-23T19:33:11.353801+0000","last_clean":"2023-09-23T19:33:11.353801+0000","last_became_active":"2023-09-23T19:18:09.015436+0000","last_became_peered":"2023-09-23T19:18:09.015436+0000","last_unstale":"2023-09-23T19:33:11.353801+0000","last_undegraded":"2023-09-23T19:33:11.353801+0000","last_fullsized":"2023-09-23T19:33:11.353801+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:59:33.840751+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:59:33.840751+0000","last_clean_scrub_stamp":"2023-09-23T18:59:33.840751+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":69,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.367953+0000","last_change":"2023-09-23T19:32:49.734148+0000","last_active":"2023-09-23T19:33:11.367953+0000","last_peered":"2023-09-23T19:33:11.367953+0000","last_clean":"2023-09-23T19:33:11.367953+0000","last_became_active":"2023-09-23T19:32:49.733512+0000","last_became_peered":"2023-09-23T19:32:49.733512+0000","last_unstale":"2023-09-23T19:33:11.367953+0000","last_undegraded":"2023-09-23T19:33:11.367953+0000","last_fullsized":"2023-09-23T19:33:11.367953+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":79,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.346472+0000","last_change":"2023-09-23T19:05:52.786771+0000","last_active":"2023-09-23T19:33:11.346472+0000","last_peered":"2023-09-23T19:33:11.346472+0000","last_clean":"2023-09-23T19:33:11.346472+0000","last_became_active":"2023-09-23T19:05:52.785797+0000","last_became_peered":"2023-09-23T19:05:52.785797+0000","last_unstale":"2023-09-23T19:33:11.346472+0000","last_undegraded":"2023-09-23T19:33:11.346472+0000","last_fullsized":"2023-09-23T19:33:11.346472+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"129'8243","reported_seq":8263,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T19:33:37.163799+0000","last_change":"2023-09-23T19:31:12.591240+0000","last_active":"2023-09-23T19:33:37.163799+0000","last_peered":"2023-09-23T19:33:37.163799+0000","last_clean":"2023-09-23T19:33:37.163799+0000","last_became_active":"2023-09-23T19:31:12.590654+0000","last_became_peered":"2023-09-23T19:31:12.590654+0000","last_unstale":"2023-09-23T19:33:37.163799+0000","last_undegraded":"2023-09-23T19:33:37.163799+0000","last_fullsized":"2023-09-23T19:33:37.163799+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_clean_scrub_stamp":"2023-09-23T19:31:11.546138+0000","objects_scrubbed":0,"log_size":8243,"log_dups_size":0,"ondisk_log_size":8243,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:03.681405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335347712,"num_objects":5117,"num_object_clones":0,"num_object_copies":15351,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5117,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8243,"num_write_kb":427520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":114,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.353701+0000","last_change":"2023-09-23T19:22:42.150636+0000","last_active":"2023-09-23T19:33:11.353701+0000","last_peered":"2023-09-23T19:33:11.353701+0000","last_clean":"2023-09-23T19:33:11.353701+0000","last_became_active":"2023-09-23T19:22:42.150045+0000","last_became_peered":"2023-09-23T19:22:42.150045+0000","last_unstale":"2023-09-23T19:33:11.353701+0000","last_undegraded":"2023-09-23T19:33:11.353701+0000","last_fullsized":"2023-09-23T19:33:11.353701+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":87,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.343651+0000","last_change":"2023-09-23T19:28:07.889194+0000","last_active":"2023-09-23T19:33:11.343651+0000","last_peered":"2023-09-23T19:33:11.343651+0000","last_clean":"2023-09-23T19:33:11.343651+0000","last_became_active":"2023-09-23T19:28:07.888823+0000","last_became_peered":"2023-09-23T19:28:07.888823+0000","last_unstale":"2023-09-23T19:33:11.343651+0000","last_undegraded":"2023-09-23T19:33:11.343651+0000","last_fullsized":"2023-09-23T19:33:11.343651+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:23.464688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":102,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:17.113461+0000","last_change":"2023-09-23T19:27:24.256410+0000","last_active":"2023-09-23T19:33:17.113461+0000","last_peered":"2023-09-23T19:33:17.113461+0000","last_clean":"2023-09-23T19:33:17.113461+0000","last_became_active":"2023-09-23T19:27:24.256052+0000","last_became_peered":"2023-09-23T19:27:24.256052+0000","last_unstale":"2023-09-23T19:33:17.113461+0000","last_undegraded":"2023-09-23T19:33:17.113461+0000","last_fullsized":"2023-09-23T19:33:17.113461+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:11.951400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"129'8340","reported_seq":8358,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T19:33:34.962319+0000","last_change":"2023-09-23T19:31:12.593664+0000","last_active":"2023-09-23T19:33:34.962319+0000","last_peered":"2023-09-23T19:33:34.962319+0000","last_clean":"2023-09-23T19:33:34.962319+0000","last_became_active":"2023-09-23T19:31:12.592662+0000","last_became_peered":"2023-09-23T19:31:12.592662+0000","last_unstale":"2023-09-23T19:33:34.962319+0000","last_undegraded":"2023-09-23T19:33:34.962319+0000","last_fullsized":"2023-09-23T19:33:34.962319+0000","mapping_epoch":119,"log_start":"122'600","ondisk_log_start":"122'600","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_clean_scrub_stamp":"2023-09-23T19:31:11.546138+0000","objects_scrubbed":0,"log_size":7740,"log_dups_size":0,"ondisk_log_size":7740,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":348389376,"num_objects":5316,"num_object_clones":0,"num_object_copies":15948,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5316,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8340,"num_write_kb":436992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":99,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.360317+0000","last_change":"2023-09-23T19:18:09.015853+0000","last_active":"2023-09-23T19:33:11.360317+0000","last_peered":"2023-09-23T19:33:11.360317+0000","last_clean":"2023-09-23T19:33:11.360317+0000","last_became_active":"2023-09-23T19:18:09.015338+0000","last_became_peered":"2023-09-23T19:18:09.015338+0000","last_unstale":"2023-09-23T19:33:11.360317+0000","last_undegraded":"2023-09-23T19:33:11.360317+0000","last_fullsized":"2023-09-23T19:33:11.360317+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":72,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.336145+0000","last_change":"2023-09-23T19:27:24.266241+0000","last_active":"2023-09-23T19:33:11.336145+0000","last_peered":"2023-09-23T19:33:11.336145+0000","last_clean":"2023-09-23T19:33:11.336145+0000","last_became_active":"2023-09-23T19:27:24.265764+0000","last_became_peered":"2023-09-23T19:27:24.265764+0000","last_unstale":"2023-09-23T19:33:11.336145+0000","last_undegraded":"2023-09-23T19:33:11.336145+0000","last_fullsized":"2023-09-23T19:33:11.336145+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:07.694732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":72,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.345790+0000","last_change":"2023-09-23T19:18:09.005823+0000","last_active":"2023-09-23T19:33:11.345790+0000","last_peered":"2023-09-23T19:33:11.345790+0000","last_clean":"2023-09-23T19:33:11.345790+0000","last_became_active":"2023-09-23T19:18:09.005207+0000","last_became_peered":"2023-09-23T19:18:09.005207+0000","last_unstale":"2023-09-23T19:33:11.345790+0000","last_undegraded":"2023-09-23T19:33:11.345790+0000","last_fullsized":"2023-09-23T19:33:11.345790+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:05.520188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"129'8274","reported_seq":8293,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T19:33:34.971553+0000","last_change":"2023-09-23T19:31:12.576193+0000","last_active":"2023-09-23T19:33:34.971553+0000","last_peered":"2023-09-23T19:33:34.971553+0000","last_clean":"2023-09-23T19:33:34.971553+0000","last_became_active":"2023-09-23T19:31:12.575804+0000","last_became_peered":"2023-09-23T19:31:12.575804+0000","last_unstale":"2023-09-23T19:33:34.971553+0000","last_undegraded":"2023-09-23T19:33:34.971553+0000","last_fullsized":"2023-09-23T19:33:34.971553+0000","mapping_epoch":119,"log_start":"122'500","ondisk_log_start":"122'500","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_clean_scrub_stamp":"2023-09-23T19:31:11.546138+0000","objects_scrubbed":0,"log_size":7774,"log_dups_size":0,"ondisk_log_size":7774,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":345047064,"num_objects":5266,"num_object_clones":0,"num_object_copies":15798,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5266,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8274,"num_write_kb":433217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":33,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.363355+0000","last_change":"2023-09-23T19:32:49.737323+0000","last_active":"2023-09-23T19:33:11.363355+0000","last_peered":"2023-09-23T19:33:11.363355+0000","last_clean":"2023-09-23T19:33:11.363355+0000","last_became_active":"2023-09-23T19:32:49.736937+0000","last_became_peered":"2023-09-23T19:32:49.736937+0000","last_unstale":"2023-09-23T19:33:11.363355+0000","last_undegraded":"2023-09-23T19:33:11.363355+0000","last_fullsized":"2023-09-23T19:33:11.363355+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":57,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T19:33:28.535282+0000","last_change":"2023-09-23T19:22:42.145506+0000","last_active":"2023-09-23T19:33:28.535282+0000","last_peered":"2023-09-23T19:33:28.535282+0000","last_clean":"2023-09-23T19:33:28.535282+0000","last_became_active":"2023-09-23T19:22:42.143991+0000","last_became_peered":"2023-09-23T19:22:42.143991+0000","last_unstale":"2023-09-23T19:33:28.535282+0000","last_undegraded":"2023-09-23T19:33:28.535282+0000","last_fullsized":"2023-09-23T19:33:28.535282+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:56.875175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":70,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.354050+0000","last_change":"2023-09-23T19:32:49.734592+0000","last_active":"2023-09-23T19:33:11.354050+0000","last_peered":"2023-09-23T19:33:11.354050+0000","last_clean":"2023-09-23T19:33:11.354050+0000","last_became_active":"2023-09-23T19:32:49.733905+0000","last_became_peered":"2023-09-23T19:32:49.733905+0000","last_unstale":"2023-09-23T19:33:11.354050+0000","last_undegraded":"2023-09-23T19:33:11.354050+0000","last_fullsized":"2023-09-23T19:33:11.354050+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.c","version":"129'8438","reported_seq":8465,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T19:33:34.949966+0000","last_change":"2023-09-23T19:32:49.758174+0000","last_active":"2023-09-23T19:33:34.949966+0000","last_peered":"2023-09-23T19:33:34.949966+0000","last_clean":"2023-09-23T19:33:34.949966+0000","last_became_active":"2023-09-23T19:32:49.736194+0000","last_became_peered":"2023-09-23T19:32:49.736194+0000","last_unstale":"2023-09-23T19:33:34.949966+0000","last_undegraded":"2023-09-23T19:33:34.949966+0000","last_fullsized":"2023-09-23T19:33:34.949966+0000","mapping_epoch":126,"log_start":"122'100","ondisk_log_start":"122'100","created":119,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_clean_scrub_stamp":"2023-09-23T19:31:11.546138+0000","objects_scrubbed":0,"log_size":8338,"log_dups_size":0,"ondisk_log_size":8338,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":349700096,"num_objects":5336,"num_object_clones":0,"num_object_copies":16008,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5336,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8438,"num_write_kb":440768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":99,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.346536+0000","last_change":"2023-09-23T19:27:24.271827+0000","last_active":"2023-09-23T19:33:11.346536+0000","last_peered":"2023-09-23T19:33:11.346536+0000","last_clean":"2023-09-23T19:33:11.346536+0000","last_became_active":"2023-09-23T19:27:24.271328+0000","last_became_peered":"2023-09-23T19:27:24.271328+0000","last_unstale":"2023-09-23T19:33:11.346536+0000","last_undegraded":"2023-09-23T19:33:11.346536+0000","last_fullsized":"2023-09-23T19:33:11.346536+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":90,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:17.113430+0000","last_change":"2023-09-23T19:22:42.149694+0000","last_active":"2023-09-23T19:33:17.113430+0000","last_peered":"2023-09-23T19:33:17.113430+0000","last_clean":"2023-09-23T19:33:17.113430+0000","last_became_active":"2023-09-23T19:22:42.148194+0000","last_became_peered":"2023-09-23T19:22:42.148194+0000","last_unstale":"2023-09-23T19:33:17.113430+0000","last_undegraded":"2023-09-23T19:33:17.113430+0000","last_fullsized":"2023-09-23T19:33:17.113430+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:47.842383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":79,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.355374+0000","last_change":"2023-09-23T19:05:52.138620+0000","last_active":"2023-09-23T19:33:11.355374+0000","last_peered":"2023-09-23T19:33:11.355374+0000","last_clean":"2023-09-23T19:33:11.355374+0000","last_became_active":"2023-09-23T19:05:52.138243+0000","last_became_peered":"2023-09-23T19:05:52.138243+0000","last_unstale":"2023-09-23T19:33:11.355374+0000","last_undegraded":"2023-09-23T19:33:11.355374+0000","last_fullsized":"2023-09-23T19:33:11.355374+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":74,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.346253+0000","last_change":"2023-09-23T19:18:09.008811+0000","last_active":"2023-09-23T19:33:11.346253+0000","last_peered":"2023-09-23T19:33:11.346253+0000","last_clean":"2023-09-23T19:33:11.346253+0000","last_became_active":"2023-09-23T19:18:09.007255+0000","last_became_peered":"2023-09-23T19:18:09.007255+0000","last_unstale":"2023-09-23T19:33:11.346253+0000","last_undegraded":"2023-09-23T19:33:11.346253+0000","last_fullsized":"2023-09-23T19:33:11.346253+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:25.874038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":111,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.341197+0000","last_change":"2023-09-23T19:28:07.876141+0000","last_active":"2023-09-23T19:33:11.341197+0000","last_peered":"2023-09-23T19:33:11.341197+0000","last_clean":"2023-09-23T19:33:11.341197+0000","last_became_active":"2023-09-23T19:28:07.875802+0000","last_became_peered":"2023-09-23T19:28:07.875802+0000","last_unstale":"2023-09-23T19:33:11.341197+0000","last_undegraded":"2023-09-23T19:33:11.341197+0000","last_fullsized":"2023-09-23T19:33:11.341197+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:38.947881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":98,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.357169+0000","last_change":"2023-09-23T19:22:42.153825+0000","last_active":"2023-09-23T19:33:11.357169+0000","last_peered":"2023-09-23T19:33:11.357169+0000","last_clean":"2023-09-23T19:33:11.357169+0000","last_became_active":"2023-09-23T19:22:42.153416+0000","last_became_peered":"2023-09-23T19:22:42.153416+0000","last_unstale":"2023-09-23T19:33:11.357169+0000","last_undegraded":"2023-09-23T19:33:11.357169+0000","last_fullsized":"2023-09-23T19:33:11.357169+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":118,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.356122+0000","last_change":"2023-09-23T19:28:07.899164+0000","last_active":"2023-09-23T19:33:11.356122+0000","last_peered":"2023-09-23T19:33:11.356122+0000","last_clean":"2023-09-23T19:33:11.356122+0000","last_became_active":"2023-09-23T19:28:07.898812+0000","last_became_peered":"2023-09-23T19:28:07.898812+0000","last_unstale":"2023-09-23T19:33:11.356122+0000","last_undegraded":"2023-09-23T19:33:11.356122+0000","last_fullsized":"2023-09-23T19:33:11.356122+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":83,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.356596+0000","last_change":"2023-09-23T19:05:52.107570+0000","last_active":"2023-09-23T19:33:11.356596+0000","last_peered":"2023-09-23T19:33:11.356596+0000","last_clean":"2023-09-23T19:33:11.356596+0000","last_became_active":"2023-09-23T19:05:52.107215+0000","last_became_peered":"2023-09-23T19:05:52.107215+0000","last_unstale":"2023-09-23T19:33:11.356596+0000","last_undegraded":"2023-09-23T19:33:11.356596+0000","last_fullsized":"2023-09-23T19:33:11.356596+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":109,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.355777+0000","last_change":"2023-09-23T19:28:07.896983+0000","last_active":"2023-09-23T19:33:11.355777+0000","last_peered":"2023-09-23T19:33:11.355777+0000","last_clean":"2023-09-23T19:33:11.355777+0000","last_became_active":"2023-09-23T19:28:07.896577+0000","last_became_peered":"2023-09-23T19:28:07.896577+0000","last_unstale":"2023-09-23T19:33:11.355777+0000","last_undegraded":"2023-09-23T19:33:11.355777+0000","last_fullsized":"2023-09-23T19:33:11.355777+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":23,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.346595+0000","last_change":"2023-09-23T19:32:49.732683+0000","last_active":"2023-09-23T19:33:11.346595+0000","last_peered":"2023-09-23T19:33:11.346595+0000","last_clean":"2023-09-23T19:33:11.346595+0000","last_became_active":"2023-09-23T19:32:49.732188+0000","last_became_peered":"2023-09-23T19:32:49.732188+0000","last_unstale":"2023-09-23T19:33:11.346595+0000","last_undegraded":"2023-09-23T19:33:11.346595+0000","last_fullsized":"2023-09-23T19:33:11.346595+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":110,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.340799+0000","last_change":"2023-09-23T19:32:49.725466+0000","last_active":"2023-09-23T19:33:11.340799+0000","last_peered":"2023-09-23T19:33:11.340799+0000","last_clean":"2023-09-23T19:33:11.340799+0000","last_became_active":"2023-09-23T19:32:49.725048+0000","last_became_peered":"2023-09-23T19:32:49.725048+0000","last_unstale":"2023-09-23T19:33:11.340799+0000","last_undegraded":"2023-09-23T19:33:11.340799+0000","last_fullsized":"2023-09-23T19:33:11.340799+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:44:08.728220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":90,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.343998+0000","last_change":"2023-09-23T19:27:24.267931+0000","last_active":"2023-09-23T19:33:11.343998+0000","last_peered":"2023-09-23T19:33:11.343998+0000","last_clean":"2023-09-23T19:33:11.343998+0000","last_became_active":"2023-09-23T19:27:24.267552+0000","last_became_peered":"2023-09-23T19:27:24.267552+0000","last_unstale":"2023-09-23T19:33:11.343998+0000","last_undegraded":"2023-09-23T19:33:11.343998+0000","last_fullsized":"2023-09-23T19:33:11.343998+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:12.425257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":88,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T19:33:11.355057+0000","last_change":"2023-09-23T19:06:36.673002+0000","last_active":"2023-09-23T19:33:11.355057+0000","last_peered":"2023-09-23T19:33:11.355057+0000","last_clean":"2023-09-23T19:33:11.355057+0000","last_became_active":"2023-09-23T19:06:36.671911+0000","last_became_peered":"2023-09-23T19:06:36.671911+0000","last_unstale":"2023-09-23T19:33:11.355057+0000","last_undegraded":"2023-09-23T19:33:11.355057+0000","last_fullsized":"2023-09-23T19:33:11.355057+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":5487984664,"num_objects":83741,"num_object_clones":0,"num_object_copies":251223,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":83741,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":133019,"num_write_kb":6936257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16467374080,"data_stored":16467361864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128419,"ondisk_log_size":128419,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430577,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3658524,"kb_used_data":3056532,"kb_used_omap":0,"kb_used_meta":601984,"kb_avail":90086628,"statfs":{"total":95995035648,"available":92248707072,"internally_reserved":0,"allocated":3129888768,"data_stored":3129413977,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":616431616},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:33:34 2023","interfaces":[{"interface":"back","average":{"1min":1.4019999999999999,"5min":1.1140000000000001,"15min":1.226},"min":{"1min":0.27600000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":7.1390000000000002,"5min":7.1390000000000002,"15min":9.6760000000000002},"last":1.0900000000000001},{"interface":"front","average":{"1min":1.742,"5min":1.2569999999999999,"15min":1.3380000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.28999999999999998},"max":{"1min":5.7690000000000001,"5min":7.907,"15min":8.9670000000000005},"last":0.48899999999999999}]},{"osd":1,"last update":"Sat Sep 23 19:33:02 2023","interfaces":[{"interface":"back","average":{"1min":1.72,"5min":1.1359999999999999,"15min":1.365},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.28499999999999998},"max":{"1min":9.8209999999999997,"5min":9.8209999999999997,"15min":9.8209999999999997},"last":0.34899999999999998},{"interface":"front","average":{"1min":1.385,"5min":1.083,"15min":1.3460000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29599999999999999},"max":{"1min":5.7000000000000002,"5min":5.7000000000000002,"15min":7.3739999999999997},"last":0.64400000000000002}]},{"osd":2,"last update":"Sat Sep 23 19:32:35 2023","interfaces":[{"interface":"back","average":{"1min":1.9990000000000001,"5min":1.0980000000000001,"15min":0.92300000000000004},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":7.9320000000000004,"5min":7.9320000000000004,"15min":7.9320000000000004},"last":0.46000000000000002},{"interface":"front","average":{"1min":1.1699999999999999,"5min":1.0149999999999999,"15min":0.98599999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":5.6719999999999997,"5min":5.6719999999999997,"15min":5.6719999999999997},"last":0.69399999999999995}]},{"osd":3,"last update":"Sat Sep 23 19:33:14 2023","interfaces":[{"interface":"back","average":{"1min":2.2149999999999999,"5min":1.502,"15min":1.206},"min":{"1min":0.47999999999999998,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":9.9260000000000002,"5min":19.637,"15min":19.637},"last":0.58299999999999996},{"interface":"front","average":{"1min":1.6439999999999999,"5min":1.482,"15min":1.246},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":6.0010000000000003,"5min":20.204000000000001,"15min":20.204000000000001},"last":0.72199999999999998}]},{"osd":4,"last update":"Sat Sep 23 19:33:02 2023","interfaces":[{"interface":"back","average":{"1min":1.407,"5min":1.046,"15min":1.107},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.28699999999999998},"max":{"1min":9.7699999999999996,"5min":9.7699999999999996,"15min":9.7699999999999996},"last":0.38},{"interface":"front","average":{"1min":1.284,"5min":1.0960000000000001,"15min":1.0169999999999999},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.33700000000000002},"max":{"1min":5.5750000000000002,"5min":5.5750000000000002,"15min":7.5090000000000003},"last":1.1519999999999999}]},{"osd":5,"last update":"Sat Sep 23 19:33:02 2023","interfaces":[{"interface":"back","average":{"1min":2.1659999999999999,"5min":1.3080000000000001,"15min":1.2250000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":7.867,"5min":7.867,"15min":7.867},"last":5.2809999999999997},{"interface":"front","average":{"1min":2.1349999999999998,"5min":1.3220000000000001,"15min":1.363},"min":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.34599999999999997},"max":{"1min":9.8979999999999997,"5min":9.8979999999999997,"15min":9.8979999999999997},"last":0.60799999999999998}]},{"osd":6,"last update":"Sat Sep 23 19:33:02 2023","interfaces":[{"interface":"back","average":{"1min":1.379,"5min":1.175,"15min":1.2749999999999999},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.34599999999999997},"max":{"1min":9.8629999999999995,"5min":9.8629999999999995,"15min":13.462},"last":0.626},{"interface":"front","average":{"1min":1.353,"5min":1.1970000000000001,"15min":1.113},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":5.6420000000000003,"5min":5.6420000000000003,"15min":6.8520000000000003},"last":0.67300000000000004}]}]},{"osd":6,"up_from":42,"seq":180388626809,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1313864,"kb_used_data":1012544,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92431288,"statfs":{"total":95995035648,"available":94649638912,"internally_reserved":0,"allocated":1036845056,"data_stored":1036390260,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:32:39 2023","interfaces":[{"interface":"back","average":{"1min":4.7880000000000003,"5min":1.6659999999999999,"15min":1.5409999999999999},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.24099999999999999},"max":{"1min":87.283000000000001,"5min":87.283000000000001,"15min":87.283000000000001},"last":0.39400000000000002},{"interface":"front","average":{"1min":5.2469999999999999,"5min":1.8049999999999999,"15min":1.5349999999999999},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.25900000000000001},"max":{"1min":86.286000000000001,"5min":86.286000000000001,"15min":86.286000000000001},"last":0.55200000000000005}]},{"osd":1,"last update":"Sat Sep 23 19:32:58 2023","interfaces":[{"interface":"back","average":{"1min":1.228,"5min":1.133,"15min":1.4930000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":5.0540000000000003,"5min":7.415,"15min":11.351000000000001},"last":0.36499999999999999},{"interface":"front","average":{"1min":1.4019999999999999,"5min":1.2370000000000001,"15min":1.6180000000000001},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.34200000000000003},"max":{"1min":4.181,"5min":7.4320000000000004,"15min":12.728999999999999},"last":0.60699999999999998}]},{"osd":2,"last update":"Sat Sep 23 19:32:33 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.85899999999999999,"15min":0.90500000000000003},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.0169999999999999,"5min":1.3120000000000001,"15min":1.3120000000000001},"last":1.4870000000000001},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.86299999999999999,"15min":0.88500000000000001},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":1.9550000000000001,"5min":1.9550000000000001,"15min":1.9550000000000001},"last":0.42899999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:33:19 2023","interfaces":[{"interface":"back","average":{"1min":4.306,"5min":1.7170000000000001,"15min":1.2350000000000001},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":60.726999999999997,"5min":60.726999999999997,"15min":60.726999999999997},"last":1.5249999999999999},{"interface":"front","average":{"1min":3.931,"5min":1.611,"15min":1.1819999999999999},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":58.304000000000002,"5min":58.304000000000002,"15min":58.304000000000002},"last":0.45900000000000002}]},{"osd":4,"last update":"Sat Sep 23 19:33:19 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":0.90500000000000003,"15min":0.97599999999999998},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.29299999999999998},"max":{"1min":3.4390000000000001,"5min":3.4390000000000001,"15min":6.5869999999999997},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.72999999999999998,"5min":0.93300000000000005,"15min":1.091},"min":{"1min":0.378,"5min":0.376,"15min":0.30499999999999999},"max":{"1min":2.9319999999999999,"5min":2.9319999999999999,"15min":8.5540000000000003},"last":1.552}]},{"osd":5,"last update":"Sat Sep 23 19:33:19 2023","interfaces":[{"interface":"back","average":{"1min":1.0189999999999999,"5min":0.997,"15min":1.1240000000000001},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.35799999999999998},"max":{"1min":2.7799999999999998,"5min":2.7799999999999998,"15min":6.6660000000000004},"last":3.7530000000000001},{"interface":"front","average":{"1min":1.8280000000000001,"5min":1.236,"15min":1.3560000000000001},"min":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.38100000000000001},"max":{"1min":6.6100000000000003,"5min":6.6100000000000003,"15min":8.5820000000000007},"last":1.5800000000000001}]},{"osd":7,"last update":"Sat Sep 23 19:32:47 2023","interfaces":[{"interface":"back","average":{"1min":1.109,"5min":1.085,"15min":1.1140000000000001},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.38300000000000001},"max":{"1min":3.3839999999999999,"5min":3.3839999999999999,"15min":8.7949999999999999},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.96899999999999997,"5min":1.0900000000000001,"15min":1.2410000000000001},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.39300000000000002},"max":{"1min":2.6400000000000001,"5min":2.6400000000000001,"15min":6.7400000000000002},"last":2.0920000000000001}]}]},{"osd":1,"up_from":98,"seq":420906795141,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3356996,"kb_used_data":3009020,"kb_used_omap":4,"kb_used_meta":347963,"kb_avail":90388156,"statfs":{"total":95995035648,"available":92557471744,"internally_reserved":0,"allocated":3081236480,"data_stored":3080669691,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":356315022},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:33:03 2023","interfaces":[{"interface":"back","average":{"1min":1.1739999999999999,"5min":0.83599999999999997,"15min":1.8100000000000001},"min":{"1min":0.253,"5min":0.22700000000000001,"15min":0.20799999999999999},"max":{"1min":7.1109999999999998,"5min":7.1109999999999998,"15min":14.99},"last":0.745},{"interface":"front","average":{"1min":1.276,"5min":0.91100000000000003,"15min":1.8089999999999999},"min":{"1min":0.26200000000000001,"5min":0.255,"15min":0.23499999999999999},"max":{"1min":5.5999999999999996,"5min":5.899,"15min":12.215},"last":0.80900000000000005}]},{"osd":2,"last update":"Sat Sep 23 19:33:35 2023","interfaces":[{"interface":"back","average":{"1min":1.3540000000000001,"5min":0.93999999999999995,"15min":0.80400000000000005},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":5.7750000000000004,"5min":6.2919999999999998,"15min":6.2919999999999998},"last":0.39800000000000002},{"interface":"front","average":{"1min":1.7010000000000001,"5min":1.2150000000000001,"15min":0.85399999999999998},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":7.3769999999999998,"5min":11.308,"15min":11.308},"last":3.823}]},{"osd":3,"last update":"Sat Sep 23 19:33:16 2023","interfaces":[{"interface":"back","average":{"1min":2.6179999999999999,"5min":1.2230000000000001,"15min":0.90900000000000003},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":10.612,"5min":10.612,"15min":10.612},"last":0.27100000000000002},{"interface":"front","average":{"1min":3.1459999999999999,"5min":1.6419999999999999,"15min":1.0960000000000001},"min":{"1min":0.41599999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":9.2949999999999999,"5min":11.372,"15min":11.372},"last":4.1390000000000002}]},{"osd":4,"last update":"Sat Sep 23 19:33:03 2023","interfaces":[{"interface":"back","average":{"1min":1.278,"5min":1.0920000000000001,"15min":1.5149999999999999},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.35899999999999999},"max":{"1min":5.9299999999999997,"5min":6.3419999999999996,"15min":7.6130000000000004},"last":0.42999999999999999},{"interface":"front","average":{"1min":1.7509999999999999,"5min":1.3560000000000001,"15min":1.4770000000000001},"min":{"1min":0.41199999999999998,"5min":0.40799999999999997,"15min":0.34399999999999997},"max":{"1min":7.6280000000000001,"5min":11.409000000000001,"15min":11.409000000000001},"last":3.8519999999999999}]},{"osd":5,"last update":"Sat Sep 23 19:33:03 2023","interfaces":[{"interface":"back","average":{"1min":3.101,"5min":1.671,"15min":1.6439999999999999},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":10.279999999999999,"5min":11.456,"15min":12.640000000000001},"last":3.8940000000000001},{"interface":"front","average":{"1min":1.669,"5min":1.23,"15min":1.5089999999999999},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.36299999999999999},"max":{"1min":7.1589999999999998,"5min":7.1589999999999998,"15min":9.218},"last":7.1070000000000002}]},{"osd":6,"last update":"Sat Sep 23 19:33:03 2023","interfaces":[{"interface":"back","average":{"1min":1.2669999999999999,"5min":1.171,"15min":1.5389999999999999},"min":{"1min":0.48099999999999998,"5min":0.44700000000000001,"15min":0.40100000000000002},"max":{"1min":7.1890000000000001,"5min":7.1890000000000001,"15min":9.3010000000000002},"last":3.214},{"interface":"front","average":{"1min":1.3340000000000001,"5min":1.198,"15min":1.6879999999999999},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.38800000000000001},"max":{"1min":5.8899999999999997,"5min":6.3659999999999997,"15min":9.1699999999999999},"last":3.1739999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:33:03 2023","interfaces":[{"interface":"back","average":{"1min":2.0259999999999998,"5min":1.3879999999999999,"15min":1.857},"min":{"1min":0.45500000000000002,"5min":0.433,"15min":0.40999999999999998},"max":{"1min":6.5359999999999996,"5min":6.5359999999999996,"15min":9.2159999999999993},"last":0.78200000000000003},{"interface":"front","average":{"1min":2.302,"5min":1.637,"15min":1.6579999999999999},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.42699999999999999},"max":{"1min":7.6749999999999998,"5min":12.709,"15min":12.709},"last":3.8719999999999999}]}]},{"osd":0,"up_from":85,"seq":365072220348,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1714712,"kb_used_data":1324240,"kb_used_omap":5,"kb_used_meta":390458,"kb_avail":92030440,"statfs":{"total":95995035648,"available":94239170560,"internally_reserved":0,"allocated":1356021760,"data_stored":1355439812,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":399829626},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 19:32:57 2023","interfaces":[{"interface":"back","average":{"1min":1.544,"5min":1.018,"15min":1.492},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":4.3739999999999997,"5min":11.939,"15min":13.557},"last":0.42999999999999999},{"interface":"front","average":{"1min":2.9830000000000001,"5min":1.254,"15min":1.8120000000000001},"min":{"1min":0.442,"5min":0.29099999999999998,"15min":0.253},"max":{"1min":12.939,"5min":12.939,"15min":12.939},"last":0.93799999999999994}]},{"osd":2,"last update":"Sat Sep 23 19:33:34 2023","interfaces":[{"interface":"back","average":{"1min":1.363,"5min":1.004,"15min":0.84899999999999998},"min":{"1min":0.28000000000000003,"5min":0.214,"15min":0.214},"max":{"1min":8.2289999999999992,"5min":11.861000000000001,"15min":11.861000000000001},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.98299999999999998,"5min":1.0429999999999999,"15min":0.873},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":3.6509999999999998,"5min":10.212,"15min":10.212},"last":0.55900000000000005}]},{"osd":3,"last update":"Sat Sep 23 19:33:13 2023","interfaces":[{"interface":"back","average":{"1min":1.7609999999999999,"5min":1.101,"15min":0.86899999999999999},"min":{"1min":0.30099999999999999,"5min":0.251,"15min":0.251},"max":{"1min":7.4580000000000002,"5min":7.4580000000000002,"15min":7.4580000000000002},"last":4.1280000000000001},{"interface":"front","average":{"1min":2.843,"5min":1.4870000000000001,"15min":1.149},"min":{"1min":0.45800000000000002,"5min":0.372,"15min":0.372},"max":{"1min":12.991,"5min":12.991,"15min":12.991},"last":1.222}]},{"osd":4,"last update":"Sat Sep 23 19:32:40 2023","interfaces":[{"interface":"back","average":{"1min":1.444,"5min":1.0660000000000001,"15min":1.5089999999999999},"min":{"1min":0.41699999999999998,"5min":0.34699999999999998,"15min":0.32100000000000001},"max":{"1min":4.1120000000000001,"5min":4.1120000000000001,"15min":11.156000000000001},"last":4.0110000000000001},{"interface":"front","average":{"1min":1.099,"5min":1.123,"15min":1.5800000000000001},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.35599999999999998},"max":{"1min":3.3140000000000001,"5min":3.3180000000000001,"15min":13.566000000000001},"last":0.61099999999999999}]},{"osd":5,"last update":"Sat Sep 23 19:32:40 2023","interfaces":[{"interface":"back","average":{"1min":2.3929999999999998,"5min":1.4119999999999999,"15min":1.76},"min":{"1min":0.436,"5min":0.436,"15min":0.33600000000000002},"max":{"1min":6.9240000000000004,"5min":6.9240000000000004,"15min":13.509},"last":0.89800000000000002},{"interface":"front","average":{"1min":2.544,"5min":1.3939999999999999,"15min":1.7270000000000001},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.33400000000000002},"max":{"1min":7.6479999999999997,"5min":7.6479999999999997,"15min":11.048999999999999},"last":4.0789999999999997}]},{"osd":6,"last update":"Sat Sep 23 19:32:40 2023","interfaces":[{"interface":"back","average":{"1min":1.524,"5min":1.1759999999999999,"15min":1.6819999999999999},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.36499999999999999},"max":{"1min":4.0350000000000001,"5min":4.0350000000000001,"15min":11.467000000000001},"last":4.0490000000000004},{"interface":"front","average":{"1min":1.3240000000000001,"5min":1.2410000000000001,"15min":1.7549999999999999},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36399999999999999},"max":{"1min":4.2670000000000003,"5min":4.2670000000000003,"15min":13.603999999999999},"last":0.65300000000000002}]},{"osd":7,"last update":"Sat Sep 23 19:32:40 2023","interfaces":[{"interface":"back","average":{"1min":2.4430000000000001,"5min":1.367,"15min":1.7889999999999999},"min":{"1min":0.55300000000000005,"5min":0.44500000000000001,"15min":0.434},"max":{"1min":12.385,"5min":12.385,"15min":16.271999999999998},"last":0.70399999999999996},{"interface":"front","average":{"1min":1.7470000000000001,"5min":1.258,"15min":1.714},"min":{"1min":0.435,"5min":0.435,"15min":0.38300000000000001},"max":{"1min":4.5380000000000003,"5min":4.5380000000000003,"15min":11.098000000000001},"last":4.1029999999999998}]}]},{"osd":2,"up_from":111,"seq":476741369933,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1165780,"kb_used_data":987404,"kb_used_omap":7,"kb_used_meta":178360,"kb_avail":92579372,"statfs":{"total":95995035648,"available":94801276928,"internally_reserved":0,"allocated":1011101696,"data_stored":1010578117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":182640721},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:33:31 2023","interfaces":[{"interface":"back","average":{"1min":0.82399999999999995,"5min":0.71799999999999997,"15min":0.70999999999999996},"min":{"1min":0.21199999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":3.613,"5min":3.613,"15min":3.613},"last":0.65600000000000003},{"interface":"front","average":{"1min":1.1930000000000001,"5min":0.83699999999999997,"15min":0.79100000000000004},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":7.7549999999999999,"5min":7.7549999999999999,"15min":7.7549999999999999},"last":0.47299999999999998}]},{"osd":1,"last update":"Sat Sep 23 19:33:31 2023","interfaces":[{"interface":"back","average":{"1min":6.9870000000000001,"5min":2.1019999999999999,"15min":1.206},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":92.933000000000007,"5min":92.933000000000007,"15min":92.933000000000007},"last":0.61199999999999999},{"interface":"front","average":{"1min":7.0149999999999997,"5min":2.1230000000000002,"15min":1.2749999999999999},"min":{"1min":0.40799999999999997,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":92.432000000000002,"5min":92.432000000000002,"15min":92.432000000000002},"last":0.86299999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:33:20 2023","interfaces":[{"interface":"back","average":{"1min":6.9029999999999996,"5min":2.1070000000000002,"15min":1.1990000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":92.350999999999999,"5min":92.350999999999999,"15min":92.350999999999999},"last":0.68200000000000005},{"interface":"front","average":{"1min":6.9939999999999998,"5min":2.1779999999999999,"15min":1.302},"min":{"1min":0.32600000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":96.236999999999995,"5min":96.236999999999995,"15min":96.236999999999995},"last":3.6200000000000001}]},{"osd":4,"last update":"Sat Sep 23 19:33:31 2023","interfaces":[{"interface":"back","average":{"1min":0.79100000000000004,"5min":0.97699999999999998,"15min":1.0069999999999999},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":2.7240000000000002,"5min":3.1240000000000001,"15min":3.1240000000000001},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.72699999999999998,"5min":1.0589999999999999,"15min":1.0249999999999999},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":2.7669999999999999,"5min":9.0199999999999996,"15min":9.0199999999999996},"last":0.53100000000000003}]},{"osd":5,"last update":"Sat Sep 23 19:33:31 2023","interfaces":[{"interface":"back","average":{"1min":1.7150000000000001,"5min":1.284,"15min":1.1879999999999999},"min":{"1min":0.42499999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":4.9000000000000004,"5min":4.9000000000000004,"15min":4.9000000000000004},"last":0.82699999999999996},{"interface":"front","average":{"1min":1.617,"5min":1.2949999999999999,"15min":1.194},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":4.8310000000000004,"5min":4.8310000000000004,"15min":4.8310000000000004},"last":0.70299999999999996}]},{"osd":6,"last update":"Sat Sep 23 19:33:31 2023","interfaces":[{"interface":"back","average":{"1min":0.81899999999999995,"5min":1.0740000000000001,"15min":1.1000000000000001},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":2.8100000000000001,"5min":9.0739999999999998,"15min":9.0739999999999998},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.92300000000000004,"5min":1.0720000000000001,"15min":1.1379999999999999},"min":{"1min":0.41899999999999998,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":3.52,"5min":3.52,"15min":3.52},"last":0.77700000000000002}]},{"osd":7,"last update":"Sat Sep 23 19:33:31 2023","interfaces":[{"interface":"back","average":{"1min":2.4700000000000002,"5min":1.4670000000000001,"15min":1.3009999999999999},"min":{"1min":0.5,"5min":0.48799999999999999,"15min":0.48799999999999999},"max":{"1min":7.8600000000000003,"5min":7.8600000000000003,"15min":7.8600000000000003},"last":0.72499999999999998},{"interface":"front","average":{"1min":2.4119999999999999,"5min":1.488,"15min":1.2849999999999999},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":7.8280000000000003,"5min":9.1150000000000002,"15min":9.1150000000000002},"last":0.59199999999999997}]}]},{"osd":3,"up_from":116,"seq":498216206404,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2912372,"kb_used_data":2668908,"kb_used_omap":2,"kb_used_meta":243453,"kb_avail":90832780,"statfs":{"total":95995035648,"available":93012766720,"internally_reserved":0,"allocated":2732961792,"data_stored":2732465542,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":249295925},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:33:16 2023","interfaces":[{"interface":"back","average":{"1min":1.7789999999999999,"5min":1.0589999999999999,"15min":0.79100000000000004},"min":{"1min":0.249,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":6.8540000000000001,"5min":6.8540000000000001,"15min":6.8540000000000001},"last":8.2370000000000001},{"interface":"front","average":{"1min":2,"5min":1.1859999999999999,"15min":0.93300000000000005},"min":{"1min":0.27900000000000003,"5min":0.251,"15min":0.251},"max":{"1min":6.3739999999999997,"5min":6.3739999999999997,"15min":6.3739999999999997},"last":4.9340000000000002}]},{"osd":1,"last update":"Sat Sep 23 19:33:16 2023","interfaces":[{"interface":"back","average":{"1min":1.754,"5min":1.2569999999999999,"15min":0.91400000000000003},"min":{"1min":0.308,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":5.4859999999999998,"5min":13.659000000000001,"15min":13.659000000000001},"last":4.133},{"interface":"front","average":{"1min":2.1619999999999999,"5min":1.4159999999999999,"15min":1.0049999999999999},"min":{"1min":0.36499999999999999,"5min":0.253,"15min":0.253},"max":{"1min":4.7389999999999999,"5min":11.989000000000001,"15min":11.989000000000001},"last":0.64400000000000002}]},{"osd":2,"last update":"Sat Sep 23 19:33:16 2023","interfaces":[{"interface":"back","average":{"1min":1.4730000000000001,"5min":0.96599999999999997,"15min":0.83899999999999997},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":6.9950000000000001,"5min":6.9950000000000001,"15min":6.9950000000000001},"last":0.57899999999999996},{"interface":"front","average":{"1min":1.5109999999999999,"5min":1.1379999999999999,"15min":0.94399999999999995},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":5.6269999999999998,"5min":5.9589999999999996,"15min":5.9589999999999996},"last":0.67000000000000004}]},{"osd":4,"last update":"Sat Sep 23 19:33:16 2023","interfaces":[{"interface":"back","average":{"1min":1.304,"5min":1.218,"15min":1.1240000000000001},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":3.4199999999999999,"5min":6.0069999999999997,"15min":6.0069999999999997},"last":0.69699999999999995},{"interface":"front","average":{"1min":1.744,"5min":1.3520000000000001,"15min":1.216},"min":{"1min":0.41499999999999998,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":5.5229999999999997,"5min":5.5229999999999997,"15min":5.5229999999999997},"last":0.52200000000000002}]},{"osd":5,"last update":"Sat Sep 23 19:33:16 2023","interfaces":[{"interface":"back","average":{"1min":3.0299999999999998,"5min":1.7390000000000001,"15min":1.3300000000000001},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":7.8559999999999999,"5min":7.8559999999999999,"15min":7.8559999999999999},"last":0.48999999999999999},{"interface":"front","average":{"1min":2.278,"5min":1.403,"15min":1.1879999999999999},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":7.9699999999999998,"5min":7.9699999999999998,"15min":7.9699999999999998},"last":0.60799999999999998}]},{"osd":6,"last update":"Sat Sep 23 19:33:16 2023","interfaces":[{"interface":"back","average":{"1min":2.0939999999999999,"5min":1.444,"15min":1.2450000000000001},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":6.319,"5min":6.319,"15min":6.319},"last":0.54800000000000004},{"interface":"front","average":{"1min":1.603,"5min":1.28,"15min":1.1919999999999999},"min":{"1min":0.40999999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":7.1210000000000004,"5min":7.1210000000000004,"15min":7.1210000000000004},"last":1.323}]},{"osd":7,"last update":"Sat Sep 23 19:33:16 2023","interfaces":[{"interface":"back","average":{"1min":2.0800000000000001,"5min":1.4119999999999999,"15min":1.244},"min":{"1min":0.376,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":11.259,"5min":11.259,"15min":11.259},"last":1.7070000000000001},{"interface":"front","average":{"1min":1.8819999999999999,"5min":1.421,"15min":1.244},"min":{"1min":0.52100000000000002,"5min":0.443,"15min":0.443},"max":{"1min":6.7309999999999999,"5min":6.7309999999999999,"15min":6.7309999999999999},"last":1.7929999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019274,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1618192,"kb_used_data":1028488,"kb_used_omap":0,"kb_used_meta":589696,"kb_avail":92126960,"statfs":{"total":95995035648,"available":94338007040,"internally_reserved":0,"allocated":1053171712,"data_stored":1052708571,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603848704},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:32:35 2023","interfaces":[{"interface":"back","average":{"1min":0.80600000000000005,"5min":0.85399999999999998,"15min":2.0510000000000002},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":3.3250000000000002,"5min":3.3250000000000002,"15min":129.16200000000001},"last":0.30399999999999999},{"interface":"front","average":{"1min":1.385,"5min":0.97499999999999998,"15min":2.2029999999999998},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.27700000000000002},"max":{"1min":3.395,"5min":3.395,"15min":130.40600000000001},"last":0.41299999999999998}]},{"osd":1,"last update":"Sat Sep 23 19:33:05 2023","interfaces":[{"interface":"back","average":{"1min":1.1799999999999999,"5min":0.95799999999999996,"15min":1.4810000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":6.6150000000000002,"5min":6.6150000000000002,"15min":12.372},"last":0.67100000000000004},{"interface":"front","average":{"1min":1.4910000000000001,"5min":1.0589999999999999,"15min":1.4630000000000001},"min":{"1min":0.48299999999999998,"5min":0.372,"15min":0.31},"max":{"1min":6.7640000000000002,"5min":6.7640000000000002,"15min":7.3360000000000003},"last":0.76000000000000001}]},{"osd":2,"last update":"Sat Sep 23 19:33:30 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":0.85799999999999998,"15min":0.876},"min":{"1min":0.34499999999999997,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":4.915,"5min":4.915,"15min":4.915},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.83199999999999996,"5min":0.84999999999999998,"15min":0.88200000000000001},"min":{"1min":0.35999999999999999,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":2.9359999999999999,"5min":2.9359999999999999,"15min":2.9359999999999999},"last":0.40100000000000002}]},{"osd":3,"last update":"Sat Sep 23 19:33:21 2023","interfaces":[{"interface":"back","average":{"1min":2.2080000000000002,"5min":1.6990000000000001,"15min":1.216},"min":{"1min":0.378,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":9.3780000000000001,"5min":49.515000000000001,"15min":49.515000000000001},"last":4.6689999999999996},{"interface":"front","average":{"1min":1.4119999999999999,"5min":1.554,"15min":1.159},"min":{"1min":0.378,"5min":0.374,"15min":0.374},"max":{"1min":8.3309999999999995,"5min":51.518999999999998,"15min":51.518999999999998},"last":1.845}]},{"osd":5,"last update":"Sat Sep 23 19:32:45 2023","interfaces":[{"interface":"back","average":{"1min":1.6970000000000001,"5min":1.1519999999999999,"15min":1.2130000000000001},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.29199999999999998},"max":{"1min":6.2539999999999996,"5min":6.2539999999999996,"15min":12.459},"last":0.89100000000000001},{"interface":"front","average":{"1min":1.3460000000000001,"5min":1.121,"15min":1.1970000000000001},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.26700000000000002},"max":{"1min":5.3639999999999999,"5min":5.3639999999999999,"15min":12.478},"last":0.65000000000000002}]},{"osd":6,"last update":"Sat Sep 23 19:33:05 2023","interfaces":[{"interface":"back","average":{"1min":1.353,"5min":1.0800000000000001,"15min":1.27},"min":{"1min":0.44900000000000001,"5min":0.35999999999999999,"15min":0.29999999999999999},"max":{"1min":6.4610000000000003,"5min":6.4610000000000003,"15min":7.0810000000000004},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.82599999999999996,"5min":1.0449999999999999,"15min":1.1799999999999999},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.34300000000000003},"max":{"1min":2.5790000000000002,"5min":2.5790000000000002,"15min":6.9050000000000002},"last":0.52300000000000002}]},{"osd":7,"last update":"Sat Sep 23 19:33:05 2023","interfaces":[{"interface":"back","average":{"1min":1.4730000000000001,"5min":1.1240000000000001,"15min":1.222},"min":{"1min":0.436,"5min":0.42699999999999999,"15min":0.374},"max":{"1min":4.9470000000000001,"5min":4.9470000000000001,"15min":7.8479999999999999},"last":0.84999999999999998},{"interface":"front","average":{"1min":1.8029999999999999,"5min":1.2629999999999999,"15min":1.181},"min":{"1min":0.44,"5min":0.44,"15min":0.36699999999999999},"max":{"1min":6.3890000000000002,"5min":6.3890000000000002,"15min":6.3890000000000002},"last":0.61499999999999999}]}]},{"osd":5,"up_from":36,"seq":154618823041,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3966780,"kb_used_data":3059828,"kb_used_omap":0,"kb_used_meta":906880,"kb_avail":89778372,"statfs":{"total":95995035648,"available":91933052928,"internally_reserved":0,"allocated":3133263872,"data_stored":3132816349,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":928645120},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:32:37 2023","interfaces":[{"interface":"back","average":{"1min":0.85399999999999998,"5min":0.89800000000000002,"15min":1.262},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":3.8119999999999998,"5min":3.8119999999999998,"15min":11.058},"last":3.665},{"interface":"front","average":{"1min":1.341,"5min":0.98899999999999999,"15min":1.5229999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29699999999999999},"max":{"1min":5.2430000000000003,"5min":5.2430000000000003,"15min":10.802},"last":0.498}]},{"osd":1,"last update":"Sat Sep 23 19:33:06 2023","interfaces":[{"interface":"back","average":{"1min":1.3799999999999999,"5min":1.012,"15min":1.2230000000000001},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.27200000000000002},"max":{"1min":3.9279999999999999,"5min":3.9279999999999999,"15min":9.5389999999999997},"last":0.32400000000000001},{"interface":"front","average":{"1min":2.3199999999999998,"5min":1.256,"15min":1.881},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.26700000000000002},"max":{"1min":7.8390000000000004,"5min":7.8390000000000004,"15min":11.228999999999999},"last":0.47099999999999997}]},{"osd":2,"last update":"Sat Sep 23 19:32:37 2023","interfaces":[{"interface":"back","average":{"1min":0.80400000000000005,"5min":0.96999999999999997,"15min":0.96199999999999997},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":3.9990000000000001,"5min":3.9990000000000001,"15min":3.9990000000000001},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.98299999999999998,"15min":1.026},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":1.034,"5min":2.335,"15min":2.335},"last":0.57499999999999996}]},{"osd":3,"last update":"Sat Sep 23 19:33:13 2023","interfaces":[{"interface":"back","average":{"1min":1.861,"5min":1.1379999999999999,"15min":1.087},"min":{"1min":0.39500000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":7.532,"5min":7.532,"15min":7.532},"last":3.4049999999999998},{"interface":"front","average":{"1min":1.3879999999999999,"5min":1.1399999999999999,"15min":1.1619999999999999},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":4.4169999999999998,"5min":4.4169999999999998,"15min":4.4169999999999998},"last":0.38}]},{"osd":4,"last update":"Sat Sep 23 19:32:37 2023","interfaces":[{"interface":"back","average":{"1min":1.103,"5min":0.95299999999999996,"15min":1.0429999999999999},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.29199999999999998},"max":{"1min":2.3610000000000002,"5min":2.3610000000000002,"15min":8.1790000000000003},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.84999999999999998,"5min":0.95999999999999996,"15min":1.0580000000000001},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.27000000000000002},"max":{"1min":3.9510000000000001,"5min":3.9510000000000001,"15min":13.714},"last":0.41199999999999998}]},{"osd":6,"last update":"Sat Sep 23 19:33:25 2023","interfaces":[{"interface":"back","average":{"1min":1.0369999999999999,"5min":1.044,"15min":1.161},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":3.7240000000000002,"5min":3.7240000000000002,"15min":26.448},"last":0.61399999999999999},{"interface":"front","average":{"1min":1.593,"5min":1.224,"15min":1.2769999999999999},"min":{"1min":0.48899999999999999,"5min":0.378,"15min":0.378},"max":{"1min":7.5750000000000002,"5min":7.5750000000000002,"15min":26.841999999999999},"last":0.66800000000000004}]},{"osd":7,"last update":"Sat Sep 23 19:32:51 2023","interfaces":[{"interface":"back","average":{"1min":1.8220000000000001,"5min":1.27,"15min":1.216},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.36599999999999999},"max":{"1min":7.6280000000000001,"5min":7.6280000000000001,"15min":7.6280000000000001},"last":0.72599999999999998},{"interface":"front","average":{"1min":1.627,"5min":1.238,"15min":1.117},"min":{"1min":0.47799999999999998,"5min":0.39900000000000002,"15min":0.35799999999999998},"max":{"1min":13.18,"5min":13.18,"15min":13.18},"last":0.35499999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1345585152,"data_stored":1345585152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3079409664,"data_stored":3079405592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1008467968,"data_stored":1008467968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2721447936,"data_stored":2721447936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1046413312,"data_stored":1046413312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3106672640,"data_stored":3106668568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1031864320,"data_stored":1031864320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3127513088,"data_stored":3127509016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T19:33:40.201 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T19:33:40.201 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd pool set device_health_metrics pg_num 11 2023-09-23T19:33:40.230 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:40.257 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:40.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:40.263 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:33:40.364 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:40.387 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:40.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:40.393 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:40.493 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:40.522 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:40.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:40.525 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:33:40.555 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:33:40.558 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:33:40.626 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:40.658 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:40.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:40.665 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:33:40.765 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:40.808 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:40.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:40.810 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:40.911 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:40.939 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:40.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:40.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:33:41.048 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:41.082 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:41.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:41.089 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:33:41.190 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:41.222 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:41.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:41.225 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:41.326 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:41.348 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:41.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:41.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:41.451 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:41.483 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:41.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:41.486 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:41.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:41 smithi092 bash[58855]: cluster 2023-09-23T19:33:40.004711+0000 mgr.y (mgr.25325) 782 : cluster 0 pgmap v685: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2023-09-23T19:33:41.586 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:41.596 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:41 smithi016 bash[75004]: cluster 2023-09-23T19:33:40.004711+0000 mgr.y (mgr.25325) 782 : cluster 0 pgmap v685: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2023-09-23T19:33:41.596 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:41 smithi016 bash[77995]: cluster 2023-09-23T19:33:40.004711+0000 mgr.y (mgr.25325) 782 : cluster 0 pgmap v685: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2023-09-23T19:33:41.620 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:41.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:41.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:33:41.727 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:41.758 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:41.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:41.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:33:41.865 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:41.914 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:41.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:41.917 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:33:42.017 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:42.046 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:42.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:42.052 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:42.152 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:42.185 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:42.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:42.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:42.203 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:33:42.288 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:42.314 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:42.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:42.320 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:42.421 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:42.445 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:42.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:42.447 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:33:42.548 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:42.576 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:42.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:42.580 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:42.680 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:42.710 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:42.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:42.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:33:42.816 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:42.851 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:42.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:42.854 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:33:42.954 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:42.989 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:42.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:42.994 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:33:43.095 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:43.119 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:43.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:43.122 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:33:43.222 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:43.256 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:43.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:43.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:43.363 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:43.393 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:43.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:43.397 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:33:43.497 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:43.530 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:43.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:43.533 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:33:43.634 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:43.659 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:43.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:43.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:33:43.762 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:43.770 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:43 smithi016 bash[75004]: cluster 2023-09-23T19:33:42.005493+0000 mgr.y (mgr.25325) 783 : cluster 0 pgmap v686: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 769 op/s 2023-09-23T19:33:43.770 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:43 smithi016 bash[77995]: cluster 2023-09-23T19:33:42.005493+0000 mgr.y (mgr.25325) 783 : cluster 0 pgmap v686: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 769 op/s 2023-09-23T19:33:43.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:43 smithi092 bash[58855]: cluster 2023-09-23T19:33:42.005493+0000 mgr.y (mgr.25325) 783 : cluster 0 pgmap v686: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 769 op/s 2023-09-23T19:33:43.792 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:43.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:43.796 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:43.897 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:43.933 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:43.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:43.938 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:33:44.038 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:44.059 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:44.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:44.061 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:44.162 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:44.214 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:44.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:44.228 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:44.329 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:44.355 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:44.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:44.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:44.461 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:44.500 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:44.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:44.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:44.604 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:44.631 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:44.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:44.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:44.735 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:44.744 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:44 smithi092 bash[58855]: audit 2023-09-23T19:33:43.643647+0000 mon.a (mon.0) 974 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:33:44.745 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:33:44.760 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:44.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:44.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:44.769 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:44 smithi016 bash[75004]: audit 2023-09-23T19:33:43.643647+0000 mon.a (mon.0) 974 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:33:44.769 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:44 smithi016 bash[77995]: audit 2023-09-23T19:33:43.643647+0000 mon.a (mon.0) 974 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:33:44.865 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:44.892 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:44.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:44.894 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:33:44.994 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:45.019 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:45.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:45.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:33:45.126 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:45.162 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:45.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:45.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:33:45.268 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:45.295 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:45.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:45.300 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:45.401 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:45.432 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:45.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:45.435 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:33:45.536 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:45.566 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:45.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:45.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:45.672 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:45.700 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:45.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:45.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:33:45.804 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:45.846 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:45 smithi092 bash[58855]: cluster 2023-09-23T19:33:44.006057+0000 mgr.y (mgr.25325) 784 : cluster 0 pgmap v687: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2023-09-23T19:33:45.869 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:45.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:45.871 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:45.972 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:45.981 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:45 smithi016 bash[75004]: cluster 2023-09-23T19:33:44.006057+0000 mgr.y (mgr.25325) 784 : cluster 0 pgmap v687: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2023-09-23T19:33:45.981 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:33:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:33:45] "GET /metrics HTTP/1.1" 200 36236 "" "Prometheus/2.33.4" 2023-09-23T19:33:45.981 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:45 smithi016 bash[77995]: cluster 2023-09-23T19:33:44.006057+0000 mgr.y (mgr.25325) 784 : cluster 0 pgmap v687: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2023-09-23T19:33:46.001 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:46.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:46.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:33:46.104 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:46.123 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:46.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:46.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:33:46.228 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:46.261 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:46.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:46.268 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:46.368 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:46.406 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:46.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:46.412 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:33:46.512 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:46.539 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:46.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:46.544 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:33:46.644 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:46.671 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:46.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:46.677 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:33:46.778 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:46.779 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:33:46.815 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:46.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:46.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:46.920 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:46.946 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:46.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:46.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:47.049 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:47.088 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:47.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:47.095 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:47.196 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:47.252 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:47.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:47.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:33:47.360 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:47.383 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:47.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:47.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:33:47.489 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:47.525 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:47.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:47.529 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:33:47.629 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:47.637 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:47 smithi092 bash[58855]: cluster 2023-09-23T19:33:46.006706+0000 mgr.y (mgr.25325) 785 : cluster 0 pgmap v688: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 801 op/s 2023-09-23T19:33:47.656 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:47.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:47.658 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:33:47.759 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:47.786 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:47.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:47.789 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:33:47.805 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:47 smithi016 bash[75004]: cluster 2023-09-23T19:33:46.006706+0000 mgr.y (mgr.25325) 785 : cluster 0 pgmap v688: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 801 op/s 2023-09-23T19:33:47.805 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:47 smithi016 bash[77995]: cluster 2023-09-23T19:33:46.006706+0000 mgr.y (mgr.25325) 785 : cluster 0 pgmap v688: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 801 op/s 2023-09-23T19:33:47.890 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:47.921 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:47.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:47.924 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:33:48.024 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:48.047 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-09-23T19:33:48.047 INFO:teuthology.orchestra.run.smithi016.stdout: "size": 20, 2023-09-23T19:33:48.047 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 600, 2023-09-23T19:33:48.047 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [ 2023-09-23T19:33:48.047 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.047 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45671.0:73805 9.6 e125/119 9:68805e72:::benchmark_data_smithi016_113233_object73804:head v 125'4489, mlcod=125'4483)", 2023-09-23T19:33:48.047 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:32:24.757289+0000", 2023-09-23T19:33:48.047 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 83.278395360000005, 2023-09-23T19:33:48.047 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.023156706999999999, 2023-09-23T19:33:48.047 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:33:48.048 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:33:48.048 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:33:48.048 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.048 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:33:48.048 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:24.757289+0000", 2023-09-23T19:33:48.048 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:33:48.048 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.048 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.048 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:33:48.048 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:24.757289+0000", 2023-09-23T19:33:48.048 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:33:48.049 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.049 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.049 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:33:48.049 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:24.757288+0000", 2023-09-23T19:33:48.049 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:33:48.049 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.049 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.049 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:33:48.049 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:24.757336+0000", 2023-09-23T19:33:48.049 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.7422000000000001e-05 2023-09-23T19:33:48.049 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.049 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.050 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:33:48.050 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:24.757352+0000", 2023-09-23T19:33:48.050 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.6656000000000001e-05 2023-09-23T19:33:48.050 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.050 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.050 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:33:48.050 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:24.757362+0000", 2023-09-23T19:33:48.050 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.2459999999999992e-06 2023-09-23T19:33:48.050 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.050 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.051 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:33:48.051 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:24.762128+0000", 2023-09-23T19:33:48.051 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.004766569 2023-09-23T19:33:48.051 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.051 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.051 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:33:48.051 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:24.762194+0000", 2023-09-23T19:33:48.051 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.5630999999999996e-05 2023-09-23T19:33:48.051 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.051 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.051 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:33:48.052 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:24.780384+0000", 2023-09-23T19:33:48.052 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.018190732000000001 2023-09-23T19:33:48.052 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.052 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.052 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:33:48.052 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:24.780446+0000", 2023-09-23T19:33:48.052 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.1245000000000001e-05 2023-09-23T19:33:48.052 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:33:48.052 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:33:48.052 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:33:48.052 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.053 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.053 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45671.0:105900 9.6 e127/119 9:6d6a6984:::benchmark_data_smithi016_113233_object105899:head v 127'6492, mlcod=127'6489)", 2023-09-23T19:33:48.053 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:33:01.232983+0000", 2023-09-23T19:33:48.053 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 46.802701423999999, 2023-09-23T19:33:48.053 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.024864173, 2023-09-23T19:33:48.053 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:33:48.053 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:33:48.053 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:33:48.053 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.053 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:33:48.053 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:01.232983+0000", 2023-09-23T19:33:48.053 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:33:48.054 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.054 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.054 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:33:48.054 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:01.232983+0000", 2023-09-23T19:33:48.054 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:33:48.054 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.054 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.054 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:33:48.054 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:01.232982+0000", 2023-09-23T19:33:48.054 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-09-23T19:33:48.055 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.055 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.055 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:33:48.055 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:01.233070+0000", 2023-09-23T19:33:48.055 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.8438999999999998e-05 2023-09-23T19:33:48.055 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.055 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.055 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:33:48.055 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:01.233092+0000", 2023-09-23T19:33:48.055 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.1651e-05 2023-09-23T19:33:48.055 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.055 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.056 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:33:48.056 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:01.233101+0000", 2023-09-23T19:33:48.056 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.4469999999999995e-06 2023-09-23T19:33:48.056 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.056 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.056 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:33:48.056 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:01.233590+0000", 2023-09-23T19:33:48.056 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00048821999999999998 2023-09-23T19:33:48.056 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.056 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.056 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:33:48.057 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:01.233652+0000", 2023-09-23T19:33:48.057 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.2131000000000005e-05 2023-09-23T19:33:48.057 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.057 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.057 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:33:48.057 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:01.257651+0000", 2023-09-23T19:33:48.057 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.023999215000000001 2023-09-23T19:33:48.057 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.057 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.057 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:33:48.057 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:01.257847+0000", 2023-09-23T19:33:48.058 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00019628700000000001 2023-09-23T19:33:48.058 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:33:48.058 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:33:48.058 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:33:48.058 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.058 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.058 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45671.0:105904 9.6 e127/119 9:68132f94:::benchmark_data_smithi016_113233_object105903:head v 127'6493, mlcod=127'6489)", 2023-09-23T19:33:48.058 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:33:01.236406+0000", 2023-09-23T19:33:48.058 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 46.799277902999997, 2023-09-23T19:33:48.058 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.021620746999999999, 2023-09-23T19:33:48.058 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:33:48.059 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:33:48.059 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:33:48.059 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.059 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:33:48.059 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:01.236406+0000", 2023-09-23T19:33:48.059 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:33:48.059 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.059 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.059 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:33:48.059 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:01.236406+0000", 2023-09-23T19:33:48.060 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:33:48.060 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.060 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.060 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:33:48.060 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:01.236405+0000", 2023-09-23T19:33:48.060 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:33:48.060 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.060 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.060 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:33:48.060 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:01.236507+0000", 2023-09-23T19:33:48.060 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000101718 2023-09-23T19:33:48.060 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.061 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.061 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:33:48.061 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:01.236529+0000", 2023-09-23T19:33:48.061 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.1699e-05 2023-09-23T19:33:48.061 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.061 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.061 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:33:48.061 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:01.236538+0000", 2023-09-23T19:33:48.061 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.9290000000000004e-06 2023-09-23T19:33:48.061 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.061 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.062 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:33:48.062 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:01.237709+0000", 2023-09-23T19:33:48.062 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0011707969999999999 2023-09-23T19:33:48.062 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.062 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.062 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:33:48.062 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:01.237760+0000", 2023-09-23T19:33:48.062 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.1807000000000001e-05 2023-09-23T19:33:48.062 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.062 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.062 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:33:48.062 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:01.257856+0000", 2023-09-23T19:33:48.063 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.020095538999999999 2023-09-23T19:33:48.063 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.063 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.063 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:33:48.063 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:01.258027+0000", 2023-09-23T19:33:48.063 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00017131899999999999 2023-09-23T19:33:48.063 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:33:48.063 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:33:48.063 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:33:48.063 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.063 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.063 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45671.0:106581 9.6 e127/119 9:6e5d9fed:::benchmark_data_smithi016_113233_object106580:head v 127'6544, mlcod=127'6541)", 2023-09-23T19:33:48.064 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:33:02.043530+0000", 2023-09-23T19:33:48.064 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 45.992154624999998, 2023-09-23T19:33:48.064 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.020100818999999999, 2023-09-23T19:33:48.064 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:33:48.064 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:33:48.064 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:33:48.064 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.064 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:33:48.064 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:02.043530+0000", 2023-09-23T19:33:48.064 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:33:48.064 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.065 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.065 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:33:48.065 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:02.043530+0000", 2023-09-23T19:33:48.065 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:33:48.065 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.065 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.065 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:33:48.065 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:02.043528+0000", 2023-09-23T19:33:48.065 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-09-23T19:33:48.065 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.065 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.065 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:33:48.066 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:02.043589+0000", 2023-09-23T19:33:48.066 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.0319999999999998e-05 2023-09-23T19:33:48.066 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.066 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.066 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:33:48.066 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:02.043609+0000", 2023-09-23T19:33:48.066 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.0293e-05 2023-09-23T19:33:48.066 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.066 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.066 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:33:48.067 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:02.043619+0000", 2023-09-23T19:33:48.067 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.7690000000000008e-06 2023-09-23T19:33:48.067 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.067 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.067 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:33:48.067 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:02.043670+0000", 2023-09-23T19:33:48.067 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.1298000000000003e-05 2023-09-23T19:33:48.067 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.067 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.067 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:33:48.067 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:02.043735+0000", 2023-09-23T19:33:48.068 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.5079000000000005e-05 2023-09-23T19:33:48.068 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.068 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.068 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:33:48.068 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:02.063390+0000", 2023-09-23T19:33:48.068 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.019654742999999999 2023-09-23T19:33:48.068 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.068 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.068 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:33:48.068 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:02.063631+0000", 2023-09-23T19:33:48.068 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000240621 2023-09-23T19:33:48.068 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:33:48.069 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:33:48.069 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:33:48.069 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.069 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.069 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45671.0:106585 9.0 e127/119 9:01e857ff:::benchmark_data_smithi016_113233_object106584:head v 127'6685, mlcod=127'6681)", 2023-09-23T19:33:48.069 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:33:02.052436+0000", 2023-09-23T19:33:48.069 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 45.983248297999999, 2023-09-23T19:33:48.069 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.021676358, 2023-09-23T19:33:48.069 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:33:48.069 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:33:48.069 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:33:48.069 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.070 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:33:48.070 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:02.052436+0000", 2023-09-23T19:33:48.070 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:33:48.070 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.070 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.070 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:33:48.070 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:02.052436+0000", 2023-09-23T19:33:48.070 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:33:48.070 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.070 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.070 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:33:48.071 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:02.052435+0000", 2023-09-23T19:33:48.071 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:33:48.071 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.071 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.071 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:33:48.071 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:02.052474+0000", 2023-09-23T19:33:48.071 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.9205000000000002e-05 2023-09-23T19:33:48.071 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.071 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.071 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:33:48.071 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:02.055250+0000", 2023-09-23T19:33:48.072 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0027755599999999998 2023-09-23T19:33:48.072 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.072 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.072 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:33:48.072 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:02.055262+0000", 2023-09-23T19:33:48.072 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2136e-05 2023-09-23T19:33:48.072 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.072 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.072 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:33:48.072 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:02.059661+0000", 2023-09-23T19:33:48.072 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0043993540000000003 2023-09-23T19:33:48.072 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.073 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.073 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:33:48.073 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:02.059715+0000", 2023-09-23T19:33:48.073 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.3520000000000003e-05 2023-09-23T19:33:48.073 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.073 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.073 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:33:48.073 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:02.074076+0000", 2023-09-23T19:33:48.073 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.014361004 2023-09-23T19:33:48.073 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.073 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.073 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:33:48.073 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:02.074112+0000", 2023-09-23T19:33:48.074 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.6633e-05 2023-09-23T19:33:48.074 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:33:48.074 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:33:48.074 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:33:48.074 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.074 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.074 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45671.0:106593 9.0 e127/119 9:07703164:::benchmark_data_smithi016_113233_object106592:head v 127'6686, mlcod=127'6681)", 2023-09-23T19:33:48.074 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:33:02.055354+0000", 2023-09-23T19:33:48.074 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 45.980330002999999, 2023-09-23T19:33:48.074 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.018779192, 2023-09-23T19:33:48.074 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:33:48.075 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:33:48.075 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:33:48.075 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.075 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:33:48.075 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:02.055354+0000", 2023-09-23T19:33:48.075 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:33:48.075 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.075 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.075 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:33:48.075 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:02.055354+0000", 2023-09-23T19:33:48.075 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:33:48.075 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.076 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.076 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:33:48.076 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:02.055354+0000", 2023-09-23T19:33:48.076 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T19:33:48.076 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.076 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.076 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:33:48.076 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:02.055429+0000", 2023-09-23T19:33:48.076 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.5387000000000002e-05 2023-09-23T19:33:48.076 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.076 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.077 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:33:48.077 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:02.055438+0000", 2023-09-23T19:33:48.077 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.2819999999999996e-06 2023-09-23T19:33:48.077 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.077 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.077 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:33:48.077 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:02.055443+0000", 2023-09-23T19:33:48.077 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.2870000000000004e-06 2023-09-23T19:33:48.077 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.077 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.077 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:33:48.077 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:02.064684+0000", 2023-09-23T19:33:48.077 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0092407889999999993 2023-09-23T19:33:48.078 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.078 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.078 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:33:48.078 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:02.064746+0000", 2023-09-23T19:33:48.078 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.1925000000000001e-05 2023-09-23T19:33:48.078 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.078 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.078 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:33:48.078 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:02.074115+0000", 2023-09-23T19:33:48.078 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0093696189999999992 2023-09-23T19:33:48.078 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.078 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.078 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:33:48.079 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:02.074134+0000", 2023-09-23T19:33:48.079 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.8204999999999999e-05 2023-09-23T19:33:48.079 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:33:48.079 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:33:48.079 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:33:48.079 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.079 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.079 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45671.0:106905 9.6 e127/119 9:62362d75:::benchmark_data_smithi016_113233_object106904:head v 127'6572, mlcod=127'6567)", 2023-09-23T19:33:48.079 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:33:02.424913+0000", 2023-09-23T19:33:48.079 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 45.610771266, 2023-09-23T19:33:48.079 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.020261370000000001, 2023-09-23T19:33:48.079 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:33:48.079 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:33:48.080 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:33:48.080 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.080 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:33:48.080 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:02.424913+0000", 2023-09-23T19:33:48.080 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:33:48.080 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.080 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.080 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:33:48.080 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:02.424913+0000", 2023-09-23T19:33:48.080 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:33:48.080 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.080 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.081 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:33:48.081 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:02.424913+0000", 2023-09-23T19:33:48.081 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T19:33:48.081 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.081 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.081 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:33:48.081 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:02.424962+0000", 2023-09-23T19:33:48.081 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.9450000000000003e-05 2023-09-23T19:33:48.081 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.081 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.081 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:33:48.082 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:02.424997+0000", 2023-09-23T19:33:48.082 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.5009000000000003e-05 2023-09-23T19:33:48.082 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.082 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.082 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:33:48.082 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:02.425004+0000", 2023-09-23T19:33:48.082 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.2840000000000001e-06 2023-09-23T19:33:48.082 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.082 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.082 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:33:48.082 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:02.429028+0000", 2023-09-23T19:33:48.082 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0040240809999999997 2023-09-23T19:33:48.083 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.083 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.083 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:33:48.083 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:02.429049+0000", 2023-09-23T19:33:48.083 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.0373999999999998e-05 2023-09-23T19:33:48.083 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.083 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.083 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:33:48.083 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:02.445039+0000", 2023-09-23T19:33:48.083 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.015990553000000001 2023-09-23T19:33:48.083 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.084 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.084 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:33:48.084 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:02.445175+0000", 2023-09-23T19:33:48.084 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00013520000000000001 2023-09-23T19:33:48.084 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:33:48.084 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:33:48.084 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:33:48.084 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.084 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.084 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45671.0:106992 9.6 e127/119 9:65433dae:::benchmark_data_smithi016_113233_object106991:head v 127'6581, mlcod=127'6579)", 2023-09-23T19:33:48.084 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:33:02.533069+0000", 2023-09-23T19:33:48.084 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 45.502615456999997, 2023-09-23T19:33:48.085 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.020679797999999999, 2023-09-23T19:33:48.085 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:33:48.085 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:33:48.085 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:33:48.085 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.085 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:33:48.085 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:02.533069+0000", 2023-09-23T19:33:48.085 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:33:48.085 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.085 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.085 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:33:48.085 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:02.533069+0000", 2023-09-23T19:33:48.086 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:33:48.086 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.086 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.086 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:33:48.086 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:02.533068+0000", 2023-09-23T19:33:48.086 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-09-23T19:33:48.086 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.086 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.086 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:33:48.086 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:02.533120+0000", 2023-09-23T19:33:48.086 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.2821000000000001e-05 2023-09-23T19:33:48.087 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.087 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.087 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:33:48.087 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:02.533139+0000", 2023-09-23T19:33:48.087 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.9059000000000002e-05 2023-09-23T19:33:48.087 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.087 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.087 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:33:48.087 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:02.533146+0000", 2023-09-23T19:33:48.087 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.0389999999999999e-06 2023-09-23T19:33:48.087 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.087 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.088 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:33:48.088 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:02.533202+0000", 2023-09-23T19:33:48.088 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.5451999999999997e-05 2023-09-23T19:33:48.088 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.088 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.088 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:33:48.088 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:02.533228+0000", 2023-09-23T19:33:48.088 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.6347999999999999e-05 2023-09-23T19:33:48.088 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.088 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.088 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:33:48.088 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:02.553450+0000", 2023-09-23T19:33:48.089 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.020221468999999999 2023-09-23T19:33:48.089 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.089 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.089 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:33:48.089 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:02.553749+0000", 2023-09-23T19:33:48.089 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00029903800000000001 2023-09-23T19:33:48.089 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:33:48.089 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:33:48.089 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:33:48.089 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.089 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.089 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45671.0:118596 9.0 e128/119 9:0acdcdd0:::benchmark_data_smithi016_113233_object10214:head v 128'7386, mlcod=128'7384)", 2023-09-23T19:33:48.089 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:33:17.173956+0000", 2023-09-23T19:33:48.090 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 30.861728575000001, 2023-09-23T19:33:48.090 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.018587184999999999, 2023-09-23T19:33:48.090 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:33:48.090 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:33:48.090 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:33:48.090 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.090 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:33:48.090 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:17.173956+0000", 2023-09-23T19:33:48.090 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:33:48.090 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.090 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.090 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:33:48.091 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:17.173956+0000", 2023-09-23T19:33:48.091 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:33:48.091 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.091 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.091 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:33:48.091 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:17.173955+0000", 2023-09-23T19:33:48.091 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:33:48.091 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.091 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.091 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:33:48.091 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:17.173963+0000", 2023-09-23T19:33:48.091 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.7349999999999996e-06 2023-09-23T19:33:48.092 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.092 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.092 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:33:48.092 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:17.173980+0000", 2023-09-23T19:33:48.092 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.7479e-05 2023-09-23T19:33:48.092 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.092 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.092 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:33:48.092 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:17.173988+0000", 2023-09-23T19:33:48.092 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.2900000000000002e-06 2023-09-23T19:33:48.092 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.093 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.093 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:33:48.093 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:17.174041+0000", 2023-09-23T19:33:48.093 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.2170999999999999e-05 2023-09-23T19:33:48.093 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.093 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.093 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:33:48.093 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:17.174062+0000", 2023-09-23T19:33:48.093 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.1333000000000001e-05 2023-09-23T19:33:48.093 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.093 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.093 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:33:48.094 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:17.192400+0000", 2023-09-23T19:33:48.094 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.018337802 2023-09-23T19:33:48.094 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.094 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.094 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:33:48.094 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:17.192543+0000", 2023-09-23T19:33:48.094 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00014322099999999999 2023-09-23T19:33:48.094 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:33:48.094 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:33:48.094 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:33:48.094 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.095 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.095 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45671.0:121258 9.0 e128/119 9:05717355:::benchmark_data_smithi016_113233_object12876:head v 128'7573, mlcod=128'7571)", 2023-09-23T19:33:48.095 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:33:20.409900+0000", 2023-09-23T19:33:48.095 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 27.625784781, 2023-09-23T19:33:48.095 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.022790564999999999, 2023-09-23T19:33:48.095 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:33:48.095 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:33:48.095 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:33:48.095 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.095 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:33:48.095 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:20.409900+0000", 2023-09-23T19:33:48.095 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:33:48.096 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.096 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.096 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:33:48.096 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:20.409900+0000", 2023-09-23T19:33:48.096 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:33:48.096 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.096 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.096 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:33:48.096 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:20.409898+0000", 2023-09-23T19:33:48.096 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-09-23T19:33:48.096 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.096 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.096 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:33:48.097 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:20.409914+0000", 2023-09-23T19:33:48.097 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.5729000000000001e-05 2023-09-23T19:33:48.097 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.097 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.097 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:33:48.097 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:20.409942+0000", 2023-09-23T19:33:48.097 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.7875999999999999e-05 2023-09-23T19:33:48.097 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.097 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.097 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:33:48.097 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:20.409953+0000", 2023-09-23T19:33:48.097 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1122e-05 2023-09-23T19:33:48.097 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.098 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.098 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:33:48.098 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:20.410023+0000", 2023-09-23T19:33:48.098 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.0245000000000003e-05 2023-09-23T19:33:48.098 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.098 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.098 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:33:48.098 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:20.410124+0000", 2023-09-23T19:33:48.098 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00010038400000000001 2023-09-23T19:33:48.098 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.098 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.098 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:33:48.099 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:20.432361+0000", 2023-09-23T19:33:48.099 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.022237684000000001 2023-09-23T19:33:48.099 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.099 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.099 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:33:48.099 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:20.432690+0000", 2023-09-23T19:33:48.099 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00032896800000000001 2023-09-23T19:33:48.099 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:33:48.099 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:33:48.099 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:33:48.099 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.099 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.100 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45671.0:121260 9.0 e128/119 9:0f69e44b:::benchmark_data_smithi016_113233_object12878:head v 128'7574, mlcod=128'7571)", 2023-09-23T19:33:48.100 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:33:20.411250+0000", 2023-09-23T19:33:48.100 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 27.624434282999999, 2023-09-23T19:33:48.100 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.021733643, 2023-09-23T19:33:48.100 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:33:48.100 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:33:48.100 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:33:48.100 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.100 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:33:48.100 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:20.411250+0000", 2023-09-23T19:33:48.100 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:33:48.100 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.101 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.101 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:33:48.101 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:20.411250+0000", 2023-09-23T19:33:48.101 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:33:48.101 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.101 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.101 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:33:48.101 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:20.411249+0000", 2023-09-23T19:33:48.101 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:33:48.101 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.101 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.101 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:33:48.102 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:20.411256+0000", 2023-09-23T19:33:48.102 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.1729999999999998e-06 2023-09-23T19:33:48.102 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.102 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.102 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:33:48.102 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:20.411273+0000", 2023-09-23T19:33:48.102 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.6969999999999998e-05 2023-09-23T19:33:48.102 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.102 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.102 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:33:48.102 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:20.411282+0000", 2023-09-23T19:33:48.102 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.0930000000000003e-06 2023-09-23T19:33:48.102 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.103 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.103 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:33:48.103 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:20.417600+0000", 2023-09-23T19:33:48.103 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0063180809999999997 2023-09-23T19:33:48.103 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.103 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.103 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:33:48.103 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:20.417625+0000", 2023-09-23T19:33:48.103 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.5748000000000001e-05 2023-09-23T19:33:48.103 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.103 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.103 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:33:48.104 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:20.432698+0000", 2023-09-23T19:33:48.104 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.015072321 2023-09-23T19:33:48.104 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.104 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.104 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:33:48.104 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:20.432984+0000", 2023-09-23T19:33:48.104 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00028605900000000002 2023-09-23T19:33:48.104 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:33:48.104 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:33:48.104 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:33:48.104 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.104 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.105 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45671.0:121262 9.0 e128/119 9:0c0eaf99:::benchmark_data_smithi016_113233_object12880:head v 128'7575, mlcod=128'7571)", 2023-09-23T19:33:48.105 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:33:20.418517+0000", 2023-09-23T19:33:48.105 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 27.617167854000002, 2023-09-23T19:33:48.105 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.022568090999999998, 2023-09-23T19:33:48.105 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:33:48.105 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:33:48.105 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:33:48.105 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.105 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:33:48.105 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:20.418517+0000", 2023-09-23T19:33:48.105 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:33:48.105 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.106 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.106 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:33:48.106 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:20.418517+0000", 2023-09-23T19:33:48.106 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:33:48.106 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.106 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.106 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:33:48.106 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:20.418516+0000", 2023-09-23T19:33:48.106 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:33:48.106 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.106 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.106 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:33:48.106 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:20.418525+0000", 2023-09-23T19:33:48.107 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.0829999999999993e-06 2023-09-23T19:33:48.107 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.107 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.107 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:33:48.107 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:20.418542+0000", 2023-09-23T19:33:48.107 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.7340999999999999e-05 2023-09-23T19:33:48.107 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.107 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.107 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:33:48.107 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:20.418550+0000", 2023-09-23T19:33:48.107 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.1529999999999997e-06 2023-09-23T19:33:48.107 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.107 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.108 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:33:48.108 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:20.433057+0000", 2023-09-23T19:33:48.108 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.014506493000000001 2023-09-23T19:33:48.108 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.108 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.108 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:33:48.108 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:20.433080+0000", 2023-09-23T19:33:48.108 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.3405999999999999e-05 2023-09-23T19:33:48.108 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.108 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.108 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:33:48.108 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:20.438297+0000", 2023-09-23T19:33:48.109 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0052171500000000003 2023-09-23T19:33:48.109 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.109 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.109 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:33:48.109 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:20.441085+0000", 2023-09-23T19:33:48.109 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.002787496 2023-09-23T19:33:48.109 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:33:48.109 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:33:48.109 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:33:48.109 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.109 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.109 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45671.0:134696 9.0 e129/119 9:07d2d9bc:::benchmark_data_smithi016_113233_object26314:head v 129'8424, mlcod=129'8420)", 2023-09-23T19:33:48.109 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:33:38.061321+0000", 2023-09-23T19:33:48.110 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 9.9743636710000008, 2023-09-23T19:33:48.110 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.018579458, 2023-09-23T19:33:48.110 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:33:48.110 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:33:48.110 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:33:48.110 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.110 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:33:48.110 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:38.061321+0000", 2023-09-23T19:33:48.110 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:33:48.110 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.110 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.110 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:33:48.111 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:38.061321+0000", 2023-09-23T19:33:48.111 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:33:48.111 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.111 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.111 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:33:48.111 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:38.061320+0000", 2023-09-23T19:33:48.111 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:33:48.111 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.111 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.111 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:33:48.111 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:38.061329+0000", 2023-09-23T19:33:48.111 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.2660000000000007e-06 2023-09-23T19:33:48.112 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.112 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.112 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:33:48.112 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:38.061341+0000", 2023-09-23T19:33:48.112 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2519e-05 2023-09-23T19:33:48.112 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.112 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.112 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:33:48.112 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:38.061350+0000", 2023-09-23T19:33:48.112 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.6410000000000008e-06 2023-09-23T19:33:48.112 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.112 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.113 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:33:48.113 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:38.063211+0000", 2023-09-23T19:33:48.113 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00186123 2023-09-23T19:33:48.113 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.113 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.113 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:33:48.113 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:38.063235+0000", 2023-09-23T19:33:48.113 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.3581999999999999e-05 2023-09-23T19:33:48.113 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.113 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.113 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:33:48.113 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:38.079860+0000", 2023-09-23T19:33:48.113 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.01662543 2023-09-23T19:33:48.114 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.114 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.114 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:33:48.114 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:38.079900+0000", 2023-09-23T19:33:48.114 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.9827999999999999e-05 2023-09-23T19:33:48.114 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:33:48.114 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:33:48.114 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:33:48.114 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.114 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.114 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45671.0:136243 9.2 e129/126 9:404b4d5c:::benchmark_data_smithi016_113233_object27861:head v 129'8399, mlcod=129'8395)", 2023-09-23T19:33:48.114 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:33:40.132321+0000", 2023-09-23T19:33:48.115 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 7.903363176, 2023-09-23T19:33:48.115 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.020371271999999999, 2023-09-23T19:33:48.115 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:33:48.115 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:33:48.115 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:33:48.115 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.115 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:33:48.115 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:40.132321+0000", 2023-09-23T19:33:48.115 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:33:48.115 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.115 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.115 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:33:48.116 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:40.132321+0000", 2023-09-23T19:33:48.116 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:33:48.116 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.116 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.116 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:33:48.116 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:40.132320+0000", 2023-09-23T19:33:48.116 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-09-23T19:33:48.116 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.116 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.116 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:33:48.116 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:40.132329+0000", 2023-09-23T19:33:48.116 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.5759999999999995e-06 2023-09-23T19:33:48.117 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.117 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.117 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:33:48.117 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:40.132349+0000", 2023-09-23T19:33:48.117 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.9278000000000001e-05 2023-09-23T19:33:48.117 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.117 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.117 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:33:48.117 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:40.132357+0000", 2023-09-23T19:33:48.117 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.2199999999999992e-06 2023-09-23T19:33:48.117 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.117 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.117 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:33:48.118 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:40.132542+0000", 2023-09-23T19:33:48.118 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00018466500000000001 2023-09-23T19:33:48.118 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.118 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.118 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:33:48.118 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:40.132795+0000", 2023-09-23T19:33:48.118 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00025365199999999999 2023-09-23T19:33:48.118 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.118 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.118 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:33:48.118 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:40.145793+0000", 2023-09-23T19:33:48.118 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.012997784 2023-09-23T19:33:48.119 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.119 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.119 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:33:48.119 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:40.152692+0000", 2023-09-23T19:33:48.119 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0068995020000000001 2023-09-23T19:33:48.119 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:33:48.119 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:33:48.119 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:33:48.119 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.119 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.119 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45671.0:136259 9.2 e129/126 9:413af13f:::benchmark_data_smithi016_113233_object27877:head v 129'8401, mlcod=129'8397)", 2023-09-23T19:33:48.120 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:33:40.143224+0000", 2023-09-23T19:33:48.120 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 7.8924605440000004, 2023-09-23T19:33:48.120 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.021100279999999999, 2023-09-23T19:33:48.120 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:33:48.120 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:33:48.120 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:33:48.120 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.120 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:33:48.120 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:40.143224+0000", 2023-09-23T19:33:48.120 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:33:48.120 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.120 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.121 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:33:48.121 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:40.143224+0000", 2023-09-23T19:33:48.121 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:33:48.121 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.121 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.121 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:33:48.121 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:40.143223+0000", 2023-09-23T19:33:48.121 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-09-23T19:33:48.121 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.121 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.121 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:33:48.121 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:40.143234+0000", 2023-09-23T19:33:48.121 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1936999999999999e-05 2023-09-23T19:33:48.122 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.122 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.122 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:33:48.122 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:40.143257+0000", 2023-09-23T19:33:48.122 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.2775000000000001e-05 2023-09-23T19:33:48.122 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.122 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.122 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:33:48.122 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:40.143268+0000", 2023-09-23T19:33:48.122 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1201e-05 2023-09-23T19:33:48.122 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.122 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.123 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:33:48.123 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:40.153089+0000", 2023-09-23T19:33:48.123 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0098203700000000001 2023-09-23T19:33:48.123 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.123 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.123 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:33:48.123 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:40.153164+0000", 2023-09-23T19:33:48.123 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.5012999999999995e-05 2023-09-23T19:33:48.123 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.123 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.123 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:33:48.123 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:40.164269+0000", 2023-09-23T19:33:48.124 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.011105265 2023-09-23T19:33:48.124 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.124 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.124 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:33:48.124 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:40.164324+0000", 2023-09-23T19:33:48.124 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.5038999999999999e-05 2023-09-23T19:33:48.124 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:33:48.124 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:33:48.124 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:33:48.124 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.124 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.124 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45671.0:139211 9.0 e129/119 9:09076ad2:::benchmark_data_smithi016_113233_object30829:head v 129'8705, mlcod=129'8703)", 2023-09-23T19:33:48.125 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:33:44.160329+0000", 2023-09-23T19:33:48.125 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 3.8753555889999998, 2023-09-23T19:33:48.125 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.02003214, 2023-09-23T19:33:48.125 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:33:48.125 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:33:48.125 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:33:48.125 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.125 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:33:48.125 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:44.160329+0000", 2023-09-23T19:33:48.125 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:33:48.125 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.125 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.126 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:33:48.126 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:44.160329+0000", 2023-09-23T19:33:48.126 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:33:48.126 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.126 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.126 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:33:48.126 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:44.160328+0000", 2023-09-23T19:33:48.126 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-09-23T19:33:48.126 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.126 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.126 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:33:48.126 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:44.160339+0000", 2023-09-23T19:33:48.127 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1251e-05 2023-09-23T19:33:48.127 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.127 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.127 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:33:48.127 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:44.160363+0000", 2023-09-23T19:33:48.127 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.4596e-05 2023-09-23T19:33:48.127 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.127 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.127 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:33:48.127 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:44.160373+0000", 2023-09-23T19:33:48.127 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.2569999999999996e-06 2023-09-23T19:33:48.127 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.127 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.128 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:33:48.128 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:44.160423+0000", 2023-09-23T19:33:48.128 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.0257999999999999e-05 2023-09-23T19:33:48.128 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.128 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.128 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:33:48.128 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:44.160447+0000", 2023-09-23T19:33:48.128 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.4082000000000001e-05 2023-09-23T19:33:48.128 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.128 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.128 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:33:48.128 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:44.180273+0000", 2023-09-23T19:33:48.129 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.019826094999999998 2023-09-23T19:33:48.129 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.129 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.129 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:33:48.129 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:44.180361+0000", 2023-09-23T19:33:48.129 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.7853000000000001e-05 2023-09-23T19:33:48.129 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:33:48.129 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:33:48.129 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:33:48.129 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.129 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.129 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45671.0:140462 9.2 e129/126 9:42865912:::benchmark_data_smithi016_113233_object32080:head v 129'8647, mlcod=129'8644)", 2023-09-23T19:33:48.130 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:33:45.843636+0000", 2023-09-23T19:33:48.130 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 2.1920479799999999, 2023-09-23T19:33:48.130 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.019068897000000001, 2023-09-23T19:33:48.130 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:33:48.130 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:33:48.130 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:33:48.130 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.130 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:33:48.130 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:45.843636+0000", 2023-09-23T19:33:48.130 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:33:48.130 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.130 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.131 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:33:48.131 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:45.843636+0000", 2023-09-23T19:33:48.131 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:33:48.131 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.131 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.131 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:33:48.131 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:45.843636+0000", 2023-09-23T19:33:48.131 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:33:48.131 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.131 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.131 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:33:48.131 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:45.843642+0000", 2023-09-23T19:33:48.132 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.5690000000000001e-06 2023-09-23T19:33:48.132 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.132 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.132 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:33:48.132 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:45.843655+0000", 2023-09-23T19:33:48.132 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3093e-05 2023-09-23T19:33:48.132 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.132 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.132 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:33:48.132 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:45.843661+0000", 2023-09-23T19:33:48.132 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.0279999999999997e-06 2023-09-23T19:33:48.132 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.133 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.133 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:33:48.133 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:45.843708+0000", 2023-09-23T19:33:48.133 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.6863999999999997e-05 2023-09-23T19:33:48.133 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.133 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.133 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:33:48.133 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:45.843755+0000", 2023-09-23T19:33:48.133 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.6576000000000003e-05 2023-09-23T19:33:48.133 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.133 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.133 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:33:48.133 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:45.862664+0000", 2023-09-23T19:33:48.134 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.018909197999999999 2023-09-23T19:33:48.134 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.134 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.134 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:33:48.134 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:45.862705+0000", 2023-09-23T19:33:48.134 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.1362999999999997e-05 2023-09-23T19:33:48.134 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:33:48.134 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:33:48.134 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:33:48.134 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.134 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.134 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45671.0:140464 9.6 e129/119 9:6fc8fe72:::benchmark_data_smithi016_113233_object32082:head v 129'8583, mlcod=129'8581)", 2023-09-23T19:33:48.134 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:33:45.845839+0000", 2023-09-23T19:33:48.135 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 2.1898453689999999, 2023-09-23T19:33:48.135 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.023345395000000001, 2023-09-23T19:33:48.135 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:33:48.135 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:33:48.135 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:33:48.135 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.135 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:33:48.135 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:45.845839+0000", 2023-09-23T19:33:48.135 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:33:48.135 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.135 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.135 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:33:48.135 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:45.845839+0000", 2023-09-23T19:33:48.136 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:33:48.136 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.136 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.136 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:33:48.136 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:45.845838+0000", 2023-09-23T19:33:48.136 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:33:48.136 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.136 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.136 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:33:48.136 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:45.845847+0000", 2023-09-23T19:33:48.136 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.3280000000000006e-06 2023-09-23T19:33:48.136 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.137 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.137 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:33:48.137 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:45.845864+0000", 2023-09-23T19:33:48.137 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.7257000000000001e-05 2023-09-23T19:33:48.137 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.137 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.137 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:33:48.137 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:45.845871+0000", 2023-09-23T19:33:48.137 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.8040000000000004e-06 2023-09-23T19:33:48.137 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.137 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.137 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:33:48.138 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:45.846016+0000", 2023-09-23T19:33:48.138 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00014544100000000001 2023-09-23T19:33:48.138 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.138 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.138 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:33:48.138 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:45.846069+0000", 2023-09-23T19:33:48.138 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.3279999999999998e-05 2023-09-23T19:33:48.138 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.138 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.138 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:33:48.138 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:45.869070+0000", 2023-09-23T19:33:48.138 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.023000902 2023-09-23T19:33:48.138 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.139 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.139 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:33:48.139 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:45.869184+0000", 2023-09-23T19:33:48.139 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000114171 2023-09-23T19:33:48.139 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:33:48.139 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:33:48.139 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:33:48.139 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.139 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.139 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45671.0:140468 9.6 e129/119 9:657dd681:::benchmark_data_smithi016_113233_object32086:head v 129'8584, mlcod=129'8581)", 2023-09-23T19:33:48.139 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:33:45.850650+0000", 2023-09-23T19:33:48.139 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 2.1850345309999999, 2023-09-23T19:33:48.140 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.018636551000000001, 2023-09-23T19:33:48.140 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:33:48.140 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:33:48.140 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:33:48.140 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.140 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:33:48.140 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:45.850650+0000", 2023-09-23T19:33:48.140 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:33:48.140 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.140 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.140 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:33:48.140 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:45.850650+0000", 2023-09-23T19:33:48.140 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:33:48.141 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.141 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.141 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:33:48.141 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:45.850649+0000", 2023-09-23T19:33:48.141 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:33:48.141 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.141 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.141 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:33:48.141 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:45.850658+0000", 2023-09-23T19:33:48.141 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.3219999999999999e-06 2023-09-23T19:33:48.141 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.141 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.141 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:33:48.142 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:45.850678+0000", 2023-09-23T19:33:48.142 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.9996999999999998e-05 2023-09-23T19:33:48.142 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.142 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.142 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:33:48.142 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:45.850688+0000", 2023-09-23T19:33:48.142 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.9599999999999995e-06 2023-09-23T19:33:48.142 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.142 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.142 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:33:48.142 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:45.858033+0000", 2023-09-23T19:33:48.142 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0073447800000000004 2023-09-23T19:33:48.143 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.143 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.143 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:33:48.143 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:45.858055+0000", 2023-09-23T19:33:48.143 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.211e-05 2023-09-23T19:33:48.143 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.143 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.143 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:33:48.143 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:45.869191+0000", 2023-09-23T19:33:48.143 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.011135845 2023-09-23T19:33:48.143 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.143 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.144 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:33:48.144 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:45.869286+0000", 2023-09-23T19:33:48.144 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.5434999999999997e-05 2023-09-23T19:33:48.144 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:33:48.145 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:33:48.145 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:33:48.145 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.145 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.145 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45671.0:140474 9.6 e129/119 9:69fd0fe1:::benchmark_data_smithi016_113233_object32092:head v 129'8586, mlcod=129'8581)", 2023-09-23T19:33:48.145 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:33:45.860928+0000", 2023-09-23T19:33:48.145 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 2.1747559160000001, 2023-09-23T19:33:48.145 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.021189187000000002, 2023-09-23T19:33:48.145 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:33:48.146 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:33:48.146 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:33:48.146 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.146 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:33:48.146 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:45.860928+0000", 2023-09-23T19:33:48.146 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:33:48.146 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.146 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.146 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:33:48.146 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:45.860928+0000", 2023-09-23T19:33:48.146 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:33:48.146 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.147 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.147 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:33:48.147 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:45.860928+0000", 2023-09-23T19:33:48.147 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T19:33:48.147 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.147 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.147 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:33:48.147 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:45.860935+0000", 2023-09-23T19:33:48.147 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.6630000000000004e-06 2023-09-23T19:33:48.147 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.147 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.147 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:33:48.147 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:45.860950+0000", 2023-09-23T19:33:48.147 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.5308e-05 2023-09-23T19:33:48.148 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.148 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.148 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:33:48.148 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:45.860957+0000", 2023-09-23T19:33:48.148 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.7800000000000003e-06 2023-09-23T19:33:48.148 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.148 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.148 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:33:48.148 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:45.869502+0000", 2023-09-23T19:33:48.148 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0085455229999999993 2023-09-23T19:33:48.148 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.148 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.148 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:33:48.149 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:45.869520+0000", 2023-09-23T19:33:48.149 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.7649999999999999e-05 2023-09-23T19:33:48.149 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.149 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.149 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:33:48.149 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:45.881952+0000", 2023-09-23T19:33:48.149 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.012431994 2023-09-23T19:33:48.149 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:33:48.149 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:33:48.149 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:33:48.149 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:45.882118+0000", 2023-09-23T19:33:48.149 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00016586800000000001 2023-09-23T19:33:48.149 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:33:48.149 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:33:48.150 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:33:48.150 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:33:48.150 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:33:48.150 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-09-23T19:33:48.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:48.151 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:48.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:48.251 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:48.277 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:48.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:48.279 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:33:48.380 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:48.408 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:48.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:48.412 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:48.513 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:48.542 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:48.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:48.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:48.650 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:48.677 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:48.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:48.680 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:33:48.780 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:48.810 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:48.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:48.814 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:33:48.914 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:49.023 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:49.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:49.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:33:49.127 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:49.156 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:49.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:49.158 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:33:49.259 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:49.289 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:49.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:49.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:33:49.394 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:49.425 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:49.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:49.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:49.529 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:49.557 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:49.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:49.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:49.573 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is unset 2023-09-23T19:33:49.666 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:49.675 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:49 smithi016 bash[77995]: cluster 2023-09-23T19:33:48.007349+0000 mgr.y (mgr.25325) 786 : cluster 0 pgmap v689: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 754 op/s 2023-09-23T19:33:49.675 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:49 smithi016 bash[77995]: audit 2023-09-23T19:33:49.146072+0000 mon.c (mon.2) 147 : audit 1 from='client.? 172.21.15.16:0/1131530721' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:33:49.675 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:49 smithi016 bash[77995]: audit 2023-09-23T19:33:49.148197+0000 mon.a (mon.0) 975 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:33:49.675 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:49 smithi016 bash[75004]: cluster 2023-09-23T19:33:48.007349+0000 mgr.y (mgr.25325) 786 : cluster 0 pgmap v689: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 754 op/s 2023-09-23T19:33:49.675 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:49 smithi016 bash[75004]: audit 2023-09-23T19:33:49.146072+0000 mon.c (mon.2) 147 : audit 1 from='client.? 172.21.15.16:0/1131530721' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:33:49.675 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:49 smithi016 bash[75004]: audit 2023-09-23T19:33:49.148197+0000 mon.a (mon.0) 975 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:33:49.693 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:49.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:49.697 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:49.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:49 smithi092 bash[58855]: cluster 2023-09-23T19:33:48.007349+0000 mgr.y (mgr.25325) 786 : cluster 0 pgmap v689: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 754 op/s 2023-09-23T19:33:49.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:49 smithi092 bash[58855]: audit 2023-09-23T19:33:49.146072+0000 mon.c (mon.2) 147 : audit 1 from='client.? 172.21.15.16:0/1131530721' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:33:49.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:49 smithi092 bash[58855]: audit 2023-09-23T19:33:49.148197+0000 mon.a (mon.0) 975 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:33:49.797 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:49.837 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:49.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:49.841 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:49.942 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:49.969 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:49.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:49.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:33:50.073 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:50.103 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:50.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:50.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:50.208 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:50.228 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:50.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:50.232 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:50.332 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:50.368 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:50.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:50.371 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:33:50.471 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:50.499 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:50.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:50.505 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:50.605 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:50.613 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:50 smithi092 bash[58855]: cluster 2023-09-23T19:33:49.552793+0000 mon.a (mon.0) 976 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:33:50.613 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:50 smithi092 bash[58855]: cluster 2023-09-23T19:33:49.552825+0000 mon.a (mon.0) 977 : cluster 1 Cluster is now healthy 2023-09-23T19:33:50.613 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:50 smithi092 bash[58855]: audit 2023-09-23T19:33:49.559677+0000 mon.a (mon.0) 978 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:33:50.613 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:50 smithi092 bash[58855]: cluster 2023-09-23T19:33:49.560463+0000 osd.4 (osd.4) 58 : cluster 3 failed to encode map e130 with expected crc 2023-09-23T19:33:50.614 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:50 smithi092 bash[58855]: cluster 2023-09-23T19:33:49.560623+0000 mon.a (mon.0) 979 : cluster 0 osdmap e130: 8 total, 8 up, 8 in 2023-09-23T19:33:50.614 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:50 smithi092 bash[58855]: cluster 2023-09-23T19:33:49.568960+0000 osd.7 (osd.7) 49 : cluster 3 failed to encode map e130 with expected crc 2023-09-23T19:33:50.614 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:50 smithi092 bash[58855]: cluster 2023-09-23T19:33:49.571557+0000 osd.5 (osd.5) 60 : cluster 3 failed to encode map e130 with expected crc 2023-09-23T19:33:50.626 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:50.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:50.629 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:33:50.729 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:50.750 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:50.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:50.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:50.758 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:50 smithi016 bash[77995]: cluster 2023-09-23T19:33:49.552793+0000 mon.a (mon.0) 976 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:33:50.758 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:50 smithi016 bash[77995]: cluster 2023-09-23T19:33:49.552825+0000 mon.a (mon.0) 977 : cluster 1 Cluster is now healthy 2023-09-23T19:33:50.758 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:50 smithi016 bash[77995]: audit 2023-09-23T19:33:49.559677+0000 mon.a (mon.0) 978 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:33:50.758 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:50 smithi016 bash[77995]: cluster 2023-09-23T19:33:49.560463+0000 osd.4 (osd.4) 58 : cluster 3 failed to encode map e130 with expected crc 2023-09-23T19:33:50.759 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:50 smithi016 bash[77995]: cluster 2023-09-23T19:33:49.560623+0000 mon.a (mon.0) 979 : cluster 0 osdmap e130: 8 total, 8 up, 8 in 2023-09-23T19:33:50.759 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:50 smithi016 bash[77995]: cluster 2023-09-23T19:33:49.568960+0000 osd.7 (osd.7) 49 : cluster 3 failed to encode map e130 with expected crc 2023-09-23T19:33:50.759 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:50 smithi016 bash[77995]: cluster 2023-09-23T19:33:49.571557+0000 osd.5 (osd.5) 60 : cluster 3 failed to encode map e130 with expected crc 2023-09-23T19:33:50.759 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:50 smithi016 bash[75004]: cluster 2023-09-23T19:33:49.552793+0000 mon.a (mon.0) 976 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:33:50.759 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:50 smithi016 bash[75004]: cluster 2023-09-23T19:33:49.552825+0000 mon.a (mon.0) 977 : cluster 1 Cluster is now healthy 2023-09-23T19:33:50.759 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:50 smithi016 bash[75004]: audit 2023-09-23T19:33:49.559677+0000 mon.a (mon.0) 978 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:33:50.759 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:50 smithi016 bash[75004]: cluster 2023-09-23T19:33:49.560463+0000 osd.4 (osd.4) 58 : cluster 3 failed to encode map e130 with expected crc 2023-09-23T19:33:50.759 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:50 smithi016 bash[75004]: cluster 2023-09-23T19:33:49.560623+0000 mon.a (mon.0) 979 : cluster 0 osdmap e130: 8 total, 8 up, 8 in 2023-09-23T19:33:50.759 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:50 smithi016 bash[75004]: cluster 2023-09-23T19:33:49.568960+0000 osd.7 (osd.7) 49 : cluster 3 failed to encode map e130 with expected crc 2023-09-23T19:33:50.759 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:50 smithi016 bash[75004]: cluster 2023-09-23T19:33:49.571557+0000 osd.5 (osd.5) 60 : cluster 3 failed to encode map e130 with expected crc 2023-09-23T19:33:50.853 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:50.880 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:50.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:50.883 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:50.984 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:51.178 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T19:33:51.220 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:51.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:51.222 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:33:51.323 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:51.357 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:51.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:51.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:33:51.460 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:51.485 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:51.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:51.488 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:51.589 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:51.620 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:51.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:51.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:33:51.725 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:51.743 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:51 smithi016 bash[77995]: cluster 2023-09-23T19:33:50.007825+0000 mgr.y (mgr.25325) 787 : cluster 0 pgmap v691: 89 pgs: 89 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 737 op/s 2023-09-23T19:33:51.745 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:51 smithi016 bash[75004]: cluster 2023-09-23T19:33:50.007825+0000 mgr.y (mgr.25325) 787 : cluster 0 pgmap v691: 89 pgs: 89 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 737 op/s 2023-09-23T19:33:51.753 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:51.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:51.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:51.857 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:51.865 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:51 smithi092 bash[58855]: cluster 2023-09-23T19:33:50.007825+0000 mgr.y (mgr.25325) 787 : cluster 0 pgmap v691: 89 pgs: 89 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 737 op/s 2023-09-23T19:33:51.891 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:51.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:51.900 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:33:52.001 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:52.024 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:52.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:52.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:33:52.129 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:52.166 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:52.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:52.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:52.271 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:52.302 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:52.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:52.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:33:52.407 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:52.434 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:52.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:52.441 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:52.541 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:52.582 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:52.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:52.588 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:33:52.689 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:52.711 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:52.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:52.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:33:52.814 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:52.838 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:52.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:52.841 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:33:52.941 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:52.970 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:52.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:52.976 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:53.076 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:53.105 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:53.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:53.110 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:53.210 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:53.250 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:53.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:53.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:53.356 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:53.387 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:53.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:53.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:53.493 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:53.525 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:53.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:53.528 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:53.629 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:53.636 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:53 smithi016 bash[75004]: cluster 2023-09-23T19:33:52.008567+0000 mgr.y (mgr.25325) 788 : cluster 0 pgmap v692: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 723 op/s 2023-09-23T19:33:53.636 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:53 smithi016 bash[77995]: cluster 2023-09-23T19:33:52.008567+0000 mgr.y (mgr.25325) 788 : cluster 0 pgmap v692: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 723 op/s 2023-09-23T19:33:53.654 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:53.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:53.659 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:33:53.760 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:53.802 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:53.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:53.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:33:53.906 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:53.937 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:53.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:53.940 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:33:54.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:53 smithi092 bash[58855]: cluster 2023-09-23T19:33:52.008567+0000 mgr.y (mgr.25325) 788 : cluster 0 pgmap v692: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 723 op/s 2023-09-23T19:33:54.040 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:54.081 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:54.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:54.083 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:33:54.184 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:54.212 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:54.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:54.217 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:33:54.318 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:54.351 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:54.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:54.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:33:54.454 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:54.487 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:54.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:54.490 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:33:54.591 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:54.614 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:54.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:54.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:54.720 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:54.745 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:54.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:54.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:33:54.853 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:54.877 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:54.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:54.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:33:54.984 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:55.008 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:55.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:55.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:55.112 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:55.139 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:55.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:55.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:33:55.242 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:55.269 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:55.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:55.274 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:55.374 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:55.402 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:55.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:55.404 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:33:55.505 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:55.541 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:55.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:55.544 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:33:55.645 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:55.680 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:55.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:55.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:33:55.783 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:55.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:55 smithi092 bash[58855]: cluster 2023-09-23T19:33:54.009194+0000 mgr.y (mgr.25325) 789 : cluster 0 pgmap v693: 89 pgs: 89 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2023-09-23T19:33:55.816 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:55.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:55.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:33:55.862 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T19:33:55.903 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:55 smithi016 bash[75004]: cluster 2023-09-23T19:33:54.009194+0000 mgr.y (mgr.25325) 789 : cluster 0 pgmap v693: 89 pgs: 89 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2023-09-23T19:33:55.903 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:55 smithi016 bash[77995]: cluster 2023-09-23T19:33:54.009194+0000 mgr.y (mgr.25325) 789 : cluster 0 pgmap v693: 89 pgs: 89 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2023-09-23T19:33:55.903 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:33:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:33:55] "GET /metrics HTTP/1.1" 200 36236 "" "Prometheus/2.33.4" 2023-09-23T19:33:55.919 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:55.948 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:55.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:55.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:56.051 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:56.093 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:56.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:56.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:56.200 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:56.230 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:56.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:56.233 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:33:56.334 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:56.366 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:56.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:56.371 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:33:56.472 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:56.496 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:56.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:56.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:33:56.601 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:56.634 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:56.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:56.637 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:56.737 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:56.787 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:56.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:56.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:33:56.894 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:56.940 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:56.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:56.945 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:57.046 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:57.077 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:57.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:57.083 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:33:57.157 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:33:57.183 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:57.192 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:33:57.206 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:57.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:57.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:33:57.231 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:33:57.294 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:33:57.306 INFO:teuthology.orchestra.run.smithi016.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-09-23T19:33:57.313 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:57.340 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:33:57.343 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:57.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:57.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:33:57.381 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:33:57.425 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:33:57.449 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:57.461 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:33:57.465 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:33:57.465 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:33:57.465 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:33:57.465 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:33:57.465 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:33:57.465 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:33:57.465 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:33:57.466 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:33:57.484 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:57.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:57.490 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:57.591 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:57.622 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:57.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:57.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:33:57.727 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:57.750 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:33:57.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:57.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:33:57.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:57 smithi016 bash[77995]: cluster 2023-09-23T19:33:56.009942+0000 mgr.y (mgr.25325) 790 : cluster 0 pgmap v694: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 714 op/s 2023-09-23T19:33:57.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:57 smithi016 bash[77995]: audit 2023-09-23T19:33:57.097849+0000 mon.c (mon.2) 148 : audit 0 from='client.? 172.21.15.16:0/761378631' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:33:57.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:57 smithi016 bash[77995]: audit 2023-09-23T19:33:57.297342+0000 mon.c (mon.2) 149 : audit 1 from='client.? 172.21.15.16:0/246039530' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T19:33:57.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:57 smithi016 bash[77995]: audit 2023-09-23T19:33:57.298778+0000 mon.a (mon.0) 980 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T19:33:57.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:57 smithi016 bash[75004]: cluster 2023-09-23T19:33:56.009942+0000 mgr.y (mgr.25325) 790 : cluster 0 pgmap v694: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 714 op/s 2023-09-23T19:33:57.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:57 smithi016 bash[75004]: audit 2023-09-23T19:33:57.097849+0000 mon.c (mon.2) 148 : audit 0 from='client.? 172.21.15.16:0/761378631' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:33:57.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:57 smithi016 bash[75004]: audit 2023-09-23T19:33:57.297342+0000 mon.c (mon.2) 149 : audit 1 from='client.? 172.21.15.16:0/246039530' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T19:33:57.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:57 smithi016 bash[75004]: audit 2023-09-23T19:33:57.298778+0000 mon.a (mon.0) 980 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T19:33:57.855 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:57.863 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:57 smithi092 bash[58855]: cluster 2023-09-23T19:33:56.009942+0000 mgr.y (mgr.25325) 790 : cluster 0 pgmap v694: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 714 op/s 2023-09-23T19:33:57.863 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:57 smithi092 bash[58855]: audit 2023-09-23T19:33:57.097849+0000 mon.c (mon.2) 148 : audit 0 from='client.? 172.21.15.16:0/761378631' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:33:57.863 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:57 smithi092 bash[58855]: audit 2023-09-23T19:33:57.297342+0000 mon.c (mon.2) 149 : audit 1 from='client.? 172.21.15.16:0/246039530' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T19:33:57.864 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:57 smithi092 bash[58855]: audit 2023-09-23T19:33:57.298778+0000 mon.a (mon.0) 980 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T19:33:57.893 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:57.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:57.897 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:33:57.998 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:58.023 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:58.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:58.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:33:58.129 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:58.162 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:33:58.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:58.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:33:58.265 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:58.298 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:58.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:58.301 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:33:58.401 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:58.436 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:58.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:58.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:33:58.544 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:58.574 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:33:58.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:58.578 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:33:58.679 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:58.726 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:58.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:58.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:33:58.833 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:58.857 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:58.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:58.860 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:58.960 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:58.986 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:58.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:58.989 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:33:59.089 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:59.219 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:33:59.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:59.225 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:33:59.326 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:59.371 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:33:59.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:59.377 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:33:59.381 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-09-23T19:33:59.477 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:59.510 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:33:59.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:59.513 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:33:59.614 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:59.645 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:33:59.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:59.650 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:33:59.750 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:59.786 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:59.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:59.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:33:59.861 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:33:59.890 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:59.899 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:59 smithi092 bash[58855]: cluster 2023-09-23T19:33:58.010556+0000 mgr.y (mgr.25325) 791 : cluster 0 pgmap v695: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 712 op/s 2023-09-23T19:33:59.899 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:59 smithi092 bash[58855]: audit 2023-09-23T19:33:58.646046+0000 mon.a (mon.0) 981 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:33:59.899 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:33:59 smithi092 bash[58855]: audit 2023-09-23T19:33:58.649564+0000 mon.a (mon.0) 982 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:33:59.901 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:59 smithi016 bash[75004]: cluster 2023-09-23T19:33:58.010556+0000 mgr.y (mgr.25325) 791 : cluster 0 pgmap v695: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 712 op/s 2023-09-23T19:33:59.901 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:59 smithi016 bash[75004]: audit 2023-09-23T19:33:58.646046+0000 mon.a (mon.0) 981 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:33:59.901 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:33:59 smithi016 bash[75004]: audit 2023-09-23T19:33:58.649564+0000 mon.a (mon.0) 982 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:33:59.902 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:59 smithi016 bash[77995]: cluster 2023-09-23T19:33:58.010556+0000 mgr.y (mgr.25325) 791 : cluster 0 pgmap v695: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 712 op/s 2023-09-23T19:33:59.902 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:59 smithi016 bash[77995]: audit 2023-09-23T19:33:58.646046+0000 mon.a (mon.0) 981 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:33:59.902 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:33:59 smithi016 bash[77995]: audit 2023-09-23T19:33:58.649564+0000 mon.a (mon.0) 982 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:33:59.920 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:33:59.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:33:59.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:00.024 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:00.057 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:00.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:00.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:00.166 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:00.193 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:00.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:00.199 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:34:00.299 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:00.324 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:00.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:00.329 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:00.429 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:00.467 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:00.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:00.473 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:34:00.574 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:00.605 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:00.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:00.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:00.711 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:00.750 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:00.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:00.753 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:00.854 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:00.880 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:00.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:00.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:00.987 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:01.016 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:01.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:01.017 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:01.118 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:01.155 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:01.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:01.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:01.261 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:01.314 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:01.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:01.317 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:01.418 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:01.443 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:34:01.451 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:01.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:01.455 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:34:01.556 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:01.599 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:01.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:01.605 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:01.705 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:01.739 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:01.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:01.745 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:01.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:01 smithi016 bash[75004]: cluster 2023-09-23T19:34:00.011159+0000 mgr.y (mgr.25325) 792 : cluster 0 pgmap v696: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2023-09-23T19:34:01.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:01 smithi016 bash[77995]: cluster 2023-09-23T19:34:00.011159+0000 mgr.y (mgr.25325) 792 : cluster 0 pgmap v696: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2023-09-23T19:34:01.846 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:01.886 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:01.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:01.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:01.996 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:02.004 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:01 smithi092 bash[58855]: cluster 2023-09-23T19:34:00.011159+0000 mgr.y (mgr.25325) 792 : cluster 0 pgmap v696: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2023-09-23T19:34:02.032 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:02.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:02.038 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:34:02.138 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:02.172 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:02.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:02.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:34:02.275 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:02.302 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:02.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:02.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:02.405 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:02.438 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:02.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:02.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:02.540 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:02.565 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:02.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:02.569 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:02.620 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:34:02.670 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:02.704 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:02.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:02.708 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:34:02.808 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:02.843 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:02.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:02.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:02.949 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:02.982 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:02.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:02.985 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:03.085 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:03.117 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:03.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:03.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:03.224 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:03.246 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:03.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:03.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:34:03.349 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:03.374 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:03.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:03.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:03.481 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:03.520 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:03.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:03.525 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:03.626 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:03.655 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:03.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:03.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:03.762 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:03.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:03 smithi016 bash[75004]: cluster 2023-09-23T19:34:02.011844+0000 mgr.y (mgr.25325) 793 : cluster 0 pgmap v697: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 773 op/s 2023-09-23T19:34:03.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:03 smithi016 bash[77995]: cluster 2023-09-23T19:34:02.011844+0000 mgr.y (mgr.25325) 793 : cluster 0 pgmap v697: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 773 op/s 2023-09-23T19:34:03.794 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:03.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:03.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:03.897 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:03.905 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:03 smithi092 bash[58855]: cluster 2023-09-23T19:34:02.011844+0000 mgr.y (mgr.25325) 793 : cluster 0 pgmap v697: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 773 op/s 2023-09-23T19:34:03.924 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:03.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:03.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:34:04.031 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:04.073 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:04.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:04.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:04.179 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:04.219 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:04.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:04.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:04.324 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:04.349 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:04.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:04.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:04.382 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T19:34:04.382 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T19:34:04.383 INFO:tasks.thrashosds.thrasher:Growing pool device_health_metrics 2023-09-23T19:34:04.383 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T19:34:04.455 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:04.480 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:04.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:04.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:04.583 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:04.604 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-09-23T19:34:04.604 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:34:04.606 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:04.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:04.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:34:04.712 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:04.742 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:04.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:04.747 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:34:04.848 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:04.885 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:04.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:04.889 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:04.990 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:05.023 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:05.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:05.029 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:05.129 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:05.153 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:05.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:05.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:34:05.262 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:05.315 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:05.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:05.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:05.420 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:05.451 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:05.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:05.456 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:05.557 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:05.588 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:05.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:05.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:05.691 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:05.717 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:05.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:05.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:34:05.824 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:05.861 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:05.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:05.868 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:05.969 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:05.979 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:05 smithi016 bash[75004]: cluster 2023-09-23T19:34:04.012538+0000 mgr.y (mgr.25325) 794 : cluster 0 pgmap v698: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2023-09-23T19:34:05.980 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:34:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:34:05] "GET /metrics HTTP/1.1" 200 36239 "" "Prometheus/2.33.4" 2023-09-23T19:34:05.980 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:05 smithi016 bash[77995]: cluster 2023-09-23T19:34:04.012538+0000 mgr.y (mgr.25325) 794 : cluster 0 pgmap v698: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2023-09-23T19:34:06.001 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:06.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:06.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:06.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:05 smithi092 bash[58855]: cluster 2023-09-23T19:34:04.012538+0000 mgr.y (mgr.25325) 794 : cluster 0 pgmap v698: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2023-09-23T19:34:06.105 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:06.136 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:06.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:06.141 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:06.146 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T19:34:06.243 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:06.283 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:06.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:06.291 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:06.392 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:06.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:06.421 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:06.421 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:06.522 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:06.553 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:06.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:06.560 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:34:06.625 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:34:06.661 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:06.695 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:06.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:06.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:06.801 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:06.850 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:06.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:06.853 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:34:06.954 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:06.989 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:06.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:06.996 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:07.097 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:07.133 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:07.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:07.137 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:07.237 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:07.331 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:07.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:07.334 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:07.435 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:07.484 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:07.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:07.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:07.588 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:07.621 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:07.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:07.625 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:07.725 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:07.755 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:07.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:07.761 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:34:07.861 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:07.888 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:07.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:07.892 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:07.993 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:08.002 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:07 smithi092 bash[58855]: cluster 2023-09-23T19:34:06.013210+0000 mgr.y (mgr.25325) 795 : cluster 0 pgmap v699: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 753 op/s 2023-09-23T19:34:08.017 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:08.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:08.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:08.040 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:07 smithi016 bash[77995]: cluster 2023-09-23T19:34:06.013210+0000 mgr.y (mgr.25325) 795 : cluster 0 pgmap v699: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 753 op/s 2023-09-23T19:34:08.040 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:07 smithi016 bash[75004]: cluster 2023-09-23T19:34:06.013210+0000 mgr.y (mgr.25325) 795 : cluster 0 pgmap v699: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 753 op/s 2023-09-23T19:34:08.124 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:08.156 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:08.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:08.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:34:08.261 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:08.300 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:08.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:08.304 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:34:08.405 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:08.434 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:08.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:08.437 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:08.537 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:08.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:08.571 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:08.571 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:08.672 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:08.703 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:08.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:08.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:08.809 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:08.841 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:08.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:08.845 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:08.946 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:08.979 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:08.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:08.982 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:09.083 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:09.111 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:09.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:09.117 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:09.217 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:09.249 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:09.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:09.252 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:09.353 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:09.379 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:09.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:09.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:09.482 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:09.505 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:09.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:09.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:09.608 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:09.641 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:09.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:09.644 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:09.692 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is set 2023-09-23T19:34:09.745 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:09.754 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:09 smithi092 bash[58855]: cluster 2023-09-23T19:34:08.013705+0000 mgr.y (mgr.25325) 796 : cluster 0 pgmap v700: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 713 op/s 2023-09-23T19:34:09.754 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:09 smithi092 bash[58855]: audit 2023-09-23T19:34:09.017968+0000 mon.a (mon.0) 983 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:34:09.754 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:09 smithi092 bash[58855]: audit 2023-09-23T19:34:09.020899+0000 mon.b (mon.1) 40 : audit 1 from='client.? 172.21.15.16:0/4092597177' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:34:09.787 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:09.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:09.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:09.891 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:09.926 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:09.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:09.930 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:09 smithi016 bash[75004]: cluster 2023-09-23T19:34:08.013705+0000 mgr.y (mgr.25325) 796 : cluster 0 pgmap v700: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 713 op/s 2023-09-23T19:34:09.930 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:09 smithi016 bash[75004]: audit 2023-09-23T19:34:09.017968+0000 mon.a (mon.0) 983 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:34:09.930 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:09 smithi016 bash[75004]: audit 2023-09-23T19:34:09.020899+0000 mon.b (mon.1) 40 : audit 1 from='client.? 172.21.15.16:0/4092597177' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:34:09.930 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:09 smithi016 bash[77995]: cluster 2023-09-23T19:34:08.013705+0000 mgr.y (mgr.25325) 796 : cluster 0 pgmap v700: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 713 op/s 2023-09-23T19:34:09.930 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:09 smithi016 bash[77995]: audit 2023-09-23T19:34:09.017968+0000 mon.a (mon.0) 983 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:34:09.930 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:09 smithi016 bash[77995]: audit 2023-09-23T19:34:09.020899+0000 mon.b (mon.1) 40 : audit 1 from='client.? 172.21.15.16:0/4092597177' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:34:09.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:10.031 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:10.059 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:10.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:10.065 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:10.165 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:10.201 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:10.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:10.204 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:10.305 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:10.338 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:10.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:10.341 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:34:10.441 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:10.490 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:10.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:10.496 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:10.596 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:10.713 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:10.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:10.720 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:10.820 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:10.846 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:10.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:10.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:10.948 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:10.958 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:10 smithi016 bash[75004]: cluster 2023-09-23T19:34:09.664508+0000 mon.a (mon.0) 984 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:34:10.958 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:10 smithi016 bash[75004]: audit 2023-09-23T19:34:09.678217+0000 mon.a (mon.0) 985 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:34:10.958 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:10 smithi016 bash[75004]: cluster 2023-09-23T19:34:09.678437+0000 mon.a (mon.0) 986 : cluster 0 osdmap e131: 8 total, 8 up, 8 in 2023-09-23T19:34:10.958 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:10 smithi016 bash[75004]: cluster 2023-09-23T19:34:09.679465+0000 osd.4 (osd.4) 59 : cluster 3 failed to encode map e131 with expected crc 2023-09-23T19:34:10.958 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:10 smithi016 bash[75004]: cluster 2023-09-23T19:34:09.681613+0000 osd.5 (osd.5) 61 : cluster 3 failed to encode map e131 with expected crc 2023-09-23T19:34:10.958 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:10 smithi016 bash[75004]: cluster 2023-09-23T19:34:09.694123+0000 osd.6 (osd.6) 51 : cluster 3 failed to encode map e131 with expected crc 2023-09-23T19:34:10.958 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:10 smithi016 bash[75004]: cluster 2023-09-23T19:34:09.701201+0000 osd.7 (osd.7) 50 : cluster 3 failed to encode map e131 with expected crc 2023-09-23T19:34:10.959 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:10 smithi016 bash[77995]: cluster 2023-09-23T19:34:09.664508+0000 mon.a (mon.0) 984 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:34:10.959 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:10 smithi016 bash[77995]: audit 2023-09-23T19:34:09.678217+0000 mon.a (mon.0) 985 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:34:10.959 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:10 smithi016 bash[77995]: cluster 2023-09-23T19:34:09.678437+0000 mon.a (mon.0) 986 : cluster 0 osdmap e131: 8 total, 8 up, 8 in 2023-09-23T19:34:10.959 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:10 smithi016 bash[77995]: cluster 2023-09-23T19:34:09.679465+0000 osd.4 (osd.4) 59 : cluster 3 failed to encode map e131 with expected crc 2023-09-23T19:34:10.959 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:10 smithi016 bash[77995]: cluster 2023-09-23T19:34:09.681613+0000 osd.5 (osd.5) 61 : cluster 3 failed to encode map e131 with expected crc 2023-09-23T19:34:10.959 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:10 smithi016 bash[77995]: cluster 2023-09-23T19:34:09.694123+0000 osd.6 (osd.6) 51 : cluster 3 failed to encode map e131 with expected crc 2023-09-23T19:34:10.959 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:10 smithi016 bash[77995]: cluster 2023-09-23T19:34:09.701201+0000 osd.7 (osd.7) 50 : cluster 3 failed to encode map e131 with expected crc 2023-09-23T19:34:10.975 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:10.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:10.982 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:34:11.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:10 smithi092 bash[58855]: cluster 2023-09-23T19:34:09.664508+0000 mon.a (mon.0) 984 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:34:11.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:10 smithi092 bash[58855]: audit 2023-09-23T19:34:09.678217+0000 mon.a (mon.0) 985 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:34:11.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:10 smithi092 bash[58855]: cluster 2023-09-23T19:34:09.678437+0000 mon.a (mon.0) 986 : cluster 0 osdmap e131: 8 total, 8 up, 8 in 2023-09-23T19:34:11.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:10 smithi092 bash[58855]: cluster 2023-09-23T19:34:09.679465+0000 osd.4 (osd.4) 59 : cluster 3 failed to encode map e131 with expected crc 2023-09-23T19:34:11.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:10 smithi092 bash[58855]: cluster 2023-09-23T19:34:09.681613+0000 osd.5 (osd.5) 61 : cluster 3 failed to encode map e131 with expected crc 2023-09-23T19:34:11.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:10 smithi092 bash[58855]: cluster 2023-09-23T19:34:09.694123+0000 osd.6 (osd.6) 51 : cluster 3 failed to encode map e131 with expected crc 2023-09-23T19:34:11.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:10 smithi092 bash[58855]: cluster 2023-09-23T19:34:09.701201+0000 osd.7 (osd.7) 50 : cluster 3 failed to encode map e131 with expected crc 2023-09-23T19:34:11.082 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:11.111 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:11.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:11.114 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:34:11.215 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:11.240 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:11.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:11.244 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:11.344 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:11.379 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:11.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:11.384 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:34:11.485 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:11.515 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:11.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:11.517 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:34:11.618 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:11.654 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:11.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:11.657 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:11.695 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:11 smithi016 bash[75004]: cluster 2023-09-23T19:34:10.014211+0000 mgr.y (mgr.25325) 797 : cluster 0 pgmap v702: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2023-09-23T19:34:11.758 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:11.767 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:11 smithi092 bash[58855]: cluster 2023-09-23T19:34:10.014211+0000 mgr.y (mgr.25325) 797 : cluster 0 pgmap v702: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2023-09-23T19:34:11.788 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:11.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:11.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:11.892 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:11.915 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:11.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:11.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:34:11.923 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:34:11.949 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:11 smithi016 bash[77995]: cluster 2023-09-23T19:34:10.014211+0000 mgr.y (mgr.25325) 797 : cluster 0 pgmap v702: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2023-09-23T19:34:12.014 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:34:12.022 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:12.028 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:34:12.053 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:12.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:12.060 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:34:12.095 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:34:12.128 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:34:12.161 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:12.177 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:34:12.187 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:12.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:12.189 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:12.223 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:34:12.263 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:34:12.290 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:12.299 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:34:12.315 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:12.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:12.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:12.338 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:34:12.347 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:34:12.347 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:34:12.347 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:34:12.348 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:34:12.348 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:34:12.348 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:34:12.348 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:34:12.348 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:34:12.420 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:12.451 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:12.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:12.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:12.560 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:12.584 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:12.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:12.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:12.687 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:12.709 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:12 smithi092 bash[58855]: audit 2023-09-23T19:34:11.961555+0000 mon.a (mon.0) 987 : audit 0 from='client.? 172.21.15.16:0/1380943360' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:34:12.720 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:12.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:12.725 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:12.825 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:12.843 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:12.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:12.845 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:12.946 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:12.997 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:13.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:13.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:13.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:12 smithi016 bash[75004]: audit 2023-09-23T19:34:11.961555+0000 mon.a (mon.0) 987 : audit 0 from='client.? 172.21.15.16:0/1380943360' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:34:13.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:12 smithi016 bash[77995]: audit 2023-09-23T19:34:11.961555+0000 mon.a (mon.0) 987 : audit 0 from='client.? 172.21.15.16:0/1380943360' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:34:13.104 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:13.130 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:13.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:13.135 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:13.235 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:13.260 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:13.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:13.262 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:34:13.362 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:13.400 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:13.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:13.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:13.507 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:13.528 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T19:34:13.529 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:13.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:13.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:13.632 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:13.662 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:13.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:13.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:34:13.770 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:13.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:13 smithi092 bash[58855]: cluster 2023-09-23T19:34:12.014894+0000 mgr.y (mgr.25325) 798 : cluster 0 pgmap v703: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2023-09-23T19:34:13.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:13 smithi092 bash[58855]: audit 2023-09-23T19:34:13.650949+0000 mon.a (mon.0) 988 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:34:13.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:13 smithi092 bash[58855]: audit 2023-09-23T19:34:13.652449+0000 mon.a (mon.0) 989 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:34:13.793 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:13.796 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:13 smithi016 bash[75004]: cluster 2023-09-23T19:34:12.014894+0000 mgr.y (mgr.25325) 798 : cluster 0 pgmap v703: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2023-09-23T19:34:13.796 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:13 smithi016 bash[75004]: audit 2023-09-23T19:34:13.650949+0000 mon.a (mon.0) 988 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:34:13.796 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:13 smithi016 bash[75004]: audit 2023-09-23T19:34:13.652449+0000 mon.a (mon.0) 989 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:34:13.796 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:13 smithi016 bash[77995]: cluster 2023-09-23T19:34:12.014894+0000 mgr.y (mgr.25325) 798 : cluster 0 pgmap v703: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2023-09-23T19:34:13.797 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:13 smithi016 bash[77995]: audit 2023-09-23T19:34:13.650949+0000 mon.a (mon.0) 988 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:34:13.797 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:13 smithi016 bash[77995]: audit 2023-09-23T19:34:13.652449+0000 mon.a (mon.0) 989 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:34:13.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:13.797 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:13.898 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:13.927 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:13.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:13.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:14.035 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:14.071 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:14.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:14.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:14.177 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:14.202 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:14.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:14.205 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:14.306 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:14.356 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:14.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:14.368 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:34:14.468 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:14.502 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:14.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:14.505 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:14.606 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:14.631 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:14.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:14.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:14.738 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:14.768 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:14.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:14.775 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:14.876 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:14.897 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:14.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:14.899 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:14.999 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:15.034 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:15.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:15.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:15.138 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:15.169 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:15.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:15.172 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:34:15.273 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:15.302 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:15.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:15.304 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:15.405 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:15.440 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:15.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:15.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:15.544 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:15.574 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:15.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:15.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:15.678 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:15.714 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:15.717 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:15 smithi016 bash[75004]: cluster 2023-09-23T19:34:14.015360+0000 mgr.y (mgr.25325) 799 : cluster 0 pgmap v704: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 704 op/s 2023-09-23T19:34:15.717 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:34:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:34:15] "GET /metrics HTTP/1.1" 200 36239 "" "Prometheus/2.33.4" 2023-09-23T19:34:15.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:15.718 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:15.819 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:15.828 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:15 smithi092 bash[58855]: cluster 2023-09-23T19:34:14.015360+0000 mgr.y (mgr.25325) 799 : cluster 0 pgmap v704: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 704 op/s 2023-09-23T19:34:15.865 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:15.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:15.871 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:15.972 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:15.981 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:15 smithi016 bash[77995]: cluster 2023-09-23T19:34:14.015360+0000 mgr.y (mgr.25325) 799 : cluster 0 pgmap v704: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 704 op/s 2023-09-23T19:34:15.998 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:16.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:16.001 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:16.102 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:16.146 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:16.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:16.150 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:16.251 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:16.277 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:16.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:16.281 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:16.382 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:16.409 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:16.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:16.412 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:34:16.513 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:16.559 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:16.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:16.565 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:16.666 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:16.695 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:16.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:16.701 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:16.801 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:16.831 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:16.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:16.836 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:16.937 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:16.958 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:16.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:16.965 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:34:17.065 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:17.096 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:17.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:17.101 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:34:17.162 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:34:17.203 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:17.263 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:17.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:17.268 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:17.368 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:17.395 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:17.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:17.400 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:17.501 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:17.552 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:17.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:17.556 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:17.656 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:17.710 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:17.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:17.717 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:34:17.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:17 smithi016 bash[77995]: cluster 2023-09-23T19:34:16.016377+0000 mgr.y (mgr.25325) 800 : cluster 0 pgmap v705: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 695 op/s 2023-09-23T19:34:17.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:17 smithi016 bash[77995]: audit 2023-09-23T19:34:17.181212+0000 mon.a (mon.0) 990 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:34:17.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:17 smithi016 bash[75004]: cluster 2023-09-23T19:34:16.016377+0000 mgr.y (mgr.25325) 800 : cluster 0 pgmap v705: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 695 op/s 2023-09-23T19:34:17.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:17 smithi016 bash[75004]: audit 2023-09-23T19:34:17.181212+0000 mon.a (mon.0) 990 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:34:17.818 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:17.857 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:17.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:17.861 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:17.962 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:17.996 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:18.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:18.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:34:18.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:17 smithi092 bash[58855]: cluster 2023-09-23T19:34:16.016377+0000 mgr.y (mgr.25325) 800 : cluster 0 pgmap v705: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 695 op/s 2023-09-23T19:34:18.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:17 smithi092 bash[58855]: audit 2023-09-23T19:34:17.181212+0000 mon.a (mon.0) 990 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:34:18.102 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:18.125 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:18.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:18.129 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:34:18.229 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:18.269 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:18.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:18.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:18.375 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:18.398 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:18.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:18.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:18.504 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:18.529 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:18.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:18.533 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:18.634 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:18.665 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:18.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:18.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:18.771 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:18.803 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:18.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:18.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:34:18.909 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:18.949 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:18.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:18.953 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:19.037 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-09-23T19:34:19.037 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [], 2023-09-23T19:34:19.038 INFO:teuthology.orchestra.run.smithi016.stdout: "complaint_time": 30, 2023-09-23T19:34:19.038 INFO:teuthology.orchestra.run.smithi016.stdout: "num_blocked_ops": 0 2023-09-23T19:34:19.038 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-09-23T19:34:19.054 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:19.091 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:19.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:19.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:19.195 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:19.220 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:19.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:19.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:19.325 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:19.359 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:19.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:19.363 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:19.440 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-09-23T19:34:19.445 INFO:teuthology.orchestra.run.smithi016.stderr:dumped all 2023-09-23T19:34:19.464 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:19.525 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:19.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:19.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:34:19.628 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:19.656 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:19.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:19.658 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:19.759 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:19.795 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:19.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:19.799 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:19.900 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:19.909 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:19 smithi016 bash[75004]: cluster 2023-09-23T19:34:18.017060+0000 mgr.y (mgr.25325) 801 : cluster 0 pgmap v706: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2023-09-23T19:34:19.910 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:19 smithi016 bash[77995]: cluster 2023-09-23T19:34:18.017060+0000 mgr.y (mgr.25325) 801 : cluster 0 pgmap v706: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2023-09-23T19:34:19.929 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:19.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:19.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:34:20.032 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:20.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:19 smithi092 bash[58855]: cluster 2023-09-23T19:34:18.017060+0000 mgr.y (mgr.25325) 801 : cluster 0 pgmap v706: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2023-09-23T19:34:20.063 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:20.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:20.066 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:20.166 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:20.197 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:20.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:20.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:34:20.301 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:20.329 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:20.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:20.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:20.432 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:20.473 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:20.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:20.479 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:34:20.579 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:20.605 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:20.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:20.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:20.708 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:20.737 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:20 smithi016 bash[75004]: audit 2023-09-23T19:34:19.418075+0000 mgr.y (mgr.25325) 802 : audit 0 from='client.45788 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:34:20.742 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:20.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:20.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:20.829 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T19:34:20.847 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:20.908 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:20.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:20.912 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:34:21.013 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:21.020 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:20 smithi092 bash[58855]: audit 2023-09-23T19:34:19.418075+0000 mgr.y (mgr.25325) 802 : audit 0 from='client.45788 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:34:21.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:20 smithi016 bash[77995]: audit 2023-09-23T19:34:19.418075+0000 mgr.y (mgr.25325) 802 : audit 0 from='client.45788 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:34:21.039 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:21.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:21.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:21.142 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:21.171 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:21.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:21.180 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:21.280 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:21.307 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:21.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:21.313 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:21.385 INFO:teuthology.orchestra.run.smithi016.stdout:{"pg_ready":true,"pg_map":{"version":706,"stamp":"2023-09-23T19:34:18.016577+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3629520824,"num_objects":55398,"num_object_clones":0,"num_object_copies":166194,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":55398,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":161647,"num_write_kb":6938703,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":143981,"ondisk_log_size":143981,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":14282728,"kb_used_data":10722408,"kb_used_omap":20,"kb_used_meta":3560107,"kb_avail":735678488,"statfs":{"total":767960285184,"available":753334771712,"internally_reserved":0,"allocated":10979745792,"data_stored":10975711590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":3645550222},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,5,10,7,3,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-446824448,"num_objects":-6818,"num_object_clones":0,"num_object_copies":-20454,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-6818,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6818,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003168"},"pg_stats":[{"pgid":"3.1a","version":"59'1","reported_seq":113,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.570368+0000","last_change":"2023-09-23T19:27:24.274696+0000","last_active":"2023-09-23T19:33:49.570368+0000","last_peered":"2023-09-23T19:33:49.570368+0000","last_clean":"2023-09-23T19:33:49.570368+0000","last_became_active":"2023-09-23T19:27:24.273561+0000","last_became_peered":"2023-09-23T19:27:24.273561+0000","last_unstale":"2023-09-23T19:33:49.570368+0000","last_undegraded":"2023-09-23T19:33:49.570368+0000","last_fullsized":"2023-09-23T19:33:49.570368+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":85,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.560173+0000","last_change":"2023-09-23T19:32:49.731128+0000","last_active":"2023-09-23T19:33:49.560173+0000","last_peered":"2023-09-23T19:33:49.560173+0000","last_clean":"2023-09-23T19:33:49.560173+0000","last_became_active":"2023-09-23T19:32:49.730757+0000","last_became_peered":"2023-09-23T19:32:49.730757+0000","last_unstale":"2023-09-23T19:33:49.560173+0000","last_undegraded":"2023-09-23T19:33:49.560173+0000","last_fullsized":"2023-09-23T19:33:49.560173+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:40.183605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":73,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.559907+0000","last_change":"2023-09-23T19:28:07.890462+0000","last_active":"2023-09-23T19:33:49.559907+0000","last_peered":"2023-09-23T19:33:49.559907+0000","last_clean":"2023-09-23T19:33:49.559907+0000","last_became_active":"2023-09-23T19:28:07.890084+0000","last_became_peered":"2023-09-23T19:28:07.890084+0000","last_unstale":"2023-09-23T19:33:49.559907+0000","last_undegraded":"2023-09-23T19:33:49.559907+0000","last_fullsized":"2023-09-23T19:33:49.559907+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:50.458218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":90,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-09-23T19:34:09.694367+0000","last_change":"2023-09-23T19:28:07.884498+0000","last_active":"2023-09-23T19:34:09.694367+0000","last_peered":"2023-09-23T19:34:09.694367+0000","last_clean":"2023-09-23T19:34:09.694367+0000","last_became_active":"2023-09-23T19:28:07.884102+0000","last_became_peered":"2023-09-23T19:28:07.884102+0000","last_unstale":"2023-09-23T19:34:09.694367+0000","last_undegraded":"2023-09-23T19:34:09.694367+0000","last_fullsized":"2023-09-23T19:34:09.694367+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:05:26.348041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":101,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.580748+0000","last_change":"2023-09-23T19:32:49.729961+0000","last_active":"2023-09-23T19:33:49.580748+0000","last_peered":"2023-09-23T19:33:49.580748+0000","last_clean":"2023-09-23T19:33:49.580748+0000","last_became_active":"2023-09-23T19:32:49.729251+0000","last_became_peered":"2023-09-23T19:32:49.729251+0000","last_unstale":"2023-09-23T19:33:49.580748+0000","last_undegraded":"2023-09-23T19:33:49.580748+0000","last_fullsized":"2023-09-23T19:33:49.580748+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:20:56.595732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":96,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.599663+0000","last_change":"2023-09-23T19:22:42.150582+0000","last_active":"2023-09-23T19:33:49.599663+0000","last_peered":"2023-09-23T19:33:49.599663+0000","last_clean":"2023-09-23T19:33:49.599663+0000","last_became_active":"2023-09-23T19:22:42.150229+0000","last_became_peered":"2023-09-23T19:22:42.150229+0000","last_unstale":"2023-09-23T19:33:49.599663+0000","last_undegraded":"2023-09-23T19:33:49.599663+0000","last_fullsized":"2023-09-23T19:33:49.599663+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":88,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.581018+0000","last_change":"2023-09-23T19:28:07.887141+0000","last_active":"2023-09-23T19:33:49.581018+0000","last_peered":"2023-09-23T19:33:49.581018+0000","last_clean":"2023-09-23T19:33:49.581018+0000","last_became_active":"2023-09-23T19:28:07.886494+0000","last_became_peered":"2023-09-23T19:28:07.886494+0000","last_unstale":"2023-09-23T19:33:49.581018+0000","last_undegraded":"2023-09-23T19:33:49.581018+0000","last_fullsized":"2023-09-23T19:33:49.581018+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:24:07.615051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":85,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.561444+0000","last_change":"2023-09-23T19:32:49.729765+0000","last_active":"2023-09-23T19:33:49.561444+0000","last_peered":"2023-09-23T19:33:49.561444+0000","last_clean":"2023-09-23T19:33:49.561444+0000","last_became_active":"2023-09-23T19:32:49.729007+0000","last_became_peered":"2023-09-23T19:32:49.729007+0000","last_unstale":"2023-09-23T19:33:49.561444+0000","last_undegraded":"2023-09-23T19:33:49.561444+0000","last_fullsized":"2023-09-23T19:33:49.561444+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:59:31.183753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":98,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.580147+0000","last_change":"2023-09-23T19:22:42.151617+0000","last_active":"2023-09-23T19:33:49.580147+0000","last_peered":"2023-09-23T19:33:49.580147+0000","last_clean":"2023-09-23T19:33:49.580147+0000","last_became_active":"2023-09-23T19:22:42.151217+0000","last_became_peered":"2023-09-23T19:22:42.151217+0000","last_unstale":"2023-09-23T19:33:49.580147+0000","last_undegraded":"2023-09-23T19:33:49.580147+0000","last_fullsized":"2023-09-23T19:33:49.580147+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":71,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.577420+0000","last_change":"2023-09-23T19:32:49.736864+0000","last_active":"2023-09-23T19:33:49.577420+0000","last_peered":"2023-09-23T19:33:49.577420+0000","last_clean":"2023-09-23T19:33:49.577420+0000","last_became_active":"2023-09-23T19:32:49.736523+0000","last_became_peered":"2023-09-23T19:32:49.736523+0000","last_unstale":"2023-09-23T19:33:49.577420+0000","last_undegraded":"2023-09-23T19:33:49.577420+0000","last_fullsized":"2023-09-23T19:33:49.577420+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":100,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.578114+0000","last_change":"2023-09-23T19:28:07.892835+0000","last_active":"2023-09-23T19:33:49.578114+0000","last_peered":"2023-09-23T19:33:49.578114+0000","last_clean":"2023-09-23T19:33:49.578114+0000","last_became_active":"2023-09-23T19:28:07.892338+0000","last_became_peered":"2023-09-23T19:28:07.892338+0000","last_unstale":"2023-09-23T19:33:49.578114+0000","last_undegraded":"2023-09-23T19:33:49.578114+0000","last_fullsized":"2023-09-23T19:33:49.578114+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":73,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.560330+0000","last_change":"2023-09-23T19:27:24.268005+0000","last_active":"2023-09-23T19:33:49.560330+0000","last_peered":"2023-09-23T19:33:49.560330+0000","last_clean":"2023-09-23T19:33:49.560330+0000","last_became_active":"2023-09-23T19:27:24.267513+0000","last_became_peered":"2023-09-23T19:27:24.267513+0000","last_unstale":"2023-09-23T19:33:49.560330+0000","last_undegraded":"2023-09-23T19:33:49.560330+0000","last_fullsized":"2023-09-23T19:33:49.560330+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.961399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'23","reported_seq":105,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.570924+0000","last_change":"2023-09-23T19:05:52.107727+0000","last_active":"2023-09-23T19:33:49.570924+0000","last_peered":"2023-09-23T19:33:49.570924+0000","last_clean":"2023-09-23T19:33:49.570924+0000","last_became_active":"2023-09-23T19:05:52.107378+0000","last_became_peered":"2023-09-23T19:05:52.107378+0000","last_unstale":"2023-09-23T19:33:49.570924+0000","last_undegraded":"2023-09-23T19:33:49.570924+0000","last_fullsized":"2023-09-23T19:33:49.570924+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":96,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.570664+0000","last_change":"2023-09-23T19:22:42.148497+0000","last_active":"2023-09-23T19:33:49.570664+0000","last_peered":"2023-09-23T19:33:49.570664+0000","last_clean":"2023-09-23T19:33:49.570664+0000","last_became_active":"2023-09-23T19:22:42.148147+0000","last_became_peered":"2023-09-23T19:22:42.148147+0000","last_unstale":"2023-09-23T19:33:49.570664+0000","last_undegraded":"2023-09-23T19:33:49.570664+0000","last_fullsized":"2023-09-23T19:33:49.570664+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":99,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.578917+0000","last_change":"2023-09-23T19:27:24.273667+0000","last_active":"2023-09-23T19:33:49.578917+0000","last_peered":"2023-09-23T19:33:49.578917+0000","last_clean":"2023-09-23T19:33:49.578917+0000","last_became_active":"2023-09-23T19:27:24.273234+0000","last_became_peered":"2023-09-23T19:27:24.273234+0000","last_unstale":"2023-09-23T19:33:49.578917+0000","last_undegraded":"2023-09-23T19:33:49.578917+0000","last_fullsized":"2023-09-23T19:33:49.578917+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":85,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.568848+0000","last_change":"2023-09-23T19:32:49.730549+0000","last_active":"2023-09-23T19:33:49.568848+0000","last_peered":"2023-09-23T19:33:49.568848+0000","last_clean":"2023-09-23T19:33:49.568848+0000","last_became_active":"2023-09-23T19:32:49.730105+0000","last_became_peered":"2023-09-23T19:32:49.730105+0000","last_unstale":"2023-09-23T19:33:49.568848+0000","last_undegraded":"2023-09-23T19:33:49.568848+0000","last_fullsized":"2023-09-23T19:33:49.568848+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:49.751893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":88,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.580704+0000","last_change":"2023-09-23T19:28:07.884394+0000","last_active":"2023-09-23T19:33:49.580704+0000","last_peered":"2023-09-23T19:33:49.580704+0000","last_clean":"2023-09-23T19:33:49.580704+0000","last_became_active":"2023-09-23T19:28:07.883944+0000","last_became_peered":"2023-09-23T19:28:07.883944+0000","last_unstale":"2023-09-23T19:33:49.580704+0000","last_undegraded":"2023-09-23T19:33:49.580704+0000","last_fullsized":"2023-09-23T19:33:49.580704+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:08:17.608926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":97,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.580272+0000","last_change":"2023-09-23T19:28:07.891056+0000","last_active":"2023-09-23T19:33:49.580272+0000","last_peered":"2023-09-23T19:33:49.580272+0000","last_clean":"2023-09-23T19:33:49.580272+0000","last_became_active":"2023-09-23T19:28:07.890625+0000","last_became_peered":"2023-09-23T19:28:07.890625+0000","last_unstale":"2023-09-23T19:33:49.580272+0000","last_undegraded":"2023-09-23T19:33:49.580272+0000","last_fullsized":"2023-09-23T19:33:49.580272+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":82,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.577526+0000","last_change":"2023-09-23T19:05:52.114186+0000","last_active":"2023-09-23T19:33:49.577526+0000","last_peered":"2023-09-23T19:33:49.577526+0000","last_clean":"2023-09-23T19:33:49.577526+0000","last_became_active":"2023-09-23T19:05:52.113635+0000","last_became_peered":"2023-09-23T19:05:52.113635+0000","last_unstale":"2023-09-23T19:33:49.577526+0000","last_undegraded":"2023-09-23T19:33:49.577526+0000","last_fullsized":"2023-09-23T19:33:49.577526+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":88,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.581016+0000","last_change":"2023-09-23T19:28:07.884512+0000","last_active":"2023-09-23T19:33:49.581016+0000","last_peered":"2023-09-23T19:33:49.581016+0000","last_clean":"2023-09-23T19:33:49.581016+0000","last_became_active":"2023-09-23T19:28:07.884155+0000","last_became_peered":"2023-09-23T19:28:07.884155+0000","last_unstale":"2023-09-23T19:33:49.581016+0000","last_undegraded":"2023-09-23T19:33:49.581016+0000","last_fullsized":"2023-09-23T19:33:49.581016+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:10.654958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":105,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.599783+0000","last_change":"2023-09-23T19:18:09.010195+0000","last_active":"2023-09-23T19:33:49.599783+0000","last_peered":"2023-09-23T19:33:49.599783+0000","last_clean":"2023-09-23T19:33:49.599783+0000","last_became_active":"2023-09-23T19:18:09.009687+0000","last_became_peered":"2023-09-23T19:18:09.009687+0000","last_unstale":"2023-09-23T19:33:49.599783+0000","last_undegraded":"2023-09-23T19:33:49.599783+0000","last_fullsized":"2023-09-23T19:33:49.599783+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":93,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.581357+0000","last_change":"2023-09-23T19:28:07.885265+0000","last_active":"2023-09-23T19:33:49.581357+0000","last_peered":"2023-09-23T19:33:49.581357+0000","last_clean":"2023-09-23T19:33:49.581357+0000","last_became_active":"2023-09-23T19:28:07.884353+0000","last_became_peered":"2023-09-23T19:28:07.884353+0000","last_unstale":"2023-09-23T19:33:49.581357+0000","last_undegraded":"2023-09-23T19:33:49.581357+0000","last_fullsized":"2023-09-23T19:33:49.581357+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:56.325352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"131'10166","reported_seq":10188,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-09-23T19:34:17.300477+0000","last_change":"2023-09-23T19:31:12.572210+0000","last_active":"2023-09-23T19:34:17.300477+0000","last_peered":"2023-09-23T19:34:17.300477+0000","last_clean":"2023-09-23T19:34:17.300477+0000","last_became_active":"2023-09-23T19:31:12.571806+0000","last_became_peered":"2023-09-23T19:31:12.571806+0000","last_unstale":"2023-09-23T19:34:17.300477+0000","last_undegraded":"2023-09-23T19:34:17.300477+0000","last_fullsized":"2023-09-23T19:34:17.300477+0000","mapping_epoch":119,"log_start":"122'100","ondisk_log_start":"122'100","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_clean_scrub_stamp":"2023-09-23T19:31:11.546138+0000","objects_scrubbed":0,"log_size":10066,"log_dups_size":0,"ondisk_log_size":10066,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:41:17.174555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222035968,"num_objects":3388,"num_object_clones":0,"num_object_copies":10164,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3388,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10166,"num_write_kb":433728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":98,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.578680+0000","last_change":"2023-09-23T19:18:09.035116+0000","last_active":"2023-09-23T19:33:49.578680+0000","last_peered":"2023-09-23T19:33:49.578680+0000","last_clean":"2023-09-23T19:33:49.578680+0000","last_became_active":"2023-09-23T19:18:09.034416+0000","last_became_peered":"2023-09-23T19:18:09.034416+0000","last_unstale":"2023-09-23T19:33:49.578680+0000","last_undegraded":"2023-09-23T19:33:49.578680+0000","last_fullsized":"2023-09-23T19:33:49.578680+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":82,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.579720+0000","last_change":"2023-09-23T19:05:52.789323+0000","last_active":"2023-09-23T19:33:49.579720+0000","last_peered":"2023-09-23T19:33:49.579720+0000","last_clean":"2023-09-23T19:33:49.579720+0000","last_became_active":"2023-09-23T19:05:52.788820+0000","last_became_peered":"2023-09-23T19:05:52.788820+0000","last_unstale":"2023-09-23T19:33:49.579720+0000","last_undegraded":"2023-09-23T19:33:49.579720+0000","last_fullsized":"2023-09-23T19:33:49.579720+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"131'10192","reported_seq":10214,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-09-23T19:34:16.556107+0000","last_change":"2023-09-23T19:31:12.589012+0000","last_active":"2023-09-23T19:34:16.556107+0000","last_peered":"2023-09-23T19:34:16.556107+0000","last_clean":"2023-09-23T19:34:16.556107+0000","last_became_active":"2023-09-23T19:31:12.588473+0000","last_became_peered":"2023-09-23T19:31:12.588473+0000","last_unstale":"2023-09-23T19:34:16.556107+0000","last_undegraded":"2023-09-23T19:34:16.556107+0000","last_fullsized":"2023-09-23T19:34:16.556107+0000","mapping_epoch":119,"log_start":"123'2800","ondisk_log_start":"123'2800","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_clean_scrub_stamp":"2023-09-23T19:31:11.546138+0000","objects_scrubbed":0,"log_size":7392,"log_dups_size":0,"ondisk_log_size":7392,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:47:07.601624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":226885632,"num_objects":3462,"num_object_clones":0,"num_object_copies":10386,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3462,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10192,"num_write_kb":436928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":99,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.580368+0000","last_change":"2023-09-23T19:22:42.151134+0000","last_active":"2023-09-23T19:33:49.580368+0000","last_peered":"2023-09-23T19:33:49.580368+0000","last_clean":"2023-09-23T19:33:49.580368+0000","last_became_active":"2023-09-23T19:22:42.150553+0000","last_became_peered":"2023-09-23T19:22:42.150553+0000","last_unstale":"2023-09-23T19:33:49.580368+0000","last_undegraded":"2023-09-23T19:33:49.580368+0000","last_fullsized":"2023-09-23T19:33:49.580368+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":111,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.570112+0000","last_change":"2023-09-23T19:28:07.899262+0000","last_active":"2023-09-23T19:33:49.570112+0000","last_peered":"2023-09-23T19:33:49.570112+0000","last_clean":"2023-09-23T19:33:49.570112+0000","last_became_active":"2023-09-23T19:28:07.898848+0000","last_became_peered":"2023-09-23T19:28:07.898848+0000","last_unstale":"2023-09-23T19:33:49.570112+0000","last_undegraded":"2023-09-23T19:33:49.570112+0000","last_fullsized":"2023-09-23T19:33:49.570112+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"131'10044","reported_seq":10064,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-09-23T19:34:15.222401+0000","last_change":"2023-09-23T19:31:12.576224+0000","last_active":"2023-09-23T19:34:15.222401+0000","last_peered":"2023-09-23T19:34:15.222401+0000","last_clean":"2023-09-23T19:34:15.222401+0000","last_became_active":"2023-09-23T19:31:12.575807+0000","last_became_peered":"2023-09-23T19:31:12.575807+0000","last_unstale":"2023-09-23T19:34:15.222401+0000","last_undegraded":"2023-09-23T19:34:15.222401+0000","last_fullsized":"2023-09-23T19:34:15.222401+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_clean_scrub_stamp":"2023-09-23T19:31:11.546138+0000","objects_scrubbed":0,"log_size":10044,"log_dups_size":0,"ondisk_log_size":10044,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":234749952,"num_objects":3582,"num_object_clones":0,"num_object_copies":10746,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3582,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10044,"num_write_kb":436032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":83,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.577916+0000","last_change":"2023-09-23T19:05:52.114108+0000","last_active":"2023-09-23T19:33:49.577916+0000","last_peered":"2023-09-23T19:33:49.577916+0000","last_clean":"2023-09-23T19:33:49.577916+0000","last_became_active":"2023-09-23T19:05:52.113609+0000","last_became_peered":"2023-09-23T19:05:52.113609+0000","last_unstale":"2023-09-23T19:33:49.577916+0000","last_undegraded":"2023-09-23T19:33:49.577916+0000","last_fullsized":"2023-09-23T19:33:49.577916+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":96,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.599476+0000","last_change":"2023-09-23T19:22:42.156825+0000","last_active":"2023-09-23T19:33:49.599476+0000","last_peered":"2023-09-23T19:33:49.599476+0000","last_clean":"2023-09-23T19:33:49.599476+0000","last_became_active":"2023-09-23T19:22:42.156474+0000","last_became_peered":"2023-09-23T19:22:42.156474+0000","last_unstale":"2023-09-23T19:33:49.599476+0000","last_undegraded":"2023-09-23T19:33:49.599476+0000","last_fullsized":"2023-09-23T19:33:49.599476+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"131'9877","reported_seq":9897,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-09-23T19:34:15.287321+0000","last_change":"2023-09-23T19:31:12.589020+0000","last_active":"2023-09-23T19:34:15.287321+0000","last_peered":"2023-09-23T19:34:15.287321+0000","last_clean":"2023-09-23T19:34:15.287321+0000","last_became_active":"2023-09-23T19:31:12.588673+0000","last_became_peered":"2023-09-23T19:31:12.588673+0000","last_unstale":"2023-09-23T19:34:15.287321+0000","last_undegraded":"2023-09-23T19:34:15.287321+0000","last_fullsized":"2023-09-23T19:34:15.287321+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_clean_scrub_stamp":"2023-09-23T19:31:11.546138+0000","objects_scrubbed":0,"log_size":9877,"log_dups_size":0,"ondisk_log_size":9877,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":226033664,"num_objects":3449,"num_object_clones":0,"num_object_copies":10347,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3449,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9877,"num_write_kb":426432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":98,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.579950+0000","last_change":"2023-09-23T19:28:07.891536+0000","last_active":"2023-09-23T19:33:49.579950+0000","last_peered":"2023-09-23T19:33:49.579950+0000","last_clean":"2023-09-23T19:33:49.579950+0000","last_became_active":"2023-09-23T19:28:07.891153+0000","last_became_peered":"2023-09-23T19:28:07.891153+0000","last_unstale":"2023-09-23T19:33:49.579950+0000","last_undegraded":"2023-09-23T19:33:49.579950+0000","last_fullsized":"2023-09-23T19:33:49.579950+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":89,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:52.231222+0000","last_change":"2023-09-23T19:22:42.135572+0000","last_active":"2023-09-23T19:33:52.231222+0000","last_peered":"2023-09-23T19:33:52.231222+0000","last_clean":"2023-09-23T19:33:52.231222+0000","last_became_active":"2023-09-23T19:22:42.135237+0000","last_became_peered":"2023-09-23T19:22:42.135237+0000","last_unstale":"2023-09-23T19:33:52.231222+0000","last_undegraded":"2023-09-23T19:33:52.231222+0000","last_fullsized":"2023-09-23T19:33:52.231222+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:35.838718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"131'9909","reported_seq":9938,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-09-23T19:34:14.526105+0000","last_change":"2023-09-23T19:32:49.767399+0000","last_active":"2023-09-23T19:34:14.526105+0000","last_peered":"2023-09-23T19:34:14.526105+0000","last_clean":"2023-09-23T19:34:14.526105+0000","last_became_active":"2023-09-23T19:32:49.735026+0000","last_became_peered":"2023-09-23T19:32:49.735026+0000","last_unstale":"2023-09-23T19:34:14.526105+0000","last_undegraded":"2023-09-23T19:34:14.526105+0000","last_fullsized":"2023-09-23T19:34:14.526105+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_clean_scrub_stamp":"2023-09-23T19:31:11.546138+0000","objects_scrubbed":0,"log_size":9909,"log_dups_size":0,"ondisk_log_size":9909,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224067584,"num_objects":3419,"num_object_clones":0,"num_object_copies":10257,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3419,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9909,"num_write_kb":426496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":35,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.592973+0000","last_change":"2023-09-23T19:32:49.728589+0000","last_active":"2023-09-23T19:33:49.592973+0000","last_peered":"2023-09-23T19:33:49.592973+0000","last_clean":"2023-09-23T19:33:49.592973+0000","last_became_active":"2023-09-23T19:32:49.728037+0000","last_became_peered":"2023-09-23T19:32:49.728037+0000","last_unstale":"2023-09-23T19:33:49.592973+0000","last_undegraded":"2023-09-23T19:33:49.592973+0000","last_fullsized":"2023-09-23T19:33:49.592973+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:26:00.957969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":96,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.571000+0000","last_change":"2023-09-23T19:28:07.896803+0000","last_active":"2023-09-23T19:33:49.571000+0000","last_peered":"2023-09-23T19:33:49.571000+0000","last_clean":"2023-09-23T19:33:49.571000+0000","last_became_active":"2023-09-23T19:28:07.896480+0000","last_became_peered":"2023-09-23T19:28:07.896480+0000","last_unstale":"2023-09-23T19:33:49.571000+0000","last_undegraded":"2023-09-23T19:33:49.571000+0000","last_fullsized":"2023-09-23T19:33:49.571000+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"131'10095","reported_seq":10117,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-09-23T19:34:16.556699+0000","last_change":"2023-09-23T19:31:12.591859+0000","last_active":"2023-09-23T19:34:16.556699+0000","last_peered":"2023-09-23T19:34:16.556699+0000","last_clean":"2023-09-23T19:34:16.556699+0000","last_became_active":"2023-09-23T19:31:12.591505+0000","last_became_peered":"2023-09-23T19:31:12.591505+0000","last_unstale":"2023-09-23T19:34:16.556699+0000","last_undegraded":"2023-09-23T19:34:16.556699+0000","last_fullsized":"2023-09-23T19:34:16.556699+0000","mapping_epoch":119,"log_start":"123'2700","ondisk_log_start":"123'2700","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_clean_scrub_stamp":"2023-09-23T19:31:11.546138+0000","objects_scrubbed":0,"log_size":7395,"log_dups_size":0,"ondisk_log_size":7395,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:33:29.412181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":227737600,"num_objects":3475,"num_object_clones":0,"num_object_copies":10425,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3475,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10095,"num_write_kb":434240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":97,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.599579+0000","last_change":"2023-09-23T19:22:42.156582+0000","last_active":"2023-09-23T19:33:49.599579+0000","last_peered":"2023-09-23T19:33:49.599579+0000","last_clean":"2023-09-23T19:33:49.599579+0000","last_became_active":"2023-09-23T19:22:42.156251+0000","last_became_peered":"2023-09-23T19:22:42.156251+0000","last_unstale":"2023-09-23T19:33:49.599579+0000","last_undegraded":"2023-09-23T19:33:49.599579+0000","last_fullsized":"2023-09-23T19:33:49.599579+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":111,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.570940+0000","last_change":"2023-09-23T19:27:24.273971+0000","last_active":"2023-09-23T19:33:49.570940+0000","last_peered":"2023-09-23T19:33:49.570940+0000","last_clean":"2023-09-23T19:33:49.570940+0000","last_became_active":"2023-09-23T19:27:24.273491+0000","last_became_peered":"2023-09-23T19:27:24.273491+0000","last_unstale":"2023-09-23T19:33:49.570940+0000","last_undegraded":"2023-09-23T19:33:49.570940+0000","last_fullsized":"2023-09-23T19:33:49.570940+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"131'10070","reported_seq":10090,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-09-23T19:34:15.076243+0000","last_change":"2023-09-23T19:31:12.573917+0000","last_active":"2023-09-23T19:34:15.076243+0000","last_peered":"2023-09-23T19:34:15.076243+0000","last_clean":"2023-09-23T19:34:15.076243+0000","last_became_active":"2023-09-23T19:31:12.573499+0000","last_became_peered":"2023-09-23T19:31:12.573499+0000","last_unstale":"2023-09-23T19:34:15.076243+0000","last_undegraded":"2023-09-23T19:34:15.076243+0000","last_fullsized":"2023-09-23T19:34:15.076243+0000","mapping_epoch":119,"log_start":"122'2300","ondisk_log_start":"122'2300","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_clean_scrub_stamp":"2023-09-23T19:31:11.546138+0000","objects_scrubbed":0,"log_size":7770,"log_dups_size":0,"ondisk_log_size":7770,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":227278848,"num_objects":3468,"num_object_clones":0,"num_object_copies":10404,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3468,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10070,"num_write_kb":433216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":97,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.569908+0000","last_change":"2023-09-23T19:27:24.274698+0000","last_active":"2023-09-23T19:33:49.569908+0000","last_peered":"2023-09-23T19:33:49.569908+0000","last_clean":"2023-09-23T19:33:49.569908+0000","last_became_active":"2023-09-23T19:27:24.273264+0000","last_became_peered":"2023-09-23T19:27:24.273264+0000","last_unstale":"2023-09-23T19:33:49.569908+0000","last_undegraded":"2023-09-23T19:33:49.569908+0000","last_fullsized":"2023-09-23T19:33:49.569908+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":96,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.571419+0000","last_change":"2023-09-23T19:22:42.149670+0000","last_active":"2023-09-23T19:33:49.571419+0000","last_peered":"2023-09-23T19:33:49.571419+0000","last_clean":"2023-09-23T19:33:49.571419+0000","last_became_active":"2023-09-23T19:22:42.149022+0000","last_became_peered":"2023-09-23T19:22:42.149022+0000","last_unstale":"2023-09-23T19:33:49.571419+0000","last_undegraded":"2023-09-23T19:33:49.571419+0000","last_fullsized":"2023-09-23T19:33:49.571419+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"131'10046","reported_seq":10074,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-09-23T19:34:16.535412+0000","last_change":"2023-09-23T19:32:49.742544+0000","last_active":"2023-09-23T19:34:16.535412+0000","last_peered":"2023-09-23T19:34:16.535412+0000","last_clean":"2023-09-23T19:34:16.535412+0000","last_became_active":"2023-09-23T19:32:49.728515+0000","last_became_peered":"2023-09-23T19:32:49.728515+0000","last_unstale":"2023-09-23T19:34:16.535412+0000","last_undegraded":"2023-09-23T19:34:16.535412+0000","last_fullsized":"2023-09-23T19:34:16.535412+0000","mapping_epoch":126,"log_start":"123'2700","ondisk_log_start":"123'2700","created":119,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_clean_scrub_stamp":"2023-09-23T19:31:11.546138+0000","objects_scrubbed":0,"log_size":7346,"log_dups_size":0,"ondisk_log_size":7346,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:01.357892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217055232,"num_objects":3312,"num_object_clones":0,"num_object_copies":9936,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3312,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10046,"num_write_kb":427456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":86,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.561327+0000","last_change":"2023-09-23T19:32:49.729244+0000","last_active":"2023-09-23T19:33:49.561327+0000","last_peered":"2023-09-23T19:33:49.561327+0000","last_clean":"2023-09-23T19:33:49.561327+0000","last_became_active":"2023-09-23T19:32:49.728810+0000","last_became_peered":"2023-09-23T19:32:49.728810+0000","last_unstale":"2023-09-23T19:33:49.561327+0000","last_undegraded":"2023-09-23T19:33:49.561327+0000","last_fullsized":"2023-09-23T19:33:49.561327+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:57.540826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":74,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:56.488569+0000","last_change":"2023-09-23T19:28:07.889095+0000","last_active":"2023-09-23T19:33:56.488569+0000","last_peered":"2023-09-23T19:33:56.488569+0000","last_clean":"2023-09-23T19:33:56.488569+0000","last_became_active":"2023-09-23T19:28:07.888742+0000","last_became_peered":"2023-09-23T19:28:07.888742+0000","last_unstale":"2023-09-23T19:33:56.488569+0000","last_undegraded":"2023-09-23T19:33:56.488569+0000","last_fullsized":"2023-09-23T19:33:56.488569+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:43.929488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"131'10056","reported_seq":10072,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-09-23T19:34:15.266360+0000","last_change":"2023-09-23T19:31:12.577456+0000","last_active":"2023-09-23T19:34:15.266360+0000","last_peered":"2023-09-23T19:34:15.266360+0000","last_clean":"2023-09-23T19:34:15.266360+0000","last_became_active":"2023-09-23T19:31:12.576998+0000","last_became_peered":"2023-09-23T19:31:12.576998+0000","last_unstale":"2023-09-23T19:34:15.266360+0000","last_undegraded":"2023-09-23T19:34:15.266360+0000","last_fullsized":"2023-09-23T19:34:15.266360+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_clean_scrub_stamp":"2023-09-23T19:31:11.546138+0000","objects_scrubbed":0,"log_size":10056,"log_dups_size":0,"ondisk_log_size":10056,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224788480,"num_objects":3430,"num_object_clones":0,"num_object_copies":10290,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3430,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10056,"num_write_kb":431552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":107,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.568806+0000","last_change":"2023-09-23T19:22:42.143289+0000","last_active":"2023-09-23T19:33:49.568806+0000","last_peered":"2023-09-23T19:33:49.568806+0000","last_clean":"2023-09-23T19:33:49.568806+0000","last_became_active":"2023-09-23T19:22:42.142656+0000","last_became_peered":"2023-09-23T19:22:42.142656+0000","last_unstale":"2023-09-23T19:33:49.568806+0000","last_undegraded":"2023-09-23T19:33:49.568806+0000","last_fullsized":"2023-09-23T19:33:49.568806+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:38.467441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":113,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.580314+0000","last_change":"2023-09-23T19:28:07.890623+0000","last_active":"2023-09-23T19:33:49.580314+0000","last_peered":"2023-09-23T19:33:49.580314+0000","last_clean":"2023-09-23T19:33:49.580314+0000","last_became_active":"2023-09-23T19:28:07.890148+0000","last_became_peered":"2023-09-23T19:28:07.890148+0000","last_unstale":"2023-09-23T19:33:49.580314+0000","last_undegraded":"2023-09-23T19:33:49.580314+0000","last_fullsized":"2023-09-23T19:33:49.580314+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":86,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.578298+0000","last_change":"2023-09-23T19:27:24.269080+0000","last_active":"2023-09-23T19:33:49.578298+0000","last_peered":"2023-09-23T19:33:49.578298+0000","last_clean":"2023-09-23T19:33:49.578298+0000","last_became_active":"2023-09-23T19:27:24.268748+0000","last_became_peered":"2023-09-23T19:27:24.268748+0000","last_unstale":"2023-09-23T19:33:49.578298+0000","last_undegraded":"2023-09-23T19:33:49.578298+0000","last_fullsized":"2023-09-23T19:33:49.578298+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:49.014902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.9","version":"131'10230","reported_seq":10252,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-09-23T19:34:17.274387+0000","last_change":"2023-09-23T19:31:12.572956+0000","last_active":"2023-09-23T19:34:17.274387+0000","last_peered":"2023-09-23T19:34:17.274387+0000","last_clean":"2023-09-23T19:34:17.274387+0000","last_became_active":"2023-09-23T19:31:12.572413+0000","last_became_peered":"2023-09-23T19:31:12.572413+0000","last_unstale":"2023-09-23T19:34:17.274387+0000","last_undegraded":"2023-09-23T19:34:17.274387+0000","last_fullsized":"2023-09-23T19:34:17.274387+0000","mapping_epoch":119,"log_start":"122'200","ondisk_log_start":"122'200","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_clean_scrub_stamp":"2023-09-23T19:31:11.546138+0000","objects_scrubbed":0,"log_size":10030,"log_dups_size":0,"ondisk_log_size":10030,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:46:33.130893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228458496,"num_objects":3486,"num_object_clones":0,"num_object_copies":10458,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3486,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10230,"num_write_kb":438912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":102,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.579992+0000","last_change":"2023-09-23T19:22:42.153647+0000","last_active":"2023-09-23T19:33:49.579992+0000","last_peered":"2023-09-23T19:33:49.579992+0000","last_clean":"2023-09-23T19:33:49.579992+0000","last_became_active":"2023-09-23T19:22:42.153317+0000","last_became_peered":"2023-09-23T19:22:42.153317+0000","last_unstale":"2023-09-23T19:33:49.579992+0000","last_undegraded":"2023-09-23T19:33:49.579992+0000","last_fullsized":"2023-09-23T19:33:49.579992+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":97,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.570624+0000","last_change":"2023-09-23T19:18:09.017525+0000","last_active":"2023-09-23T19:33:49.570624+0000","last_peered":"2023-09-23T19:33:49.570624+0000","last_clean":"2023-09-23T19:33:49.570624+0000","last_became_active":"2023-09-23T19:18:09.017193+0000","last_became_peered":"2023-09-23T19:18:09.017193+0000","last_unstale":"2023-09-23T19:33:49.570624+0000","last_undegraded":"2023-09-23T19:33:49.570624+0000","last_fullsized":"2023-09-23T19:33:49.570624+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":89,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.568760+0000","last_change":"2023-09-23T19:28:07.875796+0000","last_active":"2023-09-23T19:33:49.568760+0000","last_peered":"2023-09-23T19:33:49.568760+0000","last_clean":"2023-09-23T19:33:49.568760+0000","last_became_active":"2023-09-23T19:28:07.875310+0000","last_became_peered":"2023-09-23T19:28:07.875310+0000","last_unstale":"2023-09-23T19:33:49.568760+0000","last_undegraded":"2023-09-23T19:33:49.568760+0000","last_fullsized":"2023-09-23T19:33:49.568760+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:44:13.379633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"131'10391","reported_seq":10411,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-09-23T19:34:15.264679+0000","last_change":"2023-09-23T19:31:12.576188+0000","last_active":"2023-09-23T19:34:15.264679+0000","last_peered":"2023-09-23T19:34:15.264679+0000","last_clean":"2023-09-23T19:34:15.264679+0000","last_became_active":"2023-09-23T19:31:12.575832+0000","last_became_peered":"2023-09-23T19:31:12.575832+0000","last_unstale":"2023-09-23T19:34:15.264679+0000","last_undegraded":"2023-09-23T19:34:15.264679+0000","last_fullsized":"2023-09-23T19:34:15.264679+0000","mapping_epoch":119,"log_start":"122'300","ondisk_log_start":"122'300","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_clean_scrub_stamp":"2023-09-23T19:31:11.546138+0000","objects_scrubbed":0,"log_size":10091,"log_dups_size":0,"ondisk_log_size":10091,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":236388352,"num_objects":3607,"num_object_clones":0,"num_object_copies":10821,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3607,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10391,"num_write_kb":447936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":106,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.578298+0000","last_change":"2023-09-23T19:28:07.878688+0000","last_active":"2023-09-23T19:33:49.578298+0000","last_peered":"2023-09-23T19:33:49.578298+0000","last_clean":"2023-09-23T19:33:49.578298+0000","last_became_active":"2023-09-23T19:28:07.878309+0000","last_became_peered":"2023-09-23T19:28:07.878309+0000","last_unstale":"2023-09-23T19:33:49.578298+0000","last_undegraded":"2023-09-23T19:33:49.578298+0000","last_fullsized":"2023-09-23T19:33:49.578298+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:49:06.202657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":74,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.561251+0000","last_change":"2023-09-23T19:27:24.267897+0000","last_active":"2023-09-23T19:33:49.561251+0000","last_peered":"2023-09-23T19:33:49.561251+0000","last_clean":"2023-09-23T19:33:49.561251+0000","last_became_active":"2023-09-23T19:27:24.267535+0000","last_became_peered":"2023-09-23T19:27:24.267535+0000","last_unstale":"2023-09-23T19:33:49.561251+0000","last_undegraded":"2023-09-23T19:33:49.561251+0000","last_fullsized":"2023-09-23T19:33:49.561251+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:13:57.447808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":102,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.560882+0000","last_change":"2023-09-23T19:22:42.143733+0000","last_active":"2023-09-23T19:33:49.560882+0000","last_peered":"2023-09-23T19:33:49.560882+0000","last_clean":"2023-09-23T19:33:49.560882+0000","last_became_active":"2023-09-23T19:22:42.143077+0000","last_became_peered":"2023-09-23T19:22:42.143077+0000","last_unstale":"2023-09-23T19:33:49.560882+0000","last_undegraded":"2023-09-23T19:33:49.560882+0000","last_fullsized":"2023-09-23T19:33:49.560882+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:47:33.086994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"131'9939","reported_seq":9968,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-09-23T19:34:17.323046+0000","last_change":"2023-09-23T19:32:49.743645+0000","last_active":"2023-09-23T19:34:17.323046+0000","last_peered":"2023-09-23T19:34:17.323046+0000","last_clean":"2023-09-23T19:34:17.323046+0000","last_became_active":"2023-09-23T19:32:49.729828+0000","last_became_peered":"2023-09-23T19:32:49.729828+0000","last_unstale":"2023-09-23T19:34:17.323046+0000","last_undegraded":"2023-09-23T19:34:17.323046+0000","last_fullsized":"2023-09-23T19:34:17.323046+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_clean_scrub_stamp":"2023-09-23T19:31:11.546138+0000","objects_scrubbed":0,"log_size":9939,"log_dups_size":0,"ondisk_log_size":9939,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:44:38.489342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":218693632,"num_objects":3337,"num_object_clones":0,"num_object_copies":10011,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3337,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9939,"num_write_kb":424832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":102,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.580082+0000","last_change":"2023-09-23T19:27:24.271520+0000","last_active":"2023-09-23T19:33:49.580082+0000","last_peered":"2023-09-23T19:33:49.580082+0000","last_clean":"2023-09-23T19:33:49.580082+0000","last_became_active":"2023-09-23T19:27:24.271165+0000","last_became_peered":"2023-09-23T19:27:24.271165+0000","last_unstale":"2023-09-23T19:33:49.580082+0000","last_undegraded":"2023-09-23T19:33:49.580082+0000","last_fullsized":"2023-09-23T19:33:49.580082+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":333,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.577303+0000","last_change":"2023-09-23T19:18:09.016297+0000","last_active":"2023-09-23T19:33:49.577303+0000","last_peered":"2023-09-23T19:33:49.577303+0000","last_clean":"2023-09-23T19:33:49.577303+0000","last_became_active":"2023-09-23T19:18:09.015436+0000","last_became_peered":"2023-09-23T19:18:09.015436+0000","last_unstale":"2023-09-23T19:33:49.577303+0000","last_undegraded":"2023-09-23T19:33:49.577303+0000","last_fullsized":"2023-09-23T19:33:49.577303+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:59:33.840751+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:59:33.840751+0000","last_clean_scrub_stamp":"2023-09-23T18:59:33.840751+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":71,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.579898+0000","last_change":"2023-09-23T19:32:49.734148+0000","last_active":"2023-09-23T19:33:49.579898+0000","last_peered":"2023-09-23T19:33:49.579898+0000","last_clean":"2023-09-23T19:33:49.579898+0000","last_became_active":"2023-09-23T19:32:49.733512+0000","last_became_peered":"2023-09-23T19:32:49.733512+0000","last_unstale":"2023-09-23T19:33:49.579898+0000","last_undegraded":"2023-09-23T19:33:49.579898+0000","last_fullsized":"2023-09-23T19:33:49.579898+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":81,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.599729+0000","last_change":"2023-09-23T19:05:52.786771+0000","last_active":"2023-09-23T19:33:49.599729+0000","last_peered":"2023-09-23T19:33:49.599729+0000","last_clean":"2023-09-23T19:33:49.599729+0000","last_became_active":"2023-09-23T19:05:52.785797+0000","last_became_peered":"2023-09-23T19:05:52.785797+0000","last_unstale":"2023-09-23T19:33:49.599729+0000","last_undegraded":"2023-09-23T19:33:49.599729+0000","last_fullsized":"2023-09-23T19:33:49.599729+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"131'9983","reported_seq":10005,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-09-23T19:34:17.292665+0000","last_change":"2023-09-23T19:31:12.591240+0000","last_active":"2023-09-23T19:34:17.292665+0000","last_peered":"2023-09-23T19:34:17.292665+0000","last_clean":"2023-09-23T19:34:17.292665+0000","last_became_active":"2023-09-23T19:31:12.590654+0000","last_became_peered":"2023-09-23T19:31:12.590654+0000","last_unstale":"2023-09-23T19:34:17.292665+0000","last_undegraded":"2023-09-23T19:34:17.292665+0000","last_fullsized":"2023-09-23T19:34:17.292665+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_clean_scrub_stamp":"2023-09-23T19:31:11.546138+0000","objects_scrubbed":0,"log_size":9983,"log_dups_size":0,"ondisk_log_size":9983,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:03.681405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":221315072,"num_objects":3377,"num_object_clones":0,"num_object_copies":10131,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3377,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9983,"num_write_kb":427520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":116,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.577203+0000","last_change":"2023-09-23T19:22:42.150636+0000","last_active":"2023-09-23T19:33:49.577203+0000","last_peered":"2023-09-23T19:33:49.577203+0000","last_clean":"2023-09-23T19:33:49.577203+0000","last_became_active":"2023-09-23T19:22:42.150045+0000","last_became_peered":"2023-09-23T19:22:42.150045+0000","last_unstale":"2023-09-23T19:33:49.577203+0000","last_undegraded":"2023-09-23T19:33:49.577203+0000","last_fullsized":"2023-09-23T19:33:49.577203+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":89,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.580608+0000","last_change":"2023-09-23T19:28:07.889194+0000","last_active":"2023-09-23T19:33:49.580608+0000","last_peered":"2023-09-23T19:33:49.580608+0000","last_clean":"2023-09-23T19:33:49.580608+0000","last_became_active":"2023-09-23T19:28:07.888823+0000","last_became_peered":"2023-09-23T19:28:07.888823+0000","last_unstale":"2023-09-23T19:33:49.580608+0000","last_undegraded":"2023-09-23T19:33:49.580608+0000","last_fullsized":"2023-09-23T19:33:49.580608+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:23.464688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":104,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.580867+0000","last_change":"2023-09-23T19:27:24.256410+0000","last_active":"2023-09-23T19:33:49.580867+0000","last_peered":"2023-09-23T19:33:49.580867+0000","last_clean":"2023-09-23T19:33:49.580867+0000","last_became_active":"2023-09-23T19:27:24.256052+0000","last_became_peered":"2023-09-23T19:27:24.256052+0000","last_unstale":"2023-09-23T19:33:49.580867+0000","last_undegraded":"2023-09-23T19:33:49.580867+0000","last_fullsized":"2023-09-23T19:33:49.580867+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:11.951400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"131'10102","reported_seq":10122,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-09-23T19:34:14.981219+0000","last_change":"2023-09-23T19:31:12.593664+0000","last_active":"2023-09-23T19:34:14.981219+0000","last_peered":"2023-09-23T19:34:14.981219+0000","last_clean":"2023-09-23T19:34:14.981219+0000","last_became_active":"2023-09-23T19:31:12.592662+0000","last_became_peered":"2023-09-23T19:31:12.592662+0000","last_unstale":"2023-09-23T19:34:14.981219+0000","last_undegraded":"2023-09-23T19:34:14.981219+0000","last_fullsized":"2023-09-23T19:34:14.981219+0000","mapping_epoch":119,"log_start":"122'2400","ondisk_log_start":"122'2400","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_clean_scrub_stamp":"2023-09-23T19:31:11.546138+0000","objects_scrubbed":0,"log_size":7702,"log_dups_size":0,"ondisk_log_size":7702,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":232914944,"num_objects":3554,"num_object_clones":0,"num_object_copies":10662,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3554,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10102,"num_write_kb":436992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":101,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.578011+0000","last_change":"2023-09-23T19:18:09.015853+0000","last_active":"2023-09-23T19:33:49.578011+0000","last_peered":"2023-09-23T19:33:49.578011+0000","last_clean":"2023-09-23T19:33:49.578011+0000","last_became_active":"2023-09-23T19:18:09.015338+0000","last_became_peered":"2023-09-23T19:18:09.015338+0000","last_unstale":"2023-09-23T19:33:49.578011+0000","last_undegraded":"2023-09-23T19:33:49.578011+0000","last_fullsized":"2023-09-23T19:33:49.578011+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":74,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.568681+0000","last_change":"2023-09-23T19:27:24.266241+0000","last_active":"2023-09-23T19:33:49.568681+0000","last_peered":"2023-09-23T19:33:49.568681+0000","last_clean":"2023-09-23T19:33:49.568681+0000","last_became_active":"2023-09-23T19:27:24.265764+0000","last_became_peered":"2023-09-23T19:27:24.265764+0000","last_unstale":"2023-09-23T19:33:49.568681+0000","last_undegraded":"2023-09-23T19:33:49.568681+0000","last_fullsized":"2023-09-23T19:33:49.568681+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:07.694732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":74,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.592891+0000","last_change":"2023-09-23T19:18:09.005823+0000","last_active":"2023-09-23T19:33:49.592891+0000","last_peered":"2023-09-23T19:33:49.592891+0000","last_clean":"2023-09-23T19:33:49.592891+0000","last_became_active":"2023-09-23T19:18:09.005207+0000","last_became_peered":"2023-09-23T19:18:09.005207+0000","last_unstale":"2023-09-23T19:33:49.592891+0000","last_undegraded":"2023-09-23T19:33:49.592891+0000","last_fullsized":"2023-09-23T19:33:49.592891+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:05.520188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"131'10057","reported_seq":10078,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-09-23T19:34:15.093037+0000","last_change":"2023-09-23T19:31:12.576193+0000","last_active":"2023-09-23T19:34:15.093037+0000","last_peered":"2023-09-23T19:34:15.093037+0000","last_clean":"2023-09-23T19:34:15.093037+0000","last_became_active":"2023-09-23T19:31:12.575804+0000","last_became_peered":"2023-09-23T19:31:12.575804+0000","last_unstale":"2023-09-23T19:34:15.093037+0000","last_undegraded":"2023-09-23T19:34:15.093037+0000","last_fullsized":"2023-09-23T19:34:15.093037+0000","mapping_epoch":119,"log_start":"122'2300","ondisk_log_start":"122'2300","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_clean_scrub_stamp":"2023-09-23T19:31:11.546138+0000","objects_scrubbed":0,"log_size":7757,"log_dups_size":0,"ondisk_log_size":7757,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228196376,"num_objects":3483,"num_object_clones":0,"num_object_copies":10449,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3483,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10057,"num_write_kb":433217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":35,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.578431+0000","last_change":"2023-09-23T19:32:49.737323+0000","last_active":"2023-09-23T19:33:49.578431+0000","last_peered":"2023-09-23T19:33:49.578431+0000","last_clean":"2023-09-23T19:33:49.578431+0000","last_became_active":"2023-09-23T19:32:49.736937+0000","last_became_peered":"2023-09-23T19:32:49.736937+0000","last_unstale":"2023-09-23T19:33:49.578431+0000","last_undegraded":"2023-09-23T19:33:49.578431+0000","last_fullsized":"2023-09-23T19:33:49.578431+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":59,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.590066+0000","last_change":"2023-09-23T19:22:42.145506+0000","last_active":"2023-09-23T19:33:49.590066+0000","last_peered":"2023-09-23T19:33:49.590066+0000","last_clean":"2023-09-23T19:33:49.590066+0000","last_became_active":"2023-09-23T19:22:42.143991+0000","last_became_peered":"2023-09-23T19:22:42.143991+0000","last_unstale":"2023-09-23T19:33:49.590066+0000","last_undegraded":"2023-09-23T19:33:49.590066+0000","last_fullsized":"2023-09-23T19:33:49.590066+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:56.875175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":72,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.577516+0000","last_change":"2023-09-23T19:32:49.734592+0000","last_active":"2023-09-23T19:33:49.577516+0000","last_peered":"2023-09-23T19:33:49.577516+0000","last_clean":"2023-09-23T19:33:49.577516+0000","last_became_active":"2023-09-23T19:32:49.733905+0000","last_became_peered":"2023-09-23T19:32:49.733905+0000","last_unstale":"2023-09-23T19:33:49.577516+0000","last_undegraded":"2023-09-23T19:33:49.577516+0000","last_fullsized":"2023-09-23T19:33:49.577516+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.c","version":"131'10229","reported_seq":10258,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-09-23T19:34:14.941010+0000","last_change":"2023-09-23T19:32:49.758174+0000","last_active":"2023-09-23T19:34:14.941010+0000","last_peered":"2023-09-23T19:34:14.941010+0000","last_clean":"2023-09-23T19:34:14.941010+0000","last_became_active":"2023-09-23T19:32:49.736194+0000","last_became_peered":"2023-09-23T19:32:49.736194+0000","last_unstale":"2023-09-23T19:34:14.941010+0000","last_undegraded":"2023-09-23T19:34:14.941010+0000","last_fullsized":"2023-09-23T19:34:14.941010+0000","mapping_epoch":126,"log_start":"122'1800","ondisk_log_start":"122'1800","created":119,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:31:11.546138+0000","last_clean_scrub_stamp":"2023-09-23T19:31:11.546138+0000","objects_scrubbed":0,"log_size":8429,"log_dups_size":0,"ondisk_log_size":8429,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":232325120,"num_objects":3545,"num_object_clones":0,"num_object_copies":10635,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3545,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10229,"num_write_kb":440768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":101,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.599573+0000","last_change":"2023-09-23T19:27:24.271827+0000","last_active":"2023-09-23T19:33:49.599573+0000","last_peered":"2023-09-23T19:33:49.599573+0000","last_clean":"2023-09-23T19:33:49.599573+0000","last_became_active":"2023-09-23T19:27:24.271328+0000","last_became_peered":"2023-09-23T19:27:24.271328+0000","last_unstale":"2023-09-23T19:33:49.599573+0000","last_undegraded":"2023-09-23T19:33:49.599573+0000","last_fullsized":"2023-09-23T19:33:49.599573+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":92,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.580762+0000","last_change":"2023-09-23T19:22:42.149694+0000","last_active":"2023-09-23T19:33:49.580762+0000","last_peered":"2023-09-23T19:33:49.580762+0000","last_clean":"2023-09-23T19:33:49.580762+0000","last_became_active":"2023-09-23T19:22:42.148194+0000","last_became_peered":"2023-09-23T19:22:42.148194+0000","last_unstale":"2023-09-23T19:33:49.580762+0000","last_undegraded":"2023-09-23T19:33:49.580762+0000","last_fullsized":"2023-09-23T19:33:49.580762+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:47.842383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":81,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.569988+0000","last_change":"2023-09-23T19:05:52.138620+0000","last_active":"2023-09-23T19:33:49.569988+0000","last_peered":"2023-09-23T19:33:49.569988+0000","last_clean":"2023-09-23T19:33:49.569988+0000","last_became_active":"2023-09-23T19:05:52.138243+0000","last_became_peered":"2023-09-23T19:05:52.138243+0000","last_unstale":"2023-09-23T19:33:49.569988+0000","last_undegraded":"2023-09-23T19:33:49.569988+0000","last_fullsized":"2023-09-23T19:33:49.569988+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":76,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.593121+0000","last_change":"2023-09-23T19:18:09.008811+0000","last_active":"2023-09-23T19:33:49.593121+0000","last_peered":"2023-09-23T19:33:49.593121+0000","last_clean":"2023-09-23T19:33:49.593121+0000","last_became_active":"2023-09-23T19:18:09.007255+0000","last_became_peered":"2023-09-23T19:18:09.007255+0000","last_unstale":"2023-09-23T19:33:49.593121+0000","last_undegraded":"2023-09-23T19:33:49.593121+0000","last_fullsized":"2023-09-23T19:33:49.593121+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:25.874038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":113,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.578847+0000","last_change":"2023-09-23T19:28:07.876141+0000","last_active":"2023-09-23T19:33:49.578847+0000","last_peered":"2023-09-23T19:33:49.578847+0000","last_clean":"2023-09-23T19:33:49.578847+0000","last_became_active":"2023-09-23T19:28:07.875802+0000","last_became_peered":"2023-09-23T19:28:07.875802+0000","last_unstale":"2023-09-23T19:33:49.578847+0000","last_undegraded":"2023-09-23T19:33:49.578847+0000","last_fullsized":"2023-09-23T19:33:49.578847+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:38.947881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":100,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.580068+0000","last_change":"2023-09-23T19:22:42.153825+0000","last_active":"2023-09-23T19:33:49.580068+0000","last_peered":"2023-09-23T19:33:49.580068+0000","last_clean":"2023-09-23T19:33:49.580068+0000","last_became_active":"2023-09-23T19:22:42.153416+0000","last_became_peered":"2023-09-23T19:22:42.153416+0000","last_unstale":"2023-09-23T19:33:49.580068+0000","last_undegraded":"2023-09-23T19:33:49.580068+0000","last_fullsized":"2023-09-23T19:33:49.580068+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":120,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.570745+0000","last_change":"2023-09-23T19:28:07.899164+0000","last_active":"2023-09-23T19:33:49.570745+0000","last_peered":"2023-09-23T19:33:49.570745+0000","last_clean":"2023-09-23T19:33:49.570745+0000","last_became_active":"2023-09-23T19:28:07.898812+0000","last_became_peered":"2023-09-23T19:28:07.898812+0000","last_unstale":"2023-09-23T19:33:49.570745+0000","last_undegraded":"2023-09-23T19:33:49.570745+0000","last_fullsized":"2023-09-23T19:33:49.570745+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":85,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.580333+0000","last_change":"2023-09-23T19:05:52.107570+0000","last_active":"2023-09-23T19:33:49.580333+0000","last_peered":"2023-09-23T19:33:49.580333+0000","last_clean":"2023-09-23T19:33:49.580333+0000","last_became_active":"2023-09-23T19:05:52.107215+0000","last_became_peered":"2023-09-23T19:05:52.107215+0000","last_unstale":"2023-09-23T19:33:49.580333+0000","last_undegraded":"2023-09-23T19:33:49.580333+0000","last_fullsized":"2023-09-23T19:33:49.580333+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":111,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.570488+0000","last_change":"2023-09-23T19:28:07.896983+0000","last_active":"2023-09-23T19:33:49.570488+0000","last_peered":"2023-09-23T19:33:49.570488+0000","last_clean":"2023-09-23T19:33:49.570488+0000","last_became_active":"2023-09-23T19:28:07.896577+0000","last_became_peered":"2023-09-23T19:28:07.896577+0000","last_unstale":"2023-09-23T19:33:49.570488+0000","last_undegraded":"2023-09-23T19:33:49.570488+0000","last_fullsized":"2023-09-23T19:33:49.570488+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":25,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.599764+0000","last_change":"2023-09-23T19:32:49.732683+0000","last_active":"2023-09-23T19:33:49.599764+0000","last_peered":"2023-09-23T19:33:49.599764+0000","last_clean":"2023-09-23T19:33:49.599764+0000","last_became_active":"2023-09-23T19:32:49.732188+0000","last_became_peered":"2023-09-23T19:32:49.732188+0000","last_unstale":"2023-09-23T19:33:49.599764+0000","last_undegraded":"2023-09-23T19:33:49.599764+0000","last_fullsized":"2023-09-23T19:33:49.599764+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":112,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.578465+0000","last_change":"2023-09-23T19:32:49.725466+0000","last_active":"2023-09-23T19:33:49.578465+0000","last_peered":"2023-09-23T19:33:49.578465+0000","last_clean":"2023-09-23T19:33:49.578465+0000","last_became_active":"2023-09-23T19:32:49.725048+0000","last_became_peered":"2023-09-23T19:32:49.725048+0000","last_unstale":"2023-09-23T19:33:49.578465+0000","last_undegraded":"2023-09-23T19:33:49.578465+0000","last_fullsized":"2023-09-23T19:33:49.578465+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:44:08.728220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":92,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.580933+0000","last_change":"2023-09-23T19:27:24.267931+0000","last_active":"2023-09-23T19:33:49.580933+0000","last_peered":"2023-09-23T19:33:49.580933+0000","last_clean":"2023-09-23T19:33:49.580933+0000","last_became_active":"2023-09-23T19:27:24.267552+0000","last_became_peered":"2023-09-23T19:27:24.267552+0000","last_unstale":"2023-09-23T19:33:49.580933+0000","last_undegraded":"2023-09-23T19:33:49.580933+0000","last_fullsized":"2023-09-23T19:33:49.580933+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:12.425257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":90,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T19:33:49.569838+0000","last_change":"2023-09-23T19:06:36.673002+0000","last_active":"2023-09-23T19:33:49.569838+0000","last_peered":"2023-09-23T19:33:49.569838+0000","last_clean":"2023-09-23T19:33:49.569838+0000","last_became_active":"2023-09-23T19:06:36.671911+0000","last_became_peered":"2023-09-23T19:06:36.671911+0000","last_unstale":"2023-09-23T19:33:49.569838+0000","last_undegraded":"2023-09-23T19:33:49.569838+0000","last_fullsized":"2023-09-23T19:33:49.569838+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":3628924952,"num_objects":55374,"num_object_clones":0,"num_object_copies":166122,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":55374,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":161386,"num_write_kb":6936257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10890981376,"data_stored":10890969160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":143786,"ondisk_log_size":143786,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430585,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2626540,"kb_used_data":2032740,"kb_used_omap":0,"kb_used_meta":593792,"kb_avail":91118612,"statfs":{"total":95995035648,"available":93305458688,"internally_reserved":0,"allocated":2081525760,"data_stored":2081050149,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":608043008},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:33:34 2023","interfaces":[{"interface":"back","average":{"1min":1.4019999999999999,"5min":1.1140000000000001,"15min":1.226},"min":{"1min":0.27600000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":7.1390000000000002,"5min":7.1390000000000002,"15min":9.6760000000000002},"last":2.1949999999999998},{"interface":"front","average":{"1min":1.742,"5min":1.2569999999999999,"15min":1.3380000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.28999999999999998},"max":{"1min":5.7690000000000001,"5min":7.907,"15min":8.9670000000000005},"last":0.46300000000000002}]},{"osd":1,"last update":"Sat Sep 23 19:34:03 2023","interfaces":[{"interface":"back","average":{"1min":1.4079999999999999,"5min":1.226,"15min":1.357},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.28499999999999998},"max":{"1min":5.5970000000000004,"5min":9.8209999999999997,"15min":9.8209999999999997},"last":0.67200000000000004},{"interface":"front","average":{"1min":1.726,"5min":1.2290000000000001,"15min":1.3620000000000001},"min":{"1min":0.30099999999999999,"5min":0.29999999999999999,"15min":0.29599999999999999},"max":{"1min":8.9390000000000001,"5min":8.9390000000000001,"15min":8.9390000000000001},"last":0.64800000000000002}]},{"osd":2,"last update":"Sat Sep 23 19:33:35 2023","interfaces":[{"interface":"back","average":{"1min":2.069,"5min":1.345,"15min":1.0049999999999999},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":5.718,"5min":7.9320000000000004,"15min":7.9320000000000004},"last":0.36899999999999999},{"interface":"front","average":{"1min":1.4279999999999999,"5min":1.1060000000000001,"15min":1.016},"min":{"1min":0.36799999999999999,"5min":0.308,"15min":0.308},"max":{"1min":4.7240000000000002,"5min":5.6719999999999997,"15min":5.6719999999999997},"last":0.54200000000000004}]},{"osd":3,"last update":"Sat Sep 23 19:33:14 2023","interfaces":[{"interface":"back","average":{"1min":2.2149999999999999,"5min":1.502,"15min":1.206},"min":{"1min":0.47999999999999998,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":9.9260000000000002,"5min":19.637,"15min":19.637},"last":0.47299999999999998},{"interface":"front","average":{"1min":1.6439999999999999,"5min":1.482,"15min":1.246},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":6.0010000000000003,"5min":20.204000000000001,"15min":20.204000000000001},"last":0.48799999999999999}]},{"osd":4,"last update":"Sat Sep 23 19:34:03 2023","interfaces":[{"interface":"back","average":{"1min":1.2949999999999999,"5min":1.1259999999999999,"15min":1.1259999999999999},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":5.5590000000000002,"5min":9.7699999999999996,"15min":9.7699999999999996},"last":0.41999999999999998},{"interface":"front","average":{"1min":1.2490000000000001,"5min":1.1419999999999999,"15min":1.056},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":6.5899999999999999,"5min":6.5899999999999999,"15min":7.5090000000000003},"last":0.443}]},{"osd":5,"last update":"Sat Sep 23 19:34:03 2023","interfaces":[{"interface":"back","average":{"1min":1.6879999999999999,"5min":1.4490000000000001,"15min":1.2709999999999999},"min":{"1min":0.41199999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":5.2809999999999997,"5min":7.867,"15min":7.867},"last":6.1379999999999999},{"interface":"front","average":{"1min":1.867,"5min":1.48,"15min":1.4299999999999999},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":8.4760000000000009,"5min":9.8979999999999997,"15min":9.8979999999999997},"last":0.69799999999999995}]},{"osd":6,"last update":"Sat Sep 23 19:34:03 2023","interfaces":[{"interface":"back","average":{"1min":1.8140000000000001,"5min":1.3220000000000001,"15min":1.3200000000000001},"min":{"1min":0.41799999999999998,"5min":0.40899999999999997,"15min":0.34599999999999997},"max":{"1min":5.6589999999999998,"5min":9.8629999999999995,"15min":13.462},"last":0.63300000000000001},{"interface":"front","average":{"1min":1.6739999999999999,"5min":1.3049999999999999,"15min":1.1719999999999999},"min":{"1min":0.41299999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":8.5009999999999994,"5min":8.5009999999999994,"15min":8.5009999999999994},"last":0.51600000000000001}]}]},{"osd":6,"up_from":42,"seq":180388626817,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":981016,"kb_used_data":675600,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":92764136,"statfs":{"total":95995035648,"available":94990475264,"internally_reserved":0,"allocated":691814400,"data_stored":691358784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:33:43 2023","interfaces":[{"interface":"back","average":{"1min":1.782,"5min":1.8380000000000001,"15min":1.5589999999999999},"min":{"1min":0.313,"5min":0.25900000000000001,"15min":0.24099999999999999},"max":{"1min":8.1289999999999996,"5min":87.283000000000001,"15min":87.283000000000001},"last":0.32000000000000001},{"interface":"front","average":{"1min":1.0740000000000001,"5min":1.827,"15min":1.548},"min":{"1min":0.312,"5min":0.30399999999999999,"15min":0.25900000000000001},"max":{"1min":3.7690000000000001,"5min":86.286000000000001,"15min":86.286000000000001},"last":2.351}]},{"osd":1,"last update":"Sat Sep 23 19:34:02 2023","interfaces":[{"interface":"back","average":{"1min":1.956,"5min":1.337,"15min":1.518},"min":{"1min":0.29699999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":8.0899999999999999,"5min":8.0899999999999999,"15min":11.351000000000001},"last":0.40600000000000003},{"interface":"front","average":{"1min":1.2090000000000001,"5min":1.2709999999999999,"15min":1.5860000000000001},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.34200000000000003},"max":{"1min":4.3659999999999997,"5min":7.4320000000000004,"15min":12.728999999999999},"last":2.3940000000000001}]},{"osd":2,"last update":"Sat Sep 23 19:33:35 2023","interfaces":[{"interface":"back","average":{"1min":0.94699999999999995,"5min":0.86299999999999999,"15min":0.90600000000000003},"min":{"1min":0.34599999999999997,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":2.9670000000000001,"5min":2.9670000000000001,"15min":2.9670000000000001},"last":0.379},{"interface":"front","average":{"1min":1.4610000000000001,"5min":0.97599999999999998,"15min":0.92200000000000004},"min":{"1min":0.35499999999999998,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":8.0600000000000005,"5min":8.0600000000000005,"15min":8.0600000000000005},"last":0.45000000000000001}]},{"osd":3,"last update":"Sat Sep 23 19:33:19 2023","interfaces":[{"interface":"back","average":{"1min":4.306,"5min":1.7170000000000001,"15min":1.2350000000000001},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":60.726999999999997,"5min":60.726999999999997,"15min":60.726999999999997},"last":0.439},{"interface":"front","average":{"1min":3.931,"5min":1.611,"15min":1.1819999999999999},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":58.304000000000002,"5min":58.304000000000002,"15min":58.304000000000002},"last":0.49099999999999999}]},{"osd":4,"last update":"Sat Sep 23 19:33:19 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":0.90500000000000003,"15min":0.97599999999999998},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.29299999999999998},"max":{"1min":3.4390000000000001,"5min":3.4390000000000001,"15min":6.5869999999999997},"last":2.4249999999999998},{"interface":"front","average":{"1min":0.72999999999999998,"5min":0.93300000000000005,"15min":1.091},"min":{"1min":0.378,"5min":0.376,"15min":0.30499999999999999},"max":{"1min":2.9319999999999999,"5min":2.9319999999999999,"15min":8.5540000000000003},"last":0.64200000000000002}]},{"osd":5,"last update":"Sat Sep 23 19:33:19 2023","interfaces":[{"interface":"back","average":{"1min":1.0189999999999999,"5min":0.997,"15min":1.1240000000000001},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.35799999999999998},"max":{"1min":2.7799999999999998,"5min":2.7799999999999998,"15min":6.6660000000000004},"last":2.4510000000000001},{"interface":"front","average":{"1min":1.8280000000000001,"5min":1.236,"15min":1.3560000000000001},"min":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.38100000000000001},"max":{"1min":6.6100000000000003,"5min":6.6100000000000003,"15min":8.5820000000000007},"last":2.3740000000000001}]},{"osd":7,"last update":"Sat Sep 23 19:33:48 2023","interfaces":[{"interface":"back","average":{"1min":1.349,"5min":1.133,"15min":1.1339999999999999},"min":{"1min":0.58499999999999996,"5min":0.46100000000000002,"15min":0.38300000000000001},"max":{"1min":3.4049999999999998,"5min":3.4049999999999998,"15min":8.7949999999999999},"last":3.496},{"interface":"front","average":{"1min":2.262,"5min":1.3240000000000001,"15min":1.3220000000000001},"min":{"1min":0.53200000000000003,"5min":0.45700000000000002,"15min":0.39300000000000002},"max":{"1min":8.8059999999999992,"5min":8.8059999999999992,"15min":8.8059999999999992},"last":0.48099999999999998}]}]},{"osd":1,"up_from":98,"seq":420906795149,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2329628,"kb_used_data":1989844,"kb_used_omap":4,"kb_used_meta":339771,"kb_avail":91415524,"statfs":{"total":95995035648,"available":93609496576,"internally_reserved":0,"allocated":2037600256,"data_stored":2037025117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":347926414},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:34:06 2023","interfaces":[{"interface":"back","average":{"1min":1.5429999999999999,"5min":1.0169999999999999,"15min":1.7350000000000001},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.20799999999999999},"max":{"1min":5.1440000000000001,"5min":7.1109999999999998,"15min":14.99},"last":0.68799999999999994},{"interface":"front","average":{"1min":2.4180000000000001,"5min":1.238,"15min":1.8},"min":{"1min":0.29499999999999998,"5min":0.255,"15min":0.23499999999999999},"max":{"1min":6.8840000000000003,"5min":6.8840000000000003,"15min":12.215},"last":1.798}]},{"osd":2,"last update":"Sat Sep 23 19:33:35 2023","interfaces":[{"interface":"back","average":{"1min":1.3540000000000001,"5min":0.93999999999999995,"15min":0.80400000000000005},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":5.7750000000000004,"5min":6.2919999999999998,"15min":6.2919999999999998},"last":0.63600000000000001},{"interface":"front","average":{"1min":1.7010000000000001,"5min":1.2150000000000001,"15min":0.85399999999999998},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":7.3769999999999998,"5min":11.308,"15min":11.308},"last":0.57799999999999996}]},{"osd":3,"last update":"Sat Sep 23 19:33:16 2023","interfaces":[{"interface":"back","average":{"1min":2.6179999999999999,"5min":1.2230000000000001,"15min":0.90900000000000003},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":10.612,"5min":10.612,"15min":10.612},"last":1.607},{"interface":"front","average":{"1min":3.1459999999999999,"5min":1.6419999999999999,"15min":1.0960000000000001},"min":{"1min":0.41599999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":9.2949999999999999,"5min":11.372,"15min":11.372},"last":3.052}]},{"osd":4,"last update":"Sat Sep 23 19:34:06 2023","interfaces":[{"interface":"back","average":{"1min":1.972,"5min":1.294,"15min":1.526},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.35899999999999999},"max":{"1min":7.0190000000000001,"5min":7.0190000000000001,"15min":7.6130000000000004},"last":1.776},{"interface":"front","average":{"1min":2.097,"5min":1.5840000000000001,"15min":1.54},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.34399999999999997},"max":{"1min":5.7309999999999999,"5min":11.409000000000001,"15min":11.409000000000001},"last":0.60599999999999998}]},{"osd":5,"last update":"Sat Sep 23 19:34:06 2023","interfaces":[{"interface":"back","average":{"1min":2.452,"5min":1.966,"15min":1.7250000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":6.0389999999999997,"5min":11.456,"15min":12.640000000000001},"last":0.72499999999999998},{"interface":"front","average":{"1min":2.1659999999999999,"5min":1.4590000000000001,"15min":1.546},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":7.1070000000000002,"5min":7.1589999999999998,"15min":9.218},"last":0.86199999999999999}]},{"osd":6,"last update":"Sat Sep 23 19:34:06 2023","interfaces":[{"interface":"back","average":{"1min":1.8220000000000001,"5min":1.3200000000000001,"15min":1.5589999999999999},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.40100000000000002},"max":{"1min":5.1349999999999998,"5min":7.1890000000000001,"15min":9.3010000000000002},"last":0.66700000000000004},{"interface":"front","average":{"1min":2.2480000000000002,"5min":1.4370000000000001,"15min":1.712},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.38800000000000001},"max":{"1min":7.0819999999999999,"5min":7.0819999999999999,"15min":9.1699999999999999},"last":1.746}]},{"osd":7,"last update":"Sat Sep 23 19:34:06 2023","interfaces":[{"interface":"back","average":{"1min":2.4119999999999999,"5min":1.6479999999999999,"15min":1.865},"min":{"1min":0.432,"5min":0.432,"15min":0.40999999999999998},"max":{"1min":11.391,"5min":11.391,"15min":11.391},"last":2.1859999999999999},{"interface":"front","average":{"1min":2.5659999999999998,"5min":1.9370000000000001,"15min":1.728},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.42699999999999999},"max":{"1min":11.374000000000001,"5min":12.709,"15min":12.709},"last":0.70099999999999996}]}]},{"osd":0,"up_from":85,"seq":365072220356,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1281640,"kb_used_data":878880,"kb_used_omap":5,"kb_used_meta":402746,"kb_avail":92463512,"statfs":{"total":95995035648,"available":94682636288,"internally_reserved":0,"allocated":899973120,"data_stored":899390352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":412412538},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 19:33:58 2023","interfaces":[{"interface":"back","average":{"1min":1.425,"5min":1.171,"15min":1.4890000000000001},"min":{"1min":0.223,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":7.3680000000000003,"5min":11.939,"15min":13.557},"last":1.635},{"interface":"front","average":{"1min":1.675,"5min":1.4419999999999999,"15min":1.768},"min":{"1min":0.315,"5min":0.29099999999999998,"15min":0.253},"max":{"1min":8.3130000000000006,"5min":12.939,"15min":12.939},"last":0.64200000000000002}]},{"osd":2,"last update":"Sat Sep 23 19:33:34 2023","interfaces":[{"interface":"back","average":{"1min":1.363,"5min":1.004,"15min":0.84899999999999998},"min":{"1min":0.28000000000000003,"5min":0.214,"15min":0.214},"max":{"1min":8.2289999999999992,"5min":11.861000000000001,"15min":11.861000000000001},"last":0.85799999999999998},{"interface":"front","average":{"1min":0.98299999999999998,"5min":1.0429999999999999,"15min":0.873},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":3.6509999999999998,"5min":10.212,"15min":10.212},"last":0.67900000000000005}]},{"osd":3,"last update":"Sat Sep 23 19:34:14 2023","interfaces":[{"interface":"back","average":{"1min":2.105,"5min":1.371,"15min":0.95899999999999996},"min":{"1min":0.34399999999999997,"5min":0.251,"15min":0.251},"max":{"1min":6.1959999999999997,"5min":7.4580000000000002,"15min":7.4580000000000002},"last":3.762},{"interface":"front","average":{"1min":1.629,"5min":1.617,"15min":1.1919999999999999},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":4.8550000000000004,"5min":12.991,"15min":12.991},"last":3.706}]},{"osd":4,"last update":"Sat Sep 23 19:33:40 2023","interfaces":[{"interface":"back","average":{"1min":1.1419999999999999,"5min":1.119,"15min":1.4570000000000001},"min":{"1min":0.374,"5min":0.34699999999999998,"15min":0.32100000000000001},"max":{"1min":4.0110000000000001,"5min":4.1120000000000001,"15min":11.156000000000001},"last":3.6259999999999999},{"interface":"front","average":{"1min":1.351,"5min":1.1659999999999999,"15min":1.5269999999999999},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.35599999999999998},"max":{"1min":3.762,"5min":3.762,"15min":13.566000000000001},"last":0.78700000000000003}]},{"osd":5,"last update":"Sat Sep 23 19:33:40 2023","interfaces":[{"interface":"back","average":{"1min":2.4460000000000002,"5min":1.6619999999999999,"15min":1.798},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.33600000000000002},"max":{"1min":8.3610000000000007,"5min":8.3610000000000007,"15min":13.509},"last":0.753},{"interface":"front","average":{"1min":2.3159999999999998,"5min":1.627,"15min":1.75},"min":{"1min":0.38500000000000001,"5min":0.36899999999999999,"15min":0.33400000000000002},"max":{"1min":8.2400000000000002,"5min":8.2400000000000002,"15min":11.048999999999999},"last":3.6699999999999999}]},{"osd":6,"last update":"Sat Sep 23 19:33:40 2023","interfaces":[{"interface":"back","average":{"1min":1.2210000000000001,"5min":1.1950000000000001,"15min":1.6240000000000001},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.36499999999999999},"max":{"1min":4.0490000000000004,"5min":4.0490000000000004,"15min":11.467000000000001},"last":3.7229999999999999},{"interface":"front","average":{"1min":1.556,"5min":1.3029999999999999,"15min":1.7230000000000001},"min":{"1min":0.48499999999999999,"5min":0.36699999999999999,"15min":0.36399999999999999},"max":{"1min":4.0270000000000001,"5min":4.2670000000000003,"15min":13.603999999999999},"last":0.81599999999999995}]},{"osd":7,"last update":"Sat Sep 23 19:33:40 2023","interfaces":[{"interface":"back","average":{"1min":1.069,"5min":1.3480000000000001,"15min":1.7150000000000001},"min":{"1min":0.51800000000000002,"5min":0.44500000000000001,"15min":0.434},"max":{"1min":3.2589999999999999,"5min":12.385,"15min":16.271999999999998},"last":0.40000000000000002},{"interface":"front","average":{"1min":2.0179999999999998,"5min":1.4379999999999999,"15min":1.71},"min":{"1min":0.499,"5min":0.435,"15min":0.38300000000000001},"max":{"1min":7.1100000000000003,"5min":7.1100000000000003,"15min":11.098000000000001},"last":8.8450000000000006}]}]},{"osd":2,"up_from":111,"seq":476741369941,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":836776,"kb_used_data":654304,"kb_used_omap":7,"kb_used_meta":182456,"kb_avail":92908376,"statfs":{"total":95995035648,"available":95138177024,"internally_reserved":0,"allocated":670007296,"data_stored":669479132,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":186835025},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:33:31 2023","interfaces":[{"interface":"back","average":{"1min":0.82399999999999995,"5min":0.71799999999999997,"15min":0.70999999999999996},"min":{"1min":0.21199999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":3.613,"5min":3.613,"15min":3.613},"last":0.26000000000000001},{"interface":"front","average":{"1min":1.1930000000000001,"5min":0.83699999999999997,"15min":0.79100000000000004},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":7.7549999999999999,"5min":7.7549999999999999,"15min":7.7549999999999999},"last":0.73399999999999999}]},{"osd":1,"last update":"Sat Sep 23 19:33:31 2023","interfaces":[{"interface":"back","average":{"1min":6.9870000000000001,"5min":2.1019999999999999,"15min":1.206},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":92.933000000000007,"5min":92.933000000000007,"15min":92.933000000000007},"last":6.6349999999999998},{"interface":"front","average":{"1min":7.0149999999999997,"5min":2.1230000000000002,"15min":1.2749999999999999},"min":{"1min":0.40799999999999997,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":92.432000000000002,"5min":92.432000000000002,"15min":92.432000000000002},"last":2.8399999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:33:20 2023","interfaces":[{"interface":"back","average":{"1min":6.9029999999999996,"5min":2.1070000000000002,"15min":1.1990000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":92.350999999999999,"5min":92.350999999999999,"15min":92.350999999999999},"last":0.45700000000000002},{"interface":"front","average":{"1min":6.9939999999999998,"5min":2.1779999999999999,"15min":1.302},"min":{"1min":0.32600000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":96.236999999999995,"5min":96.236999999999995,"15min":96.236999999999995},"last":5.2329999999999997}]},{"osd":4,"last update":"Sat Sep 23 19:33:31 2023","interfaces":[{"interface":"back","average":{"1min":0.79100000000000004,"5min":0.97699999999999998,"15min":1.0069999999999999},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":2.7240000000000002,"5min":3.1240000000000001,"15min":3.1240000000000001},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.72699999999999998,"5min":1.0589999999999999,"15min":1.0249999999999999},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":2.7669999999999999,"5min":9.0199999999999996,"15min":9.0199999999999996},"last":0.79500000000000004}]},{"osd":5,"last update":"Sat Sep 23 19:33:31 2023","interfaces":[{"interface":"back","average":{"1min":1.7150000000000001,"5min":1.284,"15min":1.1879999999999999},"min":{"1min":0.42499999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":4.9000000000000004,"5min":4.9000000000000004,"15min":4.9000000000000004},"last":0.67900000000000005},{"interface":"front","average":{"1min":1.617,"5min":1.2949999999999999,"15min":1.194},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":4.8310000000000004,"5min":4.8310000000000004,"15min":4.8310000000000004},"last":0.77400000000000002}]},{"osd":6,"last update":"Sat Sep 23 19:33:31 2023","interfaces":[{"interface":"back","average":{"1min":0.81899999999999995,"5min":1.0740000000000001,"15min":1.1000000000000001},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":2.8100000000000001,"5min":9.0739999999999998,"15min":9.0739999999999998},"last":0.85599999999999998},{"interface":"front","average":{"1min":0.92300000000000004,"5min":1.0720000000000001,"15min":1.1379999999999999},"min":{"1min":0.41899999999999998,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":3.52,"5min":3.52,"15min":3.52},"last":0.61799999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:33:31 2023","interfaces":[{"interface":"back","average":{"1min":2.4700000000000002,"5min":1.4670000000000001,"15min":1.3009999999999999},"min":{"1min":0.5,"5min":0.48799999999999999,"15min":0.48799999999999999},"max":{"1min":7.8600000000000003,"5min":7.8600000000000003,"15min":7.8600000000000003},"last":0.70399999999999996},{"interface":"front","average":{"1min":2.4119999999999999,"5min":1.488,"15min":1.2849999999999999},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":7.8280000000000003,"5min":9.1150000000000002,"15min":9.1150000000000002},"last":0.82599999999999996}]}]},{"osd":3,"up_from":116,"seq":498216206412,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2015500,"kb_used_data":1771908,"kb_used_omap":2,"kb_used_meta":243453,"kb_avail":91729652,"statfs":{"total":95995035648,"available":93931163648,"internally_reserved":0,"allocated":1814433792,"data_stored":1813929192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":249295925},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,3,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:33:16 2023","interfaces":[{"interface":"back","average":{"1min":1.7789999999999999,"5min":1.0589999999999999,"15min":0.79100000000000004},"min":{"1min":0.249,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":6.8540000000000001,"5min":6.8540000000000001,"15min":6.8540000000000001},"last":0.66400000000000003},{"interface":"front","average":{"1min":2,"5min":1.1859999999999999,"15min":0.93300000000000005},"min":{"1min":0.27900000000000003,"5min":0.251,"15min":0.251},"max":{"1min":6.3739999999999997,"5min":6.3739999999999997,"15min":6.3739999999999997},"last":2.8199999999999998}]},{"osd":1,"last update":"Sat Sep 23 19:33:16 2023","interfaces":[{"interface":"back","average":{"1min":1.754,"5min":1.2569999999999999,"15min":0.91400000000000003},"min":{"1min":0.308,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":5.4859999999999998,"5min":13.659000000000001,"15min":13.659000000000001},"last":5.585},{"interface":"front","average":{"1min":2.1619999999999999,"5min":1.4159999999999999,"15min":1.0049999999999999},"min":{"1min":0.36499999999999999,"5min":0.253,"15min":0.253},"max":{"1min":4.7389999999999999,"5min":11.989000000000001,"15min":11.989000000000001},"last":6.0380000000000003}]},{"osd":2,"last update":"Sat Sep 23 19:33:16 2023","interfaces":[{"interface":"back","average":{"1min":1.4730000000000001,"5min":0.96599999999999997,"15min":0.83899999999999997},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":6.9950000000000001,"5min":6.9950000000000001,"15min":6.9950000000000001},"last":2.5830000000000002},{"interface":"front","average":{"1min":1.5109999999999999,"5min":1.1379999999999999,"15min":0.94399999999999995},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":5.6269999999999998,"5min":5.9589999999999996,"15min":5.9589999999999996},"last":6.077}]},{"osd":4,"last update":"Sat Sep 23 19:33:16 2023","interfaces":[{"interface":"back","average":{"1min":1.304,"5min":1.218,"15min":1.1240000000000001},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":3.4199999999999999,"5min":6.0069999999999997,"15min":6.0069999999999997},"last":0.628},{"interface":"front","average":{"1min":1.744,"5min":1.3520000000000001,"15min":1.216},"min":{"1min":0.41499999999999998,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":5.5229999999999997,"5min":5.5229999999999997,"15min":5.5229999999999997},"last":2.7330000000000001}]},{"osd":5,"last update":"Sat Sep 23 19:33:16 2023","interfaces":[{"interface":"back","average":{"1min":3.0299999999999998,"5min":1.7390000000000001,"15min":1.3300000000000001},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":7.8559999999999999,"5min":7.8559999999999999,"15min":7.8559999999999999},"last":2.79},{"interface":"front","average":{"1min":2.278,"5min":1.403,"15min":1.1879999999999999},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":7.9699999999999998,"5min":7.9699999999999998,"15min":7.9699999999999998},"last":2.6179999999999999}]},{"osd":6,"last update":"Sat Sep 23 19:33:16 2023","interfaces":[{"interface":"back","average":{"1min":2.0939999999999999,"5min":1.444,"15min":1.2450000000000001},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":6.319,"5min":6.319,"15min":6.319},"last":2.7629999999999999},{"interface":"front","average":{"1min":1.603,"5min":1.28,"15min":1.1919999999999999},"min":{"1min":0.40999999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":7.1210000000000004,"5min":7.1210000000000004,"15min":7.1210000000000004},"last":2.645}]},{"osd":7,"last update":"Sat Sep 23 19:33:16 2023","interfaces":[{"interface":"back","average":{"1min":2.0800000000000001,"5min":1.4119999999999999,"15min":1.244},"min":{"1min":0.376,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":11.259,"5min":11.259,"15min":11.259},"last":2.6779999999999999},{"interface":"front","average":{"1min":1.8819999999999999,"5min":1.421,"15min":1.244},"min":{"1min":0.52100000000000002,"5min":0.443,"15min":0.443},"max":{"1min":6.7309999999999999,"5min":6.7309999999999999,"15min":6.7309999999999999},"last":0.58499999999999996}]}]},{"osd":4,"up_from":30,"seq":128849019282,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1279136,"kb_used_data":685336,"kb_used_omap":0,"kb_used_meta":593792,"kb_avail":92466016,"statfs":{"total":95995035648,"available":94685200384,"internally_reserved":0,"allocated":701784064,"data_stored":701320103,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":608043008},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:33:37 2023","interfaces":[{"interface":"back","average":{"1min":1.0589999999999999,"5min":0.89300000000000002,"15min":1.647},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":3.9249999999999998,"5min":3.9249999999999998,"15min":129.16200000000001},"last":7.391},{"interface":"front","average":{"1min":0.95999999999999996,"5min":1.002,"15min":1.7430000000000001},"min":{"1min":0.307,"5min":0.307,"15min":0.27700000000000002},"max":{"1min":4.8250000000000002,"5min":4.8250000000000002,"15min":130.40600000000001},"last":0.33700000000000002}]},{"osd":1,"last update":"Sat Sep 23 19:34:09 2023","interfaces":[{"interface":"back","average":{"1min":1.1319999999999999,"5min":1,"15min":1.4450000000000001},"min":{"1min":0.30499999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":5.806,"5min":6.6150000000000002,"15min":12.372},"last":0.30499999999999999},{"interface":"front","average":{"1min":1.153,"5min":1.095,"15min":1.4359999999999999},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.31},"max":{"1min":4.423,"5min":6.7640000000000002,"15min":7.3360000000000003},"last":0.50600000000000001}]},{"osd":2,"last update":"Sat Sep 23 19:33:30 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":0.85799999999999998,"15min":0.876},"min":{"1min":0.34499999999999997,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":4.915,"5min":4.915,"15min":4.915},"last":3.5800000000000001},{"interface":"front","average":{"1min":0.83199999999999996,"5min":0.84999999999999998,"15min":0.88200000000000001},"min":{"1min":0.35999999999999999,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":2.9359999999999999,"5min":2.9359999999999999,"15min":2.9359999999999999},"last":0.48399999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:33:21 2023","interfaces":[{"interface":"back","average":{"1min":2.2080000000000002,"5min":1.6990000000000001,"15min":1.216},"min":{"1min":0.378,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":9.3780000000000001,"5min":49.515000000000001,"15min":49.515000000000001},"last":2.137},{"interface":"front","average":{"1min":1.4119999999999999,"5min":1.554,"15min":1.159},"min":{"1min":0.378,"5min":0.374,"15min":0.374},"max":{"1min":8.3309999999999995,"5min":51.518999999999998,"15min":51.518999999999998},"last":0.52500000000000002}]},{"osd":5,"last update":"Sat Sep 23 19:33:48 2023","interfaces":[{"interface":"back","average":{"1min":1.107,"5min":1.1799999999999999,"15min":1.2410000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":4.4790000000000001,"5min":6.2539999999999996,"15min":12.459},"last":8.1609999999999996},{"interface":"front","average":{"1min":1.032,"5min":1.133,"15min":1.2130000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":3.2490000000000001,"5min":5.3639999999999999,"15min":12.478},"last":8.1120000000000001}]},{"osd":6,"last update":"Sat Sep 23 19:34:09 2023","interfaces":[{"interface":"back","average":{"1min":1.1990000000000001,"5min":1.1220000000000001,"15min":1.284},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.29999999999999999},"max":{"1min":4.8070000000000004,"5min":6.4610000000000003,"15min":7.0810000000000004},"last":0.55700000000000005},{"interface":"front","average":{"1min":1.4019999999999999,"5min":1.103,"15min":1.214},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":4.8609999999999998,"5min":4.8609999999999998,"15min":6.9050000000000002},"last":0.54800000000000004}]},{"osd":7,"last update":"Sat Sep 23 19:34:09 2023","interfaces":[{"interface":"back","average":{"1min":1.218,"5min":1.171,"15min":1.24},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":5.0570000000000004,"5min":5.0570000000000004,"15min":7.8479999999999999},"last":3.766},{"interface":"front","average":{"1min":1.3939999999999999,"5min":1.3300000000000001,"15min":1.214},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.36699999999999999},"max":{"1min":6.569,"5min":6.569,"15min":6.569},"last":3.6949999999999998}]}]},{"osd":5,"up_from":36,"seq":154618823049,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2932492,"kb_used_data":2033796,"kb_used_omap":0,"kb_used_meta":898688,"kb_avail":90812660,"statfs":{"total":95995035648,"available":92992163840,"internally_reserved":0,"allocated":2082607104,"data_stored":2082158761,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920256512},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2,1,3,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:33:40 2023","interfaces":[{"interface":"back","average":{"1min":1.0169999999999999,"5min":0.90800000000000003,"15min":1.246},"min":{"1min":0.26800000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":3.665,"5min":3.8119999999999998,"15min":11.058},"last":0.311},{"interface":"front","average":{"1min":2.0699999999999998,"5min":1.238,"15min":1.5309999999999999},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":18.731000000000002,"5min":18.731000000000002,"15min":18.731000000000002},"last":1.5429999999999999}]},{"osd":1,"last update":"Sat Sep 23 19:34:07 2023","interfaces":[{"interface":"back","average":{"1min":1.1679999999999999,"5min":1.044,"15min":1.2190000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.27200000000000002},"max":{"1min":4.2439999999999998,"5min":4.2439999999999998,"15min":9.5389999999999997},"last":0.34000000000000002},{"interface":"front","average":{"1min":1.9870000000000001,"5min":1.4630000000000001,"15min":1.847},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.26700000000000002},"max":{"1min":18.765000000000001,"5min":18.765000000000001,"15min":18.765000000000001},"last":1.5760000000000001}]},{"osd":2,"last update":"Sat Sep 23 19:33:40 2023","interfaces":[{"interface":"back","average":{"1min":1.9099999999999999,"5min":1.1599999999999999,"15min":1.026},"min":{"1min":0.34000000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":10.164,"5min":10.164,"15min":10.164},"last":0.57799999999999996},{"interface":"front","average":{"1min":1.1240000000000001,"5min":0.998,"15min":1.0309999999999999},"min":{"1min":0.41399999999999998,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":9.0050000000000008,"5min":9.0050000000000008,"15min":9.0050000000000008},"last":0.36499999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:34:15 2023","interfaces":[{"interface":"back","average":{"1min":1.9339999999999999,"5min":1.3120000000000001,"15min":1.145},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":8.5020000000000007,"5min":8.5020000000000007,"15min":8.5020000000000007},"last":1.1020000000000001},{"interface":"front","average":{"1min":1.014,"5min":1.1080000000000001,"15min":1.1519999999999999},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":3.2080000000000002,"5min":4.4169999999999998,"15min":4.4169999999999998},"last":0.38500000000000001}]},{"osd":4,"last update":"Sat Sep 23 19:33:40 2023","interfaces":[{"interface":"back","average":{"1min":2.234,"5min":1.218,"15min":1.141},"min":{"1min":0.379,"5min":0.33500000000000002,"15min":0.29199999999999998},"max":{"1min":18.683,"5min":18.683,"15min":18.683},"last":1.504},{"interface":"front","average":{"1min":1.415,"5min":1.048,"15min":1.101},"min":{"1min":0.38200000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":7.4950000000000001,"5min":7.4950000000000001,"15min":13.714},"last":0.65900000000000003}]},{"osd":6,"last update":"Sat Sep 23 19:33:25 2023","interfaces":[{"interface":"back","average":{"1min":1.0369999999999999,"5min":1.044,"15min":1.161},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":3.7240000000000002,"5min":3.7240000000000002,"15min":26.448},"last":0.40799999999999997},{"interface":"front","average":{"1min":1.593,"5min":1.224,"15min":1.2769999999999999},"min":{"1min":0.48899999999999999,"5min":0.378,"15min":0.378},"max":{"1min":7.5750000000000002,"5min":7.5750000000000002,"15min":26.841999999999999},"last":0.63}]},{"osd":7,"last update":"Sat Sep 23 19:33:53 2023","interfaces":[{"interface":"back","average":{"1min":1.8560000000000001,"5min":1.4319999999999999,"15min":1.274},"min":{"1min":0.42199999999999999,"5min":0.39800000000000002,"15min":0.37},"max":{"1min":7.0739999999999998,"5min":7.6280000000000001,"15min":7.6280000000000001},"last":0.72299999999999998},{"interface":"front","average":{"1min":2.6589999999999998,"5min":1.544,"15min":1.2350000000000001},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":9.6690000000000005,"5min":13.18,"15min":13.18},"last":0.749}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":890175488,"data_stored":890175488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2023493632,"data_stored":2023489560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":659161088,"data_stored":659161088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1795883008,"data_stored":1795883008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":696516608,"data_stored":696516608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2062946304,"data_stored":2062942232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":686030848,"data_stored":686030848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2076774400,"data_stored":2076770328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T19:34:21.391 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T19:34:21.392 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd pool set device_health_metrics pg_num 21 2023-09-23T19:34:21.413 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:21.479 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:21.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:21.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:21.588 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:21.618 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:21.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:21.621 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:21.722 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:21.740 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:21 smithi016 bash[75004]: cluster 2023-09-23T19:34:20.017714+0000 mgr.y (mgr.25325) 803 : cluster 0 pgmap v707: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2023-09-23T19:34:21.764 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:21.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:21.769 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:21.806 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:34:21.870 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:21.878 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:21 smithi092 bash[58855]: cluster 2023-09-23T19:34:20.017714+0000 mgr.y (mgr.25325) 803 : cluster 0 pgmap v707: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2023-09-23T19:34:21.908 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:21.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:21.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:22.013 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:22.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:21 smithi016 bash[77995]: cluster 2023-09-23T19:34:20.017714+0000 mgr.y (mgr.25325) 803 : cluster 0 pgmap v707: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2023-09-23T19:34:22.042 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:22.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:22.045 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:22.145 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:22.180 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:22.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:22.189 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:22.290 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:22.316 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:22.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:22.320 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:22.421 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:22.454 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:22.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:22.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:22.557 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:22.599 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:22.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:22.604 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:22.705 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:22.755 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:22.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:22.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:34:22.861 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:22.894 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:22.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:22.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:34:22.997 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:23.032 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:23.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:23.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:23.137 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:23.177 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:23.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:23.180 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:23.280 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:23.315 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:23.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:23.323 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:34:23.423 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:23.594 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:23.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:23.597 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:34:23.697 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:23.724 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:23.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:23.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:34:23.744 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:23 smithi016 bash[77995]: cluster 2023-09-23T19:34:22.018402+0000 mgr.y (mgr.25325) 804 : cluster 0 pgmap v708: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 741 op/s 2023-09-23T19:34:23.828 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:23.837 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:23 smithi016 bash[75004]: cluster 2023-09-23T19:34:22.018402+0000 mgr.y (mgr.25325) 804 : cluster 0 pgmap v708: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 741 op/s 2023-09-23T19:34:23.854 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:23.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:23.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:23.964 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:23.977 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:23 smithi092 bash[58855]: cluster 2023-09-23T19:34:22.018402+0000 mgr.y (mgr.25325) 804 : cluster 0 pgmap v708: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 741 op/s 2023-09-23T19:34:24.010 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:24.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:24.039 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:24.139 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:24.174 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:24.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:24.179 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:24.279 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:24.315 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:24.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:24.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:24.421 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:24.455 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:24.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:24.459 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:34:24.560 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:24.570 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:34:24.582 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:24.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:24.585 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:24.685 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:24.804 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:24.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:24.805 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:24.906 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:24.941 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:24.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:24.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:25.046 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:25.069 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:25.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:25.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:25.173 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:25.198 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:25.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:25.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:25.301 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:25.308 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:25 smithi092 bash[58855]: cluster 2023-09-23T19:34:24.019009+0000 mgr.y (mgr.25325) 805 : cluster 0 pgmap v709: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 627 op/s 2023-09-23T19:34:25.308 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:25 smithi092 bash[58855]: audit 2023-09-23T19:34:24.292402+0000 mon.a (mon.0) 991 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:34:25.308 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:25 smithi092 bash[58855]: audit 2023-09-23T19:34:24.300640+0000 mon.a (mon.0) 992 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:34:25.324 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:25.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:25.328 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:25.429 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:25.440 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:25 smithi016 bash[77995]: cluster 2023-09-23T19:34:24.019009+0000 mgr.y (mgr.25325) 805 : cluster 0 pgmap v709: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 627 op/s 2023-09-23T19:34:25.440 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:25 smithi016 bash[77995]: audit 2023-09-23T19:34:24.292402+0000 mon.a (mon.0) 991 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:34:25.440 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:25 smithi016 bash[77995]: audit 2023-09-23T19:34:24.300640+0000 mon.a (mon.0) 992 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:34:25.440 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:25 smithi016 bash[75004]: cluster 2023-09-23T19:34:24.019009+0000 mgr.y (mgr.25325) 805 : cluster 0 pgmap v709: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 627 op/s 2023-09-23T19:34:25.440 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:25 smithi016 bash[75004]: audit 2023-09-23T19:34:24.292402+0000 mon.a (mon.0) 991 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:34:25.440 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:25 smithi016 bash[75004]: audit 2023-09-23T19:34:24.300640+0000 mon.a (mon.0) 992 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:34:25.462 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:25.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:25.467 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:25.567 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:25.594 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:25.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:25.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:25.698 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:25.708 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:34:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:34:25] "GET /metrics HTTP/1.1" 200 36239 "" "Prometheus/2.33.4" 2023-09-23T19:34:25.733 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:25.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:25.737 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:25.838 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:25.866 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:25.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:25.869 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:34:25.970 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:26.024 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:26.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:26.031 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:34:26.132 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:26.178 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:26.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:26.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:26.281 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:26.302 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:26.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:26.309 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:34:26.409 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:26.441 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:26.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:26.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:26.549 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:26.581 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:26.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:26.585 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:26.686 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:26.780 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:26.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:26.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:26.883 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:26.909 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:26.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:26.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:34:27.012 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:27.038 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:27.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:27.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:27.142 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:27.190 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:27.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:27.194 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:27.294 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:27.325 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:27.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:27.329 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:27.369 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:27 smithi016 bash[77995]: cluster 2023-09-23T19:34:26.019583+0000 mgr.y (mgr.25325) 806 : cluster 0 pgmap v710: 89 pgs: 89 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2023-09-23T19:34:27.369 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:27 smithi016 bash[77995]: audit 2023-09-23T19:34:26.364673+0000 mon.a (mon.0) 993 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:34:27.370 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:27 smithi016 bash[77995]: audit 2023-09-23T19:34:26.372008+0000 mon.a (mon.0) 994 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:34:27.370 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:27 smithi016 bash[77995]: audit 2023-09-23T19:34:27.012198+0000 mon.a (mon.0) 995 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:34:27.370 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:27 smithi016 bash[77995]: audit 2023-09-23T19:34:27.018986+0000 mon.a (mon.0) 996 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:34:27.370 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:27 smithi016 bash[77995]: audit 2023-09-23T19:34:27.025533+0000 mon.a (mon.0) 997 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:34:27.370 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:27 smithi016 bash[77995]: audit 2023-09-23T19:34:27.053741+0000 mon.c (mon.2) 150 : audit 1 from='client.? 172.21.15.16:0/1403466342' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:34:27.370 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:27 smithi016 bash[77995]: audit 2023-09-23T19:34:27.054198+0000 mon.a (mon.0) 998 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:34:27.370 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:27 smithi016 bash[75004]: cluster 2023-09-23T19:34:26.019583+0000 mgr.y (mgr.25325) 806 : cluster 0 pgmap v710: 89 pgs: 89 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2023-09-23T19:34:27.370 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:27 smithi016 bash[75004]: audit 2023-09-23T19:34:26.364673+0000 mon.a (mon.0) 993 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:34:27.370 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:27 smithi016 bash[75004]: audit 2023-09-23T19:34:26.372008+0000 mon.a (mon.0) 994 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:34:27.371 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:27 smithi016 bash[75004]: audit 2023-09-23T19:34:27.012198+0000 mon.a (mon.0) 995 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:34:27.371 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:27 smithi016 bash[75004]: audit 2023-09-23T19:34:27.018986+0000 mon.a (mon.0) 996 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:34:27.371 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:27 smithi016 bash[75004]: audit 2023-09-23T19:34:27.025533+0000 mon.a (mon.0) 997 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:34:27.371 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:27 smithi016 bash[75004]: audit 2023-09-23T19:34:27.053741+0000 mon.c (mon.2) 150 : audit 1 from='client.? 172.21.15.16:0/1403466342' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:34:27.371 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:27 smithi016 bash[75004]: audit 2023-09-23T19:34:27.054198+0000 mon.a (mon.0) 998 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:34:27.429 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:27.457 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:27.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:27.464 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:27.565 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:27.589 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:27.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:27.592 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:34:27.692 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:27.702 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:27 smithi092 bash[58855]: cluster 2023-09-23T19:34:26.019583+0000 mgr.y (mgr.25325) 806 : cluster 0 pgmap v710: 89 pgs: 89 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2023-09-23T19:34:27.702 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:27 smithi092 bash[58855]: audit 2023-09-23T19:34:26.364673+0000 mon.a (mon.0) 993 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:34:27.702 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:27 smithi092 bash[58855]: audit 2023-09-23T19:34:26.372008+0000 mon.a (mon.0) 994 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:34:27.702 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:27 smithi092 bash[58855]: audit 2023-09-23T19:34:27.012198+0000 mon.a (mon.0) 995 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:34:27.702 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:27 smithi092 bash[58855]: audit 2023-09-23T19:34:27.018986+0000 mon.a (mon.0) 996 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:34:27.702 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:27 smithi092 bash[58855]: audit 2023-09-23T19:34:27.025533+0000 mon.a (mon.0) 997 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:34:27.702 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:27 smithi092 bash[58855]: audit 2023-09-23T19:34:27.053741+0000 mon.c (mon.2) 150 : audit 1 from='client.? 172.21.15.16:0/1403466342' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:34:27.702 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:27 smithi092 bash[58855]: audit 2023-09-23T19:34:27.054198+0000 mon.a (mon.0) 998 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:34:27.720 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:27.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:27.722 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:27.823 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:27.858 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:27.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:27.861 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:34:27.961 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:27.984 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:27.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:27.986 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:28.041 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is set 2023-09-23T19:34:28.087 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:28.118 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:28.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:28.122 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:28.222 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:28.252 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:28.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:28.256 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:28.323 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:34:28.324 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:34:28.357 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:28.396 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:28.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:28.404 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:34:28.504 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:28.538 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:28.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:28.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:34:28.641 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:28.680 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:28.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:28.685 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:34:28.785 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:28.810 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:28.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:28.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:28.913 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:28.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:28.946 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:28.946 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:29.046 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:29.055 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:29 smithi016 bash[75004]: cluster 2023-09-23T19:34:28.020182+0000 mgr.y (mgr.25325) 807 : cluster 0 pgmap v711: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 664 op/s 2023-09-23T19:34:29.056 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:29 smithi016 bash[75004]: cluster 2023-09-23T19:34:28.022964+0000 mon.a (mon.0) 999 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:34:29.056 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:29 smithi016 bash[75004]: audit 2023-09-23T19:34:28.032604+0000 mon.a (mon.0) 1000 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:34:29.056 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:29 smithi016 bash[75004]: cluster 2023-09-23T19:34:28.032750+0000 mon.a (mon.0) 1001 : cluster 0 osdmap e132: 8 total, 8 up, 8 in 2023-09-23T19:34:29.056 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:29 smithi016 bash[75004]: cluster 2023-09-23T19:34:28.040190+0000 osd.7 (osd.7) 51 : cluster 3 failed to encode map e132 with expected crc 2023-09-23T19:34:29.056 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:29 smithi016 bash[75004]: cluster 2023-09-23T19:34:28.041186+0000 osd.6 (osd.6) 52 : cluster 3 failed to encode map e132 with expected crc 2023-09-23T19:34:29.056 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:29 smithi016 bash[75004]: cluster 2023-09-23T19:34:28.041545+0000 osd.4 (osd.4) 60 : cluster 3 failed to encode map e132 with expected crc 2023-09-23T19:34:29.056 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:29 smithi016 bash[75004]: cluster 2023-09-23T19:34:28.048785+0000 osd.5 (osd.5) 62 : cluster 3 failed to encode map e132 with expected crc 2023-09-23T19:34:29.056 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:29 smithi016 bash[75004]: cluster 2023-09-23T19:34:28.050537+0000 osd.5 (osd.5) 63 : cluster 3 failed to encode map e132 with expected crc 2023-09-23T19:34:29.056 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:29 smithi016 bash[75004]: audit 2023-09-23T19:34:28.641221+0000 mon.a (mon.0) 1002 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:34:29.057 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:29 smithi016 bash[77995]: cluster 2023-09-23T19:34:28.020182+0000 mgr.y (mgr.25325) 807 : cluster 0 pgmap v711: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 664 op/s 2023-09-23T19:34:29.057 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:29 smithi016 bash[77995]: cluster 2023-09-23T19:34:28.022964+0000 mon.a (mon.0) 999 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:34:29.057 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:29 smithi016 bash[77995]: audit 2023-09-23T19:34:28.032604+0000 mon.a (mon.0) 1000 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:34:29.057 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:29 smithi016 bash[77995]: cluster 2023-09-23T19:34:28.032750+0000 mon.a (mon.0) 1001 : cluster 0 osdmap e132: 8 total, 8 up, 8 in 2023-09-23T19:34:29.057 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:29 smithi016 bash[77995]: cluster 2023-09-23T19:34:28.040190+0000 osd.7 (osd.7) 51 : cluster 3 failed to encode map e132 with expected crc 2023-09-23T19:34:29.057 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:29 smithi016 bash[77995]: cluster 2023-09-23T19:34:28.041186+0000 osd.6 (osd.6) 52 : cluster 3 failed to encode map e132 with expected crc 2023-09-23T19:34:29.057 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:29 smithi016 bash[77995]: cluster 2023-09-23T19:34:28.041545+0000 osd.4 (osd.4) 60 : cluster 3 failed to encode map e132 with expected crc 2023-09-23T19:34:29.057 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:29 smithi016 bash[77995]: cluster 2023-09-23T19:34:28.048785+0000 osd.5 (osd.5) 62 : cluster 3 failed to encode map e132 with expected crc 2023-09-23T19:34:29.057 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:29 smithi016 bash[77995]: cluster 2023-09-23T19:34:28.050537+0000 osd.5 (osd.5) 63 : cluster 3 failed to encode map e132 with expected crc 2023-09-23T19:34:29.057 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:29 smithi016 bash[77995]: audit 2023-09-23T19:34:28.641221+0000 mon.a (mon.0) 1002 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:34:29.071 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:29.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:29.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:34:29.177 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:29.187 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:29 smithi092 bash[58855]: cluster 2023-09-23T19:34:28.020182+0000 mgr.y (mgr.25325) 807 : cluster 0 pgmap v711: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 664 op/s 2023-09-23T19:34:29.187 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:29 smithi092 bash[58855]: cluster 2023-09-23T19:34:28.022964+0000 mon.a (mon.0) 999 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:34:29.187 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:29 smithi092 bash[58855]: audit 2023-09-23T19:34:28.032604+0000 mon.a (mon.0) 1000 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:34:29.187 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:29 smithi092 bash[58855]: cluster 2023-09-23T19:34:28.032750+0000 mon.a (mon.0) 1001 : cluster 0 osdmap e132: 8 total, 8 up, 8 in 2023-09-23T19:34:29.187 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:29 smithi092 bash[58855]: cluster 2023-09-23T19:34:28.040190+0000 osd.7 (osd.7) 51 : cluster 3 failed to encode map e132 with expected crc 2023-09-23T19:34:29.187 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:29 smithi092 bash[58855]: cluster 2023-09-23T19:34:28.041186+0000 osd.6 (osd.6) 52 : cluster 3 failed to encode map e132 with expected crc 2023-09-23T19:34:29.187 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:29 smithi092 bash[58855]: cluster 2023-09-23T19:34:28.041545+0000 osd.4 (osd.4) 60 : cluster 3 failed to encode map e132 with expected crc 2023-09-23T19:34:29.188 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:29 smithi092 bash[58855]: cluster 2023-09-23T19:34:28.048785+0000 osd.5 (osd.5) 62 : cluster 3 failed to encode map e132 with expected crc 2023-09-23T19:34:29.188 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:29 smithi092 bash[58855]: cluster 2023-09-23T19:34:28.050537+0000 osd.5 (osd.5) 63 : cluster 3 failed to encode map e132 with expected crc 2023-09-23T19:34:29.188 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:29 smithi092 bash[58855]: audit 2023-09-23T19:34:28.641221+0000 mon.a (mon.0) 1002 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:34:29.217 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:29.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:29.218 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:34:29.319 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:29.369 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:29.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:29.373 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:29.473 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:29.499 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:29.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:29.504 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:29.604 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:29.637 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:29.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:29.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:34:29.744 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:29.771 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:29.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:29.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:34:29.882 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:29.911 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:29.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:29.917 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:30.018 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:30.041 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:30.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:30.044 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:30.145 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:30.181 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:30.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:30.183 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:34:30.284 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:30.309 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:30.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:30.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:34:30.412 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:30.445 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:30.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:30.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:34:30.550 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:30.575 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:30.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:30.582 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:34:30.682 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:30.711 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:30.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:30.717 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:34:30.818 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:30.846 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:30.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:30.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:30.950 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:30.978 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:30.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:30.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:31.081 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:31.089 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:34:31.107 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:31.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:31.110 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:34:31.137 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:34:31.185 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:34:31.210 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:31.242 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:34:31.242 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:31.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:31.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:31.277 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:34:31.318 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:34:31.350 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:31 smithi016 bash[77995]: cluster 2023-09-23T19:34:30.020782+0000 mgr.y (mgr.25325) 808 : cluster 0 pgmap v713: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 649 op/s 2023-09-23T19:34:31.350 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:31 smithi016 bash[77995]: audit 2023-09-23T19:34:31.024521+0000 mon.a (mon.0) 1003 : audit 0 from='client.? 172.21.15.16:0/4034133442' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:34:31.350 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:31 smithi016 bash[75004]: cluster 2023-09-23T19:34:30.020782+0000 mgr.y (mgr.25325) 808 : cluster 0 pgmap v713: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 649 op/s 2023-09-23T19:34:31.351 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:31 smithi016 bash[75004]: audit 2023-09-23T19:34:31.024521+0000 mon.a (mon.0) 1003 : audit 0 from='client.? 172.21.15.16:0/4034133442' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:34:31.352 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:31.357 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:34:31.381 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:31.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:31.386 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:31.405 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:34:31.412 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:34:31.412 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:34:31.412 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:34:31.412 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:34:31.413 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:34:31.413 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:34:31.413 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:34:31.413 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:34:31.486 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:31.619 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:31 smithi092 bash[58855]: cluster 2023-09-23T19:34:30.020782+0000 mgr.y (mgr.25325) 808 : cluster 0 pgmap v713: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 649 op/s 2023-09-23T19:34:31.619 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:31 smithi092 bash[58855]: audit 2023-09-23T19:34:31.024521+0000 mon.a (mon.0) 1003 : audit 0 from='client.? 172.21.15.16:0/4034133442' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:34:31.641 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:31.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:31.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:31.745 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:31.770 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:31.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:31.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:31.876 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:31.919 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:31.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:31.925 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:32.026 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:32.060 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:32.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:32.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:32.165 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:32.201 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:32.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:32.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:32.311 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:32.352 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:32.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:32.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:34:32.458 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:32.485 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:32.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:32.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:32.500 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T19:34:32.594 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:32.634 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:32.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:32.637 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:32.738 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:32.771 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:32.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:32.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:34:32.878 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:32.903 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:32.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:32.909 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:33.010 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:33.056 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:33.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:33.059 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:33.160 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:33.186 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:33.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:33.189 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:33.289 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:33.319 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:33.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:33.325 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:33.426 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:33.458 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:33.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:33.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:34:33.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:33 smithi092 bash[58855]: cluster 2023-09-23T19:34:32.021428+0000 mgr.y (mgr.25325) 809 : cluster 0 pgmap v714: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-09-23T19:34:33.563 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:33.603 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:33.605 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:33 smithi016 bash[75004]: cluster 2023-09-23T19:34:32.021428+0000 mgr.y (mgr.25325) 809 : cluster 0 pgmap v714: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-09-23T19:34:33.605 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:33 smithi016 bash[77995]: cluster 2023-09-23T19:34:32.021428+0000 mgr.y (mgr.25325) 809 : cluster 0 pgmap v714: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-09-23T19:34:33.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:33.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:33.707 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:33.752 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:33.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:33.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:33.861 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:33.896 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:33.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:33.900 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:34:34.000 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:34.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:34.034 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:34.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:34.135 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:34.173 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:34.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:34.177 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:34:34.277 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:34.300 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:34.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:34.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:34:34.403 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:34.432 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:34.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:34.438 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:34.539 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:34.565 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:34.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:34.567 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:34.667 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:34.698 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:34.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:34.701 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:34.802 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:34.838 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:34.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:34.841 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:34.941 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:34.966 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:34.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:34.971 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:34:35.071 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:35.100 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:35.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:35.105 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:34:35.206 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:35.256 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:35.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:35.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:35.355 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:35 smithi016 bash[75004]: cluster 2023-09-23T19:34:34.021933+0000 mgr.y (mgr.25325) 810 : cluster 0 pgmap v715: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 655 op/s 2023-09-23T19:34:35.355 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:35 smithi016 bash[77995]: cluster 2023-09-23T19:34:34.021933+0000 mgr.y (mgr.25325) 810 : cluster 0 pgmap v715: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 655 op/s 2023-09-23T19:34:35.359 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:35.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:35.395 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:35.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:34:35.470 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:34:35.496 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:35.505 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:35 smithi092 bash[58855]: cluster 2023-09-23T19:34:34.021933+0000 mgr.y (mgr.25325) 810 : cluster 0 pgmap v715: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 655 op/s 2023-09-23T19:34:35.521 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:35.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:35.523 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:35.623 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:35.641 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:34:35 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:34:35] "GET /metrics HTTP/1.1" 200 36234 "" "Prometheus/2.33.4" 2023-09-23T19:34:35.646 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:35.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:35.648 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:35.702 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-09-23T19:34:35.702 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:34:35.749 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:35.778 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:35.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:35.781 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:35.881 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:35.908 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:35.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:35.910 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:34:36.011 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:36.046 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:36.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:36.049 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:36.087 INFO:teuthology.orchestra.run.smithi016.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-09-23T19:34:36.150 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:36.174 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:36.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:36.176 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:34:36.277 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:36.323 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:36.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:36.329 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:36.430 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:36.439 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:36 smithi016 bash[75004]: audit 2023-09-23T19:34:36.078353+0000 mon.a (mon.0) 1004 : audit 1 from='client.? 172.21.15.16:0/1672741576' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "21"}]: dispatch 2023-09-23T19:34:36.439 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:36 smithi016 bash[77995]: audit 2023-09-23T19:34:36.078353+0000 mon.a (mon.0) 1004 : audit 1 from='client.? 172.21.15.16:0/1672741576' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "21"}]: dispatch 2023-09-23T19:34:36.465 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:36.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:36.470 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:36.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:36 smithi092 bash[58855]: audit 2023-09-23T19:34:36.078353+0000 mon.a (mon.0) 1004 : audit 1 from='client.? 172.21.15.16:0/1672741576' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "21"}]: dispatch 2023-09-23T19:34:36.571 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:36.601 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:36.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:36.604 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:36.705 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:36.743 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:36.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:36.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:36.849 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:36.879 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:36.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:36.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:36.985 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:37.009 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:37.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:37.012 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:37.049 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T19:34:37.112 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:37.159 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:37.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:37.161 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:37.262 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:37.286 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:37.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:37.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:37.390 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:37.400 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:37 smithi092 bash[58855]: cluster 2023-09-23T19:34:36.022572+0000 mgr.y (mgr.25325) 811 : cluster 0 pgmap v716: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2023-09-23T19:34:37.418 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:37.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:37.420 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:37.521 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:37.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:37 smithi016 bash[75004]: cluster 2023-09-23T19:34:36.022572+0000 mgr.y (mgr.25325) 811 : cluster 0 pgmap v716: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2023-09-23T19:34:37.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:37 smithi016 bash[77995]: cluster 2023-09-23T19:34:36.022572+0000 mgr.y (mgr.25325) 811 : cluster 0 pgmap v716: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2023-09-23T19:34:37.556 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:37.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:37.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:37.632 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-09-23T19:34:37.660 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:37.704 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:37.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:37.709 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:37.810 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:37.842 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:37.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:37.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:34:37.945 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:37.975 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:37.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:37.979 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:38.079 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:38.108 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:38.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:38.110 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:34:38.210 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:38.239 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:38.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:38.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:34:38.344 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:38.372 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:38.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:38.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:38.480 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:38.517 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:38.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:38.522 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:38.623 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:38.654 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:38.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:38.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:38.758 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:38.792 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:38.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:38.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:34:38.900 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:38.936 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:38.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:38.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:39.042 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:39.070 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:39.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:39.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:34:39.118 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:34:39.175 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:39.203 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:39.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:39.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:34:39.307 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:39.342 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:39.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:39.349 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:34:39.449 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:39.481 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:39.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:39.487 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:34:39.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:39 smithi016 bash[75004]: cluster 2023-09-23T19:34:38.023225+0000 mgr.y (mgr.25325) 812 : cluster 0 pgmap v717: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 680 op/s 2023-09-23T19:34:39.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:39 smithi016 bash[77995]: cluster 2023-09-23T19:34:38.023225+0000 mgr.y (mgr.25325) 812 : cluster 0 pgmap v717: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 680 op/s 2023-09-23T19:34:39.589 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:39.619 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:39.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:39.622 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:39.622 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:39 smithi092 bash[58855]: cluster 2023-09-23T19:34:38.023225+0000 mgr.y (mgr.25325) 812 : cluster 0 pgmap v717: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 680 op/s 2023-09-23T19:34:39.723 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:39.744 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:39.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:39.749 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:34:39.850 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:39.879 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:39.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:39.884 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:39.984 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:40.018 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:40.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:40.021 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:40.121 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:40.146 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:40.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:40.154 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:40.255 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:40.281 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:40.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:40.286 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:34:40.387 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:40.425 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:40.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:40.427 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:40.528 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:40.555 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:40.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:40.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:40.658 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:40.686 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:40.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:40.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:40.790 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:40.825 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:40.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:40.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:34:40.932 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:40.961 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:40.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:40.965 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:41.066 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:41.093 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:41.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:41.099 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:34:41.200 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:41.239 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:41.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:41.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:41.346 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:41.371 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:41 smithi016 bash[75004]: cluster 2023-09-23T19:34:40.023804+0000 mgr.y (mgr.25325) 813 : cluster 0 pgmap v718: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 601 op/s 2023-09-23T19:34:41.381 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:41.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:41.388 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:34:41.489 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:41.522 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:41.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:41.526 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:41.626 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:41.635 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:41 smithi092 bash[58855]: cluster 2023-09-23T19:34:40.023804+0000 mgr.y (mgr.25325) 813 : cluster 0 pgmap v718: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 601 op/s 2023-09-23T19:34:41.669 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:41.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:41.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:41.773 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:41.782 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:41 smithi016 bash[77995]: cluster 2023-09-23T19:34:40.023804+0000 mgr.y (mgr.25325) 813 : cluster 0 pgmap v718: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 601 op/s 2023-09-23T19:34:41.808 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:41.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:41.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:41.912 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:41.932 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:34:41.940 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:34:41.952 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:41.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:41.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:34:42.057 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:42.087 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:42.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:42.089 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:34:42.190 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:42.229 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:42.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:42.232 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:42.332 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:42.361 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:42.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:42.367 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:34:42.468 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:42.501 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:42.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:42.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:34:42.609 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:42.632 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T19:34:42.633 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T19:34:42.640 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:42.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:42.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:42.750 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:42.781 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:42.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:42.785 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:34:42.886 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:42.937 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:42.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:42.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:43.042 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:43.069 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:43.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:43.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:43.172 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:43.211 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:43.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:43.218 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:43.319 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:43.337 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:43.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:43.342 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:43.443 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:43.467 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:43.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:43.471 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:43.571 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:43.596 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:43.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:43.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:43.703 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:43.722 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:43 smithi092 bash[58855]: cluster 2023-09-23T19:34:42.024467+0000 mgr.y (mgr.25325) 814 : cluster 0 pgmap v719: 89 pgs: 89 active+clean; 2.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 747 op/s 2023-09-23T19:34:43.735 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:43.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:43.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:43.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:43 smithi016 bash[75004]: cluster 2023-09-23T19:34:42.024467+0000 mgr.y (mgr.25325) 814 : cluster 0 pgmap v719: 89 pgs: 89 active+clean; 2.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 747 op/s 2023-09-23T19:34:43.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:43 smithi016 bash[77995]: cluster 2023-09-23T19:34:42.024467+0000 mgr.y (mgr.25325) 814 : cluster 0 pgmap v719: 89 pgs: 89 active+clean; 2.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 747 op/s 2023-09-23T19:34:43.842 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:43.880 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:43.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:43.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:43.982 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:44.003 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:44.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:44.008 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:34:44.108 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:44.141 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:44.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:44.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:34:44.247 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:44.279 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:44.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:44.285 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:44.386 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:44.388 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:44 smithi092 bash[58855]: audit 2023-09-23T19:34:43.641292+0000 mon.a (mon.0) 1005 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:34:44.413 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:44.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:44.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:34:44.517 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:44.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:44 smithi016 bash[77995]: audit 2023-09-23T19:34:43.641292+0000 mon.a (mon.0) 1005 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:34:44.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:44 smithi016 bash[75004]: audit 2023-09-23T19:34:43.641292+0000 mon.a (mon.0) 1005 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:34:44.545 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:44.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:44.552 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:44.653 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:44.686 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:44.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:44.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:44.794 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:44.817 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:44.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:44.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:44.922 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:44.958 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:44.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:44.965 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:45.066 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:45.100 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:45.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:45.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:45.203 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:45.232 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:45.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:45.237 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:34:45.337 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:45.374 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:45.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:45.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:45.405 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is unset 2023-09-23T19:34:45.478 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:45.510 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:45.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:45.518 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:45.619 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:45.657 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:45.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:45.661 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:45.762 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:45.786 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:45.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:45.789 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:34:45.792 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:45 smithi016 bash[75004]: cluster 2023-09-23T19:34:44.025091+0000 mgr.y (mgr.25325) 815 : cluster 0 pgmap v720: 89 pgs: 89 active+clean; 2.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2023-09-23T19:34:45.792 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:45 smithi016 bash[75004]: audit 2023-09-23T19:34:44.492877+0000 mon.a (mon.0) 1006 : audit 1 from='client.? 172.21.15.16:0/747211833' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:34:45.792 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:45 smithi016 bash[77995]: cluster 2023-09-23T19:34:44.025091+0000 mgr.y (mgr.25325) 815 : cluster 0 pgmap v720: 89 pgs: 89 active+clean; 2.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2023-09-23T19:34:45.792 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:45 smithi016 bash[77995]: audit 2023-09-23T19:34:44.492877+0000 mon.a (mon.0) 1006 : audit 1 from='client.? 172.21.15.16:0/747211833' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:34:45.801 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:45 smithi092 bash[58855]: cluster 2023-09-23T19:34:44.025091+0000 mgr.y (mgr.25325) 815 : cluster 0 pgmap v720: 89 pgs: 89 active+clean; 2.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2023-09-23T19:34:45.801 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:45 smithi092 bash[58855]: audit 2023-09-23T19:34:44.492877+0000 mon.a (mon.0) 1006 : audit 1 from='client.? 172.21.15.16:0/747211833' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:34:45.889 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:45.913 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:45.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:45.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:34:46.017 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:46.045 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:46.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:46.049 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:46.105 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:34:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:34:45] "GET /metrics HTTP/1.1" 200 36238 "" "Prometheus/2.33.4" 2023-09-23T19:34:46.150 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:46.177 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:46.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:46.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:46.280 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:46.305 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:46.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:46.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:34:46.394 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:46 smithi016 bash[75004]: cluster 2023-09-23T19:34:45.384692+0000 mon.a (mon.0) 1007 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:34:46.394 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:46 smithi016 bash[75004]: audit 2023-09-23T19:34:45.395530+0000 mon.a (mon.0) 1008 : audit 1 from='client.? 172.21.15.16:0/747211833' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:34:46.394 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:46 smithi016 bash[75004]: cluster 2023-09-23T19:34:45.395669+0000 mon.a (mon.0) 1009 : cluster 0 osdmap e133: 8 total, 8 up, 8 in 2023-09-23T19:34:46.394 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:46 smithi016 bash[75004]: cluster 2023-09-23T19:34:45.401171+0000 osd.6 (osd.6) 53 : cluster 3 failed to encode map e133 with expected crc 2023-09-23T19:34:46.394 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:46 smithi016 bash[75004]: cluster 2023-09-23T19:34:45.407236+0000 osd.5 (osd.5) 64 : cluster 3 failed to encode map e133 with expected crc 2023-09-23T19:34:46.394 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:46 smithi016 bash[75004]: cluster 2023-09-23T19:34:45.414148+0000 osd.4 (osd.4) 61 : cluster 3 failed to encode map e133 with expected crc 2023-09-23T19:34:46.409 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:46.419 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:46 smithi092 bash[58855]: cluster 2023-09-23T19:34:45.384692+0000 mon.a (mon.0) 1007 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:34:46.419 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:46 smithi092 bash[58855]: audit 2023-09-23T19:34:45.395530+0000 mon.a (mon.0) 1008 : audit 1 from='client.? 172.21.15.16:0/747211833' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:34:46.419 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:46 smithi092 bash[58855]: cluster 2023-09-23T19:34:45.395669+0000 mon.a (mon.0) 1009 : cluster 0 osdmap e133: 8 total, 8 up, 8 in 2023-09-23T19:34:46.419 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:46 smithi092 bash[58855]: cluster 2023-09-23T19:34:45.401171+0000 osd.6 (osd.6) 53 : cluster 3 failed to encode map e133 with expected crc 2023-09-23T19:34:46.419 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:46 smithi092 bash[58855]: cluster 2023-09-23T19:34:45.407236+0000 osd.5 (osd.5) 64 : cluster 3 failed to encode map e133 with expected crc 2023-09-23T19:34:46.419 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:46 smithi092 bash[58855]: cluster 2023-09-23T19:34:45.414148+0000 osd.4 (osd.4) 61 : cluster 3 failed to encode map e133 with expected crc 2023-09-23T19:34:46.445 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:46.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:46.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:46.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:46 smithi016 bash[77995]: cluster 2023-09-23T19:34:45.384692+0000 mon.a (mon.0) 1007 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:34:46.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:46 smithi016 bash[77995]: audit 2023-09-23T19:34:45.395530+0000 mon.a (mon.0) 1008 : audit 1 from='client.? 172.21.15.16:0/747211833' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:34:46.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:46 smithi016 bash[77995]: cluster 2023-09-23T19:34:45.395669+0000 mon.a (mon.0) 1009 : cluster 0 osdmap e133: 8 total, 8 up, 8 in 2023-09-23T19:34:46.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:46 smithi016 bash[77995]: cluster 2023-09-23T19:34:45.401171+0000 osd.6 (osd.6) 53 : cluster 3 failed to encode map e133 with expected crc 2023-09-23T19:34:46.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:46 smithi016 bash[77995]: cluster 2023-09-23T19:34:45.407236+0000 osd.5 (osd.5) 64 : cluster 3 failed to encode map e133 with expected crc 2023-09-23T19:34:46.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:46 smithi016 bash[77995]: cluster 2023-09-23T19:34:45.414148+0000 osd.4 (osd.4) 61 : cluster 3 failed to encode map e133 with expected crc 2023-09-23T19:34:46.551 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:46.597 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:46.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:46.601 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:46.701 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:46.736 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:46.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:46.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:34:46.801 INFO:teuthology.orchestra.run.smithi092.stdout:ERROR: (22) Invalid argument 2023-09-23T19:34:46.801 INFO:teuthology.orchestra.run.smithi092.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:34:46.842 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:46.864 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:46.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:46.869 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:34:46.969 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:47.035 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:47.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:47.041 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:47.142 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:47.172 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:47.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:47.177 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:47.277 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:47.307 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:47.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:47.311 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:34:47.408 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:47 smithi016 bash[75004]: cluster 2023-09-23T19:34:46.025530+0000 mgr.y (mgr.25325) 816 : cluster 0 pgmap v722: 89 pgs: 89 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2023-09-23T19:34:47.412 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:47.439 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:47.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:47.445 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:34:47.545 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:47.576 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:47.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:47.579 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:47.609 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:34:47.663 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:34:47.680 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:47.688 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:47 smithi092 bash[58855]: cluster 2023-09-23T19:34:46.025530+0000 mgr.y (mgr.25325) 816 : cluster 0 pgmap v722: 89 pgs: 89 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2023-09-23T19:34:47.705 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:47.707 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:34:47.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:47.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:47.757 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:34:47.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:47 smithi016 bash[77995]: cluster 2023-09-23T19:34:46.025530+0000 mgr.y (mgr.25325) 816 : cluster 0 pgmap v722: 89 pgs: 89 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2023-09-23T19:34:47.792 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:34:47.809 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:47.822 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T19:34:47.833 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:34:47.850 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:47.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:47.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:34:47.880 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:34:47.926 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:34:47.930 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:34:47.930 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:34:47.930 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:34:47.930 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:34:47.930 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:34:47.930 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:34:47.930 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:34:47.930 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:34:47.953 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:47.984 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:47.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:47.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:34:48.087 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:48.117 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:48.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:48.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:48.222 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:48.251 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:48.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:48.259 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:48.360 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:48.385 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:48.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:48.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:48.492 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:48.523 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:48.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:48.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:48.628 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:48.666 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:48.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:48.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:48.671 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:48 smithi016 bash[77995]: audit 2023-09-23T19:34:47.558492+0000 mon.a (mon.0) 1010 : audit 0 from='client.? 172.21.15.16:0/1517743227' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:34:48.671 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:48 smithi016 bash[75004]: audit 2023-09-23T19:34:47.558492+0000 mon.a (mon.0) 1010 : audit 0 from='client.? 172.21.15.16:0/1517743227' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:34:48.769 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:48.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:48 smithi092 bash[58855]: audit 2023-09-23T19:34:47.558492+0000 mon.a (mon.0) 1010 : audit 0 from='client.? 172.21.15.16:0/1517743227' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:34:48.797 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:48.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:48.803 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:34:48.903 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:48.929 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:48.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:48.933 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:49.033 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:49.064 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:49.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:49.069 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:49.169 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:49.197 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:49.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:49.200 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:49.301 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:49.330 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:49.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:49.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:34:49.362 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T19:34:49.437 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:49.465 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:49.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:49.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:49.573 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:49.605 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:49.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:49.609 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:34:49.709 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:49.717 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:49 smithi092 bash[58855]: cluster 2023-09-23T19:34:48.026157+0000 mgr.y (mgr.25325) 817 : cluster 0 pgmap v723: 89 pgs: 89 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2023-09-23T19:34:49.735 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:49.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:49.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:49.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:49 smithi016 bash[75004]: cluster 2023-09-23T19:34:48.026157+0000 mgr.y (mgr.25325) 817 : cluster 0 pgmap v723: 89 pgs: 89 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2023-09-23T19:34:49.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:49 smithi016 bash[77995]: cluster 2023-09-23T19:34:48.026157+0000 mgr.y (mgr.25325) 817 : cluster 0 pgmap v723: 89 pgs: 89 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2023-09-23T19:34:49.841 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:49.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:49.871 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:49.871 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:34:49.972 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:50.003 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:50.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:50.010 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:50.111 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:50.143 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:50.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:50.147 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:34:50.248 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:50.272 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:50.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:50.277 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:50.378 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:50.421 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:50.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:50.425 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:34:50.526 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:50.556 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:50.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:50.564 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:34:50.664 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:50.665 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:34:50.704 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:50.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:50.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:50.809 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:50.845 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:50.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:50.848 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:50.948 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:50.976 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:50.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:50.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:51.082 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:51.106 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:51.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:51.109 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:51.157 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:34:51.209 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:51.240 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:51.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:51.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:51.349 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:51.389 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:51.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:51.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:51.496 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:51.536 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:51.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:51.542 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:51.642 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:51.651 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:51 smithi016 bash[77995]: cluster 2023-09-23T19:34:50.026737+0000 mgr.y (mgr.25325) 818 : cluster 0 pgmap v724: 89 pgs: 89 active+clean; 2.1 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2023-09-23T19:34:51.651 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:51 smithi016 bash[75004]: cluster 2023-09-23T19:34:50.026737+0000 mgr.y (mgr.25325) 818 : cluster 0 pgmap v724: 89 pgs: 89 active+clean; 2.1 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2023-09-23T19:34:51.668 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:51.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:51.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:34:51.780 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:51.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:51 smithi092 bash[58855]: cluster 2023-09-23T19:34:50.026737+0000 mgr.y (mgr.25325) 818 : cluster 0 pgmap v724: 89 pgs: 89 active+clean; 2.1 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2023-09-23T19:34:51.804 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:51.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:51.806 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:51.907 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:51.936 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:51.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:51.939 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:52.039 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:52.068 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:52.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:52.070 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:52.171 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:52.204 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:52.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:52.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:52.308 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:52.342 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:52.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:52.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:52.447 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:52.488 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:52.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:52.492 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:52.593 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:52.616 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:52.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:52.618 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:34:52.689 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:34:52.718 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:52.748 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:52.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:52.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:34:52.852 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:52.878 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:52.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:52.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:52.986 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:53.012 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:53.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:53.017 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:53.117 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:53.151 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:53.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:53.156 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:53.257 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:53.285 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:53.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:53.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:53.389 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:53.413 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:53.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:53.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:53.517 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:53.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:53 smithi016 bash[77995]: cluster 2023-09-23T19:34:52.027368+0000 mgr.y (mgr.25325) 819 : cluster 0 pgmap v725: 89 pgs: 89 active+clean; 2.0 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2023-09-23T19:34:53.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:53 smithi016 bash[75004]: cluster 2023-09-23T19:34:52.027368+0000 mgr.y (mgr.25325) 819 : cluster 0 pgmap v725: 89 pgs: 89 active+clean; 2.0 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2023-09-23T19:34:53.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:53 smithi092 bash[58855]: cluster 2023-09-23T19:34:52.027368+0000 mgr.y (mgr.25325) 819 : cluster 0 pgmap v725: 89 pgs: 89 active+clean; 2.0 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2023-09-23T19:34:53.545 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:53.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:53.553 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:34:53.654 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:53.681 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:53.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:53.683 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:34:53.746 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:34:53.784 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:53.809 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:53.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:53.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:53.912 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:53.936 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:53.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:53.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:54.041 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:54.067 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:54.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:54.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:54.173 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:54.204 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:54.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:54.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:54.311 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:54.338 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:54.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:54.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:34:54.441 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:54.495 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:54.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:54.498 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:34:54.599 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:54.629 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:54.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:54.633 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:54.734 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:54.763 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:54.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:54.766 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:34:54.866 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:54.894 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:54.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:54.897 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:54.997 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:55.038 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:55.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:55.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:55.142 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:55.174 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:55.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:55.179 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:55.279 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:55.311 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:55.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:55.314 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:34:55.415 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:55.461 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:55.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:55.466 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:34:55.561 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:55 smithi016 bash[77995]: cluster 2023-09-23T19:34:54.027930+0000 mgr.y (mgr.25325) 820 : cluster 0 pgmap v726: 89 pgs: 89 active+clean; 1.9 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 678 op/s 2023-09-23T19:34:55.562 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:55 smithi016 bash[75004]: cluster 2023-09-23T19:34:54.027930+0000 mgr.y (mgr.25325) 820 : cluster 0 pgmap v726: 89 pgs: 89 active+clean; 1.9 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 678 op/s 2023-09-23T19:34:55.567 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:55.600 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:55.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:55.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:55.705 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:55.750 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:55.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:55.760 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:55.779 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:34:55.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:55 smithi092 bash[58855]: cluster 2023-09-23T19:34:54.027930+0000 mgr.y (mgr.25325) 820 : cluster 0 pgmap v726: 89 pgs: 89 active+clean; 1.9 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 678 op/s 2023-09-23T19:34:55.815 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:34:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:34:55] "GET /metrics HTTP/1.1" 200 36238 "" "Prometheus/2.33.4" 2023-09-23T19:34:55.860 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:55.903 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:55.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:55.907 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:34:56.008 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:56.030 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:56.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:56.032 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:56.133 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:56.160 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:56.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:56.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:34:56.266 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:56.301 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:56.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:56.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:56.406 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:56.429 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:56.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:56.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:56.456 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T19:34:56.533 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:56.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:56 smithi092 bash[58855]: audit 2023-09-23T19:34:55.989850+0000 mon.c (mon.2) 151 : audit 1 from='client.? 172.21.15.16:0/3204914460' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T19:34:56.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:56 smithi092 bash[58855]: audit 2023-09-23T19:34:55.990821+0000 mon.a (mon.0) 1011 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T19:34:56.567 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:56.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:56.571 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:56.672 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:56.695 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:56.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:56.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:34:56.800 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:56.818 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:56.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:56.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:56.823 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:56 smithi016 bash[77995]: audit 2023-09-23T19:34:55.989850+0000 mon.c (mon.2) 151 : audit 1 from='client.? 172.21.15.16:0/3204914460' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T19:34:56.823 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:56 smithi016 bash[77995]: audit 2023-09-23T19:34:55.990821+0000 mon.a (mon.0) 1011 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T19:34:56.823 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:56 smithi016 bash[75004]: audit 2023-09-23T19:34:55.989850+0000 mon.c (mon.2) 151 : audit 1 from='client.? 172.21.15.16:0/3204914460' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T19:34:56.823 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:56 smithi016 bash[75004]: audit 2023-09-23T19:34:55.990821+0000 mon.a (mon.0) 1011 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T19:34:56.921 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:56.938 INFO:teuthology.orchestra.run.smithi092.stdout:{ 2023-09-23T19:34:56.938 INFO:teuthology.orchestra.run.smithi092.stdout: "ops": [], 2023-09-23T19:34:56.938 INFO:teuthology.orchestra.run.smithi092.stdout: "complaint_time": 30, 2023-09-23T19:34:56.938 INFO:teuthology.orchestra.run.smithi092.stdout: "num_blocked_ops": 0 2023-09-23T19:34:56.938 INFO:teuthology.orchestra.run.smithi092.stdout:} 2023-09-23T19:34:56.957 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:56.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:56.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:57.062 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:57.080 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:57.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:57.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:57.183 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:57.224 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:57.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:57.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:34:57.328 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:57.355 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:57.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:57.358 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:34:57.458 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:57.496 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:34:57.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:57.500 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:34:57.601 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:57.643 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:57.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:57.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:34:57.746 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:57.755 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:57 smithi016 bash[77995]: cluster 2023-09-23T19:34:56.028587+0000 mgr.y (mgr.25325) 821 : cluster 0 pgmap v727: 89 pgs: 89 active+clean; 1.8 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 640 op/s 2023-09-23T19:34:57.755 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:57 smithi016 bash[77995]: audit 2023-09-23T19:34:56.437860+0000 mon.a (mon.0) 1012 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T19:34:57.755 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:57 smithi016 bash[77995]: cluster 2023-09-23T19:34:56.442819+0000 mon.a (mon.0) 1013 : cluster 0 osdmap e134: 8 total, 8 up, 8 in 2023-09-23T19:34:57.755 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:57 smithi016 bash[77995]: cluster 2023-09-23T19:34:56.446758+0000 osd.5 (osd.5) 65 : cluster 3 failed to encode map e134 with expected crc 2023-09-23T19:34:57.755 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:57 smithi016 bash[77995]: cluster 2023-09-23T19:34:56.447582+0000 osd.7 (osd.7) 52 : cluster 3 failed to encode map e134 with expected crc 2023-09-23T19:34:57.755 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:57 smithi016 bash[77995]: cluster 2023-09-23T19:34:56.449511+0000 osd.5 (osd.5) 66 : cluster 3 failed to encode map e134 with expected crc 2023-09-23T19:34:57.755 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:57 smithi016 bash[77995]: cluster 2023-09-23T19:34:56.455866+0000 osd.4 (osd.4) 62 : cluster 3 failed to encode map e134 with expected crc 2023-09-23T19:34:57.755 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:57 smithi016 bash[77995]: cluster 2023-09-23T19:34:56.457450+0000 osd.6 (osd.6) 54 : cluster 3 failed to encode map e134 with expected crc 2023-09-23T19:34:57.755 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:57 smithi016 bash[77995]: cluster 2023-09-23T19:34:56.457669+0000 osd.4 (osd.4) 63 : cluster 3 failed to encode map e134 with expected crc 2023-09-23T19:34:57.756 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:57 smithi016 bash[75004]: cluster 2023-09-23T19:34:56.028587+0000 mgr.y (mgr.25325) 821 : cluster 0 pgmap v727: 89 pgs: 89 active+clean; 1.8 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 640 op/s 2023-09-23T19:34:57.756 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:57 smithi016 bash[75004]: audit 2023-09-23T19:34:56.437860+0000 mon.a (mon.0) 1012 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T19:34:57.756 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:57 smithi016 bash[75004]: cluster 2023-09-23T19:34:56.442819+0000 mon.a (mon.0) 1013 : cluster 0 osdmap e134: 8 total, 8 up, 8 in 2023-09-23T19:34:57.756 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:57 smithi016 bash[75004]: cluster 2023-09-23T19:34:56.446758+0000 osd.5 (osd.5) 65 : cluster 3 failed to encode map e134 with expected crc 2023-09-23T19:34:57.756 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:57 smithi016 bash[75004]: cluster 2023-09-23T19:34:56.447582+0000 osd.7 (osd.7) 52 : cluster 3 failed to encode map e134 with expected crc 2023-09-23T19:34:57.756 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:57 smithi016 bash[75004]: cluster 2023-09-23T19:34:56.449511+0000 osd.5 (osd.5) 66 : cluster 3 failed to encode map e134 with expected crc 2023-09-23T19:34:57.756 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:57 smithi016 bash[75004]: cluster 2023-09-23T19:34:56.455866+0000 osd.4 (osd.4) 62 : cluster 3 failed to encode map e134 with expected crc 2023-09-23T19:34:57.756 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:57 smithi016 bash[75004]: cluster 2023-09-23T19:34:56.457450+0000 osd.6 (osd.6) 54 : cluster 3 failed to encode map e134 with expected crc 2023-09-23T19:34:57.756 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:57 smithi016 bash[75004]: cluster 2023-09-23T19:34:56.457669+0000 osd.4 (osd.4) 63 : cluster 3 failed to encode map e134 with expected crc 2023-09-23T19:34:57.781 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:57.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:57.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:57.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:57 smithi092 bash[58855]: cluster 2023-09-23T19:34:56.028587+0000 mgr.y (mgr.25325) 821 : cluster 0 pgmap v727: 89 pgs: 89 active+clean; 1.8 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 640 op/s 2023-09-23T19:34:57.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:57 smithi092 bash[58855]: audit 2023-09-23T19:34:56.437860+0000 mon.a (mon.0) 1012 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T19:34:57.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:57 smithi092 bash[58855]: cluster 2023-09-23T19:34:56.442819+0000 mon.a (mon.0) 1013 : cluster 0 osdmap e134: 8 total, 8 up, 8 in 2023-09-23T19:34:57.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:57 smithi092 bash[58855]: cluster 2023-09-23T19:34:56.446758+0000 osd.5 (osd.5) 65 : cluster 3 failed to encode map e134 with expected crc 2023-09-23T19:34:57.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:57 smithi092 bash[58855]: cluster 2023-09-23T19:34:56.447582+0000 osd.7 (osd.7) 52 : cluster 3 failed to encode map e134 with expected crc 2023-09-23T19:34:57.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:57 smithi092 bash[58855]: cluster 2023-09-23T19:34:56.449511+0000 osd.5 (osd.5) 66 : cluster 3 failed to encode map e134 with expected crc 2023-09-23T19:34:57.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:57 smithi092 bash[58855]: cluster 2023-09-23T19:34:56.455866+0000 osd.4 (osd.4) 62 : cluster 3 failed to encode map e134 with expected crc 2023-09-23T19:34:57.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:57 smithi092 bash[58855]: cluster 2023-09-23T19:34:56.457450+0000 osd.6 (osd.6) 54 : cluster 3 failed to encode map e134 with expected crc 2023-09-23T19:34:57.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:57 smithi092 bash[58855]: cluster 2023-09-23T19:34:56.457669+0000 osd.4 (osd.4) 63 : cluster 3 failed to encode map e134 with expected crc 2023-09-23T19:34:57.873 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T19:34:57.886 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:57.908 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:34:57.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:57.910 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:34:58.011 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:58.051 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:58.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:58.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:58.156 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:58.190 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:34:58.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:58.197 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:34:58.298 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:58.322 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:58.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:58.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:58.430 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:58.457 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:34:58.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:58.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:34:58.561 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:58.600 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:58.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:58.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:34:58.704 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:58.729 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:58.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:58.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:58.836 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:58.865 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:34:58.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:58.869 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:34:58.969 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:59.003 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:59.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:59.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:59.112 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:59.158 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:59.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:59.161 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:34:59.262 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:59.295 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:34:59.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:59.301 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:34:59.401 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:59.430 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:59.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:59.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:34:59.533 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:59.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:59 smithi016 bash[75004]: cluster 2023-09-23T19:34:58.029294+0000 mgr.y (mgr.25325) 822 : cluster 0 pgmap v729: 89 pgs: 89 active+clean; 1.7 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 674 op/s 2023-09-23T19:34:59.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:34:59 smithi016 bash[75004]: audit 2023-09-23T19:34:58.641704+0000 mon.a (mon.0) 1014 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:34:59.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:59 smithi016 bash[77995]: cluster 2023-09-23T19:34:58.029294+0000 mgr.y (mgr.25325) 822 : cluster 0 pgmap v729: 89 pgs: 89 active+clean; 1.7 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 674 op/s 2023-09-23T19:34:59.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:34:59 smithi016 bash[77995]: audit 2023-09-23T19:34:58.641704+0000 mon.a (mon.0) 1014 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:34:59.557 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:59.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:59.561 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:59.662 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:59.670 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:59 smithi092 bash[58855]: cluster 2023-09-23T19:34:58.029294+0000 mgr.y (mgr.25325) 822 : cluster 0 pgmap v729: 89 pgs: 89 active+clean; 1.7 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 674 op/s 2023-09-23T19:34:59.670 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:34:59 smithi092 bash[58855]: audit 2023-09-23T19:34:58.641704+0000 mon.a (mon.0) 1014 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:34:59.690 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:34:59.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:59.697 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:34:59.729 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T19:34:59.797 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:59.834 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:59.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:59.840 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:34:59.941 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:59.973 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:34:59.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:34:59.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:00.080 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:00.108 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:00.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:00.111 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:35:00.211 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:00.250 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:00.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:00.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:35:00.355 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:00.382 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:00.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:00.384 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:00.485 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:00.510 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:00.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:00.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:00.618 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:00.639 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:00.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:00.641 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:35:00.742 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:00.765 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:00.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:00.767 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:00.868 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:00.915 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:00.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:00.920 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:35:01.021 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:01.055 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:01.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:01.060 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:01.160 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:01.192 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:01.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:01.198 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:35:01.298 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:01.337 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:01.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:01.343 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:01.444 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:01.466 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:01.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:01.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:01.498 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is unset 2023-09-23T19:35:01.564 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:01 smithi016 bash[77995]: cluster 2023-09-23T19:35:00.029803+0000 mgr.y (mgr.25325) 823 : cluster 0 pgmap v730: 89 pgs: 89 active+clean; 1.7 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 674 op/s 2023-09-23T19:35:01.564 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:01 smithi016 bash[77995]: audit 2023-09-23T19:35:00.662053+0000 mon.a (mon.0) 1015 : audit 1 from='client.? 172.21.15.16:0/1178672252' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:35:01.564 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:01 smithi016 bash[75004]: cluster 2023-09-23T19:35:00.029803+0000 mgr.y (mgr.25325) 823 : cluster 0 pgmap v730: 89 pgs: 89 active+clean; 1.7 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 674 op/s 2023-09-23T19:35:01.565 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:01 smithi016 bash[75004]: audit 2023-09-23T19:35:00.662053+0000 mon.a (mon.0) 1015 : audit 1 from='client.? 172.21.15.16:0/1178672252' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:35:01.570 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:01.599 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:01.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:01.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:35:01.705 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:01.729 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:01.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:01.731 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:01.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:01 smithi092 bash[58855]: cluster 2023-09-23T19:35:00.029803+0000 mgr.y (mgr.25325) 823 : cluster 0 pgmap v730: 89 pgs: 89 active+clean; 1.7 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 674 op/s 2023-09-23T19:35:01.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:01 smithi092 bash[58855]: audit 2023-09-23T19:35:00.662053+0000 mon.a (mon.0) 1015 : audit 1 from='client.? 172.21.15.16:0/1178672252' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:35:01.832 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:01.866 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:01.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:01.876 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:01.977 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:02.010 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:02.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:02.015 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:02.116 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:02.141 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:02.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:02.145 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:02.245 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:02.272 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:02.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:02.277 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:35:02.377 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:02.403 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:02.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:02.405 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:02.506 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:02.543 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:02.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:02.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:02.650 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:02.680 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:02.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:02.684 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:35:02.718 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:35:02.754 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:02 smithi092 bash[58855]: cluster 2023-09-23T19:35:01.464045+0000 mon.a (mon.0) 1016 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:35:02.754 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:02 smithi092 bash[58855]: cluster 2023-09-23T19:35:01.464110+0000 mon.a (mon.0) 1017 : cluster 1 Cluster is now healthy 2023-09-23T19:35:02.754 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:02 smithi092 bash[58855]: audit 2023-09-23T19:35:01.483713+0000 mon.a (mon.0) 1018 : audit 1 from='client.? 172.21.15.16:0/1178672252' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:35:02.754 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:02 smithi092 bash[58855]: cluster 2023-09-23T19:35:01.484018+0000 mon.a (mon.0) 1019 : cluster 0 osdmap e135: 8 total, 8 up, 8 in 2023-09-23T19:35:02.755 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:02 smithi092 bash[58855]: cluster 2023-09-23T19:35:01.485154+0000 osd.4 (osd.4) 64 : cluster 3 failed to encode map e135 with expected crc 2023-09-23T19:35:02.755 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:02 smithi092 bash[58855]: cluster 2023-09-23T19:35:01.505591+0000 osd.6 (osd.6) 55 : cluster 3 failed to encode map e135 with expected crc 2023-09-23T19:35:02.785 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:02.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:02 smithi016 bash[75004]: cluster 2023-09-23T19:35:01.464045+0000 mon.a (mon.0) 1016 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:35:02.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:02 smithi016 bash[75004]: cluster 2023-09-23T19:35:01.464110+0000 mon.a (mon.0) 1017 : cluster 1 Cluster is now healthy 2023-09-23T19:35:02.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:02 smithi016 bash[75004]: audit 2023-09-23T19:35:01.483713+0000 mon.a (mon.0) 1018 : audit 1 from='client.? 172.21.15.16:0/1178672252' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:35:02.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:02 smithi016 bash[75004]: cluster 2023-09-23T19:35:01.484018+0000 mon.a (mon.0) 1019 : cluster 0 osdmap e135: 8 total, 8 up, 8 in 2023-09-23T19:35:02.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:02 smithi016 bash[75004]: cluster 2023-09-23T19:35:01.485154+0000 osd.4 (osd.4) 64 : cluster 3 failed to encode map e135 with expected crc 2023-09-23T19:35:02.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:02 smithi016 bash[75004]: cluster 2023-09-23T19:35:01.505591+0000 osd.6 (osd.6) 55 : cluster 3 failed to encode map e135 with expected crc 2023-09-23T19:35:02.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:02 smithi016 bash[77995]: cluster 2023-09-23T19:35:01.464045+0000 mon.a (mon.0) 1016 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:35:02.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:02 smithi016 bash[77995]: cluster 2023-09-23T19:35:01.464110+0000 mon.a (mon.0) 1017 : cluster 1 Cluster is now healthy 2023-09-23T19:35:02.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:02 smithi016 bash[77995]: audit 2023-09-23T19:35:01.483713+0000 mon.a (mon.0) 1018 : audit 1 from='client.? 172.21.15.16:0/1178672252' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:35:02.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:02 smithi016 bash[77995]: cluster 2023-09-23T19:35:01.484018+0000 mon.a (mon.0) 1019 : cluster 0 osdmap e135: 8 total, 8 up, 8 in 2023-09-23T19:35:02.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:02 smithi016 bash[77995]: cluster 2023-09-23T19:35:01.485154+0000 osd.4 (osd.4) 64 : cluster 3 failed to encode map e135 with expected crc 2023-09-23T19:35:02.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:02 smithi016 bash[77995]: cluster 2023-09-23T19:35:01.505591+0000 osd.6 (osd.6) 55 : cluster 3 failed to encode map e135 with expected crc 2023-09-23T19:35:02.822 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:02.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:02.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:02.926 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:02.964 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:02.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:02.967 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:35:03.068 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:03.104 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:03.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:03.111 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:03.212 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:03.236 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:03.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:03.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:03.340 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:03.365 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:03.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:03.371 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:03.472 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:03.498 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:03.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:03.501 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:03.602 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:03.610 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:03 smithi016 bash[77995]: cluster 2023-09-23T19:35:02.030327+0000 mgr.y (mgr.25325) 824 : cluster 0 pgmap v732: 89 pgs: 89 active+clean; 1.6 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 732 op/s 2023-09-23T19:35:03.611 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:03 smithi016 bash[75004]: cluster 2023-09-23T19:35:02.030327+0000 mgr.y (mgr.25325) 824 : cluster 0 pgmap v732: 89 pgs: 89 active+clean; 1.6 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 732 op/s 2023-09-23T19:35:03.652 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:03.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:03.659 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:03.760 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:03.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:03 smithi092 bash[58855]: cluster 2023-09-23T19:35:02.030327+0000 mgr.y (mgr.25325) 824 : cluster 0 pgmap v732: 89 pgs: 89 active+clean; 1.6 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 732 op/s 2023-09-23T19:35:03.796 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:03.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:03.799 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:03.900 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:03.929 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:03.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:03.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:35:04.033 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:04.061 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:04.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:04.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:35:04.163 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:04.206 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:04.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:04.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:04.279 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:35:04.312 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:35:04.313 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:04.338 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:04.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:04.344 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:35:04.350 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:35:04.387 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:35:04.420 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:35:04.446 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:04.464 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:35:04.477 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:04.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:04.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:04.499 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:35:04.542 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:35:04.557 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:35:04.557 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:35:04.558 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:35:04.558 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:35:04.558 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:35:04.558 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:35:04.558 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:35:04.558 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:35:04.584 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:04.614 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:04.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:04.616 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:04.717 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:04.720 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:04 smithi092 bash[58855]: audit 2023-09-23T19:35:04.229091+0000 mon.c (mon.2) 152 : audit 0 from='client.? 172.21.15.16:0/4206475404' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:35:04.749 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:04.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:04 smithi016 bash[75004]: audit 2023-09-23T19:35:04.229091+0000 mon.c (mon.2) 152 : audit 0 from='client.? 172.21.15.16:0/4206475404' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:35:04.751 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:04 smithi016 bash[77995]: audit 2023-09-23T19:35:04.229091+0000 mon.c (mon.2) 152 : audit 0 from='client.? 172.21.15.16:0/4206475404' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:35:04.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:04.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:04.858 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:04.894 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:04.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:04.898 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:35:04.999 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:05.033 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:05.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:05.036 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:35:05.136 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:05.163 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:05.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:05.169 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:35:05.269 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:05.303 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:05.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:05.309 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:35:05.410 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:05.433 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:05.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:05.441 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:05.542 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:05.588 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:05.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:05.594 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:05.669 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T19:35:05.678 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:05 smithi016 bash[75004]: cluster 2023-09-23T19:35:04.032098+0000 mgr.y (mgr.25325) 825 : cluster 0 pgmap v733: 89 pgs: 89 active+clean; 1.5 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 589 op/s 2023-09-23T19:35:05.678 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:35:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:35:05] "GET /metrics HTTP/1.1" 200 36236 "" "Prometheus/2.33.4" 2023-09-23T19:35:05.679 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:05 smithi016 bash[77995]: cluster 2023-09-23T19:35:04.032098+0000 mgr.y (mgr.25325) 825 : cluster 0 pgmap v733: 89 pgs: 89 active+clean; 1.5 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 589 op/s 2023-09-23T19:35:05.694 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:05.729 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:05.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:05.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:05.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:05 smithi092 bash[58855]: cluster 2023-09-23T19:35:04.032098+0000 mgr.y (mgr.25325) 825 : cluster 0 pgmap v733: 89 pgs: 89 active+clean; 1.5 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 589 op/s 2023-09-23T19:35:05.833 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:05.870 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:05.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:05.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:05.977 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:06.006 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:06.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:06.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:06.112 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:06.151 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:06.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:06.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:06.255 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:06.289 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:06.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:06.293 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:35:06.394 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:06.429 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:06.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:06.432 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:35:06.532 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:06.563 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:06.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:06.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:35:06.666 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:06.700 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:06.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:06.704 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:06.804 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:06.829 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:06.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:06.833 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:06.933 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:06.971 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:06.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:06.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:07.074 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:07.103 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:35:07.112 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:07.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:07.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:35:07.216 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:07.241 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:07.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:07.244 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:35:07.344 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:07.353 INFO:teuthology.orchestra.run.smithi092.stdout:ERROR: (22) Invalid argument 2023-09-23T19:35:07.353 INFO:teuthology.orchestra.run.smithi092.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:35:07.374 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:07.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:07.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:07.477 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:07.512 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:07.513 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:07 smithi092 bash[58855]: cluster 2023-09-23T19:35:06.032777+0000 mgr.y (mgr.25325) 826 : cluster 0 pgmap v734: 89 pgs: 89 active+clean; 1.4 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 694 op/s 2023-09-23T19:35:07.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:07.516 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:35:07.616 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:07.617 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:35:07.625 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:07 smithi016 bash[77995]: cluster 2023-09-23T19:35:06.032777+0000 mgr.y (mgr.25325) 826 : cluster 0 pgmap v734: 89 pgs: 89 active+clean; 1.4 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 694 op/s 2023-09-23T19:35:07.626 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:07 smithi016 bash[75004]: cluster 2023-09-23T19:35:06.032777+0000 mgr.y (mgr.25325) 826 : cluster 0 pgmap v734: 89 pgs: 89 active+clean; 1.4 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 694 op/s 2023-09-23T19:35:07.649 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:07.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:07.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:35:07.751 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:07.781 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:07.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:07.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:07.886 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:07.922 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:07.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:07.929 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:35:08.029 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:08.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:08.059 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:08.059 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:35:08.160 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:08.194 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:08.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:08.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:35:08.296 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:08.338 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:08.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:08.341 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:35:08.351 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T19:35:08.442 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:08.468 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:08.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:08.471 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:35:08.572 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:08.609 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:08.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:08.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:35:08.712 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:08.736 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:08.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:08.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:08.840 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:08.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:08.881 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:08.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:08.982 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:09.003 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:09.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:09.009 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:35:09.109 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:09.143 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:09.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:09.146 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:35:09.246 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:09.270 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:09.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:09.276 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:35:09.377 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:09.405 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:09.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:09.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:09.512 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:09.555 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:09.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:09.561 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:09.661 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:09.721 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:09.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:09.726 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:09.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:09 smithi016 bash[77995]: cluster 2023-09-23T19:35:08.033342+0000 mgr.y (mgr.25325) 827 : cluster 0 pgmap v735: 89 pgs: 89 active+clean; 1.3 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2023-09-23T19:35:09.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:09 smithi016 bash[75004]: cluster 2023-09-23T19:35:08.033342+0000 mgr.y (mgr.25325) 827 : cluster 0 pgmap v735: 89 pgs: 89 active+clean; 1.3 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2023-09-23T19:35:09.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:09 smithi092 bash[58855]: cluster 2023-09-23T19:35:08.033342+0000 mgr.y (mgr.25325) 827 : cluster 0 pgmap v735: 89 pgs: 89 active+clean; 1.3 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2023-09-23T19:35:09.826 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:09.851 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:09.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:09.856 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:35:09.957 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:09.983 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:09.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:09.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:35:10.089 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:10.120 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:10.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:10.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:10.173 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:35:10.225 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:10.258 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:10.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:10.261 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:10.362 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:10.402 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:10.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:10.405 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:10.506 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:10.529 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:10.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:10.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:35:10.632 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:10.667 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:10.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:10.670 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:10.770 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:10.793 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:10.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:10.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:35:10.896 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:10.918 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:10.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:10.924 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:11.024 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:11.050 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:11.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:11.052 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:35:11.152 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:11.177 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:11.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:11.179 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:35:11.280 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:11.302 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:11.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:11.305 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:35:11.406 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:11.429 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:11.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:11.436 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:11.528 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:11 smithi092 bash[58855]: cluster 2023-09-23T19:35:10.033938+0000 mgr.y (mgr.25325) 828 : cluster 0 pgmap v736: 89 pgs: 89 active+clean; 1.3 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 674 op/s 2023-09-23T19:35:11.536 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:11.565 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:11.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:11.569 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:11.669 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:11.678 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:11 smithi016 bash[75004]: cluster 2023-09-23T19:35:10.033938+0000 mgr.y (mgr.25325) 828 : cluster 0 pgmap v736: 89 pgs: 89 active+clean; 1.3 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 674 op/s 2023-09-23T19:35:11.679 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:11 smithi016 bash[77995]: cluster 2023-09-23T19:35:10.033938+0000 mgr.y (mgr.25325) 828 : cluster 0 pgmap v736: 89 pgs: 89 active+clean; 1.3 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 674 op/s 2023-09-23T19:35:11.704 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:11.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:11.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:11.809 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:11.833 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:11.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:11.841 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:11.942 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:11.965 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:11.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:11.967 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:12.067 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:12.099 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:12.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:12.104 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:35:12.204 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:12.205 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:35:12.231 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:12.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:12.234 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:35:12.335 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:12.374 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:12.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:12.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:12.478 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:12.511 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:12.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:12.513 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:12.532 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:12 smithi016 bash[75004]: audit 2023-09-23T19:35:12.462990+0000 mon.c (mon.2) 153 : audit 1 from='client.? 172.21.15.16:0/3613942214' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T19:35:12.532 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:12 smithi016 bash[75004]: audit 2023-09-23T19:35:12.463420+0000 mon.a (mon.0) 1020 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T19:35:12.558 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T19:35:12.614 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:12.622 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:12 smithi092 bash[58855]: audit 2023-09-23T19:35:12.462990+0000 mon.c (mon.2) 153 : audit 1 from='client.? 172.21.15.16:0/3613942214' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T19:35:12.622 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:12 smithi092 bash[58855]: audit 2023-09-23T19:35:12.463420+0000 mon.a (mon.0) 1020 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T19:35:12.649 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:12.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:12.654 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:35:12.755 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:12.783 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:12.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:12 smithi016 bash[77995]: audit 2023-09-23T19:35:12.462990+0000 mon.c (mon.2) 153 : audit 1 from='client.? 172.21.15.16:0/3613942214' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T19:35:12.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:12 smithi016 bash[77995]: audit 2023-09-23T19:35:12.463420+0000 mon.a (mon.0) 1020 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T19:35:12.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:12.788 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:35:12.889 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:12.913 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:12.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:12.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:35:13.020 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:13.045 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:13.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:13.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:13.148 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:13.182 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:13.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:13.184 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:13.241 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:35:13.285 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:13.315 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:13.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:13.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:35:13.425 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:13.453 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:13.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:13.456 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:13.556 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:13.584 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:13.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:13.590 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:35:13.691 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:13.701 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:13 smithi092 bash[58855]: cluster 2023-09-23T19:35:12.034599+0000 mgr.y (mgr.25325) 829 : cluster 0 pgmap v737: 89 pgs: 89 active+clean; 1.2 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2023-09-23T19:35:13.701 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:13 smithi092 bash[58855]: audit 2023-09-23T19:35:12.540639+0000 mon.a (mon.0) 1021 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T19:35:13.701 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:13 smithi092 bash[58855]: cluster 2023-09-23T19:35:12.540738+0000 mon.a (mon.0) 1022 : cluster 0 osdmap e136: 8 total, 8 up, 8 in 2023-09-23T19:35:13.702 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:13 smithi092 bash[58855]: cluster 2023-09-23T19:35:12.550337+0000 osd.7 (osd.7) 53 : cluster 3 failed to encode map e136 with expected crc 2023-09-23T19:35:13.702 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:13 smithi092 bash[58855]: cluster 2023-09-23T19:35:12.552065+0000 osd.6 (osd.6) 56 : cluster 3 failed to encode map e136 with expected crc 2023-09-23T19:35:13.702 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:13 smithi092 bash[58855]: cluster 2023-09-23T19:35:12.554116+0000 osd.5 (osd.5) 67 : cluster 3 failed to encode map e136 with expected crc 2023-09-23T19:35:13.702 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:13 smithi092 bash[58855]: cluster 2023-09-23T19:35:12.555257+0000 osd.4 (osd.4) 65 : cluster 3 failed to encode map e136 with expected crc 2023-09-23T19:35:13.702 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:13 smithi092 bash[58855]: cluster 2023-09-23T19:35:12.556082+0000 osd.7 (osd.7) 54 : cluster 3 failed to encode map e136 with expected crc 2023-09-23T19:35:13.703 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:13 smithi092 bash[58855]: cluster 2023-09-23T19:35:12.557814+0000 osd.5 (osd.5) 68 : cluster 3 failed to encode map e136 with expected crc 2023-09-23T19:35:13.736 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:13.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:13.741 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:35:13.841 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:13.866 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:13.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:13.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:35:13.974 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:14.004 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:14.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:14.008 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:35:14.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:13 smithi016 bash[75004]: cluster 2023-09-23T19:35:12.034599+0000 mgr.y (mgr.25325) 829 : cluster 0 pgmap v737: 89 pgs: 89 active+clean; 1.2 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2023-09-23T19:35:14.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:13 smithi016 bash[75004]: audit 2023-09-23T19:35:12.540639+0000 mon.a (mon.0) 1021 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T19:35:14.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:13 smithi016 bash[75004]: cluster 2023-09-23T19:35:12.540738+0000 mon.a (mon.0) 1022 : cluster 0 osdmap e136: 8 total, 8 up, 8 in 2023-09-23T19:35:14.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:13 smithi016 bash[75004]: cluster 2023-09-23T19:35:12.550337+0000 osd.7 (osd.7) 53 : cluster 3 failed to encode map e136 with expected crc 2023-09-23T19:35:14.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:13 smithi016 bash[75004]: cluster 2023-09-23T19:35:12.552065+0000 osd.6 (osd.6) 56 : cluster 3 failed to encode map e136 with expected crc 2023-09-23T19:35:14.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:13 smithi016 bash[75004]: cluster 2023-09-23T19:35:12.554116+0000 osd.5 (osd.5) 67 : cluster 3 failed to encode map e136 with expected crc 2023-09-23T19:35:14.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:13 smithi016 bash[75004]: cluster 2023-09-23T19:35:12.555257+0000 osd.4 (osd.4) 65 : cluster 3 failed to encode map e136 with expected crc 2023-09-23T19:35:14.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:13 smithi016 bash[75004]: cluster 2023-09-23T19:35:12.556082+0000 osd.7 (osd.7) 54 : cluster 3 failed to encode map e136 with expected crc 2023-09-23T19:35:14.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:13 smithi016 bash[75004]: cluster 2023-09-23T19:35:12.557814+0000 osd.5 (osd.5) 68 : cluster 3 failed to encode map e136 with expected crc 2023-09-23T19:35:14.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:13 smithi016 bash[77995]: cluster 2023-09-23T19:35:12.034599+0000 mgr.y (mgr.25325) 829 : cluster 0 pgmap v737: 89 pgs: 89 active+clean; 1.2 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2023-09-23T19:35:14.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:13 smithi016 bash[77995]: audit 2023-09-23T19:35:12.540639+0000 mon.a (mon.0) 1021 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T19:35:14.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:13 smithi016 bash[77995]: cluster 2023-09-23T19:35:12.540738+0000 mon.a (mon.0) 1022 : cluster 0 osdmap e136: 8 total, 8 up, 8 in 2023-09-23T19:35:14.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:13 smithi016 bash[77995]: cluster 2023-09-23T19:35:12.550337+0000 osd.7 (osd.7) 53 : cluster 3 failed to encode map e136 with expected crc 2023-09-23T19:35:14.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:13 smithi016 bash[77995]: cluster 2023-09-23T19:35:12.552065+0000 osd.6 (osd.6) 56 : cluster 3 failed to encode map e136 with expected crc 2023-09-23T19:35:14.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:13 smithi016 bash[77995]: cluster 2023-09-23T19:35:12.554116+0000 osd.5 (osd.5) 67 : cluster 3 failed to encode map e136 with expected crc 2023-09-23T19:35:14.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:13 smithi016 bash[77995]: cluster 2023-09-23T19:35:12.555257+0000 osd.4 (osd.4) 65 : cluster 3 failed to encode map e136 with expected crc 2023-09-23T19:35:14.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:13 smithi016 bash[77995]: cluster 2023-09-23T19:35:12.556082+0000 osd.7 (osd.7) 54 : cluster 3 failed to encode map e136 with expected crc 2023-09-23T19:35:14.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:13 smithi016 bash[77995]: cluster 2023-09-23T19:35:12.557814+0000 osd.5 (osd.5) 68 : cluster 3 failed to encode map e136 with expected crc 2023-09-23T19:35:14.108 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:14.145 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:14.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:14.149 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:14.250 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:14.282 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:14.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:14.289 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:14.389 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:14.418 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:14.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:14.421 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:14.522 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:14.568 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:14.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:14.571 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:14.671 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:14.702 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:14.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:14.711 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:14.812 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:14.839 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:14.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:14.847 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:14.948 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:14.957 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:14 smithi092 bash[58855]: audit 2023-09-23T19:35:13.647876+0000 mon.a (mon.0) 1023 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:35:14.957 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:14 smithi092 bash[58855]: audit 2023-09-23T19:35:13.653122+0000 mon.a (mon.0) 1024 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:35:14.973 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:14.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:14.977 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:35:15.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:14 smithi016 bash[77995]: audit 2023-09-23T19:35:13.647876+0000 mon.a (mon.0) 1023 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:35:15.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:14 smithi016 bash[77995]: audit 2023-09-23T19:35:13.653122+0000 mon.a (mon.0) 1024 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:35:15.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:14 smithi016 bash[75004]: audit 2023-09-23T19:35:13.647876+0000 mon.a (mon.0) 1023 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:35:15.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:14 smithi016 bash[75004]: audit 2023-09-23T19:35:13.653122+0000 mon.a (mon.0) 1024 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:35:15.078 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:15.113 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:15.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:15.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:35:15.216 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:15.248 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:15.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:15.256 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:35:15.357 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:15.379 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:15.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:15.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:35:15.488 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:15.513 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:15.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:15.517 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:35:15.617 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:15.631 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T19:35:15.650 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:15.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:15.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:35:15.757 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:15.780 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:15.782 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:15 smithi092 bash[58855]: cluster 2023-09-23T19:35:14.035613+0000 mgr.y (mgr.25325) 830 : cluster 0 pgmap v739: 89 pgs: 89 active+clean; 1.1 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2023-09-23T19:35:15.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:15.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:35:15.883 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:15.894 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:15 smithi016 bash[75004]: cluster 2023-09-23T19:35:14.035613+0000 mgr.y (mgr.25325) 830 : cluster 0 pgmap v739: 89 pgs: 89 active+clean; 1.1 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2023-09-23T19:35:15.895 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:35:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:35:15] "GET /metrics HTTP/1.1" 200 36231 "" "Prometheus/2.33.4" 2023-09-23T19:35:15.895 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:15 smithi016 bash[77995]: cluster 2023-09-23T19:35:14.035613+0000 mgr.y (mgr.25325) 830 : cluster 0 pgmap v739: 89 pgs: 89 active+clean; 1.1 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2023-09-23T19:35:15.918 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:15.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:15.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:16.022 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:16.055 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:16.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:16.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:35:16.163 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:16.190 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:16.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:16.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:16.296 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:16.329 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:16.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:16.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:16.433 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:16.461 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:16.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:16.464 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:35:16.565 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:16.600 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:16.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:16.606 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:16.706 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:16.913 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:16.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:16.917 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:35:17.017 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:17.039 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:17.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:17.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:17.142 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:17.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:17.187 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:17.187 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:35:17.288 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:17.321 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:17.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:17.328 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:17.428 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:17.464 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:17.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:17.471 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:17.572 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:17.614 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:17.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:17.621 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:35:17.658 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:17 smithi016 bash[75004]: cluster 2023-09-23T19:35:16.036320+0000 mgr.y (mgr.25325) 831 : cluster 0 pgmap v740: 89 pgs: 89 active+clean; 1024 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 676 op/s 2023-09-23T19:35:17.721 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:17.725 INFO:teuthology.orchestra.run.smithi092.stdout:ERROR: (22) Invalid argument 2023-09-23T19:35:17.725 INFO:teuthology.orchestra.run.smithi092.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:35:17.730 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:17 smithi092 bash[58855]: cluster 2023-09-23T19:35:16.036320+0000 mgr.y (mgr.25325) 831 : cluster 0 pgmap v740: 89 pgs: 89 active+clean; 1024 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 676 op/s 2023-09-23T19:35:17.751 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:17.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:17.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:35:17.854 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:17.886 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:17.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:17.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:35:17.989 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:17.999 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:17 smithi016 bash[77995]: cluster 2023-09-23T19:35:16.036320+0000 mgr.y (mgr.25325) 831 : cluster 0 pgmap v740: 89 pgs: 89 active+clean; 1024 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 676 op/s 2023-09-23T19:35:18.018 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:18.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:18.024 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:18.124 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:18.150 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:18.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:18.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:18.255 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:18.288 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:18.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:18.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:18.393 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:18.428 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:18.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:18.431 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:18.531 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:18.553 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:18.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:18.564 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:18.665 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:18.679 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is set 2023-09-23T19:35:18.691 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:18.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:18.697 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:18.797 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:18.807 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:18 smithi016 bash[75004]: audit 2023-09-23T19:35:17.904803+0000 mon.c (mon.2) 154 : audit 1 from='client.? 172.21.15.16:0/503622099' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:35:18.807 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:18 smithi016 bash[75004]: audit 2023-09-23T19:35:17.905399+0000 mon.a (mon.0) 1025 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:35:18.807 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:18 smithi016 bash[77995]: audit 2023-09-23T19:35:17.904803+0000 mon.c (mon.2) 154 : audit 1 from='client.? 172.21.15.16:0/503622099' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:35:18.807 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:18 smithi016 bash[77995]: audit 2023-09-23T19:35:17.905399+0000 mon.a (mon.0) 1025 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:35:18.825 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T19:35:18.825 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:18.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:18.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:18.933 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:18.956 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:18.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:18.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:19.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:18 smithi092 bash[58855]: audit 2023-09-23T19:35:17.904803+0000 mon.c (mon.2) 154 : audit 1 from='client.? 172.21.15.16:0/503622099' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:35:19.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:18 smithi092 bash[58855]: audit 2023-09-23T19:35:17.905399+0000 mon.a (mon.0) 1025 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:35:19.062 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:19.088 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:19.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:19.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:19.194 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:19.224 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:19.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:19.232 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:19.333 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:19.358 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:19.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:19.367 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:19.460 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:35:19.468 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:19.500 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:19.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:19.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:19.505 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:35:19.546 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:35:19.596 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:35:19.604 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:19.630 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:19.631 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:35:19.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:19.632 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:19.681 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:35:19.718 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:35:19.733 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:19.755 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:35:19.758 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:35:19.758 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:35:19.758 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:35:19.758 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:35:19.758 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:35:19.758 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:35:19.758 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:35:19.758 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:35:19.783 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:19.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:19.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:35:19.889 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:19.922 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:19.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:19.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:19.929 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:19 smithi092 bash[58855]: cluster 2023-09-23T19:35:18.036892+0000 mgr.y (mgr.25325) 832 : cluster 0 pgmap v741: 89 pgs: 89 active+clean; 935 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 669 op/s 2023-09-23T19:35:19.929 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:19 smithi092 bash[58855]: cluster 2023-09-23T19:35:18.654585+0000 mon.a (mon.0) 1026 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:35:19.929 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:19 smithi092 bash[58855]: audit 2023-09-23T19:35:18.666395+0000 mon.a (mon.0) 1027 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:35:19.929 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:19 smithi092 bash[58855]: cluster 2023-09-23T19:35:18.666486+0000 mon.a (mon.0) 1028 : cluster 0 osdmap e137: 8 total, 8 up, 8 in 2023-09-23T19:35:19.929 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:19 smithi092 bash[58855]: cluster 2023-09-23T19:35:18.671800+0000 osd.5 (osd.5) 69 : cluster 3 failed to encode map e137 with expected crc 2023-09-23T19:35:19.929 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:19 smithi092 bash[58855]: cluster 2023-09-23T19:35:18.672486+0000 osd.7 (osd.7) 55 : cluster 3 failed to encode map e137 with expected crc 2023-09-23T19:35:19.930 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:19 smithi092 bash[58855]: cluster 2023-09-23T19:35:18.675775+0000 osd.7 (osd.7) 56 : cluster 3 failed to encode map e137 with expected crc 2023-09-23T19:35:19.930 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:19 smithi092 bash[58855]: cluster 2023-09-23T19:35:18.677460+0000 osd.6 (osd.6) 57 : cluster 3 failed to encode map e137 with expected crc 2023-09-23T19:35:19.930 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:19 smithi092 bash[58855]: audit 2023-09-23T19:35:19.408846+0000 mon.a (mon.0) 1029 : audit 0 from='client.? 172.21.15.16:0/1481251152' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:35:20.030 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:20.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:19 smithi016 bash[75004]: cluster 2023-09-23T19:35:18.036892+0000 mgr.y (mgr.25325) 832 : cluster 0 pgmap v741: 89 pgs: 89 active+clean; 935 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 669 op/s 2023-09-23T19:35:20.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:19 smithi016 bash[75004]: cluster 2023-09-23T19:35:18.654585+0000 mon.a (mon.0) 1026 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:35:20.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:19 smithi016 bash[75004]: audit 2023-09-23T19:35:18.666395+0000 mon.a (mon.0) 1027 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:35:20.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:19 smithi016 bash[75004]: cluster 2023-09-23T19:35:18.666486+0000 mon.a (mon.0) 1028 : cluster 0 osdmap e137: 8 total, 8 up, 8 in 2023-09-23T19:35:20.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:19 smithi016 bash[75004]: cluster 2023-09-23T19:35:18.671800+0000 osd.5 (osd.5) 69 : cluster 3 failed to encode map e137 with expected crc 2023-09-23T19:35:20.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:19 smithi016 bash[75004]: cluster 2023-09-23T19:35:18.672486+0000 osd.7 (osd.7) 55 : cluster 3 failed to encode map e137 with expected crc 2023-09-23T19:35:20.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:19 smithi016 bash[75004]: cluster 2023-09-23T19:35:18.675775+0000 osd.7 (osd.7) 56 : cluster 3 failed to encode map e137 with expected crc 2023-09-23T19:35:20.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:19 smithi016 bash[75004]: cluster 2023-09-23T19:35:18.677460+0000 osd.6 (osd.6) 57 : cluster 3 failed to encode map e137 with expected crc 2023-09-23T19:35:20.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:19 smithi016 bash[75004]: audit 2023-09-23T19:35:19.408846+0000 mon.a (mon.0) 1029 : audit 0 from='client.? 172.21.15.16:0/1481251152' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:35:20.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:19 smithi016 bash[77995]: cluster 2023-09-23T19:35:18.036892+0000 mgr.y (mgr.25325) 832 : cluster 0 pgmap v741: 89 pgs: 89 active+clean; 935 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 669 op/s 2023-09-23T19:35:20.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:19 smithi016 bash[77995]: cluster 2023-09-23T19:35:18.654585+0000 mon.a (mon.0) 1026 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:35:20.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:19 smithi016 bash[77995]: audit 2023-09-23T19:35:18.666395+0000 mon.a (mon.0) 1027 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:35:20.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:19 smithi016 bash[77995]: cluster 2023-09-23T19:35:18.666486+0000 mon.a (mon.0) 1028 : cluster 0 osdmap e137: 8 total, 8 up, 8 in 2023-09-23T19:35:20.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:19 smithi016 bash[77995]: cluster 2023-09-23T19:35:18.671800+0000 osd.5 (osd.5) 69 : cluster 3 failed to encode map e137 with expected crc 2023-09-23T19:35:20.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:19 smithi016 bash[77995]: cluster 2023-09-23T19:35:18.672486+0000 osd.7 (osd.7) 55 : cluster 3 failed to encode map e137 with expected crc 2023-09-23T19:35:20.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:19 smithi016 bash[77995]: cluster 2023-09-23T19:35:18.675775+0000 osd.7 (osd.7) 56 : cluster 3 failed to encode map e137 with expected crc 2023-09-23T19:35:20.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:19 smithi016 bash[77995]: cluster 2023-09-23T19:35:18.677460+0000 osd.6 (osd.6) 57 : cluster 3 failed to encode map e137 with expected crc 2023-09-23T19:35:20.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:19 smithi016 bash[77995]: audit 2023-09-23T19:35:19.408846+0000 mon.a (mon.0) 1029 : audit 0 from='client.? 172.21.15.16:0/1481251152' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:35:20.073 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:20.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:20.081 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:35:20.182 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:20.211 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:20.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:20.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:20.314 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:20.341 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:20.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:20.349 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:20.450 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:20.480 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:20.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:20.484 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:35:20.584 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:20.621 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:20.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:20.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:35:20.726 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:20.754 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:20.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:20.760 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:20.861 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:20.900 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:20.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:20.905 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:21.006 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:21.035 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:21.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:21.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:21.139 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:21.174 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:21.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:21.177 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:35:21.277 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:21.304 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:21.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:21.309 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:35:21.409 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:21.446 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:21.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:21.449 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:21.550 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:21.582 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:21.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:21.585 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:21.685 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:21.695 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:21 smithi016 bash[77995]: cluster 2023-09-23T19:35:20.037546+0000 mgr.y (mgr.25325) 833 : cluster 0 pgmap v743: 89 pgs: 89 active+clean; 911 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 569 op/s 2023-09-23T19:35:21.695 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:21 smithi016 bash[75004]: cluster 2023-09-23T19:35:20.037546+0000 mgr.y (mgr.25325) 833 : cluster 0 pgmap v743: 89 pgs: 89 active+clean; 911 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 569 op/s 2023-09-23T19:35:21.725 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:21.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:21.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:35:21.831 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:21.858 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:21.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:21.860 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:35:21.961 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:21.969 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:21 smithi092 bash[58855]: cluster 2023-09-23T19:35:20.037546+0000 mgr.y (mgr.25325) 833 : cluster 0 pgmap v743: 89 pgs: 89 active+clean; 911 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 569 op/s 2023-09-23T19:35:21.987 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:21.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:21.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:35:22.090 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:22.105 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T19:35:22.117 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:22.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:22.122 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:22.223 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:22.258 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:22.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:22.261 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:22.362 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:22.395 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:22.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:22.398 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:35:22.499 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:22.531 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:22.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:22.534 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:35:22.634 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:22.665 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:22.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:22.670 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:22.770 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:22.802 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:22.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:22.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:22.907 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:22.921 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:35:22.944 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:22.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:22.954 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:35:23.054 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:23.085 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:23.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:23.088 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:23.188 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:23.223 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:23.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:23.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:35:23.327 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:23.356 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:23.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:23.361 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:35:23.461 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:23.486 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:23.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:23.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:35:23.592 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:23.620 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:23.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:23.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:23.659 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:35:23.723 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:23.754 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:23.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:23.757 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:35:23.858 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:23.866 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:23 smithi092 bash[58855]: cluster 2023-09-23T19:35:22.038281+0000 mgr.y (mgr.25325) 834 : cluster 0 pgmap v744: 89 pgs: 89 active+clean; 777 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 612 op/s 2023-09-23T19:35:23.902 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:23.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:23.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:23.964 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:35:24.007 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:24.012 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:23 smithi016 bash[75004]: cluster 2023-09-23T19:35:22.038281+0000 mgr.y (mgr.25325) 834 : cluster 0 pgmap v744: 89 pgs: 89 active+clean; 777 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 612 op/s 2023-09-23T19:35:24.013 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:23 smithi016 bash[77995]: cluster 2023-09-23T19:35:22.038281+0000 mgr.y (mgr.25325) 834 : cluster 0 pgmap v744: 89 pgs: 89 active+clean; 777 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 612 op/s 2023-09-23T19:35:24.034 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:24.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:24.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:35:24.139 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:24.167 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:24.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:24.169 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:35:24.270 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:24.303 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:24.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:24.309 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:24.410 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:24.438 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:24.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:24.441 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:35:24.542 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:24.570 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:24.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:24.577 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:35:24.678 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:24.715 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:24.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:24.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:24.822 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:24.847 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:24.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:24.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:35:24.953 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:24.978 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:24.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:24.985 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:35:25.086 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:25.123 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:25.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:25.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:35:25.230 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:25.259 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:25.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:25.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:35:25.364 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:25.405 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:25.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:25.407 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:25.508 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:25.545 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:25.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:25.553 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:25.654 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:25.676 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:25.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:25.680 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:25.780 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:25.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:25 smithi092 bash[58855]: cluster 2023-09-23T19:35:24.038831+0000 mgr.y (mgr.25325) 835 : cluster 0 pgmap v745: 89 pgs: 89 active+clean; 723 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 668 op/s 2023-09-23T19:35:25.821 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:25.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:25.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:25.927 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:25.962 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:25.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:25.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:35:26.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:25 smithi016 bash[77995]: cluster 2023-09-23T19:35:24.038831+0000 mgr.y (mgr.25325) 835 : cluster 0 pgmap v745: 89 pgs: 89 active+clean; 723 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 668 op/s 2023-09-23T19:35:26.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:25 smithi016 bash[75004]: cluster 2023-09-23T19:35:24.038831+0000 mgr.y (mgr.25325) 835 : cluster 0 pgmap v745: 89 pgs: 89 active+clean; 723 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 668 op/s 2023-09-23T19:35:26.037 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:35:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:35:25] "GET /metrics HTTP/1.1" 200 36231 "" "Prometheus/2.33.4" 2023-09-23T19:35:26.068 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:26.103 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:26.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:26.109 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:35:26.210 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:26.235 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:26.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:26.240 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:35:26.340 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:26.369 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:26.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:26.372 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:35:26.472 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:26.508 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:26.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:26.513 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:26.614 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:26.649 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:26.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:26.654 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:35:26.754 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:26.774 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:26.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:26.777 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:35:26.878 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:26.924 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:26.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:26.932 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:35:26.965 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:35:27.017 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:35:27.032 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:27.067 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:27.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:27.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:27.171 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:27.209 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:27.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:27.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:27.321 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:27.367 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:27.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:27.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:27.476 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:27.501 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:27.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:27.504 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:27.605 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:27.636 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:27.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:27.639 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:27.740 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:27.776 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:27.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:27.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:35:27.789 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:27 smithi016 bash[75004]: cluster 2023-09-23T19:35:26.039464+0000 mgr.y (mgr.25325) 836 : cluster 0 pgmap v746: 89 pgs: 89 active+clean; 604 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 671 op/s 2023-09-23T19:35:27.790 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:27 smithi016 bash[75004]: audit 2023-09-23T19:35:27.111508+0000 mon.a (mon.0) 1030 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:35:27.877 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:27.920 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:27.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:27.927 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:28.027 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:28.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:27 smithi092 bash[58855]: cluster 2023-09-23T19:35:26.039464+0000 mgr.y (mgr.25325) 836 : cluster 0 pgmap v746: 89 pgs: 89 active+clean; 604 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 671 op/s 2023-09-23T19:35:28.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:27 smithi092 bash[58855]: audit 2023-09-23T19:35:27.111508+0000 mon.a (mon.0) 1030 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:35:28.051 INFO:teuthology.orchestra.run.smithi092.stdout:{ 2023-09-23T19:35:28.051 INFO:teuthology.orchestra.run.smithi092.stdout: "ops": [], 2023-09-23T19:35:28.051 INFO:teuthology.orchestra.run.smithi092.stdout: "complaint_time": 30, 2023-09-23T19:35:28.051 INFO:teuthology.orchestra.run.smithi092.stdout: "num_blocked_ops": 0 2023-09-23T19:35:28.051 INFO:teuthology.orchestra.run.smithi092.stdout:} 2023-09-23T19:35:28.052 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:28.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:28.055 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:28.133 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:27 smithi016 bash[77995]: cluster 2023-09-23T19:35:26.039464+0000 mgr.y (mgr.25325) 836 : cluster 0 pgmap v746: 89 pgs: 89 active+clean; 604 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 671 op/s 2023-09-23T19:35:28.149 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:27 smithi016 bash[77995]: audit 2023-09-23T19:35:27.111508+0000 mon.a (mon.0) 1030 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:35:28.155 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:28.190 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:28.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:28.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:28.294 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:28.322 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:28.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:28.326 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:28.427 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:28.450 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:28.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:28.454 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:28.555 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:28.577 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:28.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:28.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:28.685 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:28.726 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:28.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:28.729 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:28.830 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:28.855 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:28.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:28.859 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:35:28.958 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T19:35:28.959 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:28.986 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:28.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:28.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:29.089 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:29.129 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:29.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:29.133 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:35:29.234 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:29.265 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:29.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:29.269 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:35:29.370 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:29.397 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:29.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:29.401 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:35:29.501 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:29.538 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:29.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:29.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:29.645 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:29.650 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:29 smithi016 bash[75004]: cluster 2023-09-23T19:35:28.040204+0000 mgr.y (mgr.25325) 837 : cluster 0 pgmap v747: 89 pgs: 89 active+clean; 511 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2023-09-23T19:35:29.650 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:29 smithi016 bash[75004]: audit 2023-09-23T19:35:28.647496+0000 mon.a (mon.0) 1031 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:35:29.650 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:29 smithi016 bash[75004]: audit 2023-09-23T19:35:28.649374+0000 mon.a (mon.0) 1032 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:35:29.675 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:29.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:29.685 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:29.786 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:29.820 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:29.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:29.823 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:35:29.924 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:29.933 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:29 smithi016 bash[77995]: cluster 2023-09-23T19:35:28.040204+0000 mgr.y (mgr.25325) 837 : cluster 0 pgmap v747: 89 pgs: 89 active+clean; 511 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2023-09-23T19:35:29.933 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:29 smithi016 bash[77995]: audit 2023-09-23T19:35:28.647496+0000 mon.a (mon.0) 1031 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:35:29.934 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:29 smithi016 bash[77995]: audit 2023-09-23T19:35:28.649374+0000 mon.a (mon.0) 1032 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:35:29.952 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:29.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:29.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:35:30.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:29 smithi092 bash[58855]: cluster 2023-09-23T19:35:28.040204+0000 mgr.y (mgr.25325) 837 : cluster 0 pgmap v747: 89 pgs: 89 active+clean; 511 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2023-09-23T19:35:30.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:29 smithi092 bash[58855]: audit 2023-09-23T19:35:28.647496+0000 mon.a (mon.0) 1031 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:35:30.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:29 smithi092 bash[58855]: audit 2023-09-23T19:35:28.649374+0000 mon.a (mon.0) 1032 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:35:30.056 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:30.101 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:30.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:30.108 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:30.208 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:30.244 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:30.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:30.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:35:30.352 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:30.379 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:30.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:30.385 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:30.486 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:30.530 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:30.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:30.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:30.638 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:30.671 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:30.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:30.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:30.678 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T19:35:30.779 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:30.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:30 smithi092 bash[58855]: audit 2023-09-23T19:35:30.353657+0000 mon.a (mon.0) 1033 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T19:35:30.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:30 smithi092 bash[58855]: audit 2023-09-23T19:35:30.357257+0000 mon.b (mon.1) 41 : audit 1 from='client.? 172.21.15.16:0/1506041485' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T19:35:30.815 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:30.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:30.821 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:35:30.880 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:30 smithi016 bash[75004]: audit 2023-09-23T19:35:30.353657+0000 mon.a (mon.0) 1033 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T19:35:30.880 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:30 smithi016 bash[75004]: audit 2023-09-23T19:35:30.357257+0000 mon.b (mon.1) 41 : audit 1 from='client.? 172.21.15.16:0/1506041485' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T19:35:30.881 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:30 smithi016 bash[77995]: audit 2023-09-23T19:35:30.353657+0000 mon.a (mon.0) 1033 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T19:35:30.881 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:30 smithi016 bash[77995]: audit 2023-09-23T19:35:30.357257+0000 mon.b (mon.1) 41 : audit 1 from='client.? 172.21.15.16:0/1506041485' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T19:35:30.921 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:30.946 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:30.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:30.949 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:31.049 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:31.090 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:31.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:31.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:35:31.198 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:31.240 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:31.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:31.247 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:31.348 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:31.381 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:31.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:31.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:35:31.485 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:31.514 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:31.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:31.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:35:31.618 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:31.643 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:31.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:31.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:31.665 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:31 smithi092 bash[58855]: cluster 2023-09-23T19:35:30.040789+0000 mgr.y (mgr.25325) 838 : cluster 0 pgmap v748: 89 pgs: 89 active+clean; 487 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 630 op/s 2023-09-23T19:35:31.665 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:31 smithi092 bash[58855]: audit 2023-09-23T19:35:30.665025+0000 mon.a (mon.0) 1034 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T19:35:31.665 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:31 smithi092 bash[58855]: cluster 2023-09-23T19:35:30.665308+0000 mon.a (mon.0) 1035 : cluster 0 osdmap e138: 8 total, 8 up, 8 in 2023-09-23T19:35:31.665 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:31 smithi092 bash[58855]: cluster 2023-09-23T19:35:30.665391+0000 osd.5 (osd.5) 70 : cluster 3 failed to encode map e138 with expected crc 2023-09-23T19:35:31.666 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:31 smithi092 bash[58855]: cluster 2023-09-23T19:35:30.693367+0000 osd.4 (osd.4) 66 : cluster 3 failed to encode map e138 with expected crc 2023-09-23T19:35:31.747 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:31.780 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:31.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:31.784 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:35:31.884 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:31.896 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:31 smithi016 bash[77995]: cluster 2023-09-23T19:35:30.040789+0000 mgr.y (mgr.25325) 838 : cluster 0 pgmap v748: 89 pgs: 89 active+clean; 487 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 630 op/s 2023-09-23T19:35:31.896 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:31 smithi016 bash[77995]: audit 2023-09-23T19:35:30.665025+0000 mon.a (mon.0) 1034 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T19:35:31.896 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:31 smithi016 bash[77995]: cluster 2023-09-23T19:35:30.665308+0000 mon.a (mon.0) 1035 : cluster 0 osdmap e138: 8 total, 8 up, 8 in 2023-09-23T19:35:31.896 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:31 smithi016 bash[77995]: cluster 2023-09-23T19:35:30.665391+0000 osd.5 (osd.5) 70 : cluster 3 failed to encode map e138 with expected crc 2023-09-23T19:35:31.896 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:31 smithi016 bash[77995]: cluster 2023-09-23T19:35:30.693367+0000 osd.4 (osd.4) 66 : cluster 3 failed to encode map e138 with expected crc 2023-09-23T19:35:31.897 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:31 smithi016 bash[75004]: cluster 2023-09-23T19:35:30.040789+0000 mgr.y (mgr.25325) 838 : cluster 0 pgmap v748: 89 pgs: 89 active+clean; 487 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 630 op/s 2023-09-23T19:35:31.897 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:31 smithi016 bash[75004]: audit 2023-09-23T19:35:30.665025+0000 mon.a (mon.0) 1034 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T19:35:31.897 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:31 smithi016 bash[75004]: cluster 2023-09-23T19:35:30.665308+0000 mon.a (mon.0) 1035 : cluster 0 osdmap e138: 8 total, 8 up, 8 in 2023-09-23T19:35:31.897 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:31 smithi016 bash[75004]: cluster 2023-09-23T19:35:30.665391+0000 osd.5 (osd.5) 70 : cluster 3 failed to encode map e138 with expected crc 2023-09-23T19:35:31.897 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:31 smithi016 bash[75004]: cluster 2023-09-23T19:35:30.693367+0000 osd.4 (osd.4) 66 : cluster 3 failed to encode map e138 with expected crc 2023-09-23T19:35:31.919 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:31.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:31.927 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:32.028 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:32.062 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:32.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:32.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:32.166 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:32.196 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:32.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:32.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:32.301 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:32.327 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:32.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:32.333 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:35:32.433 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:32.466 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:32.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:32.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:32.570 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:32.599 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:32.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:32.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:32.708 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:32.736 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:32.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:32.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:32.842 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:32.868 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:32.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:32.871 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:35:32.971 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:33.016 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:33.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:33.022 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:33.123 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:33.165 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:33.167 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T19:35:33.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:33.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:33.272 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:33.314 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:33.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:33.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:33.418 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:33.455 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:33.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:33.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:35:33.525 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:35:33.560 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:33.592 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:33.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:33.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:35:33.695 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:33.711 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:33 smithi092 bash[58855]: cluster 2023-09-23T19:35:32.041363+0000 mgr.y (mgr.25325) 839 : cluster 0 pgmap v750: 89 pgs: 89 active+clean; 363 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2023-09-23T19:35:33.729 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:33.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:33.732 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:35:33.833 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:33.840 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:33 smithi016 bash[75004]: cluster 2023-09-23T19:35:32.041363+0000 mgr.y (mgr.25325) 839 : cluster 0 pgmap v750: 89 pgs: 89 active+clean; 363 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2023-09-23T19:35:33.846 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:33 smithi016 bash[77995]: cluster 2023-09-23T19:35:32.041363+0000 mgr.y (mgr.25325) 839 : cluster 0 pgmap v750: 89 pgs: 89 active+clean; 363 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2023-09-23T19:35:33.865 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:33.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:33.869 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:33.970 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:34.023 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:34.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:34.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:35:34.134 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:34.171 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:34.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:34.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:35:34.276 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:34.314 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:34.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:34.323 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:34.424 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:34.459 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:34.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:34.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:34.565 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:34.595 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:34.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:34.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:34.705 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:34.739 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:34.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:34.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:35:34.843 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:34.900 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:34.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:34.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:35.007 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:35.033 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:35.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:35.036 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:35:35.137 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:35.163 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:35.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:35.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:35.268 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:35.277 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:35 smithi092 bash[58855]: cluster 2023-09-23T19:35:34.041936+0000 mgr.y (mgr.25325) 840 : cluster 0 pgmap v751: 89 pgs: 89 active+clean; 311 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 657 op/s 2023-09-23T19:35:35.277 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:35 smithi092 bash[58855]: audit 2023-09-23T19:35:34.233975+0000 mon.a (mon.0) 1036 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:35:35.277 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:35 smithi092 bash[58855]: audit 2023-09-23T19:35:34.251523+0000 mon.a (mon.0) 1037 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:35:35.297 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:35.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:35.299 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:35.400 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:35.423 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:35.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:35.426 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:35:35.527 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:35.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:35 smithi016 bash[75004]: cluster 2023-09-23T19:35:34.041936+0000 mgr.y (mgr.25325) 840 : cluster 0 pgmap v751: 89 pgs: 89 active+clean; 311 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 657 op/s 2023-09-23T19:35:35.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:35 smithi016 bash[75004]: audit 2023-09-23T19:35:34.233975+0000 mon.a (mon.0) 1036 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:35:35.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:35 smithi016 bash[75004]: audit 2023-09-23T19:35:34.251523+0000 mon.a (mon.0) 1037 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:35:35.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:35 smithi016 bash[77995]: cluster 2023-09-23T19:35:34.041936+0000 mgr.y (mgr.25325) 840 : cluster 0 pgmap v751: 89 pgs: 89 active+clean; 311 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 657 op/s 2023-09-23T19:35:35.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:35 smithi016 bash[77995]: audit 2023-09-23T19:35:34.233975+0000 mon.a (mon.0) 1036 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:35:35.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:35 smithi016 bash[77995]: audit 2023-09-23T19:35:34.251523+0000 mon.a (mon.0) 1037 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:35:35.553 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:35.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:35.556 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:35:35.641 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:35:35 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:35:35] "GET /metrics HTTP/1.1" 200 36229 "" "Prometheus/2.33.4" 2023-09-23T19:35:35.656 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:35.679 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:35:35.693 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:35.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:35.696 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:35:35.722 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:35:35.765 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:35:35.797 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:35.822 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:35:35.836 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:35.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:35.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:35.865 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:35:35.908 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:35:35.943 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:35.943 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:35:35.966 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:35.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:35.971 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:35:35.977 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:35:35.980 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:35:35.980 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:35:35.980 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:35:35.980 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:35:35.981 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:35:35.981 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:35:35.981 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:35:35.981 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:35:36.072 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:36.100 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:36.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:36.106 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:36.206 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:36.252 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:36.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:36.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:35:36.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:36 smithi016 bash[75004]: audit 2023-09-23T19:35:35.238663+0000 mon.a (mon.0) 1038 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:35:36.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:36 smithi016 bash[75004]: audit 2023-09-23T19:35:35.246896+0000 mon.a (mon.0) 1039 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:35:36.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:36 smithi016 bash[75004]: audit 2023-09-23T19:35:35.618561+0000 mon.a (mon.0) 1040 : audit 0 from='client.? 172.21.15.16:0/2523080591' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:35:36.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:36 smithi016 bash[75004]: audit 2023-09-23T19:35:35.924854+0000 mon.a (mon.0) 1041 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:35:36.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:36 smithi016 bash[75004]: audit 2023-09-23T19:35:35.926311+0000 mon.a (mon.0) 1042 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:35:36.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:36 smithi016 bash[75004]: audit 2023-09-23T19:35:35.941226+0000 mon.a (mon.0) 1043 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:35:36.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:36 smithi016 bash[75004]: audit 2023-09-23T19:35:36.084582+0000 mon.c (mon.2) 155 : audit 1 from='client.? 172.21.15.16:0/2074378588' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:35:36.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:36 smithi016 bash[75004]: audit 2023-09-23T19:35:36.085633+0000 mon.a (mon.0) 1044 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:35:36.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:36 smithi016 bash[77995]: audit 2023-09-23T19:35:35.238663+0000 mon.a (mon.0) 1038 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:35:36.288 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:36 smithi016 bash[77995]: audit 2023-09-23T19:35:35.246896+0000 mon.a (mon.0) 1039 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:35:36.288 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:36 smithi016 bash[77995]: audit 2023-09-23T19:35:35.618561+0000 mon.a (mon.0) 1040 : audit 0 from='client.? 172.21.15.16:0/2523080591' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:35:36.288 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:36 smithi016 bash[77995]: audit 2023-09-23T19:35:35.924854+0000 mon.a (mon.0) 1041 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:35:36.288 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:36 smithi016 bash[77995]: audit 2023-09-23T19:35:35.926311+0000 mon.a (mon.0) 1042 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:35:36.288 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:36 smithi016 bash[77995]: audit 2023-09-23T19:35:35.941226+0000 mon.a (mon.0) 1043 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:35:36.288 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:36 smithi016 bash[77995]: audit 2023-09-23T19:35:36.084582+0000 mon.c (mon.2) 155 : audit 1 from='client.? 172.21.15.16:0/2074378588' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:35:36.289 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:36 smithi016 bash[77995]: audit 2023-09-23T19:35:36.085633+0000 mon.a (mon.0) 1044 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:35:36.356 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:36.393 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:36.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:36.397 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:35:36.498 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:36.509 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:36 smithi092 bash[58855]: audit 2023-09-23T19:35:35.238663+0000 mon.a (mon.0) 1038 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:35:36.509 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:36 smithi092 bash[58855]: audit 2023-09-23T19:35:35.246896+0000 mon.a (mon.0) 1039 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:35:36.509 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:36 smithi092 bash[58855]: audit 2023-09-23T19:35:35.618561+0000 mon.a (mon.0) 1040 : audit 0 from='client.? 172.21.15.16:0/2523080591' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:35:36.509 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:36 smithi092 bash[58855]: audit 2023-09-23T19:35:35.924854+0000 mon.a (mon.0) 1041 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:35:36.509 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:36 smithi092 bash[58855]: audit 2023-09-23T19:35:35.926311+0000 mon.a (mon.0) 1042 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:35:36.509 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:36 smithi092 bash[58855]: audit 2023-09-23T19:35:35.941226+0000 mon.a (mon.0) 1043 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:35:36.509 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:36 smithi092 bash[58855]: audit 2023-09-23T19:35:36.084582+0000 mon.c (mon.2) 155 : audit 1 from='client.? 172.21.15.16:0/2074378588' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:35:36.510 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:36 smithi092 bash[58855]: audit 2023-09-23T19:35:36.085633+0000 mon.a (mon.0) 1044 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:35:36.526 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:36.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:36.529 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:35:36.630 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:36.660 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:36.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:36.665 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:35:36.766 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:36.794 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:36.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:36.797 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:36.897 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:36.923 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:36.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:36.927 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:35:36.971 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is set 2023-09-23T19:35:37.028 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:37.069 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:37.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:37.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:37.176 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:37.202 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:37.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:37.205 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:37.306 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:37.330 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:37.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:37.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:35:37.437 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:37.448 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:37 smithi016 bash[75004]: cluster 2023-09-23T19:35:36.042566+0000 mgr.y (mgr.25325) 841 : cluster 0 pgmap v752: 89 pgs: 89 active+clean; 202 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 642 op/s 2023-09-23T19:35:37.448 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:37 smithi016 bash[75004]: cluster 2023-09-23T19:35:36.933461+0000 mon.a (mon.0) 1045 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:35:37.449 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:37 smithi016 bash[75004]: cluster 2023-09-23T19:35:36.951687+0000 osd.4 (osd.4) 67 : cluster 3 failed to encode map e139 with expected crc 2023-09-23T19:35:37.449 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:37 smithi016 bash[75004]: audit 2023-09-23T19:35:36.956256+0000 mon.a (mon.0) 1046 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:35:37.449 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:37 smithi016 bash[75004]: cluster 2023-09-23T19:35:36.956343+0000 mon.a (mon.0) 1047 : cluster 0 osdmap e139: 8 total, 8 up, 8 in 2023-09-23T19:35:37.449 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:37 smithi016 bash[75004]: cluster 2023-09-23T19:35:36.963363+0000 osd.5 (osd.5) 71 : cluster 3 failed to encode map e139 with expected crc 2023-09-23T19:35:37.449 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:37 smithi016 bash[75004]: cluster 2023-09-23T19:35:36.978198+0000 osd.6 (osd.6) 58 : cluster 3 failed to encode map e139 with expected crc 2023-09-23T19:35:37.449 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:37 smithi016 bash[77995]: cluster 2023-09-23T19:35:36.042566+0000 mgr.y (mgr.25325) 841 : cluster 0 pgmap v752: 89 pgs: 89 active+clean; 202 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 642 op/s 2023-09-23T19:35:37.449 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:37 smithi016 bash[77995]: cluster 2023-09-23T19:35:36.933461+0000 mon.a (mon.0) 1045 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:35:37.449 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:37 smithi016 bash[77995]: cluster 2023-09-23T19:35:36.951687+0000 osd.4 (osd.4) 67 : cluster 3 failed to encode map e139 with expected crc 2023-09-23T19:35:37.449 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:37 smithi016 bash[77995]: audit 2023-09-23T19:35:36.956256+0000 mon.a (mon.0) 1046 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:35:37.450 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:37 smithi016 bash[77995]: cluster 2023-09-23T19:35:36.956343+0000 mon.a (mon.0) 1047 : cluster 0 osdmap e139: 8 total, 8 up, 8 in 2023-09-23T19:35:37.450 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:37 smithi016 bash[77995]: cluster 2023-09-23T19:35:36.963363+0000 osd.5 (osd.5) 71 : cluster 3 failed to encode map e139 with expected crc 2023-09-23T19:35:37.450 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:37 smithi016 bash[77995]: cluster 2023-09-23T19:35:36.978198+0000 osd.6 (osd.6) 58 : cluster 3 failed to encode map e139 with expected crc 2023-09-23T19:35:37.465 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:37.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:37.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:37.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:37 smithi092 bash[58855]: cluster 2023-09-23T19:35:36.042566+0000 mgr.y (mgr.25325) 841 : cluster 0 pgmap v752: 89 pgs: 89 active+clean; 202 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 642 op/s 2023-09-23T19:35:37.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:37 smithi092 bash[58855]: cluster 2023-09-23T19:35:36.933461+0000 mon.a (mon.0) 1045 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:35:37.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:37 smithi092 bash[58855]: cluster 2023-09-23T19:35:36.951687+0000 osd.4 (osd.4) 67 : cluster 3 failed to encode map e139 with expected crc 2023-09-23T19:35:37.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:37 smithi092 bash[58855]: audit 2023-09-23T19:35:36.956256+0000 mon.a (mon.0) 1046 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:35:37.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:37 smithi092 bash[58855]: cluster 2023-09-23T19:35:36.956343+0000 mon.a (mon.0) 1047 : cluster 0 osdmap e139: 8 total, 8 up, 8 in 2023-09-23T19:35:37.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:37 smithi092 bash[58855]: cluster 2023-09-23T19:35:36.963363+0000 osd.5 (osd.5) 71 : cluster 3 failed to encode map e139 with expected crc 2023-09-23T19:35:37.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:37 smithi092 bash[58855]: cluster 2023-09-23T19:35:36.978198+0000 osd.6 (osd.6) 58 : cluster 3 failed to encode map e139 with expected crc 2023-09-23T19:35:37.568 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:37.592 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:37.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:37.599 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:35:37.700 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:37.730 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:37.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:37.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:37.833 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:37.874 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:37.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:37.877 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:35:37.978 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:38.009 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:38.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:38.012 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:35:38.112 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:38.117 INFO:teuthology.orchestra.run.smithi092.stdout:ERROR: (22) Invalid argument 2023-09-23T19:35:38.117 INFO:teuthology.orchestra.run.smithi092.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:35:38.147 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:38.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:38.153 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:38.253 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:38.285 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:38.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:38.292 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:38.393 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:38.416 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:38.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:38.420 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:35:38.520 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:38.530 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:38 smithi016 bash[77995]: cluster 2023-09-23T19:35:36.963403+0000 osd.7 (osd.7) 57 : cluster 3 failed to encode map e139 with expected crc 2023-09-23T19:35:38.530 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:38 smithi016 bash[75004]: cluster 2023-09-23T19:35:36.963403+0000 osd.7 (osd.7) 57 : cluster 3 failed to encode map e139 with expected crc 2023-09-23T19:35:38.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:38 smithi092 bash[58855]: cluster 2023-09-23T19:35:36.963403+0000 osd.7 (osd.7) 57 : cluster 3 failed to encode map e139 with expected crc 2023-09-23T19:35:38.557 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:38.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:38.566 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:35:38.666 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:38.703 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:38.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:38.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:35:38.807 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:38.841 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:38.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:38.844 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:38.868 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Removed 108379 objects 2023-09-23T19:35:38.870 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Clean up completed and total clean up time :154.561 2023-09-23T19:35:38.945 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:38.977 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:38.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:38.980 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:39.040 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:35:39.081 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:39.116 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:39.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:39.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:39.154 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T19:35:39.220 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:39.251 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:39.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:39.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:39.355 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:39.368 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:39 smithi016 bash[75004]: cluster 2023-09-23T19:35:38.043161+0000 mgr.y (mgr.25325) 842 : cluster 0 pgmap v754: 89 pgs: 89 active+clean; 115 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2023-09-23T19:35:39.368 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:39 smithi016 bash[77995]: cluster 2023-09-23T19:35:38.043161+0000 mgr.y (mgr.25325) 842 : cluster 0 pgmap v754: 89 pgs: 89 active+clean; 115 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2023-09-23T19:35:39.390 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:39.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:39.393 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:35:39.410 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:35:39.493 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:39.504 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:39 smithi092 bash[58855]: cluster 2023-09-23T19:35:38.043161+0000 mgr.y (mgr.25325) 842 : cluster 0 pgmap v754: 89 pgs: 89 active+clean; 115 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2023-09-23T19:35:39.525 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:39.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:39.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:35:39.628 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:39.659 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:39.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:39.662 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:35:39.762 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:39.796 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:39.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:39.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:39.900 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:39.914 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T19:35:39.931 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:39.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:39.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:35:40.035 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:40.064 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:40.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:40.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:35:40.167 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:40.198 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:40.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:40.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:40.301 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:40.331 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:40.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:40.334 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:35:40.435 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:40.466 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:40.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:40.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:40.570 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:40.599 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:40.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:40.602 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:40.703 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:40.734 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:40.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:40.736 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:40.837 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:40.868 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:40.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:40.872 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:35:40.972 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:41.003 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:41.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:41.005 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:35:41.106 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:41.139 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:41.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:41.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:35:41.243 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:41.269 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:41.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:41.272 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:41.372 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:41.401 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:41.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:41.404 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:41.504 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:41.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:41 smithi016 bash[77995]: cluster 2023-09-23T19:35:40.043916+0000 mgr.y (mgr.25325) 843 : cluster 0 pgmap v755: 89 pgs: 89 active+clean; 96 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 521 op/s 2023-09-23T19:35:41.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:41 smithi016 bash[75004]: cluster 2023-09-23T19:35:40.043916+0000 mgr.y (mgr.25325) 843 : cluster 0 pgmap v755: 89 pgs: 89 active+clean; 96 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 521 op/s 2023-09-23T19:35:41.538 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:41.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:41.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:35:41.642 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:41.655 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:41 smithi092 bash[58855]: cluster 2023-09-23T19:35:40.043916+0000 mgr.y (mgr.25325) 843 : cluster 0 pgmap v755: 89 pgs: 89 active+clean; 96 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 521 op/s 2023-09-23T19:35:41.675 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:41.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:41.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:35:41.779 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:41.809 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:41.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:41.813 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:41.913 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:41.941 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:41.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:41.945 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:42.045 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:42.078 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:42.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:42.082 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:42.182 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:42.212 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:42.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:42.215 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:42.316 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:42.348 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:42.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:42.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:42.452 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:42.481 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:42.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:42.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:35:42.585 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:42.615 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:42.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:42.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:42.719 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:42.751 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:42.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:42.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:35:42.856 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:42.888 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:42.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:42.891 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:42.991 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:43.023 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:43.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:43.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:43.126 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:43.159 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:43.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:43.162 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:43.263 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:43.294 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:43.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:43.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:35:43.387 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T19:35:43.398 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:43.412 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:43 smithi016 bash[77995]: cluster 2023-09-23T19:35:42.044918+0000 mgr.y (mgr.25325) 844 : cluster 0 pgmap v756: 89 pgs: 89 active+clean; 15 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 556 op/s 2023-09-23T19:35:43.412 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:43 smithi016 bash[77995]: audit 2023-09-23T19:35:43.245213+0000 mon.c (mon.2) 156 : audit 1 from='client.? 172.21.15.16:0/977471590' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T19:35:43.413 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:43 smithi016 bash[77995]: audit 2023-09-23T19:35:43.245955+0000 mon.a (mon.0) 1048 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T19:35:43.413 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:43 smithi016 bash[75004]: cluster 2023-09-23T19:35:42.044918+0000 mgr.y (mgr.25325) 844 : cluster 0 pgmap v756: 89 pgs: 89 active+clean; 15 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 556 op/s 2023-09-23T19:35:43.413 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:43 smithi016 bash[75004]: audit 2023-09-23T19:35:43.245213+0000 mon.c (mon.2) 156 : audit 1 from='client.? 172.21.15.16:0/977471590' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T19:35:43.413 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:43 smithi016 bash[75004]: audit 2023-09-23T19:35:43.245955+0000 mon.a (mon.0) 1048 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T19:35:43.433 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:43.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:43.437 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:35:43.537 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:43.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:43 smithi092 bash[58855]: cluster 2023-09-23T19:35:42.044918+0000 mgr.y (mgr.25325) 844 : cluster 0 pgmap v756: 89 pgs: 89 active+clean; 15 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 556 op/s 2023-09-23T19:35:43.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:43 smithi092 bash[58855]: audit 2023-09-23T19:35:43.245213+0000 mon.c (mon.2) 156 : audit 1 from='client.? 172.21.15.16:0/977471590' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T19:35:43.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:43 smithi092 bash[58855]: audit 2023-09-23T19:35:43.245955+0000 mon.a (mon.0) 1048 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T19:35:43.566 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:43.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:43.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:35:43.670 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:43.703 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:43.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:43.707 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:35:43.807 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:43.834 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:43.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:43.838 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:43.933 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:35:43.938 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:43.967 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:43.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:43.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:35:44.070 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:44.101 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:44.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:44.104 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:35:44.205 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:44.231 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:44.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:44.234 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:35:44.307 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T19:35:44.334 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:44.380 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:44.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:44.383 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:44.399 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_4 2023-09-23T19:35:44.400 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_4 unique_pool_4 --yes-i-really-really-mean-it 2023-09-23T19:35:44.484 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:44.496 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:44 smithi092 bash[58855]: cluster 2023-09-23T19:35:43.378184+0000 osd.7 (osd.7) 58 : cluster 3 failed to encode map e140 with expected crc 2023-09-23T19:35:44.496 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:44 smithi092 bash[58855]: audit 2023-09-23T19:35:43.378637+0000 mon.a (mon.0) 1049 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T19:35:44.496 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:44 smithi092 bash[58855]: cluster 2023-09-23T19:35:43.378858+0000 mon.a (mon.0) 1050 : cluster 0 osdmap e140: 8 total, 8 up, 8 in 2023-09-23T19:35:44.496 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:44 smithi092 bash[58855]: audit 2023-09-23T19:35:43.642954+0000 mon.a (mon.0) 1051 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:35:44.511 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:44.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:44.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:35:44.614 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:44.628 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:44 smithi016 bash[75004]: cluster 2023-09-23T19:35:43.378184+0000 osd.7 (osd.7) 58 : cluster 3 failed to encode map e140 with expected crc 2023-09-23T19:35:44.628 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:44 smithi016 bash[75004]: audit 2023-09-23T19:35:43.378637+0000 mon.a (mon.0) 1049 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T19:35:44.628 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:44 smithi016 bash[75004]: cluster 2023-09-23T19:35:43.378858+0000 mon.a (mon.0) 1050 : cluster 0 osdmap e140: 8 total, 8 up, 8 in 2023-09-23T19:35:44.628 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:44 smithi016 bash[75004]: audit 2023-09-23T19:35:43.642954+0000 mon.a (mon.0) 1051 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:35:44.628 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:44 smithi016 bash[77995]: cluster 2023-09-23T19:35:43.378184+0000 osd.7 (osd.7) 58 : cluster 3 failed to encode map e140 with expected crc 2023-09-23T19:35:44.628 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:44 smithi016 bash[77995]: audit 2023-09-23T19:35:43.378637+0000 mon.a (mon.0) 1049 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T19:35:44.629 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:44 smithi016 bash[77995]: cluster 2023-09-23T19:35:43.378858+0000 mon.a (mon.0) 1050 : cluster 0 osdmap e140: 8 total, 8 up, 8 in 2023-09-23T19:35:44.629 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:44 smithi016 bash[77995]: audit 2023-09-23T19:35:43.642954+0000 mon.a (mon.0) 1051 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:35:44.645 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:44.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:44.649 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:35:44.750 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:44.782 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:44.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:44.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:44.790 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:35:44.791 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:35:44.887 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:44.918 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:44.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:44.921 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:45.022 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:45.047 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:45.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:45.050 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:45.151 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:45.180 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:45.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:45.182 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:45.283 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:45.314 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:45.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:45.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:45.418 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:45.444 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:45.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:45.447 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:45.548 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:45.559 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:45 smithi016 bash[77995]: cluster 2023-09-23T19:35:44.045638+0000 mgr.y (mgr.25325) 845 : cluster 0 pgmap v758: 89 pgs: 89 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 402 op/s 2023-09-23T19:35:45.559 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:45 smithi016 bash[77995]: cluster 2023-09-23T19:35:44.176585+0000 osd.5 (osd.5) 72 : cluster 3 failed to encode map e140 with expected crc 2023-09-23T19:35:45.559 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:45 smithi016 bash[77995]: cluster 2023-09-23T19:35:44.559150+0000 osd.4 (osd.4) 68 : cluster 3 failed to encode map e140 with expected crc 2023-09-23T19:35:45.560 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:45 smithi016 bash[75004]: cluster 2023-09-23T19:35:44.045638+0000 mgr.y (mgr.25325) 845 : cluster 0 pgmap v758: 89 pgs: 89 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 402 op/s 2023-09-23T19:35:45.560 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:45 smithi016 bash[75004]: cluster 2023-09-23T19:35:44.176585+0000 osd.5 (osd.5) 72 : cluster 3 failed to encode map e140 with expected crc 2023-09-23T19:35:45.560 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:45 smithi016 bash[75004]: cluster 2023-09-23T19:35:44.559150+0000 osd.4 (osd.4) 68 : cluster 3 failed to encode map e140 with expected crc 2023-09-23T19:35:45.574 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:45.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:45.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:45.677 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:45.706 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:45.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:45.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:35:45.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:45 smithi092 bash[58855]: cluster 2023-09-23T19:35:44.045638+0000 mgr.y (mgr.25325) 845 : cluster 0 pgmap v758: 89 pgs: 89 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 402 op/s 2023-09-23T19:35:45.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:45 smithi092 bash[58855]: cluster 2023-09-23T19:35:44.176585+0000 osd.5 (osd.5) 72 : cluster 3 failed to encode map e140 with expected crc 2023-09-23T19:35:45.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:45 smithi092 bash[58855]: cluster 2023-09-23T19:35:44.559150+0000 osd.4 (osd.4) 68 : cluster 3 failed to encode map e140 with expected crc 2023-09-23T19:35:45.811 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:45.823 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:35:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:35:45] "GET /metrics HTTP/1.1" 200 36183 "" "Prometheus/2.33.4" 2023-09-23T19:35:45.842 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:45.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:45.845 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:45.946 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:45.974 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:45.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:45.978 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:46.078 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:46.104 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:46.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:46.106 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:46.207 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:46.232 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:46.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:46.234 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:35:46.335 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:46.362 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:46.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:46.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:46.466 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:46.494 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:46.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:46.497 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:35:46.598 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:46.628 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:46.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:46.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:35:46.732 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:46.760 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:46.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:46.763 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:35:46.863 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:46.887 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:46.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:46.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:35:46.990 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:47.011 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:47.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:47.013 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:47.114 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:47.136 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:47.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:47.139 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:35:47.239 INFO:teuthology.orchestra.run.smithi092.stdout:ERROR: (22) Invalid argument 2023-09-23T19:35:47.240 INFO:teuthology.orchestra.run.smithi092.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:35:47.240 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:47.268 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:47.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:47.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:47.371 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:47.403 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:47.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:47.406 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:47.507 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:47.516 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:47 smithi016 bash[75004]: cluster 2023-09-23T19:35:46.046715+0000 mgr.y (mgr.25325) 846 : cluster 0 pgmap v759: 89 pgs: 89 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 290 op/s 2023-09-23T19:35:47.517 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:47 smithi016 bash[77995]: cluster 2023-09-23T19:35:46.046715+0000 mgr.y (mgr.25325) 846 : cluster 0 pgmap v759: 89 pgs: 89 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 290 op/s 2023-09-23T19:35:47.534 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:47.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:47.536 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:35:47.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:47 smithi092 bash[58855]: cluster 2023-09-23T19:35:46.046715+0000 mgr.y (mgr.25325) 846 : cluster 0 pgmap v759: 89 pgs: 89 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 290 op/s 2023-09-23T19:35:47.637 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:47.670 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:47.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:47.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:47.774 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:47.800 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:47.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:47.803 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:47.903 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:47.930 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:47.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:47.933 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:35:48.033 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:48.057 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:48.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:48.060 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:48.069 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T19:35:48.161 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:48.190 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:48.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:48.193 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:48.294 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:48.324 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:48.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:48.327 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:35:48.427 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:48.456 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:48.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:48.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:35:48.559 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:48.583 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:48.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:48.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:48.688 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:48.713 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:48.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:48.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:48.817 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:48.850 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:48.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:48.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:35:48.859 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:35:48.860 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:35:48.954 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:48.986 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:48.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:48.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:35:49.089 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:49.118 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:49.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:49.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:49.222 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:49.245 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:49.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:49.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:49.347 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:49.370 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:49.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:49.373 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:49.473 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:49.501 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:49.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:49.504 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:35:49.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:49 smithi016 bash[75004]: cluster 2023-09-23T19:35:48.047490+0000 mgr.y (mgr.25325) 847 : cluster 0 pgmap v760: 89 pgs: 89 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 183 op/s 2023-09-23T19:35:49.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:49 smithi016 bash[77995]: cluster 2023-09-23T19:35:48.047490+0000 mgr.y (mgr.25325) 847 : cluster 0 pgmap v760: 89 pgs: 89 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 183 op/s 2023-09-23T19:35:49.605 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:49.642 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:49.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:49.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:49.746 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:49.759 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:49 smithi092 bash[58855]: cluster 2023-09-23T19:35:48.047490+0000 mgr.y (mgr.25325) 847 : cluster 0 pgmap v760: 89 pgs: 89 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 183 op/s 2023-09-23T19:35:49.782 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:49.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:49.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:49.886 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:49.919 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:49.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:49.921 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:35:50.022 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:50.047 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:50.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:50.050 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:35:50.150 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:50.182 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:50.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:50.185 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:50.286 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:50.318 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:50.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:50.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:50.421 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:50.453 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:50.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:50.456 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:35:50.556 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:50.586 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:50.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:50.589 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:35:50.646 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:35:50.668 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:35:50.688 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:35:50.690 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:50.709 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:35:50.717 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:50.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:50.720 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:50.732 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:35:50.759 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:35:50.786 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:35:50.813 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:35:50.815 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:35:50.815 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:35:50.815 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:35:50.816 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:35:50.816 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:35:50.816 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:35:50.816 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:35:50.816 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:35:50.821 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:50.850 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:50.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:50.853 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:35:50.954 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:50.974 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:50.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:50.977 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:35:51.078 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:51.111 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:51.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:51.114 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:51.215 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:51.250 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:51.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:51.253 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:35:51.353 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:51.387 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:51.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:51.390 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:35:51.417 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is unset 2023-09-23T19:35:51.492 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:51.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:51 smithi016 bash[75004]: cluster 2023-09-23T19:35:50.048140+0000 mgr.y (mgr.25325) 848 : cluster 0 pgmap v761: 89 pgs: 89 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 152 op/s 2023-09-23T19:35:51.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:51 smithi016 bash[75004]: audit 2023-09-23T19:35:50.621116+0000 mon.a (mon.0) 1052 : audit 0 from='client.? 172.21.15.16:0/3537000802' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:35:51.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:51 smithi016 bash[75004]: audit 2023-09-23T19:35:51.203185+0000 mon.c (mon.2) 157 : audit 1 from='client.? 172.21.15.16:0/3110387767' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:35:51.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:51 smithi016 bash[75004]: audit 2023-09-23T19:35:51.203811+0000 mon.a (mon.0) 1053 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:35:51.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:51 smithi016 bash[77995]: cluster 2023-09-23T19:35:50.048140+0000 mgr.y (mgr.25325) 848 : cluster 0 pgmap v761: 89 pgs: 89 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 152 op/s 2023-09-23T19:35:51.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:51 smithi016 bash[77995]: audit 2023-09-23T19:35:50.621116+0000 mon.a (mon.0) 1052 : audit 0 from='client.? 172.21.15.16:0/3537000802' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:35:51.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:51 smithi016 bash[77995]: audit 2023-09-23T19:35:51.203185+0000 mon.c (mon.2) 157 : audit 1 from='client.? 172.21.15.16:0/3110387767' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:35:51.503 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:51 smithi016 bash[77995]: audit 2023-09-23T19:35:51.203811+0000 mon.a (mon.0) 1053 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:35:51.515 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:51.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:51.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:51.618 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:51.632 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:51 smithi092 bash[58855]: cluster 2023-09-23T19:35:50.048140+0000 mgr.y (mgr.25325) 848 : cluster 0 pgmap v761: 89 pgs: 89 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 152 op/s 2023-09-23T19:35:51.632 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:51 smithi092 bash[58855]: audit 2023-09-23T19:35:50.621116+0000 mon.a (mon.0) 1052 : audit 0 from='client.? 172.21.15.16:0/3537000802' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:35:51.632 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:51 smithi092 bash[58855]: audit 2023-09-23T19:35:51.203185+0000 mon.c (mon.2) 157 : audit 1 from='client.? 172.21.15.16:0/3110387767' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:35:51.632 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:51 smithi092 bash[58855]: audit 2023-09-23T19:35:51.203811+0000 mon.a (mon.0) 1053 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:35:51.653 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:51.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:51.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:35:51.756 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:51.778 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:51.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:51.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:51.882 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:51.903 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:51.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:51.905 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:52.006 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:52.033 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:52.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:52.036 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:35:52.136 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:52.167 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:52.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:52.171 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:35:52.272 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:52.294 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:52.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:52.297 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:52.398 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:52.424 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:52.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:52.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:52.528 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:52.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:52 smithi016 bash[75004]: cluster 2023-09-23T19:35:51.398969+0000 mon.a (mon.0) 1054 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:35:52.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:52 smithi016 bash[75004]: audit 2023-09-23T19:35:51.409415+0000 mon.a (mon.0) 1055 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:35:52.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:52 smithi016 bash[75004]: cluster 2023-09-23T19:35:51.409541+0000 mon.a (mon.0) 1056 : cluster 0 osdmap e141: 8 total, 8 up, 8 in 2023-09-23T19:35:52.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:52 smithi016 bash[77995]: cluster 2023-09-23T19:35:51.398969+0000 mon.a (mon.0) 1054 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:35:52.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:52 smithi016 bash[77995]: audit 2023-09-23T19:35:51.409415+0000 mon.a (mon.0) 1055 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:35:52.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:52 smithi016 bash[77995]: cluster 2023-09-23T19:35:51.409541+0000 mon.a (mon.0) 1056 : cluster 0 osdmap e141: 8 total, 8 up, 8 in 2023-09-23T19:35:52.560 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:52.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:52.564 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:35:52.664 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:52.686 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:52.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:52.688 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:52.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:52 smithi092 bash[58855]: cluster 2023-09-23T19:35:51.398969+0000 mon.a (mon.0) 1054 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:35:52.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:52 smithi092 bash[58855]: audit 2023-09-23T19:35:51.409415+0000 mon.a (mon.0) 1055 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:35:52.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:52 smithi092 bash[58855]: cluster 2023-09-23T19:35:51.409541+0000 mon.a (mon.0) 1056 : cluster 0 osdmap e141: 8 total, 8 up, 8 in 2023-09-23T19:35:52.790 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:52.818 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:52.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:52.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:52.854 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:35:52.921 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:52.951 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:52.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:52.954 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:35:53.054 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:53.081 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:53.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:53.084 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:53.184 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:53.205 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:53.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:53.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:53.309 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:53.336 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:53.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:53.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:35:53.440 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:53.469 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:53.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:53.471 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:35:53.572 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:53.601 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:53.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:53.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:35:53.705 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:53.718 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:53 smithi016 bash[77995]: cluster 2023-09-23T19:35:52.048938+0000 mgr.y (mgr.25325) 849 : cluster 0 pgmap v763: 89 pgs: 89 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:35:53.718 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:53 smithi016 bash[77995]: cluster 2023-09-23T19:35:52.207267+0000 osd.5 (osd.5) 73 : cluster 3 failed to encode map e141 with expected crc 2023-09-23T19:35:53.718 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:53 smithi016 bash[77995]: cluster 2023-09-23T19:35:52.482897+0000 osd.6 (osd.6) 59 : cluster 3 failed to encode map e141 with expected crc 2023-09-23T19:35:53.719 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:53 smithi016 bash[77995]: cluster 2023-09-23T19:35:52.483064+0000 osd.4 (osd.4) 69 : cluster 3 failed to encode map e141 with expected crc 2023-09-23T19:35:53.719 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:53 smithi016 bash[75004]: cluster 2023-09-23T19:35:52.048938+0000 mgr.y (mgr.25325) 849 : cluster 0 pgmap v763: 89 pgs: 89 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:35:53.719 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:53 smithi016 bash[75004]: cluster 2023-09-23T19:35:52.207267+0000 osd.5 (osd.5) 73 : cluster 3 failed to encode map e141 with expected crc 2023-09-23T19:35:53.719 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:53 smithi016 bash[75004]: cluster 2023-09-23T19:35:52.482897+0000 osd.6 (osd.6) 59 : cluster 3 failed to encode map e141 with expected crc 2023-09-23T19:35:53.719 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:53 smithi016 bash[75004]: cluster 2023-09-23T19:35:52.483064+0000 osd.4 (osd.4) 69 : cluster 3 failed to encode map e141 with expected crc 2023-09-23T19:35:53.733 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:53.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:53.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:35:53.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:53 smithi092 bash[58855]: cluster 2023-09-23T19:35:52.048938+0000 mgr.y (mgr.25325) 849 : cluster 0 pgmap v763: 89 pgs: 89 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:35:53.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:53 smithi092 bash[58855]: cluster 2023-09-23T19:35:52.207267+0000 osd.5 (osd.5) 73 : cluster 3 failed to encode map e141 with expected crc 2023-09-23T19:35:53.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:53 smithi092 bash[58855]: cluster 2023-09-23T19:35:52.482897+0000 osd.6 (osd.6) 59 : cluster 3 failed to encode map e141 with expected crc 2023-09-23T19:35:53.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:53 smithi092 bash[58855]: cluster 2023-09-23T19:35:52.483064+0000 osd.4 (osd.4) 69 : cluster 3 failed to encode map e141 with expected crc 2023-09-23T19:35:53.836 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:53.870 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:53.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:53.874 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:53.974 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:54.005 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:54.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:54.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:54.109 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:54.136 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:54.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:54.139 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:35:54.240 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:54.269 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:54.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:54.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:54.326 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:35:54.373 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:54.395 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:54.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:54.397 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:35:54.498 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:54.521 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:54.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:54.523 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:35:54.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:54 smithi016 bash[75004]: cluster 2023-09-23T19:35:52.483195+0000 osd.7 (osd.7) 59 : cluster 3 failed to encode map e141 with expected crc 2023-09-23T19:35:54.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:54 smithi016 bash[77995]: cluster 2023-09-23T19:35:52.483195+0000 osd.7 (osd.7) 59 : cluster 3 failed to encode map e141 with expected crc 2023-09-23T19:35:54.624 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:54.637 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:54 smithi092 bash[58855]: cluster 2023-09-23T19:35:52.483195+0000 osd.7 (osd.7) 59 : cluster 3 failed to encode map e141 with expected crc 2023-09-23T19:35:54.652 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:54.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:54.655 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:35:54.756 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:54.783 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:54.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:54.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:54.887 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:54.913 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:54.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:54.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:35:55.016 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:55.041 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:55.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:55.044 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:35:55.144 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:55.176 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:55.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:55.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:35:55.279 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:55.309 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:55.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:55.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:35:55.412 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:55.445 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:55.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:55.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:35:55.548 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:55.571 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:55.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:55.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:55.677 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:55.690 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:35:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:35:55] "GET /metrics HTTP/1.1" 200 36183 "" "Prometheus/2.33.4" 2023-09-23T19:35:55.690 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:55 smithi016 bash[77995]: cluster 2023-09-23T19:35:54.049565+0000 mgr.y (mgr.25325) 850 : cluster 0 pgmap v764: 89 pgs: 89 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:35:55.690 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:55 smithi016 bash[75004]: cluster 2023-09-23T19:35:54.049565+0000 mgr.y (mgr.25325) 850 : cluster 0 pgmap v764: 89 pgs: 89 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:35:55.706 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:55.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:55.710 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:55.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:55 smithi092 bash[58855]: cluster 2023-09-23T19:35:54.049565+0000 mgr.y (mgr.25325) 850 : cluster 0 pgmap v764: 89 pgs: 89 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:35:55.810 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:55.833 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:55.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:55.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:55.937 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:55.961 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:55.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:55.965 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:56.045 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T19:35:56.066 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:56.095 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:56.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:56.098 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:56.198 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:56.209 INFO:teuthology.orchestra.run.smithi092.stdout:{ 2023-09-23T19:35:56.209 INFO:teuthology.orchestra.run.smithi092.stdout: "ops": [], 2023-09-23T19:35:56.209 INFO:teuthology.orchestra.run.smithi092.stdout: "complaint_time": 30, 2023-09-23T19:35:56.209 INFO:teuthology.orchestra.run.smithi092.stdout: "num_blocked_ops": 0 2023-09-23T19:35:56.209 INFO:teuthology.orchestra.run.smithi092.stdout:} 2023-09-23T19:35:56.224 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:56.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:56.228 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:35:56.328 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:56.355 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:56.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:56.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:56.458 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:56.490 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:56.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:56.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:56.594 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:56.616 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:56.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:56.619 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:56.720 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:56.741 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:35:56.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:56.743 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:35:56.844 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:56.876 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:56.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:56.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:35:56.975 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T19:35:56.980 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:57.013 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:57.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:57.016 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:57.116 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:57.143 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:57.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:57.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:35:57.247 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:57.281 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:57.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:57.283 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:35:57.384 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:57.419 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:57.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:57.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:57.449 INFO:teuthology.orchestra.run.smithi016.stderr:pool 'unique_pool_4' removed 2023-09-23T19:35:57.463 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T19:35:57.514 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:57 smithi016 bash[77995]: cluster 2023-09-23T19:35:56.050539+0000 mgr.y (mgr.25325) 851 : cluster 0 pgmap v765: 89 pgs: 89 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:35:57.515 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:57 smithi016 bash[77995]: audit 2023-09-23T19:35:56.433014+0000 mon.a (mon.0) 1057 : audit 1 from='client.? 172.21.15.16:0/786513126' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T19:35:57.515 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:57 smithi016 bash[77995]: audit 2023-09-23T19:35:56.437584+0000 mon.c (mon.2) 158 : audit 1 from='client.? 172.21.15.16:0/2729189700' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T19:35:57.515 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:57 smithi016 bash[77995]: audit 2023-09-23T19:35:56.438034+0000 mon.a (mon.0) 1058 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T19:35:57.515 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:57 smithi016 bash[75004]: cluster 2023-09-23T19:35:56.050539+0000 mgr.y (mgr.25325) 851 : cluster 0 pgmap v765: 89 pgs: 89 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:35:57.515 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:57 smithi016 bash[75004]: audit 2023-09-23T19:35:56.433014+0000 mon.a (mon.0) 1057 : audit 1 from='client.? 172.21.15.16:0/786513126' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T19:35:57.516 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:57 smithi016 bash[75004]: audit 2023-09-23T19:35:56.437584+0000 mon.c (mon.2) 158 : audit 1 from='client.? 172.21.15.16:0/2729189700' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T19:35:57.516 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:57 smithi016 bash[75004]: audit 2023-09-23T19:35:56.438034+0000 mon.a (mon.0) 1058 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T19:35:57.523 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:57.549 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:57.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:57.553 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:35:57.654 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:57.683 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:57.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:57.686 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:57.787 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:57.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:57 smithi092 bash[58855]: cluster 2023-09-23T19:35:56.050539+0000 mgr.y (mgr.25325) 851 : cluster 0 pgmap v765: 89 pgs: 89 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:35:57.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:57 smithi092 bash[58855]: audit 2023-09-23T19:35:56.433014+0000 mon.a (mon.0) 1057 : audit 1 from='client.? 172.21.15.16:0/786513126' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T19:35:57.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:57 smithi092 bash[58855]: audit 2023-09-23T19:35:56.437584+0000 mon.c (mon.2) 158 : audit 1 from='client.? 172.21.15.16:0/2729189700' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T19:35:57.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:57 smithi092 bash[58855]: audit 2023-09-23T19:35:56.438034+0000 mon.a (mon.0) 1058 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T19:35:57.812 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:35:57.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:57.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:35:57.916 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:57.958 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:57.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:57.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:35:58.064 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:58.087 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:58.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:58.093 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:35:58.193 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:58.225 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:58.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:58.228 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:58.328 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:58.357 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:58.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:58.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:35:58.461 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:58.475 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:58 smithi016 bash[77995]: audit 2023-09-23T19:35:57.437781+0000 mon.a (mon.0) 1059 : audit 1 from='client.? 172.21.15.16:0/786513126' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T19:35:58.475 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:58 smithi016 bash[77995]: audit 2023-09-23T19:35:57.437926+0000 mon.a (mon.0) 1060 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T19:35:58.475 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:58 smithi016 bash[77995]: cluster 2023-09-23T19:35:57.438052+0000 mon.a (mon.0) 1061 : cluster 0 osdmap e142: 8 total, 8 up, 8 in 2023-09-23T19:35:58.475 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:58 smithi016 bash[77995]: cluster 2023-09-23T19:35:57.885046+0000 osd.5 (osd.5) 74 : cluster 3 failed to encode map e142 with expected crc 2023-09-23T19:35:58.475 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:58 smithi016 bash[77995]: cluster 2023-09-23T19:35:57.885127+0000 osd.4 (osd.4) 70 : cluster 3 failed to encode map e142 with expected crc 2023-09-23T19:35:58.476 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:58 smithi016 bash[77995]: cluster 2023-09-23T19:35:57.885287+0000 osd.6 (osd.6) 60 : cluster 3 failed to encode map e142 with expected crc 2023-09-23T19:35:58.476 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:58 smithi016 bash[75004]: audit 2023-09-23T19:35:57.437781+0000 mon.a (mon.0) 1059 : audit 1 from='client.? 172.21.15.16:0/786513126' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T19:35:58.476 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:58 smithi016 bash[75004]: audit 2023-09-23T19:35:57.437926+0000 mon.a (mon.0) 1060 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T19:35:58.476 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:58 smithi016 bash[75004]: cluster 2023-09-23T19:35:57.438052+0000 mon.a (mon.0) 1061 : cluster 0 osdmap e142: 8 total, 8 up, 8 in 2023-09-23T19:35:58.476 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:58 smithi016 bash[75004]: cluster 2023-09-23T19:35:57.885046+0000 osd.5 (osd.5) 74 : cluster 3 failed to encode map e142 with expected crc 2023-09-23T19:35:58.476 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:58 smithi016 bash[75004]: cluster 2023-09-23T19:35:57.885127+0000 osd.4 (osd.4) 70 : cluster 3 failed to encode map e142 with expected crc 2023-09-23T19:35:58.476 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:58 smithi016 bash[75004]: cluster 2023-09-23T19:35:57.885287+0000 osd.6 (osd.6) 60 : cluster 3 failed to encode map e142 with expected crc 2023-09-23T19:35:58.493 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:35:58.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:58.497 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:35:58.597 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:58.609 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:58 smithi092 bash[58855]: audit 2023-09-23T19:35:57.437781+0000 mon.a (mon.0) 1059 : audit 1 from='client.? 172.21.15.16:0/786513126' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T19:35:58.609 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:58 smithi092 bash[58855]: audit 2023-09-23T19:35:57.437926+0000 mon.a (mon.0) 1060 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T19:35:58.609 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:58 smithi092 bash[58855]: cluster 2023-09-23T19:35:57.438052+0000 mon.a (mon.0) 1061 : cluster 0 osdmap e142: 8 total, 8 up, 8 in 2023-09-23T19:35:58.609 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:58 smithi092 bash[58855]: cluster 2023-09-23T19:35:57.885046+0000 osd.5 (osd.5) 74 : cluster 3 failed to encode map e142 with expected crc 2023-09-23T19:35:58.610 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:58 smithi092 bash[58855]: cluster 2023-09-23T19:35:57.885127+0000 osd.4 (osd.4) 70 : cluster 3 failed to encode map e142 with expected crc 2023-09-23T19:35:58.610 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:58 smithi092 bash[58855]: cluster 2023-09-23T19:35:57.885287+0000 osd.6 (osd.6) 60 : cluster 3 failed to encode map e142 with expected crc 2023-09-23T19:35:58.627 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:58.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:58.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:35:58.730 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:58.762 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:58.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:58.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:58.773 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T19:35:58.780 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T19:35:58.781 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T19:35:58.782 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_5 2023-09-23T19:35:58.782 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_5 16 2023-09-23T19:35:58.865 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:58.915 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:58.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:58.918 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:35:59.018 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:59.042 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:35:59.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:59.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:35:59.146 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:59.180 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:59.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:59.183 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:35:59.284 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:59.316 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:35:59.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:59.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:35:59.420 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:59.448 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:35:59.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:59.451 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:35:59.552 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:59.582 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:59.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:59.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:35:59.687 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:59.700 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:59 smithi092 bash[58855]: cluster 2023-09-23T19:35:57.885219+0000 osd.7 (osd.7) 60 : cluster 3 failed to encode map e142 with expected crc 2023-09-23T19:35:59.700 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:59 smithi092 bash[58855]: cluster 2023-09-23T19:35:58.051058+0000 mgr.y (mgr.25325) 852 : cluster 0 pgmap v767: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:35:59.700 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:35:59 smithi092 bash[58855]: audit 2023-09-23T19:35:58.644039+0000 mon.a (mon.0) 1062 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:35:59.719 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:59.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:59.724 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:35:59.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:59 smithi016 bash[75004]: cluster 2023-09-23T19:35:57.885219+0000 osd.7 (osd.7) 60 : cluster 3 failed to encode map e142 with expected crc 2023-09-23T19:35:59.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:59 smithi016 bash[75004]: cluster 2023-09-23T19:35:58.051058+0000 mgr.y (mgr.25325) 852 : cluster 0 pgmap v767: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:35:59.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:35:59 smithi016 bash[75004]: audit 2023-09-23T19:35:58.644039+0000 mon.a (mon.0) 1062 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:35:59.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:59 smithi016 bash[77995]: cluster 2023-09-23T19:35:57.885219+0000 osd.7 (osd.7) 60 : cluster 3 failed to encode map e142 with expected crc 2023-09-23T19:35:59.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:59 smithi016 bash[77995]: cluster 2023-09-23T19:35:58.051058+0000 mgr.y (mgr.25325) 852 : cluster 0 pgmap v767: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:35:59.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:35:59 smithi016 bash[77995]: audit 2023-09-23T19:35:58.644039+0000 mon.a (mon.0) 1062 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:35:59.825 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:59.857 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:35:59.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:59.860 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:35:59.961 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:59.992 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:35:59.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:35:59.995 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:36:00.096 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:00.130 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:00.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:00.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:36:00.233 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:00.263 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:00.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:00.266 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:36:00.367 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:00.396 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:00.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:00.400 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:00.500 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:00.532 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:00.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:00.536 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:00.636 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:00.668 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:00.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:00.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:36:00.772 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:00.803 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:00.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:00.806 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:00.907 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:00.941 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:00.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:00.945 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:36:01.045 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:01.078 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:01.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:01.081 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:01.182 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:01.217 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:01.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:01.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:36:01.302 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:36:01.303 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:36:01.321 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:01.353 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:01.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:01.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:01.456 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:01.482 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:01.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:01.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:36:01.586 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:01.596 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:01 smithi092 bash[58855]: cluster 2023-09-23T19:36:00.051809+0000 mgr.y (mgr.25325) 853 : cluster 0 pgmap v768: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:36:01.615 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:01.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:01.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:01.719 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:01.750 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:01.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:01.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:36:01.755 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:36:01.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:01 smithi016 bash[75004]: cluster 2023-09-23T19:36:00.051809+0000 mgr.y (mgr.25325) 853 : cluster 0 pgmap v768: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:36:01.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:01 smithi016 bash[77995]: cluster 2023-09-23T19:36:00.051809+0000 mgr.y (mgr.25325) 853 : cluster 0 pgmap v768: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:36:01.854 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:01.888 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:01.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:01.891 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:01.992 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:02.020 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:02.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:02.023 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:02.124 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:02.145 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:02.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:02.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:02.248 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:02.270 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:02.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:02.273 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:02.374 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:02.399 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:02.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:02.402 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:02.502 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:02.529 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:02.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:02.533 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:36:02.633 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:02.659 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:02.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:02.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:02.763 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:02.791 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:02.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:02.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:02.894 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:02.921 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:02.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:02.924 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:03.024 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:03.053 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:03.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:03.056 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:03.156 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:03.182 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:03.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:03.184 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:03.285 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:03.315 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:03.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:03.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:03.331 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:36:03.333 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:36:03.419 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:03.448 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:03.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:03.451 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:03.551 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:03.581 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:03.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:03.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:03.585 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:03 smithi016 bash[77995]: cluster 2023-09-23T19:36:02.052493+0000 mgr.y (mgr.25325) 854 : cluster 0 pgmap v769: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:36:03.585 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:03 smithi016 bash[75004]: cluster 2023-09-23T19:36:02.052493+0000 mgr.y (mgr.25325) 854 : cluster 0 pgmap v769: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:36:03.685 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:03.714 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:03.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:03.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:36:03.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:03 smithi092 bash[58855]: cluster 2023-09-23T19:36:02.052493+0000 mgr.y (mgr.25325) 854 : cluster 0 pgmap v769: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:36:03.817 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:03.852 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:03.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:03.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:36:03.956 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:03.983 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:03.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:03.987 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:04.088 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:04.116 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:04.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:04.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:36:04.219 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:04.247 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:04.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:04.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:04.350 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:04.378 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:04.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:04.382 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:04.482 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:04.514 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:04.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:04.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:04.617 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:04.649 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:04.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:04.651 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:04.752 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:04.778 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:04.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:04.781 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:36:04.881 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:04.904 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:04.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:04.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:36:05.007 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:05.034 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:05.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:05.037 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:05.138 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:05.165 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:05.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:05.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:05.269 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:05.298 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:05.301 INFO:teuthology.orchestra.run.smithi092.stdout:{ 2023-09-23T19:36:05.301 INFO:teuthology.orchestra.run.smithi092.stdout: "size": 20, 2023-09-23T19:36:05.301 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 600, 2023-09-23T19:36:05.302 INFO:teuthology.orchestra.run.smithi092.stdout: "ops": [ 2023-09-23T19:36:05.302 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.302 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:116153 9.1 9:89ac564e:::benchmark_data_smithi016_113233_object7771:head [delete] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-09-23T19:36:05.302 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:33:14.021107+0000", 2023-09-23T19:36:05.302 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 171.276063139, 2023-09-23T19:36:05.302 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.103312926, 2023-09-23T19:36:05.302 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:36:05.302 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:36:05.302 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:36:05.302 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:36:05.302 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:36:05.303 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 116153 2023-09-23T19:36:05.303 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.303 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:36:05.303 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.303 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:36:05.303 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.021107+0000", 2023-09-23T19:36:05.303 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:36:05.303 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.303 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.303 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:36:05.303 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.021107+0000", 2023-09-23T19:36:05.303 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T19:36:05.304 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.304 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.304 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:36:05.304 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.021105+0000", 2023-09-23T19:36:05.304 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.7860000000000002e-06 2023-09-23T19:36:05.304 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.304 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.304 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:36:05.304 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.021114+0000", 2023-09-23T19:36:05.304 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.7119e-05 2023-09-23T19:36:05.304 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.305 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.305 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:36:05.305 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.021131+0000", 2023-09-23T19:36:05.305 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.9199999999999993e-06 2023-09-23T19:36:05.305 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.305 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.305 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:36:05.305 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.021140+0000", 2023-09-23T19:36:05.305 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.7054999999999999e-05 2023-09-23T19:36:05.305 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.305 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.305 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:36:05.305 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.021227+0000", 2023-09-23T19:36:05.306 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00017417800000000001 2023-09-23T19:36:05.306 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.306 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.306 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:36:05.306 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.021401+0000", 2023-09-23T19:36:05.306 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00017338 2023-09-23T19:36:05.306 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.306 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.306 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3,7", 2023-09-23T19:36:05.306 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.021575+0000", 2023-09-23T19:36:05.306 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0089701980000000004 2023-09-23T19:36:05.306 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.307 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.307 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:36:05.307 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.030545+0000", 2023-09-23T19:36:05.307 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.003814268 2023-09-23T19:36:05.307 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.307 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.307 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:05.307 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.034359+0000", 2023-09-23T19:36:05.307 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.089844070999999998 2023-09-23T19:36:05.307 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.307 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.307 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:05.308 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.124203+0000", 2023-09-23T19:36:05.308 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000144239 2023-09-23T19:36:05.308 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.308 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.308 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:36:05.308 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.124347+0000", 2023-09-23T19:36:05.308 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.2608999999999995e-05 2023-09-23T19:36:05.308 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.308 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.308 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:36:05.308 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.124420+0000", 2023-09-23T19:36:05.309 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.103312926 2023-09-23T19:36:05.309 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:05.309 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:36:05.309 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:05.309 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.309 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.309 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:116156 9.1 9:87513286:::benchmark_data_smithi016_113233_object7774:head [delete] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-09-23T19:36:05.309 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:33:14.025628+0000", 2023-09-23T19:36:05.309 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 171.271541861, 2023-09-23T19:36:05.309 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.099121599000000005, 2023-09-23T19:36:05.309 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:36:05.310 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:36:05.310 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:36:05.310 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:36:05.310 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:36:05.310 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 116156 2023-09-23T19:36:05.310 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.310 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:36:05.310 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.310 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:36:05.310 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.025628+0000", 2023-09-23T19:36:05.310 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:36:05.311 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.311 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.311 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:36:05.311 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.025628+0000", 2023-09-23T19:36:05.311 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999976 2023-09-23T19:36:05.311 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.311 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.311 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:36:05.311 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.025626+0000", 2023-09-23T19:36:05.311 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.3989999999999993e-06 2023-09-23T19:36:05.311 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.311 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.312 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:36:05.312 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.025635+0000", 2023-09-23T19:36:05.312 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.8e-05 2023-09-23T19:36:05.312 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.312 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.312 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:36:05.312 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.025653+0000", 2023-09-23T19:36:05.312 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.6510000000000001e-06 2023-09-23T19:36:05.312 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.312 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.312 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:36:05.313 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.025661+0000", 2023-09-23T19:36:05.313 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00056569499999999996 2023-09-23T19:36:05.313 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.313 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.313 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:36:05.313 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.026227+0000", 2023-09-23T19:36:05.314 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000113001 2023-09-23T19:36:05.314 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.314 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.314 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:36:05.314 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.026340+0000", 2023-09-23T19:36:05.314 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00012759300000000001 2023-09-23T19:36:05.314 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.314 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.314 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3,7", 2023-09-23T19:36:05.314 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.026467+0000", 2023-09-23T19:36:05.314 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00761883 2023-09-23T19:36:05.315 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.315 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.315 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:36:05.315 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.034086+0000", 2023-09-23T19:36:05.315 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.010389819 2023-09-23T19:36:05.315 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.315 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.315 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:05.315 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.044476+0000", 2023-09-23T19:36:05.316 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.080098409999999995 2023-09-23T19:36:05.316 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.316 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.316 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:05.316 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.124574+0000", 2023-09-23T19:36:05.316 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00011454400000000001 2023-09-23T19:36:05.316 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.316 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.316 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:36:05.316 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.124689+0000", 2023-09-23T19:36:05.316 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.0822000000000001e-05 2023-09-23T19:36:05.317 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.317 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.317 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:36:05.317 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.124750+0000", 2023-09-23T19:36:05.317 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.099121599000000005 2023-09-23T19:36:05.317 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:05.317 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:36:05.317 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:05.318 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.318 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.318 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:116159 9.1 9:8bfe72dd:::benchmark_data_smithi016_113233_object7777:head [delete] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-09-23T19:36:05.318 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:33:14.027706+0000", 2023-09-23T19:36:05.318 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 171.269463831, 2023-09-23T19:36:05.318 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.097322597999999996, 2023-09-23T19:36:05.318 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:36:05.318 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:36:05.318 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:36:05.318 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:36:05.318 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:36:05.319 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 116159 2023-09-23T19:36:05.319 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.319 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:36:05.319 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.319 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:36:05.319 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.027706+0000", 2023-09-23T19:36:05.319 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:36:05.319 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.319 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.319 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:36:05.319 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.027706+0000", 2023-09-23T19:36:05.319 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T19:36:05.319 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.320 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.320 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:36:05.320 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.027705+0000", 2023-09-23T19:36:05.320 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.1510000000000004e-06 2023-09-23T19:36:05.320 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.320 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.320 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:36:05.320 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.027713+0000", 2023-09-23T19:36:05.320 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.2089000000000001e-05 2023-09-23T19:36:05.320 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.320 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.321 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:36:05.321 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.027725+0000", 2023-09-23T19:36:05.321 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.6850000000000004e-06 2023-09-23T19:36:05.321 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.321 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.321 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:36:05.321 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.027732+0000", 2023-09-23T19:36:05.321 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0028444759999999999 2023-09-23T19:36:05.321 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.322 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.322 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:36:05.322 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.030576+0000", 2023-09-23T19:36:05.322 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000113383 2023-09-23T19:36:05.322 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.322 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.322 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:36:05.322 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.030689+0000", 2023-09-23T19:36:05.322 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00013951600000000001 2023-09-23T19:36:05.323 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.323 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.323 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3,7", 2023-09-23T19:36:05.323 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.030829+0000", 2023-09-23T19:36:05.323 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0034693580000000001 2023-09-23T19:36:05.323 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.323 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.323 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:36:05.323 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.034298+0000", 2023-09-23T19:36:05.324 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.014735705999999999 2023-09-23T19:36:05.324 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.324 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.324 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:05.324 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.049034+0000", 2023-09-23T19:36:05.324 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.075790369999999996 2023-09-23T19:36:05.324 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.324 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.324 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:05.324 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.124824+0000", 2023-09-23T19:36:05.325 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000143576 2023-09-23T19:36:05.325 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.325 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.325 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:36:05.325 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.124968+0000", 2023-09-23T19:36:05.325 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.0945e-05 2023-09-23T19:36:05.325 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.325 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.325 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:36:05.326 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.125029+0000", 2023-09-23T19:36:05.326 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.097322597999999996 2023-09-23T19:36:05.326 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:05.326 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:36:05.326 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:05.326 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.326 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.326 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:116179 9.1 9:806a1a66:::benchmark_data_smithi016_113233_object7797:head [delete] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-09-23T19:36:05.327 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:33:14.061224+0000", 2023-09-23T19:36:05.327 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 171.23594617800001, 2023-09-23T19:36:05.327 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.067170966999999998, 2023-09-23T19:36:05.327 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:36:05.327 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:36:05.327 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:36:05.327 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:36:05.327 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:36:05.327 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 116179 2023-09-23T19:36:05.328 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.328 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:36:05.328 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.328 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:36:05.328 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.061224+0000", 2023-09-23T19:36:05.328 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:36:05.328 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.328 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.328 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:36:05.328 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.061224+0000", 2023-09-23T19:36:05.328 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999971 2023-09-23T19:36:05.329 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.329 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.329 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:36:05.329 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.061221+0000", 2023-09-23T19:36:05.329 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.1092e-05 2023-09-23T19:36:05.329 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.329 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.329 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:36:05.329 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.061232+0000", 2023-09-23T19:36:05.329 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.8338e-05 2023-09-23T19:36:05.329 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.330 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.330 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:36:05.330 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.061250+0000", 2023-09-23T19:36:05.330 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.7490000000000004e-06 2023-09-23T19:36:05.330 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.330 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.330 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:36:05.330 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.061259+0000", 2023-09-23T19:36:05.330 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.7124000000000003e-05 2023-09-23T19:36:05.330 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.330 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.330 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:36:05.331 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.061326+0000", 2023-09-23T19:36:05.331 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000124497 2023-09-23T19:36:05.331 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.331 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.331 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:36:05.331 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.061451+0000", 2023-09-23T19:36:05.331 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000133965 2023-09-23T19:36:05.331 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.331 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.331 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3,7", 2023-09-23T19:36:05.331 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.061585+0000", 2023-09-23T19:36:05.331 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0039240830000000001 2023-09-23T19:36:05.332 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.332 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.332 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:05.332 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.065509+0000", 2023-09-23T19:36:05.332 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00016223100000000001 2023-09-23T19:36:05.332 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.332 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.332 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:36:05.332 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.065671+0000", 2023-09-23T19:36:05.332 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.062461475000000002 2023-09-23T19:36:05.332 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.332 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.333 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:05.333 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.128133+0000", 2023-09-23T19:36:05.333 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000143497 2023-09-23T19:36:05.333 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.333 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.333 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:36:05.333 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.128276+0000", 2023-09-23T19:36:05.333 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000118866 2023-09-23T19:36:05.333 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.333 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.333 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:36:05.333 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:14.128395+0000", 2023-09-23T19:36:05.334 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.067170966999999998 2023-09-23T19:36:05.334 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:05.334 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:36:05.334 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:05.334 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.334 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.334 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:135769 9.1 9:8f36a9ae:::benchmark_data_smithi016_113233_object27387:head [delete] snapc 0=[] ondisk+write+known_if_redirected e129)", 2023-09-23T19:36:05.334 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:33:39.480162+0000", 2023-09-23T19:36:05.334 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 145.817008203, 2023-09-23T19:36:05.334 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.076127847999999998, 2023-09-23T19:36:05.334 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:36:05.335 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:36:05.335 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:36:05.335 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:36:05.335 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:36:05.335 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 135769 2023-09-23T19:36:05.335 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.335 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:36:05.335 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.335 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:36:05.335 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:39.480162+0000", 2023-09-23T19:36:05.335 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:36:05.336 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.336 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.336 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:36:05.336 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:39.480162+0000", 2023-09-23T19:36:05.336 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999976 2023-09-23T19:36:05.336 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.336 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.336 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:36:05.336 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:39.480160+0000", 2023-09-23T19:36:05.336 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.0927999999999999e-05 2023-09-23T19:36:05.336 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.336 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.337 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:36:05.337 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:39.480171+0000", 2023-09-23T19:36:05.337 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.6804999999999999e-05 2023-09-23T19:36:05.337 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.337 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.337 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:36:05.337 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:39.480187+0000", 2023-09-23T19:36:05.337 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.9219999999999997e-06 2023-09-23T19:36:05.337 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.337 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.337 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:36:05.337 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:39.480195+0000", 2023-09-23T19:36:05.338 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0037620309999999999 2023-09-23T19:36:05.338 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.338 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.338 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:36:05.338 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:39.483957+0000", 2023-09-23T19:36:05.338 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000236667 2023-09-23T19:36:05.338 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.338 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.338 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:36:05.338 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:39.484194+0000", 2023-09-23T19:36:05.338 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000368675 2023-09-23T19:36:05.338 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.339 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.339 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3,7", 2023-09-23T19:36:05.339 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:39.484563+0000", 2023-09-23T19:36:05.339 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0040910690000000001 2023-09-23T19:36:05.339 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.339 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.339 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:36:05.339 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:39.488654+0000", 2023-09-23T19:36:05.339 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000274868 2023-09-23T19:36:05.339 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.339 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.339 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:05.340 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:39.488929+0000", 2023-09-23T19:36:05.340 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.067245554999999999 2023-09-23T19:36:05.340 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.340 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.340 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:05.340 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:39.556174+0000", 2023-09-23T19:36:05.340 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.2657999999999995e-05 2023-09-23T19:36:05.340 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.340 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.340 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:36:05.340 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:39.556237+0000", 2023-09-23T19:36:05.340 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.2930999999999999e-05 2023-09-23T19:36:05.341 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.341 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.341 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:36:05.341 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:39.556290+0000", 2023-09-23T19:36:05.341 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.076127847999999998 2023-09-23T19:36:05.341 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:05.341 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:36:05.341 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:05.341 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.341 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.341 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:135774 9.1 9:85366549:::benchmark_data_smithi016_113233_object27392:head [delete] snapc 0=[] ondisk+write+known_if_redirected e129)", 2023-09-23T19:36:05.341 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:33:39.489415+0000", 2023-09-23T19:36:05.342 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 145.80775510300001, 2023-09-23T19:36:05.342 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.067003623999999998, 2023-09-23T19:36:05.342 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:36:05.342 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:36:05.342 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:36:05.342 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:36:05.342 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:36:05.342 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 135774 2023-09-23T19:36:05.342 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.342 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:36:05.342 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.342 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:36:05.343 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:39.489415+0000", 2023-09-23T19:36:05.343 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:36:05.343 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.343 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.343 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:36:05.343 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:39.489415+0000", 2023-09-23T19:36:05.343 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T19:36:05.343 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.343 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.343 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:36:05.343 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:39.489413+0000", 2023-09-23T19:36:05.343 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.5480000000000007e-06 2023-09-23T19:36:05.344 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.344 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.344 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:36:05.344 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:39.489423+0000", 2023-09-23T19:36:05.344 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.3147e-05 2023-09-23T19:36:05.344 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.344 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.344 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:36:05.344 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:39.489436+0000", 2023-09-23T19:36:05.344 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.3890000000000002e-06 2023-09-23T19:36:05.344 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.344 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.345 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:36:05.345 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:39.489442+0000", 2023-09-23T19:36:05.345 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.6842999999999999e-05 2023-09-23T19:36:05.345 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.345 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.345 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:36:05.345 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:39.489479+0000", 2023-09-23T19:36:05.345 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.2311000000000004e-05 2023-09-23T19:36:05.345 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.345 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.345 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:36:05.345 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:39.489572+0000", 2023-09-23T19:36:05.346 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000101226 2023-09-23T19:36:05.346 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.346 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.346 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3,7", 2023-09-23T19:36:05.346 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:39.489673+0000", 2023-09-23T19:36:05.346 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0038326329999999998 2023-09-23T19:36:05.346 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.346 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.346 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:36:05.346 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:39.493505+0000", 2023-09-23T19:36:05.346 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.001613843 2023-09-23T19:36:05.346 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.347 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.347 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:05.347 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:39.495119+0000", 2023-09-23T19:36:05.347 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.061204486000000002 2023-09-23T19:36:05.347 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.347 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.347 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:05.347 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:39.556324+0000", 2023-09-23T19:36:05.347 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.3932999999999999e-05 2023-09-23T19:36:05.347 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.347 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.347 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:36:05.347 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:39.556388+0000", 2023-09-23T19:36:05.348 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.0984e-05 2023-09-23T19:36:05.348 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.348 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.348 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:36:05.348 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:39.556419+0000", 2023-09-23T19:36:05.348 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.067003623999999998 2023-09-23T19:36:05.348 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:05.348 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:36:05.348 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:05.349 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.349 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.349 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:143792 9.1 9:88b4fb66:::benchmark_data_smithi016_113233_object35410:head [delete] snapc 0=[] ondisk+write+known_if_redirected e130)", 2023-09-23T19:36:05.349 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:33:50.594162+0000", 2023-09-23T19:36:05.349 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 134.70300826499999, 2023-09-23T19:36:05.349 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.070546708999999999, 2023-09-23T19:36:05.349 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:36:05.349 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:36:05.349 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:36:05.349 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:36:05.349 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:36:05.350 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 143792 2023-09-23T19:36:05.350 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.350 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:36:05.350 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.350 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:36:05.350 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.594162+0000", 2023-09-23T19:36:05.350 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:36:05.350 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.350 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.351 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:36:05.351 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.594162+0000", 2023-09-23T19:36:05.351 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T19:36:05.351 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.351 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.351 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:36:05.351 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.594160+0000", 2023-09-23T19:36:05.351 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.0664e-05 2023-09-23T19:36:05.351 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.351 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.351 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:36:05.352 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.594171+0000", 2023-09-23T19:36:05.352 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.2683999999999999e-05 2023-09-23T19:36:05.352 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.352 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.352 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:36:05.352 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.594184+0000", 2023-09-23T19:36:05.352 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.0879999999999997e-06 2023-09-23T19:36:05.352 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.352 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.352 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:36:05.353 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.594191+0000", 2023-09-23T19:36:05.353 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00031472 2023-09-23T19:36:05.353 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.353 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.353 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:36:05.353 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.594506+0000", 2023-09-23T19:36:05.353 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00016700999999999999 2023-09-23T19:36:05.353 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.353 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.354 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:36:05.354 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.594673+0000", 2023-09-23T19:36:05.354 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00017687700000000001 2023-09-23T19:36:05.354 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.354 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.354 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3,7", 2023-09-23T19:36:05.354 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.594850+0000", 2023-09-23T19:36:05.354 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.021483136999999999 2023-09-23T19:36:05.355 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.355 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.355 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:05.355 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.616333+0000", 2023-09-23T19:36:05.355 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0159779 2023-09-23T19:36:05.355 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.355 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.355 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:36:05.355 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.632311+0000", 2023-09-23T19:36:05.356 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.032241068999999997 2023-09-23T19:36:05.356 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.356 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.356 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:05.356 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.664552+0000", 2023-09-23T19:36:05.356 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00010323999999999999 2023-09-23T19:36:05.356 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.356 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.356 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:36:05.357 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.664655+0000", 2023-09-23T19:36:05.357 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.3687000000000003e-05 2023-09-23T19:36:05.357 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.357 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.357 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:36:05.357 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.664709+0000", 2023-09-23T19:36:05.357 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.070546708999999999 2023-09-23T19:36:05.357 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:05.357 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:36:05.357 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:05.358 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.358 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.358 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:143793 9.1 9:820544d8:::benchmark_data_smithi016_113233_object35411:head [delete] snapc 0=[] ondisk+write+known_if_redirected e130)", 2023-09-23T19:36:05.358 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:33:50.594834+0000", 2023-09-23T19:36:05.358 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 134.702336492, 2023-09-23T19:36:05.358 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.076057294999999997, 2023-09-23T19:36:05.358 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:36:05.358 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:36:05.358 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:36:05.359 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:36:05.359 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:36:05.359 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 143793 2023-09-23T19:36:05.359 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.359 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:36:05.359 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.359 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:36:05.359 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.594834+0000", 2023-09-23T19:36:05.360 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:36:05.360 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.360 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.360 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:36:05.360 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.594834+0000", 2023-09-23T19:36:05.360 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.999999 2023-09-23T19:36:05.361 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.361 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.361 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:36:05.361 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.594833+0000", 2023-09-23T19:36:05.361 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.0099999999999998e-06 2023-09-23T19:36:05.361 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.361 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.361 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:36:05.361 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.594840+0000", 2023-09-23T19:36:05.361 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.1051e-05 2023-09-23T19:36:05.361 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.362 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.362 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:36:05.362 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.594851+0000", 2023-09-23T19:36:05.362 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.1760000000000004e-06 2023-09-23T19:36:05.362 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.362 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.362 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:36:05.362 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.594858+0000", 2023-09-23T19:36:05.362 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0051093500000000003 2023-09-23T19:36:05.362 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.362 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.362 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:36:05.363 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.599967+0000", 2023-09-23T19:36:05.363 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00014624199999999999 2023-09-23T19:36:05.363 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.363 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.363 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:36:05.363 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.600113+0000", 2023-09-23T19:36:05.363 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00016062499999999999 2023-09-23T19:36:05.363 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.363 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.363 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3,7", 2023-09-23T19:36:05.363 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.600274+0000", 2023-09-23T19:36:05.363 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.032045716000000002 2023-09-23T19:36:05.364 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.364 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.364 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:36:05.364 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.632320+0000", 2023-09-23T19:36:05.364 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.1501000000000006e-05 2023-09-23T19:36:05.364 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.364 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.364 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:05.364 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.632381+0000", 2023-09-23T19:36:05.364 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.038218163999999999 2023-09-23T19:36:05.364 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.364 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.365 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:05.365 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.670599+0000", 2023-09-23T19:36:05.365 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000112247 2023-09-23T19:36:05.365 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.365 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.365 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:36:05.365 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.670712+0000", 2023-09-23T19:36:05.365 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000179245 2023-09-23T19:36:05.365 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.365 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.365 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:36:05.366 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.670891+0000", 2023-09-23T19:36:05.366 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.076057294999999997 2023-09-23T19:36:05.366 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:05.366 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:36:05.366 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:05.366 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.366 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.366 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:143795 9.1 9:8d54b8a3:::benchmark_data_smithi016_113233_object35413:head [delete] snapc 0=[] ondisk+write+known_if_redirected e130)", 2023-09-23T19:36:05.366 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:33:50.599676+0000", 2023-09-23T19:36:05.366 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 134.697494304, 2023-09-23T19:36:05.366 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.071375665000000005, 2023-09-23T19:36:05.366 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:36:05.367 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:36:05.367 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:36:05.367 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:36:05.367 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:36:05.367 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 143795 2023-09-23T19:36:05.367 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.367 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:36:05.367 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.367 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:36:05.367 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.599676+0000", 2023-09-23T19:36:05.367 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:36:05.367 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.368 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.368 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:36:05.368 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.599676+0000", 2023-09-23T19:36:05.368 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T19:36:05.368 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.368 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.368 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:36:05.368 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.599674+0000", 2023-09-23T19:36:05.368 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.3340000000000002e-06 2023-09-23T19:36:05.368 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.368 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.368 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:36:05.368 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.599682+0000", 2023-09-23T19:36:05.369 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.5216e-05 2023-09-23T19:36:05.369 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.369 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.369 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:36:05.369 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.599698+0000", 2023-09-23T19:36:05.369 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.8839999999999997e-06 2023-09-23T19:36:05.369 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.369 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.369 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:36:05.369 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.599704+0000", 2023-09-23T19:36:05.369 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0054567799999999996 2023-09-23T19:36:05.369 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.370 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.370 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:36:05.370 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.605161+0000", 2023-09-23T19:36:05.370 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00015313999999999999 2023-09-23T19:36:05.370 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.370 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.370 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:36:05.370 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.605314+0000", 2023-09-23T19:36:05.370 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00016738199999999999 2023-09-23T19:36:05.370 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.370 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.370 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3,7", 2023-09-23T19:36:05.371 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.605482+0000", 2023-09-23T19:36:05.371 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.02684547 2023-09-23T19:36:05.371 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.371 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.371 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:36:05.371 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.632327+0000", 2023-09-23T19:36:05.371 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000118094 2023-09-23T19:36:05.371 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.372 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.372 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:05.372 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.632445+0000", 2023-09-23T19:36:05.372 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.038483650000000001 2023-09-23T19:36:05.372 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.372 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.372 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:05.372 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.670929+0000", 2023-09-23T19:36:05.372 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.1574000000000002e-05 2023-09-23T19:36:05.372 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.373 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.373 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:36:05.373 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.671010+0000", 2023-09-23T19:36:05.373 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.0988999999999997e-05 2023-09-23T19:36:05.373 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.373 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.373 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:36:05.373 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.671051+0000", 2023-09-23T19:36:05.373 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.071375665000000005 2023-09-23T19:36:05.374 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:05.374 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:36:05.374 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:05.374 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.374 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.374 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:143800 9.1 9:8f104a1b:::benchmark_data_smithi016_113233_object35418:head [delete] snapc 0=[] ondisk+write+known_if_redirected e130)", 2023-09-23T19:36:05.374 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:33:50.600276+0000", 2023-09-23T19:36:05.374 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 134.69689412100001, 2023-09-23T19:36:05.374 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.070920677000000001, 2023-09-23T19:36:05.374 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:36:05.374 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:36:05.375 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:36:05.375 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:36:05.375 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:36:05.375 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 143800 2023-09-23T19:36:05.375 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.375 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:36:05.375 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.375 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:36:05.375 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.600276+0000", 2023-09-23T19:36:05.375 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:36:05.375 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.375 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.376 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:36:05.376 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.600276+0000", 2023-09-23T19:36:05.376 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T19:36:05.376 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.376 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.376 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:36:05.376 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.600275+0000", 2023-09-23T19:36:05.376 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.1960000000000001e-05 2023-09-23T19:36:05.376 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.376 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.376 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:36:05.376 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.600287+0000", 2023-09-23T19:36:05.377 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.4813e-05 2023-09-23T19:36:05.377 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.377 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.377 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:36:05.377 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.600301+0000", 2023-09-23T19:36:05.377 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.5140000000000001e-06 2023-09-23T19:36:05.377 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.377 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.377 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:36:05.377 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.600310+0000", 2023-09-23T19:36:05.377 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.010677783 2023-09-23T19:36:05.377 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.378 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.378 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:36:05.378 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.610988+0000", 2023-09-23T19:36:05.378 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00016627800000000001 2023-09-23T19:36:05.378 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.378 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.378 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:36:05.378 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.611154+0000", 2023-09-23T19:36:05.378 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000186448 2023-09-23T19:36:05.378 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.378 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.379 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3,7", 2023-09-23T19:36:05.379 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.611340+0000", 2023-09-23T19:36:05.379 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.021115648000000001 2023-09-23T19:36:05.379 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.379 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.379 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:36:05.379 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.632456+0000", 2023-09-23T19:36:05.379 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.7472999999999997e-05 2023-09-23T19:36:05.379 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.379 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.379 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:05.379 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.632514+0000", 2023-09-23T19:36:05.379 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.038572505999999999 2023-09-23T19:36:05.380 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.380 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.380 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:05.380 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.671086+0000", 2023-09-23T19:36:05.380 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.3898000000000003e-05 2023-09-23T19:36:05.380 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.380 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.380 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:36:05.380 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.671160+0000", 2023-09-23T19:36:05.380 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.6745000000000001e-05 2023-09-23T19:36:05.380 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.380 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.381 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:36:05.381 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.671197+0000", 2023-09-23T19:36:05.381 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.070920677000000001 2023-09-23T19:36:05.381 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:05.381 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:36:05.381 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:05.381 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.381 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.381 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:143801 9.1 9:8e706bd1:::benchmark_data_smithi016_113233_object35419:head [delete] snapc 0=[] ondisk+write+known_if_redirected e130)", 2023-09-23T19:36:05.381 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:33:50.600321+0000", 2023-09-23T19:36:05.382 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 134.696849458, 2023-09-23T19:36:05.382 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.071017971999999999, 2023-09-23T19:36:05.382 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:36:05.382 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:36:05.382 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:36:05.382 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:36:05.382 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:36:05.382 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 143801 2023-09-23T19:36:05.382 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.382 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:36:05.383 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.383 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:36:05.383 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.600321+0000", 2023-09-23T19:36:05.383 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:36:05.383 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.383 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.383 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:36:05.383 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.600321+0000", 2023-09-23T19:36:05.383 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.999999 2023-09-23T19:36:05.383 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.383 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.384 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:36:05.384 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.600320+0000", 2023-09-23T19:36:05.384 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.4440000000000001e-06 2023-09-23T19:36:05.384 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.384 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.384 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:36:05.384 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.600324+0000", 2023-09-23T19:36:05.384 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.9930000000000001e-06 2023-09-23T19:36:05.384 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.384 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.384 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:36:05.385 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.600333+0000", 2023-09-23T19:36:05.385 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.4709999999999998e-06 2023-09-23T19:36:05.385 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.385 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.385 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:36:05.385 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.600339+0000", 2023-09-23T19:36:05.385 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.016026133000000001 2023-09-23T19:36:05.385 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.385 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.385 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:36:05.385 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.616365+0000", 2023-09-23T19:36:05.385 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00015708599999999999 2023-09-23T19:36:05.386 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.386 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.386 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:36:05.386 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.616522+0000", 2023-09-23T19:36:05.386 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000184067 2023-09-23T19:36:05.386 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.386 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.386 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3,7", 2023-09-23T19:36:05.386 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.616706+0000", 2023-09-23T19:36:05.387 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.015857837 2023-09-23T19:36:05.387 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.387 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.387 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:05.387 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.632564+0000", 2023-09-23T19:36:05.387 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0060175130000000004 2023-09-23T19:36:05.387 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.387 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.387 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:36:05.387 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.638581+0000", 2023-09-23T19:36:05.387 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.032648203000000001 2023-09-23T19:36:05.388 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.388 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.388 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:05.388 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.671230+0000", 2023-09-23T19:36:05.388 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.2597999999999998e-05 2023-09-23T19:36:05.388 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.388 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.388 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:36:05.388 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.671302+0000", 2023-09-23T19:36:05.388 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.6505999999999999e-05 2023-09-23T19:36:05.389 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.389 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.389 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:36:05.389 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:33:50.671339+0000", 2023-09-23T19:36:05.389 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.071017971999999999 2023-09-23T19:36:05.389 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:05.389 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:36:05.389 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:05.389 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.389 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.389 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:160284 9.1 9:8f1806a3:::benchmark_data_smithi016_113233_object51902:head [delete] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-09-23T19:36:05.389 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:34:14.233830+0000", 2023-09-23T19:36:05.390 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 111.06334006599999, 2023-09-23T19:36:05.390 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.13149026899999999, 2023-09-23T19:36:05.390 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:36:05.390 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:36:05.390 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:36:05.390 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:36:05.390 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:36:05.390 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 160284 2023-09-23T19:36:05.390 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.390 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:36:05.390 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.390 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:36:05.391 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:14.233830+0000", 2023-09-23T19:36:05.391 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:36:05.391 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.391 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.391 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:36:05.391 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:14.233830+0000", 2023-09-23T19:36:05.391 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.999999 2023-09-23T19:36:05.391 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.391 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.391 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:36:05.392 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:14.233829+0000", 2023-09-23T19:36:05.392 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.0480000000000004e-06 2023-09-23T19:36:05.392 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.392 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.392 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:36:05.392 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:14.233835+0000", 2023-09-23T19:36:05.392 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.3504e-05 2023-09-23T19:36:05.392 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.392 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.392 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:36:05.393 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:14.233849+0000", 2023-09-23T19:36:05.393 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.1539999999999999e-06 2023-09-23T19:36:05.393 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.393 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.393 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:36:05.393 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:14.233855+0000", 2023-09-23T19:36:05.393 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0092985010000000007 2023-09-23T19:36:05.393 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.394 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.394 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:36:05.394 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:14.243153+0000", 2023-09-23T19:36:05.394 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00010237400000000001 2023-09-23T19:36:05.394 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.394 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.394 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:36:05.394 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:14.243256+0000", 2023-09-23T19:36:05.395 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.5850000000000001e-05 2023-09-23T19:36:05.395 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.395 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.395 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3,7", 2023-09-23T19:36:05.395 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:14.243332+0000", 2023-09-23T19:36:05.395 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.012898078 2023-09-23T19:36:05.395 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.395 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.396 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:36:05.396 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:14.256230+0000", 2023-09-23T19:36:05.396 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.010047614999999999 2023-09-23T19:36:05.396 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.396 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.396 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:05.396 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:14.266277+0000", 2023-09-23T19:36:05.396 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.098907302000000002 2023-09-23T19:36:05.396 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.397 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.397 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:05.397 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:14.365185+0000", 2023-09-23T19:36:05.397 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.5572000000000001e-05 2023-09-23T19:36:05.397 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.397 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.397 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:36:05.397 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:14.365260+0000", 2023-09-23T19:36:05.397 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.0232999999999997e-05 2023-09-23T19:36:05.397 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.397 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.397 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:36:05.398 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:14.365320+0000", 2023-09-23T19:36:05.398 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.13149026899999999 2023-09-23T19:36:05.398 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:05.398 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:36:05.398 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:05.398 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.398 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.398 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:160287 9.1 9:84572a1b:::benchmark_data_smithi016_113233_object51905:head [delete] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-09-23T19:36:05.398 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:34:14.237433+0000", 2023-09-23T19:36:05.398 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 111.059736673, 2023-09-23T19:36:05.398 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.12808333799999999, 2023-09-23T19:36:05.398 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:36:05.399 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:36:05.399 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:36:05.399 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:36:05.399 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:36:05.399 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 160287 2023-09-23T19:36:05.399 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.399 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:36:05.399 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.399 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:36:05.399 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:14.237433+0000", 2023-09-23T19:36:05.399 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:36:05.399 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.400 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.400 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:36:05.400 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:14.237433+0000", 2023-09-23T19:36:05.400 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T19:36:05.400 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.400 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.400 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:36:05.400 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:14.237432+0000", 2023-09-23T19:36:05.400 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.0870000000000007e-06 2023-09-23T19:36:05.400 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.400 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.400 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:36:05.400 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:14.237440+0000", 2023-09-23T19:36:05.401 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.5954999999999999e-05 2023-09-23T19:36:05.401 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.401 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.401 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:36:05.401 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:14.237456+0000", 2023-09-23T19:36:05.401 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.9964999999999999e-05 2023-09-23T19:36:05.401 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.401 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.401 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:36:05.401 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:14.237476+0000", 2023-09-23T19:36:05.401 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.010201297999999999 2023-09-23T19:36:05.401 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.402 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.402 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:36:05.402 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:14.247677+0000", 2023-09-23T19:36:05.402 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.8652000000000005e-05 2023-09-23T19:36:05.402 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.402 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.402 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:36:05.402 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:14.247756+0000", 2023-09-23T19:36:05.402 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.1507000000000003e-05 2023-09-23T19:36:05.402 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.402 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.402 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3,7", 2023-09-23T19:36:05.403 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:14.247847+0000", 2023-09-23T19:36:05.403 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0083857610000000003 2023-09-23T19:36:05.403 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.403 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.403 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:36:05.403 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:14.256233+0000", 2023-09-23T19:36:05.403 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.013160607 2023-09-23T19:36:05.403 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.403 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.403 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:05.403 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:14.269393+0000", 2023-09-23T19:36:05.403 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.096017149999999996 2023-09-23T19:36:05.403 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.404 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.404 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:05.404 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:14.365411+0000", 2023-09-23T19:36:05.404 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.7432999999999997e-05 2023-09-23T19:36:05.404 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.404 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.404 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:36:05.404 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:14.365468+0000", 2023-09-23T19:36:05.404 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.8723000000000001e-05 2023-09-23T19:36:05.404 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.404 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.404 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:36:05.405 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:14.365517+0000", 2023-09-23T19:36:05.405 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.12808333799999999 2023-09-23T19:36:05.405 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:05.405 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:36:05.405 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:05.405 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.405 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.405 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:160292 9.1 9:872b13a6:::benchmark_data_smithi016_113233_object51910:head [delete] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-09-23T19:36:05.405 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:34:14.246862+0000", 2023-09-23T19:36:05.405 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 111.05030857, 2023-09-23T19:36:05.405 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.118783758, 2023-09-23T19:36:05.405 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:36:05.406 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:36:05.406 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:36:05.406 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:36:05.406 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:36:05.406 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 160292 2023-09-23T19:36:05.406 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.406 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:36:05.406 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.406 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:36:05.406 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:14.246862+0000", 2023-09-23T19:36:05.406 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:36:05.406 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.406 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.407 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:36:05.407 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:14.246862+0000", 2023-09-23T19:36:05.407 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999976 2023-09-23T19:36:05.407 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.407 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.407 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:36:05.407 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:14.246859+0000", 2023-09-23T19:36:05.407 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.0173000000000001e-05 2023-09-23T19:36:05.407 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.409 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.409 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:36:05.409 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:14.246870+0000", 2023-09-23T19:36:05.409 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.0678000000000001e-05 2023-09-23T19:36:05.409 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.409 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.409 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:36:05.409 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:14.246890+0000", 2023-09-23T19:36:05.409 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.3019999999999995e-06 2023-09-23T19:36:05.409 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.409 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.410 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:36:05.410 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:14.246899+0000", 2023-09-23T19:36:05.410 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0051550149999999998 2023-09-23T19:36:05.410 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.410 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.410 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:36:05.410 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:14.252054+0000", 2023-09-23T19:36:05.410 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.5724999999999997e-05 2023-09-23T19:36:05.410 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.410 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.410 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:36:05.411 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:14.252129+0000", 2023-09-23T19:36:05.411 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.3809999999999999e-05 2023-09-23T19:36:05.411 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.411 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.411 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3,7", 2023-09-23T19:36:05.411 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:14.252213+0000", 2023-09-23T19:36:05.411 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0041130919999999996 2023-09-23T19:36:05.411 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.411 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.411 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:36:05.411 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:14.256326+0000", 2023-09-23T19:36:05.411 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.013256824 2023-09-23T19:36:05.412 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.412 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.412 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:05.412 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:14.269583+0000", 2023-09-23T19:36:05.412 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.095977933000000001 2023-09-23T19:36:05.412 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.412 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.412 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:05.412 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:14.365561+0000", 2023-09-23T19:36:05.412 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.0300999999999998e-05 2023-09-23T19:36:05.412 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.412 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.413 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:36:05.413 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:14.365611+0000", 2023-09-23T19:36:05.413 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.4057000000000003e-05 2023-09-23T19:36:05.413 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.413 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.413 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:36:05.413 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:14.365645+0000", 2023-09-23T19:36:05.413 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.118783758 2023-09-23T19:36:05.413 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:05.413 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:36:05.413 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:05.413 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.414 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.414 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:162257 9.1 9:8c85931f:::benchmark_data_smithi016_113233_object53875:head [delete] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-09-23T19:36:05.414 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:34:17.140887+0000", 2023-09-23T19:36:05.414 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 108.1562835, 2023-09-23T19:36:05.414 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.066901651000000006, 2023-09-23T19:36:05.414 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:36:05.414 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:36:05.414 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:36:05.414 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:36:05.414 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:36:05.414 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 162257 2023-09-23T19:36:05.414 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.415 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:36:05.415 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.415 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:36:05.415 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:17.140887+0000", 2023-09-23T19:36:05.415 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:36:05.415 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.415 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.415 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:36:05.415 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:17.140887+0000", 2023-09-23T19:36:05.415 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999976 2023-09-23T19:36:05.415 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.415 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.416 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:36:05.416 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:17.140884+0000", 2023-09-23T19:36:05.416 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.0923e-05 2023-09-23T19:36:05.416 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.416 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.416 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:36:05.416 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:17.140895+0000", 2023-09-23T19:36:05.416 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.3726e-05 2023-09-23T19:36:05.416 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.416 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.416 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:36:05.416 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:17.140919+0000", 2023-09-23T19:36:05.416 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.1290000000000002e-06 2023-09-23T19:36:05.417 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.417 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.417 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:36:05.417 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:17.140928+0000", 2023-09-23T19:36:05.417 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0037599669999999999 2023-09-23T19:36:05.417 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.417 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.417 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:36:05.417 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:17.144688+0000", 2023-09-23T19:36:05.417 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00014342200000000001 2023-09-23T19:36:05.417 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.418 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.418 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:36:05.418 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:17.144832+0000", 2023-09-23T19:36:05.418 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00016055299999999999 2023-09-23T19:36:05.418 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.418 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.418 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3,7", 2023-09-23T19:36:05.418 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:17.144992+0000", 2023-09-23T19:36:05.418 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.010660012999999999 2023-09-23T19:36:05.419 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.419 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.419 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:05.419 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:17.155652+0000", 2023-09-23T19:36:05.419 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.0045e-05 2023-09-23T19:36:05.419 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.419 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.419 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:36:05.419 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:17.155662+0000", 2023-09-23T19:36:05.419 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.052051871999999999 2023-09-23T19:36:05.419 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.420 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.420 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:05.420 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:17.207714+0000", 2023-09-23T19:36:05.420 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.3035999999999999e-05 2023-09-23T19:36:05.420 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.420 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.420 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:36:05.420 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:17.207757+0000", 2023-09-23T19:36:05.420 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.1129e-05 2023-09-23T19:36:05.420 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.421 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.421 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:36:05.421 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:17.207788+0000", 2023-09-23T19:36:05.421 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.066901651000000006 2023-09-23T19:36:05.421 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:05.421 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:36:05.421 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:05.421 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.421 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.422 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:162260 9.1 9:877f13ea:::benchmark_data_smithi016_113233_object53878:head [delete] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-09-23T19:36:05.422 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:34:17.141977+0000", 2023-09-23T19:36:05.422 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 108.15519275299999, 2023-09-23T19:36:05.422 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.065890569999999996, 2023-09-23T19:36:05.422 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:36:05.422 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:36:05.422 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:36:05.422 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:36:05.422 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:36:05.423 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 162260 2023-09-23T19:36:05.423 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.423 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:36:05.423 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.423 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:36:05.423 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:17.141977+0000", 2023-09-23T19:36:05.423 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:36:05.423 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.423 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.423 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:36:05.423 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:17.141977+0000", 2023-09-23T19:36:05.423 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.999999 2023-09-23T19:36:05.424 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.424 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.424 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:36:05.424 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:17.141976+0000", 2023-09-23T19:36:05.424 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.635e-06 2023-09-23T19:36:05.424 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.424 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.424 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:36:05.424 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:17.141983+0000", 2023-09-23T19:36:05.425 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.6138e-05 2023-09-23T19:36:05.425 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.425 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.425 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:36:05.425 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:17.141999+0000", 2023-09-23T19:36:05.425 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.2400000000000001e-06 2023-09-23T19:36:05.425 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.425 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.425 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:36:05.425 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:17.142006+0000", 2023-09-23T19:36:05.426 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0085407580000000007 2023-09-23T19:36:05.426 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.426 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.426 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:36:05.426 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:17.150547+0000", 2023-09-23T19:36:05.426 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000172857 2023-09-23T19:36:05.426 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.426 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.426 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:36:05.426 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:17.150720+0000", 2023-09-23T19:36:05.427 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00016480499999999999 2023-09-23T19:36:05.427 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.427 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.427 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3,7", 2023-09-23T19:36:05.427 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:17.150885+0000", 2023-09-23T19:36:05.427 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0096122529999999994 2023-09-23T19:36:05.427 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.427 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.428 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:36:05.428 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:17.160497+0000", 2023-09-23T19:36:05.428 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.003176735 2023-09-23T19:36:05.428 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.428 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.428 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:05.428 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:17.163674+0000", 2023-09-23T19:36:05.428 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.044143983999999997 2023-09-23T19:36:05.428 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.429 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.429 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:05.429 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:17.207818+0000", 2023-09-23T19:36:05.429 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.4562e-05 2023-09-23T19:36:05.429 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.429 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.429 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:36:05.429 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:17.207842+0000", 2023-09-23T19:36:05.430 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.5711000000000002e-05 2023-09-23T19:36:05.430 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.430 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.430 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:36:05.430 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:17.207868+0000", 2023-09-23T19:36:05.430 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.065890569999999996 2023-09-23T19:36:05.430 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:05.430 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:36:05.430 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:05.431 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.431 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.431 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:162261 9.1 9:8764259d:::benchmark_data_smithi016_113233_object53879:head [delete] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-09-23T19:36:05.431 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:34:17.142589+0000", 2023-09-23T19:36:05.431 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 108.15458124600001, 2023-09-23T19:36:05.431 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.065410889999999999, 2023-09-23T19:36:05.431 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:36:05.431 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:36:05.431 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:36:05.431 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:36:05.431 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:36:05.431 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 162261 2023-09-23T19:36:05.432 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.432 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:36:05.432 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.432 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:36:05.432 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:17.142589+0000", 2023-09-23T19:36:05.432 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:36:05.432 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.432 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.432 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:36:05.432 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:17.142589+0000", 2023-09-23T19:36:05.432 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.999999 2023-09-23T19:36:05.433 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.433 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.433 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:36:05.433 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:17.142588+0000", 2023-09-23T19:36:05.433 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.7680000000000003e-06 2023-09-23T19:36:05.433 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.433 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.433 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:36:05.433 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:17.142594+0000", 2023-09-23T19:36:05.433 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.6266e-05 2023-09-23T19:36:05.433 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.433 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.434 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:36:05.434 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:17.142611+0000", 2023-09-23T19:36:05.434 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.5509999999999996e-06 2023-09-23T19:36:05.434 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.434 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.434 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:36:05.434 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:17.142617+0000", 2023-09-23T19:36:05.434 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.013090467999999999 2023-09-23T19:36:05.434 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.434 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.434 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:36:05.435 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:17.155708+0000", 2023-09-23T19:36:05.435 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00012475799999999999 2023-09-23T19:36:05.435 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.435 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.435 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:36:05.435 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:17.155833+0000", 2023-09-23T19:36:05.435 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00014106000000000001 2023-09-23T19:36:05.435 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.435 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.435 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3,7", 2023-09-23T19:36:05.435 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:17.155974+0000", 2023-09-23T19:36:05.435 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0046575920000000003 2023-09-23T19:36:05.436 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.436 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.436 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:36:05.436 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:17.160631+0000", 2023-09-23T19:36:05.436 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0032175010000000002 2023-09-23T19:36:05.436 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.436 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.436 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:05.436 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:17.163849+0000", 2023-09-23T19:36:05.436 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.044069143999999998 2023-09-23T19:36:05.436 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.436 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.437 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:05.437 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:17.207918+0000", 2023-09-23T19:36:05.437 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.0561e-05 2023-09-23T19:36:05.437 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.437 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.437 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:36:05.437 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:17.207958+0000", 2023-09-23T19:36:05.437 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.1394999999999999e-05 2023-09-23T19:36:05.437 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.437 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.437 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:36:05.437 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:17.208000+0000", 2023-09-23T19:36:05.438 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.065410889999999999 2023-09-23T19:36:05.438 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:05.438 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:36:05.438 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:05.438 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.438 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.438 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:184382 9.1 9:83418b97:::benchmark_data_smithi016_113233_object76000:head [delete] snapc 0=[] ondisk+write+known_if_redirected e133)", 2023-09-23T19:36:05.438 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:34:50.109559+0000", 2023-09-23T19:36:05.438 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 75.187611098999994, 2023-09-23T19:36:05.438 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.068870347999999998, 2023-09-23T19:36:05.438 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:36:05.438 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:36:05.439 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:36:05.439 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:36:05.439 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:36:05.439 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 184382 2023-09-23T19:36:05.439 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.439 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:36:05.439 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.439 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:36:05.439 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:50.109559+0000", 2023-09-23T19:36:05.439 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:36:05.439 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.439 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.439 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:36:05.440 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:50.109559+0000", 2023-09-23T19:36:05.440 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T19:36:05.440 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.440 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.440 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:36:05.440 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:50.109557+0000", 2023-09-23T19:36:05.440 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.5359999999999998e-06 2023-09-23T19:36:05.440 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.440 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.440 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:36:05.440 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:50.109567+0000", 2023-09-23T19:36:05.441 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.9672000000000001e-05 2023-09-23T19:36:05.441 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.441 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.441 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:36:05.441 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:50.109586+0000", 2023-09-23T19:36:05.441 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.4590000000000001e-06 2023-09-23T19:36:05.441 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.441 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.441 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:36:05.441 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:50.109594+0000", 2023-09-23T19:36:05.441 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.9985999999999998e-05 2023-09-23T19:36:05.442 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.442 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.442 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:36:05.442 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:50.109654+0000", 2023-09-23T19:36:05.442 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000151201 2023-09-23T19:36:05.442 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.442 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.442 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:36:05.442 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:50.109805+0000", 2023-09-23T19:36:05.442 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00016958300000000001 2023-09-23T19:36:05.442 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.442 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.443 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3,7", 2023-09-23T19:36:05.443 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:50.109975+0000", 2023-09-23T19:36:05.443 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0055109089999999996 2023-09-23T19:36:05.443 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.443 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.443 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:36:05.443 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:50.115486+0000", 2023-09-23T19:36:05.443 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.015112811 2023-09-23T19:36:05.443 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.443 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.443 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:05.443 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:50.130598+0000", 2023-09-23T19:36:05.444 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.047728872999999998 2023-09-23T19:36:05.444 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.444 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.444 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:05.444 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:50.178327+0000", 2023-09-23T19:36:05.444 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.1926999999999999e-05 2023-09-23T19:36:05.444 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.444 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.444 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:36:05.444 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:50.178379+0000", 2023-09-23T19:36:05.444 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.0173999999999998e-05 2023-09-23T19:36:05.445 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.445 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.445 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:36:05.445 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:50.178429+0000", 2023-09-23T19:36:05.445 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.068870347999999998 2023-09-23T19:36:05.445 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:05.445 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:36:05.445 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:05.445 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.445 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.446 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:184388 9.1 9:88a45c45:::benchmark_data_smithi016_113233_object76006:head [delete] snapc 0=[] ondisk+write+known_if_redirected e133)", 2023-09-23T19:36:05.446 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:34:50.115459+0000", 2023-09-23T19:36:05.446 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 75.181711273999994, 2023-09-23T19:36:05.446 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.063084874999999999, 2023-09-23T19:36:05.446 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:36:05.446 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:36:05.446 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:36:05.446 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:36:05.446 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:36:05.446 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 184388 2023-09-23T19:36:05.446 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.447 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:36:05.447 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.447 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:36:05.447 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:50.115459+0000", 2023-09-23T19:36:05.447 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:36:05.447 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.447 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.447 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:36:05.447 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:50.115459+0000", 2023-09-23T19:36:05.447 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T19:36:05.447 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.447 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.448 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:36:05.448 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:50.115457+0000", 2023-09-23T19:36:05.448 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.6160000000000008e-06 2023-09-23T19:36:05.448 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.448 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.448 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:36:05.448 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:50.115467+0000", 2023-09-23T19:36:05.448 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.8603000000000001e-05 2023-09-23T19:36:05.448 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.448 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.448 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:36:05.448 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:50.115485+0000", 2023-09-23T19:36:05.449 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.3799999999999996e-06 2023-09-23T19:36:05.449 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.449 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.449 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:36:05.449 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:50.115493+0000", 2023-09-23T19:36:05.449 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.3837000000000003e-05 2023-09-23T19:36:05.449 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.449 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.449 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:36:05.449 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:50.115586+0000", 2023-09-23T19:36:05.449 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000144241 2023-09-23T19:36:05.449 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.450 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.450 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:36:05.450 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:50.115731+0000", 2023-09-23T19:36:05.450 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00016478599999999999 2023-09-23T19:36:05.450 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.450 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.450 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3,7", 2023-09-23T19:36:05.450 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:50.115895+0000", 2023-09-23T19:36:05.450 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.014207294000000001 2023-09-23T19:36:05.450 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.451 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.451 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:36:05.451 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:50.130103+0000", 2023-09-23T19:36:05.451 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00067519200000000002 2023-09-23T19:36:05.451 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.451 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.451 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:05.451 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:50.130778+0000", 2023-09-23T19:36:05.451 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.047689415999999998 2023-09-23T19:36:05.452 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.452 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.452 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:05.452 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:50.178467+0000", 2023-09-23T19:36:05.452 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.9854000000000002e-05 2023-09-23T19:36:05.452 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.452 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.452 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:36:05.452 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:50.178507+0000", 2023-09-23T19:36:05.452 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.6536999999999999e-05 2023-09-23T19:36:05.452 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.452 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.453 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:36:05.453 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:50.178544+0000", 2023-09-23T19:36:05.453 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.063084874999999999 2023-09-23T19:36:05.453 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:05.453 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:36:05.453 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:05.453 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.453 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.453 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:184392 9.1 9:8565a640:::benchmark_data_smithi016_113233_object76010:head [delete] snapc 0=[] ondisk+write+known_if_redirected e133)", 2023-09-23T19:36:05.454 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:34:50.119355+0000", 2023-09-23T19:36:05.454 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 75.177814982000001, 2023-09-23T19:36:05.454 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.074008462999999997, 2023-09-23T19:36:05.454 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:36:05.454 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:36:05.454 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:36:05.454 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:36:05.454 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:36:05.454 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 184392 2023-09-23T19:36:05.455 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.455 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:36:05.455 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.455 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:36:05.455 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:50.119355+0000", 2023-09-23T19:36:05.455 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:36:05.455 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.455 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.455 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:36:05.455 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:50.119355+0000", 2023-09-23T19:36:05.456 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T19:36:05.456 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.456 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.456 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:36:05.456 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:50.119353+0000", 2023-09-23T19:36:05.456 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.0036e-05 2023-09-23T19:36:05.456 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.456 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.456 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:36:05.456 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:50.119363+0000", 2023-09-23T19:36:05.456 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.9517e-05 2023-09-23T19:36:05.457 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.457 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.457 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:36:05.457 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:50.119383+0000", 2023-09-23T19:36:05.457 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.2970000000000004e-06 2023-09-23T19:36:05.457 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.457 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.457 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:36:05.457 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:50.119391+0000", 2023-09-23T19:36:05.458 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0022779639999999999 2023-09-23T19:36:05.458 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.458 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.458 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:36:05.458 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:50.121669+0000", 2023-09-23T19:36:05.458 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00030524600000000001 2023-09-23T19:36:05.458 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.458 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.459 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:36:05.459 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:50.121974+0000", 2023-09-23T19:36:05.459 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000447336 2023-09-23T19:36:05.459 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.459 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.459 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3,7", 2023-09-23T19:36:05.459 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:50.122422+0000", 2023-09-23T19:36:05.459 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0076902209999999997 2023-09-23T19:36:05.459 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.459 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.459 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:36:05.460 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:50.130112+0000", 2023-09-23T19:36:05.460 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000714984 2023-09-23T19:36:05.460 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.460 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.460 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:05.460 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:50.130827+0000", 2023-09-23T19:36:05.460 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.062430892000000002 2023-09-23T19:36:05.460 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.460 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.460 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:05.460 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:50.193258+0000", 2023-09-23T19:36:05.461 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.402e-05 2023-09-23T19:36:05.461 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.461 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.461 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:36:05.461 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:50.193322+0000", 2023-09-23T19:36:05.461 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.1967e-05 2023-09-23T19:36:05.461 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:05.461 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:05.461 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:36:05.461 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:34:50.193364+0000", 2023-09-23T19:36:05.461 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.074008462999999997 2023-09-23T19:36:05.461 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:05.462 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:36:05.462 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:05.462 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:05.462 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:36:05.462 INFO:teuthology.orchestra.run.smithi092.stdout:} 2023-09-23T19:36:05.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:05.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:05.533 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:05 smithi092 bash[58855]: cluster 2023-09-23T19:36:04.053194+0000 mgr.y (mgr.25325) 855 : cluster 0 pgmap v770: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:36:05.564 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:05.591 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:05.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:05.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:36:05.694 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:05.719 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:05.721 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:05 smithi016 bash[75004]: cluster 2023-09-23T19:36:04.053194+0000 mgr.y (mgr.25325) 855 : cluster 0 pgmap v770: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:36:05.721 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:36:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:36:05] "GET /metrics HTTP/1.1" 200 33878 "" "Prometheus/2.33.4" 2023-09-23T19:36:05.721 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:05 smithi016 bash[77995]: cluster 2023-09-23T19:36:04.053194+0000 mgr.y (mgr.25325) 855 : cluster 0 pgmap v770: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:36:05.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:05.722 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:36:05.823 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:05.848 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:05.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:05.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:36:05.951 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:05.972 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:05.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:05.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:06.075 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:06.109 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:06.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:06.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:06.131 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T19:36:06.214 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:06.247 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:06.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:06.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:06.352 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:06.380 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:06.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:06.383 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:36:06.484 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:06.510 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:06.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:06.513 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:36:06.614 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:06.637 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:06.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:06.639 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:36:06.740 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:06.771 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:06.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:06.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:06.875 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:06.906 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:06.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:06.909 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:07.010 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:07.040 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:07.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:07.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:36:07.144 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:07.177 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:07.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:07.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:36:07.281 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:07.314 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:07.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:07.317 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:07.418 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:07.448 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:07.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:07.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:36:07.552 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:07.576 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:07.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:07.579 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:36:07.680 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:07.705 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:07.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:07.707 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:07.722 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:07 smithi016 bash[75004]: cluster 2023-09-23T19:36:06.054154+0000 mgr.y (mgr.25325) 856 : cluster 0 pgmap v771: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:36:07.722 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:07 smithi016 bash[77995]: cluster 2023-09-23T19:36:06.054154+0000 mgr.y (mgr.25325) 856 : cluster 0 pgmap v771: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:36:07.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:07 smithi092 bash[58855]: cluster 2023-09-23T19:36:06.054154+0000 mgr.y (mgr.25325) 856 : cluster 0 pgmap v771: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:36:07.809 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:07.821 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:36:07.835 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:07.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:07.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:36:07.843 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:36:07.862 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:36:07.890 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:36:07.913 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:36:07.937 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:07.938 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:36:07.962 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:36:07.965 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:07.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:07.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:36:07.989 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:36:07.990 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:36:07.991 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:36:07.991 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:36:07.991 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:36:07.991 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:36:07.991 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:36:07.991 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:36:07.991 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:36:08.068 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:08.095 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:08.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:08.098 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:08.200 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:08.231 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:08.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:08.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:36:08.334 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:08.368 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:08.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:08.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:08.471 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:08.477 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is unset 2023-09-23T19:36:08.492 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:08.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:08.495 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:36:08.596 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:08.607 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:08 smithi016 bash[75004]: audit 2023-09-23T19:36:07.756840+0000 mon.a (mon.0) 1063 : audit 1 from='client.? 172.21.15.16:0/1180170847' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:36:08.607 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:08 smithi016 bash[75004]: audit 2023-09-23T19:36:07.796777+0000 mon.a (mon.0) 1064 : audit 0 from='client.? 172.21.15.16:0/2444607031' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:36:08.608 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:08 smithi016 bash[77995]: audit 2023-09-23T19:36:07.756840+0000 mon.a (mon.0) 1063 : audit 1 from='client.? 172.21.15.16:0/1180170847' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:36:08.608 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:08 smithi016 bash[77995]: audit 2023-09-23T19:36:07.796777+0000 mon.a (mon.0) 1064 : audit 0 from='client.? 172.21.15.16:0/2444607031' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:36:08.624 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:08.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:08.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:36:08.728 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:08.740 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:08 smithi092 bash[58855]: audit 2023-09-23T19:36:07.756840+0000 mon.a (mon.0) 1063 : audit 1 from='client.? 172.21.15.16:0/1180170847' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:36:08.740 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:08 smithi092 bash[58855]: audit 2023-09-23T19:36:07.796777+0000 mon.a (mon.0) 1064 : audit 0 from='client.? 172.21.15.16:0/2444607031' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:36:08.762 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:08.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:08.766 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:08.867 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:08.897 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:08.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:08.901 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:09.002 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:09.034 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:09.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:09.037 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:09.138 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:09.168 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:09.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:09.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:09.273 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:09.304 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:09.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:09.307 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:09.408 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:09.439 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:09.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:09.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:36:09.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:09 smithi092 bash[58855]: cluster 2023-09-23T19:36:08.054977+0000 mgr.y (mgr.25325) 857 : cluster 0 pgmap v772: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:36:09.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:09 smithi092 bash[58855]: cluster 2023-09-23T19:36:08.458276+0000 mon.a (mon.0) 1065 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:36:09.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:09 smithi092 bash[58855]: cluster 2023-09-23T19:36:08.458316+0000 mon.a (mon.0) 1066 : cluster 1 Cluster is now healthy 2023-09-23T19:36:09.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:09 smithi092 bash[58855]: audit 2023-09-23T19:36:08.468812+0000 mon.a (mon.0) 1067 : audit 1 from='client.? 172.21.15.16:0/1180170847' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:36:09.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:09 smithi092 bash[58855]: cluster 2023-09-23T19:36:08.468872+0000 mon.a (mon.0) 1068 : cluster 0 osdmap e143: 8 total, 8 up, 8 in 2023-09-23T19:36:09.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:09 smithi092 bash[58855]: cluster 2023-09-23T19:36:08.471704+0000 osd.6 (osd.6) 61 : cluster 3 failed to encode map e143 with expected crc 2023-09-23T19:36:09.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:09 smithi092 bash[58855]: cluster 2023-09-23T19:36:09.087899+0000 osd.4 (osd.4) 71 : cluster 3 failed to encode map e143 with expected crc 2023-09-23T19:36:09.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:09 smithi092 bash[58855]: cluster 2023-09-23T19:36:09.091738+0000 osd.4 (osd.4) 72 : cluster 3 failed to encode map e143 with expected crc 2023-09-23T19:36:09.543 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:09.569 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:09.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:09.571 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:36:09.576 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:09 smithi016 bash[75004]: cluster 2023-09-23T19:36:08.054977+0000 mgr.y (mgr.25325) 857 : cluster 0 pgmap v772: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:36:09.576 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:09 smithi016 bash[75004]: cluster 2023-09-23T19:36:08.458276+0000 mon.a (mon.0) 1065 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:36:09.577 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:09 smithi016 bash[75004]: cluster 2023-09-23T19:36:08.458316+0000 mon.a (mon.0) 1066 : cluster 1 Cluster is now healthy 2023-09-23T19:36:09.577 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:09 smithi016 bash[75004]: audit 2023-09-23T19:36:08.468812+0000 mon.a (mon.0) 1067 : audit 1 from='client.? 172.21.15.16:0/1180170847' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:36:09.577 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:09 smithi016 bash[75004]: cluster 2023-09-23T19:36:08.468872+0000 mon.a (mon.0) 1068 : cluster 0 osdmap e143: 8 total, 8 up, 8 in 2023-09-23T19:36:09.577 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:09 smithi016 bash[75004]: cluster 2023-09-23T19:36:08.471704+0000 osd.6 (osd.6) 61 : cluster 3 failed to encode map e143 with expected crc 2023-09-23T19:36:09.577 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:09 smithi016 bash[75004]: cluster 2023-09-23T19:36:09.087899+0000 osd.4 (osd.4) 71 : cluster 3 failed to encode map e143 with expected crc 2023-09-23T19:36:09.577 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:09 smithi016 bash[75004]: cluster 2023-09-23T19:36:09.091738+0000 osd.4 (osd.4) 72 : cluster 3 failed to encode map e143 with expected crc 2023-09-23T19:36:09.577 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:09 smithi016 bash[77995]: cluster 2023-09-23T19:36:08.054977+0000 mgr.y (mgr.25325) 857 : cluster 0 pgmap v772: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:36:09.577 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:09 smithi016 bash[77995]: cluster 2023-09-23T19:36:08.458276+0000 mon.a (mon.0) 1065 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:36:09.577 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:09 smithi016 bash[77995]: cluster 2023-09-23T19:36:08.458316+0000 mon.a (mon.0) 1066 : cluster 1 Cluster is now healthy 2023-09-23T19:36:09.578 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:09 smithi016 bash[77995]: audit 2023-09-23T19:36:08.468812+0000 mon.a (mon.0) 1067 : audit 1 from='client.? 172.21.15.16:0/1180170847' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:36:09.578 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:09 smithi016 bash[77995]: cluster 2023-09-23T19:36:08.468872+0000 mon.a (mon.0) 1068 : cluster 0 osdmap e143: 8 total, 8 up, 8 in 2023-09-23T19:36:09.578 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:09 smithi016 bash[77995]: cluster 2023-09-23T19:36:08.471704+0000 osd.6 (osd.6) 61 : cluster 3 failed to encode map e143 with expected crc 2023-09-23T19:36:09.578 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:09 smithi016 bash[77995]: cluster 2023-09-23T19:36:09.087899+0000 osd.4 (osd.4) 71 : cluster 3 failed to encode map e143 with expected crc 2023-09-23T19:36:09.578 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:09 smithi016 bash[77995]: cluster 2023-09-23T19:36:09.091738+0000 osd.4 (osd.4) 72 : cluster 3 failed to encode map e143 with expected crc 2023-09-23T19:36:09.672 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:09.707 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:09.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:09.711 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:09.811 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:09.847 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:09.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:09.850 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:09.951 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:09.982 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:09.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:09.985 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:10.086 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:10.110 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:10.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:10.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:36:10.214 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:10.248 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:10.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:10.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:36:10.352 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:10.381 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:10.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:10.384 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:36:10.484 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:10.506 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:10.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:10.509 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:36:10.609 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:10.632 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:10.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:10.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:10.735 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:10.760 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:10.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:10.763 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:36:10.863 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:10.896 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:10.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:10.899 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:10.951 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:36:11.000 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:11.026 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:11.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:11.029 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:11.129 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:11.162 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:11.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:11.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:11.266 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:11.288 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:11.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:11.290 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:11.391 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:11.420 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:11.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:11.422 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:11.523 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:11.549 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:11.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:11.552 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:11.653 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:11.683 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:11.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:11.685 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:11.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:11 smithi016 bash[75004]: cluster 2023-09-23T19:36:10.055648+0000 mgr.y (mgr.25325) 858 : cluster 0 pgmap v774: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:36:11.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:11 smithi016 bash[77995]: cluster 2023-09-23T19:36:10.055648+0000 mgr.y (mgr.25325) 858 : cluster 0 pgmap v774: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:36:11.786 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:11.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:11 smithi092 bash[58855]: cluster 2023-09-23T19:36:10.055648+0000 mgr.y (mgr.25325) 858 : cluster 0 pgmap v774: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:36:11.822 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:11.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:11.825 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:11.926 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:11.955 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:11.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:11.957 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:12.058 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:12.088 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:12.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:12.091 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:36:12.191 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:12.224 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:12.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:12.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:12.328 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:12.351 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:12.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:12.354 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:36:12.455 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:12.483 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:12.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:12.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:36:12.587 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:12.615 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:12.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:12.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:12.719 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:12.740 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:12.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:12.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:12.844 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:12.869 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:12.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:12.872 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:36:12.973 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:13.001 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:13.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:13.004 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:36:13.105 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:13.131 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:13.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:13.134 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:36:13.235 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:13.243 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:36:13.257 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:13.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:13.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:13.360 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:13.387 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:13.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:13.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:13.490 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:13.491 INFO:teuthology.orchestra.run.smithi016.stderr:pool 'unique_pool_5' created 2023-09-23T19:36:13.493 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T19:36:13.515 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:13.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:13.517 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:36:13.521 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:13 smithi016 bash[75004]: cluster 2023-09-23T19:36:12.056427+0000 mgr.y (mgr.25325) 859 : cluster 0 pgmap v775: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:36:13.521 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:13 smithi016 bash[75004]: audit 2023-09-23T19:36:12.613932+0000 mon.a (mon.0) 1069 : audit 1 from='client.? 172.21.15.16:0/2174489650' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T19:36:13.521 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:13 smithi016 bash[75004]: audit 2023-09-23T19:36:12.667199+0000 mon.a (mon.0) 1070 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-23T19:36:13.521 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:13 smithi016 bash[75004]: audit 2023-09-23T19:36:12.671301+0000 mon.b (mon.1) 42 : audit 1 from='client.? 172.21.15.16:0/1231760236' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-23T19:36:13.522 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:13 smithi016 bash[77995]: cluster 2023-09-23T19:36:12.056427+0000 mgr.y (mgr.25325) 859 : cluster 0 pgmap v775: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:36:13.522 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:13 smithi016 bash[77995]: audit 2023-09-23T19:36:12.613932+0000 mon.a (mon.0) 1069 : audit 1 from='client.? 172.21.15.16:0/2174489650' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T19:36:13.522 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:13 smithi016 bash[77995]: audit 2023-09-23T19:36:12.667199+0000 mon.a (mon.0) 1070 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-23T19:36:13.522 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:13 smithi016 bash[77995]: audit 2023-09-23T19:36:12.671301+0000 mon.b (mon.1) 42 : audit 1 from='client.? 172.21.15.16:0/1231760236' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-23T19:36:13.619 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:13.644 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:13.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:13.647 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:36:13.713 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T19:36:13.748 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:13.760 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:13 smithi092 bash[58855]: cluster 2023-09-23T19:36:12.056427+0000 mgr.y (mgr.25325) 859 : cluster 0 pgmap v775: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:36:13.760 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:13 smithi092 bash[58855]: audit 2023-09-23T19:36:12.613932+0000 mon.a (mon.0) 1069 : audit 1 from='client.? 172.21.15.16:0/2174489650' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T19:36:13.760 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:13 smithi092 bash[58855]: audit 2023-09-23T19:36:12.667199+0000 mon.a (mon.0) 1070 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-23T19:36:13.760 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:13 smithi092 bash[58855]: audit 2023-09-23T19:36:12.671301+0000 mon.b (mon.1) 42 : audit 1 from='client.? 172.21.15.16:0/1231760236' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-23T19:36:13.776 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:13.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:13.778 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:13.878 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:13.908 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:13.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:13.911 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:14.011 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:14.039 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:14.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:14.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:14.143 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:14.164 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:14.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:14.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:14.229 INFO:teuthology.orchestra.run.smithi092.stdout:ERROR: (22) Invalid argument 2023-09-23T19:36:14.230 INFO:teuthology.orchestra.run.smithi092.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:36:14.268 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:14.296 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:14.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:14.298 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:36:14.399 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:14.427 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:14.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:14.430 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:14.531 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:14.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:14 smithi092 bash[58855]: audit 2023-09-23T19:36:13.481563+0000 mon.a (mon.0) 1071 : audit 1 from='client.? 172.21.15.16:0/2174489650' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T19:36:14.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:14 smithi092 bash[58855]: audit 2023-09-23T19:36:13.481930+0000 mon.a (mon.0) 1072 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-09-23T19:36:14.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:14 smithi092 bash[58855]: cluster 2023-09-23T19:36:13.482066+0000 mon.a (mon.0) 1073 : cluster 0 osdmap e144: 8 total, 8 up, 8 in 2023-09-23T19:36:14.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:14 smithi092 bash[58855]: cluster 2023-09-23T19:36:13.486672+0000 osd.5 (osd.5) 75 : cluster 3 failed to encode map e144 with expected crc 2023-09-23T19:36:14.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:14 smithi092 bash[58855]: cluster 2023-09-23T19:36:13.489869+0000 osd.4 (osd.4) 73 : cluster 3 failed to encode map e144 with expected crc 2023-09-23T19:36:14.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:14 smithi092 bash[58855]: cluster 2023-09-23T19:36:13.490400+0000 osd.7 (osd.7) 61 : cluster 3 failed to encode map e144 with expected crc 2023-09-23T19:36:14.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:14 smithi092 bash[58855]: audit 2023-09-23T19:36:13.647715+0000 mon.a (mon.0) 1074 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:36:14.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:14 smithi092 bash[58855]: audit 2023-09-23T19:36:13.648883+0000 mon.a (mon.0) 1075 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:36:14.554 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:14.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:14.557 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:36:14.657 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:14.684 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:14.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:14.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:14.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:14 smithi016 bash[77995]: audit 2023-09-23T19:36:13.481563+0000 mon.a (mon.0) 1071 : audit 1 from='client.? 172.21.15.16:0/2174489650' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T19:36:14.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:14 smithi016 bash[77995]: audit 2023-09-23T19:36:13.481930+0000 mon.a (mon.0) 1072 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-09-23T19:36:14.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:14 smithi016 bash[77995]: cluster 2023-09-23T19:36:13.482066+0000 mon.a (mon.0) 1073 : cluster 0 osdmap e144: 8 total, 8 up, 8 in 2023-09-23T19:36:14.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:14 smithi016 bash[77995]: cluster 2023-09-23T19:36:13.486672+0000 osd.5 (osd.5) 75 : cluster 3 failed to encode map e144 with expected crc 2023-09-23T19:36:14.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:14 smithi016 bash[77995]: cluster 2023-09-23T19:36:13.489869+0000 osd.4 (osd.4) 73 : cluster 3 failed to encode map e144 with expected crc 2023-09-23T19:36:14.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:14 smithi016 bash[77995]: cluster 2023-09-23T19:36:13.490400+0000 osd.7 (osd.7) 61 : cluster 3 failed to encode map e144 with expected crc 2023-09-23T19:36:14.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:14 smithi016 bash[77995]: audit 2023-09-23T19:36:13.647715+0000 mon.a (mon.0) 1074 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:36:14.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:14 smithi016 bash[77995]: audit 2023-09-23T19:36:13.648883+0000 mon.a (mon.0) 1075 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:36:14.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:14 smithi016 bash[75004]: audit 2023-09-23T19:36:13.481563+0000 mon.a (mon.0) 1071 : audit 1 from='client.? 172.21.15.16:0/2174489650' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T19:36:14.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:14 smithi016 bash[75004]: audit 2023-09-23T19:36:13.481930+0000 mon.a (mon.0) 1072 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-09-23T19:36:14.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:14 smithi016 bash[75004]: cluster 2023-09-23T19:36:13.482066+0000 mon.a (mon.0) 1073 : cluster 0 osdmap e144: 8 total, 8 up, 8 in 2023-09-23T19:36:14.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:14 smithi016 bash[75004]: cluster 2023-09-23T19:36:13.486672+0000 osd.5 (osd.5) 75 : cluster 3 failed to encode map e144 with expected crc 2023-09-23T19:36:14.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:14 smithi016 bash[75004]: cluster 2023-09-23T19:36:13.489869+0000 osd.4 (osd.4) 73 : cluster 3 failed to encode map e144 with expected crc 2023-09-23T19:36:14.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:14 smithi016 bash[75004]: cluster 2023-09-23T19:36:13.490400+0000 osd.7 (osd.7) 61 : cluster 3 failed to encode map e144 with expected crc 2023-09-23T19:36:14.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:14 smithi016 bash[75004]: audit 2023-09-23T19:36:13.647715+0000 mon.a (mon.0) 1074 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:36:14.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:14 smithi016 bash[75004]: audit 2023-09-23T19:36:13.648883+0000 mon.a (mon.0) 1075 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:36:14.788 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:14.820 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:14.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:14.823 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:14.831 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T19:36:14.838 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_5 rados --yes-i-really-mean-it || true 2023-09-23T19:36:14.924 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:14.954 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:14.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:14.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:15.058 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:15.083 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:15.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:15.086 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:15.153 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T19:36:15.187 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:15.216 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:15.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:15.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:36:15.320 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:15.354 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:15.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:15.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:15.457 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:15.490 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:15.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:15.495 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:36:15.595 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:15.607 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:15 smithi092 bash[58855]: cluster 2023-09-23T19:36:14.056798+0000 mgr.y (mgr.25325) 860 : cluster 0 pgmap v777: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:36:15.607 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:15 smithi092 bash[58855]: cluster 2023-09-23T19:36:14.495585+0000 mon.a (mon.0) 1076 : cluster 0 osdmap e145: 8 total, 8 up, 8 in 2023-09-23T19:36:15.607 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:15 smithi092 bash[58855]: cluster 2023-09-23T19:36:14.501110+0000 osd.6 (osd.6) 62 : cluster 3 failed to encode map e145 with expected crc 2023-09-23T19:36:15.608 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:15 smithi092 bash[58855]: cluster 2023-09-23T19:36:14.501124+0000 osd.5 (osd.5) 76 : cluster 3 failed to encode map e145 with expected crc 2023-09-23T19:36:15.608 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:15 smithi092 bash[58855]: cluster 2023-09-23T19:36:14.501126+0000 osd.4 (osd.4) 74 : cluster 3 failed to encode map e145 with expected crc 2023-09-23T19:36:15.608 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:15 smithi092 bash[58855]: cluster 2023-09-23T19:36:14.501313+0000 osd.7 (osd.7) 62 : cluster 3 failed to encode map e145 with expected crc 2023-09-23T19:36:15.623 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:15.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:15.627 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:15.727 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:15.741 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:36:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:36:15] "GET /metrics HTTP/1.1" 200 36226 "" "Prometheus/2.33.4" 2023-09-23T19:36:15.741 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:15 smithi016 bash[75004]: cluster 2023-09-23T19:36:14.056798+0000 mgr.y (mgr.25325) 860 : cluster 0 pgmap v777: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:36:15.741 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:15 smithi016 bash[75004]: cluster 2023-09-23T19:36:14.495585+0000 mon.a (mon.0) 1076 : cluster 0 osdmap e145: 8 total, 8 up, 8 in 2023-09-23T19:36:15.742 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:15 smithi016 bash[75004]: cluster 2023-09-23T19:36:14.501110+0000 osd.6 (osd.6) 62 : cluster 3 failed to encode map e145 with expected crc 2023-09-23T19:36:15.742 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:15 smithi016 bash[75004]: cluster 2023-09-23T19:36:14.501124+0000 osd.5 (osd.5) 76 : cluster 3 failed to encode map e145 with expected crc 2023-09-23T19:36:15.742 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:15 smithi016 bash[75004]: cluster 2023-09-23T19:36:14.501126+0000 osd.4 (osd.4) 74 : cluster 3 failed to encode map e145 with expected crc 2023-09-23T19:36:15.742 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:15 smithi016 bash[75004]: cluster 2023-09-23T19:36:14.501313+0000 osd.7 (osd.7) 62 : cluster 3 failed to encode map e145 with expected crc 2023-09-23T19:36:15.742 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:15 smithi016 bash[77995]: cluster 2023-09-23T19:36:14.056798+0000 mgr.y (mgr.25325) 860 : cluster 0 pgmap v777: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:36:15.742 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:15 smithi016 bash[77995]: cluster 2023-09-23T19:36:14.495585+0000 mon.a (mon.0) 1076 : cluster 0 osdmap e145: 8 total, 8 up, 8 in 2023-09-23T19:36:15.742 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:15 smithi016 bash[77995]: cluster 2023-09-23T19:36:14.501110+0000 osd.6 (osd.6) 62 : cluster 3 failed to encode map e145 with expected crc 2023-09-23T19:36:15.742 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:15 smithi016 bash[77995]: cluster 2023-09-23T19:36:14.501124+0000 osd.5 (osd.5) 76 : cluster 3 failed to encode map e145 with expected crc 2023-09-23T19:36:15.742 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:15 smithi016 bash[77995]: cluster 2023-09-23T19:36:14.501126+0000 osd.4 (osd.4) 74 : cluster 3 failed to encode map e145 with expected crc 2023-09-23T19:36:15.742 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:15 smithi016 bash[77995]: cluster 2023-09-23T19:36:14.501313+0000 osd.7 (osd.7) 62 : cluster 3 failed to encode map e145 with expected crc 2023-09-23T19:36:15.759 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:15.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:15.765 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:36:15.866 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:15.893 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:15.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:15.897 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:15.998 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:16.029 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:16.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:16.032 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:36:16.133 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:16.164 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:16.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:16.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:16.268 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:16.299 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:16.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:16.302 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:36:16.403 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:16.436 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:16.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:16.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:16.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:16 smithi016 bash[75004]: cluster 2023-09-23T19:36:15.501029+0000 mon.a (mon.0) 1077 : cluster 0 osdmap e146: 8 total, 8 up, 8 in 2023-09-23T19:36:16.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:16 smithi016 bash[75004]: cluster 2023-09-23T19:36:16.038844+0000 osd.5 (osd.5) 77 : cluster 3 failed to encode map e146 with expected crc 2023-09-23T19:36:16.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:16 smithi016 bash[77995]: cluster 2023-09-23T19:36:15.501029+0000 mon.a (mon.0) 1077 : cluster 0 osdmap e146: 8 total, 8 up, 8 in 2023-09-23T19:36:16.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:16 smithi016 bash[77995]: cluster 2023-09-23T19:36:16.038844+0000 osd.5 (osd.5) 77 : cluster 3 failed to encode map e146 with expected crc 2023-09-23T19:36:16.541 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:16.572 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:16.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:16.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:16.676 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:16.688 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:16 smithi092 bash[58855]: cluster 2023-09-23T19:36:15.501029+0000 mon.a (mon.0) 1077 : cluster 0 osdmap e146: 8 total, 8 up, 8 in 2023-09-23T19:36:16.688 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:16 smithi092 bash[58855]: cluster 2023-09-23T19:36:16.038844+0000 osd.5 (osd.5) 77 : cluster 3 failed to encode map e146 with expected crc 2023-09-23T19:36:16.706 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:16.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:16.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:16.811 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:16.843 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:16.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:16.846 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:36:16.947 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:16.978 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:16.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:16.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:17.082 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:17.115 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:17.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:17.118 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:36:17.219 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:17.252 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:17.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:17.254 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:17.355 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:17.388 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:17.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:17.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:17.492 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:17.525 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:17.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:17.527 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:36:17.628 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:17.662 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:17.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:17.666 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:36:17.766 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:17.779 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:17 smithi016 bash[77995]: cluster 2023-09-23T19:36:16.057496+0000 mgr.y (mgr.25325) 861 : cluster 0 pgmap v780: 89 pgs: 5 unknown, 84 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:36:17.779 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:17 smithi016 bash[75004]: cluster 2023-09-23T19:36:16.057496+0000 mgr.y (mgr.25325) 861 : cluster 0 pgmap v780: 89 pgs: 5 unknown, 84 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:36:17.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:17 smithi092 bash[58855]: cluster 2023-09-23T19:36:16.057496+0000 mgr.y (mgr.25325) 861 : cluster 0 pgmap v780: 89 pgs: 5 unknown, 84 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:36:17.798 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:17.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:17.802 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:36:17.903 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:17.935 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:17.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:17.938 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:36:18.039 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:18.072 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:18.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:18.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:18.175 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:18.208 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:18.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:18.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:18.312 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:18.342 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:18.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:18.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:18.377 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:36:18.378 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:36:18.445 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:18.476 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:18.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:18.480 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:36:18.581 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:18.608 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:18.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:18.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:36:18.712 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:18.743 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:18.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:18.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:18.846 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:18.873 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:18.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:18.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:36:18.978 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:19.012 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:19.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:19.014 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:19.115 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:19.145 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:19.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:19.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:19.249 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:19.284 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:19.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:19.287 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:36:19.388 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:19.391 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:36:19.392 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:36:19.414 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:19.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:19.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:36:19.518 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:19.551 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:19.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:19.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:19.655 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:19.688 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:19.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:19.692 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:19.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:19 smithi016 bash[77995]: cluster 2023-09-23T19:36:18.058514+0000 mgr.y (mgr.25325) 862 : cluster 0 pgmap v781: 89 pgs: 89 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:36:19.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:19 smithi016 bash[75004]: cluster 2023-09-23T19:36:18.058514+0000 mgr.y (mgr.25325) 862 : cluster 0 pgmap v781: 89 pgs: 89 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:36:19.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:19 smithi092 bash[58855]: cluster 2023-09-23T19:36:18.058514+0000 mgr.y (mgr.25325) 862 : cluster 0 pgmap v781: 89 pgs: 89 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:36:19.792 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:19.822 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:19.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:19.826 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:36:19.927 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:19.932 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:36:19.953 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:19.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:19.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:36:20.056 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:20.085 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:20.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:20.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:36:20.189 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:20.217 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:20.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:20.221 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:20.321 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:20.347 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:20.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:20.350 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:36:20.451 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:20.477 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:20.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:20.480 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:20.580 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:20.608 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:20.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:20.612 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:20.713 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:20.743 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:20.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:20.746 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:20.847 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:20.875 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:20.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:20.878 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:20.979 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:21.000 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:21.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:21.003 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:36:21.103 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:21.131 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:21.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:21.135 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:21.235 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:21.269 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:21.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:21.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:36:21.372 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:21.402 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:21.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:21.404 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:21.505 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:21.530 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:21.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:21.533 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:36:21.634 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:21.644 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:21 smithi016 bash[75004]: cluster 2023-09-23T19:36:20.059662+0000 mgr.y (mgr.25325) 863 : cluster 0 pgmap v782: 89 pgs: 89 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:36:21.644 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:21 smithi016 bash[77995]: cluster 2023-09-23T19:36:20.059662+0000 mgr.y (mgr.25325) 863 : cluster 0 pgmap v782: 89 pgs: 89 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:36:21.658 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:21.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:21.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:36:21.764 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:21.787 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:21.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:21 smithi092 bash[58855]: cluster 2023-09-23T19:36:20.059662+0000 mgr.y (mgr.25325) 863 : cluster 0 pgmap v782: 89 pgs: 89 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:36:21.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:21.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:36:21.891 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:21.918 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:21.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:21.921 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:22.022 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:22.049 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:22.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:22.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:22.152 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:22.176 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:22.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:22.178 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:22.278 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:22.300 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:22.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:22.302 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:36:22.403 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:22.425 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:22.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:22.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:36:22.528 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:22.553 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:22.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:22.556 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:22.656 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:22.679 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:22.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:22.682 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:36:22.782 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:22.813 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:22.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:22.815 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:22.916 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:22.938 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:22.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:22.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:36:23.041 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:23.072 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:23.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:23.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:23.176 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:23.198 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:23.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:23.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:36:23.236 INFO:teuthology.orchestra.run.smithi092.stdout:ERROR: (22) Invalid argument 2023-09-23T19:36:23.236 INFO:teuthology.orchestra.run.smithi092.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:36:23.301 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:23.323 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:23.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:23.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:23.429 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:23.454 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:23.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:23.456 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:23.557 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:23.587 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:23.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:23.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:36:23.690 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:23.708 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:23 smithi016 bash[75004]: cluster 2023-09-23T19:36:22.060508+0000 mgr.y (mgr.25325) 864 : cluster 0 pgmap v783: 89 pgs: 89 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:36:23.708 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:23 smithi016 bash[77995]: cluster 2023-09-23T19:36:22.060508+0000 mgr.y (mgr.25325) 864 : cluster 0 pgmap v783: 89 pgs: 89 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:36:23.716 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:23.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:23.719 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:36:23.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:23 smithi092 bash[58855]: cluster 2023-09-23T19:36:22.060508+0000 mgr.y (mgr.25325) 864 : cluster 0 pgmap v783: 89 pgs: 89 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:36:23.820 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:23.850 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:23.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:23.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:36:23.954 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:23.985 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:23.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:23.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:36:24.014 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T19:36:24.088 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:24.132 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:24.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:24.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:24.237 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:24.264 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:24.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:24.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:24.367 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:24.402 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:24.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:24.405 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:24.506 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:24.543 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:24.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:24.546 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:36:24.647 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:24.676 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:24.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:24.678 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:24.779 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:24.810 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:24.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:24.813 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:24.913 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:24.944 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:24.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:24.946 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:25.047 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:25.058 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:25 smithi092 bash[58855]: audit 2023-09-23T19:36:24.034520+0000 mon.a (mon.0) 1078 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:36:25.058 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:25 smithi092 bash[58855]: cluster 2023-09-23T19:36:24.060998+0000 mgr.y (mgr.25325) 865 : cluster 0 pgmap v784: 89 pgs: 89 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:36:25.078 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:25.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:25.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:36:25.182 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:25.199 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:25 smithi016 bash[75004]: audit 2023-09-23T19:36:24.034520+0000 mon.a (mon.0) 1078 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:36:25.199 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:25 smithi016 bash[75004]: cluster 2023-09-23T19:36:24.060998+0000 mgr.y (mgr.25325) 865 : cluster 0 pgmap v784: 89 pgs: 89 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:36:25.199 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:25 smithi016 bash[77995]: audit 2023-09-23T19:36:24.034520+0000 mon.a (mon.0) 1078 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:36:25.199 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:25 smithi016 bash[77995]: cluster 2023-09-23T19:36:24.060998+0000 mgr.y (mgr.25325) 865 : cluster 0 pgmap v784: 89 pgs: 89 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:36:25.215 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:25.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:25.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:36:25.319 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:25.345 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:25.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:25.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:36:25.449 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:25.476 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:25.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:25.478 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:36:25.578 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:25.610 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:25.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:25.613 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:25.714 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:25.728 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:36:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:36:25] "GET /metrics HTTP/1.1" 200 36226 "" "Prometheus/2.33.4" 2023-09-23T19:36:25.744 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:25.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:25.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:25.847 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:25.874 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:25.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:25.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:36:25.977 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:26.000 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:26.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:26.002 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:36:26.103 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:26.125 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:26.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:26.128 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:36:26.228 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:26.259 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:26.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:26.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:26.362 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:26.396 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:26.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:26.400 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:26.500 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:26.532 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:26.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:26.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:26.637 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:26.667 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:26.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:26.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:36:26.770 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:26.802 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:26.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:26.806 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:26.906 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:26.936 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:26.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:26.938 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:27.039 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:27.066 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:27.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:27.070 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:27.170 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:27.182 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:27 smithi016 bash[75004]: cluster 2023-09-23T19:36:26.062039+0000 mgr.y (mgr.25325) 866 : cluster 0 pgmap v785: 89 pgs: 89 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:36:27.182 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:27 smithi016 bash[77995]: cluster 2023-09-23T19:36:26.062039+0000 mgr.y (mgr.25325) 866 : cluster 0 pgmap v785: 89 pgs: 89 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:36:27.196 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:27.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:27.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:36:27.299 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:27.323 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:27.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:27.326 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:36:27.426 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:27.440 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:27 smithi092 bash[58855]: cluster 2023-09-23T19:36:26.062039+0000 mgr.y (mgr.25325) 866 : cluster 0 pgmap v785: 89 pgs: 89 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:36:27.459 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:27.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:27.462 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:27.563 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:27.589 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:27.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:27.591 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:27.692 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:27.723 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:27.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:27.726 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:27.827 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:27.859 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:27.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:27.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:27.880 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:36:27.900 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:36:27.920 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:36:27.943 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:36:27.962 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:27.965 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:36:27.989 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:27.989 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:36:27.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:27.991 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:36:28.011 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:36:28.033 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:36:28.035 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:36:28.035 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:36:28.035 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:36:28.035 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:36:28.035 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:36:28.035 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:36:28.035 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:36:28.035 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:36:28.091 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:28.116 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:28.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:28.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:36:28.219 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:28.226 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:28 smithi016 bash[75004]: audit 2023-09-23T19:36:27.855208+0000 mon.c (mon.2) 159 : audit 0 from='client.? 172.21.15.16:0/1611016393' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:36:28.227 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:28 smithi016 bash[77995]: audit 2023-09-23T19:36:27.855208+0000 mon.c (mon.2) 159 : audit 0 from='client.? 172.21.15.16:0/1611016393' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:36:28.244 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:28.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:28.247 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:28.348 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:28.379 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:28.380 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:28 smithi092 bash[58855]: audit 2023-09-23T19:36:27.855208+0000 mon.c (mon.2) 159 : audit 0 from='client.? 172.21.15.16:0/1611016393' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:36:28.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:28.382 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:28.482 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:28.513 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:28.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:28.515 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:28.616 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:28.643 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:28.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:28.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:36:28.747 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:28.777 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:28.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:28.780 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:28.838 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:36:28.881 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:28.903 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:28.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:28.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:29.005 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:29.028 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:29.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:29.031 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:36:29.132 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:29.157 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:29.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:29.159 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:36:29.173 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:29 smithi016 bash[77995]: cluster 2023-09-23T19:36:28.062716+0000 mgr.y (mgr.25325) 867 : cluster 0 pgmap v786: 89 pgs: 89 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:36:29.173 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:29 smithi016 bash[77995]: audit 2023-09-23T19:36:28.644048+0000 mon.a (mon.0) 1079 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:36:29.173 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:29 smithi016 bash[75004]: cluster 2023-09-23T19:36:28.062716+0000 mgr.y (mgr.25325) 867 : cluster 0 pgmap v786: 89 pgs: 89 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:36:29.173 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:29 smithi016 bash[75004]: audit 2023-09-23T19:36:28.644048+0000 mon.a (mon.0) 1079 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:36:29.260 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:29.292 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:29.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:29.295 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:29.396 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:29.407 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:29 smithi092 bash[58855]: cluster 2023-09-23T19:36:28.062716+0000 mgr.y (mgr.25325) 867 : cluster 0 pgmap v786: 89 pgs: 89 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:36:29.407 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:29 smithi092 bash[58855]: audit 2023-09-23T19:36:28.644048+0000 mon.a (mon.0) 1079 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:36:29.426 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:29.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:29.429 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:29.530 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:29.548 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:29.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:29.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:36:29.651 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:29.687 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:29.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:29.689 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:36:29.790 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:29.815 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:29.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:29.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:29.919 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:29.949 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:29.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:29.951 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:30.052 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:30.084 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:30.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:30.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:36:30.187 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:30.209 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:30.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:30.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:30.312 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:30.338 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:30.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:30.341 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:36:30.397 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is set 2023-09-23T19:36:30.441 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:30.462 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:30.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:30.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:30.489 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:30 smithi016 bash[75004]: audit 2023-09-23T19:36:29.579439+0000 mon.a (mon.0) 1080 : audit 1 from='client.? 172.21.15.16:0/3137073177' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:36:30.489 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:30 smithi016 bash[77995]: audit 2023-09-23T19:36:29.579439+0000 mon.a (mon.0) 1080 : audit 1 from='client.? 172.21.15.16:0/3137073177' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:36:30.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:30 smithi092 bash[58855]: audit 2023-09-23T19:36:29.579439+0000 mon.a (mon.0) 1080 : audit 1 from='client.? 172.21.15.16:0/3137073177' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:36:30.566 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:30.590 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:30.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:30.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:36:30.695 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:30.731 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:30.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:30.733 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:36:30.834 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:30.864 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:30.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:30.868 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:30.968 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:30.992 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:30.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:30.995 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:36:31.095 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:31.125 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:31.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:31.128 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:31.228 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:31.252 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:31.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:31.255 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:36:31.356 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:31.387 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:31.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:31.390 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:31.491 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:31.523 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:31.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:31.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:31.627 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:31.639 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:31 smithi016 bash[77995]: cluster 2023-09-23T19:36:30.063225+0000 mgr.y (mgr.25325) 868 : cluster 0 pgmap v787: 89 pgs: 89 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:36:31.640 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:31 smithi016 bash[77995]: cluster 2023-09-23T19:36:30.378432+0000 mon.a (mon.0) 1081 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:36:31.640 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:31 smithi016 bash[77995]: audit 2023-09-23T19:36:30.389254+0000 mon.a (mon.0) 1082 : audit 1 from='client.? 172.21.15.16:0/3137073177' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:36:31.640 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:31 smithi016 bash[77995]: cluster 2023-09-23T19:36:30.389333+0000 mon.a (mon.0) 1083 : cluster 0 osdmap e147: 8 total, 8 up, 8 in 2023-09-23T19:36:31.640 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:31 smithi016 bash[77995]: cluster 2023-09-23T19:36:30.391093+0000 osd.4 (osd.4) 75 : cluster 3 failed to encode map e147 with expected crc 2023-09-23T19:36:31.640 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:31 smithi016 bash[77995]: cluster 2023-09-23T19:36:30.543341+0000 osd.6 (osd.6) 63 : cluster 3 failed to encode map e147 with expected crc 2023-09-23T19:36:31.640 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:31 smithi016 bash[75004]: cluster 2023-09-23T19:36:30.063225+0000 mgr.y (mgr.25325) 868 : cluster 0 pgmap v787: 89 pgs: 89 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:36:31.640 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:31 smithi016 bash[75004]: cluster 2023-09-23T19:36:30.378432+0000 mon.a (mon.0) 1081 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:36:31.640 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:31 smithi016 bash[75004]: audit 2023-09-23T19:36:30.389254+0000 mon.a (mon.0) 1082 : audit 1 from='client.? 172.21.15.16:0/3137073177' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:36:31.640 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:31 smithi016 bash[75004]: cluster 2023-09-23T19:36:30.389333+0000 mon.a (mon.0) 1083 : cluster 0 osdmap e147: 8 total, 8 up, 8 in 2023-09-23T19:36:31.641 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:31 smithi016 bash[75004]: cluster 2023-09-23T19:36:30.391093+0000 osd.4 (osd.4) 75 : cluster 3 failed to encode map e147 with expected crc 2023-09-23T19:36:31.641 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:31 smithi016 bash[75004]: cluster 2023-09-23T19:36:30.543341+0000 osd.6 (osd.6) 63 : cluster 3 failed to encode map e147 with expected crc 2023-09-23T19:36:31.651 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:31.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:31.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:36:31.754 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:31.773 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:31.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:31.776 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:36:31.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:31 smithi092 bash[58855]: cluster 2023-09-23T19:36:30.063225+0000 mgr.y (mgr.25325) 868 : cluster 0 pgmap v787: 89 pgs: 89 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:36:31.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:31 smithi092 bash[58855]: cluster 2023-09-23T19:36:30.378432+0000 mon.a (mon.0) 1081 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:36:31.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:31 smithi092 bash[58855]: audit 2023-09-23T19:36:30.389254+0000 mon.a (mon.0) 1082 : audit 1 from='client.? 172.21.15.16:0/3137073177' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:36:31.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:31 smithi092 bash[58855]: cluster 2023-09-23T19:36:30.389333+0000 mon.a (mon.0) 1083 : cluster 0 osdmap e147: 8 total, 8 up, 8 in 2023-09-23T19:36:31.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:31 smithi092 bash[58855]: cluster 2023-09-23T19:36:30.391093+0000 osd.4 (osd.4) 75 : cluster 3 failed to encode map e147 with expected crc 2023-09-23T19:36:31.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:31 smithi092 bash[58855]: cluster 2023-09-23T19:36:30.543341+0000 osd.6 (osd.6) 63 : cluster 3 failed to encode map e147 with expected crc 2023-09-23T19:36:31.876 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:31.897 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:31.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:31.899 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:36:32.000 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:32.027 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:32.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:32.029 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:36:32.127 INFO:teuthology.orchestra.run.smithi092.stdout:{ 2023-09-23T19:36:32.128 INFO:teuthology.orchestra.run.smithi092.stdout: "size": 20, 2023-09-23T19:36:32.128 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 600, 2023-09-23T19:36:32.128 INFO:teuthology.orchestra.run.smithi092.stdout: "ops": [ 2023-09-23T19:36:32.128 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.128 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:26585 9.3 9:cb81db0c:::benchmark_data_smithi016_113233_object26584:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e122)", 2023-09-23T19:36:32.128 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:31:46.551642+0000", 2023-09-23T19:36:32.128 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 285.572564339, 2023-09-23T19:36:32.128 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.19433394500000001, 2023-09-23T19:36:32.128 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:36:32.128 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:36:32.129 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:36:32.129 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:36:32.129 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:36:32.129 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 26585 2023-09-23T19:36:32.129 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.129 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:36:32.129 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.129 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:36:32.129 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.551642+0000", 2023-09-23T19:36:32.129 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:36:32.129 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.129 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.130 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:36:32.130 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.551642+0000", 2023-09-23T19:36:32.130 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T19:36:32.130 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.130 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.130 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:36:32.130 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.551641+0000", 2023-09-23T19:36:32.130 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.3479999999999998e-05 2023-09-23T19:36:32.130 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.130 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.130 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:36:32.130 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.551674+0000", 2023-09-23T19:36:32.131 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.2439999999999999e-06 2023-09-23T19:36:32.131 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.131 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.131 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:36:32.131 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.551683+0000", 2023-09-23T19:36:32.131 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.6189999999999999e-06 2023-09-23T19:36:32.131 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.131 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.131 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:36:32.131 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.551689+0000", 2023-09-23T19:36:32.131 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00014126500000000001 2023-09-23T19:36:32.131 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.132 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.132 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:36:32.132 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.551830+0000", 2023-09-23T19:36:32.132 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00013956600000000001 2023-09-23T19:36:32.132 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.132 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.132 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:36:32.132 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.551970+0000", 2023-09-23T19:36:32.132 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00016731800000000001 2023-09-23T19:36:32.132 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.132 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.132 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,3", 2023-09-23T19:36:32.133 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.552137+0000", 2023-09-23T19:36:32.133 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.001424504 2023-09-23T19:36:32.133 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.133 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.133 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:36:32.133 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.553562+0000", 2023-09-23T19:36:32.133 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0046421320000000002 2023-09-23T19:36:32.133 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.133 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.133 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:32.133 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.558204+0000", 2023-09-23T19:36:32.133 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.187635939 2023-09-23T19:36:32.133 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.134 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.134 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:32.134 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.745840+0000", 2023-09-23T19:36:32.134 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.9017999999999995e-05 2023-09-23T19:36:32.134 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.134 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.134 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:36:32.134 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.745919+0000", 2023-09-23T19:36:32.134 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.7176000000000003e-05 2023-09-23T19:36:32.134 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.134 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.134 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:36:32.135 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.745976+0000", 2023-09-23T19:36:32.135 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.19433394500000001 2023-09-23T19:36:32.135 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:32.135 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:36:32.135 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:32.135 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.135 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.135 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:26598 9.3 9:c53a6029:::benchmark_data_smithi016_113233_object26597:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e122)", 2023-09-23T19:36:32.135 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:31:46.556651+0000", 2023-09-23T19:36:32.135 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 285.56755517200003, 2023-09-23T19:36:32.135 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.18955604600000001, 2023-09-23T19:36:32.135 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:36:32.136 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:36:32.136 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:36:32.136 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:36:32.136 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:36:32.136 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 26598 2023-09-23T19:36:32.136 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.136 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:36:32.136 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.136 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:36:32.136 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.556651+0000", 2023-09-23T19:36:32.137 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:36:32.137 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.137 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.137 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:36:32.137 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.556651+0000", 2023-09-23T19:36:32.137 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T19:36:32.137 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.137 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.138 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:36:32.138 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.556650+0000", 2023-09-23T19:36:32.138 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.2991999999999998e-05 2023-09-23T19:36:32.138 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.138 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.138 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:36:32.138 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.556693+0000", 2023-09-23T19:36:32.138 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.5656000000000001e-05 2023-09-23T19:36:32.139 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.139 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.139 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:36:32.139 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.556708+0000", 2023-09-23T19:36:32.139 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.7762e-05 2023-09-23T19:36:32.139 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.139 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.139 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:36:32.139 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.556726+0000", 2023-09-23T19:36:32.139 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.9646000000000004e-05 2023-09-23T19:36:32.139 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.139 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.140 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:36:32.140 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.556796+0000", 2023-09-23T19:36:32.140 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00017475800000000001 2023-09-23T19:36:32.140 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.140 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.140 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:36:32.140 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.556971+0000", 2023-09-23T19:36:32.140 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00015882299999999999 2023-09-23T19:36:32.140 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.140 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.140 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,3", 2023-09-23T19:36:32.140 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.557129+0000", 2023-09-23T19:36:32.141 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00114853 2023-09-23T19:36:32.141 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.141 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.141 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:36:32.141 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.558278+0000", 2023-09-23T19:36:32.141 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.17477084800000001 2023-09-23T19:36:32.141 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.141 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.141 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:32.141 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.733049+0000", 2023-09-23T19:36:32.141 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.013031696000000001 2023-09-23T19:36:32.141 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.142 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.142 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:32.142 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.746081+0000", 2023-09-23T19:36:32.142 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.4264999999999994e-05 2023-09-23T19:36:32.142 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.142 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.142 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:36:32.142 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.746165+0000", 2023-09-23T19:36:32.142 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.2463000000000003e-05 2023-09-23T19:36:32.142 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.142 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.142 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:36:32.143 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.746207+0000", 2023-09-23T19:36:32.143 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.18955604600000001 2023-09-23T19:36:32.143 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:32.143 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:36:32.143 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:32.143 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.143 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.143 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:26599 9.d 9:bef5b10a:::benchmark_data_smithi016_113233_object26598:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e122)", 2023-09-23T19:36:32.143 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:31:46.558587+0000", 2023-09-23T19:36:32.143 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 285.56561975199998, 2023-09-23T19:36:32.143 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.173559186, 2023-09-23T19:36:32.144 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:36:32.144 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:36:32.144 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:36:32.144 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:36:32.144 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:36:32.144 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 26599 2023-09-23T19:36:32.144 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.144 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:36:32.144 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.144 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:36:32.145 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.558587+0000", 2023-09-23T19:36:32.145 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:36:32.145 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.145 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.145 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:36:32.145 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.558587+0000", 2023-09-23T19:36:32.145 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T19:36:32.145 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.145 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.145 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:36:32.146 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.558585+0000", 2023-09-23T19:36:32.146 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.0269000000000002e-05 2023-09-23T19:36:32.146 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.146 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.146 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:36:32.146 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.558646+0000", 2023-09-23T19:36:32.146 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.1790000000000001e-05 2023-09-23T19:36:32.146 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.146 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.146 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:36:32.146 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.558657+0000", 2023-09-23T19:36:32.147 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.3800000000000002e-06 2023-09-23T19:36:32.147 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.147 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.147 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:36:32.147 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.558663+0000", 2023-09-23T19:36:32.147 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.3485e-05 2023-09-23T19:36:32.147 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.147 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.147 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:36:32.147 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.558716+0000", 2023-09-23T19:36:32.148 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00015891200000000001 2023-09-23T19:36:32.148 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.148 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.148 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:36:32.148 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.558875+0000", 2023-09-23T19:36:32.148 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00016389 2023-09-23T19:36:32.148 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.148 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.148 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,5", 2023-09-23T19:36:32.148 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.559039+0000", 2023-09-23T19:36:32.148 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0013228000000000001 2023-09-23T19:36:32.148 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.149 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.149 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:36:32.149 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.560362+0000", 2023-09-23T19:36:32.149 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.001284751 2023-09-23T19:36:32.149 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.149 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.149 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:32.149 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.561647+0000", 2023-09-23T19:36:32.149 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.17030458400000001 2023-09-23T19:36:32.149 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.149 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.150 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:32.150 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.731951+0000", 2023-09-23T19:36:32.150 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000114024 2023-09-23T19:36:32.150 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.150 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.150 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:36:32.150 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.732065+0000", 2023-09-23T19:36:32.150 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.0548000000000002e-05 2023-09-23T19:36:32.150 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.150 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.150 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:36:32.150 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.732146+0000", 2023-09-23T19:36:32.151 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.173559186 2023-09-23T19:36:32.151 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:32.151 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:36:32.151 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:32.151 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.151 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.151 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:71071 9.e 9:713311a6:::benchmark_data_smithi016_113233_object71070:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e124)", 2023-09-23T19:36:32.151 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:32:22.033184+0000", 2023-09-23T19:36:32.151 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 250.09102248600001, 2023-09-23T19:36:32.151 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.181195512, 2023-09-23T19:36:32.151 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:36:32.151 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:36:32.151 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:36:32.152 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:36:32.152 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:36:32.152 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 71071 2023-09-23T19:36:32.152 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.152 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:36:32.152 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.152 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:36:32.152 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.033184+0000", 2023-09-23T19:36:32.152 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:36:32.152 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.152 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.152 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:36:32.153 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.033184+0000", 2023-09-23T19:36:32.153 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T19:36:32.153 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.153 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.153 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:36:32.153 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.033182+0000", 2023-09-23T19:36:32.153 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.2682999999999999e-05 2023-09-23T19:36:32.153 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.153 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.153 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:36:32.153 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.033215+0000", 2023-09-23T19:36:32.153 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.3942e-05 2023-09-23T19:36:32.154 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.154 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.154 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:36:32.154 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.033229+0000", 2023-09-23T19:36:32.154 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.2189999999999998e-06 2023-09-23T19:36:32.154 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.154 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.154 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:36:32.154 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.033236+0000", 2023-09-23T19:36:32.154 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000260511 2023-09-23T19:36:32.154 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.154 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.155 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:36:32.155 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.033496+0000", 2023-09-23T19:36:32.155 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00028382 2023-09-23T19:36:32.155 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.155 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.155 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:36:32.155 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.033780+0000", 2023-09-23T19:36:32.155 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000164947 2023-09-23T19:36:32.155 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.155 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.155 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,4", 2023-09-23T19:36:32.155 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.033945+0000", 2023-09-23T19:36:32.155 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0053813419999999999 2023-09-23T19:36:32.156 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.156 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.156 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:32.156 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.039326+0000", 2023-09-23T19:36:32.156 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.1019999999999997e-06 2023-09-23T19:36:32.156 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.156 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.156 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:36:32.156 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.039335+0000", 2023-09-23T19:36:32.156 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.17478982200000001 2023-09-23T19:36:32.156 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.156 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.157 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:32.157 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.214124+0000", 2023-09-23T19:36:32.157 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000114564 2023-09-23T19:36:32.157 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.157 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.157 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:36:32.157 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.214239+0000", 2023-09-23T19:36:32.157 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000140443 2023-09-23T19:36:32.157 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.157 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.157 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:36:32.157 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.214379+0000", 2023-09-23T19:36:32.157 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.181195512 2023-09-23T19:36:32.158 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:32.158 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:36:32.158 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:32.158 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.158 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.158 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:71078 9.e 9:79417d87:::benchmark_data_smithi016_113233_object71077:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e124)", 2023-09-23T19:36:32.158 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:32:22.037570+0000", 2023-09-23T19:36:32.158 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 250.086636083, 2023-09-23T19:36:32.158 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.18826123, 2023-09-23T19:36:32.158 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:36:32.158 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:36:32.158 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:36:32.159 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:36:32.159 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:36:32.159 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 71078 2023-09-23T19:36:32.159 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.159 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:36:32.159 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.159 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:36:32.159 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.037570+0000", 2023-09-23T19:36:32.159 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:36:32.159 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.159 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.159 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:36:32.160 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.037570+0000", 2023-09-23T19:36:32.160 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T19:36:32.160 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.160 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.160 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:36:32.160 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.037569+0000", 2023-09-23T19:36:32.160 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.4791999999999999e-05 2023-09-23T19:36:32.160 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.160 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.160 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:36:32.160 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.037634+0000", 2023-09-23T19:36:32.160 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.5428000000000001e-05 2023-09-23T19:36:32.160 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.161 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.161 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:36:32.161 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.037659+0000", 2023-09-23T19:36:32.161 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.238e-06 2023-09-23T19:36:32.161 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.161 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.161 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:36:32.161 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.037666+0000", 2023-09-23T19:36:32.161 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.001695876 2023-09-23T19:36:32.161 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.161 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.161 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:36:32.162 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.039362+0000", 2023-09-23T19:36:32.162 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00013776700000000001 2023-09-23T19:36:32.162 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.162 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.162 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:36:32.162 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.039500+0000", 2023-09-23T19:36:32.162 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000164258 2023-09-23T19:36:32.162 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.162 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.162 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,4", 2023-09-23T19:36:32.162 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.039664+0000", 2023-09-23T19:36:32.162 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0032891230000000001 2023-09-23T19:36:32.163 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.163 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.163 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:36:32.163 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.042953+0000", 2023-09-23T19:36:32.163 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0038750659999999999 2023-09-23T19:36:32.163 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.163 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.163 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:32.163 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.046829+0000", 2023-09-23T19:36:32.163 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.178904596 2023-09-23T19:36:32.163 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.163 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.163 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:32.164 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.225733+0000", 2023-09-23T19:36:32.164 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.3872000000000002e-05 2023-09-23T19:36:32.164 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.164 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.164 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:36:32.164 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.225787+0000", 2023-09-23T19:36:32.164 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.4545000000000001e-05 2023-09-23T19:36:32.164 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.164 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.164 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:36:32.164 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.225832+0000", 2023-09-23T19:36:32.164 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.18826123 2023-09-23T19:36:32.165 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:32.165 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:36:32.165 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:32.165 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.165 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.165 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:71087 9.e 9:7757b3dd:::benchmark_data_smithi016_113233_object71086:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e124)", 2023-09-23T19:36:32.165 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:32:22.047091+0000", 2023-09-23T19:36:32.165 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 250.07711489299999, 2023-09-23T19:36:32.165 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.17886874899999999, 2023-09-23T19:36:32.165 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:36:32.165 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:36:32.165 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:36:32.166 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:36:32.166 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:36:32.166 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 71087 2023-09-23T19:36:32.166 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.166 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:36:32.166 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.166 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:36:32.166 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.047091+0000", 2023-09-23T19:36:32.166 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:36:32.166 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.166 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.167 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:36:32.167 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.047091+0000", 2023-09-23T19:36:32.167 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T19:36:32.167 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.167 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.167 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:36:32.167 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.047089+0000", 2023-09-23T19:36:32.167 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.1995e-05 2023-09-23T19:36:32.167 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.167 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.167 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:36:32.167 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.047131+0000", 2023-09-23T19:36:32.168 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.4183e-05 2023-09-23T19:36:32.168 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.168 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.168 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:36:32.168 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.047146+0000", 2023-09-23T19:36:32.168 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.9540000000000004e-06 2023-09-23T19:36:32.168 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.168 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.168 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:36:32.168 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.047153+0000", 2023-09-23T19:36:32.168 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.8807000000000001e-05 2023-09-23T19:36:32.168 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.169 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.169 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:36:32.169 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.047212+0000", 2023-09-23T19:36:32.169 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00014279700000000001 2023-09-23T19:36:32.169 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.169 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.169 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:36:32.169 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.047355+0000", 2023-09-23T19:36:32.169 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00029963899999999998 2023-09-23T19:36:32.169 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.169 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.169 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,4", 2023-09-23T19:36:32.170 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.047655+0000", 2023-09-23T19:36:32.170 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0090374549999999998 2023-09-23T19:36:32.170 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.170 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.170 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:32.170 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.056692+0000", 2023-09-23T19:36:32.170 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0029485779999999999 2023-09-23T19:36:32.170 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.170 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.170 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:36:32.170 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.059641+0000", 2023-09-23T19:36:32.171 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.166230761 2023-09-23T19:36:32.171 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.171 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.171 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:32.171 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.225872+0000", 2023-09-23T19:36:32.171 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.0167999999999998e-05 2023-09-23T19:36:32.171 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.171 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.171 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:36:32.171 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.225922+0000", 2023-09-23T19:36:32.171 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.8529999999999999e-05 2023-09-23T19:36:32.171 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.172 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.172 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:36:32.172 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.225960+0000", 2023-09-23T19:36:32.172 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.17886874899999999 2023-09-23T19:36:32.172 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:32.172 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:36:32.172 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:32.172 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.172 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.172 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:71089 9.e 9:774585d3:::benchmark_data_smithi016_113233_object71088:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e124)", 2023-09-23T19:36:32.172 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:32:22.050155+0000", 2023-09-23T19:36:32.172 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 250.074051406, 2023-09-23T19:36:32.173 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.175930751, 2023-09-23T19:36:32.173 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:36:32.173 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:36:32.173 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:36:32.173 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:36:32.173 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:36:32.173 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 71089 2023-09-23T19:36:32.173 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.173 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:36:32.173 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.173 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:36:32.174 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.050155+0000", 2023-09-23T19:36:32.174 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:36:32.174 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.174 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.174 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:36:32.174 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.050155+0000", 2023-09-23T19:36:32.174 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.999999 2023-09-23T19:36:32.174 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.174 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.174 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:36:32.174 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.050154+0000", 2023-09-23T19:36:32.174 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.5884000000000002e-05 2023-09-23T19:36:32.175 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.175 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.175 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:36:32.175 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.050180+0000", 2023-09-23T19:36:32.175 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.8975999999999998e-05 2023-09-23T19:36:32.175 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.175 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.175 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:36:32.175 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.050249+0000", 2023-09-23T19:36:32.175 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.8759999999999999e-06 2023-09-23T19:36:32.175 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.175 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.176 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:36:32.176 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.050256+0000", 2023-09-23T19:36:32.176 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0031540100000000001 2023-09-23T19:36:32.176 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.176 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.176 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:36:32.176 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.053410+0000", 2023-09-23T19:36:32.176 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00013616100000000001 2023-09-23T19:36:32.176 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.176 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.176 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:36:32.176 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.053546+0000", 2023-09-23T19:36:32.177 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00012943899999999999 2023-09-23T19:36:32.177 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.177 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.177 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,4", 2023-09-23T19:36:32.177 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.053675+0000", 2023-09-23T19:36:32.177 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0059726470000000002 2023-09-23T19:36:32.177 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.177 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.177 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:36:32.177 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.059648+0000", 2023-09-23T19:36:32.177 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.248e-05 2023-09-23T19:36:32.177 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.178 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.178 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:32.178 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.059690+0000", 2023-09-23T19:36:32.178 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.16630597499999999 2023-09-23T19:36:32.178 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.178 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.178 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:32.178 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.225996+0000", 2023-09-23T19:36:32.178 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.7840999999999998e-05 2023-09-23T19:36:32.178 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.178 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.178 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:36:32.179 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.226044+0000", 2023-09-23T19:36:32.179 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.1561999999999999e-05 2023-09-23T19:36:32.179 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.179 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.179 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:36:32.179 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.226086+0000", 2023-09-23T19:36:32.179 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.175930751 2023-09-23T19:36:32.179 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:32.179 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:36:32.179 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:32.179 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.179 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.180 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:71092 9.e 9:7c97ae4b:::benchmark_data_smithi016_113233_object71091:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e124)", 2023-09-23T19:36:32.180 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:32:22.052519+0000", 2023-09-23T19:36:32.180 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 250.07168744200001, 2023-09-23T19:36:32.180 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.17368992999999999, 2023-09-23T19:36:32.180 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:36:32.180 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:36:32.180 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:36:32.180 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:36:32.180 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:36:32.180 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 71092 2023-09-23T19:36:32.180 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.180 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:36:32.181 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.181 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:36:32.181 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.052519+0000", 2023-09-23T19:36:32.181 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:36:32.181 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.181 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.181 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:36:32.181 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.052519+0000", 2023-09-23T19:36:32.181 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T19:36:32.181 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.181 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.181 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:36:32.182 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.052518+0000", 2023-09-23T19:36:32.182 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.9794999999999999e-05 2023-09-23T19:36:32.182 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.182 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.182 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:36:32.182 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.052577+0000", 2023-09-23T19:36:32.182 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.5046e-05 2023-09-23T19:36:32.182 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.182 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.182 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:36:32.182 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.052592+0000", 2023-09-23T19:36:32.182 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.2539999999999992e-06 2023-09-23T19:36:32.182 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.183 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.183 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:36:32.183 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.052601+0000", 2023-09-23T19:36:32.183 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0041490149999999998 2023-09-23T19:36:32.183 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.183 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.183 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:36:32.183 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.056750+0000", 2023-09-23T19:36:32.183 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000250234 2023-09-23T19:36:32.183 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.183 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.184 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:36:32.184 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.057000+0000", 2023-09-23T19:36:32.184 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00017354499999999999 2023-09-23T19:36:32.184 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.184 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.184 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,4", 2023-09-23T19:36:32.184 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.057173+0000", 2023-09-23T19:36:32.184 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0053271459999999996 2023-09-23T19:36:32.184 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.184 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.184 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:32.184 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.062501+0000", 2023-09-23T19:36:32.185 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.001941536 2023-09-23T19:36:32.185 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.185 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.185 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:36:32.185 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.064442+0000", 2023-09-23T19:36:32.185 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.16168389899999999 2023-09-23T19:36:32.185 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.185 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.185 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:32.185 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.226126+0000", 2023-09-23T19:36:32.185 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.3513999999999997e-05 2023-09-23T19:36:32.185 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.185 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.186 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:36:32.186 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.226170+0000", 2023-09-23T19:36:32.186 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.9360000000000003e-05 2023-09-23T19:36:32.186 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.186 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.186 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:36:32.186 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.226209+0000", 2023-09-23T19:36:32.186 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.17368992999999999 2023-09-23T19:36:32.186 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:32.186 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:36:32.186 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:32.186 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.187 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.187 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:71094 9.3 9:c2ab45ca:::benchmark_data_smithi016_113233_object71093:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e124)", 2023-09-23T19:36:32.187 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:32:22.056556+0000", 2023-09-23T19:36:32.187 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 250.06765034200001, 2023-09-23T19:36:32.187 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.165906251, 2023-09-23T19:36:32.187 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:36:32.187 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:36:32.187 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:36:32.187 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:36:32.187 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:36:32.187 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 71094 2023-09-23T19:36:32.188 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.188 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:36:32.188 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.188 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:36:32.188 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.056556+0000", 2023-09-23T19:36:32.188 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:36:32.188 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.188 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.188 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:36:32.188 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.056556+0000", 2023-09-23T19:36:32.188 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T19:36:32.188 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.188 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.188 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:36:32.189 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.056554+0000", 2023-09-23T19:36:32.189 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.8666999999999999e-05 2023-09-23T19:36:32.189 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.189 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.189 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:36:32.189 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.056613+0000", 2023-09-23T19:36:32.189 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.9188e-05 2023-09-23T19:36:32.189 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.189 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.189 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:36:32.189 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.056632+0000", 2023-09-23T19:36:32.189 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.396e-06 2023-09-23T19:36:32.190 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.190 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.190 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:36:32.190 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.056640+0000", 2023-09-23T19:36:32.190 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.3092e-05 2023-09-23T19:36:32.190 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.190 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.190 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:36:32.190 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.056683+0000", 2023-09-23T19:36:32.190 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000103237 2023-09-23T19:36:32.190 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.190 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.191 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:36:32.191 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.056786+0000", 2023-09-23T19:36:32.191 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000145845 2023-09-23T19:36:32.191 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.191 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.191 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,3", 2023-09-23T19:36:32.191 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.056932+0000", 2023-09-23T19:36:32.191 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0039338469999999999 2023-09-23T19:36:32.191 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.191 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.191 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:32.192 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.060866+0000", 2023-09-23T19:36:32.192 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0013245279999999999 2023-09-23T19:36:32.192 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.192 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.192 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:36:32.192 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.062190+0000", 2023-09-23T19:36:32.192 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.16003255299999999 2023-09-23T19:36:32.192 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.192 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.192 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:32.192 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.222223+0000", 2023-09-23T19:36:32.193 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00015455300000000001 2023-09-23T19:36:32.193 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.193 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.193 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:36:32.193 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.222377+0000", 2023-09-23T19:36:32.193 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.4900000000000004e-05 2023-09-23T19:36:32.193 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.193 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.193 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:36:32.193 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.222462+0000", 2023-09-23T19:36:32.193 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.165906251 2023-09-23T19:36:32.193 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:32.193 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:36:32.194 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:32.194 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.194 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.194 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:71095 9.3 9:c9176f15:::benchmark_data_smithi016_113233_object71094:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e124)", 2023-09-23T19:36:32.194 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:32:22.056791+0000", 2023-09-23T19:36:32.194 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 250.06741556599999, 2023-09-23T19:36:32.194 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.166731289, 2023-09-23T19:36:32.194 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:36:32.194 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:36:32.194 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:36:32.194 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:36:32.195 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:36:32.195 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 71095 2023-09-23T19:36:32.195 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.195 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:36:32.195 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.195 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:36:32.195 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.056791+0000", 2023-09-23T19:36:32.195 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:36:32.195 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.195 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.195 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:36:32.195 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.056791+0000", 2023-09-23T19:36:32.195 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T19:36:32.196 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.196 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.196 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:36:32.196 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.056790+0000", 2023-09-23T19:36:32.196 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.4085999999999995e-05 2023-09-23T19:36:32.196 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.196 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.196 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:36:32.196 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.056864+0000", 2023-09-23T19:36:32.196 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.1114999999999999e-05 2023-09-23T19:36:32.196 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.196 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.196 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:36:32.197 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.056875+0000", 2023-09-23T19:36:32.197 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.5230000000000001e-06 2023-09-23T19:36:32.197 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.197 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.197 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:36:32.197 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.056881+0000", 2023-09-23T19:36:32.197 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0021203960000000001 2023-09-23T19:36:32.197 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.197 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.197 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:36:32.197 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.059002+0000", 2023-09-23T19:36:32.197 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.9608000000000001e-05 2023-09-23T19:36:32.198 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.198 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.198 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:36:32.198 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.059081+0000", 2023-09-23T19:36:32.198 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000100628 2023-09-23T19:36:32.198 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.198 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.198 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,3", 2023-09-23T19:36:32.198 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.059182+0000", 2023-09-23T19:36:32.198 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0027136830000000002 2023-09-23T19:36:32.198 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.198 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.199 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:32.199 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.061896+0000", 2023-09-23T19:36:32.199 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0021113130000000001 2023-09-23T19:36:32.199 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.199 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.199 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:36:32.199 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.064007+0000", 2023-09-23T19:36:32.199 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.15940559000000001 2023-09-23T19:36:32.199 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.199 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.199 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:32.199 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.223412+0000", 2023-09-23T19:36:32.199 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.2522999999999998e-05 2023-09-23T19:36:32.200 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.200 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.200 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:36:32.200 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.223475+0000", 2023-09-23T19:36:32.200 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.7114999999999999e-05 2023-09-23T19:36:32.200 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.200 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.200 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:36:32.200 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.223522+0000", 2023-09-23T19:36:32.200 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.166731289 2023-09-23T19:36:32.200 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:32.200 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:36:32.201 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:32.201 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.201 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.201 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:87240 9.d 9:b2883303:::benchmark_data_smithi016_113233_object87239:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e125)", 2023-09-23T19:36:32.201 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:32:38.684569+0000", 2023-09-23T19:36:32.201 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 233.439637616, 2023-09-23T19:36:32.201 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.16459875199999999, 2023-09-23T19:36:32.201 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:36:32.201 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:36:32.201 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:36:32.201 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:36:32.201 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:36:32.202 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 87240 2023-09-23T19:36:32.202 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.202 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:36:32.202 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.202 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:36:32.202 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:38.684569+0000", 2023-09-23T19:36:32.202 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:36:32.202 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.202 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.202 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:36:32.202 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:38.684569+0000", 2023-09-23T19:36:32.202 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T19:36:32.203 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.203 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.203 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:36:32.203 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:38.684567+0000", 2023-09-23T19:36:32.203 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.9859e-05 2023-09-23T19:36:32.203 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.203 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.203 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:36:32.203 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:38.684597+0000", 2023-09-23T19:36:32.203 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.2280000000000004e-06 2023-09-23T19:36:32.203 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.203 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.204 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:36:32.204 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:38.684606+0000", 2023-09-23T19:36:32.204 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.251e-06 2023-09-23T19:36:32.204 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.204 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.204 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:36:32.204 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:38.684612+0000", 2023-09-23T19:36:32.204 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.4313e-05 2023-09-23T19:36:32.204 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.204 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.204 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:36:32.204 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:38.684667+0000", 2023-09-23T19:36:32.204 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000223298 2023-09-23T19:36:32.205 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.205 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.205 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:36:32.205 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:38.684890+0000", 2023-09-23T19:36:32.205 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00015868999999999999 2023-09-23T19:36:32.205 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.205 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.205 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,5", 2023-09-23T19:36:32.205 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:38.685049+0000", 2023-09-23T19:36:32.205 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.004261117 2023-09-23T19:36:32.205 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.205 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.206 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:36:32.206 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:38.689310+0000", 2023-09-23T19:36:32.206 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.004225375 2023-09-23T19:36:32.206 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.206 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.206 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:32.206 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:38.693535+0000", 2023-09-23T19:36:32.206 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.15553545199999999 2023-09-23T19:36:32.206 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.206 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.206 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:32.207 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:38.849071+0000", 2023-09-23T19:36:32.207 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.7445000000000003e-05 2023-09-23T19:36:32.207 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.207 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.207 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:36:32.207 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:38.849128+0000", 2023-09-23T19:36:32.207 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.9415999999999997e-05 2023-09-23T19:36:32.207 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.207 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.207 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:36:32.208 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:38.849168+0000", 2023-09-23T19:36:32.208 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.16459875199999999 2023-09-23T19:36:32.208 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:32.208 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:36:32.208 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:32.208 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.208 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.208 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:208985 9.d 9:b5103cf5:::benchmark_data_smithi016_113233_object100603:head [delete] snapc 0=[] ondisk+write+known_if_redirected e137)", 2023-09-23T19:36:32.208 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:35:26.601224+0000", 2023-09-23T19:36:32.208 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 65.522982831999997, 2023-09-23T19:36:32.208 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.22539637300000001, 2023-09-23T19:36:32.208 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:36:32.209 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:36:32.209 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:36:32.209 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:36:32.209 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:36:32.209 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 208985 2023-09-23T19:36:32.209 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.209 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:36:32.209 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.209 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:36:32.209 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.601224+0000", 2023-09-23T19:36:32.209 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:36:32.209 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.210 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.210 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:36:32.210 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.601224+0000", 2023-09-23T19:36:32.210 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T19:36:32.210 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.210 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.210 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:36:32.210 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.601222+0000", 2023-09-23T19:36:32.210 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.7570000000000004e-06 2023-09-23T19:36:32.210 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.210 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.211 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:36:32.211 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.601230+0000", 2023-09-23T19:36:32.211 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.5673999999999999e-05 2023-09-23T19:36:32.211 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.211 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.211 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:36:32.211 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.601245+0000", 2023-09-23T19:36:32.211 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.6920000000000002e-06 2023-09-23T19:36:32.211 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.211 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.211 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:36:32.211 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.601253+0000", 2023-09-23T19:36:32.211 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.8211999999999998e-05 2023-09-23T19:36:32.212 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.212 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.212 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:36:32.212 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.601311+0000", 2023-09-23T19:36:32.212 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00018479299999999999 2023-09-23T19:36:32.212 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.212 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.212 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:36:32.212 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.601496+0000", 2023-09-23T19:36:32.212 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00022976100000000001 2023-09-23T19:36:32.212 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.213 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.213 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,5", 2023-09-23T19:36:32.213 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.601726+0000", 2023-09-23T19:36:32.213 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0043601200000000003 2023-09-23T19:36:32.213 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.213 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.213 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:36:32.213 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.606086+0000", 2023-09-23T19:36:32.213 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.023760499000000001 2023-09-23T19:36:32.213 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.213 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.213 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:32.213 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.629847+0000", 2023-09-23T19:36:32.214 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.196654314 2023-09-23T19:36:32.214 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.214 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.214 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:32.214 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.826501+0000", 2023-09-23T19:36:32.214 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.7210000000000002e-05 2023-09-23T19:36:32.214 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.214 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.214 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:36:32.214 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.826568+0000", 2023-09-23T19:36:32.214 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.1832000000000001e-05 2023-09-23T19:36:32.215 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.215 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.215 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:36:32.215 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.826620+0000", 2023-09-23T19:36:32.215 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.22539637300000001 2023-09-23T19:36:32.215 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:32.215 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:36:32.215 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:32.215 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.215 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.215 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:208989 9.d 9:bbab1653:::benchmark_data_smithi016_113233_object100607:head [delete] snapc 0=[] ondisk+write+known_if_redirected e137)", 2023-09-23T19:36:32.215 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:35:26.612950+0000", 2023-09-23T19:36:32.216 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 65.511256165999995, 2023-09-23T19:36:32.216 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.229970548, 2023-09-23T19:36:32.216 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:36:32.216 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:36:32.216 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:36:32.216 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:36:32.216 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:36:32.216 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 208989 2023-09-23T19:36:32.216 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.216 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:36:32.216 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.216 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:36:32.216 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.612950+0000", 2023-09-23T19:36:32.217 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:36:32.217 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.217 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.217 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:36:32.217 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.612950+0000", 2023-09-23T19:36:32.217 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T19:36:32.217 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.217 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.217 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:36:32.217 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.612948+0000", 2023-09-23T19:36:32.217 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.7009999999999999e-06 2023-09-23T19:36:32.217 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.218 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.218 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:36:32.218 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.612955+0000", 2023-09-23T19:36:32.218 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.1002000000000001e-05 2023-09-23T19:36:32.218 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.218 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.218 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:36:32.218 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.612966+0000", 2023-09-23T19:36:32.218 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.4650000000000002e-06 2023-09-23T19:36:32.218 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.218 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.218 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:36:32.219 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.612972+0000", 2023-09-23T19:36:32.219 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.002833009 2023-09-23T19:36:32.219 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.219 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.219 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:36:32.219 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.615805+0000", 2023-09-23T19:36:32.219 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00013731999999999999 2023-09-23T19:36:32.219 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.219 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.219 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:36:32.219 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.615942+0000", 2023-09-23T19:36:32.219 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00015651900000000001 2023-09-23T19:36:32.220 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.220 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.220 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,5", 2023-09-23T19:36:32.220 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.616098+0000", 2023-09-23T19:36:32.220 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.013827902 2023-09-23T19:36:32.220 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.220 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.220 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:36:32.220 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.629926+0000", 2023-09-23T19:36:32.220 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.6820999999999998e-05 2023-09-23T19:36:32.220 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.221 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.221 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:32.221 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.629973+0000", 2023-09-23T19:36:32.221 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.21284078400000001 2023-09-23T19:36:32.221 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.221 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.221 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:32.221 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.842814+0000", 2023-09-23T19:36:32.221 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.5655999999999997e-05 2023-09-23T19:36:32.221 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.221 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.221 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:36:32.222 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.842870+0000", 2023-09-23T19:36:32.222 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.1178999999999999e-05 2023-09-23T19:36:32.222 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.222 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.222 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:36:32.222 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.842921+0000", 2023-09-23T19:36:32.222 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.229970548 2023-09-23T19:36:32.222 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:32.222 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:36:32.222 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:32.222 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.222 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.223 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:208990 9.d 9:b24ecd90:::benchmark_data_smithi016_113233_object100608:head [delete] snapc 0=[] ondisk+write+known_if_redirected e137)", 2023-09-23T19:36:32.223 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:35:26.612982+0000", 2023-09-23T19:36:32.223 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 65.511224815999995, 2023-09-23T19:36:32.223 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.23009431799999999, 2023-09-23T19:36:32.223 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:36:32.223 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:36:32.223 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:36:32.223 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:36:32.223 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:36:32.223 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 208990 2023-09-23T19:36:32.224 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.224 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:36:32.224 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.224 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:36:32.224 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.612982+0000", 2023-09-23T19:36:32.224 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:36:32.224 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.224 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.224 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:36:32.224 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.612982+0000", 2023-09-23T19:36:32.224 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999995 2023-09-23T19:36:32.224 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.225 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.225 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:36:32.225 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.612981+0000", 2023-09-23T19:36:32.225 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.8519999999999997e-06 2023-09-23T19:36:32.225 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.225 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.225 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:36:32.225 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.612985+0000", 2023-09-23T19:36:32.225 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.5910000000000001e-06 2023-09-23T19:36:32.225 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.225 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.225 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:36:32.225 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.612992+0000", 2023-09-23T19:36:32.226 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.2320000000000003e-06 2023-09-23T19:36:32.226 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.226 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.226 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:36:32.226 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.612996+0000", 2023-09-23T19:36:32.226 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.012239991 2023-09-23T19:36:32.226 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.226 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.226 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:36:32.226 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.625236+0000", 2023-09-23T19:36:32.226 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00014660399999999999 2023-09-23T19:36:32.226 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.227 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.227 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:36:32.227 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.625382+0000", 2023-09-23T19:36:32.227 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000100065 2023-09-23T19:36:32.227 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.227 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.227 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,5", 2023-09-23T19:36:32.227 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.625482+0000", 2023-09-23T19:36:32.227 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0045393910000000003 2023-09-23T19:36:32.227 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.227 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.228 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:36:32.228 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.630022+0000", 2023-09-23T19:36:32.228 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.021801620000000001 2023-09-23T19:36:32.228 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.228 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.228 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:32.228 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.651823+0000", 2023-09-23T19:36:32.228 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.191152778 2023-09-23T19:36:32.228 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.228 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.228 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:32.228 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.842976+0000", 2023-09-23T19:36:32.229 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.7379999999999998e-05 2023-09-23T19:36:32.229 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.229 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.229 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:36:32.229 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.843014+0000", 2023-09-23T19:36:32.229 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.2272000000000003e-05 2023-09-23T19:36:32.229 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.229 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.229 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:36:32.229 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.843076+0000", 2023-09-23T19:36:32.229 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.23009431799999999 2023-09-23T19:36:32.229 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:32.230 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:36:32.230 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:32.230 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.230 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.230 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:208995 9.e 9:7a040566:::benchmark_data_smithi016_113233_object100613:head [delete] snapc 0=[] ondisk+write+known_if_redirected e137)", 2023-09-23T19:36:32.230 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:35:26.613044+0000", 2023-09-23T19:36:32.230 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 65.511162268999996, 2023-09-23T19:36:32.230 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.21365092299999999, 2023-09-23T19:36:32.230 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:36:32.230 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:36:32.230 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:36:32.230 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:36:32.231 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:36:32.231 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 208995 2023-09-23T19:36:32.231 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.231 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:36:32.231 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.231 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:36:32.231 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.613044+0000", 2023-09-23T19:36:32.231 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:36:32.231 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.231 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.231 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:36:32.231 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.613044+0000", 2023-09-23T19:36:32.232 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999995 2023-09-23T19:36:32.232 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.232 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.232 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:36:32.232 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.613044+0000", 2023-09-23T19:36:32.232 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.824e-06 2023-09-23T19:36:32.232 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.232 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.232 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:36:32.232 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.613046+0000", 2023-09-23T19:36:32.232 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.947e-06 2023-09-23T19:36:32.232 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.233 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.233 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:36:32.233 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.613052+0000", 2023-09-23T19:36:32.233 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.7330000000000001e-06 2023-09-23T19:36:32.233 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.233 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.233 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:36:32.233 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.613056+0000", 2023-09-23T19:36:32.233 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0050268300000000004 2023-09-23T19:36:32.233 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.233 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.234 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:36:32.234 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.618083+0000", 2023-09-23T19:36:32.234 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000125809 2023-09-23T19:36:32.234 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.234 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.234 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:36:32.234 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.618209+0000", 2023-09-23T19:36:32.234 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000142208 2023-09-23T19:36:32.234 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.234 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.234 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,4", 2023-09-23T19:36:32.234 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.618351+0000", 2023-09-23T19:36:32.234 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.013098810000000001 2023-09-23T19:36:32.235 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.235 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.235 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:32.235 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.631450+0000", 2023-09-23T19:36:32.235 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000119982 2023-09-23T19:36:32.235 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.235 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.235 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:36:32.235 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.631570+0000", 2023-09-23T19:36:32.235 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.19500669600000001 2023-09-23T19:36:32.235 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.236 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.236 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:32.236 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.826577+0000", 2023-09-23T19:36:32.236 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.5329999999999998e-05 2023-09-23T19:36:32.236 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.236 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.236 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:36:32.236 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.826662+0000", 2023-09-23T19:36:32.236 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.3200000000000001e-05 2023-09-23T19:36:32.236 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.236 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.236 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:36:32.236 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.826695+0000", 2023-09-23T19:36:32.237 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.21365092299999999 2023-09-23T19:36:32.237 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:32.237 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:36:32.237 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:32.237 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.237 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.237 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:209002 9.d 9:b8ac4f6d:::benchmark_data_smithi016_113233_object100620:head [delete] snapc 0=[] ondisk+write+known_if_redirected e137)", 2023-09-23T19:36:32.237 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:35:26.632989+0000", 2023-09-23T19:36:32.237 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 65.491217806999998, 2023-09-23T19:36:32.237 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.21023080699999999, 2023-09-23T19:36:32.237 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:36:32.237 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:36:32.238 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:36:32.238 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:36:32.238 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:36:32.238 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 209002 2023-09-23T19:36:32.238 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.238 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:36:32.238 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.238 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:36:32.238 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.632989+0000", 2023-09-23T19:36:32.238 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:36:32.238 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.239 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.239 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:36:32.239 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.632989+0000", 2023-09-23T19:36:32.239 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999976 2023-09-23T19:36:32.239 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.239 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.239 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:36:32.239 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.632986+0000", 2023-09-23T19:36:32.239 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.2043e-05 2023-09-23T19:36:32.239 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.239 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.239 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:36:32.240 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.632998+0000", 2023-09-23T19:36:32.240 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.6719e-05 2023-09-23T19:36:32.240 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.240 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.240 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:36:32.240 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.633015+0000", 2023-09-23T19:36:32.240 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.1560000000000001e-06 2023-09-23T19:36:32.240 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.240 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.240 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:36:32.240 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.633024+0000", 2023-09-23T19:36:32.240 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.1647000000000002e-05 2023-09-23T19:36:32.241 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.241 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.241 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:36:32.241 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.633076+0000", 2023-09-23T19:36:32.241 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000109858 2023-09-23T19:36:32.241 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.241 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.241 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:36:32.241 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.633185+0000", 2023-09-23T19:36:32.241 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00010092199999999999 2023-09-23T19:36:32.241 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.241 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.242 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,5", 2023-09-23T19:36:32.242 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.633286+0000", 2023-09-23T19:36:32.242 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0045832210000000002 2023-09-23T19:36:32.242 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.242 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.242 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:36:32.242 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.637870+0000", 2023-09-23T19:36:32.242 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.013985238000000001 2023-09-23T19:36:32.242 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.242 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.242 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:32.242 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.651855+0000", 2023-09-23T19:36:32.243 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.191273893 2023-09-23T19:36:32.243 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.243 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.243 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:32.243 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.843129+0000", 2023-09-23T19:36:32.243 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.9422999999999998e-05 2023-09-23T19:36:32.243 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.243 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.243 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:36:32.243 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.843178+0000", 2023-09-23T19:36:32.243 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.1186999999999997e-05 2023-09-23T19:36:32.243 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.244 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.244 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:36:32.244 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.843219+0000", 2023-09-23T19:36:32.244 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.21023080699999999 2023-09-23T19:36:32.244 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:32.244 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:36:32.244 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:32.244 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.244 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.244 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:209010 9.e 9:7c5d9b11:::benchmark_data_smithi016_113233_object100628:head [delete] snapc 0=[] ondisk+write+known_if_redirected e137)", 2023-09-23T19:36:32.244 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:35:26.647717+0000", 2023-09-23T19:36:32.244 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 65.476489194999999, 2023-09-23T19:36:32.244 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.21077024899999999, 2023-09-23T19:36:32.245 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:36:32.245 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:36:32.245 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:36:32.245 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:36:32.245 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:36:32.245 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 209010 2023-09-23T19:36:32.245 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.245 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:36:32.245 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.245 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:36:32.245 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.647717+0000", 2023-09-23T19:36:32.246 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:36:32.246 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.246 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.246 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:36:32.246 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.647717+0000", 2023-09-23T19:36:32.246 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999976 2023-09-23T19:36:32.246 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.246 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.246 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:36:32.246 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.647715+0000", 2023-09-23T19:36:32.246 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.8609999999999993e-06 2023-09-23T19:36:32.246 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.247 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.247 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:36:32.247 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.647725+0000", 2023-09-23T19:36:32.247 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.2419e-05 2023-09-23T19:36:32.247 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.247 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.247 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:36:32.247 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.647747+0000", 2023-09-23T19:36:32.247 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.7399999999999993e-06 2023-09-23T19:36:32.247 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.247 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.247 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:36:32.247 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.647756+0000", 2023-09-23T19:36:32.248 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.2290000000000001e-05 2023-09-23T19:36:32.248 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.248 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.248 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:36:32.248 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.647828+0000", 2023-09-23T19:36:32.248 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000134091 2023-09-23T19:36:32.248 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.248 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.248 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:36:32.248 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.647962+0000", 2023-09-23T19:36:32.248 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00016182300000000001 2023-09-23T19:36:32.248 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.249 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.249 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,4", 2023-09-23T19:36:32.249 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.648124+0000", 2023-09-23T19:36:32.249 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.010215615000000001 2023-09-23T19:36:32.249 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.249 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.249 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:32.249 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.658340+0000", 2023-09-23T19:36:32.249 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0045123359999999996 2023-09-23T19:36:32.249 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.249 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.249 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:36:32.249 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.662852+0000", 2023-09-23T19:36:32.250 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.19550694599999999 2023-09-23T19:36:32.250 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.250 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.250 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:32.250 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.858359+0000", 2023-09-23T19:36:32.250 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.7562000000000001e-05 2023-09-23T19:36:32.250 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.250 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.250 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:36:32.250 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.858427+0000", 2023-09-23T19:36:32.250 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.0812e-05 2023-09-23T19:36:32.250 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.251 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.251 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:36:32.251 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.858487+0000", 2023-09-23T19:36:32.251 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.21077024899999999 2023-09-23T19:36:32.251 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:32.251 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:36:32.251 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:32.251 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.251 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.251 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:209012 9.3 9:c8071fe1:::benchmark_data_smithi016_113233_object100630:head [delete] snapc 0=[] ondisk+write+known_if_redirected e137)", 2023-09-23T19:36:32.251 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:35:26.654569+0000", 2023-09-23T19:36:32.251 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 65.469637665999997, 2023-09-23T19:36:32.252 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.18387530799999999, 2023-09-23T19:36:32.252 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:36:32.252 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:36:32.252 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:36:32.252 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:36:32.252 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:36:32.252 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 209012 2023-09-23T19:36:32.252 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.252 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:36:32.252 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.252 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:36:32.252 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.654569+0000", 2023-09-23T19:36:32.252 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:36:32.253 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.253 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.253 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:36:32.253 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.654569+0000", 2023-09-23T19:36:32.253 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999914 2023-09-23T19:36:32.253 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.253 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.253 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:36:32.253 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.654560+0000", 2023-09-23T19:36:32.253 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.9383999999999999e-05 2023-09-23T19:36:32.253 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.253 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.254 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:36:32.254 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.654579+0000", 2023-09-23T19:36:32.254 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.4882e-05 2023-09-23T19:36:32.254 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.254 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.254 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:36:32.254 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.654604+0000", 2023-09-23T19:36:32.254 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.9680000000000001e-06 2023-09-23T19:36:32.254 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.254 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.254 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:36:32.254 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.654614+0000", 2023-09-23T19:36:32.255 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.1459999999999998e-05 2023-09-23T19:36:32.255 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.255 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.255 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:36:32.255 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.654676+0000", 2023-09-23T19:36:32.255 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00013746799999999999 2023-09-23T19:36:32.255 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.255 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.255 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:36:32.255 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.654813+0000", 2023-09-23T19:36:32.255 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000150452 2023-09-23T19:36:32.255 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.256 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.256 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,3", 2023-09-23T19:36:32.256 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.654964+0000", 2023-09-23T19:36:32.256 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0045507359999999997 2023-09-23T19:36:32.256 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.256 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.256 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:32.256 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.659514+0000", 2023-09-23T19:36:32.256 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00013708599999999999 2023-09-23T19:36:32.256 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.256 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.256 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:36:32.256 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.659651+0000", 2023-09-23T19:36:32.257 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.17867601899999999 2023-09-23T19:36:32.257 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.257 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.257 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:32.257 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.838327+0000", 2023-09-23T19:36:32.257 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.2668000000000004e-05 2023-09-23T19:36:32.257 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.257 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.257 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:36:32.257 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.838390+0000", 2023-09-23T19:36:32.257 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.3909000000000002e-05 2023-09-23T19:36:32.258 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.258 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.258 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:36:32.258 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.838444+0000", 2023-09-23T19:36:32.258 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.18387530799999999 2023-09-23T19:36:32.258 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:32.258 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:36:32.258 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:32.258 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.258 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.258 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:209015 9.e 9:7873cd13:::benchmark_data_smithi016_113233_object100633:head [delete] snapc 0=[] ondisk+write+known_if_redirected e137)", 2023-09-23T19:36:32.258 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:35:26.658215+0000", 2023-09-23T19:36:32.259 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 65.465991138999996, 2023-09-23T19:36:32.259 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.20040670799999999, 2023-09-23T19:36:32.259 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:36:32.259 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:36:32.259 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:36:32.259 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:36:32.259 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:36:32.259 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 209015 2023-09-23T19:36:32.259 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.259 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:36:32.259 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.259 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:36:32.260 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.658215+0000", 2023-09-23T19:36:32.260 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:36:32.260 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.260 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.260 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:36:32.260 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.658215+0000", 2023-09-23T19:36:32.260 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T19:36:32.260 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.260 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.260 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:36:32.260 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.658213+0000", 2023-09-23T19:36:32.260 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.055e-06 2023-09-23T19:36:32.260 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.261 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.261 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:36:32.261 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.658221+0000", 2023-09-23T19:36:32.261 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.3132999999999999e-05 2023-09-23T19:36:32.261 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.261 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.261 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:36:32.261 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.658234+0000", 2023-09-23T19:36:32.261 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.0929999999999997e-06 2023-09-23T19:36:32.261 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.261 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.261 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:36:32.262 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.658242+0000", 2023-09-23T19:36:32.262 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000118411 2023-09-23T19:36:32.262 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.262 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.262 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:36:32.262 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.658360+0000", 2023-09-23T19:36:32.262 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.7691999999999999e-05 2023-09-23T19:36:32.262 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.262 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.262 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:36:32.262 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.658458+0000", 2023-09-23T19:36:32.262 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000115229 2023-09-23T19:36:32.262 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.263 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.263 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,4", 2023-09-23T19:36:32.263 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.658573+0000", 2023-09-23T19:36:32.263 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0045246649999999998 2023-09-23T19:36:32.263 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.263 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.263 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:36:32.263 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.663098+0000", 2023-09-23T19:36:32.263 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00023705899999999999 2023-09-23T19:36:32.263 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.263 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.263 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:32.264 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.663335+0000", 2023-09-23T19:36:32.264 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.19522404500000001 2023-09-23T19:36:32.264 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.264 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.264 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:32.264 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.858559+0000", 2023-09-23T19:36:32.264 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.4736000000000001e-05 2023-09-23T19:36:32.264 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.264 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.264 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:36:32.264 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.858593+0000", 2023-09-23T19:36:32.264 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.8591000000000001e-05 2023-09-23T19:36:32.265 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.265 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.265 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:36:32.265 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.858622+0000", 2023-09-23T19:36:32.265 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.20040670799999999 2023-09-23T19:36:32.265 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:32.265 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:36:32.265 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:32.265 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.265 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.265 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:209018 9.e 9:78e76208:::benchmark_data_smithi016_113233_object100636:head [delete] snapc 0=[] ondisk+write+known_if_redirected e137)", 2023-09-23T19:36:32.265 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:35:26.672023+0000", 2023-09-23T19:36:32.265 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 65.452183208999998, 2023-09-23T19:36:32.266 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.18676025900000001, 2023-09-23T19:36:32.266 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:36:32.266 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:36:32.266 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:36:32.266 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:36:32.266 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:36:32.266 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 209018 2023-09-23T19:36:32.266 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.266 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:36:32.266 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.266 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:36:32.267 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.672023+0000", 2023-09-23T19:36:32.267 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:36:32.267 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.267 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.267 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:36:32.267 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.672023+0000", 2023-09-23T19:36:32.267 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T19:36:32.267 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.267 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.267 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:36:32.267 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.672021+0000", 2023-09-23T19:36:32.267 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.3939999999999996e-06 2023-09-23T19:36:32.268 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.268 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.268 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:36:32.268 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.672030+0000", 2023-09-23T19:36:32.268 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.8062000000000001e-05 2023-09-23T19:36:32.268 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.268 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.268 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:36:32.268 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.672048+0000", 2023-09-23T19:36:32.268 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.4009999999999997e-06 2023-09-23T19:36:32.268 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.268 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.268 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:36:32.269 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.672056+0000", 2023-09-23T19:36:32.269 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.8889999999999999e-05 2023-09-23T19:36:32.269 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.269 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.269 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:36:32.269 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.672125+0000", 2023-09-23T19:36:32.269 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000195603 2023-09-23T19:36:32.269 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.269 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.269 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:36:32.269 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.672321+0000", 2023-09-23T19:36:32.269 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00023059600000000001 2023-09-23T19:36:32.270 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.270 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.270 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,4", 2023-09-23T19:36:32.270 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.672551+0000", 2023-09-23T19:36:32.270 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.003564219 2023-09-23T19:36:32.270 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.270 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.270 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:36:32.270 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.676116+0000", 2023-09-23T19:36:32.270 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00057282399999999999 2023-09-23T19:36:32.270 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.270 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.271 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:32.271 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.676688+0000", 2023-09-23T19:36:32.271 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.18199246699999999 2023-09-23T19:36:32.271 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.271 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.271 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:36:32.271 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.858681+0000", 2023-09-23T19:36:32.271 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.7912999999999999e-05 2023-09-23T19:36:32.271 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.271 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.271 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:36:32.271 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.858739+0000", 2023-09-23T19:36:32.272 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.4705e-05 2023-09-23T19:36:32.272 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:36:32.272 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:36:32.272 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:36:32.272 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.858783+0000", 2023-09-23T19:36:32.272 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.18676025900000001 2023-09-23T19:36:32.272 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:32.272 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:36:32.272 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:32.272 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:36:32.272 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:36:32.272 INFO:teuthology.orchestra.run.smithi092.stdout:} 2023-09-23T19:36:32.273 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:32.300 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:32.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:32.303 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:36:32.404 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:32.407 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T19:36:32.413 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:32 smithi092 bash[58855]: audit 2023-09-23T19:36:31.636143+0000 mon.c (mon.2) 160 : audit 1 from='client.? 172.21.15.16:0/3676923328' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T19:36:32.414 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:32 smithi092 bash[58855]: audit 2023-09-23T19:36:31.636756+0000 mon.a (mon.0) 1084 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T19:36:32.427 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:32.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:32.430 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:32.531 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:32.556 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:32.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:32.559 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:32.608 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:32 smithi016 bash[75004]: audit 2023-09-23T19:36:31.636143+0000 mon.c (mon.2) 160 : audit 1 from='client.? 172.21.15.16:0/3676923328' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T19:36:32.609 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:32 smithi016 bash[75004]: audit 2023-09-23T19:36:31.636756+0000 mon.a (mon.0) 1084 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T19:36:32.663 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:32.695 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:32.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:32.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:36:32.797 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:32.822 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:32.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:32.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:32.925 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:32.933 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T19:36:32.946 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:32 smithi016 bash[77995]: audit 2023-09-23T19:36:31.636143+0000 mon.c (mon.2) 160 : audit 1 from='client.? 172.21.15.16:0/3676923328' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T19:36:32.946 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:32 smithi016 bash[77995]: audit 2023-09-23T19:36:31.636756+0000 mon.a (mon.0) 1084 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T19:36:32.961 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:32.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:32.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:36:33.064 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:33.086 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:33.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:33.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:36:33.189 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:33.224 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:33.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:33.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:36:33.242 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:36:33.328 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:33.358 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:33.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:33.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:33.415 INFO:teuthology.orchestra.run.smithi016.stderr:enabled application 'rados' on pool 'unique_pool_5' 2023-09-23T19:36:33.462 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:33.485 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:33.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:33.488 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:33.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:33 smithi016 bash[75004]: cluster 2023-09-23T19:36:32.063736+0000 mgr.y (mgr.25325) 869 : cluster 0 pgmap v789: 89 pgs: 89 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:36:33.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:33 smithi016 bash[75004]: audit 2023-09-23T19:36:32.399327+0000 mon.a (mon.0) 1085 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T19:36:33.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:33 smithi016 bash[75004]: cluster 2023-09-23T19:36:32.399435+0000 mon.a (mon.0) 1086 : cluster 0 osdmap e148: 8 total, 8 up, 8 in 2023-09-23T19:36:33.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:33 smithi016 bash[75004]: cluster 2023-09-23T19:36:32.400969+0000 osd.7 (osd.7) 63 : cluster 3 failed to encode map e148 with expected crc 2023-09-23T19:36:33.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:33 smithi016 bash[75004]: audit 2023-09-23T19:36:32.944209+0000 mon.c (mon.2) 161 : audit 1 from='client.? 172.21.15.16:0/103429963' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:36:33.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:33 smithi016 bash[75004]: audit 2023-09-23T19:36:32.944847+0000 mon.a (mon.0) 1087 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:36:33.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:33 smithi016 bash[75004]: cluster 2023-09-23T19:36:33.025404+0000 osd.5 (osd.5) 78 : cluster 3 failed to encode map e148 with expected crc 2023-09-23T19:36:33.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:33 smithi016 bash[77995]: cluster 2023-09-23T19:36:32.063736+0000 mgr.y (mgr.25325) 869 : cluster 0 pgmap v789: 89 pgs: 89 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:36:33.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:33 smithi016 bash[77995]: audit 2023-09-23T19:36:32.399327+0000 mon.a (mon.0) 1085 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T19:36:33.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:33 smithi016 bash[77995]: cluster 2023-09-23T19:36:32.399435+0000 mon.a (mon.0) 1086 : cluster 0 osdmap e148: 8 total, 8 up, 8 in 2023-09-23T19:36:33.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:33 smithi016 bash[77995]: cluster 2023-09-23T19:36:32.400969+0000 osd.7 (osd.7) 63 : cluster 3 failed to encode map e148 with expected crc 2023-09-23T19:36:33.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:33 smithi016 bash[77995]: audit 2023-09-23T19:36:32.944209+0000 mon.c (mon.2) 161 : audit 1 from='client.? 172.21.15.16:0/103429963' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:36:33.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:33 smithi016 bash[77995]: audit 2023-09-23T19:36:32.944847+0000 mon.a (mon.0) 1087 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:36:33.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:33 smithi016 bash[77995]: cluster 2023-09-23T19:36:33.025404+0000 osd.5 (osd.5) 78 : cluster 3 failed to encode map e148 with expected crc 2023-09-23T19:36:33.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:33 smithi092 bash[58855]: cluster 2023-09-23T19:36:32.063736+0000 mgr.y (mgr.25325) 869 : cluster 0 pgmap v789: 89 pgs: 89 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:36:33.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:33 smithi092 bash[58855]: audit 2023-09-23T19:36:32.399327+0000 mon.a (mon.0) 1085 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T19:36:33.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:33 smithi092 bash[58855]: cluster 2023-09-23T19:36:32.399435+0000 mon.a (mon.0) 1086 : cluster 0 osdmap e148: 8 total, 8 up, 8 in 2023-09-23T19:36:33.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:33 smithi092 bash[58855]: cluster 2023-09-23T19:36:32.400969+0000 osd.7 (osd.7) 63 : cluster 3 failed to encode map e148 with expected crc 2023-09-23T19:36:33.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:33 smithi092 bash[58855]: audit 2023-09-23T19:36:32.944209+0000 mon.c (mon.2) 161 : audit 1 from='client.? 172.21.15.16:0/103429963' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:36:33.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:33 smithi092 bash[58855]: audit 2023-09-23T19:36:32.944847+0000 mon.a (mon.0) 1087 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:36:33.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:33 smithi092 bash[58855]: cluster 2023-09-23T19:36:33.025404+0000 osd.5 (osd.5) 78 : cluster 3 failed to encode map e148 with expected crc 2023-09-23T19:36:33.589 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:33.620 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:33.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:33.622 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:36:33.680 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-23T19:36:33.723 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:33.754 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:33.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:33.757 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:33.858 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:33.889 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:33.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:33.892 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:33.993 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:34.023 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:34.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:34.025 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:36:34.125 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:34.152 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:34.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:34.156 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:34.256 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:34.281 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:34.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:34.284 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:34.385 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:34.419 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:34.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:34.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:36:34.522 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:34.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:34 smithi016 bash[75004]: audit 2023-09-23T19:36:33.405890+0000 mon.a (mon.0) 1088 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T19:36:34.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:34 smithi016 bash[75004]: cluster 2023-09-23T19:36:33.405984+0000 mon.a (mon.0) 1089 : cluster 0 osdmap e149: 8 total, 8 up, 8 in 2023-09-23T19:36:34.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:34 smithi016 bash[75004]: cluster 2023-09-23T19:36:33.409317+0000 osd.6 (osd.6) 64 : cluster 3 failed to encode map e149 with expected crc 2023-09-23T19:36:34.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:34 smithi016 bash[75004]: cluster 2023-09-23T19:36:33.464145+0000 osd.4 (osd.4) 76 : cluster 3 failed to encode map e148 with expected crc 2023-09-23T19:36:34.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:34 smithi016 bash[75004]: cluster 2023-09-23T19:36:34.069973+0000 osd.5 (osd.5) 79 : cluster 3 failed to encode map e149 with expected crc 2023-09-23T19:36:34.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:34 smithi016 bash[77995]: audit 2023-09-23T19:36:33.405890+0000 mon.a (mon.0) 1088 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T19:36:34.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:34 smithi016 bash[77995]: cluster 2023-09-23T19:36:33.405984+0000 mon.a (mon.0) 1089 : cluster 0 osdmap e149: 8 total, 8 up, 8 in 2023-09-23T19:36:34.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:34 smithi016 bash[77995]: cluster 2023-09-23T19:36:33.409317+0000 osd.6 (osd.6) 64 : cluster 3 failed to encode map e149 with expected crc 2023-09-23T19:36:34.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:34 smithi016 bash[77995]: cluster 2023-09-23T19:36:33.464145+0000 osd.4 (osd.4) 76 : cluster 3 failed to encode map e148 with expected crc 2023-09-23T19:36:34.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:34 smithi016 bash[77995]: cluster 2023-09-23T19:36:34.069973+0000 osd.5 (osd.5) 79 : cluster 3 failed to encode map e149 with expected crc 2023-09-23T19:36:34.555 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:34.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:34.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:34.618 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T19:36:34.660 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:34.672 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:34 smithi092 bash[58855]: audit 2023-09-23T19:36:33.405890+0000 mon.a (mon.0) 1088 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T19:36:34.672 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:34 smithi092 bash[58855]: cluster 2023-09-23T19:36:33.405984+0000 mon.a (mon.0) 1089 : cluster 0 osdmap e149: 8 total, 8 up, 8 in 2023-09-23T19:36:34.673 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:34 smithi092 bash[58855]: cluster 2023-09-23T19:36:33.409317+0000 osd.6 (osd.6) 64 : cluster 3 failed to encode map e149 with expected crc 2023-09-23T19:36:34.673 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:34 smithi092 bash[58855]: cluster 2023-09-23T19:36:33.464145+0000 osd.4 (osd.4) 76 : cluster 3 failed to encode map e148 with expected crc 2023-09-23T19:36:34.673 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:34 smithi092 bash[58855]: cluster 2023-09-23T19:36:34.069973+0000 osd.5 (osd.5) 79 : cluster 3 failed to encode map e149 with expected crc 2023-09-23T19:36:34.691 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:34.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:34.694 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:36:34.795 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:34.827 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:34.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:34.831 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:34.932 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:34.962 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:34.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:34.965 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:36:35.066 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:35.097 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:35.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:35.100 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:36:35.201 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:35.236 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:35.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:35.238 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:36:35.339 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:35.354 DEBUG:teuthology.orchestra.run.smithi016:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_5 bench 90 write' 2023-09-23T19:36:35.366 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:35.368 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T19:36:35.368 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T19:36:35.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:35.369 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:35.470 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:35.482 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:hints = 1 2023-09-23T19:36:35.482 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T19:36:35.483 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Object prefix: benchmark_data_smithi016_136365 2023-09-23T19:36:35.483 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:36:35.483 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 0 0 0 0 0 0 - 0 2023-09-23T19:36:35.507 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:35.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:35.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:35.612 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:35.641 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:35 smithi016 bash[77995]: cluster 2023-09-23T19:36:34.064376+0000 mgr.y (mgr.25325) 870 : cluster 0 pgmap v792: 89 pgs: 89 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:36:35.641 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:35 smithi016 bash[77995]: cluster 2023-09-23T19:36:34.159376+0000 osd.7 (osd.7) 64 : cluster 3 failed to encode map e149 with expected crc 2023-09-23T19:36:35.641 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:35 smithi016 bash[75004]: cluster 2023-09-23T19:36:34.064376+0000 mgr.y (mgr.25325) 870 : cluster 0 pgmap v792: 89 pgs: 89 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:36:35.642 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:35 smithi016 bash[75004]: cluster 2023-09-23T19:36:34.159376+0000 osd.7 (osd.7) 64 : cluster 3 failed to encode map e149 with expected crc 2023-09-23T19:36:35.642 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:36:35 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:36:35] "GET /metrics HTTP/1.1" 200 36227 "" "Prometheus/2.33.4" 2023-09-23T19:36:35.647 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:35.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:35.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:36:35.752 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:35.779 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:35.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:35.782 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:35.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:35 smithi092 bash[58855]: cluster 2023-09-23T19:36:34.064376+0000 mgr.y (mgr.25325) 870 : cluster 0 pgmap v792: 89 pgs: 89 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T19:36:35.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:35 smithi092 bash[58855]: cluster 2023-09-23T19:36:34.159376+0000 osd.7 (osd.7) 64 : cluster 3 failed to encode map e149 with expected crc 2023-09-23T19:36:35.882 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:35.908 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:35.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:35.911 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:36.011 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:36.042 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:36.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:36.043 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:36.144 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:36.171 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:36.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:36.189 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:36.290 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:36.325 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:36.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:36.329 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:36.429 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:36.454 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:36.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:36.456 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:36.483 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 1 16 2721 2705 169.033 169.062 0.00358148 0.00588093 2023-09-23T19:36:36.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:36 smithi092 bash[58855]: audit 2023-09-23T19:36:36.040350+0000 mon.a (mon.0) 1090 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:36:36.557 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:36.585 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:36.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:36.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:36:36.690 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:36.720 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:36.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:36.722 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:36.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:36 smithi016 bash[77995]: audit 2023-09-23T19:36:36.040350+0000 mon.a (mon.0) 1090 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:36:36.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:36 smithi016 bash[75004]: audit 2023-09-23T19:36:36.040350+0000 mon.a (mon.0) 1090 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:36:36.822 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:36.852 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:36.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:36.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:36:36.955 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:36.986 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:36.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:36.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:36:37.090 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:37.119 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:37.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:37.123 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:36:37.223 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:37.251 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:37.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:37.257 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:36:37.358 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:37.389 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:37.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:37.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:36:37.484 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 2 16 5441 5425 169.507 170 0.00426416 0.0058875 2023-09-23T19:36:37.493 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:37.520 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:37.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:37.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:37.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:37 smithi092 bash[58855]: cluster 2023-09-23T19:36:36.065174+0000 mgr.y (mgr.25325) 871 : cluster 0 pgmap v793: 89 pgs: 89 active+clean; 2.4 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 309 KiB/s wr, 4 op/s 2023-09-23T19:36:37.624 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:37.634 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:37 smithi016 bash[77995]: cluster 2023-09-23T19:36:36.065174+0000 mgr.y (mgr.25325) 871 : cluster 0 pgmap v793: 89 pgs: 89 active+clean; 2.4 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 309 KiB/s wr, 4 op/s 2023-09-23T19:36:37.635 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:37 smithi016 bash[75004]: cluster 2023-09-23T19:36:36.065174+0000 mgr.y (mgr.25325) 871 : cluster 0 pgmap v793: 89 pgs: 89 active+clean; 2.4 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 309 KiB/s wr, 4 op/s 2023-09-23T19:36:37.657 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:37.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:37.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:36:37.762 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:37.792 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:37.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:37.795 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:36:37.896 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:37.905 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:36:37.906 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:36:37.908 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:36:37.921 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:37.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:37.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:38.025 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:38.055 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:38.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:38.059 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:36:38.159 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:38.185 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:38.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:38.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:36:38.289 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:38.320 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:38.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:38.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:36:38.425 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:38.457 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:38.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:38.460 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:36:38.484 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 3 16 7968 7952 165.645 157.938 0.00550895 0.00602522 2023-09-23T19:36:38.561 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:38.587 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:38.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:38.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:36:38.690 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:38.719 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:38.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:38.722 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:36:38.823 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:38.850 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:38.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:38.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:38.926 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:36:38.953 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:38.980 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:38.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:38.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:39.083 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:39.110 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:39.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:39.114 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:36:39.214 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:39.246 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:39.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:39.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:39.350 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:39.380 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:39.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:39.384 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:39.484 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 4 16 10298 10282 160.636 145.625 0.00406722 0.00621528 2023-09-23T19:36:39.485 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:39.510 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:39.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:39.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:39.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:39 smithi016 bash[75004]: cluster 2023-09-23T19:36:38.065938+0000 mgr.y (mgr.25325) 872 : cluster 0 pgmap v794: 89 pgs: 89 active+clean; 101 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 13 MiB/s wr, 209 op/s 2023-09-23T19:36:39.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:39 smithi016 bash[77995]: cluster 2023-09-23T19:36:38.065938+0000 mgr.y (mgr.25325) 872 : cluster 0 pgmap v794: 89 pgs: 89 active+clean; 101 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 13 MiB/s wr, 209 op/s 2023-09-23T19:36:39.613 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:39.623 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:39 smithi092 bash[58855]: cluster 2023-09-23T19:36:38.065938+0000 mgr.y (mgr.25325) 872 : cluster 0 pgmap v794: 89 pgs: 89 active+clean; 101 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 13 MiB/s wr, 209 op/s 2023-09-23T19:36:39.641 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:39.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:39.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:36:39.745 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:39.772 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:39.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:39.776 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:39.877 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:39.905 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:39.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:39.907 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:40.008 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:40.034 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:40.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:40.037 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:36:40.138 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:40.162 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:40.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:40.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:40.268 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:40.296 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:40.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:40.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:40.400 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:40.425 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:40.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:40.427 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:40.484 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 5 16 12669 12653 158.144 148.188 0.00316572 0.0063156 2023-09-23T19:36:40.529 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:40.559 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:40.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:40.560 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:36:40.661 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:40.687 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:40.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:40.689 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:36:40.790 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:40.813 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:40.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:40.815 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:40.916 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:40.944 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:40.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:40.946 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:36:41.047 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:41.082 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:41.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:41.108 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:36:41.209 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:41.229 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:41.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:41.231 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:36:41.332 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:41.356 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:41.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:41.358 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:36:41.441 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:41 smithi016 bash[75004]: cluster 2023-09-23T19:36:40.066638+0000 mgr.y (mgr.25325) 873 : cluster 0 pgmap v795: 89 pgs: 89 active+clean; 443 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 55 MiB/s wr, 884 op/s 2023-09-23T19:36:41.458 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:41.484 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 6 16 14416 14400 149.969 109.188 0.00425924 0.00666042 2023-09-23T19:36:41.492 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:41.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:41.497 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:36:41.598 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:41.663 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:41.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:41.667 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:36:41.767 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:41 smithi016 bash[77995]: cluster 2023-09-23T19:36:40.066638+0000 mgr.y (mgr.25325) 873 : cluster 0 pgmap v795: 89 pgs: 89 active+clean; 443 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 55 MiB/s wr, 884 op/s 2023-09-23T19:36:41.767 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:41.777 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:41 smithi092 bash[58855]: cluster 2023-09-23T19:36:40.066638+0000 mgr.y (mgr.25325) 873 : cluster 0 pgmap v795: 89 pgs: 89 active+clean; 443 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 55 MiB/s wr, 884 op/s 2023-09-23T19:36:41.791 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:41.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:41.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:41.894 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:41.925 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:41.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:41.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:42.029 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:42.056 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:42.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:42.060 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:42.161 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:42.186 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:42.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:42.204 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:36:42.305 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:42.341 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:42.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:42.345 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:36:42.446 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:42.475 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:42.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:42.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:42.485 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 7 16 15975 15959 142.463 97.4375 0.00615816 0.00700653 2023-09-23T19:36:42.582 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:42.621 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:42.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:42.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:42.725 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:42.767 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:42.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:42.771 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:42.872 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:42.904 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:42.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:42.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:43.008 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:43.036 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:43.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:43.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:36:43.140 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:43.165 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:43.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:43.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:43.269 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:43.276 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:43 smithi016 bash[75004]: cluster 2023-09-23T19:36:42.067325+0000 mgr.y (mgr.25325) 874 : cluster 0 pgmap v796: 89 pgs: 89 active+clean; 628 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T19:36:43.276 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:43 smithi016 bash[75004]: audit 2023-09-23T19:36:42.244161+0000 mon.a (mon.0) 1091 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:36:43.277 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:43 smithi016 bash[75004]: audit 2023-09-23T19:36:42.249905+0000 mon.a (mon.0) 1092 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:36:43.293 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:43.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:43.295 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:43.397 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:43.447 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:43.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:43.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:43.485 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 8 16 17871 17855 139.467 118.5 0.0136217 0.00716021 2023-09-23T19:36:43.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:43 smithi016 bash[77995]: cluster 2023-09-23T19:36:42.067325+0000 mgr.y (mgr.25325) 874 : cluster 0 pgmap v796: 89 pgs: 89 active+clean; 628 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T19:36:43.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:43 smithi016 bash[77995]: audit 2023-09-23T19:36:42.244161+0000 mon.a (mon.0) 1091 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:36:43.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:43 smithi016 bash[77995]: audit 2023-09-23T19:36:42.249905+0000 mon.a (mon.0) 1092 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:36:43.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:43 smithi092 bash[58855]: cluster 2023-09-23T19:36:42.067325+0000 mgr.y (mgr.25325) 874 : cluster 0 pgmap v796: 89 pgs: 89 active+clean; 628 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T19:36:43.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:43 smithi092 bash[58855]: audit 2023-09-23T19:36:42.244161+0000 mon.a (mon.0) 1091 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:36:43.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:43 smithi092 bash[58855]: audit 2023-09-23T19:36:42.249905+0000 mon.a (mon.0) 1092 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:36:43.554 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:43.576 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:43.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:43.579 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:36:43.680 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:43.712 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:43.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:43.718 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:43.819 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:43.858 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:43.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:43.861 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:43.962 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:43.991 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:43.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:43.995 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:44.096 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:44.125 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:44.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:44.128 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:36:44.228 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:44.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:44.261 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:44.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:44.362 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:44.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:44.395 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:44.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:44.485 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 9 16 19472 19456 135.088 100.062 0.00451398 0.0073959 2023-09-23T19:36:44.496 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:44.524 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:44.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:44.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:44.629 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:44.656 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:44.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:44.660 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:44.760 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:44.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:44 smithi092 bash[58855]: audit 2023-09-23T19:36:43.648205+0000 mon.a (mon.0) 1093 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:36:44.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:44 smithi092 bash[58855]: audit 2023-09-23T19:36:43.649238+0000 mon.a (mon.0) 1094 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:36:44.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:44 smithi092 bash[58855]: audit 2023-09-23T19:36:43.882107+0000 mon.a (mon.0) 1095 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:36:44.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:44 smithi092 bash[58855]: audit 2023-09-23T19:36:43.890103+0000 mon.a (mon.0) 1096 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:36:44.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:44 smithi092 bash[58855]: audit 2023-09-23T19:36:44.466671+0000 mon.a (mon.0) 1097 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:36:44.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:44 smithi092 bash[58855]: audit 2023-09-23T19:36:44.467813+0000 mon.a (mon.0) 1098 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:36:44.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:44 smithi092 bash[58855]: audit 2023-09-23T19:36:44.478022+0000 mon.a (mon.0) 1099 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:36:44.789 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:44.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:44.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:44.892 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:44.921 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:44.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:44.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:45.023 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:44 smithi016 bash[75004]: audit 2023-09-23T19:36:43.648205+0000 mon.a (mon.0) 1093 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:36:45.024 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:44 smithi016 bash[75004]: audit 2023-09-23T19:36:43.649238+0000 mon.a (mon.0) 1094 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:36:45.024 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:44 smithi016 bash[75004]: audit 2023-09-23T19:36:43.882107+0000 mon.a (mon.0) 1095 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:36:45.024 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:44 smithi016 bash[75004]: audit 2023-09-23T19:36:43.890103+0000 mon.a (mon.0) 1096 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:36:45.024 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:44 smithi016 bash[75004]: audit 2023-09-23T19:36:44.466671+0000 mon.a (mon.0) 1097 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:36:45.024 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:44 smithi016 bash[75004]: audit 2023-09-23T19:36:44.467813+0000 mon.a (mon.0) 1098 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:36:45.024 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:44 smithi016 bash[75004]: audit 2023-09-23T19:36:44.478022+0000 mon.a (mon.0) 1099 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:36:45.024 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:44 smithi016 bash[77995]: audit 2023-09-23T19:36:43.648205+0000 mon.a (mon.0) 1093 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:36:45.024 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:44 smithi016 bash[77995]: audit 2023-09-23T19:36:43.649238+0000 mon.a (mon.0) 1094 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:36:45.025 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:44 smithi016 bash[77995]: audit 2023-09-23T19:36:43.882107+0000 mon.a (mon.0) 1095 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:36:45.025 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:44 smithi016 bash[77995]: audit 2023-09-23T19:36:43.890103+0000 mon.a (mon.0) 1096 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:36:45.025 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:44 smithi016 bash[77995]: audit 2023-09-23T19:36:44.466671+0000 mon.a (mon.0) 1097 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:36:45.025 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:44 smithi016 bash[77995]: audit 2023-09-23T19:36:44.467813+0000 mon.a (mon.0) 1098 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:36:45.025 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:44 smithi016 bash[77995]: audit 2023-09-23T19:36:44.478022+0000 mon.a (mon.0) 1099 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:36:45.025 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:45.054 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:45.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:45.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:45.157 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:45.190 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:45.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:45.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:36:45.294 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:45.330 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:45.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:45.332 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:36:45.433 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:45.464 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:45.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:45.467 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:45.485 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 10 16 21347 21331 133.297 117.188 0.00826681 0.00749467 2023-09-23T19:36:45.568 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:45.597 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:45.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:45.600 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:36:45.701 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:45.728 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:45.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:45.733 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:36:45.786 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:36:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:36:45] "GET /metrics HTTP/1.1" 200 36289 "" "Prometheus/2.33.4" 2023-09-23T19:36:45.834 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:45.857 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:45.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:45.862 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:45.963 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:45.972 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:45 smithi092 bash[58855]: cluster 2023-09-23T19:36:44.067894+0000 mgr.y (mgr.25325) 875 : cluster 0 pgmap v797: 89 pgs: 89 active+clean; 790 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-09-23T19:36:45.989 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:45.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:45.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:46.098 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:46.134 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:46.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:46.137 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:36:46.238 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:46.246 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:45 smithi016 bash[75004]: cluster 2023-09-23T19:36:44.067894+0000 mgr.y (mgr.25325) 875 : cluster 0 pgmap v797: 89 pgs: 89 active+clean; 790 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-09-23T19:36:46.246 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:45 smithi016 bash[77995]: cluster 2023-09-23T19:36:44.067894+0000 mgr.y (mgr.25325) 875 : cluster 0 pgmap v797: 89 pgs: 89 active+clean; 790 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-09-23T19:36:46.265 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:46.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:46.268 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:36:46.369 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:46.398 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:46.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:46.401 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:46.485 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 11 16 23278 23262 132.15 120.688 0.00492014 0.00756007 2023-09-23T19:36:46.502 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:46.533 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:46.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:46.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:36:46.639 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:46.675 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:46.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:46.676 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:46.777 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:46.807 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:46.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:46.811 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:36:46.912 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:46.942 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:46.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:46.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:47.045 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:47.072 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:47.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:47.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:47.178 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:47.207 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:47.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:47.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:36:47.311 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:47.349 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:47.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:47.353 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:36:47.454 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:47.484 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:47.485 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 12 16 25152 25136 130.897 117.125 0.00451492 0.00763295 2023-09-23T19:36:47.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:47.486 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:47.587 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:47.616 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:47.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:47.620 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:36:47.721 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:47.750 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:47.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:47.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:36:47.853 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:47.879 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:47.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:47.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:47.982 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:48.009 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:48.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:48.012 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:48.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:47 smithi016 bash[77995]: cluster 2023-09-23T19:36:46.068579+0000 mgr.y (mgr.25325) 876 : cluster 0 pgmap v798: 89 pgs: 89 active+clean; 1.2 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-09-23T19:36:48.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:47 smithi016 bash[75004]: cluster 2023-09-23T19:36:46.068579+0000 mgr.y (mgr.25325) 876 : cluster 0 pgmap v798: 89 pgs: 89 active+clean; 1.2 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-09-23T19:36:48.114 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:48.124 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:47 smithi092 bash[58855]: cluster 2023-09-23T19:36:46.068579+0000 mgr.y (mgr.25325) 876 : cluster 0 pgmap v798: 89 pgs: 89 active+clean; 1.2 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-09-23T19:36:48.142 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:48.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:48.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:48.248 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:48.275 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:48.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:48.278 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:36:48.379 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:48.408 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:48.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:48.412 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:36:48.485 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 13 16 26907 26891 129.264 109.688 0.0103615 0.00772874 2023-09-23T19:36:48.513 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:48.538 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:48.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:48.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:48.641 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:48.668 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:48.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:48.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:48.772 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:48.795 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:48.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:48.797 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:36:48.898 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:48.923 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:48.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:48.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:36:49.031 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:49.067 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:49.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:49.069 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:49.170 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:49.198 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:49.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:49.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:49.302 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:49.327 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:49.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:49.329 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:49.430 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:49.460 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:49.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:49.462 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:49.485 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 14 16 28709 28693 128.075 112.625 0.00561393 0.00780198 2023-09-23T19:36:49.563 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:49.584 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:49.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:49.586 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:36:49.687 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:49.730 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:49.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:49.734 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:49.835 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:49.865 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:49.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:49.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:49.969 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:49.997 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:50.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:50.001 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:36:50.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:49 smithi092 bash[58855]: cluster 2023-09-23T19:36:48.069251+0000 mgr.y (mgr.25325) 877 : cluster 0 pgmap v799: 89 pgs: 89 active+clean; 1.3 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-09-23T19:36:50.101 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:50.134 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:50.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:50.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:50.237 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:50.265 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:50.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:50.269 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:36:50.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:49 smithi016 bash[75004]: cluster 2023-09-23T19:36:48.069251+0000 mgr.y (mgr.25325) 877 : cluster 0 pgmap v799: 89 pgs: 89 active+clean; 1.3 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-09-23T19:36:50.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:49 smithi016 bash[77995]: cluster 2023-09-23T19:36:48.069251+0000 mgr.y (mgr.25325) 877 : cluster 0 pgmap v799: 89 pgs: 89 active+clean; 1.3 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-09-23T19:36:50.369 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:50.402 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:50.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:50.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:36:50.485 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 15 16 30389 30373 126.536 105 0.00751741 0.00789652 2023-09-23T19:36:50.508 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:50.534 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:50.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:50.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:50.638 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:50.665 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:50.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:50.668 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:50.769 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:50.794 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:50.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:50.797 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:36:50.897 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:50.925 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:50.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:50.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:36:51.028 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:51.061 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:51.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:51.064 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:36:51.164 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:51.194 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:51.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:51.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:36:51.299 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:51.337 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:51.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:51.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:36:51.441 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:51.467 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:51.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:51.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:36:51.486 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 16 16 32094 32078 125.287 106.562 0.0164184 0.00797391 2023-09-23T19:36:51.573 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:51.604 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:51.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:51.606 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:51.707 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:51.732 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:51.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:51.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:36:51.841 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:51.866 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:51.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:51.869 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:36:51.970 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:51.978 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:51 smithi092 bash[58855]: cluster 2023-09-23T19:36:50.069881+0000 mgr.y (mgr.25325) 878 : cluster 0 pgmap v800: 89 pgs: 89 active+clean; 1.6 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2023-09-23T19:36:52.001 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:52.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:52.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:52.104 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:52.138 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:52.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:52.140 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:52.241 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:52.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:51 smithi016 bash[77995]: cluster 2023-09-23T19:36:50.069881+0000 mgr.y (mgr.25325) 878 : cluster 0 pgmap v800: 89 pgs: 89 active+clean; 1.6 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2023-09-23T19:36:52.250 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:51 smithi016 bash[75004]: cluster 2023-09-23T19:36:50.069881+0000 mgr.y (mgr.25325) 878 : cluster 0 pgmap v800: 89 pgs: 89 active+clean; 1.6 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2023-09-23T19:36:52.271 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:52.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:52.275 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:36:52.375 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:52.400 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:52.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:52.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:36:52.486 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 17 16 33771 33755 124.082 104.812 0.0119288 0.00805288 2023-09-23T19:36:52.505 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:52.541 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:52.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:52.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:52.645 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:52.677 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:52.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:52.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:36:52.780 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:52.806 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:52.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:52.808 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:36:52.910 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:52.940 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:52.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:52.944 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:53.045 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:53.064 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-09-23T19:36:53.064 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [ 2023-09-23T19:36:53.064 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:36:53.064 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45947.0:34667 10.6 e149/144 10:6d934f96:::benchmark_data_smithi016_136365_object34666:head v 149'2179, mlcod=149'2177)", 2023-09-23T19:36:53.065 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:36:53.059036+0000", 2023-09-23T19:36:53.065 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 0.0018108569999999999, 2023-09-23T19:36:53.065 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0018345740000000001, 2023-09-23T19:36:53.065 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:36:53.065 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "started", 2023-09-23T19:36:53.065 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:36:53.065 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:36:53.065 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:36:53.065 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:53.059036+0000", 2023-09-23T19:36:53.065 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:36:53.065 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:36:53.066 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:36:53.066 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:36:53.066 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:53.059036+0000", 2023-09-23T19:36:53.066 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:36:53.066 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:36:53.066 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:36:53.066 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:36:53.066 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:53.059035+0000", 2023-09-23T19:36:53.066 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:36:53.066 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:36:53.066 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:36:53.067 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:36:53.067 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:53.059133+0000", 2023-09-23T19:36:53.067 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.7844999999999996e-05 2023-09-23T19:36:53.067 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:36:53.067 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:36:53.067 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:36:53.067 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:53.059147+0000", 2023-09-23T19:36:53.067 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.4266e-05 2023-09-23T19:36:53.067 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:36:53.067 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:36:53.067 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:36:53.067 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:53.059153+0000", 2023-09-23T19:36:53.067 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.5560000000000004e-06 2023-09-23T19:36:53.068 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:36:53.068 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:36:53.068 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:36:53.068 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:53.059202+0000", 2023-09-23T19:36:53.068 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.8866000000000003e-05 2023-09-23T19:36:53.068 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:36:53.068 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:36:53.068 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:36:53.068 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:53.059229+0000", 2023-09-23T19:36:53.068 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.6529e-05 2023-09-23T19:36:53.068 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:36:53.068 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:36:53.069 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:36:53.069 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:36:53.069 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:36:53.069 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45947.0:34670 10.c e149/144 10:3e387ce4:::benchmark_data_smithi016_136365_object34669:head v 149'2213, mlcod=149'2211)", 2023-09-23T19:36:53.069 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:36:53.059044+0000", 2023-09-23T19:36:53.069 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 0.0018028580000000001, 2023-09-23T19:36:53.069 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001869512, 2023-09-23T19:36:53.069 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:36:53.069 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "started", 2023-09-23T19:36:53.069 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:36:53.069 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:36:53.069 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:36:53.070 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:53.059044+0000", 2023-09-23T19:36:53.070 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:36:53.070 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:36:53.070 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:36:53.070 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:36:53.070 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:53.059044+0000", 2023-09-23T19:36:53.070 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:36:53.070 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:36:53.070 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:36:53.070 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:36:53.070 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:53.059043+0000", 2023-09-23T19:36:53.071 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:36:53.071 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:36:53.071 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:36:53.071 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:36:53.071 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:53.059143+0000", 2023-09-23T19:36:53.071 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000100912 2023-09-23T19:36:53.071 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:36:53.071 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:36:53.071 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:36:53.071 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:53.059155+0000", 2023-09-23T19:36:53.071 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1389000000000001e-05 2023-09-23T19:36:53.072 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:36:53.072 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:36:53.072 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:36:53.072 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:53.059163+0000", 2023-09-23T19:36:53.072 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.123e-06 2023-09-23T19:36:53.072 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:36:53.072 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:36:53.072 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:36:53.072 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:53.059248+0000", 2023-09-23T19:36:53.072 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.4735999999999996e-05 2023-09-23T19:36:53.072 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:36:53.073 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:36:53.073 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:36:53.073 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:53.059269+0000", 2023-09-23T19:36:53.073 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.1708e-05 2023-09-23T19:36:53.073 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:36:53.073 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:36:53.073 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:36:53.073 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:36:53.073 INFO:teuthology.orchestra.run.smithi016.stdout: ], 2023-09-23T19:36:53.073 INFO:teuthology.orchestra.run.smithi016.stdout: "num_ops": 2 2023-09-23T19:36:53.073 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-09-23T19:36:53.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:53.074 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:53.075 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:53.175 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:53.206 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:53.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:53.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:36:53.310 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:53.331 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:53.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:53.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:36:53.435 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:53.466 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:53.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:53.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:36:53.486 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 18 16 35262 35246 122.365 93.1875 0.00374325 0.00816293 2023-09-23T19:36:53.574 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:53.613 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:53.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:53.617 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:53.718 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:53.746 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:53.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:53.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:36:53.853 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:53.890 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:53.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:53.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:36:53.939 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is set 2023-09-23T19:36:53.996 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:54.013 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:53 smithi016 bash[75004]: cluster 2023-09-23T19:36:52.070700+0000 mgr.y (mgr.25325) 879 : cluster 0 pgmap v801: 89 pgs: 89 active+clean; 1.7 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-09-23T19:36:54.013 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:53 smithi016 bash[75004]: audit 2023-09-23T19:36:53.791543+0000 mon.a (mon.0) 1100 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:36:54.013 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:53 smithi016 bash[75004]: audit 2023-09-23T19:36:53.796186+0000 mon.b (mon.1) 43 : audit 1 from='client.? 172.21.15.16:0/1871332182' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:36:54.013 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:53 smithi016 bash[77995]: cluster 2023-09-23T19:36:52.070700+0000 mgr.y (mgr.25325) 879 : cluster 0 pgmap v801: 89 pgs: 89 active+clean; 1.7 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-09-23T19:36:54.013 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:53 smithi016 bash[77995]: audit 2023-09-23T19:36:53.791543+0000 mon.a (mon.0) 1100 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:36:54.013 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:53 smithi016 bash[77995]: audit 2023-09-23T19:36:53.796186+0000 mon.b (mon.1) 43 : audit 1 from='client.? 172.21.15.16:0/1871332182' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:36:54.019 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:54.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:54.021 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:36:54.080 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:36:54.113 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:36:54.123 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:54.147 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:36:54.148 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:54.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:54.150 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:54.173 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:36:54.222 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:36:54.251 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:54.259 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:53 smithi092 bash[58855]: cluster 2023-09-23T19:36:52.070700+0000 mgr.y (mgr.25325) 879 : cluster 0 pgmap v801: 89 pgs: 89 active+clean; 1.7 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-09-23T19:36:54.260 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:53 smithi092 bash[58855]: audit 2023-09-23T19:36:53.791543+0000 mon.a (mon.0) 1100 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:36:54.260 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:53 smithi092 bash[58855]: audit 2023-09-23T19:36:53.796186+0000 mon.b (mon.1) 43 : audit 1 from='client.? 172.21.15.16:0/1871332182' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:36:54.267 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:36:54.277 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:54.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:54.280 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:54.300 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:36:54.328 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:36:54.332 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:36:54.332 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:36:54.333 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:36:54.333 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:36:54.333 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:36:54.333 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:36:54.333 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:36:54.333 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:36:54.381 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:54.408 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:54.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:54.413 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:54.486 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 19 16 36589 36573 120.29 82.9375 0.00719629 0.00830527 2023-09-23T19:36:54.515 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:54.541 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:54.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:54.543 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:54.644 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:54.673 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:54.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:54.677 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:36:54.778 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:54.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:54.808 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:54.808 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:54.909 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:54.915 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:54 smithi016 bash[75004]: cluster 2023-09-23T19:36:53.906791+0000 mon.a (mon.0) 1101 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:36:54.915 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:54 smithi016 bash[75004]: audit 2023-09-23T19:36:53.921994+0000 mon.a (mon.0) 1102 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:36:54.944 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:36:54.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:54.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:36:55.050 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:55.080 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:55.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:55.083 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:36:55.184 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:55.193 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:54 smithi016 bash[75004]: cluster 2023-09-23T19:36:53.922171+0000 mon.a (mon.0) 1103 : cluster 0 osdmap e150: 8 total, 8 up, 8 in 2023-09-23T19:36:55.193 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:54 smithi016 bash[75004]: cluster 2023-09-23T19:36:53.931021+0000 osd.6 (osd.6) 65 : cluster 3 failed to encode map e150 with expected crc 2023-09-23T19:36:55.193 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:54 smithi016 bash[75004]: cluster 2023-09-23T19:36:53.936832+0000 osd.5 (osd.5) 80 : cluster 3 failed to encode map e150 with expected crc 2023-09-23T19:36:55.193 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:54 smithi016 bash[75004]: audit 2023-09-23T19:36:53.985150+0000 mon.c (mon.2) 162 : audit 0 from='client.? 172.21.15.16:0/3423325311' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:36:55.194 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:54 smithi016 bash[75004]: audit 2023-09-23T19:36:54.500945+0000 mon.a (mon.0) 1104 : audit 1 from='client.? 172.21.15.16:0/784581600' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T19:36:55.194 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:54 smithi016 bash[77995]: cluster 2023-09-23T19:36:53.906791+0000 mon.a (mon.0) 1101 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:36:55.194 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:54 smithi016 bash[77995]: audit 2023-09-23T19:36:53.921994+0000 mon.a (mon.0) 1102 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:36:55.194 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:54 smithi016 bash[77995]: cluster 2023-09-23T19:36:53.922171+0000 mon.a (mon.0) 1103 : cluster 0 osdmap e150: 8 total, 8 up, 8 in 2023-09-23T19:36:55.195 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:54 smithi016 bash[77995]: cluster 2023-09-23T19:36:53.931021+0000 osd.6 (osd.6) 65 : cluster 3 failed to encode map e150 with expected crc 2023-09-23T19:36:55.195 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:54 smithi016 bash[77995]: cluster 2023-09-23T19:36:53.936832+0000 osd.5 (osd.5) 80 : cluster 3 failed to encode map e150 with expected crc 2023-09-23T19:36:55.195 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:54 smithi016 bash[77995]: audit 2023-09-23T19:36:53.985150+0000 mon.c (mon.2) 162 : audit 0 from='client.? 172.21.15.16:0/3423325311' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:36:55.195 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:54 smithi016 bash[77995]: audit 2023-09-23T19:36:54.500945+0000 mon.a (mon.0) 1104 : audit 1 from='client.? 172.21.15.16:0/784581600' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T19:36:55.213 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:55.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:55.217 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:55.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:54 smithi092 bash[58855]: cluster 2023-09-23T19:36:53.906791+0000 mon.a (mon.0) 1101 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:36:55.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:54 smithi092 bash[58855]: audit 2023-09-23T19:36:53.921994+0000 mon.a (mon.0) 1102 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:36:55.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:54 smithi092 bash[58855]: cluster 2023-09-23T19:36:53.922171+0000 mon.a (mon.0) 1103 : cluster 0 osdmap e150: 8 total, 8 up, 8 in 2023-09-23T19:36:55.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:54 smithi092 bash[58855]: cluster 2023-09-23T19:36:53.931021+0000 osd.6 (osd.6) 65 : cluster 3 failed to encode map e150 with expected crc 2023-09-23T19:36:55.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:54 smithi092 bash[58855]: cluster 2023-09-23T19:36:53.936832+0000 osd.5 (osd.5) 80 : cluster 3 failed to encode map e150 with expected crc 2023-09-23T19:36:55.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:54 smithi092 bash[58855]: audit 2023-09-23T19:36:53.985150+0000 mon.c (mon.2) 162 : audit 0 from='client.? 172.21.15.16:0/3423325311' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:36:55.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:54 smithi092 bash[58855]: audit 2023-09-23T19:36:54.500945+0000 mon.a (mon.0) 1104 : audit 1 from='client.? 172.21.15.16:0/784581600' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T19:36:55.318 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:55.344 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:36:55.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:55.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:36:55.400 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.7 to 1 (10000) 2023-09-23T19:36:55.447 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:55.481 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:55.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:55.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:36:55.486 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-09-23T19:36:55.483375+0000 min lat: 0.00156577 max lat: 0.17339 avg lat: 0.00837378 2023-09-23T19:36:55.486 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:36:55.486 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 20 16 38207 38191 119.331 101.125 0.00944979 0.00837378 2023-09-23T19:36:55.584 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:55.627 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:55.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:55.631 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:55.731 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:55.741 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:36:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:36:55] "GET /metrics HTTP/1.1" 200 36289 "" "Prometheus/2.33.4" 2023-09-23T19:36:55.756 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:55.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:55.758 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:36:55.859 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:55.885 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:55.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:55.889 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:55.989 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:56.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:55 smithi016 bash[75004]: cluster 2023-09-23T19:36:54.071184+0000 mgr.y (mgr.25325) 880 : cluster 0 pgmap v803: 89 pgs: 89 active+clean; 1.9 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-09-23T19:36:56.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:55 smithi016 bash[75004]: audit 2023-09-23T19:36:55.389269+0000 mon.a (mon.0) 1105 : audit 1 from='client.? 172.21.15.16:0/784581600' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T19:36:56.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:55 smithi016 bash[75004]: cluster 2023-09-23T19:36:55.389392+0000 mon.a (mon.0) 1106 : cluster 0 osdmap e151: 8 total, 8 up, 8 in 2023-09-23T19:36:56.001 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:55 smithi016 bash[75004]: cluster 2023-09-23T19:36:55.407284+0000 osd.7 (osd.7) 65 : cluster 3 failed to encode map e151 with expected crc 2023-09-23T19:36:56.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:55 smithi016 bash[77995]: cluster 2023-09-23T19:36:54.071184+0000 mgr.y (mgr.25325) 880 : cluster 0 pgmap v803: 89 pgs: 89 active+clean; 1.9 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-09-23T19:36:56.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:55 smithi016 bash[77995]: audit 2023-09-23T19:36:55.389269+0000 mon.a (mon.0) 1105 : audit 1 from='client.? 172.21.15.16:0/784581600' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T19:36:56.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:55 smithi016 bash[77995]: cluster 2023-09-23T19:36:55.389392+0000 mon.a (mon.0) 1106 : cluster 0 osdmap e151: 8 total, 8 up, 8 in 2023-09-23T19:36:56.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:55 smithi016 bash[77995]: cluster 2023-09-23T19:36:55.407284+0000 osd.7 (osd.7) 65 : cluster 3 failed to encode map e151 with expected crc 2023-09-23T19:36:56.026 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:56.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:56.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:56.132 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:56.165 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:56.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:56.169 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:56.269 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:56.282 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:55 smithi092 bash[58855]: cluster 2023-09-23T19:36:54.071184+0000 mgr.y (mgr.25325) 880 : cluster 0 pgmap v803: 89 pgs: 89 active+clean; 1.9 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-09-23T19:36:56.282 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:55 smithi092 bash[58855]: audit 2023-09-23T19:36:55.389269+0000 mon.a (mon.0) 1105 : audit 1 from='client.? 172.21.15.16:0/784581600' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T19:36:56.282 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:55 smithi092 bash[58855]: cluster 2023-09-23T19:36:55.389392+0000 mon.a (mon.0) 1106 : cluster 0 osdmap e151: 8 total, 8 up, 8 in 2023-09-23T19:36:56.282 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:55 smithi092 bash[58855]: cluster 2023-09-23T19:36:55.407284+0000 osd.7 (osd.7) 65 : cluster 3 failed to encode map e151 with expected crc 2023-09-23T19:36:56.303 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:56.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:56.310 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:56.335 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T19:36:56.410 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:56.438 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:56.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:56.446 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:56.486 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 21 16 39612 39596 117.83 87.8125 0.0134233 0.0084803 2023-09-23T19:36:56.547 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:56.573 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:56.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:56.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:56.678 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:56.706 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:56.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:56.712 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:36:56.813 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:56.840 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:56.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:56.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:56.943 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:56.975 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:56.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:56.977 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:57.078 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:57.087 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:56 smithi092 bash[58855]: cluster 2023-09-23T19:36:55.396497+0000 osd.6 (osd.6) 66 : cluster 3 failed to encode map e151 with expected crc 2023-09-23T19:36:57.087 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:56 smithi092 bash[58855]: cluster 2023-09-23T19:36:55.400031+0000 osd.4 (osd.4) 77 : cluster 3 failed to encode map e151 with expected crc 2023-09-23T19:36:57.107 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:57.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:57.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:57.213 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:57.222 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:56 smithi016 bash[75004]: cluster 2023-09-23T19:36:55.396497+0000 osd.6 (osd.6) 66 : cluster 3 failed to encode map e151 with expected crc 2023-09-23T19:36:57.222 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:56 smithi016 bash[75004]: cluster 2023-09-23T19:36:55.400031+0000 osd.4 (osd.4) 77 : cluster 3 failed to encode map e151 with expected crc 2023-09-23T19:36:57.223 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:56 smithi016 bash[77995]: cluster 2023-09-23T19:36:55.396497+0000 osd.6 (osd.6) 66 : cluster 3 failed to encode map e151 with expected crc 2023-09-23T19:36:57.223 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:56 smithi016 bash[77995]: cluster 2023-09-23T19:36:55.400031+0000 osd.4 (osd.4) 77 : cluster 3 failed to encode map e151 with expected crc 2023-09-23T19:36:57.242 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:57.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:57.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:36:57.346 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:57.371 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:57.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:57.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:57.478 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:57.486 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 22 16 41161 41145 116.874 96.8125 0.0147157 0.00855001 2023-09-23T19:36:57.508 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:36:57.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:57.510 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:36:57.611 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:57.645 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:57.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:57.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:57.753 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:57.783 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:57.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:57.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:57.887 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:57.913 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:57.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:57.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:36:57.941 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:57 smithi092 bash[58855]: cluster 2023-09-23T19:36:56.071742+0000 mgr.y (mgr.25325) 881 : cluster 0 pgmap v805: 89 pgs: 89 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-09-23T19:36:58.016 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:58.047 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:58.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:58.052 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:36:58.153 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:58.161 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:57 smithi016 bash[75004]: cluster 2023-09-23T19:36:56.071742+0000 mgr.y (mgr.25325) 881 : cluster 0 pgmap v805: 89 pgs: 89 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-09-23T19:36:58.161 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:57 smithi016 bash[77995]: cluster 2023-09-23T19:36:56.071742+0000 mgr.y (mgr.25325) 881 : cluster 0 pgmap v805: 89 pgs: 89 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-09-23T19:36:58.180 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:58.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:58.184 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:36:58.285 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:58.306 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:58.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:58.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:58.407 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T19:36:58.407 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T19:36:58.409 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:58.436 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:58.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:58.441 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:58.486 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 23 16 42695 42679 115.961 95.875 0.012527 0.00861816 2023-09-23T19:36:58.542 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:58.589 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:58.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:58.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:36:58.694 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:58.718 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:58.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:58.720 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:58.821 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:58.851 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:58.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:58.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:58.955 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:58.979 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:58.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:58.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:36:59.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:58 smithi092 bash[58855]: audit 2023-09-23T19:36:58.650916+0000 mon.a (mon.0) 1107 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:36:59.082 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:59.111 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:36:59.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:59.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:36:59.214 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:59.225 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:58 smithi016 bash[75004]: audit 2023-09-23T19:36:58.650916+0000 mon.a (mon.0) 1107 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:36:59.226 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:58 smithi016 bash[77995]: audit 2023-09-23T19:36:58.650916+0000 mon.a (mon.0) 1107 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:36:59.244 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:36:59.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:59.247 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:36:59.347 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:59.375 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:59.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:59.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:59.478 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:59.486 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 24 16 44097 44081 114.78 87.625 0.00913399 0.00870588 2023-09-23T19:36:59.509 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:36:59.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:59.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:36:59.613 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:59.642 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:59.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:59.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:36:59.746 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:59.773 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:36:59.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:59.778 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:36:59.880 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:59.910 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:36:59.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:36:59.913 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:37:00.014 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:00.041 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:00.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:00.043 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:00.144 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:00.182 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:00.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:00.184 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:37:00.284 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T19:37:00.285 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:00.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:36:59 smithi016 bash[77995]: cluster 2023-09-23T19:36:58.072426+0000 mgr.y (mgr.25325) 882 : cluster 0 pgmap v806: 89 pgs: 89 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-09-23T19:37:00.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:36:59 smithi016 bash[75004]: cluster 2023-09-23T19:36:58.072426+0000 mgr.y (mgr.25325) 882 : cluster 0 pgmap v806: 89 pgs: 89 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-09-23T19:37:00.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:36:59 smithi092 bash[58855]: cluster 2023-09-23T19:36:58.072426+0000 mgr.y (mgr.25325) 882 : cluster 0 pgmap v806: 89 pgs: 89 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-09-23T19:37:00.328 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:00.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:00.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:37:00.389 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:37:00.432 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:00.482 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:00.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:00.486 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:00.486 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 25 16 45513 45497 113.728 88.5 0.0247804 0.00878617 2023-09-23T19:37:00.588 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:00.614 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:00.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:00.618 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:00.719 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:00.748 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:00.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:00.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:37:00.854 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:00.883 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:00.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:00.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:00.988 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:01.024 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:01.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:01.030 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:37:01.131 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:01.166 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:01.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:01.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:01.176 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:37:01.270 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:01.305 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:01.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:01.309 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:01.410 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:01.437 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:01.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:01.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:37:01.487 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 26 16 46877 46861 112.633 85.25 0.00593207 0.00887204 2023-09-23T19:37:01.540 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:01.563 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:01.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:01.566 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:01.667 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:01.701 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:01.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:01.703 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:37:01.804 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:01.833 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:01.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:01.836 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:01.937 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:01.968 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:01.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:01.970 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:02.071 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:02.081 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:01 smithi092 bash[58855]: cluster 2023-09-23T19:37:00.073044+0000 mgr.y (mgr.25325) 883 : cluster 0 pgmap v807: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2023-09-23T19:37:02.106 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:02.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:02.110 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:37:02.210 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:02.235 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:02.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:02.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:37:02.279 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:02 smithi016 bash[75004]: cluster 2023-09-23T19:37:00.073044+0000 mgr.y (mgr.25325) 883 : cluster 0 pgmap v807: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2023-09-23T19:37:02.279 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:02 smithi016 bash[77995]: cluster 2023-09-23T19:37:00.073044+0000 mgr.y (mgr.25325) 883 : cluster 0 pgmap v807: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2023-09-23T19:37:02.339 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:02.365 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:02.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:02.368 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:02.469 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:02.487 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 27 16 48289 48273 111.729 88.25 0.00570253 0.00894422 2023-09-23T19:37:02.497 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:02.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:02.500 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:02.600 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:02.627 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:02.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:02.630 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:37:02.731 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:02.756 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:02.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:02.762 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:02.863 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:02.889 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:02.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:02.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:02.994 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:03.023 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:03.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:03.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:03.129 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:03.165 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:03.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:03.171 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:37:03.223 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:37:03.272 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:03.302 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:03.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:03.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:03.406 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:03.432 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:03.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:03.437 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:03.488 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 28 16 49605 49589 110.676 82.25 0.0124219 0.00902973 2023-09-23T19:37:03.537 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:03.577 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:03.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:03.580 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:37:03.681 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:03.717 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:03.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:03.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:03.821 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:03.846 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:03.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:03.850 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:37:03.950 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:03.981 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:03.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:03.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:37:04.085 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:04.128 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:04.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:04.133 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:04.234 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:04.242 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:37:04.266 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:04.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:04.270 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:04.278 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:04 smithi016 bash[75004]: cluster 2023-09-23T19:37:02.077043+0000 mgr.y (mgr.25325) 884 : cluster 0 pgmap v808: 89 pgs: 89 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-09-23T19:37:04.278 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:04 smithi016 bash[77995]: cluster 2023-09-23T19:37:02.077043+0000 mgr.y (mgr.25325) 884 : cluster 0 pgmap v808: 89 pgs: 89 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-09-23T19:37:04.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:03 smithi092 bash[58855]: cluster 2023-09-23T19:37:02.077043+0000 mgr.y (mgr.25325) 884 : cluster 0 pgmap v808: 89 pgs: 89 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-09-23T19:37:04.370 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:04.402 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:04.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:04.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:04.487 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 29 16 50732 50716 109.289 70.4375 0.0123062 0.00914469 2023-09-23T19:37:04.505 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:04.535 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:04.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:04.540 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:04.640 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:04.674 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:04.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:04.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:04.776 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:04.813 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:04.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:04.818 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:04.918 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:04.949 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:04.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:04.952 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:37:05.053 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:05.079 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:05.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:05.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:05.184 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:05.209 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:05.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:05.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:37:05.314 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:05.340 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:05.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:05.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:37:05.445 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:05.475 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:05.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:05.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:05.487 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 30 15 52017 52002 108.324 80.375 0.0157366 0.00922614 2023-09-23T19:37:05.579 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:05.607 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:05.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:05.609 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:37:05.710 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:05.744 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:05.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:05.748 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:05.786 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:37:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:37:05] "GET /metrics HTTP/1.1" 200 36289 "" "Prometheus/2.33.4" 2023-09-23T19:37:05.848 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:05.886 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:05.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:05.897 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:05.966 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:05 smithi092 bash[58855]: cluster 2023-09-23T19:37:04.077648+0000 mgr.y (mgr.25325) 885 : cluster 0 pgmap v809: 89 pgs: 89 active+clean; 2.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2023-09-23T19:37:05.998 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:06.024 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:06.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:06.031 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:06.132 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:06.166 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:06.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:06.169 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:06.249 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:05 smithi016 bash[77995]: cluster 2023-09-23T19:37:04.077648+0000 mgr.y (mgr.25325) 885 : cluster 0 pgmap v809: 89 pgs: 89 active+clean; 2.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2023-09-23T19:37:06.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:05 smithi016 bash[75004]: cluster 2023-09-23T19:37:04.077648+0000 mgr.y (mgr.25325) 885 : cluster 0 pgmap v809: 89 pgs: 89 active+clean; 2.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2023-09-23T19:37:06.269 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:06.295 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:06.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:06.298 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:37:06.399 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:06.427 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:06.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:06.430 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:37:06.487 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 31 16 53359 53343 107.534 83.8125 0.00848962 0.00929348 2023-09-23T19:37:06.531 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:06.570 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:06.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:06.573 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:06.674 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:06.708 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:06.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:06.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:06.815 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:06.847 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:06.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:06.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:06.951 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:06.984 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:06.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:06.987 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:37:07.087 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:07.116 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:07.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:07.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:07.220 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:07.247 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:07.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:07.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:07.351 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:07.378 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:07.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:07.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:07.482 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:07.487 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 32 16 54678 54662 106.749 82.4375 0.00420773 0.00936051 2023-09-23T19:37:07.510 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:07.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:07.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:07.615 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:07.650 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:07.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:07.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:37:07.755 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:07.783 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:07.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:07.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:37:07.886 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:07.912 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:07.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:07.916 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:37:08.017 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:08.026 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:07 smithi092 bash[58855]: cluster 2023-09-23T19:37:06.078347+0000 mgr.y (mgr.25325) 886 : cluster 0 pgmap v810: 89 pgs: 89 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-09-23T19:37:08.042 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:08.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:08.046 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:08.070 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:37:08.146 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:08.165 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:08.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:08.168 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:08.178 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:07 smithi016 bash[77995]: cluster 2023-09-23T19:37:06.078347+0000 mgr.y (mgr.25325) 886 : cluster 0 pgmap v810: 89 pgs: 89 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-09-23T19:37:08.178 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:07 smithi016 bash[75004]: cluster 2023-09-23T19:37:06.078347+0000 mgr.y (mgr.25325) 886 : cluster 0 pgmap v810: 89 pgs: 89 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-09-23T19:37:08.270 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:08.300 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:08.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:08.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:08.406 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:08.436 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:08.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:08.439 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:08.487 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 33 16 55853 55837 105.739 73.4375 0.00826212 0.00945073 2023-09-23T19:37:08.540 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:08.572 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:08.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:08.574 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:37:08.675 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:08.703 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:08.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:08.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:08.808 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:08.845 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:08.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:08.848 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:08.949 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:08.977 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:08.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:08.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:09.083 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:09.120 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:09.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:09.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:09.224 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:09.249 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:09.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:09.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:09.353 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:09.387 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:09.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:09.390 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:37:09.487 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 34 16 56985 56969 104.71 70.75 0.00820174 0.00954374 2023-09-23T19:37:09.491 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:09.520 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:09.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:09.523 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:37:09.624 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:09.650 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:09.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:09.654 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:09.755 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:09.781 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:09.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:09.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:37:09.884 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:09.909 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:09.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:09.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:37:10.015 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:10.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:09 smithi016 bash[75004]: cluster 2023-09-23T19:37:08.078911+0000 mgr.y (mgr.25325) 887 : cluster 0 pgmap v811: 89 pgs: 89 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s 2023-09-23T19:37:10.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:09 smithi016 bash[77995]: cluster 2023-09-23T19:37:08.078911+0000 mgr.y (mgr.25325) 887 : cluster 0 pgmap v811: 89 pgs: 89 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s 2023-09-23T19:37:10.050 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:10.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:10.054 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:37:10.155 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:10.195 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:10.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:10.202 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:10.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:09 smithi092 bash[58855]: cluster 2023-09-23T19:37:08.078911+0000 mgr.y (mgr.25325) 887 : cluster 0 pgmap v811: 89 pgs: 89 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s 2023-09-23T19:37:10.304 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:10.329 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:10.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:10.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:37:10.432 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:10.458 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:10.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:10.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:37:10.488 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 35 16 58150 58134 103.799 72.8125 0.0164938 0.00962805 2023-09-23T19:37:10.561 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:10.586 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:10.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:10.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:37:10.689 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:10.722 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:10.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:10.725 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:37:10.826 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:10.859 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:10.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:10.866 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:10.966 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:10.990 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:10.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:10.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:37:11.095 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:11.121 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:11.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:11.123 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:11.224 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:11.264 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:11.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:11.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:37:11.371 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:11.410 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:11.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:11.413 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:37:11.488 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 36 16 59372 59356 103.037 76.375 0.0166192 0.00969952 2023-09-23T19:37:11.513 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:11.542 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:11.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:11.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:11.646 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:11.677 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:11.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:11.679 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:11.780 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:11.828 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:11.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:11.833 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:11.934 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:11.960 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:11.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:11.965 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:37:11.981 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:11 smithi092 bash[58855]: cluster 2023-09-23T19:37:10.079555+0000 mgr.y (mgr.25325) 888 : cluster 0 pgmap v812: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 88 MiB/s wr, 1.42k op/s 2023-09-23T19:37:12.066 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:12.090 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:12.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:12.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:37:12.193 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:12.203 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:12 smithi016 bash[75004]: cluster 2023-09-23T19:37:10.079555+0000 mgr.y (mgr.25325) 888 : cluster 0 pgmap v812: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 88 MiB/s wr, 1.42k op/s 2023-09-23T19:37:12.203 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:12 smithi016 bash[77995]: cluster 2023-09-23T19:37:10.079555+0000 mgr.y (mgr.25325) 888 : cluster 0 pgmap v812: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 88 MiB/s wr, 1.42k op/s 2023-09-23T19:37:12.223 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:12.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:12.224 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:12.325 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:12.350 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:12.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:12.353 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:37:12.453 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:12.488 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 37 16 60582 60566 102.296 75.625 0.014978 0.00976804 2023-09-23T19:37:12.489 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:12.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:12.492 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:37:12.592 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:12.621 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:12.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:12.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:12.725 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:12.760 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:12.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:12.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:37:12.871 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:12.884 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-09-23T19:37:12.884 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:37:12.910 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:12.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:12.916 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:13.017 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:13.050 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:13.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:13.056 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:13.156 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:13.185 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:13.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:13.189 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:13.289 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:13.318 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:13.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:13.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:13.421 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:13.447 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:13.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:13.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:13.488 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 38 16 61694 61678 101.432 69.5 0.00843014 0.00984795 2023-09-23T19:37:13.551 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:13.585 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:13.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:13.588 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:37:13.689 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:13.715 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:13.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:13.720 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:37:13.821 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:13.852 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:13.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:13.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:37:13.955 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:13.988 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:13.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:13.994 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:14.089 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:13 smithi016 bash[75004]: cluster 2023-09-23T19:37:12.080192+0000 mgr.y (mgr.25325) 889 : cluster 0 pgmap v813: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-09-23T19:37:14.090 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:13 smithi016 bash[75004]: audit 2023-09-23T19:37:13.651460+0000 mon.a (mon.0) 1108 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:37:14.090 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:13 smithi016 bash[77995]: cluster 2023-09-23T19:37:12.080192+0000 mgr.y (mgr.25325) 889 : cluster 0 pgmap v813: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-09-23T19:37:14.090 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:13 smithi016 bash[77995]: audit 2023-09-23T19:37:13.651460+0000 mon.a (mon.0) 1108 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:37:14.094 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:14.124 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:14.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:14.130 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:37:14.230 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:14.240 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:13 smithi092 bash[58855]: cluster 2023-09-23T19:37:12.080192+0000 mgr.y (mgr.25325) 889 : cluster 0 pgmap v813: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-09-23T19:37:14.240 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:13 smithi092 bash[58855]: audit 2023-09-23T19:37:13.651460+0000 mon.a (mon.0) 1108 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:37:14.257 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:14.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:14.260 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:14.361 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:14.389 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:14.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:14.392 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:14.489 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 39 16 62850 62834 100.684 72.25 0.00597455 0.00992554 2023-09-23T19:37:14.494 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:14.519 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:14.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:14.524 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:37:14.625 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:14.649 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:14.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:14.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:14.756 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:14.783 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:14.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:14.785 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:14.886 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:14.915 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:14.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:14.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:15.018 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:15.045 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:15.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:15.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:37:15.148 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:15.178 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:15.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:15.181 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:15.281 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:15.307 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:15.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:15.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:37:15.410 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:15.437 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:15.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:15.439 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:15.488 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-09-23T19:37:15.485320+0000 min lat: 0.00156577 max lat: 0.17339 avg lat: 0.0100128 2023-09-23T19:37:15.488 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:37:15.488 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 40 16 63905 63889 99.8151 65.9375 0.0045867 0.0100128 2023-09-23T19:37:15.540 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:15.566 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:15.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:15.569 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:37:15.670 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:15.706 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:15.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:15.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:15.813 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:15.844 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:15.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:15.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:15.949 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:15.985 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:15.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:15.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:15.995 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:37:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:37:15] "GET /metrics HTTP/1.1" 200 36292 "" "Prometheus/2.33.4" 2023-09-23T19:37:16.090 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:16.116 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:16.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:16.120 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:16.221 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:16.253 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:16.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:16.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:37:16.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:15 smithi092 bash[58855]: cluster 2023-09-23T19:37:14.080782+0000 mgr.y (mgr.25325) 890 : cluster 0 pgmap v814: 89 pgs: 89 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-09-23T19:37:16.347 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:15 smithi016 bash[77995]: cluster 2023-09-23T19:37:14.080782+0000 mgr.y (mgr.25325) 890 : cluster 0 pgmap v814: 89 pgs: 89 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-09-23T19:37:16.347 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:15 smithi016 bash[75004]: cluster 2023-09-23T19:37:14.080782+0000 mgr.y (mgr.25325) 890 : cluster 0 pgmap v814: 89 pgs: 89 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-09-23T19:37:16.359 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:16.387 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:16.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:16.392 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:16.488 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 41 16 65116 65100 99.2264 75.6875 0.022385 0.0100723 2023-09-23T19:37:16.493 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:16.523 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:16.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:16.528 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:16.629 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:16.653 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:16.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:16.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:16.758 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:16.799 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:16.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:16.805 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:37:16.906 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:16.931 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:16.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:16.934 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:17.034 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:17.065 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:17.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:17.071 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:37:17.171 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:17.206 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:17.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:17.209 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:37:17.211 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T19:37:17.310 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:17.337 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:17.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:17.341 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:17.442 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:17.472 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:17.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:17.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:17.488 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 42 16 66248 66232 98.5483 70.75 0.0123332 0.0101421 2023-09-23T19:37:17.576 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:17.613 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:17.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:17.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:37:17.719 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:17.746 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:17.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:17.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:17.854 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:17.887 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:17.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:17.891 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:17.992 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:18.024 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:18.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:18.030 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:37:18.131 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:18.161 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:18.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:18.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:18.266 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:18.275 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:17 smithi016 bash[77995]: cluster 2023-09-23T19:37:16.081434+0000 mgr.y (mgr.25325) 891 : cluster 0 pgmap v815: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-09-23T19:37:18.275 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:17 smithi016 bash[75004]: cluster 2023-09-23T19:37:16.081434+0000 mgr.y (mgr.25325) 891 : cluster 0 pgmap v815: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-09-23T19:37:18.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:17 smithi092 bash[58855]: cluster 2023-09-23T19:37:16.081434+0000 mgr.y (mgr.25325) 891 : cluster 0 pgmap v815: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-09-23T19:37:18.297 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:18.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:18.301 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:37:18.402 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:18.423 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:18.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:18.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:18.488 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 43 16 67294 67278 97.7767 65.375 0.00721028 0.0102213 2023-09-23T19:37:18.526 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:18.555 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:18.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:18.560 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:18.661 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:18.690 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:18.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:18.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:18.796 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:18.830 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:18.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:18.833 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:18.934 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:18.969 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:18.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:18.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:19.073 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:19.093 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:19.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:19.095 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:37:19.196 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:19.228 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:19.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:19.233 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:37:19.334 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:19.358 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:19.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:19.365 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:37:19.466 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:19.488 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 44 16 68337 68321 97.0359 65.1875 0.0080758 0.0102988 2023-09-23T19:37:19.530 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:19.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:19.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:37:19.635 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:19.665 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:19.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:19.668 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:19.768 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:19.795 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:19.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:19.798 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:19.898 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:19.940 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:19.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:19.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:37:19.973 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-09-23T19:37:19.978 INFO:teuthology.orchestra.run.smithi016.stderr:dumped all 2023-09-23T19:37:20.047 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:20.076 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:20.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:20.081 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:20.113 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:37:20.156 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:37:20.182 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:20.192 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:19 smithi016 bash[75004]: cluster 2023-09-23T19:37:18.082020+0000 mgr.y (mgr.25325) 892 : cluster 0 pgmap v816: 89 pgs: 89 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-09-23T19:37:20.192 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:19 smithi016 bash[75004]: audit 2023-09-23T19:37:19.450016+0000 mon.c (mon.2) 163 : audit 1 from='client.? 172.21.15.16:0/2466600652' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:37:20.192 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:19 smithi016 bash[75004]: audit 2023-09-23T19:37:19.450853+0000 mon.a (mon.0) 1109 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:37:20.192 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:20 smithi016 bash[77995]: cluster 2023-09-23T19:37:18.082020+0000 mgr.y (mgr.25325) 892 : cluster 0 pgmap v816: 89 pgs: 89 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-09-23T19:37:20.193 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:20 smithi016 bash[77995]: audit 2023-09-23T19:37:19.450016+0000 mon.c (mon.2) 163 : audit 1 from='client.? 172.21.15.16:0/2466600652' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:37:20.193 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:20 smithi016 bash[77995]: audit 2023-09-23T19:37:19.450853+0000 mon.a (mon.0) 1109 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:37:20.213 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:20.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:20.218 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:37:20.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:37:20.262 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:37:20.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:20 smithi092 bash[58855]: cluster 2023-09-23T19:37:18.082020+0000 mgr.y (mgr.25325) 892 : cluster 0 pgmap v816: 89 pgs: 89 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-09-23T19:37:20.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:20 smithi092 bash[58855]: audit 2023-09-23T19:37:19.450016+0000 mon.c (mon.2) 163 : audit 1 from='client.? 172.21.15.16:0/2466600652' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:37:20.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:20 smithi092 bash[58855]: audit 2023-09-23T19:37:19.450853+0000 mon.a (mon.0) 1109 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:37:20.304 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:37:20.320 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:20.347 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:37:20.358 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:20.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:20.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:20.384 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:37:20.413 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is unset 2023-09-23T19:37:20.425 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:37:20.431 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:37:20.431 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:37:20.431 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:37:20.431 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:37:20.432 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:37:20.432 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:37:20.432 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:37:20.432 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:37:20.462 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:20.485 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:20.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:20.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:37:20.488 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 45 16 69258 69242 96.1585 57.5625 0.00475977 0.0103818 2023-09-23T19:37:20.588 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:20.616 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:20.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:20.620 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:37:20.720 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:20.752 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:20.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:20.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:37:20.856 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:20.892 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:20.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:20.897 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:20.997 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:21.028 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:21.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:21.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:21.131 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:21.142 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:21 smithi092 bash[58855]: audit 2023-09-23T19:37:19.941403+0000 mgr.y (mgr.25325) 893 : audit 0 from='client.55558 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:37:21.143 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:21 smithi092 bash[58855]: audit 2023-09-23T19:37:20.070096+0000 mon.a (mon.0) 1110 : audit 0 from='client.? 172.21.15.16:0/3038090842' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:37:21.143 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:21 smithi092 bash[58855]: cluster 2023-09-23T19:37:20.394777+0000 mon.a (mon.0) 1111 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:37:21.143 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:21 smithi092 bash[58855]: audit 2023-09-23T19:37:20.400417+0000 mon.a (mon.0) 1112 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:37:21.143 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:21 smithi092 bash[58855]: cluster 2023-09-23T19:37:20.400525+0000 mon.a (mon.0) 1113 : cluster 0 osdmap e152: 8 total, 8 up, 8 in 2023-09-23T19:37:21.143 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:21 smithi092 bash[58855]: cluster 2023-09-23T19:37:20.404068+0000 osd.7 (osd.7) 66 : cluster 3 failed to encode map e152 with expected crc 2023-09-23T19:37:21.162 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:21.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:21.165 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:37:21.266 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:21.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:21 smithi016 bash[77995]: audit 2023-09-23T19:37:19.941403+0000 mgr.y (mgr.25325) 893 : audit 0 from='client.55558 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:37:21.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:21 smithi016 bash[77995]: audit 2023-09-23T19:37:20.070096+0000 mon.a (mon.0) 1110 : audit 0 from='client.? 172.21.15.16:0/3038090842' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:37:21.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:21 smithi016 bash[77995]: cluster 2023-09-23T19:37:20.394777+0000 mon.a (mon.0) 1111 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:37:21.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:21 smithi016 bash[77995]: audit 2023-09-23T19:37:20.400417+0000 mon.a (mon.0) 1112 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:37:21.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:21 smithi016 bash[77995]: cluster 2023-09-23T19:37:20.400525+0000 mon.a (mon.0) 1113 : cluster 0 osdmap e152: 8 total, 8 up, 8 in 2023-09-23T19:37:21.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:21 smithi016 bash[77995]: cluster 2023-09-23T19:37:20.404068+0000 osd.7 (osd.7) 66 : cluster 3 failed to encode map e152 with expected crc 2023-09-23T19:37:21.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:21 smithi016 bash[75004]: audit 2023-09-23T19:37:19.941403+0000 mgr.y (mgr.25325) 893 : audit 0 from='client.55558 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:37:21.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:21 smithi016 bash[75004]: audit 2023-09-23T19:37:20.070096+0000 mon.a (mon.0) 1110 : audit 0 from='client.? 172.21.15.16:0/3038090842' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:37:21.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:21 smithi016 bash[75004]: cluster 2023-09-23T19:37:20.394777+0000 mon.a (mon.0) 1111 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:37:21.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:21 smithi016 bash[75004]: audit 2023-09-23T19:37:20.400417+0000 mon.a (mon.0) 1112 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:37:21.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:21 smithi016 bash[75004]: cluster 2023-09-23T19:37:20.400525+0000 mon.a (mon.0) 1113 : cluster 0 osdmap e152: 8 total, 8 up, 8 in 2023-09-23T19:37:21.288 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:21 smithi016 bash[75004]: cluster 2023-09-23T19:37:20.404068+0000 osd.7 (osd.7) 66 : cluster 3 failed to encode map e152 with expected crc 2023-09-23T19:37:21.302 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:21.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:21.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:21.408 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:21.435 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:21.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:21.443 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:21.489 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 46 16 70272 70256 95.4457 63.375 0.00741637 0.0104713 2023-09-23T19:37:21.543 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:21.583 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:21.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:21.585 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:37:21.599 INFO:teuthology.orchestra.run.smithi016.stdout:{"pg_ready":true,"pg_map":{"version":816,"stamp":"2023-09-23T19:37:18.081610+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4209055648,"num_objects":64240,"num_object_clones":0,"num_object_copies":192720,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":64240,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":64477,"num_write_kb":4112270,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":64411,"ondisk_log_size":64411,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":16707380,"kb_used_data":12358004,"kb_used_omap":20,"kb_used_meta":4349291,"kb_avail":733253836,"statfs":{"total":767960285184,"available":750851928064,"internally_reserved":0,"allocated":12654596096,"data_stored":12650242561,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":4453674638},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":910950400,"num_objects":13900,"num_object_clones":0,"num_object_copies":41700,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13900,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13900,"num_write_kb":889600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003747"},"pg_stats":[{"pgid":"3.1a","version":"59'1","reported_seq":132,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.940038+0000","last_change":"2023-09-23T19:27:24.274696+0000","last_active":"2023-09-23T19:36:53.940038+0000","last_peered":"2023-09-23T19:36:53.940038+0000","last_clean":"2023-09-23T19:36:53.940038+0000","last_became_active":"2023-09-23T19:27:24.273561+0000","last_became_peered":"2023-09-23T19:27:24.273561+0000","last_unstale":"2023-09-23T19:36:53.940038+0000","last_undegraded":"2023-09-23T19:36:53.940038+0000","last_fullsized":"2023-09-23T19:36:53.940038+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":105,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.923239+0000","last_change":"2023-09-23T19:32:49.731128+0000","last_active":"2023-09-23T19:36:53.923239+0000","last_peered":"2023-09-23T19:36:53.923239+0000","last_clean":"2023-09-23T19:36:53.923239+0000","last_became_active":"2023-09-23T19:32:49.730757+0000","last_became_peered":"2023-09-23T19:32:49.730757+0000","last_unstale":"2023-09-23T19:36:53.923239+0000","last_undegraded":"2023-09-23T19:36:53.923239+0000","last_fullsized":"2023-09-23T19:36:53.923239+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:40.183605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":94,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.389023+0000","last_change":"2023-09-23T19:28:07.890462+0000","last_active":"2023-09-23T19:36:55.389023+0000","last_peered":"2023-09-23T19:36:55.389023+0000","last_clean":"2023-09-23T19:36:55.389023+0000","last_became_active":"2023-09-23T19:28:07.890084+0000","last_became_peered":"2023-09-23T19:28:07.890084+0000","last_unstale":"2023-09-23T19:36:55.389023+0000","last_undegraded":"2023-09-23T19:36:55.389023+0000","last_fullsized":"2023-09-23T19:36:55.389023+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:50.458218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":111,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.945382+0000","last_change":"2023-09-23T19:28:07.884498+0000","last_active":"2023-09-23T19:36:53.945382+0000","last_peered":"2023-09-23T19:36:53.945382+0000","last_clean":"2023-09-23T19:36:53.945382+0000","last_became_active":"2023-09-23T19:28:07.884102+0000","last_became_peered":"2023-09-23T19:28:07.884102+0000","last_unstale":"2023-09-23T19:36:53.945382+0000","last_undegraded":"2023-09-23T19:36:53.945382+0000","last_fullsized":"2023-09-23T19:36:53.945382+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:05:26.348041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":122,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.941253+0000","last_change":"2023-09-23T19:32:49.729961+0000","last_active":"2023-09-23T19:36:53.941253+0000","last_peered":"2023-09-23T19:36:53.941253+0000","last_clean":"2023-09-23T19:36:53.941253+0000","last_became_active":"2023-09-23T19:32:49.729251+0000","last_became_peered":"2023-09-23T19:32:49.729251+0000","last_unstale":"2023-09-23T19:36:53.941253+0000","last_undegraded":"2023-09-23T19:36:53.941253+0000","last_fullsized":"2023-09-23T19:36:53.941253+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:20:56.595732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":116,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.938184+0000","last_change":"2023-09-23T19:22:42.150582+0000","last_active":"2023-09-23T19:36:53.938184+0000","last_peered":"2023-09-23T19:36:53.938184+0000","last_clean":"2023-09-23T19:36:53.938184+0000","last_became_active":"2023-09-23T19:22:42.150229+0000","last_became_peered":"2023-09-23T19:22:42.150229+0000","last_unstale":"2023-09-23T19:36:53.938184+0000","last_undegraded":"2023-09-23T19:36:53.938184+0000","last_fullsized":"2023-09-23T19:36:53.938184+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":109,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.945745+0000","last_change":"2023-09-23T19:28:07.887141+0000","last_active":"2023-09-23T19:36:53.945745+0000","last_peered":"2023-09-23T19:36:53.945745+0000","last_clean":"2023-09-23T19:36:53.945745+0000","last_became_active":"2023-09-23T19:28:07.886494+0000","last_became_peered":"2023-09-23T19:28:07.886494+0000","last_unstale":"2023-09-23T19:36:53.945745+0000","last_undegraded":"2023-09-23T19:36:53.945745+0000","last_fullsized":"2023-09-23T19:36:53.945745+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:24:07.615051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":105,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.924498+0000","last_change":"2023-09-23T19:32:49.729765+0000","last_active":"2023-09-23T19:36:53.924498+0000","last_peered":"2023-09-23T19:36:53.924498+0000","last_clean":"2023-09-23T19:36:53.924498+0000","last_became_active":"2023-09-23T19:32:49.729007+0000","last_became_peered":"2023-09-23T19:32:49.729007+0000","last_unstale":"2023-09-23T19:36:53.924498+0000","last_undegraded":"2023-09-23T19:36:53.924498+0000","last_fullsized":"2023-09-23T19:36:53.924498+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:59:31.183753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":118,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.940663+0000","last_change":"2023-09-23T19:22:42.151617+0000","last_active":"2023-09-23T19:36:53.940663+0000","last_peered":"2023-09-23T19:36:53.940663+0000","last_clean":"2023-09-23T19:36:53.940663+0000","last_became_active":"2023-09-23T19:22:42.151217+0000","last_became_peered":"2023-09-23T19:22:42.151217+0000","last_unstale":"2023-09-23T19:36:53.940663+0000","last_undegraded":"2023-09-23T19:36:53.940663+0000","last_fullsized":"2023-09-23T19:36:53.940663+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":91,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.946487+0000","last_change":"2023-09-23T19:32:49.736864+0000","last_active":"2023-09-23T19:36:53.946487+0000","last_peered":"2023-09-23T19:36:53.946487+0000","last_clean":"2023-09-23T19:36:53.946487+0000","last_became_active":"2023-09-23T19:32:49.736523+0000","last_became_peered":"2023-09-23T19:32:49.736523+0000","last_unstale":"2023-09-23T19:36:53.946487+0000","last_undegraded":"2023-09-23T19:36:53.946487+0000","last_fullsized":"2023-09-23T19:36:53.946487+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":120,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.947155+0000","last_change":"2023-09-23T19:28:07.892835+0000","last_active":"2023-09-23T19:36:53.947155+0000","last_peered":"2023-09-23T19:36:53.947155+0000","last_clean":"2023-09-23T19:36:53.947155+0000","last_became_active":"2023-09-23T19:28:07.892338+0000","last_became_peered":"2023-09-23T19:28:07.892338+0000","last_unstale":"2023-09-23T19:36:53.947155+0000","last_undegraded":"2023-09-23T19:36:53.947155+0000","last_fullsized":"2023-09-23T19:36:53.947155+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":94,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.923520+0000","last_change":"2023-09-23T19:27:24.268005+0000","last_active":"2023-09-23T19:36:53.923520+0000","last_peered":"2023-09-23T19:36:53.923520+0000","last_clean":"2023-09-23T19:36:53.923520+0000","last_became_active":"2023-09-23T19:27:24.267513+0000","last_became_peered":"2023-09-23T19:27:24.267513+0000","last_unstale":"2023-09-23T19:36:53.923520+0000","last_undegraded":"2023-09-23T19:36:53.923520+0000","last_fullsized":"2023-09-23T19:36:53.923520+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.961399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'23","reported_seq":124,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.940757+0000","last_change":"2023-09-23T19:05:52.107727+0000","last_active":"2023-09-23T19:36:53.940757+0000","last_peered":"2023-09-23T19:36:53.940757+0000","last_clean":"2023-09-23T19:36:53.940757+0000","last_became_active":"2023-09-23T19:05:52.107378+0000","last_became_peered":"2023-09-23T19:05:52.107378+0000","last_unstale":"2023-09-23T19:36:53.940757+0000","last_undegraded":"2023-09-23T19:36:53.940757+0000","last_fullsized":"2023-09-23T19:36:53.940757+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":115,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.940304+0000","last_change":"2023-09-23T19:22:42.148497+0000","last_active":"2023-09-23T19:36:53.940304+0000","last_peered":"2023-09-23T19:36:53.940304+0000","last_clean":"2023-09-23T19:36:53.940304+0000","last_became_active":"2023-09-23T19:22:42.148147+0000","last_became_peered":"2023-09-23T19:22:42.148147+0000","last_unstale":"2023-09-23T19:36:53.940304+0000","last_undegraded":"2023-09-23T19:36:53.940304+0000","last_fullsized":"2023-09-23T19:36:53.940304+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":119,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.946812+0000","last_change":"2023-09-23T19:27:24.273667+0000","last_active":"2023-09-23T19:36:53.946812+0000","last_peered":"2023-09-23T19:36:53.946812+0000","last_clean":"2023-09-23T19:36:53.946812+0000","last_became_active":"2023-09-23T19:27:24.273234+0000","last_became_peered":"2023-09-23T19:27:24.273234+0000","last_unstale":"2023-09-23T19:36:53.946812+0000","last_undegraded":"2023-09-23T19:36:53.946812+0000","last_fullsized":"2023-09-23T19:36:53.946812+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":105,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.923433+0000","last_change":"2023-09-23T19:32:49.730549+0000","last_active":"2023-09-23T19:36:53.923433+0000","last_peered":"2023-09-23T19:36:53.923433+0000","last_clean":"2023-09-23T19:36:53.923433+0000","last_became_active":"2023-09-23T19:32:49.730105+0000","last_became_peered":"2023-09-23T19:32:49.730105+0000","last_unstale":"2023-09-23T19:36:53.923433+0000","last_undegraded":"2023-09-23T19:36:53.923433+0000","last_fullsized":"2023-09-23T19:36:53.923433+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:49.751893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":110,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.948594+0000","last_change":"2023-09-23T19:28:07.884394+0000","last_active":"2023-09-23T19:36:53.948594+0000","last_peered":"2023-09-23T19:36:53.948594+0000","last_clean":"2023-09-23T19:36:53.948594+0000","last_became_active":"2023-09-23T19:28:07.883944+0000","last_became_peered":"2023-09-23T19:28:07.883944+0000","last_unstale":"2023-09-23T19:36:53.948594+0000","last_undegraded":"2023-09-23T19:36:53.948594+0000","last_fullsized":"2023-09-23T19:36:53.948594+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:08:17.608926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":117,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.940550+0000","last_change":"2023-09-23T19:28:07.891056+0000","last_active":"2023-09-23T19:36:53.940550+0000","last_peered":"2023-09-23T19:36:53.940550+0000","last_clean":"2023-09-23T19:36:53.940550+0000","last_became_active":"2023-09-23T19:28:07.890625+0000","last_became_peered":"2023-09-23T19:28:07.890625+0000","last_unstale":"2023-09-23T19:36:53.940550+0000","last_undegraded":"2023-09-23T19:36:53.940550+0000","last_fullsized":"2023-09-23T19:36:53.940550+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":102,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.946688+0000","last_change":"2023-09-23T19:05:52.114186+0000","last_active":"2023-09-23T19:36:53.946688+0000","last_peered":"2023-09-23T19:36:53.946688+0000","last_clean":"2023-09-23T19:36:53.946688+0000","last_became_active":"2023-09-23T19:05:52.113635+0000","last_became_peered":"2023-09-23T19:05:52.113635+0000","last_unstale":"2023-09-23T19:36:53.946688+0000","last_undegraded":"2023-09-23T19:36:53.946688+0000","last_fullsized":"2023-09-23T19:36:53.946688+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":109,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.945502+0000","last_change":"2023-09-23T19:28:07.884512+0000","last_active":"2023-09-23T19:36:53.945502+0000","last_peered":"2023-09-23T19:36:53.945502+0000","last_clean":"2023-09-23T19:36:53.945502+0000","last_became_active":"2023-09-23T19:28:07.884155+0000","last_became_peered":"2023-09-23T19:28:07.884155+0000","last_unstale":"2023-09-23T19:36:53.945502+0000","last_undegraded":"2023-09-23T19:36:53.945502+0000","last_fullsized":"2023-09-23T19:36:53.945502+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:10.654958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":125,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.935096+0000","last_change":"2023-09-23T19:18:09.010195+0000","last_active":"2023-09-23T19:36:53.935096+0000","last_peered":"2023-09-23T19:36:53.935096+0000","last_clean":"2023-09-23T19:36:53.935096+0000","last_became_active":"2023-09-23T19:18:09.009687+0000","last_became_peered":"2023-09-23T19:18:09.009687+0000","last_unstale":"2023-09-23T19:36:53.935096+0000","last_undegraded":"2023-09-23T19:36:53.935096+0000","last_fullsized":"2023-09-23T19:36:53.935096+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":114,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.927749+0000","last_change":"2023-09-23T19:28:07.885265+0000","last_active":"2023-09-23T19:36:53.927749+0000","last_peered":"2023-09-23T19:36:53.927749+0000","last_clean":"2023-09-23T19:36:53.927749+0000","last_became_active":"2023-09-23T19:28:07.884353+0000","last_became_peered":"2023-09-23T19:28:07.884353+0000","last_unstale":"2023-09-23T19:36:53.927749+0000","last_undegraded":"2023-09-23T19:36:53.927749+0000","last_fullsized":"2023-09-23T19:36:53.927749+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:56.325352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"151'3955","reported_seq":3968,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:37:15.594830+0000","last_change":"2023-09-23T19:36:14.521072+0000","last_active":"2023-09-23T19:37:15.594830+0000","last_peered":"2023-09-23T19:37:15.594830+0000","last_clean":"2023-09-23T19:37:15.594830+0000","last_became_active":"2023-09-23T19:36:14.520698+0000","last_became_peered":"2023-09-23T19:36:14.520698+0000","last_unstale":"2023-09-23T19:37:15.594830+0000","last_undegraded":"2023-09-23T19:37:15.594830+0000","last_fullsized":"2023-09-23T19:37:15.594830+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_clean_scrub_stamp":"2023-09-23T19:36:13.468134+0000","objects_scrubbed":0,"log_size":3955,"log_dups_size":0,"ondisk_log_size":3955,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259194880,"num_objects":3955,"num_object_clones":0,"num_object_copies":11865,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3955,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3955,"num_write_kb":253120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":118,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.947285+0000","last_change":"2023-09-23T19:18:09.035116+0000","last_active":"2023-09-23T19:36:53.947285+0000","last_peered":"2023-09-23T19:36:53.947285+0000","last_clean":"2023-09-23T19:36:53.947285+0000","last_became_active":"2023-09-23T19:18:09.034416+0000","last_became_peered":"2023-09-23T19:18:09.034416+0000","last_unstale":"2023-09-23T19:36:53.947285+0000","last_undegraded":"2023-09-23T19:36:53.947285+0000","last_fullsized":"2023-09-23T19:36:53.947285+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":102,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.940007+0000","last_change":"2023-09-23T19:05:52.789323+0000","last_active":"2023-09-23T19:36:53.940007+0000","last_peered":"2023-09-23T19:36:53.940007+0000","last_clean":"2023-09-23T19:36:53.940007+0000","last_became_active":"2023-09-23T19:05:52.788820+0000","last_became_peered":"2023-09-23T19:05:52.788820+0000","last_unstale":"2023-09-23T19:36:53.940007+0000","last_undegraded":"2023-09-23T19:36:53.940007+0000","last_fullsized":"2023-09-23T19:36:53.940007+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"151'4038","reported_seq":4053,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:37:14.616375+0000","last_change":"2023-09-23T19:36:14.525132+0000","last_active":"2023-09-23T19:37:14.616375+0000","last_peered":"2023-09-23T19:37:14.616375+0000","last_clean":"2023-09-23T19:37:14.616375+0000","last_became_active":"2023-09-23T19:36:14.524731+0000","last_became_peered":"2023-09-23T19:36:14.524731+0000","last_unstale":"2023-09-23T19:37:14.616375+0000","last_undegraded":"2023-09-23T19:37:14.616375+0000","last_fullsized":"2023-09-23T19:37:14.616375+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_clean_scrub_stamp":"2023-09-23T19:36:13.468134+0000","objects_scrubbed":0,"log_size":4038,"log_dups_size":0,"ondisk_log_size":4038,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264634368,"num_objects":4038,"num_object_clones":0,"num_object_copies":12114,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4038,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4038,"num_write_kb":258432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":119,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.947530+0000","last_change":"2023-09-23T19:22:42.151134+0000","last_active":"2023-09-23T19:36:53.947530+0000","last_peered":"2023-09-23T19:36:53.947530+0000","last_clean":"2023-09-23T19:36:53.947530+0000","last_became_active":"2023-09-23T19:22:42.150553+0000","last_became_peered":"2023-09-23T19:22:42.150553+0000","last_unstale":"2023-09-23T19:36:53.947530+0000","last_undegraded":"2023-09-23T19:36:53.947530+0000","last_fullsized":"2023-09-23T19:36:53.947530+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":130,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.940217+0000","last_change":"2023-09-23T19:28:07.899262+0000","last_active":"2023-09-23T19:36:53.940217+0000","last_peered":"2023-09-23T19:36:53.940217+0000","last_clean":"2023-09-23T19:36:53.940217+0000","last_became_active":"2023-09-23T19:28:07.898848+0000","last_became_peered":"2023-09-23T19:28:07.898848+0000","last_unstale":"2023-09-23T19:36:53.940217+0000","last_undegraded":"2023-09-23T19:36:53.940217+0000","last_fullsized":"2023-09-23T19:36:53.940217+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"151'3984","reported_seq":4000,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:37:17.538344+0000","last_change":"2023-09-23T19:36:14.506997+0000","last_active":"2023-09-23T19:37:17.538344+0000","last_peered":"2023-09-23T19:37:17.538344+0000","last_clean":"2023-09-23T19:37:17.538344+0000","last_became_active":"2023-09-23T19:36:14.506682+0000","last_became_peered":"2023-09-23T19:36:14.506682+0000","last_unstale":"2023-09-23T19:37:17.538344+0000","last_undegraded":"2023-09-23T19:37:17.538344+0000","last_fullsized":"2023-09-23T19:37:17.538344+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_clean_scrub_stamp":"2023-09-23T19:36:13.468134+0000","objects_scrubbed":0,"log_size":3984,"log_dups_size":0,"ondisk_log_size":3984,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:08:49.170182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":261095424,"num_objects":3984,"num_object_clones":0,"num_object_copies":11952,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3984,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3984,"num_write_kb":254976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":103,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.946954+0000","last_change":"2023-09-23T19:05:52.114108+0000","last_active":"2023-09-23T19:36:53.946954+0000","last_peered":"2023-09-23T19:36:53.946954+0000","last_clean":"2023-09-23T19:36:53.946954+0000","last_became_active":"2023-09-23T19:05:52.113609+0000","last_became_peered":"2023-09-23T19:05:52.113609+0000","last_unstale":"2023-09-23T19:36:53.946954+0000","last_undegraded":"2023-09-23T19:36:53.946954+0000","last_fullsized":"2023-09-23T19:36:53.946954+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":116,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.939573+0000","last_change":"2023-09-23T19:22:42.156825+0000","last_active":"2023-09-23T19:36:53.939573+0000","last_peered":"2023-09-23T19:36:53.939573+0000","last_clean":"2023-09-23T19:36:53.939573+0000","last_became_active":"2023-09-23T19:22:42.156474+0000","last_became_peered":"2023-09-23T19:22:42.156474+0000","last_unstale":"2023-09-23T19:36:53.939573+0000","last_undegraded":"2023-09-23T19:36:53.939573+0000","last_fullsized":"2023-09-23T19:36:53.939573+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"151'4107","reported_seq":4122,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:37:17.403100+0000","last_change":"2023-09-23T19:36:14.507382+0000","last_active":"2023-09-23T19:37:17.403100+0000","last_peered":"2023-09-23T19:37:17.403100+0000","last_clean":"2023-09-23T19:37:17.403100+0000","last_became_active":"2023-09-23T19:36:14.506978+0000","last_became_peered":"2023-09-23T19:36:14.506978+0000","last_unstale":"2023-09-23T19:37:17.403100+0000","last_undegraded":"2023-09-23T19:37:17.403100+0000","last_fullsized":"2023-09-23T19:37:17.403100+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_clean_scrub_stamp":"2023-09-23T19:36:13.468134+0000","objects_scrubbed":0,"log_size":4107,"log_dups_size":0,"ondisk_log_size":4107,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:49.373147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":269156352,"num_objects":4107,"num_object_clones":0,"num_object_copies":12321,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4107,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4107,"num_write_kb":262848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":118,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.940209+0000","last_change":"2023-09-23T19:28:07.891536+0000","last_active":"2023-09-23T19:36:53.940209+0000","last_peered":"2023-09-23T19:36:53.940209+0000","last_clean":"2023-09-23T19:36:53.940209+0000","last_became_active":"2023-09-23T19:28:07.891153+0000","last_became_peered":"2023-09-23T19:28:07.891153+0000","last_unstale":"2023-09-23T19:36:53.940209+0000","last_undegraded":"2023-09-23T19:36:53.940209+0000","last_fullsized":"2023-09-23T19:36:53.940209+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":110,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.945178+0000","last_change":"2023-09-23T19:22:42.135572+0000","last_active":"2023-09-23T19:36:53.945178+0000","last_peered":"2023-09-23T19:36:53.945178+0000","last_clean":"2023-09-23T19:36:53.945178+0000","last_became_active":"2023-09-23T19:22:42.135237+0000","last_became_peered":"2023-09-23T19:22:42.135237+0000","last_unstale":"2023-09-23T19:36:53.945178+0000","last_undegraded":"2023-09-23T19:36:53.945178+0000","last_fullsized":"2023-09-23T19:36:53.945178+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:35.838718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"151'4098","reported_seq":4113,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:37:16.843435+0000","last_change":"2023-09-23T19:36:14.507260+0000","last_active":"2023-09-23T19:37:16.843435+0000","last_peered":"2023-09-23T19:37:16.843435+0000","last_clean":"2023-09-23T19:37:16.843435+0000","last_became_active":"2023-09-23T19:36:14.506955+0000","last_became_peered":"2023-09-23T19:36:14.506955+0000","last_unstale":"2023-09-23T19:37:16.843435+0000","last_undegraded":"2023-09-23T19:37:16.843435+0000","last_fullsized":"2023-09-23T19:37:16.843435+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_clean_scrub_stamp":"2023-09-23T19:36:13.468134+0000","objects_scrubbed":0,"log_size":4098,"log_dups_size":0,"ondisk_log_size":4098,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:09:12.021383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":268566528,"num_objects":4098,"num_object_clones":0,"num_object_copies":12294,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4098,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4098,"num_write_kb":262272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":57,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.397136+0000","last_change":"2023-09-23T19:32:49.728589+0000","last_active":"2023-09-23T19:36:55.397136+0000","last_peered":"2023-09-23T19:36:55.397136+0000","last_clean":"2023-09-23T19:36:55.397136+0000","last_became_active":"2023-09-23T19:32:49.728037+0000","last_became_peered":"2023-09-23T19:32:49.728037+0000","last_unstale":"2023-09-23T19:36:55.397136+0000","last_undegraded":"2023-09-23T19:36:55.397136+0000","last_fullsized":"2023-09-23T19:36:55.397136+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:26:00.957969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":115,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.940794+0000","last_change":"2023-09-23T19:28:07.896803+0000","last_active":"2023-09-23T19:36:53.940794+0000","last_peered":"2023-09-23T19:36:53.940794+0000","last_clean":"2023-09-23T19:36:53.940794+0000","last_became_active":"2023-09-23T19:28:07.896480+0000","last_became_peered":"2023-09-23T19:28:07.896480+0000","last_unstale":"2023-09-23T19:36:53.940794+0000","last_undegraded":"2023-09-23T19:36:53.940794+0000","last_fullsized":"2023-09-23T19:36:53.940794+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"151'4143","reported_seq":4159,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:37:17.411523+0000","last_change":"2023-09-23T19:36:14.518837+0000","last_active":"2023-09-23T19:37:17.411523+0000","last_peered":"2023-09-23T19:37:17.411523+0000","last_clean":"2023-09-23T19:37:17.411523+0000","last_became_active":"2023-09-23T19:36:14.518503+0000","last_became_peered":"2023-09-23T19:36:14.518503+0000","last_unstale":"2023-09-23T19:37:17.411523+0000","last_undegraded":"2023-09-23T19:37:17.411523+0000","last_fullsized":"2023-09-23T19:37:17.411523+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_clean_scrub_stamp":"2023-09-23T19:36:13.468134+0000","objects_scrubbed":0,"log_size":4143,"log_dups_size":0,"ondisk_log_size":4143,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:39:05.790711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":271515648,"num_objects":4143,"num_object_clones":0,"num_object_copies":12429,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4143,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4143,"num_write_kb":265152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":117,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.939796+0000","last_change":"2023-09-23T19:22:42.156582+0000","last_active":"2023-09-23T19:36:53.939796+0000","last_peered":"2023-09-23T19:36:53.939796+0000","last_clean":"2023-09-23T19:36:53.939796+0000","last_became_active":"2023-09-23T19:22:42.156251+0000","last_became_peered":"2023-09-23T19:22:42.156251+0000","last_unstale":"2023-09-23T19:36:53.939796+0000","last_undegraded":"2023-09-23T19:36:53.939796+0000","last_fullsized":"2023-09-23T19:36:53.939796+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":130,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.940326+0000","last_change":"2023-09-23T19:27:24.273971+0000","last_active":"2023-09-23T19:36:53.940326+0000","last_peered":"2023-09-23T19:36:53.940326+0000","last_clean":"2023-09-23T19:36:53.940326+0000","last_became_active":"2023-09-23T19:27:24.273491+0000","last_became_peered":"2023-09-23T19:27:24.273491+0000","last_unstale":"2023-09-23T19:36:53.940326+0000","last_undegraded":"2023-09-23T19:36:53.940326+0000","last_fullsized":"2023-09-23T19:36:53.940326+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"151'3964","reported_seq":3979,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:37:14.606953+0000","last_change":"2023-09-23T19:36:14.526027+0000","last_active":"2023-09-23T19:37:14.606953+0000","last_peered":"2023-09-23T19:37:14.606953+0000","last_clean":"2023-09-23T19:37:14.606953+0000","last_became_active":"2023-09-23T19:36:14.525656+0000","last_became_peered":"2023-09-23T19:36:14.525656+0000","last_unstale":"2023-09-23T19:37:14.606953+0000","last_undegraded":"2023-09-23T19:37:14.606953+0000","last_fullsized":"2023-09-23T19:37:14.606953+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_clean_scrub_stamp":"2023-09-23T19:36:13.468134+0000","objects_scrubbed":0,"log_size":3964,"log_dups_size":0,"ondisk_log_size":3964,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259784704,"num_objects":3964,"num_object_clones":0,"num_object_copies":11892,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3964,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3964,"num_write_kb":253696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":116,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.939951+0000","last_change":"2023-09-23T19:27:24.274698+0000","last_active":"2023-09-23T19:36:53.939951+0000","last_peered":"2023-09-23T19:36:53.939951+0000","last_clean":"2023-09-23T19:36:53.939951+0000","last_became_active":"2023-09-23T19:27:24.273264+0000","last_became_peered":"2023-09-23T19:27:24.273264+0000","last_unstale":"2023-09-23T19:36:53.939951+0000","last_undegraded":"2023-09-23T19:36:53.939951+0000","last_fullsized":"2023-09-23T19:36:53.939951+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":115,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.941253+0000","last_change":"2023-09-23T19:22:42.149670+0000","last_active":"2023-09-23T19:36:53.941253+0000","last_peered":"2023-09-23T19:36:53.941253+0000","last_clean":"2023-09-23T19:36:53.941253+0000","last_became_active":"2023-09-23T19:22:42.149022+0000","last_became_peered":"2023-09-23T19:22:42.149022+0000","last_unstale":"2023-09-23T19:36:53.941253+0000","last_undegraded":"2023-09-23T19:36:53.941253+0000","last_fullsized":"2023-09-23T19:36:53.941253+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"151'3954","reported_seq":3968,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:37:15.030685+0000","last_change":"2023-09-23T19:36:14.524191+0000","last_active":"2023-09-23T19:37:15.030685+0000","last_peered":"2023-09-23T19:37:15.030685+0000","last_clean":"2023-09-23T19:37:15.030685+0000","last_became_active":"2023-09-23T19:36:14.523868+0000","last_became_peered":"2023-09-23T19:36:14.523868+0000","last_unstale":"2023-09-23T19:37:15.030685+0000","last_undegraded":"2023-09-23T19:37:15.030685+0000","last_fullsized":"2023-09-23T19:37:15.030685+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_clean_scrub_stamp":"2023-09-23T19:36:13.468134+0000","objects_scrubbed":0,"log_size":3954,"log_dups_size":0,"ondisk_log_size":3954,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259129344,"num_objects":3954,"num_object_clones":0,"num_object_copies":11862,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3954,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3954,"num_write_kb":253056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":107,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.390401+0000","last_change":"2023-09-23T19:32:49.729244+0000","last_active":"2023-09-23T19:36:55.390401+0000","last_peered":"2023-09-23T19:36:55.390401+0000","last_clean":"2023-09-23T19:36:55.390401+0000","last_became_active":"2023-09-23T19:32:49.728810+0000","last_became_peered":"2023-09-23T19:32:49.728810+0000","last_unstale":"2023-09-23T19:36:55.390401+0000","last_undegraded":"2023-09-23T19:36:55.390401+0000","last_fullsized":"2023-09-23T19:36:55.390401+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:57.540826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":94,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.923716+0000","last_change":"2023-09-23T19:28:07.889095+0000","last_active":"2023-09-23T19:36:53.923716+0000","last_peered":"2023-09-23T19:36:53.923716+0000","last_clean":"2023-09-23T19:36:53.923716+0000","last_became_active":"2023-09-23T19:28:07.888742+0000","last_became_peered":"2023-09-23T19:28:07.888742+0000","last_unstale":"2023-09-23T19:36:53.923716+0000","last_undegraded":"2023-09-23T19:36:53.923716+0000","last_fullsized":"2023-09-23T19:36:53.923716+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:43.929488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"151'4052","reported_seq":4067,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:37:15.317691+0000","last_change":"2023-09-23T19:36:14.521993+0000","last_active":"2023-09-23T19:37:15.317691+0000","last_peered":"2023-09-23T19:37:15.317691+0000","last_clean":"2023-09-23T19:37:15.317691+0000","last_became_active":"2023-09-23T19:36:14.521681+0000","last_became_peered":"2023-09-23T19:36:14.521681+0000","last_unstale":"2023-09-23T19:37:15.317691+0000","last_undegraded":"2023-09-23T19:37:15.317691+0000","last_fullsized":"2023-09-23T19:37:15.317691+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_clean_scrub_stamp":"2023-09-23T19:36:13.468134+0000","objects_scrubbed":0,"log_size":4052,"log_dups_size":0,"ondisk_log_size":4052,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":265551872,"num_objects":4052,"num_object_clones":0,"num_object_copies":12156,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4052,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4052,"num_write_kb":259328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":128,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.389247+0000","last_change":"2023-09-23T19:22:42.143289+0000","last_active":"2023-09-23T19:36:55.389247+0000","last_peered":"2023-09-23T19:36:55.389247+0000","last_clean":"2023-09-23T19:36:55.389247+0000","last_became_active":"2023-09-23T19:22:42.142656+0000","last_became_peered":"2023-09-23T19:22:42.142656+0000","last_unstale":"2023-09-23T19:36:55.389247+0000","last_undegraded":"2023-09-23T19:36:55.389247+0000","last_fullsized":"2023-09-23T19:36:55.389247+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:38.467441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":133,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.940716+0000","last_change":"2023-09-23T19:28:07.890623+0000","last_active":"2023-09-23T19:36:53.940716+0000","last_peered":"2023-09-23T19:36:53.940716+0000","last_clean":"2023-09-23T19:36:53.940716+0000","last_became_active":"2023-09-23T19:28:07.890148+0000","last_became_peered":"2023-09-23T19:28:07.890148+0000","last_unstale":"2023-09-23T19:36:53.940716+0000","last_undegraded":"2023-09-23T19:36:53.940716+0000","last_fullsized":"2023-09-23T19:36:53.940716+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":107,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.928294+0000","last_change":"2023-09-23T19:27:24.269080+0000","last_active":"2023-09-23T19:36:53.928294+0000","last_peered":"2023-09-23T19:36:53.928294+0000","last_clean":"2023-09-23T19:36:53.928294+0000","last_became_active":"2023-09-23T19:27:24.268748+0000","last_became_peered":"2023-09-23T19:27:24.268748+0000","last_unstale":"2023-09-23T19:36:53.928294+0000","last_undegraded":"2023-09-23T19:36:53.928294+0000","last_fullsized":"2023-09-23T19:36:53.928294+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:49.014902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.a","version":"151'3899","reported_seq":3914,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:37:14.584206+0000","last_change":"2023-09-23T19:36:14.523767+0000","last_active":"2023-09-23T19:37:14.584206+0000","last_peered":"2023-09-23T19:37:14.584206+0000","last_clean":"2023-09-23T19:37:14.584206+0000","last_became_active":"2023-09-23T19:36:14.523424+0000","last_became_peered":"2023-09-23T19:36:14.523424+0000","last_unstale":"2023-09-23T19:37:14.584206+0000","last_undegraded":"2023-09-23T19:37:14.584206+0000","last_fullsized":"2023-09-23T19:37:14.584206+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_clean_scrub_stamp":"2023-09-23T19:36:13.468134+0000","objects_scrubbed":0,"log_size":3899,"log_dups_size":0,"ondisk_log_size":3899,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":255524864,"num_objects":3899,"num_object_clones":0,"num_object_copies":11697,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3899,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3899,"num_write_kb":249536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":122,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.940643+0000","last_change":"2023-09-23T19:22:42.153647+0000","last_active":"2023-09-23T19:36:53.940643+0000","last_peered":"2023-09-23T19:36:53.940643+0000","last_clean":"2023-09-23T19:36:53.940643+0000","last_became_active":"2023-09-23T19:22:42.153317+0000","last_became_peered":"2023-09-23T19:22:42.153317+0000","last_unstale":"2023-09-23T19:36:53.940643+0000","last_undegraded":"2023-09-23T19:36:53.940643+0000","last_fullsized":"2023-09-23T19:36:53.940643+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":116,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.940607+0000","last_change":"2023-09-23T19:18:09.017525+0000","last_active":"2023-09-23T19:36:53.940607+0000","last_peered":"2023-09-23T19:36:53.940607+0000","last_clean":"2023-09-23T19:36:53.940607+0000","last_became_active":"2023-09-23T19:18:09.017193+0000","last_became_peered":"2023-09-23T19:18:09.017193+0000","last_unstale":"2023-09-23T19:36:53.940607+0000","last_undegraded":"2023-09-23T19:36:53.940607+0000","last_fullsized":"2023-09-23T19:36:53.940607+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":110,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.389200+0000","last_change":"2023-09-23T19:28:07.875796+0000","last_active":"2023-09-23T19:36:55.389200+0000","last_peered":"2023-09-23T19:36:55.389200+0000","last_clean":"2023-09-23T19:36:55.389200+0000","last_became_active":"2023-09-23T19:28:07.875310+0000","last_became_peered":"2023-09-23T19:28:07.875310+0000","last_unstale":"2023-09-23T19:36:55.389200+0000","last_undegraded":"2023-09-23T19:36:55.389200+0000","last_fullsized":"2023-09-23T19:36:55.389200+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:44:13.379633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"151'4008","reported_seq":4023,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:37:15.511257+0000","last_change":"2023-09-23T19:36:14.524117+0000","last_active":"2023-09-23T19:37:15.511257+0000","last_peered":"2023-09-23T19:37:15.511257+0000","last_clean":"2023-09-23T19:37:15.511257+0000","last_became_active":"2023-09-23T19:36:14.523735+0000","last_became_peered":"2023-09-23T19:36:14.523735+0000","last_unstale":"2023-09-23T19:37:15.511257+0000","last_undegraded":"2023-09-23T19:37:15.511257+0000","last_fullsized":"2023-09-23T19:37:15.511257+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_clean_scrub_stamp":"2023-09-23T19:36:13.468134+0000","objects_scrubbed":0,"log_size":4008,"log_dups_size":0,"ondisk_log_size":4008,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262668288,"num_objects":4008,"num_object_clones":0,"num_object_copies":12024,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4008,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4008,"num_write_kb":256512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":127,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.928704+0000","last_change":"2023-09-23T19:28:07.878688+0000","last_active":"2023-09-23T19:36:53.928704+0000","last_peered":"2023-09-23T19:36:53.928704+0000","last_clean":"2023-09-23T19:36:53.928704+0000","last_became_active":"2023-09-23T19:28:07.878309+0000","last_became_peered":"2023-09-23T19:28:07.878309+0000","last_unstale":"2023-09-23T19:36:53.928704+0000","last_undegraded":"2023-09-23T19:36:53.928704+0000","last_fullsized":"2023-09-23T19:36:53.928704+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:49:06.202657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":95,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.924336+0000","last_change":"2023-09-23T19:27:24.267897+0000","last_active":"2023-09-23T19:36:53.924336+0000","last_peered":"2023-09-23T19:36:53.924336+0000","last_clean":"2023-09-23T19:36:53.924336+0000","last_became_active":"2023-09-23T19:27:24.267535+0000","last_became_peered":"2023-09-23T19:27:24.267535+0000","last_unstale":"2023-09-23T19:36:53.924336+0000","last_undegraded":"2023-09-23T19:36:53.924336+0000","last_fullsized":"2023-09-23T19:36:53.924336+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:13:57.447808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":122,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.923796+0000","last_change":"2023-09-23T19:22:42.143733+0000","last_active":"2023-09-23T19:36:53.923796+0000","last_peered":"2023-09-23T19:36:53.923796+0000","last_clean":"2023-09-23T19:36:53.923796+0000","last_became_active":"2023-09-23T19:22:42.143077+0000","last_became_peered":"2023-09-23T19:22:42.143077+0000","last_unstale":"2023-09-23T19:36:53.923796+0000","last_undegraded":"2023-09-23T19:36:53.923796+0000","last_fullsized":"2023-09-23T19:36:53.923796+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:47:33.086994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"151'3873","reported_seq":3888,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:37:14.612150+0000","last_change":"2023-09-23T19:36:14.524573+0000","last_active":"2023-09-23T19:37:14.612150+0000","last_peered":"2023-09-23T19:37:14.612150+0000","last_clean":"2023-09-23T19:37:14.612150+0000","last_became_active":"2023-09-23T19:36:14.524235+0000","last_became_peered":"2023-09-23T19:36:14.524235+0000","last_unstale":"2023-09-23T19:37:14.612150+0000","last_undegraded":"2023-09-23T19:37:14.612150+0000","last_fullsized":"2023-09-23T19:37:14.612150+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_clean_scrub_stamp":"2023-09-23T19:36:13.468134+0000","objects_scrubbed":0,"log_size":3873,"log_dups_size":0,"ondisk_log_size":3873,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253820928,"num_objects":3873,"num_object_clones":0,"num_object_copies":11619,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3873,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3873,"num_write_kb":247872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":122,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.940899+0000","last_change":"2023-09-23T19:27:24.271520+0000","last_active":"2023-09-23T19:36:53.940899+0000","last_peered":"2023-09-23T19:36:53.940899+0000","last_clean":"2023-09-23T19:36:53.940899+0000","last_became_active":"2023-09-23T19:27:24.271165+0000","last_became_peered":"2023-09-23T19:27:24.271165+0000","last_unstale":"2023-09-23T19:36:53.940899+0000","last_undegraded":"2023-09-23T19:36:53.940899+0000","last_fullsized":"2023-09-23T19:36:53.940899+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":353,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.944929+0000","last_change":"2023-09-23T19:18:09.016297+0000","last_active":"2023-09-23T19:36:53.944929+0000","last_peered":"2023-09-23T19:36:53.944929+0000","last_clean":"2023-09-23T19:36:53.944929+0000","last_became_active":"2023-09-23T19:18:09.015436+0000","last_became_peered":"2023-09-23T19:18:09.015436+0000","last_unstale":"2023-09-23T19:36:53.944929+0000","last_undegraded":"2023-09-23T19:36:53.944929+0000","last_fullsized":"2023-09-23T19:36:53.944929+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:59:33.840751+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:59:33.840751+0000","last_clean_scrub_stamp":"2023-09-23T18:59:33.840751+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":91,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.940418+0000","last_change":"2023-09-23T19:32:49.734148+0000","last_active":"2023-09-23T19:36:53.940418+0000","last_peered":"2023-09-23T19:36:53.940418+0000","last_clean":"2023-09-23T19:36:53.940418+0000","last_became_active":"2023-09-23T19:32:49.733512+0000","last_became_peered":"2023-09-23T19:32:49.733512+0000","last_unstale":"2023-09-23T19:36:53.940418+0000","last_undegraded":"2023-09-23T19:36:53.940418+0000","last_fullsized":"2023-09-23T19:36:53.940418+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":101,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.937299+0000","last_change":"2023-09-23T19:05:52.786771+0000","last_active":"2023-09-23T19:36:53.937299+0000","last_peered":"2023-09-23T19:36:53.937299+0000","last_clean":"2023-09-23T19:36:53.937299+0000","last_became_active":"2023-09-23T19:05:52.785797+0000","last_became_peered":"2023-09-23T19:05:52.785797+0000","last_unstale":"2023-09-23T19:36:53.937299+0000","last_undegraded":"2023-09-23T19:36:53.937299+0000","last_fullsized":"2023-09-23T19:36:53.937299+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"151'4195","reported_seq":4211,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:37:17.559535+0000","last_change":"2023-09-23T19:36:14.519271+0000","last_active":"2023-09-23T19:37:17.559535+0000","last_peered":"2023-09-23T19:37:17.559535+0000","last_clean":"2023-09-23T19:37:17.559535+0000","last_became_active":"2023-09-23T19:36:14.518961+0000","last_became_peered":"2023-09-23T19:36:14.518961+0000","last_unstale":"2023-09-23T19:37:17.559535+0000","last_undegraded":"2023-09-23T19:37:17.559535+0000","last_fullsized":"2023-09-23T19:37:17.559535+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_clean_scrub_stamp":"2023-09-23T19:36:13.468134+0000","objects_scrubbed":0,"log_size":4195,"log_dups_size":0,"ondisk_log_size":4195,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:06:14.200688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274923520,"num_objects":4195,"num_object_clones":0,"num_object_copies":12585,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4195,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4195,"num_write_kb":268480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":136,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.944806+0000","last_change":"2023-09-23T19:22:42.150636+0000","last_active":"2023-09-23T19:36:53.944806+0000","last_peered":"2023-09-23T19:36:53.944806+0000","last_clean":"2023-09-23T19:36:53.944806+0000","last_became_active":"2023-09-23T19:22:42.150045+0000","last_became_peered":"2023-09-23T19:22:42.150045+0000","last_unstale":"2023-09-23T19:36:53.944806+0000","last_undegraded":"2023-09-23T19:36:53.944806+0000","last_fullsized":"2023-09-23T19:36:53.944806+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":111,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.944926+0000","last_change":"2023-09-23T19:28:07.889194+0000","last_active":"2023-09-23T19:36:53.944926+0000","last_peered":"2023-09-23T19:36:53.944926+0000","last_clean":"2023-09-23T19:36:53.944926+0000","last_became_active":"2023-09-23T19:28:07.888823+0000","last_became_peered":"2023-09-23T19:28:07.888823+0000","last_unstale":"2023-09-23T19:36:53.944926+0000","last_undegraded":"2023-09-23T19:36:53.944926+0000","last_fullsized":"2023-09-23T19:36:53.944926+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:23.464688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":125,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.945427+0000","last_change":"2023-09-23T19:27:24.256410+0000","last_active":"2023-09-23T19:36:53.945427+0000","last_peered":"2023-09-23T19:36:53.945427+0000","last_clean":"2023-09-23T19:36:53.945427+0000","last_became_active":"2023-09-23T19:27:24.256052+0000","last_became_peered":"2023-09-23T19:27:24.256052+0000","last_unstale":"2023-09-23T19:36:53.945427+0000","last_undegraded":"2023-09-23T19:36:53.945427+0000","last_fullsized":"2023-09-23T19:36:53.945427+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:11.951400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"151'3969","reported_seq":3981,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:37:14.600717+0000","last_change":"2023-09-23T19:36:14.525740+0000","last_active":"2023-09-23T19:37:14.600717+0000","last_peered":"2023-09-23T19:37:14.600717+0000","last_clean":"2023-09-23T19:37:14.600717+0000","last_became_active":"2023-09-23T19:36:14.525356+0000","last_became_peered":"2023-09-23T19:36:14.525356+0000","last_unstale":"2023-09-23T19:37:14.600717+0000","last_undegraded":"2023-09-23T19:37:14.600717+0000","last_fullsized":"2023-09-23T19:37:14.600717+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_clean_scrub_stamp":"2023-09-23T19:36:13.468134+0000","objects_scrubbed":0,"log_size":3969,"log_dups_size":0,"ondisk_log_size":3969,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":260112384,"num_objects":3969,"num_object_clones":0,"num_object_copies":11907,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3969,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3969,"num_write_kb":254016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":121,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.947052+0000","last_change":"2023-09-23T19:18:09.015853+0000","last_active":"2023-09-23T19:36:53.947052+0000","last_peered":"2023-09-23T19:36:53.947052+0000","last_clean":"2023-09-23T19:36:53.947052+0000","last_became_active":"2023-09-23T19:18:09.015338+0000","last_became_peered":"2023-09-23T19:18:09.015338+0000","last_unstale":"2023-09-23T19:36:53.947052+0000","last_undegraded":"2023-09-23T19:36:53.947052+0000","last_fullsized":"2023-09-23T19:36:53.947052+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":95,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.389169+0000","last_change":"2023-09-23T19:27:24.266241+0000","last_active":"2023-09-23T19:36:55.389169+0000","last_peered":"2023-09-23T19:36:55.389169+0000","last_clean":"2023-09-23T19:36:55.389169+0000","last_became_active":"2023-09-23T19:27:24.265764+0000","last_became_peered":"2023-09-23T19:27:24.265764+0000","last_unstale":"2023-09-23T19:36:55.389169+0000","last_undegraded":"2023-09-23T19:36:55.389169+0000","last_fullsized":"2023-09-23T19:36:55.389169+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:07.694732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":96,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.945063+0000","last_change":"2023-09-23T19:18:09.005823+0000","last_active":"2023-09-23T19:36:53.945063+0000","last_peered":"2023-09-23T19:36:53.945063+0000","last_clean":"2023-09-23T19:36:53.945063+0000","last_became_active":"2023-09-23T19:18:09.005207+0000","last_became_peered":"2023-09-23T19:18:09.005207+0000","last_unstale":"2023-09-23T19:36:53.945063+0000","last_undegraded":"2023-09-23T19:36:53.945063+0000","last_fullsized":"2023-09-23T19:36:53.945063+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:05.520188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"151'3985","reported_seq":3999,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:37:14.968521+0000","last_change":"2023-09-23T19:36:14.528771+0000","last_active":"2023-09-23T19:37:14.968521+0000","last_peered":"2023-09-23T19:37:14.968521+0000","last_clean":"2023-09-23T19:37:14.968521+0000","last_became_active":"2023-09-23T19:36:14.528397+0000","last_became_peered":"2023-09-23T19:36:14.528397+0000","last_unstale":"2023-09-23T19:37:14.968521+0000","last_undegraded":"2023-09-23T19:37:14.968521+0000","last_fullsized":"2023-09-23T19:37:14.968521+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_clean_scrub_stamp":"2023-09-23T19:36:13.468134+0000","objects_scrubbed":0,"log_size":3985,"log_dups_size":0,"ondisk_log_size":3985,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":261160960,"num_objects":3985,"num_object_clones":0,"num_object_copies":11955,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3985,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3985,"num_write_kb":255040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":55,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.946958+0000","last_change":"2023-09-23T19:32:49.737323+0000","last_active":"2023-09-23T19:36:53.946958+0000","last_peered":"2023-09-23T19:36:53.946958+0000","last_clean":"2023-09-23T19:36:53.946958+0000","last_became_active":"2023-09-23T19:32:49.736937+0000","last_became_peered":"2023-09-23T19:32:49.736937+0000","last_unstale":"2023-09-23T19:36:53.946958+0000","last_undegraded":"2023-09-23T19:36:53.946958+0000","last_fullsized":"2023-09-23T19:36:53.946958+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":79,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.923443+0000","last_change":"2023-09-23T19:22:42.145506+0000","last_active":"2023-09-23T19:36:53.923443+0000","last_peered":"2023-09-23T19:36:53.923443+0000","last_clean":"2023-09-23T19:36:53.923443+0000","last_became_active":"2023-09-23T19:22:42.143991+0000","last_became_peered":"2023-09-23T19:22:42.143991+0000","last_unstale":"2023-09-23T19:36:53.923443+0000","last_undegraded":"2023-09-23T19:36:53.923443+0000","last_fullsized":"2023-09-23T19:36:53.923443+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:56.875175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":92,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.945008+0000","last_change":"2023-09-23T19:32:49.734592+0000","last_active":"2023-09-23T19:36:53.945008+0000","last_peered":"2023-09-23T19:36:53.945008+0000","last_clean":"2023-09-23T19:36:53.945008+0000","last_became_active":"2023-09-23T19:32:49.733905+0000","last_became_peered":"2023-09-23T19:32:49.733905+0000","last_unstale":"2023-09-23T19:36:53.945008+0000","last_undegraded":"2023-09-23T19:36:53.945008+0000","last_fullsized":"2023-09-23T19:36:53.945008+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"151'3992","reported_seq":4007,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:37:14.592945+0000","last_change":"2023-09-23T19:36:14.522170+0000","last_active":"2023-09-23T19:37:14.592945+0000","last_peered":"2023-09-23T19:37:14.592945+0000","last_clean":"2023-09-23T19:37:14.592945+0000","last_became_active":"2023-09-23T19:36:14.521790+0000","last_became_peered":"2023-09-23T19:36:14.521790+0000","last_unstale":"2023-09-23T19:37:14.592945+0000","last_undegraded":"2023-09-23T19:37:14.592945+0000","last_fullsized":"2023-09-23T19:37:14.592945+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_clean_scrub_stamp":"2023-09-23T19:36:13.468134+0000","objects_scrubbed":0,"log_size":3992,"log_dups_size":0,"ondisk_log_size":3992,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":261619712,"num_objects":3992,"num_object_clones":0,"num_object_copies":11976,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3992,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3992,"num_write_kb":255488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":121,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.937761+0000","last_change":"2023-09-23T19:27:24.271827+0000","last_active":"2023-09-23T19:36:53.937761+0000","last_peered":"2023-09-23T19:36:53.937761+0000","last_clean":"2023-09-23T19:36:53.937761+0000","last_became_active":"2023-09-23T19:27:24.271328+0000","last_became_peered":"2023-09-23T19:27:24.271328+0000","last_unstale":"2023-09-23T19:36:53.937761+0000","last_undegraded":"2023-09-23T19:36:53.937761+0000","last_fullsized":"2023-09-23T19:36:53.937761+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":113,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.945330+0000","last_change":"2023-09-23T19:22:42.149694+0000","last_active":"2023-09-23T19:36:53.945330+0000","last_peered":"2023-09-23T19:36:53.945330+0000","last_clean":"2023-09-23T19:36:53.945330+0000","last_became_active":"2023-09-23T19:22:42.148194+0000","last_became_peered":"2023-09-23T19:22:42.148194+0000","last_unstale":"2023-09-23T19:36:53.945330+0000","last_undegraded":"2023-09-23T19:36:53.945330+0000","last_fullsized":"2023-09-23T19:36:53.945330+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:47.842383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":100,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.940049+0000","last_change":"2023-09-23T19:05:52.138620+0000","last_active":"2023-09-23T19:36:53.940049+0000","last_peered":"2023-09-23T19:36:53.940049+0000","last_clean":"2023-09-23T19:36:53.940049+0000","last_became_active":"2023-09-23T19:05:52.138243+0000","last_became_peered":"2023-09-23T19:05:52.138243+0000","last_unstale":"2023-09-23T19:36:53.940049+0000","last_undegraded":"2023-09-23T19:36:53.940049+0000","last_fullsized":"2023-09-23T19:36:53.940049+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":97,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.945229+0000","last_change":"2023-09-23T19:18:09.008811+0000","last_active":"2023-09-23T19:36:53.945229+0000","last_peered":"2023-09-23T19:36:53.945229+0000","last_clean":"2023-09-23T19:36:53.945229+0000","last_became_active":"2023-09-23T19:18:09.007255+0000","last_became_peered":"2023-09-23T19:18:09.007255+0000","last_unstale":"2023-09-23T19:36:53.945229+0000","last_undegraded":"2023-09-23T19:36:53.945229+0000","last_fullsized":"2023-09-23T19:36:53.945229+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:25.874038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":134,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.928445+0000","last_change":"2023-09-23T19:28:07.876141+0000","last_active":"2023-09-23T19:36:53.928445+0000","last_peered":"2023-09-23T19:36:53.928445+0000","last_clean":"2023-09-23T19:36:53.928445+0000","last_became_active":"2023-09-23T19:28:07.875802+0000","last_became_peered":"2023-09-23T19:28:07.875802+0000","last_unstale":"2023-09-23T19:36:53.928445+0000","last_undegraded":"2023-09-23T19:36:53.928445+0000","last_fullsized":"2023-09-23T19:36:53.928445+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:38.947881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":120,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.940321+0000","last_change":"2023-09-23T19:22:42.153825+0000","last_active":"2023-09-23T19:36:53.940321+0000","last_peered":"2023-09-23T19:36:53.940321+0000","last_clean":"2023-09-23T19:36:53.940321+0000","last_became_active":"2023-09-23T19:22:42.153416+0000","last_became_peered":"2023-09-23T19:22:42.153416+0000","last_unstale":"2023-09-23T19:36:53.940321+0000","last_undegraded":"2023-09-23T19:36:53.940321+0000","last_fullsized":"2023-09-23T19:36:53.940321+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":139,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.940462+0000","last_change":"2023-09-23T19:28:07.899164+0000","last_active":"2023-09-23T19:36:53.940462+0000","last_peered":"2023-09-23T19:36:53.940462+0000","last_clean":"2023-09-23T19:36:53.940462+0000","last_became_active":"2023-09-23T19:28:07.898812+0000","last_became_peered":"2023-09-23T19:28:07.898812+0000","last_unstale":"2023-09-23T19:36:53.940462+0000","last_undegraded":"2023-09-23T19:36:53.940462+0000","last_fullsized":"2023-09-23T19:36:53.940462+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":105,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.940775+0000","last_change":"2023-09-23T19:05:52.107570+0000","last_active":"2023-09-23T19:36:53.940775+0000","last_peered":"2023-09-23T19:36:53.940775+0000","last_clean":"2023-09-23T19:36:53.940775+0000","last_became_active":"2023-09-23T19:05:52.107215+0000","last_became_peered":"2023-09-23T19:05:52.107215+0000","last_unstale":"2023-09-23T19:36:53.940775+0000","last_undegraded":"2023-09-23T19:36:53.940775+0000","last_fullsized":"2023-09-23T19:36:53.940775+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":130,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.940220+0000","last_change":"2023-09-23T19:28:07.896983+0000","last_active":"2023-09-23T19:36:53.940220+0000","last_peered":"2023-09-23T19:36:53.940220+0000","last_clean":"2023-09-23T19:36:53.940220+0000","last_became_active":"2023-09-23T19:28:07.896577+0000","last_became_peered":"2023-09-23T19:28:07.896577+0000","last_unstale":"2023-09-23T19:36:53.940220+0000","last_undegraded":"2023-09-23T19:36:53.940220+0000","last_fullsized":"2023-09-23T19:36:53.940220+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":45,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.936037+0000","last_change":"2023-09-23T19:32:49.732683+0000","last_active":"2023-09-23T19:36:53.936037+0000","last_peered":"2023-09-23T19:36:53.936037+0000","last_clean":"2023-09-23T19:36:53.936037+0000","last_became_active":"2023-09-23T19:32:49.732188+0000","last_became_peered":"2023-09-23T19:32:49.732188+0000","last_unstale":"2023-09-23T19:36:53.936037+0000","last_undegraded":"2023-09-23T19:36:53.936037+0000","last_fullsized":"2023-09-23T19:36:53.936037+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":133,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.928560+0000","last_change":"2023-09-23T19:32:49.725466+0000","last_active":"2023-09-23T19:36:53.928560+0000","last_peered":"2023-09-23T19:36:53.928560+0000","last_clean":"2023-09-23T19:36:53.928560+0000","last_became_active":"2023-09-23T19:32:49.725048+0000","last_became_peered":"2023-09-23T19:32:49.725048+0000","last_unstale":"2023-09-23T19:36:53.928560+0000","last_undegraded":"2023-09-23T19:36:53.928560+0000","last_fullsized":"2023-09-23T19:36:53.928560+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:44:08.728220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":114,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.945659+0000","last_change":"2023-09-23T19:27:24.267931+0000","last_active":"2023-09-23T19:36:53.945659+0000","last_peered":"2023-09-23T19:36:53.945659+0000","last_clean":"2023-09-23T19:36:53.945659+0000","last_became_active":"2023-09-23T19:27:24.267552+0000","last_became_peered":"2023-09-23T19:27:24.267552+0000","last_unstale":"2023-09-23T19:36:53.945659+0000","last_undegraded":"2023-09-23T19:36:53.945659+0000","last_fullsized":"2023-09-23T19:36:53.945659+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:12.425257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":109,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-09-23T19:36:53.941445+0000","last_change":"2023-09-23T19:06:36.673002+0000","last_active":"2023-09-23T19:36:53.941445+0000","last_peered":"2023-09-23T19:36:53.941445+0000","last_clean":"2023-09-23T19:36:53.941445+0000","last_became_active":"2023-09-23T19:06:36.671911+0000","last_became_peered":"2023-09-23T19:06:36.671911+0000","last_unstale":"2023-09-23T19:36:53.941445+0000","last_undegraded":"2023-09-23T19:36:53.941445+0000","last_fullsized":"2023-09-23T19:36:53.941445+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":4208459776,"num_objects":64216,"num_object_clones":0,"num_object_copies":192648,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":64216,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":64216,"num_write_kb":4109824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12723617792,"data_stored":12723617792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":64216,"ondisk_log_size":64216,"up":48,"acting":48,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430621,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1925644,"kb_used_data":1025668,"kb_used_omap":0,"kb_used_meta":899968,"kb_avail":91819508,"statfs":{"total":95995035648,"available":94023176192,"internally_reserved":0,"allocated":1050284032,"data_stored":1049799787,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":921567232},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:36:38 2023","interfaces":[{"interface":"back","average":{"1min":0.83299999999999996,"5min":2.7240000000000002,"15min":1.657},"min":{"1min":0.53600000000000003,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":1.2070000000000001,"5min":92.944999999999993,"15min":92.944999999999993},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.97599999999999998,"5min":2.7389999999999999,"15min":1.661},"min":{"1min":0.55800000000000005,"5min":0.29499999999999998,"15min":0.28999999999999998},"max":{"1min":2.8690000000000002,"5min":92.763999999999996,"15min":92.763999999999996},"last":0.51700000000000002}]},{"osd":1,"last update":"Sat Sep 23 19:37:05 2023","interfaces":[{"interface":"back","average":{"1min":0.66900000000000004,"5min":3.0739999999999998,"15min":1.8680000000000001},"min":{"1min":0.37,"5min":0.29199999999999998,"15min":0.28499999999999998},"max":{"1min":1.1080000000000001,"5min":93.757999999999996,"15min":93.757999999999996},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.84399999999999997,"5min":2.9860000000000002,"15min":1.851},"min":{"1min":0.39000000000000001,"5min":0.29999999999999999,"15min":0.29599999999999999},"max":{"1min":1.726,"5min":93.835999999999999,"15min":93.835999999999999},"last":0.56299999999999994}]},{"osd":2,"last update":"Sat Sep 23 19:36:38 2023","interfaces":[{"interface":"back","average":{"1min":0.84099999999999997,"5min":1.677,"15min":1.1259999999999999},"min":{"1min":0.36899999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":1.0449999999999999,"5min":7.9320000000000004,"15min":7.9320000000000004},"last":3.5840000000000001},{"interface":"front","average":{"1min":0.96699999999999997,"5min":1.2709999999999999,"15min":1.073},"min":{"1min":0.52700000000000002,"5min":0.308,"15min":0.308},"max":{"1min":1.2210000000000001,"5min":12.574,"15min":12.574},"last":0.53200000000000003}]},{"osd":3,"last update":"Sat Sep 23 19:36:22 2023","interfaces":[{"interface":"back","average":{"1min":1.244,"5min":1.99,"15min":1.3740000000000001},"min":{"1min":0.496,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":3.2759999999999998,"5min":19.637,"15min":19.637},"last":1.4390000000000001},{"interface":"front","average":{"1min":1.496,"5min":1.726,"15min":1.3360000000000001},"min":{"1min":0.44,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":12.608000000000001,"5min":20.204000000000001,"15min":20.204000000000001},"last":1.403}]},{"osd":4,"last update":"Sat Sep 23 19:37:05 2023","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":1.377,"15min":1.1459999999999999},"min":{"1min":0.38900000000000001,"5min":0.252,"15min":0.252},"max":{"1min":1.0189999999999999,"5min":9.7699999999999996,"15min":9.7699999999999996},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.93100000000000005,"5min":1.2,"15min":1.073},"min":{"1min":0.40200000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":1.7649999999999999,"5min":12.509,"15min":12.509},"last":0.41799999999999998}]},{"osd":5,"last update":"Sat Sep 23 19:37:05 2023","interfaces":[{"interface":"back","average":{"1min":1.127,"5min":1.877,"15min":1.3799999999999999},"min":{"1min":0.40799999999999997,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.581,"5min":13.342000000000001,"15min":13.342000000000001},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.97599999999999998,"5min":1.9490000000000001,"15min":1.5109999999999999},"min":{"1min":0.47799999999999998,"5min":0.31,"15min":0.31},"max":{"1min":2.2770000000000001,"5min":9.8979999999999997,"15min":9.8979999999999997},"last":0.59999999999999998}]},{"osd":6,"last update":"Sat Sep 23 19:37:05 2023","interfaces":[{"interface":"back","average":{"1min":1.161,"5min":1.522,"15min":1.29},"min":{"1min":0.56899999999999995,"5min":0.316,"15min":0.316},"max":{"1min":4.008,"5min":9.8629999999999995,"15min":9.8629999999999995},"last":0.63500000000000001},{"interface":"front","average":{"1min":1.4950000000000001,"5min":1.4279999999999999,"15min":1.1819999999999999},"min":{"1min":0.68100000000000005,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":5.7759999999999998,"5min":12.631,"15min":12.631},"last":0.49199999999999999}]}]},{"osd":6,"up_from":42,"seq":180388626853,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2361148,"kb_used_data":1763252,"kb_used_omap":0,"kb_used_meta":597888,"kb_avail":91384004,"statfs":{"total":95995035648,"available":93577220096,"internally_reserved":0,"allocated":1805570048,"data_stored":1805101726,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":612237312},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:36:49 2023","interfaces":[{"interface":"back","average":{"1min":1.0169999999999999,"5min":2.302,"15min":1.524},"min":{"1min":0.377,"5min":0.25900000000000001,"15min":0.24099999999999999},"max":{"1min":3.3799999999999999,"5min":87.283000000000001,"15min":87.283000000000001},"last":0.38900000000000001},{"interface":"front","average":{"1min":1.7190000000000001,"5min":2.4289999999999998,"15min":1.681},"min":{"1min":0.56899999999999995,"5min":0.30399999999999999,"15min":0.26600000000000001},"max":{"1min":6.5419999999999998,"5min":86.286000000000001,"15min":86.286000000000001},"last":0.45600000000000002}]},{"osd":1,"last update":"Sat Sep 23 19:37:09 2023","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":1.5149999999999999,"15min":1.4930000000000001},"min":{"1min":0.315,"5min":0.253,"15min":0.253},"max":{"1min":2.0910000000000002,"5min":8.0899999999999999,"15min":11.351000000000001},"last":0.56899999999999995},{"interface":"front","average":{"1min":2.0310000000000001,"5min":1.53,"15min":1.585},"min":{"1min":0.60899999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":6.0949999999999998,"5min":7.4889999999999999,"15min":12.728999999999999},"last":0.67200000000000004}]},{"osd":2,"last update":"Sat Sep 23 19:36:39 2023","interfaces":[{"interface":"back","average":{"1min":0.93500000000000005,"5min":0.79600000000000004,"15min":0.88900000000000001},"min":{"1min":0.47699999999999998,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.48,"5min":4.3840000000000003,"15min":4.3840000000000003},"last":42.020000000000003},{"interface":"front","average":{"1min":0.88600000000000001,"5min":1.081,"15min":0.96699999999999997},"min":{"1min":0.34200000000000003,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":1.5589999999999999,"5min":8.0600000000000005,"15min":8.0600000000000005},"last":41.970999999999997}]},{"osd":3,"last update":"Sat Sep 23 19:36:27 2023","interfaces":[{"interface":"back","average":{"1min":0.88900000000000001,"5min":1.921,"15min":1.3080000000000001},"min":{"1min":0.38800000000000001,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.29,"5min":60.726999999999997,"15min":60.726999999999997},"last":0.64800000000000002},{"interface":"front","average":{"1min":1.01,"5min":2.2930000000000001,"15min":1.4139999999999999},"min":{"1min":0.47999999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":2.5550000000000002,"5min":58.304000000000002,"15min":58.304000000000002},"last":0.438}]},{"osd":4,"last update":"Sat Sep 23 19:36:27 2023","interfaces":[{"interface":"back","average":{"1min":0.97399999999999998,"5min":0.89900000000000002,"15min":1.002},"min":{"1min":0.42699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":2.5209999999999999,"5min":3.7320000000000002,"15min":6.5759999999999996},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.995,"5min":0.89800000000000002,"15min":1.131},"min":{"1min":0.49099999999999999,"5min":0.376,"15min":0.30499999999999999},"max":{"1min":2.141,"5min":3.6560000000000001,"15min":8.5540000000000003},"last":0.68400000000000005}]},{"osd":5,"last update":"Sat Sep 23 19:36:27 2023","interfaces":[{"interface":"back","average":{"1min":0.97299999999999998,"5min":1.2569999999999999,"15min":1.222},"min":{"1min":0.51000000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":2.0960000000000001,"5min":8.8670000000000009,"15min":8.8670000000000009},"last":0.50900000000000001},{"interface":"front","average":{"1min":1.8340000000000001,"5min":1.718,"15min":1.53},"min":{"1min":0.57699999999999996,"5min":0.41099999999999998,"15min":0.38100000000000001},"max":{"1min":9.7219999999999995,"5min":9.7219999999999995,"15min":9.7219999999999995},"last":0.71599999999999997}]},{"osd":7,"last update":"Sat Sep 23 19:36:54 2023","interfaces":[{"interface":"back","average":{"1min":1.851,"5min":1.298,"15min":1.181},"min":{"1min":0.61599999999999999,"5min":0.41599999999999998,"15min":0.38300000000000001},"max":{"1min":5.9580000000000002,"5min":5.9580000000000002,"15min":8.7949999999999999},"last":0.60899999999999999},{"interface":"front","average":{"1min":1.3049999999999999,"5min":1.464,"15min":1.3740000000000001},"min":{"1min":0.55300000000000005,"5min":0.41899999999999998,"15min":0.39300000000000002},"max":{"1min":2.5870000000000002,"5min":8.8059999999999992,"15min":8.8059999999999992},"last":0.746}]}]},{"osd":1,"up_from":98,"seq":420906795185,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1158268,"kb_used_data":774196,"kb_used_omap":4,"kb_used_meta":384059,"kb_avail":92586884,"statfs":{"total":95995035648,"available":94808969216,"internally_reserved":0,"allocated":792776704,"data_stored":792133478,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":393277326},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:37:14 2023","interfaces":[{"interface":"back","average":{"1min":0.76500000000000001,"5min":1.3360000000000001,"15min":1.466},"min":{"1min":0.24399999999999999,"5min":0.185,"15min":0.185},"max":{"1min":2.399,"5min":7.2610000000000001,"15min":14.99},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.751,"5min":1.7949999999999999,"15min":1.6399999999999999},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":2.177,"5min":9.5470000000000006,"15min":12.215},"last":0.55900000000000005}]},{"osd":2,"last update":"Sat Sep 23 19:36:43 2023","interfaces":[{"interface":"back","average":{"1min":0.79100000000000004,"5min":1.621,"15min":1.0289999999999999},"min":{"1min":0.45000000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":1.425,"5min":10.212999999999999,"15min":10.212999999999999},"last":0.28000000000000003},{"interface":"front","average":{"1min":0.66600000000000004,"5min":1.5620000000000001,"15min":0.97999999999999998},"min":{"1min":0.34100000000000003,"5min":0.246,"15min":0.246},"max":{"1min":0.95799999999999996,"5min":11.308,"15min":11.308},"last":0.39600000000000002}]},{"osd":3,"last update":"Sat Sep 23 19:36:20 2023","interfaces":[{"interface":"back","average":{"1min":1.179,"5min":1.8740000000000001,"15min":1.1220000000000001},"min":{"1min":0.27700000000000002,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":7.0780000000000003,"5min":10.612,"15min":10.612},"last":1.1240000000000001},{"interface":"front","average":{"1min":1.1100000000000001,"5min":2.3039999999999998,"15min":1.3089999999999999},"min":{"1min":0.32600000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":4.6790000000000003,"5min":12.457000000000001,"15min":12.457000000000001},"last":3.2879999999999998}]},{"osd":4,"last update":"Sat Sep 23 19:37:14 2023","interfaces":[{"interface":"back","average":{"1min":0.98699999999999999,"5min":1.772,"15min":1.5389999999999999},"min":{"1min":0.40400000000000003,"5min":0.38900000000000001,"15min":0.35899999999999999},"max":{"1min":1.948,"5min":9.6690000000000005,"15min":9.6690000000000005},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.99299999999999999,"5min":1.6140000000000001,"15min":1.5900000000000001},"min":{"1min":0.48099999999999998,"5min":0.36199999999999999,"15min":0.34399999999999997},"max":{"1min":1.9299999999999999,"5min":10.061999999999999,"15min":11.409000000000001},"last":0.72599999999999998}]},{"osd":5,"last update":"Sat Sep 23 19:37:14 2023","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":2.3980000000000001,"15min":1.905},"min":{"1min":0.55400000000000005,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":3.4950000000000001,"5min":13.273999999999999,"15min":13.273999999999999},"last":0.86799999999999999},{"interface":"front","average":{"1min":1.1120000000000001,"5min":1.9390000000000001,"15min":1.6140000000000001},"min":{"1min":0.50600000000000001,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":2.423,"5min":7.3310000000000004,"15min":9.218},"last":0.70399999999999996}]},{"osd":6,"last update":"Sat Sep 23 19:37:14 2023","interfaces":[{"interface":"back","average":{"1min":1.5549999999999999,"5min":1.796,"15min":1.6479999999999999},"min":{"1min":0.55100000000000005,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":6.1399999999999997,"5min":12.429,"15min":12.429},"last":0.66900000000000004},{"interface":"front","average":{"1min":1.5109999999999999,"5min":2.0009999999999999,"15min":1.7629999999999999},"min":{"1min":0.63200000000000001,"5min":0.39100000000000001,"15min":0.38800000000000001},"max":{"1min":6.181,"5min":9.641,"15min":9.641},"last":0.64100000000000001}]},{"osd":7,"last update":"Sat Sep 23 19:37:14 2023","interfaces":[{"interface":"back","average":{"1min":1.0269999999999999,"5min":2.343,"15min":1.891},"min":{"1min":0.54100000000000004,"5min":0.432,"15min":0.40999999999999998},"max":{"1min":1.6180000000000001,"5min":11.391,"15min":11.391},"last":0.59699999999999998},{"interface":"front","average":{"1min":1.139,"5min":2.1339999999999999,"15min":1.8089999999999999},"min":{"1min":0.48199999999999998,"5min":0.44900000000000001,"15min":0.42699999999999999},"max":{"1min":2.4340000000000002,"5min":11.374000000000001,"15min":12.709},"last":0.48199999999999998}]}]},{"osd":0,"up_from":85,"seq":365072220392,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3099724,"kb_used_data":2632772,"kb_used_omap":5,"kb_used_meta":466938,"kb_avail":90645428,"statfs":{"total":95995035648,"available":92820918272,"internally_reserved":0,"allocated":2695958528,"data_stored":2695303953,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":478145146},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 19:37:04 2023","interfaces":[{"interface":"back","average":{"1min":1.0589999999999999,"5min":1.379,"15min":1.454},"min":{"1min":0.29799999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":3.718,"5min":7.3680000000000003,"15min":13.557},"last":1.6539999999999999},{"interface":"front","average":{"1min":0.91600000000000004,"5min":1.7270000000000001,"15min":1.5660000000000001},"min":{"1min":0.28199999999999997,"5min":0.26000000000000001,"15min":0.253},"max":{"1min":2.464,"5min":12.939,"15min":12.939},"last":1.49}]},{"osd":2,"last update":"Sat Sep 23 19:36:41 2023","interfaces":[{"interface":"back","average":{"1min":1.117,"5min":1.2529999999999999,"15min":0.92100000000000004},"min":{"1min":0.37,"5min":0.214,"15min":0.214},"max":{"1min":9.827,"5min":11.861000000000001,"15min":11.861000000000001},"last":1.696},{"interface":"front","average":{"1min":1.026,"5min":1.3109999999999999,"15min":0.96399999999999997},"min":{"1min":0.41199999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":4.718,"5min":10.212,"15min":10.212},"last":1.7689999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:36:16 2023","interfaces":[{"interface":"back","average":{"1min":1.105,"5min":1.667,"15min":1.075},"min":{"1min":0.36399999999999999,"5min":0.30099999999999999,"15min":0.251},"max":{"1min":4.9509999999999996,"5min":10.430999999999999,"15min":10.430999999999999},"last":1.1839999999999999},{"interface":"front","average":{"1min":1.238,"5min":1.8600000000000001,"15min":1.2729999999999999},"min":{"1min":0.49199999999999999,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":6.5679999999999996,"5min":12.991,"15min":12.991},"last":1.5169999999999999}]},{"osd":4,"last update":"Sat Sep 23 19:36:49 2023","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":1.4750000000000001,"15min":1.371},"min":{"1min":0.503,"5min":0.37,"15min":0.32100000000000001},"max":{"1min":1.97,"5min":10.002000000000001,"15min":10.002000000000001},"last":1.2470000000000001},{"interface":"front","average":{"1min":1.1060000000000001,"5min":1.2190000000000001,"15min":1.3879999999999999},"min":{"1min":0.66100000000000003,"5min":0.36399999999999999,"15min":0.35599999999999998},"max":{"1min":2.8759999999999999,"5min":4.6529999999999996,"15min":10.282999999999999},"last":2.0379999999999998}]},{"osd":5,"last update":"Sat Sep 23 19:36:49 2023","interfaces":[{"interface":"back","average":{"1min":1.157,"5min":1.9930000000000001,"15min":1.7749999999999999},"min":{"1min":0.61299999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":2.3359999999999999,"5min":13.154,"15min":13.154},"last":1.736},{"interface":"front","average":{"1min":1.103,"5min":2.2890000000000001,"15min":1.7669999999999999},"min":{"1min":0.48199999999999998,"5min":0.36899999999999999,"15min":0.33400000000000002},"max":{"1min":1.925,"5min":13.109999999999999,"15min":13.109999999999999},"last":1.304}]},{"osd":6,"last update":"Sat Sep 23 19:36:49 2023","interfaces":[{"interface":"back","average":{"1min":1.1830000000000001,"5min":1.4950000000000001,"15min":1.466},"min":{"1min":0.53000000000000003,"5min":0.38400000000000001,"15min":0.36499999999999999},"max":{"1min":3.2080000000000002,"5min":6.2329999999999997,"15min":8.532},"last":2.778},{"interface":"front","average":{"1min":1.2889999999999999,"5min":1.411,"15min":1.5640000000000001},"min":{"1min":0.75700000000000001,"5min":0.36699999999999999,"15min":0.36399999999999999},"max":{"1min":2.8239999999999998,"5min":6.1130000000000004,"15min":10.308},"last":2.7050000000000001}]},{"osd":7,"last update":"Sat Sep 23 19:36:49 2023","interfaces":[{"interface":"back","average":{"1min":1.2370000000000001,"5min":1.7450000000000001,"15min":1.6060000000000001},"min":{"1min":0.58299999999999996,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":3.7690000000000001,"5min":12.385,"15min":13.218},"last":1.7230000000000001},{"interface":"front","average":{"1min":1.1599999999999999,"5min":2.0259999999999998,"15min":1.7050000000000001},"min":{"1min":0.58199999999999996,"5min":0.435,"15min":0.38300000000000001},"max":{"1min":1.9950000000000001,"5min":8.8450000000000006,"15min":8.8450000000000006},"last":1.2789999999999999}]}]},{"osd":2,"up_from":111,"seq":476741369977,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1817296,"kb_used_data":1584776,"kb_used_omap":7,"kb_used_meta":232504,"kb_avail":91927856,"statfs":{"total":95995035648,"available":94134124544,"internally_reserved":0,"allocated":1622810624,"data_stored":1622206561,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":238084177},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:36:37 2023","interfaces":[{"interface":"back","average":{"1min":0.71899999999999997,"5min":1.446,"15min":0.95499999999999996},"min":{"1min":0.38100000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":1.744,"5min":59.670999999999999,"15min":59.670999999999999},"last":1.474},{"interface":"front","average":{"1min":0.77900000000000003,"5min":1.4319999999999999,"15min":0.99299999999999999},"min":{"1min":0.35399999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":1.946,"5min":59.774000000000001,"15min":59.774000000000001},"last":1.609}]},{"osd":1,"last update":"Sat Sep 23 19:36:37 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":3.452,"15min":1.6459999999999999},"min":{"1min":0.29199999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.93600000000000005,"5min":92.933000000000007,"15min":92.933000000000007},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.78700000000000003,"5min":3.3340000000000001,"15min":1.6739999999999999},"min":{"1min":0.47499999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":1.827,"5min":92.432000000000002,"15min":92.432000000000002},"last":0.88300000000000001}]},{"osd":3,"last update":"Sat Sep 23 19:36:28 2023","interfaces":[{"interface":"back","average":{"1min":0.68200000000000005,"5min":2.3730000000000002,"15min":1.29},"min":{"1min":0.25700000000000001,"5min":0.22,"15min":0.22},"max":{"1min":1.056,"5min":92.350999999999999,"15min":92.350999999999999},"last":0.93600000000000005},{"interface":"front","average":{"1min":0.752,"5min":2.5569999999999999,"15min":1.4239999999999999},"min":{"1min":0.39600000000000002,"5min":0.248,"15min":0.248},"max":{"1min":1.212,"5min":96.236999999999995,"15min":96.236999999999995},"last":1.325}]},{"osd":4,"last update":"Sat Sep 23 19:36:37 2023","interfaces":[{"interface":"back","average":{"1min":0.97099999999999997,"5min":0.84399999999999997,"15min":0.96299999999999997},"min":{"1min":0.52700000000000002,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":2.0339999999999998,"5min":3.1899999999999999,"15min":3.1899999999999999},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.92500000000000004,"5min":1.054,"15min":1.022},"min":{"1min":0.41499999999999998,"5min":0.37,"15min":0.37},"max":{"1min":1.526,"5min":9.0199999999999996,"15min":9.0199999999999996},"last":0.72199999999999998}]},{"osd":5,"last update":"Sat Sep 23 19:36:37 2023","interfaces":[{"interface":"back","average":{"1min":1.089,"5min":1.569,"15min":1.266},"min":{"1min":0.64500000000000002,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":2.1629999999999998,"5min":11.058999999999999,"15min":11.058999999999999},"last":1.0389999999999999},{"interface":"front","average":{"1min":1.0940000000000001,"5min":1.544,"15min":1.262},"min":{"1min":0.67100000000000004,"5min":0.309,"15min":0.309},"max":{"1min":2.0790000000000002,"5min":6.5179999999999998,"15min":6.5179999999999998},"last":0.40300000000000002}]},{"osd":6,"last update":"Sat Sep 23 19:36:37 2023","interfaces":[{"interface":"back","average":{"1min":1.0009999999999999,"5min":1.1359999999999999,"15min":1.0960000000000001},"min":{"1min":0.63300000000000001,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":1.661,"5min":9.0739999999999998,"15min":9.0739999999999998},"last":0.78800000000000003},{"interface":"front","average":{"1min":1.181,"5min":1.0820000000000001,"15min":1.1339999999999999},"min":{"1min":0.67900000000000005,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":3.5819999999999999,"5min":5.6959999999999997,"15min":5.6959999999999997},"last":1.079}]},{"osd":7,"last update":"Sat Sep 23 19:36:37 2023","interfaces":[{"interface":"back","average":{"1min":1.161,"5min":1.6639999999999999,"15min":1.353},"min":{"1min":0.502,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.9910000000000001,"5min":12.339,"15min":12.339},"last":0.45700000000000002},{"interface":"front","average":{"1min":1.044,"5min":1.7190000000000001,"15min":1.3480000000000001},"min":{"1min":0.56499999999999995,"5min":0.37,"15min":0.37},"max":{"1min":1.423,"5min":12.236000000000001,"15min":12.236000000000001},"last":0.75900000000000001}]}]},{"osd":3,"up_from":116,"seq":498216206448,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1575012,"kb_used_data":1299548,"kb_used_omap":2,"kb_used_meta":275453,"kb_avail":92170140,"statfs":{"total":95995035648,"available":94382223360,"internally_reserved":0,"allocated":1330737152,"data_stored":1330171692,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":282063925},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:36:22 2023","interfaces":[{"interface":"back","average":{"1min":1.1240000000000001,"5min":1.8959999999999999,"15min":1.0800000000000001},"min":{"1min":0.27200000000000002,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":9.1329999999999991,"5min":13.411,"15min":13.411},"last":0.60099999999999998},{"interface":"front","average":{"1min":1.331,"5min":1.663,"15min":1.0780000000000001},"min":{"1min":0.39900000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":4.782,"5min":10.77,"15min":10.77},"last":0.34100000000000003}]},{"osd":1,"last update":"Sat Sep 23 19:36:22 2023","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":1.958,"15min":1.151},"min":{"1min":0.40600000000000003,"5min":0.246,"15min":0.246},"max":{"1min":2.5019999999999998,"5min":13.659000000000001,"15min":13.659000000000001},"last":0.29699999999999999},{"interface":"front","average":{"1min":0.83099999999999996,"5min":2.403,"15min":1.3380000000000001},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":2.9820000000000002,"5min":16.262,"15min":16.262},"last":0.23000000000000001}]},{"osd":2,"last update":"Sat Sep 23 19:36:22 2023","interfaces":[{"interface":"back","average":{"1min":1.006,"5min":1.3420000000000001,"15min":0.95499999999999996},"min":{"1min":0.31,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":4.1180000000000003,"5min":6.9950000000000001,"15min":6.9950000000000001},"last":0.245},{"interface":"front","average":{"1min":1.2110000000000001,"5min":1.7310000000000001,"15min":1.1479999999999999},"min":{"1min":0.45600000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":6.3109999999999999,"5min":7.0670000000000002,"15min":7.0670000000000002},"last":0.36399999999999999}]},{"osd":4,"last update":"Sat Sep 23 19:36:22 2023","interfaces":[{"interface":"back","average":{"1min":1.081,"5min":1.5920000000000001,"15min":1.2470000000000001},"min":{"1min":0.505,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":2.8919999999999999,"5min":7.0259999999999998,"15min":7.0259999999999998},"last":0.32500000000000001},{"interface":"front","average":{"1min":1.071,"5min":1.446,"15min":1.242},"min":{"1min":0.48799999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":2.3759999999999999,"5min":10.926,"15min":10.926},"last":0.379}]},{"osd":5,"last update":"Sat Sep 23 19:36:22 2023","interfaces":[{"interface":"back","average":{"1min":1.0980000000000001,"5min":2.1000000000000001,"15min":1.4390000000000001},"min":{"1min":0.52200000000000002,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":2.472,"5min":10.82,"15min":10.82},"last":0.41799999999999998},{"interface":"front","average":{"1min":1.0620000000000001,"5min":1.806,"15min":1.3220000000000001},"min":{"1min":0.48399999999999999,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":2.2949999999999999,"5min":8.7710000000000008,"15min":8.7710000000000008},"last":0.53700000000000003}]},{"osd":6,"last update":"Sat Sep 23 19:36:22 2023","interfaces":[{"interface":"back","average":{"1min":1.3759999999999999,"5min":1.667,"15min":1.3169999999999999},"min":{"1min":0.66000000000000003,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":4.3730000000000002,"5min":10.859,"15min":10.859},"last":1.5760000000000001},{"interface":"front","average":{"1min":1.054,"5min":1.4259999999999999,"15min":1.238},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":2.266,"5min":8.0190000000000001,"15min":8.0190000000000001},"last":0.45900000000000002}]},{"osd":7,"last update":"Sat Sep 23 19:36:22 2023","interfaces":[{"interface":"back","average":{"1min":1.113,"5min":1.79,"15min":1.369},"min":{"1min":0.53800000000000003,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":2.234,"5min":11.259,"15min":11.259},"last":2.5190000000000001},{"interface":"front","average":{"1min":1.1699999999999999,"5min":1.798,"15min":1.379},"min":{"1min":0.44600000000000001,"5min":0.432,"15min":0.432},"max":{"1min":2.944,"5min":7.0279999999999996,"15min":7.0279999999999996},"last":0.52400000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019318,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2595588,"kb_used_data":2005884,"kb_used_omap":0,"kb_used_meta":589696,"kb_avail":91149564,"statfs":{"total":95995035648,"available":93337153536,"internally_reserved":0,"allocated":2054025216,"data_stored":2053548549,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603848704},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:36:45 2023","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":1.0229999999999999,"15min":1.111},"min":{"1min":0.56699999999999995,"5min":0.255,"15min":0.255},"max":{"1min":3.0550000000000002,"5min":7.391,"15min":8.7680000000000007},"last":2.1850000000000001},{"interface":"front","average":{"1min":0.85999999999999999,"5min":1.1599999999999999,"15min":1.24},"min":{"1min":0.313,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":1.4059999999999999,"5min":7.9349999999999996,"15min":7.9349999999999996},"last":2.8029999999999999}]},{"osd":1,"last update":"Sat Sep 23 19:36:14 2023","interfaces":[{"interface":"back","average":{"1min":2.113,"5min":1.3540000000000001,"15min":1.4650000000000001},"min":{"1min":0.28799999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":11.093,"5min":11.093,"15min":12.372},"last":2.726},{"interface":"front","average":{"1min":1.954,"5min":1.502,"15min":1.4970000000000001},"min":{"1min":0.48199999999999998,"5min":0.32500000000000001,"15min":0.31},"max":{"1min":11.061,"5min":11.061,"15min":11.061},"last":0.48799999999999999}]},{"osd":2,"last update":"Sat Sep 23 19:36:34 2023","interfaces":[{"interface":"back","average":{"1min":0.84399999999999997,"5min":0.88600000000000001,"15min":0.89700000000000002},"min":{"1min":0.36799999999999999,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":1.137,"5min":7.9699999999999998,"15min":7.9699999999999998},"last":5.4660000000000002},{"interface":"front","average":{"1min":0.80500000000000005,"5min":0.85599999999999998,"15min":0.88600000000000001},"min":{"1min":0.42599999999999999,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.216,"5min":7.2069999999999999,"15min":7.2069999999999999},"last":2.2799999999999998}]},{"osd":3,"last update":"Sat Sep 23 19:36:28 2023","interfaces":[{"interface":"back","average":{"1min":3.274,"5min":2.3119999999999998,"15min":1.4159999999999999},"min":{"1min":0.45200000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":44.340000000000003,"5min":49.515000000000001,"15min":49.515000000000001},"last":0.59099999999999997},{"interface":"front","average":{"1min":3.5590000000000002,"5min":2.323,"15min":1.4179999999999999},"min":{"1min":0.56699999999999995,"5min":0.374,"15min":0.374},"max":{"1min":44.203000000000003,"5min":51.518999999999998,"15min":51.518999999999998},"last":2.226}]},{"osd":5,"last update":"Sat Sep 23 19:36:51 2023","interfaces":[{"interface":"back","average":{"1min":1.3979999999999999,"5min":1.6990000000000001,"15min":1.423},"min":{"1min":0.45700000000000002,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":8.1899999999999995,"5min":10.118,"15min":12.459},"last":0.45500000000000002},{"interface":"front","average":{"1min":1.2450000000000001,"5min":1.4730000000000001,"15min":1.3440000000000001},"min":{"1min":0.61099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":4.649,"5min":8.1120000000000001,"15min":12.478},"last":5.2560000000000002}]},{"osd":6,"last update":"Sat Sep 23 19:36:14 2023","interfaces":[{"interface":"back","average":{"1min":1.4690000000000001,"5min":1.2629999999999999,"15min":1.3049999999999999},"min":{"1min":0.379,"5min":0.34399999999999997,"15min":0.29999999999999999},"max":{"1min":6.9119999999999999,"5min":6.9119999999999999,"15min":7.0810000000000004},"last":0.56499999999999995},{"interface":"front","average":{"1min":1.2370000000000001,"5min":1.1000000000000001,"15min":1.212},"min":{"1min":0.42599999999999999,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":6.9400000000000004,"5min":7.319,"15min":7.319},"last":4.1390000000000002}]},{"osd":7,"last update":"Sat Sep 23 19:36:14 2023","interfaces":[{"interface":"back","average":{"1min":1.677,"5min":1.3420000000000001,"15min":1.2749999999999999},"min":{"1min":0.443,"5min":0.371,"15min":0.371},"max":{"1min":6.0300000000000002,"5min":7.2729999999999997,"15min":7.8479999999999999},"last":2.7730000000000001},{"interface":"front","average":{"1min":1.6100000000000001,"5min":1.5569999999999999,"15min":1.3089999999999999},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":7.242,"5min":7.242,"15min":7.242},"last":0.51800000000000002}]}]},{"osd":5,"up_from":36,"seq":154618823085,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2174700,"kb_used_data":1271908,"kb_used_omap":0,"kb_used_meta":902784,"kb_avail":91570452,"statfs":{"total":95995035648,"available":93768142848,"internally_reserved":0,"allocated":1302433792,"data_stored":1301976815,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":924450816},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:36:50 2023","interfaces":[{"interface":"back","average":{"1min":1.149,"5min":1.8899999999999999,"15min":1.4059999999999999},"min":{"1min":0.29499999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":7.024,"5min":32.643999999999998,"15min":32.643999999999998},"last":0.44400000000000001},{"interface":"front","average":{"1min":1.5760000000000001,"5min":2.657,"15min":1.849},"min":{"1min":0.60199999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":7.5869999999999997,"5min":31.876999999999999,"15min":31.876999999999999},"last":2.899}]},{"osd":1,"last update":"Sat Sep 23 19:36:16 2023","interfaces":[{"interface":"back","average":{"1min":2.266,"5min":1.8240000000000001,"15min":1.4390000000000001},"min":{"1min":0.54200000000000004,"5min":0.29499999999999998,"15min":0.27200000000000002},"max":{"1min":23.777999999999999,"5min":32.253999999999998,"15min":32.253999999999998},"last":0.36199999999999999},{"interface":"front","average":{"1min":3.165,"5min":2.6539999999999999,"15min":2.0329999999999999},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.26700000000000002},"max":{"1min":29.314,"5min":32.277999999999999,"15min":32.277999999999999},"last":0.48799999999999999}]},{"osd":2,"last update":"Sat Sep 23 19:36:50 2023","interfaces":[{"interface":"back","average":{"1min":1.056,"5min":1.476,"15min":1.145},"min":{"1min":0.51500000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":2.9340000000000002,"5min":10.164,"15min":10.164},"last":5.7060000000000004},{"interface":"front","average":{"1min":1.151,"5min":1.1120000000000001,"15min":1.079},"min":{"1min":0.39400000000000002,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":2.855,"5min":9.0050000000000008,"15min":9.0050000000000008},"last":2.7320000000000002}]},{"osd":3,"last update":"Sat Sep 23 19:36:16 2023","interfaces":[{"interface":"back","average":{"1min":1.5069999999999999,"5min":1.669,"15min":1.248},"min":{"1min":0.379,"5min":0.311,"15min":0.311},"max":{"1min":10.885999999999999,"5min":10.885999999999999,"15min":10.885999999999999},"last":5.7599999999999998},{"interface":"front","average":{"1min":1.319,"5min":1.3080000000000001,"15min":1.2030000000000001},"min":{"1min":0.51100000000000001,"5min":0.373,"15min":0.373},"max":{"1min":4.2000000000000002,"5min":6.5369999999999999,"15min":6.5369999999999999},"last":4.218}]},{"osd":4,"last update":"Sat Sep 23 19:36:50 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":1.921,"15min":1.411},"min":{"1min":0.38900000000000001,"5min":0.33500000000000002,"15min":0.29199999999999998},"max":{"1min":2.7010000000000001,"5min":18.683,"15min":18.683},"last":0.67100000000000004},{"interface":"front","average":{"1min":1.036,"5min":1.4399999999999999,"15min":1.28},"min":{"1min":0.371,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.625,"5min":16.073,"15min":16.073},"last":0.57099999999999995}]},{"osd":6,"last update":"Sat Sep 23 19:36:28 2023","interfaces":[{"interface":"back","average":{"1min":1.0740000000000001,"5min":1.131,"15min":1.2430000000000001},"min":{"1min":0.49099999999999999,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":1.661,"5min":5.7149999999999999,"15min":26.448},"last":0.81899999999999995},{"interface":"front","average":{"1min":1.0449999999999999,"5min":1.5149999999999999,"15min":1.4239999999999999},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":1.7370000000000001,"5min":9.4749999999999996,"15min":26.841999999999999},"last":0.86599999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:36:59 2023","interfaces":[{"interface":"back","average":{"1min":1.325,"5min":1.988,"15min":1.4750000000000001},"min":{"1min":0.437,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":4.9009999999999998,"5min":11.913,"15min":11.913},"last":0.61199999999999999},{"interface":"front","average":{"1min":1.099,"5min":1.756,"15min":1.357},"min":{"1min":0.371,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.972,"5min":13.18,"15min":13.18},"last":0.40699999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2702180352,"data_stored":2702180352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":801832960,"data_stored":801832960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1635909632,"data_stored":1635909632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1329725440,"data_stored":1329725440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2075000832,"data_stored":2075000832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1312423936,"data_stored":1312423936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1815085056,"data_stored":1815085056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1051459584,"data_stored":1051459584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T19:37:21.605 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T19:37:21.686 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:21.712 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:21.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:21.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:21.816 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:21.841 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:21.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:21.844 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:37:21.944 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:21.972 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:21.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:21.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:37:22.076 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:22.110 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:22.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:22.114 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:37:22.215 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:22.224 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:22 smithi016 bash[75004]: cluster 2023-09-23T19:37:20.083441+0000 mgr.y (mgr.25325) 894 : cluster 0 pgmap v817: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-09-23T19:37:22.224 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:22 smithi016 bash[75004]: cluster 2023-09-23T19:37:20.408835+0000 osd.6 (osd.6) 67 : cluster 3 failed to encode map e152 with expected crc 2023-09-23T19:37:22.224 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:22 smithi016 bash[75004]: cluster 2023-09-23T19:37:20.408957+0000 osd.4 (osd.4) 78 : cluster 3 failed to encode map e152 with expected crc 2023-09-23T19:37:22.225 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:22 smithi016 bash[75004]: cluster 2023-09-23T19:37:20.413847+0000 osd.6 (osd.6) 68 : cluster 3 failed to encode map e152 with expected crc 2023-09-23T19:37:22.225 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:22 smithi016 bash[77995]: cluster 2023-09-23T19:37:20.083441+0000 mgr.y (mgr.25325) 894 : cluster 0 pgmap v817: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-09-23T19:37:22.225 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:22 smithi016 bash[77995]: cluster 2023-09-23T19:37:20.408835+0000 osd.6 (osd.6) 67 : cluster 3 failed to encode map e152 with expected crc 2023-09-23T19:37:22.225 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:22 smithi016 bash[77995]: cluster 2023-09-23T19:37:20.408957+0000 osd.4 (osd.4) 78 : cluster 3 failed to encode map e152 with expected crc 2023-09-23T19:37:22.225 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:22 smithi016 bash[77995]: cluster 2023-09-23T19:37:20.413847+0000 osd.6 (osd.6) 68 : cluster 3 failed to encode map e152 with expected crc 2023-09-23T19:37:22.240 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:22.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:22.244 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:37:22.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:22 smithi092 bash[58855]: cluster 2023-09-23T19:37:20.083441+0000 mgr.y (mgr.25325) 894 : cluster 0 pgmap v817: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-09-23T19:37:22.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:22 smithi092 bash[58855]: cluster 2023-09-23T19:37:20.408835+0000 osd.6 (osd.6) 67 : cluster 3 failed to encode map e152 with expected crc 2023-09-23T19:37:22.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:22 smithi092 bash[58855]: cluster 2023-09-23T19:37:20.408957+0000 osd.4 (osd.4) 78 : cluster 3 failed to encode map e152 with expected crc 2023-09-23T19:37:22.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:22 smithi092 bash[58855]: cluster 2023-09-23T19:37:20.413847+0000 osd.6 (osd.6) 68 : cluster 3 failed to encode map e152 with expected crc 2023-09-23T19:37:22.345 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:22.380 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:22.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:22.385 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:22.486 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:22.489 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 47 16 71347 71331 94.8444 67.1875 0.0217462 0.0105372 2023-09-23T19:37:22.517 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:22.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:22.520 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:22.621 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:22.652 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:22.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:22.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:22.755 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:22.780 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:22.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:22.784 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:22.884 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:22.913 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:22.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:22.917 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:37:23.018 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:23.082 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:23.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:23.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:37:23.185 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:23.218 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:23.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:23.221 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:23.321 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:23.352 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:23.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:23.356 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:23.457 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:23.489 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 48 16 72476 72460 94.3383 70.5625 0.0137054 0.0105943 2023-09-23T19:37:23.496 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:23.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:23.501 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:37:23.602 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:23.631 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:23.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:23.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:23.736 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:23.762 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:23.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:23.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:37:23.865 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:23.898 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:23.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:23.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:24.002 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:24.036 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:24.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:24.039 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:24.140 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:24.167 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:24.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:24.173 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:24.274 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:24.283 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:24 smithi016 bash[77995]: cluster 2023-09-23T19:37:22.084064+0000 mgr.y (mgr.25325) 895 : cluster 0 pgmap v819: 89 pgs: 89 active+clean; 4.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T19:37:24.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:24 smithi016 bash[75004]: cluster 2023-09-23T19:37:22.084064+0000 mgr.y (mgr.25325) 895 : cluster 0 pgmap v819: 89 pgs: 89 active+clean; 4.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T19:37:24.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:24 smithi092 bash[58855]: cluster 2023-09-23T19:37:22.084064+0000 mgr.y (mgr.25325) 895 : cluster 0 pgmap v819: 89 pgs: 89 active+clean; 4.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T19:37:24.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:24.306 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:24.307 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:37:24.411 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:24.452 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:24.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:24.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:24.489 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 49 16 73549 73533 93.7816 67.0625 0.0238459 0.0106572 2023-09-23T19:37:24.558 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:24.594 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:24.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:24.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:24.701 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:24.730 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:24.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:24.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:37:24.833 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:24.866 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:24.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:24.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:37:24.972 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:24.988 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T19:37:25.002 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:25.004 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:37:25.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:25.007 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:37:25.108 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:25.163 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:25.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:25.167 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:37:25.268 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:25.295 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:25.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:25.298 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:37:25.398 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:25.428 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:25.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:25.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:37:25.489 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 50 16 74547 74531 93.1533 62.375 0.0103261 0.0107292 2023-09-23T19:37:25.516 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:37:25.536 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:25.560 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:25.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:25.564 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:37:25.665 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:25.696 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:25.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:25.702 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:25.802 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:25.833 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:25.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:25.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:25.938 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:25.965 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:25.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:25.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:26.026 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:37:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:37:25] "GET /metrics HTTP/1.1" 200 36292 "" "Prometheus/2.33.4" 2023-09-23T19:37:26.068 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:26.076 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:26 smithi092 bash[58855]: cluster 2023-09-23T19:37:24.084750+0000 mgr.y (mgr.25325) 896 : cluster 0 pgmap v820: 89 pgs: 89 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-09-23T19:37:26.094 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:26.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:26.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:37:26.198 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:26.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:26.237 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:26.238 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:26.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:26 smithi016 bash[75004]: cluster 2023-09-23T19:37:24.084750+0000 mgr.y (mgr.25325) 896 : cluster 0 pgmap v820: 89 pgs: 89 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-09-23T19:37:26.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:26 smithi016 bash[77995]: cluster 2023-09-23T19:37:24.084750+0000 mgr.y (mgr.25325) 896 : cluster 0 pgmap v820: 89 pgs: 89 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-09-23T19:37:26.338 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:26.365 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:26.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:26.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:26.468 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:26.489 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 51 16 75612 75596 92.6318 66.5625 0.0191825 0.0107894 2023-09-23T19:37:26.493 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:26.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:26.496 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:37:26.597 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:26.641 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:26.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:26.646 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:37:26.747 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:26.772 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:26.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:26.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:26.875 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:26.900 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:26.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:26.903 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:37:27.004 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:27.043 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:27.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:27.052 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:27.152 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:27.186 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:27.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:27.195 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:37:27.296 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:27.327 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:27.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:27.332 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:37:27.433 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:27.461 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:27.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:27.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:27.489 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 52 16 76649 76633 92.0967 64.8125 0.00784482 0.0108527 2023-09-23T19:37:27.568 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:27.596 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:27.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:27.600 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:27.613 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:37:27.700 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:27.730 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:27.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:27.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:27.835 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:27.873 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:27.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:27.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:27.977 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:28.005 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:28.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:28.010 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:37:28.111 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:28.142 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:28.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:28.145 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:28.245 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:28.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:28 smithi016 bash[75004]: cluster 2023-09-23T19:37:26.085618+0000 mgr.y (mgr.25325) 897 : cluster 0 pgmap v821: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-09-23T19:37:28.255 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:28 smithi016 bash[77995]: cluster 2023-09-23T19:37:26.085618+0000 mgr.y (mgr.25325) 897 : cluster 0 pgmap v821: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-09-23T19:37:28.276 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:28.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:28.278 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:28.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:28 smithi092 bash[58855]: cluster 2023-09-23T19:37:26.085618+0000 mgr.y (mgr.25325) 897 : cluster 0 pgmap v821: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-09-23T19:37:28.379 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:28.412 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:28.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:28.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:37:28.489 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 53 16 77627 77611 91.5123 61.125 0.0152084 0.0109213 2023-09-23T19:37:28.521 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:28.546 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:28.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:28.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:37:28.651 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:28.678 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:28.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:28.682 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:28.783 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:28.805 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:28.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:28.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:37:28.908 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:28.938 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:28.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:28.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:29.041 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:29.050 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:29 smithi092 bash[58855]: audit 2023-09-23T19:37:28.651813+0000 mon.a (mon.0) 1114 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:37:29.079 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:29.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:29.088 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:37:29.188 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:29.223 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:29.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:29.230 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:29.330 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:29.340 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:29 smithi016 bash[75004]: audit 2023-09-23T19:37:28.651813+0000 mon.a (mon.0) 1114 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:37:29.341 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:29 smithi016 bash[77995]: audit 2023-09-23T19:37:28.651813+0000 mon.a (mon.0) 1114 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:37:29.372 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:29.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:29.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:37:29.478 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:29.489 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 54 16 78573 78557 90.9124 59.125 0.0148037 0.0109934 2023-09-23T19:37:29.507 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:29.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:29.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:29.613 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:29.645 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:29.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:29.650 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:37:29.751 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:29.776 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:29.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:29.780 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:37:29.812 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:37:29.815 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:37:29.881 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:29.908 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:29.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:29.913 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:30.014 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:30.046 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:30.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:30.052 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:37:30.152 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:30.162 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:30 smithi016 bash[77995]: cluster 2023-09-23T19:37:28.087684+0000 mgr.y (mgr.25325) 898 : cluster 0 pgmap v822: 89 pgs: 89 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-09-23T19:37:30.162 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:30 smithi016 bash[75004]: cluster 2023-09-23T19:37:28.087684+0000 mgr.y (mgr.25325) 898 : cluster 0 pgmap v822: 89 pgs: 89 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-09-23T19:37:30.188 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:30.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:30.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:30.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:30 smithi092 bash[58855]: cluster 2023-09-23T19:37:28.087684+0000 mgr.y (mgr.25325) 898 : cluster 0 pgmap v822: 89 pgs: 89 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-09-23T19:37:30.293 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:30.318 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:30.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:30.322 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:37:30.423 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:30.457 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:30.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:30.462 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:30.489 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 55 16 79573 79557 90.3957 62.5 0.012091 0.0110565 2023-09-23T19:37:30.563 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:30.589 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:30.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:30.595 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:30.695 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:30.727 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:30.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:30.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:37:30.832 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:30.857 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:30.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:30.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:37:30.960 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:30.987 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:30.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:30.990 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:31.091 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:31.121 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:31.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:31.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:31.224 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:31.254 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:31.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:31.257 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:37:31.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:31 smithi016 bash[77995]: cluster 2023-09-23T19:37:30.088464+0000 mgr.y (mgr.25325) 899 : cluster 0 pgmap v823: 89 pgs: 89 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T19:37:31.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:31 smithi016 bash[75004]: cluster 2023-09-23T19:37:30.088464+0000 mgr.y (mgr.25325) 899 : cluster 0 pgmap v823: 89 pgs: 89 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T19:37:31.357 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:31.370 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:31 smithi092 bash[58855]: cluster 2023-09-23T19:37:30.088464+0000 mgr.y (mgr.25325) 899 : cluster 0 pgmap v823: 89 pgs: 89 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T19:37:31.387 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:31.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:31.390 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:31.490 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 56 16 80343 80327 89.6408 48.125 0.0148441 0.0111505 2023-09-23T19:37:31.491 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:31.522 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:31.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:31.525 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:37:31.625 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:31.649 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:31.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:31.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:37:31.754 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:31.776 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:31.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:31.779 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:37:31.880 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:31.917 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:31.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:31.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:32.020 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:32.045 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:32.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:32.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:32.150 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:32.175 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:32.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:32.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:32.279 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:32.309 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:32.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:32.316 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:32.417 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:32.459 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:32.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:32.464 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:37:32.490 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 57 16 81287 81271 89.1031 59 0.0148998 0.0112178 2023-09-23T19:37:32.564 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:32.595 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:32.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:32.601 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:37:32.702 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:32.738 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:32.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:32.744 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:32.845 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:32.877 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:32.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:32.881 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:37:32.982 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:33.007 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:33.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:33.014 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:37:33.115 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:33.116 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-09-23T19:37:33.116 INFO:teuthology.orchestra.run.smithi016.stdout: "size": 20, 2023-09-23T19:37:33.116 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 600, 2023-09-23T19:37:33.116 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [ 2023-09-23T19:37:33.117 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.117 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45671.0:19142 9.b 9:d3b64c3d:::benchmark_data_smithi016_113233_object19141:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e122)", 2023-09-23T19:37:33.117 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:31:42.042244+0000", 2023-09-23T19:37:33.117 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 351.068104871, 2023-09-23T19:37:33.117 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.15077631799999999, 2023-09-23T19:37:33.117 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:37:33.117 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:37:33.117 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:37:33.117 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45671", 2023-09-23T19:37:33.117 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:37:33.117 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 19142 2023-09-23T19:37:33.118 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.118 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:37:33.118 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.118 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:37:33.118 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.042244+0000", 2023-09-23T19:37:33.118 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:37:33.118 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.118 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.118 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:37:33.118 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.042244+0000", 2023-09-23T19:37:33.118 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:37:33.119 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.119 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.119 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:37:33.119 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.042244+0000", 2023-09-23T19:37:33.119 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T19:37:33.119 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.119 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.119 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:37:33.119 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.042261+0000", 2023-09-23T19:37:33.119 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.6874000000000001e-05 2023-09-23T19:37:33.119 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.119 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.120 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:37:33.120 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.042268+0000", 2023-09-23T19:37:33.120 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.8630000000000002e-06 2023-09-23T19:37:33.120 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.120 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.120 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:37:33.120 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.042274+0000", 2023-09-23T19:37:33.120 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.3380000000000001e-06 2023-09-23T19:37:33.120 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.120 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.121 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:37:33.121 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.043858+0000", 2023-09-23T19:37:33.121 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0015839669999999999 2023-09-23T19:37:33.121 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.121 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.121 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:37:33.121 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.043926+0000", 2023-09-23T19:37:33.121 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.8548999999999997e-05 2023-09-23T19:37:33.121 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.121 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.121 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 5,6", 2023-09-23T19:37:33.121 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.044006+0000", 2023-09-23T19:37:33.122 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.9386000000000002e-05 2023-09-23T19:37:33.122 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.122 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.122 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:37:33.122 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.045454+0000", 2023-09-23T19:37:33.122 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00144794 2023-09-23T19:37:33.122 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.122 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.122 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:37:33.122 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.046460+0000", 2023-09-23T19:37:33.122 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0010058529999999999 2023-09-23T19:37:33.122 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.123 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.123 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:37:33.123 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.046898+0000", 2023-09-23T19:37:33.123 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00043816100000000001 2023-09-23T19:37:33.123 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.123 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.123 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:37:33.123 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.192884+0000", 2023-09-23T19:37:33.123 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.14598634999999999 2023-09-23T19:37:33.123 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.124 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.124 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:37:33.124 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.193021+0000", 2023-09-23T19:37:33.124 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000136475 2023-09-23T19:37:33.124 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:37:33.124 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:37:33.124 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:37:33.124 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.125 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.125 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45671.0:19154 9.9 9:9fca7707:::benchmark_data_smithi016_113233_object19153:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e122)", 2023-09-23T19:37:33.125 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:31:42.048781+0000", 2023-09-23T19:37:33.125 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 351.06156814600001, 2023-09-23T19:37:33.125 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.164320364, 2023-09-23T19:37:33.125 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:37:33.125 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:37:33.125 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:37:33.126 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45671", 2023-09-23T19:37:33.126 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:37:33.126 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 19154 2023-09-23T19:37:33.126 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.126 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:37:33.126 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.126 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:37:33.126 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.048781+0000", 2023-09-23T19:37:33.126 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:37:33.126 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.126 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.127 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:37:33.127 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.048781+0000", 2023-09-23T19:37:33.127 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:37:33.127 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.127 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.127 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:37:33.127 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.048779+0000", 2023-09-23T19:37:33.127 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-09-23T19:37:33.127 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.127 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.128 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:37:33.128 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.048804+0000", 2023-09-23T19:37:33.128 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.4301e-05 2023-09-23T19:37:33.128 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.128 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.128 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:37:33.128 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.193070+0000", 2023-09-23T19:37:33.128 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.144265904 2023-09-23T19:37:33.128 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.128 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.128 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:37:33.129 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.193094+0000", 2023-09-23T19:37:33.129 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.4773000000000002e-05 2023-09-23T19:37:33.129 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.129 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.129 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:37:33.129 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.193427+0000", 2023-09-23T19:37:33.129 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00033257799999999999 2023-09-23T19:37:33.129 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.129 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.130 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:37:33.130 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.193586+0000", 2023-09-23T19:37:33.130 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00015939900000000001 2023-09-23T19:37:33.130 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.130 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.130 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 5,7", 2023-09-23T19:37:33.130 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.195215+0000", 2023-09-23T19:37:33.130 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001628359 2023-09-23T19:37:33.131 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.131 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.131 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:37:33.131 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.198925+0000", 2023-09-23T19:37:33.131 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0037104009999999999 2023-09-23T19:37:33.131 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.131 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.131 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:37:33.131 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.204277+0000", 2023-09-23T19:37:33.131 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0053514280000000001 2023-09-23T19:37:33.132 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.132 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.132 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:37:33.132 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.212966+0000", 2023-09-23T19:37:33.132 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0086890839999999997 2023-09-23T19:37:33.132 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.132 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.132 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:37:33.132 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.213045+0000", 2023-09-23T19:37:33.132 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.9141999999999995e-05 2023-09-23T19:37:33.133 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.133 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.133 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:37:33.133 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:31:42.213101+0000", 2023-09-23T19:37:33.133 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.6511999999999997e-05 2023-09-23T19:37:33.133 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:37:33.133 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:37:33.133 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:37:33.133 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.133 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.133 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45671.0:81633 9.8 e125/119 9:1d4ef9a8:::benchmark_data_smithi016_113233_object81632:head v 125'4969, mlcod=125'4963)", 2023-09-23T19:37:33.133 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:32:32.555650+0000", 2023-09-23T19:37:33.134 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 300.55469932400001, 2023-09-23T19:37:33.134 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.19491667700000001, 2023-09-23T19:37:33.134 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:37:33.134 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:37:33.134 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:37:33.134 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.134 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:37:33.134 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.555650+0000", 2023-09-23T19:37:33.134 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:37:33.134 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.135 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.136 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:37:33.136 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.555650+0000", 2023-09-23T19:37:33.136 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:37:33.136 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.136 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.136 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:37:33.136 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.555649+0000", 2023-09-23T19:37:33.137 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:37:33.137 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.137 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.137 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:37:33.137 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.555676+0000", 2023-09-23T19:37:33.137 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.7739999999999999e-05 2023-09-23T19:37:33.137 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.137 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.137 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:37:33.137 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.555688+0000", 2023-09-23T19:37:33.137 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2012e-05 2023-09-23T19:37:33.137 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.138 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.138 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:37:33.138 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.555695+0000", 2023-09-23T19:37:33.138 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.922e-06 2023-09-23T19:37:33.138 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.138 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.138 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:37:33.138 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.557740+0000", 2023-09-23T19:37:33.138 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0020449919999999998 2023-09-23T19:37:33.138 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.138 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.139 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:37:33.139 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.557761+0000", 2023-09-23T19:37:33.139 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.0628999999999998e-05 2023-09-23T19:37:33.139 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.139 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.139 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:37:33.139 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.750508+0000", 2023-09-23T19:37:33.139 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.19274656100000001 2023-09-23T19:37:33.139 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.139 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.139 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:37:33.140 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.750566+0000", 2023-09-23T19:37:33.140 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.8967e-05 2023-09-23T19:37:33.140 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:37:33.140 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:37:33.140 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:37:33.140 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.140 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.140 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45671.0:81655 9.b 9:ddafaf93:::benchmark_data_smithi016_113233_object81654:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e125)", 2023-09-23T19:37:33.140 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:32:32.555996+0000", 2023-09-23T19:37:33.140 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 300.554352795, 2023-09-23T19:37:33.140 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.19463889400000001, 2023-09-23T19:37:33.140 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:37:33.141 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:37:33.141 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:37:33.141 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45671", 2023-09-23T19:37:33.141 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:37:33.141 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 81655 2023-09-23T19:37:33.141 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.141 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:37:33.141 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.142 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:37:33.142 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.555996+0000", 2023-09-23T19:37:33.142 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:37:33.142 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.142 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.142 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:37:33.142 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.555996+0000", 2023-09-23T19:37:33.142 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:37:33.143 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.143 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.143 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:37:33.143 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.555995+0000", 2023-09-23T19:37:33.143 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-09-23T19:37:33.143 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.143 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.143 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:37:33.143 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.556026+0000", 2023-09-23T19:37:33.143 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.1396000000000002e-05 2023-09-23T19:37:33.143 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.143 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.144 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:37:33.144 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.556041+0000", 2023-09-23T19:37:33.144 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.5108e-05 2023-09-23T19:37:33.144 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.144 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.144 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:37:33.144 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.556050+0000", 2023-09-23T19:37:33.144 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.952e-06 2023-09-23T19:37:33.144 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.144 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.144 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:37:33.145 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.556099+0000", 2023-09-23T19:37:33.145 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.8668000000000002e-05 2023-09-23T19:37:33.145 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.145 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.145 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:37:33.145 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.556263+0000", 2023-09-23T19:37:33.145 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00016430399999999999 2023-09-23T19:37:33.145 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.145 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.145 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 5,6", 2023-09-23T19:37:33.145 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.556373+0000", 2023-09-23T19:37:33.146 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000109597 2023-09-23T19:37:33.146 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.146 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.146 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:37:33.146 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.559238+0000", 2023-09-23T19:37:33.146 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0028652090000000001 2023-09-23T19:37:33.146 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.146 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.146 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:37:33.146 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.560945+0000", 2023-09-23T19:37:33.147 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001707482 2023-09-23T19:37:33.147 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.147 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.147 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:37:33.147 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.750505+0000", 2023-09-23T19:37:33.147 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.18955966899999999 2023-09-23T19:37:33.147 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.147 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.147 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:37:33.147 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.750577+0000", 2023-09-23T19:37:33.147 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.2096999999999998e-05 2023-09-23T19:37:33.147 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.147 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.148 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:37:33.148 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.750635+0000", 2023-09-23T19:37:33.148 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.8183999999999997e-05 2023-09-23T19:37:33.148 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:37:33.148 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:37:33.148 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:37:33.148 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.148 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.148 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45671.0:81657 9.9 9:9b696463:::benchmark_data_smithi016_113233_object81656:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e125)", 2023-09-23T19:37:33.148 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:32:32.556062+0000", 2023-09-23T19:37:33.148 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 300.55428689600001, 2023-09-23T19:37:33.148 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.194812766, 2023-09-23T19:37:33.149 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:37:33.149 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:37:33.149 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:37:33.149 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45671", 2023-09-23T19:37:33.149 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:37:33.149 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 81657 2023-09-23T19:37:33.149 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.149 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:37:33.149 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.149 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:37:33.149 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.556062+0000", 2023-09-23T19:37:33.149 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:37:33.149 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.150 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.150 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:37:33.150 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.556062+0000", 2023-09-23T19:37:33.150 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:37:33.150 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.150 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.150 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:37:33.150 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.556062+0000", 2023-09-23T19:37:33.150 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T19:37:33.150 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.150 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.151 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:37:33.151 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.556088+0000", 2023-09-23T19:37:33.151 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.5843e-05 2023-09-23T19:37:33.151 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.151 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.151 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:37:33.151 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.556096+0000", 2023-09-23T19:37:33.151 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.9999999999999996e-06 2023-09-23T19:37:33.151 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.151 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.151 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:37:33.151 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.556102+0000", 2023-09-23T19:37:33.152 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.5780000000000004e-06 2023-09-23T19:37:33.152 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.152 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.152 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:37:33.152 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.556152+0000", 2023-09-23T19:37:33.152 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.9453999999999998e-05 2023-09-23T19:37:33.152 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.152 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.152 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:37:33.152 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.556289+0000", 2023-09-23T19:37:33.152 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00013696300000000001 2023-09-23T19:37:33.153 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.153 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.153 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 5,7", 2023-09-23T19:37:33.153 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.556422+0000", 2023-09-23T19:37:33.153 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00013310199999999999 2023-09-23T19:37:33.153 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.153 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.153 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:37:33.153 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.559800+0000", 2023-09-23T19:37:33.153 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0033779220000000002 2023-09-23T19:37:33.153 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.154 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.154 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:37:33.154 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.750689+0000", 2023-09-23T19:37:33.154 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.190889164 2023-09-23T19:37:33.154 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.154 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.154 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:37:33.154 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.750772+0000", 2023-09-23T19:37:33.154 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.3584000000000005e-05 2023-09-23T19:37:33.154 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.154 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.155 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:37:33.155 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.750820+0000", 2023-09-23T19:37:33.155 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.7422000000000001e-05 2023-09-23T19:37:33.155 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.155 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.155 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:37:33.155 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.750875+0000", 2023-09-23T19:37:33.155 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.5305e-05 2023-09-23T19:37:33.155 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:37:33.155 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:37:33.155 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:37:33.155 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.156 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.156 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45671.0:81643 9.8 e125/119 9:18565bb9:::benchmark_data_smithi016_113233_object81642:head v 125'4970, mlcod=125'4963)", 2023-09-23T19:37:33.156 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:32:32.560496+0000", 2023-09-23T19:37:33.156 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 300.54985294699998, 2023-09-23T19:37:33.156 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.19010199999999999, 2023-09-23T19:37:33.156 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:37:33.156 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:37:33.156 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:37:33.156 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.156 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:37:33.157 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.560496+0000", 2023-09-23T19:37:33.157 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:37:33.157 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.157 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.157 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:37:33.157 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.560496+0000", 2023-09-23T19:37:33.157 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:37:33.157 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.157 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.157 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:37:33.157 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.560495+0000", 2023-09-23T19:37:33.158 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:37:33.158 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.158 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.158 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:37:33.158 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.560526+0000", 2023-09-23T19:37:33.158 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.0134999999999999e-05 2023-09-23T19:37:33.158 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.158 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.158 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:37:33.158 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.560538+0000", 2023-09-23T19:37:33.158 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2747e-05 2023-09-23T19:37:33.159 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.159 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.159 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:37:33.159 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.560547+0000", 2023-09-23T19:37:33.159 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.0259999999999996e-06 2023-09-23T19:37:33.159 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.159 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.159 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:37:33.159 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.560592+0000", 2023-09-23T19:37:33.159 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.4236999999999997e-05 2023-09-23T19:37:33.159 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.160 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.160 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:37:33.160 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.560612+0000", 2023-09-23T19:37:33.160 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.0052999999999999e-05 2023-09-23T19:37:33.160 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.160 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.160 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:37:33.160 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.750574+0000", 2023-09-23T19:37:33.160 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.18996205099999999 2023-09-23T19:37:33.160 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.160 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.160 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:37:33.160 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.750598+0000", 2023-09-23T19:37:33.161 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.4499999999999999e-05 2023-09-23T19:37:33.161 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:37:33.161 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:37:33.161 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:37:33.161 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.161 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.161 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop_reply(client.45671.0:81655 9.b e125/119 ondisk, result = 0)", 2023-09-23T19:37:33.161 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:32:32.560845+0000", 2023-09-23T19:37:33.161 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 300.549504082, 2023-09-23T19:37:33.161 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.18981118399999999, 2023-09-23T19:37:33.161 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:37:33.161 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "started", 2023-09-23T19:37:33.162 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:37:33.162 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.162 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:37:33.162 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.560845+0000", 2023-09-23T19:37:33.162 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:37:33.162 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.162 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.162 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:37:33.162 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.560845+0000", 2023-09-23T19:37:33.162 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:37:33.162 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.163 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.163 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:37:33.163 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.560844+0000", 2023-09-23T19:37:33.163 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T19:37:33.163 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.163 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.163 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:37:33.163 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.560850+0000", 2023-09-23T19:37:33.163 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.2460000000000003e-06 2023-09-23T19:37:33.163 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.163 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.163 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:37:33.164 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.560860+0000", 2023-09-23T19:37:33.164 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.043e-05 2023-09-23T19:37:33.164 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.164 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.164 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:37:33.164 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.560868+0000", 2023-09-23T19:37:33.164 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.7109999999999995e-06 2023-09-23T19:37:33.164 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.164 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.164 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:37:33.164 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.750483+0000", 2023-09-23T19:37:33.164 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.18961492699999999 2023-09-23T19:37:33.165 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.165 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.165 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:37:33.165 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.750497+0000", 2023-09-23T19:37:33.165 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3788000000000001e-05 2023-09-23T19:37:33.165 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.165 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.165 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:37:33.165 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.750656+0000", 2023-09-23T19:37:33.165 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00015973800000000001 2023-09-23T19:37:33.165 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:37:33.166 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:37:33.166 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:37:33.166 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.166 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.166 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop_reply(client.45671.0:81657 9.9 e125/119 ondisk, result = 0)", 2023-09-23T19:37:33.166 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:32:32.560955+0000", 2023-09-23T19:37:33.166 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 300.54939442800003, 2023-09-23T19:37:33.166 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.18993455100000001, 2023-09-23T19:37:33.166 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:37:33.166 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "started", 2023-09-23T19:37:33.166 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:37:33.166 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.167 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:37:33.167 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.560955+0000", 2023-09-23T19:37:33.167 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:37:33.167 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.167 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.167 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:37:33.167 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.560955+0000", 2023-09-23T19:37:33.167 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:37:33.167 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.167 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.167 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:37:33.168 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.560954+0000", 2023-09-23T19:37:33.168 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:37:33.168 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.168 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.168 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:37:33.168 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.560960+0000", 2023-09-23T19:37:33.168 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.6319999999999996e-06 2023-09-23T19:37:33.168 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.168 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.168 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:37:33.168 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.750500+0000", 2023-09-23T19:37:33.168 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.18954022100000001 2023-09-23T19:37:33.169 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.169 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.169 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:37:33.169 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.750512+0000", 2023-09-23T19:37:33.169 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1995e-05 2023-09-23T19:37:33.169 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.169 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.169 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:37:33.169 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.750745+0000", 2023-09-23T19:37:33.169 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00023374100000000001 2023-09-23T19:37:33.169 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.169 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.170 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:37:33.170 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.750767+0000", 2023-09-23T19:37:33.170 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.1061e-05 2023-09-23T19:37:33.170 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.170 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.170 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:37:33.170 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.750889+0000", 2023-09-23T19:37:33.170 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000122714 2023-09-23T19:37:33.170 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:37:33.170 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:37:33.170 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:37:33.171 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.171 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.171 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45671.0:81663 9.5 9:aa237e9a:::benchmark_data_smithi016_113233_object81662:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e125)", 2023-09-23T19:37:33.171 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:32:32.561020+0000", 2023-09-23T19:37:33.171 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 300.54932874799999, 2023-09-23T19:37:33.171 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.20284819500000001, 2023-09-23T19:37:33.171 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:37:33.171 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:37:33.171 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:37:33.171 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45671", 2023-09-23T19:37:33.171 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:37:33.171 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 81663 2023-09-23T19:37:33.172 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.172 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:37:33.172 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.172 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:37:33.172 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.561020+0000", 2023-09-23T19:37:33.172 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:37:33.172 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.172 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.172 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:37:33.172 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.561020+0000", 2023-09-23T19:37:33.172 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:37:33.172 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.173 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.173 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:37:33.173 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.561019+0000", 2023-09-23T19:37:33.173 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-09-23T19:37:33.173 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.173 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.173 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:37:33.173 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.561047+0000", 2023-09-23T19:37:33.173 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.8773999999999999e-05 2023-09-23T19:37:33.173 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.173 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.173 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:37:33.174 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.750516+0000", 2023-09-23T19:37:33.174 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.189469045 2023-09-23T19:37:33.174 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.174 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.174 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:37:33.174 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.750540+0000", 2023-09-23T19:37:33.174 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.3887e-05 2023-09-23T19:37:33.174 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.174 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.174 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:37:33.175 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.750731+0000", 2023-09-23T19:37:33.175 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000191034 2023-09-23T19:37:33.175 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.175 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.175 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:37:33.175 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.750990+0000", 2023-09-23T19:37:33.175 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000258273 2023-09-23T19:37:33.175 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.175 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.175 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 5,7", 2023-09-23T19:37:33.175 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.751178+0000", 2023-09-23T19:37:33.175 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000188791 2023-09-23T19:37:33.176 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.176 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.176 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:37:33.176 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.759196+0000", 2023-09-23T19:37:33.176 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0080173769999999991 2023-09-23T19:37:33.176 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.176 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.176 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:37:33.176 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.762722+0000", 2023-09-23T19:37:33.176 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0035265560000000001 2023-09-23T19:37:33.176 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.177 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.177 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:37:33.177 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.763769+0000", 2023-09-23T19:37:33.177 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001046548 2023-09-23T19:37:33.177 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.177 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.177 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:37:33.177 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.763825+0000", 2023-09-23T19:37:33.177 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.6638999999999998e-05 2023-09-23T19:37:33.177 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.177 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.178 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:37:33.178 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.763869+0000", 2023-09-23T19:37:33.178 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.3099000000000002e-05 2023-09-23T19:37:33.178 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:37:33.178 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:37:33.178 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:37:33.178 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.178 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.178 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop_reply(client.45671.0:81657 9.9 e125/119 ondisk, result = 0)", 2023-09-23T19:37:33.179 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:32:32.566008+0000", 2023-09-23T19:37:33.179 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 300.54434127600001, 2023-09-23T19:37:33.179 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.184686036, 2023-09-23T19:37:33.179 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:37:33.179 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "started", 2023-09-23T19:37:33.179 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:37:33.179 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.179 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:37:33.179 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.566008+0000", 2023-09-23T19:37:33.179 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:37:33.179 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.180 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.180 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:37:33.180 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.566008+0000", 2023-09-23T19:37:33.180 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:37:33.180 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.180 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.180 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:37:33.180 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.566007+0000", 2023-09-23T19:37:33.180 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:37:33.180 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.180 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.181 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:37:33.181 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.566014+0000", 2023-09-23T19:37:33.181 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.5989999999999996e-06 2023-09-23T19:37:33.181 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.181 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.181 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:37:33.182 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.750447+0000", 2023-09-23T19:37:33.182 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.18443309399999999 2023-09-23T19:37:33.182 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.182 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.182 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:37:33.182 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.750470+0000", 2023-09-23T19:37:33.182 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.3006e-05 2023-09-23T19:37:33.182 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.182 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.182 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:37:33.183 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.750672+0000", 2023-09-23T19:37:33.183 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00020127099999999999 2023-09-23T19:37:33.183 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.183 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.183 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:37:33.183 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.750681+0000", 2023-09-23T19:37:33.183 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.0040000000000005e-06 2023-09-23T19:37:33.183 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.183 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.183 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:37:33.183 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:32:32.750694+0000", 2023-09-23T19:37:33.184 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3179999999999999e-05 2023-09-23T19:37:33.184 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:37:33.184 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:37:33.184 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:37:33.184 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.184 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.185 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45671.0:147000 9.9 9:98251214:::benchmark_data_smithi016_113233_object38618:head [delete] snapc 0=[] ondisk+write+known_if_redirected e130)", 2023-09-23T19:37:33.185 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:33:54.995424+0000", 2023-09-23T19:37:33.185 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 218.11492518899999, 2023-09-23T19:37:33.185 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.158760754, 2023-09-23T19:37:33.185 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:37:33.186 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:37:33.186 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:37:33.186 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45671", 2023-09-23T19:37:33.186 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:37:33.186 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 147000 2023-09-23T19:37:33.186 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.186 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:37:33.186 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.186 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:37:33.186 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:54.995424+0000", 2023-09-23T19:37:33.186 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:37:33.187 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.187 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.187 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:37:33.187 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:54.995424+0000", 2023-09-23T19:37:33.187 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:37:33.187 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.187 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.187 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:37:33.187 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:54.995422+0000", 2023-09-23T19:37:33.187 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-09-23T19:37:33.187 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.188 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.188 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:37:33.188 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:54.995434+0000", 2023-09-23T19:37:33.188 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1185999999999999e-05 2023-09-23T19:37:33.188 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.188 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.188 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:37:33.188 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:54.995445+0000", 2023-09-23T19:37:33.188 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1967000000000001e-05 2023-09-23T19:37:33.188 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.188 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.188 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:37:33.189 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:54.995453+0000", 2023-09-23T19:37:33.189 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.909e-06 2023-09-23T19:37:33.189 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.189 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.189 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:37:33.189 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:54.995499+0000", 2023-09-23T19:37:33.189 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.5513000000000003e-05 2023-09-23T19:37:33.189 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.189 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.189 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:37:33.189 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:54.995685+0000", 2023-09-23T19:37:33.189 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000185663 2023-09-23T19:37:33.190 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.190 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.190 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 5,7", 2023-09-23T19:37:33.190 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:54.995814+0000", 2023-09-23T19:37:33.190 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00012978299999999999 2023-09-23T19:37:33.190 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.190 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.190 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:37:33.190 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:55.001310+0000", 2023-09-23T19:37:33.190 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.005495214 2023-09-23T19:37:33.190 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.190 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.191 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:37:33.191 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:55.008855+0000", 2023-09-23T19:37:33.191 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0075455870000000003 2023-09-23T19:37:33.191 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.191 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.191 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:37:33.191 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:55.154042+0000", 2023-09-23T19:37:33.191 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.14518662199999999 2023-09-23T19:37:33.191 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.191 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.191 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:37:33.191 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:55.154139+0000", 2023-09-23T19:37:33.192 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.7114000000000006e-05 2023-09-23T19:37:33.192 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.192 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.192 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:37:33.192 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:55.154185+0000", 2023-09-23T19:37:33.192 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.5822000000000003e-05 2023-09-23T19:37:33.192 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:37:33.192 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:37:33.192 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:37:33.192 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.192 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.192 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45671.0:147018 9.9 9:91a8f3c5:::benchmark_data_smithi016_113233_object38636:head [delete] snapc 0=[] ondisk+write+known_if_redirected e130)", 2023-09-23T19:37:33.193 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:33:55.020631+0000", 2023-09-23T19:37:33.193 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 218.08971835599999, 2023-09-23T19:37:33.193 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.156368123, 2023-09-23T19:37:33.193 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:37:33.193 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:37:33.193 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:37:33.193 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45671", 2023-09-23T19:37:33.193 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:37:33.193 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 147018 2023-09-23T19:37:33.193 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.193 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:37:33.193 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.193 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:37:33.194 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:55.020631+0000", 2023-09-23T19:37:33.194 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:37:33.194 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.194 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.194 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:37:33.194 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:55.020631+0000", 2023-09-23T19:37:33.194 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:37:33.194 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.194 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.194 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:37:33.194 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:55.020630+0000", 2023-09-23T19:37:33.195 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:37:33.195 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.195 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.195 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:37:33.195 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:55.020635+0000", 2023-09-23T19:37:33.195 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.763e-06 2023-09-23T19:37:33.195 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.195 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.195 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:37:33.195 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:55.020644+0000", 2023-09-23T19:37:33.195 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.3270000000000007e-06 2023-09-23T19:37:33.195 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.195 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.196 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:37:33.196 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:55.020651+0000", 2023-09-23T19:37:33.196 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.9110000000000004e-06 2023-09-23T19:37:33.196 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.196 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.196 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:37:33.196 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:55.154583+0000", 2023-09-23T19:37:33.196 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.13393206699999999 2023-09-23T19:37:33.196 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.196 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.196 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:37:33.196 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:55.154718+0000", 2023-09-23T19:37:33.197 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00013544499999999999 2023-09-23T19:37:33.197 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.197 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.197 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 5,7", 2023-09-23T19:37:33.197 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:55.154838+0000", 2023-09-23T19:37:33.197 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000119305 2023-09-23T19:37:33.197 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.197 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.197 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:37:33.197 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:55.165165+0000", 2023-09-23T19:37:33.197 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.010327184999999999 2023-09-23T19:37:33.197 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.198 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.198 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:37:33.198 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:55.171013+0000", 2023-09-23T19:37:33.198 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0058484449999999999 2023-09-23T19:37:33.198 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.198 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.198 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:37:33.198 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:55.176861+0000", 2023-09-23T19:37:33.198 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0058476819999999999 2023-09-23T19:37:33.198 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.198 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.198 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:37:33.198 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:55.176933+0000", 2023-09-23T19:37:33.199 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.2070000000000006e-05 2023-09-23T19:37:33.199 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.199 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.199 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:37:33.199 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:33:55.176999+0000", 2023-09-23T19:37:33.199 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.5763e-05 2023-09-23T19:37:33.199 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:37:33.199 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:37:33.199 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:37:33.199 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.199 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.199 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45947.0:18774 10.4 10:2c86c7c7:::benchmark_data_smithi016_136365_object18773:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e149)", 2023-09-23T19:37:33.200 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:36:43.986783+0000", 2023-09-23T19:37:33.200 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 49.123566322999999, 2023-09-23T19:37:33.200 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.15847433799999999, 2023-09-23T19:37:33.200 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:37:33.200 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:37:33.200 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:37:33.200 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45947", 2023-09-23T19:37:33.200 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/4039191752", 2023-09-23T19:37:33.200 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 18774 2023-09-23T19:37:33.200 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.200 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:37:33.200 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.200 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:37:33.201 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.986783+0000", 2023-09-23T19:37:33.201 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:37:33.201 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.201 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.201 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:37:33.201 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.986783+0000", 2023-09-23T19:37:33.201 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:37:33.201 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.201 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.201 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:37:33.201 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.986782+0000", 2023-09-23T19:37:33.201 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:37:33.202 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.202 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.202 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:37:33.202 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.986802+0000", 2023-09-23T19:37:33.202 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.0472e-05 2023-09-23T19:37:33.202 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.202 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.202 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:37:33.202 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.986819+0000", 2023-09-23T19:37:33.202 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.6663e-05 2023-09-23T19:37:33.202 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.202 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.202 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:37:33.203 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.986824+0000", 2023-09-23T19:37:33.203 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.3669999999999996e-06 2023-09-23T19:37:33.203 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.203 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.203 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:37:33.203 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.986871+0000", 2023-09-23T19:37:33.203 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.6731999999999999e-05 2023-09-23T19:37:33.203 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.203 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.203 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:37:33.203 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.987001+0000", 2023-09-23T19:37:33.203 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00013023200000000001 2023-09-23T19:37:33.204 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.204 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.204 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 3,4", 2023-09-23T19:37:33.204 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.987142+0000", 2023-09-23T19:37:33.204 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00014108100000000001 2023-09-23T19:37:33.204 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.204 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.204 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:37:33.204 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.988466+0000", 2023-09-23T19:37:33.204 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0013232160000000001 2023-09-23T19:37:33.204 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.204 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.204 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:37:33.205 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.988501+0000", 2023-09-23T19:37:33.205 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.5858999999999997e-05 2023-09-23T19:37:33.205 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.205 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.205 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:37:33.205 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:44.145154+0000", 2023-09-23T19:37:33.205 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.15665296400000001 2023-09-23T19:37:33.205 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.205 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.205 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:37:33.205 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:44.145213+0000", 2023-09-23T19:37:33.205 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.8631999999999997e-05 2023-09-23T19:37:33.206 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.206 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.206 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:37:33.206 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:44.145257+0000", 2023-09-23T19:37:33.206 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.4156999999999998e-05 2023-09-23T19:37:33.206 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:37:33.206 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:37:33.206 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:37:33.206 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.206 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.206 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45947.0:18779 10.6 e149/144 10:654a3fe0:::benchmark_data_smithi016_136365_object18778:head v 149'1163, mlcod=149'1161)", 2023-09-23T19:37:33.206 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:36:43.992501+0000", 2023-09-23T19:37:33.207 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 49.117848039999998, 2023-09-23T19:37:33.207 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.15106165299999999, 2023-09-23T19:37:33.207 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:37:33.207 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:37:33.207 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:37:33.207 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.207 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:37:33.207 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.992501+0000", 2023-09-23T19:37:33.207 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:37:33.207 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.207 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.207 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:37:33.208 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.992501+0000", 2023-09-23T19:37:33.208 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:37:33.208 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.208 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.208 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:37:33.208 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.992487+0000", 2023-09-23T19:37:33.208 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999862 2023-09-23T19:37:33.208 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.208 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.208 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:37:33.208 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.992556+0000", 2023-09-23T19:37:33.208 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.8695000000000006e-05 2023-09-23T19:37:33.209 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.209 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.209 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:37:33.209 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.992567+0000", 2023-09-23T19:37:33.209 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1263999999999999e-05 2023-09-23T19:37:33.209 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.209 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.209 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:37:33.209 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.992574+0000", 2023-09-23T19:37:33.209 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.2690000000000002e-06 2023-09-23T19:37:33.209 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.209 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.210 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:37:33.210 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.992620+0000", 2023-09-23T19:37:33.210 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.6106999999999997e-05 2023-09-23T19:37:33.210 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.210 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.210 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:37:33.210 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.992640+0000", 2023-09-23T19:37:33.210 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.9949000000000002e-05 2023-09-23T19:37:33.210 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.210 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.211 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:37:33.211 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:44.143512+0000", 2023-09-23T19:37:33.211 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.150871904 2023-09-23T19:37:33.211 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.211 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.211 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:37:33.211 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:44.143563+0000", 2023-09-23T19:37:33.211 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.0445999999999998e-05 2023-09-23T19:37:33.211 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:37:33.211 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:37:33.211 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:37:33.211 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.212 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.212 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45947.0:18782 10.6 e149/144 10:658265a1:::benchmark_data_smithi016_136365_object18781:head v 149'1164, mlcod=149'1161)", 2023-09-23T19:37:33.212 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:36:43.992590+0000", 2023-09-23T19:37:33.212 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 49.117759126999999, 2023-09-23T19:37:33.212 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.151287544, 2023-09-23T19:37:33.212 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:37:33.212 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:37:33.212 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:37:33.212 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.212 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:37:33.212 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.992590+0000", 2023-09-23T19:37:33.212 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:37:33.213 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.213 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.213 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:37:33.213 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.992590+0000", 2023-09-23T19:37:33.213 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:37:33.213 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.213 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.213 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:37:33.213 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.992590+0000", 2023-09-23T19:37:33.213 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T19:37:33.213 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.213 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.213 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:37:33.214 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.992615+0000", 2023-09-23T19:37:33.214 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.4995999999999999e-05 2023-09-23T19:37:33.214 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.214 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.214 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:37:33.214 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.992621+0000", 2023-09-23T19:37:33.214 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.8460000000000002e-06 2023-09-23T19:37:33.214 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.214 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.214 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:37:33.214 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.992627+0000", 2023-09-23T19:37:33.214 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.3759999999999999e-06 2023-09-23T19:37:33.215 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.215 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.215 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:37:33.215 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.993278+0000", 2023-09-23T19:37:33.215 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00065142900000000003 2023-09-23T19:37:33.215 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.215 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.215 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:37:33.215 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.993290+0000", 2023-09-23T19:37:33.215 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1521e-05 2023-09-23T19:37:33.215 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.215 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.216 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:37:33.216 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:44.143856+0000", 2023-09-23T19:37:33.216 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.15056660999999999 2023-09-23T19:37:33.216 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.216 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.216 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:37:33.216 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:44.143878+0000", 2023-09-23T19:37:33.216 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.1154000000000001e-05 2023-09-23T19:37:33.216 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:37:33.216 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:37:33.216 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:37:33.216 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.217 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.217 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45947.0:18785 10.5 e149/144 10:a9530852:::benchmark_data_smithi016_136365_object18784:head v 149'1169, mlcod=149'1166)", 2023-09-23T19:37:33.217 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:36:43.992617+0000", 2023-09-23T19:37:33.217 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 49.117731679999999, 2023-09-23T19:37:33.217 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.151079088, 2023-09-23T19:37:33.217 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:37:33.217 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:37:33.217 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:37:33.217 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.217 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:37:33.217 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.992617+0000", 2023-09-23T19:37:33.217 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:37:33.218 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.218 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.218 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:37:33.218 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.992617+0000", 2023-09-23T19:37:33.218 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:37:33.218 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.218 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.218 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:37:33.218 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.992617+0000", 2023-09-23T19:37:33.218 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T19:37:33.218 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.218 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.219 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:37:33.219 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.992638+0000", 2023-09-23T19:37:33.219 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.1373e-05 2023-09-23T19:37:33.219 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.219 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.219 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:37:33.219 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.992646+0000", 2023-09-23T19:37:33.219 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.2290000000000003e-06 2023-09-23T19:37:33.219 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.219 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.219 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:37:33.219 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.992655+0000", 2023-09-23T19:37:33.219 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.5580000000000001e-06 2023-09-23T19:37:33.220 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.220 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.220 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:37:33.220 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.992694+0000", 2023-09-23T19:37:33.220 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.8906000000000003e-05 2023-09-23T19:37:33.220 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.220 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.220 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:37:33.220 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.992714+0000", 2023-09-23T19:37:33.220 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.9896999999999999e-05 2023-09-23T19:37:33.220 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.220 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.221 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:37:33.221 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.993436+0000", 2023-09-23T19:37:33.221 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00072195500000000004 2023-09-23T19:37:33.221 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.221 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.221 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:37:33.221 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:44.143697+0000", 2023-09-23T19:37:33.221 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.150260739 2023-09-23T19:37:33.221 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:37:33.221 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:37:33.221 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:37:33.222 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.222 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.222 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45947.0:18789 10.4 10:2981e255:::benchmark_data_smithi016_136365_object18788:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e149)", 2023-09-23T19:37:33.222 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:36:43.993117+0000", 2023-09-23T19:37:33.222 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 49.117232540000003, 2023-09-23T19:37:33.222 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.154935991, 2023-09-23T19:37:33.222 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:37:33.222 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:37:33.222 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:37:33.222 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45947", 2023-09-23T19:37:33.222 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/4039191752", 2023-09-23T19:37:33.222 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 18789 2023-09-23T19:37:33.223 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.223 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:37:33.223 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.223 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:37:33.223 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.993117+0000", 2023-09-23T19:37:33.223 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:37:33.224 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.224 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.224 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:37:33.224 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.993117+0000", 2023-09-23T19:37:33.224 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:37:33.224 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.224 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.225 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:37:33.225 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.993115+0000", 2023-09-23T19:37:33.225 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:37:33.225 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.225 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.225 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:37:33.225 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.993144+0000", 2023-09-23T19:37:33.226 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.8799e-05 2023-09-23T19:37:33.226 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.226 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.226 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:37:33.226 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.993153+0000", 2023-09-23T19:37:33.226 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.0669999999999998e-06 2023-09-23T19:37:33.227 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.227 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.227 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:37:33.227 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.993159+0000", 2023-09-23T19:37:33.227 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.5219999999999998e-06 2023-09-23T19:37:33.227 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.227 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.228 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:37:33.228 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.993191+0000", 2023-09-23T19:37:33.228 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.2503999999999999e-05 2023-09-23T19:37:33.228 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.228 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.228 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:37:33.228 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:44.143608+0000", 2023-09-23T19:37:33.229 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.15041691400000001 2023-09-23T19:37:33.229 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.229 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.229 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 3,4", 2023-09-23T19:37:33.229 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:44.143775+0000", 2023-09-23T19:37:33.229 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000166637 2023-09-23T19:37:33.229 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.230 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.230 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:37:33.230 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:44.145363+0000", 2023-09-23T19:37:33.230 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0015877070000000001 2023-09-23T19:37:33.230 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.230 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.231 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:37:33.231 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:44.145859+0000", 2023-09-23T19:37:33.231 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000496625 2023-09-23T19:37:33.231 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.231 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.231 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:37:33.231 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:44.147972+0000", 2023-09-23T19:37:33.232 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0021126840000000001 2023-09-23T19:37:33.232 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.232 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.232 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:37:33.232 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:44.148021+0000", 2023-09-23T19:37:33.232 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.9344000000000001e-05 2023-09-23T19:37:33.232 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.233 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.233 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:37:33.233 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:44.148053+0000", 2023-09-23T19:37:33.233 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.1355e-05 2023-09-23T19:37:33.233 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:37:33.233 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:37:33.233 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:37:33.234 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.234 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.234 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45947.0:18792 10.c e149/144 10:3ca0785b:::benchmark_data_smithi016_136365_object18791:head v 149'1201, mlcod=149'1198)", 2023-09-23T19:37:33.234 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:36:43.994308+0000", 2023-09-23T19:37:33.234 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 49.116040722000001, 2023-09-23T19:37:33.234 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.15077327500000001, 2023-09-23T19:37:33.235 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:37:33.235 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:37:33.235 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:37:33.235 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.235 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:37:33.235 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.994308+0000", 2023-09-23T19:37:33.235 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:37:33.236 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.236 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.236 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:37:33.236 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.994308+0000", 2023-09-23T19:37:33.236 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:37:33.236 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.237 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.237 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:37:33.237 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.994307+0000", 2023-09-23T19:37:33.237 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:37:33.237 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.237 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.237 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:37:33.238 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.994338+0000", 2023-09-23T19:37:33.238 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.0917999999999998e-05 2023-09-23T19:37:33.238 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.238 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.238 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:37:33.238 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:44.142017+0000", 2023-09-23T19:37:33.238 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.14767894000000001 2023-09-23T19:37:33.239 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.239 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.239 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:37:33.239 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:44.142028+0000", 2023-09-23T19:37:33.239 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1078e-05 2023-09-23T19:37:33.239 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.239 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.239 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:37:33.239 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:44.142654+0000", 2023-09-23T19:37:33.240 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00062544200000000003 2023-09-23T19:37:33.240 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.240 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.240 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:37:33.240 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:44.142674+0000", 2023-09-23T19:37:33.240 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.0171000000000001e-05 2023-09-23T19:37:33.240 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.240 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.240 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:37:33.240 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:44.145042+0000", 2023-09-23T19:37:33.240 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.002367933 2023-09-23T19:37:33.240 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.240 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.241 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:37:33.241 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:44.145082+0000", 2023-09-23T19:37:33.241 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.9737000000000003e-05 2023-09-23T19:37:33.241 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:37:33.241 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:37:33.241 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:37:33.241 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.241 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.242 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45947.0:18788 10.1 e149/144 10:87c8449a:::benchmark_data_smithi016_136365_object18787:head v 149'1182, mlcod=149'1180)", 2023-09-23T19:37:33.242 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:36:43.994631+0000", 2023-09-23T19:37:33.242 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 49.11571833, 2023-09-23T19:37:33.242 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.151274665, 2023-09-23T19:37:33.242 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:37:33.242 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:37:33.242 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:37:33.243 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.243 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:37:33.243 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.994631+0000", 2023-09-23T19:37:33.243 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:37:33.243 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.243 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.244 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:37:33.244 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.994631+0000", 2023-09-23T19:37:33.244 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:37:33.244 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.244 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.244 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:37:33.244 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.994630+0000", 2023-09-23T19:37:33.245 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:37:33.245 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.245 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.245 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:37:33.245 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.994692+0000", 2023-09-23T19:37:33.245 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.2263999999999999e-05 2023-09-23T19:37:33.246 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.246 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.246 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:37:33.246 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:44.143523+0000", 2023-09-23T19:37:33.246 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.14883057699999999 2023-09-23T19:37:33.246 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.247 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.247 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:37:33.247 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:44.143532+0000", 2023-09-23T19:37:33.247 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.6389999999999998e-06 2023-09-23T19:37:33.247 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.247 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.248 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:37:33.248 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:44.143598+0000", 2023-09-23T19:37:33.248 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.6691000000000002e-05 2023-09-23T19:37:33.248 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.248 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.248 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:37:33.248 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:44.143620+0000", 2023-09-23T19:37:33.249 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.2038000000000001e-05 2023-09-23T19:37:33.249 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.249 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.249 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:37:33.249 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:44.145840+0000", 2023-09-23T19:37:33.249 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00222018 2023-09-23T19:37:33.249 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.250 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.250 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:37:33.250 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:44.145905+0000", 2023-09-23T19:37:33.250 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.5015e-05 2023-09-23T19:37:33.250 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:37:33.250 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:37:33.250 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:37:33.251 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.251 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.251 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45947.0:79825 10.a e152/144 10:52f61f88:::benchmark_data_smithi016_136365_object79824:head v 152'4960, mlcod=152'4958)", 2023-09-23T19:37:33.251 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:37:30.726494+0000", 2023-09-23T19:37:33.251 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 2.383854651, 2023-09-23T19:37:33.251 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.207575123, 2023-09-23T19:37:33.252 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:37:33.252 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:37:33.252 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:37:33.252 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.252 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:37:33.252 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.726494+0000", 2023-09-23T19:37:33.252 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:37:33.253 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.253 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.253 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:37:33.253 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.726494+0000", 2023-09-23T19:37:33.253 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:37:33.253 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.253 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.254 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:37:33.254 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.726494+0000", 2023-09-23T19:37:33.254 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:37:33.254 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.254 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.254 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:37:33.255 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.726549+0000", 2023-09-23T19:37:33.255 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.5327000000000001e-05 2023-09-23T19:37:33.255 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.255 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.255 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:37:33.255 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.796059+0000", 2023-09-23T19:37:33.255 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.069509940000000006 2023-09-23T19:37:33.256 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.256 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.256 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:37:33.256 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.796078+0000", 2023-09-23T19:37:33.256 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.8663000000000001e-05 2023-09-23T19:37:33.256 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.257 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.257 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:37:33.257 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.796252+0000", 2023-09-23T19:37:33.257 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000174122 2023-09-23T19:37:33.257 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.257 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.257 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:37:33.258 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.796341+0000", 2023-09-23T19:37:33.258 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.8912000000000005e-05 2023-09-23T19:37:33.258 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.258 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.258 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:37:33.258 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.934017+0000", 2023-09-23T19:37:33.259 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.13767655000000001 2023-09-23T19:37:33.259 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:37:33.259 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:37:33.259 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:37:33.259 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.934070+0000", 2023-09-23T19:37:33.259 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.2340999999999999e-05 2023-09-23T19:37:33.259 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:37:33.260 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:37:33.260 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:37:33.260 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:37:33.260 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:37:33.260 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-09-23T19:37:33.261 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:33 smithi016 bash[75004]: cluster 2023-09-23T19:37:32.091417+0000 mgr.y (mgr.25325) 900 : cluster 0 pgmap v824: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 965 op/s 2023-09-23T19:37:33.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:33.262 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:33 smithi016 bash[77995]: cluster 2023-09-23T19:37:32.091417+0000 mgr.y (mgr.25325) 900 : cluster 0 pgmap v824: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 965 op/s 2023-09-23T19:37:33.263 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:33.263 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:37:33.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:33 smithi092 bash[58855]: cluster 2023-09-23T19:37:32.091417+0000 mgr.y (mgr.25325) 900 : cluster 0 pgmap v824: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 965 op/s 2023-09-23T19:37:33.364 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:33.388 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:33.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:33.391 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:33.490 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 58 16 82247 82231 88.6013 60 0.0131855 0.01128 2023-09-23T19:37:33.492 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:33.520 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:33.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:33.523 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:33.624 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:33.647 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:33.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:33.649 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:37:33.749 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:33.779 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:33.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:33.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:33.882 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:33.909 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:33.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:33.912 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:37:34.012 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:34.039 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:34.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:34.041 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:37:34.141 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:34.176 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:34.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:34.181 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:34.282 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:34.308 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:34.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:34.313 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:37:34.414 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:34.435 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:34.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:34.439 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:34.490 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 59 16 83195 83179 88.1037 59.25 0.0198557 0.0113447 2023-09-23T19:37:34.540 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:34.573 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:34.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:34.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:37:34.676 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:34.699 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:34.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:34.701 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:37:34.802 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:34.829 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:34.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:34.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:37:34.932 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:34.962 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:34.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:34.963 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:37:35.064 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:35.096 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:35.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:35.102 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:37:35.203 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:35.236 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:35.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:35.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:37:35.339 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:35.376 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:35.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:35.382 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:35.482 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:35.490 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-09-23T19:37:35.487291+0000 min lat: 0.00156577 max lat: 0.209545 avg lat: 0.0114034 2023-09-23T19:37:35.490 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:37:35.490 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 60 16 84163 84147 87.6435 60.5 0.0107934 0.0114034 2023-09-23T19:37:35.512 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:35.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:35.515 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:37:35.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:35 smithi016 bash[75004]: cluster 2023-09-23T19:37:34.091915+0000 mgr.y (mgr.25325) 901 : cluster 0 pgmap v825: 89 pgs: 89 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 954 op/s 2023-09-23T19:37:35.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:35 smithi016 bash[77995]: cluster 2023-09-23T19:37:34.091915+0000 mgr.y (mgr.25325) 901 : cluster 0 pgmap v825: 89 pgs: 89 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 954 op/s 2023-09-23T19:37:35.616 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:35.624 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:35 smithi092 bash[58855]: cluster 2023-09-23T19:37:34.091915+0000 mgr.y (mgr.25325) 901 : cluster 0 pgmap v825: 89 pgs: 89 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 954 op/s 2023-09-23T19:37:35.652 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:35.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:35.657 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:35.757 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:35.787 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:35.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:35.792 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:35.893 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:35.926 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:35.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:35.928 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:36.029 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:36.036 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:37:35 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:37:35] "GET /metrics HTTP/1.1" 200 36303 "" "Prometheus/2.33.4" 2023-09-23T19:37:36.057 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:36.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:36.059 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:36.160 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:36.190 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:36.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:36.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:37:36.294 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:36.320 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:36.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:36.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:36.427 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:36.461 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:36.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:36.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:36.490 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 61 16 85164 85148 87.2323 62.5625 0.0115553 0.0114582 2023-09-23T19:37:36.567 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:36.592 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:36.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:36.596 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:36.697 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:36.733 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:36.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:36.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:36.836 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:36.875 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:36.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:36.884 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:36.985 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:37.014 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:37.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:37.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:37.121 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:37.150 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:37.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:37.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:37.256 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:37.286 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:37.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:37.289 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:37:37.390 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:37.423 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:37.424 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T19:37:37.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:37.427 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:37.490 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 62 16 86126 86110 86.795 60.125 0.0104257 0.0115159 2023-09-23T19:37:37.528 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:37.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:37 smithi016 bash[75004]: cluster 2023-09-23T19:37:36.092444+0000 mgr.y (mgr.25325) 902 : cluster 0 pgmap v826: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-09-23T19:37:37.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:37 smithi016 bash[77995]: cluster 2023-09-23T19:37:36.092444+0000 mgr.y (mgr.25325) 902 : cluster 0 pgmap v826: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-09-23T19:37:37.565 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:37.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:37.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:37:37.668 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:37.678 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:37 smithi092 bash[58855]: cluster 2023-09-23T19:37:36.092444+0000 mgr.y (mgr.25325) 902 : cluster 0 pgmap v826: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-09-23T19:37:37.697 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:37.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:37.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:37.800 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:37.830 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:37.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:37.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:37.935 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:37.960 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:37.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:37.965 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:37:38.066 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:38.093 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:38.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:38.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:38.199 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:38.223 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:38.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:38.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:37:38.328 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:38.358 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:38.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:38.360 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:37:38.461 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:38.486 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:38.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:38.490 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:37:38.490 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 63 16 87066 87050 86.3497 58.75 0.0100338 0.0115739 2023-09-23T19:37:38.591 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:38.630 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:38.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:38.634 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:38.734 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:38.765 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:38.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:38.768 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:38.869 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:38.894 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:38.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:38.897 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:37:38.998 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:39.029 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:39.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:39.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:39.132 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:39.161 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:39.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:39.164 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:39.265 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:39.289 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:39.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:39.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:39.395 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:39.426 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:39.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:39.428 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:39.490 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 64 16 88031 88015 85.9428 60.3125 0.023607 0.0116295 2023-09-23T19:37:39.529 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:39.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:39 smithi016 bash[75004]: cluster 2023-09-23T19:37:38.093038+0000 mgr.y (mgr.25325) 903 : cluster 0 pgmap v827: 89 pgs: 89 active+clean; 5.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 59 MiB/s wr, 936 op/s 2023-09-23T19:37:39.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:39 smithi016 bash[77995]: cluster 2023-09-23T19:37:38.093038+0000 mgr.y (mgr.25325) 903 : cluster 0 pgmap v827: 89 pgs: 89 active+clean; 5.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 59 MiB/s wr, 936 op/s 2023-09-23T19:37:39.561 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:39.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:39.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:37:39.666 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:39.690 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:39.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:39.693 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:39.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:39 smithi092 bash[58855]: cluster 2023-09-23T19:37:38.093038+0000 mgr.y (mgr.25325) 903 : cluster 0 pgmap v827: 89 pgs: 89 active+clean; 5.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 59 MiB/s wr, 936 op/s 2023-09-23T19:37:39.793 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:39.828 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:39.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:39.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:39.934 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:39.970 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:39.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:39.973 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:37:40.074 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:40.098 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:40.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:40.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:40.201 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:40.232 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:40.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:40.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:37:40.335 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:40.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:40.366 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:40.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:40.467 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:40.493 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 65 16 88976 88960 85.5292 59.0625 0.0166297 0.0116858 2023-09-23T19:37:40.496 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:40.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:40.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:37:40.605 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:40.641 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:40.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:40.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:40.744 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:40.778 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:40.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:40.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:40.884 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:40.907 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:40.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:40.912 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:37:41.012 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:41.048 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:41.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:41.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:41.152 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:41.181 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:41.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:41.185 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:41.286 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:41.316 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:41.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:41.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:37:41.420 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:41.451 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:41.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:41.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:41.491 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 66 16 89940 89924 85.1461 60.25 0.0234136 0.011739 2023-09-23T19:37:41.556 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:41.565 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:41 smithi092 bash[58855]: cluster 2023-09-23T19:37:40.093605+0000 mgr.y (mgr.25325) 904 : cluster 0 pgmap v828: 89 pgs: 89 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T19:37:41.586 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:41.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:41.589 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:41.689 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:41.717 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:41.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:41.721 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:41.742 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:41 smithi016 bash[75004]: cluster 2023-09-23T19:37:40.093605+0000 mgr.y (mgr.25325) 904 : cluster 0 pgmap v828: 89 pgs: 89 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T19:37:41.742 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:41 smithi016 bash[77995]: cluster 2023-09-23T19:37:40.093605+0000 mgr.y (mgr.25325) 904 : cluster 0 pgmap v828: 89 pgs: 89 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T19:37:41.822 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:41.846 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:41.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:41.848 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:37:41.949 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:42.007 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:42.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:42.013 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:37:42.109 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:37:42.114 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:42.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:42.143 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:42.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:42.153 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:37:42.210 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:37:42.243 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:42.267 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:37:42.276 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:42.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:42.279 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:37:42.298 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-09-23T19:37:42.302 INFO:teuthology.orchestra.run.smithi016.stderr:dumped all 2023-09-23T19:37:42.309 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:37:42.354 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:37:42.379 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:42.387 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:37:42.405 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:42.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:42.409 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:42.432 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:37:42.435 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:37:42.435 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:37:42.436 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:37:42.436 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:37:42.436 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:37:42.436 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:37:42.436 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:37:42.436 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:37:42.452 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is unset 2023-09-23T19:37:42.491 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 67 16 90701 90685 84.5851 47.5625 0.0134289 0.0118163 2023-09-23T19:37:42.509 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:42.539 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:42.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:42.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:42.643 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:42.651 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:42 smithi092 bash[58855]: audit 2023-09-23T19:37:42.049426+0000 mon.c (mon.2) 164 : audit 0 from='client.? 172.21.15.16:0/3172467308' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:37:42.651 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:42 smithi092 bash[58855]: audit 2023-09-23T19:37:42.243819+0000 mon.a (mon.0) 1115 : audit 1 from='client.? 172.21.15.16:0/3851107921' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:37:42.680 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:42.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:42.684 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:42.785 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:42.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:42 smithi016 bash[75004]: audit 2023-09-23T19:37:42.049426+0000 mon.c (mon.2) 164 : audit 0 from='client.? 172.21.15.16:0/3172467308' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:37:42.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:42 smithi016 bash[75004]: audit 2023-09-23T19:37:42.243819+0000 mon.a (mon.0) 1115 : audit 1 from='client.? 172.21.15.16:0/3851107921' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:37:42.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:42 smithi016 bash[77995]: audit 2023-09-23T19:37:42.049426+0000 mon.c (mon.2) 164 : audit 0 from='client.? 172.21.15.16:0/3172467308' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:37:42.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:42 smithi016 bash[77995]: audit 2023-09-23T19:37:42.243819+0000 mon.a (mon.0) 1115 : audit 1 from='client.? 172.21.15.16:0/3851107921' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:37:42.816 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:42.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:42.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:42.920 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:42.961 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:42.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:42.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:43.065 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:43.095 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:43.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:43.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:43.199 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:43.228 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:43.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:43.231 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:43.332 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:43.358 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:43.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:43.364 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:37:43.465 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:43.491 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 68 16 91652 91636 84.2152 59.4375 0.0443882 0.0118679 2023-09-23T19:37:43.508 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:43.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:43.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:43.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:43 smithi016 bash[75004]: cluster 2023-09-23T19:37:42.094257+0000 mgr.y (mgr.25325) 905 : cluster 0 pgmap v829: 89 pgs: 89 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 55 MiB/s wr, 884 op/s 2023-09-23T19:37:43.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:43 smithi016 bash[75004]: audit 2023-09-23T19:37:42.265349+0000 mgr.y (mgr.25325) 906 : audit 0 from='client.55573 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:37:43.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:43 smithi016 bash[75004]: cluster 2023-09-23T19:37:42.422832+0000 mon.a (mon.0) 1116 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:37:43.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:43 smithi016 bash[75004]: cluster 2023-09-23T19:37:42.422877+0000 mon.a (mon.0) 1117 : cluster 1 Cluster is now healthy 2023-09-23T19:37:43.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:43 smithi016 bash[75004]: audit 2023-09-23T19:37:42.443373+0000 mon.a (mon.0) 1118 : audit 1 from='client.? 172.21.15.16:0/3851107921' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:37:43.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:43 smithi016 bash[75004]: cluster 2023-09-23T19:37:42.443705+0000 mon.a (mon.0) 1119 : cluster 0 osdmap e153: 8 total, 8 up, 8 in 2023-09-23T19:37:43.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:43 smithi016 bash[75004]: cluster 2023-09-23T19:37:42.457102+0000 osd.5 (osd.5) 81 : cluster 3 failed to encode map e153 with expected crc 2023-09-23T19:37:43.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:43 smithi016 bash[75004]: cluster 2023-09-23T19:37:42.463602+0000 osd.4 (osd.4) 79 : cluster 3 failed to encode map e153 with expected crc 2023-09-23T19:37:43.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:43 smithi016 bash[75004]: cluster 2023-09-23T19:37:42.464001+0000 osd.6 (osd.6) 69 : cluster 3 failed to encode map e153 with expected crc 2023-09-23T19:37:43.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:43 smithi016 bash[77995]: cluster 2023-09-23T19:37:42.094257+0000 mgr.y (mgr.25325) 905 : cluster 0 pgmap v829: 89 pgs: 89 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 55 MiB/s wr, 884 op/s 2023-09-23T19:37:43.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:43 smithi016 bash[77995]: audit 2023-09-23T19:37:42.265349+0000 mgr.y (mgr.25325) 906 : audit 0 from='client.55573 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:37:43.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:43 smithi016 bash[77995]: cluster 2023-09-23T19:37:42.422832+0000 mon.a (mon.0) 1116 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:37:43.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:43 smithi016 bash[77995]: cluster 2023-09-23T19:37:42.422877+0000 mon.a (mon.0) 1117 : cluster 1 Cluster is now healthy 2023-09-23T19:37:43.538 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:43 smithi016 bash[77995]: audit 2023-09-23T19:37:42.443373+0000 mon.a (mon.0) 1118 : audit 1 from='client.? 172.21.15.16:0/3851107921' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:37:43.538 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:43 smithi016 bash[77995]: cluster 2023-09-23T19:37:42.443705+0000 mon.a (mon.0) 1119 : cluster 0 osdmap e153: 8 total, 8 up, 8 in 2023-09-23T19:37:43.538 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:43 smithi016 bash[77995]: cluster 2023-09-23T19:37:42.457102+0000 osd.5 (osd.5) 81 : cluster 3 failed to encode map e153 with expected crc 2023-09-23T19:37:43.538 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:43 smithi016 bash[77995]: cluster 2023-09-23T19:37:42.463602+0000 osd.4 (osd.4) 79 : cluster 3 failed to encode map e153 with expected crc 2023-09-23T19:37:43.538 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:43 smithi016 bash[77995]: cluster 2023-09-23T19:37:42.464001+0000 osd.6 (osd.6) 69 : cluster 3 failed to encode map e153 with expected crc 2023-09-23T19:37:43.614 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:43.638 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:43.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:43.640 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:43.741 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:43.768 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:43.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:43.772 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:43.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:43 smithi092 bash[58855]: cluster 2023-09-23T19:37:42.094257+0000 mgr.y (mgr.25325) 905 : cluster 0 pgmap v829: 89 pgs: 89 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 55 MiB/s wr, 884 op/s 2023-09-23T19:37:43.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:43 smithi092 bash[58855]: audit 2023-09-23T19:37:42.265349+0000 mgr.y (mgr.25325) 906 : audit 0 from='client.55573 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:37:43.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:43 smithi092 bash[58855]: cluster 2023-09-23T19:37:42.422832+0000 mon.a (mon.0) 1116 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:37:43.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:43 smithi092 bash[58855]: cluster 2023-09-23T19:37:42.422877+0000 mon.a (mon.0) 1117 : cluster 1 Cluster is now healthy 2023-09-23T19:37:43.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:43 smithi092 bash[58855]: audit 2023-09-23T19:37:42.443373+0000 mon.a (mon.0) 1118 : audit 1 from='client.? 172.21.15.16:0/3851107921' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:37:43.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:43 smithi092 bash[58855]: cluster 2023-09-23T19:37:42.443705+0000 mon.a (mon.0) 1119 : cluster 0 osdmap e153: 8 total, 8 up, 8 in 2023-09-23T19:37:43.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:43 smithi092 bash[58855]: cluster 2023-09-23T19:37:42.457102+0000 osd.5 (osd.5) 81 : cluster 3 failed to encode map e153 with expected crc 2023-09-23T19:37:43.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:43 smithi092 bash[58855]: cluster 2023-09-23T19:37:42.463602+0000 osd.4 (osd.4) 79 : cluster 3 failed to encode map e153 with expected crc 2023-09-23T19:37:43.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:43 smithi092 bash[58855]: cluster 2023-09-23T19:37:42.464001+0000 osd.6 (osd.6) 69 : cluster 3 failed to encode map e153 with expected crc 2023-09-23T19:37:43.873 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:43.897 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:43.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:43.900 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:37:44.001 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:44.038 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:44.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:44.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:44.143 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:44.169 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:44.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:44.174 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:44.274 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:44.304 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:44.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:44.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:37:44.410 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:44.445 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:44.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:44.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:37:44.491 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 69 16 92601 92585 83.8542 59.3125 0.0288985 0.0119197 2023-09-23T19:37:44.550 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:44.593 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:44.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:44.599 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:37:44.653 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:44 smithi016 bash[75004]: cluster 2023-09-23T19:37:42.469281+0000 osd.7 (osd.7) 67 : cluster 3 failed to encode map e153 with expected crc 2023-09-23T19:37:44.653 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:44 smithi016 bash[75004]: audit 2023-09-23T19:37:43.658261+0000 mon.a (mon.0) 1120 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:37:44.653 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:44 smithi016 bash[75004]: audit 2023-09-23T19:37:43.659709+0000 mon.a (mon.0) 1121 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:37:44.654 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:44 smithi016 bash[77995]: cluster 2023-09-23T19:37:42.469281+0000 osd.7 (osd.7) 67 : cluster 3 failed to encode map e153 with expected crc 2023-09-23T19:37:44.654 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:44 smithi016 bash[77995]: audit 2023-09-23T19:37:43.658261+0000 mon.a (mon.0) 1120 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:37:44.654 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:44 smithi016 bash[77995]: audit 2023-09-23T19:37:43.659709+0000 mon.a (mon.0) 1121 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:37:44.700 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:44.732 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:44.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:44.738 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:37:44.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:44 smithi092 bash[58855]: cluster 2023-09-23T19:37:42.469281+0000 osd.7 (osd.7) 67 : cluster 3 failed to encode map e153 with expected crc 2023-09-23T19:37:44.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:44 smithi092 bash[58855]: audit 2023-09-23T19:37:43.658261+0000 mon.a (mon.0) 1120 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:37:44.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:44 smithi092 bash[58855]: audit 2023-09-23T19:37:43.659709+0000 mon.a (mon.0) 1121 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:37:44.839 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:44.878 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:44.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:44.880 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:37:44.981 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:45.004 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:45.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:45.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:37:45.112 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:45.155 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:45.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:45.160 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:37:45.261 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:45.293 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:45.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:45.297 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:37:45.334 INFO:teuthology.orchestra.run.smithi016.stdout:{"pg_ready":true,"pg_map":{"version":829,"stamp":"2023-09-23T19:37:42.093795+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5762455456,"num_objects":87943,"num_object_clones":0,"num_object_copies":263829,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":87943,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":88180,"num_write_kb":5629262,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":88114,"ondisk_log_size":88114,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":21196932,"kb_used_data":16826116,"kb_used_omap":20,"kb_used_meta":4370667,"kb_avail":728764284,"statfs":{"total":767960285184,"available":746254626816,"internally_reserved":0,"allocated":17229942784,"data_stored":17225570909,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":4475563662},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,2,7,6,8],"upper_bound":32},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":695926784,"num_objects":10619,"num_object_clones":0,"num_object_copies":31857,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":10619,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10619,"num_write_kb":679616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.005730"},"pg_stats":[{"pgid":"3.1a","version":"59'1","reported_seq":133,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.407465+0000","last_change":"2023-09-23T19:27:24.274696+0000","last_active":"2023-09-23T19:36:55.407465+0000","last_peered":"2023-09-23T19:36:55.407465+0000","last_clean":"2023-09-23T19:36:55.407465+0000","last_became_active":"2023-09-23T19:27:24.273561+0000","last_became_peered":"2023-09-23T19:27:24.273561+0000","last_unstale":"2023-09-23T19:36:55.407465+0000","last_undegraded":"2023-09-23T19:36:55.407465+0000","last_fullsized":"2023-09-23T19:36:55.407465+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":106,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.389335+0000","last_change":"2023-09-23T19:32:49.731128+0000","last_active":"2023-09-23T19:36:55.389335+0000","last_peered":"2023-09-23T19:36:55.389335+0000","last_clean":"2023-09-23T19:36:55.389335+0000","last_became_active":"2023-09-23T19:32:49.730757+0000","last_became_peered":"2023-09-23T19:32:49.730757+0000","last_unstale":"2023-09-23T19:36:55.389335+0000","last_undegraded":"2023-09-23T19:36:55.389335+0000","last_fullsized":"2023-09-23T19:36:55.389335+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:40.183605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":95,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:37:11.885072+0000","last_change":"2023-09-23T19:28:07.890462+0000","last_active":"2023-09-23T19:37:11.885072+0000","last_peered":"2023-09-23T19:37:11.885072+0000","last_clean":"2023-09-23T19:37:11.885072+0000","last_became_active":"2023-09-23T19:28:07.890084+0000","last_became_peered":"2023-09-23T19:28:07.890084+0000","last_unstale":"2023-09-23T19:37:11.885072+0000","last_undegraded":"2023-09-23T19:37:11.885072+0000","last_fullsized":"2023-09-23T19:37:11.885072+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:50.458218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":112,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.396126+0000","last_change":"2023-09-23T19:28:07.884498+0000","last_active":"2023-09-23T19:36:55.396126+0000","last_peered":"2023-09-23T19:36:55.396126+0000","last_clean":"2023-09-23T19:36:55.396126+0000","last_became_active":"2023-09-23T19:28:07.884102+0000","last_became_peered":"2023-09-23T19:28:07.884102+0000","last_unstale":"2023-09-23T19:36:55.396126+0000","last_undegraded":"2023-09-23T19:36:55.396126+0000","last_fullsized":"2023-09-23T19:36:55.396126+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:05:26.348041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":123,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.395702+0000","last_change":"2023-09-23T19:32:49.729961+0000","last_active":"2023-09-23T19:36:55.395702+0000","last_peered":"2023-09-23T19:36:55.395702+0000","last_clean":"2023-09-23T19:36:55.395702+0000","last_became_active":"2023-09-23T19:32:49.729251+0000","last_became_peered":"2023-09-23T19:32:49.729251+0000","last_unstale":"2023-09-23T19:36:55.395702+0000","last_undegraded":"2023-09-23T19:36:55.395702+0000","last_fullsized":"2023-09-23T19:36:55.395702+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:20:56.595732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":117,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.404414+0000","last_change":"2023-09-23T19:22:42.150582+0000","last_active":"2023-09-23T19:36:55.404414+0000","last_peered":"2023-09-23T19:36:55.404414+0000","last_clean":"2023-09-23T19:36:55.404414+0000","last_became_active":"2023-09-23T19:22:42.150229+0000","last_became_peered":"2023-09-23T19:22:42.150229+0000","last_unstale":"2023-09-23T19:36:55.404414+0000","last_undegraded":"2023-09-23T19:36:55.404414+0000","last_fullsized":"2023-09-23T19:36:55.404414+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":111,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-09-23T19:37:20.406849+0000","last_change":"2023-09-23T19:28:07.887141+0000","last_active":"2023-09-23T19:37:20.406849+0000","last_peered":"2023-09-23T19:37:20.406849+0000","last_clean":"2023-09-23T19:37:20.406849+0000","last_became_active":"2023-09-23T19:28:07.886494+0000","last_became_peered":"2023-09-23T19:28:07.886494+0000","last_unstale":"2023-09-23T19:37:20.406849+0000","last_undegraded":"2023-09-23T19:37:20.406849+0000","last_fullsized":"2023-09-23T19:37:20.406849+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:24:07.615051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":107,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-09-23T19:37:20.401158+0000","last_change":"2023-09-23T19:32:49.729765+0000","last_active":"2023-09-23T19:37:20.401158+0000","last_peered":"2023-09-23T19:37:20.401158+0000","last_clean":"2023-09-23T19:37:20.401158+0000","last_became_active":"2023-09-23T19:32:49.729007+0000","last_became_peered":"2023-09-23T19:32:49.729007+0000","last_unstale":"2023-09-23T19:37:20.401158+0000","last_undegraded":"2023-09-23T19:37:20.401158+0000","last_fullsized":"2023-09-23T19:37:20.401158+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:59:31.183753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":119,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.403641+0000","last_change":"2023-09-23T19:22:42.151617+0000","last_active":"2023-09-23T19:36:55.403641+0000","last_peered":"2023-09-23T19:36:55.403641+0000","last_clean":"2023-09-23T19:36:55.403641+0000","last_became_active":"2023-09-23T19:22:42.151217+0000","last_became_peered":"2023-09-23T19:22:42.151217+0000","last_unstale":"2023-09-23T19:36:55.403641+0000","last_undegraded":"2023-09-23T19:36:55.403641+0000","last_fullsized":"2023-09-23T19:36:55.403641+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":92,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.414713+0000","last_change":"2023-09-23T19:32:49.736864+0000","last_active":"2023-09-23T19:36:55.414713+0000","last_peered":"2023-09-23T19:36:55.414713+0000","last_clean":"2023-09-23T19:36:55.414713+0000","last_became_active":"2023-09-23T19:32:49.736523+0000","last_became_peered":"2023-09-23T19:32:49.736523+0000","last_unstale":"2023-09-23T19:36:55.414713+0000","last_undegraded":"2023-09-23T19:36:55.414713+0000","last_fullsized":"2023-09-23T19:36:55.414713+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":121,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.413258+0000","last_change":"2023-09-23T19:28:07.892835+0000","last_active":"2023-09-23T19:36:55.413258+0000","last_peered":"2023-09-23T19:36:55.413258+0000","last_clean":"2023-09-23T19:36:55.413258+0000","last_became_active":"2023-09-23T19:28:07.892338+0000","last_became_peered":"2023-09-23T19:28:07.892338+0000","last_unstale":"2023-09-23T19:36:55.413258+0000","last_undegraded":"2023-09-23T19:36:55.413258+0000","last_fullsized":"2023-09-23T19:36:55.413258+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":95,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.389290+0000","last_change":"2023-09-23T19:27:24.268005+0000","last_active":"2023-09-23T19:36:55.389290+0000","last_peered":"2023-09-23T19:36:55.389290+0000","last_clean":"2023-09-23T19:36:55.389290+0000","last_became_active":"2023-09-23T19:27:24.267513+0000","last_became_peered":"2023-09-23T19:27:24.267513+0000","last_unstale":"2023-09-23T19:36:55.389290+0000","last_undegraded":"2023-09-23T19:36:55.389290+0000","last_fullsized":"2023-09-23T19:36:55.389290+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.961399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'23","reported_seq":125,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.405572+0000","last_change":"2023-09-23T19:05:52.107727+0000","last_active":"2023-09-23T19:36:55.405572+0000","last_peered":"2023-09-23T19:36:55.405572+0000","last_clean":"2023-09-23T19:36:55.405572+0000","last_became_active":"2023-09-23T19:05:52.107378+0000","last_became_peered":"2023-09-23T19:05:52.107378+0000","last_unstale":"2023-09-23T19:36:55.405572+0000","last_undegraded":"2023-09-23T19:36:55.405572+0000","last_fullsized":"2023-09-23T19:36:55.405572+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":116,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.405671+0000","last_change":"2023-09-23T19:22:42.148497+0000","last_active":"2023-09-23T19:36:55.405671+0000","last_peered":"2023-09-23T19:36:55.405671+0000","last_clean":"2023-09-23T19:36:55.405671+0000","last_became_active":"2023-09-23T19:22:42.148147+0000","last_became_peered":"2023-09-23T19:22:42.148147+0000","last_unstale":"2023-09-23T19:36:55.405671+0000","last_undegraded":"2023-09-23T19:36:55.405671+0000","last_fullsized":"2023-09-23T19:36:55.405671+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":120,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.413846+0000","last_change":"2023-09-23T19:27:24.273667+0000","last_active":"2023-09-23T19:36:55.413846+0000","last_peered":"2023-09-23T19:36:55.413846+0000","last_clean":"2023-09-23T19:36:55.413846+0000","last_became_active":"2023-09-23T19:27:24.273234+0000","last_became_peered":"2023-09-23T19:27:24.273234+0000","last_unstale":"2023-09-23T19:36:55.413846+0000","last_undegraded":"2023-09-23T19:36:55.413846+0000","last_fullsized":"2023-09-23T19:36:55.413846+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":107,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-09-23T19:37:20.401117+0000","last_change":"2023-09-23T19:32:49.730549+0000","last_active":"2023-09-23T19:37:20.401117+0000","last_peered":"2023-09-23T19:37:20.401117+0000","last_clean":"2023-09-23T19:37:20.401117+0000","last_became_active":"2023-09-23T19:32:49.730105+0000","last_became_peered":"2023-09-23T19:32:49.730105+0000","last_unstale":"2023-09-23T19:37:20.401117+0000","last_undegraded":"2023-09-23T19:37:20.401117+0000","last_fullsized":"2023-09-23T19:37:20.401117+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:49.751893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":111,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.396091+0000","last_change":"2023-09-23T19:28:07.884394+0000","last_active":"2023-09-23T19:36:55.396091+0000","last_peered":"2023-09-23T19:36:55.396091+0000","last_clean":"2023-09-23T19:36:55.396091+0000","last_became_active":"2023-09-23T19:28:07.883944+0000","last_became_peered":"2023-09-23T19:28:07.883944+0000","last_unstale":"2023-09-23T19:36:55.396091+0000","last_undegraded":"2023-09-23T19:36:55.396091+0000","last_fullsized":"2023-09-23T19:36:55.396091+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:08:17.608926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":118,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.403161+0000","last_change":"2023-09-23T19:28:07.891056+0000","last_active":"2023-09-23T19:36:55.403161+0000","last_peered":"2023-09-23T19:36:55.403161+0000","last_clean":"2023-09-23T19:36:55.403161+0000","last_became_active":"2023-09-23T19:28:07.890625+0000","last_became_peered":"2023-09-23T19:28:07.890625+0000","last_unstale":"2023-09-23T19:36:55.403161+0000","last_undegraded":"2023-09-23T19:36:55.403161+0000","last_fullsized":"2023-09-23T19:36:55.403161+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":103,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.414878+0000","last_change":"2023-09-23T19:05:52.114186+0000","last_active":"2023-09-23T19:36:55.414878+0000","last_peered":"2023-09-23T19:36:55.414878+0000","last_clean":"2023-09-23T19:36:55.414878+0000","last_became_active":"2023-09-23T19:05:52.113635+0000","last_became_peered":"2023-09-23T19:05:52.113635+0000","last_unstale":"2023-09-23T19:36:55.414878+0000","last_undegraded":"2023-09-23T19:36:55.414878+0000","last_fullsized":"2023-09-23T19:36:55.414878+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":111,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-09-23T19:37:20.406720+0000","last_change":"2023-09-23T19:28:07.884512+0000","last_active":"2023-09-23T19:37:20.406720+0000","last_peered":"2023-09-23T19:37:20.406720+0000","last_clean":"2023-09-23T19:37:20.406720+0000","last_became_active":"2023-09-23T19:28:07.884155+0000","last_became_peered":"2023-09-23T19:28:07.884155+0000","last_unstale":"2023-09-23T19:37:20.406720+0000","last_undegraded":"2023-09-23T19:37:20.406720+0000","last_fullsized":"2023-09-23T19:37:20.406720+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:10.654958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":126,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.401321+0000","last_change":"2023-09-23T19:18:09.010195+0000","last_active":"2023-09-23T19:36:55.401321+0000","last_peered":"2023-09-23T19:36:55.401321+0000","last_clean":"2023-09-23T19:36:55.401321+0000","last_became_active":"2023-09-23T19:18:09.009687+0000","last_became_peered":"2023-09-23T19:18:09.009687+0000","last_unstale":"2023-09-23T19:36:55.401321+0000","last_undegraded":"2023-09-23T19:36:55.401321+0000","last_fullsized":"2023-09-23T19:36:55.401321+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":115,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.400616+0000","last_change":"2023-09-23T19:28:07.885265+0000","last_active":"2023-09-23T19:36:55.400616+0000","last_peered":"2023-09-23T19:36:55.400616+0000","last_clean":"2023-09-23T19:36:55.400616+0000","last_became_active":"2023-09-23T19:28:07.884353+0000","last_became_peered":"2023-09-23T19:28:07.884353+0000","last_unstale":"2023-09-23T19:36:55.400616+0000","last_undegraded":"2023-09-23T19:36:55.400616+0000","last_fullsized":"2023-09-23T19:36:55.400616+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:56.325352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"152'5500","reported_seq":5516,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-09-23T19:37:40.592076+0000","last_change":"2023-09-23T19:36:14.521072+0000","last_active":"2023-09-23T19:37:40.592076+0000","last_peered":"2023-09-23T19:37:40.592076+0000","last_clean":"2023-09-23T19:37:40.592076+0000","last_became_active":"2023-09-23T19:36:14.520698+0000","last_became_peered":"2023-09-23T19:36:14.520698+0000","last_unstale":"2023-09-23T19:37:40.592076+0000","last_undegraded":"2023-09-23T19:37:40.592076+0000","last_fullsized":"2023-09-23T19:37:40.592076+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_clean_scrub_stamp":"2023-09-23T19:36:13.468134+0000","objects_scrubbed":0,"log_size":5500,"log_dups_size":0,"ondisk_log_size":5500,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":360448000,"num_objects":5500,"num_object_clones":0,"num_object_copies":16500,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5500,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5500,"num_write_kb":352000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":119,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.413359+0000","last_change":"2023-09-23T19:18:09.035116+0000","last_active":"2023-09-23T19:36:55.413359+0000","last_peered":"2023-09-23T19:36:55.413359+0000","last_clean":"2023-09-23T19:36:55.413359+0000","last_became_active":"2023-09-23T19:18:09.034416+0000","last_became_peered":"2023-09-23T19:18:09.034416+0000","last_unstale":"2023-09-23T19:36:55.413359+0000","last_undegraded":"2023-09-23T19:36:55.413359+0000","last_fullsized":"2023-09-23T19:36:55.413359+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":103,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.403634+0000","last_change":"2023-09-23T19:05:52.789323+0000","last_active":"2023-09-23T19:36:55.403634+0000","last_peered":"2023-09-23T19:36:55.403634+0000","last_clean":"2023-09-23T19:36:55.403634+0000","last_became_active":"2023-09-23T19:05:52.788820+0000","last_became_peered":"2023-09-23T19:05:52.788820+0000","last_unstale":"2023-09-23T19:36:55.403634+0000","last_undegraded":"2023-09-23T19:36:55.403634+0000","last_fullsized":"2023-09-23T19:36:55.403634+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"152'5663","reported_seq":5678,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-09-23T19:37:39.614011+0000","last_change":"2023-09-23T19:36:14.525132+0000","last_active":"2023-09-23T19:37:39.614011+0000","last_peered":"2023-09-23T19:37:39.614011+0000","last_clean":"2023-09-23T19:37:39.614011+0000","last_became_active":"2023-09-23T19:36:14.524731+0000","last_became_peered":"2023-09-23T19:36:14.524731+0000","last_unstale":"2023-09-23T19:37:39.614011+0000","last_undegraded":"2023-09-23T19:37:39.614011+0000","last_fullsized":"2023-09-23T19:37:39.614011+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_clean_scrub_stamp":"2023-09-23T19:36:13.468134+0000","objects_scrubbed":0,"log_size":5663,"log_dups_size":0,"ondisk_log_size":5663,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":371130368,"num_objects":5663,"num_object_clones":0,"num_object_copies":16989,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5663,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5663,"num_write_kb":362432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":120,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.413608+0000","last_change":"2023-09-23T19:22:42.151134+0000","last_active":"2023-09-23T19:36:55.413608+0000","last_peered":"2023-09-23T19:36:55.413608+0000","last_clean":"2023-09-23T19:36:55.413608+0000","last_became_active":"2023-09-23T19:22:42.150553+0000","last_became_peered":"2023-09-23T19:22:42.150553+0000","last_unstale":"2023-09-23T19:36:55.413608+0000","last_undegraded":"2023-09-23T19:36:55.413608+0000","last_fullsized":"2023-09-23T19:36:55.413608+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":131,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.406411+0000","last_change":"2023-09-23T19:28:07.899262+0000","last_active":"2023-09-23T19:36:55.406411+0000","last_peered":"2023-09-23T19:36:55.406411+0000","last_clean":"2023-09-23T19:36:55.406411+0000","last_became_active":"2023-09-23T19:28:07.898848+0000","last_became_peered":"2023-09-23T19:28:07.898848+0000","last_unstale":"2023-09-23T19:36:55.406411+0000","last_undegraded":"2023-09-23T19:36:55.406411+0000","last_fullsized":"2023-09-23T19:36:55.406411+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"152'5205","reported_seq":5222,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-09-23T19:37:37.618612+0000","last_change":"2023-09-23T19:36:14.506997+0000","last_active":"2023-09-23T19:37:37.618612+0000","last_peered":"2023-09-23T19:37:37.618612+0000","last_clean":"2023-09-23T19:37:37.618612+0000","last_became_active":"2023-09-23T19:36:14.506682+0000","last_became_peered":"2023-09-23T19:36:14.506682+0000","last_unstale":"2023-09-23T19:37:37.618612+0000","last_undegraded":"2023-09-23T19:37:37.618612+0000","last_fullsized":"2023-09-23T19:37:37.618612+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_clean_scrub_stamp":"2023-09-23T19:36:13.468134+0000","objects_scrubbed":0,"log_size":5205,"log_dups_size":0,"ondisk_log_size":5205,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:08:49.170182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":341114880,"num_objects":5205,"num_object_clones":0,"num_object_copies":15615,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5205,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5205,"num_write_kb":333120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":104,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.413077+0000","last_change":"2023-09-23T19:05:52.114108+0000","last_active":"2023-09-23T19:36:55.413077+0000","last_peered":"2023-09-23T19:36:55.413077+0000","last_clean":"2023-09-23T19:36:55.413077+0000","last_became_active":"2023-09-23T19:05:52.113609+0000","last_became_peered":"2023-09-23T19:05:52.113609+0000","last_unstale":"2023-09-23T19:36:55.413077+0000","last_undegraded":"2023-09-23T19:36:55.413077+0000","last_fullsized":"2023-09-23T19:36:55.413077+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":117,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.402883+0000","last_change":"2023-09-23T19:22:42.156825+0000","last_active":"2023-09-23T19:36:55.402883+0000","last_peered":"2023-09-23T19:36:55.402883+0000","last_clean":"2023-09-23T19:36:55.402883+0000","last_became_active":"2023-09-23T19:22:42.156474+0000","last_became_peered":"2023-09-23T19:22:42.156474+0000","last_unstale":"2023-09-23T19:36:55.402883+0000","last_undegraded":"2023-09-23T19:36:55.402883+0000","last_fullsized":"2023-09-23T19:36:55.402883+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"152'5343","reported_seq":5360,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-09-23T19:37:37.404576+0000","last_change":"2023-09-23T19:36:14.507382+0000","last_active":"2023-09-23T19:37:37.404576+0000","last_peered":"2023-09-23T19:37:37.404576+0000","last_clean":"2023-09-23T19:37:37.404576+0000","last_became_active":"2023-09-23T19:36:14.506978+0000","last_became_peered":"2023-09-23T19:36:14.506978+0000","last_unstale":"2023-09-23T19:37:37.404576+0000","last_undegraded":"2023-09-23T19:37:37.404576+0000","last_fullsized":"2023-09-23T19:37:37.404576+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_clean_scrub_stamp":"2023-09-23T19:36:13.468134+0000","objects_scrubbed":0,"log_size":5343,"log_dups_size":0,"ondisk_log_size":5343,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:49.373147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":350158848,"num_objects":5343,"num_object_clones":0,"num_object_copies":16029,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5343,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5343,"num_write_kb":341952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":119,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.403874+0000","last_change":"2023-09-23T19:28:07.891536+0000","last_active":"2023-09-23T19:36:55.403874+0000","last_peered":"2023-09-23T19:36:55.403874+0000","last_clean":"2023-09-23T19:36:55.403874+0000","last_became_active":"2023-09-23T19:28:07.891153+0000","last_became_peered":"2023-09-23T19:28:07.891153+0000","last_unstale":"2023-09-23T19:36:55.403874+0000","last_undegraded":"2023-09-23T19:36:55.403874+0000","last_fullsized":"2023-09-23T19:36:55.403874+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":111,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.397234+0000","last_change":"2023-09-23T19:22:42.135572+0000","last_active":"2023-09-23T19:36:55.397234+0000","last_peered":"2023-09-23T19:36:55.397234+0000","last_clean":"2023-09-23T19:36:55.397234+0000","last_became_active":"2023-09-23T19:22:42.135237+0000","last_became_peered":"2023-09-23T19:22:42.135237+0000","last_unstale":"2023-09-23T19:36:55.397234+0000","last_undegraded":"2023-09-23T19:36:55.397234+0000","last_fullsized":"2023-09-23T19:36:55.397234+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:35.838718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"152'5550","reported_seq":5566,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-09-23T19:37:40.125422+0000","last_change":"2023-09-23T19:36:14.507260+0000","last_active":"2023-09-23T19:37:40.125422+0000","last_peered":"2023-09-23T19:37:40.125422+0000","last_clean":"2023-09-23T19:37:40.125422+0000","last_became_active":"2023-09-23T19:36:14.506955+0000","last_became_peered":"2023-09-23T19:36:14.506955+0000","last_unstale":"2023-09-23T19:37:40.125422+0000","last_undegraded":"2023-09-23T19:37:40.125422+0000","last_fullsized":"2023-09-23T19:37:40.125422+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_clean_scrub_stamp":"2023-09-23T19:36:13.468134+0000","objects_scrubbed":0,"log_size":5550,"log_dups_size":0,"ondisk_log_size":5550,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:09:12.021383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":363724800,"num_objects":5550,"num_object_clones":0,"num_object_copies":16650,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5550,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5550,"num_write_kb":355200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":58,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:37:17.562605+0000","last_change":"2023-09-23T19:32:49.728589+0000","last_active":"2023-09-23T19:37:17.562605+0000","last_peered":"2023-09-23T19:37:17.562605+0000","last_clean":"2023-09-23T19:37:17.562605+0000","last_became_active":"2023-09-23T19:32:49.728037+0000","last_became_peered":"2023-09-23T19:32:49.728037+0000","last_unstale":"2023-09-23T19:37:17.562605+0000","last_undegraded":"2023-09-23T19:37:17.562605+0000","last_fullsized":"2023-09-23T19:37:17.562605+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:26:00.957969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":116,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.405657+0000","last_change":"2023-09-23T19:28:07.896803+0000","last_active":"2023-09-23T19:36:55.405657+0000","last_peered":"2023-09-23T19:36:55.405657+0000","last_clean":"2023-09-23T19:36:55.405657+0000","last_became_active":"2023-09-23T19:28:07.896480+0000","last_became_peered":"2023-09-23T19:28:07.896480+0000","last_unstale":"2023-09-23T19:36:55.405657+0000","last_undegraded":"2023-09-23T19:36:55.405657+0000","last_fullsized":"2023-09-23T19:36:55.405657+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"152'5379","reported_seq":5396,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-09-23T19:37:37.428793+0000","last_change":"2023-09-23T19:36:14.518837+0000","last_active":"2023-09-23T19:37:37.428793+0000","last_peered":"2023-09-23T19:37:37.428793+0000","last_clean":"2023-09-23T19:37:37.428793+0000","last_became_active":"2023-09-23T19:36:14.518503+0000","last_became_peered":"2023-09-23T19:36:14.518503+0000","last_unstale":"2023-09-23T19:37:37.428793+0000","last_undegraded":"2023-09-23T19:37:37.428793+0000","last_fullsized":"2023-09-23T19:37:37.428793+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_clean_scrub_stamp":"2023-09-23T19:36:13.468134+0000","objects_scrubbed":0,"log_size":5379,"log_dups_size":0,"ondisk_log_size":5379,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:39:05.790711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":352518144,"num_objects":5379,"num_object_clones":0,"num_object_copies":16137,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5379,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5379,"num_write_kb":344256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":118,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.403035+0000","last_change":"2023-09-23T19:22:42.156582+0000","last_active":"2023-09-23T19:36:55.403035+0000","last_peered":"2023-09-23T19:36:55.403035+0000","last_clean":"2023-09-23T19:36:55.403035+0000","last_became_active":"2023-09-23T19:22:42.156251+0000","last_became_peered":"2023-09-23T19:22:42.156251+0000","last_unstale":"2023-09-23T19:36:55.403035+0000","last_undegraded":"2023-09-23T19:36:55.403035+0000","last_fullsized":"2023-09-23T19:36:55.403035+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":131,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.405849+0000","last_change":"2023-09-23T19:27:24.273971+0000","last_active":"2023-09-23T19:36:55.405849+0000","last_peered":"2023-09-23T19:36:55.405849+0000","last_clean":"2023-09-23T19:36:55.405849+0000","last_became_active":"2023-09-23T19:27:24.273491+0000","last_became_peered":"2023-09-23T19:27:24.273491+0000","last_unstale":"2023-09-23T19:36:55.405849+0000","last_undegraded":"2023-09-23T19:36:55.405849+0000","last_fullsized":"2023-09-23T19:36:55.405849+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"152'5525","reported_seq":5541,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-09-23T19:37:39.614678+0000","last_change":"2023-09-23T19:36:14.526027+0000","last_active":"2023-09-23T19:37:39.614678+0000","last_peered":"2023-09-23T19:37:39.614678+0000","last_clean":"2023-09-23T19:37:39.614678+0000","last_became_active":"2023-09-23T19:36:14.525656+0000","last_became_peered":"2023-09-23T19:36:14.525656+0000","last_unstale":"2023-09-23T19:37:39.614678+0000","last_undegraded":"2023-09-23T19:37:39.614678+0000","last_fullsized":"2023-09-23T19:37:39.614678+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_clean_scrub_stamp":"2023-09-23T19:36:13.468134+0000","objects_scrubbed":0,"log_size":5525,"log_dups_size":0,"ondisk_log_size":5525,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":362086400,"num_objects":5525,"num_object_clones":0,"num_object_copies":16575,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5525,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5525,"num_write_kb":353600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":117,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.406242+0000","last_change":"2023-09-23T19:27:24.274698+0000","last_active":"2023-09-23T19:36:55.406242+0000","last_peered":"2023-09-23T19:36:55.406242+0000","last_clean":"2023-09-23T19:36:55.406242+0000","last_became_active":"2023-09-23T19:27:24.273264+0000","last_became_peered":"2023-09-23T19:27:24.273264+0000","last_unstale":"2023-09-23T19:36:55.406242+0000","last_undegraded":"2023-09-23T19:36:55.406242+0000","last_fullsized":"2023-09-23T19:36:55.406242+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":116,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.406850+0000","last_change":"2023-09-23T19:22:42.149670+0000","last_active":"2023-09-23T19:36:55.406850+0000","last_peered":"2023-09-23T19:36:55.406850+0000","last_clean":"2023-09-23T19:36:55.406850+0000","last_became_active":"2023-09-23T19:22:42.149022+0000","last_became_peered":"2023-09-23T19:22:42.149022+0000","last_unstale":"2023-09-23T19:36:55.406850+0000","last_undegraded":"2023-09-23T19:36:55.406850+0000","last_fullsized":"2023-09-23T19:36:55.406850+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"152'5528","reported_seq":5543,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-09-23T19:37:40.049023+0000","last_change":"2023-09-23T19:36:14.524191+0000","last_active":"2023-09-23T19:37:40.049023+0000","last_peered":"2023-09-23T19:37:40.049023+0000","last_clean":"2023-09-23T19:37:40.049023+0000","last_became_active":"2023-09-23T19:36:14.523868+0000","last_became_peered":"2023-09-23T19:36:14.523868+0000","last_unstale":"2023-09-23T19:37:40.049023+0000","last_undegraded":"2023-09-23T19:37:40.049023+0000","last_fullsized":"2023-09-23T19:37:40.049023+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_clean_scrub_stamp":"2023-09-23T19:36:13.468134+0000","objects_scrubbed":0,"log_size":5528,"log_dups_size":0,"ondisk_log_size":5528,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":362283008,"num_objects":5528,"num_object_clones":0,"num_object_copies":16584,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5528,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5528,"num_write_kb":353792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":108,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:37:11.885027+0000","last_change":"2023-09-23T19:32:49.729244+0000","last_active":"2023-09-23T19:37:11.885027+0000","last_peered":"2023-09-23T19:37:11.885027+0000","last_clean":"2023-09-23T19:37:11.885027+0000","last_became_active":"2023-09-23T19:32:49.728810+0000","last_became_peered":"2023-09-23T19:32:49.728810+0000","last_unstale":"2023-09-23T19:37:11.885027+0000","last_undegraded":"2023-09-23T19:37:11.885027+0000","last_fullsized":"2023-09-23T19:37:11.885027+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:57.540826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":95,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.389811+0000","last_change":"2023-09-23T19:28:07.889095+0000","last_active":"2023-09-23T19:36:55.389811+0000","last_peered":"2023-09-23T19:36:55.389811+0000","last_clean":"2023-09-23T19:36:55.389811+0000","last_became_active":"2023-09-23T19:28:07.888742+0000","last_became_peered":"2023-09-23T19:28:07.888742+0000","last_unstale":"2023-09-23T19:36:55.389811+0000","last_undegraded":"2023-09-23T19:36:55.389811+0000","last_fullsized":"2023-09-23T19:36:55.389811+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:43.929488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"152'5541","reported_seq":5555,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-09-23T19:37:40.324342+0000","last_change":"2023-09-23T19:36:14.521993+0000","last_active":"2023-09-23T19:37:40.324342+0000","last_peered":"2023-09-23T19:37:40.324342+0000","last_clean":"2023-09-23T19:37:40.324342+0000","last_became_active":"2023-09-23T19:36:14.521681+0000","last_became_peered":"2023-09-23T19:36:14.521681+0000","last_unstale":"2023-09-23T19:37:40.324342+0000","last_undegraded":"2023-09-23T19:37:40.324342+0000","last_fullsized":"2023-09-23T19:37:40.324342+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_clean_scrub_stamp":"2023-09-23T19:36:13.468134+0000","objects_scrubbed":0,"log_size":5541,"log_dups_size":0,"ondisk_log_size":5541,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":363134976,"num_objects":5541,"num_object_clones":0,"num_object_copies":16623,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5541,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5541,"num_write_kb":354624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":129,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:37:16.886440+0000","last_change":"2023-09-23T19:22:42.143289+0000","last_active":"2023-09-23T19:37:16.886440+0000","last_peered":"2023-09-23T19:37:16.886440+0000","last_clean":"2023-09-23T19:37:16.886440+0000","last_became_active":"2023-09-23T19:22:42.142656+0000","last_became_peered":"2023-09-23T19:22:42.142656+0000","last_unstale":"2023-09-23T19:37:16.886440+0000","last_undegraded":"2023-09-23T19:37:16.886440+0000","last_fullsized":"2023-09-23T19:37:16.886440+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:38.467441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":134,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.403249+0000","last_change":"2023-09-23T19:28:07.890623+0000","last_active":"2023-09-23T19:36:55.403249+0000","last_peered":"2023-09-23T19:36:55.403249+0000","last_clean":"2023-09-23T19:36:55.403249+0000","last_became_active":"2023-09-23T19:28:07.890148+0000","last_became_peered":"2023-09-23T19:28:07.890148+0000","last_unstale":"2023-09-23T19:36:55.403249+0000","last_undegraded":"2023-09-23T19:36:55.403249+0000","last_fullsized":"2023-09-23T19:36:55.403249+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":108,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.400223+0000","last_change":"2023-09-23T19:27:24.269080+0000","last_active":"2023-09-23T19:36:55.400223+0000","last_peered":"2023-09-23T19:36:55.400223+0000","last_clean":"2023-09-23T19:36:55.400223+0000","last_became_active":"2023-09-23T19:27:24.268748+0000","last_became_peered":"2023-09-23T19:27:24.268748+0000","last_unstale":"2023-09-23T19:36:55.400223+0000","last_undegraded":"2023-09-23T19:36:55.400223+0000","last_fullsized":"2023-09-23T19:36:55.400223+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:49.014902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.a","version":"152'5515","reported_seq":5531,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-09-23T19:37:39.628085+0000","last_change":"2023-09-23T19:36:14.523767+0000","last_active":"2023-09-23T19:37:39.628085+0000","last_peered":"2023-09-23T19:37:39.628085+0000","last_clean":"2023-09-23T19:37:39.628085+0000","last_became_active":"2023-09-23T19:36:14.523424+0000","last_became_peered":"2023-09-23T19:36:14.523424+0000","last_unstale":"2023-09-23T19:37:39.628085+0000","last_undegraded":"2023-09-23T19:37:39.628085+0000","last_fullsized":"2023-09-23T19:37:39.628085+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_clean_scrub_stamp":"2023-09-23T19:36:13.468134+0000","objects_scrubbed":0,"log_size":5515,"log_dups_size":0,"ondisk_log_size":5515,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":361431040,"num_objects":5515,"num_object_clones":0,"num_object_copies":16545,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5515,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5515,"num_write_kb":352960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":123,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.404468+0000","last_change":"2023-09-23T19:22:42.153647+0000","last_active":"2023-09-23T19:36:55.404468+0000","last_peered":"2023-09-23T19:36:55.404468+0000","last_clean":"2023-09-23T19:36:55.404468+0000","last_became_active":"2023-09-23T19:22:42.153317+0000","last_became_peered":"2023-09-23T19:22:42.153317+0000","last_unstale":"2023-09-23T19:36:55.404468+0000","last_undegraded":"2023-09-23T19:36:55.404468+0000","last_fullsized":"2023-09-23T19:36:55.404468+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":117,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.405596+0000","last_change":"2023-09-23T19:18:09.017525+0000","last_active":"2023-09-23T19:36:55.405596+0000","last_peered":"2023-09-23T19:36:55.405596+0000","last_clean":"2023-09-23T19:36:55.405596+0000","last_became_active":"2023-09-23T19:18:09.017193+0000","last_became_peered":"2023-09-23T19:18:09.017193+0000","last_unstale":"2023-09-23T19:36:55.405596+0000","last_undegraded":"2023-09-23T19:36:55.405596+0000","last_fullsized":"2023-09-23T19:36:55.405596+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":111,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:37:16.886414+0000","last_change":"2023-09-23T19:28:07.875796+0000","last_active":"2023-09-23T19:37:16.886414+0000","last_peered":"2023-09-23T19:37:16.886414+0000","last_clean":"2023-09-23T19:37:16.886414+0000","last_became_active":"2023-09-23T19:28:07.875310+0000","last_became_peered":"2023-09-23T19:28:07.875310+0000","last_unstale":"2023-09-23T19:37:16.886414+0000","last_undegraded":"2023-09-23T19:37:16.886414+0000","last_fullsized":"2023-09-23T19:37:16.886414+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:44:13.379633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"152'5566","reported_seq":5580,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-09-23T19:37:40.595204+0000","last_change":"2023-09-23T19:36:14.524117+0000","last_active":"2023-09-23T19:37:40.595204+0000","last_peered":"2023-09-23T19:37:40.595204+0000","last_clean":"2023-09-23T19:37:40.595204+0000","last_became_active":"2023-09-23T19:36:14.523735+0000","last_became_peered":"2023-09-23T19:36:14.523735+0000","last_unstale":"2023-09-23T19:37:40.595204+0000","last_undegraded":"2023-09-23T19:37:40.595204+0000","last_fullsized":"2023-09-23T19:37:40.595204+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_clean_scrub_stamp":"2023-09-23T19:36:13.468134+0000","objects_scrubbed":0,"log_size":5566,"log_dups_size":0,"ondisk_log_size":5566,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":364773376,"num_objects":5566,"num_object_clones":0,"num_object_copies":16698,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5566,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5566,"num_write_kb":356224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":128,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.403364+0000","last_change":"2023-09-23T19:28:07.878688+0000","last_active":"2023-09-23T19:36:55.403364+0000","last_peered":"2023-09-23T19:36:55.403364+0000","last_clean":"2023-09-23T19:36:55.403364+0000","last_became_active":"2023-09-23T19:28:07.878309+0000","last_became_peered":"2023-09-23T19:28:07.878309+0000","last_unstale":"2023-09-23T19:36:55.403364+0000","last_undegraded":"2023-09-23T19:36:55.403364+0000","last_fullsized":"2023-09-23T19:36:55.403364+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:49:06.202657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":96,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.390323+0000","last_change":"2023-09-23T19:27:24.267897+0000","last_active":"2023-09-23T19:36:55.390323+0000","last_peered":"2023-09-23T19:36:55.390323+0000","last_clean":"2023-09-23T19:36:55.390323+0000","last_became_active":"2023-09-23T19:27:24.267535+0000","last_became_peered":"2023-09-23T19:27:24.267535+0000","last_unstale":"2023-09-23T19:36:55.390323+0000","last_undegraded":"2023-09-23T19:36:55.390323+0000","last_fullsized":"2023-09-23T19:36:55.390323+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:13:57.447808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":123,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.389878+0000","last_change":"2023-09-23T19:22:42.143733+0000","last_active":"2023-09-23T19:36:55.389878+0000","last_peered":"2023-09-23T19:36:55.389878+0000","last_clean":"2023-09-23T19:36:55.389878+0000","last_became_active":"2023-09-23T19:22:42.143077+0000","last_became_peered":"2023-09-23T19:22:42.143077+0000","last_unstale":"2023-09-23T19:36:55.389878+0000","last_undegraded":"2023-09-23T19:36:55.389878+0000","last_fullsized":"2023-09-23T19:36:55.389878+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:47:33.086994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"152'5442","reported_seq":5458,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-09-23T19:37:39.617280+0000","last_change":"2023-09-23T19:36:14.524573+0000","last_active":"2023-09-23T19:37:39.617280+0000","last_peered":"2023-09-23T19:37:39.617280+0000","last_clean":"2023-09-23T19:37:39.617280+0000","last_became_active":"2023-09-23T19:36:14.524235+0000","last_became_peered":"2023-09-23T19:36:14.524235+0000","last_unstale":"2023-09-23T19:37:39.617280+0000","last_undegraded":"2023-09-23T19:37:39.617280+0000","last_fullsized":"2023-09-23T19:37:39.617280+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_clean_scrub_stamp":"2023-09-23T19:36:13.468134+0000","objects_scrubbed":0,"log_size":5442,"log_dups_size":0,"ondisk_log_size":5442,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":356646912,"num_objects":5442,"num_object_clones":0,"num_object_copies":16326,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5442,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5442,"num_write_kb":348288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":123,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.404674+0000","last_change":"2023-09-23T19:27:24.271520+0000","last_active":"2023-09-23T19:36:55.404674+0000","last_peered":"2023-09-23T19:36:55.404674+0000","last_clean":"2023-09-23T19:36:55.404674+0000","last_became_active":"2023-09-23T19:27:24.271165+0000","last_became_peered":"2023-09-23T19:27:24.271165+0000","last_unstale":"2023-09-23T19:36:55.404674+0000","last_undegraded":"2023-09-23T19:36:55.404674+0000","last_fullsized":"2023-09-23T19:36:55.404674+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":354,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.414215+0000","last_change":"2023-09-23T19:18:09.016297+0000","last_active":"2023-09-23T19:36:55.414215+0000","last_peered":"2023-09-23T19:36:55.414215+0000","last_clean":"2023-09-23T19:36:55.414215+0000","last_became_active":"2023-09-23T19:18:09.015436+0000","last_became_peered":"2023-09-23T19:18:09.015436+0000","last_unstale":"2023-09-23T19:36:55.414215+0000","last_undegraded":"2023-09-23T19:36:55.414215+0000","last_fullsized":"2023-09-23T19:36:55.414215+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:59:33.840751+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:59:33.840751+0000","last_clean_scrub_stamp":"2023-09-23T18:59:33.840751+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":92,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.404320+0000","last_change":"2023-09-23T19:32:49.734148+0000","last_active":"2023-09-23T19:36:55.404320+0000","last_peered":"2023-09-23T19:36:55.404320+0000","last_clean":"2023-09-23T19:36:55.404320+0000","last_became_active":"2023-09-23T19:32:49.733512+0000","last_became_peered":"2023-09-23T19:32:49.733512+0000","last_unstale":"2023-09-23T19:36:55.404320+0000","last_undegraded":"2023-09-23T19:36:55.404320+0000","last_fullsized":"2023-09-23T19:36:55.404320+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":102,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.403850+0000","last_change":"2023-09-23T19:05:52.786771+0000","last_active":"2023-09-23T19:36:55.403850+0000","last_peered":"2023-09-23T19:36:55.403850+0000","last_clean":"2023-09-23T19:36:55.403850+0000","last_became_active":"2023-09-23T19:05:52.785797+0000","last_became_peered":"2023-09-23T19:05:52.785797+0000","last_unstale":"2023-09-23T19:36:55.403850+0000","last_undegraded":"2023-09-23T19:36:55.403850+0000","last_fullsized":"2023-09-23T19:36:55.403850+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"152'5475","reported_seq":5492,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-09-23T19:37:37.600532+0000","last_change":"2023-09-23T19:36:14.519271+0000","last_active":"2023-09-23T19:37:37.600532+0000","last_peered":"2023-09-23T19:37:37.600532+0000","last_clean":"2023-09-23T19:37:37.600532+0000","last_became_active":"2023-09-23T19:36:14.518961+0000","last_became_peered":"2023-09-23T19:36:14.518961+0000","last_unstale":"2023-09-23T19:37:37.600532+0000","last_undegraded":"2023-09-23T19:37:37.600532+0000","last_fullsized":"2023-09-23T19:37:37.600532+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_clean_scrub_stamp":"2023-09-23T19:36:13.468134+0000","objects_scrubbed":0,"log_size":5475,"log_dups_size":0,"ondisk_log_size":5475,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:06:14.200688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":358809600,"num_objects":5475,"num_object_clones":0,"num_object_copies":16425,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5475,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5475,"num_write_kb":350400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":137,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.414123+0000","last_change":"2023-09-23T19:22:42.150636+0000","last_active":"2023-09-23T19:36:55.414123+0000","last_peered":"2023-09-23T19:36:55.414123+0000","last_clean":"2023-09-23T19:36:55.414123+0000","last_became_active":"2023-09-23T19:22:42.150045+0000","last_became_peered":"2023-09-23T19:22:42.150045+0000","last_unstale":"2023-09-23T19:36:55.414123+0000","last_undegraded":"2023-09-23T19:36:55.414123+0000","last_fullsized":"2023-09-23T19:36:55.414123+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":112,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.395953+0000","last_change":"2023-09-23T19:28:07.889194+0000","last_active":"2023-09-23T19:36:55.395953+0000","last_peered":"2023-09-23T19:36:55.395953+0000","last_clean":"2023-09-23T19:36:55.395953+0000","last_became_active":"2023-09-23T19:28:07.888823+0000","last_became_peered":"2023-09-23T19:28:07.888823+0000","last_unstale":"2023-09-23T19:36:55.395953+0000","last_undegraded":"2023-09-23T19:36:55.395953+0000","last_fullsized":"2023-09-23T19:36:55.395953+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:23.464688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":127,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-09-23T19:37:20.406633+0000","last_change":"2023-09-23T19:27:24.256410+0000","last_active":"2023-09-23T19:37:20.406633+0000","last_peered":"2023-09-23T19:37:20.406633+0000","last_clean":"2023-09-23T19:37:20.406633+0000","last_became_active":"2023-09-23T19:27:24.256052+0000","last_became_peered":"2023-09-23T19:27:24.256052+0000","last_unstale":"2023-09-23T19:37:20.406633+0000","last_undegraded":"2023-09-23T19:37:20.406633+0000","last_fullsized":"2023-09-23T19:37:20.406633+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:11.951400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"152'5530","reported_seq":5546,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-09-23T19:37:39.640394+0000","last_change":"2023-09-23T19:36:14.525740+0000","last_active":"2023-09-23T19:37:39.640394+0000","last_peered":"2023-09-23T19:37:39.640394+0000","last_clean":"2023-09-23T19:37:39.640394+0000","last_became_active":"2023-09-23T19:36:14.525356+0000","last_became_peered":"2023-09-23T19:36:14.525356+0000","last_unstale":"2023-09-23T19:37:39.640394+0000","last_undegraded":"2023-09-23T19:37:39.640394+0000","last_fullsized":"2023-09-23T19:37:39.640394+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_clean_scrub_stamp":"2023-09-23T19:36:13.468134+0000","objects_scrubbed":0,"log_size":5530,"log_dups_size":0,"ondisk_log_size":5530,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":362414080,"num_objects":5530,"num_object_clones":0,"num_object_copies":16590,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5530,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5530,"num_write_kb":353920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":122,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.413157+0000","last_change":"2023-09-23T19:18:09.015853+0000","last_active":"2023-09-23T19:36:55.413157+0000","last_peered":"2023-09-23T19:36:55.413157+0000","last_clean":"2023-09-23T19:36:55.413157+0000","last_became_active":"2023-09-23T19:18:09.015338+0000","last_became_peered":"2023-09-23T19:18:09.015338+0000","last_unstale":"2023-09-23T19:36:55.413157+0000","last_undegraded":"2023-09-23T19:36:55.413157+0000","last_fullsized":"2023-09-23T19:36:55.413157+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":96,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:37:16.886374+0000","last_change":"2023-09-23T19:27:24.266241+0000","last_active":"2023-09-23T19:37:16.886374+0000","last_peered":"2023-09-23T19:37:16.886374+0000","last_clean":"2023-09-23T19:37:16.886374+0000","last_became_active":"2023-09-23T19:27:24.265764+0000","last_became_peered":"2023-09-23T19:27:24.265764+0000","last_unstale":"2023-09-23T19:37:16.886374+0000","last_undegraded":"2023-09-23T19:37:16.886374+0000","last_fullsized":"2023-09-23T19:37:16.886374+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:07.694732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":97,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.396636+0000","last_change":"2023-09-23T19:18:09.005823+0000","last_active":"2023-09-23T19:36:55.396636+0000","last_peered":"2023-09-23T19:36:55.396636+0000","last_clean":"2023-09-23T19:36:55.396636+0000","last_became_active":"2023-09-23T19:18:09.005207+0000","last_became_peered":"2023-09-23T19:18:09.005207+0000","last_unstale":"2023-09-23T19:36:55.396636+0000","last_undegraded":"2023-09-23T19:36:55.396636+0000","last_fullsized":"2023-09-23T19:36:55.396636+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:05.520188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"152'5548","reported_seq":5563,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-09-23T19:37:40.049570+0000","last_change":"2023-09-23T19:36:14.528771+0000","last_active":"2023-09-23T19:37:40.049570+0000","last_peered":"2023-09-23T19:37:40.049570+0000","last_clean":"2023-09-23T19:37:40.049570+0000","last_became_active":"2023-09-23T19:36:14.528397+0000","last_became_peered":"2023-09-23T19:36:14.528397+0000","last_unstale":"2023-09-23T19:37:40.049570+0000","last_undegraded":"2023-09-23T19:37:40.049570+0000","last_fullsized":"2023-09-23T19:37:40.049570+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_clean_scrub_stamp":"2023-09-23T19:36:13.468134+0000","objects_scrubbed":0,"log_size":5548,"log_dups_size":0,"ondisk_log_size":5548,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":363593728,"num_objects":5548,"num_object_clones":0,"num_object_copies":16644,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5548,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5548,"num_write_kb":355072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":56,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.413103+0000","last_change":"2023-09-23T19:32:49.737323+0000","last_active":"2023-09-23T19:36:55.413103+0000","last_peered":"2023-09-23T19:36:55.413103+0000","last_clean":"2023-09-23T19:36:55.413103+0000","last_became_active":"2023-09-23T19:32:49.736937+0000","last_became_peered":"2023-09-23T19:32:49.736937+0000","last_unstale":"2023-09-23T19:36:55.413103+0000","last_undegraded":"2023-09-23T19:36:55.413103+0000","last_fullsized":"2023-09-23T19:36:55.413103+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":80,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.389528+0000","last_change":"2023-09-23T19:22:42.145506+0000","last_active":"2023-09-23T19:36:55.389528+0000","last_peered":"2023-09-23T19:36:55.389528+0000","last_clean":"2023-09-23T19:36:55.389528+0000","last_became_active":"2023-09-23T19:22:42.143991+0000","last_became_peered":"2023-09-23T19:22:42.143991+0000","last_unstale":"2023-09-23T19:36:55.389528+0000","last_undegraded":"2023-09-23T19:36:55.389528+0000","last_fullsized":"2023-09-23T19:36:55.389528+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:56.875175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":93,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.414424+0000","last_change":"2023-09-23T19:32:49.734592+0000","last_active":"2023-09-23T19:36:55.414424+0000","last_peered":"2023-09-23T19:36:55.414424+0000","last_clean":"2023-09-23T19:36:55.414424+0000","last_became_active":"2023-09-23T19:32:49.733905+0000","last_became_peered":"2023-09-23T19:32:49.733905+0000","last_unstale":"2023-09-23T19:36:55.414424+0000","last_undegraded":"2023-09-23T19:36:55.414424+0000","last_fullsized":"2023-09-23T19:36:55.414424+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"152'5609","reported_seq":5625,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-09-23T19:37:39.642186+0000","last_change":"2023-09-23T19:36:14.522170+0000","last_active":"2023-09-23T19:37:39.642186+0000","last_peered":"2023-09-23T19:37:39.642186+0000","last_clean":"2023-09-23T19:37:39.642186+0000","last_became_active":"2023-09-23T19:36:14.521790+0000","last_became_peered":"2023-09-23T19:36:14.521790+0000","last_unstale":"2023-09-23T19:37:39.642186+0000","last_undegraded":"2023-09-23T19:37:39.642186+0000","last_fullsized":"2023-09-23T19:37:39.642186+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:36:13.468134+0000","last_clean_scrub_stamp":"2023-09-23T19:36:13.468134+0000","objects_scrubbed":0,"log_size":5609,"log_dups_size":0,"ondisk_log_size":5609,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":367591424,"num_objects":5609,"num_object_clones":0,"num_object_copies":16827,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5609,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5609,"num_write_kb":358976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":122,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.404083+0000","last_change":"2023-09-23T19:27:24.271827+0000","last_active":"2023-09-23T19:36:55.404083+0000","last_peered":"2023-09-23T19:36:55.404083+0000","last_clean":"2023-09-23T19:36:55.404083+0000","last_became_active":"2023-09-23T19:27:24.271328+0000","last_became_peered":"2023-09-23T19:27:24.271328+0000","last_unstale":"2023-09-23T19:36:55.404083+0000","last_undegraded":"2023-09-23T19:36:55.404083+0000","last_fullsized":"2023-09-23T19:36:55.404083+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":115,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-09-23T19:37:20.405270+0000","last_change":"2023-09-23T19:22:42.149694+0000","last_active":"2023-09-23T19:37:20.405270+0000","last_peered":"2023-09-23T19:37:20.405270+0000","last_clean":"2023-09-23T19:37:20.405270+0000","last_became_active":"2023-09-23T19:22:42.148194+0000","last_became_peered":"2023-09-23T19:22:42.148194+0000","last_unstale":"2023-09-23T19:37:20.405270+0000","last_undegraded":"2023-09-23T19:37:20.405270+0000","last_fullsized":"2023-09-23T19:37:20.405270+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:47.842383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":101,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.407026+0000","last_change":"2023-09-23T19:05:52.138620+0000","last_active":"2023-09-23T19:36:55.407026+0000","last_peered":"2023-09-23T19:36:55.407026+0000","last_clean":"2023-09-23T19:36:55.407026+0000","last_became_active":"2023-09-23T19:05:52.138243+0000","last_became_peered":"2023-09-23T19:05:52.138243+0000","last_unstale":"2023-09-23T19:36:55.407026+0000","last_undegraded":"2023-09-23T19:36:55.407026+0000","last_fullsized":"2023-09-23T19:36:55.407026+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":98,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.397316+0000","last_change":"2023-09-23T19:18:09.008811+0000","last_active":"2023-09-23T19:36:55.397316+0000","last_peered":"2023-09-23T19:36:55.397316+0000","last_clean":"2023-09-23T19:36:55.397316+0000","last_became_active":"2023-09-23T19:18:09.007255+0000","last_became_peered":"2023-09-23T19:18:09.007255+0000","last_unstale":"2023-09-23T19:36:55.397316+0000","last_undegraded":"2023-09-23T19:36:55.397316+0000","last_fullsized":"2023-09-23T19:36:55.397316+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:25.874038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":136,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-09-23T19:37:20.416626+0000","last_change":"2023-09-23T19:28:07.876141+0000","last_active":"2023-09-23T19:37:20.416626+0000","last_peered":"2023-09-23T19:37:20.416626+0000","last_clean":"2023-09-23T19:37:20.416626+0000","last_became_active":"2023-09-23T19:28:07.875802+0000","last_became_peered":"2023-09-23T19:28:07.875802+0000","last_unstale":"2023-09-23T19:37:20.416626+0000","last_undegraded":"2023-09-23T19:37:20.416626+0000","last_fullsized":"2023-09-23T19:37:20.416626+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:38.947881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":121,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.403975+0000","last_change":"2023-09-23T19:22:42.153825+0000","last_active":"2023-09-23T19:36:55.403975+0000","last_peered":"2023-09-23T19:36:55.403975+0000","last_clean":"2023-09-23T19:36:55.403975+0000","last_became_active":"2023-09-23T19:22:42.153416+0000","last_became_peered":"2023-09-23T19:22:42.153416+0000","last_unstale":"2023-09-23T19:36:55.403975+0000","last_undegraded":"2023-09-23T19:36:55.403975+0000","last_fullsized":"2023-09-23T19:36:55.403975+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":140,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.405855+0000","last_change":"2023-09-23T19:28:07.899164+0000","last_active":"2023-09-23T19:36:55.405855+0000","last_peered":"2023-09-23T19:36:55.405855+0000","last_clean":"2023-09-23T19:36:55.405855+0000","last_became_active":"2023-09-23T19:28:07.898812+0000","last_became_peered":"2023-09-23T19:28:07.898812+0000","last_unstale":"2023-09-23T19:36:55.405855+0000","last_undegraded":"2023-09-23T19:36:55.405855+0000","last_fullsized":"2023-09-23T19:36:55.405855+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":106,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.403430+0000","last_change":"2023-09-23T19:05:52.107570+0000","last_active":"2023-09-23T19:36:55.403430+0000","last_peered":"2023-09-23T19:36:55.403430+0000","last_clean":"2023-09-23T19:36:55.403430+0000","last_became_active":"2023-09-23T19:05:52.107215+0000","last_became_peered":"2023-09-23T19:05:52.107215+0000","last_unstale":"2023-09-23T19:36:55.403430+0000","last_undegraded":"2023-09-23T19:36:55.403430+0000","last_fullsized":"2023-09-23T19:36:55.403430+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":131,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.407598+0000","last_change":"2023-09-23T19:28:07.896983+0000","last_active":"2023-09-23T19:36:55.407598+0000","last_peered":"2023-09-23T19:36:55.407598+0000","last_clean":"2023-09-23T19:36:55.407598+0000","last_became_active":"2023-09-23T19:28:07.896577+0000","last_became_peered":"2023-09-23T19:28:07.896577+0000","last_unstale":"2023-09-23T19:36:55.407598+0000","last_undegraded":"2023-09-23T19:36:55.407598+0000","last_fullsized":"2023-09-23T19:36:55.407598+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":46,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.405174+0000","last_change":"2023-09-23T19:32:49.732683+0000","last_active":"2023-09-23T19:36:55.405174+0000","last_peered":"2023-09-23T19:36:55.405174+0000","last_clean":"2023-09-23T19:36:55.405174+0000","last_became_active":"2023-09-23T19:32:49.732188+0000","last_became_peered":"2023-09-23T19:32:49.732188+0000","last_unstale":"2023-09-23T19:36:55.405174+0000","last_undegraded":"2023-09-23T19:36:55.405174+0000","last_fullsized":"2023-09-23T19:36:55.405174+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":134,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.400431+0000","last_change":"2023-09-23T19:32:49.725466+0000","last_active":"2023-09-23T19:36:55.400431+0000","last_peered":"2023-09-23T19:36:55.400431+0000","last_clean":"2023-09-23T19:36:55.400431+0000","last_became_active":"2023-09-23T19:32:49.725048+0000","last_became_peered":"2023-09-23T19:32:49.725048+0000","last_unstale":"2023-09-23T19:36:55.400431+0000","last_undegraded":"2023-09-23T19:36:55.400431+0000","last_fullsized":"2023-09-23T19:36:55.400431+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:44:08.728220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":115,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.396181+0000","last_change":"2023-09-23T19:27:24.267931+0000","last_active":"2023-09-23T19:36:55.396181+0000","last_peered":"2023-09-23T19:36:55.396181+0000","last_clean":"2023-09-23T19:36:55.396181+0000","last_became_active":"2023-09-23T19:27:24.267552+0000","last_became_peered":"2023-09-23T19:27:24.267552+0000","last_unstale":"2023-09-23T19:36:55.396181+0000","last_undegraded":"2023-09-23T19:36:55.396181+0000","last_fullsized":"2023-09-23T19:36:55.396181+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:12.425257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":110,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-09-23T19:36:55.405145+0000","last_change":"2023-09-23T19:06:36.673002+0000","last_active":"2023-09-23T19:36:55.405145+0000","last_peered":"2023-09-23T19:36:55.405145+0000","last_clean":"2023-09-23T19:36:55.405145+0000","last_became_active":"2023-09-23T19:06:36.671911+0000","last_became_peered":"2023-09-23T19:06:36.671911+0000","last_unstale":"2023-09-23T19:36:55.405145+0000","last_undegraded":"2023-09-23T19:36:55.405145+0000","last_fullsized":"2023-09-23T19:36:55.405145+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":5761859584,"num_objects":87919,"num_object_clones":0,"num_object_copies":263757,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":87919,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":87919,"num_write_kb":5626816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17290952704,"data_stored":17290952704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":87919,"ondisk_log_size":87919,"up":48,"acting":48,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430626,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2310036,"kb_used_data":1422348,"kb_used_omap":0,"kb_used_meta":887680,"kb_avail":91435116,"statfs":{"total":95995035648,"available":93629558784,"internally_reserved":0,"allocated":1456484352,"data_stored":1455999697,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":908984320},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:36:38 2023","interfaces":[{"interface":"back","average":{"1min":0.83299999999999996,"5min":2.7240000000000002,"15min":1.657},"min":{"1min":0.53600000000000003,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":1.2070000000000001,"5min":92.944999999999993,"15min":92.944999999999993},"last":4.1459999999999999},{"interface":"front","average":{"1min":0.97599999999999998,"5min":2.7389999999999999,"15min":1.661},"min":{"1min":0.55800000000000005,"5min":0.29499999999999998,"15min":0.28999999999999998},"max":{"1min":2.8690000000000002,"5min":92.763999999999996,"15min":92.763999999999996},"last":4.5810000000000004}]},{"osd":1,"last update":"Sat Sep 23 19:37:05 2023","interfaces":[{"interface":"back","average":{"1min":0.66900000000000004,"5min":3.0739999999999998,"15min":1.8680000000000001},"min":{"1min":0.37,"5min":0.29199999999999998,"15min":0.28499999999999998},"max":{"1min":1.1080000000000001,"5min":93.757999999999996,"15min":93.757999999999996},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.84399999999999997,"5min":2.9860000000000002,"15min":1.851},"min":{"1min":0.39000000000000001,"5min":0.29999999999999999,"15min":0.29599999999999999},"max":{"1min":1.726,"5min":93.835999999999999,"15min":93.835999999999999},"last":0.60299999999999998}]},{"osd":2,"last update":"Sat Sep 23 19:36:38 2023","interfaces":[{"interface":"back","average":{"1min":0.84099999999999997,"5min":1.677,"15min":1.1259999999999999},"min":{"1min":0.36899999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":1.0449999999999999,"5min":7.9320000000000004,"15min":7.9320000000000004},"last":2.004},{"interface":"front","average":{"1min":0.96699999999999997,"5min":1.2709999999999999,"15min":1.073},"min":{"1min":0.52700000000000002,"5min":0.308,"15min":0.308},"max":{"1min":1.2210000000000001,"5min":12.574,"15min":12.574},"last":1.9139999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:37:26 2023","interfaces":[{"interface":"back","average":{"1min":0.84299999999999997,"5min":1.7509999999999999,"15min":1.3600000000000001},"min":{"1min":0.36799999999999999,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":2.8599999999999999,"5min":10.65,"15min":19.637},"last":0.438},{"interface":"front","average":{"1min":1.173,"5min":1.5109999999999999,"15min":1.339},"min":{"1min":0.41699999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":3.0609999999999999,"5min":12.608000000000001,"15min":20.204000000000001},"last":3.9300000000000002}]},{"osd":4,"last update":"Sat Sep 23 19:37:05 2023","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":1.377,"15min":1.1459999999999999},"min":{"1min":0.38900000000000001,"5min":0.252,"15min":0.252},"max":{"1min":1.0189999999999999,"5min":9.7699999999999996,"15min":9.7699999999999996},"last":0.90400000000000003},{"interface":"front","average":{"1min":0.93100000000000005,"5min":1.2,"15min":1.073},"min":{"1min":0.40200000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":1.7649999999999999,"5min":12.509,"15min":12.509},"last":0.91700000000000004}]},{"osd":5,"last update":"Sat Sep 23 19:37:05 2023","interfaces":[{"interface":"back","average":{"1min":1.127,"5min":1.877,"15min":1.3799999999999999},"min":{"1min":0.40799999999999997,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.581,"5min":13.342000000000001,"15min":13.342000000000001},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.97599999999999998,"5min":1.9490000000000001,"15min":1.5109999999999999},"min":{"1min":0.47799999999999998,"5min":0.31,"15min":0.31},"max":{"1min":2.2770000000000001,"5min":9.8979999999999997,"15min":9.8979999999999997},"last":0.58399999999999996}]},{"osd":6,"last update":"Sat Sep 23 19:37:05 2023","interfaces":[{"interface":"back","average":{"1min":1.161,"5min":1.522,"15min":1.29},"min":{"1min":0.56899999999999995,"5min":0.316,"15min":0.316},"max":{"1min":4.008,"5min":9.8629999999999995,"15min":9.8629999999999995},"last":0.76600000000000001},{"interface":"front","average":{"1min":1.4950000000000001,"5min":1.4279999999999999,"15min":1.1819999999999999},"min":{"1min":0.68100000000000005,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":5.7759999999999998,"5min":12.631,"15min":12.631},"last":0.56999999999999995}]}]},{"osd":6,"up_from":42,"seq":180388626858,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3055876,"kb_used_data":2470268,"kb_used_omap":0,"kb_used_meta":585600,"kb_avail":90689276,"statfs":{"total":95995035648,"available":92865818624,"internally_reserved":0,"allocated":2529554432,"data_stored":2529085700,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":599654400},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,2,3,6],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:36:49 2023","interfaces":[{"interface":"back","average":{"1min":1.0169999999999999,"5min":2.302,"15min":1.524},"min":{"1min":0.377,"5min":0.25900000000000001,"15min":0.24099999999999999},"max":{"1min":3.3799999999999999,"5min":87.283000000000001,"15min":87.283000000000001},"last":4.5199999999999996},{"interface":"front","average":{"1min":1.7190000000000001,"5min":2.4289999999999998,"15min":1.681},"min":{"1min":0.56899999999999995,"5min":0.30399999999999999,"15min":0.26600000000000001},"max":{"1min":6.5419999999999998,"5min":86.286000000000001,"15min":86.286000000000001},"last":1.587}]},{"osd":1,"last update":"Sat Sep 23 19:37:09 2023","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":1.5149999999999999,"15min":1.4930000000000001},"min":{"1min":0.315,"5min":0.253,"15min":0.253},"max":{"1min":2.0910000000000002,"5min":8.0899999999999999,"15min":11.351000000000001},"last":4.5720000000000001},{"interface":"front","average":{"1min":2.0310000000000001,"5min":1.53,"15min":1.585},"min":{"1min":0.60899999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":6.0949999999999998,"5min":7.4889999999999999,"15min":12.728999999999999},"last":2.1480000000000001}]},{"osd":2,"last update":"Sat Sep 23 19:36:39 2023","interfaces":[{"interface":"back","average":{"1min":0.93500000000000005,"5min":0.79600000000000004,"15min":0.88900000000000001},"min":{"1min":0.47699999999999998,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.48,"5min":4.3840000000000003,"15min":4.3840000000000003},"last":0.318},{"interface":"front","average":{"1min":0.88600000000000001,"5min":1.081,"15min":0.96699999999999997},"min":{"1min":0.34200000000000003,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":1.5589999999999999,"5min":8.0600000000000005,"15min":8.0600000000000005},"last":0.67900000000000005}]},{"osd":3,"last update":"Sat Sep 23 19:37:28 2023","interfaces":[{"interface":"back","average":{"1min":1.454,"5min":1.97,"15min":1.339},"min":{"1min":0.379,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":4.5250000000000004,"5min":60.726999999999997,"15min":60.726999999999997},"last":0.69599999999999995},{"interface":"front","average":{"1min":0.96199999999999997,"5min":2.2549999999999999,"15min":1.4139999999999999},"min":{"1min":0.36299999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":2.3740000000000001,"5min":58.304000000000002,"15min":58.304000000000002},"last":0.64100000000000001}]},{"osd":4,"last update":"Sat Sep 23 19:37:28 2023","interfaces":[{"interface":"back","average":{"1min":1.98,"5min":1.131,"15min":1.0329999999999999},"min":{"1min":0.41599999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.8929999999999998,"5min":5.8929999999999998,"15min":5.8929999999999998},"last":1.502},{"interface":"front","average":{"1min":1.5049999999999999,"5min":1.026,"15min":1.123},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.30499999999999999},"max":{"1min":5.6210000000000004,"5min":5.6210000000000004,"15min":8.5540000000000003},"last":1.5289999999999999}]},{"osd":5,"last update":"Sat Sep 23 19:37:28 2023","interfaces":[{"interface":"back","average":{"1min":2.2610000000000001,"5min":1.53,"15min":1.2769999999999999},"min":{"1min":0.436,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":8.9710000000000001,"5min":8.9710000000000001,"15min":8.9710000000000001},"last":1.548},{"interface":"front","average":{"1min":2.339,"5min":1.9510000000000001,"15min":1.585},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":6.6449999999999996,"5min":9.7219999999999995,"15min":9.7219999999999995},"last":0.73799999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:36:54 2023","interfaces":[{"interface":"back","average":{"1min":1.851,"5min":1.298,"15min":1.181},"min":{"1min":0.61599999999999999,"5min":0.41599999999999998,"15min":0.38300000000000001},"max":{"1min":5.9580000000000002,"5min":5.9580000000000002,"15min":8.7949999999999999},"last":1.617},{"interface":"front","average":{"1min":1.3049999999999999,"5min":1.464,"15min":1.3740000000000001},"min":{"1min":0.55300000000000005,"5min":0.41899999999999998,"15min":0.39300000000000002},"max":{"1min":2.5870000000000002,"5min":8.8059999999999992,"15min":8.8059999999999992},"last":2.0409999999999999}]}]},{"osd":1,"up_from":98,"seq":420906795190,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1461960,"kb_used_data":1059840,"kb_used_omap":4,"kb_used_meta":402107,"kb_avail":92283192,"statfs":{"total":95995035648,"available":94497988608,"internally_reserved":0,"allocated":1085276160,"data_stored":1084628759,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":411758478},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:37:14 2023","interfaces":[{"interface":"back","average":{"1min":0.76500000000000001,"5min":1.3360000000000001,"15min":1.466},"min":{"1min":0.24399999999999999,"5min":0.185,"15min":0.185},"max":{"1min":2.399,"5min":7.2610000000000001,"15min":14.99},"last":0.28100000000000003},{"interface":"front","average":{"1min":0.751,"5min":1.7949999999999999,"15min":1.6399999999999999},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":2.177,"5min":9.5470000000000006,"15min":12.215},"last":0.63500000000000001}]},{"osd":2,"last update":"Sat Sep 23 19:36:43 2023","interfaces":[{"interface":"back","average":{"1min":0.79100000000000004,"5min":1.621,"15min":1.0289999999999999},"min":{"1min":0.45000000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":1.425,"5min":10.212999999999999,"15min":10.212999999999999},"last":0.309},{"interface":"front","average":{"1min":0.66600000000000004,"5min":1.5620000000000001,"15min":0.97999999999999998},"min":{"1min":0.34100000000000003,"5min":0.246,"15min":0.246},"max":{"1min":0.95799999999999996,"5min":11.308,"15min":11.308},"last":1.9490000000000001}]},{"osd":3,"last update":"Sat Sep 23 19:37:23 2023","interfaces":[{"interface":"back","average":{"1min":0.86799999999999999,"5min":1.788,"15min":1.1299999999999999},"min":{"1min":0.29199999999999998,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":2.4510000000000001,"5min":10.612,"15min":10.612},"last":0.32300000000000001},{"interface":"front","average":{"1min":1.1279999999999999,"5min":1.9890000000000001,"15min":1.329},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.26500000000000001},"max":{"1min":4.7309999999999999,"5min":12.457000000000001,"15min":12.457000000000001},"last":0.40200000000000002}]},{"osd":4,"last update":"Sat Sep 23 19:37:14 2023","interfaces":[{"interface":"back","average":{"1min":0.98699999999999999,"5min":1.772,"15min":1.5389999999999999},"min":{"1min":0.40400000000000003,"5min":0.38900000000000001,"15min":0.35899999999999999},"max":{"1min":1.948,"5min":9.6690000000000005,"15min":9.6690000000000005},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.99299999999999999,"5min":1.6140000000000001,"15min":1.5900000000000001},"min":{"1min":0.48099999999999998,"5min":0.36199999999999999,"15min":0.34399999999999997},"max":{"1min":1.9299999999999999,"5min":10.061999999999999,"15min":11.409000000000001},"last":0.52900000000000003}]},{"osd":5,"last update":"Sat Sep 23 19:37:14 2023","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":2.3980000000000001,"15min":1.905},"min":{"1min":0.55400000000000005,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":3.4950000000000001,"5min":13.273999999999999,"15min":13.273999999999999},"last":0.499},{"interface":"front","average":{"1min":1.1120000000000001,"5min":1.9390000000000001,"15min":1.6140000000000001},"min":{"1min":0.50600000000000001,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":2.423,"5min":7.3310000000000004,"15min":9.218},"last":0.47299999999999998}]},{"osd":6,"last update":"Sat Sep 23 19:37:14 2023","interfaces":[{"interface":"back","average":{"1min":1.5549999999999999,"5min":1.796,"15min":1.6479999999999999},"min":{"1min":0.55100000000000005,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":6.1399999999999997,"5min":12.429,"15min":12.429},"last":0.38600000000000001},{"interface":"front","average":{"1min":1.5109999999999999,"5min":2.0009999999999999,"15min":1.7629999999999999},"min":{"1min":0.63200000000000001,"5min":0.39100000000000001,"15min":0.38800000000000001},"max":{"1min":6.181,"5min":9.641,"15min":9.641},"last":0.67300000000000004}]},{"osd":7,"last update":"Sat Sep 23 19:37:14 2023","interfaces":[{"interface":"back","average":{"1min":1.0269999999999999,"5min":2.343,"15min":1.891},"min":{"1min":0.54100000000000004,"5min":0.432,"15min":0.40999999999999998},"max":{"1min":1.6180000000000001,"5min":11.391,"15min":11.391},"last":0.60199999999999998},{"interface":"front","average":{"1min":1.139,"5min":2.1339999999999999,"15min":1.8089999999999999},"min":{"1min":0.48199999999999998,"5min":0.44900000000000001,"15min":0.42699999999999999},"max":{"1min":2.4340000000000002,"5min":11.374000000000001,"15min":12.709},"last":0.55100000000000005}]}]},{"osd":0,"up_from":85,"seq":365072220396,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3903320,"kb_used_data":3426448,"kb_used_omap":5,"kb_used_meta":476794,"kb_avail":89841832,"statfs":{"total":95995035648,"available":91998035968,"internally_reserved":0,"allocated":3508682752,"data_stored":3508024002,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":488237690},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,1,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 19:37:04 2023","interfaces":[{"interface":"back","average":{"1min":1.0589999999999999,"5min":1.379,"15min":1.454},"min":{"1min":0.29799999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":3.718,"5min":7.3680000000000003,"15min":13.557},"last":1.0740000000000001},{"interface":"front","average":{"1min":0.91600000000000004,"5min":1.7270000000000001,"15min":1.5660000000000001},"min":{"1min":0.28199999999999997,"5min":0.26000000000000001,"15min":0.253},"max":{"1min":2.464,"5min":12.939,"15min":12.939},"last":0.224}]},{"osd":2,"last update":"Sat Sep 23 19:36:41 2023","interfaces":[{"interface":"back","average":{"1min":1.117,"5min":1.2529999999999999,"15min":0.92100000000000004},"min":{"1min":0.37,"5min":0.214,"15min":0.214},"max":{"1min":9.827,"5min":11.861000000000001,"15min":11.861000000000001},"last":1.1459999999999999},{"interface":"front","average":{"1min":1.026,"5min":1.3109999999999999,"15min":0.96399999999999997},"min":{"1min":0.41199999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":4.718,"5min":10.212,"15min":10.212},"last":0.39500000000000002}]},{"osd":3,"last update":"Sat Sep 23 19:37:17 2023","interfaces":[{"interface":"back","average":{"1min":1.0589999999999999,"5min":1.6319999999999999,"15min":1.095},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.251},"max":{"1min":3.0350000000000001,"5min":10.430999999999999,"15min":10.430999999999999},"last":0.64100000000000001},{"interface":"front","average":{"1min":1.2969999999999999,"5min":1.7889999999999999,"15min":1.294},"min":{"1min":0.44600000000000001,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":2.573,"5min":12.991,"15min":12.991},"last":0.50700000000000001}]},{"osd":4,"last update":"Sat Sep 23 19:36:49 2023","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":1.4750000000000001,"15min":1.371},"min":{"1min":0.503,"5min":0.37,"15min":0.32100000000000001},"max":{"1min":1.97,"5min":10.002000000000001,"15min":10.002000000000001},"last":0.66900000000000004},{"interface":"front","average":{"1min":1.1060000000000001,"5min":1.2190000000000001,"15min":1.3879999999999999},"min":{"1min":0.66100000000000003,"5min":0.36399999999999999,"15min":0.35599999999999998},"max":{"1min":2.8759999999999999,"5min":4.6529999999999996,"15min":10.282999999999999},"last":0.58999999999999997}]},{"osd":5,"last update":"Sat Sep 23 19:36:49 2023","interfaces":[{"interface":"back","average":{"1min":1.157,"5min":1.9930000000000001,"15min":1.7749999999999999},"min":{"1min":0.61299999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":2.3359999999999999,"5min":13.154,"15min":13.154},"last":3.806},{"interface":"front","average":{"1min":1.103,"5min":2.2890000000000001,"15min":1.7669999999999999},"min":{"1min":0.48199999999999998,"5min":0.36899999999999999,"15min":0.33400000000000002},"max":{"1min":1.925,"5min":13.109999999999999,"15min":13.109999999999999},"last":0.91100000000000003}]},{"osd":6,"last update":"Sat Sep 23 19:36:49 2023","interfaces":[{"interface":"back","average":{"1min":1.1830000000000001,"5min":1.4950000000000001,"15min":1.466},"min":{"1min":0.53000000000000003,"5min":0.38400000000000001,"15min":0.36499999999999999},"max":{"1min":3.2080000000000002,"5min":6.2329999999999997,"15min":8.532},"last":0.69299999999999995},{"interface":"front","average":{"1min":1.2889999999999999,"5min":1.411,"15min":1.5640000000000001},"min":{"1min":0.75700000000000001,"5min":0.36699999999999999,"15min":0.36399999999999999},"max":{"1min":2.8239999999999998,"5min":6.1130000000000004,"15min":10.308},"last":0.624}]},{"osd":7,"last update":"Sat Sep 23 19:36:49 2023","interfaces":[{"interface":"back","average":{"1min":1.2370000000000001,"5min":1.7450000000000001,"15min":1.6060000000000001},"min":{"1min":0.58299999999999996,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":3.7690000000000001,"5min":12.385,"15min":13.218},"last":1.167},{"interface":"front","average":{"1min":1.1599999999999999,"5min":2.0259999999999998,"15min":1.7050000000000001},"min":{"1min":0.58199999999999996,"5min":0.435,"15min":0.38300000000000001},"max":{"1min":1.9950000000000001,"5min":8.8450000000000006,"15min":8.8450000000000006},"last":0.61099999999999999}]}]},{"osd":2,"up_from":111,"seq":476741369981,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2312156,"kb_used_data":2067348,"kb_used_omap":7,"kb_used_meta":244792,"kb_avail":91432996,"statfs":{"total":95995035648,"available":93627387904,"internally_reserved":0,"allocated":2116964352,"data_stored":2116356114,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":250667089},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:36:37 2023","interfaces":[{"interface":"back","average":{"1min":0.71899999999999997,"5min":1.446,"15min":0.95499999999999996},"min":{"1min":0.38100000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":1.744,"5min":59.670999999999999,"15min":59.670999999999999},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.77900000000000003,"5min":1.4319999999999999,"15min":0.99299999999999999},"min":{"1min":0.35399999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":1.946,"5min":59.774000000000001,"15min":59.774000000000001},"last":0.27300000000000002}]},{"osd":1,"last update":"Sat Sep 23 19:36:37 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":3.452,"15min":1.6459999999999999},"min":{"1min":0.29199999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.93600000000000005,"5min":92.933000000000007,"15min":92.933000000000007},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.78700000000000003,"5min":3.3340000000000001,"15min":1.6739999999999999},"min":{"1min":0.47499999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":1.827,"5min":92.432000000000002,"15min":92.432000000000002},"last":0.68400000000000005}]},{"osd":3,"last update":"Sat Sep 23 19:37:34 2023","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":2.2349999999999999,"15min":1.2849999999999999},"min":{"1min":0.23899999999999999,"5min":0.22,"15min":0.22},"max":{"1min":1.8919999999999999,"5min":92.350999999999999,"15min":92.350999999999999},"last":0.313},{"interface":"front","average":{"1min":1.1899999999999999,"5min":2.5219999999999998,"15min":1.446},"min":{"1min":0.29399999999999998,"5min":0.248,"15min":0.248},"max":{"1min":3.4359999999999999,"5min":96.236999999999995,"15min":96.236999999999995},"last":0.50800000000000001}]},{"osd":4,"last update":"Sat Sep 23 19:36:37 2023","interfaces":[{"interface":"back","average":{"1min":0.97099999999999997,"5min":0.84399999999999997,"15min":0.96299999999999997},"min":{"1min":0.52700000000000002,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":2.0339999999999998,"5min":3.1899999999999999,"15min":3.1899999999999999},"last":3.085},{"interface":"front","average":{"1min":0.92500000000000004,"5min":1.054,"15min":1.022},"min":{"1min":0.41499999999999998,"5min":0.37,"15min":0.37},"max":{"1min":1.526,"5min":9.0199999999999996,"15min":9.0199999999999996},"last":3.778}]},{"osd":5,"last update":"Sat Sep 23 19:36:37 2023","interfaces":[{"interface":"back","average":{"1min":1.089,"5min":1.569,"15min":1.266},"min":{"1min":0.64500000000000002,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":2.1629999999999998,"5min":11.058999999999999,"15min":11.058999999999999},"last":0.54500000000000004},{"interface":"front","average":{"1min":1.0940000000000001,"5min":1.544,"15min":1.262},"min":{"1min":0.67100000000000004,"5min":0.309,"15min":0.309},"max":{"1min":2.0790000000000002,"5min":6.5179999999999998,"15min":6.5179999999999998},"last":0.38300000000000001}]},{"osd":6,"last update":"Sat Sep 23 19:36:37 2023","interfaces":[{"interface":"back","average":{"1min":1.0009999999999999,"5min":1.1359999999999999,"15min":1.0960000000000001},"min":{"1min":0.63300000000000001,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":1.661,"5min":9.0739999999999998,"15min":9.0739999999999998},"last":0.70699999999999996},{"interface":"front","average":{"1min":1.181,"5min":1.0820000000000001,"15min":1.1339999999999999},"min":{"1min":0.67900000000000005,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":3.5819999999999999,"5min":5.6959999999999997,"15min":5.6959999999999997},"last":0.76400000000000001}]},{"osd":7,"last update":"Sat Sep 23 19:36:37 2023","interfaces":[{"interface":"back","average":{"1min":1.161,"5min":1.6639999999999999,"15min":1.353},"min":{"1min":0.502,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.9910000000000001,"5min":12.339,"15min":12.339},"last":0.437},{"interface":"front","average":{"1min":1.044,"5min":1.7190000000000001,"15min":1.3480000000000001},"min":{"1min":0.56499999999999995,"5min":0.37,"15min":0.37},"max":{"1min":1.423,"5min":12.236000000000001,"15min":12.236000000000001},"last":0.89200000000000002}]}]},{"osd":3,"up_from":116,"seq":498216206453,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2087344,"kb_used_data":1797928,"kb_used_omap":2,"kb_used_meta":289405,"kb_avail":91657808,"statfs":{"total":95995035648,"available":93857595392,"internally_reserved":0,"allocated":1841078272,"data_stored":1840508637,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":296350773},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:37:23 2023","interfaces":[{"interface":"back","average":{"1min":0.76700000000000002,"5min":1.7689999999999999,"15min":1.087},"min":{"1min":0.33600000000000002,"5min":0.20200000000000001,"15min":0.19900000000000001},"max":{"1min":1.8300000000000001,"5min":13.411,"15min":13.411},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.64500000000000002,"5min":1.4470000000000001,"15min":1.0669999999999999},"min":{"1min":0.34100000000000003,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":1.3360000000000001,"5min":10.77,"15min":10.77},"last":0.28100000000000003}]},{"osd":1,"last update":"Sat Sep 23 19:37:23 2023","interfaces":[{"interface":"back","average":{"1min":0.73799999999999999,"5min":1.655,"15min":1.151},"min":{"1min":0.28100000000000003,"5min":0.246,"15min":0.246},"max":{"1min":3.0859999999999999,"5min":12.119,"15min":13.659000000000001},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.53800000000000003,"5min":2.02,"15min":1.321},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.73299999999999998,"5min":16.262,"15min":16.262},"last":0.77200000000000002}]},{"osd":2,"last update":"Sat Sep 23 19:37:23 2023","interfaces":[{"interface":"back","average":{"1min":0.85099999999999998,"5min":1.2789999999999999,"15min":0.95999999999999996},"min":{"1min":0.245,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":2.8839999999999999,"5min":6.9950000000000001,"15min":6.9950000000000001},"last":0.311},{"interface":"front","average":{"1min":0.73599999999999999,"5min":1.569,"15min":1.141},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.6870000000000001,"5min":7.0670000000000002,"15min":7.0670000000000002},"last":0.51400000000000001}]},{"osd":4,"last update":"Sat Sep 23 19:37:23 2023","interfaces":[{"interface":"back","average":{"1min":0.98999999999999999,"5min":1.474,"15min":1.2410000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":3.1030000000000002,"5min":7.0259999999999998,"15min":7.0259999999999998},"last":1.4019999999999999},{"interface":"front","average":{"1min":1.1140000000000001,"5min":1.3360000000000001,"15min":1.2390000000000001},"min":{"1min":0.379,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":2.8319999999999999,"5min":10.926,"15min":10.926},"last":1.371}]},{"osd":5,"last update":"Sat Sep 23 19:37:23 2023","interfaces":[{"interface":"back","average":{"1min":0.90900000000000003,"5min":1.792,"15min":1.425},"min":{"1min":0.40300000000000002,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":2.0550000000000002,"5min":10.82,"15min":10.82},"last":0.377},{"interface":"front","average":{"1min":1.383,"5min":1.7829999999999999,"15min":1.3420000000000001},"min":{"1min":0.38,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":7.5049999999999999,"5min":8.7710000000000008,"15min":8.7710000000000008},"last":0.624}]},{"osd":6,"last update":"Sat Sep 23 19:37:23 2023","interfaces":[{"interface":"back","average":{"1min":1.3300000000000001,"5min":1.589,"15min":1.3300000000000001},"min":{"1min":0.51100000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":3.5,"5min":10.859,"15min":10.859},"last":2.766},{"interface":"front","average":{"1min":1.218,"5min":1.3899999999999999,"15min":1.242},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":3.548,"5min":8.0190000000000001,"15min":8.0190000000000001},"last":2.6240000000000001}]},{"osd":7,"last update":"Sat Sep 23 19:37:23 2023","interfaces":[{"interface":"back","average":{"1min":1.149,"5min":1.7170000000000001,"15min":1.3680000000000001},"min":{"1min":0.45200000000000001,"5min":0.376,"15min":0.32400000000000001},"max":{"1min":2.9700000000000002,"5min":11.259,"15min":11.259},"last":0.68000000000000005},{"interface":"front","average":{"1min":0.83799999999999997,"5min":1.641,"15min":1.357},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":1.5069999999999999,"5min":7.0279999999999996,"15min":7.0279999999999996},"last":0.70699999999999996}]}]},{"osd":4,"up_from":30,"seq":128849019323,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3390156,"kb_used_data":2804548,"kb_used_omap":0,"kb_used_meta":585600,"kb_avail":90354996,"statfs":{"total":95995035648,"available":92523515904,"internally_reserved":0,"allocated":2871857152,"data_stored":2871380075,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":599654400},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:36:45 2023","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":1.0229999999999999,"15min":1.111},"min":{"1min":0.56699999999999995,"5min":0.255,"15min":0.255},"max":{"1min":3.0550000000000002,"5min":7.391,"15min":8.7680000000000007},"last":2.7810000000000001},{"interface":"front","average":{"1min":0.85999999999999999,"5min":1.1599999999999999,"15min":1.24},"min":{"1min":0.313,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":1.4059999999999999,"5min":7.9349999999999996,"15min":7.9349999999999996},"last":0.40000000000000002}]},{"osd":1,"last update":"Sat Sep 23 19:37:17 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":1.377,"15min":1.4159999999999999},"min":{"1min":0.33700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":3.355,"5min":11.093,"15min":12.372},"last":0.53200000000000003},{"interface":"front","average":{"1min":1.0660000000000001,"5min":1.5449999999999999,"15min":1.4650000000000001},"min":{"1min":0.41299999999999998,"5min":0.32500000000000001,"15min":0.31},"max":{"1min":4.3319999999999999,"5min":11.061,"15min":11.061},"last":2.806}]},{"osd":2,"last update":"Sat Sep 23 19:37:38 2023","interfaces":[{"interface":"back","average":{"1min":0.93700000000000006,"5min":0.96999999999999997,"15min":0.90000000000000002},"min":{"1min":0.32800000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":5.4660000000000002,"5min":7.9699999999999998,"15min":7.9699999999999998},"last":2.4279999999999999},{"interface":"front","average":{"1min":1.4890000000000001,"5min":1.016,"15min":0.92600000000000005},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":8.0709999999999997,"5min":8.0709999999999997,"15min":8.0709999999999997},"last":3.3849999999999998}]},{"osd":3,"last update":"Sat Sep 23 19:37:30 2023","interfaces":[{"interface":"back","average":{"1min":1.159,"5min":1.857,"15min":1.4279999999999999},"min":{"1min":0.41399999999999998,"5min":0.378,"15min":0.36299999999999999},"max":{"1min":4.4749999999999996,"5min":44.340000000000003,"15min":49.515000000000001},"last":3.988},{"interface":"front","average":{"1min":1.3660000000000001,"5min":1.909,"15min":1.4450000000000001},"min":{"1min":0.41999999999999998,"5min":0.378,"15min":0.374},"max":{"1min":5.2400000000000002,"5min":44.203000000000003,"15min":51.518999999999998},"last":4.3499999999999996}]},{"osd":5,"last update":"Sat Sep 23 19:36:51 2023","interfaces":[{"interface":"back","average":{"1min":1.3979999999999999,"5min":1.6990000000000001,"15min":1.423},"min":{"1min":0.45700000000000002,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":8.1899999999999995,"5min":10.118,"15min":12.459},"last":2.9580000000000002},{"interface":"front","average":{"1min":1.2450000000000001,"5min":1.4730000000000001,"15min":1.3440000000000001},"min":{"1min":0.61099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":4.649,"5min":8.1120000000000001,"15min":12.478},"last":2.851}]},{"osd":6,"last update":"Sat Sep 23 19:37:17 2023","interfaces":[{"interface":"back","average":{"1min":1.6619999999999999,"5min":1.419,"15min":1.345},"min":{"1min":0.439,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":6.3719999999999999,"5min":6.9119999999999999,"15min":7.0810000000000004},"last":2.9119999999999999},{"interface":"front","average":{"1min":1.53,"5min":1.2190000000000001,"15min":1.25},"min":{"1min":0.45200000000000001,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":5.2729999999999997,"5min":7.319,"15min":7.319},"last":2.823}]},{"osd":7,"last update":"Sat Sep 23 19:37:17 2023","interfaces":[{"interface":"back","average":{"1min":1.0800000000000001,"5min":1.3660000000000001,"15min":1.29},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":3.3780000000000001,"5min":7.2729999999999997,"15min":7.8479999999999999},"last":2.8940000000000001},{"interface":"front","average":{"1min":1.5149999999999999,"5min":1.6379999999999999,"15min":1.331},"min":{"1min":0.47599999999999998,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":4.4340000000000002,"5min":7.242,"15min":7.242},"last":2.8610000000000002}]}]},{"osd":5,"up_from":36,"seq":154618823090,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2676084,"kb_used_data":1777388,"kb_used_omap":0,"kb_used_meta":898688,"kb_avail":91069068,"statfs":{"total":95995035648,"available":93254725632,"internally_reserved":0,"allocated":1820045312,"data_stored":1819587925,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920256512},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:36:50 2023","interfaces":[{"interface":"back","average":{"1min":1.149,"5min":1.8899999999999999,"15min":1.4059999999999999},"min":{"1min":0.29499999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":7.024,"5min":32.643999999999998,"15min":32.643999999999998},"last":3.3679999999999999},{"interface":"front","average":{"1min":1.5760000000000001,"5min":2.657,"15min":1.849},"min":{"1min":0.60199999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":7.5869999999999997,"5min":31.876999999999999,"15min":31.876999999999999},"last":4.7220000000000004}]},{"osd":1,"last update":"Sat Sep 23 19:37:17 2023","interfaces":[{"interface":"back","average":{"1min":0.76700000000000002,"5min":1.8160000000000001,"15min":1.409},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.27200000000000002},"max":{"1min":2.036,"5min":32.253999999999998,"15min":32.253999999999998},"last":3.286},{"interface":"front","average":{"1min":0.95699999999999996,"5min":2.6120000000000001,"15min":1.931},"min":{"1min":0.307,"5min":0.307,"15min":0.26700000000000002},"max":{"1min":2.665,"5min":32.277999999999999,"15min":32.277999999999999},"last":2.101}]},{"osd":2,"last update":"Sat Sep 23 19:36:50 2023","interfaces":[{"interface":"back","average":{"1min":1.056,"5min":1.476,"15min":1.145},"min":{"1min":0.51500000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":2.9340000000000002,"5min":10.164,"15min":10.164},"last":0.54500000000000004},{"interface":"front","average":{"1min":1.151,"5min":1.1120000000000001,"15min":1.079},"min":{"1min":0.39400000000000002,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":2.855,"5min":9.0050000000000008,"15min":9.0050000000000008},"last":4.335}]},{"osd":3,"last update":"Sat Sep 23 19:37:17 2023","interfaces":[{"interface":"back","average":{"1min":1.623,"5min":1.8160000000000001,"15min":1.2849999999999999},"min":{"1min":0.40999999999999998,"5min":0.311,"15min":0.311},"max":{"1min":5.7599999999999998,"5min":10.885999999999999,"15min":10.885999999999999},"last":0.58399999999999996},{"interface":"front","average":{"1min":1.1459999999999999,"5min":1.3320000000000001,"15min":1.2010000000000001},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":4.218,"5min":6.5369999999999999,"15min":6.5369999999999999},"last":3.1080000000000001}]},{"osd":4,"last update":"Sat Sep 23 19:36:50 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":1.921,"15min":1.411},"min":{"1min":0.38900000000000001,"5min":0.33500000000000002,"15min":0.29199999999999998},"max":{"1min":2.7010000000000001,"5min":18.683,"15min":18.683},"last":0.42999999999999999},{"interface":"front","average":{"1min":1.036,"5min":1.4399999999999999,"15min":1.28},"min":{"1min":0.371,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.625,"5min":16.073,"15min":16.073},"last":1.3600000000000001}]},{"osd":6,"last update":"Sat Sep 23 19:37:29 2023","interfaces":[{"interface":"back","average":{"1min":1.865,"5min":1.333,"15min":1.222},"min":{"1min":0.40200000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":6.5270000000000001,"5min":6.5270000000000001,"15min":7.3739999999999997},"last":3.0739999999999998},{"interface":"front","average":{"1min":2.3849999999999998,"5min":1.7829999999999999,"15min":1.4339999999999999},"min":{"1min":0.54900000000000004,"5min":0.309,"15min":0.309},"max":{"1min":6.5590000000000002,"5min":9.4749999999999996,"15min":9.4749999999999996},"last":1.425}]},{"osd":7,"last update":"Sat Sep 23 19:36:59 2023","interfaces":[{"interface":"back","average":{"1min":1.325,"5min":1.988,"15min":1.4750000000000001},"min":{"1min":0.437,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":4.9009999999999998,"5min":11.913,"15min":11.913},"last":0.61299999999999999},{"interface":"front","average":{"1min":1.099,"5min":1.756,"15min":1.357},"min":{"1min":0.371,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.972,"5min":13.18,"15min":13.18},"last":3.4089999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3518431232,"data_stored":3518431232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1086062592,"data_stored":1086062592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2123497472,"data_stored":2123497472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1841102848,"data_stored":1841102848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2888171520,"data_stored":2888171520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1830354944,"data_stored":1830354944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2543779840,"data_stored":2543779840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1459552256,"data_stored":1459552256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T19:37:45.340 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T19:37:45.340 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T19:37:45.340 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-09-23T19:37:45.340 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T19:37:45.341 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T19:37:45.341 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-09-23T19:37:45.398 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:45.429 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:45.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:45.432 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:45.491 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 70 16 93472 93456 83.4339 54.4375 0.0115202 0.0119799 2023-09-23T19:37:45.532 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:45.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:45 smithi016 bash[75004]: cluster 2023-09-23T19:37:44.094855+0000 mgr.y (mgr.25325) 907 : cluster 0 pgmap v831: 89 pgs: 89 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 58 MiB/s wr, 932 op/s 2023-09-23T19:37:45.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:45 smithi016 bash[75004]: audit 2023-09-23T19:37:44.564659+0000 mon.a (mon.0) 1122 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:37:45.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:45 smithi016 bash[77995]: cluster 2023-09-23T19:37:44.094855+0000 mgr.y (mgr.25325) 907 : cluster 0 pgmap v831: 89 pgs: 89 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 58 MiB/s wr, 932 op/s 2023-09-23T19:37:45.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:45 smithi016 bash[77995]: audit 2023-09-23T19:37:44.564659+0000 mon.a (mon.0) 1122 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:37:45.558 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:45.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:45.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:45.662 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:45.674 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:45 smithi092 bash[58855]: cluster 2023-09-23T19:37:44.094855+0000 mgr.y (mgr.25325) 907 : cluster 0 pgmap v831: 89 pgs: 89 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 58 MiB/s wr, 932 op/s 2023-09-23T19:37:45.674 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:45 smithi092 bash[58855]: audit 2023-09-23T19:37:44.564659+0000 mon.a (mon.0) 1122 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:37:45.688 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:45.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:45.694 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:37:45.759 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:37:45.795 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:45.822 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:45.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:45.826 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:45.928 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:45.957 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:45.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:45.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:46.036 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:37:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:37:45] "GET /metrics HTTP/1.1" 200 36305 "" "Prometheus/2.33.4" 2023-09-23T19:37:46.060 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:46.085 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:46.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:46.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:46.191 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:46.220 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:46.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:46.224 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:37:46.325 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:46.360 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:46.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:46.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:46.464 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:46.490 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:46.491 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 71 16 94374 94358 83.0527 56.375 0.0397046 0.0120343 2023-09-23T19:37:46.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:46.495 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:46.596 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:46.625 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:46.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:46.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:37:46.731 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:46.758 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:46.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:46.761 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:37:46.862 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:46.889 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:46.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:46.894 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:46.995 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:47.027 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:47.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:47.029 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:47.130 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:47.154 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:47.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:47.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:47.258 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:47.289 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:47.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:47.293 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:47.385 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:37:47.393 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:47.413 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T19:37:47.469 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:47.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:47.472 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:37:47.492 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 72 16 95270 95254 82.6757 56 0.0154045 0.012088 2023-09-23T19:37:47.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:47 smithi092 bash[58855]: cluster 2023-09-23T19:37:46.095732+0000 mgr.y (mgr.25325) 908 : cluster 0 pgmap v832: 89 pgs: 89 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 60 MiB/s wr, 954 op/s 2023-09-23T19:37:47.572 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:47.608 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:47.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:47.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:47.713 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:47.727 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:47 smithi016 bash[77995]: cluster 2023-09-23T19:37:46.095732+0000 mgr.y (mgr.25325) 908 : cluster 0 pgmap v832: 89 pgs: 89 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 60 MiB/s wr, 954 op/s 2023-09-23T19:37:47.727 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:47 smithi016 bash[75004]: cluster 2023-09-23T19:37:46.095732+0000 mgr.y (mgr.25325) 908 : cluster 0 pgmap v832: 89 pgs: 89 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 60 MiB/s wr, 954 op/s 2023-09-23T19:37:47.745 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:47.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:47.748 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:47.848 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:47.884 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:47.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:47.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:47.989 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:48.016 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:48.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:48.019 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:48.120 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:48.155 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:48.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:48.160 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:48.261 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:48.287 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:48.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:48.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:37:48.393 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:48.419 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:48.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:48.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:37:48.492 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 73 16 96134 96118 82.2828 54 0.00922199 0.0121469 2023-09-23T19:37:48.529 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:48.563 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:48.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:48.569 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:37:48.670 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:48.704 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:48.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:48.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:37:48.808 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:48.837 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:48.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:48.840 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:37:48.941 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:48.977 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:48.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:48.980 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:37:49.081 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:49.105 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:49.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:49.110 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:37:49.211 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:49.239 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:49.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:49.246 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:37:49.346 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:49.375 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:49.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:49.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:49.454 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:49 smithi092 bash[58855]: cluster 2023-09-23T19:37:48.096569+0000 mgr.y (mgr.25325) 909 : cluster 0 pgmap v833: 89 pgs: 89 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s 2023-09-23T19:37:49.480 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:49.491 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:49 smithi016 bash[75004]: cluster 2023-09-23T19:37:48.096569+0000 mgr.y (mgr.25325) 909 : cluster 0 pgmap v833: 89 pgs: 89 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s 2023-09-23T19:37:49.491 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:49 smithi016 bash[77995]: cluster 2023-09-23T19:37:48.096569+0000 mgr.y (mgr.25325) 909 : cluster 0 pgmap v833: 89 pgs: 89 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s 2023-09-23T19:37:49.493 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 74 16 97030 97014 81.9275 56 0.0285312 0.0122003 2023-09-23T19:37:49.510 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:49.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:49.515 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:37:49.615 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:49.650 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:49.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:49.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:49.754 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:49.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:49.791 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:49.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:49.892 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:49.918 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:49.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:49.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:50.024 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:50.073 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:50.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:50.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:50.182 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:50.185 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-09-23T19:37:50.185 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:37:50.211 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:50.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:50.223 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:37:50.324 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:50.355 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:50.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:50.358 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:50.459 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:50.492 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:50.492 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 75 16 97814 97798 81.4884 49 0.0301016 0.0122655 2023-09-23T19:37:50.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:50.495 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:37:50.595 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:50.630 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:50.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:50.663 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:37:50.764 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:50.787 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:50.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:50.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:37:50.894 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:50.933 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:50.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:50.937 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:51.037 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:51.105 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:51.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:51.108 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:37:51.209 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:51.254 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:51.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:51.262 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:51.301 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T19:37:51.362 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:51.400 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:51.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:51.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:37:51.493 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 76 16 98605 98589 81.0667 49.4375 0.0188707 0.0123292 2023-09-23T19:37:51.509 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:51.519 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:51 smithi092 bash[58855]: cluster 2023-09-23T19:37:50.099560+0000 mgr.y (mgr.25325) 910 : cluster 0 pgmap v834: 89 pgs: 89 active+clean; 5.8 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 55 MiB/s wr, 885 op/s 2023-09-23T19:37:51.519 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:51 smithi092 bash[58855]: audit 2023-09-23T19:37:51.302614+0000 mon.a (mon.0) 1123 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:37:51.519 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:51 smithi092 bash[58855]: audit 2023-09-23T19:37:51.321377+0000 mon.a (mon.0) 1124 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:37:51.542 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:51.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:51.547 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:51.647 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:51.657 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:51 smithi016 bash[75004]: cluster 2023-09-23T19:37:50.099560+0000 mgr.y (mgr.25325) 910 : cluster 0 pgmap v834: 89 pgs: 89 active+clean; 5.8 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 55 MiB/s wr, 885 op/s 2023-09-23T19:37:51.657 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:51 smithi016 bash[75004]: audit 2023-09-23T19:37:51.302614+0000 mon.a (mon.0) 1123 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:37:51.657 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:51 smithi016 bash[75004]: audit 2023-09-23T19:37:51.321377+0000 mon.a (mon.0) 1124 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:37:51.658 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:51 smithi016 bash[77995]: cluster 2023-09-23T19:37:50.099560+0000 mgr.y (mgr.25325) 910 : cluster 0 pgmap v834: 89 pgs: 89 active+clean; 5.8 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 55 MiB/s wr, 885 op/s 2023-09-23T19:37:51.658 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:51 smithi016 bash[77995]: audit 2023-09-23T19:37:51.302614+0000 mon.a (mon.0) 1123 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:37:51.658 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:51 smithi016 bash[77995]: audit 2023-09-23T19:37:51.321377+0000 mon.a (mon.0) 1124 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:37:51.674 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:51.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:51.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:37:51.719 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:37:51.780 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:51.804 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:51.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:51.807 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:51.908 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:51.934 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:51.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:51.938 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:52.039 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:52.071 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:52.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:52.077 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:52.177 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:52.201 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:52.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:52.205 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:52.305 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:52.351 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:52.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:52.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:37:52.455 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:52.491 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:52.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:52.494 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:52.495 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 77 16 99383 99367 80.6428 48.625 0.0335893 0.0123932 2023-09-23T19:37:52.595 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:52.624 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:52.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:52.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:52.731 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:52.743 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:37:52.743 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:37:52.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:52.776 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:52.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:37:52.877 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:52.910 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:52.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:52.914 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:53.015 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:53.040 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:53.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:53.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:53.144 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:53.178 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:53.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:53.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:53.283 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:53.313 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:53.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:53.315 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:37:53.416 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:53.457 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:53.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:53.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:53.495 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 78 16 100124 100108 80.2026 46.3125 0.01816 0.0124619 2023-09-23T19:37:53.567 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:53.574 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:53 smithi016 bash[77995]: cluster 2023-09-23T19:37:52.100197+0000 mgr.y (mgr.25325) 911 : cluster 0 pgmap v835: 89 pgs: 89 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 896 op/s 2023-09-23T19:37:53.575 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:53 smithi016 bash[75004]: cluster 2023-09-23T19:37:52.100197+0000 mgr.y (mgr.25325) 911 : cluster 0 pgmap v835: 89 pgs: 89 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 896 op/s 2023-09-23T19:37:53.596 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:53.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:53.604 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:53.704 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:53.715 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:53 smithi092 bash[58855]: cluster 2023-09-23T19:37:52.100197+0000 mgr.y (mgr.25325) 911 : cluster 0 pgmap v835: 89 pgs: 89 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 896 op/s 2023-09-23T19:37:53.731 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:53.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:53.737 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:37:53.837 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:53.877 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:53.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:53.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:53.984 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:54.022 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:54.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:54.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:37:54.126 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:54.153 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:54.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:54.159 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:54.259 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:54.287 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:54.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:54.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:37:54.396 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:54.428 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:54.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:54.436 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:37:54.496 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 79 16 100982 100966 79.8661 53.625 0.00997076 0.0125149 2023-09-23T19:37:54.536 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:54.570 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:54.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:54.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:37:54.667 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:54 smithi016 bash[75004]: audit 2023-09-23T19:37:53.563780+0000 mon.a (mon.0) 1125 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:37:54.667 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:54 smithi016 bash[75004]: audit 2023-09-23T19:37:53.575330+0000 mon.a (mon.0) 1126 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:37:54.667 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:54 smithi016 bash[75004]: audit 2023-09-23T19:37:54.155837+0000 mon.a (mon.0) 1127 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:37:54.667 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:54 smithi016 bash[75004]: audit 2023-09-23T19:37:54.157255+0000 mon.a (mon.0) 1128 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:37:54.667 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:54 smithi016 bash[75004]: audit 2023-09-23T19:37:54.168479+0000 mon.a (mon.0) 1129 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:37:54.667 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:54 smithi016 bash[77995]: audit 2023-09-23T19:37:53.563780+0000 mon.a (mon.0) 1125 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:37:54.667 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:54 smithi016 bash[77995]: audit 2023-09-23T19:37:53.575330+0000 mon.a (mon.0) 1126 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:37:54.668 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:54 smithi016 bash[77995]: audit 2023-09-23T19:37:54.155837+0000 mon.a (mon.0) 1127 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:37:54.668 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:54 smithi016 bash[77995]: audit 2023-09-23T19:37:54.157255+0000 mon.a (mon.0) 1128 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:37:54.668 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:54 smithi016 bash[77995]: audit 2023-09-23T19:37:54.168479+0000 mon.a (mon.0) 1129 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:37:54.675 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:54.706 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:54.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:54.710 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:54.810 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:54.822 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:54 smithi092 bash[58855]: audit 2023-09-23T19:37:53.563780+0000 mon.a (mon.0) 1125 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:37:54.822 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:54 smithi092 bash[58855]: audit 2023-09-23T19:37:53.575330+0000 mon.a (mon.0) 1126 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:37:54.822 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:54 smithi092 bash[58855]: audit 2023-09-23T19:37:54.155837+0000 mon.a (mon.0) 1127 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:37:54.822 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:54 smithi092 bash[58855]: audit 2023-09-23T19:37:54.157255+0000 mon.a (mon.0) 1128 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:37:54.822 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:54 smithi092 bash[58855]: audit 2023-09-23T19:37:54.168479+0000 mon.a (mon.0) 1129 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:37:54.837 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:54.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:54.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:37:54.942 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:54.971 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:54.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:54.974 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:55.074 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:55.103 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:55.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:55.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:55.210 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:55.235 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:55.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:55.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:55.338 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:55.372 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:55.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:55.375 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:55.476 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:55.496 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-09-23T19:37:55.492714+0000 min lat: 0.00156577 max lat: 0.209545 avg lat: 0.0125619 2023-09-23T19:37:55.496 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:37:55.496 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 80 16 101882 101866 79.5709 56.25 0.016522 0.0125619 2023-09-23T19:37:55.509 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:55.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:55.516 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:37:55.617 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:55.651 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:55.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:55.657 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:37:55.758 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:55.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:55 smithi092 bash[58855]: cluster 2023-09-23T19:37:54.100758+0000 mgr.y (mgr.25325) 912 : cluster 0 pgmap v836: 89 pgs: 89 active+clean; 6.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 857 op/s 2023-09-23T19:37:55.802 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:55.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:55.805 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:55.906 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:55.914 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:55 smithi016 bash[75004]: cluster 2023-09-23T19:37:54.100758+0000 mgr.y (mgr.25325) 912 : cluster 0 pgmap v836: 89 pgs: 89 active+clean; 6.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 857 op/s 2023-09-23T19:37:55.915 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:55 smithi016 bash[77995]: cluster 2023-09-23T19:37:54.100758+0000 mgr.y (mgr.25325) 912 : cluster 0 pgmap v836: 89 pgs: 89 active+clean; 6.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 857 op/s 2023-09-23T19:37:55.915 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:37:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:37:55] "GET /metrics HTTP/1.1" 200 36305 "" "Prometheus/2.33.4" 2023-09-23T19:37:55.940 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:55.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:55.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:56.048 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:56.075 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:56.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:56.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:37:56.180 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:56.208 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:56.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:56.212 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:37:56.313 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:56.344 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:56.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:56.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:56.450 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:56.468 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:56.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:56.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:56.496 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 81 16 102685 102669 79.208 50.1875 0.00442635 0.0126071 2023-09-23T19:37:56.503 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:37:56.571 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:56.601 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:56.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:56.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:37:56.705 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:56.735 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:56.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:56.737 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:37:56.838 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:56.880 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:56.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:56.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:37:56.987 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:57.014 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:57.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:57.017 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:57.118 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:57.155 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:37:57.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:57.160 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:37:57.260 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:57.288 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:57.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:57.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:57.394 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:57.428 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:57.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:57.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:57.496 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 82 16 103531 103515 78.8869 52.875 0.0141339 0.0126701 2023-09-23T19:37:57.533 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:57.569 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:57.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:57.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:57.673 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:57.704 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:57.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:57.706 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:57.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:57 smithi092 bash[58855]: cluster 2023-09-23T19:37:56.101481+0000 mgr.y (mgr.25325) 913 : cluster 0 pgmap v837: 89 pgs: 89 active+clean; 6.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 62 MiB/s wr, 990 op/s 2023-09-23T19:37:57.806 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:57.815 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:57 smithi016 bash[77995]: cluster 2023-09-23T19:37:56.101481+0000 mgr.y (mgr.25325) 913 : cluster 0 pgmap v837: 89 pgs: 89 active+clean; 6.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 62 MiB/s wr, 990 op/s 2023-09-23T19:37:57.815 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:57 smithi016 bash[75004]: cluster 2023-09-23T19:37:56.101481+0000 mgr.y (mgr.25325) 913 : cluster 0 pgmap v837: 89 pgs: 89 active+clean; 6.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 62 MiB/s wr, 990 op/s 2023-09-23T19:37:57.842 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:57.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:57.845 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:57.945 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:57.971 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:57.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:57.974 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:58.074 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:58.104 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:37:58.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:58.109 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:37:58.210 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:58.237 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:58.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:58.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:58.341 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:58.371 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:37:58.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:58.375 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:37:58.476 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:58.496 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 83 16 104320 104304 78.53 49.3125 0.0112417 0.0127274 2023-09-23T19:37:58.503 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:37:58.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:58.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:37:58.607 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:58.638 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:58.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:58.641 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:58.742 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:58.767 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:58.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:58.772 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:58.872 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:58.902 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:58.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:58.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:59.010 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:59.072 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:59.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:59.076 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:59.176 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:59.214 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:59.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:59.218 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:59.318 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:59.345 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:37:59.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:59.348 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:37:59.448 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:59.475 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:37:59.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:59.477 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:37:59.496 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 84 16 105181 105165 78.2357 53.8125 0.0186055 0.012776 2023-09-23T19:37:59.578 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:59.598 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:59 smithi092 bash[58855]: cluster 2023-09-23T19:37:58.101947+0000 mgr.y (mgr.25325) 914 : cluster 0 pgmap v838: 89 pgs: 89 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 50 MiB/s wr, 794 op/s 2023-09-23T19:37:59.598 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:37:59 smithi092 bash[58855]: audit 2023-09-23T19:37:58.652291+0000 mon.a (mon.0) 1130 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:37:59.608 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:37:59.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:59.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:37:59.711 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:59.720 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:59 smithi016 bash[77995]: cluster 2023-09-23T19:37:58.101947+0000 mgr.y (mgr.25325) 914 : cluster 0 pgmap v838: 89 pgs: 89 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 50 MiB/s wr, 794 op/s 2023-09-23T19:37:59.720 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:37:59 smithi016 bash[77995]: audit 2023-09-23T19:37:58.652291+0000 mon.a (mon.0) 1130 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:37:59.720 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:59 smithi016 bash[75004]: cluster 2023-09-23T19:37:58.101947+0000 mgr.y (mgr.25325) 914 : cluster 0 pgmap v838: 89 pgs: 89 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 50 MiB/s wr, 794 op/s 2023-09-23T19:37:59.720 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:37:59 smithi016 bash[75004]: audit 2023-09-23T19:37:58.652291+0000 mon.a (mon.0) 1130 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:37:59.734 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:59.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:59.737 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:59.837 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:59.866 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:37:59.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:37:59.869 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:37:59.969 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:00.012 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:00.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:00.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:00.120 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:00.147 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:00.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:00.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:00.256 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:00.285 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:00.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:00.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:00.390 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:00.421 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:00.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:00.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:38:00.496 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 85 16 106026 106010 77.9366 52.8125 0.0265593 0.012825 2023-09-23T19:38:00.530 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:00.556 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:00.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:00.562 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:00.663 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:00.691 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:00.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:00.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:00.795 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:00.838 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:00.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:00.841 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:00.942 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:00.974 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:00.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:00.976 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:01.077 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:01.104 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:01.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:01.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:01.209 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:01.241 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:01.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:01.246 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:01.347 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:01.379 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:01.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:01.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:38:01.485 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:01.499 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 86 16 106847 106831 77.627 51.3125 0.0312686 0.0128764 2023-09-23T19:38:01.520 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:01.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:01.522 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:01.623 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:38:01.646 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:38:01.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:01.648 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:38:01.671 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:01 smithi016 bash[75004]: cluster 2023-09-23T19:38:00.102472+0000 mgr.y (mgr.25325) 915 : cluster 0 pgmap v839: 89 pgs: 89 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 52 MiB/s wr, 831 op/s 2023-09-23T19:38:01.749 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:01.785 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:01.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:01.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:01.892 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:01.900 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:01 smithi092 bash[58855]: cluster 2023-09-23T19:38:00.102472+0000 mgr.y (mgr.25325) 915 : cluster 0 pgmap v839: 89 pgs: 89 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 52 MiB/s wr, 831 op/s 2023-09-23T19:38:01.921 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:01.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:01.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:02.024 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:02.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:01 smithi016 bash[77995]: cluster 2023-09-23T19:38:00.102472+0000 mgr.y (mgr.25325) 915 : cluster 0 pgmap v839: 89 pgs: 89 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 52 MiB/s wr, 831 op/s 2023-09-23T19:38:02.057 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:02.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:02.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:02.164 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:02.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:02.201 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:02.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:02.301 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:02.333 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:02.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:02.336 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:02.436 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:02.462 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:02.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:02.464 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:02.497 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 87 16 107657 107641 77.3166 50.625 0.0114981 0.0129284 2023-09-23T19:38:02.565 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:02.595 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:02.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:02.601 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:02.701 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:02.742 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:02.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:02.748 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:02.849 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:38:02.883 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:38:02.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:02.886 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:38:02.987 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:03.018 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:03.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:03.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:03.126 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:03.140 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-09-23T19:38:03.141 INFO:teuthology.orchestra.run.smithi016.stdout: "success": "" 2023-09-23T19:38:03.141 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-09-23T19:38:03.165 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:03.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:03.169 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:03.270 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:03.309 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:03.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:03.318 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:03.418 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:03.464 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:03.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:03.475 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:03.497 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 88 16 108437 108421 76.9919 48.75 0.0146672 0.0129824 2023-09-23T19:38:03.575 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:03.626 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:03.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:03.630 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:03.730 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:03.760 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:03.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:03.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:03.865 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:03.874 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:03 smithi016 bash[77995]: cluster 2023-09-23T19:38:02.103119+0000 mgr.y (mgr.25325) 916 : cluster 0 pgmap v840: 89 pgs: 89 active+clean; 6.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 49 MiB/s wr, 777 op/s 2023-09-23T19:38:03.875 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:03 smithi016 bash[75004]: cluster 2023-09-23T19:38:02.103119+0000 mgr.y (mgr.25325) 916 : cluster 0 pgmap v840: 89 pgs: 89 active+clean; 6.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 49 MiB/s wr, 777 op/s 2023-09-23T19:38:03.893 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:03.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:03.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:03.998 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:04.007 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:03 smithi092 bash[58855]: cluster 2023-09-23T19:38:02.103119+0000 mgr.y (mgr.25325) 916 : cluster 0 pgmap v840: 89 pgs: 89 active+clean; 6.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 49 MiB/s wr, 777 op/s 2023-09-23T19:38:04.023 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:04.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:04.028 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:04.129 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:38:04.156 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:38:04.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:04.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:38:04.262 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:04.284 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:04.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:04.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:04.391 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:04.430 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:04.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:04.435 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:04.497 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 89 16 109292 109276 76.7272 53.4375 0.0188731 0.013027 2023-09-23T19:38:04.536 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:04.560 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:04.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:04.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:04.665 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:04.698 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:04.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:04.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:04.805 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:04.828 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:04.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:04.831 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:38:04.932 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:04.955 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:04.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:04.960 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:05.061 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:05.117 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:05.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:05.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:05.221 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:05.244 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:05.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:05.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:05.351 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:05.378 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:05.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:05.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:05.484 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:05.497 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Total time run: 90.0089 2023-09-23T19:38:05.497 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Total writes made: 110131 2023-09-23T19:38:05.497 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Write size: 65536 2023-09-23T19:38:05.497 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Object size: 65536 2023-09-23T19:38:05.497 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Bandwidth (MB/sec): 76.4723 2023-09-23T19:38:05.497 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Stddev Bandwidth: 28.1598 2023-09-23T19:38:05.497 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Max bandwidth (MB/sec): 170 2023-09-23T19:38:05.497 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Min bandwidth (MB/sec): 46.3125 2023-09-23T19:38:05.498 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Average IOPS: 1223 2023-09-23T19:38:05.498 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Stddev IOPS: 450.557 2023-09-23T19:38:05.498 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Max IOPS: 2720 2023-09-23T19:38:05.498 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Min IOPS: 741 2023-09-23T19:38:05.498 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Average Latency(s): 0.0130718 2023-09-23T19:38:05.498 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Stddev Latency(s): 0.00966896 2023-09-23T19:38:05.498 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Max latency(s): 0.209545 2023-09-23T19:38:05.498 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Min latency(s): 0.00156577 2023-09-23T19:38:05.505 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:05.506 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T19:38:05.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:05.508 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:05.609 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:05.638 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:05.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:05.641 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:38:05.742 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:05.773 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:05.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:05.777 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:05.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:05 smithi016 bash[75004]: cluster 2023-09-23T19:38:04.103536+0000 mgr.y (mgr.25325) 917 : cluster 0 pgmap v841: 89 pgs: 89 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 48 MiB/s wr, 768 op/s 2023-09-23T19:38:05.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:05 smithi016 bash[77995]: cluster 2023-09-23T19:38:04.103536+0000 mgr.y (mgr.25325) 917 : cluster 0 pgmap v841: 89 pgs: 89 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 48 MiB/s wr, 768 op/s 2023-09-23T19:38:05.786 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:38:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:38:05] "GET /metrics HTTP/1.1" 200 36310 "" "Prometheus/2.33.4" 2023-09-23T19:38:05.879 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:05.888 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:05 smithi092 bash[58855]: cluster 2023-09-23T19:38:04.103536+0000 mgr.y (mgr.25325) 917 : cluster 0 pgmap v841: 89 pgs: 89 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 48 MiB/s wr, 768 op/s 2023-09-23T19:38:05.915 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:05.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:05.921 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:06.021 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:38:06.048 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:38:06.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:06.055 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:38:06.156 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:06.186 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:06.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:06.189 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:06.290 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:06.316 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:06.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:06.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:06.423 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:06.456 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:06.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:06.463 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:06.563 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:38:06.593 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:38:06.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:06.598 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:38:06.698 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:06.728 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:06.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:06.733 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:06.833 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:06.866 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:06.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:06.871 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:06.972 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:07.011 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:07.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:07.016 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:38:07.117 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:07.143 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:07.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:07.145 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:38:07.246 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:38:07.279 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:38:07.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:07.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:38:07.389 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:07.419 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:07.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:07.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:38:07.522 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:07.555 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:07.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:07.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:07.639 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is set 2023-09-23T19:38:07.661 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:07.689 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:07.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:07.692 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:07.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:07 smithi016 bash[77995]: cluster 2023-09-23T19:38:06.104398+0000 mgr.y (mgr.25325) 918 : cluster 0 pgmap v842: 89 pgs: 89 active+clean; 6.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 59 MiB/s wr, 939 op/s 2023-09-23T19:38:07.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:07 smithi016 bash[77995]: audit 2023-09-23T19:38:07.556871+0000 mon.a (mon.0) 1131 : audit 1 from='client.? 172.21.15.16:0/680577490' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:38:07.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:07 smithi016 bash[75004]: cluster 2023-09-23T19:38:06.104398+0000 mgr.y (mgr.25325) 918 : cluster 0 pgmap v842: 89 pgs: 89 active+clean; 6.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 59 MiB/s wr, 939 op/s 2023-09-23T19:38:07.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:07 smithi016 bash[75004]: audit 2023-09-23T19:38:07.556871+0000 mon.a (mon.0) 1131 : audit 1 from='client.? 172.21.15.16:0/680577490' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:38:07.794 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:07.820 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:07.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:07.824 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:07.925 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:07.952 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:07.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:07.954 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:38:08.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:07 smithi092 bash[58855]: cluster 2023-09-23T19:38:06.104398+0000 mgr.y (mgr.25325) 918 : cluster 0 pgmap v842: 89 pgs: 89 active+clean; 6.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 59 MiB/s wr, 939 op/s 2023-09-23T19:38:08.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:07 smithi092 bash[58855]: audit 2023-09-23T19:38:07.556871+0000 mon.a (mon.0) 1131 : audit 1 from='client.? 172.21.15.16:0/680577490' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:38:08.055 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:08.083 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:08.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:08.086 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:38:08.186 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:08.210 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:08.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:08.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:08.314 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:08.339 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:08.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:08.341 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:38:08.442 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:08.470 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:08.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:08.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:08.574 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:08.602 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:08.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:08.605 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:08.624 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:08 smithi092 bash[58855]: cluster 2023-09-23T19:38:07.610013+0000 mon.a (mon.0) 1132 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:38:08.625 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:08 smithi092 bash[58855]: audit 2023-09-23T19:38:07.624692+0000 mon.a (mon.0) 1133 : audit 1 from='client.? 172.21.15.16:0/680577490' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:38:08.625 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:08 smithi092 bash[58855]: cluster 2023-09-23T19:38:07.624767+0000 mon.a (mon.0) 1134 : cluster 0 osdmap e154: 8 total, 8 up, 8 in 2023-09-23T19:38:08.625 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:08 smithi092 bash[58855]: cluster 2023-09-23T19:38:07.629327+0000 osd.4 (osd.4) 80 : cluster 3 failed to encode map e154 with expected crc 2023-09-23T19:38:08.625 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:08 smithi092 bash[58855]: cluster 2023-09-23T19:38:07.634111+0000 osd.6 (osd.6) 70 : cluster 3 failed to encode map e154 with expected crc 2023-09-23T19:38:08.625 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:08 smithi092 bash[58855]: cluster 2023-09-23T19:38:07.638921+0000 osd.5 (osd.5) 82 : cluster 3 failed to encode map e154 with expected crc 2023-09-23T19:38:08.625 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:08 smithi092 bash[58855]: cluster 2023-09-23T19:38:07.640023+0000 osd.5 (osd.5) 83 : cluster 3 failed to encode map e154 with expected crc 2023-09-23T19:38:08.625 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:08 smithi092 bash[58855]: cluster 2023-09-23T19:38:07.640290+0000 osd.7 (osd.7) 68 : cluster 3 failed to encode map e154 with expected crc 2023-09-23T19:38:08.706 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:08.728 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:08.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:08.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:38:08.834 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:08.872 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:08.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:08.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:38:08.883 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:08 smithi016 bash[75004]: cluster 2023-09-23T19:38:07.610013+0000 mon.a (mon.0) 1132 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:38:08.883 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:08 smithi016 bash[75004]: audit 2023-09-23T19:38:07.624692+0000 mon.a (mon.0) 1133 : audit 1 from='client.? 172.21.15.16:0/680577490' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:38:08.883 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:08 smithi016 bash[75004]: cluster 2023-09-23T19:38:07.624767+0000 mon.a (mon.0) 1134 : cluster 0 osdmap e154: 8 total, 8 up, 8 in 2023-09-23T19:38:08.883 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:08 smithi016 bash[75004]: cluster 2023-09-23T19:38:07.629327+0000 osd.4 (osd.4) 80 : cluster 3 failed to encode map e154 with expected crc 2023-09-23T19:38:08.883 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:08 smithi016 bash[75004]: cluster 2023-09-23T19:38:07.634111+0000 osd.6 (osd.6) 70 : cluster 3 failed to encode map e154 with expected crc 2023-09-23T19:38:08.883 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:08 smithi016 bash[75004]: cluster 2023-09-23T19:38:07.638921+0000 osd.5 (osd.5) 82 : cluster 3 failed to encode map e154 with expected crc 2023-09-23T19:38:08.883 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:08 smithi016 bash[75004]: cluster 2023-09-23T19:38:07.640023+0000 osd.5 (osd.5) 83 : cluster 3 failed to encode map e154 with expected crc 2023-09-23T19:38:08.884 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:08 smithi016 bash[75004]: cluster 2023-09-23T19:38:07.640290+0000 osd.7 (osd.7) 68 : cluster 3 failed to encode map e154 with expected crc 2023-09-23T19:38:08.884 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:08 smithi016 bash[77995]: cluster 2023-09-23T19:38:07.610013+0000 mon.a (mon.0) 1132 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:38:08.884 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:08 smithi016 bash[77995]: audit 2023-09-23T19:38:07.624692+0000 mon.a (mon.0) 1133 : audit 1 from='client.? 172.21.15.16:0/680577490' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:38:08.884 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:08 smithi016 bash[77995]: cluster 2023-09-23T19:38:07.624767+0000 mon.a (mon.0) 1134 : cluster 0 osdmap e154: 8 total, 8 up, 8 in 2023-09-23T19:38:08.884 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:08 smithi016 bash[77995]: cluster 2023-09-23T19:38:07.629327+0000 osd.4 (osd.4) 80 : cluster 3 failed to encode map e154 with expected crc 2023-09-23T19:38:08.884 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:08 smithi016 bash[77995]: cluster 2023-09-23T19:38:07.634111+0000 osd.6 (osd.6) 70 : cluster 3 failed to encode map e154 with expected crc 2023-09-23T19:38:08.884 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:08 smithi016 bash[77995]: cluster 2023-09-23T19:38:07.638921+0000 osd.5 (osd.5) 82 : cluster 3 failed to encode map e154 with expected crc 2023-09-23T19:38:08.884 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:08 smithi016 bash[77995]: cluster 2023-09-23T19:38:07.640023+0000 osd.5 (osd.5) 83 : cluster 3 failed to encode map e154 with expected crc 2023-09-23T19:38:08.884 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:08 smithi016 bash[77995]: cluster 2023-09-23T19:38:07.640290+0000 osd.7 (osd.7) 68 : cluster 3 failed to encode map e154 with expected crc 2023-09-23T19:38:08.977 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:09.006 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:09.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:09.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:09.112 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:38:09.139 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:38:09.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:09.144 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:38:09.185 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:38:09.225 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:38:09.245 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:09.272 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:38:09.273 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:09.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:09.277 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:09.315 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:38:09.355 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:38:09.379 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:09.394 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:38:09.404 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:09.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:09.410 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:38:09.429 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:38:09.467 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:38:09.471 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:38:09.471 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:38:09.471 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:38:09.471 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:38:09.471 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:38:09.471 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:38:09.471 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:38:09.471 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:38:09.511 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:09.541 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:09.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:09.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:09.645 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:09.675 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:09.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:09.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:09.779 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:09.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:09 smithi016 bash[77995]: cluster 2023-09-23T19:38:08.104801+0000 mgr.y (mgr.25325) 919 : cluster 0 pgmap v844: 89 pgs: 89 active+clean; 6.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 49 MiB/s wr, 828 op/s 2023-09-23T19:38:09.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:09 smithi016 bash[77995]: audit 2023-09-23T19:38:09.154864+0000 mon.b (mon.1) 44 : audit 0 from='client.? 172.21.15.16:0/922528917' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:38:09.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:09 smithi016 bash[75004]: cluster 2023-09-23T19:38:08.104801+0000 mgr.y (mgr.25325) 919 : cluster 0 pgmap v844: 89 pgs: 89 active+clean; 6.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 49 MiB/s wr, 828 op/s 2023-09-23T19:38:09.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:09 smithi016 bash[75004]: audit 2023-09-23T19:38:09.154864+0000 mon.b (mon.1) 44 : audit 0 from='client.? 172.21.15.16:0/922528917' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:38:09.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:09 smithi092 bash[58855]: cluster 2023-09-23T19:38:08.104801+0000 mgr.y (mgr.25325) 919 : cluster 0 pgmap v844: 89 pgs: 89 active+clean; 6.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 49 MiB/s wr, 828 op/s 2023-09-23T19:38:09.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:09 smithi092 bash[58855]: audit 2023-09-23T19:38:09.154864+0000 mon.b (mon.1) 44 : audit 0 from='client.? 172.21.15.16:0/922528917' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:38:09.807 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:09.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:09.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:09.917 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:09.951 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:09.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:09.953 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:10.053 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:10.083 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:10.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:10.088 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:10.188 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:10.218 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:10.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:10.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:10.325 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:38:10.357 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:38:10.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:10.363 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:38:10.464 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:38:10.482 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:38:10.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:10.484 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:38:10.584 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:10.612 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:10.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:10.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:10.719 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:38:10.752 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:38:10.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:10.758 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:38:10.859 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:38:10.884 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:38:10.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:10.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:38:10.989 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:38:11.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:11.017 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:38:11.017 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:38:11.118 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:11.149 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:11.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:11.154 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:11.255 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:11.284 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:11.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:11.289 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:11.389 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:38:11.415 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:38:11.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:11.485 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:38:11.586 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:11.619 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:11.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:11.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:38:11.726 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:11.755 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:11.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:11.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:11.864 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:38:11.895 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:38:11.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:11.898 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:11 smithi016 bash[75004]: cluster 2023-09-23T19:38:10.106995+0000 mgr.y (mgr.25325) 920 : cluster 0 pgmap v845: 89 pgs: 89 active+clean; 6.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 41 MiB/s wr, 829 op/s 2023-09-23T19:38:11.898 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:11 smithi016 bash[77995]: cluster 2023-09-23T19:38:10.106995+0000 mgr.y (mgr.25325) 920 : cluster 0 pgmap v845: 89 pgs: 89 active+clean; 6.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 41 MiB/s wr, 829 op/s 2023-09-23T19:38:11.899 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:38:12.000 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:12.009 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:11 smithi092 bash[58855]: cluster 2023-09-23T19:38:10.106995+0000 mgr.y (mgr.25325) 920 : cluster 0 pgmap v845: 89 pgs: 89 active+clean; 6.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 41 MiB/s wr, 829 op/s 2023-09-23T19:38:12.030 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:12.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:12.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:12.055 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-09-23T19:38:12.055 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [], 2023-09-23T19:38:12.056 INFO:teuthology.orchestra.run.smithi016.stdout: "complaint_time": 30, 2023-09-23T19:38:12.056 INFO:teuthology.orchestra.run.smithi016.stdout: "num_blocked_ops": 0 2023-09-23T19:38:12.056 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-09-23T19:38:12.133 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:12.159 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:12.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:12.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:12.262 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:12.296 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:12.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:12.300 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:12.400 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:12.433 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:12.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:12.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:12.474 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T19:38:12.475 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T19:38:12.475 INFO:tasks.thrashosds.thrasher:Killing osd 2, live_osds are [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T19:38:12.475 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2023-09-23T19:38:12.475 DEBUG:teuthology.orchestra.run.smithi016:> sudo systemctl stop ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@osd.2 2023-09-23T19:38:12.536 INFO:journalctl@ceph.osd.2.smithi016.stdout:Sep 23 19:38:12 smithi016 systemd[1]: Stopping Ceph osd.2 for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7... 2023-09-23T19:38:12.536 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:12.571 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:12.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:12.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:12.673 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:12.698 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:12.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:12.703 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:38:12.804 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:12.835 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:12.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:12.841 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:12.937 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T19:38:12.941 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:12.965 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:12.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:12.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:13.069 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:13.104 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:13.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:13.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:13.158 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T19:38:13.210 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:13.247 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:13.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:13.253 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:13.286 INFO:journalctl@ceph.osd.2.smithi016.stdout:Sep 23 19:38:13 smithi016 bash[97918]: debug 2023-09-23T19:38:13.064+0000 7f6c09f6a700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-09-23T19:38:13.286 INFO:journalctl@ceph.osd.2.smithi016.stdout:Sep 23 19:38:13 smithi016 bash[97918]: debug 2023-09-23T19:38:13.064+0000 7f6c09f6a700 -1 osd.2 154 *** Got signal Terminated *** 2023-09-23T19:38:13.286 INFO:journalctl@ceph.osd.2.smithi016.stdout:Sep 23 19:38:13 smithi016 bash[97918]: debug 2023-09-23T19:38:13.064+0000 7f6c09f6a700 -1 osd.2 154 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T19:38:13.353 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:13.377 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:13.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:13.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:13.485 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:13.517 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:13.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:13.520 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:13.620 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:13.654 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:13.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:13.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:13.744 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:38:13.758 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:13 smithi016 bash[75004]: cluster 2023-09-23T19:38:12.107657+0000 mgr.y (mgr.25325) 921 : cluster 0 pgmap v846: 89 pgs: 89 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 32 MiB/s wr, 828 op/s 2023-09-23T19:38:13.758 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:13 smithi016 bash[75004]: cluster 2023-09-23T19:38:13.072783+0000 mon.a (mon.0) 1135 : cluster 1 osd.2 marked itself down and dead 2023-09-23T19:38:13.759 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:13 smithi016 bash[77995]: cluster 2023-09-23T19:38:12.107657+0000 mgr.y (mgr.25325) 921 : cluster 0 pgmap v846: 89 pgs: 89 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 32 MiB/s wr, 828 op/s 2023-09-23T19:38:13.759 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:13 smithi016 bash[77995]: cluster 2023-09-23T19:38:13.072783+0000 mon.a (mon.0) 1135 : cluster 1 osd.2 marked itself down and dead 2023-09-23T19:38:13.760 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:13.788 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:13.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:13.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:13.891 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:13.903 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:13 smithi092 bash[58855]: cluster 2023-09-23T19:38:12.107657+0000 mgr.y (mgr.25325) 921 : cluster 0 pgmap v846: 89 pgs: 89 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 32 MiB/s wr, 828 op/s 2023-09-23T19:38:13.903 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:13 smithi092 bash[58855]: cluster 2023-09-23T19:38:13.072783+0000 mon.a (mon.0) 1135 : cluster 1 osd.2 marked itself down and dead 2023-09-23T19:38:13.922 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:13.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:13.924 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:38:14.025 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:14.057 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:14.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:14.060 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:14.161 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:14.191 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:14.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:14.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:14.295 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:14.324 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:14.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:14.327 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:38:14.429 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:14.455 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:14.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:14.458 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:14.559 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:14.583 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:14.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:14.586 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:14.687 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:14.716 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:14.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:14.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:14.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:14 smithi016 bash[77995]: cluster 2023-09-23T19:38:13.634163+0000 mon.a (mon.0) 1136 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T19:38:14.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:14 smithi016 bash[77995]: cluster 2023-09-23T19:38:13.650281+0000 mon.a (mon.0) 1137 : cluster 0 osdmap e155: 8 total, 7 up, 8 in 2023-09-23T19:38:14.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:14 smithi016 bash[77995]: cluster 2023-09-23T19:38:13.656137+0000 osd.6 (osd.6) 71 : cluster 3 failed to encode map e155 with expected crc 2023-09-23T19:38:14.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:14 smithi016 bash[77995]: cluster 2023-09-23T19:38:13.658537+0000 osd.5 (osd.5) 84 : cluster 3 failed to encode map e155 with expected crc 2023-09-23T19:38:14.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:14 smithi016 bash[77995]: cluster 2023-09-23T19:38:13.659662+0000 osd.5 (osd.5) 85 : cluster 3 failed to encode map e155 with expected crc 2023-09-23T19:38:14.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:14 smithi016 bash[77995]: cluster 2023-09-23T19:38:13.662150+0000 osd.4 (osd.4) 81 : cluster 3 failed to encode map e155 with expected crc 2023-09-23T19:38:14.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:14 smithi016 bash[77995]: cluster 2023-09-23T19:38:13.663004+0000 osd.6 (osd.6) 72 : cluster 3 failed to encode map e155 with expected crc 2023-09-23T19:38:14.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:14 smithi016 bash[77995]: audit 2023-09-23T19:38:13.663062+0000 mon.a (mon.0) 1138 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:38:14.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:14 smithi016 bash[77995]: cluster 2023-09-23T19:38:13.664258+0000 osd.4 (osd.4) 82 : cluster 3 failed to encode map e155 with expected crc 2023-09-23T19:38:14.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:14 smithi016 bash[77995]: audit 2023-09-23T19:38:13.664846+0000 mon.a (mon.0) 1139 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:38:14.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:14 smithi016 bash[75004]: cluster 2023-09-23T19:38:13.634163+0000 mon.a (mon.0) 1136 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T19:38:14.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:14 smithi016 bash[75004]: cluster 2023-09-23T19:38:13.650281+0000 mon.a (mon.0) 1137 : cluster 0 osdmap e155: 8 total, 7 up, 8 in 2023-09-23T19:38:14.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:14 smithi016 bash[75004]: cluster 2023-09-23T19:38:13.656137+0000 osd.6 (osd.6) 71 : cluster 3 failed to encode map e155 with expected crc 2023-09-23T19:38:14.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:14 smithi016 bash[75004]: cluster 2023-09-23T19:38:13.658537+0000 osd.5 (osd.5) 84 : cluster 3 failed to encode map e155 with expected crc 2023-09-23T19:38:14.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:14 smithi016 bash[75004]: cluster 2023-09-23T19:38:13.659662+0000 osd.5 (osd.5) 85 : cluster 3 failed to encode map e155 with expected crc 2023-09-23T19:38:14.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:14 smithi016 bash[75004]: cluster 2023-09-23T19:38:13.662150+0000 osd.4 (osd.4) 81 : cluster 3 failed to encode map e155 with expected crc 2023-09-23T19:38:14.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:14 smithi016 bash[75004]: cluster 2023-09-23T19:38:13.663004+0000 osd.6 (osd.6) 72 : cluster 3 failed to encode map e155 with expected crc 2023-09-23T19:38:14.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:14 smithi016 bash[75004]: audit 2023-09-23T19:38:13.663062+0000 mon.a (mon.0) 1138 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:38:14.788 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:14 smithi016 bash[75004]: cluster 2023-09-23T19:38:13.664258+0000 osd.4 (osd.4) 82 : cluster 3 failed to encode map e155 with expected crc 2023-09-23T19:38:14.788 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:14 smithi016 bash[75004]: audit 2023-09-23T19:38:13.664846+0000 mon.a (mon.0) 1139 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:38:14.826 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:14.849 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:14.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:14.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:14.952 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:14.962 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:14 smithi092 bash[58855]: cluster 2023-09-23T19:38:13.634163+0000 mon.a (mon.0) 1136 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T19:38:14.962 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:14 smithi092 bash[58855]: cluster 2023-09-23T19:38:13.650281+0000 mon.a (mon.0) 1137 : cluster 0 osdmap e155: 8 total, 7 up, 8 in 2023-09-23T19:38:14.962 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:14 smithi092 bash[58855]: cluster 2023-09-23T19:38:13.656137+0000 osd.6 (osd.6) 71 : cluster 3 failed to encode map e155 with expected crc 2023-09-23T19:38:14.962 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:14 smithi092 bash[58855]: cluster 2023-09-23T19:38:13.658537+0000 osd.5 (osd.5) 84 : cluster 3 failed to encode map e155 with expected crc 2023-09-23T19:38:14.962 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:14 smithi092 bash[58855]: cluster 2023-09-23T19:38:13.659662+0000 osd.5 (osd.5) 85 : cluster 3 failed to encode map e155 with expected crc 2023-09-23T19:38:14.962 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:14 smithi092 bash[58855]: cluster 2023-09-23T19:38:13.662150+0000 osd.4 (osd.4) 81 : cluster 3 failed to encode map e155 with expected crc 2023-09-23T19:38:14.962 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:14 smithi092 bash[58855]: cluster 2023-09-23T19:38:13.663004+0000 osd.6 (osd.6) 72 : cluster 3 failed to encode map e155 with expected crc 2023-09-23T19:38:14.962 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:14 smithi092 bash[58855]: audit 2023-09-23T19:38:13.663062+0000 mon.a (mon.0) 1138 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:38:14.962 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:14 smithi092 bash[58855]: cluster 2023-09-23T19:38:13.664258+0000 osd.4 (osd.4) 82 : cluster 3 failed to encode map e155 with expected crc 2023-09-23T19:38:14.963 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:14 smithi092 bash[58855]: audit 2023-09-23T19:38:13.664846+0000 mon.a (mon.0) 1139 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:38:14.986 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:14.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:14.992 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:15.093 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:15.118 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:15.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:15.120 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:38:15.221 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:15.254 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:15.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:15.258 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:38:15.359 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:15.386 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:15.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:15.389 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:38:15.489 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:15.498 INFO:journalctl@ceph.osd.2.smithi016.stdout:Sep 23 19:38:15 smithi016 bash[143539]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd-2 2023-09-23T19:38:15.523 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:15.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:15.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:15.626 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:15.658 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:15.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:15.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:38:15.764 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:15.773 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:15 smithi016 bash[77995]: cluster 2023-09-23T19:38:14.108345+0000 mgr.y (mgr.25325) 922 : cluster 0 pgmap v848: 89 pgs: 6 stale+active+clean, 83 active+clean; 6.5 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 127 B/s rd, 7.4 MiB/s wr, 643 op/s 2023-09-23T19:38:15.773 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:15 smithi016 bash[77995]: cluster 2023-09-23T19:38:14.649806+0000 mon.a (mon.0) 1140 : cluster 0 osdmap e156: 8 total, 7 up, 8 in 2023-09-23T19:38:15.773 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:15 smithi016 bash[77995]: cluster 2023-09-23T19:38:14.652934+0000 osd.6 (osd.6) 73 : cluster 3 failed to encode map e156 with expected crc 2023-09-23T19:38:15.773 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:15 smithi016 bash[77995]: cluster 2023-09-23T19:38:14.655404+0000 osd.6 (osd.6) 74 : cluster 3 failed to encode map e156 with expected crc 2023-09-23T19:38:15.773 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:15 smithi016 bash[77995]: cluster 2023-09-23T19:38:14.655543+0000 osd.7 (osd.7) 69 : cluster 3 failed to encode map e156 with expected crc 2023-09-23T19:38:15.774 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:15 smithi016 bash[77995]: cluster 2023-09-23T19:38:14.655565+0000 osd.4 (osd.4) 83 : cluster 3 failed to encode map e156 with expected crc 2023-09-23T19:38:15.774 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:15 smithi016 bash[77995]: cluster 2023-09-23T19:38:14.655591+0000 osd.5 (osd.5) 86 : cluster 3 failed to encode map e156 with expected crc 2023-09-23T19:38:15.774 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:15 smithi016 bash[77995]: cluster 2023-09-23T19:38:14.658553+0000 osd.4 (osd.4) 84 : cluster 3 failed to encode map e156 with expected crc 2023-09-23T19:38:15.774 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:15 smithi016 bash[75004]: cluster 2023-09-23T19:38:14.108345+0000 mgr.y (mgr.25325) 922 : cluster 0 pgmap v848: 89 pgs: 6 stale+active+clean, 83 active+clean; 6.5 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 127 B/s rd, 7.4 MiB/s wr, 643 op/s 2023-09-23T19:38:15.774 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:15 smithi016 bash[75004]: cluster 2023-09-23T19:38:14.649806+0000 mon.a (mon.0) 1140 : cluster 0 osdmap e156: 8 total, 7 up, 8 in 2023-09-23T19:38:15.774 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:15 smithi016 bash[75004]: cluster 2023-09-23T19:38:14.652934+0000 osd.6 (osd.6) 73 : cluster 3 failed to encode map e156 with expected crc 2023-09-23T19:38:15.774 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:15 smithi016 bash[75004]: cluster 2023-09-23T19:38:14.655404+0000 osd.6 (osd.6) 74 : cluster 3 failed to encode map e156 with expected crc 2023-09-23T19:38:15.775 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:15 smithi016 bash[75004]: cluster 2023-09-23T19:38:14.655543+0000 osd.7 (osd.7) 69 : cluster 3 failed to encode map e156 with expected crc 2023-09-23T19:38:15.775 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:15 smithi016 bash[75004]: cluster 2023-09-23T19:38:14.655565+0000 osd.4 (osd.4) 83 : cluster 3 failed to encode map e156 with expected crc 2023-09-23T19:38:15.775 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:15 smithi016 bash[75004]: cluster 2023-09-23T19:38:14.655591+0000 osd.5 (osd.5) 86 : cluster 3 failed to encode map e156 with expected crc 2023-09-23T19:38:15.775 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:15 smithi016 bash[75004]: cluster 2023-09-23T19:38:14.658553+0000 osd.4 (osd.4) 84 : cluster 3 failed to encode map e156 with expected crc 2023-09-23T19:38:15.775 INFO:journalctl@ceph.osd.2.smithi016.stdout:Sep 23 19:38:15 smithi016 bash[143731]: Error response from daemon: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd-2 2023-09-23T19:38:15.775 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:38:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:38:15] "GET /metrics HTTP/1.1" 200 36303 "" "Prometheus/2.33.4" 2023-09-23T19:38:15.802 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:15.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:15.806 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:38:15.906 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:15.938 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:15.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:15.941 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:38:16.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:15 smithi092 bash[58855]: cluster 2023-09-23T19:38:14.108345+0000 mgr.y (mgr.25325) 922 : cluster 0 pgmap v848: 89 pgs: 6 stale+active+clean, 83 active+clean; 6.5 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 127 B/s rd, 7.4 MiB/s wr, 643 op/s 2023-09-23T19:38:16.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:15 smithi092 bash[58855]: cluster 2023-09-23T19:38:14.649806+0000 mon.a (mon.0) 1140 : cluster 0 osdmap e156: 8 total, 7 up, 8 in 2023-09-23T19:38:16.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:15 smithi092 bash[58855]: cluster 2023-09-23T19:38:14.652934+0000 osd.6 (osd.6) 73 : cluster 3 failed to encode map e156 with expected crc 2023-09-23T19:38:16.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:15 smithi092 bash[58855]: cluster 2023-09-23T19:38:14.655404+0000 osd.6 (osd.6) 74 : cluster 3 failed to encode map e156 with expected crc 2023-09-23T19:38:16.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:15 smithi092 bash[58855]: cluster 2023-09-23T19:38:14.655543+0000 osd.7 (osd.7) 69 : cluster 3 failed to encode map e156 with expected crc 2023-09-23T19:38:16.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:15 smithi092 bash[58855]: cluster 2023-09-23T19:38:14.655565+0000 osd.4 (osd.4) 83 : cluster 3 failed to encode map e156 with expected crc 2023-09-23T19:38:16.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:15 smithi092 bash[58855]: cluster 2023-09-23T19:38:14.655591+0000 osd.5 (osd.5) 86 : cluster 3 failed to encode map e156 with expected crc 2023-09-23T19:38:16.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:15 smithi092 bash[58855]: cluster 2023-09-23T19:38:14.658553+0000 osd.4 (osd.4) 84 : cluster 3 failed to encode map e156 with expected crc 2023-09-23T19:38:16.041 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:16.080 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:16.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:16.084 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:16.185 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:16.229 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:16.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:16.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:16.335 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:16.357 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:16.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:16.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:38:16.462 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:16.494 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:16.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:16.496 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:16.596 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:16.631 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:16.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:16.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:16.736 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:16.746 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:16 smithi092 bash[58855]: cluster 2023-09-23T19:38:16.648404+0000 mon.a (mon.0) 1141 : cluster 3 Health check failed: Degraded data redundancy: 25897/310236 objects degraded (8.348%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T19:38:16.772 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:16.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:16.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:16.859 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:16 smithi016 bash[75004]: cluster 2023-09-23T19:38:16.648404+0000 mon.a (mon.0) 1141 : cluster 3 Health check failed: Degraded data redundancy: 25897/310236 objects degraded (8.348%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T19:38:16.859 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:16 smithi016 bash[77995]: cluster 2023-09-23T19:38:16.648404+0000 mon.a (mon.0) 1141 : cluster 3 Health check failed: Degraded data redundancy: 25897/310236 objects degraded (8.348%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T19:38:16.876 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:16.903 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:16.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:16.909 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:17.010 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:17.052 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:17.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:17.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:17.158 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:17.185 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:17.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:17.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:38:17.290 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:17.319 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:17.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:17.322 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:17.423 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:17.452 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:17.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:17.457 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:17.557 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:17.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:17.590 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:17.591 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:17.691 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:17.725 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:17.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:17.729 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:17.829 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:17.838 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:17 smithi092 bash[58855]: cluster 2023-09-23T19:38:16.109129+0000 mgr.y (mgr.25325) 923 : cluster 0 pgmap v850: 89 pgs: 5 active+undersized+degraded, 3 stale+active+clean, 14 active+undersized, 67 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 127 B/s rd, 2.7 MiB/s wr, 829 op/s; 25897/310236 objects degraded (8.348%) 2023-09-23T19:38:17.863 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:17.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:17.870 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:17.970 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:18.005 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:18.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:18.009 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:18.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:17 smithi016 bash[75004]: cluster 2023-09-23T19:38:16.109129+0000 mgr.y (mgr.25325) 923 : cluster 0 pgmap v850: 89 pgs: 5 active+undersized+degraded, 3 stale+active+clean, 14 active+undersized, 67 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 127 B/s rd, 2.7 MiB/s wr, 829 op/s; 25897/310236 objects degraded (8.348%) 2023-09-23T19:38:18.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:17 smithi016 bash[77995]: cluster 2023-09-23T19:38:16.109129+0000 mgr.y (mgr.25325) 923 : cluster 0 pgmap v850: 89 pgs: 5 active+undersized+degraded, 3 stale+active+clean, 14 active+undersized, 67 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 127 B/s rd, 2.7 MiB/s wr, 829 op/s; 25897/310236 objects degraded (8.348%) 2023-09-23T19:38:18.110 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:18.138 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:18.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:18.143 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:18.244 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:18.271 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:18.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:18.274 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:38:18.375 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:18.401 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:18.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:18.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:38:18.507 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:18.530 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:18.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:18.536 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:18.637 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:18.662 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:18.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:18.668 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:18.740 DEBUG:teuthology.orchestra.run.smithi016:> sudo pkill -f 'journalctl -f -n 0 -u ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@osd.2.service' 2023-09-23T19:38:18.769 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:18.775 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T19:38:18.776 INFO:tasks.cephadm.osd.2:Stopped osd.2 2023-09-23T19:38:18.854 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:18.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:18.858 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:18.959 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:18.985 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:18.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:18.989 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:19.090 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:19.120 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:19.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:19.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:38:19.187 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:38:19.187 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:38:19.188 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:38:19.227 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:19.255 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:19.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:19.263 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:38:19.364 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:19.390 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:19.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:19.394 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:38:19.494 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:19.522 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:19.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:19.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:19.628 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:19.657 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:19.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:19.660 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:19.714 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:19 smithi016 bash[77995]: cluster 2023-09-23T19:38:18.109796+0000 mgr.y (mgr.25325) 924 : cluster 0 pgmap v851: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 520 KiB/s wr, 715 op/s; 38552/308157 objects degraded (12.511%); 0 B/s, 0 objects/s recovering 2023-09-23T19:38:19.761 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:19.770 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:19 smithi092 bash[58855]: cluster 2023-09-23T19:38:18.109796+0000 mgr.y (mgr.25325) 924 : cluster 0 pgmap v851: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 520 KiB/s wr, 715 op/s; 38552/308157 objects degraded (12.511%); 0 B/s, 0 objects/s recovering 2023-09-23T19:38:19.787 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:19.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:19.795 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:19.895 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:19.929 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:19.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:19.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:20.035 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:20.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:19 smithi016 bash[75004]: cluster 2023-09-23T19:38:18.109796+0000 mgr.y (mgr.25325) 924 : cluster 0 pgmap v851: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 520 KiB/s wr, 715 op/s; 38552/308157 objects degraded (12.511%); 0 B/s, 0 objects/s recovering 2023-09-23T19:38:20.068 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:20.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:20.071 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:20.172 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:20.215 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:20.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:20.221 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:20.321 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:20.349 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:20.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:20.352 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:20.453 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:20.493 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:20.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:20.496 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:20.597 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:20.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:20.635 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:20.635 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:20.736 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:20.767 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:20.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:20.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:20.875 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:20.904 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:20.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:20.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:38:21.007 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:21.040 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:21.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:21.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:21.143 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:21.167 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:21.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:21.170 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:38:21.271 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:21.297 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:21.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:21.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:21.402 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:21.447 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:21.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:21.449 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:21.550 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:21.589 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:21.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:21.594 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:21.694 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:21.719 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:21.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:21.724 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:21.824 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:21.866 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:21.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:21.872 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:21.916 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:21 smithi016 bash[75004]: cluster 2023-09-23T19:38:20.110365+0000 mgr.y (mgr.25325) 925 : cluster 0 pgmap v852: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 38271/303243 objects degraded (12.621%); 0 B/s, 0 objects/s recovering 2023-09-23T19:38:21.916 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:21 smithi016 bash[77995]: cluster 2023-09-23T19:38:20.110365+0000 mgr.y (mgr.25325) 925 : cluster 0 pgmap v852: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 38271/303243 objects degraded (12.621%); 0 B/s, 0 objects/s recovering 2023-09-23T19:38:21.972 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:21.982 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:21 smithi092 bash[58855]: cluster 2023-09-23T19:38:20.110365+0000 mgr.y (mgr.25325) 925 : cluster 0 pgmap v852: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 38271/303243 objects degraded (12.621%); 0 B/s, 0 objects/s recovering 2023-09-23T19:38:21.997 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:22.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:22.004 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:38:22.105 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:22.135 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:22.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:22.141 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:22.241 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:22.290 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:22.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:22.296 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:38:22.397 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:22.441 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:22.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:22.446 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:22.547 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:22.575 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:22.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:22.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:38:22.678 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:22.703 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:22.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:22.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:22.807 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:22.815 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:22 smithi016 bash[75004]: cluster 2023-09-23T19:38:22.720317+0000 mon.a (mon.0) 1142 : cluster 3 Health check update: Degraded data redundancy: 37478/297690 objects degraded (12.590%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T19:38:22.815 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:22 smithi016 bash[77995]: cluster 2023-09-23T19:38:22.720317+0000 mon.a (mon.0) 1142 : cluster 3 Health check update: Degraded data redundancy: 37478/297690 objects degraded (12.590%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T19:38:22.834 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:22.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:22.837 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:38:22.938 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:22.969 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:22.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:22.972 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:38:23.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:22 smithi092 bash[58855]: cluster 2023-09-23T19:38:22.720317+0000 mon.a (mon.0) 1142 : cluster 3 Health check update: Degraded data redundancy: 37478/297690 objects degraded (12.590%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T19:38:23.073 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:23.106 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:23.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:23.110 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:23.211 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:23.241 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:23.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:23.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:23.345 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:23.379 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:23.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:23.383 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:38:23.483 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:23.509 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:23.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:23.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:23.613 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:23.653 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:23.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:23.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:23.756 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:23.777 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [2] live_osds: [0, 1, 3, 4, 5, 6, 7] 2023-09-23T19:38:23.777 INFO:tasks.thrashosds.thrasher:Reviving osd 2 2023-09-23T19:38:23.777 INFO:tasks.cephadm.osd.2:Restarting osd.2 (starting--it wasn't running)... 2023-09-23T19:38:23.777 DEBUG:teuthology.orchestra.run.smithi016:osd.2> sudo journalctl -f -n 0 -u ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@osd.2.service 2023-09-23T19:38:23.779 DEBUG:teuthology.orchestra.run.smithi016:> sudo systemctl start ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@osd.2 2023-09-23T19:38:23.782 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:23.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:23.785 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:38:23.847 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T19:38:23.885 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:23.893 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:23 smithi092 bash[58855]: cluster 2023-09-23T19:38:22.111070+0000 mgr.y (mgr.25325) 926 : cluster 0 pgmap v853: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.1 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 37478/297690 objects degraded (12.590%); 0 B/s, 0 objects/s recovering 2023-09-23T19:38:23.912 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:23.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:23.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:24.019 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:24.050 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:24.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:24.054 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:24.089 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:23 smithi016 bash[75004]: cluster 2023-09-23T19:38:22.111070+0000 mgr.y (mgr.25325) 926 : cluster 0 pgmap v853: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.1 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 37478/297690 objects degraded (12.590%); 0 B/s, 0 objects/s recovering 2023-09-23T19:38:24.089 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:23 smithi016 bash[77995]: cluster 2023-09-23T19:38:22.111070+0000 mgr.y (mgr.25325) 926 : cluster 0 pgmap v853: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.1 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 37478/297690 objects degraded (12.590%); 0 B/s, 0 objects/s recovering 2023-09-23T19:38:24.155 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:24.183 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:24.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:24.185 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:24.286 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:24.315 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:24.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:24.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:24.419 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:24.445 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:24.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:24.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:24.549 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:24.574 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:24.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:24.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:24.677 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:24.703 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:24.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:24.712 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:24.812 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:24.833 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:24.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:24.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:24.939 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:24.966 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:24.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:24.968 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:25.046 INFO:journalctl@ceph.osd.2.smithi016.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-09-23T19:38:25.070 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:25.106 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:25.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:25.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:25.213 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:25.249 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:25.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:25.254 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:38:25.354 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:25.389 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:25.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:25.392 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:25.493 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:25.534 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:25.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:25.539 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:25.639 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:25.671 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:25.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:25.676 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:25.777 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:25.807 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:25.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:25.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:25.910 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:25.948 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:25.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:25.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:26.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:25 smithi092 bash[58855]: cluster 2023-09-23T19:38:24.111978+0000 mgr.y (mgr.25325) 927 : cluster 0 pgmap v854: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 36973/295398 objects degraded (12.516%); 0 B/s, 0 objects/s recovering 2023-09-23T19:38:26.053 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:26.082 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:26.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:26.088 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:38:26.101 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:25 smithi016 bash[75004]: cluster 2023-09-23T19:38:24.111978+0000 mgr.y (mgr.25325) 927 : cluster 0 pgmap v854: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 36973/295398 objects degraded (12.516%); 0 B/s, 0 objects/s recovering 2023-09-23T19:38:26.101 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:25 smithi016 bash[77995]: cluster 2023-09-23T19:38:24.111978+0000 mgr.y (mgr.25325) 927 : cluster 0 pgmap v854: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 36973/295398 objects degraded (12.516%); 0 B/s, 0 objects/s recovering 2023-09-23T19:38:26.102 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:38:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:38:25] "GET /metrics HTTP/1.1" 200 36303 "" "Prometheus/2.33.4" 2023-09-23T19:38:26.188 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:26.219 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:26.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:26.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:26.325 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:26.353 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:26.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:26.356 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:26.456 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:26.492 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:26.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:26.500 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:26.601 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:26.630 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:26.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:26.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:38:26.733 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:26.756 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:26.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:26.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:26.860 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:26.897 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:26.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:26.901 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:27.002 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:27.038 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:27.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:27.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:27.143 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:27.177 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:27.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:27.181 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:27.282 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:27.317 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:27.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:27.327 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:27.428 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:27.464 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:27.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:27.467 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:27.568 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:27.614 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:27.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:27.627 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:27.697 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-09-23T19:38:27.698 INFO:teuthology.orchestra.run.smithi016.stdout: "size": 20, 2023-09-23T19:38:27.698 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 600, 2023-09-23T19:38:27.698 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [ 2023-09-23T19:38:27.698 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.698 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45671.0:159321 9.e e131/119 9:779c740b:::benchmark_data_smithi016_113233_object50939:head v 131'10024, mlcod=131'10022)", 2023-09-23T19:38:27.698 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:34:12.753284+0000", 2023-09-23T19:38:27.698 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 254.93875491, 2023-09-23T19:38:27.698 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.13932908899999999, 2023-09-23T19:38:27.698 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:38:27.698 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:38:27.698 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:38:27.699 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.699 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:38:27.699 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:34:12.753284+0000", 2023-09-23T19:38:27.699 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:38:27.699 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.699 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.699 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:38:27.699 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:34:12.753284+0000", 2023-09-23T19:38:27.699 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:38:27.699 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.699 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.699 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:38:27.699 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:34:12.753283+0000", 2023-09-23T19:38:27.699 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:38:27.700 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.700 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.700 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:38:27.700 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:34:12.753292+0000", 2023-09-23T19:38:27.700 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.115e-06 2023-09-23T19:38:27.700 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.700 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.700 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:38:27.700 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:34:12.753300+0000", 2023-09-23T19:38:27.700 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.7090000000000002e-06 2023-09-23T19:38:27.700 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.700 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.701 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:38:27.701 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:34:12.753305+0000", 2023-09-23T19:38:27.701 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.305e-06 2023-09-23T19:38:27.701 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.701 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.701 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:38:27.701 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:34:12.753344+0000", 2023-09-23T19:38:27.701 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.9239999999999997e-05 2023-09-23T19:38:27.701 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.701 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.701 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:38:27.701 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:34:12.753365+0000", 2023-09-23T19:38:27.701 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.0619e-05 2023-09-23T19:38:27.702 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.702 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.702 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:38:27.702 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:34:12.892537+0000", 2023-09-23T19:38:27.702 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.13917205299999999 2023-09-23T19:38:27.702 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.702 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.702 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:38:27.702 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:34:12.892613+0000", 2023-09-23T19:38:27.702 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.5865999999999995e-05 2023-09-23T19:38:27.702 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:38:27.702 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:38:27.703 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:38:27.703 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.703 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.703 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45671.0:159338 9.6 e131/119 9:6625ebd7:::benchmark_data_smithi016_113233_object50956:head v 131'9757, mlcod=131'9755)", 2023-09-23T19:38:27.703 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:34:12.757621+0000", 2023-09-23T19:38:27.703 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 254.934417905, 2023-09-23T19:38:27.703 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.141925938, 2023-09-23T19:38:27.703 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:38:27.703 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:38:27.703 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:38:27.703 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.703 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:38:27.703 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:34:12.757621+0000", 2023-09-23T19:38:27.704 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:38:27.704 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.704 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.704 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:38:27.704 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:34:12.757621+0000", 2023-09-23T19:38:27.704 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:38:27.704 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.704 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.704 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:38:27.704 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:34:12.757620+0000", 2023-09-23T19:38:27.704 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:38:27.704 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.704 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.705 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:38:27.705 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:34:12.757626+0000", 2023-09-23T19:38:27.705 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.8660000000000002e-06 2023-09-23T19:38:27.705 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.705 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.705 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:38:27.705 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:34:12.757636+0000", 2023-09-23T19:38:27.705 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.057e-05 2023-09-23T19:38:27.705 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.705 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.705 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:38:27.705 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:34:12.757643+0000", 2023-09-23T19:38:27.706 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.4579999999999998e-06 2023-09-23T19:38:27.706 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.706 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.706 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:38:27.706 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:34:12.758369+0000", 2023-09-23T19:38:27.706 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00072655400000000002 2023-09-23T19:38:27.706 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.706 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.706 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:38:27.706 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:34:12.758388+0000", 2023-09-23T19:38:27.706 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.874e-05 2023-09-23T19:38:27.706 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.707 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.707 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:38:27.707 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:34:12.899489+0000", 2023-09-23T19:38:27.707 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.141101326 2023-09-23T19:38:27.707 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.707 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.707 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:38:27.707 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:34:12.899547+0000", 2023-09-23T19:38:27.707 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.7195999999999999e-05 2023-09-23T19:38:27.707 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:38:27.707 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:38:27.707 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:38:27.707 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.708 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.708 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45671.0:159333 9.f e131/119 9:f0a3169a:::benchmark_data_smithi016_113233_object50951:head v 131'9947, mlcod=131'9943)", 2023-09-23T19:38:27.708 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:34:12.760362+0000", 2023-09-23T19:38:27.708 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 254.93167664800001, 2023-09-23T19:38:27.708 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.13645136999999999, 2023-09-23T19:38:27.708 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:38:27.708 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:38:27.708 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:38:27.708 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.708 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:38:27.708 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:34:12.760362+0000", 2023-09-23T19:38:27.708 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:38:27.708 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.709 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.709 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:38:27.709 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:34:12.760362+0000", 2023-09-23T19:38:27.709 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:38:27.709 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.709 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.709 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:38:27.709 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:34:12.760361+0000", 2023-09-23T19:38:27.709 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:38:27.709 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.709 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.709 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:38:27.709 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:34:12.760369+0000", 2023-09-23T19:38:27.710 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.072e-06 2023-09-23T19:38:27.710 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.710 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.710 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:38:27.710 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:34:12.883326+0000", 2023-09-23T19:38:27.710 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.122956909 2023-09-23T19:38:27.710 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.710 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.710 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:38:27.710 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:34:12.883340+0000", 2023-09-23T19:38:27.710 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3784999999999999e-05 2023-09-23T19:38:27.710 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.711 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.711 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:38:27.711 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:34:12.883406+0000", 2023-09-23T19:38:27.711 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.6651999999999998e-05 2023-09-23T19:38:27.711 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.711 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.711 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:38:27.711 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:34:12.883429+0000", 2023-09-23T19:38:27.711 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.2892e-05 2023-09-23T19:38:27.711 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.711 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.711 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:38:27.712 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:34:12.896721+0000", 2023-09-23T19:38:27.712 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.013291858 2023-09-23T19:38:27.712 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.712 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.712 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:38:27.712 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:34:12.896813+0000", 2023-09-23T19:38:27.712 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.1946000000000004e-05 2023-09-23T19:38:27.712 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:38:27.712 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:38:27.712 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:38:27.712 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.712 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.712 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45671.0:208985 9.d e137/119 9:b5103cf5:::benchmark_data_smithi016_113233_object100603:head v 137'13051, mlcod=137'13049)", 2023-09-23T19:38:27.712 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:35:26.608279+0000", 2023-09-23T19:38:27.713 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 181.08375942399999, 2023-09-23T19:38:27.713 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.21262897, 2023-09-23T19:38:27.713 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:38:27.713 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:38:27.713 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:38:27.713 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.713 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:38:27.713 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.608279+0000", 2023-09-23T19:38:27.713 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:38:27.713 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.713 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.713 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:38:27.713 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.608279+0000", 2023-09-23T19:38:27.714 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:38:27.714 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.714 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.714 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:38:27.714 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.608278+0000", 2023-09-23T19:38:27.714 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:38:27.714 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.714 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.714 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:38:27.714 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.608287+0000", 2023-09-23T19:38:27.714 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.4009999999999997e-06 2023-09-23T19:38:27.714 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.714 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.715 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:38:27.715 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.608298+0000", 2023-09-23T19:38:27.715 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1348e-05 2023-09-23T19:38:27.715 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.715 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.715 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:38:27.715 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.608305+0000", 2023-09-23T19:38:27.715 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.6510000000000004e-06 2023-09-23T19:38:27.715 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.715 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.715 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:38:27.715 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.608350+0000", 2023-09-23T19:38:27.715 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.5500999999999997e-05 2023-09-23T19:38:27.716 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.716 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.716 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:38:27.716 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.608378+0000", 2023-09-23T19:38:27.716 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.8374e-05 2023-09-23T19:38:27.716 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.716 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.716 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:38:27.716 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.820849+0000", 2023-09-23T19:38:27.716 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.21247050300000001 2023-09-23T19:38:27.716 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.716 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.716 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:38:27.717 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.820908+0000", 2023-09-23T19:38:27.717 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.9021999999999998e-05 2023-09-23T19:38:27.717 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:38:27.717 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:38:27.717 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:38:27.717 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.717 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.717 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45671.0:208989 9.d e137/119 9:bbab1653:::benchmark_data_smithi016_113233_object100607:head v 137'13052, mlcod=137'13049)", 2023-09-23T19:38:27.717 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:35:26.612132+0000", 2023-09-23T19:38:27.717 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 181.07990638999999, 2023-09-23T19:38:27.717 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.225652405, 2023-09-23T19:38:27.717 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:38:27.717 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:38:27.718 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:38:27.718 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.718 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:38:27.718 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.612132+0000", 2023-09-23T19:38:27.718 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:38:27.718 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.718 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.718 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:38:27.718 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.612132+0000", 2023-09-23T19:38:27.718 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:38:27.718 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.718 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.718 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:38:27.719 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.612132+0000", 2023-09-23T19:38:27.719 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T19:38:27.719 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.719 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.719 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:38:27.719 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.612136+0000", 2023-09-23T19:38:27.719 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.391e-06 2023-09-23T19:38:27.719 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.719 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.719 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:38:27.719 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.612143+0000", 2023-09-23T19:38:27.719 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.8770000000000004e-06 2023-09-23T19:38:27.719 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.720 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.720 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:38:27.720 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.612148+0000", 2023-09-23T19:38:27.720 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.4410000000000003e-06 2023-09-23T19:38:27.720 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.720 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.720 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:38:27.720 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.612558+0000", 2023-09-23T19:38:27.720 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000410448 2023-09-23T19:38:27.720 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.720 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.720 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:38:27.720 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.612577+0000", 2023-09-23T19:38:27.720 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.8706999999999999e-05 2023-09-23T19:38:27.721 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.721 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.721 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:38:27.721 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.837673+0000", 2023-09-23T19:38:27.721 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.22509628600000001 2023-09-23T19:38:27.721 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.721 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.721 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:38:27.721 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.837784+0000", 2023-09-23T19:38:27.721 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000111533 2023-09-23T19:38:27.721 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:38:27.721 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:38:27.721 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:38:27.722 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.722 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.722 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45671.0:208997 9.0 9:05dc60b0:::benchmark_data_smithi016_113233_object100615:head [delete] snapc 0=[] ondisk+write+known_if_redirected e137)", 2023-09-23T19:38:27.722 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:35:26.615043+0000", 2023-09-23T19:38:27.722 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 181.07699569600001, 2023-09-23T19:38:27.722 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.21171452399999999, 2023-09-23T19:38:27.722 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:38:27.722 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:38:27.722 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:38:27.722 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45671", 2023-09-23T19:38:27.722 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:38:27.723 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 208997 2023-09-23T19:38:27.723 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.723 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:38:27.723 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.723 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:38:27.723 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.615043+0000", 2023-09-23T19:38:27.723 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:38:27.723 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.723 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.723 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:38:27.723 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.615043+0000", 2023-09-23T19:38:27.723 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:38:27.723 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.723 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.724 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:38:27.724 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.615041+0000", 2023-09-23T19:38:27.724 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-09-23T19:38:27.724 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.724 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.724 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:38:27.724 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.615052+0000", 2023-09-23T19:38:27.724 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0886e-05 2023-09-23T19:38:27.724 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.724 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.724 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:38:27.724 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.615071+0000", 2023-09-23T19:38:27.724 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.9140999999999999e-05 2023-09-23T19:38:27.725 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.725 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.725 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:38:27.725 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.615083+0000", 2023-09-23T19:38:27.725 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2182e-05 2023-09-23T19:38:27.725 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.725 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.725 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:38:27.725 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.615166+0000", 2023-09-23T19:38:27.725 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.2787999999999994e-05 2023-09-23T19:38:27.725 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.725 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.726 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:38:27.726 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.615315+0000", 2023-09-23T19:38:27.726 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00014920299999999999 2023-09-23T19:38:27.726 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.726 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.726 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 2,6", 2023-09-23T19:38:27.726 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.615426+0000", 2023-09-23T19:38:27.726 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000111018 2023-09-23T19:38:27.726 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.726 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.726 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:38:27.726 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.619749+0000", 2023-09-23T19:38:27.726 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0043230849999999999 2023-09-23T19:38:27.727 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.727 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.727 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:38:27.727 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.820806+0000", 2023-09-23T19:38:27.727 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.20105654000000001 2023-09-23T19:38:27.727 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.727 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.727 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:38:27.727 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.826602+0000", 2023-09-23T19:38:27.727 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0057966609999999998 2023-09-23T19:38:27.727 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.727 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.727 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:38:27.728 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.826682+0000", 2023-09-23T19:38:27.728 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.9899000000000002e-05 2023-09-23T19:38:27.728 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.728 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.728 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:38:27.728 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.826757+0000", 2023-09-23T19:38:27.728 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.5217999999999997e-05 2023-09-23T19:38:27.728 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:38:27.728 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:38:27.728 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:38:27.728 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.728 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.728 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45671.0:208995 9.e e137/119 9:7a040566:::benchmark_data_smithi016_113233_object100613:head v 137'13188, mlcod=137'13185)", 2023-09-23T19:38:27.729 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:35:26.615077+0000", 2023-09-23T19:38:27.729 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 181.076961302, 2023-09-23T19:38:27.729 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.205860126, 2023-09-23T19:38:27.729 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:38:27.729 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:38:27.729 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:38:27.729 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.729 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:38:27.729 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.615077+0000", 2023-09-23T19:38:27.729 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:38:27.729 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.729 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.729 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:38:27.729 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.615077+0000", 2023-09-23T19:38:27.730 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:38:27.730 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.730 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.730 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:38:27.730 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.615076+0000", 2023-09-23T19:38:27.730 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:38:27.730 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.730 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.730 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:38:27.730 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.615098+0000", 2023-09-23T19:38:27.730 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.1946e-05 2023-09-23T19:38:27.730 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.731 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.731 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:38:27.731 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.615111+0000", 2023-09-23T19:38:27.731 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2544999999999999e-05 2023-09-23T19:38:27.731 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.731 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.731 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:38:27.731 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.615119+0000", 2023-09-23T19:38:27.731 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.5059999999999995e-06 2023-09-23T19:38:27.731 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.731 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.731 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:38:27.731 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.619925+0000", 2023-09-23T19:38:27.732 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0048052499999999996 2023-09-23T19:38:27.732 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.732 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.732 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:38:27.732 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.619968+0000", 2023-09-23T19:38:27.732 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.3316999999999999e-05 2023-09-23T19:38:27.732 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.732 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.732 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:38:27.732 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.820865+0000", 2023-09-23T19:38:27.732 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.20089712900000001 2023-09-23T19:38:27.732 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.733 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.733 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:38:27.733 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.820937+0000", 2023-09-23T19:38:27.733 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.2251999999999999e-05 2023-09-23T19:38:27.733 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:38:27.733 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:38:27.733 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:38:27.733 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.733 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.733 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45671.0:208998 9.f e137/119 9:fe02e78b:::benchmark_data_smithi016_113233_object100616:head v 137'13030, mlcod=137'13028)", 2023-09-23T19:38:27.733 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:35:26.620103+0000", 2023-09-23T19:38:27.733 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 181.07193553100001, 2023-09-23T19:38:27.733 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.20097216000000001, 2023-09-23T19:38:27.734 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:38:27.734 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:38:27.734 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:38:27.734 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.734 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:38:27.734 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.620103+0000", 2023-09-23T19:38:27.734 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:38:27.734 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.734 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.734 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:38:27.734 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.620103+0000", 2023-09-23T19:38:27.734 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:38:27.735 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.735 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.735 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:38:27.735 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.620101+0000", 2023-09-23T19:38:27.735 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-09-23T19:38:27.735 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.735 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.735 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:38:27.735 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.620138+0000", 2023-09-23T19:38:27.735 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.6656999999999998e-05 2023-09-23T19:38:27.735 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.735 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.736 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:38:27.736 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.620158+0000", 2023-09-23T19:38:27.736 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.9624e-05 2023-09-23T19:38:27.736 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.736 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.736 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:38:27.736 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.620172+0000", 2023-09-23T19:38:27.736 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.394e-05 2023-09-23T19:38:27.736 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.736 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.736 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:38:27.736 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.620272+0000", 2023-09-23T19:38:27.736 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000100338 2023-09-23T19:38:27.737 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.737 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.737 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:38:27.737 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.620286+0000", 2023-09-23T19:38:27.737 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3937999999999999e-05 2023-09-23T19:38:27.737 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.737 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.737 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:38:27.737 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.820990+0000", 2023-09-23T19:38:27.737 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.20070369399999999 2023-09-23T19:38:27.737 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.737 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.737 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:38:27.738 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.821075+0000", 2023-09-23T19:38:27.738 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.5578e-05 2023-09-23T19:38:27.738 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:38:27.738 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:38:27.738 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:38:27.738 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.738 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.738 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop_reply(client.45671.0:208997 9.0 e137/119 ondisk, result = 0)", 2023-09-23T19:38:27.738 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:35:26.620282+0000", 2023-09-23T19:38:27.738 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 181.071756072, 2023-09-23T19:38:27.738 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.20053090500000001, 2023-09-23T19:38:27.738 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:38:27.739 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "started", 2023-09-23T19:38:27.739 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:38:27.739 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.739 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:38:27.739 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.620282+0000", 2023-09-23T19:38:27.739 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:38:27.739 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.739 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.739 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:38:27.739 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.620282+0000", 2023-09-23T19:38:27.739 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:38:27.739 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.739 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.740 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:38:27.740 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.620281+0000", 2023-09-23T19:38:27.740 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:38:27.740 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.740 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.740 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:38:27.740 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.620291+0000", 2023-09-23T19:38:27.740 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.0259999999999996e-06 2023-09-23T19:38:27.740 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.740 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.740 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:38:27.740 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.620307+0000", 2023-09-23T19:38:27.740 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.6857000000000001e-05 2023-09-23T19:38:27.740 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.741 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.741 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:38:27.741 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.620339+0000", 2023-09-23T19:38:27.741 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.1537e-05 2023-09-23T19:38:27.741 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.741 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.741 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:38:27.741 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.620385+0000", 2023-09-23T19:38:27.741 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.6177999999999997e-05 2023-09-23T19:38:27.741 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.741 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.741 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:38:27.741 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.820782+0000", 2023-09-23T19:38:27.742 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.20039693 2023-09-23T19:38:27.742 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.742 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.742 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:38:27.742 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.820813+0000", 2023-09-23T19:38:27.742 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.1294000000000002e-05 2023-09-23T19:38:27.742 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:38:27.742 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:38:27.742 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:38:27.742 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.742 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.743 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45671.0:208990 9.d e137/119 9:b24ecd90:::benchmark_data_smithi016_113233_object100608:head v 137'13053, mlcod=137'13049)", 2023-09-23T19:38:27.743 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:35:26.620706+0000", 2023-09-23T19:38:27.743 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 181.071332904, 2023-09-23T19:38:27.743 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.217123129, 2023-09-23T19:38:27.743 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:38:27.743 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:38:27.743 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:38:27.743 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.743 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:38:27.743 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.620706+0000", 2023-09-23T19:38:27.743 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:38:27.743 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.743 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.744 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:38:27.744 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.620706+0000", 2023-09-23T19:38:27.744 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:38:27.744 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.744 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.744 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:38:27.744 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.620704+0000", 2023-09-23T19:38:27.744 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:38:27.744 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.744 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.744 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:38:27.744 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.620714+0000", 2023-09-23T19:38:27.744 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.4199999999999996e-06 2023-09-23T19:38:27.745 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.745 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.745 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:38:27.745 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.820827+0000", 2023-09-23T19:38:27.745 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.20011351099999999 2023-09-23T19:38:27.745 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.745 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.745 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:38:27.745 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.820864+0000", 2023-09-23T19:38:27.745 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.6662999999999998e-05 2023-09-23T19:38:27.745 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.745 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.746 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:38:27.746 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.820931+0000", 2023-09-23T19:38:27.746 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.7158999999999999e-05 2023-09-23T19:38:27.746 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.746 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.746 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:38:27.746 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.820961+0000", 2023-09-23T19:38:27.746 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.976e-05 2023-09-23T19:38:27.746 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.746 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.746 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:38:27.746 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.837794+0000", 2023-09-23T19:38:27.747 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.016833013000000001 2023-09-23T19:38:27.747 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.747 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.747 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:38:27.747 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.837829+0000", 2023-09-23T19:38:27.747 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.4709999999999998e-05 2023-09-23T19:38:27.749 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:38:27.749 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:38:27.749 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:38:27.749 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.749 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.749 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45671.0:209000 9.4 9:2109823a:::benchmark_data_smithi016_113233_object100618:head [delete] snapc 0=[] ondisk+write+known_if_redirected e137)", 2023-09-23T19:38:27.749 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:35:26.621957+0000", 2023-09-23T19:38:27.749 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 181.07008187100001, 2023-09-23T19:38:27.749 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.21210473799999999, 2023-09-23T19:38:27.750 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:38:27.750 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:38:27.750 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:38:27.750 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45671", 2023-09-23T19:38:27.750 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:38:27.750 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 209000 2023-09-23T19:38:27.750 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.750 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:38:27.750 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.750 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:38:27.750 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.621957+0000", 2023-09-23T19:38:27.750 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:38:27.751 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.751 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.751 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:38:27.751 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.621957+0000", 2023-09-23T19:38:27.751 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:38:27.751 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.751 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.751 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:38:27.751 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.621955+0000", 2023-09-23T19:38:27.751 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-09-23T19:38:27.751 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.751 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.752 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:38:27.752 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.621962+0000", 2023-09-23T19:38:27.752 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.5680000000000002e-06 2023-09-23T19:38:27.752 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.752 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.752 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:38:27.752 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.820839+0000", 2023-09-23T19:38:27.752 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.198876935 2023-09-23T19:38:27.752 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.752 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.752 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:38:27.752 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.820877+0000", 2023-09-23T19:38:27.752 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.8115999999999999e-05 2023-09-23T19:38:27.753 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.753 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.753 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:38:27.753 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.821003+0000", 2023-09-23T19:38:27.753 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000125551 2023-09-23T19:38:27.753 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.753 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.753 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:38:27.753 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.821254+0000", 2023-09-23T19:38:27.753 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00025106199999999998 2023-09-23T19:38:27.753 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.753 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.754 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 3,4", 2023-09-23T19:38:27.754 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.821458+0000", 2023-09-23T19:38:27.754 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00020381800000000001 2023-09-23T19:38:27.754 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.754 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.754 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:38:27.754 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.827288+0000", 2023-09-23T19:38:27.754 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0058305459999999998 2023-09-23T19:38:27.754 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.754 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.754 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:38:27.754 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.832114+0000", 2023-09-23T19:38:27.755 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0048260860000000003 2023-09-23T19:38:27.755 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.755 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.755 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:38:27.755 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.833956+0000", 2023-09-23T19:38:27.755 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0018413189999999999 2023-09-23T19:38:27.755 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.755 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.755 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:38:27.755 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.834010+0000", 2023-09-23T19:38:27.755 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.465e-05 2023-09-23T19:38:27.756 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.756 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.756 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:38:27.756 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.834061+0000", 2023-09-23T19:38:27.756 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.1112000000000001e-05 2023-09-23T19:38:27.756 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:38:27.756 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:38:27.756 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:38:27.756 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.756 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.756 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45947.0:18778 10.2 10:43b1c768:::benchmark_data_smithi016_136365_object18777:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e149)", 2023-09-23T19:38:27.756 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:36:43.988360+0000", 2023-09-23T19:38:27.756 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 103.7036787, 2023-09-23T19:38:27.757 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.15361467700000001, 2023-09-23T19:38:27.757 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:38:27.757 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:38:27.757 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:38:27.757 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45947", 2023-09-23T19:38:27.757 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/4039191752", 2023-09-23T19:38:27.757 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 18778 2023-09-23T19:38:27.757 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.757 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:38:27.757 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.757 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:38:27.757 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.988360+0000", 2023-09-23T19:38:27.757 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:38:27.758 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.758 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.758 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:38:27.758 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.988360+0000", 2023-09-23T19:38:27.758 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:38:27.758 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.758 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.758 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:38:27.758 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.988359+0000", 2023-09-23T19:38:27.758 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-09-23T19:38:27.758 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.758 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.758 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:38:27.759 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.988391+0000", 2023-09-23T19:38:27.759 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.2845000000000001e-05 2023-09-23T19:38:27.759 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.759 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.759 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:38:27.759 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.988404+0000", 2023-09-23T19:38:27.759 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2207e-05 2023-09-23T19:38:27.759 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.759 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.759 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:38:27.759 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.988411+0000", 2023-09-23T19:38:27.759 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.7880000000000007e-06 2023-09-23T19:38:27.759 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.760 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.760 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:38:27.760 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.988458+0000", 2023-09-23T19:38:27.760 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.6211000000000002e-05 2023-09-23T19:38:27.760 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.760 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.760 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:38:27.760 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.988587+0000", 2023-09-23T19:38:27.760 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00012913599999999999 2023-09-23T19:38:27.760 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.760 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.760 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,4", 2023-09-23T19:38:27.760 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.988726+0000", 2023-09-23T19:38:27.761 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00013903 2023-09-23T19:38:27.761 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.761 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.761 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:38:27.761 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.989794+0000", 2023-09-23T19:38:27.761 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0010681340000000001 2023-09-23T19:38:27.761 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.761 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.761 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:38:27.761 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.991743+0000", 2023-09-23T19:38:27.761 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0019494479999999999 2023-09-23T19:38:27.761 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.761 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.762 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:38:27.762 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:44.141871+0000", 2023-09-23T19:38:27.762 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.15012731900000001 2023-09-23T19:38:27.762 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.762 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.762 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:38:27.762 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:44.141931+0000", 2023-09-23T19:38:27.762 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.9939000000000003e-05 2023-09-23T19:38:27.762 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.762 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.762 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:38:27.762 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:44.141974+0000", 2023-09-23T19:38:27.763 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.3828000000000001e-05 2023-09-23T19:38:27.763 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:38:27.763 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:38:27.763 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:38:27.763 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.763 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.763 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop_reply(client.45947.0:18778 10.2 e149/144 ondisk, result = 0)", 2023-09-23T19:38:27.763 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:36:43.992615+0000", 2023-09-23T19:38:27.763 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 103.69942354, 2023-09-23T19:38:27.763 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.14936886399999999, 2023-09-23T19:38:27.763 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:38:27.763 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "started", 2023-09-23T19:38:27.764 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:38:27.764 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.764 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:38:27.764 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.992615+0000", 2023-09-23T19:38:27.764 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:38:27.764 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.764 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.764 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:38:27.764 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.992615+0000", 2023-09-23T19:38:27.764 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:38:27.764 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.764 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.764 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:38:27.764 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.992615+0000", 2023-09-23T19:38:27.765 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T19:38:27.765 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.765 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.765 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:38:27.765 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.992618+0000", 2023-09-23T19:38:27.765 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.1329999999999999e-06 2023-09-23T19:38:27.765 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.765 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.765 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:38:27.765 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.992624+0000", 2023-09-23T19:38:27.765 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.3239999999999999e-06 2023-09-23T19:38:27.765 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.765 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.766 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:38:27.766 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.992629+0000", 2023-09-23T19:38:27.766 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.0209999999999999e-06 2023-09-23T19:38:27.766 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.766 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.766 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:38:27.766 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:44.141852+0000", 2023-09-23T19:38:27.766 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.14922295899999999 2023-09-23T19:38:27.766 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.766 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.766 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:38:27.766 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:44.141863+0000", 2023-09-23T19:38:27.767 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1219e-05 2023-09-23T19:38:27.767 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.767 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.767 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:38:27.767 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:44.141984+0000", 2023-09-23T19:38:27.767 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000120537 2023-09-23T19:38:27.767 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:38:27.767 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:38:27.767 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:38:27.767 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.767 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.767 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45947.0:18787 10.2 10:4ba09bef:::benchmark_data_smithi016_136365_object18786:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e149)", 2023-09-23T19:38:27.767 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:36:43.992846+0000", 2023-09-23T19:38:27.768 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 103.699192407, 2023-09-23T19:38:27.768 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.15537915999999999, 2023-09-23T19:38:27.768 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:38:27.768 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:38:27.768 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:38:27.768 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45947", 2023-09-23T19:38:27.768 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/4039191752", 2023-09-23T19:38:27.768 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 18787 2023-09-23T19:38:27.768 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.768 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:38:27.768 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.768 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:38:27.768 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.992846+0000", 2023-09-23T19:38:27.769 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:38:27.769 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.769 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.769 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:38:27.769 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.992846+0000", 2023-09-23T19:38:27.769 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:38:27.769 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.769 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.769 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:38:27.769 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.992845+0000", 2023-09-23T19:38:27.769 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-09-23T19:38:27.769 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.770 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.770 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:38:27.770 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:43.992871+0000", 2023-09-23T19:38:27.770 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.6798999999999999e-05 2023-09-23T19:38:27.770 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.770 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.770 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:38:27.770 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:44.142041+0000", 2023-09-23T19:38:27.770 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.14916921999999999 2023-09-23T19:38:27.770 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.770 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.770 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:38:27.770 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:44.142055+0000", 2023-09-23T19:38:27.771 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.4511e-05 2023-09-23T19:38:27.771 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.771 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.771 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:38:27.771 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:44.142118+0000", 2023-09-23T19:38:27.771 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.2936999999999997e-05 2023-09-23T19:38:27.771 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.771 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.771 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:38:27.771 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:44.142341+0000", 2023-09-23T19:38:27.771 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000223265 2023-09-23T19:38:27.771 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.772 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.772 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,4", 2023-09-23T19:38:27.772 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:44.142506+0000", 2023-09-23T19:38:27.772 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00016427600000000001 2023-09-23T19:38:27.772 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.772 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.772 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:38:27.772 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:44.144104+0000", 2023-09-23T19:38:27.772 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001598686 2023-09-23T19:38:27.772 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.772 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.772 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:38:27.772 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:44.146855+0000", 2023-09-23T19:38:27.773 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0027505889999999999 2023-09-23T19:38:27.773 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.773 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.773 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:38:27.773 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:44.148130+0000", 2023-09-23T19:38:27.773 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001275167 2023-09-23T19:38:27.773 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.773 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.773 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:38:27.773 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:44.148195+0000", 2023-09-23T19:38:27.773 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.5322999999999998e-05 2023-09-23T19:38:27.773 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.774 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.774 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:38:27.774 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:44.148225+0000", 2023-09-23T19:38:27.774 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.9878000000000001e-05 2023-09-23T19:38:27.774 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:38:27.774 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:38:27.774 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:38:27.774 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.774 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.774 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45947.0:79834 10.2 10:432191b2:::benchmark_data_smithi016_136365_object79833:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e152)", 2023-09-23T19:38:27.774 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:37:30.744209+0000", 2023-09-23T19:38:27.774 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 56.947829628999997, 2023-09-23T19:38:27.774 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.190664373, 2023-09-23T19:38:27.775 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:38:27.775 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:38:27.775 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:38:27.775 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45947", 2023-09-23T19:38:27.775 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/4039191752", 2023-09-23T19:38:27.775 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 79834 2023-09-23T19:38:27.775 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.775 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:38:27.775 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.775 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:38:27.775 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.744209+0000", 2023-09-23T19:38:27.775 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:38:27.775 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.776 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.776 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:38:27.776 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.744209+0000", 2023-09-23T19:38:27.776 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:38:27.776 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.776 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.776 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:38:27.776 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.744206+0000", 2023-09-23T19:38:27.776 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999976 2023-09-23T19:38:27.776 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.776 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.776 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:38:27.777 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.744261+0000", 2023-09-23T19:38:27.777 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.4487999999999997e-05 2023-09-23T19:38:27.777 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.777 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.777 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:38:27.777 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.744290+0000", 2023-09-23T19:38:27.777 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.9420999999999998e-05 2023-09-23T19:38:27.777 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.777 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.777 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:38:27.777 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.744304+0000", 2023-09-23T19:38:27.777 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3823e-05 2023-09-23T19:38:27.777 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.778 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.778 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:38:27.778 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.744371+0000", 2023-09-23T19:38:27.778 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.6662000000000007e-05 2023-09-23T19:38:27.778 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.778 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.778 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:38:27.778 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.744500+0000", 2023-09-23T19:38:27.778 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00012901500000000001 2023-09-23T19:38:27.778 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.778 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.778 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,4", 2023-09-23T19:38:27.779 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.744612+0000", 2023-09-23T19:38:27.779 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00011184999999999999 2023-09-23T19:38:27.779 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.779 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.779 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:38:27.779 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.747045+0000", 2023-09-23T19:38:27.779 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0024333179999999999 2023-09-23T19:38:27.779 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.779 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.779 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:38:27.779 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.749209+0000", 2023-09-23T19:38:27.779 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0021636889999999999 2023-09-23T19:38:27.780 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.780 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.780 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:38:27.780 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.934762+0000", 2023-09-23T19:38:27.780 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.18555321599999999 2023-09-23T19:38:27.780 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.780 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.780 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:38:27.780 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.934821+0000", 2023-09-23T19:38:27.780 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.9378e-05 2023-09-23T19:38:27.780 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.780 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.780 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:38:27.781 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.934873+0000", 2023-09-23T19:38:27.781 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.2002000000000001e-05 2023-09-23T19:38:27.781 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:38:27.781 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:38:27.781 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:38:27.781 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.781 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.781 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45947.0:79835 10.2 10:4a2b11b9:::benchmark_data_smithi016_136365_object79834:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e152)", 2023-09-23T19:38:27.781 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:37:30.747176+0000", 2023-09-23T19:38:27.781 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 56.944862897, 2023-09-23T19:38:27.781 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.18782270100000001, 2023-09-23T19:38:27.781 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:38:27.782 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:38:27.782 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:38:27.782 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45947", 2023-09-23T19:38:27.782 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/4039191752", 2023-09-23T19:38:27.782 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 79835 2023-09-23T19:38:27.782 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.782 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:38:27.782 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.782 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:38:27.782 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.747176+0000", 2023-09-23T19:38:27.782 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:38:27.782 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.782 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.783 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:38:27.783 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.747176+0000", 2023-09-23T19:38:27.783 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:38:27.783 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.783 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.783 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:38:27.783 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.747174+0000", 2023-09-23T19:38:27.783 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-09-23T19:38:27.783 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.783 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.783 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:38:27.783 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.747207+0000", 2023-09-23T19:38:27.783 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.3448999999999998e-05 2023-09-23T19:38:27.784 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.784 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.784 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:38:27.784 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.747225+0000", 2023-09-23T19:38:27.784 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.7697999999999999e-05 2023-09-23T19:38:27.784 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.784 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.784 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:38:27.784 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.747235+0000", 2023-09-23T19:38:27.784 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.022e-05 2023-09-23T19:38:27.784 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.784 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.785 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:38:27.785 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.747286+0000", 2023-09-23T19:38:27.785 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.0828000000000002e-05 2023-09-23T19:38:27.785 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.785 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.785 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:38:27.785 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.747391+0000", 2023-09-23T19:38:27.785 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00010524200000000001 2023-09-23T19:38:27.785 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.785 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.785 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,4", 2023-09-23T19:38:27.785 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.747501+0000", 2023-09-23T19:38:27.785 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000109966 2023-09-23T19:38:27.786 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.786 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.786 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:38:27.786 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.749264+0000", 2023-09-23T19:38:27.786 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0017630059999999999 2023-09-23T19:38:27.786 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.786 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.786 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:38:27.786 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.749372+0000", 2023-09-23T19:38:27.786 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000107371 2023-09-23T19:38:27.786 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.787 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.787 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:38:27.787 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.934919+0000", 2023-09-23T19:38:27.787 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.185547394 2023-09-23T19:38:27.787 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.787 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.787 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:38:27.787 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.934961+0000", 2023-09-23T19:38:27.787 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.1940000000000002e-05 2023-09-23T19:38:27.787 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.787 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.787 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:38:27.787 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.934998+0000", 2023-09-23T19:38:27.788 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.7277000000000002e-05 2023-09-23T19:38:27.788 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:38:27.788 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:38:27.788 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:38:27.788 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.788 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.788 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45947.0:113830 10.2 10:4dcf8a3c:::benchmark_data_smithi016_136365_object3696:head [delete] snapc 0=[] ondisk+write+known_if_redirected e154)", 2023-09-23T19:38:27.788 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:38:09.954636+0000", 2023-09-23T19:38:27.788 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 17.737402251999999, 2023-09-23T19:38:27.788 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.131516461, 2023-09-23T19:38:27.788 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:38:27.788 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:38:27.788 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:38:27.789 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45947", 2023-09-23T19:38:27.789 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/4039191752", 2023-09-23T19:38:27.789 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 113830 2023-09-23T19:38:27.789 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.789 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:38:27.789 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.789 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:38:27.789 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:09.954636+0000", 2023-09-23T19:38:27.789 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:38:27.789 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.789 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.789 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:38:27.789 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:09.954636+0000", 2023-09-23T19:38:27.790 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:38:27.790 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.790 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.790 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:38:27.790 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:09.954635+0000", 2023-09-23T19:38:27.790 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-09-23T19:38:27.790 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.790 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.790 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:38:27.790 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:09.954640+0000", 2023-09-23T19:38:27.790 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.4219999999999999e-06 2023-09-23T19:38:27.790 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.791 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.791 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:38:27.791 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:09.954648+0000", 2023-09-23T19:38:27.791 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.7260000000000002e-06 2023-09-23T19:38:27.791 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.791 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.791 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:38:27.791 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:09.954654+0000", 2023-09-23T19:38:27.791 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.6529999999999999e-06 2023-09-23T19:38:27.791 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.791 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.791 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:38:27.792 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:09.954693+0000", 2023-09-23T19:38:27.792 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.9381000000000001e-05 2023-09-23T19:38:27.792 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.792 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.792 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:38:27.792 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:09.954789+0000", 2023-09-23T19:38:27.792 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.6490000000000006e-05 2023-09-23T19:38:27.792 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.792 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.792 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,4", 2023-09-23T19:38:27.792 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:09.954872+0000", 2023-09-23T19:38:27.792 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.2134999999999999e-05 2023-09-23T19:38:27.792 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.793 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.793 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:38:27.793 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:09.969741+0000", 2023-09-23T19:38:27.793 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.014869867 2023-09-23T19:38:27.793 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.793 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.793 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:38:27.793 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:09.984369+0000", 2023-09-23T19:38:27.793 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.014627195000000001 2023-09-23T19:38:27.793 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.793 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.793 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:38:27.793 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:10.086030+0000", 2023-09-23T19:38:27.794 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.101661553 2023-09-23T19:38:27.794 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.794 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.794 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:38:27.794 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:10.086100+0000", 2023-09-23T19:38:27.794 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.0007999999999998e-05 2023-09-23T19:38:27.794 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.794 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.794 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:38:27.794 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:10.086153+0000", 2023-09-23T19:38:27.794 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.2516999999999999e-05 2023-09-23T19:38:27.794 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:38:27.795 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:38:27.795 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:38:27.795 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.795 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.795 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45947.0:113831 10.2 10:49fb92f2:::benchmark_data_smithi016_136365_object3697:head [delete] snapc 0=[] ondisk+write+known_if_redirected e154)", 2023-09-23T19:38:27.795 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:38:09.955457+0000", 2023-09-23T19:38:27.795 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 17.736581055999999, 2023-09-23T19:38:27.795 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.13085756000000001, 2023-09-23T19:38:27.795 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:38:27.795 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:38:27.795 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:38:27.795 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45947", 2023-09-23T19:38:27.796 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/4039191752", 2023-09-23T19:38:27.796 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 113831 2023-09-23T19:38:27.796 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.796 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:38:27.796 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.796 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:38:27.796 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:09.955457+0000", 2023-09-23T19:38:27.796 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:38:27.796 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.796 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.796 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:38:27.796 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:09.955457+0000", 2023-09-23T19:38:27.797 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:38:27.797 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.797 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.797 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:38:27.797 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:09.955456+0000", 2023-09-23T19:38:27.797 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:38:27.797 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.797 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.797 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:38:27.797 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:09.955462+0000", 2023-09-23T19:38:27.797 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.801e-06 2023-09-23T19:38:27.797 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.797 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.798 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:38:27.798 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:09.955474+0000", 2023-09-23T19:38:27.798 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1921e-05 2023-09-23T19:38:27.798 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.798 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.798 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:38:27.798 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:09.955482+0000", 2023-09-23T19:38:27.798 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.8730000000000007e-06 2023-09-23T19:38:27.798 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.798 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.798 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:38:27.798 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:09.958247+0000", 2023-09-23T19:38:27.798 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0027645970000000001 2023-09-23T19:38:27.799 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.799 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.799 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:38:27.799 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:09.958391+0000", 2023-09-23T19:38:27.799 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00014486100000000001 2023-09-23T19:38:27.799 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.799 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.799 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,4", 2023-09-23T19:38:27.799 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:09.958501+0000", 2023-09-23T19:38:27.799 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000109302 2023-09-23T19:38:27.799 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.799 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.800 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:38:27.800 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:09.969747+0000", 2023-09-23T19:38:27.800 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.011246713 2023-09-23T19:38:27.800 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.800 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.800 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:38:27.800 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:09.984402+0000", 2023-09-23T19:38:27.800 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.014654645000000001 2023-09-23T19:38:27.800 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.800 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.800 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:38:27.800 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:10.086217+0000", 2023-09-23T19:38:27.800 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.10181522699999999 2023-09-23T19:38:27.801 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.801 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.801 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:38:27.801 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:10.086268+0000", 2023-09-23T19:38:27.801 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.0210999999999997e-05 2023-09-23T19:38:27.801 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.801 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.801 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:38:27.801 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:10.086315+0000", 2023-09-23T19:38:27.801 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.7411999999999999e-05 2023-09-23T19:38:27.801 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:38:27.801 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:38:27.801 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:38:27.802 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.802 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.802 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45947.0:113832 10.2 10:4ae498cd:::benchmark_data_smithi016_136365_object3698:head [delete] snapc 0=[] ondisk+write+known_if_redirected e154)", 2023-09-23T19:38:27.802 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:38:09.956225+0000", 2023-09-23T19:38:27.802 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 17.735813789000002, 2023-09-23T19:38:27.802 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.130248315, 2023-09-23T19:38:27.802 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:38:27.802 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:38:27.802 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:38:27.802 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45947", 2023-09-23T19:38:27.802 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/4039191752", 2023-09-23T19:38:27.802 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 113832 2023-09-23T19:38:27.802 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.803 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:38:27.803 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.803 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:38:27.803 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:09.956225+0000", 2023-09-23T19:38:27.803 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:38:27.803 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.803 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.803 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:38:27.803 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:09.956225+0000", 2023-09-23T19:38:27.803 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:38:27.803 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.803 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.803 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:38:27.804 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:09.956224+0000", 2023-09-23T19:38:27.804 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:38:27.804 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.804 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.804 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:38:27.804 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:09.956242+0000", 2023-09-23T19:38:27.804 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.8502e-05 2023-09-23T19:38:27.804 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.804 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.804 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:38:27.804 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:09.956254+0000", 2023-09-23T19:38:27.804 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1287000000000001e-05 2023-09-23T19:38:27.804 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.805 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.805 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:38:27.805 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:09.956261+0000", 2023-09-23T19:38:27.805 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.6550000000000004e-06 2023-09-23T19:38:27.805 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.805 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.805 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:38:27.805 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:09.962541+0000", 2023-09-23T19:38:27.805 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0062796739999999998 2023-09-23T19:38:27.805 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.805 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.805 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:38:27.806 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:09.962693+0000", 2023-09-23T19:38:27.806 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00015251300000000001 2023-09-23T19:38:27.806 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.806 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.806 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,4", 2023-09-23T19:38:27.806 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:09.962819+0000", 2023-09-23T19:38:27.806 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000125671 2023-09-23T19:38:27.806 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.806 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.806 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:38:27.806 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:09.969751+0000", 2023-09-23T19:38:27.806 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0069317509999999999 2023-09-23T19:38:27.807 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.807 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.807 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:38:27.807 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:09.984433+0000", 2023-09-23T19:38:27.807 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.014681856 2023-09-23T19:38:27.807 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.807 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.807 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:38:27.807 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:10.086378+0000", 2023-09-23T19:38:27.807 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.10194561000000001 2023-09-23T19:38:27.807 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.807 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.808 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:38:27.808 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:10.086429+0000", 2023-09-23T19:38:27.808 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.1149000000000001e-05 2023-09-23T19:38:27.808 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.808 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.808 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:38:27.808 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:10.086473+0000", 2023-09-23T19:38:27.808 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.3556000000000001e-05 2023-09-23T19:38:27.808 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:38:27.808 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:38:27.808 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:38:27.808 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.808 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.809 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45947.0:113838 10.2 10:42f36b91:::benchmark_data_smithi016_136365_object3704:head [delete] snapc 0=[] ondisk+write+known_if_redirected e154)", 2023-09-23T19:38:27.809 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:38:09.961097+0000", 2023-09-23T19:38:27.809 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 17.730941001000001, 2023-09-23T19:38:27.809 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.12551258400000001, 2023-09-23T19:38:27.809 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:38:27.809 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:38:27.809 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:38:27.809 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45947", 2023-09-23T19:38:27.809 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/4039191752", 2023-09-23T19:38:27.809 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 113838 2023-09-23T19:38:27.809 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.809 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:38:27.809 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.810 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:38:27.810 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:09.961097+0000", 2023-09-23T19:38:27.810 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:38:27.810 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.810 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.810 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:38:27.810 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:09.961097+0000", 2023-09-23T19:38:27.810 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:38:27.810 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.810 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.810 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:38:27.810 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:09.961096+0000", 2023-09-23T19:38:27.811 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-09-23T19:38:27.811 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.811 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.811 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:38:27.811 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:09.961104+0000", 2023-09-23T19:38:27.811 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.3760000000000008e-06 2023-09-23T19:38:27.811 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.811 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.811 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:38:27.811 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:09.961117+0000", 2023-09-23T19:38:27.811 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2782000000000001e-05 2023-09-23T19:38:27.811 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.811 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.812 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:38:27.812 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:09.961124+0000", 2023-09-23T19:38:27.812 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.7569999999999998e-06 2023-09-23T19:38:27.812 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.812 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.812 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:38:27.812 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:09.966696+0000", 2023-09-23T19:38:27.812 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0055720409999999998 2023-09-23T19:38:27.812 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.812 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.812 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:38:27.812 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:09.966785+0000", 2023-09-23T19:38:27.813 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.9534e-05 2023-09-23T19:38:27.813 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.813 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.813 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,4", 2023-09-23T19:38:27.813 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:09.966863+0000", 2023-09-23T19:38:27.813 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.7739999999999998e-05 2023-09-23T19:38:27.813 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.813 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.813 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:38:27.813 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:09.969905+0000", 2023-09-23T19:38:27.813 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0030424219999999999 2023-09-23T19:38:27.813 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.813 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.814 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:38:27.814 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:09.984475+0000", 2023-09-23T19:38:27.814 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.014569367999999999 2023-09-23T19:38:27.814 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.814 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.814 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:38:27.814 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:10.086520+0000", 2023-09-23T19:38:27.814 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.102045625 2023-09-23T19:38:27.814 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.814 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.814 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:38:27.814 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:10.086569+0000", 2023-09-23T19:38:27.814 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.8563000000000002e-05 2023-09-23T19:38:27.815 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:38:27.815 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:38:27.815 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:38:27.815 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:10.086610+0000", 2023-09-23T19:38:27.815 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.1232000000000001e-05 2023-09-23T19:38:27.815 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:38:27.815 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:38:27.815 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:38:27.815 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:38:27.815 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:38:27.815 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-09-23T19:38:27.816 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:27.847 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:27.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:27.853 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:27 smithi092 bash[58855]: cluster 2023-09-23T19:38:26.112706+0000 mgr.y (mgr.25325) 928 : cluster 0 pgmap v855: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 944 op/s; 35931/285339 objects degraded (12.592%); 0 B/s, 0 objects/s recovering 2023-09-23T19:38:27.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:38:27.954 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:27.993 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:27.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:27.999 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:38:28.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:27 smithi016 bash[77995]: cluster 2023-09-23T19:38:26.112706+0000 mgr.y (mgr.25325) 928 : cluster 0 pgmap v855: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 944 op/s; 35931/285339 objects degraded (12.592%); 0 B/s, 0 objects/s recovering 2023-09-23T19:38:28.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:27 smithi016 bash[75004]: cluster 2023-09-23T19:38:26.112706+0000 mgr.y (mgr.25325) 928 : cluster 0 pgmap v855: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 944 op/s; 35931/285339 objects degraded (12.592%); 0 B/s, 0 objects/s recovering 2023-09-23T19:38:28.099 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:28.122 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:28.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:28.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:28.226 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:28.258 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:28.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:28.265 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:38:28.365 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:28.393 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:28.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:28.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:28.412 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:38:28.454 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:38:28.480 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:38:28.499 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:28.516 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:38:28.524 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:28.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:28.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:28.563 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:38:28.602 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:38:28.632 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:28.646 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:38:28.646 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:38:28.647 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:38:28.647 INFO:teuthology.orchestra.run.smithi016.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T19:38:28.647 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:38:28.647 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:38:28.647 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:38:28.647 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:38:28.647 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:38:28.665 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:28.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:28.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:38:28.773 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:28.803 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:28.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:28.810 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:28.910 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:28.920 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:28 smithi016 bash[75004]: audit 2023-09-23T19:38:28.368040+0000 mon.a (mon.0) 1143 : audit 0 from='client.? 172.21.15.16:0/3457332070' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:38:28.920 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:28 smithi016 bash[75004]: audit 2023-09-23T19:38:28.659890+0000 mon.a (mon.0) 1144 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:38:28.921 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:28 smithi016 bash[75004]: audit 2023-09-23T19:38:28.666544+0000 mon.a (mon.0) 1145 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:38:28.921 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:28 smithi016 bash[77995]: audit 2023-09-23T19:38:28.368040+0000 mon.a (mon.0) 1143 : audit 0 from='client.? 172.21.15.16:0/3457332070' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:38:28.921 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:28 smithi016 bash[77995]: audit 2023-09-23T19:38:28.659890+0000 mon.a (mon.0) 1144 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:38:28.921 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:28 smithi016 bash[77995]: audit 2023-09-23T19:38:28.666544+0000 mon.a (mon.0) 1145 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:38:28.940 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:28.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:28.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:29.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:28 smithi092 bash[58855]: audit 2023-09-23T19:38:28.368040+0000 mon.a (mon.0) 1143 : audit 0 from='client.? 172.21.15.16:0/3457332070' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:38:29.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:28 smithi092 bash[58855]: audit 2023-09-23T19:38:28.659890+0000 mon.a (mon.0) 1144 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:38:29.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:28 smithi092 bash[58855]: audit 2023-09-23T19:38:28.666544+0000 mon.a (mon.0) 1145 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:38:29.044 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:29.078 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:29.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:29.081 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:29.181 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:29.206 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:29.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:29.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:29.310 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:29.345 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:29.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:29.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:29.450 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:29.477 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:29.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:29.480 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:38:29.581 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:29.610 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:29.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:29.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:29.719 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:29.728 INFO:journalctl@ceph.osd.2.smithi016.stdout:Sep 23 19:38:29 smithi016 bash[144380]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T19:38:29.728 INFO:journalctl@ceph.osd.2.smithi016.stdout:Sep 23 19:38:29 smithi016 bash[144380]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-09-23T19:38:29.728 INFO:journalctl@ceph.osd.2.smithi016.stdout:Sep 23 19:38:29 smithi016 bash[144380]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2023-09-23T19:38:29.729 INFO:journalctl@ceph.osd.2.smithi016.stdout:Sep 23 19:38:29 smithi016 bash[144380]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2023-09-23T19:38:29.729 INFO:journalctl@ceph.osd.2.smithi016.stdout:Sep 23 19:38:29 smithi016 bash[144380]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2023-09-23T19:38:29.729 INFO:journalctl@ceph.osd.2.smithi016.stdout:Sep 23 19:38:29 smithi016 bash[144380]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2023-09-23T19:38:29.729 INFO:journalctl@ceph.osd.2.smithi016.stdout:Sep 23 19:38:29 smithi016 bash[144380]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-09-23T19:38:29.729 INFO:journalctl@ceph.osd.2.smithi016.stdout:Sep 23 19:38:29 smithi016 bash[144380]: --> ceph-volume lvm activate successful for osd ID: 2 2023-09-23T19:38:29.744 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:29.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:29.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:29.822 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is set 2023-09-23T19:38:29.848 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:29.886 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:29.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:29.894 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:29.995 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:30.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:29 smithi016 bash[77995]: cluster 2023-09-23T19:38:28.113370+0000 mgr.y (mgr.25325) 929 : cluster 0 pgmap v856: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 35460/283197 objects degraded (12.521%); 0 B/s, 0 objects/s recovering 2023-09-23T19:38:30.002 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:29 smithi016 bash[77995]: cluster 2023-09-23T19:38:28.779450+0000 mon.a (mon.0) 1146 : cluster 3 Health check update: Degraded data redundancy: 35460/283197 objects degraded (12.521%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T19:38:30.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:29 smithi016 bash[77995]: audit 2023-09-23T19:38:29.410911+0000 mon.c (mon.2) 165 : audit 1 from='client.? 172.21.15.16:0/684018966' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:38:30.003 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:29 smithi016 bash[77995]: audit 2023-09-23T19:38:29.411308+0000 mon.a (mon.0) 1147 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:38:30.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:29 smithi016 bash[75004]: cluster 2023-09-23T19:38:28.113370+0000 mgr.y (mgr.25325) 929 : cluster 0 pgmap v856: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 35460/283197 objects degraded (12.521%); 0 B/s, 0 objects/s recovering 2023-09-23T19:38:30.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:29 smithi016 bash[75004]: cluster 2023-09-23T19:38:28.779450+0000 mon.a (mon.0) 1146 : cluster 3 Health check update: Degraded data redundancy: 35460/283197 objects degraded (12.521%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T19:38:30.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:29 smithi016 bash[75004]: audit 2023-09-23T19:38:29.410911+0000 mon.c (mon.2) 165 : audit 1 from='client.? 172.21.15.16:0/684018966' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:38:30.003 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:29 smithi016 bash[75004]: audit 2023-09-23T19:38:29.411308+0000 mon.a (mon.0) 1147 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:38:30.017 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:30.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:30.021 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:38:30.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:29 smithi092 bash[58855]: cluster 2023-09-23T19:38:28.113370+0000 mgr.y (mgr.25325) 929 : cluster 0 pgmap v856: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 35460/283197 objects degraded (12.521%); 0 B/s, 0 objects/s recovering 2023-09-23T19:38:30.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:29 smithi092 bash[58855]: cluster 2023-09-23T19:38:28.779450+0000 mon.a (mon.0) 1146 : cluster 3 Health check update: Degraded data redundancy: 35460/283197 objects degraded (12.521%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T19:38:30.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:29 smithi092 bash[58855]: audit 2023-09-23T19:38:29.410911+0000 mon.c (mon.2) 165 : audit 1 from='client.? 172.21.15.16:0/684018966' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:38:30.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:29 smithi092 bash[58855]: audit 2023-09-23T19:38:29.411308+0000 mon.a (mon.0) 1147 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:38:30.122 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:30.147 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:30.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:30.149 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:30.250 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:30.282 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:30.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:30.285 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:30.386 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:30.410 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:30.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:30.416 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:38:30.517 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:30.543 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:30.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:30.545 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:30.608 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T19:38:30.646 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:30.702 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:30.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:30.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:30.808 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:30.837 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:30.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:30.840 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:38:30.941 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:30.950 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:30 smithi016 bash[77995]: cluster 2023-09-23T19:38:29.797565+0000 mon.a (mon.0) 1148 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:38:30.950 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:30 smithi016 bash[77995]: audit 2023-09-23T19:38:29.809448+0000 mon.a (mon.0) 1149 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:38:30.951 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:30 smithi016 bash[77995]: cluster 2023-09-23T19:38:29.809808+0000 mon.a (mon.0) 1150 : cluster 0 osdmap e157: 8 total, 7 up, 8 in 2023-09-23T19:38:30.951 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:30 smithi016 bash[77995]: cluster 2023-09-23T19:38:29.810520+0000 osd.7 (osd.7) 70 : cluster 3 failed to encode map e157 with expected crc 2023-09-23T19:38:30.951 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:30 smithi016 bash[77995]: cluster 2023-09-23T19:38:29.813848+0000 osd.6 (osd.6) 75 : cluster 3 failed to encode map e157 with expected crc 2023-09-23T19:38:30.951 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:30 smithi016 bash[75004]: cluster 2023-09-23T19:38:29.797565+0000 mon.a (mon.0) 1148 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:38:30.951 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:30 smithi016 bash[75004]: audit 2023-09-23T19:38:29.809448+0000 mon.a (mon.0) 1149 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:38:30.951 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:30 smithi016 bash[75004]: cluster 2023-09-23T19:38:29.809808+0000 mon.a (mon.0) 1150 : cluster 0 osdmap e157: 8 total, 7 up, 8 in 2023-09-23T19:38:30.951 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:30 smithi016 bash[75004]: cluster 2023-09-23T19:38:29.810520+0000 osd.7 (osd.7) 70 : cluster 3 failed to encode map e157 with expected crc 2023-09-23T19:38:30.951 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:30 smithi016 bash[75004]: cluster 2023-09-23T19:38:29.813848+0000 osd.6 (osd.6) 75 : cluster 3 failed to encode map e157 with expected crc 2023-09-23T19:38:30.976 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:30.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:30.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:31.085 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:31.119 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:31.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:31.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:38:31.223 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:31.249 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:31.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:31.253 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:31.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:30 smithi092 bash[58855]: cluster 2023-09-23T19:38:29.797565+0000 mon.a (mon.0) 1148 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:38:31.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:30 smithi092 bash[58855]: audit 2023-09-23T19:38:29.809448+0000 mon.a (mon.0) 1149 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:38:31.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:30 smithi092 bash[58855]: cluster 2023-09-23T19:38:29.809808+0000 mon.a (mon.0) 1150 : cluster 0 osdmap e157: 8 total, 7 up, 8 in 2023-09-23T19:38:31.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:30 smithi092 bash[58855]: cluster 2023-09-23T19:38:29.810520+0000 osd.7 (osd.7) 70 : cluster 3 failed to encode map e157 with expected crc 2023-09-23T19:38:31.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:30 smithi092 bash[58855]: cluster 2023-09-23T19:38:29.813848+0000 osd.6 (osd.6) 75 : cluster 3 failed to encode map e157 with expected crc 2023-09-23T19:38:31.354 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:31.377 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:31.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:31.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:31.480 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:31.505 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:31.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:31.515 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:31.616 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:31.646 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:31.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:31.649 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:38:31.749 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:31.777 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:31.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:31.779 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:38:31.880 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:31.890 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:31 smithi016 bash[77995]: cluster 2023-09-23T19:38:30.113983+0000 mgr.y (mgr.25325) 930 : cluster 0 pgmap v858: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 816 op/s; 35205/278742 objects degraded (12.630%) 2023-09-23T19:38:31.891 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:31 smithi016 bash[75004]: cluster 2023-09-23T19:38:30.113983+0000 mgr.y (mgr.25325) 930 : cluster 0 pgmap v858: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 816 op/s; 35205/278742 objects degraded (12.630%) 2023-09-23T19:38:31.910 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:31.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:31.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:32.014 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:32.049 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:32.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:32.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:32.111 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T19:38:32.112 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T19:38:32.152 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:32.180 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:32.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:32.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:32.284 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:32.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:31 smithi092 bash[58855]: cluster 2023-09-23T19:38:30.113983+0000 mgr.y (mgr.25325) 930 : cluster 0 pgmap v858: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 816 op/s; 35205/278742 objects degraded (12.630%) 2023-09-23T19:38:32.317 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:32.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:32.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:32.423 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:32.452 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:32.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:32.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:32.557 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:32.594 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:32.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:32.601 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:32.702 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:32.731 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:32.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:32.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:32.839 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:32.870 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:32.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:32.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:32.977 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:33.009 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:33.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:33.015 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:33.116 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:33.141 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:33.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:33.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:33.163 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:38:33.247 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:33.279 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:33.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:33.284 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:38:33.384 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:33.422 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:33.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:33.429 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:33.530 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:33.555 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:33.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:33.557 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:33.658 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:33.681 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:33.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:33.689 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:33.790 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:33.813 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:33.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:33.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:33.918 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:33.943 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:33.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:33.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:34.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:33 smithi092 bash[58855]: cluster 2023-09-23T19:38:32.114896+0000 mgr.y (mgr.25325) 931 : cluster 0 pgmap v859: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 810 op/s; 34462/273366 objects degraded (12.607%) 2023-09-23T19:38:34.046 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:34.069 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:34.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:34.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:38:34.113 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:38:34.125 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:33 smithi016 bash[75004]: cluster 2023-09-23T19:38:32.114896+0000 mgr.y (mgr.25325) 931 : cluster 0 pgmap v859: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 810 op/s; 34462/273366 objects degraded (12.607%) 2023-09-23T19:38:34.125 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:33 smithi016 bash[77995]: cluster 2023-09-23T19:38:32.114896+0000 mgr.y (mgr.25325) 931 : cluster 0 pgmap v859: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 810 op/s; 34462/273366 objects degraded (12.607%) 2023-09-23T19:38:34.176 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:34.211 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:34.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:34.218 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:38:34.319 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:34.350 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:34.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:34.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:34.456 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:34.482 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:34.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:34.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:34.593 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:34.640 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:34.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:34.644 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:34.745 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:34.771 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:34.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:34.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:34.786 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T19:38:34.879 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:34.905 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:34.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:34.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:38:35.012 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:35.020 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:34 smithi092 bash[58855]: cluster 2023-09-23T19:38:34.426481+0000 mon.a (mon.0) 1151 : cluster 3 Health check update: Degraded data redundancy: 34462/273366 objects degraded (12.607%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T19:38:35.036 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:35.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:35.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:38:35.139 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:35.167 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:35.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:35.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:35.192 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:34 smithi016 bash[75004]: cluster 2023-09-23T19:38:34.426481+0000 mon.a (mon.0) 1151 : cluster 3 Health check update: Degraded data redundancy: 34462/273366 objects degraded (12.607%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T19:38:35.193 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:34 smithi016 bash[77995]: cluster 2023-09-23T19:38:34.426481+0000 mon.a (mon.0) 1151 : cluster 3 Health check update: Degraded data redundancy: 34462/273366 objects degraded (12.607%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T19:38:35.272 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:35.300 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:35.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:35.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:35.408 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:35.442 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:35.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:35.448 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:35.535 INFO:journalctl@ceph.osd.2.smithi016.stdout:Sep 23 19:38:35 smithi016 bash[145232]: debug 2023-09-23T19:38:35.292+0000 7f316d29f700 1 -- 172.21.15.16:0/332587350 <== mon.0 v2:172.21.15.16:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x55a7c9287600 con 0x55a7c9296800 2023-09-23T19:38:35.536 INFO:journalctl@ceph.osd.2.smithi016.stdout:Sep 23 19:38:35 smithi016 bash[145232]: debug 2023-09-23T19:38:35.292+0000 7f316d29f700 1 -- 172.21.15.16:0/332587350 <== mon.0 v2:172.21.15.16:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55a7c923c4e0 con 0x55a7c9296800 2023-09-23T19:38:35.548 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:35.574 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:35.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:35.576 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:35.676 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:35.706 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:35.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:35.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:38:35.812 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:35.821 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:38:35 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:38:35] "GET /metrics HTTP/1.1" 200 36318 "" "Prometheus/2.33.4" 2023-09-23T19:38:35.850 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:35.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:35.853 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:35.954 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:35.961 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:35 smithi092 bash[58855]: cluster 2023-09-23T19:38:34.115514+0000 mgr.y (mgr.25325) 932 : cluster 0 pgmap v860: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 807 op/s; 33955/271161 objects degraded (12.522%) 2023-09-23T19:38:35.991 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:35.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:35.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:36.036 INFO:journalctl@ceph.osd.2.smithi016.stdout:Sep 23 19:38:35 smithi016 bash[145232]: debug 2023-09-23T19:38:35.848+0000 7f3176424640 -1 Falling back to public interface 2023-09-23T19:38:36.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:35 smithi016 bash[75004]: cluster 2023-09-23T19:38:34.115514+0000 mgr.y (mgr.25325) 932 : cluster 0 pgmap v860: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 807 op/s; 33955/271161 objects degraded (12.522%) 2023-09-23T19:38:36.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:35 smithi016 bash[77995]: cluster 2023-09-23T19:38:34.115514+0000 mgr.y (mgr.25325) 932 : cluster 0 pgmap v860: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 807 op/s; 33955/271161 objects degraded (12.522%) 2023-09-23T19:38:36.094 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:36.118 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:36.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:36.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:36.222 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:36.243 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:36.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:36.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:36.346 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:36.383 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:36.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:36.386 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:36.487 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:36.518 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:36.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:36.521 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:36.622 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:36.647 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:36.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:36.649 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:38:36.749 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:36.780 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:36.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:36.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:36.885 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:36.913 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:36.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:36.920 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:37.020 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:37.053 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:37.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:37.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:37.159 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:37.188 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:37.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:37.191 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:37.292 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:37.316 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:38:37.331 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:37.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:37.333 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:37.434 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:37.484 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:37.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:37.489 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:37.590 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:37.612 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:37.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:37.615 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:37.715 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:37.751 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:37.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:37.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:38:37.858 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:37.865 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:37 smithi016 bash[77995]: cluster 2023-09-23T19:38:36.116371+0000 mgr.y (mgr.25325) 933 : cluster 0 pgmap v861: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 797 op/s; 32976/261420 objects degraded (12.614%) 2023-09-23T19:38:37.866 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:37 smithi016 bash[75004]: cluster 2023-09-23T19:38:36.116371+0000 mgr.y (mgr.25325) 933 : cluster 0 pgmap v861: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 797 op/s; 32976/261420 objects degraded (12.614%) 2023-09-23T19:38:37.889 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:37.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:37.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:37.994 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:38.004 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:37 smithi092 bash[58855]: cluster 2023-09-23T19:38:36.116371+0000 mgr.y (mgr.25325) 933 : cluster 0 pgmap v861: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 797 op/s; 32976/261420 objects degraded (12.614%) 2023-09-23T19:38:38.028 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:38.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:38.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:38.133 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:38.158 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:38.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:38.161 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:38.262 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:38.287 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:38.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:38.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:38.391 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:38.416 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:38.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:38.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:38:38.522 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:38.548 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:38.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:38.552 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:38.652 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:38.679 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:38.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:38.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:38.787 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:38.826 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:38.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:38.832 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:38.933 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:38.972 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:38.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:38.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:38:39.078 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:39.104 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:39.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:39.109 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:39.210 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:39.247 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:39.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:39.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:39.352 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:39.380 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:38:39.380 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:38:39.383 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:39.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:39.385 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:38:39.486 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:39.519 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:39.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:39.524 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:38:39.617 INFO:teuthology.orchestra.run.smithi016.stderr:no valid command found; 10 closest matches: 2023-09-23T19:38:39.617 INFO:teuthology.orchestra.run.smithi016.stderr:0 2023-09-23T19:38:39.617 INFO:teuthology.orchestra.run.smithi016.stderr:1 2023-09-23T19:38:39.618 INFO:teuthology.orchestra.run.smithi016.stderr:2 2023-09-23T19:38:39.618 INFO:teuthology.orchestra.run.smithi016.stderr:abort 2023-09-23T19:38:39.618 INFO:teuthology.orchestra.run.smithi016.stderr:assert 2023-09-23T19:38:39.618 INFO:teuthology.orchestra.run.smithi016.stderr:bluefs debug_inject_read_zeros 2023-09-23T19:38:39.618 INFO:teuthology.orchestra.run.smithi016.stderr:bluefs files list 2023-09-23T19:38:39.618 INFO:teuthology.orchestra.run.smithi016.stderr:bluefs stats 2023-09-23T19:38:39.618 INFO:teuthology.orchestra.run.smithi016.stderr:bluestore allocator dump block 2023-09-23T19:38:39.618 INFO:teuthology.orchestra.run.smithi016.stderr:bluestore allocator fragmentation block 2023-09-23T19:38:39.618 INFO:teuthology.orchestra.run.smithi016.stderr:admin_socket: invalid command 2023-09-23T19:38:39.625 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:39.664 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:39.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:39.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:39.770 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:39.800 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:39.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:39.806 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:39.907 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:39.940 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:39.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:39.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:38:40.018 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:39 smithi016 bash[75004]: cluster 2023-09-23T19:38:38.116914+0000 mgr.y (mgr.25325) 934 : cluster 0 pgmap v862: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 5.3 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 793 op/s; 32512/259371 objects degraded (12.535%) 2023-09-23T19:38:40.018 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:39 smithi016 bash[75004]: cluster 2023-09-23T19:38:39.427431+0000 mon.a (mon.0) 1152 : cluster 3 Health check update: Degraded data redundancy: 32512/259371 objects degraded (12.535%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T19:38:40.019 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:39 smithi016 bash[77995]: cluster 2023-09-23T19:38:38.116914+0000 mgr.y (mgr.25325) 934 : cluster 0 pgmap v862: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 5.3 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 793 op/s; 32512/259371 objects degraded (12.535%) 2023-09-23T19:38:40.019 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:39 smithi016 bash[77995]: cluster 2023-09-23T19:38:39.427431+0000 mon.a (mon.0) 1152 : cluster 3 Health check update: Degraded data redundancy: 32512/259371 objects degraded (12.535%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T19:38:40.045 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:40.084 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:40.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:40.089 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:39 smithi092 bash[58855]: cluster 2023-09-23T19:38:38.116914+0000 mgr.y (mgr.25325) 934 : cluster 0 pgmap v862: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 5.3 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 793 op/s; 32512/259371 objects degraded (12.535%) 2023-09-23T19:38:40.089 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:39 smithi092 bash[58855]: cluster 2023-09-23T19:38:39.427431+0000 mon.a (mon.0) 1152 : cluster 3 Health check update: Degraded data redundancy: 32512/259371 objects degraded (12.535%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T19:38:40.089 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:40.190 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:40.220 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:40.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:40.223 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:40.324 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:40.350 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:40.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:40.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:40.454 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:40.481 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:40.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:40.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:40.584 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:40.612 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:40.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:40.616 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:40.717 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:40.744 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:40.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:40.749 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:38:40.850 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:40.870 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:40.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:40.872 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:40.972 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:41.015 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:41.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:41.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:38:41.121 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:41.156 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:41.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:41.159 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:41.259 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:41.289 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:41.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:41.293 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:41.394 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:41.430 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:41.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:41.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:41.535 INFO:journalctl@ceph.osd.2.smithi016.stdout:Sep 23 19:38:41 smithi016 bash[145232]: debug 2023-09-23T19:38:41.436+0000 7f3176424640 -1 osd.2 154 log_to_monitors true 2023-09-23T19:38:41.536 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:41.563 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:41.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:41.565 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:38:41.666 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:41.703 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:41.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:41.705 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:41.806 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:41.835 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:41.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:41.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:41.941 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:41.966 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:41.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:41.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:38:42.036 INFO:journalctl@ceph.osd.2.smithi016.stdout:Sep 23 19:38:41 smithi016 bash[145232]: debug 2023-09-23T19:38:41.884+0000 7f3163889700 -1 osd.2 154 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T19:38:42.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:41 smithi016 bash[77995]: cluster 2023-09-23T19:38:40.117678+0000 mgr.y (mgr.25325) 935 : cluster 0 pgmap v863: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 764 op/s; 32259/255093 objects degraded (12.646%) 2023-09-23T19:38:42.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:41 smithi016 bash[77995]: audit 2023-09-23T19:38:41.439818+0000 mon.c (mon.2) 166 : audit 1 from='osd.2 [v2:172.21.15.16:6818/4244964590,v1:172.21.15.16:6819/4244964590]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T19:38:42.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:41 smithi016 bash[77995]: audit 2023-09-23T19:38:41.440183+0000 mon.a (mon.0) 1153 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T19:38:42.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:41 smithi016 bash[75004]: cluster 2023-09-23T19:38:40.117678+0000 mgr.y (mgr.25325) 935 : cluster 0 pgmap v863: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 764 op/s; 32259/255093 objects degraded (12.646%) 2023-09-23T19:38:42.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:41 smithi016 bash[75004]: audit 2023-09-23T19:38:41.439818+0000 mon.c (mon.2) 166 : audit 1 from='osd.2 [v2:172.21.15.16:6818/4244964590,v1:172.21.15.16:6819/4244964590]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T19:38:42.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:41 smithi016 bash[75004]: audit 2023-09-23T19:38:41.440183+0000 mon.a (mon.0) 1153 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T19:38:42.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:41 smithi092 bash[58855]: cluster 2023-09-23T19:38:40.117678+0000 mgr.y (mgr.25325) 935 : cluster 0 pgmap v863: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 764 op/s; 32259/255093 objects degraded (12.646%) 2023-09-23T19:38:42.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:41 smithi092 bash[58855]: audit 2023-09-23T19:38:41.439818+0000 mon.c (mon.2) 166 : audit 1 from='osd.2 [v2:172.21.15.16:6818/4244964590,v1:172.21.15.16:6819/4244964590]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T19:38:42.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:41 smithi092 bash[58855]: audit 2023-09-23T19:38:41.440183+0000 mon.a (mon.0) 1153 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T19:38:42.071 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:42.105 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:42.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:42.112 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:42.212 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:42.246 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:42.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:42.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:42.349 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:42.375 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:42.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:42.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:42.480 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:42.518 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:42.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:42.521 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:42.622 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:42.650 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:42.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:42.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:42.754 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:42.781 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:42.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:42.788 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:38:42.888 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:42.912 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:42.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:42.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:42.934 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T19:38:42.934 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-2, ['dump_ops_in_flight'] 2023-09-23T19:38:43.016 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:43.036 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:43.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:43.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:43.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:42 smithi092 bash[58855]: audit 2023-09-23T19:38:41.840680+0000 mon.a (mon.0) 1154 : audit 1 from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T19:38:43.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:42 smithi092 bash[58855]: cluster 2023-09-23T19:38:41.840735+0000 mon.a (mon.0) 1155 : cluster 0 osdmap e158: 8 total, 7 up, 8 in 2023-09-23T19:38:43.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:42 smithi092 bash[58855]: audit 2023-09-23T19:38:41.841859+0000 mon.c (mon.2) 167 : audit 1 from='osd.2 [v2:172.21.15.16:6818/4244964590,v1:172.21.15.16:6819/4244964590]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-09-23T19:38:43.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:42 smithi092 bash[58855]: cluster 2023-09-23T19:38:41.850308+0000 osd.5 (osd.5) 87 : cluster 3 failed to encode map e158 with expected crc 2023-09-23T19:38:43.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:42 smithi092 bash[58855]: cluster 2023-09-23T19:38:41.852441+0000 osd.7 (osd.7) 71 : cluster 3 failed to encode map e158 with expected crc 2023-09-23T19:38:43.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:42 smithi092 bash[58855]: audit 2023-09-23T19:38:41.852695+0000 mon.a (mon.0) 1156 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-09-23T19:38:43.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:42 smithi092 bash[58855]: cluster 2023-09-23T19:38:41.853626+0000 osd.6 (osd.6) 76 : cluster 3 failed to encode map e158 with expected crc 2023-09-23T19:38:43.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:42 smithi092 bash[58855]: cluster 2023-09-23T19:38:41.857106+0000 osd.4 (osd.4) 85 : cluster 3 failed to encode map e158 with expected crc 2023-09-23T19:38:43.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:42 smithi092 bash[58855]: cluster 2023-09-23T19:38:41.858152+0000 osd.4 (osd.4) 86 : cluster 3 failed to encode map e158 with expected crc 2023-09-23T19:38:43.139 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:43.168 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:43.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:43.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:43.272 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:43.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:42 smithi016 bash[77995]: audit 2023-09-23T19:38:41.840680+0000 mon.a (mon.0) 1154 : audit 1 from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T19:38:43.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:42 smithi016 bash[77995]: cluster 2023-09-23T19:38:41.840735+0000 mon.a (mon.0) 1155 : cluster 0 osdmap e158: 8 total, 7 up, 8 in 2023-09-23T19:38:43.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:42 smithi016 bash[77995]: audit 2023-09-23T19:38:41.841859+0000 mon.c (mon.2) 167 : audit 1 from='osd.2 [v2:172.21.15.16:6818/4244964590,v1:172.21.15.16:6819/4244964590]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-09-23T19:38:43.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:42 smithi016 bash[77995]: cluster 2023-09-23T19:38:41.850308+0000 osd.5 (osd.5) 87 : cluster 3 failed to encode map e158 with expected crc 2023-09-23T19:38:43.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:42 smithi016 bash[77995]: cluster 2023-09-23T19:38:41.852441+0000 osd.7 (osd.7) 71 : cluster 3 failed to encode map e158 with expected crc 2023-09-23T19:38:43.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:42 smithi016 bash[77995]: audit 2023-09-23T19:38:41.852695+0000 mon.a (mon.0) 1156 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-09-23T19:38:43.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:42 smithi016 bash[77995]: cluster 2023-09-23T19:38:41.853626+0000 osd.6 (osd.6) 76 : cluster 3 failed to encode map e158 with expected crc 2023-09-23T19:38:43.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:42 smithi016 bash[77995]: cluster 2023-09-23T19:38:41.857106+0000 osd.4 (osd.4) 85 : cluster 3 failed to encode map e158 with expected crc 2023-09-23T19:38:43.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:42 smithi016 bash[77995]: cluster 2023-09-23T19:38:41.858152+0000 osd.4 (osd.4) 86 : cluster 3 failed to encode map e158 with expected crc 2023-09-23T19:38:43.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:42 smithi016 bash[75004]: audit 2023-09-23T19:38:41.840680+0000 mon.a (mon.0) 1154 : audit 1 from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T19:38:43.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:42 smithi016 bash[75004]: cluster 2023-09-23T19:38:41.840735+0000 mon.a (mon.0) 1155 : cluster 0 osdmap e158: 8 total, 7 up, 8 in 2023-09-23T19:38:43.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:42 smithi016 bash[75004]: audit 2023-09-23T19:38:41.841859+0000 mon.c (mon.2) 167 : audit 1 from='osd.2 [v2:172.21.15.16:6818/4244964590,v1:172.21.15.16:6819/4244964590]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-09-23T19:38:43.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:42 smithi016 bash[75004]: cluster 2023-09-23T19:38:41.850308+0000 osd.5 (osd.5) 87 : cluster 3 failed to encode map e158 with expected crc 2023-09-23T19:38:43.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:42 smithi016 bash[75004]: cluster 2023-09-23T19:38:41.852441+0000 osd.7 (osd.7) 71 : cluster 3 failed to encode map e158 with expected crc 2023-09-23T19:38:43.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:42 smithi016 bash[75004]: audit 2023-09-23T19:38:41.852695+0000 mon.a (mon.0) 1156 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi016", "root=default"]}]: dispatch 2023-09-23T19:38:43.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:42 smithi016 bash[75004]: cluster 2023-09-23T19:38:41.853626+0000 osd.6 (osd.6) 76 : cluster 3 failed to encode map e158 with expected crc 2023-09-23T19:38:43.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:42 smithi016 bash[75004]: cluster 2023-09-23T19:38:41.857106+0000 osd.4 (osd.4) 85 : cluster 3 failed to encode map e158 with expected crc 2023-09-23T19:38:43.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:42 smithi016 bash[75004]: cluster 2023-09-23T19:38:41.858152+0000 osd.4 (osd.4) 86 : cluster 3 failed to encode map e158 with expected crc 2023-09-23T19:38:43.302 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:43.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:43.306 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:43.407 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:43.439 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:43.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:43.441 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:38:43.542 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:43.574 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:43.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:43.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:43.678 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:43.706 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:43.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:43.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:43.811 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:43.842 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:43.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:43.846 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:43.947 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:43.971 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:43.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:43.973 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:44.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:43 smithi092 bash[58855]: cluster 2023-09-23T19:38:42.118276+0000 mgr.y (mgr.25325) 936 : cluster 0 pgmap v865: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 31522/249906 objects degraded (12.614%) 2023-09-23T19:38:44.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:43 smithi092 bash[58855]: cluster 2023-09-23T19:38:42.843074+0000 mon.a (mon.0) 1157 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T19:38:44.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:43 smithi092 bash[58855]: cluster 2023-09-23T19:38:42.848848+0000 mon.a (mon.0) 1158 : cluster 1 osd.2 [v2:172.21.15.16:6818/4244964590,v1:172.21.15.16:6819/4244964590] boot 2023-09-23T19:38:44.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:43 smithi092 bash[58855]: cluster 2023-09-23T19:38:42.848916+0000 mon.a (mon.0) 1159 : cluster 0 osdmap e159: 8 total, 8 up, 8 in 2023-09-23T19:38:44.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:43 smithi092 bash[58855]: audit 2023-09-23T19:38:42.850171+0000 mon.a (mon.0) 1160 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T19:38:44.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:43 smithi092 bash[58855]: cluster 2023-09-23T19:38:42.850860+0000 osd.4 (osd.4) 87 : cluster 3 failed to encode map e159 with expected crc 2023-09-23T19:38:44.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:43 smithi092 bash[58855]: cluster 2023-09-23T19:38:42.858902+0000 osd.5 (osd.5) 88 : cluster 3 failed to encode map e159 with expected crc 2023-09-23T19:38:44.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:43 smithi092 bash[58855]: cluster 2023-09-23T19:38:42.861763+0000 osd.6 (osd.6) 77 : cluster 3 failed to encode map e159 with expected crc 2023-09-23T19:38:44.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:43 smithi092 bash[58855]: cluster 2023-09-23T19:38:42.868731+0000 osd.6 (osd.6) 78 : cluster 3 failed to encode map e159 with expected crc 2023-09-23T19:38:44.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:43 smithi092 bash[58855]: audit 2023-09-23T19:38:43.659460+0000 mon.a (mon.0) 1161 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:38:44.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:43 smithi092 bash[58855]: audit 2023-09-23T19:38:43.660648+0000 mon.a (mon.0) 1162 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:38:44.073 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:44.098 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:44.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:44.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:38:44.201 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:44.230 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:44.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:44.233 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:44.267 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:43 smithi016 bash[75004]: cluster 2023-09-23T19:38:42.118276+0000 mgr.y (mgr.25325) 936 : cluster 0 pgmap v865: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 31522/249906 objects degraded (12.614%) 2023-09-23T19:38:44.267 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:43 smithi016 bash[75004]: cluster 2023-09-23T19:38:42.843074+0000 mon.a (mon.0) 1157 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T19:38:44.267 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:43 smithi016 bash[75004]: cluster 2023-09-23T19:38:42.848848+0000 mon.a (mon.0) 1158 : cluster 1 osd.2 [v2:172.21.15.16:6818/4244964590,v1:172.21.15.16:6819/4244964590] boot 2023-09-23T19:38:44.267 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:43 smithi016 bash[75004]: cluster 2023-09-23T19:38:42.848916+0000 mon.a (mon.0) 1159 : cluster 0 osdmap e159: 8 total, 8 up, 8 in 2023-09-23T19:38:44.267 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:43 smithi016 bash[75004]: audit 2023-09-23T19:38:42.850171+0000 mon.a (mon.0) 1160 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T19:38:44.267 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:43 smithi016 bash[75004]: cluster 2023-09-23T19:38:42.850860+0000 osd.4 (osd.4) 87 : cluster 3 failed to encode map e159 with expected crc 2023-09-23T19:38:44.267 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:43 smithi016 bash[75004]: cluster 2023-09-23T19:38:42.858902+0000 osd.5 (osd.5) 88 : cluster 3 failed to encode map e159 with expected crc 2023-09-23T19:38:44.267 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:43 smithi016 bash[75004]: cluster 2023-09-23T19:38:42.861763+0000 osd.6 (osd.6) 77 : cluster 3 failed to encode map e159 with expected crc 2023-09-23T19:38:44.268 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:43 smithi016 bash[75004]: cluster 2023-09-23T19:38:42.868731+0000 osd.6 (osd.6) 78 : cluster 3 failed to encode map e159 with expected crc 2023-09-23T19:38:44.268 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:43 smithi016 bash[75004]: audit 2023-09-23T19:38:43.659460+0000 mon.a (mon.0) 1161 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:38:44.268 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:43 smithi016 bash[75004]: audit 2023-09-23T19:38:43.660648+0000 mon.a (mon.0) 1162 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:38:44.268 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:43 smithi016 bash[77995]: cluster 2023-09-23T19:38:42.118276+0000 mgr.y (mgr.25325) 936 : cluster 0 pgmap v865: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 31522/249906 objects degraded (12.614%) 2023-09-23T19:38:44.268 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:43 smithi016 bash[77995]: cluster 2023-09-23T19:38:42.843074+0000 mon.a (mon.0) 1157 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T19:38:44.268 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:43 smithi016 bash[77995]: cluster 2023-09-23T19:38:42.848848+0000 mon.a (mon.0) 1158 : cluster 1 osd.2 [v2:172.21.15.16:6818/4244964590,v1:172.21.15.16:6819/4244964590] boot 2023-09-23T19:38:44.268 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:43 smithi016 bash[77995]: cluster 2023-09-23T19:38:42.848916+0000 mon.a (mon.0) 1159 : cluster 0 osdmap e159: 8 total, 8 up, 8 in 2023-09-23T19:38:44.268 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:43 smithi016 bash[77995]: audit 2023-09-23T19:38:42.850171+0000 mon.a (mon.0) 1160 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T19:38:44.268 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:43 smithi016 bash[77995]: cluster 2023-09-23T19:38:42.850860+0000 osd.4 (osd.4) 87 : cluster 3 failed to encode map e159 with expected crc 2023-09-23T19:38:44.268 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:43 smithi016 bash[77995]: cluster 2023-09-23T19:38:42.858902+0000 osd.5 (osd.5) 88 : cluster 3 failed to encode map e159 with expected crc 2023-09-23T19:38:44.269 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:43 smithi016 bash[77995]: cluster 2023-09-23T19:38:42.861763+0000 osd.6 (osd.6) 77 : cluster 3 failed to encode map e159 with expected crc 2023-09-23T19:38:44.269 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:43 smithi016 bash[77995]: cluster 2023-09-23T19:38:42.868731+0000 osd.6 (osd.6) 78 : cluster 3 failed to encode map e159 with expected crc 2023-09-23T19:38:44.269 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:43 smithi016 bash[77995]: audit 2023-09-23T19:38:43.659460+0000 mon.a (mon.0) 1161 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:38:44.269 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:43 smithi016 bash[77995]: audit 2023-09-23T19:38:43.660648+0000 mon.a (mon.0) 1162 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:38:44.334 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:44.366 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:44.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:44.369 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:44.470 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:44.505 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:44.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:44.508 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:44.609 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:44.638 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:44.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:44.641 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:44.741 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:44.764 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-09-23T19:38:44.764 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:38:44.774 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:44.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:44.779 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:44.880 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:44.907 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:44.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:44.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:45.010 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:45.036 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:45.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:44 smithi092 bash[58855]: cluster 2023-09-23T19:38:43.863749+0000 mon.a (mon.0) 1163 : cluster 0 osdmap e160: 8 total, 8 up, 8 in 2023-09-23T19:38:45.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:44 smithi092 bash[58855]: cluster 2023-09-23T19:38:43.870904+0000 osd.5 (osd.5) 89 : cluster 3 failed to encode map e160 with expected crc 2023-09-23T19:38:45.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:44 smithi092 bash[58855]: cluster 2023-09-23T19:38:43.870988+0000 osd.4 (osd.4) 88 : cluster 3 failed to encode map e160 with expected crc 2023-09-23T19:38:45.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:44 smithi092 bash[58855]: cluster 2023-09-23T19:38:43.871201+0000 osd.6 (osd.6) 79 : cluster 3 failed to encode map e160 with expected crc 2023-09-23T19:38:45.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:44 smithi092 bash[58855]: cluster 2023-09-23T19:38:43.871236+0000 osd.7 (osd.7) 72 : cluster 3 failed to encode map e160 with expected crc 2023-09-23T19:38:45.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:44 smithi092 bash[58855]: cluster 2023-09-23T19:38:43.874610+0000 osd.5 (osd.5) 90 : cluster 3 failed to encode map e160 with expected crc 2023-09-23T19:38:45.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:44 smithi092 bash[58855]: cluster 2023-09-23T19:38:43.875638+0000 osd.6 (osd.6) 80 : cluster 3 failed to encode map e160 with expected crc 2023-09-23T19:38:45.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:44 smithi092 bash[58855]: cluster 2023-09-23T19:38:43.876784+0000 osd.7 (osd.7) 73 : cluster 3 failed to encode map e160 with expected crc 2023-09-23T19:38:45.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:44 smithi092 bash[58855]: cluster 2023-09-23T19:38:43.876979+0000 osd.4 (osd.4) 89 : cluster 3 failed to encode map e160 with expected crc 2023-09-23T19:38:45.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:44 smithi092 bash[58855]: cluster 2023-09-23T19:38:43.877814+0000 osd.5 (osd.5) 91 : cluster 3 failed to encode map e160 with expected crc 2023-09-23T19:38:45.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:44 smithi092 bash[58855]: cluster 2023-09-23T19:38:43.877941+0000 osd.6 (osd.6) 81 : cluster 3 failed to encode map e160 with expected crc 2023-09-23T19:38:45.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:44 smithi092 bash[58855]: cluster 2023-09-23T19:38:43.879269+0000 osd.4 (osd.4) 90 : cluster 3 failed to encode map e160 with expected crc 2023-09-23T19:38:45.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:44 smithi092 bash[58855]: cluster 2023-09-23T19:38:43.879782+0000 osd.5 (osd.5) 92 : cluster 3 failed to encode map e160 with expected crc 2023-09-23T19:38:45.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:44 smithi092 bash[58855]: cluster 2023-09-23T19:38:43.880507+0000 osd.4 (osd.4) 91 : cluster 3 failed to encode map e160 with expected crc 2023-09-23T19:38:45.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:44 smithi092 bash[58855]: cluster 2023-09-23T19:38:43.881292+0000 osd.7 (osd.7) 74 : cluster 3 failed to encode map e160 with expected crc 2023-09-23T19:38:45.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:44 smithi092 bash[58855]: cluster 2023-09-23T19:38:43.881728+0000 osd.4 (osd.4) 92 : cluster 3 failed to encode map e160 with expected crc 2023-09-23T19:38:45.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:44 smithi092 bash[58855]: cluster 2023-09-23T19:38:44.429212+0000 mon.a (mon.0) 1164 : cluster 3 Health check update: Degraded data redundancy: 31522/249906 objects degraded (12.614%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T19:38:45.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:44 smithi092 bash[58855]: cluster 2023-09-23T19:38:44.865933+0000 mon.a (mon.0) 1165 : cluster 0 osdmap e161: 8 total, 8 up, 8 in 2023-09-23T19:38:45.041 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:44 smithi016 bash[75004]: cluster 2023-09-23T19:38:43.863749+0000 mon.a (mon.0) 1163 : cluster 0 osdmap e160: 8 total, 8 up, 8 in 2023-09-23T19:38:45.041 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:44 smithi016 bash[75004]: cluster 2023-09-23T19:38:43.870904+0000 osd.5 (osd.5) 89 : cluster 3 failed to encode map e160 with expected crc 2023-09-23T19:38:45.041 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:44 smithi016 bash[75004]: cluster 2023-09-23T19:38:43.870988+0000 osd.4 (osd.4) 88 : cluster 3 failed to encode map e160 with expected crc 2023-09-23T19:38:45.041 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:44 smithi016 bash[75004]: cluster 2023-09-23T19:38:43.871201+0000 osd.6 (osd.6) 79 : cluster 3 failed to encode map e160 with expected crc 2023-09-23T19:38:45.041 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:44 smithi016 bash[75004]: cluster 2023-09-23T19:38:43.871236+0000 osd.7 (osd.7) 72 : cluster 3 failed to encode map e160 with expected crc 2023-09-23T19:38:45.041 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:44 smithi016 bash[75004]: cluster 2023-09-23T19:38:43.874610+0000 osd.5 (osd.5) 90 : cluster 3 failed to encode map e160 with expected crc 2023-09-23T19:38:45.042 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:44 smithi016 bash[75004]: cluster 2023-09-23T19:38:43.875638+0000 osd.6 (osd.6) 80 : cluster 3 failed to encode map e160 with expected crc 2023-09-23T19:38:45.042 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:44 smithi016 bash[75004]: cluster 2023-09-23T19:38:43.876784+0000 osd.7 (osd.7) 73 : cluster 3 failed to encode map e160 with expected crc 2023-09-23T19:38:45.042 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:44 smithi016 bash[75004]: cluster 2023-09-23T19:38:43.876979+0000 osd.4 (osd.4) 89 : cluster 3 failed to encode map e160 with expected crc 2023-09-23T19:38:45.042 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:44 smithi016 bash[75004]: cluster 2023-09-23T19:38:43.877814+0000 osd.5 (osd.5) 91 : cluster 3 failed to encode map e160 with expected crc 2023-09-23T19:38:45.042 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:44 smithi016 bash[75004]: cluster 2023-09-23T19:38:43.877941+0000 osd.6 (osd.6) 81 : cluster 3 failed to encode map e160 with expected crc 2023-09-23T19:38:45.042 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:44 smithi016 bash[75004]: cluster 2023-09-23T19:38:43.879269+0000 osd.4 (osd.4) 90 : cluster 3 failed to encode map e160 with expected crc 2023-09-23T19:38:45.043 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:44 smithi016 bash[75004]: cluster 2023-09-23T19:38:43.879782+0000 osd.5 (osd.5) 92 : cluster 3 failed to encode map e160 with expected crc 2023-09-23T19:38:45.043 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:44 smithi016 bash[75004]: cluster 2023-09-23T19:38:43.880507+0000 osd.4 (osd.4) 91 : cluster 3 failed to encode map e160 with expected crc 2023-09-23T19:38:45.043 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:44 smithi016 bash[75004]: cluster 2023-09-23T19:38:43.881292+0000 osd.7 (osd.7) 74 : cluster 3 failed to encode map e160 with expected crc 2023-09-23T19:38:45.043 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:44 smithi016 bash[75004]: cluster 2023-09-23T19:38:43.881728+0000 osd.4 (osd.4) 92 : cluster 3 failed to encode map e160 with expected crc 2023-09-23T19:38:45.043 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:44 smithi016 bash[75004]: cluster 2023-09-23T19:38:44.429212+0000 mon.a (mon.0) 1164 : cluster 3 Health check update: Degraded data redundancy: 31522/249906 objects degraded (12.614%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T19:38:45.043 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:44 smithi016 bash[75004]: cluster 2023-09-23T19:38:44.865933+0000 mon.a (mon.0) 1165 : cluster 0 osdmap e161: 8 total, 8 up, 8 in 2023-09-23T19:38:45.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:45.044 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:45.045 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:44 smithi016 bash[77995]: cluster 2023-09-23T19:38:43.863749+0000 mon.a (mon.0) 1163 : cluster 0 osdmap e160: 8 total, 8 up, 8 in 2023-09-23T19:38:45.045 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:44 smithi016 bash[77995]: cluster 2023-09-23T19:38:43.870904+0000 osd.5 (osd.5) 89 : cluster 3 failed to encode map e160 with expected crc 2023-09-23T19:38:45.045 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:44 smithi016 bash[77995]: cluster 2023-09-23T19:38:43.870988+0000 osd.4 (osd.4) 88 : cluster 3 failed to encode map e160 with expected crc 2023-09-23T19:38:45.045 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:44 smithi016 bash[77995]: cluster 2023-09-23T19:38:43.871201+0000 osd.6 (osd.6) 79 : cluster 3 failed to encode map e160 with expected crc 2023-09-23T19:38:45.045 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:44 smithi016 bash[77995]: cluster 2023-09-23T19:38:43.871236+0000 osd.7 (osd.7) 72 : cluster 3 failed to encode map e160 with expected crc 2023-09-23T19:38:45.045 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:44 smithi016 bash[77995]: cluster 2023-09-23T19:38:43.874610+0000 osd.5 (osd.5) 90 : cluster 3 failed to encode map e160 with expected crc 2023-09-23T19:38:45.045 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:44 smithi016 bash[77995]: cluster 2023-09-23T19:38:43.875638+0000 osd.6 (osd.6) 80 : cluster 3 failed to encode map e160 with expected crc 2023-09-23T19:38:45.045 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:44 smithi016 bash[77995]: cluster 2023-09-23T19:38:43.876784+0000 osd.7 (osd.7) 73 : cluster 3 failed to encode map e160 with expected crc 2023-09-23T19:38:45.045 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:44 smithi016 bash[77995]: cluster 2023-09-23T19:38:43.876979+0000 osd.4 (osd.4) 89 : cluster 3 failed to encode map e160 with expected crc 2023-09-23T19:38:45.045 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:44 smithi016 bash[77995]: cluster 2023-09-23T19:38:43.877814+0000 osd.5 (osd.5) 91 : cluster 3 failed to encode map e160 with expected crc 2023-09-23T19:38:45.046 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:44 smithi016 bash[77995]: cluster 2023-09-23T19:38:43.877941+0000 osd.6 (osd.6) 81 : cluster 3 failed to encode map e160 with expected crc 2023-09-23T19:38:45.046 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:44 smithi016 bash[77995]: cluster 2023-09-23T19:38:43.879269+0000 osd.4 (osd.4) 90 : cluster 3 failed to encode map e160 with expected crc 2023-09-23T19:38:45.046 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:44 smithi016 bash[77995]: cluster 2023-09-23T19:38:43.879782+0000 osd.5 (osd.5) 92 : cluster 3 failed to encode map e160 with expected crc 2023-09-23T19:38:45.046 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:44 smithi016 bash[77995]: cluster 2023-09-23T19:38:43.880507+0000 osd.4 (osd.4) 91 : cluster 3 failed to encode map e160 with expected crc 2023-09-23T19:38:45.046 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:44 smithi016 bash[77995]: cluster 2023-09-23T19:38:43.881292+0000 osd.7 (osd.7) 74 : cluster 3 failed to encode map e160 with expected crc 2023-09-23T19:38:45.046 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:44 smithi016 bash[77995]: cluster 2023-09-23T19:38:43.881728+0000 osd.4 (osd.4) 92 : cluster 3 failed to encode map e160 with expected crc 2023-09-23T19:38:45.046 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:44 smithi016 bash[77995]: cluster 2023-09-23T19:38:44.429212+0000 mon.a (mon.0) 1164 : cluster 3 Health check update: Degraded data redundancy: 31522/249906 objects degraded (12.614%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T19:38:45.046 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:44 smithi016 bash[77995]: cluster 2023-09-23T19:38:44.865933+0000 mon.a (mon.0) 1165 : cluster 0 osdmap e161: 8 total, 8 up, 8 in 2023-09-23T19:38:45.145 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:45.172 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:45.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:45.177 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:45.278 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:45.300 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:45.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:45.306 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:45.407 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:45.447 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:45.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:45.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:45.552 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:45.584 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:45.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:45.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:45.644 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:38:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:38:45] "GET /metrics HTTP/1.1" 200 36163 "" "Prometheus/2.33.4" 2023-09-23T19:38:45.690 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:45.725 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:45.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:45.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:45.829 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:45.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:45.867 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:45.867 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:45.894 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:45 smithi016 bash[75004]: cluster 2023-09-23T19:38:44.119010+0000 mgr.y (mgr.25325) 937 : cluster 0 pgmap v868: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 31044/247848 objects degraded (12.525%) 2023-09-23T19:38:45.894 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:45 smithi016 bash[75004]: cluster 2023-09-23T19:38:44.868602+0000 osd.4 (osd.4) 93 : cluster 3 failed to encode map e161 with expected crc 2023-09-23T19:38:45.895 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:45 smithi016 bash[75004]: cluster 2023-09-23T19:38:44.871905+0000 osd.4 (osd.4) 94 : cluster 3 failed to encode map e161 with expected crc 2023-09-23T19:38:45.895 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:45 smithi016 bash[75004]: cluster 2023-09-23T19:38:44.872223+0000 osd.5 (osd.5) 93 : cluster 3 failed to encode map e161 with expected crc 2023-09-23T19:38:45.895 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:45 smithi016 bash[75004]: cluster 2023-09-23T19:38:44.872348+0000 osd.6 (osd.6) 82 : cluster 3 failed to encode map e161 with expected crc 2023-09-23T19:38:45.895 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:45 smithi016 bash[75004]: cluster 2023-09-23T19:38:44.877709+0000 osd.6 (osd.6) 83 : cluster 3 failed to encode map e161 with expected crc 2023-09-23T19:38:45.969 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:46.003 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:46.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:46.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:46.110 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:46.118 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:45 smithi092 bash[58855]: cluster 2023-09-23T19:38:44.119010+0000 mgr.y (mgr.25325) 937 : cluster 0 pgmap v868: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 31044/247848 objects degraded (12.525%) 2023-09-23T19:38:46.118 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:45 smithi092 bash[58855]: cluster 2023-09-23T19:38:44.868602+0000 osd.4 (osd.4) 93 : cluster 3 failed to encode map e161 with expected crc 2023-09-23T19:38:46.118 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:45 smithi092 bash[58855]: cluster 2023-09-23T19:38:44.871905+0000 osd.4 (osd.4) 94 : cluster 3 failed to encode map e161 with expected crc 2023-09-23T19:38:46.118 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:45 smithi092 bash[58855]: cluster 2023-09-23T19:38:44.872223+0000 osd.5 (osd.5) 93 : cluster 3 failed to encode map e161 with expected crc 2023-09-23T19:38:46.118 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:45 smithi092 bash[58855]: cluster 2023-09-23T19:38:44.872348+0000 osd.6 (osd.6) 82 : cluster 3 failed to encode map e161 with expected crc 2023-09-23T19:38:46.119 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:45 smithi092 bash[58855]: cluster 2023-09-23T19:38:44.877709+0000 osd.6 (osd.6) 83 : cluster 3 failed to encode map e161 with expected crc 2023-09-23T19:38:46.141 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:46.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:46.143 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:46.245 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:46.275 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:46.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:46.278 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:46.290 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:45 smithi016 bash[77995]: cluster 2023-09-23T19:38:44.119010+0000 mgr.y (mgr.25325) 937 : cluster 0 pgmap v868: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 31044/247848 objects degraded (12.525%) 2023-09-23T19:38:46.290 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:45 smithi016 bash[77995]: cluster 2023-09-23T19:38:44.868602+0000 osd.4 (osd.4) 93 : cluster 3 failed to encode map e161 with expected crc 2023-09-23T19:38:46.290 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:45 smithi016 bash[77995]: cluster 2023-09-23T19:38:44.871905+0000 osd.4 (osd.4) 94 : cluster 3 failed to encode map e161 with expected crc 2023-09-23T19:38:46.291 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:45 smithi016 bash[77995]: cluster 2023-09-23T19:38:44.872223+0000 osd.5 (osd.5) 93 : cluster 3 failed to encode map e161 with expected crc 2023-09-23T19:38:46.291 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:45 smithi016 bash[77995]: cluster 2023-09-23T19:38:44.872348+0000 osd.6 (osd.6) 82 : cluster 3 failed to encode map e161 with expected crc 2023-09-23T19:38:46.291 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:45 smithi016 bash[77995]: cluster 2023-09-23T19:38:44.877709+0000 osd.6 (osd.6) 83 : cluster 3 failed to encode map e161 with expected crc 2023-09-23T19:38:46.378 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:46.414 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:46.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:46.417 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:46.517 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:46.553 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:46.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:46.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:46.660 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:46.688 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:46.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:46.690 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:46.791 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:46.821 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:46.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:46.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:46.926 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:46.956 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:46.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:46.963 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:47.064 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:47.100 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:47.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:47.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:47.207 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:47.240 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:47.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:47.242 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:47.246 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T19:38:47.343 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:47.371 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:47.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:47.376 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:38:47.477 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:47.503 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:47.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:47.510 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:38:47.611 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:47.637 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:47.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:47.642 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:47.742 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:47.768 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:47.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:47.772 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:47.872 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:47.896 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:47.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:47.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:47.935 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T19:38:48.003 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:48.037 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:48.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:48.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:48.140 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:48.150 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:47 smithi092 bash[58855]: cluster 2023-09-23T19:38:46.119757+0000 mgr.y (mgr.25325) 938 : cluster 0 pgmap v870: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 4 active+undersized+degraded, 5 active+undersized, 77 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 538 op/s; 18172/242994 objects degraded (7.478%); 0 B/s, 0 objects/s recovering 2023-09-23T19:38:48.172 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:48.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:48.175 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:48.276 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:48.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:47 smithi016 bash[75004]: cluster 2023-09-23T19:38:46.119757+0000 mgr.y (mgr.25325) 938 : cluster 0 pgmap v870: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 4 active+undersized+degraded, 5 active+undersized, 77 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 538 op/s; 18172/242994 objects degraded (7.478%); 0 B/s, 0 objects/s recovering 2023-09-23T19:38:48.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:47 smithi016 bash[77995]: cluster 2023-09-23T19:38:46.119757+0000 mgr.y (mgr.25325) 938 : cluster 0 pgmap v870: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 4 active+undersized+degraded, 5 active+undersized, 77 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 538 op/s; 18172/242994 objects degraded (7.478%); 0 B/s, 0 objects/s recovering 2023-09-23T19:38:48.307 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:48.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:48.320 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:48.421 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:48.460 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:48.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:48.468 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:48.569 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:48.594 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:48.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:48.600 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:48.700 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:48.735 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:48.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:48.740 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:48.840 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:48.865 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:48.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:48.868 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:48.969 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:48.991 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:48.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:48.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:38:49.094 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:49.116 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:49.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:49.120 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:49.184 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:38:49.221 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:49.231 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:38:49.263 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:49.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:49.267 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:49.304 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:38:49.364 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:38:49.368 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:49.396 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:49.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:49.400 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:49.403 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:38:49.442 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:38:49.478 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:38:49.501 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:49.523 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:38:49.531 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:38:49.531 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:38:49.531 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:38:49.531 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:38:49.531 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:38:49.531 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:38:49.531 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:38:49.531 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:38:49.541 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:49.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:49.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:49.651 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:49.679 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:49.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:49.681 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:49.782 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:49.814 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:49.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:49.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:49.919 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:49.946 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:49.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:49.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:38:50.051 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:50.083 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:50.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:50.088 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:50.121 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:49 smithi016 bash[77995]: cluster 2023-09-23T19:38:48.120454+0000 mgr.y (mgr.25325) 939 : cluster 0 pgmap v871: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 83 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 477 op/s; 5636/241305 objects degraded (2.336%); 0 B/s, 25 objects/s recovering 2023-09-23T19:38:50.121 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:49 smithi016 bash[77995]: audit 2023-09-23T19:38:49.137437+0000 mon.a (mon.0) 1166 : audit 0 from='client.? 172.21.15.16:0/678276222' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:38:50.122 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:49 smithi016 bash[77995]: cluster 2023-09-23T19:38:49.430289+0000 mon.a (mon.0) 1167 : cluster 3 Health check update: Degraded data redundancy: 5636/241305 objects degraded (2.336%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T19:38:50.122 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:49 smithi016 bash[77995]: audit 2023-09-23T19:38:49.604474+0000 mon.a (mon.0) 1168 : audit 1 from='client.? 172.21.15.16:0/3222922643' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:38:50.122 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:49 smithi016 bash[75004]: cluster 2023-09-23T19:38:48.120454+0000 mgr.y (mgr.25325) 939 : cluster 0 pgmap v871: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 83 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 477 op/s; 5636/241305 objects degraded (2.336%); 0 B/s, 25 objects/s recovering 2023-09-23T19:38:50.122 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:49 smithi016 bash[75004]: audit 2023-09-23T19:38:49.137437+0000 mon.a (mon.0) 1166 : audit 0 from='client.? 172.21.15.16:0/678276222' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:38:50.122 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:49 smithi016 bash[75004]: cluster 2023-09-23T19:38:49.430289+0000 mon.a (mon.0) 1167 : cluster 3 Health check update: Degraded data redundancy: 5636/241305 objects degraded (2.336%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T19:38:50.122 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:49 smithi016 bash[75004]: audit 2023-09-23T19:38:49.604474+0000 mon.a (mon.0) 1168 : audit 1 from='client.? 172.21.15.16:0/3222922643' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:38:50.189 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:50.198 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:49 smithi092 bash[58855]: cluster 2023-09-23T19:38:48.120454+0000 mgr.y (mgr.25325) 939 : cluster 0 pgmap v871: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 83 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 477 op/s; 5636/241305 objects degraded (2.336%); 0 B/s, 25 objects/s recovering 2023-09-23T19:38:50.198 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:49 smithi092 bash[58855]: audit 2023-09-23T19:38:49.137437+0000 mon.a (mon.0) 1166 : audit 0 from='client.? 172.21.15.16:0/678276222' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:38:50.198 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:49 smithi092 bash[58855]: cluster 2023-09-23T19:38:49.430289+0000 mon.a (mon.0) 1167 : cluster 3 Health check update: Degraded data redundancy: 5636/241305 objects degraded (2.336%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T19:38:50.199 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:49 smithi092 bash[58855]: audit 2023-09-23T19:38:49.604474+0000 mon.a (mon.0) 1168 : audit 1 from='client.? 172.21.15.16:0/3222922643' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:38:50.244 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:50.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:50.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:50.352 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:50.388 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:50.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:50.394 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:38:50.450 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is unset 2023-09-23T19:38:50.494 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:50.534 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:50.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:50.536 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:50.637 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:50.660 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:50.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:50.666 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:50.767 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:50.797 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:50.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:50.801 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:50.902 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:50.928 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:50.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:50.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:51.034 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:51.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:50 smithi016 bash[75004]: cluster 2023-09-23T19:38:50.435763+0000 mon.a (mon.0) 1169 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:38:51.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:50 smithi016 bash[75004]: audit 2023-09-23T19:38:50.442671+0000 mon.a (mon.0) 1170 : audit 1 from='client.? 172.21.15.16:0/3222922643' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:38:51.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:50 smithi016 bash[75004]: cluster 2023-09-23T19:38:50.442745+0000 mon.a (mon.0) 1171 : cluster 0 osdmap e162: 8 total, 8 up, 8 in 2023-09-23T19:38:51.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:50 smithi016 bash[75004]: cluster 2023-09-23T19:38:50.474086+0000 osd.7 (osd.7) 75 : cluster 3 failed to encode map e162 with expected crc 2023-09-23T19:38:51.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:50 smithi016 bash[77995]: cluster 2023-09-23T19:38:50.435763+0000 mon.a (mon.0) 1169 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:38:51.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:50 smithi016 bash[77995]: audit 2023-09-23T19:38:50.442671+0000 mon.a (mon.0) 1170 : audit 1 from='client.? 172.21.15.16:0/3222922643' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:38:51.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:50 smithi016 bash[77995]: cluster 2023-09-23T19:38:50.442745+0000 mon.a (mon.0) 1171 : cluster 0 osdmap e162: 8 total, 8 up, 8 in 2023-09-23T19:38:51.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:50 smithi016 bash[77995]: cluster 2023-09-23T19:38:50.474086+0000 osd.7 (osd.7) 75 : cluster 3 failed to encode map e162 with expected crc 2023-09-23T19:38:51.073 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:51.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:51.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:51.177 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:51.209 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:51.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:51.215 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:38:51.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:50 smithi092 bash[58855]: cluster 2023-09-23T19:38:50.435763+0000 mon.a (mon.0) 1169 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:38:51.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:50 smithi092 bash[58855]: audit 2023-09-23T19:38:50.442671+0000 mon.a (mon.0) 1170 : audit 1 from='client.? 172.21.15.16:0/3222922643' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:38:51.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:50 smithi092 bash[58855]: cluster 2023-09-23T19:38:50.442745+0000 mon.a (mon.0) 1171 : cluster 0 osdmap e162: 8 total, 8 up, 8 in 2023-09-23T19:38:51.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:50 smithi092 bash[58855]: cluster 2023-09-23T19:38:50.474086+0000 osd.7 (osd.7) 75 : cluster 3 failed to encode map e162 with expected crc 2023-09-23T19:38:51.315 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:51.354 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:51.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:51.356 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:51.457 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:51.481 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:51.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:51.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:38:51.586 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:51.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:51.613 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:51.614 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:51.714 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:51.739 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:51.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:51.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:51.850 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:51.876 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:51.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:51.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:51.982 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:52.009 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:52.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:52.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:52.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:51 smithi016 bash[75004]: cluster 2023-09-23T19:38:50.121065+0000 mgr.y (mgr.25325) 940 : cluster 0 pgmap v872: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 7083/237747 objects degraded (2.979%); 0 B/s, 21 objects/s recovering 2023-09-23T19:38:52.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:51 smithi016 bash[75004]: cluster 2023-09-23T19:38:50.473504+0000 osd.4 (osd.4) 95 : cluster 3 failed to encode map e162 with expected crc 2023-09-23T19:38:52.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:51 smithi016 bash[75004]: cluster 2023-09-23T19:38:50.481287+0000 osd.5 (osd.5) 94 : cluster 3 failed to encode map e162 with expected crc 2023-09-23T19:38:52.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:51 smithi016 bash[75004]: cluster 2023-09-23T19:38:50.483740+0000 osd.5 (osd.5) 95 : cluster 3 failed to encode map e162 with expected crc 2023-09-23T19:38:52.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:51 smithi016 bash[75004]: cluster 2023-09-23T19:38:50.485094+0000 osd.6 (osd.6) 84 : cluster 3 failed to encode map e162 with expected crc 2023-09-23T19:38:52.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:51 smithi016 bash[77995]: cluster 2023-09-23T19:38:50.121065+0000 mgr.y (mgr.25325) 940 : cluster 0 pgmap v872: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 7083/237747 objects degraded (2.979%); 0 B/s, 21 objects/s recovering 2023-09-23T19:38:52.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:51 smithi016 bash[77995]: cluster 2023-09-23T19:38:50.473504+0000 osd.4 (osd.4) 95 : cluster 3 failed to encode map e162 with expected crc 2023-09-23T19:38:52.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:51 smithi016 bash[77995]: cluster 2023-09-23T19:38:50.481287+0000 osd.5 (osd.5) 94 : cluster 3 failed to encode map e162 with expected crc 2023-09-23T19:38:52.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:51 smithi016 bash[77995]: cluster 2023-09-23T19:38:50.483740+0000 osd.5 (osd.5) 95 : cluster 3 failed to encode map e162 with expected crc 2023-09-23T19:38:52.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:51 smithi016 bash[77995]: cluster 2023-09-23T19:38:50.485094+0000 osd.6 (osd.6) 84 : cluster 3 failed to encode map e162 with expected crc 2023-09-23T19:38:52.113 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:52.122 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:51 smithi092 bash[58855]: cluster 2023-09-23T19:38:50.121065+0000 mgr.y (mgr.25325) 940 : cluster 0 pgmap v872: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 7083/237747 objects degraded (2.979%); 0 B/s, 21 objects/s recovering 2023-09-23T19:38:52.123 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:51 smithi092 bash[58855]: cluster 2023-09-23T19:38:50.473504+0000 osd.4 (osd.4) 95 : cluster 3 failed to encode map e162 with expected crc 2023-09-23T19:38:52.123 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:51 smithi092 bash[58855]: cluster 2023-09-23T19:38:50.481287+0000 osd.5 (osd.5) 94 : cluster 3 failed to encode map e162 with expected crc 2023-09-23T19:38:52.123 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:51 smithi092 bash[58855]: cluster 2023-09-23T19:38:50.483740+0000 osd.5 (osd.5) 95 : cluster 3 failed to encode map e162 with expected crc 2023-09-23T19:38:52.123 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:51 smithi092 bash[58855]: cluster 2023-09-23T19:38:50.485094+0000 osd.6 (osd.6) 84 : cluster 3 failed to encode map e162 with expected crc 2023-09-23T19:38:52.147 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:52.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:52.149 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:38:52.250 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:52.294 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:52.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:52.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:52.397 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:52.420 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:52.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:52.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:52.522 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:52.562 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:52.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:52.568 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:52.669 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:52.706 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:52.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:52.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:52.810 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:52.849 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:52.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:52.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:52.952 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:52.981 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:52.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:52.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:53.044 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:38:53.087 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:53.121 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:53.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:53.126 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:53.226 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:53.251 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:53.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:53.257 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:53.358 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:53.388 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:53.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:53.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:53.494 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:53.528 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:53.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:53.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:53.633 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:53.670 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:53.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:53.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:53.729 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T19:38:53.774 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:53.850 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:53.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:53.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:53.964 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:53.990 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:53.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:53.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:54.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:53 smithi016 bash[77995]: cluster 2023-09-23T19:38:52.121701+0000 mgr.y (mgr.25325) 941 : cluster 0 pgmap v874: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 7083/233409 objects degraded (3.035%); 0 B/s, 19 objects/s recovering 2023-09-23T19:38:54.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:53 smithi016 bash[75004]: cluster 2023-09-23T19:38:52.121701+0000 mgr.y (mgr.25325) 941 : cluster 0 pgmap v874: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 7083/233409 objects degraded (3.035%); 0 B/s, 19 objects/s recovering 2023-09-23T19:38:54.100 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:54.113 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:53 smithi092 bash[58855]: cluster 2023-09-23T19:38:52.121701+0000 mgr.y (mgr.25325) 941 : cluster 0 pgmap v874: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 7083/233409 objects degraded (3.035%); 0 B/s, 19 objects/s recovering 2023-09-23T19:38:54.128 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:54.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:54.135 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:54.231 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:38:54.233 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:38:54.235 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:54.262 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:54.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:54.264 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:38:54.365 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:54.408 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:54.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:54.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:54.512 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:54.567 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:54.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:54.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:54.672 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:54.716 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:54.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:54.720 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:54.821 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:54.853 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:54.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:54.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:38:54.931 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:54 smithi092 bash[58855]: audit 2023-09-23T19:38:54.264956+0000 mon.a (mon.0) 1172 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:38:54.932 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:54 smithi092 bash[58855]: cluster 2023-09-23T19:38:54.431526+0000 mon.a (mon.0) 1173 : cluster 3 Health check update: Degraded data redundancy: 7083/233409 objects degraded (3.035%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T19:38:54.932 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:54 smithi092 bash[58855]: audit 2023-09-23T19:38:54.794204+0000 mon.a (mon.0) 1174 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:38:54.932 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:54 smithi092 bash[58855]: audit 2023-09-23T19:38:54.803655+0000 mon.a (mon.0) 1175 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:38:54.932 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:54 smithi092 bash[58855]: audit 2023-09-23T19:38:54.817791+0000 mon.a (mon.0) 1176 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:38:54.932 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:54 smithi092 bash[58855]: audit 2023-09-23T19:38:54.834882+0000 mon.a (mon.0) 1177 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:38:54.959 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:54.997 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:55.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:55.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:38:55.103 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:55.134 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:55.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:55.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:55.242 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:55.277 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:55.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:55.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:55.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:54 smithi016 bash[75004]: audit 2023-09-23T19:38:54.264956+0000 mon.a (mon.0) 1172 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:38:55.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:54 smithi016 bash[75004]: cluster 2023-09-23T19:38:54.431526+0000 mon.a (mon.0) 1173 : cluster 3 Health check update: Degraded data redundancy: 7083/233409 objects degraded (3.035%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T19:38:55.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:54 smithi016 bash[75004]: audit 2023-09-23T19:38:54.794204+0000 mon.a (mon.0) 1174 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:38:55.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:54 smithi016 bash[75004]: audit 2023-09-23T19:38:54.803655+0000 mon.a (mon.0) 1175 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:38:55.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:54 smithi016 bash[75004]: audit 2023-09-23T19:38:54.817791+0000 mon.a (mon.0) 1176 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:38:55.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:54 smithi016 bash[75004]: audit 2023-09-23T19:38:54.834882+0000 mon.a (mon.0) 1177 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:38:55.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:54 smithi016 bash[77995]: audit 2023-09-23T19:38:54.264956+0000 mon.a (mon.0) 1172 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:38:55.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:54 smithi016 bash[77995]: cluster 2023-09-23T19:38:54.431526+0000 mon.a (mon.0) 1173 : cluster 3 Health check update: Degraded data redundancy: 7083/233409 objects degraded (3.035%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T19:38:55.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:54 smithi016 bash[77995]: audit 2023-09-23T19:38:54.794204+0000 mon.a (mon.0) 1174 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:38:55.288 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:54 smithi016 bash[77995]: audit 2023-09-23T19:38:54.803655+0000 mon.a (mon.0) 1175 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:38:55.288 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:54 smithi016 bash[77995]: audit 2023-09-23T19:38:54.817791+0000 mon.a (mon.0) 1176 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:38:55.288 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:54 smithi016 bash[77995]: audit 2023-09-23T19:38:54.834882+0000 mon.a (mon.0) 1177 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:38:55.380 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:55.413 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:55.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:55.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:55.517 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:55.540 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:55.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:55.544 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:55.645 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:55.683 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:55.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:55.689 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:38:55.790 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:55.822 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:55.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:55.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:38:55.927 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:55.944 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:38:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:38:55] "GET /metrics HTTP/1.1" 200 36163 "" "Prometheus/2.33.4" 2023-09-23T19:38:55.945 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:55 smithi016 bash[77995]: cluster 2023-09-23T19:38:54.122615+0000 mgr.y (mgr.25325) 942 : cluster 0 pgmap v875: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 493 op/s; 7083/231591 objects degraded (3.058%); 0 B/s, 41 objects/s recovering 2023-09-23T19:38:55.946 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:55 smithi016 bash[75004]: cluster 2023-09-23T19:38:54.122615+0000 mgr.y (mgr.25325) 942 : cluster 0 pgmap v875: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 493 op/s; 7083/231591 objects degraded (3.058%); 0 B/s, 41 objects/s recovering 2023-09-23T19:38:55.954 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:55.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:55.956 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:56.057 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:56.088 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:56.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:56.095 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:56.196 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:56.237 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:56.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:56.241 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:56.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:55 smithi092 bash[58855]: cluster 2023-09-23T19:38:54.122615+0000 mgr.y (mgr.25325) 942 : cluster 0 pgmap v875: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 493 op/s; 7083/231591 objects degraded (3.058%); 0 B/s, 41 objects/s recovering 2023-09-23T19:38:56.342 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:56.380 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:56.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:56.389 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:56.490 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:56.522 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:56.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:56.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:56.624 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:56.649 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:56.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:56.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:56.752 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:56.800 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:56.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:56.811 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:56.912 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:56.946 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:56.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:56.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:57.050 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:57.094 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:57.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:57.105 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:57.206 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:57.245 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:57.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:57.252 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:57.297 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:38:57.352 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:57.380 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:57.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:57.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:38:57.485 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:57.508 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:57.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:57.511 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:57.611 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:57.647 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:57.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:57.654 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:57.755 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:57.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:57.795 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:38:57.795 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:38:57.895 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:57.927 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:57.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:57.931 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:58.032 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:58.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:57 smithi092 bash[58855]: cluster 2023-09-23T19:38:56.123417+0000 mgr.y (mgr.25325) 943 : cluster 0 pgmap v876: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 7083/224058 objects degraded (3.161%); 0 B/s, 38 objects/s recovering 2023-09-23T19:38:58.061 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:58.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:58.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:58.165 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:58.192 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:58.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:58.195 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:58.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:57 smithi016 bash[77995]: cluster 2023-09-23T19:38:56.123417+0000 mgr.y (mgr.25325) 943 : cluster 0 pgmap v876: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 7083/224058 objects degraded (3.161%); 0 B/s, 38 objects/s recovering 2023-09-23T19:38:58.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:57 smithi016 bash[75004]: cluster 2023-09-23T19:38:56.123417+0000 mgr.y (mgr.25325) 943 : cluster 0 pgmap v876: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 7083/224058 objects degraded (3.161%); 0 B/s, 38 objects/s recovering 2023-09-23T19:38:58.295 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:58.332 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:58.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:58.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:58.440 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:58.464 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:58.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:58.469 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:58.570 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:58.606 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:58.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:58.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:58.712 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:58.737 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:58.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:58.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:38:58.843 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:58.874 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:58.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:58.877 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:38:58.978 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:58.986 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:58 smithi092 bash[58855]: audit 2023-09-23T19:38:58.653941+0000 mon.a (mon.0) 1178 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:38:59.008 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:38:59.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:59.013 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:38:59.113 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:59.144 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:38:59.148 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:58 smithi016 bash[77995]: audit 2023-09-23T19:38:58.653941+0000 mon.a (mon.0) 1178 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:38:59.148 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:58 smithi016 bash[75004]: audit 2023-09-23T19:38:58.653941+0000 mon.a (mon.0) 1178 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:38:59.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:59.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:38:59.250 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:59.277 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:38:59.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:59.280 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:38:59.380 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:59.417 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:59.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:59.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:59.521 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:59.546 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:38:59.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:59.549 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:38:59.649 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:59.674 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:59.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:59.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:38:59.778 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:59.831 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:38:59.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:59.833 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:38:59.933 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:59.989 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:38:59.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:38:59.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:00.095 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:00.139 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:00.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:00.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:39:00.243 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:00.257 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:59 smithi092 bash[58855]: cluster 2023-09-23T19:38:58.124729+0000 mgr.y (mgr.25325) 944 : cluster 0 pgmap v877: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 7083/221382 objects degraded (3.199%); 0 B/s, 48 objects/s recovering 2023-09-23T19:39:00.257 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:38:59 smithi092 bash[58855]: cluster 2023-09-23T19:38:59.432714+0000 mon.a (mon.0) 1179 : cluster 3 Health check update: Degraded data redundancy: 7083/221382 objects degraded (3.199%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T19:39:00.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:59 smithi016 bash[77995]: cluster 2023-09-23T19:38:58.124729+0000 mgr.y (mgr.25325) 944 : cluster 0 pgmap v877: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 7083/221382 objects degraded (3.199%); 0 B/s, 48 objects/s recovering 2023-09-23T19:39:00.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:38:59 smithi016 bash[77995]: cluster 2023-09-23T19:38:59.432714+0000 mon.a (mon.0) 1179 : cluster 3 Health check update: Degraded data redundancy: 7083/221382 objects degraded (3.199%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T19:39:00.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:59 smithi016 bash[75004]: cluster 2023-09-23T19:38:58.124729+0000 mgr.y (mgr.25325) 944 : cluster 0 pgmap v877: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 7083/221382 objects degraded (3.199%); 0 B/s, 48 objects/s recovering 2023-09-23T19:39:00.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:38:59 smithi016 bash[75004]: cluster 2023-09-23T19:38:59.432714+0000 mon.a (mon.0) 1179 : cluster 3 Health check update: Degraded data redundancy: 7083/221382 objects degraded (3.199%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T19:39:00.287 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:00.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:00.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:39:00.392 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:00.425 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:00.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:00.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:00.532 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:00.575 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:00.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:00.578 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:00.679 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:00.710 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:00.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:00.713 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:00.814 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:00.840 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:00.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:00.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:39:00.943 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:00.958 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:39:00.991 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:00.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:00.994 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:39:01.095 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:01.138 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:01.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:01.141 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:39:01.242 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:01.270 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:01.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:01.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:01.380 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:01.415 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:01.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:01.420 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:39:01.521 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:01.545 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:01.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:01.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:39:01.648 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:01.686 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:01.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:01.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:01.790 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:01.819 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:01.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:01.850 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:39:01.950 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:01.976 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:01.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:01.978 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:39:02.079 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:02.111 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:02.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:02.117 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:39:02.218 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:02.227 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:01 smithi016 bash[75004]: cluster 2023-09-23T19:39:00.125349+0000 mgr.y (mgr.25325) 945 : cluster 0 pgmap v878: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 674 op/s; 7083/217515 objects degraded (3.256%); 0 B/s, 48 objects/s recovering 2023-09-23T19:39:02.227 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:01 smithi016 bash[75004]: audit 2023-09-23T19:39:01.584415+0000 mon.a (mon.0) 1180 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:39:02.228 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:01 smithi016 bash[75004]: audit 2023-09-23T19:39:01.594068+0000 mon.a (mon.0) 1181 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:39:02.228 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:01 smithi016 bash[77995]: cluster 2023-09-23T19:39:00.125349+0000 mgr.y (mgr.25325) 945 : cluster 0 pgmap v878: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 674 op/s; 7083/217515 objects degraded (3.256%); 0 B/s, 48 objects/s recovering 2023-09-23T19:39:02.228 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:01 smithi016 bash[77995]: audit 2023-09-23T19:39:01.584415+0000 mon.a (mon.0) 1180 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:39:02.228 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:01 smithi016 bash[77995]: audit 2023-09-23T19:39:01.594068+0000 mon.a (mon.0) 1181 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:39:02.262 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:02.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:02.269 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:39:02.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:01 smithi092 bash[58855]: cluster 2023-09-23T19:39:00.125349+0000 mgr.y (mgr.25325) 945 : cluster 0 pgmap v878: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 674 op/s; 7083/217515 objects degraded (3.256%); 0 B/s, 48 objects/s recovering 2023-09-23T19:39:02.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:01 smithi092 bash[58855]: audit 2023-09-23T19:39:01.584415+0000 mon.a (mon.0) 1180 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:39:02.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:01 smithi092 bash[58855]: audit 2023-09-23T19:39:01.594068+0000 mon.a (mon.0) 1181 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:39:02.370 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:02.390 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:02.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:02.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:02.493 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:02.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:02.538 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:02.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:02.639 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:02.678 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:02.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:02.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:02.783 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:02.809 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:02.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:02.812 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:02.913 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:02.940 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:02.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:02.943 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:39:03.044 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:03.067 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:03.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:03.070 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:39:03.170 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:03.205 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:03.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:03.207 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:03.308 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:03.342 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:03.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:03.344 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:39:03.445 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:03.468 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:03.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:03.470 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:03.571 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:03.595 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:03.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:03.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:03.699 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:03.708 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:03 smithi092 bash[58855]: cluster 2023-09-23T19:39:02.129699+0000 mgr.y (mgr.25325) 946 : cluster 0 pgmap v879: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 7083/212973 objects degraded (3.326%); 0 B/s, 41 objects/s recovering 2023-09-23T19:39:03.708 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:03 smithi092 bash[58855]: audit 2023-09-23T19:39:02.447569+0000 mon.a (mon.0) 1182 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:39:03.708 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:03 smithi092 bash[58855]: audit 2023-09-23T19:39:02.466410+0000 mon.a (mon.0) 1183 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:39:03.708 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:03 smithi092 bash[58855]: audit 2023-09-23T19:39:03.037229+0000 mon.a (mon.0) 1184 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:39:03.708 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:03 smithi092 bash[58855]: audit 2023-09-23T19:39:03.038202+0000 mon.a (mon.0) 1185 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:39:03.709 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:03 smithi092 bash[58855]: audit 2023-09-23T19:39:03.043320+0000 mon.a (mon.0) 1186 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:39:03.722 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:03.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:03.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:03.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:03 smithi016 bash[77995]: cluster 2023-09-23T19:39:02.129699+0000 mgr.y (mgr.25325) 946 : cluster 0 pgmap v879: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 7083/212973 objects degraded (3.326%); 0 B/s, 41 objects/s recovering 2023-09-23T19:39:03.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:03 smithi016 bash[77995]: audit 2023-09-23T19:39:02.447569+0000 mon.a (mon.0) 1182 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:39:03.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:03 smithi016 bash[77995]: audit 2023-09-23T19:39:02.466410+0000 mon.a (mon.0) 1183 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:39:03.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:03 smithi016 bash[77995]: audit 2023-09-23T19:39:03.037229+0000 mon.a (mon.0) 1184 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:39:03.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:03 smithi016 bash[77995]: audit 2023-09-23T19:39:03.038202+0000 mon.a (mon.0) 1185 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:39:03.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:03 smithi016 bash[77995]: audit 2023-09-23T19:39:03.043320+0000 mon.a (mon.0) 1186 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:39:03.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:03 smithi016 bash[75004]: cluster 2023-09-23T19:39:02.129699+0000 mgr.y (mgr.25325) 946 : cluster 0 pgmap v879: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 7083/212973 objects degraded (3.326%); 0 B/s, 41 objects/s recovering 2023-09-23T19:39:03.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:03 smithi016 bash[75004]: audit 2023-09-23T19:39:02.447569+0000 mon.a (mon.0) 1182 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:39:03.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:03 smithi016 bash[75004]: audit 2023-09-23T19:39:02.466410+0000 mon.a (mon.0) 1183 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:39:03.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:03 smithi016 bash[75004]: audit 2023-09-23T19:39:03.037229+0000 mon.a (mon.0) 1184 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:39:03.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:03 smithi016 bash[75004]: audit 2023-09-23T19:39:03.038202+0000 mon.a (mon.0) 1185 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:39:03.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:03 smithi016 bash[75004]: audit 2023-09-23T19:39:03.043320+0000 mon.a (mon.0) 1186 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:39:03.832 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:03.857 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:03.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:03.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:03.960 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:04.002 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:04.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:04.007 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:39:04.108 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:04.139 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:04.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:04.148 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:39:04.248 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:04.272 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:04.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:04.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:39:04.377 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:04.406 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:04.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:04.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:39:04.516 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:04.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:04 smithi092 bash[58855]: cluster 2023-09-23T19:39:04.434103+0000 mon.a (mon.0) 1187 : cluster 3 Health check update: Degraded data redundancy: 7083/212973 objects degraded (3.326%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T19:39:04.551 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:04.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:04.555 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:04.656 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:04.691 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:04.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:04.698 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:39:04.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:04 smithi016 bash[77995]: cluster 2023-09-23T19:39:04.434103+0000 mon.a (mon.0) 1187 : cluster 3 Health check update: Degraded data redundancy: 7083/212973 objects degraded (3.326%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T19:39:04.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:04 smithi016 bash[75004]: cluster 2023-09-23T19:39:04.434103+0000 mon.a (mon.0) 1187 : cluster 3 Health check update: Degraded data redundancy: 7083/212973 objects degraded (3.326%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T19:39:04.799 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:04.825 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:04.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:04.829 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:04.929 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:04.957 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:04.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:04.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:05.059 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:05.094 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:05.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:05.096 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:39:05.197 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:05.230 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:05.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:05.236 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:39:05.337 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:05.360 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:05.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:05.366 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:05.466 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:05.496 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:05.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:05.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:05.600 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:05.631 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:05.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:05.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:39:05.738 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:05.763 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:05.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:05.766 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:05.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:05 smithi092 bash[58855]: cluster 2023-09-23T19:39:04.130308+0000 mgr.y (mgr.25325) 947 : cluster 0 pgmap v880: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 7083/211194 objects degraded (3.354%); 0 B/s, 66 objects/s recovering 2023-09-23T19:39:05.789 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:39:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:39:05] "GET /metrics HTTP/1.1" 200 36316 "" "Prometheus/2.33.4" 2023-09-23T19:39:05.789 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:05 smithi016 bash[75004]: cluster 2023-09-23T19:39:04.130308+0000 mgr.y (mgr.25325) 947 : cluster 0 pgmap v880: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 7083/211194 objects degraded (3.354%); 0 B/s, 66 objects/s recovering 2023-09-23T19:39:05.790 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:05 smithi016 bash[77995]: cluster 2023-09-23T19:39:04.130308+0000 mgr.y (mgr.25325) 947 : cluster 0 pgmap v880: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 7083/211194 objects degraded (3.354%); 0 B/s, 66 objects/s recovering 2023-09-23T19:39:05.867 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:05.909 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:05.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:05.912 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:39:06.013 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:06.041 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:06.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:06.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:06.144 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:06.168 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:06.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:06.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:39:06.271 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:06.291 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:06.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:06.293 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:06.394 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:06.415 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-09-23T19:39:06.415 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [], 2023-09-23T19:39:06.415 INFO:teuthology.orchestra.run.smithi016.stdout: "complaint_time": 30, 2023-09-23T19:39:06.415 INFO:teuthology.orchestra.run.smithi016.stdout: "num_blocked_ops": 0 2023-09-23T19:39:06.415 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-09-23T19:39:06.419 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:06.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:06.426 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:06.527 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:06.561 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:06.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:06.563 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:39:06.664 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:06.693 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:06.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:06.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:06.797 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:06.836 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:06.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:06.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:39:06.947 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:06.984 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:06.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:06.991 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:39:07.092 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:07.118 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:07.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:07.120 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:07.221 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:07.274 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:07.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:07.276 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:07.377 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:07.404 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:07.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:07.407 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:07.507 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:07.517 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:07 smithi092 bash[58855]: cluster 2023-09-23T19:39:06.131057+0000 mgr.y (mgr.25325) 948 : cluster 0 pgmap v881: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 763 op/s; 7083/204084 objects degraded (3.471%); 0 B/s, 47 objects/s recovering 2023-09-23T19:39:07.532 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:07.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:07.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:39:07.641 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:07.670 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:07.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:07.676 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:39:07.777 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:07.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:07 smithi016 bash[75004]: cluster 2023-09-23T19:39:06.131057+0000 mgr.y (mgr.25325) 948 : cluster 0 pgmap v881: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 763 op/s; 7083/204084 objects degraded (3.471%); 0 B/s, 47 objects/s recovering 2023-09-23T19:39:07.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:07 smithi016 bash[77995]: cluster 2023-09-23T19:39:06.131057+0000 mgr.y (mgr.25325) 948 : cluster 0 pgmap v881: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 763 op/s; 7083/204084 objects degraded (3.471%); 0 B/s, 47 objects/s recovering 2023-09-23T19:39:07.802 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:07.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:07.805 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:07.906 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:07.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:07.935 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:07.935 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:39:08.036 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:08.066 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:08.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:08.069 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:08.170 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:08.196 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:08.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:08.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:39:08.300 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:08.336 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:08.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:08.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:39:08.442 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:08.461 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-09-23T19:39:08.461 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [ 2023-09-23T19:39:08.461 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:39:08.461 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45947.0:154940 10.9 e162/160 10:9f90b9c6:::benchmark_data_smithi016_136365_object44806:head v 162'9647, mlcod=154'7276)", 2023-09-23T19:39:08.461 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:39:08.454067+0000", 2023-09-23T19:39:08.462 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 0.003588866, 2023-09-23T19:39:08.462 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0036193060000000001, 2023-09-23T19:39:08.462 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:39:08.462 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "started", 2023-09-23T19:39:08.462 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:39:08.462 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:39:08.462 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:39:08.462 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:39:08.454067+0000", 2023-09-23T19:39:08.462 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:39:08.462 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:39:08.462 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:39:08.462 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:39:08.463 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:39:08.454067+0000", 2023-09-23T19:39:08.463 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:39:08.463 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:39:08.463 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:39:08.463 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:39:08.463 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:39:08.454066+0000", 2023-09-23T19:39:08.463 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:39:08.463 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:39:08.463 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:39:08.463 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:39:08.463 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:39:08.454075+0000", 2023-09-23T19:39:08.463 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.4449999999999996e-06 2023-09-23T19:39:08.464 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:39:08.464 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:39:08.464 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:39:08.464 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:39:08.454086+0000", 2023-09-23T19:39:08.464 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1711e-05 2023-09-23T19:39:08.464 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:39:08.464 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:39:08.464 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:39:08.464 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:39:08.454095+0000", 2023-09-23T19:39:08.464 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.3869999999999995e-06 2023-09-23T19:39:08.464 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:39:08.464 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:39:08.464 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:39:08.465 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:39:08.454139+0000", 2023-09-23T19:39:08.465 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.4688999999999999e-05 2023-09-23T19:39:08.465 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:39:08.465 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:39:08.465 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:39:08.465 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:39:08.454164+0000", 2023-09-23T19:39:08.465 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.4468999999999999e-05 2023-09-23T19:39:08.465 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:39:08.465 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:39:08.465 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:39:08.465 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:39:08.465 INFO:teuthology.orchestra.run.smithi016.stdout: ], 2023-09-23T19:39:08.466 INFO:teuthology.orchestra.run.smithi016.stdout: "num_ops": 1 2023-09-23T19:39:08.466 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-09-23T19:39:08.479 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:08.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:08.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:39:08.584 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:08.619 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:08.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:08.626 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:08.727 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:08.760 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:08.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:08.764 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:08.865 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:08.893 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:08.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:08.897 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:39:08.997 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:09.038 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:09.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:09.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:09.142 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:09.174 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:09.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:09.177 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:39:09.278 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:09.281 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T19:39:09.307 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:09.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:09.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:39:09.412 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:09.452 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:09.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:09.468 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:09.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:09 smithi016 bash[77995]: cluster 2023-09-23T19:39:08.131657+0000 mgr.y (mgr.25325) 949 : cluster 0 pgmap v882: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 624 op/s; 7083/201570 objects degraded (3.514%); 0 B/s, 76 objects/s recovering 2023-09-23T19:39:09.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:09 smithi016 bash[77995]: cluster 2023-09-23T19:39:09.435583+0000 mon.a (mon.0) 1188 : cluster 3 Health check update: Degraded data redundancy: 7083/201570 objects degraded (3.514%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T19:39:09.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:09 smithi016 bash[75004]: cluster 2023-09-23T19:39:08.131657+0000 mgr.y (mgr.25325) 949 : cluster 0 pgmap v882: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 624 op/s; 7083/201570 objects degraded (3.514%); 0 B/s, 76 objects/s recovering 2023-09-23T19:39:09.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:09 smithi016 bash[75004]: cluster 2023-09-23T19:39:09.435583+0000 mon.a (mon.0) 1188 : cluster 3 Health check update: Degraded data redundancy: 7083/201570 objects degraded (3.514%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T19:39:09.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:09 smithi092 bash[58855]: cluster 2023-09-23T19:39:08.131657+0000 mgr.y (mgr.25325) 949 : cluster 0 pgmap v882: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 624 op/s; 7083/201570 objects degraded (3.514%); 0 B/s, 76 objects/s recovering 2023-09-23T19:39:09.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:09 smithi092 bash[58855]: cluster 2023-09-23T19:39:09.435583+0000 mon.a (mon.0) 1188 : cluster 3 Health check update: Degraded data redundancy: 7083/201570 objects degraded (3.514%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T19:39:09.569 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:09.598 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:09.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:09.601 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:39:09.702 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:09.728 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:09.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:09.730 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:09.830 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:09.859 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:09.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:09.863 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:39:09.881 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:39:09.921 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:39:09.963 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:09.964 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:39:10.000 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:10.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:10.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:10.009 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:39:10.047 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:39:10.094 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:39:10.109 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:10.131 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:39:10.142 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:10.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:10.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:10.176 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:39:10.180 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:39:10.180 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:39:10.181 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:39:10.181 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:39:10.181 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:39:10.181 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:39:10.181 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:39:10.181 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:39:10.245 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:10.294 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:10.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:10.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:10.400 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:10.432 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:10.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:10.437 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:39:10.487 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:10 smithi016 bash[77995]: audit 2023-09-23T19:39:09.828529+0000 mon.c (mon.2) 168 : audit 0 from='client.? 172.21.15.16:0/3937081973' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:39:10.487 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:10 smithi092 bash[58855]: audit 2023-09-23T19:39:09.828529+0000 mon.c (mon.2) 168 : audit 0 from='client.? 172.21.15.16:0/3937081973' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:39:10.537 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:10.572 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:10.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:10.577 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:39:10.677 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:10.706 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:10.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:10.708 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:39:10.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:10 smithi016 bash[75004]: audit 2023-09-23T19:39:09.828529+0000 mon.c (mon.2) 168 : audit 0 from='client.? 172.21.15.16:0/3937081973' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:39:10.809 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:10.829 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:10.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:10.833 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:39:10.934 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:10.961 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:10.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:10.964 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:39:11.064 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:11.095 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:11.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:11.097 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:39:11.198 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:11.199 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T19:39:11.273 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:11.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:11.275 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:11.376 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:11.403 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:11.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:11.405 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:39:11.506 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:11.535 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:11.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:11.539 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:39:11.640 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:11.648 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:11 smithi092 bash[58855]: cluster 2023-09-23T19:39:10.133068+0000 mgr.y (mgr.25325) 950 : cluster 0 pgmap v883: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 7083/197811 objects degraded (3.581%); 0 B/s, 55 objects/s recovering 2023-09-23T19:39:11.648 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:11 smithi092 bash[58855]: cluster 2023-09-23T19:39:10.492788+0000 mon.a (mon.0) 1189 : cluster 0 osdmap e163: 8 total, 8 up, 8 in 2023-09-23T19:39:11.649 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:11 smithi092 bash[58855]: cluster 2023-09-23T19:39:10.503706+0000 osd.7 (osd.7) 76 : cluster 3 failed to encode map e163 with expected crc 2023-09-23T19:39:11.649 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:11 smithi092 bash[58855]: cluster 2023-09-23T19:39:10.503866+0000 osd.5 (osd.5) 96 : cluster 3 failed to encode map e163 with expected crc 2023-09-23T19:39:11.649 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:11 smithi092 bash[58855]: cluster 2023-09-23T19:39:10.504611+0000 osd.6 (osd.6) 85 : cluster 3 failed to encode map e163 with expected crc 2023-09-23T19:39:11.649 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:11 smithi092 bash[58855]: cluster 2023-09-23T19:39:10.505718+0000 osd.5 (osd.5) 97 : cluster 3 failed to encode map e163 with expected crc 2023-09-23T19:39:11.649 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:11 smithi092 bash[58855]: cluster 2023-09-23T19:39:10.506354+0000 osd.6 (osd.6) 86 : cluster 3 failed to encode map e163 with expected crc 2023-09-23T19:39:11.649 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:11 smithi092 bash[58855]: cluster 2023-09-23T19:39:10.506868+0000 osd.4 (osd.4) 96 : cluster 3 failed to encode map e163 with expected crc 2023-09-23T19:39:11.649 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:11 smithi092 bash[58855]: cluster 2023-09-23T19:39:10.508536+0000 osd.4 (osd.4) 97 : cluster 3 failed to encode map e163 with expected crc 2023-09-23T19:39:11.649 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:11 smithi092 bash[58855]: cluster 2023-09-23T19:39:10.510221+0000 osd.4 (osd.4) 98 : cluster 3 failed to encode map e163 with expected crc 2023-09-23T19:39:11.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:11.679 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:11.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:39:11.780 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:11.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:11 smithi016 bash[75004]: cluster 2023-09-23T19:39:10.133068+0000 mgr.y (mgr.25325) 950 : cluster 0 pgmap v883: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 7083/197811 objects degraded (3.581%); 0 B/s, 55 objects/s recovering 2023-09-23T19:39:11.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:11 smithi016 bash[75004]: cluster 2023-09-23T19:39:10.492788+0000 mon.a (mon.0) 1189 : cluster 0 osdmap e163: 8 total, 8 up, 8 in 2023-09-23T19:39:11.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:11 smithi016 bash[75004]: cluster 2023-09-23T19:39:10.503706+0000 osd.7 (osd.7) 76 : cluster 3 failed to encode map e163 with expected crc 2023-09-23T19:39:11.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:11 smithi016 bash[75004]: cluster 2023-09-23T19:39:10.503866+0000 osd.5 (osd.5) 96 : cluster 3 failed to encode map e163 with expected crc 2023-09-23T19:39:11.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:11 smithi016 bash[75004]: cluster 2023-09-23T19:39:10.504611+0000 osd.6 (osd.6) 85 : cluster 3 failed to encode map e163 with expected crc 2023-09-23T19:39:11.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:11 smithi016 bash[75004]: cluster 2023-09-23T19:39:10.505718+0000 osd.5 (osd.5) 97 : cluster 3 failed to encode map e163 with expected crc 2023-09-23T19:39:11.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:11 smithi016 bash[75004]: cluster 2023-09-23T19:39:10.506354+0000 osd.6 (osd.6) 86 : cluster 3 failed to encode map e163 with expected crc 2023-09-23T19:39:11.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:11 smithi016 bash[75004]: cluster 2023-09-23T19:39:10.506868+0000 osd.4 (osd.4) 96 : cluster 3 failed to encode map e163 with expected crc 2023-09-23T19:39:11.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:11 smithi016 bash[75004]: cluster 2023-09-23T19:39:10.508536+0000 osd.4 (osd.4) 97 : cluster 3 failed to encode map e163 with expected crc 2023-09-23T19:39:11.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:11 smithi016 bash[75004]: cluster 2023-09-23T19:39:10.510221+0000 osd.4 (osd.4) 98 : cluster 3 failed to encode map e163 with expected crc 2023-09-23T19:39:11.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:11 smithi016 bash[77995]: cluster 2023-09-23T19:39:10.133068+0000 mgr.y (mgr.25325) 950 : cluster 0 pgmap v883: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 7083/197811 objects degraded (3.581%); 0 B/s, 55 objects/s recovering 2023-09-23T19:39:11.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:11 smithi016 bash[77995]: cluster 2023-09-23T19:39:10.492788+0000 mon.a (mon.0) 1189 : cluster 0 osdmap e163: 8 total, 8 up, 8 in 2023-09-23T19:39:11.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:11 smithi016 bash[77995]: cluster 2023-09-23T19:39:10.503706+0000 osd.7 (osd.7) 76 : cluster 3 failed to encode map e163 with expected crc 2023-09-23T19:39:11.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:11 smithi016 bash[77995]: cluster 2023-09-23T19:39:10.503866+0000 osd.5 (osd.5) 96 : cluster 3 failed to encode map e163 with expected crc 2023-09-23T19:39:11.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:11 smithi016 bash[77995]: cluster 2023-09-23T19:39:10.504611+0000 osd.6 (osd.6) 85 : cluster 3 failed to encode map e163 with expected crc 2023-09-23T19:39:11.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:11 smithi016 bash[77995]: cluster 2023-09-23T19:39:10.505718+0000 osd.5 (osd.5) 97 : cluster 3 failed to encode map e163 with expected crc 2023-09-23T19:39:11.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:11 smithi016 bash[77995]: cluster 2023-09-23T19:39:10.506354+0000 osd.6 (osd.6) 86 : cluster 3 failed to encode map e163 with expected crc 2023-09-23T19:39:11.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:11 smithi016 bash[77995]: cluster 2023-09-23T19:39:10.506868+0000 osd.4 (osd.4) 96 : cluster 3 failed to encode map e163 with expected crc 2023-09-23T19:39:11.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:11 smithi016 bash[77995]: cluster 2023-09-23T19:39:10.508536+0000 osd.4 (osd.4) 97 : cluster 3 failed to encode map e163 with expected crc 2023-09-23T19:39:11.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:11 smithi016 bash[77995]: cluster 2023-09-23T19:39:10.510221+0000 osd.4 (osd.4) 98 : cluster 3 failed to encode map e163 with expected crc 2023-09-23T19:39:11.805 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:11.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:11.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:11.916 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:11.942 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:11.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:11.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:12.048 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:12.072 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:12.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:12.078 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:12.179 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:12.228 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:12.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:12.231 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:39:12.331 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:12.355 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:12.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:12.361 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:39:12.461 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:12.489 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:12.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:12.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:12.593 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:12.601 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:12 smithi092 bash[58855]: cluster 2023-09-23T19:39:11.486188+0000 mon.a (mon.0) 1190 : cluster 0 osdmap e164: 8 total, 8 up, 8 in 2023-09-23T19:39:12.601 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:12 smithi092 bash[58855]: cluster 2023-09-23T19:39:11.500295+0000 osd.7 (osd.7) 77 : cluster 3 failed to encode map e164 with expected crc 2023-09-23T19:39:12.601 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:12 smithi092 bash[58855]: cluster 2023-09-23T19:39:11.524013+0000 osd.4 (osd.4) 99 : cluster 3 failed to encode map e164 with expected crc 2023-09-23T19:39:12.602 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:12 smithi092 bash[58855]: cluster 2023-09-23T19:39:11.547409+0000 osd.6 (osd.6) 87 : cluster 3 failed to encode map e164 with expected crc 2023-09-23T19:39:12.626 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:12.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:12.629 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:39:12.729 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:12.739 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:12 smithi016 bash[75004]: cluster 2023-09-23T19:39:11.486188+0000 mon.a (mon.0) 1190 : cluster 0 osdmap e164: 8 total, 8 up, 8 in 2023-09-23T19:39:12.739 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:12 smithi016 bash[75004]: cluster 2023-09-23T19:39:11.500295+0000 osd.7 (osd.7) 77 : cluster 3 failed to encode map e164 with expected crc 2023-09-23T19:39:12.739 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:12 smithi016 bash[75004]: cluster 2023-09-23T19:39:11.524013+0000 osd.4 (osd.4) 99 : cluster 3 failed to encode map e164 with expected crc 2023-09-23T19:39:12.739 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:12 smithi016 bash[75004]: cluster 2023-09-23T19:39:11.547409+0000 osd.6 (osd.6) 87 : cluster 3 failed to encode map e164 with expected crc 2023-09-23T19:39:12.739 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:12 smithi016 bash[77995]: cluster 2023-09-23T19:39:11.486188+0000 mon.a (mon.0) 1190 : cluster 0 osdmap e164: 8 total, 8 up, 8 in 2023-09-23T19:39:12.739 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:12 smithi016 bash[77995]: cluster 2023-09-23T19:39:11.500295+0000 osd.7 (osd.7) 77 : cluster 3 failed to encode map e164 with expected crc 2023-09-23T19:39:12.740 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:12 smithi016 bash[77995]: cluster 2023-09-23T19:39:11.524013+0000 osd.4 (osd.4) 99 : cluster 3 failed to encode map e164 with expected crc 2023-09-23T19:39:12.740 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:12 smithi016 bash[77995]: cluster 2023-09-23T19:39:11.547409+0000 osd.6 (osd.6) 87 : cluster 3 failed to encode map e164 with expected crc 2023-09-23T19:39:12.761 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:12.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:12.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:12.864 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:12.894 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:12.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:12.897 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:12.998 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:13.026 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:13.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:13.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:13.129 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:13.167 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:13.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:13.173 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:13.274 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:13.295 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:13.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:13.302 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:39:13.403 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:13.434 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:13.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:13.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:13.517 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:13 smithi092 bash[58855]: cluster 2023-09-23T19:39:12.133852+0000 mgr.y (mgr.25325) 951 : cluster 0 pgmap v886: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 732 op/s; 5681/193596 objects degraded (2.934%); 0 B/s, 45 objects/s recovering 2023-09-23T19:39:13.541 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:13.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:13.574 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:13.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:13.675 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:13.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:13.702 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:13.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:39:13.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:13 smithi016 bash[75004]: cluster 2023-09-23T19:39:12.133852+0000 mgr.y (mgr.25325) 951 : cluster 0 pgmap v886: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 732 op/s; 5681/193596 objects degraded (2.934%); 0 B/s, 45 objects/s recovering 2023-09-23T19:39:13.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:13 smithi016 bash[77995]: cluster 2023-09-23T19:39:12.133852+0000 mgr.y (mgr.25325) 951 : cluster 0 pgmap v886: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 732 op/s; 5681/193596 objects degraded (2.934%); 0 B/s, 45 objects/s recovering 2023-09-23T19:39:13.803 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:13.835 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:13.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:13.840 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:39:13.941 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:13.974 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:13.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:13.978 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:39:14.079 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:14.113 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:14.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:14.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:14.217 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:14.257 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:14.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:14.259 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:14.360 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:14.385 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:14.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:14.388 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:39:14.489 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:14.519 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:14 smithi016 bash[75004]: audit 2023-09-23T19:39:13.654096+0000 mon.a (mon.0) 1191 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:39:14.519 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:14 smithi016 bash[75004]: audit 2023-09-23T19:39:13.935196+0000 mon.a (mon.0) 1192 : audit 1 from='client.? 172.21.15.16:0/1206020874' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:39:14.519 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:14 smithi016 bash[75004]: cluster 2023-09-23T19:39:14.438652+0000 mon.a (mon.0) 1193 : cluster 3 Health check update: Degraded data redundancy: 5681/193596 objects degraded (2.934%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T19:39:14.525 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:14.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:14.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:14.549 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is unset 2023-09-23T19:39:14.631 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:14.640 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:14 smithi092 bash[58855]: audit 2023-09-23T19:39:13.654096+0000 mon.a (mon.0) 1191 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:39:14.640 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:14 smithi092 bash[58855]: audit 2023-09-23T19:39:13.935196+0000 mon.a (mon.0) 1192 : audit 1 from='client.? 172.21.15.16:0/1206020874' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:39:14.640 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:14 smithi092 bash[58855]: cluster 2023-09-23T19:39:14.438652+0000 mon.a (mon.0) 1193 : cluster 3 Health check update: Degraded data redundancy: 5681/193596 objects degraded (2.934%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T19:39:14.669 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:14.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:14.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:14.776 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:14.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:14 smithi016 bash[77995]: audit 2023-09-23T19:39:13.654096+0000 mon.a (mon.0) 1191 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:39:14.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:14 smithi016 bash[77995]: audit 2023-09-23T19:39:13.935196+0000 mon.a (mon.0) 1192 : audit 1 from='client.? 172.21.15.16:0/1206020874' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:39:14.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:14 smithi016 bash[77995]: cluster 2023-09-23T19:39:14.438652+0000 mon.a (mon.0) 1193 : cluster 3 Health check update: Degraded data redundancy: 5681/193596 objects degraded (2.934%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T19:39:14.799 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:14.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:14.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:39:14.866 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:39:14.910 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:14.938 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:14.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:14.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:39:15.041 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:15.076 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:15.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:15.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:39:15.186 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:15.216 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:15.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:15.224 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:39:15.325 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:15.354 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:15.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:15.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:39:15.466 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:15.503 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:15.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:15.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:39:15.606 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:15.614 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:15 smithi092 bash[58855]: cluster 2023-09-23T19:39:14.134782+0000 mgr.y (mgr.25325) 952 : cluster 0 pgmap v887: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 508 op/s; 5681/191877 objects degraded (2.961%); 0 B/s, 59 objects/s recovering 2023-09-23T19:39:15.614 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:15 smithi092 bash[58855]: cluster 2023-09-23T19:39:14.512920+0000 mon.a (mon.0) 1194 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:39:15.615 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:15 smithi092 bash[58855]: audit 2023-09-23T19:39:14.533008+0000 mon.a (mon.0) 1195 : audit 1 from='client.? 172.21.15.16:0/1206020874' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:39:15.615 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:15 smithi092 bash[58855]: cluster 2023-09-23T19:39:14.533071+0000 mon.a (mon.0) 1196 : cluster 0 osdmap e165: 8 total, 8 up, 8 in 2023-09-23T19:39:15.615 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:15 smithi092 bash[58855]: cluster 2023-09-23T19:39:14.536225+0000 osd.6 (osd.6) 88 : cluster 3 failed to encode map e165 with expected crc 2023-09-23T19:39:15.615 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:15 smithi092 bash[58855]: cluster 2023-09-23T19:39:14.550886+0000 osd.7 (osd.7) 78 : cluster 3 failed to encode map e165 with expected crc 2023-09-23T19:39:15.637 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:15.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:15.639 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:39:15.740 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:15.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:15 smithi016 bash[77995]: cluster 2023-09-23T19:39:14.134782+0000 mgr.y (mgr.25325) 952 : cluster 0 pgmap v887: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 508 op/s; 5681/191877 objects degraded (2.961%); 0 B/s, 59 objects/s recovering 2023-09-23T19:39:15.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:15 smithi016 bash[77995]: cluster 2023-09-23T19:39:14.512920+0000 mon.a (mon.0) 1194 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:39:15.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:15 smithi016 bash[77995]: audit 2023-09-23T19:39:14.533008+0000 mon.a (mon.0) 1195 : audit 1 from='client.? 172.21.15.16:0/1206020874' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:39:15.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:15 smithi016 bash[77995]: cluster 2023-09-23T19:39:14.533071+0000 mon.a (mon.0) 1196 : cluster 0 osdmap e165: 8 total, 8 up, 8 in 2023-09-23T19:39:15.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:15 smithi016 bash[77995]: cluster 2023-09-23T19:39:14.536225+0000 osd.6 (osd.6) 88 : cluster 3 failed to encode map e165 with expected crc 2023-09-23T19:39:15.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:15 smithi016 bash[77995]: cluster 2023-09-23T19:39:14.550886+0000 osd.7 (osd.7) 78 : cluster 3 failed to encode map e165 with expected crc 2023-09-23T19:39:15.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:15 smithi016 bash[75004]: cluster 2023-09-23T19:39:14.134782+0000 mgr.y (mgr.25325) 952 : cluster 0 pgmap v887: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 508 op/s; 5681/191877 objects degraded (2.961%); 0 B/s, 59 objects/s recovering 2023-09-23T19:39:15.750 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:15 smithi016 bash[75004]: cluster 2023-09-23T19:39:14.512920+0000 mon.a (mon.0) 1194 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:39:15.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:15 smithi016 bash[75004]: audit 2023-09-23T19:39:14.533008+0000 mon.a (mon.0) 1195 : audit 1 from='client.? 172.21.15.16:0/1206020874' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:39:15.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:15 smithi016 bash[75004]: cluster 2023-09-23T19:39:14.533071+0000 mon.a (mon.0) 1196 : cluster 0 osdmap e165: 8 total, 8 up, 8 in 2023-09-23T19:39:15.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:15 smithi016 bash[75004]: cluster 2023-09-23T19:39:14.536225+0000 osd.6 (osd.6) 88 : cluster 3 failed to encode map e165 with expected crc 2023-09-23T19:39:15.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:15 smithi016 bash[75004]: cluster 2023-09-23T19:39:14.550886+0000 osd.7 (osd.7) 78 : cluster 3 failed to encode map e165 with expected crc 2023-09-23T19:39:15.751 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:39:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:39:15] "GET /metrics HTTP/1.1" 200 36314 "" "Prometheus/2.33.4" 2023-09-23T19:39:15.767 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:15.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:15.772 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:15.873 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:15.925 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:15.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:15.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:16.033 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:16.061 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:16.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:16.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:16.166 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:16.201 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:16.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:16.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:16.307 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:16.354 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:16.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:16.356 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:16.457 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:16.497 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:16.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:16.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:39:16.602 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:16.632 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:16.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:16.639 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:16.739 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:16.765 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:16.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:16.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:39:16.869 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:16.906 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:16.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:16.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:17.008 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:17.047 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:17.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:17.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:39:17.153 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:17.184 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:17.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:17.189 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:39:17.290 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:17.321 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:17.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:17.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:17.429 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:17.463 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:17.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:17.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:39:17.569 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:17.580 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:17 smithi016 bash[75004]: cluster 2023-09-23T19:39:16.135901+0000 mgr.y (mgr.25325) 953 : cluster 0 pgmap v889: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 5681/185241 objects degraded (3.067%); 0 B/s, 38 objects/s recovering 2023-09-23T19:39:17.580 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:17 smithi016 bash[77995]: cluster 2023-09-23T19:39:16.135901+0000 mgr.y (mgr.25325) 953 : cluster 0 pgmap v889: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 5681/185241 objects degraded (3.067%); 0 B/s, 38 objects/s recovering 2023-09-23T19:39:17.603 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:17.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:17.610 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:17.710 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:17.740 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:17.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:17.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:39:17.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:17 smithi092 bash[58855]: cluster 2023-09-23T19:39:16.135901+0000 mgr.y (mgr.25325) 953 : cluster 0 pgmap v889: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 5681/185241 objects degraded (3.067%); 0 B/s, 38 objects/s recovering 2023-09-23T19:39:17.845 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:17.884 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:17.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:17.888 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:17.934 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T19:39:17.990 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:18.026 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:18.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:18.029 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:39:18.130 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:18.169 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:18.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:18.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:39:18.276 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:18.305 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:18.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:18.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:18.408 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:18.436 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:18.441 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:39:18.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:18.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:39:18.444 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:39:18.545 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:18.588 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:18.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:18.593 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:18.694 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:18.721 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:18.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:18.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:39:18.826 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:18.862 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:18.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:18.865 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:39:18.966 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:19.002 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:19.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:19.009 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:39:19.109 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:19.139 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:19.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:19.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:39:19.249 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:19.274 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:19.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:19.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:39:19.379 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:19.414 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:19.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:19.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:39:19.517 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:19.544 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:19.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:19.547 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:39:19.648 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:19.656 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:19 smithi092 bash[58855]: cluster 2023-09-23T19:39:18.136504+0000 mgr.y (mgr.25325) 954 : cluster 0 pgmap v890: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 5681/182637 objects degraded (3.111%); 0 B/s, 29 objects/s recovering 2023-09-23T19:39:19.656 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:19 smithi092 bash[58855]: cluster 2023-09-23T19:39:19.440075+0000 mon.a (mon.0) 1197 : cluster 3 Health check update: Degraded data redundancy: 5681/182637 objects degraded (3.111%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T19:39:19.678 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:19.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:19.683 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:39:19.784 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:19.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:19 smithi016 bash[77995]: cluster 2023-09-23T19:39:18.136504+0000 mgr.y (mgr.25325) 954 : cluster 0 pgmap v890: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 5681/182637 objects degraded (3.111%); 0 B/s, 29 objects/s recovering 2023-09-23T19:39:19.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:19 smithi016 bash[77995]: cluster 2023-09-23T19:39:19.440075+0000 mon.a (mon.0) 1197 : cluster 3 Health check update: Degraded data redundancy: 5681/182637 objects degraded (3.111%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T19:39:19.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:19 smithi016 bash[75004]: cluster 2023-09-23T19:39:18.136504+0000 mgr.y (mgr.25325) 954 : cluster 0 pgmap v890: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 5681/182637 objects degraded (3.111%); 0 B/s, 29 objects/s recovering 2023-09-23T19:39:19.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:19 smithi016 bash[75004]: cluster 2023-09-23T19:39:19.440075+0000 mon.a (mon.0) 1197 : cluster 3 Health check update: Degraded data redundancy: 5681/182637 objects degraded (3.111%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T19:39:19.816 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:19.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:19.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:19.926 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:19.960 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:19.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:19.968 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:39:20.069 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:20.105 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:20.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:20.111 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:39:20.212 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:20.248 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:20.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:20.253 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:39:20.354 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:20.385 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:20.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:20.388 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:39:20.479 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:39:20.488 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:20.522 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:20.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:20.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:39:20.627 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:20.650 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:20.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:20.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:39:20.754 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:20.789 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:20.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:20.793 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:20.893 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:20.921 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:20.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:20.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:21.027 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:21.056 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:21.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:21.059 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:21.159 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:21.189 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:21.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:21.192 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:39:21.292 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:21.322 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:21.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:21.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:39:21.425 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:21.456 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:21.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:21.459 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:39:21.560 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:21.601 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:21.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:21.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:39:21.706 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:21.741 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:21.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:21.747 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:39:21.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:21 smithi092 bash[58855]: cluster 2023-09-23T19:39:20.137183+0000 mgr.y (mgr.25325) 955 : cluster 0 pgmap v891: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 5681/178752 objects degraded (3.178%); 0 B/s, 25 objects/s recovering 2023-09-23T19:39:21.800 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:21 smithi016 bash[75004]: cluster 2023-09-23T19:39:20.137183+0000 mgr.y (mgr.25325) 955 : cluster 0 pgmap v891: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 5681/178752 objects degraded (3.178%); 0 B/s, 25 objects/s recovering 2023-09-23T19:39:21.800 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:21 smithi016 bash[77995]: cluster 2023-09-23T19:39:20.137183+0000 mgr.y (mgr.25325) 955 : cluster 0 pgmap v891: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 5681/178752 objects degraded (3.178%); 0 B/s, 25 objects/s recovering 2023-09-23T19:39:21.848 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:21.878 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:21.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:21.886 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:39:21.986 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:22.026 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:22.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:22.033 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:39:22.134 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:22.163 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:22.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:22.168 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:39:22.269 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:22.301 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:22.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:22.308 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:22.409 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:22.440 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:22.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:22.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:39:22.510 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:39:22.546 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:22.586 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:22.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:22.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:22.693 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:22.716 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:22.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:22.723 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:39:22.824 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:22.850 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:22.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:22.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:39:22.955 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:22.987 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:22.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:22.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:23.090 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:23.128 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:23.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:23.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:23.235 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:23.259 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:23.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:23.262 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:39:23.363 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:23.402 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:23.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:23.404 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:39:23.505 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:23.551 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:23.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:23.556 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:39:23.656 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:23.665 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:23 smithi092 bash[58855]: cluster 2023-09-23T19:39:22.141119+0000 mgr.y (mgr.25325) 956 : cluster 0 pgmap v892: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 651 op/s; 5681/174039 objects degraded (3.264%); 0 B/s, 33 objects/s recovering 2023-09-23T19:39:23.688 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:23.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:23.692 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:23.793 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:23.808 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:23 smithi016 bash[75004]: cluster 2023-09-23T19:39:22.141119+0000 mgr.y (mgr.25325) 956 : cluster 0 pgmap v892: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 651 op/s; 5681/174039 objects degraded (3.264%); 0 B/s, 33 objects/s recovering 2023-09-23T19:39:23.808 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:23 smithi016 bash[77995]: cluster 2023-09-23T19:39:22.141119+0000 mgr.y (mgr.25325) 956 : cluster 0 pgmap v892: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 651 op/s; 5681/174039 objects degraded (3.264%); 0 B/s, 33 objects/s recovering 2023-09-23T19:39:23.817 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:23.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:23.822 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:39:23.923 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:23.952 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:23.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:23.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:24.056 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:24.089 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:24.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:24.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:39:24.192 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:24.241 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:24.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:24.243 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:39:24.343 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:24.374 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:24.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:24.381 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:24.482 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:24.506 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:24.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:24.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:39:24.610 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:24.649 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:24.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:24.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:24.756 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:24.764 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:24 smithi092 bash[58855]: cluster 2023-09-23T19:39:24.441339+0000 mon.a (mon.0) 1198 : cluster 3 Health check update: Degraded data redundancy: 5681/174039 objects degraded (3.264%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T19:39:24.781 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:24.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:24.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:39:24.884 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:24.891 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:24 smithi016 bash[77995]: cluster 2023-09-23T19:39:24.441339+0000 mon.a (mon.0) 1198 : cluster 3 Health check update: Degraded data redundancy: 5681/174039 objects degraded (3.264%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T19:39:24.891 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:24 smithi016 bash[75004]: cluster 2023-09-23T19:39:24.441339+0000 mon.a (mon.0) 1198 : cluster 3 Health check update: Degraded data redundancy: 5681/174039 objects degraded (3.264%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T19:39:24.907 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:24.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:24.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:39:25.011 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:25.036 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:25.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:25.038 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:25.138 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:25.168 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:25.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:25.171 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:25.272 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:25.307 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:25.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:25.312 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:39:25.413 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:25.448 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:25.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:25.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:25.554 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:25.586 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:25.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:25.589 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:25.690 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:25.700 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:25 smithi092 bash[58855]: cluster 2023-09-23T19:39:24.141668+0000 mgr.y (mgr.25325) 957 : cluster 0 pgmap v893: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 5681/172197 objects degraded (3.299%); 0 B/s, 23 objects/s recovering 2023-09-23T19:39:25.734 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:25.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:25.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:25.840 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:25.863 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:25.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:25.865 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:39:25.966 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:25.999 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:26.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:26.007 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:39:26.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:25 smithi016 bash[75004]: cluster 2023-09-23T19:39:24.141668+0000 mgr.y (mgr.25325) 957 : cluster 0 pgmap v893: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 5681/172197 objects degraded (3.299%); 0 B/s, 23 objects/s recovering 2023-09-23T19:39:26.036 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:39:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:39:25] "GET /metrics HTTP/1.1" 200 36314 "" "Prometheus/2.33.4" 2023-09-23T19:39:26.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:25 smithi016 bash[77995]: cluster 2023-09-23T19:39:24.141668+0000 mgr.y (mgr.25325) 957 : cluster 0 pgmap v893: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 5681/172197 objects degraded (3.299%); 0 B/s, 23 objects/s recovering 2023-09-23T19:39:26.107 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:26.130 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:26.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:26.132 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:26.233 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:26.262 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:26.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:26.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:39:26.364 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:26.391 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:26.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:26.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:26.496 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:26.523 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:26.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:26.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:39:26.625 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:26.657 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:26.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:26.661 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:39:26.762 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:26.804 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:26.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:26.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:39:26.911 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:26.944 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:26.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:26.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:27.049 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:27.078 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:27.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:27.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:39:27.181 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:27.210 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:27.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:27.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:39:27.313 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:27.342 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:27.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:27.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:27.445 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:27.476 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:27.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:27.485 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:39:27.586 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:27.617 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:27.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:27.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:27.724 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:27.760 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:27.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:27.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:27.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:27 smithi016 bash[77995]: cluster 2023-09-23T19:39:26.142403+0000 mgr.y (mgr.25325) 958 : cluster 0 pgmap v894: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 775 op/s; 5681/164850 objects degraded (3.446%); 0 B/s, 29 objects/s recovering 2023-09-23T19:39:27.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:27 smithi016 bash[75004]: cluster 2023-09-23T19:39:26.142403+0000 mgr.y (mgr.25325) 958 : cluster 0 pgmap v894: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 775 op/s; 5681/164850 objects degraded (3.446%); 0 B/s, 29 objects/s recovering 2023-09-23T19:39:27.864 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:27.901 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:27.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:27.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:39:28.006 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:28.031 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:28.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:28.035 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:28.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:27 smithi092 bash[58855]: cluster 2023-09-23T19:39:26.142403+0000 mgr.y (mgr.25325) 958 : cluster 0 pgmap v894: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 775 op/s; 5681/164850 objects degraded (3.446%); 0 B/s, 29 objects/s recovering 2023-09-23T19:39:28.136 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:28.171 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:28.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:28.174 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:39:28.275 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:28.310 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:28.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:28.312 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:39:28.412 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:28.438 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:28.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:28.441 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:39:28.541 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:28.570 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:28.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:28.573 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:39:28.674 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:28.711 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:28.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:28.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:28.815 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:28.834 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:28.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:28.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:28.937 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:28.967 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:28.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:28.972 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:39:29.073 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:29.102 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:29.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:29.104 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:39:29.205 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:29.237 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:29.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:29.240 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:29.341 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:29.385 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:29.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:29.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:29.491 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:29.521 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:29.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:29.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:29.624 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:29.674 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:29.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:29.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:29.780 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:29.804 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:29.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:29.806 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:39:29.813 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:29 smithi016 bash[75004]: cluster 2023-09-23T19:39:28.143095+0000 mgr.y (mgr.25325) 959 : cluster 0 pgmap v895: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 5681/162423 objects degraded (3.498%); 0 B/s, 19 objects/s recovering 2023-09-23T19:39:29.813 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:29 smithi016 bash[75004]: audit 2023-09-23T19:39:28.659457+0000 mon.a (mon.0) 1199 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:39:29.814 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:29 smithi016 bash[75004]: audit 2023-09-23T19:39:28.660816+0000 mon.a (mon.0) 1200 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:39:29.814 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:29 smithi016 bash[75004]: cluster 2023-09-23T19:39:29.443068+0000 mon.a (mon.0) 1201 : cluster 3 Health check update: Degraded data redundancy: 5681/162423 objects degraded (3.498%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T19:39:29.814 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:29 smithi016 bash[77995]: cluster 2023-09-23T19:39:28.143095+0000 mgr.y (mgr.25325) 959 : cluster 0 pgmap v895: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 5681/162423 objects degraded (3.498%); 0 B/s, 19 objects/s recovering 2023-09-23T19:39:29.814 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:29 smithi016 bash[77995]: audit 2023-09-23T19:39:28.659457+0000 mon.a (mon.0) 1199 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:39:29.814 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:29 smithi016 bash[77995]: audit 2023-09-23T19:39:28.660816+0000 mon.a (mon.0) 1200 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:39:29.814 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:29 smithi016 bash[77995]: cluster 2023-09-23T19:39:29.443068+0000 mon.a (mon.0) 1201 : cluster 3 Health check update: Degraded data redundancy: 5681/162423 objects degraded (3.498%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T19:39:29.907 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:29.915 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:29 smithi092 bash[58855]: cluster 2023-09-23T19:39:28.143095+0000 mgr.y (mgr.25325) 959 : cluster 0 pgmap v895: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 5681/162423 objects degraded (3.498%); 0 B/s, 19 objects/s recovering 2023-09-23T19:39:29.915 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:29 smithi092 bash[58855]: audit 2023-09-23T19:39:28.659457+0000 mon.a (mon.0) 1199 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:39:29.915 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:29 smithi092 bash[58855]: audit 2023-09-23T19:39:28.660816+0000 mon.a (mon.0) 1200 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:39:29.915 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:29 smithi092 bash[58855]: cluster 2023-09-23T19:39:29.443068+0000 mon.a (mon.0) 1201 : cluster 3 Health check update: Degraded data redundancy: 5681/162423 objects degraded (3.498%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T19:39:29.949 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:29.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:29.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:30.057 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:30.085 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:30.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:30.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:39:30.190 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:30.220 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:30.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:30.226 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:39:30.326 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:30.356 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:30.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:30.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:39:30.462 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:30.490 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:30.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:30.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:39:30.594 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:30.624 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:30.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:30.628 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:30.729 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:30.765 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:30.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:30.770 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:30.870 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:30.893 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:30.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:30.895 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:39:30.996 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:31.031 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:31.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:31.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:31.140 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:31.187 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:31.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:31.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:39:31.299 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:31.330 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:31.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:31.333 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:39:31.434 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:31.467 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:31.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:31.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:39:31.575 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:31.610 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:31.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:31.611 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:39:31.665 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:31 smithi016 bash[75004]: cluster 2023-09-23T19:39:30.143594+0000 mgr.y (mgr.25325) 960 : cluster 0 pgmap v896: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 5681/158469 objects degraded (3.585%); 0 B/s, 19 objects/s recovering 2023-09-23T19:39:31.712 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:31.742 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:31.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:31.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:31.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:31 smithi092 bash[58855]: cluster 2023-09-23T19:39:30.143594+0000 mgr.y (mgr.25325) 960 : cluster 0 pgmap v896: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 5681/158469 objects degraded (3.585%); 0 B/s, 19 objects/s recovering 2023-09-23T19:39:31.846 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:31.887 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:31.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:31.892 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:39:31.967 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:31 smithi016 bash[77995]: cluster 2023-09-23T19:39:30.143594+0000 mgr.y (mgr.25325) 960 : cluster 0 pgmap v896: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 5681/158469 objects degraded (3.585%); 0 B/s, 19 objects/s recovering 2023-09-23T19:39:31.993 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:32.018 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:32.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:32.021 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:39:32.122 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:32.151 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:32.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:32.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:39:32.255 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:32.280 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:32.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:32.284 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:39:32.385 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:32.404 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-09-23T19:39:32.405 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:39:32.412 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:32.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:32.416 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:32.516 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:32.543 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:32.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:32.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:32.649 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:32.698 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:32.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:32.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:39:32.803 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:32.831 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:32.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:32.837 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:32.937 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:32.963 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:32.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:32.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:33.071 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:33.101 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:33.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:33.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:33.204 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:33.230 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:33.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:33.234 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:39:33.334 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:33.360 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:33.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:33.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:39:33.463 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:33.492 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:33.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:33.496 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:39:33.597 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:33.624 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:33.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:33.628 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:39:33.729 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:33.767 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:33.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:33.770 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:39:33.830 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:33 smithi016 bash[77995]: cluster 2023-09-23T19:39:32.144311+0000 mgr.y (mgr.25325) 961 : cluster 0 pgmap v897: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 5681/154080 objects degraded (3.687%); 0 B/s, 28 objects/s recovering 2023-09-23T19:39:33.831 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:33 smithi016 bash[75004]: cluster 2023-09-23T19:39:32.144311+0000 mgr.y (mgr.25325) 961 : cluster 0 pgmap v897: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 5681/154080 objects degraded (3.687%); 0 B/s, 28 objects/s recovering 2023-09-23T19:39:33.871 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:33.873 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:39:33.893 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:33.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:33.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:33.923 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:39:33.961 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:39:33.996 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:33.999 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:39:34.005 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:33 smithi092 bash[58855]: cluster 2023-09-23T19:39:32.144311+0000 mgr.y (mgr.25325) 961 : cluster 0 pgmap v897: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 5681/154080 objects degraded (3.687%); 0 B/s, 28 objects/s recovering 2023-09-23T19:39:34.024 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:39:34.033 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:34.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:34.041 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:34.059 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:39:34.107 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:39:34.142 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:34.154 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:39:34.164 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:39:34.164 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:39:34.164 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:39:34.164 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:39:34.164 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:39:34.164 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:39:34.164 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:39:34.165 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:39:34.178 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:34.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:34.184 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:39:34.285 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:34.324 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:34.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:34.328 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:34.429 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:34.462 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:34.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:34.465 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:39:34.565 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:34.594 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:34.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:34.601 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:39:34.688 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:34 smithi092 bash[58855]: audit 2023-09-23T19:39:33.758053+0000 mon.c (mon.2) 169 : audit 0 from='client.? 172.21.15.16:0/2093056329' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:39:34.688 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:34 smithi092 bash[58855]: cluster 2023-09-23T19:39:34.444956+0000 mon.a (mon.0) 1202 : cluster 3 Health check update: Degraded data redundancy: 5681/154080 objects degraded (3.687%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T19:39:34.702 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:34.745 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:34.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:34.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:34.851 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:34.859 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:34 smithi016 bash[77995]: audit 2023-09-23T19:39:33.758053+0000 mon.c (mon.2) 169 : audit 0 from='client.? 172.21.15.16:0/2093056329' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:39:34.860 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:34 smithi016 bash[77995]: cluster 2023-09-23T19:39:34.444956+0000 mon.a (mon.0) 1202 : cluster 3 Health check update: Degraded data redundancy: 5681/154080 objects degraded (3.687%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T19:39:34.860 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:34 smithi016 bash[75004]: audit 2023-09-23T19:39:33.758053+0000 mon.c (mon.2) 169 : audit 0 from='client.? 172.21.15.16:0/2093056329' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:39:34.860 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:34 smithi016 bash[75004]: cluster 2023-09-23T19:39:34.444956+0000 mon.a (mon.0) 1202 : cluster 3 Health check update: Degraded data redundancy: 5681/154080 objects degraded (3.687%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T19:39:34.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:34.898 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:34.898 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:39:34.999 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:35.030 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:35.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:35.035 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:39:35.135 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:35.168 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:35.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:35.172 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:35.273 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:35.304 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:35.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:35.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:35.411 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:35.437 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:35.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:35.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:39:35.540 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:35.569 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:35.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:35.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:39:35.619 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T19:39:35.675 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:35.685 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:35 smithi016 bash[75004]: cluster 2023-09-23T19:39:34.144923+0000 mgr.y (mgr.25325) 962 : cluster 0 pgmap v898: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 5681/152223 objects degraded (3.732%); 0 B/s, 19 objects/s recovering 2023-09-23T19:39:35.685 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:35 smithi016 bash[75004]: audit 2023-09-23T19:39:34.756533+0000 mon.a (mon.0) 1203 : audit 1 from='client.? 172.21.15.16:0/1631268955' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T19:39:35.686 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:39:35 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:39:35] "GET /metrics HTTP/1.1" 200 36319 "" "Prometheus/2.33.4" 2023-09-23T19:39:35.701 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T19:39:35.717 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:35.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:35.722 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:39:35.741 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:35 smithi016 bash[77995]: cluster 2023-09-23T19:39:34.144923+0000 mgr.y (mgr.25325) 962 : cluster 0 pgmap v898: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 5681/152223 objects degraded (3.732%); 0 B/s, 19 objects/s recovering 2023-09-23T19:39:35.741 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:35 smithi016 bash[77995]: audit 2023-09-23T19:39:34.756533+0000 mon.a (mon.0) 1203 : audit 1 from='client.? 172.21.15.16:0/1631268955' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T19:39:35.822 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:35.848 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:35.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:35.853 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:35.954 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:35.962 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:35 smithi092 bash[58855]: cluster 2023-09-23T19:39:34.144923+0000 mgr.y (mgr.25325) 962 : cluster 0 pgmap v898: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 5681/152223 objects degraded (3.732%); 0 B/s, 19 objects/s recovering 2023-09-23T19:39:35.962 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:35 smithi092 bash[58855]: audit 2023-09-23T19:39:34.756533+0000 mon.a (mon.0) 1203 : audit 1 from='client.? 172.21.15.16:0/1631268955' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T19:39:35.989 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:35.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:35.994 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:39:36.095 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:36.135 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:36.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:36.139 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:36.240 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:36.281 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:36.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:36.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:36.385 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:36.429 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:36.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:36.433 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:39:36.534 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:36.567 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:36.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:36.570 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:39:36.671 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:36.698 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:36.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:36.701 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:36.716 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is set 2023-09-23T19:39:36.802 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:36.837 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:36.839 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:36 smithi016 bash[77995]: audit 2023-09-23T19:39:35.689550+0000 mon.a (mon.0) 1204 : audit 1 from='client.? 172.21.15.16:0/1631268955' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T19:39:36.840 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:36 smithi016 bash[77995]: cluster 2023-09-23T19:39:35.689885+0000 mon.a (mon.0) 1205 : cluster 0 osdmap e166: 8 total, 8 up, 8 in 2023-09-23T19:39:36.840 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:36 smithi016 bash[77995]: cluster 2023-09-23T19:39:35.691949+0000 osd.7 (osd.7) 79 : cluster 3 failed to encode map e166 with expected crc 2023-09-23T19:39:36.840 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:36 smithi016 bash[77995]: audit 2023-09-23T19:39:36.114416+0000 mon.c (mon.2) 170 : audit 1 from='client.? 172.21.15.16:0/3557536829' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:39:36.840 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:36 smithi016 bash[77995]: audit 2023-09-23T19:39:36.114859+0000 mon.a (mon.0) 1206 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:39:36.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:36.841 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:36.841 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:36 smithi016 bash[75004]: audit 2023-09-23T19:39:35.689550+0000 mon.a (mon.0) 1204 : audit 1 from='client.? 172.21.15.16:0/1631268955' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T19:39:36.841 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:36 smithi016 bash[75004]: cluster 2023-09-23T19:39:35.689885+0000 mon.a (mon.0) 1205 : cluster 0 osdmap e166: 8 total, 8 up, 8 in 2023-09-23T19:39:36.841 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:36 smithi016 bash[75004]: cluster 2023-09-23T19:39:35.691949+0000 osd.7 (osd.7) 79 : cluster 3 failed to encode map e166 with expected crc 2023-09-23T19:39:36.841 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:36 smithi016 bash[75004]: audit 2023-09-23T19:39:36.114416+0000 mon.c (mon.2) 170 : audit 1 from='client.? 172.21.15.16:0/3557536829' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:39:36.841 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:36 smithi016 bash[75004]: audit 2023-09-23T19:39:36.114859+0000 mon.a (mon.0) 1206 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:39:36.941 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:36.970 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:36.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:36.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:37.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:36 smithi092 bash[58855]: audit 2023-09-23T19:39:35.689550+0000 mon.a (mon.0) 1204 : audit 1 from='client.? 172.21.15.16:0/1631268955' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T19:39:37.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:36 smithi092 bash[58855]: cluster 2023-09-23T19:39:35.689885+0000 mon.a (mon.0) 1205 : cluster 0 osdmap e166: 8 total, 8 up, 8 in 2023-09-23T19:39:37.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:36 smithi092 bash[58855]: cluster 2023-09-23T19:39:35.691949+0000 osd.7 (osd.7) 79 : cluster 3 failed to encode map e166 with expected crc 2023-09-23T19:39:37.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:36 smithi092 bash[58855]: audit 2023-09-23T19:39:36.114416+0000 mon.c (mon.2) 170 : audit 1 from='client.? 172.21.15.16:0/3557536829' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:39:37.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:36 smithi092 bash[58855]: audit 2023-09-23T19:39:36.114859+0000 mon.a (mon.0) 1206 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:39:37.073 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:37.121 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:37.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:37.127 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:39:37.227 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:37.260 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:37.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:37.262 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:37.363 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:37.386 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:37.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:37.390 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:39:37.491 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:37.520 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:37.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:37.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:39:37.627 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:37.663 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:37.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:37.668 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:39:37.769 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:37.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:37 smithi092 bash[58855]: cluster 2023-09-23T19:39:36.145466+0000 mgr.y (mgr.25325) 963 : cluster 0 pgmap v900: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 5681/144480 objects degraded (3.932%); 0 B/s, 23 objects/s recovering 2023-09-23T19:39:37.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:37 smithi092 bash[58855]: cluster 2023-09-23T19:39:36.686048+0000 mon.a (mon.0) 1207 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:39:37.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:37 smithi092 bash[58855]: audit 2023-09-23T19:39:36.706618+0000 mon.a (mon.0) 1208 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:39:37.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:37 smithi092 bash[58855]: cluster 2023-09-23T19:39:36.706819+0000 mon.a (mon.0) 1209 : cluster 0 osdmap e167: 8 total, 8 up, 8 in 2023-09-23T19:39:37.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:37 smithi092 bash[58855]: cluster 2023-09-23T19:39:36.707160+0000 osd.4 (osd.4) 100 : cluster 3 failed to encode map e167 with expected crc 2023-09-23T19:39:37.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:37 smithi092 bash[58855]: cluster 2023-09-23T19:39:36.717407+0000 osd.6 (osd.6) 89 : cluster 3 failed to encode map e167 with expected crc 2023-09-23T19:39:37.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:37 smithi092 bash[58855]: cluster 2023-09-23T19:39:36.723203+0000 osd.5 (osd.5) 98 : cluster 3 failed to encode map e167 with expected crc 2023-09-23T19:39:37.815 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:37.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:37.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:39:37.922 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:37.958 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:37.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:37.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:39:38.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:37 smithi016 bash[75004]: cluster 2023-09-23T19:39:36.145466+0000 mgr.y (mgr.25325) 963 : cluster 0 pgmap v900: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 5681/144480 objects degraded (3.932%); 0 B/s, 23 objects/s recovering 2023-09-23T19:39:38.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:37 smithi016 bash[75004]: cluster 2023-09-23T19:39:36.686048+0000 mon.a (mon.0) 1207 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:39:38.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:37 smithi016 bash[75004]: audit 2023-09-23T19:39:36.706618+0000 mon.a (mon.0) 1208 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:39:38.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:37 smithi016 bash[75004]: cluster 2023-09-23T19:39:36.706819+0000 mon.a (mon.0) 1209 : cluster 0 osdmap e167: 8 total, 8 up, 8 in 2023-09-23T19:39:38.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:37 smithi016 bash[75004]: cluster 2023-09-23T19:39:36.707160+0000 osd.4 (osd.4) 100 : cluster 3 failed to encode map e167 with expected crc 2023-09-23T19:39:38.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:37 smithi016 bash[75004]: cluster 2023-09-23T19:39:36.717407+0000 osd.6 (osd.6) 89 : cluster 3 failed to encode map e167 with expected crc 2023-09-23T19:39:38.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:37 smithi016 bash[75004]: cluster 2023-09-23T19:39:36.723203+0000 osd.5 (osd.5) 98 : cluster 3 failed to encode map e167 with expected crc 2023-09-23T19:39:38.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:37 smithi016 bash[77995]: cluster 2023-09-23T19:39:36.145466+0000 mgr.y (mgr.25325) 963 : cluster 0 pgmap v900: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 5681/144480 objects degraded (3.932%); 0 B/s, 23 objects/s recovering 2023-09-23T19:39:38.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:37 smithi016 bash[77995]: cluster 2023-09-23T19:39:36.686048+0000 mon.a (mon.0) 1207 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:39:38.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:37 smithi016 bash[77995]: audit 2023-09-23T19:39:36.706618+0000 mon.a (mon.0) 1208 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:39:38.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:37 smithi016 bash[77995]: cluster 2023-09-23T19:39:36.706819+0000 mon.a (mon.0) 1209 : cluster 0 osdmap e167: 8 total, 8 up, 8 in 2023-09-23T19:39:38.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:37 smithi016 bash[77995]: cluster 2023-09-23T19:39:36.707160+0000 osd.4 (osd.4) 100 : cluster 3 failed to encode map e167 with expected crc 2023-09-23T19:39:38.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:37 smithi016 bash[77995]: cluster 2023-09-23T19:39:36.717407+0000 osd.6 (osd.6) 89 : cluster 3 failed to encode map e167 with expected crc 2023-09-23T19:39:38.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:37 smithi016 bash[77995]: cluster 2023-09-23T19:39:36.723203+0000 osd.5 (osd.5) 98 : cluster 3 failed to encode map e167 with expected crc 2023-09-23T19:39:38.064 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:38.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:38.104 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:38.104 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:39:38.205 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:38.236 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:38.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:38.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:39:38.336 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T19:39:38.339 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:38.409 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:38.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:38.414 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:39:38.515 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:38.555 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:38.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:38.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:39:38.658 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:38.692 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:38.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:38.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:38.795 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:38.831 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:38.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:38.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:38.913 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:39:38.934 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:38.963 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:38.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:38.967 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:39:39.068 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:39.097 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:39.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:39.104 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:39.204 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:39.233 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:39.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:39.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:39:39.341 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:39.369 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:39.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:39.374 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:39:39.475 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:39.524 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:39.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:39.528 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:39:39.629 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:39.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:39.680 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:39.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:39.780 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:39.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:39 smithi092 bash[58855]: cluster 2023-09-23T19:39:38.146339+0000 mgr.y (mgr.25325) 964 : cluster 0 pgmap v902: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 691 op/s; 5681/141864 objects degraded (4.005%); 0 B/s, 28 objects/s recovering 2023-09-23T19:39:39.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:39 smithi092 bash[58855]: cluster 2023-09-23T19:39:39.446064+0000 mon.a (mon.0) 1210 : cluster 3 Health check update: Degraded data redundancy: 5681/141864 objects degraded (4.005%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T19:39:39.826 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:39.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:39.830 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:39:39.930 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:39.939 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:39 smithi016 bash[77995]: cluster 2023-09-23T19:39:38.146339+0000 mgr.y (mgr.25325) 964 : cluster 0 pgmap v902: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 691 op/s; 5681/141864 objects degraded (4.005%); 0 B/s, 28 objects/s recovering 2023-09-23T19:39:39.939 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:39 smithi016 bash[77995]: cluster 2023-09-23T19:39:39.446064+0000 mon.a (mon.0) 1210 : cluster 3 Health check update: Degraded data redundancy: 5681/141864 objects degraded (4.005%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T19:39:39.939 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:39 smithi016 bash[75004]: cluster 2023-09-23T19:39:38.146339+0000 mgr.y (mgr.25325) 964 : cluster 0 pgmap v902: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 691 op/s; 5681/141864 objects degraded (4.005%); 0 B/s, 28 objects/s recovering 2023-09-23T19:39:39.940 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:39 smithi016 bash[75004]: cluster 2023-09-23T19:39:39.446064+0000 mon.a (mon.0) 1210 : cluster 3 Health check update: Degraded data redundancy: 5681/141864 objects degraded (4.005%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T19:39:39.962 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:39.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:39.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:39:40.070 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:40.099 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:40.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:40.105 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:39:40.206 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:40.239 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:40.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:40.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:39:40.342 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:40.386 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:40.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:40.391 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:40.439 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:39:40.491 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:40.537 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:40.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:40.540 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:39:40.641 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:40.680 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:40.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:40.685 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:39:40.786 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:40.825 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:40.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:40.828 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:39:40.928 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:40.944 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T19:39:40.953 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:40.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:40.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:41.056 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:41.106 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:41.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:41.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:39:41.209 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:41.234 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:41.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:41.236 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:39:41.337 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:41.374 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:41.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:41.381 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:39:41.481 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:41.513 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:41.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:41.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:39:41.618 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:41.671 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:41.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:41.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:39:41.773 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:41.783 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:41 smithi016 bash[75004]: cluster 2023-09-23T19:39:40.146897+0000 mgr.y (mgr.25325) 965 : cluster 0 pgmap v903: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 5681/138018 objects degraded (4.116%); 0 B/s, 14 objects/s recovering 2023-09-23T19:39:41.783 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:41 smithi016 bash[77995]: cluster 2023-09-23T19:39:40.146897+0000 mgr.y (mgr.25325) 965 : cluster 0 pgmap v903: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 5681/138018 objects degraded (4.116%); 0 B/s, 14 objects/s recovering 2023-09-23T19:39:41.814 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:41.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:41.817 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:39:41.918 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:41.953 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:41.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:41.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:39:42.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:41 smithi092 bash[58855]: cluster 2023-09-23T19:39:40.146897+0000 mgr.y (mgr.25325) 965 : cluster 0 pgmap v903: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 5681/138018 objects degraded (4.116%); 0 B/s, 14 objects/s recovering 2023-09-23T19:39:42.060 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:42.089 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:42.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:42.091 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:42.192 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:42.241 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:42.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:42.246 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:42.346 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:42.378 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:42.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:42.380 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:39:42.481 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:42.518 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:42.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:42.522 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:42.623 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:42.654 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:42.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:42.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:42.758 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:42.793 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:42.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:42.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:42.897 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:42.924 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:42.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:42.932 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:39:43.033 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:43.058 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:43.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:43.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:39:43.165 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:43.208 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:39:43.208 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:39:43.209 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:43.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:43.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:43.317 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:43.360 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:43.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:43.367 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:43.468 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:43.496 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:43.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:43.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:39:43.602 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:43.632 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:43.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:43.636 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:39:43.737 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:43.746 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:43 smithi016 bash[77995]: cluster 2023-09-23T19:39:42.147820+0000 mgr.y (mgr.25325) 966 : cluster 0 pgmap v904: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 5681/133455 objects degraded (4.257%); 0 B/s, 28 objects/s recovering 2023-09-23T19:39:43.746 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:43 smithi016 bash[77995]: audit 2023-09-23T19:39:43.666354+0000 mon.a (mon.0) 1211 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:39:43.746 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:43 smithi016 bash[77995]: audit 2023-09-23T19:39:43.667649+0000 mon.a (mon.0) 1212 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:39:43.747 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:43 smithi016 bash[75004]: cluster 2023-09-23T19:39:42.147820+0000 mgr.y (mgr.25325) 966 : cluster 0 pgmap v904: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 5681/133455 objects degraded (4.257%); 0 B/s, 28 objects/s recovering 2023-09-23T19:39:43.747 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:43 smithi016 bash[75004]: audit 2023-09-23T19:39:43.666354+0000 mon.a (mon.0) 1211 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:39:43.747 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:43 smithi016 bash[75004]: audit 2023-09-23T19:39:43.667649+0000 mon.a (mon.0) 1212 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:39:43.770 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:43.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:43.777 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:39:43.870 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:43 smithi092 bash[58855]: cluster 2023-09-23T19:39:42.147820+0000 mgr.y (mgr.25325) 966 : cluster 0 pgmap v904: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 5681/133455 objects degraded (4.257%); 0 B/s, 28 objects/s recovering 2023-09-23T19:39:43.871 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:43 smithi092 bash[58855]: audit 2023-09-23T19:39:43.666354+0000 mon.a (mon.0) 1211 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:39:43.871 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:43 smithi092 bash[58855]: audit 2023-09-23T19:39:43.667649+0000 mon.a (mon.0) 1212 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:39:43.877 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:43.910 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:43.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:43.914 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:39:44.014 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:44.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:44.047 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:44.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:44.148 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:44.182 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:44.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:44.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:44.288 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:44.316 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:44.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:44.319 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:44.419 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:44.454 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:44.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:44.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:44.560 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:44.586 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:44.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:44.589 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:44.689 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:44.721 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:44.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:44.724 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:39:44.727 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:44 smithi016 bash[75004]: cluster 2023-09-23T19:39:44.447188+0000 mon.a (mon.0) 1213 : cluster 3 Health check update: Degraded data redundancy: 5681/133455 objects degraded (4.257%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T19:39:44.825 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:44.861 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:44.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:44.867 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:39:44.968 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:44.971 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:44 smithi092 bash[58855]: cluster 2023-09-23T19:39:44.447188+0000 mon.a (mon.0) 1213 : cluster 3 Health check update: Degraded data redundancy: 5681/133455 objects degraded (4.257%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T19:39:44.977 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:44 smithi016 bash[77995]: cluster 2023-09-23T19:39:44.447188+0000 mon.a (mon.0) 1213 : cluster 3 Health check update: Degraded data redundancy: 5681/133455 objects degraded (4.257%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T19:39:45.003 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:45.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:45.006 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:45.106 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:45.133 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:45.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:45.138 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:39:45.238 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:45.243 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:39:45.293 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:45.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:45.298 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:45.399 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:45.424 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:45.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:45.427 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:45.528 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:45.563 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:45.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:45.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:39:45.668 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:45.694 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:45.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:45.699 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:39:45.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:45 smithi092 bash[58855]: cluster 2023-09-23T19:39:44.148472+0000 mgr.y (mgr.25325) 967 : cluster 0 pgmap v905: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 561 op/s; 5681/131562 objects degraded (4.318%); 0 B/s, 13 objects/s recovering 2023-09-23T19:39:45.799 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:45.836 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:45.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:45.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:45.939 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:45.970 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:45.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:45.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:39:46.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:45 smithi016 bash[77995]: cluster 2023-09-23T19:39:44.148472+0000 mgr.y (mgr.25325) 967 : cluster 0 pgmap v905: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 561 op/s; 5681/131562 objects degraded (4.318%); 0 B/s, 13 objects/s recovering 2023-09-23T19:39:46.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:45 smithi016 bash[75004]: cluster 2023-09-23T19:39:44.148472+0000 mgr.y (mgr.25325) 967 : cluster 0 pgmap v905: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 561 op/s; 5681/131562 objects degraded (4.318%); 0 B/s, 13 objects/s recovering 2023-09-23T19:39:46.036 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:39:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:39:45] "GET /metrics HTTP/1.1" 200 36314 "" "Prometheus/2.33.4" 2023-09-23T19:39:46.074 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:46.107 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:46.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:46.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:46.217 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:46.249 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:46.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:46.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:39:46.355 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:46.402 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:46.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:46.404 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:39:46.505 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:46.518 INFO:teuthology.orchestra.run.smithi092.stdout:{ 2023-09-23T19:39:46.518 INFO:teuthology.orchestra.run.smithi092.stdout: "ops": [ 2023-09-23T19:39:46.518 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:39:46.518 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_repop(client.45947.0:180369 10.4 e167/144 10:256953e1:::benchmark_data_smithi016_136365_object70235:head v 167'10940, mlcod=167'10937)", 2023-09-23T19:39:46.518 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:39:46.517958+0000", 2023-09-23T19:39:46.518 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 0.00200045, 2023-09-23T19:39:46.518 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0020283609999999998, 2023-09-23T19:39:46.518 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:39:46.519 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "started", 2023-09-23T19:39:46.519 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:39:46.519 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:39:46.519 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:39:46.519 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:39:46.517958+0000", 2023-09-23T19:39:46.519 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:39:46.519 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:39:46.519 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:39:46.519 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:39:46.519 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:39:46.517958+0000", 2023-09-23T19:39:46.519 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T19:39:46.520 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:39:46.520 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:39:46.520 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:39:46.520 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:39:46.517957+0000", 2023-09-23T19:39:46.520 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.2768999999999999e-05 2023-09-23T19:39:46.520 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:39:46.520 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:39:46.520 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:39:46.520 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:39:46.517970+0000", 2023-09-23T19:39:46.520 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.8267e-05 2023-09-23T19:39:46.520 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:39:46.521 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:39:46.521 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:39:46.521 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:39:46.517988+0000", 2023-09-23T19:39:46.521 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.7689999999999996e-06 2023-09-23T19:39:46.521 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:39:46.521 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:39:46.521 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:39:46.521 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:39:46.517996+0000", 2023-09-23T19:39:46.521 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00013123 2023-09-23T19:39:46.521 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:39:46.521 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:39:46.521 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:39:46.522 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:39:46.518127+0000", 2023-09-23T19:39:46.522 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.6208e-05 2023-09-23T19:39:46.522 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:39:46.522 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:39:46.522 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:39:46.522 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:39:46.518153+0000", 2023-09-23T19:39:46.522 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00019478599999999999 2023-09-23T19:39:46.522 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:39:46.522 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:39:46.522 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:39:46.522 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:39:46.522 INFO:teuthology.orchestra.run.smithi092.stdout: ], 2023-09-23T19:39:46.523 INFO:teuthology.orchestra.run.smithi092.stdout: "num_ops": 1 2023-09-23T19:39:46.523 INFO:teuthology.orchestra.run.smithi092.stdout:} 2023-09-23T19:39:46.543 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:46.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:46.549 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:39:46.652 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:46.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:46.691 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:46.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:46.792 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:46.826 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:46.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:46.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:46.931 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:46.972 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:46.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:46.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:39:47.076 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:47.115 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:47.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:47.120 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:47.221 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:47.249 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:47.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:47.253 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:39:47.354 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:47.401 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:47.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:47.406 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:47.507 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:47.539 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:47.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:47.542 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:39:47.599 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T19:39:47.642 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:47.676 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:47.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:47.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:39:47.780 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:47.811 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:47.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:47.816 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:47.883 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:47 smithi016 bash[75004]: cluster 2023-09-23T19:39:46.152781+0000 mgr.y (mgr.25325) 968 : cluster 0 pgmap v906: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 687 op/s; 5681/123843 objects degraded (4.587%); 0 B/s, 23 objects/s recovering 2023-09-23T19:39:47.884 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:47 smithi016 bash[77995]: cluster 2023-09-23T19:39:46.152781+0000 mgr.y (mgr.25325) 968 : cluster 0 pgmap v906: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 687 op/s; 5681/123843 objects degraded (4.587%); 0 B/s, 23 objects/s recovering 2023-09-23T19:39:47.917 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:47.967 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:47.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:47.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:48.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:47 smithi092 bash[58855]: cluster 2023-09-23T19:39:46.152781+0000 mgr.y (mgr.25325) 968 : cluster 0 pgmap v906: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 687 op/s; 5681/123843 objects degraded (4.587%); 0 B/s, 23 objects/s recovering 2023-09-23T19:39:48.073 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:48.104 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:48.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:48.111 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:39:48.212 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:48.248 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:48.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:48.252 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:39:48.353 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:48.385 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:48.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:48.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:39:48.488 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:48.516 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:48.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:48.520 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:48.620 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:48.644 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:48.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:48.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:39:48.749 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:48.784 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:48.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:48.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:39:48.892 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:48.921 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:48.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:48.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:39:49.024 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:49.058 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:49.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:49.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:39:49.161 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:49.194 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:49.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:49.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:39:49.298 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:49.323 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:49.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:49.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:49.427 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:49.465 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:49.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:49.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:49.573 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:49.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:49.608 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:49.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:39:49.709 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:49.753 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:49.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:49.759 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:49.859 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:49.891 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:49.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:49.896 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:49.997 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:50.006 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:49 smithi092 bash[58855]: cluster 2023-09-23T19:39:48.153536+0000 mgr.y (mgr.25325) 969 : cluster 0 pgmap v907: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 5681/121233 objects degraded (4.686%); 0 B/s, 20 objects/s recovering 2023-09-23T19:39:50.006 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:49 smithi092 bash[58855]: cluster 2023-09-23T19:39:49.448519+0000 mon.a (mon.0) 1214 : cluster 3 Health check update: Degraded data redundancy: 5681/121233 objects degraded (4.686%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:39:50.029 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:50.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:50.034 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:50.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:49 smithi016 bash[75004]: cluster 2023-09-23T19:39:48.153536+0000 mgr.y (mgr.25325) 969 : cluster 0 pgmap v907: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 5681/121233 objects degraded (4.686%); 0 B/s, 20 objects/s recovering 2023-09-23T19:39:50.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:49 smithi016 bash[75004]: cluster 2023-09-23T19:39:49.448519+0000 mon.a (mon.0) 1214 : cluster 3 Health check update: Degraded data redundancy: 5681/121233 objects degraded (4.686%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:39:50.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:49 smithi016 bash[77995]: cluster 2023-09-23T19:39:48.153536+0000 mgr.y (mgr.25325) 969 : cluster 0 pgmap v907: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 5681/121233 objects degraded (4.686%); 0 B/s, 20 objects/s recovering 2023-09-23T19:39:50.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:49 smithi016 bash[77995]: cluster 2023-09-23T19:39:49.448519+0000 mon.a (mon.0) 1214 : cluster 3 Health check update: Degraded data redundancy: 5681/121233 objects degraded (4.686%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:39:50.135 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:50.171 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:50.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:50.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:50.278 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:50.310 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:50.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:50.312 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:39:50.413 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:50.451 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:50.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:50.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:50.560 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:50.601 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:50.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:50.610 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:39:50.711 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:50.735 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:50.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:50.739 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:39:50.840 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:50.889 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:50.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:50.891 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:39:50.992 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:51.019 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:51.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:51.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:51.123 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:51.146 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:51.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:51.148 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:51.249 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:51.289 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:51.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:51.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:39:51.395 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:51.420 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:51.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:51.424 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:51.525 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:51.548 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:51.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:51.551 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:39:51.651 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:51.681 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:51.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:51.684 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:51.785 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:51.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:51 smithi092 bash[58855]: cluster 2023-09-23T19:39:50.154227+0000 mgr.y (mgr.25325) 970 : cluster 0 pgmap v908: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 5681/117237 objects degraded (4.846%); 0 B/s, 19 objects/s recovering 2023-09-23T19:39:51.845 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:51.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:51.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:51.949 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:51.978 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:51.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:51.981 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:39:52.007 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:51 smithi016 bash[75004]: cluster 2023-09-23T19:39:50.154227+0000 mgr.y (mgr.25325) 970 : cluster 0 pgmap v908: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 5681/117237 objects degraded (4.846%); 0 B/s, 19 objects/s recovering 2023-09-23T19:39:52.008 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:51 smithi016 bash[77995]: cluster 2023-09-23T19:39:50.154227+0000 mgr.y (mgr.25325) 970 : cluster 0 pgmap v908: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 5681/117237 objects degraded (4.846%); 0 B/s, 19 objects/s recovering 2023-09-23T19:39:52.082 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:52.117 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:52.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:52.123 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:39:52.224 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:52.251 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:52.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:52.257 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:52.358 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:52.385 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:52.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:52.389 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:52.423 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:39:52.489 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:52.535 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:52.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:52.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:52.638 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:52.676 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:52.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:52.680 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:39:52.781 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:52.810 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:52.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:52.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:39:52.913 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:52.946 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:52.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:52.949 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:39:53.050 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:53.090 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:53.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:53.096 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:53.197 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:53.222 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:53.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:53.228 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:53.329 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:53.359 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:53.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:53.361 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:53.461 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:53.500 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:53.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:53.506 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:39:53.606 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:53.629 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:53.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:53.635 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:53.736 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:53.777 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:53.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:53.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:39:53.881 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:53.889 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:53 smithi092 bash[58855]: cluster 2023-09-23T19:39:52.155245+0000 mgr.y (mgr.25325) 971 : cluster 0 pgmap v909: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 5681/112824 objects degraded (5.035%); 0 B/s, 28 objects/s recovering 2023-09-23T19:39:53.910 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:53.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:53.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:39:54.017 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:54.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:53 smithi016 bash[75004]: cluster 2023-09-23T19:39:52.155245+0000 mgr.y (mgr.25325) 971 : cluster 0 pgmap v909: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 5681/112824 objects degraded (5.035%); 0 B/s, 28 objects/s recovering 2023-09-23T19:39:54.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:53 smithi016 bash[77995]: cluster 2023-09-23T19:39:52.155245+0000 mgr.y (mgr.25325) 971 : cluster 0 pgmap v909: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 5681/112824 objects degraded (5.035%); 0 B/s, 28 objects/s recovering 2023-09-23T19:39:54.041 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:54.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:54.046 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:39:54.147 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:54.191 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:54.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:54.194 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:54.269 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:39:54.297 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:54.318 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:39:54.333 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:54.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:54.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:39:54.361 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:39:54.411 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:39:54.438 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:54.451 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:39:54.461 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:54.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:54.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:39:54.499 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:39:54.534 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:39:54.565 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:54.578 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:39:54.586 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:39:54.586 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:39:54.586 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:39:54.586 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:39:54.586 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:39:54.587 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:39:54.587 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:39:54.587 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:39:54.587 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:54.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:54.596 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:54.697 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:54.743 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:54.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:54.748 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:54.849 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:54.885 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:54.886 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:54 smithi092 bash[58855]: audit 2023-09-23T19:39:54.192816+0000 mon.c (mon.2) 171 : audit 0 from='client.? 172.21.15.16:0/3660877093' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:39:54.887 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:54 smithi092 bash[58855]: cluster 2023-09-23T19:39:54.452642+0000 mon.a (mon.0) 1215 : cluster 3 Health check update: Degraded data redundancy: 5681/112824 objects degraded (5.035%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:39:54.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:54.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:54.890 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:54 smithi016 bash[75004]: audit 2023-09-23T19:39:54.192816+0000 mon.c (mon.2) 171 : audit 0 from='client.? 172.21.15.16:0/3660877093' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:39:54.890 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:54 smithi016 bash[75004]: cluster 2023-09-23T19:39:54.452642+0000 mon.a (mon.0) 1215 : cluster 3 Health check update: Degraded data redundancy: 5681/112824 objects degraded (5.035%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:39:54.890 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:54 smithi016 bash[77995]: audit 2023-09-23T19:39:54.192816+0000 mon.c (mon.2) 171 : audit 0 from='client.? 172.21.15.16:0/3660877093' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:39:54.890 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:54 smithi016 bash[77995]: cluster 2023-09-23T19:39:54.452642+0000 mon.a (mon.0) 1215 : cluster 3 Health check update: Degraded data redundancy: 5681/112824 objects degraded (5.035%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:39:54.991 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:55.034 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:55.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:55.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:55.139 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:55.178 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:55.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:55.185 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:55.286 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:55.323 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:55.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:55.324 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:55.425 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:55.454 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:55.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:55.457 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:39:55.557 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:55.585 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:55.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:55.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:55.690 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:55.719 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:55.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:55.722 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:55.789 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is set 2023-09-23T19:39:55.795 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T19:39:55.824 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:55.826 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:55 smithi016 bash[75004]: cluster 2023-09-23T19:39:54.156002+0000 mgr.y (mgr.25325) 972 : cluster 0 pgmap v910: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 627 op/s; 5681/110838 objects degraded (5.125%); 0 B/s, 19 objects/s recovering 2023-09-23T19:39:55.826 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:55 smithi016 bash[75004]: audit 2023-09-23T19:39:54.993311+0000 mon.a (mon.0) 1216 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:39:55.826 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:55 smithi016 bash[75004]: audit 2023-09-23T19:39:54.998216+0000 mon.b (mon.1) 45 : audit 1 from='client.? 172.21.15.16:0/2284132101' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:39:55.827 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:55 smithi016 bash[75004]: audit 2023-09-23T19:39:55.102214+0000 mon.c (mon.2) 172 : audit 1 from='client.? 172.21.15.16:0/2128761054' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T19:39:55.827 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:55 smithi016 bash[75004]: audit 2023-09-23T19:39:55.102912+0000 mon.a (mon.0) 1217 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T19:39:55.827 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:39:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:39:55] "GET /metrics HTTP/1.1" 200 36314 "" "Prometheus/2.33.4" 2023-09-23T19:39:55.827 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:55 smithi016 bash[77995]: cluster 2023-09-23T19:39:54.156002+0000 mgr.y (mgr.25325) 972 : cluster 0 pgmap v910: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 627 op/s; 5681/110838 objects degraded (5.125%); 0 B/s, 19 objects/s recovering 2023-09-23T19:39:55.827 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:55 smithi016 bash[77995]: audit 2023-09-23T19:39:54.993311+0000 mon.a (mon.0) 1216 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:39:55.827 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:55 smithi016 bash[77995]: audit 2023-09-23T19:39:54.998216+0000 mon.b (mon.1) 45 : audit 1 from='client.? 172.21.15.16:0/2284132101' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:39:55.829 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:55 smithi016 bash[77995]: audit 2023-09-23T19:39:55.102214+0000 mon.c (mon.2) 172 : audit 1 from='client.? 172.21.15.16:0/2128761054' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T19:39:55.829 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:55 smithi016 bash[77995]: audit 2023-09-23T19:39:55.102912+0000 mon.a (mon.0) 1217 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T19:39:55.874 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:55.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:55.879 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:39:55.980 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:56.004 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:56.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:56.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:39:56.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:55 smithi092 bash[58855]: cluster 2023-09-23T19:39:54.156002+0000 mgr.y (mgr.25325) 972 : cluster 0 pgmap v910: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 627 op/s; 5681/110838 objects degraded (5.125%); 0 B/s, 19 objects/s recovering 2023-09-23T19:39:56.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:55 smithi092 bash[58855]: audit 2023-09-23T19:39:54.993311+0000 mon.a (mon.0) 1216 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:39:56.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:55 smithi092 bash[58855]: audit 2023-09-23T19:39:54.998216+0000 mon.b (mon.1) 45 : audit 1 from='client.? 172.21.15.16:0/2284132101' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:39:56.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:55 smithi092 bash[58855]: audit 2023-09-23T19:39:55.102214+0000 mon.c (mon.2) 172 : audit 1 from='client.? 172.21.15.16:0/2128761054' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T19:39:56.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:55 smithi092 bash[58855]: audit 2023-09-23T19:39:55.102912+0000 mon.a (mon.0) 1217 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T19:39:56.114 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:56.157 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:56.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:56.164 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:56.265 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:56.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:56.292 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:56.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:56.393 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:56.426 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:56.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:56.432 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:56.532 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:56.556 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:56.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:56.561 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:39:56.661 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:56.686 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:56.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:56.689 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:56.790 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:56.798 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:56 smithi092 bash[58855]: cluster 2023-09-23T19:39:55.765231+0000 mon.a (mon.0) 1218 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:39:56.798 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:56 smithi092 bash[58855]: audit 2023-09-23T19:39:55.780029+0000 mon.a (mon.0) 1219 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:39:56.799 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:56 smithi092 bash[58855]: audit 2023-09-23T19:39:55.780317+0000 mon.a (mon.0) 1220 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T19:39:56.799 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:56 smithi092 bash[58855]: cluster 2023-09-23T19:39:55.780523+0000 mon.a (mon.0) 1221 : cluster 0 osdmap e168: 8 total, 8 up, 8 in 2023-09-23T19:39:56.799 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:56 smithi092 bash[58855]: cluster 2023-09-23T19:39:55.783602+0000 osd.4 (osd.4) 101 : cluster 3 failed to encode map e168 with expected crc 2023-09-23T19:39:56.799 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:56 smithi092 bash[58855]: cluster 2023-09-23T19:39:55.785742+0000 osd.4 (osd.4) 102 : cluster 3 failed to encode map e168 with expected crc 2023-09-23T19:39:56.799 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:56 smithi092 bash[58855]: cluster 2023-09-23T19:39:55.786724+0000 osd.6 (osd.6) 90 : cluster 3 failed to encode map e168 with expected crc 2023-09-23T19:39:56.799 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:56 smithi092 bash[58855]: cluster 2023-09-23T19:39:55.800575+0000 osd.5 (osd.5) 99 : cluster 3 failed to encode map e168 with expected crc 2023-09-23T19:39:56.818 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:56.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:56.824 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:39:56.925 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:56.935 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:56 smithi016 bash[77995]: cluster 2023-09-23T19:39:55.765231+0000 mon.a (mon.0) 1218 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:39:56.935 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:56 smithi016 bash[77995]: audit 2023-09-23T19:39:55.780029+0000 mon.a (mon.0) 1219 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:39:56.935 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:56 smithi016 bash[77995]: audit 2023-09-23T19:39:55.780317+0000 mon.a (mon.0) 1220 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T19:39:56.935 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:56 smithi016 bash[77995]: cluster 2023-09-23T19:39:55.780523+0000 mon.a (mon.0) 1221 : cluster 0 osdmap e168: 8 total, 8 up, 8 in 2023-09-23T19:39:56.935 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:56 smithi016 bash[77995]: cluster 2023-09-23T19:39:55.783602+0000 osd.4 (osd.4) 101 : cluster 3 failed to encode map e168 with expected crc 2023-09-23T19:39:56.935 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:56 smithi016 bash[77995]: cluster 2023-09-23T19:39:55.785742+0000 osd.4 (osd.4) 102 : cluster 3 failed to encode map e168 with expected crc 2023-09-23T19:39:56.936 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:56 smithi016 bash[77995]: cluster 2023-09-23T19:39:55.786724+0000 osd.6 (osd.6) 90 : cluster 3 failed to encode map e168 with expected crc 2023-09-23T19:39:56.936 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:56 smithi016 bash[77995]: cluster 2023-09-23T19:39:55.800575+0000 osd.5 (osd.5) 99 : cluster 3 failed to encode map e168 with expected crc 2023-09-23T19:39:56.936 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:56 smithi016 bash[75004]: cluster 2023-09-23T19:39:55.765231+0000 mon.a (mon.0) 1218 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:39:56.936 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:56 smithi016 bash[75004]: audit 2023-09-23T19:39:55.780029+0000 mon.a (mon.0) 1219 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:39:56.936 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:56 smithi016 bash[75004]: audit 2023-09-23T19:39:55.780317+0000 mon.a (mon.0) 1220 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T19:39:56.936 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:56 smithi016 bash[75004]: cluster 2023-09-23T19:39:55.780523+0000 mon.a (mon.0) 1221 : cluster 0 osdmap e168: 8 total, 8 up, 8 in 2023-09-23T19:39:56.936 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:56 smithi016 bash[75004]: cluster 2023-09-23T19:39:55.783602+0000 osd.4 (osd.4) 101 : cluster 3 failed to encode map e168 with expected crc 2023-09-23T19:39:56.936 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:56 smithi016 bash[75004]: cluster 2023-09-23T19:39:55.785742+0000 osd.4 (osd.4) 102 : cluster 3 failed to encode map e168 with expected crc 2023-09-23T19:39:56.937 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:56 smithi016 bash[75004]: cluster 2023-09-23T19:39:55.786724+0000 osd.6 (osd.6) 90 : cluster 3 failed to encode map e168 with expected crc 2023-09-23T19:39:56.937 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:56 smithi016 bash[75004]: cluster 2023-09-23T19:39:55.800575+0000 osd.5 (osd.5) 99 : cluster 3 failed to encode map e168 with expected crc 2023-09-23T19:39:56.961 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:56.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:56.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:39:57.067 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:57.095 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:57.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:57.103 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:39:57.204 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:57.236 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:57.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:57.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:39:57.342 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:57.386 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:57.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:57.390 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:39:57.490 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:57.525 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:39:57.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:57.530 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:39:57.532 INFO:teuthology.orchestra.run.smithi092.stdout:ERROR: (22) Invalid argument 2023-09-23T19:39:57.532 INFO:teuthology.orchestra.run.smithi092.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:39:57.631 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:57.660 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:57.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:57.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:57.706 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T19:39:57.764 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:57.847 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:39:57.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:57.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:39:57.953 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:57.962 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:57 smithi092 bash[58855]: cluster 2023-09-23T19:39:56.156719+0000 mgr.y (mgr.25325) 973 : cluster 0 pgmap v912: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 5681/103467 objects degraded (5.491%); 0 B/s, 23 objects/s recovering 2023-09-23T19:39:57.982 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:57.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:57.987 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:39:57.988 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:39:58.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:57 smithi016 bash[77995]: cluster 2023-09-23T19:39:56.156719+0000 mgr.y (mgr.25325) 973 : cluster 0 pgmap v912: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 5681/103467 objects degraded (5.491%); 0 B/s, 23 objects/s recovering 2023-09-23T19:39:58.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:57 smithi016 bash[75004]: cluster 2023-09-23T19:39:56.156719+0000 mgr.y (mgr.25325) 973 : cluster 0 pgmap v912: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 5681/103467 objects degraded (5.491%); 0 B/s, 23 objects/s recovering 2023-09-23T19:39:58.088 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:58.115 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:58.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:58.121 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:58.222 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:58.263 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:58.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:58.267 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:39:58.367 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:58.392 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:39:58.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:58.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:39:58.496 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:58.538 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:58.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:58.543 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:39:58.643 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:58.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:58.674 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:58.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:39:58.751 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T19:39:58.775 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:58.824 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:39:58.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:58.831 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:39:58.932 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:58.960 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:58.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:58.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:39:59.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:58 smithi092 bash[58855]: audit 2023-09-23T19:39:58.657466+0000 mon.a (mon.0) 1222 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:39:59.063 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:59.103 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:59.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:59.104 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:59.204 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:59.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:59.239 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:59.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:39:59.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:58 smithi016 bash[77995]: audit 2023-09-23T19:39:58.657466+0000 mon.a (mon.0) 1222 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:39:59.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:58 smithi016 bash[75004]: audit 2023-09-23T19:39:58.657466+0000 mon.a (mon.0) 1222 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:39:59.340 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:59.375 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:59.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:59.378 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:39:59.478 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:59.524 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:39:59.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:59.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:39:59.629 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:59.656 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:39:59.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:59.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:39:59.696 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T19:39:59.760 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:59.789 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:39:59.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:59.792 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:39:59.892 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:59.934 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:39:59.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:39:59.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:40:00.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:59 smithi016 bash[75004]: cluster 2023-09-23T19:39:58.157666+0000 mgr.y (mgr.25325) 974 : cluster 0 pgmap v913: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 5681/101022 objects degraded (5.624%); 0 B/s, 23 objects/s recovering 2023-09-23T19:40:00.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:39:59 smithi016 bash[75004]: cluster 2023-09-23T19:39:59.453792+0000 mon.a (mon.0) 1223 : cluster 3 Health check update: Degraded data redundancy: 5681/101022 objects degraded (5.624%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:40:00.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:59 smithi016 bash[77995]: cluster 2023-09-23T19:39:58.157666+0000 mgr.y (mgr.25325) 974 : cluster 0 pgmap v913: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 5681/101022 objects degraded (5.624%); 0 B/s, 23 objects/s recovering 2023-09-23T19:40:00.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:39:59 smithi016 bash[77995]: cluster 2023-09-23T19:39:59.453792+0000 mon.a (mon.0) 1223 : cluster 3 Health check update: Degraded data redundancy: 5681/101022 objects degraded (5.624%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:40:00.037 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:00.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:59 smithi092 bash[58855]: cluster 2023-09-23T19:39:58.157666+0000 mgr.y (mgr.25325) 974 : cluster 0 pgmap v913: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 5681/101022 objects degraded (5.624%); 0 B/s, 23 objects/s recovering 2023-09-23T19:40:00.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:39:59 smithi092 bash[58855]: cluster 2023-09-23T19:39:59.453792+0000 mon.a (mon.0) 1223 : cluster 3 Health check update: Degraded data redundancy: 5681/101022 objects degraded (5.624%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:40:00.068 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:00.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:00.073 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:00.173 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:00.215 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:00.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:00.222 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:40:00.322 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:00.360 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:00.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:00.365 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:40:00.465 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:00.497 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:00.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:00.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:00.604 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:00.638 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:00.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:00.641 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:40:00.742 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:00.790 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:00.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:00.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:00.898 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:00.921 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:00.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:00.924 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:40:01.025 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:01.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:00 smithi092 bash[58855]: cluster 2023-09-23T19:40:00.000113+0000 mon.a (mon.0) 1224 : cluster 3 Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; Degraded data redundancy: 5681/101022 objects degraded (5.624%), 4 pgs degraded, 5 pgs undersized 2023-09-23T19:40:01.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:00 smithi092 bash[58855]: cluster 2023-09-23T19:40:00.000145+0000 mon.a (mon.0) 1225 : cluster 3 [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2023-09-23T19:40:01.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:00 smithi092 bash[58855]: cluster 2023-09-23T19:40:00.000157+0000 mon.a (mon.0) 1226 : cluster 3 [WRN] PG_DEGRADED: Degraded data redundancy: 5681/101022 objects degraded (5.624%), 4 pgs degraded, 5 pgs undersized 2023-09-23T19:40:01.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:00 smithi092 bash[58855]: cluster 2023-09-23T19:40:00.000164+0000 mon.a (mon.0) 1227 : cluster 3 pg 10.3 is stuck undersized for 73s, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,5] 2023-09-23T19:40:01.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:00 smithi092 bash[58855]: cluster 2023-09-23T19:40:00.000171+0000 mon.a (mon.0) 1228 : cluster 3 pg 10.5 is stuck undersized for 73s, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,3] 2023-09-23T19:40:01.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:00 smithi092 bash[58855]: cluster 2023-09-23T19:40:00.000182+0000 mon.a (mon.0) 1229 : cluster 3 pg 10.9 is stuck undersized for 73s, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,7] 2023-09-23T19:40:01.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:00 smithi092 bash[58855]: cluster 2023-09-23T19:40:00.000198+0000 mon.a (mon.0) 1230 : cluster 3 pg 10.e is stuck undersized for 73s, current state active+recovering+undersized+remapped, last acting [4,6] 2023-09-23T19:40:01.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:00 smithi092 bash[58855]: cluster 2023-09-23T19:40:00.000207+0000 mon.a (mon.0) 1231 : cluster 3 pg 10.f is stuck undersized for 73s, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,3] 2023-09-23T19:40:01.063 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:01.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:01.066 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:40:01.166 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:01.197 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:01.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:01.200 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:01.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:00 smithi016 bash[75004]: cluster 2023-09-23T19:40:00.000113+0000 mon.a (mon.0) 1224 : cluster 3 Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; Degraded data redundancy: 5681/101022 objects degraded (5.624%), 4 pgs degraded, 5 pgs undersized 2023-09-23T19:40:01.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:00 smithi016 bash[75004]: cluster 2023-09-23T19:40:00.000145+0000 mon.a (mon.0) 1225 : cluster 3 [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2023-09-23T19:40:01.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:00 smithi016 bash[75004]: cluster 2023-09-23T19:40:00.000157+0000 mon.a (mon.0) 1226 : cluster 3 [WRN] PG_DEGRADED: Degraded data redundancy: 5681/101022 objects degraded (5.624%), 4 pgs degraded, 5 pgs undersized 2023-09-23T19:40:01.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:00 smithi016 bash[75004]: cluster 2023-09-23T19:40:00.000164+0000 mon.a (mon.0) 1227 : cluster 3 pg 10.3 is stuck undersized for 73s, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,5] 2023-09-23T19:40:01.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:00 smithi016 bash[75004]: cluster 2023-09-23T19:40:00.000171+0000 mon.a (mon.0) 1228 : cluster 3 pg 10.5 is stuck undersized for 73s, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,3] 2023-09-23T19:40:01.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:00 smithi016 bash[75004]: cluster 2023-09-23T19:40:00.000182+0000 mon.a (mon.0) 1229 : cluster 3 pg 10.9 is stuck undersized for 73s, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,7] 2023-09-23T19:40:01.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:00 smithi016 bash[75004]: cluster 2023-09-23T19:40:00.000198+0000 mon.a (mon.0) 1230 : cluster 3 pg 10.e is stuck undersized for 73s, current state active+recovering+undersized+remapped, last acting [4,6] 2023-09-23T19:40:01.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:00 smithi016 bash[75004]: cluster 2023-09-23T19:40:00.000207+0000 mon.a (mon.0) 1231 : cluster 3 pg 10.f is stuck undersized for 73s, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,3] 2023-09-23T19:40:01.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:00 smithi016 bash[77995]: cluster 2023-09-23T19:40:00.000113+0000 mon.a (mon.0) 1224 : cluster 3 Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; Degraded data redundancy: 5681/101022 objects degraded (5.624%), 4 pgs degraded, 5 pgs undersized 2023-09-23T19:40:01.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:00 smithi016 bash[77995]: cluster 2023-09-23T19:40:00.000145+0000 mon.a (mon.0) 1225 : cluster 3 [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2023-09-23T19:40:01.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:00 smithi016 bash[77995]: cluster 2023-09-23T19:40:00.000157+0000 mon.a (mon.0) 1226 : cluster 3 [WRN] PG_DEGRADED: Degraded data redundancy: 5681/101022 objects degraded (5.624%), 4 pgs degraded, 5 pgs undersized 2023-09-23T19:40:01.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:00 smithi016 bash[77995]: cluster 2023-09-23T19:40:00.000164+0000 mon.a (mon.0) 1227 : cluster 3 pg 10.3 is stuck undersized for 73s, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,5] 2023-09-23T19:40:01.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:00 smithi016 bash[77995]: cluster 2023-09-23T19:40:00.000171+0000 mon.a (mon.0) 1228 : cluster 3 pg 10.5 is stuck undersized for 73s, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,3] 2023-09-23T19:40:01.288 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:00 smithi016 bash[77995]: cluster 2023-09-23T19:40:00.000182+0000 mon.a (mon.0) 1229 : cluster 3 pg 10.9 is stuck undersized for 73s, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,7] 2023-09-23T19:40:01.288 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:00 smithi016 bash[77995]: cluster 2023-09-23T19:40:00.000198+0000 mon.a (mon.0) 1230 : cluster 3 pg 10.e is stuck undersized for 73s, current state active+recovering+undersized+remapped, last acting [4,6] 2023-09-23T19:40:01.288 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:00 smithi016 bash[77995]: cluster 2023-09-23T19:40:00.000207+0000 mon.a (mon.0) 1231 : cluster 3 pg 10.f is stuck undersized for 73s, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,3] 2023-09-23T19:40:01.301 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:01.331 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:01.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:01.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:40:01.438 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:01.470 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:01.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:01.476 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:01.576 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:01.613 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:01.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:01.616 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:01.717 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:01.746 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:01.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:01.750 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:40:01.851 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:01.885 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:01.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:01.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:01.989 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:02.024 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:02.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:02.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:40:02.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:01 smithi016 bash[75004]: cluster 2023-09-23T19:40:00.158605+0000 mgr.y (mgr.25325) 975 : cluster 0 pgmap v914: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 5681/97176 objects degraded (5.846%); 0 B/s, 23 objects/s recovering 2023-09-23T19:40:02.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:01 smithi016 bash[77995]: cluster 2023-09-23T19:40:00.158605+0000 mgr.y (mgr.25325) 975 : cluster 0 pgmap v914: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 5681/97176 objects degraded (5.846%); 0 B/s, 23 objects/s recovering 2023-09-23T19:40:02.126 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:02.172 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:02.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:02.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:40:02.276 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:02.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:01 smithi092 bash[58855]: cluster 2023-09-23T19:40:00.158605+0000 mgr.y (mgr.25325) 975 : cluster 0 pgmap v914: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 5681/97176 objects degraded (5.846%); 0 B/s, 23 objects/s recovering 2023-09-23T19:40:02.314 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:02.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:02.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:40:02.425 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:02.490 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:02.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:02.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:40:02.593 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:02.595 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:40:02.595 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:40:02.623 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:02.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:02.627 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:02.728 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:02.758 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:02.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:02.770 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:02.871 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:02.919 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:02.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:02.923 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:40:03.024 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:03.055 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:03.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:03.064 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:03.164 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:03.194 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:03.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:03.197 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:40:03.298 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:03.334 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:03.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:03.338 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:40:03.438 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:03.477 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:03.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:03.480 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:40:03.581 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:03.587 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:40:03.616 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:40:03.624 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:03.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:03.626 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:40:03.727 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:03.756 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:03.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:03.760 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:40:03.860 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:03.885 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:03.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:03.890 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:03.991 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:04.000 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:03 smithi092 bash[58855]: cluster 2023-09-23T19:40:02.159397+0000 mgr.y (mgr.25325) 976 : cluster 0 pgmap v915: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 5681/92811 objects degraded (6.121%); 0 B/s, 22 objects/s recovering 2023-09-23T19:40:04.000 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:03 smithi092 bash[58855]: audit 2023-09-23T19:40:03.085125+0000 mon.a (mon.0) 1232 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:40:04.030 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:04.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:04.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:04.133 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:04.143 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:03 smithi016 bash[77995]: cluster 2023-09-23T19:40:02.159397+0000 mgr.y (mgr.25325) 976 : cluster 0 pgmap v915: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 5681/92811 objects degraded (6.121%); 0 B/s, 22 objects/s recovering 2023-09-23T19:40:04.143 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:03 smithi016 bash[77995]: audit 2023-09-23T19:40:03.085125+0000 mon.a (mon.0) 1232 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:40:04.143 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:03 smithi016 bash[75004]: cluster 2023-09-23T19:40:02.159397+0000 mgr.y (mgr.25325) 976 : cluster 0 pgmap v915: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 5681/92811 objects degraded (6.121%); 0 B/s, 22 objects/s recovering 2023-09-23T19:40:04.144 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:03 smithi016 bash[75004]: audit 2023-09-23T19:40:03.085125+0000 mon.a (mon.0) 1232 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:40:04.165 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:04.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:04.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:40:04.268 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:04.293 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:04.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:04.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:40:04.396 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:04.429 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:04.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:04.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:04.533 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:04.562 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:04.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:04.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:40:04.665 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:04.688 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:04.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:04.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:40:04.792 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:04.830 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:04.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:04.843 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:04.944 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:04.989 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:04.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:04.996 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:40:05.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:04 smithi092 bash[58855]: cluster 2023-09-23T19:40:04.454796+0000 mon.a (mon.0) 1233 : cluster 3 Health check update: Degraded data redundancy: 5681/92811 objects degraded (6.121%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:40:05.097 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:05.131 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:05.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:05.138 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:05.239 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:05.267 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:05.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:05.272 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:05.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:04 smithi016 bash[77995]: cluster 2023-09-23T19:40:04.454796+0000 mon.a (mon.0) 1233 : cluster 3 Health check update: Degraded data redundancy: 5681/92811 objects degraded (6.121%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:40:05.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:04 smithi016 bash[75004]: cluster 2023-09-23T19:40:04.454796+0000 mon.a (mon.0) 1233 : cluster 3 Health check update: Degraded data redundancy: 5681/92811 objects degraded (6.121%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:40:05.372 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:05.406 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:05.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:05.412 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:40:05.513 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:05.537 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:05.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:05.543 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:05.643 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:05.681 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:05.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:05.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:05.786 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:05.832 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:05.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:05.835 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:40:05.936 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:05.970 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:05.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:05.972 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:40:06.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:05 smithi016 bash[77995]: cluster 2023-09-23T19:40:04.160040+0000 mgr.y (mgr.25325) 977 : cluster 0 pgmap v916: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 5681/90942 objects degraded (6.247%); 0 B/s, 22 objects/s recovering 2023-09-23T19:40:06.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:05 smithi016 bash[75004]: cluster 2023-09-23T19:40:04.160040+0000 mgr.y (mgr.25325) 977 : cluster 0 pgmap v916: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 5681/90942 objects degraded (6.247%); 0 B/s, 22 objects/s recovering 2023-09-23T19:40:06.036 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:40:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:40:05] "GET /metrics HTTP/1.1" 200 36315 "" "Prometheus/2.33.4" 2023-09-23T19:40:06.073 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:06.107 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:06.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:06.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:40:06.191 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:05 smithi092 bash[58855]: cluster 2023-09-23T19:40:04.160040+0000 mgr.y (mgr.25325) 977 : cluster 0 pgmap v916: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 5681/90942 objects degraded (6.247%); 0 B/s, 22 objects/s recovering 2023-09-23T19:40:06.211 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:06.254 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:06.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:06.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:06.358 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:06.392 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:06.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:06.396 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:40:06.497 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:06.532 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:06.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:06.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:40:06.638 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:06.674 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:06.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:06.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:40:06.777 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:06.806 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:06.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:06.813 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:06.914 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:06.948 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:06.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:06.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:40:07.051 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:07.093 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:07.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:07.096 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:40:07.197 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:07.225 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:07.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:07.229 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:40:07.330 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:07.357 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:07.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:07.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:07.461 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:07.501 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:07.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:07.506 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:40:07.607 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:07.635 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:07.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:07.640 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:40:07.741 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:07.785 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:07.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:07.790 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:40:07.891 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:07.900 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:07 smithi016 bash[77995]: cluster 2023-09-23T19:40:06.160934+0000 mgr.y (mgr.25325) 978 : cluster 0 pgmap v917: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 5681/83148 objects degraded (6.832%); 0 B/s, 22 objects/s recovering 2023-09-23T19:40:07.900 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:07 smithi016 bash[75004]: cluster 2023-09-23T19:40:06.160934+0000 mgr.y (mgr.25325) 978 : cluster 0 pgmap v917: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 5681/83148 objects degraded (6.832%); 0 B/s, 22 objects/s recovering 2023-09-23T19:40:07.919 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:07.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:07.922 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:40:08.022 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:08.063 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:08.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:08.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:40:08.168 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:08.178 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:07 smithi092 bash[58855]: cluster 2023-09-23T19:40:06.160934+0000 mgr.y (mgr.25325) 978 : cluster 0 pgmap v917: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 5681/83148 objects degraded (6.832%); 0 B/s, 22 objects/s recovering 2023-09-23T19:40:08.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:08.237 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:08.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:40:08.337 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:08.375 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:08.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:08.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:40:08.479 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:08.489 INFO:teuthology.orchestra.run.smithi092.stdout:ERROR: (22) Invalid argument 2023-09-23T19:40:08.489 INFO:teuthology.orchestra.run.smithi092.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:40:08.506 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:08.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:08.511 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:40:08.611 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:08.656 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:08.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:08.666 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:40:08.767 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:08.793 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:08.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:08.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:40:08.899 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:08.925 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:08.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:08.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:09.028 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:09.073 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:09.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:09.078 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:09.178 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:09.203 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:09.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:09.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:09.307 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:09.342 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:09.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:09.348 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:40:09.448 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:09.470 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:09.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:09.473 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:09.574 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:09.602 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:09.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:09.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:09.640 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T19:40:09.706 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:09.746 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:09.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:09.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:09.850 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:09 smithi016 bash[75004]: cluster 2023-09-23T19:40:08.161611+0000 mgr.y (mgr.25325) 979 : cluster 0 pgmap v918: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.6 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 5681/80751 objects degraded (7.035%); 0 B/s, 19 objects/s recovering 2023-09-23T19:40:09.850 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:09 smithi016 bash[75004]: audit 2023-09-23T19:40:09.434878+0000 mon.a (mon.0) 1234 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:40:09.850 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:09 smithi016 bash[75004]: audit 2023-09-23T19:40:09.442791+0000 mon.a (mon.0) 1235 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:40:09.850 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:09 smithi016 bash[75004]: cluster 2023-09-23T19:40:09.456323+0000 mon.a (mon.0) 1236 : cluster 3 Health check update: Degraded data redundancy: 5681/80751 objects degraded (7.035%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:40:09.851 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:09 smithi016 bash[77995]: cluster 2023-09-23T19:40:08.161611+0000 mgr.y (mgr.25325) 979 : cluster 0 pgmap v918: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.6 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 5681/80751 objects degraded (7.035%); 0 B/s, 19 objects/s recovering 2023-09-23T19:40:09.851 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:09 smithi016 bash[77995]: audit 2023-09-23T19:40:09.434878+0000 mon.a (mon.0) 1234 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:40:09.851 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:09 smithi016 bash[77995]: audit 2023-09-23T19:40:09.442791+0000 mon.a (mon.0) 1235 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:40:09.851 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:09 smithi016 bash[77995]: cluster 2023-09-23T19:40:09.456323+0000 mon.a (mon.0) 1236 : cluster 3 Health check update: Degraded data redundancy: 5681/80751 objects degraded (7.035%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:40:09.854 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:09.890 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:09.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:09.909 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:10.009 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:10.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:09 smithi092 bash[58855]: cluster 2023-09-23T19:40:08.161611+0000 mgr.y (mgr.25325) 979 : cluster 0 pgmap v918: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.6 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 5681/80751 objects degraded (7.035%); 0 B/s, 19 objects/s recovering 2023-09-23T19:40:10.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:09 smithi092 bash[58855]: audit 2023-09-23T19:40:09.434878+0000 mon.a (mon.0) 1234 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:40:10.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:09 smithi092 bash[58855]: audit 2023-09-23T19:40:09.442791+0000 mon.a (mon.0) 1235 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:40:10.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:09 smithi092 bash[58855]: cluster 2023-09-23T19:40:09.456323+0000 mon.a (mon.0) 1236 : cluster 3 Health check update: Degraded data redundancy: 5681/80751 objects degraded (7.035%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:40:10.048 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:10.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:10.054 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:40:10.154 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:10.190 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:10.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:10.196 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:40:10.297 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:10.328 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:10.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:10.335 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:40:10.436 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:10.470 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:10.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:10.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:40:10.573 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:10.596 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:10.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:10.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:10.703 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:10.735 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:10.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:10.737 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:10.838 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:10.864 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:10.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:10.869 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:40:10.970 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:10.999 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:11.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:11.004 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:11.105 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:11.136 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:11.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:11.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:40:11.239 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:11.265 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:11.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:11.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:40:11.368 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:11.395 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:11.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:11.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:40:11.498 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:11.521 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:11.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:11.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:40:11.624 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:11.650 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:11.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:11.654 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:40:11.754 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:11.783 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:11.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:11.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:40:11.884 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:11 smithi016 bash[75004]: cluster 2023-09-23T19:40:10.162221+0000 mgr.y (mgr.25325) 980 : cluster 0 pgmap v919: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.6 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 667 op/s; 5681/76998 objects degraded (7.378%); 0 B/s, 19 objects/s recovering 2023-09-23T19:40:11.885 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:11 smithi016 bash[75004]: cluster 2023-09-23T19:40:10.865605+0000 osd.4 (osd.4) 103 : cluster 3 failed to encode map e169 with expected crc 2023-09-23T19:40:11.885 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:11 smithi016 bash[75004]: cluster 2023-09-23T19:40:10.871114+0000 osd.6 (osd.6) 91 : cluster 3 failed to encode map e169 with expected crc 2023-09-23T19:40:11.885 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:11 smithi016 bash[75004]: cluster 2023-09-23T19:40:10.872254+0000 mon.a (mon.0) 1237 : cluster 0 osdmap e169: 8 total, 8 up, 8 in 2023-09-23T19:40:11.885 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:11 smithi016 bash[75004]: cluster 2023-09-23T19:40:10.875193+0000 osd.4 (osd.4) 104 : cluster 3 failed to encode map e169 with expected crc 2023-09-23T19:40:11.885 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:11 smithi016 bash[75004]: cluster 2023-09-23T19:40:10.877904+0000 osd.4 (osd.4) 105 : cluster 3 failed to encode map e169 with expected crc 2023-09-23T19:40:11.885 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:11 smithi016 bash[75004]: cluster 2023-09-23T19:40:10.895516+0000 osd.7 (osd.7) 80 : cluster 3 failed to encode map e169 with expected crc 2023-09-23T19:40:11.885 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:11 smithi016 bash[75004]: audit 2023-09-23T19:40:11.215493+0000 mon.a (mon.0) 1238 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:40:11.885 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:11 smithi016 bash[75004]: audit 2023-09-23T19:40:11.222104+0000 mon.a (mon.0) 1239 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:40:11.885 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:11 smithi016 bash[75004]: audit 2023-09-23T19:40:11.743994+0000 mon.a (mon.0) 1240 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:40:11.885 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:11 smithi016 bash[75004]: audit 2023-09-23T19:40:11.745094+0000 mon.a (mon.0) 1241 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:40:11.886 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:11 smithi016 bash[75004]: audit 2023-09-23T19:40:11.751141+0000 mon.a (mon.0) 1242 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:40:11.886 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:11 smithi016 bash[77995]: cluster 2023-09-23T19:40:10.162221+0000 mgr.y (mgr.25325) 980 : cluster 0 pgmap v919: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.6 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 667 op/s; 5681/76998 objects degraded (7.378%); 0 B/s, 19 objects/s recovering 2023-09-23T19:40:11.886 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:11 smithi016 bash[77995]: cluster 2023-09-23T19:40:10.865605+0000 osd.4 (osd.4) 103 : cluster 3 failed to encode map e169 with expected crc 2023-09-23T19:40:11.886 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:11 smithi016 bash[77995]: cluster 2023-09-23T19:40:10.871114+0000 osd.6 (osd.6) 91 : cluster 3 failed to encode map e169 with expected crc 2023-09-23T19:40:11.886 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:11 smithi016 bash[77995]: cluster 2023-09-23T19:40:10.872254+0000 mon.a (mon.0) 1237 : cluster 0 osdmap e169: 8 total, 8 up, 8 in 2023-09-23T19:40:11.886 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:11 smithi016 bash[77995]: cluster 2023-09-23T19:40:10.875193+0000 osd.4 (osd.4) 104 : cluster 3 failed to encode map e169 with expected crc 2023-09-23T19:40:11.886 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:11 smithi016 bash[77995]: cluster 2023-09-23T19:40:10.877904+0000 osd.4 (osd.4) 105 : cluster 3 failed to encode map e169 with expected crc 2023-09-23T19:40:11.886 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:11 smithi016 bash[77995]: cluster 2023-09-23T19:40:10.895516+0000 osd.7 (osd.7) 80 : cluster 3 failed to encode map e169 with expected crc 2023-09-23T19:40:11.886 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:11 smithi016 bash[77995]: audit 2023-09-23T19:40:11.215493+0000 mon.a (mon.0) 1238 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:40:11.886 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:11 smithi016 bash[77995]: audit 2023-09-23T19:40:11.222104+0000 mon.a (mon.0) 1239 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:40:11.887 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:11 smithi016 bash[77995]: audit 2023-09-23T19:40:11.743994+0000 mon.a (mon.0) 1240 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:40:11.887 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:11 smithi016 bash[77995]: audit 2023-09-23T19:40:11.745094+0000 mon.a (mon.0) 1241 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:40:11.887 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:11 smithi016 bash[77995]: audit 2023-09-23T19:40:11.751141+0000 mon.a (mon.0) 1242 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:40:11.887 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:11.921 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:11.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:11.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:11.945 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:40:11.973 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:40:11.999 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:40:12.024 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:12.049 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:40:12.059 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:12.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:12.065 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:40:12.119 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:40:12.166 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:12.168 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:40:12.198 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:12.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:12.202 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:40:12.213 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:40:12.251 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:40:12.271 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:40:12.272 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:40:12.272 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:40:12.272 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:40:12.272 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:40:12.272 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:40:12.272 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:40:12.272 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:40:12.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:11 smithi092 bash[58855]: cluster 2023-09-23T19:40:10.162221+0000 mgr.y (mgr.25325) 980 : cluster 0 pgmap v919: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.6 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 667 op/s; 5681/76998 objects degraded (7.378%); 0 B/s, 19 objects/s recovering 2023-09-23T19:40:12.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:11 smithi092 bash[58855]: cluster 2023-09-23T19:40:10.865605+0000 osd.4 (osd.4) 103 : cluster 3 failed to encode map e169 with expected crc 2023-09-23T19:40:12.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:11 smithi092 bash[58855]: cluster 2023-09-23T19:40:10.871114+0000 osd.6 (osd.6) 91 : cluster 3 failed to encode map e169 with expected crc 2023-09-23T19:40:12.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:11 smithi092 bash[58855]: cluster 2023-09-23T19:40:10.872254+0000 mon.a (mon.0) 1237 : cluster 0 osdmap e169: 8 total, 8 up, 8 in 2023-09-23T19:40:12.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:11 smithi092 bash[58855]: cluster 2023-09-23T19:40:10.875193+0000 osd.4 (osd.4) 104 : cluster 3 failed to encode map e169 with expected crc 2023-09-23T19:40:12.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:11 smithi092 bash[58855]: cluster 2023-09-23T19:40:10.877904+0000 osd.4 (osd.4) 105 : cluster 3 failed to encode map e169 with expected crc 2023-09-23T19:40:12.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:11 smithi092 bash[58855]: cluster 2023-09-23T19:40:10.895516+0000 osd.7 (osd.7) 80 : cluster 3 failed to encode map e169 with expected crc 2023-09-23T19:40:12.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:11 smithi092 bash[58855]: audit 2023-09-23T19:40:11.215493+0000 mon.a (mon.0) 1238 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:40:12.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:11 smithi092 bash[58855]: audit 2023-09-23T19:40:11.222104+0000 mon.a (mon.0) 1239 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:40:12.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:11 smithi092 bash[58855]: audit 2023-09-23T19:40:11.743994+0000 mon.a (mon.0) 1240 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:40:12.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:11 smithi092 bash[58855]: audit 2023-09-23T19:40:11.745094+0000 mon.a (mon.0) 1241 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:40:12.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:11 smithi092 bash[58855]: audit 2023-09-23T19:40:11.751141+0000 mon.a (mon.0) 1242 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:40:12.303 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:12.330 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:12.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:12.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:12.438 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:12.470 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:12.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:12.474 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:40:12.574 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:12.602 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:12.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:12.605 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:12.706 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:12.745 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:12.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:12.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:40:12.848 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:12.902 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:12.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:12.906 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:12.908 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is unset 2023-09-23T19:40:12.915 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T19:40:12.977 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:12 smithi016 bash[75004]: cluster 2023-09-23T19:40:11.881313+0000 mon.a (mon.0) 1243 : cluster 0 osdmap e170: 8 total, 8 up, 8 in 2023-09-23T19:40:12.978 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:12 smithi016 bash[75004]: cluster 2023-09-23T19:40:11.887292+0000 osd.4 (osd.4) 106 : cluster 3 failed to encode map e170 with expected crc 2023-09-23T19:40:12.978 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:12 smithi016 bash[75004]: cluster 2023-09-23T19:40:11.890127+0000 osd.4 (osd.4) 107 : cluster 3 failed to encode map e170 with expected crc 2023-09-23T19:40:12.978 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:12 smithi016 bash[75004]: audit 2023-09-23T19:40:11.914974+0000 mon.c (mon.2) 173 : audit 0 from='client.? 172.21.15.16:0/2293608550' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:40:12.978 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:12 smithi016 bash[75004]: cluster 2023-09-23T19:40:12.002940+0000 osd.7 (osd.7) 81 : cluster 3 failed to encode map e170 with expected crc 2023-09-23T19:40:12.978 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:12 smithi016 bash[75004]: audit 2023-09-23T19:40:12.319371+0000 mon.c (mon.2) 174 : audit 1 from='client.? 172.21.15.16:0/1805029457' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:40:12.978 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:12 smithi016 bash[75004]: audit 2023-09-23T19:40:12.320106+0000 mon.a (mon.0) 1244 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:40:12.978 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:12 smithi016 bash[75004]: audit 2023-09-23T19:40:12.528705+0000 mon.a (mon.0) 1245 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T19:40:12.978 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:12 smithi016 bash[75004]: audit 2023-09-23T19:40:12.530630+0000 mon.b (mon.1) 46 : audit 1 from='client.? 172.21.15.16:0/4031390931' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T19:40:12.979 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:12 smithi016 bash[77995]: cluster 2023-09-23T19:40:11.881313+0000 mon.a (mon.0) 1243 : cluster 0 osdmap e170: 8 total, 8 up, 8 in 2023-09-23T19:40:12.979 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:12 smithi016 bash[77995]: cluster 2023-09-23T19:40:11.887292+0000 osd.4 (osd.4) 106 : cluster 3 failed to encode map e170 with expected crc 2023-09-23T19:40:12.979 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:12 smithi016 bash[77995]: cluster 2023-09-23T19:40:11.890127+0000 osd.4 (osd.4) 107 : cluster 3 failed to encode map e170 with expected crc 2023-09-23T19:40:12.979 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:12 smithi016 bash[77995]: audit 2023-09-23T19:40:11.914974+0000 mon.c (mon.2) 173 : audit 0 from='client.? 172.21.15.16:0/2293608550' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:40:12.979 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:12 smithi016 bash[77995]: cluster 2023-09-23T19:40:12.002940+0000 osd.7 (osd.7) 81 : cluster 3 failed to encode map e170 with expected crc 2023-09-23T19:40:12.979 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:12 smithi016 bash[77995]: audit 2023-09-23T19:40:12.319371+0000 mon.c (mon.2) 174 : audit 1 from='client.? 172.21.15.16:0/1805029457' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:40:12.979 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:12 smithi016 bash[77995]: audit 2023-09-23T19:40:12.320106+0000 mon.a (mon.0) 1244 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:40:12.979 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:12 smithi016 bash[77995]: audit 2023-09-23T19:40:12.528705+0000 mon.a (mon.0) 1245 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T19:40:12.979 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:12 smithi016 bash[77995]: audit 2023-09-23T19:40:12.530630+0000 mon.b (mon.1) 46 : audit 1 from='client.? 172.21.15.16:0/4031390931' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T19:40:13.007 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:13.035 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:13.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:13.041 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:40:13.142 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:13.174 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:13.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:13.176 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:13.277 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:13.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:12 smithi092 bash[58855]: cluster 2023-09-23T19:40:11.881313+0000 mon.a (mon.0) 1243 : cluster 0 osdmap e170: 8 total, 8 up, 8 in 2023-09-23T19:40:13.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:12 smithi092 bash[58855]: cluster 2023-09-23T19:40:11.887292+0000 osd.4 (osd.4) 106 : cluster 3 failed to encode map e170 with expected crc 2023-09-23T19:40:13.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:12 smithi092 bash[58855]: cluster 2023-09-23T19:40:11.890127+0000 osd.4 (osd.4) 107 : cluster 3 failed to encode map e170 with expected crc 2023-09-23T19:40:13.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:12 smithi092 bash[58855]: audit 2023-09-23T19:40:11.914974+0000 mon.c (mon.2) 173 : audit 0 from='client.? 172.21.15.16:0/2293608550' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:40:13.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:12 smithi092 bash[58855]: cluster 2023-09-23T19:40:12.002940+0000 osd.7 (osd.7) 81 : cluster 3 failed to encode map e170 with expected crc 2023-09-23T19:40:13.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:12 smithi092 bash[58855]: audit 2023-09-23T19:40:12.319371+0000 mon.c (mon.2) 174 : audit 1 from='client.? 172.21.15.16:0/1805029457' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:40:13.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:12 smithi092 bash[58855]: audit 2023-09-23T19:40:12.320106+0000 mon.a (mon.0) 1244 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:40:13.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:12 smithi092 bash[58855]: audit 2023-09-23T19:40:12.528705+0000 mon.a (mon.0) 1245 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T19:40:13.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:12 smithi092 bash[58855]: audit 2023-09-23T19:40:12.530630+0000 mon.b (mon.1) 46 : audit 1 from='client.? 172.21.15.16:0/4031390931' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T19:40:13.313 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:13.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:13.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:40:13.420 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:13.458 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:13.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:13.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:40:13.564 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:13.591 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:13.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:13.599 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:13.700 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:13.734 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:13.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:13.737 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:40:13.837 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:13.865 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:13.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:13.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:40:13.977 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:14.003 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:14.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:14.005 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:40:14.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:13 smithi016 bash[75004]: cluster 2023-09-23T19:40:12.162826+0000 mgr.y (mgr.25325) 981 : cluster 0 pgmap v922: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.5 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 4246/73071 objects degraded (5.811%); 0 B/s, 46 objects/s recovering 2023-09-23T19:40:14.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:13 smithi016 bash[75004]: cluster 2023-09-23T19:40:12.872593+0000 mon.a (mon.0) 1246 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:40:14.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:13 smithi016 bash[75004]: audit 2023-09-23T19:40:12.898733+0000 mon.a (mon.0) 1247 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:40:14.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:13 smithi016 bash[75004]: audit 2023-09-23T19:40:12.899545+0000 mon.a (mon.0) 1248 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T19:40:14.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:13 smithi016 bash[75004]: cluster 2023-09-23T19:40:12.901299+0000 mon.a (mon.0) 1249 : cluster 0 osdmap e171: 8 total, 8 up, 8 in 2023-09-23T19:40:14.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:13 smithi016 bash[75004]: cluster 2023-09-23T19:40:12.906792+0000 osd.6 (osd.6) 92 : cluster 3 failed to encode map e171 with expected crc 2023-09-23T19:40:14.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:13 smithi016 bash[75004]: cluster 2023-09-23T19:40:12.906963+0000 osd.4 (osd.4) 108 : cluster 3 failed to encode map e171 with expected crc 2023-09-23T19:40:14.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:13 smithi016 bash[75004]: cluster 2023-09-23T19:40:12.909461+0000 osd.5 (osd.5) 100 : cluster 3 failed to encode map e171 with expected crc 2023-09-23T19:40:14.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:13 smithi016 bash[75004]: cluster 2023-09-23T19:40:12.911015+0000 osd.4 (osd.4) 109 : cluster 3 failed to encode map e171 with expected crc 2023-09-23T19:40:14.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:13 smithi016 bash[75004]: cluster 2023-09-23T19:40:12.912466+0000 osd.6 (osd.6) 93 : cluster 3 failed to encode map e171 with expected crc 2023-09-23T19:40:14.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:13 smithi016 bash[75004]: cluster 2023-09-23T19:40:12.915839+0000 osd.5 (osd.5) 101 : cluster 3 failed to encode map e171 with expected crc 2023-09-23T19:40:14.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:13 smithi016 bash[75004]: cluster 2023-09-23T19:40:12.915869+0000 osd.6 (osd.6) 94 : cluster 3 failed to encode map e171 with expected crc 2023-09-23T19:40:14.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:13 smithi016 bash[75004]: cluster 2023-09-23T19:40:12.917968+0000 osd.7 (osd.7) 82 : cluster 3 failed to encode map e171 with expected crc 2023-09-23T19:40:14.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:13 smithi016 bash[75004]: audit 2023-09-23T19:40:13.657812+0000 mon.a (mon.0) 1250 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:40:14.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:13 smithi016 bash[77995]: cluster 2023-09-23T19:40:12.162826+0000 mgr.y (mgr.25325) 981 : cluster 0 pgmap v922: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.5 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 4246/73071 objects degraded (5.811%); 0 B/s, 46 objects/s recovering 2023-09-23T19:40:14.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:13 smithi016 bash[77995]: cluster 2023-09-23T19:40:12.872593+0000 mon.a (mon.0) 1246 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:40:14.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:13 smithi016 bash[77995]: audit 2023-09-23T19:40:12.898733+0000 mon.a (mon.0) 1247 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:40:14.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:13 smithi016 bash[77995]: audit 2023-09-23T19:40:12.899545+0000 mon.a (mon.0) 1248 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T19:40:14.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:13 smithi016 bash[77995]: cluster 2023-09-23T19:40:12.901299+0000 mon.a (mon.0) 1249 : cluster 0 osdmap e171: 8 total, 8 up, 8 in 2023-09-23T19:40:14.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:13 smithi016 bash[77995]: cluster 2023-09-23T19:40:12.906792+0000 osd.6 (osd.6) 92 : cluster 3 failed to encode map e171 with expected crc 2023-09-23T19:40:14.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:13 smithi016 bash[77995]: cluster 2023-09-23T19:40:12.906963+0000 osd.4 (osd.4) 108 : cluster 3 failed to encode map e171 with expected crc 2023-09-23T19:40:14.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:13 smithi016 bash[77995]: cluster 2023-09-23T19:40:12.909461+0000 osd.5 (osd.5) 100 : cluster 3 failed to encode map e171 with expected crc 2023-09-23T19:40:14.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:13 smithi016 bash[77995]: cluster 2023-09-23T19:40:12.911015+0000 osd.4 (osd.4) 109 : cluster 3 failed to encode map e171 with expected crc 2023-09-23T19:40:14.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:13 smithi016 bash[77995]: cluster 2023-09-23T19:40:12.912466+0000 osd.6 (osd.6) 93 : cluster 3 failed to encode map e171 with expected crc 2023-09-23T19:40:14.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:13 smithi016 bash[77995]: cluster 2023-09-23T19:40:12.915839+0000 osd.5 (osd.5) 101 : cluster 3 failed to encode map e171 with expected crc 2023-09-23T19:40:14.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:13 smithi016 bash[77995]: cluster 2023-09-23T19:40:12.915869+0000 osd.6 (osd.6) 94 : cluster 3 failed to encode map e171 with expected crc 2023-09-23T19:40:14.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:13 smithi016 bash[77995]: cluster 2023-09-23T19:40:12.917968+0000 osd.7 (osd.7) 82 : cluster 3 failed to encode map e171 with expected crc 2023-09-23T19:40:14.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:13 smithi016 bash[77995]: audit 2023-09-23T19:40:13.657812+0000 mon.a (mon.0) 1250 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:40:14.106 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:14.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:14.137 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:14.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:40:14.239 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:14.247 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:13 smithi092 bash[58855]: cluster 2023-09-23T19:40:12.162826+0000 mgr.y (mgr.25325) 981 : cluster 0 pgmap v922: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.5 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 4246/73071 objects degraded (5.811%); 0 B/s, 46 objects/s recovering 2023-09-23T19:40:14.248 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:13 smithi092 bash[58855]: cluster 2023-09-23T19:40:12.872593+0000 mon.a (mon.0) 1246 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:40:14.248 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:13 smithi092 bash[58855]: audit 2023-09-23T19:40:12.898733+0000 mon.a (mon.0) 1247 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:40:14.248 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:13 smithi092 bash[58855]: audit 2023-09-23T19:40:12.899545+0000 mon.a (mon.0) 1248 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T19:40:14.248 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:13 smithi092 bash[58855]: cluster 2023-09-23T19:40:12.901299+0000 mon.a (mon.0) 1249 : cluster 0 osdmap e171: 8 total, 8 up, 8 in 2023-09-23T19:40:14.248 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:13 smithi092 bash[58855]: cluster 2023-09-23T19:40:12.906792+0000 osd.6 (osd.6) 92 : cluster 3 failed to encode map e171 with expected crc 2023-09-23T19:40:14.248 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:13 smithi092 bash[58855]: cluster 2023-09-23T19:40:12.906963+0000 osd.4 (osd.4) 108 : cluster 3 failed to encode map e171 with expected crc 2023-09-23T19:40:14.248 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:13 smithi092 bash[58855]: cluster 2023-09-23T19:40:12.909461+0000 osd.5 (osd.5) 100 : cluster 3 failed to encode map e171 with expected crc 2023-09-23T19:40:14.248 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:13 smithi092 bash[58855]: cluster 2023-09-23T19:40:12.911015+0000 osd.4 (osd.4) 109 : cluster 3 failed to encode map e171 with expected crc 2023-09-23T19:40:14.248 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:13 smithi092 bash[58855]: cluster 2023-09-23T19:40:12.912466+0000 osd.6 (osd.6) 93 : cluster 3 failed to encode map e171 with expected crc 2023-09-23T19:40:14.248 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:13 smithi092 bash[58855]: cluster 2023-09-23T19:40:12.915839+0000 osd.5 (osd.5) 101 : cluster 3 failed to encode map e171 with expected crc 2023-09-23T19:40:14.249 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:13 smithi092 bash[58855]: cluster 2023-09-23T19:40:12.915869+0000 osd.6 (osd.6) 94 : cluster 3 failed to encode map e171 with expected crc 2023-09-23T19:40:14.249 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:13 smithi092 bash[58855]: cluster 2023-09-23T19:40:12.917968+0000 osd.7 (osd.7) 82 : cluster 3 failed to encode map e171 with expected crc 2023-09-23T19:40:14.249 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:13 smithi092 bash[58855]: audit 2023-09-23T19:40:13.657812+0000 mon.a (mon.0) 1250 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:40:14.266 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:14.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:14.269 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:40:14.369 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:14.402 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:14.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:14.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:14.484 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:40:14.508 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:14.553 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:14.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:14.558 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:40:14.659 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:14.682 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:14.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:14.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:40:14.790 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:14.824 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:14.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:14.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:40:14.927 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:14.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:14.961 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:14.961 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:40:15.062 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:15.069 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:14 smithi092 bash[58855]: cluster 2023-09-23T19:40:14.457453+0000 mon.a (mon.0) 1251 : cluster 3 Health check update: Degraded data redundancy: 4246/73071 objects degraded (5.811%), 3 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:40:15.085 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:15.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:15.087 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:40:15.189 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:15.216 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:15.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:15.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:15.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:14 smithi016 bash[75004]: cluster 2023-09-23T19:40:14.457453+0000 mon.a (mon.0) 1251 : cluster 3 Health check update: Degraded data redundancy: 4246/73071 objects degraded (5.811%), 3 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:40:15.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:14 smithi016 bash[77995]: cluster 2023-09-23T19:40:14.457453+0000 mon.a (mon.0) 1251 : cluster 3 Health check update: Degraded data redundancy: 4246/73071 objects degraded (5.811%), 3 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:40:15.325 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:15.362 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:15.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:15.365 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:40:15.466 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:15.500 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:15.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:15.501 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:40:15.601 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T19:40:15.602 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:15.682 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:15.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:15.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:40:15.787 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:15.814 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:15.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:15.816 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:40:15.914 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:15 smithi016 bash[75004]: cluster 2023-09-23T19:40:14.164703+0000 mgr.y (mgr.25325) 982 : cluster 0 pgmap v924: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.5 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 506 op/s; 4246/71637 objects degraded (5.927%); 0 B/s, 42 objects/s recovering 2023-09-23T19:40:15.914 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:40:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:40:15] "GET /metrics HTTP/1.1" 200 36312 "" "Prometheus/2.33.4" 2023-09-23T19:40:15.918 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:15.941 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:15.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:15.946 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:16.047 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:16.079 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:16.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:16.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:40:16.182 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:16.190 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:15 smithi092 bash[58855]: cluster 2023-09-23T19:40:14.164703+0000 mgr.y (mgr.25325) 982 : cluster 0 pgmap v924: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.5 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 506 op/s; 4246/71637 objects degraded (5.927%); 0 B/s, 42 objects/s recovering 2023-09-23T19:40:16.217 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:16.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:16.221 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:16.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:15 smithi016 bash[77995]: cluster 2023-09-23T19:40:14.164703+0000 mgr.y (mgr.25325) 982 : cluster 0 pgmap v924: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.5 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 506 op/s; 4246/71637 objects degraded (5.927%); 0 B/s, 42 objects/s recovering 2023-09-23T19:40:16.302 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:40:16.322 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:16.373 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:16.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:16.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:40:16.478 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:16.515 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:16.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:16.520 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:40:16.620 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:16.657 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:16.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:16.662 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:40:16.763 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:16.788 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:16.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:16.795 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:40:16.896 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:16.925 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:16.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:16.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:17.033 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:17.065 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:17.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:17.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:17.173 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:17.207 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:17.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:17.218 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:40:17.318 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:17.354 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:17.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:17.364 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:40:17.465 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:17.502 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:17.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:17.507 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:17.581 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T19:40:17.608 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:17.641 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:17.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:17.648 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:40:17.749 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:17.788 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:17.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:17.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:17.892 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:17.923 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:17.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:17.933 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:40:18.034 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:18.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:17 smithi092 bash[58855]: cluster 2023-09-23T19:40:16.165502+0000 mgr.y (mgr.25325) 983 : cluster 0 pgmap v925: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 4246/65562 objects degraded (6.476%); 0 B/s, 43 objects/s recovering 2023-09-23T19:40:18.068 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:18.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:18.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:40:18.174 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:18.201 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:18.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:18.203 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:18.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:17 smithi016 bash[75004]: cluster 2023-09-23T19:40:16.165502+0000 mgr.y (mgr.25325) 983 : cluster 0 pgmap v925: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 4246/65562 objects degraded (6.476%); 0 B/s, 43 objects/s recovering 2023-09-23T19:40:18.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:17 smithi016 bash[77995]: cluster 2023-09-23T19:40:16.165502+0000 mgr.y (mgr.25325) 983 : cluster 0 pgmap v925: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 4246/65562 objects degraded (6.476%); 0 B/s, 43 objects/s recovering 2023-09-23T19:40:18.304 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:18.339 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:18.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:18.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:18.446 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:18.469 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:18.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:18.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:40:18.572 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:18.600 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:18.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:18.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:40:18.703 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:18.736 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:18.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:18.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:18.841 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:18.874 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:18.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:18.878 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:40:18.953 INFO:teuthology.orchestra.run.smithi092.stdout:{ 2023-09-23T19:40:18.953 INFO:teuthology.orchestra.run.smithi092.stdout: "ops": [ 2023-09-23T19:40:18.954 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:40:18.954 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45947.0:201263 10.9 10:9c73de64:::benchmark_data_smithi016_136365_object91129:head [delete] snapc 0=[] ondisk+write+known_if_redirected e171)", 2023-09-23T19:40:18.954 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:40:18.950012+0000", 2023-09-23T19:40:18.954 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 0.005002556, 2023-09-23T19:40:18.954 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.005038453, 2023-09-23T19:40:18.954 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:40:18.954 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "waiting for sub ops", 2023-09-23T19:40:18.954 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:40:18.954 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45947", 2023-09-23T19:40:18.954 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/4039191752", 2023-09-23T19:40:18.954 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 201263 2023-09-23T19:40:18.955 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:40:18.955 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:40:18.955 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:40:18.955 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:40:18.955 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:18.950012+0000", 2023-09-23T19:40:18.955 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:40:18.955 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:40:18.955 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:40:18.955 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:40:18.955 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:18.950012+0000", 2023-09-23T19:40:18.955 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T19:40:18.955 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:40:18.955 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:40:18.956 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:40:18.956 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:18.950010+0000", 2023-09-23T19:40:18.956 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.4230000000000005e-06 2023-09-23T19:40:18.956 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:40:18.956 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:40:18.956 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:40:18.956 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:18.950018+0000", 2023-09-23T19:40:18.956 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.3692999999999999e-05 2023-09-23T19:40:18.956 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:40:18.956 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:40:18.956 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:40:18.956 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:18.950032+0000", 2023-09-23T19:40:18.957 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.0910000000000004e-06 2023-09-23T19:40:18.957 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:40:18.957 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:40:18.957 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:40:18.957 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:18.950039+0000", 2023-09-23T19:40:18.957 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.6249999999999998e-05 2023-09-23T19:40:18.957 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:40:18.957 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:40:18.957 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:40:18.957 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:18.950095+0000", 2023-09-23T19:40:18.957 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000146928 2023-09-23T19:40:18.957 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:40:18.957 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:40:18.958 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:40:18.958 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:18.950242+0000", 2023-09-23T19:40:18.958 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00017079700000000001 2023-09-23T19:40:18.958 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:40:18.958 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:40:18.958 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 2,7", 2023-09-23T19:40:18.958 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:18.950413+0000", 2023-09-23T19:40:18.958 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000401214 2023-09-23T19:40:18.958 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:40:18.958 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:40:18.958 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:40:18.958 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:40:18.959 INFO:teuthology.orchestra.run.smithi092.stdout: ], 2023-09-23T19:40:18.959 INFO:teuthology.orchestra.run.smithi092.stdout: "num_ops": 1 2023-09-23T19:40:18.959 INFO:teuthology.orchestra.run.smithi092.stdout:} 2023-09-23T19:40:18.979 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:19.010 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:19.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:19.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:19.114 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:19.159 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:19.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:19.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:40:19.265 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:19.289 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:19.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:19.292 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:19.392 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:19.418 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:19.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:19.423 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:19.523 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:19.569 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:19.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:19.572 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:40:19.673 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:19.712 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:19.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:19.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:19.816 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:19.845 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:19.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:19.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:19.886 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T19:40:19.952 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:19.981 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:19.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:19.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:20.085 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:20.114 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:20.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:20.116 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:40:20.216 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:20.225 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:19 smithi092 bash[58855]: cluster 2023-09-23T19:40:18.166120+0000 mgr.y (mgr.25325) 984 : cluster 0 pgmap v926: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 476 op/s; 4246/63177 objects degraded (6.721%); 0 B/s, 59 objects/s recovering 2023-09-23T19:40:20.225 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:19 smithi092 bash[58855]: cluster 2023-09-23T19:40:19.458652+0000 mon.a (mon.0) 1252 : cluster 3 Health check update: Degraded data redundancy: 4246/63177 objects degraded (6.721%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:40:20.251 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:20.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:20.253 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:40:20.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:19 smithi016 bash[75004]: cluster 2023-09-23T19:40:18.166120+0000 mgr.y (mgr.25325) 984 : cluster 0 pgmap v926: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 476 op/s; 4246/63177 objects degraded (6.721%); 0 B/s, 59 objects/s recovering 2023-09-23T19:40:20.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:19 smithi016 bash[75004]: cluster 2023-09-23T19:40:19.458652+0000 mon.a (mon.0) 1252 : cluster 3 Health check update: Degraded data redundancy: 4246/63177 objects degraded (6.721%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:40:20.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:19 smithi016 bash[77995]: cluster 2023-09-23T19:40:18.166120+0000 mgr.y (mgr.25325) 984 : cluster 0 pgmap v926: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 476 op/s; 4246/63177 objects degraded (6.721%); 0 B/s, 59 objects/s recovering 2023-09-23T19:40:20.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:19 smithi016 bash[77995]: cluster 2023-09-23T19:40:19.458652+0000 mon.a (mon.0) 1252 : cluster 3 Health check update: Degraded data redundancy: 4246/63177 objects degraded (6.721%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:40:20.354 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:20.386 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:20.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:20.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:20.489 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:20.508 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:40:20.508 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:40:20.532 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:20.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:20.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:40:20.642 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:20.682 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:20.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:20.687 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:40:20.788 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:20.839 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:20.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:20.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:20.944 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:20.970 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:20.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:20.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:21.075 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:21.107 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:21.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:21.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:40:21.214 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:21.236 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:21.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:21.241 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:21.342 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:21.381 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:21.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:21.384 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:21.485 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:21.519 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:21.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:21.522 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:40:21.522 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:40:21.623 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:21.657 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:21.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:21.661 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:21.761 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:21.792 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:21.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:21.805 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:40:21.905 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:21.932 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:21 smithi016 bash[75004]: cluster 2023-09-23T19:40:20.166713+0000 mgr.y (mgr.25325) 985 : cluster 0 pgmap v927: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 544 op/s; 4246/59532 objects degraded (7.132%); 0 B/s, 35 objects/s recovering 2023-09-23T19:40:21.933 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:21 smithi016 bash[77995]: cluster 2023-09-23T19:40:20.166713+0000 mgr.y (mgr.25325) 985 : cluster 0 pgmap v927: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 544 op/s; 4246/59532 objects degraded (7.132%); 0 B/s, 35 objects/s recovering 2023-09-23T19:40:21.942 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:21.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:21.947 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:22.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:21 smithi092 bash[58855]: cluster 2023-09-23T19:40:20.166713+0000 mgr.y (mgr.25325) 985 : cluster 0 pgmap v927: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 544 op/s; 4246/59532 objects degraded (7.132%); 0 B/s, 35 objects/s recovering 2023-09-23T19:40:22.048 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:22.086 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:22.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:22.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:22.189 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:22.215 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:22.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:22.218 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:40:22.319 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:22.352 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:22.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:22.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:22.459 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:22.485 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:22.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:22.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:40:22.592 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:22.614 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:22.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:22.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:40:22.720 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:22.747 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:22.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:22.750 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:22.851 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:22.899 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:22.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:22.902 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:23.002 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:23.026 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:23.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:23.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:23.129 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:23.154 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:23.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:23.160 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:23.261 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:23.289 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:23.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:23.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:23.392 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:23.430 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:23.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:23.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:23.538 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:23.572 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:23.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:23.576 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:23.677 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:23.702 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:23.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:23.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:40:23.806 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:23.848 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:23.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:23.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:40:23.938 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:23 smithi016 bash[75004]: cluster 2023-09-23T19:40:22.170052+0000 mgr.y (mgr.25325) 986 : cluster 0 pgmap v928: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 4246/55134 objects degraded (7.701%); 0 B/s, 59 objects/s recovering 2023-09-23T19:40:23.953 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:23.981 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:23.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:23.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:40:24.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:23 smithi016 bash[77995]: cluster 2023-09-23T19:40:22.170052+0000 mgr.y (mgr.25325) 986 : cluster 0 pgmap v928: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 4246/55134 objects degraded (7.701%); 0 B/s, 59 objects/s recovering 2023-09-23T19:40:24.086 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:24.119 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:24.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:24.125 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:24.226 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:24.232 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:23 smithi092 bash[58855]: cluster 2023-09-23T19:40:22.170052+0000 mgr.y (mgr.25325) 986 : cluster 0 pgmap v928: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 4246/55134 objects degraded (7.701%); 0 B/s, 59 objects/s recovering 2023-09-23T19:40:24.249 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:24.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:24.256 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:24.357 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:24.388 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:24.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:24.393 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:40:24.494 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:24.542 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:24.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:24.543 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:24.644 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:24.671 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:24.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:24.676 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:40:24.774 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:40:24.777 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:24.821 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:24.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:24.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:40:24.926 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:24.945 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:24 smithi016 bash[75004]: cluster 2023-09-23T19:40:24.460327+0000 mon.a (mon.0) 1253 : cluster 3 Health check update: Degraded data redundancy: 4246/55134 objects degraded (7.701%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:40:24.946 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:24 smithi016 bash[77995]: cluster 2023-09-23T19:40:24.460327+0000 mon.a (mon.0) 1253 : cluster 3 Health check update: Degraded data redundancy: 4246/55134 objects degraded (7.701%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:40:24.961 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:24.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:24.964 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:40:25.064 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:25.092 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:25.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:25.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:40:25.198 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:25.206 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:24 smithi092 bash[58855]: cluster 2023-09-23T19:40:24.460327+0000 mon.a (mon.0) 1253 : cluster 3 Health check update: Degraded data redundancy: 4246/55134 objects degraded (7.701%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:40:25.251 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:25.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:25.258 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:25.359 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:25.385 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:25.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:25.388 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:25.488 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:25.527 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:25.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:25.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:40:25.631 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:25.641 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:40:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:40:25] "GET /metrics HTTP/1.1" 200 36312 "" "Prometheus/2.33.4" 2023-09-23T19:40:25.660 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:25.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:25.665 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:40:25.766 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:25.810 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:25.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:25.811 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:40:25.912 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:25.936 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:25.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:25.941 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:25.947 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:25 smithi016 bash[75004]: cluster 2023-09-23T19:40:24.170690+0000 mgr.y (mgr.25325) 987 : cluster 0 pgmap v929: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.1 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 4246/53127 objects degraded (7.992%); 0 B/s, 52 objects/s recovering 2023-09-23T19:40:25.947 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:25 smithi016 bash[77995]: cluster 2023-09-23T19:40:24.170690+0000 mgr.y (mgr.25325) 987 : cluster 0 pgmap v929: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.1 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 4246/53127 objects degraded (7.992%); 0 B/s, 52 objects/s recovering 2023-09-23T19:40:26.042 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:26.074 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:26.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:26.077 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:26.178 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:26.186 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:25 smithi092 bash[58855]: cluster 2023-09-23T19:40:24.170690+0000 mgr.y (mgr.25325) 987 : cluster 0 pgmap v929: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.1 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 4246/53127 objects degraded (7.992%); 0 B/s, 52 objects/s recovering 2023-09-23T19:40:26.203 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:26.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:26.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:26.308 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:26.345 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:26.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:26.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:40:26.451 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:26.475 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:26.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:26.480 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:40:26.581 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:26.603 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:26.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:26.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:26.712 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:26.739 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:26.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:26.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:40:26.842 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:26.886 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:26.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:26.890 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:40:26.991 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:27.015 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:27.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:27.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:40:27.120 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:27.154 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:27.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:27.156 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:27.257 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:27.307 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:27.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:27.311 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:40:27.412 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:27.437 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:27.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:27.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:40:27.543 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:27.567 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:27.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:27.570 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:27.670 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:27.702 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:27.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:27.705 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:40:27.806 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:27.831 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:27.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:27.837 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:27.938 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:27.949 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:27 smithi016 bash[75004]: cluster 2023-09-23T19:40:26.171451+0000 mgr.y (mgr.25325) 988 : cluster 0 pgmap v930: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 947 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 4246/45498 objects degraded (9.332%); 0 B/s, 49 objects/s recovering 2023-09-23T19:40:27.950 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:27 smithi016 bash[77995]: cluster 2023-09-23T19:40:26.171451+0000 mgr.y (mgr.25325) 988 : cluster 0 pgmap v930: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 947 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 4246/45498 objects degraded (9.332%); 0 B/s, 49 objects/s recovering 2023-09-23T19:40:27.984 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:27.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:27.994 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:28.094 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:28.116 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:28.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:28.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:40:28.219 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:28.232 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:27 smithi092 bash[58855]: cluster 2023-09-23T19:40:26.171451+0000 mgr.y (mgr.25325) 988 : cluster 0 pgmap v930: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 947 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 4246/45498 objects degraded (9.332%); 0 B/s, 49 objects/s recovering 2023-09-23T19:40:28.250 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:28.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:28.253 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:28.354 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:28.405 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:28.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:28.412 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:28.513 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:28.549 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:28.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:28.555 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:40:28.656 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:28.680 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:28.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:28.682 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:28.783 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:28.810 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:28.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:28.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:40:28.913 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:28.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:28.949 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:28.949 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:28.958 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:28 smithi016 bash[77995]: audit 2023-09-23T19:40:28.657949+0000 mon.a (mon.0) 1254 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:40:29.049 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:29.089 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:29.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:29.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:40:29.176 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:28 smithi092 bash[58855]: audit 2023-09-23T19:40:28.657949+0000 mon.a (mon.0) 1254 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:40:29.193 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:29.227 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:28 smithi016 bash[75004]: audit 2023-09-23T19:40:28.657949+0000 mon.a (mon.0) 1254 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:40:29.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:29.228 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:29.228 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:29.328 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:29.361 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:29.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:29.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:40:29.419 INFO:teuthology.orchestra.run.smithi092.stdout:{ 2023-09-23T19:40:29.420 INFO:teuthology.orchestra.run.smithi092.stdout: "ops": [], 2023-09-23T19:40:29.420 INFO:teuthology.orchestra.run.smithi092.stdout: "complaint_time": 30, 2023-09-23T19:40:29.420 INFO:teuthology.orchestra.run.smithi092.stdout: "num_blocked_ops": 0 2023-09-23T19:40:29.420 INFO:teuthology.orchestra.run.smithi092.stdout:} 2023-09-23T19:40:29.466 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:29.491 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:29.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:29.495 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:29.596 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:29.624 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:29.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:29.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:40:29.727 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:29.762 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:29.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:29.767 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:40:29.868 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:29.910 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:29.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:29.917 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:29.961 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:29 smithi016 bash[75004]: cluster 2023-09-23T19:40:28.172155+0000 mgr.y (mgr.25325) 989 : cluster 0 pgmap v931: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 935 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 4246/44928 objects degraded (9.451%); 0 B/s, 75 objects/s recovering 2023-09-23T19:40:30.018 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:30.051 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:30.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:30.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:40:30.154 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:30.182 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:30.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:30.187 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:30.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:29 smithi016 bash[75004]: cluster 2023-09-23T19:40:29.461565+0000 mon.a (mon.0) 1255 : cluster 3 Health check update: Degraded data redundancy: 4246/44928 objects degraded (9.451%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:40:30.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:29 smithi016 bash[77995]: cluster 2023-09-23T19:40:28.172155+0000 mgr.y (mgr.25325) 989 : cluster 0 pgmap v931: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 935 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 4246/44928 objects degraded (9.451%); 0 B/s, 75 objects/s recovering 2023-09-23T19:40:30.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:29 smithi016 bash[77995]: cluster 2023-09-23T19:40:29.461565+0000 mon.a (mon.0) 1255 : cluster 3 Health check update: Degraded data redundancy: 4246/44928 objects degraded (9.451%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:40:30.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:29 smithi092 bash[58855]: cluster 2023-09-23T19:40:28.172155+0000 mgr.y (mgr.25325) 989 : cluster 0 pgmap v931: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 935 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 4246/44928 objects degraded (9.451%); 0 B/s, 75 objects/s recovering 2023-09-23T19:40:30.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:29 smithi092 bash[58855]: cluster 2023-09-23T19:40:29.461565+0000 mon.a (mon.0) 1255 : cluster 3 Health check update: Degraded data redundancy: 4246/44928 objects degraded (9.451%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:40:30.288 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:30.327 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:30.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:30.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:40:30.370 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:40:30.408 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T19:40:30.412 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:40:30.436 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:30.476 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:40:30.504 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:30.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:30.510 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:30.522 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:40:30.568 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:40:30.596 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:40:30.610 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:30.653 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:30.655 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:40:30.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:30.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:40:30.707 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:40:30.727 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:40:30.727 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:40:30.727 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:40:30.727 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:40:30.727 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:40:30.727 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:40:30.727 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:40:30.727 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:40:30.757 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:30.786 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:30.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:30.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:40:30.891 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:30.923 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:30.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:30.931 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:40:30.990 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is unset 2023-09-23T19:40:30.990 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T19:40:31.030 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:30 smithi016 bash[77995]: audit 2023-09-23T19:40:30.317233+0000 mon.a (mon.0) 1256 : audit 0 from='client.? 172.21.15.16:0/33797417' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:40:31.030 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:30 smithi016 bash[77995]: audit 2023-09-23T19:40:30.440538+0000 mon.a (mon.0) 1257 : audit 1 from='client.? 172.21.15.16:0/672772201' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T19:40:31.030 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:30 smithi016 bash[77995]: audit 2023-09-23T19:40:30.771938+0000 mon.a (mon.0) 1258 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:40:31.030 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:30 smithi016 bash[77995]: audit 2023-09-23T19:40:30.776473+0000 mon.b (mon.1) 47 : audit 1 from='client.? 172.21.15.16:0/179895324' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:40:31.031 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:30 smithi016 bash[75004]: audit 2023-09-23T19:40:30.317233+0000 mon.a (mon.0) 1256 : audit 0 from='client.? 172.21.15.16:0/33797417' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:40:31.031 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:30 smithi016 bash[75004]: audit 2023-09-23T19:40:30.440538+0000 mon.a (mon.0) 1257 : audit 1 from='client.? 172.21.15.16:0/672772201' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T19:40:31.031 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:30 smithi016 bash[75004]: audit 2023-09-23T19:40:30.771938+0000 mon.a (mon.0) 1258 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:40:31.031 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:30 smithi016 bash[75004]: audit 2023-09-23T19:40:30.776473+0000 mon.b (mon.1) 47 : audit 1 from='client.? 172.21.15.16:0/179895324' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:40:31.031 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:31.063 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:31.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:31.066 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:31.167 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:31.189 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:31.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:31.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:40:31.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:30 smithi092 bash[58855]: audit 2023-09-23T19:40:30.317233+0000 mon.a (mon.0) 1256 : audit 0 from='client.? 172.21.15.16:0/33797417' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:40:31.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:30 smithi092 bash[58855]: audit 2023-09-23T19:40:30.440538+0000 mon.a (mon.0) 1257 : audit 1 from='client.? 172.21.15.16:0/672772201' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T19:40:31.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:30 smithi092 bash[58855]: audit 2023-09-23T19:40:30.771938+0000 mon.a (mon.0) 1258 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:40:31.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:30 smithi092 bash[58855]: audit 2023-09-23T19:40:30.776473+0000 mon.b (mon.1) 47 : audit 1 from='client.? 172.21.15.16:0/179895324' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:40:31.296 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:31.325 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:31.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:31.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:40:31.428 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:31.456 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:31.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:31.458 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:40:31.559 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:31.587 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:31.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:31.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:40:31.690 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:31.719 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:31.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:31.721 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:31.821 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:31.850 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:31.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:31.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:40:31.954 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:31.987 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:31.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:31.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:32.091 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:32.099 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:31 smithi092 bash[58855]: cluster 2023-09-23T19:40:30.173279+0000 mgr.y (mgr.25325) 990 : cluster 0 pgmap v932: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 822 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 4246/39489 objects degraded (10.752%); 0 B/s, 51 objects/s recovering 2023-09-23T19:40:32.099 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:31 smithi092 bash[58855]: cluster 2023-09-23T19:40:30.957700+0000 mon.a (mon.0) 1259 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:40:32.099 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:31 smithi092 bash[58855]: audit 2023-09-23T19:40:30.977330+0000 mon.a (mon.0) 1260 : audit 1 from='client.? 172.21.15.16:0/672772201' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T19:40:32.099 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:31 smithi092 bash[58855]: cluster 2023-09-23T19:40:30.977450+0000 osd.4 (osd.4) 110 : cluster 3 failed to encode map e172 with expected crc 2023-09-23T19:40:32.099 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:31 smithi092 bash[58855]: audit 2023-09-23T19:40:30.977545+0000 mon.a (mon.0) 1261 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:40:32.100 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:31 smithi092 bash[58855]: cluster 2023-09-23T19:40:30.977973+0000 mon.a (mon.0) 1262 : cluster 0 osdmap e172: 8 total, 8 up, 8 in 2023-09-23T19:40:32.100 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:31 smithi092 bash[58855]: cluster 2023-09-23T19:40:30.993841+0000 osd.5 (osd.5) 102 : cluster 3 failed to encode map e172 with expected crc 2023-09-23T19:40:32.100 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:31 smithi092 bash[58855]: cluster 2023-09-23T19:40:31.004076+0000 osd.7 (osd.7) 83 : cluster 3 failed to encode map e172 with expected crc 2023-09-23T19:40:32.122 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:32.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:32.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:40:32.226 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:32.251 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:32.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:32.253 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:32.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:31 smithi016 bash[75004]: cluster 2023-09-23T19:40:30.173279+0000 mgr.y (mgr.25325) 990 : cluster 0 pgmap v932: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 822 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 4246/39489 objects degraded (10.752%); 0 B/s, 51 objects/s recovering 2023-09-23T19:40:32.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:31 smithi016 bash[75004]: cluster 2023-09-23T19:40:30.957700+0000 mon.a (mon.0) 1259 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:40:32.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:31 smithi016 bash[75004]: audit 2023-09-23T19:40:30.977330+0000 mon.a (mon.0) 1260 : audit 1 from='client.? 172.21.15.16:0/672772201' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T19:40:32.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:31 smithi016 bash[75004]: cluster 2023-09-23T19:40:30.977450+0000 osd.4 (osd.4) 110 : cluster 3 failed to encode map e172 with expected crc 2023-09-23T19:40:32.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:31 smithi016 bash[75004]: audit 2023-09-23T19:40:30.977545+0000 mon.a (mon.0) 1261 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:40:32.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:31 smithi016 bash[75004]: cluster 2023-09-23T19:40:30.977973+0000 mon.a (mon.0) 1262 : cluster 0 osdmap e172: 8 total, 8 up, 8 in 2023-09-23T19:40:32.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:31 smithi016 bash[75004]: cluster 2023-09-23T19:40:30.993841+0000 osd.5 (osd.5) 102 : cluster 3 failed to encode map e172 with expected crc 2023-09-23T19:40:32.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:31 smithi016 bash[75004]: cluster 2023-09-23T19:40:31.004076+0000 osd.7 (osd.7) 83 : cluster 3 failed to encode map e172 with expected crc 2023-09-23T19:40:32.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:31 smithi016 bash[77995]: cluster 2023-09-23T19:40:30.173279+0000 mgr.y (mgr.25325) 990 : cluster 0 pgmap v932: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 822 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 4246/39489 objects degraded (10.752%); 0 B/s, 51 objects/s recovering 2023-09-23T19:40:32.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:31 smithi016 bash[77995]: cluster 2023-09-23T19:40:30.957700+0000 mon.a (mon.0) 1259 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:40:32.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:31 smithi016 bash[77995]: audit 2023-09-23T19:40:30.977330+0000 mon.a (mon.0) 1260 : audit 1 from='client.? 172.21.15.16:0/672772201' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T19:40:32.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:31 smithi016 bash[77995]: cluster 2023-09-23T19:40:30.977450+0000 osd.4 (osd.4) 110 : cluster 3 failed to encode map e172 with expected crc 2023-09-23T19:40:32.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:31 smithi016 bash[77995]: audit 2023-09-23T19:40:30.977545+0000 mon.a (mon.0) 1261 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:40:32.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:31 smithi016 bash[77995]: cluster 2023-09-23T19:40:30.977973+0000 mon.a (mon.0) 1262 : cluster 0 osdmap e172: 8 total, 8 up, 8 in 2023-09-23T19:40:32.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:31 smithi016 bash[77995]: cluster 2023-09-23T19:40:30.993841+0000 osd.5 (osd.5) 102 : cluster 3 failed to encode map e172 with expected crc 2023-09-23T19:40:32.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:31 smithi016 bash[77995]: cluster 2023-09-23T19:40:31.004076+0000 osd.7 (osd.7) 83 : cluster 3 failed to encode map e172 with expected crc 2023-09-23T19:40:32.354 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:32.382 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:32.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:32.385 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:40:32.486 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:32.517 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:32.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:32.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:40:32.621 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:32.666 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:32.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:32.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:40:32.774 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:32.798 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:32.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:32.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:32.905 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:32.936 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:32.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:32.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:32.993 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:32 smithi016 bash[75004]: cluster 2023-09-23T19:40:31.975442+0000 mon.a (mon.0) 1263 : cluster 0 osdmap e173: 8 total, 8 up, 8 in 2023-09-23T19:40:32.993 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:32 smithi016 bash[75004]: cluster 2023-09-23T19:40:31.984733+0000 osd.5 (osd.5) 103 : cluster 3 failed to encode map e173 with expected crc 2023-09-23T19:40:32.993 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:32 smithi016 bash[75004]: cluster 2023-09-23T19:40:32.037720+0000 osd.6 (osd.6) 95 : cluster 3 failed to encode map e173 with expected crc 2023-09-23T19:40:32.993 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:32 smithi016 bash[77995]: cluster 2023-09-23T19:40:31.975442+0000 mon.a (mon.0) 1263 : cluster 0 osdmap e173: 8 total, 8 up, 8 in 2023-09-23T19:40:32.993 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:32 smithi016 bash[77995]: cluster 2023-09-23T19:40:31.984733+0000 osd.5 (osd.5) 103 : cluster 3 failed to encode map e173 with expected crc 2023-09-23T19:40:32.993 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:32 smithi016 bash[77995]: cluster 2023-09-23T19:40:32.037720+0000 osd.6 (osd.6) 95 : cluster 3 failed to encode map e173 with expected crc 2023-09-23T19:40:33.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:32 smithi092 bash[58855]: cluster 2023-09-23T19:40:31.975442+0000 mon.a (mon.0) 1263 : cluster 0 osdmap e173: 8 total, 8 up, 8 in 2023-09-23T19:40:33.062 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:32 smithi092 bash[58855]: cluster 2023-09-23T19:40:31.984733+0000 osd.5 (osd.5) 103 : cluster 3 failed to encode map e173 with expected crc 2023-09-23T19:40:33.063 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:32 smithi092 bash[58855]: cluster 2023-09-23T19:40:32.037720+0000 osd.6 (osd.6) 95 : cluster 3 failed to encode map e173 with expected crc 2023-09-23T19:40:33.063 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:33.095 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:33.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:33.098 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:40:33.199 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:33.236 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:33.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:33.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:40:33.344 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:33.373 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:33.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:33.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:40:33.477 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:33.514 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:33.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:33.516 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:40:33.617 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:33.651 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:33.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:33.658 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:40:33.758 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:33.789 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:33.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:33.793 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:33.892 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T19:40:33.894 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:33.970 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:33.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:33.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:34.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:33 smithi092 bash[58855]: cluster 2023-09-23T19:40:32.173830+0000 mgr.y (mgr.25325) 991 : cluster 0 pgmap v935: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 737 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 4246/35400 objects degraded (11.994%); 0 B/s, 88 objects/s recovering 2023-09-23T19:40:34.074 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:34.104 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:34.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:34.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:34.207 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:34.219 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:33 smithi016 bash[75004]: cluster 2023-09-23T19:40:32.173830+0000 mgr.y (mgr.25325) 991 : cluster 0 pgmap v935: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 737 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 4246/35400 objects degraded (11.994%); 0 B/s, 88 objects/s recovering 2023-09-23T19:40:34.219 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:33 smithi016 bash[77995]: cluster 2023-09-23T19:40:32.173830+0000 mgr.y (mgr.25325) 991 : cluster 0 pgmap v935: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 737 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 4246/35400 objects degraded (11.994%); 0 B/s, 88 objects/s recovering 2023-09-23T19:40:34.245 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:34.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:34.254 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:40:34.355 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:34.389 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:34.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:34.396 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:40:34.496 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:34.529 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:34.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:34.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:40:34.635 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:34.677 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:34.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:34.683 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:34.784 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:34.820 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:34.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:34.826 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:34.926 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:34.971 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:34.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:34.976 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:35.077 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:35.087 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:35 smithi016 bash[77995]: cluster 2023-09-23T19:40:34.462792+0000 mon.a (mon.0) 1264 : cluster 3 Health check update: Degraded data redundancy: 4246/35400 objects degraded (11.994%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T19:40:35.087 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:35 smithi016 bash[75004]: cluster 2023-09-23T19:40:34.462792+0000 mon.a (mon.0) 1264 : cluster 3 Health check update: Degraded data redundancy: 4246/35400 objects degraded (11.994%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T19:40:35.111 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:35.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:35.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:35.215 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:35.244 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:40:35.256 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:35.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:35.261 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:40:35.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:35 smithi092 bash[58855]: cluster 2023-09-23T19:40:34.462792+0000 mon.a (mon.0) 1264 : cluster 3 Health check update: Degraded data redundancy: 4246/35400 objects degraded (11.994%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T19:40:35.361 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:35.387 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:35.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:35.392 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:35.493 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:35.525 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:35.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:35.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:35.632 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:35.668 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:35.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:35.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:35.761 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:40:35.776 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:35.835 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:35.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:35.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:40:35.872 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T19:40:35.895 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:40:35 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:40:35] "GET /metrics HTTP/1.1" 200 36308 "" "Prometheus/2.33.4" 2023-09-23T19:40:35.943 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:35.978 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:35.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:35.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:40:36.088 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:36.134 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:36.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:36.136 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:40:36.237 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:36.244 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:36 smithi092 bash[58855]: cluster 2023-09-23T19:40:34.174501+0000 mgr.y (mgr.25325) 992 : cluster 0 pgmap v936: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 705 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 483 op/s; 4246/33894 objects degraded (12.527%); 0 B/s, 88 objects/s recovering 2023-09-23T19:40:36.273 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:36.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:36.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:40:36.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:36 smithi016 bash[75004]: cluster 2023-09-23T19:40:34.174501+0000 mgr.y (mgr.25325) 992 : cluster 0 pgmap v936: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 705 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 483 op/s; 4246/33894 objects degraded (12.527%); 0 B/s, 88 objects/s recovering 2023-09-23T19:40:36.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:36 smithi016 bash[77995]: cluster 2023-09-23T19:40:34.174501+0000 mgr.y (mgr.25325) 992 : cluster 0 pgmap v936: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 705 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 483 op/s; 4246/33894 objects degraded (12.527%); 0 B/s, 88 objects/s recovering 2023-09-23T19:40:36.378 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:36.406 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:36.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:36.413 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:40:36.513 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:36.550 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:36.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:36.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:36.655 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:36.683 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:36.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:36.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:40:36.789 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:36.820 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:36.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:36.822 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:40:36.923 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:36.948 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:36.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:36.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:40:37.052 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:37.075 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:37.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:37.078 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:37.178 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:37.202 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:37.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:37.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:37.304 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:37.337 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:37.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:37.341 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:40:37.442 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:37.465 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:37.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:37.470 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:37.571 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:37.603 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:37.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:37.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:37.710 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:37.741 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:37.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:37.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:40:37.846 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:37.876 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:37.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:37.881 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:40:37.982 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:38.014 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:38.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:38.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:38.117 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:38.124 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:38 smithi092 bash[58855]: cluster 2023-09-23T19:40:36.175308+0000 mgr.y (mgr.25325) 993 : cluster 0 pgmap v937: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 567 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 2833/27267 objects degraded (10.390%); 0 B/s, 87 objects/s recovering 2023-09-23T19:40:38.169 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:38.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:38.173 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:38.274 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:38.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:38 smithi016 bash[77995]: cluster 2023-09-23T19:40:36.175308+0000 mgr.y (mgr.25325) 993 : cluster 0 pgmap v937: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 567 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 2833/27267 objects degraded (10.390%); 0 B/s, 87 objects/s recovering 2023-09-23T19:40:38.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:38 smithi016 bash[75004]: cluster 2023-09-23T19:40:36.175308+0000 mgr.y (mgr.25325) 993 : cluster 0 pgmap v937: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 567 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 2833/27267 objects degraded (10.390%); 0 B/s, 87 objects/s recovering 2023-09-23T19:40:38.309 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:38.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:38.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:40:38.413 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:38.451 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:38.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:38.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:38.556 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:38.596 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:38.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:38.601 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:38.701 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:38.733 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:38.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:38.741 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:40:38.821 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:40:38.841 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:38.878 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:38.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:38.885 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:40:38.985 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:39.020 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:39.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:39.021 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:40:39.122 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:39.150 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:39.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:39.154 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:39.254 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:39.309 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:39.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:39.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:39.416 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:39.451 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:39.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:39.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:40:39.557 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:39.587 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:39.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:39.592 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:40:39.693 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:39.716 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:39.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:39.718 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:39.819 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:39.851 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:40:39.851 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:40:39.852 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:39.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:39.856 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:40:39.957 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:40.007 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:40.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:40.014 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:40.115 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:40.139 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:40.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:40.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:40.244 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:40.253 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:40 smithi092 bash[58855]: cluster 2023-09-23T19:40:38.175912+0000 mgr.y (mgr.25325) 994 : cluster 0 pgmap v938: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 554 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 535 op/s; 2833/26640 objects degraded (10.634%); 0 B/s, 87 objects/s recovering 2023-09-23T19:40:40.253 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:40 smithi092 bash[58855]: cluster 2023-09-23T19:40:39.464062+0000 mon.a (mon.0) 1265 : cluster 3 Health check update: Degraded data redundancy: 2833/26640 objects degraded (10.634%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T19:40:40.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:40 smithi016 bash[77995]: cluster 2023-09-23T19:40:38.175912+0000 mgr.y (mgr.25325) 994 : cluster 0 pgmap v938: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 554 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 535 op/s; 2833/26640 objects degraded (10.634%); 0 B/s, 87 objects/s recovering 2023-09-23T19:40:40.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:40 smithi016 bash[77995]: cluster 2023-09-23T19:40:39.464062+0000 mon.a (mon.0) 1265 : cluster 3 Health check update: Degraded data redundancy: 2833/26640 objects degraded (10.634%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T19:40:40.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:40 smithi016 bash[75004]: cluster 2023-09-23T19:40:38.175912+0000 mgr.y (mgr.25325) 994 : cluster 0 pgmap v938: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 554 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 535 op/s; 2833/26640 objects degraded (10.634%); 0 B/s, 87 objects/s recovering 2023-09-23T19:40:40.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:40 smithi016 bash[75004]: cluster 2023-09-23T19:40:39.464062+0000 mon.a (mon.0) 1265 : cluster 3 Health check update: Degraded data redundancy: 2833/26640 objects degraded (10.634%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T19:40:40.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:40.287 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:40.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:40.388 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:40.412 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:40.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:40.419 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:40:40.519 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:40.544 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:40.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:40.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:40:40.649 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:40.679 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:40.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:40.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:40:40.703 INFO:teuthology.orchestra.run.smithi092.stdout:ERROR: (22) Invalid argument 2023-09-23T19:40:40.704 INFO:teuthology.orchestra.run.smithi092.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:40:40.789 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:40.829 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:40.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:40.833 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:40:40.933 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:40.966 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:40.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:40.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:41.069 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:41.110 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:41.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:41.113 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:41.213 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:41.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:41.242 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:41.243 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:41.343 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:41.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:41.377 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:41.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:41.477 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:41.512 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:41.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:41.518 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:40:41.618 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:41.647 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:41.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:41.656 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:40:41.757 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:41.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:41.799 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:41.799 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:40:41.900 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:41.937 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:41.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:41.941 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:40:41.960 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T19:40:42.022 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:42 smithi016 bash[77995]: cluster 2023-09-23T19:40:40.176552+0000 mgr.y (mgr.25325) 995 : cluster 0 pgmap v939: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 432 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 548 op/s; 2833/20784 objects degraded (13.631%); 0 B/s, 75 objects/s recovering 2023-09-23T19:40:42.023 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:42 smithi016 bash[75004]: cluster 2023-09-23T19:40:40.176552+0000 mgr.y (mgr.25325) 995 : cluster 0 pgmap v939: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 432 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 548 op/s; 2833/20784 objects degraded (13.631%); 0 B/s, 75 objects/s recovering 2023-09-23T19:40:42.042 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:42.105 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:42.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:42.117 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:40:42.218 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:42.227 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:42 smithi092 bash[58855]: cluster 2023-09-23T19:40:40.176552+0000 mgr.y (mgr.25325) 995 : cluster 0 pgmap v939: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 432 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 548 op/s; 2833/20784 objects degraded (13.631%); 0 B/s, 75 objects/s recovering 2023-09-23T19:40:42.261 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:42.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:42.264 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:40:42.365 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:42.400 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:42.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:42.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:40:42.507 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:42.552 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:42.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:42.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:40:42.655 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:42.681 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:42.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:42.684 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:40:42.785 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:42.808 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:42.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:42.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:42.910 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:42.945 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:42.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:42.952 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:43.053 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:43.085 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:43.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:43.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:43.189 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:43.229 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:43.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:43.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:43.334 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:43.354 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:43.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:43.356 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:40:43.457 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:43.486 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:43.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:43.492 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:40:43.593 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:43.617 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:43.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:43.624 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:40:43.724 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:43.764 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:43.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:43.768 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:43.869 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:43.895 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:43.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:43.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:44.003 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:44.037 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:44.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:44.041 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:40:44.142 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:44.150 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:44 smithi016 bash[77995]: cluster 2023-09-23T19:40:42.177369+0000 mgr.y (mgr.25325) 996 : cluster 0 pgmap v940: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 340 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 638 op/s; 2833/16380 objects degraded (17.295%); 0 B/s, 92 objects/s recovering 2023-09-23T19:40:44.150 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:44 smithi016 bash[77995]: audit 2023-09-23T19:40:43.667370+0000 mon.a (mon.0) 1266 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:40:44.150 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:44 smithi016 bash[77995]: audit 2023-09-23T19:40:43.668674+0000 mon.a (mon.0) 1267 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:40:44.150 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:44 smithi016 bash[75004]: cluster 2023-09-23T19:40:42.177369+0000 mgr.y (mgr.25325) 996 : cluster 0 pgmap v940: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 340 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 638 op/s; 2833/16380 objects degraded (17.295%); 0 B/s, 92 objects/s recovering 2023-09-23T19:40:44.151 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:44 smithi016 bash[75004]: audit 2023-09-23T19:40:43.667370+0000 mon.a (mon.0) 1266 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:40:44.151 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:44 smithi016 bash[75004]: audit 2023-09-23T19:40:43.668674+0000 mon.a (mon.0) 1267 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:40:44.172 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:44.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:44.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:40:44.276 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:44.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:44 smithi092 bash[58855]: cluster 2023-09-23T19:40:42.177369+0000 mgr.y (mgr.25325) 996 : cluster 0 pgmap v940: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 340 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 638 op/s; 2833/16380 objects degraded (17.295%); 0 B/s, 92 objects/s recovering 2023-09-23T19:40:44.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:44 smithi092 bash[58855]: audit 2023-09-23T19:40:43.667370+0000 mon.a (mon.0) 1266 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:40:44.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:44 smithi092 bash[58855]: audit 2023-09-23T19:40:43.668674+0000 mon.a (mon.0) 1267 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:40:44.309 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:44.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:44.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:44.412 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:44.451 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:44.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:44.456 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:44.557 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:44.584 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:44.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:44.591 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:40:44.691 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:44.714 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:44.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:44.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:44.821 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:44.845 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:44.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:44.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:40:44.950 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:44.998 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:45.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:45.001 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:40:45.102 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:45.137 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:45.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:45.141 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:40:45.241 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:45.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:45 smithi016 bash[75004]: cluster 2023-09-23T19:40:44.465172+0000 mon.a (mon.0) 1268 : cluster 3 Health check update: Degraded data redundancy: 2833/16380 objects degraded (17.295%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T19:40:45.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:45 smithi016 bash[77995]: cluster 2023-09-23T19:40:44.465172+0000 mon.a (mon.0) 1268 : cluster 3 Health check update: Degraded data redundancy: 2833/16380 objects degraded (17.295%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T19:40:45.283 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:45.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:45.287 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:40:45.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:45 smithi092 bash[58855]: cluster 2023-09-23T19:40:44.465172+0000 mon.a (mon.0) 1268 : cluster 3 Health check update: Degraded data redundancy: 2833/16380 objects degraded (17.295%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T19:40:45.387 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:45.421 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:45.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:45.427 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:40:45.528 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:45.554 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:45.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:45.557 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:40:45.658 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:45.685 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:45.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:45.692 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:45.793 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:45.821 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:45.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:45.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:45.888 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:40:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:40:45] "GET /metrics HTTP/1.1" 200 36301 "" "Prometheus/2.33.4" 2023-09-23T19:40:45.930 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:45.960 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:45.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:45.964 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:40:46.065 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:46.110 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:46.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:46.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:40:46.214 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:46.223 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:46 smithi092 bash[58855]: cluster 2023-09-23T19:40:44.177903+0000 mgr.y (mgr.25325) 997 : cluster 0 pgmap v941: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 300 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 2833/14439 objects degraded (19.620%); 0 B/s, 46 objects/s recovering 2023-09-23T19:40:46.236 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:46.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:46.241 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:40:46.341 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:46.371 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:46.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:46.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:40:46.475 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:46.511 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:46.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:46.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:46.549 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:46 smithi016 bash[77995]: cluster 2023-09-23T19:40:44.177903+0000 mgr.y (mgr.25325) 997 : cluster 0 pgmap v941: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 300 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 2833/14439 objects degraded (19.620%); 0 B/s, 46 objects/s recovering 2023-09-23T19:40:46.549 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:46 smithi016 bash[75004]: cluster 2023-09-23T19:40:44.177903+0000 mgr.y (mgr.25325) 997 : cluster 0 pgmap v941: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 300 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 2833/14439 objects degraded (19.620%); 0 B/s, 46 objects/s recovering 2023-09-23T19:40:46.617 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:46.648 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:46.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:46.656 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:40:46.757 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:46.793 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:46.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:46.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:46.874 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:40:46.898 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:46.937 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:46.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:46.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:47.041 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:47.067 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:47.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:47.073 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:40:47.174 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:47.210 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:47.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:47.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:47.317 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:47.341 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:47.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:47.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:40:47.446 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:47.487 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:47.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:47.493 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:40:47.594 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:47.633 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:47.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:47.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:47.737 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:47.768 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:47.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:47.776 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:47.877 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:47.907 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:47.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:47.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:48.010 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:48.037 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:48.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:48.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:48.077 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Removed 110131 objects 2023-09-23T19:40:48.078 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Clean up completed and total clean up time :162.572 2023-09-23T19:40:48.139 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:48.163 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:48.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:48.165 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:40:48.265 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:48.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:48 smithi016 bash[75004]: cluster 2023-09-23T19:40:46.178651+0000 mgr.y (mgr.25325) 998 : cluster 0 pgmap v942: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 141 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 751 op/s; 2833/6834 objects degraded (41.454%); 0 B/s, 68 objects/s recovering 2023-09-23T19:40:48.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:48 smithi016 bash[77995]: cluster 2023-09-23T19:40:46.178651+0000 mgr.y (mgr.25325) 998 : cluster 0 pgmap v942: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 141 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 751 op/s; 2833/6834 objects degraded (41.454%); 0 B/s, 68 objects/s recovering 2023-09-23T19:40:48.295 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:48.297 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:48 smithi092 bash[58855]: cluster 2023-09-23T19:40:46.178651+0000 mgr.y (mgr.25325) 998 : cluster 0 pgmap v942: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 141 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 751 op/s; 2833/6834 objects degraded (41.454%); 0 B/s, 68 objects/s recovering 2023-09-23T19:40:48.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:48.298 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:40:48.398 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:48.420 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:48.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:48.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:48.523 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:48.552 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:48.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:48.554 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:48.656 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:48.682 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:48.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:48.685 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:48.785 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:48.813 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:48.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:48.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:48.918 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:48.947 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:48.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:48.949 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:49.050 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:49.071 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:49.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:49.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:40:49.174 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:49.180 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:40:49.195 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:49.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:49.198 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:40:49.202 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:40:49.225 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:40:49.247 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:40:49.270 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:40:49.291 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:40:49.299 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:49.313 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:40:49.329 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:49.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:49.332 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:49.335 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:40:49.336 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:40:49.336 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:40:49.337 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:40:49.337 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:40:49.337 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:40:49.337 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:40:49.337 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:40:49.337 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:40:49.432 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:49.460 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:49.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:49.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:49.563 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:49.588 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:49.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:49.592 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:49.692 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:49.715 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:49.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:49.719 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:49.819 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:49.840 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:49.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:49.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:40:49.943 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:49.971 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:49.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:49.973 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:40:50.049 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T19:40:50.052 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is set 2023-09-23T19:40:50.074 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:50.098 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:50.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:50.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:40:50.201 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:50.225 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:50.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:50.228 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:40:50.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:50 smithi092 bash[58855]: cluster 2023-09-23T19:40:48.179369+0000 mgr.y (mgr.25325) 999 : cluster 0 pgmap v943: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 129 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 2833/6240 objects degraded (45.401%); 0 B/s, 42 objects/s recovering 2023-09-23T19:40:50.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:50 smithi092 bash[58855]: audit 2023-09-23T19:40:49.151802+0000 mon.c (mon.2) 175 : audit 0 from='client.? 172.21.15.16:0/127890057' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:40:50.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:50 smithi092 bash[58855]: audit 2023-09-23T19:40:49.239749+0000 mon.a (mon.0) 1269 : audit 1 from='client.? 172.21.15.16:0/1841504546' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T19:40:50.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:50 smithi092 bash[58855]: cluster 2023-09-23T19:40:49.466245+0000 mon.a (mon.0) 1270 : cluster 3 Health check update: Degraded data redundancy: 2833/6240 objects degraded (45.401%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T19:40:50.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:50 smithi092 bash[58855]: audit 2023-09-23T19:40:49.664867+0000 mon.c (mon.2) 176 : audit 1 from='client.? 172.21.15.16:0/93976912' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:40:50.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:50 smithi092 bash[58855]: audit 2023-09-23T19:40:49.665264+0000 mon.a (mon.0) 1271 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:40:50.329 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:50.356 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:50.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:50.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:40:50.457 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:50 smithi016 bash[77995]: cluster 2023-09-23T19:40:48.179369+0000 mgr.y (mgr.25325) 999 : cluster 0 pgmap v943: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 129 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 2833/6240 objects degraded (45.401%); 0 B/s, 42 objects/s recovering 2023-09-23T19:40:50.457 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:50 smithi016 bash[77995]: audit 2023-09-23T19:40:49.151802+0000 mon.c (mon.2) 175 : audit 0 from='client.? 172.21.15.16:0/127890057' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:40:50.457 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:50 smithi016 bash[77995]: audit 2023-09-23T19:40:49.239749+0000 mon.a (mon.0) 1269 : audit 1 from='client.? 172.21.15.16:0/1841504546' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T19:40:50.457 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:50 smithi016 bash[77995]: cluster 2023-09-23T19:40:49.466245+0000 mon.a (mon.0) 1270 : cluster 3 Health check update: Degraded data redundancy: 2833/6240 objects degraded (45.401%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T19:40:50.457 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:50 smithi016 bash[77995]: audit 2023-09-23T19:40:49.664867+0000 mon.c (mon.2) 176 : audit 1 from='client.? 172.21.15.16:0/93976912' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:40:50.458 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:50 smithi016 bash[77995]: audit 2023-09-23T19:40:49.665264+0000 mon.a (mon.0) 1271 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:40:50.458 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:50 smithi016 bash[75004]: cluster 2023-09-23T19:40:48.179369+0000 mgr.y (mgr.25325) 999 : cluster 0 pgmap v943: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 129 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 2833/6240 objects degraded (45.401%); 0 B/s, 42 objects/s recovering 2023-09-23T19:40:50.458 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:50 smithi016 bash[75004]: audit 2023-09-23T19:40:49.151802+0000 mon.c (mon.2) 175 : audit 0 from='client.? 172.21.15.16:0/127890057' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:40:50.458 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:50 smithi016 bash[75004]: audit 2023-09-23T19:40:49.239749+0000 mon.a (mon.0) 1269 : audit 1 from='client.? 172.21.15.16:0/1841504546' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T19:40:50.458 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:50 smithi016 bash[75004]: cluster 2023-09-23T19:40:49.466245+0000 mon.a (mon.0) 1270 : cluster 3 Health check update: Degraded data redundancy: 2833/6240 objects degraded (45.401%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T19:40:50.458 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:50 smithi016 bash[75004]: audit 2023-09-23T19:40:49.664867+0000 mon.c (mon.2) 176 : audit 1 from='client.? 172.21.15.16:0/93976912' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:40:50.459 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:50 smithi016 bash[75004]: audit 2023-09-23T19:40:49.665264+0000 mon.a (mon.0) 1271 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:40:50.460 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:50.482 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:50.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:50.486 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:40:50.586 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:50.611 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:50.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:50.613 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:40:50.713 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:50.735 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:50.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:50.738 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:40:50.839 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:50.861 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:50.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:50.864 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:50.965 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:50.999 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:51.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:51.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:40:51.055 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:51 smithi016 bash[75004]: cluster 2023-09-23T19:40:50.035835+0000 mon.a (mon.0) 1272 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:40:51.056 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:51 smithi016 bash[75004]: audit 2023-09-23T19:40:50.041091+0000 mon.a (mon.0) 1273 : audit 1 from='client.? 172.21.15.16:0/1841504546' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T19:40:51.056 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:51 smithi016 bash[75004]: audit 2023-09-23T19:40:50.041176+0000 mon.a (mon.0) 1274 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:40:51.103 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:51.113 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:51 smithi092 bash[58855]: cluster 2023-09-23T19:40:50.035835+0000 mon.a (mon.0) 1272 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:40:51.113 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:51 smithi092 bash[58855]: audit 2023-09-23T19:40:50.041091+0000 mon.a (mon.0) 1273 : audit 1 from='client.? 172.21.15.16:0/1841504546' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T19:40:51.114 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:51 smithi092 bash[58855]: audit 2023-09-23T19:40:50.041176+0000 mon.a (mon.0) 1274 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:40:51.114 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:51 smithi092 bash[58855]: cluster 2023-09-23T19:40:50.041244+0000 mon.a (mon.0) 1275 : cluster 0 osdmap e174: 8 total, 8 up, 8 in 2023-09-23T19:40:51.114 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:51 smithi092 bash[58855]: cluster 2023-09-23T19:40:50.047102+0000 osd.5 (osd.5) 104 : cluster 3 failed to encode map e174 with expected crc 2023-09-23T19:40:51.114 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:51 smithi092 bash[58855]: cluster 2023-09-23T19:40:50.312122+0000 osd.7 (osd.7) 84 : cluster 3 failed to encode map e174 with expected crc 2023-09-23T19:40:51.128 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:51.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:51.130 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:51.231 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:51.250 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:51.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:51.253 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:40:51.354 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:51.366 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:51 smithi016 bash[77995]: cluster 2023-09-23T19:40:50.035835+0000 mon.a (mon.0) 1272 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:40:51.366 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:51 smithi016 bash[77995]: audit 2023-09-23T19:40:50.041091+0000 mon.a (mon.0) 1273 : audit 1 from='client.? 172.21.15.16:0/1841504546' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T19:40:51.366 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:51 smithi016 bash[77995]: audit 2023-09-23T19:40:50.041176+0000 mon.a (mon.0) 1274 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:40:51.366 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:51 smithi016 bash[77995]: cluster 2023-09-23T19:40:50.041244+0000 mon.a (mon.0) 1275 : cluster 0 osdmap e174: 8 total, 8 up, 8 in 2023-09-23T19:40:51.366 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:51 smithi016 bash[77995]: cluster 2023-09-23T19:40:50.047102+0000 osd.5 (osd.5) 104 : cluster 3 failed to encode map e174 with expected crc 2023-09-23T19:40:51.366 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:51 smithi016 bash[77995]: cluster 2023-09-23T19:40:50.312122+0000 osd.7 (osd.7) 84 : cluster 3 failed to encode map e174 with expected crc 2023-09-23T19:40:51.367 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:51 smithi016 bash[75004]: cluster 2023-09-23T19:40:50.041244+0000 mon.a (mon.0) 1275 : cluster 0 osdmap e174: 8 total, 8 up, 8 in 2023-09-23T19:40:51.367 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:51 smithi016 bash[75004]: cluster 2023-09-23T19:40:50.047102+0000 osd.5 (osd.5) 104 : cluster 3 failed to encode map e174 with expected crc 2023-09-23T19:40:51.367 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:51 smithi016 bash[75004]: cluster 2023-09-23T19:40:50.312122+0000 osd.7 (osd.7) 84 : cluster 3 failed to encode map e174 with expected crc 2023-09-23T19:40:51.384 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:51.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:51.386 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:40:51.487 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:51.513 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:51.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:51.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:40:51.618 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:51.620 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_5 2023-09-23T19:40:51.621 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_5 unique_pool_5 --yes-i-really-really-mean-it 2023-09-23T19:40:51.648 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:51.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:51.651 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:51.741 INFO:teuthology.orchestra.run.smithi092.stdout:{ 2023-09-23T19:40:51.741 INFO:teuthology.orchestra.run.smithi092.stdout: "ops": [], 2023-09-23T19:40:51.741 INFO:teuthology.orchestra.run.smithi092.stdout: "num_ops": 0 2023-09-23T19:40:51.741 INFO:teuthology.orchestra.run.smithi092.stdout:} 2023-09-23T19:40:51.751 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:51.777 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:51.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:51.780 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:40:51.881 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:51.911 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:51.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:51.913 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:40:52.014 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:52.046 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:52.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:52.048 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:40:52.049 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T19:40:52.149 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:52.172 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:52.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:52.175 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:52.275 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:52.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:52 smithi016 bash[75004]: cluster 2023-09-23T19:40:50.179900+0000 mgr.y (mgr.25325) 1000 : cluster 0 pgmap v945: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 44 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 2833/2145 objects degraded (132.075%); 0 B/s, 51 objects/s recovering 2023-09-23T19:40:52.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:52 smithi016 bash[75004]: cluster 2023-09-23T19:40:50.456996+0000 osd.6 (osd.6) 96 : cluster 3 failed to encode map e174 with expected crc 2023-09-23T19:40:52.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:52 smithi016 bash[75004]: cluster 2023-09-23T19:40:50.457124+0000 osd.4 (osd.4) 111 : cluster 3 failed to encode map e174 with expected crc 2023-09-23T19:40:52.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:52 smithi016 bash[75004]: cluster 2023-09-23T19:40:51.057317+0000 osd.6 (osd.6) 97 : cluster 3 failed to encode map e175 with expected crc 2023-09-23T19:40:52.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:52 smithi016 bash[75004]: cluster 2023-09-23T19:40:51.057584+0000 mon.a (mon.0) 1276 : cluster 0 osdmap e175: 8 total, 8 up, 8 in 2023-09-23T19:40:52.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:52 smithi016 bash[75004]: cluster 2023-09-23T19:40:51.060680+0000 osd.5 (osd.5) 105 : cluster 3 failed to encode map e175 with expected crc 2023-09-23T19:40:52.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:52 smithi016 bash[75004]: cluster 2023-09-23T19:40:51.063366+0000 osd.5 (osd.5) 106 : cluster 3 failed to encode map e175 with expected crc 2023-09-23T19:40:52.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:52 smithi016 bash[75004]: cluster 2023-09-23T19:40:51.200386+0000 osd.4 (osd.4) 112 : cluster 3 failed to encode map e175 with expected crc 2023-09-23T19:40:52.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:52 smithi016 bash[77995]: cluster 2023-09-23T19:40:50.179900+0000 mgr.y (mgr.25325) 1000 : cluster 0 pgmap v945: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 44 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 2833/2145 objects degraded (132.075%); 0 B/s, 51 objects/s recovering 2023-09-23T19:40:52.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:52 smithi016 bash[77995]: cluster 2023-09-23T19:40:50.456996+0000 osd.6 (osd.6) 96 : cluster 3 failed to encode map e174 with expected crc 2023-09-23T19:40:52.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:52 smithi016 bash[77995]: cluster 2023-09-23T19:40:50.457124+0000 osd.4 (osd.4) 111 : cluster 3 failed to encode map e174 with expected crc 2023-09-23T19:40:52.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:52 smithi016 bash[77995]: cluster 2023-09-23T19:40:51.057317+0000 osd.6 (osd.6) 97 : cluster 3 failed to encode map e175 with expected crc 2023-09-23T19:40:52.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:52 smithi016 bash[77995]: cluster 2023-09-23T19:40:51.057584+0000 mon.a (mon.0) 1276 : cluster 0 osdmap e175: 8 total, 8 up, 8 in 2023-09-23T19:40:52.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:52 smithi016 bash[77995]: cluster 2023-09-23T19:40:51.060680+0000 osd.5 (osd.5) 105 : cluster 3 failed to encode map e175 with expected crc 2023-09-23T19:40:52.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:52 smithi016 bash[77995]: cluster 2023-09-23T19:40:51.063366+0000 osd.5 (osd.5) 106 : cluster 3 failed to encode map e175 with expected crc 2023-09-23T19:40:52.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:52 smithi016 bash[77995]: cluster 2023-09-23T19:40:51.200386+0000 osd.4 (osd.4) 112 : cluster 3 failed to encode map e175 with expected crc 2023-09-23T19:40:52.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:52 smithi092 bash[58855]: cluster 2023-09-23T19:40:50.179900+0000 mgr.y (mgr.25325) 1000 : cluster 0 pgmap v945: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 44 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 2833/2145 objects degraded (132.075%); 0 B/s, 51 objects/s recovering 2023-09-23T19:40:52.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:52 smithi092 bash[58855]: cluster 2023-09-23T19:40:50.456996+0000 osd.6 (osd.6) 96 : cluster 3 failed to encode map e174 with expected crc 2023-09-23T19:40:52.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:52 smithi092 bash[58855]: cluster 2023-09-23T19:40:50.457124+0000 osd.4 (osd.4) 111 : cluster 3 failed to encode map e174 with expected crc 2023-09-23T19:40:52.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:52 smithi092 bash[58855]: cluster 2023-09-23T19:40:51.057317+0000 osd.6 (osd.6) 97 : cluster 3 failed to encode map e175 with expected crc 2023-09-23T19:40:52.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:52 smithi092 bash[58855]: cluster 2023-09-23T19:40:51.057584+0000 mon.a (mon.0) 1276 : cluster 0 osdmap e175: 8 total, 8 up, 8 in 2023-09-23T19:40:52.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:52 smithi092 bash[58855]: cluster 2023-09-23T19:40:51.060680+0000 osd.5 (osd.5) 105 : cluster 3 failed to encode map e175 with expected crc 2023-09-23T19:40:52.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:52 smithi092 bash[58855]: cluster 2023-09-23T19:40:51.063366+0000 osd.5 (osd.5) 106 : cluster 3 failed to encode map e175 with expected crc 2023-09-23T19:40:52.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:52 smithi092 bash[58855]: cluster 2023-09-23T19:40:51.200386+0000 osd.4 (osd.4) 112 : cluster 3 failed to encode map e175 with expected crc 2023-09-23T19:40:52.304 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:52.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:52.307 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:40:52.408 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:52.438 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:52.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:52.441 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:40:52.460 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:40:52.542 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:52.568 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:52.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:52.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:52.585 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T19:40:52.673 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:52.701 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:52.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:52.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:52.804 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:52.833 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:52.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:52.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:40:52.937 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:52.972 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:52.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:52.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:53.076 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:53.104 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:53.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:53.107 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:40:53.208 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:53.238 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:53.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:53.242 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:40:53.342 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:53.356 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:53 smithi016 bash[75004]: cluster 2023-09-23T19:40:52.057398+0000 mon.a (mon.0) 1277 : cluster 0 osdmap e176: 8 total, 8 up, 8 in 2023-09-23T19:40:53.356 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:53 smithi016 bash[75004]: cluster 2023-09-23T19:40:52.063453+0000 osd.5 (osd.5) 107 : cluster 3 failed to encode map e176 with expected crc 2023-09-23T19:40:53.356 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:53 smithi016 bash[77995]: cluster 2023-09-23T19:40:52.057398+0000 mon.a (mon.0) 1277 : cluster 0 osdmap e176: 8 total, 8 up, 8 in 2023-09-23T19:40:53.356 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:53 smithi016 bash[77995]: cluster 2023-09-23T19:40:52.063453+0000 osd.5 (osd.5) 107 : cluster 3 failed to encode map e176 with expected crc 2023-09-23T19:40:53.377 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:53.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:53.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:53.481 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:53.494 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:53 smithi092 bash[58855]: cluster 2023-09-23T19:40:52.057398+0000 mon.a (mon.0) 1277 : cluster 0 osdmap e176: 8 total, 8 up, 8 in 2023-09-23T19:40:53.494 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:53 smithi092 bash[58855]: cluster 2023-09-23T19:40:52.063453+0000 osd.5 (osd.5) 107 : cluster 3 failed to encode map e176 with expected crc 2023-09-23T19:40:53.511 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:53.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:53.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:53.614 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:53.639 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:53.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:53.642 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:53.742 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:53.768 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:53.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:53.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:40:53.871 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:53.904 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:53.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:53.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:40:54.008 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:54.039 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:54.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:54.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:40:54.067 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T19:40:54.143 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:54.171 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:54.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:54.173 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:54.274 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:54.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:54 smithi092 bash[58855]: cluster 2023-09-23T19:40:52.180550+0000 mgr.y (mgr.25325) 1001 : cluster 0 pgmap v948: 89 pgs: 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 375 op/s; 2833/72 objects degraded (3934.722%); 0 B/s, 98 objects/s recovering 2023-09-23T19:40:54.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:54 smithi092 bash[58855]: cluster 2023-09-23T19:40:52.524290+0000 osd.4 (osd.4) 113 : cluster 3 failed to encode map e176 with expected crc 2023-09-23T19:40:54.303 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:54.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:54.306 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:54.407 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:54.432 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:54.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:54.435 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:40:54.535 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:54.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:54 smithi016 bash[77995]: cluster 2023-09-23T19:40:52.180550+0000 mgr.y (mgr.25325) 1001 : cluster 0 pgmap v948: 89 pgs: 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 375 op/s; 2833/72 objects degraded (3934.722%); 0 B/s, 98 objects/s recovering 2023-09-23T19:40:54.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:54 smithi016 bash[77995]: cluster 2023-09-23T19:40:52.524290+0000 osd.4 (osd.4) 113 : cluster 3 failed to encode map e176 with expected crc 2023-09-23T19:40:54.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:54 smithi016 bash[75004]: cluster 2023-09-23T19:40:52.180550+0000 mgr.y (mgr.25325) 1001 : cluster 0 pgmap v948: 89 pgs: 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 375 op/s; 2833/72 objects degraded (3934.722%); 0 B/s, 98 objects/s recovering 2023-09-23T19:40:54.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:54 smithi016 bash[75004]: cluster 2023-09-23T19:40:52.524290+0000 osd.4 (osd.4) 113 : cluster 3 failed to encode map e176 with expected crc 2023-09-23T19:40:54.566 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:54.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:54.569 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:54.669 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:54.698 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:54.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:54.701 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:54.802 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:54.832 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:54.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:54.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:40:54.936 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:54.969 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:54.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:54.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:55.074 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:55.103 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:55.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:55.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:40:55.207 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:55.235 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:55.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:55.239 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:55.339 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:55.352 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:55 smithi016 bash[75004]: cluster 2023-09-23T19:40:54.467281+0000 mon.a (mon.0) 1278 : cluster 3 Health check update: Degraded data redundancy: 2833/72 objects degraded (3934.722%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T19:40:55.352 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:55 smithi016 bash[77995]: cluster 2023-09-23T19:40:54.467281+0000 mon.a (mon.0) 1278 : cluster 3 Health check update: Degraded data redundancy: 2833/72 objects degraded (3934.722%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T19:40:55.376 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:55.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:55.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:55.480 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:55.492 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:55 smithi092 bash[58855]: cluster 2023-09-23T19:40:54.467281+0000 mon.a (mon.0) 1278 : cluster 3 Health check update: Degraded data redundancy: 2833/72 objects degraded (3934.722%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T19:40:55.510 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:55.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:55.513 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:40:55.614 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:55.646 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:55.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:55.649 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:55.750 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:55.778 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:55.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:55.782 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:40:55.882 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:55.894 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:40:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:40:55] "GET /metrics HTTP/1.1" 200 36301 "" "Prometheus/2.33.4" 2023-09-23T19:40:55.915 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:55.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:55.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:40:56.020 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:56.050 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:56.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:56.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:56.154 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:56.166 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:56 smithi016 bash[75004]: cluster 2023-09-23T19:40:54.181388+0000 mgr.y (mgr.25325) 1002 : cluster 0 pgmap v949: 89 pgs: 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 342 op/s; 2833/72 objects degraded (3934.722%); 0 B/s, 98 objects/s recovering 2023-09-23T19:40:56.166 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:56 smithi016 bash[77995]: cluster 2023-09-23T19:40:54.181388+0000 mgr.y (mgr.25325) 1002 : cluster 0 pgmap v949: 89 pgs: 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 342 op/s; 2833/72 objects degraded (3934.722%); 0 B/s, 98 objects/s recovering 2023-09-23T19:40:56.185 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:56.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:56.189 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:40:56.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:56 smithi092 bash[58855]: cluster 2023-09-23T19:40:54.181388+0000 mgr.y (mgr.25325) 1002 : cluster 0 pgmap v949: 89 pgs: 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 342 op/s; 2833/72 objects degraded (3934.722%); 0 B/s, 98 objects/s recovering 2023-09-23T19:40:56.289 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:56.320 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:56.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:56.322 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:56.423 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:56.434 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:40:56.436 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:40:56.437 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:40:56.456 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:56.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:56.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:40:56.560 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:56.591 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:56.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:56.594 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:40:56.695 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:56.720 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:56.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:56.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:40:56.824 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:56.852 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:56.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:56.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:40:56.956 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:56.981 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:56.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:56.985 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:40:57.086 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:57.117 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:57.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:57.119 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:40:57.220 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:57.248 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:40:57.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:57.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:40:57.352 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:57.387 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:40:57.387 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:57.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:57.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:57.490 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:57.518 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:57.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:57.521 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:40:57.622 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:57.645 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:57.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:57.648 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:57.750 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:57.774 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:57.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:57.777 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:40:57.877 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:57.909 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:57.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:57.912 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:40:58.013 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:58.039 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:58.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:58.041 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:58.141 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:58.169 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:58.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:58.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:58.272 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:58.281 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:58 smithi092 bash[58855]: cluster 2023-09-23T19:40:56.182829+0000 mgr.y (mgr.25325) 1003 : cluster 0 pgmap v950: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 238 op/s; 1386/72 objects degraded (1925.000%); 0 B/s, 113 objects/s recovering 2023-09-23T19:40:58.294 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:58.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:58.298 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:58.387 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:58 smithi016 bash[75004]: cluster 2023-09-23T19:40:56.182829+0000 mgr.y (mgr.25325) 1003 : cluster 0 pgmap v950: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 238 op/s; 1386/72 objects degraded (1925.000%); 0 B/s, 113 objects/s recovering 2023-09-23T19:40:58.387 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:58 smithi016 bash[77995]: cluster 2023-09-23T19:40:56.182829+0000 mgr.y (mgr.25325) 1003 : cluster 0 pgmap v950: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 238 op/s; 1386/72 objects degraded (1925.000%); 0 B/s, 113 objects/s recovering 2023-09-23T19:40:58.399 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:58.430 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:58.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:58.434 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:40:58.462 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:40:58.535 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:58.556 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:58.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:58.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:58.659 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:58.677 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:58.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:58.680 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:58.781 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:58.807 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:40:58.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:58.810 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:40:58.911 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:58.937 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:58.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:58.939 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:40:59.040 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:59.066 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:59.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:59.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:40:59.169 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:59.196 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:40:59.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:59.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:40:59.300 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:59.328 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:40:59.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:59.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:40:59.431 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:59.457 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:40:59.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:59.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:40:59.560 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:59.587 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:40:59.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:59.590 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:40:59.691 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:59.718 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:40:59.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:59.721 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:40:59.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:59 smithi016 bash[75004]: cluster 2023-09-23T19:40:58.183567+0000 mgr.y (mgr.25325) 1004 : cluster 0 pgmap v951: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 86 op/s; 1386/72 objects degraded (1925.000%); 0 B/s, 86 objects/s recovering 2023-09-23T19:40:59.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:59 smithi016 bash[75004]: audit 2023-09-23T19:40:58.662580+0000 mon.a (mon.0) 1279 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:40:59.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:59 smithi016 bash[75004]: audit 2023-09-23T19:40:58.663771+0000 mon.a (mon.0) 1280 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:40:59.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:40:59 smithi016 bash[75004]: cluster 2023-09-23T19:40:59.468870+0000 mon.a (mon.0) 1281 : cluster 3 Health check update: Degraded data redundancy: 1386/72 objects degraded (1925.000%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T19:40:59.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:59 smithi016 bash[77995]: cluster 2023-09-23T19:40:58.183567+0000 mgr.y (mgr.25325) 1004 : cluster 0 pgmap v951: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 86 op/s; 1386/72 objects degraded (1925.000%); 0 B/s, 86 objects/s recovering 2023-09-23T19:40:59.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:59 smithi016 bash[77995]: audit 2023-09-23T19:40:58.662580+0000 mon.a (mon.0) 1279 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:40:59.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:59 smithi016 bash[77995]: audit 2023-09-23T19:40:58.663771+0000 mon.a (mon.0) 1280 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:40:59.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:40:59 smithi016 bash[77995]: cluster 2023-09-23T19:40:59.468870+0000 mon.a (mon.0) 1281 : cluster 3 Health check update: Degraded data redundancy: 1386/72 objects degraded (1925.000%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T19:40:59.822 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:59.846 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:59.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:59.848 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:40:59.949 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:59.975 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:40:59.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:40:59.978 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:41:00.005 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:59 smithi092 bash[58855]: cluster 2023-09-23T19:40:58.183567+0000 mgr.y (mgr.25325) 1004 : cluster 0 pgmap v951: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 86 op/s; 1386/72 objects degraded (1925.000%); 0 B/s, 86 objects/s recovering 2023-09-23T19:41:00.006 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:59 smithi092 bash[58855]: audit 2023-09-23T19:40:58.662580+0000 mon.a (mon.0) 1279 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:41:00.006 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:59 smithi092 bash[58855]: audit 2023-09-23T19:40:58.663771+0000 mon.a (mon.0) 1280 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:41:00.006 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:40:59 smithi092 bash[58855]: cluster 2023-09-23T19:40:59.468870+0000 mon.a (mon.0) 1281 : cluster 3 Health check update: Degraded data redundancy: 1386/72 objects degraded (1925.000%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T19:41:00.078 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:00.106 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:00.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:00.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:00.210 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:00.234 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:00.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:00.236 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:00.337 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:00.363 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:00.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:00.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:41:00.466 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:00.486 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:00.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:00.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:00.589 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:00.611 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:00.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:00.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:41:00.714 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:00.738 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:00.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:00.740 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:00.841 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:00.868 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:00.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:00.870 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:00.971 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:00.994 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:00.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:00.998 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:41:01.098 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:01.128 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:01.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:01.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:01.231 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:01.253 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:01.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:01.256 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:01.357 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:01.372 INFO:teuthology.orchestra.run.smithi092.stdout:{ 2023-09-23T19:41:01.372 INFO:teuthology.orchestra.run.smithi092.stdout: "ops": [], 2023-09-23T19:41:01.372 INFO:teuthology.orchestra.run.smithi092.stdout: "complaint_time": 30, 2023-09-23T19:41:01.372 INFO:teuthology.orchestra.run.smithi092.stdout: "num_blocked_ops": 0 2023-09-23T19:41:01.372 INFO:teuthology.orchestra.run.smithi092.stdout:} 2023-09-23T19:41:01.384 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:01.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:01.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:41:01.488 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:01.515 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:01.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:01.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:41:01.618 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:01.644 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:01.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:01.647 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:01.747 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:01.772 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:01.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:01.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:41:01.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:01 smithi016 bash[75004]: cluster 2023-09-23T19:41:00.184544+0000 mgr.y (mgr.25325) 1005 : cluster 0 pgmap v952: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 5 op/s; 1386/72 objects degraded (1925.000%); 0 B/s, 24 objects/s recovering 2023-09-23T19:41:01.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:01 smithi016 bash[77995]: cluster 2023-09-23T19:41:00.184544+0000 mgr.y (mgr.25325) 1005 : cluster 0 pgmap v952: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 5 op/s; 1386/72 objects degraded (1925.000%); 0 B/s, 24 objects/s recovering 2023-09-23T19:41:01.876 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:01.883 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:01 smithi092 bash[58855]: cluster 2023-09-23T19:41:00.184544+0000 mgr.y (mgr.25325) 1005 : cluster 0 pgmap v952: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 5 op/s; 1386/72 objects degraded (1925.000%); 0 B/s, 24 objects/s recovering 2023-09-23T19:41:01.898 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:01.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:01.900 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:02.001 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:02.028 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:02.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:02.031 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:02.131 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:02.152 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:02.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:02.155 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:02.169 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T19:41:02.256 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:02.284 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:02.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:02.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:41:02.387 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:02.418 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:02.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:02.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:02.522 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:02.548 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:02.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:02.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:41:02.652 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:02.684 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:02.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:02.687 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:02.787 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:02.816 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:02.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:02.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:41:02.920 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:02.946 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:02.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:02.949 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:03.050 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:03.079 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:03.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:03.081 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:41:03.182 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:03.212 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:03.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:03.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:41:03.315 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:03.343 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:03.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:03.345 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:41:03.446 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:03.472 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:03.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:03.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:41:03.576 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:03.602 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:03.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:03.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:03.706 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:03.706 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:03 smithi016 bash[77995]: cluster 2023-09-23T19:41:02.185292+0000 mgr.y (mgr.25325) 1006 : cluster 0 pgmap v953: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 4 op/s; 1386/72 objects degraded (1925.000%); 0 B/s, 21 objects/s recovering 2023-09-23T19:41:03.707 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:03 smithi016 bash[75004]: cluster 2023-09-23T19:41:02.185292+0000 mgr.y (mgr.25325) 1006 : cluster 0 pgmap v953: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 4 op/s; 1386/72 objects degraded (1925.000%); 0 B/s, 21 objects/s recovering 2023-09-23T19:41:03.731 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:03.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:03.733 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:03.834 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:03.859 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:03.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:03.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:41:03.962 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:03.991 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:03.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:03.994 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:41:04.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:03 smithi092 bash[58855]: cluster 2023-09-23T19:41:02.185292+0000 mgr.y (mgr.25325) 1006 : cluster 0 pgmap v953: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 4 op/s; 1386/72 objects degraded (1925.000%); 0 B/s, 21 objects/s recovering 2023-09-23T19:41:04.094 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:04.118 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:04.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:04.121 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:04.221 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:04.251 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:04.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:04.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:04.396 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:04.420 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:04.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:04.422 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:41:04.523 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:04.551 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:04.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:04.553 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:41:04.654 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:04.680 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:04.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:04.682 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:41:04.783 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:04.812 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:04.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:04.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:04.915 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:04.944 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:04.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:04.946 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:05.047 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:05.077 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:05.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:05.080 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:41:05.181 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:05.213 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:05.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:05.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:05.316 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:05.348 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:05.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:05.351 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:05.452 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:05.483 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:05.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:05.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:05.587 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:05.614 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:05.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:05.616 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:05.717 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:05.745 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:05.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:05.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:05.848 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:05.870 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:05.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:05.872 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:41:05.959 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:41:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:41:05] "GET /metrics HTTP/1.1" 200 36301 "" "Prometheus/2.33.4" 2023-09-23T19:41:05.960 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:05 smithi016 bash[77995]: cluster 2023-09-23T19:41:04.185965+0000 mgr.y (mgr.25325) 1007 : cluster 0 pgmap v954: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 1386/72 objects degraded (1925.000%); 0 B/s, 18 objects/s recovering 2023-09-23T19:41:05.960 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:05 smithi016 bash[75004]: cluster 2023-09-23T19:41:04.185965+0000 mgr.y (mgr.25325) 1007 : cluster 0 pgmap v954: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 1386/72 objects degraded (1925.000%); 0 B/s, 18 objects/s recovering 2023-09-23T19:41:05.974 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:05.985 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:05 smithi092 bash[58855]: cluster 2023-09-23T19:41:04.185965+0000 mgr.y (mgr.25325) 1007 : cluster 0 pgmap v954: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 1386/72 objects degraded (1925.000%); 0 B/s, 18 objects/s recovering 2023-09-23T19:41:06.005 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:06.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:06.008 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:06.108 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:06.140 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:06.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:06.142 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:06.243 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:06.274 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:06.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:06.277 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:06.377 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:06.408 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:06.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:06.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:06.512 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:06.531 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:06.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:06.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:41:06.634 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:06.664 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:06.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:06.666 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:06.709 INFO:teuthology.orchestra.run.smithi016.stderr:pool 'unique_pool_5' removed 2023-09-23T19:41:06.767 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:06.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:06 smithi092 bash[58855]: audit 2023-09-23T19:41:06.506134+0000 mon.a (mon.0) 1282 : audit 1 from='client.? 172.21.15.16:0/1539929513' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T19:41:06.793 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:06.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:06.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:41:06.897 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:06.921 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:06.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:06.923 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:06.944 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:41:07.024 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:07.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:06 smithi016 bash[75004]: audit 2023-09-23T19:41:06.506134+0000 mon.a (mon.0) 1282 : audit 1 from='client.? 172.21.15.16:0/1539929513' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T19:41:07.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:06 smithi016 bash[77995]: audit 2023-09-23T19:41:06.506134+0000 mon.a (mon.0) 1282 : audit 1 from='client.? 172.21.15.16:0/1539929513' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T19:41:07.054 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:07.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:07.056 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:41:07.157 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:07.176 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:07.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:07.179 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:41:07.280 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:07.303 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:07.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:07.305 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:07.406 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:07.431 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:07.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:07.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:07.534 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:07.561 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:07.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:07.564 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:07.665 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:07.690 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:07.691 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:07 smithi092 bash[58855]: cluster 2023-09-23T19:41:06.186983+0000 mgr.y (mgr.25325) 1008 : cluster 0 pgmap v955: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 1386/72 objects degraded (1925.000%); 0 B/s, 28 objects/s recovering 2023-09-23T19:41:07.691 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:07 smithi092 bash[58855]: audit 2023-09-23T19:41:06.683484+0000 mon.a (mon.0) 1283 : audit 1 from='client.? 172.21.15.16:0/1539929513' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T19:41:07.692 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:07 smithi092 bash[58855]: cluster 2023-09-23T19:41:06.683601+0000 mon.a (mon.0) 1284 : cluster 0 osdmap e177: 8 total, 8 up, 8 in 2023-09-23T19:41:07.692 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:07 smithi092 bash[58855]: cluster 2023-09-23T19:41:06.852214+0000 osd.7 (osd.7) 85 : cluster 3 failed to encode map e177 with expected crc 2023-09-23T19:41:07.692 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:07 smithi092 bash[58855]: cluster 2023-09-23T19:41:06.855051+0000 osd.7 (osd.7) 86 : cluster 3 failed to encode map e177 with expected crc 2023-09-23T19:41:07.692 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:07 smithi092 bash[58855]: cluster 2023-09-23T19:41:07.083207+0000 osd.6 (osd.6) 98 : cluster 3 failed to encode map e177 with expected crc 2023-09-23T19:41:07.692 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:07 smithi092 bash[58855]: cluster 2023-09-23T19:41:07.085106+0000 osd.6 (osd.6) 99 : cluster 3 failed to encode map e177 with expected crc 2023-09-23T19:41:07.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:07.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:41:07.794 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:07.829 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:07.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:07.832 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:07.932 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:07.960 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:07.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:07.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:08.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:07 smithi016 bash[77995]: cluster 2023-09-23T19:41:06.186983+0000 mgr.y (mgr.25325) 1008 : cluster 0 pgmap v955: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 1386/72 objects degraded (1925.000%); 0 B/s, 28 objects/s recovering 2023-09-23T19:41:08.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:07 smithi016 bash[77995]: audit 2023-09-23T19:41:06.683484+0000 mon.a (mon.0) 1283 : audit 1 from='client.? 172.21.15.16:0/1539929513' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T19:41:08.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:07 smithi016 bash[77995]: cluster 2023-09-23T19:41:06.683601+0000 mon.a (mon.0) 1284 : cluster 0 osdmap e177: 8 total, 8 up, 8 in 2023-09-23T19:41:08.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:07 smithi016 bash[77995]: cluster 2023-09-23T19:41:06.852214+0000 osd.7 (osd.7) 85 : cluster 3 failed to encode map e177 with expected crc 2023-09-23T19:41:08.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:07 smithi016 bash[77995]: cluster 2023-09-23T19:41:06.855051+0000 osd.7 (osd.7) 86 : cluster 3 failed to encode map e177 with expected crc 2023-09-23T19:41:08.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:07 smithi016 bash[77995]: cluster 2023-09-23T19:41:07.083207+0000 osd.6 (osd.6) 98 : cluster 3 failed to encode map e177 with expected crc 2023-09-23T19:41:08.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:07 smithi016 bash[77995]: cluster 2023-09-23T19:41:07.085106+0000 osd.6 (osd.6) 99 : cluster 3 failed to encode map e177 with expected crc 2023-09-23T19:41:08.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:07 smithi016 bash[75004]: cluster 2023-09-23T19:41:06.186983+0000 mgr.y (mgr.25325) 1008 : cluster 0 pgmap v955: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 1386/72 objects degraded (1925.000%); 0 B/s, 28 objects/s recovering 2023-09-23T19:41:08.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:07 smithi016 bash[75004]: audit 2023-09-23T19:41:06.683484+0000 mon.a (mon.0) 1283 : audit 1 from='client.? 172.21.15.16:0/1539929513' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T19:41:08.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:07 smithi016 bash[75004]: cluster 2023-09-23T19:41:06.683601+0000 mon.a (mon.0) 1284 : cluster 0 osdmap e177: 8 total, 8 up, 8 in 2023-09-23T19:41:08.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:07 smithi016 bash[75004]: cluster 2023-09-23T19:41:06.852214+0000 osd.7 (osd.7) 85 : cluster 3 failed to encode map e177 with expected crc 2023-09-23T19:41:08.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:07 smithi016 bash[75004]: cluster 2023-09-23T19:41:06.855051+0000 osd.7 (osd.7) 86 : cluster 3 failed to encode map e177 with expected crc 2023-09-23T19:41:08.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:07 smithi016 bash[75004]: cluster 2023-09-23T19:41:07.083207+0000 osd.6 (osd.6) 98 : cluster 3 failed to encode map e177 with expected crc 2023-09-23T19:41:08.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:07 smithi016 bash[75004]: cluster 2023-09-23T19:41:07.085106+0000 osd.6 (osd.6) 99 : cluster 3 failed to encode map e177 with expected crc 2023-09-23T19:41:08.064 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:08.098 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:08.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:08.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:08.201 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:08.227 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:08.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:08.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:08.331 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:08.355 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:08.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:08.358 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:08.459 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:08.483 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:08.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:08.485 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:08.586 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:08.611 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:08.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:08.614 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:08.628 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:41:08.648 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:41:08.669 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:41:08.692 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:41:08.700 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T19:41:08.715 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:08.724 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:41:08.740 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:08.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:08.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:41:08.752 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:41:08.782 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:41:08.808 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:41:08.810 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:41:08.810 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:41:08.810 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:41:08.810 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:41:08.810 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:41:08.810 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:41:08.810 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:41:08.810 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:41:08.844 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:08.868 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:08.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:08.871 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:41:08.971 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:08.978 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:08 smithi016 bash[75004]: cluster 2023-09-23T19:41:07.498769+0000 osd.4 (osd.4) 114 : cluster 3 failed to encode map e177 with expected crc 2023-09-23T19:41:08.979 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:08 smithi016 bash[75004]: audit 2023-09-23T19:41:08.188561+0000 mon.a (mon.0) 1285 : audit 1 from='client.? 172.21.15.16:0/3046383556' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T19:41:08.979 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:08 smithi016 bash[75004]: cluster 2023-09-23T19:41:08.244459+0000 osd.5 (osd.5) 108 : cluster 3 failed to encode map e177 with expected crc 2023-09-23T19:41:08.979 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:08 smithi016 bash[75004]: audit 2023-09-23T19:41:08.599959+0000 mon.a (mon.0) 1286 : audit 0 from='client.? 172.21.15.16:0/2450955318' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:41:08.979 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:08 smithi016 bash[77995]: cluster 2023-09-23T19:41:07.498769+0000 osd.4 (osd.4) 114 : cluster 3 failed to encode map e177 with expected crc 2023-09-23T19:41:08.979 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:08 smithi016 bash[77995]: audit 2023-09-23T19:41:08.188561+0000 mon.a (mon.0) 1285 : audit 1 from='client.? 172.21.15.16:0/3046383556' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T19:41:08.979 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:08 smithi016 bash[77995]: cluster 2023-09-23T19:41:08.244459+0000 osd.5 (osd.5) 108 : cluster 3 failed to encode map e177 with expected crc 2023-09-23T19:41:08.979 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:08 smithi016 bash[77995]: audit 2023-09-23T19:41:08.599959+0000 mon.a (mon.0) 1286 : audit 0 from='client.? 172.21.15.16:0/2450955318' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:41:08.980 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T19:41:08.981 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T19:41:08.981 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_6 2023-09-23T19:41:08.981 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_6 16 2023-09-23T19:41:08.996 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:08.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:08.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:09.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:08 smithi092 bash[58855]: cluster 2023-09-23T19:41:07.498769+0000 osd.4 (osd.4) 114 : cluster 3 failed to encode map e177 with expected crc 2023-09-23T19:41:09.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:08 smithi092 bash[58855]: audit 2023-09-23T19:41:08.188561+0000 mon.a (mon.0) 1285 : audit 1 from='client.? 172.21.15.16:0/3046383556' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T19:41:09.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:08 smithi092 bash[58855]: cluster 2023-09-23T19:41:08.244459+0000 osd.5 (osd.5) 108 : cluster 3 failed to encode map e177 with expected crc 2023-09-23T19:41:09.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:08 smithi092 bash[58855]: audit 2023-09-23T19:41:08.599959+0000 mon.a (mon.0) 1286 : audit 0 from='client.? 172.21.15.16:0/2450955318' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:41:09.100 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:09.123 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:09.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:09.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:41:09.226 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:09.256 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:09.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:09.259 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:41:09.360 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:09.392 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:09.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:09.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:09.497 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:09.522 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:09.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:09.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:09.625 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:09.653 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:09.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:09.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:09.757 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:09.783 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:09.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:09.785 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:09.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:09 smithi092 bash[58855]: cluster 2023-09-23T19:41:08.187421+0000 mgr.y (mgr.25325) 1009 : cluster 0 pgmap v957: 73 pgs: 73 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2023-09-23T19:41:09.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:09 smithi092 bash[58855]: cluster 2023-09-23T19:41:08.682292+0000 mon.a (mon.0) 1287 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1386/72 objects degraded (1925.000%), 1 pg degraded, 2 pgs undersized) 2023-09-23T19:41:09.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:09 smithi092 bash[58855]: audit 2023-09-23T19:41:08.691726+0000 mon.a (mon.0) 1288 : audit 1 from='client.? 172.21.15.16:0/3046383556' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T19:41:09.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:09 smithi092 bash[58855]: cluster 2023-09-23T19:41:08.691768+0000 mon.a (mon.0) 1289 : cluster 0 osdmap e178: 8 total, 8 up, 8 in 2023-09-23T19:41:09.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:09 smithi092 bash[58855]: audit 2023-09-23T19:41:09.586531+0000 mon.a (mon.0) 1290 : audit 1 from='client.? 172.21.15.16:0/2589070423' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:41:09.887 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:09.921 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:09.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:09.924 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:10.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:09 smithi016 bash[75004]: cluster 2023-09-23T19:41:08.187421+0000 mgr.y (mgr.25325) 1009 : cluster 0 pgmap v957: 73 pgs: 73 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2023-09-23T19:41:10.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:09 smithi016 bash[75004]: cluster 2023-09-23T19:41:08.682292+0000 mon.a (mon.0) 1287 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1386/72 objects degraded (1925.000%), 1 pg degraded, 2 pgs undersized) 2023-09-23T19:41:10.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:09 smithi016 bash[75004]: audit 2023-09-23T19:41:08.691726+0000 mon.a (mon.0) 1288 : audit 1 from='client.? 172.21.15.16:0/3046383556' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T19:41:10.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:09 smithi016 bash[75004]: cluster 2023-09-23T19:41:08.691768+0000 mon.a (mon.0) 1289 : cluster 0 osdmap e178: 8 total, 8 up, 8 in 2023-09-23T19:41:10.000 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:09 smithi016 bash[75004]: audit 2023-09-23T19:41:09.586531+0000 mon.a (mon.0) 1290 : audit 1 from='client.? 172.21.15.16:0/2589070423' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:41:10.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:09 smithi016 bash[77995]: cluster 2023-09-23T19:41:08.187421+0000 mgr.y (mgr.25325) 1009 : cluster 0 pgmap v957: 73 pgs: 73 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2023-09-23T19:41:10.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:09 smithi016 bash[77995]: cluster 2023-09-23T19:41:08.682292+0000 mon.a (mon.0) 1287 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1386/72 objects degraded (1925.000%), 1 pg degraded, 2 pgs undersized) 2023-09-23T19:41:10.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:09 smithi016 bash[77995]: audit 2023-09-23T19:41:08.691726+0000 mon.a (mon.0) 1288 : audit 1 from='client.? 172.21.15.16:0/3046383556' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T19:41:10.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:09 smithi016 bash[77995]: cluster 2023-09-23T19:41:08.691768+0000 mon.a (mon.0) 1289 : cluster 0 osdmap e178: 8 total, 8 up, 8 in 2023-09-23T19:41:10.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:09 smithi016 bash[77995]: audit 2023-09-23T19:41:09.586531+0000 mon.a (mon.0) 1290 : audit 1 from='client.? 172.21.15.16:0/2589070423' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:41:10.026 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:10.054 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:10.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:10.056 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:10.157 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:10.186 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:10.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:10.190 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:41:10.290 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:10.321 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:10.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:10.324 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:10.425 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:10.457 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:10.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:10.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:41:10.490 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is set 2023-09-23T19:41:10.521 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T19:41:10.560 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:10.586 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:10.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:10.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:10.689 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:10.717 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:10.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:10.721 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:10.819 INFO:teuthology.orchestra.run.smithi092.stdout:ERROR: (22) Invalid argument 2023-09-23T19:41:10.819 INFO:teuthology.orchestra.run.smithi092.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:41:10.822 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:10.835 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:10 smithi092 bash[58855]: cluster 2023-09-23T19:41:09.752467+0000 osd.7 (osd.7) 87 : cluster 3 failed to encode map e178 with expected crc 2023-09-23T19:41:10.835 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:10 smithi092 bash[58855]: cluster 2023-09-23T19:41:09.753580+0000 osd.7 (osd.7) 88 : cluster 3 failed to encode map e178 with expected crc 2023-09-23T19:41:10.835 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:10 smithi092 bash[58855]: cluster 2023-09-23T19:41:09.859610+0000 osd.5 (osd.5) 109 : cluster 3 failed to encode map e178 with expected crc 2023-09-23T19:41:10.835 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:10 smithi092 bash[58855]: cluster 2023-09-23T19:41:09.859615+0000 osd.6 (osd.6) 100 : cluster 3 failed to encode map e178 with expected crc 2023-09-23T19:41:10.835 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:10 smithi092 bash[58855]: cluster 2023-09-23T19:41:09.859929+0000 osd.4 (osd.4) 115 : cluster 3 failed to encode map e178 with expected crc 2023-09-23T19:41:10.836 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:10 smithi092 bash[58855]: cluster 2023-09-23T19:41:10.473424+0000 mon.a (mon.0) 1291 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:41:10.836 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:10 smithi092 bash[58855]: audit 2023-09-23T19:41:10.482132+0000 mon.a (mon.0) 1292 : audit 1 from='client.? 172.21.15.16:0/2589070423' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:41:10.836 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:10 smithi092 bash[58855]: cluster 2023-09-23T19:41:10.482230+0000 mon.a (mon.0) 1293 : cluster 0 osdmap e179: 8 total, 8 up, 8 in 2023-09-23T19:41:10.836 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:10 smithi092 bash[58855]: cluster 2023-09-23T19:41:10.503328+0000 osd.7 (osd.7) 89 : cluster 3 failed to encode map e179 with expected crc 2023-09-23T19:41:10.852 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:10.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:10.855 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:10.955 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:10.986 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:10.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:10.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:11.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:10 smithi016 bash[75004]: cluster 2023-09-23T19:41:09.752467+0000 osd.7 (osd.7) 87 : cluster 3 failed to encode map e178 with expected crc 2023-09-23T19:41:11.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:10 smithi016 bash[75004]: cluster 2023-09-23T19:41:09.753580+0000 osd.7 (osd.7) 88 : cluster 3 failed to encode map e178 with expected crc 2023-09-23T19:41:11.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:10 smithi016 bash[75004]: cluster 2023-09-23T19:41:09.859610+0000 osd.5 (osd.5) 109 : cluster 3 failed to encode map e178 with expected crc 2023-09-23T19:41:11.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:10 smithi016 bash[75004]: cluster 2023-09-23T19:41:09.859615+0000 osd.6 (osd.6) 100 : cluster 3 failed to encode map e178 with expected crc 2023-09-23T19:41:11.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:10 smithi016 bash[75004]: cluster 2023-09-23T19:41:09.859929+0000 osd.4 (osd.4) 115 : cluster 3 failed to encode map e178 with expected crc 2023-09-23T19:41:11.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:10 smithi016 bash[75004]: cluster 2023-09-23T19:41:10.473424+0000 mon.a (mon.0) 1291 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:41:11.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:10 smithi016 bash[75004]: audit 2023-09-23T19:41:10.482132+0000 mon.a (mon.0) 1292 : audit 1 from='client.? 172.21.15.16:0/2589070423' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:41:11.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:10 smithi016 bash[75004]: cluster 2023-09-23T19:41:10.482230+0000 mon.a (mon.0) 1293 : cluster 0 osdmap e179: 8 total, 8 up, 8 in 2023-09-23T19:41:11.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:10 smithi016 bash[75004]: cluster 2023-09-23T19:41:10.503328+0000 osd.7 (osd.7) 89 : cluster 3 failed to encode map e179 with expected crc 2023-09-23T19:41:11.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:10 smithi016 bash[77995]: cluster 2023-09-23T19:41:09.752467+0000 osd.7 (osd.7) 87 : cluster 3 failed to encode map e178 with expected crc 2023-09-23T19:41:11.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:10 smithi016 bash[77995]: cluster 2023-09-23T19:41:09.753580+0000 osd.7 (osd.7) 88 : cluster 3 failed to encode map e178 with expected crc 2023-09-23T19:41:11.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:10 smithi016 bash[77995]: cluster 2023-09-23T19:41:09.859610+0000 osd.5 (osd.5) 109 : cluster 3 failed to encode map e178 with expected crc 2023-09-23T19:41:11.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:10 smithi016 bash[77995]: cluster 2023-09-23T19:41:09.859615+0000 osd.6 (osd.6) 100 : cluster 3 failed to encode map e178 with expected crc 2023-09-23T19:41:11.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:10 smithi016 bash[77995]: cluster 2023-09-23T19:41:09.859929+0000 osd.4 (osd.4) 115 : cluster 3 failed to encode map e178 with expected crc 2023-09-23T19:41:11.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:10 smithi016 bash[77995]: cluster 2023-09-23T19:41:10.473424+0000 mon.a (mon.0) 1291 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:41:11.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:10 smithi016 bash[77995]: audit 2023-09-23T19:41:10.482132+0000 mon.a (mon.0) 1292 : audit 1 from='client.? 172.21.15.16:0/2589070423' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:41:11.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:10 smithi016 bash[77995]: cluster 2023-09-23T19:41:10.482230+0000 mon.a (mon.0) 1293 : cluster 0 osdmap e179: 8 total, 8 up, 8 in 2023-09-23T19:41:11.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:10 smithi016 bash[77995]: cluster 2023-09-23T19:41:10.503328+0000 osd.7 (osd.7) 89 : cluster 3 failed to encode map e179 with expected crc 2023-09-23T19:41:11.090 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:11.115 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:11.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:11.118 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:11.218 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:11.246 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:11.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:11.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:41:11.350 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:11.380 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:11.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:11.382 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:11.483 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:11.517 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:11.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:11.519 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:41:11.620 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:11.653 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:11.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:11.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:11.757 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:11.784 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T19:41:11.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:11 smithi016 bash[75004]: cluster 2023-09-23T19:41:10.188051+0000 mgr.y (mgr.25325) 1010 : cluster 0 pgmap v959: 73 pgs: 73 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2023-09-23T19:41:11.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:11 smithi016 bash[75004]: cluster 2023-09-23T19:41:10.503429+0000 osd.4 (osd.4) 116 : cluster 3 failed to encode map e179 with expected crc 2023-09-23T19:41:11.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:11 smithi016 bash[75004]: cluster 2023-09-23T19:41:10.503438+0000 osd.5 (osd.5) 110 : cluster 3 failed to encode map e179 with expected crc 2023-09-23T19:41:11.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:11 smithi016 bash[75004]: cluster 2023-09-23T19:41:10.503441+0000 osd.6 (osd.6) 101 : cluster 3 failed to encode map e179 with expected crc 2023-09-23T19:41:11.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:11 smithi016 bash[77995]: cluster 2023-09-23T19:41:10.188051+0000 mgr.y (mgr.25325) 1010 : cluster 0 pgmap v959: 73 pgs: 73 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2023-09-23T19:41:11.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:11 smithi016 bash[77995]: cluster 2023-09-23T19:41:10.503429+0000 osd.4 (osd.4) 116 : cluster 3 failed to encode map e179 with expected crc 2023-09-23T19:41:11.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:11 smithi016 bash[77995]: cluster 2023-09-23T19:41:10.503438+0000 osd.5 (osd.5) 110 : cluster 3 failed to encode map e179 with expected crc 2023-09-23T19:41:11.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:11 smithi016 bash[77995]: cluster 2023-09-23T19:41:10.503441+0000 osd.6 (osd.6) 101 : cluster 3 failed to encode map e179 with expected crc 2023-09-23T19:41:11.787 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:11.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:11.789 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:11.890 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:11.919 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:11.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:11.922 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:41:12.023 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:12.034 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:11 smithi092 bash[58855]: cluster 2023-09-23T19:41:10.188051+0000 mgr.y (mgr.25325) 1010 : cluster 0 pgmap v959: 73 pgs: 73 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2023-09-23T19:41:12.034 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:11 smithi092 bash[58855]: cluster 2023-09-23T19:41:10.503429+0000 osd.4 (osd.4) 116 : cluster 3 failed to encode map e179 with expected crc 2023-09-23T19:41:12.034 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:11 smithi092 bash[58855]: cluster 2023-09-23T19:41:10.503438+0000 osd.5 (osd.5) 110 : cluster 3 failed to encode map e179 with expected crc 2023-09-23T19:41:12.034 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:11 smithi092 bash[58855]: cluster 2023-09-23T19:41:10.503441+0000 osd.6 (osd.6) 101 : cluster 3 failed to encode map e179 with expected crc 2023-09-23T19:41:12.050 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:12.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:12.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:12.154 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:12.179 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:12.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:12.181 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:12.282 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:12.312 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:12.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:12.315 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:41:12.415 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:12.446 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:12.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:12.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:12.527 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:41:12.549 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:12.579 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:12.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:12.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:12.684 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:12.717 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:12.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:12.719 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:41:12.820 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:12.831 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:12 smithi016 bash[75004]: audit 2023-09-23T19:41:11.837986+0000 mon.a (mon.0) 1294 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:41:12.832 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:12 smithi016 bash[77995]: audit 2023-09-23T19:41:11.837986+0000 mon.a (mon.0) 1294 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:41:12.854 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:12.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:12.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:41:12.958 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:12.973 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:12 smithi092 bash[58855]: audit 2023-09-23T19:41:11.837986+0000 mon.a (mon.0) 1294 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:41:12.990 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:12.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:12.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:13.094 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:13.129 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:13.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:13.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:13.233 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:13.267 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:13.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:13.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:13.371 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:13.403 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:13.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:13.406 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:13.467 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T19:41:13.506 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:13.545 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:13.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:13.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:41:13.649 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:13.678 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:13.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:13.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:13.781 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:13.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:13 smithi016 bash[75004]: cluster 2023-09-23T19:41:12.188891+0000 mgr.y (mgr.25325) 1011 : cluster 0 pgmap v961: 73 pgs: 73 active+clean; 582 KiB data, 4.7 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:41:13.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:13 smithi016 bash[75004]: audit 2023-09-23T19:41:13.662773+0000 mon.a (mon.0) 1295 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:41:13.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:13 smithi016 bash[75004]: audit 2023-09-23T19:41:13.663930+0000 mon.a (mon.0) 1296 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:41:13.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:13 smithi016 bash[77995]: cluster 2023-09-23T19:41:12.188891+0000 mgr.y (mgr.25325) 1011 : cluster 0 pgmap v961: 73 pgs: 73 active+clean; 582 KiB data, 4.7 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:41:13.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:13 smithi016 bash[77995]: audit 2023-09-23T19:41:13.662773+0000 mon.a (mon.0) 1295 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:41:13.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:13 smithi016 bash[77995]: audit 2023-09-23T19:41:13.663930+0000 mon.a (mon.0) 1296 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:41:13.813 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:13.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:13.816 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:41:13.917 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:13.950 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:13.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:13.953 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:13.991 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:41:14.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:13 smithi092 bash[58855]: cluster 2023-09-23T19:41:12.188891+0000 mgr.y (mgr.25325) 1011 : cluster 0 pgmap v961: 73 pgs: 73 active+clean; 582 KiB data, 4.7 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:41:14.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:13 smithi092 bash[58855]: audit 2023-09-23T19:41:13.662773+0000 mon.a (mon.0) 1295 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:41:14.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:13 smithi092 bash[58855]: audit 2023-09-23T19:41:13.663930+0000 mon.a (mon.0) 1296 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:41:14.054 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:14.080 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:14.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:14.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:41:14.184 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:14.212 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:14.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:14.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:14.316 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:14.350 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:14.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:14.353 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:14.454 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:14.484 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:14.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:14.486 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:14.587 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:14.617 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:14.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:14.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:14.721 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:14.753 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:14.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:14.757 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:14.857 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:14.889 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:14.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:14.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:14.992 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:15.007 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:41:15.029 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:15.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:15.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:41:15.132 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:15.145 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:15 smithi016 bash[75004]: audit 2023-09-23T19:41:14.107085+0000 mon.a (mon.0) 1297 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:41:15.145 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:15 smithi016 bash[75004]: cluster 2023-09-23T19:41:14.189638+0000 mgr.y (mgr.25325) 1012 : cluster 0 pgmap v962: 73 pgs: 73 active+clean; 582 KiB data, 4.7 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:41:15.145 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:15 smithi016 bash[77995]: audit 2023-09-23T19:41:14.107085+0000 mon.a (mon.0) 1297 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:41:15.145 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:15 smithi016 bash[77995]: cluster 2023-09-23T19:41:14.189638+0000 mgr.y (mgr.25325) 1012 : cluster 0 pgmap v962: 73 pgs: 73 active+clean; 582 KiB data, 4.7 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:41:15.160 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:15.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:15.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:41:15.264 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:15.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:15 smithi092 bash[58855]: audit 2023-09-23T19:41:14.107085+0000 mon.a (mon.0) 1297 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:41:15.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:15 smithi092 bash[58855]: cluster 2023-09-23T19:41:14.189638+0000 mgr.y (mgr.25325) 1012 : cluster 0 pgmap v962: 73 pgs: 73 active+clean; 582 KiB data, 4.7 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:41:15.290 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:15.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:15.293 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:15.394 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:15.425 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:15.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:15.429 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:41:15.530 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:15.562 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:15.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:15.565 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:15.666 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:15.694 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:41:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:41:15] "GET /metrics HTTP/1.1" 200 33899 "" "Prometheus/2.33.4" 2023-09-23T19:41:15.702 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:15.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:15.705 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:15.806 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:15.831 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:15.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:15.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:41:15.936 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:15.967 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:15.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:15.971 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:16.071 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:16.102 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:16.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:16.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:16.206 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:16.232 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:16.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:16.235 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:41:16.336 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:16.367 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:16.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:16.370 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:16.470 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:16.492 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:16.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:16.495 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:16.558 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:41:16.596 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:16.618 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:16.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:16.621 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:16.722 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:16.748 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:16.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:16.751 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:16.852 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:16.876 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:16.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:16.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:16.979 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:17.003 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:17.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:17.006 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:41:17.107 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:17.131 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:17.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:17.133 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:41:17.234 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:17.257 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:17.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:17.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:17.360 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:17.384 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:17.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:17.387 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:17.388 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:17 smithi016 bash[77995]: cluster 2023-09-23T19:41:16.190477+0000 mgr.y (mgr.25325) 1013 : cluster 0 pgmap v963: 73 pgs: 73 active+clean; 582 KiB data, 4.7 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:41:17.388 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:17 smithi016 bash[77995]: audit 2023-09-23T19:41:17.179510+0000 mon.a (mon.0) 1298 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:41:17.388 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:17 smithi016 bash[77995]: audit 2023-09-23T19:41:17.185870+0000 mon.a (mon.0) 1299 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:41:17.389 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:17 smithi016 bash[75004]: cluster 2023-09-23T19:41:16.190477+0000 mgr.y (mgr.25325) 1013 : cluster 0 pgmap v963: 73 pgs: 73 active+clean; 582 KiB data, 4.7 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:41:17.390 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:17 smithi016 bash[75004]: audit 2023-09-23T19:41:17.179510+0000 mon.a (mon.0) 1298 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:41:17.390 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:17 smithi016 bash[75004]: audit 2023-09-23T19:41:17.185870+0000 mon.a (mon.0) 1299 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:41:17.488 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:17.507 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:17.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:17.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:17.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:17 smithi092 bash[58855]: cluster 2023-09-23T19:41:16.190477+0000 mgr.y (mgr.25325) 1013 : cluster 0 pgmap v963: 73 pgs: 73 active+clean; 582 KiB data, 4.7 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:41:17.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:17 smithi092 bash[58855]: audit 2023-09-23T19:41:17.179510+0000 mon.a (mon.0) 1298 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:41:17.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:17 smithi092 bash[58855]: audit 2023-09-23T19:41:17.185870+0000 mon.a (mon.0) 1299 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:41:17.610 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:17.642 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:17.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:17.645 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:17.745 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:17.775 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:17.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:17.778 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:41:17.840 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:41:17.878 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:17.902 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:17.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:17.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:41:18.006 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:18.035 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:18.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:18.038 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:18.048 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:41:18.139 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:18.163 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:18.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:18.166 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:18.266 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:18.292 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:18.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:18.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:18.396 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:18.422 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:18.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:18.424 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:18.524 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:18.556 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:18.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:18.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:18.658 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:18.688 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:18.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:18.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:41:18.792 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:18.800 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:18 smithi016 bash[75004]: audit 2023-09-23T19:41:17.785694+0000 mon.a (mon.0) 1300 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:41:18.800 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:18 smithi016 bash[75004]: audit 2023-09-23T19:41:17.792376+0000 mon.a (mon.0) 1301 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:41:18.800 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:18 smithi016 bash[75004]: audit 2023-09-23T19:41:18.264124+0000 mon.a (mon.0) 1302 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:41:18.800 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:18 smithi016 bash[75004]: audit 2023-09-23T19:41:18.265017+0000 mon.a (mon.0) 1303 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:41:18.801 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:18 smithi016 bash[75004]: audit 2023-09-23T19:41:18.271916+0000 mon.a (mon.0) 1304 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:41:18.801 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:18 smithi016 bash[77995]: audit 2023-09-23T19:41:17.785694+0000 mon.a (mon.0) 1300 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:41:18.801 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:18 smithi016 bash[77995]: audit 2023-09-23T19:41:17.792376+0000 mon.a (mon.0) 1301 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:41:18.801 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:18 smithi016 bash[77995]: audit 2023-09-23T19:41:18.264124+0000 mon.a (mon.0) 1302 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:41:18.801 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:18 smithi016 bash[77995]: audit 2023-09-23T19:41:18.265017+0000 mon.a (mon.0) 1303 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:41:18.801 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:18 smithi016 bash[77995]: audit 2023-09-23T19:41:18.271916+0000 mon.a (mon.0) 1304 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:41:18.813 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:18.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:18.816 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:41:18.917 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:18.942 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:18.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:18.945 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:41:19.045 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:19.058 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:18 smithi092 bash[58855]: audit 2023-09-23T19:41:17.785694+0000 mon.a (mon.0) 1300 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:41:19.059 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:18 smithi092 bash[58855]: audit 2023-09-23T19:41:17.792376+0000 mon.a (mon.0) 1301 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:41:19.059 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:18 smithi092 bash[58855]: audit 2023-09-23T19:41:18.264124+0000 mon.a (mon.0) 1302 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:41:19.059 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:18 smithi092 bash[58855]: audit 2023-09-23T19:41:18.265017+0000 mon.a (mon.0) 1303 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:41:19.059 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:18 smithi092 bash[58855]: audit 2023-09-23T19:41:18.271916+0000 mon.a (mon.0) 1304 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:41:19.072 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:19.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:19.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:19.175 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:19.201 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:19.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:19.204 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:41:19.305 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:19.332 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:19.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:19.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:19.435 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:19.467 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:19.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:19.470 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:19.571 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:19.603 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:19.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:19.606 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:19.707 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:19.737 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:19.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:19.739 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:41:19.840 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:19.848 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:19 smithi016 bash[75004]: cluster 2023-09-23T19:41:18.191064+0000 mgr.y (mgr.25325) 1014 : cluster 0 pgmap v964: 73 pgs: 73 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2023-09-23T19:41:19.848 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:19 smithi016 bash[77995]: cluster 2023-09-23T19:41:18.191064+0000 mgr.y (mgr.25325) 1014 : cluster 0 pgmap v964: 73 pgs: 73 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2023-09-23T19:41:19.862 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:19.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:19.865 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:19.965 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:19.976 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:19 smithi092 bash[58855]: cluster 2023-09-23T19:41:18.191064+0000 mgr.y (mgr.25325) 1014 : cluster 0 pgmap v964: 73 pgs: 73 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2023-09-23T19:41:19.994 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:19.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:19.997 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:20.098 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:20.131 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:20.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:20.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:20.234 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:20.260 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:20.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:20.263 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:20.364 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:20.377 INFO:teuthology.orchestra.run.smithi092.stdout:ERROR: (22) Invalid argument 2023-09-23T19:41:20.377 INFO:teuthology.orchestra.run.smithi092.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:41:20.388 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:20.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:20.390 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:20.491 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:20.516 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:20.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:20.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:20.620 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:20.648 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:20.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:20.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:41:20.751 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:20.778 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:20.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:20.781 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:20.881 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:20.909 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:20.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:20.911 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:21.012 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:21.037 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:21.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:21.039 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:21.140 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:21.166 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:21.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:21.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:21.269 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:21.273 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T19:41:21.297 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:21.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:21.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:21.401 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:21.432 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:21.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:21.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:41:21.535 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:21.565 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:21.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:21.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:21.669 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:21.697 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:21.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:21.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:41:21.800 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:21.827 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:21.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:21.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:41:21.930 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:21.963 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:21.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:21.967 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:41:22.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:21 smithi016 bash[77995]: cluster 2023-09-23T19:41:20.191658+0000 mgr.y (mgr.25325) 1015 : cluster 0 pgmap v965: 73 pgs: 73 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2023-09-23T19:41:22.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:21 smithi016 bash[75004]: cluster 2023-09-23T19:41:20.191658+0000 mgr.y (mgr.25325) 1015 : cluster 0 pgmap v965: 73 pgs: 73 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2023-09-23T19:41:22.067 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:22.081 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:21 smithi092 bash[58855]: cluster 2023-09-23T19:41:20.191658+0000 mgr.y (mgr.25325) 1015 : cluster 0 pgmap v965: 73 pgs: 73 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2023-09-23T19:41:22.103 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:22.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:22.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:22.207 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:22.239 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:22.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:22.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:22.343 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:22.377 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:22.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:22.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:41:22.480 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:22.509 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:22.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:22.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:22.613 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:22.636 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:22.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:22.640 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:41:22.740 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:22.773 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:22.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:22.776 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:22.876 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:22.909 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:22.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:22.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:23.012 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:23.042 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:23.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:23.045 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:23.145 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:23.170 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:23.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:23.172 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:41:23.273 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:23.303 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:23.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:23.306 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:23.406 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:23.428 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:23.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:23.431 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:41:23.532 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:23.559 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:23.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:23.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:41:23.663 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:23.692 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:23.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:23.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:23.797 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:23.825 INFO:teuthology.orchestra.run.smithi016.stderr:pool 'unique_pool_6' created 2023-09-23T19:41:23.826 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T19:41:23.830 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:23.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:23.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:41:23.933 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:23.964 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:23.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:23.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:24.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:23 smithi092 bash[58855]: cluster 2023-09-23T19:41:22.192214+0000 mgr.y (mgr.25325) 1016 : cluster 0 pgmap v966: 73 pgs: 73 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2023-09-23T19:41:24.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:23 smithi092 bash[58855]: audit 2023-09-23T19:41:22.849468+0000 mon.c (mon.2) 177 : audit 1 from='client.? 172.21.15.16:0/664254450' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]: dispatch 2023-09-23T19:41:24.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:23 smithi092 bash[58855]: audit 2023-09-23T19:41:22.857220+0000 mon.a (mon.0) 1305 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]: dispatch 2023-09-23T19:41:24.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:23 smithi092 bash[58855]: audit 2023-09-23T19:41:23.299372+0000 mon.a (mon.0) 1306 : audit 1 from='client.? 172.21.15.16:0/2448525257' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T19:41:24.069 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:24.081 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:23 smithi016 bash[77995]: cluster 2023-09-23T19:41:22.192214+0000 mgr.y (mgr.25325) 1016 : cluster 0 pgmap v966: 73 pgs: 73 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2023-09-23T19:41:24.081 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:23 smithi016 bash[77995]: audit 2023-09-23T19:41:22.849468+0000 mon.c (mon.2) 177 : audit 1 from='client.? 172.21.15.16:0/664254450' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]: dispatch 2023-09-23T19:41:24.081 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:23 smithi016 bash[77995]: audit 2023-09-23T19:41:22.857220+0000 mon.a (mon.0) 1305 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]: dispatch 2023-09-23T19:41:24.081 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:23 smithi016 bash[77995]: audit 2023-09-23T19:41:23.299372+0000 mon.a (mon.0) 1306 : audit 1 from='client.? 172.21.15.16:0/2448525257' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T19:41:24.081 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:23 smithi016 bash[75004]: cluster 2023-09-23T19:41:22.192214+0000 mgr.y (mgr.25325) 1016 : cluster 0 pgmap v966: 73 pgs: 73 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2023-09-23T19:41:24.081 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:23 smithi016 bash[75004]: audit 2023-09-23T19:41:22.849468+0000 mon.c (mon.2) 177 : audit 1 from='client.? 172.21.15.16:0/664254450' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]: dispatch 2023-09-23T19:41:24.081 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:23 smithi016 bash[75004]: audit 2023-09-23T19:41:22.857220+0000 mon.a (mon.0) 1305 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]: dispatch 2023-09-23T19:41:24.081 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:23 smithi016 bash[75004]: audit 2023-09-23T19:41:23.299372+0000 mon.a (mon.0) 1306 : audit 1 from='client.? 172.21.15.16:0/2448525257' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T19:41:24.096 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:24.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:24.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:41:24.199 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:24.227 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:24.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:24.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:24.330 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:24.351 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:24.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:24.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:41:24.455 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:24.486 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:24.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:24.489 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:24.590 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:24.623 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:24.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:24.626 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:24.727 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:24.756 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:24.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:24.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:41:24.860 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:24.885 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:24.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:24.888 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:41:24.989 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:25.001 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:24 smithi092 bash[58855]: audit 2023-09-23T19:41:23.813201+0000 mon.a (mon.0) 1307 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]': finished 2023-09-23T19:41:25.001 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:24 smithi092 bash[58855]: audit 2023-09-23T19:41:23.813323+0000 mon.a (mon.0) 1308 : audit 1 from='client.? 172.21.15.16:0/2448525257' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T19:41:25.001 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:24 smithi092 bash[58855]: cluster 2023-09-23T19:41:23.813412+0000 mon.a (mon.0) 1309 : cluster 0 osdmap e180: 8 total, 8 up, 8 in 2023-09-23T19:41:25.001 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:24 smithi092 bash[58855]: cluster 2023-09-23T19:41:23.818067+0000 osd.4 (osd.4) 117 : cluster 3 failed to encode map e180 with expected crc 2023-09-23T19:41:25.001 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:24 smithi092 bash[58855]: cluster 2023-09-23T19:41:23.818115+0000 osd.7 (osd.7) 90 : cluster 3 failed to encode map e180 with expected crc 2023-09-23T19:41:25.001 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:24 smithi092 bash[58855]: cluster 2023-09-23T19:41:23.823147+0000 osd.4 (osd.4) 118 : cluster 3 failed to encode map e180 with expected crc 2023-09-23T19:41:25.002 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:24 smithi092 bash[58855]: cluster 2023-09-23T19:41:23.823290+0000 osd.6 (osd.6) 102 : cluster 3 failed to encode map e180 with expected crc 2023-09-23T19:41:25.020 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:25.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:25.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:25.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:24 smithi016 bash[75004]: audit 2023-09-23T19:41:23.813201+0000 mon.a (mon.0) 1307 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]': finished 2023-09-23T19:41:25.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:24 smithi016 bash[75004]: audit 2023-09-23T19:41:23.813323+0000 mon.a (mon.0) 1308 : audit 1 from='client.? 172.21.15.16:0/2448525257' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T19:41:25.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:24 smithi016 bash[75004]: cluster 2023-09-23T19:41:23.813412+0000 mon.a (mon.0) 1309 : cluster 0 osdmap e180: 8 total, 8 up, 8 in 2023-09-23T19:41:25.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:24 smithi016 bash[75004]: cluster 2023-09-23T19:41:23.818067+0000 osd.4 (osd.4) 117 : cluster 3 failed to encode map e180 with expected crc 2023-09-23T19:41:25.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:24 smithi016 bash[75004]: cluster 2023-09-23T19:41:23.818115+0000 osd.7 (osd.7) 90 : cluster 3 failed to encode map e180 with expected crc 2023-09-23T19:41:25.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:24 smithi016 bash[75004]: cluster 2023-09-23T19:41:23.823147+0000 osd.4 (osd.4) 118 : cluster 3 failed to encode map e180 with expected crc 2023-09-23T19:41:25.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:24 smithi016 bash[75004]: cluster 2023-09-23T19:41:23.823290+0000 osd.6 (osd.6) 102 : cluster 3 failed to encode map e180 with expected crc 2023-09-23T19:41:25.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:24 smithi016 bash[77995]: audit 2023-09-23T19:41:23.813201+0000 mon.a (mon.0) 1307 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]': finished 2023-09-23T19:41:25.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:24 smithi016 bash[77995]: audit 2023-09-23T19:41:23.813323+0000 mon.a (mon.0) 1308 : audit 1 from='client.? 172.21.15.16:0/2448525257' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T19:41:25.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:24 smithi016 bash[77995]: cluster 2023-09-23T19:41:23.813412+0000 mon.a (mon.0) 1309 : cluster 0 osdmap e180: 8 total, 8 up, 8 in 2023-09-23T19:41:25.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:24 smithi016 bash[77995]: cluster 2023-09-23T19:41:23.818067+0000 osd.4 (osd.4) 117 : cluster 3 failed to encode map e180 with expected crc 2023-09-23T19:41:25.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:24 smithi016 bash[77995]: cluster 2023-09-23T19:41:23.818115+0000 osd.7 (osd.7) 90 : cluster 3 failed to encode map e180 with expected crc 2023-09-23T19:41:25.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:24 smithi016 bash[77995]: cluster 2023-09-23T19:41:23.823147+0000 osd.4 (osd.4) 118 : cluster 3 failed to encode map e180 with expected crc 2023-09-23T19:41:25.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:24 smithi016 bash[77995]: cluster 2023-09-23T19:41:23.823290+0000 osd.6 (osd.6) 102 : cluster 3 failed to encode map e180 with expected crc 2023-09-23T19:41:25.125 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:25.154 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:25.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:25.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:41:25.257 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:25.291 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:25.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:25.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:25.394 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:25.424 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:25.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:25.426 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:25.527 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:25.554 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:25.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:25.557 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:25.657 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:25.665 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:41:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:41:25] "GET /metrics HTTP/1.1" 200 33899 "" "Prometheus/2.33.4" 2023-09-23T19:41:25.686 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:25.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:25.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:25.788 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:25.818 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:25.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:25.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:25.921 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:25.954 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:25.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:25.957 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:26.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:25 smithi016 bash[75004]: cluster 2023-09-23T19:41:24.192611+0000 mgr.y (mgr.25325) 1017 : cluster 0 pgmap v968: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2023-09-23T19:41:26.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:25 smithi016 bash[75004]: cluster 2023-09-23T19:41:24.824250+0000 mon.a (mon.0) 1310 : cluster 0 osdmap e181: 8 total, 8 up, 8 in 2023-09-23T19:41:26.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:25 smithi016 bash[75004]: cluster 2023-09-23T19:41:24.830370+0000 osd.5 (osd.5) 111 : cluster 3 failed to encode map e181 with expected crc 2023-09-23T19:41:26.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:25 smithi016 bash[75004]: cluster 2023-09-23T19:41:24.830411+0000 osd.6 (osd.6) 103 : cluster 3 failed to encode map e181 with expected crc 2023-09-23T19:41:26.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:25 smithi016 bash[75004]: cluster 2023-09-23T19:41:24.830500+0000 osd.7 (osd.7) 91 : cluster 3 failed to encode map e181 with expected crc 2023-09-23T19:41:26.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:25 smithi016 bash[75004]: cluster 2023-09-23T19:41:24.830502+0000 osd.4 (osd.4) 119 : cluster 3 failed to encode map e181 with expected crc 2023-09-23T19:41:26.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:25 smithi016 bash[77995]: cluster 2023-09-23T19:41:24.192611+0000 mgr.y (mgr.25325) 1017 : cluster 0 pgmap v968: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2023-09-23T19:41:26.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:25 smithi016 bash[77995]: cluster 2023-09-23T19:41:24.824250+0000 mon.a (mon.0) 1310 : cluster 0 osdmap e181: 8 total, 8 up, 8 in 2023-09-23T19:41:26.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:25 smithi016 bash[77995]: cluster 2023-09-23T19:41:24.830370+0000 osd.5 (osd.5) 111 : cluster 3 failed to encode map e181 with expected crc 2023-09-23T19:41:26.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:25 smithi016 bash[77995]: cluster 2023-09-23T19:41:24.830411+0000 osd.6 (osd.6) 103 : cluster 3 failed to encode map e181 with expected crc 2023-09-23T19:41:26.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:25 smithi016 bash[77995]: cluster 2023-09-23T19:41:24.830500+0000 osd.7 (osd.7) 91 : cluster 3 failed to encode map e181 with expected crc 2023-09-23T19:41:26.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:25 smithi016 bash[77995]: cluster 2023-09-23T19:41:24.830502+0000 osd.4 (osd.4) 119 : cluster 3 failed to encode map e181 with expected crc 2023-09-23T19:41:26.047 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:41:26.058 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:26.079 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:25 smithi092 bash[58855]: cluster 2023-09-23T19:41:24.192611+0000 mgr.y (mgr.25325) 1017 : cluster 0 pgmap v968: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2023-09-23T19:41:26.080 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:25 smithi092 bash[58855]: cluster 2023-09-23T19:41:24.824250+0000 mon.a (mon.0) 1310 : cluster 0 osdmap e181: 8 total, 8 up, 8 in 2023-09-23T19:41:26.080 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:25 smithi092 bash[58855]: cluster 2023-09-23T19:41:24.830370+0000 osd.5 (osd.5) 111 : cluster 3 failed to encode map e181 with expected crc 2023-09-23T19:41:26.080 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:25 smithi092 bash[58855]: cluster 2023-09-23T19:41:24.830411+0000 osd.6 (osd.6) 103 : cluster 3 failed to encode map e181 with expected crc 2023-09-23T19:41:26.080 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:25 smithi092 bash[58855]: cluster 2023-09-23T19:41:24.830500+0000 osd.7 (osd.7) 91 : cluster 3 failed to encode map e181 with expected crc 2023-09-23T19:41:26.080 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:25 smithi092 bash[58855]: cluster 2023-09-23T19:41:24.830502+0000 osd.4 (osd.4) 119 : cluster 3 failed to encode map e181 with expected crc 2023-09-23T19:41:26.085 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:26.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:26.088 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:26.188 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:26.213 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:26.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:26.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:41:26.317 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:26.345 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:26.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:26.348 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:26.403 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-23T19:41:26.449 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:26.476 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:26.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:26.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:26.580 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:26.600 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_6 rados --yes-i-really-mean-it || true 2023-09-23T19:41:26.612 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:26.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:26.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:26.715 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:26.748 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:26.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:26.751 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:26.851 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:26.877 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:26.879 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:26 smithi092 bash[58855]: cluster 2023-09-23T19:41:25.828504+0000 mon.a (mon.0) 1311 : cluster 0 osdmap e182: 8 total, 8 up, 8 in 2023-09-23T19:41:26.879 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:26 smithi092 bash[58855]: cluster 2023-09-23T19:41:26.056579+0000 osd.7 (osd.7) 92 : cluster 3 failed to encode map e182 with expected crc 2023-09-23T19:41:26.879 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:26 smithi092 bash[58855]: cluster 2023-09-23T19:41:26.057664+0000 osd.7 (osd.7) 93 : cluster 3 failed to encode map e182 with expected crc 2023-09-23T19:41:26.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:26.880 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:26.980 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:27.012 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:27.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:27.015 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:41:27.115 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:27.145 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:27.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:27.149 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:27.249 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:27.281 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:27.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:27.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:27.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:26 smithi016 bash[75004]: cluster 2023-09-23T19:41:25.828504+0000 mon.a (mon.0) 1311 : cluster 0 osdmap e182: 8 total, 8 up, 8 in 2023-09-23T19:41:27.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:26 smithi016 bash[75004]: cluster 2023-09-23T19:41:26.056579+0000 osd.7 (osd.7) 92 : cluster 3 failed to encode map e182 with expected crc 2023-09-23T19:41:27.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:26 smithi016 bash[75004]: cluster 2023-09-23T19:41:26.057664+0000 osd.7 (osd.7) 93 : cluster 3 failed to encode map e182 with expected crc 2023-09-23T19:41:27.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:26 smithi016 bash[77995]: cluster 2023-09-23T19:41:25.828504+0000 mon.a (mon.0) 1311 : cluster 0 osdmap e182: 8 total, 8 up, 8 in 2023-09-23T19:41:27.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:26 smithi016 bash[77995]: cluster 2023-09-23T19:41:26.056579+0000 osd.7 (osd.7) 92 : cluster 3 failed to encode map e182 with expected crc 2023-09-23T19:41:27.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:26 smithi016 bash[77995]: cluster 2023-09-23T19:41:26.057664+0000 osd.7 (osd.7) 93 : cluster 3 failed to encode map e182 with expected crc 2023-09-23T19:41:27.385 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:27.409 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:27.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:27.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:27.514 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:27.546 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:27.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:27.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:27.649 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:27.678 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:27.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:27.681 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:27.727 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:41:27.755 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:41:27.781 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:27.784 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:41:27.809 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:27.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:27.812 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:41:27.814 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:41:27.839 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:41:27.877 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:41:27.901 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:41:27.912 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:27.924 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:27 smithi092 bash[58855]: cluster 2023-09-23T19:41:26.193073+0000 mgr.y (mgr.25325) 1018 : cluster 0 pgmap v971: 89 pgs: 3 unknown, 86 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2023-09-23T19:41:27.924 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:27 smithi092 bash[58855]: cluster 2023-09-23T19:41:26.302134+0000 osd.4 (osd.4) 120 : cluster 3 failed to encode map e182 with expected crc 2023-09-23T19:41:27.925 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:27 smithi092 bash[58855]: cluster 2023-09-23T19:41:26.304977+0000 osd.4 (osd.4) 121 : cluster 3 failed to encode map e182 with expected crc 2023-09-23T19:41:27.925 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:27 smithi092 bash[58855]: cluster 2023-09-23T19:41:26.887857+0000 osd.6 (osd.6) 104 : cluster 3 failed to encode map e182 with expected crc 2023-09-23T19:41:27.925 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:27 smithi092 bash[58855]: audit 2023-09-23T19:41:27.693482+0000 mon.a (mon.0) 1312 : audit 0 from='client.? 172.21.15.16:0/2663253875' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:41:27.925 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:41:27.927 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:41:27.927 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:41:27.927 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:41:27.927 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:41:27.927 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:41:27.927 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:41:27.927 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:41:27.927 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:41:27.942 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:27.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:27.944 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:28.045 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:28.075 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:28.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:28.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:28.167 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:27 smithi016 bash[75004]: cluster 2023-09-23T19:41:26.193073+0000 mgr.y (mgr.25325) 1018 : cluster 0 pgmap v971: 89 pgs: 3 unknown, 86 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2023-09-23T19:41:28.167 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:27 smithi016 bash[75004]: cluster 2023-09-23T19:41:26.302134+0000 osd.4 (osd.4) 120 : cluster 3 failed to encode map e182 with expected crc 2023-09-23T19:41:28.167 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:27 smithi016 bash[75004]: cluster 2023-09-23T19:41:26.304977+0000 osd.4 (osd.4) 121 : cluster 3 failed to encode map e182 with expected crc 2023-09-23T19:41:28.167 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:27 smithi016 bash[75004]: cluster 2023-09-23T19:41:26.887857+0000 osd.6 (osd.6) 104 : cluster 3 failed to encode map e182 with expected crc 2023-09-23T19:41:28.167 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:27 smithi016 bash[75004]: audit 2023-09-23T19:41:27.693482+0000 mon.a (mon.0) 1312 : audit 0 from='client.? 172.21.15.16:0/2663253875' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:41:28.167 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:27 smithi016 bash[77995]: cluster 2023-09-23T19:41:26.193073+0000 mgr.y (mgr.25325) 1018 : cluster 0 pgmap v971: 89 pgs: 3 unknown, 86 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2023-09-23T19:41:28.168 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:27 smithi016 bash[77995]: cluster 2023-09-23T19:41:26.302134+0000 osd.4 (osd.4) 120 : cluster 3 failed to encode map e182 with expected crc 2023-09-23T19:41:28.168 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:27 smithi016 bash[77995]: cluster 2023-09-23T19:41:26.304977+0000 osd.4 (osd.4) 121 : cluster 3 failed to encode map e182 with expected crc 2023-09-23T19:41:28.168 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:27 smithi016 bash[77995]: cluster 2023-09-23T19:41:26.887857+0000 osd.6 (osd.6) 104 : cluster 3 failed to encode map e182 with expected crc 2023-09-23T19:41:28.168 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:27 smithi016 bash[77995]: audit 2023-09-23T19:41:27.693482+0000 mon.a (mon.0) 1312 : audit 0 from='client.? 172.21.15.16:0/2663253875' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:41:28.179 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:28.205 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:28.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:28.208 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:28.308 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:28.329 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:28.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:28.332 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:41:28.432 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:28.459 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:28.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:28.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:28.562 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:28.589 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:28.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:28.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:28.693 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:28.721 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:28.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:28.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:28.825 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:28.856 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is unset 2023-09-23T19:41:28.860 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:28.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:28.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:41:28.964 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:28.994 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:28.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:28.997 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:29.098 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:29.100 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:28 smithi016 bash[75004]: cluster 2023-09-23T19:41:27.932452+0000 osd.5 (osd.5) 112 : cluster 3 failed to encode map e182 with expected crc 2023-09-23T19:41:29.100 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:28 smithi016 bash[75004]: audit 2023-09-23T19:41:27.966493+0000 mon.c (mon.2) 178 : audit 1 from='client.? 172.21.15.16:0/2372915754' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:41:29.100 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:28 smithi016 bash[75004]: audit 2023-09-23T19:41:27.967236+0000 mon.a (mon.0) 1313 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:41:29.101 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:28 smithi016 bash[75004]: audit 2023-09-23T19:41:28.660391+0000 mon.a (mon.0) 1314 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:41:29.101 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:28 smithi016 bash[77995]: cluster 2023-09-23T19:41:27.932452+0000 osd.5 (osd.5) 112 : cluster 3 failed to encode map e182 with expected crc 2023-09-23T19:41:29.101 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:28 smithi016 bash[77995]: audit 2023-09-23T19:41:27.966493+0000 mon.c (mon.2) 178 : audit 1 from='client.? 172.21.15.16:0/2372915754' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:41:29.101 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:28 smithi016 bash[77995]: audit 2023-09-23T19:41:27.967236+0000 mon.a (mon.0) 1313 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:41:29.101 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:28 smithi016 bash[77995]: audit 2023-09-23T19:41:28.660391+0000 mon.a (mon.0) 1314 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:41:29.122 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:29.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:29.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:41:29.226 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:29.234 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:28 smithi092 bash[58855]: cluster 2023-09-23T19:41:27.932452+0000 osd.5 (osd.5) 112 : cluster 3 failed to encode map e182 with expected crc 2023-09-23T19:41:29.234 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:28 smithi092 bash[58855]: audit 2023-09-23T19:41:27.966493+0000 mon.c (mon.2) 178 : audit 1 from='client.? 172.21.15.16:0/2372915754' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:41:29.235 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:28 smithi092 bash[58855]: audit 2023-09-23T19:41:27.967236+0000 mon.a (mon.0) 1313 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:41:29.235 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:28 smithi092 bash[58855]: audit 2023-09-23T19:41:28.660391+0000 mon.a (mon.0) 1314 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:41:29.253 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:29.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:29.256 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:29.356 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:29.383 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:29.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:29.385 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:29.486 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:29.511 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:29.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:29.514 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:41:29.603 INFO:teuthology.orchestra.run.smithi092.stdout:{ 2023-09-23T19:41:29.603 INFO:teuthology.orchestra.run.smithi092.stdout: "ops": [], 2023-09-23T19:41:29.603 INFO:teuthology.orchestra.run.smithi092.stdout: "complaint_time": 30, 2023-09-23T19:41:29.603 INFO:teuthology.orchestra.run.smithi092.stdout: "num_blocked_ops": 0 2023-09-23T19:41:29.603 INFO:teuthology.orchestra.run.smithi092.stdout:} 2023-09-23T19:41:29.614 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:29.643 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:29.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:29.646 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:29.747 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:29.776 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:29.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:29.780 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:41:29.880 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:29.912 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:29.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:29.915 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:41:30.016 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:30.024 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:29 smithi092 bash[58855]: cluster 2023-09-23T19:41:28.193620+0000 mgr.y (mgr.25325) 1019 : cluster 0 pgmap v972: 89 pgs: 1 unknown, 88 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2023-09-23T19:41:30.024 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:29 smithi092 bash[58855]: cluster 2023-09-23T19:41:28.836790+0000 mon.a (mon.0) 1315 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:41:30.025 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:29 smithi092 bash[58855]: audit 2023-09-23T19:41:28.848289+0000 mon.a (mon.0) 1316 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:41:30.025 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:29 smithi092 bash[58855]: cluster 2023-09-23T19:41:28.848397+0000 mon.a (mon.0) 1317 : cluster 0 osdmap e183: 8 total, 8 up, 8 in 2023-09-23T19:41:30.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:29 smithi016 bash[75004]: cluster 2023-09-23T19:41:28.193620+0000 mgr.y (mgr.25325) 1019 : cluster 0 pgmap v972: 89 pgs: 1 unknown, 88 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2023-09-23T19:41:30.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:29 smithi016 bash[75004]: cluster 2023-09-23T19:41:28.836790+0000 mon.a (mon.0) 1315 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:41:30.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:29 smithi016 bash[75004]: audit 2023-09-23T19:41:28.848289+0000 mon.a (mon.0) 1316 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:41:30.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:29 smithi016 bash[75004]: cluster 2023-09-23T19:41:28.848397+0000 mon.a (mon.0) 1317 : cluster 0 osdmap e183: 8 total, 8 up, 8 in 2023-09-23T19:41:30.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:29 smithi016 bash[77995]: cluster 2023-09-23T19:41:28.193620+0000 mgr.y (mgr.25325) 1019 : cluster 0 pgmap v972: 89 pgs: 1 unknown, 88 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2023-09-23T19:41:30.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:29 smithi016 bash[77995]: cluster 2023-09-23T19:41:28.836790+0000 mon.a (mon.0) 1315 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:41:30.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:29 smithi016 bash[77995]: audit 2023-09-23T19:41:28.848289+0000 mon.a (mon.0) 1316 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:41:30.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:29 smithi016 bash[77995]: cluster 2023-09-23T19:41:28.848397+0000 mon.a (mon.0) 1317 : cluster 0 osdmap e183: 8 total, 8 up, 8 in 2023-09-23T19:41:30.041 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:30.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:30.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:30.144 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:30.175 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:30.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:30.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:30.279 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:30.307 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:30.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:30.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:30.410 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:30.444 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:30.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:30.448 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:30.534 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T19:41:30.549 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:30.607 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:30.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:30.611 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:30.712 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:30.739 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:30.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:30.745 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:41:30.845 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:30.879 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:30.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:30.883 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:41:30.983 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:31.017 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:31.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:31.021 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:31.102 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:41:31.116 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:30 smithi016 bash[75004]: cluster 2023-09-23T19:41:29.967062+0000 osd.5 (osd.5) 113 : cluster 3 failed to encode map e183 with expected crc 2023-09-23T19:41:31.116 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:30 smithi016 bash[75004]: cluster 2023-09-23T19:41:30.469271+0000 osd.7 (osd.7) 94 : cluster 3 failed to encode map e183 with expected crc 2023-09-23T19:41:31.116 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:30 smithi016 bash[77995]: cluster 2023-09-23T19:41:29.967062+0000 osd.5 (osd.5) 113 : cluster 3 failed to encode map e183 with expected crc 2023-09-23T19:41:31.117 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:30 smithi016 bash[77995]: cluster 2023-09-23T19:41:30.469271+0000 osd.7 (osd.7) 94 : cluster 3 failed to encode map e183 with expected crc 2023-09-23T19:41:31.122 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:31.133 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:30 smithi092 bash[58855]: cluster 2023-09-23T19:41:29.967062+0000 osd.5 (osd.5) 113 : cluster 3 failed to encode map e183 with expected crc 2023-09-23T19:41:31.134 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:30 smithi092 bash[58855]: cluster 2023-09-23T19:41:30.469271+0000 osd.7 (osd.7) 94 : cluster 3 failed to encode map e183 with expected crc 2023-09-23T19:41:31.151 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:31.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:31.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:31.256 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:31.290 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:31.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:31.293 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:41:31.394 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:31.425 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:31.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:31.428 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:41:31.529 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:31.560 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:31.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:31.564 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:31.664 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:31.695 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:31.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:31.698 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:31.798 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:31.821 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T19:41:31.826 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:31.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:31.829 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:31.930 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:31.941 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:31 smithi016 bash[75004]: cluster 2023-09-23T19:41:30.194385+0000 mgr.y (mgr.25325) 1020 : cluster 0 pgmap v974: 89 pgs: 89 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2023-09-23T19:41:31.941 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:31 smithi016 bash[75004]: cluster 2023-09-23T19:41:30.299431+0000 osd.4 (osd.4) 122 : cluster 3 failed to encode map e183 with expected crc 2023-09-23T19:41:31.941 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:31 smithi016 bash[75004]: cluster 2023-09-23T19:41:30.607188+0000 osd.6 (osd.6) 105 : cluster 3 failed to encode map e183 with expected crc 2023-09-23T19:41:31.941 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:31 smithi016 bash[77995]: cluster 2023-09-23T19:41:30.194385+0000 mgr.y (mgr.25325) 1020 : cluster 0 pgmap v974: 89 pgs: 89 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2023-09-23T19:41:31.941 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:31 smithi016 bash[77995]: cluster 2023-09-23T19:41:30.299431+0000 osd.4 (osd.4) 122 : cluster 3 failed to encode map e183 with expected crc 2023-09-23T19:41:31.941 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:31 smithi016 bash[77995]: cluster 2023-09-23T19:41:30.607188+0000 osd.6 (osd.6) 105 : cluster 3 failed to encode map e183 with expected crc 2023-09-23T19:41:31.959 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:31.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:31.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:41:32.062 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:32.075 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:31 smithi092 bash[58855]: cluster 2023-09-23T19:41:30.194385+0000 mgr.y (mgr.25325) 1020 : cluster 0 pgmap v974: 89 pgs: 89 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2023-09-23T19:41:32.075 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:31 smithi092 bash[58855]: cluster 2023-09-23T19:41:30.299431+0000 osd.4 (osd.4) 122 : cluster 3 failed to encode map e183 with expected crc 2023-09-23T19:41:32.075 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:31 smithi092 bash[58855]: cluster 2023-09-23T19:41:30.607188+0000 osd.6 (osd.6) 105 : cluster 3 failed to encode map e183 with expected crc 2023-09-23T19:41:32.095 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:32.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:32.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:32.199 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:32.230 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:32.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:32.234 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:41:32.331 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:41:32.332 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:41:32.335 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:32.365 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:32.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:32.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:32.468 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:32.494 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:32.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:32.497 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:41:32.598 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:32.625 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:32.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:32.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:32.729 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:32.752 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:32.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:32.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:32.856 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:32.883 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:32.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:32.885 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:41:32.986 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:33.015 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:33.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:33.019 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:33.119 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:33.152 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:33.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:33.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:41:33.256 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:33.289 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:33.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:33.292 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:33.392 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:33.431 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:33.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:33.433 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:33.534 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:33.566 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:33.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:33.570 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:33.671 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:33.706 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:33.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:33.709 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:33.810 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:33.835 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:33.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:33.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:41:33.938 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:33.969 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:33.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:33.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:41:34.073 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:34.082 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:33 smithi016 bash[77995]: cluster 2023-09-23T19:41:32.195472+0000 mgr.y (mgr.25325) 1021 : cluster 0 pgmap v975: 89 pgs: 89 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2023-09-23T19:41:34.082 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:33 smithi016 bash[75004]: cluster 2023-09-23T19:41:32.195472+0000 mgr.y (mgr.25325) 1021 : cluster 0 pgmap v975: 89 pgs: 89 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2023-09-23T19:41:34.098 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:34.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:34.101 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:34.201 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:34.214 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:33 smithi092 bash[58855]: cluster 2023-09-23T19:41:32.195472+0000 mgr.y (mgr.25325) 1021 : cluster 0 pgmap v975: 89 pgs: 89 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2023-09-23T19:41:34.235 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:34.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:34.237 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:34.338 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:34.369 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:34.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:34.372 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:34.473 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:34.503 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:34.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:34.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:34.606 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:34.634 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:34.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:34.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:34.737 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:34.768 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:34.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:34.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:34.871 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:34.903 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:34.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:34.906 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:35.007 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:35.037 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:35.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:35.040 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:41:35.141 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:35.166 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:35.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:35.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:41:35.270 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:35.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:35 smithi016 bash[77995]: audit 2023-09-23T19:41:34.122298+0000 mon.a (mon.0) 1318 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:41:35.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:35 smithi016 bash[77995]: cluster 2023-09-23T19:41:34.196127+0000 mgr.y (mgr.25325) 1022 : cluster 0 pgmap v976: 89 pgs: 89 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2023-09-23T19:41:35.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:35 smithi016 bash[75004]: audit 2023-09-23T19:41:34.122298+0000 mon.a (mon.0) 1318 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:41:35.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:35 smithi016 bash[75004]: cluster 2023-09-23T19:41:34.196127+0000 mgr.y (mgr.25325) 1022 : cluster 0 pgmap v976: 89 pgs: 89 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2023-09-23T19:41:35.302 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:35.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:35.305 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:41:35.318 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:41:35.406 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:35.419 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:35 smithi092 bash[58855]: audit 2023-09-23T19:41:34.122298+0000 mon.a (mon.0) 1318 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:41:35.419 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:35 smithi092 bash[58855]: cluster 2023-09-23T19:41:34.196127+0000 mgr.y (mgr.25325) 1022 : cluster 0 pgmap v976: 89 pgs: 89 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2023-09-23T19:41:35.441 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:35.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:35.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:41:35.543 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:35.572 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:35.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:35.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:35.676 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:35.709 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:35.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:35.712 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:41:35.785 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:41:35 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:41:35] "GET /metrics HTTP/1.1" 200 36233 "" "Prometheus/2.33.4" 2023-09-23T19:41:35.813 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:35.838 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:35.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:35.842 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:35.918 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:41:35.942 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:35.975 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:35.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:35.977 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:36.078 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:36.100 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:36.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:36.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:36.203 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:36.235 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:36.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:36.237 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:36.338 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:36.367 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:36.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:36.369 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:36.394 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:41:36.470 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:36.496 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:36.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:36.498 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:36.599 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:36.626 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:36.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:36.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:41:36.730 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:36.757 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:36.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:36.759 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:36.860 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:36.889 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:36.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:36.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:41:36.993 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:37.017 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:37.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:37.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:41:37.120 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:37.149 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:37.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:37.151 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:37.251 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:37.281 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:37.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:37.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:37.384 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:37.409 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:37.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:37.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:41:37.512 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:37.522 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:37 smithi092 bash[58855]: cluster 2023-09-23T19:41:36.196894+0000 mgr.y (mgr.25325) 1023 : cluster 0 pgmap v977: 89 pgs: 89 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2023-09-23T19:41:37.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:37 smithi016 bash[75004]: cluster 2023-09-23T19:41:36.196894+0000 mgr.y (mgr.25325) 1023 : cluster 0 pgmap v977: 89 pgs: 89 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2023-09-23T19:41:37.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:37 smithi016 bash[77995]: cluster 2023-09-23T19:41:36.196894+0000 mgr.y (mgr.25325) 1023 : cluster 0 pgmap v977: 89 pgs: 89 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2023-09-23T19:41:37.539 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:37.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:37.542 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:37.642 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:37.668 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:37.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:37.670 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:37.771 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:37.801 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:37.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:37.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:41:37.905 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:37.938 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:37.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:37.942 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:38.043 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:38.071 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:38.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:38.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:38.174 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:38.203 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:38.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:38.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:38.307 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:38.337 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:38.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:38.339 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:38.440 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:38.471 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:38.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:38.474 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:41:38.575 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:38.606 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:38.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:38.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:38.710 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:38.744 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:38.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:38.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:38.848 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:38.870 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:38.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:38.872 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:41:38.958 INFO:teuthology.orchestra.run.smithi092.stdout:{ 2023-09-23T19:41:38.958 INFO:teuthology.orchestra.run.smithi092.stdout: "size": 20, 2023-09-23T19:41:38.958 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 600, 2023-09-23T19:41:38.958 INFO:teuthology.orchestra.run.smithi092.stdout: "ops": [ 2023-09-23T19:41:38.958 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.958 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:26585 9.3 9:cb81db0c:::benchmark_data_smithi016_113233_object26584:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e122)", 2023-09-23T19:41:38.958 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:31:46.551642+0000", 2023-09-23T19:41:38.958 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 592.40291592799997, 2023-09-23T19:41:38.959 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.19433394500000001, 2023-09-23T19:41:38.959 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:41:38.959 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:41:38.959 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:41:38.959 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:41:38.959 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:41:38.959 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 26585 2023-09-23T19:41:38.959 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.959 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:41:38.959 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.959 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:41:38.959 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.551642+0000", 2023-09-23T19:41:38.960 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:41:38.960 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.960 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.960 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:41:38.960 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.551642+0000", 2023-09-23T19:41:38.960 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T19:41:38.960 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.960 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.960 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:41:38.960 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.551641+0000", 2023-09-23T19:41:38.960 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.3479999999999998e-05 2023-09-23T19:41:38.960 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.961 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.961 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:41:38.961 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.551674+0000", 2023-09-23T19:41:38.961 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.2439999999999999e-06 2023-09-23T19:41:38.961 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.961 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.961 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:41:38.961 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.551683+0000", 2023-09-23T19:41:38.961 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.6189999999999999e-06 2023-09-23T19:41:38.961 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.961 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.961 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:41:38.962 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.551689+0000", 2023-09-23T19:41:38.962 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00014126500000000001 2023-09-23T19:41:38.962 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.962 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.962 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:41:38.962 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.551830+0000", 2023-09-23T19:41:38.962 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00013956600000000001 2023-09-23T19:41:38.962 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.962 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.962 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:41:38.962 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.551970+0000", 2023-09-23T19:41:38.963 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00016731800000000001 2023-09-23T19:41:38.963 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.963 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.963 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,3", 2023-09-23T19:41:38.963 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.552137+0000", 2023-09-23T19:41:38.963 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.001424504 2023-09-23T19:41:38.963 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.963 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.963 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:41:38.963 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.553562+0000", 2023-09-23T19:41:38.963 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0046421320000000002 2023-09-23T19:41:38.963 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.964 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.964 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:41:38.964 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.558204+0000", 2023-09-23T19:41:38.964 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.187635939 2023-09-23T19:41:38.964 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.964 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.964 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:41:38.964 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.745840+0000", 2023-09-23T19:41:38.964 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.9017999999999995e-05 2023-09-23T19:41:38.964 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.964 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.964 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:41:38.965 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.745919+0000", 2023-09-23T19:41:38.965 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.7176000000000003e-05 2023-09-23T19:41:38.965 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.965 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.965 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:41:38.965 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.745976+0000", 2023-09-23T19:41:38.965 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.19433394500000001 2023-09-23T19:41:38.965 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:41:38.965 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:41:38.965 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:41:38.965 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.966 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.966 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:26598 9.3 9:c53a6029:::benchmark_data_smithi016_113233_object26597:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e122)", 2023-09-23T19:41:38.966 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:31:46.556651+0000", 2023-09-23T19:41:38.966 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 592.39790676099994, 2023-09-23T19:41:38.966 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.18955604600000001, 2023-09-23T19:41:38.966 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:41:38.966 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:41:38.966 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:41:38.966 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:41:38.966 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:41:38.966 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 26598 2023-09-23T19:41:38.966 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.967 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:41:38.967 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.967 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:41:38.967 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.556651+0000", 2023-09-23T19:41:38.967 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:41:38.967 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.967 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.967 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:41:38.967 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.556651+0000", 2023-09-23T19:41:38.967 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T19:41:38.967 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.967 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.968 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:41:38.968 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.556650+0000", 2023-09-23T19:41:38.968 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.2991999999999998e-05 2023-09-23T19:41:38.968 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.968 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.968 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:41:38.968 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.556693+0000", 2023-09-23T19:41:38.968 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.5656000000000001e-05 2023-09-23T19:41:38.968 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.968 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.968 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:41:38.968 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.556708+0000", 2023-09-23T19:41:38.968 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.7762e-05 2023-09-23T19:41:38.969 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.969 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.969 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:41:38.969 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.556726+0000", 2023-09-23T19:41:38.969 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.9646000000000004e-05 2023-09-23T19:41:38.969 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.969 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.969 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:41:38.969 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.556796+0000", 2023-09-23T19:41:38.969 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00017475800000000001 2023-09-23T19:41:38.969 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.969 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.969 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:41:38.970 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.556971+0000", 2023-09-23T19:41:38.970 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00015882299999999999 2023-09-23T19:41:38.970 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.970 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.970 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,3", 2023-09-23T19:41:38.970 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.557129+0000", 2023-09-23T19:41:38.970 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00114853 2023-09-23T19:41:38.970 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.970 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.970 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:41:38.970 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.558278+0000", 2023-09-23T19:41:38.970 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.17477084800000001 2023-09-23T19:41:38.971 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.971 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.971 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:41:38.971 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.733049+0000", 2023-09-23T19:41:38.971 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.013031696000000001 2023-09-23T19:41:38.971 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.971 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.971 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:41:38.971 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.746081+0000", 2023-09-23T19:41:38.971 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.4264999999999994e-05 2023-09-23T19:41:38.971 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.971 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.971 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:41:38.971 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.746165+0000", 2023-09-23T19:41:38.972 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.2463000000000003e-05 2023-09-23T19:41:38.972 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.972 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.972 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:41:38.972 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.746207+0000", 2023-09-23T19:41:38.972 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.18955604600000001 2023-09-23T19:41:38.972 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:41:38.972 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:41:38.972 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:41:38.972 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.972 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.972 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:26599 9.d 9:bef5b10a:::benchmark_data_smithi016_113233_object26598:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e122)", 2023-09-23T19:41:38.972 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:31:46.558587+0000", 2023-09-23T19:41:38.973 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 592.39597134099995, 2023-09-23T19:41:38.973 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.173559186, 2023-09-23T19:41:38.973 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:41:38.973 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:41:38.973 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:41:38.973 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:41:38.973 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:41:38.973 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 26599 2023-09-23T19:41:38.973 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.973 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:41:38.973 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.973 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:41:38.973 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.558587+0000", 2023-09-23T19:41:38.974 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:41:38.974 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.974 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.974 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:41:38.974 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.558587+0000", 2023-09-23T19:41:38.974 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T19:41:38.974 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.974 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.974 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:41:38.974 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.558585+0000", 2023-09-23T19:41:38.974 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.0269000000000002e-05 2023-09-23T19:41:38.974 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.974 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.974 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:41:38.975 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.558646+0000", 2023-09-23T19:41:38.975 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.1790000000000001e-05 2023-09-23T19:41:38.975 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.975 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.975 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:41:38.975 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.558657+0000", 2023-09-23T19:41:38.975 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.3800000000000002e-06 2023-09-23T19:41:38.975 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.975 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.975 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:41:38.975 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.558663+0000", 2023-09-23T19:41:38.975 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.3485e-05 2023-09-23T19:41:38.975 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.976 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.976 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:41:38.976 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.558716+0000", 2023-09-23T19:41:38.976 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00015891200000000001 2023-09-23T19:41:38.976 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.976 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.976 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:41:38.976 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.558875+0000", 2023-09-23T19:41:38.976 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00016389 2023-09-23T19:41:38.976 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.976 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.976 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,5", 2023-09-23T19:41:38.976 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.559039+0000", 2023-09-23T19:41:38.976 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0013228000000000001 2023-09-23T19:41:38.977 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.977 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.977 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:41:38.977 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.560362+0000", 2023-09-23T19:41:38.977 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.001284751 2023-09-23T19:41:38.977 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.977 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.977 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:41:38.977 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.561647+0000", 2023-09-23T19:41:38.977 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.17030458400000001 2023-09-23T19:41:38.977 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.977 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.977 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:41:38.978 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.731951+0000", 2023-09-23T19:41:38.978 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000114024 2023-09-23T19:41:38.978 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.978 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.978 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:41:38.978 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.732065+0000", 2023-09-23T19:41:38.978 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.0548000000000002e-05 2023-09-23T19:41:38.978 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.978 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.978 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:41:38.978 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:31:46.732146+0000", 2023-09-23T19:41:38.978 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.173559186 2023-09-23T19:41:38.978 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:41:38.979 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:41:38.979 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:41:38.979 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.979 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.979 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:71071 9.e 9:713311a6:::benchmark_data_smithi016_113233_object71070:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e124)", 2023-09-23T19:41:38.979 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:32:22.033184+0000", 2023-09-23T19:41:38.979 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 556.92137407500002, 2023-09-23T19:41:38.979 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.181195512, 2023-09-23T19:41:38.979 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:41:38.979 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:41:38.979 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:41:38.979 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:41:38.979 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:41:38.980 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 71071 2023-09-23T19:41:38.980 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.980 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:41:38.980 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.980 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:41:38.980 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.033184+0000", 2023-09-23T19:41:38.980 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:41:38.980 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.980 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.980 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:41:38.980 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.033184+0000", 2023-09-23T19:41:38.980 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T19:41:38.980 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.980 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.981 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:41:38.981 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.033182+0000", 2023-09-23T19:41:38.981 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.2682999999999999e-05 2023-09-23T19:41:38.981 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.981 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.981 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:41:38.981 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.033215+0000", 2023-09-23T19:41:38.981 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.3942e-05 2023-09-23T19:41:38.981 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.981 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.981 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:41:38.981 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.033229+0000", 2023-09-23T19:41:38.981 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.2189999999999998e-06 2023-09-23T19:41:38.981 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.982 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.982 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:41:38.982 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.033236+0000", 2023-09-23T19:41:38.982 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000260511 2023-09-23T19:41:38.982 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.982 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.982 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:41:38.982 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.033496+0000", 2023-09-23T19:41:38.982 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00028382 2023-09-23T19:41:38.982 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.982 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.982 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:41:38.982 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.033780+0000", 2023-09-23T19:41:38.982 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000164947 2023-09-23T19:41:38.983 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.983 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.983 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,4", 2023-09-23T19:41:38.983 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.033945+0000", 2023-09-23T19:41:38.983 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0053813419999999999 2023-09-23T19:41:38.983 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.983 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.983 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:41:38.983 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.039326+0000", 2023-09-23T19:41:38.983 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.1019999999999997e-06 2023-09-23T19:41:38.983 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.983 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.983 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:41:38.984 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.039335+0000", 2023-09-23T19:41:38.984 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.17478982200000001 2023-09-23T19:41:38.984 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.984 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.984 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:41:38.984 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.214124+0000", 2023-09-23T19:41:38.984 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000114564 2023-09-23T19:41:38.984 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.984 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.984 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:41:38.984 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.214239+0000", 2023-09-23T19:41:38.984 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000140443 2023-09-23T19:41:38.984 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.984 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.985 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:41:38.985 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.214379+0000", 2023-09-23T19:41:38.985 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.181195512 2023-09-23T19:41:38.985 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:41:38.985 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:41:38.985 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:41:38.985 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.985 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.985 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:71078 9.e 9:79417d87:::benchmark_data_smithi016_113233_object71077:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e124)", 2023-09-23T19:41:38.985 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:32:22.037570+0000", 2023-09-23T19:41:38.985 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 556.916987672, 2023-09-23T19:41:38.985 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.18826123, 2023-09-23T19:41:38.985 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:41:38.985 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:41:38.986 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:41:38.986 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:41:38.986 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:41:38.986 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 71078 2023-09-23T19:41:38.986 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.986 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:41:38.986 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.986 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:41:38.986 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.037570+0000", 2023-09-23T19:41:38.986 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:41:38.986 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.986 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.986 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:41:38.987 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.037570+0000", 2023-09-23T19:41:38.987 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T19:41:38.987 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.987 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.987 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:41:38.987 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.037569+0000", 2023-09-23T19:41:38.987 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.4791999999999999e-05 2023-09-23T19:41:38.987 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.987 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.987 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:41:38.987 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.037634+0000", 2023-09-23T19:41:38.987 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.5428000000000001e-05 2023-09-23T19:41:38.987 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.987 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.988 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:41:38.988 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.037659+0000", 2023-09-23T19:41:38.988 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.238e-06 2023-09-23T19:41:38.988 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.988 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.988 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:41:38.988 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.037666+0000", 2023-09-23T19:41:38.988 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.001695876 2023-09-23T19:41:38.988 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.988 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.988 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:41:38.988 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.039362+0000", 2023-09-23T19:41:38.988 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00013776700000000001 2023-09-23T19:41:38.989 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.989 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.989 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:41:38.989 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.039500+0000", 2023-09-23T19:41:38.989 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000164258 2023-09-23T19:41:38.989 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.989 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.989 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,4", 2023-09-23T19:41:38.989 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.039664+0000", 2023-09-23T19:41:38.989 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0032891230000000001 2023-09-23T19:41:38.989 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.989 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.989 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:41:38.989 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.042953+0000", 2023-09-23T19:41:38.990 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0038750659999999999 2023-09-23T19:41:38.990 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.990 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.990 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:41:38.990 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.046829+0000", 2023-09-23T19:41:38.990 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.178904596 2023-09-23T19:41:38.990 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.990 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.990 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:41:38.990 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.225733+0000", 2023-09-23T19:41:38.990 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.3872000000000002e-05 2023-09-23T19:41:38.990 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.990 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.991 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:41:38.991 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.225787+0000", 2023-09-23T19:41:38.991 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.4545000000000001e-05 2023-09-23T19:41:38.991 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.991 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.991 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:41:38.991 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.225832+0000", 2023-09-23T19:41:38.991 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.18826123 2023-09-23T19:41:38.991 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:41:38.991 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:41:38.991 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:41:38.991 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.991 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.991 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:71087 9.e 9:7757b3dd:::benchmark_data_smithi016_113233_object71086:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e124)", 2023-09-23T19:41:38.992 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:32:22.047091+0000", 2023-09-23T19:41:38.992 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 556.90746648200002, 2023-09-23T19:41:38.992 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.17886874899999999, 2023-09-23T19:41:38.992 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:41:38.992 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:41:38.992 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:41:38.992 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:41:38.992 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:41:38.992 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 71087 2023-09-23T19:41:38.992 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.992 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:41:38.992 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.992 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:41:38.992 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.047091+0000", 2023-09-23T19:41:38.993 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:41:38.993 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.993 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.993 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:41:38.993 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.047091+0000", 2023-09-23T19:41:38.993 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T19:41:38.993 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.993 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.993 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:41:38.993 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.047089+0000", 2023-09-23T19:41:38.993 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.1995e-05 2023-09-23T19:41:38.993 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.993 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.993 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:41:38.994 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.047131+0000", 2023-09-23T19:41:38.994 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.4183e-05 2023-09-23T19:41:38.994 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.994 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.994 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:41:38.994 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.047146+0000", 2023-09-23T19:41:38.994 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.9540000000000004e-06 2023-09-23T19:41:38.994 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.994 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.994 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:41:38.994 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.047153+0000", 2023-09-23T19:41:38.994 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.8807000000000001e-05 2023-09-23T19:41:38.994 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.995 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.995 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:41:38.995 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.047212+0000", 2023-09-23T19:41:38.995 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00014279700000000001 2023-09-23T19:41:38.995 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.995 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.995 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:41:38.995 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.047355+0000", 2023-09-23T19:41:38.995 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00029963899999999998 2023-09-23T19:41:38.995 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.995 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.995 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,4", 2023-09-23T19:41:38.995 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.047655+0000", 2023-09-23T19:41:38.995 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0090374549999999998 2023-09-23T19:41:38.996 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.996 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.996 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:41:38.996 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.056692+0000", 2023-09-23T19:41:38.996 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0029485779999999999 2023-09-23T19:41:38.996 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.996 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.996 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:41:38.996 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.059641+0000", 2023-09-23T19:41:38.996 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.166230761 2023-09-23T19:41:38.996 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.996 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.996 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:41:38.997 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.225872+0000", 2023-09-23T19:41:38.997 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.0167999999999998e-05 2023-09-23T19:41:38.997 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.997 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.997 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:41:38.997 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.225922+0000", 2023-09-23T19:41:38.997 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.8529999999999999e-05 2023-09-23T19:41:38.997 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.997 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.997 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:41:38.997 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.225960+0000", 2023-09-23T19:41:38.997 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.17886874899999999 2023-09-23T19:41:38.997 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:41:38.997 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:41:38.998 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:41:38.998 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.998 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.998 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:71089 9.e 9:774585d3:::benchmark_data_smithi016_113233_object71088:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e124)", 2023-09-23T19:41:38.998 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:32:22.050155+0000", 2023-09-23T19:41:38.998 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 556.90440299500005, 2023-09-23T19:41:38.998 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.175930751, 2023-09-23T19:41:38.998 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:41:38.998 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:41:38.998 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:41:38.998 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:41:38.998 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:41:38.998 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 71089 2023-09-23T19:41:38.999 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.999 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:41:38.999 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.999 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:41:38.999 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.050155+0000", 2023-09-23T19:41:38.999 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:41:38.999 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.999 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:38.999 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:41:38.999 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.050155+0000", 2023-09-23T19:41:38.999 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.999999 2023-09-23T19:41:38.999 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:38.999 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.000 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:41:39.000 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.050154+0000", 2023-09-23T19:41:39.000 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.5884000000000002e-05 2023-09-23T19:41:39.000 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.000 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.000 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:41:39.000 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.050180+0000", 2023-09-23T19:41:39.000 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.8975999999999998e-05 2023-09-23T19:41:39.000 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.000 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.000 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:41:39.000 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.050249+0000", 2023-09-23T19:41:39.000 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.8759999999999999e-06 2023-09-23T19:41:39.000 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.001 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.001 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:41:39.001 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.050256+0000", 2023-09-23T19:41:39.001 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0031540100000000001 2023-09-23T19:41:39.001 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.001 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.001 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:41:39.001 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.053410+0000", 2023-09-23T19:41:39.001 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00013616100000000001 2023-09-23T19:41:39.001 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.001 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.001 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:41:39.001 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.053546+0000", 2023-09-23T19:41:39.002 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00012943899999999999 2023-09-23T19:41:39.002 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.002 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.002 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,4", 2023-09-23T19:41:39.002 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.053675+0000", 2023-09-23T19:41:39.002 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0059726470000000002 2023-09-23T19:41:39.002 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.002 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.002 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:41:39.002 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.059648+0000", 2023-09-23T19:41:39.002 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.248e-05 2023-09-23T19:41:39.002 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.002 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.003 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:41:39.003 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.059690+0000", 2023-09-23T19:41:39.003 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.16630597499999999 2023-09-23T19:41:39.003 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.003 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.003 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:41:39.003 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.225996+0000", 2023-09-23T19:41:39.003 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.7840999999999998e-05 2023-09-23T19:41:39.003 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.003 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.003 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:41:39.003 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.226044+0000", 2023-09-23T19:41:39.003 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.1561999999999999e-05 2023-09-23T19:41:39.004 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.004 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.004 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:41:39.004 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.226086+0000", 2023-09-23T19:41:39.004 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.175930751 2023-09-23T19:41:39.004 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:41:39.004 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:41:39.004 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:41:39.004 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.004 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.004 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:71092 9.e 9:7c97ae4b:::benchmark_data_smithi016_113233_object71091:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e124)", 2023-09-23T19:41:39.004 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:32:22.052519+0000", 2023-09-23T19:41:39.004 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 556.90203903099996, 2023-09-23T19:41:39.005 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.17368992999999999, 2023-09-23T19:41:39.005 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:41:39.005 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:41:39.005 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:41:39.005 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:41:39.005 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:41:39.005 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 71092 2023-09-23T19:41:39.005 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.005 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:41:39.005 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.005 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:41:39.005 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.052519+0000", 2023-09-23T19:41:39.006 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:41:39.006 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.006 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.006 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:41:39.006 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.052519+0000", 2023-09-23T19:41:39.006 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T19:41:39.006 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.006 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.006 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:41:39.006 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.052518+0000", 2023-09-23T19:41:39.006 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.9794999999999999e-05 2023-09-23T19:41:39.006 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.007 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.007 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:41:39.007 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.052577+0000", 2023-09-23T19:41:39.007 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.5046e-05 2023-09-23T19:41:39.007 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.007 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.007 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:41:39.007 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.052592+0000", 2023-09-23T19:41:39.007 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.2539999999999992e-06 2023-09-23T19:41:39.007 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.007 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.007 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:41:39.007 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.052601+0000", 2023-09-23T19:41:39.007 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0041490149999999998 2023-09-23T19:41:39.008 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.008 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.008 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:41:39.008 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.056750+0000", 2023-09-23T19:41:39.008 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000250234 2023-09-23T19:41:39.008 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.008 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.008 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:41:39.008 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.057000+0000", 2023-09-23T19:41:39.008 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00017354499999999999 2023-09-23T19:41:39.008 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.008 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.008 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,4", 2023-09-23T19:41:39.009 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.057173+0000", 2023-09-23T19:41:39.009 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0053271459999999996 2023-09-23T19:41:39.009 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.009 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.009 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:41:39.009 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.062501+0000", 2023-09-23T19:41:39.009 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.001941536 2023-09-23T19:41:39.009 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.009 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.009 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:41:39.009 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.064442+0000", 2023-09-23T19:41:39.009 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.16168389899999999 2023-09-23T19:41:39.009 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.010 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.010 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:41:39.010 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.226126+0000", 2023-09-23T19:41:39.010 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.3513999999999997e-05 2023-09-23T19:41:39.010 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.010 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.010 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:41:39.010 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.226170+0000", 2023-09-23T19:41:39.010 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.9360000000000003e-05 2023-09-23T19:41:39.010 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.010 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.010 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:41:39.010 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.226209+0000", 2023-09-23T19:41:39.011 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.17368992999999999 2023-09-23T19:41:39.011 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:41:39.011 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:41:39.011 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:41:39.011 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.011 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.011 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:71094 9.3 9:c2ab45ca:::benchmark_data_smithi016_113233_object71093:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e124)", 2023-09-23T19:41:39.011 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:32:22.056556+0000", 2023-09-23T19:41:39.011 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 556.89800193099995, 2023-09-23T19:41:39.011 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.165906251, 2023-09-23T19:41:39.011 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:41:39.011 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:41:39.011 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:41:39.012 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:41:39.012 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:41:39.012 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 71094 2023-09-23T19:41:39.012 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.012 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:41:39.012 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.012 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:41:39.012 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.056556+0000", 2023-09-23T19:41:39.012 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:41:39.012 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.012 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.012 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:41:39.012 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.056556+0000", 2023-09-23T19:41:39.012 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T19:41:39.013 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.013 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.013 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:41:39.013 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.056554+0000", 2023-09-23T19:41:39.013 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.8666999999999999e-05 2023-09-23T19:41:39.013 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.013 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.013 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:41:39.013 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.056613+0000", 2023-09-23T19:41:39.013 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.9188e-05 2023-09-23T19:41:39.013 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.013 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.013 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:41:39.014 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.056632+0000", 2023-09-23T19:41:39.014 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.396e-06 2023-09-23T19:41:39.014 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.014 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.014 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:41:39.014 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.056640+0000", 2023-09-23T19:41:39.014 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.3092e-05 2023-09-23T19:41:39.014 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.014 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.014 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:41:39.014 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.056683+0000", 2023-09-23T19:41:39.014 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000103237 2023-09-23T19:41:39.014 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.014 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.015 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:41:39.015 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.056786+0000", 2023-09-23T19:41:39.015 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000145845 2023-09-23T19:41:39.015 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.015 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.015 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,3", 2023-09-23T19:41:39.015 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.056932+0000", 2023-09-23T19:41:39.015 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0039338469999999999 2023-09-23T19:41:39.015 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.015 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.015 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:41:39.015 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.060866+0000", 2023-09-23T19:41:39.015 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0013245279999999999 2023-09-23T19:41:39.016 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.016 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.016 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:41:39.016 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.062190+0000", 2023-09-23T19:41:39.016 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.16003255299999999 2023-09-23T19:41:39.016 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.016 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.016 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:41:39.016 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.222223+0000", 2023-09-23T19:41:39.016 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00015455300000000001 2023-09-23T19:41:39.016 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.016 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.016 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:41:39.017 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.222377+0000", 2023-09-23T19:41:39.017 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.4900000000000004e-05 2023-09-23T19:41:39.017 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.017 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.017 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:41:39.017 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.222462+0000", 2023-09-23T19:41:39.017 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.165906251 2023-09-23T19:41:39.017 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:41:39.017 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:41:39.017 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:41:39.017 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.017 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.017 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:71095 9.3 9:c9176f15:::benchmark_data_smithi016_113233_object71094:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e124)", 2023-09-23T19:41:39.017 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:32:22.056791+0000", 2023-09-23T19:41:39.018 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 556.897767155, 2023-09-23T19:41:39.018 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.166731289, 2023-09-23T19:41:39.018 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:41:39.018 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:41:39.018 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:41:39.018 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:41:39.018 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:41:39.018 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 71095 2023-09-23T19:41:39.018 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.018 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:41:39.018 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.018 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:41:39.018 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.056791+0000", 2023-09-23T19:41:39.019 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:41:39.019 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.019 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.019 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:41:39.019 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.056791+0000", 2023-09-23T19:41:39.019 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T19:41:39.019 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.019 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.019 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:41:39.019 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.056790+0000", 2023-09-23T19:41:39.019 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.4085999999999995e-05 2023-09-23T19:41:39.019 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.019 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.020 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:41:39.020 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.056864+0000", 2023-09-23T19:41:39.020 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.1114999999999999e-05 2023-09-23T19:41:39.020 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.020 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.020 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:41:39.020 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.056875+0000", 2023-09-23T19:41:39.020 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.5230000000000001e-06 2023-09-23T19:41:39.020 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.020 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.020 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:41:39.020 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.056881+0000", 2023-09-23T19:41:39.020 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0021203960000000001 2023-09-23T19:41:39.021 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.021 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.021 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:41:39.021 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.059002+0000", 2023-09-23T19:41:39.021 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.9608000000000001e-05 2023-09-23T19:41:39.021 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.021 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.021 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:41:39.021 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.059081+0000", 2023-09-23T19:41:39.021 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000100628 2023-09-23T19:41:39.021 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.021 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.021 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,3", 2023-09-23T19:41:39.021 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.059182+0000", 2023-09-23T19:41:39.022 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0027136830000000002 2023-09-23T19:41:39.022 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.022 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.022 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:41:39.022 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.061896+0000", 2023-09-23T19:41:39.022 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0021113130000000001 2023-09-23T19:41:39.022 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.022 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.022 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:41:39.022 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.064007+0000", 2023-09-23T19:41:39.022 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.15940559000000001 2023-09-23T19:41:39.022 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.022 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.022 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:41:39.023 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.223412+0000", 2023-09-23T19:41:39.023 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.2522999999999998e-05 2023-09-23T19:41:39.023 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.023 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.023 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:41:39.023 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.223475+0000", 2023-09-23T19:41:39.023 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.7114999999999999e-05 2023-09-23T19:41:39.023 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.023 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.023 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:41:39.023 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:32:22.223522+0000", 2023-09-23T19:41:39.023 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.166731289 2023-09-23T19:41:39.023 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:41:39.023 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:41:39.024 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:41:39.024 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.024 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.024 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:208985 9.d 9:b5103cf5:::benchmark_data_smithi016_113233_object100603:head [delete] snapc 0=[] ondisk+write+known_if_redirected e137)", 2023-09-23T19:41:39.024 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:35:26.601224+0000", 2023-09-23T19:41:39.024 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 372.353334421, 2023-09-23T19:41:39.024 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.22539637300000001, 2023-09-23T19:41:39.024 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:41:39.024 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:41:39.024 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:41:39.024 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:41:39.024 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:41:39.024 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 208985 2023-09-23T19:41:39.025 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.025 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:41:39.025 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.025 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:41:39.025 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.601224+0000", 2023-09-23T19:41:39.025 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:41:39.025 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.025 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.025 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:41:39.025 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.601224+0000", 2023-09-23T19:41:39.025 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T19:41:39.025 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.025 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.025 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:41:39.026 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.601222+0000", 2023-09-23T19:41:39.026 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.7570000000000004e-06 2023-09-23T19:41:39.026 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.026 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.026 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:41:39.026 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.601230+0000", 2023-09-23T19:41:39.026 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.5673999999999999e-05 2023-09-23T19:41:39.026 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.026 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.026 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:41:39.026 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.601245+0000", 2023-09-23T19:41:39.026 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.6920000000000002e-06 2023-09-23T19:41:39.026 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.027 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.027 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:41:39.027 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.601253+0000", 2023-09-23T19:41:39.027 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.8211999999999998e-05 2023-09-23T19:41:39.027 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.027 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.027 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:41:39.027 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.601311+0000", 2023-09-23T19:41:39.027 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00018479299999999999 2023-09-23T19:41:39.027 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.027 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.027 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:41:39.027 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.601496+0000", 2023-09-23T19:41:39.027 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00022976100000000001 2023-09-23T19:41:39.028 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.028 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.028 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,5", 2023-09-23T19:41:39.028 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.601726+0000", 2023-09-23T19:41:39.028 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0043601200000000003 2023-09-23T19:41:39.028 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.028 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.028 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:41:39.028 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.606086+0000", 2023-09-23T19:41:39.028 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.023760499000000001 2023-09-23T19:41:39.028 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.028 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.028 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:41:39.028 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.629847+0000", 2023-09-23T19:41:39.029 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.196654314 2023-09-23T19:41:39.029 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.029 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.029 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:41:39.029 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.826501+0000", 2023-09-23T19:41:39.029 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.7210000000000002e-05 2023-09-23T19:41:39.029 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.029 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.029 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:41:39.029 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.826568+0000", 2023-09-23T19:41:39.029 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.1832000000000001e-05 2023-09-23T19:41:39.029 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.029 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.030 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:41:39.030 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.826620+0000", 2023-09-23T19:41:39.030 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.22539637300000001 2023-09-23T19:41:39.030 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:41:39.030 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:41:39.030 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:41:39.030 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.030 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.030 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:208989 9.d 9:bbab1653:::benchmark_data_smithi016_113233_object100607:head [delete] snapc 0=[] ondisk+write+known_if_redirected e137)", 2023-09-23T19:41:39.030 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:35:26.612950+0000", 2023-09-23T19:41:39.030 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 372.34160775499998, 2023-09-23T19:41:39.030 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.229970548, 2023-09-23T19:41:39.030 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:41:39.031 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:41:39.031 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:41:39.031 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:41:39.031 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:41:39.031 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 208989 2023-09-23T19:41:39.031 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.031 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:41:39.031 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.031 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:41:39.031 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.612950+0000", 2023-09-23T19:41:39.031 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:41:39.031 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.031 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.031 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:41:39.032 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.612950+0000", 2023-09-23T19:41:39.032 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T19:41:39.032 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.032 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.032 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:41:39.032 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.612948+0000", 2023-09-23T19:41:39.032 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.7009999999999999e-06 2023-09-23T19:41:39.032 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.032 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.032 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:41:39.032 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.612955+0000", 2023-09-23T19:41:39.032 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.1002000000000001e-05 2023-09-23T19:41:39.032 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.032 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.033 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:41:39.033 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.612966+0000", 2023-09-23T19:41:39.033 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.4650000000000002e-06 2023-09-23T19:41:39.033 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.033 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.033 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:41:39.033 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.612972+0000", 2023-09-23T19:41:39.033 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.002833009 2023-09-23T19:41:39.033 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.033 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.033 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:41:39.033 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.615805+0000", 2023-09-23T19:41:39.033 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00013731999999999999 2023-09-23T19:41:39.033 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.034 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.034 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:41:39.034 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.615942+0000", 2023-09-23T19:41:39.034 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00015651900000000001 2023-09-23T19:41:39.034 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.034 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.034 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,5", 2023-09-23T19:41:39.034 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.616098+0000", 2023-09-23T19:41:39.034 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.013827902 2023-09-23T19:41:39.034 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.034 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.034 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:41:39.034 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.629926+0000", 2023-09-23T19:41:39.035 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.6820999999999998e-05 2023-09-23T19:41:39.035 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.035 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.035 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:41:39.035 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.629973+0000", 2023-09-23T19:41:39.035 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.21284078400000001 2023-09-23T19:41:39.035 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.035 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.035 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:41:39.035 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.842814+0000", 2023-09-23T19:41:39.035 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.5655999999999997e-05 2023-09-23T19:41:39.035 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.035 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.035 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:41:39.036 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.842870+0000", 2023-09-23T19:41:39.036 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.1178999999999999e-05 2023-09-23T19:41:39.036 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.036 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.036 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:41:39.036 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.842921+0000", 2023-09-23T19:41:39.036 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.229970548 2023-09-23T19:41:39.036 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:41:39.036 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:41:39.036 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:41:39.036 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.036 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.036 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:208990 9.d 9:b24ecd90:::benchmark_data_smithi016_113233_object100608:head [delete] snapc 0=[] ondisk+write+known_if_redirected e137)", 2023-09-23T19:41:39.037 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:35:26.612982+0000", 2023-09-23T19:41:39.037 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 372.34157640500001, 2023-09-23T19:41:39.037 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.23009431799999999, 2023-09-23T19:41:39.037 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:41:39.037 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:41:39.037 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:41:39.037 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:41:39.037 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:41:39.037 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 208990 2023-09-23T19:41:39.037 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.037 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:41:39.037 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.037 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:41:39.037 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.612982+0000", 2023-09-23T19:41:39.038 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:41:39.038 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.038 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.038 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:41:39.038 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.612982+0000", 2023-09-23T19:41:39.038 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999995 2023-09-23T19:41:39.038 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.038 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.038 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:41:39.038 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.612981+0000", 2023-09-23T19:41:39.038 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.8519999999999997e-06 2023-09-23T19:41:39.038 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.038 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.039 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:41:39.039 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.612985+0000", 2023-09-23T19:41:39.039 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.5910000000000001e-06 2023-09-23T19:41:39.039 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.039 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.039 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:41:39.039 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.612992+0000", 2023-09-23T19:41:39.039 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.2320000000000003e-06 2023-09-23T19:41:39.039 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.039 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.039 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:41:39.039 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.612996+0000", 2023-09-23T19:41:39.039 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.012239991 2023-09-23T19:41:39.039 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.040 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.040 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:41:39.040 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.625236+0000", 2023-09-23T19:41:39.040 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00014660399999999999 2023-09-23T19:41:39.040 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.040 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.040 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:41:39.040 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.625382+0000", 2023-09-23T19:41:39.040 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000100065 2023-09-23T19:41:39.040 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.040 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.040 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,5", 2023-09-23T19:41:39.040 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.625482+0000", 2023-09-23T19:41:39.041 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0045393910000000003 2023-09-23T19:41:39.041 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.041 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.041 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:41:39.041 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.630022+0000", 2023-09-23T19:41:39.041 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.021801620000000001 2023-09-23T19:41:39.041 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.041 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.041 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:41:39.041 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.651823+0000", 2023-09-23T19:41:39.041 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.191152778 2023-09-23T19:41:39.041 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.041 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.041 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:41:39.042 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.842976+0000", 2023-09-23T19:41:39.042 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.7379999999999998e-05 2023-09-23T19:41:39.042 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.042 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.042 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:41:39.042 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.843014+0000", 2023-09-23T19:41:39.042 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.2272000000000003e-05 2023-09-23T19:41:39.042 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.042 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.042 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:41:39.042 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.843076+0000", 2023-09-23T19:41:39.042 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.23009431799999999 2023-09-23T19:41:39.042 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:41:39.042 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:41:39.043 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:41:39.043 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.043 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.043 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:208995 9.e 9:7a040566:::benchmark_data_smithi016_113233_object100613:head [delete] snapc 0=[] ondisk+write+known_if_redirected e137)", 2023-09-23T19:41:39.043 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:35:26.613044+0000", 2023-09-23T19:41:39.043 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 372.34151385799998, 2023-09-23T19:41:39.043 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.21365092299999999, 2023-09-23T19:41:39.043 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:41:39.043 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:41:39.043 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:41:39.043 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:41:39.043 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:41:39.043 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 208995 2023-09-23T19:41:39.044 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.044 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:41:39.044 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.044 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:41:39.044 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.613044+0000", 2023-09-23T19:41:39.044 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:41:39.044 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.044 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.044 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:41:39.044 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.613044+0000", 2023-09-23T19:41:39.044 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999995 2023-09-23T19:41:39.044 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.044 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.045 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:41:39.045 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.613044+0000", 2023-09-23T19:41:39.045 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.824e-06 2023-09-23T19:41:39.045 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.045 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.045 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:41:39.045 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.613046+0000", 2023-09-23T19:41:39.045 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.947e-06 2023-09-23T19:41:39.045 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.045 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.045 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:41:39.045 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.613052+0000", 2023-09-23T19:41:39.045 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.7330000000000001e-06 2023-09-23T19:41:39.045 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.046 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.046 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:41:39.046 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.613056+0000", 2023-09-23T19:41:39.046 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0050268300000000004 2023-09-23T19:41:39.046 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.046 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.046 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:41:39.046 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.618083+0000", 2023-09-23T19:41:39.046 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000125809 2023-09-23T19:41:39.046 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.046 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.046 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:41:39.047 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.618209+0000", 2023-09-23T19:41:39.047 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000142208 2023-09-23T19:41:39.047 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.047 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.047 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,4", 2023-09-23T19:41:39.047 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.618351+0000", 2023-09-23T19:41:39.047 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.013098810000000001 2023-09-23T19:41:39.047 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.047 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.047 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:41:39.047 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.631450+0000", 2023-09-23T19:41:39.047 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000119982 2023-09-23T19:41:39.047 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.047 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.048 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:41:39.048 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.631570+0000", 2023-09-23T19:41:39.048 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.19500669600000001 2023-09-23T19:41:39.048 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.048 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.048 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:41:39.048 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.826577+0000", 2023-09-23T19:41:39.048 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.5329999999999998e-05 2023-09-23T19:41:39.048 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.048 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.048 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:41:39.048 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.826662+0000", 2023-09-23T19:41:39.048 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.3200000000000001e-05 2023-09-23T19:41:39.049 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.049 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.049 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:41:39.049 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.826695+0000", 2023-09-23T19:41:39.049 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.21365092299999999 2023-09-23T19:41:39.049 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:41:39.049 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:41:39.049 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:41:39.049 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.049 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.049 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:209002 9.d 9:b8ac4f6d:::benchmark_data_smithi016_113233_object100620:head [delete] snapc 0=[] ondisk+write+known_if_redirected e137)", 2023-09-23T19:41:39.049 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:35:26.632989+0000", 2023-09-23T19:41:39.049 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 372.32156939599997, 2023-09-23T19:41:39.049 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.21023080699999999, 2023-09-23T19:41:39.050 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:41:39.050 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:41:39.050 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:41:39.050 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:41:39.050 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:41:39.050 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 209002 2023-09-23T19:41:39.050 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.050 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:41:39.050 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.050 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:41:39.050 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.632989+0000", 2023-09-23T19:41:39.050 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:41:39.050 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.051 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.051 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:41:39.051 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.632989+0000", 2023-09-23T19:41:39.051 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999976 2023-09-23T19:41:39.051 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.051 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.051 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:41:39.051 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.632986+0000", 2023-09-23T19:41:39.051 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.2043e-05 2023-09-23T19:41:39.051 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.051 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.051 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:41:39.051 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.632998+0000", 2023-09-23T19:41:39.051 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.6719e-05 2023-09-23T19:41:39.052 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.052 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.052 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:41:39.052 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.633015+0000", 2023-09-23T19:41:39.052 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.1560000000000001e-06 2023-09-23T19:41:39.052 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.052 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.052 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:41:39.052 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.633024+0000", 2023-09-23T19:41:39.052 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.1647000000000002e-05 2023-09-23T19:41:39.052 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.052 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.052 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:41:39.053 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.633076+0000", 2023-09-23T19:41:39.053 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000109858 2023-09-23T19:41:39.053 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.053 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.053 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:41:39.053 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.633185+0000", 2023-09-23T19:41:39.053 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00010092199999999999 2023-09-23T19:41:39.053 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.053 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.053 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,5", 2023-09-23T19:41:39.053 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.633286+0000", 2023-09-23T19:41:39.053 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0045832210000000002 2023-09-23T19:41:39.053 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.053 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.054 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:41:39.054 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.637870+0000", 2023-09-23T19:41:39.054 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.013985238000000001 2023-09-23T19:41:39.054 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.054 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.054 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:41:39.054 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.651855+0000", 2023-09-23T19:41:39.054 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.191273893 2023-09-23T19:41:39.054 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.054 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.054 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:41:39.054 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.843129+0000", 2023-09-23T19:41:39.054 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.9422999999999998e-05 2023-09-23T19:41:39.055 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.055 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.055 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:41:39.055 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.843178+0000", 2023-09-23T19:41:39.055 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.1186999999999997e-05 2023-09-23T19:41:39.055 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.055 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.055 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:41:39.055 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.843219+0000", 2023-09-23T19:41:39.055 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.21023080699999999 2023-09-23T19:41:39.055 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:41:39.055 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:41:39.055 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:41:39.055 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.056 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.056 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:209010 9.e 9:7c5d9b11:::benchmark_data_smithi016_113233_object100628:head [delete] snapc 0=[] ondisk+write+known_if_redirected e137)", 2023-09-23T19:41:39.056 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:35:26.647717+0000", 2023-09-23T19:41:39.056 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 372.30684078399997, 2023-09-23T19:41:39.056 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.21077024899999999, 2023-09-23T19:41:39.056 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:41:39.056 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:41:39.056 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:41:39.056 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:41:39.056 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:41:39.056 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 209010 2023-09-23T19:41:39.056 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.057 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:41:39.057 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.057 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:41:39.057 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.647717+0000", 2023-09-23T19:41:39.057 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:41:39.057 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.057 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.057 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:41:39.057 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.647717+0000", 2023-09-23T19:41:39.057 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999976 2023-09-23T19:41:39.057 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.057 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.057 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:41:39.057 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.647715+0000", 2023-09-23T19:41:39.058 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.8609999999999993e-06 2023-09-23T19:41:39.058 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.058 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.058 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:41:39.058 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.647725+0000", 2023-09-23T19:41:39.058 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.2419e-05 2023-09-23T19:41:39.058 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.058 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.058 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:41:39.058 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.647747+0000", 2023-09-23T19:41:39.058 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.7399999999999993e-06 2023-09-23T19:41:39.058 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.058 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.059 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:41:39.059 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.647756+0000", 2023-09-23T19:41:39.059 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.2290000000000001e-05 2023-09-23T19:41:39.059 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.059 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.059 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:41:39.059 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.647828+0000", 2023-09-23T19:41:39.059 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000134091 2023-09-23T19:41:39.059 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.059 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.059 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:41:39.059 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.647962+0000", 2023-09-23T19:41:39.059 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00016182300000000001 2023-09-23T19:41:39.059 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.060 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.060 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,4", 2023-09-23T19:41:39.060 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.648124+0000", 2023-09-23T19:41:39.060 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.010215615000000001 2023-09-23T19:41:39.060 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.060 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.060 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:41:39.060 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.658340+0000", 2023-09-23T19:41:39.060 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0045123359999999996 2023-09-23T19:41:39.060 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.060 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.060 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:41:39.060 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.662852+0000", 2023-09-23T19:41:39.061 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.19550694599999999 2023-09-23T19:41:39.061 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.061 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.061 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:41:39.061 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.858359+0000", 2023-09-23T19:41:39.061 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.7562000000000001e-05 2023-09-23T19:41:39.061 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.061 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.061 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:41:39.061 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.858427+0000", 2023-09-23T19:41:39.061 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.0812e-05 2023-09-23T19:41:39.061 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.061 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.062 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:41:39.062 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.858487+0000", 2023-09-23T19:41:39.062 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.21077024899999999 2023-09-23T19:41:39.062 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:41:39.062 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:41:39.062 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:41:39.062 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.062 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.062 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:209012 9.3 9:c8071fe1:::benchmark_data_smithi016_113233_object100630:head [delete] snapc 0=[] ondisk+write+known_if_redirected e137)", 2023-09-23T19:41:39.062 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:35:26.654569+0000", 2023-09-23T19:41:39.062 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 372.29998925500001, 2023-09-23T19:41:39.062 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.18387530799999999, 2023-09-23T19:41:39.062 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:41:39.063 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:41:39.063 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:41:39.063 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:41:39.063 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:41:39.063 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 209012 2023-09-23T19:41:39.063 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.063 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:41:39.063 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.063 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:41:39.063 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.654569+0000", 2023-09-23T19:41:39.063 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:41:39.063 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.063 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.063 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:41:39.064 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.654569+0000", 2023-09-23T19:41:39.064 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999914 2023-09-23T19:41:39.064 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.064 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.064 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:41:39.064 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.654560+0000", 2023-09-23T19:41:39.064 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.9383999999999999e-05 2023-09-23T19:41:39.064 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.064 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.064 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:41:39.064 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.654579+0000", 2023-09-23T19:41:39.064 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.4882e-05 2023-09-23T19:41:39.065 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.065 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.065 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:41:39.065 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.654604+0000", 2023-09-23T19:41:39.065 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.9680000000000001e-06 2023-09-23T19:41:39.065 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.065 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.065 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:41:39.065 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.654614+0000", 2023-09-23T19:41:39.065 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.1459999999999998e-05 2023-09-23T19:41:39.065 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.065 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.066 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:41:39.066 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.654676+0000", 2023-09-23T19:41:39.066 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00013746799999999999 2023-09-23T19:41:39.066 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.066 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.066 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:41:39.066 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.654813+0000", 2023-09-23T19:41:39.066 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000150452 2023-09-23T19:41:39.066 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.066 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.066 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,3", 2023-09-23T19:41:39.066 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.654964+0000", 2023-09-23T19:41:39.066 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0045507359999999997 2023-09-23T19:41:39.067 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.067 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.067 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:41:39.067 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.659514+0000", 2023-09-23T19:41:39.067 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00013708599999999999 2023-09-23T19:41:39.067 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.067 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.067 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:41:39.067 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.659651+0000", 2023-09-23T19:41:39.067 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.17867601899999999 2023-09-23T19:41:39.067 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.067 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.067 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:41:39.068 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.838327+0000", 2023-09-23T19:41:39.068 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.2668000000000004e-05 2023-09-23T19:41:39.068 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.068 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.068 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:41:39.068 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.838390+0000", 2023-09-23T19:41:39.068 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.3909000000000002e-05 2023-09-23T19:41:39.068 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.068 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.068 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:41:39.068 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.838444+0000", 2023-09-23T19:41:39.068 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.18387530799999999 2023-09-23T19:41:39.068 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:41:39.069 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:41:39.069 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:41:39.069 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.069 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.069 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:209015 9.e 9:7873cd13:::benchmark_data_smithi016_113233_object100633:head [delete] snapc 0=[] ondisk+write+known_if_redirected e137)", 2023-09-23T19:41:39.069 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:35:26.658215+0000", 2023-09-23T19:41:39.069 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 372.29634272800001, 2023-09-23T19:41:39.069 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.20040670799999999, 2023-09-23T19:41:39.069 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:41:39.069 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:41:39.069 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:41:39.069 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:41:39.069 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:41:39.070 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 209015 2023-09-23T19:41:39.070 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.070 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:41:39.070 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.070 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:41:39.070 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.658215+0000", 2023-09-23T19:41:39.070 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:41:39.070 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.070 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.070 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:41:39.070 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.658215+0000", 2023-09-23T19:41:39.070 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T19:41:39.071 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.071 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.071 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:41:39.071 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.658213+0000", 2023-09-23T19:41:39.071 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.055e-06 2023-09-23T19:41:39.071 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.071 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.071 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:41:39.071 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.658221+0000", 2023-09-23T19:41:39.071 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.3132999999999999e-05 2023-09-23T19:41:39.071 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.071 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.071 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:41:39.071 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.658234+0000", 2023-09-23T19:41:39.072 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.0929999999999997e-06 2023-09-23T19:41:39.072 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.072 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.072 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:41:39.072 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.658242+0000", 2023-09-23T19:41:39.072 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000118411 2023-09-23T19:41:39.072 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.072 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.072 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:41:39.072 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.658360+0000", 2023-09-23T19:41:39.072 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.7691999999999999e-05 2023-09-23T19:41:39.072 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.072 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.073 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:41:39.073 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.658458+0000", 2023-09-23T19:41:39.073 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000115229 2023-09-23T19:41:39.073 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.073 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.073 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,4", 2023-09-23T19:41:39.073 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.658573+0000", 2023-09-23T19:41:39.073 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0045246649999999998 2023-09-23T19:41:39.073 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.073 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.073 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:41:39.073 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.663098+0000", 2023-09-23T19:41:39.073 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00023705899999999999 2023-09-23T19:41:39.073 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.074 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.074 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:41:39.074 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.663335+0000", 2023-09-23T19:41:39.074 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.19522404500000001 2023-09-23T19:41:39.074 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.074 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.074 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:41:39.074 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.858559+0000", 2023-09-23T19:41:39.074 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.4736000000000001e-05 2023-09-23T19:41:39.074 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.074 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.074 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:41:39.075 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.858593+0000", 2023-09-23T19:41:39.075 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.8591000000000001e-05 2023-09-23T19:41:39.075 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.075 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.075 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:41:39.075 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.858622+0000", 2023-09-23T19:41:39.075 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.20040670799999999 2023-09-23T19:41:39.075 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:41:39.075 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:41:39.075 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:41:39.075 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.075 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.075 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45671.0:209018 9.e 9:78e76208:::benchmark_data_smithi016_113233_object100636:head [delete] snapc 0=[] ondisk+write+known_if_redirected e137)", 2023-09-23T19:41:39.076 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:35:26.672023+0000", 2023-09-23T19:41:39.076 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 372.28253479799997, 2023-09-23T19:41:39.076 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.18676025900000001, 2023-09-23T19:41:39.076 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:41:39.076 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:41:39.076 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:41:39.076 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45671", 2023-09-23T19:41:39.076 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:41:39.076 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 209018 2023-09-23T19:41:39.076 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.076 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:41:39.076 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.077 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:41:39.077 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.672023+0000", 2023-09-23T19:41:39.077 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:41:39.077 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.077 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.077 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:41:39.077 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.672023+0000", 2023-09-23T19:41:39.077 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T19:41:39.077 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.077 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.077 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:41:39.077 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.672021+0000", 2023-09-23T19:41:39.077 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.3939999999999996e-06 2023-09-23T19:41:39.077 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.078 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.078 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:41:39.078 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.672030+0000", 2023-09-23T19:41:39.078 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.8062000000000001e-05 2023-09-23T19:41:39.078 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.078 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.078 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:41:39.078 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.672048+0000", 2023-09-23T19:41:39.078 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.4009999999999997e-06 2023-09-23T19:41:39.078 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.078 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.078 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:41:39.079 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.672056+0000", 2023-09-23T19:41:39.079 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.8889999999999999e-05 2023-09-23T19:41:39.079 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.079 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.079 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:41:39.079 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.672125+0000", 2023-09-23T19:41:39.079 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000195603 2023-09-23T19:41:39.079 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.079 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.079 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:41:39.079 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.672321+0000", 2023-09-23T19:41:39.079 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00023059600000000001 2023-09-23T19:41:39.079 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.080 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.080 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,4", 2023-09-23T19:41:39.080 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.672551+0000", 2023-09-23T19:41:39.080 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.003564219 2023-09-23T19:41:39.080 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.080 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.080 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:41:39.080 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.676116+0000", 2023-09-23T19:41:39.080 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00057282399999999999 2023-09-23T19:41:39.080 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.080 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.080 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:41:39.080 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.676688+0000", 2023-09-23T19:41:39.081 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.18199246699999999 2023-09-23T19:41:39.081 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.081 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.081 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:41:39.081 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.858681+0000", 2023-09-23T19:41:39.081 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.7912999999999999e-05 2023-09-23T19:41:39.081 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.081 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.081 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:41:39.081 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.858739+0000", 2023-09-23T19:41:39.081 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.4705e-05 2023-09-23T19:41:39.081 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.081 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.082 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:41:39.082 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:35:26.858783+0000", 2023-09-23T19:41:39.082 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.18676025900000001 2023-09-23T19:41:39.082 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:41:39.082 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:41:39.082 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:41:39.082 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.082 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.082 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45947.0:207476 10.c 10:32805061:::benchmark_data_smithi016_136365_object97342:head [delete] snapc 0=[] ondisk+write+known_if_redirected e171)", 2023-09-23T19:41:39.082 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:40:28.095050+0000", 2023-09-23T19:41:39.082 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 70.859507832999995, 2023-09-23T19:41:39.082 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.26265255700000001, 2023-09-23T19:41:39.083 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:41:39.083 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:41:39.083 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:41:39.083 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45947", 2023-09-23T19:41:39.083 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/4039191752", 2023-09-23T19:41:39.083 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 207476 2023-09-23T19:41:39.083 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.083 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:41:39.083 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.083 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:41:39.083 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.095050+0000", 2023-09-23T19:41:39.083 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:41:39.083 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.084 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.084 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:41:39.084 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.095050+0000", 2023-09-23T19:41:39.084 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T19:41:39.084 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.084 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.084 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:41:39.084 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.095048+0000", 2023-09-23T19:41:39.084 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.7920000000000004e-06 2023-09-23T19:41:39.084 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.084 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.085 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:41:39.085 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.095058+0000", 2023-09-23T19:41:39.085 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.6631999999999999e-05 2023-09-23T19:41:39.085 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.085 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.085 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:41:39.085 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.095075+0000", 2023-09-23T19:41:39.085 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.0339999999999997e-06 2023-09-23T19:41:39.085 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.085 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.085 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:41:39.085 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.095083+0000", 2023-09-23T19:41:39.085 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.1377999999999994e-05 2023-09-23T19:41:39.086 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.086 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.086 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:41:39.086 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.095144+0000", 2023-09-23T19:41:39.086 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000116507 2023-09-23T19:41:39.086 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.086 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.086 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:41:39.086 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.095261+0000", 2023-09-23T19:41:39.086 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00013625599999999999 2023-09-23T19:41:39.086 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.086 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.087 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 0,4", 2023-09-23T19:41:39.087 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.095397+0000", 2023-09-23T19:41:39.087 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0044904259999999996 2023-09-23T19:41:39.087 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.087 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.087 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:41:39.087 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.099887+0000", 2023-09-23T19:41:39.087 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0090943039999999992 2023-09-23T19:41:39.087 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.087 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.087 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:41:39.087 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.108982+0000", 2023-09-23T19:41:39.088 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.24851672599999999 2023-09-23T19:41:39.088 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.088 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.088 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:41:39.088 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.357498+0000", 2023-09-23T19:41:39.088 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00014587500000000001 2023-09-23T19:41:39.088 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.088 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.088 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:41:39.088 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.357644+0000", 2023-09-23T19:41:39.088 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.8529999999999997e-05 2023-09-23T19:41:39.088 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:41:39.088 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:41:39.088 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:41:39.089 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.357703+0000", 2023-09-23T19:41:39.089 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.26265255700000001 2023-09-23T19:41:39.089 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:41:39.089 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:41:39.089 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:41:39.089 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:41:39.089 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:41:39.089 INFO:teuthology.orchestra.run.smithi092.stdout:} 2023-09-23T19:41:39.089 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:39.124 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:39.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:39.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:39.227 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:39.255 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:39.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:39.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:39.358 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:39.389 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:39.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:39.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:41:39.492 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:39.505 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:39 smithi092 bash[58855]: cluster 2023-09-23T19:41:38.197591+0000 mgr.y (mgr.25325) 1024 : cluster 0 pgmap v978: 89 pgs: 89 active+clean; 582 KiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2023-09-23T19:41:39.524 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:39.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:39.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:41:39.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:39 smithi016 bash[75004]: cluster 2023-09-23T19:41:38.197591+0000 mgr.y (mgr.25325) 1024 : cluster 0 pgmap v978: 89 pgs: 89 active+clean; 582 KiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2023-09-23T19:41:39.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:39 smithi016 bash[77995]: cluster 2023-09-23T19:41:38.197591+0000 mgr.y (mgr.25325) 1024 : cluster 0 pgmap v978: 89 pgs: 89 active+clean; 582 KiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2023-09-23T19:41:39.627 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:39.658 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:39.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:39.660 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:39.761 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:39.794 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:39.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:39.797 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:39.893 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T19:41:39.898 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:39.931 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:39.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:39.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:40.035 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:40.053 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:40.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:40.055 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:41:40.156 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:40.190 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:40.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:40.193 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:41:40.294 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:40.312 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:40.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:40.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:41:40.414 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:40.436 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:40.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:40.438 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:40.505 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.7 to 1 (10000) 2023-09-23T19:41:40.506 INFO:teuthology.orchestra.run.smithi016.stderr:enabled application 'rados' on pool 'unique_pool_6' 2023-09-23T19:41:40.539 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:40.570 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:40.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:40.574 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:40.594 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:40 smithi016 bash[77995]: audit 2023-09-23T19:41:39.993520+0000 mon.a (mon.0) 1319 : audit 1 from='client.? 172.21.15.16:0/573014355' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T19:41:40.594 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:40 smithi016 bash[77995]: audit 2023-09-23T19:41:40.474938+0000 mon.c (mon.2) 179 : audit 1 from='client.? 172.21.15.16:0/2395643415' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:41:40.594 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:40 smithi016 bash[77995]: audit 2023-09-23T19:41:40.475663+0000 mon.a (mon.0) 1320 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:41:40.594 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:40 smithi016 bash[75004]: audit 2023-09-23T19:41:39.993520+0000 mon.a (mon.0) 1319 : audit 1 from='client.? 172.21.15.16:0/573014355' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T19:41:40.594 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:40 smithi016 bash[75004]: audit 2023-09-23T19:41:40.474938+0000 mon.c (mon.2) 179 : audit 1 from='client.? 172.21.15.16:0/2395643415' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:41:40.594 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:40 smithi016 bash[75004]: audit 2023-09-23T19:41:40.475663+0000 mon.a (mon.0) 1320 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:41:40.675 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:40.707 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:40.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:40.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:40.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:40 smithi092 bash[58855]: audit 2023-09-23T19:41:39.993520+0000 mon.a (mon.0) 1319 : audit 1 from='client.? 172.21.15.16:0/573014355' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T19:41:40.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:40 smithi092 bash[58855]: audit 2023-09-23T19:41:40.474938+0000 mon.c (mon.2) 179 : audit 1 from='client.? 172.21.15.16:0/2395643415' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:41:40.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:40 smithi092 bash[58855]: audit 2023-09-23T19:41:40.475663+0000 mon.a (mon.0) 1320 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:41:40.811 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:40.841 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:40.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:40.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:40.944 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:40.977 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:40.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:40.980 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:41.081 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:41.103 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:41.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:41.106 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:41.206 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:41.237 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:41.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:41.240 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:41.341 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:41.372 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:41.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:41.375 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:41.475 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:41.508 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:41.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:41.511 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:41.612 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:41.623 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:41 smithi016 bash[77995]: cluster 2023-09-23T19:41:40.198389+0000 mgr.y (mgr.25325) 1025 : cluster 0 pgmap v979: 89 pgs: 89 active+clean; 582 KiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2023-09-23T19:41:41.623 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:41 smithi016 bash[77995]: audit 2023-09-23T19:41:40.495799+0000 mon.a (mon.0) 1321 : audit 1 from='client.? 172.21.15.16:0/573014355' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T19:41:41.623 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:41 smithi016 bash[77995]: audit 2023-09-23T19:41:40.495873+0000 mon.a (mon.0) 1322 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T19:41:41.623 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:41 smithi016 bash[77995]: cluster 2023-09-23T19:41:40.495933+0000 mon.a (mon.0) 1323 : cluster 0 osdmap e184: 8 total, 8 up, 8 in 2023-09-23T19:41:41.623 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:41 smithi016 bash[77995]: cluster 2023-09-23T19:41:40.499491+0000 osd.6 (osd.6) 106 : cluster 3 failed to encode map e184 with expected crc 2023-09-23T19:41:41.623 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:41 smithi016 bash[77995]: cluster 2023-09-23T19:41:40.568473+0000 osd.7 (osd.7) 95 : cluster 3 failed to encode map e184 with expected crc 2023-09-23T19:41:41.623 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:41 smithi016 bash[77995]: cluster 2023-09-23T19:41:41.006678+0000 osd.5 (osd.5) 114 : cluster 3 failed to encode map e184 with expected crc 2023-09-23T19:41:41.624 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:41 smithi016 bash[75004]: cluster 2023-09-23T19:41:40.198389+0000 mgr.y (mgr.25325) 1025 : cluster 0 pgmap v979: 89 pgs: 89 active+clean; 582 KiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2023-09-23T19:41:41.624 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:41 smithi016 bash[75004]: audit 2023-09-23T19:41:40.495799+0000 mon.a (mon.0) 1321 : audit 1 from='client.? 172.21.15.16:0/573014355' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T19:41:41.624 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:41 smithi016 bash[75004]: audit 2023-09-23T19:41:40.495873+0000 mon.a (mon.0) 1322 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T19:41:41.624 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:41 smithi016 bash[75004]: cluster 2023-09-23T19:41:40.495933+0000 mon.a (mon.0) 1323 : cluster 0 osdmap e184: 8 total, 8 up, 8 in 2023-09-23T19:41:41.624 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:41 smithi016 bash[75004]: cluster 2023-09-23T19:41:40.499491+0000 osd.6 (osd.6) 106 : cluster 3 failed to encode map e184 with expected crc 2023-09-23T19:41:41.624 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:41 smithi016 bash[75004]: cluster 2023-09-23T19:41:40.568473+0000 osd.7 (osd.7) 95 : cluster 3 failed to encode map e184 with expected crc 2023-09-23T19:41:41.624 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:41 smithi016 bash[75004]: cluster 2023-09-23T19:41:41.006678+0000 osd.5 (osd.5) 114 : cluster 3 failed to encode map e184 with expected crc 2023-09-23T19:41:41.637 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:41.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:41.639 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:41.740 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:41.762 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:41.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:41.764 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:41:41.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:41 smithi092 bash[58855]: cluster 2023-09-23T19:41:40.198389+0000 mgr.y (mgr.25325) 1025 : cluster 0 pgmap v979: 89 pgs: 89 active+clean; 582 KiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2023-09-23T19:41:41.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:41 smithi092 bash[58855]: audit 2023-09-23T19:41:40.495799+0000 mon.a (mon.0) 1321 : audit 1 from='client.? 172.21.15.16:0/573014355' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T19:41:41.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:41 smithi092 bash[58855]: audit 2023-09-23T19:41:40.495873+0000 mon.a (mon.0) 1322 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T19:41:41.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:41 smithi092 bash[58855]: cluster 2023-09-23T19:41:40.495933+0000 mon.a (mon.0) 1323 : cluster 0 osdmap e184: 8 total, 8 up, 8 in 2023-09-23T19:41:41.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:41 smithi092 bash[58855]: cluster 2023-09-23T19:41:40.499491+0000 osd.6 (osd.6) 106 : cluster 3 failed to encode map e184 with expected crc 2023-09-23T19:41:41.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:41 smithi092 bash[58855]: cluster 2023-09-23T19:41:40.568473+0000 osd.7 (osd.7) 95 : cluster 3 failed to encode map e184 with expected crc 2023-09-23T19:41:41.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:41 smithi092 bash[58855]: cluster 2023-09-23T19:41:41.006678+0000 osd.5 (osd.5) 114 : cluster 3 failed to encode map e184 with expected crc 2023-09-23T19:41:41.865 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:41.896 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:41.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:41.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:42.000 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:42.028 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:42.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:42.031 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:42.131 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:42.165 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:42.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:42.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:42.268 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:42.300 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:42.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:42.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:42.403 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:42.430 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:42.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:42.433 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:42.534 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:42.569 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:42.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:42.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:42.673 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:42.704 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:42.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:42.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:42.807 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:42.842 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:42.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:42.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:41:42.945 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:42.970 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:42.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:42.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:41:43.073 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:43.106 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:43.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:43.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:41:43.147 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T19:41:43.147 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T19:41:43.210 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:43.244 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:43.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:43.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:43.347 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:43.378 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:43.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:43.381 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:41:43.482 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:43.512 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:43.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:43.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:43.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:43 smithi016 bash[77995]: cluster 2023-09-23T19:41:42.199192+0000 mgr.y (mgr.25325) 1026 : cluster 0 pgmap v981: 89 pgs: 89 active+clean; 582 KiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2023-09-23T19:41:43.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:43 smithi016 bash[75004]: cluster 2023-09-23T19:41:42.199192+0000 mgr.y (mgr.25325) 1026 : cluster 0 pgmap v981: 89 pgs: 89 active+clean; 582 KiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2023-09-23T19:41:43.617 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:43.645 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:43.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:43.648 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:41:43.749 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:43.773 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:43.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:43.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:41:43.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:43 smithi092 bash[58855]: cluster 2023-09-23T19:41:42.199192+0000 mgr.y (mgr.25325) 1026 : cluster 0 pgmap v981: 89 pgs: 89 active+clean; 582 KiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2023-09-23T19:41:43.876 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:43.908 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:43.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:43.912 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:44.012 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:44.034 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:44.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:44.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:41:44.129 DEBUG:teuthology.orchestra.run.smithi016:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_6 bench 90 write' 2023-09-23T19:41:44.131 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T19:41:44.131 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T19:41:44.137 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:44.160 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:44.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:44.162 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:44.219 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:hints = 1 2023-09-23T19:41:44.219 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T19:41:44.219 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Object prefix: benchmark_data_smithi016_160148 2023-09-23T19:41:44.219 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:41:44.220 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 0 0 0 0 0 0 - 0 2023-09-23T19:41:44.263 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:44.288 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:44.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:44.291 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:44.392 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:44.425 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:44.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:44.427 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:44.528 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:44.560 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:44.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:44.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:41:44.663 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:44.702 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:44.706 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:44 smithi016 bash[77995]: audit 2023-09-23T19:41:43.660453+0000 mon.a (mon.0) 1324 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:41:44.707 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:44 smithi016 bash[75004]: audit 2023-09-23T19:41:43.660453+0000 mon.a (mon.0) 1324 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:41:44.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:44.707 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:41:44.777 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:41:44.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:44 smithi092 bash[58855]: audit 2023-09-23T19:41:43.660453+0000 mon.a (mon.0) 1324 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:41:44.808 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:44.835 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:44.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:44.842 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:44.851 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:41:44.917 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:41:44.943 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:44.971 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:44.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:44.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:44.975 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:41:45.028 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:41:45.068 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:41:45.074 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:45.103 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:45.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:45.105 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:45.110 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:41:45.147 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:41:45.153 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:41:45.153 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:41:45.153 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:41:45.153 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:41:45.153 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:41:45.154 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:41:45.154 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:41:45.154 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:41:45.206 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:45.220 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 1 16 2473 2457 153.554 153.562 0.0109533 0.00645748 2023-09-23T19:41:45.234 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:45.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:45.238 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:41:45.339 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:45.379 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:45.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:45.386 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:45.486 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:45.508 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:45.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:45.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:45.610 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:45.637 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:45.639 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:45 smithi016 bash[75004]: cluster 2023-09-23T19:41:44.199934+0000 mgr.y (mgr.25325) 1027 : cluster 0 pgmap v982: 89 pgs: 89 active+clean; 582 KiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2023-09-23T19:41:45.639 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:45 smithi016 bash[75004]: audit 2023-09-23T19:41:44.706670+0000 mon.c (mon.2) 180 : audit 0 from='client.? 172.21.15.16:0/4261441495' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:41:45.639 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:45 smithi016 bash[77995]: cluster 2023-09-23T19:41:44.199934+0000 mgr.y (mgr.25325) 1027 : cluster 0 pgmap v982: 89 pgs: 89 active+clean; 582 KiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2023-09-23T19:41:45.639 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:45 smithi016 bash[77995]: audit 2023-09-23T19:41:44.706670+0000 mon.c (mon.2) 180 : audit 0 from='client.? 172.21.15.16:0/4261441495' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:41:45.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:45.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:45.741 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:45.750 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:45 smithi092 bash[58855]: cluster 2023-09-23T19:41:44.199934+0000 mgr.y (mgr.25325) 1027 : cluster 0 pgmap v982: 89 pgs: 89 active+clean; 582 KiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2023-09-23T19:41:45.751 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:45 smithi092 bash[58855]: audit 2023-09-23T19:41:44.706670+0000 mon.c (mon.2) 180 : audit 0 from='client.? 172.21.15.16:0/4261441495' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:41:45.768 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:45.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:45.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:45.786 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:41:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:41:45] "GET /metrics HTTP/1.1" 200 36239 "" "Prometheus/2.33.4" 2023-09-23T19:41:45.872 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:45.909 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:45.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:45.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:41:46.012 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:46.040 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:46.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:46.043 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:46.143 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:46.170 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:46.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:46.173 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:46.220 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 2 16 5111 5095 159.207 164.875 0.0041723 0.00626668 2023-09-23T19:41:46.274 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:46.304 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:46.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:46.307 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:46.408 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:46.438 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:46.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:46.441 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:46.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:46 smithi016 bash[75004]: audit 2023-09-23T19:41:45.546667+0000 mon.c (mon.2) 181 : audit 1 from='client.? 172.21.15.16:0/3359408749' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:41:46.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:46 smithi016 bash[75004]: audit 2023-09-23T19:41:45.547353+0000 mon.a (mon.0) 1325 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:41:46.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:46 smithi016 bash[77995]: audit 2023-09-23T19:41:45.546667+0000 mon.c (mon.2) 181 : audit 1 from='client.? 172.21.15.16:0/3359408749' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:41:46.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:46 smithi016 bash[77995]: audit 2023-09-23T19:41:45.547353+0000 mon.a (mon.0) 1325 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:41:46.536 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is unset 2023-09-23T19:41:46.542 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:46.577 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:46.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:46.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:41:46.682 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:46.712 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:46.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:46.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:46.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:46 smithi092 bash[58855]: audit 2023-09-23T19:41:45.546667+0000 mon.c (mon.2) 181 : audit 1 from='client.? 172.21.15.16:0/3359408749' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:41:46.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:46 smithi092 bash[58855]: audit 2023-09-23T19:41:45.547353+0000 mon.a (mon.0) 1325 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:41:46.816 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:46.836 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:46.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:46.838 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:46.938 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:46.973 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:46.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:46.977 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:47.078 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:47.102 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:47.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:47.109 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:47.209 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:47.220 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 3 16 7265 7249 151.008 134.625 0.00431513 0.00660872 2023-09-23T19:41:47.237 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:47.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:47.241 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:41:47.342 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:47.372 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:47.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:47.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:41:47.476 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:47.503 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:47.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:47.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:41:47.606 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:47.637 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:47 smithi016 bash[75004]: cluster 2023-09-23T19:41:46.200574+0000 mgr.y (mgr.25325) 1028 : cluster 0 pgmap v983: 89 pgs: 89 active+clean; 137 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 14 MiB/s wr, 217 op/s 2023-09-23T19:41:47.637 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:47 smithi016 bash[75004]: cluster 2023-09-23T19:41:46.515956+0000 mon.a (mon.0) 1326 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:41:47.637 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:47 smithi016 bash[75004]: cluster 2023-09-23T19:41:46.516003+0000 mon.a (mon.0) 1327 : cluster 1 Cluster is now healthy 2023-09-23T19:41:47.638 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:47 smithi016 bash[75004]: audit 2023-09-23T19:41:46.526676+0000 mon.a (mon.0) 1328 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:41:47.638 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:47 smithi016 bash[75004]: cluster 2023-09-23T19:41:46.526771+0000 mon.a (mon.0) 1329 : cluster 0 osdmap e185: 8 total, 8 up, 8 in 2023-09-23T19:41:47.638 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:47 smithi016 bash[75004]: cluster 2023-09-23T19:41:46.528945+0000 osd.5 (osd.5) 115 : cluster 3 failed to encode map e185 with expected crc 2023-09-23T19:41:47.638 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:47 smithi016 bash[75004]: cluster 2023-09-23T19:41:46.560357+0000 osd.4 (osd.4) 123 : cluster 3 failed to encode map e185 with expected crc 2023-09-23T19:41:47.638 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:47 smithi016 bash[77995]: cluster 2023-09-23T19:41:46.200574+0000 mgr.y (mgr.25325) 1028 : cluster 0 pgmap v983: 89 pgs: 89 active+clean; 137 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 14 MiB/s wr, 217 op/s 2023-09-23T19:41:47.638 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:47 smithi016 bash[77995]: cluster 2023-09-23T19:41:46.515956+0000 mon.a (mon.0) 1326 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:41:47.638 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:47 smithi016 bash[77995]: cluster 2023-09-23T19:41:46.516003+0000 mon.a (mon.0) 1327 : cluster 1 Cluster is now healthy 2023-09-23T19:41:47.638 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:47 smithi016 bash[77995]: audit 2023-09-23T19:41:46.526676+0000 mon.a (mon.0) 1328 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:41:47.638 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:47 smithi016 bash[77995]: cluster 2023-09-23T19:41:46.526771+0000 mon.a (mon.0) 1329 : cluster 0 osdmap e185: 8 total, 8 up, 8 in 2023-09-23T19:41:47.638 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:47 smithi016 bash[77995]: cluster 2023-09-23T19:41:46.528945+0000 osd.5 (osd.5) 115 : cluster 3 failed to encode map e185 with expected crc 2023-09-23T19:41:47.638 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:47 smithi016 bash[77995]: cluster 2023-09-23T19:41:46.560357+0000 osd.4 (osd.4) 123 : cluster 3 failed to encode map e185 with expected crc 2023-09-23T19:41:47.641 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:47.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:47.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:47.747 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:47.756 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:47 smithi092 bash[58855]: cluster 2023-09-23T19:41:46.200574+0000 mgr.y (mgr.25325) 1028 : cluster 0 pgmap v983: 89 pgs: 89 active+clean; 137 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 14 MiB/s wr, 217 op/s 2023-09-23T19:41:47.756 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:47 smithi092 bash[58855]: cluster 2023-09-23T19:41:46.515956+0000 mon.a (mon.0) 1326 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:41:47.756 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:47 smithi092 bash[58855]: cluster 2023-09-23T19:41:46.516003+0000 mon.a (mon.0) 1327 : cluster 1 Cluster is now healthy 2023-09-23T19:41:47.756 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:47 smithi092 bash[58855]: audit 2023-09-23T19:41:46.526676+0000 mon.a (mon.0) 1328 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:41:47.756 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:47 smithi092 bash[58855]: cluster 2023-09-23T19:41:46.526771+0000 mon.a (mon.0) 1329 : cluster 0 osdmap e185: 8 total, 8 up, 8 in 2023-09-23T19:41:47.757 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:47 smithi092 bash[58855]: cluster 2023-09-23T19:41:46.528945+0000 osd.5 (osd.5) 115 : cluster 3 failed to encode map e185 with expected crc 2023-09-23T19:41:47.757 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:47 smithi092 bash[58855]: cluster 2023-09-23T19:41:46.560357+0000 osd.4 (osd.4) 123 : cluster 3 failed to encode map e185 with expected crc 2023-09-23T19:41:47.773 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:47.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:47.776 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:47.877 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:47.908 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:47.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:47.911 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:41:48.012 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:48.044 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:48.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:48.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:48.149 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:48.173 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:48.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:48.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:41:48.220 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 4 16 9777 9761 152.502 157 0.00841798 0.006547 2023-09-23T19:41:48.276 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:48.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:48.315 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:48.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:48.416 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:48.440 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:48.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:48.444 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:48.544 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:48.571 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:48.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:48.574 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:41:48.674 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:48.708 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:48.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:48.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:48.802 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:41:48.812 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:48.839 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:48.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:48.841 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:48.942 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:48.970 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:48.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:48.974 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:49.075 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:49.101 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:49.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:49.105 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:49.137 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:41:49.206 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:49.220 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 5 16 12062 12046 150.561 142.812 0.0142755 0.00663339 2023-09-23T19:41:49.241 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:49.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:49.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:41:49.344 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:49.369 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:49.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:49.373 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:49.473 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:49.499 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:49.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:49.502 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:41:49.603 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:49.626 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:49.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:49.631 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:49.731 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:49.740 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:49 smithi016 bash[77995]: cluster 2023-09-23T19:41:48.201017+0000 mgr.y (mgr.25325) 1029 : cluster 0 pgmap v985: 89 pgs: 89 active+clean; 190 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 24 MiB/s wr, 378 op/s 2023-09-23T19:41:49.741 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:49 smithi016 bash[75004]: cluster 2023-09-23T19:41:48.201017+0000 mgr.y (mgr.25325) 1029 : cluster 0 pgmap v985: 89 pgs: 89 active+clean; 190 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 24 MiB/s wr, 378 op/s 2023-09-23T19:41:49.757 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:49.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:49.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:41:49.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:49 smithi092 bash[58855]: cluster 2023-09-23T19:41:48.201017+0000 mgr.y (mgr.25325) 1029 : cluster 0 pgmap v985: 89 pgs: 89 active+clean; 190 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 24 MiB/s wr, 378 op/s 2023-09-23T19:41:49.862 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:49.889 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:49.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:49.893 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:49.994 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:50.021 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:50.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:50.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:50.125 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:50.155 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:50.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:50.158 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:41:50.220 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 6 16 14225 14209 147.997 135.188 0.00368356 0.00674804 2023-09-23T19:41:50.259 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:50.286 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:50.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:50.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:50.320 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T19:41:50.390 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:50.417 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:50.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:50.420 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:50.520 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:50.546 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:50.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:50.549 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:50.650 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:50.690 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:50.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:50.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:50.793 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:50.821 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:50.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:50.825 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:41:50.842 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:41:50.926 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:50.949 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:50.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:50.953 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:51.054 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:51.085 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:51.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:51.088 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:51.188 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:51.215 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:51.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:51.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:41:51.220 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 7 16 16343 16327 145.763 132.375 0.00924073 0.00685392 2023-09-23T19:41:51.320 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:51.344 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:51.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:51.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:41:51.448 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:51.473 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:51.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:51.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:51.575 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:51.613 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:51.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:51.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:51.716 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:51.726 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:51 smithi092 bash[58855]: cluster 2023-09-23T19:41:50.201651+0000 mgr.y (mgr.25325) 1030 : cluster 0 pgmap v986: 89 pgs: 89 active+clean; 463 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 48 MiB/s wr, 761 op/s 2023-09-23T19:41:51.743 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:51.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:51.745 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:41:51.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:51 smithi016 bash[75004]: cluster 2023-09-23T19:41:50.201651+0000 mgr.y (mgr.25325) 1030 : cluster 0 pgmap v986: 89 pgs: 89 active+clean; 463 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 48 MiB/s wr, 761 op/s 2023-09-23T19:41:51.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:51 smithi016 bash[77995]: cluster 2023-09-23T19:41:50.201651+0000 mgr.y (mgr.25325) 1030 : cluster 0 pgmap v986: 89 pgs: 89 active+clean; 463 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 48 MiB/s wr, 761 op/s 2023-09-23T19:41:51.846 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:51.873 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:51.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:51.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:51.976 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:52.006 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:52.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:52.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:52.110 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:52.134 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:52.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:52.138 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:41:52.220 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 8 16 18469 18453 144.151 132.875 0.00892222 0.0069284 2023-09-23T19:41:52.239 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:52.268 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:52.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:52.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:52.371 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:52.396 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:52.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:52.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:52.500 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:52.533 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:52.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:52.536 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:52.637 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:52.678 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:52.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:52.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:41:52.783 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:52.818 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:52.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:52.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:52.922 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:52.943 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:52.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:52.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:53.045 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:53.070 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:53.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:53.074 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:53.174 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:53.203 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:53.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:53.206 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:41:53.220 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 9 16 20452 20436 141.904 123.938 0.00599429 0.0070403 2023-09-23T19:41:53.307 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:53.333 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:53.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:53.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:53.439 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:53.478 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:53.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:53.481 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:41:53.581 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:53.605 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:53.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:53.608 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:41:53.708 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:53.736 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:53.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:53.739 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:53.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:53 smithi016 bash[77995]: cluster 2023-09-23T19:41:52.202451+0000 mgr.y (mgr.25325) 1031 : cluster 0 pgmap v987: 89 pgs: 89 active+clean; 913 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-09-23T19:41:53.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:53 smithi016 bash[75004]: cluster 2023-09-23T19:41:52.202451+0000 mgr.y (mgr.25325) 1031 : cluster 0 pgmap v987: 89 pgs: 89 active+clean; 913 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-09-23T19:41:53.839 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:53.877 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:53.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:53.881 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:41:53.982 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:53.991 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:53 smithi092 bash[58855]: cluster 2023-09-23T19:41:52.202451+0000 mgr.y (mgr.25325) 1031 : cluster 0 pgmap v987: 89 pgs: 89 active+clean; 913 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-09-23T19:41:54.010 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:54.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:54.013 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:54.114 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:54.141 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:54.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:54.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:54.221 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 10 16 22450 22434 140.2 124.875 0.0118061 0.0071265 2023-09-23T19:41:54.247 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:54.271 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:54.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:54.274 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:41:54.375 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:54.403 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:54.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:54.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:54.507 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:54.540 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:54.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:54.544 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:41:54.644 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:54.672 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:54.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:54.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:54.775 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:54.802 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:54.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:54.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:54.905 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:54.932 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:54.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:54.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:41:55.036 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:55.064 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:55.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:55.069 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:55.121 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:41:55.170 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:55.215 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:55.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:55.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:55.221 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 11 16 24396 24380 138.51 121.625 0.00689613 0.00721265 2023-09-23T19:41:55.320 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:55.347 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:41:55.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:55.349 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:41:55.450 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:55.482 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:55.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:55.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:55.586 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:55.594 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:55 smithi016 bash[77995]: cluster 2023-09-23T19:41:54.203149+0000 mgr.y (mgr.25325) 1032 : cluster 0 pgmap v988: 89 pgs: 89 active+clean; 954 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-09-23T19:41:55.594 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:55 smithi016 bash[75004]: cluster 2023-09-23T19:41:54.203149+0000 mgr.y (mgr.25325) 1032 : cluster 0 pgmap v988: 89 pgs: 89 active+clean; 954 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-09-23T19:41:55.610 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:55.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:55.613 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:41:55.714 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:55.723 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:41:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:41:55] "GET /metrics HTTP/1.1" 200 36239 "" "Prometheus/2.33.4" 2023-09-23T19:41:55.741 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:55.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:55.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:41:55.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:55 smithi092 bash[58855]: cluster 2023-09-23T19:41:54.203149+0000 mgr.y (mgr.25325) 1032 : cluster 0 pgmap v988: 89 pgs: 89 active+clean; 954 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-09-23T19:41:55.845 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:55.872 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:55.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:55.875 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:55.975 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:56.007 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:56.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:56.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:56.110 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:56.132 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:56.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:56.135 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:56.221 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 12 16 26175 26159 136.232 111.188 0.00642501 0.00733441 2023-09-23T19:41:56.235 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:56.258 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:56.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:56.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:41:56.361 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:56.396 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:56.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:56.400 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:56.500 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:56.527 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:56.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:56.532 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:41:56.632 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:56.660 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:56.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:56.662 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:56.763 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:56.793 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:56.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:56.795 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:41:56.896 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:56.919 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:56.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:56.923 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:41:57.023 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:57.049 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:57.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:57.054 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:57.154 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:57.188 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:57.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:57.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:57.221 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 13 16 28035 28019 134.694 116.25 0.00455001 0.00741854 2023-09-23T19:41:57.294 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:57.319 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:57.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:57.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:57.424 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:57.461 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:57.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:57.467 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:57.568 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:57.592 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:57.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:57.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:41:57.696 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:57.714 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:57.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:57.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:41:57.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:57 smithi092 bash[58855]: cluster 2023-09-23T19:41:56.204757+0000 mgr.y (mgr.25325) 1033 : cluster 0 pgmap v989: 89 pgs: 89 active+clean; 1.4 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 133 MiB/s wr, 2.13k op/s 2023-09-23T19:41:57.817 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:57.826 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:57 smithi016 bash[75004]: cluster 2023-09-23T19:41:56.204757+0000 mgr.y (mgr.25325) 1033 : cluster 0 pgmap v989: 89 pgs: 89 active+clean; 1.4 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 133 MiB/s wr, 2.13k op/s 2023-09-23T19:41:57.826 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:57 smithi016 bash[77995]: cluster 2023-09-23T19:41:56.204757+0000 mgr.y (mgr.25325) 1033 : cluster 0 pgmap v989: 89 pgs: 89 active+clean; 1.4 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 133 MiB/s wr, 2.13k op/s 2023-09-23T19:41:57.846 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:57.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:57.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:57.949 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:57.975 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:57.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:57.978 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:41:58.078 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:58.112 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:58.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:58.117 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:41:58.218 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:58.221 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 14 16 29486 29470 131.55 90.6875 0.0126203 0.0075958 2023-09-23T19:41:58.246 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:41:58.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:58.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:41:58.351 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:58.388 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:58.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:58.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:58.491 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:58.517 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:58.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:58.520 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:58.621 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:58.655 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:58.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:58.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:58.760 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:58.785 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:41:58.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:58.788 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:41:58.888 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:58.917 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:58.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:58.921 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:59.022 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:59.048 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:59.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:59.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:59.154 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:59.178 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:59.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:59.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:41:59.221 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 15 16 31185 31169 129.859 106.188 0.0127916 0.00769348 2023-09-23T19:41:59.281 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:59.315 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:41:59.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:59.320 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:41:59.421 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:59.442 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:41:59.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:59.445 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:41:59.545 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:59.578 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:41:59.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:59.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:41:59.681 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:59.708 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:59.712 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:59 smithi016 bash[77995]: cluster 2023-09-23T19:41:58.205281+0000 mgr.y (mgr.25325) 1034 : cluster 0 pgmap v990: 89 pgs: 89 active+clean; 1.5 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 121 MiB/s wr, 1.93k op/s 2023-09-23T19:41:59.712 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:59 smithi016 bash[77995]: audit 2023-09-23T19:41:58.667092+0000 mon.a (mon.0) 1330 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:41:59.712 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:41:59 smithi016 bash[77995]: audit 2023-09-23T19:41:58.672187+0000 mon.a (mon.0) 1331 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:41:59.712 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:59 smithi016 bash[75004]: cluster 2023-09-23T19:41:58.205281+0000 mgr.y (mgr.25325) 1034 : cluster 0 pgmap v990: 89 pgs: 89 active+clean; 1.5 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 121 MiB/s wr, 1.93k op/s 2023-09-23T19:41:59.713 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:59 smithi016 bash[75004]: audit 2023-09-23T19:41:58.667092+0000 mon.a (mon.0) 1330 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:41:59.713 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:41:59 smithi016 bash[75004]: audit 2023-09-23T19:41:58.672187+0000 mon.a (mon.0) 1331 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:41:59.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:59.713 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:59.814 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:59.843 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:41:59.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:59.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:41:59.948 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:59.957 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:59 smithi092 bash[58855]: cluster 2023-09-23T19:41:58.205281+0000 mgr.y (mgr.25325) 1034 : cluster 0 pgmap v990: 89 pgs: 89 active+clean; 1.5 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 121 MiB/s wr, 1.93k op/s 2023-09-23T19:41:59.957 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:59 smithi092 bash[58855]: audit 2023-09-23T19:41:58.667092+0000 mon.a (mon.0) 1330 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:41:59.957 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:41:59 smithi092 bash[58855]: audit 2023-09-23T19:41:58.672187+0000 mon.a (mon.0) 1331 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:41:59.981 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:41:59.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:41:59.983 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:00.084 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:00.112 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:00.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:00.115 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:00.216 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:00.221 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 16 15 32635 32620 127.41 90.6875 0.00338944 0.0078415 2023-09-23T19:42:00.244 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:00.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:00.248 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:00.349 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:00.361 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:42:00.390 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:00.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:00.395 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:42:00.495 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:00.524 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:00.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:00.528 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:00.629 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:00.661 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:00.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:00.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:42:00.766 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:00.798 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:00.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:00.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:42:00.901 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:00.940 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:00.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:00.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:42:00.961 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-09-23T19:42:00.961 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:42:01.046 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:01.077 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:01.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:01.081 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:42:01.181 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:01.217 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:01.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:01.222 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:42:01.222 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 17 16 33889 33873 124.522 78.3125 0.0167637 0.00802531 2023-09-23T19:42:01.322 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:01.350 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:01.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:01.353 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:42:01.454 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:01.485 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:01.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:01.488 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:01.589 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:01.626 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:01.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:01.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:42:01.732 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:01.762 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:01.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:01.764 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:01.864 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:01.873 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:01 smithi092 bash[58855]: cluster 2023-09-23T19:42:00.205909+0000 mgr.y (mgr.25325) 1035 : cluster 0 pgmap v991: 89 pgs: 89 active+clean; 1.7 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 129 MiB/s wr, 2.07k op/s 2023-09-23T19:42:01.896 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:01.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:01.899 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:42:02.000 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:02.025 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:02.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:02.029 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:02.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:01 smithi016 bash[75004]: cluster 2023-09-23T19:42:00.205909+0000 mgr.y (mgr.25325) 1035 : cluster 0 pgmap v991: 89 pgs: 89 active+clean; 1.7 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 129 MiB/s wr, 2.07k op/s 2023-09-23T19:42:02.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:01 smithi016 bash[77995]: cluster 2023-09-23T19:42:00.205909+0000 mgr.y (mgr.25325) 1035 : cluster 0 pgmap v991: 89 pgs: 89 active+clean; 1.7 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 129 MiB/s wr, 2.07k op/s 2023-09-23T19:42:02.130 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:02.155 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:02.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:02.160 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:42:02.221 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 18 16 35293 35277 122.478 87.75 0.00671476 0.00815857 2023-09-23T19:42:02.261 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:02.293 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:02.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:02.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:42:02.395 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:02.425 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:02.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:02.428 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:42:02.529 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:02.562 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:02.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:02.567 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:02.667 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:02.693 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:02.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:02.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:42:02.797 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:02.828 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:02.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:02.830 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:02.931 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:02.957 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:02.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:02.962 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:03.063 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:03.092 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:03.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:03.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:42:03.199 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:03.223 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 19 16 36853 36837 121.154 97.5 0.0125332 0.00824888 2023-09-23T19:42:03.227 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:03.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:03.229 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:42:03.330 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:03.354 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:03.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:03.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:03.467 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:03.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:03.495 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:03.496 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:03.596 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:03.624 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:03.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:03.627 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:42:03.728 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:03.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:03 smithi016 bash[77995]: cluster 2023-09-23T19:42:02.206532+0000 mgr.y (mgr.25325) 1036 : cluster 0 pgmap v992: 89 pgs: 89 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 133 MiB/s wr, 2.12k op/s 2023-09-23T19:42:03.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:03 smithi016 bash[75004]: cluster 2023-09-23T19:42:02.206532+0000 mgr.y (mgr.25325) 1036 : cluster 0 pgmap v992: 89 pgs: 89 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 133 MiB/s wr, 2.12k op/s 2023-09-23T19:42:03.753 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:03.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:03.757 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:42:03.858 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:03.885 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:03.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:03.891 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:03.992 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:04.014 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:04.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:04.017 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:04.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:03 smithi092 bash[58855]: cluster 2023-09-23T19:42:02.206532+0000 mgr.y (mgr.25325) 1036 : cluster 0 pgmap v992: 89 pgs: 89 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 133 MiB/s wr, 2.12k op/s 2023-09-23T19:42:04.117 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:04.144 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:04.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:04.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:04.223 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-09-23T19:42:04.220341+0000 min lat: 0.00130197 max lat: 0.16206 avg lat: 0.00831794 2023-09-23T19:42:04.223 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:42:04.223 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 20 16 38467 38451 120.139 100.875 0.0133564 0.00831794 2023-09-23T19:42:04.249 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:04.273 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:04.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:04.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:42:04.376 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:04.402 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:04.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:04.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:04.509 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:04.533 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:04.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:04.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:04.636 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:04.664 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:04.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:04.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:04.769 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:04.797 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:04.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:04.800 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:42:04.901 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:04.931 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T19:42:04.932 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:04.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:04.935 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:05.036 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:05.066 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:05.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:05.068 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:42:05.169 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:05.196 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:05.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:05.200 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:42:05.223 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 21 16 39993 39977 118.96 95.375 0.018016 0.00840064 2023-09-23T19:42:05.301 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:05.325 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:05.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:05.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:05.430 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:05.457 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:05.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:05.460 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:05.561 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:05.590 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:05.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:05.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:05.695 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:05.723 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:05.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:05.726 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:05.827 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:05.854 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:05.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:05.857 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:05.958 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:05.966 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:05 smithi092 bash[58855]: cluster 2023-09-23T19:42:04.207029+0000 mgr.y (mgr.25325) 1037 : cluster 0 pgmap v993: 89 pgs: 89 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2023-09-23T19:42:05.994 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:05.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:05.999 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:42:06.080 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:42:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:42:05] "GET /metrics HTTP/1.1" 200 36302 "" "Prometheus/2.33.4" 2023-09-23T19:42:06.080 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:05 smithi016 bash[77995]: cluster 2023-09-23T19:42:04.207029+0000 mgr.y (mgr.25325) 1037 : cluster 0 pgmap v993: 89 pgs: 89 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2023-09-23T19:42:06.080 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:05 smithi016 bash[75004]: cluster 2023-09-23T19:42:04.207029+0000 mgr.y (mgr.25325) 1037 : cluster 0 pgmap v993: 89 pgs: 89 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2023-09-23T19:42:06.100 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:06.133 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:06.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:06.137 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:06.223 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 22 16 41293 41277 117.246 81.25 0.00768501 0.00852329 2023-09-23T19:42:06.266 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:06.299 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:06.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:06.304 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:06.404 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:06.431 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:06.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:06.434 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:42:06.535 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:06.577 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:06.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:06.579 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:06.680 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:06.709 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:06.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:06.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:42:06.815 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:06.849 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:06.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:06.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:42:06.953 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:06.984 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:06.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:06.988 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:07.088 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:07.122 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:07.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:07.128 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:42:07.223 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 23 16 42469 42453 115.343 73.5 0.0130443 0.00866352 2023-09-23T19:42:07.229 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:07.258 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:07.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:07.265 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:07.310 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-09-23T19:42:07.312 INFO:teuthology.orchestra.run.smithi016.stderr:dumped all 2023-09-23T19:42:07.366 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:07.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:07.400 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:07.400 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:07.500 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:07.523 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:07.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:07.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:07.627 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:07.653 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:07.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:07.657 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:07.758 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:07.766 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:07 smithi016 bash[77995]: cluster 2023-09-23T19:42:06.208520+0000 mgr.y (mgr.25325) 1038 : cluster 0 pgmap v994: 89 pgs: 89 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 125 MiB/s wr, 2.00k op/s 2023-09-23T19:42:07.767 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:07 smithi016 bash[75004]: cluster 2023-09-23T19:42:06.208520+0000 mgr.y (mgr.25325) 1038 : cluster 0 pgmap v994: 89 pgs: 89 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 125 MiB/s wr, 2.00k op/s 2023-09-23T19:42:07.782 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:07.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:07.784 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:07.885 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:07.894 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:07 smithi092 bash[58855]: cluster 2023-09-23T19:42:06.208520+0000 mgr.y (mgr.25325) 1038 : cluster 0 pgmap v994: 89 pgs: 89 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 125 MiB/s wr, 2.00k op/s 2023-09-23T19:42:07.922 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:07.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:07.926 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:42:08.027 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:08.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:08.063 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:08.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:42:08.163 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:08.192 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:08.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:08.194 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:08.223 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 24 15 43930 43915 114.344 91.375 0.0106656 0.0087399 2023-09-23T19:42:08.295 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:08.317 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:08.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:08.323 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:42:08.424 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:08.453 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:08.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:08.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:08.560 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:08.583 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:08.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:08.585 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:08.686 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:08.723 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:08.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:08.729 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:08.830 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:08.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:08.870 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:08.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:42:08.971 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:08.980 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:08 smithi016 bash[77995]: audit 2023-09-23T19:42:07.285852+0000 mgr.y (mgr.25325) 1039 : audit 0 from='client.55747 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:42:08.980 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:08 smithi016 bash[75004]: audit 2023-09-23T19:42:07.285852+0000 mgr.y (mgr.25325) 1039 : audit 0 from='client.55747 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:42:08.999 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:09.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:09.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:42:09.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:08 smithi092 bash[58855]: audit 2023-09-23T19:42:07.285852+0000 mgr.y (mgr.25325) 1039 : audit 0 from='client.55747 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:42:09.105 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:09.131 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:09.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:09.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:09.223 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 25 16 45370 45354 113.368 89.9375 0.0112224 0.00881537 2023-09-23T19:42:09.235 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:09.270 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:09.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:09.273 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:09.374 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:09.400 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:09.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:09.404 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:42:09.505 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:09.523 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:09.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:09.525 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:42:09.625 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:09.650 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:09.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:09.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:09.753 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:09.780 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:09.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:09.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:42:09.884 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:09.914 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:09.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:09.917 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:09.953 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:09 smithi016 bash[75004]: cluster 2023-09-23T19:42:08.210147+0000 mgr.y (mgr.25325) 1040 : cluster 0 pgmap v995: 89 pgs: 89 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-09-23T19:42:09.953 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:09 smithi016 bash[77995]: cluster 2023-09-23T19:42:08.210147+0000 mgr.y (mgr.25325) 1040 : cluster 0 pgmap v995: 89 pgs: 89 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-09-23T19:42:10.018 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:10.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:09 smithi092 bash[58855]: cluster 2023-09-23T19:42:08.210147+0000 mgr.y (mgr.25325) 1040 : cluster 0 pgmap v995: 89 pgs: 89 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-09-23T19:42:10.050 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:10.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:10.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:42:10.154 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:10.183 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:10.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:10.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:10.224 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 26 16 46478 46462 111.671 69.25 0.0131983 0.00894843 2023-09-23T19:42:10.288 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:10.310 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:10.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:10.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:42:10.413 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:10.449 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:10.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:10.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:10.553 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:10.579 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:10.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:10.582 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:42:10.682 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:10.718 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:10.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:10.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:10.787 INFO:teuthology.orchestra.run.smithi016.stdout:{"pg_ready":true,"pg_map":{"version":994,"stamp":"2023-09-23T19:42:06.208014+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2577405856,"num_objects":39343,"num_object_clones":0,"num_object_copies":118029,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":39343,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":39580,"num_write_kb":2518862,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39514,"ondisk_log_size":39514,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":12448956,"kb_used_data":7254524,"kb_used_omap":20,"kb_used_meta":5194347,"kb_avail":737512260,"statfs":{"total":767960285184,"available":755212554240,"internally_reserved":0,"allocated":7428632576,"data_stored":7423701203,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":5319011982},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1577385984,"num_objects":24069,"num_object_clones":0,"num_object_copies":72207,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24069,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":24069,"num_write_kb":1540416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.005367"},"pg_stats":[{"pgid":"3.1a","version":"59'1","reported_seq":181,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:41.262012+0000","last_change":"2023-09-23T19:38:43.897548+0000","last_active":"2023-09-23T19:41:41.262012+0000","last_peered":"2023-09-23T19:41:41.262012+0000","last_clean":"2023-09-23T19:41:41.262012+0000","last_became_active":"2023-09-23T19:38:43.896868+0000","last_became_peered":"2023-09-23T19:38:43.896868+0000","last_unstale":"2023-09-23T19:41:41.262012+0000","last_undegraded":"2023-09-23T19:41:41.262012+0000","last_fullsized":"2023-09-23T19:41:41.262012+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":141,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.496153+0000","last_change":"2023-09-23T19:32:49.731128+0000","last_active":"2023-09-23T19:41:40.496153+0000","last_peered":"2023-09-23T19:41:40.496153+0000","last_clean":"2023-09-23T19:41:40.496153+0000","last_became_active":"2023-09-23T19:32:49.730757+0000","last_became_peered":"2023-09-23T19:32:49.730757+0000","last_unstale":"2023-09-23T19:41:40.496153+0000","last_undegraded":"2023-09-23T19:41:40.496153+0000","last_fullsized":"2023-09-23T19:41:40.496153+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:40.183605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":130,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.524912+0000","last_change":"2023-09-23T19:28:07.890462+0000","last_active":"2023-09-23T19:41:46.524912+0000","last_peered":"2023-09-23T19:41:46.524912+0000","last_clean":"2023-09-23T19:41:46.524912+0000","last_became_active":"2023-09-23T19:28:07.890084+0000","last_became_peered":"2023-09-23T19:28:07.890084+0000","last_unstale":"2023-09-23T19:41:46.524912+0000","last_undegraded":"2023-09-23T19:41:46.524912+0000","last_fullsized":"2023-09-23T19:41:46.524912+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:50.458218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":147,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:41.257265+0000","last_change":"2023-09-23T19:28:07.884498+0000","last_active":"2023-09-23T19:41:41.257265+0000","last_peered":"2023-09-23T19:41:41.257265+0000","last_clean":"2023-09-23T19:41:41.257265+0000","last_became_active":"2023-09-23T19:28:07.884102+0000","last_became_peered":"2023-09-23T19:28:07.884102+0000","last_unstale":"2023-09-23T19:41:41.257265+0000","last_undegraded":"2023-09-23T19:41:41.257265+0000","last_fullsized":"2023-09-23T19:41:41.257265+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:05:26.348041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":158,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:41.256429+0000","last_change":"2023-09-23T19:32:49.729961+0000","last_active":"2023-09-23T19:41:41.256429+0000","last_peered":"2023-09-23T19:41:41.256429+0000","last_clean":"2023-09-23T19:41:41.256429+0000","last_became_active":"2023-09-23T19:32:49.729251+0000","last_became_peered":"2023-09-23T19:32:49.729251+0000","last_unstale":"2023-09-23T19:41:41.256429+0000","last_undegraded":"2023-09-23T19:41:41.256429+0000","last_fullsized":"2023-09-23T19:41:41.256429+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:20:56.595732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":150,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.504471+0000","last_change":"2023-09-23T19:22:42.150582+0000","last_active":"2023-09-23T19:41:40.504471+0000","last_peered":"2023-09-23T19:41:40.504471+0000","last_clean":"2023-09-23T19:41:40.504471+0000","last_became_active":"2023-09-23T19:22:42.150229+0000","last_became_peered":"2023-09-23T19:22:42.150229+0000","last_unstale":"2023-09-23T19:41:40.504471+0000","last_undegraded":"2023-09-23T19:41:40.504471+0000","last_fullsized":"2023-09-23T19:41:40.504471+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":145,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:41.257577+0000","last_change":"2023-09-23T19:28:07.887141+0000","last_active":"2023-09-23T19:41:41.257577+0000","last_peered":"2023-09-23T19:41:41.257577+0000","last_clean":"2023-09-23T19:41:41.257577+0000","last_became_active":"2023-09-23T19:28:07.886494+0000","last_became_peered":"2023-09-23T19:28:07.886494+0000","last_unstale":"2023-09-23T19:41:41.257577+0000","last_undegraded":"2023-09-23T19:41:41.257577+0000","last_fullsized":"2023-09-23T19:41:41.257577+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:24:07.615051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":141,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.495623+0000","last_change":"2023-09-23T19:32:49.729765+0000","last_active":"2023-09-23T19:41:40.495623+0000","last_peered":"2023-09-23T19:41:40.495623+0000","last_clean":"2023-09-23T19:41:40.495623+0000","last_became_active":"2023-09-23T19:32:49.729007+0000","last_became_peered":"2023-09-23T19:32:49.729007+0000","last_unstale":"2023-09-23T19:41:40.495623+0000","last_undegraded":"2023-09-23T19:41:40.495623+0000","last_fullsized":"2023-09-23T19:41:40.495623+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:59:31.183753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":152,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:41.012037+0000","last_change":"2023-09-23T19:22:42.151617+0000","last_active":"2023-09-23T19:41:41.012037+0000","last_peered":"2023-09-23T19:41:41.012037+0000","last_clean":"2023-09-23T19:41:41.012037+0000","last_became_active":"2023-09-23T19:22:42.151217+0000","last_became_peered":"2023-09-23T19:22:42.151217+0000","last_unstale":"2023-09-23T19:41:41.012037+0000","last_undegraded":"2023-09-23T19:41:41.012037+0000","last_fullsized":"2023-09-23T19:41:41.012037+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":125,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.572754+0000","last_change":"2023-09-23T19:32:49.736864+0000","last_active":"2023-09-23T19:41:40.572754+0000","last_peered":"2023-09-23T19:41:40.572754+0000","last_clean":"2023-09-23T19:41:40.572754+0000","last_became_active":"2023-09-23T19:32:49.736523+0000","last_became_peered":"2023-09-23T19:32:49.736523+0000","last_unstale":"2023-09-23T19:41:40.572754+0000","last_undegraded":"2023-09-23T19:41:40.572754+0000","last_fullsized":"2023-09-23T19:41:40.572754+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":154,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.573329+0000","last_change":"2023-09-23T19:28:07.892835+0000","last_active":"2023-09-23T19:41:40.573329+0000","last_peered":"2023-09-23T19:41:40.573329+0000","last_clean":"2023-09-23T19:41:40.573329+0000","last_became_active":"2023-09-23T19:28:07.892338+0000","last_became_peered":"2023-09-23T19:28:07.892338+0000","last_unstale":"2023-09-23T19:41:40.573329+0000","last_undegraded":"2023-09-23T19:41:40.573329+0000","last_fullsized":"2023-09-23T19:41:40.573329+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":145,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.495849+0000","last_change":"2023-09-23T19:38:43.890137+0000","last_active":"2023-09-23T19:41:40.495849+0000","last_peered":"2023-09-23T19:41:40.495849+0000","last_clean":"2023-09-23T19:41:40.495849+0000","last_became_active":"2023-09-23T19:38:43.889800+0000","last_became_peered":"2023-09-23T19:38:43.889800+0000","last_unstale":"2023-09-23T19:41:40.495849+0000","last_undegraded":"2023-09-23T19:41:40.495849+0000","last_fullsized":"2023-09-23T19:41:40.495849+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:59:20.468913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'23","reported_seq":158,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:41.262922+0000","last_change":"2023-09-23T19:05:52.107727+0000","last_active":"2023-09-23T19:41:41.262922+0000","last_peered":"2023-09-23T19:41:41.262922+0000","last_clean":"2023-09-23T19:41:41.262922+0000","last_became_active":"2023-09-23T19:05:52.107378+0000","last_became_peered":"2023-09-23T19:05:52.107378+0000","last_unstale":"2023-09-23T19:41:41.262922+0000","last_undegraded":"2023-09-23T19:41:41.262922+0000","last_fullsized":"2023-09-23T19:41:41.262922+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":149,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:41.262646+0000","last_change":"2023-09-23T19:22:42.148497+0000","last_active":"2023-09-23T19:41:41.262646+0000","last_peered":"2023-09-23T19:41:41.262646+0000","last_clean":"2023-09-23T19:41:41.262646+0000","last_became_active":"2023-09-23T19:22:42.148147+0000","last_became_peered":"2023-09-23T19:22:42.148147+0000","last_unstale":"2023-09-23T19:41:41.262646+0000","last_undegraded":"2023-09-23T19:41:41.262646+0000","last_fullsized":"2023-09-23T19:41:41.262646+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":168,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.573318+0000","last_change":"2023-09-23T19:38:43.897378+0000","last_active":"2023-09-23T19:41:40.573318+0000","last_peered":"2023-09-23T19:41:40.573318+0000","last_clean":"2023-09-23T19:41:40.573318+0000","last_became_active":"2023-09-23T19:38:43.897012+0000","last_became_peered":"2023-09-23T19:38:43.897012+0000","last_unstale":"2023-09-23T19:41:40.573318+0000","last_undegraded":"2023-09-23T19:41:40.573318+0000","last_fullsized":"2023-09-23T19:41:40.573318+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":141,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.497057+0000","last_change":"2023-09-23T19:32:49.730549+0000","last_active":"2023-09-23T19:41:40.497057+0000","last_peered":"2023-09-23T19:41:40.497057+0000","last_clean":"2023-09-23T19:41:40.497057+0000","last_became_active":"2023-09-23T19:32:49.730105+0000","last_became_peered":"2023-09-23T19:32:49.730105+0000","last_unstale":"2023-09-23T19:41:40.497057+0000","last_undegraded":"2023-09-23T19:41:40.497057+0000","last_fullsized":"2023-09-23T19:41:40.497057+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:49.751893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":146,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:41.256825+0000","last_change":"2023-09-23T19:28:07.884394+0000","last_active":"2023-09-23T19:41:41.256825+0000","last_peered":"2023-09-23T19:41:41.256825+0000","last_clean":"2023-09-23T19:41:41.256825+0000","last_became_active":"2023-09-23T19:28:07.883944+0000","last_became_peered":"2023-09-23T19:28:07.883944+0000","last_unstale":"2023-09-23T19:41:41.256825+0000","last_undegraded":"2023-09-23T19:41:41.256825+0000","last_fullsized":"2023-09-23T19:41:41.256825+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:08:17.608926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":151,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:41.011817+0000","last_change":"2023-09-23T19:28:07.891056+0000","last_active":"2023-09-23T19:41:41.011817+0000","last_peered":"2023-09-23T19:41:41.011817+0000","last_clean":"2023-09-23T19:41:41.011817+0000","last_became_active":"2023-09-23T19:28:07.890625+0000","last_became_peered":"2023-09-23T19:28:07.890625+0000","last_unstale":"2023-09-23T19:41:41.011817+0000","last_undegraded":"2023-09-23T19:41:41.011817+0000","last_fullsized":"2023-09-23T19:41:41.011817+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":136,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.572809+0000","last_change":"2023-09-23T19:05:52.114186+0000","last_active":"2023-09-23T19:41:40.572809+0000","last_peered":"2023-09-23T19:41:40.572809+0000","last_clean":"2023-09-23T19:41:40.572809+0000","last_became_active":"2023-09-23T19:05:52.113635+0000","last_became_peered":"2023-09-23T19:05:52.113635+0000","last_unstale":"2023-09-23T19:41:40.572809+0000","last_undegraded":"2023-09-23T19:41:40.572809+0000","last_fullsized":"2023-09-23T19:41:40.572809+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":145,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:41.257470+0000","last_change":"2023-09-23T19:28:07.884512+0000","last_active":"2023-09-23T19:41:41.257470+0000","last_peered":"2023-09-23T19:41:41.257470+0000","last_clean":"2023-09-23T19:41:41.257470+0000","last_became_active":"2023-09-23T19:28:07.884155+0000","last_became_peered":"2023-09-23T19:28:07.884155+0000","last_unstale":"2023-09-23T19:41:41.257470+0000","last_undegraded":"2023-09-23T19:41:41.257470+0000","last_fullsized":"2023-09-23T19:41:41.257470+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:10.654958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":159,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.503550+0000","last_change":"2023-09-23T19:18:09.010195+0000","last_active":"2023-09-23T19:41:40.503550+0000","last_peered":"2023-09-23T19:41:40.503550+0000","last_clean":"2023-09-23T19:41:40.503550+0000","last_became_active":"2023-09-23T19:18:09.009687+0000","last_became_peered":"2023-09-23T19:18:09.009687+0000","last_unstale":"2023-09-23T19:41:40.503550+0000","last_undegraded":"2023-09-23T19:41:40.503550+0000","last_fullsized":"2023-09-23T19:41:40.503550+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":150,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.563756+0000","last_change":"2023-09-23T19:28:07.885265+0000","last_active":"2023-09-23T19:41:40.563756+0000","last_peered":"2023-09-23T19:41:40.563756+0000","last_clean":"2023-09-23T19:41:40.563756+0000","last_became_active":"2023-09-23T19:28:07.884353+0000","last_became_peered":"2023-09-23T19:28:07.884353+0000","last_unstale":"2023-09-23T19:41:40.563756+0000","last_undegraded":"2023-09-23T19:41:40.563756+0000","last_fullsized":"2023-09-23T19:41:40.563756+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:56.325352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"185'2490","reported_seq":2502,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:42:05.851875+0000","last_change":"2023-09-23T19:41:24.846275+0000","last_active":"2023-09-23T19:42:05.851875+0000","last_peered":"2023-09-23T19:42:05.851875+0000","last_clean":"2023-09-23T19:42:05.851875+0000","last_became_active":"2023-09-23T19:41:24.845875+0000","last_became_peered":"2023-09-23T19:41:24.845875+0000","last_unstale":"2023-09-23T19:42:05.851875+0000","last_undegraded":"2023-09-23T19:42:05.851875+0000","last_fullsized":"2023-09-23T19:42:05.851875+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":180,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_clean_scrub_stamp":"2023-09-23T19:41:23.799472+0000","objects_scrubbed":0,"log_size":2490,"log_dups_size":0,"ondisk_log_size":2490,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163184640,"num_objects":2490,"num_object_clones":0,"num_object_copies":7470,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2490,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2490,"num_write_kb":159360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":152,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.573607+0000","last_change":"2023-09-23T19:18:09.035116+0000","last_active":"2023-09-23T19:41:40.573607+0000","last_peered":"2023-09-23T19:41:40.573607+0000","last_clean":"2023-09-23T19:41:40.573607+0000","last_became_active":"2023-09-23T19:18:09.034416+0000","last_became_peered":"2023-09-23T19:18:09.034416+0000","last_unstale":"2023-09-23T19:41:40.573607+0000","last_undegraded":"2023-09-23T19:41:40.573607+0000","last_fullsized":"2023-09-23T19:41:40.573607+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":136,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:41.011675+0000","last_change":"2023-09-23T19:05:52.789323+0000","last_active":"2023-09-23T19:41:41.011675+0000","last_peered":"2023-09-23T19:41:41.011675+0000","last_clean":"2023-09-23T19:41:41.011675+0000","last_became_active":"2023-09-23T19:05:52.788820+0000","last_became_peered":"2023-09-23T19:05:52.788820+0000","last_unstale":"2023-09-23T19:41:41.011675+0000","last_undegraded":"2023-09-23T19:41:41.011675+0000","last_fullsized":"2023-09-23T19:41:41.011675+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"185'2296","reported_seq":2310,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:42:03.451383+0000","last_change":"2023-09-23T19:41:24.830623+0000","last_active":"2023-09-23T19:42:03.451383+0000","last_peered":"2023-09-23T19:42:03.451383+0000","last_clean":"2023-09-23T19:42:03.451383+0000","last_became_active":"2023-09-23T19:41:24.830263+0000","last_became_peered":"2023-09-23T19:41:24.830263+0000","last_unstale":"2023-09-23T19:42:03.451383+0000","last_undegraded":"2023-09-23T19:42:03.451383+0000","last_fullsized":"2023-09-23T19:42:03.451383+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":180,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_clean_scrub_stamp":"2023-09-23T19:41:23.799472+0000","objects_scrubbed":0,"log_size":2296,"log_dups_size":0,"ondisk_log_size":2296,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:15:06.852546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150470656,"num_objects":2296,"num_object_clones":0,"num_object_copies":6888,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2296,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2296,"num_write_kb":146944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":153,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.573626+0000","last_change":"2023-09-23T19:22:42.151134+0000","last_active":"2023-09-23T19:41:40.573626+0000","last_peered":"2023-09-23T19:41:40.573626+0000","last_clean":"2023-09-23T19:41:40.573626+0000","last_became_active":"2023-09-23T19:22:42.150553+0000","last_became_peered":"2023-09-23T19:22:42.150553+0000","last_unstale":"2023-09-23T19:41:40.573626+0000","last_undegraded":"2023-09-23T19:41:40.573626+0000","last_fullsized":"2023-09-23T19:41:40.573626+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":164,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:41.261979+0000","last_change":"2023-09-23T19:28:07.899262+0000","last_active":"2023-09-23T19:41:41.261979+0000","last_peered":"2023-09-23T19:41:41.261979+0000","last_clean":"2023-09-23T19:41:41.261979+0000","last_became_active":"2023-09-23T19:28:07.898848+0000","last_became_peered":"2023-09-23T19:28:07.898848+0000","last_unstale":"2023-09-23T19:41:41.261979+0000","last_undegraded":"2023-09-23T19:41:41.261979+0000","last_fullsized":"2023-09-23T19:41:41.261979+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"185'2505","reported_seq":2518,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:42:05.498780+0000","last_change":"2023-09-23T19:41:24.846144+0000","last_active":"2023-09-23T19:42:05.498780+0000","last_peered":"2023-09-23T19:42:05.498780+0000","last_clean":"2023-09-23T19:42:05.498780+0000","last_became_active":"2023-09-23T19:41:24.845725+0000","last_became_peered":"2023-09-23T19:41:24.845725+0000","last_unstale":"2023-09-23T19:42:05.498780+0000","last_undegraded":"2023-09-23T19:42:05.498780+0000","last_fullsized":"2023-09-23T19:42:05.498780+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":180,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_clean_scrub_stamp":"2023-09-23T19:41:23.799472+0000","objects_scrubbed":0,"log_size":2505,"log_dups_size":0,"ondisk_log_size":2505,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164167680,"num_objects":2505,"num_object_clones":0,"num_object_copies":7515,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2505,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2505,"num_write_kb":160320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":137,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.573140+0000","last_change":"2023-09-23T19:05:52.114108+0000","last_active":"2023-09-23T19:41:40.573140+0000","last_peered":"2023-09-23T19:41:40.573140+0000","last_clean":"2023-09-23T19:41:40.573140+0000","last_became_active":"2023-09-23T19:05:52.113609+0000","last_became_peered":"2023-09-23T19:05:52.113609+0000","last_unstale":"2023-09-23T19:41:40.573140+0000","last_undegraded":"2023-09-23T19:41:40.573140+0000","last_fullsized":"2023-09-23T19:41:40.573140+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":150,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.503823+0000","last_change":"2023-09-23T19:22:42.156825+0000","last_active":"2023-09-23T19:41:40.503823+0000","last_peered":"2023-09-23T19:41:40.503823+0000","last_clean":"2023-09-23T19:41:40.503823+0000","last_became_active":"2023-09-23T19:22:42.156474+0000","last_became_peered":"2023-09-23T19:22:42.156474+0000","last_unstale":"2023-09-23T19:41:40.503823+0000","last_undegraded":"2023-09-23T19:41:40.503823+0000","last_fullsized":"2023-09-23T19:41:40.503823+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"185'2455","reported_seq":2468,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:42:05.476454+0000","last_change":"2023-09-23T19:41:24.846316+0000","last_active":"2023-09-23T19:42:05.476454+0000","last_peered":"2023-09-23T19:42:05.476454+0000","last_clean":"2023-09-23T19:42:05.476454+0000","last_became_active":"2023-09-23T19:41:24.845849+0000","last_became_peered":"2023-09-23T19:41:24.845849+0000","last_unstale":"2023-09-23T19:42:05.476454+0000","last_undegraded":"2023-09-23T19:42:05.476454+0000","last_fullsized":"2023-09-23T19:42:05.476454+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":180,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_clean_scrub_stamp":"2023-09-23T19:41:23.799472+0000","objects_scrubbed":0,"log_size":2455,"log_dups_size":0,"ondisk_log_size":2455,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160890880,"num_objects":2455,"num_object_clones":0,"num_object_copies":7365,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2455,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2455,"num_write_kb":157120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":152,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:41.011669+0000","last_change":"2023-09-23T19:28:07.891536+0000","last_active":"2023-09-23T19:41:41.011669+0000","last_peered":"2023-09-23T19:41:41.011669+0000","last_clean":"2023-09-23T19:41:41.011669+0000","last_became_active":"2023-09-23T19:28:07.891153+0000","last_became_peered":"2023-09-23T19:28:07.891153+0000","last_unstale":"2023-09-23T19:41:41.011669+0000","last_undegraded":"2023-09-23T19:41:41.011669+0000","last_fullsized":"2023-09-23T19:41:41.011669+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":146,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:41.257354+0000","last_change":"2023-09-23T19:22:42.135572+0000","last_active":"2023-09-23T19:41:41.257354+0000","last_peered":"2023-09-23T19:41:41.257354+0000","last_clean":"2023-09-23T19:41:41.257354+0000","last_became_active":"2023-09-23T19:22:42.135237+0000","last_became_peered":"2023-09-23T19:22:42.135237+0000","last_unstale":"2023-09-23T19:41:41.257354+0000","last_undegraded":"2023-09-23T19:41:41.257354+0000","last_fullsized":"2023-09-23T19:41:41.257354+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:35.838718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"185'2450","reported_seq":2463,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:42:05.059690+0000","last_change":"2023-09-23T19:41:24.846345+0000","last_active":"2023-09-23T19:42:05.059690+0000","last_peered":"2023-09-23T19:42:05.059690+0000","last_clean":"2023-09-23T19:42:05.059690+0000","last_became_active":"2023-09-23T19:41:24.844550+0000","last_became_peered":"2023-09-23T19:41:24.844550+0000","last_unstale":"2023-09-23T19:42:05.059690+0000","last_undegraded":"2023-09-23T19:42:05.059690+0000","last_fullsized":"2023-09-23T19:42:05.059690+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":180,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_clean_scrub_stamp":"2023-09-23T19:41:23.799472+0000","objects_scrubbed":0,"log_size":2450,"log_dups_size":0,"ondisk_log_size":2450,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160563200,"num_objects":2450,"num_object_clones":0,"num_object_copies":7350,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2450,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2450,"num_write_kb":156800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":93,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.552284+0000","last_change":"2023-09-23T19:32:49.728589+0000","last_active":"2023-09-23T19:41:46.552284+0000","last_peered":"2023-09-23T19:41:46.552284+0000","last_clean":"2023-09-23T19:41:46.552284+0000","last_became_active":"2023-09-23T19:32:49.728037+0000","last_became_peered":"2023-09-23T19:32:49.728037+0000","last_unstale":"2023-09-23T19:41:46.552284+0000","last_undegraded":"2023-09-23T19:41:46.552284+0000","last_fullsized":"2023-09-23T19:41:46.552284+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:26:00.957969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":149,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:41.263000+0000","last_change":"2023-09-23T19:28:07.896803+0000","last_active":"2023-09-23T19:41:41.263000+0000","last_peered":"2023-09-23T19:41:41.263000+0000","last_clean":"2023-09-23T19:41:41.263000+0000","last_became_active":"2023-09-23T19:28:07.896480+0000","last_became_peered":"2023-09-23T19:28:07.896480+0000","last_unstale":"2023-09-23T19:41:41.263000+0000","last_undegraded":"2023-09-23T19:41:41.263000+0000","last_fullsized":"2023-09-23T19:41:41.263000+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"185'2190","reported_seq":2204,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:42:02.133155+0000","last_change":"2023-09-23T19:41:24.829547+0000","last_active":"2023-09-23T19:42:02.133155+0000","last_peered":"2023-09-23T19:42:02.133155+0000","last_clean":"2023-09-23T19:42:02.133155+0000","last_became_active":"2023-09-23T19:41:24.829169+0000","last_became_peered":"2023-09-23T19:41:24.829169+0000","last_unstale":"2023-09-23T19:42:02.133155+0000","last_undegraded":"2023-09-23T19:42:02.133155+0000","last_fullsized":"2023-09-23T19:42:02.133155+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":180,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_clean_scrub_stamp":"2023-09-23T19:41:23.799472+0000","objects_scrubbed":0,"log_size":2190,"log_dups_size":0,"ondisk_log_size":2190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:24.473495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143523840,"num_objects":2190,"num_object_clones":0,"num_object_copies":6570,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2190,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2190,"num_write_kb":140160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":151,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.503825+0000","last_change":"2023-09-23T19:22:42.156582+0000","last_active":"2023-09-23T19:41:40.503825+0000","last_peered":"2023-09-23T19:41:40.503825+0000","last_clean":"2023-09-23T19:41:40.503825+0000","last_became_active":"2023-09-23T19:22:42.156251+0000","last_became_peered":"2023-09-23T19:22:42.156251+0000","last_unstale":"2023-09-23T19:41:40.503825+0000","last_undegraded":"2023-09-23T19:41:40.503825+0000","last_fullsized":"2023-09-23T19:41:40.503825+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":179,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:41.263012+0000","last_change":"2023-09-23T19:38:43.891593+0000","last_active":"2023-09-23T19:41:41.263012+0000","last_peered":"2023-09-23T19:41:41.263012+0000","last_clean":"2023-09-23T19:41:41.263012+0000","last_became_active":"2023-09-23T19:38:43.891063+0000","last_became_peered":"2023-09-23T19:38:43.891063+0000","last_unstale":"2023-09-23T19:41:41.263012+0000","last_undegraded":"2023-09-23T19:41:41.263012+0000","last_fullsized":"2023-09-23T19:41:41.263012+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"185'2488","reported_seq":2501,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:42:05.043924+0000","last_change":"2023-09-23T19:41:24.841200+0000","last_active":"2023-09-23T19:42:05.043924+0000","last_peered":"2023-09-23T19:42:05.043924+0000","last_clean":"2023-09-23T19:42:05.043924+0000","last_became_active":"2023-09-23T19:41:24.840724+0000","last_became_peered":"2023-09-23T19:41:24.840724+0000","last_unstale":"2023-09-23T19:42:05.043924+0000","last_undegraded":"2023-09-23T19:42:05.043924+0000","last_fullsized":"2023-09-23T19:42:05.043924+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":180,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_clean_scrub_stamp":"2023-09-23T19:41:23.799472+0000","objects_scrubbed":0,"log_size":2488,"log_dups_size":0,"ondisk_log_size":2488,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163053568,"num_objects":2488,"num_object_clones":0,"num_object_copies":7464,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2488,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2488,"num_write_kb":159232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":165,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:41.261700+0000","last_change":"2023-09-23T19:38:43.892972+0000","last_active":"2023-09-23T19:41:41.261700+0000","last_peered":"2023-09-23T19:41:41.261700+0000","last_clean":"2023-09-23T19:41:41.261700+0000","last_became_active":"2023-09-23T19:38:43.892451+0000","last_became_peered":"2023-09-23T19:38:43.892451+0000","last_unstale":"2023-09-23T19:41:41.261700+0000","last_undegraded":"2023-09-23T19:41:41.261700+0000","last_fullsized":"2023-09-23T19:41:41.261700+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":149,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:41.263468+0000","last_change":"2023-09-23T19:22:42.149670+0000","last_active":"2023-09-23T19:41:41.263468+0000","last_peered":"2023-09-23T19:41:41.263468+0000","last_clean":"2023-09-23T19:41:41.263468+0000","last_became_active":"2023-09-23T19:22:42.149022+0000","last_became_peered":"2023-09-23T19:22:42.149022+0000","last_unstale":"2023-09-23T19:41:41.263468+0000","last_undegraded":"2023-09-23T19:41:41.263468+0000","last_fullsized":"2023-09-23T19:41:41.263468+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"185'2548","reported_seq":2562,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:42:05.263005+0000","last_change":"2023-09-23T19:41:24.843247+0000","last_active":"2023-09-23T19:42:05.263005+0000","last_peered":"2023-09-23T19:42:05.263005+0000","last_clean":"2023-09-23T19:42:05.263005+0000","last_became_active":"2023-09-23T19:41:24.842923+0000","last_became_peered":"2023-09-23T19:41:24.842923+0000","last_unstale":"2023-09-23T19:42:05.263005+0000","last_undegraded":"2023-09-23T19:42:05.263005+0000","last_fullsized":"2023-09-23T19:42:05.263005+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":180,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_clean_scrub_stamp":"2023-09-23T19:41:23.799472+0000","objects_scrubbed":0,"log_size":2548,"log_dups_size":0,"ondisk_log_size":2548,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:40:00.822642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166985728,"num_objects":2548,"num_object_clones":0,"num_object_copies":7644,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2548,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2548,"num_write_kb":163072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":143,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.495830+0000","last_change":"2023-09-23T19:32:49.729244+0000","last_active":"2023-09-23T19:41:40.495830+0000","last_peered":"2023-09-23T19:41:40.495830+0000","last_clean":"2023-09-23T19:41:40.495830+0000","last_became_active":"2023-09-23T19:32:49.728810+0000","last_became_peered":"2023-09-23T19:32:49.728810+0000","last_unstale":"2023-09-23T19:41:40.495830+0000","last_undegraded":"2023-09-23T19:41:40.495830+0000","last_fullsized":"2023-09-23T19:41:40.495830+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:57.540826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":130,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.496594+0000","last_change":"2023-09-23T19:28:07.889095+0000","last_active":"2023-09-23T19:41:40.496594+0000","last_peered":"2023-09-23T19:41:40.496594+0000","last_clean":"2023-09-23T19:41:40.496594+0000","last_became_active":"2023-09-23T19:28:07.888742+0000","last_became_peered":"2023-09-23T19:28:07.888742+0000","last_unstale":"2023-09-23T19:41:40.496594+0000","last_undegraded":"2023-09-23T19:41:40.496594+0000","last_fullsized":"2023-09-23T19:41:40.496594+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:43.929488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.d","version":"185'2159","reported_seq":2173,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:42:02.102141+0000","last_change":"2023-09-23T19:41:24.839428+0000","last_active":"2023-09-23T19:42:02.102141+0000","last_peered":"2023-09-23T19:42:02.102141+0000","last_clean":"2023-09-23T19:42:02.102141+0000","last_became_active":"2023-09-23T19:41:24.838617+0000","last_became_peered":"2023-09-23T19:41:24.838617+0000","last_unstale":"2023-09-23T19:42:02.102141+0000","last_undegraded":"2023-09-23T19:42:02.102141+0000","last_fullsized":"2023-09-23T19:42:02.102141+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":180,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_clean_scrub_stamp":"2023-09-23T19:41:23.799472+0000","objects_scrubbed":0,"log_size":2159,"log_dups_size":0,"ondisk_log_size":2159,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:05:40.117492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141492224,"num_objects":2159,"num_object_clones":0,"num_object_copies":6477,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2159,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2159,"num_write_kb":138176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":164,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.496923+0000","last_change":"2023-09-23T19:22:42.143289+0000","last_active":"2023-09-23T19:41:40.496923+0000","last_peered":"2023-09-23T19:41:40.496923+0000","last_clean":"2023-09-23T19:41:40.496923+0000","last_became_active":"2023-09-23T19:22:42.142656+0000","last_became_peered":"2023-09-23T19:22:42.142656+0000","last_unstale":"2023-09-23T19:41:40.496923+0000","last_undegraded":"2023-09-23T19:41:40.496923+0000","last_fullsized":"2023-09-23T19:41:40.496923+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:38.467441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":182,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:41.012058+0000","last_change":"2023-09-23T19:38:43.894691+0000","last_active":"2023-09-23T19:41:41.012058+0000","last_peered":"2023-09-23T19:41:41.012058+0000","last_clean":"2023-09-23T19:41:41.012058+0000","last_became_active":"2023-09-23T19:38:43.894384+0000","last_became_peered":"2023-09-23T19:38:43.894384+0000","last_unstale":"2023-09-23T19:41:41.012058+0000","last_undegraded":"2023-09-23T19:41:41.012058+0000","last_fullsized":"2023-09-23T19:41:41.012058+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":149,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.805290+0000","last_change":"2023-09-23T19:38:43.898519+0000","last_active":"2023-09-23T19:41:40.805290+0000","last_peered":"2023-09-23T19:41:40.805290+0000","last_clean":"2023-09-23T19:41:40.805290+0000","last_became_active":"2023-09-23T19:38:43.896814+0000","last_became_peered":"2023-09-23T19:38:43.896814+0000","last_unstale":"2023-09-23T19:41:40.805290+0000","last_undegraded":"2023-09-23T19:41:40.805290+0000","last_fullsized":"2023-09-23T19:41:40.805290+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:40.465038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.b","version":"185'2555","reported_seq":2569,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:42:05.267394+0000","last_change":"2023-09-23T19:41:24.828810+0000","last_active":"2023-09-23T19:42:05.267394+0000","last_peered":"2023-09-23T19:42:05.267394+0000","last_clean":"2023-09-23T19:42:05.267394+0000","last_became_active":"2023-09-23T19:41:24.827984+0000","last_became_peered":"2023-09-23T19:41:24.827984+0000","last_unstale":"2023-09-23T19:42:05.267394+0000","last_undegraded":"2023-09-23T19:42:05.267394+0000","last_fullsized":"2023-09-23T19:42:05.267394+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":180,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_clean_scrub_stamp":"2023-09-23T19:41:23.799472+0000","objects_scrubbed":0,"log_size":2555,"log_dups_size":0,"ondisk_log_size":2555,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:09.027812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167444480,"num_objects":2555,"num_object_clones":0,"num_object_copies":7665,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2555,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2555,"num_write_kb":163520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":156,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:41.011766+0000","last_change":"2023-09-23T19:22:42.153647+0000","last_active":"2023-09-23T19:41:41.011766+0000","last_peered":"2023-09-23T19:41:41.011766+0000","last_clean":"2023-09-23T19:41:41.011766+0000","last_became_active":"2023-09-23T19:22:42.153317+0000","last_became_peered":"2023-09-23T19:22:42.153317+0000","last_unstale":"2023-09-23T19:41:41.011766+0000","last_undegraded":"2023-09-23T19:41:41.011766+0000","last_fullsized":"2023-09-23T19:41:41.011766+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":150,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:41.262619+0000","last_change":"2023-09-23T19:18:09.017525+0000","last_active":"2023-09-23T19:41:41.262619+0000","last_peered":"2023-09-23T19:41:41.262619+0000","last_clean":"2023-09-23T19:41:41.262619+0000","last_became_active":"2023-09-23T19:18:09.017193+0000","last_became_peered":"2023-09-23T19:18:09.017193+0000","last_unstale":"2023-09-23T19:41:41.262619+0000","last_undegraded":"2023-09-23T19:41:41.262619+0000","last_fullsized":"2023-09-23T19:41:41.262619+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":161,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.496782+0000","last_change":"2023-09-23T19:38:43.877088+0000","last_active":"2023-09-23T19:41:40.496782+0000","last_peered":"2023-09-23T19:41:40.496782+0000","last_clean":"2023-09-23T19:41:40.496782+0000","last_became_active":"2023-09-23T19:38:43.876719+0000","last_became_peered":"2023-09-23T19:38:43.876719+0000","last_unstale":"2023-09-23T19:41:40.496782+0000","last_undegraded":"2023-09-23T19:41:40.496782+0000","last_fullsized":"2023-09-23T19:41:40.496782+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:05.423256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.8","version":"185'2561","reported_seq":2574,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:42:05.027497+0000","last_change":"2023-09-23T19:41:24.841123+0000","last_active":"2023-09-23T19:42:05.027497+0000","last_peered":"2023-09-23T19:42:05.027497+0000","last_clean":"2023-09-23T19:42:05.027497+0000","last_became_active":"2023-09-23T19:41:24.840707+0000","last_became_peered":"2023-09-23T19:41:24.840707+0000","last_unstale":"2023-09-23T19:42:05.027497+0000","last_undegraded":"2023-09-23T19:42:05.027497+0000","last_fullsized":"2023-09-23T19:42:05.027497+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":180,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_clean_scrub_stamp":"2023-09-23T19:41:23.799472+0000","objects_scrubbed":0,"log_size":2561,"log_dups_size":0,"ondisk_log_size":2561,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167837696,"num_objects":2561,"num_object_clones":0,"num_object_copies":7683,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2561,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2561,"num_write_kb":163904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":129,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.804900+0000","last_change":"2023-09-23T19:38:43.883663+0000","last_active":"2023-09-23T19:41:40.804900+0000","last_peered":"2023-09-23T19:41:40.804900+0000","last_clean":"2023-09-23T19:41:40.804900+0000","last_became_active":"2023-09-23T19:38:43.882421+0000","last_became_peered":"2023-09-23T19:38:43.882421+0000","last_unstale":"2023-09-23T19:41:40.804900+0000","last_undegraded":"2023-09-23T19:41:40.804900+0000","last_fullsized":"2023-09-23T19:41:40.804900+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:49:01.607659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":146,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.495340+0000","last_change":"2023-09-23T19:38:43.890600+0000","last_active":"2023-09-23T19:41:40.495340+0000","last_peered":"2023-09-23T19:41:40.495340+0000","last_clean":"2023-09-23T19:41:40.495340+0000","last_became_active":"2023-09-23T19:38:43.890253+0000","last_became_peered":"2023-09-23T19:38:43.890253+0000","last_unstale":"2023-09-23T19:41:40.495340+0000","last_undegraded":"2023-09-23T19:41:40.495340+0000","last_fullsized":"2023-09-23T19:41:40.495340+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:46.751120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":158,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.496740+0000","last_change":"2023-09-23T19:22:42.143733+0000","last_active":"2023-09-23T19:41:40.496740+0000","last_peered":"2023-09-23T19:41:40.496740+0000","last_clean":"2023-09-23T19:41:40.496740+0000","last_became_active":"2023-09-23T19:22:42.143077+0000","last_became_peered":"2023-09-23T19:22:42.143077+0000","last_unstale":"2023-09-23T19:41:40.496740+0000","last_undegraded":"2023-09-23T19:41:40.496740+0000","last_fullsized":"2023-09-23T19:41:40.496740+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:47:33.086994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"185'2529","reported_seq":2541,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:42:05.057266+0000","last_change":"2023-09-23T19:41:24.841442+0000","last_active":"2023-09-23T19:42:05.057266+0000","last_peered":"2023-09-23T19:42:05.057266+0000","last_clean":"2023-09-23T19:42:05.057266+0000","last_became_active":"2023-09-23T19:41:24.841069+0000","last_became_peered":"2023-09-23T19:41:24.841069+0000","last_unstale":"2023-09-23T19:42:05.057266+0000","last_undegraded":"2023-09-23T19:42:05.057266+0000","last_fullsized":"2023-09-23T19:42:05.057266+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":180,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_clean_scrub_stamp":"2023-09-23T19:41:23.799472+0000","objects_scrubbed":0,"log_size":2529,"log_dups_size":0,"ondisk_log_size":2529,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165740544,"num_objects":2529,"num_object_clones":0,"num_object_copies":7587,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2529,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2529,"num_write_kb":161856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":171,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:41.012152+0000","last_change":"2023-09-23T19:38:43.947487+0000","last_active":"2023-09-23T19:41:41.012152+0000","last_peered":"2023-09-23T19:41:41.012152+0000","last_clean":"2023-09-23T19:41:41.012152+0000","last_became_active":"2023-09-23T19:38:43.946493+0000","last_became_peered":"2023-09-23T19:38:43.946493+0000","last_unstale":"2023-09-23T19:41:41.012152+0000","last_undegraded":"2023-09-23T19:41:41.012152+0000","last_fullsized":"2023-09-23T19:41:41.012152+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":387,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.572751+0000","last_change":"2023-09-23T19:18:09.016297+0000","last_active":"2023-09-23T19:41:40.572751+0000","last_peered":"2023-09-23T19:41:40.572751+0000","last_clean":"2023-09-23T19:41:40.572751+0000","last_became_active":"2023-09-23T19:18:09.015436+0000","last_became_peered":"2023-09-23T19:18:09.015436+0000","last_unstale":"2023-09-23T19:41:40.572751+0000","last_undegraded":"2023-09-23T19:41:40.572751+0000","last_fullsized":"2023-09-23T19:41:40.572751+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:59:33.840751+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:59:33.840751+0000","last_clean_scrub_stamp":"2023-09-23T18:59:33.840751+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":125,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:41.011491+0000","last_change":"2023-09-23T19:32:49.734148+0000","last_active":"2023-09-23T19:41:41.011491+0000","last_peered":"2023-09-23T19:41:41.011491+0000","last_clean":"2023-09-23T19:41:41.011491+0000","last_became_active":"2023-09-23T19:32:49.733512+0000","last_became_peered":"2023-09-23T19:32:49.733512+0000","last_unstale":"2023-09-23T19:41:41.011491+0000","last_undegraded":"2023-09-23T19:41:41.011491+0000","last_fullsized":"2023-09-23T19:41:41.011491+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":135,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.504897+0000","last_change":"2023-09-23T19:05:52.786771+0000","last_active":"2023-09-23T19:41:40.504897+0000","last_peered":"2023-09-23T19:41:40.504897+0000","last_clean":"2023-09-23T19:41:40.504897+0000","last_became_active":"2023-09-23T19:05:52.785797+0000","last_became_peered":"2023-09-23T19:05:52.785797+0000","last_unstale":"2023-09-23T19:41:40.504897+0000","last_undegraded":"2023-09-23T19:41:40.504897+0000","last_fullsized":"2023-09-23T19:41:40.504897+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.9","version":"185'2532","reported_seq":2545,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:42:05.022777+0000","last_change":"2023-09-23T19:41:24.846496+0000","last_active":"2023-09-23T19:42:05.022777+0000","last_peered":"2023-09-23T19:42:05.022777+0000","last_clean":"2023-09-23T19:42:05.022777+0000","last_became_active":"2023-09-23T19:41:24.846126+0000","last_became_peered":"2023-09-23T19:41:24.846126+0000","last_unstale":"2023-09-23T19:42:05.022777+0000","last_undegraded":"2023-09-23T19:42:05.022777+0000","last_fullsized":"2023-09-23T19:42:05.022777+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":180,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_clean_scrub_stamp":"2023-09-23T19:41:23.799472+0000","objects_scrubbed":0,"log_size":2532,"log_dups_size":0,"ondisk_log_size":2532,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165937152,"num_objects":2532,"num_object_clones":0,"num_object_copies":7596,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2532,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2532,"num_write_kb":162048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":170,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.572689+0000","last_change":"2023-09-23T19:22:42.150636+0000","last_active":"2023-09-23T19:41:40.572689+0000","last_peered":"2023-09-23T19:41:40.572689+0000","last_clean":"2023-09-23T19:41:40.572689+0000","last_became_active":"2023-09-23T19:22:42.150045+0000","last_became_peered":"2023-09-23T19:22:42.150045+0000","last_unstale":"2023-09-23T19:41:40.572689+0000","last_undegraded":"2023-09-23T19:41:40.572689+0000","last_fullsized":"2023-09-23T19:41:40.572689+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":147,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:41.256495+0000","last_change":"2023-09-23T19:28:07.889194+0000","last_active":"2023-09-23T19:41:41.256495+0000","last_peered":"2023-09-23T19:41:41.256495+0000","last_clean":"2023-09-23T19:41:41.256495+0000","last_became_active":"2023-09-23T19:28:07.888823+0000","last_became_peered":"2023-09-23T19:28:07.888823+0000","last_unstale":"2023-09-23T19:41:41.256495+0000","last_undegraded":"2023-09-23T19:41:41.256495+0000","last_fullsized":"2023-09-23T19:41:41.256495+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:23.464688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":177,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.553585+0000","last_change":"2023-09-23T19:38:43.876930+0000","last_active":"2023-09-23T19:41:46.553585+0000","last_peered":"2023-09-23T19:41:46.553585+0000","last_clean":"2023-09-23T19:41:46.553585+0000","last_became_active":"2023-09-23T19:38:43.876571+0000","last_became_peered":"2023-09-23T19:38:43.876571+0000","last_unstale":"2023-09-23T19:41:46.553585+0000","last_undegraded":"2023-09-23T19:41:46.553585+0000","last_fullsized":"2023-09-23T19:41:46.553585+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:30.924536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"185'2460","reported_seq":2473,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:42:05.254157+0000","last_change":"2023-09-23T19:41:24.838695+0000","last_active":"2023-09-23T19:42:05.254157+0000","last_peered":"2023-09-23T19:42:05.254157+0000","last_clean":"2023-09-23T19:42:05.254157+0000","last_became_active":"2023-09-23T19:41:24.838311+0000","last_became_peered":"2023-09-23T19:41:24.838311+0000","last_unstale":"2023-09-23T19:42:05.254157+0000","last_undegraded":"2023-09-23T19:42:05.254157+0000","last_fullsized":"2023-09-23T19:42:05.254157+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":180,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_clean_scrub_stamp":"2023-09-23T19:41:23.799472+0000","objects_scrubbed":0,"log_size":2460,"log_dups_size":0,"ondisk_log_size":2460,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:54.968784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161218560,"num_objects":2460,"num_object_clones":0,"num_object_copies":7380,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2460,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2460,"num_write_kb":157440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":155,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.573346+0000","last_change":"2023-09-23T19:18:09.015853+0000","last_active":"2023-09-23T19:41:40.573346+0000","last_peered":"2023-09-23T19:41:40.573346+0000","last_clean":"2023-09-23T19:41:40.573346+0000","last_became_active":"2023-09-23T19:18:09.015338+0000","last_became_peered":"2023-09-23T19:18:09.015338+0000","last_unstale":"2023-09-23T19:41:40.573346+0000","last_undegraded":"2023-09-23T19:41:40.573346+0000","last_fullsized":"2023-09-23T19:41:40.573346+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":146,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.496648+0000","last_change":"2023-09-23T19:38:43.881233+0000","last_active":"2023-09-23T19:41:40.496648+0000","last_peered":"2023-09-23T19:41:40.496648+0000","last_clean":"2023-09-23T19:41:40.496648+0000","last_became_active":"2023-09-23T19:38:43.880893+0000","last_became_peered":"2023-09-23T19:38:43.880893+0000","last_unstale":"2023-09-23T19:41:40.496648+0000","last_undegraded":"2023-09-23T19:41:40.496648+0000","last_fullsized":"2023-09-23T19:41:40.496648+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:11.206947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":132,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:41.257098+0000","last_change":"2023-09-23T19:18:09.005823+0000","last_active":"2023-09-23T19:41:41.257098+0000","last_peered":"2023-09-23T19:41:41.257098+0000","last_clean":"2023-09-23T19:41:41.257098+0000","last_became_active":"2023-09-23T19:18:09.005207+0000","last_became_peered":"2023-09-23T19:18:09.005207+0000","last_unstale":"2023-09-23T19:41:41.257098+0000","last_undegraded":"2023-09-23T19:41:41.257098+0000","last_fullsized":"2023-09-23T19:41:41.257098+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:05.520188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"185'2591","reported_seq":2604,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:42:05.467988+0000","last_change":"2023-09-23T19:41:24.848461+0000","last_active":"2023-09-23T19:42:05.467988+0000","last_peered":"2023-09-23T19:42:05.467988+0000","last_clean":"2023-09-23T19:42:05.467988+0000","last_became_active":"2023-09-23T19:41:24.848117+0000","last_became_peered":"2023-09-23T19:41:24.848117+0000","last_unstale":"2023-09-23T19:42:05.467988+0000","last_undegraded":"2023-09-23T19:42:05.467988+0000","last_fullsized":"2023-09-23T19:42:05.467988+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":180,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_clean_scrub_stamp":"2023-09-23T19:41:23.799472+0000","objects_scrubbed":0,"log_size":2591,"log_dups_size":0,"ondisk_log_size":2591,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169803776,"num_objects":2591,"num_object_clones":0,"num_object_copies":7773,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2591,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2591,"num_write_kb":165824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":89,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.573539+0000","last_change":"2023-09-23T19:32:49.737323+0000","last_active":"2023-09-23T19:41:40.573539+0000","last_peered":"2023-09-23T19:41:40.573539+0000","last_clean":"2023-09-23T19:41:40.573539+0000","last_became_active":"2023-09-23T19:32:49.736937+0000","last_became_peered":"2023-09-23T19:32:49.736937+0000","last_unstale":"2023-09-23T19:41:40.573539+0000","last_undegraded":"2023-09-23T19:41:40.573539+0000","last_fullsized":"2023-09-23T19:41:40.573539+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":115,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.496630+0000","last_change":"2023-09-23T19:22:42.145506+0000","last_active":"2023-09-23T19:41:40.496630+0000","last_peered":"2023-09-23T19:41:40.496630+0000","last_clean":"2023-09-23T19:41:40.496630+0000","last_became_active":"2023-09-23T19:22:42.143991+0000","last_became_peered":"2023-09-23T19:22:42.143991+0000","last_unstale":"2023-09-23T19:41:40.496630+0000","last_undegraded":"2023-09-23T19:41:40.496630+0000","last_fullsized":"2023-09-23T19:41:40.496630+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:56.875175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":126,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.573302+0000","last_change":"2023-09-23T19:32:49.734592+0000","last_active":"2023-09-23T19:41:40.573302+0000","last_peered":"2023-09-23T19:41:40.573302+0000","last_clean":"2023-09-23T19:41:40.573302+0000","last_became_active":"2023-09-23T19:32:49.733905+0000","last_became_peered":"2023-09-23T19:32:49.733905+0000","last_unstale":"2023-09-23T19:41:40.573302+0000","last_undegraded":"2023-09-23T19:41:40.573302+0000","last_fullsized":"2023-09-23T19:41:40.573302+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.e","version":"185'2510","reported_seq":2523,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:42:05.439801+0000","last_change":"2023-09-23T19:41:24.849220+0000","last_active":"2023-09-23T19:42:05.439801+0000","last_peered":"2023-09-23T19:42:05.439801+0000","last_clean":"2023-09-23T19:42:05.439801+0000","last_became_active":"2023-09-23T19:41:24.848808+0000","last_became_peered":"2023-09-23T19:41:24.848808+0000","last_unstale":"2023-09-23T19:42:05.439801+0000","last_undegraded":"2023-09-23T19:42:05.439801+0000","last_fullsized":"2023-09-23T19:42:05.439801+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":180,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_clean_scrub_stamp":"2023-09-23T19:41:23.799472+0000","objects_scrubbed":0,"log_size":2510,"log_dups_size":0,"ondisk_log_size":2510,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164495360,"num_objects":2510,"num_object_clones":0,"num_object_copies":7530,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2510,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2510,"num_write_kb":160640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":170,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.504372+0000","last_change":"2023-09-23T19:38:43.895887+0000","last_active":"2023-09-23T19:41:40.504372+0000","last_peered":"2023-09-23T19:41:40.504372+0000","last_clean":"2023-09-23T19:41:40.504372+0000","last_became_active":"2023-09-23T19:38:43.895486+0000","last_became_peered":"2023-09-23T19:38:43.895486+0000","last_unstale":"2023-09-23T19:41:40.504372+0000","last_undegraded":"2023-09-23T19:41:40.504372+0000","last_fullsized":"2023-09-23T19:41:40.504372+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":150,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.553797+0000","last_change":"2023-09-23T19:22:42.149694+0000","last_active":"2023-09-23T19:41:46.553797+0000","last_peered":"2023-09-23T19:41:46.553797+0000","last_clean":"2023-09-23T19:41:46.553797+0000","last_became_active":"2023-09-23T19:22:42.148194+0000","last_became_peered":"2023-09-23T19:22:42.148194+0000","last_unstale":"2023-09-23T19:41:46.553797+0000","last_undegraded":"2023-09-23T19:41:46.553797+0000","last_fullsized":"2023-09-23T19:41:46.553797+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:47.842383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":134,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:41.261683+0000","last_change":"2023-09-23T19:05:52.138620+0000","last_active":"2023-09-23T19:41:41.261683+0000","last_peered":"2023-09-23T19:41:41.261683+0000","last_clean":"2023-09-23T19:41:41.261683+0000","last_became_active":"2023-09-23T19:05:52.138243+0000","last_became_peered":"2023-09-23T19:05:52.138243+0000","last_unstale":"2023-09-23T19:41:41.261683+0000","last_undegraded":"2023-09-23T19:41:41.261683+0000","last_fullsized":"2023-09-23T19:41:41.261683+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":133,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:41.257425+0000","last_change":"2023-09-23T19:18:09.008811+0000","last_active":"2023-09-23T19:41:41.257425+0000","last_peered":"2023-09-23T19:41:41.257425+0000","last_clean":"2023-09-23T19:41:41.257425+0000","last_became_active":"2023-09-23T19:18:09.007255+0000","last_became_peered":"2023-09-23T19:18:09.007255+0000","last_unstale":"2023-09-23T19:41:41.257425+0000","last_undegraded":"2023-09-23T19:41:41.257425+0000","last_fullsized":"2023-09-23T19:41:41.257425+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:25.874038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":137,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.805301+0000","last_change":"2023-09-23T19:38:43.877244+0000","last_active":"2023-09-23T19:41:40.805301+0000","last_peered":"2023-09-23T19:41:40.805301+0000","last_clean":"2023-09-23T19:41:40.805301+0000","last_became_active":"2023-09-23T19:38:43.876896+0000","last_became_peered":"2023-09-23T19:38:43.876896+0000","last_unstale":"2023-09-23T19:41:40.805301+0000","last_undegraded":"2023-09-23T19:41:40.805301+0000","last_fullsized":"2023-09-23T19:41:40.805301+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:22.288276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":154,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:41.011945+0000","last_change":"2023-09-23T19:22:42.153825+0000","last_active":"2023-09-23T19:41:41.011945+0000","last_peered":"2023-09-23T19:41:41.011945+0000","last_clean":"2023-09-23T19:41:41.011945+0000","last_became_active":"2023-09-23T19:22:42.153416+0000","last_became_peered":"2023-09-23T19:22:42.153416+0000","last_unstale":"2023-09-23T19:41:41.011945+0000","last_undegraded":"2023-09-23T19:41:41.011945+0000","last_fullsized":"2023-09-23T19:41:41.011945+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":188,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:41.262868+0000","last_change":"2023-09-23T19:38:43.947451+0000","last_active":"2023-09-23T19:41:41.262868+0000","last_peered":"2023-09-23T19:41:41.262868+0000","last_clean":"2023-09-23T19:41:41.262868+0000","last_became_active":"2023-09-23T19:38:43.946420+0000","last_became_peered":"2023-09-23T19:38:43.946420+0000","last_unstale":"2023-09-23T19:41:41.262868+0000","last_undegraded":"2023-09-23T19:41:41.262868+0000","last_fullsized":"2023-09-23T19:41:41.262868+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":139,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:41.012309+0000","last_change":"2023-09-23T19:05:52.107570+0000","last_active":"2023-09-23T19:41:41.012309+0000","last_peered":"2023-09-23T19:41:41.012309+0000","last_clean":"2023-09-23T19:41:41.012309+0000","last_became_active":"2023-09-23T19:05:52.107215+0000","last_became_peered":"2023-09-23T19:05:52.107215+0000","last_unstale":"2023-09-23T19:41:41.012309+0000","last_undegraded":"2023-09-23T19:41:41.012309+0000","last_fullsized":"2023-09-23T19:41:41.012309+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":164,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:41.262298+0000","last_change":"2023-09-23T19:28:07.896983+0000","last_active":"2023-09-23T19:41:41.262298+0000","last_peered":"2023-09-23T19:41:41.262298+0000","last_clean":"2023-09-23T19:41:41.262298+0000","last_became_active":"2023-09-23T19:28:07.896577+0000","last_became_peered":"2023-09-23T19:28:07.896577+0000","last_unstale":"2023-09-23T19:41:41.262298+0000","last_undegraded":"2023-09-23T19:41:41.262298+0000","last_fullsized":"2023-09-23T19:41:41.262298+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":94,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.504670+0000","last_change":"2023-09-23T19:38:43.900225+0000","last_active":"2023-09-23T19:41:40.504670+0000","last_peered":"2023-09-23T19:41:40.504670+0000","last_clean":"2023-09-23T19:41:40.504670+0000","last_became_active":"2023-09-23T19:38:43.899907+0000","last_became_peered":"2023-09-23T19:38:43.899907+0000","last_unstale":"2023-09-23T19:41:40.504670+0000","last_undegraded":"2023-09-23T19:41:40.504670+0000","last_fullsized":"2023-09-23T19:41:40.504670+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":146,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.804904+0000","last_change":"2023-09-23T19:38:43.882487+0000","last_active":"2023-09-23T19:41:40.804904+0000","last_peered":"2023-09-23T19:41:40.804904+0000","last_clean":"2023-09-23T19:41:40.804904+0000","last_became_active":"2023-09-23T19:38:43.881931+0000","last_became_peered":"2023-09-23T19:38:43.881931+0000","last_unstale":"2023-09-23T19:41:40.804904+0000","last_undegraded":"2023-09-23T19:41:40.804904+0000","last_fullsized":"2023-09-23T19:41:40.804904+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:02:36.906535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":165,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:41.257510+0000","last_change":"2023-09-23T19:38:43.893983+0000","last_active":"2023-09-23T19:41:41.257510+0000","last_peered":"2023-09-23T19:41:41.257510+0000","last_clean":"2023-09-23T19:41:41.257510+0000","last_became_active":"2023-09-23T19:38:43.893527+0000","last_became_peered":"2023-09-23T19:38:43.893527+0000","last_unstale":"2023-09-23T19:41:41.257510+0000","last_undegraded":"2023-09-23T19:41:41.257510+0000","last_fullsized":"2023-09-23T19:41:41.257510+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:12.156531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":143,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T19:41:41.261428+0000","last_change":"2023-09-23T19:06:36.673002+0000","last_active":"2023-09-23T19:41:41.261428+0000","last_peered":"2023-09-23T19:41:41.261428+0000","last_clean":"2023-09-23T19:41:41.261428+0000","last_became_active":"2023-09-23T19:06:36.671911+0000","last_became_peered":"2023-09-23T19:06:36.671911+0000","last_unstale":"2023-09-23T19:41:41.261428+0000","last_undegraded":"2023-09-23T19:41:41.261428+0000","last_fullsized":"2023-09-23T19:41:41.261428+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":2576809984,"num_objects":39319,"num_object_clones":0,"num_object_copies":117957,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":39319,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":39319,"num_write_kb":2516416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7557218304,"data_stored":7557218304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39319,"ondisk_log_size":39319,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430679,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1857696,"kb_used_data":970008,"kb_used_omap":0,"kb_used_meta":887680,"kb_avail":91887456,"statfs":{"total":95995035648,"available":94092754944,"internally_reserved":0,"allocated":993288192,"data_stored":992787266,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":908984320},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:41:51 2023","interfaces":[{"interface":"back","average":{"1min":0.89100000000000001,"5min":1.825,"15min":1.8520000000000001},"min":{"1min":0.39300000000000002,"5min":0.27300000000000002,"15min":0.23100000000000001},"max":{"1min":2.0760000000000001,"5min":18.271000000000001,"15min":92.944999999999993},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.78600000000000003,"5min":1.7270000000000001,"15min":1.774},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.665,"5min":18.242000000000001,"15min":92.763999999999996},"last":0.66100000000000003}]},{"osd":1,"last update":"Sat Sep 23 19:41:09 2023","interfaces":[{"interface":"back","average":{"1min":1.1699999999999999,"5min":1.1950000000000001,"15min":1.788},"min":{"1min":0.34999999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":3.6400000000000001,"5min":6.4219999999999997,"15min":93.757999999999996},"last":1.0429999999999999},{"interface":"front","average":{"1min":1.0720000000000001,"5min":1.3029999999999999,"15min":1.806},"min":{"1min":0.32800000000000001,"5min":0.30599999999999999,"15min":0.29999999999999999},"max":{"1min":4.6699999999999999,"5min":6.4390000000000001,"15min":93.835999999999999},"last":1.1259999999999999}]},{"osd":2,"last update":"Sat Sep 23 19:41:51 2023","interfaces":[{"interface":"back","average":{"1min":1.0309999999999999,"5min":1.7829999999999999,"15min":1.9850000000000001},"min":{"1min":0.52000000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":3.2690000000000001,"5min":6.851,"15min":6.851},"last":1.1970000000000001},{"interface":"front","average":{"1min":0.79800000000000004,"5min":1.2569999999999999,"15min":1.2929999999999999},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":1.1160000000000001,"5min":10.74,"15min":10.74},"last":0.69399999999999995}]},{"osd":3,"last update":"Sat Sep 23 19:41:34 2023","interfaces":[{"interface":"back","average":{"1min":1.056,"5min":1.2090000000000001,"15min":1.4239999999999999},"min":{"1min":0.621,"5min":0.35399999999999998,"15min":0.33800000000000002},"max":{"1min":3.4689999999999999,"5min":6.0999999999999996,"15min":19.637},"last":0.378},{"interface":"front","average":{"1min":1.0980000000000001,"5min":1.4750000000000001,"15min":1.452},"min":{"1min":0.36699999999999999,"5min":0.34100000000000003,"15min":0.33900000000000002},"max":{"1min":4.6219999999999999,"5min":6.4630000000000001,"15min":20.204000000000001},"last":0.42999999999999999}]},{"osd":4,"last update":"Sat Sep 23 19:41:09 2023","interfaces":[{"interface":"back","average":{"1min":1.109,"5min":1.196,"15min":1.194},"min":{"1min":0.254,"5min":0.254,"15min":0.252},"max":{"1min":4.1929999999999996,"5min":5.4210000000000003,"15min":9.7699999999999996},"last":0.621},{"interface":"front","average":{"1min":1.0549999999999999,"5min":1.575,"15min":1.306},"min":{"1min":0.27600000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":4.7880000000000003,"5min":13.385,"15min":13.385},"last":0.39100000000000001}]},{"osd":5,"last update":"Sat Sep 23 19:41:09 2023","interfaces":[{"interface":"back","average":{"1min":1.079,"5min":1.595,"15min":1.5169999999999999},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.28499999999999998},"max":{"1min":4.6790000000000003,"5min":13.273999999999999,"15min":13.342000000000001},"last":0.72199999999999998},{"interface":"front","average":{"1min":1.478,"5min":1.492,"15min":1.5720000000000001},"min":{"1min":0.39800000000000002,"5min":0.36499999999999999,"15min":0.31},"max":{"1min":5.5270000000000001,"5min":6.3609999999999998,"15min":9.8979999999999997},"last":0.41699999999999998}]},{"osd":6,"last update":"Sat Sep 23 19:41:09 2023","interfaces":[{"interface":"back","average":{"1min":2.2280000000000002,"5min":1.804,"15min":1.488},"min":{"1min":0.378,"5min":0.378,"15min":0.316},"max":{"1min":22.725000000000001,"5min":22.725000000000001,"15min":22.725000000000001},"last":0.58599999999999997},{"interface":"front","average":{"1min":2.5299999999999998,"5min":2.2669999999999999,"15min":1.587},"min":{"1min":0.39300000000000002,"5min":0.308,"15min":0.308},"max":{"1min":22.707000000000001,"5min":22.707000000000001,"15min":22.707000000000001},"last":1.1659999999999999}]}]},{"osd":6,"up_from":42,"seq":180388626911,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2145744,"kb_used_data":1254536,"kb_used_omap":0,"kb_used_meta":891200,"kb_avail":91599408,"statfs":{"total":95995035648,"available":93797793792,"internally_reserved":0,"allocated":1284644864,"data_stored":1284159861,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":912588800},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:41:59 2023","interfaces":[{"interface":"back","average":{"1min":0.72699999999999998,"5min":2.2200000000000002,"15min":1.796},"min":{"1min":0.32100000000000001,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":1.236,"5min":15.140000000000001,"15min":87.283000000000001},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.92200000000000004,"5min":1.6839999999999999,"15min":1.6870000000000001},"min":{"1min":0.28299999999999997,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":1.5029999999999999,"5min":8.9909999999999997,"15min":86.286000000000001},"last":1.1120000000000001}]},{"osd":1,"last update":"Sat Sep 23 19:41:21 2023","interfaces":[{"interface":"back","average":{"1min":1.393,"5min":1.4730000000000001,"15min":1.429},"min":{"1min":0.40000000000000002,"5min":0.26300000000000001,"15min":0.253},"max":{"1min":6.0860000000000003,"5min":8.4299999999999997,"15min":8.4299999999999997},"last":1.3069999999999999},{"interface":"front","average":{"1min":1.056,"5min":1.5069999999999999,"15min":1.452},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.28999999999999998},"max":{"1min":2.585,"5min":7.9340000000000002,"15min":12.728999999999999},"last":1.2769999999999999}]},{"osd":2,"last update":"Sat Sep 23 19:41:46 2023","interfaces":[{"interface":"back","average":{"1min":6.8310000000000004,"5min":3.206,"15min":2.6030000000000002},"min":{"1min":0.52800000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":117.587,"5min":117.587,"15min":117.587},"last":0.30399999999999999},{"interface":"front","average":{"1min":6.8209999999999997,"5min":3.21,"15min":2.637},"min":{"1min":0.54400000000000004,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":117.26900000000001,"5min":117.26900000000001,"15min":117.26900000000001},"last":0.93799999999999994}]},{"osd":3,"last update":"Sat Sep 23 19:41:39 2023","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":1.9179999999999999,"15min":1.6160000000000001},"min":{"1min":0.46300000000000002,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":2.9430000000000001,"5min":8.7560000000000002,"15min":60.726999999999997},"last":1.675},{"interface":"front","average":{"1min":1.2150000000000001,"5min":1.8140000000000001,"15min":1.696},"min":{"1min":0.46600000000000003,"5min":0.34200000000000003,"15min":0.28799999999999998},"max":{"1min":4.1040000000000001,"5min":12.956,"15min":58.304000000000002},"last":1.345}]},{"osd":4,"last update":"Sat Sep 23 19:41:39 2023","interfaces":[{"interface":"back","average":{"1min":1.1819999999999999,"5min":1.825,"15min":1.212},"min":{"1min":0.38500000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":4.2000000000000002,"5min":10.138999999999999,"15min":10.138999999999999},"last":0.81100000000000005},{"interface":"front","average":{"1min":0.97599999999999998,"5min":1.98,"15min":1.3100000000000001},"min":{"1min":0.34899999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":2.9910000000000001,"5min":9.3399999999999999,"15min":9.3399999999999999},"last":1.1319999999999999}]},{"osd":5,"last update":"Sat Sep 23 19:41:39 2023","interfaces":[{"interface":"back","average":{"1min":1.0389999999999999,"5min":1.8280000000000001,"15min":1.3839999999999999},"min":{"1min":0.39800000000000002,"5min":0.39200000000000002,"15min":0.32600000000000001},"max":{"1min":2.6709999999999998,"5min":12.827,"15min":12.827},"last":0.83999999999999997},{"interface":"front","average":{"1min":1.177,"5min":2.4540000000000002,"15min":1.7470000000000001},"min":{"1min":0.50700000000000001,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":4.2110000000000003,"5min":10.433999999999999,"15min":10.433999999999999},"last":1.2909999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:41:03 2023","interfaces":[{"interface":"back","average":{"1min":1.49,"5min":1.5840000000000001,"15min":1.296},"min":{"1min":0.443,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":5.4699999999999998,"5min":5.9580000000000002,"15min":8.7949999999999999},"last":0.96899999999999997},{"interface":"front","average":{"1min":2.1850000000000001,"5min":1.9630000000000001,"15min":1.512},"min":{"1min":0.496,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":10.382999999999999,"5min":11.439,"15min":11.439},"last":1.371}]}]},{"osd":1,"up_from":98,"seq":420906795243,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1693576,"kb_used_data":1235840,"kb_used_omap":4,"kb_used_meta":457723,"kb_avail":92051576,"statfs":{"total":95995035648,"available":94260813824,"internally_reserved":0,"allocated":1265500160,"data_stored":1264731366,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":468709262},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:41:25 2023","interfaces":[{"interface":"back","average":{"1min":3.8839999999999999,"5min":4.6740000000000004,"15min":2.3439999999999999},"min":{"1min":0.28999999999999998,"5min":0.21299999999999999,"15min":0.185},"max":{"1min":53.064999999999998,"5min":65.079999999999998,"15min":65.079999999999998},"last":1.982},{"interface":"front","average":{"1min":3.6499999999999999,"5min":4.3310000000000004,"15min":2.3820000000000001},"min":{"1min":0.34599999999999997,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":53.130000000000003,"5min":65.730000000000004,"15min":65.730000000000004},"last":2.6579999999999999}]},{"osd":2,"last update":"Sat Sep 23 19:41:46 2023","interfaces":[{"interface":"back","average":{"1min":0.70699999999999996,"5min":1.3049999999999999,"15min":1.28},"min":{"1min":0.43099999999999999,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":1.516,"5min":11.143000000000001,"15min":11.143000000000001},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.69999999999999996,"5min":2.2349999999999999,"15min":2.7480000000000002},"min":{"1min":0.33200000000000002,"5min":0.251,"15min":0.251},"max":{"1min":1.3759999999999999,"5min":48.546999999999997,"15min":48.546999999999997},"last":2.286}]},{"osd":3,"last update":"Sat Sep 23 19:41:27 2023","interfaces":[{"interface":"back","average":{"1min":0.84299999999999997,"5min":1.157,"15min":1.2569999999999999},"min":{"1min":0.33000000000000002,"5min":0.255,"15min":0.21199999999999999},"max":{"1min":3.0569999999999999,"5min":9.1790000000000003,"15min":10.612},"last":2.3210000000000002},{"interface":"front","average":{"1min":1.105,"5min":1.744,"15min":1.6160000000000001},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.26500000000000001},"max":{"1min":5.2930000000000001,"5min":48.692,"15min":48.692},"last":0.61499999999999999}]},{"osd":4,"last update":"Sat Sep 23 19:41:25 2023","interfaces":[{"interface":"back","average":{"1min":0.89300000000000002,"5min":1.7490000000000001,"15min":1.5669999999999999},"min":{"1min":0.36499999999999999,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":2.9020000000000001,"5min":53.128999999999998,"15min":53.128999999999998},"last":2.5920000000000001},{"interface":"front","average":{"1min":1.077,"5min":2.0899999999999999,"15min":1.6990000000000001},"min":{"1min":0.373,"5min":0.373,"15min":0.36199999999999999},"max":{"1min":3.9100000000000001,"5min":48.637,"15min":48.637},"last":0.82399999999999995}]},{"osd":5,"last update":"Sat Sep 23 19:41:25 2023","interfaces":[{"interface":"back","average":{"1min":0.98199999999999998,"5min":1.873,"15min":1.9159999999999999},"min":{"1min":0.46400000000000002,"5min":0.42499999999999999,"15min":0.33900000000000002},"max":{"1min":3.1150000000000002,"5min":48.738,"15min":48.738},"last":0.76800000000000002},{"interface":"front","average":{"1min":1.1930000000000001,"5min":1.851,"15min":1.6759999999999999},"min":{"1min":0.40899999999999997,"5min":0.40400000000000003,"15min":0.35999999999999999},"max":{"1min":3.5800000000000001,"5min":48.726999999999997,"15min":48.726999999999997},"last":2.3519999999999999}]},{"osd":6,"last update":"Sat Sep 23 19:41:25 2023","interfaces":[{"interface":"back","average":{"1min":2.6219999999999999,"5min":2.3250000000000002,"15min":1.7729999999999999},"min":{"1min":0.503,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":26.145,"5min":48.762,"15min":48.762},"last":2.2519999999999998},{"interface":"front","average":{"1min":2.5859999999999999,"5min":2.343,"15min":1.8859999999999999},"min":{"1min":0.43099999999999999,"5min":0.39200000000000002,"15min":0.39100000000000001},"max":{"1min":26.312999999999999,"5min":53.076999999999998,"15min":53.076999999999998},"last":2.5369999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:41:25 2023","interfaces":[{"interface":"back","average":{"1min":1.0700000000000001,"5min":1.8,"15min":1.843},"min":{"1min":0.48399999999999999,"5min":0.38,"15min":0.38},"max":{"1min":3.0190000000000001,"5min":48.677,"15min":48.677},"last":2.6240000000000001},{"interface":"front","average":{"1min":1.1839999999999999,"5min":1.948,"15min":1.861},"min":{"1min":0.5,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":5.2610000000000001,"5min":48.805,"15min":48.805},"last":0.98299999999999998}]}]},{"osd":0,"up_from":85,"seq":365072220449,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1353820,"kb_used_data":724436,"kb_used_omap":5,"kb_used_meta":629370,"kb_avail":92391332,"statfs":{"total":95995035648,"available":94608723968,"internally_reserved":0,"allocated":741822464,"data_stored":741038216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":644475514},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 19:41:16 2023","interfaces":[{"interface":"back","average":{"1min":1.0840000000000001,"5min":1.077,"15min":1.173},"min":{"1min":0.22800000000000001,"5min":0.20799999999999999,"15min":0.20399999999999999},"max":{"1min":4.5599999999999996,"5min":5.4020000000000001,"15min":11.939},"last":14.970000000000001},{"interface":"front","average":{"1min":1.4730000000000001,"5min":1.5940000000000001,"15min":1.401},"min":{"1min":0.27100000000000002,"5min":0.223,"15min":0.223},"max":{"1min":12.848000000000001,"5min":12.848000000000001,"15min":12.939},"last":14.667999999999999}]},{"osd":2,"last update":"Sat Sep 23 19:41:48 2023","interfaces":[{"interface":"back","average":{"1min":0.91800000000000004,"5min":1.7569999999999999,"15min":1.859},"min":{"1min":0.317,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":3.5859999999999999,"5min":7.1100000000000003,"15min":7.1100000000000003},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.68000000000000005,"5min":2.0579999999999998,"15min":2.2599999999999998},"min":{"1min":0.30099999999999999,"5min":0.247,"15min":0.247},"max":{"1min":1.3380000000000001,"5min":15.124000000000001,"15min":15.124000000000001},"last":0.65200000000000002}]},{"osd":3,"last update":"Sat Sep 23 19:41:23 2023","interfaces":[{"interface":"back","average":{"1min":1.5629999999999999,"5min":2.1509999999999998,"15min":1.5409999999999999},"min":{"1min":0.27900000000000003,"5min":0.26400000000000001,"15min":0.251},"max":{"1min":14.212,"5min":14.212,"15min":14.212},"last":0.55300000000000005},{"interface":"front","average":{"1min":1.413,"5min":1.8660000000000001,"15min":1.569},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.34300000000000003},"max":{"1min":10.968999999999999,"5min":10.968999999999999,"15min":12.991},"last":0.57499999999999996}]},{"osd":4,"last update":"Sat Sep 23 19:41:56 2023","interfaces":[{"interface":"back","average":{"1min":0.83799999999999997,"5min":2.032,"15min":1.4890000000000001},"min":{"1min":0.36799999999999999,"5min":0.36599999999999999,"15min":0.34699999999999998},"max":{"1min":1.5269999999999999,"5min":11.792999999999999,"15min":11.792999999999999},"last":0.97799999999999998},{"interface":"front","average":{"1min":0.90200000000000002,"5min":2.077,"15min":1.47},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":1.385,"5min":13.641999999999999,"15min":13.641999999999999},"last":0.79100000000000004}]},{"osd":5,"last update":"Sat Sep 23 19:41:56 2023","interfaces":[{"interface":"back","average":{"1min":0.91100000000000003,"5min":2.0099999999999998,"15min":1.7370000000000001},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":1.6339999999999999,"5min":8.6229999999999993,"15min":13.154},"last":0.85299999999999998},{"interface":"front","average":{"1min":0.96399999999999997,"5min":2.165,"15min":1.845},"min":{"1min":0.42799999999999999,"5min":0.40200000000000002,"15min":0.36899999999999999},"max":{"1min":1.8740000000000001,"5min":10.411,"15min":13.109999999999999},"last":1.0449999999999999}]},{"osd":6,"last update":"Sat Sep 23 19:41:56 2023","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":2.673,"15min":1.748},"min":{"1min":0.54900000000000004,"5min":0.40500000000000003,"15min":0.38400000000000001},"max":{"1min":5.375,"5min":40.935000000000002,"15min":40.935000000000002},"last":0.86099999999999999},{"interface":"front","average":{"1min":1.127,"5min":2.6200000000000001,"15min":1.746},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.36699999999999999},"max":{"1min":3.9780000000000002,"5min":40.787999999999997,"15min":40.787999999999997},"last":0.88}]},{"osd":7,"last update":"Sat Sep 23 19:41:56 2023","interfaces":[{"interface":"back","average":{"1min":1.2010000000000001,"5min":1.6559999999999999,"15min":1.5049999999999999},"min":{"1min":0.69699999999999995,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":2.5910000000000002,"5min":7.2400000000000002,"15min":12.385},"last":1.0549999999999999},{"interface":"front","average":{"1min":0.97899999999999998,"5min":1.9770000000000001,"15min":1.7090000000000001},"min":{"1min":0.43099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":1.772,"5min":10.561,"15min":10.561},"last":1.0720000000000001}]}]},{"osd":2,"up_from":159,"seq":682899800105,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1018204,"kb_used_data":835796,"kb_used_omap":7,"kb_used_meta":182392,"kb_avail":92726948,"statfs":{"total":95995035648,"available":94952394752,"internally_reserved":0,"allocated":855855104,"data_stored":855124541,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":186769489},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:41:51 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":1.9590000000000001,"15min":2.0950000000000002},"min":{"1min":0.38300000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.0680000000000001,"5min":10.194000000000001,"15min":10.194000000000001},"last":0.621},{"interface":"front","average":{"1min":0.75900000000000001,"5min":1.8120000000000001,"15min":1.9219999999999999},"min":{"1min":0.379,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":1.1819999999999999,"5min":8.8049999999999997,"15min":8.8049999999999997},"last":0.68999999999999995}]},{"osd":1,"last update":"Sat Sep 23 19:41:51 2023","interfaces":[{"interface":"back","average":{"1min":0.72299999999999998,"5min":0.97499999999999998,"15min":0.93500000000000005},"min":{"1min":0.46000000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":2.294,"5min":5.4320000000000004,"15min":5.4320000000000004},"last":1.3009999999999999},{"interface":"front","average":{"1min":0.75600000000000001,"5min":1.23,"15min":1.3540000000000001},"min":{"1min":0.58799999999999997,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.98999999999999999,"5min":8.7880000000000003,"15min":8.7880000000000003},"last":0.34999999999999998}]},{"osd":3,"last update":"Sat Sep 23 19:41:51 2023","interfaces":[{"interface":"back","average":{"1min":0.82499999999999996,"5min":1.6799999999999999,"15min":1.891},"min":{"1min":0.438,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":2.3140000000000001,"5min":8.0489999999999995,"15min":8.0489999999999995},"last":1.2649999999999999},{"interface":"front","average":{"1min":1.3420000000000001,"5min":1.9930000000000001,"15min":2.0670000000000002},"min":{"1min":0.497,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":9.3780000000000001,"5min":9.3780000000000001,"15min":9.3780000000000001},"last":0.70699999999999996}]},{"osd":4,"last update":"Sat Sep 23 19:41:51 2023","interfaces":[{"interface":"back","average":{"1min":1.3129999999999999,"5min":1.651,"15min":1.526},"min":{"1min":0.75600000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":9.1020000000000003,"5min":9.1020000000000003,"15min":9.1020000000000003},"last":0.60699999999999998},{"interface":"front","average":{"1min":1.002,"5min":1.5369999999999999,"15min":1.579},"min":{"1min":0.46300000000000002,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":2.2149999999999999,"5min":6.8570000000000002,"15min":6.8570000000000002},"last":0.76100000000000001}]},{"osd":5,"last update":"Sat Sep 23 19:41:51 2023","interfaces":[{"interface":"back","average":{"1min":0.95799999999999996,"5min":1.5669999999999999,"15min":1.7250000000000001},"min":{"1min":0.42199999999999999,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.3660000000000001,"5min":9.1180000000000003,"15min":9.1180000000000003},"last":0.66600000000000004},{"interface":"front","average":{"1min":1.343,"5min":1.849,"15min":1.863},"min":{"1min":0.55600000000000005,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":8.9930000000000003,"5min":8.9930000000000003,"15min":8.9930000000000003},"last":0.65600000000000003}]},{"osd":6,"last update":"Sat Sep 23 19:41:51 2023","interfaces":[{"interface":"back","average":{"1min":1.4470000000000001,"5min":2.0499999999999998,"15min":1.9259999999999999},"min":{"1min":0.72099999999999997,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":9.9659999999999993,"5min":11.851000000000001,"15min":11.851000000000001},"last":3.5409999999999999},{"interface":"front","average":{"1min":1.52,"5min":1.827,"15min":1.7430000000000001},"min":{"1min":0.71499999999999997,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":9.4909999999999997,"5min":11.826000000000001,"15min":11.826000000000001},"last":1.7410000000000001}]},{"osd":7,"last update":"Sat Sep 23 19:41:51 2023","interfaces":[{"interface":"back","average":{"1min":1.5089999999999999,"5min":1.5489999999999999,"15min":1.5589999999999999},"min":{"1min":0.69199999999999995,"5min":0.441,"15min":0.441},"max":{"1min":9.4570000000000007,"5min":9.4570000000000007,"15min":9.4570000000000007},"last":2.4870000000000001},{"interface":"front","average":{"1min":1.081,"5min":1.403,"15min":1.46},"min":{"1min":0.48899999999999999,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":2.2469999999999999,"5min":7.9169999999999998,"15min":7.9169999999999998},"last":1.008}]}]},{"osd":3,"up_from":116,"seq":498216206505,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1313268,"kb_used_data":961772,"kb_used_omap":2,"kb_used_meta":351485,"kb_avail":92431884,"statfs":{"total":95995035648,"available":94650249216,"internally_reserved":0,"allocated":984854528,"data_stored":984159695,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":359920693},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:41:32 2023","interfaces":[{"interface":"back","average":{"1min":1.2230000000000001,"5min":1.552,"15min":1.3779999999999999},"min":{"1min":0.33200000000000002,"5min":0.23200000000000001,"15min":0.19900000000000001},"max":{"1min":7.8360000000000003,"5min":11.439,"15min":13.411},"last":0.61499999999999999},{"interface":"front","average":{"1min":1.099,"5min":1.6819999999999999,"15min":1.3700000000000001},"min":{"1min":0.35399999999999998,"5min":0.23300000000000001,"15min":0.23100000000000001},"max":{"1min":5.2130000000000001,"5min":9.5069999999999997,"15min":10.77},"last":1.536}]},{"osd":1,"last update":"Sat Sep 23 19:41:32 2023","interfaces":[{"interface":"back","average":{"1min":1.1359999999999999,"5min":1.0780000000000001,"15min":1.2629999999999999},"min":{"1min":0.499,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":5.109,"5min":6.4390000000000001,"15min":13.659000000000001},"last":1.5640000000000001},{"interface":"front","average":{"1min":1.1040000000000001,"5min":1.1930000000000001,"15min":1.47},"min":{"1min":0.435,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":4.1369999999999996,"5min":9.4130000000000003,"15min":16.262},"last":1.2729999999999999}]},{"osd":2,"last update":"Sat Sep 23 19:41:49 2023","interfaces":[{"interface":"back","average":{"1min":0.85799999999999998,"5min":2.5539999999999998,"15min":2.5600000000000001},"min":{"1min":0.624,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":1.522,"5min":10.960000000000001,"15min":10.960000000000001},"last":1.391},{"interface":"front","average":{"1min":0.89900000000000002,"5min":1.6180000000000001,"15min":1.71},"min":{"1min":0.64200000000000002,"5min":0.254,"15min":0.254},"max":{"1min":2.0579999999999998,"5min":8.7560000000000002,"15min":8.7560000000000002},"last":1.2290000000000001}]},{"osd":4,"last update":"Sat Sep 23 19:41:32 2023","interfaces":[{"interface":"back","average":{"1min":1.387,"5min":1.8360000000000001,"15min":1.5},"min":{"1min":0.48899999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":4.1790000000000003,"5min":9.6229999999999993,"15min":9.6229999999999993},"last":0.55800000000000005},{"interface":"front","average":{"1min":1.2709999999999999,"5min":1.905,"15min":1.494},"min":{"1min":0.59199999999999997,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":3.4670000000000001,"5min":9.6539999999999999,"15min":10.926},"last":1.4239999999999999}]},{"osd":5,"last update":"Sat Sep 23 19:41:32 2023","interfaces":[{"interface":"back","average":{"1min":1.4650000000000001,"5min":1.7110000000000001,"15min":1.635},"min":{"1min":0.66300000000000003,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":5.181,"5min":8.0169999999999995,"15min":10.82},"last":1.49},{"interface":"front","average":{"1min":1.4139999999999999,"5min":1.55,"15min":1.4790000000000001},"min":{"1min":0.53000000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":3.9550000000000001,"5min":7.5049999999999999,"15min":8.7710000000000008},"last":1.26}]},{"osd":6,"last update":"Sat Sep 23 19:41:32 2023","interfaces":[{"interface":"back","average":{"1min":1.621,"5min":1.9359999999999999,"15min":1.581},"min":{"1min":0.68999999999999995,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":5.1520000000000001,"5min":7.4409999999999998,"15min":10.859},"last":3.0529999999999999},{"interface":"front","average":{"1min":1.6539999999999999,"5min":1.494,"15min":1.353},"min":{"1min":0.58699999999999997,"5min":0.315,"15min":0.315},"max":{"1min":6.6310000000000002,"5min":6.6310000000000002,"15min":8.0190000000000001},"last":1.175}]},{"osd":7,"last update":"Sat Sep 23 19:41:32 2023","interfaces":[{"interface":"back","average":{"1min":1.337,"5min":1.395,"15min":1.448},"min":{"1min":0.67600000000000005,"5min":0.35799999999999998,"15min":0.32400000000000001},"max":{"1min":2.8109999999999999,"5min":7.7670000000000003,"15min":11.259},"last":1.325},{"interface":"front","average":{"1min":1.5660000000000001,"5min":1.6439999999999999,"15min":1.5409999999999999},"min":{"1min":0.55400000000000005,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":5.5970000000000004,"5min":9.5510000000000002,"15min":9.5510000000000002},"last":0.71799999999999997}]}]},{"osd":4,"up_from":30,"seq":128849019376,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1535704,"kb_used_data":635792,"kb_used_omap":0,"kb_used_meta":899904,"kb_avail":92209448,"statfs":{"total":95995035648,"available":94422474752,"internally_reserved":0,"allocated":651051008,"data_stored":650557660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":921501696},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:41:57 2023","interfaces":[{"interface":"back","average":{"1min":0.80100000000000005,"5min":1.917,"15min":1.2589999999999999},"min":{"1min":0.42999999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":1.341,"5min":12.975,"15min":12.975},"last":0.27600000000000002},{"interface":"front","average":{"1min":0.76400000000000001,"5min":1.714,"15min":1.244},"min":{"1min":0.41099999999999998,"5min":0.27700000000000002,"15min":0.26500000000000001},"max":{"1min":1.5349999999999999,"5min":11.380000000000001,"15min":11.380000000000001},"last":0.29499999999999998}]},{"osd":1,"last update":"Sat Sep 23 19:41:26 2023","interfaces":[{"interface":"back","average":{"1min":1.4259999999999999,"5min":1.4850000000000001,"15min":1.371},"min":{"1min":0.40100000000000002,"5min":0.29299999999999998,"15min":0.27700000000000002},"max":{"1min":6.5039999999999996,"5min":9.1859999999999999,"15min":12.372},"last":0.379},{"interface":"front","average":{"1min":1.054,"5min":1.571,"15min":1.454},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":4.8920000000000003,"5min":11.047000000000001,"15min":11.061},"last":0.316}]},{"osd":2,"last update":"Sat Sep 23 19:41:45 2023","interfaces":[{"interface":"back","average":{"1min":0.74199999999999999,"5min":2.6080000000000001,"15min":3.2719999999999998},"min":{"1min":0.35899999999999999,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":1.3049999999999999,"5min":11.768000000000001,"15min":11.768000000000001},"last":0.41499999999999998},{"interface":"front","average":{"1min":1.1819999999999999,"5min":1.929,"15min":1.893},"min":{"1min":0.42199999999999999,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":8.1699999999999999,"5min":9.1479999999999997,"15min":9.1479999999999997},"last":1.1970000000000001}]},{"osd":3,"last update":"Sat Sep 23 19:41:39 2023","interfaces":[{"interface":"back","average":{"1min":1.123,"5min":1.7869999999999999,"15min":1.6859999999999999},"min":{"1min":0.46600000000000003,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":5.0060000000000002,"5min":12.973000000000001,"15min":49.515000000000001},"last":0.39200000000000002},{"interface":"front","average":{"1min":1.0409999999999999,"5min":1.839,"15min":1.7110000000000001},"min":{"1min":0.52600000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":4.0990000000000002,"5min":10.266,"15min":51.518999999999998},"last":0.437}]},{"osd":5,"last update":"Sat Sep 23 19:41:57 2023","interfaces":[{"interface":"back","average":{"1min":0.874,"5min":1.825,"15min":1.5109999999999999},"min":{"1min":0.48899999999999999,"5min":0.315,"15min":0.28699999999999998},"max":{"1min":1.1579999999999999,"5min":10.944000000000001,"15min":10.944000000000001},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.86899999999999999,"5min":1.925,"15min":1.488},"min":{"1min":0.46899999999999997,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":1.2889999999999999,"5min":10.362,"15min":10.362},"last":0.46500000000000002}]},{"osd":6,"last update":"Sat Sep 23 19:41:26 2023","interfaces":[{"interface":"back","average":{"1min":1.49,"5min":2.2490000000000001,"15min":1.5469999999999999},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":7.6779999999999999,"5min":11.076000000000001,"15min":11.076000000000001},"last":0.45500000000000002},{"interface":"front","average":{"1min":1.1000000000000001,"5min":2.0720000000000001,"15min":1.4570000000000001},"min":{"1min":0.46899999999999997,"5min":0.39100000000000001,"15min":0.34000000000000002},"max":{"1min":3.9889999999999999,"5min":10.952,"15min":10.952},"last":0.505}]},{"osd":7,"last update":"Sat Sep 23 19:41:26 2023","interfaces":[{"interface":"back","average":{"1min":1.284,"5min":1.54,"15min":1.3620000000000001},"min":{"1min":0.44600000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":6.4320000000000004,"5min":9.2149999999999999,"15min":9.2149999999999999},"last":6.3650000000000002},{"interface":"front","average":{"1min":1.375,"5min":1.893,"15min":1.5569999999999999},"min":{"1min":0.374,"5min":0.374,"15min":0.34499999999999997},"max":{"1min":6.5529999999999999,"5min":10.989000000000001,"15min":10.989000000000001},"last":4.3310000000000004}]}]},{"osd":5,"up_from":36,"seq":154618823143,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1530944,"kb_used_data":636344,"kb_used_omap":0,"kb_used_meta":894592,"kb_avail":92214208,"statfs":{"total":95995035648,"available":94427348992,"internally_reserved":0,"allocated":651616256,"data_stored":651142598,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":916062208},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":1.8,"15min":1.532},"min":{"1min":0.42499999999999999,"5min":0.26300000000000001,"15min":0.26100000000000001},"max":{"1min":2.9900000000000002,"5min":43.094000000000001,"15min":43.094000000000001},"last":0.312},{"interface":"front","average":{"1min":0.77800000000000002,"5min":2.3940000000000001,"15min":1.9910000000000001},"min":{"1min":0.32300000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":1.206,"5min":43.802999999999997,"15min":43.802999999999997},"last":0.32900000000000001}]},{"osd":1,"last update":"Sat Sep 23 19:41:23 2023","interfaces":[{"interface":"back","average":{"1min":1.371,"5min":1.3100000000000001,"15min":1.3759999999999999},"min":{"1min":0.315,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":5.2190000000000003,"5min":6.8689999999999998,"15min":32.253999999999998},"last":0.40899999999999997},{"interface":"front","average":{"1min":1.2689999999999999,"5min":1.5580000000000001,"15min":1.8240000000000001},"min":{"1min":0.51600000000000001,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":7.2039999999999997,"5min":18.207000000000001,"15min":32.277999999999999},"last":0.39600000000000002}]},{"osd":2,"last update":"Sat Sep 23 19:41:46 2023","interfaces":[{"interface":"back","average":{"1min":0.84399999999999997,"5min":2.5430000000000001,"15min":2.8889999999999998},"min":{"1min":0.56599999999999995,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":1.208,"5min":10.611000000000001,"15min":10.611000000000001},"last":1.0089999999999999},{"interface":"front","average":{"1min":1.528,"5min":1.9950000000000001,"15min":1.9770000000000001},"min":{"1min":0.56000000000000005,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":7.2519999999999998,"5min":8.6120000000000001,"15min":8.6120000000000001},"last":0.45000000000000001}]},{"osd":3,"last update":"Sat Sep 23 19:41:23 2023","interfaces":[{"interface":"back","average":{"1min":1.367,"5min":1.877,"15min":1.5189999999999999},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.311},"max":{"1min":5.7560000000000002,"5min":10.686,"15min":10.885999999999999},"last":0.56100000000000005},{"interface":"front","average":{"1min":2.028,"5min":1.865,"15min":1.4330000000000001},"min":{"1min":0.48999999999999999,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":8.5879999999999992,"5min":8.5879999999999992,"15min":8.5879999999999992},"last":0.53900000000000003}]},{"osd":4,"last update":"Sat Sep 23 19:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.80200000000000005,"5min":2.4870000000000001,"15min":1.776},"min":{"1min":0.36199999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":2.3199999999999998,"5min":74.221000000000004,"15min":74.221000000000004},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.86399999999999999,"5min":2.8940000000000001,"15min":1.778},"min":{"1min":0.432,"5min":0.33700000000000002,"15min":0.30599999999999999},"max":{"1min":1.603,"5min":73.772999999999996,"15min":73.772999999999996},"last":0.52500000000000002}]},{"osd":6,"last update":"Sat Sep 23 19:41:39 2023","interfaces":[{"interface":"back","average":{"1min":1.2070000000000001,"5min":2.1019999999999999,"15min":1.4379999999999999},"min":{"1min":0.45000000000000001,"5min":0.35899999999999999,"15min":0.32000000000000001},"max":{"1min":4.484,"5min":16.222000000000001,"15min":16.222000000000001},"last":0.48699999999999999},{"interface":"front","average":{"1min":1.468,"5min":2.3010000000000002,"15min":1.649},"min":{"1min":0.63300000000000001,"5min":0.33500000000000002,"15min":0.309},"max":{"1min":5.7270000000000003,"5min":14.178000000000001,"15min":14.178000000000001},"last":0.59099999999999997}]},{"osd":7,"last update":"Sat Sep 23 19:41:05 2023","interfaces":[{"interface":"back","average":{"1min":1.8109999999999999,"5min":1.883,"15min":1.657},"min":{"1min":0.38500000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":5.782,"5min":14.127000000000001,"15min":14.127000000000001},"last":0.628},{"interface":"front","average":{"1min":2.2629999999999999,"5min":1.8169999999999999,"15min":1.5840000000000001},"min":{"1min":0.40699999999999997,"5min":0.307,"15min":0.307},"max":{"1min":4.9260000000000002,"5min":9.0440000000000005,"15min":13.18},"last":0.57999999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":765788160,"data_stored":765788160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1311309824,"data_stored":1311309824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":854523904,"data_stored":854523904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1010302976,"data_stored":1010302976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":651034624,"data_stored":651034624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":670629888,"data_stored":670629888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1302331392,"data_stored":1302331392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":991297536,"data_stored":991297536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T19:42:10.792 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T19:42:10.821 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:10.880 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:10.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:10.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:10.904 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:42:10.963 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:42:10.985 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:11.012 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:42:11.026 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:11.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:11.030 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:11.063 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:42:11.100 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:42:11.131 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:11.144 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:42:11.162 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:11.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:11.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:42:11.186 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:42:11.220 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:42:11.223 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:42:11.225 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:42:11.225 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:42:11.225 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:42:11.225 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:42:11.225 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:42:11.225 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:42:11.225 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:42:11.226 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 27 16 47580 47564 110.086 68.875 0.0113245 0.00907799 2023-09-23T19:42:11.268 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:11.294 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:11.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:11.302 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:42:11.403 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:11.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:11.437 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:11.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:42:11.538 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:11.567 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:11.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:11.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:42:11.671 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:11.699 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:11.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:11.701 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:11.802 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:11.811 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:11 smithi092 bash[58855]: cluster 2023-09-23T19:42:10.210691+0000 mgr.y (mgr.25325) 1041 : cluster 0 pgmap v996: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2023-09-23T19:42:11.811 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:11 smithi092 bash[58855]: audit 2023-09-23T19:42:10.869360+0000 mon.b (mon.1) 48 : audit 0 from='client.? 172.21.15.16:0/3617218801' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:42:11.834 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:11.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:11.837 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:42:11.938 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:11.965 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:11.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:11.967 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:42:12.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:11 smithi016 bash[77995]: cluster 2023-09-23T19:42:10.210691+0000 mgr.y (mgr.25325) 1041 : cluster 0 pgmap v996: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2023-09-23T19:42:12.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:11 smithi016 bash[77995]: audit 2023-09-23T19:42:10.869360+0000 mon.b (mon.1) 48 : audit 0 from='client.? 172.21.15.16:0/3617218801' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:42:12.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:11 smithi016 bash[75004]: cluster 2023-09-23T19:42:10.210691+0000 mgr.y (mgr.25325) 1041 : cluster 0 pgmap v996: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2023-09-23T19:42:12.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:11 smithi016 bash[75004]: audit 2023-09-23T19:42:10.869360+0000 mon.b (mon.1) 48 : audit 0 from='client.? 172.21.15.16:0/3617218801' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:42:12.069 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:12.089 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:12.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:12.091 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:12.191 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:12.221 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:12.224 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 28 16 48828 48812 108.94 78 0.00984696 0.00917078 2023-09-23T19:42:12.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:12.225 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:12.325 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:12.359 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:12.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:12.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:12.464 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:12.492 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:12.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:12.495 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:12.596 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:12.624 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:12.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:12.628 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:12.729 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:12.762 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:12.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:12.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:12.869 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:12.906 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:12.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:12.911 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:13.011 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:13.137 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:13.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:13.142 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:13.224 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 29 16 50184 50168 108.105 84.75 0.0102805 0.00924383 2023-09-23T19:42:13.243 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:13.271 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:13.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:13.276 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:13.377 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:13.402 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:13.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:13.409 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:42:13.509 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:13.543 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:13.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:13.547 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:13.647 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:13.674 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:13.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:13.677 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:13.707 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:13 smithi016 bash[75004]: cluster 2023-09-23T19:42:12.211444+0000 mgr.y (mgr.25325) 1042 : cluster 0 pgmap v997: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-09-23T19:42:13.778 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:13.804 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:13.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:13.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:42:13.908 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:13.931 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:13.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:13.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:42:14.035 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:14.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:13 smithi016 bash[77995]: cluster 2023-09-23T19:42:12.211444+0000 mgr.y (mgr.25325) 1042 : cluster 0 pgmap v997: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-09-23T19:42:14.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:13 smithi016 bash[77995]: audit 2023-09-23T19:42:13.660722+0000 mon.a (mon.0) 1332 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:42:14.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:13 smithi016 bash[75004]: audit 2023-09-23T19:42:13.660722+0000 mon.a (mon.0) 1332 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:42:14.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:13 smithi092 bash[58855]: cluster 2023-09-23T19:42:12.211444+0000 mgr.y (mgr.25325) 1042 : cluster 0 pgmap v997: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-09-23T19:42:14.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:13 smithi092 bash[58855]: audit 2023-09-23T19:42:13.660722+0000 mon.a (mon.0) 1332 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:42:14.075 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:14.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:14.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:42:14.183 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:14.214 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:14.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:14.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:42:14.224 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 30 16 51356 51340 106.943 73.25 0.00942157 0.00934503 2023-09-23T19:42:14.321 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:14.362 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:14.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:14.367 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:42:14.468 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:14.497 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:14.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:14.502 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:14.603 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:14.634 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:14.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:14.641 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:42:14.737 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is set 2023-09-23T19:42:14.742 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:14.765 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:14.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:14.770 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:42:14.775 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:14 smithi016 bash[75004]: audit 2023-09-23T19:42:14.158415+0000 mon.c (mon.2) 182 : audit 1 from='client.? 172.21.15.16:0/1302199705' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:42:14.775 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:14 smithi016 bash[75004]: audit 2023-09-23T19:42:14.159372+0000 mon.a (mon.0) 1333 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:42:14.776 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:14 smithi016 bash[77995]: audit 2023-09-23T19:42:14.158415+0000 mon.c (mon.2) 182 : audit 1 from='client.? 172.21.15.16:0/1302199705' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:42:14.776 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:14 smithi016 bash[77995]: audit 2023-09-23T19:42:14.159372+0000 mon.a (mon.0) 1333 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:42:14.870 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:14.896 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:14.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:14.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:42:15.004 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:15.013 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:14 smithi092 bash[58855]: audit 2023-09-23T19:42:14.158415+0000 mon.c (mon.2) 182 : audit 1 from='client.? 172.21.15.16:0/1302199705' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:42:15.014 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:14 smithi092 bash[58855]: audit 2023-09-23T19:42:14.159372+0000 mon.a (mon.0) 1333 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:42:15.039 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:15.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:15.044 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:42:15.145 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:15.176 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:15.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:15.183 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:42:15.224 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 31 16 52604 52588 106.009 78 0.011496 0.0094276 2023-09-23T19:42:15.283 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:15.318 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:15.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:15.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:42:15.364 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:42:15.423 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:15.468 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:15.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:15.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:15.576 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:15.617 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:15.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:15.621 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:42:15.721 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:15.757 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:15.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:15.765 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:15.865 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:15.900 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:15.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:15.901 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:15.932 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:15 smithi016 bash[75004]: cluster 2023-09-23T19:42:14.211984+0000 mgr.y (mgr.25325) 1043 : cluster 0 pgmap v998: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-09-23T19:42:15.932 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:15 smithi016 bash[75004]: cluster 2023-09-23T19:42:14.703647+0000 mon.a (mon.0) 1334 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:42:15.932 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:15 smithi016 bash[75004]: cluster 2023-09-23T19:42:14.721852+0000 osd.4 (osd.4) 124 : cluster 3 failed to encode map e186 with expected crc 2023-09-23T19:42:15.932 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:15 smithi016 bash[75004]: audit 2023-09-23T19:42:14.722196+0000 mon.a (mon.0) 1335 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:42:15.932 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:15 smithi016 bash[75004]: cluster 2023-09-23T19:42:14.724359+0000 mon.a (mon.0) 1336 : cluster 0 osdmap e186: 8 total, 8 up, 8 in 2023-09-23T19:42:15.932 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:15 smithi016 bash[75004]: cluster 2023-09-23T19:42:14.728011+0000 osd.7 (osd.7) 96 : cluster 3 failed to encode map e186 with expected crc 2023-09-23T19:42:15.932 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:15 smithi016 bash[75004]: cluster 2023-09-23T19:42:14.728011+0000 osd.4 (osd.4) 125 : cluster 3 failed to encode map e186 with expected crc 2023-09-23T19:42:15.932 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:15 smithi016 bash[75004]: cluster 2023-09-23T19:42:14.729240+0000 osd.5 (osd.5) 116 : cluster 3 failed to encode map e186 with expected crc 2023-09-23T19:42:15.932 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:15 smithi016 bash[75004]: cluster 2023-09-23T19:42:14.739202+0000 osd.6 (osd.6) 107 : cluster 3 failed to encode map e186 with expected crc 2023-09-23T19:42:15.933 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:42:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:42:15] "GET /metrics HTTP/1.1" 200 36303 "" "Prometheus/2.33.4" 2023-09-23T19:42:15.933 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:15 smithi016 bash[77995]: cluster 2023-09-23T19:42:14.211984+0000 mgr.y (mgr.25325) 1043 : cluster 0 pgmap v998: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-09-23T19:42:15.933 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:15 smithi016 bash[77995]: cluster 2023-09-23T19:42:14.703647+0000 mon.a (mon.0) 1334 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:42:15.933 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:15 smithi016 bash[77995]: cluster 2023-09-23T19:42:14.721852+0000 osd.4 (osd.4) 124 : cluster 3 failed to encode map e186 with expected crc 2023-09-23T19:42:15.933 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:15 smithi016 bash[77995]: audit 2023-09-23T19:42:14.722196+0000 mon.a (mon.0) 1335 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:42:15.933 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:15 smithi016 bash[77995]: cluster 2023-09-23T19:42:14.724359+0000 mon.a (mon.0) 1336 : cluster 0 osdmap e186: 8 total, 8 up, 8 in 2023-09-23T19:42:15.933 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:15 smithi016 bash[77995]: cluster 2023-09-23T19:42:14.728011+0000 osd.7 (osd.7) 96 : cluster 3 failed to encode map e186 with expected crc 2023-09-23T19:42:15.933 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:15 smithi016 bash[77995]: cluster 2023-09-23T19:42:14.728011+0000 osd.4 (osd.4) 125 : cluster 3 failed to encode map e186 with expected crc 2023-09-23T19:42:15.933 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:15 smithi016 bash[77995]: cluster 2023-09-23T19:42:14.729240+0000 osd.5 (osd.5) 116 : cluster 3 failed to encode map e186 with expected crc 2023-09-23T19:42:15.934 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:15 smithi016 bash[77995]: cluster 2023-09-23T19:42:14.739202+0000 osd.6 (osd.6) 107 : cluster 3 failed to encode map e186 with expected crc 2023-09-23T19:42:16.002 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:16.010 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:15 smithi092 bash[58855]: cluster 2023-09-23T19:42:14.211984+0000 mgr.y (mgr.25325) 1043 : cluster 0 pgmap v998: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-09-23T19:42:16.010 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:15 smithi092 bash[58855]: cluster 2023-09-23T19:42:14.703647+0000 mon.a (mon.0) 1334 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:42:16.010 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:15 smithi092 bash[58855]: cluster 2023-09-23T19:42:14.721852+0000 osd.4 (osd.4) 124 : cluster 3 failed to encode map e186 with expected crc 2023-09-23T19:42:16.011 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:15 smithi092 bash[58855]: audit 2023-09-23T19:42:14.722196+0000 mon.a (mon.0) 1335 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:42:16.011 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:15 smithi092 bash[58855]: cluster 2023-09-23T19:42:14.724359+0000 mon.a (mon.0) 1336 : cluster 0 osdmap e186: 8 total, 8 up, 8 in 2023-09-23T19:42:16.011 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:15 smithi092 bash[58855]: cluster 2023-09-23T19:42:14.728011+0000 osd.7 (osd.7) 96 : cluster 3 failed to encode map e186 with expected crc 2023-09-23T19:42:16.011 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:15 smithi092 bash[58855]: cluster 2023-09-23T19:42:14.728011+0000 osd.4 (osd.4) 125 : cluster 3 failed to encode map e186 with expected crc 2023-09-23T19:42:16.011 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:15 smithi092 bash[58855]: cluster 2023-09-23T19:42:14.729240+0000 osd.5 (osd.5) 116 : cluster 3 failed to encode map e186 with expected crc 2023-09-23T19:42:16.011 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:15 smithi092 bash[58855]: cluster 2023-09-23T19:42:14.739202+0000 osd.6 (osd.6) 107 : cluster 3 failed to encode map e186 with expected crc 2023-09-23T19:42:16.032 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:16.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:16.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:42:16.137 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:16.164 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:16.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:16.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:42:16.224 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 32 16 53825 53809 105.081 76.3125 0.0140684 0.00951046 2023-09-23T19:42:16.269 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:16.297 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:16.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:16.303 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:16.404 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:16.446 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:16.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:16.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:16.549 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:16.574 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:16.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:16.579 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:16.679 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:16.711 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:16.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:16.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:16.815 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:16.848 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:16.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:16.850 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:42:16.953 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:16.981 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:16.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:16.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:16.988 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:42:17.089 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:17.118 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:17.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:17.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:17.222 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:17.224 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 33 16 55040 55024 104.198 75.9375 0.009045 0.00959067 2023-09-23T19:42:17.244 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:17.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:17.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:42:17.347 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:17.374 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:17.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:17.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:17.481 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:17.506 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:17.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:17.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:42:17.609 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:17.640 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:17.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:17.645 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:17.746 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:17.754 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:17 smithi092 bash[58855]: cluster 2023-09-23T19:42:16.212853+0000 mgr.y (mgr.25325) 1044 : cluster 0 pgmap v1000: 89 pgs: 89 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s 2023-09-23T19:42:17.775 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:17.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:17.778 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:42:17.879 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:17.888 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:17 smithi016 bash[77995]: cluster 2023-09-23T19:42:16.212853+0000 mgr.y (mgr.25325) 1044 : cluster 0 pgmap v1000: 89 pgs: 89 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s 2023-09-23T19:42:17.888 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:17 smithi016 bash[75004]: cluster 2023-09-23T19:42:16.212853+0000 mgr.y (mgr.25325) 1044 : cluster 0 pgmap v1000: 89 pgs: 89 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s 2023-09-23T19:42:17.911 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:17.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:17.914 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:42:18.015 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:18.046 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:18.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:18.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:42:18.151 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:18.175 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:18.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:18.178 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:42:18.224 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 34 15 56305 56290 103.46 79.125 0.0088774 0.00965859 2023-09-23T19:42:18.279 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:18.311 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:18.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:18.314 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:18.415 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:18.462 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:18.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:18.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:42:18.508 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T19:42:18.566 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:18.617 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:18.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:18.621 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:18.722 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:18.757 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:18.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:18.761 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:18.862 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:18.896 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:18.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:18.900 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:42:18.947 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:18 smithi016 bash[77995]: audit 2023-09-23T19:42:18.330519+0000 mon.a (mon.0) 1337 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:42:18.947 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:18 smithi016 bash[75004]: audit 2023-09-23T19:42:18.330519+0000 mon.a (mon.0) 1337 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:42:19.001 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:19.011 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:18 smithi092 bash[58855]: audit 2023-09-23T19:42:18.330519+0000 mon.a (mon.0) 1337 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:42:19.029 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:19.031 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:42:19.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:19.034 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:19.135 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:19.182 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:19.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:19.184 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:19.224 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 35 16 57348 57332 102.365 65.125 0.0184504 0.00976295 2023-09-23T19:42:19.285 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:19.310 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:19.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:19.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:42:19.416 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:19.444 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:19.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:19.446 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:19.547 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:19.575 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:19.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:19.581 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:19.682 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:19.712 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:19.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:19.716 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:19.817 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:19.850 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:19.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:19.856 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:42:19.956 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:19.983 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:19.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:19.990 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:20.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:19 smithi092 bash[58855]: cluster 2023-09-23T19:42:18.213459+0000 mgr.y (mgr.25325) 1045 : cluster 0 pgmap v1001: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-09-23T19:42:20.091 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:20.122 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:20.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:20.125 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:20.225 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 36 16 58529 58513 101.571 73.8125 0.00876493 0.00983928 2023-09-23T19:42:20.226 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:20.263 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:20.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:20.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:20.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:19 smithi016 bash[75004]: cluster 2023-09-23T19:42:18.213459+0000 mgr.y (mgr.25325) 1045 : cluster 0 pgmap v1001: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-09-23T19:42:20.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:19 smithi016 bash[77995]: cluster 2023-09-23T19:42:18.213459+0000 mgr.y (mgr.25325) 1045 : cluster 0 pgmap v1001: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-09-23T19:42:20.366 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:20.401 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:20.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:20.406 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:42:20.507 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:20.544 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:20.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:20.547 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:20.647 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:20.673 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:20.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:20.679 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:42:20.780 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:20.812 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:20.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:20.814 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:42:20.915 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:20.941 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:20.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:20.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:42:21.044 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:21.074 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:21.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:21.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:42:21.181 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:21.209 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:21.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:21.211 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:21.225 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 37 16 59725 59709 100.846 74.75 0.00404476 0.00990873 2023-09-23T19:42:21.312 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:21.341 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:21.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:21.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:42:21.400 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:42:21.447 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:21.473 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:21.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:21.476 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:21.577 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:21.609 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:21.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:21.611 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:42:21.712 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:21.744 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:21.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:21.748 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:42:21.848 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:21.858 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:21 smithi092 bash[58855]: cluster 2023-09-23T19:42:20.213999+0000 mgr.y (mgr.25325) 1046 : cluster 0 pgmap v1002: 89 pgs: 89 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-09-23T19:42:21.873 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:21.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:21.881 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:42:21.981 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:21.991 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:21 smithi016 bash[75004]: cluster 2023-09-23T19:42:20.213999+0000 mgr.y (mgr.25325) 1046 : cluster 0 pgmap v1002: 89 pgs: 89 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-09-23T19:42:21.991 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:21 smithi016 bash[77995]: cluster 2023-09-23T19:42:20.213999+0000 mgr.y (mgr.25325) 1046 : cluster 0 pgmap v1002: 89 pgs: 89 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-09-23T19:42:22.005 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:22.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:22.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:22.112 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:22.154 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:22.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:22.158 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:42:22.225 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 38 16 60787 60771 99.9392 66.375 0.015675 0.0100003 2023-09-23T19:42:22.258 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:22.284 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:22.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:22.288 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:42:22.389 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:22.418 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:22.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:22.422 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:22.523 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:22.557 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:22.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:22.560 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:22.661 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:22.700 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:22.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:22.705 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:22.805 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:22.830 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:22.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:22.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:42:22.933 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:22.961 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:22.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:22.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:42:23.064 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:23.090 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:23.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:23.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:23.105 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:42:23.194 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:23.218 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:23.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:23.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:42:23.225 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 39 16 61880 61864 99.1281 68.3125 0.0260545 0.0100816 2023-09-23T19:42:23.321 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:23.353 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:23.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:23.357 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:42:23.457 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:23.492 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:23.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:23.501 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:23.602 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:23.633 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:23.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:23.637 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:23.737 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:23.744 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:23 smithi016 bash[75004]: cluster 2023-09-23T19:42:22.214623+0000 mgr.y (mgr.25325) 1047 : cluster 0 pgmap v1003: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2023-09-23T19:42:23.762 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:23.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:23.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:42:23.866 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:23.907 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:23.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:23.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:24.011 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:24.018 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:23 smithi092 bash[58855]: cluster 2023-09-23T19:42:22.214623+0000 mgr.y (mgr.25325) 1047 : cluster 0 pgmap v1003: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2023-09-23T19:42:24.035 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:24.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:23 smithi016 bash[77995]: cluster 2023-09-23T19:42:22.214623+0000 mgr.y (mgr.25325) 1047 : cluster 0 pgmap v1003: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2023-09-23T19:42:24.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:24.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:42:24.138 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:24.166 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:24.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:24.172 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:24.225 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-09-23T19:42:24.222226+0000 min lat: 0.00130197 max lat: 0.165975 avg lat: 0.0101854 2023-09-23T19:42:24.225 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:42:24.225 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 40 16 62823 62807 98.1232 58.9375 0.0116002 0.0101854 2023-09-23T19:42:24.273 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:24.299 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:24.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:24.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:42:24.405 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:24.451 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:24.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:24.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:42:24.557 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:24.582 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:24.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:24.584 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:24.685 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:24.713 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:24.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:24.716 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:24.817 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:24.848 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:24.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:24.852 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:24.953 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:25.010 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:25.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:25.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:25.120 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:25.145 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:25.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:25.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:25.225 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 41 16 63808 63792 97.2314 61.5625 0.0214387 0.0102795 2023-09-23T19:42:25.226 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-09-23T19:42:25.226 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [], 2023-09-23T19:42:25.227 INFO:teuthology.orchestra.run.smithi016.stdout: "complaint_time": 30, 2023-09-23T19:42:25.227 INFO:teuthology.orchestra.run.smithi016.stdout: "num_blocked_ops": 0 2023-09-23T19:42:25.227 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-09-23T19:42:25.249 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:25.281 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:25.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:25.286 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:42:25.387 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:25.426 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:25.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:25.427 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:42:25.528 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:25.560 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:25.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:25.564 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:42:25.665 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:25.702 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:25.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:25.705 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:25.789 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:42:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:42:25] "GET /metrics HTTP/1.1" 200 36303 "" "Prometheus/2.33.4" 2023-09-23T19:42:25.806 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:25.841 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:25.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:25.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:25.951 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:25.977 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:25.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:25.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:42:26.081 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:26.108 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:26.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:26.111 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:26.134 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:25 smithi016 bash[75004]: cluster 2023-09-23T19:42:24.215144+0000 mgr.y (mgr.25325) 1048 : cluster 0 pgmap v1004: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-09-23T19:42:26.134 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:25 smithi016 bash[75004]: audit 2023-09-23T19:42:24.804313+0000 mon.a (mon.0) 1338 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:42:26.134 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:25 smithi016 bash[75004]: audit 2023-09-23T19:42:24.817966+0000 mon.a (mon.0) 1339 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:42:26.134 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:25 smithi016 bash[75004]: audit 2023-09-23T19:42:24.840472+0000 mon.a (mon.0) 1340 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:42:26.134 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:25 smithi016 bash[75004]: audit 2023-09-23T19:42:24.849450+0000 mon.a (mon.0) 1341 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:42:26.134 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:25 smithi016 bash[75004]: audit 2023-09-23T19:42:25.530127+0000 mon.a (mon.0) 1342 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:42:26.135 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:25 smithi016 bash[75004]: audit 2023-09-23T19:42:25.531168+0000 mon.a (mon.0) 1343 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:42:26.135 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:25 smithi016 bash[75004]: audit 2023-09-23T19:42:25.542721+0000 mon.a (mon.0) 1344 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:42:26.135 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:25 smithi016 bash[77995]: cluster 2023-09-23T19:42:24.215144+0000 mgr.y (mgr.25325) 1048 : cluster 0 pgmap v1004: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-09-23T19:42:26.135 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:25 smithi016 bash[77995]: audit 2023-09-23T19:42:24.804313+0000 mon.a (mon.0) 1338 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:42:26.135 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:25 smithi016 bash[77995]: audit 2023-09-23T19:42:24.817966+0000 mon.a (mon.0) 1339 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:42:26.135 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:25 smithi016 bash[77995]: audit 2023-09-23T19:42:24.840472+0000 mon.a (mon.0) 1340 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:42:26.135 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:25 smithi016 bash[77995]: audit 2023-09-23T19:42:24.849450+0000 mon.a (mon.0) 1341 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:42:26.135 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:25 smithi016 bash[77995]: audit 2023-09-23T19:42:25.530127+0000 mon.a (mon.0) 1342 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:42:26.135 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:25 smithi016 bash[77995]: audit 2023-09-23T19:42:25.531168+0000 mon.a (mon.0) 1343 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:42:26.135 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:25 smithi016 bash[77995]: audit 2023-09-23T19:42:25.542721+0000 mon.a (mon.0) 1344 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:42:26.212 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:26.227 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 42 16 64875 64859 96.504 66.6875 0.00890329 0.0103563 2023-09-23T19:42:26.235 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:26.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:26.239 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:42:26.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:25 smithi092 bash[58855]: cluster 2023-09-23T19:42:24.215144+0000 mgr.y (mgr.25325) 1048 : cluster 0 pgmap v1004: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-09-23T19:42:26.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:25 smithi092 bash[58855]: audit 2023-09-23T19:42:24.804313+0000 mon.a (mon.0) 1338 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:42:26.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:25 smithi092 bash[58855]: audit 2023-09-23T19:42:24.817966+0000 mon.a (mon.0) 1339 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:42:26.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:25 smithi092 bash[58855]: audit 2023-09-23T19:42:24.840472+0000 mon.a (mon.0) 1340 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:42:26.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:25 smithi092 bash[58855]: audit 2023-09-23T19:42:24.849450+0000 mon.a (mon.0) 1341 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:42:26.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:25 smithi092 bash[58855]: audit 2023-09-23T19:42:25.530127+0000 mon.a (mon.0) 1342 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:42:26.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:25 smithi092 bash[58855]: audit 2023-09-23T19:42:25.531168+0000 mon.a (mon.0) 1343 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:42:26.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:25 smithi092 bash[58855]: audit 2023-09-23T19:42:25.542721+0000 mon.a (mon.0) 1344 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:42:26.340 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:26.367 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:26.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:26.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:26.477 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:26.510 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:26.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:26.515 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:26.615 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:26.655 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:26.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:26.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:42:26.765 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:26.795 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:26.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:26.800 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:26.901 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:26.930 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:26.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:26.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:27.035 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:27.062 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:27.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:27.071 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:42:27.171 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:27.201 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:27.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:27.204 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:42:27.225 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 43 16 65977 65961 95.8613 68.875 0.0040462 0.010417 2023-09-23T19:42:27.305 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:27.335 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:27.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:27.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:27.440 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:27.469 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:27.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:27.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:42:27.573 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:27.605 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:27.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:27.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:27.708 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:27.731 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:27.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:27.736 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:42:27.837 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:27.862 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:27.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:27.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:42:27.967 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:27.976 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:27 smithi016 bash[75004]: cluster 2023-09-23T19:42:26.215830+0000 mgr.y (mgr.25325) 1049 : cluster 0 pgmap v1005: 89 pgs: 89 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-09-23T19:42:27.976 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:27 smithi016 bash[77995]: cluster 2023-09-23T19:42:26.215830+0000 mgr.y (mgr.25325) 1049 : cluster 0 pgmap v1005: 89 pgs: 89 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-09-23T19:42:27.994 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:27.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:27.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:28.099 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:28.134 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:28.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:28.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:42:28.225 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 44 16 67013 66997 95.1541 64.75 0.0109211 0.0105026 2023-09-23T19:42:28.238 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:28.256 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:28.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:28.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:28.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:27 smithi092 bash[58855]: cluster 2023-09-23T19:42:26.215830+0000 mgr.y (mgr.25325) 1049 : cluster 0 pgmap v1005: 89 pgs: 89 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-09-23T19:42:28.358 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:28.390 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:28.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:28.396 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:28.496 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:28.521 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:28.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:28.523 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:28.624 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:28.655 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:28.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:28.658 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:42:28.759 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:28.788 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:28.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:28.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:28.891 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:28.936 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:28.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:28.940 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:29.041 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:29.068 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:29.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:29.072 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:29.173 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:29.206 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:29.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:29.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:29.225 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 45 16 67886 67870 94.252 54.5625 0.0171277 0.0106039 2023-09-23T19:42:29.312 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:29.334 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:29.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:29.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:42:29.437 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:29.476 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:29.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:29.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:42:29.481 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T19:42:29.579 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:29.621 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:29.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:29.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:29.727 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:29.759 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:29.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:29.763 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:42:29.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:29 smithi016 bash[77995]: cluster 2023-09-23T19:42:28.216218+0000 mgr.y (mgr.25325) 1050 : cluster 0 pgmap v1006: 89 pgs: 89 active+clean; 3.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-09-23T19:42:29.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:29 smithi016 bash[77995]: audit 2023-09-23T19:42:28.666591+0000 mon.a (mon.0) 1345 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:42:29.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:29 smithi016 bash[77995]: audit 2023-09-23T19:42:28.668218+0000 mon.a (mon.0) 1346 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:42:29.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:29 smithi016 bash[75004]: cluster 2023-09-23T19:42:28.216218+0000 mgr.y (mgr.25325) 1050 : cluster 0 pgmap v1006: 89 pgs: 89 active+clean; 3.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-09-23T19:42:29.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:29 smithi016 bash[75004]: audit 2023-09-23T19:42:28.666591+0000 mon.a (mon.0) 1345 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:42:29.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:29 smithi016 bash[75004]: audit 2023-09-23T19:42:28.668218+0000 mon.a (mon.0) 1346 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:42:29.864 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:29.895 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:29.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:29.900 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:30.000 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:30.008 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:29 smithi092 bash[58855]: cluster 2023-09-23T19:42:28.216218+0000 mgr.y (mgr.25325) 1050 : cluster 0 pgmap v1006: 89 pgs: 89 active+clean; 3.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-09-23T19:42:30.008 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:29 smithi092 bash[58855]: audit 2023-09-23T19:42:28.666591+0000 mon.a (mon.0) 1345 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:42:30.008 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:29 smithi092 bash[58855]: audit 2023-09-23T19:42:28.668218+0000 mon.a (mon.0) 1346 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:42:30.026 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:30.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:30.031 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:42:30.132 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:30.166 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:30.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:30.169 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:42:30.226 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 46 16 68947 68931 93.6445 66.3125 0.0150282 0.0106732 2023-09-23T19:42:30.270 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:30.300 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:30.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:30.303 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:30.404 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:30.435 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:30.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:30.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:42:30.542 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:30.582 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:30.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:30.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:42:30.684 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:30.728 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:30.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:30.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:42:30.832 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:30.865 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:30.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:30.870 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:42:30.971 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:30.993 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:30.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:30.995 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:31.096 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:31.124 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:31.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:31.128 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:31.226 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 47 16 69916 69900 92.9405 60.5625 0.00989414 0.0107536 2023-09-23T19:42:31.228 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:31.260 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:31.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:31.265 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:31.366 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:31.389 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:31.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:31.393 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:31.494 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:31.517 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:31.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:31.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:31.619 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:31.642 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:31.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:31.646 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:42:31.675 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:31 smithi016 bash[77995]: cluster 2023-09-23T19:42:30.216853+0000 mgr.y (mgr.25325) 1051 : cluster 0 pgmap v1007: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-23T19:42:31.675 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:31 smithi016 bash[75004]: cluster 2023-09-23T19:42:30.216853+0000 mgr.y (mgr.25325) 1051 : cluster 0 pgmap v1007: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-23T19:42:31.747 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:31.779 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:31.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:31.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:42:31.883 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:31.892 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:31 smithi092 bash[58855]: cluster 2023-09-23T19:42:30.216853+0000 mgr.y (mgr.25325) 1051 : cluster 0 pgmap v1007: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-23T19:42:31.912 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:31.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:31.919 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:42:32.019 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:32.048 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:32.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:32.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:32.152 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:32.181 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:32.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:32.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:32.226 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 48 16 71013 70997 92.4325 68.5625 0.0112032 0.0108127 2023-09-23T19:42:32.288 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:32.319 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:32.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:32.323 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:32.423 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:32.453 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:32.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:32.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:42:32.558 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:32.579 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:32.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:32.581 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:32.682 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:32.719 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:32.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:32.722 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:42:32.822 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:32.855 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:32.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:32.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:42:32.960 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:32.986 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:32.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:32.992 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:42:33.093 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:33.123 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:33.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:33.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:33.226 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:33.226 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 49 16 71871 71855 91.6404 53.625 0.0936849 0.0108976 2023-09-23T19:42:33.257 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:33.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:33.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:42:33.367 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:33.404 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:33.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:33.407 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:33.508 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:33.537 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:33.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:33.540 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:33.641 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:33.673 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:33.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:33.678 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:33.779 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:33.817 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:33.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:33.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:42:33.921 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:33.945 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:33.951 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:33 smithi092 bash[58855]: cluster 2023-09-23T19:42:32.217706+0000 mgr.y (mgr.25325) 1052 : cluster 0 pgmap v1008: 89 pgs: 89 active+clean; 4.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-09-23T19:42:33.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:33.952 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:42:34.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:33 smithi016 bash[75004]: cluster 2023-09-23T19:42:32.217706+0000 mgr.y (mgr.25325) 1052 : cluster 0 pgmap v1008: 89 pgs: 89 active+clean; 4.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-09-23T19:42:34.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:33 smithi016 bash[77995]: cluster 2023-09-23T19:42:32.217706+0000 mgr.y (mgr.25325) 1052 : cluster 0 pgmap v1008: 89 pgs: 89 active+clean; 4.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-09-23T19:42:34.053 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:34.103 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:34.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:34.105 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:42:34.206 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:34.226 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 50 16 72889 72873 91.0799 63.625 0.0108421 0.0109726 2023-09-23T19:42:34.233 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:34.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:34.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:34.340 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:34.365 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:34.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:34.369 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:42:34.469 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:34.488 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:34.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:34.490 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:42:34.591 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:34.623 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:34.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:34.624 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:42:34.725 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:34.753 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:34.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:34.755 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:42:34.856 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:34.863 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-09-23T19:42:34.876 INFO:teuthology.orchestra.run.smithi016.stderr:dumped all 2023-09-23T19:42:34.894 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:34.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:34.900 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:35.001 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:35.022 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:35.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:35.026 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:35.126 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:35.149 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:35.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:35.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:42:35.226 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 51 16 73777 73761 90.3822 55.5 0.00748553 0.011058 2023-09-23T19:42:35.252 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:35.285 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:35.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:35.288 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:42:35.389 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:35.415 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:35.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:35.417 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:42:35.518 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:35.545 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:35.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:35.548 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:42:35.649 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:35.681 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:35.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:35.683 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:42:35.784 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:35.816 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:35.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:35.824 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:35.861 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:42:35.893 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:35 smithi016 bash[75004]: cluster 2023-09-23T19:42:34.218250+0000 mgr.y (mgr.25325) 1053 : cluster 0 pgmap v1009: 89 pgs: 89 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 56 MiB/s wr, 899 op/s 2023-09-23T19:42:35.893 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:35 smithi016 bash[75004]: audit 2023-09-23T19:42:34.841499+0000 mgr.y (mgr.25325) 1054 : audit 0 from='client.46226 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:42:35.894 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:35 smithi016 bash[77995]: cluster 2023-09-23T19:42:34.218250+0000 mgr.y (mgr.25325) 1053 : cluster 0 pgmap v1009: 89 pgs: 89 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 56 MiB/s wr, 899 op/s 2023-09-23T19:42:35.894 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:35 smithi016 bash[77995]: audit 2023-09-23T19:42:34.841499+0000 mgr.y (mgr.25325) 1054 : audit 0 from='client.46226 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:42:35.894 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:42:35 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:42:35] "GET /metrics HTTP/1.1" 200 36302 "" "Prometheus/2.33.4" 2023-09-23T19:42:35.906 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:42:35.925 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:35.933 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:35 smithi092 bash[58855]: cluster 2023-09-23T19:42:34.218250+0000 mgr.y (mgr.25325) 1053 : cluster 0 pgmap v1009: 89 pgs: 89 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 56 MiB/s wr, 899 op/s 2023-09-23T19:42:35.933 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:35 smithi092 bash[58855]: audit 2023-09-23T19:42:34.841499+0000 mgr.y (mgr.25325) 1054 : audit 0 from='client.46226 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:42:35.947 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:42:35.949 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:35.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:35.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:42:35.985 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:42:36.022 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:42:36.052 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:36.058 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:42:36.078 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:36.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:36.084 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:36.099 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:42:36.128 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:42:36.131 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:42:36.131 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:42:36.132 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:42:36.132 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:42:36.132 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:42:36.132 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:42:36.132 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:42:36.132 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:42:36.184 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:36.215 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:36.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:36.218 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:42:36.226 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 52 16 74700 74684 89.7534 57.6875 0.0151063 0.011136 2023-09-23T19:42:36.319 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:36.356 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:36.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:36.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:42:36.461 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:36.490 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:36.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:36.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:42:36.592 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:36.629 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:36.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:36.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:36.734 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:36.759 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:36.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:36.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:36.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:36 smithi092 bash[58855]: audit 2023-09-23T19:42:35.831466+0000 mon.c (mon.2) 183 : audit 0 from='client.? 172.21.15.16:0/2646269066' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:42:36.865 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:36.895 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:36.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:36.904 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:37.004 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:37.014 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:36 smithi016 bash[75004]: audit 2023-09-23T19:42:35.831466+0000 mon.c (mon.2) 183 : audit 0 from='client.? 172.21.15.16:0/2646269066' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:42:37.014 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:36 smithi016 bash[77995]: audit 2023-09-23T19:42:35.831466+0000 mon.c (mon.2) 183 : audit 0 from='client.? 172.21.15.16:0/2646269066' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:42:37.033 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:37.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:37.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:37.138 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:37.161 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:37.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:37.165 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:42:37.226 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 53 16 75758 75742 89.3075 66.125 0.00855499 0.0111909 2023-09-23T19:42:37.267 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:37.286 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:37.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:37.288 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:37.389 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:37.412 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:37.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:37.419 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:42:37.519 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:37.555 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:37.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:37.558 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:37.618 INFO:teuthology.orchestra.run.smithi016.stdout:{"pg_ready":true,"pg_map":{"version":1009,"stamp":"2023-09-23T19:42:34.217946+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4556986272,"num_objects":69549,"num_object_clones":0,"num_object_copies":208647,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":69549,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":69786,"num_write_kb":4452046,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":69720,"ondisk_log_size":69720,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":18652748,"kb_used_data":13372172,"kb_used_omap":20,"kb_used_meta":5280491,"kb_avail":731308468,"statfs":{"total":767960285184,"available":748859871232,"internally_reserved":0,"allocated":13693104128,"data_stored":13688154415,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":5407223438},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[6,2,3,4,2,6,0,7],"upper_bound":256},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":707657728,"num_objects":10798,"num_object_clones":0,"num_object_copies":32394,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":10798,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10798,"num_write_kb":691072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003766"},"pg_stats":[{"pgid":"3.1a","version":"59'1","reported_seq":182,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.565166+0000","last_change":"2023-09-23T19:38:43.897548+0000","last_active":"2023-09-23T19:41:46.565166+0000","last_peered":"2023-09-23T19:41:46.565166+0000","last_clean":"2023-09-23T19:41:46.565166+0000","last_became_active":"2023-09-23T19:38:43.896868+0000","last_became_peered":"2023-09-23T19:38:43.896868+0000","last_unstale":"2023-09-23T19:41:46.565166+0000","last_undegraded":"2023-09-23T19:41:46.565166+0000","last_fullsized":"2023-09-23T19:41:46.565166+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":142,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.525275+0000","last_change":"2023-09-23T19:32:49.731128+0000","last_active":"2023-09-23T19:41:46.525275+0000","last_peered":"2023-09-23T19:41:46.525275+0000","last_clean":"2023-09-23T19:41:46.525275+0000","last_became_active":"2023-09-23T19:32:49.730757+0000","last_became_peered":"2023-09-23T19:32:49.730757+0000","last_unstale":"2023-09-23T19:41:46.525275+0000","last_undegraded":"2023-09-23T19:41:46.525275+0000","last_fullsized":"2023-09-23T19:41:46.525275+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:40.183605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":131,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:42:05.277142+0000","last_change":"2023-09-23T19:28:07.890462+0000","last_active":"2023-09-23T19:42:05.277142+0000","last_peered":"2023-09-23T19:42:05.277142+0000","last_clean":"2023-09-23T19:42:05.277142+0000","last_became_active":"2023-09-23T19:28:07.890084+0000","last_became_peered":"2023-09-23T19:28:07.890084+0000","last_unstale":"2023-09-23T19:42:05.277142+0000","last_undegraded":"2023-09-23T19:42:05.277142+0000","last_fullsized":"2023-09-23T19:42:05.277142+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:50.458218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":149,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:42:13.465798+0000","last_change":"2023-09-23T19:28:07.884498+0000","last_active":"2023-09-23T19:42:13.465798+0000","last_peered":"2023-09-23T19:42:13.465798+0000","last_clean":"2023-09-23T19:42:13.465798+0000","last_became_active":"2023-09-23T19:28:07.884102+0000","last_became_peered":"2023-09-23T19:28:07.884102+0000","last_unstale":"2023-09-23T19:42:13.465798+0000","last_undegraded":"2023-09-23T19:42:13.465798+0000","last_fullsized":"2023-09-23T19:42:13.465798+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:05:26.348041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":160,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T19:42:14.731664+0000","last_change":"2023-09-23T19:32:49.729961+0000","last_active":"2023-09-23T19:42:14.731664+0000","last_peered":"2023-09-23T19:42:14.731664+0000","last_clean":"2023-09-23T19:42:14.731664+0000","last_became_active":"2023-09-23T19:32:49.729251+0000","last_became_peered":"2023-09-23T19:32:49.729251+0000","last_unstale":"2023-09-23T19:42:14.731664+0000","last_undegraded":"2023-09-23T19:42:14.731664+0000","last_fullsized":"2023-09-23T19:42:14.731664+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:20:56.595732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":151,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.560932+0000","last_change":"2023-09-23T19:22:42.150582+0000","last_active":"2023-09-23T19:41:46.560932+0000","last_peered":"2023-09-23T19:41:46.560932+0000","last_clean":"2023-09-23T19:41:46.560932+0000","last_became_active":"2023-09-23T19:22:42.150229+0000","last_became_peered":"2023-09-23T19:22:42.150229+0000","last_unstale":"2023-09-23T19:41:46.560932+0000","last_undegraded":"2023-09-23T19:41:46.560932+0000","last_fullsized":"2023-09-23T19:41:46.560932+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":147,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T19:42:14.732439+0000","last_change":"2023-09-23T19:28:07.887141+0000","last_active":"2023-09-23T19:42:14.732439+0000","last_peered":"2023-09-23T19:42:14.732439+0000","last_clean":"2023-09-23T19:42:14.732439+0000","last_became_active":"2023-09-23T19:28:07.886494+0000","last_became_peered":"2023-09-23T19:28:07.886494+0000","last_unstale":"2023-09-23T19:42:14.732439+0000","last_undegraded":"2023-09-23T19:42:14.732439+0000","last_fullsized":"2023-09-23T19:42:14.732439+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:24:07.615051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":143,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:42:10.341068+0000","last_change":"2023-09-23T19:32:49.729765+0000","last_active":"2023-09-23T19:42:10.341068+0000","last_peered":"2023-09-23T19:42:10.341068+0000","last_clean":"2023-09-23T19:42:10.341068+0000","last_became_active":"2023-09-23T19:32:49.729007+0000","last_became_peered":"2023-09-23T19:32:49.729007+0000","last_unstale":"2023-09-23T19:42:10.341068+0000","last_undegraded":"2023-09-23T19:42:10.341068+0000","last_fullsized":"2023-09-23T19:42:10.341068+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:59:31.183753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":153,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.534771+0000","last_change":"2023-09-23T19:22:42.151617+0000","last_active":"2023-09-23T19:41:46.534771+0000","last_peered":"2023-09-23T19:41:46.534771+0000","last_clean":"2023-09-23T19:41:46.534771+0000","last_became_active":"2023-09-23T19:22:42.151217+0000","last_became_peered":"2023-09-23T19:22:42.151217+0000","last_unstale":"2023-09-23T19:41:46.534771+0000","last_undegraded":"2023-09-23T19:41:46.534771+0000","last_fullsized":"2023-09-23T19:41:46.534771+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":126,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.562329+0000","last_change":"2023-09-23T19:32:49.736864+0000","last_active":"2023-09-23T19:41:46.562329+0000","last_peered":"2023-09-23T19:41:46.562329+0000","last_clean":"2023-09-23T19:41:46.562329+0000","last_became_active":"2023-09-23T19:32:49.736523+0000","last_became_peered":"2023-09-23T19:32:49.736523+0000","last_unstale":"2023-09-23T19:41:46.562329+0000","last_undegraded":"2023-09-23T19:41:46.562329+0000","last_fullsized":"2023-09-23T19:41:46.562329+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":155,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.564461+0000","last_change":"2023-09-23T19:28:07.892835+0000","last_active":"2023-09-23T19:41:46.564461+0000","last_peered":"2023-09-23T19:41:46.564461+0000","last_clean":"2023-09-23T19:41:46.564461+0000","last_became_active":"2023-09-23T19:28:07.892338+0000","last_became_peered":"2023-09-23T19:28:07.892338+0000","last_unstale":"2023-09-23T19:41:46.564461+0000","last_undegraded":"2023-09-23T19:41:46.564461+0000","last_fullsized":"2023-09-23T19:41:46.564461+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":146,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.525134+0000","last_change":"2023-09-23T19:38:43.890137+0000","last_active":"2023-09-23T19:41:46.525134+0000","last_peered":"2023-09-23T19:41:46.525134+0000","last_clean":"2023-09-23T19:41:46.525134+0000","last_became_active":"2023-09-23T19:38:43.889800+0000","last_became_peered":"2023-09-23T19:38:43.889800+0000","last_unstale":"2023-09-23T19:41:46.525134+0000","last_undegraded":"2023-09-23T19:41:46.525134+0000","last_fullsized":"2023-09-23T19:41:46.525134+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:59:20.468913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'23","reported_seq":159,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.565561+0000","last_change":"2023-09-23T19:05:52.107727+0000","last_active":"2023-09-23T19:41:46.565561+0000","last_peered":"2023-09-23T19:41:46.565561+0000","last_clean":"2023-09-23T19:41:46.565561+0000","last_became_active":"2023-09-23T19:05:52.107378+0000","last_became_peered":"2023-09-23T19:05:52.107378+0000","last_unstale":"2023-09-23T19:41:46.565561+0000","last_undegraded":"2023-09-23T19:41:46.565561+0000","last_fullsized":"2023-09-23T19:41:46.565561+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":150,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.565491+0000","last_change":"2023-09-23T19:22:42.148497+0000","last_active":"2023-09-23T19:41:46.565491+0000","last_peered":"2023-09-23T19:41:46.565491+0000","last_clean":"2023-09-23T19:41:46.565491+0000","last_became_active":"2023-09-23T19:22:42.148147+0000","last_became_peered":"2023-09-23T19:22:42.148147+0000","last_unstale":"2023-09-23T19:41:46.565491+0000","last_undegraded":"2023-09-23T19:41:46.565491+0000","last_fullsized":"2023-09-23T19:41:46.565491+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":169,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.563290+0000","last_change":"2023-09-23T19:38:43.897378+0000","last_active":"2023-09-23T19:41:46.563290+0000","last_peered":"2023-09-23T19:41:46.563290+0000","last_clean":"2023-09-23T19:41:46.563290+0000","last_became_active":"2023-09-23T19:38:43.897012+0000","last_became_peered":"2023-09-23T19:38:43.897012+0000","last_unstale":"2023-09-23T19:41:46.563290+0000","last_undegraded":"2023-09-23T19:41:46.563290+0000","last_fullsized":"2023-09-23T19:41:46.563290+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":143,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T19:42:14.725908+0000","last_change":"2023-09-23T19:32:49.730549+0000","last_active":"2023-09-23T19:42:14.725908+0000","last_peered":"2023-09-23T19:42:14.725908+0000","last_clean":"2023-09-23T19:42:14.725908+0000","last_became_active":"2023-09-23T19:32:49.730105+0000","last_became_peered":"2023-09-23T19:32:49.730105+0000","last_unstale":"2023-09-23T19:42:14.725908+0000","last_undegraded":"2023-09-23T19:42:14.725908+0000","last_fullsized":"2023-09-23T19:42:14.725908+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:49.751893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":147,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.552050+0000","last_change":"2023-09-23T19:28:07.884394+0000","last_active":"2023-09-23T19:41:46.552050+0000","last_peered":"2023-09-23T19:41:46.552050+0000","last_clean":"2023-09-23T19:41:46.552050+0000","last_became_active":"2023-09-23T19:28:07.883944+0000","last_became_peered":"2023-09-23T19:28:07.883944+0000","last_unstale":"2023-09-23T19:41:46.552050+0000","last_undegraded":"2023-09-23T19:41:46.552050+0000","last_fullsized":"2023-09-23T19:41:46.552050+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:08:17.608926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":152,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.534700+0000","last_change":"2023-09-23T19:28:07.891056+0000","last_active":"2023-09-23T19:41:46.534700+0000","last_peered":"2023-09-23T19:41:46.534700+0000","last_clean":"2023-09-23T19:41:46.534700+0000","last_became_active":"2023-09-23T19:28:07.890625+0000","last_became_peered":"2023-09-23T19:28:07.890625+0000","last_unstale":"2023-09-23T19:41:46.534700+0000","last_undegraded":"2023-09-23T19:41:46.534700+0000","last_fullsized":"2023-09-23T19:41:46.534700+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":137,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.562434+0000","last_change":"2023-09-23T19:05:52.114186+0000","last_active":"2023-09-23T19:41:46.562434+0000","last_peered":"2023-09-23T19:41:46.562434+0000","last_clean":"2023-09-23T19:41:46.562434+0000","last_became_active":"2023-09-23T19:05:52.113635+0000","last_became_peered":"2023-09-23T19:05:52.113635+0000","last_unstale":"2023-09-23T19:41:46.562434+0000","last_undegraded":"2023-09-23T19:41:46.562434+0000","last_fullsized":"2023-09-23T19:41:46.562434+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":147,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T19:42:14.737042+0000","last_change":"2023-09-23T19:28:07.884512+0000","last_active":"2023-09-23T19:42:14.737042+0000","last_peered":"2023-09-23T19:42:14.737042+0000","last_clean":"2023-09-23T19:42:14.737042+0000","last_became_active":"2023-09-23T19:28:07.884155+0000","last_became_peered":"2023-09-23T19:28:07.884155+0000","last_unstale":"2023-09-23T19:42:14.737042+0000","last_undegraded":"2023-09-23T19:42:14.737042+0000","last_fullsized":"2023-09-23T19:42:14.737042+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:10.654958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":160,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.561097+0000","last_change":"2023-09-23T19:18:09.010195+0000","last_active":"2023-09-23T19:41:46.561097+0000","last_peered":"2023-09-23T19:41:46.561097+0000","last_clean":"2023-09-23T19:41:46.561097+0000","last_became_active":"2023-09-23T19:18:09.009687+0000","last_became_peered":"2023-09-23T19:18:09.009687+0000","last_unstale":"2023-09-23T19:41:46.561097+0000","last_undegraded":"2023-09-23T19:41:46.561097+0000","last_fullsized":"2023-09-23T19:41:46.561097+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":151,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.557481+0000","last_change":"2023-09-23T19:28:07.885265+0000","last_active":"2023-09-23T19:41:46.557481+0000","last_peered":"2023-09-23T19:41:46.557481+0000","last_clean":"2023-09-23T19:41:46.557481+0000","last_became_active":"2023-09-23T19:28:07.884353+0000","last_became_peered":"2023-09-23T19:28:07.884353+0000","last_unstale":"2023-09-23T19:41:46.557481+0000","last_undegraded":"2023-09-23T19:41:46.557481+0000","last_fullsized":"2023-09-23T19:41:46.557481+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:56.325352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"186'4342","reported_seq":4354,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T19:42:30.865356+0000","last_change":"2023-09-23T19:41:24.846275+0000","last_active":"2023-09-23T19:42:30.865356+0000","last_peered":"2023-09-23T19:42:30.865356+0000","last_clean":"2023-09-23T19:42:30.865356+0000","last_became_active":"2023-09-23T19:41:24.845875+0000","last_became_peered":"2023-09-23T19:41:24.845875+0000","last_unstale":"2023-09-23T19:42:30.865356+0000","last_undegraded":"2023-09-23T19:42:30.865356+0000","last_fullsized":"2023-09-23T19:42:30.865356+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":180,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_clean_scrub_stamp":"2023-09-23T19:41:23.799472+0000","objects_scrubbed":0,"log_size":4342,"log_dups_size":0,"ondisk_log_size":4342,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":284557312,"num_objects":4342,"num_object_clones":0,"num_object_copies":13026,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4342,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4342,"num_write_kb":277888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":153,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.563424+0000","last_change":"2023-09-23T19:18:09.035116+0000","last_active":"2023-09-23T19:41:46.563424+0000","last_peered":"2023-09-23T19:41:46.563424+0000","last_clean":"2023-09-23T19:41:46.563424+0000","last_became_active":"2023-09-23T19:18:09.034416+0000","last_became_peered":"2023-09-23T19:18:09.034416+0000","last_unstale":"2023-09-23T19:41:46.563424+0000","last_undegraded":"2023-09-23T19:41:46.563424+0000","last_fullsized":"2023-09-23T19:41:46.563424+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":137,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.534430+0000","last_change":"2023-09-23T19:05:52.789323+0000","last_active":"2023-09-23T19:41:46.534430+0000","last_peered":"2023-09-23T19:41:46.534430+0000","last_clean":"2023-09-23T19:41:46.534430+0000","last_became_active":"2023-09-23T19:05:52.788820+0000","last_became_peered":"2023-09-23T19:05:52.788820+0000","last_unstale":"2023-09-23T19:41:46.534430+0000","last_undegraded":"2023-09-23T19:41:46.534430+0000","last_fullsized":"2023-09-23T19:41:46.534430+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"186'4471","reported_seq":4486,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T19:42:33.429271+0000","last_change":"2023-09-23T19:41:24.830623+0000","last_active":"2023-09-23T19:42:33.429271+0000","last_peered":"2023-09-23T19:42:33.429271+0000","last_clean":"2023-09-23T19:42:33.429271+0000","last_became_active":"2023-09-23T19:41:24.830263+0000","last_became_peered":"2023-09-23T19:41:24.830263+0000","last_unstale":"2023-09-23T19:42:33.429271+0000","last_undegraded":"2023-09-23T19:42:33.429271+0000","last_fullsized":"2023-09-23T19:42:33.429271+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":180,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_clean_scrub_stamp":"2023-09-23T19:41:23.799472+0000","objects_scrubbed":0,"log_size":4471,"log_dups_size":0,"ondisk_log_size":4471,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:15:06.852546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293011456,"num_objects":4471,"num_object_clones":0,"num_object_copies":13413,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4471,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4471,"num_write_kb":286144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":154,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.562986+0000","last_change":"2023-09-23T19:22:42.151134+0000","last_active":"2023-09-23T19:41:46.562986+0000","last_peered":"2023-09-23T19:41:46.562986+0000","last_clean":"2023-09-23T19:41:46.562986+0000","last_became_active":"2023-09-23T19:22:42.150553+0000","last_became_peered":"2023-09-23T19:22:42.150553+0000","last_unstale":"2023-09-23T19:41:46.562986+0000","last_undegraded":"2023-09-23T19:41:46.562986+0000","last_fullsized":"2023-09-23T19:41:46.562986+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":165,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.565069+0000","last_change":"2023-09-23T19:28:07.899262+0000","last_active":"2023-09-23T19:41:46.565069+0000","last_peered":"2023-09-23T19:41:46.565069+0000","last_clean":"2023-09-23T19:41:46.565069+0000","last_became_active":"2023-09-23T19:28:07.898848+0000","last_became_peered":"2023-09-23T19:28:07.898848+0000","last_unstale":"2023-09-23T19:41:46.565069+0000","last_undegraded":"2023-09-23T19:41:46.565069+0000","last_fullsized":"2023-09-23T19:41:46.565069+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"186'4297","reported_seq":4311,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T19:42:30.521885+0000","last_change":"2023-09-23T19:41:24.846144+0000","last_active":"2023-09-23T19:42:30.521885+0000","last_peered":"2023-09-23T19:42:30.521885+0000","last_clean":"2023-09-23T19:42:30.521885+0000","last_became_active":"2023-09-23T19:41:24.845725+0000","last_became_peered":"2023-09-23T19:41:24.845725+0000","last_unstale":"2023-09-23T19:42:30.521885+0000","last_undegraded":"2023-09-23T19:42:30.521885+0000","last_fullsized":"2023-09-23T19:42:30.521885+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":180,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_clean_scrub_stamp":"2023-09-23T19:41:23.799472+0000","objects_scrubbed":0,"log_size":4297,"log_dups_size":0,"ondisk_log_size":4297,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":281608192,"num_objects":4297,"num_object_clones":0,"num_object_copies":12891,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4297,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4297,"num_write_kb":275008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":138,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.562928+0000","last_change":"2023-09-23T19:05:52.114108+0000","last_active":"2023-09-23T19:41:46.562928+0000","last_peered":"2023-09-23T19:41:46.562928+0000","last_clean":"2023-09-23T19:41:46.562928+0000","last_became_active":"2023-09-23T19:05:52.113609+0000","last_became_peered":"2023-09-23T19:05:52.113609+0000","last_unstale":"2023-09-23T19:41:46.562928+0000","last_undegraded":"2023-09-23T19:41:46.562928+0000","last_fullsized":"2023-09-23T19:41:46.562928+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":151,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.559420+0000","last_change":"2023-09-23T19:22:42.156825+0000","last_active":"2023-09-23T19:41:46.559420+0000","last_peered":"2023-09-23T19:41:46.559420+0000","last_clean":"2023-09-23T19:41:46.559420+0000","last_became_active":"2023-09-23T19:22:42.156474+0000","last_became_peered":"2023-09-23T19:22:42.156474+0000","last_unstale":"2023-09-23T19:41:46.559420+0000","last_undegraded":"2023-09-23T19:41:46.559420+0000","last_fullsized":"2023-09-23T19:41:46.559420+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"186'4228","reported_seq":4242,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T19:42:30.546757+0000","last_change":"2023-09-23T19:41:24.846316+0000","last_active":"2023-09-23T19:42:30.546757+0000","last_peered":"2023-09-23T19:42:30.546757+0000","last_clean":"2023-09-23T19:42:30.546757+0000","last_became_active":"2023-09-23T19:41:24.845849+0000","last_became_peered":"2023-09-23T19:41:24.845849+0000","last_unstale":"2023-09-23T19:42:30.546757+0000","last_undegraded":"2023-09-23T19:42:30.546757+0000","last_fullsized":"2023-09-23T19:42:30.546757+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":180,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_clean_scrub_stamp":"2023-09-23T19:41:23.799472+0000","objects_scrubbed":0,"log_size":4228,"log_dups_size":0,"ondisk_log_size":4228,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":277086208,"num_objects":4228,"num_object_clones":0,"num_object_copies":12684,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4228,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4228,"num_write_kb":270592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":153,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.534942+0000","last_change":"2023-09-23T19:28:07.891536+0000","last_active":"2023-09-23T19:41:46.534942+0000","last_peered":"2023-09-23T19:41:46.534942+0000","last_clean":"2023-09-23T19:41:46.534942+0000","last_became_active":"2023-09-23T19:28:07.891153+0000","last_became_peered":"2023-09-23T19:28:07.891153+0000","last_unstale":"2023-09-23T19:41:46.534942+0000","last_undegraded":"2023-09-23T19:41:46.534942+0000","last_fullsized":"2023-09-23T19:41:46.534942+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":147,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.552558+0000","last_change":"2023-09-23T19:22:42.135572+0000","last_active":"2023-09-23T19:41:46.552558+0000","last_peered":"2023-09-23T19:41:46.552558+0000","last_clean":"2023-09-23T19:41:46.552558+0000","last_became_active":"2023-09-23T19:22:42.135237+0000","last_became_peered":"2023-09-23T19:22:42.135237+0000","last_unstale":"2023-09-23T19:41:46.552558+0000","last_undegraded":"2023-09-23T19:41:46.552558+0000","last_fullsized":"2023-09-23T19:41:46.552558+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:35.838718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"186'4253","reported_seq":4266,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T19:42:30.077443+0000","last_change":"2023-09-23T19:41:24.846345+0000","last_active":"2023-09-23T19:42:30.077443+0000","last_peered":"2023-09-23T19:42:30.077443+0000","last_clean":"2023-09-23T19:42:30.077443+0000","last_became_active":"2023-09-23T19:41:24.844550+0000","last_became_peered":"2023-09-23T19:41:24.844550+0000","last_unstale":"2023-09-23T19:42:30.077443+0000","last_undegraded":"2023-09-23T19:42:30.077443+0000","last_fullsized":"2023-09-23T19:42:30.077443+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":180,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_clean_scrub_stamp":"2023-09-23T19:41:23.799472+0000","objects_scrubbed":0,"log_size":4253,"log_dups_size":0,"ondisk_log_size":4253,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":278724608,"num_objects":4253,"num_object_clones":0,"num_object_copies":12759,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4253,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4253,"num_write_kb":272192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":94,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:48.451492+0000","last_change":"2023-09-23T19:32:49.728589+0000","last_active":"2023-09-23T19:41:48.451492+0000","last_peered":"2023-09-23T19:41:48.451492+0000","last_clean":"2023-09-23T19:41:48.451492+0000","last_became_active":"2023-09-23T19:32:49.728037+0000","last_became_peered":"2023-09-23T19:32:49.728037+0000","last_unstale":"2023-09-23T19:41:48.451492+0000","last_undegraded":"2023-09-23T19:41:48.451492+0000","last_fullsized":"2023-09-23T19:41:48.451492+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:26:00.957969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":150,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.565841+0000","last_change":"2023-09-23T19:28:07.896803+0000","last_active":"2023-09-23T19:41:46.565841+0000","last_peered":"2023-09-23T19:41:46.565841+0000","last_clean":"2023-09-23T19:41:46.565841+0000","last_became_active":"2023-09-23T19:28:07.896480+0000","last_became_peered":"2023-09-23T19:28:07.896480+0000","last_unstale":"2023-09-23T19:41:46.565841+0000","last_undegraded":"2023-09-23T19:41:46.565841+0000","last_fullsized":"2023-09-23T19:41:46.565841+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"186'4377","reported_seq":4392,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T19:42:32.141011+0000","last_change":"2023-09-23T19:41:24.829547+0000","last_active":"2023-09-23T19:42:32.141011+0000","last_peered":"2023-09-23T19:42:32.141011+0000","last_clean":"2023-09-23T19:42:32.141011+0000","last_became_active":"2023-09-23T19:41:24.829169+0000","last_became_peered":"2023-09-23T19:41:24.829169+0000","last_unstale":"2023-09-23T19:42:32.141011+0000","last_undegraded":"2023-09-23T19:42:32.141011+0000","last_fullsized":"2023-09-23T19:42:32.141011+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":180,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_clean_scrub_stamp":"2023-09-23T19:41:23.799472+0000","objects_scrubbed":0,"log_size":4377,"log_dups_size":0,"ondisk_log_size":4377,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:24.473495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":286851072,"num_objects":4377,"num_object_clones":0,"num_object_copies":13131,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4377,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4377,"num_write_kb":280128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":152,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.559268+0000","last_change":"2023-09-23T19:22:42.156582+0000","last_active":"2023-09-23T19:41:46.559268+0000","last_peered":"2023-09-23T19:41:46.559268+0000","last_clean":"2023-09-23T19:41:46.559268+0000","last_became_active":"2023-09-23T19:22:42.156251+0000","last_became_peered":"2023-09-23T19:22:42.156251+0000","last_unstale":"2023-09-23T19:41:46.559268+0000","last_undegraded":"2023-09-23T19:41:46.559268+0000","last_fullsized":"2023-09-23T19:41:46.559268+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":180,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.566034+0000","last_change":"2023-09-23T19:38:43.891593+0000","last_active":"2023-09-23T19:41:46.566034+0000","last_peered":"2023-09-23T19:41:46.566034+0000","last_clean":"2023-09-23T19:41:46.566034+0000","last_became_active":"2023-09-23T19:38:43.891063+0000","last_became_peered":"2023-09-23T19:38:43.891063+0000","last_unstale":"2023-09-23T19:41:46.566034+0000","last_undegraded":"2023-09-23T19:41:46.566034+0000","last_fullsized":"2023-09-23T19:41:46.566034+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"186'4333","reported_seq":4347,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T19:42:30.038538+0000","last_change":"2023-09-23T19:41:24.841200+0000","last_active":"2023-09-23T19:42:30.038538+0000","last_peered":"2023-09-23T19:42:30.038538+0000","last_clean":"2023-09-23T19:42:30.038538+0000","last_became_active":"2023-09-23T19:41:24.840724+0000","last_became_peered":"2023-09-23T19:41:24.840724+0000","last_unstale":"2023-09-23T19:42:30.038538+0000","last_undegraded":"2023-09-23T19:42:30.038538+0000","last_fullsized":"2023-09-23T19:42:30.038538+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":180,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_clean_scrub_stamp":"2023-09-23T19:41:23.799472+0000","objects_scrubbed":0,"log_size":4333,"log_dups_size":0,"ondisk_log_size":4333,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":283967488,"num_objects":4333,"num_object_clones":0,"num_object_copies":12999,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4333,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4333,"num_write_kb":277312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":166,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.564945+0000","last_change":"2023-09-23T19:38:43.892972+0000","last_active":"2023-09-23T19:41:46.564945+0000","last_peered":"2023-09-23T19:41:46.564945+0000","last_clean":"2023-09-23T19:41:46.564945+0000","last_became_active":"2023-09-23T19:38:43.892451+0000","last_became_peered":"2023-09-23T19:38:43.892451+0000","last_unstale":"2023-09-23T19:41:46.564945+0000","last_undegraded":"2023-09-23T19:41:46.564945+0000","last_fullsized":"2023-09-23T19:41:46.564945+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":150,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.566597+0000","last_change":"2023-09-23T19:22:42.149670+0000","last_active":"2023-09-23T19:41:46.566597+0000","last_peered":"2023-09-23T19:41:46.566597+0000","last_clean":"2023-09-23T19:41:46.566597+0000","last_became_active":"2023-09-23T19:22:42.149022+0000","last_became_peered":"2023-09-23T19:22:42.149022+0000","last_unstale":"2023-09-23T19:41:46.566597+0000","last_undegraded":"2023-09-23T19:41:46.566597+0000","last_fullsized":"2023-09-23T19:41:46.566597+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"186'4340","reported_seq":4355,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T19:42:30.359086+0000","last_change":"2023-09-23T19:41:24.843247+0000","last_active":"2023-09-23T19:42:30.359086+0000","last_peered":"2023-09-23T19:42:30.359086+0000","last_clean":"2023-09-23T19:42:30.359086+0000","last_became_active":"2023-09-23T19:41:24.842923+0000","last_became_peered":"2023-09-23T19:41:24.842923+0000","last_unstale":"2023-09-23T19:42:30.359086+0000","last_undegraded":"2023-09-23T19:42:30.359086+0000","last_fullsized":"2023-09-23T19:42:30.359086+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":180,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_clean_scrub_stamp":"2023-09-23T19:41:23.799472+0000","objects_scrubbed":0,"log_size":4340,"log_dups_size":0,"ondisk_log_size":4340,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:40:00.822642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":284426240,"num_objects":4340,"num_object_clones":0,"num_object_copies":13020,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4340,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4340,"num_write_kb":277760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":144,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.524855+0000","last_change":"2023-09-23T19:32:49.729244+0000","last_active":"2023-09-23T19:41:46.524855+0000","last_peered":"2023-09-23T19:41:46.524855+0000","last_clean":"2023-09-23T19:41:46.524855+0000","last_became_active":"2023-09-23T19:32:49.728810+0000","last_became_peered":"2023-09-23T19:32:49.728810+0000","last_unstale":"2023-09-23T19:41:46.524855+0000","last_undegraded":"2023-09-23T19:41:46.524855+0000","last_fullsized":"2023-09-23T19:41:46.524855+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:57.540826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":131,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.525582+0000","last_change":"2023-09-23T19:28:07.889095+0000","last_active":"2023-09-23T19:41:46.525582+0000","last_peered":"2023-09-23T19:41:46.525582+0000","last_clean":"2023-09-23T19:41:46.525582+0000","last_became_active":"2023-09-23T19:28:07.888742+0000","last_became_peered":"2023-09-23T19:28:07.888742+0000","last_unstale":"2023-09-23T19:41:46.525582+0000","last_undegraded":"2023-09-23T19:41:46.525582+0000","last_fullsized":"2023-09-23T19:41:46.525582+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:43.929488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.d","version":"186'4391","reported_seq":4406,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T19:42:32.140977+0000","last_change":"2023-09-23T19:41:24.839428+0000","last_active":"2023-09-23T19:42:32.140977+0000","last_peered":"2023-09-23T19:42:32.140977+0000","last_clean":"2023-09-23T19:42:32.140977+0000","last_became_active":"2023-09-23T19:41:24.838617+0000","last_became_peered":"2023-09-23T19:41:24.838617+0000","last_unstale":"2023-09-23T19:42:32.140977+0000","last_undegraded":"2023-09-23T19:42:32.140977+0000","last_fullsized":"2023-09-23T19:42:32.140977+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":180,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_clean_scrub_stamp":"2023-09-23T19:41:23.799472+0000","objects_scrubbed":0,"log_size":4391,"log_dups_size":0,"ondisk_log_size":4391,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:05:40.117492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":287768576,"num_objects":4391,"num_object_clones":0,"num_object_copies":13173,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4391,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4391,"num_write_kb":281024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":165,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.525815+0000","last_change":"2023-09-23T19:22:42.143289+0000","last_active":"2023-09-23T19:41:46.525815+0000","last_peered":"2023-09-23T19:41:46.525815+0000","last_clean":"2023-09-23T19:41:46.525815+0000","last_became_active":"2023-09-23T19:22:42.142656+0000","last_became_peered":"2023-09-23T19:22:42.142656+0000","last_unstale":"2023-09-23T19:41:46.525815+0000","last_undegraded":"2023-09-23T19:41:46.525815+0000","last_fullsized":"2023-09-23T19:41:46.525815+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:38.467441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":183,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.534826+0000","last_change":"2023-09-23T19:38:43.894691+0000","last_active":"2023-09-23T19:41:46.534826+0000","last_peered":"2023-09-23T19:41:46.534826+0000","last_clean":"2023-09-23T19:41:46.534826+0000","last_became_active":"2023-09-23T19:38:43.894384+0000","last_became_peered":"2023-09-23T19:38:43.894384+0000","last_unstale":"2023-09-23T19:41:46.534826+0000","last_undegraded":"2023-09-23T19:41:46.534826+0000","last_fullsized":"2023-09-23T19:41:46.534826+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":150,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.555996+0000","last_change":"2023-09-23T19:38:43.898519+0000","last_active":"2023-09-23T19:41:46.555996+0000","last_peered":"2023-09-23T19:41:46.555996+0000","last_clean":"2023-09-23T19:41:46.555996+0000","last_became_active":"2023-09-23T19:38:43.896814+0000","last_became_peered":"2023-09-23T19:38:43.896814+0000","last_unstale":"2023-09-23T19:41:46.555996+0000","last_undegraded":"2023-09-23T19:41:46.555996+0000","last_fullsized":"2023-09-23T19:41:46.555996+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:40.465038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.b","version":"186'4373","reported_seq":4388,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T19:42:30.364083+0000","last_change":"2023-09-23T19:41:24.828810+0000","last_active":"2023-09-23T19:42:30.364083+0000","last_peered":"2023-09-23T19:42:30.364083+0000","last_clean":"2023-09-23T19:42:30.364083+0000","last_became_active":"2023-09-23T19:41:24.827984+0000","last_became_peered":"2023-09-23T19:41:24.827984+0000","last_unstale":"2023-09-23T19:42:30.364083+0000","last_undegraded":"2023-09-23T19:42:30.364083+0000","last_fullsized":"2023-09-23T19:42:30.364083+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":180,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_clean_scrub_stamp":"2023-09-23T19:41:23.799472+0000","objects_scrubbed":0,"log_size":4373,"log_dups_size":0,"ondisk_log_size":4373,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:09.027812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":286588928,"num_objects":4373,"num_object_clones":0,"num_object_copies":13119,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4373,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4373,"num_write_kb":279872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":157,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.535117+0000","last_change":"2023-09-23T19:22:42.153647+0000","last_active":"2023-09-23T19:41:46.535117+0000","last_peered":"2023-09-23T19:41:46.535117+0000","last_clean":"2023-09-23T19:41:46.535117+0000","last_became_active":"2023-09-23T19:22:42.153317+0000","last_became_peered":"2023-09-23T19:22:42.153317+0000","last_unstale":"2023-09-23T19:41:46.535117+0000","last_undegraded":"2023-09-23T19:41:46.535117+0000","last_fullsized":"2023-09-23T19:41:46.535117+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":151,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.565402+0000","last_change":"2023-09-23T19:18:09.017525+0000","last_active":"2023-09-23T19:41:46.565402+0000","last_peered":"2023-09-23T19:41:46.565402+0000","last_clean":"2023-09-23T19:41:46.565402+0000","last_became_active":"2023-09-23T19:18:09.017193+0000","last_became_peered":"2023-09-23T19:18:09.017193+0000","last_unstale":"2023-09-23T19:41:46.565402+0000","last_undegraded":"2023-09-23T19:41:46.565402+0000","last_fullsized":"2023-09-23T19:41:46.565402+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":162,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.525726+0000","last_change":"2023-09-23T19:38:43.877088+0000","last_active":"2023-09-23T19:41:46.525726+0000","last_peered":"2023-09-23T19:41:46.525726+0000","last_clean":"2023-09-23T19:41:46.525726+0000","last_became_active":"2023-09-23T19:38:43.876719+0000","last_became_peered":"2023-09-23T19:38:43.876719+0000","last_unstale":"2023-09-23T19:41:46.525726+0000","last_undegraded":"2023-09-23T19:41:46.525726+0000","last_fullsized":"2023-09-23T19:41:46.525726+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:05.423256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.8","version":"186'4373","reported_seq":4387,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T19:42:30.041489+0000","last_change":"2023-09-23T19:41:24.841123+0000","last_active":"2023-09-23T19:42:30.041489+0000","last_peered":"2023-09-23T19:42:30.041489+0000","last_clean":"2023-09-23T19:42:30.041489+0000","last_became_active":"2023-09-23T19:41:24.840707+0000","last_became_peered":"2023-09-23T19:41:24.840707+0000","last_unstale":"2023-09-23T19:42:30.041489+0000","last_undegraded":"2023-09-23T19:42:30.041489+0000","last_fullsized":"2023-09-23T19:42:30.041489+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":180,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_clean_scrub_stamp":"2023-09-23T19:41:23.799472+0000","objects_scrubbed":0,"log_size":4373,"log_dups_size":0,"ondisk_log_size":4373,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":286588928,"num_objects":4373,"num_object_clones":0,"num_object_copies":13119,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4373,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4373,"num_write_kb":279872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":130,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.554799+0000","last_change":"2023-09-23T19:38:43.883663+0000","last_active":"2023-09-23T19:41:46.554799+0000","last_peered":"2023-09-23T19:41:46.554799+0000","last_clean":"2023-09-23T19:41:46.554799+0000","last_became_active":"2023-09-23T19:38:43.882421+0000","last_became_peered":"2023-09-23T19:38:43.882421+0000","last_unstale":"2023-09-23T19:41:46.554799+0000","last_undegraded":"2023-09-23T19:41:46.554799+0000","last_fullsized":"2023-09-23T19:41:46.554799+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:49:01.607659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":147,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.524743+0000","last_change":"2023-09-23T19:38:43.890600+0000","last_active":"2023-09-23T19:41:46.524743+0000","last_peered":"2023-09-23T19:41:46.524743+0000","last_clean":"2023-09-23T19:41:46.524743+0000","last_became_active":"2023-09-23T19:38:43.890253+0000","last_became_peered":"2023-09-23T19:38:43.890253+0000","last_unstale":"2023-09-23T19:41:46.524743+0000","last_undegraded":"2023-09-23T19:41:46.524743+0000","last_fullsized":"2023-09-23T19:41:46.524743+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:46.751120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":160,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:42:10.342719+0000","last_change":"2023-09-23T19:22:42.143733+0000","last_active":"2023-09-23T19:42:10.342719+0000","last_peered":"2023-09-23T19:42:10.342719+0000","last_clean":"2023-09-23T19:42:10.342719+0000","last_became_active":"2023-09-23T19:22:42.143077+0000","last_became_peered":"2023-09-23T19:22:42.143077+0000","last_unstale":"2023-09-23T19:42:10.342719+0000","last_undegraded":"2023-09-23T19:42:10.342719+0000","last_fullsized":"2023-09-23T19:42:10.342719+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:47:33.086994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"186'4303","reported_seq":4317,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T19:42:30.070432+0000","last_change":"2023-09-23T19:41:24.841442+0000","last_active":"2023-09-23T19:42:30.070432+0000","last_peered":"2023-09-23T19:42:30.070432+0000","last_clean":"2023-09-23T19:42:30.070432+0000","last_became_active":"2023-09-23T19:41:24.841069+0000","last_became_peered":"2023-09-23T19:41:24.841069+0000","last_unstale":"2023-09-23T19:42:30.070432+0000","last_undegraded":"2023-09-23T19:42:30.070432+0000","last_fullsized":"2023-09-23T19:42:30.070432+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":180,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_clean_scrub_stamp":"2023-09-23T19:41:23.799472+0000","objects_scrubbed":0,"log_size":4303,"log_dups_size":0,"ondisk_log_size":4303,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":282001408,"num_objects":4303,"num_object_clones":0,"num_object_copies":12909,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4303,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4303,"num_write_kb":275392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":172,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.534501+0000","last_change":"2023-09-23T19:38:43.947487+0000","last_active":"2023-09-23T19:41:46.534501+0000","last_peered":"2023-09-23T19:41:46.534501+0000","last_clean":"2023-09-23T19:41:46.534501+0000","last_became_active":"2023-09-23T19:38:43.946493+0000","last_became_peered":"2023-09-23T19:38:43.946493+0000","last_unstale":"2023-09-23T19:41:46.534501+0000","last_undegraded":"2023-09-23T19:41:46.534501+0000","last_fullsized":"2023-09-23T19:41:46.534501+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":388,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.562577+0000","last_change":"2023-09-23T19:18:09.016297+0000","last_active":"2023-09-23T19:41:46.562577+0000","last_peered":"2023-09-23T19:41:46.562577+0000","last_clean":"2023-09-23T19:41:46.562577+0000","last_became_active":"2023-09-23T19:18:09.015436+0000","last_became_peered":"2023-09-23T19:18:09.015436+0000","last_unstale":"2023-09-23T19:41:46.562577+0000","last_undegraded":"2023-09-23T19:41:46.562577+0000","last_fullsized":"2023-09-23T19:41:46.562577+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:59:33.840751+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:59:33.840751+0000","last_clean_scrub_stamp":"2023-09-23T18:59:33.840751+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":126,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.534908+0000","last_change":"2023-09-23T19:32:49.734148+0000","last_active":"2023-09-23T19:41:46.534908+0000","last_peered":"2023-09-23T19:41:46.534908+0000","last_clean":"2023-09-23T19:41:46.534908+0000","last_became_active":"2023-09-23T19:32:49.733512+0000","last_became_peered":"2023-09-23T19:32:49.733512+0000","last_unstale":"2023-09-23T19:41:46.534908+0000","last_undegraded":"2023-09-23T19:41:46.534908+0000","last_fullsized":"2023-09-23T19:41:46.534908+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":136,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.560765+0000","last_change":"2023-09-23T19:05:52.786771+0000","last_active":"2023-09-23T19:41:46.560765+0000","last_peered":"2023-09-23T19:41:46.560765+0000","last_clean":"2023-09-23T19:41:46.560765+0000","last_became_active":"2023-09-23T19:05:52.785797+0000","last_became_peered":"2023-09-23T19:05:52.785797+0000","last_unstale":"2023-09-23T19:41:46.560765+0000","last_undegraded":"2023-09-23T19:41:46.560765+0000","last_fullsized":"2023-09-23T19:41:46.560765+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.9","version":"186'4385","reported_seq":4399,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T19:42:30.046828+0000","last_change":"2023-09-23T19:41:24.846496+0000","last_active":"2023-09-23T19:42:30.046828+0000","last_peered":"2023-09-23T19:42:30.046828+0000","last_clean":"2023-09-23T19:42:30.046828+0000","last_became_active":"2023-09-23T19:41:24.846126+0000","last_became_peered":"2023-09-23T19:41:24.846126+0000","last_unstale":"2023-09-23T19:42:30.046828+0000","last_undegraded":"2023-09-23T19:42:30.046828+0000","last_fullsized":"2023-09-23T19:42:30.046828+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":180,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_clean_scrub_stamp":"2023-09-23T19:41:23.799472+0000","objects_scrubbed":0,"log_size":4385,"log_dups_size":0,"ondisk_log_size":4385,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":287375360,"num_objects":4385,"num_object_clones":0,"num_object_copies":13155,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4385,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4385,"num_write_kb":280640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":171,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.562384+0000","last_change":"2023-09-23T19:22:42.150636+0000","last_active":"2023-09-23T19:41:46.562384+0000","last_peered":"2023-09-23T19:41:46.562384+0000","last_clean":"2023-09-23T19:41:46.562384+0000","last_became_active":"2023-09-23T19:22:42.150045+0000","last_became_peered":"2023-09-23T19:22:42.150045+0000","last_unstale":"2023-09-23T19:41:46.562384+0000","last_undegraded":"2023-09-23T19:41:46.562384+0000","last_fullsized":"2023-09-23T19:41:46.562384+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":148,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.552827+0000","last_change":"2023-09-23T19:28:07.889194+0000","last_active":"2023-09-23T19:41:46.552827+0000","last_peered":"2023-09-23T19:41:46.552827+0000","last_clean":"2023-09-23T19:41:46.552827+0000","last_became_active":"2023-09-23T19:28:07.888823+0000","last_became_peered":"2023-09-23T19:28:07.888823+0000","last_unstale":"2023-09-23T19:41:46.552827+0000","last_undegraded":"2023-09-23T19:41:46.552827+0000","last_fullsized":"2023-09-23T19:41:46.552827+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:23.464688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":178,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:58.456519+0000","last_change":"2023-09-23T19:38:43.876930+0000","last_active":"2023-09-23T19:41:58.456519+0000","last_peered":"2023-09-23T19:41:58.456519+0000","last_clean":"2023-09-23T19:41:58.456519+0000","last_became_active":"2023-09-23T19:38:43.876571+0000","last_became_peered":"2023-09-23T19:38:43.876571+0000","last_unstale":"2023-09-23T19:41:58.456519+0000","last_undegraded":"2023-09-23T19:41:58.456519+0000","last_fullsized":"2023-09-23T19:41:58.456519+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:30.924536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"186'4270","reported_seq":4285,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T19:42:30.401164+0000","last_change":"2023-09-23T19:41:24.838695+0000","last_active":"2023-09-23T19:42:30.401164+0000","last_peered":"2023-09-23T19:42:30.401164+0000","last_clean":"2023-09-23T19:42:30.401164+0000","last_became_active":"2023-09-23T19:41:24.838311+0000","last_became_peered":"2023-09-23T19:41:24.838311+0000","last_unstale":"2023-09-23T19:42:30.401164+0000","last_undegraded":"2023-09-23T19:42:30.401164+0000","last_fullsized":"2023-09-23T19:42:30.401164+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":180,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_clean_scrub_stamp":"2023-09-23T19:41:23.799472+0000","objects_scrubbed":0,"log_size":4270,"log_dups_size":0,"ondisk_log_size":4270,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:54.968784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":279838720,"num_objects":4270,"num_object_clones":0,"num_object_copies":12810,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4270,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4270,"num_write_kb":273280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":156,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.563023+0000","last_change":"2023-09-23T19:18:09.015853+0000","last_active":"2023-09-23T19:41:46.563023+0000","last_peered":"2023-09-23T19:41:46.563023+0000","last_clean":"2023-09-23T19:41:46.563023+0000","last_became_active":"2023-09-23T19:18:09.015338+0000","last_became_peered":"2023-09-23T19:18:09.015338+0000","last_unstale":"2023-09-23T19:41:46.563023+0000","last_undegraded":"2023-09-23T19:41:46.563023+0000","last_fullsized":"2023-09-23T19:41:46.563023+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":147,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.525647+0000","last_change":"2023-09-23T19:38:43.881233+0000","last_active":"2023-09-23T19:41:46.525647+0000","last_peered":"2023-09-23T19:41:46.525647+0000","last_clean":"2023-09-23T19:41:46.525647+0000","last_became_active":"2023-09-23T19:38:43.880893+0000","last_became_peered":"2023-09-23T19:38:43.880893+0000","last_unstale":"2023-09-23T19:41:46.525647+0000","last_undegraded":"2023-09-23T19:41:46.525647+0000","last_fullsized":"2023-09-23T19:41:46.525647+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:11.206947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":133,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.552174+0000","last_change":"2023-09-23T19:18:09.005823+0000","last_active":"2023-09-23T19:41:46.552174+0000","last_peered":"2023-09-23T19:41:46.552174+0000","last_clean":"2023-09-23T19:41:46.552174+0000","last_became_active":"2023-09-23T19:18:09.005207+0000","last_became_peered":"2023-09-23T19:18:09.005207+0000","last_unstale":"2023-09-23T19:41:46.552174+0000","last_undegraded":"2023-09-23T19:41:46.552174+0000","last_fullsized":"2023-09-23T19:41:46.552174+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:05.520188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"186'4427","reported_seq":4441,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T19:42:30.538910+0000","last_change":"2023-09-23T19:41:24.848461+0000","last_active":"2023-09-23T19:42:30.538910+0000","last_peered":"2023-09-23T19:42:30.538910+0000","last_clean":"2023-09-23T19:42:30.538910+0000","last_became_active":"2023-09-23T19:41:24.848117+0000","last_became_peered":"2023-09-23T19:41:24.848117+0000","last_unstale":"2023-09-23T19:42:30.538910+0000","last_undegraded":"2023-09-23T19:42:30.538910+0000","last_fullsized":"2023-09-23T19:42:30.538910+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":180,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_clean_scrub_stamp":"2023-09-23T19:41:23.799472+0000","objects_scrubbed":0,"log_size":4427,"log_dups_size":0,"ondisk_log_size":4427,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":290127872,"num_objects":4427,"num_object_clones":0,"num_object_copies":13281,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4427,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4427,"num_write_kb":283328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":90,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.562701+0000","last_change":"2023-09-23T19:32:49.737323+0000","last_active":"2023-09-23T19:41:46.562701+0000","last_peered":"2023-09-23T19:41:46.562701+0000","last_clean":"2023-09-23T19:41:46.562701+0000","last_became_active":"2023-09-23T19:32:49.736937+0000","last_became_peered":"2023-09-23T19:32:49.736937+0000","last_unstale":"2023-09-23T19:41:46.562701+0000","last_undegraded":"2023-09-23T19:41:46.562701+0000","last_fullsized":"2023-09-23T19:41:46.562701+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":117,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:42:10.342689+0000","last_change":"2023-09-23T19:22:42.145506+0000","last_active":"2023-09-23T19:42:10.342689+0000","last_peered":"2023-09-23T19:42:10.342689+0000","last_clean":"2023-09-23T19:42:10.342689+0000","last_became_active":"2023-09-23T19:22:42.143991+0000","last_became_peered":"2023-09-23T19:22:42.143991+0000","last_unstale":"2023-09-23T19:42:10.342689+0000","last_undegraded":"2023-09-23T19:42:10.342689+0000","last_fullsized":"2023-09-23T19:42:10.342689+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:56.875175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":127,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.563189+0000","last_change":"2023-09-23T19:32:49.734592+0000","last_active":"2023-09-23T19:41:46.563189+0000","last_peered":"2023-09-23T19:41:46.563189+0000","last_clean":"2023-09-23T19:41:46.563189+0000","last_became_active":"2023-09-23T19:32:49.733905+0000","last_became_peered":"2023-09-23T19:32:49.733905+0000","last_unstale":"2023-09-23T19:41:46.563189+0000","last_undegraded":"2023-09-23T19:41:46.563189+0000","last_fullsized":"2023-09-23T19:41:46.563189+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.e","version":"186'4362","reported_seq":4376,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T19:42:30.428419+0000","last_change":"2023-09-23T19:41:24.849220+0000","last_active":"2023-09-23T19:42:30.428419+0000","last_peered":"2023-09-23T19:42:30.428419+0000","last_clean":"2023-09-23T19:42:30.428419+0000","last_became_active":"2023-09-23T19:41:24.848808+0000","last_became_peered":"2023-09-23T19:41:24.848808+0000","last_unstale":"2023-09-23T19:42:30.428419+0000","last_undegraded":"2023-09-23T19:42:30.428419+0000","last_fullsized":"2023-09-23T19:42:30.428419+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":180,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:23.799472+0000","last_clean_scrub_stamp":"2023-09-23T19:41:23.799472+0000","objects_scrubbed":0,"log_size":4362,"log_dups_size":0,"ondisk_log_size":4362,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":285868032,"num_objects":4362,"num_object_clones":0,"num_object_copies":13086,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4362,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4362,"num_write_kb":279168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":171,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.561150+0000","last_change":"2023-09-23T19:38:43.895887+0000","last_active":"2023-09-23T19:41:46.561150+0000","last_peered":"2023-09-23T19:41:46.561150+0000","last_clean":"2023-09-23T19:41:46.561150+0000","last_became_active":"2023-09-23T19:38:43.895486+0000","last_became_peered":"2023-09-23T19:38:43.895486+0000","last_unstale":"2023-09-23T19:41:46.561150+0000","last_undegraded":"2023-09-23T19:41:46.561150+0000","last_fullsized":"2023-09-23T19:41:46.561150+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":151,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:58.456475+0000","last_change":"2023-09-23T19:22:42.149694+0000","last_active":"2023-09-23T19:41:58.456475+0000","last_peered":"2023-09-23T19:41:58.456475+0000","last_clean":"2023-09-23T19:41:58.456475+0000","last_became_active":"2023-09-23T19:22:42.148194+0000","last_became_peered":"2023-09-23T19:22:42.148194+0000","last_unstale":"2023-09-23T19:41:58.456475+0000","last_undegraded":"2023-09-23T19:41:58.456475+0000","last_fullsized":"2023-09-23T19:41:58.456475+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:47.842383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":135,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.564955+0000","last_change":"2023-09-23T19:05:52.138620+0000","last_active":"2023-09-23T19:41:46.564955+0000","last_peered":"2023-09-23T19:41:46.564955+0000","last_clean":"2023-09-23T19:41:46.564955+0000","last_became_active":"2023-09-23T19:05:52.138243+0000","last_became_peered":"2023-09-23T19:05:52.138243+0000","last_unstale":"2023-09-23T19:41:46.564955+0000","last_undegraded":"2023-09-23T19:41:46.564955+0000","last_fullsized":"2023-09-23T19:41:46.564955+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":134,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.552671+0000","last_change":"2023-09-23T19:18:09.008811+0000","last_active":"2023-09-23T19:41:46.552671+0000","last_peered":"2023-09-23T19:41:46.552671+0000","last_clean":"2023-09-23T19:41:46.552671+0000","last_became_active":"2023-09-23T19:18:09.007255+0000","last_became_peered":"2023-09-23T19:18:09.007255+0000","last_unstale":"2023-09-23T19:41:46.552671+0000","last_undegraded":"2023-09-23T19:41:46.552671+0000","last_fullsized":"2023-09-23T19:41:46.552671+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:25.874038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":138,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.555052+0000","last_change":"2023-09-23T19:38:43.877244+0000","last_active":"2023-09-23T19:41:46.555052+0000","last_peered":"2023-09-23T19:41:46.555052+0000","last_clean":"2023-09-23T19:41:46.555052+0000","last_became_active":"2023-09-23T19:38:43.876896+0000","last_became_peered":"2023-09-23T19:38:43.876896+0000","last_unstale":"2023-09-23T19:41:46.555052+0000","last_undegraded":"2023-09-23T19:41:46.555052+0000","last_fullsized":"2023-09-23T19:41:46.555052+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:22.288276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":155,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.535220+0000","last_change":"2023-09-23T19:22:42.153825+0000","last_active":"2023-09-23T19:41:46.535220+0000","last_peered":"2023-09-23T19:41:46.535220+0000","last_clean":"2023-09-23T19:41:46.535220+0000","last_became_active":"2023-09-23T19:22:42.153416+0000","last_became_peered":"2023-09-23T19:22:42.153416+0000","last_unstale":"2023-09-23T19:41:46.535220+0000","last_undegraded":"2023-09-23T19:41:46.535220+0000","last_fullsized":"2023-09-23T19:41:46.535220+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":189,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.565647+0000","last_change":"2023-09-23T19:38:43.947451+0000","last_active":"2023-09-23T19:41:46.565647+0000","last_peered":"2023-09-23T19:41:46.565647+0000","last_clean":"2023-09-23T19:41:46.565647+0000","last_became_active":"2023-09-23T19:38:43.946420+0000","last_became_peered":"2023-09-23T19:38:43.946420+0000","last_unstale":"2023-09-23T19:41:46.565647+0000","last_undegraded":"2023-09-23T19:41:46.565647+0000","last_fullsized":"2023-09-23T19:41:46.565647+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":140,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.535033+0000","last_change":"2023-09-23T19:05:52.107570+0000","last_active":"2023-09-23T19:41:46.535033+0000","last_peered":"2023-09-23T19:41:46.535033+0000","last_clean":"2023-09-23T19:41:46.535033+0000","last_became_active":"2023-09-23T19:05:52.107215+0000","last_became_peered":"2023-09-23T19:05:52.107215+0000","last_unstale":"2023-09-23T19:41:46.535033+0000","last_undegraded":"2023-09-23T19:41:46.535033+0000","last_fullsized":"2023-09-23T19:41:46.535033+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":165,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.565215+0000","last_change":"2023-09-23T19:28:07.896983+0000","last_active":"2023-09-23T19:41:46.565215+0000","last_peered":"2023-09-23T19:41:46.565215+0000","last_clean":"2023-09-23T19:41:46.565215+0000","last_became_active":"2023-09-23T19:28:07.896577+0000","last_became_peered":"2023-09-23T19:28:07.896577+0000","last_unstale":"2023-09-23T19:41:46.565215+0000","last_undegraded":"2023-09-23T19:41:46.565215+0000","last_fullsized":"2023-09-23T19:41:46.565215+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":95,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.560921+0000","last_change":"2023-09-23T19:38:43.900225+0000","last_active":"2023-09-23T19:41:46.560921+0000","last_peered":"2023-09-23T19:41:46.560921+0000","last_clean":"2023-09-23T19:41:46.560921+0000","last_became_active":"2023-09-23T19:38:43.899907+0000","last_became_peered":"2023-09-23T19:38:43.899907+0000","last_unstale":"2023-09-23T19:41:46.560921+0000","last_undegraded":"2023-09-23T19:41:46.560921+0000","last_fullsized":"2023-09-23T19:41:46.560921+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":147,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.554861+0000","last_change":"2023-09-23T19:38:43.882487+0000","last_active":"2023-09-23T19:41:46.554861+0000","last_peered":"2023-09-23T19:41:46.554861+0000","last_clean":"2023-09-23T19:41:46.554861+0000","last_became_active":"2023-09-23T19:38:43.881931+0000","last_became_peered":"2023-09-23T19:38:43.881931+0000","last_unstale":"2023-09-23T19:41:46.554861+0000","last_undegraded":"2023-09-23T19:41:46.554861+0000","last_fullsized":"2023-09-23T19:41:46.554861+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:02:36.906535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":166,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.554280+0000","last_change":"2023-09-23T19:38:43.893983+0000","last_active":"2023-09-23T19:41:46.554280+0000","last_peered":"2023-09-23T19:41:46.554280+0000","last_clean":"2023-09-23T19:41:46.554280+0000","last_became_active":"2023-09-23T19:38:43.893527+0000","last_became_peered":"2023-09-23T19:38:43.893527+0000","last_unstale":"2023-09-23T19:41:46.554280+0000","last_undegraded":"2023-09-23T19:41:46.554280+0000","last_fullsized":"2023-09-23T19:41:46.554280+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:12.156531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":144,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T19:41:46.564932+0000","last_change":"2023-09-23T19:06:36.673002+0000","last_active":"2023-09-23T19:41:46.564932+0000","last_peered":"2023-09-23T19:41:46.564932+0000","last_clean":"2023-09-23T19:41:46.564932+0000","last_became_active":"2023-09-23T19:06:36.671911+0000","last_became_peered":"2023-09-23T19:06:36.671911+0000","last_unstale":"2023-09-23T19:41:46.564932+0000","last_undegraded":"2023-09-23T19:41:46.564932+0000","last_fullsized":"2023-09-23T19:41:46.564932+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":4556390400,"num_objects":69525,"num_object_clones":0,"num_object_copies":208575,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":69525,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":69525,"num_write_kb":4449600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13741195264,"data_stored":13741195264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":69525,"ondisk_log_size":69525,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430684,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2559272,"kb_used_data":1663392,"kb_used_omap":0,"kb_used_meta":895872,"kb_avail":91185880,"statfs":{"total":95995035648,"available":93374341120,"internally_reserved":0,"allocated":1703313408,"data_stored":1702812072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":917372928},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:41:51 2023","interfaces":[{"interface":"back","average":{"1min":0.89100000000000001,"5min":1.825,"15min":1.8520000000000001},"min":{"1min":0.39300000000000002,"5min":0.27300000000000002,"15min":0.23100000000000001},"max":{"1min":2.0760000000000001,"5min":18.271000000000001,"15min":92.944999999999993},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.78600000000000003,"5min":1.7270000000000001,"15min":1.774},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.665,"5min":18.242000000000001,"15min":92.763999999999996},"last":0.63600000000000001}]},{"osd":1,"last update":"Sat Sep 23 19:42:10 2023","interfaces":[{"interface":"back","average":{"1min":1.498,"5min":1.361,"15min":1.8049999999999999},"min":{"1min":0.39900000000000002,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":5.4470000000000001,"5min":6.4219999999999997,"15min":93.757999999999996},"last":30.780999999999999},{"interface":"front","average":{"1min":1.645,"5min":1.464,"15min":1.819},"min":{"1min":0.46400000000000002,"5min":0.30599999999999999,"15min":0.29999999999999999},"max":{"1min":5.4269999999999996,"5min":6.4390000000000001,"15min":93.835999999999999},"last":30.827999999999999}]},{"osd":2,"last update":"Sat Sep 23 19:41:51 2023","interfaces":[{"interface":"back","average":{"1min":1.0309999999999999,"5min":1.7829999999999999,"15min":1.9850000000000001},"min":{"1min":0.52000000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":3.2690000000000001,"5min":6.851,"15min":6.851},"last":0.78300000000000003},{"interface":"front","average":{"1min":0.79800000000000004,"5min":1.2569999999999999,"15min":1.2929999999999999},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":1.1160000000000001,"5min":10.74,"15min":10.74},"last":0.71899999999999997}]},{"osd":3,"last update":"Sat Sep 23 19:41:34 2023","interfaces":[{"interface":"back","average":{"1min":1.056,"5min":1.2090000000000001,"15min":1.4239999999999999},"min":{"1min":0.621,"5min":0.35399999999999998,"15min":0.33800000000000002},"max":{"1min":3.4689999999999999,"5min":6.0999999999999996,"15min":19.637},"last":0.82899999999999996},{"interface":"front","average":{"1min":1.0980000000000001,"5min":1.4750000000000001,"15min":1.452},"min":{"1min":0.36699999999999999,"5min":0.34100000000000003,"15min":0.33900000000000002},"max":{"1min":4.6219999999999999,"5min":6.4630000000000001,"15min":20.204000000000001},"last":0.876}]},{"osd":4,"last update":"Sat Sep 23 19:42:10 2023","interfaces":[{"interface":"back","average":{"1min":0.998,"5min":1.26,"15min":1.1910000000000001},"min":{"1min":0.29199999999999998,"5min":0.254,"15min":0.252},"max":{"1min":4.0300000000000002,"5min":5.4210000000000003,"15min":9.7699999999999996},"last":0.66000000000000003},{"interface":"front","average":{"1min":1.2090000000000001,"5min":1.631,"15min":1.2889999999999999},"min":{"1min":0.39100000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":3.464,"5min":13.385,"15min":13.385},"last":0.996}]},{"osd":5,"last update":"Sat Sep 23 19:42:10 2023","interfaces":[{"interface":"back","average":{"1min":1.165,"5min":1.6020000000000001,"15min":1.514},"min":{"1min":0.48999999999999999,"5min":0.36699999999999999,"15min":0.28499999999999998},"max":{"1min":3.5019999999999998,"5min":13.273999999999999,"15min":13.342000000000001},"last":0.80900000000000005},{"interface":"front","average":{"1min":1.109,"5min":1.5189999999999999,"15min":1.5229999999999999},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.31},"max":{"1min":3.9900000000000002,"5min":6.3609999999999998,"15min":9.8979999999999997},"last":0.90100000000000002}]},{"osd":6,"last update":"Sat Sep 23 19:42:10 2023","interfaces":[{"interface":"back","average":{"1min":1.089,"5min":1.7889999999999999,"15min":1.4730000000000001},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.316},"max":{"1min":4.1050000000000004,"5min":22.725000000000001,"15min":22.725000000000001},"last":0.93799999999999994},{"interface":"front","average":{"1min":1.5069999999999999,"5min":2.27,"15min":1.6180000000000001},"min":{"1min":0.66800000000000004,"5min":0.308,"15min":0.308},"max":{"1min":5.3840000000000003,"5min":22.707000000000001,"15min":22.707000000000001},"last":1.1459999999999999}]}]},{"osd":6,"up_from":42,"seq":180388626916,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3083288,"kb_used_data":2187984,"kb_used_omap":0,"kb_used_meta":895296,"kb_avail":90661864,"statfs":{"total":95995035648,"available":92837748736,"internally_reserved":0,"allocated":2240495616,"data_stored":2240010203,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":916783104},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,1,2,5],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:41:59 2023","interfaces":[{"interface":"back","average":{"1min":0.72699999999999998,"5min":2.2200000000000002,"15min":1.796},"min":{"1min":0.32100000000000001,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":1.236,"5min":15.140000000000001,"15min":87.283000000000001},"last":0.316},{"interface":"front","average":{"1min":0.92200000000000004,"5min":1.6839999999999999,"15min":1.6870000000000001},"min":{"1min":0.28299999999999997,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":1.5029999999999999,"5min":8.9909999999999997,"15min":86.286000000000001},"last":1.458}]},{"osd":1,"last update":"Sat Sep 23 19:42:22 2023","interfaces":[{"interface":"back","average":{"1min":8.4480000000000004,"5min":2.9780000000000002,"15min":1.859},"min":{"1min":0.35099999999999998,"5min":0.26300000000000001,"15min":0.253},"max":{"1min":116.102,"5min":116.102,"15min":116.102},"last":0.28399999999999997},{"interface":"front","average":{"1min":8.5229999999999997,"5min":2.8050000000000002,"15min":1.8360000000000001},"min":{"1min":0.35399999999999998,"5min":0.29399999999999998,"15min":0.28999999999999998},"max":{"1min":116.06,"5min":116.06,"15min":116.06},"last":0.38500000000000001}]},{"osd":2,"last update":"Sat Sep 23 19:41:46 2023","interfaces":[{"interface":"back","average":{"1min":6.8310000000000004,"5min":3.206,"15min":2.6030000000000002},"min":{"1min":0.52800000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":117.587,"5min":117.587,"15min":117.587},"last":0.34699999999999998},{"interface":"front","average":{"1min":6.8209999999999997,"5min":3.21,"15min":2.637},"min":{"1min":0.54400000000000004,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":117.26900000000001,"5min":117.26900000000001,"15min":117.26900000000001},"last":0.86599999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:41:39 2023","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":1.9179999999999999,"15min":1.6160000000000001},"min":{"1min":0.46300000000000002,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":2.9430000000000001,"5min":8.7560000000000002,"15min":60.726999999999997},"last":3.9249999999999998},{"interface":"front","average":{"1min":1.2150000000000001,"5min":1.8140000000000001,"15min":1.696},"min":{"1min":0.46600000000000003,"5min":0.34200000000000003,"15min":0.28799999999999998},"max":{"1min":4.1040000000000001,"5min":12.956,"15min":58.304000000000002},"last":1.0569999999999999}]},{"osd":4,"last update":"Sat Sep 23 19:41:39 2023","interfaces":[{"interface":"back","average":{"1min":1.1819999999999999,"5min":1.825,"15min":1.212},"min":{"1min":0.38500000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":4.2000000000000002,"5min":10.138999999999999,"15min":10.138999999999999},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.97599999999999998,"5min":1.98,"15min":1.3100000000000001},"min":{"1min":0.34899999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":2.9910000000000001,"5min":9.3399999999999999,"15min":9.3399999999999999},"last":1.639}]},{"osd":5,"last update":"Sat Sep 23 19:41:39 2023","interfaces":[{"interface":"back","average":{"1min":1.0389999999999999,"5min":1.8280000000000001,"15min":1.3839999999999999},"min":{"1min":0.39800000000000002,"5min":0.39200000000000002,"15min":0.32600000000000001},"max":{"1min":2.6709999999999998,"5min":12.827,"15min":12.827},"last":0.42199999999999999},{"interface":"front","average":{"1min":1.177,"5min":2.4540000000000002,"15min":1.7470000000000001},"min":{"1min":0.50700000000000001,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":4.2110000000000003,"5min":10.433999999999999,"15min":10.433999999999999},"last":1.7190000000000001}]},{"osd":7,"last update":"Sat Sep 23 19:42:05 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":1.4099999999999999,"15min":1.264},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":1.466,"5min":5.75,"15min":5.9580000000000002},"last":0.49299999999999999},{"interface":"front","average":{"1min":1.1599999999999999,"5min":1.9339999999999999,"15min":1.5029999999999999},"min":{"1min":0.53600000000000003,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":3.3759999999999999,"5min":11.439,"15min":11.439},"last":1.6919999999999999}]}]},{"osd":1,"up_from":98,"seq":420906795248,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2690132,"kb_used_data":2214348,"kb_used_omap":4,"kb_used_meta":475771,"kb_avail":91055020,"statfs":{"total":95995035648,"available":93240340480,"internally_reserved":0,"allocated":2267492352,"data_stored":2266719383,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":487190414},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,6],"upper_bound":256},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:42:26 2023","interfaces":[{"interface":"back","average":{"1min":1.3440000000000001,"5min":4.79,"15min":2.2829999999999999},"min":{"1min":0.36799999999999999,"5min":0.21299999999999999,"15min":0.185},"max":{"1min":4.7249999999999996,"5min":65.079999999999998,"15min":65.079999999999998},"last":0.73199999999999998},{"interface":"front","average":{"1min":1.2669999999999999,"5min":4.4340000000000002,"15min":2.3380000000000001},"min":{"1min":0.27500000000000002,"5min":0.24399999999999999,"15min":0.20300000000000001},"max":{"1min":3.347,"5min":65.730000000000004,"15min":65.730000000000004},"last":1.306}]},{"osd":2,"last update":"Sat Sep 23 19:41:46 2023","interfaces":[{"interface":"back","average":{"1min":0.70699999999999996,"5min":1.3049999999999999,"15min":1.28},"min":{"1min":0.43099999999999999,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":1.516,"5min":11.143000000000001,"15min":11.143000000000001},"last":1.3500000000000001},{"interface":"front","average":{"1min":0.69999999999999996,"5min":2.2349999999999999,"15min":2.7480000000000002},"min":{"1min":0.33200000000000002,"5min":0.251,"15min":0.251},"max":{"1min":1.3759999999999999,"5min":48.546999999999997,"15min":48.546999999999997},"last":0.79100000000000004}]},{"osd":3,"last update":"Sat Sep 23 19:42:28 2023","interfaces":[{"interface":"back","average":{"1min":1.6319999999999999,"5min":1.3089999999999999,"15min":1.3160000000000001},"min":{"1min":0.39200000000000002,"5min":0.255,"15min":0.21199999999999999},"max":{"1min":4.9720000000000004,"5min":9.1790000000000003,"15min":10.612},"last":4.9720000000000004},{"interface":"front","average":{"1min":2.1789999999999998,"5min":1.954,"15min":1.706},"min":{"1min":0.46700000000000003,"5min":0.27100000000000002,"15min":0.26500000000000001},"max":{"1min":13.366,"5min":48.692,"15min":48.692},"last":0.71199999999999997}]},{"osd":4,"last update":"Sat Sep 23 19:42:26 2023","interfaces":[{"interface":"back","average":{"1min":1.3009999999999999,"5min":1.8120000000000001,"15min":1.538},"min":{"1min":0.39400000000000002,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":3.8969999999999998,"5min":53.128999999999998,"15min":53.128999999999998},"last":1.613},{"interface":"front","average":{"1min":1.26,"5min":2.1429999999999998,"15min":1.6539999999999999},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":4.7999999999999998,"5min":48.637,"15min":48.637},"last":0.89300000000000002}]},{"osd":5,"last update":"Sat Sep 23 19:42:26 2023","interfaces":[{"interface":"back","average":{"1min":1.4419999999999999,"5min":1.9570000000000001,"15min":1.867},"min":{"1min":0.45100000000000001,"5min":0.42499999999999999,"15min":0.33900000000000002},"max":{"1min":4.867,"5min":48.738,"15min":48.738},"last":0.85799999999999998},{"interface":"front","average":{"1min":1.198,"5min":1.8680000000000001,"15min":1.633},"min":{"1min":0.50900000000000001,"5min":0.40400000000000003,"15min":0.35999999999999999},"max":{"1min":2.6659999999999999,"5min":48.726999999999997,"15min":48.726999999999997},"last":1.081}]},{"osd":6,"last update":"Sat Sep 23 19:42:26 2023","interfaces":[{"interface":"back","average":{"1min":1.784,"5min":2.371,"15min":1.7629999999999999},"min":{"1min":0.60499999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":7.6959999999999997,"5min":48.762,"15min":48.762},"last":1.1240000000000001},{"interface":"front","average":{"1min":1.722,"5min":2.3849999999999998,"15min":1.843},"min":{"1min":0.53200000000000003,"5min":0.39200000000000002,"15min":0.39100000000000001},"max":{"1min":7.7480000000000002,"5min":53.076999999999998,"15min":53.076999999999998},"last":1.5529999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:42:26 2023","interfaces":[{"interface":"back","average":{"1min":1.492,"5min":1.893,"15min":1.8120000000000001},"min":{"1min":0.57999999999999996,"5min":0.38,"15min":0.38},"max":{"1min":3.2709999999999999,"5min":48.677,"15min":48.677},"last":1.5860000000000001},{"interface":"front","average":{"1min":1.391,"5min":1.9990000000000001,"15min":1.8440000000000001},"min":{"1min":0.46500000000000002,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":4.835,"5min":48.805,"15min":48.805},"last":5.0220000000000002}]}]},{"osd":0,"up_from":85,"seq":365072220455,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2070376,"kb_used_data":1435232,"kb_used_omap":5,"kb_used_meta":635130,"kb_avail":91674776,"statfs":{"total":95995035648,"available":93874970624,"internally_reserved":0,"allocated":1469677568,"data_stored":1468889145,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":650373754},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 19:42:19 2023","interfaces":[{"interface":"back","average":{"1min":3.7989999999999999,"5min":1.625,"15min":1.284},"min":{"1min":0.33200000000000002,"5min":0.20799999999999999,"15min":0.20399999999999999},"max":{"1min":57.234999999999999,"5min":57.234999999999999,"15min":57.234999999999999},"last":0.57099999999999995},{"interface":"front","average":{"1min":4.077,"5min":2.226,"15min":1.585},"min":{"1min":0.28100000000000003,"5min":0.223,"15min":0.223},"max":{"1min":58.825000000000003,"5min":58.825000000000003,"15min":58.825000000000003},"last":2.278}]},{"osd":2,"last update":"Sat Sep 23 19:41:48 2023","interfaces":[{"interface":"back","average":{"1min":0.91800000000000004,"5min":1.7569999999999999,"15min":1.859},"min":{"1min":0.317,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":3.5859999999999999,"5min":7.1100000000000003,"15min":7.1100000000000003},"last":0.76500000000000001},{"interface":"front","average":{"1min":0.68000000000000005,"5min":2.0579999999999998,"15min":2.2599999999999998},"min":{"1min":0.30099999999999999,"5min":0.247,"15min":0.247},"max":{"1min":1.3380000000000001,"5min":15.124000000000001,"15min":15.124000000000001},"last":0.25800000000000001}]},{"osd":3,"last update":"Sat Sep 23 19:42:25 2023","interfaces":[{"interface":"back","average":{"1min":1.4359999999999999,"5min":2.2269999999999999,"15min":1.5860000000000001},"min":{"1min":0.29399999999999998,"5min":0.27900000000000003,"15min":0.251},"max":{"1min":8.2590000000000003,"5min":14.212,"15min":14.212},"last":2.8820000000000001},{"interface":"front","average":{"1min":1.351,"5min":1.877,"15min":1.5940000000000001},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":6.5659999999999998,"5min":10.968999999999999,"15min":12.991},"last":3.0459999999999998}]},{"osd":4,"last update":"Sat Sep 23 19:41:56 2023","interfaces":[{"interface":"back","average":{"1min":0.83799999999999997,"5min":2.032,"15min":1.4890000000000001},"min":{"1min":0.36799999999999999,"5min":0.36599999999999999,"15min":0.34699999999999998},"max":{"1min":1.5269999999999999,"5min":11.792999999999999,"15min":11.792999999999999},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.90200000000000002,"5min":2.077,"15min":1.47},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":1.385,"5min":13.641999999999999,"15min":13.641999999999999},"last":2.4209999999999998}]},{"osd":5,"last update":"Sat Sep 23 19:41:56 2023","interfaces":[{"interface":"back","average":{"1min":0.91100000000000003,"5min":2.0099999999999998,"15min":1.7370000000000001},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":1.6339999999999999,"5min":8.6229999999999993,"15min":13.154},"last":2.3799999999999999},{"interface":"front","average":{"1min":0.96399999999999997,"5min":2.165,"15min":1.845},"min":{"1min":0.42799999999999999,"5min":0.40200000000000002,"15min":0.36899999999999999},"max":{"1min":1.8740000000000001,"5min":10.411,"15min":13.109999999999999},"last":0.52300000000000002}]},{"osd":6,"last update":"Sat Sep 23 19:41:56 2023","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":2.673,"15min":1.748},"min":{"1min":0.54900000000000004,"5min":0.40500000000000003,"15min":0.38400000000000001},"max":{"1min":5.375,"5min":40.935000000000002,"15min":40.935000000000002},"last":0.48099999999999998},{"interface":"front","average":{"1min":1.127,"5min":2.6200000000000001,"15min":1.746},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.36699999999999999},"max":{"1min":3.9780000000000002,"5min":40.787999999999997,"15min":40.787999999999997},"last":6.7430000000000003}]},{"osd":7,"last update":"Sat Sep 23 19:41:56 2023","interfaces":[{"interface":"back","average":{"1min":1.2010000000000001,"5min":1.6559999999999999,"15min":1.5049999999999999},"min":{"1min":0.69699999999999995,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":2.5910000000000002,"5min":7.2400000000000002,"15min":12.385},"last":0.621},{"interface":"front","average":{"1min":0.97899999999999998,"5min":1.9770000000000001,"15min":1.7090000000000001},"min":{"1min":0.43099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":1.772,"5min":10.561,"15min":10.561},"last":1.673}]}]},{"osd":2,"up_from":159,"seq":682899800111,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1892392,"kb_used_data":1691936,"kb_used_omap":7,"kb_used_meta":200440,"kb_avail":91852760,"statfs":{"total":95995035648,"available":94057226240,"internally_reserved":0,"allocated":1732542464,"data_stored":1731807726,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":205250641},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:41:51 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":1.9590000000000001,"15min":2.0950000000000002},"min":{"1min":0.38300000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.0680000000000001,"5min":10.194000000000001,"15min":10.194000000000001},"last":0.435},{"interface":"front","average":{"1min":0.75900000000000001,"5min":1.8120000000000001,"15min":1.9219999999999999},"min":{"1min":0.379,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":1.1819999999999999,"5min":8.8049999999999997,"15min":8.8049999999999997},"last":1.9830000000000001}]},{"osd":1,"last update":"Sat Sep 23 19:41:51 2023","interfaces":[{"interface":"back","average":{"1min":0.72299999999999998,"5min":0.97499999999999998,"15min":0.93500000000000005},"min":{"1min":0.46000000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":2.294,"5min":5.4320000000000004,"15min":5.4320000000000004},"last":0.78000000000000003},{"interface":"front","average":{"1min":0.75600000000000001,"5min":1.23,"15min":1.3540000000000001},"min":{"1min":0.58799999999999997,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.98999999999999999,"5min":8.7880000000000003,"15min":8.7880000000000003},"last":0.36399999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:41:51 2023","interfaces":[{"interface":"back","average":{"1min":0.82499999999999996,"5min":1.6799999999999999,"15min":1.891},"min":{"1min":0.438,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":2.3140000000000001,"5min":8.0489999999999995,"15min":8.0489999999999995},"last":0.52900000000000003},{"interface":"front","average":{"1min":1.3420000000000001,"5min":1.9930000000000001,"15min":2.0670000000000002},"min":{"1min":0.497,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":9.3780000000000001,"5min":9.3780000000000001,"15min":9.3780000000000001},"last":0.33100000000000002}]},{"osd":4,"last update":"Sat Sep 23 19:41:51 2023","interfaces":[{"interface":"back","average":{"1min":1.3129999999999999,"5min":1.651,"15min":1.526},"min":{"1min":0.75600000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":9.1020000000000003,"5min":9.1020000000000003,"15min":9.1020000000000003},"last":3.105},{"interface":"front","average":{"1min":1.002,"5min":1.5369999999999999,"15min":1.579},"min":{"1min":0.46300000000000002,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":2.2149999999999999,"5min":6.8570000000000002,"15min":6.8570000000000002},"last":0.60599999999999998}]},{"osd":5,"last update":"Sat Sep 23 19:41:51 2023","interfaces":[{"interface":"back","average":{"1min":0.95799999999999996,"5min":1.5669999999999999,"15min":1.7250000000000001},"min":{"1min":0.42199999999999999,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.3660000000000001,"5min":9.1180000000000003,"15min":9.1180000000000003},"last":0.41199999999999998},{"interface":"front","average":{"1min":1.343,"5min":1.849,"15min":1.863},"min":{"1min":0.55600000000000005,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":8.9930000000000003,"5min":8.9930000000000003,"15min":8.9930000000000003},"last":2.02}]},{"osd":6,"last update":"Sat Sep 23 19:41:51 2023","interfaces":[{"interface":"back","average":{"1min":1.4470000000000001,"5min":2.0499999999999998,"15min":1.9259999999999999},"min":{"1min":0.72099999999999997,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":9.9659999999999993,"5min":11.851000000000001,"15min":11.851000000000001},"last":2.0419999999999998},{"interface":"front","average":{"1min":1.52,"5min":1.827,"15min":1.7430000000000001},"min":{"1min":0.71499999999999997,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":9.4909999999999997,"5min":11.826000000000001,"15min":11.826000000000001},"last":1.9239999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:41:51 2023","interfaces":[{"interface":"back","average":{"1min":1.5089999999999999,"5min":1.5489999999999999,"15min":1.5589999999999999},"min":{"1min":0.69199999999999995,"5min":0.441,"15min":0.441},"max":{"1min":9.4570000000000007,"5min":9.4570000000000007,"15min":9.4570000000000007},"last":1.9630000000000001},{"interface":"front","average":{"1min":1.081,"5min":1.403,"15min":1.46},"min":{"1min":0.48899999999999999,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":2.2469999999999999,"5min":7.9169999999999998,"15min":7.9169999999999998},"last":0.56799999999999995}]}]},{"osd":3,"up_from":116,"seq":498216206511,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2347328,"kb_used_data":1972024,"kb_used_omap":2,"kb_used_meta":375293,"kb_avail":91397824,"statfs":{"total":95995035648,"available":93591371776,"internally_reserved":0,"allocated":2019352576,"data_stored":2018653568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":384300085},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:41:32 2023","interfaces":[{"interface":"back","average":{"1min":1.2230000000000001,"5min":1.552,"15min":1.3779999999999999},"min":{"1min":0.33200000000000002,"5min":0.23200000000000001,"15min":0.19900000000000001},"max":{"1min":7.8360000000000003,"5min":11.439,"15min":13.411},"last":0.40500000000000003},{"interface":"front","average":{"1min":1.099,"5min":1.6819999999999999,"15min":1.3700000000000001},"min":{"1min":0.35399999999999998,"5min":0.23300000000000001,"15min":0.23100000000000001},"max":{"1min":5.2130000000000001,"5min":9.5069999999999997,"15min":10.77},"last":2.5819999999999999}]},{"osd":1,"last update":"Sat Sep 23 19:41:32 2023","interfaces":[{"interface":"back","average":{"1min":1.1359999999999999,"5min":1.0780000000000001,"15min":1.2629999999999999},"min":{"1min":0.499,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":5.109,"5min":6.4390000000000001,"15min":13.659000000000001},"last":0.53600000000000003},{"interface":"front","average":{"1min":1.1040000000000001,"5min":1.1930000000000001,"15min":1.47},"min":{"1min":0.435,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":4.1369999999999996,"5min":9.4130000000000003,"15min":16.262},"last":0.43099999999999999}]},{"osd":2,"last update":"Sat Sep 23 19:41:49 2023","interfaces":[{"interface":"back","average":{"1min":0.85799999999999998,"5min":2.5539999999999998,"15min":2.5600000000000001},"min":{"1min":0.624,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":1.522,"5min":10.960000000000001,"15min":10.960000000000001},"last":2.617},{"interface":"front","average":{"1min":0.89900000000000002,"5min":1.6180000000000001,"15min":1.71},"min":{"1min":0.64200000000000002,"5min":0.254,"15min":0.254},"max":{"1min":2.0579999999999998,"5min":8.7560000000000002,"15min":8.7560000000000002},"last":1.8859999999999999}]},{"osd":4,"last update":"Sat Sep 23 19:41:32 2023","interfaces":[{"interface":"back","average":{"1min":1.387,"5min":1.8360000000000001,"15min":1.5},"min":{"1min":0.48899999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":4.1790000000000003,"5min":9.6229999999999993,"15min":9.6229999999999993},"last":0.45700000000000002},{"interface":"front","average":{"1min":1.2709999999999999,"5min":1.905,"15min":1.494},"min":{"1min":0.59199999999999997,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":3.4670000000000001,"5min":9.6539999999999999,"15min":10.926},"last":0.57099999999999995}]},{"osd":5,"last update":"Sat Sep 23 19:41:32 2023","interfaces":[{"interface":"back","average":{"1min":1.4650000000000001,"5min":1.7110000000000001,"15min":1.635},"min":{"1min":0.66300000000000003,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":5.181,"5min":8.0169999999999995,"15min":10.82},"last":0.70299999999999996},{"interface":"front","average":{"1min":1.4139999999999999,"5min":1.55,"15min":1.4790000000000001},"min":{"1min":0.53000000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":3.9550000000000001,"5min":7.5049999999999999,"15min":8.7710000000000008},"last":3.3069999999999999}]},{"osd":6,"last update":"Sat Sep 23 19:41:32 2023","interfaces":[{"interface":"back","average":{"1min":1.621,"5min":1.9359999999999999,"15min":1.581},"min":{"1min":0.68999999999999995,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":5.1520000000000001,"5min":7.4409999999999998,"15min":10.859},"last":2.6589999999999998},{"interface":"front","average":{"1min":1.6539999999999999,"5min":1.494,"15min":1.353},"min":{"1min":0.58699999999999997,"5min":0.315,"15min":0.315},"max":{"1min":6.6310000000000002,"5min":6.6310000000000002,"15min":8.0190000000000001},"last":6.5579999999999998}]},{"osd":7,"last update":"Sat Sep 23 19:41:32 2023","interfaces":[{"interface":"back","average":{"1min":1.337,"5min":1.395,"15min":1.448},"min":{"1min":0.67600000000000005,"5min":0.35799999999999998,"15min":0.32400000000000001},"max":{"1min":2.8109999999999999,"5min":7.7670000000000003,"15min":11.259},"last":1.917},{"interface":"front","average":{"1min":1.5660000000000001,"5min":1.6439999999999999,"15min":1.5409999999999999},"min":{"1min":0.55400000000000005,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":5.5970000000000004,"5min":9.5510000000000002,"15min":9.5510000000000002},"last":0.504}]}]},{"osd":4,"up_from":30,"seq":128849019381,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1998048,"kb_used_data":1102232,"kb_used_omap":0,"kb_used_meta":895808,"kb_avail":91747104,"statfs":{"total":95995035648,"available":93949034496,"internally_reserved":0,"allocated":1128685568,"data_stored":1128191810,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":917307392},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,0,0,0,0,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:41:57 2023","interfaces":[{"interface":"back","average":{"1min":0.80100000000000005,"5min":1.917,"15min":1.2589999999999999},"min":{"1min":0.42999999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":1.341,"5min":12.975,"15min":12.975},"last":1.2350000000000001},{"interface":"front","average":{"1min":0.76400000000000001,"5min":1.714,"15min":1.244},"min":{"1min":0.41099999999999998,"5min":0.27700000000000002,"15min":0.26500000000000001},"max":{"1min":1.5349999999999999,"5min":11.380000000000001,"15min":11.380000000000001},"last":0.59599999999999997}]},{"osd":1,"last update":"Sat Sep 23 19:42:26 2023","interfaces":[{"interface":"back","average":{"1min":7.6219999999999999,"5min":2.8239999999999998,"15min":1.706},"min":{"1min":0.30099999999999999,"5min":0.29299999999999998,"15min":0.27700000000000002},"max":{"1min":101.786,"5min":101.786,"15min":101.786},"last":0.747},{"interface":"front","average":{"1min":7.7329999999999997,"5min":2.9049999999999998,"15min":1.8009999999999999},"min":{"1min":0.316,"5min":0.309,"15min":0.309},"max":{"1min":101.726,"5min":101.726,"15min":101.726},"last":0.79900000000000004}]},{"osd":2,"last update":"Sat Sep 23 19:41:45 2023","interfaces":[{"interface":"back","average":{"1min":0.74199999999999999,"5min":2.6080000000000001,"15min":3.2719999999999998},"min":{"1min":0.35899999999999999,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":1.3049999999999999,"5min":11.768000000000001,"15min":11.768000000000001},"last":0.63200000000000001},{"interface":"front","average":{"1min":1.1819999999999999,"5min":1.929,"15min":1.893},"min":{"1min":0.42199999999999999,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":8.1699999999999999,"5min":9.1479999999999997,"15min":9.1479999999999997},"last":1.0629999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:41:39 2023","interfaces":[{"interface":"back","average":{"1min":1.123,"5min":1.7869999999999999,"15min":1.6859999999999999},"min":{"1min":0.46600000000000003,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":5.0060000000000002,"5min":12.973000000000001,"15min":49.515000000000001},"last":0.92600000000000005},{"interface":"front","average":{"1min":1.0409999999999999,"5min":1.839,"15min":1.7110000000000001},"min":{"1min":0.52600000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":4.0990000000000002,"5min":10.266,"15min":51.518999999999998},"last":1.151}]},{"osd":5,"last update":"Sat Sep 23 19:41:57 2023","interfaces":[{"interface":"back","average":{"1min":0.874,"5min":1.825,"15min":1.5109999999999999},"min":{"1min":0.48899999999999999,"5min":0.315,"15min":0.28699999999999998},"max":{"1min":1.1579999999999999,"5min":10.944000000000001,"15min":10.944000000000001},"last":1.2070000000000001},{"interface":"front","average":{"1min":0.86899999999999999,"5min":1.925,"15min":1.488},"min":{"1min":0.46899999999999997,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":1.2889999999999999,"5min":10.362,"15min":10.362},"last":1.3140000000000001}]},{"osd":6,"last update":"Sat Sep 23 19:42:26 2023","interfaces":[{"interface":"back","average":{"1min":1.0840000000000001,"5min":2.133,"15min":1.5209999999999999},"min":{"1min":0.45500000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":3.2789999999999999,"5min":11.076000000000001,"15min":11.076000000000001},"last":1.286},{"interface":"front","average":{"1min":0.90600000000000003,"5min":1.9470000000000001,"15min":1.4199999999999999},"min":{"1min":0.49199999999999999,"5min":0.39100000000000001,"15min":0.34000000000000002},"max":{"1min":3.2610000000000001,"5min":10.952,"15min":10.952},"last":1.349}]},{"osd":7,"last update":"Sat Sep 23 19:42:26 2023","interfaces":[{"interface":"back","average":{"1min":1.399,"5min":1.6040000000000001,"15min":1.335},"min":{"1min":0.50900000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":6.3650000000000002,"5min":9.2149999999999999,"15min":9.2149999999999999},"last":1.9710000000000001},{"interface":"front","average":{"1min":1.1459999999999999,"5min":1.819,"15min":1.5249999999999999},"min":{"1min":0.56200000000000006,"5min":0.374,"15min":0.34499999999999997},"max":{"1min":4.3310000000000004,"5min":10.989000000000001,"15min":10.989000000000001},"last":1.956}]}]},{"osd":5,"up_from":36,"seq":154618823148,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2011912,"kb_used_data":1105024,"kb_used_omap":0,"kb_used_meta":906880,"kb_avail":91733240,"statfs":{"total":95995035648,"available":93934837760,"internally_reserved":0,"allocated":1131544576,"data_stored":1131070508,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":928645120},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":1.8,"15min":1.532},"min":{"1min":0.42499999999999999,"5min":0.26300000000000001,"15min":0.26100000000000001},"max":{"1min":2.9900000000000002,"5min":43.094000000000001,"15min":43.094000000000001},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.77800000000000002,"5min":2.3940000000000001,"15min":1.9910000000000001},"min":{"1min":0.32300000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":1.206,"5min":43.802999999999997,"15min":43.802999999999997},"last":0.84799999999999998}]},{"osd":1,"last update":"Sat Sep 23 19:42:26 2023","interfaces":[{"interface":"back","average":{"1min":0.88900000000000001,"5min":1.335,"15min":1.3540000000000001},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":2.1589999999999998,"5min":6.8689999999999998,"15min":32.253999999999998},"last":0.78400000000000003},{"interface":"front","average":{"1min":1.121,"5min":1.591,"15min":1.7250000000000001},"min":{"1min":0.39600000000000002,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":2.3820000000000001,"5min":18.207000000000001,"15min":32.277999999999999},"last":0.82199999999999995}]},{"osd":2,"last update":"Sat Sep 23 19:41:46 2023","interfaces":[{"interface":"back","average":{"1min":0.84399999999999997,"5min":2.5430000000000001,"15min":2.8889999999999998},"min":{"1min":0.56599999999999995,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":1.208,"5min":10.611000000000001,"15min":10.611000000000001},"last":0.35099999999999998},{"interface":"front","average":{"1min":1.528,"5min":1.9950000000000001,"15min":1.9770000000000001},"min":{"1min":0.56000000000000005,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":7.2519999999999998,"5min":8.6120000000000001,"15min":8.6120000000000001},"last":0.80700000000000005}]},{"osd":3,"last update":"Sat Sep 23 19:42:26 2023","interfaces":[{"interface":"back","average":{"1min":0.99399999999999999,"5min":1.7509999999999999,"15min":1.5149999999999999},"min":{"1min":0.379,"5min":0.36799999999999999,"15min":0.311},"max":{"1min":2.2360000000000002,"5min":10.686,"15min":10.885999999999999},"last":0.622},{"interface":"front","average":{"1min":1.298,"5min":1.895,"15min":1.4410000000000001},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.33800000000000002},"max":{"1min":6.9260000000000002,"5min":8.5879999999999992,"15min":8.5879999999999992},"last":6.9260000000000002}]},{"osd":4,"last update":"Sat Sep 23 19:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.80200000000000005,"5min":2.4870000000000001,"15min":1.776},"min":{"1min":0.36199999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":2.3199999999999998,"5min":74.221000000000004,"15min":74.221000000000004},"last":1.7529999999999999},{"interface":"front","average":{"1min":0.86399999999999999,"5min":2.8940000000000001,"15min":1.778},"min":{"1min":0.432,"5min":0.33700000000000002,"15min":0.30599999999999999},"max":{"1min":1.603,"5min":73.772999999999996,"15min":73.772999999999996},"last":0.65600000000000003}]},{"osd":6,"last update":"Sat Sep 23 19:41:39 2023","interfaces":[{"interface":"back","average":{"1min":1.2070000000000001,"5min":2.1019999999999999,"15min":1.4379999999999999},"min":{"1min":0.45000000000000001,"5min":0.35899999999999999,"15min":0.32000000000000001},"max":{"1min":4.484,"5min":16.222000000000001,"15min":16.222000000000001},"last":2.621},{"interface":"front","average":{"1min":1.468,"5min":2.3010000000000002,"15min":1.649},"min":{"1min":0.63300000000000001,"5min":0.33500000000000002,"15min":0.309},"max":{"1min":5.7270000000000003,"5min":14.178000000000001,"15min":14.178000000000001},"last":1.131}]},{"osd":7,"last update":"Sat Sep 23 19:42:06 2023","interfaces":[{"interface":"back","average":{"1min":0.92700000000000005,"5min":1.8029999999999999,"15min":1.6379999999999999},"min":{"1min":0.47899999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":1.671,"5min":14.127000000000001,"15min":14.127000000000001},"last":1.379},{"interface":"front","average":{"1min":0.98999999999999999,"5min":1.7949999999999999,"15min":1.5680000000000001},"min":{"1min":0.46300000000000002,"5min":0.307,"15min":0.307},"max":{"1min":2.2570000000000001,"5min":9.0440000000000005,"15min":13.18},"last":2.573}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1482489856,"data_stored":1482489856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2267283456,"data_stored":2267283456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1730740224,"data_stored":1730740224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2036531200,"data_stored":2036531200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1128464384,"data_stored":1128464384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1142882304,"data_stored":1142882304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2251685888,"data_stored":2251685888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1701117952,"data_stored":1701117952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T19:42:37.625 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T19:42:37.625 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T19:42:37.625 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T19:42:37.626 INFO:tasks.thrashosds.thrasher:Setting osd 0 primary_affinity to 0.456309 2023-09-23T19:42:37.626 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 0 0.4563093629283942 2023-09-23T19:42:37.659 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:37.706 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:37.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:37.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:37.809 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:37 smithi016 bash[75004]: cluster 2023-09-23T19:42:36.218924+0000 mgr.y (mgr.25325) 1055 : cluster 0 pgmap v1010: 89 pgs: 89 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-09-23T19:42:37.809 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:37 smithi016 bash[77995]: cluster 2023-09-23T19:42:36.218924+0000 mgr.y (mgr.25325) 1055 : cluster 0 pgmap v1010: 89 pgs: 89 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-09-23T19:42:37.811 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:37.854 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:37.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:37.858 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:37.959 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:37.993 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:37.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:37.997 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:38.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:37 smithi092 bash[58855]: cluster 2023-09-23T19:42:36.218924+0000 mgr.y (mgr.25325) 1055 : cluster 0 pgmap v1010: 89 pgs: 89 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-09-23T19:42:38.098 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:38.129 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:38.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:38.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:42:38.226 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 54 16 76663 76647 88.701 56.5625 0.0101539 0.0112675 2023-09-23T19:42:38.232 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:38.259 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:38.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:38.263 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:42:38.364 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:38.391 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:38.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:38.396 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:38.497 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:38.523 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:38.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:38.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:38.626 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:38.654 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:38.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:38.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:42:38.719 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is set 2023-09-23T19:42:38.759 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:38 smithi016 bash[75004]: audit 2023-09-23T19:42:38.385883+0000 mon.a (mon.0) 1347 : audit 1 from='client.? 172.21.15.16:0/1314564431' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:42:38.759 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:38 smithi016 bash[77995]: audit 2023-09-23T19:42:38.385883+0000 mon.a (mon.0) 1347 : audit 1 from='client.? 172.21.15.16:0/1314564431' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:42:38.759 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:38.786 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:38.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:38.789 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:42:38.890 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:38.915 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:38.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:38.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:39.019 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:39.028 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:38 smithi092 bash[58855]: audit 2023-09-23T19:42:38.385883+0000 mon.a (mon.0) 1347 : audit 1 from='client.? 172.21.15.16:0/1314564431' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:42:39.061 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:39.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:39.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:42:39.164 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:39.192 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:39.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:39.197 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:39.226 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 55 16 77679 77663 88.2427 63.5 0.0106472 0.0113271 2023-09-23T19:42:39.298 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:39.323 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:39.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:39.328 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:39.428 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:39.451 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:39.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:39.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:42:39.558 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:39.592 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:39.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:39.595 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:39.696 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:39.702 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:39 smithi092 bash[58855]: cluster 2023-09-23T19:42:38.219352+0000 mgr.y (mgr.25325) 1056 : cluster 0 pgmap v1011: 89 pgs: 89 active+clean; 4.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 878 op/s 2023-09-23T19:42:39.703 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:39 smithi092 bash[58855]: cluster 2023-09-23T19:42:38.694413+0000 mon.a (mon.0) 1348 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:42:39.703 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:39 smithi092 bash[58855]: audit 2023-09-23T19:42:38.707876+0000 mon.a (mon.0) 1349 : audit 1 from='client.? 172.21.15.16:0/1314564431' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:42:39.703 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:39 smithi092 bash[58855]: cluster 2023-09-23T19:42:38.708060+0000 mon.a (mon.0) 1350 : cluster 0 osdmap e187: 8 total, 8 up, 8 in 2023-09-23T19:42:39.703 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:39 smithi092 bash[58855]: cluster 2023-09-23T19:42:38.710154+0000 osd.4 (osd.4) 126 : cluster 3 failed to encode map e187 with expected crc 2023-09-23T19:42:39.703 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:39 smithi092 bash[58855]: cluster 2023-09-23T19:42:38.725840+0000 osd.6 (osd.6) 108 : cluster 3 failed to encode map e187 with expected crc 2023-09-23T19:42:39.720 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:39.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:39.722 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:39.823 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:39.832 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:39 smithi016 bash[75004]: cluster 2023-09-23T19:42:38.219352+0000 mgr.y (mgr.25325) 1056 : cluster 0 pgmap v1011: 89 pgs: 89 active+clean; 4.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 878 op/s 2023-09-23T19:42:39.832 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:39 smithi016 bash[75004]: cluster 2023-09-23T19:42:38.694413+0000 mon.a (mon.0) 1348 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:42:39.832 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:39 smithi016 bash[75004]: audit 2023-09-23T19:42:38.707876+0000 mon.a (mon.0) 1349 : audit 1 from='client.? 172.21.15.16:0/1314564431' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:42:39.832 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:39 smithi016 bash[75004]: cluster 2023-09-23T19:42:38.708060+0000 mon.a (mon.0) 1350 : cluster 0 osdmap e187: 8 total, 8 up, 8 in 2023-09-23T19:42:39.832 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:39 smithi016 bash[75004]: cluster 2023-09-23T19:42:38.710154+0000 osd.4 (osd.4) 126 : cluster 3 failed to encode map e187 with expected crc 2023-09-23T19:42:39.833 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:39 smithi016 bash[75004]: cluster 2023-09-23T19:42:38.725840+0000 osd.6 (osd.6) 108 : cluster 3 failed to encode map e187 with expected crc 2023-09-23T19:42:39.833 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:39 smithi016 bash[77995]: cluster 2023-09-23T19:42:38.219352+0000 mgr.y (mgr.25325) 1056 : cluster 0 pgmap v1011: 89 pgs: 89 active+clean; 4.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 878 op/s 2023-09-23T19:42:39.833 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:39 smithi016 bash[77995]: cluster 2023-09-23T19:42:38.694413+0000 mon.a (mon.0) 1348 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:42:39.833 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:39 smithi016 bash[77995]: audit 2023-09-23T19:42:38.707876+0000 mon.a (mon.0) 1349 : audit 1 from='client.? 172.21.15.16:0/1314564431' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:42:39.833 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:39 smithi016 bash[77995]: cluster 2023-09-23T19:42:38.708060+0000 mon.a (mon.0) 1350 : cluster 0 osdmap e187: 8 total, 8 up, 8 in 2023-09-23T19:42:39.833 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:39 smithi016 bash[77995]: cluster 2023-09-23T19:42:38.710154+0000 osd.4 (osd.4) 126 : cluster 3 failed to encode map e187 with expected crc 2023-09-23T19:42:39.833 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:39 smithi016 bash[77995]: cluster 2023-09-23T19:42:38.725840+0000 osd.6 (osd.6) 108 : cluster 3 failed to encode map e187 with expected crc 2023-09-23T19:42:39.850 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:39.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:39.857 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:39.958 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:39.992 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:39.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:39.996 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:40.097 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:40.123 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:40.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:40.128 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:42:40.227 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 56 16 78667 78651 87.7695 61.75 0.0070289 0.0113876 2023-09-23T19:42:40.229 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:40.254 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:40.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:40.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:40.323 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:42:40.359 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:40.396 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:40.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:40.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:40.499 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:40.530 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:40.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:40.537 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:40.637 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:40.663 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:40.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:40.670 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:40.674 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:42:40.771 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:40.795 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:40.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:40.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:40.905 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:40.935 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:40.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:40.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:41.040 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:41.064 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:41.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:41.068 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:42:41.169 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:41.201 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:41.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:41.203 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:42:41.227 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 57 16 79564 79548 87.2132 56.0625 0.0038395 0.0114589 2023-09-23T19:42:41.305 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:41.332 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:41.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:41.336 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:41.436 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:41.468 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:41.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:41.474 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:41.574 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:41.601 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:41.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:41.604 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:41.705 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:41.708 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:41 smithi016 bash[75004]: cluster 2023-09-23T19:42:40.219954+0000 mgr.y (mgr.25325) 1057 : cluster 0 pgmap v1013: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 978 op/s 2023-09-23T19:42:41.708 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:41 smithi016 bash[77995]: cluster 2023-09-23T19:42:40.219954+0000 mgr.y (mgr.25325) 1057 : cluster 0 pgmap v1013: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 978 op/s 2023-09-23T19:42:41.736 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:41.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:41.742 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:41.842 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:41.867 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:41.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:41.870 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:41.908 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T19:42:41.971 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:41.979 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:41 smithi092 bash[58855]: cluster 2023-09-23T19:42:40.219954+0000 mgr.y (mgr.25325) 1057 : cluster 0 pgmap v1013: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 978 op/s 2023-09-23T19:42:42.003 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:42.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:42.005 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:42:42.106 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:42.132 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:42.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:42.135 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:42.227 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 58 16 80508 80492 86.7262 59 0.0189956 0.011525 2023-09-23T19:42:42.236 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:42.271 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:42.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:42.277 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:42:42.378 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:42.405 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:42.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:42.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:42.509 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:42.536 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:42.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:42.538 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:42:42.639 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:42.664 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:42.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:42.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:42:42.772 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:42.799 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:42.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:42.801 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:42.902 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:42.932 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:42.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:42.937 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:42:43.037 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:43.066 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:43.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:43.070 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:43.170 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:43.205 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:43.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:43.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:43.227 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 59 16 81496 81480 86.3025 61.75 0.0141211 0.0115813 2023-09-23T19:42:43.311 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:43.334 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:43.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:43.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:43.441 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:43.477 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:43.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:43.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:43.580 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:43.614 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:43.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:43.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:42:43.633 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:42:43.720 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:43.748 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:43.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:43.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:43.852 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:43.861 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:43 smithi092 bash[58855]: cluster 2023-09-23T19:42:42.224860+0000 mgr.y (mgr.25325) 1058 : cluster 0 pgmap v1014: 89 pgs: 89 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 971 op/s 2023-09-23T19:42:43.861 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:43 smithi092 bash[58855]: audit 2023-09-23T19:42:43.661496+0000 mon.a (mon.0) 1351 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:42:43.888 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:43.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:43.892 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:42:43.993 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:44.018 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:44.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:44.021 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:42:44.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:43 smithi016 bash[75004]: cluster 2023-09-23T19:42:42.224860+0000 mgr.y (mgr.25325) 1058 : cluster 0 pgmap v1014: 89 pgs: 89 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 971 op/s 2023-09-23T19:42:44.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:43 smithi016 bash[75004]: audit 2023-09-23T19:42:43.661496+0000 mon.a (mon.0) 1351 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:42:44.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:43 smithi016 bash[77995]: cluster 2023-09-23T19:42:42.224860+0000 mgr.y (mgr.25325) 1058 : cluster 0 pgmap v1014: 89 pgs: 89 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 971 op/s 2023-09-23T19:42:44.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:43 smithi016 bash[77995]: audit 2023-09-23T19:42:43.661496+0000 mon.a (mon.0) 1351 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:42:44.122 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:44.150 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:44.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:44.156 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:44.227 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-09-23T19:42:44.224705+0000 min lat: 0.00130197 max lat: 0.165975 avg lat: 0.0116413 2023-09-23T19:42:44.228 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:42:44.228 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 60 16 82452 82436 85.8599 59.75 0.022814 0.0116413 2023-09-23T19:42:44.256 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:44.285 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:44.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:44.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:44.390 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:44.415 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:44.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:44.424 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:42:44.525 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:44.548 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:44.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:44.550 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:42:44.651 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:44.685 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:44.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:44.689 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:44.790 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:44.809 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:44.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:44.812 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:44.912 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:44.942 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:44.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:44.945 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:45.045 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:45.066 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:45.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:45.068 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:45.169 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:45.195 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:45.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:45.199 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:42:45.228 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 61 16 83219 83203 85.2381 47.9375 0.00935716 0.0117259 2023-09-23T19:42:45.299 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:45.329 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:45.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:45.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:42:45.434 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:45.464 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:45.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:45.468 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:45.555 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:42:45.568 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:45.592 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:45.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:45.596 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:42:45.696 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:45.725 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:45.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:45.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:42:45.811 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:45 smithi016 bash[77995]: cluster 2023-09-23T19:42:44.225449+0000 mgr.y (mgr.25325) 1059 : cluster 0 pgmap v1015: 89 pgs: 89 active+clean; 4.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 966 op/s 2023-09-23T19:42:45.812 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:45 smithi016 bash[75004]: cluster 2023-09-23T19:42:44.225449+0000 mgr.y (mgr.25325) 1059 : cluster 0 pgmap v1015: 89 pgs: 89 active+clean; 4.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 966 op/s 2023-09-23T19:42:45.812 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:42:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:42:45] "GET /metrics HTTP/1.1" 200 36304 "" "Prometheus/2.33.4" 2023-09-23T19:42:45.831 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:45.864 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-09-23T19:42:45.864 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:42:45.864 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:45.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:45.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:45.969 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:45.977 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:45 smithi092 bash[58855]: cluster 2023-09-23T19:42:44.225449+0000 mgr.y (mgr.25325) 1059 : cluster 0 pgmap v1015: 89 pgs: 89 active+clean; 4.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 966 op/s 2023-09-23T19:42:46.006 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:46.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:46.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:46.111 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:46.137 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:46.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:46.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:46.228 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 62 16 84134 84118 84.7856 57.1875 0.0154054 0.0117887 2023-09-23T19:42:46.241 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:46.280 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:46.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:46.285 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:42:46.386 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:46.413 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:46.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:46.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:42:46.517 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:46.557 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:46.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:46.563 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:42:46.664 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:46.691 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:46.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:46.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:46.796 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:46.819 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:46.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:46.823 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:46.924 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:46.946 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:46.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:46.950 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:42:47.051 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:47.078 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:47.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:47.081 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:47.182 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:47.215 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:47.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:47.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:47.228 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 63 16 85104 85088 84.402 60.625 0.00731656 0.0118424 2023-09-23T19:42:47.320 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:47.346 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:47.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:47.349 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:42:47.450 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:47.509 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:47.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:47.512 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:47.613 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:47.646 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:47.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:47.651 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:47.724 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:47 smithi016 bash[75004]: cluster 2023-09-23T19:42:46.226219+0000 mgr.y (mgr.25325) 1060 : cluster 0 pgmap v1016: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 913 op/s 2023-09-23T19:42:47.752 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:47.786 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:47.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:47.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:47.891 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:47.920 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:47.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:47.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:48.024 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:48.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:47 smithi016 bash[77995]: cluster 2023-09-23T19:42:46.226219+0000 mgr.y (mgr.25325) 1060 : cluster 0 pgmap v1016: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 913 op/s 2023-09-23T19:42:48.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:47 smithi092 bash[58855]: cluster 2023-09-23T19:42:46.226219+0000 mgr.y (mgr.25325) 1060 : cluster 0 pgmap v1016: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 913 op/s 2023-09-23T19:42:48.048 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:48.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:48.052 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:48.152 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:48.175 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:48.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:48.179 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:42:48.228 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 64 16 86098 86082 84.0539 62.125 0.0198993 0.011892 2023-09-23T19:42:48.281 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:48.314 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:48.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:48.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:42:48.420 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:48.447 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:48.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:48.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:48.550 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:48.577 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:48.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:48.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:48.681 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:48.705 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:48.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:48.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:42:48.809 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:48.847 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:48.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:48.848 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:42:48.949 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:48.976 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:48.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:48.980 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:42:49.080 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:49.111 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:49.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:49.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:42:49.216 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:49.228 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 65 16 87056 87040 83.6818 59.875 0.00928967 0.0119442 2023-09-23T19:42:49.241 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:49.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:49.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:42:49.348 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:49.375 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:49.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:49.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:42:49.479 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:49.519 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:49.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:49.524 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:49.557 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T19:42:49.625 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:49.696 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:49.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:49.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:49.732 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:49 smithi092 bash[58855]: cluster 2023-09-23T19:42:48.226723+0000 mgr.y (mgr.25325) 1061 : cluster 0 pgmap v1017: 89 pgs: 89 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 907 op/s 2023-09-23T19:42:49.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:49 smithi016 bash[75004]: cluster 2023-09-23T19:42:48.226723+0000 mgr.y (mgr.25325) 1061 : cluster 0 pgmap v1017: 89 pgs: 89 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 907 op/s 2023-09-23T19:42:49.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:49 smithi016 bash[77995]: cluster 2023-09-23T19:42:48.226723+0000 mgr.y (mgr.25325) 1061 : cluster 0 pgmap v1017: 89 pgs: 89 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 907 op/s 2023-09-23T19:42:49.801 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:49.827 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:49.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:49.832 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:42:49.933 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:49.959 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:49.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:49.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:42:50.036 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:42:50.064 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:50.100 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:50.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:50.104 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:50.205 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:50.228 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 66 16 87861 87845 83.1761 50.3125 0.0087551 0.0120158 2023-09-23T19:42:50.230 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:50.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:50.236 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:42:50.336 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:50.363 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:50.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:50.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:42:50.468 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:50.495 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:50.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:50.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:42:50.600 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:50.637 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:50.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:50.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:42:50.742 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:50.796 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:50.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:50.800 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:42:50.900 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:50.937 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:50.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:50.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:51.041 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:51.065 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:51.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:51.068 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:51.169 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:51.197 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:51.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:51.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:42:51.228 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 67 16 88732 88716 82.7471 54.4375 0.00372204 0.0120665 2023-09-23T19:42:51.300 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:51.329 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:51.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:51.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:51.432 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:51.462 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:51.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:51.465 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:51.566 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:51.599 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:51.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:51.605 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:51.705 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:51.734 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:51.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:51.738 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:51.838 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:51.846 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:51 smithi016 bash[77995]: cluster 2023-09-23T19:42:50.227449+0000 mgr.y (mgr.25325) 1062 : cluster 0 pgmap v1018: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T19:42:51.846 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:51 smithi016 bash[75004]: cluster 2023-09-23T19:42:50.227449+0000 mgr.y (mgr.25325) 1062 : cluster 0 pgmap v1018: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T19:42:51.862 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:51.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:51.866 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:51.966 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:51.996 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:52.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:52.002 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:52.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:51 smithi092 bash[58855]: cluster 2023-09-23T19:42:50.227449+0000 mgr.y (mgr.25325) 1062 : cluster 0 pgmap v1018: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T19:42:52.103 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:52.131 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:52.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:52.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:52.228 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 68 16 89653 89637 82.3767 57.5625 0.0185499 0.0121328 2023-09-23T19:42:52.235 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:52.273 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:52.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:52.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:42:52.377 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:52.406 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:52.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:52.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:42:52.509 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:52.531 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:52.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:52.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:52.638 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:52.665 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:52.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:52.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:52.770 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:52.801 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:52.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:52.807 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:52.908 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:52.937 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:52.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:52.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:53.043 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:53.074 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:53.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:53.077 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:42:53.177 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:53.201 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:53.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:53.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:53.228 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 69 16 90574 90558 82.017 57.5625 0.00842846 0.0121871 2023-09-23T19:42:53.307 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:53.337 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:53.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:53.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:42:53.444 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:53.473 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:53.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:53.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:53.576 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:53.619 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:53.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:53.624 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:53.725 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:53.756 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:53.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:53.762 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:42:53.862 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:53.896 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:53.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:53.900 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:54.001 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:54.009 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:53 smithi092 bash[58855]: cluster 2023-09-23T19:42:52.228029+0000 mgr.y (mgr.25325) 1063 : cluster 0 pgmap v1019: 89 pgs: 89 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 62 MiB/s wr, 989 op/s 2023-09-23T19:42:54.032 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:54.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:53 smithi016 bash[77995]: cluster 2023-09-23T19:42:52.228029+0000 mgr.y (mgr.25325) 1063 : cluster 0 pgmap v1019: 89 pgs: 89 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 62 MiB/s wr, 989 op/s 2023-09-23T19:42:54.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:53 smithi016 bash[75004]: cluster 2023-09-23T19:42:52.228029+0000 mgr.y (mgr.25325) 1063 : cluster 0 pgmap v1019: 89 pgs: 89 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 62 MiB/s wr, 989 op/s 2023-09-23T19:42:54.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:54.040 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:54.140 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:54.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:54.178 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:54.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:42:54.229 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 70 16 91394 91378 81.5774 51.25 0.0105741 0.0122526 2023-09-23T19:42:54.278 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:54.314 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:54.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:54.320 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:42:54.421 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:54.452 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:54.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:54.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:42:54.556 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:54.584 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:54.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:54.586 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:54.687 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:54.712 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:54.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:54.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:42:54.766 INFO:teuthology.orchestra.run.smithi016.stderr:set osd.0 primary-affinity to 0.456309 (8299042) 2023-09-23T19:42:54.805 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:54 smithi016 bash[77995]: audit 2023-09-23T19:42:53.962027+0000 mon.a (mon.0) 1352 : audit 1 from='client.? 172.21.15.16:0/101800245' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.4563093629283942}]: dispatch 2023-09-23T19:42:54.806 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:54 smithi016 bash[75004]: audit 2023-09-23T19:42:53.962027+0000 mon.a (mon.0) 1352 : audit 1 from='client.? 172.21.15.16:0/101800245' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.4563093629283942}]: dispatch 2023-09-23T19:42:54.821 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:54.860 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:54.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:54.865 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:42:54.966 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:54.978 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:54 smithi092 bash[58855]: audit 2023-09-23T19:42:53.962027+0000 mon.a (mon.0) 1352 : audit 1 from='client.? 172.21.15.16:0/101800245' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.4563093629283942}]: dispatch 2023-09-23T19:42:54.999 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:55.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:55.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:42:55.102 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:55.130 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:55.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:55.134 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:55.229 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 71 16 92067 92051 81.0208 42.0625 0.00456414 0.0122975 2023-09-23T19:42:55.234 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:55.265 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:55.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:55.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:42:55.369 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:55.393 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:55.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:55.396 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:55.497 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:55.527 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:55.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:55.529 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:55.630 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:55.666 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:55.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:55.669 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:42:55.770 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:55.797 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:55.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:55.806 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:55.907 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:55.913 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:42:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:42:55] "GET /metrics HTTP/1.1" 200 36304 "" "Prometheus/2.33.4" 2023-09-23T19:42:55.913 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:55 smithi016 bash[77995]: cluster 2023-09-23T19:42:54.228761+0000 mgr.y (mgr.25325) 1064 : cluster 0 pgmap v1020: 89 pgs: 89 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 47 MiB/s wr, 753 op/s 2023-09-23T19:42:55.914 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:55 smithi016 bash[77995]: audit 2023-09-23T19:42:54.752743+0000 mon.a (mon.0) 1353 : audit 1 from='client.? 172.21.15.16:0/101800245' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.4563093629283942}]': finished 2023-09-23T19:42:55.914 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:55 smithi016 bash[77995]: cluster 2023-09-23T19:42:54.754081+0000 osd.4 (osd.4) 127 : cluster 3 failed to encode map e188 with expected crc 2023-09-23T19:42:55.914 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:55 smithi016 bash[77995]: cluster 2023-09-23T19:42:54.757435+0000 mon.a (mon.0) 1354 : cluster 0 osdmap e188: 8 total, 8 up, 8 in 2023-09-23T19:42:55.914 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:55 smithi016 bash[77995]: cluster 2023-09-23T19:42:54.767463+0000 osd.7 (osd.7) 97 : cluster 3 failed to encode map e188 with expected crc 2023-09-23T19:42:55.914 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:55 smithi016 bash[77995]: cluster 2023-09-23T19:42:54.768360+0000 osd.5 (osd.5) 117 : cluster 3 failed to encode map e188 with expected crc 2023-09-23T19:42:55.914 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:55 smithi016 bash[75004]: cluster 2023-09-23T19:42:54.228761+0000 mgr.y (mgr.25325) 1064 : cluster 0 pgmap v1020: 89 pgs: 89 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 47 MiB/s wr, 753 op/s 2023-09-23T19:42:55.914 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:55 smithi016 bash[75004]: audit 2023-09-23T19:42:54.752743+0000 mon.a (mon.0) 1353 : audit 1 from='client.? 172.21.15.16:0/101800245' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.4563093629283942}]': finished 2023-09-23T19:42:55.915 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:55 smithi016 bash[75004]: cluster 2023-09-23T19:42:54.754081+0000 osd.4 (osd.4) 127 : cluster 3 failed to encode map e188 with expected crc 2023-09-23T19:42:55.915 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:55 smithi016 bash[75004]: cluster 2023-09-23T19:42:54.757435+0000 mon.a (mon.0) 1354 : cluster 0 osdmap e188: 8 total, 8 up, 8 in 2023-09-23T19:42:55.915 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:55 smithi016 bash[75004]: cluster 2023-09-23T19:42:54.767463+0000 osd.7 (osd.7) 97 : cluster 3 failed to encode map e188 with expected crc 2023-09-23T19:42:55.915 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:55 smithi016 bash[75004]: cluster 2023-09-23T19:42:54.768360+0000 osd.5 (osd.5) 117 : cluster 3 failed to encode map e188 with expected crc 2023-09-23T19:42:55.932 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:55.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:55.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:42:56.035 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:56.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:55 smithi092 bash[58855]: cluster 2023-09-23T19:42:54.228761+0000 mgr.y (mgr.25325) 1064 : cluster 0 pgmap v1020: 89 pgs: 89 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 47 MiB/s wr, 753 op/s 2023-09-23T19:42:56.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:55 smithi092 bash[58855]: audit 2023-09-23T19:42:54.752743+0000 mon.a (mon.0) 1353 : audit 1 from='client.? 172.21.15.16:0/101800245' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.4563093629283942}]': finished 2023-09-23T19:42:56.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:55 smithi092 bash[58855]: cluster 2023-09-23T19:42:54.754081+0000 osd.4 (osd.4) 127 : cluster 3 failed to encode map e188 with expected crc 2023-09-23T19:42:56.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:55 smithi092 bash[58855]: cluster 2023-09-23T19:42:54.757435+0000 mon.a (mon.0) 1354 : cluster 0 osdmap e188: 8 total, 8 up, 8 in 2023-09-23T19:42:56.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:55 smithi092 bash[58855]: cluster 2023-09-23T19:42:54.767463+0000 osd.7 (osd.7) 97 : cluster 3 failed to encode map e188 with expected crc 2023-09-23T19:42:56.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:55 smithi092 bash[58855]: cluster 2023-09-23T19:42:54.768360+0000 osd.5 (osd.5) 117 : cluster 3 failed to encode map e188 with expected crc 2023-09-23T19:42:56.067 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:56.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:56.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:56.174 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:56.200 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:56.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:56.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:56.229 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 72 16 92524 92508 80.2922 28.5625 0.00426378 0.0124415 2023-09-23T19:42:56.304 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:56.326 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:56.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:56.329 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:42:56.430 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:56.469 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:56.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:56.473 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:42:56.574 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:56.611 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:56.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:56.616 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:56.717 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:56.751 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:56.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:56.754 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:56.855 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:56.894 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:56.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:56.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:42:57.001 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:57.027 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:57.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:57.030 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:57.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:56 smithi092 bash[58855]: cluster 2023-09-23T19:42:55.752503+0000 mon.a (mon.0) 1355 : cluster 0 osdmap e189: 8 total, 8 up, 8 in 2023-09-23T19:42:57.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:56 smithi092 bash[58855]: cluster 2023-09-23T19:42:55.755102+0000 osd.5 (osd.5) 118 : cluster 3 failed to encode map e189 with expected crc 2023-09-23T19:42:57.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:56 smithi092 bash[58855]: cluster 2023-09-23T19:42:55.758679+0000 osd.4 (osd.4) 128 : cluster 3 failed to encode map e189 with expected crc 2023-09-23T19:42:57.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:56 smithi092 bash[58855]: cluster 2023-09-23T19:42:55.758736+0000 osd.7 (osd.7) 98 : cluster 3 failed to encode map e189 with expected crc 2023-09-23T19:42:57.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:56 smithi092 bash[58855]: cluster 2023-09-23T19:42:55.758743+0000 osd.5 (osd.5) 119 : cluster 3 failed to encode map e189 with expected crc 2023-09-23T19:42:57.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:56 smithi092 bash[58855]: cluster 2023-09-23T19:42:55.765617+0000 osd.4 (osd.4) 129 : cluster 3 failed to encode map e189 with expected crc 2023-09-23T19:42:57.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:56 smithi092 bash[58855]: cluster 2023-09-23T19:42:55.766064+0000 osd.7 (osd.7) 99 : cluster 3 failed to encode map e189 with expected crc 2023-09-23T19:42:57.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:56 smithi092 bash[58855]: cluster 2023-09-23T19:42:55.802174+0000 osd.6 (osd.6) 109 : cluster 3 failed to encode map e189 with expected crc 2023-09-23T19:42:57.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:56 smithi092 bash[58855]: cluster 2023-09-23T19:42:55.805260+0000 osd.6 (osd.6) 110 : cluster 3 failed to encode map e189 with expected crc 2023-09-23T19:42:57.130 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:57.156 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:57.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:57.158 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:42:57.229 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 73 16 93370 93354 79.9165 52.875 0.0150234 0.0125074 2023-09-23T19:42:57.259 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:57.268 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:56 smithi016 bash[75004]: cluster 2023-09-23T19:42:55.752503+0000 mon.a (mon.0) 1355 : cluster 0 osdmap e189: 8 total, 8 up, 8 in 2023-09-23T19:42:57.268 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:56 smithi016 bash[75004]: cluster 2023-09-23T19:42:55.755102+0000 osd.5 (osd.5) 118 : cluster 3 failed to encode map e189 with expected crc 2023-09-23T19:42:57.269 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:56 smithi016 bash[75004]: cluster 2023-09-23T19:42:55.758679+0000 osd.4 (osd.4) 128 : cluster 3 failed to encode map e189 with expected crc 2023-09-23T19:42:57.269 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:56 smithi016 bash[75004]: cluster 2023-09-23T19:42:55.758736+0000 osd.7 (osd.7) 98 : cluster 3 failed to encode map e189 with expected crc 2023-09-23T19:42:57.269 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:56 smithi016 bash[75004]: cluster 2023-09-23T19:42:55.758743+0000 osd.5 (osd.5) 119 : cluster 3 failed to encode map e189 with expected crc 2023-09-23T19:42:57.269 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:56 smithi016 bash[75004]: cluster 2023-09-23T19:42:55.765617+0000 osd.4 (osd.4) 129 : cluster 3 failed to encode map e189 with expected crc 2023-09-23T19:42:57.269 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:56 smithi016 bash[75004]: cluster 2023-09-23T19:42:55.766064+0000 osd.7 (osd.7) 99 : cluster 3 failed to encode map e189 with expected crc 2023-09-23T19:42:57.269 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:56 smithi016 bash[75004]: cluster 2023-09-23T19:42:55.802174+0000 osd.6 (osd.6) 109 : cluster 3 failed to encode map e189 with expected crc 2023-09-23T19:42:57.269 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:56 smithi016 bash[75004]: cluster 2023-09-23T19:42:55.805260+0000 osd.6 (osd.6) 110 : cluster 3 failed to encode map e189 with expected crc 2023-09-23T19:42:57.269 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:56 smithi016 bash[77995]: cluster 2023-09-23T19:42:55.752503+0000 mon.a (mon.0) 1355 : cluster 0 osdmap e189: 8 total, 8 up, 8 in 2023-09-23T19:42:57.270 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:56 smithi016 bash[77995]: cluster 2023-09-23T19:42:55.755102+0000 osd.5 (osd.5) 118 : cluster 3 failed to encode map e189 with expected crc 2023-09-23T19:42:57.270 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:56 smithi016 bash[77995]: cluster 2023-09-23T19:42:55.758679+0000 osd.4 (osd.4) 128 : cluster 3 failed to encode map e189 with expected crc 2023-09-23T19:42:57.270 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:56 smithi016 bash[77995]: cluster 2023-09-23T19:42:55.758736+0000 osd.7 (osd.7) 98 : cluster 3 failed to encode map e189 with expected crc 2023-09-23T19:42:57.270 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:56 smithi016 bash[77995]: cluster 2023-09-23T19:42:55.758743+0000 osd.5 (osd.5) 119 : cluster 3 failed to encode map e189 with expected crc 2023-09-23T19:42:57.270 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:56 smithi016 bash[77995]: cluster 2023-09-23T19:42:55.765617+0000 osd.4 (osd.4) 129 : cluster 3 failed to encode map e189 with expected crc 2023-09-23T19:42:57.270 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:56 smithi016 bash[77995]: cluster 2023-09-23T19:42:55.766064+0000 osd.7 (osd.7) 99 : cluster 3 failed to encode map e189 with expected crc 2023-09-23T19:42:57.270 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:56 smithi016 bash[77995]: cluster 2023-09-23T19:42:55.802174+0000 osd.6 (osd.6) 109 : cluster 3 failed to encode map e189 with expected crc 2023-09-23T19:42:57.270 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:56 smithi016 bash[77995]: cluster 2023-09-23T19:42:55.805260+0000 osd.6 (osd.6) 110 : cluster 3 failed to encode map e189 with expected crc 2023-09-23T19:42:57.293 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:57.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:57.300 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:57.401 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:57.434 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:57.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:57.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:57.540 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:57.571 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:57.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:57.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:42:57.678 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:57.707 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:42:57.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:57.709 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:42:57.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:57 smithi016 bash[77995]: cluster 2023-09-23T19:42:56.229342+0000 mgr.y (mgr.25325) 1065 : cluster 0 pgmap v1023: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-09-23T19:42:57.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:57 smithi016 bash[75004]: cluster 2023-09-23T19:42:56.229342+0000 mgr.y (mgr.25325) 1065 : cluster 0 pgmap v1023: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-09-23T19:42:57.810 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:57.819 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:57 smithi092 bash[58855]: cluster 2023-09-23T19:42:56.229342+0000 mgr.y (mgr.25325) 1065 : cluster 0 pgmap v1023: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-09-23T19:42:57.847 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:57.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:57.850 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:57.854 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:42:57.902 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:42:57.948 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:42:57.951 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:57.977 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:57.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:57.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:42:57.985 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:42:58.015 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:42:58.064 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:42:58.081 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:58.102 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:42:58.111 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:58.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:58.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:42:58.144 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:42:58.152 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:42:58.152 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:42:58.152 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:42:58.152 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:42:58.152 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:42:58.152 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:42:58.152 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:42:58.152 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:42:58.216 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:58.229 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 74 16 94209 94193 79.5452 52.4375 0.00729342 0.0125657 2023-09-23T19:42:58.246 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:42:58.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:58.250 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:42:58.350 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:58.384 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:42:58.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:58.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:42:58.489 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:58.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:58.528 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:58.528 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:58.629 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:58.656 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:42:58.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:58.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:42:58.760 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:58.781 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:58.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:58.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:42:58.800 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:58 smithi016 bash[77995]: audit 2023-09-23T19:42:57.807702+0000 mon.c (mon.2) 184 : audit 0 from='client.? 172.21.15.16:0/3892272088' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:42:58.800 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:58 smithi016 bash[77995]: audit 2023-09-23T19:42:58.661666+0000 mon.a (mon.0) 1356 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:42:58.800 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:58 smithi016 bash[75004]: audit 2023-09-23T19:42:57.807702+0000 mon.c (mon.2) 184 : audit 0 from='client.? 172.21.15.16:0/3892272088' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:42:58.800 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:58 smithi016 bash[75004]: audit 2023-09-23T19:42:58.661666+0000 mon.a (mon.0) 1356 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:42:58.884 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:58.914 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:58.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:58.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:59.019 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:59.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:58 smithi092 bash[58855]: audit 2023-09-23T19:42:57.807702+0000 mon.c (mon.2) 184 : audit 0 from='client.? 172.21.15.16:0/3892272088' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:42:59.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:58 smithi092 bash[58855]: audit 2023-09-23T19:42:58.661666+0000 mon.a (mon.0) 1356 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:42:59.056 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:59.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:59.060 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:59.161 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:59.190 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:59.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:59.197 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:59.229 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 75 16 95006 94990 79.1487 49.8125 0.0349631 0.0126279 2023-09-23T19:42:59.298 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:59.334 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:42:59.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:59.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:42:59.437 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:59.490 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:42:59.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:59.493 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:42:59.594 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:59.621 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:59.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:59.626 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:42:59.727 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:59.762 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:42:59.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:59.768 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:42:59.778 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:42:59 smithi016 bash[75004]: cluster 2023-09-23T19:42:58.229897+0000 mgr.y (mgr.25325) 1066 : cluster 0 pgmap v1024: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 45 MiB/s wr, 714 op/s 2023-09-23T19:42:59.870 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:59.878 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:42:59 smithi092 bash[58855]: cluster 2023-09-23T19:42:58.229897+0000 mgr.y (mgr.25325) 1066 : cluster 0 pgmap v1024: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 45 MiB/s wr, 714 op/s 2023-09-23T19:42:59.893 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:42:59.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:59.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:00.001 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:00.035 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:00.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:42:59 smithi016 bash[77995]: cluster 2023-09-23T19:42:58.229897+0000 mgr.y (mgr.25325) 1066 : cluster 0 pgmap v1024: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 45 MiB/s wr, 714 op/s 2023-09-23T19:43:00.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:00.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:43:00.140 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:00.171 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:00.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:00.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:43:00.229 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 76 16 95829 95813 78.784 51.4375 0.0177319 0.012687 2023-09-23T19:43:00.278 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:00.312 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:00.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:00.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:43:00.416 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:00.441 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:00.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:00.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:00.548 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:00.580 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:00.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:00.586 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:43:00.687 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:00.732 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:00.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:00.737 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:43:00.743 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:43:00.839 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:00.869 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:00.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:00.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:00.973 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:01.020 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:01.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:01.024 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:43:01.125 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:01.158 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:01.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:01.161 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:43:01.229 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 77 16 96653 96637 78.4296 51.5 0.0103703 0.0127452 2023-09-23T19:43:01.263 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:01.294 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:01.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:01.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:43:01.398 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:01.427 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:01.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:01.435 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:01.536 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:01.563 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:01.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:01.569 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:01.669 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:01.695 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:01.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:01.701 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:43:01.795 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is unset 2023-09-23T19:43:01.801 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:01.836 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:01.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:01.840 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:01.940 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:01.947 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:01 smithi092 bash[58855]: cluster 2023-09-23T19:43:00.230578+0000 mgr.y (mgr.25325) 1067 : cluster 0 pgmap v1025: 89 pgs: 89 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 52 MiB/s wr, 824 op/s 2023-09-23T19:43:01.948 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:01 smithi092 bash[58855]: audit 2023-09-23T19:43:01.091023+0000 mon.a (mon.0) 1357 : audit 1 from='client.? 172.21.15.16:0/4222172019' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:43:01.975 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:01.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:01.981 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:02.081 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:02.092 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:01 smithi016 bash[75004]: cluster 2023-09-23T19:43:00.230578+0000 mgr.y (mgr.25325) 1067 : cluster 0 pgmap v1025: 89 pgs: 89 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 52 MiB/s wr, 824 op/s 2023-09-23T19:43:02.092 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:01 smithi016 bash[75004]: audit 2023-09-23T19:43:01.091023+0000 mon.a (mon.0) 1357 : audit 1 from='client.? 172.21.15.16:0/4222172019' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:43:02.092 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:01 smithi016 bash[77995]: cluster 2023-09-23T19:43:00.230578+0000 mgr.y (mgr.25325) 1067 : cluster 0 pgmap v1025: 89 pgs: 89 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 52 MiB/s wr, 824 op/s 2023-09-23T19:43:02.092 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:01 smithi016 bash[77995]: audit 2023-09-23T19:43:01.091023+0000 mon.a (mon.0) 1357 : audit 1 from='client.? 172.21.15.16:0/4222172019' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:43:02.115 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:02.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:02.120 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:43:02.220 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:02.229 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 78 16 97488 97472 78.0931 52.1875 0.0146414 0.0127991 2023-09-23T19:43:02.248 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:02.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:02.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:43:02.349 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:43:02.352 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:02.381 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:02.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:02.385 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:02.485 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:02.513 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:02.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:02.518 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:43:02.580 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-09-23T19:43:02.580 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T19:43:02.580 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2023-09-23T19:43:02.581 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T19:43:02.581 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T19:43:02.581 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 config set filestore_inject_stall 3 2023-09-23T19:43:02.618 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:02.660 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:02.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:02.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:02.767 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:02.797 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:02.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:02.806 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:43:02.806 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:02 smithi016 bash[75004]: cluster 2023-09-23T19:43:01.777556+0000 mon.a (mon.0) 1358 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:43:02.806 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:02 smithi016 bash[75004]: audit 2023-09-23T19:43:01.785153+0000 mon.a (mon.0) 1359 : audit 1 from='client.? 172.21.15.16:0/4222172019' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:43:02.806 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:02 smithi016 bash[75004]: cluster 2023-09-23T19:43:01.785386+0000 mon.a (mon.0) 1360 : cluster 0 osdmap e190: 8 total, 8 up, 8 in 2023-09-23T19:43:02.806 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:02 smithi016 bash[75004]: cluster 2023-09-23T19:43:01.802484+0000 osd.6 (osd.6) 111 : cluster 3 failed to encode map e190 with expected crc 2023-09-23T19:43:02.806 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:02 smithi016 bash[75004]: cluster 2023-09-23T19:43:01.808061+0000 osd.7 (osd.7) 100 : cluster 3 failed to encode map e190 with expected crc 2023-09-23T19:43:02.806 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:02 smithi016 bash[75004]: cluster 2023-09-23T19:43:01.809071+0000 osd.4 (osd.4) 130 : cluster 3 failed to encode map e190 with expected crc 2023-09-23T19:43:02.807 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:02 smithi016 bash[77995]: cluster 2023-09-23T19:43:01.777556+0000 mon.a (mon.0) 1358 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:43:02.807 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:02 smithi016 bash[77995]: audit 2023-09-23T19:43:01.785153+0000 mon.a (mon.0) 1359 : audit 1 from='client.? 172.21.15.16:0/4222172019' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:43:02.807 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:02 smithi016 bash[77995]: cluster 2023-09-23T19:43:01.785386+0000 mon.a (mon.0) 1360 : cluster 0 osdmap e190: 8 total, 8 up, 8 in 2023-09-23T19:43:02.807 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:02 smithi016 bash[77995]: cluster 2023-09-23T19:43:01.802484+0000 osd.6 (osd.6) 111 : cluster 3 failed to encode map e190 with expected crc 2023-09-23T19:43:02.807 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:02 smithi016 bash[77995]: cluster 2023-09-23T19:43:01.808061+0000 osd.7 (osd.7) 100 : cluster 3 failed to encode map e190 with expected crc 2023-09-23T19:43:02.807 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:02 smithi016 bash[77995]: cluster 2023-09-23T19:43:01.809071+0000 osd.4 (osd.4) 130 : cluster 3 failed to encode map e190 with expected crc 2023-09-23T19:43:02.907 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:02.934 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:02.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:02.938 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:03.039 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:03.071 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:03.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:03.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:43:03.177 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:03.189 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:02 smithi092 bash[58855]: cluster 2023-09-23T19:43:01.777556+0000 mon.a (mon.0) 1358 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:43:03.189 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:02 smithi092 bash[58855]: audit 2023-09-23T19:43:01.785153+0000 mon.a (mon.0) 1359 : audit 1 from='client.? 172.21.15.16:0/4222172019' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:43:03.190 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:02 smithi092 bash[58855]: cluster 2023-09-23T19:43:01.785386+0000 mon.a (mon.0) 1360 : cluster 0 osdmap e190: 8 total, 8 up, 8 in 2023-09-23T19:43:03.190 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:02 smithi092 bash[58855]: cluster 2023-09-23T19:43:01.802484+0000 osd.6 (osd.6) 111 : cluster 3 failed to encode map e190 with expected crc 2023-09-23T19:43:03.190 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:02 smithi092 bash[58855]: cluster 2023-09-23T19:43:01.808061+0000 osd.7 (osd.7) 100 : cluster 3 failed to encode map e190 with expected crc 2023-09-23T19:43:03.190 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:02 smithi092 bash[58855]: cluster 2023-09-23T19:43:01.809071+0000 osd.4 (osd.4) 130 : cluster 3 failed to encode map e190 with expected crc 2023-09-23T19:43:03.205 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:03.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:03.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:43:03.229 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 79 16 98309 98293 77.7541 51.3125 0.0128449 0.0128557 2023-09-23T19:43:03.309 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:03.344 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:03.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:03.348 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:03.449 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:03.478 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:03.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:03.481 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:43:03.582 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:03.622 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:03.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:03.628 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:03.729 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:03.755 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:03.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:03.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:03.860 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:03.868 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:03 smithi092 bash[58855]: cluster 2023-09-23T19:43:02.231097+0000 mgr.y (mgr.25325) 1068 : cluster 0 pgmap v1027: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 44 MiB/s wr, 698 op/s 2023-09-23T19:43:03.894 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:03.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:03.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:03.999 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:04.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:03 smithi016 bash[77995]: cluster 2023-09-23T19:43:02.231097+0000 mgr.y (mgr.25325) 1068 : cluster 0 pgmap v1027: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 44 MiB/s wr, 698 op/s 2023-09-23T19:43:04.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:03 smithi016 bash[75004]: cluster 2023-09-23T19:43:02.231097+0000 mgr.y (mgr.25325) 1068 : cluster 0 pgmap v1027: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 44 MiB/s wr, 698 op/s 2023-09-23T19:43:04.050 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:04.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:04.056 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:04.157 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:04.189 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:04.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:04.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:43:04.229 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-09-23T19:43:04.226653+0000 min lat: 0.00130197 max lat: 0.981669 avg lat: 0.0129034 2023-09-23T19:43:04.230 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:43:04.230 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 80 16 99186 99170 77.4672 54.8125 0.0258141 0.0129034 2023-09-23T19:43:04.295 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:04.337 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:04.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:04.339 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:43:04.440 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:04.487 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:04.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:04.490 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:43:04.591 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:04.622 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:04.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:04.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:43:04.726 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:04.761 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:04.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:04.764 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:04.865 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:04.893 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:04.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:04.896 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:43:04.997 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:05.030 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:05.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:05.037 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:05.138 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:05.168 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:05.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:05.171 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:05.231 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 81 16 100021 100005 77.1538 52.1875 0.0130183 0.0129546 2023-09-23T19:43:05.271 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:05.296 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:05.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:05.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:43:05.400 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:05.428 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:05.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:05.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:43:05.536 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:05.560 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:05.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:05.563 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:43:05.592 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T19:43:05.664 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:05.692 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:05.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:05.694 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:05.795 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:05.805 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:05 smithi016 bash[75004]: cluster 2023-09-23T19:43:04.231904+0000 mgr.y (mgr.25325) 1069 : cluster 0 pgmap v1028: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 33 MiB/s wr, 525 op/s 2023-09-23T19:43:05.806 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:43:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:43:05] "GET /metrics HTTP/1.1" 200 36305 "" "Prometheus/2.33.4" 2023-09-23T19:43:05.819 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:05.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:05.827 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:05.927 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:05.936 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:05 smithi092 bash[58855]: cluster 2023-09-23T19:43:04.231904+0000 mgr.y (mgr.25325) 1069 : cluster 0 pgmap v1028: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 33 MiB/s wr, 525 op/s 2023-09-23T19:43:05.947 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-09-23T19:43:05.948 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [ 2023-09-23T19:43:05.948 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:05.948 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.36243.0:100578 11.b e190/180 11:dc7336c5:::benchmark_data_smithi016_160148_object100577:head v 190'6294, mlcod=190'6292)", 2023-09-23T19:43:05.948 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:43:05.937839+0000", 2023-09-23T19:43:05.948 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 0.0048190230000000004, 2023-09-23T19:43:05.948 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0048495760000000004, 2023-09-23T19:43:05.949 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:43:05.949 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "started", 2023-09-23T19:43:05.949 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:43:05.949 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:05.949 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:43:05.949 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:05.937839+0000", 2023-09-23T19:43:05.949 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:43:05.950 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:05.950 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:05.950 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:43:05.950 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:05.937839+0000", 2023-09-23T19:43:05.950 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:43:05.950 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:05.951 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:05.951 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:43:05.951 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:05.937837+0000", 2023-09-23T19:43:05.951 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-09-23T19:43:05.951 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:05.951 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:05.951 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:43:05.952 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:05.937869+0000", 2023-09-23T19:43:05.952 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.1155999999999998e-05 2023-09-23T19:43:05.952 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:05.952 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:05.952 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:43:05.952 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:05.937881+0000", 2023-09-23T19:43:05.953 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.219e-05 2023-09-23T19:43:05.953 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:05.953 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:05.953 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:43:05.953 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:05.937889+0000", 2023-09-23T19:43:05.953 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.1310000000000006e-06 2023-09-23T19:43:05.953 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:05.954 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:05.954 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:43:05.954 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:05.937969+0000", 2023-09-23T19:43:05.954 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.0153000000000003e-05 2023-09-23T19:43:05.954 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:05.954 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:05.954 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:43:05.955 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:05.937992+0000", 2023-09-23T19:43:05.955 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.3053000000000001e-05 2023-09-23T19:43:05.955 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:43:05.955 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:43:05.955 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:43:05.955 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:43:05.955 INFO:teuthology.orchestra.run.smithi016.stdout: ], 2023-09-23T19:43:05.956 INFO:teuthology.orchestra.run.smithi016.stdout: "num_ops": 1 2023-09-23T19:43:05.956 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-09-23T19:43:05.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:05.957 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:05.957 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:43:06.058 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:06.068 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:05 smithi016 bash[77995]: cluster 2023-09-23T19:43:04.231904+0000 mgr.y (mgr.25325) 1069 : cluster 0 pgmap v1028: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 33 MiB/s wr, 525 op/s 2023-09-23T19:43:06.086 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:06.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:06.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:43:06.189 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:06.219 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:06.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:06.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:06.231 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 82 16 100824 100808 76.8249 50.1875 0.040136 0.0130105 2023-09-23T19:43:06.326 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:06.359 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:06.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:06.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:06.462 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:06.500 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:06.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:06.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:43:06.605 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:06.632 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:06.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:06.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:43:06.737 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:06.766 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:06.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:06.772 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:06.873 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:06.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:06.914 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:06.915 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:07.015 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T19:43:07.016 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:07.056 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:07.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:07.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:07.165 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:07.190 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:07.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:07.195 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:07.231 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 83 16 101661 101645 76.5296 52.3125 0.0116828 0.0130607 2023-09-23T19:43:07.297 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:07.328 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:07.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:07.332 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:43:07.420 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:43:07.432 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:07.476 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:07.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:07.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:07.494 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:43:07.583 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:07.616 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:07.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:07.622 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:07.723 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:07.756 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:07.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:07.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:43:07.859 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:07.889 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:07.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:07.892 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:07.992 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:08.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:07 smithi016 bash[77995]: cluster 2023-09-23T19:43:06.232740+0000 mgr.y (mgr.25325) 1070 : cluster 0 pgmap v1029: 89 pgs: 89 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 49 MiB/s wr, 785 op/s 2023-09-23T19:43:08.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:07 smithi016 bash[75004]: cluster 2023-09-23T19:43:06.232740+0000 mgr.y (mgr.25325) 1070 : cluster 0 pgmap v1029: 89 pgs: 89 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 49 MiB/s wr, 785 op/s 2023-09-23T19:43:08.038 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:08.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:08.040 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:08.140 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:08.148 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:07 smithi092 bash[58855]: cluster 2023-09-23T19:43:06.232740+0000 mgr.y (mgr.25325) 1070 : cluster 0 pgmap v1029: 89 pgs: 89 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 49 MiB/s wr, 785 op/s 2023-09-23T19:43:08.170 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:08.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:08.175 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:08.231 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 84 16 102492 102476 76.2368 51.9375 0.00984618 0.0131103 2023-09-23T19:43:08.276 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:08.301 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:08.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:08.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:43:08.405 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:08.438 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:08.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:08.444 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:43:08.544 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:08.579 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:08.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:08.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:43:08.689 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:08.718 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:08.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:08.723 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:43:08.823 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:08.860 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:08.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:08.865 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:08.966 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:08.997 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:09.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:09.003 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:43:09.104 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:09.134 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:09.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:09.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:09.231 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 85 16 103333 103317 75.9582 52.5625 0.0155175 0.0131597 2023-09-23T19:43:09.242 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:09.262 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:09.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:09.264 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:09.365 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:09.399 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:09.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:09.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:09.505 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:09.535 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:09.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:09.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:09.640 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:09.678 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:09.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:09.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:09.786 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:09.831 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:09.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:09.836 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:09.937 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:09.962 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:09.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:09.966 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:10.067 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:10.074 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:09 smithi092 bash[58855]: cluster 2023-09-23T19:43:08.233287+0000 mgr.y (mgr.25325) 1071 : cluster 0 pgmap v1030: 89 pgs: 89 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 50 MiB/s wr, 801 op/s 2023-09-23T19:43:10.106 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:10.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:10.111 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:10.212 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:10.231 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 86 16 104120 104104 75.6469 49.1875 0.0135276 0.0132138 2023-09-23T19:43:10.235 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:10.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:10.241 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:43:10.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:09 smithi016 bash[77995]: cluster 2023-09-23T19:43:08.233287+0000 mgr.y (mgr.25325) 1071 : cluster 0 pgmap v1030: 89 pgs: 89 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 50 MiB/s wr, 801 op/s 2023-09-23T19:43:10.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:09 smithi016 bash[75004]: cluster 2023-09-23T19:43:08.233287+0000 mgr.y (mgr.25325) 1071 : cluster 0 pgmap v1030: 89 pgs: 89 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 50 MiB/s wr, 801 op/s 2023-09-23T19:43:10.341 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:10.374 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:10.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:10.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:10.478 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:10.505 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:10.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:10.509 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:10.565 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:43:10.610 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:10.657 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:10.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:10.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:43:10.760 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:10.795 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:10.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:10.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:10.898 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:10.928 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:10.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:10.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:11.032 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:11.055 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:11.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:11.057 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:11.158 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:11.187 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:11.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:11.195 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:11.231 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 87 16 104930 104914 75.3592 50.625 0.0183137 0.0132648 2023-09-23T19:43:11.296 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:11.328 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:11.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:11.333 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:43:11.433 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:11.465 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:11.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:11.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:43:11.569 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:11.603 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:11.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:11.606 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:11.707 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:11.746 INFO:teuthology.orchestra.run.smithi092.stdout:{ 2023-09-23T19:43:11.746 INFO:teuthology.orchestra.run.smithi092.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T19:43:11.746 INFO:teuthology.orchestra.run.smithi092.stdout:} 2023-09-23T19:43:11.750 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:11.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:11.754 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:43:11.854 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:11.905 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:11.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:11.909 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:43:12.010 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:12.035 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:12.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:11 smithi016 bash[77995]: cluster 2023-09-23T19:43:10.234008+0000 mgr.y (mgr.25325) 1072 : cluster 0 pgmap v1031: 89 pgs: 89 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 52 MiB/s wr, 831 op/s 2023-09-23T19:43:12.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:11 smithi016 bash[75004]: cluster 2023-09-23T19:43:10.234008+0000 mgr.y (mgr.25325) 1072 : cluster 0 pgmap v1031: 89 pgs: 89 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 52 MiB/s wr, 831 op/s 2023-09-23T19:43:12.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:12.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:12.139 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:12.170 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:12.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:12.174 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:12.231 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 88 16 105639 105623 75.0064 44.3125 0.0261207 0.0133261 2023-09-23T19:43:12.275 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:12.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:11 smithi092 bash[58855]: cluster 2023-09-23T19:43:10.234008+0000 mgr.y (mgr.25325) 1072 : cluster 0 pgmap v1031: 89 pgs: 89 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 52 MiB/s wr, 831 op/s 2023-09-23T19:43:12.299 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:12.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:12.303 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:43:12.403 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:12.427 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:12.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:12.429 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:43:12.530 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:12.557 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:12.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:12.559 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:12.599 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:43:12.659 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:12.700 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:12.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:12.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:12.812 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:12.818 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:43:12.837 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:12.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:12.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:12.881 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:43:12.927 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:43:12.940 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:12.972 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:12.976 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:43:12.976 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:12 smithi016 bash[77995]: audit 2023-09-23T19:43:12.778033+0000 mon.c (mon.2) 185 : audit 0 from='client.? 172.21.15.16:0/3106144907' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:43:12.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:12.977 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:43:12.977 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:12 smithi016 bash[75004]: audit 2023-09-23T19:43:12.778033+0000 mon.c (mon.2) 185 : audit 0 from='client.? 172.21.15.16:0/3106144907' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:43:13.019 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:43:13.067 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:43:13.078 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:13.088 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:12 smithi092 bash[58855]: audit 2023-09-23T19:43:12.778033+0000 mon.c (mon.2) 185 : audit 0 from='client.? 172.21.15.16:0/3106144907' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:43:13.103 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:13.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:13.111 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:13.112 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:43:13.148 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:43:13.155 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:43:13.156 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:43:13.156 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:43:13.156 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:43:13.156 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:43:13.156 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:43:13.156 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:43:13.156 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:43:13.212 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:13.231 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 89 16 106366 106350 74.6741 45.4375 0.0149267 0.0133855 2023-09-23T19:43:13.246 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:13.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:13.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:13.353 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:13.378 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:13.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:13.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:13.482 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:13.519 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:13.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:13.529 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:13.630 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:13.666 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:13.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:13.671 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:43:13.772 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:13.800 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:13.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:13.804 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:43:13.905 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:13.914 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:13 smithi016 bash[75004]: cluster 2023-09-23T19:43:12.234732+0000 mgr.y (mgr.25325) 1073 : cluster 0 pgmap v1032: 89 pgs: 89 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 54 MiB/s wr, 871 op/s 2023-09-23T19:43:13.915 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:13 smithi016 bash[75004]: audit 2023-09-23T19:43:13.661561+0000 mon.a (mon.0) 1361 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:43:13.915 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:13 smithi016 bash[77995]: cluster 2023-09-23T19:43:12.234732+0000 mgr.y (mgr.25325) 1073 : cluster 0 pgmap v1032: 89 pgs: 89 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 54 MiB/s wr, 871 op/s 2023-09-23T19:43:13.915 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:13 smithi016 bash[77995]: audit 2023-09-23T19:43:13.661561+0000 mon.a (mon.0) 1361 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:43:13.947 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:13.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:13.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:43:14.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:13 smithi092 bash[58855]: cluster 2023-09-23T19:43:12.234732+0000 mgr.y (mgr.25325) 1073 : cluster 0 pgmap v1032: 89 pgs: 89 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 54 MiB/s wr, 871 op/s 2023-09-23T19:43:14.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:13 smithi092 bash[58855]: audit 2023-09-23T19:43:13.661561+0000 mon.a (mon.0) 1361 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:43:14.050 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:14.092 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:14.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:14.096 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:14.197 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:14.222 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:14.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:14.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:43:14.232 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Total time run: 90.0107 2023-09-23T19:43:14.232 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Total writes made: 107200 2023-09-23T19:43:14.232 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Write size: 65536 2023-09-23T19:43:14.232 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Object size: 65536 2023-09-23T19:43:14.232 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Bandwidth (MB/sec): 74.4356 2023-09-23T19:43:14.232 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Stddev Bandwidth: 29.2196 2023-09-23T19:43:14.232 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Max bandwidth (MB/sec): 164.875 2023-09-23T19:43:14.232 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Min bandwidth (MB/sec): 28.5625 2023-09-23T19:43:14.233 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Average IOPS: 1190 2023-09-23T19:43:14.233 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Stddev IOPS: 467.514 2023-09-23T19:43:14.233 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Max IOPS: 2638 2023-09-23T19:43:14.233 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Min IOPS: 457 2023-09-23T19:43:14.233 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Average Latency(s): 0.0134297 2023-09-23T19:43:14.233 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Stddev Latency(s): 0.014589 2023-09-23T19:43:14.233 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Max latency(s): 0.981669 2023-09-23T19:43:14.233 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Min latency(s): 0.00130197 2023-09-23T19:43:14.237 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T19:43:14.326 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:14.351 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:14.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:14.353 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:14.454 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:14.486 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:14.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:14.492 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:14.593 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:14.616 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:14.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:14.618 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:43:14.719 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:14.752 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:14.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:14.756 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:43:14.856 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:14.887 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:14.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:14.891 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:43:14.992 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:15.023 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:15.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:15.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:15.131 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:15.166 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:15.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:15.172 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:43:15.272 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:15.298 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:15.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:15.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:43:15.403 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:15.443 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:15.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:15.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:43:15.547 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:15.574 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:15.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:15.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:15.681 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:15.708 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:15.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:15.717 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:43:15.818 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:15.851 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:15.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:15.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:15.870 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:43:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:43:15] "GET /metrics HTTP/1.1" 200 36306 "" "Prometheus/2.33.4" 2023-09-23T19:43:15.955 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:15.983 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:15.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:15.988 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:16.089 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:16.109 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:16.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:16.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:43:16.212 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:16.222 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:15 smithi092 bash[58855]: cluster 2023-09-23T19:43:14.235209+0000 mgr.y (mgr.25325) 1074 : cluster 0 pgmap v1033: 89 pgs: 89 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 43 MiB/s wr, 693 op/s 2023-09-23T19:43:16.241 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:16.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:16.247 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:16.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:15 smithi016 bash[75004]: cluster 2023-09-23T19:43:14.235209+0000 mgr.y (mgr.25325) 1074 : cluster 0 pgmap v1033: 89 pgs: 89 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 43 MiB/s wr, 693 op/s 2023-09-23T19:43:16.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:15 smithi016 bash[77995]: cluster 2023-09-23T19:43:14.235209+0000 mgr.y (mgr.25325) 1074 : cluster 0 pgmap v1033: 89 pgs: 89 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 43 MiB/s wr, 693 op/s 2023-09-23T19:43:16.348 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:16.373 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:16.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:16.376 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:16.477 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:16.507 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:16.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:16.512 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:43:16.613 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:16.641 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:16.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:16.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:16.744 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:16.775 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:16.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:16.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:16.881 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:16.920 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:16.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:16.925 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:17.025 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:17.050 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:17.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:17.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:17.153 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:17.174 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:17.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:17.185 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:43:17.285 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:17.326 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:17.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:17.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:17.432 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:17.457 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:17.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:17.463 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:17.564 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:17.594 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:17.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:17.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:17.704 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:17.736 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:17.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:17.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:17.811 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-09-23T19:43:17.811 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T19:43:17.812 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T19:43:17.812 INFO:tasks.thrashosds.thrasher:Setting osd 5 primary_affinity to 0.000000 2023-09-23T19:43:17.812 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 5 0 2023-09-23T19:43:17.842 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:17.871 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:17.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:17.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:17.976 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:18.007 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:18.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:18.011 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:18.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:17 smithi016 bash[75004]: cluster 2023-09-23T19:43:16.236014+0000 mgr.y (mgr.25325) 1075 : cluster 0 pgmap v1034: 89 pgs: 89 active+clean; 6.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 58 MiB/s wr, 955 op/s 2023-09-23T19:43:18.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:17 smithi016 bash[77995]: cluster 2023-09-23T19:43:16.236014+0000 mgr.y (mgr.25325) 1075 : cluster 0 pgmap v1034: 89 pgs: 89 active+clean; 6.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 58 MiB/s wr, 955 op/s 2023-09-23T19:43:18.112 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:18.119 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:17 smithi092 bash[58855]: cluster 2023-09-23T19:43:16.236014+0000 mgr.y (mgr.25325) 1075 : cluster 0 pgmap v1034: 89 pgs: 89 active+clean; 6.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 58 MiB/s wr, 955 op/s 2023-09-23T19:43:18.138 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:18.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:18.142 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:18.243 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:18.278 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:18.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:18.281 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:18.383 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:18.410 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:18.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:18.415 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:43:18.472 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:43:18.516 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:18.539 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:18.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:18.543 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:18.643 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:18.677 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:18.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:18.681 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:18.782 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:18.812 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:18.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:18.813 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:18.914 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:18.952 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:18.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:18.958 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:19.059 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:19.108 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:19.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:19.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:19.214 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:19.241 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:19.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:19.244 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:19.345 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:19.371 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:19.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:19.376 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:19.477 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:19.505 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:19.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:19.511 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:19.611 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:19.652 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:19.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:19.658 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:43:19.758 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:19.796 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:19.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:19.800 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:19.901 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:19.909 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:19 smithi016 bash[77995]: cluster 2023-09-23T19:43:18.236527+0000 mgr.y (mgr.25325) 1076 : cluster 0 pgmap v1035: 89 pgs: 89 active+clean; 6.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 42 MiB/s wr, 737 op/s 2023-09-23T19:43:19.910 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:19 smithi016 bash[75004]: cluster 2023-09-23T19:43:18.236527+0000 mgr.y (mgr.25325) 1076 : cluster 0 pgmap v1035: 89 pgs: 89 active+clean; 6.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 42 MiB/s wr, 737 op/s 2023-09-23T19:43:19.928 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:19.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:19.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:20.035 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:20.061 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:20.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:20.064 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:43:20.165 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:20.198 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:20.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:20.201 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:43:20.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:19 smithi092 bash[58855]: cluster 2023-09-23T19:43:18.236527+0000 mgr.y (mgr.25325) 1076 : cluster 0 pgmap v1035: 89 pgs: 89 active+clean; 6.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 42 MiB/s wr, 737 op/s 2023-09-23T19:43:20.301 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:20.325 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:20.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:20.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:43:20.430 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:20.457 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:20.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:20.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:20.568 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:20.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:20.603 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:20.604 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:20.705 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:20.735 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:20.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:20.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:20.840 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:20.864 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:20.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:20.869 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:20.905 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is unset 2023-09-23T19:43:20.969 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:20.977 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:20 smithi092 bash[58855]: audit 2023-09-23T19:43:20.126630+0000 mon.c (mon.2) 186 : audit 1 from='client.? 172.21.15.16:0/4267027029' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:43:20.978 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:20 smithi092 bash[58855]: audit 2023-09-23T19:43:20.127375+0000 mon.a (mon.0) 1362 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:43:20.994 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:20.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:20.999 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:43:21.099 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:21.108 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:20 smithi016 bash[75004]: audit 2023-09-23T19:43:20.126630+0000 mon.c (mon.2) 186 : audit 1 from='client.? 172.21.15.16:0/4267027029' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:43:21.108 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:20 smithi016 bash[75004]: audit 2023-09-23T19:43:20.127375+0000 mon.a (mon.0) 1362 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:43:21.108 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:20 smithi016 bash[77995]: audit 2023-09-23T19:43:20.126630+0000 mon.c (mon.2) 186 : audit 1 from='client.? 172.21.15.16:0/4267027029' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:43:21.108 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:20 smithi016 bash[77995]: audit 2023-09-23T19:43:20.127375+0000 mon.a (mon.0) 1362 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:43:21.126 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:21.126 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-09-23T19:43:21.126 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:43:21.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:21.133 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:43:21.234 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:21.262 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:21.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:21.268 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:21.369 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:21.408 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:21.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:21.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:43:21.515 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:21.551 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:21.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:21.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:43:21.656 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:21.678 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:21.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:21.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:43:21.781 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:21.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:21.822 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:21.823 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:21.924 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:21.954 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:21.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:21.956 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:22.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:21 smithi016 bash[75004]: cluster 2023-09-23T19:43:20.240700+0000 mgr.y (mgr.25325) 1077 : cluster 0 pgmap v1036: 89 pgs: 89 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 38 MiB/s wr, 897 op/s 2023-09-23T19:43:22.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:21 smithi016 bash[75004]: cluster 2023-09-23T19:43:20.849424+0000 mon.a (mon.0) 1363 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:43:22.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:21 smithi016 bash[75004]: cluster 2023-09-23T19:43:20.849456+0000 mon.a (mon.0) 1364 : cluster 1 Cluster is now healthy 2023-09-23T19:43:22.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:21 smithi016 bash[75004]: cluster 2023-09-23T19:43:20.874233+0000 osd.6 (osd.6) 112 : cluster 3 failed to encode map e191 with expected crc 2023-09-23T19:43:22.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:21 smithi016 bash[75004]: audit 2023-09-23T19:43:20.879523+0000 mon.a (mon.0) 1365 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:43:22.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:21 smithi016 bash[75004]: cluster 2023-09-23T19:43:20.879711+0000 mon.a (mon.0) 1366 : cluster 0 osdmap e191: 8 total, 8 up, 8 in 2023-09-23T19:43:22.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:21 smithi016 bash[75004]: cluster 2023-09-23T19:43:20.895069+0000 osd.4 (osd.4) 131 : cluster 3 failed to encode map e191 with expected crc 2023-09-23T19:43:22.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:21 smithi016 bash[75004]: cluster 2023-09-23T19:43:20.897042+0000 osd.7 (osd.7) 101 : cluster 3 failed to encode map e191 with expected crc 2023-09-23T19:43:22.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:21 smithi016 bash[75004]: cluster 2023-09-23T19:43:20.897376+0000 osd.5 (osd.5) 120 : cluster 3 failed to encode map e191 with expected crc 2023-09-23T19:43:22.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:21 smithi016 bash[77995]: cluster 2023-09-23T19:43:20.240700+0000 mgr.y (mgr.25325) 1077 : cluster 0 pgmap v1036: 89 pgs: 89 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 38 MiB/s wr, 897 op/s 2023-09-23T19:43:22.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:21 smithi016 bash[77995]: cluster 2023-09-23T19:43:20.849424+0000 mon.a (mon.0) 1363 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:43:22.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:21 smithi016 bash[77995]: cluster 2023-09-23T19:43:20.849456+0000 mon.a (mon.0) 1364 : cluster 1 Cluster is now healthy 2023-09-23T19:43:22.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:21 smithi016 bash[77995]: cluster 2023-09-23T19:43:20.874233+0000 osd.6 (osd.6) 112 : cluster 3 failed to encode map e191 with expected crc 2023-09-23T19:43:22.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:21 smithi016 bash[77995]: audit 2023-09-23T19:43:20.879523+0000 mon.a (mon.0) 1365 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:43:22.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:21 smithi016 bash[77995]: cluster 2023-09-23T19:43:20.879711+0000 mon.a (mon.0) 1366 : cluster 0 osdmap e191: 8 total, 8 up, 8 in 2023-09-23T19:43:22.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:21 smithi016 bash[77995]: cluster 2023-09-23T19:43:20.895069+0000 osd.4 (osd.4) 131 : cluster 3 failed to encode map e191 with expected crc 2023-09-23T19:43:22.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:21 smithi016 bash[77995]: cluster 2023-09-23T19:43:20.897042+0000 osd.7 (osd.7) 101 : cluster 3 failed to encode map e191 with expected crc 2023-09-23T19:43:22.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:21 smithi016 bash[77995]: cluster 2023-09-23T19:43:20.897376+0000 osd.5 (osd.5) 120 : cluster 3 failed to encode map e191 with expected crc 2023-09-23T19:43:22.057 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:22.086 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:22.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:22.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:22.190 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:22.198 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:21 smithi092 bash[58855]: cluster 2023-09-23T19:43:20.240700+0000 mgr.y (mgr.25325) 1077 : cluster 0 pgmap v1036: 89 pgs: 89 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 38 MiB/s wr, 897 op/s 2023-09-23T19:43:22.198 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:21 smithi092 bash[58855]: cluster 2023-09-23T19:43:20.849424+0000 mon.a (mon.0) 1363 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:43:22.198 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:21 smithi092 bash[58855]: cluster 2023-09-23T19:43:20.849456+0000 mon.a (mon.0) 1364 : cluster 1 Cluster is now healthy 2023-09-23T19:43:22.198 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:21 smithi092 bash[58855]: cluster 2023-09-23T19:43:20.874233+0000 osd.6 (osd.6) 112 : cluster 3 failed to encode map e191 with expected crc 2023-09-23T19:43:22.199 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:21 smithi092 bash[58855]: audit 2023-09-23T19:43:20.879523+0000 mon.a (mon.0) 1365 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:43:22.199 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:21 smithi092 bash[58855]: cluster 2023-09-23T19:43:20.879711+0000 mon.a (mon.0) 1366 : cluster 0 osdmap e191: 8 total, 8 up, 8 in 2023-09-23T19:43:22.199 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:21 smithi092 bash[58855]: cluster 2023-09-23T19:43:20.895069+0000 osd.4 (osd.4) 131 : cluster 3 failed to encode map e191 with expected crc 2023-09-23T19:43:22.199 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:21 smithi092 bash[58855]: cluster 2023-09-23T19:43:20.897042+0000 osd.7 (osd.7) 101 : cluster 3 failed to encode map e191 with expected crc 2023-09-23T19:43:22.199 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:21 smithi092 bash[58855]: cluster 2023-09-23T19:43:20.897376+0000 osd.5 (osd.5) 120 : cluster 3 failed to encode map e191 with expected crc 2023-09-23T19:43:22.216 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:22.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:22.218 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:22.319 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:22.344 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:22.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:22.351 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:43:22.452 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:22.485 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:22.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:22.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:22.589 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:22.613 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:22.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:22.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:43:22.720 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:22.758 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:22.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:22.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:43:22.861 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:22.890 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:22.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:22.893 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:22.993 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:23.026 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:23.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:23.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:43:23.132 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:23.165 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:23.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:23.170 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:23.201 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T19:43:23.271 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:23.315 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:23.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:23.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:43:23.418 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:23.447 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:23.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:23.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:23.551 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:23.597 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:23.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:23.604 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:43:23.705 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:23.740 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:23.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:23.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:23.844 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:23.869 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:23.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:23.872 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:23.881 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:23 smithi016 bash[75004]: cluster 2023-09-23T19:43:22.241096+0000 mgr.y (mgr.25325) 1078 : cluster 0 pgmap v1038: 89 pgs: 89 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 19 MiB/s wr, 811 op/s 2023-09-23T19:43:23.882 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:23 smithi016 bash[77995]: cluster 2023-09-23T19:43:22.241096+0000 mgr.y (mgr.25325) 1078 : cluster 0 pgmap v1038: 89 pgs: 89 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 19 MiB/s wr, 811 op/s 2023-09-23T19:43:23.973 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:24.000 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:24.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:24.002 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:43:24.103 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:24.134 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:24.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:24.137 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:43:24.238 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:24.268 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:24.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:24.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:43:24.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:23 smithi092 bash[58855]: cluster 2023-09-23T19:43:22.241096+0000 mgr.y (mgr.25325) 1078 : cluster 0 pgmap v1038: 89 pgs: 89 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 19 MiB/s wr, 811 op/s 2023-09-23T19:43:24.371 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:24.413 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:24.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:24.415 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:43:24.516 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:24.548 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:24.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:24.550 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:43:24.651 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:24.689 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:24.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:24.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:24.795 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:24.822 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:24.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:24.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:24.927 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:24.954 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:24.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:24.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:43:25.057 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:25.086 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:25.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:25.088 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:25.148 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T19:43:25.189 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:25.226 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:25.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:25.231 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:25.331 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:25.371 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:25.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:25.378 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:43:25.478 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:25.515 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:25.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:25.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:25.621 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:25.647 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:25.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:25.649 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:25.652 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:43:25.656 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:43:25.750 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:25.787 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:25.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:25.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:25.887 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:25 smithi016 bash[75004]: cluster 2023-09-23T19:43:24.241587+0000 mgr.y (mgr.25325) 1079 : cluster 0 pgmap v1039: 89 pgs: 89 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 19 MiB/s wr, 811 op/s 2023-09-23T19:43:25.887 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:25 smithi016 bash[75004]: audit 2023-09-23T19:43:25.624359+0000 mon.a (mon.0) 1367 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:43:25.888 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:43:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:43:25] "GET /metrics HTTP/1.1" 200 36306 "" "Prometheus/2.33.4" 2023-09-23T19:43:25.888 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:25 smithi016 bash[77995]: cluster 2023-09-23T19:43:24.241587+0000 mgr.y (mgr.25325) 1079 : cluster 0 pgmap v1039: 89 pgs: 89 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 19 MiB/s wr, 811 op/s 2023-09-23T19:43:25.888 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:25 smithi016 bash[77995]: audit 2023-09-23T19:43:25.624359+0000 mon.a (mon.0) 1367 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:43:25.892 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:25.926 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:25.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:25.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:26.030 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:26.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:25 smithi092 bash[58855]: cluster 2023-09-23T19:43:24.241587+0000 mgr.y (mgr.25325) 1079 : cluster 0 pgmap v1039: 89 pgs: 89 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 19 MiB/s wr, 811 op/s 2023-09-23T19:43:26.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:25 smithi092 bash[58855]: audit 2023-09-23T19:43:25.624359+0000 mon.a (mon.0) 1367 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:43:26.058 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:26.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:26.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:26.160 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:26.192 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:26.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:26.198 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:26.299 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:26.327 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:26.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:26.331 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:43:26.431 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:26.472 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:26.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:26.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:26.575 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:26.603 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:26.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:26.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:26.706 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:26.741 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:26.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:26.745 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:26.846 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:26.872 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:26.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:26.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:26.977 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:27.000 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:27.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:27.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:27.103 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:27.146 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:27.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:27.148 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:27.249 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:27.275 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:27.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:27.278 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:43:27.379 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:27.418 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:27.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:27.423 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:43:27.523 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:27.553 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:27.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:27.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:43:27.660 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:27.683 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:27.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:27.686 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:43:27.697 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:43:27.787 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:27.834 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:27.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:27.840 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:27.928 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:27 smithi016 bash[75004]: cluster 2023-09-23T19:43:26.244899+0000 mgr.y (mgr.25325) 1080 : cluster 0 pgmap v1040: 89 pgs: 89 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 1.8 MiB/s wr, 829 op/s 2023-09-23T19:43:27.929 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:27 smithi016 bash[77995]: cluster 2023-09-23T19:43:26.244899+0000 mgr.y (mgr.25325) 1080 : cluster 0 pgmap v1040: 89 pgs: 89 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 1.8 MiB/s wr, 829 op/s 2023-09-23T19:43:27.941 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:27.970 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:27.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:27.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:28.076 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:28.105 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:28.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:28.108 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:28.208 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:28.236 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:28.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:28.240 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:43:28.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:27 smithi092 bash[58855]: cluster 2023-09-23T19:43:26.244899+0000 mgr.y (mgr.25325) 1080 : cluster 0 pgmap v1040: 89 pgs: 89 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 1.8 MiB/s wr, 829 op/s 2023-09-23T19:43:28.340 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:28.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:28.375 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:28.375 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:28.476 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:28.504 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:28.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:28.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:43:28.607 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:28.627 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:28.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:28.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:28.730 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:28.761 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:28.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:28.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:43:28.864 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:28.889 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:28.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:28.893 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:43:28.994 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:29.023 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:29.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:29.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:43:29.126 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:29.152 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:29.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:29.156 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:29.257 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:29.287 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:29.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:29.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:43:29.393 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:29.433 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:29.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:29.437 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:43:29.538 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:29.573 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:29.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:29.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:29.680 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:29.707 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:29.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:29.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:29.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:29 smithi016 bash[77995]: cluster 2023-09-23T19:43:28.245385+0000 mgr.y (mgr.25325) 1081 : cluster 0 pgmap v1041: 89 pgs: 89 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 827 op/s 2023-09-23T19:43:29.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:29 smithi016 bash[77995]: audit 2023-09-23T19:43:28.666127+0000 mon.a (mon.0) 1368 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:43:29.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:29 smithi016 bash[77995]: audit 2023-09-23T19:43:28.667301+0000 mon.a (mon.0) 1369 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:43:29.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:29 smithi016 bash[75004]: cluster 2023-09-23T19:43:28.245385+0000 mgr.y (mgr.25325) 1081 : cluster 0 pgmap v1041: 89 pgs: 89 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 827 op/s 2023-09-23T19:43:29.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:29 smithi016 bash[75004]: audit 2023-09-23T19:43:28.666127+0000 mon.a (mon.0) 1368 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:43:29.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:29 smithi016 bash[75004]: audit 2023-09-23T19:43:28.667301+0000 mon.a (mon.0) 1369 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:43:29.812 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:29.819 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:29 smithi092 bash[58855]: cluster 2023-09-23T19:43:28.245385+0000 mgr.y (mgr.25325) 1081 : cluster 0 pgmap v1041: 89 pgs: 89 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 827 op/s 2023-09-23T19:43:29.820 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:29 smithi092 bash[58855]: audit 2023-09-23T19:43:28.666127+0000 mon.a (mon.0) 1368 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:43:29.820 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:29 smithi092 bash[58855]: audit 2023-09-23T19:43:28.667301+0000 mon.a (mon.0) 1369 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:43:29.839 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:29.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:29.847 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:29.947 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:29.978 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:29.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:29.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:43:30.085 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:30.113 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:30.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:30.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:30.218 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:30.242 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:30.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:30.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:30.345 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:30.378 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:30.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:30.381 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:30.482 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:30.507 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:30.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:30.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:43:30.611 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:30.642 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:30.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:30.647 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:43:30.747 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:30.788 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:30.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:30.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:43:30.896 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:30.923 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:30.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:30.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:43:31.025 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:31.054 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:31.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:31.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:43:31.158 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:31.203 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:31.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:31.211 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:31.212 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:43:31.312 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:31.352 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:31.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:31.353 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:43:31.454 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:31.485 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:31.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:31.491 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:43:31.592 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:31.627 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:31.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:31.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:31.731 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:31.741 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:31 smithi016 bash[75004]: cluster 2023-09-23T19:43:30.246167+0000 mgr.y (mgr.25325) 1082 : cluster 0 pgmap v1042: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 788 op/s 2023-09-23T19:43:31.742 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:31 smithi016 bash[77995]: cluster 2023-09-23T19:43:30.246167+0000 mgr.y (mgr.25325) 1082 : cluster 0 pgmap v1042: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 788 op/s 2023-09-23T19:43:31.770 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:31.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:31.775 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:43:31.875 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:31.884 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:31 smithi092 bash[58855]: cluster 2023-09-23T19:43:30.246167+0000 mgr.y (mgr.25325) 1082 : cluster 0 pgmap v1042: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 788 op/s 2023-09-23T19:43:31.916 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:31.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:31.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:32.021 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:32.063 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:32.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:32.067 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:43:32.167 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:32.194 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:32.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:32.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:43:32.301 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:32.367 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:32.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:32.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:32.470 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:32.491 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:32.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:32.496 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:32.597 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:32.623 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:32.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:32.625 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:32.726 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:32.765 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:32.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:32.771 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:43:32.871 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:32.925 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:32.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:32.930 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:43:33.031 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:33.063 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:33.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:33.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:43:33.167 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:33.200 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:33.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:33.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:33.304 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:33.312 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:33 smithi092 bash[58855]: audit 2023-09-23T19:43:32.209282+0000 mon.a (mon.0) 1370 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:43:33.312 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:33 smithi092 bash[58855]: audit 2023-09-23T19:43:32.229049+0000 mon.a (mon.0) 1371 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:43:33.312 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:33 smithi092 bash[58855]: cluster 2023-09-23T19:43:32.246884+0000 mgr.y (mgr.25325) 1083 : cluster 0 pgmap v1043: 89 pgs: 89 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 745 op/s 2023-09-23T19:43:33.312 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:33 smithi092 bash[58855]: audit 2023-09-23T19:43:32.607016+0000 mon.a (mon.0) 1372 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:43:33.312 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:33 smithi092 bash[58855]: audit 2023-09-23T19:43:32.619894+0000 mon.a (mon.0) 1373 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:43:33.335 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:33.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:33.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:33.443 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:33.468 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:33.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:33.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:33.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:33 smithi016 bash[75004]: audit 2023-09-23T19:43:32.209282+0000 mon.a (mon.0) 1370 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:43:33.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:33 smithi016 bash[75004]: audit 2023-09-23T19:43:32.229049+0000 mon.a (mon.0) 1371 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:43:33.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:33 smithi016 bash[75004]: cluster 2023-09-23T19:43:32.246884+0000 mgr.y (mgr.25325) 1083 : cluster 0 pgmap v1043: 89 pgs: 89 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 745 op/s 2023-09-23T19:43:33.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:33 smithi016 bash[75004]: audit 2023-09-23T19:43:32.607016+0000 mon.a (mon.0) 1372 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:43:33.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:33 smithi016 bash[75004]: audit 2023-09-23T19:43:32.619894+0000 mon.a (mon.0) 1373 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:43:33.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:33 smithi016 bash[77995]: audit 2023-09-23T19:43:32.209282+0000 mon.a (mon.0) 1370 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:43:33.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:33 smithi016 bash[77995]: audit 2023-09-23T19:43:32.229049+0000 mon.a (mon.0) 1371 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:43:33.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:33 smithi016 bash[77995]: cluster 2023-09-23T19:43:32.246884+0000 mgr.y (mgr.25325) 1083 : cluster 0 pgmap v1043: 89 pgs: 89 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 745 op/s 2023-09-23T19:43:33.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:33 smithi016 bash[77995]: audit 2023-09-23T19:43:32.607016+0000 mon.a (mon.0) 1372 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:43:33.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:33 smithi016 bash[77995]: audit 2023-09-23T19:43:32.619894+0000 mon.a (mon.0) 1373 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:43:33.573 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:33.602 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:33.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:33.609 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:33.710 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:33.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:33.740 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:33.741 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:43:33.841 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:33.868 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:33.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:33.870 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:43:33.971 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:34.009 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:34.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:34.011 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:34.112 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:34.137 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:34.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:34.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:43:34.240 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:34.247 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:34 smithi092 bash[58855]: audit 2023-09-23T19:43:33.224498+0000 mon.a (mon.0) 1374 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:43:34.247 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:34 smithi092 bash[58855]: audit 2023-09-23T19:43:33.225679+0000 mon.a (mon.0) 1375 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:43:34.247 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:34 smithi092 bash[58855]: audit 2023-09-23T19:43:33.235602+0000 mon.a (mon.0) 1376 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:43:34.268 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:34.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:34.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:34.375 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:34.397 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:34.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:34.405 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:43:34.505 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:34.514 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:34 smithi016 bash[77995]: audit 2023-09-23T19:43:33.224498+0000 mon.a (mon.0) 1374 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:43:34.514 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:34 smithi016 bash[77995]: audit 2023-09-23T19:43:33.225679+0000 mon.a (mon.0) 1375 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:43:34.514 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:34 smithi016 bash[77995]: audit 2023-09-23T19:43:33.235602+0000 mon.a (mon.0) 1376 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:43:34.514 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:34 smithi016 bash[75004]: audit 2023-09-23T19:43:33.224498+0000 mon.a (mon.0) 1374 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:43:34.514 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:34 smithi016 bash[75004]: audit 2023-09-23T19:43:33.225679+0000 mon.a (mon.0) 1375 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:43:34.515 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:34 smithi016 bash[75004]: audit 2023-09-23T19:43:33.235602+0000 mon.a (mon.0) 1376 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:43:34.543 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:34.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:34.546 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:43:34.646 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:34.687 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:34.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:34.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:34.792 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:34.824 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:34.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:34.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:43:34.930 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:34.968 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:34.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:34.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:35.074 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:35.104 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:35.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:35.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:35.207 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:35.240 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:35.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:35.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:35.345 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:35.373 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:35.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:35.380 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:35.481 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:35.519 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:35.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:35.524 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:43:35.624 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:35.633 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:35 smithi092 bash[58855]: cluster 2023-09-23T19:43:34.247368+0000 mgr.y (mgr.25325) 1084 : cluster 0 pgmap v1044: 89 pgs: 89 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 644 op/s 2023-09-23T19:43:35.655 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:35.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:35.660 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:35.761 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:35.769 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:35 smithi016 bash[77995]: cluster 2023-09-23T19:43:34.247368+0000 mgr.y (mgr.25325) 1084 : cluster 0 pgmap v1044: 89 pgs: 89 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 644 op/s 2023-09-23T19:43:35.770 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:35 smithi016 bash[75004]: cluster 2023-09-23T19:43:34.247368+0000 mgr.y (mgr.25325) 1084 : cluster 0 pgmap v1044: 89 pgs: 89 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 644 op/s 2023-09-23T19:43:35.770 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:43:35 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:43:35] "GET /metrics HTTP/1.1" 200 36307 "" "Prometheus/2.33.4" 2023-09-23T19:43:35.788 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:35.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:35.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:35.895 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:35.928 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:35.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:35.931 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:36.032 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:36.061 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:36.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:36.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:36.167 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:36.193 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:36.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:36.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:36.299 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:36.321 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:36.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:36.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:43:36.424 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:36.455 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:36.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:36.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:36.565 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:36.598 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:36.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:36.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:36.702 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:36.722 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:36.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:36.724 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:43:36.825 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:36.866 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:36.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:36.868 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:43:36.969 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:37.001 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:37.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:37.004 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:43:37.105 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:37.138 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:37.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:37.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:37.245 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:37.278 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:37.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:37.283 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:37.383 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:37.404 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:37.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:37.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:37.509 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:37.541 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:37.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:37.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:37.562 INFO:teuthology.orchestra.run.smithi016.stderr:set osd.5 primary-affinity to 0 (802) 2023-09-23T19:43:37.646 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:37.654 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:37 smithi016 bash[77995]: cluster 2023-09-23T19:43:36.248756+0000 mgr.y (mgr.25325) 1085 : cluster 0 pgmap v1045: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 894 op/s 2023-09-23T19:43:37.654 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:37 smithi016 bash[77995]: audit 2023-09-23T19:43:37.432436+0000 mon.c (mon.2) 187 : audit 1 from='client.? 172.21.15.16:0/771541498' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]: dispatch 2023-09-23T19:43:37.655 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:37 smithi016 bash[77995]: audit 2023-09-23T19:43:37.432925+0000 mon.a (mon.0) 1377 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]: dispatch 2023-09-23T19:43:37.655 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:37 smithi016 bash[75004]: cluster 2023-09-23T19:43:36.248756+0000 mgr.y (mgr.25325) 1085 : cluster 0 pgmap v1045: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 894 op/s 2023-09-23T19:43:37.655 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:37 smithi016 bash[75004]: audit 2023-09-23T19:43:37.432436+0000 mon.c (mon.2) 187 : audit 1 from='client.? 172.21.15.16:0/771541498' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]: dispatch 2023-09-23T19:43:37.655 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:37 smithi016 bash[75004]: audit 2023-09-23T19:43:37.432925+0000 mon.a (mon.0) 1377 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]: dispatch 2023-09-23T19:43:37.671 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:37.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:37.677 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:43:37.777 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:37.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:37 smithi092 bash[58855]: cluster 2023-09-23T19:43:36.248756+0000 mgr.y (mgr.25325) 1085 : cluster 0 pgmap v1045: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 894 op/s 2023-09-23T19:43:37.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:37 smithi092 bash[58855]: audit 2023-09-23T19:43:37.432436+0000 mon.c (mon.2) 187 : audit 1 from='client.? 172.21.15.16:0/771541498' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]: dispatch 2023-09-23T19:43:37.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:37 smithi092 bash[58855]: audit 2023-09-23T19:43:37.432925+0000 mon.a (mon.0) 1377 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]: dispatch 2023-09-23T19:43:37.812 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:37.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:37.816 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:43:37.855 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:43:37.884 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:43:37.905 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:43:37.917 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:37.935 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:37.935 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:43:37.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:37.937 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:37.960 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:43:37.990 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:43:38.011 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:43:38.036 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:43:38.038 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:38.039 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:43:38.039 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:43:38.039 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:43:38.039 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:43:38.039 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:43:38.039 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:43:38.039 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:43:38.039 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:43:38.057 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:38.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:38.059 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:38.160 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:38.179 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:38.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:38.180 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:38.281 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:38.302 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:38.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:38.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:38.406 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:38.438 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:38.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:38.441 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:38.542 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:38.543 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-09-23T19:43:38.544 INFO:teuthology.orchestra.run.smithi016.stdout: "size": 20, 2023-09-23T19:43:38.544 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 600, 2023-09-23T19:43:38.544 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [ 2023-09-23T19:43:38.544 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.544 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45947.0:79825 10.a e152/144 10:52f61f88:::benchmark_data_smithi016_136365_object79824:head v 152'4960, mlcod=152'4958)", 2023-09-23T19:43:38.544 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:37:30.726494+0000", 2023-09-23T19:43:38.544 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 367.81175623399997, 2023-09-23T19:43:38.544 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.207575123, 2023-09-23T19:43:38.544 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:43:38.544 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:43:38.545 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:43:38.545 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.545 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:43:38.545 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.726494+0000", 2023-09-23T19:43:38.545 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:43:38.545 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.545 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.545 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:43:38.545 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.726494+0000", 2023-09-23T19:43:38.545 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:43:38.545 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.546 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.546 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:43:38.546 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.726494+0000", 2023-09-23T19:43:38.546 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:43:38.546 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.546 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.546 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:43:38.546 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.726549+0000", 2023-09-23T19:43:38.546 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.5327000000000001e-05 2023-09-23T19:43:38.546 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.546 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.547 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:43:38.547 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.796059+0000", 2023-09-23T19:43:38.547 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.069509940000000006 2023-09-23T19:43:38.547 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.547 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.547 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:43:38.547 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.796078+0000", 2023-09-23T19:43:38.547 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.8663000000000001e-05 2023-09-23T19:43:38.547 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.547 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.547 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:43:38.547 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.796252+0000", 2023-09-23T19:43:38.548 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000174122 2023-09-23T19:43:38.548 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.548 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.548 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:43:38.548 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.796341+0000", 2023-09-23T19:43:38.548 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.8912000000000005e-05 2023-09-23T19:43:38.548 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.548 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.548 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:43:38.549 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.934017+0000", 2023-09-23T19:43:38.549 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.13767655000000001 2023-09-23T19:43:38.549 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.549 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.549 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:43:38.549 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.934070+0000", 2023-09-23T19:43:38.549 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.2340999999999999e-05 2023-09-23T19:43:38.549 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:43:38.549 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:43:38.549 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:43:38.549 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.549 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.550 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45947.0:116706 10.5 10:a85d7450:::benchmark_data_smithi016_136365_object6572:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e155)", 2023-09-23T19:43:38.550 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:38:13.657092+0000", 2023-09-23T19:43:38.550 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 324.881158449, 2023-09-23T19:43:38.550 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.027455478, 2023-09-23T19:43:38.550 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:43:38.550 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "reached pg", 2023-09-23T19:43:38.550 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:43:38.550 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45947", 2023-09-23T19:43:38.550 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/4039191752", 2023-09-23T19:43:38.550 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 116706 2023-09-23T19:43:38.550 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.551 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:43:38.551 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.551 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:43:38.551 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:13.657092+0000", 2023-09-23T19:43:38.551 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:43:38.551 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.551 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.551 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:43:38.551 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:13.657092+0000", 2023-09-23T19:43:38.551 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:43:38.551 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.551 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.552 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:43:38.552 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:13.657090+0000", 2023-09-23T19:43:38.552 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999976 2023-09-23T19:43:38.552 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.552 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.552 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:43:38.552 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:13.657098+0000", 2023-09-23T19:43:38.552 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.9519999999999994e-06 2023-09-23T19:43:38.552 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.552 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.552 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:43:38.552 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:13.657113+0000", 2023-09-23T19:43:38.553 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.4826e-05 2023-09-23T19:43:38.553 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.553 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.553 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:43:38.553 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:13.657122+0000", 2023-09-23T19:43:38.553 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.0909999999999999e-06 2023-09-23T19:43:38.553 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.553 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.553 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:43:38.553 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:13.661249+0000", 2023-09-23T19:43:38.554 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0041270090000000001 2023-09-23T19:43:38.554 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.554 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.554 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op must wait for map", 2023-09-23T19:43:38.554 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:13.661257+0000", 2023-09-23T19:43:38.554 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.3909999999999998e-06 2023-09-23T19:43:38.554 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.554 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.554 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:43:38.554 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:13.668735+0000", 2023-09-23T19:43:38.554 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.007478111 2023-09-23T19:43:38.555 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.555 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.555 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-09-23T19:43:38.555 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:13.668749+0000", 2023-09-23T19:43:38.555 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3753e-05 2023-09-23T19:43:38.555 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.555 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.555 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:43:38.555 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:14.682464+0000", 2023-09-23T19:43:38.555 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0137152970000001 2023-09-23T19:43:38.555 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.555 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.556 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:43:38.556 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:14.684548+0000", 2023-09-23T19:43:38.556 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0020833890000000002 2023-09-23T19:43:38.556 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:43:38.556 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:43:38.556 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:43:38.556 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.556 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.556 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45947.0:116711 10.5 10:ae789e54:::benchmark_data_smithi016_136365_object6577:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e155)", 2023-09-23T19:43:38.556 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:38:13.657135+0000", 2023-09-23T19:43:38.556 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 324.881115658, 2023-09-23T19:43:38.556 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0523677629999999, 2023-09-23T19:43:38.557 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:43:38.557 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "reached pg", 2023-09-23T19:43:38.557 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:43:38.557 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45947", 2023-09-23T19:43:38.557 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/4039191752", 2023-09-23T19:43:38.557 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 116711 2023-09-23T19:43:38.557 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.557 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:43:38.557 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.557 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:43:38.557 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:13.657135+0000", 2023-09-23T19:43:38.557 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:43:38.558 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.558 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.558 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:43:38.558 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:13.657135+0000", 2023-09-23T19:43:38.558 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:43:38.558 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.558 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.558 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:43:38.558 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:13.657135+0000", 2023-09-23T19:43:38.558 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T19:43:38.558 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.559 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.559 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:43:38.559 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:13.657138+0000", 2023-09-23T19:43:38.559 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.3459999999999998e-06 2023-09-23T19:43:38.559 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.559 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.559 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:43:38.559 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:13.657146+0000", 2023-09-23T19:43:38.559 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.1750000000000005e-06 2023-09-23T19:43:38.559 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.559 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.559 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:43:38.560 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:13.657153+0000", 2023-09-23T19:43:38.560 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.4359999999999999e-06 2023-09-23T19:43:38.560 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.560 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.560 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:43:38.560 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:13.661404+0000", 2023-09-23T19:43:38.560 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0042519660000000003 2023-09-23T19:43:38.560 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.560 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.560 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting_for_map not empty", 2023-09-23T19:43:38.560 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:13.661411+0000", 2023-09-23T19:43:38.560 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.9369999999999998e-06 2023-09-23T19:43:38.561 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.561 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.561 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:43:38.561 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:13.672808+0000", 2023-09-23T19:43:38.561 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.011396279 2023-09-23T19:43:38.561 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.561 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.561 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-09-23T19:43:38.561 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:13.672819+0000", 2023-09-23T19:43:38.561 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0924000000000001e-05 2023-09-23T19:43:38.561 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.561 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.562 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:43:38.562 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:14.684849+0000", 2023-09-23T19:43:38.562 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.012029965 2023-09-23T19:43:38.562 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.562 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.562 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for missing object", 2023-09-23T19:43:38.562 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:14.685003+0000", 2023-09-23T19:43:38.562 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00015428100000000001 2023-09-23T19:43:38.562 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.562 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.562 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:43:38.562 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:14.709444+0000", 2023-09-23T19:43:38.563 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.024440670000000001 2023-09-23T19:43:38.563 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.563 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.563 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:43:38.563 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:14.709503+0000", 2023-09-23T19:43:38.563 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.9299000000000003e-05 2023-09-23T19:43:38.563 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:43:38.563 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:43:38.563 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:43:38.563 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.563 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.564 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45947.0:116719 10.5 10:a9484678:::benchmark_data_smithi016_136365_object6585:head [delete] snapc 0=[] ondisk+write+known_if_redirected e155)", 2023-09-23T19:43:38.564 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:38:13.661977+0000", 2023-09-23T19:43:38.564 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 324.87627399399997, 2023-09-23T19:43:38.564 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0370644170000001, 2023-09-23T19:43:38.564 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:43:38.564 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:43:38.564 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:43:38.564 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45947", 2023-09-23T19:43:38.564 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/4039191752", 2023-09-23T19:43:38.564 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 116719 2023-09-23T19:43:38.564 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.564 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:43:38.565 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.565 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:43:38.565 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:13.661977+0000", 2023-09-23T19:43:38.565 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:43:38.565 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.565 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.565 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:43:38.565 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:13.661977+0000", 2023-09-23T19:43:38.565 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:43:38.565 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.565 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.566 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:43:38.566 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:13.661975+0000", 2023-09-23T19:43:38.566 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-09-23T19:43:38.566 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.566 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.566 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:43:38.566 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:13.661981+0000", 2023-09-23T19:43:38.566 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.4420000000000003e-06 2023-09-23T19:43:38.566 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.566 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.567 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:43:38.567 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:13.661991+0000", 2023-09-23T19:43:38.567 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.9490000000000008e-06 2023-09-23T19:43:38.567 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.567 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.567 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:43:38.567 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:13.661998+0000", 2023-09-23T19:43:38.567 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.2500000000000003e-06 2023-09-23T19:43:38.567 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.567 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.567 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:43:38.568 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:13.662041+0000", 2023-09-23T19:43:38.568 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.3223000000000003e-05 2023-09-23T19:43:38.568 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.568 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.568 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting_for_map not empty", 2023-09-23T19:43:38.568 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:13.662045+0000", 2023-09-23T19:43:38.568 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.817e-06 2023-09-23T19:43:38.568 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.568 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.568 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:43:38.568 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:13.672842+0000", 2023-09-23T19:43:38.568 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.010796798 2023-09-23T19:43:38.569 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.569 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.569 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-09-23T19:43:38.569 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:13.672851+0000", 2023-09-23T19:43:38.569 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.1290000000000002e-06 2023-09-23T19:43:38.569 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.569 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.569 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:43:38.569 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:14.685102+0000", 2023-09-23T19:43:38.569 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0122516100000001 2023-09-23T19:43:38.569 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.569 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.570 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:43:38.570 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:14.686002+0000", 2023-09-23T19:43:38.570 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00089964500000000002 2023-09-23T19:43:38.570 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.570 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.570 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 3", 2023-09-23T19:43:38.570 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:14.686089+0000", 2023-09-23T19:43:38.570 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.7540000000000006e-05 2023-09-23T19:43:38.570 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.570 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.570 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:43:38.570 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:14.693742+0000", 2023-09-23T19:43:38.571 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0076530310000000002 2023-09-23T19:43:38.571 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.571 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.571 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:43:38.571 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:14.698858+0000", 2023-09-23T19:43:38.571 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0051158159999999996 2023-09-23T19:43:38.571 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.571 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.572 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:43:38.572 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:14.698945+0000", 2023-09-23T19:43:38.572 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.6442999999999999e-05 2023-09-23T19:43:38.572 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.572 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.572 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:43:38.573 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:14.699041+0000", 2023-09-23T19:43:38.573 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.6409999999999993e-05 2023-09-23T19:43:38.573 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:43:38.573 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:43:38.573 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:43:38.573 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.573 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.573 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45947.0:116732 10.5 10:ad4b454d:::benchmark_data_smithi016_136365_object6598:head [delete] snapc 0=[] ondisk+write+known_if_redirected e155)", 2023-09-23T19:43:38.573 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:38:13.688275+0000", 2023-09-23T19:43:38.573 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 324.84997554799997, 2023-09-23T19:43:38.573 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.011000721, 2023-09-23T19:43:38.574 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:43:38.574 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:43:38.574 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:43:38.574 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45947", 2023-09-23T19:43:38.574 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/4039191752", 2023-09-23T19:43:38.574 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 116732 2023-09-23T19:43:38.574 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.574 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:43:38.574 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.574 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:43:38.574 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:13.688275+0000", 2023-09-23T19:43:38.574 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:43:38.575 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.575 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.575 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:43:38.575 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:13.688275+0000", 2023-09-23T19:43:38.575 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:43:38.575 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.575 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.575 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:43:38.575 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:13.688274+0000", 2023-09-23T19:43:38.575 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-09-23T19:43:38.575 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.575 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.576 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:43:38.576 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:13.688282+0000", 2023-09-23T19:43:38.576 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.8639999999999996e-06 2023-09-23T19:43:38.576 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.576 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.576 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:43:38.576 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:13.688295+0000", 2023-09-23T19:43:38.576 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3365e-05 2023-09-23T19:43:38.576 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.576 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.576 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:43:38.576 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:13.688302+0000", 2023-09-23T19:43:38.577 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.2230000000000001e-06 2023-09-23T19:43:38.577 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.577 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.577 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:43:38.577 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:13.690250+0000", 2023-09-23T19:43:38.577 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0019476700000000001 2023-09-23T19:43:38.577 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.577 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.577 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-09-23T19:43:38.577 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:13.690259+0000", 2023-09-23T19:43:38.577 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.5570000000000001e-06 2023-09-23T19:43:38.577 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.577 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.578 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:43:38.578 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:14.689760+0000", 2023-09-23T19:43:38.578 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.99950072599999995 2023-09-23T19:43:38.578 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.578 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.578 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:43:38.578 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:14.689873+0000", 2023-09-23T19:43:38.578 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000112642 2023-09-23T19:43:38.578 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.578 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.578 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 3", 2023-09-23T19:43:38.578 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:14.689960+0000", 2023-09-23T19:43:38.579 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.7754e-05 2023-09-23T19:43:38.579 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.579 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.579 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:43:38.579 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:14.693750+0000", 2023-09-23T19:43:38.579 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.003789316 2023-09-23T19:43:38.579 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.579 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.579 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:43:38.579 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:14.699112+0000", 2023-09-23T19:43:38.579 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0053622940000000001 2023-09-23T19:43:38.579 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.579 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.580 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:43:38.580 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:14.699168+0000", 2023-09-23T19:43:38.580 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.5773999999999998e-05 2023-09-23T19:43:38.580 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.580 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.580 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:43:38.580 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:14.699276+0000", 2023-09-23T19:43:38.580 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000108026 2023-09-23T19:43:38.580 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:43:38.580 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:43:38.580 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:43:38.581 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.581 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.581 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45947.0:116734 10.8 10:1b758e78:::benchmark_data_smithi016_136365_object6600:head [delete] snapc 0=[] ondisk+write+known_if_redirected e155)", 2023-09-23T19:43:38.581 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:38:13.688477+0000", 2023-09-23T19:43:38.581 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 324.84977363500002, 2023-09-23T19:43:38.581 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.98948107500000004, 2023-09-23T19:43:38.581 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:43:38.581 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:43:38.581 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:43:38.581 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45947", 2023-09-23T19:43:38.581 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/4039191752", 2023-09-23T19:43:38.581 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 116734 2023-09-23T19:43:38.581 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.582 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:43:38.582 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.582 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:43:38.582 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:13.688477+0000", 2023-09-23T19:43:38.582 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:43:38.582 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.582 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.582 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:43:38.582 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:13.688477+0000", 2023-09-23T19:43:38.582 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:43:38.582 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.582 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.583 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:43:38.583 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:13.688476+0000", 2023-09-23T19:43:38.583 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T19:43:38.583 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.583 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.583 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:43:38.583 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:13.688481+0000", 2023-09-23T19:43:38.583 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.3560000000000003e-06 2023-09-23T19:43:38.583 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.583 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.583 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:43:38.583 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:13.688491+0000", 2023-09-23T19:43:38.583 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0042e-05 2023-09-23T19:43:38.584 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.584 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.584 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:43:38.584 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:13.688496+0000", 2023-09-23T19:43:38.584 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.6579999999999999e-06 2023-09-23T19:43:38.584 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.584 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.584 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:43:38.584 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:13.688559+0000", 2023-09-23T19:43:38.584 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.2922999999999994e-05 2023-09-23T19:43:38.584 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.584 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.584 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-09-23T19:43:38.585 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:13.688570+0000", 2023-09-23T19:43:38.585 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1032e-05 2023-09-23T19:43:38.585 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.585 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.585 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:43:38.585 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:14.669124+0000", 2023-09-23T19:43:38.585 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.98055320199999996 2023-09-23T19:43:38.585 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.585 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.585 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:43:38.585 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:14.669301+0000", 2023-09-23T19:43:38.585 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000177004 2023-09-23T19:43:38.585 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.586 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.586 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 7", 2023-09-23T19:43:38.586 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:14.669460+0000", 2023-09-23T19:43:38.586 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00015990000000000001 2023-09-23T19:43:38.586 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.586 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.586 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:43:38.586 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:14.677765+0000", 2023-09-23T19:43:38.586 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0083046479999999995 2023-09-23T19:43:38.586 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.586 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.587 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:43:38.587 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:14.677813+0000", 2023-09-23T19:43:38.587 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.8310000000000003e-05 2023-09-23T19:43:38.587 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.587 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.587 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:43:38.587 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:14.677870+0000", 2023-09-23T19:43:38.587 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.6255000000000003e-05 2023-09-23T19:43:38.587 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.587 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.587 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:43:38.587 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:14.677958+0000", 2023-09-23T19:43:38.587 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.8448000000000004e-05 2023-09-23T19:43:38.588 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:43:38.588 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:43:38.588 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:43:38.588 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.588 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.588 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45947.0:116743 10.8 10:142413be:::benchmark_data_smithi016_136365_object6609:head [delete] snapc 0=[] ondisk+write+known_if_redirected e155)", 2023-09-23T19:43:38.588 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:38:13.701671+0000", 2023-09-23T19:43:38.588 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 324.836579814, 2023-09-23T19:43:38.588 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.97721656499999998, 2023-09-23T19:43:38.588 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:43:38.588 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:43:38.588 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:43:38.588 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45947", 2023-09-23T19:43:38.589 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/4039191752", 2023-09-23T19:43:38.589 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 116743 2023-09-23T19:43:38.589 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.589 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:43:38.589 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.589 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:43:38.589 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:13.701671+0000", 2023-09-23T19:43:38.589 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:43:38.589 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.589 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.589 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:43:38.589 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:13.701671+0000", 2023-09-23T19:43:38.589 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:43:38.589 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.590 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.590 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:43:38.590 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:13.701669+0000", 2023-09-23T19:43:38.590 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-09-23T19:43:38.590 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.590 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.590 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:43:38.590 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:13.701677+0000", 2023-09-23T19:43:38.590 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.1540000000000002e-06 2023-09-23T19:43:38.590 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.590 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.590 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:43:38.591 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:13.701693+0000", 2023-09-23T19:43:38.591 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.6313000000000001e-05 2023-09-23T19:43:38.591 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.591 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.591 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:43:38.591 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:13.701702+0000", 2023-09-23T19:43:38.591 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.8929999999999994e-06 2023-09-23T19:43:38.591 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.591 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.591 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:43:38.591 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:13.701752+0000", 2023-09-23T19:43:38.591 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.9731999999999998e-05 2023-09-23T19:43:38.592 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.592 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.592 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-09-23T19:43:38.592 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:13.701764+0000", 2023-09-23T19:43:38.592 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1837e-05 2023-09-23T19:43:38.592 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.592 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.592 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:43:38.592 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:14.673148+0000", 2023-09-23T19:43:38.592 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.97138450700000001 2023-09-23T19:43:38.592 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.592 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.592 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:43:38.592 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:14.673351+0000", 2023-09-23T19:43:38.593 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00020258900000000001 2023-09-23T19:43:38.593 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.593 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.593 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 7", 2023-09-23T19:43:38.593 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:14.673529+0000", 2023-09-23T19:43:38.593 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00017852699999999999 2023-09-23T19:43:38.593 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.593 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.593 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:43:38.593 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:14.678015+0000", 2023-09-23T19:43:38.593 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0044856160000000004 2023-09-23T19:43:38.593 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.593 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.594 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:43:38.594 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:14.678761+0000", 2023-09-23T19:43:38.594 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00074599599999999996 2023-09-23T19:43:38.594 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.594 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.594 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:43:38.594 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:14.678809+0000", 2023-09-23T19:43:38.594 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.7475999999999997e-05 2023-09-23T19:43:38.594 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.594 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.594 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:43:38.594 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:38:14.678887+0000", 2023-09-23T19:43:38.594 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.8914999999999999e-05 2023-09-23T19:43:38.595 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:43:38.595 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:43:38.595 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:43:38.595 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.595 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.595 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45947.0:207465 10.6 e171/144 10:652cdf93:::benchmark_data_smithi016_136365_object97331:head v 171'12792, mlcod=171'12790)", 2023-09-23T19:43:38.595 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:40:28.081262+0000", 2023-09-23T19:43:38.595 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 190.45698825299999, 2023-09-23T19:43:38.595 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.25118891300000001, 2023-09-23T19:43:38.595 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:43:38.595 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:43:38.595 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:43:38.595 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.595 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:43:38.596 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.081262+0000", 2023-09-23T19:43:38.596 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:43:38.596 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.596 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.596 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:43:38.596 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.081262+0000", 2023-09-23T19:43:38.596 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:43:38.596 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.596 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.596 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:43:38.596 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.081261+0000", 2023-09-23T19:43:38.596 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-09-23T19:43:38.596 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.597 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.597 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:43:38.597 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.081268+0000", 2023-09-23T19:43:38.597 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.6630000000000004e-06 2023-09-23T19:43:38.597 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.597 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.597 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:43:38.597 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.081277+0000", 2023-09-23T19:43:38.597 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.0240000000000003e-06 2023-09-23T19:43:38.597 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.597 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.597 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:43:38.598 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.081284+0000", 2023-09-23T19:43:38.598 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.3640000000000001e-06 2023-09-23T19:43:38.598 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.598 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.598 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:43:38.598 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.081327+0000", 2023-09-23T19:43:38.598 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.2799999999999997e-05 2023-09-23T19:43:38.598 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.598 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.598 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:43:38.598 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.081349+0000", 2023-09-23T19:43:38.598 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.1566999999999999e-05 2023-09-23T19:43:38.598 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.599 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.599 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:43:38.599 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.329197+0000", 2023-09-23T19:43:38.599 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.24784800300000001 2023-09-23T19:43:38.599 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.599 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.599 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:43:38.599 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.332451+0000", 2023-09-23T19:43:38.599 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0032547000000000001 2023-09-23T19:43:38.599 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:43:38.599 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:43:38.599 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:43:38.599 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.600 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.600 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45947.0:207464 10.1 e171/144 10:8927d421:::benchmark_data_smithi016_136365_object97330:head v 171'12975, mlcod=171'12973)", 2023-09-23T19:43:38.600 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:40:28.083389+0000", 2023-09-23T19:43:38.600 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 190.45486147700001, 2023-09-23T19:43:38.600 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.24909103899999999, 2023-09-23T19:43:38.600 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:43:38.600 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:43:38.600 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:43:38.600 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.600 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:43:38.600 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.083389+0000", 2023-09-23T19:43:38.600 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:43:38.600 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.601 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.601 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:43:38.601 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.083389+0000", 2023-09-23T19:43:38.601 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:43:38.601 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.601 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.601 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:43:38.601 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.083388+0000", 2023-09-23T19:43:38.601 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:43:38.601 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.601 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.601 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:43:38.602 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.083397+0000", 2023-09-23T19:43:38.602 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.8049999999999996e-06 2023-09-23T19:43:38.602 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.602 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.602 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:43:38.602 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.083409+0000", 2023-09-23T19:43:38.602 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2189e-05 2023-09-23T19:43:38.602 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.602 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.602 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:43:38.602 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.083417+0000", 2023-09-23T19:43:38.602 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.3170000000000004e-06 2023-09-23T19:43:38.602 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.603 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.603 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:43:38.603 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.083467+0000", 2023-09-23T19:43:38.603 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.0550000000000002e-05 2023-09-23T19:43:38.603 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.603 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.603 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:43:38.603 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.083488+0000", 2023-09-23T19:43:38.603 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.1146999999999999e-05 2023-09-23T19:43:38.603 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.603 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.603 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:43:38.604 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.332448+0000", 2023-09-23T19:43:38.604 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.24895941499999999 2023-09-23T19:43:38.604 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.604 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.604 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:43:38.604 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.332480+0000", 2023-09-23T19:43:38.604 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.2663999999999998e-05 2023-09-23T19:43:38.604 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:43:38.604 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:43:38.604 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:43:38.604 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.604 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.605 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45947.0:207469 10.6 e171/144 10:6cee13bb:::benchmark_data_smithi016_136365_object97335:head v 171'12793, mlcod=171'12790)", 2023-09-23T19:43:38.605 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:40:28.086193+0000", 2023-09-23T19:43:38.605 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 190.45205820699999, 2023-09-23T19:43:38.605 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.25814531200000002, 2023-09-23T19:43:38.605 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:43:38.605 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:43:38.605 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:43:38.605 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.605 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:43:38.605 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.086193+0000", 2023-09-23T19:43:38.605 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:43:38.605 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.605 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.606 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:43:38.606 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.086193+0000", 2023-09-23T19:43:38.606 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:43:38.606 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.606 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.606 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:43:38.606 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.086191+0000", 2023-09-23T19:43:38.606 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-09-23T19:43:38.606 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.606 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.606 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:43:38.606 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.086202+0000", 2023-09-23T19:43:38.607 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1627999999999999e-05 2023-09-23T19:43:38.607 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.607 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.607 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:43:38.607 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.328356+0000", 2023-09-23T19:43:38.607 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.242153286 2023-09-23T19:43:38.607 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.607 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.607 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:43:38.607 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.328384+0000", 2023-09-23T19:43:38.607 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.8586999999999999e-05 2023-09-23T19:43:38.607 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.607 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.608 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:43:38.608 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.332530+0000", 2023-09-23T19:43:38.608 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.004145475 2023-09-23T19:43:38.608 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.608 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.609 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:43:38.609 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.332552+0000", 2023-09-23T19:43:38.609 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.2208000000000002e-05 2023-09-23T19:43:38.609 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.610 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.610 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:43:38.610 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.344289+0000", 2023-09-23T19:43:38.610 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.011736961000000001 2023-09-23T19:43:38.610 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.610 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.610 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:43:38.610 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.344338+0000", 2023-09-23T19:43:38.610 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.8965000000000003e-05 2023-09-23T19:43:38.610 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:43:38.610 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:43:38.611 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:43:38.611 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.611 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.611 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45947.0:207474 10.4 10:2b46c204:::benchmark_data_smithi016_136365_object97340:head [delete] snapc 0=[] ondisk+write+known_if_redirected e171)", 2023-09-23T19:43:38.611 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:40:28.088271+0000", 2023-09-23T19:43:38.611 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 190.449979676, 2023-09-23T19:43:38.611 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.26161290700000001, 2023-09-23T19:43:38.611 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:43:38.611 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:43:38.611 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:43:38.611 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45947", 2023-09-23T19:43:38.611 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/4039191752", 2023-09-23T19:43:38.612 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 207474 2023-09-23T19:43:38.612 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.612 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:43:38.612 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.612 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:43:38.612 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.088271+0000", 2023-09-23T19:43:38.612 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:43:38.612 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.612 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.612 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:43:38.612 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.088271+0000", 2023-09-23T19:43:38.612 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:43:38.613 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.613 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.613 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:43:38.613 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.088269+0000", 2023-09-23T19:43:38.613 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-09-23T19:43:38.613 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.613 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.613 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:43:38.613 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.088288+0000", 2023-09-23T19:43:38.613 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.8822e-05 2023-09-23T19:43:38.613 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.613 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.613 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:43:38.614 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.328378+0000", 2023-09-23T19:43:38.614 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.24009019000000001 2023-09-23T19:43:38.614 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.614 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.614 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:43:38.614 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.328411+0000", 2023-09-23T19:43:38.614 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.2830000000000002e-05 2023-09-23T19:43:38.614 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.614 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.614 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:43:38.614 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.328682+0000", 2023-09-23T19:43:38.614 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00027039000000000002 2023-09-23T19:43:38.615 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.615 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.615 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:43:38.615 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.328832+0000", 2023-09-23T19:43:38.615 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000150736 2023-09-23T19:43:38.615 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.615 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.615 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 3,4", 2023-09-23T19:43:38.615 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.328968+0000", 2023-09-23T19:43:38.615 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00013575800000000001 2023-09-23T19:43:38.615 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.615 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.615 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:43:38.616 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.336992+0000", 2023-09-23T19:43:38.616 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0080238370000000007 2023-09-23T19:43:38.616 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.616 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.616 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:43:38.616 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.349737+0000", 2023-09-23T19:43:38.616 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.012745126000000001 2023-09-23T19:43:38.616 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.616 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.616 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:43:38.616 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.349791+0000", 2023-09-23T19:43:38.617 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.3619999999999998e-05 2023-09-23T19:43:38.617 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.617 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.617 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:43:38.617 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.349837+0000", 2023-09-23T19:43:38.617 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.6062e-05 2023-09-23T19:43:38.617 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.617 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.617 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:43:38.617 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.349884+0000", 2023-09-23T19:43:38.617 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.7209000000000002e-05 2023-09-23T19:43:38.617 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:43:38.617 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:43:38.618 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:43:38.618 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.618 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.618 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45947.0:207476 10.c e171/144 10:32805061:::benchmark_data_smithi016_136365_object97342:head v 171'12959, mlcod=171'12957)", 2023-09-23T19:43:38.618 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:40:28.092017+0000", 2023-09-23T19:43:38.618 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 190.44623337199999, 2023-09-23T19:43:38.618 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.257763352, 2023-09-23T19:43:38.618 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:43:38.618 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:43:38.618 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:43:38.618 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.618 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:43:38.619 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.092017+0000", 2023-09-23T19:43:38.619 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:43:38.621 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.621 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.621 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:43:38.621 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.092017+0000", 2023-09-23T19:43:38.621 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:43:38.621 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.622 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.622 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:43:38.622 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.092016+0000", 2023-09-23T19:43:38.622 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-09-23T19:43:38.622 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.622 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.622 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:43:38.622 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.092027+0000", 2023-09-23T19:43:38.622 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0849999999999999e-05 2023-09-23T19:43:38.622 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.622 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.622 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:43:38.622 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.328377+0000", 2023-09-23T19:43:38.623 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.23635049399999999 2023-09-23T19:43:38.623 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.623 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.623 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:43:38.623 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.328397+0000", 2023-09-23T19:43:38.623 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.9259999999999999e-05 2023-09-23T19:43:38.623 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.623 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.623 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:43:38.623 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.328543+0000", 2023-09-23T19:43:38.623 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00014627499999999999 2023-09-23T19:43:38.623 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.624 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.624 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:43:38.624 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.328592+0000", 2023-09-23T19:43:38.624 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.9187000000000002e-05 2023-09-23T19:43:38.624 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.624 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.624 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:43:38.624 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.349742+0000", 2023-09-23T19:43:38.624 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.021149748999999999 2023-09-23T19:43:38.624 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.624 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.624 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:43:38.624 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.349781+0000", 2023-09-23T19:43:38.625 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.8847999999999998e-05 2023-09-23T19:43:38.625 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:43:38.625 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:43:38.625 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:43:38.625 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.625 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.625 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.36243.0:71857 11.6 11:636773fc:::benchmark_data_smithi016_160148_object71856:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e186)", 2023-09-23T19:43:38.625 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:42:33.131851+0000", 2023-09-23T19:43:38.625 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 65.406399475000001, 2023-09-23T19:43:38.625 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.093090425000000004, 2023-09-23T19:43:38.625 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:43:38.625 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:43:38.625 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:43:38.626 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.36243", 2023-09-23T19:43:38.626 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1280038109", 2023-09-23T19:43:38.626 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 71857 2023-09-23T19:43:38.626 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.626 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:43:38.626 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.626 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:43:38.626 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:33.131851+0000", 2023-09-23T19:43:38.626 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:43:38.626 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.626 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.626 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:43:38.627 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:33.131851+0000", 2023-09-23T19:43:38.627 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:43:38.627 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.627 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.627 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:43:38.627 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:33.131850+0000", 2023-09-23T19:43:38.627 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-09-23T19:43:38.627 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.627 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.627 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:43:38.627 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:33.131871+0000", 2023-09-23T19:43:38.627 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.1240999999999999e-05 2023-09-23T19:43:38.627 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.628 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.628 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:43:38.628 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:33.131881+0000", 2023-09-23T19:43:38.628 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.4339999999999998e-06 2023-09-23T19:43:38.628 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.628 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.628 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:43:38.628 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:33.131900+0000", 2023-09-23T19:43:38.628 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.9137999999999999e-05 2023-09-23T19:43:38.628 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.628 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.628 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:43:38.628 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:33.131944+0000", 2023-09-23T19:43:38.629 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.4076000000000003e-05 2023-09-23T19:43:38.629 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.629 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.629 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:43:38.629 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:33.132044+0000", 2023-09-23T19:43:38.629 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.9625999999999998e-05 2023-09-23T19:43:38.629 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.629 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.629 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 1,2", 2023-09-23T19:43:38.629 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:33.132132+0000", 2023-09-23T19:43:38.629 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.8357999999999997e-05 2023-09-23T19:43:38.629 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.629 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.630 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:43:38.630 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:33.134814+0000", 2023-09-23T19:43:38.630 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0026820400000000001 2023-09-23T19:43:38.630 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.630 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.630 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:43:38.630 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:33.223637+0000", 2023-09-23T19:43:38.630 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.088823009999999994 2023-09-23T19:43:38.630 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.630 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.630 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:43:38.630 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:33.224689+0000", 2023-09-23T19:43:38.631 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0010522699999999999 2023-09-23T19:43:38.631 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.631 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.631 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:43:38.631 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:33.224886+0000", 2023-09-23T19:43:38.631 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00019712700000000001 2023-09-23T19:43:38.631 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.631 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.631 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:43:38.631 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:33.224942+0000", 2023-09-23T19:43:38.631 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.5352999999999997e-05 2023-09-23T19:43:38.631 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:43:38.632 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:43:38.632 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:43:38.632 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.632 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.632 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.36243.0:73076 11.6 11:6f44afb5:::benchmark_data_smithi016_160148_object73075:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e186)", 2023-09-23T19:43:38.632 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:42:34.423754+0000", 2023-09-23T19:43:38.632 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 64.114496220999996, 2023-09-23T19:43:38.632 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.10642233600000001, 2023-09-23T19:43:38.632 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:43:38.632 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:43:38.632 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:43:38.632 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.36243", 2023-09-23T19:43:38.632 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1280038109", 2023-09-23T19:43:38.632 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 73076 2023-09-23T19:43:38.633 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.633 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:43:38.633 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.633 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:43:38.633 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.423754+0000", 2023-09-23T19:43:38.633 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:43:38.633 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.633 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.633 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:43:38.633 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.423754+0000", 2023-09-23T19:43:38.633 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:43:38.633 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.634 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.634 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:43:38.634 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.423752+0000", 2023-09-23T19:43:38.634 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-09-23T19:43:38.634 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.634 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.634 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:43:38.634 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.423785+0000", 2023-09-23T19:43:38.634 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.2370999999999999e-05 2023-09-23T19:43:38.634 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.634 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.634 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:43:38.635 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.423804+0000", 2023-09-23T19:43:38.635 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.8918000000000001e-05 2023-09-23T19:43:38.635 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.635 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.635 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:43:38.635 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.423814+0000", 2023-09-23T19:43:38.635 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.8689999999999999e-06 2023-09-23T19:43:38.635 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.635 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.635 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:43:38.635 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.423865+0000", 2023-09-23T19:43:38.635 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.1814999999999998e-05 2023-09-23T19:43:38.635 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.635 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.636 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:43:38.636 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.424034+0000", 2023-09-23T19:43:38.636 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00016892799999999999 2023-09-23T19:43:38.636 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.636 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.636 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 1,2", 2023-09-23T19:43:38.636 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.424209+0000", 2023-09-23T19:43:38.636 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00017483700000000001 2023-09-23T19:43:38.636 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.636 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.636 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:43:38.636 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.429977+0000", 2023-09-23T19:43:38.636 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.005768217 2023-09-23T19:43:38.637 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.637 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.637 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:43:38.637 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.430024+0000", 2023-09-23T19:43:38.637 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.6859e-05 2023-09-23T19:43:38.637 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.637 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.637 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:43:38.637 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.530051+0000", 2023-09-23T19:43:38.637 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.100026752 2023-09-23T19:43:38.637 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.637 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.637 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:43:38.638 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.530121+0000", 2023-09-23T19:43:38.638 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.0222000000000006e-05 2023-09-23T19:43:38.638 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.638 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.638 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:43:38.638 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.530177+0000", 2023-09-23T19:43:38.638 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.5668999999999998e-05 2023-09-23T19:43:38.638 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:43:38.638 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:43:38.638 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:43:38.638 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.638 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.638 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.36243.0:73094 11.6 11:6d11af52:::benchmark_data_smithi016_160148_object73093:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e186)", 2023-09-23T19:43:38.639 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:42:34.439454+0000", 2023-09-23T19:43:38.639 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 64.098796723999996, 2023-09-23T19:43:38.639 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.090897679999999995, 2023-09-23T19:43:38.639 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:43:38.639 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:43:38.639 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:43:38.639 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.36243", 2023-09-23T19:43:38.639 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1280038109", 2023-09-23T19:43:38.639 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 73094 2023-09-23T19:43:38.639 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.639 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:43:38.639 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.640 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:43:38.640 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.439454+0000", 2023-09-23T19:43:38.640 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:43:38.640 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.640 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.640 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:43:38.640 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.439454+0000", 2023-09-23T19:43:38.640 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:43:38.640 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.640 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.640 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:43:38.640 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.439452+0000", 2023-09-23T19:43:38.640 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-09-23T19:43:38.641 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.641 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.641 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:43:38.641 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.439488+0000", 2023-09-23T19:43:38.641 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.6137999999999999e-05 2023-09-23T19:43:38.641 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.641 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.641 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:43:38.641 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.439506+0000", 2023-09-23T19:43:38.641 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.7713000000000001e-05 2023-09-23T19:43:38.641 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.641 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.641 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:43:38.642 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.439514+0000", 2023-09-23T19:43:38.642 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.0299999999999994e-06 2023-09-23T19:43:38.642 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.642 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.642 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:43:38.642 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.439561+0000", 2023-09-23T19:43:38.642 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.7420999999999999e-05 2023-09-23T19:43:38.642 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.642 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.642 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:43:38.642 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.439702+0000", 2023-09-23T19:43:38.643 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00014056800000000001 2023-09-23T19:43:38.643 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.643 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.643 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 1,2", 2023-09-23T19:43:38.643 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.439978+0000", 2023-09-23T19:43:38.643 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00027598100000000001 2023-09-23T19:43:38.643 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.643 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.643 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:43:38.643 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.445566+0000", 2023-09-23T19:43:38.643 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00558773 2023-09-23T19:43:38.643 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.643 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.644 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:43:38.644 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.445737+0000", 2023-09-23T19:43:38.644 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00017080799999999999 2023-09-23T19:43:38.644 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.644 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.644 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:43:38.644 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.530252+0000", 2023-09-23T19:43:38.644 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.084515321000000004 2023-09-23T19:43:38.644 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.644 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.644 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:43:38.644 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.530309+0000", 2023-09-23T19:43:38.644 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.6669999999999998e-05 2023-09-23T19:43:38.645 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.645 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.645 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:43:38.645 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.530352+0000", 2023-09-23T19:43:38.645 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.3137999999999999e-05 2023-09-23T19:43:38.645 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:43:38.645 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:43:38.645 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:43:38.645 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.645 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.645 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.36243.0:73096 11.6 11:6b40700a:::benchmark_data_smithi016_160148_object73095:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e186)", 2023-09-23T19:43:38.645 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:42:34.442017+0000", 2023-09-23T19:43:38.646 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 64.096233810000001, 2023-09-23T19:43:38.646 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.092005972000000005, 2023-09-23T19:43:38.646 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:43:38.646 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:43:38.646 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:43:38.646 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.36243", 2023-09-23T19:43:38.646 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1280038109", 2023-09-23T19:43:38.646 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 73096 2023-09-23T19:43:38.646 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.646 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:43:38.646 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.646 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:43:38.647 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.442017+0000", 2023-09-23T19:43:38.647 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:43:38.647 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.647 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.647 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:43:38.647 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.442017+0000", 2023-09-23T19:43:38.647 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:43:38.647 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.647 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.647 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:43:38.647 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.442015+0000", 2023-09-23T19:43:38.647 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-09-23T19:43:38.647 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.648 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.648 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:43:38.648 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.442038+0000", 2023-09-23T19:43:38.648 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.2870999999999998e-05 2023-09-23T19:43:38.648 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.648 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.648 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:43:38.648 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.442051+0000", 2023-09-23T19:43:38.648 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3199e-05 2023-09-23T19:43:38.648 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.648 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.648 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:43:38.649 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.442058+0000", 2023-09-23T19:43:38.649 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.6969999999999996e-06 2023-09-23T19:43:38.649 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.649 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.649 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:43:38.649 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.442887+0000", 2023-09-23T19:43:38.649 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00082837000000000004 2023-09-23T19:43:38.649 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.649 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.649 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:43:38.649 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.443058+0000", 2023-09-23T19:43:38.649 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00017153099999999999 2023-09-23T19:43:38.649 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.650 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.650 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 1,2", 2023-09-23T19:43:38.650 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.443266+0000", 2023-09-23T19:43:38.650 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00020746299999999999 2023-09-23T19:43:38.650 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.650 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.650 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:43:38.650 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.445819+0000", 2023-09-23T19:43:38.650 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0025534189999999999 2023-09-23T19:43:38.650 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.650 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.650 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:43:38.651 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.447220+0000", 2023-09-23T19:43:38.651 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00140116 2023-09-23T19:43:38.651 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.651 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.651 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:43:38.651 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.533926+0000", 2023-09-23T19:43:38.651 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.086706189000000003 2023-09-23T19:43:38.651 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.651 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.651 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:43:38.651 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.533981+0000", 2023-09-23T19:43:38.651 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.5090999999999998e-05 2023-09-23T19:43:38.651 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.652 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.652 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:43:38.652 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.534023+0000", 2023-09-23T19:43:38.652 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.1488e-05 2023-09-23T19:43:38.652 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:43:38.652 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:43:38.652 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:43:38.652 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.652 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.652 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.36243.0:73310 11.6 11:68d62697:::benchmark_data_smithi016_160148_object73309:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e186)", 2023-09-23T19:43:38.652 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:42:34.736359+0000", 2023-09-23T19:43:38.652 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 63.801891718, 2023-09-23T19:43:38.653 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.046413111, 2023-09-23T19:43:38.653 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:43:38.653 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:43:38.653 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:43:38.653 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.36243", 2023-09-23T19:43:38.653 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1280038109", 2023-09-23T19:43:38.653 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 73310 2023-09-23T19:43:38.653 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.653 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:43:38.653 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.653 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:43:38.653 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.736359+0000", 2023-09-23T19:43:38.653 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:43:38.654 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.654 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.654 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:43:38.654 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.736359+0000", 2023-09-23T19:43:38.654 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:43:38.654 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.654 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.654 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:43:38.654 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.736282+0000", 2023-09-23T19:43:38.654 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999228 2023-09-23T19:43:38.654 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.654 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.655 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:43:38.655 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.736391+0000", 2023-09-23T19:43:38.655 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000109307 2023-09-23T19:43:38.655 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.655 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.655 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:43:38.655 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.736406+0000", 2023-09-23T19:43:38.655 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.4727e-05 2023-09-23T19:43:38.655 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.655 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.655 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:43:38.655 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.736420+0000", 2023-09-23T19:43:38.656 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3706999999999999e-05 2023-09-23T19:43:38.656 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.656 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.656 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:43:38.656 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.736466+0000", 2023-09-23T19:43:38.656 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.6489999999999997e-05 2023-09-23T19:43:38.656 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.656 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.656 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:43:38.656 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.736757+0000", 2023-09-23T19:43:38.656 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00029131900000000001 2023-09-23T19:43:38.656 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.657 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.657 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 1,2", 2023-09-23T19:43:38.657 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.736938+0000", 2023-09-23T19:43:38.657 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00018104199999999999 2023-09-23T19:43:38.657 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.657 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.657 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:43:38.657 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.744455+0000", 2023-09-23T19:43:38.657 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0075167419999999999 2023-09-23T19:43:38.657 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.657 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.657 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:43:38.658 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.744646+0000", 2023-09-23T19:43:38.658 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000191079 2023-09-23T19:43:38.658 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.658 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.658 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:43:38.658 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.782642+0000", 2023-09-23T19:43:38.658 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.037995585999999998 2023-09-23T19:43:38.658 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.658 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.658 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:43:38.658 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.782706+0000", 2023-09-23T19:43:38.658 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.4228999999999998e-05 2023-09-23T19:43:38.659 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.659 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.659 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:43:38.659 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.782772+0000", 2023-09-23T19:43:38.659 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.6058999999999999e-05 2023-09-23T19:43:38.659 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:43:38.659 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:43:38.659 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:43:38.659 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.659 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.659 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.36243.0:73311 11.6 11:6f956300:::benchmark_data_smithi016_160148_object73310:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e186)", 2023-09-23T19:43:38.659 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:42:34.736441+0000", 2023-09-23T19:43:38.659 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 63.801809298999999, 2023-09-23T19:43:38.660 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.046459955999999997, 2023-09-23T19:43:38.660 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:43:38.660 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:43:38.660 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:43:38.660 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.36243", 2023-09-23T19:43:38.660 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1280038109", 2023-09-23T19:43:38.660 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 73311 2023-09-23T19:43:38.660 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.660 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:43:38.660 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.660 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:43:38.660 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.736441+0000", 2023-09-23T19:43:38.661 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:43:38.661 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.661 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.661 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:43:38.661 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.736441+0000", 2023-09-23T19:43:38.661 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:43:38.661 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.661 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.661 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:43:38.661 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.736441+0000", 2023-09-23T19:43:38.661 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T19:43:38.661 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.661 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.662 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:43:38.662 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.736464+0000", 2023-09-23T19:43:38.662 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.3552000000000001e-05 2023-09-23T19:43:38.662 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.662 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.662 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:43:38.662 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.736487+0000", 2023-09-23T19:43:38.662 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.3078000000000002e-05 2023-09-23T19:43:38.662 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.662 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.662 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:43:38.662 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.736607+0000", 2023-09-23T19:43:38.663 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00011972300000000001 2023-09-23T19:43:38.663 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.663 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.663 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:43:38.663 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.739626+0000", 2023-09-23T19:43:38.663 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0030185910000000002 2023-09-23T19:43:38.663 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.663 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.663 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:43:38.663 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.739966+0000", 2023-09-23T19:43:38.663 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00034036299999999999 2023-09-23T19:43:38.663 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.663 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.664 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 1,2", 2023-09-23T19:43:38.664 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.740437+0000", 2023-09-23T19:43:38.664 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000470889 2023-09-23T19:43:38.664 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.664 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.664 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:43:38.664 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.744488+0000", 2023-09-23T19:43:38.664 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00405142 2023-09-23T19:43:38.664 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.664 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.664 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:43:38.664 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.747918+0000", 2023-09-23T19:43:38.665 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0034296859999999999 2023-09-23T19:43:38.665 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.665 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.665 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:43:38.665 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.782821+0000", 2023-09-23T19:43:38.665 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.034903269000000001 2023-09-23T19:43:38.665 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.665 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.665 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:43:38.665 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.782865+0000", 2023-09-23T19:43:38.665 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.3618999999999997e-05 2023-09-23T19:43:38.665 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.666 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.666 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:43:38.666 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:34.782901+0000", 2023-09-23T19:43:38.666 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.6455000000000002e-05 2023-09-23T19:43:38.666 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:43:38.666 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:43:38.666 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:43:38.666 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.666 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.666 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.36243.0:74051 11.6 11:65c258ad:::benchmark_data_smithi016_160148_object74050:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e186)", 2023-09-23T19:43:38.666 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:42:35.484110+0000", 2023-09-23T19:43:38.666 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 63.054140834000002, 2023-09-23T19:43:38.667 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.12983930900000001, 2023-09-23T19:43:38.667 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:43:38.667 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:43:38.667 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:43:38.667 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.36243", 2023-09-23T19:43:38.667 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1280038109", 2023-09-23T19:43:38.667 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 74051 2023-09-23T19:43:38.667 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.667 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:43:38.667 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.667 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:43:38.667 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:35.484110+0000", 2023-09-23T19:43:38.667 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:43:38.668 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.668 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.668 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:43:38.668 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:35.484110+0000", 2023-09-23T19:43:38.668 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:43:38.668 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.668 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.668 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:43:38.668 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:35.484108+0000", 2023-09-23T19:43:38.668 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999976 2023-09-23T19:43:38.668 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.668 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.668 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:43:38.669 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:35.484150+0000", 2023-09-23T19:43:38.669 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.1820999999999998e-05 2023-09-23T19:43:38.669 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.669 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.669 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:43:38.669 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:35.484177+0000", 2023-09-23T19:43:38.669 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.7209e-05 2023-09-23T19:43:38.669 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.669 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.669 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:43:38.669 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:35.484184+0000", 2023-09-23T19:43:38.669 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.9970000000000001e-06 2023-09-23T19:43:38.669 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.669 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.670 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:43:38.670 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:35.484312+0000", 2023-09-23T19:43:38.670 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00012781199999999999 2023-09-23T19:43:38.670 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.670 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.670 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:43:38.670 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:35.484523+0000", 2023-09-23T19:43:38.670 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00021110599999999999 2023-09-23T19:43:38.670 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.670 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.670 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 1,2", 2023-09-23T19:43:38.670 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:35.484670+0000", 2023-09-23T19:43:38.671 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00014690300000000001 2023-09-23T19:43:38.671 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.671 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.671 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:43:38.671 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:35.492525+0000", 2023-09-23T19:43:38.671 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0078551160000000005 2023-09-23T19:43:38.671 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.671 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.671 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:43:38.671 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:35.492531+0000", 2023-09-23T19:43:38.671 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.1009999999999997e-06 2023-09-23T19:43:38.671 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.671 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.672 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:43:38.672 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:35.613498+0000", 2023-09-23T19:43:38.672 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.120967027 2023-09-23T19:43:38.672 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.672 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.672 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:43:38.672 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:35.613673+0000", 2023-09-23T19:43:38.672 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00017543300000000001 2023-09-23T19:43:38.672 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.672 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.672 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:43:38.672 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:35.613949+0000", 2023-09-23T19:43:38.673 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00027597400000000001 2023-09-23T19:43:38.673 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:43:38.673 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:43:38.673 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:43:38.673 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.673 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.673 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.36243.0:74063 11.6 11:643a7a6b:::benchmark_data_smithi016_160148_object74062:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e186)", 2023-09-23T19:43:38.673 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:42:35.495615+0000", 2023-09-23T19:43:38.673 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 63.042635936000003, 2023-09-23T19:43:38.673 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.119084572, 2023-09-23T19:43:38.673 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:43:38.673 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:43:38.673 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:43:38.674 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.36243", 2023-09-23T19:43:38.674 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1280038109", 2023-09-23T19:43:38.674 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 74063 2023-09-23T19:43:38.674 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.674 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:43:38.674 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.674 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:43:38.674 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:35.495615+0000", 2023-09-23T19:43:38.674 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:43:38.674 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.674 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.674 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:43:38.675 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:35.495615+0000", 2023-09-23T19:43:38.675 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:43:38.675 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.675 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.675 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:43:38.675 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:35.495613+0000", 2023-09-23T19:43:38.675 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-09-23T19:43:38.675 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.675 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.675 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:43:38.675 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:35.495634+0000", 2023-09-23T19:43:38.675 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.0995000000000001e-05 2023-09-23T19:43:38.675 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.676 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.676 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:43:38.676 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:35.495647+0000", 2023-09-23T19:43:38.676 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2574e-05 2023-09-23T19:43:38.676 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.676 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.676 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:43:38.676 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:35.495653+0000", 2023-09-23T19:43:38.676 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.3910000000000003e-06 2023-09-23T19:43:38.676 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.676 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.676 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:43:38.677 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:35.495702+0000", 2023-09-23T19:43:38.677 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.8118000000000002e-05 2023-09-23T19:43:38.677 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.677 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.677 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:43:38.677 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:35.495799+0000", 2023-09-23T19:43:38.677 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.7646999999999995e-05 2023-09-23T19:43:38.677 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.677 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.677 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 1,2", 2023-09-23T19:43:38.677 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:35.495898+0000", 2023-09-23T19:43:38.677 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.8913999999999995e-05 2023-09-23T19:43:38.677 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.677 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.678 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:43:38.678 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:35.498257+0000", 2023-09-23T19:43:38.678 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0023592299999999999 2023-09-23T19:43:38.678 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.678 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.678 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:43:38.678 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:35.589875+0000", 2023-09-23T19:43:38.678 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.091617686000000004 2023-09-23T19:43:38.678 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.678 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.678 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:43:38.678 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:35.614212+0000", 2023-09-23T19:43:38.679 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.024337467000000002 2023-09-23T19:43:38.679 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.679 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.679 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:43:38.679 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:35.614543+0000", 2023-09-23T19:43:38.679 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00033081499999999998 2023-09-23T19:43:38.679 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:43:38.679 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:43:38.679 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:43:38.679 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:35.614699+0000", 2023-09-23T19:43:38.679 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00015606600000000001 2023-09-23T19:43:38.679 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:43:38.679 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:43:38.679 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:43:38.680 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:43:38.680 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:43:38.680 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-09-23T19:43:38.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:38.680 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:38.681 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:43:38.781 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:38.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:38 smithi092 bash[58855]: audit 2023-09-23T19:43:37.543304+0000 mon.a (mon.0) 1378 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]': finished 2023-09-23T19:43:38.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:38 smithi092 bash[58855]: cluster 2023-09-23T19:43:37.543579+0000 mon.a (mon.0) 1379 : cluster 0 osdmap e192: 8 total, 8 up, 8 in 2023-09-23T19:43:38.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:38 smithi092 bash[58855]: cluster 2023-09-23T19:43:37.550058+0000 osd.6 (osd.6) 113 : cluster 3 failed to encode map e192 with expected crc 2023-09-23T19:43:38.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:38 smithi092 bash[58855]: cluster 2023-09-23T19:43:37.552043+0000 osd.7 (osd.7) 102 : cluster 3 failed to encode map e192 with expected crc 2023-09-23T19:43:38.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:38 smithi092 bash[58855]: cluster 2023-09-23T19:43:37.555112+0000 osd.5 (osd.5) 121 : cluster 3 failed to encode map e192 with expected crc 2023-09-23T19:43:38.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:38 smithi092 bash[58855]: cluster 2023-09-23T19:43:37.555206+0000 osd.7 (osd.7) 103 : cluster 3 failed to encode map e192 with expected crc 2023-09-23T19:43:38.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:38 smithi092 bash[58855]: cluster 2023-09-23T19:43:37.557121+0000 osd.5 (osd.5) 122 : cluster 3 failed to encode map e192 with expected crc 2023-09-23T19:43:38.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:38 smithi092 bash[58855]: cluster 2023-09-23T19:43:37.557169+0000 osd.7 (osd.7) 104 : cluster 3 failed to encode map e192 with expected crc 2023-09-23T19:43:38.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:38 smithi092 bash[58855]: cluster 2023-09-23T19:43:37.558277+0000 osd.6 (osd.6) 114 : cluster 3 failed to encode map e192 with expected crc 2023-09-23T19:43:38.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:38 smithi092 bash[58855]: cluster 2023-09-23T19:43:37.564224+0000 osd.7 (osd.7) 105 : cluster 3 failed to encode map e192 with expected crc 2023-09-23T19:43:38.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:38 smithi092 bash[58855]: cluster 2023-09-23T19:43:37.564584+0000 osd.4 (osd.4) 132 : cluster 3 failed to encode map e192 with expected crc 2023-09-23T19:43:38.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:38 smithi092 bash[58855]: audit 2023-09-23T19:43:37.824961+0000 mon.a (mon.0) 1380 : audit 0 from='client.? 172.21.15.16:0/3104293201' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:43:38.808 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:38.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:38.810 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:38.838 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:38 smithi016 bash[75004]: audit 2023-09-23T19:43:37.543304+0000 mon.a (mon.0) 1378 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]': finished 2023-09-23T19:43:38.838 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:38 smithi016 bash[75004]: cluster 2023-09-23T19:43:37.543579+0000 mon.a (mon.0) 1379 : cluster 0 osdmap e192: 8 total, 8 up, 8 in 2023-09-23T19:43:38.838 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:38 smithi016 bash[75004]: cluster 2023-09-23T19:43:37.550058+0000 osd.6 (osd.6) 113 : cluster 3 failed to encode map e192 with expected crc 2023-09-23T19:43:38.839 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:38 smithi016 bash[75004]: cluster 2023-09-23T19:43:37.552043+0000 osd.7 (osd.7) 102 : cluster 3 failed to encode map e192 with expected crc 2023-09-23T19:43:38.839 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:38 smithi016 bash[75004]: cluster 2023-09-23T19:43:37.555112+0000 osd.5 (osd.5) 121 : cluster 3 failed to encode map e192 with expected crc 2023-09-23T19:43:38.839 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:38 smithi016 bash[75004]: cluster 2023-09-23T19:43:37.555206+0000 osd.7 (osd.7) 103 : cluster 3 failed to encode map e192 with expected crc 2023-09-23T19:43:38.839 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:38 smithi016 bash[75004]: cluster 2023-09-23T19:43:37.557121+0000 osd.5 (osd.5) 122 : cluster 3 failed to encode map e192 with expected crc 2023-09-23T19:43:38.839 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:38 smithi016 bash[75004]: cluster 2023-09-23T19:43:37.557169+0000 osd.7 (osd.7) 104 : cluster 3 failed to encode map e192 with expected crc 2023-09-23T19:43:38.839 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:38 smithi016 bash[75004]: cluster 2023-09-23T19:43:37.558277+0000 osd.6 (osd.6) 114 : cluster 3 failed to encode map e192 with expected crc 2023-09-23T19:43:38.839 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:38 smithi016 bash[75004]: cluster 2023-09-23T19:43:37.564224+0000 osd.7 (osd.7) 105 : cluster 3 failed to encode map e192 with expected crc 2023-09-23T19:43:38.839 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:38 smithi016 bash[75004]: cluster 2023-09-23T19:43:37.564584+0000 osd.4 (osd.4) 132 : cluster 3 failed to encode map e192 with expected crc 2023-09-23T19:43:38.839 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:38 smithi016 bash[75004]: audit 2023-09-23T19:43:37.824961+0000 mon.a (mon.0) 1380 : audit 0 from='client.? 172.21.15.16:0/3104293201' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:43:38.839 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:38 smithi016 bash[77995]: audit 2023-09-23T19:43:37.543304+0000 mon.a (mon.0) 1378 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]': finished 2023-09-23T19:43:38.840 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:38 smithi016 bash[77995]: cluster 2023-09-23T19:43:37.543579+0000 mon.a (mon.0) 1379 : cluster 0 osdmap e192: 8 total, 8 up, 8 in 2023-09-23T19:43:38.840 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:38 smithi016 bash[77995]: cluster 2023-09-23T19:43:37.550058+0000 osd.6 (osd.6) 113 : cluster 3 failed to encode map e192 with expected crc 2023-09-23T19:43:38.840 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:38 smithi016 bash[77995]: cluster 2023-09-23T19:43:37.552043+0000 osd.7 (osd.7) 102 : cluster 3 failed to encode map e192 with expected crc 2023-09-23T19:43:38.840 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:38 smithi016 bash[77995]: cluster 2023-09-23T19:43:37.555112+0000 osd.5 (osd.5) 121 : cluster 3 failed to encode map e192 with expected crc 2023-09-23T19:43:38.840 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:38 smithi016 bash[77995]: cluster 2023-09-23T19:43:37.555206+0000 osd.7 (osd.7) 103 : cluster 3 failed to encode map e192 with expected crc 2023-09-23T19:43:38.840 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:38 smithi016 bash[77995]: cluster 2023-09-23T19:43:37.557121+0000 osd.5 (osd.5) 122 : cluster 3 failed to encode map e192 with expected crc 2023-09-23T19:43:38.840 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:38 smithi016 bash[77995]: cluster 2023-09-23T19:43:37.557169+0000 osd.7 (osd.7) 104 : cluster 3 failed to encode map e192 with expected crc 2023-09-23T19:43:38.840 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:38 smithi016 bash[77995]: cluster 2023-09-23T19:43:37.558277+0000 osd.6 (osd.6) 114 : cluster 3 failed to encode map e192 with expected crc 2023-09-23T19:43:38.840 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:38 smithi016 bash[77995]: cluster 2023-09-23T19:43:37.564224+0000 osd.7 (osd.7) 105 : cluster 3 failed to encode map e192 with expected crc 2023-09-23T19:43:38.840 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:38 smithi016 bash[77995]: cluster 2023-09-23T19:43:37.564584+0000 osd.4 (osd.4) 132 : cluster 3 failed to encode map e192 with expected crc 2023-09-23T19:43:38.840 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:38 smithi016 bash[77995]: audit 2023-09-23T19:43:37.824961+0000 mon.a (mon.0) 1380 : audit 0 from='client.? 172.21.15.16:0/3104293201' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:43:38.910 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:38.942 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:38.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:38.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:43:39.049 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:39.083 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:39.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:39.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:39.189 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:39.210 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:39.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:39.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:39.316 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:39.341 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:39.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:39.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:39.445 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:39.473 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:39.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:39.480 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:43:39.581 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:39.627 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:39.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:39.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:39.718 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:39 smithi016 bash[77995]: cluster 2023-09-23T19:43:38.249348+0000 mgr.y (mgr.25325) 1086 : cluster 0 pgmap v1047: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2023-09-23T19:43:39.718 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:39 smithi016 bash[77995]: cluster 2023-09-23T19:43:38.551866+0000 mon.a (mon.0) 1381 : cluster 0 osdmap e193: 8 total, 8 up, 8 in 2023-09-23T19:43:39.718 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:39 smithi016 bash[77995]: cluster 2023-09-23T19:43:38.557720+0000 osd.6 (osd.6) 115 : cluster 3 failed to encode map e193 with expected crc 2023-09-23T19:43:39.718 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:39 smithi016 bash[77995]: cluster 2023-09-23T19:43:38.558251+0000 osd.7 (osd.7) 106 : cluster 3 failed to encode map e193 with expected crc 2023-09-23T19:43:39.718 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:39 smithi016 bash[77995]: cluster 2023-09-23T19:43:38.558658+0000 osd.4 (osd.4) 133 : cluster 3 failed to encode map e193 with expected crc 2023-09-23T19:43:39.718 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:39 smithi016 bash[77995]: cluster 2023-09-23T19:43:38.561051+0000 osd.6 (osd.6) 116 : cluster 3 failed to encode map e193 with expected crc 2023-09-23T19:43:39.718 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:39 smithi016 bash[77995]: cluster 2023-09-23T19:43:38.562938+0000 osd.6 (osd.6) 117 : cluster 3 failed to encode map e193 with expected crc 2023-09-23T19:43:39.719 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:39 smithi016 bash[77995]: cluster 2023-09-23T19:43:38.563484+0000 osd.7 (osd.7) 107 : cluster 3 failed to encode map e193 with expected crc 2023-09-23T19:43:39.719 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:39 smithi016 bash[77995]: cluster 2023-09-23T19:43:38.563715+0000 osd.4 (osd.4) 134 : cluster 3 failed to encode map e193 with expected crc 2023-09-23T19:43:39.719 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:39 smithi016 bash[77995]: cluster 2023-09-23T19:43:38.565195+0000 osd.7 (osd.7) 108 : cluster 3 failed to encode map e193 with expected crc 2023-09-23T19:43:39.719 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:39 smithi016 bash[75004]: cluster 2023-09-23T19:43:38.249348+0000 mgr.y (mgr.25325) 1086 : cluster 0 pgmap v1047: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2023-09-23T19:43:39.719 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:39 smithi016 bash[75004]: cluster 2023-09-23T19:43:38.551866+0000 mon.a (mon.0) 1381 : cluster 0 osdmap e193: 8 total, 8 up, 8 in 2023-09-23T19:43:39.719 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:39 smithi016 bash[75004]: cluster 2023-09-23T19:43:38.557720+0000 osd.6 (osd.6) 115 : cluster 3 failed to encode map e193 with expected crc 2023-09-23T19:43:39.719 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:39 smithi016 bash[75004]: cluster 2023-09-23T19:43:38.558251+0000 osd.7 (osd.7) 106 : cluster 3 failed to encode map e193 with expected crc 2023-09-23T19:43:39.719 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:39 smithi016 bash[75004]: cluster 2023-09-23T19:43:38.558658+0000 osd.4 (osd.4) 133 : cluster 3 failed to encode map e193 with expected crc 2023-09-23T19:43:39.719 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:39 smithi016 bash[75004]: cluster 2023-09-23T19:43:38.561051+0000 osd.6 (osd.6) 116 : cluster 3 failed to encode map e193 with expected crc 2023-09-23T19:43:39.719 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:39 smithi016 bash[75004]: cluster 2023-09-23T19:43:38.562938+0000 osd.6 (osd.6) 117 : cluster 3 failed to encode map e193 with expected crc 2023-09-23T19:43:39.719 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:39 smithi016 bash[75004]: cluster 2023-09-23T19:43:38.563484+0000 osd.7 (osd.7) 107 : cluster 3 failed to encode map e193 with expected crc 2023-09-23T19:43:39.719 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:39 smithi016 bash[75004]: cluster 2023-09-23T19:43:38.563715+0000 osd.4 (osd.4) 134 : cluster 3 failed to encode map e193 with expected crc 2023-09-23T19:43:39.720 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:39 smithi016 bash[75004]: cluster 2023-09-23T19:43:38.565195+0000 osd.7 (osd.7) 108 : cluster 3 failed to encode map e193 with expected crc 2023-09-23T19:43:39.730 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:39.749 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:39.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:39.753 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:39.854 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:39.863 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:39 smithi092 bash[58855]: cluster 2023-09-23T19:43:38.249348+0000 mgr.y (mgr.25325) 1086 : cluster 0 pgmap v1047: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2023-09-23T19:43:39.863 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:39 smithi092 bash[58855]: cluster 2023-09-23T19:43:38.551866+0000 mon.a (mon.0) 1381 : cluster 0 osdmap e193: 8 total, 8 up, 8 in 2023-09-23T19:43:39.863 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:39 smithi092 bash[58855]: cluster 2023-09-23T19:43:38.557720+0000 osd.6 (osd.6) 115 : cluster 3 failed to encode map e193 with expected crc 2023-09-23T19:43:39.863 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:39 smithi092 bash[58855]: cluster 2023-09-23T19:43:38.558251+0000 osd.7 (osd.7) 106 : cluster 3 failed to encode map e193 with expected crc 2023-09-23T19:43:39.863 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:39 smithi092 bash[58855]: cluster 2023-09-23T19:43:38.558658+0000 osd.4 (osd.4) 133 : cluster 3 failed to encode map e193 with expected crc 2023-09-23T19:43:39.863 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:39 smithi092 bash[58855]: cluster 2023-09-23T19:43:38.561051+0000 osd.6 (osd.6) 116 : cluster 3 failed to encode map e193 with expected crc 2023-09-23T19:43:39.863 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:39 smithi092 bash[58855]: cluster 2023-09-23T19:43:38.562938+0000 osd.6 (osd.6) 117 : cluster 3 failed to encode map e193 with expected crc 2023-09-23T19:43:39.863 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:39 smithi092 bash[58855]: cluster 2023-09-23T19:43:38.563484+0000 osd.7 (osd.7) 107 : cluster 3 failed to encode map e193 with expected crc 2023-09-23T19:43:39.863 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:39 smithi092 bash[58855]: cluster 2023-09-23T19:43:38.563715+0000 osd.4 (osd.4) 134 : cluster 3 failed to encode map e193 with expected crc 2023-09-23T19:43:39.863 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:39 smithi092 bash[58855]: cluster 2023-09-23T19:43:38.565195+0000 osd.7 (osd.7) 108 : cluster 3 failed to encode map e193 with expected crc 2023-09-23T19:43:39.887 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:39.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:39.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:39.992 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:40.021 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:40.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:40.024 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:40.125 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:40.157 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:40.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:40.161 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:40.262 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:40.289 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:40.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:40.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:43:40.392 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:40.421 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:40.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:40.425 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:40.526 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:40.558 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:40.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:40.566 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:40.667 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:40.700 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:40.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:40.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:40.804 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:40.837 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:40.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:40.840 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:43:40.941 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:40.967 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:40.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:40.971 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:41.072 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:41.094 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:41.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:41.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:41.199 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:41.222 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:41.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:41.225 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:41.327 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:41.352 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:41.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:41.359 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:43:41.421 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T19:43:41.460 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:41.488 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:41.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:41.490 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:41.591 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:41.626 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:41.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:41.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:41.734 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:41.746 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:41 smithi092 bash[58855]: cluster 2023-09-23T19:43:40.249917+0000 mgr.y (mgr.25325) 1087 : cluster 0 pgmap v1049: 89 pgs: 89 active+clean; 5.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 674 op/s 2023-09-23T19:43:41.766 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:41.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:41.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:41.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:41 smithi016 bash[75004]: cluster 2023-09-23T19:43:40.249917+0000 mgr.y (mgr.25325) 1087 : cluster 0 pgmap v1049: 89 pgs: 89 active+clean; 5.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 674 op/s 2023-09-23T19:43:41.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:41 smithi016 bash[77995]: cluster 2023-09-23T19:43:40.249917+0000 mgr.y (mgr.25325) 1087 : cluster 0 pgmap v1049: 89 pgs: 89 active+clean; 5.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 674 op/s 2023-09-23T19:43:41.869 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:41.904 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:41.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:41.912 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:43:42.012 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:42.038 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:42.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:42.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:43:42.141 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:42.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:42.184 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:42.185 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:43:42.285 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:42.315 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:42.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:42.318 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:43:42.418 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:42.442 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:42.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:42.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:42.544 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:42.573 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:42.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:42.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:42.677 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:42.716 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:42.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:42.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:43:42.822 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:42.854 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:42.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:42.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:42.957 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:42.993 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:42.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:43.000 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:43:43.100 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:43.132 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:43.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:43.136 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:43.237 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:43.262 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:43.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:43.267 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:43.368 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:43.399 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:43:43.410 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:43.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:43.411 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:43.512 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:43.578 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:43.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:43.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:43.591 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is set 2023-09-23T19:43:43.681 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:43.715 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:43.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:43.722 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:43.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:43 smithi016 bash[75004]: cluster 2023-09-23T19:43:42.250578+0000 mgr.y (mgr.25325) 1088 : cluster 0 pgmap v1050: 89 pgs: 89 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 776 op/s 2023-09-23T19:43:43.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:43 smithi016 bash[75004]: audit 2023-09-23T19:43:43.356907+0000 mon.c (mon.2) 188 : audit 1 from='client.? 172.21.15.16:0/735593783' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:43:43.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:43 smithi016 bash[75004]: audit 2023-09-23T19:43:43.357848+0000 mon.a (mon.0) 1382 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:43:43.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:43 smithi016 bash[77995]: cluster 2023-09-23T19:43:42.250578+0000 mgr.y (mgr.25325) 1088 : cluster 0 pgmap v1050: 89 pgs: 89 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 776 op/s 2023-09-23T19:43:43.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:43 smithi016 bash[77995]: audit 2023-09-23T19:43:43.356907+0000 mon.c (mon.2) 188 : audit 1 from='client.? 172.21.15.16:0/735593783' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:43:43.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:43 smithi016 bash[77995]: audit 2023-09-23T19:43:43.357848+0000 mon.a (mon.0) 1382 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:43:43.823 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:43.832 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:43 smithi092 bash[58855]: cluster 2023-09-23T19:43:42.250578+0000 mgr.y (mgr.25325) 1088 : cluster 0 pgmap v1050: 89 pgs: 89 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 776 op/s 2023-09-23T19:43:43.832 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:43 smithi092 bash[58855]: audit 2023-09-23T19:43:43.356907+0000 mon.c (mon.2) 188 : audit 1 from='client.? 172.21.15.16:0/735593783' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:43:43.832 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:43 smithi092 bash[58855]: audit 2023-09-23T19:43:43.357848+0000 mon.a (mon.0) 1382 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:43:43.851 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:43.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:43.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:43.955 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:43.979 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:43.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:43.982 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:43:44.083 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:44.115 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:44.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:44.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:43:44.223 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:44.271 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:44.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:44.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:44.377 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:44.404 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:44.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:44.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:43:44.509 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:44.538 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:44.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:44.544 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:44.645 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:44.656 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:44 smithi016 bash[75004]: cluster 2023-09-23T19:43:43.567697+0000 mon.a (mon.0) 1383 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:43:44.656 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:44 smithi016 bash[75004]: audit 2023-09-23T19:43:43.574030+0000 mon.a (mon.0) 1384 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:43:44.656 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:44 smithi016 bash[75004]: cluster 2023-09-23T19:43:43.574196+0000 mon.a (mon.0) 1385 : cluster 0 osdmap e194: 8 total, 8 up, 8 in 2023-09-23T19:43:44.656 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:44 smithi016 bash[75004]: cluster 2023-09-23T19:43:43.579884+0000 osd.7 (osd.7) 109 : cluster 3 failed to encode map e194 with expected crc 2023-09-23T19:43:44.656 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:44 smithi016 bash[75004]: cluster 2023-09-23T19:43:43.582616+0000 osd.6 (osd.6) 118 : cluster 3 failed to encode map e194 with expected crc 2023-09-23T19:43:44.656 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:44 smithi016 bash[75004]: cluster 2023-09-23T19:43:43.585073+0000 osd.6 (osd.6) 119 : cluster 3 failed to encode map e194 with expected crc 2023-09-23T19:43:44.656 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:44 smithi016 bash[75004]: cluster 2023-09-23T19:43:43.585892+0000 osd.4 (osd.4) 135 : cluster 3 failed to encode map e194 with expected crc 2023-09-23T19:43:44.657 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:44 smithi016 bash[75004]: cluster 2023-09-23T19:43:43.588741+0000 osd.5 (osd.5) 123 : cluster 3 failed to encode map e194 with expected crc 2023-09-23T19:43:44.657 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:44 smithi016 bash[75004]: audit 2023-09-23T19:43:43.673191+0000 mon.a (mon.0) 1386 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:43:44.657 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:44 smithi016 bash[75004]: audit 2023-09-23T19:43:43.674877+0000 mon.a (mon.0) 1387 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:43:44.657 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:44 smithi016 bash[77995]: cluster 2023-09-23T19:43:43.567697+0000 mon.a (mon.0) 1383 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:43:44.657 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:44 smithi016 bash[77995]: audit 2023-09-23T19:43:43.574030+0000 mon.a (mon.0) 1384 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:43:44.657 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:44 smithi016 bash[77995]: cluster 2023-09-23T19:43:43.574196+0000 mon.a (mon.0) 1385 : cluster 0 osdmap e194: 8 total, 8 up, 8 in 2023-09-23T19:43:44.657 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:44 smithi016 bash[77995]: cluster 2023-09-23T19:43:43.579884+0000 osd.7 (osd.7) 109 : cluster 3 failed to encode map e194 with expected crc 2023-09-23T19:43:44.657 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:44 smithi016 bash[77995]: cluster 2023-09-23T19:43:43.582616+0000 osd.6 (osd.6) 118 : cluster 3 failed to encode map e194 with expected crc 2023-09-23T19:43:44.657 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:44 smithi016 bash[77995]: cluster 2023-09-23T19:43:43.585073+0000 osd.6 (osd.6) 119 : cluster 3 failed to encode map e194 with expected crc 2023-09-23T19:43:44.657 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:44 smithi016 bash[77995]: cluster 2023-09-23T19:43:43.585892+0000 osd.4 (osd.4) 135 : cluster 3 failed to encode map e194 with expected crc 2023-09-23T19:43:44.658 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:44 smithi016 bash[77995]: cluster 2023-09-23T19:43:43.588741+0000 osd.5 (osd.5) 123 : cluster 3 failed to encode map e194 with expected crc 2023-09-23T19:43:44.658 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:44 smithi016 bash[77995]: audit 2023-09-23T19:43:43.673191+0000 mon.a (mon.0) 1386 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:43:44.658 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:44 smithi016 bash[77995]: audit 2023-09-23T19:43:43.674877+0000 mon.a (mon.0) 1387 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:43:44.673 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:44.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:44.678 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:43:44.778 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:44.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:44 smithi092 bash[58855]: cluster 2023-09-23T19:43:43.567697+0000 mon.a (mon.0) 1383 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:43:44.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:44 smithi092 bash[58855]: audit 2023-09-23T19:43:43.574030+0000 mon.a (mon.0) 1384 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:43:44.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:44 smithi092 bash[58855]: cluster 2023-09-23T19:43:43.574196+0000 mon.a (mon.0) 1385 : cluster 0 osdmap e194: 8 total, 8 up, 8 in 2023-09-23T19:43:44.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:44 smithi092 bash[58855]: cluster 2023-09-23T19:43:43.579884+0000 osd.7 (osd.7) 109 : cluster 3 failed to encode map e194 with expected crc 2023-09-23T19:43:44.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:44 smithi092 bash[58855]: cluster 2023-09-23T19:43:43.582616+0000 osd.6 (osd.6) 118 : cluster 3 failed to encode map e194 with expected crc 2023-09-23T19:43:44.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:44 smithi092 bash[58855]: cluster 2023-09-23T19:43:43.585073+0000 osd.6 (osd.6) 119 : cluster 3 failed to encode map e194 with expected crc 2023-09-23T19:43:44.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:44 smithi092 bash[58855]: cluster 2023-09-23T19:43:43.585892+0000 osd.4 (osd.4) 135 : cluster 3 failed to encode map e194 with expected crc 2023-09-23T19:43:44.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:44 smithi092 bash[58855]: cluster 2023-09-23T19:43:43.588741+0000 osd.5 (osd.5) 123 : cluster 3 failed to encode map e194 with expected crc 2023-09-23T19:43:44.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:44 smithi092 bash[58855]: audit 2023-09-23T19:43:43.673191+0000 mon.a (mon.0) 1386 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:43:44.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:44 smithi092 bash[58855]: audit 2023-09-23T19:43:43.674877+0000 mon.a (mon.0) 1387 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:43:44.807 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:44.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:44.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:44.910 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:44.946 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:44.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:44.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:43:45.050 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:45.079 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:45.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:45.082 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:43:45.183 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:45.219 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:45.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:45.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:45.328 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:45.358 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:45.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:45.365 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:43:45.465 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:45.495 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:45.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:45.501 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:43:45.601 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:45.641 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:45.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:45.645 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:43:45.745 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:45.775 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:45.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:45.780 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:43:45.881 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:45.908 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:45.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:45.912 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:43:46.013 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:46.021 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:45 smithi092 bash[58855]: cluster 2023-09-23T19:43:44.251024+0000 mgr.y (mgr.25325) 1089 : cluster 0 pgmap v1052: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 468 op/s 2023-09-23T19:43:46.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:45 smithi016 bash[77995]: cluster 2023-09-23T19:43:44.251024+0000 mgr.y (mgr.25325) 1089 : cluster 0 pgmap v1052: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 468 op/s 2023-09-23T19:43:46.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:45 smithi016 bash[75004]: cluster 2023-09-23T19:43:44.251024+0000 mgr.y (mgr.25325) 1089 : cluster 0 pgmap v1052: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 468 op/s 2023-09-23T19:43:46.036 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:43:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:43:45] "GET /metrics HTTP/1.1" 200 36306 "" "Prometheus/2.33.4" 2023-09-23T19:43:46.039 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:46.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:46.043 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:46.144 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:46.162 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-09-23T19:43:46.162 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T19:43:46.162 INFO:tasks.thrashosds.thrasher:inject_pause on osd.2 2023-09-23T19:43:46.162 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T19:43:46.162 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T19:43:46.162 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 config set filestore_inject_stall 3 2023-09-23T19:43:46.172 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:46.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:46.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:46.276 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:46.308 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:46.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:46.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:43:46.425 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:46.455 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:46.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:46.460 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:46.561 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:46.591 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:46.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:46.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:46.695 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:46.722 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:46.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:46.725 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:43:46.826 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:46.852 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:46.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:46.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:46.902 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T19:43:46.956 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:46.981 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:46.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:46.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:43:47.085 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:47.118 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:47.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:47.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:47.222 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:47.246 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:47.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:47.248 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:47.349 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:47.379 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:47.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:47.384 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:47.422 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:43:47.484 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:47.512 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:47.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:47.519 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:43:47.619 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:47.656 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:47.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:47.661 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:43:47.762 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:47.771 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:47 smithi016 bash[75004]: cluster 2023-09-23T19:43:46.255161+0000 mgr.y (mgr.25325) 1090 : cluster 0 pgmap v1053: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 769 op/s 2023-09-23T19:43:47.771 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:47 smithi016 bash[77995]: cluster 2023-09-23T19:43:46.255161+0000 mgr.y (mgr.25325) 1090 : cluster 0 pgmap v1053: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 769 op/s 2023-09-23T19:43:47.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:47 smithi092 bash[58855]: cluster 2023-09-23T19:43:46.255161+0000 mgr.y (mgr.25325) 1090 : cluster 0 pgmap v1053: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 769 op/s 2023-09-23T19:43:47.792 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:47.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:47.795 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:43:47.895 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:47.920 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:47.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:47.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:48.025 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:48.065 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:48.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:48.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:48.169 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:48.195 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:48.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:48.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:43:48.298 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:48.322 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:48.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:48.324 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:43:48.425 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:48.439 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:43:48.464 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:48.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:48.467 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:48.567 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:48.592 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:48.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:48.600 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:48.701 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:48.745 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:48.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:48.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:48.851 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:48.881 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:48.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:48.887 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:43:48.988 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:49.021 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:49.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:49.026 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:49.127 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:49.153 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:49.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:49.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:43:49.262 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:49.297 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:49.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:49.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:43:49.400 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:49.433 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:49.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:49.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:49.537 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:49.573 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:49.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:49.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:43:49.676 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:49.718 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:49.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:49.721 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:43:49.821 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:49.830 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:49 smithi016 bash[75004]: cluster 2023-09-23T19:43:48.255647+0000 mgr.y (mgr.25325) 1091 : cluster 0 pgmap v1054: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 702 op/s 2023-09-23T19:43:49.830 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:49 smithi016 bash[77995]: cluster 2023-09-23T19:43:48.255647+0000 mgr.y (mgr.25325) 1091 : cluster 0 pgmap v1054: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 702 op/s 2023-09-23T19:43:49.846 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:49.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:49.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:43:49.949 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:49.983 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:49.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:49.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:50.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:49 smithi092 bash[58855]: cluster 2023-09-23T19:43:48.255647+0000 mgr.y (mgr.25325) 1091 : cluster 0 pgmap v1054: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 702 op/s 2023-09-23T19:43:50.088 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:50.129 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:50.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:50.134 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:43:50.235 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:50.265 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:50.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:50.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:43:50.371 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:50.399 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:50.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:50.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:50.477 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:43:50.502 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:50.539 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:50.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:50.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:43:50.649 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:50.693 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:50.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:50.696 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:50.797 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:50.832 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:50.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:50.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:43:50.935 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:50.969 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:50.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:50.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:51.073 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:51.113 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:51.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:51.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:51.219 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:51.250 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:51.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:51.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:51.355 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:51.392 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:51.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:51.396 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:51.495 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:43:51.497 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:51.525 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:51.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:51.533 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:51.633 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:51.673 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:51.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:51.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:43:51.780 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:51.806 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:51.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:51.808 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:43:51.909 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:51.918 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:51 smithi016 bash[75004]: cluster 2023-09-23T19:43:50.256443+0000 mgr.y (mgr.25325) 1092 : cluster 0 pgmap v1055: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2023-09-23T19:43:51.918 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:51 smithi016 bash[77995]: cluster 2023-09-23T19:43:50.256443+0000 mgr.y (mgr.25325) 1092 : cluster 0 pgmap v1055: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2023-09-23T19:43:51.939 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:51.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:51.941 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:43:52.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:51 smithi092 bash[58855]: cluster 2023-09-23T19:43:50.256443+0000 mgr.y (mgr.25325) 1092 : cluster 0 pgmap v1055: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2023-09-23T19:43:52.042 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:52.072 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:52.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:52.074 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:52.175 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:52.203 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:52.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:52.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:52.309 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:52.337 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:52.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:52.341 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:43:52.442 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:52.473 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:52.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:52.476 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:52.576 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:52.608 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:52.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:52.618 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:52.718 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:52.740 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:52.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:52.744 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:43:52.845 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:52.878 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:52.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:52.883 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:43:52.985 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:53.016 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:53.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:53.020 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:43:53.120 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:53.149 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:53.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:53.155 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:53.255 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:53.282 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:53.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:53.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:53.388 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:53.422 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:53.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:53.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:43:53.526 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:53.562 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:53.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:53.564 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:43:53.665 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:53.695 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:53.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:53.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:53.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:53 smithi092 bash[58855]: cluster 2023-09-23T19:43:52.256964+0000 mgr.y (mgr.25325) 1093 : cluster 0 pgmap v1056: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2023-09-23T19:43:53.801 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:53.826 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:53.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:53.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:43:53.929 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:53.959 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:53.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:53.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:54.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:53 smithi016 bash[75004]: cluster 2023-09-23T19:43:52.256964+0000 mgr.y (mgr.25325) 1093 : cluster 0 pgmap v1056: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2023-09-23T19:43:54.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:53 smithi016 bash[77995]: cluster 2023-09-23T19:43:52.256964+0000 mgr.y (mgr.25325) 1093 : cluster 0 pgmap v1056: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2023-09-23T19:43:54.064 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:54.096 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:54.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:54.100 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:54.201 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:54.228 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:54.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:54.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:54.336 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:54.353 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:54.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:54.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:43:54.456 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:54.488 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:54.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:54.496 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:43:54.596 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:54.632 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:54.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:54.634 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:54.735 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:54.758 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:54.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:54.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:54.861 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:54.892 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:54.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:54.894 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:43:54.995 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:55.028 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:55.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:55.032 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:55.133 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:55.159 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:55.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:55.161 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:43:55.261 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:55.288 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:55.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:55.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:43:55.397 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:55.422 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:55.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:55.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:43:55.526 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:55.550 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:55.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:55.553 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:55.653 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:55.662 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:43:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:43:55] "GET /metrics HTTP/1.1" 200 36306 "" "Prometheus/2.33.4" 2023-09-23T19:43:55.690 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:55.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:55.693 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:43:55.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:55 smithi092 bash[58855]: cluster 2023-09-23T19:43:54.257572+0000 mgr.y (mgr.25325) 1094 : cluster 0 pgmap v1057: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2023-09-23T19:43:55.793 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:55.821 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:55.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:55.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:43:55.924 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:55.930 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:55 smithi016 bash[77995]: cluster 2023-09-23T19:43:54.257572+0000 mgr.y (mgr.25325) 1094 : cluster 0 pgmap v1057: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2023-09-23T19:43:55.930 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:55 smithi016 bash[75004]: cluster 2023-09-23T19:43:54.257572+0000 mgr.y (mgr.25325) 1094 : cluster 0 pgmap v1057: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2023-09-23T19:43:55.969 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:55.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:55.972 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:43:56.073 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:56.096 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:56.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:56.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:56.204 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:56.232 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:56.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:56.237 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:56.338 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:56.370 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:56.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:56.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:43:56.474 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:56.508 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:56.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:56.511 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:43:56.612 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:56.640 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:56.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:56.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:43:56.744 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:56.791 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:56.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:56.797 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:43:56.898 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:56.925 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:56.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:56.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:43:57.032 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:57.057 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:57.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:57.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:57.160 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:57.191 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:57.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:57.196 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:57.297 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:57.334 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:57.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:57.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:43:57.437 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:57.473 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:57.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:57.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:57.577 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:57.615 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:57.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:57.621 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:57.712 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:57 smithi016 bash[75004]: cluster 2023-09-23T19:43:56.259089+0000 mgr.y (mgr.25325) 1095 : cluster 0 pgmap v1058: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 842 op/s 2023-09-23T19:43:57.722 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:57.747 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:43:57.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:57.752 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:43:57.852 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:57.886 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:57.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:57.891 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:43:57.991 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:58.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:57 smithi016 bash[77995]: cluster 2023-09-23T19:43:56.259089+0000 mgr.y (mgr.25325) 1095 : cluster 0 pgmap v1058: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 842 op/s 2023-09-23T19:43:58.020 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:58.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:58.026 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:43:58.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:57 smithi092 bash[58855]: cluster 2023-09-23T19:43:56.259089+0000 mgr.y (mgr.25325) 1095 : cluster 0 pgmap v1058: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 842 op/s 2023-09-23T19:43:58.128 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:58.153 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:58.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:58.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:58.256 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:58.281 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:58.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:58.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:43:58.387 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:58.425 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:58.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:58.430 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:58.533 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:58.561 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:58.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:58.564 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:58.665 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:58.694 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:58.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:58.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:43:58.798 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:58.820 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:58.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:58.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:58.923 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:58.933 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:58 smithi016 bash[75004]: audit 2023-09-23T19:43:58.662727+0000 mon.a (mon.0) 1388 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:43:58.934 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:58 smithi016 bash[77995]: audit 2023-09-23T19:43:58.662727+0000 mon.a (mon.0) 1388 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:43:58.946 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:43:58.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:58.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:43:59.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:58 smithi092 bash[58855]: audit 2023-09-23T19:43:58.662727+0000 mon.a (mon.0) 1388 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:43:59.049 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:59.083 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:43:59.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:59.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:43:59.189 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:59.211 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:43:59.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:59.217 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:43:59.318 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:59.363 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:59.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:59.369 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:43:59.470 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:59.511 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:43:59.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:59.516 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:43:59.617 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:59.641 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:43:59.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:59.643 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:43:59.733 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:43:59 smithi016 bash[77995]: cluster 2023-09-23T19:43:58.259699+0000 mgr.y (mgr.25325) 1096 : cluster 0 pgmap v1059: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 646 op/s 2023-09-23T19:43:59.733 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:43:59 smithi016 bash[75004]: cluster 2023-09-23T19:43:58.259699+0000 mgr.y (mgr.25325) 1096 : cluster 0 pgmap v1059: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 646 op/s 2023-09-23T19:43:59.745 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:59.767 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:43:59.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:59.774 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:43:59.875 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:59.918 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:43:59.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:43:59.920 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:44:00.021 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:00.030 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:43:59 smithi092 bash[58855]: cluster 2023-09-23T19:43:58.259699+0000 mgr.y (mgr.25325) 1096 : cluster 0 pgmap v1059: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 646 op/s 2023-09-23T19:44:00.063 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:00.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:00.069 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:44:00.170 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:00.201 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:00.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:00.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:44:00.305 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:00.339 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:00.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:00.346 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:44:00.447 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:00.472 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:00.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:00.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:44:00.580 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:00.611 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:00.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:00.613 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:44:00.714 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:00.760 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:00.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:00.762 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:44:00.863 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:00.896 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:00.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:00.902 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:44:01.003 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:01.039 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:01.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:01.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:44:01.143 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:01.190 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:01.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:01.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:44:01.295 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:01.318 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:01.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:01.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:01.422 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:01.447 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:01.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:01.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:44:01.550 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:01.580 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:01.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:01.583 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:44:01.684 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:01.708 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:01.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:01.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:44:01.813 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:01.841 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:01.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:01.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:44:01.879 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-09-23T19:44:01.879 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [ 2023-09-23T19:44:01.879 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.879 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.36243.0:142277 11.c 11:3c31ed47:::benchmark_data_smithi016_160148_object35074:head [delete] snapc 0=[] ondisk+write+known_if_redirected e194)", 2023-09-23T19:44:01.880 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:44:01.860389+0000", 2023-09-23T19:44:01.880 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 0.014263002, 2023-09-23T19:44:01.880 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.014307136999999999, 2023-09-23T19:44:01.880 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:44:01.880 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "waiting for sub ops", 2023-09-23T19:44:01.880 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:44:01.880 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.36243", 2023-09-23T19:44:01.880 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1280038109", 2023-09-23T19:44:01.880 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 142277 2023-09-23T19:44:01.880 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.880 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:44:01.881 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.881 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:44:01.881 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:01.860389+0000", 2023-09-23T19:44:01.881 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:44:01.881 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.881 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.881 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:44:01.881 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:01.860389+0000", 2023-09-23T19:44:01.881 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:44:01.881 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.881 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.881 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:44:01.882 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:01.860387+0000", 2023-09-23T19:44:01.882 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-09-23T19:44:01.882 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.882 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.882 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:44:01.882 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:01.860395+0000", 2023-09-23T19:44:01.882 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.25e-06 2023-09-23T19:44:01.882 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.882 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.882 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:44:01.882 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:01.860407+0000", 2023-09-23T19:44:01.883 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2228999999999999e-05 2023-09-23T19:44:01.883 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.883 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.883 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:44:01.883 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:01.860415+0000", 2023-09-23T19:44:01.883 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.6499999999999996e-06 2023-09-23T19:44:01.883 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.883 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.883 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:44:01.883 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:01.860484+0000", 2023-09-23T19:44:01.883 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.9187e-05 2023-09-23T19:44:01.883 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.884 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.884 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:44:01.884 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:01.860625+0000", 2023-09-23T19:44:01.884 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000141135 2023-09-23T19:44:01.884 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.884 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.884 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 2,4", 2023-09-23T19:44:01.884 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:01.860761+0000", 2023-09-23T19:44:01.884 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00013613199999999999 2023-09-23T19:44:01.884 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.884 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.884 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:44:01.884 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:01.869372+0000", 2023-09-23T19:44:01.885 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0086113660000000005 2023-09-23T19:44:01.885 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:44:01.885 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:44:01.885 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:44:01.885 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.885 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.885 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.36243.0:142278 11.c 11:347c966e:::benchmark_data_smithi016_160148_object35075:head [delete] snapc 0=[] ondisk+write+known_if_redirected e194)", 2023-09-23T19:44:01.885 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:44:01.863144+0000", 2023-09-23T19:44:01.885 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 0.011507880999999999, 2023-09-23T19:44:01.885 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.011648036000000001, 2023-09-23T19:44:01.885 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:44:01.885 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "waiting for sub ops", 2023-09-23T19:44:01.886 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:44:01.886 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.36243", 2023-09-23T19:44:01.886 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1280038109", 2023-09-23T19:44:01.886 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 142278 2023-09-23T19:44:01.886 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.886 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:44:01.886 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.886 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:44:01.886 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:01.863144+0000", 2023-09-23T19:44:01.886 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:44:01.886 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.886 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.887 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:44:01.887 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:01.863144+0000", 2023-09-23T19:44:01.887 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:44:01.887 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.887 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.887 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:44:01.887 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:01.863143+0000", 2023-09-23T19:44:01.887 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:44:01.887 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.887 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.887 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:44:01.887 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:01.863149+0000", 2023-09-23T19:44:01.887 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.677e-06 2023-09-23T19:44:01.888 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.888 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.888 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:44:01.888 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:01.863159+0000", 2023-09-23T19:44:01.888 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.082e-05 2023-09-23T19:44:01.888 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.888 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.888 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:44:01.888 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:01.863165+0000", 2023-09-23T19:44:01.888 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.8649999999999998e-06 2023-09-23T19:44:01.888 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.888 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.889 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:44:01.889 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:01.864504+0000", 2023-09-23T19:44:01.889 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0013389140000000001 2023-09-23T19:44:01.889 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.889 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.889 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:44:01.889 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:01.864616+0000", 2023-09-23T19:44:01.889 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000112241 2023-09-23T19:44:01.889 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.889 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.889 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 2,4", 2023-09-23T19:44:01.889 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:01.864725+0000", 2023-09-23T19:44:01.890 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00010825800000000001 2023-09-23T19:44:01.890 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.890 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.890 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:44:01.890 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:01.871139+0000", 2023-09-23T19:44:01.890 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0064139330000000001 2023-09-23T19:44:01.890 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:44:01.890 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:44:01.890 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:44:01.890 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.890 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.891 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.36243.0:142279 11.0 11:0e6e211f:::benchmark_data_smithi016_160148_object35076:head [delete] snapc 0=[] ondisk+write+known_if_redirected e194)", 2023-09-23T19:44:01.891 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:44:01.863491+0000", 2023-09-23T19:44:01.891 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 0.011161132000000001, 2023-09-23T19:44:01.891 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.011391754, 2023-09-23T19:44:01.891 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:44:01.891 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "waiting for sub ops", 2023-09-23T19:44:01.891 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:44:01.891 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.36243", 2023-09-23T19:44:01.891 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1280038109", 2023-09-23T19:44:01.891 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 142279 2023-09-23T19:44:01.891 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.891 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:44:01.892 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.892 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:44:01.892 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:01.863491+0000", 2023-09-23T19:44:01.892 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:44:01.892 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.892 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.892 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:44:01.892 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:01.863491+0000", 2023-09-23T19:44:01.892 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:44:01.892 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.892 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.893 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:44:01.893 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:01.863490+0000", 2023-09-23T19:44:01.893 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:44:01.893 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.893 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.893 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:44:01.893 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:01.863495+0000", 2023-09-23T19:44:01.893 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.9060000000000001e-06 2023-09-23T19:44:01.893 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.893 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.893 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:44:01.893 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:01.863503+0000", 2023-09-23T19:44:01.894 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.8079999999999994e-06 2023-09-23T19:44:01.894 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.894 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.894 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:44:01.894 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:01.863509+0000", 2023-09-23T19:44:01.894 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.1379999999999998e-06 2023-09-23T19:44:01.894 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.894 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.894 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:44:01.894 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:01.863553+0000", 2023-09-23T19:44:01.894 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.3921000000000002e-05 2023-09-23T19:44:01.894 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.895 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.895 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:44:01.895 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:01.863685+0000", 2023-09-23T19:44:01.895 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000132791 2023-09-23T19:44:01.895 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.895 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.895 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 3,7", 2023-09-23T19:44:01.895 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:01.863810+0000", 2023-09-23T19:44:01.895 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000125003 2023-09-23T19:44:01.895 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.895 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.895 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:44:01.895 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:01.872468+0000", 2023-09-23T19:44:01.896 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0086575809999999993 2023-09-23T19:44:01.896 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:44:01.896 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:44:01.896 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:44:01.896 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.896 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.896 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.36243.0:142274 11.e e194/180 11:7b808d05:::benchmark_data_smithi016_160148_object35071:head v 194'8963, mlcod=194'8961)", 2023-09-23T19:44:01.896 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:44:01.864791+0000", 2023-09-23T19:44:01.896 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 0.0098603270000000003, 2023-09-23T19:44:01.896 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.010174953, 2023-09-23T19:44:01.896 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:44:01.896 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:44:01.897 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:44:01.897 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.897 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:44:01.897 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:01.864791+0000", 2023-09-23T19:44:01.897 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:44:01.897 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.897 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.897 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:44:01.897 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:01.864791+0000", 2023-09-23T19:44:01.897 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:44:01.897 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.897 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.898 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:44:01.898 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:01.864790+0000", 2023-09-23T19:44:01.898 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:44:01.898 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.898 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.898 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:44:01.898 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:01.864799+0000", 2023-09-23T19:44:01.898 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.3159999999999997e-06 2023-09-23T19:44:01.898 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.898 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.898 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:44:01.898 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:01.864808+0000", 2023-09-23T19:44:01.899 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.9430000000000006e-06 2023-09-23T19:44:01.899 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.899 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.899 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:44:01.899 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:01.864815+0000", 2023-09-23T19:44:01.899 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.187e-06 2023-09-23T19:44:01.899 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.899 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.899 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:44:01.899 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:01.864937+0000", 2023-09-23T19:44:01.899 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000122127 2023-09-23T19:44:01.899 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.900 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.900 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:44:01.900 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:01.864971+0000", 2023-09-23T19:44:01.900 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.4517000000000002e-05 2023-09-23T19:44:01.900 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.900 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.900 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:44:01.900 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:01.874248+0000", 2023-09-23T19:44:01.900 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0092767249999999996 2023-09-23T19:44:01.900 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:44:01.900 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:44:01.900 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:44:01.900 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.901 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.901 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.36243.0:142283 11.f e194/180 11:f644f4d0:::benchmark_data_smithi016_160148_object35080:head v 194'9137, mlcod=194'9135)", 2023-09-23T19:44:01.901 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:44:01.864870+0000", 2023-09-23T19:44:01.901 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 0.0097820779999999996, 2023-09-23T19:44:01.901 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.010165749, 2023-09-23T19:44:01.901 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:44:01.901 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "started", 2023-09-23T19:44:01.901 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:44:01.901 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.901 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:44:01.901 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:01.864870+0000", 2023-09-23T19:44:01.902 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:44:01.902 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.902 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.902 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:44:01.902 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:01.864870+0000", 2023-09-23T19:44:01.902 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:44:01.902 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.902 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.902 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:44:01.902 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:01.864869+0000", 2023-09-23T19:44:01.902 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967296 2023-09-23T19:44:01.902 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.903 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.903 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:44:01.903 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:01.864873+0000", 2023-09-23T19:44:01.903 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.036e-06 2023-09-23T19:44:01.903 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.903 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.903 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:44:01.903 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:01.864879+0000", 2023-09-23T19:44:01.903 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.4770000000000002e-06 2023-09-23T19:44:01.903 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.903 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.904 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:44:01.904 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:01.864883+0000", 2023-09-23T19:44:01.904 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.0489999999999997e-06 2023-09-23T19:44:01.904 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.904 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.904 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:44:01.904 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:01.865854+0000", 2023-09-23T19:44:01.904 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000971078 2023-09-23T19:44:01.904 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.904 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.904 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:44:01.904 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:01.865873+0000", 2023-09-23T19:44:01.905 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.9174e-05 2023-09-23T19:44:01.905 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:44:01.905 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:44:01.905 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:44:01.905 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.905 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.905 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.36243.0:142284 11.f e194/180)", 2023-09-23T19:44:01.905 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:44:01.870886+0000", 2023-09-23T19:44:01.905 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 0.0037654490000000001, 2023-09-23T19:44:01.905 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0042075649999999999, 2023-09-23T19:44:01.905 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:44:01.906 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "reached pg", 2023-09-23T19:44:01.906 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:44:01.906 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.906 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:44:01.906 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:01.870886+0000", 2023-09-23T19:44:01.906 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:44:01.906 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.906 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.906 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:44:01.906 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:01.870886+0000", 2023-09-23T19:44:01.906 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:44:01.906 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.907 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.907 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:44:01.907 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:01.870885+0000", 2023-09-23T19:44:01.907 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:44:01.907 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.907 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.907 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:44:01.907 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:01.870896+0000", 2023-09-23T19:44:01.907 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0567000000000001e-05 2023-09-23T19:44:01.907 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.907 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.907 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:44:01.907 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:01.870908+0000", 2023-09-23T19:44:01.908 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1673e-05 2023-09-23T19:44:01.908 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.908 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.908 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:44:01.908 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:01.870915+0000", 2023-09-23T19:44:01.908 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.6199999999999999e-06 2023-09-23T19:44:01.908 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:01.908 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:01.908 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:44:01.908 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:01.873091+0000", 2023-09-23T19:44:01.908 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0021755250000000002 2023-09-23T19:44:01.908 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:44:01.909 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:44:01.909 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:44:01.909 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:44:01.909 INFO:teuthology.orchestra.run.smithi016.stdout: ], 2023-09-23T19:44:01.909 INFO:teuthology.orchestra.run.smithi016.stdout: "num_ops": 6 2023-09-23T19:44:01.909 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-09-23T19:44:01.944 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:01.954 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:01 smithi092 bash[58855]: cluster 2023-09-23T19:44:00.260367+0000 mgr.y (mgr.25325) 1097 : cluster 0 pgmap v1060: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 763 op/s 2023-09-23T19:44:01.977 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:01.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:01.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:44:02.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:01 smithi016 bash[75004]: cluster 2023-09-23T19:44:00.260367+0000 mgr.y (mgr.25325) 1097 : cluster 0 pgmap v1060: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 763 op/s 2023-09-23T19:44:02.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:01 smithi016 bash[77995]: cluster 2023-09-23T19:44:00.260367+0000 mgr.y (mgr.25325) 1097 : cluster 0 pgmap v1060: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 763 op/s 2023-09-23T19:44:02.080 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:02.106 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:02.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:02.109 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:44:02.210 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:02.243 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:02.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:02.247 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:44:02.348 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:02.372 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:02.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:02.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:44:02.477 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:02.521 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:02.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:02.524 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:44:02.624 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:02.658 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:02.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:02.661 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:44:02.761 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:02.784 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:02.790 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:44:02.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:02.793 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:44:02.838 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:44:02.882 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:44:02.894 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:02.918 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:44:02.925 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:02.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:02.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:44:02.956 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:44:02.985 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:44:03.016 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:44:03.029 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:03.053 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:03.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:03.056 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:03.071 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:44:03.086 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:44:03.086 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:44:03.086 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:44:03.086 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:44:03.086 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:44:03.087 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:44:03.087 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:44:03.087 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:44:03.157 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:03.187 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:03.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:03.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:03.292 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:03.314 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:03.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:03.316 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:44:03.417 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:03.439 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:03.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:03.441 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:03.542 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:03.570 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:03.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:03.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:44:03.676 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:03.707 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:03.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:03.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:44:03.739 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:03 smithi016 bash[75004]: cluster 2023-09-23T19:44:02.260921+0000 mgr.y (mgr.25325) 1098 : cluster 0 pgmap v1061: 89 pgs: 89 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 667 op/s 2023-09-23T19:44:03.739 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:03 smithi016 bash[75004]: audit 2023-09-23T19:44:02.750331+0000 mon.a (mon.0) 1389 : audit 0 from='client.? 172.21.15.16:0/2608232063' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:44:03.739 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:03 smithi016 bash[77995]: cluster 2023-09-23T19:44:02.260921+0000 mgr.y (mgr.25325) 1098 : cluster 0 pgmap v1061: 89 pgs: 89 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 667 op/s 2023-09-23T19:44:03.739 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:03 smithi016 bash[77995]: audit 2023-09-23T19:44:02.750331+0000 mon.a (mon.0) 1389 : audit 0 from='client.? 172.21.15.16:0/2608232063' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:44:03.810 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:03.852 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:03.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:03.858 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:03.958 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:03.988 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:03.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:03.994 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:44:04.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:03 smithi092 bash[58855]: cluster 2023-09-23T19:44:02.260921+0000 mgr.y (mgr.25325) 1098 : cluster 0 pgmap v1061: 89 pgs: 89 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 667 op/s 2023-09-23T19:44:04.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:03 smithi092 bash[58855]: audit 2023-09-23T19:44:02.750331+0000 mon.a (mon.0) 1389 : audit 0 from='client.? 172.21.15.16:0/2608232063' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:44:04.094 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:04.130 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:04.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:04.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:04.150 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T19:44:04.234 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:04.264 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:04.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:04.268 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:44:04.369 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:04.403 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:04.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:04.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:04.509 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:04.533 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:04.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:04.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:44:04.640 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:04.669 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:04.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:04.671 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:44:04.772 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:04.801 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:04.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:04.805 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:44:04.906 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:04.936 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:04.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:04.941 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:44:05.041 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:05.070 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:05.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:05.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:44:05.173 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:05.192 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:05.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:05.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:05.294 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:05.319 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:05.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:05.322 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:05.423 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:05.458 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:05.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:05.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:44:05.562 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:05.573 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-09-23T19:44:05.573 INFO:teuthology.orchestra.run.smithi016.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T19:44:05.573 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-09-23T19:44:05.588 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:05.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:05.592 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:44:05.693 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:05.727 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:05.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:05.733 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:44:05.833 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:05.843 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:05 smithi092 bash[58855]: cluster 2023-09-23T19:44:04.261542+0000 mgr.y (mgr.25325) 1099 : cluster 0 pgmap v1062: 89 pgs: 89 active+clean; 4.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 647 op/s 2023-09-23T19:44:05.867 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:05.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:05.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:44:05.922 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:44:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:44:05] "GET /metrics HTTP/1.1" 200 36307 "" "Prometheus/2.33.4" 2023-09-23T19:44:05.922 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:05 smithi016 bash[75004]: cluster 2023-09-23T19:44:04.261542+0000 mgr.y (mgr.25325) 1099 : cluster 0 pgmap v1062: 89 pgs: 89 active+clean; 4.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 647 op/s 2023-09-23T19:44:05.922 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:05 smithi016 bash[77995]: cluster 2023-09-23T19:44:04.261542+0000 mgr.y (mgr.25325) 1099 : cluster 0 pgmap v1062: 89 pgs: 89 active+clean; 4.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 647 op/s 2023-09-23T19:44:05.975 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:06.001 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:06.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:06.006 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:44:06.107 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:06.138 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:06.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:06.142 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:06.242 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:06.274 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:06.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:06.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:44:06.377 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:06.405 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:06.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:06.410 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:44:06.511 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:06.566 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:06.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:06.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:06.672 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:06.697 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:06.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:06.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:44:06.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:06 smithi016 bash[75004]: audit 2023-09-23T19:44:06.247244+0000 mon.a (mon.0) 1390 : audit 1 from='client.? 172.21.15.16:0/377793614' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:44:06.766 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is set 2023-09-23T19:44:06.801 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:06.831 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:06.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:06.834 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:44:06.935 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:06.959 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:06.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:06.962 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:44:07.031 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:06 smithi016 bash[77995]: audit 2023-09-23T19:44:06.247244+0000 mon.a (mon.0) 1390 : audit 1 from='client.? 172.21.15.16:0/377793614' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:44:07.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:06 smithi092 bash[58855]: audit 2023-09-23T19:44:06.247244+0000 mon.a (mon.0) 1390 : audit 1 from='client.? 172.21.15.16:0/377793614' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:44:07.063 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:07.103 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:07.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:07.107 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:44:07.208 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:07.241 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:07.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:07.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:44:07.345 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:07.367 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:07.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:07.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:07.476 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:07.512 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:07.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:07.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:44:07.615 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:07.650 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:07.652 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:44:07.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:07.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:07.755 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:07.797 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:07.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:07.800 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:44:07.901 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:07.929 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:07.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:07.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:44:08.036 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:08.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:07 smithi016 bash[75004]: cluster 2023-09-23T19:44:06.262183+0000 mgr.y (mgr.25325) 1100 : cluster 0 pgmap v1063: 89 pgs: 89 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 832 op/s 2023-09-23T19:44:08.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:07 smithi016 bash[75004]: cluster 2023-09-23T19:44:06.736918+0000 mon.a (mon.0) 1391 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:44:08.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:07 smithi016 bash[75004]: audit 2023-09-23T19:44:06.749582+0000 mon.a (mon.0) 1392 : audit 1 from='client.? 172.21.15.16:0/377793614' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:44:08.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:07 smithi016 bash[75004]: cluster 2023-09-23T19:44:06.750358+0000 mon.a (mon.0) 1393 : cluster 0 osdmap e195: 8 total, 8 up, 8 in 2023-09-23T19:44:08.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:07 smithi016 bash[75004]: cluster 2023-09-23T19:44:06.750786+0000 osd.4 (osd.4) 136 : cluster 3 failed to encode map e195 with expected crc 2023-09-23T19:44:08.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:07 smithi016 bash[75004]: cluster 2023-09-23T19:44:06.764137+0000 osd.6 (osd.6) 120 : cluster 3 failed to encode map e195 with expected crc 2023-09-23T19:44:08.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:07 smithi016 bash[75004]: cluster 2023-09-23T19:44:06.768584+0000 osd.7 (osd.7) 110 : cluster 3 failed to encode map e195 with expected crc 2023-09-23T19:44:08.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:07 smithi016 bash[77995]: cluster 2023-09-23T19:44:06.262183+0000 mgr.y (mgr.25325) 1100 : cluster 0 pgmap v1063: 89 pgs: 89 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 832 op/s 2023-09-23T19:44:08.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:07 smithi016 bash[77995]: cluster 2023-09-23T19:44:06.736918+0000 mon.a (mon.0) 1391 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:44:08.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:07 smithi016 bash[77995]: audit 2023-09-23T19:44:06.749582+0000 mon.a (mon.0) 1392 : audit 1 from='client.? 172.21.15.16:0/377793614' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:44:08.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:07 smithi016 bash[77995]: cluster 2023-09-23T19:44:06.750358+0000 mon.a (mon.0) 1393 : cluster 0 osdmap e195: 8 total, 8 up, 8 in 2023-09-23T19:44:08.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:07 smithi016 bash[77995]: cluster 2023-09-23T19:44:06.750786+0000 osd.4 (osd.4) 136 : cluster 3 failed to encode map e195 with expected crc 2023-09-23T19:44:08.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:07 smithi016 bash[77995]: cluster 2023-09-23T19:44:06.764137+0000 osd.6 (osd.6) 120 : cluster 3 failed to encode map e195 with expected crc 2023-09-23T19:44:08.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:07 smithi016 bash[77995]: cluster 2023-09-23T19:44:06.768584+0000 osd.7 (osd.7) 110 : cluster 3 failed to encode map e195 with expected crc 2023-09-23T19:44:08.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:07 smithi092 bash[58855]: cluster 2023-09-23T19:44:06.262183+0000 mgr.y (mgr.25325) 1100 : cluster 0 pgmap v1063: 89 pgs: 89 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 832 op/s 2023-09-23T19:44:08.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:07 smithi092 bash[58855]: cluster 2023-09-23T19:44:06.736918+0000 mon.a (mon.0) 1391 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:44:08.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:07 smithi092 bash[58855]: audit 2023-09-23T19:44:06.749582+0000 mon.a (mon.0) 1392 : audit 1 from='client.? 172.21.15.16:0/377793614' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:44:08.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:07 smithi092 bash[58855]: cluster 2023-09-23T19:44:06.750358+0000 mon.a (mon.0) 1393 : cluster 0 osdmap e195: 8 total, 8 up, 8 in 2023-09-23T19:44:08.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:07 smithi092 bash[58855]: cluster 2023-09-23T19:44:06.750786+0000 osd.4 (osd.4) 136 : cluster 3 failed to encode map e195 with expected crc 2023-09-23T19:44:08.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:07 smithi092 bash[58855]: cluster 2023-09-23T19:44:06.764137+0000 osd.6 (osd.6) 120 : cluster 3 failed to encode map e195 with expected crc 2023-09-23T19:44:08.040 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:07 smithi092 bash[58855]: cluster 2023-09-23T19:44:06.768584+0000 osd.7 (osd.7) 110 : cluster 3 failed to encode map e195 with expected crc 2023-09-23T19:44:08.072 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:08.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:08.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:44:08.177 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:08.204 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:08.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:08.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:08.312 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:08.338 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:08.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:08.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:44:08.446 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:08.493 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:08.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:08.495 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:08.596 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:08.623 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:08.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:08.631 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:44:08.732 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:08.768 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:08.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:08.771 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:08.871 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:08.908 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:08.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:08.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:09.013 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:09.042 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:09.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:09.046 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:44:09.146 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:09.186 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:09.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:09.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:44:09.289 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:09.323 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:09.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:09.329 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:44:09.430 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:09.454 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:09.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:09.457 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:44:09.557 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:09.590 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:09.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:09.595 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:44:09.695 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:09.723 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:09.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:09.730 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:44:09.817 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T19:44:09.831 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:09.896 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:09.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:09.900 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:44:10.001 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:10.010 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:09 smithi016 bash[77995]: cluster 2023-09-23T19:44:08.262638+0000 mgr.y (mgr.25325) 1101 : cluster 0 pgmap v1065: 89 pgs: 89 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 706 op/s 2023-09-23T19:44:10.010 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:09 smithi016 bash[75004]: cluster 2023-09-23T19:44:08.262638+0000 mgr.y (mgr.25325) 1101 : cluster 0 pgmap v1065: 89 pgs: 89 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 706 op/s 2023-09-23T19:44:10.036 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:10.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:09 smithi092 bash[58855]: cluster 2023-09-23T19:44:08.262638+0000 mgr.y (mgr.25325) 1101 : cluster 0 pgmap v1065: 89 pgs: 89 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 706 op/s 2023-09-23T19:44:10.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:10.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:44:10.140 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:10.167 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:10.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:10.170 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:44:10.270 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:10.304 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:10.305 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:44:10.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:10.310 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:44:10.411 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:10.448 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:10.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:10.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:44:10.551 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:10.581 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:10.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:10.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:10.690 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:10.720 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:10.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:10.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:10.824 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:10.849 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:10.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:10.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:44:10.954 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:10.982 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:10.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:10.986 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:44:11.087 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:11.115 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:11.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:11.118 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:44:11.219 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:11.244 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:11.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:11.247 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:11.347 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:11.388 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:11.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:11.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:44:11.494 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:11.531 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:11.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:11.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:44:11.638 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:11.667 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:11.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:11.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:44:11.768 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:11 smithi016 bash[75004]: cluster 2023-09-23T19:44:10.263403+0000 mgr.y (mgr.25325) 1102 : cluster 0 pgmap v1066: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 695 op/s 2023-09-23T19:44:11.768 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:11 smithi016 bash[77995]: cluster 2023-09-23T19:44:10.263403+0000 mgr.y (mgr.25325) 1102 : cluster 0 pgmap v1066: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 695 op/s 2023-09-23T19:44:11.771 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:11.797 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:11.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:11.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:44:11.903 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:11.931 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:11.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:11.934 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:11.963 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-09-23T19:44:11.963 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T19:44:11.964 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2023-09-23T19:44:11.964 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T19:44:11.964 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T19:44:11.964 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 config set filestore_inject_stall 3 2023-09-23T19:44:12.035 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:12.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:11 smithi092 bash[58855]: cluster 2023-09-23T19:44:10.263403+0000 mgr.y (mgr.25325) 1102 : cluster 0 pgmap v1066: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 695 op/s 2023-09-23T19:44:12.059 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:12.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:12.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:44:12.164 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:12.206 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:12.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:12.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:44:12.313 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:12.344 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:12.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:12.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:44:12.354 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:44:12.451 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:12.475 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:12.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:12.481 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:44:12.582 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:12.605 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:12.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:12.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:44:12.708 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:12.755 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:12.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:12.759 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:44:12.859 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:12.893 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:12.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:12.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:44:13.001 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:13.027 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:13.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:13.032 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:44:13.133 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:13.170 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:13.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:13.173 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:13.274 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:13.301 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:13.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:13.304 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:44:13.405 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:13.430 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:13.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:13.435 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:13.536 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:13.569 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:13.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:13.576 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:44:13.677 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:13.722 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:13.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:13.728 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:13.828 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:13.861 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:13.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:13.866 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:13.967 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:13.975 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:13 smithi016 bash[77995]: cluster 2023-09-23T19:44:12.263968+0000 mgr.y (mgr.25325) 1103 : cluster 0 pgmap v1067: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2023-09-23T19:44:13.976 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:13 smithi016 bash[77995]: audit 2023-09-23T19:44:13.663330+0000 mon.a (mon.0) 1394 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:44:13.976 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:13 smithi016 bash[75004]: cluster 2023-09-23T19:44:12.263968+0000 mgr.y (mgr.25325) 1103 : cluster 0 pgmap v1067: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2023-09-23T19:44:13.976 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:13 smithi016 bash[75004]: audit 2023-09-23T19:44:13.663330+0000 mon.a (mon.0) 1394 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:44:13.995 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:13.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:13.998 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:14.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:13 smithi092 bash[58855]: cluster 2023-09-23T19:44:12.263968+0000 mgr.y (mgr.25325) 1103 : cluster 0 pgmap v1067: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2023-09-23T19:44:14.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:13 smithi092 bash[58855]: audit 2023-09-23T19:44:13.663330+0000 mon.a (mon.0) 1394 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:44:14.099 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:14.128 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:14.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:14.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:44:14.233 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:14.259 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:14.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:14.262 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:14.362 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:14.387 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:14.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:14.389 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:44:14.490 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:14.527 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:14.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:14.529 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:44:14.630 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:14.654 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:14.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:14.658 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:44:14.759 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:14.790 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:14.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:14.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:44:14.894 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:14.923 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:14.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:14.929 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:44:15.029 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:15.062 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:15.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:15.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:44:15.110 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:44:15.168 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:15.197 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:15.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:15.199 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:44:15.300 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:15.319 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:15.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:15.321 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:15.422 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:15.449 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:15.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:15.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:44:15.553 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:15.584 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:15.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:15.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:44:15.642 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:44:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:44:15] "GET /metrics HTTP/1.1" 200 36306 "" "Prometheus/2.33.4" 2023-09-23T19:44:15.689 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:15.719 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:15.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:15.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:44:15.827 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:15.854 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:15.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:15.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:44:15.957 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:15.966 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:15 smithi092 bash[58855]: cluster 2023-09-23T19:44:14.264506+0000 mgr.y (mgr.25325) 1104 : cluster 0 pgmap v1068: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2023-09-23T19:44:15.993 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:15.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:15.996 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:44:16.097 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:16.115 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:16.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:16.117 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:44:16.122 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:15 smithi016 bash[75004]: cluster 2023-09-23T19:44:14.264506+0000 mgr.y (mgr.25325) 1104 : cluster 0 pgmap v1068: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2023-09-23T19:44:16.122 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:15 smithi016 bash[77995]: cluster 2023-09-23T19:44:14.264506+0000 mgr.y (mgr.25325) 1104 : cluster 0 pgmap v1068: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2023-09-23T19:44:16.218 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:16.252 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:16.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:16.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:44:16.356 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:16.379 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:16.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:16.381 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:44:16.481 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:16.516 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:16.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:16.520 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:44:16.621 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:16.663 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:16.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:16.668 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:44:16.769 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:16.783 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:44:16.796 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:16.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:16.798 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:44:16.899 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:16.925 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:16.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:16.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:17.033 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:17.062 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:17.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:17.064 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:44:17.165 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:17.193 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:17.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:17.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:17.298 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:17.322 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:17.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:17.329 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:44:17.430 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:17.461 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:17.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:17.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:44:17.564 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:17.602 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:17.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:17.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:17.709 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:17.734 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:17.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:17.747 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:44:17.848 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:17.856 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:17 smithi016 bash[75004]: cluster 2023-09-23T19:44:16.265257+0000 mgr.y (mgr.25325) 1105 : cluster 0 pgmap v1069: 89 pgs: 89 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 704 op/s 2023-09-23T19:44:17.856 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:17 smithi016 bash[77995]: cluster 2023-09-23T19:44:16.265257+0000 mgr.y (mgr.25325) 1105 : cluster 0 pgmap v1069: 89 pgs: 89 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 704 op/s 2023-09-23T19:44:17.879 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:17.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:17.885 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:44:17.985 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:17.995 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:17 smithi092 bash[58855]: cluster 2023-09-23T19:44:16.265257+0000 mgr.y (mgr.25325) 1105 : cluster 0 pgmap v1069: 89 pgs: 89 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 704 op/s 2023-09-23T19:44:18.000 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-09-23T19:44:18.000 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:44:18.017 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:18.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:18.021 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:44:18.122 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:18.147 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:18.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:18.153 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:44:18.254 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:18.288 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:18.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:18.293 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:44:18.394 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:18.416 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:18.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:18.420 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:44:18.521 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:18.545 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:18.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:18.548 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:44:18.648 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:18.680 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:18.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:18.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:44:18.784 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:18.806 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:18.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:18.808 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:44:18.909 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:18.944 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:18.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:18.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:44:19.049 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:19.073 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:19.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:19.075 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:44:19.176 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:19.200 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:19.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:19.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:44:19.306 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:19.336 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:19.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:19.338 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:44:19.439 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:19.471 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:19.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:19.476 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:19.577 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:19.609 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:19.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:19.618 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:44:19.718 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:19.755 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:19.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:19.763 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:44:19.864 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:19.872 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:19 smithi016 bash[77995]: cluster 2023-09-23T19:44:18.265813+0000 mgr.y (mgr.25325) 1106 : cluster 0 pgmap v1070: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 667 op/s 2023-09-23T19:44:19.872 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:19 smithi016 bash[75004]: cluster 2023-09-23T19:44:18.265813+0000 mgr.y (mgr.25325) 1106 : cluster 0 pgmap v1070: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 667 op/s 2023-09-23T19:44:19.891 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:19.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:19.893 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:44:19.993 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:20.024 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:20.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:20.032 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:20.133 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:20.164 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:20.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:20.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:20.269 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:20.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:19 smithi092 bash[58855]: cluster 2023-09-23T19:44:18.265813+0000 mgr.y (mgr.25325) 1106 : cluster 0 pgmap v1070: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 667 op/s 2023-09-23T19:44:20.300 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:20.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:20.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:20.406 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:20.434 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:20.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:20.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:20.537 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:20.570 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:20.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:20.572 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:44:20.673 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:20.710 INFO:teuthology.orchestra.run.smithi092.stdout:{ 2023-09-23T19:44:20.710 INFO:teuthology.orchestra.run.smithi092.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T19:44:20.710 INFO:teuthology.orchestra.run.smithi092.stdout:} 2023-09-23T19:44:20.734 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:20.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:20.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:44:20.839 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:20.863 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:20.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:20.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:44:20.976 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:21.002 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:21.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:21.007 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:44:21.108 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:21.138 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:21.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:21.141 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:44:21.242 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:21.287 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:21.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:21.292 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:44:21.393 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:21.423 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:21.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:21.426 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:21.527 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:21.561 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T19:44:21.567 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:21.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:21.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:21.673 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:21.707 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:21.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:21.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:21.814 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:21.850 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:21.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:21.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:44:21.955 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:21.963 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:21 smithi016 bash[77995]: cluster 2023-09-23T19:44:20.266488+0000 mgr.y (mgr.25325) 1107 : cluster 0 pgmap v1071: 89 pgs: 89 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 754 op/s 2023-09-23T19:44:21.963 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:21 smithi016 bash[75004]: cluster 2023-09-23T19:44:20.266488+0000 mgr.y (mgr.25325) 1107 : cluster 0 pgmap v1071: 89 pgs: 89 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 754 op/s 2023-09-23T19:44:21.991 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:21.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:21.995 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:44:22.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:21 smithi092 bash[58855]: cluster 2023-09-23T19:44:20.266488+0000 mgr.y (mgr.25325) 1107 : cluster 0 pgmap v1071: 89 pgs: 89 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 754 op/s 2023-09-23T19:44:22.096 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:22.150 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:22.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:22.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:44:22.257 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:22.284 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:22.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:22.292 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:44:22.392 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:22.418 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:22.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:22.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:22.526 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:22.550 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:22.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:22.552 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:44:22.653 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:22.676 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:22.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:22.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:44:22.784 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:22.812 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:22.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:22.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:44:22.915 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:22.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:22.945 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:22.945 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:44:23.046 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:23.068 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:23.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:23.072 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:44:23.172 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:23.210 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:44:23.212 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:23.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:23.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:23.277 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:44:23.316 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:44:23.318 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:23.358 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:23.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:23.366 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:23.367 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:44:23.402 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:44:23.439 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:44:23.467 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:23.469 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:44:23.498 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:23.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:23.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:44:23.503 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:44:23.505 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:44:23.505 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:44:23.505 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:44:23.505 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:44:23.505 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:44:23.505 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:44:23.505 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:44:23.505 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:44:23.602 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:23.627 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:23.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:23.633 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:44:23.733 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:23.771 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:23.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:23.776 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:44:23.877 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:23.918 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:23.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:23.922 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:44:24.022 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:24.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:23 smithi016 bash[75004]: cluster 2023-09-23T19:44:22.267080+0000 mgr.y (mgr.25325) 1108 : cluster 0 pgmap v1072: 89 pgs: 89 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2023-09-23T19:44:24.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:23 smithi016 bash[75004]: audit 2023-09-23T19:44:23.157660+0000 mon.a (mon.0) 1395 : audit 0 from='client.? 172.21.15.16:0/3704371550' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:44:24.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:23 smithi016 bash[77995]: cluster 2023-09-23T19:44:22.267080+0000 mgr.y (mgr.25325) 1108 : cluster 0 pgmap v1072: 89 pgs: 89 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2023-09-23T19:44:24.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:23 smithi016 bash[77995]: audit 2023-09-23T19:44:23.157660+0000 mon.a (mon.0) 1395 : audit 0 from='client.? 172.21.15.16:0/3704371550' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:44:24.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:23 smithi092 bash[58855]: cluster 2023-09-23T19:44:22.267080+0000 mgr.y (mgr.25325) 1108 : cluster 0 pgmap v1072: 89 pgs: 89 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2023-09-23T19:44:24.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:23 smithi092 bash[58855]: audit 2023-09-23T19:44:23.157660+0000 mon.a (mon.0) 1395 : audit 0 from='client.? 172.21.15.16:0/3704371550' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:44:24.059 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:24.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:24.066 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:24.166 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:24.193 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:24.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:24.197 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:44:24.298 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:24.324 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:24.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:24.332 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:44:24.433 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:24.463 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:24.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:24.487 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:44:24.589 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:24.617 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:24.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:24.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:44:24.720 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:24.752 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:24.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:24.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:24.830 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is unset 2023-09-23T19:44:24.862 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:24.891 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:24.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:24.894 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:44:24.994 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:25.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:24 smithi016 bash[75004]: audit 2023-09-23T19:44:24.409563+0000 mon.a (mon.0) 1396 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:44:25.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:24 smithi016 bash[75004]: audit 2023-09-23T19:44:24.414738+0000 mon.b (mon.1) 49 : audit 1 from='client.? 172.21.15.16:0/4194163782' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:44:25.004 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:24 smithi016 bash[77995]: audit 2023-09-23T19:44:24.409563+0000 mon.a (mon.0) 1396 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:44:25.004 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:24 smithi016 bash[77995]: audit 2023-09-23T19:44:24.414738+0000 mon.b (mon.1) 49 : audit 1 from='client.? 172.21.15.16:0/4194163782' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:44:25.035 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:25.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:25.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:44:25.142 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:25.177 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:25.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:25.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:44:25.286 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:25.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:24 smithi092 bash[58855]: audit 2023-09-23T19:44:24.409563+0000 mon.a (mon.0) 1396 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:44:25.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:24 smithi092 bash[58855]: audit 2023-09-23T19:44:24.414738+0000 mon.b (mon.1) 49 : audit 1 from='client.? 172.21.15.16:0/4194163782' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:44:25.314 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:25.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:25.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:44:25.416 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:25.444 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:25.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:25.456 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:44:25.557 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:25.591 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:25.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:25.595 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:44:25.696 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:25.726 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:25.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:25.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:44:25.786 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:44:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:44:25] "GET /metrics HTTP/1.1" 200 36306 "" "Prometheus/2.33.4" 2023-09-23T19:44:25.829 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:25.868 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:25.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:25.872 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:25.973 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:26.008 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:26.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:26.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:26.114 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:26.124 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:25 smithi016 bash[75004]: cluster 2023-09-23T19:44:24.267578+0000 mgr.y (mgr.25325) 1109 : cluster 0 pgmap v1073: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2023-09-23T19:44:26.124 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:25 smithi016 bash[75004]: cluster 2023-09-23T19:44:24.808760+0000 mon.a (mon.0) 1397 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:44:26.124 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:25 smithi016 bash[75004]: audit 2023-09-23T19:44:24.819162+0000 mon.a (mon.0) 1398 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:44:26.124 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:25 smithi016 bash[75004]: cluster 2023-09-23T19:44:24.819398+0000 mon.a (mon.0) 1399 : cluster 0 osdmap e196: 8 total, 8 up, 8 in 2023-09-23T19:44:26.124 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:25 smithi016 bash[75004]: cluster 2023-09-23T19:44:24.830962+0000 osd.6 (osd.6) 121 : cluster 3 failed to encode map e196 with expected crc 2023-09-23T19:44:26.124 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:25 smithi016 bash[75004]: cluster 2023-09-23T19:44:24.833038+0000 osd.7 (osd.7) 111 : cluster 3 failed to encode map e196 with expected crc 2023-09-23T19:44:26.124 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:25 smithi016 bash[75004]: cluster 2023-09-23T19:44:24.836371+0000 osd.4 (osd.4) 137 : cluster 3 failed to encode map e196 with expected crc 2023-09-23T19:44:26.125 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:25 smithi016 bash[75004]: cluster 2023-09-23T19:44:24.836659+0000 osd.5 (osd.5) 124 : cluster 3 failed to encode map e196 with expected crc 2023-09-23T19:44:26.125 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:25 smithi016 bash[77995]: cluster 2023-09-23T19:44:24.267578+0000 mgr.y (mgr.25325) 1109 : cluster 0 pgmap v1073: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2023-09-23T19:44:26.125 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:25 smithi016 bash[77995]: cluster 2023-09-23T19:44:24.808760+0000 mon.a (mon.0) 1397 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:44:26.125 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:25 smithi016 bash[77995]: audit 2023-09-23T19:44:24.819162+0000 mon.a (mon.0) 1398 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:44:26.125 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:25 smithi016 bash[77995]: cluster 2023-09-23T19:44:24.819398+0000 mon.a (mon.0) 1399 : cluster 0 osdmap e196: 8 total, 8 up, 8 in 2023-09-23T19:44:26.125 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:25 smithi016 bash[77995]: cluster 2023-09-23T19:44:24.830962+0000 osd.6 (osd.6) 121 : cluster 3 failed to encode map e196 with expected crc 2023-09-23T19:44:26.125 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:25 smithi016 bash[77995]: cluster 2023-09-23T19:44:24.833038+0000 osd.7 (osd.7) 111 : cluster 3 failed to encode map e196 with expected crc 2023-09-23T19:44:26.125 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:25 smithi016 bash[77995]: cluster 2023-09-23T19:44:24.836371+0000 osd.4 (osd.4) 137 : cluster 3 failed to encode map e196 with expected crc 2023-09-23T19:44:26.125 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:25 smithi016 bash[77995]: cluster 2023-09-23T19:44:24.836659+0000 osd.5 (osd.5) 124 : cluster 3 failed to encode map e196 with expected crc 2023-09-23T19:44:26.141 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:26.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:26.145 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:44:26.245 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:26.253 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:25 smithi092 bash[58855]: cluster 2023-09-23T19:44:24.267578+0000 mgr.y (mgr.25325) 1109 : cluster 0 pgmap v1073: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2023-09-23T19:44:26.253 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:25 smithi092 bash[58855]: cluster 2023-09-23T19:44:24.808760+0000 mon.a (mon.0) 1397 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:44:26.253 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:25 smithi092 bash[58855]: audit 2023-09-23T19:44:24.819162+0000 mon.a (mon.0) 1398 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:44:26.253 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:25 smithi092 bash[58855]: cluster 2023-09-23T19:44:24.819398+0000 mon.a (mon.0) 1399 : cluster 0 osdmap e196: 8 total, 8 up, 8 in 2023-09-23T19:44:26.253 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:25 smithi092 bash[58855]: cluster 2023-09-23T19:44:24.830962+0000 osd.6 (osd.6) 121 : cluster 3 failed to encode map e196 with expected crc 2023-09-23T19:44:26.253 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:25 smithi092 bash[58855]: cluster 2023-09-23T19:44:24.833038+0000 osd.7 (osd.7) 111 : cluster 3 failed to encode map e196 with expected crc 2023-09-23T19:44:26.253 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:25 smithi092 bash[58855]: cluster 2023-09-23T19:44:24.836371+0000 osd.4 (osd.4) 137 : cluster 3 failed to encode map e196 with expected crc 2023-09-23T19:44:26.253 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:25 smithi092 bash[58855]: cluster 2023-09-23T19:44:24.836659+0000 osd.5 (osd.5) 124 : cluster 3 failed to encode map e196 with expected crc 2023-09-23T19:44:26.277 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:26.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:26.280 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:44:26.380 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:26.408 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:26.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:26.417 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:26.517 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:26.561 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:26.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:26.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:26.665 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:26.688 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-09-23T19:44:26.688 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T19:44:26.698 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:26.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:26.703 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:44:26.773 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:44:26.804 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:26.860 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:26.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:26.868 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:26.969 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:27.009 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:27.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:27.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:44:27.111 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:27.143 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:27.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:27.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:44:27.248 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:27.290 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:27.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:27.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:44:27.395 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:27.430 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:27.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:27.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:44:27.536 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:27.577 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:27.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:27.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:44:27.681 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:27.711 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:27.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:27.714 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:44:27.814 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:27.825 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:27 smithi016 bash[75004]: cluster 2023-09-23T19:44:26.268119+0000 mgr.y (mgr.25325) 1110 : cluster 0 pgmap v1075: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2023-09-23T19:44:27.825 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:27 smithi016 bash[77995]: cluster 2023-09-23T19:44:26.268119+0000 mgr.y (mgr.25325) 1110 : cluster 0 pgmap v1075: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2023-09-23T19:44:27.845 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:27.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:27.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:44:27.949 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:27.990 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:27.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:27.994 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:28.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:27 smithi092 bash[58855]: cluster 2023-09-23T19:44:26.268119+0000 mgr.y (mgr.25325) 1110 : cluster 0 pgmap v1075: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2023-09-23T19:44:28.054 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T19:44:28.094 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:28.135 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:28.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:28.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:28.239 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:28.286 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:28.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:28.289 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:44:28.389 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:28.417 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:28.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:28.421 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:28.507 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:44:28.522 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:28.554 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:28.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:28.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:44:28.659 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:28.702 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:28.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:28.706 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:28.807 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:28.835 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:28.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:28.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:44:28.937 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:28.976 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:28.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:28.978 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:44:29.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:28 smithi016 bash[75004]: audit 2023-09-23T19:44:28.663730+0000 mon.a (mon.0) 1400 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:44:29.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:28 smithi016 bash[77995]: audit 2023-09-23T19:44:28.663730+0000 mon.a (mon.0) 1400 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:44:29.079 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:29.090 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:28 smithi092 bash[58855]: audit 2023-09-23T19:44:28.663730+0000 mon.a (mon.0) 1400 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:44:29.109 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:29.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:29.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:44:29.213 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:29.240 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:29.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:29.242 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:44:29.343 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:29.387 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:29.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:29.390 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:44:29.491 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:29.515 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:29.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:29.524 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:29.624 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:29.648 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:29.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:29.653 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:44:29.754 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:29.779 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:29.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:29.782 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:44:29.882 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:29.912 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:29.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:29.917 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:30.018 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:30.026 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:29 smithi016 bash[75004]: cluster 2023-09-23T19:44:28.268721+0000 mgr.y (mgr.25325) 1111 : cluster 0 pgmap v1076: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 702 op/s 2023-09-23T19:44:30.026 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:29 smithi016 bash[77995]: cluster 2023-09-23T19:44:28.268721+0000 mgr.y (mgr.25325) 1111 : cluster 0 pgmap v1076: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 702 op/s 2023-09-23T19:44:30.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:29 smithi092 bash[58855]: cluster 2023-09-23T19:44:28.268721+0000 mgr.y (mgr.25325) 1111 : cluster 0 pgmap v1076: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 702 op/s 2023-09-23T19:44:30.051 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:30.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:30.054 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:30.155 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:30.179 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:30.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:30.183 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:44:30.283 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:30.309 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:30.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:30.312 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:44:30.412 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:30.451 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:30.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:30.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:30.557 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:30.581 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:30.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:30.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:44:30.691 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:30.728 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:30.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:30.731 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:44:30.832 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:30.863 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:30.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:30.868 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:44:30.969 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:31.000 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:31.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:31.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:44:31.103 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:31.130 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:31.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:31.135 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:31.235 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:31.265 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:31.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:31.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:44:31.371 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:31.396 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:31.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:31.405 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:44:31.506 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:31.546 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:31.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:31.552 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:31.652 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:31.687 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:31.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:31.692 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:44:31.777 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:44:31.777 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:44:31.793 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:31.825 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:31.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:31.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:44:31.928 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:31.964 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:31.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:31.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:44:32.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:31 smithi092 bash[58855]: cluster 2023-09-23T19:44:30.269521+0000 mgr.y (mgr.25325) 1112 : cluster 0 pgmap v1077: 89 pgs: 89 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2023-09-23T19:44:32.067 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:32.077 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:31 smithi016 bash[75004]: cluster 2023-09-23T19:44:30.269521+0000 mgr.y (mgr.25325) 1112 : cluster 0 pgmap v1077: 89 pgs: 89 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2023-09-23T19:44:32.078 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:31 smithi016 bash[77995]: cluster 2023-09-23T19:44:30.269521+0000 mgr.y (mgr.25325) 1112 : cluster 0 pgmap v1077: 89 pgs: 89 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2023-09-23T19:44:32.092 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:32.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:32.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:32.200 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:32.235 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:32.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:32.243 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:44:32.343 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:32.390 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:32.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:32.394 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:32.494 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:32.519 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:32.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:32.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:44:32.591 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:44:32.625 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:32.658 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:32.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:32.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:44:32.760 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:32.791 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:32.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:32.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:44:32.897 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:32.920 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:32.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:32.925 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:44:33.025 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:33.049 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:33.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:33.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:44:33.152 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:33.175 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:33.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:33.177 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:44:33.278 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:33.317 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:33.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:33.324 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:44:33.425 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:33.449 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:33.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:33.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:33.553 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:33.590 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:33.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:33.598 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:44:33.699 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:33.741 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:33.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:33.743 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:44:33.844 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:33.880 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:33.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:33.883 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:33.984 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:34.011 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:34.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:34.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:44:34.117 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:34.126 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:33 smithi092 bash[58855]: cluster 2023-09-23T19:44:32.270051+0000 mgr.y (mgr.25325) 1113 : cluster 0 pgmap v1078: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 690 op/s 2023-09-23T19:44:34.126 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:33 smithi092 bash[58855]: audit 2023-09-23T19:44:33.302611+0000 mon.a (mon.0) 1401 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:44:34.145 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:34.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:34.150 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:44:34.250 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:34.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:33 smithi016 bash[77995]: cluster 2023-09-23T19:44:32.270051+0000 mgr.y (mgr.25325) 1113 : cluster 0 pgmap v1078: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 690 op/s 2023-09-23T19:44:34.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:33 smithi016 bash[77995]: audit 2023-09-23T19:44:33.302611+0000 mon.a (mon.0) 1401 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:44:34.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:33 smithi016 bash[75004]: cluster 2023-09-23T19:44:32.270051+0000 mgr.y (mgr.25325) 1113 : cluster 0 pgmap v1078: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 690 op/s 2023-09-23T19:44:34.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:33 smithi016 bash[75004]: audit 2023-09-23T19:44:33.302611+0000 mon.a (mon.0) 1401 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:44:34.287 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:34.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:34.291 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:44:34.392 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:34.420 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:34.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:34.424 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:34.525 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:34.550 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:34.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:34.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:44:34.655 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:34.687 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:34.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:34.689 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:34.790 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:34.812 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:34.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:34.816 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:44:34.916 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:34.950 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:34.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:34.956 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:44:35.057 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:35.087 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:35.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:35.092 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:35.192 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:35.218 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:35.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:35.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:44:35.323 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:35.359 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:35.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:35.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:44:35.464 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:35.487 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:35.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:35.492 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:44:35.592 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:35.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:35.648 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-09-23T19:44:35.648 INFO:teuthology.orchestra.run.smithi016.stdout: "size": 20, 2023-09-23T19:44:35.648 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 600, 2023-09-23T19:44:35.648 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [ 2023-09-23T19:44:35.648 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.648 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45671.0:208985 9.d e137/119 9:b5103cf5:::benchmark_data_smithi016_113233_object100603:head v 137'13051, mlcod=137'13049)", 2023-09-23T19:44:35.649 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:35:26.608279+0000", 2023-09-23T19:44:35.649 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 549.02506289400003, 2023-09-23T19:44:35.649 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.21262897, 2023-09-23T19:44:35.649 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:44:35.649 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:44:35.649 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:44:35.649 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.649 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:44:35.650 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.608279+0000", 2023-09-23T19:44:35.650 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:44:35.650 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.650 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.650 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:44:35.650 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.608279+0000", 2023-09-23T19:44:35.650 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:44:35.650 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.651 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.651 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:44:35.651 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.608278+0000", 2023-09-23T19:44:35.651 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:44:35.651 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.651 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.651 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:44:35.652 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.608287+0000", 2023-09-23T19:44:35.652 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.4009999999999997e-06 2023-09-23T19:44:35.652 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.652 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.652 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:44:35.652 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.608298+0000", 2023-09-23T19:44:35.652 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1348e-05 2023-09-23T19:44:35.652 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.653 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.653 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:44:35.653 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.608305+0000", 2023-09-23T19:44:35.653 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.6510000000000004e-06 2023-09-23T19:44:35.653 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.653 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.653 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:44:35.653 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.608350+0000", 2023-09-23T19:44:35.654 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.5500999999999997e-05 2023-09-23T19:44:35.654 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.654 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.654 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:44:35.654 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.608378+0000", 2023-09-23T19:44:35.654 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.8374e-05 2023-09-23T19:44:35.654 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.655 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.655 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:44:35.655 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.820849+0000", 2023-09-23T19:44:35.655 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.21247050300000001 2023-09-23T19:44:35.655 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.655 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.655 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:44:35.655 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.820908+0000", 2023-09-23T19:44:35.656 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.9021999999999998e-05 2023-09-23T19:44:35.656 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:44:35.656 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:44:35.656 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:44:35.656 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.656 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.656 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45671.0:208989 9.d e137/119 9:bbab1653:::benchmark_data_smithi016_113233_object100607:head v 137'13052, mlcod=137'13049)", 2023-09-23T19:44:35.657 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:35:26.612132+0000", 2023-09-23T19:44:35.657 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 549.02120986, 2023-09-23T19:44:35.657 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.225652405, 2023-09-23T19:44:35.657 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:44:35.657 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:44:35.657 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:44:35.657 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.657 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:44:35.657 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.612132+0000", 2023-09-23T19:44:35.658 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:44:35.658 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.658 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.658 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:44:35.658 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.612132+0000", 2023-09-23T19:44:35.658 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:44:35.658 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.659 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.659 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:44:35.659 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.612132+0000", 2023-09-23T19:44:35.659 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T19:44:35.659 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.659 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.659 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:44:35.659 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.612136+0000", 2023-09-23T19:44:35.660 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.391e-06 2023-09-23T19:44:35.660 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.660 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.660 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:44:35.660 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.612143+0000", 2023-09-23T19:44:35.660 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.8770000000000004e-06 2023-09-23T19:44:35.660 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.661 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.661 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:44:35.661 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.612148+0000", 2023-09-23T19:44:35.661 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.4410000000000003e-06 2023-09-23T19:44:35.661 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.661 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.661 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:44:35.661 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.612558+0000", 2023-09-23T19:44:35.662 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000410448 2023-09-23T19:44:35.662 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.662 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.662 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:44:35.662 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.612577+0000", 2023-09-23T19:44:35.662 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.8706999999999999e-05 2023-09-23T19:44:35.662 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.663 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.663 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:44:35.663 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.837673+0000", 2023-09-23T19:44:35.663 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.22509628600000001 2023-09-23T19:44:35.663 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.663 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.663 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:44:35.663 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.837784+0000", 2023-09-23T19:44:35.664 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000111533 2023-09-23T19:44:35.664 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:44:35.664 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:44:35.664 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:44:35.664 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.664 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.664 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45671.0:208997 9.0 9:05dc60b0:::benchmark_data_smithi016_113233_object100615:head [delete] snapc 0=[] ondisk+write+known_if_redirected e137)", 2023-09-23T19:44:35.664 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:35:26.615043+0000", 2023-09-23T19:44:35.665 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 549.01829916600002, 2023-09-23T19:44:35.665 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.21171452399999999, 2023-09-23T19:44:35.665 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:44:35.665 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:44:35.665 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:44:35.665 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45671", 2023-09-23T19:44:35.665 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:44:35.666 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 208997 2023-09-23T19:44:35.666 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.666 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:44:35.666 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.666 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:44:35.666 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.615043+0000", 2023-09-23T19:44:35.667 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:44:35.667 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.667 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.667 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:44:35.667 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.615043+0000", 2023-09-23T19:44:35.667 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:44:35.667 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.668 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.668 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:44:35.668 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.615041+0000", 2023-09-23T19:44:35.668 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-09-23T19:44:35.668 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.668 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.668 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:44:35.668 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.615052+0000", 2023-09-23T19:44:35.669 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0886e-05 2023-09-23T19:44:35.669 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.669 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.669 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:44:35.669 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.615071+0000", 2023-09-23T19:44:35.669 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.9140999999999999e-05 2023-09-23T19:44:35.669 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.670 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.670 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:44:35.670 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.615083+0000", 2023-09-23T19:44:35.670 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2182e-05 2023-09-23T19:44:35.670 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.670 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.671 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:44:35.671 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.615166+0000", 2023-09-23T19:44:35.671 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.2787999999999994e-05 2023-09-23T19:44:35.671 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.671 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.671 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:44:35.671 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.615315+0000", 2023-09-23T19:44:35.672 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00014920299999999999 2023-09-23T19:44:35.672 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.672 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.672 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 2,6", 2023-09-23T19:44:35.672 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.615426+0000", 2023-09-23T19:44:35.672 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000111018 2023-09-23T19:44:35.673 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.673 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.673 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:44:35.673 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.619749+0000", 2023-09-23T19:44:35.673 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0043230849999999999 2023-09-23T19:44:35.673 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.673 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.674 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:44:35.674 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.820806+0000", 2023-09-23T19:44:35.674 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.20105654000000001 2023-09-23T19:44:35.674 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.674 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.674 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:44:35.675 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.826602+0000", 2023-09-23T19:44:35.675 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0057966609999999998 2023-09-23T19:44:35.675 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.675 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.675 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:44:35.675 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.826682+0000", 2023-09-23T19:44:35.676 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.9899000000000002e-05 2023-09-23T19:44:35.676 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.676 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.676 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:44:35.676 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.826757+0000", 2023-09-23T19:44:35.676 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.5217999999999997e-05 2023-09-23T19:44:35.676 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:44:35.677 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:44:35.677 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:44:35.677 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.677 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.677 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45671.0:208995 9.e e137/119 9:7a040566:::benchmark_data_smithi016_113233_object100613:head v 137'13188, mlcod=137'13185)", 2023-09-23T19:44:35.677 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:35:26.615077+0000", 2023-09-23T19:44:35.677 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 549.01826477199995, 2023-09-23T19:44:35.678 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.205860126, 2023-09-23T19:44:35.678 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:44:35.678 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:44:35.678 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:44:35.678 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.678 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:44:35.679 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.615077+0000", 2023-09-23T19:44:35.679 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:44:35.679 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.679 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.679 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:44:35.679 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.615077+0000", 2023-09-23T19:44:35.679 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:44:35.680 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.680 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.680 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:44:35.680 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.615076+0000", 2023-09-23T19:44:35.680 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:44:35.680 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.680 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.681 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:44:35.681 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.615098+0000", 2023-09-23T19:44:35.681 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.1946e-05 2023-09-23T19:44:35.681 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.681 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.681 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:44:35.682 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.615111+0000", 2023-09-23T19:44:35.682 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2544999999999999e-05 2023-09-23T19:44:35.682 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.682 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.682 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:44:35.682 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.615119+0000", 2023-09-23T19:44:35.683 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.5059999999999995e-06 2023-09-23T19:44:35.683 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.683 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.683 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:44:35.683 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.619925+0000", 2023-09-23T19:44:35.683 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0048052499999999996 2023-09-23T19:44:35.683 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.684 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.684 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:44:35.684 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.619968+0000", 2023-09-23T19:44:35.684 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.3316999999999999e-05 2023-09-23T19:44:35.684 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.684 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.684 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:44:35.685 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.820865+0000", 2023-09-23T19:44:35.685 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.20089712900000001 2023-09-23T19:44:35.685 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.685 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.685 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:44:35.685 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.820937+0000", 2023-09-23T19:44:35.685 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.2251999999999999e-05 2023-09-23T19:44:35.686 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:44:35.686 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:44:35.686 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:44:35.686 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.686 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.686 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45671.0:208998 9.f e137/119 9:fe02e78b:::benchmark_data_smithi016_113233_object100616:head v 137'13030, mlcod=137'13028)", 2023-09-23T19:44:35.686 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:35:26.620103+0000", 2023-09-23T19:44:35.687 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 549.01323900099999, 2023-09-23T19:44:35.687 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.20097216000000001, 2023-09-23T19:44:35.687 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:44:35.687 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:44:35.687 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:44:35.687 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.688 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:44:35.688 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.620103+0000", 2023-09-23T19:44:35.688 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:44:35.688 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.688 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.688 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:44:35.688 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.620103+0000", 2023-09-23T19:44:35.689 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:44:35.689 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.689 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.689 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:44:35.689 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.620101+0000", 2023-09-23T19:44:35.689 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-09-23T19:44:35.690 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.690 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.690 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:44:35.690 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.620138+0000", 2023-09-23T19:44:35.690 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.6656999999999998e-05 2023-09-23T19:44:35.690 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.690 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.691 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:44:35.691 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.620158+0000", 2023-09-23T19:44:35.691 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.9624e-05 2023-09-23T19:44:35.691 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.691 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.691 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:44:35.691 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.620172+0000", 2023-09-23T19:44:35.692 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.394e-05 2023-09-23T19:44:35.692 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.692 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.692 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:44:35.692 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.620272+0000", 2023-09-23T19:44:35.692 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000100338 2023-09-23T19:44:35.692 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.693 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.693 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:44:35.693 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.620286+0000", 2023-09-23T19:44:35.693 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3937999999999999e-05 2023-09-23T19:44:35.693 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.693 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.694 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:44:35.694 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.820990+0000", 2023-09-23T19:44:35.694 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.20070369399999999 2023-09-23T19:44:35.694 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.694 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.694 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:44:35.694 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.821075+0000", 2023-09-23T19:44:35.695 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.5578e-05 2023-09-23T19:44:35.695 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:44:35.695 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:44:35.695 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:44:35.695 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.695 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.695 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop_reply(client.45671.0:208997 9.0 e137/119 ondisk, result = 0)", 2023-09-23T19:44:35.696 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:35:26.620282+0000", 2023-09-23T19:44:35.696 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 549.01305954199995, 2023-09-23T19:44:35.696 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.20053090500000001, 2023-09-23T19:44:35.696 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:44:35.696 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "started", 2023-09-23T19:44:35.696 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:44:35.697 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.697 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:44:35.697 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.620282+0000", 2023-09-23T19:44:35.697 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:44:35.697 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.697 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.698 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:44:35.698 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.620282+0000", 2023-09-23T19:44:35.698 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:44:35.698 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.698 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.698 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:44:35.698 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.620281+0000", 2023-09-23T19:44:35.699 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:44:35.699 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.699 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.699 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:44:35.699 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.620291+0000", 2023-09-23T19:44:35.699 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.0259999999999996e-06 2023-09-23T19:44:35.700 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.700 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.700 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:44:35.700 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.620307+0000", 2023-09-23T19:44:35.700 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.6857000000000001e-05 2023-09-23T19:44:35.700 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.700 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.701 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:44:35.701 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.620339+0000", 2023-09-23T19:44:35.701 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.1537e-05 2023-09-23T19:44:35.701 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.701 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.701 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:44:35.702 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.620385+0000", 2023-09-23T19:44:35.702 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.6177999999999997e-05 2023-09-23T19:44:35.702 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.702 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.702 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:44:35.702 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.820782+0000", 2023-09-23T19:44:35.702 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.20039693 2023-09-23T19:44:35.703 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.703 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.703 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:44:35.703 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.820813+0000", 2023-09-23T19:44:35.703 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.1294000000000002e-05 2023-09-23T19:44:35.703 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:44:35.703 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:44:35.704 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:44:35.704 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.704 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.704 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:35.705 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45671.0:208990 9.d e137/119 9:b24ecd90:::benchmark_data_smithi016_113233_object100608:head v 137'13053, mlcod=137'13049)", 2023-09-23T19:44:35.705 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:35:26.620706+0000", 2023-09-23T19:44:35.705 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 549.01263637399995, 2023-09-23T19:44:35.705 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.217123129, 2023-09-23T19:44:35.705 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:44:35.706 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:44:35.706 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:44:35.706 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.706 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:44:35.706 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.620706+0000", 2023-09-23T19:44:35.706 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:44:35.707 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.707 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.707 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:44:35.707 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.620706+0000", 2023-09-23T19:44:35.707 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:44:35.707 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.708 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.708 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:44:35.708 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.620704+0000", 2023-09-23T19:44:35.708 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:44:35.708 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.708 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.709 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:44:35.709 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.620714+0000", 2023-09-23T19:44:35.709 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.4199999999999996e-06 2023-09-23T19:44:35.709 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.709 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.709 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:44:35.709 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.820827+0000", 2023-09-23T19:44:35.709 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.20011351099999999 2023-09-23T19:44:35.710 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.710 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.710 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:44:35.710 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.820864+0000", 2023-09-23T19:44:35.710 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.6662999999999998e-05 2023-09-23T19:44:35.710 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.711 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.711 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:44:35.711 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.820931+0000", 2023-09-23T19:44:35.711 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.7158999999999999e-05 2023-09-23T19:44:35.711 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.711 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.712 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:44:35.712 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.820961+0000", 2023-09-23T19:44:35.712 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.976e-05 2023-09-23T19:44:35.712 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.712 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.712 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:44:35.712 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.837794+0000", 2023-09-23T19:44:35.713 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.016833013000000001 2023-09-23T19:44:35.713 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.713 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.713 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:44:35.713 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.837829+0000", 2023-09-23T19:44:35.713 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.4709999999999998e-05 2023-09-23T19:44:35.713 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:44:35.714 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:44:35.714 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:44:35.714 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.714 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.714 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45671.0:209000 9.4 9:2109823a:::benchmark_data_smithi016_113233_object100618:head [delete] snapc 0=[] ondisk+write+known_if_redirected e137)", 2023-09-23T19:44:35.714 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:35:26.621957+0000", 2023-09-23T19:44:35.714 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 549.01138534100005, 2023-09-23T19:44:35.715 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.21210473799999999, 2023-09-23T19:44:35.715 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:44:35.715 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:44:35.715 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:44:35.715 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45671", 2023-09-23T19:44:35.715 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2663554924", 2023-09-23T19:44:35.716 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 209000 2023-09-23T19:44:35.716 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.716 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:44:35.716 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.716 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:44:35.716 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.621957+0000", 2023-09-23T19:44:35.717 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:44:35.717 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.717 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.717 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:44:35.717 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.621957+0000", 2023-09-23T19:44:35.717 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:44:35.717 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.718 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.718 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:44:35.718 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.621955+0000", 2023-09-23T19:44:35.718 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-09-23T19:44:35.718 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.718 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.719 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:44:35.719 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.621962+0000", 2023-09-23T19:44:35.719 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.5680000000000002e-06 2023-09-23T19:44:35.719 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.719 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.719 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:44:35.719 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.820839+0000", 2023-09-23T19:44:35.719 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.198876935 2023-09-23T19:44:35.720 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.720 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.720 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:44:35.720 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.820877+0000", 2023-09-23T19:44:35.720 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.8115999999999999e-05 2023-09-23T19:44:35.720 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.721 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.721 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:44:35.721 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.821003+0000", 2023-09-23T19:44:35.721 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000125551 2023-09-23T19:44:35.721 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.721 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.721 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:44:35.722 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.821254+0000", 2023-09-23T19:44:35.722 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00025106199999999998 2023-09-23T19:44:35.722 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.722 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.722 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 3,4", 2023-09-23T19:44:35.722 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.821458+0000", 2023-09-23T19:44:35.723 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00020381800000000001 2023-09-23T19:44:35.723 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.723 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.723 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:44:35.723 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.827288+0000", 2023-09-23T19:44:35.723 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0058305459999999998 2023-09-23T19:44:35.723 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.723 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.724 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:44:35.724 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.832114+0000", 2023-09-23T19:44:35.724 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0048260860000000003 2023-09-23T19:44:35.724 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.724 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.724 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:44:35.724 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.833956+0000", 2023-09-23T19:44:35.725 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0018413189999999999 2023-09-23T19:44:35.725 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.725 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.725 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:44:35.725 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.834010+0000", 2023-09-23T19:44:35.725 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.465e-05 2023-09-23T19:44:35.725 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.726 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.726 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:44:35.726 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:35:26.834061+0000", 2023-09-23T19:44:35.726 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.1112000000000001e-05 2023-09-23T19:44:35.726 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:44:35.726 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:44:35.727 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:44:35.727 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.727 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.727 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45947.0:79834 10.2 10:432191b2:::benchmark_data_smithi016_136365_object79833:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e152)", 2023-09-23T19:44:35.727 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:37:30.744209+0000", 2023-09-23T19:44:35.727 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 424.88913309899999, 2023-09-23T19:44:35.727 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.190664373, 2023-09-23T19:44:35.728 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:44:35.728 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:44:35.728 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:44:35.728 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45947", 2023-09-23T19:44:35.728 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/4039191752", 2023-09-23T19:44:35.728 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 79834 2023-09-23T19:44:35.728 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.729 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:44:35.729 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.729 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:44:35.729 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.744209+0000", 2023-09-23T19:44:35.729 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:44:35.729 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.729 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.730 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:44:35.730 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.744209+0000", 2023-09-23T19:44:35.730 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:44:35.730 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.730 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.730 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:44:35.730 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.744206+0000", 2023-09-23T19:44:35.731 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999976 2023-09-23T19:44:35.731 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.731 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.731 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:44:35.731 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.744261+0000", 2023-09-23T19:44:35.731 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.4487999999999997e-05 2023-09-23T19:44:35.731 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.732 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.732 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:44:35.732 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.744290+0000", 2023-09-23T19:44:35.732 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.9420999999999998e-05 2023-09-23T19:44:35.732 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.732 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.732 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:44:35.733 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.744304+0000", 2023-09-23T19:44:35.733 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3823e-05 2023-09-23T19:44:35.733 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.733 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.733 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:44:35.733 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.744371+0000", 2023-09-23T19:44:35.733 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.6662000000000007e-05 2023-09-23T19:44:35.734 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.734 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.734 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:44:35.734 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.744500+0000", 2023-09-23T19:44:35.734 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00012901500000000001 2023-09-23T19:44:35.734 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.735 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.735 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,4", 2023-09-23T19:44:35.735 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.744612+0000", 2023-09-23T19:44:35.735 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00011184999999999999 2023-09-23T19:44:35.735 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.735 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.735 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:44:35.736 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.747045+0000", 2023-09-23T19:44:35.736 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0024333179999999999 2023-09-23T19:44:35.736 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.736 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.736 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:44:35.736 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.749209+0000", 2023-09-23T19:44:35.736 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0021636889999999999 2023-09-23T19:44:35.737 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.737 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.737 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:44:35.737 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.934762+0000", 2023-09-23T19:44:35.737 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.18555321599999999 2023-09-23T19:44:35.737 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.737 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.738 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:44:35.738 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.934821+0000", 2023-09-23T19:44:35.738 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.9378e-05 2023-09-23T19:44:35.738 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.738 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.738 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:44:35.739 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.934873+0000", 2023-09-23T19:44:35.739 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.2002000000000001e-05 2023-09-23T19:44:35.739 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:44:35.739 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:44:35.739 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:44:35.739 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.739 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.739 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45947.0:79835 10.2 10:4a2b11b9:::benchmark_data_smithi016_136365_object79834:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e152)", 2023-09-23T19:44:35.740 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:37:30.747176+0000", 2023-09-23T19:44:35.740 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 424.88616636699999, 2023-09-23T19:44:35.740 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.18782270100000001, 2023-09-23T19:44:35.740 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:44:35.740 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:44:35.740 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:44:35.740 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45947", 2023-09-23T19:44:35.740 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/4039191752", 2023-09-23T19:44:35.740 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 79835 2023-09-23T19:44:35.740 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.740 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:44:35.740 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.741 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:44:35.741 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.747176+0000", 2023-09-23T19:44:35.741 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:44:35.741 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.741 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.741 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:44:35.741 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.747176+0000", 2023-09-23T19:44:35.741 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:44:35.741 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.741 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.741 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:44:35.741 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.747174+0000", 2023-09-23T19:44:35.742 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-09-23T19:44:35.742 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.742 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.742 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:44:35.742 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.747207+0000", 2023-09-23T19:44:35.742 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.3448999999999998e-05 2023-09-23T19:44:35.742 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.742 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.742 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:44:35.742 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.747225+0000", 2023-09-23T19:44:35.742 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.7697999999999999e-05 2023-09-23T19:44:35.742 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.743 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.743 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:44:35.743 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.747235+0000", 2023-09-23T19:44:35.743 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.022e-05 2023-09-23T19:44:35.743 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.743 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.743 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:44:35.743 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.747286+0000", 2023-09-23T19:44:35.743 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.0828000000000002e-05 2023-09-23T19:44:35.743 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.743 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.743 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:44:35.744 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.747391+0000", 2023-09-23T19:44:35.744 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00010524200000000001 2023-09-23T19:44:35.744 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.744 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.744 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,4", 2023-09-23T19:44:35.744 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.747501+0000", 2023-09-23T19:44:35.744 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000109966 2023-09-23T19:44:35.744 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.744 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.744 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:44:35.744 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.749264+0000", 2023-09-23T19:44:35.744 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0017630059999999999 2023-09-23T19:44:35.745 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.745 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.745 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:44:35.745 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.749372+0000", 2023-09-23T19:44:35.745 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000107371 2023-09-23T19:44:35.745 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.745 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.745 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:44:35.745 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.934919+0000", 2023-09-23T19:44:35.745 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.185547394 2023-09-23T19:44:35.745 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.745 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.746 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:44:35.746 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.934961+0000", 2023-09-23T19:44:35.746 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.1940000000000002e-05 2023-09-23T19:44:35.746 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.746 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.746 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:44:35.746 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:37:30.934998+0000", 2023-09-23T19:44:35.746 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.7277000000000002e-05 2023-09-23T19:44:35.746 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:44:35.746 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:44:35.746 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:44:35.746 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.747 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.747 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45947.0:207473 10.2 10:42a949ac:::benchmark_data_smithi016_136365_object97339:head [delete] snapc 0=[] ondisk+write+known_if_redirected e171)", 2023-09-23T19:44:35.747 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:40:28.088246+0000", 2023-09-23T19:44:35.747 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 247.54509593099999, 2023-09-23T19:44:35.747 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.26844422600000001, 2023-09-23T19:44:35.747 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:44:35.747 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:44:35.747 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:44:35.747 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45947", 2023-09-23T19:44:35.747 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/4039191752", 2023-09-23T19:44:35.747 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 207473 2023-09-23T19:44:35.747 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.748 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:44:35.748 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.748 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:44:35.748 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.088246+0000", 2023-09-23T19:44:35.748 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:44:35.748 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.748 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.748 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:44:35.748 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.088246+0000", 2023-09-23T19:44:35.748 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:44:35.748 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.748 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.749 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:44:35.749 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.088244+0000", 2023-09-23T19:44:35.749 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-09-23T19:44:35.749 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.749 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.749 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:44:35.749 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.088253+0000", 2023-09-23T19:44:35.749 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.8589999999999994e-06 2023-09-23T19:44:35.749 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.749 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.749 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:44:35.749 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.088268+0000", 2023-09-23T19:44:35.750 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.519e-05 2023-09-23T19:44:35.750 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.750 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.750 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:44:35.750 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.088277+0000", 2023-09-23T19:44:35.750 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.4679999999999992e-06 2023-09-23T19:44:35.750 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.750 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.750 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:44:35.750 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.088434+0000", 2023-09-23T19:44:35.750 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000157236 2023-09-23T19:44:35.750 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.751 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.751 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:44:35.751 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.088540+0000", 2023-09-23T19:44:35.751 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000105517 2023-09-23T19:44:35.751 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.751 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.751 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,4", 2023-09-23T19:44:35.751 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.088617+0000", 2023-09-23T19:44:35.751 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.7578999999999997e-05 2023-09-23T19:44:35.751 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.751 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.751 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:44:35.751 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.092417+0000", 2023-09-23T19:44:35.752 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:35.753 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0038002679999999999 2023-09-23T19:44:35.753 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.753 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.753 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:44:35.753 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.104312+0000", 2023-09-23T19:44:35.753 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.011894978 2023-09-23T19:44:35.753 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.753 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.753 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:44:35.754 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.355871+0000", 2023-09-23T19:44:35.754 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.251558848 2023-09-23T19:44:35.754 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.754 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.754 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:44:35.754 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.356637+0000", 2023-09-23T19:44:35.754 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00076531500000000003 2023-09-23T19:44:35.754 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.754 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.754 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:44:35.754 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.356690+0000", 2023-09-23T19:44:35.754 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.3726e-05 2023-09-23T19:44:35.755 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:44:35.755 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:44:35.755 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:44:35.755 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.755 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.755 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.36243.0:105197 11.f e190/180 11:f6c01383:::benchmark_data_smithi016_160148_object105196:head v 190'6752, mlcod=190'6749)", 2023-09-23T19:44:35.755 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:43:11.556733+0000", 2023-09-23T19:44:35.755 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 84.076609167000001, 2023-09-23T19:44:35.755 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.17318994600000001, 2023-09-23T19:44:35.755 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:44:35.755 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:44:35.755 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:44:35.756 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.756 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:44:35.756 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.556733+0000", 2023-09-23T19:44:35.756 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:44:35.756 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.756 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.756 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:44:35.756 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.556733+0000", 2023-09-23T19:44:35.756 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:44:35.756 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.756 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.756 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:44:35.757 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.556732+0000", 2023-09-23T19:44:35.757 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:44:35.757 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.757 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.757 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:44:35.757 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.556765+0000", 2023-09-23T19:44:35.757 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.3494000000000002e-05 2023-09-23T19:44:35.757 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.757 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.757 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:44:35.757 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.556775+0000", 2023-09-23T19:44:35.757 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.3530000000000001e-06 2023-09-23T19:44:35.758 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.758 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.758 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:44:35.758 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.556780+0000", 2023-09-23T19:44:35.758 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.9339999999999997e-06 2023-09-23T19:44:35.758 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.758 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.758 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:44:35.758 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.556830+0000", 2023-09-23T19:44:35.758 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.9876000000000002e-05 2023-09-23T19:44:35.758 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.759 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.759 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:44:35.759 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.556855+0000", 2023-09-23T19:44:35.759 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.4870000000000001e-05 2023-09-23T19:44:35.759 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.759 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.759 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:44:35.759 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.729792+0000", 2023-09-23T19:44:35.759 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.17293725300000001 2023-09-23T19:44:35.759 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.759 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.759 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:44:35.760 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.729923+0000", 2023-09-23T19:44:35.760 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00013093499999999999 2023-09-23T19:44:35.760 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:44:35.760 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:44:35.760 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:44:35.760 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.760 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.760 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.36243.0:105204 11.b 11:d2046e17:::benchmark_data_smithi016_160148_object105203:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e190)", 2023-09-23T19:44:35.760 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:43:11.556961+0000", 2023-09-23T19:44:35.760 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 84.076381201000004, 2023-09-23T19:44:35.760 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.16188370899999999, 2023-09-23T19:44:35.760 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:44:35.761 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:44:35.761 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:44:35.761 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.36243", 2023-09-23T19:44:35.761 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1280038109", 2023-09-23T19:44:35.761 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 105204 2023-09-23T19:44:35.761 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.761 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:44:35.761 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.761 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:44:35.761 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.556961+0000", 2023-09-23T19:44:35.761 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:44:35.762 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.762 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.762 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:44:35.762 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.556961+0000", 2023-09-23T19:44:35.762 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:44:35.762 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.762 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.762 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:44:35.762 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.556959+0000", 2023-09-23T19:44:35.762 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-09-23T19:44:35.762 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.762 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.763 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:44:35.763 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.556988+0000", 2023-09-23T19:44:35.763 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.8432000000000002e-05 2023-09-23T19:44:35.763 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.763 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.763 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:44:35.763 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.557000+0000", 2023-09-23T19:44:35.763 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2332e-05 2023-09-23T19:44:35.763 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.763 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.763 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:44:35.763 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.557008+0000", 2023-09-23T19:44:35.764 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.0730000000000005e-06 2023-09-23T19:44:35.764 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.764 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.764 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:44:35.764 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.557051+0000", 2023-09-23T19:44:35.764 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.2781000000000003e-05 2023-09-23T19:44:35.764 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.764 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.764 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:44:35.764 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.557195+0000", 2023-09-23T19:44:35.764 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00014452700000000001 2023-09-23T19:44:35.764 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.765 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.765 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,3", 2023-09-23T19:44:35.765 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.557349+0000", 2023-09-23T19:44:35.765 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00015327599999999999 2023-09-23T19:44:35.765 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.765 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.765 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:44:35.765 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.717731+0000", 2023-09-23T19:44:35.765 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.16038263699999999 2023-09-23T19:44:35.765 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.765 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.765 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:44:35.766 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.717740+0000", 2023-09-23T19:44:35.766 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.5620000000000004e-06 2023-09-23T19:44:35.766 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.766 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.766 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:44:35.766 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.718556+0000", 2023-09-23T19:44:35.766 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00081608099999999997 2023-09-23T19:44:35.766 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.766 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.766 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:44:35.766 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.718675+0000", 2023-09-23T19:44:35.767 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000118829 2023-09-23T19:44:35.767 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.767 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.767 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:44:35.767 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.718844+0000", 2023-09-23T19:44:35.767 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00016959000000000001 2023-09-23T19:44:35.767 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:44:35.767 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:44:35.767 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:44:35.767 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.767 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.767 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.36243.0:105208 11.c 11:3eb35dca:::benchmark_data_smithi016_160148_object105207:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e190)", 2023-09-23T19:44:35.768 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:43:11.557274+0000", 2023-09-23T19:44:35.768 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 84.076068118999999, 2023-09-23T19:44:35.768 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.17526666900000001, 2023-09-23T19:44:35.768 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:44:35.768 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:44:35.768 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:44:35.768 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.36243", 2023-09-23T19:44:35.768 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1280038109", 2023-09-23T19:44:35.768 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 105208 2023-09-23T19:44:35.768 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.768 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:44:35.768 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.769 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:44:35.769 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.557274+0000", 2023-09-23T19:44:35.769 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:44:35.769 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.769 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.769 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:44:35.769 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.557274+0000", 2023-09-23T19:44:35.769 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:44:35.769 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.769 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.769 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:44:35.770 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.557272+0000", 2023-09-23T19:44:35.770 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-09-23T19:44:35.770 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.770 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.770 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:44:35.770 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.557305+0000", 2023-09-23T19:44:35.770 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.2396e-05 2023-09-23T19:44:35.770 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.770 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.770 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:44:35.770 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.557324+0000", 2023-09-23T19:44:35.770 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.9276e-05 2023-09-23T19:44:35.771 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.771 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.771 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:44:35.771 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.557336+0000", 2023-09-23T19:44:35.771 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1483e-05 2023-09-23T19:44:35.771 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.771 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.771 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:44:35.771 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.557584+0000", 2023-09-23T19:44:35.771 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00024880600000000002 2023-09-23T19:44:35.771 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.771 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.772 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:44:35.772 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.557919+0000", 2023-09-23T19:44:35.772 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00033508399999999998 2023-09-23T19:44:35.772 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.772 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.772 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 2,4", 2023-09-23T19:44:35.772 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.558122+0000", 2023-09-23T19:44:35.772 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00020218100000000001 2023-09-23T19:44:35.772 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.772 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.772 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:44:35.772 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.722644+0000", 2023-09-23T19:44:35.773 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.16452285 2023-09-23T19:44:35.773 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.773 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.773 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:44:35.773 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.722700+0000", 2023-09-23T19:44:35.773 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.5795999999999999e-05 2023-09-23T19:44:35.773 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.773 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.773 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:44:35.773 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.732455+0000", 2023-09-23T19:44:35.773 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0097542770000000004 2023-09-23T19:44:35.774 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.774 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.774 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:44:35.774 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.732500+0000", 2023-09-23T19:44:35.774 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.5219000000000002e-05 2023-09-23T19:44:35.774 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.774 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.774 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:44:35.774 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.732540+0000", 2023-09-23T19:44:35.774 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.0751000000000003e-05 2023-09-23T19:44:35.774 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:44:35.774 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:44:35.775 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:44:35.775 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.775 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.775 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.36243.0:105209 11.b 11:d81bb78d:::benchmark_data_smithi016_160148_object105208:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e190)", 2023-09-23T19:44:35.775 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:43:11.557379+0000", 2023-09-23T19:44:35.775 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 84.075962672000003, 2023-09-23T19:44:35.775 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.175150837, 2023-09-23T19:44:35.775 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:44:35.775 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:44:35.775 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:44:35.775 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.36243", 2023-09-23T19:44:35.775 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1280038109", 2023-09-23T19:44:35.776 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 105209 2023-09-23T19:44:35.776 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.776 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:44:35.776 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.776 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:44:35.776 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.557379+0000", 2023-09-23T19:44:35.776 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:44:35.776 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.776 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.776 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:44:35.776 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.557379+0000", 2023-09-23T19:44:35.777 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:44:35.777 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.777 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.777 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:44:35.777 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.557378+0000", 2023-09-23T19:44:35.777 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:44:35.777 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.777 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.777 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:44:35.777 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.557405+0000", 2023-09-23T19:44:35.777 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.6313999999999999e-05 2023-09-23T19:44:35.777 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.778 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.778 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:44:35.778 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.557416+0000", 2023-09-23T19:44:35.778 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1399999999999999e-05 2023-09-23T19:44:35.778 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.778 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.778 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:44:35.778 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.557424+0000", 2023-09-23T19:44:35.778 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.2260000000000005e-06 2023-09-23T19:44:35.778 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.778 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.778 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:44:35.779 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.711733+0000", 2023-09-23T19:44:35.779 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.154309165 2023-09-23T19:44:35.779 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.779 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.779 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:44:35.779 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.711935+0000", 2023-09-23T19:44:35.779 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00020122099999999999 2023-09-23T19:44:35.779 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.779 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.779 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,3", 2023-09-23T19:44:35.779 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.712109+0000", 2023-09-23T19:44:35.779 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00017477700000000001 2023-09-23T19:44:35.780 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.780 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.780 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:44:35.780 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.727385+0000", 2023-09-23T19:44:35.780 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.015275222 2023-09-23T19:44:35.780 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.780 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.780 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:44:35.780 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.727441+0000", 2023-09-23T19:44:35.780 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.6638000000000002e-05 2023-09-23T19:44:35.780 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.780 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.781 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:44:35.781 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.732425+0000", 2023-09-23T19:44:35.781 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0049837329999999997 2023-09-23T19:44:35.781 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.781 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.781 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:44:35.781 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.732480+0000", 2023-09-23T19:44:35.781 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.5133000000000002e-05 2023-09-23T19:44:35.781 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.781 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.781 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:44:35.781 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.732530+0000", 2023-09-23T19:44:35.782 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.9926000000000003e-05 2023-09-23T19:44:35.782 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:44:35.782 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:44:35.782 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:44:35.782 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.782 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.782 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.36243.0:105211 11.b 11:dbc04c90:::benchmark_data_smithi016_160148_object105210:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e190)", 2023-09-23T19:44:35.782 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:43:11.557436+0000", 2023-09-23T19:44:35.782 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 84.075905953000003, 2023-09-23T19:44:35.782 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.17530274700000001, 2023-09-23T19:44:35.782 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:44:35.783 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:44:35.783 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:44:35.783 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.36243", 2023-09-23T19:44:35.783 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1280038109", 2023-09-23T19:44:35.783 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 105211 2023-09-23T19:44:35.783 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.783 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:44:35.783 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.783 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:44:35.783 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.557436+0000", 2023-09-23T19:44:35.783 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:44:35.783 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.784 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.784 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:44:35.784 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.557436+0000", 2023-09-23T19:44:35.784 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:44:35.784 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.784 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.784 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:44:35.784 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.557435+0000", 2023-09-23T19:44:35.784 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:44:35.784 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.784 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.784 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:44:35.784 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.557463+0000", 2023-09-23T19:44:35.785 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.7682e-05 2023-09-23T19:44:35.785 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.785 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.785 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:44:35.785 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.557478+0000", 2023-09-23T19:44:35.785 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.5227e-05 2023-09-23T19:44:35.785 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.785 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.785 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:44:35.785 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.557486+0000", 2023-09-23T19:44:35.785 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.5619999999999998e-06 2023-09-23T19:44:35.785 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.786 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.786 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:44:35.786 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.718999+0000", 2023-09-23T19:44:35.786 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.16151301900000001 2023-09-23T19:44:35.786 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.786 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.786 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:44:35.786 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.719465+0000", 2023-09-23T19:44:35.786 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00046604699999999998 2023-09-23T19:44:35.786 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.786 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.786 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,3", 2023-09-23T19:44:35.787 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.720506+0000", 2023-09-23T19:44:35.787 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001041375 2023-09-23T19:44:35.787 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.787 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.787 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:44:35.787 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.732576+0000", 2023-09-23T19:44:35.787 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.012069458 2023-09-23T19:44:35.787 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.787 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.787 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:44:35.787 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.732620+0000", 2023-09-23T19:44:35.787 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.4672999999999997e-05 2023-09-23T19:44:35.788 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.788 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.788 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:44:35.788 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.732660+0000", 2023-09-23T19:44:35.788 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.9847e-05 2023-09-23T19:44:35.788 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.788 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.788 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:44:35.788 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.732702+0000", 2023-09-23T19:44:35.788 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.1894000000000002e-05 2023-09-23T19:44:35.788 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.788 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.789 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:44:35.789 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.732739+0000", 2023-09-23T19:44:35.789 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.6711000000000001e-05 2023-09-23T19:44:35.789 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:44:35.789 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:44:35.789 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:44:35.789 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.789 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.789 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.36243.0:105214 11.c 11:3bbc54f0:::benchmark_data_smithi016_160148_object105213:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e190)", 2023-09-23T19:44:35.789 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:43:11.557819+0000", 2023-09-23T19:44:35.789 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 84.075522801000005, 2023-09-23T19:44:35.789 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.17480869199999999, 2023-09-23T19:44:35.790 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:44:35.790 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:44:35.790 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:44:35.790 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.36243", 2023-09-23T19:44:35.790 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1280038109", 2023-09-23T19:44:35.790 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 105214 2023-09-23T19:44:35.790 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.790 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:44:35.790 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.790 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:44:35.790 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.557819+0000", 2023-09-23T19:44:35.790 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:44:35.791 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.791 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.791 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:44:35.791 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.557819+0000", 2023-09-23T19:44:35.791 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:44:35.791 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.791 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.791 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:44:35.791 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.557818+0000", 2023-09-23T19:44:35.791 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:44:35.791 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.791 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.792 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:44:35.792 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.557848+0000", 2023-09-23T19:44:35.792 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.9553999999999999e-05 2023-09-23T19:44:35.792 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.792 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.792 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:44:35.792 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.557865+0000", 2023-09-23T19:44:35.792 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.753e-05 2023-09-23T19:44:35.792 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.792 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.792 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:44:35.792 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.557875+0000", 2023-09-23T19:44:35.792 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.024e-05 2023-09-23T19:44:35.793 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.793 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.793 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:44:35.793 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.715629+0000", 2023-09-23T19:44:35.793 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.15775339899999999 2023-09-23T19:44:35.793 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.793 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.793 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:44:35.793 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.715842+0000", 2023-09-23T19:44:35.793 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00021323300000000001 2023-09-23T19:44:35.793 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.793 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.794 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 2,4", 2023-09-23T19:44:35.794 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.716465+0000", 2023-09-23T19:44:35.794 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00062334499999999998 2023-09-23T19:44:35.794 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.794 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.794 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:44:35.794 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.732409+0000", 2023-09-23T19:44:35.794 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.015943507999999999 2023-09-23T19:44:35.794 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.794 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.794 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:44:35.794 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.732447+0000", 2023-09-23T19:44:35.795 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.8643999999999998e-05 2023-09-23T19:44:35.795 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.795 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.795 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:44:35.795 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.732549+0000", 2023-09-23T19:44:35.795 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000101253 2023-09-23T19:44:35.795 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.795 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.795 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:44:35.795 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.732594+0000", 2023-09-23T19:44:35.795 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.4805999999999998e-05 2023-09-23T19:44:35.795 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.796 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.796 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:44:35.796 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.732628+0000", 2023-09-23T19:44:35.796 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.4294000000000001e-05 2023-09-23T19:44:35.796 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:44:35.796 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:44:35.796 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:44:35.796 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.796 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.796 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.36243.0:105212 11.f e190/180 11:fb6f74d3:::benchmark_data_smithi016_160148_object105211:head v 190'6753, mlcod=190'6750)", 2023-09-23T19:44:35.796 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:43:11.560288+0000", 2023-09-23T19:44:35.796 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 84.073053999999999, 2023-09-23T19:44:35.797 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.16978679399999999, 2023-09-23T19:44:35.797 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:44:35.797 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:44:35.797 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:44:35.797 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.797 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:44:35.797 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.560288+0000", 2023-09-23T19:44:35.797 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:44:35.797 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.797 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.797 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:44:35.797 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.560288+0000", 2023-09-23T19:44:35.797 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:44:35.798 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.798 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.798 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:44:35.798 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.560287+0000", 2023-09-23T19:44:35.798 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:44:35.798 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.798 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.798 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:44:35.798 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.560337+0000", 2023-09-23T19:44:35.798 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.0368999999999998e-05 2023-09-23T19:44:35.798 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.799 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.799 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:44:35.799 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.709478+0000", 2023-09-23T19:44:35.799 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.14914037099999999 2023-09-23T19:44:35.799 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.799 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.799 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:44:35.799 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.709497+0000", 2023-09-23T19:44:35.799 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.9556999999999999e-05 2023-09-23T19:44:35.799 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.799 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.799 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:44:35.800 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.711648+0000", 2023-09-23T19:44:35.800 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0021509929999999999 2023-09-23T19:44:35.800 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.800 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.800 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:44:35.800 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.711671+0000", 2023-09-23T19:44:35.800 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.2565000000000002e-05 2023-09-23T19:44:35.800 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.800 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.800 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:44:35.800 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.729930+0000", 2023-09-23T19:44:35.800 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.018259272999999999 2023-09-23T19:44:35.801 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.801 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.801 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:44:35.801 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.730075+0000", 2023-09-23T19:44:35.801 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00014471800000000001 2023-09-23T19:44:35.801 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:44:35.801 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:44:35.801 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:44:35.801 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.801 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.801 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.36243.0:105206 11.6 e190/188 11:6e53ba42:::benchmark_data_smithi016_160148_object105205:head v 190'6499, mlcod=190'6497)", 2023-09-23T19:44:35.801 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:43:11.560414+0000", 2023-09-23T19:44:35.802 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 84.072927913000001, 2023-09-23T19:44:35.802 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.16748537899999999, 2023-09-23T19:44:35.802 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:44:35.802 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:44:35.802 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:44:35.802 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.802 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:44:35.802 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.560414+0000", 2023-09-23T19:44:35.802 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:44:35.802 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.802 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.802 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:44:35.803 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.560414+0000", 2023-09-23T19:44:35.803 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:44:35.803 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.803 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.803 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:44:35.803 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.560413+0000", 2023-09-23T19:44:35.803 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T19:44:35.803 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.803 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.803 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:44:35.803 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.560441+0000", 2023-09-23T19:44:35.803 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.7285e-05 2023-09-23T19:44:35.804 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.804 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.804 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:44:35.804 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.709782+0000", 2023-09-23T19:44:35.804 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.149341528 2023-09-23T19:44:35.804 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.804 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.804 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:44:35.804 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.709796+0000", 2023-09-23T19:44:35.804 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.4173e-05 2023-09-23T19:44:35.804 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.804 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.805 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:44:35.805 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.709992+0000", 2023-09-23T19:44:35.805 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000195892 2023-09-23T19:44:35.805 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.805 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.805 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:44:35.805 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.710017+0000", 2023-09-23T19:44:35.805 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.4479999999999999e-05 2023-09-23T19:44:35.805 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.805 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.805 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:44:35.805 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.727632+0000", 2023-09-23T19:44:35.806 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.017615307 2023-09-23T19:44:35.806 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.806 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.806 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:44:35.806 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:11.727899+0000", 2023-09-23T19:44:35.806 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00026738800000000002 2023-09-23T19:44:35.806 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:44:35.806 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:44:35.806 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:44:35.806 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.806 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.806 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.36243.0:125592 11.7 11:ec806389:::benchmark_data_smithi016_160148_object18389:head [delete] snapc 0=[] ondisk+write+known_if_redirected e192)", 2023-09-23T19:44:35.807 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:43:37.891033+0000", 2023-09-23T19:44:35.807 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 57.742309273000004, 2023-09-23T19:44:35.807 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.80448627699999997, 2023-09-23T19:44:35.807 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:44:35.807 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:44:35.807 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:44:35.807 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.36243", 2023-09-23T19:44:35.807 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1280038109", 2023-09-23T19:44:35.807 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 125592 2023-09-23T19:44:35.807 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.807 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:44:35.807 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.808 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:44:35.808 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:37.891033+0000", 2023-09-23T19:44:35.808 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:44:35.808 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.808 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.808 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:44:35.808 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:37.891033+0000", 2023-09-23T19:44:35.808 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:44:35.808 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.808 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.808 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:44:35.808 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:37.891031+0000", 2023-09-23T19:44:35.809 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-09-23T19:44:35.809 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.809 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.809 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:44:35.809 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:37.891038+0000", 2023-09-23T19:44:35.809 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.8730000000000001e-06 2023-09-23T19:44:35.809 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.809 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.809 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:44:35.809 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:37.891051+0000", 2023-09-23T19:44:35.809 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3115999999999999e-05 2023-09-23T19:44:35.809 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.810 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.810 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:44:35.810 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:37.891061+0000", 2023-09-23T19:44:35.810 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0383999999999999e-05 2023-09-23T19:44:35.810 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.810 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.810 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:44:35.810 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:37.891118+0000", 2023-09-23T19:44:35.810 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.6879999999999998e-05 2023-09-23T19:44:35.810 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.810 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.810 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for peered", 2023-09-23T19:44:35.811 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:37.891128+0000", 2023-09-23T19:44:35.811 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.533e-06 2023-09-23T19:44:35.811 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.811 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.811 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:44:35.811 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:38.657297+0000", 2023-09-23T19:44:35.811 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.76616915500000005 2023-09-23T19:44:35.811 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.811 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.811 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:44:35.811 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:38.657357+0000", 2023-09-23T19:44:35.811 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.9515000000000002e-05 2023-09-23T19:44:35.812 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.812 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.812 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 4,5", 2023-09-23T19:44:35.812 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:38.657420+0000", 2023-09-23T19:44:35.812 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.3678999999999998e-05 2023-09-23T19:44:35.812 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.812 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.812 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:44:35.812 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:38.660604+0000", 2023-09-23T19:44:35.812 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0031842009999999998 2023-09-23T19:44:35.812 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.813 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.813 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:44:35.813 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:38.689129+0000", 2023-09-23T19:44:35.813 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.028524429 2023-09-23T19:44:35.813 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.813 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.813 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:44:35.813 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:38.695440+0000", 2023-09-23T19:44:35.813 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0063112070000000001 2023-09-23T19:44:35.813 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.813 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.813 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:44:35.814 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:38.695483+0000", 2023-09-23T19:44:35.814 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.2589999999999997e-05 2023-09-23T19:44:35.814 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:44:35.814 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:44:35.814 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:44:35.814 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:38.695519+0000", 2023-09-23T19:44:35.814 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.6266000000000001e-05 2023-09-23T19:44:35.814 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:44:35.814 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:44:35.814 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:44:35.814 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:44:35.814 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:44:35.815 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-09-23T19:44:35.853 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:35.859 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:35 smithi016 bash[75004]: cluster 2023-09-23T19:44:34.270552+0000 mgr.y (mgr.25325) 1114 : cluster 0 pgmap v1079: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 693 op/s 2023-09-23T19:44:35.859 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:44:35 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:44:35] "GET /metrics HTTP/1.1" 200 36300 "" "Prometheus/2.33.4" 2023-09-23T19:44:35.884 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:35.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:35.889 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:35.990 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:36.030 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:36.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:36.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:44:36.135 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:36.143 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:35 smithi092 bash[58855]: cluster 2023-09-23T19:44:34.270552+0000 mgr.y (mgr.25325) 1114 : cluster 0 pgmap v1079: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 693 op/s 2023-09-23T19:44:36.154 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:35 smithi016 bash[77995]: cluster 2023-09-23T19:44:34.270552+0000 mgr.y (mgr.25325) 1114 : cluster 0 pgmap v1079: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 693 op/s 2023-09-23T19:44:36.158 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:36.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:36.163 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:44:36.264 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:36.299 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:36.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:36.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:36.406 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:36.430 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:36.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:36.433 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:44:36.534 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:36.565 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:36.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:36.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:44:36.672 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:36.715 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:36.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:36.718 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:44:36.818 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:36.847 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:36.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:36.849 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:36.950 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:36.980 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:36.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:36.984 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:44:37.085 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:37.117 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:37.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:37.124 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:37.224 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:37.256 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:37.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:37.259 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:37.360 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:37.387 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:37.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:37.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:37.490 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:37.542 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:37.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:37.546 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:44:37.647 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:37.680 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:37.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:37.684 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:44:37.784 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:37.817 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:37.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:37.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:37.921 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:37.952 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:37.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:37.963 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:38.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:37 smithi092 bash[58855]: cluster 2023-09-23T19:44:36.272204+0000 mgr.y (mgr.25325) 1115 : cluster 0 pgmap v1080: 89 pgs: 89 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 674 op/s 2023-09-23T19:44:38.065 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:38.099 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:38.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:38.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:44:38.207 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:38.241 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:38.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:38.247 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:44:38.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:37 smithi016 bash[75004]: cluster 2023-09-23T19:44:36.272204+0000 mgr.y (mgr.25325) 1115 : cluster 0 pgmap v1080: 89 pgs: 89 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 674 op/s 2023-09-23T19:44:38.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:37 smithi016 bash[77995]: cluster 2023-09-23T19:44:36.272204+0000 mgr.y (mgr.25325) 1115 : cluster 0 pgmap v1080: 89 pgs: 89 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 674 op/s 2023-09-23T19:44:38.348 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:38.378 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:38.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:38.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:38.481 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:38.510 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:38.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:38.514 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:38.615 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:38.643 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:38.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:38.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:44:38.749 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:38.785 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:38.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:38.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:44:38.892 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:38.917 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:38.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:38.920 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:39.021 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:39.055 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:39.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:39.058 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:39.159 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:39.186 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:39.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:39.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:39.291 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:39.322 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:39.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:39.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:44:39.430 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:39.454 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:39.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:39.458 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:39.558 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:39.596 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:39.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:39.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:39.675 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T19:44:39.699 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:39.737 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:39.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:39.744 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:44:39.844 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:39.880 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:39.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:39.883 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:44:39.984 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:40.013 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:40 smithi016 bash[75004]: cluster 2023-09-23T19:44:38.272695+0000 mgr.y (mgr.25325) 1116 : cluster 0 pgmap v1081: 89 pgs: 89 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-09-23T19:44:40.017 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:40.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:40.021 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:40.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:39 smithi092 bash[58855]: cluster 2023-09-23T19:44:38.272695+0000 mgr.y (mgr.25325) 1116 : cluster 0 pgmap v1081: 89 pgs: 89 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-09-23T19:44:40.123 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:40.158 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:40.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:40.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:40.262 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:40.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:40 smithi016 bash[77995]: cluster 2023-09-23T19:44:38.272695+0000 mgr.y (mgr.25325) 1116 : cluster 0 pgmap v1081: 89 pgs: 89 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-09-23T19:44:40.289 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:40.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:40.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:44:40.393 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:40.434 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:40.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:40.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:40.545 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:40.574 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:40.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:40.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:40.678 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:40.711 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:40.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:40.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:40.817 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:40.863 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:40.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:40.869 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:44:40.970 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:40.979 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:40 smithi016 bash[77995]: audit 2023-09-23T19:44:39.885252+0000 mon.a (mon.0) 1402 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:44:40.979 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:40 smithi016 bash[77995]: audit 2023-09-23T19:44:39.903565+0000 mon.a (mon.0) 1403 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:44:40.979 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:40 smithi016 bash[75004]: audit 2023-09-23T19:44:39.885252+0000 mon.a (mon.0) 1402 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:44:40.979 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:40 smithi016 bash[75004]: audit 2023-09-23T19:44:39.903565+0000 mon.a (mon.0) 1403 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:44:40.995 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:40.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:40.997 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:44:41.098 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:41.137 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:41.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:41.140 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:41.241 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:41.247 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:40 smithi092 bash[58855]: audit 2023-09-23T19:44:39.885252+0000 mon.a (mon.0) 1402 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:44:41.248 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:40 smithi092 bash[58855]: audit 2023-09-23T19:44:39.903565+0000 mon.a (mon.0) 1403 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:44:41.263 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:41.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:41.266 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:44:41.367 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:41.406 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:41.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:41.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:41.511 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:41.538 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:41.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:41.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:44:41.641 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:41.667 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:41.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:41.669 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:44:41.769 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:41.805 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:41.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:41.810 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:41.911 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:41.932 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:41.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:41.936 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:44:42.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:41 smithi016 bash[75004]: cluster 2023-09-23T19:44:40.273358+0000 mgr.y (mgr.25325) 1117 : cluster 0 pgmap v1082: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 725 op/s 2023-09-23T19:44:42.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:41 smithi016 bash[77995]: cluster 2023-09-23T19:44:40.273358+0000 mgr.y (mgr.25325) 1117 : cluster 0 pgmap v1082: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 725 op/s 2023-09-23T19:44:42.037 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:42.095 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:42.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:42.098 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:44:42.199 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:42.209 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:41 smithi092 bash[58855]: cluster 2023-09-23T19:44:40.273358+0000 mgr.y (mgr.25325) 1117 : cluster 0 pgmap v1082: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 725 op/s 2023-09-23T19:44:42.235 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:42.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:42.237 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:44:42.338 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:42.370 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:42.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:42.373 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:42.473 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:42.509 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:42.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:42.515 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:44:42.617 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:42.644 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:42.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:42.645 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:44:42.746 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:42.775 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:42.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:42.778 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:42.878 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:42.911 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:42.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:42.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:44:43.018 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:43.047 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:43.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:43.049 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:44:43.150 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:43.186 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:43.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:43.190 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:44:43.291 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:43.330 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:43.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:43.334 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:43.434 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:43.465 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:43.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:43.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:44:43.572 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:43.603 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:43.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:43.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:43.708 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:43.748 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:43.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:43.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:44:43.854 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:43.889 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:43.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:43.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:43.994 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:44.026 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:44.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:44.032 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:44:44.133 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:44.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:44.172 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:44.172 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:44:44.173 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:43 smithi092 bash[58855]: cluster 2023-09-23T19:44:42.273872+0000 mgr.y (mgr.25325) 1118 : cluster 0 pgmap v1083: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 643 op/s 2023-09-23T19:44:44.173 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:43 smithi092 bash[58855]: audit 2023-09-23T19:44:43.663787+0000 mon.a (mon.0) 1404 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:44:44.213 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:44:44.273 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:44.279 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:43 smithi016 bash[75004]: cluster 2023-09-23T19:44:42.273872+0000 mgr.y (mgr.25325) 1118 : cluster 0 pgmap v1083: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 643 op/s 2023-09-23T19:44:44.279 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:43 smithi016 bash[75004]: audit 2023-09-23T19:44:43.663787+0000 mon.a (mon.0) 1404 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:44:44.280 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:43 smithi016 bash[77995]: cluster 2023-09-23T19:44:42.273872+0000 mgr.y (mgr.25325) 1118 : cluster 0 pgmap v1083: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 643 op/s 2023-09-23T19:44:44.280 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:43 smithi016 bash[77995]: audit 2023-09-23T19:44:43.663787+0000 mon.a (mon.0) 1404 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:44:44.299 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:44.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:44.307 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:44.408 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:44.433 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:44.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:44.438 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:44:44.539 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:44.583 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:44.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:44.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:44.689 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:44.725 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:44.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:44.732 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:44.833 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:44.856 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:44.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:44.857 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:44:44.958 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:44.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:44.998 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:44.998 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:44:45.101 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:45.133 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:45.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:45.138 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:44:45.239 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:45.269 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:45.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:45.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:44:45.375 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:45.440 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:45.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:45.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:45.544 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:45.585 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:45.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:45.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:45.647 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:44:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:44:45] "GET /metrics HTTP/1.1" 200 36306 "" "Prometheus/2.33.4" 2023-09-23T19:44:45.691 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:45.715 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:45.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:45.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:45.819 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:45.850 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:45.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:45.856 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:44:45.919 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:45 smithi016 bash[75004]: cluster 2023-09-23T19:44:44.274361+0000 mgr.y (mgr.25325) 1119 : cluster 0 pgmap v1084: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 622 op/s 2023-09-23T19:44:45.919 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:45 smithi016 bash[75004]: audit 2023-09-23T19:44:45.624010+0000 mon.a (mon.0) 1405 : audit 1 from='client.? 172.21.15.16:0/428021396' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T19:44:45.919 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:45 smithi016 bash[75004]: audit 2023-09-23T19:44:45.675338+0000 mon.a (mon.0) 1406 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:44:45.920 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:45 smithi016 bash[75004]: audit 2023-09-23T19:44:45.684577+0000 mon.a (mon.0) 1407 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:44:45.945 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T19:44:45.956 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:46.016 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:46.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:46.021 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:46.122 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:46.154 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:46.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:46.160 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:44:46.261 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:46.270 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:45 smithi092 bash[58855]: cluster 2023-09-23T19:44:44.274361+0000 mgr.y (mgr.25325) 1119 : cluster 0 pgmap v1084: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 622 op/s 2023-09-23T19:44:46.270 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:45 smithi092 bash[58855]: audit 2023-09-23T19:44:45.624010+0000 mon.a (mon.0) 1405 : audit 1 from='client.? 172.21.15.16:0/428021396' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T19:44:46.270 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:45 smithi092 bash[58855]: audit 2023-09-23T19:44:45.675338+0000 mon.a (mon.0) 1406 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:44:46.270 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:45 smithi092 bash[58855]: audit 2023-09-23T19:44:45.684577+0000 mon.a (mon.0) 1407 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:44:46.284 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:46.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:46.286 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:44:46.336 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:45 smithi016 bash[77995]: cluster 2023-09-23T19:44:44.274361+0000 mgr.y (mgr.25325) 1119 : cluster 0 pgmap v1084: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 622 op/s 2023-09-23T19:44:46.336 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:45 smithi016 bash[77995]: audit 2023-09-23T19:44:45.624010+0000 mon.a (mon.0) 1405 : audit 1 from='client.? 172.21.15.16:0/428021396' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T19:44:46.336 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:45 smithi016 bash[77995]: audit 2023-09-23T19:44:45.675338+0000 mon.a (mon.0) 1406 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:44:46.336 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:45 smithi016 bash[77995]: audit 2023-09-23T19:44:45.684577+0000 mon.a (mon.0) 1407 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:44:46.386 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:46.410 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:46.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:46.414 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:44:46.515 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:46.555 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:46.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:46.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:44:46.659 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:46.681 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:46.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:46.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:44:46.787 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:46.832 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:46.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:46.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:46.938 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:46.969 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:46.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:46.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:44:47.009 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:46 smithi016 bash[75004]: audit 2023-09-23T19:44:45.931558+0000 mon.a (mon.0) 1408 : audit 1 from='client.? 172.21.15.16:0/428021396' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T19:44:47.009 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:46 smithi016 bash[75004]: cluster 2023-09-23T19:44:45.931784+0000 mon.a (mon.0) 1409 : cluster 0 osdmap e197: 8 total, 8 up, 8 in 2023-09-23T19:44:47.009 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:47 smithi016 bash[75004]: cluster 2023-09-23T19:44:45.945514+0000 osd.7 (osd.7) 112 : cluster 3 failed to encode map e197 with expected crc 2023-09-23T19:44:47.010 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:47 smithi016 bash[77995]: audit 2023-09-23T19:44:45.931558+0000 mon.a (mon.0) 1408 : audit 1 from='client.? 172.21.15.16:0/428021396' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T19:44:47.010 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:47 smithi016 bash[77995]: cluster 2023-09-23T19:44:45.931784+0000 mon.a (mon.0) 1409 : cluster 0 osdmap e197: 8 total, 8 up, 8 in 2023-09-23T19:44:47.011 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:47 smithi016 bash[77995]: cluster 2023-09-23T19:44:45.945514+0000 osd.7 (osd.7) 112 : cluster 3 failed to encode map e197 with expected crc 2023-09-23T19:44:47.011 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:47 smithi016 bash[77995]: cluster 2023-09-23T19:44:45.945736+0000 osd.6 (osd.6) 122 : cluster 3 failed to encode map e197 with expected crc 2023-09-23T19:44:47.011 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:47 smithi016 bash[77995]: cluster 2023-09-23T19:44:45.947532+0000 osd.4 (osd.4) 138 : cluster 3 failed to encode map e197 with expected crc 2023-09-23T19:44:47.011 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:47 smithi016 bash[77995]: cluster 2023-09-23T19:44:45.949559+0000 osd.6 (osd.6) 123 : cluster 3 failed to encode map e197 with expected crc 2023-09-23T19:44:47.011 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:47 smithi016 bash[77995]: cluster 2023-09-23T19:44:45.958464+0000 osd.5 (osd.5) 125 : cluster 3 failed to encode map e197 with expected crc 2023-09-23T19:44:47.011 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:47 smithi016 bash[77995]: audit 2023-09-23T19:44:46.268707+0000 mon.a (mon.0) 1410 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:44:47.011 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:47 smithi016 bash[77995]: audit 2023-09-23T19:44:46.269592+0000 mon.a (mon.0) 1411 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:44:47.011 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:47 smithi016 bash[77995]: audit 2023-09-23T19:44:46.274733+0000 mon.a (mon.0) 1412 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:44:47.041 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:44:47.069 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:44:47.076 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:47.094 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:44:47.105 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:47.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:47.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:44:47.126 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:44:47.167 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:44:47.206 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:44:47.211 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:47.222 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:46 smithi092 bash[58855]: audit 2023-09-23T19:44:45.931558+0000 mon.a (mon.0) 1408 : audit 1 from='client.? 172.21.15.16:0/428021396' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T19:44:47.222 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:46 smithi092 bash[58855]: cluster 2023-09-23T19:44:45.931784+0000 mon.a (mon.0) 1409 : cluster 0 osdmap e197: 8 total, 8 up, 8 in 2023-09-23T19:44:47.222 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:46 smithi092 bash[58855]: cluster 2023-09-23T19:44:45.945514+0000 osd.7 (osd.7) 112 : cluster 3 failed to encode map e197 with expected crc 2023-09-23T19:44:47.222 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:46 smithi092 bash[58855]: cluster 2023-09-23T19:44:45.945736+0000 osd.6 (osd.6) 122 : cluster 3 failed to encode map e197 with expected crc 2023-09-23T19:44:47.222 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:46 smithi092 bash[58855]: cluster 2023-09-23T19:44:45.947532+0000 osd.4 (osd.4) 138 : cluster 3 failed to encode map e197 with expected crc 2023-09-23T19:44:47.223 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:46 smithi092 bash[58855]: cluster 2023-09-23T19:44:45.949559+0000 osd.6 (osd.6) 123 : cluster 3 failed to encode map e197 with expected crc 2023-09-23T19:44:47.223 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:46 smithi092 bash[58855]: cluster 2023-09-23T19:44:45.958464+0000 osd.5 (osd.5) 125 : cluster 3 failed to encode map e197 with expected crc 2023-09-23T19:44:47.223 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:46 smithi092 bash[58855]: audit 2023-09-23T19:44:46.268707+0000 mon.a (mon.0) 1410 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:44:47.223 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:46 smithi092 bash[58855]: audit 2023-09-23T19:44:46.269592+0000 mon.a (mon.0) 1411 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:44:47.223 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:46 smithi092 bash[58855]: audit 2023-09-23T19:44:46.274733+0000 mon.a (mon.0) 1412 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:44:47.241 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:47.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:47.244 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:44:47.251 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:44:47.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:47 smithi016 bash[75004]: cluster 2023-09-23T19:44:45.945736+0000 osd.6 (osd.6) 122 : cluster 3 failed to encode map e197 with expected crc 2023-09-23T19:44:47.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:47 smithi016 bash[75004]: cluster 2023-09-23T19:44:45.947532+0000 osd.4 (osd.4) 138 : cluster 3 failed to encode map e197 with expected crc 2023-09-23T19:44:47.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:47 smithi016 bash[75004]: cluster 2023-09-23T19:44:45.949559+0000 osd.6 (osd.6) 123 : cluster 3 failed to encode map e197 with expected crc 2023-09-23T19:44:47.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:47 smithi016 bash[75004]: cluster 2023-09-23T19:44:45.958464+0000 osd.5 (osd.5) 125 : cluster 3 failed to encode map e197 with expected crc 2023-09-23T19:44:47.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:47 smithi016 bash[75004]: audit 2023-09-23T19:44:46.268707+0000 mon.a (mon.0) 1410 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:44:47.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:47 smithi016 bash[75004]: audit 2023-09-23T19:44:46.269592+0000 mon.a (mon.0) 1411 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:44:47.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:47 smithi016 bash[75004]: audit 2023-09-23T19:44:46.274733+0000 mon.a (mon.0) 1412 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:44:47.288 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:44:47.295 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:44:47.295 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:44:47.296 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:44:47.296 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:44:47.296 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:44:47.296 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:44:47.296 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:44:47.296 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:44:47.344 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:47.379 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:47.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:47.388 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:44:47.489 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:47.513 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:47.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:47.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:47.620 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:47.653 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:47.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:47.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:44:47.757 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:47.785 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:47.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:47.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:44:47.892 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:47.924 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:47.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:47.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:44:47.932 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:47 smithi016 bash[77995]: cluster 2023-09-23T19:44:46.274705+0000 mgr.y (mgr.25325) 1120 : cluster 0 pgmap v1086: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 682 op/s 2023-09-23T19:44:47.932 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:47 smithi016 bash[77995]: audit 2023-09-23T19:44:46.999795+0000 mon.b (mon.1) 50 : audit 0 from='client.? 172.21.15.16:0/3210382347' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:44:47.932 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:47 smithi016 bash[77995]: audit 2023-09-23T19:44:47.014914+0000 mon.a (mon.0) 1413 : audit 1 from='client.? 172.21.15.16:0/1511238724' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:44:47.944 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is unset 2023-09-23T19:44:48.032 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:48.057 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:48.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:48.061 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:44:48.161 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:48.183 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:48.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:48.192 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:48.192 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:47 smithi016 bash[75004]: cluster 2023-09-23T19:44:46.274705+0000 mgr.y (mgr.25325) 1120 : cluster 0 pgmap v1086: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 682 op/s 2023-09-23T19:44:48.192 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:47 smithi016 bash[75004]: audit 2023-09-23T19:44:46.999795+0000 mon.b (mon.1) 50 : audit 0 from='client.? 172.21.15.16:0/3210382347' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:44:48.192 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:47 smithi016 bash[75004]: audit 2023-09-23T19:44:47.014914+0000 mon.a (mon.0) 1413 : audit 1 from='client.? 172.21.15.16:0/1511238724' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:44:48.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:47 smithi092 bash[58855]: cluster 2023-09-23T19:44:46.274705+0000 mgr.y (mgr.25325) 1120 : cluster 0 pgmap v1086: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 682 op/s 2023-09-23T19:44:48.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:47 smithi092 bash[58855]: audit 2023-09-23T19:44:46.999795+0000 mon.b (mon.1) 50 : audit 0 from='client.? 172.21.15.16:0/3210382347' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:44:48.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:47 smithi092 bash[58855]: audit 2023-09-23T19:44:47.014914+0000 mon.a (mon.0) 1413 : audit 1 from='client.? 172.21.15.16:0/1511238724' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:44:48.292 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:48.316 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:48.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:48.318 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:44:48.419 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:48.455 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:48.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:48.458 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:48.558 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:48.594 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:48.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:48.598 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:44:48.698 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:48.724 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:48.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:48.726 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:44:48.827 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:48.852 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:48.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:48.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:44:48.918 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T19:44:48.958 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:48.966 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:48 smithi092 bash[58855]: cluster 2023-09-23T19:44:47.925144+0000 mon.a (mon.0) 1414 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:44:48.967 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:48 smithi092 bash[58855]: cluster 2023-09-23T19:44:47.925188+0000 mon.a (mon.0) 1415 : cluster 1 Cluster is now healthy 2023-09-23T19:44:48.967 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:48 smithi092 bash[58855]: audit 2023-09-23T19:44:47.936632+0000 mon.a (mon.0) 1416 : audit 1 from='client.? 172.21.15.16:0/1511238724' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:44:48.967 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:48 smithi092 bash[58855]: cluster 2023-09-23T19:44:47.936926+0000 mon.a (mon.0) 1417 : cluster 0 osdmap e198: 8 total, 8 up, 8 in 2023-09-23T19:44:48.967 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:48 smithi092 bash[58855]: cluster 2023-09-23T19:44:47.943782+0000 osd.6 (osd.6) 124 : cluster 3 failed to encode map e198 with expected crc 2023-09-23T19:44:48.967 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:48 smithi092 bash[58855]: cluster 2023-09-23T19:44:47.944283+0000 osd.4 (osd.4) 139 : cluster 3 failed to encode map e198 with expected crc 2023-09-23T19:44:48.967 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:48 smithi092 bash[58855]: cluster 2023-09-23T19:44:47.950679+0000 osd.5 (osd.5) 126 : cluster 3 failed to encode map e198 with expected crc 2023-09-23T19:44:48.967 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:48 smithi092 bash[58855]: cluster 2023-09-23T19:44:47.951658+0000 osd.7 (osd.7) 113 : cluster 3 failed to encode map e198 with expected crc 2023-09-23T19:44:48.968 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:48 smithi092 bash[58855]: cluster 2023-09-23T19:44:47.953487+0000 osd.7 (osd.7) 114 : cluster 3 failed to encode map e198 with expected crc 2023-09-23T19:44:48.968 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:48 smithi092 bash[58855]: cluster 2023-09-23T19:44:47.953872+0000 osd.6 (osd.6) 125 : cluster 3 failed to encode map e198 with expected crc 2023-09-23T19:44:48.980 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:48.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:48.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:44:49.082 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:49.120 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:49.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:49.125 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:44:49.226 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:49.233 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:48 smithi016 bash[75004]: cluster 2023-09-23T19:44:47.925144+0000 mon.a (mon.0) 1414 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:44:49.234 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:48 smithi016 bash[75004]: cluster 2023-09-23T19:44:47.925188+0000 mon.a (mon.0) 1415 : cluster 1 Cluster is now healthy 2023-09-23T19:44:49.234 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:48 smithi016 bash[75004]: audit 2023-09-23T19:44:47.936632+0000 mon.a (mon.0) 1416 : audit 1 from='client.? 172.21.15.16:0/1511238724' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:44:49.234 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:48 smithi016 bash[75004]: cluster 2023-09-23T19:44:47.936926+0000 mon.a (mon.0) 1417 : cluster 0 osdmap e198: 8 total, 8 up, 8 in 2023-09-23T19:44:49.234 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:48 smithi016 bash[75004]: cluster 2023-09-23T19:44:47.943782+0000 osd.6 (osd.6) 124 : cluster 3 failed to encode map e198 with expected crc 2023-09-23T19:44:49.234 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:48 smithi016 bash[75004]: cluster 2023-09-23T19:44:47.944283+0000 osd.4 (osd.4) 139 : cluster 3 failed to encode map e198 with expected crc 2023-09-23T19:44:49.234 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:48 smithi016 bash[75004]: cluster 2023-09-23T19:44:47.950679+0000 osd.5 (osd.5) 126 : cluster 3 failed to encode map e198 with expected crc 2023-09-23T19:44:49.234 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:48 smithi016 bash[75004]: cluster 2023-09-23T19:44:47.951658+0000 osd.7 (osd.7) 113 : cluster 3 failed to encode map e198 with expected crc 2023-09-23T19:44:49.234 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:48 smithi016 bash[75004]: cluster 2023-09-23T19:44:47.953487+0000 osd.7 (osd.7) 114 : cluster 3 failed to encode map e198 with expected crc 2023-09-23T19:44:49.234 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:48 smithi016 bash[75004]: cluster 2023-09-23T19:44:47.953872+0000 osd.6 (osd.6) 125 : cluster 3 failed to encode map e198 with expected crc 2023-09-23T19:44:49.235 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:48 smithi016 bash[77995]: cluster 2023-09-23T19:44:47.925144+0000 mon.a (mon.0) 1414 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:44:49.235 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:48 smithi016 bash[77995]: cluster 2023-09-23T19:44:47.925188+0000 mon.a (mon.0) 1415 : cluster 1 Cluster is now healthy 2023-09-23T19:44:49.235 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:48 smithi016 bash[77995]: audit 2023-09-23T19:44:47.936632+0000 mon.a (mon.0) 1416 : audit 1 from='client.? 172.21.15.16:0/1511238724' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:44:49.235 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:48 smithi016 bash[77995]: cluster 2023-09-23T19:44:47.936926+0000 mon.a (mon.0) 1417 : cluster 0 osdmap e198: 8 total, 8 up, 8 in 2023-09-23T19:44:49.235 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:48 smithi016 bash[77995]: cluster 2023-09-23T19:44:47.943782+0000 osd.6 (osd.6) 124 : cluster 3 failed to encode map e198 with expected crc 2023-09-23T19:44:49.235 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:48 smithi016 bash[77995]: cluster 2023-09-23T19:44:47.944283+0000 osd.4 (osd.4) 139 : cluster 3 failed to encode map e198 with expected crc 2023-09-23T19:44:49.235 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:48 smithi016 bash[77995]: cluster 2023-09-23T19:44:47.950679+0000 osd.5 (osd.5) 126 : cluster 3 failed to encode map e198 with expected crc 2023-09-23T19:44:49.235 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:48 smithi016 bash[77995]: cluster 2023-09-23T19:44:47.951658+0000 osd.7 (osd.7) 113 : cluster 3 failed to encode map e198 with expected crc 2023-09-23T19:44:49.235 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:48 smithi016 bash[77995]: cluster 2023-09-23T19:44:47.953487+0000 osd.7 (osd.7) 114 : cluster 3 failed to encode map e198 with expected crc 2023-09-23T19:44:49.235 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:48 smithi016 bash[77995]: cluster 2023-09-23T19:44:47.953872+0000 osd.6 (osd.6) 125 : cluster 3 failed to encode map e198 with expected crc 2023-09-23T19:44:49.258 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:49.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:49.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:44:49.363 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:49.397 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:49.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:49.400 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:44:49.501 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:49.519 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:49.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:49.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:44:49.622 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:49.646 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:49.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:49.652 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:44:49.752 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:49.786 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:49.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:49.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:49.890 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:49.928 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:49.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:49.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:44:49.950 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:49 smithi092 bash[58855]: cluster 2023-09-23T19:44:48.275115+0000 mgr.y (mgr.25325) 1121 : cluster 0 pgmap v1088: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2023-09-23T19:44:50.034 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:50.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:49 smithi016 bash[75004]: cluster 2023-09-23T19:44:48.275115+0000 mgr.y (mgr.25325) 1121 : cluster 0 pgmap v1088: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2023-09-23T19:44:50.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:49 smithi016 bash[77995]: cluster 2023-09-23T19:44:48.275115+0000 mgr.y (mgr.25325) 1121 : cluster 0 pgmap v1088: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2023-09-23T19:44:50.079 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:50.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:50.084 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:44:50.185 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:50.213 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:50.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:50.217 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:44:50.318 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:50.347 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:50.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:50.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:50.450 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:50.476 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:50.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:50.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:44:50.579 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:50.605 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:50.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:50.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:44:50.713 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:50.739 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:50.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:50.742 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:44:50.843 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:50.875 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:50.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:50.879 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:44:50.980 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:51.020 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:51.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:51.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:44:51.123 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:51.147 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:51.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:51.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:44:51.256 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:51.279 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:51.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:51.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:44:51.384 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:51.411 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:51.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:51.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:44:51.518 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:51.562 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:51.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:51.568 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:51.669 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:51.692 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:51.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:51.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:44:51.797 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:51.828 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:51.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:51.835 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:44:51.936 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:51.966 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:51.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:51.972 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:44:52.073 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:52.108 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:52.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:52.115 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:44:52.210 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:51 smithi016 bash[75004]: cluster 2023-09-23T19:44:50.275652+0000 mgr.y (mgr.25325) 1122 : cluster 0 pgmap v1089: 89 pgs: 89 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2023-09-23T19:44:52.210 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:51 smithi016 bash[77995]: cluster 2023-09-23T19:44:50.275652+0000 mgr.y (mgr.25325) 1122 : cluster 0 pgmap v1089: 89 pgs: 89 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2023-09-23T19:44:52.216 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:52.255 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:52.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:52.258 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:52.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:51 smithi092 bash[58855]: cluster 2023-09-23T19:44:50.275652+0000 mgr.y (mgr.25325) 1122 : cluster 0 pgmap v1089: 89 pgs: 89 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2023-09-23T19:44:52.358 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:52.398 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:52.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:52.405 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:44:52.485 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-09-23T19:44:52.485 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:44:52.507 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:52.533 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:52.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:52.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:44:52.640 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:52.670 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:52.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:52.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:44:52.773 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:52.795 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:52.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:52.797 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:44:52.897 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:52.932 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:52.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:52.936 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:52.976 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T19:44:52.979 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:44:53.037 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:53.080 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:53.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:53.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:44:53.181 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:53.204 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:53.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:53.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:44:53.310 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:53.343 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:53.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:53.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:53.450 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:53.491 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:53.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:53.494 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:44:53.597 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:53.625 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:53.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:53.627 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:44:53.668 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T19:44:53.728 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:53.755 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:53.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:53.759 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:44:53.860 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:53.902 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:53.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:53.911 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:54.012 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:54.068 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:54.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:54.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:54.147 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:44:54.173 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:54.211 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:54.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:54.217 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:44:54.217 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:53 smithi016 bash[77995]: cluster 2023-09-23T19:44:52.276164+0000 mgr.y (mgr.25325) 1123 : cluster 0 pgmap v1090: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 751 op/s 2023-09-23T19:44:54.217 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:53 smithi016 bash[75004]: cluster 2023-09-23T19:44:52.276164+0000 mgr.y (mgr.25325) 1123 : cluster 0 pgmap v1090: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 751 op/s 2023-09-23T19:44:54.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:53 smithi092 bash[58855]: cluster 2023-09-23T19:44:52.276164+0000 mgr.y (mgr.25325) 1123 : cluster 0 pgmap v1090: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 751 op/s 2023-09-23T19:44:54.318 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:54.359 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:54.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:54.361 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:44:54.462 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:54.486 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:54.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:54.490 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:44:54.591 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:54.613 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:54.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:54.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:44:54.716 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:54.750 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:54.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:54.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:44:54.853 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:54.884 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:54.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:54.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:44:54.987 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:55.016 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:55.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:55.020 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:44:55.121 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:55.170 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:55.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:55.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:44:55.280 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:55.304 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:55.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:55.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:44:55.407 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:55.448 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:55.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:55.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:44:55.552 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:55.580 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:55.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:55.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:44:55.688 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:55.713 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:55.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:55.718 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:44:55.819 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:55.834 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:44:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:44:55] "GET /metrics HTTP/1.1" 200 36306 "" "Prometheus/2.33.4" 2023-09-23T19:44:55.862 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:44:55.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:55.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:44:55.961 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:55 smithi092 bash[58855]: cluster 2023-09-23T19:44:54.276663+0000 mgr.y (mgr.25325) 1124 : cluster 0 pgmap v1091: 89 pgs: 89 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 456 op/s 2023-09-23T19:44:55.965 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:56.006 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:56.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:56.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:56.112 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:56.138 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:56.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:56.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:44:56.244 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:56.268 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:56.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:56.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:44:56.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:55 smithi016 bash[77995]: cluster 2023-09-23T19:44:54.276663+0000 mgr.y (mgr.25325) 1124 : cluster 0 pgmap v1091: 89 pgs: 89 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 456 op/s 2023-09-23T19:44:56.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:55 smithi016 bash[75004]: cluster 2023-09-23T19:44:54.276663+0000 mgr.y (mgr.25325) 1124 : cluster 0 pgmap v1091: 89 pgs: 89 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 456 op/s 2023-09-23T19:44:56.374 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:56.416 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:56.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:56.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:44:56.524 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:56.553 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:56.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:56.560 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:44:56.661 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:56.687 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:56.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:56.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:44:56.791 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:56.825 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:56.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:56.828 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:56.929 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:56.959 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:56.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:56.962 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:44:57.063 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:57.094 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:57.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:57.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:44:57.200 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:57.225 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:57.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:57.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:44:57.328 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:57.359 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:57.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:57.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:57.462 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:57.485 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:57.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:57.488 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:44:57.588 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:57.616 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:57.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:57.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:44:57.723 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:57.755 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:57.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:57.762 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:57.863 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:57.907 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:57.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:57.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:58.013 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:58.042 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:44:58.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:58.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:44:58.147 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:58.155 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:57 smithi016 bash[75004]: cluster 2023-09-23T19:44:56.277395+0000 mgr.y (mgr.25325) 1125 : cluster 0 pgmap v1092: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2023-09-23T19:44:58.155 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:57 smithi016 bash[77995]: cluster 2023-09-23T19:44:56.277395+0000 mgr.y (mgr.25325) 1125 : cluster 0 pgmap v1092: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2023-09-23T19:44:58.184 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:58.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:58.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:58.235 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:44:58.235 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:44:58.239 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:44:58.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:57 smithi092 bash[58855]: cluster 2023-09-23T19:44:56.277395+0000 mgr.y (mgr.25325) 1125 : cluster 0 pgmap v1092: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2023-09-23T19:44:58.291 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:58.316 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:58.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:58.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:44:58.420 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:58.442 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:58.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:58.444 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:44:58.545 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:58.569 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:44:58.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:58.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:44:58.672 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:58.713 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:58.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:58.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:44:58.818 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:58.851 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:58.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:58.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:58.961 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:58.975 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:58 smithi016 bash[75004]: audit 2023-09-23T19:44:58.630506+0000 mon.a (mon.0) 1418 : audit 1 from='client.? 172.21.15.16:0/4218179794' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T19:44:58.975 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:58 smithi016 bash[75004]: audit 2023-09-23T19:44:58.669052+0000 mon.a (mon.0) 1419 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:44:58.975 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:58 smithi016 bash[75004]: audit 2023-09-23T19:44:58.670967+0000 mon.a (mon.0) 1420 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:44:58.976 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:58 smithi016 bash[77995]: audit 2023-09-23T19:44:58.630506+0000 mon.a (mon.0) 1418 : audit 1 from='client.? 172.21.15.16:0/4218179794' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T19:44:58.976 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:58 smithi016 bash[77995]: audit 2023-09-23T19:44:58.669052+0000 mon.a (mon.0) 1419 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:44:58.976 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:44:58 smithi016 bash[77995]: audit 2023-09-23T19:44:58.670967+0000 mon.a (mon.0) 1420 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:44:58.985 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:58.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:58.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:44:59.009 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T19:44:59.088 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:59.116 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:59.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:59.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:59.220 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:59.251 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:44:59.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:59.258 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:44:59.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:58 smithi092 bash[58855]: audit 2023-09-23T19:44:58.630506+0000 mon.a (mon.0) 1418 : audit 1 from='client.? 172.21.15.16:0/4218179794' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T19:44:59.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:58 smithi092 bash[58855]: audit 2023-09-23T19:44:58.669052+0000 mon.a (mon.0) 1419 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:44:59.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:58 smithi092 bash[58855]: audit 2023-09-23T19:44:58.670967+0000 mon.a (mon.0) 1420 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:44:59.358 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:59.391 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:59.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:59.396 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:44:59.497 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:59.532 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:44:59.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:59.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:44:59.639 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:59.667 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:44:59.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:59.676 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:44:59.776 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:59.802 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:44:59.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:59.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:44:59.905 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:59.931 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:44:59.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:44:59.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:45:00.035 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:00.057 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:00.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:00.062 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:45:00.163 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:00.189 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:00.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:00.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:45:00.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:00 smithi016 bash[77995]: cluster 2023-09-23T19:44:58.281734+0000 mgr.y (mgr.25325) 1126 : cluster 0 pgmap v1093: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 629 op/s 2023-09-23T19:45:00.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:00 smithi016 bash[77995]: audit 2023-09-23T19:44:58.992052+0000 mon.a (mon.0) 1421 : audit 1 from='client.? 172.21.15.16:0/4218179794' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T19:45:00.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:00 smithi016 bash[77995]: cluster 2023-09-23T19:44:58.992100+0000 mon.a (mon.0) 1422 : cluster 0 osdmap e199: 8 total, 8 up, 8 in 2023-09-23T19:45:00.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:00 smithi016 bash[77995]: cluster 2023-09-23T19:44:58.994272+0000 osd.7 (osd.7) 115 : cluster 3 failed to encode map e199 with expected crc 2023-09-23T19:45:00.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:00 smithi016 bash[77995]: cluster 2023-09-23T19:44:59.004022+0000 osd.4 (osd.4) 140 : cluster 3 failed to encode map e199 with expected crc 2023-09-23T19:45:00.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:00 smithi016 bash[77995]: cluster 2023-09-23T19:44:59.004169+0000 osd.7 (osd.7) 116 : cluster 3 failed to encode map e199 with expected crc 2023-09-23T19:45:00.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:00 smithi016 bash[77995]: cluster 2023-09-23T19:44:59.011976+0000 osd.6 (osd.6) 126 : cluster 3 failed to encode map e199 with expected crc 2023-09-23T19:45:00.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:00 smithi016 bash[77995]: cluster 2023-09-23T19:44:59.014633+0000 osd.6 (osd.6) 127 : cluster 3 failed to encode map e199 with expected crc 2023-09-23T19:45:00.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:00 smithi016 bash[77995]: cluster 2023-09-23T19:44:59.017082+0000 osd.5 (osd.5) 127 : cluster 3 failed to encode map e199 with expected crc 2023-09-23T19:45:00.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:59 smithi016 bash[75004]: cluster 2023-09-23T19:44:58.281734+0000 mgr.y (mgr.25325) 1126 : cluster 0 pgmap v1093: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 629 op/s 2023-09-23T19:45:00.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:44:59 smithi016 bash[75004]: audit 2023-09-23T19:44:58.992052+0000 mon.a (mon.0) 1421 : audit 1 from='client.? 172.21.15.16:0/4218179794' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T19:45:00.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:00 smithi016 bash[75004]: cluster 2023-09-23T19:44:58.992100+0000 mon.a (mon.0) 1422 : cluster 0 osdmap e199: 8 total, 8 up, 8 in 2023-09-23T19:45:00.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:00 smithi016 bash[75004]: cluster 2023-09-23T19:44:58.994272+0000 osd.7 (osd.7) 115 : cluster 3 failed to encode map e199 with expected crc 2023-09-23T19:45:00.288 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:00 smithi016 bash[75004]: cluster 2023-09-23T19:44:59.004022+0000 osd.4 (osd.4) 140 : cluster 3 failed to encode map e199 with expected crc 2023-09-23T19:45:00.288 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:00 smithi016 bash[75004]: cluster 2023-09-23T19:44:59.004169+0000 osd.7 (osd.7) 116 : cluster 3 failed to encode map e199 with expected crc 2023-09-23T19:45:00.288 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:00 smithi016 bash[75004]: cluster 2023-09-23T19:44:59.011976+0000 osd.6 (osd.6) 126 : cluster 3 failed to encode map e199 with expected crc 2023-09-23T19:45:00.288 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:00 smithi016 bash[75004]: cluster 2023-09-23T19:44:59.014633+0000 osd.6 (osd.6) 127 : cluster 3 failed to encode map e199 with expected crc 2023-09-23T19:45:00.288 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:00 smithi016 bash[75004]: cluster 2023-09-23T19:44:59.017082+0000 osd.5 (osd.5) 127 : cluster 3 failed to encode map e199 with expected crc 2023-09-23T19:45:00.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:59 smithi092 bash[58855]: cluster 2023-09-23T19:44:58.281734+0000 mgr.y (mgr.25325) 1126 : cluster 0 pgmap v1093: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 629 op/s 2023-09-23T19:45:00.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:59 smithi092 bash[58855]: audit 2023-09-23T19:44:58.992052+0000 mon.a (mon.0) 1421 : audit 1 from='client.? 172.21.15.16:0/4218179794' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T19:45:00.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:59 smithi092 bash[58855]: cluster 2023-09-23T19:44:58.992100+0000 mon.a (mon.0) 1422 : cluster 0 osdmap e199: 8 total, 8 up, 8 in 2023-09-23T19:45:00.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:59 smithi092 bash[58855]: cluster 2023-09-23T19:44:58.994272+0000 osd.7 (osd.7) 115 : cluster 3 failed to encode map e199 with expected crc 2023-09-23T19:45:00.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:59 smithi092 bash[58855]: cluster 2023-09-23T19:44:59.004022+0000 osd.4 (osd.4) 140 : cluster 3 failed to encode map e199 with expected crc 2023-09-23T19:45:00.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:59 smithi092 bash[58855]: cluster 2023-09-23T19:44:59.004169+0000 osd.7 (osd.7) 116 : cluster 3 failed to encode map e199 with expected crc 2023-09-23T19:45:00.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:59 smithi092 bash[58855]: cluster 2023-09-23T19:44:59.011976+0000 osd.6 (osd.6) 126 : cluster 3 failed to encode map e199 with expected crc 2023-09-23T19:45:00.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:59 smithi092 bash[58855]: cluster 2023-09-23T19:44:59.014633+0000 osd.6 (osd.6) 127 : cluster 3 failed to encode map e199 with expected crc 2023-09-23T19:45:00.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:44:59 smithi092 bash[58855]: cluster 2023-09-23T19:44:59.017082+0000 osd.5 (osd.5) 127 : cluster 3 failed to encode map e199 with expected crc 2023-09-23T19:45:00.295 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:00.324 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:00.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:00.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:00.428 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:00.465 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:00.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:00.469 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:00.569 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:00.614 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:00.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:00.617 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:00.718 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:00.743 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:00.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:00.746 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:00.847 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:00.875 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:00.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:00.879 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:00.980 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:01.020 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:01.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:01.023 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:45:01.123 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:01.125 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T19:45:01.152 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:01.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:01.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:01.257 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:01.284 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:01.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:01.289 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:45:01.389 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:01.445 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:01.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:01.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:45:01.552 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:01.578 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:01.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:01.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:01.681 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:01.713 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:01.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:01.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:45:01.816 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:01.839 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:01.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:01.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:45:01.945 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:01.972 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:01.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:01.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:45:02.079 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:02.087 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:02 smithi092 bash[58855]: cluster 2023-09-23T19:45:00.284882+0000 mgr.y (mgr.25325) 1127 : cluster 0 pgmap v1095: 89 pgs: 89 active+clean; 2.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2023-09-23T19:45:02.109 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:02.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:02.115 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:45:02.215 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:02.225 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:01 smithi016 bash[77995]: cluster 2023-09-23T19:45:00.284882+0000 mgr.y (mgr.25325) 1127 : cluster 0 pgmap v1095: 89 pgs: 89 active+clean; 2.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2023-09-23T19:45:02.226 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:01 smithi016 bash[75004]: cluster 2023-09-23T19:45:00.284882+0000 mgr.y (mgr.25325) 1127 : cluster 0 pgmap v1095: 89 pgs: 89 active+clean; 2.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2023-09-23T19:45:02.245 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:02.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:02.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:02.352 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:02.380 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:02.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:02.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:02.485 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:02.514 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:02.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:02.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:02.617 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:02.652 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:02.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:02.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:45:02.756 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:02.781 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:02.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:02.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:02.886 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:02.919 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:02.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:02.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:03.025 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:03.056 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:03.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:03.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:03.161 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:03.191 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:03.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:03.193 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:03.294 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:03.318 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:03.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:03.322 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:45:03.423 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:03.458 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:03.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:03.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:03.564 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:03.588 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:03.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:03.590 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:45:03.691 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:03.727 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:03.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:03.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:03.831 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:03.858 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:03.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:03.863 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:03.964 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:04.002 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:04.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:04.005 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:04.106 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:04.119 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:03 smithi016 bash[75004]: cluster 2023-09-23T19:45:02.285423+0000 mgr.y (mgr.25325) 1128 : cluster 0 pgmap v1096: 89 pgs: 89 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 674 op/s 2023-09-23T19:45:04.120 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:03 smithi016 bash[77995]: cluster 2023-09-23T19:45:02.285423+0000 mgr.y (mgr.25325) 1128 : cluster 0 pgmap v1096: 89 pgs: 89 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 674 op/s 2023-09-23T19:45:04.155 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:04.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:04.161 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:04.262 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:04.270 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:04 smithi092 bash[58855]: cluster 2023-09-23T19:45:02.285423+0000 mgr.y (mgr.25325) 1128 : cluster 0 pgmap v1096: 89 pgs: 89 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 674 op/s 2023-09-23T19:45:04.302 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:04.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:04.312 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:04.413 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:04.446 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:04.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:04.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:45:04.552 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:04.581 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:04.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:04.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:04.684 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:04.714 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:04.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:04.720 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:04.820 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:04.850 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:04.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:04.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:45:04.958 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:04.981 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:04.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:04.989 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:45:05.089 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:05.117 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:05.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:05.120 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:45:05.220 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:05.247 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:05.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:05.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:05.353 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:05.389 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:05.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:05.395 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:45:05.496 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:05.529 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:05.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:05.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:05.634 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:05.659 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:05.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:05.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:05.765 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:05.806 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:05.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:05.810 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:05.910 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:05.953 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:05.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:05.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:06.009 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:45:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:45:05] "GET /metrics HTTP/1.1" 200 36306 "" "Prometheus/2.33.4" 2023-09-23T19:45:06.060 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:06.093 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:06.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:06.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:06.197 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:06.229 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:06.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:06.235 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:45:06.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:06 smithi016 bash[75004]: cluster 2023-09-23T19:45:04.285997+0000 mgr.y (mgr.25325) 1129 : cluster 0 pgmap v1097: 89 pgs: 89 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 672 op/s 2023-09-23T19:45:06.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:06 smithi016 bash[77995]: cluster 2023-09-23T19:45:04.285997+0000 mgr.y (mgr.25325) 1129 : cluster 0 pgmap v1097: 89 pgs: 89 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 672 op/s 2023-09-23T19:45:06.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:06 smithi092 bash[58855]: cluster 2023-09-23T19:45:04.285997+0000 mgr.y (mgr.25325) 1129 : cluster 0 pgmap v1097: 89 pgs: 89 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 672 op/s 2023-09-23T19:45:06.336 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:06.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:06.374 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:06.375 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:06.475 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:06.499 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:06.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:06.503 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:45:06.604 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:06.639 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:06.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:06.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:06.745 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:06.778 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:06.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:06.784 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:06.885 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:06.917 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:06.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:06.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:45:07.020 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:07.062 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:07.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:07.067 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:45:07.162 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:45:07.168 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:07.209 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:07.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:07.216 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:45:07.317 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:07.368 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:07.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:07.372 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:07.473 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:07.500 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:07.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:07.502 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:07.603 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:07.645 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:07.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:07.652 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:45:07.753 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:07.781 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:07.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:07.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:07.884 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:07.927 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:07.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:07.933 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:45:08.034 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:08.051 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:08.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:08.053 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:08.154 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:08.168 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:08 smithi016 bash[75004]: cluster 2023-09-23T19:45:06.286706+0000 mgr.y (mgr.25325) 1130 : cluster 0 pgmap v1098: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 698 op/s 2023-09-23T19:45:08.184 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:08.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:08.186 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:08.287 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:08.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:08 smithi092 bash[58855]: cluster 2023-09-23T19:45:06.286706+0000 mgr.y (mgr.25325) 1130 : cluster 0 pgmap v1098: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 698 op/s 2023-09-23T19:45:08.321 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:08.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:08.325 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:45:08.425 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:08.451 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:08.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:08.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:08.484 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:08 smithi016 bash[77995]: cluster 2023-09-23T19:45:06.286706+0000 mgr.y (mgr.25325) 1130 : cluster 0 pgmap v1098: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 698 op/s 2023-09-23T19:45:08.560 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:08.586 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:08.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:08.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:45:08.689 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:08.714 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:08.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:08.720 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:45:08.759 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-09-23T19:45:08.759 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:45:08.820 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:08.859 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:08.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:08.866 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:08.967 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:08.991 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:08.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:08.998 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:09.099 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:09.130 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:09.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:09.132 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:09.186 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:45:09.228 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:45:09.233 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:09.259 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:09.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:09.262 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:45:09.267 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:45:09.332 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:45:09.363 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:09.373 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:45:09.387 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:09.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:09.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:09.407 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:45:09.449 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:45:09.487 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:45:09.492 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:09.500 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:45:09.500 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:45:09.500 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:45:09.500 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:45:09.500 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:45:09.501 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:45:09.501 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:45:09.501 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:45:09.516 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:09.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:09.521 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:09.621 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:09.645 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:09.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:09.652 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:45:09.753 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:09.786 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:09.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:09.788 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:09.889 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:09.916 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:09.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:09.921 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:45:10.017 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:10 smithi016 bash[75004]: cluster 2023-09-23T19:45:08.287145+0000 mgr.y (mgr.25325) 1131 : cluster 0 pgmap v1099: 89 pgs: 89 active+clean; 1.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 702 op/s 2023-09-23T19:45:10.018 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:10 smithi016 bash[75004]: audit 2023-09-23T19:45:09.107703+0000 mon.b (mon.1) 51 : audit 0 from='client.? 172.21.15.16:0/2080474041' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:45:10.018 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:10 smithi016 bash[75004]: audit 2023-09-23T19:45:09.681940+0000 mon.a (mon.0) 1423 : audit 1 from='client.? 172.21.15.16:0/3858607380' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:45:10.018 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:10 smithi016 bash[77995]: cluster 2023-09-23T19:45:08.287145+0000 mgr.y (mgr.25325) 1131 : cluster 0 pgmap v1099: 89 pgs: 89 active+clean; 1.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 702 op/s 2023-09-23T19:45:10.018 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:10 smithi016 bash[77995]: audit 2023-09-23T19:45:09.107703+0000 mon.b (mon.1) 51 : audit 0 from='client.? 172.21.15.16:0/2080474041' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:45:10.018 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:10 smithi016 bash[77995]: audit 2023-09-23T19:45:09.681940+0000 mon.a (mon.0) 1423 : audit 1 from='client.? 172.21.15.16:0/3858607380' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:45:10.021 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:10.052 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:10.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:10.054 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:45:10.155 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:10.164 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:10 smithi092 bash[58855]: cluster 2023-09-23T19:45:08.287145+0000 mgr.y (mgr.25325) 1131 : cluster 0 pgmap v1099: 89 pgs: 89 active+clean; 1.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 702 op/s 2023-09-23T19:45:10.164 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:10 smithi092 bash[58855]: audit 2023-09-23T19:45:09.107703+0000 mon.b (mon.1) 51 : audit 0 from='client.? 172.21.15.16:0/2080474041' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:45:10.164 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:10 smithi092 bash[58855]: audit 2023-09-23T19:45:09.681940+0000 mon.a (mon.0) 1423 : audit 1 from='client.? 172.21.15.16:0/3858607380' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:45:10.187 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:10.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:10.192 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:45:10.293 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:10.323 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:10.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:10.328 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:45:10.428 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:10.456 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:10.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:10.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:10.562 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:10.565 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is set 2023-09-23T19:45:10.594 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:10.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:10.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:10.696 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:10.729 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:10.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:10.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:10.833 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:10.865 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:10.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:10.868 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:45:10.969 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:10.993 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:10.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:10.999 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:11.099 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:11.143 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:11.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:11.149 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:45:11.244 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:11 smithi016 bash[75004]: cluster 2023-09-23T19:45:10.551819+0000 mon.a (mon.0) 1424 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:45:11.244 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:11 smithi016 bash[75004]: audit 2023-09-23T19:45:10.558950+0000 mon.a (mon.0) 1425 : audit 1 from='client.? 172.21.15.16:0/3858607380' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:45:11.250 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:11.258 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:11 smithi092 bash[58855]: cluster 2023-09-23T19:45:10.551819+0000 mon.a (mon.0) 1424 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:45:11.258 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:11 smithi092 bash[58855]: audit 2023-09-23T19:45:10.558950+0000 mon.a (mon.0) 1425 : audit 1 from='client.? 172.21.15.16:0/3858607380' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:45:11.258 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:11 smithi092 bash[58855]: cluster 2023-09-23T19:45:10.559038+0000 mon.a (mon.0) 1426 : cluster 0 osdmap e200: 8 total, 8 up, 8 in 2023-09-23T19:45:11.258 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:11 smithi092 bash[58855]: cluster 2023-09-23T19:45:10.575929+0000 osd.7 (osd.7) 117 : cluster 3 failed to encode map e200 with expected crc 2023-09-23T19:45:11.279 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:11.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:11.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:45:11.385 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:11.410 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:11.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:11.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:45:11.514 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:11.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:11 smithi016 bash[75004]: cluster 2023-09-23T19:45:10.559038+0000 mon.a (mon.0) 1426 : cluster 0 osdmap e200: 8 total, 8 up, 8 in 2023-09-23T19:45:11.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:11 smithi016 bash[75004]: cluster 2023-09-23T19:45:10.575929+0000 osd.7 (osd.7) 117 : cluster 3 failed to encode map e200 with expected crc 2023-09-23T19:45:11.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:11 smithi016 bash[77995]: cluster 2023-09-23T19:45:10.551819+0000 mon.a (mon.0) 1424 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:45:11.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:11 smithi016 bash[77995]: audit 2023-09-23T19:45:10.558950+0000 mon.a (mon.0) 1425 : audit 1 from='client.? 172.21.15.16:0/3858607380' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:45:11.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:11 smithi016 bash[77995]: cluster 2023-09-23T19:45:10.559038+0000 mon.a (mon.0) 1426 : cluster 0 osdmap e200: 8 total, 8 up, 8 in 2023-09-23T19:45:11.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:11 smithi016 bash[77995]: cluster 2023-09-23T19:45:10.575929+0000 osd.7 (osd.7) 117 : cluster 3 failed to encode map e200 with expected crc 2023-09-23T19:45:11.541 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:11.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:11.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:45:11.650 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:11.685 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:11.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:11.689 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:11.789 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:11.822 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:11.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:11.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:45:11.926 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:11.955 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:11.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:11.959 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:45:12.060 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:12.089 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:12.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:12.091 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:12.192 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:12.214 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:12.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:12.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:12.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:12 smithi092 bash[58855]: cluster 2023-09-23T19:45:10.287751+0000 mgr.y (mgr.25325) 1132 : cluster 0 pgmap v1100: 89 pgs: 89 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 789 op/s 2023-09-23T19:45:12.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:12 smithi092 bash[58855]: cluster 2023-09-23T19:45:10.562351+0000 osd.5 (osd.5) 128 : cluster 3 failed to encode map e200 with expected crc 2023-09-23T19:45:12.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:12 smithi092 bash[58855]: cluster 2023-09-23T19:45:10.581182+0000 osd.4 (osd.4) 141 : cluster 3 failed to encode map e200 with expected crc 2023-09-23T19:45:12.316 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:12.350 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:12.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:12.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:45:12.455 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:12.482 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:12.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:12.487 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:12.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:12 smithi016 bash[75004]: cluster 2023-09-23T19:45:10.287751+0000 mgr.y (mgr.25325) 1132 : cluster 0 pgmap v1100: 89 pgs: 89 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 789 op/s 2023-09-23T19:45:12.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:12 smithi016 bash[75004]: cluster 2023-09-23T19:45:10.562351+0000 osd.5 (osd.5) 128 : cluster 3 failed to encode map e200 with expected crc 2023-09-23T19:45:12.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:12 smithi016 bash[75004]: cluster 2023-09-23T19:45:10.581182+0000 osd.4 (osd.4) 141 : cluster 3 failed to encode map e200 with expected crc 2023-09-23T19:45:12.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:12 smithi016 bash[77995]: cluster 2023-09-23T19:45:10.287751+0000 mgr.y (mgr.25325) 1132 : cluster 0 pgmap v1100: 89 pgs: 89 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 789 op/s 2023-09-23T19:45:12.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:12 smithi016 bash[77995]: cluster 2023-09-23T19:45:10.562351+0000 osd.5 (osd.5) 128 : cluster 3 failed to encode map e200 with expected crc 2023-09-23T19:45:12.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:12 smithi016 bash[77995]: cluster 2023-09-23T19:45:10.581182+0000 osd.4 (osd.4) 141 : cluster 3 failed to encode map e200 with expected crc 2023-09-23T19:45:12.588 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:12.614 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:12.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:12.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:12.718 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:12.746 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:12.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:12.749 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:45:12.849 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:12.879 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:12.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:12.881 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:45:12.982 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:13.017 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:13.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:13.020 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:13.094 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T19:45:13.121 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:13.147 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:13.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:13.149 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:13.250 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:13.283 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:13.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:13.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:45:13.389 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:13.435 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:13.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:13.440 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:45:13.541 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:13.573 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:13.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:13.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:45:13.676 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:13.700 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:13.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:13.707 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:13.807 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:13.847 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:13.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:13.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:13.950 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:13.993 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:13.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:13.997 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:14.098 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:14.129 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:14.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:14.132 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:14.233 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:14.264 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:14.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:14.272 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:14.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:14 smithi092 bash[58855]: cluster 2023-09-23T19:45:12.288280+0000 mgr.y (mgr.25325) 1133 : cluster 0 pgmap v1102: 89 pgs: 89 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2023-09-23T19:45:14.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:14 smithi092 bash[58855]: audit 2023-09-23T19:45:13.674867+0000 mon.a (mon.0) 1427 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:45:14.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:14 smithi092 bash[58855]: audit 2023-09-23T19:45:13.680566+0000 mon.a (mon.0) 1428 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:45:14.372 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:14.383 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:14 smithi016 bash[75004]: cluster 2023-09-23T19:45:12.288280+0000 mgr.y (mgr.25325) 1133 : cluster 0 pgmap v1102: 89 pgs: 89 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2023-09-23T19:45:14.384 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:14 smithi016 bash[75004]: audit 2023-09-23T19:45:13.674867+0000 mon.a (mon.0) 1427 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:45:14.384 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:14 smithi016 bash[75004]: audit 2023-09-23T19:45:13.680566+0000 mon.a (mon.0) 1428 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:45:14.384 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:14 smithi016 bash[77995]: cluster 2023-09-23T19:45:12.288280+0000 mgr.y (mgr.25325) 1133 : cluster 0 pgmap v1102: 89 pgs: 89 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2023-09-23T19:45:14.384 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:14 smithi016 bash[77995]: audit 2023-09-23T19:45:13.674867+0000 mon.a (mon.0) 1427 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:45:14.384 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:14 smithi016 bash[77995]: audit 2023-09-23T19:45:13.680566+0000 mon.a (mon.0) 1428 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:45:14.397 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:14.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:14.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:45:14.500 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:14.523 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:14.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:14.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:45:14.628 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:14.655 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:14.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:14.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:14.758 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:14.789 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:14.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:14.794 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:45:14.894 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:14.926 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:14.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:14.935 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:45:15.036 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:15.064 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:15.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:15.068 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:45:15.168 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:15.195 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:15.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:15.202 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:15.302 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:15.334 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:15.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:15.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:15.436 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:15.442 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:45:15.467 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:15.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:15.471 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:15.573 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:15.609 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:15.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:15.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:15.713 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:15.723 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:45:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:45:15] "GET /metrics HTTP/1.1" 200 36304 "" "Prometheus/2.33.4" 2023-09-23T19:45:15.759 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:15.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:15.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:15.865 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:15.907 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:15.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:15.913 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:45:16.013 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:16.042 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:16.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:16.045 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:45:16.076 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T19:45:16.145 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:16.178 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:16.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:16.181 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:16.282 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:16.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:16 smithi092 bash[58855]: cluster 2023-09-23T19:45:14.289074+0000 mgr.y (mgr.25325) 1134 : cluster 0 pgmap v1103: 89 pgs: 89 active+clean; 1.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 691 op/s 2023-09-23T19:45:16.319 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:16.327 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:16 smithi016 bash[75004]: cluster 2023-09-23T19:45:14.289074+0000 mgr.y (mgr.25325) 1134 : cluster 0 pgmap v1103: 89 pgs: 89 active+clean; 1.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 691 op/s 2023-09-23T19:45:16.327 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:16 smithi016 bash[77995]: cluster 2023-09-23T19:45:14.289074+0000 mgr.y (mgr.25325) 1134 : cluster 0 pgmap v1103: 89 pgs: 89 active+clean; 1.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 691 op/s 2023-09-23T19:45:16.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:16.328 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:45:16.429 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:16.460 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:16.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:16.469 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:16.570 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:16.593 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:16.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:16.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:45:16.697 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:16.732 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:16.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:16.737 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:16.838 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:16.873 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:16.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:16.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:16.977 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:17.019 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:17.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:17.020 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:45:17.077 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T19:45:17.110 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:17 smithi016 bash[75004]: audit 2023-09-23T19:45:16.115399+0000 mon.a (mon.0) 1429 : audit 1 from='client.? 172.21.15.16:0/3353046668' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T19:45:17.111 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:17 smithi016 bash[77995]: audit 2023-09-23T19:45:16.115399+0000 mon.a (mon.0) 1429 : audit 1 from='client.? 172.21.15.16:0/3353046668' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T19:45:17.121 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:17.145 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:17.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:17.149 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:17.249 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:17.286 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:17.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:17.288 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:17.389 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:17.416 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:17.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:17.425 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:45:17.526 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:17.535 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:17 smithi092 bash[58855]: audit 2023-09-23T19:45:16.115399+0000 mon.a (mon.0) 1429 : audit 1 from='client.? 172.21.15.16:0/3353046668' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T19:45:17.568 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:17.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:17.574 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:45:17.675 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:17.710 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:17.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:17.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:17.813 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:17.857 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:17.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:17.861 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:17.962 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:18.001 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:18.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:18.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:18.108 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:18.158 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:18.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:18.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:18.265 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:18.274 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:18 smithi092 bash[58855]: cluster 2023-09-23T19:45:16.290097+0000 mgr.y (mgr.25325) 1135 : cluster 0 pgmap v1104: 89 pgs: 89 active+clean; 1.4 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 690 op/s 2023-09-23T19:45:18.274 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:18 smithi092 bash[58855]: audit 2023-09-23T19:45:17.062631+0000 mon.a (mon.0) 1430 : audit 1 from='client.? 172.21.15.16:0/3353046668' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T19:45:18.274 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:18 smithi092 bash[58855]: cluster 2023-09-23T19:45:17.062934+0000 mon.a (mon.0) 1431 : cluster 0 osdmap e201: 8 total, 8 up, 8 in 2023-09-23T19:45:18.274 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:18 smithi092 bash[58855]: cluster 2023-09-23T19:45:17.076578+0000 osd.6 (osd.6) 128 : cluster 3 failed to encode map e201 with expected crc 2023-09-23T19:45:18.274 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:18 smithi092 bash[58855]: cluster 2023-09-23T19:45:17.082927+0000 osd.4 (osd.4) 142 : cluster 3 failed to encode map e201 with expected crc 2023-09-23T19:45:18.298 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:18.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:18.304 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:18.405 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:18.433 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T19:45:18.433 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:18 smithi016 bash[77995]: cluster 2023-09-23T19:45:16.290097+0000 mgr.y (mgr.25325) 1135 : cluster 0 pgmap v1104: 89 pgs: 89 active+clean; 1.4 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 690 op/s 2023-09-23T19:45:18.433 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:18 smithi016 bash[77995]: audit 2023-09-23T19:45:17.062631+0000 mon.a (mon.0) 1430 : audit 1 from='client.? 172.21.15.16:0/3353046668' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T19:45:18.433 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:18 smithi016 bash[77995]: cluster 2023-09-23T19:45:17.062934+0000 mon.a (mon.0) 1431 : cluster 0 osdmap e201: 8 total, 8 up, 8 in 2023-09-23T19:45:18.433 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:18 smithi016 bash[77995]: cluster 2023-09-23T19:45:17.076578+0000 osd.6 (osd.6) 128 : cluster 3 failed to encode map e201 with expected crc 2023-09-23T19:45:18.433 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:18 smithi016 bash[77995]: cluster 2023-09-23T19:45:17.082927+0000 osd.4 (osd.4) 142 : cluster 3 failed to encode map e201 with expected crc 2023-09-23T19:45:18.434 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:18 smithi016 bash[75004]: cluster 2023-09-23T19:45:16.290097+0000 mgr.y (mgr.25325) 1135 : cluster 0 pgmap v1104: 89 pgs: 89 active+clean; 1.4 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 690 op/s 2023-09-23T19:45:18.434 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:18 smithi016 bash[75004]: audit 2023-09-23T19:45:17.062631+0000 mon.a (mon.0) 1430 : audit 1 from='client.? 172.21.15.16:0/3353046668' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T19:45:18.434 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:18 smithi016 bash[75004]: cluster 2023-09-23T19:45:17.062934+0000 mon.a (mon.0) 1431 : cluster 0 osdmap e201: 8 total, 8 up, 8 in 2023-09-23T19:45:18.434 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:18 smithi016 bash[75004]: cluster 2023-09-23T19:45:17.076578+0000 osd.6 (osd.6) 128 : cluster 3 failed to encode map e201 with expected crc 2023-09-23T19:45:18.434 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:18 smithi016 bash[75004]: cluster 2023-09-23T19:45:17.082927+0000 osd.4 (osd.4) 142 : cluster 3 failed to encode map e201 with expected crc 2023-09-23T19:45:18.447 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:18.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:18.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:18.555 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:18.582 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:18.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:18.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:18.688 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:18.719 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:18.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:18.723 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:45:18.824 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:18.855 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:18.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:18.858 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:18.915 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:45:18.959 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:18.985 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:18.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:18.992 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:19.093 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:19.124 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:19.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:19.130 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:19.230 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:19.269 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:19.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:19.272 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:45:19.372 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:19.395 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:19.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:19.401 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:19.502 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:19.539 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:19.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:19.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:19.646 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:19.676 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:19.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:19.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:19.779 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:19.810 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:19.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:19.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:19.915 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:19.953 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:19.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:19.958 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:45:20.059 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:20.094 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:20.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:20.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:45:20.198 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:20.235 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:20.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:20.240 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:20.341 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:20.345 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:20 smithi016 bash[77995]: cluster 2023-09-23T19:45:18.290527+0000 mgr.y (mgr.25325) 1136 : cluster 0 pgmap v1106: 89 pgs: 89 active+clean; 1.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 611 op/s 2023-09-23T19:45:20.346 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:20 smithi016 bash[75004]: cluster 2023-09-23T19:45:18.290527+0000 mgr.y (mgr.25325) 1136 : cluster 0 pgmap v1106: 89 pgs: 89 active+clean; 1.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 611 op/s 2023-09-23T19:45:20.365 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:20.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:20.367 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:45:20.467 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:20.503 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:20.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:20.505 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:45:20.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:20 smithi092 bash[58855]: cluster 2023-09-23T19:45:18.290527+0000 mgr.y (mgr.25325) 1136 : cluster 0 pgmap v1106: 89 pgs: 89 active+clean; 1.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 611 op/s 2023-09-23T19:45:20.605 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:20.625 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:20.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:20.629 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:45:20.729 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:20.764 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:20.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:20.766 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:20.867 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:20.902 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:20.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:20.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:45:20.959 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:45:20.960 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:45:21.008 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:21.032 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:21.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:21.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:45:21.144 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:21.175 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:21.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:21.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:21.280 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:21.309 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:21.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:21.315 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:21.416 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:21.449 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:21.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:21.453 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:45:21.554 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:21.582 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:21.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:21.586 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:21.687 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:21.717 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:21.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:21.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:45:21.823 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:21.857 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:21.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:21.860 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:21.961 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:22.010 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:22.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:22.014 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:45:22.115 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:22.124 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:22 smithi092 bash[58855]: cluster 2023-09-23T19:45:20.291193+0000 mgr.y (mgr.25325) 1137 : cluster 0 pgmap v1107: 89 pgs: 89 active+clean; 1.2 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 693 op/s 2023-09-23T19:45:22.147 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:22.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:22.152 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:45:22.253 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:22.278 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:22.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:22.281 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:22.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:22 smithi016 bash[75004]: cluster 2023-09-23T19:45:20.291193+0000 mgr.y (mgr.25325) 1137 : cluster 0 pgmap v1107: 89 pgs: 89 active+clean; 1.2 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 693 op/s 2023-09-23T19:45:22.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:22 smithi016 bash[77995]: cluster 2023-09-23T19:45:20.291193+0000 mgr.y (mgr.25325) 1137 : cluster 0 pgmap v1107: 89 pgs: 89 active+clean; 1.2 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 693 op/s 2023-09-23T19:45:22.382 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:22.409 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:22.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:22.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:45:22.516 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:22.549 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:22.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:22.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:22.652 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:22.689 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:22.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:22.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:45:22.793 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:22.818 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:22.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:22.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:22.922 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:22.958 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:22.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:22.965 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:45:23.066 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:23.089 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:23.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:23.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:45:23.162 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:45:23.193 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:23.241 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:23.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:23.246 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:23.347 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:23.385 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:23.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:23.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:45:23.490 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:23.533 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:23.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:23.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:45:23.640 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:23.664 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:23.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:23.670 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:23.771 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:23.798 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:23.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:23.801 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:45:23.902 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:23.933 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:23.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:23.944 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:24.045 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:24.073 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:24.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:24.076 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:24.088 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:24 smithi016 bash[77995]: cluster 2023-09-23T19:45:22.291654+0000 mgr.y (mgr.25325) 1138 : cluster 0 pgmap v1108: 89 pgs: 89 active+clean; 1.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 683 op/s 2023-09-23T19:45:24.088 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:24 smithi016 bash[75004]: cluster 2023-09-23T19:45:22.291654+0000 mgr.y (mgr.25325) 1138 : cluster 0 pgmap v1108: 89 pgs: 89 active+clean; 1.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 683 op/s 2023-09-23T19:45:24.178 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:24.208 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:24.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:24.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:24.312 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:24.343 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:24.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:24.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:45:24.447 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:24.484 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:24.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:24.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:45:24.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:24 smithi092 bash[58855]: cluster 2023-09-23T19:45:22.291654+0000 mgr.y (mgr.25325) 1138 : cluster 0 pgmap v1108: 89 pgs: 89 active+clean; 1.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 683 op/s 2023-09-23T19:45:24.588 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:24.630 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:24.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:24.634 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:45:24.734 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:24.760 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:24.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:24.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:45:24.869 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:24.893 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:24.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:24.896 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:45:24.997 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:25.032 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:25.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:25.036 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:45:25.137 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:25.172 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:25.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:25.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:45:25.276 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:25.298 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:25.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:25.300 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:45:25.401 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:25.428 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:25.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:25.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:25.532 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:25.553 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:25.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:25.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:25.656 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:25.682 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:25.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:25.684 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:25.785 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:25.826 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:25.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:25.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:25.935 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:25.971 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:25.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:25.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:45:26.036 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:45:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:45:25] "GET /metrics HTTP/1.1" 200 36304 "" "Prometheus/2.33.4" 2023-09-23T19:45:26.078 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:26.108 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:26.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:26.111 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:26.212 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:26.220 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:26 smithi016 bash[77995]: cluster 2023-09-23T19:45:24.292140+0000 mgr.y (mgr.25325) 1139 : cluster 0 pgmap v1109: 89 pgs: 89 active+clean; 1.1 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2023-09-23T19:45:26.221 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:26 smithi016 bash[75004]: cluster 2023-09-23T19:45:24.292140+0000 mgr.y (mgr.25325) 1139 : cluster 0 pgmap v1109: 89 pgs: 89 active+clean; 1.1 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2023-09-23T19:45:26.240 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:26.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:26.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:45:26.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:26 smithi092 bash[58855]: cluster 2023-09-23T19:45:24.292140+0000 mgr.y (mgr.25325) 1139 : cluster 0 pgmap v1109: 89 pgs: 89 active+clean; 1.1 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2023-09-23T19:45:26.346 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:26.379 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:26.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:26.385 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:26.486 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:26.518 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:26.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:26.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:45:26.621 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:26.652 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:26.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:26.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:26.756 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:26.789 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:26.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:26.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:26.892 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:26.918 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:26.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:26.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:26.931 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-09-23T19:45:26.931 INFO:teuthology.orchestra.run.smithi016.stdout: "size": 20, 2023-09-23T19:45:26.931 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 600, 2023-09-23T19:45:26.931 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [ 2023-09-23T19:45:26.931 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.932 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45947.0:14112 10.5 e149/144 10:a053e224:::benchmark_data_smithi016_136365_object14111:head v 149'896, mlcod=149'894)", 2023-09-23T19:45:26.932 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:36:41.195266+0000", 2023-09-23T19:45:26.932 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 525.72971271799997, 2023-09-23T19:45:26.932 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.15845964700000001, 2023-09-23T19:45:26.932 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:45:26.932 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:45:26.932 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:45:26.932 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.932 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:45:26.932 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:41.195266+0000", 2023-09-23T19:45:26.932 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:45:26.933 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.933 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.933 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:45:26.933 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:41.195266+0000", 2023-09-23T19:45:26.933 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:45:26.933 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.933 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.933 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:45:26.933 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:41.195266+0000", 2023-09-23T19:45:26.933 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T19:45:26.933 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.934 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.934 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:45:26.934 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:41.195291+0000", 2023-09-23T19:45:26.934 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.4984e-05 2023-09-23T19:45:26.934 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.934 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.934 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:45:26.934 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:41.195305+0000", 2023-09-23T19:45:26.934 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.4562999999999999e-05 2023-09-23T19:45:26.934 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.934 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.935 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:45:26.935 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:41.195310+0000", 2023-09-23T19:45:26.935 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.096e-06 2023-09-23T19:45:26.935 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.935 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.935 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:45:26.935 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:41.195355+0000", 2023-09-23T19:45:26.935 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.4904999999999998e-05 2023-09-23T19:45:26.935 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.935 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.935 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:45:26.936 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:41.195371+0000", 2023-09-23T19:45:26.936 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.525e-05 2023-09-23T19:45:26.936 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.936 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.936 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:45:26.936 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:41.353633+0000", 2023-09-23T19:45:26.936 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.15826241899999999 2023-09-23T19:45:26.936 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.936 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.936 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:45:26.936 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:41.353726+0000", 2023-09-23T19:45:26.936 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.2832999999999997e-05 2023-09-23T19:45:26.937 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:45:26.937 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:45:26.937 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:45:26.937 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.937 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.937 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45947.0:14114 10.5 e149/144 10:ac11c1de:::benchmark_data_smithi016_136365_object14113:head v 149'897, mlcod=149'894)", 2023-09-23T19:45:26.937 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:36:41.195330+0000", 2023-09-23T19:45:26.937 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 525.72964850699998, 2023-09-23T19:45:26.937 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.15916371000000001, 2023-09-23T19:45:26.937 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:45:26.937 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:45:26.938 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:45:26.938 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.938 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:45:26.938 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:41.195330+0000", 2023-09-23T19:45:26.938 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:45:26.938 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.938 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.938 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:45:26.938 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:41.195330+0000", 2023-09-23T19:45:26.938 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:45:26.938 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.939 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.939 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:45:26.939 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:41.195330+0000", 2023-09-23T19:45:26.939 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T19:45:26.939 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.939 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.939 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:45:26.939 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:41.195350+0000", 2023-09-23T19:45:26.939 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.0327999999999999e-05 2023-09-23T19:45:26.939 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.939 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.939 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:45:26.940 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:41.195355+0000", 2023-09-23T19:45:26.940 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.8069999999999999e-06 2023-09-23T19:45:26.940 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.940 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.940 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:45:26.940 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:41.195358+0000", 2023-09-23T19:45:26.940 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.0249999999999998e-06 2023-09-23T19:45:26.940 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.940 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.940 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:45:26.940 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:41.352008+0000", 2023-09-23T19:45:26.941 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.15665005700000001 2023-09-23T19:45:26.941 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.941 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.941 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:45:26.941 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:41.352043+0000", 2023-09-23T19:45:26.941 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.4606e-05 2023-09-23T19:45:26.941 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.941 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.941 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:45:26.941 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:41.354462+0000", 2023-09-23T19:45:26.941 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0024194149999999998 2023-09-23T19:45:26.941 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.942 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.942 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:45:26.942 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:41.354494+0000", 2023-09-23T19:45:26.942 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.1758000000000003e-05 2023-09-23T19:45:26.942 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:45:26.942 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:45:26.942 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:45:26.942 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.942 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.942 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45947.0:14094 10.7 e149/144 10:e5c68248:::benchmark_data_smithi016_136365_object14093:head v 149'915, mlcod=149'913)", 2023-09-23T19:45:26.942 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:36:41.195404+0000", 2023-09-23T19:45:26.943 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 525.72957522700005, 2023-09-23T19:45:26.943 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.15998005500000001, 2023-09-23T19:45:26.943 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:45:26.943 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:45:26.943 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:45:26.943 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.943 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:45:26.943 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:41.195404+0000", 2023-09-23T19:45:26.943 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:45:26.943 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.943 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.943 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:45:26.943 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:41.195404+0000", 2023-09-23T19:45:26.944 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:45:26.944 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.944 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.944 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:45:26.944 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:41.195403+0000", 2023-09-23T19:45:26.944 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T19:45:26.944 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.944 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.944 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:45:26.944 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:41.195434+0000", 2023-09-23T19:45:26.944 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.0463999999999998e-05 2023-09-23T19:45:26.944 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.944 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.945 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:45:26.945 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:41.195441+0000", 2023-09-23T19:45:26.945 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.7680000000000003e-06 2023-09-23T19:45:26.945 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.945 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.945 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:45:26.945 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:41.195444+0000", 2023-09-23T19:45:26.945 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.8249999999999998e-06 2023-09-23T19:45:26.945 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.945 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.945 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:45:26.945 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:41.195498+0000", 2023-09-23T19:45:26.946 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.3776000000000001e-05 2023-09-23T19:45:26.946 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.946 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.946 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:45:26.946 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:41.195520+0000", 2023-09-23T19:45:26.946 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.2078000000000001e-05 2023-09-23T19:45:26.946 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.946 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.946 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:45:26.946 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:41.355359+0000", 2023-09-23T19:45:26.946 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.15983919799999999 2023-09-23T19:45:26.947 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.947 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.947 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:45:26.947 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:41.355384+0000", 2023-09-23T19:45:26.947 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.4329999999999999e-05 2023-09-23T19:45:26.947 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:45:26.947 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:45:26.947 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:45:26.947 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.947 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.947 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45947.0:14092 10.f e149/144 10:f9ca31ec:::benchmark_data_smithi016_136365_object14091:head v 149'906, mlcod=149'904)", 2023-09-23T19:45:26.947 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:36:41.195456+0000", 2023-09-23T19:45:26.948 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 525.72952240699999, 2023-09-23T19:45:26.948 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.15989957699999999, 2023-09-23T19:45:26.948 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:45:26.948 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:45:26.948 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:45:26.948 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.948 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:45:26.948 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:41.195456+0000", 2023-09-23T19:45:26.948 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:45:26.948 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.948 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.949 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:45:26.949 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:41.195456+0000", 2023-09-23T19:45:26.949 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:45:26.949 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.949 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.949 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:45:26.949 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:41.195456+0000", 2023-09-23T19:45:26.949 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967296 2023-09-23T19:45:26.949 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.949 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.949 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:45:26.950 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:41.195483+0000", 2023-09-23T19:45:26.950 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.7098e-05 2023-09-23T19:45:26.950 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.950 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.950 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:45:26.950 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:41.195488+0000", 2023-09-23T19:45:26.950 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.2889999999999999e-06 2023-09-23T19:45:26.950 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.950 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.950 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:45:26.950 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:41.195492+0000", 2023-09-23T19:45:26.951 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.873e-06 2023-09-23T19:45:26.951 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.951 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.951 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:45:26.951 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:41.195548+0000", 2023-09-23T19:45:26.951 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.6765000000000003e-05 2023-09-23T19:45:26.951 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.951 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.951 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:45:26.951 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:41.195565+0000", 2023-09-23T19:45:26.951 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.6744000000000001e-05 2023-09-23T19:45:26.952 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.952 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.952 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:45:26.952 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:41.355306+0000", 2023-09-23T19:45:26.952 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.15974112500000001 2023-09-23T19:45:26.952 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.952 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.952 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:45:26.952 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:41.355356+0000", 2023-09-23T19:45:26.952 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.9903e-05 2023-09-23T19:45:26.952 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:45:26.952 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:45:26.953 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:45:26.953 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.953 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.953 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45947.0:14101 10.f e149/144 10:fc6e3296:::benchmark_data_smithi016_136365_object14100:head v 149'907, mlcod=149'904)", 2023-09-23T19:45:26.953 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:36:41.195673+0000", 2023-09-23T19:45:26.953 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 525.72930555699998, 2023-09-23T19:45:26.953 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.16042768600000001, 2023-09-23T19:45:26.953 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:45:26.953 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:45:26.953 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:45:26.953 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.953 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:45:26.954 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:41.195673+0000", 2023-09-23T19:45:26.954 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:45:26.954 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.954 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.954 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:45:26.954 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:41.195673+0000", 2023-09-23T19:45:26.954 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:45:26.954 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.954 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.954 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:45:26.954 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:41.195673+0000", 2023-09-23T19:45:26.955 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:45:26.955 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.955 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.955 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:45:26.955 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:41.195776+0000", 2023-09-23T19:45:26.955 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000103865 2023-09-23T19:45:26.955 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.955 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.955 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:45:26.955 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:41.351711+0000", 2023-09-23T19:45:26.955 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.15593510199999999 2023-09-23T19:45:26.955 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.956 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.956 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:45:26.956 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:41.351735+0000", 2023-09-23T19:45:26.956 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.3366e-05 2023-09-23T19:45:26.956 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.956 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.956 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:45:26.956 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:41.352538+0000", 2023-09-23T19:45:26.956 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00080316999999999997 2023-09-23T19:45:26.956 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.956 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.957 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:45:26.957 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:41.352557+0000", 2023-09-23T19:45:26.957 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.9284999999999999e-05 2023-09-23T19:45:26.957 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.957 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.957 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:45:26.957 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:41.356059+0000", 2023-09-23T19:45:26.957 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0035019140000000001 2023-09-23T19:45:26.957 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.957 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.957 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:45:26.957 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:36:41.356101+0000", 2023-09-23T19:45:26.958 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.1826000000000002e-05 2023-09-23T19:45:26.958 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:45:26.958 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:45:26.958 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:45:26.958 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.958 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.958 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.36243.0:66594 11.8 e186/180 11:1f0e70a2:::benchmark_data_smithi016_160148_object66593:head v 186'4234, mlcod=186'4231)", 2023-09-23T19:45:26.958 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:42:27.795500+0000", 2023-09-23T19:45:26.958 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 179.12947906400001, 2023-09-23T19:45:26.958 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.116549746, 2023-09-23T19:45:26.958 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:45:26.958 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:45:26.959 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:45:26.959 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.959 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:45:26.959 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.795500+0000", 2023-09-23T19:45:26.959 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:45:26.959 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.959 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.959 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:45:26.959 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.795500+0000", 2023-09-23T19:45:26.959 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:45:26.959 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.959 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.960 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:45:26.960 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.795499+0000", 2023-09-23T19:45:26.960 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T19:45:26.960 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.960 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.960 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:45:26.960 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.800845+0000", 2023-09-23T19:45:26.960 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0053454970000000003 2023-09-23T19:45:26.960 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.960 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.960 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:45:26.960 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.800858+0000", 2023-09-23T19:45:26.961 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.291e-05 2023-09-23T19:45:26.961 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.961 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.961 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:45:26.961 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.800866+0000", 2023-09-23T19:45:26.961 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.9999999999999996e-06 2023-09-23T19:45:26.961 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.961 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.961 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:45:26.961 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.801209+0000", 2023-09-23T19:45:26.961 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00034295700000000002 2023-09-23T19:45:26.961 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.962 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.962 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:45:26.962 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.801231+0000", 2023-09-23T19:45:26.962 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.198e-05 2023-09-23T19:45:26.962 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.962 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.962 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:45:26.962 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.911893+0000", 2023-09-23T19:45:26.962 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.110662076 2023-09-23T19:45:26.962 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.962 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.963 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:45:26.963 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.912050+0000", 2023-09-23T19:45:26.963 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00015690600000000001 2023-09-23T19:45:26.963 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:45:26.963 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:45:26.963 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:45:26.963 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.963 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.963 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.36243.0:66598 11.8 e186/180 11:15a3a476:::benchmark_data_smithi016_160148_object66597:head v 186'4235, mlcod=186'4231)", 2023-09-23T19:45:26.963 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:42:27.800888+0000", 2023-09-23T19:45:26.963 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 179.12409063600001, 2023-09-23T19:45:26.963 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.11120917900000001, 2023-09-23T19:45:26.964 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:45:26.964 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:45:26.964 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:45:26.964 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.964 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:45:26.964 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.800888+0000", 2023-09-23T19:45:26.964 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:45:26.964 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.964 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.964 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:45:26.964 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.800888+0000", 2023-09-23T19:45:26.965 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:45:26.965 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.965 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.965 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:45:26.965 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.800888+0000", 2023-09-23T19:45:26.965 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T19:45:26.965 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.965 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.965 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:45:26.965 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.800918+0000", 2023-09-23T19:45:26.965 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.9940000000000001e-05 2023-09-23T19:45:26.965 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.966 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.966 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:45:26.966 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.800925+0000", 2023-09-23T19:45:26.966 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.1069999999999999e-06 2023-09-23T19:45:26.966 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.966 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.966 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:45:26.966 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.800930+0000", 2023-09-23T19:45:26.966 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.5280000000000003e-06 2023-09-23T19:45:26.966 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.966 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.966 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:45:26.967 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.806034+0000", 2023-09-23T19:45:26.967 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0051039299999999996 2023-09-23T19:45:26.967 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.967 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.967 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:45:26.967 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.806057+0000", 2023-09-23T19:45:26.967 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.2314999999999999e-05 2023-09-23T19:45:26.967 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.967 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.967 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:45:26.967 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.912055+0000", 2023-09-23T19:45:26.967 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.105998727 2023-09-23T19:45:26.967 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.968 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.968 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:45:26.968 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.912097+0000", 2023-09-23T19:45:26.968 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.2196000000000001e-05 2023-09-23T19:45:26.968 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:45:26.968 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:45:26.968 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:45:26.968 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.968 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.968 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.36243.0:66608 11.0 e186/180 11:0c2a2515:::benchmark_data_smithi016_160148_object66607:head v 186'4188, mlcod=186'4186)", 2023-09-23T19:45:26.968 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:42:27.805597+0000", 2023-09-23T19:45:26.968 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 179.11938198199999, 2023-09-23T19:45:26.969 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.108608076, 2023-09-23T19:45:26.969 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:45:26.969 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:45:26.969 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:45:26.969 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.969 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:45:26.969 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.805597+0000", 2023-09-23T19:45:26.969 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:45:26.969 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.969 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.969 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:45:26.969 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.805597+0000", 2023-09-23T19:45:26.969 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:45:26.970 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.970 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.970 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:45:26.970 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.805596+0000", 2023-09-23T19:45:26.970 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:45:26.970 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.970 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.970 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:45:26.970 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.805647+0000", 2023-09-23T19:45:26.970 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.1523999999999997e-05 2023-09-23T19:45:26.971 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.971 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.971 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:45:26.971 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.805657+0000", 2023-09-23T19:45:26.971 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.9850000000000001e-06 2023-09-23T19:45:26.971 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.971 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.971 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:45:26.971 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.805663+0000", 2023-09-23T19:45:26.971 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.9800000000000003e-06 2023-09-23T19:45:26.971 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.971 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.971 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:45:26.972 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.807282+0000", 2023-09-23T19:45:26.972 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001618626 2023-09-23T19:45:26.972 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.972 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.972 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:45:26.972 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.807388+0000", 2023-09-23T19:45:26.972 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000105518 2023-09-23T19:45:26.972 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.972 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.972 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:45:26.972 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.914161+0000", 2023-09-23T19:45:26.972 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.10677305199999999 2023-09-23T19:45:26.973 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.973 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.973 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:45:26.973 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.914205+0000", 2023-09-23T19:45:26.973 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.4345000000000003e-05 2023-09-23T19:45:26.973 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:45:26.973 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:45:26.973 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:45:26.973 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.973 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.973 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.36243.0:66616 11.0 e186/180 11:0463f26c:::benchmark_data_smithi016_160148_object66615:head v 186'4189, mlcod=186'4186)", 2023-09-23T19:45:26.973 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:42:27.808950+0000", 2023-09-23T19:45:26.974 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 179.11602848000001, 2023-09-23T19:45:26.974 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.10536155, 2023-09-23T19:45:26.974 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:45:26.974 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:45:26.974 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:45:26.974 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.974 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:45:26.974 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.808950+0000", 2023-09-23T19:45:26.974 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:45:26.974 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.974 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.975 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:45:26.975 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.808950+0000", 2023-09-23T19:45:26.975 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:45:26.975 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.975 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.975 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:45:26.975 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.808949+0000", 2023-09-23T19:45:26.975 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:45:26.975 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.975 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.975 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:45:26.975 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.808973+0000", 2023-09-23T19:45:26.976 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.3289999999999999e-05 2023-09-23T19:45:26.976 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.976 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.976 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:45:26.976 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.808984+0000", 2023-09-23T19:45:26.976 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1433e-05 2023-09-23T19:45:26.976 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.976 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.976 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:45:26.976 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.808992+0000", 2023-09-23T19:45:26.976 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.8280000000000003e-06 2023-09-23T19:45:26.976 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.977 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.977 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:45:26.977 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.909421+0000", 2023-09-23T19:45:26.977 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.100429382 2023-09-23T19:45:26.977 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.977 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.977 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:45:26.977 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.909445+0000", 2023-09-23T19:45:26.977 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.3264999999999998e-05 2023-09-23T19:45:26.977 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.977 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.977 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:45:26.978 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.914283+0000", 2023-09-23T19:45:26.978 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0048385709999999998 2023-09-23T19:45:26.978 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.978 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.978 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:45:26.978 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.914312+0000", 2023-09-23T19:45:26.978 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.8821000000000001e-05 2023-09-23T19:45:26.978 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:45:26.978 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:45:26.978 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:45:26.979 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.979 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.979 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.36243.0:66617 11.8 e186/180 11:1d35c8b4:::benchmark_data_smithi016_160148_object66616:head v 186'4236, mlcod=186'4232)", 2023-09-23T19:45:26.979 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:42:27.810503+0000", 2023-09-23T19:45:26.979 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 179.114476209, 2023-09-23T19:45:26.979 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.103777889, 2023-09-23T19:45:26.979 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:45:26.979 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:45:26.979 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:45:26.979 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.979 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:45:26.979 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.810503+0000", 2023-09-23T19:45:26.980 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:45:26.980 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.980 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.980 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:45:26.980 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.810503+0000", 2023-09-23T19:45:26.980 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:45:26.980 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.980 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.980 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:45:26.980 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.810502+0000", 2023-09-23T19:45:26.980 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T19:45:26.980 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.981 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.981 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:45:26.981 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.810563+0000", 2023-09-23T19:45:26.981 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.0531e-05 2023-09-23T19:45:26.981 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.981 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.981 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:45:26.981 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.810572+0000", 2023-09-23T19:45:26.981 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.2860000000000005e-06 2023-09-23T19:45:26.981 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.981 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.981 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:45:26.982 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.810587+0000", 2023-09-23T19:45:26.982 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.518e-05 2023-09-23T19:45:26.982 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.982 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.982 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:45:26.982 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.909401+0000", 2023-09-23T19:45:26.982 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.098813521000000001 2023-09-23T19:45:26.982 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.982 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.982 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:45:26.982 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.909430+0000", 2023-09-23T19:45:26.982 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.9043999999999998e-05 2023-09-23T19:45:26.983 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.983 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.983 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:45:26.983 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.914211+0000", 2023-09-23T19:45:26.983 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0047816589999999997 2023-09-23T19:45:26.983 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.983 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.983 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:45:26.983 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.914281+0000", 2023-09-23T19:45:26.983 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.9320999999999996e-05 2023-09-23T19:45:26.983 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:45:26.983 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:45:26.984 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:45:26.984 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.984 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.984 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.36243.0:66614 11.9 e186/180 11:9c605adb:::benchmark_data_smithi016_160148_object66613:head v 186'4256, mlcod=186'4252)", 2023-09-23T19:45:26.984 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:42:27.810593+0000", 2023-09-23T19:45:26.984 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 179.11438542900001, 2023-09-23T19:45:26.984 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.098796657999999996, 2023-09-23T19:45:26.984 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:45:26.984 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:45:26.984 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:45:26.984 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.985 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:45:26.985 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.810593+0000", 2023-09-23T19:45:26.985 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:45:26.985 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.985 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.985 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:45:26.985 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.810593+0000", 2023-09-23T19:45:26.985 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:45:26.985 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.985 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.985 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:45:26.985 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.810593+0000", 2023-09-23T19:45:26.986 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T19:45:26.986 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.986 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.986 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:45:26.986 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.810620+0000", 2023-09-23T19:45:26.986 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.65e-05 2023-09-23T19:45:26.986 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.986 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.986 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:45:26.986 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.810626+0000", 2023-09-23T19:45:26.986 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.6279999999999999e-06 2023-09-23T19:45:26.987 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.987 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.987 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:45:26.987 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.810630+0000", 2023-09-23T19:45:26.987 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.4170000000000001e-06 2023-09-23T19:45:26.987 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.987 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.987 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:45:26.987 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.810671+0000", 2023-09-23T19:45:26.987 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.1083e-05 2023-09-23T19:45:26.987 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.987 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.988 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:45:26.988 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.810685+0000", 2023-09-23T19:45:26.988 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3995000000000001e-05 2023-09-23T19:45:26.988 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.988 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.988 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:45:26.988 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.820834+0000", 2023-09-23T19:45:26.988 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.010149459 2023-09-23T19:45:26.988 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.988 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.988 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:45:26.988 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.909390+0000", 2023-09-23T19:45:26.989 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.088555828000000003 2023-09-23T19:45:26.989 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:45:26.989 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:45:26.989 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:45:26.989 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.989 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.989 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.36243.0:66611 11.b e186/180 11:d3e853b9:::benchmark_data_smithi016_160148_object66610:head v 186'4215, mlcod=186'4211)", 2023-09-23T19:45:26.989 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:42:27.812487+0000", 2023-09-23T19:45:26.989 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 179.11249140699999, 2023-09-23T19:45:26.989 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.096952996999999999, 2023-09-23T19:45:26.990 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:45:26.990 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:45:26.990 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:45:26.990 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.990 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:45:26.990 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.812487+0000", 2023-09-23T19:45:26.990 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:45:26.990 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.990 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.990 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:45:26.990 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.812487+0000", 2023-09-23T19:45:26.990 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:45:26.991 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.991 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.991 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:45:26.991 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.812487+0000", 2023-09-23T19:45:26.991 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T19:45:26.991 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.991 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.991 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:45:26.991 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.812520+0000", 2023-09-23T19:45:26.991 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.2656e-05 2023-09-23T19:45:26.991 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.991 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.992 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:45:26.992 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.812528+0000", 2023-09-23T19:45:26.992 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.8119999999999997e-06 2023-09-23T19:45:26.992 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.992 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.992 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:45:26.992 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.812533+0000", 2023-09-23T19:45:26.992 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.9139999999999999e-06 2023-09-23T19:45:26.992 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.992 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.993 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:45:26.993 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.812584+0000", 2023-09-23T19:45:26.993 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.1175999999999999e-05 2023-09-23T19:45:26.993 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.993 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.993 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:45:26.993 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.812599+0000", 2023-09-23T19:45:26.993 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.411e-05 2023-09-23T19:45:26.993 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.993 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.993 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:45:26.993 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.820843+0000", 2023-09-23T19:45:26.994 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0082448139999999996 2023-09-23T19:45:26.994 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.994 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.994 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:45:26.994 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.909440+0000", 2023-09-23T19:45:26.994 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.088597123 2023-09-23T19:45:26.994 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:45:26.994 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:45:26.994 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:45:26.994 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.994 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.994 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.36243.0:66619 11.2 e186/180 11:4c92318d:::benchmark_data_smithi016_160148_object66618:head v 186'4120, mlcod=186'4118)", 2023-09-23T19:45:26.995 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:42:27.813146+0000", 2023-09-23T19:45:26.995 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 179.11183324800001, 2023-09-23T19:45:26.995 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.101680728, 2023-09-23T19:45:26.995 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:45:26.995 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:45:26.995 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:45:26.995 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.995 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:45:26.995 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.813146+0000", 2023-09-23T19:45:26.995 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:45:26.995 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.995 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.996 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:45:26.996 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.813146+0000", 2023-09-23T19:45:26.996 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:45:26.996 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.996 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.996 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:45:26.996 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.813145+0000", 2023-09-23T19:45:26.996 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T19:45:26.996 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.996 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.996 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:45:26.996 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.813162+0000", 2023-09-23T19:45:26.997 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.6738000000000001e-05 2023-09-23T19:45:26.997 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.997 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.997 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:45:26.997 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.813169+0000", 2023-09-23T19:45:26.997 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.9639999999999997e-06 2023-09-23T19:45:26.997 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.997 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.997 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:45:26.997 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.813173+0000", 2023-09-23T19:45:26.997 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.6800000000000001e-06 2023-09-23T19:45:26.997 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.998 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.998 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:45:26.998 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.813207+0000", 2023-09-23T19:45:26.998 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.3430999999999999e-05 2023-09-23T19:45:26.998 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.998 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.998 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:45:26.998 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.813220+0000", 2023-09-23T19:45:26.998 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3485999999999999e-05 2023-09-23T19:45:26.998 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.998 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.998 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:45:26.999 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.914722+0000", 2023-09-23T19:45:26.999 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.10150184700000001 2023-09-23T19:45:26.999 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.999 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:26.999 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:45:26.999 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.914826+0000", 2023-09-23T19:45:26.999 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00010412999999999999 2023-09-23T19:45:26.999 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:45:26.999 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:45:26.999 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:45:26.999 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:26.999 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.000 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.36243.0:66623 11.a e186/180 11:5acb3f7e:::benchmark_data_smithi016_160148_object66622:head v 186'4181, mlcod=186'4179)", 2023-09-23T19:45:27.000 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:42:27.820308+0000", 2023-09-23T19:45:27.000 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 179.104670889, 2023-09-23T19:45:27.000 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.102558178, 2023-09-23T19:45:27.000 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:45:27.000 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:45:27.000 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:45:27.000 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.000 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:45:27.000 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.820308+0000", 2023-09-23T19:45:27.000 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:45:27.000 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.001 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.001 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:45:27.001 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.820308+0000", 2023-09-23T19:45:27.001 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:45:27.001 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.001 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.001 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:45:27.001 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.820307+0000", 2023-09-23T19:45:27.001 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:45:27.001 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.001 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.001 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:45:27.001 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.820347+0000", 2023-09-23T19:45:27.002 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.9889999999999999e-05 2023-09-23T19:45:27.002 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.002 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.002 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:45:27.002 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.909352+0000", 2023-09-23T19:45:27.002 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.089005168999999995 2023-09-23T19:45:27.002 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.002 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.002 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:45:27.002 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.909368+0000", 2023-09-23T19:45:27.002 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.6164999999999999e-05 2023-09-23T19:45:27.002 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.003 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.003 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:45:27.003 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.909417+0000", 2023-09-23T19:45:27.003 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.9273000000000001e-05 2023-09-23T19:45:27.003 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.003 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.003 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:45:27.003 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.909436+0000", 2023-09-23T19:45:27.003 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.8411e-05 2023-09-23T19:45:27.003 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.003 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.003 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:45:27.004 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.922684+0000", 2023-09-23T19:45:27.004 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.013247836000000001 2023-09-23T19:45:27.004 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.004 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.004 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:45:27.004 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:42:27.922866+0000", 2023-09-23T19:45:27.004 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000182447 2023-09-23T19:45:27.004 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:45:27.004 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:45:27.004 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:45:27.004 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.004 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.005 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.36243.0:128933 11.9 e193/180 11:91dd381a:::benchmark_data_smithi016_160148_object21730:head v 193'8160, mlcod=193'8156)", 2023-09-23T19:45:27.005 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:43:43.120152+0000", 2023-09-23T19:45:27.005 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 103.804827304, 2023-09-23T19:45:27.005 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.066193769, 2023-09-23T19:45:27.005 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:45:27.005 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:45:27.005 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:45:27.005 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.005 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:45:27.005 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:43.120152+0000", 2023-09-23T19:45:27.005 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:45:27.006 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.006 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.006 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:45:27.006 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:43.120152+0000", 2023-09-23T19:45:27.006 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:45:27.006 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.006 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.006 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:45:27.006 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:43.120151+0000", 2023-09-23T19:45:27.006 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:45:27.006 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.006 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.007 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:45:27.007 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:43.120164+0000", 2023-09-23T19:45:27.007 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3570000000000001e-05 2023-09-23T19:45:27.007 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.007 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.007 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:45:27.007 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:43.120185+0000", 2023-09-23T19:45:27.007 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.0519000000000001e-05 2023-09-23T19:45:27.007 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.007 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.007 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:45:27.007 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:43.120194+0000", 2023-09-23T19:45:27.008 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.7820000000000005e-06 2023-09-23T19:45:27.008 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.008 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.008 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:45:27.008 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:43.126198+0000", 2023-09-23T19:45:27.008 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0060036660000000004 2023-09-23T19:45:27.008 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.008 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.008 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:45:27.008 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:43.126263+0000", 2023-09-23T19:45:27.008 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.4474000000000006e-05 2023-09-23T19:45:27.009 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.009 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.009 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:45:27.009 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:43.186256+0000", 2023-09-23T19:45:27.009 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.059993023999999999 2023-09-23T19:45:27.009 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.009 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.009 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:45:27.009 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:43.186345+0000", 2023-09-23T19:45:27.009 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.9703999999999994e-05 2023-09-23T19:45:27.009 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:45:27.010 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:45:27.010 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:45:27.010 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.010 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.010 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.36243.0:128951 11.1 e193/180 11:842006af:::benchmark_data_smithi016_160148_object21748:head v 193'7986, mlcod=193'7983)", 2023-09-23T19:45:27.010 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:43:43.144548+0000", 2023-09-23T19:45:27.010 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 103.78043053099999, 2023-09-23T19:45:27.010 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.078182834000000007, 2023-09-23T19:45:27.010 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:45:27.010 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:45:27.010 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:45:27.010 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.011 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:45:27.011 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:43.144548+0000", 2023-09-23T19:45:27.011 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:45:27.011 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.011 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.011 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:45:27.011 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:43.144548+0000", 2023-09-23T19:45:27.011 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:45:27.011 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.011 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.012 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:45:27.012 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:43.144548+0000", 2023-09-23T19:45:27.012 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967296 2023-09-23T19:45:27.012 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.012 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.012 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:45:27.012 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:43.144555+0000", 2023-09-23T19:45:27.012 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.2659999999999998e-06 2023-09-23T19:45:27.012 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.013 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.013 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:45:27.013 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:43.144558+0000", 2023-09-23T19:45:27.013 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.5939999999999998e-06 2023-09-23T19:45:27.013 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.013 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.013 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:45:27.013 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:43.144561+0000", 2023-09-23T19:45:27.013 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.6539999999999999e-06 2023-09-23T19:45:27.013 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.013 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.014 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:45:27.014 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:43.176156+0000", 2023-09-23T19:45:27.014 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.031594920999999998 2023-09-23T19:45:27.014 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.014 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.014 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:45:27.014 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:43.176177+0000", 2023-09-23T19:45:27.014 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.1625999999999999e-05 2023-09-23T19:45:27.014 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.014 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.014 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:45:27.014 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:43.222599+0000", 2023-09-23T19:45:27.015 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.046422021000000001 2023-09-23T19:45:27.015 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.015 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.015 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:45:27.015 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:43.222731+0000", 2023-09-23T19:45:27.015 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00013187799999999999 2023-09-23T19:45:27.015 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:45:27.015 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:45:27.015 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:45:27.015 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.015 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.016 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.36243.0:128960 11.9 e193/180 11:97fb6315:::benchmark_data_smithi016_160148_object21757:head v 193'8164, mlcod=193'8158)", 2023-09-23T19:45:27.016 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:43:43.160277+0000", 2023-09-23T19:45:27.016 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 103.764701985, 2023-09-23T19:45:27.016 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.074931972999999999, 2023-09-23T19:45:27.016 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:45:27.016 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:45:27.016 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:45:27.016 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.016 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:45:27.016 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:43.160277+0000", 2023-09-23T19:45:27.016 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:45:27.016 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.016 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.017 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:45:27.017 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:43.160277+0000", 2023-09-23T19:45:27.017 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:45:27.017 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.017 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.017 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:45:27.017 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:43.160276+0000", 2023-09-23T19:45:27.017 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-09-23T19:45:27.017 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.017 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.017 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:45:27.017 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:43.160285+0000", 2023-09-23T19:45:27.018 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.6760000000000002e-06 2023-09-23T19:45:27.018 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.018 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.018 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:45:27.018 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:43.160304+0000", 2023-09-23T19:45:27.018 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.8275e-05 2023-09-23T19:45:27.018 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.018 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.018 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:45:27.018 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:43.160312+0000", 2023-09-23T19:45:27.018 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.4419999999999998e-06 2023-09-23T19:45:27.019 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.019 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.019 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:45:27.019 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:43.181148+0000", 2023-09-23T19:45:27.019 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.020836453000000001 2023-09-23T19:45:27.019 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.019 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.019 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:45:27.019 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:43.181171+0000", 2023-09-23T19:45:27.019 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.2011999999999999e-05 2023-09-23T19:45:27.019 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.019 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.019 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:45:27.020 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:43.235115+0000", 2023-09-23T19:45:27.020 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.053944472 2023-09-23T19:45:27.020 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.020 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.020 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:45:27.020 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:43.235209+0000", 2023-09-23T19:45:27.020 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.3912999999999993e-05 2023-09-23T19:45:27.020 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:45:27.020 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:45:27.020 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:45:27.020 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.020 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.021 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.36243.0:128966 11.1 e193/180 11:8318a9e5:::benchmark_data_smithi016_160148_object21763:head v 193'7987, mlcod=193'7983)", 2023-09-23T19:45:27.021 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:43:43.167511+0000", 2023-09-23T19:45:27.021 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 103.75746811400001, 2023-09-23T19:45:27.021 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.067763417000000006, 2023-09-23T19:45:27.021 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:45:27.021 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:45:27.021 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:45:27.021 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.021 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:45:27.021 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:43.167511+0000", 2023-09-23T19:45:27.021 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:45:27.021 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.021 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.022 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:45:27.022 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:43.167511+0000", 2023-09-23T19:45:27.022 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:45:27.022 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.022 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.022 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:45:27.022 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:43.167510+0000", 2023-09-23T19:45:27.022 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:45:27.022 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.022 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.022 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:45:27.023 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:43.167518+0000", 2023-09-23T19:45:27.023 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.7419999999999997e-06 2023-09-23T19:45:27.023 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.023 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.023 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:45:27.023 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:43.167536+0000", 2023-09-23T19:45:27.023 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.8389999999999998e-05 2023-09-23T19:45:27.023 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.023 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.023 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:45:27.023 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:43.167545+0000", 2023-09-23T19:45:27.023 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.8759999999999994e-06 2023-09-23T19:45:27.024 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.024 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.024 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:45:27.024 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:43.186193+0000", 2023-09-23T19:45:27.024 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.018648392999999999 2023-09-23T19:45:27.024 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.024 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.024 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:45:27.024 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:43.186216+0000", 2023-09-23T19:45:27.024 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.3095000000000002e-05 2023-09-23T19:45:27.024 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.025 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.025 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:45:27.025 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:43.235214+0000", 2023-09-23T19:45:27.025 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.048997751999999999 2023-09-23T19:45:27.025 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.025 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.025 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:45:27.025 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:43:43.235274+0000", 2023-09-23T19:45:27.025 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.0046000000000001e-05 2023-09-23T19:45:27.025 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:45:27.025 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:45:27.025 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:45:27.026 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.026 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.026 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.36243.0:168534 11.9 e196/180 11:9216e318:::benchmark_data_smithi016_160148_object61331:head v 196'10717, mlcod=196'10714)", 2023-09-23T19:45:27.026 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:44:39.832152+0000", 2023-09-23T19:45:27.026 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 47.092827319999998, 2023-09-23T19:45:27.026 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.067337843999999994, 2023-09-23T19:45:27.026 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:45:27.026 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:45:27.026 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:45:27.026 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.026 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:45:27.026 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:39.832152+0000", 2023-09-23T19:45:27.027 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:45:27.027 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.027 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.027 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:45:27.027 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:39.832152+0000", 2023-09-23T19:45:27.027 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:45:27.027 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.027 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.027 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:45:27.027 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:39.832151+0000", 2023-09-23T19:45:27.027 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:45:27.027 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.027 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.028 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:45:27.028 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:39.832160+0000", 2023-09-23T19:45:27.028 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.8400000000000001e-06 2023-09-23T19:45:27.028 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.028 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.028 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:45:27.028 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:39.832178+0000", 2023-09-23T19:45:27.028 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.7992999999999999e-05 2023-09-23T19:45:27.028 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.028 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.028 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:45:27.028 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:39.832186+0000", 2023-09-23T19:45:27.029 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.2900000000000002e-06 2023-09-23T19:45:27.029 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.029 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.029 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:45:27.029 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:39.853857+0000", 2023-09-23T19:45:27.029 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.021670906 2023-09-23T19:45:27.029 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.029 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.029 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:45:27.029 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:39.853924+0000", 2023-09-23T19:45:27.029 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.7156000000000006e-05 2023-09-23T19:45:27.029 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.030 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.030 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:45:27.030 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:39.899273+0000", 2023-09-23T19:45:27.030 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.045348871999999998 2023-09-23T19:45:27.030 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.030 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.030 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:45:27.030 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:39.899489+0000", 2023-09-23T19:45:27.030 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000216646 2023-09-23T19:45:27.030 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:45:27.030 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:45:27.031 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:45:27.031 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.031 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.031 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.36243.0:172293 11.2 e196/180 11:4621267c:::benchmark_data_smithi016_160148_object65090:head v 196'10635, mlcod=196'10632)", 2023-09-23T19:45:27.031 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:44:45.344125+0000", 2023-09-23T19:45:27.031 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 41.580853943000001, 2023-09-23T19:45:27.031 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.072040113000000003, 2023-09-23T19:45:27.031 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:45:27.031 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:45:27.031 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:45:27.031 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.032 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:45:27.032 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:45.344125+0000", 2023-09-23T19:45:27.032 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:45:27.032 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.032 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.032 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:45:27.032 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:45.344125+0000", 2023-09-23T19:45:27.032 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:45:27.032 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.032 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.032 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:45:27.033 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:45.344124+0000", 2023-09-23T19:45:27.033 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T19:45:27.033 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.033 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.033 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:45:27.033 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:45.344131+0000", 2023-09-23T19:45:27.033 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.4819999999999999e-06 2023-09-23T19:45:27.033 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.033 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.033 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:45:27.033 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:45.344143+0000", 2023-09-23T19:45:27.033 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2109999999999999e-05 2023-09-23T19:45:27.034 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.034 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.034 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:45:27.034 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:45.344151+0000", 2023-09-23T19:45:27.034 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.0320000000000003e-06 2023-09-23T19:45:27.034 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.034 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.034 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:45:27.034 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:45.344932+0000", 2023-09-23T19:45:27.034 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00078083299999999998 2023-09-23T19:45:27.034 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.035 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.035 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:45:27.035 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:45.344951+0000", 2023-09-23T19:45:27.035 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.8858000000000001e-05 2023-09-23T19:45:27.035 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.035 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.035 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:45:27.035 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:45.416106+0000", 2023-09-23T19:45:27.035 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.071155431000000005 2023-09-23T19:45:27.035 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:45:27.035 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:45:27.035 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:45:27.035 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:44:45.416165+0000", 2023-09-23T19:45:27.036 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.9069e-05 2023-09-23T19:45:27.036 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:45:27.036 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:45:27.036 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:45:27.036 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:45:27.036 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:45:27.036 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-09-23T19:45:27.036 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:27.065 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:27.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:27.071 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:45:27.171 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:27.205 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:27.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:27.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:45:27.310 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:27.348 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:27.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:27.351 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:27.451 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:27.487 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:27.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:27.494 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:45:27.595 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:27.637 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:27.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:27.644 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:27.744 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:27.768 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:27.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:27.771 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:45:27.873 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:27.928 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:27.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:27.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:28.034 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:28.051 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:28.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:28.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:45:28.153 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:28.184 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:28.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:28.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:45:28.203 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:28 smithi016 bash[75004]: cluster 2023-09-23T19:45:26.292756+0000 mgr.y (mgr.25325) 1140 : cluster 0 pgmap v1110: 89 pgs: 89 active+clean; 968 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 679 op/s 2023-09-23T19:45:28.203 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:28 smithi016 bash[77995]: cluster 2023-09-23T19:45:26.292756+0000 mgr.y (mgr.25325) 1140 : cluster 0 pgmap v1110: 89 pgs: 89 active+clean; 968 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 679 op/s 2023-09-23T19:45:28.286 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:28.312 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:28.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:28.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:45:28.416 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:28.447 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:28.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:28.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:28.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:28 smithi092 bash[58855]: cluster 2023-09-23T19:45:26.292756+0000 mgr.y (mgr.25325) 1140 : cluster 0 pgmap v1110: 89 pgs: 89 active+clean; 968 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 679 op/s 2023-09-23T19:45:28.551 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:28.581 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:28.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:28.585 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:45:28.685 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:28.712 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:28.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:28.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:28.814 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:28.838 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:28.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:28.842 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:28.942 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:28.972 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:28.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:28.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:29.074 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:29.111 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:29.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:29.114 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:29.215 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:29.223 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:29 smithi016 bash[75004]: audit 2023-09-23T19:45:28.665242+0000 mon.a (mon.0) 1432 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:45:29.223 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:29 smithi016 bash[77995]: audit 2023-09-23T19:45:28.665242+0000 mon.a (mon.0) 1432 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:45:29.257 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:29.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:29.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:29.361 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:29.395 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:29.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:29.400 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:29.501 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:29.509 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:29 smithi092 bash[58855]: audit 2023-09-23T19:45:28.665242+0000 mon.a (mon.0) 1432 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:45:29.525 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:29.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:29.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:29.627 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:29.665 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:29.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:29.668 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:45:29.769 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:29.796 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:29.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:29.802 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:45:29.903 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:29.933 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:29.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:29.936 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:45:30.037 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:30.064 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:30.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:30.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:30.167 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:30.216 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:30.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:30.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:30.321 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:30.341 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:30.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:30.343 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:30.444 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:30.453 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:30 smithi092 bash[58855]: cluster 2023-09-23T19:45:28.293173+0000 mgr.y (mgr.25325) 1141 : cluster 0 pgmap v1111: 89 pgs: 89 active+clean; 929 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2023-09-23T19:45:30.474 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:30.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:30.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:30.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:30 smithi016 bash[75004]: cluster 2023-09-23T19:45:28.293173+0000 mgr.y (mgr.25325) 1141 : cluster 0 pgmap v1111: 89 pgs: 89 active+clean; 929 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2023-09-23T19:45:30.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:30 smithi016 bash[77995]: cluster 2023-09-23T19:45:28.293173+0000 mgr.y (mgr.25325) 1141 : cluster 0 pgmap v1111: 89 pgs: 89 active+clean; 929 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2023-09-23T19:45:30.579 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:30.626 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:30.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:30.631 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:30.697 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T19:45:30.733 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:30.754 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:30.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:30.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:30.856 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:30.911 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:30.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:30.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:31.017 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:31.043 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:31.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:31.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:31.146 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:31.185 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:31.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:31.189 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:31.289 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:31.330 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:31.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:31.335 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:45:31.435 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:31.465 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:31.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:31.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:45:31.568 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:31.602 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:31.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:31.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:31.710 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:31.746 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:31.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:31.748 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:31.850 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:31.878 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:31.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:31.881 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:31.982 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:32.019 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:32.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:32.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:32.125 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:32.160 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:32.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:32.162 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:45:32.262 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:32.272 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:32 smithi092 bash[58855]: cluster 2023-09-23T19:45:30.293859+0000 mgr.y (mgr.25325) 1142 : cluster 0 pgmap v1112: 89 pgs: 89 active+clean; 809 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 730 op/s 2023-09-23T19:45:32.300 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:32.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:32.305 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:32.405 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:32.414 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:32 smithi016 bash[77995]: cluster 2023-09-23T19:45:30.293859+0000 mgr.y (mgr.25325) 1142 : cluster 0 pgmap v1112: 89 pgs: 89 active+clean; 809 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 730 op/s 2023-09-23T19:45:32.415 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:32 smithi016 bash[75004]: cluster 2023-09-23T19:45:30.293859+0000 mgr.y (mgr.25325) 1142 : cluster 0 pgmap v1112: 89 pgs: 89 active+clean; 809 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 730 op/s 2023-09-23T19:45:32.438 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:32.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:32.441 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:32.541 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:32.573 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:32.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:32.577 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:32.678 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:32.712 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:32.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:32.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:45:32.821 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:32.851 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:32.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:32.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:45:32.954 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:32.977 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:32.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:32.985 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:33.085 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:33.119 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:33.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:33.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:45:33.223 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:33.249 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:33.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:33.252 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:33.353 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:33.383 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:33.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:33.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:33.488 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:33.489 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:45:33.532 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:33.534 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:45:33.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:33.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:33.588 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:45:33.630 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:45:33.637 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:33.666 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:33.671 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:45:33.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:33.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:33.714 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:45:33.743 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:45:33.773 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:33.779 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:45:33.794 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:45:33.794 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:45:33.794 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:45:33.794 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:45:33.794 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:45:33.795 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:45:33.796 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:45:33.796 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:45:33.796 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:33.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:33.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:45:33.905 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:33.947 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:33.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:33.951 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:34.052 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:34.086 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:34.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:34.091 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:34.192 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:34.225 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:34.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:34.230 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:34.331 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:34.341 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:34 smithi092 bash[58855]: cluster 2023-09-23T19:45:32.294391+0000 mgr.y (mgr.25325) 1143 : cluster 0 pgmap v1113: 89 pgs: 89 active+clean; 757 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2023-09-23T19:45:34.341 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:34 smithi092 bash[58855]: audit 2023-09-23T19:45:33.427203+0000 mon.c (mon.2) 189 : audit 0 from='client.? 172.21.15.16:0/3160294086' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:45:34.366 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:34.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:34.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:34.473 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:34.482 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:34 smithi016 bash[75004]: cluster 2023-09-23T19:45:32.294391+0000 mgr.y (mgr.25325) 1143 : cluster 0 pgmap v1113: 89 pgs: 89 active+clean; 757 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2023-09-23T19:45:34.482 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:34 smithi016 bash[75004]: audit 2023-09-23T19:45:33.427203+0000 mon.c (mon.2) 189 : audit 0 from='client.? 172.21.15.16:0/3160294086' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:45:34.482 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:34 smithi016 bash[77995]: cluster 2023-09-23T19:45:32.294391+0000 mgr.y (mgr.25325) 1143 : cluster 0 pgmap v1113: 89 pgs: 89 active+clean; 757 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2023-09-23T19:45:34.482 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:34 smithi016 bash[77995]: audit 2023-09-23T19:45:33.427203+0000 mon.c (mon.2) 189 : audit 0 from='client.? 172.21.15.16:0/3160294086' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:45:34.497 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:34.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:34.504 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:45:34.605 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:34.639 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:34.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:34.649 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:45:34.750 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:34.779 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:34.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:34.782 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:45:34.883 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:34.910 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:34.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:34.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:35.013 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:35.046 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:35.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:35.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:45:35.152 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:35.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:35.209 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:35.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:45:35.310 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:35.349 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:35.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:35.352 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:45:35.453 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:35.484 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:35.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:35.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:35.504 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:45:35.588 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:35.611 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:35.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:35.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:45:35.715 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:35.723 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:45:35 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:45:35] "GET /metrics HTTP/1.1" 200 36299 "" "Prometheus/2.33.4" 2023-09-23T19:45:35.746 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:35.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:35.748 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:45:35.849 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:35.873 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:35.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:35.876 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:45:35.977 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:36.010 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:36.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:36.014 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:45:36.115 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:36.153 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T19:45:36.155 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:36.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:36.160 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:36.162 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is set 2023-09-23T19:45:36.260 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:36.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:36 smithi016 bash[75004]: cluster 2023-09-23T19:45:34.294896+0000 mgr.y (mgr.25325) 1144 : cluster 0 pgmap v1114: 89 pgs: 89 active+clean; 716 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 625 op/s 2023-09-23T19:45:36.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:36 smithi016 bash[75004]: audit 2023-09-23T19:45:36.073073+0000 mon.a (mon.0) 1433 : audit 1 from='client.? 172.21.15.16:0/832085826' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T19:45:36.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:36 smithi016 bash[75004]: audit 2023-09-23T19:45:36.122902+0000 mon.c (mon.2) 190 : audit 1 from='client.? 172.21.15.16:0/6488509' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:45:36.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:36 smithi016 bash[75004]: audit 2023-09-23T19:45:36.124130+0000 mon.a (mon.0) 1434 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:45:36.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:36 smithi016 bash[77995]: cluster 2023-09-23T19:45:34.294896+0000 mgr.y (mgr.25325) 1144 : cluster 0 pgmap v1114: 89 pgs: 89 active+clean; 716 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 625 op/s 2023-09-23T19:45:36.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:36 smithi016 bash[77995]: audit 2023-09-23T19:45:36.073073+0000 mon.a (mon.0) 1433 : audit 1 from='client.? 172.21.15.16:0/832085826' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T19:45:36.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:36 smithi016 bash[77995]: audit 2023-09-23T19:45:36.122902+0000 mon.c (mon.2) 190 : audit 1 from='client.? 172.21.15.16:0/6488509' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:45:36.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:36 smithi016 bash[77995]: audit 2023-09-23T19:45:36.124130+0000 mon.a (mon.0) 1434 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:45:36.291 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:36.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:36.293 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:45:36.394 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:36.402 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:36 smithi092 bash[58855]: cluster 2023-09-23T19:45:34.294896+0000 mgr.y (mgr.25325) 1144 : cluster 0 pgmap v1114: 89 pgs: 89 active+clean; 716 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 625 op/s 2023-09-23T19:45:36.402 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:36 smithi092 bash[58855]: audit 2023-09-23T19:45:36.073073+0000 mon.a (mon.0) 1433 : audit 1 from='client.? 172.21.15.16:0/832085826' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T19:45:36.402 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:36 smithi092 bash[58855]: audit 2023-09-23T19:45:36.122902+0000 mon.c (mon.2) 190 : audit 1 from='client.? 172.21.15.16:0/6488509' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:45:36.402 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:36 smithi092 bash[58855]: audit 2023-09-23T19:45:36.124130+0000 mon.a (mon.0) 1434 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:45:36.443 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:36.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:36.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:45:36.549 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:36.579 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:36.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:36.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:45:36.685 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:36.709 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:36.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:36.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:45:36.812 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:36.837 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:36.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:36.841 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:45:36.942 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:36.971 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:36.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:36.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:37.076 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:37.117 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:37.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:37.120 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:37.220 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:37.245 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:37.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:37.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:37.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:37 smithi092 bash[58855]: cluster 2023-09-23T19:45:36.125952+0000 mon.a (mon.0) 1435 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:45:37.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:37 smithi092 bash[58855]: audit 2023-09-23T19:45:36.143017+0000 mon.a (mon.0) 1436 : audit 1 from='client.? 172.21.15.16:0/832085826' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T19:45:37.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:37 smithi092 bash[58855]: audit 2023-09-23T19:45:36.143279+0000 mon.a (mon.0) 1437 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:45:37.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:37 smithi092 bash[58855]: cluster 2023-09-23T19:45:36.143344+0000 mon.a (mon.0) 1438 : cluster 0 osdmap e202: 8 total, 8 up, 8 in 2023-09-23T19:45:37.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:37 smithi092 bash[58855]: cluster 2023-09-23T19:45:36.158923+0000 osd.6 (osd.6) 129 : cluster 3 failed to encode map e202 with expected crc 2023-09-23T19:45:37.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:37 smithi092 bash[58855]: cluster 2023-09-23T19:45:36.160893+0000 osd.6 (osd.6) 130 : cluster 3 failed to encode map e202 with expected crc 2023-09-23T19:45:37.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:37 smithi092 bash[58855]: cluster 2023-09-23T19:45:36.174515+0000 osd.7 (osd.7) 118 : cluster 3 failed to encode map e202 with expected crc 2023-09-23T19:45:37.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:37 smithi092 bash[58855]: cluster 2023-09-23T19:45:36.177846+0000 osd.4 (osd.4) 143 : cluster 3 failed to encode map e202 with expected crc 2023-09-23T19:45:37.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:37 smithi092 bash[58855]: cluster 2023-09-23T19:45:36.179747+0000 osd.4 (osd.4) 144 : cluster 3 failed to encode map e202 with expected crc 2023-09-23T19:45:37.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:37 smithi092 bash[58855]: cluster 2023-09-23T19:45:36.181579+0000 osd.4 (osd.4) 145 : cluster 3 failed to encode map e202 with expected crc 2023-09-23T19:45:37.350 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:37.359 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:37 smithi016 bash[75004]: cluster 2023-09-23T19:45:36.125952+0000 mon.a (mon.0) 1435 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:45:37.359 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:37 smithi016 bash[75004]: audit 2023-09-23T19:45:36.143017+0000 mon.a (mon.0) 1436 : audit 1 from='client.? 172.21.15.16:0/832085826' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T19:45:37.359 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:37 smithi016 bash[75004]: audit 2023-09-23T19:45:36.143279+0000 mon.a (mon.0) 1437 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:45:37.359 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:37 smithi016 bash[75004]: cluster 2023-09-23T19:45:36.143344+0000 mon.a (mon.0) 1438 : cluster 0 osdmap e202: 8 total, 8 up, 8 in 2023-09-23T19:45:37.359 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:37 smithi016 bash[75004]: cluster 2023-09-23T19:45:36.158923+0000 osd.6 (osd.6) 129 : cluster 3 failed to encode map e202 with expected crc 2023-09-23T19:45:37.359 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:37 smithi016 bash[75004]: cluster 2023-09-23T19:45:36.160893+0000 osd.6 (osd.6) 130 : cluster 3 failed to encode map e202 with expected crc 2023-09-23T19:45:37.359 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:37 smithi016 bash[75004]: cluster 2023-09-23T19:45:36.174515+0000 osd.7 (osd.7) 118 : cluster 3 failed to encode map e202 with expected crc 2023-09-23T19:45:37.359 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:37 smithi016 bash[75004]: cluster 2023-09-23T19:45:36.177846+0000 osd.4 (osd.4) 143 : cluster 3 failed to encode map e202 with expected crc 2023-09-23T19:45:37.359 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:37 smithi016 bash[75004]: cluster 2023-09-23T19:45:36.179747+0000 osd.4 (osd.4) 144 : cluster 3 failed to encode map e202 with expected crc 2023-09-23T19:45:37.359 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:37 smithi016 bash[75004]: cluster 2023-09-23T19:45:36.181579+0000 osd.4 (osd.4) 145 : cluster 3 failed to encode map e202 with expected crc 2023-09-23T19:45:37.360 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:37 smithi016 bash[77995]: cluster 2023-09-23T19:45:36.125952+0000 mon.a (mon.0) 1435 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:45:37.360 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:37 smithi016 bash[77995]: audit 2023-09-23T19:45:36.143017+0000 mon.a (mon.0) 1436 : audit 1 from='client.? 172.21.15.16:0/832085826' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T19:45:37.360 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:37 smithi016 bash[77995]: audit 2023-09-23T19:45:36.143279+0000 mon.a (mon.0) 1437 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:45:37.360 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:37 smithi016 bash[77995]: cluster 2023-09-23T19:45:36.143344+0000 mon.a (mon.0) 1438 : cluster 0 osdmap e202: 8 total, 8 up, 8 in 2023-09-23T19:45:37.360 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:37 smithi016 bash[77995]: cluster 2023-09-23T19:45:36.158923+0000 osd.6 (osd.6) 129 : cluster 3 failed to encode map e202 with expected crc 2023-09-23T19:45:37.360 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:37 smithi016 bash[77995]: cluster 2023-09-23T19:45:36.160893+0000 osd.6 (osd.6) 130 : cluster 3 failed to encode map e202 with expected crc 2023-09-23T19:45:37.360 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:37 smithi016 bash[77995]: cluster 2023-09-23T19:45:36.174515+0000 osd.7 (osd.7) 118 : cluster 3 failed to encode map e202 with expected crc 2023-09-23T19:45:37.360 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:37 smithi016 bash[77995]: cluster 2023-09-23T19:45:36.177846+0000 osd.4 (osd.4) 143 : cluster 3 failed to encode map e202 with expected crc 2023-09-23T19:45:37.360 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:37 smithi016 bash[77995]: cluster 2023-09-23T19:45:36.179747+0000 osd.4 (osd.4) 144 : cluster 3 failed to encode map e202 with expected crc 2023-09-23T19:45:37.360 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:37 smithi016 bash[77995]: cluster 2023-09-23T19:45:36.181579+0000 osd.4 (osd.4) 145 : cluster 3 failed to encode map e202 with expected crc 2023-09-23T19:45:37.377 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:37.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:37.383 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:45:37.420 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:45:37.484 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:37.514 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:37.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:37.520 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:37.621 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:37.650 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:37.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:37.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:45:37.754 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:37.787 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:37.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:37.795 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:45:37.895 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:37.921 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:37.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:37.922 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:38.023 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:38.045 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:38.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:38.048 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:45:38.149 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:38.158 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:38 smithi016 bash[75004]: cluster 2023-09-23T19:45:36.295317+0000 mgr.y (mgr.25325) 1145 : cluster 0 pgmap v1116: 89 pgs: 89 active+clean; 545 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2023-09-23T19:45:38.158 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:38 smithi016 bash[77995]: cluster 2023-09-23T19:45:36.295317+0000 mgr.y (mgr.25325) 1145 : cluster 0 pgmap v1116: 89 pgs: 89 active+clean; 545 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2023-09-23T19:45:38.176 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:38.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:38.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:38.288 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:38.312 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:38.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:38.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:38.360 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T19:45:38.417 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:38.424 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:38 smithi092 bash[58855]: cluster 2023-09-23T19:45:36.295317+0000 mgr.y (mgr.25325) 1145 : cluster 0 pgmap v1116: 89 pgs: 89 active+clean; 545 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2023-09-23T19:45:38.456 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:38.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:38.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:45:38.564 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:38.601 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:38.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:38.605 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:38.705 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:38.742 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:38.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:38.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:38.845 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:38.875 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:38.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:38.881 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:45:38.982 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:39.009 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:39.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:39.015 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:39.115 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:39.151 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:39.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:39.158 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:39.258 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:39.282 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:39.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:39.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:45:39.387 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:39.412 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:39.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:39.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:45:39.524 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:39.551 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:39.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:39.557 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:39.658 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:39.684 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:39.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:39.692 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:39.793 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:39.821 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:39.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:39.823 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:45:39.924 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:39.955 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:39.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:39.965 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:45:40.066 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:40.096 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:40.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:40.105 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:45:40.206 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:40.241 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:40.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:40.245 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:40.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:40 smithi016 bash[75004]: cluster 2023-09-23T19:45:38.297381+0000 mgr.y (mgr.25325) 1146 : cluster 0 pgmap v1117: 89 pgs: 89 active+clean; 507 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2023-09-23T19:45:40.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:40 smithi016 bash[77995]: cluster 2023-09-23T19:45:38.297381+0000 mgr.y (mgr.25325) 1146 : cluster 0 pgmap v1117: 89 pgs: 89 active+clean; 507 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2023-09-23T19:45:40.346 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:40.354 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:40 smithi092 bash[58855]: cluster 2023-09-23T19:45:38.297381+0000 mgr.y (mgr.25325) 1146 : cluster 0 pgmap v1117: 89 pgs: 89 active+clean; 507 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2023-09-23T19:45:40.377 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:40.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:40.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:45:40.397 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T19:45:40.482 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:40.532 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:40.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:40.541 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:45:40.642 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:40.663 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:40.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:40.668 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:40.769 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:40.800 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:40.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:40.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:40.903 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:40.929 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:40.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:40.932 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:45:41.033 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:41.060 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:41.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:41.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:41.168 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:41.198 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:41.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:41.205 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:41.306 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:41.359 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:41.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:41.361 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:41.462 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:41.495 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:41.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:41.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:45:41.598 INFO:teuthology.orchestra.run.smithi092.stdout:ERROR: (22) Invalid argument 2023-09-23T19:45:41.598 INFO:teuthology.orchestra.run.smithi092.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:45:41.601 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:41.650 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:41.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:41.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:45:41.754 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:41.796 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:41.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:41.800 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:41.900 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:41.939 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:41.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:41.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:45:42.044 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:42.068 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:42.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:42.073 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:42.174 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:42.204 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:42.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:42.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:42.309 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:42.314 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:45:42.315 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:45:42.317 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:42 smithi092 bash[58855]: cluster 2023-09-23T19:45:40.298287+0000 mgr.y (mgr.25325) 1147 : cluster 0 pgmap v1118: 89 pgs: 89 active+clean; 382 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 682 op/s 2023-09-23T19:45:42.332 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:42.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:42.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:42.439 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:42.464 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:42.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:42.468 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:42.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:42 smithi016 bash[77995]: cluster 2023-09-23T19:45:40.298287+0000 mgr.y (mgr.25325) 1147 : cluster 0 pgmap v1118: 89 pgs: 89 active+clean; 382 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 682 op/s 2023-09-23T19:45:42.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:42 smithi016 bash[75004]: cluster 2023-09-23T19:45:40.298287+0000 mgr.y (mgr.25325) 1147 : cluster 0 pgmap v1118: 89 pgs: 89 active+clean; 382 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 682 op/s 2023-09-23T19:45:42.569 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:42.619 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:42.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:42.625 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:42.645 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T19:45:42.727 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:42.750 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:42.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:42.754 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:45:42.855 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:42.894 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:42.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:42.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:45:43.000 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:43.039 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:43.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:43.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:45:43.144 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:43.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:43.181 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:43.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:43.281 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:43.306 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:43.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:43.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:43.413 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:43.435 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:43.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:43.443 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:43.543 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:43.575 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:43.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:43.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:43.678 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:43.700 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:43.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:43.705 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:43.806 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:43.833 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:43.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:43.835 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:45:43.936 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:43.970 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:43.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:43.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:45:44.073 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:44.103 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:44.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:44.112 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:44.159 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:44 smithi092 bash[58855]: cluster 2023-09-23T19:45:42.298844+0000 mgr.y (mgr.25325) 1148 : cluster 0 pgmap v1119: 89 pgs: 89 active+clean; 327 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2023-09-23T19:45:44.159 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:44 smithi092 bash[58855]: audit 2023-09-23T19:45:43.665361+0000 mon.a (mon.0) 1439 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:45:44.213 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:44.250 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:44.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:44.251 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:44.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:44 smithi016 bash[77995]: cluster 2023-09-23T19:45:42.298844+0000 mgr.y (mgr.25325) 1148 : cluster 0 pgmap v1119: 89 pgs: 89 active+clean; 327 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2023-09-23T19:45:44.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:44 smithi016 bash[77995]: audit 2023-09-23T19:45:43.665361+0000 mon.a (mon.0) 1439 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:45:44.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:44 smithi016 bash[75004]: cluster 2023-09-23T19:45:42.298844+0000 mgr.y (mgr.25325) 1148 : cluster 0 pgmap v1119: 89 pgs: 89 active+clean; 327 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2023-09-23T19:45:44.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:44 smithi016 bash[75004]: audit 2023-09-23T19:45:43.665361+0000 mon.a (mon.0) 1439 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:45:44.349 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:45:44.352 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:44.381 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:44.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:44.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:45:44.490 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:44.517 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:44.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:44.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:45:44.621 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:44.654 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:44.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:44.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:44.757 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:44.790 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:44.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:44.793 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:45:44.893 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:44.923 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:44.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:44.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:45:45.026 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:45.081 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:45.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:45.087 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:45:45.188 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:45.222 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:45.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:45.228 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:45:45.329 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:45.358 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:45.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:45.360 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:45:45.460 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:45.493 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:45.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:45.496 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:45:45.596 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:45.626 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:45.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:45.632 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:45:45.733 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:45.764 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:45.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:45.766 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:45.866 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:45.903 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:45.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:45.906 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:46.007 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:46.036 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:45:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:45:45] "GET /metrics HTTP/1.1" 200 36294 "" "Prometheus/2.33.4" 2023-09-23T19:45:46.040 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:46.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:46.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:46.145 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:46.170 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:46.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:46.176 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:46.277 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:46.285 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:46 smithi092 bash[58855]: cluster 2023-09-23T19:45:44.299331+0000 mgr.y (mgr.25325) 1149 : cluster 0 pgmap v1120: 89 pgs: 89 active+clean; 287 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 685 op/s 2023-09-23T19:45:46.309 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:46.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:46.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:45:46.415 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:46.451 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:46.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:46.460 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:45:46.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:46 smithi016 bash[77995]: cluster 2023-09-23T19:45:44.299331+0000 mgr.y (mgr.25325) 1149 : cluster 0 pgmap v1120: 89 pgs: 89 active+clean; 287 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 685 op/s 2023-09-23T19:45:46.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:46 smithi016 bash[75004]: cluster 2023-09-23T19:45:44.299331+0000 mgr.y (mgr.25325) 1149 : cluster 0 pgmap v1120: 89 pgs: 89 active+clean; 287 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 685 op/s 2023-09-23T19:45:46.560 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:46.596 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:46.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:46.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:46.704 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:46.747 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:46.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:46.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:46.853 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:46.885 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:46.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:46.893 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:46.994 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:47.022 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:47.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:47.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:45:47.129 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:47.162 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:47.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:47.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:45:47.229 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Removed 107200 objects 2023-09-23T19:45:47.229 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Clean up completed and total clean up time :152.991 2023-09-23T19:45:47.241 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:47 smithi016 bash[77995]: audit 2023-09-23T19:45:46.332543+0000 mon.a (mon.0) 1440 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:45:47.241 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:47 smithi016 bash[75004]: audit 2023-09-23T19:45:46.332543+0000 mon.a (mon.0) 1440 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:45:47.267 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:47.294 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:47.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:47.298 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:45:47.398 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:47.426 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:47.427 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:47 smithi092 bash[58855]: audit 2023-09-23T19:45:46.332543+0000 mon.a (mon.0) 1440 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:45:47.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:47.428 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:47.453 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:45:47.529 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:47.557 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:47.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:47.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:47.661 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:47.687 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:47.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:47.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:45:47.791 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:47.820 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:47.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:47.823 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:45:47.924 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:47.951 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:47.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:47.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:48.054 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:48.080 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:48.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:48.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:45:48.181 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_6 2023-09-23T19:45:48.181 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_6 unique_pool_6 --yes-i-really-really-mean-it 2023-09-23T19:45:48.185 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:48.190 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:48 smithi016 bash[75004]: cluster 2023-09-23T19:45:46.300054+0000 mgr.y (mgr.25325) 1150 : cluster 0 pgmap v1121: 89 pgs: 89 active+clean; 113 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2023-09-23T19:45:48.190 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:48 smithi016 bash[77995]: cluster 2023-09-23T19:45:46.300054+0000 mgr.y (mgr.25325) 1150 : cluster 0 pgmap v1121: 89 pgs: 89 active+clean; 113 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2023-09-23T19:45:48.214 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:48.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:48.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:48.319 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:48.347 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:48.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:48.350 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:45:48.450 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:48.462 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:48 smithi092 bash[58855]: cluster 2023-09-23T19:45:46.300054+0000 mgr.y (mgr.25325) 1150 : cluster 0 pgmap v1121: 89 pgs: 89 active+clean; 113 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2023-09-23T19:45:48.481 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:48.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:48.484 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:45:48.585 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:48.611 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:48.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:48.614 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:48.714 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:48.742 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:48.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:48.745 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:48.846 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:48.871 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:48.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:48.874 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:48.977 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:49.001 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:49.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:49.005 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:49.105 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:49.136 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:49.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:49.138 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:45:49.239 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:49.267 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:49.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:49.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:49.371 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:49.392 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:49.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:49.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:49.495 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:49.518 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:49.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:49.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:49.622 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:49.650 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:49.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:49.652 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:45:49.753 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:49.785 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:49.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:49.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:49.888 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:49.921 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:49.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:49.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:50.025 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:50.060 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:50.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:50.063 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:45:50.164 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:50.194 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:50.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:50.196 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:45:50.297 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:50.326 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:50.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:50.329 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:50.430 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:50.438 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:50 smithi016 bash[77995]: cluster 2023-09-23T19:45:48.300898+0000 mgr.y (mgr.25325) 1151 : cluster 0 pgmap v1122: 89 pgs: 89 active+clean; 75 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 627 op/s 2023-09-23T19:45:50.439 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:50 smithi016 bash[75004]: cluster 2023-09-23T19:45:48.300898+0000 mgr.y (mgr.25325) 1151 : cluster 0 pgmap v1122: 89 pgs: 89 active+clean; 75 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 627 op/s 2023-09-23T19:45:50.455 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:50.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:50.458 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:50.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:50 smithi092 bash[58855]: cluster 2023-09-23T19:45:48.300898+0000 mgr.y (mgr.25325) 1151 : cluster 0 pgmap v1122: 89 pgs: 89 active+clean; 75 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 627 op/s 2023-09-23T19:45:50.558 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:50.583 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:50.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:50.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:45:50.686 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:50.716 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:50.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:50.720 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:45:50.821 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:50.841 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:50.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:50.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:50.944 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:50.973 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:50.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:50.976 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:45:51.076 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:51.105 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:51.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:51.108 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:45:51.209 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:51.235 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:51.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:51.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:51.338 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:51.360 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:51.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:51.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:51.463 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:51.488 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:45:51.506 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:51.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:51.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:51.520 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:45:51.550 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:45:51.577 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:45:51.604 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:45:51.610 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:51.631 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:45:51.636 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:51.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:51.639 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:51.654 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:45:51.681 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:45:51.684 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:45:51.684 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:45:51.684 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:45:51.684 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:45:51.684 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:45:51.684 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:45:51.685 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:45:51.685 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:45:51.740 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:51.763 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:51.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:51.766 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:51.867 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:51.922 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:51.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:51.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:45:51.966 INFO:teuthology.orchestra.run.smithi092.stdout:{ 2023-09-23T19:45:51.966 INFO:teuthology.orchestra.run.smithi092.stdout: "ops": [], 2023-09-23T19:45:51.966 INFO:teuthology.orchestra.run.smithi092.stdout: "complaint_time": 30, 2023-09-23T19:45:51.966 INFO:teuthology.orchestra.run.smithi092.stdout: "num_blocked_ops": 0 2023-09-23T19:45:51.967 INFO:teuthology.orchestra.run.smithi092.stdout:} 2023-09-23T19:45:52.024 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:52.052 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:52.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:52.056 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:52.157 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:52.181 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:52.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:52.185 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:52.200 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is unset 2023-09-23T19:45:52.200 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T19:45:52.262 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:52 smithi016 bash[75004]: cluster 2023-09-23T19:45:50.301922+0000 mgr.y (mgr.25325) 1152 : cluster 0 pgmap v1123: 89 pgs: 89 active+clean; 15 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 654 op/s 2023-09-23T19:45:52.262 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:52 smithi016 bash[75004]: audit 2023-09-23T19:45:51.402220+0000 mon.c (mon.2) 191 : audit 1 from='client.? 172.21.15.16:0/2622590723' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:45:52.262 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:52 smithi016 bash[75004]: audit 2023-09-23T19:45:51.402641+0000 mon.a (mon.0) 1441 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:45:52.262 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:52 smithi016 bash[75004]: audit 2023-09-23T19:45:51.440780+0000 mon.a (mon.0) 1442 : audit 0 from='client.? 172.21.15.16:0/3473364679' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:45:52.262 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:52 smithi016 bash[75004]: audit 2023-09-23T19:45:51.705454+0000 mon.a (mon.0) 1443 : audit 1 from='client.? 172.21.15.16:0/3057418596' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T19:45:52.263 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:52 smithi016 bash[75004]: audit 2023-09-23T19:45:52.081474+0000 mon.a (mon.0) 1444 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:45:52.263 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:52 smithi016 bash[75004]: audit 2023-09-23T19:45:52.089372+0000 mon.a (mon.0) 1445 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:45:52.263 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:52 smithi016 bash[77995]: cluster 2023-09-23T19:45:50.301922+0000 mgr.y (mgr.25325) 1152 : cluster 0 pgmap v1123: 89 pgs: 89 active+clean; 15 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 654 op/s 2023-09-23T19:45:52.263 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:52 smithi016 bash[77995]: audit 2023-09-23T19:45:51.402220+0000 mon.c (mon.2) 191 : audit 1 from='client.? 172.21.15.16:0/2622590723' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:45:52.263 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:52 smithi016 bash[77995]: audit 2023-09-23T19:45:51.402641+0000 mon.a (mon.0) 1441 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:45:52.263 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:52 smithi016 bash[77995]: audit 2023-09-23T19:45:51.440780+0000 mon.a (mon.0) 1442 : audit 0 from='client.? 172.21.15.16:0/3473364679' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:45:52.263 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:52 smithi016 bash[77995]: audit 2023-09-23T19:45:51.705454+0000 mon.a (mon.0) 1443 : audit 1 from='client.? 172.21.15.16:0/3057418596' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T19:45:52.264 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:52 smithi016 bash[77995]: audit 2023-09-23T19:45:52.081474+0000 mon.a (mon.0) 1444 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:45:52.264 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:52 smithi016 bash[77995]: audit 2023-09-23T19:45:52.089372+0000 mon.a (mon.0) 1445 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:45:52.285 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:52.306 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:52.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:52.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:52.409 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:52.429 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:52.432 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:52 smithi092 bash[58855]: cluster 2023-09-23T19:45:50.301922+0000 mgr.y (mgr.25325) 1152 : cluster 0 pgmap v1123: 89 pgs: 89 active+clean; 15 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 654 op/s 2023-09-23T19:45:52.432 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:52 smithi092 bash[58855]: audit 2023-09-23T19:45:51.402220+0000 mon.c (mon.2) 191 : audit 1 from='client.? 172.21.15.16:0/2622590723' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:45:52.432 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:52 smithi092 bash[58855]: audit 2023-09-23T19:45:51.402641+0000 mon.a (mon.0) 1441 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:45:52.433 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:52 smithi092 bash[58855]: audit 2023-09-23T19:45:51.440780+0000 mon.a (mon.0) 1442 : audit 0 from='client.? 172.21.15.16:0/3473364679' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:45:52.433 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:52 smithi092 bash[58855]: audit 2023-09-23T19:45:51.705454+0000 mon.a (mon.0) 1443 : audit 1 from='client.? 172.21.15.16:0/3057418596' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T19:45:52.433 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:52 smithi092 bash[58855]: audit 2023-09-23T19:45:52.081474+0000 mon.a (mon.0) 1444 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:45:52.433 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:52 smithi092 bash[58855]: audit 2023-09-23T19:45:52.089372+0000 mon.a (mon.0) 1445 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:45:52.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:52.434 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:52.534 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:52.560 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:52.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:52.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:45:52.663 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:52.691 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:52.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:52.693 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:52.793 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:52.825 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:52.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:52.828 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:52.927 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T19:45:52.928 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:52.955 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:52.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:52.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:53.059 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:53.088 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:53.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:53.090 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:53.191 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:53.220 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:53.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:53.222 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:45:53.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:53 smithi016 bash[75004]: cluster 2023-09-23T19:45:52.176465+0000 mon.a (mon.0) 1446 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:45:53.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:53 smithi016 bash[75004]: audit 2023-09-23T19:45:52.192046+0000 mon.a (mon.0) 1447 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:45:53.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:53 smithi016 bash[75004]: audit 2023-09-23T19:45:52.192195+0000 mon.a (mon.0) 1448 : audit 1 from='client.? 172.21.15.16:0/3057418596' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T19:45:53.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:53 smithi016 bash[75004]: cluster 2023-09-23T19:45:52.192297+0000 mon.a (mon.0) 1449 : cluster 0 osdmap e203: 8 total, 8 up, 8 in 2023-09-23T19:45:53.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:53 smithi016 bash[75004]: audit 2023-09-23T19:45:52.485226+0000 mon.a (mon.0) 1450 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:45:53.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:53 smithi016 bash[75004]: audit 2023-09-23T19:45:52.493458+0000 mon.a (mon.0) 1451 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:45:53.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:53 smithi016 bash[75004]: cluster 2023-09-23T19:45:52.622315+0000 osd.7 (osd.7) 119 : cluster 3 failed to encode map e203 with expected crc 2023-09-23T19:45:53.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:53 smithi016 bash[75004]: cluster 2023-09-23T19:45:52.623340+0000 osd.7 (osd.7) 120 : cluster 3 failed to encode map e203 with expected crc 2023-09-23T19:45:53.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:53 smithi016 bash[75004]: audit 2023-09-23T19:45:52.980011+0000 mon.a (mon.0) 1452 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:45:53.288 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:53 smithi016 bash[75004]: audit 2023-09-23T19:45:52.981151+0000 mon.a (mon.0) 1453 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:45:53.288 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:53 smithi016 bash[75004]: audit 2023-09-23T19:45:52.987258+0000 mon.a (mon.0) 1454 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:45:53.288 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:53 smithi016 bash[77995]: cluster 2023-09-23T19:45:52.176465+0000 mon.a (mon.0) 1446 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:45:53.288 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:53 smithi016 bash[77995]: audit 2023-09-23T19:45:52.192046+0000 mon.a (mon.0) 1447 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:45:53.288 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:53 smithi016 bash[77995]: audit 2023-09-23T19:45:52.192195+0000 mon.a (mon.0) 1448 : audit 1 from='client.? 172.21.15.16:0/3057418596' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T19:45:53.288 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:53 smithi016 bash[77995]: cluster 2023-09-23T19:45:52.192297+0000 mon.a (mon.0) 1449 : cluster 0 osdmap e203: 8 total, 8 up, 8 in 2023-09-23T19:45:53.288 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:53 smithi016 bash[77995]: audit 2023-09-23T19:45:52.485226+0000 mon.a (mon.0) 1450 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:45:53.288 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:53 smithi016 bash[77995]: audit 2023-09-23T19:45:52.493458+0000 mon.a (mon.0) 1451 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:45:53.288 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:53 smithi016 bash[77995]: cluster 2023-09-23T19:45:52.622315+0000 osd.7 (osd.7) 119 : cluster 3 failed to encode map e203 with expected crc 2023-09-23T19:45:53.289 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:53 smithi016 bash[77995]: cluster 2023-09-23T19:45:52.623340+0000 osd.7 (osd.7) 120 : cluster 3 failed to encode map e203 with expected crc 2023-09-23T19:45:53.289 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:53 smithi016 bash[77995]: audit 2023-09-23T19:45:52.980011+0000 mon.a (mon.0) 1452 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:45:53.289 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:53 smithi016 bash[77995]: audit 2023-09-23T19:45:52.981151+0000 mon.a (mon.0) 1453 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:45:53.289 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:53 smithi016 bash[77995]: audit 2023-09-23T19:45:52.987258+0000 mon.a (mon.0) 1454 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:45:53.323 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:53.349 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:53.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:53.352 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:45:53.452 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:53.476 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:53.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:53.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:53.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:53 smithi092 bash[58855]: cluster 2023-09-23T19:45:52.176465+0000 mon.a (mon.0) 1446 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:45:53.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:53 smithi092 bash[58855]: audit 2023-09-23T19:45:52.192046+0000 mon.a (mon.0) 1447 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:45:53.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:53 smithi092 bash[58855]: audit 2023-09-23T19:45:52.192195+0000 mon.a (mon.0) 1448 : audit 1 from='client.? 172.21.15.16:0/3057418596' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T19:45:53.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:53 smithi092 bash[58855]: cluster 2023-09-23T19:45:52.192297+0000 mon.a (mon.0) 1449 : cluster 0 osdmap e203: 8 total, 8 up, 8 in 2023-09-23T19:45:53.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:53 smithi092 bash[58855]: audit 2023-09-23T19:45:52.485226+0000 mon.a (mon.0) 1450 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:45:53.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:53 smithi092 bash[58855]: audit 2023-09-23T19:45:52.493458+0000 mon.a (mon.0) 1451 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:45:53.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:53 smithi092 bash[58855]: cluster 2023-09-23T19:45:52.622315+0000 osd.7 (osd.7) 119 : cluster 3 failed to encode map e203 with expected crc 2023-09-23T19:45:53.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:53 smithi092 bash[58855]: cluster 2023-09-23T19:45:52.623340+0000 osd.7 (osd.7) 120 : cluster 3 failed to encode map e203 with expected crc 2023-09-23T19:45:53.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:53 smithi092 bash[58855]: audit 2023-09-23T19:45:52.980011+0000 mon.a (mon.0) 1452 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:45:53.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:53 smithi092 bash[58855]: audit 2023-09-23T19:45:52.981151+0000 mon.a (mon.0) 1453 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:45:53.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:53 smithi092 bash[58855]: audit 2023-09-23T19:45:52.987258+0000 mon.a (mon.0) 1454 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:45:53.580 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:53.604 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:53.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:53.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:45:53.708 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:53.739 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:53.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:53.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:45:53.842 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:53.876 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:53.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:53.879 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:53.980 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:54.010 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:54.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:54.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:45:54.114 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:54.141 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:54.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:54.145 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:54.245 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:54.277 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:54.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:54.280 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:54.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:54 smithi016 bash[75004]: cluster 2023-09-23T19:45:52.302465+0000 mgr.y (mgr.25325) 1153 : cluster 0 pgmap v1125: 89 pgs: 89 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 522 op/s 2023-09-23T19:45:54.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:54 smithi016 bash[75004]: cluster 2023-09-23T19:45:53.253758+0000 osd.4 (osd.4) 146 : cluster 3 failed to encode map e203 with expected crc 2023-09-23T19:45:54.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:54 smithi016 bash[75004]: cluster 2023-09-23T19:45:53.256467+0000 osd.4 (osd.4) 147 : cluster 3 failed to encode map e203 with expected crc 2023-09-23T19:45:54.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:54 smithi016 bash[75004]: cluster 2023-09-23T19:45:53.428538+0000 osd.6 (osd.6) 131 : cluster 3 failed to encode map e203 with expected crc 2023-09-23T19:45:54.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:54 smithi016 bash[77995]: cluster 2023-09-23T19:45:52.302465+0000 mgr.y (mgr.25325) 1153 : cluster 0 pgmap v1125: 89 pgs: 89 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 522 op/s 2023-09-23T19:45:54.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:54 smithi016 bash[77995]: cluster 2023-09-23T19:45:53.253758+0000 osd.4 (osd.4) 146 : cluster 3 failed to encode map e203 with expected crc 2023-09-23T19:45:54.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:54 smithi016 bash[77995]: cluster 2023-09-23T19:45:53.256467+0000 osd.4 (osd.4) 147 : cluster 3 failed to encode map e203 with expected crc 2023-09-23T19:45:54.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:54 smithi016 bash[77995]: cluster 2023-09-23T19:45:53.428538+0000 osd.6 (osd.6) 131 : cluster 3 failed to encode map e203 with expected crc 2023-09-23T19:45:54.321 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T19:45:54.381 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:54.438 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:54.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:54.441 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:45:54.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:54 smithi092 bash[58855]: cluster 2023-09-23T19:45:52.302465+0000 mgr.y (mgr.25325) 1153 : cluster 0 pgmap v1125: 89 pgs: 89 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 522 op/s 2023-09-23T19:45:54.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:54 smithi092 bash[58855]: cluster 2023-09-23T19:45:53.253758+0000 osd.4 (osd.4) 146 : cluster 3 failed to encode map e203 with expected crc 2023-09-23T19:45:54.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:54 smithi092 bash[58855]: cluster 2023-09-23T19:45:53.256467+0000 osd.4 (osd.4) 147 : cluster 3 failed to encode map e203 with expected crc 2023-09-23T19:45:54.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:54 smithi092 bash[58855]: cluster 2023-09-23T19:45:53.428538+0000 osd.6 (osd.6) 131 : cluster 3 failed to encode map e203 with expected crc 2023-09-23T19:45:54.542 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:54.575 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:54.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:54.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:45:54.678 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:54.709 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:54.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:54.714 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:54.780 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:45:54.787 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:45:54.814 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:54.857 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:54.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:54.860 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:45:54.961 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:54.988 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:54.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:54.990 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:55.091 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:55.125 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:55.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:55.126 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:55.227 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:55.260 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:55.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:55.263 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:45:55.364 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:55.377 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:55 smithi092 bash[58855]: cluster 2023-09-23T19:45:54.198332+0000 osd.5 (osd.5) 129 : cluster 3 failed to encode map e203 with expected crc 2023-09-23T19:45:55.378 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:55 smithi092 bash[58855]: cluster 2023-09-23T19:45:54.303283+0000 mgr.y (mgr.25325) 1154 : cluster 0 pgmap v1126: 89 pgs: 89 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 457 op/s 2023-09-23T19:45:55.395 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:55.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:55.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:45:55.499 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:55.531 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:55.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:55.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:55.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:55 smithi016 bash[75004]: cluster 2023-09-23T19:45:54.198332+0000 osd.5 (osd.5) 129 : cluster 3 failed to encode map e203 with expected crc 2023-09-23T19:45:55.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:55 smithi016 bash[75004]: cluster 2023-09-23T19:45:54.303283+0000 mgr.y (mgr.25325) 1154 : cluster 0 pgmap v1126: 89 pgs: 89 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 457 op/s 2023-09-23T19:45:55.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:55 smithi016 bash[77995]: cluster 2023-09-23T19:45:54.198332+0000 osd.5 (osd.5) 129 : cluster 3 failed to encode map e203 with expected crc 2023-09-23T19:45:55.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:55 smithi016 bash[77995]: cluster 2023-09-23T19:45:54.303283+0000 mgr.y (mgr.25325) 1154 : cluster 0 pgmap v1126: 89 pgs: 89 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 457 op/s 2023-09-23T19:45:55.635 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:55.667 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:55.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:55.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:55.772 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:55.804 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:55.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:55.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:55.908 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:55.919 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:45:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:45:55] "GET /metrics HTTP/1.1" 200 36294 "" "Prometheus/2.33.4" 2023-09-23T19:45:55.942 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:55.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:55.944 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:56.045 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:56.077 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:56.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:56.081 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:45:56.181 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:56.209 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:56.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:56.212 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:45:56.310 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T19:45:56.312 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:56.344 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:56.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:56.348 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:56.449 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:56.476 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:56.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:56.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:56.580 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:56.609 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:56.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:56.612 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:45:56.713 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:56.740 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:56.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:56.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:45:56.843 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:56.867 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:56.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:56.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:45:56.971 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:56.995 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:56.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:56.998 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:57.099 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:57.129 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:57.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:57.131 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:57.232 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:57.266 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:57.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:57.269 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:57.370 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:57.401 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:57.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:57.405 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:45:57.505 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:57.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:57 smithi016 bash[75004]: cluster 2023-09-23T19:45:56.304058+0000 mgr.y (mgr.25325) 1155 : cluster 0 pgmap v1127: 89 pgs: 89 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 179 op/s 2023-09-23T19:45:57.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:57 smithi016 bash[77995]: cluster 2023-09-23T19:45:56.304058+0000 mgr.y (mgr.25325) 1155 : cluster 0 pgmap v1127: 89 pgs: 89 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 179 op/s 2023-09-23T19:45:57.542 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:45:57.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:57.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:45:57.646 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:57.670 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:57.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:57.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:45:57.706 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:45:57.738 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:57 smithi092 bash[58855]: cluster 2023-09-23T19:45:56.304058+0000 mgr.y (mgr.25325) 1155 : cluster 0 pgmap v1127: 89 pgs: 89 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 179 op/s 2023-09-23T19:45:57.774 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:57.807 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:57.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:57.810 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:57.911 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:57.936 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:57.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:57.938 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:58.039 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:58.068 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:58.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:58.070 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:45:58.171 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:58.205 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:58.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:58.208 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:45:58.309 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:58.342 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:58.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:58.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:45:58.445 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:58.480 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:58.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:58.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:45:58.583 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:58.615 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:58.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:58.618 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:45:58.718 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:58.753 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:58.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:58.756 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:58.846 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:45:58.847 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:45:58.856 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:58.885 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:58.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:58.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:45:58.989 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:59.021 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:59.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:59.026 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:45:59.126 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:59.156 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:45:59.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:59.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:45:59.260 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:59.294 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:45:59.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:59.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:45:59.397 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:59.400 INFO:teuthology.orchestra.run.smithi016.stderr:pool 'unique_pool_6' removed 2023-09-23T19:45:59.442 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:45:59.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:59.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:45:59.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:59 smithi092 bash[58855]: cluster 2023-09-23T19:45:58.304761+0000 mgr.y (mgr.25325) 1156 : cluster 0 pgmap v1128: 89 pgs: 89 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 118 op/s 2023-09-23T19:45:59.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:59 smithi092 bash[58855]: audit 2023-09-23T19:45:58.665657+0000 mon.a (mon.0) 1455 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:45:59.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:45:59 smithi092 bash[58855]: audit 2023-09-23T19:45:59.150353+0000 mon.a (mon.0) 1456 : audit 1 from='client.? 172.21.15.16:0/3849121559' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T19:45:59.546 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:59.570 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:45:59.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:59.572 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:45:59.672 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:59.685 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:59 smithi016 bash[75004]: cluster 2023-09-23T19:45:58.304761+0000 mgr.y (mgr.25325) 1156 : cluster 0 pgmap v1128: 89 pgs: 89 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 118 op/s 2023-09-23T19:45:59.685 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:59 smithi016 bash[75004]: audit 2023-09-23T19:45:58.665657+0000 mon.a (mon.0) 1455 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:45:59.685 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:45:59 smithi016 bash[75004]: audit 2023-09-23T19:45:59.150353+0000 mon.a (mon.0) 1456 : audit 1 from='client.? 172.21.15.16:0/3849121559' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T19:45:59.685 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:59 smithi016 bash[77995]: cluster 2023-09-23T19:45:58.304761+0000 mgr.y (mgr.25325) 1156 : cluster 0 pgmap v1128: 89 pgs: 89 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 118 op/s 2023-09-23T19:45:59.685 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:59 smithi016 bash[77995]: audit 2023-09-23T19:45:58.665657+0000 mon.a (mon.0) 1455 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:45:59.685 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:45:59 smithi016 bash[77995]: audit 2023-09-23T19:45:59.150353+0000 mon.a (mon.0) 1456 : audit 1 from='client.? 172.21.15.16:0/3849121559' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T19:45:59.700 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:45:59.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:59.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:45:59.803 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:59.829 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:45:59.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:59.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:45:59.933 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:59.981 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:45:59.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:45:59.985 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:46:00.085 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:00.106 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:00.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:00.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:46:00.208 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:00.235 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:00.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:00.237 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:46:00.338 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:00.364 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:00.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:00.367 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:46:00.468 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:00.475 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:00 smithi092 bash[58855]: audit 2023-09-23T19:45:59.368892+0000 mon.a (mon.0) 1457 : audit 1 from='client.? 172.21.15.16:0/3849121559' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T19:46:00.476 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:00 smithi092 bash[58855]: cluster 2023-09-23T19:45:59.369445+0000 mon.a (mon.0) 1458 : cluster 0 osdmap e204: 8 total, 8 up, 8 in 2023-09-23T19:46:00.476 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:00 smithi092 bash[58855]: cluster 2023-09-23T19:45:59.849853+0000 osd.4 (osd.4) 148 : cluster 3 failed to encode map e204 with expected crc 2023-09-23T19:46:00.494 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:00.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:00.496 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:00.597 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:00.631 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:00.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:00.634 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:00.735 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:00.766 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:00.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:00.769 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:46:00.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:00 smithi016 bash[75004]: audit 2023-09-23T19:45:59.368892+0000 mon.a (mon.0) 1457 : audit 1 from='client.? 172.21.15.16:0/3849121559' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T19:46:00.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:00 smithi016 bash[75004]: cluster 2023-09-23T19:45:59.369445+0000 mon.a (mon.0) 1458 : cluster 0 osdmap e204: 8 total, 8 up, 8 in 2023-09-23T19:46:00.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:00 smithi016 bash[75004]: cluster 2023-09-23T19:45:59.849853+0000 osd.4 (osd.4) 148 : cluster 3 failed to encode map e204 with expected crc 2023-09-23T19:46:00.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:00 smithi016 bash[77995]: audit 2023-09-23T19:45:59.368892+0000 mon.a (mon.0) 1457 : audit 1 from='client.? 172.21.15.16:0/3849121559' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T19:46:00.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:00 smithi016 bash[77995]: cluster 2023-09-23T19:45:59.369445+0000 mon.a (mon.0) 1458 : cluster 0 osdmap e204: 8 total, 8 up, 8 in 2023-09-23T19:46:00.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:00 smithi016 bash[77995]: cluster 2023-09-23T19:45:59.849853+0000 osd.4 (osd.4) 148 : cluster 3 failed to encode map e204 with expected crc 2023-09-23T19:46:00.870 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:00.895 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:00.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:00.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:00.999 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:01.026 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:01.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:01.030 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:46:01.131 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:01.162 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:01.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:01.165 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:01.265 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:01.293 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:01.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:01.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:01.382 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T19:46:01.383 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T19:46:01.383 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_7 2023-09-23T19:46:01.383 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_7 16 2023-09-23T19:46:01.398 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:01.429 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:01.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:01.432 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:01.533 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:01.535 INFO:teuthology.orchestra.run.smithi092.stdout:{ 2023-09-23T19:46:01.535 INFO:teuthology.orchestra.run.smithi092.stdout: "size": 20, 2023-09-23T19:46:01.535 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 600, 2023-09-23T19:46:01.535 INFO:teuthology.orchestra.run.smithi092.stdout: "ops": [ 2023-09-23T19:46:01.536 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.536 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45947.0:14914 10.e 10:7610796e:::benchmark_data_smithi016_136365_object14913:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e149)", 2023-09-23T19:46:01.536 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:36:41.730834+0000", 2023-09-23T19:46:01.536 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 559.80310900999996, 2023-09-23T19:46:01.536 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.116764646, 2023-09-23T19:46:01.536 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:46:01.536 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:46:01.536 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:46:01.536 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45947", 2023-09-23T19:46:01.536 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/4039191752", 2023-09-23T19:46:01.536 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 14914 2023-09-23T19:46:01.536 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.537 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:46:01.537 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.537 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:46:01.537 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:41.730834+0000", 2023-09-23T19:46:01.537 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:46:01.537 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.537 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.537 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:46:01.537 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:41.730834+0000", 2023-09-23T19:46:01.537 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999919 2023-09-23T19:46:01.537 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.537 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.538 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:46:01.538 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:41.730826+0000", 2023-09-23T19:46:01.538 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.9828999999999998e-05 2023-09-23T19:46:01.538 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.538 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.538 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:46:01.538 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:41.730906+0000", 2023-09-23T19:46:01.538 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.7773e-05 2023-09-23T19:46:01.538 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.538 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.538 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:46:01.538 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:41.730934+0000", 2023-09-23T19:46:01.538 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.8179999999999998e-06 2023-09-23T19:46:01.539 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.539 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.539 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:46:01.539 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:41.730941+0000", 2023-09-23T19:46:01.539 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.9309999999999999e-05 2023-09-23T19:46:01.539 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.539 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.539 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:46:01.539 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:41.731010+0000", 2023-09-23T19:46:01.539 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00038705 2023-09-23T19:46:01.539 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.539 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.540 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:46:01.540 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:41.731397+0000", 2023-09-23T19:46:01.540 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00018300900000000001 2023-09-23T19:46:01.540 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.540 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.540 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 2,6", 2023-09-23T19:46:01.540 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:41.731580+0000", 2023-09-23T19:46:01.540 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.001202891 2023-09-23T19:46:01.540 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.540 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.540 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:46:01.540 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:41.732783+0000", 2023-09-23T19:46:01.540 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0032828219999999999 2023-09-23T19:46:01.541 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.541 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.541 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:01.541 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:41.736066+0000", 2023-09-23T19:46:01.541 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.111441674 2023-09-23T19:46:01.541 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.541 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.541 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:01.541 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:41.847507+0000", 2023-09-23T19:46:01.541 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.4809000000000003e-05 2023-09-23T19:46:01.541 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.542 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.542 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:46:01.542 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:41.847562+0000", 2023-09-23T19:46:01.542 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.6834e-05 2023-09-23T19:46:01.542 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.542 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.542 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:46:01.542 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:41.847599+0000", 2023-09-23T19:46:01.542 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.116764646 2023-09-23T19:46:01.542 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:01.542 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:46:01.542 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:01.542 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.543 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.543 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45947.0:14920 10.e 10:7b288f96:::benchmark_data_smithi016_136365_object14919:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e149)", 2023-09-23T19:46:01.543 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:36:41.734345+0000", 2023-09-23T19:46:01.543 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 559.79959826799995, 2023-09-23T19:46:01.543 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.113359249, 2023-09-23T19:46:01.543 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:46:01.543 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:46:01.543 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:46:01.543 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45947", 2023-09-23T19:46:01.543 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/4039191752", 2023-09-23T19:46:01.543 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 14920 2023-09-23T19:46:01.543 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.543 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:46:01.544 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.544 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:46:01.544 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:41.734345+0000", 2023-09-23T19:46:01.544 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:46:01.544 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.544 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.544 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:46:01.544 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:41.734345+0000", 2023-09-23T19:46:01.544 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T19:46:01.544 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.544 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.544 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:46:01.545 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:41.734344+0000", 2023-09-23T19:46:01.545 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.7948000000000001e-05 2023-09-23T19:46:01.545 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.545 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.545 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:46:01.545 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:41.734372+0000", 2023-09-23T19:46:01.545 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.0434999999999999e-05 2023-09-23T19:46:01.545 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.545 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.545 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:46:01.545 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:41.734382+0000", 2023-09-23T19:46:01.545 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.0749999999999997e-06 2023-09-23T19:46:01.545 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.546 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.546 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:46:01.546 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:41.734387+0000", 2023-09-23T19:46:01.546 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.6844e-05 2023-09-23T19:46:01.546 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.546 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.546 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:46:01.546 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:41.734444+0000", 2023-09-23T19:46:01.546 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00021890700000000001 2023-09-23T19:46:01.546 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.546 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.546 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:46:01.547 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:41.734663+0000", 2023-09-23T19:46:01.547 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00034033000000000002 2023-09-23T19:46:01.547 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.547 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.547 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 2,6", 2023-09-23T19:46:01.547 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:41.735003+0000", 2023-09-23T19:46:01.547 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0015364 2023-09-23T19:46:01.547 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.547 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.547 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:46:01.547 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:41.736540+0000", 2023-09-23T19:46:01.547 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0017523339999999999 2023-09-23T19:46:01.547 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.548 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.548 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:01.548 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:41.738292+0000", 2023-09-23T19:46:01.548 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.10935062800000001 2023-09-23T19:46:01.548 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.548 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.548 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:01.548 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:41.847643+0000", 2023-09-23T19:46:01.548 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.5126000000000002e-05 2023-09-23T19:46:01.548 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.548 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.548 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:46:01.548 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:41.847678+0000", 2023-09-23T19:46:01.549 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.6582999999999999e-05 2023-09-23T19:46:01.549 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.549 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.549 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:46:01.549 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:41.847704+0000", 2023-09-23T19:46:01.549 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.113359249 2023-09-23T19:46:01.549 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:01.549 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:46:01.549 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:01.549 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.549 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.549 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45947.0:15656 10.1 10:84c790d8:::benchmark_data_smithi016_136365_object15655:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e149)", 2023-09-23T19:46:01.550 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:36:42.205431+0000", 2023-09-23T19:46:01.550 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 559.32851208399995, 2023-09-23T19:46:01.550 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.12081094000000001, 2023-09-23T19:46:01.550 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:46:01.550 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:46:01.550 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:46:01.550 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45947", 2023-09-23T19:46:01.550 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/4039191752", 2023-09-23T19:46:01.550 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 15656 2023-09-23T19:46:01.550 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.550 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:46:01.550 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.551 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:46:01.551 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:42.205431+0000", 2023-09-23T19:46:01.551 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:46:01.551 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.551 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.551 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:46:01.551 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:42.205431+0000", 2023-09-23T19:46:01.551 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T19:46:01.551 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.551 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.551 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:46:01.551 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:42.205430+0000", 2023-09-23T19:46:01.552 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.6378999999999994e-05 2023-09-23T19:46:01.552 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.552 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.552 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:46:01.552 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:42.205516+0000", 2023-09-23T19:46:01.552 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.1644999999999999e-05 2023-09-23T19:46:01.552 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.552 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.552 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:46:01.552 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:42.205528+0000", 2023-09-23T19:46:01.553 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.1070000000000004e-06 2023-09-23T19:46:01.553 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.553 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.553 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:46:01.553 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:42.205533+0000", 2023-09-23T19:46:01.553 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.3306000000000002e-05 2023-09-23T19:46:01.553 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.553 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.553 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:46:01.553 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:42.205576+0000", 2023-09-23T19:46:01.553 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00015099800000000001 2023-09-23T19:46:01.554 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.554 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.554 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:46:01.554 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:42.205727+0000", 2023-09-23T19:46:01.554 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000201864 2023-09-23T19:46:01.554 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.554 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.554 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 0,7", 2023-09-23T19:46:01.554 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:42.205929+0000", 2023-09-23T19:46:01.554 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0031567589999999999 2023-09-23T19:46:01.554 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.554 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.555 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:01.555 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:42.209086+0000", 2023-09-23T19:46:01.555 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.8349999999999999e-06 2023-09-23T19:46:01.555 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.555 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.555 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:46:01.555 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:42.209096+0000", 2023-09-23T19:46:01.555 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.116850168 2023-09-23T19:46:01.555 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.555 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.556 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:01.556 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:42.325946+0000", 2023-09-23T19:46:01.556 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00015483900000000001 2023-09-23T19:46:01.556 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.556 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.556 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:46:01.556 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:42.326101+0000", 2023-09-23T19:46:01.556 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00014134999999999999 2023-09-23T19:46:01.556 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.556 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.556 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:46:01.556 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:42.326242+0000", 2023-09-23T19:46:01.556 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.12081094000000001 2023-09-23T19:46:01.557 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:01.557 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:46:01.557 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:01.557 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.557 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.557 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45947.0:15658 10.1 10:810b04f7:::benchmark_data_smithi016_136365_object15657:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e149)", 2023-09-23T19:46:01.557 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:36:42.205625+0000", 2023-09-23T19:46:01.557 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 559.32831850399998, 2023-09-23T19:46:01.557 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.121165729, 2023-09-23T19:46:01.557 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:46:01.557 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:46:01.557 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:46:01.558 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45947", 2023-09-23T19:46:01.558 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/4039191752", 2023-09-23T19:46:01.558 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 15658 2023-09-23T19:46:01.558 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.558 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:46:01.558 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.558 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:46:01.558 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:42.205625+0000", 2023-09-23T19:46:01.558 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:46:01.558 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.558 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.558 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:46:01.558 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:42.205625+0000", 2023-09-23T19:46:01.559 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.999999 2023-09-23T19:46:01.559 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.559 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.559 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:46:01.559 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:42.205624+0000", 2023-09-23T19:46:01.559 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.4408999999999999e-05 2023-09-23T19:46:01.559 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.559 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.559 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:46:01.559 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:42.205698+0000", 2023-09-23T19:46:01.559 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.2119999999999999e-05 2023-09-23T19:46:01.559 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.560 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.560 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:46:01.560 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:42.205710+0000", 2023-09-23T19:46:01.560 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.9369999999999998e-06 2023-09-23T19:46:01.560 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.560 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.560 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:46:01.560 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:42.205717+0000", 2023-09-23T19:46:01.560 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0022343490000000001 2023-09-23T19:46:01.560 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.560 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.560 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:46:01.560 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:42.207952+0000", 2023-09-23T19:46:01.561 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000145611 2023-09-23T19:46:01.561 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.561 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.561 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:46:01.561 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:42.208097+0000", 2023-09-23T19:46:01.561 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00019040499999999999 2023-09-23T19:46:01.561 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.561 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.561 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 0,7", 2023-09-23T19:46:01.561 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:42.208288+0000", 2023-09-23T19:46:01.561 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00097637799999999997 2023-09-23T19:46:01.561 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.561 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.562 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:46:01.562 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:42.209264+0000", 2023-09-23T19:46:01.562 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.001660885 2023-09-23T19:46:01.562 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.562 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.562 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:01.562 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:42.210925+0000", 2023-09-23T19:46:01.562 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.11546416299999999 2023-09-23T19:46:01.562 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.562 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.562 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:01.562 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:42.326389+0000", 2023-09-23T19:46:01.563 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00025573099999999998 2023-09-23T19:46:01.563 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.563 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.563 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:46:01.563 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:42.326645+0000", 2023-09-23T19:46:01.563 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00014573599999999999 2023-09-23T19:46:01.563 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.563 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.563 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:46:01.563 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:42.326791+0000", 2023-09-23T19:46:01.563 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.121165729 2023-09-23T19:46:01.563 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:01.563 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:46:01.564 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:01.564 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.564 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.564 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45947.0:15666 10.1 10:8d0daa67:::benchmark_data_smithi016_136365_object15665:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e149)", 2023-09-23T19:46:01.564 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:36:42.209510+0000", 2023-09-23T19:46:01.564 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 559.32443334599998, 2023-09-23T19:46:01.564 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.11912863999999999, 2023-09-23T19:46:01.564 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:46:01.564 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:46:01.564 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:46:01.564 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45947", 2023-09-23T19:46:01.564 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/4039191752", 2023-09-23T19:46:01.565 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 15666 2023-09-23T19:46:01.565 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.565 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:46:01.565 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.565 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:46:01.565 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:42.209510+0000", 2023-09-23T19:46:01.565 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:46:01.565 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.565 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.565 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:46:01.565 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:42.209510+0000", 2023-09-23T19:46:01.565 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T19:46:01.565 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.566 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.566 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:46:01.566 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:42.209508+0000", 2023-09-23T19:46:01.566 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.8251999999999995e-05 2023-09-23T19:46:01.566 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.566 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.566 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:46:01.566 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:42.209586+0000", 2023-09-23T19:46:01.566 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.5516000000000001e-05 2023-09-23T19:46:01.566 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.566 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.567 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:46:01.567 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:42.209612+0000", 2023-09-23T19:46:01.567 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.2058999999999999e-05 2023-09-23T19:46:01.567 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.567 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.567 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:46:01.567 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:42.209624+0000", 2023-09-23T19:46:01.567 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.9655999999999999e-05 2023-09-23T19:46:01.567 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.567 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.567 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:46:01.567 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:42.209693+0000", 2023-09-23T19:46:01.568 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00016190200000000001 2023-09-23T19:46:01.568 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.568 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.568 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:46:01.568 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:42.209855+0000", 2023-09-23T19:46:01.568 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000150667 2023-09-23T19:46:01.568 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.568 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.568 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 0,7", 2023-09-23T19:46:01.568 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:42.210006+0000", 2023-09-23T19:46:01.568 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00093496499999999995 2023-09-23T19:46:01.568 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.569 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.569 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:46:01.569 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:42.210941+0000", 2023-09-23T19:46:01.569 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.001243755 2023-09-23T19:46:01.569 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.569 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.569 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:01.569 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:42.212185+0000", 2023-09-23T19:46:01.569 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.11619895099999999 2023-09-23T19:46:01.569 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.569 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.569 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:01.570 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:42.328384+0000", 2023-09-23T19:46:01.570 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000140653 2023-09-23T19:46:01.570 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.570 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.570 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:46:01.570 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:42.328524+0000", 2023-09-23T19:46:01.570 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000114294 2023-09-23T19:46:01.570 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.570 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.570 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:46:01.570 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:42.328639+0000", 2023-09-23T19:46:01.570 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.11912863999999999 2023-09-23T19:46:01.571 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:01.571 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:46:01.571 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:01.571 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.571 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.571 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45947.0:18788 10.1 10:87c8449a:::benchmark_data_smithi016_136365_object18787:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e149)", 2023-09-23T19:46:01.571 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:36:43.998195+0000", 2023-09-23T19:46:01.571 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 557.53574840700003, 2023-09-23T19:46:01.571 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.15930808499999999, 2023-09-23T19:46:01.572 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:46:01.572 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:46:01.572 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:46:01.572 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45947", 2023-09-23T19:46:01.572 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/4039191752", 2023-09-23T19:46:01.572 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 18788 2023-09-23T19:46:01.572 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.572 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:46:01.572 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.572 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:46:01.572 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:43.998195+0000", 2023-09-23T19:46:01.572 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:46:01.573 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.573 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.573 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:46:01.573 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:43.998195+0000", 2023-09-23T19:46:01.573 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T19:46:01.573 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.573 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.573 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:46:01.573 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:43.998193+0000", 2023-09-23T19:46:01.573 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.1143999999999999e-05 2023-09-23T19:46:01.573 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.573 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.573 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:46:01.574 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:43.998225+0000", 2023-09-23T19:46:01.574 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.1184e-05 2023-09-23T19:46:01.574 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.574 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.574 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:46:01.574 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:43.998236+0000", 2023-09-23T19:46:01.574 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.4439999999999996e-06 2023-09-23T19:46:01.574 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.574 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.574 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:46:01.574 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:43.998242+0000", 2023-09-23T19:46:01.574 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.6482000000000001e-05 2023-09-23T19:46:01.575 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.575 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.575 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:46:01.575 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:43.998279+0000", 2023-09-23T19:46:01.575 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.7400999999999999e-05 2023-09-23T19:46:01.575 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.575 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.575 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:46:01.575 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:43.998366+0000", 2023-09-23T19:46:01.575 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000106959 2023-09-23T19:46:01.575 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.575 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.575 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 0,7", 2023-09-23T19:46:01.576 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:43.998473+0000", 2023-09-23T19:46:01.576 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0016823949999999999 2023-09-23T19:46:01.576 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.576 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.576 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:46:01.576 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:44.000155+0000", 2023-09-23T19:46:01.576 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.002716375 2023-09-23T19:46:01.576 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.576 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.576 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:01.576 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:44.002872+0000", 2023-09-23T19:46:01.576 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.15454966000000001 2023-09-23T19:46:01.576 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.577 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.577 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:01.577 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:44.157421+0000", 2023-09-23T19:46:01.577 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.4792e-05 2023-09-23T19:46:01.577 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.577 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.577 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:46:01.577 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:44.157476+0000", 2023-09-23T19:46:01.577 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.6809e-05 2023-09-23T19:46:01.577 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.577 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.577 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:46:01.577 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:44.157503+0000", 2023-09-23T19:46:01.578 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.15930808499999999 2023-09-23T19:46:01.578 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:01.578 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:46:01.578 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:01.578 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.578 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.578 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45947.0:18795 10.1 10:84e92114:::benchmark_data_smithi016_136365_object18794:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e149)", 2023-09-23T19:46:01.578 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:36:44.001532+0000", 2023-09-23T19:46:01.578 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 557.53241177899997, 2023-09-23T19:46:01.578 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.156039805, 2023-09-23T19:46:01.578 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:46:01.578 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:46:01.579 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:46:01.579 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45947", 2023-09-23T19:46:01.579 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/4039191752", 2023-09-23T19:46:01.579 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 18795 2023-09-23T19:46:01.579 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.579 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:46:01.579 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.579 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:46:01.579 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:44.001532+0000", 2023-09-23T19:46:01.579 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:46:01.579 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.579 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.579 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:46:01.580 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:44.001532+0000", 2023-09-23T19:46:01.580 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999976 2023-09-23T19:46:01.580 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.580 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.580 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:46:01.580 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:44.001529+0000", 2023-09-23T19:46:01.580 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.8973000000000001e-05 2023-09-23T19:46:01.580 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.580 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.580 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:46:01.580 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:44.001568+0000", 2023-09-23T19:46:01.580 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.628e-05 2023-09-23T19:46:01.581 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.581 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.581 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:46:01.581 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:44.001585+0000", 2023-09-23T19:46:01.581 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.452e-06 2023-09-23T19:46:01.581 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.581 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.581 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:46:01.581 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:44.001592+0000", 2023-09-23T19:46:01.581 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.8476000000000001e-05 2023-09-23T19:46:01.581 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.581 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.581 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:46:01.582 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:44.001651+0000", 2023-09-23T19:46:01.582 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000142762 2023-09-23T19:46:01.582 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.582 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.582 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:46:01.582 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:44.001793+0000", 2023-09-23T19:46:01.582 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00017180099999999999 2023-09-23T19:46:01.582 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.582 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.582 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 0,7", 2023-09-23T19:46:01.582 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:44.001965+0000", 2023-09-23T19:46:01.582 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0011739560000000001 2023-09-23T19:46:01.583 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.583 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.583 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:46:01.583 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:44.003139+0000", 2023-09-23T19:46:01.583 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.001417892 2023-09-23T19:46:01.583 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.583 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.583 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:01.583 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:44.004557+0000", 2023-09-23T19:46:01.583 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.152971252 2023-09-23T19:46:01.583 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.583 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.583 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:01.584 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:44.157528+0000", 2023-09-23T19:46:01.584 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.3754e-05 2023-09-23T19:46:01.584 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.584 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.584 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:46:01.584 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:44.157552+0000", 2023-09-23T19:46:01.584 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.9398e-05 2023-09-23T19:46:01.584 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.584 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.584 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:46:01.584 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:36:44.157571+0000", 2023-09-23T19:46:01.584 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.156039805 2023-09-23T19:46:01.584 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:01.585 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:46:01.585 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:01.585 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.585 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.585 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45947.0:79848 10.1 10:85c01a6b:::benchmark_data_smithi016_136365_object79847:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e152)", 2023-09-23T19:46:01.585 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:37:30.819887+0000", 2023-09-23T19:46:01.585 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 510.71405678500003, 2023-09-23T19:46:01.585 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.12732828900000001, 2023-09-23T19:46:01.585 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:46:01.585 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:46:01.585 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:46:01.585 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45947", 2023-09-23T19:46:01.586 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/4039191752", 2023-09-23T19:46:01.586 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 79848 2023-09-23T19:46:01.586 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.586 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:46:01.586 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.586 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:46:01.586 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:37:30.819887+0000", 2023-09-23T19:46:01.586 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:46:01.586 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.586 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.586 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:46:01.586 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:37:30.819887+0000", 2023-09-23T19:46:01.586 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999976 2023-09-23T19:46:01.587 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.587 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.587 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:46:01.587 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:37:30.819884+0000", 2023-09-23T19:46:01.587 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.4422000000000003e-05 2023-09-23T19:46:01.587 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.587 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.587 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:46:01.587 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:37:30.819929+0000", 2023-09-23T19:46:01.587 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.6303e-05 2023-09-23T19:46:01.587 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.587 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.587 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:46:01.588 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:37:30.819945+0000", 2023-09-23T19:46:01.588 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.3339999999999996e-06 2023-09-23T19:46:01.588 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.588 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.588 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:46:01.588 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:37:30.819952+0000", 2023-09-23T19:46:01.588 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.2163000000000002e-05 2023-09-23T19:46:01.588 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.588 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.588 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:46:01.588 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:37:30.820004+0000", 2023-09-23T19:46:01.588 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00014154200000000001 2023-09-23T19:46:01.589 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.589 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.589 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:46:01.589 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:37:30.820146+0000", 2023-09-23T19:46:01.589 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000172676 2023-09-23T19:46:01.589 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.589 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.589 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 0,7", 2023-09-23T19:46:01.589 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:37:30.820319+0000", 2023-09-23T19:46:01.589 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0034049760000000001 2023-09-23T19:46:01.589 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.589 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.590 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:46:01.590 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:37:30.823724+0000", 2023-09-23T19:46:01.590 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00054667500000000005 2023-09-23T19:46:01.590 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.590 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.590 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:01.590 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:37:30.824270+0000", 2023-09-23T19:46:01.590 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.122844155 2023-09-23T19:46:01.590 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.590 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.590 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:01.590 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:37:30.947114+0000", 2023-09-23T19:46:01.591 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.2706e-05 2023-09-23T19:46:01.591 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.591 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.591 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:46:01.591 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:37:30.947167+0000", 2023-09-23T19:46:01.591 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.7763999999999999e-05 2023-09-23T19:46:01.591 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.591 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.591 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:46:01.591 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:37:30.947215+0000", 2023-09-23T19:46:01.591 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.12732828900000001 2023-09-23T19:46:01.592 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:01.592 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:46:01.592 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:01.592 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.592 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.592 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45947.0:116724 10.e 10:7a8b48b3:::benchmark_data_smithi016_136365_object6590:head [delete] snapc 0=[] ondisk+write+known_if_redirected e155)", 2023-09-23T19:46:01.592 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:38:13.681867+0000", 2023-09-23T19:46:01.592 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 467.85207672299998, 2023-09-23T19:46:01.592 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.010121531, 2023-09-23T19:46:01.592 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:46:01.592 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:46:01.592 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:46:01.593 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45947", 2023-09-23T19:46:01.593 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/4039191752", 2023-09-23T19:46:01.593 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 116724 2023-09-23T19:46:01.593 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.593 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:46:01.593 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.593 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:46:01.593 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.681867+0000", 2023-09-23T19:46:01.593 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:46:01.593 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.593 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.593 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:46:01.594 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.681867+0000", 2023-09-23T19:46:01.594 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T19:46:01.594 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.594 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.594 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:46:01.594 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.681865+0000", 2023-09-23T19:46:01.594 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.6080000000000001e-06 2023-09-23T19:46:01.594 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.594 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.594 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:46:01.594 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.681872+0000", 2023-09-23T19:46:01.594 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.7940000000000003e-06 2023-09-23T19:46:01.594 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.595 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.595 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:46:01.595 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.681880+0000", 2023-09-23T19:46:01.595 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.9399999999999999e-06 2023-09-23T19:46:01.595 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.595 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.595 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:46:01.595 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.681886+0000", 2023-09-23T19:46:01.595 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.011089631000000001 2023-09-23T19:46:01.595 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.595 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.595 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:46:01.596 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.692975+0000", 2023-09-23T19:46:01.596 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.3869999999999995e-06 2023-09-23T19:46:01.596 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.596 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.596 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for peered", 2023-09-23T19:46:01.596 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.692984+0000", 2023-09-23T19:46:01.596 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.993392571 2023-09-23T19:46:01.596 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.596 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.596 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:46:01.596 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:14.686376+0000", 2023-09-23T19:46:01.596 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000149989 2023-09-23T19:46:01.596 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.597 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.597 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:46:01.597 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:14.686526+0000", 2023-09-23T19:46:01.597 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000105047 2023-09-23T19:46:01.597 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.597 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.597 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 6", 2023-09-23T19:46:01.597 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:14.686631+0000", 2023-09-23T19:46:01.597 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0050247699999999996 2023-09-23T19:46:01.597 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.597 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.597 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:46:01.598 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:14.691656+0000", 2023-09-23T19:46:01.598 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00022000700000000001 2023-09-23T19:46:01.598 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.598 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.598 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:01.598 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:14.691876+0000", 2023-09-23T19:46:01.598 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.0068999999999998e-05 2023-09-23T19:46:01.598 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.598 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.598 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:46:01.598 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:14.691926+0000", 2023-09-23T19:46:01.599 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.2075000000000005e-05 2023-09-23T19:46:01.599 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.599 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.599 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:46:01.599 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:14.691988+0000", 2023-09-23T19:46:01.599 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.010121531 2023-09-23T19:46:01.599 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:01.599 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:46:01.599 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:01.599 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.599 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.599 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45947.0:125789 10.1 10:84c790d8:::benchmark_data_smithi016_136365_object15655:head [delete] snapc 0=[] ondisk+write+known_if_redirected e156)", 2023-09-23T19:46:01.600 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:38:25.398758+0000", 2023-09-23T19:46:01.600 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 456.13518545699998, 2023-09-23T19:46:01.600 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.113308696, 2023-09-23T19:46:01.600 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:46:01.600 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:46:01.600 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:46:01.600 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45947", 2023-09-23T19:46:01.600 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/4039191752", 2023-09-23T19:46:01.600 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 125789 2023-09-23T19:46:01.600 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.600 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:46:01.600 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.600 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:46:01.601 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:25.398758+0000", 2023-09-23T19:46:01.601 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:46:01.601 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.601 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.601 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:46:01.601 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:25.398758+0000", 2023-09-23T19:46:01.601 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T19:46:01.601 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.601 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.601 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:46:01.601 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:25.398756+0000", 2023-09-23T19:46:01.602 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.6249999999999998e-06 2023-09-23T19:46:01.602 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.602 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.602 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:46:01.602 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:25.398763+0000", 2023-09-23T19:46:01.602 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.8337e-05 2023-09-23T19:46:01.602 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.602 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.602 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:46:01.602 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:25.398792+0000", 2023-09-23T19:46:01.602 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.9920000000000001e-06 2023-09-23T19:46:01.602 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.602 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.603 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:46:01.603 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:25.398799+0000", 2023-09-23T19:46:01.603 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000180481 2023-09-23T19:46:01.603 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.603 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.603 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:46:01.603 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:25.398979+0000", 2023-09-23T19:46:01.603 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00011345600000000001 2023-09-23T19:46:01.603 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.603 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.603 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:46:01.603 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:25.399093+0000", 2023-09-23T19:46:01.604 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00013220699999999999 2023-09-23T19:46:01.604 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.604 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.604 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 0,7", 2023-09-23T19:46:01.604 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:25.399225+0000", 2023-09-23T19:46:01.604 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0040978730000000001 2023-09-23T19:46:01.604 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.604 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.604 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:01.604 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:25.403323+0000", 2023-09-23T19:46:01.604 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0042405200000000002 2023-09-23T19:46:01.604 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.604 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.605 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:46:01.605 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:25.407563+0000", 2023-09-23T19:46:01.605 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.104320731 2023-09-23T19:46:01.605 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.605 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.605 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:01.605 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:25.511884+0000", 2023-09-23T19:46:01.605 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.7819999999999996e-05 2023-09-23T19:46:01.605 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.605 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.605 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:46:01.605 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:25.511972+0000", 2023-09-23T19:46:01.606 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.4792999999999999e-05 2023-09-23T19:46:01.606 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.606 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.606 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:46:01.606 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:25.512067+0000", 2023-09-23T19:46:01.606 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.113308696 2023-09-23T19:46:01.606 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:01.606 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:46:01.606 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:01.606 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.606 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.606 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45947.0:125791 10.1 10:810b04f7:::benchmark_data_smithi016_136365_object15657:head [delete] snapc 0=[] ondisk+write+known_if_redirected e156)", 2023-09-23T19:46:01.607 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:38:25.401202+0000", 2023-09-23T19:46:01.607 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 456.132741108, 2023-09-23T19:46:01.607 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.120507158, 2023-09-23T19:46:01.607 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:46:01.607 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:46:01.607 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:46:01.607 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45947", 2023-09-23T19:46:01.607 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/4039191752", 2023-09-23T19:46:01.607 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 125791 2023-09-23T19:46:01.607 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.607 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:46:01.607 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.607 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:46:01.608 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:25.401202+0000", 2023-09-23T19:46:01.608 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:46:01.608 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.608 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.608 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:46:01.608 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:25.401202+0000", 2023-09-23T19:46:01.608 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T19:46:01.608 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.608 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.608 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:46:01.608 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:25.401200+0000", 2023-09-23T19:46:01.608 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.0988000000000001e-05 2023-09-23T19:46:01.608 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.609 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.609 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:46:01.609 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:25.401211+0000", 2023-09-23T19:46:01.609 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.4175000000000001e-05 2023-09-23T19:46:01.609 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.609 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.609 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:46:01.609 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:25.401235+0000", 2023-09-23T19:46:01.609 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.7561e-05 2023-09-23T19:46:01.609 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.609 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.609 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:46:01.609 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:25.401253+0000", 2023-09-23T19:46:01.610 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0020908530000000002 2023-09-23T19:46:01.610 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.610 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.610 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:46:01.610 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:25.403344+0000", 2023-09-23T19:46:01.610 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00010599699999999999 2023-09-23T19:46:01.610 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.610 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.610 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:46:01.610 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:25.403450+0000", 2023-09-23T19:46:01.610 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00012367199999999999 2023-09-23T19:46:01.610 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.611 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.611 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 0,7", 2023-09-23T19:46:01.611 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:25.403573+0000", 2023-09-23T19:46:01.611 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.004474123 2023-09-23T19:46:01.611 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.611 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.611 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:01.611 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:25.408048+0000", 2023-09-23T19:46:01.611 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0016013069999999999 2023-09-23T19:46:01.611 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.611 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.611 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:46:01.611 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:25.409649+0000", 2023-09-23T19:46:01.612 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.11195121500000001 2023-09-23T19:46:01.612 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.612 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.612 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:01.612 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:25.521600+0000", 2023-09-23T19:46:01.612 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.4783e-05 2023-09-23T19:46:01.612 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.612 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.612 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:46:01.612 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:25.521655+0000", 2023-09-23T19:46:01.612 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.4512000000000003e-05 2023-09-23T19:46:01.612 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.613 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.613 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:46:01.613 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:25.521709+0000", 2023-09-23T19:46:01.613 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.120507158 2023-09-23T19:46:01.613 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:01.613 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:46:01.613 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:01.613 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.613 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.613 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45947.0:155677 10.1 10:80829ae8:::benchmark_data_smithi016_136365_object45543:head [delete] snapc 0=[] ondisk+write+known_if_redirected e162)", 2023-09-23T19:46:01.613 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:39:09.535964+0000", 2023-09-23T19:46:01.613 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 411.99797906399999, 2023-09-23T19:46:01.613 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.118889934, 2023-09-23T19:46:01.614 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:46:01.614 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:46:01.614 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:46:01.614 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45947", 2023-09-23T19:46:01.614 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/4039191752", 2023-09-23T19:46:01.614 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 155677 2023-09-23T19:46:01.614 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.614 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:46:01.614 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.614 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:46:01.614 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:39:09.535964+0000", 2023-09-23T19:46:01.614 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:46:01.615 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.615 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.615 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:46:01.615 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:39:09.535964+0000", 2023-09-23T19:46:01.615 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999976 2023-09-23T19:46:01.615 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.615 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.615 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:46:01.615 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:39:09.535962+0000", 2023-09-23T19:46:01.615 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.2879e-05 2023-09-23T19:46:01.615 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.615 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.616 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:46:01.616 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:39:09.535975+0000", 2023-09-23T19:46:01.616 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.2062e-05 2023-09-23T19:46:01.616 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.616 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.616 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:46:01.616 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:39:09.535997+0000", 2023-09-23T19:46:01.616 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.7929999999999992e-06 2023-09-23T19:46:01.616 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.616 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.616 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:46:01.616 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:39:09.536006+0000", 2023-09-23T19:46:01.616 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00095477400000000003 2023-09-23T19:46:01.617 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.617 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.617 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:46:01.617 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:39:09.536961+0000", 2023-09-23T19:46:01.617 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00014224300000000001 2023-09-23T19:46:01.617 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.617 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.617 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:46:01.617 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:39:09.537103+0000", 2023-09-23T19:46:01.617 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000167345 2023-09-23T19:46:01.617 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.618 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.618 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 0,7", 2023-09-23T19:46:01.618 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:39:09.537271+0000", 2023-09-23T19:46:01.618 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0037990189999999998 2023-09-23T19:46:01.618 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.618 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.618 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:46:01.618 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:39:09.541070+0000", 2023-09-23T19:46:01.618 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0076574629999999998 2023-09-23T19:46:01.618 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.618 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.618 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:01.619 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:39:09.548727+0000", 2023-09-23T19:46:01.619 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.106007689 2023-09-23T19:46:01.619 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.619 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.619 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:01.619 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:39:09.654735+0000", 2023-09-23T19:46:01.619 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.2517999999999999e-05 2023-09-23T19:46:01.619 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.619 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.619 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:46:01.619 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:39:09.654808+0000", 2023-09-23T19:46:01.619 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.6739000000000001e-05 2023-09-23T19:46:01.619 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.620 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.620 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:46:01.620 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:39:09.654854+0000", 2023-09-23T19:46:01.620 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.118889934 2023-09-23T19:46:01.620 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:01.620 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:46:01.620 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:01.620 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.620 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.620 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45947.0:155679 10.1 10:83737fc5:::benchmark_data_smithi016_136365_object45545:head [delete] snapc 0=[] ondisk+write+known_if_redirected e162)", 2023-09-23T19:46:01.620 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:39:09.541810+0000", 2023-09-23T19:46:01.620 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 411.99213350500003, 2023-09-23T19:46:01.620 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.113171365, 2023-09-23T19:46:01.621 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:46:01.621 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:46:01.621 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:46:01.621 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45947", 2023-09-23T19:46:01.621 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/4039191752", 2023-09-23T19:46:01.621 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 155679 2023-09-23T19:46:01.621 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.621 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:46:01.621 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.621 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:46:01.621 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:39:09.541810+0000", 2023-09-23T19:46:01.621 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:46:01.622 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.622 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.622 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:46:01.622 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:39:09.541810+0000", 2023-09-23T19:46:01.622 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T19:46:01.622 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.622 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.622 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:46:01.622 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:39:09.541808+0000", 2023-09-23T19:46:01.622 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.7589999999999992e-06 2023-09-23T19:46:01.622 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.623 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.623 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:46:01.623 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:39:09.541818+0000", 2023-09-23T19:46:01.623 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.6424999999999999e-05 2023-09-23T19:46:01.623 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.623 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.623 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:46:01.623 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:39:09.541834+0000", 2023-09-23T19:46:01.623 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.2840000000000006e-06 2023-09-23T19:46:01.623 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.623 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.623 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:46:01.624 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:39:09.541842+0000", 2023-09-23T19:46:01.624 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.9920999999999998e-05 2023-09-23T19:46:01.624 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.624 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.624 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:46:01.624 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:39:09.541902+0000", 2023-09-23T19:46:01.624 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00016026199999999999 2023-09-23T19:46:01.624 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.624 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.624 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:46:01.624 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:39:09.542063+0000", 2023-09-23T19:46:01.624 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000163852 2023-09-23T19:46:01.624 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.625 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.625 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 0,7", 2023-09-23T19:46:01.625 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:39:09.542226+0000", 2023-09-23T19:46:01.625 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0041484360000000001 2023-09-23T19:46:01.625 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.625 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.625 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:46:01.625 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:39:09.546375+0000", 2023-09-23T19:46:01.625 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0024992740000000001 2023-09-23T19:46:01.625 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.625 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.625 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:01.625 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:39:09.548874+0000", 2023-09-23T19:46:01.626 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.106021463 2023-09-23T19:46:01.626 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.626 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.626 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:01.626 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:39:09.654896+0000", 2023-09-23T19:46:01.626 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.8603999999999997e-05 2023-09-23T19:46:01.626 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.626 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.626 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:46:01.626 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:39:09.654944+0000", 2023-09-23T19:46:01.626 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.7079000000000002e-05 2023-09-23T19:46:01.626 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.627 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.627 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:46:01.627 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:39:09.654981+0000", 2023-09-23T19:46:01.627 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.113171365 2023-09-23T19:46:01.627 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:01.627 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:46:01.627 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:01.627 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.627 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.627 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45947.0:207464 10.1 10:8927d421:::benchmark_data_smithi016_136365_object97330:head [delete] snapc 0=[] ondisk+write+known_if_redirected e171)", 2023-09-23T19:46:01.627 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:40:28.084666+0000", 2023-09-23T19:46:01.628 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 333.44927749999999, 2023-09-23T19:46:01.628 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.25394785199999997, 2023-09-23T19:46:01.628 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:46:01.628 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:46:01.628 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:46:01.628 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45947", 2023-09-23T19:46:01.628 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/4039191752", 2023-09-23T19:46:01.628 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 207464 2023-09-23T19:46:01.628 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.628 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:46:01.628 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.628 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:46:01.629 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.084666+0000", 2023-09-23T19:46:01.629 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:46:01.629 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.629 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.629 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:46:01.629 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.084666+0000", 2023-09-23T19:46:01.629 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.999999 2023-09-23T19:46:01.629 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.629 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.629 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:46:01.629 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.084665+0000", 2023-09-23T19:46:01.630 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.4840000000000001e-06 2023-09-23T19:46:01.630 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.630 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.630 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:46:01.630 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.084671+0000", 2023-09-23T19:46:01.630 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.0995000000000001e-05 2023-09-23T19:46:01.630 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.630 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.630 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:46:01.630 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.084682+0000", 2023-09-23T19:46:01.630 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.8779999999999997e-06 2023-09-23T19:46:01.630 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.631 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.631 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:46:01.631 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.084687+0000", 2023-09-23T19:46:01.631 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.6968e-05 2023-09-23T19:46:01.631 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.631 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.631 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:46:01.631 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.084714+0000", 2023-09-23T19:46:01.631 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.0439999999999996e-05 2023-09-23T19:46:01.631 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.631 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.632 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:46:01.632 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.084795+0000", 2023-09-23T19:46:01.632 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000112695 2023-09-23T19:46:01.632 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.632 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.632 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 0,7", 2023-09-23T19:46:01.632 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.084907+0000", 2023-09-23T19:46:01.632 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0083150900000000007 2023-09-23T19:46:01.632 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.632 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.632 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:46:01.632 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.093222+0000", 2023-09-23T19:46:01.633 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.020486167999999999 2023-09-23T19:46:01.633 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.633 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.633 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:01.633 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.113709+0000", 2023-09-23T19:46:01.633 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.22466473000000001 2023-09-23T19:46:01.633 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.633 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.633 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:01.633 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.338373+0000", 2023-09-23T19:46:01.633 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00015006899999999999 2023-09-23T19:46:01.633 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.633 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.634 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:46:01.634 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.338523+0000", 2023-09-23T19:46:01.634 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.0434999999999997e-05 2023-09-23T19:46:01.634 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.634 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.634 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:46:01.634 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.338614+0000", 2023-09-23T19:46:01.634 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.25394785199999997 2023-09-23T19:46:01.634 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:01.634 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:46:01.634 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:01.634 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.635 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.635 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45947.0:207475 10.1 10:863264cd:::benchmark_data_smithi016_136365_object97341:head [delete] snapc 0=[] ondisk+write+known_if_redirected e171)", 2023-09-23T19:46:01.635 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:40:28.093935+0000", 2023-09-23T19:46:01.635 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 333.44000834799999, 2023-09-23T19:46:01.635 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.256250439, 2023-09-23T19:46:01.635 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:46:01.635 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:46:01.635 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:46:01.635 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45947", 2023-09-23T19:46:01.635 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/4039191752", 2023-09-23T19:46:01.635 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 207475 2023-09-23T19:46:01.635 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.636 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:46:01.636 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.636 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:46:01.636 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.093935+0000", 2023-09-23T19:46:01.636 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:46:01.636 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.636 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.636 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:46:01.636 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.093935+0000", 2023-09-23T19:46:01.636 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.999999 2023-09-23T19:46:01.636 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.636 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.636 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:46:01.637 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.093934+0000", 2023-09-23T19:46:01.637 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.2740000000000004e-06 2023-09-23T19:46:01.637 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.637 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.637 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:46:01.637 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.093940+0000", 2023-09-23T19:46:01.637 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.5599000000000001e-05 2023-09-23T19:46:01.637 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.637 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.637 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:46:01.637 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.093956+0000", 2023-09-23T19:46:01.637 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.8959999999999997e-06 2023-09-23T19:46:01.637 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.638 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.638 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:46:01.638 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.093963+0000", 2023-09-23T19:46:01.638 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.8640000000000003e-05 2023-09-23T19:46:01.638 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.638 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.638 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:46:01.638 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.094001+0000", 2023-09-23T19:46:01.638 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000118341 2023-09-23T19:46:01.638 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.638 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.638 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:46:01.639 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.094120+0000", 2023-09-23T19:46:01.639 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000134139 2023-09-23T19:46:01.639 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.639 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.639 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 0,7", 2023-09-23T19:46:01.639 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.094254+0000", 2023-09-23T19:46:01.639 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.011365685 2023-09-23T19:46:01.639 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.639 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.639 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:46:01.639 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.105619+0000", 2023-09-23T19:46:01.639 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.012941401 2023-09-23T19:46:01.640 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.640 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.640 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:01.640 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.118561+0000", 2023-09-23T19:46:01.640 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.231469493 2023-09-23T19:46:01.640 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.640 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.640 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:01.640 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.350030+0000", 2023-09-23T19:46:01.640 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.3146000000000007e-05 2023-09-23T19:46:01.640 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.640 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.641 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:46:01.641 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.350114+0000", 2023-09-23T19:46:01.641 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.1976000000000003e-05 2023-09-23T19:46:01.641 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.641 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.641 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:46:01.641 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.350185+0000", 2023-09-23T19:46:01.641 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.256250439 2023-09-23T19:46:01.641 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:01.641 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:46:01.641 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:01.641 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.642 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.642 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.36243.0:71346 11.e 11:746a046b:::benchmark_data_smithi016_160148_object71345:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e186)", 2023-09-23T19:46:01.642 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:42:32.554051+0000", 2023-09-23T19:46:01.642 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 208.97989281900001, 2023-09-23T19:46:01.642 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.13345866200000001, 2023-09-23T19:46:01.642 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:46:01.642 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:46:01.642 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:46:01.642 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.36243", 2023-09-23T19:46:01.642 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/1280038109", 2023-09-23T19:46:01.642 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 71346 2023-09-23T19:46:01.642 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.643 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:46:01.643 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.643 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:46:01.643 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:32.554051+0000", 2023-09-23T19:46:01.643 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:46:01.643 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.643 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.643 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:46:01.643 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:32.554051+0000", 2023-09-23T19:46:01.643 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T19:46:01.643 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.643 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.643 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:46:01.644 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:32.554049+0000", 2023-09-23T19:46:01.644 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.0380000000000001e-05 2023-09-23T19:46:01.644 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.644 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.644 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:46:01.644 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:32.554109+0000", 2023-09-23T19:46:01.644 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.7346999999999998e-05 2023-09-23T19:46:01.644 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.644 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.644 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:46:01.644 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:32.554126+0000", 2023-09-23T19:46:01.644 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.08e-06 2023-09-23T19:46:01.645 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.645 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.645 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:46:01.645 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:32.554133+0000", 2023-09-23T19:46:01.645 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00055516400000000003 2023-09-23T19:46:01.645 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.645 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.645 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:46:01.645 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:32.554689+0000", 2023-09-23T19:46:01.645 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000173048 2023-09-23T19:46:01.645 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.645 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.645 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:46:01.646 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:32.554862+0000", 2023-09-23T19:46:01.646 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000177116 2023-09-23T19:46:01.646 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.646 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.646 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,7", 2023-09-23T19:46:01.646 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:32.555039+0000", 2023-09-23T19:46:01.646 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0024590179999999999 2023-09-23T19:46:01.646 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.646 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.646 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:46:01.646 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:32.557498+0000", 2023-09-23T19:46:01.646 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0096484099999999996 2023-09-23T19:46:01.647 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.647 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.647 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:01.647 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:32.567146+0000", 2023-09-23T19:46:01.647 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.120187312 2023-09-23T19:46:01.647 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.647 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.647 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:01.647 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:32.687334+0000", 2023-09-23T19:46:01.647 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00011732500000000001 2023-09-23T19:46:01.647 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.647 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.647 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:46:01.648 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:32.687451+0000", 2023-09-23T19:46:01.648 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.8372999999999998e-05 2023-09-23T19:46:01.648 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.648 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.648 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:46:01.648 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:32.687509+0000", 2023-09-23T19:46:01.648 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.13345866200000001 2023-09-23T19:46:01.648 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:01.648 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:46:01.648 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:01.648 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.648 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.649 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.36243.0:74043 11.e 11:719f709f:::benchmark_data_smithi016_160148_object74042:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e186)", 2023-09-23T19:46:01.649 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:42:35.485704+0000", 2023-09-23T19:46:01.649 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 206.048238892, 2023-09-23T19:46:01.649 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.119737947, 2023-09-23T19:46:01.649 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:46:01.649 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:46:01.649 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:46:01.649 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.36243", 2023-09-23T19:46:01.649 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/1280038109", 2023-09-23T19:46:01.649 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 74043 2023-09-23T19:46:01.649 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.649 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:46:01.649 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.650 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:46:01.650 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:35.485704+0000", 2023-09-23T19:46:01.650 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:46:01.650 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.650 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.650 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:46:01.650 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:35.485704+0000", 2023-09-23T19:46:01.650 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T19:46:01.650 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.650 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.650 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:46:01.651 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:35.485703+0000", 2023-09-23T19:46:01.651 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.6063999999999996e-05 2023-09-23T19:46:01.651 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.651 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.651 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:46:01.651 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:35.485769+0000", 2023-09-23T19:46:01.651 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.9275000000000001e-05 2023-09-23T19:46:01.651 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.651 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.651 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:46:01.652 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:35.485788+0000", 2023-09-23T19:46:01.652 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.1219999999999995e-06 2023-09-23T19:46:01.652 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.652 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.652 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:46:01.652 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:35.485796+0000", 2023-09-23T19:46:01.652 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.7423999999999998e-05 2023-09-23T19:46:01.652 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.652 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.652 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:46:01.652 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:35.485854+0000", 2023-09-23T19:46:01.653 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000169233 2023-09-23T19:46:01.653 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.653 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.653 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:46:01.653 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:35.486023+0000", 2023-09-23T19:46:01.653 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00022597000000000001 2023-09-23T19:46:01.653 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.653 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.653 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,7", 2023-09-23T19:46:01.653 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:35.486249+0000", 2023-09-23T19:46:01.654 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.002579989 2023-09-23T19:46:01.654 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.654 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.654 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:46:01.654 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:35.488829+0000", 2023-09-23T19:46:01.654 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00094848000000000003 2023-09-23T19:46:01.654 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.654 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.654 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:01.654 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:35.489778+0000", 2023-09-23T19:46:01.654 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.11552668200000001 2023-09-23T19:46:01.655 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.655 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.655 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:01.655 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:35.605304+0000", 2023-09-23T19:46:01.655 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.3707000000000003e-05 2023-09-23T19:46:01.655 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.655 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.655 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:46:01.655 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:35.605388+0000", 2023-09-23T19:46:01.655 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.4534000000000003e-05 2023-09-23T19:46:01.656 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.656 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.656 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:46:01.656 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:35.605442+0000", 2023-09-23T19:46:01.656 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.119737947 2023-09-23T19:46:01.656 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:01.656 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:46:01.656 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:01.656 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.656 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.656 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.36243.0:105219 11.e 11:72ef5cba:::benchmark_data_smithi016_160148_object105218:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e190)", 2023-09-23T19:46:01.656 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:43:11.584057+0000", 2023-09-23T19:46:01.657 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 169.94988664300001, 2023-09-23T19:46:01.657 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.14955489999999999, 2023-09-23T19:46:01.657 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:46:01.657 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:46:01.657 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:46:01.657 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.36243", 2023-09-23T19:46:01.657 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/1280038109", 2023-09-23T19:46:01.657 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 105219 2023-09-23T19:46:01.657 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.657 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:46:01.657 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.658 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:46:01.658 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:11.584057+0000", 2023-09-23T19:46:01.658 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:46:01.658 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.658 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.658 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:46:01.658 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:11.584057+0000", 2023-09-23T19:46:01.658 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999976 2023-09-23T19:46:01.658 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.658 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.659 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:46:01.659 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:11.584054+0000", 2023-09-23T19:46:01.659 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.5451999999999998e-05 2023-09-23T19:46:01.659 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.659 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.659 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:46:01.659 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:11.584100+0000", 2023-09-23T19:46:01.659 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.5624000000000001e-05 2023-09-23T19:46:01.659 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.659 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.660 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:46:01.660 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:11.584125+0000", 2023-09-23T19:46:01.660 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.0162e-05 2023-09-23T19:46:01.660 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.660 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.660 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:46:01.660 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:11.584136+0000", 2023-09-23T19:46:01.660 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.6305999999999998e-05 2023-09-23T19:46:01.660 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.660 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.660 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:46:01.661 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:11.584202+0000", 2023-09-23T19:46:01.661 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00032370000000000001 2023-09-23T19:46:01.661 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.661 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.661 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:46:01.661 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:11.584526+0000", 2023-09-23T19:46:01.661 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000216465 2023-09-23T19:46:01.661 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.661 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.661 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,7", 2023-09-23T19:46:01.662 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:11.584742+0000", 2023-09-23T19:46:01.662 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0035166419999999999 2023-09-23T19:46:01.662 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.662 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.662 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:46:01.662 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:11.588259+0000", 2023-09-23T19:46:01.662 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0086689790000000003 2023-09-23T19:46:01.662 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.662 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.662 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:01.662 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:11.596928+0000", 2023-09-23T19:46:01.663 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.13654532699999999 2023-09-23T19:46:01.663 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.663 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.663 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:01.663 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:11.733473+0000", 2023-09-23T19:46:01.663 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.5894000000000002e-05 2023-09-23T19:46:01.663 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.663 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.663 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:46:01.663 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:11.733549+0000", 2023-09-23T19:46:01.664 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.2768999999999995e-05 2023-09-23T19:46:01.664 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.664 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.664 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:46:01.664 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:11.733612+0000", 2023-09-23T19:46:01.664 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.14955489999999999 2023-09-23T19:46:01.664 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:01.664 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:46:01.664 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:01.664 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.665 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.665 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.36243.0:136887 11.e 11:7f36b154:::benchmark_data_smithi016_160148_object29684:head [delete] snapc 0=[] ondisk+write+known_if_redirected e194)", 2023-09-23T19:46:01.665 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:43:54.273102+0000", 2023-09-23T19:46:01.665 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 127.260841001, 2023-09-23T19:46:01.665 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.113880035, 2023-09-23T19:46:01.665 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:46:01.665 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:46:01.665 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:46:01.665 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.36243", 2023-09-23T19:46:01.665 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/1280038109", 2023-09-23T19:46:01.665 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 136887 2023-09-23T19:46:01.666 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.666 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:46:01.666 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.666 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:46:01.666 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:54.273102+0000", 2023-09-23T19:46:01.666 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:46:01.666 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.666 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.666 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:46:01.666 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:54.273102+0000", 2023-09-23T19:46:01.666 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999976 2023-09-23T19:46:01.666 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.667 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.667 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:46:01.667 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:54.273100+0000", 2023-09-23T19:46:01.667 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.0803e-05 2023-09-23T19:46:01.667 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.667 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.667 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:46:01.667 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:54.273111+0000", 2023-09-23T19:46:01.667 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.1776e-05 2023-09-23T19:46:01.667 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.667 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.667 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:46:01.667 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:54.273132+0000", 2023-09-23T19:46:01.668 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.5059999999999998e-06 2023-09-23T19:46:01.668 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.668 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.668 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:46:01.668 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:54.273140+0000", 2023-09-23T19:46:01.668 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.2147000000000001e-05 2023-09-23T19:46:01.668 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.668 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.668 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:46:01.668 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:54.273192+0000", 2023-09-23T19:46:01.668 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000124979 2023-09-23T19:46:01.668 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.669 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.669 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:46:01.669 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:54.273317+0000", 2023-09-23T19:46:01.669 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00012845599999999999 2023-09-23T19:46:01.669 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.669 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.669 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,7", 2023-09-23T19:46:01.669 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:54.273446+0000", 2023-09-23T19:46:01.669 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0082808880000000001 2023-09-23T19:46:01.669 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.669 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.669 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:46:01.669 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:54.281726+0000", 2023-09-23T19:46:01.670 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0062855150000000002 2023-09-23T19:46:01.670 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.670 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.670 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:01.670 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:54.288012+0000", 2023-09-23T19:46:01.670 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.098084202999999995 2023-09-23T19:46:01.670 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.670 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.670 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:01.670 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:54.386096+0000", 2023-09-23T19:46:01.670 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00061483600000000001 2023-09-23T19:46:01.670 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.671 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.671 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:46:01.671 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:54.386711+0000", 2023-09-23T19:46:01.671 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00027140600000000002 2023-09-23T19:46:01.671 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.671 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.671 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:46:01.671 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:54.386982+0000", 2023-09-23T19:46:01.671 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.113880035 2023-09-23T19:46:01.671 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:01.671 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:46:01.671 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:01.672 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.672 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.672 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.36243.0:138157 11.e 11:731c0253:::benchmark_data_smithi016_160148_object30954:head [delete] snapc 0=[] ondisk+write+known_if_redirected e194)", 2023-09-23T19:46:01.672 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:43:56.128224+0000", 2023-09-23T19:46:01.672 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 125.405719453, 2023-09-23T19:46:01.672 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.11564439999999999, 2023-09-23T19:46:01.672 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:46:01.672 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:46:01.672 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:46:01.672 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.36243", 2023-09-23T19:46:01.672 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/1280038109", 2023-09-23T19:46:01.672 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 138157 2023-09-23T19:46:01.673 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.673 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:46:01.673 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.673 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:46:01.673 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:56.128224+0000", 2023-09-23T19:46:01.673 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:46:01.673 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.673 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.673 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:46:01.673 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:56.128224+0000", 2023-09-23T19:46:01.673 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T19:46:01.673 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.674 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.674 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:46:01.674 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:56.128222+0000", 2023-09-23T19:46:01.674 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.5590000000000002e-06 2023-09-23T19:46:01.674 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.674 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.674 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:46:01.674 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:56.128229+0000", 2023-09-23T19:46:01.674 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.1104000000000001e-05 2023-09-23T19:46:01.674 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.674 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.674 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:46:01.674 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:56.128240+0000", 2023-09-23T19:46:01.675 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.9390000000000002e-06 2023-09-23T19:46:01.675 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.675 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.675 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:46:01.675 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:56.128246+0000", 2023-09-23T19:46:01.675 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00025384300000000002 2023-09-23T19:46:01.675 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.675 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.675 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:46:01.675 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:56.128500+0000", 2023-09-23T19:46:01.675 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000106239 2023-09-23T19:46:01.675 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.676 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.676 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:46:01.676 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:56.128606+0000", 2023-09-23T19:46:01.676 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000122624 2023-09-23T19:46:01.676 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.676 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.676 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,7", 2023-09-23T19:46:01.676 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:56.128729+0000", 2023-09-23T19:46:01.676 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0081609270000000001 2023-09-23T19:46:01.676 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.676 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.676 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:46:01.676 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:56.136890+0000", 2023-09-23T19:46:01.677 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0019891399999999999 2023-09-23T19:46:01.677 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.677 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.677 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:01.677 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:56.138879+0000", 2023-09-23T19:46:01.677 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.104821188 2023-09-23T19:46:01.677 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.677 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.677 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:01.677 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:56.243700+0000", 2023-09-23T19:46:01.677 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00010370999999999999 2023-09-23T19:46:01.677 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.677 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.678 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:46:01.678 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:56.243804+0000", 2023-09-23T19:46:01.678 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.4679999999999997e-05 2023-09-23T19:46:01.678 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:01.678 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:01.678 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:46:01.678 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:56.243868+0000", 2023-09-23T19:46:01.678 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.11564439999999999 2023-09-23T19:46:01.678 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:01.678 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:46:01.678 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:01.678 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:01.679 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:46:01.679 INFO:teuthology.orchestra.run.smithi092.stdout:} 2023-09-23T19:46:01.680 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:01 smithi092 bash[58855]: cluster 2023-09-23T19:45:59.849635+0000 osd.5 (osd.5) 130 : cluster 3 failed to encode map e204 with expected crc 2023-09-23T19:46:01.680 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:01 smithi092 bash[58855]: cluster 2023-09-23T19:45:59.849760+0000 osd.6 (osd.6) 132 : cluster 3 failed to encode map e204 with expected crc 2023-09-23T19:46:01.680 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:01 smithi092 bash[58855]: cluster 2023-09-23T19:45:59.849890+0000 osd.7 (osd.7) 121 : cluster 3 failed to encode map e204 with expected crc 2023-09-23T19:46:01.680 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:01 smithi092 bash[58855]: cluster 2023-09-23T19:46:00.305730+0000 mgr.y (mgr.25325) 1157 : cluster 0 pgmap v1130: 73 pgs: 73 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:46:01.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:01.681 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:01.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:46:01.681 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:01 smithi016 bash[77995]: cluster 2023-09-23T19:45:59.849635+0000 osd.5 (osd.5) 130 : cluster 3 failed to encode map e204 with expected crc 2023-09-23T19:46:01.681 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:01 smithi016 bash[77995]: cluster 2023-09-23T19:45:59.849760+0000 osd.6 (osd.6) 132 : cluster 3 failed to encode map e204 with expected crc 2023-09-23T19:46:01.682 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:01 smithi016 bash[77995]: cluster 2023-09-23T19:45:59.849890+0000 osd.7 (osd.7) 121 : cluster 3 failed to encode map e204 with expected crc 2023-09-23T19:46:01.682 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:01 smithi016 bash[77995]: cluster 2023-09-23T19:46:00.305730+0000 mgr.y (mgr.25325) 1157 : cluster 0 pgmap v1130: 73 pgs: 73 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:46:01.682 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:01 smithi016 bash[75004]: cluster 2023-09-23T19:45:59.849635+0000 osd.5 (osd.5) 130 : cluster 3 failed to encode map e204 with expected crc 2023-09-23T19:46:01.682 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:01 smithi016 bash[75004]: cluster 2023-09-23T19:45:59.849760+0000 osd.6 (osd.6) 132 : cluster 3 failed to encode map e204 with expected crc 2023-09-23T19:46:01.682 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:01 smithi016 bash[75004]: cluster 2023-09-23T19:45:59.849890+0000 osd.7 (osd.7) 121 : cluster 3 failed to encode map e204 with expected crc 2023-09-23T19:46:01.682 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:01 smithi016 bash[75004]: cluster 2023-09-23T19:46:00.305730+0000 mgr.y (mgr.25325) 1157 : cluster 0 pgmap v1130: 73 pgs: 73 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:46:01.782 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:01.818 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:01.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:01.820 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:01.875 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:46:01.921 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:01.948 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:01.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:01.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:02.051 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:02.074 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:02.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:02.077 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:02.178 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:02.210 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:02.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:02.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:02.313 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:02.337 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:02.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:02.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:02.440 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:02.472 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:02.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:02.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:02.495 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T19:46:02.576 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:02.613 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:02.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:02.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:46:02.716 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:02.740 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:02.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:02.743 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:46:02.843 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:02.873 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:02.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:02.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:02.976 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:03.007 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:03.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:03.011 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:03.114 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:03.144 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:03.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:03.148 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:03.248 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:03.278 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:03.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:03.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:46:03.382 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:03.414 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:03.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:03.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:46:03.518 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:03.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:03 smithi016 bash[75004]: cluster 2023-09-23T19:46:02.306405+0000 mgr.y (mgr.25325) 1158 : cluster 0 pgmap v1131: 73 pgs: 73 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:46:03.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:03 smithi016 bash[77995]: cluster 2023-09-23T19:46:02.306405+0000 mgr.y (mgr.25325) 1158 : cluster 0 pgmap v1131: 73 pgs: 73 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:46:03.551 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:03.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:03.555 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:03.656 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:03.668 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:03 smithi092 bash[58855]: cluster 2023-09-23T19:46:02.306405+0000 mgr.y (mgr.25325) 1158 : cluster 0 pgmap v1131: 73 pgs: 73 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:46:03.687 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:03.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:03.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:03.791 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:03.822 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:03.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:03.826 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:46:03.927 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:03.958 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:03.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:03.961 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:46:04.062 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:04.094 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:04.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:04.096 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:04.197 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:04.222 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:04.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:04.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:46:04.326 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:04.361 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:04.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:04.363 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:46:04.464 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:04.495 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:04.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:04.498 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:04.599 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:04.633 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:04.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:04.636 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:04.736 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:04.770 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:04.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:04.774 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:04.875 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:04.898 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:04.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:04.901 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:05.001 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:05.037 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:05.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:05.040 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:46:05.141 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:05.169 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:05.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:05.172 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:46:05.273 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:05.295 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:05.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:05.298 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:46:05.399 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:05.428 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:05.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:05.433 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:46:05.534 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:05.572 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:05.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:05.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:05.676 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:05.707 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:05.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:05.710 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:05.810 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:05.821 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:05 smithi092 bash[58855]: cluster 2023-09-23T19:46:04.307016+0000 mgr.y (mgr.25325) 1159 : cluster 0 pgmap v1132: 73 pgs: 73 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:46:05.841 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:05.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:05.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:05.926 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:05 smithi016 bash[77995]: cluster 2023-09-23T19:46:04.307016+0000 mgr.y (mgr.25325) 1159 : cluster 0 pgmap v1132: 73 pgs: 73 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:46:05.927 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:05 smithi016 bash[75004]: cluster 2023-09-23T19:46:04.307016+0000 mgr.y (mgr.25325) 1159 : cluster 0 pgmap v1132: 73 pgs: 73 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:46:05.927 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:46:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:46:05] "GET /metrics HTTP/1.1" 200 36254 "" "Prometheus/2.33.4" 2023-09-23T19:46:05.944 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:05.978 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:05.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:05.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:06.082 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:06.120 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:06.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:06.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:06.224 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:06.258 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:06.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:06.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:06.362 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:06.394 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:06.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:06.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:06.499 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:06.531 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:06.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:06.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:06.635 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:06.658 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:06.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:06.660 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:46:06.761 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:06.800 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:06.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:06.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:46:06.904 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:06.935 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:06.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:06.938 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:07.039 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:07.068 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:07.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:07.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:46:07.172 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:07.207 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:07.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:07.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:46:07.311 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:07.314 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:46:07.337 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:07.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:07.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:07.441 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:07.464 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:07.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:07.467 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:07.568 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:07.598 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:07.600 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:07 smithi092 bash[58855]: cluster 2023-09-23T19:46:06.307700+0000 mgr.y (mgr.25325) 1160 : cluster 0 pgmap v1133: 73 pgs: 73 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:46:07.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:07.601 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:07.650 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:46:07.674 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:46:07.700 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:46:07.702 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:07.721 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:07.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:07.724 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:07.725 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:46:07.746 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:46:07.766 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:46:07.788 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:46:07.810 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:46:07.812 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:46:07.812 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:46:07.812 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:46:07.812 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:46:07.812 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:46:07.813 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:46:07.813 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:46:07.813 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:46:07.825 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:07.835 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:07 smithi016 bash[77995]: cluster 2023-09-23T19:46:06.307700+0000 mgr.y (mgr.25325) 1160 : cluster 0 pgmap v1133: 73 pgs: 73 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:46:07.835 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:07 smithi016 bash[75004]: cluster 2023-09-23T19:46:06.307700+0000 mgr.y (mgr.25325) 1160 : cluster 0 pgmap v1133: 73 pgs: 73 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:46:07.853 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:07.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:07.856 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:46:07.957 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:07.983 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:07.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:07.985 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:46:08.086 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:08.119 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:08.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:08.122 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:08.223 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:08.249 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:08.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:08.253 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:46:08.354 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:08.381 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:08.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:08.383 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:08.484 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:08.514 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:08.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:08.517 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:08.592 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T19:46:08.618 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:08.638 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:08.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:08.641 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:46:08.739 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:08 smithi016 bash[77995]: audit 2023-09-23T19:46:07.625512+0000 mon.a (mon.0) 1459 : audit 0 from='client.? 172.21.15.16:0/4155196308' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:46:08.739 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:08 smithi016 bash[77995]: audit 2023-09-23T19:46:07.709370+0000 mon.a (mon.0) 1460 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T19:46:08.739 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:08 smithi016 bash[77995]: audit 2023-09-23T19:46:07.714647+0000 mon.b (mon.1) 52 : audit 1 from='client.? 172.21.15.16:0/2999111179' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T19:46:08.739 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:08 smithi016 bash[75004]: audit 2023-09-23T19:46:07.625512+0000 mon.a (mon.0) 1459 : audit 0 from='client.? 172.21.15.16:0/4155196308' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:46:08.739 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:08 smithi016 bash[75004]: audit 2023-09-23T19:46:07.709370+0000 mon.a (mon.0) 1460 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T19:46:08.740 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:08 smithi016 bash[75004]: audit 2023-09-23T19:46:07.714647+0000 mon.b (mon.1) 52 : audit 1 from='client.? 172.21.15.16:0/2999111179' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T19:46:08.742 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:08.764 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:08.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:08.768 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:46:08.869 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:08.892 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:08.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:08.896 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:08.941 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:08 smithi092 bash[58855]: audit 2023-09-23T19:46:07.625512+0000 mon.a (mon.0) 1459 : audit 0 from='client.? 172.21.15.16:0/4155196308' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:46:08.941 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:08 smithi092 bash[58855]: audit 2023-09-23T19:46:07.709370+0000 mon.a (mon.0) 1460 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T19:46:08.941 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:08 smithi092 bash[58855]: audit 2023-09-23T19:46:07.714647+0000 mon.b (mon.1) 52 : audit 1 from='client.? 172.21.15.16:0/2999111179' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T19:46:08.996 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:09.019 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:09.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:09.021 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:09.122 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:09.147 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:09.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:09.150 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:09.251 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:09.277 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:09.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:09.280 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:09.381 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:09.414 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:09.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:09.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:09.517 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:09.540 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:09.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:09.542 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:09.643 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:09.669 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:09.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:09.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:46:09.773 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:09.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:09 smithi016 bash[75004]: cluster 2023-09-23T19:46:08.308420+0000 mgr.y (mgr.25325) 1161 : cluster 0 pgmap v1134: 73 pgs: 73 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:46:09.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:09 smithi016 bash[75004]: audit 2023-09-23T19:46:08.580087+0000 mon.a (mon.0) 1461 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T19:46:09.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:09 smithi016 bash[75004]: cluster 2023-09-23T19:46:08.580337+0000 mon.a (mon.0) 1462 : cluster 0 osdmap e205: 8 total, 8 up, 8 in 2023-09-23T19:46:09.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:09 smithi016 bash[75004]: cluster 2023-09-23T19:46:08.581810+0000 osd.5 (osd.5) 131 : cluster 3 failed to encode map e205 with expected crc 2023-09-23T19:46:09.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:09 smithi016 bash[75004]: cluster 2023-09-23T19:46:09.315650+0000 osd.6 (osd.6) 133 : cluster 3 failed to encode map e205 with expected crc 2023-09-23T19:46:09.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:09 smithi016 bash[77995]: cluster 2023-09-23T19:46:08.308420+0000 mgr.y (mgr.25325) 1161 : cluster 0 pgmap v1134: 73 pgs: 73 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:46:09.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:09 smithi016 bash[77995]: audit 2023-09-23T19:46:08.580087+0000 mon.a (mon.0) 1461 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T19:46:09.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:09 smithi016 bash[77995]: cluster 2023-09-23T19:46:08.580337+0000 mon.a (mon.0) 1462 : cluster 0 osdmap e205: 8 total, 8 up, 8 in 2023-09-23T19:46:09.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:09 smithi016 bash[77995]: cluster 2023-09-23T19:46:08.581810+0000 osd.5 (osd.5) 131 : cluster 3 failed to encode map e205 with expected crc 2023-09-23T19:46:09.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:09 smithi016 bash[77995]: cluster 2023-09-23T19:46:09.315650+0000 osd.6 (osd.6) 133 : cluster 3 failed to encode map e205 with expected crc 2023-09-23T19:46:09.802 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:09.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:09.806 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:09.906 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:09.919 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:09 smithi092 bash[58855]: cluster 2023-09-23T19:46:08.308420+0000 mgr.y (mgr.25325) 1161 : cluster 0 pgmap v1134: 73 pgs: 73 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:46:09.919 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:09 smithi092 bash[58855]: audit 2023-09-23T19:46:08.580087+0000 mon.a (mon.0) 1461 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T19:46:09.919 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:09 smithi092 bash[58855]: cluster 2023-09-23T19:46:08.580337+0000 mon.a (mon.0) 1462 : cluster 0 osdmap e205: 8 total, 8 up, 8 in 2023-09-23T19:46:09.919 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:09 smithi092 bash[58855]: cluster 2023-09-23T19:46:08.581810+0000 osd.5 (osd.5) 131 : cluster 3 failed to encode map e205 with expected crc 2023-09-23T19:46:09.919 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:09 smithi092 bash[58855]: cluster 2023-09-23T19:46:09.315650+0000 osd.6 (osd.6) 133 : cluster 3 failed to encode map e205 with expected crc 2023-09-23T19:46:09.936 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:09.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:09.939 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:10.040 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:10.070 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:10.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:10.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:10.173 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:10.198 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:10.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:10.201 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:10.302 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:10.332 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:10.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:10.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:10.436 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:10.469 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:10.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:10.471 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:46:10.572 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:10.605 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:10.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:10.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:10.708 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:10.739 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:10.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:10.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:10.843 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:10.850 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:10 smithi092 bash[58855]: cluster 2023-09-23T19:46:09.454124+0000 osd.4 (osd.4) 149 : cluster 3 failed to encode map e205 with expected crc 2023-09-23T19:46:10.857 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:10 smithi016 bash[75004]: cluster 2023-09-23T19:46:09.454124+0000 osd.4 (osd.4) 149 : cluster 3 failed to encode map e205 with expected crc 2023-09-23T19:46:10.858 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:10 smithi016 bash[77995]: cluster 2023-09-23T19:46:09.454124+0000 osd.4 (osd.4) 149 : cluster 3 failed to encode map e205 with expected crc 2023-09-23T19:46:10.866 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:10.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:10.869 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:46:10.941 INFO:teuthology.orchestra.run.smithi092.stdout:ERROR: (22) Invalid argument 2023-09-23T19:46:10.941 INFO:teuthology.orchestra.run.smithi092.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:46:10.970 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:10.992 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:10.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:10.995 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:11.096 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:11.131 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:11.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:11.134 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:46:11.179 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T19:46:11.235 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:11.260 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:11.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:11.263 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:46:11.364 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:11.389 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:11.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:11.393 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:46:11.494 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:11.522 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:11.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:11.525 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:46:11.563 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:46:11.625 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:11.650 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:11.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:11.652 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:11.753 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:11.762 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:11 smithi092 bash[58855]: cluster 2023-09-23T19:46:10.309349+0000 mgr.y (mgr.25325) 1162 : cluster 0 pgmap v1136: 73 pgs: 73 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:46:11.780 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:11.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:11.783 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:46:11.884 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:11.893 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:11 smithi016 bash[75004]: cluster 2023-09-23T19:46:10.309349+0000 mgr.y (mgr.25325) 1162 : cluster 0 pgmap v1136: 73 pgs: 73 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:46:11.893 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:11 smithi016 bash[77995]: cluster 2023-09-23T19:46:10.309349+0000 mgr.y (mgr.25325) 1162 : cluster 0 pgmap v1136: 73 pgs: 73 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:46:11.910 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:11.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:11.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:12.008 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T19:46:12.029 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:12.057 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:12.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:12.060 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:46:12.161 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:12.185 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:12.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:12.187 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:12.288 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:12.321 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:12.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:12.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:12.424 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:12.456 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:12.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:12.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:46:12.560 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:12.598 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:12.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:12.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:12.610 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is unset 2023-09-23T19:46:12.702 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:12.731 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:12.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:12.733 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:46:12.834 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:12.860 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:46:12.866 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:12.867 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:12 smithi016 bash[77995]: audit 2023-09-23T19:46:11.602746+0000 mon.c (mon.2) 192 : audit 1 from='client.? 172.21.15.16:0/1548030116' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:46:12.867 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:12 smithi016 bash[77995]: audit 2023-09-23T19:46:11.603275+0000 mon.a (mon.0) 1463 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:46:12.868 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:12 smithi016 bash[75004]: audit 2023-09-23T19:46:11.602746+0000 mon.c (mon.2) 192 : audit 1 from='client.? 172.21.15.16:0/1548030116' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:46:12.868 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:12 smithi016 bash[75004]: audit 2023-09-23T19:46:11.603275+0000 mon.a (mon.0) 1463 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:46:12.869 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:12 smithi092 bash[58855]: audit 2023-09-23T19:46:11.602746+0000 mon.c (mon.2) 192 : audit 1 from='client.? 172.21.15.16:0/1548030116' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:46:12.869 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:12 smithi092 bash[58855]: audit 2023-09-23T19:46:11.603275+0000 mon.a (mon.0) 1463 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:46:12.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:12.870 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:12.970 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:12.992 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:12.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:12.995 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:46:13.096 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:13.131 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:13.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:13.134 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:13.235 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:13.264 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:13.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:13.266 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:13.367 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:13.397 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:13.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:13.400 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:13.500 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:13.532 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:13.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:13.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:13.636 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:13.661 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:13.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:13.664 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:13.765 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:13.779 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:13 smithi092 bash[58855]: cluster 2023-09-23T19:46:12.310093+0000 mgr.y (mgr.25325) 1163 : cluster 0 pgmap v1137: 73 pgs: 73 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:46:13.779 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:13 smithi092 bash[58855]: cluster 2023-09-23T19:46:12.587294+0000 mon.a (mon.0) 1464 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:46:13.779 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:13 smithi092 bash[58855]: cluster 2023-09-23T19:46:12.587371+0000 mon.a (mon.0) 1465 : cluster 1 Cluster is now healthy 2023-09-23T19:46:13.779 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:13 smithi092 bash[58855]: audit 2023-09-23T19:46:12.600558+0000 mon.a (mon.0) 1466 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:46:13.779 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:13 smithi092 bash[58855]: cluster 2023-09-23T19:46:12.600675+0000 mon.a (mon.0) 1467 : cluster 0 osdmap e206: 8 total, 8 up, 8 in 2023-09-23T19:46:13.779 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:13 smithi092 bash[58855]: cluster 2023-09-23T19:46:12.602205+0000 osd.7 (osd.7) 122 : cluster 3 failed to encode map e206 with expected crc 2023-09-23T19:46:13.779 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:13 smithi092 bash[58855]: cluster 2023-09-23T19:46:12.636028+0000 osd.5 (osd.5) 132 : cluster 3 failed to encode map e206 with expected crc 2023-09-23T19:46:13.780 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:13 smithi092 bash[58855]: cluster 2023-09-23T19:46:12.973174+0000 osd.6 (osd.6) 134 : cluster 3 failed to encode map e206 with expected crc 2023-09-23T19:46:13.780 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:13 smithi092 bash[58855]: cluster 2023-09-23T19:46:12.976169+0000 osd.6 (osd.6) 135 : cluster 3 failed to encode map e206 with expected crc 2023-09-23T19:46:13.800 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:13.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:13.802 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:46:13.903 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:13.913 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:13 smithi016 bash[75004]: cluster 2023-09-23T19:46:12.310093+0000 mgr.y (mgr.25325) 1163 : cluster 0 pgmap v1137: 73 pgs: 73 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:46:13.913 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:13 smithi016 bash[75004]: cluster 2023-09-23T19:46:12.587294+0000 mon.a (mon.0) 1464 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:46:13.913 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:13 smithi016 bash[75004]: cluster 2023-09-23T19:46:12.587371+0000 mon.a (mon.0) 1465 : cluster 1 Cluster is now healthy 2023-09-23T19:46:13.913 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:13 smithi016 bash[75004]: audit 2023-09-23T19:46:12.600558+0000 mon.a (mon.0) 1466 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:46:13.914 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:13 smithi016 bash[75004]: cluster 2023-09-23T19:46:12.600675+0000 mon.a (mon.0) 1467 : cluster 0 osdmap e206: 8 total, 8 up, 8 in 2023-09-23T19:46:13.914 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:13 smithi016 bash[75004]: cluster 2023-09-23T19:46:12.602205+0000 osd.7 (osd.7) 122 : cluster 3 failed to encode map e206 with expected crc 2023-09-23T19:46:13.914 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:13 smithi016 bash[75004]: cluster 2023-09-23T19:46:12.636028+0000 osd.5 (osd.5) 132 : cluster 3 failed to encode map e206 with expected crc 2023-09-23T19:46:13.914 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:13 smithi016 bash[75004]: cluster 2023-09-23T19:46:12.973174+0000 osd.6 (osd.6) 134 : cluster 3 failed to encode map e206 with expected crc 2023-09-23T19:46:13.914 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:13 smithi016 bash[75004]: cluster 2023-09-23T19:46:12.976169+0000 osd.6 (osd.6) 135 : cluster 3 failed to encode map e206 with expected crc 2023-09-23T19:46:13.914 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:13 smithi016 bash[77995]: cluster 2023-09-23T19:46:12.310093+0000 mgr.y (mgr.25325) 1163 : cluster 0 pgmap v1137: 73 pgs: 73 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:46:13.914 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:13 smithi016 bash[77995]: cluster 2023-09-23T19:46:12.587294+0000 mon.a (mon.0) 1464 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:46:13.914 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:13 smithi016 bash[77995]: cluster 2023-09-23T19:46:12.587371+0000 mon.a (mon.0) 1465 : cluster 1 Cluster is now healthy 2023-09-23T19:46:13.914 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:13 smithi016 bash[77995]: audit 2023-09-23T19:46:12.600558+0000 mon.a (mon.0) 1466 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:46:13.914 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:13 smithi016 bash[77995]: cluster 2023-09-23T19:46:12.600675+0000 mon.a (mon.0) 1467 : cluster 0 osdmap e206: 8 total, 8 up, 8 in 2023-09-23T19:46:13.915 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:13 smithi016 bash[77995]: cluster 2023-09-23T19:46:12.602205+0000 osd.7 (osd.7) 122 : cluster 3 failed to encode map e206 with expected crc 2023-09-23T19:46:13.915 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:13 smithi016 bash[77995]: cluster 2023-09-23T19:46:12.636028+0000 osd.5 (osd.5) 132 : cluster 3 failed to encode map e206 with expected crc 2023-09-23T19:46:13.915 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:13 smithi016 bash[77995]: cluster 2023-09-23T19:46:12.973174+0000 osd.6 (osd.6) 134 : cluster 3 failed to encode map e206 with expected crc 2023-09-23T19:46:13.915 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:13 smithi016 bash[77995]: cluster 2023-09-23T19:46:12.976169+0000 osd.6 (osd.6) 135 : cluster 3 failed to encode map e206 with expected crc 2023-09-23T19:46:13.931 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:13.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:13.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:14.035 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:14.061 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:14.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:14.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:14.165 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:14.200 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:14.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:14.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:14.304 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:14.333 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:14.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:14.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:14.437 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:14.468 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:14.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:14.471 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:46:14.571 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:14.601 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:14.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:14.604 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:46:14.705 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:14.717 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:14 smithi092 bash[58855]: audit 2023-09-23T19:46:13.670069+0000 mon.a (mon.0) 1468 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:46:14.717 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:14 smithi092 bash[58855]: audit 2023-09-23T19:46:13.671214+0000 mon.a (mon.0) 1469 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:46:14.734 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:14.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:14.737 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:46:14.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:14 smithi016 bash[77995]: audit 2023-09-23T19:46:13.670069+0000 mon.a (mon.0) 1468 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:46:14.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:14 smithi016 bash[77995]: audit 2023-09-23T19:46:13.671214+0000 mon.a (mon.0) 1469 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:46:14.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:14 smithi016 bash[75004]: audit 2023-09-23T19:46:13.670069+0000 mon.a (mon.0) 1468 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:46:14.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:14 smithi016 bash[75004]: audit 2023-09-23T19:46:13.671214+0000 mon.a (mon.0) 1469 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:46:14.838 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:14.865 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:14.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:14.868 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:46:14.969 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:15.004 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:15.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:15.007 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:46:15.108 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:15.137 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:15.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:15.139 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:46:15.240 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:15.278 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:15.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:15.281 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:15.381 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:15.414 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:15.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:15.417 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:46:15.518 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:15.549 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:15.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:15.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:15.653 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:15.665 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:46:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:46:15] "GET /metrics HTTP/1.1" 200 33888 "" "Prometheus/2.33.4" 2023-09-23T19:46:15.684 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:15.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:15.688 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:15.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:15 smithi092 bash[58855]: cluster 2023-09-23T19:46:14.310850+0000 mgr.y (mgr.25325) 1164 : cluster 0 pgmap v1139: 73 pgs: 73 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:46:15.790 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:15.811 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T19:46:15.820 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:15.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:15.824 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:15.924 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:15.956 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:15.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:15.959 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:16.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:15 smithi016 bash[75004]: cluster 2023-09-23T19:46:14.310850+0000 mgr.y (mgr.25325) 1164 : cluster 0 pgmap v1139: 73 pgs: 73 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:46:16.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:15 smithi016 bash[77995]: cluster 2023-09-23T19:46:14.310850+0000 mgr.y (mgr.25325) 1164 : cluster 0 pgmap v1139: 73 pgs: 73 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:46:16.060 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:16.120 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:16.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:16.123 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:16.224 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:16.258 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:16.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:16.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:16.361 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:16.395 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:16.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:16.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:16.498 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:16.528 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:16.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:16.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:46:16.632 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:16.663 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:16.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:16.667 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:16.707 INFO:teuthology.orchestra.run.smithi016.stderr:pool 'unique_pool_7' created 2023-09-23T19:46:16.768 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:16.799 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:46:16.801 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:16.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:16.804 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:16.905 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:16.917 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:16 smithi016 bash[77995]: audit 2023-09-23T19:46:16.158534+0000 mon.c (mon.2) 193 : audit 1 from='client.? 172.21.15.16:0/1679329218' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]: dispatch 2023-09-23T19:46:16.917 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:16 smithi016 bash[77995]: audit 2023-09-23T19:46:16.159169+0000 mon.a (mon.0) 1470 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]: dispatch 2023-09-23T19:46:16.918 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:16 smithi016 bash[75004]: audit 2023-09-23T19:46:16.158534+0000 mon.c (mon.2) 193 : audit 1 from='client.? 172.21.15.16:0/1679329218' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]: dispatch 2023-09-23T19:46:16.918 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:16 smithi016 bash[75004]: audit 2023-09-23T19:46:16.159169+0000 mon.a (mon.0) 1470 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]: dispatch 2023-09-23T19:46:16.935 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:16.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:16.938 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:46:17.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:16 smithi092 bash[58855]: audit 2023-09-23T19:46:16.158534+0000 mon.c (mon.2) 193 : audit 1 from='client.? 172.21.15.16:0/1679329218' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]: dispatch 2023-09-23T19:46:17.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:16 smithi092 bash[58855]: audit 2023-09-23T19:46:16.159169+0000 mon.a (mon.0) 1470 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]: dispatch 2023-09-23T19:46:17.038 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:17.064 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:17.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:17.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:46:17.167 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:17.192 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:17.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:17.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:46:17.296 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:17.317 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:17.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:17.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:46:17.420 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:17.450 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:17.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:17.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:46:17.554 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:17.584 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:17.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:17.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:17.688 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:17.720 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:17.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:17.726 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:46:17.827 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:17.836 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:17 smithi016 bash[75004]: cluster 2023-09-23T19:46:16.311836+0000 mgr.y (mgr.25325) 1165 : cluster 0 pgmap v1140: 73 pgs: 73 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:46:17.836 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:17 smithi016 bash[75004]: audit 2023-09-23T19:46:16.691751+0000 mon.a (mon.0) 1471 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]': finished 2023-09-23T19:46:17.836 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:17 smithi016 bash[75004]: cluster 2023-09-23T19:46:16.691859+0000 mon.a (mon.0) 1472 : cluster 0 osdmap e207: 8 total, 8 up, 8 in 2023-09-23T19:46:17.836 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:17 smithi016 bash[75004]: cluster 2023-09-23T19:46:16.695269+0000 osd.7 (osd.7) 123 : cluster 3 failed to encode map e207 with expected crc 2023-09-23T19:46:17.836 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:17 smithi016 bash[75004]: cluster 2023-09-23T19:46:16.698062+0000 osd.7 (osd.7) 124 : cluster 3 failed to encode map e207 with expected crc 2023-09-23T19:46:17.836 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:17 smithi016 bash[75004]: cluster 2023-09-23T19:46:16.703450+0000 osd.4 (osd.4) 150 : cluster 3 failed to encode map e207 with expected crc 2023-09-23T19:46:17.837 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:17 smithi016 bash[75004]: cluster 2023-09-23T19:46:16.703826+0000 osd.5 (osd.5) 133 : cluster 3 failed to encode map e207 with expected crc 2023-09-23T19:46:17.837 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:17 smithi016 bash[75004]: cluster 2023-09-23T19:46:16.703997+0000 osd.6 (osd.6) 136 : cluster 3 failed to encode map e207 with expected crc 2023-09-23T19:46:17.837 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:17 smithi016 bash[77995]: cluster 2023-09-23T19:46:16.311836+0000 mgr.y (mgr.25325) 1165 : cluster 0 pgmap v1140: 73 pgs: 73 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:46:17.837 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:17 smithi016 bash[77995]: audit 2023-09-23T19:46:16.691751+0000 mon.a (mon.0) 1471 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]': finished 2023-09-23T19:46:17.837 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:17 smithi016 bash[77995]: cluster 2023-09-23T19:46:16.691859+0000 mon.a (mon.0) 1472 : cluster 0 osdmap e207: 8 total, 8 up, 8 in 2023-09-23T19:46:17.837 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:17 smithi016 bash[77995]: cluster 2023-09-23T19:46:16.695269+0000 osd.7 (osd.7) 123 : cluster 3 failed to encode map e207 with expected crc 2023-09-23T19:46:17.837 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:17 smithi016 bash[77995]: cluster 2023-09-23T19:46:16.698062+0000 osd.7 (osd.7) 124 : cluster 3 failed to encode map e207 with expected crc 2023-09-23T19:46:17.837 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:17 smithi016 bash[77995]: cluster 2023-09-23T19:46:16.703450+0000 osd.4 (osd.4) 150 : cluster 3 failed to encode map e207 with expected crc 2023-09-23T19:46:17.838 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:17 smithi016 bash[77995]: cluster 2023-09-23T19:46:16.703826+0000 osd.5 (osd.5) 133 : cluster 3 failed to encode map e207 with expected crc 2023-09-23T19:46:17.838 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:17 smithi016 bash[77995]: cluster 2023-09-23T19:46:16.703997+0000 osd.6 (osd.6) 136 : cluster 3 failed to encode map e207 with expected crc 2023-09-23T19:46:17.839 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_7 rados --yes-i-really-mean-it || true 2023-09-23T19:46:17.852 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:17.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:17.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:17.956 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:17.989 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:17.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:17.992 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:18.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:17 smithi092 bash[58855]: cluster 2023-09-23T19:46:16.311836+0000 mgr.y (mgr.25325) 1165 : cluster 0 pgmap v1140: 73 pgs: 73 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:46:18.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:17 smithi092 bash[58855]: audit 2023-09-23T19:46:16.691751+0000 mon.a (mon.0) 1471 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]': finished 2023-09-23T19:46:18.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:17 smithi092 bash[58855]: cluster 2023-09-23T19:46:16.691859+0000 mon.a (mon.0) 1472 : cluster 0 osdmap e207: 8 total, 8 up, 8 in 2023-09-23T19:46:18.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:17 smithi092 bash[58855]: cluster 2023-09-23T19:46:16.695269+0000 osd.7 (osd.7) 123 : cluster 3 failed to encode map e207 with expected crc 2023-09-23T19:46:18.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:17 smithi092 bash[58855]: cluster 2023-09-23T19:46:16.698062+0000 osd.7 (osd.7) 124 : cluster 3 failed to encode map e207 with expected crc 2023-09-23T19:46:18.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:17 smithi092 bash[58855]: cluster 2023-09-23T19:46:16.703450+0000 osd.4 (osd.4) 150 : cluster 3 failed to encode map e207 with expected crc 2023-09-23T19:46:18.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:17 smithi092 bash[58855]: cluster 2023-09-23T19:46:16.703826+0000 osd.5 (osd.5) 133 : cluster 3 failed to encode map e207 with expected crc 2023-09-23T19:46:18.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:17 smithi092 bash[58855]: cluster 2023-09-23T19:46:16.703997+0000 osd.6 (osd.6) 136 : cluster 3 failed to encode map e207 with expected crc 2023-09-23T19:46:18.093 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:18.119 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:18.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:18.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:18.222 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:18.253 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:18.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:18.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:46:18.259 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:46:18.260 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:46:18.360 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:18.387 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:18.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:18.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:46:18.490 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:18.522 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:18.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:18.525 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:46:18.625 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:18.657 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:18.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:18.659 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:46:18.760 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:18.791 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:18.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:18.795 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:46:18.896 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:18.924 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:18.927 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:18 smithi092 bash[58855]: cluster 2023-09-23T19:46:17.700462+0000 mon.a (mon.0) 1473 : cluster 0 osdmap e208: 8 total, 8 up, 8 in 2023-09-23T19:46:18.927 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:18 smithi092 bash[58855]: cluster 2023-09-23T19:46:17.705127+0000 osd.7 (osd.7) 125 : cluster 3 failed to encode map e208 with expected crc 2023-09-23T19:46:18.927 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:18 smithi092 bash[58855]: cluster 2023-09-23T19:46:17.706247+0000 osd.6 (osd.6) 137 : cluster 3 failed to encode map e208 with expected crc 2023-09-23T19:46:18.927 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:18 smithi092 bash[58855]: cluster 2023-09-23T19:46:17.706823+0000 osd.4 (osd.4) 151 : cluster 3 failed to encode map e208 with expected crc 2023-09-23T19:46:18.927 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:18 smithi092 bash[58855]: cluster 2023-09-23T19:46:17.889350+0000 osd.5 (osd.5) 134 : cluster 3 failed to encode map e208 with expected crc 2023-09-23T19:46:18.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:18.928 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:19.029 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:19.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:18 smithi016 bash[75004]: cluster 2023-09-23T19:46:17.700462+0000 mon.a (mon.0) 1473 : cluster 0 osdmap e208: 8 total, 8 up, 8 in 2023-09-23T19:46:19.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:18 smithi016 bash[75004]: cluster 2023-09-23T19:46:17.705127+0000 osd.7 (osd.7) 125 : cluster 3 failed to encode map e208 with expected crc 2023-09-23T19:46:19.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:18 smithi016 bash[75004]: cluster 2023-09-23T19:46:17.706247+0000 osd.6 (osd.6) 137 : cluster 3 failed to encode map e208 with expected crc 2023-09-23T19:46:19.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:18 smithi016 bash[75004]: cluster 2023-09-23T19:46:17.706823+0000 osd.4 (osd.4) 151 : cluster 3 failed to encode map e208 with expected crc 2023-09-23T19:46:19.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:18 smithi016 bash[75004]: cluster 2023-09-23T19:46:17.889350+0000 osd.5 (osd.5) 134 : cluster 3 failed to encode map e208 with expected crc 2023-09-23T19:46:19.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:18 smithi016 bash[77995]: cluster 2023-09-23T19:46:17.700462+0000 mon.a (mon.0) 1473 : cluster 0 osdmap e208: 8 total, 8 up, 8 in 2023-09-23T19:46:19.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:18 smithi016 bash[77995]: cluster 2023-09-23T19:46:17.705127+0000 osd.7 (osd.7) 125 : cluster 3 failed to encode map e208 with expected crc 2023-09-23T19:46:19.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:18 smithi016 bash[77995]: cluster 2023-09-23T19:46:17.706247+0000 osd.6 (osd.6) 137 : cluster 3 failed to encode map e208 with expected crc 2023-09-23T19:46:19.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:18 smithi016 bash[77995]: cluster 2023-09-23T19:46:17.706823+0000 osd.4 (osd.4) 151 : cluster 3 failed to encode map e208 with expected crc 2023-09-23T19:46:19.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:18 smithi016 bash[77995]: cluster 2023-09-23T19:46:17.889350+0000 osd.5 (osd.5) 134 : cluster 3 failed to encode map e208 with expected crc 2023-09-23T19:46:19.059 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:19.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:19.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:19.163 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:19.194 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:19.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:19.197 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:46:19.298 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:19.320 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:19.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:19.322 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:19.423 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:19.450 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:19.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:19.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:19.553 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:19.585 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:19.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:19.588 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:19.689 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:19.725 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:19.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:19.728 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:19.829 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:19.862 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:19.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:19.865 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:19.966 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:19.987 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:19 smithi092 bash[58855]: cluster 2023-09-23T19:46:18.312669+0000 mgr.y (mgr.25325) 1166 : cluster 0 pgmap v1143: 89 pgs: 4 creating+activating, 12 unknown, 73 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:46:19.987 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:19 smithi092 bash[58855]: cluster 2023-09-23T19:46:18.713778+0000 mon.a (mon.0) 1474 : cluster 0 osdmap e209: 8 total, 8 up, 8 in 2023-09-23T19:46:19.987 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:19 smithi092 bash[58855]: cluster 2023-09-23T19:46:19.374311+0000 osd.6 (osd.6) 138 : cluster 3 failed to encode map e209 with expected crc 2023-09-23T19:46:19.987 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:19 smithi092 bash[58855]: cluster 2023-09-23T19:46:19.375347+0000 osd.6 (osd.6) 139 : cluster 3 failed to encode map e209 with expected crc 2023-09-23T19:46:20.007 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:20.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:20.011 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:20.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:19 smithi016 bash[75004]: cluster 2023-09-23T19:46:18.312669+0000 mgr.y (mgr.25325) 1166 : cluster 0 pgmap v1143: 89 pgs: 4 creating+activating, 12 unknown, 73 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:46:20.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:19 smithi016 bash[75004]: cluster 2023-09-23T19:46:18.713778+0000 mon.a (mon.0) 1474 : cluster 0 osdmap e209: 8 total, 8 up, 8 in 2023-09-23T19:46:20.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:19 smithi016 bash[75004]: cluster 2023-09-23T19:46:19.374311+0000 osd.6 (osd.6) 138 : cluster 3 failed to encode map e209 with expected crc 2023-09-23T19:46:20.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:19 smithi016 bash[75004]: cluster 2023-09-23T19:46:19.375347+0000 osd.6 (osd.6) 139 : cluster 3 failed to encode map e209 with expected crc 2023-09-23T19:46:20.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:19 smithi016 bash[77995]: cluster 2023-09-23T19:46:18.312669+0000 mgr.y (mgr.25325) 1166 : cluster 0 pgmap v1143: 89 pgs: 4 creating+activating, 12 unknown, 73 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:46:20.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:19 smithi016 bash[77995]: cluster 2023-09-23T19:46:18.713778+0000 mon.a (mon.0) 1474 : cluster 0 osdmap e209: 8 total, 8 up, 8 in 2023-09-23T19:46:20.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:19 smithi016 bash[77995]: cluster 2023-09-23T19:46:19.374311+0000 osd.6 (osd.6) 138 : cluster 3 failed to encode map e209 with expected crc 2023-09-23T19:46:20.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:19 smithi016 bash[77995]: cluster 2023-09-23T19:46:19.375347+0000 osd.6 (osd.6) 139 : cluster 3 failed to encode map e209 with expected crc 2023-09-23T19:46:20.114 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:20.139 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:20.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:20.141 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:46:20.242 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:20.274 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:20.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:20.277 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:20.378 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:20.404 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:20.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:20.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:46:20.507 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:20.519 INFO:teuthology.orchestra.run.smithi092.stdout:ERROR: (22) Invalid argument 2023-09-23T19:46:20.519 INFO:teuthology.orchestra.run.smithi092.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:46:20.534 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:20.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:20.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:20.638 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:20.673 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:20.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:20.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:20.776 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:20.803 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:20.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:20.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:46:20.906 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:20.917 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:20 smithi092 bash[58855]: cluster 2023-09-23T19:46:19.705415+0000 osd.4 (osd.4) 152 : cluster 3 failed to encode map e209 with expected crc 2023-09-23T19:46:20.917 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:20 smithi092 bash[58855]: cluster 2023-09-23T19:46:19.705541+0000 osd.5 (osd.5) 135 : cluster 3 failed to encode map e209 with expected crc 2023-09-23T19:46:20.918 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:20 smithi092 bash[58855]: cluster 2023-09-23T19:46:19.705770+0000 osd.7 (osd.7) 126 : cluster 3 failed to encode map e209 with expected crc 2023-09-23T19:46:20.934 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:20.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:20.937 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:21.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:20 smithi016 bash[75004]: cluster 2023-09-23T19:46:19.705415+0000 osd.4 (osd.4) 152 : cluster 3 failed to encode map e209 with expected crc 2023-09-23T19:46:21.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:20 smithi016 bash[75004]: cluster 2023-09-23T19:46:19.705541+0000 osd.5 (osd.5) 135 : cluster 3 failed to encode map e209 with expected crc 2023-09-23T19:46:21.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:20 smithi016 bash[75004]: cluster 2023-09-23T19:46:19.705770+0000 osd.7 (osd.7) 126 : cluster 3 failed to encode map e209 with expected crc 2023-09-23T19:46:21.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:20 smithi016 bash[77995]: cluster 2023-09-23T19:46:19.705415+0000 osd.4 (osd.4) 152 : cluster 3 failed to encode map e209 with expected crc 2023-09-23T19:46:21.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:20 smithi016 bash[77995]: cluster 2023-09-23T19:46:19.705541+0000 osd.5 (osd.5) 135 : cluster 3 failed to encode map e209 with expected crc 2023-09-23T19:46:21.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:20 smithi016 bash[77995]: cluster 2023-09-23T19:46:19.705770+0000 osd.7 (osd.7) 126 : cluster 3 failed to encode map e209 with expected crc 2023-09-23T19:46:21.039 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:21.073 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:21.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:21.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:21.177 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:21.210 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:21.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:21.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:21.314 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:21.350 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:21.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:21.353 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:21.454 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:21.468 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T19:46:21.487 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:21.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:21.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:46:21.591 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:21.625 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:21.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:21.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:21.729 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:21.755 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:21.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:21.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:46:21.859 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:21.870 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:21 smithi016 bash[75004]: cluster 2023-09-23T19:46:20.313292+0000 mgr.y (mgr.25325) 1167 : cluster 0 pgmap v1145: 89 pgs: 4 creating+activating, 2 unknown, 83 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:46:21.870 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:21 smithi016 bash[77995]: cluster 2023-09-23T19:46:20.313292+0000 mgr.y (mgr.25325) 1167 : cluster 0 pgmap v1145: 89 pgs: 4 creating+activating, 2 unknown, 83 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:46:21.887 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:21.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:21.891 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:46:21.992 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:22.006 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:21 smithi092 bash[58855]: cluster 2023-09-23T19:46:20.313292+0000 mgr.y (mgr.25325) 1167 : cluster 0 pgmap v1145: 89 pgs: 4 creating+activating, 2 unknown, 83 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:46:22.028 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:22.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:22.032 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:22.132 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:22.162 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:22.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:22.165 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:22.266 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:22.294 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:22.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:22.297 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:46:22.350 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:46:22.351 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:46:22.398 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:22.429 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:22.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:22.431 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:22.532 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:22.561 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:22.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:22.564 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:22.665 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:22.696 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:22.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:22.698 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:22.799 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:22.830 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:22.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:22.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:46:22.933 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:22.968 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:22.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:22.971 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:46:23.072 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:23.104 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:23.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:23.107 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:46:23.207 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:23.238 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:23.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:23.242 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:46:23.342 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:23.377 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:23.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:23.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:23.480 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:23.513 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:23.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:23.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:46:23.616 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:23.642 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:23.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:23.644 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:46:23.745 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:23.755 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:23 smithi092 bash[58855]: cluster 2023-09-23T19:46:22.314060+0000 mgr.y (mgr.25325) 1168 : cluster 0 pgmap v1146: 89 pgs: 4 creating+activating, 85 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:46:23.782 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:23.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:23.785 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:46:23.885 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:23.918 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:23.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:23.921 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:46:24.022 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:24.030 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:23 smithi016 bash[77995]: cluster 2023-09-23T19:46:22.314060+0000 mgr.y (mgr.25325) 1168 : cluster 0 pgmap v1146: 89 pgs: 4 creating+activating, 85 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:46:24.030 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:23 smithi016 bash[75004]: cluster 2023-09-23T19:46:22.314060+0000 mgr.y (mgr.25325) 1168 : cluster 0 pgmap v1146: 89 pgs: 4 creating+activating, 85 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:46:24.051 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:24.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:24.054 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:46:24.155 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:24.185 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:24.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:24.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:24.289 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:24.321 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:24.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:24.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:24.425 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:24.459 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:24.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:24.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:46:24.479 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:46:24.507 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:46:24.530 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:46:24.556 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:46:24.562 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:24.583 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:24.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:24.585 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:24.587 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:46:24.613 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:46:24.638 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:46:24.664 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:46:24.666 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:46:24.666 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:46:24.666 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:46:24.666 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:46:24.666 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:46:24.666 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:46:24.667 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:46:24.667 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:46:24.686 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:24.708 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:24.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:24.711 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:24.811 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:24.821 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:24 smithi016 bash[77995]: audit 2023-09-23T19:46:24.447715+0000 mon.c (mon.2) 194 : audit 0 from='client.? 172.21.15.16:0/2431418228' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:46:24.821 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:24 smithi016 bash[75004]: audit 2023-09-23T19:46:24.447715+0000 mon.c (mon.2) 194 : audit 0 from='client.? 172.21.15.16:0/2431418228' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:46:24.836 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:24.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:24.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:46:24.940 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:24.970 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:24.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:24.973 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:46:25.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:24 smithi092 bash[58855]: audit 2023-09-23T19:46:24.447715+0000 mon.c (mon.2) 194 : audit 0 from='client.? 172.21.15.16:0/2431418228' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:46:25.074 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:25.105 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:25.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:25.108 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:46:25.209 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:25.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:25.230 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:25.230 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:46:25.331 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:25.365 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:25.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:25.368 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:25.468 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:25.503 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:25.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:25.506 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:25.607 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:25.630 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:25.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:25.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:25.734 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:25.757 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T19:46:25.758 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:25.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:25.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:46:25.812 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:25 smithi016 bash[77995]: cluster 2023-09-23T19:46:24.314988+0000 mgr.y (mgr.25325) 1169 : cluster 0 pgmap v1147: 89 pgs: 89 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:46:25.812 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:25 smithi016 bash[77995]: audit 2023-09-23T19:46:25.211449+0000 mon.a (mon.0) 1475 : audit 1 from='client.? 172.21.15.16:0/3079084050' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T19:46:25.813 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:46:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:46:25] "GET /metrics HTTP/1.1" 200 33888 "" "Prometheus/2.33.4" 2023-09-23T19:46:25.813 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:25 smithi016 bash[75004]: cluster 2023-09-23T19:46:24.314988+0000 mgr.y (mgr.25325) 1169 : cluster 0 pgmap v1147: 89 pgs: 89 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:46:25.813 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:25 smithi016 bash[75004]: audit 2023-09-23T19:46:25.211449+0000 mon.a (mon.0) 1475 : audit 1 from='client.? 172.21.15.16:0/3079084050' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T19:46:25.862 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:25.893 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:25.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:25.896 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:25.996 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:26.028 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:26.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:26.031 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:26.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:25 smithi092 bash[58855]: cluster 2023-09-23T19:46:24.314988+0000 mgr.y (mgr.25325) 1169 : cluster 0 pgmap v1147: 89 pgs: 89 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:46:26.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:25 smithi092 bash[58855]: audit 2023-09-23T19:46:25.211449+0000 mon.a (mon.0) 1475 : audit 1 from='client.? 172.21.15.16:0/3079084050' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T19:46:26.131 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:26.158 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:26.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:26.161 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:46:26.252 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:46:26.262 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:26.292 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:26.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:26.294 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:46:26.395 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:26.417 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:26.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:26.419 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:26.520 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:26.550 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:26.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:26.553 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:26.654 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:26.677 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:26.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:26.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:26.780 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:26.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:26 smithi092 bash[58855]: audit 2023-09-23T19:46:25.748298+0000 mon.a (mon.0) 1476 : audit 1 from='client.? 172.21.15.16:0/3079084050' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T19:46:26.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:26 smithi092 bash[58855]: cluster 2023-09-23T19:46:25.748452+0000 mon.a (mon.0) 1477 : cluster 0 osdmap e210: 8 total, 8 up, 8 in 2023-09-23T19:46:26.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:26 smithi092 bash[58855]: cluster 2023-09-23T19:46:25.832479+0000 osd.7 (osd.7) 127 : cluster 3 failed to encode map e210 with expected crc 2023-09-23T19:46:26.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:26 smithi092 bash[58855]: cluster 2023-09-23T19:46:26.059885+0000 osd.4 (osd.4) 153 : cluster 3 failed to encode map e210 with expected crc 2023-09-23T19:46:26.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:26 smithi092 bash[58855]: cluster 2023-09-23T19:46:26.062788+0000 osd.4 (osd.4) 154 : cluster 3 failed to encode map e210 with expected crc 2023-09-23T19:46:26.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:26 smithi092 bash[58855]: cluster 2023-09-23T19:46:26.302088+0000 osd.6 (osd.6) 140 : cluster 3 failed to encode map e210 with expected crc 2023-09-23T19:46:26.809 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:26.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:26.812 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:46:26.913 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:26.940 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:26.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:26.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:27.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:26 smithi016 bash[75004]: audit 2023-09-23T19:46:25.748298+0000 mon.a (mon.0) 1476 : audit 1 from='client.? 172.21.15.16:0/3079084050' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T19:46:27.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:26 smithi016 bash[75004]: cluster 2023-09-23T19:46:25.748452+0000 mon.a (mon.0) 1477 : cluster 0 osdmap e210: 8 total, 8 up, 8 in 2023-09-23T19:46:27.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:26 smithi016 bash[75004]: cluster 2023-09-23T19:46:25.832479+0000 osd.7 (osd.7) 127 : cluster 3 failed to encode map e210 with expected crc 2023-09-23T19:46:27.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:26 smithi016 bash[75004]: cluster 2023-09-23T19:46:26.059885+0000 osd.4 (osd.4) 153 : cluster 3 failed to encode map e210 with expected crc 2023-09-23T19:46:27.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:26 smithi016 bash[75004]: cluster 2023-09-23T19:46:26.062788+0000 osd.4 (osd.4) 154 : cluster 3 failed to encode map e210 with expected crc 2023-09-23T19:46:27.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:26 smithi016 bash[75004]: cluster 2023-09-23T19:46:26.302088+0000 osd.6 (osd.6) 140 : cluster 3 failed to encode map e210 with expected crc 2023-09-23T19:46:27.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:26 smithi016 bash[77995]: audit 2023-09-23T19:46:25.748298+0000 mon.a (mon.0) 1476 : audit 1 from='client.? 172.21.15.16:0/3079084050' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T19:46:27.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:26 smithi016 bash[77995]: cluster 2023-09-23T19:46:25.748452+0000 mon.a (mon.0) 1477 : cluster 0 osdmap e210: 8 total, 8 up, 8 in 2023-09-23T19:46:27.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:26 smithi016 bash[77995]: cluster 2023-09-23T19:46:25.832479+0000 osd.7 (osd.7) 127 : cluster 3 failed to encode map e210 with expected crc 2023-09-23T19:46:27.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:26 smithi016 bash[77995]: cluster 2023-09-23T19:46:26.059885+0000 osd.4 (osd.4) 153 : cluster 3 failed to encode map e210 with expected crc 2023-09-23T19:46:27.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:26 smithi016 bash[77995]: cluster 2023-09-23T19:46:26.062788+0000 osd.4 (osd.4) 154 : cluster 3 failed to encode map e210 with expected crc 2023-09-23T19:46:27.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:26 smithi016 bash[77995]: cluster 2023-09-23T19:46:26.302088+0000 osd.6 (osd.6) 140 : cluster 3 failed to encode map e210 with expected crc 2023-09-23T19:46:27.044 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:27.072 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:27.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:27.074 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:27.175 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:27.203 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:27.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:27.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:27.308 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:27.337 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:27.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:27.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:46:27.441 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:27.475 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:27.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:27.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:27.578 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:27.604 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:27.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:27.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:27.708 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:27.739 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:27.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:27.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:27.843 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:27.872 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:27.875 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:27 smithi092 bash[58855]: cluster 2023-09-23T19:46:26.315814+0000 mgr.y (mgr.25325) 1170 : cluster 0 pgmap v1149: 89 pgs: 89 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:46:27.875 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:27 smithi092 bash[58855]: cluster 2023-09-23T19:46:26.649433+0000 osd.5 (osd.5) 136 : cluster 3 failed to encode map e210 with expected crc 2023-09-23T19:46:27.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:27.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:27.977 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:28.011 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:28.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:28.014 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:28.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:27 smithi016 bash[75004]: cluster 2023-09-23T19:46:26.315814+0000 mgr.y (mgr.25325) 1170 : cluster 0 pgmap v1149: 89 pgs: 89 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:46:28.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:27 smithi016 bash[75004]: cluster 2023-09-23T19:46:26.649433+0000 osd.5 (osd.5) 136 : cluster 3 failed to encode map e210 with expected crc 2023-09-23T19:46:28.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:27 smithi016 bash[77995]: cluster 2023-09-23T19:46:26.315814+0000 mgr.y (mgr.25325) 1170 : cluster 0 pgmap v1149: 89 pgs: 89 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:46:28.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:27 smithi016 bash[77995]: cluster 2023-09-23T19:46:26.649433+0000 osd.5 (osd.5) 136 : cluster 3 failed to encode map e210 with expected crc 2023-09-23T19:46:28.114 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:28.139 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:28.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:28.142 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:46:28.243 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:28.269 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:28.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:28.272 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:46:28.306 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-23T19:46:28.372 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:28.417 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:28.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:28.419 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:46:28.520 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:28.543 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:28.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:28.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:46:28.646 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:28.672 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:28.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:28.674 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:46:28.775 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:28.800 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:28.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:28.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:46:28.845 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:46:28.903 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:28.933 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:28.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:28.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:46:29.036 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:29.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:28 smithi016 bash[75004]: audit 2023-09-23T19:46:28.666915+0000 mon.a (mon.0) 1478 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:46:29.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:28 smithi016 bash[77995]: audit 2023-09-23T19:46:28.666915+0000 mon.a (mon.0) 1478 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:46:29.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:28 smithi092 bash[58855]: audit 2023-09-23T19:46:28.666915+0000 mon.a (mon.0) 1478 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:46:29.066 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:29.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:29.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:29.170 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:29.192 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:29.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:29.194 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:46:29.295 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:29.316 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:29.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:29.318 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:29.419 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:29.448 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:29.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:29.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:46:29.551 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:29.578 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:29.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:29.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:46:29.681 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:29.713 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:29.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:29.717 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:46:29.817 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:29.834 INFO:teuthology.orchestra.run.smithi092.stdout:{ 2023-09-23T19:46:29.835 INFO:teuthology.orchestra.run.smithi092.stdout: "size": 20, 2023-09-23T19:46:29.835 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 600, 2023-09-23T19:46:29.835 INFO:teuthology.orchestra.run.smithi092.stdout: "ops": [ 2023-09-23T19:46:29.835 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.835 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45947.0:116715 10.9 10:959e17e7:::benchmark_data_smithi016_136365_object6581:head [delete] snapc 0=[] ondisk+write+known_if_redirected e154)", 2023-09-23T19:46:29.835 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:38:13.661306+0000", 2023-09-23T19:46:29.835 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 496.17009082999999, 2023-09-23T19:46:29.835 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.019912347, 2023-09-23T19:46:29.835 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:46:29.835 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "reached pg", 2023-09-23T19:46:29.836 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:46:29.836 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45947", 2023-09-23T19:46:29.836 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/4039191752", 2023-09-23T19:46:29.836 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 116715 2023-09-23T19:46:29.836 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.836 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:46:29.836 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.836 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:46:29.836 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.661306+0000", 2023-09-23T19:46:29.836 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:46:29.836 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.837 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.837 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:46:29.837 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.661306+0000", 2023-09-23T19:46:29.837 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T19:46:29.837 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.837 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.837 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:46:29.837 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.661304+0000", 2023-09-23T19:46:29.837 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.6919999999999997e-06 2023-09-23T19:46:29.837 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.837 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.837 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:46:29.838 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.661314+0000", 2023-09-23T19:46:29.838 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.5917999999999999e-05 2023-09-23T19:46:29.838 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.838 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.838 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:46:29.838 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.661330+0000", 2023-09-23T19:46:29.838 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.5140000000000001e-06 2023-09-23T19:46:29.838 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.838 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.838 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:46:29.838 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.661338+0000", 2023-09-23T19:46:29.838 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.4846000000000003e-05 2023-09-23T19:46:29.839 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.839 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.839 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:46:29.839 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.661393+0000", 2023-09-23T19:46:29.839 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00016160600000000001 2023-09-23T19:46:29.839 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.839 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.839 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:46:29.839 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.661555+0000", 2023-09-23T19:46:29.839 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000307965 2023-09-23T19:46:29.840 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.840 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.840 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 2,7", 2023-09-23T19:46:29.840 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.661863+0000", 2023-09-23T19:46:29.840 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0096936660000000001 2023-09-23T19:46:29.840 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.840 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.840 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:46:29.840 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.671556+0000", 2023-09-23T19:46:29.840 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0054527810000000003 2023-09-23T19:46:29.840 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.840 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.841 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:46:29.841 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.677009+0000", 2023-09-23T19:46:29.841 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.8539999999999992e-06 2023-09-23T19:46:29.841 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.841 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.841 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for peered", 2023-09-23T19:46:29.841 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.677019+0000", 2023-09-23T19:46:29.841 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.0041209550000001 2023-09-23T19:46:29.841 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.841 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.841 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:46:29.841 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:14.681140+0000", 2023-09-23T19:46:29.842 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.8568999999999999e-05 2023-09-23T19:46:29.842 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.842 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.842 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:46:29.842 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:14.681219+0000", 2023-09-23T19:46:29.842 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.019912347 2023-09-23T19:46:29.842 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:29.842 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:46:29.842 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:29.842 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.842 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.843 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45947.0:116729 10.3 10:c1afa68f:::benchmark_data_smithi016_136365_object6595:head [delete] snapc 0=[] ondisk+write+known_if_redirected e155)", 2023-09-23T19:46:29.843 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:38:13.685076+0000", 2023-09-23T19:46:29.843 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 496.14632079900002, 2023-09-23T19:46:29.843 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.0362053280000001, 2023-09-23T19:46:29.843 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:46:29.843 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:46:29.843 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:46:29.843 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45947", 2023-09-23T19:46:29.843 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/4039191752", 2023-09-23T19:46:29.843 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 116729 2023-09-23T19:46:29.843 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.843 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:46:29.843 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.844 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:46:29.844 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.685076+0000", 2023-09-23T19:46:29.844 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:46:29.844 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.844 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.844 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:46:29.844 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.685076+0000", 2023-09-23T19:46:29.844 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T19:46:29.844 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.844 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.844 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:46:29.844 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.685074+0000", 2023-09-23T19:46:29.845 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.7159999999999992e-06 2023-09-23T19:46:29.845 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.845 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.845 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:46:29.845 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.685083+0000", 2023-09-23T19:46:29.845 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.2413e-05 2023-09-23T19:46:29.845 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.845 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.845 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:46:29.845 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.685096+0000", 2023-09-23T19:46:29.845 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.6299999999999998e-06 2023-09-23T19:46:29.845 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.846 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.846 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:46:29.846 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.685103+0000", 2023-09-23T19:46:29.846 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00011823 2023-09-23T19:46:29.846 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.846 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.846 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:46:29.846 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.685221+0000", 2023-09-23T19:46:29.846 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.054e-06 2023-09-23T19:46:29.846 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.846 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.847 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for peered", 2023-09-23T19:46:29.847 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.685231+0000", 2023-09-23T19:46:29.847 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.99700242900000002 2023-09-23T19:46:29.847 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.847 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.847 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:46:29.847 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:14.682233+0000", 2023-09-23T19:46:29.847 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0040735809999999997 2023-09-23T19:46:29.847 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.847 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.847 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:46:29.847 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:14.686307+0000", 2023-09-23T19:46:29.847 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00014805299999999999 2023-09-23T19:46:29.848 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.848 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.848 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3", 2023-09-23T19:46:29.848 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:14.686455+0000", 2023-09-23T19:46:29.848 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0048098660000000003 2023-09-23T19:46:29.848 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.848 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.848 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:29.848 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:14.691264+0000", 2023-09-23T19:46:29.848 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.029862791 2023-09-23T19:46:29.848 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.848 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.849 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:46:29.849 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:14.721127+0000", 2023-09-23T19:46:29.849 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.3599e-05 2023-09-23T19:46:29.849 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.849 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.849 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:46:29.849 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:14.721181+0000", 2023-09-23T19:46:29.849 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00010074200000000001 2023-09-23T19:46:29.849 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.849 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.849 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:46:29.849 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:14.721282+0000", 2023-09-23T19:46:29.850 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.0362053280000001 2023-09-23T19:46:29.850 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:29.850 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:46:29.850 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:29.850 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.850 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.850 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45947.0:116733 10.3 10:c441ff1c:::benchmark_data_smithi016_136365_object6599:head [delete] snapc 0=[] ondisk+write+known_if_redirected e155)", 2023-09-23T19:46:29.850 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:38:13.692125+0000", 2023-09-23T19:46:29.850 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 496.13927182200001, 2023-09-23T19:46:29.850 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.029330611, 2023-09-23T19:46:29.850 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:46:29.850 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:46:29.851 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:46:29.851 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45947", 2023-09-23T19:46:29.851 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/4039191752", 2023-09-23T19:46:29.851 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 116733 2023-09-23T19:46:29.851 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.851 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:46:29.851 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.851 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:46:29.851 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.692125+0000", 2023-09-23T19:46:29.851 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:46:29.851 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.851 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.852 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:46:29.852 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.692125+0000", 2023-09-23T19:46:29.852 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T19:46:29.852 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.852 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.852 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:46:29.852 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.692124+0000", 2023-09-23T19:46:29.852 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.0979999999999996e-06 2023-09-23T19:46:29.852 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.852 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.852 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:46:29.852 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.692131+0000", 2023-09-23T19:46:29.853 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.4905999999999999e-05 2023-09-23T19:46:29.853 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.853 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.853 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:46:29.853 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.692146+0000", 2023-09-23T19:46:29.853 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.9449999999999998e-06 2023-09-23T19:46:29.853 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.853 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.853 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:46:29.853 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.692151+0000", 2023-09-23T19:46:29.853 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.7141999999999998e-05 2023-09-23T19:46:29.853 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.854 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.854 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:46:29.854 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.692189+0000", 2023-09-23T19:46:29.854 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.8889999999999999e-06 2023-09-23T19:46:29.854 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.854 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.854 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for peered", 2023-09-23T19:46:29.854 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.692195+0000", 2023-09-23T19:46:29.854 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.99909799200000005 2023-09-23T19:46:29.854 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.854 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.855 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:46:29.855 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:14.691293+0000", 2023-09-23T19:46:29.855 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000113662 2023-09-23T19:46:29.855 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.855 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.855 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:46:29.855 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:14.691406+0000", 2023-09-23T19:46:29.855 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000154485 2023-09-23T19:46:29.855 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.855 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.855 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3", 2023-09-23T19:46:29.856 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:14.691561+0000", 2023-09-23T19:46:29.856 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.009067828 2023-09-23T19:46:29.856 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.856 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.856 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:29.856 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:14.700628+0000", 2023-09-23T19:46:29.856 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.020668559999999999 2023-09-23T19:46:29.856 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.856 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.856 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:46:29.856 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:14.721297+0000", 2023-09-23T19:46:29.856 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.8945e-05 2023-09-23T19:46:29.857 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.857 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.857 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:46:29.857 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:14.721346+0000", 2023-09-23T19:46:29.857 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000109833 2023-09-23T19:46:29.857 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.857 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.857 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:46:29.857 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:14.721456+0000", 2023-09-23T19:46:29.857 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.029330611 2023-09-23T19:46:29.857 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:29.857 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:46:29.858 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:29.858 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.858 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.858 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45947.0:116737 10.3 10:ca9efb98:::benchmark_data_smithi016_136365_object6603:head [delete] snapc 0=[] ondisk+write+known_if_redirected e155)", 2023-09-23T19:46:29.858 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:38:13.699309+0000", 2023-09-23T19:46:29.858 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 496.13208817100002, 2023-09-23T19:46:29.858 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.0222978979999999, 2023-09-23T19:46:29.858 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:46:29.858 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:46:29.858 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:46:29.858 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45947", 2023-09-23T19:46:29.859 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/4039191752", 2023-09-23T19:46:29.859 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 116737 2023-09-23T19:46:29.859 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.859 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:46:29.859 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.859 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:46:29.859 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.699309+0000", 2023-09-23T19:46:29.859 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:46:29.859 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.859 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.859 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:46:29.860 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.699309+0000", 2023-09-23T19:46:29.860 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T19:46:29.860 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.860 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.860 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:46:29.860 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.699307+0000", 2023-09-23T19:46:29.860 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.2989999999999999e-06 2023-09-23T19:46:29.860 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.860 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.860 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:46:29.860 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.699315+0000", 2023-09-23T19:46:29.860 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.4414e-05 2023-09-23T19:46:29.861 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.861 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.861 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:46:29.861 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.699329+0000", 2023-09-23T19:46:29.861 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.5960000000000003e-06 2023-09-23T19:46:29.861 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.861 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.861 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:46:29.861 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.699335+0000", 2023-09-23T19:46:29.861 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.54e-05 2023-09-23T19:46:29.861 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.862 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.862 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:46:29.862 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.699370+0000", 2023-09-23T19:46:29.862 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.0360000000000003e-06 2023-09-23T19:46:29.862 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.862 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.862 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for peered", 2023-09-23T19:46:29.862 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.699376+0000", 2023-09-23T19:46:29.862 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.99668330400000005 2023-09-23T19:46:29.862 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.862 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.862 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:46:29.863 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:14.696060+0000", 2023-09-23T19:46:29.863 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00011095300000000001 2023-09-23T19:46:29.863 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.863 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.863 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:46:29.863 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:14.696170+0000", 2023-09-23T19:46:29.863 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00012792200000000001 2023-09-23T19:46:29.863 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.863 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.863 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3", 2023-09-23T19:46:29.863 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:14.696298+0000", 2023-09-23T19:46:29.863 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.024776975999999999 2023-09-23T19:46:29.864 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.864 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.864 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:29.864 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:14.721075+0000", 2023-09-23T19:46:29.864 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00039015499999999999 2023-09-23T19:46:29.864 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.864 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.864 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:46:29.864 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:14.721466+0000", 2023-09-23T19:46:29.864 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.5995999999999998e-05 2023-09-23T19:46:29.864 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.864 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.865 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:46:29.865 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:14.721512+0000", 2023-09-23T19:46:29.865 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.5249999999999998e-05 2023-09-23T19:46:29.865 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.865 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.865 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:46:29.865 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:14.721607+0000", 2023-09-23T19:46:29.865 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.0222978979999999 2023-09-23T19:46:29.865 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:29.865 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:46:29.865 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:29.865 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.866 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.866 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45947.0:116741 10.9 10:99063b1a:::benchmark_data_smithi016_136365_object6607:head [delete] snapc 0=[] ondisk+write+known_if_redirected e155)", 2023-09-23T19:46:29.866 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:38:13.702167+0000", 2023-09-23T19:46:29.866 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 496.12923011999999, 2023-09-23T19:46:29.866 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.98450871399999995, 2023-09-23T19:46:29.866 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:46:29.866 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:46:29.866 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:46:29.866 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45947", 2023-09-23T19:46:29.866 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/4039191752", 2023-09-23T19:46:29.866 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 116741 2023-09-23T19:46:29.867 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.867 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:46:29.867 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.867 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:46:29.867 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.702167+0000", 2023-09-23T19:46:29.867 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:46:29.867 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.867 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.867 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:46:29.867 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.702167+0000", 2023-09-23T19:46:29.867 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.999999 2023-09-23T19:46:29.867 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.868 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.868 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:46:29.868 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.702166+0000", 2023-09-23T19:46:29.868 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.0839999999999997e-06 2023-09-23T19:46:29.868 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.868 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.868 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:46:29.868 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.702172+0000", 2023-09-23T19:46:29.868 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.2591e-05 2023-09-23T19:46:29.868 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.868 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.869 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:46:29.869 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.702184+0000", 2023-09-23T19:46:29.869 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.5979999999999996e-06 2023-09-23T19:46:29.869 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.869 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.869 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:46:29.869 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.702190+0000", 2023-09-23T19:46:29.869 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.7716999999999997e-05 2023-09-23T19:46:29.870 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.870 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.870 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:46:29.870 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.702238+0000", 2023-09-23T19:46:29.870 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.6649999999999992e-06 2023-09-23T19:46:29.870 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.870 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.870 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for peered", 2023-09-23T19:46:29.870 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.702246+0000", 2023-09-23T19:46:29.871 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.979302758 2023-09-23T19:46:29.871 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.871 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.871 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:46:29.871 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:14.681549+0000", 2023-09-23T19:46:29.871 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00014356699999999999 2023-09-23T19:46:29.871 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.871 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.871 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:46:29.871 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:14.681693+0000", 2023-09-23T19:46:29.871 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000172984 2023-09-23T19:46:29.872 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.872 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.872 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 7", 2023-09-23T19:46:29.872 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:14.681866+0000", 2023-09-23T19:46:29.872 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0042647850000000001 2023-09-23T19:46:29.872 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.872 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.872 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:46:29.872 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:14.686131+0000", 2023-09-23T19:46:29.872 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00038800400000000001 2023-09-23T19:46:29.873 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.873 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.873 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:29.873 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:14.686519+0000", 2023-09-23T19:46:29.873 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.0823999999999999e-05 2023-09-23T19:46:29.873 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.873 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.873 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:46:29.873 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:14.686579+0000", 2023-09-23T19:46:29.874 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.624e-05 2023-09-23T19:46:29.874 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.874 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.874 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:46:29.874 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:14.686676+0000", 2023-09-23T19:46:29.874 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.98450871399999995 2023-09-23T19:46:29.874 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:29.874 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:46:29.874 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:29.874 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.875 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.875 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45947.0:116744 10.3 10:c31fc165:::benchmark_data_smithi016_136365_object6610:head [delete] snapc 0=[] ondisk+write+known_if_redirected e155)", 2023-09-23T19:46:29.875 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:38:13.709494+0000", 2023-09-23T19:46:29.875 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 496.12190326899997, 2023-09-23T19:46:29.875 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.0122597360000001, 2023-09-23T19:46:29.875 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:46:29.876 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:46:29.876 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:46:29.876 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45947", 2023-09-23T19:46:29.876 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/4039191752", 2023-09-23T19:46:29.876 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 116744 2023-09-23T19:46:29.876 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.876 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:46:29.876 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.876 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:46:29.877 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.709494+0000", 2023-09-23T19:46:29.877 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:46:29.877 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.877 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.877 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:46:29.877 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.709494+0000", 2023-09-23T19:46:29.877 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T19:46:29.877 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.877 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.878 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:46:29.878 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.709492+0000", 2023-09-23T19:46:29.878 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.3033999999999999e-05 2023-09-23T19:46:29.878 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.878 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.878 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:46:29.878 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.709505+0000", 2023-09-23T19:46:29.878 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.3215e-05 2023-09-23T19:46:29.878 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.879 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.879 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:46:29.879 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.709538+0000", 2023-09-23T19:46:29.879 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.6271999999999999e-05 2023-09-23T19:46:29.879 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.879 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.879 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:46:29.879 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.709554+0000", 2023-09-23T19:46:29.880 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.4932000000000005e-05 2023-09-23T19:46:29.880 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.880 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.880 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:46:29.880 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.709649+0000", 2023-09-23T19:46:29.880 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.9358999999999999e-05 2023-09-23T19:46:29.881 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.881 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.881 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for peered", 2023-09-23T19:46:29.881 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.709668+0000", 2023-09-23T19:46:29.881 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.99098472500000001 2023-09-23T19:46:29.881 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.881 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.881 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:46:29.881 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:14.700653+0000", 2023-09-23T19:46:29.882 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00013872600000000001 2023-09-23T19:46:29.882 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.882 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.882 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:46:29.882 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:14.700792+0000", 2023-09-23T19:46:29.882 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00019278599999999999 2023-09-23T19:46:29.882 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.882 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.883 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3", 2023-09-23T19:46:29.883 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:14.700985+0000", 2023-09-23T19:46:29.883 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.020134394 2023-09-23T19:46:29.883 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.883 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.883 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:29.883 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:14.721119+0000", 2023-09-23T19:46:29.883 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00049582300000000005 2023-09-23T19:46:29.883 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.883 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.883 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:46:29.883 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:14.721615+0000", 2023-09-23T19:46:29.884 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.0620000000000001e-05 2023-09-23T19:46:29.884 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.884 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.884 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:46:29.884 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:14.721656+0000", 2023-09-23T19:46:29.884 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.7969999999999999e-05 2023-09-23T19:46:29.884 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.884 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.884 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:46:29.884 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:14.721754+0000", 2023-09-23T19:46:29.884 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.0122597360000001 2023-09-23T19:46:29.884 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:29.885 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:46:29.885 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:29.885 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.885 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.885 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45947.0:116747 10.3 10:c0cbfb92:::benchmark_data_smithi016_136365_object6613:head [delete] snapc 0=[] ondisk+write+known_if_redirected e155)", 2023-09-23T19:46:29.885 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:38:13.720103+0000", 2023-09-23T19:46:29.885 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 496.11129400900001, 2023-09-23T19:46:29.885 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.0018429360000001, 2023-09-23T19:46:29.885 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:46:29.886 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:46:29.886 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:46:29.886 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45947", 2023-09-23T19:46:29.886 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/4039191752", 2023-09-23T19:46:29.886 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 116747 2023-09-23T19:46:29.886 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.886 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:46:29.886 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.887 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:46:29.887 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.720103+0000", 2023-09-23T19:46:29.887 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:46:29.887 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.887 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.887 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:46:29.887 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.720103+0000", 2023-09-23T19:46:29.887 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999971 2023-09-23T19:46:29.888 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.888 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.888 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:46:29.888 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.720100+0000", 2023-09-23T19:46:29.888 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.4569000000000001e-05 2023-09-23T19:46:29.888 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.888 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.888 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:46:29.888 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.720115+0000", 2023-09-23T19:46:29.889 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.7813e-05 2023-09-23T19:46:29.889 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.889 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.889 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:46:29.889 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.720143+0000", 2023-09-23T19:46:29.889 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.5160000000000001e-05 2023-09-23T19:46:29.889 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.889 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.889 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:46:29.889 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.720158+0000", 2023-09-23T19:46:29.889 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.7565000000000002e-05 2023-09-23T19:46:29.889 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.890 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.890 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:46:29.890 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.720215+0000", 2023-09-23T19:46:29.890 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.2179999999999996e-06 2023-09-23T19:46:29.890 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.890 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.890 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for peered", 2023-09-23T19:46:29.890 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:13.720222+0000", 2023-09-23T19:46:29.890 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.98421170800000002 2023-09-23T19:46:29.890 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.890 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.891 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:46:29.891 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:14.704433+0000", 2023-09-23T19:46:29.891 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000133993 2023-09-23T19:46:29.891 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.891 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.891 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:46:29.891 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:14.704567+0000", 2023-09-23T19:46:29.891 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00014388500000000001 2023-09-23T19:46:29.891 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.891 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.891 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3", 2023-09-23T19:46:29.891 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:14.704711+0000", 2023-09-23T19:46:29.892 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.017053158999999998 2023-09-23T19:46:29.892 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.892 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.892 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:46:29.892 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:14.721764+0000", 2023-09-23T19:46:29.892 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.8118000000000002e-05 2023-09-23T19:46:29.892 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.892 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.892 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:29.892 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:14.721812+0000", 2023-09-23T19:46:29.892 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.2794999999999999e-05 2023-09-23T19:46:29.893 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.893 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.893 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:46:29.893 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:14.721855+0000", 2023-09-23T19:46:29.893 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.0750999999999999e-05 2023-09-23T19:46:29.893 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.893 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.893 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:46:29.893 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:38:14.721946+0000", 2023-09-23T19:46:29.893 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.0018429360000001 2023-09-23T19:46:29.893 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:29.893 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:46:29.894 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:29.894 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.894 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.894 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45947.0:207465 10.6 10:652cdf93:::benchmark_data_smithi016_136365_object97331:head [delete] snapc 0=[] ondisk+write+known_if_redirected e171)", 2023-09-23T19:46:29.894 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:40:28.086329+0000", 2023-09-23T19:46:29.894 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 361.745068054, 2023-09-23T19:46:29.894 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.25209948100000001, 2023-09-23T19:46:29.894 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:46:29.894 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:46:29.894 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:46:29.894 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45947", 2023-09-23T19:46:29.895 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/4039191752", 2023-09-23T19:46:29.895 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 207465 2023-09-23T19:46:29.895 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.895 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:46:29.895 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.895 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:46:29.895 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.086329+0000", 2023-09-23T19:46:29.895 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:46:29.895 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.895 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.895 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:46:29.895 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.086329+0000", 2023-09-23T19:46:29.896 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T19:46:29.896 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.896 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.896 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:46:29.896 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.086328+0000", 2023-09-23T19:46:29.896 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.5549999999999997e-06 2023-09-23T19:46:29.896 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.896 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.896 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:46:29.896 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.086335+0000", 2023-09-23T19:46:29.897 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.3028e-05 2023-09-23T19:46:29.897 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.897 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.897 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:46:29.897 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.086348+0000", 2023-09-23T19:46:29.897 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.6070000000000002e-06 2023-09-23T19:46:29.897 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.897 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.897 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:46:29.897 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.086356+0000", 2023-09-23T19:46:29.897 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.7197000000000001e-05 2023-09-23T19:46:29.897 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.898 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.898 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:46:29.898 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.086413+0000", 2023-09-23T19:46:29.898 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000111433 2023-09-23T19:46:29.898 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.898 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.898 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:46:29.898 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.086524+0000", 2023-09-23T19:46:29.898 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.5847000000000007e-05 2023-09-23T19:46:29.898 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.898 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.899 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 0,4", 2023-09-23T19:46:29.899 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.086610+0000", 2023-09-23T19:46:29.899 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.011316638 2023-09-23T19:46:29.899 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.899 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.899 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:29.899 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.097927+0000", 2023-09-23T19:46:29.899 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.2525999999999999e-05 2023-09-23T19:46:29.899 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.899 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.899 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:46:29.899 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.097969+0000", 2023-09-23T19:46:29.900 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.24033410499999999 2023-09-23T19:46:29.900 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.900 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.900 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:29.900 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.338304+0000", 2023-09-23T19:46:29.900 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.6575999999999994e-05 2023-09-23T19:46:29.900 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.900 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.900 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:46:29.900 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.338370+0000", 2023-09-23T19:46:29.900 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.8288000000000002e-05 2023-09-23T19:46:29.900 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.901 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.901 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:46:29.901 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.338428+0000", 2023-09-23T19:46:29.901 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.25209948100000001 2023-09-23T19:46:29.901 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:29.901 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:46:29.901 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:29.901 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.901 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.901 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.45947.0:207469 10.6 10:6cee13bb:::benchmark_data_smithi016_136365_object97335:head [delete] snapc 0=[] ondisk+write+known_if_redirected e171)", 2023-09-23T19:46:29.901 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:40:28.087878+0000", 2023-09-23T19:46:29.902 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 361.74351893199997, 2023-09-23T19:46:29.902 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.26224475200000003, 2023-09-23T19:46:29.902 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:46:29.902 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:46:29.902 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:46:29.902 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.45947", 2023-09-23T19:46:29.902 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/4039191752", 2023-09-23T19:46:29.902 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 207469 2023-09-23T19:46:29.902 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.903 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:46:29.903 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.903 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:46:29.903 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.087878+0000", 2023-09-23T19:46:29.903 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:46:29.903 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.903 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.903 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:46:29.903 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.087878+0000", 2023-09-23T19:46:29.903 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.999999 2023-09-23T19:46:29.904 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.904 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.904 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:46:29.904 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.087877+0000", 2023-09-23T19:46:29.904 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.9900000000000002e-06 2023-09-23T19:46:29.904 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.904 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.904 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:46:29.904 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.087883+0000", 2023-09-23T19:46:29.904 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.1080999999999999e-05 2023-09-23T19:46:29.904 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.905 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.905 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:46:29.905 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.087894+0000", 2023-09-23T19:46:29.905 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.0730000000000001e-06 2023-09-23T19:46:29.905 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.905 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.905 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:46:29.905 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.087900+0000", 2023-09-23T19:46:29.905 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00340209 2023-09-23T19:46:29.905 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.905 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.906 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:46:29.906 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.091302+0000", 2023-09-23T19:46:29.906 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00011048399999999999 2023-09-23T19:46:29.906 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.906 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.906 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:46:29.906 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.091413+0000", 2023-09-23T19:46:29.906 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.9942e-05 2023-09-23T19:46:29.906 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.906 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.906 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 0,4", 2023-09-23T19:46:29.906 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.091513+0000", 2023-09-23T19:46:29.907 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0064842179999999999 2023-09-23T19:46:29.907 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.907 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.907 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:46:29.907 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.097997+0000", 2023-09-23T19:46:29.907 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0020859810000000002 2023-09-23T19:46:29.907 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.907 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.907 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:29.907 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.100083+0000", 2023-09-23T19:46:29.907 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.249933297 2023-09-23T19:46:29.908 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.908 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.908 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:29.908 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.350016+0000", 2023-09-23T19:46:29.908 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.9857e-05 2023-09-23T19:46:29.908 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.908 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.908 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:46:29.908 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.350076+0000", 2023-09-23T19:46:29.908 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.6782e-05 2023-09-23T19:46:29.908 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.908 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.909 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:46:29.909 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:40:28.350123+0000", 2023-09-23T19:46:29.909 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.26224475200000003 2023-09-23T19:46:29.909 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:29.909 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:46:29.909 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:29.909 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.909 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.909 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.36243.0:60661 11.7 11:e442658e:::benchmark_data_smithi016_160148_object60660:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e186)", 2023-09-23T19:46:29.910 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:42:22.039096+0000", 2023-09-23T19:46:29.910 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 247.79230136699999, 2023-09-23T19:46:29.910 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.103415098, 2023-09-23T19:46:29.910 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:46:29.910 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:46:29.910 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:46:29.910 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.36243", 2023-09-23T19:46:29.910 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/1280038109", 2023-09-23T19:46:29.910 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 60661 2023-09-23T19:46:29.911 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.911 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:46:29.911 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.911 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:46:29.911 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:22.039096+0000", 2023-09-23T19:46:29.911 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:46:29.911 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.911 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.911 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:46:29.912 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:22.039096+0000", 2023-09-23T19:46:29.912 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T19:46:29.912 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.912 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.912 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:46:29.912 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:22.039094+0000", 2023-09-23T19:46:29.912 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.6505999999999998e-05 2023-09-23T19:46:29.912 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.912 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.912 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:46:29.913 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:22.039140+0000", 2023-09-23T19:46:29.913 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.6835000000000001e-05 2023-09-23T19:46:29.913 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.913 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.913 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:46:29.913 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:22.039157+0000", 2023-09-23T19:46:29.913 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.7610000000000007e-06 2023-09-23T19:46:29.913 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.913 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.914 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:46:29.914 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:22.039165+0000", 2023-09-23T19:46:29.914 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.5132e-05 2023-09-23T19:46:29.914 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.914 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.914 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:46:29.914 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:22.039220+0000", 2023-09-23T19:46:29.914 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00012289199999999999 2023-09-23T19:46:29.914 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.914 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.915 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:46:29.915 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:22.039343+0000", 2023-09-23T19:46:29.915 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00020640099999999999 2023-09-23T19:46:29.915 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.915 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.915 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,4", 2023-09-23T19:46:29.915 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:22.039549+0000", 2023-09-23T19:46:29.915 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0049130179999999999 2023-09-23T19:46:29.915 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.916 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.916 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:46:29.916 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:22.044462+0000", 2023-09-23T19:46:29.916 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0072093929999999997 2023-09-23T19:46:29.916 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.916 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.916 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:29.916 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:22.051672+0000", 2023-09-23T19:46:29.916 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.090658509999999998 2023-09-23T19:46:29.916 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.917 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.917 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:29.917 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:22.142330+0000", 2023-09-23T19:46:29.917 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000103835 2023-09-23T19:46:29.917 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.917 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.917 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:46:29.917 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:22.142434+0000", 2023-09-23T19:46:29.917 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.6575999999999993e-05 2023-09-23T19:46:29.918 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.918 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.918 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:46:29.918 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:22.142511+0000", 2023-09-23T19:46:29.918 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.103415098 2023-09-23T19:46:29.918 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:29.918 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:46:29.918 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:29.918 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.919 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.919 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.36243.0:60669 11.7 11:ecf59873:::benchmark_data_smithi016_160148_object60668:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e186)", 2023-09-23T19:46:29.919 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:42:22.047214+0000", 2023-09-23T19:46:29.919 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 247.784183504, 2023-09-23T19:46:29.919 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.095535110000000006, 2023-09-23T19:46:29.919 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:46:29.919 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:46:29.919 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:46:29.920 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.36243", 2023-09-23T19:46:29.920 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/1280038109", 2023-09-23T19:46:29.920 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 60669 2023-09-23T19:46:29.920 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.920 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:46:29.920 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.920 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:46:29.920 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:22.047214+0000", 2023-09-23T19:46:29.920 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:46:29.921 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.921 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.921 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:46:29.921 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:22.047214+0000", 2023-09-23T19:46:29.921 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999938 2023-09-23T19:46:29.921 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.921 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.921 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:46:29.921 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:22.047207+0000", 2023-09-23T19:46:29.922 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.4524999999999995e-05 2023-09-23T19:46:29.922 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.922 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.922 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:46:29.922 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:22.047282+0000", 2023-09-23T19:46:29.922 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.5519000000000002e-05 2023-09-23T19:46:29.922 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.922 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.922 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:46:29.923 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:22.047298+0000", 2023-09-23T19:46:29.923 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.6529999999999994e-06 2023-09-23T19:46:29.923 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.923 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.923 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:46:29.923 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:22.047305+0000", 2023-09-23T19:46:29.923 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.6215999999999998e-05 2023-09-23T19:46:29.923 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.923 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.923 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:46:29.923 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:22.047361+0000", 2023-09-23T19:46:29.923 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00015182499999999999 2023-09-23T19:46:29.923 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.924 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.924 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:46:29.924 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:22.047513+0000", 2023-09-23T19:46:29.924 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00018416299999999999 2023-09-23T19:46:29.924 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.924 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.924 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,4", 2023-09-23T19:46:29.924 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:22.047697+0000", 2023-09-23T19:46:29.924 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0048189599999999997 2023-09-23T19:46:29.924 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.924 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.925 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:29.925 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:22.052516+0000", 2023-09-23T19:46:29.925 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0035091979999999998 2023-09-23T19:46:29.925 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.925 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.925 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:46:29.925 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:22.056026+0000", 2023-09-23T19:46:29.925 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.086558528999999995 2023-09-23T19:46:29.925 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.925 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.925 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:29.925 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:22.142584+0000", 2023-09-23T19:46:29.926 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.2009000000000006e-05 2023-09-23T19:46:29.926 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.926 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.926 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:46:29.926 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:22.142676+0000", 2023-09-23T19:46:29.926 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.2550000000000002e-05 2023-09-23T19:46:29.926 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.926 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.926 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:46:29.926 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:22.142749+0000", 2023-09-23T19:46:29.926 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.095535110000000006 2023-09-23T19:46:29.926 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:29.927 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:46:29.927 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:29.927 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.927 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.927 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.36243.0:67329 11.7 11:e47bf749:::benchmark_data_smithi016_160148_object67328:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e186)", 2023-09-23T19:46:29.927 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:42:28.575538+0000", 2023-09-23T19:46:29.927 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 241.25585918300001, 2023-09-23T19:46:29.927 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.112775024, 2023-09-23T19:46:29.927 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:46:29.927 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:46:29.927 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:46:29.927 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.36243", 2023-09-23T19:46:29.928 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/1280038109", 2023-09-23T19:46:29.928 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 67329 2023-09-23T19:46:29.928 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.928 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:46:29.928 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.928 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:46:29.928 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:28.575538+0000", 2023-09-23T19:46:29.928 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:46:29.928 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.928 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.928 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:46:29.928 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:28.575538+0000", 2023-09-23T19:46:29.929 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T19:46:29.929 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.929 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.929 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:46:29.929 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:28.575536+0000", 2023-09-23T19:46:29.929 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.8891999999999999e-05 2023-09-23T19:46:29.929 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.929 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.929 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:46:29.929 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:28.575585+0000", 2023-09-23T19:46:29.929 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.5764999999999999e-05 2023-09-23T19:46:29.930 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.930 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.930 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:46:29.930 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:28.575601+0000", 2023-09-23T19:46:29.930 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.8209999999999997e-06 2023-09-23T19:46:29.930 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.930 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.930 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:46:29.930 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:28.575610+0000", 2023-09-23T19:46:29.930 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.1975999999999999e-05 2023-09-23T19:46:29.930 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.930 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.931 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:46:29.931 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:28.575652+0000", 2023-09-23T19:46:29.931 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000132392 2023-09-23T19:46:29.931 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.931 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.931 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:46:29.931 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:28.575785+0000", 2023-09-23T19:46:29.931 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00016992600000000001 2023-09-23T19:46:29.931 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.931 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.931 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,4", 2023-09-23T19:46:29.931 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:28.575955+0000", 2023-09-23T19:46:29.932 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0038486420000000002 2023-09-23T19:46:29.932 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.932 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.932 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:46:29.932 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:28.579803+0000", 2023-09-23T19:46:29.932 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0027080860000000002 2023-09-23T19:46:29.932 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.932 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.932 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:29.932 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:28.582511+0000", 2023-09-23T19:46:29.932 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.10569285 2023-09-23T19:46:29.932 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.933 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.933 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:29.933 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:28.688204+0000", 2023-09-23T19:46:29.933 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.8250999999999994e-05 2023-09-23T19:46:29.933 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.933 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.933 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:46:29.933 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:28.688273+0000", 2023-09-23T19:46:29.933 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.0364999999999997e-05 2023-09-23T19:46:29.933 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.933 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.933 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:46:29.934 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:28.688313+0000", 2023-09-23T19:46:29.934 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.112775024 2023-09-23T19:46:29.934 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:29.934 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:46:29.934 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:29.934 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.934 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.934 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.36243.0:71366 11.7 11:e4033890:::benchmark_data_smithi016_160148_object71365:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e186)", 2023-09-23T19:46:29.934 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:42:32.579973+0000", 2023-09-23T19:46:29.934 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 237.251424523, 2023-09-23T19:46:29.935 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.115774133, 2023-09-23T19:46:29.935 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:46:29.935 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:46:29.935 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:46:29.935 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.36243", 2023-09-23T19:46:29.935 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/1280038109", 2023-09-23T19:46:29.935 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 71366 2023-09-23T19:46:29.935 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.935 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:46:29.935 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.935 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:46:29.935 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:32.579973+0000", 2023-09-23T19:46:29.936 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:46:29.936 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.936 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.936 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:46:29.936 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:32.579973+0000", 2023-09-23T19:46:29.936 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T19:46:29.936 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.936 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.936 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:46:29.936 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:32.579971+0000", 2023-09-23T19:46:29.936 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.9957999999999998e-05 2023-09-23T19:46:29.936 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.937 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.937 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:46:29.937 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:32.580021+0000", 2023-09-23T19:46:29.937 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.4301000000000001e-05 2023-09-23T19:46:29.937 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.937 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.937 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:46:29.937 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:32.580035+0000", 2023-09-23T19:46:29.937 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.0310000000000001e-06 2023-09-23T19:46:29.937 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.937 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.937 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:46:29.938 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:32.580042+0000", 2023-09-23T19:46:29.938 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.9972999999999998e-05 2023-09-23T19:46:29.938 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.938 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.938 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:46:29.938 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:32.580082+0000", 2023-09-23T19:46:29.938 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000120359 2023-09-23T19:46:29.938 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.938 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.938 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:46:29.938 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:32.580203+0000", 2023-09-23T19:46:29.938 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00013284700000000001 2023-09-23T19:46:29.939 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.939 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.939 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,4", 2023-09-23T19:46:29.939 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:32.580336+0000", 2023-09-23T19:46:29.939 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.004911197 2023-09-23T19:46:29.939 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.939 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.939 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:46:29.939 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:32.585247+0000", 2023-09-23T19:46:29.939 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.3553000000000005e-05 2023-09-23T19:46:29.940 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.940 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.940 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:29.940 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:32.585330+0000", 2023-09-23T19:46:29.940 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.110314749 2023-09-23T19:46:29.940 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.940 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.940 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:29.940 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:32.695645+0000", 2023-09-23T19:46:29.940 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.9836000000000002e-05 2023-09-23T19:46:29.940 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.940 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.941 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:46:29.941 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:32.695695+0000", 2023-09-23T19:46:29.941 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.1789999999999997e-05 2023-09-23T19:46:29.941 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.941 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.941 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:46:29.941 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:32.695747+0000", 2023-09-23T19:46:29.941 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.115774133 2023-09-23T19:46:29.941 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:29.941 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:46:29.941 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:29.941 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.942 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.942 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.36243.0:71367 11.7 11:ef4bc6f2:::benchmark_data_smithi016_160148_object71366:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e186)", 2023-09-23T19:46:29.942 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:42:32.580052+0000", 2023-09-23T19:46:29.942 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 237.25134539699999, 2023-09-23T19:46:29.942 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.11587554, 2023-09-23T19:46:29.942 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:46:29.942 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:46:29.942 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:46:29.942 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.36243", 2023-09-23T19:46:29.942 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/1280038109", 2023-09-23T19:46:29.942 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 71367 2023-09-23T19:46:29.943 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.943 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:46:29.943 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.943 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:46:29.943 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:32.580052+0000", 2023-09-23T19:46:29.943 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:46:29.943 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.943 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.943 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:46:29.943 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:32.580052+0000", 2023-09-23T19:46:29.944 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999995 2023-09-23T19:46:29.944 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.944 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.944 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:46:29.944 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:32.580051+0000", 2023-09-23T19:46:29.944 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.3582999999999998e-05 2023-09-23T19:46:29.944 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.944 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.944 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:46:29.945 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:32.580105+0000", 2023-09-23T19:46:29.945 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.9599999999999997e-06 2023-09-23T19:46:29.945 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.945 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.945 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:46:29.945 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:32.580111+0000", 2023-09-23T19:46:29.945 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.7299999999999999e-06 2023-09-23T19:46:29.945 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.945 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.946 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:46:29.946 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:32.580114+0000", 2023-09-23T19:46:29.946 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0024937700000000002 2023-09-23T19:46:29.946 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.946 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.946 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:46:29.946 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:32.582608+0000", 2023-09-23T19:46:29.946 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000221482 2023-09-23T19:46:29.946 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.947 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.949 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:46:29.949 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:32.582830+0000", 2023-09-23T19:46:29.949 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000224348 2023-09-23T19:46:29.949 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.949 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.949 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,4", 2023-09-23T19:46:29.949 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:32.583054+0000", 2023-09-23T19:46:29.949 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0023124880000000001 2023-09-23T19:46:29.950 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.950 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.950 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:29.950 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:32.585366+0000", 2023-09-23T19:46:29.950 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.6581000000000004e-05 2023-09-23T19:46:29.950 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.950 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.950 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:46:29.950 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:32.585443+0000", 2023-09-23T19:46:29.950 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.11038326599999999 2023-09-23T19:46:29.950 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.951 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.951 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:29.951 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:32.695826+0000", 2023-09-23T19:46:29.951 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.897e-05 2023-09-23T19:46:29.951 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.951 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.951 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:46:29.951 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:32.695885+0000", 2023-09-23T19:46:29.951 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.1916999999999999e-05 2023-09-23T19:46:29.952 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.952 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.952 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:46:29.952 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:32.695927+0000", 2023-09-23T19:46:29.952 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.11587554 2023-09-23T19:46:29.952 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:29.952 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:46:29.952 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:29.953 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.953 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.953 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.36243.0:71860 11.7 11:eb701d37:::benchmark_data_smithi016_160148_object71859:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e186)", 2023-09-23T19:46:29.953 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:42:33.138388+0000", 2023-09-23T19:46:29.953 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 236.693008994, 2023-09-23T19:46:29.953 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.094089550999999993, 2023-09-23T19:46:29.953 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:46:29.953 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:46:29.953 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:46:29.953 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.36243", 2023-09-23T19:46:29.954 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/1280038109", 2023-09-23T19:46:29.954 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 71860 2023-09-23T19:46:29.954 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.954 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:46:29.954 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.954 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:46:29.954 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:33.138388+0000", 2023-09-23T19:46:29.954 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:46:29.955 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.955 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.955 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:46:29.955 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:33.138388+0000", 2023-09-23T19:46:29.955 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T19:46:29.955 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.955 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.955 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:46:29.956 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:33.138386+0000", 2023-09-23T19:46:29.956 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.6730999999999998e-05 2023-09-23T19:46:29.956 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.956 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.956 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:46:29.956 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:33.138423+0000", 2023-09-23T19:46:29.956 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.5843000000000001e-05 2023-09-23T19:46:29.956 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.956 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.956 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:46:29.957 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:33.138439+0000", 2023-09-23T19:46:29.957 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.6229999999999999e-06 2023-09-23T19:46:29.957 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.957 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.957 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:46:29.957 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:33.138446+0000", 2023-09-23T19:46:29.957 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.9045999999999998e-05 2023-09-23T19:46:29.957 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.957 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.957 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:46:29.957 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:33.138495+0000", 2023-09-23T19:46:29.957 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00010806099999999999 2023-09-23T19:46:29.958 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.958 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.958 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:46:29.958 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:33.138603+0000", 2023-09-23T19:46:29.958 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000121653 2023-09-23T19:46:29.958 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.958 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.958 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,4", 2023-09-23T19:46:29.958 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:33.138724+0000", 2023-09-23T19:46:29.958 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00266994 2023-09-23T19:46:29.959 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.959 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.959 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:46:29.959 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:33.141394+0000", 2023-09-23T19:46:29.959 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0023070719999999999 2023-09-23T19:46:29.959 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.959 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.959 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:29.959 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:33.143701+0000", 2023-09-23T19:46:29.959 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.088666742000000007 2023-09-23T19:46:29.960 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.960 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.960 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:29.960 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:33.232368+0000", 2023-09-23T19:46:29.960 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.2867000000000006e-05 2023-09-23T19:46:29.960 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.960 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.960 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:46:29.960 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:33.232431+0000", 2023-09-23T19:46:29.960 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.6588999999999997e-05 2023-09-23T19:46:29.960 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.961 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.961 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:46:29.961 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:33.232478+0000", 2023-09-23T19:46:29.961 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.094089550999999993 2023-09-23T19:46:29.961 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:29.961 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:46:29.961 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:29.961 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.961 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.961 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.36243.0:74059 11.7 11:e12aa103:::benchmark_data_smithi016_160148_object74058:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e186)", 2023-09-23T19:46:29.961 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:42:35.497249+0000", 2023-09-23T19:46:29.962 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 234.334147696, 2023-09-23T19:46:29.962 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.105958249, 2023-09-23T19:46:29.962 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:46:29.962 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:46:29.962 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:46:29.962 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.36243", 2023-09-23T19:46:29.962 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/1280038109", 2023-09-23T19:46:29.962 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 74059 2023-09-23T19:46:29.962 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.962 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:46:29.962 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.963 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:46:29.963 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:35.497249+0000", 2023-09-23T19:46:29.963 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:46:29.963 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.963 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.963 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:46:29.963 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:35.497249+0000", 2023-09-23T19:46:29.963 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T19:46:29.963 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.963 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.963 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:46:29.963 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:35.497247+0000", 2023-09-23T19:46:29.964 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.4177000000000006e-05 2023-09-23T19:46:29.964 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.964 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.964 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:46:29.964 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:35.497311+0000", 2023-09-23T19:46:29.964 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.9933e-05 2023-09-23T19:46:29.964 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.964 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.964 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:46:29.964 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:35.497331+0000", 2023-09-23T19:46:29.964 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.4870000000000002e-06 2023-09-23T19:46:29.965 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.965 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.965 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:46:29.965 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:35.497340+0000", 2023-09-23T19:46:29.965 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.0740000000000004e-05 2023-09-23T19:46:29.965 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.965 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.965 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:46:29.965 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:35.497411+0000", 2023-09-23T19:46:29.965 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00016692399999999999 2023-09-23T19:46:29.965 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.966 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.966 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:46:29.966 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:35.497578+0000", 2023-09-23T19:46:29.966 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000201349 2023-09-23T19:46:29.966 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.966 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.966 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,4", 2023-09-23T19:46:29.966 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:35.497779+0000", 2023-09-23T19:46:29.966 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0029889109999999999 2023-09-23T19:46:29.966 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.966 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.966 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:46:29.967 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:35.500768+0000", 2023-09-23T19:46:29.967 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0031771130000000001 2023-09-23T19:46:29.967 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.967 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.967 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:29.967 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:35.503945+0000", 2023-09-23T19:46:29.967 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.099147003999999997 2023-09-23T19:46:29.967 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.967 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.967 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:29.967 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:35.603092+0000", 2023-09-23T19:46:29.967 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.0983999999999997e-05 2023-09-23T19:46:29.968 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.968 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.968 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:46:29.968 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:35.603163+0000", 2023-09-23T19:46:29.968 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.4694999999999998e-05 2023-09-23T19:46:29.968 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.968 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.968 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:46:29.968 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:42:35.603208+0000", 2023-09-23T19:46:29.968 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.105958249 2023-09-23T19:46:29.968 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:29.968 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:46:29.969 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:29.969 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.969 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.969 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_repop(client.36243.0:204020 11.5 e201/180 11:a0258c2c:::benchmark_data_smithi016_160148_object96817:head v 201'12661, mlcod=201'12657)", 2023-09-23T19:46:29.969 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:45:32.084807+0000", 2023-09-23T19:46:29.969 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 57.746589962999998, 2023-09-23T19:46:29.969 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.025361287, 2023-09-23T19:46:29.969 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:46:29.969 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:46:29.969 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:46:29.969 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.969 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:46:29.970 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:45:32.084807+0000", 2023-09-23T19:46:29.970 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:46:29.970 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.970 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.970 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:46:29.970 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:45:32.084807+0000", 2023-09-23T19:46:29.970 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.999999 2023-09-23T19:46:29.970 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.970 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.970 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:46:29.970 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:45:32.084806+0000", 2023-09-23T19:46:29.970 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.8980000000000002e-06 2023-09-23T19:46:29.971 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.971 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.971 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:46:29.971 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:45:32.084815+0000", 2023-09-23T19:46:29.971 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.2628e-05 2023-09-23T19:46:29.971 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.971 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.971 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:46:29.971 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:45:32.084828+0000", 2023-09-23T19:46:29.971 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.4450000000000001e-06 2023-09-23T19:46:29.971 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.972 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.972 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:46:29.972 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:45:32.084834+0000", 2023-09-23T19:46:29.972 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.4839999999999995e-05 2023-09-23T19:46:29.972 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.972 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.972 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:46:29.972 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:45:32.084909+0000", 2023-09-23T19:46:29.972 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.0959999999999999e-05 2023-09-23T19:46:29.972 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.972 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.972 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:46:29.973 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:45:32.084930+0000", 2023-09-23T19:46:29.973 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.025142642999999999 2023-09-23T19:46:29.973 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.973 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.973 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:46:29.973 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:45:32.110072+0000", 2023-09-23T19:46:29.973 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.5880999999999999e-05 2023-09-23T19:46:29.973 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.973 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.973 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:46:29.973 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:45:32.110168+0000", 2023-09-23T19:46:29.973 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.025361287 2023-09-23T19:46:29.973 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:29.974 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:46:29.974 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:29.974 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.974 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.974 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_repop(client.36243.0:204242 11.7 e201/192 11:e8f284c3:::benchmark_data_smithi016_160148_object97039:head v 201'12727, mlcod=201'12724)", 2023-09-23T19:46:29.974 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:45:32.432371+0000", 2023-09-23T19:46:29.974 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 57.399026345000003, 2023-09-23T19:46:29.974 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.024149805999999999, 2023-09-23T19:46:29.974 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:46:29.974 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:46:29.974 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:46:29.975 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.975 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:46:29.975 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:45:32.432371+0000", 2023-09-23T19:46:29.975 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:46:29.975 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.975 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.975 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:46:29.975 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:45:32.432371+0000", 2023-09-23T19:46:29.975 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T19:46:29.975 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.975 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.976 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:46:29.976 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:45:32.432369+0000", 2023-09-23T19:46:29.976 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.5671e-05 2023-09-23T19:46:29.976 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.976 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.976 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:46:29.976 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:45:32.432385+0000", 2023-09-23T19:46:29.976 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.5576e-05 2023-09-23T19:46:29.976 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.976 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.976 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:46:29.976 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:45:32.432420+0000", 2023-09-23T19:46:29.977 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.434e-05 2023-09-23T19:46:29.977 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.977 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.977 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:46:29.977 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:45:32.432435+0000", 2023-09-23T19:46:29.977 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0030857079999999999 2023-09-23T19:46:29.977 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.977 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.977 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:46:29.977 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:45:32.435520+0000", 2023-09-23T19:46:29.977 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.0869999999999999e-05 2023-09-23T19:46:29.977 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.978 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.978 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:46:29.978 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:45:32.435571+0000", 2023-09-23T19:46:29.978 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.020850202000000002 2023-09-23T19:46:29.978 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.978 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.978 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:46:29.978 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:45:32.456421+0000", 2023-09-23T19:46:29.978 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.9182e-05 2023-09-23T19:46:29.978 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.978 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.979 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:46:29.979 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:45:32.456520+0000", 2023-09-23T19:46:29.979 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.024149805999999999 2023-09-23T19:46:29.979 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:29.979 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:46:29.979 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:29.979 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.979 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.979 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_repop(client.36243.0:204704 11.7 e201/192 11:e91bcea4:::benchmark_data_smithi016_160148_object97501:head v 201'12754, mlcod=201'12751)", 2023-09-23T19:46:29.979 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:45:33.118333+0000", 2023-09-23T19:46:29.980 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 56.713063851999998, 2023-09-23T19:46:29.980 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.043003349000000003, 2023-09-23T19:46:29.980 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:46:29.980 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:46:29.980 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:46:29.980 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.980 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:46:29.980 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:45:33.118333+0000", 2023-09-23T19:46:29.980 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:46:29.980 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.980 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.981 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:46:29.981 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:45:33.118333+0000", 2023-09-23T19:46:29.981 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999995 2023-09-23T19:46:29.981 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.981 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.981 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:46:29.981 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:45:33.118333+0000", 2023-09-23T19:46:29.981 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.0730000000000004e-06 2023-09-23T19:46:29.981 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.981 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.981 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:46:29.982 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:45:33.118338+0000", 2023-09-23T19:46:29.982 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.4980000000000006e-06 2023-09-23T19:46:29.982 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.982 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.982 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:46:29.982 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:45:33.118346+0000", 2023-09-23T19:46:29.982 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.1989999999999997e-06 2023-09-23T19:46:29.982 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.983 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.983 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:46:29.983 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:45:33.118352+0000", 2023-09-23T19:46:29.983 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0046368540000000001 2023-09-23T19:46:29.983 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.983 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.983 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:46:29.983 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:45:33.122988+0000", 2023-09-23T19:46:29.983 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.5670999999999998e-05 2023-09-23T19:46:29.984 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.984 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.984 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:46:29.984 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:45:33.123024+0000", 2023-09-23T19:46:29.984 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.038229086000000002 2023-09-23T19:46:29.985 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.985 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.985 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:46:29.985 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:45:33.161253+0000", 2023-09-23T19:46:29.985 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.3385000000000002e-05 2023-09-23T19:46:29.985 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.986 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.986 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:46:29.986 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:45:33.161337+0000", 2023-09-23T19:46:29.986 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.043003349000000003 2023-09-23T19:46:29.986 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:29.986 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:46:29.986 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:29.986 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.986 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.986 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_repop(client.36243.0:204710 11.7 e201/192 11:e144b2ea:::benchmark_data_smithi016_160148_object97507:head v 201'12755, mlcod=201'12751)", 2023-09-23T19:46:29.987 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:45:33.128315+0000", 2023-09-23T19:46:29.987 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 56.703082522999999, 2023-09-23T19:46:29.987 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.033177086000000001, 2023-09-23T19:46:29.987 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:46:29.987 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:46:29.987 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:46:29.987 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.987 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:46:29.988 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:45:33.128315+0000", 2023-09-23T19:46:29.988 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:46:29.988 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.988 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.988 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:46:29.988 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:45:33.128315+0000", 2023-09-23T19:46:29.988 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.999999 2023-09-23T19:46:29.988 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.988 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.989 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:46:29.989 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:45:33.128313+0000", 2023-09-23T19:46:29.989 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.6829999999999997e-06 2023-09-23T19:46:29.989 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.989 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.989 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:46:29.989 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:45:33.128322+0000", 2023-09-23T19:46:29.990 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.6934000000000001e-05 2023-09-23T19:46:29.990 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.990 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.990 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:46:29.990 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:45:33.128339+0000", 2023-09-23T19:46:29.990 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.6090000000000003e-06 2023-09-23T19:46:29.990 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.990 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.990 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:46:29.991 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:45:33.128347+0000", 2023-09-23T19:46:29.991 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00053600000000000002 2023-09-23T19:46:29.991 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.991 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.991 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:46:29.991 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:45:33.128883+0000", 2023-09-23T19:46:29.991 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.0451999999999996e-05 2023-09-23T19:46:29.991 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.992 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.992 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:46:29.992 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:45:33.128953+0000", 2023-09-23T19:46:29.992 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.032389222000000002 2023-09-23T19:46:29.992 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.992 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.992 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:46:29.992 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:45:33.161342+0000", 2023-09-23T19:46:29.992 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00014923099999999999 2023-09-23T19:46:29.993 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:29.993 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:29.993 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:46:29.993 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:45:33.161492+0000", 2023-09-23T19:46:29.993 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.033177086000000001 2023-09-23T19:46:29.993 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:29.993 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:46:29.993 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:29.993 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:29.994 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:46:29.994 INFO:teuthology.orchestra.run.smithi092.stdout:} 2023-09-23T19:46:29.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:29.995 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:29.995 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:30.096 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:30.126 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:30.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:30.129 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:46:30.230 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:30.261 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:30.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:30.265 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:30.366 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:30.377 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:30 smithi092 bash[58855]: cluster 2023-09-23T19:46:28.316646+0000 mgr.y (mgr.25325) 1171 : cluster 0 pgmap v1150: 89 pgs: 89 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:46:30.378 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:30 smithi092 bash[58855]: audit 2023-09-23T19:46:29.185901+0000 mon.a (mon.0) 1479 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:46:30.391 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:30.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:30.395 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:30.496 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:30.532 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:30 smithi016 bash[77995]: cluster 2023-09-23T19:46:28.316646+0000 mgr.y (mgr.25325) 1171 : cluster 0 pgmap v1150: 89 pgs: 89 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:46:30.532 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:30 smithi016 bash[77995]: audit 2023-09-23T19:46:29.185901+0000 mon.a (mon.0) 1479 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:46:30.533 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:30 smithi016 bash[75004]: cluster 2023-09-23T19:46:28.316646+0000 mgr.y (mgr.25325) 1171 : cluster 0 pgmap v1150: 89 pgs: 89 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:46:30.533 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:30 smithi016 bash[75004]: audit 2023-09-23T19:46:29.185901+0000 mon.a (mon.0) 1479 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:46:30.533 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:30.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:30.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:30.636 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:30.669 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:30.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:30.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:30.686 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T19:46:30.773 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:30.802 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:30.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:30.805 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:46:30.905 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:30.931 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:30.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:30.933 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:31.034 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:31.066 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:31.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:31.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:31.169 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:31.202 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:31.208 INFO:teuthology.orchestra.run.smithi016.stderr:enabled application 'rados' on pool 'unique_pool_7' 2023-09-23T19:46:31.208 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is set 2023-09-23T19:46:31.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:31.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:31.310 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:31.324 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:31 smithi016 bash[75004]: audit 2023-09-23T19:46:30.967875+0000 mon.a (mon.0) 1480 : audit 1 from='client.? 172.21.15.16:0/4138613031' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:46:31.324 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:31 smithi016 bash[75004]: audit 2023-09-23T19:46:30.989939+0000 mon.a (mon.0) 1481 : audit 1 from='client.? 172.21.15.16:0/4022801673' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:46:31.325 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:31 smithi016 bash[77995]: audit 2023-09-23T19:46:30.967875+0000 mon.a (mon.0) 1480 : audit 1 from='client.? 172.21.15.16:0/4138613031' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:46:31.325 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:31 smithi016 bash[77995]: audit 2023-09-23T19:46:30.989939+0000 mon.a (mon.0) 1481 : audit 1 from='client.? 172.21.15.16:0/4022801673' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:46:31.339 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:31.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:31.342 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:31.443 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:31.456 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:31 smithi092 bash[58855]: audit 2023-09-23T19:46:30.967875+0000 mon.a (mon.0) 1480 : audit 1 from='client.? 172.21.15.16:0/4138613031' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:46:31.456 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:31 smithi092 bash[58855]: audit 2023-09-23T19:46:30.989939+0000 mon.a (mon.0) 1481 : audit 1 from='client.? 172.21.15.16:0/4022801673' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:46:31.476 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:31.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:31.478 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:31.579 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:31.609 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:31.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:31.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:31.713 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:31.739 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:31.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:31.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:31.843 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:31.872 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:31.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:31.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:31.976 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:32.006 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:32.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:32.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:46:32.110 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:32.139 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:32.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:32.142 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:46:32.243 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:32.256 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:32 smithi092 bash[58855]: cluster 2023-09-23T19:46:30.317735+0000 mgr.y (mgr.25325) 1172 : cluster 0 pgmap v1151: 89 pgs: 89 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:46:32.256 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:32 smithi092 bash[58855]: cluster 2023-09-23T19:46:31.185443+0000 mon.a (mon.0) 1482 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:46:32.256 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:32 smithi092 bash[58855]: audit 2023-09-23T19:46:31.198391+0000 mon.a (mon.0) 1483 : audit 1 from='client.? 172.21.15.16:0/4138613031' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T19:46:32.256 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:32 smithi092 bash[58855]: audit 2023-09-23T19:46:31.198505+0000 mon.a (mon.0) 1484 : audit 1 from='client.? 172.21.15.16:0/4022801673' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:46:32.256 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:32 smithi092 bash[58855]: cluster 2023-09-23T19:46:31.198594+0000 mon.a (mon.0) 1485 : cluster 0 osdmap e211: 8 total, 8 up, 8 in 2023-09-23T19:46:32.276 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:32.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:32.280 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:32.381 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:32.413 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:32.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:32.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:32.517 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:32.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:32 smithi016 bash[77995]: cluster 2023-09-23T19:46:30.317735+0000 mgr.y (mgr.25325) 1172 : cluster 0 pgmap v1151: 89 pgs: 89 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:46:32.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:32 smithi016 bash[77995]: cluster 2023-09-23T19:46:31.185443+0000 mon.a (mon.0) 1482 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:46:32.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:32 smithi016 bash[77995]: audit 2023-09-23T19:46:31.198391+0000 mon.a (mon.0) 1483 : audit 1 from='client.? 172.21.15.16:0/4138613031' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T19:46:32.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:32 smithi016 bash[77995]: audit 2023-09-23T19:46:31.198505+0000 mon.a (mon.0) 1484 : audit 1 from='client.? 172.21.15.16:0/4022801673' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:46:32.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:32 smithi016 bash[77995]: cluster 2023-09-23T19:46:31.198594+0000 mon.a (mon.0) 1485 : cluster 0 osdmap e211: 8 total, 8 up, 8 in 2023-09-23T19:46:32.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:32 smithi016 bash[75004]: cluster 2023-09-23T19:46:30.317735+0000 mgr.y (mgr.25325) 1172 : cluster 0 pgmap v1151: 89 pgs: 89 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:46:32.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:32 smithi016 bash[75004]: cluster 2023-09-23T19:46:31.185443+0000 mon.a (mon.0) 1482 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:46:32.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:32 smithi016 bash[75004]: audit 2023-09-23T19:46:31.198391+0000 mon.a (mon.0) 1483 : audit 1 from='client.? 172.21.15.16:0/4138613031' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T19:46:32.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:32 smithi016 bash[75004]: audit 2023-09-23T19:46:31.198505+0000 mon.a (mon.0) 1484 : audit 1 from='client.? 172.21.15.16:0/4022801673' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:46:32.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:32 smithi016 bash[75004]: cluster 2023-09-23T19:46:31.198594+0000 mon.a (mon.0) 1485 : cluster 0 osdmap e211: 8 total, 8 up, 8 in 2023-09-23T19:46:32.552 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:32.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:32.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:46:32.656 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:32.687 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:32.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:32.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:32.792 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:32.825 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:32.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:32.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:32.927 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:32.949 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:32.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:32.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:46:33.052 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:33.081 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:33.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:33.084 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:33.185 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:33.218 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:33.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:33.221 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:33.322 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:33.354 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:33.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:33.357 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:46:33.458 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:33.471 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:33 smithi016 bash[75004]: cluster 2023-09-23T19:46:31.830706+0000 osd.7 (osd.7) 128 : cluster 3 failed to encode map e211 with expected crc 2023-09-23T19:46:33.471 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:33 smithi016 bash[75004]: cluster 2023-09-23T19:46:31.833491+0000 osd.7 (osd.7) 129 : cluster 3 failed to encode map e211 with expected crc 2023-09-23T19:46:33.472 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:33 smithi016 bash[75004]: cluster 2023-09-23T19:46:31.892215+0000 osd.4 (osd.4) 155 : cluster 3 failed to encode map e211 with expected crc 2023-09-23T19:46:33.472 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:33 smithi016 bash[75004]: cluster 2023-09-23T19:46:31.892382+0000 osd.6 (osd.6) 141 : cluster 3 failed to encode map e211 with expected crc 2023-09-23T19:46:33.472 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:33 smithi016 bash[75004]: cluster 2023-09-23T19:46:31.892382+0000 osd.5 (osd.5) 137 : cluster 3 failed to encode map e211 with expected crc 2023-09-23T19:46:33.472 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:33 smithi016 bash[75004]: cluster 2023-09-23T19:46:32.318454+0000 mgr.y (mgr.25325) 1173 : cluster 0 pgmap v1153: 89 pgs: 89 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:46:33.472 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:33 smithi016 bash[77995]: cluster 2023-09-23T19:46:31.830706+0000 osd.7 (osd.7) 128 : cluster 3 failed to encode map e211 with expected crc 2023-09-23T19:46:33.472 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:33 smithi016 bash[77995]: cluster 2023-09-23T19:46:31.833491+0000 osd.7 (osd.7) 129 : cluster 3 failed to encode map e211 with expected crc 2023-09-23T19:46:33.472 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:33 smithi016 bash[77995]: cluster 2023-09-23T19:46:31.892215+0000 osd.4 (osd.4) 155 : cluster 3 failed to encode map e211 with expected crc 2023-09-23T19:46:33.472 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:33 smithi016 bash[77995]: cluster 2023-09-23T19:46:31.892382+0000 osd.6 (osd.6) 141 : cluster 3 failed to encode map e211 with expected crc 2023-09-23T19:46:33.472 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:33 smithi016 bash[77995]: cluster 2023-09-23T19:46:31.892382+0000 osd.5 (osd.5) 137 : cluster 3 failed to encode map e211 with expected crc 2023-09-23T19:46:33.473 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:33 smithi016 bash[77995]: cluster 2023-09-23T19:46:32.318454+0000 mgr.y (mgr.25325) 1173 : cluster 0 pgmap v1153: 89 pgs: 89 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:46:33.490 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:33.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:33.494 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:33.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:33 smithi092 bash[58855]: cluster 2023-09-23T19:46:31.830706+0000 osd.7 (osd.7) 128 : cluster 3 failed to encode map e211 with expected crc 2023-09-23T19:46:33.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:33 smithi092 bash[58855]: cluster 2023-09-23T19:46:31.833491+0000 osd.7 (osd.7) 129 : cluster 3 failed to encode map e211 with expected crc 2023-09-23T19:46:33.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:33 smithi092 bash[58855]: cluster 2023-09-23T19:46:31.892215+0000 osd.4 (osd.4) 155 : cluster 3 failed to encode map e211 with expected crc 2023-09-23T19:46:33.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:33 smithi092 bash[58855]: cluster 2023-09-23T19:46:31.892382+0000 osd.6 (osd.6) 141 : cluster 3 failed to encode map e211 with expected crc 2023-09-23T19:46:33.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:33 smithi092 bash[58855]: cluster 2023-09-23T19:46:31.892382+0000 osd.5 (osd.5) 137 : cluster 3 failed to encode map e211 with expected crc 2023-09-23T19:46:33.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:33 smithi092 bash[58855]: cluster 2023-09-23T19:46:32.318454+0000 mgr.y (mgr.25325) 1173 : cluster 0 pgmap v1153: 89 pgs: 89 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:46:33.594 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:33.626 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:33.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:33.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:46:33.731 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:33.763 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:33.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:33.765 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:33.866 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:33.899 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:33.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:33.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:34.003 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:34.038 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:34.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:34.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:46:34.048 DEBUG:teuthology.orchestra.run.smithi016:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_7 bench 90 write' 2023-09-23T19:46:34.086 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T19:46:34.086 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T19:46:34.142 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:hints = 1 2023-09-23T19:46:34.142 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T19:46:34.142 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Object prefix: benchmark_data_smithi016_181952 2023-09-23T19:46:34.142 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:34.143 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:46:34.143 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 0 0 0 0 0 0 - 0 2023-09-23T19:46:34.184 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:34.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:34.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:46:34.289 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:34.319 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:34.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:34.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:46:34.423 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:34.450 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:34.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:34.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:46:34.544 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:46:34.549 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:46:34.555 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:34.595 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:34.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:34.601 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:34.701 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:34.731 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:34.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:34.734 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:34.835 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:34.863 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:34.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:34.866 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:34.967 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:34.996 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:34.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:34.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:35.035 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T19:46:35.100 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:35.135 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:35.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:35.138 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:35.142 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 1 16 2629 2613 163.303 163.312 0.0046009 0.00608707 2023-09-23T19:46:35.239 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:35.269 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:35.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:35.272 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:46:35.373 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:35.399 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:35.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:35.401 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:35.502 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:35.519 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:46:35.532 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:35.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:35.534 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:35.635 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:35.653 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:35.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:35.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:46:35.756 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:35.786 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:35.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:35 smithi092 bash[58855]: cluster 2023-09-23T19:46:34.319003+0000 mgr.y (mgr.25325) 1174 : cluster 0 pgmap v1154: 89 pgs: 89 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:46:35.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:35.789 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:35.889 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:35.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:35.921 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:35.921 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:36.022 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:36.054 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:36.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:36.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:46:36.082 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:35 smithi016 bash[77995]: cluster 2023-09-23T19:46:34.319003+0000 mgr.y (mgr.25325) 1174 : cluster 0 pgmap v1154: 89 pgs: 89 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:46:36.082 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:35 smithi016 bash[75004]: cluster 2023-09-23T19:46:34.319003+0000 mgr.y (mgr.25325) 1174 : cluster 0 pgmap v1154: 89 pgs: 89 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T19:46:36.083 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:46:35 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:46:35] "GET /metrics HTTP/1.1" 200 36235 "" "Prometheus/2.33.4" 2023-09-23T19:46:36.142 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 2 16 5183 5167 161.457 159.625 0.00423722 0.00617871 2023-09-23T19:46:36.158 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:36.185 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:36.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:36.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:46:36.289 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:36.322 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:36.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:36.325 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:46:36.426 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:36.463 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:36.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:36.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:46:36.567 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:36.593 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:36.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:36.596 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:36.697 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:36.728 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:36.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:36.731 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:46:36.832 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:36.857 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:36.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:36.860 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:36.960 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:36.991 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:36.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:36.994 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:37.095 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:37.128 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:37.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:37.131 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:46:37.142 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 3 16 7532 7516 156.57 146.812 0.0067327 0.00637369 2023-09-23T19:46:37.232 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:37.266 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:37.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:37.269 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:37.370 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:37.394 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:37.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:37.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:37.499 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:37.528 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:37.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:37.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:46:37.632 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:37.660 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:37.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:37.664 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:46:37.764 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:37.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:37 smithi092 bash[58855]: cluster 2023-09-23T19:46:36.319713+0000 mgr.y (mgr.25325) 1175 : cluster 0 pgmap v1155: 89 pgs: 89 active+clean; 76 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 7.6 MiB/s wr, 120 op/s 2023-09-23T19:46:37.813 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:37.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:37.816 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:37.871 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:37 smithi016 bash[77995]: cluster 2023-09-23T19:46:36.319713+0000 mgr.y (mgr.25325) 1175 : cluster 0 pgmap v1155: 89 pgs: 89 active+clean; 76 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 7.6 MiB/s wr, 120 op/s 2023-09-23T19:46:37.871 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:37 smithi016 bash[75004]: cluster 2023-09-23T19:46:36.319713+0000 mgr.y (mgr.25325) 1175 : cluster 0 pgmap v1155: 89 pgs: 89 active+clean; 76 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 7.6 MiB/s wr, 120 op/s 2023-09-23T19:46:37.916 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:37.945 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:37.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:37.947 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:46:38.048 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:38.077 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:38.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:38.080 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:38.143 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 4 16 9788 9772 152.674 141 0.00418668 0.00654062 2023-09-23T19:46:38.180 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:38.214 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:38.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:38.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:46:38.319 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:38.347 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:38.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:38.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:46:38.450 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:38.482 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:38.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:38.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:38.587 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:38.617 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:38.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:38.623 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:38.724 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:38.759 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:38.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:38.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:46:38.864 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:38.893 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:38.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:38.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:38.996 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:39.018 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:39.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:39.021 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:39.121 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:39.145 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 5 16 11323 11307 141.271 95.9375 0.00938942 0.00706749 2023-09-23T19:46:39.156 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:39.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:39.161 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:46:39.262 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:39.289 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:39.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:39.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:39.392 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:39.438 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:39.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:39.441 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:39.542 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:39.569 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:39.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:39.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:46:39.672 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:39.682 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:39 smithi016 bash[75004]: cluster 2023-09-23T19:46:38.320145+0000 mgr.y (mgr.25325) 1176 : cluster 0 pgmap v1156: 89 pgs: 89 active+clean; 221 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 22 MiB/s wr, 352 op/s 2023-09-23T19:46:39.682 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:39 smithi016 bash[77995]: cluster 2023-09-23T19:46:38.320145+0000 mgr.y (mgr.25325) 1176 : cluster 0 pgmap v1156: 89 pgs: 89 active+clean; 221 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 22 MiB/s wr, 352 op/s 2023-09-23T19:46:39.705 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:39.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:39.709 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:39.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:39 smithi092 bash[58855]: cluster 2023-09-23T19:46:38.320145+0000 mgr.y (mgr.25325) 1176 : cluster 0 pgmap v1156: 89 pgs: 89 active+clean; 221 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 22 MiB/s wr, 352 op/s 2023-09-23T19:46:39.809 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:39.834 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:39.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:39.837 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:46:39.938 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:39.967 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:39.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:39.970 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:46:40.071 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:40.103 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:40.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:40.106 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:40.145 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 6 16 13348 13332 138.818 126.562 0.00390832 0.00719484 2023-09-23T19:46:40.208 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:40.235 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:40.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:40.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:46:40.339 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:40.422 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:40.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:40.424 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:40.525 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:40.557 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:40.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:40.560 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:40.661 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:40.690 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:40.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:40.694 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:40.794 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:40.819 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:40.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:40.822 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:40.923 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:40.956 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:40.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:40.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:41.059 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:41.082 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:41.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:41.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:41.145 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 7 16 15424 15408 137.521 129.75 0.00710856 0.00726538 2023-09-23T19:46:41.185 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:41.214 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:41.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:41.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:46:41.272 INFO:teuthology.orchestra.run.smithi092.stdout:{ 2023-09-23T19:46:41.272 INFO:teuthology.orchestra.run.smithi092.stdout: "ops": [ 2023-09-23T19:46:41.272 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.272 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.36480.0:15657 12.f 12:faeb51f0:::benchmark_data_smithi016_181952_object15656:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e211)", 2023-09-23T19:46:41.272 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:46:41.262943+0000", 2023-09-23T19:46:41.272 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 0.0096553160000000006, 2023-09-23T19:46:41.273 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.009693274, 2023-09-23T19:46:41.273 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:46:41.273 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "waiting for sub ops", 2023-09-23T19:46:41.273 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:46:41.273 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.36480", 2023-09-23T19:46:41.273 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/1035295215", 2023-09-23T19:46:41.273 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 15657 2023-09-23T19:46:41.273 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.274 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:46:41.274 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.274 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:46:41.274 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.262943+0000", 2023-09-23T19:46:41.274 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:46:41.274 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.274 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.275 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:46:41.275 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.262943+0000", 2023-09-23T19:46:41.275 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.999999 2023-09-23T19:46:41.275 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.275 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.275 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:46:41.276 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.262942+0000", 2023-09-23T19:46:41.276 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.6927e-05 2023-09-23T19:46:41.276 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.276 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.276 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:46:41.276 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.262989+0000", 2023-09-23T19:46:41.276 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.1081999999999999e-05 2023-09-23T19:46:41.277 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.277 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.277 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:46:41.277 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.263010+0000", 2023-09-23T19:46:41.277 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.6988000000000001e-05 2023-09-23T19:46:41.277 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.278 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.278 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:46:41.278 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.263027+0000", 2023-09-23T19:46:41.278 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0034329069999999998 2023-09-23T19:46:41.278 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.278 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.278 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:46:41.279 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.266460+0000", 2023-09-23T19:46:41.279 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00016899499999999999 2023-09-23T19:46:41.279 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.279 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.279 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:46:41.279 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.266629+0000", 2023-09-23T19:46:41.279 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00019004300000000001 2023-09-23T19:46:41.279 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.280 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.280 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 2,5", 2023-09-23T19:46:41.280 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.266819+0000", 2023-09-23T19:46:41.280 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.003267879 2023-09-23T19:46:41.280 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.280 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.280 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:41.280 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.270087+0000", 2023-09-23T19:46:41.280 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.1313e-05 2023-09-23T19:46:41.280 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.280 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.280 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:46:41.281 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.270098+0000", 2023-09-23T19:46:41.281 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0071554000000000001 2023-09-23T19:46:41.281 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:41.281 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:46:41.282 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:41.282 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.282 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.282 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.36480.0:15661 12.f 12:fc99bd45:::benchmark_data_smithi016_181952_object15660:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e211)", 2023-09-23T19:46:41.282 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:46:41.265377+0000", 2023-09-23T19:46:41.282 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 0.0072216650000000004, 2023-09-23T19:46:41.282 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0073754240000000002, 2023-09-23T19:46:41.282 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:46:41.282 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "waiting for sub ops", 2023-09-23T19:46:41.282 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:46:41.283 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.36480", 2023-09-23T19:46:41.283 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/1035295215", 2023-09-23T19:46:41.283 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 15661 2023-09-23T19:46:41.283 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.283 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:46:41.283 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.283 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:46:41.283 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.265377+0000", 2023-09-23T19:46:41.283 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:46:41.283 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.283 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.283 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:46:41.283 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.265377+0000", 2023-09-23T19:46:41.284 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T19:46:41.284 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.284 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.284 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:46:41.284 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.265375+0000", 2023-09-23T19:46:41.284 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000252551 2023-09-23T19:46:41.284 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.284 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.284 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:46:41.284 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.265628+0000", 2023-09-23T19:46:41.284 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.7334e-05 2023-09-23T19:46:41.284 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.284 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.285 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:46:41.285 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.265645+0000", 2023-09-23T19:46:41.285 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.4159999999999998e-06 2023-09-23T19:46:41.285 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.285 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.285 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:46:41.285 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.265652+0000", 2023-09-23T19:46:41.285 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0024313749999999999 2023-09-23T19:46:41.285 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.285 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.285 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:46:41.285 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.268084+0000", 2023-09-23T19:46:41.286 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00014425000000000001 2023-09-23T19:46:41.286 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.286 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.286 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:46:41.286 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.268228+0000", 2023-09-23T19:46:41.286 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000183034 2023-09-23T19:46:41.286 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.286 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.286 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 2,5", 2023-09-23T19:46:41.287 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.268411+0000", 2023-09-23T19:46:41.287 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.001976181 2023-09-23T19:46:41.287 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.287 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.287 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:46:41.287 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.270387+0000", 2023-09-23T19:46:41.287 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000541893 2023-09-23T19:46:41.287 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.287 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.287 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:41.287 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.270929+0000", 2023-09-23T19:46:41.287 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0055525369999999998 2023-09-23T19:46:41.288 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:41.288 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:46:41.288 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:41.288 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.288 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.288 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.36480.0:15666 12.5 12:aaf308c9:::benchmark_data_smithi016_181952_object15665:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e211)", 2023-09-23T19:46:41.288 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:46:41.267358+0000", 2023-09-23T19:46:41.288 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 0.0052407620000000004, 2023-09-23T19:46:41.288 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0055061090000000004, 2023-09-23T19:46:41.288 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:46:41.288 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "waiting for sub ops", 2023-09-23T19:46:41.288 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:46:41.289 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.36480", 2023-09-23T19:46:41.289 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/1035295215", 2023-09-23T19:46:41.289 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 15666 2023-09-23T19:46:41.289 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.289 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:46:41.289 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.289 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:46:41.289 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.267358+0000", 2023-09-23T19:46:41.289 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:46:41.289 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.289 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.289 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:46:41.289 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.267358+0000", 2023-09-23T19:46:41.290 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T19:46:41.290 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.290 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.290 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:46:41.290 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.267356+0000", 2023-09-23T19:46:41.290 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.5595999999999998e-05 2023-09-23T19:46:41.290 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.290 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.290 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:46:41.291 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.267442+0000", 2023-09-23T19:46:41.291 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.6900000000000001e-05 2023-09-23T19:46:41.291 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.291 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.291 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:46:41.291 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.267459+0000", 2023-09-23T19:46:41.291 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.5329999999999997e-06 2023-09-23T19:46:41.291 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.291 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.291 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:46:41.291 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.267466+0000", 2023-09-23T19:46:41.292 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.6944000000000003e-05 2023-09-23T19:46:41.292 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.292 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.292 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:46:41.292 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.267513+0000", 2023-09-23T19:46:41.292 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00013423800000000001 2023-09-23T19:46:41.292 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.293 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.293 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:46:41.293 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.267647+0000", 2023-09-23T19:46:41.293 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00070584300000000005 2023-09-23T19:46:41.293 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.293 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.293 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,2", 2023-09-23T19:46:41.293 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.268353+0000", 2023-09-23T19:46:41.293 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.001736884 2023-09-23T19:46:41.293 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.293 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.293 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:46:41.294 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.270090+0000", 2023-09-23T19:46:41.294 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0010513569999999999 2023-09-23T19:46:41.294 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.294 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.294 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:46:41.294 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.271141+0000", 2023-09-23T19:46:41.294 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0037838580000000002 2023-09-23T19:46:41.294 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:41.294 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:46:41.294 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:41.294 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.294 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.295 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_repop(client.36480.0:15652 12.b e211/207 12:d5677c09:::benchmark_data_smithi016_181952_object15651:head v 211'1015, mlcod=211'1013)", 2023-09-23T19:46:41.295 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:46:41.271541+0000", 2023-09-23T19:46:41.295 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 0.001057033, 2023-09-23T19:46:41.295 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.001412565, 2023-09-23T19:46:41.295 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:46:41.295 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "started", 2023-09-23T19:46:41.295 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:46:41.295 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.295 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:46:41.295 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.271541+0000", 2023-09-23T19:46:41.295 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:46:41.296 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.296 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.296 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:46:41.296 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.271541+0000", 2023-09-23T19:46:41.296 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999995 2023-09-23T19:46:41.296 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.296 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.296 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:46:41.296 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.271541+0000", 2023-09-23T19:46:41.296 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.6640000000000001e-05 2023-09-23T19:46:41.297 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.297 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.297 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:46:41.297 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.271588+0000", 2023-09-23T19:46:41.297 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.4339999999999998e-06 2023-09-23T19:46:41.297 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.297 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.297 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:46:41.297 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.271597+0000", 2023-09-23T19:46:41.297 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.3539999999999999e-06 2023-09-23T19:46:41.297 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.297 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.298 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:46:41.298 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.271602+0000", 2023-09-23T19:46:41.298 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.2645999999999994e-05 2023-09-23T19:46:41.298 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.298 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.298 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:46:41.298 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.271695+0000", 2023-09-23T19:46:41.299 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.0585e-05 2023-09-23T19:46:41.299 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.299 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.300 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:46:41.300 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.271716+0000", 2023-09-23T19:46:41.300 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00017429699999999999 2023-09-23T19:46:41.300 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:41.300 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:46:41.300 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:41.301 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.301 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.301 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_repop(client.36480.0:15655 12.9 e211/207 12:93d88b9e:::benchmark_data_smithi016_181952_object15654:head v 211'1025, mlcod=211'1023)", 2023-09-23T19:46:41.301 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:46:41.271617+0000", 2023-09-23T19:46:41.301 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 0.000980952, 2023-09-23T19:46:41.301 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.001398736, 2023-09-23T19:46:41.301 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:46:41.301 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "started", 2023-09-23T19:46:41.301 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:46:41.301 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.301 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:46:41.301 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.271617+0000", 2023-09-23T19:46:41.302 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:46:41.302 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.302 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.302 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:46:41.302 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.271617+0000", 2023-09-23T19:46:41.302 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999995 2023-09-23T19:46:41.302 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.302 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.302 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:46:41.302 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.271617+0000", 2023-09-23T19:46:41.302 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.0324000000000002e-05 2023-09-23T19:46:41.303 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.303 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.303 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:46:41.303 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.271657+0000", 2023-09-23T19:46:41.303 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.4939999999999997e-06 2023-09-23T19:46:41.303 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.303 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.303 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:46:41.303 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.271665+0000", 2023-09-23T19:46:41.304 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.0549999999999999e-06 2023-09-23T19:46:41.304 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.304 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.304 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:46:41.304 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.271670+0000", 2023-09-23T19:46:41.304 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.5093999999999997e-05 2023-09-23T19:46:41.304 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.304 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.305 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:46:41.305 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.271735+0000", 2023-09-23T19:46:41.305 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.1362e-05 2023-09-23T19:46:41.305 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.305 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.305 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:46:41.306 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.271756+0000", 2023-09-23T19:46:41.306 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00013888400000000001 2023-09-23T19:46:41.306 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:41.306 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:46:41.306 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:41.306 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.306 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.307 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_repop(client.36480.0:15662 12.9 e211/207 12:9b817687:::benchmark_data_smithi016_181952_object15661:head v 211'1026, mlcod=211'1023)", 2023-09-23T19:46:41.307 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:46:41.271684+0000", 2023-09-23T19:46:41.307 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 0.000914765, 2023-09-23T19:46:41.307 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0013949959999999999, 2023-09-23T19:46:41.307 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:46:41.307 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "started", 2023-09-23T19:46:41.308 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:46:41.308 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.308 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:46:41.308 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.271684+0000", 2023-09-23T19:46:41.308 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:46:41.308 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.309 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.309 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:46:41.309 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.271684+0000", 2023-09-23T19:46:41.309 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999995 2023-09-23T19:46:41.309 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.309 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.310 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:46:41.310 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.271683+0000", 2023-09-23T19:46:41.310 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.0309000000000001e-05 2023-09-23T19:46:41.310 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.310 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.310 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:46:41.311 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.271713+0000", 2023-09-23T19:46:41.311 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.8769999999999993e-06 2023-09-23T19:46:41.311 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.311 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.311 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:46:41.311 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.271721+0000", 2023-09-23T19:46:41.312 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.9230000000000001e-06 2023-09-23T19:46:41.312 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.312 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.312 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:46:41.312 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.271726+0000", 2023-09-23T19:46:41.312 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00095162999999999997 2023-09-23T19:46:41.312 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.312 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.313 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:46:41.313 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.272678+0000", 2023-09-23T19:46:41.313 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.1675000000000002e-05 2023-09-23T19:46:41.313 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:46:41.313 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:46:41.313 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:46:41.313 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:41.272700+0000", 2023-09-23T19:46:41.314 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0010160239999999999 2023-09-23T19:46:41.314 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:41.314 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:46:41.314 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:41.314 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:46:41.314 INFO:teuthology.orchestra.run.smithi092.stdout: ], 2023-09-23T19:46:41.314 INFO:teuthology.orchestra.run.smithi092.stdout: "num_ops": 6 2023-09-23T19:46:41.314 INFO:teuthology.orchestra.run.smithi092.stdout:} 2023-09-23T19:46:41.318 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:41.349 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:41.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:41.352 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:46:41.452 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:41.486 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:41.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:41.490 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:46:41.591 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:41.621 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:41.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:41.624 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:46:41.725 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:41.735 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:41 smithi092 bash[58855]: cluster 2023-09-23T19:46:40.320771+0000 mgr.y (mgr.25325) 1177 : cluster 0 pgmap v1157: 89 pgs: 89 active+clean; 641 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-09-23T19:46:41.759 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:41.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:41.762 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:46:41.863 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:41.889 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:41.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:41.893 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:41.993 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:42.004 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:41 smithi016 bash[77995]: cluster 2023-09-23T19:46:40.320771+0000 mgr.y (mgr.25325) 1177 : cluster 0 pgmap v1157: 89 pgs: 89 active+clean; 641 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-09-23T19:46:42.004 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:41 smithi016 bash[75004]: cluster 2023-09-23T19:46:40.320771+0000 mgr.y (mgr.25325) 1177 : cluster 0 pgmap v1157: 89 pgs: 89 active+clean; 641 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-09-23T19:46:42.023 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:42.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:42.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:46:42.126 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:42.145 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 8 16 17567 17551 137.072 133.938 0.0117539 0.00728777 2023-09-23T19:46:42.152 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:42.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:42.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:42.255 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:42.280 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:42.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:42.287 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:46:42.387 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:42.423 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:42.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:42.428 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:46:42.529 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:42.552 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:42.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:42.556 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:46:42.656 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:42.684 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T19:46:42.686 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:42.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:42.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:42.790 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:42.817 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:42.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:42.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:46:42.843 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:46:42.923 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:42.961 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:42.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:42.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:46:43.064 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:43.094 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:43.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:43.097 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:43.145 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 9 16 19455 19439 134.952 118 0.016176 0.00740294 2023-09-23T19:46:43.198 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:43.225 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:43.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:43.228 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:43.329 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:43.356 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:43.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:43.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:43.459 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:43.485 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:43.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:43.488 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:46:43.589 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:43.613 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:43.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:43.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:43.716 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:44.016 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:43 smithi092 bash[58855]: cluster 2023-09-23T19:46:42.322061+0000 mgr.y (mgr.25325) 1178 : cluster 0 pgmap v1158: 89 pgs: 89 active+clean; 723 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T19:46:44.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:43 smithi016 bash[77995]: cluster 2023-09-23T19:46:42.322061+0000 mgr.y (mgr.25325) 1178 : cluster 0 pgmap v1158: 89 pgs: 89 active+clean; 723 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T19:46:44.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:43 smithi016 bash[75004]: cluster 2023-09-23T19:46:42.322061+0000 mgr.y (mgr.25325) 1178 : cluster 0 pgmap v1158: 89 pgs: 89 active+clean; 723 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T19:46:44.046 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:44.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:44.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:44.145 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 10 16 20906 20890 130.526 90.6875 0.00558555 0.0076514 2023-09-23T19:46:44.150 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:44.186 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:44.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:44.189 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:44.290 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:44.319 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:44.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:44.322 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:44.423 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:44.454 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:44.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:44.459 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:44.559 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:44.588 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:44.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:44.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:44.689 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:44 smithi016 bash[75004]: audit 2023-09-23T19:46:43.670348+0000 mon.a (mon.0) 1486 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:46:44.689 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:44 smithi016 bash[75004]: audit 2023-09-23T19:46:43.671515+0000 mon.a (mon.0) 1487 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:46:44.692 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:44.720 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:44.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:44.723 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:46:44.824 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:44.853 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:44.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:44.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:46:44.956 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:44.969 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:44 smithi016 bash[77995]: audit 2023-09-23T19:46:43.670348+0000 mon.a (mon.0) 1486 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:46:44.969 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:44 smithi016 bash[77995]: audit 2023-09-23T19:46:43.671515+0000 mon.a (mon.0) 1487 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:46:44.993 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:44.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:44.996 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:45.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:44 smithi092 bash[58855]: audit 2023-09-23T19:46:43.670348+0000 mon.a (mon.0) 1486 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:46:45.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:44 smithi092 bash[58855]: audit 2023-09-23T19:46:43.671515+0000 mon.a (mon.0) 1487 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:46:45.097 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:45.121 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:45.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:45.125 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:45.145 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 11 16 22362 22346 126.932 91 0.00844282 0.00787199 2023-09-23T19:46:45.226 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:45.257 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:45.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:45.260 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:45.361 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:45.389 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:45.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:45.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:45.492 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:45.523 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:45.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:45.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:46:45.627 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:45.655 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:45.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:45.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:46:45.759 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:45.788 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:45.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:45.792 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:46:45.892 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:45.902 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:45 smithi092 bash[58855]: cluster 2023-09-23T19:46:44.322698+0000 mgr.y (mgr.25325) 1179 : cluster 0 pgmap v1159: 89 pgs: 89 active+clean; 988 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-09-23T19:46:45.926 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:45.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:45.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:46:46.030 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:46.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:45 smithi016 bash[75004]: cluster 2023-09-23T19:46:44.322698+0000 mgr.y (mgr.25325) 1179 : cluster 0 pgmap v1159: 89 pgs: 89 active+clean; 988 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-09-23T19:46:46.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:45 smithi016 bash[77995]: cluster 2023-09-23T19:46:44.322698+0000 mgr.y (mgr.25325) 1179 : cluster 0 pgmap v1159: 89 pgs: 89 active+clean; 988 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-09-23T19:46:46.036 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:46:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:46:45] "GET /metrics HTTP/1.1" 200 36293 "" "Prometheus/2.33.4" 2023-09-23T19:46:46.057 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:46.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:46.061 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:46.145 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 12 16 24203 24187 125.943 115.062 0.00716625 0.00793359 2023-09-23T19:46:46.163 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:46.181 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:46.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:46.183 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:46.284 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:46.314 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:46.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:46.318 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:46.419 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:46.449 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:46.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:46.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:46:46.554 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:46.574 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:46.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:46.578 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:46.679 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:46.708 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:46.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:46.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:46.811 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:46.840 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:46.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:46.846 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:46:46.946 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:46.972 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:46.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:46.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:46:47.075 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:47.102 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:47.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:47.107 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:47.146 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 13 16 25682 25666 123.36 92.4375 0.00807848 0.0080988 2023-09-23T19:46:47.207 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:47.237 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:47.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:47.241 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:46:47.342 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:47.377 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:47.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:47.381 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:46:47.482 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:47.488 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:46:47.510 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:47.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:47.512 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:47.524 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:46:47.525 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:46:47.571 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:46:47.613 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:47.620 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:46:47.645 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:47.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:47.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:47.660 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:46:47.703 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:46:47.738 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:46:47.748 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:47.756 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:47 smithi092 bash[58855]: cluster 2023-09-23T19:46:46.323424+0000 mgr.y (mgr.25325) 1180 : cluster 0 pgmap v1160: 89 pgs: 89 active+clean; 1.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 110 MiB/s wr, 1.75k op/s 2023-09-23T19:46:47.756 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:47 smithi092 bash[58855]: audit 2023-09-23T19:46:47.447437+0000 mon.a (mon.0) 1488 : audit 0 from='client.? 172.21.15.16:0/1683092708' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:46:47.774 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:47.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:47.777 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:46:47.780 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:46:47.785 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:46:47.785 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:46:47.785 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:46:47.785 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:46:47.785 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:46:47.785 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:46:47.785 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:46:47.785 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:46:47.878 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:47.904 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:47.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:47.908 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:47.914 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:47 smithi016 bash[75004]: cluster 2023-09-23T19:46:46.323424+0000 mgr.y (mgr.25325) 1180 : cluster 0 pgmap v1160: 89 pgs: 89 active+clean; 1.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 110 MiB/s wr, 1.75k op/s 2023-09-23T19:46:47.914 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:47 smithi016 bash[75004]: audit 2023-09-23T19:46:47.447437+0000 mon.a (mon.0) 1488 : audit 0 from='client.? 172.21.15.16:0/1683092708' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:46:47.915 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:47 smithi016 bash[77995]: cluster 2023-09-23T19:46:46.323424+0000 mgr.y (mgr.25325) 1180 : cluster 0 pgmap v1160: 89 pgs: 89 active+clean; 1.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 110 MiB/s wr, 1.75k op/s 2023-09-23T19:46:47.915 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:47 smithi016 bash[77995]: audit 2023-09-23T19:46:47.447437+0000 mon.a (mon.0) 1488 : audit 0 from='client.? 172.21.15.16:0/1683092708' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:46:48.008 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:48.035 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:48.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:48.037 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:46:48.139 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:48.147 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 14 16 27057 27041 120.687 85.9375 0.014199 0.00827936 2023-09-23T19:46:48.172 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:48.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:48.176 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:46:48.278 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:48.307 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:48.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:48.311 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:46:48.412 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:48.459 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:48.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:48.462 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:48.563 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:48.600 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:48.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:48.602 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:46:48.703 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:48.728 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.7 to 1 (10000) 2023-09-23T19:46:48.747 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:48.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:48.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:46:48.758 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:48 smithi016 bash[75004]: audit 2023-09-23T19:46:48.323525+0000 mon.a (mon.0) 1489 : audit 1 from='client.? 172.21.15.16:0/1203582243' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T19:46:48.758 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:48 smithi016 bash[77995]: audit 2023-09-23T19:46:48.323525+0000 mon.a (mon.0) 1489 : audit 1 from='client.? 172.21.15.16:0/1203582243' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T19:46:48.850 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:48.881 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:48.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:48.885 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:46:48.986 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:49.016 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:49.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:49.018 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:49.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:48 smithi092 bash[58855]: audit 2023-09-23T19:46:48.323525+0000 mon.a (mon.0) 1489 : audit 1 from='client.? 172.21.15.16:0/1203582243' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T19:46:49.119 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:49.146 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 15 16 28751 28735 119.699 105.875 0.0061829 0.008348 2023-09-23T19:46:49.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:49.147 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:49.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:46:49.248 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:49.271 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:49.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:49.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:49.373 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:49.400 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:49.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:49.404 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:46:49.504 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:49.532 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:49.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:49.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:49.639 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:49.665 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:49.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:49.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:46:49.768 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:49.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:49 smithi016 bash[75004]: cluster 2023-09-23T19:46:48.324610+0000 mgr.y (mgr.25325) 1181 : cluster 0 pgmap v1161: 89 pgs: 89 active+clean; 1.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-09-23T19:46:49.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:49 smithi016 bash[75004]: audit 2023-09-23T19:46:48.718010+0000 mon.a (mon.0) 1490 : audit 1 from='client.? 172.21.15.16:0/1203582243' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T19:46:49.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:49 smithi016 bash[75004]: cluster 2023-09-23T19:46:48.718181+0000 mon.a (mon.0) 1491 : cluster 0 osdmap e212: 8 total, 8 up, 8 in 2023-09-23T19:46:49.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:49 smithi016 bash[75004]: cluster 2023-09-23T19:46:48.720648+0000 osd.5 (osd.5) 138 : cluster 3 failed to encode map e212 with expected crc 2023-09-23T19:46:49.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:49 smithi016 bash[75004]: cluster 2023-09-23T19:46:48.724769+0000 osd.6 (osd.6) 142 : cluster 3 failed to encode map e212 with expected crc 2023-09-23T19:46:49.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:49 smithi016 bash[75004]: cluster 2023-09-23T19:46:48.730658+0000 osd.7 (osd.7) 130 : cluster 3 failed to encode map e212 with expected crc 2023-09-23T19:46:49.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:49 smithi016 bash[75004]: cluster 2023-09-23T19:46:48.733665+0000 osd.7 (osd.7) 131 : cluster 3 failed to encode map e212 with expected crc 2023-09-23T19:46:49.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:49 smithi016 bash[77995]: cluster 2023-09-23T19:46:48.324610+0000 mgr.y (mgr.25325) 1181 : cluster 0 pgmap v1161: 89 pgs: 89 active+clean; 1.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-09-23T19:46:49.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:49 smithi016 bash[77995]: audit 2023-09-23T19:46:48.718010+0000 mon.a (mon.0) 1490 : audit 1 from='client.? 172.21.15.16:0/1203582243' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T19:46:49.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:49 smithi016 bash[77995]: cluster 2023-09-23T19:46:48.718181+0000 mon.a (mon.0) 1491 : cluster 0 osdmap e212: 8 total, 8 up, 8 in 2023-09-23T19:46:49.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:49 smithi016 bash[77995]: cluster 2023-09-23T19:46:48.720648+0000 osd.5 (osd.5) 138 : cluster 3 failed to encode map e212 with expected crc 2023-09-23T19:46:49.788 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:49 smithi016 bash[77995]: cluster 2023-09-23T19:46:48.724769+0000 osd.6 (osd.6) 142 : cluster 3 failed to encode map e212 with expected crc 2023-09-23T19:46:49.788 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:49 smithi016 bash[77995]: cluster 2023-09-23T19:46:48.730658+0000 osd.7 (osd.7) 130 : cluster 3 failed to encode map e212 with expected crc 2023-09-23T19:46:49.788 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:49 smithi016 bash[77995]: cluster 2023-09-23T19:46:48.733665+0000 osd.7 (osd.7) 131 : cluster 3 failed to encode map e212 with expected crc 2023-09-23T19:46:49.794 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:49.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:49.798 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:46:49.899 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:49.909 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:49 smithi092 bash[58855]: cluster 2023-09-23T19:46:48.324610+0000 mgr.y (mgr.25325) 1181 : cluster 0 pgmap v1161: 89 pgs: 89 active+clean; 1.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-09-23T19:46:49.909 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:49 smithi092 bash[58855]: audit 2023-09-23T19:46:48.718010+0000 mon.a (mon.0) 1490 : audit 1 from='client.? 172.21.15.16:0/1203582243' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T19:46:49.909 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:49 smithi092 bash[58855]: cluster 2023-09-23T19:46:48.718181+0000 mon.a (mon.0) 1491 : cluster 0 osdmap e212: 8 total, 8 up, 8 in 2023-09-23T19:46:49.909 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:49 smithi092 bash[58855]: cluster 2023-09-23T19:46:48.720648+0000 osd.5 (osd.5) 138 : cluster 3 failed to encode map e212 with expected crc 2023-09-23T19:46:49.909 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:49 smithi092 bash[58855]: cluster 2023-09-23T19:46:48.724769+0000 osd.6 (osd.6) 142 : cluster 3 failed to encode map e212 with expected crc 2023-09-23T19:46:49.909 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:49 smithi092 bash[58855]: cluster 2023-09-23T19:46:48.730658+0000 osd.7 (osd.7) 130 : cluster 3 failed to encode map e212 with expected crc 2023-09-23T19:46:49.909 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:49 smithi092 bash[58855]: cluster 2023-09-23T19:46:48.733665+0000 osd.7 (osd.7) 131 : cluster 3 failed to encode map e212 with expected crc 2023-09-23T19:46:49.928 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:49.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:49.930 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:46:50.033 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:50.059 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:50.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:50.061 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:46:50.146 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 16 16 30283 30267 118.202 95.75 0.00647427 0.00845246 2023-09-23T19:46:50.163 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:50.190 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:50.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:50.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:50.293 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:50.338 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:50.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:50.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:46:50.441 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:50.472 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:50.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:50.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:50.575 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:50.600 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:50.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:50.605 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:46:50.705 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:50.732 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:50.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:50.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:50.835 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:50.864 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:50.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:50.868 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:50.969 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:50.993 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:50.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:50.995 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:51.096 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:51.125 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:51.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:51.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:51.146 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 17 16 31973 31957 117.461 105.625 0.00743308 0.00850747 2023-09-23T19:46:51.232 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:51.260 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:51.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:51.261 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:51.362 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:51.388 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:51.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:51.391 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:46:51.492 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:51.525 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:51.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:51.528 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:46:51.629 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:51.655 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:51.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:51.658 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:46:51.759 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:51.768 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:51 smithi016 bash[77995]: cluster 2023-09-23T19:46:50.325242+0000 mgr.y (mgr.25325) 1182 : cluster 0 pgmap v1163: 89 pgs: 89 active+clean; 1.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s 2023-09-23T19:46:51.769 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:51 smithi016 bash[75004]: cluster 2023-09-23T19:46:50.325242+0000 mgr.y (mgr.25325) 1182 : cluster 0 pgmap v1163: 89 pgs: 89 active+clean; 1.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s 2023-09-23T19:46:51.792 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:51.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:51.798 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:46:51.899 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:51.944 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:51.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:51.950 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:52.035 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T19:46:52.035 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T19:46:52.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:51 smithi092 bash[58855]: cluster 2023-09-23T19:46:50.325242+0000 mgr.y (mgr.25325) 1182 : cluster 0 pgmap v1163: 89 pgs: 89 active+clean; 1.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s 2023-09-23T19:46:52.051 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:52.083 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:52.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:52.086 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:52.146 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 18 16 33403 33387 115.901 89.375 0.00562383 0.00862101 2023-09-23T19:46:52.186 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:52.223 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:52.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:52.228 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:46:52.329 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:52.360 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:52.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:52.363 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:52.463 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:52.500 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:52.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:52.503 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:52.604 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:52.628 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:52.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:52.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:46:52.732 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:52.761 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:52.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:52.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:52.864 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:52.888 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:52.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:52.891 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:52.991 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:53.020 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:53.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:53.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:46:53.090 INFO:teuthology.orchestra.run.smithi092.stdout:ERROR: (22) Invalid argument 2023-09-23T19:46:53.091 INFO:teuthology.orchestra.run.smithi092.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:46:53.125 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:53.147 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 19 16 34824 34808 114.469 88.8125 0.0161499 0.00872929 2023-09-23T19:46:53.153 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:53.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:53.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:46:53.257 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:53.284 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:53.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:53.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:53.389 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:53.422 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:53.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:53.426 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:46:53.527 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:53.555 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:53.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:53.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:53.659 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:53.680 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:53.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:53.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:46:53.784 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:53.807 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:53.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:53.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:53.911 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:53.920 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:53 smithi092 bash[58855]: cluster 2023-09-23T19:46:52.325807+0000 mgr.y (mgr.25325) 1183 : cluster 0 pgmap v1164: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s 2023-09-23T19:46:53.920 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:53 smithi092 bash[58855]: audit 2023-09-23T19:46:53.061703+0000 mon.a (mon.0) 1492 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:46:53.934 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:46:53.940 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:53.943 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:53 smithi016 bash[75004]: cluster 2023-09-23T19:46:52.325807+0000 mgr.y (mgr.25325) 1183 : cluster 0 pgmap v1164: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s 2023-09-23T19:46:53.944 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:53 smithi016 bash[75004]: audit 2023-09-23T19:46:53.061703+0000 mon.a (mon.0) 1492 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:46:53.944 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:53 smithi016 bash[77995]: cluster 2023-09-23T19:46:52.325807+0000 mgr.y (mgr.25325) 1183 : cluster 0 pgmap v1164: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s 2023-09-23T19:46:53.944 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:53 smithi016 bash[77995]: audit 2023-09-23T19:46:53.061703+0000 mon.a (mon.0) 1492 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:46:53.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:53.945 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:54.046 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:54.085 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:54.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:54.089 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:46:54.148 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-09-23T19:46:54.144263+0000 min lat: 0.00140724 max lat: 0.307847 avg lat: 0.0088488 2023-09-23T19:46:54.148 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:46:54.148 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 20 16 36153 36137 112.899 83.0625 0.00534304 0.0088488 2023-09-23T19:46:54.189 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:54.218 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:54.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:54.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:54.243 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T19:46:54.321 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:54.358 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:54.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:54.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:54.462 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:54.491 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:54.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:54.497 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:46:54.597 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:54.629 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:54.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:54.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:46:54.735 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:54.745 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is set 2023-09-23T19:46:54.762 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:54.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:54.767 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:46:54.867 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:54.877 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:54 smithi016 bash[77995]: audit 2023-09-23T19:46:54.389167+0000 mon.a (mon.0) 1493 : audit 1 from='client.? 172.21.15.16:0/3081282836' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:46:54.878 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:54 smithi016 bash[75004]: audit 2023-09-23T19:46:54.389167+0000 mon.a (mon.0) 1493 : audit 1 from='client.? 172.21.15.16:0/3081282836' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:46:54.896 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:54.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:54.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:46:54.999 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:55.034 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:55.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:55.038 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:55.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:54 smithi092 bash[58855]: audit 2023-09-23T19:46:54.389167+0000 mon.a (mon.0) 1493 : audit 1 from='client.? 172.21.15.16:0/3081282836' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:46:55.139 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:55.147 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 21 16 37474 37458 111.454 82.5625 0.0106026 0.00896505 2023-09-23T19:46:55.165 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:55.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:55.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:46:55.267 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:55.295 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:55.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:55.298 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:55.399 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:55.426 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:55.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:55.430 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:55.530 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:55.562 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:55.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:55.565 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:55.666 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:55.692 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:55.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:55.695 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:55.796 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:55.809 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:55 smithi016 bash[77995]: cluster 2023-09-23T19:46:54.326336+0000 mgr.y (mgr.25325) 1184 : cluster 0 pgmap v1165: 89 pgs: 89 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-09-23T19:46:55.809 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:55 smithi016 bash[77995]: cluster 2023-09-23T19:46:54.720648+0000 mon.a (mon.0) 1494 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:46:55.809 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:55 smithi016 bash[77995]: audit 2023-09-23T19:46:54.733903+0000 mon.a (mon.0) 1495 : audit 1 from='client.? 172.21.15.16:0/3081282836' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:46:55.809 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:55 smithi016 bash[77995]: cluster 2023-09-23T19:46:54.734185+0000 mon.a (mon.0) 1496 : cluster 0 osdmap e213: 8 total, 8 up, 8 in 2023-09-23T19:46:55.809 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:55 smithi016 bash[77995]: cluster 2023-09-23T19:46:54.741120+0000 osd.7 (osd.7) 132 : cluster 3 failed to encode map e213 with expected crc 2023-09-23T19:46:55.809 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:55 smithi016 bash[77995]: cluster 2023-09-23T19:46:54.741820+0000 osd.6 (osd.6) 143 : cluster 3 failed to encode map e213 with expected crc 2023-09-23T19:46:55.809 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:55 smithi016 bash[77995]: cluster 2023-09-23T19:46:54.742078+0000 osd.5 (osd.5) 139 : cluster 3 failed to encode map e213 with expected crc 2023-09-23T19:46:55.809 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:55 smithi016 bash[77995]: cluster 2023-09-23T19:46:54.743493+0000 osd.5 (osd.5) 140 : cluster 3 failed to encode map e213 with expected crc 2023-09-23T19:46:55.810 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:55 smithi016 bash[75004]: cluster 2023-09-23T19:46:54.326336+0000 mgr.y (mgr.25325) 1184 : cluster 0 pgmap v1165: 89 pgs: 89 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-09-23T19:46:55.810 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:55 smithi016 bash[75004]: cluster 2023-09-23T19:46:54.720648+0000 mon.a (mon.0) 1494 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:46:55.810 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:55 smithi016 bash[75004]: audit 2023-09-23T19:46:54.733903+0000 mon.a (mon.0) 1495 : audit 1 from='client.? 172.21.15.16:0/3081282836' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:46:55.810 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:55 smithi016 bash[75004]: cluster 2023-09-23T19:46:54.734185+0000 mon.a (mon.0) 1496 : cluster 0 osdmap e213: 8 total, 8 up, 8 in 2023-09-23T19:46:55.810 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:55 smithi016 bash[75004]: cluster 2023-09-23T19:46:54.741120+0000 osd.7 (osd.7) 132 : cluster 3 failed to encode map e213 with expected crc 2023-09-23T19:46:55.810 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:55 smithi016 bash[75004]: cluster 2023-09-23T19:46:54.741820+0000 osd.6 (osd.6) 143 : cluster 3 failed to encode map e213 with expected crc 2023-09-23T19:46:55.810 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:55 smithi016 bash[75004]: cluster 2023-09-23T19:46:54.742078+0000 osd.5 (osd.5) 139 : cluster 3 failed to encode map e213 with expected crc 2023-09-23T19:46:55.810 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:55 smithi016 bash[75004]: cluster 2023-09-23T19:46:54.743493+0000 osd.5 (osd.5) 140 : cluster 3 failed to encode map e213 with expected crc 2023-09-23T19:46:55.811 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:46:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:46:55] "GET /metrics HTTP/1.1" 200 36293 "" "Prometheus/2.33.4" 2023-09-23T19:46:55.823 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:55.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:55.826 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:55.926 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:55.958 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:55.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:55.961 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:46:56.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:55 smithi092 bash[58855]: cluster 2023-09-23T19:46:54.326336+0000 mgr.y (mgr.25325) 1184 : cluster 0 pgmap v1165: 89 pgs: 89 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-09-23T19:46:56.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:55 smithi092 bash[58855]: cluster 2023-09-23T19:46:54.720648+0000 mon.a (mon.0) 1494 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:46:56.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:55 smithi092 bash[58855]: audit 2023-09-23T19:46:54.733903+0000 mon.a (mon.0) 1495 : audit 1 from='client.? 172.21.15.16:0/3081282836' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:46:56.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:55 smithi092 bash[58855]: cluster 2023-09-23T19:46:54.734185+0000 mon.a (mon.0) 1496 : cluster 0 osdmap e213: 8 total, 8 up, 8 in 2023-09-23T19:46:56.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:55 smithi092 bash[58855]: cluster 2023-09-23T19:46:54.741120+0000 osd.7 (osd.7) 132 : cluster 3 failed to encode map e213 with expected crc 2023-09-23T19:46:56.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:55 smithi092 bash[58855]: cluster 2023-09-23T19:46:54.741820+0000 osd.6 (osd.6) 143 : cluster 3 failed to encode map e213 with expected crc 2023-09-23T19:46:56.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:55 smithi092 bash[58855]: cluster 2023-09-23T19:46:54.742078+0000 osd.5 (osd.5) 139 : cluster 3 failed to encode map e213 with expected crc 2023-09-23T19:46:56.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:55 smithi092 bash[58855]: cluster 2023-09-23T19:46:54.743493+0000 osd.5 (osd.5) 140 : cluster 3 failed to encode map e213 with expected crc 2023-09-23T19:46:56.062 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:56.089 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:56.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:56.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:46:56.148 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 22 16 38509 38493 109.328 64.6875 0.0130125 0.0091388 2023-09-23T19:46:56.195 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:56.225 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:56.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:56.228 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:56.328 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:56.360 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:56.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:56.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:46:56.466 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:56.494 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:56.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:56.497 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:56.598 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:56.635 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:56.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:56.638 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:46:56.738 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:56.771 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:56.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:56.775 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:56.876 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:56.906 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:56.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:56.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:46:57.012 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:57.039 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:57.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:57.043 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:57.144 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:57.147 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 23 16 39848 39832 108.213 83.6875 0.0111835 0.00923343 2023-09-23T19:46:57.176 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:57.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:57.180 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:57.281 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:57.313 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:57.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:57.316 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:57.417 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:57.448 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:46:57.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:57.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:46:57.553 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:57.578 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:57.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:57.585 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:57.686 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:57.717 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:57.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:57.719 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:57.732 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:57 smithi016 bash[75004]: cluster 2023-09-23T19:46:56.326873+0000 mgr.y (mgr.25325) 1185 : cluster 0 pgmap v1167: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-09-23T19:46:57.821 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:57.847 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:46:57.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:57.850 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:46:57.951 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:57.960 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:57 smithi092 bash[58855]: cluster 2023-09-23T19:46:56.326873+0000 mgr.y (mgr.25325) 1185 : cluster 0 pgmap v1167: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-09-23T19:46:57.979 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:57.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:57.983 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:58.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:57 smithi016 bash[77995]: cluster 2023-09-23T19:46:56.326873+0000 mgr.y (mgr.25325) 1185 : cluster 0 pgmap v1167: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-09-23T19:46:58.084 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:58.114 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:46:58.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:58.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:46:58.148 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 24 16 41205 41189 107.238 84.8125 0.0113621 0.0093194 2023-09-23T19:46:58.222 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:58.258 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:58.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:58.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:46:58.361 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:58.397 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:58.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:58.401 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:58.502 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:58.537 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:58.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:58.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:46:58.641 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:58.671 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T19:46:58.680 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:46:58.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:58.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:46:58.784 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:58.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:58 smithi016 bash[77995]: audit 2023-09-23T19:46:58.668846+0000 mon.a (mon.0) 1497 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:46:58.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:58 smithi016 bash[75004]: audit 2023-09-23T19:46:58.668846+0000 mon.a (mon.0) 1497 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:46:58.817 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:58.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:58.820 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:46:58.921 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:58.958 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:58.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:58.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:59.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:58 smithi092 bash[58855]: audit 2023-09-23T19:46:58.668846+0000 mon.a (mon.0) 1497 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:46:59.050 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:46:59.063 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:59.107 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:59.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:59.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:59.148 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 25 16 42480 42464 106.136 79.6875 0.0115774 0.00941584 2023-09-23T19:46:59.182 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:46:59.183 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:46:59.212 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:59.240 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:59.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:59.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:46:59.345 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:59.384 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:46:59.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:59.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:46:59.488 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:59.515 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:59.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:59.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:59.618 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:59.644 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:46:59.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:59.647 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:46:59.748 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:59.757 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:59 smithi092 bash[58855]: cluster 2023-09-23T19:46:58.327545+0000 mgr.y (mgr.25325) 1186 : cluster 0 pgmap v1168: 89 pgs: 89 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-09-23T19:46:59.757 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:59 smithi092 bash[58855]: audit 2023-09-23T19:46:59.628370+0000 mon.a (mon.0) 1498 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:46:59.757 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:46:59 smithi092 bash[58855]: audit 2023-09-23T19:46:59.646309+0000 mon.a (mon.0) 1499 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:46:59.778 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:46:59.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:59.782 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:46:59.883 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:59.930 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:46:59.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:46:59.933 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:47:00.033 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:00.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:59 smithi016 bash[75004]: cluster 2023-09-23T19:46:58.327545+0000 mgr.y (mgr.25325) 1186 : cluster 0 pgmap v1168: 89 pgs: 89 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-09-23T19:47:00.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:59 smithi016 bash[75004]: audit 2023-09-23T19:46:59.628370+0000 mon.a (mon.0) 1498 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:47:00.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:46:59 smithi016 bash[75004]: audit 2023-09-23T19:46:59.646309+0000 mon.a (mon.0) 1499 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:47:00.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:59 smithi016 bash[77995]: cluster 2023-09-23T19:46:58.327545+0000 mgr.y (mgr.25325) 1186 : cluster 0 pgmap v1168: 89 pgs: 89 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-09-23T19:47:00.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:59 smithi016 bash[77995]: audit 2023-09-23T19:46:59.628370+0000 mon.a (mon.0) 1498 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:47:00.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:46:59 smithi016 bash[77995]: audit 2023-09-23T19:46:59.646309+0000 mon.a (mon.0) 1499 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:47:00.058 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:00.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:00.064 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:47:00.148 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 26 16 43499 43483 104.503 63.6875 0.00368826 0.0095431 2023-09-23T19:47:00.164 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:00.186 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:00.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:00.189 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:00.289 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:00.320 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:00.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:00.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:47:00.424 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:00.453 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:00.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:00.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:47:00.557 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:00.585 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:00.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:00.588 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:47:00.689 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:00.712 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:00.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:00.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:47:00.817 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:00.843 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:00.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:00.846 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:47:00.947 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:00.971 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:00.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:00.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:47:01.077 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:01.112 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:01.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:01.118 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:01.148 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 27 16 44282 44266 102.445 48.9375 0.0131401 0.00975135 2023-09-23T19:47:01.219 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:01.246 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:01.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:01.248 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:47:01.349 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:01.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:01.387 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:01.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:47:01.488 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:01.513 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:01.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:01.516 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:01.616 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:01.647 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:01.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:01.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:01.752 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:01.762 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:01 smithi016 bash[77995]: cluster 2023-09-23T19:47:00.328167+0000 mgr.y (mgr.25325) 1187 : cluster 0 pgmap v1169: 89 pgs: 89 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-09-23T19:47:01.762 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:01 smithi016 bash[77995]: audit 2023-09-23T19:47:00.554923+0000 mon.a (mon.0) 1500 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:47:01.762 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:01 smithi016 bash[77995]: audit 2023-09-23T19:47:00.566942+0000 mon.a (mon.0) 1501 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:47:01.762 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:01 smithi016 bash[77995]: audit 2023-09-23T19:47:01.052388+0000 mon.a (mon.0) 1502 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:47:01.762 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:01 smithi016 bash[77995]: audit 2023-09-23T19:47:01.064482+0000 mon.a (mon.0) 1503 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:47:01.762 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:01 smithi016 bash[75004]: cluster 2023-09-23T19:47:00.328167+0000 mgr.y (mgr.25325) 1187 : cluster 0 pgmap v1169: 89 pgs: 89 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-09-23T19:47:01.763 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:01 smithi016 bash[75004]: audit 2023-09-23T19:47:00.554923+0000 mon.a (mon.0) 1500 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:47:01.763 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:01 smithi016 bash[75004]: audit 2023-09-23T19:47:00.566942+0000 mon.a (mon.0) 1501 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:47:01.763 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:01 smithi016 bash[75004]: audit 2023-09-23T19:47:01.052388+0000 mon.a (mon.0) 1502 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:47:01.763 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:01 smithi016 bash[75004]: audit 2023-09-23T19:47:01.064482+0000 mon.a (mon.0) 1503 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:47:01.779 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:01.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:01 smithi092 bash[58855]: cluster 2023-09-23T19:47:00.328167+0000 mgr.y (mgr.25325) 1187 : cluster 0 pgmap v1169: 89 pgs: 89 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-09-23T19:47:01.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:01 smithi092 bash[58855]: audit 2023-09-23T19:47:00.554923+0000 mon.a (mon.0) 1500 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:47:01.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:01 smithi092 bash[58855]: audit 2023-09-23T19:47:00.566942+0000 mon.a (mon.0) 1501 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:47:01.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:01 smithi092 bash[58855]: audit 2023-09-23T19:47:01.052388+0000 mon.a (mon.0) 1502 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:47:01.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:01 smithi092 bash[58855]: audit 2023-09-23T19:47:01.064482+0000 mon.a (mon.0) 1503 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:47:01.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:01.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:47:01.890 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:01.913 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:01.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:01.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:02.017 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:02.049 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:02.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:02.051 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:02.148 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 28 16 45593 45577 101.713 81.9375 0.00990314 0.00982648 2023-09-23T19:47:02.152 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:02.177 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:02.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:02.179 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:47:02.280 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:02.308 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:02.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:02.317 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:02.418 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:02.444 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:02.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:02.452 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:47:02.553 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:02.574 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:02.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:02.581 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:47:02.682 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:02.705 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:02.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:02.707 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:47:02.807 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:02.831 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:02.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:02.833 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:02.934 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:02.962 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:02.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:02.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:03.064 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:03.101 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:03.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:03.104 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:47:03.148 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 29 16 46808 46792 100.824 75.9375 0.0178981 0.00991024 2023-09-23T19:47:03.204 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:03.238 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:03.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:03.241 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:03.342 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:03.374 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:03.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:03.381 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:03.481 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:03.510 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:03.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:03.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:47:03.613 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:03.646 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:03.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:03.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:03.755 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:03 smithi092 bash[58855]: cluster 2023-09-23T19:47:02.328703+0000 mgr.y (mgr.25325) 1188 : cluster 0 pgmap v1170: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-09-23T19:47:03.825 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:03.861 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:03.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:03.868 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:03.969 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:03.972 INFO:teuthology.orchestra.run.smithi092.stdout:ERROR: (22) Invalid argument 2023-09-23T19:47:03.972 INFO:teuthology.orchestra.run.smithi092.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:47:03.985 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:03 smithi016 bash[77995]: cluster 2023-09-23T19:47:02.328703+0000 mgr.y (mgr.25325) 1188 : cluster 0 pgmap v1170: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-09-23T19:47:03.985 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:03 smithi016 bash[75004]: cluster 2023-09-23T19:47:02.328703+0000 mgr.y (mgr.25325) 1188 : cluster 0 pgmap v1170: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-09-23T19:47:03.992 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:03.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:03.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:04.100 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:04.135 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:04.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:04.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:04.148 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 30 16 48069 48053 100.09 78.8125 0.0119515 0.00998466 2023-09-23T19:47:04.239 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:04.267 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:04.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:04.270 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:47:04.371 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:04.405 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:04.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:04.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:04.509 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:04.547 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:04.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:04.549 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:47:04.650 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:04.676 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:04.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:04.684 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:04.785 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:04.815 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:04.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:04.824 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:04.925 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:04.961 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:04.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:04.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:05.064 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:05.066 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T19:47:05.144 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:05.148 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 31 16 49360 49344 99.4639 80.6875 0.0143744 0.0100479 2023-09-23T19:47:05.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:05.151 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:05.251 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:05.279 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:05.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:05.282 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:05.282 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:47:05.383 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:05.410 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:05.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:05.420 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:05.520 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:05.546 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:05.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:05.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:05.651 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:05.662 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:05 smithi092 bash[58855]: cluster 2023-09-23T19:47:04.329520+0000 mgr.y (mgr.25325) 1189 : cluster 0 pgmap v1171: 89 pgs: 89 active+clean; 2.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s 2023-09-23T19:47:05.693 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:05.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:05.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:47:05.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:05 smithi016 bash[75004]: cluster 2023-09-23T19:47:04.329520+0000 mgr.y (mgr.25325) 1189 : cluster 0 pgmap v1171: 89 pgs: 89 active+clean; 2.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s 2023-09-23T19:47:05.787 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:47:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:47:05] "GET /metrics HTTP/1.1" 200 36300 "" "Prometheus/2.33.4" 2023-09-23T19:47:05.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:05 smithi016 bash[77995]: cluster 2023-09-23T19:47:04.329520+0000 mgr.y (mgr.25325) 1189 : cluster 0 pgmap v1171: 89 pgs: 89 active+clean; 2.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s 2023-09-23T19:47:05.800 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:05.825 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:05.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:05.830 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:47:05.930 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:05.955 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:05.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:05.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:47:06.058 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:06.083 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:06.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:06.086 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:47:06.149 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 32 16 50356 50340 98.3007 62.25 0.00327589 0.0101007 2023-09-23T19:47:06.186 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:06.215 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:06.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:06.217 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:47:06.318 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:06.347 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:06.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:06.351 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:47:06.452 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:06.480 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:06.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:06.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:06.585 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:06.612 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:06.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:06.616 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:47:06.717 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:06.741 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:06.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:06.744 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:06.844 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:06.871 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:06.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:06.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:47:06.974 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:07.002 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:07.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:07.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:47:07.106 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:07.133 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:07.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:07.137 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:07.149 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 33 16 51421 51405 97.3388 66.5625 0.0119503 0.0102668 2023-09-23T19:47:07.237 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:07.270 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:07.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:07.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:47:07.375 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:07.407 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:07.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:07.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:07.511 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:07.546 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:07.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:07.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:47:07.642 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:07 smithi016 bash[77995]: cluster 2023-09-23T19:47:06.330342+0000 mgr.y (mgr.25325) 1190 : cluster 0 pgmap v1172: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-09-23T19:47:07.643 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:07 smithi016 bash[75004]: cluster 2023-09-23T19:47:06.330342+0000 mgr.y (mgr.25325) 1190 : cluster 0 pgmap v1172: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-09-23T19:47:07.655 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:07.700 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:07.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:07.707 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:07.807 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:07.840 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:07.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:07.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:07.881 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:47:07.930 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:47:07.944 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:07.982 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:47:07.989 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:07.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:07.994 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:07.999 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:07 smithi092 bash[58855]: cluster 2023-09-23T19:47:06.330342+0000 mgr.y (mgr.25325) 1190 : cluster 0 pgmap v1172: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-09-23T19:47:08.023 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:47:08.068 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:47:08.095 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:08.105 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:47:08.119 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:08.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:08.122 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:08.140 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:47:08.149 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 34 16 52419 52403 96.3104 62.375 0.00812007 0.010375 2023-09-23T19:47:08.173 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:47:08.175 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:47:08.176 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:47:08.176 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:47:08.176 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:47:08.176 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:47:08.176 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:47:08.176 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:47:08.176 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:47:08.223 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:08.264 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:08.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:08.271 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:08.368 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-09-23T19:47:08.372 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:08.377 INFO:teuthology.orchestra.run.smithi016.stderr:dumped all 2023-09-23T19:47:08.406 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:08.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:08.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:47:08.509 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:08.532 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:08.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:08.535 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:08.636 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:08.668 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:08.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:08.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:47:08.774 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:08.781 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:08 smithi092 bash[58855]: audit 2023-09-23T19:47:07.839071+0000 mon.a (mon.0) 1504 : audit 0 from='client.? 172.21.15.16:0/951397771' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:47:08.794 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:08.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:08.796 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:08.897 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:08.905 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:08 smithi016 bash[75004]: audit 2023-09-23T19:47:07.839071+0000 mon.a (mon.0) 1504 : audit 0 from='client.? 172.21.15.16:0/951397771' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:47:08.906 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:08 smithi016 bash[77995]: audit 2023-09-23T19:47:07.839071+0000 mon.a (mon.0) 1504 : audit 0 from='client.? 172.21.15.16:0/951397771' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:47:08.921 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:08.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:08.925 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:09.026 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:09.054 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:09.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:09.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:09.149 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 35 16 53506 53490 95.4997 67.9375 0.00955933 0.010465 2023-09-23T19:47:09.158 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:09.178 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:09.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:09.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:09.281 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:09.298 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:09.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:09.301 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:09.402 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:09.429 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:09.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:09.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:47:09.432 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:47:09.533 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:09.562 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:09.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:09.566 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:09.666 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:09.684 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:09.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:09.686 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:09.786 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:09.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:09 smithi092 bash[58855]: cluster 2023-09-23T19:47:08.330931+0000 mgr.y (mgr.25325) 1191 : cluster 0 pgmap v1173: 89 pgs: 89 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-09-23T19:47:09.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:09 smithi092 bash[58855]: audit 2023-09-23T19:47:08.342865+0000 mgr.y (mgr.25325) 1192 : audit 0 from='client.36507 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:47:09.815 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:09.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:09.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:09.918 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:09.927 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:09 smithi016 bash[75004]: cluster 2023-09-23T19:47:08.330931+0000 mgr.y (mgr.25325) 1191 : cluster 0 pgmap v1173: 89 pgs: 89 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-09-23T19:47:09.927 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:09 smithi016 bash[75004]: audit 2023-09-23T19:47:08.342865+0000 mgr.y (mgr.25325) 1192 : audit 0 from='client.36507 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:47:09.929 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:09 smithi016 bash[77995]: cluster 2023-09-23T19:47:08.330931+0000 mgr.y (mgr.25325) 1191 : cluster 0 pgmap v1173: 89 pgs: 89 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-09-23T19:47:09.929 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:09 smithi016 bash[77995]: audit 2023-09-23T19:47:08.342865+0000 mgr.y (mgr.25325) 1192 : audit 0 from='client.36507 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:47:09.952 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:09.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:09.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:10.059 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:10.086 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:10.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:10.090 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:10.149 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 36 16 54436 54420 94.4614 58.125 0.0183616 0.0105804 2023-09-23T19:47:10.191 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:10.213 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:10.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:10.216 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:47:10.316 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:10.338 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:10.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:10.341 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:10.441 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:10.464 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:10.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:10.468 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:10.569 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:10.591 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:10.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:10.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:47:10.694 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:10.717 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:10.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:10.719 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:47:10.819 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:10.856 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:10.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:10.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:10.964 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:10.990 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:10.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:10.992 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:47:11.093 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:11.121 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:11.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:11.127 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:11.149 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 37 16 55394 55378 93.5266 59.875 0.0306261 0.010686 2023-09-23T19:47:11.228 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:11.252 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:11.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:11.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:11.358 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:11.385 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:11.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:11.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:11.491 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:11.521 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:11.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:11.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:47:11.624 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:11.661 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:11.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:11.664 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:47:11.765 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:11.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:11 smithi092 bash[58855]: cluster 2023-09-23T19:47:10.331592+0000 mgr.y (mgr.25325) 1193 : cluster 0 pgmap v1174: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-09-23T19:47:11.791 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:11.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:11.794 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:11.895 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:11.922 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:11.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:11.924 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:47:12.025 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:12.034 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:11 smithi016 bash[75004]: cluster 2023-09-23T19:47:10.331592+0000 mgr.y (mgr.25325) 1193 : cluster 0 pgmap v1174: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-09-23T19:47:12.034 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:11 smithi016 bash[77995]: cluster 2023-09-23T19:47:10.331592+0000 mgr.y (mgr.25325) 1193 : cluster 0 pgmap v1174: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-09-23T19:47:12.052 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:12.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:12.054 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:12.149 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 38 16 56540 56524 92.9501 71.625 0.00963692 0.0107524 2023-09-23T19:47:12.156 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:12.182 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:12.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:12.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:12.288 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:12.316 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:12.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:12.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:12.421 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:12.457 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:12.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:12.462 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:47:12.563 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:12.584 INFO:teuthology.orchestra.run.smithi016.stdout:{"pg_ready":true,"pg_map":{"version":1173,"stamp":"2023-09-23T19:47:08.330608+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3244103584,"num_objects":49516,"num_object_clones":0,"num_object_copies":148548,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":49516,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":49753,"num_write_kb":3169934,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":49687,"ondisk_log_size":49687,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":15190640,"kb_used_data":9482992,"kb_used_omap":20,"kb_used_meta":5707563,"kb_avail":734770576,"statfs":{"total":767960285184,"available":752405069824,"internally_reserved":0,"allocated":9710583808,"data_stored":9705146350,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":5844545166},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":877133824,"num_objects":13384,"num_object_clones":0,"num_object_copies":40152,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13384,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13384,"num_write_kb":856576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004089"},"pg_stats":[{"pgid":"3.1a","version":"59'1","reported_seq":209,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.733703+0000","last_change":"2023-09-23T19:38:43.897548+0000","last_active":"2023-09-23T19:46:48.733703+0000","last_peered":"2023-09-23T19:46:48.733703+0000","last_clean":"2023-09-23T19:46:48.733703+0000","last_became_active":"2023-09-23T19:38:43.896868+0000","last_became_peered":"2023-09-23T19:38:43.896868+0000","last_unstale":"2023-09-23T19:46:48.733703+0000","last_undegraded":"2023-09-23T19:46:48.733703+0000","last_fullsized":"2023-09-23T19:46:48.733703+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":171,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.719135+0000","last_change":"2023-09-23T19:32:49.731128+0000","last_active":"2023-09-23T19:46:48.719135+0000","last_peered":"2023-09-23T19:46:48.719135+0000","last_clean":"2023-09-23T19:46:48.719135+0000","last_became_active":"2023-09-23T19:32:49.730757+0000","last_became_peered":"2023-09-23T19:32:49.730757+0000","last_unstale":"2023-09-23T19:46:48.719135+0000","last_undegraded":"2023-09-23T19:46:48.719135+0000","last_fullsized":"2023-09-23T19:46:48.719135+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:40.183605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":160,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.734576+0000","last_change":"2023-09-23T19:28:07.890462+0000","last_active":"2023-09-23T19:46:54.734576+0000","last_peered":"2023-09-23T19:46:54.734576+0000","last_clean":"2023-09-23T19:46:54.734576+0000","last_became_active":"2023-09-23T19:28:07.890084+0000","last_became_peered":"2023-09-23T19:28:07.890084+0000","last_unstale":"2023-09-23T19:46:54.734576+0000","last_undegraded":"2023-09-23T19:46:54.734576+0000","last_fullsized":"2023-09-23T19:46:54.734576+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:50.458218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":99,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.737929+0000","last_change":"2023-09-23T19:42:55.772143+0000","last_active":"2023-09-23T19:46:48.737929+0000","last_peered":"2023-09-23T19:46:48.737929+0000","last_clean":"2023-09-23T19:46:48.737929+0000","last_became_active":"2023-09-23T19:42:55.771813+0000","last_became_peered":"2023-09-23T19:42:55.771813+0000","last_unstale":"2023-09-23T19:46:48.737929+0000","last_undegraded":"2023-09-23T19:46:48.737929+0000","last_fullsized":"2023-09-23T19:46:48.737929+0000","mapping_epoch":188,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":188,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.726263+0000","last_change":"2023-09-23T19:32:49.729961+0000","last_active":"2023-09-23T19:46:48.726263+0000","last_peered":"2023-09-23T19:46:48.726263+0000","last_clean":"2023-09-23T19:46:48.726263+0000","last_became_active":"2023-09-23T19:32:49.729251+0000","last_became_peered":"2023-09-23T19:32:49.729251+0000","last_unstale":"2023-09-23T19:46:48.726263+0000","last_undegraded":"2023-09-23T19:46:48.726263+0000","last_fullsized":"2023-09-23T19:46:48.726263+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:20:56.595732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":178,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.731362+0000","last_change":"2023-09-23T19:22:42.150582+0000","last_active":"2023-09-23T19:46:48.731362+0000","last_peered":"2023-09-23T19:46:48.731362+0000","last_clean":"2023-09-23T19:46:48.731362+0000","last_became_active":"2023-09-23T19:22:42.150229+0000","last_became_peered":"2023-09-23T19:22:42.150229+0000","last_unstale":"2023-09-23T19:46:48.731362+0000","last_undegraded":"2023-09-23T19:46:48.731362+0000","last_fullsized":"2023-09-23T19:46:48.731362+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":179,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.733260+0000","last_change":"2023-09-23T19:43:38.571327+0000","last_active":"2023-09-23T19:46:48.733260+0000","last_peered":"2023-09-23T19:46:48.733260+0000","last_clean":"2023-09-23T19:46:48.733260+0000","last_became_active":"2023-09-23T19:43:38.570994+0000","last_became_peered":"2023-09-23T19:43:38.570994+0000","last_unstale":"2023-09-23T19:46:48.733260+0000","last_undegraded":"2023-09-23T19:46:48.733260+0000","last_fullsized":"2023-09-23T19:46:48.733260+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:50:11.363431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":172,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:50.993205+0000","last_change":"2023-09-23T19:32:49.729765+0000","last_active":"2023-09-23T19:46:50.993205+0000","last_peered":"2023-09-23T19:46:50.993205+0000","last_clean":"2023-09-23T19:46:50.993205+0000","last_became_active":"2023-09-23T19:32:49.729007+0000","last_became_peered":"2023-09-23T19:32:49.729007+0000","last_unstale":"2023-09-23T19:46:50.993205+0000","last_undegraded":"2023-09-23T19:46:50.993205+0000","last_fullsized":"2023-09-23T19:46:50.993205+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:59:31.183753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":89,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.738034+0000","last_change":"2023-09-23T19:43:38.620503+0000","last_active":"2023-09-23T19:46:48.738034+0000","last_peered":"2023-09-23T19:46:48.738034+0000","last_clean":"2023-09-23T19:46:48.738034+0000","last_became_active":"2023-09-23T19:43:38.619167+0000","last_became_peered":"2023-09-23T19:43:38.619167+0000","last_unstale":"2023-09-23T19:46:48.738034+0000","last_undegraded":"2023-09-23T19:46:48.738034+0000","last_fullsized":"2023-09-23T19:46:48.738034+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":153,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.736576+0000","last_change":"2023-09-23T19:32:49.736864+0000","last_active":"2023-09-23T19:46:48.736576+0000","last_peered":"2023-09-23T19:46:48.736576+0000","last_clean":"2023-09-23T19:46:48.736576+0000","last_became_active":"2023-09-23T19:32:49.736523+0000","last_became_peered":"2023-09-23T19:32:49.736523+0000","last_unstale":"2023-09-23T19:46:48.736576+0000","last_undegraded":"2023-09-23T19:46:48.736576+0000","last_fullsized":"2023-09-23T19:46:48.736576+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":182,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.737021+0000","last_change":"2023-09-23T19:28:07.892835+0000","last_active":"2023-09-23T19:46:48.737021+0000","last_peered":"2023-09-23T19:46:48.737021+0000","last_clean":"2023-09-23T19:46:48.737021+0000","last_became_active":"2023-09-23T19:28:07.892338+0000","last_became_peered":"2023-09-23T19:28:07.892338+0000","last_unstale":"2023-09-23T19:46:48.737021+0000","last_undegraded":"2023-09-23T19:46:48.737021+0000","last_fullsized":"2023-09-23T19:46:48.737021+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":175,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.718919+0000","last_change":"2023-09-23T19:38:43.890137+0000","last_active":"2023-09-23T19:46:48.718919+0000","last_peered":"2023-09-23T19:46:48.718919+0000","last_clean":"2023-09-23T19:46:48.718919+0000","last_became_active":"2023-09-23T19:38:43.889800+0000","last_became_peered":"2023-09-23T19:38:43.889800+0000","last_unstale":"2023-09-23T19:46:48.718919+0000","last_undegraded":"2023-09-23T19:46:48.718919+0000","last_fullsized":"2023-09-23T19:46:48.718919+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:59:20.468913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'23","reported_seq":186,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.733503+0000","last_change":"2023-09-23T19:05:52.107727+0000","last_active":"2023-09-23T19:46:48.733503+0000","last_peered":"2023-09-23T19:46:48.733503+0000","last_clean":"2023-09-23T19:46:48.733503+0000","last_became_active":"2023-09-23T19:05:52.107378+0000","last_became_peered":"2023-09-23T19:05:52.107378+0000","last_unstale":"2023-09-23T19:46:48.733503+0000","last_undegraded":"2023-09-23T19:46:48.733503+0000","last_fullsized":"2023-09-23T19:46:48.733503+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":177,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.733817+0000","last_change":"2023-09-23T19:22:42.148497+0000","last_active":"2023-09-23T19:46:48.733817+0000","last_peered":"2023-09-23T19:46:48.733817+0000","last_clean":"2023-09-23T19:46:48.733817+0000","last_became_active":"2023-09-23T19:22:42.148147+0000","last_became_peered":"2023-09-23T19:22:42.148147+0000","last_unstale":"2023-09-23T19:46:48.733817+0000","last_undegraded":"2023-09-23T19:46:48.733817+0000","last_fullsized":"2023-09-23T19:46:48.733817+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":196,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.736916+0000","last_change":"2023-09-23T19:38:43.897378+0000","last_active":"2023-09-23T19:46:48.736916+0000","last_peered":"2023-09-23T19:46:48.736916+0000","last_clean":"2023-09-23T19:46:48.736916+0000","last_became_active":"2023-09-23T19:38:43.897012+0000","last_became_peered":"2023-09-23T19:38:43.897012+0000","last_unstale":"2023-09-23T19:46:48.736916+0000","last_undegraded":"2023-09-23T19:46:48.736916+0000","last_fullsized":"2023-09-23T19:46:48.736916+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":172,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.735732+0000","last_change":"2023-09-23T19:32:49.730549+0000","last_active":"2023-09-23T19:46:54.735732+0000","last_peered":"2023-09-23T19:46:54.735732+0000","last_clean":"2023-09-23T19:46:54.735732+0000","last_became_active":"2023-09-23T19:32:49.730105+0000","last_became_peered":"2023-09-23T19:32:49.730105+0000","last_unstale":"2023-09-23T19:46:54.735732+0000","last_undegraded":"2023-09-23T19:46:54.735732+0000","last_fullsized":"2023-09-23T19:46:54.735732+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:49.751893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":99,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.736595+0000","last_change":"2023-09-23T19:42:55.772691+0000","last_active":"2023-09-23T19:46:48.736595+0000","last_peered":"2023-09-23T19:46:48.736595+0000","last_clean":"2023-09-23T19:46:48.736595+0000","last_became_active":"2023-09-23T19:42:55.772309+0000","last_became_peered":"2023-09-23T19:42:55.772309+0000","last_unstale":"2023-09-23T19:46:48.736595+0000","last_undegraded":"2023-09-23T19:46:48.736595+0000","last_fullsized":"2023-09-23T19:46:48.736595+0000","mapping_epoch":188,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":106,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.737159+0000","last_change":"2023-09-23T19:43:38.577960+0000","last_active":"2023-09-23T19:46:48.737159+0000","last_peered":"2023-09-23T19:46:48.737159+0000","last_clean":"2023-09-23T19:46:48.737159+0000","last_became_active":"2023-09-23T19:43:38.577391+0000","last_became_peered":"2023-09-23T19:43:38.577391+0000","last_unstale":"2023-09-23T19:46:48.737159+0000","last_undegraded":"2023-09-23T19:46:48.737159+0000","last_fullsized":"2023-09-23T19:46:48.737159+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":164,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.736752+0000","last_change":"2023-09-23T19:05:52.114186+0000","last_active":"2023-09-23T19:46:48.736752+0000","last_peered":"2023-09-23T19:46:48.736752+0000","last_clean":"2023-09-23T19:46:48.736752+0000","last_became_active":"2023-09-23T19:05:52.113635+0000","last_became_peered":"2023-09-23T19:05:52.113635+0000","last_unstale":"2023-09-23T19:46:48.736752+0000","last_undegraded":"2023-09-23T19:46:48.736752+0000","last_fullsized":"2023-09-23T19:46:48.736752+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":99,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.737859+0000","last_change":"2023-09-23T19:42:55.772912+0000","last_active":"2023-09-23T19:46:48.737859+0000","last_peered":"2023-09-23T19:46:48.737859+0000","last_clean":"2023-09-23T19:46:48.737859+0000","last_became_active":"2023-09-23T19:42:55.772434+0000","last_became_peered":"2023-09-23T19:42:55.772434+0000","last_unstale":"2023-09-23T19:46:48.737859+0000","last_undegraded":"2023-09-23T19:46:48.737859+0000","last_fullsized":"2023-09-23T19:46:48.737859+0000","mapping_epoch":188,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":187,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.730549+0000","last_change":"2023-09-23T19:18:09.010195+0000","last_active":"2023-09-23T19:46:48.730549+0000","last_peered":"2023-09-23T19:46:48.730549+0000","last_clean":"2023-09-23T19:46:48.730549+0000","last_became_active":"2023-09-23T19:18:09.009687+0000","last_became_peered":"2023-09-23T19:18:09.009687+0000","last_unstale":"2023-09-23T19:46:48.730549+0000","last_undegraded":"2023-09-23T19:46:48.730549+0000","last_fullsized":"2023-09-23T19:46:48.730549+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":180,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.737491+0000","last_change":"2023-09-23T19:28:07.885265+0000","last_active":"2023-09-23T19:46:48.737491+0000","last_peered":"2023-09-23T19:46:48.737491+0000","last_clean":"2023-09-23T19:46:48.737491+0000","last_became_active":"2023-09-23T19:28:07.884353+0000","last_became_peered":"2023-09-23T19:28:07.884353+0000","last_unstale":"2023-09-23T19:46:48.737491+0000","last_undegraded":"2023-09-23T19:46:48.737491+0000","last_fullsized":"2023-09-23T19:46:48.737491+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:56.325352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.0","version":"213'3088","reported_seq":3102,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:47:04.559696+0000","last_change":"2023-09-23T19:46:17.720507+0000","last_active":"2023-09-23T19:47:04.559696+0000","last_peered":"2023-09-23T19:47:04.559696+0000","last_clean":"2023-09-23T19:47:04.559696+0000","last_became_active":"2023-09-23T19:46:17.720145+0000","last_became_peered":"2023-09-23T19:46:17.720145+0000","last_unstale":"2023-09-23T19:47:04.559696+0000","last_undegraded":"2023-09-23T19:47:04.559696+0000","last_fullsized":"2023-09-23T19:47:04.559696+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_clean_scrub_stamp":"2023-09-23T19:46:16.677636+0000","objects_scrubbed":0,"log_size":3088,"log_dups_size":0,"ondisk_log_size":3088,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":202375168,"num_objects":3088,"num_object_clones":0,"num_object_copies":9264,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3088,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3088,"num_write_kb":197632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":180,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.737943+0000","last_change":"2023-09-23T19:18:09.035116+0000","last_active":"2023-09-23T19:46:48.737943+0000","last_peered":"2023-09-23T19:46:48.737943+0000","last_clean":"2023-09-23T19:46:48.737943+0000","last_became_active":"2023-09-23T19:18:09.034416+0000","last_became_peered":"2023-09-23T19:18:09.034416+0000","last_unstale":"2023-09-23T19:46:48.737943+0000","last_undegraded":"2023-09-23T19:46:48.737943+0000","last_fullsized":"2023-09-23T19:46:48.737943+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":32,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.736608+0000","last_change":"2023-09-23T19:43:38.579831+0000","last_active":"2023-09-23T19:46:48.736608+0000","last_peered":"2023-09-23T19:46:48.736608+0000","last_clean":"2023-09-23T19:46:48.736608+0000","last_became_active":"2023-09-23T19:43:38.579398+0000","last_became_peered":"2023-09-23T19:43:38.579398+0000","last_unstale":"2023-09-23T19:46:48.736608+0000","last_undegraded":"2023-09-23T19:46:48.736608+0000","last_fullsized":"2023-09-23T19:46:48.736608+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.1","version":"213'2968","reported_seq":2982,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:47:04.737866+0000","last_change":"2023-09-23T19:46:17.828580+0000","last_active":"2023-09-23T19:47:04.737866+0000","last_peered":"2023-09-23T19:47:04.737866+0000","last_clean":"2023-09-23T19:47:04.737866+0000","last_became_active":"2023-09-23T19:46:17.827806+0000","last_became_peered":"2023-09-23T19:46:17.827806+0000","last_unstale":"2023-09-23T19:47:04.737866+0000","last_undegraded":"2023-09-23T19:47:04.737866+0000","last_fullsized":"2023-09-23T19:47:04.737866+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_clean_scrub_stamp":"2023-09-23T19:46:16.677636+0000","objects_scrubbed":0,"log_size":2968,"log_dups_size":0,"ondisk_log_size":2968,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":194510848,"num_objects":2968,"num_object_clones":0,"num_object_copies":8904,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2968,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2968,"num_write_kb":189952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":181,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.737983+0000","last_change":"2023-09-23T19:22:42.151134+0000","last_active":"2023-09-23T19:46:48.737983+0000","last_peered":"2023-09-23T19:46:48.737983+0000","last_clean":"2023-09-23T19:46:48.737983+0000","last_became_active":"2023-09-23T19:22:42.150553+0000","last_became_peered":"2023-09-23T19:22:42.150553+0000","last_unstale":"2023-09-23T19:46:48.737983+0000","last_undegraded":"2023-09-23T19:46:48.737983+0000","last_fullsized":"2023-09-23T19:46:48.737983+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":192,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.733161+0000","last_change":"2023-09-23T19:28:07.899262+0000","last_active":"2023-09-23T19:46:48.733161+0000","last_peered":"2023-09-23T19:46:48.733161+0000","last_clean":"2023-09-23T19:46:48.733161+0000","last_became_active":"2023-09-23T19:28:07.898848+0000","last_became_peered":"2023-09-23T19:28:07.898848+0000","last_unstale":"2023-09-23T19:46:48.733161+0000","last_undegraded":"2023-09-23T19:46:48.733161+0000","last_fullsized":"2023-09-23T19:46:48.733161+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"213'3009","reported_seq":3023,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:47:04.604910+0000","last_change":"2023-09-23T19:46:17.828372+0000","last_active":"2023-09-23T19:47:04.604910+0000","last_peered":"2023-09-23T19:47:04.604910+0000","last_clean":"2023-09-23T19:47:04.604910+0000","last_became_active":"2023-09-23T19:46:17.827674+0000","last_became_peered":"2023-09-23T19:46:17.827674+0000","last_unstale":"2023-09-23T19:47:04.604910+0000","last_undegraded":"2023-09-23T19:47:04.604910+0000","last_fullsized":"2023-09-23T19:47:04.604910+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_clean_scrub_stamp":"2023-09-23T19:46:16.677636+0000","objects_scrubbed":0,"log_size":3009,"log_dups_size":0,"ondisk_log_size":3009,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197197824,"num_objects":3009,"num_object_clones":0,"num_object_copies":9027,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3009,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3009,"num_write_kb":192576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":165,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.736923+0000","last_change":"2023-09-23T19:05:52.114108+0000","last_active":"2023-09-23T19:46:48.736923+0000","last_peered":"2023-09-23T19:46:48.736923+0000","last_clean":"2023-09-23T19:46:48.736923+0000","last_became_active":"2023-09-23T19:05:52.113609+0000","last_became_peered":"2023-09-23T19:05:52.113609+0000","last_unstale":"2023-09-23T19:46:48.736923+0000","last_undegraded":"2023-09-23T19:46:48.736923+0000","last_fullsized":"2023-09-23T19:46:48.736923+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":178,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.730283+0000","last_change":"2023-09-23T19:22:42.156825+0000","last_active":"2023-09-23T19:46:48.730283+0000","last_peered":"2023-09-23T19:46:48.730283+0000","last_clean":"2023-09-23T19:46:48.730283+0000","last_became_active":"2023-09-23T19:22:42.156474+0000","last_became_peered":"2023-09-23T19:22:42.156474+0000","last_unstale":"2023-09-23T19:46:48.730283+0000","last_undegraded":"2023-09-23T19:46:48.730283+0000","last_fullsized":"2023-09-23T19:46:48.730283+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"213'2957","reported_seq":2972,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:47:03.680281+0000","last_change":"2023-09-23T19:46:17.826712+0000","last_active":"2023-09-23T19:47:03.680281+0000","last_peered":"2023-09-23T19:47:03.680281+0000","last_clean":"2023-09-23T19:47:03.680281+0000","last_became_active":"2023-09-23T19:46:17.826319+0000","last_became_peered":"2023-09-23T19:46:17.826319+0000","last_unstale":"2023-09-23T19:47:03.680281+0000","last_undegraded":"2023-09-23T19:47:03.680281+0000","last_fullsized":"2023-09-23T19:47:03.680281+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_clean_scrub_stamp":"2023-09-23T19:46:16.677636+0000","objects_scrubbed":0,"log_size":2957,"log_dups_size":0,"ondisk_log_size":2957,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:22:46.891438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":193789952,"num_objects":2957,"num_object_clones":0,"num_object_copies":8871,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2957,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2957,"num_write_kb":189248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":107,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.730712+0000","last_change":"2023-09-23T19:43:38.576462+0000","last_active":"2023-09-23T19:46:48.730712+0000","last_peered":"2023-09-23T19:46:48.730712+0000","last_clean":"2023-09-23T19:46:48.730712+0000","last_became_active":"2023-09-23T19:43:38.576061+0000","last_became_peered":"2023-09-23T19:43:38.576061+0000","last_unstale":"2023-09-23T19:46:48.730712+0000","last_undegraded":"2023-09-23T19:46:48.730712+0000","last_fullsized":"2023-09-23T19:46:48.730712+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":176,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.729655+0000","last_change":"2023-09-23T19:22:42.135572+0000","last_active":"2023-09-23T19:46:48.729655+0000","last_peered":"2023-09-23T19:46:48.729655+0000","last_clean":"2023-09-23T19:46:48.729655+0000","last_became_active":"2023-09-23T19:22:42.135237+0000","last_became_peered":"2023-09-23T19:22:42.135237+0000","last_unstale":"2023-09-23T19:46:48.729655+0000","last_undegraded":"2023-09-23T19:46:48.729655+0000","last_fullsized":"2023-09-23T19:46:48.729655+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:35.838718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"213'3030","reported_seq":3044,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:47:04.736475+0000","last_change":"2023-09-23T19:46:17.828985+0000","last_active":"2023-09-23T19:47:04.736475+0000","last_peered":"2023-09-23T19:47:04.736475+0000","last_clean":"2023-09-23T19:47:04.736475+0000","last_became_active":"2023-09-23T19:46:17.827748+0000","last_became_peered":"2023-09-23T19:46:17.827748+0000","last_unstale":"2023-09-23T19:47:04.736475+0000","last_undegraded":"2023-09-23T19:47:04.736475+0000","last_fullsized":"2023-09-23T19:47:04.736475+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_clean_scrub_stamp":"2023-09-23T19:46:16.677636+0000","objects_scrubbed":0,"log_size":3030,"log_dups_size":0,"ondisk_log_size":3030,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":198574080,"num_objects":3030,"num_object_clones":0,"num_object_copies":9090,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3030,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3030,"num_write_kb":193920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":46,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.733422+0000","last_change":"2023-09-23T19:42:55.771907+0000","last_active":"2023-09-23T19:46:48.733422+0000","last_peered":"2023-09-23T19:46:48.733422+0000","last_clean":"2023-09-23T19:46:48.733422+0000","last_became_active":"2023-09-23T19:42:55.771503+0000","last_became_peered":"2023-09-23T19:42:55.771503+0000","last_unstale":"2023-09-23T19:46:48.733422+0000","last_undegraded":"2023-09-23T19:46:48.733422+0000","last_fullsized":"2023-09-23T19:46:48.733422+0000","mapping_epoch":188,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":177,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.733644+0000","last_change":"2023-09-23T19:28:07.896803+0000","last_active":"2023-09-23T19:46:48.733644+0000","last_peered":"2023-09-23T19:46:48.733644+0000","last_clean":"2023-09-23T19:46:48.733644+0000","last_became_active":"2023-09-23T19:28:07.896480+0000","last_became_peered":"2023-09-23T19:28:07.896480+0000","last_unstale":"2023-09-23T19:46:48.733644+0000","last_undegraded":"2023-09-23T19:46:48.733644+0000","last_fullsized":"2023-09-23T19:46:48.733644+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"213'2928","reported_seq":2942,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:47:04.714652+0000","last_change":"2023-09-23T19:46:17.717309+0000","last_active":"2023-09-23T19:47:04.714652+0000","last_peered":"2023-09-23T19:47:04.714652+0000","last_clean":"2023-09-23T19:47:04.714652+0000","last_became_active":"2023-09-23T19:46:17.716913+0000","last_became_peered":"2023-09-23T19:46:17.716913+0000","last_unstale":"2023-09-23T19:47:04.714652+0000","last_undegraded":"2023-09-23T19:47:04.714652+0000","last_fullsized":"2023-09-23T19:47:04.714652+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_clean_scrub_stamp":"2023-09-23T19:46:16.677636+0000","objects_scrubbed":0,"log_size":2928,"log_dups_size":0,"ondisk_log_size":2928,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191889408,"num_objects":2928,"num_object_clones":0,"num_object_copies":8784,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2928,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2928,"num_write_kb":187392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":179,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.730276+0000","last_change":"2023-09-23T19:22:42.156582+0000","last_active":"2023-09-23T19:46:48.730276+0000","last_peered":"2023-09-23T19:46:48.730276+0000","last_clean":"2023-09-23T19:46:48.730276+0000","last_became_active":"2023-09-23T19:22:42.156251+0000","last_became_peered":"2023-09-23T19:22:42.156251+0000","last_unstale":"2023-09-23T19:46:48.730276+0000","last_undegraded":"2023-09-23T19:46:48.730276+0000","last_fullsized":"2023-09-23T19:46:48.730276+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":207,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.733761+0000","last_change":"2023-09-23T19:38:43.891593+0000","last_active":"2023-09-23T19:46:48.733761+0000","last_peered":"2023-09-23T19:46:48.733761+0000","last_clean":"2023-09-23T19:46:48.733761+0000","last_became_active":"2023-09-23T19:38:43.891063+0000","last_became_peered":"2023-09-23T19:38:43.891063+0000","last_unstale":"2023-09-23T19:46:48.733761+0000","last_undegraded":"2023-09-23T19:46:48.733761+0000","last_fullsized":"2023-09-23T19:46:48.733761+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"213'3195","reported_seq":3210,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:47:05.897915+0000","last_change":"2023-09-23T19:46:17.707948+0000","last_active":"2023-09-23T19:47:05.897915+0000","last_peered":"2023-09-23T19:47:05.897915+0000","last_clean":"2023-09-23T19:47:05.897915+0000","last_became_active":"2023-09-23T19:46:17.707550+0000","last_became_peered":"2023-09-23T19:46:17.707550+0000","last_unstale":"2023-09-23T19:47:05.897915+0000","last_undegraded":"2023-09-23T19:47:05.897915+0000","last_fullsized":"2023-09-23T19:47:05.897915+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_clean_scrub_stamp":"2023-09-23T19:46:16.677636+0000","objects_scrubbed":0,"log_size":3195,"log_dups_size":0,"ondisk_log_size":3195,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:17:23.514645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209387520,"num_objects":3195,"num_object_clones":0,"num_object_copies":9585,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3195,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3195,"num_write_kb":204480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":193,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.732835+0000","last_change":"2023-09-23T19:38:43.892972+0000","last_active":"2023-09-23T19:46:48.732835+0000","last_peered":"2023-09-23T19:46:48.732835+0000","last_clean":"2023-09-23T19:46:48.732835+0000","last_became_active":"2023-09-23T19:38:43.892451+0000","last_became_peered":"2023-09-23T19:38:43.892451+0000","last_unstale":"2023-09-23T19:46:48.732835+0000","last_undegraded":"2023-09-23T19:46:48.732835+0000","last_fullsized":"2023-09-23T19:46:48.732835+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":177,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.733635+0000","last_change":"2023-09-23T19:22:42.149670+0000","last_active":"2023-09-23T19:46:48.733635+0000","last_peered":"2023-09-23T19:46:48.733635+0000","last_clean":"2023-09-23T19:46:48.733635+0000","last_became_active":"2023-09-23T19:22:42.149022+0000","last_became_peered":"2023-09-23T19:22:42.149022+0000","last_unstale":"2023-09-23T19:46:48.733635+0000","last_undegraded":"2023-09-23T19:46:48.733635+0000","last_fullsized":"2023-09-23T19:46:48.733635+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"213'3094","reported_seq":3109,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:47:05.943264+0000","last_change":"2023-09-23T19:46:17.707167+0000","last_active":"2023-09-23T19:47:05.943264+0000","last_peered":"2023-09-23T19:47:05.943264+0000","last_clean":"2023-09-23T19:47:05.943264+0000","last_became_active":"2023-09-23T19:46:17.706818+0000","last_became_peered":"2023-09-23T19:46:17.706818+0000","last_unstale":"2023-09-23T19:47:05.943264+0000","last_undegraded":"2023-09-23T19:47:05.943264+0000","last_fullsized":"2023-09-23T19:47:05.943264+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_clean_scrub_stamp":"2023-09-23T19:46:16.677636+0000","objects_scrubbed":0,"log_size":3094,"log_dups_size":0,"ondisk_log_size":3094,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:34:00.432106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":202768384,"num_objects":3094,"num_object_clones":0,"num_object_copies":9282,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3094,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3094,"num_write_kb":198016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":173,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.716911+0000","last_change":"2023-09-23T19:32:49.729244+0000","last_active":"2023-09-23T19:46:48.716911+0000","last_peered":"2023-09-23T19:46:48.716911+0000","last_clean":"2023-09-23T19:46:48.716911+0000","last_became_active":"2023-09-23T19:32:49.728810+0000","last_became_peered":"2023-09-23T19:32:49.728810+0000","last_unstale":"2023-09-23T19:46:48.716911+0000","last_undegraded":"2023-09-23T19:46:48.716911+0000","last_fullsized":"2023-09-23T19:46:48.716911+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:57.540826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":160,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.719571+0000","last_change":"2023-09-23T19:28:07.889095+0000","last_active":"2023-09-23T19:46:48.719571+0000","last_peered":"2023-09-23T19:46:48.719571+0000","last_clean":"2023-09-23T19:46:48.719571+0000","last_became_active":"2023-09-23T19:28:07.888742+0000","last_became_peered":"2023-09-23T19:28:07.888742+0000","last_unstale":"2023-09-23T19:46:48.719571+0000","last_undegraded":"2023-09-23T19:46:48.719571+0000","last_fullsized":"2023-09-23T19:46:48.719571+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:43.929488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.a","version":"213'3234","reported_seq":3249,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:47:07.707171+0000","last_change":"2023-09-23T19:46:17.821771+0000","last_active":"2023-09-23T19:47:07.707171+0000","last_peered":"2023-09-23T19:47:07.707171+0000","last_clean":"2023-09-23T19:47:07.707171+0000","last_became_active":"2023-09-23T19:46:17.821143+0000","last_became_peered":"2023-09-23T19:46:17.821143+0000","last_unstale":"2023-09-23T19:47:07.707171+0000","last_undegraded":"2023-09-23T19:47:07.707171+0000","last_fullsized":"2023-09-23T19:47:07.707171+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_clean_scrub_stamp":"2023-09-23T19:46:16.677636+0000","objects_scrubbed":0,"log_size":3234,"log_dups_size":0,"ondisk_log_size":3234,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:18:56.268551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":211943424,"num_objects":3234,"num_object_clones":0,"num_object_copies":9702,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3234,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3234,"num_write_kb":206976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":194,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.719278+0000","last_change":"2023-09-23T19:22:42.143289+0000","last_active":"2023-09-23T19:46:48.719278+0000","last_peered":"2023-09-23T19:46:48.719278+0000","last_clean":"2023-09-23T19:46:48.719278+0000","last_became_active":"2023-09-23T19:22:42.142656+0000","last_became_peered":"2023-09-23T19:22:42.142656+0000","last_unstale":"2023-09-23T19:46:48.719278+0000","last_undegraded":"2023-09-23T19:46:48.719278+0000","last_fullsized":"2023-09-23T19:46:48.719278+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:38.467441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":181,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.734844+0000","last_change":"2023-09-23T19:43:38.571832+0000","last_active":"2023-09-23T19:46:48.734844+0000","last_peered":"2023-09-23T19:46:48.734844+0000","last_clean":"2023-09-23T19:46:48.734844+0000","last_became_active":"2023-09-23T19:43:38.571403+0000","last_became_peered":"2023-09-23T19:43:38.571403+0000","last_unstale":"2023-09-23T19:46:48.734844+0000","last_undegraded":"2023-09-23T19:46:48.734844+0000","last_fullsized":"2023-09-23T19:46:48.734844+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:14:50.291822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":180,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.740910+0000","last_change":"2023-09-23T19:38:43.898519+0000","last_active":"2023-09-23T19:46:54.740910+0000","last_peered":"2023-09-23T19:46:54.740910+0000","last_clean":"2023-09-23T19:46:54.740910+0000","last_became_active":"2023-09-23T19:38:43.896814+0000","last_became_peered":"2023-09-23T19:38:43.896814+0000","last_unstale":"2023-09-23T19:46:54.740910+0000","last_undegraded":"2023-09-23T19:46:54.740910+0000","last_fullsized":"2023-09-23T19:46:54.740910+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:40.465038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.c","version":"213'2951","reported_seq":2966,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:47:03.695491+0000","last_change":"2023-09-23T19:46:17.890125+0000","last_active":"2023-09-23T19:47:03.695491+0000","last_peered":"2023-09-23T19:47:03.695491+0000","last_clean":"2023-09-23T19:47:03.695491+0000","last_became_active":"2023-09-23T19:46:17.889428+0000","last_became_peered":"2023-09-23T19:46:17.889428+0000","last_unstale":"2023-09-23T19:47:03.695491+0000","last_undegraded":"2023-09-23T19:47:03.695491+0000","last_fullsized":"2023-09-23T19:47:03.695491+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_clean_scrub_stamp":"2023-09-23T19:46:16.677636+0000","objects_scrubbed":0,"log_size":2951,"log_dups_size":0,"ondisk_log_size":2951,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:12:54.937796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":193396736,"num_objects":2951,"num_object_clones":0,"num_object_copies":8853,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2951,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2951,"num_write_kb":188864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":94,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.716919+0000","last_change":"2023-09-23T19:43:38.612700+0000","last_active":"2023-09-23T19:46:48.716919+0000","last_peered":"2023-09-23T19:46:48.716919+0000","last_clean":"2023-09-23T19:46:48.716919+0000","last_became_active":"2023-09-23T19:43:38.612424+0000","last_became_peered":"2023-09-23T19:43:38.612424+0000","last_unstale":"2023-09-23T19:46:48.716919+0000","last_undegraded":"2023-09-23T19:46:48.716919+0000","last_fullsized":"2023-09-23T19:46:48.716919+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:24.833490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":178,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.733696+0000","last_change":"2023-09-23T19:18:09.017525+0000","last_active":"2023-09-23T19:46:48.733696+0000","last_peered":"2023-09-23T19:46:48.733696+0000","last_clean":"2023-09-23T19:46:48.733696+0000","last_became_active":"2023-09-23T19:18:09.017193+0000","last_became_peered":"2023-09-23T19:18:09.017193+0000","last_unstale":"2023-09-23T19:46:48.733696+0000","last_undegraded":"2023-09-23T19:46:48.733696+0000","last_fullsized":"2023-09-23T19:46:48.733696+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":191,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.719202+0000","last_change":"2023-09-23T19:38:43.877088+0000","last_active":"2023-09-23T19:46:48.719202+0000","last_peered":"2023-09-23T19:46:48.719202+0000","last_clean":"2023-09-23T19:46:48.719202+0000","last_became_active":"2023-09-23T19:38:43.876719+0000","last_became_peered":"2023-09-23T19:38:43.876719+0000","last_unstale":"2023-09-23T19:46:48.719202+0000","last_undegraded":"2023-09-23T19:46:48.719202+0000","last_fullsized":"2023-09-23T19:46:48.719202+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:05.423256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.f","version":"213'3121","reported_seq":3135,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:47:04.737002+0000","last_change":"2023-09-23T19:46:17.895413+0000","last_active":"2023-09-23T19:47:04.737002+0000","last_peered":"2023-09-23T19:47:04.737002+0000","last_clean":"2023-09-23T19:47:04.737002+0000","last_became_active":"2023-09-23T19:46:17.894888+0000","last_became_peered":"2023-09-23T19:46:17.894888+0000","last_unstale":"2023-09-23T19:47:04.737002+0000","last_undegraded":"2023-09-23T19:47:04.737002+0000","last_fullsized":"2023-09-23T19:47:04.737002+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_clean_scrub_stamp":"2023-09-23T19:46:16.677636+0000","objects_scrubbed":0,"log_size":3121,"log_dups_size":0,"ondisk_log_size":3121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":204537856,"num_objects":3121,"num_object_clones":0,"num_object_copies":9363,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3121,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3121,"num_write_kb":199744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":160,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.742002+0000","last_change":"2023-09-23T19:38:43.883663+0000","last_active":"2023-09-23T19:46:54.742002+0000","last_peered":"2023-09-23T19:46:54.742002+0000","last_clean":"2023-09-23T19:46:54.742002+0000","last_became_active":"2023-09-23T19:38:43.882421+0000","last_became_peered":"2023-09-23T19:38:43.882421+0000","last_unstale":"2023-09-23T19:46:54.742002+0000","last_undegraded":"2023-09-23T19:46:54.742002+0000","last_fullsized":"2023-09-23T19:46:54.742002+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:49:01.607659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":177,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.716832+0000","last_change":"2023-09-23T19:38:43.890600+0000","last_active":"2023-09-23T19:46:48.716832+0000","last_peered":"2023-09-23T19:46:48.716832+0000","last_clean":"2023-09-23T19:46:48.716832+0000","last_became_active":"2023-09-23T19:38:43.890253+0000","last_became_peered":"2023-09-23T19:38:43.890253+0000","last_unstale":"2023-09-23T19:46:48.716832+0000","last_undegraded":"2023-09-23T19:46:48.716832+0000","last_fullsized":"2023-09-23T19:46:48.716832+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:46.751120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":189,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.719647+0000","last_change":"2023-09-23T19:22:42.143733+0000","last_active":"2023-09-23T19:46:48.719647+0000","last_peered":"2023-09-23T19:46:48.719647+0000","last_clean":"2023-09-23T19:46:48.719647+0000","last_became_active":"2023-09-23T19:22:42.143077+0000","last_became_peered":"2023-09-23T19:22:42.143077+0000","last_unstale":"2023-09-23T19:46:48.719647+0000","last_undegraded":"2023-09-23T19:46:48.719647+0000","last_fullsized":"2023-09-23T19:46:48.719647+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:47:33.086994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.d","version":"213'3034","reported_seq":3047,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:47:04.714285+0000","last_change":"2023-09-23T19:46:17.714589+0000","last_active":"2023-09-23T19:47:04.714285+0000","last_peered":"2023-09-23T19:47:04.714285+0000","last_clean":"2023-09-23T19:47:04.714285+0000","last_became_active":"2023-09-23T19:46:17.714169+0000","last_became_peered":"2023-09-23T19:46:17.714169+0000","last_unstale":"2023-09-23T19:47:04.714285+0000","last_undegraded":"2023-09-23T19:47:04.714285+0000","last_fullsized":"2023-09-23T19:47:04.714285+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_clean_scrub_stamp":"2023-09-23T19:46:16.677636+0000","objects_scrubbed":0,"log_size":3034,"log_dups_size":0,"ondisk_log_size":3034,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":198836224,"num_objects":3034,"num_object_clones":0,"num_object_copies":9102,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3034,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3034,"num_write_kb":194176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":170,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.735115+0000","last_change":"2023-09-23T19:43:38.571764+0000","last_active":"2023-09-23T19:46:48.735115+0000","last_peered":"2023-09-23T19:46:48.735115+0000","last_clean":"2023-09-23T19:46:48.735115+0000","last_became_active":"2023-09-23T19:43:38.571417+0000","last_became_peered":"2023-09-23T19:43:38.571417+0000","last_unstale":"2023-09-23T19:46:48.735115+0000","last_undegraded":"2023-09-23T19:46:48.735115+0000","last_fullsized":"2023-09-23T19:46:48.735115+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:22.165556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":415,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.736469+0000","last_change":"2023-09-23T19:18:09.016297+0000","last_active":"2023-09-23T19:46:48.736469+0000","last_peered":"2023-09-23T19:46:48.736469+0000","last_clean":"2023-09-23T19:46:48.736469+0000","last_became_active":"2023-09-23T19:18:09.015436+0000","last_became_peered":"2023-09-23T19:18:09.015436+0000","last_unstale":"2023-09-23T19:46:48.736469+0000","last_undegraded":"2023-09-23T19:46:48.736469+0000","last_fullsized":"2023-09-23T19:46:48.736469+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:59:33.840751+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:59:33.840751+0000","last_clean_scrub_stamp":"2023-09-23T18:59:33.840751+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":90,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.731036+0000","last_change":"2023-09-23T19:43:38.576727+0000","last_active":"2023-09-23T19:46:48.731036+0000","last_peered":"2023-09-23T19:46:48.731036+0000","last_clean":"2023-09-23T19:46:48.731036+0000","last_became_active":"2023-09-23T19:43:38.576349+0000","last_became_peered":"2023-09-23T19:43:38.576349+0000","last_unstale":"2023-09-23T19:46:48.731036+0000","last_undegraded":"2023-09-23T19:46:48.731036+0000","last_fullsized":"2023-09-23T19:46:48.731036+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":163,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.731154+0000","last_change":"2023-09-23T19:05:52.786771+0000","last_active":"2023-09-23T19:46:48.731154+0000","last_peered":"2023-09-23T19:46:48.731154+0000","last_clean":"2023-09-23T19:46:48.731154+0000","last_became_active":"2023-09-23T19:05:52.785797+0000","last_became_peered":"2023-09-23T19:05:52.785797+0000","last_unstale":"2023-09-23T19:46:48.731154+0000","last_undegraded":"2023-09-23T19:46:48.731154+0000","last_fullsized":"2023-09-23T19:46:48.731154+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"213'2926","reported_seq":2939,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:47:03.693440+0000","last_change":"2023-09-23T19:46:17.713260+0000","last_active":"2023-09-23T19:47:03.693440+0000","last_peered":"2023-09-23T19:47:03.693440+0000","last_clean":"2023-09-23T19:47:03.693440+0000","last_became_active":"2023-09-23T19:46:17.712923+0000","last_became_peered":"2023-09-23T19:46:17.712923+0000","last_unstale":"2023-09-23T19:47:03.693440+0000","last_undegraded":"2023-09-23T19:47:03.693440+0000","last_fullsized":"2023-09-23T19:47:03.693440+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_clean_scrub_stamp":"2023-09-23T19:46:16.677636+0000","objects_scrubbed":0,"log_size":2926,"log_dups_size":0,"ondisk_log_size":2926,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:25:32.003178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191758336,"num_objects":2926,"num_object_clones":0,"num_object_copies":8778,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2926,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2926,"num_write_kb":187264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":198,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.736453+0000","last_change":"2023-09-23T19:22:42.150636+0000","last_active":"2023-09-23T19:46:48.736453+0000","last_peered":"2023-09-23T19:46:48.736453+0000","last_clean":"2023-09-23T19:46:48.736453+0000","last_became_active":"2023-09-23T19:22:42.150045+0000","last_became_peered":"2023-09-23T19:22:42.150045+0000","last_unstale":"2023-09-23T19:46:48.736453+0000","last_undegraded":"2023-09-23T19:46:48.736453+0000","last_fullsized":"2023-09-23T19:46:48.736453+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":99,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.733306+0000","last_change":"2023-09-23T19:42:55.771906+0000","last_active":"2023-09-23T19:46:48.733306+0000","last_peered":"2023-09-23T19:46:48.733306+0000","last_clean":"2023-09-23T19:46:48.733306+0000","last_became_active":"2023-09-23T19:42:55.771599+0000","last_became_peered":"2023-09-23T19:42:55.771599+0000","last_unstale":"2023-09-23T19:46:48.733306+0000","last_undegraded":"2023-09-23T19:46:48.733306+0000","last_fullsized":"2023-09-23T19:46:48.733306+0000","mapping_epoch":188,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":207,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.733220+0000","last_change":"2023-09-23T19:38:43.876930+0000","last_active":"2023-09-23T19:46:48.733220+0000","last_peered":"2023-09-23T19:46:48.733220+0000","last_clean":"2023-09-23T19:46:48.733220+0000","last_became_active":"2023-09-23T19:38:43.876571+0000","last_became_peered":"2023-09-23T19:38:43.876571+0000","last_unstale":"2023-09-23T19:46:48.733220+0000","last_undegraded":"2023-09-23T19:46:48.733220+0000","last_fullsized":"2023-09-23T19:46:48.733220+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:30.924536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.b","version":"213'3319","reported_seq":3334,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:47:07.752945+0000","last_change":"2023-09-23T19:46:17.828761+0000","last_active":"2023-09-23T19:47:07.752945+0000","last_peered":"2023-09-23T19:47:07.752945+0000","last_clean":"2023-09-23T19:47:07.752945+0000","last_became_active":"2023-09-23T19:46:17.828132+0000","last_became_peered":"2023-09-23T19:46:17.828132+0000","last_unstale":"2023-09-23T19:47:07.752945+0000","last_undegraded":"2023-09-23T19:47:07.752945+0000","last_fullsized":"2023-09-23T19:47:07.752945+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_clean_scrub_stamp":"2023-09-23T19:46:16.677636+0000","objects_scrubbed":0,"log_size":3319,"log_dups_size":0,"ondisk_log_size":3319,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:50:22.166366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217513984,"num_objects":3319,"num_object_clones":0,"num_object_copies":9957,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3319,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3319,"num_write_kb":212416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":183,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.736974+0000","last_change":"2023-09-23T19:18:09.015853+0000","last_active":"2023-09-23T19:46:48.736974+0000","last_peered":"2023-09-23T19:46:48.736974+0000","last_clean":"2023-09-23T19:46:48.736974+0000","last_became_active":"2023-09-23T19:18:09.015338+0000","last_became_peered":"2023-09-23T19:18:09.015338+0000","last_unstale":"2023-09-23T19:46:48.736974+0000","last_undegraded":"2023-09-23T19:46:48.736974+0000","last_fullsized":"2023-09-23T19:46:48.736974+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":176,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.719120+0000","last_change":"2023-09-23T19:38:43.881233+0000","last_active":"2023-09-23T19:46:48.719120+0000","last_peered":"2023-09-23T19:46:48.719120+0000","last_clean":"2023-09-23T19:46:48.719120+0000","last_became_active":"2023-09-23T19:38:43.880893+0000","last_became_peered":"2023-09-23T19:38:43.880893+0000","last_unstale":"2023-09-23T19:46:48.719120+0000","last_undegraded":"2023-09-23T19:46:48.719120+0000","last_fullsized":"2023-09-23T19:46:48.719120+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:11.206947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":48,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.736633+0000","last_change":"2023-09-23T19:43:38.578086+0000","last_active":"2023-09-23T19:46:48.736633+0000","last_peered":"2023-09-23T19:46:48.736633+0000","last_clean":"2023-09-23T19:46:48.736633+0000","last_became_active":"2023-09-23T19:43:38.577623+0000","last_became_peered":"2023-09-23T19:43:38.577623+0000","last_unstale":"2023-09-23T19:46:48.736633+0000","last_undegraded":"2023-09-23T19:46:48.736633+0000","last_fullsized":"2023-09-23T19:46:48.736633+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.8","version":"213'3313","reported_seq":3328,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:47:07.646912+0000","last_change":"2023-09-23T19:46:17.821909+0000","last_active":"2023-09-23T19:47:07.646912+0000","last_peered":"2023-09-23T19:47:07.646912+0000","last_clean":"2023-09-23T19:47:07.646912+0000","last_became_active":"2023-09-23T19:46:17.821585+0000","last_became_peered":"2023-09-23T19:46:17.821585+0000","last_unstale":"2023-09-23T19:47:07.646912+0000","last_undegraded":"2023-09-23T19:47:07.646912+0000","last_fullsized":"2023-09-23T19:47:07.646912+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_clean_scrub_stamp":"2023-09-23T19:46:16.677636+0000","objects_scrubbed":0,"log_size":3313,"log_dups_size":0,"ondisk_log_size":3313,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:49.735671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217120768,"num_objects":3313,"num_object_clones":0,"num_object_copies":9939,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3313,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3313,"num_write_kb":212032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":117,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.737772+0000","last_change":"2023-09-23T19:32:49.737323+0000","last_active":"2023-09-23T19:46:48.737772+0000","last_peered":"2023-09-23T19:46:48.737772+0000","last_clean":"2023-09-23T19:46:48.737772+0000","last_became_active":"2023-09-23T19:32:49.736937+0000","last_became_peered":"2023-09-23T19:32:49.736937+0000","last_unstale":"2023-09-23T19:46:48.737772+0000","last_undegraded":"2023-09-23T19:46:48.737772+0000","last_fullsized":"2023-09-23T19:46:48.737772+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":146,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.720817+0000","last_change":"2023-09-23T19:22:42.145506+0000","last_active":"2023-09-23T19:46:48.720817+0000","last_peered":"2023-09-23T19:46:48.720817+0000","last_clean":"2023-09-23T19:46:48.720817+0000","last_became_active":"2023-09-23T19:22:42.143991+0000","last_became_peered":"2023-09-23T19:22:42.143991+0000","last_unstale":"2023-09-23T19:46:48.720817+0000","last_undegraded":"2023-09-23T19:46:48.720817+0000","last_fullsized":"2023-09-23T19:46:48.720817+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:56.875175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":154,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.736809+0000","last_change":"2023-09-23T19:32:49.734592+0000","last_active":"2023-09-23T19:46:48.736809+0000","last_peered":"2023-09-23T19:46:48.736809+0000","last_clean":"2023-09-23T19:46:48.736809+0000","last_became_active":"2023-09-23T19:32:49.733905+0000","last_became_peered":"2023-09-23T19:32:49.733905+0000","last_unstale":"2023-09-23T19:46:48.736809+0000","last_undegraded":"2023-09-23T19:46:48.736809+0000","last_fullsized":"2023-09-23T19:46:48.736809+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.9","version":"213'3325","reported_seq":3340,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:47:07.687291+0000","last_change":"2023-09-23T19:46:17.826857+0000","last_active":"2023-09-23T19:47:07.687291+0000","last_peered":"2023-09-23T19:47:07.687291+0000","last_clean":"2023-09-23T19:47:07.687291+0000","last_became_active":"2023-09-23T19:46:17.826182+0000","last_became_peered":"2023-09-23T19:46:17.826182+0000","last_unstale":"2023-09-23T19:47:07.687291+0000","last_undegraded":"2023-09-23T19:47:07.687291+0000","last_fullsized":"2023-09-23T19:47:07.687291+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_clean_scrub_stamp":"2023-09-23T19:46:16.677636+0000","objects_scrubbed":0,"log_size":3325,"log_dups_size":0,"ondisk_log_size":3325,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:32:44.646144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217907200,"num_objects":3325,"num_object_clones":0,"num_object_copies":9975,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3325,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3325,"num_write_kb":212800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":198,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.731308+0000","last_change":"2023-09-23T19:38:43.895887+0000","last_active":"2023-09-23T19:46:48.731308+0000","last_peered":"2023-09-23T19:46:48.731308+0000","last_clean":"2023-09-23T19:46:48.731308+0000","last_became_active":"2023-09-23T19:38:43.895486+0000","last_became_peered":"2023-09-23T19:38:43.895486+0000","last_unstale":"2023-09-23T19:46:48.731308+0000","last_undegraded":"2023-09-23T19:46:48.731308+0000","last_fullsized":"2023-09-23T19:46:48.731308+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":180,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.732891+0000","last_change":"2023-09-23T19:22:42.149694+0000","last_active":"2023-09-23T19:46:48.732891+0000","last_peered":"2023-09-23T19:46:48.732891+0000","last_clean":"2023-09-23T19:46:48.732891+0000","last_became_active":"2023-09-23T19:22:42.148194+0000","last_became_peered":"2023-09-23T19:22:42.148194+0000","last_unstale":"2023-09-23T19:46:48.732891+0000","last_undegraded":"2023-09-23T19:46:48.732891+0000","last_fullsized":"2023-09-23T19:46:48.732891+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:47.842383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":162,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.732932+0000","last_change":"2023-09-23T19:05:52.138620+0000","last_active":"2023-09-23T19:46:48.732932+0000","last_peered":"2023-09-23T19:46:48.732932+0000","last_clean":"2023-09-23T19:46:48.732932+0000","last_became_active":"2023-09-23T19:05:52.138243+0000","last_became_peered":"2023-09-23T19:05:52.138243+0000","last_unstale":"2023-09-23T19:46:48.732932+0000","last_undegraded":"2023-09-23T19:46:48.732932+0000","last_fullsized":"2023-09-23T19:46:48.732932+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":163,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.729744+0000","last_change":"2023-09-23T19:18:09.008811+0000","last_active":"2023-09-23T19:46:48.729744+0000","last_peered":"2023-09-23T19:46:48.729744+0000","last_clean":"2023-09-23T19:46:48.729744+0000","last_became_active":"2023-09-23T19:18:09.007255+0000","last_became_peered":"2023-09-23T19:18:09.007255+0000","last_unstale":"2023-09-23T19:46:48.729744+0000","last_undegraded":"2023-09-23T19:46:48.729744+0000","last_fullsized":"2023-09-23T19:46:48.729744+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:25.874038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":167,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.734838+0000","last_change":"2023-09-23T19:38:43.877244+0000","last_active":"2023-09-23T19:46:48.734838+0000","last_peered":"2023-09-23T19:46:48.734838+0000","last_clean":"2023-09-23T19:46:48.734838+0000","last_became_active":"2023-09-23T19:38:43.876896+0000","last_became_peered":"2023-09-23T19:38:43.876896+0000","last_unstale":"2023-09-23T19:46:48.734838+0000","last_undegraded":"2023-09-23T19:46:48.734838+0000","last_fullsized":"2023-09-23T19:46:48.734838+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:22.288276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":42,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.733331+0000","last_change":"2023-09-23T19:43:38.620732+0000","last_active":"2023-09-23T19:46:48.733331+0000","last_peered":"2023-09-23T19:46:48.733331+0000","last_clean":"2023-09-23T19:46:48.733331+0000","last_became_active":"2023-09-23T19:43:38.619424+0000","last_became_peered":"2023-09-23T19:43:38.619424+0000","last_unstale":"2023-09-23T19:46:48.733331+0000","last_undegraded":"2023-09-23T19:46:48.733331+0000","last_fullsized":"2023-09-23T19:46:48.733331+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":216,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.734028+0000","last_change":"2023-09-23T19:38:43.947451+0000","last_active":"2023-09-23T19:46:48.734028+0000","last_peered":"2023-09-23T19:46:48.734028+0000","last_clean":"2023-09-23T19:46:48.734028+0000","last_became_active":"2023-09-23T19:38:43.946420+0000","last_became_peered":"2023-09-23T19:38:43.946420+0000","last_unstale":"2023-09-23T19:46:48.734028+0000","last_undegraded":"2023-09-23T19:46:48.734028+0000","last_fullsized":"2023-09-23T19:46:48.734028+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":42,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.733898+0000","last_change":"2023-09-23T19:43:38.577015+0000","last_active":"2023-09-23T19:46:48.733898+0000","last_peered":"2023-09-23T19:46:48.733898+0000","last_clean":"2023-09-23T19:46:48.733898+0000","last_became_active":"2023-09-23T19:43:38.576358+0000","last_became_peered":"2023-09-23T19:43:38.576358+0000","last_unstale":"2023-09-23T19:46:48.733898+0000","last_undegraded":"2023-09-23T19:46:48.733898+0000","last_fullsized":"2023-09-23T19:46:48.733898+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":192,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.734598+0000","last_change":"2023-09-23T19:28:07.896983+0000","last_active":"2023-09-23T19:46:48.734598+0000","last_peered":"2023-09-23T19:46:48.734598+0000","last_clean":"2023-09-23T19:46:48.734598+0000","last_became_active":"2023-09-23T19:28:07.896577+0000","last_became_peered":"2023-09-23T19:28:07.896577+0000","last_unstale":"2023-09-23T19:46:48.734598+0000","last_undegraded":"2023-09-23T19:46:48.734598+0000","last_fullsized":"2023-09-23T19:46:48.734598+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":122,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.730807+0000","last_change":"2023-09-23T19:38:43.900225+0000","last_active":"2023-09-23T19:46:48.730807+0000","last_peered":"2023-09-23T19:46:48.730807+0000","last_clean":"2023-09-23T19:46:48.730807+0000","last_became_active":"2023-09-23T19:38:43.899907+0000","last_became_peered":"2023-09-23T19:38:43.899907+0000","last_unstale":"2023-09-23T19:46:48.730807+0000","last_undegraded":"2023-09-23T19:46:48.730807+0000","last_fullsized":"2023-09-23T19:46:48.730807+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":176,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.735016+0000","last_change":"2023-09-23T19:38:43.882487+0000","last_active":"2023-09-23T19:46:48.735016+0000","last_peered":"2023-09-23T19:46:48.735016+0000","last_clean":"2023-09-23T19:46:48.735016+0000","last_became_active":"2023-09-23T19:38:43.881931+0000","last_became_peered":"2023-09-23T19:38:43.881931+0000","last_unstale":"2023-09-23T19:46:48.735016+0000","last_undegraded":"2023-09-23T19:46:48.735016+0000","last_fullsized":"2023-09-23T19:46:48.735016+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:02:36.906535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":195,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.733123+0000","last_change":"2023-09-23T19:38:43.893983+0000","last_active":"2023-09-23T19:46:48.733123+0000","last_peered":"2023-09-23T19:46:48.733123+0000","last_clean":"2023-09-23T19:46:48.733123+0000","last_became_active":"2023-09-23T19:38:43.893527+0000","last_became_peered":"2023-09-23T19:38:43.893527+0000","last_unstale":"2023-09-23T19:46:48.733123+0000","last_undegraded":"2023-09-23T19:46:48.733123+0000","last_fullsized":"2023-09-23T19:46:48.733123+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:12.156531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":171,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T19:46:48.732873+0000","last_change":"2023-09-23T19:06:36.673002+0000","last_active":"2023-09-23T19:46:48.732873+0000","last_peered":"2023-09-23T19:46:48.732873+0000","last_clean":"2023-09-23T19:46:48.732873+0000","last_became_active":"2023-09-23T19:06:36.671911+0000","last_became_peered":"2023-09-23T19:06:36.671911+0000","last_unstale":"2023-09-23T19:46:48.732873+0000","last_undegraded":"2023-09-23T19:46:48.732873+0000","last_fullsized":"2023-09-23T19:46:48.732873+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":3243507712,"num_objects":49492,"num_object_clones":0,"num_object_copies":148476,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":49492,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":49492,"num_write_kb":3167488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9773776896,"data_stored":9773776896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":49492,"ondisk_log_size":49492,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430739,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2252100,"kb_used_data":1352124,"kb_used_omap":0,"kb_used_meta":899968,"kb_avail":91493052,"statfs":{"total":95995035648,"available":93688885248,"internally_reserved":0,"allocated":1384574976,"data_stored":1384057712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":921567232},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:46:59 2023","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":1.351,"15min":1.9670000000000001},"min":{"1min":0.30399999999999999,"5min":0.26400000000000001,"15min":0.23100000000000001},"max":{"1min":4.0350000000000001,"5min":6.8940000000000001,"15min":92.944999999999993},"last":0.441},{"interface":"front","average":{"1min":0.95499999999999996,"5min":1.7050000000000001,"15min":2.0569999999999999},"min":{"1min":0.34699999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":3.476,"5min":10.542,"15min":92.763999999999996},"last":3.476}]},{"osd":1,"last update":"Sat Sep 23 19:46:21 2023","interfaces":[{"interface":"back","average":{"1min":1.9690000000000001,"5min":2.7120000000000002,"15min":2.3580000000000001},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.28799999999999998},"max":{"1min":6.7220000000000004,"5min":30.780999999999999,"15min":93.757999999999996},"last":1.0429999999999999},{"interface":"front","average":{"1min":2.157,"5min":2.7509999999999999,"15min":2.3620000000000001},"min":{"1min":0.53700000000000003,"5min":0.35399999999999998,"15min":0.29999999999999999},"max":{"1min":6.9199999999999999,"5min":30.827999999999999,"15min":93.835999999999999},"last":0.47799999999999998}]},{"osd":2,"last update":"Sat Sep 23 19:46:59 2023","interfaces":[{"interface":"back","average":{"1min":0.83699999999999997,"5min":2.0350000000000001,"15min":1.968},"min":{"1min":0.308,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":1.2869999999999999,"5min":21.163,"15min":21.163},"last":1.169},{"interface":"front","average":{"1min":0.96899999999999997,"5min":2.097,"15min":1.5549999999999999},"min":{"1min":0.42299999999999999,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":3.625,"5min":21.233000000000001,"15min":21.233000000000001},"last":0.42299999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:46:41 2023","interfaces":[{"interface":"back","average":{"1min":1.3600000000000001,"5min":1.806,"15min":1.6679999999999999},"min":{"1min":0.44800000000000001,"5min":0.312,"15min":0.312},"max":{"1min":4.7770000000000001,"5min":8.843,"15min":19.637},"last":2.2069999999999999},{"interface":"front","average":{"1min":1.157,"5min":1.6970000000000001,"15min":1.633},"min":{"1min":0.55100000000000005,"5min":0.41199999999999998,"15min":0.33900000000000002},"max":{"1min":3.6400000000000001,"5min":7.4279999999999999,"15min":20.204000000000001},"last":2.1909999999999998}]},{"osd":4,"last update":"Sat Sep 23 19:46:21 2023","interfaces":[{"interface":"back","average":{"1min":1.357,"5min":1.321,"15min":1.319},"min":{"1min":0.38500000000000001,"5min":0.29199999999999998,"15min":0.252},"max":{"1min":4.7439999999999998,"5min":8.8010000000000002,"15min":9.7699999999999996},"last":0.95599999999999996},{"interface":"front","average":{"1min":1.796,"5min":1.401,"15min":1.3979999999999999},"min":{"1min":0.374,"5min":0.36199999999999999,"15min":0.27400000000000002},"max":{"1min":6.7779999999999996,"5min":8.0579999999999998,"15min":13.385},"last":0.501}]},{"osd":5,"last update":"Sat Sep 23 19:46:21 2023","interfaces":[{"interface":"back","average":{"1min":2.1680000000000001,"5min":1.3560000000000001,"15min":1.625},"min":{"1min":0.41799999999999998,"5min":0.34200000000000003,"15min":0.28499999999999998},"max":{"1min":10.233000000000001,"5min":10.233000000000001,"15min":13.342000000000001},"last":1.972},{"interface":"front","average":{"1min":1.4650000000000001,"5min":1.411,"15min":1.631},"min":{"1min":0.495,"5min":0.34200000000000003,"15min":0.31},"max":{"1min":4.8090000000000002,"5min":8.9100000000000001,"15min":9.8979999999999997},"last":1.097}]},{"osd":6,"last update":"Sat Sep 23 19:46:21 2023","interfaces":[{"interface":"back","average":{"1min":2.2749999999999999,"5min":1.903,"15min":1.7390000000000001},"min":{"1min":0.42099999999999999,"5min":0.33300000000000002,"15min":0.316},"max":{"1min":7.5250000000000004,"5min":8.8829999999999991,"15min":22.725000000000001},"last":1.387},{"interface":"front","average":{"1min":2.0030000000000001,"5min":1.9239999999999999,"15min":1.8440000000000001},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.308},"max":{"1min":6.8659999999999997,"5min":10.073,"15min":22.707000000000001},"last":0.71099999999999997}]}]},{"osd":6,"up_from":42,"seq":180388626971,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2241012,"kb_used_data":1353900,"kb_used_omap":0,"kb_used_meta":887104,"kb_avail":91504140,"statfs":{"total":95995035648,"available":93700239360,"internally_reserved":0,"allocated":1386393600,"data_stored":1385892259,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":908394496},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:47:03 2023","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":1.9259999999999999,"15min":2.149},"min":{"1min":0.34100000000000003,"5min":0.26800000000000002,"15min":0.24199999999999999},"max":{"1min":3.7250000000000001,"5min":13.375,"15min":87.283000000000001},"last":0.34100000000000003},{"interface":"front","average":{"1min":1.155,"5min":1.7949999999999999,"15min":1.9690000000000001},"min":{"1min":0.32700000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":3.218,"5min":11.589,"15min":86.286000000000001},"last":0.34999999999999998}]},{"osd":1,"last update":"Sat Sep 23 19:46:32 2023","interfaces":[{"interface":"back","average":{"1min":1.7569999999999999,"5min":4.6760000000000002,"15min":2.5920000000000001},"min":{"1min":0.32200000000000001,"5min":0.28399999999999997,"15min":0.253},"max":{"1min":8.6820000000000004,"5min":117.58199999999999,"15min":117.58199999999999},"last":0.48399999999999999},{"interface":"front","average":{"1min":1.044,"5min":4.5819999999999999,"15min":2.5099999999999998},"min":{"1min":0.53100000000000003,"5min":0.29199999999999998,"15min":0.28999999999999998},"max":{"1min":2.0190000000000001,"5min":117.633,"15min":117.633},"last":0.53200000000000003}]},{"osd":2,"last update":"Sat Sep 23 19:46:52 2023","interfaces":[{"interface":"back","average":{"1min":1.3029999999999999,"5min":4.4690000000000003,"15min":3.3250000000000002},"min":{"1min":0.40699999999999997,"5min":0.30399999999999999,"15min":0.30299999999999999},"max":{"1min":3.6040000000000001,"5min":119.121,"15min":119.121},"last":0.76700000000000002},{"interface":"front","average":{"1min":1.4750000000000001,"5min":4.3949999999999996,"15min":3.319},"min":{"1min":0.32500000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":6.8520000000000003,"5min":118.947,"15min":118.947},"last":1.819}]},{"osd":3,"last update":"Sat Sep 23 19:46:49 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":1.9419999999999999,"15min":1.927},"min":{"1min":0.56899999999999995,"5min":0.33500000000000002,"15min":0.30199999999999999},"max":{"1min":3.105,"5min":8.4039999999999999,"15min":60.726999999999997},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.88400000000000001,"5min":1.9319999999999999,"15min":2.0129999999999999},"min":{"1min":0.311,"5min":0.307,"15min":0.28799999999999998},"max":{"1min":1.766,"5min":18.917000000000002,"15min":58.304000000000002},"last":0.39400000000000002}]},{"osd":4,"last update":"Sat Sep 23 19:46:49 2023","interfaces":[{"interface":"back","average":{"1min":0.83199999999999996,"5min":1.4399999999999999,"15min":1.3879999999999999},"min":{"1min":0.371,"5min":0.28999999999999998,"15min":0.28799999999999998},"max":{"1min":1.647,"5min":11.779,"15min":11.779},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.88400000000000001,"5min":1.712,"15min":1.53},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":1.4550000000000001,"5min":11.401999999999999,"15min":11.401999999999999},"last":0.42699999999999999}]},{"osd":5,"last update":"Sat Sep 23 19:46:49 2023","interfaces":[{"interface":"back","average":{"1min":0.99299999999999999,"5min":1.6830000000000001,"15min":1.589},"min":{"1min":0.42099999999999999,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":1.542,"5min":11.743,"15min":12.827},"last":0.499},{"interface":"front","average":{"1min":1.196,"5min":1.6220000000000001,"15min":1.931},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":2.8559999999999999,"5min":8.6950000000000003,"15min":10.433999999999999},"last":1.99}]},{"osd":7,"last update":"Sat Sep 23 19:46:12 2023","interfaces":[{"interface":"back","average":{"1min":1.802,"5min":1.8360000000000001,"15min":1.518},"min":{"1min":0.54900000000000004,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":6.6459999999999999,"5min":11.663,"15min":11.663},"last":0.45500000000000002},{"interface":"front","average":{"1min":2.3479999999999999,"5min":2.4020000000000001,"15min":1.929},"min":{"1min":0.439,"5min":0.40600000000000003,"15min":0.36199999999999999},"max":{"1min":10.382,"5min":10.382,"15min":11.439},"last":0.57199999999999995}]}]},{"osd":1,"up_from":98,"seq":420906795303,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1602908,"kb_used_data":982740,"kb_used_omap":4,"kb_used_meta":620155,"kb_avail":92142244,"statfs":{"total":95995035648,"available":94353657856,"internally_reserved":0,"allocated":1006325760,"data_stored":1005436443,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":635039630},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:46:31 2023","interfaces":[{"interface":"back","average":{"1min":1.004,"5min":1.8240000000000001,"15min":2.617},"min":{"1min":0.20000000000000001,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":4.7450000000000001,"5min":9.0700000000000003,"15min":65.079999999999998},"last":2.2330000000000001},{"interface":"front","average":{"1min":1.3919999999999999,"5min":1.881,"15min":2.6880000000000002},"min":{"1min":0.38900000000000001,"5min":0.26200000000000001,"15min":0.20300000000000001},"max":{"1min":10.747999999999999,"5min":11.353,"15min":65.730000000000004},"last":0.34000000000000002}]},{"osd":2,"last update":"Sat Sep 23 19:46:58 2023","interfaces":[{"interface":"back","average":{"1min":1.5489999999999999,"5min":2.4740000000000002,"15min":1.6819999999999999},"min":{"1min":0.32400000000000001,"5min":0.29699999999999999,"15min":0.23200000000000001},"max":{"1min":4.5419999999999998,"5min":11.388,"15min":11.388},"last":0.42299999999999999},{"interface":"front","average":{"1min":1.101,"5min":1.879,"15min":2.3730000000000002},"min":{"1min":0.371,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":5.6760000000000002,"5min":9.016,"15min":48.546999999999997},"last":1.141}]},{"osd":3,"last update":"Sat Sep 23 19:46:39 2023","interfaces":[{"interface":"back","average":{"1min":1.1240000000000001,"5min":2.0579999999999998,"15min":1.696},"min":{"1min":0.44700000000000001,"5min":0.251,"15min":0.21199999999999999},"max":{"1min":4.7990000000000004,"5min":15.004,"15min":15.004},"last":1.2250000000000001},{"interface":"front","average":{"1min":0.879,"5min":2.1699999999999999,"15min":2.073},"min":{"1min":0.29399999999999998,"5min":0.246,"15min":0.246},"max":{"1min":4.7590000000000003,"5min":18.018000000000001,"15min":48.692},"last":5.0119999999999996}]},{"osd":4,"last update":"Sat Sep 23 19:46:31 2023","interfaces":[{"interface":"back","average":{"1min":1.5720000000000001,"5min":1.6259999999999999,"15min":1.7310000000000001},"min":{"1min":0.505,"5min":0.35999999999999999,"15min":0.34300000000000003},"max":{"1min":10.842000000000001,"5min":11.26,"15min":53.128999999999998},"last":0.46200000000000002},{"interface":"front","average":{"1min":1.119,"5min":1.4550000000000001,"15min":1.7909999999999999},"min":{"1min":0.34599999999999997,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":4.8959999999999999,"5min":7.8280000000000003,"15min":48.637},"last":1.4650000000000001}]},{"osd":5,"last update":"Sat Sep 23 19:46:31 2023","interfaces":[{"interface":"back","average":{"1min":1.1459999999999999,"5min":1.671,"15min":2.0670000000000002},"min":{"1min":0.38100000000000001,"5min":0.34899999999999998,"15min":0.33900000000000002},"max":{"1min":5.1020000000000003,"5min":11.145,"15min":48.738},"last":1.51},{"interface":"front","average":{"1min":1.3080000000000001,"5min":1.712,"15min":1.8480000000000001},"min":{"1min":0.47399999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":4.7729999999999997,"5min":13.470000000000001,"15min":48.726999999999997},"last":1.1850000000000001}]},{"osd":6,"last update":"Sat Sep 23 19:46:31 2023","interfaces":[{"interface":"back","average":{"1min":1.464,"5min":2.2370000000000001,"15min":2.101},"min":{"1min":0.54600000000000004,"5min":0.318,"15min":0.318},"max":{"1min":5.6799999999999997,"5min":13.004,"15min":48.762},"last":1.093},{"interface":"front","average":{"1min":1.7849999999999999,"5min":2.125,"15min":2.1459999999999999},"min":{"1min":0.71199999999999997,"5min":0.44900000000000001,"15min":0.39100000000000001},"max":{"1min":10.814,"5min":11.676,"15min":53.076999999999998},"last":0.55500000000000005}]},{"osd":7,"last update":"Sat Sep 23 19:46:31 2023","interfaces":[{"interface":"back","average":{"1min":1.728,"5min":1.9239999999999999,"15min":2.056},"min":{"1min":0.53400000000000003,"5min":0.38600000000000001,"15min":0.38},"max":{"1min":10.865,"5min":11.317,"15min":48.677},"last":0.59299999999999997},{"interface":"front","average":{"1min":1.2509999999999999,"5min":1.8140000000000001,"15min":2.0619999999999998},"min":{"1min":0.40799999999999997,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":5.2599999999999998,"5min":11.26,"15min":48.805},"last":1.4199999999999999}]}]},{"osd":0,"up_from":85,"seq":365072220509,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1862500,"kb_used_data":1135068,"kb_used_omap":5,"kb_used_meta":727418,"kb_avail":91882652,"statfs":{"total":95995035648,"available":94087835648,"internally_reserved":0,"allocated":1162309632,"data_stored":1161416148,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":744876666},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 19:46:25 2023","interfaces":[{"interface":"back","average":{"1min":1.556,"5min":2.593,"15min":1.7190000000000001},"min":{"1min":0.35299999999999998,"5min":0.22500000000000001,"15min":0.20399999999999999},"max":{"1min":6.6440000000000001,"5min":57.234999999999999,"15min":57.234999999999999},"last":2.7890000000000001},{"interface":"front","average":{"1min":1.319,"5min":2.649,"15min":2.0019999999999998},"min":{"1min":0.22600000000000001,"5min":0.189,"15min":0.189},"max":{"1min":7.9119999999999999,"5min":58.825000000000003,"15min":58.825000000000003},"last":2.9249999999999998}]},{"osd":2,"last update":"Sat Sep 23 19:46:59 2023","interfaces":[{"interface":"back","average":{"1min":1.1399999999999999,"5min":4.7670000000000003,"15min":2.8119999999999998},"min":{"1min":0.33900000000000002,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":3.5790000000000002,"5min":262.59500000000003,"15min":262.59500000000003},"last":5.4000000000000004},{"interface":"front","average":{"1min":0.86499999999999999,"5min":4.7220000000000004,"15min":3.0470000000000002},"min":{"1min":0.34300000000000003,"5min":0.245,"15min":0.245},"max":{"1min":2.202,"5min":263.25700000000001,"15min":263.25700000000001},"last":2.6899999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:46:30 2023","interfaces":[{"interface":"back","average":{"1min":1.4159999999999999,"5min":1.962,"15min":1.927},"min":{"1min":0.36099999999999999,"5min":0.29399999999999998,"15min":0.26400000000000001},"max":{"1min":5.5220000000000002,"5min":8.2590000000000003,"15min":14.212},"last":2.8639999999999999},{"interface":"front","average":{"1min":1.0860000000000001,"5min":1.9530000000000001,"15min":1.893},"min":{"1min":0.28199999999999997,"5min":0.223,"15min":0.223},"max":{"1min":4.1669999999999998,"5min":14.84,"15min":14.84},"last":2.8799999999999999}]},{"osd":4,"last update":"Sat Sep 23 19:46:03 2023","interfaces":[{"interface":"back","average":{"1min":1.395,"5min":1.3899999999999999,"15min":1.637},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":5.4669999999999996,"5min":7.4290000000000003,"15min":11.792999999999999},"last":2.8330000000000002},{"interface":"front","average":{"1min":1.252,"5min":1.385,"15min":1.5760000000000001},"min":{"1min":0.36499999999999999,"5min":0.315,"15min":0.315},"max":{"1min":4.4340000000000002,"5min":6.0659999999999998,"15min":13.641999999999999},"last":2.1040000000000001}]},{"osd":5,"last update":"Sat Sep 23 19:46:03 2023","interfaces":[{"interface":"back","average":{"1min":1.264,"5min":1.5189999999999999,"15min":1.859},"min":{"1min":0.38600000000000001,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":4.5179999999999998,"5min":6.2430000000000003,"15min":13.154},"last":2.1469999999999998},{"interface":"front","average":{"1min":1.415,"5min":1.5720000000000001,"15min":2.0129999999999999},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.36899999999999999},"max":{"1min":7.4450000000000003,"5min":8.6120000000000001,"15min":13.109999999999999},"last":2.7709999999999999}]},{"osd":6,"last update":"Sat Sep 23 19:46:03 2023","interfaces":[{"interface":"back","average":{"1min":1.53,"5min":1.6120000000000001,"15min":1.9179999999999999},"min":{"1min":0.438,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":4.0999999999999996,"5min":13.651999999999999,"15min":40.935000000000002},"last":2.8050000000000002},{"interface":"front","average":{"1min":1.6339999999999999,"5min":1.6519999999999999,"15min":1.899},"min":{"1min":0.42099999999999999,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":4.4809999999999999,"5min":6.7430000000000003,"15min":40.787999999999997},"last":2.1299999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:46:03 2023","interfaces":[{"interface":"back","average":{"1min":2.819,"5min":1.929,"15min":1.762},"min":{"1min":0.44900000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":13.672000000000001,"5min":13.672000000000001,"15min":13.672000000000001},"last":0.63200000000000001},{"interface":"front","average":{"1min":1.9810000000000001,"5min":1.663,"15min":1.8959999999999999},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":7.4729999999999999,"5min":8.3759999999999994,"15min":10.561},"last":2.7309999999999999}]}]},{"osd":2,"up_from":159,"seq":682899800166,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2368868,"kb_used_data":2072348,"kb_used_omap":7,"kb_used_meta":296504,"kb_avail":91376284,"statfs":{"total":95995035648,"available":93569314816,"internally_reserved":0,"allocated":2122084352,"data_stored":2121244561,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":303620177},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:46:04 2023","interfaces":[{"interface":"back","average":{"1min":1.746,"5min":1.6399999999999999,"15min":2.0190000000000001},"min":{"1min":0.26100000000000001,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":8.0340000000000007,"5min":16.702000000000002,"15min":16.702000000000002},"last":0.42799999999999999},{"interface":"front","average":{"1min":2.488,"5min":1.9730000000000001,"15min":2.0019999999999998},"min":{"1min":0.33200000000000002,"5min":0.254,"15min":0.23599999999999999},"max":{"1min":8.157,"5min":9.7370000000000001,"15min":9.7370000000000001},"last":2.1139999999999999}]},{"osd":1,"last update":"Sat Sep 23 19:46:04 2023","interfaces":[{"interface":"back","average":{"1min":2.2320000000000002,"5min":2.9249999999999998,"15min":1.6180000000000001},"min":{"1min":0.33800000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":12.391,"5min":58.542000000000002,"15min":58.542000000000002},"last":5.734},{"interface":"front","average":{"1min":1.851,"5min":2.4239999999999999,"15min":1.734},"min":{"1min":0.34499999999999997,"5min":0.22,"15min":0.22},"max":{"1min":7.931,"5min":57.551000000000002,"15min":57.551000000000002},"last":0.39400000000000002}]},{"osd":3,"last update":"Sat Sep 23 19:46:04 2023","interfaces":[{"interface":"back","average":{"1min":2.1560000000000001,"5min":2.0569999999999999,"15min":1.9890000000000001},"min":{"1min":0.35599999999999998,"5min":0.28000000000000003,"15min":0.26200000000000001},"max":{"1min":10.096,"5min":17.736999999999998,"15min":17.736999999999998},"last":1.1799999999999999},{"interface":"front","average":{"1min":2.3639999999999999,"5min":2.0339999999999998,"15min":2.0939999999999999},"min":{"1min":0.39900000000000002,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":9.8460000000000001,"5min":16.742000000000001,"15min":16.742000000000001},"last":8.9770000000000003}]},{"osd":4,"last update":"Sat Sep 23 19:46:04 2023","interfaces":[{"interface":"back","average":{"1min":2.0009999999999999,"5min":1.863,"15min":1.669},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":8.1910000000000007,"5min":18.318999999999999,"15min":18.318999999999999},"last":2.282},{"interface":"front","average":{"1min":1.494,"5min":1.603,"15min":1.6200000000000001},"min":{"1min":0.36299999999999999,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":4.5810000000000004,"5min":7.4299999999999997,"15min":7.4299999999999997},"last":1.2170000000000001}]},{"osd":5,"last update":"Sat Sep 23 19:46:04 2023","interfaces":[{"interface":"back","average":{"1min":1.3879999999999999,"5min":1.675,"15min":1.738},"min":{"1min":0.42699999999999999,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":4.6420000000000003,"5min":16.577000000000002,"15min":16.577000000000002},"last":1.0469999999999999},{"interface":"front","average":{"1min":1.9039999999999999,"5min":1.8180000000000001,"15min":1.881},"min":{"1min":0.41199999999999998,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":8.1080000000000005,"5min":8.9930000000000003,"15min":8.9930000000000003},"last":2.3340000000000001}]},{"osd":6,"last update":"Sat Sep 23 19:46:04 2023","interfaces":[{"interface":"back","average":{"1min":2.1640000000000001,"5min":2.1880000000000002,"15min":2.0619999999999998},"min":{"1min":0.37,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":8.2210000000000001,"5min":9.9659999999999993,"15min":11.851000000000001},"last":2.3109999999999999},{"interface":"front","average":{"1min":2.6309999999999998,"5min":2.347,"15min":1.97},"min":{"1min":0.63100000000000001,"5min":0.39600000000000002,"15min":0.36699999999999999},"max":{"1min":12.297000000000001,"5min":12.297000000000001,"15min":12.297000000000001},"last":5.7009999999999996}]},{"osd":7,"last update":"Sat Sep 23 19:46:04 2023","interfaces":[{"interface":"back","average":{"1min":1.5660000000000001,"5min":2.1349999999999998,"15min":1.7529999999999999},"min":{"1min":0.48399999999999999,"5min":0.436,"15min":0.436},"max":{"1min":7.8940000000000001,"5min":16.457999999999998,"15min":16.457999999999998},"last":0.89900000000000002},{"interface":"front","average":{"1min":2.7709999999999999,"5min":2.1190000000000002,"15min":1.6970000000000001},"min":{"1min":0.63400000000000001,"5min":0.48899999999999999,"15min":0.41299999999999998},"max":{"1min":12.362,"5min":12.362,"15min":12.362},"last":1.325}]}]},{"osd":3,"up_from":116,"seq":498216206566,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2076980,"kb_used_data":1615532,"kb_used_omap":2,"kb_used_meta":461437,"kb_avail":91668172,"statfs":{"total":95995035648,"available":93868208128,"internally_reserved":0,"allocated":1654304768,"data_stored":1653508189,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":472511541},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:46:42 2023","interfaces":[{"interface":"back","average":{"1min":1.4790000000000001,"5min":1.8220000000000001,"15min":1.7569999999999999},"min":{"1min":0.42599999999999999,"5min":0.21199999999999999,"15min":0.19900000000000001},"max":{"1min":12.193,"5min":12.193,"15min":13.411},"last":0.23799999999999999},{"interface":"front","average":{"1min":1.7549999999999999,"5min":1.865,"15min":1.7370000000000001},"min":{"1min":0.45100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":16.713000000000001,"5min":16.713000000000001,"15min":16.713000000000001},"last":3.5790000000000002}]},{"osd":1,"last update":"Sat Sep 23 19:46:42 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":2.7799999999999998,"15min":1.9379999999999999},"min":{"1min":0.503,"5min":0.28299999999999997,"15min":0.22600000000000001},"max":{"1min":2.9950000000000001,"5min":92.171000000000006,"15min":92.171000000000006},"last":1.363},{"interface":"front","average":{"1min":0.95399999999999996,"5min":3.073,"15min":2.2229999999999999},"min":{"1min":0.373,"5min":0.29799999999999999,"15min":0.23000000000000001},"max":{"1min":3.258,"5min":87.771000000000001,"15min":87.771000000000001},"last":1.915}]},{"osd":2,"last update":"Sat Sep 23 19:46:59 2023","interfaces":[{"interface":"back","average":{"1min":1.171,"5min":1.917,"15min":2.3450000000000002},"min":{"1min":0.495,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":4.1680000000000001,"5min":9.4740000000000002,"15min":10.960000000000001},"last":3.6320000000000001},{"interface":"front","average":{"1min":1.351,"5min":1.889,"15min":1.754},"min":{"1min":0.41299999999999998,"5min":0.252,"15min":0.252},"max":{"1min":5.7789999999999999,"5min":9.0410000000000004,"15min":9.0410000000000004},"last":2.593}]},{"osd":4,"last update":"Sat Sep 23 19:46:42 2023","interfaces":[{"interface":"back","average":{"1min":1.212,"5min":1.7150000000000001,"15min":1.714},"min":{"1min":0.62,"5min":0.34799999999999998,"15min":0.32500000000000001},"max":{"1min":3.3199999999999998,"5min":11.666,"15min":11.666},"last":0.48699999999999999},{"interface":"front","average":{"1min":1.1639999999999999,"5min":1.5569999999999999,"15min":1.6359999999999999},"min":{"1min":0.69899999999999995,"5min":0.33000000000000002,"15min":0.32300000000000001},"max":{"1min":3.1259999999999999,"5min":9.391,"15min":10.926},"last":1.425}]},{"osd":5,"last update":"Sat Sep 23 19:46:42 2023","interfaces":[{"interface":"back","average":{"1min":1.1699999999999999,"5min":1.528,"15min":1.7789999999999999},"min":{"1min":0.63900000000000001,"5min":0.36599999999999999,"15min":0.35299999999999998},"max":{"1min":3.1560000000000001,"5min":9.4269999999999996,"15min":10.82},"last":1.3919999999999999},{"interface":"front","average":{"1min":1.0269999999999999,"5min":1.909,"15min":1.7549999999999999},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":2.3559999999999999,"5min":13.146000000000001,"15min":13.146000000000001},"last":0.38900000000000001}]},{"osd":6,"last update":"Sat Sep 23 19:46:42 2023","interfaces":[{"interface":"back","average":{"1min":1.21,"5min":2.3730000000000002,"15min":1.992},"min":{"1min":0.61699999999999999,"5min":0.39100000000000001,"15min":0.36299999999999999},"max":{"1min":3.2370000000000001,"5min":15.919,"15min":15.919},"last":4.7949999999999999},{"interface":"front","average":{"1min":1.0880000000000001,"5min":2.2810000000000001,"15min":1.734},"min":{"1min":0.40000000000000002,"5min":0.36399999999999999,"15min":0.315},"max":{"1min":2.3130000000000002,"5min":9.0909999999999993,"15min":9.0909999999999993},"last":2.6240000000000001}]},{"osd":7,"last update":"Sat Sep 23 19:46:42 2023","interfaces":[{"interface":"back","average":{"1min":1.2709999999999999,"5min":2.488,"15min":1.891},"min":{"1min":0.433,"5min":0.39600000000000002,"15min":0.32400000000000001},"max":{"1min":3.0270000000000001,"5min":14.098000000000001,"15min":14.098000000000001},"last":0.42299999999999999},{"interface":"front","average":{"1min":1.284,"5min":2.2429999999999999,"15min":1.895},"min":{"1min":0.66600000000000004,"5min":0.44,"15min":0.40300000000000002},"max":{"1min":3.2909999999999999,"5min":11.744999999999999,"15min":11.744999999999999},"last":2.5369999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019436,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1484796,"kb_used_data":576692,"kb_used_omap":0,"kb_used_meta":908096,"kb_avail":92260356,"statfs":{"total":95995035648,"available":94474604544,"internally_reserved":0,"allocated":590532608,"data_stored":590022922,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":929890304},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:46:12 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":1.0640000000000001,"15min":1.3380000000000001},"min":{"1min":0.32400000000000001,"5min":0.25700000000000001,"15min":0.23300000000000001},"max":{"1min":3.9630000000000001,"5min":7.3899999999999997,"15min":12.975},"last":0.35899999999999999},{"interface":"front","average":{"1min":1.087,"5min":0.90200000000000002,"15min":1.264},"min":{"1min":0.316,"5min":0.29499999999999998,"15min":0.26500000000000001},"max":{"1min":6.5069999999999997,"5min":6.5069999999999997,"15min":11.380000000000001},"last":0.45600000000000002}]},{"osd":1,"last update":"Sat Sep 23 19:46:38 2023","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":2.7360000000000002,"15min":1.8580000000000001},"min":{"1min":0.373,"5min":0.27800000000000002,"15min":0.27700000000000002},"max":{"1min":5.1630000000000003,"5min":101.786,"15min":101.786},"last":1.9039999999999999},{"interface":"front","average":{"1min":1.23,"5min":3.1269999999999998,"15min":2.0670000000000002},"min":{"1min":0.375,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":5.2009999999999996,"5min":101.726,"15min":101.726},"last":1.9710000000000001}]},{"osd":2,"last update":"Sat Sep 23 19:46:55 2023","interfaces":[{"interface":"back","average":{"1min":1.107,"5min":2.573,"15min":2.9279999999999999},"min":{"1min":0.34300000000000003,"5min":0.313,"15min":0.313},"max":{"1min":2.6880000000000002,"5min":101.92700000000001,"15min":101.92700000000001},"last":0.78700000000000003},{"interface":"front","average":{"1min":1.327,"5min":2.4380000000000002,"15min":2.0800000000000001},"min":{"1min":0.53400000000000003,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":5.2039999999999997,"5min":103.88200000000001,"15min":103.88200000000001},"last":0.54200000000000004}]},{"osd":3,"last update":"Sat Sep 23 19:46:50 2023","interfaces":[{"interface":"back","average":{"1min":0.95999999999999996,"5min":2.1930000000000001,"15min":2.097},"min":{"1min":0.44500000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.53,"5min":27.789000000000001,"15min":49.515000000000001},"last":1.3169999999999999},{"interface":"front","average":{"1min":0.999,"5min":1.839,"15min":2},"min":{"1min":0.40699999999999997,"5min":0.34200000000000003,"15min":0.32100000000000001},"max":{"1min":1.875,"5min":27.808,"15min":51.518999999999998},"last":0.40699999999999997}]},{"osd":5,"last update":"Sat Sep 23 19:46:12 2023","interfaces":[{"interface":"back","average":{"1min":1.4279999999999999,"5min":1.4099999999999999,"15min":1.6539999999999999},"min":{"1min":0.376,"5min":0.376,"15min":0.28699999999999998},"max":{"1min":7.0629999999999997,"5min":22.442,"15min":22.442},"last":0.41999999999999998},{"interface":"front","average":{"1min":1.46,"5min":1.1930000000000001,"15min":1.542},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.28899999999999998},"max":{"1min":7.6900000000000004,"5min":7.6900000000000004,"15min":10.362},"last":0.53200000000000003}]},{"osd":6,"last update":"Sat Sep 23 19:46:38 2023","interfaces":[{"interface":"back","average":{"1min":1.1060000000000001,"5min":1.5620000000000001,"15min":1.6910000000000001},"min":{"1min":0.56899999999999995,"5min":0.372,"15min":0.33900000000000002},"max":{"1min":2.0369999999999999,"5min":22.544,"15min":22.544},"last":0.47999999999999998},{"interface":"front","average":{"1min":1.2370000000000001,"5min":1.1899999999999999,"15min":1.454},"min":{"1min":0.48999999999999999,"5min":0.36499999999999999,"15min":0.34000000000000002},"max":{"1min":3.992,"5min":7.4880000000000004,"15min":10.952},"last":0.49299999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:46:39 2023","interfaces":[{"interface":"back","average":{"1min":18.457000000000001,"5min":4.9050000000000002,"15min":2.5960000000000001},"min":{"1min":0.49399999999999999,"5min":0.36199999999999999,"15min":0.30299999999999999},"max":{"1min":294.76499999999999,"5min":294.76499999999999,"15min":294.76499999999999},"last":0.50700000000000001},{"interface":"front","average":{"1min":18.574000000000002,"5min":5.1849999999999996,"15min":2.8780000000000001},"min":{"1min":0.68899999999999995,"5min":0.39700000000000002,"15min":0.34499999999999997},"max":{"1min":294.47699999999998,"5min":294.47699999999998,"15min":294.47699999999998},"last":0.80000000000000004}]}]},{"osd":5,"up_from":36,"seq":154618823203,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1301476,"kb_used_data":394588,"kb_used_omap":0,"kb_used_meta":906880,"kb_avail":92443676,"statfs":{"total":95995035648,"available":94662324224,"internally_reserved":0,"allocated":404058112,"data_stored":403568116,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":928645120},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:46:09 2023","interfaces":[{"interface":"back","average":{"1min":1.734,"5min":1.337,"15min":1.6719999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":4.5419999999999998,"5min":5.476,"15min":43.094000000000001},"last":0.35199999999999998},{"interface":"front","average":{"1min":1.6619999999999999,"5min":1.4219999999999999,"15min":2.1640000000000001},"min":{"1min":0.36699999999999999,"5min":0.317,"15min":0.27600000000000002},"max":{"1min":4.6619999999999999,"5min":8.2899999999999991,"15min":43.802999999999997},"last":0.51100000000000001}]},{"osd":1,"last update":"Sat Sep 23 19:46:29 2023","interfaces":[{"interface":"back","average":{"1min":1.0920000000000001,"5min":1.5109999999999999,"15min":1.548},"min":{"1min":0.28599999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":4.2999999999999998,"5min":11.266999999999999,"15min":32.253999999999998},"last":0.879},{"interface":"front","average":{"1min":1.7030000000000001,"5min":1.8129999999999999,"15min":2.008},"min":{"1min":0.378,"5min":0.28100000000000003,"15min":0.27900000000000003},"max":{"1min":7.3949999999999996,"5min":10.891999999999999,"15min":32.277999999999999},"last":0.29899999999999999}]},{"osd":2,"last update":"Sat Sep 23 19:46:54 2023","interfaces":[{"interface":"back","average":{"1min":1.246,"5min":2.7850000000000001,"15min":2.7970000000000002},"min":{"1min":0.46000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":3.2290000000000001,"5min":49.011000000000003,"15min":49.011000000000003},"last":1.6579999999999999},{"interface":"front","average":{"1min":1.4079999999999999,"5min":2.5800000000000001,"15min":2.181},"min":{"1min":0.39600000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":4.5389999999999997,"5min":48.914000000000001,"15min":48.914000000000001},"last":0.40200000000000002}]},{"osd":3,"last update":"Sat Sep 23 19:46:29 2023","interfaces":[{"interface":"back","average":{"1min":1.113,"5min":1.3380000000000001,"15min":1.6279999999999999},"min":{"1min":0.34799999999999998,"5min":0.33700000000000002,"15min":0.311},"max":{"1min":2.7490000000000001,"5min":7.29,"15min":10.885999999999999},"last":0.92900000000000005},{"interface":"front","average":{"1min":1.115,"5min":1.613,"15min":1.595},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.33800000000000002},"max":{"1min":4.4809999999999999,"5min":7.0129999999999999,"15min":8.5879999999999992},"last":0.84899999999999998}]},{"osd":4,"last update":"Sat Sep 23 19:46:09 2023","interfaces":[{"interface":"back","average":{"1min":1.2050000000000001,"5min":1.1479999999999999,"15min":1.8620000000000001},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":4.5970000000000004,"5min":7.0060000000000002,"15min":74.221000000000004},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.96499999999999997,"5min":0.93700000000000006,"15min":1.7649999999999999},"min":{"1min":0.38400000000000001,"5min":0.36899999999999999,"15min":0.30599999999999999},"max":{"1min":2.4420000000000002,"5min":5.3879999999999999,"15min":73.772999999999996},"last":0.499}]},{"osd":6,"last update":"Sat Sep 23 19:46:53 2023","interfaces":[{"interface":"back","average":{"1min":1.008,"5min":1.081,"15min":1.4379999999999999},"min":{"1min":0.51300000000000001,"5min":0.35599999999999998,"15min":0.32000000000000001},"max":{"1min":2.161,"5min":5.6360000000000001,"15min":16.222000000000001},"last":0.48699999999999999},{"interface":"front","average":{"1min":1.044,"5min":1.163,"15min":1.659},"min":{"1min":0.443,"5min":0.36099999999999999,"15min":0.309},"max":{"1min":2.5289999999999999,"5min":5.6239999999999997,"15min":14.178000000000001},"last":0.46100000000000002}]},{"osd":7,"last update":"Sat Sep 23 19:46:09 2023","interfaces":[{"interface":"back","average":{"1min":1.542,"5min":1.466,"15min":1.776},"min":{"1min":0.41499999999999998,"5min":0.40500000000000003,"15min":0.32100000000000001},"max":{"1min":13.771000000000001,"5min":13.771000000000001,"15min":14.127000000000001},"last":0.41599999999999998},{"interface":"front","average":{"1min":1.5129999999999999,"5min":1.4219999999999999,"15min":1.663},"min":{"1min":0.432,"5min":0.40500000000000003,"15min":0.307},"max":{"1min":5.9889999999999999,"5min":7.2599999999999998,"15min":13.18},"last":0.55800000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1170538496,"data_stored":1170538496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1020592128,"data_stored":1020592128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2139291648,"data_stored":2139291648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1652686848,"data_stored":1652686848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":596901888,"data_stored":596901888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":402194432,"data_stored":402194432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1403781120,"data_stored":1403781120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1387790336,"data_stored":1387790336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T19:47:12.591 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T19:47:12.597 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:12.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:12.600 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:12.701 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:12.734 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:12.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:12.736 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:12.837 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:12.865 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:12.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:12.869 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:47:12.970 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:12.995 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:12.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:12.998 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:13.099 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:13.127 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:13.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:13.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:47:13.149 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 39 16 57594 57578 92.2558 65.875 0.0244379 0.0108323 2023-09-23T19:47:13.232 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:13.263 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:13.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:13.265 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:13.366 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:13.392 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:13.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:13.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:13.497 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:13.521 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:13.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:13.524 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:13.624 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:13.634 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:13 smithi092 bash[58855]: cluster 2023-09-23T19:47:12.332144+0000 mgr.y (mgr.25325) 1194 : cluster 0 pgmap v1175: 89 pgs: 89 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 62 MiB/s wr, 985 op/s 2023-09-23T19:47:13.634 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:13 smithi092 bash[58855]: audit 2023-09-23T19:47:13.576424+0000 mon.a (mon.0) 1505 : audit 1 from='client.? 172.21.15.16:0/2202205133' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:47:13.640 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is unset 2023-09-23T19:47:13.658 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:13.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:13.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:47:13.763 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:13.798 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:13.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:13.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:47:13.902 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:13.911 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:13 smithi016 bash[75004]: cluster 2023-09-23T19:47:12.332144+0000 mgr.y (mgr.25325) 1194 : cluster 0 pgmap v1175: 89 pgs: 89 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 62 MiB/s wr, 985 op/s 2023-09-23T19:47:13.911 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:13 smithi016 bash[75004]: audit 2023-09-23T19:47:13.576424+0000 mon.a (mon.0) 1505 : audit 1 from='client.? 172.21.15.16:0/2202205133' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:47:13.911 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:13 smithi016 bash[77995]: cluster 2023-09-23T19:47:12.332144+0000 mgr.y (mgr.25325) 1194 : cluster 0 pgmap v1175: 89 pgs: 89 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 62 MiB/s wr, 985 op/s 2023-09-23T19:47:13.911 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:13 smithi016 bash[77995]: audit 2023-09-23T19:47:13.576424+0000 mon.a (mon.0) 1505 : audit 1 from='client.? 172.21.15.16:0/2202205133' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:47:13.933 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:13.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:13.938 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:14.039 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:14.068 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:14.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:14.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:14.149 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-09-23T19:47:14.146205+0000 min lat: 0.00140724 max lat: 0.376785 avg lat: 0.0109098 2023-09-23T19:47:14.150 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:47:14.150 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 40 16 58658 58642 91.6118 66.5 0.0191284 0.0109098 2023-09-23T19:47:14.173 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:14.203 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:14.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:14.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:14.310 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:14.338 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:14.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:14.340 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:47:14.441 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:14.473 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:14.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:14.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:47:14.527 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:47:14.579 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:14.614 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:14.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:14.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:14.724 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:14.756 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:14.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:14.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:14.789 INFO:teuthology.orchestra.run.smithi092.stdout:{ 2023-09-23T19:47:14.789 INFO:teuthology.orchestra.run.smithi092.stdout: "ops": [], 2023-09-23T19:47:14.789 INFO:teuthology.orchestra.run.smithi092.stdout: "num_ops": 0 2023-09-23T19:47:14.789 INFO:teuthology.orchestra.run.smithi092.stdout:} 2023-09-23T19:47:14.862 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:14.880 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:14 smithi092 bash[58855]: cluster 2023-09-23T19:47:13.609710+0000 mon.a (mon.0) 1506 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:47:14.880 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:14 smithi092 bash[58855]: audit 2023-09-23T19:47:13.630602+0000 mon.a (mon.0) 1507 : audit 1 from='client.? 172.21.15.16:0/2202205133' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:47:14.880 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:14 smithi092 bash[58855]: cluster 2023-09-23T19:47:13.630896+0000 mon.a (mon.0) 1508 : cluster 0 osdmap e214: 8 total, 8 up, 8 in 2023-09-23T19:47:14.880 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:14 smithi092 bash[58855]: cluster 2023-09-23T19:47:13.636338+0000 osd.7 (osd.7) 133 : cluster 3 failed to encode map e214 with expected crc 2023-09-23T19:47:14.880 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:14 smithi092 bash[58855]: cluster 2023-09-23T19:47:13.638046+0000 osd.4 (osd.4) 156 : cluster 3 failed to encode map e214 with expected crc 2023-09-23T19:47:14.881 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:14 smithi092 bash[58855]: cluster 2023-09-23T19:47:13.638226+0000 osd.6 (osd.6) 144 : cluster 3 failed to encode map e214 with expected crc 2023-09-23T19:47:14.881 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:14 smithi092 bash[58855]: audit 2023-09-23T19:47:13.669101+0000 mon.a (mon.0) 1509 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:47:14.885 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:14.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:14.889 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:47:14.990 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:15.012 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:15.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:15.015 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:15.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:14 smithi016 bash[75004]: cluster 2023-09-23T19:47:13.609710+0000 mon.a (mon.0) 1506 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:47:15.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:14 smithi016 bash[75004]: audit 2023-09-23T19:47:13.630602+0000 mon.a (mon.0) 1507 : audit 1 from='client.? 172.21.15.16:0/2202205133' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:47:15.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:14 smithi016 bash[75004]: cluster 2023-09-23T19:47:13.630896+0000 mon.a (mon.0) 1508 : cluster 0 osdmap e214: 8 total, 8 up, 8 in 2023-09-23T19:47:15.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:14 smithi016 bash[75004]: cluster 2023-09-23T19:47:13.636338+0000 osd.7 (osd.7) 133 : cluster 3 failed to encode map e214 with expected crc 2023-09-23T19:47:15.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:14 smithi016 bash[75004]: cluster 2023-09-23T19:47:13.638046+0000 osd.4 (osd.4) 156 : cluster 3 failed to encode map e214 with expected crc 2023-09-23T19:47:15.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:14 smithi016 bash[75004]: cluster 2023-09-23T19:47:13.638226+0000 osd.6 (osd.6) 144 : cluster 3 failed to encode map e214 with expected crc 2023-09-23T19:47:15.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:14 smithi016 bash[75004]: audit 2023-09-23T19:47:13.669101+0000 mon.a (mon.0) 1509 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:47:15.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:14 smithi016 bash[77995]: cluster 2023-09-23T19:47:13.609710+0000 mon.a (mon.0) 1506 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:47:15.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:14 smithi016 bash[77995]: audit 2023-09-23T19:47:13.630602+0000 mon.a (mon.0) 1507 : audit 1 from='client.? 172.21.15.16:0/2202205133' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:47:15.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:14 smithi016 bash[77995]: cluster 2023-09-23T19:47:13.630896+0000 mon.a (mon.0) 1508 : cluster 0 osdmap e214: 8 total, 8 up, 8 in 2023-09-23T19:47:15.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:14 smithi016 bash[77995]: cluster 2023-09-23T19:47:13.636338+0000 osd.7 (osd.7) 133 : cluster 3 failed to encode map e214 with expected crc 2023-09-23T19:47:15.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:14 smithi016 bash[77995]: cluster 2023-09-23T19:47:13.638046+0000 osd.4 (osd.4) 156 : cluster 3 failed to encode map e214 with expected crc 2023-09-23T19:47:15.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:14 smithi016 bash[77995]: cluster 2023-09-23T19:47:13.638226+0000 osd.6 (osd.6) 144 : cluster 3 failed to encode map e214 with expected crc 2023-09-23T19:47:15.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:14 smithi016 bash[77995]: audit 2023-09-23T19:47:13.669101+0000 mon.a (mon.0) 1509 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:47:15.115 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:15.145 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:15.150 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 41 16 59579 59563 90.7812 57.5625 0.0111786 0.0110088 2023-09-23T19:47:15.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:15.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:15.252 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:15.279 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:15.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:15.282 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:47:15.383 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:15.408 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:15.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:15.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:47:15.511 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:15.534 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:15.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:15.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:47:15.642 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:15.671 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:15.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:15.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:15.774 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:15.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:15 smithi092 bash[58855]: cluster 2023-09-23T19:47:14.332887+0000 mgr.y (mgr.25325) 1195 : cluster 0 pgmap v1177: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-09-23T19:47:15.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:15 smithi092 bash[58855]: audit 2023-09-23T19:47:15.492015+0000 mon.a (mon.0) 1510 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:47:15.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:15 smithi092 bash[58855]: audit 2023-09-23T19:47:15.501796+0000 mon.a (mon.0) 1511 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:47:15.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:15 smithi092 bash[58855]: audit 2023-09-23T19:47:15.503567+0000 mon.a (mon.0) 1512 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:15.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:15 smithi092 bash[58855]: audit 2023-09-23T19:47:15.505445+0000 mon.a (mon.0) 1513 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:47:15.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:15 smithi092 bash[58855]: audit 2023-09-23T19:47:15.513603+0000 mon.a (mon.0) 1514 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:47:15.799 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:15.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:15.801 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:15.902 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:15.912 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:15 smithi016 bash[75004]: cluster 2023-09-23T19:47:14.332887+0000 mgr.y (mgr.25325) 1195 : cluster 0 pgmap v1177: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-09-23T19:47:15.913 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:15 smithi016 bash[75004]: audit 2023-09-23T19:47:15.492015+0000 mon.a (mon.0) 1510 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:47:15.913 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:15 smithi016 bash[75004]: audit 2023-09-23T19:47:15.501796+0000 mon.a (mon.0) 1511 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:47:15.913 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:15 smithi016 bash[75004]: audit 2023-09-23T19:47:15.503567+0000 mon.a (mon.0) 1512 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:15.913 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:15 smithi016 bash[75004]: audit 2023-09-23T19:47:15.505445+0000 mon.a (mon.0) 1513 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:47:15.913 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:15 smithi016 bash[75004]: audit 2023-09-23T19:47:15.513603+0000 mon.a (mon.0) 1514 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:47:15.913 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:47:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:47:15] "GET /metrics HTTP/1.1" 200 36305 "" "Prometheus/2.33.4" 2023-09-23T19:47:15.913 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:15 smithi016 bash[77995]: cluster 2023-09-23T19:47:14.332887+0000 mgr.y (mgr.25325) 1195 : cluster 0 pgmap v1177: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-09-23T19:47:15.913 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:15 smithi016 bash[77995]: audit 2023-09-23T19:47:15.492015+0000 mon.a (mon.0) 1510 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:47:15.914 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:15 smithi016 bash[77995]: audit 2023-09-23T19:47:15.501796+0000 mon.a (mon.0) 1511 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:47:15.914 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:15 smithi016 bash[77995]: audit 2023-09-23T19:47:15.503567+0000 mon.a (mon.0) 1512 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:15.914 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:15 smithi016 bash[77995]: audit 2023-09-23T19:47:15.505445+0000 mon.a (mon.0) 1513 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:47:15.914 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:15 smithi016 bash[77995]: audit 2023-09-23T19:47:15.513603+0000 mon.a (mon.0) 1514 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:47:15.933 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:15.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:15.938 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:16.039 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:16.071 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:16.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:16.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:16.150 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 42 16 60686 60670 90.2669 69.1875 0.0218647 0.011072 2023-09-23T19:47:16.178 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:16.204 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:16.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:16.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:47:16.298 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T19:47:16.308 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:16.372 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:16.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:16.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:47:16.479 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:16.512 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:16.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:16.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:16.618 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:16.644 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:16.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:16.646 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:47:16.747 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:16.778 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:16.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:16.780 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:47:16.881 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:16.905 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:16.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:16.909 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:47:17.010 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:17.050 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:17.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:17.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:17.150 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 43 16 61741 61725 89.701 65.9375 0.00963092 0.0111419 2023-09-23T19:47:17.156 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:17.181 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:17.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:17.183 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:47:17.284 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:17.293 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T19:47:17.307 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:17.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:17.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:17.413 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:17.439 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:17.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:17.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:47:17.542 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:17.580 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:17.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:17.586 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:47:17.687 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:17.726 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:17.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:17.734 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:17.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:17 smithi092 bash[58855]: cluster 2023-09-23T19:47:16.333603+0000 mgr.y (mgr.25325) 1196 : cluster 0 pgmap v1178: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-09-23T19:47:17.835 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:17.861 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:17.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:17.864 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:17.965 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:17.975 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:17 smithi016 bash[77995]: cluster 2023-09-23T19:47:16.333603+0000 mgr.y (mgr.25325) 1196 : cluster 0 pgmap v1178: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-09-23T19:47:17.975 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:17 smithi016 bash[75004]: cluster 2023-09-23T19:47:16.333603+0000 mgr.y (mgr.25325) 1196 : cluster 0 pgmap v1178: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-09-23T19:47:17.995 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:17.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:18.000 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:18.101 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:18.129 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:18.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:18.132 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:18.150 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 44 16 62741 62725 89.0827 62.5 0.0101194 0.0112195 2023-09-23T19:47:18.233 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:18.257 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:18.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:18.262 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:47:18.362 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:18.395 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:18.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:18.397 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:47:18.498 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:18.524 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:18.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:18.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:47:18.628 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:18.658 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:18.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:18.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:18.764 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:18.795 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:18.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:18.797 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:18.828 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:47:18.898 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:18.926 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:18.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:18.932 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:47:19.033 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:19.070 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:19.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:19.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:19.150 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 45 16 63798 63782 88.5711 66.0625 0.0181075 0.0112853 2023-09-23T19:47:19.175 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:19.212 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:19.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:19.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:19.317 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:19.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:19.347 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:19.347 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:19.448 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:19.473 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:19.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:19.476 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:47:19.577 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:19.602 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:19.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:19.606 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:47:19.707 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:19.726 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:19.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:19.728 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:19.829 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:19.842 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:19 smithi092 bash[58855]: cluster 2023-09-23T19:47:18.334213+0000 mgr.y (mgr.25325) 1197 : cluster 0 pgmap v1179: 89 pgs: 89 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-09-23T19:47:19.851 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:47:19.860 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:19.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:19.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:47:19.964 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:19.995 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:19.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:19.998 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:47:20.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:19 smithi016 bash[75004]: cluster 2023-09-23T19:47:18.334213+0000 mgr.y (mgr.25325) 1197 : cluster 0 pgmap v1179: 89 pgs: 89 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-09-23T19:47:20.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:19 smithi016 bash[77995]: cluster 2023-09-23T19:47:18.334213+0000 mgr.y (mgr.25325) 1197 : cluster 0 pgmap v1179: 89 pgs: 89 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-09-23T19:47:20.098 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:20.125 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:20.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:20.132 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:20.150 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 46 16 64433 64417 87.5084 39.6875 0.00734959 0.0114215 2023-09-23T19:47:20.233 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:20.265 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:20.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:20.269 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:47:20.370 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:20.401 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:20.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:20.405 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:20.506 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:20.534 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:20.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:20.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:47:20.638 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:20.662 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:20.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:20.669 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:47:20.770 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:20.803 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:20.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:20.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:20.913 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:20.939 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:20.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:20.946 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:47:21.047 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:21.075 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:21.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:21.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:21.105 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:47:21.150 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 47 16 65470 65454 87.0254 64.8125 0.00930392 0.0114842 2023-09-23T19:47:21.180 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:21.205 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:21.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:21.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:21.309 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:21.338 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:21.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:21.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:21.443 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:21.468 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:21.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:21.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:21.574 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:21.600 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:21.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:21.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:47:21.704 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:21.732 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:21.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:21.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:21.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:21 smithi092 bash[58855]: cluster 2023-09-23T19:47:20.335065+0000 mgr.y (mgr.25325) 1198 : cluster 0 pgmap v1180: 89 pgs: 89 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-09-23T19:47:21.835 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:21.860 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:21.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:21.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:47:21.899 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:47:21.925 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:21 smithi016 bash[77995]: cluster 2023-09-23T19:47:20.335065+0000 mgr.y (mgr.25325) 1198 : cluster 0 pgmap v1180: 89 pgs: 89 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-09-23T19:47:21.925 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:21 smithi016 bash[75004]: cluster 2023-09-23T19:47:20.335065+0000 mgr.y (mgr.25325) 1198 : cluster 0 pgmap v1180: 89 pgs: 89 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-09-23T19:47:21.963 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:21.986 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:21.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:21.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:22.088 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:22.116 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:22.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:22.119 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:22.150 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 48 16 66523 66507 86.5833 65.8125 0.0125339 0.0115436 2023-09-23T19:47:22.220 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:22.252 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:22.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:22.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:47:22.357 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:22.387 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:22.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:22.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:22.494 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:22.526 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:22.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:22.530 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:22.630 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:22.668 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:22.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:22.673 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:22.774 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:22.800 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:22.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:22.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:47:22.904 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:22.941 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:22.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:22.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:23.044 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:23.071 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:23.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:23.075 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:23.150 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 49 16 67586 67570 86.1721 66.4375 0.0111809 0.0115989 2023-09-23T19:47:23.176 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:23.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:23.215 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:23.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:47:23.316 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:23.349 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:23.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:23.354 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:47:23.454 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:23.486 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:23.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:23.491 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:23.591 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:23.619 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:23.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:23.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:23.725 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:23.735 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:23 smithi092 bash[58855]: cluster 2023-09-23T19:47:22.335422+0000 mgr.y (mgr.25325) 1199 : cluster 0 pgmap v1181: 89 pgs: 89 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-09-23T19:47:23.754 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:23.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:23.756 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:47:23.857 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:23.867 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:23 smithi016 bash[77995]: cluster 2023-09-23T19:47:22.335422+0000 mgr.y (mgr.25325) 1199 : cluster 0 pgmap v1181: 89 pgs: 89 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-09-23T19:47:23.867 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:23 smithi016 bash[75004]: cluster 2023-09-23T19:47:22.335422+0000 mgr.y (mgr.25325) 1199 : cluster 0 pgmap v1181: 89 pgs: 89 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-09-23T19:47:23.889 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:23.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:23.891 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:23.992 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:24.019 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:24.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:24.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:24.125 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:24.151 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 50 16 68579 68563 85.6894 62.0625 0.0106279 0.011663 2023-09-23T19:47:24.155 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:24.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:24.158 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:24.258 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:24.285 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:24.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:24.288 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:24.389 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:24.418 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:24.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:24.423 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:24.523 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:24.562 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:24.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:24.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:47:24.667 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:24.695 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:24.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:24.704 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:24.805 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:24.841 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:24.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:24.844 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:24.945 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:24.969 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:24.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:24.972 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:47:25.072 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:25.102 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:25.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:25.108 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:25.151 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 51 16 69507 69491 85.1464 58 0.00872474 0.0117379 2023-09-23T19:47:25.210 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:25.248 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:25.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:25.253 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:47:25.354 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:25.395 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:25.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:25.398 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:47:25.499 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:25.529 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:25.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:25.533 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:47:25.634 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:25.657 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:25.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:25.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:47:25.765 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:25.774 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:25 smithi016 bash[77995]: cluster 2023-09-23T19:47:24.335928+0000 mgr.y (mgr.25325) 1200 : cluster 0 pgmap v1182: 89 pgs: 89 active+clean; 4.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T19:47:25.774 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:25 smithi016 bash[75004]: cluster 2023-09-23T19:47:24.335928+0000 mgr.y (mgr.25325) 1200 : cluster 0 pgmap v1182: 89 pgs: 89 active+clean; 4.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T19:47:25.774 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:47:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:47:25] "GET /metrics HTTP/1.1" 200 36305 "" "Prometheus/2.33.4" 2023-09-23T19:47:25.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:25 smithi092 bash[58855]: cluster 2023-09-23T19:47:24.335928+0000 mgr.y (mgr.25325) 1200 : cluster 0 pgmap v1182: 89 pgs: 89 active+clean; 4.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T19:47:25.798 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:25.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:25.802 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:25.903 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:25.907 INFO:teuthology.orchestra.run.smithi092.stdout:{ 2023-09-23T19:47:25.907 INFO:teuthology.orchestra.run.smithi092.stdout: "ops": [], 2023-09-23T19:47:25.907 INFO:teuthology.orchestra.run.smithi092.stdout: "complaint_time": 30, 2023-09-23T19:47:25.907 INFO:teuthology.orchestra.run.smithi092.stdout: "num_blocked_ops": 0 2023-09-23T19:47:25.907 INFO:teuthology.orchestra.run.smithi092.stdout:} 2023-09-23T19:47:25.932 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:25.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:25.938 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:26.039 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:26.060 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:26.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:26.062 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:47:26.151 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 52 16 70548 70532 84.7601 65.0625 0.012822 0.011792 2023-09-23T19:47:26.162 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:26.194 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:26.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:26.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:26.299 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:26.327 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:26.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:26.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:47:26.432 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:26.464 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:26.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:26.466 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:47:26.567 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:26.601 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:26.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:26.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:26.707 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:26.738 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:26.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:26.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:26.843 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:26.863 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:26.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:26.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:26.966 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:26.990 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:26.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:26.992 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:47:27.087 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T19:47:27.092 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:27.120 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:27.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:27.123 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:27.151 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 53 16 71445 71429 84.2185 56.0625 0.00836165 0.0118679 2023-09-23T19:47:27.225 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:27.264 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:27.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:27.269 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:47:27.369 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:27.402 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:27.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:27.407 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:27.508 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:27.534 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:27.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:27.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:27.638 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:27.654 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:27 smithi016 bash[75004]: cluster 2023-09-23T19:47:26.336957+0000 mgr.y (mgr.25325) 1201 : cluster 0 pgmap v1183: 89 pgs: 89 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-09-23T19:47:27.663 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:27.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:27.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:47:27.766 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:27.806 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:27.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:27.809 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:27.910 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:27.919 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:27 smithi092 bash[58855]: cluster 2023-09-23T19:47:26.336957+0000 mgr.y (mgr.25325) 1201 : cluster 0 pgmap v1183: 89 pgs: 89 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-09-23T19:47:27.939 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:27.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:27.945 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:47:28.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:27 smithi016 bash[77995]: cluster 2023-09-23T19:47:26.336957+0000 mgr.y (mgr.25325) 1201 : cluster 0 pgmap v1183: 89 pgs: 89 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-09-23T19:47:28.046 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:28.069 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:28.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:28.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:28.151 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 54 16 72439 72423 83.8093 62.125 0.0117311 0.0119256 2023-09-23T19:47:28.176 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:28.198 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:28.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:28.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:28.300 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:28.327 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:28.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:28.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:47:28.430 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:28.458 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:28.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:28.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:47:28.565 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:28.590 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:28.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:28.592 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:47:28.693 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:28.719 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:28.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:28.722 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:28.823 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:28.853 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:28.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:28.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:28.955 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:28.981 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:28.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:28.985 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:29.085 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:29.114 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:29.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:29.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:29.151 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 55 16 73431 73415 83.4127 62 0.00609904 0.0119828 2023-09-23T19:47:29.219 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:29.248 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:29.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:29.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:47:29.353 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:29.389 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:29.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:29.391 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:47:29.492 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:29.517 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:29.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:29.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:29.621 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:29.649 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:29.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:29.653 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:47:29.753 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:29.763 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:29 smithi016 bash[75004]: cluster 2023-09-23T19:47:28.337478+0000 mgr.y (mgr.25325) 1202 : cluster 0 pgmap v1184: 89 pgs: 89 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 58 MiB/s wr, 926 op/s 2023-09-23T19:47:29.763 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:29 smithi016 bash[75004]: audit 2023-09-23T19:47:28.667556+0000 mon.a (mon.0) 1515 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:47:29.764 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:29 smithi016 bash[77995]: cluster 2023-09-23T19:47:28.337478+0000 mgr.y (mgr.25325) 1202 : cluster 0 pgmap v1184: 89 pgs: 89 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 58 MiB/s wr, 926 op/s 2023-09-23T19:47:29.764 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:29 smithi016 bash[77995]: audit 2023-09-23T19:47:28.667556+0000 mon.a (mon.0) 1515 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:47:29.781 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:29.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:29.783 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:29.884 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:29.908 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:29.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:29.912 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:30.013 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:30.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:29 smithi092 bash[58855]: cluster 2023-09-23T19:47:28.337478+0000 mgr.y (mgr.25325) 1202 : cluster 0 pgmap v1184: 89 pgs: 89 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 58 MiB/s wr, 926 op/s 2023-09-23T19:47:30.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:29 smithi092 bash[58855]: audit 2023-09-23T19:47:28.667556+0000 mon.a (mon.0) 1515 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:47:30.041 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:30.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:30.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:30.146 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:30.151 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 56 16 74399 74383 83.0035 60.5 0.0213464 0.0120415 2023-09-23T19:47:30.191 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:30.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:30.195 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:47:30.296 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:30.333 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:30.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:30.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:30.438 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:30.471 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:30.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:30.477 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:30.577 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:30.606 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:30.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:30.612 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:30.713 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:30.744 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:30.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:30.747 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:47:30.847 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:30.897 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:30.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:30.900 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:31.001 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:31.039 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:31.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:31.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:47:31.143 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:31.151 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 57 16 75282 75266 82.5154 55.1875 0.011793 0.0121119 2023-09-23T19:47:31.180 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:31.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:31.184 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:31.214 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:47:31.269 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:47:31.285 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:31.312 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:31.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:31.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:47:31.325 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:47:31.374 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:47:31.415 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:31.423 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:47:31.438 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:31.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:31.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:47:31.466 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:47:31.505 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:47:31.542 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:31.556 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:47:31.563 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:47:31.563 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:47:31.563 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:47:31.563 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:47:31.563 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:47:31.563 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:47:31.563 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:47:31.563 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:47:31.599 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:31.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:31.604 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:31.705 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:31.739 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:31.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:31.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:47:31.842 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:31.853 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:31 smithi016 bash[75004]: cluster 2023-09-23T19:47:30.338179+0000 mgr.y (mgr.25325) 1203 : cluster 0 pgmap v1185: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-09-23T19:47:31.853 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:31 smithi016 bash[75004]: audit 2023-09-23T19:47:31.147228+0000 mon.c (mon.2) 195 : audit 0 from='client.? 172.21.15.16:0/1400066581' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:47:31.853 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:31 smithi016 bash[77995]: cluster 2023-09-23T19:47:30.338179+0000 mgr.y (mgr.25325) 1203 : cluster 0 pgmap v1185: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-09-23T19:47:31.854 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:31 smithi016 bash[77995]: audit 2023-09-23T19:47:31.147228+0000 mon.c (mon.2) 195 : audit 0 from='client.? 172.21.15.16:0/1400066581' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:47:31.873 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-09-23T19:47:31.883 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:31.884 INFO:teuthology.orchestra.run.smithi016.stderr:dumped all 2023-09-23T19:47:31.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:31.885 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:31.918 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:47:31.943 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:31 smithi092 bash[58855]: cluster 2023-09-23T19:47:30.338179+0000 mgr.y (mgr.25325) 1203 : cluster 0 pgmap v1185: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-09-23T19:47:31.943 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:31 smithi092 bash[58855]: audit 2023-09-23T19:47:31.147228+0000 mon.c (mon.2) 195 : audit 0 from='client.? 172.21.15.16:0/1400066581' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:47:31.986 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:32.014 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:32.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:32.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:47:32.121 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:32.143 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:32.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:32.146 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:47:32.151 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 58 16 76036 76020 81.9051 47.125 0.0240108 0.0122032 2023-09-23T19:47:32.247 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:32.290 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:32.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:32.293 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:47:32.393 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:32.423 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:32.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:32.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:32.528 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:32.561 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:32.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:32.565 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:47:32.666 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:32.691 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is unset 2023-09-23T19:47:32.694 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:32.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:32.698 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:47:32.799 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:32.823 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:32.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:32.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:32.927 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:32.954 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:32.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:32.959 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:47:33.059 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:33.085 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:33.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:33.087 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:47:33.152 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 59 16 76882 76866 81.4129 52.875 0.00332743 0.0122572 2023-09-23T19:47:33.167 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:32 smithi016 bash[75004]: audit 2023-09-23T19:47:31.778001+0000 mon.a (mon.0) 1516 : audit 1 from='client.? 172.21.15.16:0/3301369627' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:47:33.168 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:32 smithi016 bash[75004]: audit 2023-09-23T19:47:31.849446+0000 mgr.y (mgr.25325) 1204 : audit 0 from='client.46472 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:47:33.168 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:32 smithi016 bash[77995]: audit 2023-09-23T19:47:31.778001+0000 mon.a (mon.0) 1516 : audit 1 from='client.? 172.21.15.16:0/3301369627' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:47:33.168 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:32 smithi016 bash[77995]: audit 2023-09-23T19:47:31.849446+0000 mgr.y (mgr.25325) 1204 : audit 0 from='client.46472 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:47:33.188 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:33.220 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:33.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:33.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:33.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:32 smithi092 bash[58855]: audit 2023-09-23T19:47:31.778001+0000 mon.a (mon.0) 1516 : audit 1 from='client.? 172.21.15.16:0/3301369627' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:47:33.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:32 smithi092 bash[58855]: audit 2023-09-23T19:47:31.849446+0000 mgr.y (mgr.25325) 1204 : audit 0 from='client.46472 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:47:33.326 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:33.352 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:33.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:33.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:33.456 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:33.489 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:33.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:33.493 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:33.593 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:33.618 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:33.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:33.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:47:33.724 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:33.746 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:33.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:33.749 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:47:33.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:33 smithi016 bash[75004]: cluster 2023-09-23T19:47:32.338708+0000 mgr.y (mgr.25325) 1205 : cluster 0 pgmap v1186: 89 pgs: 89 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 54 MiB/s wr, 860 op/s 2023-09-23T19:47:33.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:33 smithi016 bash[75004]: cluster 2023-09-23T19:47:32.658858+0000 mon.a (mon.0) 1517 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:47:33.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:33 smithi016 bash[75004]: cluster 2023-09-23T19:47:32.659724+0000 mon.a (mon.0) 1518 : cluster 1 Cluster is now healthy 2023-09-23T19:47:33.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:33 smithi016 bash[75004]: audit 2023-09-23T19:47:32.683629+0000 mon.a (mon.0) 1519 : audit 1 from='client.? 172.21.15.16:0/3301369627' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:47:33.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:33 smithi016 bash[75004]: cluster 2023-09-23T19:47:32.683742+0000 mon.a (mon.0) 1520 : cluster 0 osdmap e215: 8 total, 8 up, 8 in 2023-09-23T19:47:33.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:33 smithi016 bash[75004]: cluster 2023-09-23T19:47:32.696043+0000 osd.6 (osd.6) 145 : cluster 3 failed to encode map e215 with expected crc 2023-09-23T19:47:33.788 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:33 smithi016 bash[75004]: cluster 2023-09-23T19:47:32.707223+0000 osd.4 (osd.4) 157 : cluster 3 failed to encode map e215 with expected crc 2023-09-23T19:47:33.788 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:33 smithi016 bash[77995]: cluster 2023-09-23T19:47:32.338708+0000 mgr.y (mgr.25325) 1205 : cluster 0 pgmap v1186: 89 pgs: 89 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 54 MiB/s wr, 860 op/s 2023-09-23T19:47:33.788 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:33 smithi016 bash[77995]: cluster 2023-09-23T19:47:32.658858+0000 mon.a (mon.0) 1517 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:47:33.788 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:33 smithi016 bash[77995]: cluster 2023-09-23T19:47:32.659724+0000 mon.a (mon.0) 1518 : cluster 1 Cluster is now healthy 2023-09-23T19:47:33.788 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:33 smithi016 bash[77995]: audit 2023-09-23T19:47:32.683629+0000 mon.a (mon.0) 1519 : audit 1 from='client.? 172.21.15.16:0/3301369627' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:47:33.788 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:33 smithi016 bash[77995]: cluster 2023-09-23T19:47:32.683742+0000 mon.a (mon.0) 1520 : cluster 0 osdmap e215: 8 total, 8 up, 8 in 2023-09-23T19:47:33.788 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:33 smithi016 bash[77995]: cluster 2023-09-23T19:47:32.696043+0000 osd.6 (osd.6) 145 : cluster 3 failed to encode map e215 with expected crc 2023-09-23T19:47:33.789 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:33 smithi016 bash[77995]: cluster 2023-09-23T19:47:32.707223+0000 osd.4 (osd.4) 157 : cluster 3 failed to encode map e215 with expected crc 2023-09-23T19:47:33.849 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:33.887 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:33.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:33.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:33.980 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:33 smithi092 bash[58855]: cluster 2023-09-23T19:47:32.338708+0000 mgr.y (mgr.25325) 1205 : cluster 0 pgmap v1186: 89 pgs: 89 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 54 MiB/s wr, 860 op/s 2023-09-23T19:47:33.980 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:33 smithi092 bash[58855]: cluster 2023-09-23T19:47:32.658858+0000 mon.a (mon.0) 1517 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:47:33.980 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:33 smithi092 bash[58855]: cluster 2023-09-23T19:47:32.659724+0000 mon.a (mon.0) 1518 : cluster 1 Cluster is now healthy 2023-09-23T19:47:33.980 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:33 smithi092 bash[58855]: audit 2023-09-23T19:47:32.683629+0000 mon.a (mon.0) 1519 : audit 1 from='client.? 172.21.15.16:0/3301369627' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:47:33.980 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:33 smithi092 bash[58855]: cluster 2023-09-23T19:47:32.683742+0000 mon.a (mon.0) 1520 : cluster 0 osdmap e215: 8 total, 8 up, 8 in 2023-09-23T19:47:33.980 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:33 smithi092 bash[58855]: cluster 2023-09-23T19:47:32.696043+0000 osd.6 (osd.6) 145 : cluster 3 failed to encode map e215 with expected crc 2023-09-23T19:47:33.981 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:33 smithi092 bash[58855]: cluster 2023-09-23T19:47:32.707223+0000 osd.4 (osd.4) 157 : cluster 3 failed to encode map e215 with expected crc 2023-09-23T19:47:33.993 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:34.039 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:34.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:34.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:34.143 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:34.151 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-09-23T19:47:34.148504+0000 min lat: 0.00140724 max lat: 0.404369 avg lat: 0.012345 2023-09-23T19:47:34.152 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:47:34.152 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 60 16 77749 77733 80.9591 54.1875 0.0110159 0.012345 2023-09-23T19:47:34.162 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:34.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:34.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:34.264 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:34.292 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:34.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:34.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:47:34.396 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:34.431 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:34.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:34.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:34.535 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:34.566 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:34.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:34.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:34.675 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:34.816 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:34 smithi016 bash[75004]: cluster 2023-09-23T19:47:32.690773+0000 osd.7 (osd.7) 134 : cluster 3 failed to encode map e215 with expected crc 2023-09-23T19:47:34.817 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:34 smithi016 bash[77995]: cluster 2023-09-23T19:47:32.690773+0000 osd.7 (osd.7) 134 : cluster 3 failed to encode map e215 with expected crc 2023-09-23T19:47:34.840 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:34.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:34.843 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:34.943 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:34.974 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:34.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:34.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:47:35.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:34 smithi092 bash[58855]: cluster 2023-09-23T19:47:32.690773+0000 osd.7 (osd.7) 134 : cluster 3 failed to encode map e215 with expected crc 2023-09-23T19:47:35.080 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:35.102 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:35.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:35.105 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:47:35.152 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 61 16 78718 78702 80.6247 60.5625 0.011235 0.012396 2023-09-23T19:47:35.206 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:35.243 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:35.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:35.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:35.350 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:35.393 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:35.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:35.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:35.498 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:35.528 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:35.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:35.533 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:47:35.604 INFO:teuthology.orchestra.run.smithi016.stdout:{"pg_ready":true,"pg_map":{"version":1185,"stamp":"2023-09-23T19:47:30.337676+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4774893472,"num_objects":72874,"num_object_clones":0,"num_object_copies":218622,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":72874,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":73111,"num_write_kb":4664846,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":73045,"ondisk_log_size":73045,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":19682876,"kb_used_data":13865276,"kb_used_omap":20,"kb_used_meta":5817515,"kb_avail":730278340,"statfs":{"total":767960285184,"available":747805020160,"internally_reserved":0,"allocated":14198042624,"data_stored":14192590591,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":5957136014},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,3,12,9,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":841154560,"num_objects":12835,"num_object_clones":0,"num_object_copies":38505,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12835,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12835,"num_write_kb":821440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003827"},"pg_stats":[{"pgid":"3.1a","version":"59'1","reported_seq":210,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.749223+0000","last_change":"2023-09-23T19:38:43.897548+0000","last_active":"2023-09-23T19:46:54.749223+0000","last_peered":"2023-09-23T19:46:54.749223+0000","last_clean":"2023-09-23T19:46:54.749223+0000","last_became_active":"2023-09-23T19:38:43.896868+0000","last_became_peered":"2023-09-23T19:38:43.896868+0000","last_unstale":"2023-09-23T19:46:54.749223+0000","last_undegraded":"2023-09-23T19:46:54.749223+0000","last_fullsized":"2023-09-23T19:46:54.749223+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":172,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.734693+0000","last_change":"2023-09-23T19:32:49.731128+0000","last_active":"2023-09-23T19:46:54.734693+0000","last_peered":"2023-09-23T19:46:54.734693+0000","last_clean":"2023-09-23T19:46:54.734693+0000","last_became_active":"2023-09-23T19:32:49.730757+0000","last_became_peered":"2023-09-23T19:32:49.730757+0000","last_unstale":"2023-09-23T19:46:54.734693+0000","last_undegraded":"2023-09-23T19:46:54.734693+0000","last_fullsized":"2023-09-23T19:46:54.734693+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:40.183605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":161,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:55.993846+0000","last_change":"2023-09-23T19:28:07.890462+0000","last_active":"2023-09-23T19:46:55.993846+0000","last_peered":"2023-09-23T19:46:55.993846+0000","last_clean":"2023-09-23T19:46:55.993846+0000","last_became_active":"2023-09-23T19:28:07.890084+0000","last_became_peered":"2023-09-23T19:28:07.890084+0000","last_unstale":"2023-09-23T19:46:55.993846+0000","last_undegraded":"2023-09-23T19:46:55.993846+0000","last_fullsized":"2023-09-23T19:46:55.993846+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:50.458218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":100,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.744869+0000","last_change":"2023-09-23T19:42:55.772143+0000","last_active":"2023-09-23T19:46:54.744869+0000","last_peered":"2023-09-23T19:46:54.744869+0000","last_clean":"2023-09-23T19:46:54.744869+0000","last_became_active":"2023-09-23T19:42:55.771813+0000","last_became_peered":"2023-09-23T19:42:55.771813+0000","last_unstale":"2023-09-23T19:46:54.744869+0000","last_undegraded":"2023-09-23T19:46:54.744869+0000","last_fullsized":"2023-09-23T19:46:54.744869+0000","mapping_epoch":188,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":190,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T19:47:13.645649+0000","last_change":"2023-09-23T19:32:49.729961+0000","last_active":"2023-09-23T19:47:13.645649+0000","last_peered":"2023-09-23T19:47:13.645649+0000","last_clean":"2023-09-23T19:47:13.645649+0000","last_became_active":"2023-09-23T19:32:49.729251+0000","last_became_peered":"2023-09-23T19:32:49.729251+0000","last_unstale":"2023-09-23T19:47:13.645649+0000","last_undegraded":"2023-09-23T19:47:13.645649+0000","last_fullsized":"2023-09-23T19:47:13.645649+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:20:56.595732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":179,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.746423+0000","last_change":"2023-09-23T19:22:42.150582+0000","last_active":"2023-09-23T19:46:54.746423+0000","last_peered":"2023-09-23T19:46:54.746423+0000","last_clean":"2023-09-23T19:46:54.746423+0000","last_became_active":"2023-09-23T19:22:42.150229+0000","last_became_peered":"2023-09-23T19:22:42.150229+0000","last_unstale":"2023-09-23T19:46:54.746423+0000","last_undegraded":"2023-09-23T19:46:54.746423+0000","last_fullsized":"2023-09-23T19:46:54.746423+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":181,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:47:08.728615+0000","last_change":"2023-09-23T19:43:38.571327+0000","last_active":"2023-09-23T19:47:08.728615+0000","last_peered":"2023-09-23T19:47:08.728615+0000","last_clean":"2023-09-23T19:47:08.728615+0000","last_became_active":"2023-09-23T19:43:38.570994+0000","last_became_peered":"2023-09-23T19:43:38.570994+0000","last_unstale":"2023-09-23T19:47:08.728615+0000","last_undegraded":"2023-09-23T19:47:08.728615+0000","last_fullsized":"2023-09-23T19:47:08.728615+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:50:11.363431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":173,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.734861+0000","last_change":"2023-09-23T19:32:49.729765+0000","last_active":"2023-09-23T19:46:54.734861+0000","last_peered":"2023-09-23T19:46:54.734861+0000","last_clean":"2023-09-23T19:46:54.734861+0000","last_became_active":"2023-09-23T19:32:49.729007+0000","last_became_peered":"2023-09-23T19:32:49.729007+0000","last_unstale":"2023-09-23T19:46:54.734861+0000","last_undegraded":"2023-09-23T19:46:54.734861+0000","last_fullsized":"2023-09-23T19:46:54.734861+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:59:31.183753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":90,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.745096+0000","last_change":"2023-09-23T19:43:38.620503+0000","last_active":"2023-09-23T19:46:54.745096+0000","last_peered":"2023-09-23T19:46:54.745096+0000","last_clean":"2023-09-23T19:46:54.745096+0000","last_became_active":"2023-09-23T19:43:38.619167+0000","last_became_peered":"2023-09-23T19:43:38.619167+0000","last_unstale":"2023-09-23T19:46:54.745096+0000","last_undegraded":"2023-09-23T19:46:54.745096+0000","last_fullsized":"2023-09-23T19:46:54.745096+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":154,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.743816+0000","last_change":"2023-09-23T19:32:49.736864+0000","last_active":"2023-09-23T19:46:54.743816+0000","last_peered":"2023-09-23T19:46:54.743816+0000","last_clean":"2023-09-23T19:46:54.743816+0000","last_became_active":"2023-09-23T19:32:49.736523+0000","last_became_peered":"2023-09-23T19:32:49.736523+0000","last_unstale":"2023-09-23T19:46:54.743816+0000","last_undegraded":"2023-09-23T19:46:54.743816+0000","last_fullsized":"2023-09-23T19:46:54.743816+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":183,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.744720+0000","last_change":"2023-09-23T19:28:07.892835+0000","last_active":"2023-09-23T19:46:54.744720+0000","last_peered":"2023-09-23T19:46:54.744720+0000","last_clean":"2023-09-23T19:46:54.744720+0000","last_became_active":"2023-09-23T19:28:07.892338+0000","last_became_peered":"2023-09-23T19:28:07.892338+0000","last_unstale":"2023-09-23T19:46:54.744720+0000","last_undegraded":"2023-09-23T19:46:54.744720+0000","last_fullsized":"2023-09-23T19:46:54.744720+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":176,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.735294+0000","last_change":"2023-09-23T19:38:43.890137+0000","last_active":"2023-09-23T19:46:54.735294+0000","last_peered":"2023-09-23T19:46:54.735294+0000","last_clean":"2023-09-23T19:46:54.735294+0000","last_became_active":"2023-09-23T19:38:43.889800+0000","last_became_peered":"2023-09-23T19:38:43.889800+0000","last_unstale":"2023-09-23T19:46:54.735294+0000","last_undegraded":"2023-09-23T19:46:54.735294+0000","last_fullsized":"2023-09-23T19:46:54.735294+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:59:20.468913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'23","reported_seq":187,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.749208+0000","last_change":"2023-09-23T19:05:52.107727+0000","last_active":"2023-09-23T19:46:54.749208+0000","last_peered":"2023-09-23T19:46:54.749208+0000","last_clean":"2023-09-23T19:46:54.749208+0000","last_became_active":"2023-09-23T19:05:52.107378+0000","last_became_peered":"2023-09-23T19:05:52.107378+0000","last_unstale":"2023-09-23T19:46:54.749208+0000","last_undegraded":"2023-09-23T19:46:54.749208+0000","last_fullsized":"2023-09-23T19:46:54.749208+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":178,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.749526+0000","last_change":"2023-09-23T19:22:42.148497+0000","last_active":"2023-09-23T19:46:54.749526+0000","last_peered":"2023-09-23T19:46:54.749526+0000","last_clean":"2023-09-23T19:46:54.749526+0000","last_became_active":"2023-09-23T19:22:42.148147+0000","last_became_peered":"2023-09-23T19:22:42.148147+0000","last_unstale":"2023-09-23T19:46:54.749526+0000","last_undegraded":"2023-09-23T19:46:54.749526+0000","last_fullsized":"2023-09-23T19:46:54.749526+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":197,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.744564+0000","last_change":"2023-09-23T19:38:43.897378+0000","last_active":"2023-09-23T19:46:54.744564+0000","last_peered":"2023-09-23T19:46:54.744564+0000","last_clean":"2023-09-23T19:46:54.744564+0000","last_became_active":"2023-09-23T19:38:43.897012+0000","last_became_peered":"2023-09-23T19:38:43.897012+0000","last_unstale":"2023-09-23T19:46:54.744564+0000","last_undegraded":"2023-09-23T19:46:54.744564+0000","last_fullsized":"2023-09-23T19:46:54.744564+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":173,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:55.993932+0000","last_change":"2023-09-23T19:32:49.730549+0000","last_active":"2023-09-23T19:46:55.993932+0000","last_peered":"2023-09-23T19:46:55.993932+0000","last_clean":"2023-09-23T19:46:55.993932+0000","last_became_active":"2023-09-23T19:32:49.730105+0000","last_became_peered":"2023-09-23T19:32:49.730105+0000","last_unstale":"2023-09-23T19:46:55.993932+0000","last_undegraded":"2023-09-23T19:46:55.993932+0000","last_fullsized":"2023-09-23T19:46:55.993932+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:49.751893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":100,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.744391+0000","last_change":"2023-09-23T19:42:55.772691+0000","last_active":"2023-09-23T19:46:54.744391+0000","last_peered":"2023-09-23T19:46:54.744391+0000","last_clean":"2023-09-23T19:46:54.744391+0000","last_became_active":"2023-09-23T19:42:55.772309+0000","last_became_peered":"2023-09-23T19:42:55.772309+0000","last_unstale":"2023-09-23T19:46:54.744391+0000","last_undegraded":"2023-09-23T19:46:54.744391+0000","last_fullsized":"2023-09-23T19:46:54.744391+0000","mapping_epoch":188,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":107,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.745144+0000","last_change":"2023-09-23T19:43:38.577960+0000","last_active":"2023-09-23T19:46:54.745144+0000","last_peered":"2023-09-23T19:46:54.745144+0000","last_clean":"2023-09-23T19:46:54.745144+0000","last_became_active":"2023-09-23T19:43:38.577391+0000","last_became_peered":"2023-09-23T19:43:38.577391+0000","last_unstale":"2023-09-23T19:46:54.745144+0000","last_undegraded":"2023-09-23T19:46:54.745144+0000","last_fullsized":"2023-09-23T19:46:54.745144+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":165,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.744287+0000","last_change":"2023-09-23T19:05:52.114186+0000","last_active":"2023-09-23T19:46:54.744287+0000","last_peered":"2023-09-23T19:46:54.744287+0000","last_clean":"2023-09-23T19:46:54.744287+0000","last_became_active":"2023-09-23T19:05:52.113635+0000","last_became_peered":"2023-09-23T19:05:52.113635+0000","last_unstale":"2023-09-23T19:46:54.744287+0000","last_undegraded":"2023-09-23T19:46:54.744287+0000","last_fullsized":"2023-09-23T19:46:54.744287+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":100,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.744996+0000","last_change":"2023-09-23T19:42:55.772912+0000","last_active":"2023-09-23T19:46:54.744996+0000","last_peered":"2023-09-23T19:46:54.744996+0000","last_clean":"2023-09-23T19:46:54.744996+0000","last_became_active":"2023-09-23T19:42:55.772434+0000","last_became_peered":"2023-09-23T19:42:55.772434+0000","last_unstale":"2023-09-23T19:46:54.744996+0000","last_undegraded":"2023-09-23T19:46:54.744996+0000","last_fullsized":"2023-09-23T19:46:54.744996+0000","mapping_epoch":188,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":188,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.744007+0000","last_change":"2023-09-23T19:18:09.010195+0000","last_active":"2023-09-23T19:46:54.744007+0000","last_peered":"2023-09-23T19:46:54.744007+0000","last_clean":"2023-09-23T19:46:54.744007+0000","last_became_active":"2023-09-23T19:18:09.009687+0000","last_became_peered":"2023-09-23T19:18:09.009687+0000","last_unstale":"2023-09-23T19:46:54.744007+0000","last_undegraded":"2023-09-23T19:46:54.744007+0000","last_fullsized":"2023-09-23T19:46:54.744007+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":182,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T19:47:13.631936+0000","last_change":"2023-09-23T19:28:07.885265+0000","last_active":"2023-09-23T19:47:13.631936+0000","last_peered":"2023-09-23T19:47:13.631936+0000","last_clean":"2023-09-23T19:47:13.631936+0000","last_became_active":"2023-09-23T19:28:07.884353+0000","last_became_peered":"2023-09-23T19:28:07.884353+0000","last_unstale":"2023-09-23T19:47:13.631936+0000","last_undegraded":"2023-09-23T19:47:13.631936+0000","last_fullsized":"2023-09-23T19:47:13.631936+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:56.325352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.0","version":"214'4734","reported_seq":4749,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T19:47:29.574975+0000","last_change":"2023-09-23T19:46:17.720507+0000","last_active":"2023-09-23T19:47:29.574975+0000","last_peered":"2023-09-23T19:47:29.574975+0000","last_clean":"2023-09-23T19:47:29.574975+0000","last_became_active":"2023-09-23T19:46:17.720145+0000","last_became_peered":"2023-09-23T19:46:17.720145+0000","last_unstale":"2023-09-23T19:47:29.574975+0000","last_undegraded":"2023-09-23T19:47:29.574975+0000","last_fullsized":"2023-09-23T19:47:29.574975+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_clean_scrub_stamp":"2023-09-23T19:46:16.677636+0000","objects_scrubbed":0,"log_size":4734,"log_dups_size":0,"ondisk_log_size":4734,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":310247424,"num_objects":4734,"num_object_clones":0,"num_object_copies":14202,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4734,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4734,"num_write_kb":302976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":181,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.745204+0000","last_change":"2023-09-23T19:18:09.035116+0000","last_active":"2023-09-23T19:46:54.745204+0000","last_peered":"2023-09-23T19:46:54.745204+0000","last_clean":"2023-09-23T19:46:54.745204+0000","last_became_active":"2023-09-23T19:18:09.034416+0000","last_became_peered":"2023-09-23T19:18:09.034416+0000","last_unstale":"2023-09-23T19:46:54.745204+0000","last_undegraded":"2023-09-23T19:46:54.745204+0000","last_fullsized":"2023-09-23T19:46:54.745204+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":33,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.743947+0000","last_change":"2023-09-23T19:43:38.579831+0000","last_active":"2023-09-23T19:46:54.743947+0000","last_peered":"2023-09-23T19:46:54.743947+0000","last_clean":"2023-09-23T19:46:54.743947+0000","last_became_active":"2023-09-23T19:43:38.579398+0000","last_became_peered":"2023-09-23T19:43:38.579398+0000","last_unstale":"2023-09-23T19:46:54.743947+0000","last_undegraded":"2023-09-23T19:46:54.743947+0000","last_fullsized":"2023-09-23T19:46:54.743947+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.1","version":"214'4545","reported_seq":4560,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T19:47:29.750473+0000","last_change":"2023-09-23T19:46:17.828580+0000","last_active":"2023-09-23T19:47:29.750473+0000","last_peered":"2023-09-23T19:47:29.750473+0000","last_clean":"2023-09-23T19:47:29.750473+0000","last_became_active":"2023-09-23T19:46:17.827806+0000","last_became_peered":"2023-09-23T19:46:17.827806+0000","last_unstale":"2023-09-23T19:47:29.750473+0000","last_undegraded":"2023-09-23T19:47:29.750473+0000","last_fullsized":"2023-09-23T19:47:29.750473+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_clean_scrub_stamp":"2023-09-23T19:46:16.677636+0000","objects_scrubbed":0,"log_size":4545,"log_dups_size":0,"ondisk_log_size":4545,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297861120,"num_objects":4545,"num_object_clones":0,"num_object_copies":13635,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4545,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4545,"num_write_kb":290880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":182,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.746867+0000","last_change":"2023-09-23T19:22:42.151134+0000","last_active":"2023-09-23T19:46:54.746867+0000","last_peered":"2023-09-23T19:46:54.746867+0000","last_clean":"2023-09-23T19:46:54.746867+0000","last_became_active":"2023-09-23T19:22:42.150553+0000","last_became_peered":"2023-09-23T19:22:42.150553+0000","last_unstale":"2023-09-23T19:46:54.746867+0000","last_undegraded":"2023-09-23T19:46:54.746867+0000","last_fullsized":"2023-09-23T19:46:54.746867+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":193,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.748934+0000","last_change":"2023-09-23T19:28:07.899262+0000","last_active":"2023-09-23T19:46:54.748934+0000","last_peered":"2023-09-23T19:46:54.748934+0000","last_clean":"2023-09-23T19:46:54.748934+0000","last_became_active":"2023-09-23T19:28:07.898848+0000","last_became_peered":"2023-09-23T19:28:07.898848+0000","last_unstale":"2023-09-23T19:46:54.748934+0000","last_undegraded":"2023-09-23T19:46:54.748934+0000","last_fullsized":"2023-09-23T19:46:54.748934+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"214'4605","reported_seq":4620,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T19:47:29.581058+0000","last_change":"2023-09-23T19:46:17.828372+0000","last_active":"2023-09-23T19:47:29.581058+0000","last_peered":"2023-09-23T19:47:29.581058+0000","last_clean":"2023-09-23T19:47:29.581058+0000","last_became_active":"2023-09-23T19:46:17.827674+0000","last_became_peered":"2023-09-23T19:46:17.827674+0000","last_unstale":"2023-09-23T19:47:29.581058+0000","last_undegraded":"2023-09-23T19:47:29.581058+0000","last_fullsized":"2023-09-23T19:47:29.581058+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_clean_scrub_stamp":"2023-09-23T19:46:16.677636+0000","objects_scrubbed":0,"log_size":4605,"log_dups_size":0,"ondisk_log_size":4605,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301793280,"num_objects":4605,"num_object_clones":0,"num_object_copies":13815,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4605,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4605,"num_write_kb":294720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":166,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.744525+0000","last_change":"2023-09-23T19:05:52.114108+0000","last_active":"2023-09-23T19:46:54.744525+0000","last_peered":"2023-09-23T19:46:54.744525+0000","last_clean":"2023-09-23T19:46:54.744525+0000","last_became_active":"2023-09-23T19:05:52.113609+0000","last_became_peered":"2023-09-23T19:05:52.113609+0000","last_unstale":"2023-09-23T19:46:54.744525+0000","last_undegraded":"2023-09-23T19:46:54.744525+0000","last_fullsized":"2023-09-23T19:46:54.744525+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":179,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.744190+0000","last_change":"2023-09-23T19:22:42.156825+0000","last_active":"2023-09-23T19:46:54.744190+0000","last_peered":"2023-09-23T19:46:54.744190+0000","last_clean":"2023-09-23T19:46:54.744190+0000","last_became_active":"2023-09-23T19:22:42.156474+0000","last_became_peered":"2023-09-23T19:22:42.156474+0000","last_unstale":"2023-09-23T19:46:54.744190+0000","last_undegraded":"2023-09-23T19:46:54.744190+0000","last_fullsized":"2023-09-23T19:46:54.744190+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"214'4507","reported_seq":4523,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T19:47:28.715092+0000","last_change":"2023-09-23T19:46:17.826712+0000","last_active":"2023-09-23T19:47:28.715092+0000","last_peered":"2023-09-23T19:47:28.715092+0000","last_clean":"2023-09-23T19:47:28.715092+0000","last_became_active":"2023-09-23T19:46:17.826319+0000","last_became_peered":"2023-09-23T19:46:17.826319+0000","last_unstale":"2023-09-23T19:47:28.715092+0000","last_undegraded":"2023-09-23T19:47:28.715092+0000","last_fullsized":"2023-09-23T19:47:28.715092+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_clean_scrub_stamp":"2023-09-23T19:46:16.677636+0000","objects_scrubbed":0,"log_size":4507,"log_dups_size":0,"ondisk_log_size":4507,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:22:46.891438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295370752,"num_objects":4507,"num_object_clones":0,"num_object_copies":13521,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4507,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4507,"num_write_kb":288448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":108,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.745467+0000","last_change":"2023-09-23T19:43:38.576462+0000","last_active":"2023-09-23T19:46:54.745467+0000","last_peered":"2023-09-23T19:46:54.745467+0000","last_clean":"2023-09-23T19:46:54.745467+0000","last_became_active":"2023-09-23T19:43:38.576061+0000","last_became_peered":"2023-09-23T19:43:38.576061+0000","last_unstale":"2023-09-23T19:46:54.745467+0000","last_undegraded":"2023-09-23T19:46:54.745467+0000","last_fullsized":"2023-09-23T19:46:54.745467+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":178,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T19:47:13.648791+0000","last_change":"2023-09-23T19:22:42.135572+0000","last_active":"2023-09-23T19:47:13.648791+0000","last_peered":"2023-09-23T19:47:13.648791+0000","last_clean":"2023-09-23T19:47:13.648791+0000","last_became_active":"2023-09-23T19:22:42.135237+0000","last_became_peered":"2023-09-23T19:22:42.135237+0000","last_unstale":"2023-09-23T19:47:13.648791+0000","last_undegraded":"2023-09-23T19:47:13.648791+0000","last_fullsized":"2023-09-23T19:47:13.648791+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:35.838718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"214'4610","reported_seq":4625,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T19:47:29.713861+0000","last_change":"2023-09-23T19:46:17.828985+0000","last_active":"2023-09-23T19:47:29.713861+0000","last_peered":"2023-09-23T19:47:29.713861+0000","last_clean":"2023-09-23T19:47:29.713861+0000","last_became_active":"2023-09-23T19:46:17.827748+0000","last_became_peered":"2023-09-23T19:46:17.827748+0000","last_unstale":"2023-09-23T19:47:29.713861+0000","last_undegraded":"2023-09-23T19:47:29.713861+0000","last_fullsized":"2023-09-23T19:47:29.713861+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_clean_scrub_stamp":"2023-09-23T19:46:16.677636+0000","objects_scrubbed":0,"log_size":4610,"log_dups_size":0,"ondisk_log_size":4610,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302120960,"num_objects":4610,"num_object_clones":0,"num_object_copies":13830,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4610,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4610,"num_write_kb":295040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":47,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.749240+0000","last_change":"2023-09-23T19:42:55.771907+0000","last_active":"2023-09-23T19:46:54.749240+0000","last_peered":"2023-09-23T19:46:54.749240+0000","last_clean":"2023-09-23T19:46:54.749240+0000","last_became_active":"2023-09-23T19:42:55.771503+0000","last_became_peered":"2023-09-23T19:42:55.771503+0000","last_unstale":"2023-09-23T19:46:54.749240+0000","last_undegraded":"2023-09-23T19:46:54.749240+0000","last_fullsized":"2023-09-23T19:46:54.749240+0000","mapping_epoch":188,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":178,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.749947+0000","last_change":"2023-09-23T19:28:07.896803+0000","last_active":"2023-09-23T19:46:54.749947+0000","last_peered":"2023-09-23T19:46:54.749947+0000","last_clean":"2023-09-23T19:46:54.749947+0000","last_became_active":"2023-09-23T19:28:07.896480+0000","last_became_peered":"2023-09-23T19:28:07.896480+0000","last_unstale":"2023-09-23T19:46:54.749947+0000","last_undegraded":"2023-09-23T19:46:54.749947+0000","last_fullsized":"2023-09-23T19:46:54.749947+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"214'4454","reported_seq":4469,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T19:47:29.749834+0000","last_change":"2023-09-23T19:46:17.717309+0000","last_active":"2023-09-23T19:47:29.749834+0000","last_peered":"2023-09-23T19:47:29.749834+0000","last_clean":"2023-09-23T19:47:29.749834+0000","last_became_active":"2023-09-23T19:46:17.716913+0000","last_became_peered":"2023-09-23T19:46:17.716913+0000","last_unstale":"2023-09-23T19:47:29.749834+0000","last_undegraded":"2023-09-23T19:47:29.749834+0000","last_fullsized":"2023-09-23T19:47:29.749834+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_clean_scrub_stamp":"2023-09-23T19:46:16.677636+0000","objects_scrubbed":0,"log_size":4454,"log_dups_size":0,"ondisk_log_size":4454,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":291897344,"num_objects":4454,"num_object_clones":0,"num_object_copies":13362,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4454,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4454,"num_write_kb":285056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":180,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.744093+0000","last_change":"2023-09-23T19:22:42.156582+0000","last_active":"2023-09-23T19:46:54.744093+0000","last_peered":"2023-09-23T19:46:54.744093+0000","last_clean":"2023-09-23T19:46:54.744093+0000","last_became_active":"2023-09-23T19:22:42.156251+0000","last_became_peered":"2023-09-23T19:22:42.156251+0000","last_unstale":"2023-09-23T19:46:54.744093+0000","last_undegraded":"2023-09-23T19:46:54.744093+0000","last_fullsized":"2023-09-23T19:46:54.744093+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":208,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.750006+0000","last_change":"2023-09-23T19:38:43.891593+0000","last_active":"2023-09-23T19:46:54.750006+0000","last_peered":"2023-09-23T19:46:54.750006+0000","last_clean":"2023-09-23T19:46:54.750006+0000","last_became_active":"2023-09-23T19:38:43.891063+0000","last_became_peered":"2023-09-23T19:38:43.891063+0000","last_unstale":"2023-09-23T19:46:54.750006+0000","last_undegraded":"2023-09-23T19:46:54.750006+0000","last_fullsized":"2023-09-23T19:46:54.750006+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"214'4477","reported_seq":4493,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T19:47:25.967326+0000","last_change":"2023-09-23T19:46:17.707948+0000","last_active":"2023-09-23T19:47:25.967326+0000","last_peered":"2023-09-23T19:47:25.967326+0000","last_clean":"2023-09-23T19:47:25.967326+0000","last_became_active":"2023-09-23T19:46:17.707550+0000","last_became_peered":"2023-09-23T19:46:17.707550+0000","last_unstale":"2023-09-23T19:47:25.967326+0000","last_undegraded":"2023-09-23T19:47:25.967326+0000","last_fullsized":"2023-09-23T19:47:25.967326+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_clean_scrub_stamp":"2023-09-23T19:46:16.677636+0000","objects_scrubbed":0,"log_size":4477,"log_dups_size":0,"ondisk_log_size":4477,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:17:23.514645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293404672,"num_objects":4477,"num_object_clones":0,"num_object_copies":13431,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4477,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4477,"num_write_kb":286528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":194,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.748778+0000","last_change":"2023-09-23T19:38:43.892972+0000","last_active":"2023-09-23T19:46:54.748778+0000","last_peered":"2023-09-23T19:46:54.748778+0000","last_clean":"2023-09-23T19:46:54.748778+0000","last_became_active":"2023-09-23T19:38:43.892451+0000","last_became_peered":"2023-09-23T19:38:43.892451+0000","last_unstale":"2023-09-23T19:46:54.748778+0000","last_undegraded":"2023-09-23T19:46:54.748778+0000","last_fullsized":"2023-09-23T19:46:54.748778+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":178,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.750092+0000","last_change":"2023-09-23T19:22:42.149670+0000","last_active":"2023-09-23T19:46:54.750092+0000","last_peered":"2023-09-23T19:46:54.750092+0000","last_clean":"2023-09-23T19:46:54.750092+0000","last_became_active":"2023-09-23T19:22:42.149022+0000","last_became_peered":"2023-09-23T19:22:42.149022+0000","last_unstale":"2023-09-23T19:46:54.750092+0000","last_undegraded":"2023-09-23T19:46:54.750092+0000","last_fullsized":"2023-09-23T19:46:54.750092+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"214'4311","reported_seq":4327,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T19:47:25.983746+0000","last_change":"2023-09-23T19:46:17.707167+0000","last_active":"2023-09-23T19:47:25.983746+0000","last_peered":"2023-09-23T19:47:25.983746+0000","last_clean":"2023-09-23T19:47:25.983746+0000","last_became_active":"2023-09-23T19:46:17.706818+0000","last_became_peered":"2023-09-23T19:46:17.706818+0000","last_unstale":"2023-09-23T19:47:25.983746+0000","last_undegraded":"2023-09-23T19:47:25.983746+0000","last_fullsized":"2023-09-23T19:47:25.983746+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_clean_scrub_stamp":"2023-09-23T19:46:16.677636+0000","objects_scrubbed":0,"log_size":4311,"log_dups_size":0,"ondisk_log_size":4311,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:34:00.432106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":282525696,"num_objects":4311,"num_object_clones":0,"num_object_copies":12933,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4311,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4311,"num_write_kb":275904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":174,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.734712+0000","last_change":"2023-09-23T19:32:49.729244+0000","last_active":"2023-09-23T19:46:54.734712+0000","last_peered":"2023-09-23T19:46:54.734712+0000","last_clean":"2023-09-23T19:46:54.734712+0000","last_became_active":"2023-09-23T19:32:49.728810+0000","last_became_peered":"2023-09-23T19:32:49.728810+0000","last_unstale":"2023-09-23T19:46:54.734712+0000","last_undegraded":"2023-09-23T19:46:54.734712+0000","last_fullsized":"2023-09-23T19:46:54.734712+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:57.540826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":161,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.735641+0000","last_change":"2023-09-23T19:28:07.889095+0000","last_active":"2023-09-23T19:46:54.735641+0000","last_peered":"2023-09-23T19:46:54.735641+0000","last_clean":"2023-09-23T19:46:54.735641+0000","last_became_active":"2023-09-23T19:28:07.888742+0000","last_became_peered":"2023-09-23T19:28:07.888742+0000","last_unstale":"2023-09-23T19:46:54.735641+0000","last_undegraded":"2023-09-23T19:46:54.735641+0000","last_fullsized":"2023-09-23T19:46:54.735641+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:43.929488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.a","version":"214'4486","reported_seq":4502,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T19:47:27.796404+0000","last_change":"2023-09-23T19:46:17.821771+0000","last_active":"2023-09-23T19:47:27.796404+0000","last_peered":"2023-09-23T19:47:27.796404+0000","last_clean":"2023-09-23T19:47:27.796404+0000","last_became_active":"2023-09-23T19:46:17.821143+0000","last_became_peered":"2023-09-23T19:46:17.821143+0000","last_unstale":"2023-09-23T19:47:27.796404+0000","last_undegraded":"2023-09-23T19:47:27.796404+0000","last_fullsized":"2023-09-23T19:47:27.796404+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_clean_scrub_stamp":"2023-09-23T19:46:16.677636+0000","objects_scrubbed":0,"log_size":4486,"log_dups_size":0,"ondisk_log_size":4486,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:18:56.268551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293994496,"num_objects":4486,"num_object_clones":0,"num_object_copies":13458,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4486,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4486,"num_write_kb":287104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":195,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.735659+0000","last_change":"2023-09-23T19:22:42.143289+0000","last_active":"2023-09-23T19:46:54.735659+0000","last_peered":"2023-09-23T19:46:54.735659+0000","last_clean":"2023-09-23T19:46:54.735659+0000","last_became_active":"2023-09-23T19:22:42.142656+0000","last_became_peered":"2023-09-23T19:22:42.142656+0000","last_unstale":"2023-09-23T19:46:54.735659+0000","last_undegraded":"2023-09-23T19:46:54.735659+0000","last_fullsized":"2023-09-23T19:46:54.735659+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:38.467441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":182,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.742941+0000","last_change":"2023-09-23T19:43:38.571832+0000","last_active":"2023-09-23T19:46:54.742941+0000","last_peered":"2023-09-23T19:46:54.742941+0000","last_clean":"2023-09-23T19:46:54.742941+0000","last_became_active":"2023-09-23T19:43:38.571403+0000","last_became_peered":"2023-09-23T19:43:38.571403+0000","last_unstale":"2023-09-23T19:46:54.742941+0000","last_undegraded":"2023-09-23T19:46:54.742941+0000","last_fullsized":"2023-09-23T19:46:54.742941+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:14:50.291822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":181,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:47:02.745633+0000","last_change":"2023-09-23T19:38:43.898519+0000","last_active":"2023-09-23T19:47:02.745633+0000","last_peered":"2023-09-23T19:47:02.745633+0000","last_clean":"2023-09-23T19:47:02.745633+0000","last_became_active":"2023-09-23T19:38:43.896814+0000","last_became_peered":"2023-09-23T19:38:43.896814+0000","last_unstale":"2023-09-23T19:47:02.745633+0000","last_undegraded":"2023-09-23T19:47:02.745633+0000","last_fullsized":"2023-09-23T19:47:02.745633+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:40.465038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.c","version":"214'4486","reported_seq":4502,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T19:47:28.716912+0000","last_change":"2023-09-23T19:46:17.890125+0000","last_active":"2023-09-23T19:47:28.716912+0000","last_peered":"2023-09-23T19:47:28.716912+0000","last_clean":"2023-09-23T19:47:28.716912+0000","last_became_active":"2023-09-23T19:46:17.889428+0000","last_became_peered":"2023-09-23T19:46:17.889428+0000","last_unstale":"2023-09-23T19:47:28.716912+0000","last_undegraded":"2023-09-23T19:47:28.716912+0000","last_fullsized":"2023-09-23T19:47:28.716912+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_clean_scrub_stamp":"2023-09-23T19:46:16.677636+0000","objects_scrubbed":0,"log_size":4486,"log_dups_size":0,"ondisk_log_size":4486,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:12:54.937796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293994496,"num_objects":4486,"num_object_clones":0,"num_object_copies":13458,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4486,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4486,"num_write_kb":287104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":95,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.735078+0000","last_change":"2023-09-23T19:43:38.612700+0000","last_active":"2023-09-23T19:46:54.735078+0000","last_peered":"2023-09-23T19:46:54.735078+0000","last_clean":"2023-09-23T19:46:54.735078+0000","last_became_active":"2023-09-23T19:43:38.612424+0000","last_became_peered":"2023-09-23T19:43:38.612424+0000","last_unstale":"2023-09-23T19:46:54.735078+0000","last_undegraded":"2023-09-23T19:46:54.735078+0000","last_fullsized":"2023-09-23T19:46:54.735078+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:24.833490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":179,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.749390+0000","last_change":"2023-09-23T19:18:09.017525+0000","last_active":"2023-09-23T19:46:54.749390+0000","last_peered":"2023-09-23T19:46:54.749390+0000","last_clean":"2023-09-23T19:46:54.749390+0000","last_became_active":"2023-09-23T19:18:09.017193+0000","last_became_peered":"2023-09-23T19:18:09.017193+0000","last_unstale":"2023-09-23T19:46:54.749390+0000","last_undegraded":"2023-09-23T19:46:54.749390+0000","last_fullsized":"2023-09-23T19:46:54.749390+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":192,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.735580+0000","last_change":"2023-09-23T19:38:43.877088+0000","last_active":"2023-09-23T19:46:54.735580+0000","last_peered":"2023-09-23T19:46:54.735580+0000","last_clean":"2023-09-23T19:46:54.735580+0000","last_became_active":"2023-09-23T19:38:43.876719+0000","last_became_peered":"2023-09-23T19:38:43.876719+0000","last_unstale":"2023-09-23T19:46:54.735580+0000","last_undegraded":"2023-09-23T19:46:54.735580+0000","last_fullsized":"2023-09-23T19:46:54.735580+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:05.423256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.f","version":"214'4699","reported_seq":4714,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T19:47:29.750747+0000","last_change":"2023-09-23T19:46:17.895413+0000","last_active":"2023-09-23T19:47:29.750747+0000","last_peered":"2023-09-23T19:47:29.750747+0000","last_clean":"2023-09-23T19:47:29.750747+0000","last_became_active":"2023-09-23T19:46:17.894888+0000","last_became_peered":"2023-09-23T19:46:17.894888+0000","last_unstale":"2023-09-23T19:47:29.750747+0000","last_undegraded":"2023-09-23T19:47:29.750747+0000","last_fullsized":"2023-09-23T19:47:29.750747+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_clean_scrub_stamp":"2023-09-23T19:46:16.677636+0000","objects_scrubbed":0,"log_size":4699,"log_dups_size":0,"ondisk_log_size":4699,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":307953664,"num_objects":4699,"num_object_clones":0,"num_object_copies":14097,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4699,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4699,"num_write_kb":300736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":161,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:47:02.745614+0000","last_change":"2023-09-23T19:38:43.883663+0000","last_active":"2023-09-23T19:47:02.745614+0000","last_peered":"2023-09-23T19:47:02.745614+0000","last_clean":"2023-09-23T19:47:02.745614+0000","last_became_active":"2023-09-23T19:38:43.882421+0000","last_became_peered":"2023-09-23T19:38:43.882421+0000","last_unstale":"2023-09-23T19:47:02.745614+0000","last_undegraded":"2023-09-23T19:47:02.745614+0000","last_fullsized":"2023-09-23T19:47:02.745614+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:49:01.607659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":178,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.734625+0000","last_change":"2023-09-23T19:38:43.890600+0000","last_active":"2023-09-23T19:46:54.734625+0000","last_peered":"2023-09-23T19:46:54.734625+0000","last_clean":"2023-09-23T19:46:54.734625+0000","last_became_active":"2023-09-23T19:38:43.890253+0000","last_became_peered":"2023-09-23T19:38:43.890253+0000","last_unstale":"2023-09-23T19:46:54.734625+0000","last_undegraded":"2023-09-23T19:46:54.734625+0000","last_fullsized":"2023-09-23T19:46:54.734625+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:46.751120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":190,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.735714+0000","last_change":"2023-09-23T19:22:42.143733+0000","last_active":"2023-09-23T19:46:54.735714+0000","last_peered":"2023-09-23T19:46:54.735714+0000","last_clean":"2023-09-23T19:46:54.735714+0000","last_became_active":"2023-09-23T19:22:42.143077+0000","last_became_peered":"2023-09-23T19:22:42.143077+0000","last_unstale":"2023-09-23T19:46:54.735714+0000","last_undegraded":"2023-09-23T19:46:54.735714+0000","last_fullsized":"2023-09-23T19:46:54.735714+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:47:33.086994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.d","version":"214'4636","reported_seq":4650,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T19:47:29.714576+0000","last_change":"2023-09-23T19:46:17.714589+0000","last_active":"2023-09-23T19:47:29.714576+0000","last_peered":"2023-09-23T19:47:29.714576+0000","last_clean":"2023-09-23T19:47:29.714576+0000","last_became_active":"2023-09-23T19:46:17.714169+0000","last_became_peered":"2023-09-23T19:46:17.714169+0000","last_unstale":"2023-09-23T19:47:29.714576+0000","last_undegraded":"2023-09-23T19:47:29.714576+0000","last_fullsized":"2023-09-23T19:47:29.714576+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_clean_scrub_stamp":"2023-09-23T19:46:16.677636+0000","objects_scrubbed":0,"log_size":4636,"log_dups_size":0,"ondisk_log_size":4636,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303824896,"num_objects":4636,"num_object_clones":0,"num_object_copies":13908,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4636,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4636,"num_write_kb":296704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":171,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.741613+0000","last_change":"2023-09-23T19:43:38.571764+0000","last_active":"2023-09-23T19:46:54.741613+0000","last_peered":"2023-09-23T19:46:54.741613+0000","last_clean":"2023-09-23T19:46:54.741613+0000","last_became_active":"2023-09-23T19:43:38.571417+0000","last_became_peered":"2023-09-23T19:43:38.571417+0000","last_unstale":"2023-09-23T19:46:54.741613+0000","last_undegraded":"2023-09-23T19:46:54.741613+0000","last_fullsized":"2023-09-23T19:46:54.741613+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:22.165556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":416,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.743850+0000","last_change":"2023-09-23T19:18:09.016297+0000","last_active":"2023-09-23T19:46:54.743850+0000","last_peered":"2023-09-23T19:46:54.743850+0000","last_clean":"2023-09-23T19:46:54.743850+0000","last_became_active":"2023-09-23T19:18:09.015436+0000","last_became_peered":"2023-09-23T19:18:09.015436+0000","last_unstale":"2023-09-23T19:46:54.743850+0000","last_undegraded":"2023-09-23T19:46:54.743850+0000","last_fullsized":"2023-09-23T19:46:54.743850+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:59:33.840751+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:59:33.840751+0000","last_clean_scrub_stamp":"2023-09-23T18:59:33.840751+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":91,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.747398+0000","last_change":"2023-09-23T19:43:38.576727+0000","last_active":"2023-09-23T19:46:54.747398+0000","last_peered":"2023-09-23T19:46:54.747398+0000","last_clean":"2023-09-23T19:46:54.747398+0000","last_became_active":"2023-09-23T19:43:38.576349+0000","last_became_peered":"2023-09-23T19:43:38.576349+0000","last_unstale":"2023-09-23T19:46:54.747398+0000","last_undegraded":"2023-09-23T19:46:54.747398+0000","last_fullsized":"2023-09-23T19:46:54.747398+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":164,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.746494+0000","last_change":"2023-09-23T19:05:52.786771+0000","last_active":"2023-09-23T19:46:54.746494+0000","last_peered":"2023-09-23T19:46:54.746494+0000","last_clean":"2023-09-23T19:46:54.746494+0000","last_became_active":"2023-09-23T19:05:52.785797+0000","last_became_peered":"2023-09-23T19:05:52.785797+0000","last_unstale":"2023-09-23T19:46:54.746494+0000","last_undegraded":"2023-09-23T19:46:54.746494+0000","last_fullsized":"2023-09-23T19:46:54.746494+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"214'4516","reported_seq":4532,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T19:47:28.703942+0000","last_change":"2023-09-23T19:46:17.713260+0000","last_active":"2023-09-23T19:47:28.703942+0000","last_peered":"2023-09-23T19:47:28.703942+0000","last_clean":"2023-09-23T19:47:28.703942+0000","last_became_active":"2023-09-23T19:46:17.712923+0000","last_became_peered":"2023-09-23T19:46:17.712923+0000","last_unstale":"2023-09-23T19:47:28.703942+0000","last_undegraded":"2023-09-23T19:47:28.703942+0000","last_fullsized":"2023-09-23T19:47:28.703942+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_clean_scrub_stamp":"2023-09-23T19:46:16.677636+0000","objects_scrubbed":0,"log_size":4516,"log_dups_size":0,"ondisk_log_size":4516,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:25:32.003178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295960576,"num_objects":4516,"num_object_clones":0,"num_object_copies":13548,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4516,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4516,"num_write_kb":289024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":199,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.743741+0000","last_change":"2023-09-23T19:22:42.150636+0000","last_active":"2023-09-23T19:46:54.743741+0000","last_peered":"2023-09-23T19:46:54.743741+0000","last_clean":"2023-09-23T19:46:54.743741+0000","last_became_active":"2023-09-23T19:22:42.150045+0000","last_became_peered":"2023-09-23T19:22:42.150045+0000","last_unstale":"2023-09-23T19:46:54.743741+0000","last_undegraded":"2023-09-23T19:46:54.743741+0000","last_fullsized":"2023-09-23T19:46:54.743741+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":100,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.749044+0000","last_change":"2023-09-23T19:42:55.771906+0000","last_active":"2023-09-23T19:46:54.749044+0000","last_peered":"2023-09-23T19:46:54.749044+0000","last_clean":"2023-09-23T19:46:54.749044+0000","last_became_active":"2023-09-23T19:42:55.771599+0000","last_became_peered":"2023-09-23T19:42:55.771599+0000","last_unstale":"2023-09-23T19:46:54.749044+0000","last_undegraded":"2023-09-23T19:46:54.749044+0000","last_fullsized":"2023-09-23T19:46:54.749044+0000","mapping_epoch":188,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":208,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.745840+0000","last_change":"2023-09-23T19:38:43.876930+0000","last_active":"2023-09-23T19:46:54.745840+0000","last_peered":"2023-09-23T19:46:54.745840+0000","last_clean":"2023-09-23T19:46:54.745840+0000","last_became_active":"2023-09-23T19:38:43.876571+0000","last_became_peered":"2023-09-23T19:38:43.876571+0000","last_unstale":"2023-09-23T19:46:54.745840+0000","last_undegraded":"2023-09-23T19:46:54.745840+0000","last_fullsized":"2023-09-23T19:46:54.745840+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:30.924536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.b","version":"214'4607","reported_seq":4622,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T19:47:27.811940+0000","last_change":"2023-09-23T19:46:17.828761+0000","last_active":"2023-09-23T19:47:27.811940+0000","last_peered":"2023-09-23T19:47:27.811940+0000","last_clean":"2023-09-23T19:47:27.811940+0000","last_became_active":"2023-09-23T19:46:17.828132+0000","last_became_peered":"2023-09-23T19:46:17.828132+0000","last_unstale":"2023-09-23T19:47:27.811940+0000","last_undegraded":"2023-09-23T19:47:27.811940+0000","last_fullsized":"2023-09-23T19:47:27.811940+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_clean_scrub_stamp":"2023-09-23T19:46:16.677636+0000","objects_scrubbed":0,"log_size":4607,"log_dups_size":0,"ondisk_log_size":4607,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:50:22.166366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301924352,"num_objects":4607,"num_object_clones":0,"num_object_copies":13821,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4607,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4607,"num_write_kb":294848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":184,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.744619+0000","last_change":"2023-09-23T19:18:09.015853+0000","last_active":"2023-09-23T19:46:54.744619+0000","last_peered":"2023-09-23T19:46:54.744619+0000","last_clean":"2023-09-23T19:46:54.744619+0000","last_became_active":"2023-09-23T19:18:09.015338+0000","last_became_peered":"2023-09-23T19:18:09.015338+0000","last_unstale":"2023-09-23T19:46:54.744619+0000","last_undegraded":"2023-09-23T19:46:54.744619+0000","last_fullsized":"2023-09-23T19:46:54.744619+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":177,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.735478+0000","last_change":"2023-09-23T19:38:43.881233+0000","last_active":"2023-09-23T19:46:54.735478+0000","last_peered":"2023-09-23T19:46:54.735478+0000","last_clean":"2023-09-23T19:46:54.735478+0000","last_became_active":"2023-09-23T19:38:43.880893+0000","last_became_peered":"2023-09-23T19:38:43.880893+0000","last_unstale":"2023-09-23T19:46:54.735478+0000","last_undegraded":"2023-09-23T19:46:54.735478+0000","last_fullsized":"2023-09-23T19:46:54.735478+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:11.206947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":49,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.744397+0000","last_change":"2023-09-23T19:43:38.578086+0000","last_active":"2023-09-23T19:46:54.744397+0000","last_peered":"2023-09-23T19:46:54.744397+0000","last_clean":"2023-09-23T19:46:54.744397+0000","last_became_active":"2023-09-23T19:43:38.577623+0000","last_became_peered":"2023-09-23T19:43:38.577623+0000","last_unstale":"2023-09-23T19:46:54.744397+0000","last_undegraded":"2023-09-23T19:46:54.744397+0000","last_fullsized":"2023-09-23T19:46:54.744397+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.8","version":"214'4569","reported_seq":4585,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T19:47:27.736933+0000","last_change":"2023-09-23T19:46:17.821909+0000","last_active":"2023-09-23T19:47:27.736933+0000","last_peered":"2023-09-23T19:47:27.736933+0000","last_clean":"2023-09-23T19:47:27.736933+0000","last_became_active":"2023-09-23T19:46:17.821585+0000","last_became_peered":"2023-09-23T19:46:17.821585+0000","last_unstale":"2023-09-23T19:47:27.736933+0000","last_undegraded":"2023-09-23T19:47:27.736933+0000","last_fullsized":"2023-09-23T19:47:27.736933+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_clean_scrub_stamp":"2023-09-23T19:46:16.677636+0000","objects_scrubbed":0,"log_size":4569,"log_dups_size":0,"ondisk_log_size":4569,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:49.735671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299433984,"num_objects":4569,"num_object_clones":0,"num_object_copies":13707,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4569,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4569,"num_write_kb":292416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":118,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.745025+0000","last_change":"2023-09-23T19:32:49.737323+0000","last_active":"2023-09-23T19:46:54.745025+0000","last_peered":"2023-09-23T19:46:54.745025+0000","last_clean":"2023-09-23T19:46:54.745025+0000","last_became_active":"2023-09-23T19:32:49.736937+0000","last_became_peered":"2023-09-23T19:32:49.736937+0000","last_unstale":"2023-09-23T19:46:54.745025+0000","last_undegraded":"2023-09-23T19:46:54.745025+0000","last_fullsized":"2023-09-23T19:46:54.745025+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":147,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.739210+0000","last_change":"2023-09-23T19:22:42.145506+0000","last_active":"2023-09-23T19:46:54.739210+0000","last_peered":"2023-09-23T19:46:54.739210+0000","last_clean":"2023-09-23T19:46:54.739210+0000","last_became_active":"2023-09-23T19:22:42.143991+0000","last_became_peered":"2023-09-23T19:22:42.143991+0000","last_unstale":"2023-09-23T19:46:54.739210+0000","last_undegraded":"2023-09-23T19:46:54.739210+0000","last_fullsized":"2023-09-23T19:46:54.739210+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:56.875175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":155,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.744156+0000","last_change":"2023-09-23T19:32:49.734592+0000","last_active":"2023-09-23T19:46:54.744156+0000","last_peered":"2023-09-23T19:46:54.744156+0000","last_clean":"2023-09-23T19:46:54.744156+0000","last_became_active":"2023-09-23T19:32:49.733905+0000","last_became_peered":"2023-09-23T19:32:49.733905+0000","last_unstale":"2023-09-23T19:46:54.744156+0000","last_undegraded":"2023-09-23T19:46:54.744156+0000","last_fullsized":"2023-09-23T19:46:54.744156+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.9","version":"214'4608","reported_seq":4624,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T19:47:27.816982+0000","last_change":"2023-09-23T19:46:17.826857+0000","last_active":"2023-09-23T19:47:27.816982+0000","last_peered":"2023-09-23T19:47:27.816982+0000","last_clean":"2023-09-23T19:47:27.816982+0000","last_became_active":"2023-09-23T19:46:17.826182+0000","last_became_peered":"2023-09-23T19:46:17.826182+0000","last_unstale":"2023-09-23T19:47:27.816982+0000","last_undegraded":"2023-09-23T19:47:27.816982+0000","last_fullsized":"2023-09-23T19:47:27.816982+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:46:16.677636+0000","last_clean_scrub_stamp":"2023-09-23T19:46:16.677636+0000","objects_scrubbed":0,"log_size":4608,"log_dups_size":0,"ondisk_log_size":4608,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:32:44.646144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301989888,"num_objects":4608,"num_object_clones":0,"num_object_copies":13824,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4608,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4608,"num_write_kb":294912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":199,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.746311+0000","last_change":"2023-09-23T19:38:43.895887+0000","last_active":"2023-09-23T19:46:54.746311+0000","last_peered":"2023-09-23T19:46:54.746311+0000","last_clean":"2023-09-23T19:46:54.746311+0000","last_became_active":"2023-09-23T19:38:43.895486+0000","last_became_peered":"2023-09-23T19:38:43.895486+0000","last_unstale":"2023-09-23T19:46:54.746311+0000","last_undegraded":"2023-09-23T19:46:54.746311+0000","last_fullsized":"2023-09-23T19:46:54.746311+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":181,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.741905+0000","last_change":"2023-09-23T19:22:42.149694+0000","last_active":"2023-09-23T19:46:54.741905+0000","last_peered":"2023-09-23T19:46:54.741905+0000","last_clean":"2023-09-23T19:46:54.741905+0000","last_became_active":"2023-09-23T19:22:42.148194+0000","last_became_peered":"2023-09-23T19:22:42.148194+0000","last_unstale":"2023-09-23T19:46:54.741905+0000","last_undegraded":"2023-09-23T19:46:54.741905+0000","last_fullsized":"2023-09-23T19:46:54.741905+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:47.842383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":163,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.748760+0000","last_change":"2023-09-23T19:05:52.138620+0000","last_active":"2023-09-23T19:46:54.748760+0000","last_peered":"2023-09-23T19:46:54.748760+0000","last_clean":"2023-09-23T19:46:54.748760+0000","last_became_active":"2023-09-23T19:05:52.138243+0000","last_became_peered":"2023-09-23T19:05:52.138243+0000","last_unstale":"2023-09-23T19:46:54.748760+0000","last_undegraded":"2023-09-23T19:46:54.748760+0000","last_fullsized":"2023-09-23T19:46:54.748760+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":164,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.746510+0000","last_change":"2023-09-23T19:18:09.008811+0000","last_active":"2023-09-23T19:46:54.746510+0000","last_peered":"2023-09-23T19:46:54.746510+0000","last_clean":"2023-09-23T19:46:54.746510+0000","last_became_active":"2023-09-23T19:18:09.007255+0000","last_became_peered":"2023-09-23T19:18:09.007255+0000","last_unstale":"2023-09-23T19:46:54.746510+0000","last_undegraded":"2023-09-23T19:46:54.746510+0000","last_fullsized":"2023-09-23T19:46:54.746510+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:25.874038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":168,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.741217+0000","last_change":"2023-09-23T19:38:43.877244+0000","last_active":"2023-09-23T19:46:54.741217+0000","last_peered":"2023-09-23T19:46:54.741217+0000","last_clean":"2023-09-23T19:46:54.741217+0000","last_became_active":"2023-09-23T19:38:43.876896+0000","last_became_peered":"2023-09-23T19:38:43.876896+0000","last_unstale":"2023-09-23T19:46:54.741217+0000","last_undegraded":"2023-09-23T19:46:54.741217+0000","last_fullsized":"2023-09-23T19:46:54.741217+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:22.288276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":43,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.749118+0000","last_change":"2023-09-23T19:43:38.620732+0000","last_active":"2023-09-23T19:46:54.749118+0000","last_peered":"2023-09-23T19:46:54.749118+0000","last_clean":"2023-09-23T19:46:54.749118+0000","last_became_active":"2023-09-23T19:43:38.619424+0000","last_became_peered":"2023-09-23T19:43:38.619424+0000","last_unstale":"2023-09-23T19:46:54.749118+0000","last_undegraded":"2023-09-23T19:46:54.749118+0000","last_fullsized":"2023-09-23T19:46:54.749118+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":217,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.749704+0000","last_change":"2023-09-23T19:38:43.947451+0000","last_active":"2023-09-23T19:46:54.749704+0000","last_peered":"2023-09-23T19:46:54.749704+0000","last_clean":"2023-09-23T19:46:54.749704+0000","last_became_active":"2023-09-23T19:38:43.946420+0000","last_became_peered":"2023-09-23T19:38:43.946420+0000","last_unstale":"2023-09-23T19:46:54.749704+0000","last_undegraded":"2023-09-23T19:46:54.749704+0000","last_fullsized":"2023-09-23T19:46:54.749704+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":43,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.750102+0000","last_change":"2023-09-23T19:43:38.577015+0000","last_active":"2023-09-23T19:46:54.750102+0000","last_peered":"2023-09-23T19:46:54.750102+0000","last_clean":"2023-09-23T19:46:54.750102+0000","last_became_active":"2023-09-23T19:43:38.576358+0000","last_became_peered":"2023-09-23T19:43:38.576358+0000","last_unstale":"2023-09-23T19:46:54.750102+0000","last_undegraded":"2023-09-23T19:46:54.750102+0000","last_fullsized":"2023-09-23T19:46:54.750102+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":193,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.749321+0000","last_change":"2023-09-23T19:28:07.896983+0000","last_active":"2023-09-23T19:46:54.749321+0000","last_peered":"2023-09-23T19:46:54.749321+0000","last_clean":"2023-09-23T19:46:54.749321+0000","last_became_active":"2023-09-23T19:28:07.896577+0000","last_became_peered":"2023-09-23T19:28:07.896577+0000","last_unstale":"2023-09-23T19:46:54.749321+0000","last_undegraded":"2023-09-23T19:46:54.749321+0000","last_fullsized":"2023-09-23T19:46:54.749321+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":123,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.746212+0000","last_change":"2023-09-23T19:38:43.900225+0000","last_active":"2023-09-23T19:46:54.746212+0000","last_peered":"2023-09-23T19:46:54.746212+0000","last_clean":"2023-09-23T19:46:54.746212+0000","last_became_active":"2023-09-23T19:38:43.899907+0000","last_became_peered":"2023-09-23T19:38:43.899907+0000","last_unstale":"2023-09-23T19:46:54.746212+0000","last_undegraded":"2023-09-23T19:46:54.746212+0000","last_fullsized":"2023-09-23T19:46:54.746212+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":177,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.741738+0000","last_change":"2023-09-23T19:38:43.882487+0000","last_active":"2023-09-23T19:46:54.741738+0000","last_peered":"2023-09-23T19:46:54.741738+0000","last_clean":"2023-09-23T19:46:54.741738+0000","last_became_active":"2023-09-23T19:38:43.881931+0000","last_became_peered":"2023-09-23T19:38:43.881931+0000","last_unstale":"2023-09-23T19:46:54.741738+0000","last_undegraded":"2023-09-23T19:46:54.741738+0000","last_fullsized":"2023-09-23T19:46:54.741738+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:02:36.906535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":197,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T19:47:13.645246+0000","last_change":"2023-09-23T19:38:43.893983+0000","last_active":"2023-09-23T19:47:13.645246+0000","last_peered":"2023-09-23T19:47:13.645246+0000","last_clean":"2023-09-23T19:47:13.645246+0000","last_became_active":"2023-09-23T19:38:43.893527+0000","last_became_peered":"2023-09-23T19:38:43.893527+0000","last_unstale":"2023-09-23T19:47:13.645246+0000","last_undegraded":"2023-09-23T19:47:13.645246+0000","last_fullsized":"2023-09-23T19:47:13.645246+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:12.156531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":172,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T19:46:54.748689+0000","last_change":"2023-09-23T19:06:36.673002+0000","last_active":"2023-09-23T19:46:54.748689+0000","last_peered":"2023-09-23T19:46:54.748689+0000","last_clean":"2023-09-23T19:46:54.748689+0000","last_became_active":"2023-09-23T19:06:36.671911+0000","last_became_peered":"2023-09-23T19:06:36.671911+0000","last_unstale":"2023-09-23T19:46:54.748689+0000","last_undegraded":"2023-09-23T19:46:54.748689+0000","last_fullsized":"2023-09-23T19:46:54.748689+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":4774297600,"num_objects":72850,"num_object_clones":0,"num_object_copies":218550,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":72850,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":72850,"num_write_kb":4662400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14264762368,"data_stored":14264762368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72850,"ondisk_log_size":72850,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430744,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2952076,"kb_used_data":2060292,"kb_used_omap":0,"kb_used_meta":891776,"kb_avail":90793076,"statfs":{"total":95995035648,"available":92972109824,"internally_reserved":0,"allocated":2109739008,"data_stored":2109221334,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":913178624},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:46:59 2023","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":1.351,"15min":1.9670000000000001},"min":{"1min":0.30399999999999999,"5min":0.26400000000000001,"15min":0.23100000000000001},"max":{"1min":4.0350000000000001,"5min":6.8940000000000001,"15min":92.944999999999993},"last":3.8100000000000001},{"interface":"front","average":{"1min":0.95499999999999996,"5min":1.7050000000000001,"15min":2.0569999999999999},"min":{"1min":0.34699999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":3.476,"5min":10.542,"15min":92.763999999999996},"last":2.9510000000000001}]},{"osd":1,"last update":"Sat Sep 23 19:47:24 2023","interfaces":[{"interface":"back","average":{"1min":1.0820000000000001,"5min":2.6280000000000001,"15min":2.3540000000000001},"min":{"1min":0.38500000000000001,"5min":0.32000000000000001,"15min":0.28799999999999998},"max":{"1min":4.0460000000000003,"5min":30.780999999999999,"15min":93.757999999999996},"last":3.8260000000000001},{"interface":"front","average":{"1min":0.96299999999999997,"5min":2.6139999999999999,"15min":2.3540000000000001},"min":{"1min":0.314,"5min":0.314,"15min":0.29999999999999999},"max":{"1min":4.0140000000000002,"5min":30.827999999999999,"15min":93.835999999999999},"last":3.7759999999999998}]},{"osd":2,"last update":"Sat Sep 23 19:46:59 2023","interfaces":[{"interface":"back","average":{"1min":0.83699999999999997,"5min":2.0350000000000001,"15min":1.968},"min":{"1min":0.308,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":1.2869999999999999,"5min":21.163,"15min":21.163},"last":0.311},{"interface":"front","average":{"1min":0.96899999999999997,"5min":2.097,"15min":1.5549999999999999},"min":{"1min":0.42299999999999999,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":3.625,"5min":21.233000000000001,"15min":21.233000000000001},"last":0.48399999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:46:41 2023","interfaces":[{"interface":"back","average":{"1min":1.3600000000000001,"5min":1.806,"15min":1.6679999999999999},"min":{"1min":0.44800000000000001,"5min":0.312,"15min":0.312},"max":{"1min":4.7770000000000001,"5min":8.843,"15min":19.637},"last":3.1429999999999998},{"interface":"front","average":{"1min":1.157,"5min":1.6970000000000001,"15min":1.633},"min":{"1min":0.55100000000000005,"5min":0.41199999999999998,"15min":0.33900000000000002},"max":{"1min":3.6400000000000001,"5min":7.4279999999999999,"15min":20.204000000000001},"last":3.1309999999999998}]},{"osd":4,"last update":"Sat Sep 23 19:47:24 2023","interfaces":[{"interface":"back","average":{"1min":0.73099999999999998,"5min":1.268,"15min":1.3009999999999999},"min":{"1min":0.34200000000000003,"5min":0.30599999999999999,"15min":0.252},"max":{"1min":2.4239999999999999,"5min":8.8010000000000002,"15min":9.7699999999999996},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.72899999999999998,"5min":1.3049999999999999,"15min":1.379},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.27400000000000002},"max":{"1min":1.806,"5min":8.0579999999999998,"15min":13.385},"last":0.34799999999999998}]},{"osd":5,"last update":"Sat Sep 23 19:47:24 2023","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":1.3169999999999999,"15min":1.599},"min":{"1min":0.44600000000000001,"5min":0.34200000000000003,"15min":0.28499999999999998},"max":{"1min":2.4900000000000002,"5min":10.233000000000001,"15min":13.342000000000001},"last":0.437},{"interface":"front","average":{"1min":0.81499999999999995,"5min":1.3520000000000001,"15min":1.6060000000000001},"min":{"1min":0.39800000000000002,"5min":0.34200000000000003,"15min":0.31},"max":{"1min":1.8720000000000001,"5min":8.9100000000000001,"15min":9.8979999999999997},"last":0.44900000000000001}]},{"osd":6,"last update":"Sat Sep 23 19:47:24 2023","interfaces":[{"interface":"back","average":{"1min":0.81999999999999995,"5min":1.849,"15min":1.72},"min":{"1min":0.42999999999999999,"5min":0.38100000000000001,"15min":0.316},"max":{"1min":1.387,"5min":8.8829999999999991,"15min":22.725000000000001},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.81799999999999995,"5min":1.786,"15min":1.8280000000000001},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.308},"max":{"1min":2.1019999999999999,"5min":10.073,"15min":22.707000000000001},"last":0.56200000000000006}]}]},{"osd":6,"up_from":42,"seq":180388626976,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2958908,"kb_used_data":2063604,"kb_used_omap":0,"kb_used_meta":895296,"kb_avail":90786244,"statfs":{"total":95995035648,"available":92965113856,"internally_reserved":0,"allocated":2113130496,"data_stored":2112628745,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":916783104},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,3,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:47:03 2023","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":1.9259999999999999,"15min":2.149},"min":{"1min":0.34100000000000003,"5min":0.26800000000000002,"15min":0.24199999999999999},"max":{"1min":3.7250000000000001,"5min":13.375,"15min":87.283000000000001},"last":1.379},{"interface":"front","average":{"1min":1.155,"5min":1.7949999999999999,"15min":1.9690000000000001},"min":{"1min":0.32700000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":3.218,"5min":11.589,"15min":86.286000000000001},"last":0.29599999999999999}]},{"osd":1,"last update":"Sat Sep 23 19:46:32 2023","interfaces":[{"interface":"back","average":{"1min":1.7569999999999999,"5min":4.6760000000000002,"15min":2.5920000000000001},"min":{"1min":0.32200000000000001,"5min":0.28399999999999997,"15min":0.253},"max":{"1min":8.6820000000000004,"5min":117.58199999999999,"15min":117.58199999999999},"last":0.49399999999999999},{"interface":"front","average":{"1min":1.044,"5min":4.5819999999999999,"15min":2.5099999999999998},"min":{"1min":0.53100000000000003,"5min":0.29199999999999998,"15min":0.28999999999999998},"max":{"1min":2.0190000000000001,"5min":117.633,"15min":117.633},"last":0.55000000000000004}]},{"osd":2,"last update":"Sat Sep 23 19:46:52 2023","interfaces":[{"interface":"back","average":{"1min":1.3029999999999999,"5min":4.4690000000000003,"15min":3.3250000000000002},"min":{"1min":0.40699999999999997,"5min":0.30399999999999999,"15min":0.30299999999999999},"max":{"1min":3.6040000000000001,"5min":119.121,"15min":119.121},"last":0.87},{"interface":"front","average":{"1min":1.4750000000000001,"5min":4.3949999999999996,"15min":3.319},"min":{"1min":0.32500000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":6.8520000000000003,"5min":118.947,"15min":118.947},"last":0.63300000000000001}]},{"osd":3,"last update":"Sat Sep 23 19:46:49 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":1.9419999999999999,"15min":1.927},"min":{"1min":0.56899999999999995,"5min":0.33500000000000002,"15min":0.30199999999999999},"max":{"1min":3.105,"5min":8.4039999999999999,"15min":60.726999999999997},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.88400000000000001,"5min":1.9319999999999999,"15min":2.0129999999999999},"min":{"1min":0.311,"5min":0.307,"15min":0.28799999999999998},"max":{"1min":1.766,"5min":18.917000000000002,"15min":58.304000000000002},"last":0.53800000000000003}]},{"osd":4,"last update":"Sat Sep 23 19:46:49 2023","interfaces":[{"interface":"back","average":{"1min":0.83199999999999996,"5min":1.4399999999999999,"15min":1.3879999999999999},"min":{"1min":0.371,"5min":0.28999999999999998,"15min":0.28799999999999998},"max":{"1min":1.647,"5min":11.779,"15min":11.779},"last":0.373},{"interface":"front","average":{"1min":0.88400000000000001,"5min":1.712,"15min":1.53},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":1.4550000000000001,"5min":11.401999999999999,"15min":11.401999999999999},"last":0.55700000000000005}]},{"osd":5,"last update":"Sat Sep 23 19:46:49 2023","interfaces":[{"interface":"back","average":{"1min":0.99299999999999999,"5min":1.6830000000000001,"15min":1.589},"min":{"1min":0.42099999999999999,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":1.542,"5min":11.743,"15min":12.827},"last":0.47399999999999998},{"interface":"front","average":{"1min":1.196,"5min":1.6220000000000001,"15min":1.931},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":2.8559999999999999,"5min":8.6950000000000003,"15min":10.433999999999999},"last":0.52600000000000002}]},{"osd":7,"last update":"Sat Sep 23 19:47:12 2023","interfaces":[{"interface":"back","average":{"1min":1.0409999999999999,"5min":1.8480000000000001,"15min":1.5189999999999999},"min":{"1min":0.41399999999999998,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":3.1739999999999999,"5min":11.663,"15min":11.663},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.98699999999999999,"5min":2.367,"15min":1.9219999999999999},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.36199999999999999},"max":{"1min":2.6629999999999998,"5min":10.382,"15min":11.439},"last":0.48299999999999998}]}]},{"osd":1,"up_from":98,"seq":420906795307,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1997928,"kb_used_data":1385952,"kb_used_omap":4,"kb_used_meta":611963,"kb_avail":91747224,"statfs":{"total":95995035648,"available":93949157376,"internally_reserved":0,"allocated":1419214848,"data_stored":1418321356,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":626651022},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:46:31 2023","interfaces":[{"interface":"back","average":{"1min":1.004,"5min":1.8240000000000001,"15min":2.617},"min":{"1min":0.20000000000000001,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":4.7450000000000001,"5min":9.0700000000000003,"15min":65.079999999999998},"last":0.90900000000000003},{"interface":"front","average":{"1min":1.3919999999999999,"5min":1.881,"15min":2.6880000000000002},"min":{"1min":0.38900000000000001,"5min":0.26200000000000001,"15min":0.20300000000000001},"max":{"1min":10.747999999999999,"5min":11.353,"15min":65.730000000000004},"last":1.044}]},{"osd":2,"last update":"Sat Sep 23 19:46:58 2023","interfaces":[{"interface":"back","average":{"1min":1.5489999999999999,"5min":2.4740000000000002,"15min":1.6819999999999999},"min":{"1min":0.32400000000000001,"5min":0.29699999999999999,"15min":0.23200000000000001},"max":{"1min":4.5419999999999998,"5min":11.388,"15min":11.388},"last":1.413},{"interface":"front","average":{"1min":1.101,"5min":1.879,"15min":2.3730000000000002},"min":{"1min":0.371,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":5.6760000000000002,"5min":9.016,"15min":48.546999999999997},"last":3.157}]},{"osd":3,"last update":"Sat Sep 23 19:46:39 2023","interfaces":[{"interface":"back","average":{"1min":1.1240000000000001,"5min":2.0579999999999998,"15min":1.696},"min":{"1min":0.44700000000000001,"5min":0.251,"15min":0.21199999999999999},"max":{"1min":4.7990000000000004,"5min":15.004,"15min":15.004},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.879,"5min":2.1699999999999999,"15min":2.073},"min":{"1min":0.29399999999999998,"5min":0.246,"15min":0.246},"max":{"1min":4.7590000000000003,"5min":18.018000000000001,"15min":48.692},"last":2.7919999999999998}]},{"osd":4,"last update":"Sat Sep 23 19:46:31 2023","interfaces":[{"interface":"back","average":{"1min":1.5720000000000001,"5min":1.6259999999999999,"15min":1.7310000000000001},"min":{"1min":0.505,"5min":0.35999999999999999,"15min":0.34300000000000003},"max":{"1min":10.842000000000001,"5min":11.26,"15min":53.128999999999998},"last":0.38700000000000001},{"interface":"front","average":{"1min":1.119,"5min":1.4550000000000001,"15min":1.7909999999999999},"min":{"1min":0.34599999999999997,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":4.8959999999999999,"5min":7.8280000000000003,"15min":48.637},"last":0.57799999999999996}]},{"osd":5,"last update":"Sat Sep 23 19:46:31 2023","interfaces":[{"interface":"back","average":{"1min":1.1459999999999999,"5min":1.671,"15min":2.0670000000000002},"min":{"1min":0.38100000000000001,"5min":0.34899999999999998,"15min":0.33900000000000002},"max":{"1min":5.1020000000000003,"5min":11.145,"15min":48.738},"last":0.54100000000000004},{"interface":"front","average":{"1min":1.3080000000000001,"5min":1.712,"15min":1.8480000000000001},"min":{"1min":0.47399999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":4.7729999999999997,"5min":13.470000000000001,"15min":48.726999999999997},"last":0.69999999999999996}]},{"osd":6,"last update":"Sat Sep 23 19:46:31 2023","interfaces":[{"interface":"back","average":{"1min":1.464,"5min":2.2370000000000001,"15min":2.101},"min":{"1min":0.54600000000000004,"5min":0.318,"15min":0.318},"max":{"1min":5.6799999999999997,"5min":13.004,"15min":48.762},"last":0.49199999999999999},{"interface":"front","average":{"1min":1.7849999999999999,"5min":2.125,"15min":2.1459999999999999},"min":{"1min":0.71199999999999997,"5min":0.44900000000000001,"15min":0.39100000000000001},"max":{"1min":10.814,"5min":11.676,"15min":53.076999999999998},"last":0.61199999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:46:31 2023","interfaces":[{"interface":"back","average":{"1min":1.728,"5min":1.9239999999999999,"15min":2.056},"min":{"1min":0.53400000000000003,"5min":0.38600000000000001,"15min":0.38},"max":{"1min":10.865,"5min":11.317,"15min":48.677},"last":0.69699999999999995},{"interface":"front","average":{"1min":1.2509999999999999,"5min":1.8140000000000001,"15min":2.0619999999999998},"min":{"1min":0.40799999999999997,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":5.2599999999999998,"5min":11.26,"15min":48.805},"last":2.7250000000000001}]}]},{"osd":0,"up_from":85,"seq":365072220514,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2469548,"kb_used_data":1742116,"kb_used_omap":5,"kb_used_meta":727418,"kb_avail":91275604,"statfs":{"total":95995035648,"available":93466218496,"internally_reserved":0,"allocated":1783926784,"data_stored":1783032890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":744876666},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2,1,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 19:46:25 2023","interfaces":[{"interface":"back","average":{"1min":1.556,"5min":2.593,"15min":1.7190000000000001},"min":{"1min":0.35299999999999998,"5min":0.22500000000000001,"15min":0.20399999999999999},"max":{"1min":6.6440000000000001,"5min":57.234999999999999,"15min":57.234999999999999},"last":2.653},{"interface":"front","average":{"1min":1.319,"5min":2.649,"15min":2.0019999999999998},"min":{"1min":0.22600000000000001,"5min":0.189,"15min":0.189},"max":{"1min":7.9119999999999999,"5min":58.825000000000003,"15min":58.825000000000003},"last":3.9620000000000002}]},{"osd":2,"last update":"Sat Sep 23 19:46:59 2023","interfaces":[{"interface":"back","average":{"1min":1.1399999999999999,"5min":4.7670000000000003,"15min":2.8119999999999998},"min":{"1min":0.33900000000000002,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":3.5790000000000002,"5min":262.59500000000003,"15min":262.59500000000003},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.86499999999999999,"5min":4.7220000000000004,"15min":3.0470000000000002},"min":{"1min":0.34300000000000003,"5min":0.245,"15min":0.245},"max":{"1min":2.202,"5min":263.25700000000001,"15min":263.25700000000001},"last":6.7850000000000001}]},{"osd":3,"last update":"Sat Sep 23 19:46:30 2023","interfaces":[{"interface":"back","average":{"1min":1.4159999999999999,"5min":1.962,"15min":1.927},"min":{"1min":0.36099999999999999,"5min":0.29399999999999998,"15min":0.26400000000000001},"max":{"1min":5.5220000000000002,"5min":8.2590000000000003,"15min":14.212},"last":4.4729999999999999},{"interface":"front","average":{"1min":1.0860000000000001,"5min":1.9530000000000001,"15min":1.893},"min":{"1min":0.28199999999999997,"5min":0.223,"15min":0.223},"max":{"1min":4.1669999999999998,"5min":14.84,"15min":14.84},"last":0.97699999999999998}]},{"osd":4,"last update":"Sat Sep 23 19:47:07 2023","interfaces":[{"interface":"back","average":{"1min":0.94199999999999995,"5min":1.411,"15min":1.639},"min":{"1min":0.39400000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":2.8330000000000002,"5min":7.4290000000000003,"15min":11.792999999999999},"last":4.0579999999999998},{"interface":"front","average":{"1min":1.4059999999999999,"5min":1.486,"15min":1.5940000000000001},"min":{"1min":0.82299999999999995,"5min":0.36499999999999999,"15min":0.315},"max":{"1min":3.8500000000000001,"5min":6.0659999999999998,"15min":13.641999999999999},"last":0.94599999999999995}]},{"osd":5,"last update":"Sat Sep 23 19:47:07 2023","interfaces":[{"interface":"back","average":{"1min":1.351,"5min":1.607,"15min":1.8700000000000001},"min":{"1min":0.56299999999999994,"5min":0.38600000000000001,"15min":0.33700000000000002},"max":{"1min":3.9249999999999998,"5min":6.2430000000000003,"15min":13.154},"last":1.026},{"interface":"front","average":{"1min":1.0089999999999999,"5min":1.581,"15min":2.012},"min":{"1min":0.45800000000000002,"5min":0.40500000000000003,"15min":0.36899999999999999},"max":{"1min":2.7709999999999999,"5min":8.6120000000000001,"15min":13.109999999999999},"last":4.0990000000000002}]},{"osd":6,"last update":"Sat Sep 23 19:47:07 2023","interfaces":[{"interface":"back","average":{"1min":1.167,"5min":1.611,"15min":1.9259999999999999},"min":{"1min":0.42799999999999999,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":2.8050000000000002,"5min":13.651999999999999,"15min":40.935000000000002},"last":4.1289999999999996},{"interface":"front","average":{"1min":1.744,"5min":1.7749999999999999,"15min":1.9350000000000001},"min":{"1min":0.67300000000000004,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":5.5899999999999999,"5min":6.7430000000000003,"15min":40.787999999999997},"last":1.0009999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:47:07 2023","interfaces":[{"interface":"back","average":{"1min":1.145,"5min":1.9179999999999999,"15min":1.7729999999999999},"min":{"1min":0.48399999999999999,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":2.7280000000000002,"5min":13.672000000000001,"15min":13.672000000000001},"last":0.41799999999999998},{"interface":"front","average":{"1min":1.093,"5min":1.6859999999999999,"15min":1.8959999999999999},"min":{"1min":0.59299999999999997,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":2.7309999999999999,"5min":8.3759999999999994,"15min":10.561},"last":4.1609999999999996}]}]},{"osd":2,"up_from":159,"seq":682899800170,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3238256,"kb_used_data":2877736,"kb_used_omap":7,"kb_used_meta":360504,"kb_avail":90506896,"statfs":{"total":95995035648,"available":92679061504,"internally_reserved":0,"allocated":2946801664,"data_stored":2945957698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":369156177},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,4,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:47:07 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":1.714,"15min":1.9450000000000001},"min":{"1min":0.42799999999999999,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":3.4329999999999998,"5min":16.702000000000002,"15min":16.702000000000002},"last":0.17299999999999999},{"interface":"front","average":{"1min":2.0430000000000001,"5min":2.23,"15min":2.0070000000000001},"min":{"1min":0.32600000000000001,"5min":0.254,"15min":0.23599999999999999},"max":{"1min":7.1749999999999998,"5min":9.7370000000000001,"15min":9.7370000000000001},"last":2.7730000000000001}]},{"osd":1,"last update":"Sat Sep 23 19:47:07 2023","interfaces":[{"interface":"back","average":{"1min":1.256,"5min":3.032,"15min":1.641},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":5.734,"5min":58.542000000000002,"15min":58.542000000000002},"last":3.2210000000000001},{"interface":"front","average":{"1min":0.85199999999999998,"5min":2.4430000000000001,"15min":1.696},"min":{"1min":0.375,"5min":0.22,"15min":0.22},"max":{"1min":1.8300000000000001,"5min":57.551000000000002,"15min":57.551000000000002},"last":2.0840000000000001}]},{"osd":3,"last update":"Sat Sep 23 19:47:07 2023","interfaces":[{"interface":"back","average":{"1min":1.5109999999999999,"5min":2.194,"15min":1.9570000000000001},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26200000000000001},"max":{"1min":4.5970000000000004,"5min":17.736999999999998,"15min":17.736999999999998},"last":0.90200000000000002},{"interface":"front","average":{"1min":1.736,"5min":2.113,"15min":2.0699999999999998},"min":{"1min":0.50700000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":8.9770000000000003,"5min":16.742000000000001,"15min":16.742000000000001},"last":4.1100000000000003}]},{"osd":4,"last update":"Sat Sep 23 19:47:07 2023","interfaces":[{"interface":"back","average":{"1min":1.9390000000000001,"5min":1.9890000000000001,"15min":1.7010000000000001},"min":{"1min":0.441,"5min":0.317,"15min":0.317},"max":{"1min":6.9119999999999999,"5min":18.318999999999999,"15min":18.318999999999999},"last":2.8220000000000001},{"interface":"front","average":{"1min":1.3480000000000001,"5min":1.6719999999999999,"15min":1.603},"min":{"1min":0.38700000000000001,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":4.6970000000000001,"5min":7.4299999999999997,"15min":7.4299999999999997},"last":1.0009999999999999}]},{"osd":5,"last update":"Sat Sep 23 19:47:07 2023","interfaces":[{"interface":"back","average":{"1min":1.0169999999999999,"5min":1.6870000000000001,"15min":1.6859999999999999},"min":{"1min":0.56799999999999995,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":1.8700000000000001,"5min":16.577000000000002,"15min":16.577000000000002},"last":0.27600000000000002},{"interface":"front","average":{"1min":2.1699999999999999,"5min":1.9830000000000001,"15min":1.901},"min":{"1min":0.36799999999999999,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":7.1269999999999998,"5min":8.1080000000000005,"15min":8.9930000000000003},"last":2.8679999999999999}]},{"osd":6,"last update":"Sat Sep 23 19:47:07 2023","interfaces":[{"interface":"back","average":{"1min":2.1480000000000001,"5min":2.3279999999999998,"15min":2.0800000000000001},"min":{"1min":0.40400000000000003,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":6.9640000000000004,"5min":9.7059999999999995,"15min":11.851000000000001},"last":6.7599999999999998},{"interface":"front","average":{"1min":1.6890000000000001,"5min":2.3809999999999998,"15min":1.97},"min":{"1min":0.498,"5min":0.39600000000000002,"15min":0.36699999999999999},"max":{"1min":5.7009999999999996,"5min":12.297000000000001,"15min":12.297000000000001},"last":5.8099999999999996}]},{"osd":7,"last update":"Sat Sep 23 19:47:07 2023","interfaces":[{"interface":"back","average":{"1min":1.173,"5min":2.0680000000000001,"15min":1.7270000000000001},"min":{"1min":0.52700000000000002,"5min":0.436,"15min":0.436},"max":{"1min":2.1760000000000002,"5min":16.457999999999998,"15min":16.457999999999998},"last":3.052},{"interface":"front","average":{"1min":1.6200000000000001,"5min":2.226,"15min":1.706},"min":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.41299999999999998},"max":{"1min":5.0819999999999999,"5min":12.362,"15min":12.362},"last":0.96499999999999997}]}]},{"osd":3,"up_from":116,"seq":498216206570,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2780928,"kb_used_data":2265336,"kb_used_omap":2,"kb_used_meta":515581,"kb_avail":90964224,"statfs":{"total":95995035648,"available":93147365376,"internally_reserved":0,"allocated":2319704064,"data_stored":2318903310,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":527954997},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:46:42 2023","interfaces":[{"interface":"back","average":{"1min":1.4790000000000001,"5min":1.8220000000000001,"15min":1.7569999999999999},"min":{"1min":0.42599999999999999,"5min":0.21199999999999999,"15min":0.19900000000000001},"max":{"1min":12.193,"5min":12.193,"15min":13.411},"last":0.495},{"interface":"front","average":{"1min":1.7549999999999999,"5min":1.865,"15min":1.7370000000000001},"min":{"1min":0.45100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":16.713000000000001,"5min":16.713000000000001,"15min":16.713000000000001},"last":0.33800000000000002}]},{"osd":1,"last update":"Sat Sep 23 19:46:42 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":2.7799999999999998,"15min":1.9379999999999999},"min":{"1min":0.503,"5min":0.28299999999999997,"15min":0.22600000000000001},"max":{"1min":2.9950000000000001,"5min":92.171000000000006,"15min":92.171000000000006},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.95399999999999996,"5min":3.073,"15min":2.2229999999999999},"min":{"1min":0.373,"5min":0.29799999999999999,"15min":0.23000000000000001},"max":{"1min":3.258,"5min":87.771000000000001,"15min":87.771000000000001},"last":0.67000000000000004}]},{"osd":2,"last update":"Sat Sep 23 19:46:59 2023","interfaces":[{"interface":"back","average":{"1min":1.171,"5min":1.917,"15min":2.3450000000000002},"min":{"1min":0.495,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":4.1680000000000001,"5min":9.4740000000000002,"15min":10.960000000000001},"last":0.61599999999999999},{"interface":"front","average":{"1min":1.351,"5min":1.889,"15min":1.754},"min":{"1min":0.41299999999999998,"5min":0.252,"15min":0.252},"max":{"1min":5.7789999999999999,"5min":9.0410000000000004,"15min":9.0410000000000004},"last":0.64500000000000002}]},{"osd":4,"last update":"Sat Sep 23 19:46:42 2023","interfaces":[{"interface":"back","average":{"1min":1.212,"5min":1.7150000000000001,"15min":1.714},"min":{"1min":0.62,"5min":0.34799999999999998,"15min":0.32500000000000001},"max":{"1min":3.3199999999999998,"5min":11.666,"15min":11.666},"last":0.71199999999999997},{"interface":"front","average":{"1min":1.1639999999999999,"5min":1.5569999999999999,"15min":1.6359999999999999},"min":{"1min":0.69899999999999995,"5min":0.33000000000000002,"15min":0.32300000000000001},"max":{"1min":3.1259999999999999,"5min":9.391,"15min":10.926},"last":0.56899999999999995}]},{"osd":5,"last update":"Sat Sep 23 19:46:42 2023","interfaces":[{"interface":"back","average":{"1min":1.1699999999999999,"5min":1.528,"15min":1.7789999999999999},"min":{"1min":0.63900000000000001,"5min":0.36599999999999999,"15min":0.35299999999999998},"max":{"1min":3.1560000000000001,"5min":9.4269999999999996,"15min":10.82},"last":0.59299999999999997},{"interface":"front","average":{"1min":1.0269999999999999,"5min":1.909,"15min":1.7549999999999999},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":2.3559999999999999,"5min":13.146000000000001,"15min":13.146000000000001},"last":3.7890000000000001}]},{"osd":6,"last update":"Sat Sep 23 19:46:42 2023","interfaces":[{"interface":"back","average":{"1min":1.21,"5min":2.3730000000000002,"15min":1.992},"min":{"1min":0.61699999999999999,"5min":0.39100000000000001,"15min":0.36299999999999999},"max":{"1min":3.2370000000000001,"5min":15.919,"15min":15.919},"last":2.3100000000000001},{"interface":"front","average":{"1min":1.0880000000000001,"5min":2.2810000000000001,"15min":1.734},"min":{"1min":0.40000000000000002,"5min":0.36399999999999999,"15min":0.315},"max":{"1min":2.3130000000000002,"5min":9.0909999999999993,"15min":9.0909999999999993},"last":1.72}]},{"osd":7,"last update":"Sat Sep 23 19:46:42 2023","interfaces":[{"interface":"back","average":{"1min":1.2709999999999999,"5min":2.488,"15min":1.891},"min":{"1min":0.433,"5min":0.39600000000000002,"15min":0.32400000000000001},"max":{"1min":3.0270000000000001,"5min":14.098000000000001,"15min":14.098000000000001},"last":1.5629999999999999},{"interface":"front","average":{"1min":1.284,"5min":2.2429999999999999,"15min":1.895},"min":{"1min":0.66600000000000004,"5min":0.44,"15min":0.40300000000000002},"max":{"1min":3.2909999999999999,"5min":11.744999999999999,"15min":11.744999999999999},"last":1.746}]}]},{"osd":4,"up_from":30,"seq":128849019441,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1796036,"kb_used_data":883836,"kb_used_omap":0,"kb_used_meta":912192,"kb_avail":91949116,"statfs":{"total":95995035648,"available":94155894784,"internally_reserved":0,"allocated":905048064,"data_stored":904537968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":934084608},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:47:14 2023","interfaces":[{"interface":"back","average":{"1min":8.8580000000000005,"5min":2.6749999999999998,"15min":1.871},"min":{"1min":0.29899999999999999,"5min":0.25700000000000001,"15min":0.23300000000000001},"max":{"1min":150.155,"5min":150.155,"15min":150.155},"last":1.766},{"interface":"front","average":{"1min":8.4979999999999993,"5min":2.448,"15min":1.774},"min":{"1min":0.32100000000000001,"5min":0.29499999999999998,"15min":0.26500000000000001},"max":{"1min":149.47900000000001,"5min":149.47900000000001,"15min":149.47900000000001},"last":0.751}]},{"osd":1,"last update":"Sat Sep 23 19:46:38 2023","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":2.7360000000000002,"15min":1.8580000000000001},"min":{"1min":0.373,"5min":0.27800000000000002,"15min":0.27700000000000002},"max":{"1min":5.1630000000000003,"5min":101.786,"15min":101.786},"last":0.40200000000000002},{"interface":"front","average":{"1min":1.23,"5min":3.1269999999999998,"15min":2.0670000000000002},"min":{"1min":0.375,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":5.2009999999999996,"5min":101.726,"15min":101.726},"last":0.41799999999999998}]},{"osd":2,"last update":"Sat Sep 23 19:46:55 2023","interfaces":[{"interface":"back","average":{"1min":1.107,"5min":2.573,"15min":2.9279999999999999},"min":{"1min":0.34300000000000003,"5min":0.313,"15min":0.313},"max":{"1min":2.6880000000000002,"5min":101.92700000000001,"15min":101.92700000000001},"last":0.45600000000000002},{"interface":"front","average":{"1min":1.327,"5min":2.4380000000000002,"15min":2.0800000000000001},"min":{"1min":0.53400000000000003,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":5.2039999999999997,"5min":103.88200000000001,"15min":103.88200000000001},"last":1.4950000000000001}]},{"osd":3,"last update":"Sat Sep 23 19:46:50 2023","interfaces":[{"interface":"back","average":{"1min":0.95999999999999996,"5min":2.1930000000000001,"15min":2.097},"min":{"1min":0.44500000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.53,"5min":27.789000000000001,"15min":49.515000000000001},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.999,"5min":1.839,"15min":2},"min":{"1min":0.40699999999999997,"5min":0.34200000000000003,"15min":0.32100000000000001},"max":{"1min":1.875,"5min":27.808,"15min":51.518999999999998},"last":0.53700000000000003}]},{"osd":5,"last update":"Sat Sep 23 19:47:14 2023","interfaces":[{"interface":"back","average":{"1min":0.89500000000000002,"5min":1.4139999999999999,"15min":1.6459999999999999},"min":{"1min":0.41999999999999998,"5min":0.376,"15min":0.28699999999999998},"max":{"1min":3.3399999999999999,"5min":22.442,"15min":22.442},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.94499999999999995,"5min":1.2090000000000001,"15min":1.536},"min":{"1min":0.48799999999999999,"5min":0.35499999999999998,"15min":0.28899999999999998},"max":{"1min":2.649,"5min":7.6900000000000004,"15min":10.362},"last":0.58199999999999996}]},{"osd":6,"last update":"Sat Sep 23 19:46:38 2023","interfaces":[{"interface":"back","average":{"1min":1.1060000000000001,"5min":1.5620000000000001,"15min":1.6910000000000001},"min":{"1min":0.56899999999999995,"5min":0.372,"15min":0.33900000000000002},"max":{"1min":2.0369999999999999,"5min":22.544,"15min":22.544},"last":0.85499999999999998},{"interface":"front","average":{"1min":1.2370000000000001,"5min":1.1899999999999999,"15min":1.454},"min":{"1min":0.48999999999999999,"5min":0.36499999999999999,"15min":0.34000000000000002},"max":{"1min":3.992,"5min":7.4880000000000004,"15min":10.952},"last":0.84499999999999997}]},{"osd":7,"last update":"Sat Sep 23 19:46:39 2023","interfaces":[{"interface":"back","average":{"1min":18.457000000000001,"5min":4.9050000000000002,"15min":2.5960000000000001},"min":{"1min":0.49399999999999999,"5min":0.36199999999999999,"15min":0.30299999999999999},"max":{"1min":294.76499999999999,"5min":294.76499999999999,"15min":294.76499999999999},"last":0.63600000000000001},{"interface":"front","average":{"1min":18.574000000000002,"5min":5.1849999999999996,"15min":2.8780000000000001},"min":{"1min":0.68899999999999995,"5min":0.39700000000000002,"15min":0.34499999999999997},"max":{"1min":294.47699999999998,"5min":294.47699999999998,"15min":294.47699999999998},"last":0.55600000000000005}]}]},{"osd":5,"up_from":36,"seq":154618823208,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1489196,"kb_used_data":586404,"kb_used_omap":0,"kb_used_meta":902784,"kb_avail":92255956,"statfs":{"total":95995035648,"available":94470098944,"internally_reserved":0,"allocated":600477696,"data_stored":599987290,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":924450816},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:47:12 2023","interfaces":[{"interface":"back","average":{"1min":0.95099999999999996,"5min":1.343,"15min":1.6779999999999999},"min":{"1min":0.35199999999999998,"5min":0.253,"15min":0.253},"max":{"1min":4.1070000000000002,"5min":5.476,"15min":43.094000000000001},"last":1.4450000000000001},{"interface":"front","average":{"1min":0.98699999999999999,"5min":1.464,"15min":2.1720000000000002},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":3.8380000000000001,"5min":8.2899999999999991,"15min":43.802999999999997},"last":0.93000000000000005}]},{"osd":1,"last update":"Sat Sep 23 19:46:29 2023","interfaces":[{"interface":"back","average":{"1min":1.0920000000000001,"5min":1.5109999999999999,"15min":1.548},"min":{"1min":0.28599999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":4.2999999999999998,"5min":11.266999999999999,"15min":32.253999999999998},"last":0.59499999999999997},{"interface":"front","average":{"1min":1.7030000000000001,"5min":1.8129999999999999,"15min":2.008},"min":{"1min":0.378,"5min":0.28100000000000003,"15min":0.27900000000000003},"max":{"1min":7.3949999999999996,"5min":10.891999999999999,"15min":32.277999999999999},"last":0.47199999999999998}]},{"osd":2,"last update":"Sat Sep 23 19:46:54 2023","interfaces":[{"interface":"back","average":{"1min":1.246,"5min":2.7850000000000001,"15min":2.7970000000000002},"min":{"1min":0.46000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":3.2290000000000001,"5min":49.011000000000003,"15min":49.011000000000003},"last":0.58099999999999996},{"interface":"front","average":{"1min":1.4079999999999999,"5min":2.5800000000000001,"15min":2.181},"min":{"1min":0.39600000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":4.5389999999999997,"5min":48.914000000000001,"15min":48.914000000000001},"last":0.76800000000000002}]},{"osd":3,"last update":"Sat Sep 23 19:46:29 2023","interfaces":[{"interface":"back","average":{"1min":1.113,"5min":1.3380000000000001,"15min":1.6279999999999999},"min":{"1min":0.34799999999999998,"5min":0.33700000000000002,"15min":0.311},"max":{"1min":2.7490000000000001,"5min":7.29,"15min":10.885999999999999},"last":1.321},{"interface":"front","average":{"1min":1.115,"5min":1.613,"15min":1.595},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.33800000000000002},"max":{"1min":4.4809999999999999,"5min":7.0129999999999999,"15min":8.5879999999999992},"last":2.4100000000000001}]},{"osd":4,"last update":"Sat Sep 23 19:47:12 2023","interfaces":[{"interface":"back","average":{"1min":0.86199999999999999,"5min":1.1599999999999999,"15min":1.8560000000000001},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":2.431,"5min":7.0060000000000002,"15min":74.221000000000004},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.71899999999999997,"5min":0.90800000000000003,"15min":1.7470000000000001},"min":{"1min":0.39800000000000002,"5min":0.36899999999999999,"15min":0.30599999999999999},"max":{"1min":1.9430000000000001,"5min":5.3879999999999999,"15min":73.772999999999996},"last":0.51800000000000002}]},{"osd":6,"last update":"Sat Sep 23 19:46:53 2023","interfaces":[{"interface":"back","average":{"1min":1.008,"5min":1.081,"15min":1.4379999999999999},"min":{"1min":0.51300000000000001,"5min":0.35599999999999998,"15min":0.32000000000000001},"max":{"1min":2.161,"5min":5.6360000000000001,"15min":16.222000000000001},"last":0.81999999999999995},{"interface":"front","average":{"1min":1.044,"5min":1.163,"15min":1.659},"min":{"1min":0.443,"5min":0.36099999999999999,"15min":0.309},"max":{"1min":2.5289999999999999,"5min":5.6239999999999997,"15min":14.178000000000001},"last":0.73999999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:47:12 2023","interfaces":[{"interface":"back","average":{"1min":0.80500000000000005,"5min":1.4419999999999999,"15min":1.744},"min":{"1min":0.41599999999999998,"5min":0.40500000000000003,"15min":0.32100000000000001},"max":{"1min":1.825,"5min":13.771000000000001,"15min":14.127000000000001},"last":0.79700000000000004},{"interface":"front","average":{"1min":0.78000000000000003,"5min":1.3799999999999999,"15min":1.6439999999999999},"min":{"1min":0.503,"5min":0.40500000000000003,"15min":0.307},"max":{"1min":1.286,"5min":7.2599999999999998,"15min":13.18},"last":0.72499999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1791754240,"data_stored":1791754240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1430126592,"data_stored":1430126592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2962358272,"data_stored":2962358272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2322792448,"data_stored":2322792448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":911736832,"data_stored":911736832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":604766208,"data_stored":604766208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2130116608,"data_stored":2130116608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2111111168,"data_stored":2111111168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T19:47:35.611 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T19:47:35.611 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T19:47:35.611 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T19:47:35.611 INFO:tasks.thrashosds.thrasher:Setting osd 5 primary_affinity to 0.833357 2023-09-23T19:47:35.611 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 5 0.8333573677874015 2023-09-23T19:47:35.634 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:35.656 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:35.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:35.660 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:47:35.762 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:35.772 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:35 smithi092 bash[58855]: cluster 2023-09-23T19:47:34.339277+0000 mgr.y (mgr.25325) 1206 : cluster 0 pgmap v1188: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 955 op/s 2023-09-23T19:47:35.787 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:35.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:35.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:47:35.892 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:35.913 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:35.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:35.915 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:47:36.016 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:36.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:35 smithi016 bash[77995]: cluster 2023-09-23T19:47:34.339277+0000 mgr.y (mgr.25325) 1206 : cluster 0 pgmap v1188: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 955 op/s 2023-09-23T19:47:36.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:35 smithi016 bash[75004]: cluster 2023-09-23T19:47:34.339277+0000 mgr.y (mgr.25325) 1206 : cluster 0 pgmap v1188: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 955 op/s 2023-09-23T19:47:36.037 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:47:35 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:47:35] "GET /metrics HTTP/1.1" 200 36301 "" "Prometheus/2.33.4" 2023-09-23T19:47:36.041 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:36.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:36.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:47:36.144 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:36.152 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 62 16 79671 79655 80.2849 59.5625 0.00780481 0.0124496 2023-09-23T19:47:36.170 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:36.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:36.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:36.275 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:36.298 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:36.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:36.300 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:36.401 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:36.430 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:36.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:36.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:47:36.529 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:47:36.535 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:36.566 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:36.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:36.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:47:36.669 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:36.694 INFO:teuthology.orchestra.run.smithi092.stdout:ERROR: (22) Invalid argument 2023-09-23T19:47:36.694 INFO:teuthology.orchestra.run.smithi092.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:47:36.694 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:36.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:36.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:47:36.803 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:36.848 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:36.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:36.853 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:47:36.954 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:36.977 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:36.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:36.981 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:47:37.081 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:37.105 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:37.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:37.110 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:47:37.152 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 63 16 80551 80535 79.8835 55 0.0164496 0.0125121 2023-09-23T19:47:37.210 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:37.241 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:37.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:37.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:37.346 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:37.374 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:37.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:37.377 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:37.478 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:37.510 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:37.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:37.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:47:37.614 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:37.648 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:37.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:37.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:47:37.690 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T19:47:37.697 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T19:47:37.756 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:37.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:37 smithi092 bash[58855]: cluster 2023-09-23T19:47:36.340003+0000 mgr.y (mgr.25325) 1207 : cluster 0 pgmap v1189: 89 pgs: 89 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 58 MiB/s wr, 926 op/s 2023-09-23T19:47:37.831 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:37.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:37.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:37.936 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:37.955 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:37 smithi016 bash[77995]: cluster 2023-09-23T19:47:36.340003+0000 mgr.y (mgr.25325) 1207 : cluster 0 pgmap v1189: 89 pgs: 89 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 58 MiB/s wr, 926 op/s 2023-09-23T19:47:37.955 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:37 smithi016 bash[75004]: cluster 2023-09-23T19:47:36.340003+0000 mgr.y (mgr.25325) 1207 : cluster 0 pgmap v1189: 89 pgs: 89 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 58 MiB/s wr, 926 op/s 2023-09-23T19:47:37.978 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:37.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:37.982 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:38.084 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:38.114 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:38.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:38.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:47:38.152 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 64 16 81384 81368 79.4487 52.0625 0.0233092 0.0125819 2023-09-23T19:47:38.224 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:38.249 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:38.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:38.252 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:47:38.353 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:38.393 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:38.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:38.396 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:38.497 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:38.531 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:38.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:38.537 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:47:38.637 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:38.676 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:38.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:38.680 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:38.780 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:38.808 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:38.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:38.814 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:47:38.915 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:38.941 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:38.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:38.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:47:39.045 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:39.072 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:39.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:39.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:47:39.152 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 65 16 82287 82271 79.0947 56.4375 0.0165453 0.0126367 2023-09-23T19:47:39.178 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:39.205 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:39.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:39.209 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:39.310 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:39.341 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:39.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:39.344 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:47:39.445 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:39.478 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:39.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:39.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:47:39.582 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:39.607 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:39.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:39.613 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:47:39.714 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:39.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:39.744 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:39.744 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:39.845 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:39.873 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:39.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:39.877 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:39.977 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:39.987 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:39 smithi092 bash[58855]: cluster 2023-09-23T19:47:38.340528+0000 mgr.y (mgr.25325) 1208 : cluster 0 pgmap v1190: 89 pgs: 89 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 916 op/s 2023-09-23T19:47:40.002 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:40.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:40.005 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:47:40.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:39 smithi016 bash[75004]: cluster 2023-09-23T19:47:38.340528+0000 mgr.y (mgr.25325) 1208 : cluster 0 pgmap v1190: 89 pgs: 89 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 916 op/s 2023-09-23T19:47:40.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:39 smithi016 bash[77995]: cluster 2023-09-23T19:47:38.340528+0000 mgr.y (mgr.25325) 1208 : cluster 0 pgmap v1190: 89 pgs: 89 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 916 op/s 2023-09-23T19:47:40.106 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:40.131 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:40.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:40.136 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:47:40.152 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 66 16 83162 83146 78.7248 54.6875 0.0051003 0.0126964 2023-09-23T19:47:40.237 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:40.261 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:40.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:40.264 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:47:40.365 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:40.388 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:40.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:40.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:40.474 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:47:40.480 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:47:40.491 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:40.517 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:40.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:40.521 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:47:40.622 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:40.651 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:40.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:40.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:40.754 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:40.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:40.792 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:40.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:40.893 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:40.931 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:40.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:40.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:47:41.035 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:41.068 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:41.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:41.071 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:47:41.152 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 67 16 84021 84005 78.3511 53.6875 0.0339459 0.0127565 2023-09-23T19:47:41.172 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:41.197 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:41.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:41.200 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:41.300 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:41.331 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:41.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:41.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:41.437 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:41.467 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:41.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:41.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:41.574 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:41.600 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:41.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:41.605 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:41.705 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:41.716 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:41 smithi016 bash[75004]: cluster 2023-09-23T19:47:40.341258+0000 mgr.y (mgr.25325) 1209 : cluster 0 pgmap v1191: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 898 op/s 2023-09-23T19:47:41.738 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:41.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:41.746 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:41.846 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:41.884 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:41.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:41.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:41.990 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:41.998 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:41 smithi092 bash[58855]: cluster 2023-09-23T19:47:40.341258+0000 mgr.y (mgr.25325) 1209 : cluster 0 pgmap v1191: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 898 op/s 2023-09-23T19:47:42.021 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:42.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:42.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:42.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:41 smithi016 bash[77995]: cluster 2023-09-23T19:47:40.341258+0000 mgr.y (mgr.25325) 1209 : cluster 0 pgmap v1191: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 898 op/s 2023-09-23T19:47:42.125 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:42.152 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 68 16 84907 84891 78.0131 55.375 0.00948691 0.0128126 2023-09-23T19:47:42.153 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:42.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:42.156 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:42.257 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:42.294 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:42.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:42.302 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:42.402 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:42.432 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:42.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:42.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:42.519 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:47:42.519 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:47:42.537 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:42.559 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:42.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:42.562 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:47:42.663 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:42.691 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:42.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:42.695 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:47:42.795 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:42.843 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:42.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:42.848 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:42.949 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:42.978 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:42.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:42.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:47:43.083 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:43.105 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:43.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:43.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:43.153 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 69 16 85818 85802 77.7076 56.9375 0.0145867 0.0128611 2023-09-23T19:47:43.211 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:43.241 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:43.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:43.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:43.345 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:43.379 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:43.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:43.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:47:43.484 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:43.526 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:43.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:43.528 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:43.628 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:43.661 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:43.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:43.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:43.768 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:43.798 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:43.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:43.801 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:47:43.897 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:43 smithi016 bash[75004]: cluster 2023-09-23T19:47:42.341826+0000 mgr.y (mgr.25325) 1210 : cluster 0 pgmap v1192: 89 pgs: 89 active+clean; 5.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 55 MiB/s wr, 887 op/s 2023-09-23T19:47:43.897 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:43 smithi016 bash[75004]: audit 2023-09-23T19:47:43.675911+0000 mon.a (mon.0) 1521 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:47:43.897 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:43 smithi016 bash[75004]: audit 2023-09-23T19:47:43.677553+0000 mon.a (mon.0) 1522 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:47:43.897 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:43 smithi016 bash[77995]: cluster 2023-09-23T19:47:42.341826+0000 mgr.y (mgr.25325) 1210 : cluster 0 pgmap v1192: 89 pgs: 89 active+clean; 5.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 55 MiB/s wr, 887 op/s 2023-09-23T19:47:43.898 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:43 smithi016 bash[77995]: audit 2023-09-23T19:47:43.675911+0000 mon.a (mon.0) 1521 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:47:43.898 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:43 smithi016 bash[77995]: audit 2023-09-23T19:47:43.677553+0000 mon.a (mon.0) 1522 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:47:43.902 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:43.930 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:43.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:43.936 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:44.037 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:44.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:43 smithi092 bash[58855]: cluster 2023-09-23T19:47:42.341826+0000 mgr.y (mgr.25325) 1210 : cluster 0 pgmap v1192: 89 pgs: 89 active+clean; 5.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 55 MiB/s wr, 887 op/s 2023-09-23T19:47:44.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:43 smithi092 bash[58855]: audit 2023-09-23T19:47:43.675911+0000 mon.a (mon.0) 1521 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:47:44.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:43 smithi092 bash[58855]: audit 2023-09-23T19:47:43.677553+0000 mon.a (mon.0) 1522 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:47:44.071 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:44.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:44.076 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:44.153 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 70 16 86706 86690 77.3903 55.5 0.00937502 0.0129158 2023-09-23T19:47:44.177 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:44.207 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:44.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:44.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:44.312 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:44.342 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:44.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:44.345 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:47:44.446 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:44.481 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:44.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:44.485 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:47:44.586 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:44.626 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:44.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:44.631 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:44.731 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:44.759 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:44.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:44.762 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:47:44.863 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:44.896 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:44.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:44.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:47:44.999 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:45.026 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:45.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:45.029 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:47:45.129 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:45.153 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 71 16 87637 87621 77.1198 58.1875 0.0159315 0.0129607 2023-09-23T19:47:45.159 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:45.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:45.163 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:45.264 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:45.294 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:45.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:45.297 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:45.398 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:45.425 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:45.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:45.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:47:45.530 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:45.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:45.676 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:45.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:45.776 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:45.806 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:45.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:45.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:47:45.811 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:45 smithi016 bash[75004]: cluster 2023-09-23T19:47:44.342356+0000 mgr.y (mgr.25325) 1211 : cluster 0 pgmap v1193: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 921 op/s 2023-09-23T19:47:45.812 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:47:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:47:45] "GET /metrics HTTP/1.1" 200 36302 "" "Prometheus/2.33.4" 2023-09-23T19:47:45.812 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:45 smithi016 bash[77995]: cluster 2023-09-23T19:47:44.342356+0000 mgr.y (mgr.25325) 1211 : cluster 0 pgmap v1193: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 921 op/s 2023-09-23T19:47:45.913 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:45.953 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:45.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:45.957 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:46.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:45 smithi092 bash[58855]: cluster 2023-09-23T19:47:44.342356+0000 mgr.y (mgr.25325) 1211 : cluster 0 pgmap v1193: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 921 op/s 2023-09-23T19:47:46.058 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:46.087 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:46.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:46.090 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:47:46.153 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 72 16 88550 88534 76.8411 57.0625 0.0100925 0.0130087 2023-09-23T19:47:46.190 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:46.219 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:46.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:46.225 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:46.325 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:46.350 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:46.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:46.354 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:46.454 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:46.484 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:46.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:46.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:47:46.588 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:46.621 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:46.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:46.626 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:46.727 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:46.760 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:46.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:46.762 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:46.863 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:46.890 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:46.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:46.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:47:46.994 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:47.011 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:47.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:47.013 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:47.114 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:47.149 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:47.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:47.152 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:47.153 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 73 16 89316 89300 76.4443 47.875 0.0207067 0.0130763 2023-09-23T19:47:47.252 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:47.276 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:47.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:47.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:47.379 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:47.403 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:47.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:47.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:47:47.512 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:47.534 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:47.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:47.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:47.639 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:47.662 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:47.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:47.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:47:47.768 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:47.777 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:47 smithi016 bash[75004]: cluster 2023-09-23T19:47:46.345419+0000 mgr.y (mgr.25325) 1212 : cluster 0 pgmap v1194: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 59 MiB/s wr, 946 op/s 2023-09-23T19:47:47.777 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:47 smithi016 bash[77995]: cluster 2023-09-23T19:47:46.345419+0000 mgr.y (mgr.25325) 1212 : cluster 0 pgmap v1194: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 59 MiB/s wr, 946 op/s 2023-09-23T19:47:47.790 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:47.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:47.795 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:47.895 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:47.903 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:47 smithi092 bash[58855]: cluster 2023-09-23T19:47:46.345419+0000 mgr.y (mgr.25325) 1212 : cluster 0 pgmap v1194: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 59 MiB/s wr, 946 op/s 2023-09-23T19:47:47.926 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:47.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:47.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:47:48.031 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:48.061 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:48.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:48.068 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:47:48.153 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 74 16 90214 90198 76.1697 56.125 0.0149949 0.0131233 2023-09-23T19:47:48.169 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:48.199 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:48.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:48.209 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:47:48.310 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:48.339 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:48.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:48.341 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:48.442 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:48.471 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:48.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:48.474 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:48.574 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:48.604 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:48.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:48.606 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:47:48.706 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:48.738 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:48.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:48.748 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:47:48.848 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:48.882 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:48.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:48.885 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:48.986 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:49.014 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:49.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:49.020 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:49.121 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:49.144 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:49.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:49.148 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:49.153 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 75 16 91087 91071 75.8815 54.5625 0.0124839 0.0131727 2023-09-23T19:47:49.249 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:49.274 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:49.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:49.280 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:49.381 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:49.410 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:49.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:49.414 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:49.515 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:49.545 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:49.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:49.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:49.650 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:49.684 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:49.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:49.690 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:49.790 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:49.815 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:49.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:49.820 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:47:49.921 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:49.943 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:49.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:49.947 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:47:50.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:49 smithi016 bash[75004]: cluster 2023-09-23T19:47:48.346022+0000 mgr.y (mgr.25325) 1213 : cluster 0 pgmap v1195: 89 pgs: 89 active+clean; 5.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 52 MiB/s wr, 835 op/s 2023-09-23T19:47:50.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:49 smithi016 bash[77995]: cluster 2023-09-23T19:47:48.346022+0000 mgr.y (mgr.25325) 1213 : cluster 0 pgmap v1195: 89 pgs: 89 active+clean; 5.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 52 MiB/s wr, 835 op/s 2023-09-23T19:47:50.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:49 smithi092 bash[58855]: cluster 2023-09-23T19:47:48.346022+0000 mgr.y (mgr.25325) 1213 : cluster 0 pgmap v1195: 89 pgs: 89 active+clean; 5.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 52 MiB/s wr, 835 op/s 2023-09-23T19:47:50.048 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:50.074 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:50.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:50.081 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:47:50.153 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 76 16 91945 91929 75.5886 53.625 0.0167302 0.0132232 2023-09-23T19:47:50.182 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:50.211 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:50.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:50.214 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:50.315 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:50.350 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:50.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:50.354 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:50.455 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:50.491 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:50.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:50.495 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:50.595 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:50.626 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:50.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:50.629 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:47:50.730 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:50.765 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:50.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:50.771 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:47:50.872 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:50.896 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:50.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:50.902 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:51.003 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:51.029 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:51.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:51.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:47:51.133 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:51.153 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 77 16 92838 92822 75.3317 55.8125 0.0234401 0.0132688 2023-09-23T19:47:51.160 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:51.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:51.162 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:47:51.263 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:51.292 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:51.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:51.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:47:51.397 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:51.420 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:51.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:51.425 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:47:51.525 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:51.551 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:51.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:51.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:51.661 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:51.705 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:51.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:51.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:51.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:51 smithi092 bash[58855]: cluster 2023-09-23T19:47:50.346728+0000 mgr.y (mgr.25325) 1214 : cluster 0 pgmap v1196: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 60 MiB/s wr, 966 op/s 2023-09-23T19:47:51.808 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:51.833 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:51.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:51.836 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:47:51.920 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:51 smithi016 bash[75004]: cluster 2023-09-23T19:47:50.346728+0000 mgr.y (mgr.25325) 1214 : cluster 0 pgmap v1196: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 60 MiB/s wr, 966 op/s 2023-09-23T19:47:51.921 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:51 smithi016 bash[77995]: cluster 2023-09-23T19:47:50.346728+0000 mgr.y (mgr.25325) 1214 : cluster 0 pgmap v1196: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 60 MiB/s wr, 966 op/s 2023-09-23T19:47:51.937 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:51.968 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:51.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:51.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:52.071 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:52.096 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:52.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:52.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:47:52.153 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 78 16 93691 93675 75.0494 53.3125 0.0254411 0.0133183 2023-09-23T19:47:52.199 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:52.227 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:52.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:52.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:52.330 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:52.361 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:52.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:52.366 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:52.467 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:52.493 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:52.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:52.498 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:47:52.599 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:52.627 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:52.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:52.629 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:52.730 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:52.761 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:52.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:52.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:52.866 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:52.899 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:47:52.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:52.902 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:47:53.003 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:53.049 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:53.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:53.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:53.152 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:53.154 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 79 16 94402 94386 74.6618 44.4375 0.00533489 0.0133797 2023-09-23T19:47:53.179 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:53.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:53.187 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:53.288 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:53.311 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:53.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:53.313 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:53.414 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:53.439 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:53.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:53.449 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:53.550 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:53.573 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:53.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:53.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:53.677 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:53.706 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:53.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:53.711 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:53.812 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:53.838 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:53.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:53.840 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:53.941 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:53.950 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:53 smithi092 bash[58855]: cluster 2023-09-23T19:47:52.347244+0000 mgr.y (mgr.25325) 1215 : cluster 0 pgmap v1197: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 48 MiB/s wr, 775 op/s 2023-09-23T19:47:53.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:53.978 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:53.978 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:54.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:53 smithi016 bash[75004]: cluster 2023-09-23T19:47:52.347244+0000 mgr.y (mgr.25325) 1215 : cluster 0 pgmap v1197: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 48 MiB/s wr, 775 op/s 2023-09-23T19:47:54.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:53 smithi016 bash[77995]: cluster 2023-09-23T19:47:52.347244+0000 mgr.y (mgr.25325) 1215 : cluster 0 pgmap v1197: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 48 MiB/s wr, 775 op/s 2023-09-23T19:47:54.079 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:54.118 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:54.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:54.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:54.154 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-09-23T19:47:54.150401+0000 min lat: 0.00140724 max lat: 0.404369 avg lat: 0.013437 2023-09-23T19:47:54.154 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:47:54.154 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 80 16 95243 95227 74.3856 52.5625 0.0233728 0.013437 2023-09-23T19:47:54.223 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:54.250 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:54.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:54.253 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:47:54.354 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:54.387 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:54.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:54.390 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:47:54.491 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:54.519 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:54.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:54.525 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:47:54.626 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:54.658 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:54.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:54.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:47:54.762 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:54.789 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:54.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:54.795 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:47:54.896 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:54.928 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:54.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:54.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:55.031 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:55.063 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:55.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:55.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:55.154 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 81 16 95988 95972 74.042 46.5625 0.0119744 0.0134992 2023-09-23T19:47:55.168 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:55.192 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:55.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:55.197 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:47:55.297 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:55.323 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:55.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:55.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:55.426 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:55.451 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:55.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:55.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:47:55.559 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:55.600 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:55.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:55.605 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:55.706 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:55.733 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:55.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:55.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:47:55.758 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:55 smithi092 bash[58855]: cluster 2023-09-23T19:47:54.347996+0000 mgr.y (mgr.25325) 1216 : cluster 0 pgmap v1198: 89 pgs: 89 active+clean; 5.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 55 MiB/s wr, 886 op/s 2023-09-23T19:47:55.761 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:55 smithi092 bash[58855]: audit 2023-09-23T19:47:55.667135+0000 mon.c (mon.2) 196 : audit 1 from='client.? 172.21.15.16:0/3561819757' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.8333573677874015}]: dispatch 2023-09-23T19:47:55.761 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:55 smithi092 bash[58855]: audit 2023-09-23T19:47:55.667767+0000 mon.a (mon.0) 1523 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.8333573677874015}]: dispatch 2023-09-23T19:47:55.767 INFO:teuthology.orchestra.run.smithi016.stderr:set osd.5 primary-affinity to 0.833357 (8546142) 2023-09-23T19:47:55.841 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:55.869 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:55.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:55.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:47:55.973 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:55.983 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:55 smithi016 bash[75004]: cluster 2023-09-23T19:47:54.347996+0000 mgr.y (mgr.25325) 1216 : cluster 0 pgmap v1198: 89 pgs: 89 active+clean; 5.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 55 MiB/s wr, 886 op/s 2023-09-23T19:47:55.984 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:55 smithi016 bash[75004]: audit 2023-09-23T19:47:55.667135+0000 mon.c (mon.2) 196 : audit 1 from='client.? 172.21.15.16:0/3561819757' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.8333573677874015}]: dispatch 2023-09-23T19:47:55.984 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:55 smithi016 bash[75004]: audit 2023-09-23T19:47:55.667767+0000 mon.a (mon.0) 1523 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.8333573677874015}]: dispatch 2023-09-23T19:47:55.984 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:47:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:47:55] "GET /metrics HTTP/1.1" 200 36302 "" "Prometheus/2.33.4" 2023-09-23T19:47:55.984 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:55 smithi016 bash[77995]: cluster 2023-09-23T19:47:54.347996+0000 mgr.y (mgr.25325) 1216 : cluster 0 pgmap v1198: 89 pgs: 89 active+clean; 5.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 55 MiB/s wr, 886 op/s 2023-09-23T19:47:55.984 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:55 smithi016 bash[77995]: audit 2023-09-23T19:47:55.667135+0000 mon.c (mon.2) 196 : audit 1 from='client.? 172.21.15.16:0/3561819757' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.8333573677874015}]: dispatch 2023-09-23T19:47:55.985 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:55 smithi016 bash[77995]: audit 2023-09-23T19:47:55.667767+0000 mon.a (mon.0) 1523 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.8333573677874015}]: dispatch 2023-09-23T19:47:56.012 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:56.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:56.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:56.120 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:56.152 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:56.154 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 82 16 96706 96690 73.6863 44.875 0.011665 0.0135637 2023-09-23T19:47:56.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:56.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:56.258 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:56.286 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:56.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:56.288 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:47:56.389 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:56.414 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:56.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:56.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:56.521 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:56.545 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:56.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:56.549 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:47:56.650 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:56.681 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:56.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:56.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:56.784 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is set 2023-09-23T19:47:56.788 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:56.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:56 smithi092 bash[58855]: audit 2023-09-23T19:47:55.748422+0000 mon.a (mon.0) 1524 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.8333573677874015}]': finished 2023-09-23T19:47:56.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:56 smithi092 bash[58855]: cluster 2023-09-23T19:47:55.748528+0000 mon.a (mon.0) 1525 : cluster 0 osdmap e216: 8 total, 8 up, 8 in 2023-09-23T19:47:56.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:56 smithi092 bash[58855]: cluster 2023-09-23T19:47:55.758479+0000 osd.5 (osd.5) 141 : cluster 3 failed to encode map e216 with expected crc 2023-09-23T19:47:56.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:56 smithi092 bash[58855]: cluster 2023-09-23T19:47:55.760081+0000 osd.4 (osd.4) 158 : cluster 3 failed to encode map e216 with expected crc 2023-09-23T19:47:56.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:56 smithi092 bash[58855]: cluster 2023-09-23T19:47:55.761507+0000 osd.6 (osd.6) 146 : cluster 3 failed to encode map e216 with expected crc 2023-09-23T19:47:56.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:56 smithi092 bash[58855]: cluster 2023-09-23T19:47:55.763010+0000 osd.7 (osd.7) 135 : cluster 3 failed to encode map e216 with expected crc 2023-09-23T19:47:56.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:56 smithi092 bash[58855]: cluster 2023-09-23T19:47:55.764601+0000 osd.5 (osd.5) 142 : cluster 3 failed to encode map e216 with expected crc 2023-09-23T19:47:56.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:56 smithi092 bash[58855]: audit 2023-09-23T19:47:56.633470+0000 mon.a (mon.0) 1526 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:47:56.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:56 smithi092 bash[58855]: audit 2023-09-23T19:47:56.639044+0000 mon.b (mon.1) 53 : audit 1 from='client.? 172.21.15.16:0/3713316395' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:47:56.820 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:56.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:56.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:47:56.924 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:56.933 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:56 smithi016 bash[75004]: audit 2023-09-23T19:47:55.748422+0000 mon.a (mon.0) 1524 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.8333573677874015}]': finished 2023-09-23T19:47:56.933 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:56 smithi016 bash[75004]: cluster 2023-09-23T19:47:55.748528+0000 mon.a (mon.0) 1525 : cluster 0 osdmap e216: 8 total, 8 up, 8 in 2023-09-23T19:47:56.934 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:56 smithi016 bash[75004]: cluster 2023-09-23T19:47:55.758479+0000 osd.5 (osd.5) 141 : cluster 3 failed to encode map e216 with expected crc 2023-09-23T19:47:56.934 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:56 smithi016 bash[75004]: cluster 2023-09-23T19:47:55.760081+0000 osd.4 (osd.4) 158 : cluster 3 failed to encode map e216 with expected crc 2023-09-23T19:47:56.934 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:56 smithi016 bash[75004]: cluster 2023-09-23T19:47:55.761507+0000 osd.6 (osd.6) 146 : cluster 3 failed to encode map e216 with expected crc 2023-09-23T19:47:56.934 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:56 smithi016 bash[75004]: cluster 2023-09-23T19:47:55.763010+0000 osd.7 (osd.7) 135 : cluster 3 failed to encode map e216 with expected crc 2023-09-23T19:47:56.934 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:56 smithi016 bash[75004]: cluster 2023-09-23T19:47:55.764601+0000 osd.5 (osd.5) 142 : cluster 3 failed to encode map e216 with expected crc 2023-09-23T19:47:56.934 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:56 smithi016 bash[75004]: audit 2023-09-23T19:47:56.633470+0000 mon.a (mon.0) 1526 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:47:56.934 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:56 smithi016 bash[75004]: audit 2023-09-23T19:47:56.639044+0000 mon.b (mon.1) 53 : audit 1 from='client.? 172.21.15.16:0/3713316395' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:47:56.934 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:56 smithi016 bash[77995]: audit 2023-09-23T19:47:55.748422+0000 mon.a (mon.0) 1524 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.8333573677874015}]': finished 2023-09-23T19:47:56.934 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:56 smithi016 bash[77995]: cluster 2023-09-23T19:47:55.748528+0000 mon.a (mon.0) 1525 : cluster 0 osdmap e216: 8 total, 8 up, 8 in 2023-09-23T19:47:56.935 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:56 smithi016 bash[77995]: cluster 2023-09-23T19:47:55.758479+0000 osd.5 (osd.5) 141 : cluster 3 failed to encode map e216 with expected crc 2023-09-23T19:47:56.935 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:56 smithi016 bash[77995]: cluster 2023-09-23T19:47:55.760081+0000 osd.4 (osd.4) 158 : cluster 3 failed to encode map e216 with expected crc 2023-09-23T19:47:56.935 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:56 smithi016 bash[77995]: cluster 2023-09-23T19:47:55.761507+0000 osd.6 (osd.6) 146 : cluster 3 failed to encode map e216 with expected crc 2023-09-23T19:47:56.935 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:56 smithi016 bash[77995]: cluster 2023-09-23T19:47:55.763010+0000 osd.7 (osd.7) 135 : cluster 3 failed to encode map e216 with expected crc 2023-09-23T19:47:56.935 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:56 smithi016 bash[77995]: cluster 2023-09-23T19:47:55.764601+0000 osd.5 (osd.5) 142 : cluster 3 failed to encode map e216 with expected crc 2023-09-23T19:47:56.935 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:56 smithi016 bash[77995]: audit 2023-09-23T19:47:56.633470+0000 mon.a (mon.0) 1526 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:47:56.935 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:56 smithi016 bash[77995]: audit 2023-09-23T19:47:56.639044+0000 mon.b (mon.1) 53 : audit 1 from='client.? 172.21.15.16:0/3713316395' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:47:56.950 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:56.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:56.954 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:57.055 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:57.090 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:57.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:57.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:47:57.154 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 83 16 97440 97424 73.3512 45.875 0.0180658 0.0136269 2023-09-23T19:47:57.195 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:57.219 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:57.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:57.224 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:57.325 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:57.355 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:57.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:57.358 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:47:57.459 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:57.485 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:57.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:57.488 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:57.588 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:57.633 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:57.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:57.636 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:47:57.737 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:57.764 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:57.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:57.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:57.824 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:47:57.862 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:47:57.867 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:57.900 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:57.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:57.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:57.912 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:47:57.965 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:47:58.003 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:47:58.006 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:58.029 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:58.034 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:57 smithi092 bash[58855]: cluster 2023-09-23T19:47:56.348562+0000 mgr.y (mgr.25325) 1217 : cluster 0 pgmap v1200: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 53 MiB/s wr, 849 op/s 2023-09-23T19:47:58.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:57 smithi092 bash[58855]: cluster 2023-09-23T19:47:56.753393+0000 mon.a (mon.0) 1527 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:47:58.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:57 smithi092 bash[58855]: audit 2023-09-23T19:47:56.772584+0000 mon.a (mon.0) 1528 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:47:58.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:57 smithi092 bash[58855]: cluster 2023-09-23T19:47:56.772817+0000 mon.a (mon.0) 1529 : cluster 0 osdmap e217: 8 total, 8 up, 8 in 2023-09-23T19:47:58.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:57 smithi092 bash[58855]: cluster 2023-09-23T19:47:56.774769+0000 osd.5 (osd.5) 143 : cluster 3 failed to encode map e217 with expected crc 2023-09-23T19:47:58.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:57 smithi092 bash[58855]: cluster 2023-09-23T19:47:56.776007+0000 osd.6 (osd.6) 147 : cluster 3 failed to encode map e217 with expected crc 2023-09-23T19:47:58.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:57 smithi092 bash[58855]: cluster 2023-09-23T19:47:56.776037+0000 osd.7 (osd.7) 136 : cluster 3 failed to encode map e217 with expected crc 2023-09-23T19:47:58.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:57 smithi092 bash[58855]: cluster 2023-09-23T19:47:56.777002+0000 osd.5 (osd.5) 144 : cluster 3 failed to encode map e217 with expected crc 2023-09-23T19:47:58.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:57 smithi092 bash[58855]: cluster 2023-09-23T19:47:56.778866+0000 osd.5 (osd.5) 145 : cluster 3 failed to encode map e217 with expected crc 2023-09-23T19:47:58.035 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:57 smithi092 bash[58855]: cluster 2023-09-23T19:47:56.779988+0000 osd.6 (osd.6) 148 : cluster 3 failed to encode map e217 with expected crc 2023-09-23T19:47:58.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:57 smithi092 bash[58855]: cluster 2023-09-23T19:47:56.780158+0000 osd.7 (osd.7) 137 : cluster 3 failed to encode map e217 with expected crc 2023-09-23T19:47:58.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:57 smithi092 bash[58855]: cluster 2023-09-23T19:47:56.782970+0000 osd.4 (osd.4) 159 : cluster 3 failed to encode map e217 with expected crc 2023-09-23T19:47:58.036 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:57 smithi092 bash[58855]: audit 2023-09-23T19:47:57.752066+0000 mon.a (mon.0) 1530 : audit 0 from='client.? 172.21.15.16:0/801445213' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:47:58.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:58.037 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:58.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:57 smithi016 bash[75004]: cluster 2023-09-23T19:47:56.348562+0000 mgr.y (mgr.25325) 1217 : cluster 0 pgmap v1200: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 53 MiB/s wr, 849 op/s 2023-09-23T19:47:58.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:57 smithi016 bash[75004]: cluster 2023-09-23T19:47:56.753393+0000 mon.a (mon.0) 1527 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:47:58.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:57 smithi016 bash[75004]: audit 2023-09-23T19:47:56.772584+0000 mon.a (mon.0) 1528 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:47:58.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:57 smithi016 bash[75004]: cluster 2023-09-23T19:47:56.772817+0000 mon.a (mon.0) 1529 : cluster 0 osdmap e217: 8 total, 8 up, 8 in 2023-09-23T19:47:58.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:57 smithi016 bash[75004]: cluster 2023-09-23T19:47:56.774769+0000 osd.5 (osd.5) 143 : cluster 3 failed to encode map e217 with expected crc 2023-09-23T19:47:58.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:57 smithi016 bash[75004]: cluster 2023-09-23T19:47:56.776007+0000 osd.6 (osd.6) 147 : cluster 3 failed to encode map e217 with expected crc 2023-09-23T19:47:58.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:57 smithi016 bash[75004]: cluster 2023-09-23T19:47:56.776037+0000 osd.7 (osd.7) 136 : cluster 3 failed to encode map e217 with expected crc 2023-09-23T19:47:58.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:57 smithi016 bash[75004]: cluster 2023-09-23T19:47:56.777002+0000 osd.5 (osd.5) 144 : cluster 3 failed to encode map e217 with expected crc 2023-09-23T19:47:58.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:57 smithi016 bash[75004]: cluster 2023-09-23T19:47:56.778866+0000 osd.5 (osd.5) 145 : cluster 3 failed to encode map e217 with expected crc 2023-09-23T19:47:58.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:57 smithi016 bash[75004]: cluster 2023-09-23T19:47:56.779988+0000 osd.6 (osd.6) 148 : cluster 3 failed to encode map e217 with expected crc 2023-09-23T19:47:58.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:57 smithi016 bash[75004]: cluster 2023-09-23T19:47:56.780158+0000 osd.7 (osd.7) 137 : cluster 3 failed to encode map e217 with expected crc 2023-09-23T19:47:58.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:57 smithi016 bash[75004]: cluster 2023-09-23T19:47:56.782970+0000 osd.4 (osd.4) 159 : cluster 3 failed to encode map e217 with expected crc 2023-09-23T19:47:58.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:57 smithi016 bash[75004]: audit 2023-09-23T19:47:57.752066+0000 mon.a (mon.0) 1530 : audit 0 from='client.? 172.21.15.16:0/801445213' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:47:58.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:57 smithi016 bash[77995]: cluster 2023-09-23T19:47:56.348562+0000 mgr.y (mgr.25325) 1217 : cluster 0 pgmap v1200: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 53 MiB/s wr, 849 op/s 2023-09-23T19:47:58.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:57 smithi016 bash[77995]: cluster 2023-09-23T19:47:56.753393+0000 mon.a (mon.0) 1527 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:47:58.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:57 smithi016 bash[77995]: audit 2023-09-23T19:47:56.772584+0000 mon.a (mon.0) 1528 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:47:58.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:57 smithi016 bash[77995]: cluster 2023-09-23T19:47:56.772817+0000 mon.a (mon.0) 1529 : cluster 0 osdmap e217: 8 total, 8 up, 8 in 2023-09-23T19:47:58.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:57 smithi016 bash[77995]: cluster 2023-09-23T19:47:56.774769+0000 osd.5 (osd.5) 143 : cluster 3 failed to encode map e217 with expected crc 2023-09-23T19:47:58.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:57 smithi016 bash[77995]: cluster 2023-09-23T19:47:56.776007+0000 osd.6 (osd.6) 147 : cluster 3 failed to encode map e217 with expected crc 2023-09-23T19:47:58.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:57 smithi016 bash[77995]: cluster 2023-09-23T19:47:56.776037+0000 osd.7 (osd.7) 136 : cluster 3 failed to encode map e217 with expected crc 2023-09-23T19:47:58.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:57 smithi016 bash[77995]: cluster 2023-09-23T19:47:56.777002+0000 osd.5 (osd.5) 144 : cluster 3 failed to encode map e217 with expected crc 2023-09-23T19:47:58.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:57 smithi016 bash[77995]: cluster 2023-09-23T19:47:56.778866+0000 osd.5 (osd.5) 145 : cluster 3 failed to encode map e217 with expected crc 2023-09-23T19:47:58.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:57 smithi016 bash[77995]: cluster 2023-09-23T19:47:56.779988+0000 osd.6 (osd.6) 148 : cluster 3 failed to encode map e217 with expected crc 2023-09-23T19:47:58.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:57 smithi016 bash[77995]: cluster 2023-09-23T19:47:56.780158+0000 osd.7 (osd.7) 137 : cluster 3 failed to encode map e217 with expected crc 2023-09-23T19:47:58.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:57 smithi016 bash[77995]: cluster 2023-09-23T19:47:56.782970+0000 osd.4 (osd.4) 159 : cluster 3 failed to encode map e217 with expected crc 2023-09-23T19:47:58.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:57 smithi016 bash[77995]: audit 2023-09-23T19:47:57.752066+0000 mon.a (mon.0) 1530 : audit 0 from='client.? 172.21.15.16:0/801445213' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:47:58.044 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:47:58.084 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:47:58.119 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:47:58.132 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:47:58.132 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:47:58.132 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:47:58.132 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:47:58.132 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:47:58.133 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:47:58.133 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:47:58.133 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:47:58.137 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:58.154 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 84 16 98236 98220 73.0702 49.75 0.0216918 0.0136798 2023-09-23T19:47:58.168 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:58.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:58.170 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:58.255 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-09-23T19:47:58.256 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [ 2023-09-23T19:47:58.256 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.256 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.36480.0:98296 12.5 e217/207)", 2023-09-23T19:47:58.256 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:47:58.230975+0000", 2023-09-23T19:47:58.256 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 0.014954784, 2023-09-23T19:47:58.257 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.014985619, 2023-09-23T19:47:58.257 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:47:58.257 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "queued for pg", 2023-09-23T19:47:58.257 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:47:58.257 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.257 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:47:58.258 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.230975+0000", 2023-09-23T19:47:58.258 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:47:58.258 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.258 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.258 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:47:58.259 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.230975+0000", 2023-09-23T19:47:58.259 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:47:58.259 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.259 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.259 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:47:58.259 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.230974+0000", 2023-09-23T19:47:58.260 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:47:58.260 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.260 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.260 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:47:58.260 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.231196+0000", 2023-09-23T19:47:58.260 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000222783 2023-09-23T19:47:58.261 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.261 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.261 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:47:58.261 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.231207+0000", 2023-09-23T19:47:58.261 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0869e-05 2023-09-23T19:47:58.261 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.261 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.262 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:47:58.262 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.231215+0000", 2023-09-23T19:47:58.262 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.7559999999999999e-06 2023-09-23T19:47:58.262 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:47:58.262 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:47:58.263 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:47:58.263 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.263 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.263 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.36480.0:98299 12.b 12:d33532e6:::benchmark_data_smithi016_181952_object98298:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e217)", 2023-09-23T19:47:58.263 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:47:58.235119+0000", 2023-09-23T19:47:58.263 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 0.010810652, 2023-09-23T19:47:58.264 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.010909719999999999, 2023-09-23T19:47:58.264 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:47:58.264 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "waiting for sub ops", 2023-09-23T19:47:58.264 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:47:58.264 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.36480", 2023-09-23T19:47:58.264 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1035295215", 2023-09-23T19:47:58.264 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 98299 2023-09-23T19:47:58.265 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.265 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:47:58.265 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.265 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:47:58.265 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.235119+0000", 2023-09-23T19:47:58.265 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:47:58.266 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.266 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.266 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:47:58.266 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.235119+0000", 2023-09-23T19:47:58.266 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:47:58.266 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.267 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.267 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:47:58.267 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.235117+0000", 2023-09-23T19:47:58.267 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-09-23T19:47:58.267 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.267 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.267 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:47:58.267 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.235140+0000", 2023-09-23T19:47:58.267 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.3119e-05 2023-09-23T19:47:58.267 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.267 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.268 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:47:58.268 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.235155+0000", 2023-09-23T19:47:58.268 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.4324999999999999e-05 2023-09-23T19:47:58.268 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.268 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.268 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:47:58.268 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.235163+0000", 2023-09-23T19:47:58.268 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.3250000000000008e-06 2023-09-23T19:47:58.268 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.268 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.268 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:47:58.268 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.240219+0000", 2023-09-23T19:47:58.269 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0050559139999999999 2023-09-23T19:47:58.269 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.269 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.269 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:47:58.269 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.240503+0000", 2023-09-23T19:47:58.269 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00028420000000000002 2023-09-23T19:47:58.269 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.270 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.270 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 3,6", 2023-09-23T19:47:58.270 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.240680+0000", 2023-09-23T19:47:58.270 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00017723899999999999 2023-09-23T19:47:58.270 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:47:58.270 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:47:58.270 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:47:58.271 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.271 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.271 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.36480.0:98302 12.5 e217/207)", 2023-09-23T19:47:58.271 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:47:58.236027+0000", 2023-09-23T19:47:58.271 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 0.0099020789999999994, 2023-09-23T19:47:58.271 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.010077533, 2023-09-23T19:47:58.271 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:47:58.272 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "queued for pg", 2023-09-23T19:47:58.272 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:47:58.272 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.272 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:47:58.272 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.236027+0000", 2023-09-23T19:47:58.272 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:47:58.272 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.272 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.272 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:47:58.273 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.236027+0000", 2023-09-23T19:47:58.273 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:47:58.273 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.273 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.273 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:47:58.273 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.236027+0000", 2023-09-23T19:47:58.273 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T19:47:58.273 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.273 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.273 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:47:58.274 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.236102+0000", 2023-09-23T19:47:58.274 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.5370000000000005e-05 2023-09-23T19:47:58.274 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.274 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.274 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:47:58.274 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.236118+0000", 2023-09-23T19:47:58.274 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.5522000000000001e-05 2023-09-23T19:47:58.274 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.274 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.274 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:47:58.274 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.236124+0000", 2023-09-23T19:47:58.275 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.8360000000000003e-06 2023-09-23T19:47:58.275 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:47:58.275 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:47:58.275 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:47:58.275 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.275 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.275 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.36480.0:98303 12.8 12:148c2288:::benchmark_data_smithi016_181952_object98302:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e217)", 2023-09-23T19:47:58.275 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:47:58.236574+0000", 2023-09-23T19:47:58.275 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 0.0093554039999999995, 2023-09-23T19:47:58.275 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0095880470000000006, 2023-09-23T19:47:58.275 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:47:58.276 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "waiting for sub ops", 2023-09-23T19:47:58.276 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:47:58.276 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.36480", 2023-09-23T19:47:58.276 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1035295215", 2023-09-23T19:47:58.276 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 98303 2023-09-23T19:47:58.276 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.276 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:47:58.276 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.276 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:47:58.276 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.236574+0000", 2023-09-23T19:47:58.276 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:47:58.276 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.277 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.277 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:47:58.277 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.236574+0000", 2023-09-23T19:47:58.277 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:47:58.277 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.277 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.277 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:47:58.277 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.236573+0000", 2023-09-23T19:47:58.277 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:47:58.277 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.277 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.277 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:47:58.278 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.236597+0000", 2023-09-23T19:47:58.278 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.3609999999999999e-05 2023-09-23T19:47:58.278 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.278 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.278 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:47:58.278 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.236608+0000", 2023-09-23T19:47:58.278 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.181e-05 2023-09-23T19:47:58.278 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.278 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.278 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:47:58.278 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.236616+0000", 2023-09-23T19:47:58.279 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.1509999999999998e-06 2023-09-23T19:47:58.279 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.279 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.279 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:47:58.279 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.236701+0000", 2023-09-23T19:47:58.279 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.5339000000000004e-05 2023-09-23T19:47:58.280 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.280 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.280 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:47:58.280 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.236834+0000", 2023-09-23T19:47:58.280 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00013285 2023-09-23T19:47:58.280 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.280 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.280 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,3", 2023-09-23T19:47:58.280 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.236974+0000", 2023-09-23T19:47:58.280 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000140285 2023-09-23T19:47:58.280 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.281 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.281 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:47:58.281 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.241830+0000", 2023-09-23T19:47:58.281 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0048556449999999996 2023-09-23T19:47:58.281 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:47:58.281 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:47:58.281 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:47:58.281 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.281 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.281 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.36480.0:98304 12.b 12.eaff703b (undecoded) ondisk+write+known_if_redirected e217)", 2023-09-23T19:47:58.281 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:47:58.236631+0000", 2023-09-23T19:47:58.281 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 0.0092978920000000003, 2023-09-23T19:47:58.281 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0096125680000000002, 2023-09-23T19:47:58.282 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:47:58.282 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "queued for pg", 2023-09-23T19:47:58.282 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:47:58.282 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.36480", 2023-09-23T19:47:58.282 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1035295215", 2023-09-23T19:47:58.282 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 98304 2023-09-23T19:47:58.282 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.282 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:47:58.282 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.282 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:47:58.282 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.236631+0000", 2023-09-23T19:47:58.283 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:47:58.283 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.283 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.283 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:47:58.283 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.236631+0000", 2023-09-23T19:47:58.283 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:47:58.283 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.283 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.283 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:47:58.284 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.236631+0000", 2023-09-23T19:47:58.284 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T19:47:58.284 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.284 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.284 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:47:58.284 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.236651+0000", 2023-09-23T19:47:58.284 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.9865999999999999e-05 2023-09-23T19:47:58.284 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.284 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.284 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:47:58.284 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.236658+0000", 2023-09-23T19:47:58.285 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.2509999999999997e-06 2023-09-23T19:47:58.285 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.285 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.285 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:47:58.285 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.236663+0000", 2023-09-23T19:47:58.285 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.3340000000000001e-06 2023-09-23T19:47:58.285 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:47:58.285 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:47:58.285 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:47:58.285 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.285 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.286 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.36480.0:98305 12.5 e217/207)", 2023-09-23T19:47:58.286 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:47:58.242374+0000", 2023-09-23T19:47:58.286 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 0.003555477, 2023-09-23T19:47:58.286 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00392451, 2023-09-23T19:47:58.286 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:47:58.286 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "queued for pg", 2023-09-23T19:47:58.286 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:47:58.286 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.286 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:47:58.286 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.242374+0000", 2023-09-23T19:47:58.286 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:47:58.287 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.287 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.287 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:47:58.287 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.242374+0000", 2023-09-23T19:47:58.287 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:47:58.287 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.287 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.287 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:47:58.287 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.242373+0000", 2023-09-23T19:47:58.287 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T19:47:58.287 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.287 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.288 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:47:58.288 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.242402+0000", 2023-09-23T19:47:58.288 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.9085e-05 2023-09-23T19:47:58.288 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.288 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.288 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:47:58.288 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.242415+0000", 2023-09-23T19:47:58.288 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2601e-05 2023-09-23T19:47:58.288 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.288 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.288 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:47:58.289 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.242421+0000", 2023-09-23T19:47:58.289 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.8279999999999999e-06 2023-09-23T19:47:58.289 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:47:58.289 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:47:58.289 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:47:58.289 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.289 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.289 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.36480.0:98307 12.d e217/207)", 2023-09-23T19:47:58.289 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:47:58.245932+0000", 2023-09-23T19:47:58.289 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 4294967295.9999976, 2023-09-23T19:47:58.289 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00041548900000000001, 2023-09-23T19:47:58.290 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:47:58.290 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "queued for pg", 2023-09-23T19:47:58.290 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:47:58.290 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.290 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:47:58.290 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.245932+0000", 2023-09-23T19:47:58.290 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:47:58.290 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.290 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.290 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:47:58.290 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.245932+0000", 2023-09-23T19:47:58.291 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:47:58.291 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.291 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.291 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:47:58.291 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.245931+0000", 2023-09-23T19:47:58.291 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:47:58.291 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.291 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.291 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:47:58.291 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.245967+0000", 2023-09-23T19:47:58.291 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.6037000000000001e-05 2023-09-23T19:47:58.292 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.292 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.292 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:47:58.292 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.245987+0000", 2023-09-23T19:47:58.292 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.0129e-05 2023-09-23T19:47:58.292 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.292 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.292 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:47:58.292 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.245995+0000", 2023-09-23T19:47:58.292 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.3350000000000007e-06 2023-09-23T19:47:58.292 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:47:58.292 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:47:58.293 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:47:58.293 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.293 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.293 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.36480.0:98308 12.a 12:5aee1358:::benchmark_data_smithi016_181952_object98307:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e217)", 2023-09-23T19:47:58.293 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:47:58.246075+0000", 2023-09-23T19:47:58.293 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 4294967295.9998541, 2023-09-23T19:47:58.293 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000329735, 2023-09-23T19:47:58.293 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:47:58.293 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "started", 2023-09-23T19:47:58.293 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:47:58.294 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.36480", 2023-09-23T19:47:58.294 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1035295215", 2023-09-23T19:47:58.294 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 98308 2023-09-23T19:47:58.294 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.294 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:47:58.294 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.294 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:47:58.294 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.246075+0000", 2023-09-23T19:47:58.294 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:47:58.294 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.295 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.295 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:47:58.295 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.246075+0000", 2023-09-23T19:47:58.295 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:47:58.295 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.295 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.295 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:47:58.295 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.246074+0000", 2023-09-23T19:47:58.295 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:47:58.295 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.295 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.295 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:47:58.295 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.246093+0000", 2023-09-23T19:47:58.296 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.9349999999999999e-05 2023-09-23T19:47:58.296 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.296 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.296 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:47:58.296 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.246100+0000", 2023-09-23T19:47:58.296 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.55e-06 2023-09-23T19:47:58.296 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.296 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.296 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:47:58.296 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.246104+0000", 2023-09-23T19:47:58.296 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.0690000000000003e-06 2023-09-23T19:47:58.296 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.297 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.297 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:47:58.297 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.246162+0000", 2023-09-23T19:47:58.297 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.7565000000000002e-05 2023-09-23T19:47:58.297 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.297 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.297 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:47:58.297 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.246309+0000", 2023-09-23T19:47:58.297 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00014735700000000001 2023-09-23T19:47:58.297 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.297 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.297 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,3", 2023-09-23T19:47:58.298 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.246411+0000", 2023-09-23T19:47:58.298 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000101643 2023-09-23T19:47:58.298 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:47:58.298 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:47:58.298 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:47:58.298 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.298 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.298 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.36480.0:98289 12.5 e217/207 12:ae6d363e:::benchmark_data_smithi016_181952_object98288:head v 217'5954, mlcod=217'5952)", 2023-09-23T19:47:58.298 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:47:58.219084+0000", 2023-09-23T19:47:58.298 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 0.026845187999999999, 2023-09-23T19:47:58.298 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.027400224000000001, 2023-09-23T19:47:58.298 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:47:58.299 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "started", 2023-09-23T19:47:58.299 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:47:58.299 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.299 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:47:58.299 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.219084+0000", 2023-09-23T19:47:58.299 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:47:58.299 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.299 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.299 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:47:58.299 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.219084+0000", 2023-09-23T19:47:58.300 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:47:58.300 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.300 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.300 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:47:58.300 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.219083+0000", 2023-09-23T19:47:58.300 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T19:47:58.300 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.300 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.301 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:47:58.301 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.219129+0000", 2023-09-23T19:47:58.301 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.5878999999999999e-05 2023-09-23T19:47:58.301 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.301 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.301 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:47:58.301 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.219142+0000", 2023-09-23T19:47:58.301 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2808e-05 2023-09-23T19:47:58.301 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.301 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.301 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:47:58.302 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.219148+0000", 2023-09-23T19:47:58.302 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.8250000000000001e-06 2023-09-23T19:47:58.302 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.302 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.302 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:47:58.302 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.219269+0000", 2023-09-23T19:47:58.302 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000120678 2023-09-23T19:47:58.302 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.302 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.302 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:47:58.303 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.219306+0000", 2023-09-23T19:47:58.303 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.7341e-05 2023-09-23T19:47:58.303 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:47:58.303 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:47:58.303 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:47:58.304 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.304 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.304 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.36480.0:98293 12.d e217/207 12:bebb7b1b:::benchmark_data_smithi016_181952_object98292:head v 217'6181, mlcod=217'6177)", 2023-09-23T19:47:58.304 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:47:58.221995+0000", 2023-09-23T19:47:58.304 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 0.023934278, 2023-09-23T19:47:58.304 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.024553044999999999, 2023-09-23T19:47:58.304 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:47:58.305 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "started", 2023-09-23T19:47:58.305 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:47:58.305 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.305 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:47:58.305 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.221995+0000", 2023-09-23T19:47:58.305 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:47:58.305 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.305 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.305 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:47:58.305 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.221995+0000", 2023-09-23T19:47:58.305 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:47:58.306 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.306 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.306 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:47:58.306 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.221994+0000", 2023-09-23T19:47:58.306 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:47:58.306 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.306 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.306 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:47:58.306 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.222028+0000", 2023-09-23T19:47:58.306 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.3586e-05 2023-09-23T19:47:58.306 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.307 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.307 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:47:58.307 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.222042+0000", 2023-09-23T19:47:58.307 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.4606999999999999e-05 2023-09-23T19:47:58.307 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.307 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.307 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:47:58.307 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.222051+0000", 2023-09-23T19:47:58.307 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.1359999999999997e-06 2023-09-23T19:47:58.307 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.307 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.308 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:47:58.308 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.226638+0000", 2023-09-23T19:47:58.308 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.004587331 2023-09-23T19:47:58.308 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.308 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.308 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:47:58.308 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.226657+0000", 2023-09-23T19:47:58.308 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.8983000000000001e-05 2023-09-23T19:47:58.308 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:47:58.308 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:47:58.309 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:47:58.309 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.309 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.309 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.36480.0:98294 12.5 e217/207 12:af68f508:::benchmark_data_smithi016_181952_object98293:head v 217'5955, mlcod=217'5952)", 2023-09-23T19:47:58.309 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:47:58.223845+0000", 2023-09-23T19:47:58.309 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 0.022084734000000002, 2023-09-23T19:47:58.309 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.022769805000000001, 2023-09-23T19:47:58.309 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:47:58.309 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "started", 2023-09-23T19:47:58.309 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:47:58.309 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.309 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:47:58.310 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.223845+0000", 2023-09-23T19:47:58.310 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:47:58.310 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.310 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.310 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:47:58.310 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.223845+0000", 2023-09-23T19:47:58.310 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:47:58.310 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.310 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.310 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:47:58.310 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.223844+0000", 2023-09-23T19:47:58.311 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T19:47:58.311 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.311 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.311 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:47:58.311 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.223920+0000", 2023-09-23T19:47:58.311 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.5809999999999994e-05 2023-09-23T19:47:58.311 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.311 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.311 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:47:58.311 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.226436+0000", 2023-09-23T19:47:58.311 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0025164530000000001 2023-09-23T19:47:58.311 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.312 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.312 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:47:58.312 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.226445+0000", 2023-09-23T19:47:58.312 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.6819999999999992e-06 2023-09-23T19:47:58.312 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.312 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.312 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:47:58.312 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.229902+0000", 2023-09-23T19:47:58.312 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.003456672 2023-09-23T19:47:58.312 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.312 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.312 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:47:58.313 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.229996+0000", 2023-09-23T19:47:58.313 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.4419000000000005e-05 2023-09-23T19:47:58.313 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:47:58.313 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:47:58.313 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:47:58.313 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.313 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.313 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.36480.0:98297 12.a 12:5ffa3217:::benchmark_data_smithi016_181952_object98296:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e217)", 2023-09-23T19:47:58.313 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:47:58.230465+0000", 2023-09-23T19:47:58.313 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 0.015464053, 2023-09-23T19:47:58.313 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.016223185000000001, 2023-09-23T19:47:58.313 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:47:58.314 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "waiting for sub ops", 2023-09-23T19:47:58.314 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:47:58.314 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.36480", 2023-09-23T19:47:58.314 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1035295215", 2023-09-23T19:47:58.314 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 98297 2023-09-23T19:47:58.314 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.314 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:47:58.314 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.314 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:47:58.314 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.230465+0000", 2023-09-23T19:47:58.314 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:47:58.315 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.315 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.315 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:47:58.315 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.230465+0000", 2023-09-23T19:47:58.315 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:47:58.315 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.315 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.315 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:47:58.315 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.230464+0000", 2023-09-23T19:47:58.315 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-09-23T19:47:58.315 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.315 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.316 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:47:58.316 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.230489+0000", 2023-09-23T19:47:58.316 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.5661999999999999e-05 2023-09-23T19:47:58.316 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.316 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.316 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:47:58.316 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.230507+0000", 2023-09-23T19:47:58.316 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.8263000000000001e-05 2023-09-23T19:47:58.316 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.316 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.316 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:47:58.316 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.230521+0000", 2023-09-23T19:47:58.317 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3587999999999999e-05 2023-09-23T19:47:58.317 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.317 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.317 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:47:58.317 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.230575+0000", 2023-09-23T19:47:58.317 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.4357999999999997e-05 2023-09-23T19:47:58.317 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.317 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.317 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:47:58.317 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.230694+0000", 2023-09-23T19:47:58.317 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00011879600000000001 2023-09-23T19:47:58.317 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.317 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.318 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,3", 2023-09-23T19:47:58.318 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.230895+0000", 2023-09-23T19:47:58.318 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00020082499999999999 2023-09-23T19:47:58.318 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.318 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.318 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:47:58.318 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.234483+0000", 2023-09-23T19:47:58.318 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0035875170000000001 2023-09-23T19:47:58.318 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:47:58.318 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:47:58.318 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:47:58.318 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:58.246128+0000", 2023-09-23T19:47:58.319 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.011645463 2023-09-23T19:47:58.319 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:47:58.319 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:47:58.319 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:47:58.319 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:47:58.319 INFO:teuthology.orchestra.run.smithi016.stdout: ], 2023-09-23T19:47:58.319 INFO:teuthology.orchestra.run.smithi016.stdout: "num_ops": 12 2023-09-23T19:47:58.319 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-09-23T19:47:58.319 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:58.343 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:58.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:58.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:47:58.446 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:58.490 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:47:58.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:58.493 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:47:58.594 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:58.623 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:58.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:58.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:58.726 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:58.754 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:58.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:58.757 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:58.858 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:58.888 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:47:58.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:58.894 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:47:58.995 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:59.020 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:59.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:59.022 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:59.123 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:59.154 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 85 16 99000 98984 72.7723 47.75 0.0144621 0.0137353 2023-09-23T19:47:59.154 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:59.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:59.160 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:59.261 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:59.307 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:47:59.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:59.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:47:59.411 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:59.436 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:47:59.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:59.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:47:59.541 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:59.559 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:47:59.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:59.562 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:47:59.663 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:59.692 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:59.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:59.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:47:59.715 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:59 smithi016 bash[75004]: cluster 2023-09-23T19:47:58.352111+0000 mgr.y (mgr.25325) 1218 : cluster 0 pgmap v1202: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 44 MiB/s wr, 710 op/s 2023-09-23T19:47:59.715 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:59 smithi016 bash[75004]: audit 2023-09-23T19:47:58.680356+0000 mon.a (mon.0) 1531 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:47:59.715 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:47:59 smithi016 bash[75004]: audit 2023-09-23T19:47:58.682238+0000 mon.a (mon.0) 1532 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:47:59.795 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:59.820 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:47:59.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:59.823 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:47:59.924 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:59.934 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:59 smithi092 bash[58855]: cluster 2023-09-23T19:47:58.352111+0000 mgr.y (mgr.25325) 1218 : cluster 0 pgmap v1202: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 44 MiB/s wr, 710 op/s 2023-09-23T19:47:59.934 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:59 smithi092 bash[58855]: audit 2023-09-23T19:47:58.680356+0000 mon.a (mon.0) 1531 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:47:59.934 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:47:59 smithi092 bash[58855]: audit 2023-09-23T19:47:58.682238+0000 mon.a (mon.0) 1532 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:47:59.958 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:47:59.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:47:59.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:48:00.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:59 smithi016 bash[77995]: cluster 2023-09-23T19:47:58.352111+0000 mgr.y (mgr.25325) 1218 : cluster 0 pgmap v1202: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 44 MiB/s wr, 710 op/s 2023-09-23T19:48:00.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:59 smithi016 bash[77995]: audit 2023-09-23T19:47:58.680356+0000 mon.a (mon.0) 1531 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:48:00.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:47:59 smithi016 bash[77995]: audit 2023-09-23T19:47:58.682238+0000 mon.a (mon.0) 1532 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:48:00.065 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:00.092 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:00.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:00.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:48:00.154 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 86 16 99653 99637 72.4006 40.8125 0.0302862 0.0138057 2023-09-23T19:48:00.198 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:00.226 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:00.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:00.233 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:48:00.333 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:00.376 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:00.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:00.381 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:48:00.481 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:00.518 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:00.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:00.522 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:00.623 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:00.648 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:00.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:00.651 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:00.752 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:00.790 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:00.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:00.798 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:48:00.899 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:00.939 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:00.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:00.947 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:01.048 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:01.073 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:01.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:01.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:48:01.154 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 87 16 100453 100437 72.1431 50 0.0219107 0.0138555 2023-09-23T19:48:01.180 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:01.206 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:01.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:01.209 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:48:01.309 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:01.337 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:01.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:01.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:48:01.444 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:01.473 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:01.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:01.476 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:01.530 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T19:48:01.577 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:01.657 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:01.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:01.661 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:01.762 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:01.771 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:01 smithi016 bash[77995]: cluster 2023-09-23T19:48:00.352736+0000 mgr.y (mgr.25325) 1219 : cluster 0 pgmap v1203: 89 pgs: 89 active+clean; 6.0 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 58 MiB/s wr, 927 op/s 2023-09-23T19:48:01.772 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:01 smithi016 bash[75004]: cluster 2023-09-23T19:48:00.352736+0000 mgr.y (mgr.25325) 1219 : cluster 0 pgmap v1203: 89 pgs: 89 active+clean; 6.0 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 58 MiB/s wr, 927 op/s 2023-09-23T19:48:01.798 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:01.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:01.803 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:01.904 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:01.942 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:01.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:01.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:02.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:01 smithi092 bash[58855]: cluster 2023-09-23T19:48:00.352736+0000 mgr.y (mgr.25325) 1219 : cluster 0 pgmap v1203: 89 pgs: 89 active+clean; 6.0 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 58 MiB/s wr, 927 op/s 2023-09-23T19:48:02.048 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:02.081 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:02.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:02.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:02.155 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 88 16 101200 101184 71.8538 46.6875 0.0241659 0.0139116 2023-09-23T19:48:02.184 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:02.219 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:02.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:02.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:02.325 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:02.355 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:02.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:02.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:02.461 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:02.490 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:02.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:02.497 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:48:02.599 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:02.627 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:02.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:02.633 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:48:02.733 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:02.755 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T19:48:02.767 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:02.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:02.774 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:48:02.875 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:02.907 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:02.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:02.910 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:03.011 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:03.048 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:03.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:03.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:03.154 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:03.154 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 89 16 101950 101934 71.5731 46.875 0.0180051 0.0139662 2023-09-23T19:48:03.190 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:03.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:03.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:03.296 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:03.331 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:03.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:03.338 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:03.438 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:03.471 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:03.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:03.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:03.513 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:48:03.576 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:03.603 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:03.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:03.605 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:48:03.692 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:03 smithi016 bash[75004]: cluster 2023-09-23T19:48:02.353275+0000 mgr.y (mgr.25325) 1220 : cluster 0 pgmap v1204: 89 pgs: 89 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 47 MiB/s wr, 747 op/s 2023-09-23T19:48:03.706 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:03.735 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:03.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:03.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:03.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:03 smithi092 bash[58855]: cluster 2023-09-23T19:48:02.353275+0000 mgr.y (mgr.25325) 1220 : cluster 0 pgmap v1204: 89 pgs: 89 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 47 MiB/s wr, 747 op/s 2023-09-23T19:48:03.843 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:03.863 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-09-23T19:48:03.863 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T19:48:03.863 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-09-23T19:48:03.863 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T19:48:03.863 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T19:48:03.863 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-09-23T19:48:03.887 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:03.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:03.891 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:03.992 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:04.001 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:03 smithi016 bash[77995]: cluster 2023-09-23T19:48:02.353275+0000 mgr.y (mgr.25325) 1220 : cluster 0 pgmap v1204: 89 pgs: 89 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 47 MiB/s wr, 747 op/s 2023-09-23T19:48:04.039 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:04.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:04.042 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:04.142 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:04.155 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Total time run: 90.0086 2023-09-23T19:48:04.155 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Total writes made: 102678 2023-09-23T19:48:04.155 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Write size: 65536 2023-09-23T19:48:04.155 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Object size: 65536 2023-09-23T19:48:04.155 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Bandwidth (MB/sec): 71.2974 2023-09-23T19:48:04.155 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Stddev Bandwidth: 26.9932 2023-09-23T19:48:04.155 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Max bandwidth (MB/sec): 163.312 2023-09-23T19:48:04.155 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Min bandwidth (MB/sec): 39.6875 2023-09-23T19:48:04.155 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Average IOPS: 1140 2023-09-23T19:48:04.155 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Stddev IOPS: 431.892 2023-09-23T19:48:04.156 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Max IOPS: 2613 2023-09-23T19:48:04.156 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Min IOPS: 635 2023-09-23T19:48:04.156 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Average Latency(s): 0.0140209 2023-09-23T19:48:04.156 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Stddev Latency(s): 0.0139094 2023-09-23T19:48:04.156 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Max latency(s): 0.404369 2023-09-23T19:48:04.156 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Min latency(s): 0.00140724 2023-09-23T19:48:04.162 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T19:48:04.174 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:04.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:04.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:48:04.277 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:04.311 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:04.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:04.313 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:04.414 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:04.437 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:04.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:04.444 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:04.545 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:04.581 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:04.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:04.584 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:04.684 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:04.714 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:04.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:04.718 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:04.818 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:04.850 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:04.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:04.856 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:04.957 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:04.989 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:04.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:04.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:48:05.091 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:05.119 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:05.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:05.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:05.223 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:05.252 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:05.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:05.254 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:05.355 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:05.398 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:05.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:05.405 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:05.506 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:05.537 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:05.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:05.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:05.641 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:05.642 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:48:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:48:05] "GET /metrics HTTP/1.1" 200 36300 "" "Prometheus/2.33.4" 2023-09-23T19:48:05.669 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:05.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:05.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:05.774 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:05.785 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:05 smithi092 bash[58855]: cluster 2023-09-23T19:48:04.353738+0000 mgr.y (mgr.25325) 1221 : cluster 0 pgmap v1205: 89 pgs: 89 active+clean; 6.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 42 MiB/s wr, 678 op/s 2023-09-23T19:48:05.805 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:05.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:05.807 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:48:05.908 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:05.961 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:05 smithi016 bash[77995]: cluster 2023-09-23T19:48:04.353738+0000 mgr.y (mgr.25325) 1221 : cluster 0 pgmap v1205: 89 pgs: 89 active+clean; 6.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 42 MiB/s wr, 678 op/s 2023-09-23T19:48:05.962 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:05 smithi016 bash[75004]: cluster 2023-09-23T19:48:04.353738+0000 mgr.y (mgr.25325) 1221 : cluster 0 pgmap v1205: 89 pgs: 89 active+clean; 6.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 42 MiB/s wr, 678 op/s 2023-09-23T19:48:05.983 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:05.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:05.989 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:06.090 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:06.114 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:06.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:06.119 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:06.219 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:06.266 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:06.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:06.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:06.373 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:06.404 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:06.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:06.409 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:48:06.433 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:48:06.510 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:06.531 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:06.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:06.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:48:06.639 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:06.675 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:06.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:06.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:06.779 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:06.811 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:06.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:06.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:06.918 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:06.941 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:06.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:06.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:48:07.044 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:07.072 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:07.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:07.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:07.176 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:07.203 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:07.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:07.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:48:07.306 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:07.332 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:07.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:07.335 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:48:07.435 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:07.449 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:48:07.453 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:48:07.477 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:07.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:07.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:07.584 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:07.617 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:07.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:07.621 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:07.722 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:07.751 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:07.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:07.756 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:07.856 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:07.877 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:07 smithi016 bash[75004]: cluster 2023-09-23T19:48:06.354462+0000 mgr.y (mgr.25325) 1222 : cluster 0 pgmap v1206: 89 pgs: 89 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 45 MiB/s wr, 751 op/s 2023-09-23T19:48:07.878 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:07 smithi016 bash[77995]: cluster 2023-09-23T19:48:06.354462+0000 mgr.y (mgr.25325) 1222 : cluster 0 pgmap v1206: 89 pgs: 89 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 45 MiB/s wr, 751 op/s 2023-09-23T19:48:07.883 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:07.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:07.886 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:07.987 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:08.023 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:08.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:08.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:08.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:07 smithi092 bash[58855]: cluster 2023-09-23T19:48:06.354462+0000 mgr.y (mgr.25325) 1222 : cluster 0 pgmap v1206: 89 pgs: 89 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 45 MiB/s wr, 751 op/s 2023-09-23T19:48:08.127 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:08.152 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:08.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:08.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:48:08.256 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:08.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:08.290 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:08.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:08.391 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:08.418 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:08.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:08.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:48:08.526 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:08.549 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:08.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:08.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:08.656 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:08.671 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:48:08.694 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:08.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:08.697 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:08.798 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:08.840 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:08.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:08.846 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:08.947 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:08.976 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:08.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:08.981 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:48:09.082 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:09.110 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:09.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:09.112 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:48:09.213 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:09.243 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:09.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:09.248 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:48:09.349 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:09.373 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:09.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:09.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:48:09.477 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:09.500 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:09.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:09.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:09.603 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:09.632 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:09.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:09.642 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:48:09.743 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:09.770 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:09.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:09.772 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:09.873 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:09.882 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:09 smithi016 bash[75004]: cluster 2023-09-23T19:48:08.354971+0000 mgr.y (mgr.25325) 1223 : cluster 0 pgmap v1207: 89 pgs: 89 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 88 B/s rd, 40 MiB/s wr, 777 op/s 2023-09-23T19:48:09.882 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:09 smithi016 bash[77995]: cluster 2023-09-23T19:48:08.354971+0000 mgr.y (mgr.25325) 1223 : cluster 0 pgmap v1207: 89 pgs: 89 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 88 B/s rd, 40 MiB/s wr, 777 op/s 2023-09-23T19:48:09.898 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:09.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:09.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:10.003 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:10.011 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:09 smithi092 bash[58855]: cluster 2023-09-23T19:48:08.354971+0000 mgr.y (mgr.25325) 1223 : cluster 0 pgmap v1207: 89 pgs: 89 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 88 B/s rd, 40 MiB/s wr, 777 op/s 2023-09-23T19:48:10.037 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:10.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:10.042 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:48:10.143 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:10.178 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:10.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:10.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:48:10.286 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:10.309 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:10.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:10.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:10.414 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:10.443 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:10.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:10.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:48:10.547 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:10.575 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:10.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:10.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:10.678 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:10.705 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:10.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:10.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:48:10.808 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:10.845 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:10.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:10.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:10.951 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:10.987 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:10.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:10.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:48:11.091 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:11.114 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:11.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:11.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:11.217 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:11.244 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:11.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:11.247 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:11.348 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:11.377 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:11.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:11.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:48:11.480 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:11.505 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:11.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:11.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:48:11.612 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:11.647 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:11.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:11.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:11.750 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:11.758 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:11 smithi092 bash[58855]: cluster 2023-09-23T19:48:10.355712+0000 mgr.y (mgr.25325) 1224 : cluster 0 pgmap v1208: 89 pgs: 89 active+clean; 6.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 33 MiB/s wr, 806 op/s 2023-09-23T19:48:11.788 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:11.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:11.792 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:11.893 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:11.917 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:11.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:11.921 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:12.022 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:12.032 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:11 smithi016 bash[75004]: cluster 2023-09-23T19:48:10.355712+0000 mgr.y (mgr.25325) 1224 : cluster 0 pgmap v1208: 89 pgs: 89 active+clean; 6.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 33 MiB/s wr, 806 op/s 2023-09-23T19:48:12.032 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:11 smithi016 bash[77995]: cluster 2023-09-23T19:48:10.355712+0000 mgr.y (mgr.25325) 1224 : cluster 0 pgmap v1208: 89 pgs: 89 active+clean; 6.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 33 MiB/s wr, 806 op/s 2023-09-23T19:48:12.056 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:12.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:12.063 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:12.164 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:12.191 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:12.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:12.197 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:48:12.297 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:12.325 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:12.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:12.330 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:12.430 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:12.452 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:12.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:12.457 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:12.557 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:12.583 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:12.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:12.586 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:48:12.687 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:12.714 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:12.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:12.717 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:48:12.817 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:12.849 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:12.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:12.853 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:12.954 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:12.990 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:12.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:12.994 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:13.095 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:13.119 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:13.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:13.121 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:48:13.222 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:13.251 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:13.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:13.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:48:13.351 INFO:teuthology.orchestra.run.smithi092.stdout:{ 2023-09-23T19:48:13.351 INFO:teuthology.orchestra.run.smithi092.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T19:48:13.353 INFO:teuthology.orchestra.run.smithi092.stdout:} 2023-09-23T19:48:13.355 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:13.384 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:13.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:13.390 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:48:13.491 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:13.526 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:13.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:13.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:13.631 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:13.653 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:13.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:13.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:48:13.756 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:13.779 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:13.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:13.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:13.884 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:13.894 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:13 smithi092 bash[58855]: cluster 2023-09-23T19:48:12.358697+0000 mgr.y (mgr.25325) 1225 : cluster 0 pgmap v1209: 89 pgs: 89 active+clean; 6.0 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 21 MiB/s wr, 714 op/s 2023-09-23T19:48:13.895 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:13 smithi092 bash[58855]: audit 2023-09-23T19:48:13.668615+0000 mon.a (mon.0) 1533 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:48:13.910 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:13.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:13.916 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:48:14.017 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:14.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:13 smithi016 bash[75004]: cluster 2023-09-23T19:48:12.358697+0000 mgr.y (mgr.25325) 1225 : cluster 0 pgmap v1209: 89 pgs: 89 active+clean; 6.0 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 21 MiB/s wr, 714 op/s 2023-09-23T19:48:14.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:13 smithi016 bash[75004]: audit 2023-09-23T19:48:13.668615+0000 mon.a (mon.0) 1533 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:48:14.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:13 smithi016 bash[77995]: cluster 2023-09-23T19:48:12.358697+0000 mgr.y (mgr.25325) 1225 : cluster 0 pgmap v1209: 89 pgs: 89 active+clean; 6.0 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 21 MiB/s wr, 714 op/s 2023-09-23T19:48:14.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:13 smithi016 bash[77995]: audit 2023-09-23T19:48:13.668615+0000 mon.a (mon.0) 1533 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:48:14.046 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:14.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:14.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:48:14.149 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:14.178 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:14.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:14.181 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:48:14.282 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:14.307 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:14.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:14.309 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:14.410 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:14.451 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:14.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:14.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:48:14.554 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:14.581 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:14.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:14.586 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:14.686 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:14.714 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:14.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:14.718 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:14.819 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:14.845 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:14.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:14.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:14.953 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:14.979 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:14.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:14.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:48:15.086 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:15.123 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:15.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:15.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:15.226 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:15.251 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:15.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:15.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:48:15.361 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:15.385 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:15.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:15.388 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:48:15.489 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:15.518 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:15.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:15.521 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:48:15.622 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:15.645 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:15.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:15.653 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:15.754 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:15.762 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:15 smithi092 bash[58855]: cluster 2023-09-23T19:48:14.359382+0000 mgr.y (mgr.25325) 1226 : cluster 0 pgmap v1210: 89 pgs: 89 active+clean; 5.9 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 19 MiB/s wr, 762 op/s 2023-09-23T19:48:15.762 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:15 smithi092 bash[58855]: audit 2023-09-23T19:48:15.600472+0000 mon.a (mon.0) 1534 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:48:15.779 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:15.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:15.785 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:48:15.886 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:15.899 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:15 smithi016 bash[75004]: cluster 2023-09-23T19:48:14.359382+0000 mgr.y (mgr.25325) 1226 : cluster 0 pgmap v1210: 89 pgs: 89 active+clean; 5.9 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 19 MiB/s wr, 762 op/s 2023-09-23T19:48:15.899 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:15 smithi016 bash[75004]: audit 2023-09-23T19:48:15.600472+0000 mon.a (mon.0) 1534 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:48:15.901 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:48:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:48:15] "GET /metrics HTTP/1.1" 200 36310 "" "Prometheus/2.33.4" 2023-09-23T19:48:15.902 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:15 smithi016 bash[77995]: cluster 2023-09-23T19:48:14.359382+0000 mgr.y (mgr.25325) 1226 : cluster 0 pgmap v1210: 89 pgs: 89 active+clean; 5.9 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 19 MiB/s wr, 762 op/s 2023-09-23T19:48:15.902 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:15 smithi016 bash[77995]: audit 2023-09-23T19:48:15.600472+0000 mon.a (mon.0) 1534 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:48:15.920 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:15.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:15.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:16.026 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:16.058 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:16.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:16.061 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:48:16.162 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:16.197 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:16.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:16.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:48:16.303 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:16.340 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:16.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:16.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:16.444 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:16.471 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:16.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:16.478 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:16.579 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:16.607 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:16.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:16.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:16.711 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:16.742 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:16.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:16.744 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:16.845 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:16.879 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:16.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:16.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:16.981 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:17.014 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:17.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:17.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:17.117 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:17.144 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:17.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:17.148 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:17.249 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:17.271 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:17.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:17.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:17.375 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:17.400 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:17.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:17.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:17.508 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:17.534 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:17.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:17.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:17.640 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:17.666 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:17.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:17.669 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:48:17.769 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:17.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:17 smithi092 bash[58855]: cluster 2023-09-23T19:48:16.360087+0000 mgr.y (mgr.25325) 1227 : cluster 0 pgmap v1211: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 11 MiB/s wr, 813 op/s 2023-09-23T19:48:17.793 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:17.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:17.795 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:17.896 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:17.931 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:17.933 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:17 smithi016 bash[75004]: cluster 2023-09-23T19:48:16.360087+0000 mgr.y (mgr.25325) 1227 : cluster 0 pgmap v1211: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 11 MiB/s wr, 813 op/s 2023-09-23T19:48:17.933 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:17 smithi016 bash[77995]: cluster 2023-09-23T19:48:16.360087+0000 mgr.y (mgr.25325) 1227 : cluster 0 pgmap v1211: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 11 MiB/s wr, 813 op/s 2023-09-23T19:48:17.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:17.938 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:48:18.038 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:18.066 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:18.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:18.073 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:48:18.175 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:18.208 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:18.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:18.213 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:18.314 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:18.346 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:18.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:18.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:18.449 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:18.479 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:18.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:18.484 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:18.585 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:18.631 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:18.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:18.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:48:18.734 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:18.758 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:18.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:18.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:18.861 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:18.886 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:18.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:18.891 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:18.992 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:19.026 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:19.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:19.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:19.106 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-09-23T19:48:19.106 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [], 2023-09-23T19:48:19.106 INFO:teuthology.orchestra.run.smithi016.stdout: "complaint_time": 30, 2023-09-23T19:48:19.106 INFO:teuthology.orchestra.run.smithi016.stdout: "num_blocked_ops": 0 2023-09-23T19:48:19.106 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-09-23T19:48:19.128 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:19.163 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:19.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:19.166 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:19.267 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:19.302 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:19.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:19.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:48:19.306 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-09-23T19:48:19.306 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T19:48:19.306 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2023-09-23T19:48:19.306 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T19:48:19.306 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T19:48:19.306 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 config set filestore_inject_stall 3 2023-09-23T19:48:19.405 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:19.432 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:19.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:19.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:48:19.536 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:19.563 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:19.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:19.569 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:19.670 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:19.697 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:19.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:19.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:48:19.801 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:19.810 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:19 smithi016 bash[77995]: cluster 2023-09-23T19:48:18.360593+0000 mgr.y (mgr.25325) 1228 : cluster 0 pgmap v1212: 89 pgs: 89 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 1.4 MiB/s wr, 766 op/s 2023-09-23T19:48:19.810 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:19 smithi016 bash[75004]: cluster 2023-09-23T19:48:18.360593+0000 mgr.y (mgr.25325) 1228 : cluster 0 pgmap v1212: 89 pgs: 89 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 1.4 MiB/s wr, 766 op/s 2023-09-23T19:48:19.838 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:19.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:19.844 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:19.945 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:19.985 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:19.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:19.990 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:20.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:19 smithi092 bash[58855]: cluster 2023-09-23T19:48:18.360593+0000 mgr.y (mgr.25325) 1228 : cluster 0 pgmap v1212: 89 pgs: 89 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 1.4 MiB/s wr, 766 op/s 2023-09-23T19:48:20.091 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:20.122 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:20.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:20.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:48:20.230 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:20.254 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:20.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:20.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:48:20.360 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:20.385 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:20.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:20.387 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:48:20.488 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:20.513 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:20.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:20.518 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:48:20.619 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:20.644 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:20.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:20.649 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:20.750 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:20.785 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:20.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:20.788 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:48:20.888 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:20.924 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:20.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:20.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:21.030 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:21.063 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:21.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:21.070 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:21.154 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:48:21.171 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:21.196 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:21.197 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:48:21.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:21.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:48:21.227 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:48:21.277 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:48:21.305 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:21.312 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:48:21.339 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:21.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:21.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:48:21.347 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:48:21.397 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:48:21.438 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:48:21.441 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:48:21.441 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:48:21.441 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:48:21.441 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:48:21.441 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:48:21.441 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:48:21.441 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:48:21.441 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:48:21.441 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:21.474 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:21.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:21.477 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:21.578 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:21.604 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:21.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:21.607 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:21.708 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:21.719 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T19:48:21.742 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:21.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:21.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:21.773 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is set 2023-09-23T19:48:21.846 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:21.896 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:21.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:21.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:48:22.000 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:22.015 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:21 smithi016 bash[75004]: cluster 2023-09-23T19:48:20.361202+0000 mgr.y (mgr.25325) 1229 : cluster 0 pgmap v1213: 89 pgs: 89 active+clean; 5.6 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 776 op/s 2023-09-23T19:48:22.015 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:21 smithi016 bash[75004]: audit 2023-09-23T19:48:20.858393+0000 mon.a (mon.0) 1535 : audit 1 from='client.? 172.21.15.16:0/2299446686' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:48:22.015 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:21 smithi016 bash[75004]: audit 2023-09-23T19:48:21.101376+0000 mon.a (mon.0) 1536 : audit 0 from='client.? 172.21.15.16:0/2296699668' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:48:22.015 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:21 smithi016 bash[77995]: cluster 2023-09-23T19:48:20.361202+0000 mgr.y (mgr.25325) 1229 : cluster 0 pgmap v1213: 89 pgs: 89 active+clean; 5.6 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 776 op/s 2023-09-23T19:48:22.016 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:21 smithi016 bash[77995]: audit 2023-09-23T19:48:20.858393+0000 mon.a (mon.0) 1535 : audit 1 from='client.? 172.21.15.16:0/2299446686' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:48:22.016 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:21 smithi016 bash[77995]: audit 2023-09-23T19:48:21.101376+0000 mon.a (mon.0) 1536 : audit 0 from='client.? 172.21.15.16:0/2296699668' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:48:22.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:21 smithi092 bash[58855]: cluster 2023-09-23T19:48:20.361202+0000 mgr.y (mgr.25325) 1229 : cluster 0 pgmap v1213: 89 pgs: 89 active+clean; 5.6 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 776 op/s 2023-09-23T19:48:22.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:21 smithi092 bash[58855]: audit 2023-09-23T19:48:20.858393+0000 mon.a (mon.0) 1535 : audit 1 from='client.? 172.21.15.16:0/2299446686' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:48:22.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:21 smithi092 bash[58855]: audit 2023-09-23T19:48:21.101376+0000 mon.a (mon.0) 1536 : audit 0 from='client.? 172.21.15.16:0/2296699668' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:48:22.056 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:22.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:22.060 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:48:22.161 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:22.185 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:22.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:22.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:22.291 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:22.328 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:22.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:22.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:22.432 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:22.473 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:22.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:22.481 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:48:22.582 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:22.604 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:22.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:22.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:48:22.707 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:22.736 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:22.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:22.740 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:22.840 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:22.868 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:22.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:22.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:48:22.966 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:22 smithi016 bash[77995]: cluster 2023-09-23T19:48:21.729419+0000 mon.a (mon.0) 1537 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:48:22.966 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:22 smithi016 bash[77995]: audit 2023-09-23T19:48:21.759805+0000 mon.a (mon.0) 1538 : audit 1 from='client.? 172.21.15.16:0/2299446686' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:48:22.966 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:22 smithi016 bash[77995]: cluster 2023-09-23T19:48:21.760122+0000 mon.a (mon.0) 1539 : cluster 0 osdmap e218: 8 total, 8 up, 8 in 2023-09-23T19:48:22.967 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:22 smithi016 bash[77995]: cluster 2023-09-23T19:48:21.761221+0000 osd.7 (osd.7) 138 : cluster 3 failed to encode map e218 with expected crc 2023-09-23T19:48:22.967 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:22 smithi016 bash[77995]: cluster 2023-09-23T19:48:21.767082+0000 osd.6 (osd.6) 149 : cluster 3 failed to encode map e218 with expected crc 2023-09-23T19:48:22.967 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:22 smithi016 bash[77995]: cluster 2023-09-23T19:48:21.773063+0000 osd.4 (osd.4) 160 : cluster 3 failed to encode map e218 with expected crc 2023-09-23T19:48:22.967 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:22 smithi016 bash[77995]: cluster 2023-09-23T19:48:21.799898+0000 osd.5 (osd.5) 146 : cluster 3 failed to encode map e218 with expected crc 2023-09-23T19:48:22.967 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:22 smithi016 bash[77995]: audit 2023-09-23T19:48:22.056778+0000 mon.a (mon.0) 1540 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:48:22.967 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:22 smithi016 bash[77995]: audit 2023-09-23T19:48:22.070731+0000 mon.a (mon.0) 1541 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:48:22.967 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:22 smithi016 bash[75004]: cluster 2023-09-23T19:48:21.729419+0000 mon.a (mon.0) 1537 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:48:22.968 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:22 smithi016 bash[75004]: audit 2023-09-23T19:48:21.759805+0000 mon.a (mon.0) 1538 : audit 1 from='client.? 172.21.15.16:0/2299446686' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:48:22.968 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:22 smithi016 bash[75004]: cluster 2023-09-23T19:48:21.760122+0000 mon.a (mon.0) 1539 : cluster 0 osdmap e218: 8 total, 8 up, 8 in 2023-09-23T19:48:22.968 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:22 smithi016 bash[75004]: cluster 2023-09-23T19:48:21.761221+0000 osd.7 (osd.7) 138 : cluster 3 failed to encode map e218 with expected crc 2023-09-23T19:48:22.968 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:22 smithi016 bash[75004]: cluster 2023-09-23T19:48:21.767082+0000 osd.6 (osd.6) 149 : cluster 3 failed to encode map e218 with expected crc 2023-09-23T19:48:22.968 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:22 smithi016 bash[75004]: cluster 2023-09-23T19:48:21.773063+0000 osd.4 (osd.4) 160 : cluster 3 failed to encode map e218 with expected crc 2023-09-23T19:48:22.968 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:22 smithi016 bash[75004]: cluster 2023-09-23T19:48:21.799898+0000 osd.5 (osd.5) 146 : cluster 3 failed to encode map e218 with expected crc 2023-09-23T19:48:22.968 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:22 smithi016 bash[75004]: audit 2023-09-23T19:48:22.056778+0000 mon.a (mon.0) 1540 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:48:22.968 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:22 smithi016 bash[75004]: audit 2023-09-23T19:48:22.070731+0000 mon.a (mon.0) 1541 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:48:22.973 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:22.998 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:23.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:23.004 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:22 smithi092 bash[58855]: cluster 2023-09-23T19:48:21.729419+0000 mon.a (mon.0) 1537 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:48:23.004 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:22 smithi092 bash[58855]: audit 2023-09-23T19:48:21.759805+0000 mon.a (mon.0) 1538 : audit 1 from='client.? 172.21.15.16:0/2299446686' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:48:23.005 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:22 smithi092 bash[58855]: cluster 2023-09-23T19:48:21.760122+0000 mon.a (mon.0) 1539 : cluster 0 osdmap e218: 8 total, 8 up, 8 in 2023-09-23T19:48:23.005 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:22 smithi092 bash[58855]: cluster 2023-09-23T19:48:21.761221+0000 osd.7 (osd.7) 138 : cluster 3 failed to encode map e218 with expected crc 2023-09-23T19:48:23.005 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:22 smithi092 bash[58855]: cluster 2023-09-23T19:48:21.767082+0000 osd.6 (osd.6) 149 : cluster 3 failed to encode map e218 with expected crc 2023-09-23T19:48:23.005 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:22 smithi092 bash[58855]: cluster 2023-09-23T19:48:21.773063+0000 osd.4 (osd.4) 160 : cluster 3 failed to encode map e218 with expected crc 2023-09-23T19:48:23.005 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:22 smithi092 bash[58855]: cluster 2023-09-23T19:48:21.799898+0000 osd.5 (osd.5) 146 : cluster 3 failed to encode map e218 with expected crc 2023-09-23T19:48:23.005 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:22 smithi092 bash[58855]: audit 2023-09-23T19:48:22.056778+0000 mon.a (mon.0) 1540 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:48:23.005 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:22 smithi092 bash[58855]: audit 2023-09-23T19:48:22.070731+0000 mon.a (mon.0) 1541 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:48:23.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:23.107 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:23.134 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:23.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:23.140 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:23.241 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:23.277 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:23.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:23.282 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:23.383 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:23.408 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:23.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:23.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:48:23.516 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:23.544 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:23.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:23.546 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:48:23.647 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:23.676 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:23.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:23.679 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:23.781 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:23.810 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:23.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:23.816 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:48:23.917 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:23.942 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:23.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:23.944 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:24.045 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:24.069 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:24.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:24.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:24.181 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:24.211 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:24.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:24.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:48:24.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:24 smithi016 bash[75004]: cluster 2023-09-23T19:48:22.361637+0000 mgr.y (mgr.25325) 1230 : cluster 0 pgmap v1215: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 757 op/s 2023-09-23T19:48:24.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:24 smithi016 bash[77995]: cluster 2023-09-23T19:48:22.361637+0000 mgr.y (mgr.25325) 1230 : cluster 0 pgmap v1215: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 757 op/s 2023-09-23T19:48:24.315 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:24.325 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:24 smithi092 bash[58855]: cluster 2023-09-23T19:48:22.361637+0000 mgr.y (mgr.25325) 1230 : cluster 0 pgmap v1215: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 757 op/s 2023-09-23T19:48:24.345 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:24.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:24.351 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:48:24.452 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:24.481 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:24.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:24.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:48:24.584 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:24.637 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:24.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:24.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:24.746 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:24.778 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:24.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:24.782 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:24.883 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:24.907 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:24.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:24.910 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:48:25.010 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:25.038 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:25.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:25.041 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:25.142 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:25.182 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:25.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:25.189 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:25.290 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:25.325 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:25.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:25.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:25.414 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:48:25.436 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:25.496 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:25.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:25.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:48:25.604 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:25.651 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:25.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:25.654 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:48:25.755 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:25.782 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:25.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:25.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:25.786 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:48:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:48:25] "GET /metrics HTTP/1.1" 200 36310 "" "Prometheus/2.33.4" 2023-09-23T19:48:25.885 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:25.909 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:25.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:25.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:48:26.013 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:26.048 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:26.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:26.050 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:48:26.151 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:26.167 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T19:48:26.173 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:26.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:26.177 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:48:26.277 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:26.286 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:26 smithi092 bash[58855]: cluster 2023-09-23T19:48:24.362039+0000 mgr.y (mgr.25325) 1231 : cluster 0 pgmap v1216: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 744 op/s 2023-09-23T19:48:26.307 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:26.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:26.309 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:48:26.409 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:26.428 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:26 smithi016 bash[75004]: cluster 2023-09-23T19:48:24.362039+0000 mgr.y (mgr.25325) 1231 : cluster 0 pgmap v1216: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 744 op/s 2023-09-23T19:48:26.428 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:26 smithi016 bash[77995]: cluster 2023-09-23T19:48:24.362039+0000 mgr.y (mgr.25325) 1231 : cluster 0 pgmap v1216: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 744 op/s 2023-09-23T19:48:26.459 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:26.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:26.468 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:26.568 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:26.596 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:26.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:26.601 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:48:26.686 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:48:26.701 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:26.736 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:26.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:26.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:26.843 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:26.874 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:26.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:26.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:26.984 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:27.022 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:27.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:27.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:27.128 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:27.157 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:27.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:27.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:27.257 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:27 smithi016 bash[75004]: audit 2023-09-23T19:48:26.254459+0000 mon.a (mon.0) 1542 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:48:27.257 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:27 smithi016 bash[75004]: audit 2023-09-23T19:48:26.273302+0000 mon.a (mon.0) 1543 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:48:27.257 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:27 smithi016 bash[75004]: cluster 2023-09-23T19:48:26.362777+0000 mgr.y (mgr.25325) 1232 : cluster 0 pgmap v1217: 89 pgs: 89 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2023-09-23T19:48:27.257 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:27 smithi016 bash[75004]: audit 2023-09-23T19:48:26.983659+0000 mon.a (mon.0) 1544 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:48:27.257 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:27 smithi016 bash[75004]: audit 2023-09-23T19:48:26.989315+0000 mon.a (mon.0) 1545 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:48:27.257 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:27 smithi016 bash[75004]: audit 2023-09-23T19:48:26.999628+0000 mon.a (mon.0) 1546 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:48:27.259 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:27.294 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:27.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:27.297 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:27.397 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:27.406 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:27 smithi092 bash[58855]: audit 2023-09-23T19:48:26.254459+0000 mon.a (mon.0) 1542 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:48:27.406 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:27 smithi092 bash[58855]: audit 2023-09-23T19:48:26.273302+0000 mon.a (mon.0) 1543 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:48:27.406 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:27 smithi092 bash[58855]: cluster 2023-09-23T19:48:26.362777+0000 mgr.y (mgr.25325) 1232 : cluster 0 pgmap v1217: 89 pgs: 89 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2023-09-23T19:48:27.407 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:27 smithi092 bash[58855]: audit 2023-09-23T19:48:26.983659+0000 mon.a (mon.0) 1544 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:48:27.407 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:27 smithi092 bash[58855]: audit 2023-09-23T19:48:26.989315+0000 mon.a (mon.0) 1545 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:48:27.407 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:27 smithi092 bash[58855]: audit 2023-09-23T19:48:26.999628+0000 mon.a (mon.0) 1546 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:48:27.431 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:27.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:27.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:48:27.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:27 smithi016 bash[77995]: audit 2023-09-23T19:48:26.254459+0000 mon.a (mon.0) 1542 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:48:27.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:27 smithi016 bash[77995]: audit 2023-09-23T19:48:26.273302+0000 mon.a (mon.0) 1543 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:48:27.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:27 smithi016 bash[77995]: cluster 2023-09-23T19:48:26.362777+0000 mgr.y (mgr.25325) 1232 : cluster 0 pgmap v1217: 89 pgs: 89 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2023-09-23T19:48:27.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:27 smithi016 bash[77995]: audit 2023-09-23T19:48:26.983659+0000 mon.a (mon.0) 1544 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:48:27.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:27 smithi016 bash[77995]: audit 2023-09-23T19:48:26.989315+0000 mon.a (mon.0) 1545 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:48:27.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:27 smithi016 bash[77995]: audit 2023-09-23T19:48:26.999628+0000 mon.a (mon.0) 1546 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:48:27.537 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:27.570 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:27.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:27.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:48:27.674 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:27.703 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:27.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:27.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:48:27.710 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:48:27.809 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:27.835 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:27.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:27.839 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:27.940 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:27.967 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:27.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:27.973 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:48:28.073 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:28.103 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:28.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:28.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:28.208 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:28.266 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:28.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:28.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:28.372 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:28.417 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:28.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:28.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:28.524 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:28.551 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:28.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:28.557 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:48:28.657 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:28.688 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:28.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:28.697 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:28.797 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:28.820 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:28.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:28.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:48:28.924 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:28.950 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:28.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:28.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:48:29.052 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:29.096 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:29.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:29.105 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:29.206 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:29.225 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:29.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:29.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:29.328 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:29.356 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:29.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:29.361 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:48:29.423 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:29 smithi016 bash[75004]: cluster 2023-09-23T19:48:28.363376+0000 mgr.y (mgr.25325) 1233 : cluster 0 pgmap v1218: 89 pgs: 89 active+clean; 5.3 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2023-09-23T19:48:29.423 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:29 smithi016 bash[75004]: audit 2023-09-23T19:48:28.669035+0000 mon.a (mon.0) 1547 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:48:29.423 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:29 smithi016 bash[77995]: cluster 2023-09-23T19:48:28.363376+0000 mgr.y (mgr.25325) 1233 : cluster 0 pgmap v1218: 89 pgs: 89 active+clean; 5.3 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2023-09-23T19:48:29.423 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:29 smithi016 bash[77995]: audit 2023-09-23T19:48:28.669035+0000 mon.a (mon.0) 1547 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:48:29.461 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:29.486 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:29.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:29.493 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:48:29.593 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:29.601 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:29 smithi092 bash[58855]: cluster 2023-09-23T19:48:28.363376+0000 mgr.y (mgr.25325) 1233 : cluster 0 pgmap v1218: 89 pgs: 89 active+clean; 5.3 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2023-09-23T19:48:29.601 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:29 smithi092 bash[58855]: audit 2023-09-23T19:48:28.669035+0000 mon.a (mon.0) 1547 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:48:29.617 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:29.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:29.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:48:29.720 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:29.740 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:48:29.746 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:29.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:29.749 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:48:29.849 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:29.884 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:29.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:29.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:48:29.988 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:30.014 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:30.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:30.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:48:30.119 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:30.149 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:30.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:30.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:48:30.255 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:30.286 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:30.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:30.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:48:30.394 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:30.433 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:30.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:30.436 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:48:30.537 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:30.566 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:30.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:30.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:30.669 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:30.695 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:30.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:30.700 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:48:30.753 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:48:30.801 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:30.826 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:30.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:30.829 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:48:30.930 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:30.963 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:30.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:30.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:48:31.071 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:31.097 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:31.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:31.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:31.201 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:31.226 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:31.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:31.231 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:31.331 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:31.359 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:31.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:31.366 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:48:31.467 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:31.498 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:31.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:31.502 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:31.602 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:31.630 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:31.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:31.635 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:48:31.736 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:31.744 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:31 smithi016 bash[75004]: cluster 2023-09-23T19:48:30.363990+0000 mgr.y (mgr.25325) 1234 : cluster 0 pgmap v1219: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2023-09-23T19:48:31.745 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:31 smithi016 bash[77995]: cluster 2023-09-23T19:48:30.363990+0000 mgr.y (mgr.25325) 1234 : cluster 0 pgmap v1219: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2023-09-23T19:48:31.765 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:31.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:31.766 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:31.866 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:31.892 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:31.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:31.896 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:48:31.997 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:32.009 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:31 smithi092 bash[58855]: cluster 2023-09-23T19:48:30.363990+0000 mgr.y (mgr.25325) 1234 : cluster 0 pgmap v1219: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2023-09-23T19:48:32.026 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:32.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:32.032 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:32.132 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:32.161 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:32.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:32.164 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:32.265 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:32.292 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:32.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:32.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:32.393 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:32.410 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:32.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:32.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:32.513 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:32.538 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:32.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:32.543 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:48:32.643 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:32.671 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:32.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:32.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:32.775 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:32.805 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:32.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:32.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:32.908 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:32.938 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:32.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:32.947 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:48:33.048 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:33.078 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:33.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:33.080 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:33.181 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:33.213 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:33.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:33.220 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:33.321 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:33.347 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:33.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:33.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:33.450 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:33.478 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:33.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:33.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:33.580 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:33.601 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:33.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:33.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:33.707 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:33.732 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:33.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:33.736 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:33.837 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:33.845 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:33 smithi092 bash[58855]: cluster 2023-09-23T19:48:32.364520+0000 mgr.y (mgr.25325) 1235 : cluster 0 pgmap v1220: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2023-09-23T19:48:33.871 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:33.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:33.874 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:48:33.974 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:33.982 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:33 smithi016 bash[75004]: cluster 2023-09-23T19:48:32.364520+0000 mgr.y (mgr.25325) 1235 : cluster 0 pgmap v1220: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2023-09-23T19:48:33.983 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:33 smithi016 bash[77995]: cluster 2023-09-23T19:48:32.364520+0000 mgr.y (mgr.25325) 1235 : cluster 0 pgmap v1220: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2023-09-23T19:48:34.004 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:34.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:34.009 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:34.110 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:34.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:34.138 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:34.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:34.239 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:34.266 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:34.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:34.269 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:34.369 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:34.401 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:34.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:34.403 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:48:34.504 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:34.529 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:34.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:34.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:34.636 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:34.667 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:34.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:34.672 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:34.773 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:34.806 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:34.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:34.810 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:34.911 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:34.937 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:34.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:34.941 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:35.041 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:35.073 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:35.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:35.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:35.177 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:35.211 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:35.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:35.217 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:35.317 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:35.341 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:35.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:35.346 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:48:35.447 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:35.487 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:35.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:35.489 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:48:35.589 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:35.615 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:35.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:35.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:35.723 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:35.749 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:35.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:35.756 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:35.857 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:35.864 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:35 smithi016 bash[77995]: cluster 2023-09-23T19:48:34.365180+0000 mgr.y (mgr.25325) 1236 : cluster 0 pgmap v1221: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2023-09-23T19:48:35.864 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:35 smithi016 bash[75004]: cluster 2023-09-23T19:48:34.365180+0000 mgr.y (mgr.25325) 1236 : cluster 0 pgmap v1221: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2023-09-23T19:48:35.864 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:48:35 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:48:35] "GET /metrics HTTP/1.1" 200 36303 "" "Prometheus/2.33.4" 2023-09-23T19:48:35.874 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:35.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:35.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:48:35.977 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:35.985 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:35 smithi092 bash[58855]: cluster 2023-09-23T19:48:34.365180+0000 mgr.y (mgr.25325) 1236 : cluster 0 pgmap v1221: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2023-09-23T19:48:36.014 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:36.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:36.022 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:48:36.123 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:36.150 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:36.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:36.153 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:36.253 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:36.279 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:36.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:36.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:36.384 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:36.435 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:36.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:36.438 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:48:36.538 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:36.572 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:36.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:36.574 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:48:36.675 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:36.703 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:36.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:36.705 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:48:36.806 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:36.833 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:36.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:36.836 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:48:36.936 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:36.954 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:36.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:36.958 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:37.059 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:37.093 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:37.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:37.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:48:37.197 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:37.219 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:37.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:37.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:37.326 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:37.349 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:37.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:37.353 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:48:37.453 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:37.477 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:37.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:37.481 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:37.582 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:37.606 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:37.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:37.609 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:48:37.710 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:37.717 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:37 smithi092 bash[58855]: cluster 2023-09-23T19:48:36.365945+0000 mgr.y (mgr.25325) 1237 : cluster 0 pgmap v1222: 89 pgs: 89 active+clean; 4.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 728 op/s 2023-09-23T19:48:37.737 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:37.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:37.744 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:48:37.845 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:37.876 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:37.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:37.884 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:37 smithi016 bash[75004]: cluster 2023-09-23T19:48:36.365945+0000 mgr.y (mgr.25325) 1237 : cluster 0 pgmap v1222: 89 pgs: 89 active+clean; 4.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 728 op/s 2023-09-23T19:48:37.884 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:37 smithi016 bash[77995]: cluster 2023-09-23T19:48:36.365945+0000 mgr.y (mgr.25325) 1237 : cluster 0 pgmap v1222: 89 pgs: 89 active+clean; 4.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 728 op/s 2023-09-23T19:48:37.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:37.985 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:38.018 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:38.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:38.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:38.126 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:38.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:38.160 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:38.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:38.261 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:38.296 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:38.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:38.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:38.398 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:38.429 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:38.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:38.436 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:38.537 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:38.568 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:38.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:38.576 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:38.677 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:38.711 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:38.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:38.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:48:38.815 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:38.836 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:38.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:38.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:38.941 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:38.965 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:38.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:38.967 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:48:39.068 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:39.097 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:39.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:39.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:39.201 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:39.225 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:39.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:39.230 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:39.331 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:39.363 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:39.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:39.366 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:48:39.467 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:39.498 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:39.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:39.504 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:39.605 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:39.642 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:39.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:39.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:39.747 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:39.771 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:39.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:39.773 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:39.874 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:39.883 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:39 smithi016 bash[75004]: cluster 2023-09-23T19:48:38.366457+0000 mgr.y (mgr.25325) 1238 : cluster 0 pgmap v1223: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 690 op/s 2023-09-23T19:48:39.883 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:39 smithi016 bash[77995]: cluster 2023-09-23T19:48:38.366457+0000 mgr.y (mgr.25325) 1238 : cluster 0 pgmap v1223: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 690 op/s 2023-09-23T19:48:39.903 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:39.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:39.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:40.010 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:40.018 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:39 smithi092 bash[58855]: cluster 2023-09-23T19:48:38.366457+0000 mgr.y (mgr.25325) 1238 : cluster 0 pgmap v1223: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 690 op/s 2023-09-23T19:48:40.031 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:40.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:40.034 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:48:40.135 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:40.161 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:40.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:40.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:48:40.265 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:40.289 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:40.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:40.296 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:40.397 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:40.422 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:40.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:40.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:48:40.526 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:40.563 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:40.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:40.566 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:40.667 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:40.689 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:40.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:40.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:48:40.796 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:40.822 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:40.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:40.832 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:48:40.933 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:40.957 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:40.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:40.962 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:48:41.063 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:41.089 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:41.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:41.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:41.197 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:41.233 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:41.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:41.235 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:48:41.336 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:41.364 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:41.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:41.367 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:48:41.468 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:41.496 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:41.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:41.502 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:48:41.603 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:41.626 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:41.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:41.629 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:41.729 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:41.738 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:41 smithi016 bash[75004]: cluster 2023-09-23T19:48:40.367082+0000 mgr.y (mgr.25325) 1239 : cluster 0 pgmap v1224: 89 pgs: 89 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2023-09-23T19:48:41.739 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:41 smithi016 bash[77995]: cluster 2023-09-23T19:48:40.367082+0000 mgr.y (mgr.25325) 1239 : cluster 0 pgmap v1224: 89 pgs: 89 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2023-09-23T19:48:41.787 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:41.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:41 smithi092 bash[58855]: cluster 2023-09-23T19:48:40.367082+0000 mgr.y (mgr.25325) 1239 : cluster 0 pgmap v1224: 89 pgs: 89 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2023-09-23T19:48:41.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:41.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:48:41.892 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:41.916 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:41.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:41.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:42.020 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:42.049 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:42.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:42.052 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:42.153 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:42.188 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:42.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:42.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:42.295 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:42.322 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:42.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:42.324 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:48:42.425 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:42.451 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:42.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:42.454 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:48:42.555 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:42.592 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:42.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:42.599 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:42.699 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:42.730 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:42.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:42.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:48:42.834 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:42.864 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:42.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:42.872 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:42.973 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:43.007 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:43.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:43.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:48:43.111 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:43.145 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:43.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:43.148 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:48:43.249 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:43.279 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:43.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:43.280 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:43.381 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:43.413 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:43.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:43.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:43.519 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:43.560 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:43.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:43.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:48:43.642 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:43 smithi016 bash[75004]: cluster 2023-09-23T19:48:42.367664+0000 mgr.y (mgr.25325) 1240 : cluster 0 pgmap v1225: 89 pgs: 89 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 654 op/s 2023-09-23T19:48:43.643 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:43 smithi016 bash[77995]: cluster 2023-09-23T19:48:42.367664+0000 mgr.y (mgr.25325) 1240 : cluster 0 pgmap v1225: 89 pgs: 89 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 654 op/s 2023-09-23T19:48:43.663 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:43.686 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:43.687 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-09-23T19:48:43.687 INFO:teuthology.orchestra.run.smithi016.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T19:48:43.687 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-09-23T19:48:43.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:43.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:48:43.790 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:43.822 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:43.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:43.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:48:43.925 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:43.935 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:43 smithi092 bash[58855]: cluster 2023-09-23T19:48:42.367664+0000 mgr.y (mgr.25325) 1240 : cluster 0 pgmap v1225: 89 pgs: 89 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 654 op/s 2023-09-23T19:48:43.949 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:43.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:43.956 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:48:44.057 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:44.095 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:44.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:44.102 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:44.202 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:44.231 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:44.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:44.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:44.337 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:44.373 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:44.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:44.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:44.480 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:44.508 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:44.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:44.511 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:44.612 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:44.633 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:44.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:44.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:44.739 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:44.760 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:44.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:44.764 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:48:44.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:44 smithi016 bash[75004]: audit 2023-09-23T19:48:43.669116+0000 mon.a (mon.0) 1548 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:48:44.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:44 smithi016 bash[77995]: audit 2023-09-23T19:48:43.669116+0000 mon.a (mon.0) 1548 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:48:44.865 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:44.891 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:44.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:44.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:48:44.998 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:45.007 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:44 smithi092 bash[58855]: audit 2023-09-23T19:48:43.669116+0000 mon.a (mon.0) 1548 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:48:45.030 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:45.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:45.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:45.134 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:45.168 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:45.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:45.176 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:45.277 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:45.304 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:45.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:45.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:48:45.318 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-09-23T19:48:45.318 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:48:45.410 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:45.443 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:45.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:45.449 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:48:45.550 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:45.583 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:45.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:45.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:45.690 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:45.721 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:45.725 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:45 smithi016 bash[75004]: cluster 2023-09-23T19:48:44.368148+0000 mgr.y (mgr.25325) 1241 : cluster 0 pgmap v1226: 89 pgs: 89 active+clean; 4.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 633 op/s 2023-09-23T19:48:45.725 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:45 smithi016 bash[77995]: cluster 2023-09-23T19:48:44.368148+0000 mgr.y (mgr.25325) 1241 : cluster 0 pgmap v1226: 89 pgs: 89 active+clean; 4.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 633 op/s 2023-09-23T19:48:45.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:45.726 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:45.727 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:48:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:48:45] "GET /metrics HTTP/1.1" 200 36306 "" "Prometheus/2.33.4" 2023-09-23T19:48:45.827 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:45.861 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:45.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:45.864 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:45.964 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:45.972 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:45 smithi092 bash[58855]: cluster 2023-09-23T19:48:44.368148+0000 mgr.y (mgr.25325) 1241 : cluster 0 pgmap v1226: 89 pgs: 89 active+clean; 4.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 633 op/s 2023-09-23T19:48:45.991 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:45.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:45.994 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:48:46.095 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:46.119 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:46.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:46.122 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:48:46.222 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:46.251 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:46.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:46.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:46.356 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:46.384 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:46.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:46.386 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:46.487 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:46.519 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:46.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:46.522 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:46.622 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:46.648 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:46.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:46.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:48:46.753 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:46.782 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:46.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:46.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:48:46.886 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:46.909 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:46.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:46.914 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:48:47.014 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:47.046 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:47.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:47.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:47.157 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:47.184 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:47.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:47.189 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:47.290 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:47.313 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:47.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:47.320 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:48:47.420 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:47.457 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:47.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:47.462 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:47.562 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:47.592 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:47.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:47.593 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:48:47.660 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:47 smithi016 bash[75004]: cluster 2023-09-23T19:48:46.372874+0000 mgr.y (mgr.25325) 1242 : cluster 0 pgmap v1227: 89 pgs: 89 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 722 op/s 2023-09-23T19:48:47.694 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:47.725 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:47.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:47.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:47.829 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:47.863 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:47.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:47.873 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:47.974 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:47.983 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:47 smithi016 bash[77995]: cluster 2023-09-23T19:48:46.372874+0000 mgr.y (mgr.25325) 1242 : cluster 0 pgmap v1227: 89 pgs: 89 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 722 op/s 2023-09-23T19:48:48.006 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:48.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:48.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:48.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:47 smithi092 bash[58855]: cluster 2023-09-23T19:48:46.372874+0000 mgr.y (mgr.25325) 1242 : cluster 0 pgmap v1227: 89 pgs: 89 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 722 op/s 2023-09-23T19:48:48.109 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:48.133 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:48.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:48.135 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:48:48.236 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:48.258 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:48.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:48.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:48:48.361 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:48.391 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:48.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:48.394 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:48:48.494 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:48.523 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:48.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:48.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:48.632 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:48.654 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:48.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:48.664 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:48.765 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:48.797 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:48.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:48.804 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:48:48.905 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:48.934 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:48.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:48.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:48:49.041 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:49.069 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:49.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:49.074 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:49.175 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:49.200 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:49.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:49.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:48:49.307 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:49.349 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:49.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:49.354 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:49.437 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T19:48:49.455 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:49.526 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:49.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:49.532 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:48:49.632 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:49.661 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:49.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:49.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:49.734 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:49 smithi016 bash[75004]: cluster 2023-09-23T19:48:48.373390+0000 mgr.y (mgr.25325) 1243 : cluster 0 pgmap v1228: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2023-09-23T19:48:49.734 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:49 smithi016 bash[77995]: cluster 2023-09-23T19:48:48.373390+0000 mgr.y (mgr.25325) 1243 : cluster 0 pgmap v1228: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2023-09-23T19:48:49.763 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:49.794 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:49.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:49.796 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:48:49.897 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:49.915 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:48:49.930 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:49.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:49.935 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:49.962 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:48:50.030 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:48:50.036 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:50.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:49 smithi092 bash[58855]: cluster 2023-09-23T19:48:48.373390+0000 mgr.y (mgr.25325) 1243 : cluster 0 pgmap v1228: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2023-09-23T19:48:50.065 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:50.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:50.072 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:48:50.084 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:48:50.123 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:48:50.169 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:48:50.172 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:50.208 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:48:50.214 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:50.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:50.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:50.255 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:48:50.260 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:48:50.260 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:48:50.260 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:48:50.260 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:48:50.260 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:48:50.260 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:48:50.261 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:48:50.261 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:48:50.318 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:50.342 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:50.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:50.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:48:50.446 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:50.485 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:50.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:50.491 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:50.591 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:50.618 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:50.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:50.625 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:50.699 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is unset 2023-09-23T19:48:50.725 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:50.734 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:50 smithi016 bash[77995]: audit 2023-09-23T19:48:49.852614+0000 mon.c (mon.2) 197 : audit 0 from='client.? 172.21.15.16:0/462807486' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:48:50.734 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:50 smithi016 bash[77995]: audit 2023-09-23T19:48:50.366288+0000 mon.c (mon.2) 198 : audit 1 from='client.? 172.21.15.16:0/2220387832' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:48:50.734 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:50 smithi016 bash[77995]: audit 2023-09-23T19:48:50.366780+0000 mon.a (mon.0) 1549 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:48:50.735 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:50 smithi016 bash[75004]: audit 2023-09-23T19:48:49.852614+0000 mon.c (mon.2) 197 : audit 0 from='client.? 172.21.15.16:0/462807486' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:48:50.735 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:50 smithi016 bash[75004]: audit 2023-09-23T19:48:50.366288+0000 mon.c (mon.2) 198 : audit 1 from='client.? 172.21.15.16:0/2220387832' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:48:50.735 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:50 smithi016 bash[75004]: audit 2023-09-23T19:48:50.366780+0000 mon.a (mon.0) 1549 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:48:50.754 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:50.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:50.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:50.862 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:50.887 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:50.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:50.893 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:48:50.994 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:51.029 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:51.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:51.036 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:51.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:50 smithi092 bash[58855]: audit 2023-09-23T19:48:49.852614+0000 mon.c (mon.2) 197 : audit 0 from='client.? 172.21.15.16:0/462807486' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:48:51.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:50 smithi092 bash[58855]: audit 2023-09-23T19:48:50.366288+0000 mon.c (mon.2) 198 : audit 1 from='client.? 172.21.15.16:0/2220387832' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:48:51.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:50 smithi092 bash[58855]: audit 2023-09-23T19:48:50.366780+0000 mon.a (mon.0) 1549 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:48:51.137 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:51.165 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:51.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:51.170 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:48:51.271 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:51.299 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:51.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:51.302 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:51.403 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:51.431 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:51.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:51.436 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:48:51.537 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:51.567 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:51.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:51.574 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:48:51.675 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:51.700 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:51.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:51.703 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:48:51.804 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:51.833 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:51.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:51.837 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:48:51.937 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:51.945 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:51 smithi016 bash[75004]: cluster 2023-09-23T19:48:50.374183+0000 mgr.y (mgr.25325) 1244 : cluster 0 pgmap v1229: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2023-09-23T19:48:51.945 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:51 smithi016 bash[75004]: cluster 2023-09-23T19:48:50.662574+0000 mon.a (mon.0) 1550 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:48:51.945 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:51 smithi016 bash[75004]: audit 2023-09-23T19:48:50.687127+0000 mon.a (mon.0) 1551 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:48:51.946 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:51 smithi016 bash[75004]: cluster 2023-09-23T19:48:50.687334+0000 mon.a (mon.0) 1552 : cluster 0 osdmap e219: 8 total, 8 up, 8 in 2023-09-23T19:48:51.946 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:51 smithi016 bash[75004]: cluster 2023-09-23T19:48:50.688761+0000 osd.4 (osd.4) 161 : cluster 3 failed to encode map e219 with expected crc 2023-09-23T19:48:51.946 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:51 smithi016 bash[77995]: cluster 2023-09-23T19:48:50.374183+0000 mgr.y (mgr.25325) 1244 : cluster 0 pgmap v1229: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2023-09-23T19:48:51.946 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:51 smithi016 bash[77995]: cluster 2023-09-23T19:48:50.662574+0000 mon.a (mon.0) 1550 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:48:51.946 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:51 smithi016 bash[77995]: audit 2023-09-23T19:48:50.687127+0000 mon.a (mon.0) 1551 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:48:51.947 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:51 smithi016 bash[77995]: cluster 2023-09-23T19:48:50.687334+0000 mon.a (mon.0) 1552 : cluster 0 osdmap e219: 8 total, 8 up, 8 in 2023-09-23T19:48:51.947 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:51 smithi016 bash[77995]: cluster 2023-09-23T19:48:50.688761+0000 osd.4 (osd.4) 161 : cluster 3 failed to encode map e219 with expected crc 2023-09-23T19:48:51.970 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:51.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:51.974 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:52.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:51 smithi092 bash[58855]: cluster 2023-09-23T19:48:50.374183+0000 mgr.y (mgr.25325) 1244 : cluster 0 pgmap v1229: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2023-09-23T19:48:52.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:51 smithi092 bash[58855]: cluster 2023-09-23T19:48:50.662574+0000 mon.a (mon.0) 1550 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:48:52.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:51 smithi092 bash[58855]: audit 2023-09-23T19:48:50.687127+0000 mon.a (mon.0) 1551 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:48:52.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:51 smithi092 bash[58855]: cluster 2023-09-23T19:48:50.687334+0000 mon.a (mon.0) 1552 : cluster 0 osdmap e219: 8 total, 8 up, 8 in 2023-09-23T19:48:52.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:51 smithi092 bash[58855]: cluster 2023-09-23T19:48:50.688761+0000 osd.4 (osd.4) 161 : cluster 3 failed to encode map e219 with expected crc 2023-09-23T19:48:52.075 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:52.105 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:52.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:52.108 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:48:52.208 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:52.230 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:52.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:52.232 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:52.334 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:52.359 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:52.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:52.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:52.468 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:52.495 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:52.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:52.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:48:52.601 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:52.623 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:52.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:52.630 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:48:52.730 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:52.752 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:52.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:52.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:52.859 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:52.886 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:52.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:52.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:48:52.992 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:53.037 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:53.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:53.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:48:53.142 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:53.165 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:53.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:53.169 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:48:53.269 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:53.299 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:53.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:53.301 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:53.402 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:53.443 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:53.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:53.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:53.551 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:53.587 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:53.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:53.594 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:48:53.689 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:53 smithi016 bash[75004]: cluster 2023-09-23T19:48:52.374725+0000 mgr.y (mgr.25325) 1245 : cluster 0 pgmap v1231: 89 pgs: 89 active+clean; 4.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 670 op/s 2023-09-23T19:48:53.695 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:53.722 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:53.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:53.726 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:48:53.826 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:53.835 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:53 smithi092 bash[58855]: cluster 2023-09-23T19:48:52.374725+0000 mgr.y (mgr.25325) 1245 : cluster 0 pgmap v1231: 89 pgs: 89 active+clean; 4.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 670 op/s 2023-09-23T19:48:53.854 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:53.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:53.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:48:53.957 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:53.984 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:53.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:53.987 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:48:54.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:53 smithi016 bash[77995]: cluster 2023-09-23T19:48:52.374725+0000 mgr.y (mgr.25325) 1245 : cluster 0 pgmap v1231: 89 pgs: 89 active+clean; 4.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 670 op/s 2023-09-23T19:48:54.088 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:54.114 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:54.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:54.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:48:54.223 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:54.257 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:54.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:54.260 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:54.261 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-09-23T19:48:54.261 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T19:48:54.277 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:48:54.361 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:54.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:54.400 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:54.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:54.501 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:54.530 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:54.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:54.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:48:54.633 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:54.698 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:54.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:54.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:54.804 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:54.834 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:54.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:54.841 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:54.917 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T19:48:54.942 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:55.009 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:55.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:55.017 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:55.117 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:55.152 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:55.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:55.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:48:55.258 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:55.286 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:55.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:55.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:48:55.391 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:55.429 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:55.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:55.431 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:48:55.532 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:55.577 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:55.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:55.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:55.681 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:55.690 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:48:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:48:55] "GET /metrics HTTP/1.1" 200 36306 "" "Prometheus/2.33.4" 2023-09-23T19:48:55.730 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:55.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:55.737 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:48:55.838 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:55.860 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:55.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:55.866 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:55.967 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:55.977 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:55 smithi092 bash[58855]: cluster 2023-09-23T19:48:54.375209+0000 mgr.y (mgr.25325) 1246 : cluster 0 pgmap v1232: 89 pgs: 89 active+clean; 4.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 670 op/s 2023-09-23T19:48:56.003 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:56.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:56.008 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:56.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:55 smithi016 bash[77995]: cluster 2023-09-23T19:48:54.375209+0000 mgr.y (mgr.25325) 1246 : cluster 0 pgmap v1232: 89 pgs: 89 active+clean; 4.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 670 op/s 2023-09-23T19:48:56.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:55 smithi016 bash[75004]: cluster 2023-09-23T19:48:54.375209+0000 mgr.y (mgr.25325) 1246 : cluster 0 pgmap v1232: 89 pgs: 89 active+clean; 4.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 670 op/s 2023-09-23T19:48:56.109 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:56.143 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:56.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:56.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:56.247 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:56.281 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:56.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:56.283 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:56.384 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:56.414 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:56.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:56.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:48:56.442 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:48:56.517 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:56.540 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:56.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:56.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:48:56.646 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:56.682 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:56.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:56.685 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:56.786 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:56.824 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:56.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:56.828 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:48:56.929 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:56.968 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:56.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:56.972 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:57.073 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:57.104 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:57.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:57.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:48:57.207 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:57.234 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:48:57.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:57.240 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:48:57.340 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:57.371 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:48:57.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:57.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:48:57.476 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:57.506 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:57.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:57.512 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:48:57.612 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:57.637 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:57.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:57.641 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:48:57.742 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:57.785 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:57.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:57.790 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:48:57.891 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:57.900 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:57 smithi016 bash[75004]: cluster 2023-09-23T19:48:56.375988+0000 mgr.y (mgr.25325) 1247 : cluster 0 pgmap v1233: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 668 op/s 2023-09-23T19:48:57.900 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:57 smithi016 bash[77995]: cluster 2023-09-23T19:48:56.375988+0000 mgr.y (mgr.25325) 1247 : cluster 0 pgmap v1233: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 668 op/s 2023-09-23T19:48:57.930 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:57.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:57.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:58.035 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:58.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:57 smithi092 bash[58855]: cluster 2023-09-23T19:48:56.375988+0000 mgr.y (mgr.25325) 1247 : cluster 0 pgmap v1233: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 668 op/s 2023-09-23T19:48:58.060 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:58.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:58.065 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:48:58.166 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:58.207 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:58.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:58.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:58.313 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:58.337 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:58.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:58.341 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:48:58.441 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:58.478 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:48:58.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:58.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:48:58.488 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:48:58.488 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:48:58.584 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:58.607 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:58.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:58.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:58.709 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:58 smithi016 bash[75004]: audit 2023-09-23T19:48:58.669643+0000 mon.a (mon.0) 1553 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:48:58.713 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:58.742 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:48:58.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:58.745 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:48:58.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:58 smithi016 bash[77995]: audit 2023-09-23T19:48:58.669643+0000 mon.a (mon.0) 1553 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:48:58.846 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:58.855 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:58 smithi092 bash[58855]: audit 2023-09-23T19:48:58.669643+0000 mon.a (mon.0) 1553 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:48:58.873 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:58.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:58.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:58.979 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:59.007 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:59.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:59.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:59.113 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:59.150 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:59.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:59.153 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:59.254 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:59.280 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:59.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:59.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:59.383 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:59.409 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:48:59.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:59.411 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:48:59.512 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:59.557 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:59.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:59.560 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:48:59.661 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:59.682 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:48:59.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:59.684 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:48:59.784 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:59.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:48:59 smithi016 bash[75004]: cluster 2023-09-23T19:48:58.376519+0000 mgr.y (mgr.25325) 1248 : cluster 0 pgmap v1234: 89 pgs: 89 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 665 op/s 2023-09-23T19:48:59.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:48:59 smithi016 bash[77995]: cluster 2023-09-23T19:48:58.376519+0000 mgr.y (mgr.25325) 1248 : cluster 0 pgmap v1234: 89 pgs: 89 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 665 op/s 2023-09-23T19:48:59.823 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:48:59.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:59.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:48:59.926 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:59.965 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:48:59.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:48:59.970 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:00.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:48:59 smithi092 bash[58855]: cluster 2023-09-23T19:48:58.376519+0000 mgr.y (mgr.25325) 1248 : cluster 0 pgmap v1234: 89 pgs: 89 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 665 op/s 2023-09-23T19:49:00.071 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:00.095 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:00.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:00.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:00.198 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:00.231 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:00.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:00.233 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:00.334 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:00.364 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:00.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:00.366 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:00.467 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:00.493 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:00.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:00.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:49:00.599 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:00.626 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:00.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:00.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:00.728 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:00.755 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:00.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:00.760 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:49:00.860 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:00.898 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:00.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:00.901 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:49:01.001 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:01.029 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:01.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:01.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:01.132 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:01.154 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:01.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:01.156 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:49:01.257 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:01.300 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:01.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:01.303 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:01.404 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:01.433 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:01.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:01.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:49:01.541 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:01.580 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:01.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:01.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:49:01.690 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:01.714 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:01 smithi016 bash[75004]: cluster 2023-09-23T19:49:00.377366+0000 mgr.y (mgr.25325) 1249 : cluster 0 pgmap v1235: 89 pgs: 89 active+clean; 4.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 658 op/s 2023-09-23T19:49:01.714 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:01 smithi016 bash[77995]: cluster 2023-09-23T19:49:00.377366+0000 mgr.y (mgr.25325) 1249 : cluster 0 pgmap v1235: 89 pgs: 89 active+clean; 4.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 658 op/s 2023-09-23T19:49:01.719 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:01.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:01.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:49:01.825 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:01.861 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:01.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:01.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:01.937 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:49:01.965 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:01.999 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:02.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:02.007 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:49:02.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:01 smithi092 bash[58855]: cluster 2023-09-23T19:49:00.377366+0000 mgr.y (mgr.25325) 1249 : cluster 0 pgmap v1235: 89 pgs: 89 active+clean; 4.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 658 op/s 2023-09-23T19:49:02.108 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:02.126 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:02.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:02.128 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:02.229 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:02.261 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:02.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:02.265 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:49:02.365 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:02.390 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:02.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:02.395 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:02.496 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:02.529 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:02.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:02.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:49:02.632 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:02.666 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:02.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:02.670 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:02.770 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:02.795 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:02.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:02.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:49:02.901 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:02.934 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:02.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:02.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:03.038 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:03.062 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:03.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:03.068 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:49:03.169 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:03.194 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:03.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:03.197 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:49:03.298 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:03.315 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:03.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:03.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:03.418 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:03.442 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:03.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:03.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:03.545 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:03.569 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:03.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:03.572 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:03.672 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:03.705 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:03.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:03.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:49:03.811 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:03.821 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:03 smithi092 bash[58855]: cluster 2023-09-23T19:49:02.378036+0000 mgr.y (mgr.25325) 1250 : cluster 0 pgmap v1236: 89 pgs: 89 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 649 op/s 2023-09-23T19:49:03.847 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:03.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:03.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:03.951 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:03.957 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:03 smithi016 bash[77995]: cluster 2023-09-23T19:49:02.378036+0000 mgr.y (mgr.25325) 1250 : cluster 0 pgmap v1236: 89 pgs: 89 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 649 op/s 2023-09-23T19:49:03.958 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:03 smithi016 bash[75004]: cluster 2023-09-23T19:49:02.378036+0000 mgr.y (mgr.25325) 1250 : cluster 0 pgmap v1236: 89 pgs: 89 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 649 op/s 2023-09-23T19:49:03.979 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:03.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:03.981 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:49:04.082 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:04.113 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:04.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:04.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:04.222 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:04.257 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:04.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:04.263 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:04.363 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:04.397 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:04.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:04.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:49:04.508 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:04.536 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:04.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:04.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:49:04.641 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:04.678 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:04.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:04.682 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:49:04.783 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:04.808 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:04.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:04.814 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:49:04.915 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:04.948 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:04.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:04.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:05.052 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:05.088 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:05.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:05.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:05.192 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:05.220 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:05.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:05.225 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:05.326 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:05.348 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:05.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:05.352 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:05.453 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:05.475 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:05.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:05.477 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:05.578 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:05.611 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:05.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:05.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:05.718 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:05.745 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:05.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:05.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:05.816 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:05 smithi016 bash[75004]: cluster 2023-09-23T19:49:04.378481+0000 mgr.y (mgr.25325) 1251 : cluster 0 pgmap v1237: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 622 op/s 2023-09-23T19:49:05.817 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:49:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:49:05] "GET /metrics HTTP/1.1" 200 36306 "" "Prometheus/2.33.4" 2023-09-23T19:49:05.817 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:05 smithi016 bash[77995]: cluster 2023-09-23T19:49:04.378481+0000 mgr.y (mgr.25325) 1251 : cluster 0 pgmap v1237: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 622 op/s 2023-09-23T19:49:05.850 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:05.887 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:05.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:05.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:49:05.991 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:05.999 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:05 smithi092 bash[58855]: cluster 2023-09-23T19:49:04.378481+0000 mgr.y (mgr.25325) 1251 : cluster 0 pgmap v1237: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 622 op/s 2023-09-23T19:49:06.024 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:06.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:06.026 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:06.127 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:06.163 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:06.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:06.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:49:06.266 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:06.288 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:06.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:06.292 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:06.393 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:06.423 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:06.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:06.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:06.527 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:06.561 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:06.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:06.563 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:06.664 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:06.690 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:06.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:06.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:49:06.796 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:06.827 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:06.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:06.833 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:06.934 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:06.965 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:06.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:06.967 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:07.068 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:07.098 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:07.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:07.102 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:49:07.203 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:07.232 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:07.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:07.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:07.337 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:07.377 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:07.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:07.379 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:49:07.480 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:07.506 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:07.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:07.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:49:07.611 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:07.637 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:07.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:07.642 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:07.743 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:07.767 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:07.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:07.773 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:07.874 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:07.893 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:07 smithi016 bash[75004]: cluster 2023-09-23T19:49:06.379227+0000 mgr.y (mgr.25325) 1252 : cluster 0 pgmap v1238: 89 pgs: 89 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 691 op/s 2023-09-23T19:49:07.893 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:07 smithi016 bash[77995]: cluster 2023-09-23T19:49:06.379227+0000 mgr.y (mgr.25325) 1252 : cluster 0 pgmap v1238: 89 pgs: 89 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 691 op/s 2023-09-23T19:49:07.910 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:07.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:07.915 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:49:08.016 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:08.024 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:07 smithi092 bash[58855]: cluster 2023-09-23T19:49:06.379227+0000 mgr.y (mgr.25325) 1252 : cluster 0 pgmap v1238: 89 pgs: 89 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 691 op/s 2023-09-23T19:49:08.042 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:08.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:08.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:08.145 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:08.174 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:08.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:08.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:08.277 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:08.309 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:08.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:08.312 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:08.413 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:08.437 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:08.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:08.440 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:08.467 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-09-23T19:49:08.467 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [ 2023-09-23T19:49:08.467 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:08.468 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.36480.0:146972 12.c 12:333799ee:::benchmark_data_smithi016_181952_object44291:head [delete] snapc 0=[] ondisk+write+known_if_redirected e219)", 2023-09-23T19:49:08.468 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:49:08.446131+0000", 2023-09-23T19:49:08.468 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 0.016152805999999999, 2023-09-23T19:49:08.468 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.016177674, 2023-09-23T19:49:08.468 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:49:08.468 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "waiting for sub ops", 2023-09-23T19:49:08.468 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:49:08.468 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.36480", 2023-09-23T19:49:08.468 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1035295215", 2023-09-23T19:49:08.468 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 146972 2023-09-23T19:49:08.468 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:08.469 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:49:08.469 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:08.469 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:49:08.469 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:08.446131+0000", 2023-09-23T19:49:08.469 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:49:08.469 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:08.469 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:08.469 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:49:08.469 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:08.446131+0000", 2023-09-23T19:49:08.469 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:49:08.469 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:08.470 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:08.470 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:49:08.470 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:08.446130+0000", 2023-09-23T19:49:08.470 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-09-23T19:49:08.470 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:08.470 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:08.470 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:49:08.470 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:08.446137+0000", 2023-09-23T19:49:08.470 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.99e-06 2023-09-23T19:49:08.470 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:08.470 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:08.470 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:49:08.471 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:08.446149+0000", 2023-09-23T19:49:08.471 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2145e-05 2023-09-23T19:49:08.471 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:08.471 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:08.471 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:49:08.471 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:08.446156+0000", 2023-09-23T19:49:08.471 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.503e-06 2023-09-23T19:49:08.471 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:08.471 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:08.471 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:49:08.471 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:08.446297+0000", 2023-09-23T19:49:08.472 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00014092 2023-09-23T19:49:08.472 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:08.472 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:08.472 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:49:08.472 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:08.446608+0000", 2023-09-23T19:49:08.472 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00031055999999999998 2023-09-23T19:49:08.472 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:08.472 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:08.472 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 2,5", 2023-09-23T19:49:08.472 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:08.446961+0000", 2023-09-23T19:49:08.472 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00035355400000000001 2023-09-23T19:49:08.473 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:08.473 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:08.473 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:49:08.473 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:08.456170+0000", 2023-09-23T19:49:08.473 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0092085120000000003 2023-09-23T19:49:08.473 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:08.473 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:08.473 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:49:08.473 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:08.456178+0000", 2023-09-23T19:49:08.473 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.5080000000000005e-06 2023-09-23T19:49:08.473 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:49:08.473 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:49:08.474 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:49:08.474 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:08.474 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:08.474 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.36480.0:146973 12.3 12:cdacf72d:::benchmark_data_smithi016_181952_object44292:head [delete] snapc 0=[] ondisk+write+known_if_redirected e219)", 2023-09-23T19:49:08.474 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:49:08.451204+0000", 2023-09-23T19:49:08.474 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 0.011079706999999999, 2023-09-23T19:49:08.474 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.011164556000000001, 2023-09-23T19:49:08.474 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:49:08.474 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "waiting for sub ops", 2023-09-23T19:49:08.474 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:49:08.474 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.36480", 2023-09-23T19:49:08.474 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1035295215", 2023-09-23T19:49:08.475 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 146973 2023-09-23T19:49:08.475 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:08.475 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:49:08.475 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:08.475 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:49:08.475 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:08.451204+0000", 2023-09-23T19:49:08.475 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:49:08.475 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:08.475 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:08.475 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:49:08.475 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:08.451204+0000", 2023-09-23T19:49:08.476 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:49:08.476 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:08.476 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:08.476 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:49:08.476 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:08.451202+0000", 2023-09-23T19:49:08.476 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-09-23T19:49:08.476 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:08.476 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:08.476 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:49:08.476 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:08.451210+0000", 2023-09-23T19:49:08.476 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.0129999999999993e-06 2023-09-23T19:49:08.476 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:08.477 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:08.477 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:49:08.477 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:08.451224+0000", 2023-09-23T19:49:08.477 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3627e-05 2023-09-23T19:49:08.477 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:08.477 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:08.477 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:49:08.477 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:08.451235+0000", 2023-09-23T19:49:08.477 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1742e-05 2023-09-23T19:49:08.477 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:08.477 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:08.478 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:49:08.478 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:08.451413+0000", 2023-09-23T19:49:08.478 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00017741999999999999 2023-09-23T19:49:08.478 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:08.478 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:08.478 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:49:08.478 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:08.451673+0000", 2023-09-23T19:49:08.478 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000260208 2023-09-23T19:49:08.478 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:08.479 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:08.479 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 3,7", 2023-09-23T19:49:08.479 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:08.451934+0000", 2023-09-23T19:49:08.479 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00026068 2023-09-23T19:49:08.479 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:49:08.479 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:49:08.479 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:49:08.479 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:08.479 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:08.479 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.36480.0:146981 12.a e219/207 12:529ee975:::benchmark_data_smithi016_181952_object44300:head v 219'9102, mlcod=219'9100)", 2023-09-23T19:49:08.479 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:49:08.460317+0000", 2023-09-23T19:49:08.479 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 0.001966597, 2023-09-23T19:49:08.480 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.002117575, 2023-09-23T19:49:08.480 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:49:08.480 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "started", 2023-09-23T19:49:08.480 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:49:08.480 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:08.480 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:49:08.480 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:08.460317+0000", 2023-09-23T19:49:08.480 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:49:08.480 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:08.480 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:08.480 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:49:08.481 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:08.460317+0000", 2023-09-23T19:49:08.481 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:49:08.481 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:08.481 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:08.481 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:49:08.481 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:08.460316+0000", 2023-09-23T19:49:08.481 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:49:08.481 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:08.481 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:08.481 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:49:08.481 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:08.460326+0000", 2023-09-23T19:49:08.481 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0016e-05 2023-09-23T19:49:08.482 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:08.482 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:08.482 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:49:08.482 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:08.460343+0000", 2023-09-23T19:49:08.482 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.6974999999999999e-05 2023-09-23T19:49:08.482 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:08.482 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:08.482 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:49:08.482 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:08.460353+0000", 2023-09-23T19:49:08.482 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.8169999999999994e-06 2023-09-23T19:49:08.482 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:08.482 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:08.483 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:49:08.483 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:08.460433+0000", 2023-09-23T19:49:08.483 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.0141000000000004e-05 2023-09-23T19:49:08.483 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:08.483 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:08.483 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:49:08.483 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:08.460459+0000", 2023-09-23T19:49:08.483 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.5992000000000001e-05 2023-09-23T19:49:08.483 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:49:08.483 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:49:08.483 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:49:08.483 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:49:08.483 INFO:teuthology.orchestra.run.smithi016.stdout: ], 2023-09-23T19:49:08.484 INFO:teuthology.orchestra.run.smithi016.stdout: "num_ops": 3 2023-09-23T19:49:08.484 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-09-23T19:49:08.541 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:08.566 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:08.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:08.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:08.677 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:08.715 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:08.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:08.721 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:08.821 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:08.853 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:08.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:08.856 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:49:08.956 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:08.981 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:08.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:08.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:09.085 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:09.108 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:09.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:09.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:09.213 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:09.252 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:09.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:09.261 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:49:09.362 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:09.388 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:09.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:09.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:49:09.492 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:09.518 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:09.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:09.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:09.621 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:09.648 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:09.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:09.652 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:09.730 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:09 smithi016 bash[75004]: cluster 2023-09-23T19:49:08.379722+0000 mgr.y (mgr.25325) 1253 : cluster 0 pgmap v1239: 89 pgs: 89 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 585 op/s 2023-09-23T19:49:09.753 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:09.790 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:09.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:09.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:09.897 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:09.906 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:09 smithi092 bash[58855]: cluster 2023-09-23T19:49:08.379722+0000 mgr.y (mgr.25325) 1253 : cluster 0 pgmap v1239: 89 pgs: 89 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 585 op/s 2023-09-23T19:49:09.925 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:09.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:09.927 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:10.028 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:10.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:09 smithi016 bash[77995]: cluster 2023-09-23T19:49:08.379722+0000 mgr.y (mgr.25325) 1253 : cluster 0 pgmap v1239: 89 pgs: 89 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 585 op/s 2023-09-23T19:49:10.067 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:10.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:10.070 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:10.171 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:10.196 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:10.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:10.198 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:10.299 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:10.323 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:10.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:10.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:10.430 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:10.458 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:10.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:10.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:49:10.564 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:10.591 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:10.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:10.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:49:10.698 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:10.733 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:10.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:10.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:10.842 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:10.874 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:10.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:10.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:10.979 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:11.006 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:11.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:11.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:49:11.111 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:11.146 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:11.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:11.150 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:11.251 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:11.287 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:11.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:11.291 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:11.392 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:11.418 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:11.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:11.420 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:11.521 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:11.542 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:11.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:11.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:49:11.645 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:11.681 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:11.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:11.685 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:11.749 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T19:49:11.785 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:11.814 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:11.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:11.820 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:49:11.907 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:11 smithi016 bash[77995]: cluster 2023-09-23T19:49:10.380459+0000 mgr.y (mgr.25325) 1254 : cluster 0 pgmap v1240: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 647 op/s 2023-09-23T19:49:11.908 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:11 smithi016 bash[75004]: cluster 2023-09-23T19:49:10.380459+0000 mgr.y (mgr.25325) 1254 : cluster 0 pgmap v1240: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 647 op/s 2023-09-23T19:49:11.921 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:11.955 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:11.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:11.958 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:12.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:11 smithi092 bash[58855]: cluster 2023-09-23T19:49:10.380459+0000 mgr.y (mgr.25325) 1254 : cluster 0 pgmap v1240: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 647 op/s 2023-09-23T19:49:12.059 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:12.091 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:12.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:12.101 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:49:12.202 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:12.224 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:12.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:12.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:12.331 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:12.351 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:12.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:12.357 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:49:12.457 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:12.492 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:12.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:12.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:49:12.595 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:12.623 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:12.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:12.626 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:12.726 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:12.752 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:12.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:12.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:49:12.855 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:12.893 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:12.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:12.901 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:13.001 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:13.025 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:13.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:13.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:49:13.129 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:13.164 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:13.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:13.173 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:49:13.274 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:13.305 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:13.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:13.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:49:13.414 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:13.438 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:13.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:13.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:13.543 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:13.567 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:13.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:13.570 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:13.670 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:13.697 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:13.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:13.701 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:13.802 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:13.849 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:13.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:13.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:13.955 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:13.960 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:13 smithi016 bash[75004]: cluster 2023-09-23T19:49:12.381003+0000 mgr.y (mgr.25325) 1255 : cluster 0 pgmap v1241: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 617 op/s 2023-09-23T19:49:13.960 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:13 smithi016 bash[75004]: audit 2023-09-23T19:49:13.669918+0000 mon.a (mon.0) 1554 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:49:13.961 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:13 smithi016 bash[77995]: cluster 2023-09-23T19:49:12.381003+0000 mgr.y (mgr.25325) 1255 : cluster 0 pgmap v1241: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 617 op/s 2023-09-23T19:49:13.961 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:13 smithi016 bash[77995]: audit 2023-09-23T19:49:13.669918+0000 mon.a (mon.0) 1554 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:49:13.964 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:13 smithi092 bash[58855]: cluster 2023-09-23T19:49:12.381003+0000 mgr.y (mgr.25325) 1255 : cluster 0 pgmap v1241: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 617 op/s 2023-09-23T19:49:13.965 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:13 smithi092 bash[58855]: audit 2023-09-23T19:49:13.669918+0000 mon.a (mon.0) 1554 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:49:13.982 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:13.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:13.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:14.088 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:14.114 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:14.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:14.120 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:14.217 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:49:14.221 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:14.250 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:14.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:14.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:14.293 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:49:14.347 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:49:14.353 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:14.380 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:14.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:14.385 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:14.406 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:49:14.442 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:49:14.484 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:49:14.486 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:14.522 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:14.525 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:49:14.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:14.526 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:14.567 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:49:14.571 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:49:14.571 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:49:14.571 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:49:14.571 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:49:14.571 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:49:14.571 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:49:14.571 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:49:14.572 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:49:14.627 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:14.653 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:14.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:14.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:14.755 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T19:49:14.756 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:14.784 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:14.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:14.787 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:49:14.888 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:14.896 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:14 smithi016 bash[75004]: audit 2023-09-23T19:49:14.165540+0000 mon.a (mon.0) 1555 : audit 0 from='client.? 172.21.15.16:0/4094854102' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:49:14.896 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:14 smithi016 bash[75004]: audit 2023-09-23T19:49:14.399865+0000 mon.a (mon.0) 1556 : audit 1 from='client.? 172.21.15.16:0/3975310538' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T19:49:14.897 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:14 smithi016 bash[77995]: audit 2023-09-23T19:49:14.165540+0000 mon.a (mon.0) 1555 : audit 0 from='client.? 172.21.15.16:0/4094854102' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:49:14.897 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:14 smithi016 bash[77995]: audit 2023-09-23T19:49:14.399865+0000 mon.a (mon.0) 1556 : audit 1 from='client.? 172.21.15.16:0/3975310538' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T19:49:14.922 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:14.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:14.924 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:15.025 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:15.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:14 smithi092 bash[58855]: audit 2023-09-23T19:49:14.165540+0000 mon.a (mon.0) 1555 : audit 0 from='client.? 172.21.15.16:0/4094854102' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:49:15.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:14 smithi092 bash[58855]: audit 2023-09-23T19:49:14.399865+0000 mon.a (mon.0) 1556 : audit 1 from='client.? 172.21.15.16:0/3975310538' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T19:49:15.060 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:15.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:15.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:15.167 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:15.192 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:15.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:15.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:49:15.294 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:15.325 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:15.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:15.332 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:15.433 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:15.473 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:15.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:15.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:15.578 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:15.610 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:15.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:15.616 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:15.717 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:15.741 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:15.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:15.747 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:49:15.781 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is unset 2023-09-23T19:49:15.848 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:15.877 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:15.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:15.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:15.896 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:15 smithi016 bash[75004]: cluster 2023-09-23T19:49:14.381457+0000 mgr.y (mgr.25325) 1256 : cluster 0 pgmap v1242: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 599 op/s 2023-09-23T19:49:15.897 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:15 smithi016 bash[75004]: audit 2023-09-23T19:49:14.743700+0000 mon.a (mon.0) 1557 : audit 1 from='client.? 172.21.15.16:0/3975310538' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T19:49:15.897 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:49:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:49:15] "GET /metrics HTTP/1.1" 200 36307 "" "Prometheus/2.33.4" 2023-09-23T19:49:15.981 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:16.021 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:16.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:16.023 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:49:16.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:15 smithi092 bash[58855]: cluster 2023-09-23T19:49:14.381457+0000 mgr.y (mgr.25325) 1256 : cluster 0 pgmap v1242: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 599 op/s 2023-09-23T19:49:16.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:15 smithi092 bash[58855]: audit 2023-09-23T19:49:14.743700+0000 mon.a (mon.0) 1557 : audit 1 from='client.? 172.21.15.16:0/3975310538' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T19:49:16.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:15 smithi092 bash[58855]: cluster 2023-09-23T19:49:14.743806+0000 mon.a (mon.0) 1558 : cluster 0 osdmap e220: 8 total, 8 up, 8 in 2023-09-23T19:49:16.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:15 smithi092 bash[58855]: cluster 2023-09-23T19:49:14.747229+0000 osd.5 (osd.5) 147 : cluster 3 failed to encode map e220 with expected crc 2023-09-23T19:49:16.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:15 smithi092 bash[58855]: cluster 2023-09-23T19:49:14.754583+0000 osd.6 (osd.6) 150 : cluster 3 failed to encode map e220 with expected crc 2023-09-23T19:49:16.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:15 smithi092 bash[58855]: cluster 2023-09-23T19:49:14.756251+0000 osd.7 (osd.7) 139 : cluster 3 failed to encode map e220 with expected crc 2023-09-23T19:49:16.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:15 smithi092 bash[58855]: audit 2023-09-23T19:49:14.922305+0000 mon.c (mon.2) 199 : audit 1 from='client.? 172.21.15.16:0/2260032380' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:49:16.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:15 smithi092 bash[58855]: audit 2023-09-23T19:49:14.923423+0000 mon.a (mon.0) 1559 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:49:16.124 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:16.149 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:16.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:16.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:16.253 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:16.262 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:15 smithi016 bash[75004]: cluster 2023-09-23T19:49:14.743806+0000 mon.a (mon.0) 1558 : cluster 0 osdmap e220: 8 total, 8 up, 8 in 2023-09-23T19:49:16.262 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:15 smithi016 bash[75004]: cluster 2023-09-23T19:49:14.747229+0000 osd.5 (osd.5) 147 : cluster 3 failed to encode map e220 with expected crc 2023-09-23T19:49:16.263 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:15 smithi016 bash[75004]: cluster 2023-09-23T19:49:14.754583+0000 osd.6 (osd.6) 150 : cluster 3 failed to encode map e220 with expected crc 2023-09-23T19:49:16.263 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:15 smithi016 bash[75004]: cluster 2023-09-23T19:49:14.756251+0000 osd.7 (osd.7) 139 : cluster 3 failed to encode map e220 with expected crc 2023-09-23T19:49:16.263 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:15 smithi016 bash[75004]: audit 2023-09-23T19:49:14.922305+0000 mon.c (mon.2) 199 : audit 1 from='client.? 172.21.15.16:0/2260032380' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:49:16.263 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:15 smithi016 bash[75004]: audit 2023-09-23T19:49:14.923423+0000 mon.a (mon.0) 1559 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:49:16.263 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:15 smithi016 bash[77995]: cluster 2023-09-23T19:49:14.381457+0000 mgr.y (mgr.25325) 1256 : cluster 0 pgmap v1242: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 599 op/s 2023-09-23T19:49:16.263 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:15 smithi016 bash[77995]: audit 2023-09-23T19:49:14.743700+0000 mon.a (mon.0) 1557 : audit 1 from='client.? 172.21.15.16:0/3975310538' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T19:49:16.263 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:15 smithi016 bash[77995]: cluster 2023-09-23T19:49:14.743806+0000 mon.a (mon.0) 1558 : cluster 0 osdmap e220: 8 total, 8 up, 8 in 2023-09-23T19:49:16.263 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:15 smithi016 bash[77995]: cluster 2023-09-23T19:49:14.747229+0000 osd.5 (osd.5) 147 : cluster 3 failed to encode map e220 with expected crc 2023-09-23T19:49:16.263 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:15 smithi016 bash[77995]: cluster 2023-09-23T19:49:14.754583+0000 osd.6 (osd.6) 150 : cluster 3 failed to encode map e220 with expected crc 2023-09-23T19:49:16.263 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:15 smithi016 bash[77995]: cluster 2023-09-23T19:49:14.756251+0000 osd.7 (osd.7) 139 : cluster 3 failed to encode map e220 with expected crc 2023-09-23T19:49:16.263 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:15 smithi016 bash[77995]: audit 2023-09-23T19:49:14.922305+0000 mon.c (mon.2) 199 : audit 1 from='client.? 172.21.15.16:0/2260032380' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:49:16.264 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:15 smithi016 bash[77995]: audit 2023-09-23T19:49:14.923423+0000 mon.a (mon.0) 1559 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:49:16.284 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:16.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:16.286 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:49:16.387 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:16.416 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:16.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:16.419 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:16.520 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:16.552 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:16.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:16.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:16.658 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:16.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:16.692 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:16.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:16.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:16 smithi016 bash[75004]: cluster 2023-09-23T19:49:15.749790+0000 mon.a (mon.0) 1560 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:49:16.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:16 smithi016 bash[75004]: cluster 2023-09-23T19:49:15.749812+0000 mon.a (mon.0) 1561 : cluster 1 Cluster is now healthy 2023-09-23T19:49:16.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:16 smithi016 bash[75004]: audit 2023-09-23T19:49:15.763904+0000 mon.a (mon.0) 1562 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:49:16.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:16 smithi016 bash[75004]: cluster 2023-09-23T19:49:15.764004+0000 mon.a (mon.0) 1563 : cluster 0 osdmap e221: 8 total, 8 up, 8 in 2023-09-23T19:49:16.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:16 smithi016 bash[75004]: cluster 2023-09-23T19:49:15.766897+0000 osd.7 (osd.7) 140 : cluster 3 failed to encode map e221 with expected crc 2023-09-23T19:49:16.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:16 smithi016 bash[75004]: cluster 2023-09-23T19:49:15.899983+0000 osd.5 (osd.5) 148 : cluster 3 failed to encode map e221 with expected crc 2023-09-23T19:49:16.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:16 smithi016 bash[75004]: cluster 2023-09-23T19:49:15.902575+0000 osd.6 (osd.6) 151 : cluster 3 failed to encode map e221 with expected crc 2023-09-23T19:49:16.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:16 smithi016 bash[75004]: cluster 2023-09-23T19:49:15.906026+0000 osd.6 (osd.6) 152 : cluster 3 failed to encode map e221 with expected crc 2023-09-23T19:49:16.788 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:16 smithi016 bash[77995]: cluster 2023-09-23T19:49:15.749790+0000 mon.a (mon.0) 1560 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:49:16.788 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:16 smithi016 bash[77995]: cluster 2023-09-23T19:49:15.749812+0000 mon.a (mon.0) 1561 : cluster 1 Cluster is now healthy 2023-09-23T19:49:16.788 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:16 smithi016 bash[77995]: audit 2023-09-23T19:49:15.763904+0000 mon.a (mon.0) 1562 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:49:16.788 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:16 smithi016 bash[77995]: cluster 2023-09-23T19:49:15.764004+0000 mon.a (mon.0) 1563 : cluster 0 osdmap e221: 8 total, 8 up, 8 in 2023-09-23T19:49:16.788 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:16 smithi016 bash[77995]: cluster 2023-09-23T19:49:15.766897+0000 osd.7 (osd.7) 140 : cluster 3 failed to encode map e221 with expected crc 2023-09-23T19:49:16.788 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:16 smithi016 bash[77995]: cluster 2023-09-23T19:49:15.899983+0000 osd.5 (osd.5) 148 : cluster 3 failed to encode map e221 with expected crc 2023-09-23T19:49:16.788 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:16 smithi016 bash[77995]: cluster 2023-09-23T19:49:15.902575+0000 osd.6 (osd.6) 151 : cluster 3 failed to encode map e221 with expected crc 2023-09-23T19:49:16.788 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:16 smithi016 bash[77995]: cluster 2023-09-23T19:49:15.906026+0000 osd.6 (osd.6) 152 : cluster 3 failed to encode map e221 with expected crc 2023-09-23T19:49:16.792 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:16.823 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:16.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:16.826 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:16.927 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:16.948 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:16.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:16.951 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:17.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:16 smithi092 bash[58855]: cluster 2023-09-23T19:49:15.749790+0000 mon.a (mon.0) 1560 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:49:17.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:16 smithi092 bash[58855]: cluster 2023-09-23T19:49:15.749812+0000 mon.a (mon.0) 1561 : cluster 1 Cluster is now healthy 2023-09-23T19:49:17.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:16 smithi092 bash[58855]: audit 2023-09-23T19:49:15.763904+0000 mon.a (mon.0) 1562 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:49:17.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:16 smithi092 bash[58855]: cluster 2023-09-23T19:49:15.764004+0000 mon.a (mon.0) 1563 : cluster 0 osdmap e221: 8 total, 8 up, 8 in 2023-09-23T19:49:17.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:16 smithi092 bash[58855]: cluster 2023-09-23T19:49:15.766897+0000 osd.7 (osd.7) 140 : cluster 3 failed to encode map e221 with expected crc 2023-09-23T19:49:17.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:16 smithi092 bash[58855]: cluster 2023-09-23T19:49:15.899983+0000 osd.5 (osd.5) 148 : cluster 3 failed to encode map e221 with expected crc 2023-09-23T19:49:17.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:16 smithi092 bash[58855]: cluster 2023-09-23T19:49:15.902575+0000 osd.6 (osd.6) 151 : cluster 3 failed to encode map e221 with expected crc 2023-09-23T19:49:17.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:16 smithi092 bash[58855]: cluster 2023-09-23T19:49:15.906026+0000 osd.6 (osd.6) 152 : cluster 3 failed to encode map e221 with expected crc 2023-09-23T19:49:17.052 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:17.076 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:17.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:17.084 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:17.185 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:17.225 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:17.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:17.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:17.331 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:17.353 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:17.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:17.356 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:17.456 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:17.483 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:17.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:17.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:17.587 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:17.615 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:17.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:17.619 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:17.720 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:17.744 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:17.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:17.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:17.768 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:17 smithi016 bash[75004]: cluster 2023-09-23T19:49:16.381919+0000 mgr.y (mgr.25325) 1257 : cluster 0 pgmap v1245: 89 pgs: 89 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2023-09-23T19:49:17.792 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T19:49:17.849 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:17.857 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:17 smithi092 bash[58855]: cluster 2023-09-23T19:49:16.381919+0000 mgr.y (mgr.25325) 1257 : cluster 0 pgmap v1245: 89 pgs: 89 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2023-09-23T19:49:17.875 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:17.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:17.879 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:17.979 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:18.020 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:18.022 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:17 smithi016 bash[77995]: cluster 2023-09-23T19:49:16.381919+0000 mgr.y (mgr.25325) 1257 : cluster 0 pgmap v1245: 89 pgs: 89 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2023-09-23T19:49:18.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:18.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:18.124 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:18.164 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:18.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:18.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:18.269 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:18.294 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:18.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:18.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:18.397 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:18.424 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:18.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:18.427 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:18.528 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:18.559 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:18.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:18.567 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:18.632 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:49:18.669 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:18.694 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:18.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:18.701 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:49:18.801 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:18.831 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:18.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:18.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:49:18.936 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:18.975 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:18.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:18.978 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:49:19.079 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:19.109 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:19.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:19.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:49:19.214 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:19.238 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:19.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:19.240 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:49:19.341 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:19.391 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:19.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:19.397 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:19.498 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:19.526 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:19.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:19.528 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:19.629 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:19.668 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:19.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:19.674 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:49:19.775 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:19.811 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:19.812 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T19:49:19.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:19.814 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:49:19.915 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:19.939 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:19.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:19.941 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:49:20.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:19 smithi016 bash[77995]: cluster 2023-09-23T19:49:18.382406+0000 mgr.y (mgr.25325) 1258 : cluster 0 pgmap v1246: 89 pgs: 89 active+clean; 3.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 513 op/s 2023-09-23T19:49:20.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:19 smithi016 bash[75004]: cluster 2023-09-23T19:49:18.382406+0000 mgr.y (mgr.25325) 1258 : cluster 0 pgmap v1246: 89 pgs: 89 active+clean; 3.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 513 op/s 2023-09-23T19:49:20.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:19 smithi092 bash[58855]: cluster 2023-09-23T19:49:18.382406+0000 mgr.y (mgr.25325) 1258 : cluster 0 pgmap v1246: 89 pgs: 89 active+clean; 3.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 513 op/s 2023-09-23T19:49:20.042 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:20.063 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:20.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:20.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:20.173 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:20.201 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:20.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:20.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:20.310 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:20.319 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:49:20.336 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:20.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:20.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:20.439 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:20.486 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:20.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:20.492 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:20.593 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:20.631 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:20.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:20.634 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:20.735 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:20.757 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:20.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:20.762 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:49:20.863 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:20.902 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:20.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:20.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:21.006 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:21.041 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:21.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:21.048 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:21.149 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:21.184 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:21.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:21.191 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:21.291 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:21.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:21.330 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:21.330 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:21.431 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:21.470 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:21.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:21.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:21.578 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:21.601 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:21.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:21.607 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:21.707 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:21.730 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:21.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:21.738 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:21.838 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:21.849 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:21 smithi092 bash[58855]: cluster 2023-09-23T19:49:20.383106+0000 mgr.y (mgr.25325) 1259 : cluster 0 pgmap v1247: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 657 op/s 2023-09-23T19:49:21.873 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:21.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:21.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:21.978 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:22.010 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:22.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:22.017 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:22.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:21 smithi016 bash[77995]: cluster 2023-09-23T19:49:20.383106+0000 mgr.y (mgr.25325) 1259 : cluster 0 pgmap v1247: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 657 op/s 2023-09-23T19:49:22.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:21 smithi016 bash[75004]: cluster 2023-09-23T19:49:20.383106+0000 mgr.y (mgr.25325) 1259 : cluster 0 pgmap v1247: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 657 op/s 2023-09-23T19:49:22.118 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:22.156 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:22.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:22.161 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:49:22.262 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:22.295 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:22.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:22.302 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:22.354 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:49:22.404 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:22.435 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:22.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:22.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:22.540 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:22.572 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:22.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:22.575 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:22.676 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:22.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:22.712 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:22.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:22.813 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:22.845 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:22.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:22.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:22.949 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:22.982 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:22.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:22.986 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:23.086 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:23.110 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:23.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:23.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:23.216 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:23.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:23.247 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:23.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:23.348 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:23.392 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:23.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:23.394 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:23.495 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:23.533 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:23.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:23.536 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:23.637 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:23.665 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:23.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:23.667 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:49:23.768 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:23.795 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:23.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:23.798 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:49:23.899 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:23.938 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:23.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:23.940 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:24.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:23 smithi016 bash[77995]: cluster 2023-09-23T19:49:22.383565+0000 mgr.y (mgr.25325) 1260 : cluster 0 pgmap v1248: 89 pgs: 89 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 678 op/s 2023-09-23T19:49:24.039 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:23 smithi016 bash[75004]: cluster 2023-09-23T19:49:22.383565+0000 mgr.y (mgr.25325) 1260 : cluster 0 pgmap v1248: 89 pgs: 89 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 678 op/s 2023-09-23T19:49:24.041 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:24.048 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:23 smithi092 bash[58855]: cluster 2023-09-23T19:49:22.383565+0000 mgr.y (mgr.25325) 1260 : cluster 0 pgmap v1248: 89 pgs: 89 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 678 op/s 2023-09-23T19:49:24.065 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:24.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:24.068 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:24.169 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:24.205 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:24.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:24.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:49:24.309 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:24.336 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:24.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:24.341 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:24.442 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:24.469 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:24.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:24.474 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:24.575 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:24.601 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:24.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:24.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:49:24.705 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:24.732 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:24.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:24.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:49:24.836 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:24.872 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:24.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:24.878 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:49:24.881 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:49:24.882 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:49:24.979 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:25.020 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:25.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:25.024 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:25.125 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:25.158 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:25.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:25.165 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:25.266 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:25.297 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:25.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:25.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:25.400 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:25.433 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:25.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:25.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:25.538 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:25.561 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:25.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:25.569 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:25.669 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:25.699 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:25.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:25.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:25.786 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:49:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:49:25] "GET /metrics HTTP/1.1" 200 36307 "" "Prometheus/2.33.4" 2023-09-23T19:49:25.804 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:25.813 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:25 smithi092 bash[58855]: cluster 2023-09-23T19:49:24.386683+0000 mgr.y (mgr.25325) 1261 : cluster 0 pgmap v1249: 89 pgs: 89 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 601 op/s 2023-09-23T19:49:25.837 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:25.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:25.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:49:25.943 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:25.964 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:25.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:25.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:49:26.067 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:26.078 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:25 smithi016 bash[75004]: cluster 2023-09-23T19:49:24.386683+0000 mgr.y (mgr.25325) 1261 : cluster 0 pgmap v1249: 89 pgs: 89 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 601 op/s 2023-09-23T19:49:26.078 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:25 smithi016 bash[77995]: cluster 2023-09-23T19:49:24.386683+0000 mgr.y (mgr.25325) 1261 : cluster 0 pgmap v1249: 89 pgs: 89 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 601 op/s 2023-09-23T19:49:26.108 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:26.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:26.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:49:26.213 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:26.244 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:26.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:26.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:26.347 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:26.374 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:26.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:26.377 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:26.478 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:26.508 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:26.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:26.513 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:49:26.613 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:26.645 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:26.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:26.649 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:49:26.750 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:26.788 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:26.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:26.793 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:26.894 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:26.919 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:26.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:26.922 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:49:27.023 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:27.053 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:27.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:27.057 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:49:27.158 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:27.221 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:27.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:27.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:27.325 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:27.350 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:27.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:27.354 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:27.454 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:27.492 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:27.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:27.501 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:27.602 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:27.635 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:27.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:27.638 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:49:27.739 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:27.772 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:27.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:27.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:49:27.876 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:27.908 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:27.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:27.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:49:28.019 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:28.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:27 smithi016 bash[75004]: cluster 2023-09-23T19:49:26.387327+0000 mgr.y (mgr.25325) 1262 : cluster 0 pgmap v1250: 89 pgs: 89 active+clean; 3.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 632 op/s 2023-09-23T19:49:28.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:27 smithi016 bash[75004]: audit 2023-09-23T19:49:27.084729+0000 mon.a (mon.0) 1564 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:49:28.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:27 smithi016 bash[75004]: audit 2023-09-23T19:49:27.617773+0000 mon.a (mon.0) 1565 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:49:28.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:27 smithi016 bash[75004]: audit 2023-09-23T19:49:27.625160+0000 mon.a (mon.0) 1566 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:49:28.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:27 smithi016 bash[75004]: audit 2023-09-23T19:49:27.644044+0000 mon.a (mon.0) 1567 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:49:28.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:27 smithi016 bash[75004]: audit 2023-09-23T19:49:27.653535+0000 mon.a (mon.0) 1568 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:49:28.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:27 smithi016 bash[77995]: cluster 2023-09-23T19:49:26.387327+0000 mgr.y (mgr.25325) 1262 : cluster 0 pgmap v1250: 89 pgs: 89 active+clean; 3.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 632 op/s 2023-09-23T19:49:28.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:27 smithi016 bash[77995]: audit 2023-09-23T19:49:27.084729+0000 mon.a (mon.0) 1564 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:49:28.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:27 smithi016 bash[77995]: audit 2023-09-23T19:49:27.617773+0000 mon.a (mon.0) 1565 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:49:28.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:27 smithi016 bash[77995]: audit 2023-09-23T19:49:27.625160+0000 mon.a (mon.0) 1566 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:49:28.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:27 smithi016 bash[77995]: audit 2023-09-23T19:49:27.644044+0000 mon.a (mon.0) 1567 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:49:28.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:27 smithi016 bash[77995]: audit 2023-09-23T19:49:27.653535+0000 mon.a (mon.0) 1568 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:49:28.049 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:28.051 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:27 smithi092 bash[58855]: cluster 2023-09-23T19:49:26.387327+0000 mgr.y (mgr.25325) 1262 : cluster 0 pgmap v1250: 89 pgs: 89 active+clean; 3.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 632 op/s 2023-09-23T19:49:28.051 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:27 smithi092 bash[58855]: audit 2023-09-23T19:49:27.084729+0000 mon.a (mon.0) 1564 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:49:28.051 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:27 smithi092 bash[58855]: audit 2023-09-23T19:49:27.617773+0000 mon.a (mon.0) 1565 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:49:28.051 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:27 smithi092 bash[58855]: audit 2023-09-23T19:49:27.625160+0000 mon.a (mon.0) 1566 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:49:28.052 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:27 smithi092 bash[58855]: audit 2023-09-23T19:49:27.644044+0000 mon.a (mon.0) 1567 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:49:28.052 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:27 smithi092 bash[58855]: audit 2023-09-23T19:49:27.653535+0000 mon.a (mon.0) 1568 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:49:28.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:28.053 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:49:28.153 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:28.189 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:28.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:28.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:28.295 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:28.318 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:28.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:28.320 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:28.421 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:28.463 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:28.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:28.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:28.569 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:28.594 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:28.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:28.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:28.701 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:28.735 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:28.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:28.738 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:28.839 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:28.861 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:28.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:28.865 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:28.966 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:28.998 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:29.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:29.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:29.103 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:29.150 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:29.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:29.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:29.255 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:29.288 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:29.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:29.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:49:29.393 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:29.430 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:29.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:29.431 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:29.531 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:29.558 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:29.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:29.561 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:49:29.661 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:29.705 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:29.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:29.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:29.811 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:29.837 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:29.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:29.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:49:29.941 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:29.949 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:29 smithi016 bash[77995]: cluster 2023-09-23T19:49:28.387786+0000 mgr.y (mgr.25325) 1263 : cluster 0 pgmap v1251: 89 pgs: 89 active+clean; 2.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 574 op/s 2023-09-23T19:49:29.949 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:29 smithi016 bash[77995]: audit 2023-09-23T19:49:28.674667+0000 mon.a (mon.0) 1569 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:49:29.949 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:29 smithi016 bash[77995]: audit 2023-09-23T19:49:28.676388+0000 mon.a (mon.0) 1570 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:49:29.949 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:29 smithi016 bash[75004]: cluster 2023-09-23T19:49:28.387786+0000 mgr.y (mgr.25325) 1263 : cluster 0 pgmap v1251: 89 pgs: 89 active+clean; 2.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 574 op/s 2023-09-23T19:49:29.949 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:29 smithi016 bash[75004]: audit 2023-09-23T19:49:28.674667+0000 mon.a (mon.0) 1569 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:49:29.950 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:29 smithi016 bash[75004]: audit 2023-09-23T19:49:28.676388+0000 mon.a (mon.0) 1570 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:49:29.968 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:29.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:29.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:30.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:29 smithi092 bash[58855]: cluster 2023-09-23T19:49:28.387786+0000 mgr.y (mgr.25325) 1263 : cluster 0 pgmap v1251: 89 pgs: 89 active+clean; 2.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 574 op/s 2023-09-23T19:49:30.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:29 smithi092 bash[58855]: audit 2023-09-23T19:49:28.674667+0000 mon.a (mon.0) 1569 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:49:30.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:29 smithi092 bash[58855]: audit 2023-09-23T19:49:28.676388+0000 mon.a (mon.0) 1570 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:49:30.073 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:30.102 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:30.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:30.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:30.209 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:30.239 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:30.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:30.243 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:30.343 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:30.370 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:30.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:30.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:49:30.477 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:30.514 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:30.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:30.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:30.621 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:30.654 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:30.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:30.659 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:30.760 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:30.779 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:30.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:30.781 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:49:30.881 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:30.913 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:30.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:30.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:49:31.020 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:31.053 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:31.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:31.057 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:31.158 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:31.180 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:31.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:31.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:49:31.283 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:31.314 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:31.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:31.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:31.417 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:31.440 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:31.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:31.443 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:31.543 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:31.572 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:31.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:31.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:31.680 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:31.683 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:31 smithi092 bash[58855]: cluster 2023-09-23T19:49:30.388458+0000 mgr.y (mgr.25325) 1264 : cluster 0 pgmap v1252: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2023-09-23T19:49:31.714 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:31.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:31.722 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:31.823 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:31.831 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:31 smithi016 bash[77995]: cluster 2023-09-23T19:49:30.388458+0000 mgr.y (mgr.25325) 1264 : cluster 0 pgmap v1252: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2023-09-23T19:49:31.831 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:31 smithi016 bash[75004]: cluster 2023-09-23T19:49:30.388458+0000 mgr.y (mgr.25325) 1264 : cluster 0 pgmap v1252: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2023-09-23T19:49:31.857 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:31.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:31.862 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:31.963 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:31.995 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:32.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:32.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:32.103 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:32.125 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:32.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:32.127 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:32.228 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:32.229 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-09-23T19:49:32.229 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:49:32.252 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:32.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:32.256 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:32.357 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:32.387 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:32.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:32.391 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:49:32.491 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:32.526 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:32.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:32.532 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:32.633 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:32.664 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:32.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:32.668 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:32.769 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:32.799 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:32.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:32.802 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:32.902 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:32.935 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:32.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:32.938 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:33.039 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:33.096 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:33.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:33.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:33.201 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:33.246 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:33.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:33.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:33.352 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:33.380 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:33.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:33.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:33.483 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:33.531 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:33.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:33.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:33.639 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:33.681 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:33.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:33.688 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:49:33.701 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T19:49:33.704 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:33 smithi092 bash[58855]: cluster 2023-09-23T19:49:32.388988+0000 mgr.y (mgr.25325) 1265 : cluster 0 pgmap v1253: 89 pgs: 89 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 605 op/s 2023-09-23T19:49:33.704 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:33 smithi092 bash[58855]: audit 2023-09-23T19:49:33.386693+0000 mon.c (mon.2) 200 : audit 1 from='client.? 172.21.15.16:0/1991987928' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T19:49:33.704 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:33 smithi092 bash[58855]: audit 2023-09-23T19:49:33.387669+0000 mon.a (mon.0) 1571 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T19:49:33.772 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:33 smithi016 bash[77995]: cluster 2023-09-23T19:49:32.388988+0000 mgr.y (mgr.25325) 1265 : cluster 0 pgmap v1253: 89 pgs: 89 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 605 op/s 2023-09-23T19:49:33.772 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:33 smithi016 bash[77995]: audit 2023-09-23T19:49:33.386693+0000 mon.c (mon.2) 200 : audit 1 from='client.? 172.21.15.16:0/1991987928' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T19:49:33.772 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:33 smithi016 bash[77995]: audit 2023-09-23T19:49:33.387669+0000 mon.a (mon.0) 1571 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T19:49:33.773 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:33 smithi016 bash[75004]: cluster 2023-09-23T19:49:32.388988+0000 mgr.y (mgr.25325) 1265 : cluster 0 pgmap v1253: 89 pgs: 89 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 605 op/s 2023-09-23T19:49:33.773 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:33 smithi016 bash[75004]: audit 2023-09-23T19:49:33.386693+0000 mon.c (mon.2) 200 : audit 1 from='client.? 172.21.15.16:0/1991987928' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T19:49:33.773 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:33 smithi016 bash[75004]: audit 2023-09-23T19:49:33.387669+0000 mon.a (mon.0) 1571 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T19:49:33.789 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:33.831 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:33.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:33.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:49:33.967 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:34.001 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:34.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:34.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:49:34.111 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:34.145 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:34.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:34.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:34.247 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:34.279 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:34.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:34.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:49:34.389 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:34.414 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:34.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:34.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:34.517 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:34.537 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:34.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:34.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:49:34.641 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:34.669 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:34.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:34.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:49:34.774 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:34.810 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:34.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:34.814 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:49:34.915 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:34.923 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:34 smithi092 bash[58855]: audit 2023-09-23T19:49:33.690947+0000 mon.a (mon.0) 1572 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T19:49:34.924 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:34 smithi092 bash[58855]: cluster 2023-09-23T19:49:33.691034+0000 mon.a (mon.0) 1573 : cluster 0 osdmap e222: 8 total, 8 up, 8 in 2023-09-23T19:49:34.924 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:34 smithi092 bash[58855]: cluster 2023-09-23T19:49:33.703213+0000 osd.7 (osd.7) 141 : cluster 3 failed to encode map e222 with expected crc 2023-09-23T19:49:34.924 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:34 smithi092 bash[58855]: cluster 2023-09-23T19:49:33.718821+0000 osd.6 (osd.6) 153 : cluster 3 failed to encode map e222 with expected crc 2023-09-23T19:49:34.924 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:34 smithi092 bash[58855]: cluster 2023-09-23T19:49:33.719423+0000 osd.5 (osd.5) 149 : cluster 3 failed to encode map e222 with expected crc 2023-09-23T19:49:34.924 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:34 smithi092 bash[58855]: cluster 2023-09-23T19:49:33.761891+0000 osd.4 (osd.4) 162 : cluster 3 failed to encode map e222 with expected crc 2023-09-23T19:49:34.924 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:34 smithi092 bash[58855]: audit 2023-09-23T19:49:34.063866+0000 mon.a (mon.0) 1574 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:49:34.924 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:34 smithi092 bash[58855]: audit 2023-09-23T19:49:34.072454+0000 mon.a (mon.0) 1575 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:49:34.924 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:34 smithi092 bash[58855]: audit 2023-09-23T19:49:34.124403+0000 mon.a (mon.0) 1576 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:49:34.924 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:34 smithi092 bash[58855]: audit 2023-09-23T19:49:34.136472+0000 mon.a (mon.0) 1577 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:49:34.945 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:34.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:34.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:49:35.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:34 smithi016 bash[75004]: audit 2023-09-23T19:49:33.690947+0000 mon.a (mon.0) 1572 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T19:49:35.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:34 smithi016 bash[75004]: cluster 2023-09-23T19:49:33.691034+0000 mon.a (mon.0) 1573 : cluster 0 osdmap e222: 8 total, 8 up, 8 in 2023-09-23T19:49:35.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:34 smithi016 bash[75004]: cluster 2023-09-23T19:49:33.703213+0000 osd.7 (osd.7) 141 : cluster 3 failed to encode map e222 with expected crc 2023-09-23T19:49:35.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:34 smithi016 bash[75004]: cluster 2023-09-23T19:49:33.718821+0000 osd.6 (osd.6) 153 : cluster 3 failed to encode map e222 with expected crc 2023-09-23T19:49:35.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:34 smithi016 bash[75004]: cluster 2023-09-23T19:49:33.719423+0000 osd.5 (osd.5) 149 : cluster 3 failed to encode map e222 with expected crc 2023-09-23T19:49:35.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:34 smithi016 bash[75004]: cluster 2023-09-23T19:49:33.761891+0000 osd.4 (osd.4) 162 : cluster 3 failed to encode map e222 with expected crc 2023-09-23T19:49:35.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:34 smithi016 bash[75004]: audit 2023-09-23T19:49:34.063866+0000 mon.a (mon.0) 1574 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:49:35.039 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:34 smithi016 bash[75004]: audit 2023-09-23T19:49:34.072454+0000 mon.a (mon.0) 1575 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:49:35.039 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:34 smithi016 bash[75004]: audit 2023-09-23T19:49:34.124403+0000 mon.a (mon.0) 1576 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:49:35.039 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:34 smithi016 bash[75004]: audit 2023-09-23T19:49:34.136472+0000 mon.a (mon.0) 1577 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:49:35.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:34 smithi016 bash[77995]: audit 2023-09-23T19:49:33.690947+0000 mon.a (mon.0) 1572 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T19:49:35.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:34 smithi016 bash[77995]: cluster 2023-09-23T19:49:33.691034+0000 mon.a (mon.0) 1573 : cluster 0 osdmap e222: 8 total, 8 up, 8 in 2023-09-23T19:49:35.040 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:34 smithi016 bash[77995]: cluster 2023-09-23T19:49:33.703213+0000 osd.7 (osd.7) 141 : cluster 3 failed to encode map e222 with expected crc 2023-09-23T19:49:35.040 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:34 smithi016 bash[77995]: cluster 2023-09-23T19:49:33.718821+0000 osd.6 (osd.6) 153 : cluster 3 failed to encode map e222 with expected crc 2023-09-23T19:49:35.040 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:34 smithi016 bash[77995]: cluster 2023-09-23T19:49:33.719423+0000 osd.5 (osd.5) 149 : cluster 3 failed to encode map e222 with expected crc 2023-09-23T19:49:35.040 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:34 smithi016 bash[77995]: cluster 2023-09-23T19:49:33.761891+0000 osd.4 (osd.4) 162 : cluster 3 failed to encode map e222 with expected crc 2023-09-23T19:49:35.040 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:34 smithi016 bash[77995]: audit 2023-09-23T19:49:34.063866+0000 mon.a (mon.0) 1574 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:49:35.040 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:34 smithi016 bash[77995]: audit 2023-09-23T19:49:34.072454+0000 mon.a (mon.0) 1575 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:49:35.040 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:34 smithi016 bash[77995]: audit 2023-09-23T19:49:34.124403+0000 mon.a (mon.0) 1576 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:49:35.040 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:34 smithi016 bash[77995]: audit 2023-09-23T19:49:34.136472+0000 mon.a (mon.0) 1577 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:49:35.049 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:35.079 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:35.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:35.082 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:35.183 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:35.218 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:35.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:35.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:35.325 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:35.348 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:35.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:35.351 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:35.451 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:35.479 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:35.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:35.484 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:35.584 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:35.614 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:35.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:35.617 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:35.708 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:35 smithi092 bash[58855]: cluster 2023-09-23T19:49:34.389390+0000 mgr.y (mgr.25325) 1266 : cluster 0 pgmap v1255: 89 pgs: 89 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 627 op/s 2023-09-23T19:49:35.709 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:35 smithi092 bash[58855]: audit 2023-09-23T19:49:34.771920+0000 mon.a (mon.0) 1578 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:49:35.709 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:35 smithi092 bash[58855]: audit 2023-09-23T19:49:34.773127+0000 mon.a (mon.0) 1579 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:49:35.709 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:35 smithi092 bash[58855]: audit 2023-09-23T19:49:34.785113+0000 mon.a (mon.0) 1580 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:49:35.717 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:35.748 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:35.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:35.754 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:49:35.854 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:35.887 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:35.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:35.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:35.991 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:36.014 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:36.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:36.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:36.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:35 smithi016 bash[75004]: cluster 2023-09-23T19:49:34.389390+0000 mgr.y (mgr.25325) 1266 : cluster 0 pgmap v1255: 89 pgs: 89 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 627 op/s 2023-09-23T19:49:36.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:35 smithi016 bash[75004]: audit 2023-09-23T19:49:34.771920+0000 mon.a (mon.0) 1578 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:49:36.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:35 smithi016 bash[75004]: audit 2023-09-23T19:49:34.773127+0000 mon.a (mon.0) 1579 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:49:36.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:35 smithi016 bash[75004]: audit 2023-09-23T19:49:34.785113+0000 mon.a (mon.0) 1580 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:49:36.037 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:49:35 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:49:35] "GET /metrics HTTP/1.1" 200 36308 "" "Prometheus/2.33.4" 2023-09-23T19:49:36.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:35 smithi016 bash[77995]: cluster 2023-09-23T19:49:34.389390+0000 mgr.y (mgr.25325) 1266 : cluster 0 pgmap v1255: 89 pgs: 89 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 627 op/s 2023-09-23T19:49:36.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:35 smithi016 bash[77995]: audit 2023-09-23T19:49:34.771920+0000 mon.a (mon.0) 1578 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:49:36.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:35 smithi016 bash[77995]: audit 2023-09-23T19:49:34.773127+0000 mon.a (mon.0) 1579 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:49:36.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:35 smithi016 bash[77995]: audit 2023-09-23T19:49:34.785113+0000 mon.a (mon.0) 1580 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:49:36.117 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:36.144 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:36.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:36.146 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:36.247 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:36.272 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:36.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:36.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:36.375 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:36.397 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:36.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:36.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:36.435 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T19:49:36.504 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:36.558 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:36.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:36.564 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:36.665 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:36.689 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:36.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:36.697 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:49:36.798 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:36.832 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:36.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:36.834 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:36.935 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:36.968 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:36.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:36.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:49:37.073 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:37.121 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:37.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:37.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:37.156 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T19:49:37.233 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:37.258 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:37.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:37.264 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:37.365 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:37.392 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:37.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:37.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:49:37.495 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:37.552 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:37.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:37.556 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:37.657 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:37.683 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:37.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:37.685 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:37.786 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:37.810 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:37 smithi016 bash[75004]: cluster 2023-09-23T19:49:36.395141+0000 mgr.y (mgr.25325) 1267 : cluster 0 pgmap v1256: 89 pgs: 89 active+clean; 2.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 603 op/s 2023-09-23T19:49:37.811 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:37 smithi016 bash[77995]: cluster 2023-09-23T19:49:36.395141+0000 mgr.y (mgr.25325) 1267 : cluster 0 pgmap v1256: 89 pgs: 89 active+clean; 2.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 603 op/s 2023-09-23T19:49:37.817 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:37.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:37.822 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:49:37.922 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:37.960 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:37.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:37.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:38.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:37 smithi092 bash[58855]: cluster 2023-09-23T19:49:36.395141+0000 mgr.y (mgr.25325) 1267 : cluster 0 pgmap v1256: 89 pgs: 89 active+clean; 2.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 603 op/s 2023-09-23T19:49:38.062 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:38.089 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:38.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:38.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:49:38.194 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:38.225 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:38.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:38.229 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:38.330 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:38.365 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:38.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:38.368 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:49:38.469 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:38.498 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:38.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:38.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:49:38.607 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:38.638 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:38.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:38.641 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:38.741 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:38.776 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:38.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:38.779 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:38.880 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:38.909 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:38.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:38.916 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:39.017 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:39.061 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:39.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:39.069 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:49:39.169 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:39.201 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:39.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:39.208 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:39.308 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:39.361 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:39.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:39.376 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:39.477 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:39.512 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:39.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:39.517 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:49:39.618 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:39.647 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:39.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:39.652 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:39.753 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:39.788 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:39.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:39.792 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:39.892 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:39.902 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:39 smithi092 bash[58855]: cluster 2023-09-23T19:49:38.395626+0000 mgr.y (mgr.25325) 1268 : cluster 0 pgmap v1257: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 603 op/s 2023-09-23T19:49:39.917 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:39.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:39.920 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:40.021 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:40.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:39 smithi016 bash[75004]: cluster 2023-09-23T19:49:38.395626+0000 mgr.y (mgr.25325) 1268 : cluster 0 pgmap v1257: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 603 op/s 2023-09-23T19:49:40.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:39 smithi016 bash[77995]: cluster 2023-09-23T19:49:38.395626+0000 mgr.y (mgr.25325) 1268 : cluster 0 pgmap v1257: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 603 op/s 2023-09-23T19:49:40.050 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:40.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:40.055 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:40.156 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:40.185 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:40.187 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:49:40.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:40.187 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:40.245 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:49:40.275 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:49:40.288 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:40.320 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:40.321 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:49:40.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:40.322 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:40.368 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:49:40.416 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:49:40.424 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:40.445 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:40.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:40.449 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:40.455 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:49:40.494 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:49:40.503 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:49:40.504 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:49:40.504 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:49:40.504 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:49:40.504 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:49:40.504 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:49:40.504 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:49:40.504 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:49:40.550 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:40.573 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:40.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:40.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:40.680 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:40.717 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:40.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:40.718 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:49:40.741 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is set 2023-09-23T19:49:40.783 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:40 smithi016 bash[77995]: audit 2023-09-23T19:49:40.138910+0000 mon.b (mon.1) 54 : audit 0 from='client.? 172.21.15.16:0/1870756579' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:49:40.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:40 smithi016 bash[77995]: audit 2023-09-23T19:49:40.447094+0000 mon.c (mon.2) 201 : audit 1 from='client.? 172.21.15.16:0/3557832334' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:49:40.784 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:40 smithi016 bash[77995]: audit 2023-09-23T19:49:40.448253+0000 mon.a (mon.0) 1581 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:49:40.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:40 smithi016 bash[75004]: audit 2023-09-23T19:49:40.138910+0000 mon.b (mon.1) 54 : audit 0 from='client.? 172.21.15.16:0/1870756579' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:49:40.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:40 smithi016 bash[75004]: audit 2023-09-23T19:49:40.447094+0000 mon.c (mon.2) 201 : audit 1 from='client.? 172.21.15.16:0/3557832334' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:49:40.784 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:40 smithi016 bash[75004]: audit 2023-09-23T19:49:40.448253+0000 mon.a (mon.0) 1581 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:49:40.818 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:40.860 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:40.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:40.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:49:40.964 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:40.994 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:40.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:40.995 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:41.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:40 smithi092 bash[58855]: audit 2023-09-23T19:49:40.138910+0000 mon.b (mon.1) 54 : audit 0 from='client.? 172.21.15.16:0/1870756579' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:49:41.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:40 smithi092 bash[58855]: audit 2023-09-23T19:49:40.447094+0000 mon.c (mon.2) 201 : audit 1 from='client.? 172.21.15.16:0/3557832334' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:49:41.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:40 smithi092 bash[58855]: audit 2023-09-23T19:49:40.448253+0000 mon.a (mon.0) 1581 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:49:41.095 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:41.125 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:41.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:41.130 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:41.231 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:41.271 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:41.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:41.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:41.377 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:41.412 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:41.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:41.414 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:41.515 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:41.546 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:41.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:41.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:49:41.652 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:41.676 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:41.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:41.680 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:49:41.781 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:41.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:41 smithi016 bash[77995]: cluster 2023-09-23T19:49:40.396213+0000 mgr.y (mgr.25325) 1269 : cluster 0 pgmap v1258: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 604 op/s 2023-09-23T19:49:41.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:41 smithi016 bash[77995]: cluster 2023-09-23T19:49:40.715032+0000 mon.a (mon.0) 1582 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:49:41.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:41 smithi016 bash[77995]: cluster 2023-09-23T19:49:40.728615+0000 osd.6 (osd.6) 154 : cluster 3 failed to encode map e223 with expected crc 2023-09-23T19:49:41.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:41 smithi016 bash[77995]: audit 2023-09-23T19:49:40.730686+0000 mon.a (mon.0) 1583 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:49:41.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:41 smithi016 bash[77995]: cluster 2023-09-23T19:49:40.730935+0000 mon.a (mon.0) 1584 : cluster 0 osdmap e223: 8 total, 8 up, 8 in 2023-09-23T19:49:41.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:41 smithi016 bash[77995]: cluster 2023-09-23T19:49:40.743146+0000 osd.7 (osd.7) 142 : cluster 3 failed to encode map e223 with expected crc 2023-09-23T19:49:41.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:41 smithi016 bash[77995]: cluster 2023-09-23T19:49:40.763590+0000 osd.4 (osd.4) 163 : cluster 3 failed to encode map e223 with expected crc 2023-09-23T19:49:41.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:41 smithi016 bash[75004]: cluster 2023-09-23T19:49:40.396213+0000 mgr.y (mgr.25325) 1269 : cluster 0 pgmap v1258: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 604 op/s 2023-09-23T19:49:41.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:41 smithi016 bash[75004]: cluster 2023-09-23T19:49:40.715032+0000 mon.a (mon.0) 1582 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:49:41.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:41 smithi016 bash[75004]: cluster 2023-09-23T19:49:40.728615+0000 osd.6 (osd.6) 154 : cluster 3 failed to encode map e223 with expected crc 2023-09-23T19:49:41.788 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:41 smithi016 bash[75004]: audit 2023-09-23T19:49:40.730686+0000 mon.a (mon.0) 1583 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:49:41.788 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:41 smithi016 bash[75004]: cluster 2023-09-23T19:49:40.730935+0000 mon.a (mon.0) 1584 : cluster 0 osdmap e223: 8 total, 8 up, 8 in 2023-09-23T19:49:41.788 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:41 smithi016 bash[75004]: cluster 2023-09-23T19:49:40.743146+0000 osd.7 (osd.7) 142 : cluster 3 failed to encode map e223 with expected crc 2023-09-23T19:49:41.788 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:41 smithi016 bash[75004]: cluster 2023-09-23T19:49:40.763590+0000 osd.4 (osd.4) 163 : cluster 3 failed to encode map e223 with expected crc 2023-09-23T19:49:41.816 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:41.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:41.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:49:41.925 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:41.935 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:41 smithi092 bash[58855]: cluster 2023-09-23T19:49:40.396213+0000 mgr.y (mgr.25325) 1269 : cluster 0 pgmap v1258: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 604 op/s 2023-09-23T19:49:41.936 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:41 smithi092 bash[58855]: cluster 2023-09-23T19:49:40.715032+0000 mon.a (mon.0) 1582 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:49:41.936 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:41 smithi092 bash[58855]: cluster 2023-09-23T19:49:40.728615+0000 osd.6 (osd.6) 154 : cluster 3 failed to encode map e223 with expected crc 2023-09-23T19:49:41.936 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:41 smithi092 bash[58855]: audit 2023-09-23T19:49:40.730686+0000 mon.a (mon.0) 1583 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:49:41.936 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:41 smithi092 bash[58855]: cluster 2023-09-23T19:49:40.730935+0000 mon.a (mon.0) 1584 : cluster 0 osdmap e223: 8 total, 8 up, 8 in 2023-09-23T19:49:41.936 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:41 smithi092 bash[58855]: cluster 2023-09-23T19:49:40.743146+0000 osd.7 (osd.7) 142 : cluster 3 failed to encode map e223 with expected crc 2023-09-23T19:49:41.937 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:41 smithi092 bash[58855]: cluster 2023-09-23T19:49:40.763590+0000 osd.4 (osd.4) 163 : cluster 3 failed to encode map e223 with expected crc 2023-09-23T19:49:41.950 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:41.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:41.952 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:42.053 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:42.081 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:42.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:42.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:42.189 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:42.229 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:42.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:42.236 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:49:42.336 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:42.364 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:42.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:42.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:42.472 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:42.499 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:42.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:42.506 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:42.607 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:42.634 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:42.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:42.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:42.737 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:42.772 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:42.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:42.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:49:42.876 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:42.905 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:42.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:42.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:43.013 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:43.041 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:43.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:43.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:43.145 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:43.170 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:43.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:43.173 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:43.273 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:43.310 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:43.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:43.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:43.378 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:49:43.384 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:49:43.417 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:43.446 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:43.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:43.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:43.552 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:43.581 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:43.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:43.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:43.688 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:43.730 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:43.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:43.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:43.738 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:43 smithi092 bash[58855]: cluster 2023-09-23T19:49:42.396739+0000 mgr.y (mgr.25325) 1270 : cluster 0 pgmap v1260: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 609 op/s 2023-09-23T19:49:43.738 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:43 smithi092 bash[58855]: audit 2023-09-23T19:49:43.680671+0000 mon.a (mon.0) 1585 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:49:43.739 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:43 smithi092 bash[58855]: audit 2023-09-23T19:49:43.684410+0000 mon.a (mon.0) 1586 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:49:43.835 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:43.873 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:43.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:43.879 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:43.980 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:43.990 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:43 smithi016 bash[75004]: cluster 2023-09-23T19:49:42.396739+0000 mgr.y (mgr.25325) 1270 : cluster 0 pgmap v1260: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 609 op/s 2023-09-23T19:49:43.990 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:43 smithi016 bash[75004]: audit 2023-09-23T19:49:43.680671+0000 mon.a (mon.0) 1585 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:49:43.990 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:43 smithi016 bash[75004]: audit 2023-09-23T19:49:43.684410+0000 mon.a (mon.0) 1586 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:49:43.990 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:43 smithi016 bash[77995]: cluster 2023-09-23T19:49:42.396739+0000 mgr.y (mgr.25325) 1270 : cluster 0 pgmap v1260: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 609 op/s 2023-09-23T19:49:43.990 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:43 smithi016 bash[77995]: audit 2023-09-23T19:49:43.680671+0000 mon.a (mon.0) 1585 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:49:43.990 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:43 smithi016 bash[77995]: audit 2023-09-23T19:49:43.684410+0000 mon.a (mon.0) 1586 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:49:44.009 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:44.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:44.013 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:44.114 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:44.163 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:44.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:44.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:49:44.267 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:44.293 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:44.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:44.296 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:44.397 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:44.421 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:44.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:44.424 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:44.525 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:44.551 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:44.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:44.556 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:44.657 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:44.684 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:44.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:44.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:49:44.789 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:44.817 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:44.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:44.823 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:44.853 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:49:44.897 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T19:49:44.923 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:44.944 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:44.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:44.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:45.049 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:45.107 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:45.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:45.114 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:49:45.214 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:45.244 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:45.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:45.249 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:45.350 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:45.378 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:45.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:45.385 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:45.486 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:45.525 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:45.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:45.530 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:49:45.631 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:45.656 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:45.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:45.664 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:49:45.755 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:45 smithi016 bash[75004]: cluster 2023-09-23T19:49:44.397200+0000 mgr.y (mgr.25325) 1271 : cluster 0 pgmap v1261: 89 pgs: 89 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 610 op/s 2023-09-23T19:49:45.755 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:49:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:49:45] "GET /metrics HTTP/1.1" 200 36306 "" "Prometheus/2.33.4" 2023-09-23T19:49:45.764 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:45.816 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:45.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:45.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:49:45.921 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:45.928 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:45 smithi092 bash[58855]: cluster 2023-09-23T19:49:44.397200+0000 mgr.y (mgr.25325) 1271 : cluster 0 pgmap v1261: 89 pgs: 89 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 610 op/s 2023-09-23T19:49:45.950 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:45.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:45.957 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:46.022 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:45 smithi016 bash[77995]: cluster 2023-09-23T19:49:44.397200+0000 mgr.y (mgr.25325) 1271 : cluster 0 pgmap v1261: 89 pgs: 89 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 610 op/s 2023-09-23T19:49:46.058 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:46.082 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:46.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:46.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:46.190 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:46.213 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:46.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:46.222 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:46.322 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:46.345 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:46.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:46.353 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:49:46.453 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:46.480 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:46.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:46.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:49:46.582 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:46.609 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:46.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:46.612 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:49:46.713 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:46.754 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:46.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:46.758 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:46.859 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:46.884 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:46.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:46.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:46.991 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:47.020 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:47.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:47.022 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:47.123 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:47.157 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:47.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:47.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:47.262 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:47.290 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:47.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:47.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:47.394 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:47.416 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:47.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:47.419 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:49:47.521 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:47.548 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:47.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:47.556 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:47.656 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:47.685 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:47.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:47.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:47.788 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:47.810 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:47.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:47.812 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:47.912 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:47.920 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:47 smithi016 bash[75004]: cluster 2023-09-23T19:49:46.397899+0000 mgr.y (mgr.25325) 1272 : cluster 0 pgmap v1262: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-09-23T19:49:47.920 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:47 smithi016 bash[77995]: cluster 2023-09-23T19:49:46.397899+0000 mgr.y (mgr.25325) 1272 : cluster 0 pgmap v1262: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-09-23T19:49:47.946 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:47.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:47.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:49:48.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:47 smithi092 bash[58855]: cluster 2023-09-23T19:49:46.397899+0000 mgr.y (mgr.25325) 1272 : cluster 0 pgmap v1262: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-09-23T19:49:48.050 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:48.076 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:48.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:48.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:48.185 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:48.220 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:48.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:48.223 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:48.323 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:48.350 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:48.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:48.356 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:48.457 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:48.494 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:48.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:48.507 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:49:48.607 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:48.637 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:48.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:48.640 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:48.741 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:48.766 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:48.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:48.774 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:48.875 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:48.902 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:48.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:48.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:49:49.009 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:49.055 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:49.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:49.058 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:49:49.159 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:49.187 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:49.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:49.190 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:49.291 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:49.327 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:49.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:49.332 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:49:49.433 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:49.461 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:49.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:49.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:49:49.563 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:49.606 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:49.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:49.608 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:49:49.709 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:49.737 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:49.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:49.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:49.840 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:49.850 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:49 smithi016 bash[75004]: cluster 2023-09-23T19:49:48.398366+0000 mgr.y (mgr.25325) 1273 : cluster 0 pgmap v1263: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 625 op/s 2023-09-23T19:49:49.850 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:49 smithi016 bash[77995]: cluster 2023-09-23T19:49:48.398366+0000 mgr.y (mgr.25325) 1273 : cluster 0 pgmap v1263: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 625 op/s 2023-09-23T19:49:49.882 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:49.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:49.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:49.988 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:50.000 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-09-23T19:49:50.000 INFO:teuthology.orchestra.run.smithi016.stdout: "size": 20, 2023-09-23T19:49:50.000 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 600, 2023-09-23T19:49:50.001 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [ 2023-09-23T19:49:50.001 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.001 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45947.0:207465 10.6 e171/144 10:652cdf93:::benchmark_data_smithi016_136365_object97331:head v 171'12792, mlcod=171'12790)", 2023-09-23T19:49:50.001 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:40:28.081262+0000", 2023-09-23T19:49:50.001 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 561.91202239899997, 2023-09-23T19:49:50.001 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.25118891300000001, 2023-09-23T19:49:50.001 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:49:50.001 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:49:50.001 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:49:50.001 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.001 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:49:50.002 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.081262+0000", 2023-09-23T19:49:50.002 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:49:50.002 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.002 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.002 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:49:50.002 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.081262+0000", 2023-09-23T19:49:50.002 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:49:50.002 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.002 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.002 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:49:50.002 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.081261+0000", 2023-09-23T19:49:50.002 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-09-23T19:49:50.002 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.003 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.003 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:49:50.003 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.081268+0000", 2023-09-23T19:49:50.003 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.6630000000000004e-06 2023-09-23T19:49:50.003 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.003 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.003 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:49:50.003 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.081277+0000", 2023-09-23T19:49:50.003 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.0240000000000003e-06 2023-09-23T19:49:50.003 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.003 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.003 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:49:50.004 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.081284+0000", 2023-09-23T19:49:50.004 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.3640000000000001e-06 2023-09-23T19:49:50.004 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.004 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.004 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:49:50.004 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.081327+0000", 2023-09-23T19:49:50.004 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.2799999999999997e-05 2023-09-23T19:49:50.004 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.004 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.004 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:49:50.004 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.081349+0000", 2023-09-23T19:49:50.004 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.1566999999999999e-05 2023-09-23T19:49:50.005 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.005 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.005 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:49:50.005 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.329197+0000", 2023-09-23T19:49:50.005 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.24784800300000001 2023-09-23T19:49:50.005 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.005 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.005 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:49:50.005 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.332451+0000", 2023-09-23T19:49:50.005 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0032547000000000001 2023-09-23T19:49:50.005 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:49:50.005 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:49:50.006 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:49:50.006 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.006 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.006 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45947.0:207464 10.1 e171/144 10:8927d421:::benchmark_data_smithi016_136365_object97330:head v 171'12975, mlcod=171'12973)", 2023-09-23T19:49:50.006 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:40:28.083389+0000", 2023-09-23T19:49:50.006 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 561.90989562300001, 2023-09-23T19:49:50.006 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.24909103899999999, 2023-09-23T19:49:50.006 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:49:50.006 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:49:50.006 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:49:50.006 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.006 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:49:50.007 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.083389+0000", 2023-09-23T19:49:50.007 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:49:50.007 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.007 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.007 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:49:50.007 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.083389+0000", 2023-09-23T19:49:50.007 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:49:50.007 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.007 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.007 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:49:50.007 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.083388+0000", 2023-09-23T19:49:50.007 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:49:50.007 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.008 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.008 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:49:50.008 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.083397+0000", 2023-09-23T19:49:50.008 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.8049999999999996e-06 2023-09-23T19:49:50.008 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.008 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.008 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:49:50.008 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.083409+0000", 2023-09-23T19:49:50.010 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2189e-05 2023-09-23T19:49:50.010 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.010 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.010 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:49:50.010 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.083417+0000", 2023-09-23T19:49:50.010 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.3170000000000004e-06 2023-09-23T19:49:50.010 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.010 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.010 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:49:50.011 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.083467+0000", 2023-09-23T19:49:50.011 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.0550000000000002e-05 2023-09-23T19:49:50.011 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.011 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.011 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:49:50.011 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.083488+0000", 2023-09-23T19:49:50.011 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.1146999999999999e-05 2023-09-23T19:49:50.011 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.011 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.011 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:49:50.011 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.332448+0000", 2023-09-23T19:49:50.011 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.24895941499999999 2023-09-23T19:49:50.012 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.012 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.012 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:49:50.012 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.332480+0000", 2023-09-23T19:49:50.012 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.2663999999999998e-05 2023-09-23T19:49:50.012 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:49:50.012 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:49:50.012 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:49:50.012 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.012 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.012 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45947.0:207469 10.6 e171/144 10:6cee13bb:::benchmark_data_smithi016_136365_object97335:head v 171'12793, mlcod=171'12790)", 2023-09-23T19:49:50.012 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:40:28.086193+0000", 2023-09-23T19:49:50.013 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 561.90709235300005, 2023-09-23T19:49:50.013 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.25814531200000002, 2023-09-23T19:49:50.013 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:49:50.013 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:49:50.013 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:49:50.013 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.013 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:49:50.013 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.086193+0000", 2023-09-23T19:49:50.013 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:49:50.013 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.013 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.013 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:49:50.014 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.086193+0000", 2023-09-23T19:49:50.014 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:49:50.014 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.014 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.014 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:49:50.014 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.086191+0000", 2023-09-23T19:49:50.014 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-09-23T19:49:50.014 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.014 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.014 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:49:50.014 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.086202+0000", 2023-09-23T19:49:50.014 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1627999999999999e-05 2023-09-23T19:49:50.015 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.015 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.015 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:49:50.015 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.328356+0000", 2023-09-23T19:49:50.015 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.242153286 2023-09-23T19:49:50.015 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.015 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.015 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:49:50.015 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.328384+0000", 2023-09-23T19:49:50.015 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.8586999999999999e-05 2023-09-23T19:49:50.015 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.015 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.015 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:49:50.016 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.332530+0000", 2023-09-23T19:49:50.016 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.004145475 2023-09-23T19:49:50.016 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.016 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.016 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:49:50.016 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.332552+0000", 2023-09-23T19:49:50.016 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.2208000000000002e-05 2023-09-23T19:49:50.016 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.016 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.016 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:49:50.016 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.344289+0000", 2023-09-23T19:49:50.016 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.011736961000000001 2023-09-23T19:49:50.017 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.017 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.017 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:49:50.017 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.344338+0000", 2023-09-23T19:49:50.017 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.8965000000000003e-05 2023-09-23T19:49:50.017 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:49:50.017 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:49:50.017 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:49:50.017 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.017 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.017 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.45947.0:207474 10.4 10:2b46c204:::benchmark_data_smithi016_136365_object97340:head [delete] snapc 0=[] ondisk+write+known_if_redirected e171)", 2023-09-23T19:49:50.017 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:40:28.088271+0000", 2023-09-23T19:49:50.017 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 561.905013822, 2023-09-23T19:49:50.018 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.26161290700000001, 2023-09-23T19:49:50.018 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:49:50.018 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:49:50.018 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:49:50.018 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.45947", 2023-09-23T19:49:50.018 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/4039191752", 2023-09-23T19:49:50.018 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 207474 2023-09-23T19:49:50.018 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.018 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:49:50.018 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.018 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:49:50.018 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.088271+0000", 2023-09-23T19:49:50.019 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:49:50.019 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.019 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.019 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:49:50.019 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.088271+0000", 2023-09-23T19:49:50.019 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:49:50.019 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.019 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.019 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:49:50.019 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.088269+0000", 2023-09-23T19:49:50.019 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-09-23T19:49:50.019 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.020 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.020 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:49:50.020 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.088288+0000", 2023-09-23T19:49:50.020 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.8822e-05 2023-09-23T19:49:50.020 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.020 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.020 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:49:50.020 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.328378+0000", 2023-09-23T19:49:50.020 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.24009019000000001 2023-09-23T19:49:50.020 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.020 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.020 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:49:50.021 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.328411+0000", 2023-09-23T19:49:50.021 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.2830000000000002e-05 2023-09-23T19:49:50.021 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.021 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.021 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:49:50.021 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.328682+0000", 2023-09-23T19:49:50.021 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00027039000000000002 2023-09-23T19:49:50.021 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.021 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.021 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:49:50.021 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.328832+0000", 2023-09-23T19:49:50.021 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000150736 2023-09-23T19:49:50.022 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.022 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.022 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 3,4", 2023-09-23T19:49:50.022 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.328968+0000", 2023-09-23T19:49:50.022 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00013575800000000001 2023-09-23T19:49:50.022 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.022 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.022 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:49:50.022 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.336992+0000", 2023-09-23T19:49:50.022 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0080238370000000007 2023-09-23T19:49:50.022 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.022 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.023 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:49:50.023 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.349737+0000", 2023-09-23T19:49:50.023 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.012745126000000001 2023-09-23T19:49:50.023 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.023 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.023 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:49:50.023 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.349791+0000", 2023-09-23T19:49:50.023 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.3619999999999998e-05 2023-09-23T19:49:50.023 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.023 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.023 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:49:50.023 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.349837+0000", 2023-09-23T19:49:50.024 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.6062e-05 2023-09-23T19:49:50.024 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.024 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.024 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:49:50.024 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.349884+0000", 2023-09-23T19:49:50.024 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.7209000000000002e-05 2023-09-23T19:49:50.024 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:49:50.024 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:49:50.024 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:49:50.024 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.024 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.024 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.45947.0:207476 10.c e171/144 10:32805061:::benchmark_data_smithi016_136365_object97342:head v 171'12959, mlcod=171'12957)", 2023-09-23T19:49:50.025 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:40:28.092017+0000", 2023-09-23T19:49:50.025 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 561.90126751800005, 2023-09-23T19:49:50.025 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.257763352, 2023-09-23T19:49:50.025 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:49:50.025 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:49:50.025 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:49:50.025 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.025 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:49:50.025 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.092017+0000", 2023-09-23T19:49:50.025 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:49:50.025 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.025 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.026 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:49:50.026 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.092017+0000", 2023-09-23T19:49:50.026 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:49:50.026 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.026 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.026 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:49:50.026 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.092016+0000", 2023-09-23T19:49:50.026 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-09-23T19:49:50.026 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.026 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.026 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:49:50.027 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.092027+0000", 2023-09-23T19:49:50.027 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0849999999999999e-05 2023-09-23T19:49:50.027 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.027 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.027 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:49:50.027 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.328377+0000", 2023-09-23T19:49:50.027 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.23635049399999999 2023-09-23T19:49:50.027 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.027 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.027 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:49:50.027 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.328397+0000", 2023-09-23T19:49:50.027 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.9259999999999999e-05 2023-09-23T19:49:50.027 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.028 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.028 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:49:50.028 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.328543+0000", 2023-09-23T19:49:50.028 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00014627499999999999 2023-09-23T19:49:50.028 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.028 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.028 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:49:50.028 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.328592+0000", 2023-09-23T19:49:50.028 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.9187000000000002e-05 2023-09-23T19:49:50.028 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.028 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.028 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:49:50.029 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.349742+0000", 2023-09-23T19:49:50.029 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.021149748999999999 2023-09-23T19:49:50.029 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.029 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.029 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:49:50.029 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:40:28.349781+0000", 2023-09-23T19:49:50.029 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.8847999999999998e-05 2023-09-23T19:49:50.029 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:49:50.029 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:49:50.029 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:49:50.029 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.029 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.029 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.36480.0:11084 12.3 12:c2885d86:::benchmark_data_smithi016_181952_object11083:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e211)", 2023-09-23T19:49:50.030 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:46:38.748265+0000", 2023-09-23T19:49:50.030 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 191.245019692, 2023-09-23T19:49:50.030 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.30349094799999998, 2023-09-23T19:49:50.030 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:49:50.030 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:49:50.030 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:49:50.030 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.36480", 2023-09-23T19:49:50.030 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1035295215", 2023-09-23T19:49:50.030 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 11084 2023-09-23T19:49:50.030 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.030 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:49:50.030 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.031 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:49:50.031 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:46:38.748265+0000", 2023-09-23T19:49:50.031 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:49:50.031 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.031 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.031 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:49:50.031 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:46:38.748265+0000", 2023-09-23T19:49:50.031 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:49:50.031 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.031 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.031 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:49:50.031 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:46:38.748263+0000", 2023-09-23T19:49:50.031 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-09-23T19:49:50.032 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.032 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.032 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:49:50.032 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:46:38.748291+0000", 2023-09-23T19:49:50.032 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.7328e-05 2023-09-23T19:49:50.032 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.032 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.032 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:49:50.032 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:46:38.748311+0000", 2023-09-23T19:49:50.032 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.9748000000000001e-05 2023-09-23T19:49:50.032 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.032 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.033 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:49:50.033 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:46:38.748320+0000", 2023-09-23T19:49:50.033 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.4399999999999994e-06 2023-09-23T19:49:50.033 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.033 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.033 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:49:50.033 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:46:38.748368+0000", 2023-09-23T19:49:50.033 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.7969000000000001e-05 2023-09-23T19:49:50.033 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.033 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.033 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:49:50.033 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:46:38.748472+0000", 2023-09-23T19:49:50.033 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000103931 2023-09-23T19:49:50.034 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.034 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.034 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 3,7", 2023-09-23T19:49:50.034 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:46:38.748571+0000", 2023-09-23T19:49:50.034 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.8722999999999997e-05 2023-09-23T19:49:50.034 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.034 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.034 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:49:50.034 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:46:38.749370+0000", 2023-09-23T19:49:50.034 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00079896399999999999 2023-09-23T19:49:50.034 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.034 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.035 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:49:50.035 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:46:38.755663+0000", 2023-09-23T19:49:50.035 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0062930599999999996 2023-09-23T19:49:50.035 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.035 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.035 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:49:50.035 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:46:39.051614+0000", 2023-09-23T19:49:50.035 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.29595117100000001 2023-09-23T19:49:50.035 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.035 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.035 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:49:50.035 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:46:39.051694+0000", 2023-09-23T19:49:50.036 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.0597000000000001e-05 2023-09-23T19:49:50.036 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.036 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.036 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:49:50.036 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:46:39.051756+0000", 2023-09-23T19:49:50.036 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.1727e-05 2023-09-23T19:49:50.036 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:49:50.036 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:49:50.036 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:49:50.036 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.036 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.036 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.36480.0:11105 12.3 12:c4f216e6:::benchmark_data_smithi016_181952_object11104:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e211)", 2023-09-23T19:49:50.036 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:46:38.760906+0000", 2023-09-23T19:49:50.037 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 191.232379237, 2023-09-23T19:49:50.037 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.29100776499999997, 2023-09-23T19:49:50.037 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:49:50.037 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:49:50.037 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:49:50.037 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.36480", 2023-09-23T19:49:50.037 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1035295215", 2023-09-23T19:49:50.037 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 11105 2023-09-23T19:49:50.037 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.037 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:49:50.037 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.037 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:49:50.038 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:46:38.760906+0000", 2023-09-23T19:49:50.038 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:49:50.038 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.038 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.038 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:49:50.038 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:46:38.760906+0000", 2023-09-23T19:49:50.038 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:49:50.038 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.038 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.038 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:49:50.038 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:46:38.760904+0000", 2023-09-23T19:49:50.039 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-09-23T19:49:50.039 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.039 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.039 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:49:50.039 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:46:38.760930+0000", 2023-09-23T19:49:50.039 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.6455999999999999e-05 2023-09-23T19:49:50.039 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.039 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.039 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:49:50.039 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:46:38.760945+0000", 2023-09-23T19:49:50.039 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.4666e-05 2023-09-23T19:49:50.039 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.039 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.040 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:49:50.040 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:46:38.760954+0000", 2023-09-23T19:49:50.040 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.3489999999999998e-06 2023-09-23T19:49:50.040 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.040 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.040 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:49:50.040 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:46:38.761067+0000", 2023-09-23T19:49:50.040 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000113286 2023-09-23T19:49:50.040 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.040 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.040 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:49:50.040 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:46:38.761215+0000", 2023-09-23T19:49:50.041 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00014752400000000001 2023-09-23T19:49:50.041 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.041 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.041 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 3,7", 2023-09-23T19:49:50.041 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:46:38.761373+0000", 2023-09-23T19:49:50.041 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000158422 2023-09-23T19:49:50.041 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.041 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.041 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:49:50.041 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:46:38.762797+0000", 2023-09-23T19:49:50.041 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0014238320000000001 2023-09-23T19:49:50.041 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.042 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.042 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:49:50.042 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:46:38.764402+0000", 2023-09-23T19:49:50.042 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0016047710000000001 2023-09-23T19:49:50.042 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.042 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.042 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:49:50.042 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:46:39.051815+0000", 2023-09-23T19:49:50.042 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.28741314200000001 2023-09-23T19:49:50.042 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.042 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.042 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:49:50.042 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:46:39.051871+0000", 2023-09-23T19:49:50.043 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.6178999999999999e-05 2023-09-23T19:49:50.043 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.043 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.043 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:49:50.043 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:46:39.051913+0000", 2023-09-23T19:49:50.043 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.2150999999999997e-05 2023-09-23T19:49:50.043 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:49:50.043 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:49:50.043 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:49:50.043 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.043 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.043 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.36480.0:22016 12.c 12:358f85cd:::benchmark_data_smithi016_181952_object22015:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e211)", 2023-09-23T19:49:50.043 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:46:44.757283+0000", 2023-09-23T19:49:50.044 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 185.23600180899999, 2023-09-23T19:49:50.044 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.21635954900000001, 2023-09-23T19:49:50.044 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:49:50.044 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:49:50.044 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:49:50.044 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.36480", 2023-09-23T19:49:50.044 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1035295215", 2023-09-23T19:49:50.044 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 22016 2023-09-23T19:49:50.044 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.044 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:49:50.044 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.044 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:49:50.045 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:46:44.757283+0000", 2023-09-23T19:49:50.045 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:49:50.045 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.045 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.045 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:49:50.045 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:46:44.757283+0000", 2023-09-23T19:49:50.045 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:49:50.045 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.045 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.045 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:49:50.045 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:46:44.757281+0000", 2023-09-23T19:49:50.045 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-09-23T19:49:50.046 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.046 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.046 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:49:50.046 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:46:44.757320+0000", 2023-09-23T19:49:50.046 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.8968999999999999e-05 2023-09-23T19:49:50.046 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.046 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.046 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:49:50.046 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:46:44.757430+0000", 2023-09-23T19:49:50.046 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00010964 2023-09-23T19:49:50.046 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.046 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.046 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:49:50.047 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:46:44.757438+0000", 2023-09-23T19:49:50.047 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.0690000000000002e-06 2023-09-23T19:49:50.047 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.047 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.047 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:49:50.047 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:46:44.757560+0000", 2023-09-23T19:49:50.047 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00012222200000000001 2023-09-23T19:49:50.047 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.047 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.047 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:49:50.047 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:46:44.757713+0000", 2023-09-23T19:49:50.047 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000153041 2023-09-23T19:49:50.048 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.048 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.048 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 2,5", 2023-09-23T19:49:50.048 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:46:44.759252+0000", 2023-09-23T19:49:50.048 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0015384750000000001 2023-09-23T19:49:50.048 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.048 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.048 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:49:50.048 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:46:44.762768+0000", 2023-09-23T19:49:50.048 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0035161279999999999 2023-09-23T19:49:50.048 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.048 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.048 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:49:50.049 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:46:44.762781+0000", 2023-09-23T19:49:50.049 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2840000000000001e-05 2023-09-23T19:49:50.049 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.049 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.049 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:49:50.049 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:46:44.973513+0000", 2023-09-23T19:49:50.049 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.210732115 2023-09-23T19:49:50.049 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.049 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.049 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:49:50.049 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:46:44.973589+0000", 2023-09-23T19:49:50.049 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.5661000000000006e-05 2023-09-23T19:49:50.050 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.050 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.050 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:49:50.050 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:46:44.973643+0000", 2023-09-23T19:49:50.050 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.4054000000000001e-05 2023-09-23T19:49:50.050 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:49:50.050 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:49:50.050 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:49:50.050 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.050 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.050 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.36480.0:53532 12.c 12:3c58b8d5:::benchmark_data_smithi016_181952_object53531:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e213)", 2023-09-23T19:49:50.050 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:47:09.166877+0000", 2023-09-23T19:49:50.051 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 160.82640825999999, 2023-09-23T19:49:50.051 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.156107255, 2023-09-23T19:49:50.051 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:49:50.051 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:49:50.051 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:49:50.051 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.36480", 2023-09-23T19:49:50.051 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1035295215", 2023-09-23T19:49:50.051 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 53532 2023-09-23T19:49:50.051 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.051 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:49:50.051 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.051 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:49:50.051 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:09.166877+0000", 2023-09-23T19:49:50.052 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:49:50.052 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.052 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.052 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:49:50.052 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:09.166877+0000", 2023-09-23T19:49:50.052 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:49:50.052 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.052 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.052 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:49:50.052 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:09.166876+0000", 2023-09-23T19:49:50.052 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T19:49:50.052 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.052 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.053 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:49:50.053 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:09.166899+0000", 2023-09-23T19:49:50.053 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.3524e-05 2023-09-23T19:49:50.053 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.053 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.053 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:49:50.053 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:09.166907+0000", 2023-09-23T19:49:50.053 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.7700000000000001e-06 2023-09-23T19:49:50.053 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.053 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.053 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:49:50.054 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:09.166913+0000", 2023-09-23T19:49:50.054 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.5779999999999998e-06 2023-09-23T19:49:50.054 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.054 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.054 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:49:50.054 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:09.166975+0000", 2023-09-23T19:49:50.054 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.1990999999999997e-05 2023-09-23T19:49:50.054 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.054 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.054 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:49:50.054 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:09.167083+0000", 2023-09-23T19:49:50.054 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00010785600000000001 2023-09-23T19:49:50.055 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.055 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.055 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 2,5", 2023-09-23T19:49:50.055 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:09.167166+0000", 2023-09-23T19:49:50.055 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.3395999999999999e-05 2023-09-23T19:49:50.055 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.055 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.055 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:49:50.055 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:09.169069+0000", 2023-09-23T19:49:50.055 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0019033909999999999 2023-09-23T19:49:50.055 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.055 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.056 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:49:50.056 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:09.169818+0000", 2023-09-23T19:49:50.056 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00074889399999999995 2023-09-23T19:49:50.056 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.056 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.056 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:49:50.056 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:09.322889+0000", 2023-09-23T19:49:50.056 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.153071085 2023-09-23T19:49:50.056 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.056 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.056 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:49:50.056 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:09.322942+0000", 2023-09-23T19:49:50.057 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.2751999999999999e-05 2023-09-23T19:49:50.057 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.057 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.057 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:49:50.057 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:09.322984+0000", 2023-09-23T19:49:50.057 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.1672999999999999e-05 2023-09-23T19:49:50.057 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:49:50.057 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:49:50.057 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:49:50.057 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.057 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.057 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.36480.0:54968 12.c 12:31eff732:::benchmark_data_smithi016_181952_object54967:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e213)", 2023-09-23T19:49:50.057 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:47:10.658202+0000", 2023-09-23T19:49:50.058 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 159.33508298999999, 2023-09-23T19:49:50.058 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.16324369499999999, 2023-09-23T19:49:50.058 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:49:50.058 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:49:50.058 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:49:50.058 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.36480", 2023-09-23T19:49:50.058 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1035295215", 2023-09-23T19:49:50.058 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 54968 2023-09-23T19:49:50.058 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.058 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:49:50.058 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.058 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:49:50.059 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:10.658202+0000", 2023-09-23T19:49:50.059 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:49:50.059 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.059 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.059 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:49:50.059 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:10.658202+0000", 2023-09-23T19:49:50.059 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:49:50.059 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.059 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.059 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:49:50.059 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:10.658200+0000", 2023-09-23T19:49:50.059 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-09-23T19:49:50.060 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.060 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.060 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:49:50.060 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:10.658224+0000", 2023-09-23T19:49:50.060 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.3997000000000001e-05 2023-09-23T19:49:50.060 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.060 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.060 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:49:50.060 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:10.658236+0000", 2023-09-23T19:49:50.060 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1925e-05 2023-09-23T19:49:50.060 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.060 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.061 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:49:50.061 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:10.658243+0000", 2023-09-23T19:49:50.061 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.7120000000000003e-06 2023-09-23T19:49:50.061 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.061 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.061 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:49:50.061 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:10.658283+0000", 2023-09-23T19:49:50.061 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.9610999999999997e-05 2023-09-23T19:49:50.061 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.061 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.061 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:49:50.061 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:10.804576+0000", 2023-09-23T19:49:50.062 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.14629372199999999 2023-09-23T19:49:50.062 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.062 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.062 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 2,5", 2023-09-23T19:49:50.062 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:10.804997+0000", 2023-09-23T19:49:50.062 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00042067899999999997 2023-09-23T19:49:50.062 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.062 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.062 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:49:50.062 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:10.811886+0000", 2023-09-23T19:49:50.062 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0068890959999999999 2023-09-23T19:49:50.062 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.063 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.063 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:49:50.063 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:10.811894+0000", 2023-09-23T19:49:50.063 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.5850000000000002e-06 2023-09-23T19:49:50.063 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.063 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.063 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:49:50.063 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:10.821331+0000", 2023-09-23T19:49:50.063 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0094371360000000005 2023-09-23T19:49:50.063 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.063 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.063 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:49:50.064 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:10.821392+0000", 2023-09-23T19:49:50.064 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.1142000000000005e-05 2023-09-23T19:49:50.064 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.064 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.064 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:49:50.064 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:10.821446+0000", 2023-09-23T19:49:50.064 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.3588000000000003e-05 2023-09-23T19:49:50.064 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:49:50.064 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:49:50.064 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:49:50.064 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.064 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.064 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.36480.0:64348 12.c 12:3a618180:::benchmark_data_smithi016_181952_object64347:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e214)", 2023-09-23T19:49:50.065 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:47:19.780043+0000", 2023-09-23T19:49:50.065 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 150.21324170400001, 2023-09-23T19:49:50.065 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.30939561199999999, 2023-09-23T19:49:50.065 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:49:50.065 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:49:50.065 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:49:50.065 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.36480", 2023-09-23T19:49:50.065 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1035295215", 2023-09-23T19:49:50.065 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 64348 2023-09-23T19:49:50.065 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.065 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:49:50.065 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.065 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:49:50.066 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:19.780043+0000", 2023-09-23T19:49:50.066 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:49:50.066 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.066 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.066 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:49:50.066 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:19.780043+0000", 2023-09-23T19:49:50.066 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:49:50.066 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.066 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.066 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:49:50.066 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:19.780043+0000", 2023-09-23T19:49:50.067 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T19:49:50.067 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.067 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.067 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:49:50.067 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:19.780135+0000", 2023-09-23T19:49:50.067 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.2516999999999995e-05 2023-09-23T19:49:50.067 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.067 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.067 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:49:50.067 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:19.780145+0000", 2023-09-23T19:49:50.067 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0428000000000001e-05 2023-09-23T19:49:50.067 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.067 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.068 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:49:50.068 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:19.780154+0000", 2023-09-23T19:49:50.068 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.2120000000000003e-06 2023-09-23T19:49:50.068 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.068 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.068 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:49:50.068 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:19.780226+0000", 2023-09-23T19:49:50.068 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.2102999999999997e-05 2023-09-23T19:49:50.068 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.068 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.068 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:49:50.068 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:19.780394+0000", 2023-09-23T19:49:50.069 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00016856999999999999 2023-09-23T19:49:50.069 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.069 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.069 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 2,5", 2023-09-23T19:49:50.069 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:19.780539+0000", 2023-09-23T19:49:50.069 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00014424199999999999 2023-09-23T19:49:50.069 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.069 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.069 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:49:50.069 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:19.785177+0000", 2023-09-23T19:49:50.069 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0046380070000000004 2023-09-23T19:49:50.069 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.069 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.070 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:49:50.070 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:19.786259+0000", 2023-09-23T19:49:50.070 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0010824389999999999 2023-09-23T19:49:50.070 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.070 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.070 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:49:50.070 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:20.089276+0000", 2023-09-23T19:49:50.070 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.303017288 2023-09-23T19:49:50.070 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.070 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.070 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:49:50.071 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:20.089363+0000", 2023-09-23T19:49:50.071 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.7078999999999998e-05 2023-09-23T19:49:50.071 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.071 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.071 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:49:50.071 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:20.089439+0000", 2023-09-23T19:49:50.071 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.5337000000000001e-05 2023-09-23T19:49:50.071 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:49:50.071 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:49:50.071 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:49:50.071 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.071 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.072 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.36480.0:64351 12.c 12:3b15c5e4:::benchmark_data_smithi016_181952_object64350:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e214)", 2023-09-23T19:49:50.072 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:47:19.781724+0000", 2023-09-23T19:49:50.072 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 150.21156070399999, 2023-09-23T19:49:50.072 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.307914886, 2023-09-23T19:49:50.072 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:49:50.072 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:49:50.072 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:49:50.072 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.36480", 2023-09-23T19:49:50.072 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1035295215", 2023-09-23T19:49:50.072 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 64351 2023-09-23T19:49:50.072 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.072 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:49:50.072 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.073 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:49:50.073 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:19.781724+0000", 2023-09-23T19:49:50.073 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:49:50.073 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.073 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.073 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:49:50.073 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:19.781724+0000", 2023-09-23T19:49:50.073 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:49:50.073 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.073 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.073 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:49:50.073 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:19.781723+0000", 2023-09-23T19:49:50.074 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-09-23T19:49:50.074 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.074 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.074 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:49:50.074 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:19.781781+0000", 2023-09-23T19:49:50.074 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.8628000000000002e-05 2023-09-23T19:49:50.074 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.074 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.074 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:49:50.074 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:19.781795+0000", 2023-09-23T19:49:50.074 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3696000000000001e-05 2023-09-23T19:49:50.075 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.075 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.075 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:49:50.075 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:19.781803+0000", 2023-09-23T19:49:50.075 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.2600000000000005e-06 2023-09-23T19:49:50.075 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.075 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.075 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:49:50.075 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:19.783142+0000", 2023-09-23T19:49:50.075 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0013387220000000001 2023-09-23T19:49:50.075 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.075 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.075 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:49:50.076 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:19.783274+0000", 2023-09-23T19:49:50.076 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00013161999999999999 2023-09-23T19:49:50.076 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.076 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.076 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 2,5", 2023-09-23T19:49:50.076 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:19.783375+0000", 2023-09-23T19:49:50.076 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0001013 2023-09-23T19:49:50.076 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.076 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.076 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:49:50.076 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:19.785392+0000", 2023-09-23T19:49:50.077 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0020175050000000002 2023-09-23T19:49:50.077 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.077 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.077 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:49:50.077 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:19.786495+0000", 2023-09-23T19:49:50.077 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0011029989999999999 2023-09-23T19:49:50.077 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.077 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.077 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:49:50.077 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:20.089522+0000", 2023-09-23T19:49:50.077 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.30302667 2023-09-23T19:49:50.077 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.077 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.078 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:49:50.078 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:20.089586+0000", 2023-09-23T19:49:50.078 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.3427000000000001e-05 2023-09-23T19:49:50.078 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.078 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.078 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:49:50.078 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:20.089639+0000", 2023-09-23T19:49:50.078 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.3463e-05 2023-09-23T19:49:50.078 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:49:50.078 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:49:50.078 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:49:50.078 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.079 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.079 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.36480.0:71269 12.3 12:cf2269b4:::benchmark_data_smithi016_181952_object71268:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e214)", 2023-09-23T19:49:50.079 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:47:26.815563+0000", 2023-09-23T19:49:50.079 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 143.177721531, 2023-09-23T19:49:50.079 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.20306829100000001, 2023-09-23T19:49:50.079 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:49:50.079 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:49:50.079 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:49:50.079 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.36480", 2023-09-23T19:49:50.079 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1035295215", 2023-09-23T19:49:50.079 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 71269 2023-09-23T19:49:50.079 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.080 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:49:50.080 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.080 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:49:50.080 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:26.815563+0000", 2023-09-23T19:49:50.080 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:49:50.080 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.080 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.080 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:49:50.080 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:26.815563+0000", 2023-09-23T19:49:50.080 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:49:50.080 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.080 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.080 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:49:50.081 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:26.815562+0000", 2023-09-23T19:49:50.081 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-09-23T19:49:50.081 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.081 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.081 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:49:50.081 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:26.815581+0000", 2023-09-23T19:49:50.081 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.9847000000000002e-05 2023-09-23T19:49:50.081 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.081 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.081 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:49:50.081 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:26.815603+0000", 2023-09-23T19:49:50.081 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.1423000000000001e-05 2023-09-23T19:49:50.082 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.082 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.082 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:49:50.082 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:26.815614+0000", 2023-09-23T19:49:50.082 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1542000000000001e-05 2023-09-23T19:49:50.082 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.082 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.082 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:49:50.082 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:26.816011+0000", 2023-09-23T19:49:50.082 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000397041 2023-09-23T19:49:50.082 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.082 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.082 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:49:50.083 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:26.816342+0000", 2023-09-23T19:49:50.083 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00033061400000000002 2023-09-23T19:49:50.083 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.083 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.083 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 3,7", 2023-09-23T19:49:50.083 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:26.816763+0000", 2023-09-23T19:49:50.083 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00042107000000000001 2023-09-23T19:49:50.083 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.083 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.083 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:49:50.083 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:26.821704+0000", 2023-09-23T19:49:50.083 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0049404330000000002 2023-09-23T19:49:50.084 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.084 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.084 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:49:50.084 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:26.823530+0000", 2023-09-23T19:49:50.084 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001826512 2023-09-23T19:49:50.084 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.084 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.084 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:49:50.084 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:27.018442+0000", 2023-09-23T19:49:50.084 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.194912371 2023-09-23T19:49:50.084 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.084 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.085 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:49:50.085 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:27.018582+0000", 2023-09-23T19:49:50.085 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000139603 2023-09-23T19:49:50.085 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.085 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.085 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:49:50.085 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:47:27.018632+0000", 2023-09-23T19:49:50.085 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.9585e-05 2023-09-23T19:49:50.085 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:49:50.085 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:49:50.085 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:49:50.085 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.086 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.086 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.36480.0:167687 12.a e223/207 12:5cb9be1f:::benchmark_data_smithi016_181952_object65006:head v 223'10440, mlcod=223'10437)", 2023-09-23T19:49:50.086 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:49:41.874042+0000", 2023-09-23T19:49:50.086 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 8.1192426900000001, 2023-09-23T19:49:50.086 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.166475553, 2023-09-23T19:49:50.086 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:49:50.086 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:49:50.086 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:49:50.086 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.086 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:49:50.086 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:41.874042+0000", 2023-09-23T19:49:50.086 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:49:50.087 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.087 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.087 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:49:50.087 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:41.874042+0000", 2023-09-23T19:49:50.087 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:49:50.087 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.087 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.087 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:49:50.087 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:41.874041+0000", 2023-09-23T19:49:50.087 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:49:50.087 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.087 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.087 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:49:50.088 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:41.874049+0000", 2023-09-23T19:49:50.088 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.7680000000000008e-06 2023-09-23T19:49:50.088 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.088 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.088 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:49:50.088 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:41.874061+0000", 2023-09-23T19:49:50.088 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2456999999999999e-05 2023-09-23T19:49:50.088 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.088 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.088 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:49:50.088 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:41.874069+0000", 2023-09-23T19:49:50.089 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.8650000000000001e-06 2023-09-23T19:49:50.089 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.089 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.089 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:49:50.089 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:41.874144+0000", 2023-09-23T19:49:50.089 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.5233000000000003e-05 2023-09-23T19:49:50.089 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.089 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.089 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:49:50.089 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:41.874165+0000", 2023-09-23T19:49:50.089 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.0499000000000001e-05 2023-09-23T19:49:50.089 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.089 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.090 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:49:50.090 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:42.040479+0000", 2023-09-23T19:49:50.090 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.166313761 2023-09-23T19:49:50.090 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.090 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.090 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:49:50.090 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:42.040518+0000", 2023-09-23T19:49:50.090 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.9134000000000002e-05 2023-09-23T19:49:50.090 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:49:50.090 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:49:50.090 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:49:50.090 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.091 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.091 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.36480.0:167702 12.c 12:3196f19e:::benchmark_data_smithi016_181952_object65021:head [delete] snapc 0=[] ondisk+write+known_if_redirected e223)", 2023-09-23T19:49:50.091 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:49:41.880810+0000", 2023-09-23T19:49:50.091 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 8.1124745259999997, 2023-09-23T19:49:50.091 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.15995031900000001, 2023-09-23T19:49:50.091 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:49:50.091 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:49:50.091 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:49:50.091 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.36480", 2023-09-23T19:49:50.091 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1035295215", 2023-09-23T19:49:50.091 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 167702 2023-09-23T19:49:50.091 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.092 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:49:50.092 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.092 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:49:50.092 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:41.880810+0000", 2023-09-23T19:49:50.092 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:49:50.092 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.092 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.092 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:49:50.092 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:41.880810+0000", 2023-09-23T19:49:50.092 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:49:50.092 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.092 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.093 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:49:50.093 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:41.880809+0000", 2023-09-23T19:49:50.093 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:49:50.093 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.093 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.093 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:49:50.093 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:41.880814+0000", 2023-09-23T19:49:50.093 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.814e-06 2023-09-23T19:49:50.093 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.093 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.093 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:49:50.093 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:41.880825+0000", 2023-09-23T19:49:50.094 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0857e-05 2023-09-23T19:49:50.094 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.094 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.094 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:49:50.094 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:41.880832+0000", 2023-09-23T19:49:50.094 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.2629999999999997e-06 2023-09-23T19:49:50.094 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.094 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.094 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:49:50.094 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:41.880867+0000", 2023-09-23T19:49:50.094 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.5148999999999998e-05 2023-09-23T19:49:50.094 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.095 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.095 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:49:50.095 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:41.880981+0000", 2023-09-23T19:49:50.095 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00011391 2023-09-23T19:49:50.095 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.095 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.095 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 2,5", 2023-09-23T19:49:50.095 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:41.881085+0000", 2023-09-23T19:49:50.095 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00010414800000000001 2023-09-23T19:49:50.095 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.095 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.095 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:49:50.095 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:42.040453+0000", 2023-09-23T19:49:50.096 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.159367591 2023-09-23T19:49:50.096 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.096 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.096 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:49:50.096 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:42.040646+0000", 2023-09-23T19:49:50.096 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000193024 2023-09-23T19:49:50.096 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.096 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.096 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:49:50.096 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:42.040680+0000", 2023-09-23T19:49:50.096 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.4326999999999998e-05 2023-09-23T19:49:50.096 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.097 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.097 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:49:50.097 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:42.040721+0000", 2023-09-23T19:49:50.097 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.0460999999999998e-05 2023-09-23T19:49:50.097 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.097 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.097 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:49:50.097 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:42.040761+0000", 2023-09-23T19:49:50.097 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.9925999999999997e-05 2023-09-23T19:49:50.097 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:49:50.097 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:49:50.097 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:49:50.097 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.098 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.098 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.36480.0:167701 12.6 e223/207 12:669f4b43:::benchmark_data_smithi016_181952_object65020:head v 223'10680, mlcod=223'10678)", 2023-09-23T19:49:50.098 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:49:41.883807+0000", 2023-09-23T19:49:50.098 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 8.1094777370000006, 2023-09-23T19:49:50.098 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.15670276599999999, 2023-09-23T19:49:50.098 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:49:50.098 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:49:50.098 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:49:50.098 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.098 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:49:50.098 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:41.883807+0000", 2023-09-23T19:49:50.098 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:49:50.099 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.099 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.099 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:49:50.099 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:41.883807+0000", 2023-09-23T19:49:50.099 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:49:50.099 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.099 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.099 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:49:50.099 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:41.883806+0000", 2023-09-23T19:49:50.099 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-09-23T19:49:50.099 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.099 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.100 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:49:50.100 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:41.883813+0000", 2023-09-23T19:49:50.100 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.7399999999999998e-06 2023-09-23T19:49:50.100 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.100 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.100 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:49:50.100 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:41.883823+0000", 2023-09-23T19:49:50.100 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0152999999999999e-05 2023-09-23T19:49:50.100 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.100 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.100 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:49:50.100 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:41.883828+0000", 2023-09-23T19:49:50.100 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.5690000000000004e-06 2023-09-23T19:49:50.101 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.101 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.101 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:49:50.101 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:41.883907+0000", 2023-09-23T19:49:50.101 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.8724e-05 2023-09-23T19:49:50.101 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.101 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.101 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:49:50.101 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:41.883929+0000", 2023-09-23T19:49:50.101 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.2274000000000001e-05 2023-09-23T19:49:50.101 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.101 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.102 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:49:50.102 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:42.040470+0000", 2023-09-23T19:49:50.102 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.15654047600000001 2023-09-23T19:49:50.102 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.102 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.102 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:49:50.102 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:42.040510+0000", 2023-09-23T19:49:50.102 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.0059999999999999e-05 2023-09-23T19:49:50.102 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:49:50.102 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:49:50.102 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:49:50.102 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.103 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.103 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.36480.0:167697 12.a e223/207 12:5e2c3fc1:::benchmark_data_smithi016_181952_object65016:head v 223'10441, mlcod=223'10438)", 2023-09-23T19:49:50.103 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:49:41.883989+0000", 2023-09-23T19:49:50.103 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 8.1092961579999994, 2023-09-23T19:49:50.103 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.15655640100000001, 2023-09-23T19:49:50.103 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:49:50.103 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:49:50.103 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:49:50.103 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.103 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:49:50.103 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:41.883989+0000", 2023-09-23T19:49:50.103 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:49:50.103 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.104 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.104 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:49:50.104 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:41.883989+0000", 2023-09-23T19:49:50.104 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:49:50.104 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.104 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.104 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:49:50.104 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:41.883988+0000", 2023-09-23T19:49:50.104 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T19:49:50.104 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.104 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.104 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:49:50.105 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:41.883994+0000", 2023-09-23T19:49:50.105 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.9599999999999997e-06 2023-09-23T19:49:50.105 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.105 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.105 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:49:50.105 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:41.884007+0000", 2023-09-23T19:49:50.105 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2815e-05 2023-09-23T19:49:50.105 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.105 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.105 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:49:50.105 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:41.884015+0000", 2023-09-23T19:49:50.105 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.3739999999999998e-06 2023-09-23T19:49:50.105 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.106 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.106 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:49:50.106 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:41.884084+0000", 2023-09-23T19:49:50.106 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.8728999999999999e-05 2023-09-23T19:49:50.106 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.106 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.106 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:49:50.106 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:41.884104+0000", 2023-09-23T19:49:50.106 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.9816000000000001e-05 2023-09-23T19:49:50.106 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.106 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.106 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:49:50.106 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:42.040523+0000", 2023-09-23T19:49:50.107 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.156419051 2023-09-23T19:49:50.107 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.107 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.107 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:49:50.107 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:42.040545+0000", 2023-09-23T19:49:50.107 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.2323999999999998e-05 2023-09-23T19:49:50.107 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:49:50.107 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:49:50.107 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:49:50.107 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.107 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.107 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop_reply(client.36480.0:167702 12.c e223/207 ondisk, result = 0)", 2023-09-23T19:49:50.108 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:49:41.886420+0000", 2023-09-23T19:49:50.108 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 8.1068646120000007, 2023-09-23T19:49:50.108 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.154232113, 2023-09-23T19:49:50.108 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:49:50.108 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "started", 2023-09-23T19:49:50.108 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:49:50.108 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.108 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:49:50.108 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:41.886420+0000", 2023-09-23T19:49:50.108 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:49:50.108 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.108 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.108 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:49:50.109 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:41.886420+0000", 2023-09-23T19:49:50.109 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:49:50.109 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.109 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.109 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:49:50.109 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:41.886419+0000", 2023-09-23T19:49:50.109 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:49:50.109 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.109 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.109 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:49:50.109 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:41.886425+0000", 2023-09-23T19:49:50.109 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.8289999999999996e-06 2023-09-23T19:49:50.110 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.110 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.110 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:49:50.110 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:41.886437+0000", 2023-09-23T19:49:50.110 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1829e-05 2023-09-23T19:49:50.110 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.110 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.110 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:49:50.110 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:41.886507+0000", 2023-09-23T19:49:50.110 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.9936000000000003e-05 2023-09-23T19:49:50.110 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.110 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.110 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:49:50.111 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:42.040632+0000", 2023-09-23T19:49:50.111 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.154124919 2023-09-23T19:49:50.111 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.111 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.111 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:49:50.111 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:42.040640+0000", 2023-09-23T19:49:50.111 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.9470000000000003e-06 2023-09-23T19:49:50.111 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.111 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.111 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:49:50.111 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:42.040652+0000", 2023-09-23T19:49:50.111 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2587e-05 2023-09-23T19:49:50.112 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:49:50.112 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:49:50.112 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:49:50.112 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.112 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.112 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.36480.0:167704 12.3 12:c05bee11:::benchmark_data_smithi016_181952_object65023:head [delete] snapc 0=[] ondisk+write+known_if_redirected e223)", 2023-09-23T19:49:50.112 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:49:41.886598+0000", 2023-09-23T19:49:50.112 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 8.1066865559999997, 2023-09-23T19:49:50.112 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.17205069200000001, 2023-09-23T19:49:50.112 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:49:50.112 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:49:50.112 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:49:50.112 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.36480", 2023-09-23T19:49:50.113 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/1035295215", 2023-09-23T19:49:50.113 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 167704 2023-09-23T19:49:50.113 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.113 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:49:50.113 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.113 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:49:50.113 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:41.886598+0000", 2023-09-23T19:49:50.113 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:49:50.113 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.113 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.113 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:49:50.113 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:41.886598+0000", 2023-09-23T19:49:50.114 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:49:50.114 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.114 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.114 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:49:50.114 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:41.886597+0000", 2023-09-23T19:49:50.114 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-09-23T19:49:50.114 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.114 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.114 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:49:50.114 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:41.886603+0000", 2023-09-23T19:49:50.114 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.3840000000000002e-06 2023-09-23T19:49:50.114 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.114 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.115 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:49:50.115 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:41.886614+0000", 2023-09-23T19:49:50.115 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0740000000000001e-05 2023-09-23T19:49:50.115 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.115 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.115 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:49:50.115 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:42.040374+0000", 2023-09-23T19:49:50.115 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.15375992799999999 2023-09-23T19:49:50.115 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.115 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.115 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:49:50.115 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:42.040797+0000", 2023-09-23T19:49:50.115 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00042316800000000003 2023-09-23T19:49:50.116 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.116 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.116 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:49:50.116 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:42.040927+0000", 2023-09-23T19:49:50.116 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00012933899999999999 2023-09-23T19:49:50.116 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.116 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.116 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 3,7", 2023-09-23T19:49:50.116 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:42.041046+0000", 2023-09-23T19:49:50.116 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000119432 2023-09-23T19:49:50.116 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.116 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.116 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:49:50.117 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:42.052379+0000", 2023-09-23T19:49:50.117 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.011332778 2023-09-23T19:49:50.117 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.117 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.117 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:49:50.117 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:42.057977+0000", 2023-09-23T19:49:50.117 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0055978950000000003 2023-09-23T19:49:50.117 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.117 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.117 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:49:50.117 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:42.058550+0000", 2023-09-23T19:49:50.117 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00057359499999999999 2023-09-23T19:49:50.118 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.118 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.118 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:49:50.118 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:42.058602+0000", 2023-09-23T19:49:50.118 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.1725999999999999e-05 2023-09-23T19:49:50.118 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.118 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.118 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:49:50.118 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:42.058649+0000", 2023-09-23T19:49:50.118 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.7030000000000002e-05 2023-09-23T19:49:50.118 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:49:50.118 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:49:50.119 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:49:50.119 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.119 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.119 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.36480.0:167703 12.8 e223/207 12:1b42c1af:::benchmark_data_smithi016_181952_object65022:head v 223'10638, mlcod=223'10636)", 2023-09-23T19:49:50.119 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:49:41.890376+0000", 2023-09-23T19:49:50.119 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 8.1029085900000002, 2023-09-23T19:49:50.119 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.18893697200000001, 2023-09-23T19:49:50.119 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:49:50.119 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:49:50.119 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:49:50.119 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.119 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:49:50.119 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:41.890376+0000", 2023-09-23T19:49:50.120 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:49:50.120 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.120 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.120 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:49:50.120 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:41.890376+0000", 2023-09-23T19:49:50.120 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:49:50.120 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.120 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.120 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:49:50.120 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:41.890376+0000", 2023-09-23T19:49:50.120 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:49:50.120 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.120 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.121 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:49:50.121 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:41.890382+0000", 2023-09-23T19:49:50.121 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.9280000000000004e-06 2023-09-23T19:49:50.121 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.121 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.121 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:49:50.121 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:42.040375+0000", 2023-09-23T19:49:50.121 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.14999220299999999 2023-09-23T19:49:50.121 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.121 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.121 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:49:50.121 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:42.040417+0000", 2023-09-23T19:49:50.122 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.2531999999999999e-05 2023-09-23T19:49:50.122 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.122 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.122 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:49:50.122 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:42.040467+0000", 2023-09-23T19:49:50.122 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.0262000000000001e-05 2023-09-23T19:49:50.122 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.122 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.122 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:49:50.122 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:42.040493+0000", 2023-09-23T19:49:50.122 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.5425000000000001e-05 2023-09-23T19:49:50.122 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.123 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.123 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:49:50.123 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:42.075891+0000", 2023-09-23T19:49:50.123 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.035398578999999999 2023-09-23T19:49:50.123 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:49:50.123 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:49:50.123 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:49:50.123 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:49:42.079313+0000", 2023-09-23T19:49:50.123 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0034218030000000002 2023-09-23T19:49:50.123 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:49:50.123 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:49:50.123 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:49:50.123 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:49:50.124 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:49:50.124 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-09-23T19:49:50.124 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:50.125 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:49 smithi092 bash[58855]: cluster 2023-09-23T19:49:48.398366+0000 mgr.y (mgr.25325) 1273 : cluster 0 pgmap v1263: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 625 op/s 2023-09-23T19:49:50.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:50.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:49:50.226 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:50.252 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:50.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:50.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:50.359 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:50.401 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:50.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:50.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:50.507 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:50.546 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:50.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:50.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:50.651 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:50.684 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:50.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:50.689 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:50.766 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T19:49:50.790 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:50.820 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:50.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:50.823 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:50.824 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:50 smithi016 bash[77995]: audit 2023-09-23T19:49:50.277147+0000 mon.a (mon.0) 1587 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T19:49:50.824 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:50 smithi016 bash[77995]: audit 2023-09-23T19:49:50.282937+0000 mon.b (mon.1) 55 : audit 1 from='client.? 172.21.15.16:0/185299389' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T19:49:50.825 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:50 smithi016 bash[75004]: audit 2023-09-23T19:49:50.277147+0000 mon.a (mon.0) 1587 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T19:49:50.825 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:50 smithi016 bash[75004]: audit 2023-09-23T19:49:50.282937+0000 mon.b (mon.1) 55 : audit 1 from='client.? 172.21.15.16:0/185299389' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T19:49:50.924 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:50.963 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:50.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:50.968 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:51.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:50 smithi092 bash[58855]: audit 2023-09-23T19:49:50.277147+0000 mon.a (mon.0) 1587 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T19:49:51.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:50 smithi092 bash[58855]: audit 2023-09-23T19:49:50.282937+0000 mon.b (mon.1) 55 : audit 1 from='client.? 172.21.15.16:0/185299389' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T19:49:51.069 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:51.096 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:51.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:51.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:49:51.200 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:51.229 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:51.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:51.232 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:49:51.332 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:51.364 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:51.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:51.366 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:51.467 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:51.473 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T19:49:51.493 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:51.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:51.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:51.596 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:51.632 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:51.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:51.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:51.737 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:51.770 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:51.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:51.773 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:49:51.874 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:51.903 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:51.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:51.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:49:52.010 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:52.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:51 smithi016 bash[75004]: cluster 2023-09-23T19:49:50.398983+0000 mgr.y (mgr.25325) 1274 : cluster 0 pgmap v1264: 89 pgs: 89 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-09-23T19:49:52.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:51 smithi016 bash[75004]: audit 2023-09-23T19:49:50.754475+0000 mon.a (mon.0) 1588 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T19:49:52.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:51 smithi016 bash[75004]: cluster 2023-09-23T19:49:50.754810+0000 mon.a (mon.0) 1589 : cluster 0 osdmap e224: 8 total, 8 up, 8 in 2023-09-23T19:49:52.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:51 smithi016 bash[75004]: cluster 2023-09-23T19:49:50.765081+0000 osd.6 (osd.6) 155 : cluster 3 failed to encode map e224 with expected crc 2023-09-23T19:49:52.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:51 smithi016 bash[75004]: cluster 2023-09-23T19:49:50.771908+0000 osd.7 (osd.7) 143 : cluster 3 failed to encode map e224 with expected crc 2023-09-23T19:49:52.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:51 smithi016 bash[75004]: cluster 2023-09-23T19:49:50.777518+0000 osd.7 (osd.7) 144 : cluster 3 failed to encode map e224 with expected crc 2023-09-23T19:49:52.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:51 smithi016 bash[75004]: cluster 2023-09-23T19:49:50.779122+0000 osd.4 (osd.4) 164 : cluster 3 failed to encode map e224 with expected crc 2023-09-23T19:49:52.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:51 smithi016 bash[75004]: cluster 2023-09-23T19:49:50.794487+0000 osd.5 (osd.5) 150 : cluster 3 failed to encode map e224 with expected crc 2023-09-23T19:49:52.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:51 smithi016 bash[77995]: cluster 2023-09-23T19:49:50.398983+0000 mgr.y (mgr.25325) 1274 : cluster 0 pgmap v1264: 89 pgs: 89 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-09-23T19:49:52.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:51 smithi016 bash[77995]: audit 2023-09-23T19:49:50.754475+0000 mon.a (mon.0) 1588 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T19:49:52.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:51 smithi016 bash[77995]: cluster 2023-09-23T19:49:50.754810+0000 mon.a (mon.0) 1589 : cluster 0 osdmap e224: 8 total, 8 up, 8 in 2023-09-23T19:49:52.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:51 smithi016 bash[77995]: cluster 2023-09-23T19:49:50.765081+0000 osd.6 (osd.6) 155 : cluster 3 failed to encode map e224 with expected crc 2023-09-23T19:49:52.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:51 smithi016 bash[77995]: cluster 2023-09-23T19:49:50.771908+0000 osd.7 (osd.7) 143 : cluster 3 failed to encode map e224 with expected crc 2023-09-23T19:49:52.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:51 smithi016 bash[77995]: cluster 2023-09-23T19:49:50.777518+0000 osd.7 (osd.7) 144 : cluster 3 failed to encode map e224 with expected crc 2023-09-23T19:49:52.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:51 smithi016 bash[77995]: cluster 2023-09-23T19:49:50.779122+0000 osd.4 (osd.4) 164 : cluster 3 failed to encode map e224 with expected crc 2023-09-23T19:49:52.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:51 smithi016 bash[77995]: cluster 2023-09-23T19:49:50.794487+0000 osd.5 (osd.5) 150 : cluster 3 failed to encode map e224 with expected crc 2023-09-23T19:49:52.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:51 smithi092 bash[58855]: cluster 2023-09-23T19:49:50.398983+0000 mgr.y (mgr.25325) 1274 : cluster 0 pgmap v1264: 89 pgs: 89 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-09-23T19:49:52.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:51 smithi092 bash[58855]: audit 2023-09-23T19:49:50.754475+0000 mon.a (mon.0) 1588 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T19:49:52.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:51 smithi092 bash[58855]: cluster 2023-09-23T19:49:50.754810+0000 mon.a (mon.0) 1589 : cluster 0 osdmap e224: 8 total, 8 up, 8 in 2023-09-23T19:49:52.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:51 smithi092 bash[58855]: cluster 2023-09-23T19:49:50.765081+0000 osd.6 (osd.6) 155 : cluster 3 failed to encode map e224 with expected crc 2023-09-23T19:49:52.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:51 smithi092 bash[58855]: cluster 2023-09-23T19:49:50.771908+0000 osd.7 (osd.7) 143 : cluster 3 failed to encode map e224 with expected crc 2023-09-23T19:49:52.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:51 smithi092 bash[58855]: cluster 2023-09-23T19:49:50.777518+0000 osd.7 (osd.7) 144 : cluster 3 failed to encode map e224 with expected crc 2023-09-23T19:49:52.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:51 smithi092 bash[58855]: cluster 2023-09-23T19:49:50.779122+0000 osd.4 (osd.4) 164 : cluster 3 failed to encode map e224 with expected crc 2023-09-23T19:49:52.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:51 smithi092 bash[58855]: cluster 2023-09-23T19:49:50.794487+0000 osd.5 (osd.5) 150 : cluster 3 failed to encode map e224 with expected crc 2023-09-23T19:49:52.043 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:52.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:52.047 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:52.148 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:52.173 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:52.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:52.178 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:49:52.231 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T19:49:52.279 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:52.319 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:52.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:52.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:49:52.430 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:52.453 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:52.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:52.456 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:52.557 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:52.595 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:52.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:52.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:52.704 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:52.709 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:49:52.711 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:49:52.763 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:52.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:52.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:49:52.869 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:52.893 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:52.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:52.895 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:52.996 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:53.033 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:53.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:53.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:49:53.138 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:53.162 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:53.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:53.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:49:53.270 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:53.294 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:53.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:53.295 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:49:53.396 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:53.424 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:53.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:53.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:49:53.527 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:53.552 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:53.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:53.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:53.659 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:53.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:53.692 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:53.693 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:49:53.793 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:53.815 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:53.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:53.818 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:49:53.919 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:53.954 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:53.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:53.956 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:49:54.057 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:54.066 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:53 smithi016 bash[77995]: cluster 2023-09-23T19:49:52.399485+0000 mgr.y (mgr.25325) 1275 : cluster 0 pgmap v1266: 89 pgs: 89 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 566 op/s 2023-09-23T19:49:54.066 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:53 smithi016 bash[75004]: cluster 2023-09-23T19:49:52.399485+0000 mgr.y (mgr.25325) 1275 : cluster 0 pgmap v1266: 89 pgs: 89 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 566 op/s 2023-09-23T19:49:54.089 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:54.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:54.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:49:54.195 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:54.238 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:54.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:54.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:54.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:53 smithi092 bash[58855]: cluster 2023-09-23T19:49:52.399485+0000 mgr.y (mgr.25325) 1275 : cluster 0 pgmap v1266: 89 pgs: 89 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 566 op/s 2023-09-23T19:49:54.343 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:54.375 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:54.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:54.378 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:54.479 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:54.503 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:54.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:54.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:54.609 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:54.647 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:54.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:54.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:54.754 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:54.795 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:54.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:54.803 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:54.905 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:54.932 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:54.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:54.937 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:55.038 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:55.064 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:55.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:55.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:49:55.171 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:55.195 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:55.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:55.198 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:55.299 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:55.322 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:55.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:55.329 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:49:55.429 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:55.454 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:55.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:55.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:55.559 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:55.584 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:55.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:55.588 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:55.641 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:49:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:49:55] "GET /metrics HTTP/1.1" 200 36306 "" "Prometheus/2.33.4" 2023-09-23T19:49:55.690 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:55.716 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:55.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:55.725 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:55.826 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:55.854 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:55.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:55.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:55.958 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:55.987 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:55.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:55.992 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:56.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:55 smithi016 bash[75004]: cluster 2023-09-23T19:49:54.399940+0000 mgr.y (mgr.25325) 1276 : cluster 0 pgmap v1267: 89 pgs: 89 active+clean; 1.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-09-23T19:49:56.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:55 smithi016 bash[77995]: cluster 2023-09-23T19:49:54.399940+0000 mgr.y (mgr.25325) 1276 : cluster 0 pgmap v1267: 89 pgs: 89 active+clean; 1.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-09-23T19:49:56.093 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:56.104 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:55 smithi092 bash[58855]: cluster 2023-09-23T19:49:54.399940+0000 mgr.y (mgr.25325) 1276 : cluster 0 pgmap v1267: 89 pgs: 89 active+clean; 1.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-09-23T19:49:56.119 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:56.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:56.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:56.222 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:56.244 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:56.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:56.248 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:56.349 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:56.380 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:56.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:56.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:56.484 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:56.528 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:56.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:56.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:56.631 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:56.662 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:56.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:56.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:56.770 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:56.810 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:56.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:56.819 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:56.919 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:56.942 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:56.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:56.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:57.049 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:57.077 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:57.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:57.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:57.183 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:57.208 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:57.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:57.211 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:57.311 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:49:57.312 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:57.315 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:49:57.372 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:57.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:57.378 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:57.479 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:57.506 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:57.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:57.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:49:57.612 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:57.640 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:57.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:57.642 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:57.742 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:57.767 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:57.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:57.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:57.872 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:57.901 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:49:57.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:57.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:49:58.004 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:58.027 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:49:58.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:58.032 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:49:58.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:57 smithi092 bash[58855]: cluster 2023-09-23T19:49:56.400617+0000 mgr.y (mgr.25325) 1277 : cluster 0 pgmap v1268: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 628 op/s 2023-09-23T19:49:58.104 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:57 smithi016 bash[77995]: cluster 2023-09-23T19:49:56.400617+0000 mgr.y (mgr.25325) 1277 : cluster 0 pgmap v1268: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 628 op/s 2023-09-23T19:49:58.105 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:57 smithi016 bash[75004]: cluster 2023-09-23T19:49:56.400617+0000 mgr.y (mgr.25325) 1277 : cluster 0 pgmap v1268: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 628 op/s 2023-09-23T19:49:58.132 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:58.161 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:58.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:58.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:49:58.268 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:58.289 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:58.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:58.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:49:58.393 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:58.422 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:58.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:58.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:58.526 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:58.558 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:58.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:58.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:49:58.666 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:58.689 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:58.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:58.697 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:49:58.798 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:58.826 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:49:58.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:58.832 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:49:58.933 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:58.956 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:58.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:58.958 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:59.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:58 smithi016 bash[77995]: audit 2023-09-23T19:49:58.672306+0000 mon.a (mon.0) 1590 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:49:59.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:58 smithi016 bash[75004]: audit 2023-09-23T19:49:58.672306+0000 mon.a (mon.0) 1590 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:49:59.059 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:59.066 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:58 smithi092 bash[58855]: audit 2023-09-23T19:49:58.672306+0000 mon.a (mon.0) 1590 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:49:59.086 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:59.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:59.092 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:59.193 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:59.220 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:49:59.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:59.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:49:59.324 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:59.352 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:59.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:59.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:59.456 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:59.483 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:49:59.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:59.488 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:49:59.589 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:59.620 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:49:59.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:59.623 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:49:59.724 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:59.747 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:49:59.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:59.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:49:59.850 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:59.888 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:59.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:49:59.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:49:59.990 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:49:59.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:49:59 smithi016 bash[75004]: cluster 2023-09-23T19:49:58.401288+0000 mgr.y (mgr.25325) 1278 : cluster 0 pgmap v1269: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 630 op/s 2023-09-23T19:49:59.999 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:49:59 smithi016 bash[77995]: cluster 2023-09-23T19:49:58.401288+0000 mgr.y (mgr.25325) 1278 : cluster 0 pgmap v1269: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 630 op/s 2023-09-23T19:50:00.022 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:00.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:00.030 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:50:00.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:49:59 smithi092 bash[58855]: cluster 2023-09-23T19:49:58.401288+0000 mgr.y (mgr.25325) 1278 : cluster 0 pgmap v1269: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 630 op/s 2023-09-23T19:50:00.131 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:00.158 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:00.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:00.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:00.262 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:00.290 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:00.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:00.299 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:50:00.400 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:00.427 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:00.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:00.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:00.530 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:00.551 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:00.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:00.557 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:00.658 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:00.682 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:00.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:00.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:50:00.774 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:50:00.790 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:00.818 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:00.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:00.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:50:00.851 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:50:00.923 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:00.924 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:50:00.932 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:00 smithi092 bash[58855]: cluster 2023-09-23T19:50:00.000110+0000 mon.a (mon.0) 1591 : cluster 3 Health detail: HEALTH_WARN noscrub flag(s) set 2023-09-23T19:50:00.932 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:00 smithi092 bash[58855]: cluster 2023-09-23T19:50:00.000167+0000 mon.a (mon.0) 1592 : cluster 3 [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-09-23T19:50:00.932 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:00 smithi092 bash[58855]: audit 2023-09-23T19:50:00.700085+0000 mon.c (mon.2) 202 : audit 0 from='client.? 172.21.15.16:0/299726101' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:50:00.949 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:00.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:00.952 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:00.955 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:00 smithi016 bash[77995]: cluster 2023-09-23T19:50:00.000110+0000 mon.a (mon.0) 1591 : cluster 3 Health detail: HEALTH_WARN noscrub flag(s) set 2023-09-23T19:50:00.955 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:00 smithi016 bash[77995]: cluster 2023-09-23T19:50:00.000167+0000 mon.a (mon.0) 1592 : cluster 3 [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-09-23T19:50:00.956 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:00 smithi016 bash[77995]: audit 2023-09-23T19:50:00.700085+0000 mon.c (mon.2) 202 : audit 0 from='client.? 172.21.15.16:0/299726101' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:50:00.956 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:00 smithi016 bash[75004]: cluster 2023-09-23T19:50:00.000110+0000 mon.a (mon.0) 1591 : cluster 3 Health detail: HEALTH_WARN noscrub flag(s) set 2023-09-23T19:50:00.956 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:00 smithi016 bash[75004]: cluster 2023-09-23T19:50:00.000167+0000 mon.a (mon.0) 1592 : cluster 3 [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-09-23T19:50:00.956 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:00 smithi016 bash[75004]: audit 2023-09-23T19:50:00.700085+0000 mon.c (mon.2) 202 : audit 0 from='client.? 172.21.15.16:0/299726101' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:50:00.990 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:50:01.037 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:50:01.053 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:01.082 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:50:01.082 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:01.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:01.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:50:01.132 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:50:01.176 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:50:01.183 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:50:01.183 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:50:01.183 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:50:01.183 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:50:01.183 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:50:01.183 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:50:01.183 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:50:01.183 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:50:01.186 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:01.224 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:01.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:01.228 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:50:01.328 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:01.359 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:01.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:01.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:50:01.463 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:01.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:01.501 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:01.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:01.602 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:01.631 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:01.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:01.641 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:01.741 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:01.771 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:01.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:01.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:50:01.846 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is set 2023-09-23T19:50:01.875 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:01.883 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:01 smithi092 bash[58855]: cluster 2023-09-23T19:50:00.401880+0000 mgr.y (mgr.25325) 1279 : cluster 0 pgmap v1270: 89 pgs: 89 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 629 op/s 2023-09-23T19:50:01.883 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:01 smithi092 bash[58855]: audit 2023-09-23T19:50:00.997812+0000 mon.c (mon.2) 203 : audit 1 from='client.? 172.21.15.16:0/720589915' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:50:01.883 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:01 smithi092 bash[58855]: audit 2023-09-23T19:50:00.998994+0000 mon.a (mon.0) 1593 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:50:01.908 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:01.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:01.911 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:50:02.012 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:02.036 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:02.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:02.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:02.068 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:01 smithi016 bash[75004]: cluster 2023-09-23T19:50:00.401880+0000 mgr.y (mgr.25325) 1279 : cluster 0 pgmap v1270: 89 pgs: 89 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 629 op/s 2023-09-23T19:50:02.068 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:01 smithi016 bash[75004]: audit 2023-09-23T19:50:00.997812+0000 mon.c (mon.2) 203 : audit 1 from='client.? 172.21.15.16:0/720589915' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:50:02.068 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:01 smithi016 bash[75004]: audit 2023-09-23T19:50:00.998994+0000 mon.a (mon.0) 1593 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:50:02.068 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:01 smithi016 bash[77995]: cluster 2023-09-23T19:50:00.401880+0000 mgr.y (mgr.25325) 1279 : cluster 0 pgmap v1270: 89 pgs: 89 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 629 op/s 2023-09-23T19:50:02.068 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:01 smithi016 bash[77995]: audit 2023-09-23T19:50:00.997812+0000 mon.c (mon.2) 203 : audit 1 from='client.? 172.21.15.16:0/720589915' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:50:02.068 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:01 smithi016 bash[77995]: audit 2023-09-23T19:50:00.998994+0000 mon.a (mon.0) 1593 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:50:02.145 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:02.171 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:02.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:02.173 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:02.274 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:02.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:02.312 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:02.313 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:02.413 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:02.445 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:02.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:02.448 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:50:02.549 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:02.574 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:02.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:02.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:50:02.681 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:02.705 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:02.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:02.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:50:02.809 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:02.829 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:02.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:02.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:02.933 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:02.960 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:02.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:02.963 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:03.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:02 smithi092 bash[58855]: cluster 2023-09-23T19:50:01.822362+0000 mon.a (mon.0) 1594 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:50:03.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:02 smithi092 bash[58855]: audit 2023-09-23T19:50:01.833503+0000 mon.a (mon.0) 1595 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:50:03.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:02 smithi092 bash[58855]: cluster 2023-09-23T19:50:01.833686+0000 mon.a (mon.0) 1596 : cluster 0 osdmap e225: 8 total, 8 up, 8 in 2023-09-23T19:50:03.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:02 smithi092 bash[58855]: cluster 2023-09-23T19:50:01.837211+0000 osd.4 (osd.4) 165 : cluster 3 failed to encode map e225 with expected crc 2023-09-23T19:50:03.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:02 smithi092 bash[58855]: cluster 2023-09-23T19:50:01.852659+0000 osd.6 (osd.6) 156 : cluster 3 failed to encode map e225 with expected crc 2023-09-23T19:50:03.065 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:03.073 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:02 smithi016 bash[77995]: cluster 2023-09-23T19:50:01.822362+0000 mon.a (mon.0) 1594 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:50:03.073 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:02 smithi016 bash[77995]: audit 2023-09-23T19:50:01.833503+0000 mon.a (mon.0) 1595 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:50:03.073 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:02 smithi016 bash[77995]: cluster 2023-09-23T19:50:01.833686+0000 mon.a (mon.0) 1596 : cluster 0 osdmap e225: 8 total, 8 up, 8 in 2023-09-23T19:50:03.074 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:02 smithi016 bash[77995]: cluster 2023-09-23T19:50:01.837211+0000 osd.4 (osd.4) 165 : cluster 3 failed to encode map e225 with expected crc 2023-09-23T19:50:03.074 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:02 smithi016 bash[77995]: cluster 2023-09-23T19:50:01.852659+0000 osd.6 (osd.6) 156 : cluster 3 failed to encode map e225 with expected crc 2023-09-23T19:50:03.074 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:02 smithi016 bash[75004]: cluster 2023-09-23T19:50:01.822362+0000 mon.a (mon.0) 1594 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:50:03.074 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:02 smithi016 bash[75004]: audit 2023-09-23T19:50:01.833503+0000 mon.a (mon.0) 1595 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:50:03.074 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:02 smithi016 bash[75004]: cluster 2023-09-23T19:50:01.833686+0000 mon.a (mon.0) 1596 : cluster 0 osdmap e225: 8 total, 8 up, 8 in 2023-09-23T19:50:03.074 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:02 smithi016 bash[75004]: cluster 2023-09-23T19:50:01.837211+0000 osd.4 (osd.4) 165 : cluster 3 failed to encode map e225 with expected crc 2023-09-23T19:50:03.074 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:02 smithi016 bash[75004]: cluster 2023-09-23T19:50:01.852659+0000 osd.6 (osd.6) 156 : cluster 3 failed to encode map e225 with expected crc 2023-09-23T19:50:03.102 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:03.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:03.106 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:03.207 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:03.258 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:03.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:03.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:03.366 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:03.393 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:03.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:03.396 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:03.497 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:03.530 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:03.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:03.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:50:03.635 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:03.663 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:03.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:03.665 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:50:03.766 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:03.797 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:03.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:03.803 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:03.904 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:03.969 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:03.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:03.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:50:04.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:03 smithi092 bash[58855]: cluster 2023-09-23T19:50:02.402292+0000 mgr.y (mgr.25325) 1280 : cluster 0 pgmap v1272: 89 pgs: 89 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 631 op/s 2023-09-23T19:50:04.072 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:04.094 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:04.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:04.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:50:04.111 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:03 smithi016 bash[77995]: cluster 2023-09-23T19:50:02.402292+0000 mgr.y (mgr.25325) 1280 : cluster 0 pgmap v1272: 89 pgs: 89 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 631 op/s 2023-09-23T19:50:04.112 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:03 smithi016 bash[75004]: cluster 2023-09-23T19:50:02.402292+0000 mgr.y (mgr.25325) 1280 : cluster 0 pgmap v1272: 89 pgs: 89 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 631 op/s 2023-09-23T19:50:04.197 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:04.231 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:04.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:04.236 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:50:04.336 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:04.368 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:04.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:04.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:50:04.471 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:04.504 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:04.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:04.514 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:50:04.615 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:04.649 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:04.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:04.656 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:50:04.756 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:04.793 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:04.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:04.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:50:04.899 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:04.919 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:04.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:04.922 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:05.022 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:05.050 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:05.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:05.055 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:05.156 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:05.189 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:05.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:05.193 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:50:05.293 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:05.329 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:05.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:05.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:05.435 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:05.464 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:05.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:05.468 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:50:05.569 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:05.590 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:05.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:05.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:50:05.642 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:50:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:50:05] "GET /metrics HTTP/1.1" 200 36298 "" "Prometheus/2.33.4" 2023-09-23T19:50:05.701 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:05.727 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:05.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:05.730 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:05.831 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:05.845 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:05 smithi092 bash[58855]: cluster 2023-09-23T19:50:04.402820+0000 mgr.y (mgr.25325) 1281 : cluster 0 pgmap v1273: 89 pgs: 89 active+clean; 1.6 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 629 op/s 2023-09-23T19:50:05.853 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:05.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:05.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:50:05.957 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:05.981 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:05.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:05.984 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:50:06.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:05 smithi016 bash[75004]: cluster 2023-09-23T19:50:04.402820+0000 mgr.y (mgr.25325) 1281 : cluster 0 pgmap v1273: 89 pgs: 89 active+clean; 1.6 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 629 op/s 2023-09-23T19:50:06.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:05 smithi016 bash[77995]: cluster 2023-09-23T19:50:04.402820+0000 mgr.y (mgr.25325) 1281 : cluster 0 pgmap v1273: 89 pgs: 89 active+clean; 1.6 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 629 op/s 2023-09-23T19:50:06.085 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:06.120 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:06.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:06.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:06.224 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:06.257 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:06.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:06.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:06.363 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:06.388 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:06.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:06.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:50:06.492 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:06.527 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:06.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:06.533 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:06.634 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:06.661 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:06.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:06.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:06.764 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:06.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:06.800 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:06.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:50:06.901 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:06.931 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:06.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:06.934 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:50:07.035 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:07.061 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:07.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:07.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:50:07.164 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:07.190 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:07.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:07.197 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:50:07.298 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:07.320 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:07.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:07.323 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:50:07.424 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:07.459 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:07.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:07.467 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:50:07.568 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:07.598 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:07.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:07.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:07.702 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:07.734 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:07.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:07.739 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:50:07.840 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:07.864 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:07.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:07.866 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:07.967 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:07.975 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:07 smithi092 bash[58855]: cluster 2023-09-23T19:50:06.403492+0000 mgr.y (mgr.25325) 1282 : cluster 0 pgmap v1274: 89 pgs: 89 active+clean; 1.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 622 op/s 2023-09-23T19:50:07.990 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:07.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:07.997 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:50:08.055 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:07 smithi016 bash[75004]: cluster 2023-09-23T19:50:06.403492+0000 mgr.y (mgr.25325) 1282 : cluster 0 pgmap v1274: 89 pgs: 89 active+clean; 1.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 622 op/s 2023-09-23T19:50:08.056 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:07 smithi016 bash[77995]: cluster 2023-09-23T19:50:06.403492+0000 mgr.y (mgr.25325) 1282 : cluster 0 pgmap v1274: 89 pgs: 89 active+clean; 1.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 622 op/s 2023-09-23T19:50:08.097 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:08.132 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:08.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:08.137 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:50:08.204 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:50:08.237 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:08.267 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:08.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:08.269 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:50:08.370 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:08.395 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:08.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:08.400 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:08.501 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:08.541 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:08.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:08.544 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:50:08.645 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:08.680 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:08.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:08.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:50:08.788 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:08.812 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:08.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:08.814 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:50:08.818 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T19:50:08.915 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:08.942 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:08.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:08.945 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:09.046 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:09.075 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:09.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:09.078 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:09.179 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:09.233 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:09.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:09.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:09.343 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:09.348 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-09-23T19:50:09.348 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:50:09.368 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:09.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:09.370 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:50:09.471 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:09.495 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:09.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:09.497 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:50:09.598 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:09.631 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:09.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:09.634 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:50:09.735 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:09.777 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:09.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:09.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:50:09.863 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T19:50:09.884 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:09.914 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:09.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:09.920 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:10.021 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:10.030 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:09 smithi016 bash[77995]: cluster 2023-09-23T19:50:08.403954+0000 mgr.y (mgr.25325) 1283 : cluster 0 pgmap v1275: 89 pgs: 89 active+clean; 1.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-09-23T19:50:10.030 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:09 smithi016 bash[77995]: audit 2023-09-23T19:50:09.571330+0000 mon.a (mon.0) 1597 : audit 1 from='client.? 172.21.15.16:0/354634224' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T19:50:10.030 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:09 smithi016 bash[75004]: cluster 2023-09-23T19:50:08.403954+0000 mgr.y (mgr.25325) 1283 : cluster 0 pgmap v1275: 89 pgs: 89 active+clean; 1.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-09-23T19:50:10.030 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:09 smithi016 bash[75004]: audit 2023-09-23T19:50:09.571330+0000 mon.a (mon.0) 1597 : audit 1 from='client.? 172.21.15.16:0/354634224' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T19:50:10.050 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:10.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:10.057 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:50:10.158 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:10.165 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:09 smithi092 bash[58855]: cluster 2023-09-23T19:50:08.403954+0000 mgr.y (mgr.25325) 1283 : cluster 0 pgmap v1275: 89 pgs: 89 active+clean; 1.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-09-23T19:50:10.165 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:09 smithi092 bash[58855]: audit 2023-09-23T19:50:09.571330+0000 mon.a (mon.0) 1597 : audit 1 from='client.? 172.21.15.16:0/354634224' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T19:50:10.189 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:10.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:10.192 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:10.293 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:10.317 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:10.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:10.321 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:50:10.421 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:10.454 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:10.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:10.457 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:50:10.558 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:10.599 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:10.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:10.604 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:50:10.705 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:10.731 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:10.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:10.733 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:50:10.834 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:10.862 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:10.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:10.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:50:10.962 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:10 smithi016 bash[75004]: audit 2023-09-23T19:50:09.850117+0000 mon.a (mon.0) 1598 : audit 1 from='client.? 172.21.15.16:0/354634224' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T19:50:10.963 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:10 smithi016 bash[75004]: cluster 2023-09-23T19:50:09.850185+0000 mon.a (mon.0) 1599 : cluster 0 osdmap e226: 8 total, 8 up, 8 in 2023-09-23T19:50:10.963 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:10 smithi016 bash[75004]: cluster 2023-09-23T19:50:09.867063+0000 osd.4 (osd.4) 166 : cluster 3 failed to encode map e226 with expected crc 2023-09-23T19:50:10.963 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:10 smithi016 bash[75004]: cluster 2023-09-23T19:50:09.870830+0000 osd.6 (osd.6) 157 : cluster 3 failed to encode map e226 with expected crc 2023-09-23T19:50:10.963 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:10 smithi016 bash[75004]: cluster 2023-09-23T19:50:09.872404+0000 osd.6 (osd.6) 158 : cluster 3 failed to encode map e226 with expected crc 2023-09-23T19:50:10.963 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:10 smithi016 bash[75004]: cluster 2023-09-23T19:50:09.879607+0000 osd.7 (osd.7) 145 : cluster 3 failed to encode map e226 with expected crc 2023-09-23T19:50:10.963 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:10 smithi016 bash[77995]: audit 2023-09-23T19:50:09.850117+0000 mon.a (mon.0) 1598 : audit 1 from='client.? 172.21.15.16:0/354634224' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T19:50:10.964 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:10 smithi016 bash[77995]: cluster 2023-09-23T19:50:09.850185+0000 mon.a (mon.0) 1599 : cluster 0 osdmap e226: 8 total, 8 up, 8 in 2023-09-23T19:50:10.964 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:10 smithi016 bash[77995]: cluster 2023-09-23T19:50:09.867063+0000 osd.4 (osd.4) 166 : cluster 3 failed to encode map e226 with expected crc 2023-09-23T19:50:10.964 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:10 smithi016 bash[77995]: cluster 2023-09-23T19:50:09.870830+0000 osd.6 (osd.6) 157 : cluster 3 failed to encode map e226 with expected crc 2023-09-23T19:50:10.964 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:10 smithi016 bash[77995]: cluster 2023-09-23T19:50:09.872404+0000 osd.6 (osd.6) 158 : cluster 3 failed to encode map e226 with expected crc 2023-09-23T19:50:10.964 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:10 smithi016 bash[77995]: cluster 2023-09-23T19:50:09.879607+0000 osd.7 (osd.7) 145 : cluster 3 failed to encode map e226 with expected crc 2023-09-23T19:50:10.965 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T19:50:10.965 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:10.972 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:10 smithi092 bash[58855]: audit 2023-09-23T19:50:09.850117+0000 mon.a (mon.0) 1598 : audit 1 from='client.? 172.21.15.16:0/354634224' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T19:50:10.973 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:10 smithi092 bash[58855]: cluster 2023-09-23T19:50:09.850185+0000 mon.a (mon.0) 1599 : cluster 0 osdmap e226: 8 total, 8 up, 8 in 2023-09-23T19:50:10.973 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:10 smithi092 bash[58855]: cluster 2023-09-23T19:50:09.867063+0000 osd.4 (osd.4) 166 : cluster 3 failed to encode map e226 with expected crc 2023-09-23T19:50:10.973 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:10 smithi092 bash[58855]: cluster 2023-09-23T19:50:09.870830+0000 osd.6 (osd.6) 157 : cluster 3 failed to encode map e226 with expected crc 2023-09-23T19:50:10.973 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:10 smithi092 bash[58855]: cluster 2023-09-23T19:50:09.872404+0000 osd.6 (osd.6) 158 : cluster 3 failed to encode map e226 with expected crc 2023-09-23T19:50:10.973 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:10 smithi092 bash[58855]: cluster 2023-09-23T19:50:09.879607+0000 osd.7 (osd.7) 145 : cluster 3 failed to encode map e226 with expected crc 2023-09-23T19:50:10.990 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:10.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:10.992 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:11.093 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:11.115 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:11.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:11.119 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:11.220 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:11.275 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:11.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:11.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:50:11.382 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:11.421 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:11.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:11.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:50:11.526 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:11.558 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:11.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:11.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:11.666 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:11.697 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:11.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:11.700 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:50:11.801 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:11.830 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:11.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:11.834 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:11.897 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T19:50:11.935 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:11.945 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:11 smithi092 bash[58855]: cluster 2023-09-23T19:50:10.404499+0000 mgr.y (mgr.25325) 1284 : cluster 0 pgmap v1277: 89 pgs: 89 active+clean; 1.3 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 667 op/s 2023-09-23T19:50:11.974 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:11.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:11.977 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:50:12.078 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:12.101 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:12.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:12.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:50:12.204 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:12.214 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:11 smithi016 bash[75004]: cluster 2023-09-23T19:50:10.404499+0000 mgr.y (mgr.25325) 1284 : cluster 0 pgmap v1277: 89 pgs: 89 active+clean; 1.3 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 667 op/s 2023-09-23T19:50:12.215 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:11 smithi016 bash[77995]: cluster 2023-09-23T19:50:10.404499+0000 mgr.y (mgr.25325) 1284 : cluster 0 pgmap v1277: 89 pgs: 89 active+clean; 1.3 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 667 op/s 2023-09-23T19:50:12.236 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:12.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:12.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:12.346 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:12.371 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:12.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:12.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:50:12.473 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:12.499 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:12.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:12.501 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:50:12.602 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:12.637 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:12.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:12.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:12.744 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:12.787 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:12.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:12.792 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:12.893 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:12.933 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:12.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:12.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:50:13.037 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:13.065 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:13.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:13.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:13.169 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:13.197 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:13.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:13.201 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:50:13.302 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:13.332 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:13.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:13.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:13.437 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:13.496 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:13.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:13.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:13.601 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:13.628 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:13.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:13.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:50:13.732 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:13.759 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:13.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:13.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:13.865 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:13.906 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:13.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:13.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:50:14.009 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:14.034 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:14.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:14.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:50:14.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:13 smithi016 bash[75004]: cluster 2023-09-23T19:50:12.404989+0000 mgr.y (mgr.25325) 1285 : cluster 0 pgmap v1278: 89 pgs: 89 active+clean; 1.3 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 609 op/s 2023-09-23T19:50:14.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:13 smithi016 bash[75004]: audit 2023-09-23T19:50:13.676495+0000 mon.a (mon.0) 1600 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:50:14.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:13 smithi016 bash[77995]: cluster 2023-09-23T19:50:12.404989+0000 mgr.y (mgr.25325) 1285 : cluster 0 pgmap v1278: 89 pgs: 89 active+clean; 1.3 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 609 op/s 2023-09-23T19:50:14.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:13 smithi016 bash[77995]: audit 2023-09-23T19:50:13.676495+0000 mon.a (mon.0) 1600 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:50:14.137 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:14.146 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:13 smithi092 bash[58855]: cluster 2023-09-23T19:50:12.404989+0000 mgr.y (mgr.25325) 1285 : cluster 0 pgmap v1278: 89 pgs: 89 active+clean; 1.3 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 609 op/s 2023-09-23T19:50:14.146 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:13 smithi092 bash[58855]: audit 2023-09-23T19:50:13.676495+0000 mon.a (mon.0) 1600 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:50:14.165 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:14.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:14.169 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:14.270 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:14.293 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:14.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:14.300 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:50:14.401 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:14.431 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:14.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:14.434 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:50:14.535 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:14.561 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:14.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:14.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:14.668 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:14.700 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:14.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:14.705 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:14.805 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:14.839 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:14.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:14.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:14.943 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:14.975 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:14.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:14.978 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:50:15.079 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:15.101 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:15.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:15.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:15.204 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:15.230 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:15.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:15.233 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:50:15.334 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:15.368 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:15.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:15.373 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:50:15.440 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:50:15.441 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:50:15.474 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:15.512 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:15.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:15.515 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:15.615 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:15.645 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:15.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:15.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:50:15.747 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:15.774 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:15.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:15.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:50:15.787 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:50:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:50:15] "GET /metrics HTTP/1.1" 200 36302 "" "Prometheus/2.33.4" 2023-09-23T19:50:15.875 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:15.886 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:15 smithi092 bash[58855]: cluster 2023-09-23T19:50:14.405493+0000 mgr.y (mgr.25325) 1286 : cluster 0 pgmap v1279: 89 pgs: 89 active+clean; 1.2 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 608 op/s 2023-09-23T19:50:15.906 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:15.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:15.912 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:50:16.013 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:16.053 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:16.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:16.056 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:50:16.157 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:16.192 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:16.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:16.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:50:16.295 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:16.324 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:16.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:16.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:50:16.431 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:16.448 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:15 smithi016 bash[77995]: cluster 2023-09-23T19:50:14.405493+0000 mgr.y (mgr.25325) 1286 : cluster 0 pgmap v1279: 89 pgs: 89 active+clean; 1.2 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 608 op/s 2023-09-23T19:50:16.448 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:15 smithi016 bash[75004]: cluster 2023-09-23T19:50:14.405493+0000 mgr.y (mgr.25325) 1286 : cluster 0 pgmap v1279: 89 pgs: 89 active+clean; 1.2 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 608 op/s 2023-09-23T19:50:16.457 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:50:16.461 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:50:16.467 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:16.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:16.469 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:50:16.570 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:16.597 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:16.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:16.599 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:50:16.700 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:16.730 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:16.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:16.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:16.835 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:16.865 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:16.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:16.868 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:50:16.968 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:16.996 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:17.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:17.004 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:17.104 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:17.131 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:17.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:17.134 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:50:17.235 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:17.270 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:17.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:17.277 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:17.378 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:17.404 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:17.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:17.410 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:17.511 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:17.533 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:17.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:17.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:50:17.641 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:17.665 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:17.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:17.670 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:50:17.770 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:17.799 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:17.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:17.801 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:17.902 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:17.936 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:17.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:17.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:50:18.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:17 smithi092 bash[58855]: cluster 2023-09-23T19:50:16.406204+0000 mgr.y (mgr.25325) 1287 : cluster 0 pgmap v1280: 89 pgs: 89 active+clean; 1.1 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-09-23T19:50:18.041 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:18.082 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:18.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:18.088 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:50:18.189 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:18.198 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:17 smithi016 bash[75004]: cluster 2023-09-23T19:50:16.406204+0000 mgr.y (mgr.25325) 1287 : cluster 0 pgmap v1280: 89 pgs: 89 active+clean; 1.1 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-09-23T19:50:18.198 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:17 smithi016 bash[77995]: cluster 2023-09-23T19:50:16.406204+0000 mgr.y (mgr.25325) 1287 : cluster 0 pgmap v1280: 89 pgs: 89 active+clean; 1.1 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-09-23T19:50:18.219 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:18.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:18.224 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:18.324 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:18.350 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:18.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:18.356 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:18.457 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:18.485 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:18.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:18.490 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:50:18.590 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:18.619 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:18.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:18.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:18.725 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:18.773 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:18.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:18.775 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:50:18.876 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:18.914 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:18.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:18.919 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:19.019 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:19.054 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:19.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:19.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:50:19.162 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:19.192 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:19.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:19.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:50:19.296 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:19.319 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:19.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:19.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:50:19.425 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:19.464 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:19.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:19.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:19.569 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:19.604 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:19.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:19.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:50:19.708 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:19.730 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:19.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:19.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:19.833 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:19.874 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:19.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:19.880 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:50:19.981 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:19.990 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:19 smithi092 bash[58855]: cluster 2023-09-23T19:50:18.406686+0000 mgr.y (mgr.25325) 1288 : cluster 0 pgmap v1281: 89 pgs: 89 active+clean; 1.0 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 622 op/s 2023-09-23T19:50:20.007 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:20.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:20.010 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:20.111 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:20.119 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:19 smithi016 bash[75004]: cluster 2023-09-23T19:50:18.406686+0000 mgr.y (mgr.25325) 1288 : cluster 0 pgmap v1281: 89 pgs: 89 active+clean; 1.0 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 622 op/s 2023-09-23T19:50:20.120 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:19 smithi016 bash[77995]: cluster 2023-09-23T19:50:18.406686+0000 mgr.y (mgr.25325) 1288 : cluster 0 pgmap v1281: 89 pgs: 89 active+clean; 1.0 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 622 op/s 2023-09-23T19:50:20.136 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:20.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:20.138 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:20.239 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:20.267 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:20.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:20.269 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:50:20.370 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:20.394 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:20.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:20.400 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:20.500 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:20.520 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:20.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:20.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:50:20.623 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:20.656 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:20.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:20.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:20.762 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:20.786 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:20.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:20.794 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:50:20.895 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:20.927 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:20.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:20.936 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:21.036 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:21.062 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:21.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:21.064 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:50:21.165 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:21.194 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:21.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:21.196 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:50:21.297 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:21.326 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:21.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:21.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:50:21.431 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:21.474 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:21.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:21.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:50:21.585 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:21.621 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:21.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:21.624 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:50:21.724 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:21.750 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:21.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:21.754 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:21.855 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:21.881 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:21.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:21.884 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:50:21.985 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:22.014 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:22.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:22.017 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:50:22.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:21 smithi016 bash[77995]: cluster 2023-09-23T19:50:20.407305+0000 mgr.y (mgr.25325) 1289 : cluster 0 pgmap v1282: 89 pgs: 89 active+clean; 916 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 691 op/s 2023-09-23T19:50:22.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:21 smithi016 bash[75004]: cluster 2023-09-23T19:50:20.407305+0000 mgr.y (mgr.25325) 1289 : cluster 0 pgmap v1282: 89 pgs: 89 active+clean; 916 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 691 op/s 2023-09-23T19:50:22.117 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:22.146 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:22.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:22.153 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:22.254 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:22.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:21 smithi092 bash[58855]: cluster 2023-09-23T19:50:20.407305+0000 mgr.y (mgr.25325) 1289 : cluster 0 pgmap v1282: 89 pgs: 89 active+clean; 916 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 691 op/s 2023-09-23T19:50:22.288 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:22.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:22.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:22.395 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:22.428 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:22.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:22.431 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:50:22.532 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:22.561 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:22.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:22.565 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:22.665 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:22.705 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:22.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:22.711 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:22.811 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:22.841 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:22.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:22.847 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:50:22.947 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:22.978 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:22.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:22.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:23.081 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:23.117 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:23.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:23.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:23.219 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:23.245 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:23.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:23.247 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:50:23.348 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:23.373 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:23.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:23.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:50:23.477 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:23.521 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:23.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:23.529 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:50:23.630 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:23.653 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:23.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:23.657 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:50:23.758 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:23.793 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:23.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:23.798 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:23.899 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:24.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:23 smithi016 bash[75004]: cluster 2023-09-23T19:50:22.407857+0000 mgr.y (mgr.25325) 1290 : cluster 0 pgmap v1283: 89 pgs: 89 active+clean; 889 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 566 op/s 2023-09-23T19:50:24.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:23 smithi016 bash[77995]: cluster 2023-09-23T19:50:22.407857+0000 mgr.y (mgr.25325) 1290 : cluster 0 pgmap v1283: 89 pgs: 89 active+clean; 889 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 566 op/s 2023-09-23T19:50:24.165 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:24.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:24.171 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:24.271 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:24.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:23 smithi092 bash[58855]: cluster 2023-09-23T19:50:22.407857+0000 mgr.y (mgr.25325) 1290 : cluster 0 pgmap v1283: 89 pgs: 89 active+clean; 889 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 566 op/s 2023-09-23T19:50:24.298 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:24.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:24.301 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:50:24.401 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:24.430 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:24.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:24.433 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:24.534 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:24.561 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:24.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:24.566 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:50:24.667 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:24.702 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:24.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:24.705 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:50:24.805 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:24.850 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:24.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:24.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:24.957 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:24.985 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:24.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:24.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:50:25.088 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:25.133 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:25.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:25.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:25.238 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:25.268 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:25.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:25.270 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:25.371 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:25.400 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:25.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:25.403 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:25.504 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:25.538 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:25.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:25.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:50:25.641 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:25.670 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:25.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:25.676 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:50:25.777 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:25.804 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:25.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:25.808 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:25.905 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:50:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:50:25] "GET /metrics HTTP/1.1" 200 36302 "" "Prometheus/2.33.4" 2023-09-23T19:50:25.909 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:25.941 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:25.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:25.946 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:50:26.046 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:26.071 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:26.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:26.074 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:26.174 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:26.183 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:25 smithi092 bash[58855]: cluster 2023-09-23T19:50:24.408416+0000 mgr.y (mgr.25325) 1291 : cluster 0 pgmap v1284: 89 pgs: 89 active+clean; 800 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 653 op/s 2023-09-23T19:50:26.202 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:26.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:26.203 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:50:26.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:25 smithi016 bash[75004]: cluster 2023-09-23T19:50:24.408416+0000 mgr.y (mgr.25325) 1291 : cluster 0 pgmap v1284: 89 pgs: 89 active+clean; 800 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 653 op/s 2023-09-23T19:50:26.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:26 smithi016 bash[77995]: cluster 2023-09-23T19:50:24.408416+0000 mgr.y (mgr.25325) 1291 : cluster 0 pgmap v1284: 89 pgs: 89 active+clean; 800 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 653 op/s 2023-09-23T19:50:26.304 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:26.345 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:26.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:26.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:50:26.451 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:26.474 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:26.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:26.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:26.578 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:26.610 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:26.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:26.613 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:50:26.714 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:26.736 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:26.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:26.741 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:50:26.841 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:26.869 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:26.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:26.878 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:50:26.979 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:27.007 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:27.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:27.014 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:27.115 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:27.151 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:27.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:27.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:27.257 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:27.287 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:27.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:27.289 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:27.390 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:27.432 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:27.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:27.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:50:27.536 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:27.561 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:27.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:27.568 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:50:27.669 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:27.697 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:27.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:27.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:27.806 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:27.838 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:27.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:27.842 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:50:27.943 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:27.969 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:27.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:27.972 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:50:28.073 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:28.107 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:28.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:28.108 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:50:28.208 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:28.216 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:27 smithi092 bash[58855]: cluster 2023-09-23T19:50:26.409219+0000 mgr.y (mgr.25325) 1292 : cluster 0 pgmap v1285: 89 pgs: 89 active+clean; 687 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2023-09-23T19:50:28.241 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:28.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:28.245 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:28.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:28 smithi016 bash[77995]: cluster 2023-09-23T19:50:26.409219+0000 mgr.y (mgr.25325) 1292 : cluster 0 pgmap v1285: 89 pgs: 89 active+clean; 687 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2023-09-23T19:50:28.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:28 smithi016 bash[75004]: cluster 2023-09-23T19:50:26.409219+0000 mgr.y (mgr.25325) 1292 : cluster 0 pgmap v1285: 89 pgs: 89 active+clean; 687 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2023-09-23T19:50:28.346 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:28.373 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:28.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:28.378 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:28.478 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:28.508 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:28.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:28.514 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:28.615 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:28.642 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:28.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:28.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:50:28.745 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:28.780 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:28.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:28.785 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:28.886 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:28.922 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:28.925 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:28 smithi092 bash[58855]: audit 2023-09-23T19:50:28.672943+0000 mon.a (mon.0) 1601 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:50:28.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:28.929 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:50:29.030 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:29.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:28 smithi016 bash[75004]: audit 2023-09-23T19:50:28.672943+0000 mon.a (mon.0) 1601 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:50:29.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:28 smithi016 bash[77995]: audit 2023-09-23T19:50:28.672943+0000 mon.a (mon.0) 1601 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:50:29.058 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:29.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:29.061 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:29.161 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:29.189 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:29.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:29.191 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:29.292 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:29.332 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:29.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:29.338 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:50:29.438 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:29.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:29.467 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:29.467 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:50:29.568 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:29.595 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:29.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:29.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:50:29.701 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:29.727 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:29.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:29.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:29.830 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:29.856 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:29.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:29.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:50:29.962 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:29.993 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:29.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:29.996 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:50:30.097 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:30.123 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:30.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:30.128 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:30.189 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:29 smithi016 bash[77995]: cluster 2023-09-23T19:50:28.409702+0000 mgr.y (mgr.25325) 1293 : cluster 0 pgmap v1286: 89 pgs: 89 active+clean; 648 MiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 589 op/s 2023-09-23T19:50:30.189 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:29 smithi016 bash[75004]: cluster 2023-09-23T19:50:28.409702+0000 mgr.y (mgr.25325) 1293 : cluster 0 pgmap v1286: 89 pgs: 89 active+clean; 648 MiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 589 op/s 2023-09-23T19:50:30.228 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:30.238 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:29 smithi092 bash[58855]: cluster 2023-09-23T19:50:28.409702+0000 mgr.y (mgr.25325) 1293 : cluster 0 pgmap v1286: 89 pgs: 89 active+clean; 648 MiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 589 op/s 2023-09-23T19:50:30.255 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:30.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:30.260 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:50:30.360 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:30.385 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:30.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:30.394 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:50:30.495 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:30.519 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:50:30.536 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:30.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:30.542 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:30.571 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:50:30.626 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:50:30.643 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:30.669 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:50:30.677 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:30.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:30.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:50:30.712 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:50:30.747 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:50:30.783 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:30.820 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:30.823 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:50:30.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:30.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:30.868 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:50:30.872 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:50:30.872 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:50:30.872 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:50:30.872 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:50:30.873 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:50:30.873 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:50:30.873 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:50:30.873 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:50:30.926 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:30.965 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:30.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:30.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:31.064 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:30 smithi016 bash[75004]: audit 2023-09-23T19:50:30.428912+0000 mon.a (mon.0) 1602 : audit 0 from='client.? 172.21.15.16:0/3262498199' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:50:31.064 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:30 smithi016 bash[77995]: audit 2023-09-23T19:50:30.428912+0000 mon.a (mon.0) 1602 : audit 0 from='client.? 172.21.15.16:0/3262498199' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:50:31.072 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:31.088 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-09-23T19:50:31.089 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [], 2023-09-23T19:50:31.089 INFO:teuthology.orchestra.run.smithi016.stdout: "complaint_time": 30, 2023-09-23T19:50:31.089 INFO:teuthology.orchestra.run.smithi016.stdout: "num_blocked_ops": 0 2023-09-23T19:50:31.089 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-09-23T19:50:31.097 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:31.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:31.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:50:31.200 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:31.209 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:30 smithi092 bash[58855]: audit 2023-09-23T19:50:30.428912+0000 mon.a (mon.0) 1602 : audit 0 from='client.? 172.21.15.16:0/3262498199' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:50:31.224 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:31.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:31.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:50:31.328 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:31.623 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:31.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:31.631 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:50:31.731 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:31.763 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:31.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:31.769 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:31.869 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:31.903 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:31.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:31.908 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:31.954 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T19:50:31.960 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is unset 2023-09-23T19:50:32.009 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:32.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:31 smithi092 bash[58855]: cluster 2023-09-23T19:50:30.410396+0000 mgr.y (mgr.25325) 1294 : cluster 0 pgmap v1287: 89 pgs: 89 active+clean; 507 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 728 op/s 2023-09-23T19:50:32.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:31 smithi092 bash[58855]: audit 2023-09-23T19:50:31.476923+0000 mon.a (mon.0) 1603 : audit 1 from='client.? 172.21.15.16:0/506417415' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:50:32.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:31 smithi092 bash[58855]: audit 2023-09-23T19:50:31.547470+0000 mon.a (mon.0) 1604 : audit 1 from='client.? 172.21.15.16:0/2120203952' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T19:50:32.043 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:32.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:32.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:50:32.149 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:32.177 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:32.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:32.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:32.281 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:32.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:31 smithi016 bash[75004]: cluster 2023-09-23T19:50:30.410396+0000 mgr.y (mgr.25325) 1294 : cluster 0 pgmap v1287: 89 pgs: 89 active+clean; 507 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 728 op/s 2023-09-23T19:50:32.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:31 smithi016 bash[75004]: audit 2023-09-23T19:50:31.476923+0000 mon.a (mon.0) 1603 : audit 1 from='client.? 172.21.15.16:0/506417415' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:50:32.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:31 smithi016 bash[75004]: audit 2023-09-23T19:50:31.547470+0000 mon.a (mon.0) 1604 : audit 1 from='client.? 172.21.15.16:0/2120203952' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T19:50:32.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:31 smithi016 bash[77995]: cluster 2023-09-23T19:50:30.410396+0000 mgr.y (mgr.25325) 1294 : cluster 0 pgmap v1287: 89 pgs: 89 active+clean; 507 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 728 op/s 2023-09-23T19:50:32.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:31 smithi016 bash[77995]: audit 2023-09-23T19:50:31.476923+0000 mon.a (mon.0) 1603 : audit 1 from='client.? 172.21.15.16:0/506417415' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:50:32.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:31 smithi016 bash[77995]: audit 2023-09-23T19:50:31.547470+0000 mon.a (mon.0) 1604 : audit 1 from='client.? 172.21.15.16:0/2120203952' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T19:50:32.305 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:32.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:32.311 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:32.412 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:32.435 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:32.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:32.440 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:50:32.541 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:32.569 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:32.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:32.576 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:50:32.677 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:32.708 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:32.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:32.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:50:32.812 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:32.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:32.853 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:32.853 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:32.954 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:32.980 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:32.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:32.983 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:50:33.084 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:33.109 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:33.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:33.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:33.214 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:33.240 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:33.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:33.247 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:50:33.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:32 smithi016 bash[75004]: cluster 2023-09-23T19:50:31.934001+0000 mon.a (mon.0) 1605 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:50:33.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:32 smithi016 bash[75004]: audit 2023-09-23T19:50:31.942692+0000 mon.a (mon.0) 1606 : audit 1 from='client.? 172.21.15.16:0/506417415' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:50:33.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:32 smithi016 bash[75004]: audit 2023-09-23T19:50:31.942781+0000 mon.a (mon.0) 1607 : audit 1 from='client.? 172.21.15.16:0/2120203952' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T19:50:33.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:32 smithi016 bash[75004]: cluster 2023-09-23T19:50:31.942843+0000 mon.a (mon.0) 1608 : cluster 0 osdmap e227: 8 total, 8 up, 8 in 2023-09-23T19:50:33.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:32 smithi016 bash[75004]: cluster 2023-09-23T19:50:31.946341+0000 osd.4 (osd.4) 167 : cluster 3 failed to encode map e227 with expected crc 2023-09-23T19:50:33.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:32 smithi016 bash[75004]: cluster 2023-09-23T19:50:31.951105+0000 osd.6 (osd.6) 159 : cluster 3 failed to encode map e227 with expected crc 2023-09-23T19:50:33.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:32 smithi016 bash[75004]: cluster 2023-09-23T19:50:31.972308+0000 osd.5 (osd.5) 151 : cluster 3 failed to encode map e227 with expected crc 2023-09-23T19:50:33.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:32 smithi016 bash[77995]: cluster 2023-09-23T19:50:31.934001+0000 mon.a (mon.0) 1605 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:50:33.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:32 smithi016 bash[77995]: audit 2023-09-23T19:50:31.942692+0000 mon.a (mon.0) 1606 : audit 1 from='client.? 172.21.15.16:0/506417415' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:50:33.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:32 smithi016 bash[77995]: audit 2023-09-23T19:50:31.942781+0000 mon.a (mon.0) 1607 : audit 1 from='client.? 172.21.15.16:0/2120203952' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T19:50:33.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:32 smithi016 bash[77995]: cluster 2023-09-23T19:50:31.942843+0000 mon.a (mon.0) 1608 : cluster 0 osdmap e227: 8 total, 8 up, 8 in 2023-09-23T19:50:33.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:32 smithi016 bash[77995]: cluster 2023-09-23T19:50:31.946341+0000 osd.4 (osd.4) 167 : cluster 3 failed to encode map e227 with expected crc 2023-09-23T19:50:33.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:32 smithi016 bash[77995]: cluster 2023-09-23T19:50:31.951105+0000 osd.6 (osd.6) 159 : cluster 3 failed to encode map e227 with expected crc 2023-09-23T19:50:33.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:32 smithi016 bash[77995]: cluster 2023-09-23T19:50:31.972308+0000 osd.5 (osd.5) 151 : cluster 3 failed to encode map e227 with expected crc 2023-09-23T19:50:33.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:32 smithi092 bash[58855]: cluster 2023-09-23T19:50:31.934001+0000 mon.a (mon.0) 1605 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:50:33.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:32 smithi092 bash[58855]: audit 2023-09-23T19:50:31.942692+0000 mon.a (mon.0) 1606 : audit 1 from='client.? 172.21.15.16:0/506417415' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:50:33.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:32 smithi092 bash[58855]: audit 2023-09-23T19:50:31.942781+0000 mon.a (mon.0) 1607 : audit 1 from='client.? 172.21.15.16:0/2120203952' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T19:50:33.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:32 smithi092 bash[58855]: cluster 2023-09-23T19:50:31.942843+0000 mon.a (mon.0) 1608 : cluster 0 osdmap e227: 8 total, 8 up, 8 in 2023-09-23T19:50:33.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:32 smithi092 bash[58855]: cluster 2023-09-23T19:50:31.946341+0000 osd.4 (osd.4) 167 : cluster 3 failed to encode map e227 with expected crc 2023-09-23T19:50:33.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:32 smithi092 bash[58855]: cluster 2023-09-23T19:50:31.951105+0000 osd.6 (osd.6) 159 : cluster 3 failed to encode map e227 with expected crc 2023-09-23T19:50:33.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:32 smithi092 bash[58855]: cluster 2023-09-23T19:50:31.972308+0000 osd.5 (osd.5) 151 : cluster 3 failed to encode map e227 with expected crc 2023-09-23T19:50:33.348 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:33.385 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:33.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:33.386 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:33.487 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:33.516 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:33.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:33.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:33.619 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:33.652 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:33.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:33.657 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:50:33.757 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:33.781 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:33.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:33.786 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:33.887 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:33.927 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:33.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:33.930 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:34.030 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:34.057 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:34.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:34.058 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:34.159 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:34.168 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:33 smithi016 bash[75004]: cluster 2023-09-23T19:50:32.410779+0000 mgr.y (mgr.25325) 1295 : cluster 0 pgmap v1289: 89 pgs: 89 active+clean; 483 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 649 op/s 2023-09-23T19:50:34.168 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:33 smithi016 bash[77995]: cluster 2023-09-23T19:50:32.410779+0000 mgr.y (mgr.25325) 1295 : cluster 0 pgmap v1289: 89 pgs: 89 active+clean; 483 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 649 op/s 2023-09-23T19:50:34.193 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:34.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:34.198 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:50:34.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:33 smithi092 bash[58855]: cluster 2023-09-23T19:50:32.410779+0000 mgr.y (mgr.25325) 1295 : cluster 0 pgmap v1289: 89 pgs: 89 active+clean; 483 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 649 op/s 2023-09-23T19:50:34.299 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:34.326 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:34.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:34.329 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:34.429 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:34.458 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:34.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:34.462 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:34.562 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:34.585 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:34.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:34.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:34.688 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:34.710 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:34.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:34.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:50:34.814 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:34.841 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:34.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:34.844 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:34.945 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:34.969 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:34.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:34.972 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:50:34.989 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T19:50:35.073 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:35.105 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:35.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:35.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:35.214 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:35.255 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:35.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:35.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:50:35.365 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:35.407 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:35.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:35.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:50:35.515 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:35.553 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:35.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:35.555 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:50:35.640 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:50:35 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:50:35] "GET /metrics HTTP/1.1" 200 36299 "" "Prometheus/2.33.4" 2023-09-23T19:50:35.656 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:35.684 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:35.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:35.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:50:35.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:35 smithi092 bash[58855]: cluster 2023-09-23T19:50:34.411413+0000 mgr.y (mgr.25325) 1296 : cluster 0 pgmap v1290: 89 pgs: 89 active+clean; 399 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2023-09-23T19:50:35.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:35 smithi092 bash[58855]: audit 2023-09-23T19:50:34.875671+0000 mon.a (mon.0) 1609 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:50:35.790 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:35.813 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:35.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:35.818 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:50:35.919 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:35.941 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:35.943 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T19:50:35.943 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:35 smithi016 bash[77995]: cluster 2023-09-23T19:50:34.411413+0000 mgr.y (mgr.25325) 1296 : cluster 0 pgmap v1290: 89 pgs: 89 active+clean; 399 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2023-09-23T19:50:35.943 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:35 smithi016 bash[77995]: audit 2023-09-23T19:50:34.875671+0000 mon.a (mon.0) 1609 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:50:35.944 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:35 smithi016 bash[75004]: cluster 2023-09-23T19:50:34.411413+0000 mgr.y (mgr.25325) 1296 : cluster 0 pgmap v1290: 89 pgs: 89 active+clean; 399 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2023-09-23T19:50:35.944 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:35 smithi016 bash[75004]: audit 2023-09-23T19:50:34.875671+0000 mon.a (mon.0) 1609 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:50:35.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:35.945 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:50:36.046 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:36.079 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:36.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:36.087 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:36.187 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:36.253 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:36.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:36.256 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:36.357 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:36.378 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:36.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:36.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:36.481 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:36.520 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:36.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:36.525 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:50:36.626 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:36.655 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:36.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:36.660 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:36.762 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:36.787 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:36.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:36.790 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:50:36.875 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:50:36.891 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:36.966 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:36.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:36.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:37.074 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:37.098 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:37.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:37.100 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:50:37.201 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:37.229 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:37.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:37.234 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:37.335 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:37.370 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:37.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:37.375 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:50:37.476 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:37.512 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:37.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:37.515 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:37.616 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:37.647 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:37.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:37.649 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:50:37.750 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:37.783 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:37.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:37.786 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:37.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:37 smithi016 bash[77995]: cluster 2023-09-23T19:50:36.412164+0000 mgr.y (mgr.25325) 1297 : cluster 0 pgmap v1291: 89 pgs: 89 active+clean; 296 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 625 op/s 2023-09-23T19:50:37.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:37 smithi016 bash[75004]: cluster 2023-09-23T19:50:36.412164+0000 mgr.y (mgr.25325) 1297 : cluster 0 pgmap v1291: 89 pgs: 89 active+clean; 296 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 625 op/s 2023-09-23T19:50:37.866 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T19:50:37.887 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:37.952 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:37.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:37.961 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:38.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:37 smithi092 bash[58855]: cluster 2023-09-23T19:50:36.412164+0000 mgr.y (mgr.25325) 1297 : cluster 0 pgmap v1291: 89 pgs: 89 active+clean; 296 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 625 op/s 2023-09-23T19:50:38.061 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:38.084 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:38.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:38.089 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:38.190 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:38.213 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:38.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:38.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:50:38.321 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:38.359 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:38.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:38.363 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:38.463 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:38.500 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:38.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:38.506 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:38.607 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:38.634 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:38.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:38.638 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:38.739 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:38.766 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:38.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:38.771 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:38.872 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:38.903 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:38.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:38.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:50:39.009 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:39.037 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:39.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:39.041 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:39.141 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:39.174 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:39.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:39.177 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:39.278 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:39.316 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:39.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:39.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:39.425 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:39.454 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:39.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:39.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:39.557 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:39.582 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:39.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:39.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:39.650 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:39 smithi016 bash[75004]: cluster 2023-09-23T19:50:38.416662+0000 mgr.y (mgr.25325) 1298 : cluster 0 pgmap v1292: 89 pgs: 89 active+clean; 263 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 616 op/s 2023-09-23T19:50:39.651 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:39 smithi016 bash[77995]: cluster 2023-09-23T19:50:38.416662+0000 mgr.y (mgr.25325) 1298 : cluster 0 pgmap v1292: 89 pgs: 89 active+clean; 263 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 616 op/s 2023-09-23T19:50:39.686 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:39.718 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:39.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:39.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:39.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:39 smithi092 bash[58855]: cluster 2023-09-23T19:50:38.416662+0000 mgr.y (mgr.25325) 1298 : cluster 0 pgmap v1292: 89 pgs: 89 active+clean; 263 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 616 op/s 2023-09-23T19:50:39.821 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:39.860 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:39.870 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:50:39.870 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:50:39.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:39.874 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:50:39.975 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:40.017 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:40.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:40.020 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:40.121 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:40.153 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:40.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:40.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:50:40.257 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:40.287 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:40.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:40.314 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:50:40.415 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:40.439 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:40.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:40.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:40.543 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:40.579 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:40.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:40.587 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:50:40.687 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:40.722 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:40.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:40.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:40.828 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:40.870 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:40.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:40.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:40.886 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:50:40.974 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:40.996 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:40.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:40.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:50:41.099 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:41.130 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:41.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:41.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:50:41.239 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:41.280 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:41.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:41.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:41.389 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:41.442 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:41.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:41.443 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:50:41.543 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:41.587 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:41.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:41.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:41.694 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:41.764 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:41.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:41.773 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:41.873 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:41.901 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:41.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:41.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:50:41.911 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:50:42.005 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:42.039 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:42.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:42.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:42.146 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:42.176 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:42.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:42.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:42.243 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:42 smithi016 bash[75004]: cluster 2023-09-23T19:50:40.417449+0000 mgr.y (mgr.25325) 1299 : cluster 0 pgmap v1293: 89 pgs: 89 active+clean; 130 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 603 op/s 2023-09-23T19:50:42.243 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:42 smithi016 bash[75004]: audit 2023-09-23T19:50:41.239687+0000 mon.a (mon.0) 1610 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:50:42.243 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:42 smithi016 bash[75004]: audit 2023-09-23T19:50:41.252479+0000 mon.a (mon.0) 1611 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:50:42.244 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:42 smithi016 bash[75004]: audit 2023-09-23T19:50:41.462494+0000 mon.a (mon.0) 1612 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:50:42.244 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:42 smithi016 bash[75004]: audit 2023-09-23T19:50:41.473162+0000 mon.a (mon.0) 1613 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:50:42.244 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:42 smithi016 bash[75004]: audit 2023-09-23T19:50:42.117682+0000 mon.a (mon.0) 1614 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:50:42.244 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:42 smithi016 bash[75004]: audit 2023-09-23T19:50:42.118675+0000 mon.a (mon.0) 1615 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:50:42.244 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:42 smithi016 bash[75004]: audit 2023-09-23T19:50:42.125412+0000 mon.a (mon.0) 1616 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:50:42.280 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:42.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:42 smithi092 bash[58855]: cluster 2023-09-23T19:50:40.417449+0000 mgr.y (mgr.25325) 1299 : cluster 0 pgmap v1293: 89 pgs: 89 active+clean; 130 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 603 op/s 2023-09-23T19:50:42.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:42 smithi092 bash[58855]: audit 2023-09-23T19:50:41.239687+0000 mon.a (mon.0) 1610 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:50:42.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:42 smithi092 bash[58855]: audit 2023-09-23T19:50:41.252479+0000 mon.a (mon.0) 1611 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:50:42.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:42 smithi092 bash[58855]: audit 2023-09-23T19:50:41.462494+0000 mon.a (mon.0) 1612 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:50:42.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:42 smithi092 bash[58855]: audit 2023-09-23T19:50:41.473162+0000 mon.a (mon.0) 1613 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:50:42.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:42 smithi092 bash[58855]: audit 2023-09-23T19:50:42.117682+0000 mon.a (mon.0) 1614 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:50:42.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:42 smithi092 bash[58855]: audit 2023-09-23T19:50:42.118675+0000 mon.a (mon.0) 1615 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:50:42.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:42 smithi092 bash[58855]: audit 2023-09-23T19:50:42.125412+0000 mon.a (mon.0) 1616 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:50:42.316 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:42.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:42.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:50:42.419 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:42.428 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Removed 102678 objects 2023-09-23T19:50:42.428 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Clean up completed and total clean up time :158.265 2023-09-23T19:50:42.440 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:42.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:42.443 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:42.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:42 smithi016 bash[77995]: cluster 2023-09-23T19:50:40.417449+0000 mgr.y (mgr.25325) 1299 : cluster 0 pgmap v1293: 89 pgs: 89 active+clean; 130 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 603 op/s 2023-09-23T19:50:42.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:42 smithi016 bash[77995]: audit 2023-09-23T19:50:41.239687+0000 mon.a (mon.0) 1610 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:50:42.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:42 smithi016 bash[77995]: audit 2023-09-23T19:50:41.252479+0000 mon.a (mon.0) 1611 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:50:42.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:42 smithi016 bash[77995]: audit 2023-09-23T19:50:41.462494+0000 mon.a (mon.0) 1612 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:50:42.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:42 smithi016 bash[77995]: audit 2023-09-23T19:50:41.473162+0000 mon.a (mon.0) 1613 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:50:42.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:42 smithi016 bash[77995]: audit 2023-09-23T19:50:42.117682+0000 mon.a (mon.0) 1614 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:50:42.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:42 smithi016 bash[77995]: audit 2023-09-23T19:50:42.118675+0000 mon.a (mon.0) 1615 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:50:42.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:42 smithi016 bash[77995]: audit 2023-09-23T19:50:42.125412+0000 mon.a (mon.0) 1616 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:50:42.543 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:42.571 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:42.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:42.575 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:50:42.676 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:42.706 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:42.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:42.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:50:42.810 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:42.843 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:42.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:42.846 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:50:42.947 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:42.974 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:42.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:42.977 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:43.078 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:43.103 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:43.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:43.106 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:43.206 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:43.227 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:43.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:43.230 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:43.331 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:43.352 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:43.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:43.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:43.456 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:43.478 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:43.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:43.480 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:43.581 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:43.609 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:43.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:43.612 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:43.713 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:43.746 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:43.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:43.749 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:50:43.850 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:43.883 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:43.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:43.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:50:43.987 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:44.014 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:44.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:44.017 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:44.118 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:44.144 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:44.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:44.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:50:44.241 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_7 2023-09-23T19:50:44.241 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_7 unique_pool_7 --yes-i-really-really-mean-it 2023-09-23T19:50:44.247 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:44.277 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:44.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:44.280 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:44.382 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:44.421 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:44.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:44.423 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:50:44.524 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:44.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:44 smithi016 bash[75004]: cluster 2023-09-23T19:50:42.417959+0000 mgr.y (mgr.25325) 1300 : cluster 0 pgmap v1294: 89 pgs: 89 active+clean; 108 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 571 op/s 2023-09-23T19:50:44.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:44 smithi016 bash[75004]: audit 2023-09-23T19:50:43.674121+0000 mon.a (mon.0) 1617 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:50:44.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:44 smithi016 bash[77995]: cluster 2023-09-23T19:50:42.417959+0000 mgr.y (mgr.25325) 1300 : cluster 0 pgmap v1294: 89 pgs: 89 active+clean; 108 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 571 op/s 2023-09-23T19:50:44.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:44 smithi016 bash[77995]: audit 2023-09-23T19:50:43.674121+0000 mon.a (mon.0) 1617 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:50:44.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:44 smithi092 bash[58855]: cluster 2023-09-23T19:50:42.417959+0000 mgr.y (mgr.25325) 1300 : cluster 0 pgmap v1294: 89 pgs: 89 active+clean; 108 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 571 op/s 2023-09-23T19:50:44.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:44 smithi092 bash[58855]: audit 2023-09-23T19:50:43.674121+0000 mon.a (mon.0) 1617 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:50:44.557 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:44.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:44.561 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:50:44.661 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:44.691 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:44.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:44.694 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:44.795 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:44.823 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:44.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:44.827 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:50:44.929 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:44.955 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:44.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:44.958 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:45.059 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:45.091 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:45.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:45.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:50:45.194 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:45.226 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:45.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:45.228 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:50:45.329 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:45.361 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:45.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:45.364 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:50:45.465 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:45.490 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:45.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:45.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:45.593 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:45.629 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:45.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:45.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:45.732 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:45.762 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:45.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:45.765 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:50:45.865 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:45.896 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:45.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:45.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:50:46.000 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:46.034 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:46.036 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:50:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:50:45] "GET /metrics HTTP/1.1" 200 36297 "" "Prometheus/2.33.4" 2023-09-23T19:50:46.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:46.038 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:50:46.138 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:46.163 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:46.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:46.165 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:46.266 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:46.299 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:46.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:46.303 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:46.339 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:46 smithi016 bash[75004]: cluster 2023-09-23T19:50:44.418400+0000 mgr.y (mgr.25325) 1301 : cluster 0 pgmap v1295: 89 pgs: 89 active+clean; 47 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 580 op/s 2023-09-23T19:50:46.403 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:46.433 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:46.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:46.437 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:46.537 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:46.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:46 smithi092 bash[58855]: cluster 2023-09-23T19:50:44.418400+0000 mgr.y (mgr.25325) 1301 : cluster 0 pgmap v1295: 89 pgs: 89 active+clean; 47 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 580 op/s 2023-09-23T19:50:46.567 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:46.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:46.569 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:46.670 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:46.696 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:46.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:46.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:50:46.728 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:46 smithi016 bash[77995]: cluster 2023-09-23T19:50:44.418400+0000 mgr.y (mgr.25325) 1301 : cluster 0 pgmap v1295: 89 pgs: 89 active+clean; 47 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 580 op/s 2023-09-23T19:50:46.800 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:46.828 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:46.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:46.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:50:46.931 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:46.957 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:46.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:46.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:47.060 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:47.085 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:47.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:47.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:47.188 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:47.214 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:47.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:47.218 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:47.318 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:47.350 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:47.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:47.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:50:47.453 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:47.486 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:47.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:47.489 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:47.589 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:47.615 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:47.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:47.618 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:50:47.719 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:47.744 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:47.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:47.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:50:47.847 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:47.880 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:47.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:47.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:50:47.983 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:48.015 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:48.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:48.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:50:48.119 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:48.147 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:48.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:48.150 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:50:48.251 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:48.281 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:48.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:48.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:50:48.384 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:48.395 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:48 smithi092 bash[58855]: cluster 2023-09-23T19:50:46.419259+0000 mgr.y (mgr.25325) 1302 : cluster 0 pgmap v1296: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 531 op/s 2023-09-23T19:50:48.413 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:48.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:48.415 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:48.516 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:48.527 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:48 smithi016 bash[75004]: cluster 2023-09-23T19:50:46.419259+0000 mgr.y (mgr.25325) 1302 : cluster 0 pgmap v1296: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 531 op/s 2023-09-23T19:50:48.528 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:48 smithi016 bash[77995]: cluster 2023-09-23T19:50:46.419259+0000 mgr.y (mgr.25325) 1302 : cluster 0 pgmap v1296: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 531 op/s 2023-09-23T19:50:48.541 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:48.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:48.544 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:48.645 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:48.671 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:48.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:48.674 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:50:48.774 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:48.807 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:48.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:48.810 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:48.911 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:48.942 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:48.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:48.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:50:49.045 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:49.067 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:49.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:49.070 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:49.170 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:49.202 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:49.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:49.205 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:49.306 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:49.335 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:49.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:49.338 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:49.439 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:49.474 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:49.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:49.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:49.578 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:49.606 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:49.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:49.610 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:49.710 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:49.745 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:49.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:49.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:50:49.849 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:49.882 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:49.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:49.885 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:50:49.986 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:50.015 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:50.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:50.017 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:50:50.118 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:50.148 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:50.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:50.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:50:50.251 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:50.277 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:50.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:50.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:50:50.382 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:50.393 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:50 smithi016 bash[75004]: cluster 2023-09-23T19:50:48.419983+0000 mgr.y (mgr.25325) 1303 : cluster 0 pgmap v1297: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 393 op/s 2023-09-23T19:50:50.393 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:50 smithi016 bash[77995]: cluster 2023-09-23T19:50:48.419983+0000 mgr.y (mgr.25325) 1303 : cluster 0 pgmap v1297: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 393 op/s 2023-09-23T19:50:50.406 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:50.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:50.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:50.510 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:50.523 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:50 smithi092 bash[58855]: cluster 2023-09-23T19:50:48.419983+0000 mgr.y (mgr.25325) 1303 : cluster 0 pgmap v1297: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 393 op/s 2023-09-23T19:50:50.539 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:50.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:50.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:50.644 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:50.675 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:50.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:50.678 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:50:50.778 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:50.804 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:50.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:50.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:50:50.908 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:50.938 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:50.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:50.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:50:51.042 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:51.074 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:51.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:51.077 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:50:51.178 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:51.209 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:51.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:51.212 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:51.313 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:51.349 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:51.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:51.351 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:50:51.452 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:51.488 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:51.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:51.491 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:50:51.592 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:51.620 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:51.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:51.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:50:51.723 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:51.747 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:51.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:51.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:50:51.851 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:51.885 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:51.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:51.889 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:50:51.989 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:52.016 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:52.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:52.019 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:50:52.119 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:52.153 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:52.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:52.156 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:50:52.257 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:52.260 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:50:52.262 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:52 smithi016 bash[75004]: cluster 2023-09-23T19:50:50.420796+0000 mgr.y (mgr.25325) 1304 : cluster 0 pgmap v1298: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 349 op/s 2023-09-23T19:50:52.262 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:52 smithi016 bash[75004]: audit 2023-09-23T19:50:52.230241+0000 mon.c (mon.2) 204 : audit 0 from='client.? 172.21.15.16:0/121076319' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:50:52.280 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:50:52.288 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:52.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:52.293 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:50:52.301 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:50:52.340 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:50:52.369 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:50:52.393 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:52.395 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:50:52.415 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:52.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:52.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:52.418 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:50:52.442 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:50:52.445 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:50:52.445 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:50:52.445 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:50:52.445 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:50:52.445 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:50:52.445 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:50:52.445 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:50:52.446 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:50:52.519 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:52.526 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:52 smithi016 bash[77995]: cluster 2023-09-23T19:50:50.420796+0000 mgr.y (mgr.25325) 1304 : cluster 0 pgmap v1298: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 349 op/s 2023-09-23T19:50:52.526 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:52 smithi016 bash[77995]: audit 2023-09-23T19:50:52.230241+0000 mon.c (mon.2) 204 : audit 0 from='client.? 172.21.15.16:0/121076319' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:50:52.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:52 smithi092 bash[58855]: cluster 2023-09-23T19:50:50.420796+0000 mgr.y (mgr.25325) 1304 : cluster 0 pgmap v1298: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 349 op/s 2023-09-23T19:50:52.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:52 smithi092 bash[58855]: audit 2023-09-23T19:50:52.230241+0000 mon.c (mon.2) 204 : audit 0 from='client.? 172.21.15.16:0/121076319' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:50:52.547 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:52.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:52.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:50:52.565 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-09-23T19:50:52.566 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:50:52.611 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:50:52.650 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:52.681 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:52.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:52.684 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:52.785 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:52.817 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:52.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:52.820 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:50:52.920 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:52.941 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:52.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:52.944 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:53.045 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:53.077 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:53.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:53.080 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:53.180 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:53.203 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:53.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:53.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:53.264 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:53 smithi016 bash[75004]: audit 2023-09-23T19:50:52.470182+0000 mon.a (mon.0) 1618 : audit 1 from='client.? 172.21.15.16:0/3870928267' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T19:50:53.264 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:53 smithi016 bash[75004]: audit 2023-09-23T19:50:52.901566+0000 mon.a (mon.0) 1619 : audit 1 from='client.? 172.21.15.16:0/903660472' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:50:53.278 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T19:50:53.278 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is unset 2023-09-23T19:50:53.307 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:53.328 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:53.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:53.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:50:53.431 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:53.443 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:53 smithi016 bash[77995]: audit 2023-09-23T19:50:52.470182+0000 mon.a (mon.0) 1618 : audit 1 from='client.? 172.21.15.16:0/3870928267' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T19:50:53.443 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:53 smithi016 bash[77995]: audit 2023-09-23T19:50:52.901566+0000 mon.a (mon.0) 1619 : audit 1 from='client.? 172.21.15.16:0/903660472' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:50:53.461 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:53.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:53.464 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:50:53.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:53 smithi092 bash[58855]: audit 2023-09-23T19:50:52.470182+0000 mon.a (mon.0) 1618 : audit 1 from='client.? 172.21.15.16:0/3870928267' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T19:50:53.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:53 smithi092 bash[58855]: audit 2023-09-23T19:50:52.901566+0000 mon.a (mon.0) 1619 : audit 1 from='client.? 172.21.15.16:0/903660472' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:50:53.565 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:53.595 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:53.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:53.598 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:53.699 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:53.733 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:53.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:53.737 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:50:53.838 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:53.867 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:53.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:53.870 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:50:53.971 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:54.006 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:54.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:54.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:50:54.110 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:54.141 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:54.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:54.144 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:50:54.244 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:54.267 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:54.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:54.273 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:50:54.374 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:54.410 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:54.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:54.413 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:50:54.513 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:54.527 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:54 smithi092 bash[58855]: cluster 2023-09-23T19:50:52.421334+0000 mgr.y (mgr.25325) 1305 : cluster 0 pgmap v1299: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 172 op/s 2023-09-23T19:50:54.528 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:54 smithi092 bash[58855]: cluster 2023-09-23T19:50:53.257837+0000 mon.a (mon.0) 1620 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:50:54.528 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:54 smithi092 bash[58855]: cluster 2023-09-23T19:50:53.257885+0000 mon.a (mon.0) 1621 : cluster 1 Cluster is now healthy 2023-09-23T19:50:54.528 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:54 smithi092 bash[58855]: audit 2023-09-23T19:50:53.268925+0000 mon.a (mon.0) 1622 : audit 1 from='client.? 172.21.15.16:0/3870928267' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T19:50:54.528 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:54 smithi092 bash[58855]: audit 2023-09-23T19:50:53.269034+0000 mon.a (mon.0) 1623 : audit 1 from='client.? 172.21.15.16:0/903660472' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:50:54.528 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:54 smithi092 bash[58855]: cluster 2023-09-23T19:50:53.269118+0000 mon.a (mon.0) 1624 : cluster 0 osdmap e228: 8 total, 8 up, 8 in 2023-09-23T19:50:54.528 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:54 smithi092 bash[58855]: cluster 2023-09-23T19:50:53.273590+0000 osd.5 (osd.5) 152 : cluster 3 failed to encode map e228 with expected crc 2023-09-23T19:50:54.528 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:54 smithi092 bash[58855]: cluster 2023-09-23T19:50:53.354957+0000 osd.7 (osd.7) 146 : cluster 3 failed to encode map e228 with expected crc 2023-09-23T19:50:54.528 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:54 smithi092 bash[58855]: cluster 2023-09-23T19:50:53.355062+0000 osd.4 (osd.4) 168 : cluster 3 failed to encode map e228 with expected crc 2023-09-23T19:50:54.528 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:54 smithi092 bash[58855]: cluster 2023-09-23T19:50:53.355153+0000 osd.6 (osd.6) 160 : cluster 3 failed to encode map e228 with expected crc 2023-09-23T19:50:54.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:54 smithi016 bash[75004]: cluster 2023-09-23T19:50:52.421334+0000 mgr.y (mgr.25325) 1305 : cluster 0 pgmap v1299: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 172 op/s 2023-09-23T19:50:54.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:54 smithi016 bash[75004]: cluster 2023-09-23T19:50:53.257837+0000 mon.a (mon.0) 1620 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:50:54.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:54 smithi016 bash[75004]: cluster 2023-09-23T19:50:53.257885+0000 mon.a (mon.0) 1621 : cluster 1 Cluster is now healthy 2023-09-23T19:50:54.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:54 smithi016 bash[75004]: audit 2023-09-23T19:50:53.268925+0000 mon.a (mon.0) 1622 : audit 1 from='client.? 172.21.15.16:0/3870928267' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T19:50:54.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:54 smithi016 bash[75004]: audit 2023-09-23T19:50:53.269034+0000 mon.a (mon.0) 1623 : audit 1 from='client.? 172.21.15.16:0/903660472' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:50:54.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:54 smithi016 bash[75004]: cluster 2023-09-23T19:50:53.269118+0000 mon.a (mon.0) 1624 : cluster 0 osdmap e228: 8 total, 8 up, 8 in 2023-09-23T19:50:54.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:54 smithi016 bash[75004]: cluster 2023-09-23T19:50:53.273590+0000 osd.5 (osd.5) 152 : cluster 3 failed to encode map e228 with expected crc 2023-09-23T19:50:54.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:54 smithi016 bash[75004]: cluster 2023-09-23T19:50:53.354957+0000 osd.7 (osd.7) 146 : cluster 3 failed to encode map e228 with expected crc 2023-09-23T19:50:54.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:54 smithi016 bash[75004]: cluster 2023-09-23T19:50:53.355062+0000 osd.4 (osd.4) 168 : cluster 3 failed to encode map e228 with expected crc 2023-09-23T19:50:54.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:54 smithi016 bash[75004]: cluster 2023-09-23T19:50:53.355153+0000 osd.6 (osd.6) 160 : cluster 3 failed to encode map e228 with expected crc 2023-09-23T19:50:54.538 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:54 smithi016 bash[77995]: cluster 2023-09-23T19:50:52.421334+0000 mgr.y (mgr.25325) 1305 : cluster 0 pgmap v1299: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 172 op/s 2023-09-23T19:50:54.538 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:54 smithi016 bash[77995]: cluster 2023-09-23T19:50:53.257837+0000 mon.a (mon.0) 1620 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:50:54.538 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:54 smithi016 bash[77995]: cluster 2023-09-23T19:50:53.257885+0000 mon.a (mon.0) 1621 : cluster 1 Cluster is now healthy 2023-09-23T19:50:54.538 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:54 smithi016 bash[77995]: audit 2023-09-23T19:50:53.268925+0000 mon.a (mon.0) 1622 : audit 1 from='client.? 172.21.15.16:0/3870928267' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T19:50:54.538 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:54 smithi016 bash[77995]: audit 2023-09-23T19:50:53.269034+0000 mon.a (mon.0) 1623 : audit 1 from='client.? 172.21.15.16:0/903660472' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:50:54.538 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:54 smithi016 bash[77995]: cluster 2023-09-23T19:50:53.269118+0000 mon.a (mon.0) 1624 : cluster 0 osdmap e228: 8 total, 8 up, 8 in 2023-09-23T19:50:54.538 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:54 smithi016 bash[77995]: cluster 2023-09-23T19:50:53.273590+0000 osd.5 (osd.5) 152 : cluster 3 failed to encode map e228 with expected crc 2023-09-23T19:50:54.538 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:54 smithi016 bash[77995]: cluster 2023-09-23T19:50:53.354957+0000 osd.7 (osd.7) 146 : cluster 3 failed to encode map e228 with expected crc 2023-09-23T19:50:54.538 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:54 smithi016 bash[77995]: cluster 2023-09-23T19:50:53.355062+0000 osd.4 (osd.4) 168 : cluster 3 failed to encode map e228 with expected crc 2023-09-23T19:50:54.539 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:54 smithi016 bash[77995]: cluster 2023-09-23T19:50:53.355153+0000 osd.6 (osd.6) 160 : cluster 3 failed to encode map e228 with expected crc 2023-09-23T19:50:54.544 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:54.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:54.547 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:50:54.648 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:54.676 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:54.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:54.678 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:50:54.779 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:54.809 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:54.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:54.813 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:50:54.914 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:54.939 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:54.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:54.941 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:50:55.042 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:55.067 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:55.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:55.071 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:50:55.171 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:55.202 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:55.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:55.204 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:55.305 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:55.338 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:55.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:55.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:50:55.442 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:55.479 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:55.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:55.482 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:55.583 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:55.607 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:55.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:55.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:55.638 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T19:50:55.712 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:55.737 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:50:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:50:55] "GET /metrics HTTP/1.1" 200 36297 "" "Prometheus/2.33.4" 2023-09-23T19:50:55.755 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:55.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:55.758 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:50:55.858 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:55.890 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:55.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:55.893 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:50:55.993 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:56.023 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:56.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:56.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:56.128 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:56.163 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:56.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:56.168 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:50:56.269 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:56.298 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:56.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:56.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:56.401 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:56.436 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:50:56.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:56.438 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:50:56.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:56 smithi016 bash[77995]: cluster 2023-09-23T19:50:54.422043+0000 mgr.y (mgr.25325) 1306 : cluster 0 pgmap v1301: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 74 op/s 2023-09-23T19:50:56.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:56 smithi016 bash[75004]: cluster 2023-09-23T19:50:54.422043+0000 mgr.y (mgr.25325) 1306 : cluster 0 pgmap v1301: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 74 op/s 2023-09-23T19:50:56.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:56 smithi092 bash[58855]: cluster 2023-09-23T19:50:54.422043+0000 mgr.y (mgr.25325) 1306 : cluster 0 pgmap v1301: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 74 op/s 2023-09-23T19:50:56.539 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:56.570 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:56.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:56.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:50:56.674 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:56.701 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:56.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:56.703 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:50:56.789 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T19:50:56.804 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:56.864 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:56.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:56.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:50:56.968 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:57.006 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:57.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:57.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:50:57.110 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:57.140 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:57.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:57.143 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:57.245 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:57.278 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:57.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:57.281 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:57.382 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:57.392 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:57 smithi016 bash[77995]: cluster 2023-09-23T19:50:56.422820+0000 mgr.y (mgr.25325) 1307 : cluster 0 pgmap v1302: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:50:57.392 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:57 smithi016 bash[75004]: cluster 2023-09-23T19:50:56.422820+0000 mgr.y (mgr.25325) 1307 : cluster 0 pgmap v1302: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:50:57.412 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:57.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:57.415 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:50:57.509 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:50:57.516 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:57.549 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:57.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:57.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:50:57.653 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:57.667 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:57 smithi092 bash[58855]: cluster 2023-09-23T19:50:56.422820+0000 mgr.y (mgr.25325) 1307 : cluster 0 pgmap v1302: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:50:57.688 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:57.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:57.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:50:57.792 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:57.825 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:57.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:57.828 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:50:57.929 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:57.953 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:57.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:57.956 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:50:58.057 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:58.087 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:50:58.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:58.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:50:58.191 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:58.220 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:58.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:58.223 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:50:58.324 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:58.347 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:58.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:58.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:50:58.451 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:58.479 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:58.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:58.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:58.582 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:58.614 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:58.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:58.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:50:58.718 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:58.749 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T19:50:58.752 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:58.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:58.756 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:50:58.856 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:58.881 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:50:58.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:58.884 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:50:58.985 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:59.021 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:50:59.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:59.024 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:50:59.125 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:59.156 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:59.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:59.159 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:50:59.260 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:59.295 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:59.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:59.298 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:59.398 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:59.428 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:50:59.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:59.431 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:50:59.532 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:59.562 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:50:59.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:59.566 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:50:59.667 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:59.699 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:50:59.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:59.702 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:50:59.802 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:59.837 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:59.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:59.840 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:50:59.941 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:59.952 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:59 smithi016 bash[75004]: cluster 2023-09-23T19:50:58.423553+0000 mgr.y (mgr.25325) 1308 : cluster 0 pgmap v1303: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:50:59.952 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:59 smithi016 bash[75004]: audit 2023-09-23T19:50:58.679113+0000 mon.a (mon.0) 1625 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:50:59.952 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:50:59 smithi016 bash[75004]: audit 2023-09-23T19:50:58.680348+0000 mon.a (mon.0) 1626 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:50:59.952 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:59 smithi016 bash[77995]: cluster 2023-09-23T19:50:58.423553+0000 mgr.y (mgr.25325) 1308 : cluster 0 pgmap v1303: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:50:59.952 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:59 smithi016 bash[77995]: audit 2023-09-23T19:50:58.679113+0000 mon.a (mon.0) 1625 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:50:59.952 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:50:59 smithi016 bash[77995]: audit 2023-09-23T19:50:58.680348+0000 mon.a (mon.0) 1626 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:50:59.968 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:50:59.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:50:59.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:00.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:59 smithi092 bash[58855]: cluster 2023-09-23T19:50:58.423553+0000 mgr.y (mgr.25325) 1308 : cluster 0 pgmap v1303: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:00.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:59 smithi092 bash[58855]: audit 2023-09-23T19:50:58.679113+0000 mon.a (mon.0) 1625 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:51:00.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:50:59 smithi092 bash[58855]: audit 2023-09-23T19:50:58.680348+0000 mon.a (mon.0) 1626 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:51:00.071 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:00.099 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:00.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:00.102 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:00.203 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:00.235 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:00.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:00.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:00.339 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:00.374 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:00.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:00.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:00.478 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:00.511 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:00.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:00.514 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:00.615 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:00.646 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:00.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:00.649 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:51:00.717 INFO:teuthology.orchestra.run.smithi016.stderr:pool 'unique_pool_7' removed 2023-09-23T19:51:00.750 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:00.784 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:00.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:00.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:51:00.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:00 smithi092 bash[58855]: audit 2023-09-23T19:50:59.866671+0000 mon.a (mon.0) 1627 : audit 1 from='client.? 172.21.15.16:0/3221631209' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T19:51:00.887 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:00.921 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:00.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:00.925 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:00.967 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:00 smithi016 bash[75004]: audit 2023-09-23T19:50:59.866671+0000 mon.a (mon.0) 1627 : audit 1 from='client.? 172.21.15.16:0/3221631209' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T19:51:00.969 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:00 smithi016 bash[77995]: audit 2023-09-23T19:50:59.866671+0000 mon.a (mon.0) 1627 : audit 1 from='client.? 172.21.15.16:0/3221631209' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T19:51:01.025 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:01.051 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:01.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:01.054 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:51:01.154 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:01.183 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:01.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:01.185 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:01.286 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:01.318 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:01.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:01.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:51:01.422 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:01.450 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:01.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:01.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:51:01.553 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:01.577 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:01.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:01.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:01.682 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:01.705 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:01.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:01.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:51:01.809 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:01.821 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:01 smithi092 bash[58855]: cluster 2023-09-23T19:51:00.424676+0000 mgr.y (mgr.25325) 1309 : cluster 0 pgmap v1304: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:01.821 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:01 smithi092 bash[58855]: audit 2023-09-23T19:51:00.690383+0000 mon.a (mon.0) 1628 : audit 1 from='client.? 172.21.15.16:0/3221631209' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T19:51:01.822 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:01 smithi092 bash[58855]: cluster 2023-09-23T19:51:00.690489+0000 mon.a (mon.0) 1629 : cluster 0 osdmap e229: 8 total, 8 up, 8 in 2023-09-23T19:51:01.822 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:01 smithi092 bash[58855]: cluster 2023-09-23T19:51:00.693613+0000 osd.4 (osd.4) 169 : cluster 3 failed to encode map e229 with expected crc 2023-09-23T19:51:01.822 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:01 smithi092 bash[58855]: cluster 2023-09-23T19:51:01.025482+0000 osd.5 (osd.5) 153 : cluster 3 failed to encode map e229 with expected crc 2023-09-23T19:51:01.822 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:01 smithi092 bash[58855]: cluster 2023-09-23T19:51:01.129567+0000 osd.6 (osd.6) 161 : cluster 3 failed to encode map e229 with expected crc 2023-09-23T19:51:01.831 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T19:51:01.831 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T19:51:01.832 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_8 2023-09-23T19:51:01.832 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_8 16 2023-09-23T19:51:01.840 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:01.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:01.843 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:51:01.944 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:01.955 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:01 smithi016 bash[75004]: cluster 2023-09-23T19:51:00.424676+0000 mgr.y (mgr.25325) 1309 : cluster 0 pgmap v1304: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:01.955 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:01 smithi016 bash[75004]: audit 2023-09-23T19:51:00.690383+0000 mon.a (mon.0) 1628 : audit 1 from='client.? 172.21.15.16:0/3221631209' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T19:51:01.956 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:01 smithi016 bash[75004]: cluster 2023-09-23T19:51:00.690489+0000 mon.a (mon.0) 1629 : cluster 0 osdmap e229: 8 total, 8 up, 8 in 2023-09-23T19:51:01.956 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:01 smithi016 bash[75004]: cluster 2023-09-23T19:51:00.693613+0000 osd.4 (osd.4) 169 : cluster 3 failed to encode map e229 with expected crc 2023-09-23T19:51:01.956 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:01 smithi016 bash[75004]: cluster 2023-09-23T19:51:01.025482+0000 osd.5 (osd.5) 153 : cluster 3 failed to encode map e229 with expected crc 2023-09-23T19:51:01.956 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:01 smithi016 bash[75004]: cluster 2023-09-23T19:51:01.129567+0000 osd.6 (osd.6) 161 : cluster 3 failed to encode map e229 with expected crc 2023-09-23T19:51:01.957 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:01 smithi016 bash[77995]: cluster 2023-09-23T19:51:00.424676+0000 mgr.y (mgr.25325) 1309 : cluster 0 pgmap v1304: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:01.957 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:01 smithi016 bash[77995]: audit 2023-09-23T19:51:00.690383+0000 mon.a (mon.0) 1628 : audit 1 from='client.? 172.21.15.16:0/3221631209' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T19:51:01.957 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:01 smithi016 bash[77995]: cluster 2023-09-23T19:51:00.690489+0000 mon.a (mon.0) 1629 : cluster 0 osdmap e229: 8 total, 8 up, 8 in 2023-09-23T19:51:01.957 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:01 smithi016 bash[77995]: cluster 2023-09-23T19:51:00.693613+0000 osd.4 (osd.4) 169 : cluster 3 failed to encode map e229 with expected crc 2023-09-23T19:51:01.957 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:01 smithi016 bash[77995]: cluster 2023-09-23T19:51:01.025482+0000 osd.5 (osd.5) 153 : cluster 3 failed to encode map e229 with expected crc 2023-09-23T19:51:01.957 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:01 smithi016 bash[77995]: cluster 2023-09-23T19:51:01.129567+0000 osd.6 (osd.6) 161 : cluster 3 failed to encode map e229 with expected crc 2023-09-23T19:51:01.974 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:01.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:01.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:51:02.078 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:02.111 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:02.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:02.114 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:02.215 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:02.247 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:02.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:02.250 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:02.252 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:51:02.253 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:51:02.254 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:51:02.351 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:02.384 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:02.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:02.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:02.488 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:02.522 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:02.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:02.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:02.626 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:02.652 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:02.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:02.655 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:02.755 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:02.786 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:02.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:02.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:51:02.890 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:02.924 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:02.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:02.926 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:03.027 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:03.056 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:03.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:03.059 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:03.159 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:03.191 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:03.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:03.193 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:03.294 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:03.326 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:03.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:03.329 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:51:03.430 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:03.460 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:03.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:03.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:51:03.564 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:03.590 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:03.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:03.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:03.694 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:03.726 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:03.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:03.728 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:03.829 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:03.838 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:03 smithi016 bash[75004]: cluster 2023-09-23T19:51:02.425419+0000 mgr.y (mgr.25325) 1310 : cluster 0 pgmap v1306: 73 pgs: 73 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:03.838 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:03 smithi016 bash[77995]: cluster 2023-09-23T19:51:02.425419+0000 mgr.y (mgr.25325) 1310 : cluster 0 pgmap v1306: 73 pgs: 73 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:03.853 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:03.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:03.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:03.956 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:03.980 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:03.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:03.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:51:04.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:03 smithi092 bash[58855]: cluster 2023-09-23T19:51:02.425419+0000 mgr.y (mgr.25325) 1310 : cluster 0 pgmap v1306: 73 pgs: 73 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:04.083 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:04.113 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:04.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:04.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:04.217 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:04.246 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:04.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:04.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:04.333 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:51:04.351 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:04.374 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:04.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:04.377 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:51:04.477 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:04.508 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:04.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:04.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:51:04.611 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:04.648 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:04.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:04.651 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:04.752 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:04.786 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:04.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:04.789 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:04.890 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:04.919 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:04.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:04.922 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:05.023 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:05.053 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:05.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:05.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:51:05.156 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:05.181 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:05.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:05.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:05.285 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:05.315 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:05.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:05.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:51:05.419 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:05.451 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:05.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:05.455 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:05.555 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:05.591 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:05.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:05.595 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:05.696 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:05.699 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:51:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:51:05] "GET /metrics HTTP/1.1" 200 36254 "" "Prometheus/2.33.4" 2023-09-23T19:51:05.725 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:05.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:05.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:05.829 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:05.843 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:05 smithi092 bash[58855]: cluster 2023-09-23T19:51:04.426019+0000 mgr.y (mgr.25325) 1311 : cluster 0 pgmap v1307: 73 pgs: 73 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:05.863 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:05.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:05.866 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:05.967 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:05.975 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:05 smithi016 bash[75004]: cluster 2023-09-23T19:51:04.426019+0000 mgr.y (mgr.25325) 1311 : cluster 0 pgmap v1307: 73 pgs: 73 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:05.975 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:05 smithi016 bash[77995]: cluster 2023-09-23T19:51:04.426019+0000 mgr.y (mgr.25325) 1311 : cluster 0 pgmap v1307: 73 pgs: 73 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:05.992 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:05.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:05.995 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:06.095 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:06.121 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:06.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:06.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:06.225 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:06.246 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:06.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:06.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:06.348 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:06.375 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:06.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:06.378 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:51:06.479 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:06.505 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:06.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:06.508 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:06.609 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:06.635 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:06.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:06.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:06.739 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:06.765 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:06.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:06.768 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:06.868 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:06.897 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:06.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:06.900 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:07.000 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:07.027 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:07.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:07.030 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:07.131 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:07.158 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:07.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:07.160 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:07.261 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:07.288 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:07.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:07.291 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:07.392 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:07.425 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:07.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:07.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:51:07.529 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:07.559 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:07.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:07.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:07.663 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:07.695 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:07.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:07.698 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:07.799 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:07.829 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:07.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:07.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:07.934 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:07.942 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:07 smithi016 bash[77995]: cluster 2023-09-23T19:51:06.426934+0000 mgr.y (mgr.25325) 1312 : cluster 0 pgmap v1308: 73 pgs: 73 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:07.942 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:07 smithi016 bash[75004]: cluster 2023-09-23T19:51:06.426934+0000 mgr.y (mgr.25325) 1312 : cluster 0 pgmap v1308: 73 pgs: 73 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:07.958 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:07.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:07.961 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:51:08.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:07 smithi092 bash[58855]: cluster 2023-09-23T19:51:06.426934+0000 mgr.y (mgr.25325) 1312 : cluster 0 pgmap v1308: 73 pgs: 73 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:08.062 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:08.095 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:08.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:08.098 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:51:08.199 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:08.221 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:08.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:08.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:08.324 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:08.350 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:08.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:08.353 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:08.454 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:08.485 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:08.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:08.488 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:08.589 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:08.621 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:08.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:08.624 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:08.725 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:08.763 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:08.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:08.766 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:08.867 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:08.898 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:08.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:08.900 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:09.001 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:09.036 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:09.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:09.039 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:51:09.140 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:09.164 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:09.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:09.166 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:09.267 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:09.292 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:09.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:09.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:09.396 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:09.400 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:51:09.429 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:09.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:09.432 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:51:09.533 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:09.565 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:09.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:09.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:09.668 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:09.689 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:09.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:09.692 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:09.793 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:09.827 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:09.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:09.831 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:51:09.932 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:09.964 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:09.965 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:09 smithi092 bash[58855]: cluster 2023-09-23T19:51:08.427603+0000 mgr.y (mgr.25325) 1313 : cluster 0 pgmap v1309: 73 pgs: 73 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:09.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:09.966 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:10.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:09 smithi016 bash[75004]: cluster 2023-09-23T19:51:08.427603+0000 mgr.y (mgr.25325) 1313 : cluster 0 pgmap v1309: 73 pgs: 73 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:10.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:09 smithi016 bash[77995]: cluster 2023-09-23T19:51:08.427603+0000 mgr.y (mgr.25325) 1313 : cluster 0 pgmap v1309: 73 pgs: 73 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:10.067 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:10.087 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:10.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:10.091 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:10.191 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:10.225 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:10.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:10.227 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:10.328 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:10.348 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:10.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:10.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:10.451 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:10.482 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:10.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:10.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:10.586 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:10.610 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:10.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:10.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:10.713 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:10.748 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:10.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:10.751 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:51:10.851 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:10.877 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:10.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:10.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:10.980 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:11.013 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:11.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:11.017 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:11.118 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:11.151 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:11.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:11.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:51:11.255 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:11.288 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:11.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:11.291 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:11.392 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:11.420 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:11.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:11.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:11.526 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:11.553 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:11.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:11.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:51:11.656 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:11.679 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:11.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:11.682 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:11.783 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:11.808 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:11.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:11.810 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:51:11.911 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:11.924 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:11 smithi092 bash[58855]: cluster 2023-09-23T19:51:10.428560+0000 mgr.y (mgr.25325) 1314 : cluster 0 pgmap v1310: 73 pgs: 73 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:11.946 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:11.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:11.948 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:12.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:11 smithi016 bash[75004]: cluster 2023-09-23T19:51:10.428560+0000 mgr.y (mgr.25325) 1314 : cluster 0 pgmap v1310: 73 pgs: 73 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:12.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:11 smithi016 bash[77995]: cluster 2023-09-23T19:51:10.428560+0000 mgr.y (mgr.25325) 1314 : cluster 0 pgmap v1310: 73 pgs: 73 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:12.049 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:12.074 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:12.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:12.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:12.177 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:12.211 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:12.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:12.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:12.316 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:12.344 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:12.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:12.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:12.449 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:12.472 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:12.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:12.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:12.576 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:12.597 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:12.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:12.599 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:12.697 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-09-23T19:51:12.697 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:51:12.700 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:12.732 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:12.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:12.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:12.836 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:12.861 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:12.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:12.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:12.963 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:12.995 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:12.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:12.998 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:13.099 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:13.131 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:13.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:13.134 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:13.235 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:13.263 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:13.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:13.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:51:13.366 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:13.392 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:13.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:13.394 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:51:13.495 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:13.531 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:13.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:13.535 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:13.635 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:13.670 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:13.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:13.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:51:13.774 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:13.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:13 smithi092 bash[58855]: cluster 2023-09-23T19:51:12.429026+0000 mgr.y (mgr.25325) 1315 : cluster 0 pgmap v1311: 73 pgs: 73 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:13.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:13 smithi092 bash[58855]: audit 2023-09-23T19:51:13.674874+0000 mon.a (mon.0) 1630 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:51:13.807 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:13.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:13.810 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:51:13.911 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:13.933 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:13.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:13.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:14.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:13 smithi016 bash[75004]: cluster 2023-09-23T19:51:12.429026+0000 mgr.y (mgr.25325) 1315 : cluster 0 pgmap v1311: 73 pgs: 73 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:14.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:13 smithi016 bash[75004]: audit 2023-09-23T19:51:13.674874+0000 mon.a (mon.0) 1630 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:51:14.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:13 smithi016 bash[77995]: cluster 2023-09-23T19:51:12.429026+0000 mgr.y (mgr.25325) 1315 : cluster 0 pgmap v1311: 73 pgs: 73 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:14.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:13 smithi016 bash[77995]: audit 2023-09-23T19:51:13.674874+0000 mon.a (mon.0) 1630 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:51:14.038 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:14.067 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:14.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:14.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:51:14.171 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:14.201 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:14.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:14.205 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:51:14.306 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:14.328 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:14.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:14.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:14.431 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:14.464 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:14.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:14.467 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:51:14.568 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:14.602 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:14.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:14.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:14.707 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:14.735 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:14.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:14.737 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:14.838 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:14.870 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:14.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:14.873 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:14.974 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:15.000 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:15.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:15.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:15.103 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:15.128 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:15.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:15.130 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:15.231 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:15.265 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:15.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:15.269 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:51:15.370 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:15.391 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:15.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:15.394 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:15.495 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:15.527 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:15.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:15.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:15.631 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:15.662 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:15.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:15.665 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:15.766 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:15.779 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:15 smithi092 bash[58855]: cluster 2023-09-23T19:51:14.429809+0000 mgr.y (mgr.25325) 1316 : cluster 0 pgmap v1312: 73 pgs: 73 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:15.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:15 smithi016 bash[75004]: cluster 2023-09-23T19:51:14.429809+0000 mgr.y (mgr.25325) 1316 : cluster 0 pgmap v1312: 73 pgs: 73 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:15.787 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:51:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:51:15] "GET /metrics HTTP/1.1" 200 33887 "" "Prometheus/2.33.4" 2023-09-23T19:51:15.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:15 smithi016 bash[77995]: cluster 2023-09-23T19:51:14.429809+0000 mgr.y (mgr.25325) 1316 : cluster 0 pgmap v1312: 73 pgs: 73 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:15.797 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:15.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:15.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:51:15.835 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:51:15.858 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:51:15.878 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:51:15.898 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:51:15.901 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:15.920 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:51:15.925 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:15.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:15.927 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:15.943 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:51:15.971 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:51:15.998 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:51:16.000 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:51:16.000 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:51:16.000 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:51:16.000 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:51:16.000 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:51:16.000 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:51:16.000 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:51:16.001 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:51:16.028 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:16.053 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:16.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:16.056 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:16.157 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:16.184 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:16.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:16.186 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:16.287 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:16.313 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:16.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:16.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:51:16.354 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T19:51:16.417 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:16.462 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:16.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:16.464 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:16.565 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:16.589 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:16.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:16.591 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:51:16.692 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:16.715 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:16.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:16.717 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:16.727 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T19:51:16.818 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:16.848 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:16.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:16.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:51:16.885 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:16 smithi016 bash[77995]: audit 2023-09-23T19:51:15.807474+0000 mon.a (mon.0) 1631 : audit 0 from='client.? 172.21.15.16:0/2380776361' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:51:16.885 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:16 smithi016 bash[77995]: audit 2023-09-23T19:51:16.709113+0000 mon.c (mon.2) 205 : audit 1 from='client.? 172.21.15.16:0/2336074162' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T19:51:16.885 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:16 smithi016 bash[77995]: audit 2023-09-23T19:51:16.709542+0000 mon.a (mon.0) 1632 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T19:51:16.886 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:16 smithi016 bash[75004]: audit 2023-09-23T19:51:15.807474+0000 mon.a (mon.0) 1631 : audit 0 from='client.? 172.21.15.16:0/2380776361' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:51:16.886 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:16 smithi016 bash[75004]: audit 2023-09-23T19:51:16.709113+0000 mon.c (mon.2) 205 : audit 1 from='client.? 172.21.15.16:0/2336074162' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T19:51:16.886 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:16 smithi016 bash[75004]: audit 2023-09-23T19:51:16.709542+0000 mon.a (mon.0) 1632 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T19:51:16.952 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:16.980 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:16.982 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:16 smithi092 bash[58855]: audit 2023-09-23T19:51:15.807474+0000 mon.a (mon.0) 1631 : audit 0 from='client.? 172.21.15.16:0/2380776361' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:51:16.982 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:16 smithi092 bash[58855]: audit 2023-09-23T19:51:16.709113+0000 mon.c (mon.2) 205 : audit 1 from='client.? 172.21.15.16:0/2336074162' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T19:51:16.982 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:16 smithi092 bash[58855]: audit 2023-09-23T19:51:16.709542+0000 mon.a (mon.0) 1632 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T19:51:16.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:16.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:17.084 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:17.106 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:17.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:17.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:51:17.209 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:17.231 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:17.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:17.233 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:17.334 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:17.355 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:17.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:17.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:17.458 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:17.487 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:17.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:17.490 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:51:17.591 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:17.623 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:17.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:17.626 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:51:17.726 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:17.757 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:17.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:17.761 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:17.861 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:17.888 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:17.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:17.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:17.992 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:18.004 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:17 smithi092 bash[58855]: cluster 2023-09-23T19:51:16.430500+0000 mgr.y (mgr.25325) 1317 : cluster 0 pgmap v1313: 73 pgs: 73 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:18.004 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:17 smithi092 bash[58855]: audit 2023-09-23T19:51:16.718387+0000 mon.a (mon.0) 1633 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T19:51:18.004 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:17 smithi092 bash[58855]: cluster 2023-09-23T19:51:16.718463+0000 mon.a (mon.0) 1634 : cluster 0 osdmap e230: 8 total, 8 up, 8 in 2023-09-23T19:51:18.004 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:17 smithi092 bash[58855]: cluster 2023-09-23T19:51:16.887187+0000 osd.5 (osd.5) 154 : cluster 3 failed to encode map e230 with expected crc 2023-09-23T19:51:18.004 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:17 smithi092 bash[58855]: cluster 2023-09-23T19:51:16.889705+0000 osd.5 (osd.5) 155 : cluster 3 failed to encode map e230 with expected crc 2023-09-23T19:51:18.004 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:17 smithi092 bash[58855]: cluster 2023-09-23T19:51:16.962008+0000 osd.4 (osd.4) 170 : cluster 3 failed to encode map e230 with expected crc 2023-09-23T19:51:18.004 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:17 smithi092 bash[58855]: cluster 2023-09-23T19:51:16.962221+0000 osd.7 (osd.7) 147 : cluster 3 failed to encode map e230 with expected crc 2023-09-23T19:51:18.005 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:17 smithi092 bash[58855]: cluster 2023-09-23T19:51:16.962268+0000 osd.6 (osd.6) 162 : cluster 3 failed to encode map e230 with expected crc 2023-09-23T19:51:18.023 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:18.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:18.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:18.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:17 smithi016 bash[77995]: cluster 2023-09-23T19:51:16.430500+0000 mgr.y (mgr.25325) 1317 : cluster 0 pgmap v1313: 73 pgs: 73 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:18.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:17 smithi016 bash[77995]: audit 2023-09-23T19:51:16.718387+0000 mon.a (mon.0) 1633 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T19:51:18.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:17 smithi016 bash[77995]: cluster 2023-09-23T19:51:16.718463+0000 mon.a (mon.0) 1634 : cluster 0 osdmap e230: 8 total, 8 up, 8 in 2023-09-23T19:51:18.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:17 smithi016 bash[77995]: cluster 2023-09-23T19:51:16.887187+0000 osd.5 (osd.5) 154 : cluster 3 failed to encode map e230 with expected crc 2023-09-23T19:51:18.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:17 smithi016 bash[77995]: cluster 2023-09-23T19:51:16.889705+0000 osd.5 (osd.5) 155 : cluster 3 failed to encode map e230 with expected crc 2023-09-23T19:51:18.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:17 smithi016 bash[77995]: cluster 2023-09-23T19:51:16.962008+0000 osd.4 (osd.4) 170 : cluster 3 failed to encode map e230 with expected crc 2023-09-23T19:51:18.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:17 smithi016 bash[77995]: cluster 2023-09-23T19:51:16.962221+0000 osd.7 (osd.7) 147 : cluster 3 failed to encode map e230 with expected crc 2023-09-23T19:51:18.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:17 smithi016 bash[77995]: cluster 2023-09-23T19:51:16.962268+0000 osd.6 (osd.6) 162 : cluster 3 failed to encode map e230 with expected crc 2023-09-23T19:51:18.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:17 smithi016 bash[75004]: cluster 2023-09-23T19:51:16.430500+0000 mgr.y (mgr.25325) 1317 : cluster 0 pgmap v1313: 73 pgs: 73 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:18.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:17 smithi016 bash[75004]: audit 2023-09-23T19:51:16.718387+0000 mon.a (mon.0) 1633 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T19:51:18.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:17 smithi016 bash[75004]: cluster 2023-09-23T19:51:16.718463+0000 mon.a (mon.0) 1634 : cluster 0 osdmap e230: 8 total, 8 up, 8 in 2023-09-23T19:51:18.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:17 smithi016 bash[75004]: cluster 2023-09-23T19:51:16.887187+0000 osd.5 (osd.5) 154 : cluster 3 failed to encode map e230 with expected crc 2023-09-23T19:51:18.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:17 smithi016 bash[75004]: cluster 2023-09-23T19:51:16.889705+0000 osd.5 (osd.5) 155 : cluster 3 failed to encode map e230 with expected crc 2023-09-23T19:51:18.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:17 smithi016 bash[75004]: cluster 2023-09-23T19:51:16.962008+0000 osd.4 (osd.4) 170 : cluster 3 failed to encode map e230 with expected crc 2023-09-23T19:51:18.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:17 smithi016 bash[75004]: cluster 2023-09-23T19:51:16.962221+0000 osd.7 (osd.7) 147 : cluster 3 failed to encode map e230 with expected crc 2023-09-23T19:51:18.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:17 smithi016 bash[75004]: cluster 2023-09-23T19:51:16.962268+0000 osd.6 (osd.6) 162 : cluster 3 failed to encode map e230 with expected crc 2023-09-23T19:51:18.126 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:18.145 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:18.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:18.147 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:18.248 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:18.269 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:18.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:18.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:18.372 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:18.395 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:18.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:18.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:18.498 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:18.528 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:18.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:18.530 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:51:18.631 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:18.662 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:18.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:18.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:18.744 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is set 2023-09-23T19:51:18.760 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:18 smithi016 bash[75004]: audit 2023-09-23T19:51:18.243442+0000 mon.c (mon.2) 206 : audit 1 from='client.? 172.21.15.16:0/635619071' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:51:18.760 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:18 smithi016 bash[75004]: audit 2023-09-23T19:51:18.243831+0000 mon.a (mon.0) 1635 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:51:18.766 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:18.792 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:18.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:18.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:51:18.895 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:18.919 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:18.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:18.921 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:51:19.022 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:19.030 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:18 smithi016 bash[77995]: audit 2023-09-23T19:51:18.243442+0000 mon.c (mon.2) 206 : audit 1 from='client.? 172.21.15.16:0/635619071' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:51:19.030 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:18 smithi016 bash[77995]: audit 2023-09-23T19:51:18.243831+0000 mon.a (mon.0) 1635 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:51:19.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:18 smithi092 bash[58855]: audit 2023-09-23T19:51:18.243442+0000 mon.c (mon.2) 206 : audit 1 from='client.? 172.21.15.16:0/635619071' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:51:19.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:18 smithi092 bash[58855]: audit 2023-09-23T19:51:18.243831+0000 mon.a (mon.0) 1635 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:51:19.048 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:19.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:19.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:19.151 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:19.184 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:19.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:19.186 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:51:19.287 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:19.317 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:19.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:19.320 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:19.420 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:19.455 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:19.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:19.458 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:19.559 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:19.589 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:19.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:19.592 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:51:19.692 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:19.720 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:19.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:19.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:19.824 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:19.854 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:19.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:19.858 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:19.959 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:19.991 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:19.994 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:19 smithi092 bash[58855]: cluster 2023-09-23T19:51:18.431086+0000 mgr.y (mgr.25325) 1318 : cluster 0 pgmap v1315: 73 pgs: 73 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:19.994 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:19 smithi092 bash[58855]: cluster 2023-09-23T19:51:18.722519+0000 mon.a (mon.0) 1636 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:51:19.994 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:19 smithi092 bash[58855]: audit 2023-09-23T19:51:18.735371+0000 mon.a (mon.0) 1637 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:51:19.995 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:19 smithi092 bash[58855]: cluster 2023-09-23T19:51:18.735449+0000 mon.a (mon.0) 1638 : cluster 0 osdmap e231: 8 total, 8 up, 8 in 2023-09-23T19:51:19.995 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:19 smithi092 bash[58855]: cluster 2023-09-23T19:51:18.816384+0000 osd.4 (osd.4) 171 : cluster 3 failed to encode map e231 with expected crc 2023-09-23T19:51:19.995 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:19 smithi092 bash[58855]: cluster 2023-09-23T19:51:18.819118+0000 osd.4 (osd.4) 172 : cluster 3 failed to encode map e231 with expected crc 2023-09-23T19:51:19.995 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:19 smithi092 bash[58855]: cluster 2023-09-23T19:51:19.110639+0000 osd.5 (osd.5) 156 : cluster 3 failed to encode map e231 with expected crc 2023-09-23T19:51:19.995 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:19 smithi092 bash[58855]: cluster 2023-09-23T19:51:19.110872+0000 osd.6 (osd.6) 163 : cluster 3 failed to encode map e231 with expected crc 2023-09-23T19:51:19.995 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:19 smithi092 bash[58855]: cluster 2023-09-23T19:51:19.111000+0000 osd.7 (osd.7) 148 : cluster 3 failed to encode map e231 with expected crc 2023-09-23T19:51:19.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:19.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:20.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:19 smithi016 bash[75004]: cluster 2023-09-23T19:51:18.431086+0000 mgr.y (mgr.25325) 1318 : cluster 0 pgmap v1315: 73 pgs: 73 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:20.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:19 smithi016 bash[75004]: cluster 2023-09-23T19:51:18.722519+0000 mon.a (mon.0) 1636 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:51:20.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:19 smithi016 bash[75004]: audit 2023-09-23T19:51:18.735371+0000 mon.a (mon.0) 1637 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:51:20.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:19 smithi016 bash[75004]: cluster 2023-09-23T19:51:18.735449+0000 mon.a (mon.0) 1638 : cluster 0 osdmap e231: 8 total, 8 up, 8 in 2023-09-23T19:51:20.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:19 smithi016 bash[75004]: cluster 2023-09-23T19:51:18.816384+0000 osd.4 (osd.4) 171 : cluster 3 failed to encode map e231 with expected crc 2023-09-23T19:51:20.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:19 smithi016 bash[75004]: cluster 2023-09-23T19:51:18.819118+0000 osd.4 (osd.4) 172 : cluster 3 failed to encode map e231 with expected crc 2023-09-23T19:51:20.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:19 smithi016 bash[75004]: cluster 2023-09-23T19:51:19.110639+0000 osd.5 (osd.5) 156 : cluster 3 failed to encode map e231 with expected crc 2023-09-23T19:51:20.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:19 smithi016 bash[75004]: cluster 2023-09-23T19:51:19.110872+0000 osd.6 (osd.6) 163 : cluster 3 failed to encode map e231 with expected crc 2023-09-23T19:51:20.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:19 smithi016 bash[75004]: cluster 2023-09-23T19:51:19.111000+0000 osd.7 (osd.7) 148 : cluster 3 failed to encode map e231 with expected crc 2023-09-23T19:51:20.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:19 smithi016 bash[77995]: cluster 2023-09-23T19:51:18.431086+0000 mgr.y (mgr.25325) 1318 : cluster 0 pgmap v1315: 73 pgs: 73 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:20.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:19 smithi016 bash[77995]: cluster 2023-09-23T19:51:18.722519+0000 mon.a (mon.0) 1636 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:51:20.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:19 smithi016 bash[77995]: audit 2023-09-23T19:51:18.735371+0000 mon.a (mon.0) 1637 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:51:20.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:19 smithi016 bash[77995]: cluster 2023-09-23T19:51:18.735449+0000 mon.a (mon.0) 1638 : cluster 0 osdmap e231: 8 total, 8 up, 8 in 2023-09-23T19:51:20.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:19 smithi016 bash[77995]: cluster 2023-09-23T19:51:18.816384+0000 osd.4 (osd.4) 171 : cluster 3 failed to encode map e231 with expected crc 2023-09-23T19:51:20.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:19 smithi016 bash[77995]: cluster 2023-09-23T19:51:18.819118+0000 osd.4 (osd.4) 172 : cluster 3 failed to encode map e231 with expected crc 2023-09-23T19:51:20.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:19 smithi016 bash[77995]: cluster 2023-09-23T19:51:19.110639+0000 osd.5 (osd.5) 156 : cluster 3 failed to encode map e231 with expected crc 2023-09-23T19:51:20.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:19 smithi016 bash[77995]: cluster 2023-09-23T19:51:19.110872+0000 osd.6 (osd.6) 163 : cluster 3 failed to encode map e231 with expected crc 2023-09-23T19:51:20.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:19 smithi016 bash[77995]: cluster 2023-09-23T19:51:19.111000+0000 osd.7 (osd.7) 148 : cluster 3 failed to encode map e231 with expected crc 2023-09-23T19:51:20.096 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:20.124 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:20.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:20.128 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:51:20.229 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:20.260 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:20.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:20.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:20.363 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:20.389 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:20.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:20.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:20.493 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:20.525 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:20.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:20.529 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:20.629 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:20.659 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:20.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:20.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:20.739 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-23T19:51:20.763 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:20.795 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:20.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:20.797 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:51:20.898 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:20.922 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:20.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:20.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:21.024 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:21.055 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:21.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:21.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:21.140 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:51:21.159 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:21.209 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:21.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:21.213 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:21.314 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:21.346 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:21.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:21.349 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:51:21.449 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:21.484 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:21.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:21.487 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:51:21.588 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:21.624 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:21.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:21.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:21.728 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:21.761 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:21.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:21.764 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:51:21.864 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:21.876 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:21 smithi016 bash[77995]: cluster 2023-09-23T19:51:20.432166+0000 mgr.y (mgr.25325) 1319 : cluster 0 pgmap v1317: 73 pgs: 73 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:21.876 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:21 smithi016 bash[75004]: cluster 2023-09-23T19:51:20.432166+0000 mgr.y (mgr.25325) 1319 : cluster 0 pgmap v1317: 73 pgs: 73 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:21.894 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:21.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:21.898 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:51:21.999 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:22.030 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:22.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:22.033 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:22.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:21 smithi092 bash[58855]: cluster 2023-09-23T19:51:20.432166+0000 mgr.y (mgr.25325) 1319 : cluster 0 pgmap v1317: 73 pgs: 73 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:22.134 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:22.168 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:22.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:22.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:22.272 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:22.296 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:22.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:22.300 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:22.400 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:22.432 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:22.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:22.434 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:51:22.535 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:22.571 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:22.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:22.574 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:51:22.675 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:22.707 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:22.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:22.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:22.811 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:22.843 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:22.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:22.846 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:22.946 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:22.977 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:22.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:22.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:23.082 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:23.110 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:23.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:23.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:51:23.214 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:23.246 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:23.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:23.249 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:51:23.350 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:23.383 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:23.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:23.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:51:23.487 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:23.523 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:23.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:23.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:23.627 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:23.660 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:23.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:23.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:23.729 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T19:51:23.763 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:23.771 INFO:teuthology.orchestra.run.smithi016.stderr:pool 'unique_pool_8' created 2023-09-23T19:51:23.779 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:23 smithi016 bash[77995]: cluster 2023-09-23T19:51:22.433064+0000 mgr.y (mgr.25325) 1320 : cluster 0 pgmap v1318: 73 pgs: 73 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:23.780 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:23 smithi016 bash[77995]: audit 2023-09-23T19:51:22.853105+0000 mon.c (mon.2) 207 : audit 1 from='client.? 172.21.15.16:0/2837973013' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]: dispatch 2023-09-23T19:51:23.780 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:23 smithi016 bash[77995]: audit 2023-09-23T19:51:22.853760+0000 mon.a (mon.0) 1639 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]: dispatch 2023-09-23T19:51:23.780 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:23 smithi016 bash[75004]: cluster 2023-09-23T19:51:22.433064+0000 mgr.y (mgr.25325) 1320 : cluster 0 pgmap v1318: 73 pgs: 73 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:23.780 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:23 smithi016 bash[75004]: audit 2023-09-23T19:51:22.853105+0000 mon.c (mon.2) 207 : audit 1 from='client.? 172.21.15.16:0/2837973013' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]: dispatch 2023-09-23T19:51:23.781 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:23 smithi016 bash[75004]: audit 2023-09-23T19:51:22.853760+0000 mon.a (mon.0) 1639 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]: dispatch 2023-09-23T19:51:23.795 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:23.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:23.798 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:23.899 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:23.910 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:23 smithi092 bash[58855]: cluster 2023-09-23T19:51:22.433064+0000 mgr.y (mgr.25325) 1320 : cluster 0 pgmap v1318: 73 pgs: 73 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:23.910 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:23 smithi092 bash[58855]: audit 2023-09-23T19:51:22.853105+0000 mon.c (mon.2) 207 : audit 1 from='client.? 172.21.15.16:0/2837973013' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]: dispatch 2023-09-23T19:51:23.910 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:23 smithi092 bash[58855]: audit 2023-09-23T19:51:22.853760+0000 mon.a (mon.0) 1639 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]: dispatch 2023-09-23T19:51:23.933 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:23.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:23.936 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:24.037 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:24.074 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:24.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:24.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:24.178 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:24.210 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:24.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:24.213 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:51:24.281 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:51:24.314 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:24.337 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:24.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:24.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:24.440 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:24.476 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:24.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:24.479 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:24.580 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:24.608 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:24.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:24.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:24.711 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:24.736 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:24.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:24.739 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:24.840 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:24.872 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:24.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:24.875 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:51:24.976 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:25.010 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:25.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:25.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:25.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:24 smithi016 bash[75004]: audit 2023-09-23T19:51:23.754864+0000 mon.a (mon.0) 1640 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]': finished 2023-09-23T19:51:25.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:24 smithi016 bash[75004]: cluster 2023-09-23T19:51:23.754960+0000 mon.a (mon.0) 1641 : cluster 0 osdmap e232: 8 total, 8 up, 8 in 2023-09-23T19:51:25.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:24 smithi016 bash[75004]: cluster 2023-09-23T19:51:23.757052+0000 osd.5 (osd.5) 157 : cluster 3 failed to encode map e232 with expected crc 2023-09-23T19:51:25.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:24 smithi016 bash[75004]: cluster 2023-09-23T19:51:23.759951+0000 osd.5 (osd.5) 158 : cluster 3 failed to encode map e232 with expected crc 2023-09-23T19:51:25.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:24 smithi016 bash[75004]: cluster 2023-09-23T19:51:23.764972+0000 osd.4 (osd.4) 173 : cluster 3 failed to encode map e232 with expected crc 2023-09-23T19:51:25.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:24 smithi016 bash[75004]: cluster 2023-09-23T19:51:23.766306+0000 osd.6 (osd.6) 164 : cluster 3 failed to encode map e232 with expected crc 2023-09-23T19:51:25.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:24 smithi016 bash[77995]: audit 2023-09-23T19:51:23.754864+0000 mon.a (mon.0) 1640 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]': finished 2023-09-23T19:51:25.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:24 smithi016 bash[77995]: cluster 2023-09-23T19:51:23.754960+0000 mon.a (mon.0) 1641 : cluster 0 osdmap e232: 8 total, 8 up, 8 in 2023-09-23T19:51:25.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:24 smithi016 bash[77995]: cluster 2023-09-23T19:51:23.757052+0000 osd.5 (osd.5) 157 : cluster 3 failed to encode map e232 with expected crc 2023-09-23T19:51:25.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:24 smithi016 bash[77995]: cluster 2023-09-23T19:51:23.759951+0000 osd.5 (osd.5) 158 : cluster 3 failed to encode map e232 with expected crc 2023-09-23T19:51:25.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:24 smithi016 bash[77995]: cluster 2023-09-23T19:51:23.764972+0000 osd.4 (osd.4) 173 : cluster 3 failed to encode map e232 with expected crc 2023-09-23T19:51:25.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:24 smithi016 bash[77995]: cluster 2023-09-23T19:51:23.766306+0000 osd.6 (osd.6) 164 : cluster 3 failed to encode map e232 with expected crc 2023-09-23T19:51:25.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:24 smithi092 bash[58855]: audit 2023-09-23T19:51:23.754864+0000 mon.a (mon.0) 1640 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]': finished 2023-09-23T19:51:25.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:24 smithi092 bash[58855]: cluster 2023-09-23T19:51:23.754960+0000 mon.a (mon.0) 1641 : cluster 0 osdmap e232: 8 total, 8 up, 8 in 2023-09-23T19:51:25.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:24 smithi092 bash[58855]: cluster 2023-09-23T19:51:23.757052+0000 osd.5 (osd.5) 157 : cluster 3 failed to encode map e232 with expected crc 2023-09-23T19:51:25.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:24 smithi092 bash[58855]: cluster 2023-09-23T19:51:23.759951+0000 osd.5 (osd.5) 158 : cluster 3 failed to encode map e232 with expected crc 2023-09-23T19:51:25.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:24 smithi092 bash[58855]: cluster 2023-09-23T19:51:23.764972+0000 osd.4 (osd.4) 173 : cluster 3 failed to encode map e232 with expected crc 2023-09-23T19:51:25.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:24 smithi092 bash[58855]: cluster 2023-09-23T19:51:23.766306+0000 osd.6 (osd.6) 164 : cluster 3 failed to encode map e232 with expected crc 2023-09-23T19:51:25.114 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:25.137 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:25.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:25.141 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:25.241 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:25.274 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:25.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:25.277 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:51:25.378 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:25.412 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:25.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:25.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:25.515 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:25.549 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:25.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:25.553 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:25.621 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_8 rados --yes-i-really-mean-it || true 2023-09-23T19:51:25.654 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:25.683 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:25.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:25.685 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:51:25.786 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:25.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:25 smithi092 bash[58855]: cluster 2023-09-23T19:51:24.433833+0000 mgr.y (mgr.25325) 1321 : cluster 0 pgmap v1320: 89 pgs: 2 creating+peering, 14 unknown, 73 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:25.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:25 smithi092 bash[58855]: cluster 2023-09-23T19:51:24.761401+0000 mon.a (mon.0) 1642 : cluster 0 osdmap e233: 8 total, 8 up, 8 in 2023-09-23T19:51:25.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:25 smithi092 bash[58855]: cluster 2023-09-23T19:51:24.763887+0000 osd.6 (osd.6) 165 : cluster 3 failed to encode map e233 with expected crc 2023-09-23T19:51:25.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:25 smithi092 bash[58855]: cluster 2023-09-23T19:51:24.768751+0000 osd.4 (osd.4) 174 : cluster 3 failed to encode map e233 with expected crc 2023-09-23T19:51:25.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:25 smithi092 bash[58855]: cluster 2023-09-23T19:51:24.768861+0000 osd.6 (osd.6) 166 : cluster 3 failed to encode map e233 with expected crc 2023-09-23T19:51:25.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:25 smithi092 bash[58855]: cluster 2023-09-23T19:51:24.769019+0000 osd.5 (osd.5) 159 : cluster 3 failed to encode map e233 with expected crc 2023-09-23T19:51:25.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:25 smithi092 bash[58855]: cluster 2023-09-23T19:51:24.771156+0000 osd.7 (osd.7) 149 : cluster 3 failed to encode map e233 with expected crc 2023-09-23T19:51:25.814 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:25.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:25.817 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:25.917 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:25.926 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:51:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:51:25] "GET /metrics HTTP/1.1" 200 33887 "" "Prometheus/2.33.4" 2023-09-23T19:51:25.926 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:25 smithi016 bash[75004]: cluster 2023-09-23T19:51:24.433833+0000 mgr.y (mgr.25325) 1321 : cluster 0 pgmap v1320: 89 pgs: 2 creating+peering, 14 unknown, 73 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:25.926 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:25 smithi016 bash[75004]: cluster 2023-09-23T19:51:24.761401+0000 mon.a (mon.0) 1642 : cluster 0 osdmap e233: 8 total, 8 up, 8 in 2023-09-23T19:51:25.926 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:25 smithi016 bash[75004]: cluster 2023-09-23T19:51:24.763887+0000 osd.6 (osd.6) 165 : cluster 3 failed to encode map e233 with expected crc 2023-09-23T19:51:25.926 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:25 smithi016 bash[75004]: cluster 2023-09-23T19:51:24.768751+0000 osd.4 (osd.4) 174 : cluster 3 failed to encode map e233 with expected crc 2023-09-23T19:51:25.926 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:25 smithi016 bash[75004]: cluster 2023-09-23T19:51:24.768861+0000 osd.6 (osd.6) 166 : cluster 3 failed to encode map e233 with expected crc 2023-09-23T19:51:25.926 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:25 smithi016 bash[75004]: cluster 2023-09-23T19:51:24.769019+0000 osd.5 (osd.5) 159 : cluster 3 failed to encode map e233 with expected crc 2023-09-23T19:51:25.926 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:25 smithi016 bash[75004]: cluster 2023-09-23T19:51:24.771156+0000 osd.7 (osd.7) 149 : cluster 3 failed to encode map e233 with expected crc 2023-09-23T19:51:25.927 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:25 smithi016 bash[77995]: cluster 2023-09-23T19:51:24.433833+0000 mgr.y (mgr.25325) 1321 : cluster 0 pgmap v1320: 89 pgs: 2 creating+peering, 14 unknown, 73 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:25.927 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:25 smithi016 bash[77995]: cluster 2023-09-23T19:51:24.761401+0000 mon.a (mon.0) 1642 : cluster 0 osdmap e233: 8 total, 8 up, 8 in 2023-09-23T19:51:25.927 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:25 smithi016 bash[77995]: cluster 2023-09-23T19:51:24.763887+0000 osd.6 (osd.6) 165 : cluster 3 failed to encode map e233 with expected crc 2023-09-23T19:51:25.927 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:25 smithi016 bash[77995]: cluster 2023-09-23T19:51:24.768751+0000 osd.4 (osd.4) 174 : cluster 3 failed to encode map e233 with expected crc 2023-09-23T19:51:25.927 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:25 smithi016 bash[77995]: cluster 2023-09-23T19:51:24.768861+0000 osd.6 (osd.6) 166 : cluster 3 failed to encode map e233 with expected crc 2023-09-23T19:51:25.927 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:25 smithi016 bash[77995]: cluster 2023-09-23T19:51:24.769019+0000 osd.5 (osd.5) 159 : cluster 3 failed to encode map e233 with expected crc 2023-09-23T19:51:25.927 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:25 smithi016 bash[77995]: cluster 2023-09-23T19:51:24.771156+0000 osd.7 (osd.7) 149 : cluster 3 failed to encode map e233 with expected crc 2023-09-23T19:51:25.941 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:25.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:25.944 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:26.045 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:26.075 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:26.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:26.079 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:26.181 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:26.210 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:26.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:26.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:26.314 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:26.317 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:51:26.317 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:51:26.341 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:26.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:26.343 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:26.444 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:26.473 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:26.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:26.476 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:26.577 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:26.604 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:26.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:26.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:26.707 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:26.738 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:26.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:26.740 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:51:26.841 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:26.854 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:26 smithi016 bash[75004]: cluster 2023-09-23T19:51:25.770080+0000 mon.a (mon.0) 1643 : cluster 0 osdmap e234: 8 total, 8 up, 8 in 2023-09-23T19:51:26.854 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:26 smithi016 bash[75004]: cluster 2023-09-23T19:51:25.990997+0000 osd.5 (osd.5) 160 : cluster 3 failed to encode map e234 with expected crc 2023-09-23T19:51:26.854 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:26 smithi016 bash[75004]: cluster 2023-09-23T19:51:25.994005+0000 osd.5 (osd.5) 161 : cluster 3 failed to encode map e234 with expected crc 2023-09-23T19:51:26.855 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:26 smithi016 bash[75004]: cluster 2023-09-23T19:51:26.417766+0000 osd.4 (osd.4) 175 : cluster 3 failed to encode map e234 with expected crc 2023-09-23T19:51:26.855 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:26 smithi016 bash[75004]: cluster 2023-09-23T19:51:26.420604+0000 osd.4 (osd.4) 176 : cluster 3 failed to encode map e234 with expected crc 2023-09-23T19:51:26.855 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:26 smithi016 bash[75004]: cluster 2023-09-23T19:51:26.593970+0000 osd.7 (osd.7) 150 : cluster 3 failed to encode map e234 with expected crc 2023-09-23T19:51:26.855 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:26 smithi016 bash[75004]: cluster 2023-09-23T19:51:26.596909+0000 osd.7 (osd.7) 151 : cluster 3 failed to encode map e234 with expected crc 2023-09-23T19:51:26.855 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:26 smithi016 bash[77995]: cluster 2023-09-23T19:51:25.770080+0000 mon.a (mon.0) 1643 : cluster 0 osdmap e234: 8 total, 8 up, 8 in 2023-09-23T19:51:26.855 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:26 smithi016 bash[77995]: cluster 2023-09-23T19:51:25.990997+0000 osd.5 (osd.5) 160 : cluster 3 failed to encode map e234 with expected crc 2023-09-23T19:51:26.855 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:26 smithi016 bash[77995]: cluster 2023-09-23T19:51:25.994005+0000 osd.5 (osd.5) 161 : cluster 3 failed to encode map e234 with expected crc 2023-09-23T19:51:26.855 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:26 smithi016 bash[77995]: cluster 2023-09-23T19:51:26.417766+0000 osd.4 (osd.4) 175 : cluster 3 failed to encode map e234 with expected crc 2023-09-23T19:51:26.855 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:26 smithi016 bash[77995]: cluster 2023-09-23T19:51:26.420604+0000 osd.4 (osd.4) 176 : cluster 3 failed to encode map e234 with expected crc 2023-09-23T19:51:26.855 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:26 smithi016 bash[77995]: cluster 2023-09-23T19:51:26.593970+0000 osd.7 (osd.7) 150 : cluster 3 failed to encode map e234 with expected crc 2023-09-23T19:51:26.855 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:26 smithi016 bash[77995]: cluster 2023-09-23T19:51:26.596909+0000 osd.7 (osd.7) 151 : cluster 3 failed to encode map e234 with expected crc 2023-09-23T19:51:26.872 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:26.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:26.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:26.975 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:26.988 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:26 smithi092 bash[58855]: cluster 2023-09-23T19:51:25.770080+0000 mon.a (mon.0) 1643 : cluster 0 osdmap e234: 8 total, 8 up, 8 in 2023-09-23T19:51:26.988 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:26 smithi092 bash[58855]: cluster 2023-09-23T19:51:25.990997+0000 osd.5 (osd.5) 160 : cluster 3 failed to encode map e234 with expected crc 2023-09-23T19:51:26.988 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:26 smithi092 bash[58855]: cluster 2023-09-23T19:51:25.994005+0000 osd.5 (osd.5) 161 : cluster 3 failed to encode map e234 with expected crc 2023-09-23T19:51:26.988 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:26 smithi092 bash[58855]: cluster 2023-09-23T19:51:26.417766+0000 osd.4 (osd.4) 175 : cluster 3 failed to encode map e234 with expected crc 2023-09-23T19:51:26.988 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:26 smithi092 bash[58855]: cluster 2023-09-23T19:51:26.420604+0000 osd.4 (osd.4) 176 : cluster 3 failed to encode map e234 with expected crc 2023-09-23T19:51:26.989 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:26 smithi092 bash[58855]: cluster 2023-09-23T19:51:26.593970+0000 osd.7 (osd.7) 150 : cluster 3 failed to encode map e234 with expected crc 2023-09-23T19:51:26.989 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:26 smithi092 bash[58855]: cluster 2023-09-23T19:51:26.596909+0000 osd.7 (osd.7) 151 : cluster 3 failed to encode map e234 with expected crc 2023-09-23T19:51:27.009 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:27.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:27.013 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:51:27.113 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:27.146 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:27.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:27.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:27.249 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:27.279 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:27.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:27.283 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:27.383 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:27.412 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:27.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:27.414 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:27.515 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:27.548 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:27.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:27.551 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:27.652 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:27.680 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:27.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:27.683 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:27.784 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:27.815 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:27.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:27.818 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:27.919 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:27.942 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:27.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:27.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:27.971 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:27 smithi016 bash[77995]: cluster 2023-09-23T19:51:26.434664+0000 mgr.y (mgr.25325) 1322 : cluster 0 pgmap v1323: 89 pgs: 7 creating+peering, 2 unknown, 80 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:27.971 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:27 smithi016 bash[77995]: cluster 2023-09-23T19:51:26.712427+0000 osd.6 (osd.6) 167 : cluster 3 failed to encode map e234 with expected crc 2023-09-23T19:51:27.971 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:27 smithi016 bash[75004]: cluster 2023-09-23T19:51:26.434664+0000 mgr.y (mgr.25325) 1322 : cluster 0 pgmap v1323: 89 pgs: 7 creating+peering, 2 unknown, 80 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:27.971 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:27 smithi016 bash[75004]: cluster 2023-09-23T19:51:26.712427+0000 osd.6 (osd.6) 167 : cluster 3 failed to encode map e234 with expected crc 2023-09-23T19:51:28.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:27 smithi092 bash[58855]: cluster 2023-09-23T19:51:26.434664+0000 mgr.y (mgr.25325) 1322 : cluster 0 pgmap v1323: 89 pgs: 7 creating+peering, 2 unknown, 80 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:28.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:27 smithi092 bash[58855]: cluster 2023-09-23T19:51:26.712427+0000 osd.6 (osd.6) 167 : cluster 3 failed to encode map e234 with expected crc 2023-09-23T19:51:28.045 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:28.073 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:28.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:28.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:28.176 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:28.204 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:28.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:28.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:28.307 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:28.338 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:28.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:28.341 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:51:28.442 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:28.473 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:28.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:28.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:28.576 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:28.607 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:28.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:28.611 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:51:28.711 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:28.746 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:28.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:28.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:51:28.849 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:28.881 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:28.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:28.884 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:51:28.985 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:29.013 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:29.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:29.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:51:29.116 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:29.149 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:29.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:29.152 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:29.186 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:51:29.253 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:29.286 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:29.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:29.289 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:51:29.389 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:29.421 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:29.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:29.424 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:29.525 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:29.551 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:29.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:29.553 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:29.654 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:29.690 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:29.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:29.694 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:51:29.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:29 smithi016 bash[77995]: cluster 2023-09-23T19:51:28.435425+0000 mgr.y (mgr.25325) 1323 : cluster 0 pgmap v1324: 89 pgs: 7 creating+peering, 82 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:29.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:29 smithi016 bash[77995]: audit 2023-09-23T19:51:28.681665+0000 mon.a (mon.0) 1644 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:51:29.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:29 smithi016 bash[77995]: audit 2023-09-23T19:51:28.683477+0000 mon.a (mon.0) 1645 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:51:29.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:29 smithi016 bash[75004]: cluster 2023-09-23T19:51:28.435425+0000 mgr.y (mgr.25325) 1323 : cluster 0 pgmap v1324: 89 pgs: 7 creating+peering, 82 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:29.788 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:29 smithi016 bash[75004]: audit 2023-09-23T19:51:28.681665+0000 mon.a (mon.0) 1644 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:51:29.788 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:29 smithi016 bash[75004]: audit 2023-09-23T19:51:28.683477+0000 mon.a (mon.0) 1645 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:51:29.794 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:29.824 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:29.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:29.827 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:51:29.928 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:29.940 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:29 smithi092 bash[58855]: cluster 2023-09-23T19:51:28.435425+0000 mgr.y (mgr.25325) 1323 : cluster 0 pgmap v1324: 89 pgs: 7 creating+peering, 82 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:29.940 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:29 smithi092 bash[58855]: audit 2023-09-23T19:51:28.681665+0000 mon.a (mon.0) 1644 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:51:29.940 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:29 smithi092 bash[58855]: audit 2023-09-23T19:51:28.683477+0000 mon.a (mon.0) 1645 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:51:29.959 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:29.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:29.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:30.065 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:30.099 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:30.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:30.102 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:30.203 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:30.233 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:30.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:30.236 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:51:30.337 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:30.373 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:30.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:30.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:51:30.476 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:30.500 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:30.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:30.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:30.604 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:30.639 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:30.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:30.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:30.743 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:30.775 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:30.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:30.778 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:51:30.879 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:30.902 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:30.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:30.905 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:31.006 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:31.028 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:31.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:31.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:31.131 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:31.158 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:31.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:31.160 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:31.261 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:31.294 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:31.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:31.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:51:31.397 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:31.427 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:31.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:31.430 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:51:31.531 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:31.561 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:31.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:31.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:31.665 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:31.695 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:31.698 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-09-23T19:51:31.698 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [], 2023-09-23T19:51:31.698 INFO:teuthology.orchestra.run.smithi016.stdout: "complaint_time": 30, 2023-09-23T19:51:31.698 INFO:teuthology.orchestra.run.smithi016.stdout: "num_blocked_ops": 0 2023-09-23T19:51:31.699 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-09-23T19:51:31.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:31.699 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:31.800 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:31.830 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:31.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:31.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:31.935 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:31.944 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:31 smithi016 bash[77995]: cluster 2023-09-23T19:51:30.436141+0000 mgr.y (mgr.25325) 1324 : cluster 0 pgmap v1325: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:31.944 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:31 smithi016 bash[75004]: cluster 2023-09-23T19:51:30.436141+0000 mgr.y (mgr.25325) 1324 : cluster 0 pgmap v1325: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:31.960 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:31.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:31.963 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:51:32.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:31 smithi092 bash[58855]: cluster 2023-09-23T19:51:30.436141+0000 mgr.y (mgr.25325) 1324 : cluster 0 pgmap v1325: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:32.063 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:32.087 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:32.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:32.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:51:32.191 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:32.212 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:32.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:32.215 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:51:32.317 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:32.339 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:32.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:32.342 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:51:32.443 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:32.467 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:32.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:32.470 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:32.571 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:32.602 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:32.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:32.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:32.706 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:32.738 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:32.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:32.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:32.841 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:32.871 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:32.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:32.874 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:32.975 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:33.006 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:33.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:33.009 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:33.110 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:33.131 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:33.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:33.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:33.234 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:33.260 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:33.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:33.263 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:51:33.363 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:33.384 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:33.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:33.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:33.487 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:33.510 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:33.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:33.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:33.612 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:33.639 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:33.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:33.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:51:33.745 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:33.774 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:33.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:33.778 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:51:33.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:33 smithi016 bash[75004]: cluster 2023-09-23T19:51:32.436678+0000 mgr.y (mgr.25325) 1325 : cluster 0 pgmap v1326: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:33.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:33 smithi016 bash[77995]: cluster 2023-09-23T19:51:32.436678+0000 mgr.y (mgr.25325) 1325 : cluster 0 pgmap v1326: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:33.878 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:33.904 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:33.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:33.908 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:34.008 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:34.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:33 smithi092 bash[58855]: cluster 2023-09-23T19:51:32.436678+0000 mgr.y (mgr.25325) 1325 : cluster 0 pgmap v1326: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:34.039 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:34.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:34.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:34.142 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:34.171 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:34.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:34.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:34.275 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:34.306 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:34.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:34.309 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:34.410 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:34.441 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:34.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:34.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:51:34.513 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T19:51:34.545 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:34.577 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:34.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:34.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:51:34.680 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:34.699 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:34.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:34.702 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:34.802 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:34.827 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:34.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:34.829 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:34.930 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:34.965 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:34.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:34.970 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:35.071 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:35.101 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:35.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:35.105 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:35.206 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:35.233 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:35.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:35.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:35.338 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:35.369 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:35.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:35.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:51:35.473 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:35.510 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:35.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:35.513 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:51:35.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:35 smithi016 bash[77995]: audit 2023-09-23T19:51:34.257595+0000 mon.a (mon.0) 1646 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:51:35.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:35 smithi016 bash[75004]: audit 2023-09-23T19:51:34.257595+0000 mon.a (mon.0) 1646 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:51:35.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:35 smithi092 bash[58855]: audit 2023-09-23T19:51:34.257595+0000 mon.a (mon.0) 1646 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:51:35.614 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:35.641 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:35.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:35.644 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:35.703 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:51:35.745 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:35.778 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:35.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:35.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:35.881 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:35.893 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:51:35 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:51:35] "GET /metrics HTTP/1.1" 200 36237 "" "Prometheus/2.33.4" 2023-09-23T19:51:35.911 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:35.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:35.914 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:36.015 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:36.045 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:36.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:36.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:51:36.148 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:36.182 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:36.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:36.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:51:36.287 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:36.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:36 smithi092 bash[58855]: cluster 2023-09-23T19:51:34.437425+0000 mgr.y (mgr.25325) 1326 : cluster 0 pgmap v1327: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:36.318 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:36.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:36.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:51:36.422 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:36.453 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:36.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:36.455 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:51:36.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:36 smithi016 bash[77995]: cluster 2023-09-23T19:51:34.437425+0000 mgr.y (mgr.25325) 1326 : cluster 0 pgmap v1327: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:36.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:36 smithi016 bash[75004]: cluster 2023-09-23T19:51:34.437425+0000 mgr.y (mgr.25325) 1326 : cluster 0 pgmap v1327: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:36.556 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:36.577 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:36.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:36.580 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:36.681 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:36.708 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:36.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:36.711 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:36.811 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:36.841 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:36.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:36.844 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:36.944 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:36.966 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:36.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:36.968 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:37.069 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:37.096 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:37.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:37.099 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:51:37.199 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:37.226 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:37.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:37.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:51:37.330 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:37.354 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:37.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:37.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:37.458 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:37.490 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:37.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:37.493 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:37.594 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:37.622 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:37.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:37.625 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:37.726 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:37.761 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:37.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:37.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:51:37.866 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:37.894 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:37.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:37.898 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:37.999 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:38.024 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:38.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:38.027 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:38.128 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:38.156 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:38.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:38.159 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:51:38.259 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:38.291 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:38.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:38.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:51:38.395 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:38.425 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:38.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:38.428 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:38.529 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:38.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:38 smithi016 bash[77995]: cluster 2023-09-23T19:51:36.438546+0000 mgr.y (mgr.25325) 1327 : cluster 0 pgmap v1328: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:38.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:38 smithi016 bash[75004]: cluster 2023-09-23T19:51:36.438546+0000 mgr.y (mgr.25325) 1327 : cluster 0 pgmap v1328: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:38.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:38 smithi092 bash[58855]: cluster 2023-09-23T19:51:36.438546+0000 mgr.y (mgr.25325) 1327 : cluster 0 pgmap v1328: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:38.554 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:38.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:38.557 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:38.658 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:38.690 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:38.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:38.693 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:38.794 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:38.829 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:38.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:38.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:38.933 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:38.963 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:38.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:38.966 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:39.044 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:51:39.066 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:39.086 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:39.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:39.089 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:39.189 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:39.212 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:39.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:39.215 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:39.287 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.7 to 1 (10000) 2023-09-23T19:51:39.316 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:39.323 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:39 smithi016 bash[77995]: audit 2023-09-23T19:51:39.116220+0000 mon.c (mon.2) 208 : audit 1 from='client.? 172.21.15.16:0/1253338017' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T19:51:39.324 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:39 smithi016 bash[77995]: audit 2023-09-23T19:51:39.116717+0000 mon.a (mon.0) 1647 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T19:51:39.324 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:39 smithi016 bash[75004]: audit 2023-09-23T19:51:39.116220+0000 mon.c (mon.2) 208 : audit 1 from='client.? 172.21.15.16:0/1253338017' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T19:51:39.324 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:39 smithi016 bash[75004]: audit 2023-09-23T19:51:39.116717+0000 mon.a (mon.0) 1647 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T19:51:39.337 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:39.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:39.341 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:39.441 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:39.471 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:39.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:39.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:39.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:39 smithi092 bash[58855]: audit 2023-09-23T19:51:39.116220+0000 mon.c (mon.2) 208 : audit 1 from='client.? 172.21.15.16:0/1253338017' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T19:51:39.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:39 smithi092 bash[58855]: audit 2023-09-23T19:51:39.116717+0000 mon.a (mon.0) 1647 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T19:51:39.575 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:39.607 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:39.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:39.611 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:51:39.711 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:39.738 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:39.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:39.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:51:39.842 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:39.867 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:39.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:39.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:51:39.971 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:40.002 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:40.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:40.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:40.106 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:40.134 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:40.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:40.137 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:40.237 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:40.262 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:40.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:40.265 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:40.366 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:40.396 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:40.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:40.400 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:51:40.500 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:40.508 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:40 smithi016 bash[77995]: cluster 2023-09-23T19:51:38.439090+0000 mgr.y (mgr.25325) 1328 : cluster 0 pgmap v1329: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:40.509 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:40 smithi016 bash[77995]: audit 2023-09-23T19:51:39.278184+0000 mon.a (mon.0) 1648 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T19:51:40.509 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:40 smithi016 bash[77995]: cluster 2023-09-23T19:51:39.278269+0000 mon.a (mon.0) 1649 : cluster 0 osdmap e235: 8 total, 8 up, 8 in 2023-09-23T19:51:40.509 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:40 smithi016 bash[75004]: cluster 2023-09-23T19:51:38.439090+0000 mgr.y (mgr.25325) 1328 : cluster 0 pgmap v1329: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:40.509 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:40 smithi016 bash[75004]: audit 2023-09-23T19:51:39.278184+0000 mon.a (mon.0) 1648 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T19:51:40.509 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:40 smithi016 bash[75004]: cluster 2023-09-23T19:51:39.278269+0000 mon.a (mon.0) 1649 : cluster 0 osdmap e235: 8 total, 8 up, 8 in 2023-09-23T19:51:40.523 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:40.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:40.525 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:40.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:40 smithi092 bash[58855]: cluster 2023-09-23T19:51:38.439090+0000 mgr.y (mgr.25325) 1328 : cluster 0 pgmap v1329: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:40.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:40 smithi092 bash[58855]: audit 2023-09-23T19:51:39.278184+0000 mon.a (mon.0) 1648 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T19:51:40.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:40 smithi092 bash[58855]: cluster 2023-09-23T19:51:39.278269+0000 mon.a (mon.0) 1649 : cluster 0 osdmap e235: 8 total, 8 up, 8 in 2023-09-23T19:51:40.626 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:40.660 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:40.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:40.662 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:51:40.763 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:40.790 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:40.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:40.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:40.894 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:40.915 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:40.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:40.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:40.930 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:51:40.951 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:51:40.972 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:51:40.993 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:51:41.015 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:51:41.019 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:41.037 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:51:41.053 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:41.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:41.056 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:51:41.067 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:51:41.091 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:51:41.093 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:51:41.093 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:51:41.093 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:51:41.093 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:51:41.093 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:51:41.093 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:51:41.093 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:51:41.093 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:51:41.157 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:41.190 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:41.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:41.193 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:41.293 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:41.297 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is set 2023-09-23T19:51:41.319 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:41.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:41.322 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:41.423 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:41.451 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:41.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:41.454 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:41.555 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:41.568 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:41 smithi092 bash[58855]: cluster 2023-09-23T19:51:39.748291+0000 osd.6 (osd.6) 168 : cluster 3 failed to encode map e235 with expected crc 2023-09-23T19:51:41.568 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:41 smithi092 bash[58855]: cluster 2023-09-23T19:51:39.751154+0000 osd.6 (osd.6) 169 : cluster 3 failed to encode map e235 with expected crc 2023-09-23T19:51:41.568 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:41 smithi092 bash[58855]: cluster 2023-09-23T19:51:39.941352+0000 osd.4 (osd.4) 177 : cluster 3 failed to encode map e235 with expected crc 2023-09-23T19:51:41.568 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:41 smithi092 bash[58855]: cluster 2023-09-23T19:51:40.115046+0000 osd.7 (osd.7) 152 : cluster 3 failed to encode map e235 with expected crc 2023-09-23T19:51:41.568 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:41 smithi092 bash[58855]: cluster 2023-09-23T19:51:40.115252+0000 osd.5 (osd.5) 162 : cluster 3 failed to encode map e235 with expected crc 2023-09-23T19:51:41.568 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:41 smithi092 bash[58855]: cluster 2023-09-23T19:51:40.439722+0000 mgr.y (mgr.25325) 1329 : cluster 0 pgmap v1331: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:41.568 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:41 smithi092 bash[58855]: audit 2023-09-23T19:51:40.903098+0000 mon.c (mon.2) 209 : audit 0 from='client.? 172.21.15.16:0/1263554096' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:51:41.568 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:41 smithi092 bash[58855]: audit 2023-09-23T19:51:41.045011+0000 mon.c (mon.2) 210 : audit 1 from='client.? 172.21.15.16:0/2731556864' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:51:41.568 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:41 smithi092 bash[58855]: audit 2023-09-23T19:51:41.045458+0000 mon.a (mon.0) 1650 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:51:41.589 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:41.590 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:41 smithi016 bash[75004]: cluster 2023-09-23T19:51:39.748291+0000 osd.6 (osd.6) 168 : cluster 3 failed to encode map e235 with expected crc 2023-09-23T19:51:41.590 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:41 smithi016 bash[75004]: cluster 2023-09-23T19:51:39.751154+0000 osd.6 (osd.6) 169 : cluster 3 failed to encode map e235 with expected crc 2023-09-23T19:51:41.590 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:41 smithi016 bash[75004]: cluster 2023-09-23T19:51:39.941352+0000 osd.4 (osd.4) 177 : cluster 3 failed to encode map e235 with expected crc 2023-09-23T19:51:41.590 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:41 smithi016 bash[75004]: cluster 2023-09-23T19:51:40.115046+0000 osd.7 (osd.7) 152 : cluster 3 failed to encode map e235 with expected crc 2023-09-23T19:51:41.590 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:41 smithi016 bash[75004]: cluster 2023-09-23T19:51:40.115252+0000 osd.5 (osd.5) 162 : cluster 3 failed to encode map e235 with expected crc 2023-09-23T19:51:41.590 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:41 smithi016 bash[75004]: cluster 2023-09-23T19:51:40.439722+0000 mgr.y (mgr.25325) 1329 : cluster 0 pgmap v1331: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:41.590 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:41 smithi016 bash[75004]: audit 2023-09-23T19:51:40.903098+0000 mon.c (mon.2) 209 : audit 0 from='client.? 172.21.15.16:0/1263554096' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:51:41.590 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:41 smithi016 bash[75004]: audit 2023-09-23T19:51:41.045011+0000 mon.c (mon.2) 210 : audit 1 from='client.? 172.21.15.16:0/2731556864' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:51:41.591 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:41 smithi016 bash[75004]: audit 2023-09-23T19:51:41.045458+0000 mon.a (mon.0) 1650 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:51:41.591 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:41 smithi016 bash[77995]: cluster 2023-09-23T19:51:39.748291+0000 osd.6 (osd.6) 168 : cluster 3 failed to encode map e235 with expected crc 2023-09-23T19:51:41.591 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:41 smithi016 bash[77995]: cluster 2023-09-23T19:51:39.751154+0000 osd.6 (osd.6) 169 : cluster 3 failed to encode map e235 with expected crc 2023-09-23T19:51:41.591 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:41 smithi016 bash[77995]: cluster 2023-09-23T19:51:39.941352+0000 osd.4 (osd.4) 177 : cluster 3 failed to encode map e235 with expected crc 2023-09-23T19:51:41.591 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:41 smithi016 bash[77995]: cluster 2023-09-23T19:51:40.115046+0000 osd.7 (osd.7) 152 : cluster 3 failed to encode map e235 with expected crc 2023-09-23T19:51:41.591 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:41 smithi016 bash[77995]: cluster 2023-09-23T19:51:40.115252+0000 osd.5 (osd.5) 162 : cluster 3 failed to encode map e235 with expected crc 2023-09-23T19:51:41.591 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:41 smithi016 bash[77995]: cluster 2023-09-23T19:51:40.439722+0000 mgr.y (mgr.25325) 1329 : cluster 0 pgmap v1331: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:41.591 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:41 smithi016 bash[77995]: audit 2023-09-23T19:51:40.903098+0000 mon.c (mon.2) 209 : audit 0 from='client.? 172.21.15.16:0/1263554096' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:51:41.591 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:41 smithi016 bash[77995]: audit 2023-09-23T19:51:41.045011+0000 mon.c (mon.2) 210 : audit 1 from='client.? 172.21.15.16:0/2731556864' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:51:41.591 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:41 smithi016 bash[77995]: audit 2023-09-23T19:51:41.045458+0000 mon.a (mon.0) 1650 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:51:41.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:41.593 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:41.693 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:41.724 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:41.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:41.727 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:41.828 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:41.861 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:41.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:41.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:51:41.964 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:41.991 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:41.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:41.993 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:42.094 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:42.127 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:42.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:42.130 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:51:42.230 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:42.254 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:42.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:42.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:51:42.357 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:42.370 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:42 smithi092 bash[58855]: cluster 2023-09-23T19:51:41.276367+0000 mon.a (mon.0) 1651 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:51:42.370 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:42 smithi092 bash[58855]: audit 2023-09-23T19:51:41.287826+0000 mon.a (mon.0) 1652 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:51:42.370 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:42 smithi092 bash[58855]: cluster 2023-09-23T19:51:41.287918+0000 mon.a (mon.0) 1653 : cluster 0 osdmap e236: 8 total, 8 up, 8 in 2023-09-23T19:51:42.371 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:42 smithi092 bash[58855]: audit 2023-09-23T19:51:42.213785+0000 mon.a (mon.0) 1654 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:51:42.387 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:42.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:42.389 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:42.490 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:42.517 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:42.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:42.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:42.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:42 smithi016 bash[75004]: cluster 2023-09-23T19:51:41.276367+0000 mon.a (mon.0) 1651 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:51:42.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:42 smithi016 bash[75004]: audit 2023-09-23T19:51:41.287826+0000 mon.a (mon.0) 1652 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:51:42.538 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:42 smithi016 bash[75004]: cluster 2023-09-23T19:51:41.287918+0000 mon.a (mon.0) 1653 : cluster 0 osdmap e236: 8 total, 8 up, 8 in 2023-09-23T19:51:42.538 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:42 smithi016 bash[75004]: audit 2023-09-23T19:51:42.213785+0000 mon.a (mon.0) 1654 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:51:42.538 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:42 smithi016 bash[77995]: cluster 2023-09-23T19:51:41.276367+0000 mon.a (mon.0) 1651 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:51:42.538 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:42 smithi016 bash[77995]: audit 2023-09-23T19:51:41.287826+0000 mon.a (mon.0) 1652 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:51:42.538 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:42 smithi016 bash[77995]: cluster 2023-09-23T19:51:41.287918+0000 mon.a (mon.0) 1653 : cluster 0 osdmap e236: 8 total, 8 up, 8 in 2023-09-23T19:51:42.539 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:42 smithi016 bash[77995]: audit 2023-09-23T19:51:42.213785+0000 mon.a (mon.0) 1654 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:51:42.620 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:42.643 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:42.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:42.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:42.747 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:42.775 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:42.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:42.779 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:42.880 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:42.903 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:42.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:42.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:43.005 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:43.028 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:43.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:43.030 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:43.131 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:43.164 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:43.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:43.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:51:43.268 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:43.303 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:43.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:43.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:43.406 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:43.419 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:43 smithi092 bash[58855]: cluster 2023-09-23T19:51:42.197951+0000 osd.7 (osd.7) 153 : cluster 3 failed to encode map e236 with expected crc 2023-09-23T19:51:43.420 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:43 smithi092 bash[58855]: cluster 2023-09-23T19:51:42.200401+0000 osd.7 (osd.7) 154 : cluster 3 failed to encode map e236 with expected crc 2023-09-23T19:51:43.420 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:43 smithi092 bash[58855]: cluster 2023-09-23T19:51:42.440135+0000 mgr.y (mgr.25325) 1330 : cluster 0 pgmap v1333: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:43.420 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:43 smithi092 bash[58855]: cluster 2023-09-23T19:51:42.549312+0000 osd.6 (osd.6) 170 : cluster 3 failed to encode map e236 with expected crc 2023-09-23T19:51:43.420 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:43 smithi092 bash[58855]: cluster 2023-09-23T19:51:42.551726+0000 osd.6 (osd.6) 171 : cluster 3 failed to encode map e236 with expected crc 2023-09-23T19:51:43.440 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:43.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:43.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:43.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:43 smithi016 bash[77995]: cluster 2023-09-23T19:51:42.197951+0000 osd.7 (osd.7) 153 : cluster 3 failed to encode map e236 with expected crc 2023-09-23T19:51:43.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:43 smithi016 bash[77995]: cluster 2023-09-23T19:51:42.200401+0000 osd.7 (osd.7) 154 : cluster 3 failed to encode map e236 with expected crc 2023-09-23T19:51:43.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:43 smithi016 bash[77995]: cluster 2023-09-23T19:51:42.440135+0000 mgr.y (mgr.25325) 1330 : cluster 0 pgmap v1333: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:43.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:43 smithi016 bash[77995]: cluster 2023-09-23T19:51:42.549312+0000 osd.6 (osd.6) 170 : cluster 3 failed to encode map e236 with expected crc 2023-09-23T19:51:43.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:43 smithi016 bash[77995]: cluster 2023-09-23T19:51:42.551726+0000 osd.6 (osd.6) 171 : cluster 3 failed to encode map e236 with expected crc 2023-09-23T19:51:43.538 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:43 smithi016 bash[75004]: cluster 2023-09-23T19:51:42.197951+0000 osd.7 (osd.7) 153 : cluster 3 failed to encode map e236 with expected crc 2023-09-23T19:51:43.538 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:43 smithi016 bash[75004]: cluster 2023-09-23T19:51:42.200401+0000 osd.7 (osd.7) 154 : cluster 3 failed to encode map e236 with expected crc 2023-09-23T19:51:43.538 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:43 smithi016 bash[75004]: cluster 2023-09-23T19:51:42.440135+0000 mgr.y (mgr.25325) 1330 : cluster 0 pgmap v1333: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:43.538 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:43 smithi016 bash[75004]: cluster 2023-09-23T19:51:42.549312+0000 osd.6 (osd.6) 170 : cluster 3 failed to encode map e236 with expected crc 2023-09-23T19:51:43.538 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:43 smithi016 bash[75004]: cluster 2023-09-23T19:51:42.551726+0000 osd.6 (osd.6) 171 : cluster 3 failed to encode map e236 with expected crc 2023-09-23T19:51:43.543 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:43.573 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:43.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:43.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:43.677 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:43.680 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T19:51:43.681 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T19:51:43.702 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:43.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:43.704 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:43.805 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:43.823 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:43.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:43.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:43.927 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:43.957 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:43.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:43.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:51:44.060 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:44.085 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:44.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:44.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:51:44.188 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:44.214 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:44.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:44.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:51:44.317 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:44.352 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:44.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:44.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:44.455 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:44.486 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:44.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:44.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:51:44.590 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:44.603 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:44 smithi092 bash[58855]: cluster 2023-09-23T19:51:42.820821+0000 osd.4 (osd.4) 178 : cluster 3 failed to encode map e236 with expected crc 2023-09-23T19:51:44.604 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:44 smithi092 bash[58855]: cluster 2023-09-23T19:51:42.823736+0000 osd.4 (osd.4) 179 : cluster 3 failed to encode map e236 with expected crc 2023-09-23T19:51:44.604 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:44 smithi092 bash[58855]: audit 2023-09-23T19:51:43.675697+0000 mon.a (mon.0) 1655 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:51:44.622 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:44.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:44.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:51:44.725 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:44.754 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:44.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:44.757 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:44.789 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:44 smithi016 bash[77995]: cluster 2023-09-23T19:51:42.820821+0000 osd.4 (osd.4) 178 : cluster 3 failed to encode map e236 with expected crc 2023-09-23T19:51:44.789 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:44 smithi016 bash[77995]: cluster 2023-09-23T19:51:42.823736+0000 osd.4 (osd.4) 179 : cluster 3 failed to encode map e236 with expected crc 2023-09-23T19:51:44.789 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:44 smithi016 bash[77995]: audit 2023-09-23T19:51:43.675697+0000 mon.a (mon.0) 1655 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:51:44.789 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:44 smithi016 bash[75004]: cluster 2023-09-23T19:51:42.820821+0000 osd.4 (osd.4) 178 : cluster 3 failed to encode map e236 with expected crc 2023-09-23T19:51:44.789 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:44 smithi016 bash[75004]: cluster 2023-09-23T19:51:42.823736+0000 osd.4 (osd.4) 179 : cluster 3 failed to encode map e236 with expected crc 2023-09-23T19:51:44.790 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:44 smithi016 bash[75004]: audit 2023-09-23T19:51:43.675697+0000 mon.a (mon.0) 1655 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:51:44.858 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:44.881 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:44.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:44.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:51:44.984 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:45.019 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:45.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:45.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:45.122 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:45.149 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:45.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:45.152 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:45.253 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:45.286 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:45.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:45.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:45.389 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:45.418 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:45.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:45.422 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:45.522 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:45.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:45 smithi092 bash[58855]: cluster 2023-09-23T19:51:43.867534+0000 osd.5 (osd.5) 163 : cluster 3 failed to encode map e236 with expected crc 2023-09-23T19:51:45.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:45 smithi092 bash[58855]: cluster 2023-09-23T19:51:44.440718+0000 mgr.y (mgr.25325) 1331 : cluster 0 pgmap v1334: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:45.553 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:45.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:45.556 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:51:45.640 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:45 smithi016 bash[75004]: cluster 2023-09-23T19:51:43.867534+0000 osd.5 (osd.5) 163 : cluster 3 failed to encode map e236 with expected crc 2023-09-23T19:51:45.640 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:45 smithi016 bash[75004]: cluster 2023-09-23T19:51:44.440718+0000 mgr.y (mgr.25325) 1331 : cluster 0 pgmap v1334: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:45.641 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:45 smithi016 bash[77995]: cluster 2023-09-23T19:51:43.867534+0000 osd.5 (osd.5) 163 : cluster 3 failed to encode map e236 with expected crc 2023-09-23T19:51:45.641 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:45 smithi016 bash[77995]: cluster 2023-09-23T19:51:44.440718+0000 mgr.y (mgr.25325) 1331 : cluster 0 pgmap v1334: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:45.657 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:45.693 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:45.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:45.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:51:45.796 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:45.827 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:45.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:45.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:51:45.930 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:45.958 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:51:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:51:45] "GET /metrics HTTP/1.1" 200 36236 "" "Prometheus/2.33.4" 2023-09-23T19:51:45.960 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:45.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:45.964 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:46.065 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:46.093 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:46.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:46.096 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:46.196 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:46.225 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:46.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:46.228 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:51:46.329 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:46.351 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:46.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:46.353 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:46.454 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:46.486 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:46.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:46.489 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:46.590 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:46.612 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:46.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:46.614 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:46.715 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:46.737 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:46.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:46.740 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:46.841 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:46.874 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:46.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:46.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:51:46.977 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:47.008 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:47.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:47.011 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:47.112 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:47.136 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:47.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:47.139 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:47.240 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:47.303 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:47.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:47.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:51:47.333 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T19:51:47.406 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:47.426 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:47.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:47.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:47.530 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:47.552 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:47.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:47.555 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:47.655 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:47.684 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:47.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:47.687 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:51:47.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:47 smithi016 bash[77995]: cluster 2023-09-23T19:51:46.441864+0000 mgr.y (mgr.25325) 1332 : cluster 0 pgmap v1335: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:47.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:47 smithi016 bash[75004]: cluster 2023-09-23T19:51:46.441864+0000 mgr.y (mgr.25325) 1332 : cluster 0 pgmap v1335: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:47.788 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:47.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:47 smithi092 bash[58855]: cluster 2023-09-23T19:51:46.441864+0000 mgr.y (mgr.25325) 1332 : cluster 0 pgmap v1335: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:47.817 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:47.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:47.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:47.922 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:47.974 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:47.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:47.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:48.078 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:48.105 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:48.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:48.108 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:48.208 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:48.236 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:48.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:48.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:48.340 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:48.361 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:48.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:48.364 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:48.465 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:48.496 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:48.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:48.499 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:48.600 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:48.625 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:48.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:48.628 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:48.729 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:48.758 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:48.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:48.761 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:48.861 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:48.886 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:48.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:48.889 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:51:48.990 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:49.014 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:49.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:49.016 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:49.117 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:49.148 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:49.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:49.152 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:51:49.252 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:49.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:49.286 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:49.287 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:49.387 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:49.419 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:49.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:49.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:49.517 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:49 smithi016 bash[77995]: cluster 2023-09-23T19:51:48.442368+0000 mgr.y (mgr.25325) 1333 : cluster 0 pgmap v1336: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:49.517 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:49 smithi016 bash[77995]: audit 2023-09-23T19:51:48.506316+0000 mon.a (mon.0) 1656 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:51:49.517 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:49 smithi016 bash[77995]: audit 2023-09-23T19:51:48.512766+0000 mon.a (mon.0) 1657 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:51:49.517 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:49 smithi016 bash[77995]: audit 2023-09-23T19:51:49.086593+0000 mon.a (mon.0) 1658 : audit 1 from='client.? 172.21.15.16:0/921208244' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:51:49.517 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:49 smithi016 bash[75004]: cluster 2023-09-23T19:51:48.442368+0000 mgr.y (mgr.25325) 1333 : cluster 0 pgmap v1336: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:49.517 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:49 smithi016 bash[75004]: audit 2023-09-23T19:51:48.506316+0000 mon.a (mon.0) 1656 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:51:49.518 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:49 smithi016 bash[75004]: audit 2023-09-23T19:51:48.512766+0000 mon.a (mon.0) 1657 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:51:49.518 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:49 smithi016 bash[75004]: audit 2023-09-23T19:51:49.086593+0000 mon.a (mon.0) 1658 : audit 1 from='client.? 172.21.15.16:0/921208244' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:51:49.522 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:49.527 INFO:teuthology.orchestra.run.smithi016.stderr:enabled application 'rados' on pool 'unique_pool_8' 2023-09-23T19:51:49.541 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:51:49.552 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:49.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:49.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:51:49.655 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:49.679 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:49.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:49.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:49.769 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-09-23T19:51:49.769 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:51:49.783 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:49.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:49 smithi092 bash[58855]: cluster 2023-09-23T19:51:48.442368+0000 mgr.y (mgr.25325) 1333 : cluster 0 pgmap v1336: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:49.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:49 smithi092 bash[58855]: audit 2023-09-23T19:51:48.506316+0000 mon.a (mon.0) 1656 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:51:49.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:49 smithi092 bash[58855]: audit 2023-09-23T19:51:48.512766+0000 mon.a (mon.0) 1657 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:51:49.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:49 smithi092 bash[58855]: audit 2023-09-23T19:51:49.086593+0000 mon.a (mon.0) 1658 : audit 1 from='client.? 172.21.15.16:0/921208244' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:51:49.811 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:49.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:49.814 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:49.915 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:49.947 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:49.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:49.949 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:50.050 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:50.070 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:50.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:50.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:50.173 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:50.207 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:50.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:50.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:50.311 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:50.337 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:50.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:50.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:51:50.440 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:50.471 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:50.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:50.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:50.574 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:50.606 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:50.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:50.609 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:51:50.710 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:50.741 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:50.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:50.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:51:50.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:50 smithi016 bash[75004]: audit 2023-09-23T19:51:49.517984+0000 mon.a (mon.0) 1659 : audit 1 from='client.? 172.21.15.16:0/921208244' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T19:51:50.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:50 smithi016 bash[75004]: cluster 2023-09-23T19:51:49.518096+0000 mon.a (mon.0) 1660 : cluster 0 osdmap e237: 8 total, 8 up, 8 in 2023-09-23T19:51:50.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:50 smithi016 bash[75004]: cluster 2023-09-23T19:51:49.797524+0000 osd.5 (osd.5) 164 : cluster 3 failed to encode map e237 with expected crc 2023-09-23T19:51:50.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:50 smithi016 bash[75004]: cluster 2023-09-23T19:51:49.800164+0000 osd.5 (osd.5) 165 : cluster 3 failed to encode map e237 with expected crc 2023-09-23T19:51:50.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:50 smithi016 bash[77995]: audit 2023-09-23T19:51:49.517984+0000 mon.a (mon.0) 1659 : audit 1 from='client.? 172.21.15.16:0/921208244' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T19:51:50.788 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:50 smithi016 bash[77995]: cluster 2023-09-23T19:51:49.518096+0000 mon.a (mon.0) 1660 : cluster 0 osdmap e237: 8 total, 8 up, 8 in 2023-09-23T19:51:50.788 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:50 smithi016 bash[77995]: cluster 2023-09-23T19:51:49.797524+0000 osd.5 (osd.5) 164 : cluster 3 failed to encode map e237 with expected crc 2023-09-23T19:51:50.788 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:50 smithi016 bash[77995]: cluster 2023-09-23T19:51:49.800164+0000 osd.5 (osd.5) 165 : cluster 3 failed to encode map e237 with expected crc 2023-09-23T19:51:50.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:50 smithi092 bash[58855]: audit 2023-09-23T19:51:49.517984+0000 mon.a (mon.0) 1659 : audit 1 from='client.? 172.21.15.16:0/921208244' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T19:51:50.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:50 smithi092 bash[58855]: cluster 2023-09-23T19:51:49.518096+0000 mon.a (mon.0) 1660 : cluster 0 osdmap e237: 8 total, 8 up, 8 in 2023-09-23T19:51:50.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:50 smithi092 bash[58855]: cluster 2023-09-23T19:51:49.797524+0000 osd.5 (osd.5) 164 : cluster 3 failed to encode map e237 with expected crc 2023-09-23T19:51:50.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:50 smithi092 bash[58855]: cluster 2023-09-23T19:51:49.800164+0000 osd.5 (osd.5) 165 : cluster 3 failed to encode map e237 with expected crc 2023-09-23T19:51:50.845 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:50.873 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:50.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:50.875 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:50.976 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:51.004 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:51.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:51.006 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:51.107 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:51.134 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:51.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:51.136 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:51.237 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:51.266 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:51.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:51.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:51.370 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:51.402 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:51.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:51.405 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:51.506 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:51.537 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:51.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:51.540 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:51:51.607 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T19:51:51.641 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:51.688 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:51.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:51.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:51.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:51 smithi016 bash[75004]: cluster 2023-09-23T19:51:50.442993+0000 mgr.y (mgr.25325) 1334 : cluster 0 pgmap v1338: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:51.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:51 smithi016 bash[75004]: cluster 2023-09-23T19:51:51.351019+0000 osd.6 (osd.6) 172 : cluster 3 failed to encode map e237 with expected crc 2023-09-23T19:51:51.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:51 smithi016 bash[75004]: cluster 2023-09-23T19:51:51.353676+0000 osd.6 (osd.6) 173 : cluster 3 failed to encode map e237 with expected crc 2023-09-23T19:51:51.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:51 smithi016 bash[77995]: cluster 2023-09-23T19:51:50.442993+0000 mgr.y (mgr.25325) 1334 : cluster 0 pgmap v1338: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:51.788 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:51 smithi016 bash[77995]: cluster 2023-09-23T19:51:51.351019+0000 osd.6 (osd.6) 172 : cluster 3 failed to encode map e237 with expected crc 2023-09-23T19:51:51.788 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:51 smithi016 bash[77995]: cluster 2023-09-23T19:51:51.353676+0000 osd.6 (osd.6) 173 : cluster 3 failed to encode map e237 with expected crc 2023-09-23T19:51:51.791 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:51.816 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:51.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:51.819 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:51.920 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:51.950 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:51.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:51.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:52.027 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:51:52.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:51 smithi092 bash[58855]: cluster 2023-09-23T19:51:50.442993+0000 mgr.y (mgr.25325) 1334 : cluster 0 pgmap v1338: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:52.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:51 smithi092 bash[58855]: cluster 2023-09-23T19:51:51.351019+0000 osd.6 (osd.6) 172 : cluster 3 failed to encode map e237 with expected crc 2023-09-23T19:51:52.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:51 smithi092 bash[58855]: cluster 2023-09-23T19:51:51.353676+0000 osd.6 (osd.6) 173 : cluster 3 failed to encode map e237 with expected crc 2023-09-23T19:51:52.054 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:52.086 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:52.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:52.089 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:52.190 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:52.218 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:52.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:52.220 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:52.321 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:52.340 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:52.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:52.343 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:51:52.432 DEBUG:teuthology.orchestra.run.smithi016:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_8 bench 90 write' 2023-09-23T19:51:52.434 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T19:51:52.434 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T19:51:52.444 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:52.473 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:hints = 1 2023-09-23T19:51:52.473 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T19:51:52.473 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Object prefix: benchmark_data_smithi016_205181 2023-09-23T19:51:52.474 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:51:52.474 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 0 0 0 0 0 0 - 0 2023-09-23T19:51:52.476 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:52.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:52.481 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:51:52.582 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:52.610 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:52.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:52.613 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:52.714 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:52.723 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:52 smithi092 bash[58855]: cluster 2023-09-23T19:51:51.516156+0000 osd.4 (osd.4) 180 : cluster 3 failed to encode map e237 with expected crc 2023-09-23T19:51:52.724 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:52 smithi092 bash[58855]: cluster 2023-09-23T19:51:51.516355+0000 osd.7 (osd.7) 155 : cluster 3 failed to encode map e237 with expected crc 2023-09-23T19:51:52.740 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:52.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:52.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:52.843 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:52.852 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:52 smithi016 bash[77995]: cluster 2023-09-23T19:51:51.516156+0000 osd.4 (osd.4) 180 : cluster 3 failed to encode map e237 with expected crc 2023-09-23T19:51:52.852 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:52 smithi016 bash[77995]: cluster 2023-09-23T19:51:51.516355+0000 osd.7 (osd.7) 155 : cluster 3 failed to encode map e237 with expected crc 2023-09-23T19:51:52.852 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:52 smithi016 bash[75004]: cluster 2023-09-23T19:51:51.516156+0000 osd.4 (osd.4) 180 : cluster 3 failed to encode map e237 with expected crc 2023-09-23T19:51:52.852 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:52 smithi016 bash[75004]: cluster 2023-09-23T19:51:51.516355+0000 osd.7 (osd.7) 155 : cluster 3 failed to encode map e237 with expected crc 2023-09-23T19:51:52.888 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:52.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:52.894 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:52.994 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:53.020 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:53.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:53.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:51:53.124 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:53.152 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:53.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:53.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:51:53.256 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:53.296 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:53.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:53.298 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:51:53.399 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:53.425 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:53.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:53.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:51:53.474 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 1 16 2575 2559 159.931 159.938 0.00271697 0.00621031 2023-09-23T19:51:53.528 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:53.565 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:53.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:53.567 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:53.668 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:53.700 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:53.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:53.703 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:53.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:53 smithi092 bash[58855]: cluster 2023-09-23T19:51:52.443560+0000 mgr.y (mgr.25325) 1335 : cluster 0 pgmap v1339: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:53.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:53 smithi092 bash[58855]: audit 2023-09-23T19:51:53.383869+0000 mon.a (mon.0) 1661 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:51:53.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:53 smithi092 bash[58855]: audit 2023-09-23T19:51:53.401511+0000 mon.a (mon.0) 1662 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:51:53.805 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:53.836 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:53.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:53.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:53.939 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:53.950 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:53 smithi016 bash[75004]: cluster 2023-09-23T19:51:52.443560+0000 mgr.y (mgr.25325) 1335 : cluster 0 pgmap v1339: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:53.950 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:53 smithi016 bash[75004]: audit 2023-09-23T19:51:53.383869+0000 mon.a (mon.0) 1661 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:51:53.950 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:53 smithi016 bash[75004]: audit 2023-09-23T19:51:53.401511+0000 mon.a (mon.0) 1662 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:51:53.950 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:53 smithi016 bash[77995]: cluster 2023-09-23T19:51:52.443560+0000 mgr.y (mgr.25325) 1335 : cluster 0 pgmap v1339: 89 pgs: 89 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:51:53.950 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:53 smithi016 bash[77995]: audit 2023-09-23T19:51:53.383869+0000 mon.a (mon.0) 1661 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:51:53.951 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:53 smithi016 bash[77995]: audit 2023-09-23T19:51:53.401511+0000 mon.a (mon.0) 1662 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:51:53.970 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:53.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:53.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:51:54.074 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:54.103 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:54.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:54.106 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:54.207 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:54.235 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:54.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:54.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:54.339 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:54.360 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:54.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:54.362 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:54.463 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:54.474 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 2 16 5112 5096 159.239 158.562 0.00216525 0.00625574 2023-09-23T19:51:54.491 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:54.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:54.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:51:54.594 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:54.621 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:54.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:54.625 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:51:54.681 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:54 smithi016 bash[75004]: audit 2023-09-23T19:51:54.032544+0000 mon.a (mon.0) 1663 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:51:54.681 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:54 smithi016 bash[75004]: audit 2023-09-23T19:51:54.034400+0000 mon.a (mon.0) 1664 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:51:54.681 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:54 smithi016 bash[75004]: audit 2023-09-23T19:51:54.045295+0000 mon.a (mon.0) 1665 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:51:54.681 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:54 smithi016 bash[77995]: audit 2023-09-23T19:51:54.032544+0000 mon.a (mon.0) 1663 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:51:54.681 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:54 smithi016 bash[77995]: audit 2023-09-23T19:51:54.034400+0000 mon.a (mon.0) 1664 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:51:54.681 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:54 smithi016 bash[77995]: audit 2023-09-23T19:51:54.045295+0000 mon.a (mon.0) 1665 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:51:54.726 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:54.757 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:54.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:54.760 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:54.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:54 smithi092 bash[58855]: audit 2023-09-23T19:51:54.032544+0000 mon.a (mon.0) 1663 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:51:54.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:54 smithi092 bash[58855]: audit 2023-09-23T19:51:54.034400+0000 mon.a (mon.0) 1664 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:51:54.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:54 smithi092 bash[58855]: audit 2023-09-23T19:51:54.045295+0000 mon.a (mon.0) 1665 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:51:54.861 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:54.896 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:54.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:54.898 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:51:54.999 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:55.026 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:55.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:55.029 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:55.130 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:55.157 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:55.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:55.161 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:55.262 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:55.290 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:55.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:55.295 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:55.396 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:55.427 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:55.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:55.429 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:55.474 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 3 16 7689 7673 159.842 161.062 0.005408 0.00624362 2023-09-23T19:51:55.512 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:51:55.530 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:55.573 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:55.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:55.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:55.676 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:55.706 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:55.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:55.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:55.787 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:51:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:51:55] "GET /metrics HTTP/1.1" 200 36236 "" "Prometheus/2.33.4" 2023-09-23T19:51:55.810 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:55.838 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:55.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:55.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:51:55.941 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:55.969 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:55.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:55.972 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:51:56.072 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:56.082 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:56 smithi016 bash[75004]: cluster 2023-09-23T19:51:54.444147+0000 mgr.y (mgr.25325) 1336 : cluster 0 pgmap v1340: 89 pgs: 89 active+clean; 36 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 3.5 MiB/s wr, 56 op/s 2023-09-23T19:51:56.082 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:56 smithi016 bash[77995]: cluster 2023-09-23T19:51:54.444147+0000 mgr.y (mgr.25325) 1336 : cluster 0 pgmap v1340: 89 pgs: 89 active+clean; 36 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 3.5 MiB/s wr, 56 op/s 2023-09-23T19:51:56.098 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:51:56.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:56.101 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:51:56.202 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:56.211 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:56 smithi092 bash[58855]: cluster 2023-09-23T19:51:54.444147+0000 mgr.y (mgr.25325) 1336 : cluster 0 pgmap v1340: 89 pgs: 89 active+clean; 36 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 3.5 MiB/s wr, 56 op/s 2023-09-23T19:51:56.240 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:56.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:56.243 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:51:56.344 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:56.367 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:56.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:56.370 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:56.401 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:51:56.470 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:56.474 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 4 15 10204 10189 159.19 157.25 0.005782 0.00627204 2023-09-23T19:51:56.499 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:56.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:56.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:56.604 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:56.638 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:56.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:56.642 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:51:56.743 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:56.781 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:56.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:56.783 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:51:56.884 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:56.922 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:56.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:56.925 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:51:57.025 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:57.054 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:57.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:57.058 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:57.159 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:57.193 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:57.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:57.197 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:51:57.297 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:57.327 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:57.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:57.330 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:51:57.431 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:57.465 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:51:57.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:57.467 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:51:57.475 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 5 16 12554 12538 156.711 146.812 0.00637533 0.00637431 2023-09-23T19:51:57.568 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:57.592 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:57.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:57.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:51:57.695 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:57.723 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:57.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:57.726 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:51:57.826 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:57.851 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:57.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:57.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:57.955 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:57.987 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:57.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:57.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:58.084 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:51:58.090 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:58.116 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:58 smithi016 bash[77995]: cluster 2023-09-23T19:51:56.444868+0000 mgr.y (mgr.25325) 1337 : cluster 0 pgmap v1341: 89 pgs: 89 active+clean; 330 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 33 MiB/s wr, 526 op/s 2023-09-23T19:51:58.118 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:58 smithi016 bash[75004]: cluster 2023-09-23T19:51:56.444868+0000 mgr.y (mgr.25325) 1337 : cluster 0 pgmap v1341: 89 pgs: 89 active+clean; 330 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 33 MiB/s wr, 526 op/s 2023-09-23T19:51:58.126 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:58.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:58.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:58.232 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:58.261 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:58.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:58.264 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:51:58.364 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:58.373 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:58 smithi092 bash[58855]: cluster 2023-09-23T19:51:56.444868+0000 mgr.y (mgr.25325) 1337 : cluster 0 pgmap v1341: 89 pgs: 89 active+clean; 330 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 33 MiB/s wr, 526 op/s 2023-09-23T19:51:58.390 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:58.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:58.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:58.475 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 6 16 14374 14358 149.55 113.75 0.00465315 0.00667699 2023-09-23T19:51:58.493 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:58.522 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:58.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:58.524 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:58.625 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:58.655 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:58.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:58.658 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:58.759 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:58.785 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:58.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:58.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:51:58.889 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:58.918 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:58.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:58.921 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:51:59.021 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:59.050 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:59.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:59.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:59.154 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:59.181 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:51:59.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:59.186 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:51:59.286 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:59.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:51:59 smithi092 bash[58855]: audit 2023-09-23T19:51:58.675276+0000 mon.a (mon.0) 1666 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:51:59.316 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:51:59.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:59.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:51:59.420 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:59.453 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:59.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:59.455 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:51:59.475 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 7 16 16402 16386 146.291 126.75 0.00452701 0.00682742 2023-09-23T19:51:59.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:51:59 smithi016 bash[75004]: audit 2023-09-23T19:51:58.675276+0000 mon.a (mon.0) 1666 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:51:59.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:51:59 smithi016 bash[77995]: audit 2023-09-23T19:51:58.675276+0000 mon.a (mon.0) 1666 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:51:59.556 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:59.584 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:51:59.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:59.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:51:59.687 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:59.713 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:51:59.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:59.718 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:51:59.818 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:59.843 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:51:59.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:59.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:51:59.946 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:59.972 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:51:59.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:59.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:00.075 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:00.083 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:00 smithi016 bash[75004]: cluster 2023-09-23T19:51:58.445400+0000 mgr.y (mgr.25325) 1338 : cluster 0 pgmap v1342: 89 pgs: 89 active+clean; 421 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 42 MiB/s wr, 673 op/s 2023-09-23T19:52:00.083 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:00 smithi016 bash[77995]: cluster 2023-09-23T19:51:58.445400+0000 mgr.y (mgr.25325) 1338 : cluster 0 pgmap v1342: 89 pgs: 89 active+clean; 421 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 42 MiB/s wr, 673 op/s 2023-09-23T19:52:00.097 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:00.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:00.099 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:00.160 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:00 smithi092 bash[58855]: cluster 2023-09-23T19:51:58.445400+0000 mgr.y (mgr.25325) 1338 : cluster 0 pgmap v1342: 89 pgs: 89 active+clean; 421 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 42 MiB/s wr, 673 op/s 2023-09-23T19:52:00.200 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:00.235 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:00.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:00.238 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:52:00.339 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:00.365 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:00.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:00.368 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:52:00.469 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:00.475 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 8 16 18295 18279 142.792 118.312 0.00512146 0.00699707 2023-09-23T19:52:00.492 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:00.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:00.494 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:00.595 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:00.618 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:00.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:00.621 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:00.722 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:00.741 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:00.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:00.744 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:00.844 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:00.867 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:00.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:00.869 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:00.970 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:01.002 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:01.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:01.004 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:01.105 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:01.131 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:01.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:01.135 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:01.236 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:01.261 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:01.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:01.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:01.365 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:01.398 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:01.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:01.407 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:01.475 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 9 16 19950 19934 138.418 103.438 0.00746308 0.00721189 2023-09-23T19:52:01.509 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:01.534 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:01.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:01.540 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:01.641 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:01.688 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:01.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:01.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:01.792 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:01.825 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:01.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:01.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:52:01.931 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:01.987 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-09-23T19:52:01.991 INFO:teuthology.orchestra.run.smithi016.stderr:dumped all 2023-09-23T19:52:02.101 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:02 smithi016 bash[75004]: cluster 2023-09-23T19:52:00.446091+0000 mgr.y (mgr.25325) 1339 : cluster 0 pgmap v1343: 89 pgs: 89 active+clean; 993 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2023-09-23T19:52:02.241 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:02.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:02.243 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:52:02.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:02 smithi016 bash[77995]: cluster 2023-09-23T19:52:00.446091+0000 mgr.y (mgr.25325) 1339 : cluster 0 pgmap v1343: 89 pgs: 89 active+clean; 993 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2023-09-23T19:52:02.344 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:02.370 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:02.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:02.374 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:02.475 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:02.475 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 10 16 21825 21809 136.294 117.188 0.0094007 0.00733045 2023-09-23T19:52:02.499 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:02.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:02.502 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:02.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:02 smithi092 bash[58855]: cluster 2023-09-23T19:52:00.446091+0000 mgr.y (mgr.25325) 1339 : cluster 0 pgmap v1343: 89 pgs: 89 active+clean; 993 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2023-09-23T19:52:02.602 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:02.626 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:02.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:02.630 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:02.731 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:02.762 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:02.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:02.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:02.865 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:02.892 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:02.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:02.894 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:02.995 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:03.020 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:03.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:03.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:03.124 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:03.158 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:03.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:03.161 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:52:03.262 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:03.270 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:03 smithi092 bash[58855]: audit 2023-09-23T19:52:01.962192+0000 mgr.y (mgr.25325) 1340 : audit 0 from='client.36708 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:52:03.286 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:03.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:03 smithi016 bash[75004]: audit 2023-09-23T19:52:01.962192+0000 mgr.y (mgr.25325) 1340 : audit 0 from='client.36708 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:52:03.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:03 smithi016 bash[77995]: audit 2023-09-23T19:52:01.962192+0000 mgr.y (mgr.25325) 1340 : audit 0 from='client.36708 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:52:03.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:03.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:03.311 INFO:teuthology.orchestra.run.smithi092.stdout:ERROR: (22) Invalid argument 2023-09-23T19:52:03.312 INFO:teuthology.orchestra.run.smithi092.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:52:03.389 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:03.417 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:03.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:03.421 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:52:03.475 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 11 16 23698 23682 134.545 117.062 0.00597064 0.00742447 2023-09-23T19:52:03.521 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:03.548 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:03.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:03.556 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:03.657 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:03.686 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:03.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:03.688 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:03.789 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:03.815 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:03.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:03.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:03.920 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:03.949 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:03.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:03.954 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:04.055 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:04.088 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:04.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:04.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:04.191 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:04.198 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:04 smithi092 bash[58855]: cluster 2023-09-23T19:52:02.446639+0000 mgr.y (mgr.25325) 1341 : cluster 0 pgmap v1344: 89 pgs: 89 active+clean; 1.0 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-09-23T19:52:04.213 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:04.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:04.215 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:52:04.316 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:04.342 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:04.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:04.345 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:04.446 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:04.475 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 12 16 25005 24989 130.139 81.6875 0.00898864 0.00767564 2023-09-23T19:52:04.478 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:04.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:04.481 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:52:04.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:04 smithi016 bash[75004]: cluster 2023-09-23T19:52:02.446639+0000 mgr.y (mgr.25325) 1341 : cluster 0 pgmap v1344: 89 pgs: 89 active+clean; 1.0 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-09-23T19:52:04.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:04 smithi016 bash[77995]: cluster 2023-09-23T19:52:02.446639+0000 mgr.y (mgr.25325) 1341 : cluster 0 pgmap v1344: 89 pgs: 89 active+clean; 1.0 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-09-23T19:52:04.582 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:04.610 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:04.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:04.614 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:52:04.714 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:04.746 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:04.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:04.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:04.836 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T19:52:04.850 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:04.911 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:04.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:04.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:05.018 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:05.049 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:05.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:05.052 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:52:05.153 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:05.181 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:05.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:05.183 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:05.284 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:05.308 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:05.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:05.311 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:05.412 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:05.448 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:05.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:05.451 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:05.476 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 13 16 26739 26723 128.464 108.375 0.00602728 0.00777719 2023-09-23T19:52:05.552 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:05.583 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:05.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:05.587 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:05.688 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:05.713 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:05.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:05.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:05.787 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:52:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:52:05] "GET /metrics HTTP/1.1" 200 36292 "" "Prometheus/2.33.4" 2023-09-23T19:52:05.817 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:05.842 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:05.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:05.848 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:05.949 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:05.982 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:05.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:05.985 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:06.085 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:06.087 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:06 smithi016 bash[77995]: cluster 2023-09-23T19:52:04.447257+0000 mgr.y (mgr.25325) 1342 : cluster 0 pgmap v1345: 89 pgs: 89 active+clean; 1.1 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-09-23T19:52:06.087 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:06 smithi016 bash[75004]: cluster 2023-09-23T19:52:04.447257+0000 mgr.y (mgr.25325) 1342 : cluster 0 pgmap v1345: 89 pgs: 89 active+clean; 1.1 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-09-23T19:52:06.109 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:06.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:06.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:06.212 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:06.241 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:06.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:06.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:06.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:06 smithi092 bash[58855]: cluster 2023-09-23T19:52:04.447257+0000 mgr.y (mgr.25325) 1342 : cluster 0 pgmap v1345: 89 pgs: 89 active+clean; 1.1 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-09-23T19:52:06.345 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:06.372 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:06.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:06.374 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:06.475 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:06.476 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 14 16 28406 28390 126.729 104.188 0.00591605 0.00788365 2023-09-23T19:52:06.503 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:06.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:06.506 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:06.607 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:06.636 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:06.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:06.640 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:06.741 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:06.742 INFO:teuthology.orchestra.run.smithi016.stdout:{"pg_ready":true,"pg_map":{"version":1343,"stamp":"2023-09-23T19:52:00.445589+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1040783264,"num_objects":15896,"num_object_clones":0,"num_object_copies":47688,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15896,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":16133,"num_write_kb":1018254,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":16067,"ondisk_log_size":16067,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":8782780,"kb_used_data":2658492,"kb_used_omap":20,"kb_used_meta":6124203,"kb_avail":741178436,"statfs":{"total":767960285184,"available":758966718464,"internally_reserved":0,"allocated":2722295808,"data_stored":2716462227,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":6271184526},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1040187392,"num_objects":15872,"num_object_clones":0,"num_object_copies":47616,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15872,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15872,"num_write_kb":1015808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.931186"},"pg_stats":[{"pgid":"3.1a","version":"59'1","reported_seq":233,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:42.828141+0000","last_change":"2023-09-23T19:38:43.897548+0000","last_active":"2023-09-23T19:51:42.828141+0000","last_peered":"2023-09-23T19:51:42.828141+0000","last_clean":"2023-09-23T19:51:42.828141+0000","last_became_active":"2023-09-23T19:38:43.896868+0000","last_became_peered":"2023-09-23T19:38:43.896868+0000","last_unstale":"2023-09-23T19:51:42.828141+0000","last_undegraded":"2023-09-23T19:51:42.828141+0000","last_fullsized":"2023-09-23T19:51:42.828141+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":197,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:42.311439+0000","last_change":"2023-09-23T19:32:49.731128+0000","last_active":"2023-09-23T19:51:42.311439+0000","last_peered":"2023-09-23T19:51:42.311439+0000","last_clean":"2023-09-23T19:51:42.311439+0000","last_became_active":"2023-09-23T19:32:49.730757+0000","last_became_peered":"2023-09-23T19:32:49.730757+0000","last_unstale":"2023-09-23T19:51:42.311439+0000","last_undegraded":"2023-09-23T19:51:42.311439+0000","last_fullsized":"2023-09-23T19:51:42.311439+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:40.183605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":186,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:46.386995+0000","last_change":"2023-09-23T19:28:07.890462+0000","last_active":"2023-09-23T19:51:46.386995+0000","last_peered":"2023-09-23T19:51:46.386995+0000","last_clean":"2023-09-23T19:51:46.386995+0000","last_became_active":"2023-09-23T19:28:07.890084+0000","last_became_peered":"2023-09-23T19:28:07.890084+0000","last_unstale":"2023-09-23T19:51:46.386995+0000","last_undegraded":"2023-09-23T19:51:46.386995+0000","last_fullsized":"2023-09-23T19:51:46.386995+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:50.458218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":123,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:42.205159+0000","last_change":"2023-09-23T19:42:55.772143+0000","last_active":"2023-09-23T19:51:42.205159+0000","last_peered":"2023-09-23T19:51:42.205159+0000","last_clean":"2023-09-23T19:51:42.205159+0000","last_became_active":"2023-09-23T19:42:55.771813+0000","last_became_peered":"2023-09-23T19:42:55.771813+0000","last_unstale":"2023-09-23T19:51:42.205159+0000","last_undegraded":"2023-09-23T19:51:42.205159+0000","last_fullsized":"2023-09-23T19:51:42.205159+0000","mapping_epoch":188,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":214,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:42.307162+0000","last_change":"2023-09-23T19:32:49.729961+0000","last_active":"2023-09-23T19:51:42.307162+0000","last_peered":"2023-09-23T19:51:42.307162+0000","last_clean":"2023-09-23T19:51:42.307162+0000","last_became_active":"2023-09-23T19:32:49.729251+0000","last_became_peered":"2023-09-23T19:32:49.729251+0000","last_unstale":"2023-09-23T19:51:42.307162+0000","last_undegraded":"2023-09-23T19:51:42.307162+0000","last_fullsized":"2023-09-23T19:51:42.307162+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:20:56.595732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":202,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:42.555958+0000","last_change":"2023-09-23T19:22:42.150582+0000","last_active":"2023-09-23T19:51:42.555958+0000","last_peered":"2023-09-23T19:51:42.555958+0000","last_clean":"2023-09-23T19:51:42.555958+0000","last_became_active":"2023-09-23T19:22:42.150229+0000","last_became_peered":"2023-09-23T19:22:42.150229+0000","last_unstale":"2023-09-23T19:51:42.555958+0000","last_undegraded":"2023-09-23T19:51:42.555958+0000","last_fullsized":"2023-09-23T19:51:42.555958+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":181,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:43.873620+0000","last_change":"2023-09-23T19:47:56.793944+0000","last_active":"2023-09-23T19:51:43.873620+0000","last_peered":"2023-09-23T19:51:43.873620+0000","last_clean":"2023-09-23T19:51:43.873620+0000","last_became_active":"2023-09-23T19:47:56.793618+0000","last_became_peered":"2023-09-23T19:47:56.793618+0000","last_unstale":"2023-09-23T19:51:43.873620+0000","last_undegraded":"2023-09-23T19:51:43.873620+0000","last_fullsized":"2023-09-23T19:51:43.873620+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":198,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:42.311192+0000","last_change":"2023-09-23T19:32:49.729765+0000","last_active":"2023-09-23T19:51:42.311192+0000","last_peered":"2023-09-23T19:51:42.311192+0000","last_clean":"2023-09-23T19:51:42.311192+0000","last_became_active":"2023-09-23T19:32:49.729007+0000","last_became_peered":"2023-09-23T19:32:49.729007+0000","last_unstale":"2023-09-23T19:51:42.311192+0000","last_undegraded":"2023-09-23T19:51:42.311192+0000","last_fullsized":"2023-09-23T19:51:42.311192+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:59:31.183753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":189,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:43.873407+0000","last_change":"2023-09-23T19:47:56.791464+0000","last_active":"2023-09-23T19:51:43.873407+0000","last_peered":"2023-09-23T19:51:43.873407+0000","last_clean":"2023-09-23T19:51:43.873407+0000","last_became_active":"2023-09-23T19:47:56.791144+0000","last_became_peered":"2023-09-23T19:47:56.791144+0000","last_unstale":"2023-09-23T19:51:43.873407+0000","last_undegraded":"2023-09-23T19:51:43.873407+0000","last_fullsized":"2023-09-23T19:51:43.873407+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":177,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:42.204265+0000","last_change":"2023-09-23T19:32:49.736864+0000","last_active":"2023-09-23T19:51:42.204265+0000","last_peered":"2023-09-23T19:51:42.204265+0000","last_clean":"2023-09-23T19:51:42.204265+0000","last_became_active":"2023-09-23T19:32:49.736523+0000","last_became_peered":"2023-09-23T19:32:49.736523+0000","last_unstale":"2023-09-23T19:51:42.204265+0000","last_undegraded":"2023-09-23T19:51:42.204265+0000","last_fullsized":"2023-09-23T19:51:42.204265+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":206,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:42.205181+0000","last_change":"2023-09-23T19:28:07.892835+0000","last_active":"2023-09-23T19:51:42.205181+0000","last_peered":"2023-09-23T19:51:42.205181+0000","last_clean":"2023-09-23T19:51:42.205181+0000","last_became_active":"2023-09-23T19:28:07.892338+0000","last_became_peered":"2023-09-23T19:28:07.892338+0000","last_unstale":"2023-09-23T19:51:42.205181+0000","last_undegraded":"2023-09-23T19:51:42.205181+0000","last_fullsized":"2023-09-23T19:51:42.205181+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":201,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:42.311428+0000","last_change":"2023-09-23T19:38:43.890137+0000","last_active":"2023-09-23T19:51:42.311428+0000","last_peered":"2023-09-23T19:51:42.311428+0000","last_clean":"2023-09-23T19:51:42.311428+0000","last_became_active":"2023-09-23T19:38:43.889800+0000","last_became_peered":"2023-09-23T19:38:43.889800+0000","last_unstale":"2023-09-23T19:51:42.311428+0000","last_undegraded":"2023-09-23T19:51:42.311428+0000","last_fullsized":"2023-09-23T19:51:42.311428+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:59:20.468913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'23","reported_seq":210,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:42.828422+0000","last_change":"2023-09-23T19:05:52.107727+0000","last_active":"2023-09-23T19:51:42.828422+0000","last_peered":"2023-09-23T19:51:42.828422+0000","last_clean":"2023-09-23T19:51:42.828422+0000","last_became_active":"2023-09-23T19:05:52.107378+0000","last_became_peered":"2023-09-23T19:05:52.107378+0000","last_unstale":"2023-09-23T19:51:42.828422+0000","last_undegraded":"2023-09-23T19:51:42.828422+0000","last_fullsized":"2023-09-23T19:51:42.828422+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":201,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:42.828183+0000","last_change":"2023-09-23T19:22:42.148497+0000","last_active":"2023-09-23T19:51:42.828183+0000","last_peered":"2023-09-23T19:51:42.828183+0000","last_clean":"2023-09-23T19:51:42.828183+0000","last_became_active":"2023-09-23T19:22:42.148147+0000","last_became_peered":"2023-09-23T19:22:42.148147+0000","last_unstale":"2023-09-23T19:51:42.828183+0000","last_undegraded":"2023-09-23T19:51:42.828183+0000","last_fullsized":"2023-09-23T19:51:42.828183+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":220,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:42.204800+0000","last_change":"2023-09-23T19:38:43.897378+0000","last_active":"2023-09-23T19:51:42.204800+0000","last_peered":"2023-09-23T19:51:42.204800+0000","last_clean":"2023-09-23T19:51:42.204800+0000","last_became_active":"2023-09-23T19:38:43.897012+0000","last_became_peered":"2023-09-23T19:38:43.897012+0000","last_unstale":"2023-09-23T19:51:42.204800+0000","last_undegraded":"2023-09-23T19:51:42.204800+0000","last_fullsized":"2023-09-23T19:51:42.204800+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":198,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:42.311642+0000","last_change":"2023-09-23T19:32:49.730549+0000","last_active":"2023-09-23T19:51:42.311642+0000","last_peered":"2023-09-23T19:51:42.311642+0000","last_clean":"2023-09-23T19:51:42.311642+0000","last_became_active":"2023-09-23T19:32:49.730105+0000","last_became_peered":"2023-09-23T19:32:49.730105+0000","last_unstale":"2023-09-23T19:51:42.311642+0000","last_undegraded":"2023-09-23T19:51:42.311642+0000","last_fullsized":"2023-09-23T19:51:42.311642+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:49.751893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":123,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:42.204359+0000","last_change":"2023-09-23T19:42:55.772691+0000","last_active":"2023-09-23T19:51:42.204359+0000","last_peered":"2023-09-23T19:51:42.204359+0000","last_clean":"2023-09-23T19:51:42.204359+0000","last_became_active":"2023-09-23T19:42:55.772309+0000","last_became_peered":"2023-09-23T19:42:55.772309+0000","last_unstale":"2023-09-23T19:51:42.204359+0000","last_undegraded":"2023-09-23T19:51:42.204359+0000","last_fullsized":"2023-09-23T19:51:42.204359+0000","mapping_epoch":188,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":188,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:43.873436+0000","last_change":"2023-09-23T19:47:56.794658+0000","last_active":"2023-09-23T19:51:43.873436+0000","last_peered":"2023-09-23T19:51:43.873436+0000","last_clean":"2023-09-23T19:51:43.873436+0000","last_became_active":"2023-09-23T19:47:56.793633+0000","last_became_peered":"2023-09-23T19:47:56.793633+0000","last_unstale":"2023-09-23T19:51:43.873436+0000","last_undegraded":"2023-09-23T19:51:43.873436+0000","last_fullsized":"2023-09-23T19:51:43.873436+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":188,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:42.204408+0000","last_change":"2023-09-23T19:05:52.114186+0000","last_active":"2023-09-23T19:51:42.204408+0000","last_peered":"2023-09-23T19:51:42.204408+0000","last_clean":"2023-09-23T19:51:42.204408+0000","last_became_active":"2023-09-23T19:05:52.113635+0000","last_became_peered":"2023-09-23T19:05:52.113635+0000","last_unstale":"2023-09-23T19:51:42.204408+0000","last_undegraded":"2023-09-23T19:51:42.204408+0000","last_fullsized":"2023-09-23T19:51:42.204408+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":123,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:42.205127+0000","last_change":"2023-09-23T19:42:55.772912+0000","last_active":"2023-09-23T19:51:42.205127+0000","last_peered":"2023-09-23T19:51:42.205127+0000","last_clean":"2023-09-23T19:51:42.205127+0000","last_became_active":"2023-09-23T19:42:55.772434+0000","last_became_peered":"2023-09-23T19:42:55.772434+0000","last_unstale":"2023-09-23T19:51:42.205127+0000","last_undegraded":"2023-09-23T19:51:42.205127+0000","last_fullsized":"2023-09-23T19:51:42.205127+0000","mapping_epoch":188,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":211,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:42.554906+0000","last_change":"2023-09-23T19:18:09.010195+0000","last_active":"2023-09-23T19:51:42.554906+0000","last_peered":"2023-09-23T19:51:42.554906+0000","last_clean":"2023-09-23T19:51:42.554906+0000","last_became_active":"2023-09-23T19:18:09.009687+0000","last_became_peered":"2023-09-23T19:18:09.009687+0000","last_unstale":"2023-09-23T19:51:42.554906+0000","last_undegraded":"2023-09-23T19:51:42.554906+0000","last_fullsized":"2023-09-23T19:51:42.554906+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":206,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:41.289487+0000","last_change":"2023-09-23T19:28:07.885265+0000","last_active":"2023-09-23T19:51:41.289487+0000","last_peered":"2023-09-23T19:51:41.289487+0000","last_clean":"2023-09-23T19:51:41.289487+0000","last_became_active":"2023-09-23T19:28:07.884353+0000","last_became_peered":"2023-09-23T19:28:07.884353+0000","last_unstale":"2023-09-23T19:51:41.289487+0000","last_undegraded":"2023-09-23T19:51:41.289487+0000","last_fullsized":"2023-09-23T19:51:41.289487+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:56.325352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.1","version":"237'1089","reported_seq":1102,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:59.920507+0000","last_change":"2023-09-23T19:51:24.780294+0000","last_active":"2023-09-23T19:51:59.920507+0000","last_peered":"2023-09-23T19:51:59.920507+0000","last_clean":"2023-09-23T19:51:59.920507+0000","last_became_active":"2023-09-23T19:51:24.779889+0000","last_became_peered":"2023-09-23T19:51:24.779889+0000","last_unstale":"2023-09-23T19:51:59.920507+0000","last_undegraded":"2023-09-23T19:51:59.920507+0000","last_fullsized":"2023-09-23T19:51:59.920507+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":232,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_clean_scrub_stamp":"2023-09-23T19:51:23.737731+0000","objects_scrubbed":0,"log_size":1089,"log_dups_size":0,"ondisk_log_size":1089,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71368704,"num_objects":1089,"num_object_clones":0,"num_object_copies":3267,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1089,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1089,"num_write_kb":69696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":204,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:42.205499+0000","last_change":"2023-09-23T19:18:09.035116+0000","last_active":"2023-09-23T19:51:42.205499+0000","last_peered":"2023-09-23T19:51:42.205499+0000","last_clean":"2023-09-23T19:51:42.205499+0000","last_became_active":"2023-09-23T19:18:09.034416+0000","last_became_peered":"2023-09-23T19:18:09.034416+0000","last_unstale":"2023-09-23T19:51:42.205499+0000","last_undegraded":"2023-09-23T19:51:42.205499+0000","last_fullsized":"2023-09-23T19:51:42.205499+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":173,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:43.872799+0000","last_change":"2023-09-23T19:47:56.790227+0000","last_active":"2023-09-23T19:51:43.872799+0000","last_peered":"2023-09-23T19:51:43.872799+0000","last_clean":"2023-09-23T19:51:43.872799+0000","last_became_active":"2023-09-23T19:47:56.789727+0000","last_became_peered":"2023-09-23T19:47:56.789727+0000","last_unstale":"2023-09-23T19:51:43.872799+0000","last_undegraded":"2023-09-23T19:51:43.872799+0000","last_fullsized":"2023-09-23T19:51:43.872799+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"237'1131","reported_seq":1144,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:59.894969+0000","last_change":"2023-09-23T19:51:24.781980+0000","last_active":"2023-09-23T19:51:59.894969+0000","last_peered":"2023-09-23T19:51:59.894969+0000","last_clean":"2023-09-23T19:51:59.894969+0000","last_became_active":"2023-09-23T19:51:24.781639+0000","last_became_peered":"2023-09-23T19:51:24.781639+0000","last_unstale":"2023-09-23T19:51:59.894969+0000","last_undegraded":"2023-09-23T19:51:59.894969+0000","last_fullsized":"2023-09-23T19:51:59.894969+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":232,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_clean_scrub_stamp":"2023-09-23T19:51:23.737731+0000","objects_scrubbed":0,"log_size":1131,"log_dups_size":0,"ondisk_log_size":1131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74121216,"num_objects":1131,"num_object_clones":0,"num_object_copies":3393,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1131,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1131,"num_write_kb":72384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":205,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:42.205338+0000","last_change":"2023-09-23T19:22:42.151134+0000","last_active":"2023-09-23T19:51:42.205338+0000","last_peered":"2023-09-23T19:51:42.205338+0000","last_clean":"2023-09-23T19:51:42.205338+0000","last_became_active":"2023-09-23T19:22:42.150553+0000","last_became_peered":"2023-09-23T19:22:42.150553+0000","last_unstale":"2023-09-23T19:51:42.205338+0000","last_undegraded":"2023-09-23T19:51:42.205338+0000","last_fullsized":"2023-09-23T19:51:42.205338+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":216,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:42.827987+0000","last_change":"2023-09-23T19:28:07.899262+0000","last_active":"2023-09-23T19:51:42.827987+0000","last_peered":"2023-09-23T19:51:42.827987+0000","last_clean":"2023-09-23T19:51:42.827987+0000","last_became_active":"2023-09-23T19:28:07.898848+0000","last_became_peered":"2023-09-23T19:28:07.898848+0000","last_unstale":"2023-09-23T19:51:42.827987+0000","last_undegraded":"2023-09-23T19:51:42.827987+0000","last_fullsized":"2023-09-23T19:51:42.827987+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.3","version":"237'807","reported_seq":821,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:57.521702+0000","last_change":"2023-09-23T19:51:24.773320+0000","last_active":"2023-09-23T19:51:57.521702+0000","last_peered":"2023-09-23T19:51:57.521702+0000","last_clean":"2023-09-23T19:51:57.521702+0000","last_became_active":"2023-09-23T19:51:24.772965+0000","last_became_peered":"2023-09-23T19:51:24.772965+0000","last_unstale":"2023-09-23T19:51:57.521702+0000","last_undegraded":"2023-09-23T19:51:57.521702+0000","last_fullsized":"2023-09-23T19:51:57.521702+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":232,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_clean_scrub_stamp":"2023-09-23T19:51:23.737731+0000","objects_scrubbed":0,"log_size":807,"log_dups_size":0,"ondisk_log_size":807,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:07.426463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52887552,"num_objects":807,"num_object_clones":0,"num_object_copies":2421,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":807,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":807,"num_write_kb":51648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":189,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:42.204875+0000","last_change":"2023-09-23T19:05:52.114108+0000","last_active":"2023-09-23T19:51:42.204875+0000","last_peered":"2023-09-23T19:51:42.204875+0000","last_clean":"2023-09-23T19:51:42.204875+0000","last_became_active":"2023-09-23T19:05:52.113609+0000","last_became_peered":"2023-09-23T19:05:52.113609+0000","last_unstale":"2023-09-23T19:51:42.204875+0000","last_undegraded":"2023-09-23T19:51:42.204875+0000","last_fullsized":"2023-09-23T19:51:42.204875+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":202,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:42.555240+0000","last_change":"2023-09-23T19:22:42.156825+0000","last_active":"2023-09-23T19:51:42.555240+0000","last_peered":"2023-09-23T19:51:42.555240+0000","last_clean":"2023-09-23T19:51:42.555240+0000","last_became_active":"2023-09-23T19:22:42.156474+0000","last_became_peered":"2023-09-23T19:22:42.156474+0000","last_unstale":"2023-09-23T19:51:42.555240+0000","last_undegraded":"2023-09-23T19:51:42.555240+0000","last_fullsized":"2023-09-23T19:51:42.555240+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.2","version":"237'666","reported_seq":680,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:56.708335+0000","last_change":"2023-09-23T19:51:24.779170+0000","last_active":"2023-09-23T19:51:56.708335+0000","last_peered":"2023-09-23T19:51:56.708335+0000","last_clean":"2023-09-23T19:51:56.708335+0000","last_became_active":"2023-09-23T19:51:24.778769+0000","last_became_peered":"2023-09-23T19:51:24.778769+0000","last_unstale":"2023-09-23T19:51:56.708335+0000","last_undegraded":"2023-09-23T19:51:56.708335+0000","last_fullsized":"2023-09-23T19:51:56.708335+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":232,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_clean_scrub_stamp":"2023-09-23T19:51:23.737731+0000","objects_scrubbed":0,"log_size":666,"log_dups_size":0,"ondisk_log_size":666,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:12:30.926023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43646976,"num_objects":666,"num_object_clones":0,"num_object_copies":1998,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":666,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":666,"num_write_kb":42624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":189,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:43.873269+0000","last_change":"2023-09-23T19:47:56.794638+0000","last_active":"2023-09-23T19:51:43.873269+0000","last_peered":"2023-09-23T19:51:43.873269+0000","last_clean":"2023-09-23T19:51:43.873269+0000","last_became_active":"2023-09-23T19:47:56.794305+0000","last_became_peered":"2023-09-23T19:47:56.794305+0000","last_unstale":"2023-09-23T19:51:43.873269+0000","last_undegraded":"2023-09-23T19:51:43.873269+0000","last_fullsized":"2023-09-23T19:51:43.873269+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":203,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:49.809458+0000","last_change":"2023-09-23T19:22:42.135572+0000","last_active":"2023-09-23T19:51:49.809458+0000","last_peered":"2023-09-23T19:51:49.809458+0000","last_clean":"2023-09-23T19:51:49.809458+0000","last_became_active":"2023-09-23T19:22:42.135237+0000","last_became_peered":"2023-09-23T19:22:42.135237+0000","last_unstale":"2023-09-23T19:51:49.809458+0000","last_undegraded":"2023-09-23T19:51:49.809458+0000","last_fullsized":"2023-09-23T19:51:49.809458+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:35.838718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.5","version":"237'1073","reported_seq":1086,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:59.725379+0000","last_change":"2023-09-23T19:51:24.784350+0000","last_active":"2023-09-23T19:51:59.725379+0000","last_peered":"2023-09-23T19:51:59.725379+0000","last_clean":"2023-09-23T19:51:59.725379+0000","last_became_active":"2023-09-23T19:51:24.783918+0000","last_became_peered":"2023-09-23T19:51:24.783918+0000","last_unstale":"2023-09-23T19:51:59.725379+0000","last_undegraded":"2023-09-23T19:51:59.725379+0000","last_fullsized":"2023-09-23T19:51:59.725379+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":232,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_clean_scrub_stamp":"2023-09-23T19:51:23.737731+0000","objects_scrubbed":0,"log_size":1073,"log_dups_size":0,"ondisk_log_size":1073,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70320128,"num_objects":1073,"num_object_clones":0,"num_object_copies":3219,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1073,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1073,"num_write_kb":68672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":70,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:42.828148+0000","last_change":"2023-09-23T19:42:55.771907+0000","last_active":"2023-09-23T19:51:42.828148+0000","last_peered":"2023-09-23T19:51:42.828148+0000","last_clean":"2023-09-23T19:51:42.828148+0000","last_became_active":"2023-09-23T19:42:55.771503+0000","last_became_peered":"2023-09-23T19:42:55.771503+0000","last_unstale":"2023-09-23T19:51:42.828148+0000","last_undegraded":"2023-09-23T19:51:42.828148+0000","last_fullsized":"2023-09-23T19:51:42.828148+0000","mapping_epoch":188,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":201,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:42.828566+0000","last_change":"2023-09-23T19:28:07.896803+0000","last_active":"2023-09-23T19:51:42.828566+0000","last_peered":"2023-09-23T19:51:42.828566+0000","last_clean":"2023-09-23T19:51:42.828566+0000","last_became_active":"2023-09-23T19:28:07.896480+0000","last_became_peered":"2023-09-23T19:28:07.896480+0000","last_unstale":"2023-09-23T19:51:42.828566+0000","last_undegraded":"2023-09-23T19:51:42.828566+0000","last_fullsized":"2023-09-23T19:51:42.828566+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.4","version":"237'1084","reported_seq":1097,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:59.922476+0000","last_change":"2023-09-23T19:51:24.775881+0000","last_active":"2023-09-23T19:51:59.922476+0000","last_peered":"2023-09-23T19:51:59.922476+0000","last_clean":"2023-09-23T19:51:59.922476+0000","last_became_active":"2023-09-23T19:51:24.775309+0000","last_became_peered":"2023-09-23T19:51:24.775309+0000","last_unstale":"2023-09-23T19:51:59.922476+0000","last_undegraded":"2023-09-23T19:51:59.922476+0000","last_fullsized":"2023-09-23T19:51:59.922476+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":232,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_clean_scrub_stamp":"2023-09-23T19:51:23.737731+0000","objects_scrubbed":0,"log_size":1084,"log_dups_size":0,"ondisk_log_size":1084,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71041024,"num_objects":1084,"num_object_clones":0,"num_object_copies":3252,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1084,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1084,"num_write_kb":69376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":203,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:42.555288+0000","last_change":"2023-09-23T19:22:42.156582+0000","last_active":"2023-09-23T19:51:42.555288+0000","last_peered":"2023-09-23T19:51:42.555288+0000","last_clean":"2023-09-23T19:51:42.555288+0000","last_became_active":"2023-09-23T19:22:42.156251+0000","last_became_peered":"2023-09-23T19:22:42.156251+0000","last_unstale":"2023-09-23T19:51:42.555288+0000","last_undegraded":"2023-09-23T19:51:42.555288+0000","last_fullsized":"2023-09-23T19:51:42.555288+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":231,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:42.828571+0000","last_change":"2023-09-23T19:38:43.891593+0000","last_active":"2023-09-23T19:51:42.828571+0000","last_peered":"2023-09-23T19:51:42.828571+0000","last_clean":"2023-09-23T19:51:42.828571+0000","last_became_active":"2023-09-23T19:38:43.891063+0000","last_became_peered":"2023-09-23T19:38:43.891063+0000","last_unstale":"2023-09-23T19:51:42.828571+0000","last_undegraded":"2023-09-23T19:51:42.828571+0000","last_fullsized":"2023-09-23T19:51:42.828571+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.7","version":"237'997","reported_seq":1010,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:59.865673+0000","last_change":"2023-09-23T19:51:24.784793+0000","last_active":"2023-09-23T19:51:59.865673+0000","last_peered":"2023-09-23T19:51:59.865673+0000","last_clean":"2023-09-23T19:51:59.865673+0000","last_became_active":"2023-09-23T19:51:24.784503+0000","last_became_peered":"2023-09-23T19:51:24.784503+0000","last_unstale":"2023-09-23T19:51:59.865673+0000","last_undegraded":"2023-09-23T19:51:59.865673+0000","last_fullsized":"2023-09-23T19:51:59.865673+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":232,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_clean_scrub_stamp":"2023-09-23T19:51:23.737731+0000","objects_scrubbed":0,"log_size":997,"log_dups_size":0,"ondisk_log_size":997,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65339392,"num_objects":997,"num_object_clones":0,"num_object_copies":2991,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":997,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":997,"num_write_kb":63808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":217,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:42.827814+0000","last_change":"2023-09-23T19:38:43.892972+0000","last_active":"2023-09-23T19:51:42.827814+0000","last_peered":"2023-09-23T19:51:42.827814+0000","last_clean":"2023-09-23T19:51:42.827814+0000","last_became_active":"2023-09-23T19:38:43.892451+0000","last_became_peered":"2023-09-23T19:38:43.892451+0000","last_unstale":"2023-09-23T19:51:42.827814+0000","last_undegraded":"2023-09-23T19:51:42.827814+0000","last_fullsized":"2023-09-23T19:51:42.827814+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":201,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:42.828865+0000","last_change":"2023-09-23T19:22:42.149670+0000","last_active":"2023-09-23T19:51:42.828865+0000","last_peered":"2023-09-23T19:51:42.828865+0000","last_clean":"2023-09-23T19:51:42.828865+0000","last_became_active":"2023-09-23T19:22:42.149022+0000","last_became_peered":"2023-09-23T19:22:42.149022+0000","last_unstale":"2023-09-23T19:51:42.828865+0000","last_undegraded":"2023-09-23T19:51:42.828865+0000","last_fullsized":"2023-09-23T19:51:42.828865+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.6","version":"237'1081","reported_seq":1093,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:59.725403+0000","last_change":"2023-09-23T19:51:24.783467+0000","last_active":"2023-09-23T19:51:59.725403+0000","last_peered":"2023-09-23T19:51:59.725403+0000","last_clean":"2023-09-23T19:51:59.725403+0000","last_became_active":"2023-09-23T19:51:24.783123+0000","last_became_peered":"2023-09-23T19:51:24.783123+0000","last_unstale":"2023-09-23T19:51:59.725403+0000","last_undegraded":"2023-09-23T19:51:59.725403+0000","last_fullsized":"2023-09-23T19:51:59.725403+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":232,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_clean_scrub_stamp":"2023-09-23T19:51:23.737731+0000","objects_scrubbed":0,"log_size":1081,"log_dups_size":0,"ondisk_log_size":1081,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70844416,"num_objects":1081,"num_object_clones":0,"num_object_copies":3243,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1081,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1081,"num_write_kb":69184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":199,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:42.310997+0000","last_change":"2023-09-23T19:32:49.729244+0000","last_active":"2023-09-23T19:51:42.310997+0000","last_peered":"2023-09-23T19:51:42.310997+0000","last_clean":"2023-09-23T19:51:42.310997+0000","last_became_active":"2023-09-23T19:32:49.728810+0000","last_became_peered":"2023-09-23T19:32:49.728810+0000","last_unstale":"2023-09-23T19:51:42.310997+0000","last_undegraded":"2023-09-23T19:51:42.310997+0000","last_fullsized":"2023-09-23T19:51:42.310997+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:57.540826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":186,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:42.311703+0000","last_change":"2023-09-23T19:28:07.889095+0000","last_active":"2023-09-23T19:51:42.311703+0000","last_peered":"2023-09-23T19:51:42.311703+0000","last_clean":"2023-09-23T19:51:42.311703+0000","last_became_active":"2023-09-23T19:28:07.888742+0000","last_became_peered":"2023-09-23T19:28:07.888742+0000","last_unstale":"2023-09-23T19:51:42.311703+0000","last_undegraded":"2023-09-23T19:51:42.311703+0000","last_fullsized":"2023-09-23T19:51:42.311703+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:43.929488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.b","version":"237'1115","reported_seq":1128,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:59.687742+0000","last_change":"2023-09-23T19:51:24.785370+0000","last_active":"2023-09-23T19:51:59.687742+0000","last_peered":"2023-09-23T19:51:59.687742+0000","last_clean":"2023-09-23T19:51:59.687742+0000","last_became_active":"2023-09-23T19:51:24.784883+0000","last_became_peered":"2023-09-23T19:51:24.784883+0000","last_unstale":"2023-09-23T19:51:59.687742+0000","last_undegraded":"2023-09-23T19:51:59.687742+0000","last_fullsized":"2023-09-23T19:51:59.687742+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":232,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_clean_scrub_stamp":"2023-09-23T19:51:23.737731+0000","objects_scrubbed":0,"log_size":1115,"log_dups_size":0,"ondisk_log_size":1115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73072640,"num_objects":1115,"num_object_clones":0,"num_object_copies":3345,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1115,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1115,"num_write_kb":71360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":221,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:49.517628+0000","last_change":"2023-09-23T19:22:42.143289+0000","last_active":"2023-09-23T19:51:49.517628+0000","last_peered":"2023-09-23T19:51:49.517628+0000","last_clean":"2023-09-23T19:51:49.517628+0000","last_became_active":"2023-09-23T19:22:42.142656+0000","last_became_peered":"2023-09-23T19:22:42.142656+0000","last_unstale":"2023-09-23T19:51:49.517628+0000","last_undegraded":"2023-09-23T19:51:49.517628+0000","last_fullsized":"2023-09-23T19:51:49.517628+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:38.467441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":219,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:43.873462+0000","last_change":"2023-09-23T19:47:56.794437+0000","last_active":"2023-09-23T19:51:43.873462+0000","last_peered":"2023-09-23T19:51:43.873462+0000","last_clean":"2023-09-23T19:51:43.873462+0000","last_became_active":"2023-09-23T19:47:56.794118+0000","last_became_peered":"2023-09-23T19:47:56.794118+0000","last_unstale":"2023-09-23T19:51:43.873462+0000","last_undegraded":"2023-09-23T19:51:43.873462+0000","last_fullsized":"2023-09-23T19:51:43.873462+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":206,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:41.286684+0000","last_change":"2023-09-23T19:38:43.898519+0000","last_active":"2023-09-23T19:51:41.286684+0000","last_peered":"2023-09-23T19:51:41.286684+0000","last_clean":"2023-09-23T19:51:41.286684+0000","last_became_active":"2023-09-23T19:38:43.896814+0000","last_became_peered":"2023-09-23T19:38:43.896814+0000","last_unstale":"2023-09-23T19:51:41.286684+0000","last_undegraded":"2023-09-23T19:51:41.286684+0000","last_fullsized":"2023-09-23T19:51:41.286684+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:40.465038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.d","version":"237'1050","reported_seq":1063,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:59.708529+0000","last_change":"2023-09-23T19:51:24.782799+0000","last_active":"2023-09-23T19:51:59.708529+0000","last_peered":"2023-09-23T19:51:59.708529+0000","last_clean":"2023-09-23T19:51:59.708529+0000","last_became_active":"2023-09-23T19:51:24.782419+0000","last_became_peered":"2023-09-23T19:51:24.782419+0000","last_unstale":"2023-09-23T19:51:59.708529+0000","last_undegraded":"2023-09-23T19:51:59.708529+0000","last_fullsized":"2023-09-23T19:51:59.708529+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":232,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_clean_scrub_stamp":"2023-09-23T19:51:23.737731+0000","objects_scrubbed":0,"log_size":1050,"log_dups_size":0,"ondisk_log_size":1050,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68812800,"num_objects":1050,"num_object_clones":0,"num_object_copies":3150,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1050,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1050,"num_write_kb":67200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":120,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:42.311194+0000","last_change":"2023-09-23T19:43:38.612700+0000","last_active":"2023-09-23T19:51:42.311194+0000","last_peered":"2023-09-23T19:51:42.311194+0000","last_clean":"2023-09-23T19:51:42.311194+0000","last_became_active":"2023-09-23T19:43:38.612424+0000","last_became_peered":"2023-09-23T19:43:38.612424+0000","last_unstale":"2023-09-23T19:51:42.311194+0000","last_undegraded":"2023-09-23T19:51:42.311194+0000","last_fullsized":"2023-09-23T19:51:42.311194+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:24.833490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":202,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:42.828093+0000","last_change":"2023-09-23T19:18:09.017525+0000","last_active":"2023-09-23T19:51:42.828093+0000","last_peered":"2023-09-23T19:51:42.828093+0000","last_clean":"2023-09-23T19:51:42.828093+0000","last_became_active":"2023-09-23T19:18:09.017193+0000","last_became_peered":"2023-09-23T19:18:09.017193+0000","last_unstale":"2023-09-23T19:51:42.828093+0000","last_undegraded":"2023-09-23T19:51:42.828093+0000","last_fullsized":"2023-09-23T19:51:42.828093+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":218,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:49.517516+0000","last_change":"2023-09-23T19:38:43.877088+0000","last_active":"2023-09-23T19:51:49.517516+0000","last_peered":"2023-09-23T19:51:49.517516+0000","last_clean":"2023-09-23T19:51:49.517516+0000","last_became_active":"2023-09-23T19:38:43.876719+0000","last_became_peered":"2023-09-23T19:38:43.876719+0000","last_unstale":"2023-09-23T19:51:49.517516+0000","last_undegraded":"2023-09-23T19:51:49.517516+0000","last_fullsized":"2023-09-23T19:51:49.517516+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:05.423256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.e","version":"237'1034","reported_seq":1047,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:59.880173+0000","last_change":"2023-09-23T19:51:24.785601+0000","last_active":"2023-09-23T19:51:59.880173+0000","last_peered":"2023-09-23T19:51:59.880173+0000","last_clean":"2023-09-23T19:51:59.880173+0000","last_became_active":"2023-09-23T19:51:24.785163+0000","last_became_peered":"2023-09-23T19:51:24.785163+0000","last_unstale":"2023-09-23T19:51:59.880173+0000","last_undegraded":"2023-09-23T19:51:59.880173+0000","last_fullsized":"2023-09-23T19:51:59.880173+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":232,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_clean_scrub_stamp":"2023-09-23T19:51:23.737731+0000","objects_scrubbed":0,"log_size":1034,"log_dups_size":0,"ondisk_log_size":1034,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67764224,"num_objects":1034,"num_object_clones":0,"num_object_copies":3102,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1034,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1034,"num_write_kb":66176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":186,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:41.287435+0000","last_change":"2023-09-23T19:38:43.883663+0000","last_active":"2023-09-23T19:51:41.287435+0000","last_peered":"2023-09-23T19:51:41.287435+0000","last_clean":"2023-09-23T19:51:41.287435+0000","last_became_active":"2023-09-23T19:38:43.882421+0000","last_became_peered":"2023-09-23T19:38:43.882421+0000","last_unstale":"2023-09-23T19:51:41.287435+0000","last_undegraded":"2023-09-23T19:51:41.287435+0000","last_fullsized":"2023-09-23T19:51:41.287435+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:49:01.607659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":203,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:42.310994+0000","last_change":"2023-09-23T19:38:43.890600+0000","last_active":"2023-09-23T19:51:42.310994+0000","last_peered":"2023-09-23T19:51:42.310994+0000","last_clean":"2023-09-23T19:51:42.310994+0000","last_became_active":"2023-09-23T19:38:43.890253+0000","last_became_peered":"2023-09-23T19:38:43.890253+0000","last_unstale":"2023-09-23T19:51:42.310994+0000","last_undegraded":"2023-09-23T19:51:42.310994+0000","last_fullsized":"2023-09-23T19:51:42.310994+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:46.751120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":215,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:42.311665+0000","last_change":"2023-09-23T19:22:42.143733+0000","last_active":"2023-09-23T19:51:42.311665+0000","last_peered":"2023-09-23T19:51:42.311665+0000","last_clean":"2023-09-23T19:51:42.311665+0000","last_became_active":"2023-09-23T19:22:42.143077+0000","last_became_peered":"2023-09-23T19:22:42.143077+0000","last_unstale":"2023-09-23T19:51:42.311665+0000","last_undegraded":"2023-09-23T19:51:42.311665+0000","last_fullsized":"2023-09-23T19:51:42.311665+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:47:33.086994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.c","version":"237'614","reported_seq":628,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:56.378273+0000","last_change":"2023-09-23T19:51:24.778822+0000","last_active":"2023-09-23T19:51:56.378273+0000","last_peered":"2023-09-23T19:51:56.378273+0000","last_clean":"2023-09-23T19:51:56.378273+0000","last_became_active":"2023-09-23T19:51:24.778464+0000","last_became_peered":"2023-09-23T19:51:24.778464+0000","last_unstale":"2023-09-23T19:51:56.378273+0000","last_undegraded":"2023-09-23T19:51:56.378273+0000","last_fullsized":"2023-09-23T19:51:56.378273+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":232,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_clean_scrub_stamp":"2023-09-23T19:51:23.737731+0000","objects_scrubbed":0,"log_size":614,"log_dups_size":0,"ondisk_log_size":614,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:47.174791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40239104,"num_objects":614,"num_object_clones":0,"num_object_copies":1842,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":614,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":614,"num_write_kb":39296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":208,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:43.872965+0000","last_change":"2023-09-23T19:47:56.791259+0000","last_active":"2023-09-23T19:51:43.872965+0000","last_peered":"2023-09-23T19:51:43.872965+0000","last_clean":"2023-09-23T19:51:43.872965+0000","last_became_active":"2023-09-23T19:47:56.790954+0000","last_became_peered":"2023-09-23T19:47:56.790954+0000","last_unstale":"2023-09-23T19:51:43.872965+0000","last_undegraded":"2023-09-23T19:51:43.872965+0000","last_fullsized":"2023-09-23T19:51:43.872965+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":439,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:42.204498+0000","last_change":"2023-09-23T19:18:09.016297+0000","last_active":"2023-09-23T19:51:42.204498+0000","last_peered":"2023-09-23T19:51:42.204498+0000","last_clean":"2023-09-23T19:51:42.204498+0000","last_became_active":"2023-09-23T19:18:09.015436+0000","last_became_peered":"2023-09-23T19:18:09.015436+0000","last_unstale":"2023-09-23T19:51:42.204498+0000","last_undegraded":"2023-09-23T19:51:42.204498+0000","last_fullsized":"2023-09-23T19:51:42.204498+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:59:33.840751+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:59:33.840751+0000","last_clean_scrub_stamp":"2023-09-23T18:59:33.840751+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":162,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:43.872889+0000","last_change":"2023-09-23T19:47:56.811861+0000","last_active":"2023-09-23T19:51:43.872889+0000","last_peered":"2023-09-23T19:51:43.872889+0000","last_clean":"2023-09-23T19:51:43.872889+0000","last_became_active":"2023-09-23T19:47:56.811527+0000","last_became_peered":"2023-09-23T19:47:56.811527+0000","last_unstale":"2023-09-23T19:51:43.872889+0000","last_undegraded":"2023-09-23T19:51:43.872889+0000","last_fullsized":"2023-09-23T19:51:43.872889+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":187,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:42.555781+0000","last_change":"2023-09-23T19:05:52.786771+0000","last_active":"2023-09-23T19:51:42.555781+0000","last_peered":"2023-09-23T19:51:42.555781+0000","last_clean":"2023-09-23T19:51:42.555781+0000","last_became_active":"2023-09-23T19:05:52.785797+0000","last_became_peered":"2023-09-23T19:05:52.785797+0000","last_unstale":"2023-09-23T19:51:42.555781+0000","last_undegraded":"2023-09-23T19:51:42.555781+0000","last_fullsized":"2023-09-23T19:51:42.555781+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.f","version":"237'1111","reported_seq":1124,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:59.909038+0000","last_change":"2023-09-23T19:51:24.783251+0000","last_active":"2023-09-23T19:51:59.909038+0000","last_peered":"2023-09-23T19:51:59.909038+0000","last_clean":"2023-09-23T19:51:59.909038+0000","last_became_active":"2023-09-23T19:51:24.783051+0000","last_became_peered":"2023-09-23T19:51:24.783051+0000","last_unstale":"2023-09-23T19:51:59.909038+0000","last_undegraded":"2023-09-23T19:51:59.909038+0000","last_fullsized":"2023-09-23T19:51:59.909038+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":232,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_clean_scrub_stamp":"2023-09-23T19:51:23.737731+0000","objects_scrubbed":0,"log_size":1111,"log_dups_size":0,"ondisk_log_size":1111,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72810496,"num_objects":1111,"num_object_clones":0,"num_object_copies":3333,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1111,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1111,"num_write_kb":71104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":222,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:42.204254+0000","last_change":"2023-09-23T19:22:42.150636+0000","last_active":"2023-09-23T19:51:42.204254+0000","last_peered":"2023-09-23T19:51:42.204254+0000","last_clean":"2023-09-23T19:51:42.204254+0000","last_became_active":"2023-09-23T19:22:42.150045+0000","last_became_peered":"2023-09-23T19:22:42.150045+0000","last_unstale":"2023-09-23T19:51:42.204254+0000","last_undegraded":"2023-09-23T19:51:42.204254+0000","last_fullsized":"2023-09-23T19:51:42.204254+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":123,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:42.828051+0000","last_change":"2023-09-23T19:42:55.771906+0000","last_active":"2023-09-23T19:51:42.828051+0000","last_peered":"2023-09-23T19:51:42.828051+0000","last_clean":"2023-09-23T19:51:42.828051+0000","last_became_active":"2023-09-23T19:42:55.771599+0000","last_became_peered":"2023-09-23T19:42:55.771599+0000","last_unstale":"2023-09-23T19:51:42.828051+0000","last_undegraded":"2023-09-23T19:51:42.828051+0000","last_fullsized":"2023-09-23T19:51:42.828051+0000","mapping_epoch":188,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":233,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:42.307693+0000","last_change":"2023-09-23T19:38:43.876930+0000","last_active":"2023-09-23T19:51:42.307693+0000","last_peered":"2023-09-23T19:51:42.307693+0000","last_clean":"2023-09-23T19:51:42.307693+0000","last_became_active":"2023-09-23T19:38:43.876571+0000","last_became_peered":"2023-09-23T19:38:43.876571+0000","last_unstale":"2023-09-23T19:51:42.307693+0000","last_undegraded":"2023-09-23T19:51:42.307693+0000","last_fullsized":"2023-09-23T19:51:42.307693+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:30.924536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.a","version":"237'971","reported_seq":985,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:59.131378+0000","last_change":"2023-09-23T19:51:24.773510+0000","last_active":"2023-09-23T19:51:59.131378+0000","last_peered":"2023-09-23T19:51:59.131378+0000","last_clean":"2023-09-23T19:51:59.131378+0000","last_became_active":"2023-09-23T19:51:24.773049+0000","last_became_peered":"2023-09-23T19:51:24.773049+0000","last_unstale":"2023-09-23T19:51:59.131378+0000","last_undegraded":"2023-09-23T19:51:59.131378+0000","last_fullsized":"2023-09-23T19:51:59.131378+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":232,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_clean_scrub_stamp":"2023-09-23T19:51:23.737731+0000","objects_scrubbed":0,"log_size":971,"log_dups_size":0,"ondisk_log_size":971,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:19:22.115341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63635456,"num_objects":971,"num_object_clones":0,"num_object_copies":2913,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":971,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":971,"num_write_kb":62144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":207,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:42.204878+0000","last_change":"2023-09-23T19:18:09.015853+0000","last_active":"2023-09-23T19:51:42.204878+0000","last_peered":"2023-09-23T19:51:42.204878+0000","last_clean":"2023-09-23T19:51:42.204878+0000","last_became_active":"2023-09-23T19:18:09.015338+0000","last_became_peered":"2023-09-23T19:18:09.015338+0000","last_unstale":"2023-09-23T19:51:42.204878+0000","last_undegraded":"2023-09-23T19:51:42.204878+0000","last_fullsized":"2023-09-23T19:51:42.204878+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":203,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:49.517507+0000","last_change":"2023-09-23T19:38:43.881233+0000","last_active":"2023-09-23T19:51:49.517507+0000","last_peered":"2023-09-23T19:51:49.517507+0000","last_clean":"2023-09-23T19:51:49.517507+0000","last_became_active":"2023-09-23T19:38:43.880893+0000","last_became_peered":"2023-09-23T19:38:43.880893+0000","last_unstale":"2023-09-23T19:51:49.517507+0000","last_undegraded":"2023-09-23T19:51:49.517507+0000","last_fullsized":"2023-09-23T19:51:49.517507+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:11.206947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":58,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:43.873254+0000","last_change":"2023-09-23T19:47:56.790340+0000","last_active":"2023-09-23T19:51:43.873254+0000","last_peered":"2023-09-23T19:51:43.873254+0000","last_clean":"2023-09-23T19:51:43.873254+0000","last_became_active":"2023-09-23T19:47:56.789785+0000","last_became_peered":"2023-09-23T19:47:56.789785+0000","last_unstale":"2023-09-23T19:51:43.873254+0000","last_undegraded":"2023-09-23T19:51:43.873254+0000","last_fullsized":"2023-09-23T19:51:43.873254+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.9","version":"237'1066","reported_seq":1078,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:59.703786+0000","last_change":"2023-09-23T19:51:24.785521+0000","last_active":"2023-09-23T19:51:59.703786+0000","last_peered":"2023-09-23T19:51:59.703786+0000","last_clean":"2023-09-23T19:51:59.703786+0000","last_became_active":"2023-09-23T19:51:24.784935+0000","last_became_peered":"2023-09-23T19:51:24.784935+0000","last_unstale":"2023-09-23T19:51:59.703786+0000","last_undegraded":"2023-09-23T19:51:59.703786+0000","last_fullsized":"2023-09-23T19:51:59.703786+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":232,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_clean_scrub_stamp":"2023-09-23T19:51:23.737731+0000","objects_scrubbed":0,"log_size":1066,"log_dups_size":0,"ondisk_log_size":1066,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69861376,"num_objects":1066,"num_object_clones":0,"num_object_copies":3198,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1066,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1066,"num_write_kb":68224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":141,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:42.205279+0000","last_change":"2023-09-23T19:32:49.737323+0000","last_active":"2023-09-23T19:51:42.205279+0000","last_peered":"2023-09-23T19:51:42.205279+0000","last_clean":"2023-09-23T19:51:42.205279+0000","last_became_active":"2023-09-23T19:32:49.736937+0000","last_became_peered":"2023-09-23T19:32:49.736937+0000","last_unstale":"2023-09-23T19:51:42.205279+0000","last_undegraded":"2023-09-23T19:51:42.205279+0000","last_fullsized":"2023-09-23T19:51:42.205279+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":172,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:42.311570+0000","last_change":"2023-09-23T19:22:42.145506+0000","last_active":"2023-09-23T19:51:42.311570+0000","last_peered":"2023-09-23T19:51:42.311570+0000","last_clean":"2023-09-23T19:51:42.311570+0000","last_became_active":"2023-09-23T19:22:42.143991+0000","last_became_peered":"2023-09-23T19:22:42.143991+0000","last_unstale":"2023-09-23T19:51:42.311570+0000","last_undegraded":"2023-09-23T19:51:42.311570+0000","last_fullsized":"2023-09-23T19:51:42.311570+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:56.875175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":178,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:42.204813+0000","last_change":"2023-09-23T19:32:49.734592+0000","last_active":"2023-09-23T19:51:42.204813+0000","last_peered":"2023-09-23T19:51:42.204813+0000","last_clean":"2023-09-23T19:51:42.204813+0000","last_became_active":"2023-09-23T19:32:49.733905+0000","last_became_peered":"2023-09-23T19:32:49.733905+0000","last_unstale":"2023-09-23T19:51:42.204813+0000","last_undegraded":"2023-09-23T19:51:42.204813+0000","last_fullsized":"2023-09-23T19:51:42.204813+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.8","version":"237'983","reported_seq":997,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:59.130969+0000","last_change":"2023-09-23T19:51:24.780616+0000","last_active":"2023-09-23T19:51:59.130969+0000","last_peered":"2023-09-23T19:51:59.130969+0000","last_clean":"2023-09-23T19:51:59.130969+0000","last_became_active":"2023-09-23T19:51:24.780250+0000","last_became_peered":"2023-09-23T19:51:24.780250+0000","last_unstale":"2023-09-23T19:51:59.130969+0000","last_undegraded":"2023-09-23T19:51:59.130969+0000","last_fullsized":"2023-09-23T19:51:59.130969+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":232,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_clean_scrub_stamp":"2023-09-23T19:51:23.737731+0000","objects_scrubbed":0,"log_size":983,"log_dups_size":0,"ondisk_log_size":983,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:10.216601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64421888,"num_objects":983,"num_object_clones":0,"num_object_copies":2949,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":983,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":983,"num_write_kb":62912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":222,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:42.555860+0000","last_change":"2023-09-23T19:38:43.895887+0000","last_active":"2023-09-23T19:51:42.555860+0000","last_peered":"2023-09-23T19:51:42.555860+0000","last_clean":"2023-09-23T19:51:42.555860+0000","last_became_active":"2023-09-23T19:38:43.895486+0000","last_became_peered":"2023-09-23T19:38:43.895486+0000","last_unstale":"2023-09-23T19:51:42.555860+0000","last_undegraded":"2023-09-23T19:51:42.555860+0000","last_fullsized":"2023-09-23T19:51:42.555860+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":206,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:42.307652+0000","last_change":"2023-09-23T19:22:42.149694+0000","last_active":"2023-09-23T19:51:42.307652+0000","last_peered":"2023-09-23T19:51:42.307652+0000","last_clean":"2023-09-23T19:51:42.307652+0000","last_became_active":"2023-09-23T19:22:42.148194+0000","last_became_peered":"2023-09-23T19:22:42.148194+0000","last_unstale":"2023-09-23T19:51:42.307652+0000","last_undegraded":"2023-09-23T19:51:42.307652+0000","last_fullsized":"2023-09-23T19:51:42.307652+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:47.842383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":186,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:42.827759+0000","last_change":"2023-09-23T19:05:52.138620+0000","last_active":"2023-09-23T19:51:42.827759+0000","last_peered":"2023-09-23T19:51:42.827759+0000","last_clean":"2023-09-23T19:51:42.827759+0000","last_became_active":"2023-09-23T19:05:52.138243+0000","last_became_peered":"2023-09-23T19:05:52.138243+0000","last_unstale":"2023-09-23T19:51:42.827759+0000","last_undegraded":"2023-09-23T19:51:42.827759+0000","last_fullsized":"2023-09-23T19:51:42.827759+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":189,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:42.307690+0000","last_change":"2023-09-23T19:18:09.008811+0000","last_active":"2023-09-23T19:51:42.307690+0000","last_peered":"2023-09-23T19:51:42.307690+0000","last_clean":"2023-09-23T19:51:42.307690+0000","last_became_active":"2023-09-23T19:18:09.007255+0000","last_became_peered":"2023-09-23T19:18:09.007255+0000","last_unstale":"2023-09-23T19:51:42.307690+0000","last_undegraded":"2023-09-23T19:51:42.307690+0000","last_fullsized":"2023-09-23T19:51:42.307690+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:25.874038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":193,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:41.287693+0000","last_change":"2023-09-23T19:38:43.877244+0000","last_active":"2023-09-23T19:51:41.287693+0000","last_peered":"2023-09-23T19:51:41.287693+0000","last_clean":"2023-09-23T19:51:41.287693+0000","last_became_active":"2023-09-23T19:38:43.876896+0000","last_became_peered":"2023-09-23T19:38:43.876896+0000","last_unstale":"2023-09-23T19:51:41.287693+0000","last_undegraded":"2023-09-23T19:51:41.287693+0000","last_fullsized":"2023-09-23T19:51:41.287693+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:22.288276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":66,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:42.828356+0000","last_change":"2023-09-23T19:43:38.620732+0000","last_active":"2023-09-23T19:51:42.828356+0000","last_peered":"2023-09-23T19:51:42.828356+0000","last_clean":"2023-09-23T19:51:42.828356+0000","last_became_active":"2023-09-23T19:43:38.619424+0000","last_became_peered":"2023-09-23T19:43:38.619424+0000","last_unstale":"2023-09-23T19:51:42.828356+0000","last_undegraded":"2023-09-23T19:51:42.828356+0000","last_fullsized":"2023-09-23T19:51:42.828356+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":240,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:42.828452+0000","last_change":"2023-09-23T19:38:43.947451+0000","last_active":"2023-09-23T19:51:42.828452+0000","last_peered":"2023-09-23T19:51:42.828452+0000","last_clean":"2023-09-23T19:51:42.828452+0000","last_became_active":"2023-09-23T19:38:43.946420+0000","last_became_peered":"2023-09-23T19:38:43.946420+0000","last_unstale":"2023-09-23T19:51:42.828452+0000","last_undegraded":"2023-09-23T19:51:42.828452+0000","last_fullsized":"2023-09-23T19:51:42.828452+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":176,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:43.873632+0000","last_change":"2023-09-23T19:47:56.793347+0000","last_active":"2023-09-23T19:51:43.873632+0000","last_peered":"2023-09-23T19:51:43.873632+0000","last_clean":"2023-09-23T19:51:43.873632+0000","last_became_active":"2023-09-23T19:47:56.792866+0000","last_became_peered":"2023-09-23T19:47:56.792866+0000","last_unstale":"2023-09-23T19:51:43.873632+0000","last_undegraded":"2023-09-23T19:51:43.873632+0000","last_fullsized":"2023-09-23T19:51:43.873632+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":216,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:42.828343+0000","last_change":"2023-09-23T19:28:07.896983+0000","last_active":"2023-09-23T19:51:42.828343+0000","last_peered":"2023-09-23T19:51:42.828343+0000","last_clean":"2023-09-23T19:51:42.828343+0000","last_became_active":"2023-09-23T19:28:07.896577+0000","last_became_peered":"2023-09-23T19:28:07.896577+0000","last_unstale":"2023-09-23T19:51:42.828343+0000","last_undegraded":"2023-09-23T19:51:42.828343+0000","last_fullsized":"2023-09-23T19:51:42.828343+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":146,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:42.555846+0000","last_change":"2023-09-23T19:38:43.900225+0000","last_active":"2023-09-23T19:51:42.555846+0000","last_peered":"2023-09-23T19:51:42.555846+0000","last_clean":"2023-09-23T19:51:42.555846+0000","last_became_active":"2023-09-23T19:38:43.899907+0000","last_became_peered":"2023-09-23T19:38:43.899907+0000","last_unstale":"2023-09-23T19:51:42.555846+0000","last_undegraded":"2023-09-23T19:51:42.555846+0000","last_fullsized":"2023-09-23T19:51:42.555846+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":202,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:41.287326+0000","last_change":"2023-09-23T19:38:43.882487+0000","last_active":"2023-09-23T19:51:41.287326+0000","last_peered":"2023-09-23T19:51:41.287326+0000","last_clean":"2023-09-23T19:51:41.287326+0000","last_became_active":"2023-09-23T19:38:43.881931+0000","last_became_peered":"2023-09-23T19:38:43.881931+0000","last_unstale":"2023-09-23T19:51:41.287326+0000","last_undegraded":"2023-09-23T19:51:41.287326+0000","last_fullsized":"2023-09-23T19:51:41.287326+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:02:36.906535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":222,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:42.307755+0000","last_change":"2023-09-23T19:38:43.893983+0000","last_active":"2023-09-23T19:51:42.307755+0000","last_peered":"2023-09-23T19:51:42.307755+0000","last_clean":"2023-09-23T19:51:42.307755+0000","last_became_active":"2023-09-23T19:38:43.893527+0000","last_became_peered":"2023-09-23T19:38:43.893527+0000","last_unstale":"2023-09-23T19:51:42.307755+0000","last_undegraded":"2023-09-23T19:51:42.307755+0000","last_fullsized":"2023-09-23T19:51:42.307755+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:12.156531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":195,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T19:51:42.827681+0000","last_change":"2023-09-23T19:06:36.673002+0000","last_active":"2023-09-23T19:51:42.827681+0000","last_peered":"2023-09-23T19:51:42.827681+0000","last_clean":"2023-09-23T19:51:42.827681+0000","last_became_active":"2023-09-23T19:06:36.671911+0000","last_became_peered":"2023-09-23T19:06:36.671911+0000","last_unstale":"2023-09-23T19:51:42.827681+0000","last_undegraded":"2023-09-23T19:51:42.827681+0000","last_fullsized":"2023-09-23T19:51:42.827681+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":1040187392,"num_objects":15872,"num_object_clones":0,"num_object_copies":47616,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15872,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15872,"num_write_kb":1015808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3004235776,"data_stored":3004235776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":15872,"ondisk_log_size":15872,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430798,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1223048,"kb_used_data":335360,"kb_used_omap":0,"kb_used_meta":887680,"kb_avail":92522104,"statfs":{"total":95995035648,"available":94742634496,"internally_reserved":0,"allocated":343408640,"data_stored":342876665,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":908984320},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:51:09 2023","interfaces":[{"interface":"back","average":{"1min":1.52,"5min":1.3200000000000001,"15min":1.4890000000000001},"min":{"1min":0.307,"5min":0.28699999999999998,"15min":0.26400000000000001},"max":{"1min":4.2039999999999997,"5min":6.8419999999999996,"15min":18.271000000000001},"last":1.1319999999999999},{"interface":"front","average":{"1min":2.0859999999999999,"5min":1.274,"15min":1.5700000000000001},"min":{"1min":0.47799999999999998,"5min":0.29299999999999998,"15min":0.25600000000000001},"max":{"1min":12.705,"5min":12.705,"15min":18.242000000000001},"last":2.8140000000000001}]},{"osd":1,"last update":"Sat Sep 23 19:51:28 2023","interfaces":[{"interface":"back","average":{"1min":1.0640000000000001,"5min":1.5609999999999999,"15min":1.823},"min":{"1min":0.63400000000000001,"5min":0.33900000000000002,"15min":0.28799999999999998},"max":{"1min":2.0720000000000001,"5min":22.611999999999998,"15min":30.780999999999999},"last":0.51600000000000001},{"interface":"front","average":{"1min":1.2310000000000001,"5min":1.4510000000000001,"15min":1.835},"min":{"1min":0.626,"5min":0.314,"15min":0.30599999999999999},"max":{"1min":4.3840000000000003,"5min":22.687999999999999,"15min":30.827999999999999},"last":1.512}]},{"osd":2,"last update":"Sat Sep 23 19:51:09 2023","interfaces":[{"interface":"back","average":{"1min":1.8720000000000001,"5min":6.0060000000000002,"15min":3.3580000000000001},"min":{"1min":0.55500000000000005,"5min":0.308,"15min":0.28100000000000003},"max":{"1min":6.5380000000000003,"5min":246.982,"15min":246.982},"last":1.5580000000000001},{"interface":"front","average":{"1min":1.3959999999999999,"5min":5.7969999999999997,"15min":3.073},"min":{"1min":0.37,"5min":0.35999999999999999,"15min":0.35199999999999998},"max":{"1min":6.7190000000000003,"5min":247.751,"15min":247.751},"last":2.6600000000000001}]},{"osd":3,"last update":"Sat Sep 23 19:51:59 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":1.5860000000000001,"15min":1.5329999999999999},"min":{"1min":0.54300000000000004,"5min":0.39000000000000001,"15min":0.312},"max":{"1min":1.2589999999999999,"5min":7.593,"15min":8.843},"last":0.54300000000000004},{"interface":"front","average":{"1min":1.085,"5min":1.6659999999999999,"15min":1.613},"min":{"1min":0.77600000000000002,"5min":0.42999999999999999,"15min":0.34100000000000003},"max":{"1min":1.613,"5min":7.609,"15min":7.609},"last":1.2}]},{"osd":4,"last update":"Sat Sep 23 19:51:28 2023","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":0.91800000000000004,"15min":1.145},"min":{"1min":0.40300000000000002,"5min":0.34200000000000003,"15min":0.254},"max":{"1min":1.708,"5min":5.3230000000000004,"15min":8.8010000000000002},"last":1.6950000000000001},{"interface":"front","average":{"1min":1.1419999999999999,"5min":1.02,"15min":1.3320000000000001},"min":{"1min":0.39100000000000001,"5min":0.34799999999999998,"15min":0.27400000000000002},"max":{"1min":4.4269999999999996,"5min":5.2599999999999998,"15min":13.385},"last":1.169}]},{"osd":5,"last update":"Sat Sep 23 19:51:28 2023","interfaces":[{"interface":"back","average":{"1min":0.94099999999999995,"5min":0.996,"15min":1.3149999999999999},"min":{"1min":0.42699999999999999,"5min":0.36399999999999999,"15min":0.34200000000000003},"max":{"1min":1.54,"5min":4.3769999999999998,"15min":13.273999999999999},"last":1.641},{"interface":"front","average":{"1min":1.0549999999999999,"5min":0.999,"15min":1.3009999999999999},"min":{"1min":0.59799999999999998,"5min":0.38800000000000001,"15min":0.34200000000000003},"max":{"1min":1.738,"5min":5.4279999999999999,"15min":8.9100000000000001},"last":1.391}]},{"osd":6,"last update":"Sat Sep 23 19:51:28 2023","interfaces":[{"interface":"back","average":{"1min":1.1020000000000001,"5min":1.101,"15min":1.6020000000000001},"min":{"1min":0.63600000000000001,"5min":0.33900000000000002,"15min":0.33300000000000002},"max":{"1min":1.8049999999999999,"5min":5.4569999999999999,"15min":22.725000000000001},"last":0.47999999999999998},{"interface":"front","average":{"1min":1.3300000000000001,"5min":1.2609999999999999,"15min":1.8169999999999999},"min":{"1min":0.49099999999999999,"5min":0.38800000000000001,"15min":0.308},"max":{"1min":6.1769999999999996,"5min":9.4510000000000005,"15min":22.707000000000001},"last":1.589}]}]},{"osd":6,"up_from":42,"seq":180388627030,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1204984,"kb_used_data":317872,"kb_used_omap":0,"kb_used_meta":887104,"kb_avail":92540168,"statfs":{"total":95995035648,"available":94761132032,"internally_reserved":0,"allocated":325500928,"data_stored":324984876,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":908394496},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:51:12 2023","interfaces":[{"interface":"back","average":{"1min":1.238,"5min":1.8999999999999999,"15min":2.012},"min":{"1min":0.35099999999999998,"5min":0.247,"15min":0.24199999999999999},"max":{"1min":4.1150000000000002,"5min":45.600000000000001,"15min":45.600000000000001},"last":1.5680000000000001},{"interface":"front","average":{"1min":1.343,"5min":2.0190000000000001,"15min":1.8700000000000001},"min":{"1min":0.45400000000000001,"5min":0.26600000000000001,"15min":0.25800000000000001},"max":{"1min":4.8250000000000002,"5min":45.347999999999999,"15min":45.347999999999999},"last":1.123}]},{"osd":1,"last update":"Sat Sep 23 19:51:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0229999999999999,"5min":1.573,"15min":2.5739999999999998},"min":{"1min":0.70499999999999996,"5min":0.309,"15min":0.26300000000000001},"max":{"1min":1.417,"5min":11.541,"15min":117.58199999999999},"last":1.585},{"interface":"front","average":{"1min":1.0620000000000001,"5min":1.8009999999999999,"15min":2.6299999999999999},"min":{"1min":0.54300000000000004,"5min":0.32500000000000001,"15min":0.29199999999999998},"max":{"1min":1.2929999999999999,"5min":11.529,"15min":117.633},"last":1.161}]},{"osd":2,"last update":"Sat Sep 23 19:51:02 2023","interfaces":[{"interface":"back","average":{"1min":1.1839999999999999,"5min":5.399,"15min":4.4240000000000004},"min":{"1min":0.308,"5min":0.308,"15min":0.30299999999999999},"max":{"1min":5.6749999999999998,"5min":165.09800000000001,"15min":165.09800000000001},"last":1.546},{"interface":"front","average":{"1min":2.0430000000000001,"5min":5.3680000000000003,"15min":4.383},"min":{"1min":0.42399999999999999,"5min":0.32500000000000001,"15min":0.32400000000000001},"max":{"1min":7.0839999999999996,"5min":163.93000000000001,"15min":163.93000000000001},"last":1.1930000000000001}]},{"osd":3,"last update":"Sat Sep 23 19:50:57 2023","interfaces":[{"interface":"back","average":{"1min":3.0939999999999999,"5min":2.077,"15min":1.964},"min":{"1min":0.41999999999999998,"5min":0.31,"15min":0.30199999999999999},"max":{"1min":16.428000000000001,"5min":16.428000000000001,"15min":16.428000000000001},"last":0.53800000000000003},{"interface":"front","average":{"1min":2.4180000000000001,"5min":1.6899999999999999,"15min":1.821},"min":{"1min":0.39700000000000002,"5min":0.311,"15min":0.307},"max":{"1min":7.8680000000000003,"5min":10.192,"15min":18.917000000000002},"last":1.508}]},{"osd":4,"last update":"Sat Sep 23 19:50:57 2023","interfaces":[{"interface":"back","average":{"1min":1.143,"5min":0.91200000000000003,"15min":1.4019999999999999},"min":{"1min":0.35899999999999999,"5min":0.33200000000000002,"15min":0.28799999999999998},"max":{"1min":6.6630000000000003,"5min":6.6630000000000003,"15min":11.779},"last":1.22},{"interface":"front","average":{"1min":1.9359999999999999,"5min":1.339,"15min":1.6839999999999999},"min":{"1min":0.29399999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":12.067,"5min":12.067,"15min":12.067},"last":0.60399999999999998}]},{"osd":5,"last update":"Sat Sep 23 19:50:57 2023","interfaces":[{"interface":"back","average":{"1min":1.2010000000000001,"5min":1.0960000000000001,"15min":1.534},"min":{"1min":0.38900000000000001,"5min":0.38500000000000001,"15min":0.32000000000000001},"max":{"1min":6.641,"5min":6.641,"15min":12.827},"last":1.252},{"interface":"front","average":{"1min":1.9710000000000001,"5min":1.4139999999999999,"15min":1.8720000000000001},"min":{"1min":0.35299999999999998,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":12.138999999999999,"5min":12.138999999999999,"15min":12.138999999999999},"last":0.63}]},{"osd":7,"last update":"Sat Sep 23 19:51:24 2023","interfaces":[{"interface":"back","average":{"1min":1.272,"5min":1.25,"15min":1.556},"min":{"1min":0.59899999999999998,"5min":0.40500000000000003,"15min":0.33200000000000002},"max":{"1min":4.4020000000000001,"5min":6.9009999999999998,"15min":11.663},"last":1.28},{"interface":"front","average":{"1min":1.554,"5min":1.4850000000000001,"15min":1.95},"min":{"1min":0.46500000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":6.7539999999999996,"5min":12.327999999999999,"15min":12.327999999999999},"last":0.57599999999999996}]}]},{"osd":1,"up_from":98,"seq":420906795361,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":826484,"kb_used_data":156140,"kb_used_omap":4,"kb_used_meta":670331,"kb_avail":92918668,"statfs":{"total":95995035648,"available":95148716032,"internally_reserved":0,"allocated":159887360,"data_stored":158905430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":686419854},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:51:42 2023","interfaces":[{"interface":"back","average":{"1min":0.79600000000000004,"5min":3.0449999999999999,"15min":3.181},"min":{"1min":0.433,"5min":0.221,"15min":0.18099999999999999},"max":{"1min":2.2759999999999998,"5min":84.195999999999998,"15min":84.195999999999998},"last":1.0289999999999999},{"interface":"front","average":{"1min":0.79400000000000004,"5min":2.6949999999999998,"15min":2.9689999999999999},"min":{"1min":0.41099999999999998,"5min":0.22600000000000001,"15min":0.20300000000000001},"max":{"1min":2.2410000000000001,"5min":84.016000000000005,"15min":84.016000000000005},"last":4.016}]},{"osd":2,"last update":"Sat Sep 23 19:51:05 2023","interfaces":[{"interface":"back","average":{"1min":2.4990000000000001,"5min":3.8969999999999998,"15min":2.54},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.23200000000000001},"max":{"1min":10.314,"5min":72.859999999999999,"15min":72.859999999999999},"last":3.9510000000000001},{"interface":"front","average":{"1min":2.5870000000000002,"5min":3.331,"15min":2.6080000000000001},"min":{"1min":0.44800000000000001,"5min":0.25800000000000001,"15min":0.23300000000000001},"max":{"1min":10.654,"5min":73.665999999999997,"15min":73.665999999999997},"last":1.484}]},{"osd":3,"last update":"Sat Sep 23 19:50:54 2023","interfaces":[{"interface":"back","average":{"1min":2.0950000000000002,"5min":1.986,"15min":1.7370000000000001},"min":{"1min":0.28699999999999998,"5min":0.25,"15min":0.25},"max":{"1min":10.273999999999999,"5min":10.273999999999999,"15min":15.004},"last":4.3399999999999999},{"interface":"front","average":{"1min":1.7689999999999999,"5min":1.8149999999999999,"15min":1.925},"min":{"1min":0.28799999999999998,"5min":0.217,"15min":0.217},"max":{"1min":6.8730000000000002,"5min":11.683999999999999,"15min":48.692},"last":5.4370000000000003}]},{"osd":4,"last update":"Sat Sep 23 19:51:42 2023","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":1.4950000000000001,"15min":1.623},"min":{"1min":0.47899999999999998,"5min":0.313,"15min":0.313},"max":{"1min":1.659,"5min":10.378,"15min":53.128999999999998},"last":4.0880000000000001},{"interface":"front","average":{"1min":0.92200000000000004,"5min":2.274,"15min":1.9399999999999999},"min":{"1min":0.49199999999999999,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":1.3160000000000001,"5min":82.239999999999995,"15min":82.239999999999995},"last":0.67700000000000005}]},{"osd":5,"last update":"Sat Sep 23 19:51:42 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":1.4730000000000001,"15min":1.673},"min":{"1min":0.432,"5min":0.36199999999999999,"15min":0.34899999999999998},"max":{"1min":1.2529999999999999,"5min":11.750999999999999,"15min":48.738},"last":0.57099999999999995},{"interface":"front","average":{"1min":1.0489999999999999,"5min":1.675,"15min":1.746},"min":{"1min":0.60099999999999998,"5min":0.36199999999999999,"15min":0.35799999999999998},"max":{"1min":1.552,"5min":10.611000000000001,"15min":48.726999999999997},"last":0.63100000000000001}]},{"osd":6,"last update":"Sat Sep 23 19:51:42 2023","interfaces":[{"interface":"back","average":{"1min":1.129,"5min":2.0979999999999999,"15min":2.2200000000000002},"min":{"1min":0.70099999999999996,"5min":0.41699999999999998,"15min":0.318},"max":{"1min":1.6120000000000001,"5min":10.565,"15min":48.762},"last":0.69399999999999995},{"interface":"front","average":{"1min":1.149,"5min":1.8300000000000001,"15min":2.0990000000000002},"min":{"1min":0.54900000000000004,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":1.712,"5min":7.8799999999999999,"15min":53.076999999999998},"last":4.1269999999999998}]},{"osd":7,"last update":"Sat Sep 23 19:51:42 2023","interfaces":[{"interface":"back","average":{"1min":1.167,"5min":1.8520000000000001,"15min":1.859},"min":{"1min":0.51600000000000001,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":1.579,"5min":10.475,"15min":48.677},"last":4.2480000000000002},{"interface":"front","average":{"1min":1.0569999999999999,"5min":1.8460000000000001,"15min":1.869},"min":{"1min":0.46500000000000002,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":1.405,"5min":11.778,"15min":48.805},"last":0.89500000000000002}]}]},{"osd":0,"up_from":85,"seq":365072220568,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":924332,"kb_used_data":138660,"kb_used_omap":5,"kb_used_meta":785658,"kb_avail":92820820,"statfs":{"total":95995035648,"available":95048519680,"internally_reserved":0,"allocated":141987840,"data_stored":141016451,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":804514426},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 19:51:35 2023","interfaces":[{"interface":"back","average":{"1min":0.96599999999999997,"5min":1.6299999999999999,"15min":1.7669999999999999},"min":{"1min":0.34999999999999998,"5min":0.216,"15min":0.20799999999999999},"max":{"1min":3.9580000000000002,"5min":10.111000000000001,"15min":57.234999999999999},"last":1.4419999999999999},{"interface":"front","average":{"1min":0.97299999999999998,"5min":1.702,"15min":1.982},"min":{"1min":0.45600000000000002,"5min":0.20399999999999999,"15min":0.189},"max":{"1min":5.2270000000000003,"5min":8.6750000000000007,"15min":58.825000000000003},"last":0.315}]},{"osd":2,"last update":"Sat Sep 23 19:51:08 2023","interfaces":[{"interface":"back","average":{"1min":1.9970000000000001,"5min":4.4880000000000004,"15min":3.722},"min":{"1min":0.26900000000000002,"5min":0.245,"15min":0.21299999999999999},"max":{"1min":7.6159999999999997,"5min":213.20599999999999,"15min":262.59500000000003},"last":0.53900000000000003},{"interface":"front","average":{"1min":1.1060000000000001,"5min":3.944,"15min":3.6739999999999999},"min":{"1min":0.29399999999999998,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":3.7490000000000001,"5min":209.64400000000001,"15min":263.25700000000001},"last":0.45700000000000002}]},{"osd":3,"last update":"Sat Sep 23 19:51:41 2023","interfaces":[{"interface":"back","average":{"1min":0.877,"5min":1.8280000000000001,"15min":1.98},"min":{"1min":0.52200000000000002,"5min":0.222,"15min":0.222},"max":{"1min":2.294,"5min":11.436,"15min":14.212},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.754,"5min":1.909,"15min":1.909},"min":{"1min":0.48799999999999999,"5min":0.28999999999999998,"15min":0.223},"max":{"1min":1.1930000000000001,"5min":8.0809999999999995,"15min":14.84},"last":0.34300000000000003}]},{"osd":4,"last update":"Sat Sep 23 19:51:19 2023","interfaces":[{"interface":"back","average":{"1min":1.1180000000000001,"5min":1.4670000000000001,"15min":1.635},"min":{"1min":0.376,"5min":0.35199999999999998,"15min":0.32800000000000001},"max":{"1min":3.6869999999999998,"5min":9.0679999999999996,"15min":11.792999999999999},"last":2.073},{"interface":"front","average":{"1min":1.581,"5min":1.6930000000000001,"15min":1.732},"min":{"1min":0.50900000000000001,"5min":0.41399999999999998,"15min":0.315},"max":{"1min":5.3140000000000001,"5min":8.6289999999999996,"15min":13.641999999999999},"last":2.2999999999999998}]},{"osd":5,"last update":"Sat Sep 23 19:51:19 2023","interfaces":[{"interface":"back","average":{"1min":1.738,"5min":1.599,"15min":1.7250000000000001},"min":{"1min":0.442,"5min":0.39200000000000002,"15min":0.33700000000000002},"max":{"1min":5.3689999999999998,"5min":6.9370000000000003,"15min":8.6229999999999993},"last":2.2709999999999999},{"interface":"front","average":{"1min":1.1120000000000001,"5min":1.4710000000000001,"15min":1.7450000000000001},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":3.629,"5min":9.0329999999999995,"15min":10.411},"last":1.5549999999999999}]},{"osd":6,"last update":"Sat Sep 23 19:51:19 2023","interfaces":[{"interface":"back","average":{"1min":1.357,"5min":2.206,"15min":2.1640000000000001},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":3.823,"5min":66.170000000000002,"15min":66.170000000000002},"last":0.67000000000000004},{"interface":"front","average":{"1min":2.004,"5min":2.472,"15min":2.2589999999999999},"min":{"1min":0.47299999999999998,"5min":0.45900000000000002,"15min":0.35299999999999998},"max":{"1min":5.3419999999999996,"5min":66.260999999999996,"15min":66.260999999999996},"last":0.59199999999999997}]},{"osd":7,"last update":"Sat Sep 23 19:51:19 2023","interfaces":[{"interface":"back","average":{"1min":1.5860000000000001,"5min":1.6819999999999999,"15min":1.758},"min":{"1min":0.44800000000000001,"5min":0.40400000000000003,"15min":0.38200000000000001},"max":{"1min":4.6879999999999997,"5min":10.082000000000001,"15min":13.672000000000001},"last":1.506},{"interface":"front","average":{"1min":1.411,"5min":1.786,"15min":1.821},"min":{"1min":0.439,"5min":0.41199999999999998,"15min":0.40100000000000002},"max":{"1min":5.5830000000000002,"5min":11.863,"15min":11.863},"last":0.61699999999999999}]}]},{"osd":2,"up_from":159,"seq":682899800224,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":981496,"kb_used_data":522544,"kb_used_omap":7,"kb_used_meta":458936,"kb_avail":92763656,"statfs":{"total":95995035648,"available":94989983744,"internally_reserved":0,"allocated":535085056,"data_stored":534156417,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":469950545},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:51:21 2023","interfaces":[{"interface":"back","average":{"1min":1.804,"5min":3.0739999999999998,"15min":2.323},"min":{"1min":0.35199999999999998,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":15.249000000000001,"5min":117.77500000000001,"15min":117.77500000000001},"last":0.437},{"interface":"front","average":{"1min":1.7050000000000001,"5min":3.8780000000000001,"15min":2.6360000000000001},"min":{"1min":0.40400000000000003,"5min":0.214,"15min":0.214},"max":{"1min":10.914999999999999,"5min":117.91800000000001,"15min":117.91800000000001},"last":0.46300000000000002}]},{"osd":1,"last update":"Sat Sep 23 19:51:21 2023","interfaces":[{"interface":"back","average":{"1min":1.387,"5min":3.4209999999999998,"15min":2.4529999999999998},"min":{"1min":0.34999999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":6.9089999999999998,"5min":114.107,"15min":114.107},"last":1.7310000000000001},{"interface":"front","average":{"1min":1.4379999999999999,"5min":3.044,"15min":2.2770000000000001},"min":{"1min":0.38600000000000001,"5min":0.23699999999999999,"15min":0.22},"max":{"1min":8.6560000000000006,"5min":117.172,"15min":117.172},"last":0.53500000000000003}]},{"osd":3,"last update":"Sat Sep 23 19:51:21 2023","interfaces":[{"interface":"back","average":{"1min":1.5,"5min":2.4569999999999999,"15min":2.1429999999999998},"min":{"1min":0.41799999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":6.9459999999999997,"5min":11.128,"15min":17.736999999999998},"last":0.58499999999999996},{"interface":"front","average":{"1min":1.411,"5min":2.0680000000000001,"15min":2.0830000000000002},"min":{"1min":0.41499999999999998,"5min":0.24099999999999999,"15min":0.21299999999999999},"max":{"1min":9.4009999999999998,"5min":20.286999999999999,"15min":20.286999999999999},"last":0.60799999999999998}]},{"osd":4,"last update":"Sat Sep 23 19:51:21 2023","interfaces":[{"interface":"back","average":{"1min":1.2729999999999999,"5min":2.1160000000000001,"15min":1.887},"min":{"1min":0.52100000000000002,"5min":0.317,"15min":0.317},"max":{"1min":5.6360000000000001,"5min":10.622999999999999,"15min":18.318999999999999},"last":0.56699999999999995},{"interface":"front","average":{"1min":1.5109999999999999,"5min":2.0680000000000001,"15min":1.776},"min":{"1min":0.51000000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":4.7240000000000002,"5min":12.302,"15min":12.302},"last":0.70599999999999996}]},{"osd":5,"last update":"Sat Sep 23 19:51:21 2023","interfaces":[{"interface":"back","average":{"1min":1.296,"5min":1.216,"15min":1.542},"min":{"1min":0.47999999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":5.5099999999999998,"5min":5.5099999999999998,"15min":16.577000000000002},"last":1.411},{"interface":"front","average":{"1min":1.2749999999999999,"5min":2.0470000000000002,"15min":1.9399999999999999},"min":{"1min":0.48699999999999999,"5min":0.36799999999999999,"15min":0.33600000000000002},"max":{"1min":5.6849999999999996,"5min":10.193,"15min":10.193},"last":0.60099999999999998}]},{"osd":6,"last update":"Sat Sep 23 19:51:21 2023","interfaces":[{"interface":"back","average":{"1min":1.4339999999999999,"5min":2.1920000000000002,"15min":2.1709999999999998},"min":{"1min":0.81899999999999995,"5min":0.38900000000000001,"15min":0.36899999999999999},"max":{"1min":5.7839999999999998,"5min":10.166,"15min":11.851000000000001},"last":0.72699999999999998},{"interface":"front","average":{"1min":1.6990000000000001,"5min":2.3420000000000001,"15min":2.1840000000000002},"min":{"1min":0.48499999999999999,"5min":0.38,"15min":0.36699999999999999},"max":{"1min":5.1109999999999998,"5min":12.256,"15min":12.297000000000001},"last":0.81399999999999995}]},{"osd":7,"last update":"Sat Sep 23 19:51:21 2023","interfaces":[{"interface":"back","average":{"1min":1.391,"5min":1.9139999999999999,"15min":1.8700000000000001},"min":{"1min":0.45600000000000002,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":5.468,"5min":32.945,"15min":32.945},"last":0.73499999999999999},{"interface":"front","average":{"1min":1.635,"5min":2.3100000000000001,"15min":1.9710000000000001},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.41299999999999998},"max":{"1min":6.7910000000000004,"5min":16.593,"15min":16.593},"last":1.6040000000000001}]}]},{"osd":3,"up_from":116,"seq":498216206624,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":766332,"kb_used_data":146804,"kb_used_omap":2,"kb_used_meta":619517,"kb_avail":92978820,"statfs":{"total":95995035648,"available":95210311680,"internally_reserved":0,"allocated":150327296,"data_stored":149452804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":634385461},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:51:54 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":2.7559999999999998,"15min":2.0430000000000001},"min":{"1min":0.36599999999999999,"5min":0.23799999999999999,"15min":0.21199999999999999},"max":{"1min":0.89700000000000002,"5min":50.771999999999998,"15min":50.771999999999998},"last":2.077},{"interface":"front","average":{"1min":0.60999999999999999,"5min":3.2589999999999999,"15min":2.2690000000000001},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.20100000000000001},"max":{"1min":1.282,"5min":48.887,"15min":48.887},"last":0.58199999999999996}]},{"osd":1,"last update":"Sat Sep 23 19:51:54 2023","interfaces":[{"interface":"back","average":{"1min":0.73599999999999999,"5min":2.4969999999999999,"15min":2.1179999999999999},"min":{"1min":0.28299999999999997,"5min":0.25800000000000001,"15min":0.22600000000000001},"max":{"1min":1.9359999999999999,"5min":48.918999999999997,"15min":92.171000000000006},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.69899999999999995,"5min":2.4540000000000002,"15min":2.2400000000000002},"min":{"1min":0.42699999999999999,"5min":0.25800000000000001,"15min":0.23000000000000001},"max":{"1min":0.92900000000000005,"5min":50.722999999999999,"15min":87.771000000000001},"last":1.548}]},{"osd":2,"last update":"Sat Sep 23 19:51:09 2023","interfaces":[{"interface":"back","average":{"1min":3.7109999999999999,"5min":4.9699999999999998,"15min":3.2400000000000002},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":36.826000000000001,"5min":116.363,"15min":116.363},"last":0.54900000000000004},{"interface":"front","average":{"1min":4.0540000000000003,"5min":4.7069999999999999,"15min":2.7650000000000001},"min":{"1min":0.29199999999999998,"5min":0.27300000000000002,"15min":0.252},"max":{"1min":36.628999999999998,"5min":118.474,"15min":118.474},"last":0.67100000000000004}]},{"osd":4,"last update":"Sat Sep 23 19:51:54 2023","interfaces":[{"interface":"back","average":{"1min":1.1140000000000001,"5min":1.5840000000000001,"15min":1.712},"min":{"1min":0.67500000000000004,"5min":0.33100000000000002,"15min":0.32500000000000001},"max":{"1min":2.645,"5min":8.6639999999999997,"15min":11.666},"last":1.7649999999999999},{"interface":"front","average":{"1min":1.1439999999999999,"5min":1.9930000000000001,"15min":1.8180000000000001},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.32300000000000001},"max":{"1min":2.7549999999999999,"5min":9.9990000000000006,"15min":9.9990000000000006},"last":0.77900000000000003}]},{"osd":5,"last update":"Sat Sep 23 19:51:54 2023","interfaces":[{"interface":"back","average":{"1min":0.94499999999999995,"5min":1.998,"15min":1.746},"min":{"1min":0.373,"5min":0.373,"15min":0.35299999999999998},"max":{"1min":1.891,"5min":10.022,"15min":10.022},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.97199999999999998,"5min":1.8160000000000001,"15min":1.758},"min":{"1min":0.46200000000000002,"5min":0.38900000000000001,"15min":0.34000000000000002},"max":{"1min":1.615,"5min":11.507,"15min":13.146000000000001},"last":0.64100000000000001}]},{"osd":6,"last update":"Sat Sep 23 19:51:54 2023","interfaces":[{"interface":"back","average":{"1min":1.0780000000000001,"5min":2.423,"15min":2.2440000000000002},"min":{"1min":0.63,"5min":0.42499999999999999,"15min":0.36299999999999999},"max":{"1min":1.8580000000000001,"5min":10.052,"15min":15.919},"last":2.2829999999999999},{"interface":"front","average":{"1min":1.0129999999999999,"5min":2.0489999999999999,"15min":1.9410000000000001},"min":{"1min":0.44800000000000001,"5min":0.40400000000000003,"15min":0.315},"max":{"1min":1.5309999999999999,"5min":11.456,"15min":11.456},"last":0.69899999999999995}]},{"osd":7,"last update":"Sat Sep 23 19:51:54 2023","interfaces":[{"interface":"back","average":{"1min":1.0549999999999999,"5min":1.8959999999999999,"15min":1.9259999999999999},"min":{"1min":0.42899999999999999,"5min":0.42299999999999999,"15min":0.35799999999999998},"max":{"1min":1.5389999999999999,"5min":11.486000000000001,"15min":14.098000000000001},"last":1.0900000000000001},{"interface":"front","average":{"1min":1.0129999999999999,"5min":1.885,"15min":1.9239999999999999},"min":{"1min":0.70399999999999996,"5min":0.40600000000000003,"15min":0.40300000000000002},"max":{"1min":1.361,"5min":8.7059999999999995,"15min":11.744999999999999},"last":1.611}]}]},{"osd":4,"up_from":30,"seq":128849019495,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1455360,"kb_used_data":551352,"kb_used_omap":0,"kb_used_meta":904000,"kb_avail":92289792,"statfs":{"total":95995035648,"available":94504747008,"internally_reserved":0,"allocated":564584448,"data_stored":564060051,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":925696000},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:51:24 2023","interfaces":[{"interface":"back","average":{"1min":1.1839999999999999,"5min":2.9830000000000001,"15min":1.994},"min":{"1min":0.311,"5min":0.25600000000000001,"15min":0.23300000000000001},"max":{"1min":7.694,"5min":150.155,"15min":150.155},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.91800000000000004,"5min":2.8839999999999999,"15min":1.839},"min":{"1min":0.48399999999999999,"5min":0.28100000000000003,"15min":0.27700000000000002},"max":{"1min":2.0019999999999998,"5min":149.47900000000001,"15min":149.47900000000001},"last":0.78500000000000003}]},{"osd":1,"last update":"Sat Sep 23 19:51:46 2023","interfaces":[{"interface":"back","average":{"1min":0.91400000000000003,"5min":1.27,"15min":1.8300000000000001},"min":{"1min":0.60899999999999999,"5min":0.309,"15min":0.27800000000000002},"max":{"1min":1.3600000000000001,"5min":9.0589999999999993,"15min":101.786},"last":0.74199999999999999},{"interface":"front","average":{"1min":1.0549999999999999,"5min":1.248,"15min":1.982},"min":{"1min":0.71299999999999997,"5min":0.34899999999999998,"15min":0.28499999999999998},"max":{"1min":1.4490000000000001,"5min":9.0869999999999997,"15min":101.726},"last":0.64500000000000002}]},{"osd":2,"last update":"Sat Sep 23 19:51:05 2023","interfaces":[{"interface":"back","average":{"1min":1.6140000000000001,"5min":6.7439999999999998,"15min":4.141},"min":{"1min":0.35299999999999998,"5min":0.318,"15min":0.313},"max":{"1min":6.3819999999999997,"5min":219.02799999999999,"15min":219.02799999999999},"last":4.5259999999999998},{"interface":"front","average":{"1min":1.629,"5min":6.8369999999999997,"15min":3.7709999999999999},"min":{"1min":0.36199999999999999,"5min":0.318,"15min":0.29899999999999999},"max":{"1min":5.3700000000000001,"5min":219.65600000000001,"15min":219.65600000000001},"last":2.2610000000000001}]},{"osd":3,"last update":"Sat Sep 23 19:51:05 2023","interfaces":[{"interface":"back","average":{"1min":1.264,"5min":1.5149999999999999,"15min":1.986},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32600000000000001},"max":{"1min":6.0720000000000001,"5min":17.873999999999999,"15min":44.340000000000003},"last":1.1679999999999999},{"interface":"front","average":{"1min":1.3320000000000001,"5min":1.492,"15min":1.8939999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32100000000000001},"max":{"1min":4.5789999999999997,"5min":13.859,"15min":44.203000000000003},"last":1.0589999999999999}]},{"osd":5,"last update":"Sat Sep 23 19:51:24 2023","interfaces":[{"interface":"back","average":{"1min":0.86799999999999999,"5min":0.92700000000000005,"15min":1.4219999999999999},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.315},"max":{"1min":1.226,"5min":4.242,"15min":22.442},"last":0.435},{"interface":"front","average":{"1min":0.93600000000000005,"5min":1.008,"15min":1.401},"min":{"1min":0.41199999999999998,"5min":0.33900000000000002,"15min":0.28899999999999998},"max":{"1min":2.04,"5min":5.1440000000000001,"15min":10.362},"last":0.67400000000000004}]},{"osd":6,"last update":"Sat Sep 23 19:51:46 2023","interfaces":[{"interface":"back","average":{"1min":1.077,"5min":1.2789999999999999,"15min":1.696},"min":{"1min":0.69999999999999996,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":1.4690000000000001,"5min":18.103000000000002,"15min":22.544},"last":0.48299999999999998},{"interface":"front","average":{"1min":1.1359999999999999,"5min":1.327,"15min":1.53},"min":{"1min":0.68000000000000005,"5min":0.31,"15min":0.31},"max":{"1min":2.0830000000000002,"5min":19.706,"15min":19.706},"last":0.72299999999999998}]},{"osd":7,"last update":"Sat Sep 23 19:51:46 2023","interfaces":[{"interface":"back","average":{"1min":1.0429999999999999,"5min":1.248,"15min":2.5640000000000001},"min":{"1min":0.78100000000000003,"5min":0.33100000000000002,"15min":0.30299999999999999},"max":{"1min":1.3420000000000001,"5min":8.3399999999999999,"15min":294.76499999999999},"last":0.69599999999999995},{"interface":"front","average":{"1min":1.1830000000000001,"5min":1.2430000000000001,"15min":2.774},"min":{"1min":0.94099999999999995,"5min":0.39500000000000002,"15min":0.374},"max":{"1min":1.464,"5min":10.371,"15min":294.47699999999998},"last":0.52700000000000002}]}]},{"osd":5,"up_from":36,"seq":154618823262,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1400744,"kb_used_data":489760,"kb_used_omap":0,"kb_used_meta":910976,"kb_avail":92344408,"statfs":{"total":95995035648,"available":94560673792,"internally_reserved":0,"allocated":501514240,"data_stored":501009533,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":932839424},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:51:16 2023","interfaces":[{"interface":"back","average":{"1min":1.0920000000000001,"5min":1.5049999999999999,"15min":1.5620000000000001},"min":{"1min":0.311,"5min":0.28699999999999998,"15min":0.253},"max":{"1min":3.0009999999999999,"5min":28.280000000000001,"15min":43.094000000000001},"last":1.2190000000000001},{"interface":"front","average":{"1min":1.8340000000000001,"5min":1.8859999999999999,"15min":1.954},"min":{"1min":0.36099999999999999,"5min":0.252,"15min":0.252},"max":{"1min":6.9240000000000004,"5min":28.309999999999999,"15min":43.802999999999997},"last":1.6220000000000001}]},{"osd":1,"last update":"Sat Sep 23 19:51:43 2023","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":1.248,"15min":1.3560000000000001},"min":{"1min":0.60799999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":1.226,"5min":6.9660000000000002,"15min":11.266999999999999},"last":1.3160000000000001},{"interface":"front","average":{"1min":0.92500000000000004,"5min":1.3759999999999999,"15min":1.5820000000000001},"min":{"1min":0.68400000000000005,"5min":0.28799999999999998,"15min":0.27900000000000003},"max":{"1min":1.2549999999999999,"5min":9.0210000000000008,"15min":18.207000000000001},"last":3.7610000000000001}]},{"osd":2,"last update":"Sat Sep 23 19:51:00 2023","interfaces":[{"interface":"back","average":{"1min":1.6479999999999999,"5min":4.5970000000000004,"15min":3.4289999999999998},"min":{"1min":0.315,"5min":0.309,"15min":0.309},"max":{"1min":6.0309999999999997,"5min":241.59700000000001,"15min":241.59700000000001},"last":0.59399999999999997},{"interface":"front","average":{"1min":1.645,"5min":4.3040000000000003,"15min":2.9969999999999999},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.32000000000000001},"max":{"1min":4.282,"5min":242.13999999999999,"15min":242.13999999999999},"last":1.3620000000000001}]},{"osd":3,"last update":"Sat Sep 23 19:51:43 2023","interfaces":[{"interface":"back","average":{"1min":0.98599999999999999,"5min":1.222,"15min":1.4790000000000001},"min":{"1min":0.63200000000000001,"5min":0.35099999999999998,"15min":0.33700000000000002},"max":{"1min":1.226,"5min":7.4580000000000002,"15min":10.686},"last":0.495},{"interface":"front","average":{"1min":0.996,"5min":1.234,"15min":1.5700000000000001},"min":{"1min":0.61399999999999999,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":1.284,"5min":6.1870000000000003,"15min":8.5879999999999992},"last":1.252}]},{"osd":4,"last update":"Sat Sep 23 19:51:16 2023","interfaces":[{"interface":"back","average":{"1min":1.3169999999999999,"5min":0.98099999999999998,"15min":1.556},"min":{"1min":0.377,"5min":0.307,"15min":0.307},"max":{"1min":6.585,"5min":6.585,"15min":74.221000000000004},"last":3.7959999999999998},{"interface":"front","average":{"1min":1.0880000000000001,"5min":0.86899999999999999,"15min":1.5780000000000001},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":6.266,"5min":6.266,"15min":73.772999999999996},"last":2.4369999999999998}]},{"osd":6,"last update":"Sat Sep 23 19:51:00 2023","interfaces":[{"interface":"back","average":{"1min":1.0860000000000001,"5min":0.999,"15min":1.3979999999999999},"min":{"1min":0.371,"5min":0.36099999999999999,"15min":0.35599999999999998},"max":{"1min":3.2480000000000002,"5min":3.8010000000000002,"15min":16.222000000000001},"last":1.288},{"interface":"front","average":{"1min":1.355,"5min":1.1399999999999999,"15min":1.534},"min":{"1min":0.39600000000000002,"5min":0.39300000000000002,"15min":0.309},"max":{"1min":5.984,"5min":5.984,"15min":14.178000000000001},"last":0.436}]},{"osd":7,"last update":"Sat Sep 23 19:51:16 2023","interfaces":[{"interface":"back","average":{"1min":1.1950000000000001,"5min":1.069,"15min":1.4730000000000001},"min":{"1min":0.437,"5min":0.35299999999999998,"15min":0.32100000000000001},"max":{"1min":5.9249999999999998,"5min":9.8819999999999997,"15min":14.127000000000001},"last":0.46999999999999997},{"interface":"front","average":{"1min":1.115,"5min":1.0509999999999999,"15min":1.4299999999999999},"min":{"1min":0.53500000000000003,"5min":0.33900000000000002,"15min":0.307},"max":{"1min":3.3479999999999999,"5min":5.5439999999999996,"15min":9.0440000000000005},"last":2.0569999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":159711232,"data_stored":159711232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":206372864,"data_stored":206372864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":585957376,"data_stored":585957376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":178716672,"data_stored":178716672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":614727680,"data_stored":614727680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":556466176,"data_stored":556466176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":353042432,"data_stored":353042432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":349241344,"data_stored":349241344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T19:52:06.749 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T19:52:06.779 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:06.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:06.781 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:06.882 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:06.912 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:06.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:06.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:07.016 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:07.041 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:07.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:07.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:07.145 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:07.173 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:07.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:07.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:07.277 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:07.307 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:07.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:07.314 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:07.415 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:07.448 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:07.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:07.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:52:07.476 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 15 16 29883 29867 124.434 92.3125 0.00509083 0.00803048 2023-09-23T19:52:07.551 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:07.576 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:07.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:07.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:07.679 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:07.713 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:07.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:07.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:07.816 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:07.843 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:07.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:07.846 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:07.947 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:07.980 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:07.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:07.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:08.083 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:08.110 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:08.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:08.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:08.214 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:08.240 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:08.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:08.243 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:08.344 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:08.353 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:08 smithi092 bash[58855]: cluster 2023-09-23T19:52:06.447933+0000 mgr.y (mgr.25325) 1343 : cluster 0 pgmap v1346: 89 pgs: 89 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2023-09-23T19:52:08.378 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:08.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:08.380 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:08.476 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 16 16 31551 31535 123.172 104.25 0.00878986 0.00811174 2023-09-23T19:52:08.480 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:08.491 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:08 smithi016 bash[77995]: cluster 2023-09-23T19:52:06.447933+0000 mgr.y (mgr.25325) 1343 : cluster 0 pgmap v1346: 89 pgs: 89 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2023-09-23T19:52:08.491 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:08 smithi016 bash[75004]: cluster 2023-09-23T19:52:06.447933+0000 mgr.y (mgr.25325) 1343 : cluster 0 pgmap v1346: 89 pgs: 89 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2023-09-23T19:52:08.508 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:08.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:08.514 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:08.614 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:08.640 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:08.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:08.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:08.743 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:08.773 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:08.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:08.775 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:08.876 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:08.907 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:08.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:08.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:52:09.011 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:09.037 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:09.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:09.040 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:09.140 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:09.169 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:09.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:09.173 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:52:09.274 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:09.311 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:09.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:09.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:09.416 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:09.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:09.451 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:09.451 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:09.476 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 17 16 33163 33147 121.853 100.75 0.0150436 0.00820067 2023-09-23T19:52:09.551 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:09.583 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:09.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:09.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:09.685 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:09.694 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:52:09.710 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:52:09.723 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:09.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:09.729 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:09.751 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:52:09.811 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:52:09.829 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:09.843 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:52:09.860 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:09.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:09.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:09.887 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:52:09.923 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:52:09.964 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:52:09.964 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:09.990 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:09.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:09.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:09.997 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:52:10.003 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:52:10.003 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:52:10.003 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:52:10.003 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:52:10.003 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:52:10.003 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:52:10.003 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:52:10.004 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:52:10.097 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:10.126 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:10.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:10.129 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:10.230 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:10.239 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:10 smithi092 bash[58855]: cluster 2023-09-23T19:52:08.448526+0000 mgr.y (mgr.25325) 1344 : cluster 0 pgmap v1347: 89 pgs: 89 active+clean; 1.6 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s 2023-09-23T19:52:10.239 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:10 smithi092 bash[58855]: audit 2023-09-23T19:52:09.666693+0000 mon.a (mon.0) 1667 : audit 0 from='client.? 172.21.15.16:0/3487984019' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:52:10.240 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:10 smithi092 bash[58855]: audit 2023-09-23T19:52:09.796056+0000 mon.c (mon.2) 211 : audit 1 from='client.? 172.21.15.16:0/2913229025' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:52:10.240 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:10 smithi092 bash[58855]: audit 2023-09-23T19:52:09.796723+0000 mon.a (mon.0) 1668 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:52:10.257 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:10.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:10.260 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:10.361 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:10.371 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:10 smithi016 bash[75004]: cluster 2023-09-23T19:52:08.448526+0000 mgr.y (mgr.25325) 1344 : cluster 0 pgmap v1347: 89 pgs: 89 active+clean; 1.6 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s 2023-09-23T19:52:10.371 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:10 smithi016 bash[75004]: audit 2023-09-23T19:52:09.666693+0000 mon.a (mon.0) 1667 : audit 0 from='client.? 172.21.15.16:0/3487984019' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:52:10.371 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:10 smithi016 bash[75004]: audit 2023-09-23T19:52:09.796056+0000 mon.c (mon.2) 211 : audit 1 from='client.? 172.21.15.16:0/2913229025' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:52:10.371 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:10 smithi016 bash[75004]: audit 2023-09-23T19:52:09.796723+0000 mon.a (mon.0) 1668 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:52:10.371 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:10 smithi016 bash[77995]: cluster 2023-09-23T19:52:08.448526+0000 mgr.y (mgr.25325) 1344 : cluster 0 pgmap v1347: 89 pgs: 89 active+clean; 1.6 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s 2023-09-23T19:52:10.371 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:10 smithi016 bash[77995]: audit 2023-09-23T19:52:09.666693+0000 mon.a (mon.0) 1667 : audit 0 from='client.? 172.21.15.16:0/3487984019' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:52:10.371 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:10 smithi016 bash[77995]: audit 2023-09-23T19:52:09.796056+0000 mon.c (mon.2) 211 : audit 1 from='client.? 172.21.15.16:0/2913229025' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:52:10.372 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:10 smithi016 bash[77995]: audit 2023-09-23T19:52:09.796723+0000 mon.a (mon.0) 1668 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:52:10.391 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:10.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:10.394 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:10.476 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 18 16 34576 34560 119.989 88.3125 0.00695086 0.00832758 2023-09-23T19:52:10.494 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:10.526 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:10.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:10.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:10.631 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:10.669 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:10.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:10.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:10.685 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is unset 2023-09-23T19:52:10.773 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:10.803 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:10.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:10.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:10.911 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:10.936 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:10.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:10.940 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:52:11.041 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:11.073 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:11.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:11.077 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:11.178 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:11.211 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:11.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:11.219 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:11.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:11 smithi016 bash[75004]: cluster 2023-09-23T19:52:10.663728+0000 mon.a (mon.0) 1669 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:52:11.288 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:11 smithi016 bash[75004]: audit 2023-09-23T19:52:10.670233+0000 mon.a (mon.0) 1670 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:52:11.288 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:11 smithi016 bash[75004]: cluster 2023-09-23T19:52:10.670348+0000 mon.a (mon.0) 1671 : cluster 0 osdmap e238: 8 total, 8 up, 8 in 2023-09-23T19:52:11.288 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:11 smithi016 bash[75004]: cluster 2023-09-23T19:52:10.692090+0000 osd.7 (osd.7) 156 : cluster 3 failed to encode map e238 with expected crc 2023-09-23T19:52:11.288 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:11 smithi016 bash[77995]: cluster 2023-09-23T19:52:10.663728+0000 mon.a (mon.0) 1669 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:52:11.289 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:11 smithi016 bash[77995]: audit 2023-09-23T19:52:10.670233+0000 mon.a (mon.0) 1670 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:52:11.289 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:11 smithi016 bash[77995]: cluster 2023-09-23T19:52:10.670348+0000 mon.a (mon.0) 1671 : cluster 0 osdmap e238: 8 total, 8 up, 8 in 2023-09-23T19:52:11.289 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:11 smithi016 bash[77995]: cluster 2023-09-23T19:52:10.692090+0000 osd.7 (osd.7) 156 : cluster 3 failed to encode map e238 with expected crc 2023-09-23T19:52:11.319 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:11.344 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:11.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:11.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:11.448 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:11.456 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:11 smithi092 bash[58855]: cluster 2023-09-23T19:52:10.663728+0000 mon.a (mon.0) 1669 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:52:11.456 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:11 smithi092 bash[58855]: audit 2023-09-23T19:52:10.670233+0000 mon.a (mon.0) 1670 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:52:11.457 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:11 smithi092 bash[58855]: cluster 2023-09-23T19:52:10.670348+0000 mon.a (mon.0) 1671 : cluster 0 osdmap e238: 8 total, 8 up, 8 in 2023-09-23T19:52:11.457 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:11 smithi092 bash[58855]: cluster 2023-09-23T19:52:10.692090+0000 osd.7 (osd.7) 156 : cluster 3 failed to encode map e238 with expected crc 2023-09-23T19:52:11.473 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:11.476 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 19 16 36070 36054 118.588 93.375 0.0143069 0.00842539 2023-09-23T19:52:11.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:11.479 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:11.580 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:11.612 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:11.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:11.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:11.716 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:11.742 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:11.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:11.745 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:11.846 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:11.870 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:11.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:11.876 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:11.976 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:12.006 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:12.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:12.009 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:12.109 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:12.142 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:12.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:12.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:52:12.248 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:12.271 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:12.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:12.274 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:12.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:12 smithi092 bash[58855]: cluster 2023-09-23T19:52:10.449218+0000 mgr.y (mgr.25325) 1345 : cluster 0 pgmap v1348: 89 pgs: 89 active+clean; 2.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 135 MiB/s wr, 2.16k op/s 2023-09-23T19:52:12.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:12 smithi092 bash[58855]: cluster 2023-09-23T19:52:10.686088+0000 osd.4 (osd.4) 181 : cluster 3 failed to encode map e238 with expected crc 2023-09-23T19:52:12.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:12 smithi092 bash[58855]: cluster 2023-09-23T19:52:10.687500+0000 osd.5 (osd.5) 166 : cluster 3 failed to encode map e238 with expected crc 2023-09-23T19:52:12.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:12 smithi092 bash[58855]: cluster 2023-09-23T19:52:10.688774+0000 osd.6 (osd.6) 174 : cluster 3 failed to encode map e238 with expected crc 2023-09-23T19:52:12.374 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:12.384 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:12 smithi016 bash[75004]: cluster 2023-09-23T19:52:10.449218+0000 mgr.y (mgr.25325) 1345 : cluster 0 pgmap v1348: 89 pgs: 89 active+clean; 2.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 135 MiB/s wr, 2.16k op/s 2023-09-23T19:52:12.384 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:12 smithi016 bash[75004]: cluster 2023-09-23T19:52:10.686088+0000 osd.4 (osd.4) 181 : cluster 3 failed to encode map e238 with expected crc 2023-09-23T19:52:12.384 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:12 smithi016 bash[75004]: cluster 2023-09-23T19:52:10.687500+0000 osd.5 (osd.5) 166 : cluster 3 failed to encode map e238 with expected crc 2023-09-23T19:52:12.384 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:12 smithi016 bash[75004]: cluster 2023-09-23T19:52:10.688774+0000 osd.6 (osd.6) 174 : cluster 3 failed to encode map e238 with expected crc 2023-09-23T19:52:12.384 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:12 smithi016 bash[77995]: cluster 2023-09-23T19:52:10.449218+0000 mgr.y (mgr.25325) 1345 : cluster 0 pgmap v1348: 89 pgs: 89 active+clean; 2.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 135 MiB/s wr, 2.16k op/s 2023-09-23T19:52:12.384 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:12 smithi016 bash[77995]: cluster 2023-09-23T19:52:10.686088+0000 osd.4 (osd.4) 181 : cluster 3 failed to encode map e238 with expected crc 2023-09-23T19:52:12.384 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:12 smithi016 bash[77995]: cluster 2023-09-23T19:52:10.687500+0000 osd.5 (osd.5) 166 : cluster 3 failed to encode map e238 with expected crc 2023-09-23T19:52:12.385 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:12 smithi016 bash[77995]: cluster 2023-09-23T19:52:10.688774+0000 osd.6 (osd.6) 174 : cluster 3 failed to encode map e238 with expected crc 2023-09-23T19:52:12.404 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:12.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:12.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:52:12.476 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-09-23T19:52:12.472266+0000 min lat: 0.00144627 max lat: 0.277643 avg lat: 0.00851823 2023-09-23T19:52:12.477 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:52:12.477 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 20 16 37555 37539 117.299 92.8125 0.00834664 0.00851823 2023-09-23T19:52:12.507 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:12.539 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:12.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:12.545 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:12.645 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:12.678 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:12.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:12.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:12.782 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:12.811 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:12.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:12.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:12.914 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:12.946 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:12.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:12.950 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:52:13.051 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:13.086 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:13.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:13.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:52:13.190 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:13.225 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:13.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:13.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:13.331 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:13.357 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:13.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:13.361 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:52:13.461 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:13.476 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 21 16 39058 39042 116.186 93.9375 0.00602778 0.00859991 2023-09-23T19:52:13.491 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:13.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:13.496 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:13.597 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:13.630 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:13.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:13.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:13.734 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:13.743 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:52:13.759 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:13.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:13.761 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:13.862 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:13.893 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:13.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:13.896 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:13.997 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:14.024 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:14.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:14.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:14.084 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:52:14.129 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:14.154 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:14.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:14.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:14.257 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:14.266 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:14 smithi016 bash[77995]: cluster 2023-09-23T19:52:12.449843+0000 mgr.y (mgr.25325) 1346 : cluster 0 pgmap v1350: 89 pgs: 89 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2023-09-23T19:52:14.266 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:14 smithi016 bash[77995]: audit 2023-09-23T19:52:13.675727+0000 mon.a (mon.0) 1672 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:52:14.266 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:14 smithi016 bash[75004]: cluster 2023-09-23T19:52:12.449843+0000 mgr.y (mgr.25325) 1346 : cluster 0 pgmap v1350: 89 pgs: 89 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2023-09-23T19:52:14.266 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:14 smithi016 bash[75004]: audit 2023-09-23T19:52:13.675727+0000 mon.a (mon.0) 1672 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:52:14.284 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:14.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:14.287 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:14.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:14 smithi092 bash[58855]: cluster 2023-09-23T19:52:12.449843+0000 mgr.y (mgr.25325) 1346 : cluster 0 pgmap v1350: 89 pgs: 89 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2023-09-23T19:52:14.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:14 smithi092 bash[58855]: audit 2023-09-23T19:52:13.675727+0000 mon.a (mon.0) 1672 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:52:14.388 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:14.415 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:14.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:14.418 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:14.476 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 22 16 40494 40478 114.984 89.75 0.00557489 0.00868969 2023-09-23T19:52:14.519 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:14.546 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:14.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:14.550 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:14.597 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T19:52:14.652 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:14.688 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:14.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:14.692 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:52:14.793 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:14.824 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:14.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:14.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:52:14.928 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:14.936 INFO:teuthology.orchestra.run.smithi092.stdout:{ 2023-09-23T19:52:14.936 INFO:teuthology.orchestra.run.smithi092.stdout: "ops": [], 2023-09-23T19:52:14.936 INFO:teuthology.orchestra.run.smithi092.stdout: "complaint_time": 30, 2023-09-23T19:52:14.936 INFO:teuthology.orchestra.run.smithi092.stdout: "num_blocked_ops": 0 2023-09-23T19:52:14.936 INFO:teuthology.orchestra.run.smithi092.stdout:} 2023-09-23T19:52:14.954 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:14.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:14.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:15.059 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:15.088 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:15.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:15.091 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:15.191 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:15.217 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:15.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:15.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:15.320 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:15.352 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:15.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:15.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:52:15.456 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:15.476 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 23 16 41929 41913 113.883 89.6875 0.0199134 0.00877322 2023-09-23T19:52:15.487 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:15.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:15.491 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:52:15.591 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:15.622 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:15.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:15.626 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:15.727 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:15.756 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:15.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:15.759 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:15.860 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:15.870 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:52:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:52:15] "GET /metrics HTTP/1.1" 200 36299 "" "Prometheus/2.33.4" 2023-09-23T19:52:15.892 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:15.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:15.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:15.995 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:16.024 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:16.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:16.029 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:52:16.129 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:16.140 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:16 smithi016 bash[75004]: cluster 2023-09-23T19:52:14.450485+0000 mgr.y (mgr.25325) 1347 : cluster 0 pgmap v1351: 89 pgs: 89 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2023-09-23T19:52:16.140 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:16 smithi016 bash[77995]: cluster 2023-09-23T19:52:14.450485+0000 mgr.y (mgr.25325) 1347 : cluster 0 pgmap v1351: 89 pgs: 89 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2023-09-23T19:52:16.163 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:16.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:16.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:16.266 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:16.293 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:16.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:16.296 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:16.314 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T19:52:16.397 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:16.436 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:16.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:16.439 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:16.476 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 24 16 43236 43220 112.542 81.6875 0.00432012 0.00887792 2023-09-23T19:52:16.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:16 smithi092 bash[58855]: cluster 2023-09-23T19:52:14.450485+0000 mgr.y (mgr.25325) 1347 : cluster 0 pgmap v1351: 89 pgs: 89 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2023-09-23T19:52:16.540 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:16.573 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:16.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:16.576 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:16.677 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:16.709 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:16.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:16.713 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:16.814 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:16.850 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:16.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:16.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:52:16.955 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:16.983 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:16.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:16.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:17.086 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:17.117 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:17.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:17.120 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:17.221 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:17.248 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:17.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:17.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:17.352 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:17.378 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:17.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:17.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:17.477 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 25 16 44635 44619 111.537 87.4375 0.0118361 0.00895861 2023-09-23T19:52:17.481 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:17.507 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:17.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:17.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:17.610 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:17.648 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:17.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:17.650 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:17.751 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:17.777 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:17.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:17.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:17.886 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:17.916 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:17.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:17.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:52:18.019 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:18.046 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:18.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:18.048 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:18.149 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:18.176 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:18.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:18.180 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:18.280 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:18.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:18 smithi016 bash[77995]: cluster 2023-09-23T19:52:16.451241+0000 mgr.y (mgr.25325) 1348 : cluster 0 pgmap v1352: 89 pgs: 89 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-09-23T19:52:18.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:18 smithi016 bash[75004]: cluster 2023-09-23T19:52:16.451241+0000 mgr.y (mgr.25325) 1348 : cluster 0 pgmap v1352: 89 pgs: 89 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-09-23T19:52:18.302 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:52:18.310 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:18.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:18.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:18.416 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:18.425 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:18 smithi092 bash[58855]: cluster 2023-09-23T19:52:16.451241+0000 mgr.y (mgr.25325) 1348 : cluster 0 pgmap v1352: 89 pgs: 89 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-09-23T19:52:18.443 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:18.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:18.448 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:18.477 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 26 16 45984 45968 110.49 84.3125 0.00903887 0.0090433 2023-09-23T19:52:18.548 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:18.579 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:18.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:18.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:18.684 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:18.711 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:18.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:18.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:18.817 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:18.847 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:18.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:18.849 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:52:18.950 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:18.979 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:18.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:18.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:19.083 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:19.119 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:19.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:19.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:19.225 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:19.254 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:19.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:19.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:52:19.360 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:19.389 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:19.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:19.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:19.477 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 27 16 47329 47313 109.511 84.0625 0.0086917 0.00912511 2023-09-23T19:52:19.495 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:19.524 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:19.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:19.528 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:19.628 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:19.659 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:19.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:19.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:19.764 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:19.790 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:19.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:19.792 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:19.893 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:19.916 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:19.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:19.918 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:20.019 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:20.045 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:20.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:20.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:20.148 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:20.179 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:20.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:20.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:20.194 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:52:20.287 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:20.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:20 smithi092 bash[58855]: cluster 2023-09-23T19:52:18.454211+0000 mgr.y (mgr.25325) 1349 : cluster 0 pgmap v1353: 89 pgs: 89 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 95 MiB/s wr, 1.53k op/s 2023-09-23T19:52:20.317 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:20.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:20.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:20.423 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:20.451 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:20.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:20.456 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:20.477 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 28 16 48591 48575 108.416 78.875 0.0142337 0.00921668 2023-09-23T19:52:20.558 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:20.586 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:20.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:20.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:20.603 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:20 smithi016 bash[77995]: cluster 2023-09-23T19:52:18.454211+0000 mgr.y (mgr.25325) 1349 : cluster 0 pgmap v1353: 89 pgs: 89 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 95 MiB/s wr, 1.53k op/s 2023-09-23T19:52:20.604 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:20 smithi016 bash[75004]: cluster 2023-09-23T19:52:18.454211+0000 mgr.y (mgr.25325) 1349 : cluster 0 pgmap v1353: 89 pgs: 89 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 95 MiB/s wr, 1.53k op/s 2023-09-23T19:52:20.692 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:20.714 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:20.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:20.716 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:20.817 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:20.844 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:20.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:20.848 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:52:20.949 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:20.985 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:20.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:20.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:52:21.089 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:21.113 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:21.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:21.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:21.220 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:21.221 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:52:21.248 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:21.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:21.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:52:21.351 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:21.377 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:21.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:21.386 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:21.477 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 29 16 49883 49867 107.462 80.75 0.00627408 0.00929818 2023-09-23T19:52:21.487 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:21.514 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:21.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:21.516 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:21.617 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:21.639 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:21.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:21.642 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:52:21.742 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:21.768 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:21.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:21.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:21.871 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:21.902 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:21.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:21.905 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:52:22.006 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:22.033 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:22.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:22.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:22.138 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:22.164 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:22.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:22.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:22.225 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:22 smithi016 bash[75004]: cluster 2023-09-23T19:52:20.454868+0000 mgr.y (mgr.25325) 1350 : cluster 0 pgmap v1354: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-09-23T19:52:22.226 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:22 smithi016 bash[77995]: cluster 2023-09-23T19:52:20.454868+0000 mgr.y (mgr.25325) 1350 : cluster 0 pgmap v1354: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-09-23T19:52:22.269 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:22.299 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:22.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:22.304 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:22.405 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:22.431 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:22.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:22.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:22.477 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 30 16 51047 51031 106.305 72.75 0.00691354 0.00940183 2023-09-23T19:52:22.535 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:22.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:22 smithi092 bash[58855]: cluster 2023-09-23T19:52:20.454868+0000 mgr.y (mgr.25325) 1350 : cluster 0 pgmap v1354: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-09-23T19:52:22.562 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:22.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:22.564 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:22.665 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:22.691 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:22.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:22.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:52:22.795 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:22.829 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:22.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:22.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:52:22.932 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:22.958 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:22.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:22.961 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:23.061 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:23.090 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:23.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:23.092 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:23.192 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:23.229 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:23.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:23.233 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:23.334 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:23.365 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:23.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:23.368 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:52:23.443 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-09-23T19:52:23.456 INFO:teuthology.orchestra.run.smithi016.stderr:dumped all 2023-09-23T19:52:23.469 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:23.477 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 31 16 52063 52047 104.924 63.5 0.0114533 0.00952315 2023-09-23T19:52:23.497 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:23.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:23.504 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:23.604 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:23.627 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:23.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:23.630 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:23.731 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:23.756 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:23.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:23.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:23.861 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:23.892 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:23.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:23.894 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:23.995 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:24.033 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:24.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:24.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:24.139 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:24.171 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:24.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:24.176 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:24.277 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:24.309 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:24.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:24.311 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:52:24.412 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:24.422 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:24 smithi016 bash[77995]: cluster 2023-09-23T19:52:22.455579+0000 mgr.y (mgr.25325) 1351 : cluster 0 pgmap v1355: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-09-23T19:52:24.422 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:24 smithi016 bash[75004]: cluster 2023-09-23T19:52:22.455579+0000 mgr.y (mgr.25325) 1351 : cluster 0 pgmap v1355: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-09-23T19:52:24.437 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:24.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:24.440 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:24.441 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:24 smithi092 bash[58855]: cluster 2023-09-23T19:52:22.455579+0000 mgr.y (mgr.25325) 1351 : cluster 0 pgmap v1355: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-09-23T19:52:24.477 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 32 16 53301 53285 104.063 77.375 0.00857607 0.00960362 2023-09-23T19:52:24.541 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:24.566 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:24.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:24.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:52:24.669 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:24.703 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:24.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:24.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:52:24.808 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:24.839 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:24.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:24.843 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:52:24.943 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:24.966 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:24.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:24.973 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:25.074 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:25.102 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:25.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:25.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:25.194 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:25 smithi092 bash[58855]: audit 2023-09-23T19:52:23.412405+0000 mgr.y (mgr.25325) 1352 : audit 0 from='client.46664 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:52:25.208 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:25.217 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:25 smithi016 bash[77995]: audit 2023-09-23T19:52:23.412405+0000 mgr.y (mgr.25325) 1352 : audit 0 from='client.46664 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:52:25.217 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:25 smithi016 bash[75004]: audit 2023-09-23T19:52:23.412405+0000 mgr.y (mgr.25325) 1352 : audit 0 from='client.46664 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:52:25.235 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:25.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:25.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:52:25.339 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:25.366 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:25.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:25.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:52:25.473 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:25.477 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 33 16 54586 54570 103.343 80.3125 0.0100846 0.00966873 2023-09-23T19:52:25.498 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:25.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:25.501 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:52:25.601 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:25.626 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:25.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:25.628 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:52:25.728 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:25.737 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:52:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:52:25] "GET /metrics HTTP/1.1" 200 36299 "" "Prometheus/2.33.4" 2023-09-23T19:52:25.755 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:25.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:25.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:25.860 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:25.888 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:25.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:25.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:52:25.992 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:26.022 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:26.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:26.026 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:26.127 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:26.158 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:26.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:26.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:26.250 INFO:teuthology.orchestra.run.smithi016.stdout:{"pg_ready":true,"pg_map":{"version":1355,"stamp":"2023-09-23T19:52:22.455123+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3129743264,"num_objects":47771,"num_object_clones":0,"num_object_copies":143313,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":47771,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":48008,"num_write_kb":3058254,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":47942,"ondisk_log_size":47942,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":15184900,"kb_used_data":9024516,"kb_used_omap":20,"kb_used_meta":6160299,"kb_avail":734776316,"statfs":{"total":767960285184,"available":752410947584,"internally_reserved":0,"allocated":9241104384,"data_stored":9235259993,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":6308146830},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[12,3,7,3,5,6],"upper_bound":64},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":989986816,"num_objects":15106,"num_object_clones":0,"num_object_copies":45318,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15106,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15106,"num_write_kb":966784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"11.788818"},"pg_stats":[{"pgid":"3.1a","version":"59'1","reported_seq":234,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:51.521592+0000","last_change":"2023-09-23T19:38:43.897548+0000","last_active":"2023-09-23T19:51:51.521592+0000","last_peered":"2023-09-23T19:51:51.521592+0000","last_clean":"2023-09-23T19:51:51.521592+0000","last_became_active":"2023-09-23T19:38:43.896868+0000","last_became_peered":"2023-09-23T19:38:43.896868+0000","last_unstale":"2023-09-23T19:51:51.521592+0000","last_undegraded":"2023-09-23T19:51:51.521592+0000","last_fullsized":"2023-09-23T19:51:51.521592+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":198,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:49.517167+0000","last_change":"2023-09-23T19:32:49.731128+0000","last_active":"2023-09-23T19:51:49.517167+0000","last_peered":"2023-09-23T19:51:49.517167+0000","last_clean":"2023-09-23T19:51:49.517167+0000","last_became_active":"2023-09-23T19:32:49.730757+0000","last_became_peered":"2023-09-23T19:32:49.730757+0000","last_unstale":"2023-09-23T19:51:49.517167+0000","last_undegraded":"2023-09-23T19:51:49.517167+0000","last_fullsized":"2023-09-23T19:51:49.517167+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:40.183605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":187,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:49.516564+0000","last_change":"2023-09-23T19:28:07.890462+0000","last_active":"2023-09-23T19:51:49.516564+0000","last_peered":"2023-09-23T19:51:49.516564+0000","last_clean":"2023-09-23T19:51:49.516564+0000","last_became_active":"2023-09-23T19:28:07.890084+0000","last_became_peered":"2023-09-23T19:28:07.890084+0000","last_unstale":"2023-09-23T19:51:49.516564+0000","last_undegraded":"2023-09-23T19:51:49.516564+0000","last_fullsized":"2023-09-23T19:51:49.516564+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:50.458218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":124,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:51.522844+0000","last_change":"2023-09-23T19:42:55.772143+0000","last_active":"2023-09-23T19:51:51.522844+0000","last_peered":"2023-09-23T19:51:51.522844+0000","last_clean":"2023-09-23T19:51:51.522844+0000","last_became_active":"2023-09-23T19:42:55.771813+0000","last_became_peered":"2023-09-23T19:42:55.771813+0000","last_unstale":"2023-09-23T19:51:51.522844+0000","last_undegraded":"2023-09-23T19:51:51.522844+0000","last_fullsized":"2023-09-23T19:51:51.522844+0000","mapping_epoch":188,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":216,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T19:52:10.677330+0000","last_change":"2023-09-23T19:32:49.729961+0000","last_active":"2023-09-23T19:52:10.677330+0000","last_peered":"2023-09-23T19:52:10.677330+0000","last_clean":"2023-09-23T19:52:10.677330+0000","last_became_active":"2023-09-23T19:32:49.729251+0000","last_became_peered":"2023-09-23T19:32:49.729251+0000","last_unstale":"2023-09-23T19:52:10.677330+0000","last_undegraded":"2023-09-23T19:52:10.677330+0000","last_fullsized":"2023-09-23T19:52:10.677330+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:20:56.595732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":203,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:51.358637+0000","last_change":"2023-09-23T19:22:42.150582+0000","last_active":"2023-09-23T19:51:51.358637+0000","last_peered":"2023-09-23T19:51:51.358637+0000","last_clean":"2023-09-23T19:51:51.358637+0000","last_became_active":"2023-09-23T19:22:42.150229+0000","last_became_peered":"2023-09-23T19:22:42.150229+0000","last_unstale":"2023-09-23T19:51:51.358637+0000","last_undegraded":"2023-09-23T19:51:51.358637+0000","last_fullsized":"2023-09-23T19:51:51.358637+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":182,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:49.804182+0000","last_change":"2023-09-23T19:47:56.793944+0000","last_active":"2023-09-23T19:51:49.804182+0000","last_peered":"2023-09-23T19:51:49.804182+0000","last_clean":"2023-09-23T19:51:49.804182+0000","last_became_active":"2023-09-23T19:47:56.793618+0000","last_became_peered":"2023-09-23T19:47:56.793618+0000","last_unstale":"2023-09-23T19:51:49.804182+0000","last_undegraded":"2023-09-23T19:51:49.804182+0000","last_fullsized":"2023-09-23T19:51:49.804182+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":199,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:49.517169+0000","last_change":"2023-09-23T19:32:49.729765+0000","last_active":"2023-09-23T19:51:49.517169+0000","last_peered":"2023-09-23T19:51:49.517169+0000","last_clean":"2023-09-23T19:51:49.517169+0000","last_became_active":"2023-09-23T19:32:49.729007+0000","last_became_peered":"2023-09-23T19:32:49.729007+0000","last_unstale":"2023-09-23T19:51:49.517169+0000","last_undegraded":"2023-09-23T19:51:49.517169+0000","last_fullsized":"2023-09-23T19:51:49.517169+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:59:31.183753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":190,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:49.803990+0000","last_change":"2023-09-23T19:47:56.791464+0000","last_active":"2023-09-23T19:51:49.803990+0000","last_peered":"2023-09-23T19:51:49.803990+0000","last_clean":"2023-09-23T19:51:49.803990+0000","last_became_active":"2023-09-23T19:47:56.791144+0000","last_became_peered":"2023-09-23T19:47:56.791144+0000","last_unstale":"2023-09-23T19:51:49.803990+0000","last_undegraded":"2023-09-23T19:51:49.803990+0000","last_fullsized":"2023-09-23T19:51:49.803990+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":178,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:51.521375+0000","last_change":"2023-09-23T19:32:49.736864+0000","last_active":"2023-09-23T19:51:51.521375+0000","last_peered":"2023-09-23T19:51:51.521375+0000","last_clean":"2023-09-23T19:51:51.521375+0000","last_became_active":"2023-09-23T19:32:49.736523+0000","last_became_peered":"2023-09-23T19:32:49.736523+0000","last_unstale":"2023-09-23T19:51:51.521375+0000","last_undegraded":"2023-09-23T19:51:51.521375+0000","last_fullsized":"2023-09-23T19:51:51.521375+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":207,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:51.522086+0000","last_change":"2023-09-23T19:28:07.892835+0000","last_active":"2023-09-23T19:51:51.522086+0000","last_peered":"2023-09-23T19:51:51.522086+0000","last_clean":"2023-09-23T19:51:51.522086+0000","last_became_active":"2023-09-23T19:28:07.892338+0000","last_became_peered":"2023-09-23T19:28:07.892338+0000","last_unstale":"2023-09-23T19:51:51.522086+0000","last_undegraded":"2023-09-23T19:51:51.522086+0000","last_fullsized":"2023-09-23T19:51:51.522086+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":202,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:49.517115+0000","last_change":"2023-09-23T19:38:43.890137+0000","last_active":"2023-09-23T19:51:49.517115+0000","last_peered":"2023-09-23T19:51:49.517115+0000","last_clean":"2023-09-23T19:51:49.517115+0000","last_became_active":"2023-09-23T19:38:43.889800+0000","last_became_peered":"2023-09-23T19:38:43.889800+0000","last_unstale":"2023-09-23T19:51:49.517115+0000","last_undegraded":"2023-09-23T19:51:49.517115+0000","last_fullsized":"2023-09-23T19:51:49.517115+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:59:20.468913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'23","reported_seq":211,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:51.521286+0000","last_change":"2023-09-23T19:05:52.107727+0000","last_active":"2023-09-23T19:51:51.521286+0000","last_peered":"2023-09-23T19:51:51.521286+0000","last_clean":"2023-09-23T19:51:51.521286+0000","last_became_active":"2023-09-23T19:05:52.107378+0000","last_became_peered":"2023-09-23T19:05:52.107378+0000","last_unstale":"2023-09-23T19:51:51.521286+0000","last_undegraded":"2023-09-23T19:51:51.521286+0000","last_fullsized":"2023-09-23T19:51:51.521286+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":202,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:51.521378+0000","last_change":"2023-09-23T19:22:42.148497+0000","last_active":"2023-09-23T19:51:51.521378+0000","last_peered":"2023-09-23T19:51:51.521378+0000","last_clean":"2023-09-23T19:51:51.521378+0000","last_became_active":"2023-09-23T19:22:42.148147+0000","last_became_peered":"2023-09-23T19:22:42.148147+0000","last_unstale":"2023-09-23T19:51:51.521378+0000","last_undegraded":"2023-09-23T19:51:51.521378+0000","last_fullsized":"2023-09-23T19:51:51.521378+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":221,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:51.522540+0000","last_change":"2023-09-23T19:38:43.897378+0000","last_active":"2023-09-23T19:51:51.522540+0000","last_peered":"2023-09-23T19:51:51.522540+0000","last_clean":"2023-09-23T19:51:51.522540+0000","last_became_active":"2023-09-23T19:38:43.897012+0000","last_became_peered":"2023-09-23T19:38:43.897012+0000","last_unstale":"2023-09-23T19:51:51.522540+0000","last_undegraded":"2023-09-23T19:51:51.522540+0000","last_fullsized":"2023-09-23T19:51:51.522540+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":199,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:49.517626+0000","last_change":"2023-09-23T19:32:49.730549+0000","last_active":"2023-09-23T19:51:49.517626+0000","last_peered":"2023-09-23T19:51:49.517626+0000","last_clean":"2023-09-23T19:51:49.517626+0000","last_became_active":"2023-09-23T19:32:49.730105+0000","last_became_peered":"2023-09-23T19:32:49.730105+0000","last_unstale":"2023-09-23T19:51:49.517626+0000","last_undegraded":"2023-09-23T19:51:49.517626+0000","last_fullsized":"2023-09-23T19:51:49.517626+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:49.751893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":124,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:51.521677+0000","last_change":"2023-09-23T19:42:55.772691+0000","last_active":"2023-09-23T19:51:51.521677+0000","last_peered":"2023-09-23T19:51:51.521677+0000","last_clean":"2023-09-23T19:51:51.521677+0000","last_became_active":"2023-09-23T19:42:55.772309+0000","last_became_peered":"2023-09-23T19:42:55.772309+0000","last_unstale":"2023-09-23T19:51:51.521677+0000","last_undegraded":"2023-09-23T19:51:51.521677+0000","last_fullsized":"2023-09-23T19:51:51.521677+0000","mapping_epoch":188,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":189,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:49.803892+0000","last_change":"2023-09-23T19:47:56.794658+0000","last_active":"2023-09-23T19:51:49.803892+0000","last_peered":"2023-09-23T19:51:49.803892+0000","last_clean":"2023-09-23T19:51:49.803892+0000","last_became_active":"2023-09-23T19:47:56.793633+0000","last_became_peered":"2023-09-23T19:47:56.793633+0000","last_unstale":"2023-09-23T19:51:49.803892+0000","last_undegraded":"2023-09-23T19:51:49.803892+0000","last_fullsized":"2023-09-23T19:51:49.803892+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":189,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:51.521479+0000","last_change":"2023-09-23T19:05:52.114186+0000","last_active":"2023-09-23T19:51:51.521479+0000","last_peered":"2023-09-23T19:51:51.521479+0000","last_clean":"2023-09-23T19:51:51.521479+0000","last_became_active":"2023-09-23T19:05:52.113635+0000","last_became_peered":"2023-09-23T19:05:52.113635+0000","last_unstale":"2023-09-23T19:51:51.521479+0000","last_undegraded":"2023-09-23T19:51:51.521479+0000","last_fullsized":"2023-09-23T19:51:51.521479+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":124,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:51.522366+0000","last_change":"2023-09-23T19:42:55.772912+0000","last_active":"2023-09-23T19:51:51.522366+0000","last_peered":"2023-09-23T19:51:51.522366+0000","last_clean":"2023-09-23T19:51:51.522366+0000","last_became_active":"2023-09-23T19:42:55.772434+0000","last_became_peered":"2023-09-23T19:42:55.772434+0000","last_unstale":"2023-09-23T19:51:51.522366+0000","last_undegraded":"2023-09-23T19:51:51.522366+0000","last_fullsized":"2023-09-23T19:51:51.522366+0000","mapping_epoch":188,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":212,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:51.357598+0000","last_change":"2023-09-23T19:18:09.010195+0000","last_active":"2023-09-23T19:51:51.357598+0000","last_peered":"2023-09-23T19:51:51.357598+0000","last_clean":"2023-09-23T19:51:51.357598+0000","last_became_active":"2023-09-23T19:18:09.009687+0000","last_became_peered":"2023-09-23T19:18:09.009687+0000","last_unstale":"2023-09-23T19:51:51.357598+0000","last_undegraded":"2023-09-23T19:51:51.357598+0000","last_fullsized":"2023-09-23T19:51:51.357598+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":208,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T19:52:10.685141+0000","last_change":"2023-09-23T19:28:07.885265+0000","last_active":"2023-09-23T19:52:10.685141+0000","last_peered":"2023-09-23T19:52:10.685141+0000","last_clean":"2023-09-23T19:52:10.685141+0000","last_became_active":"2023-09-23T19:28:07.884353+0000","last_became_peered":"2023-09-23T19:28:07.884353+0000","last_unstale":"2023-09-23T19:52:10.685141+0000","last_undegraded":"2023-09-23T19:52:10.685141+0000","last_fullsized":"2023-09-23T19:52:10.685141+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:56.325352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.1","version":"238'2957","reported_seq":2971,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T19:52:19.875561+0000","last_change":"2023-09-23T19:51:24.780294+0000","last_active":"2023-09-23T19:52:19.875561+0000","last_peered":"2023-09-23T19:52:19.875561+0000","last_clean":"2023-09-23T19:52:19.875561+0000","last_became_active":"2023-09-23T19:51:24.779889+0000","last_became_peered":"2023-09-23T19:51:24.779889+0000","last_unstale":"2023-09-23T19:52:19.875561+0000","last_undegraded":"2023-09-23T19:52:19.875561+0000","last_fullsized":"2023-09-23T19:52:19.875561+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":232,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_clean_scrub_stamp":"2023-09-23T19:51:23.737731+0000","objects_scrubbed":0,"log_size":2957,"log_dups_size":0,"ondisk_log_size":2957,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":193789952,"num_objects":2957,"num_object_clones":0,"num_object_copies":8871,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2957,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2957,"num_write_kb":189248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":205,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:51.523061+0000","last_change":"2023-09-23T19:18:09.035116+0000","last_active":"2023-09-23T19:51:51.523061+0000","last_peered":"2023-09-23T19:51:51.523061+0000","last_clean":"2023-09-23T19:51:51.523061+0000","last_became_active":"2023-09-23T19:18:09.034416+0000","last_became_peered":"2023-09-23T19:18:09.034416+0000","last_unstale":"2023-09-23T19:51:51.523061+0000","last_undegraded":"2023-09-23T19:51:51.523061+0000","last_fullsized":"2023-09-23T19:51:51.523061+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":174,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:49.803361+0000","last_change":"2023-09-23T19:47:56.790227+0000","last_active":"2023-09-23T19:51:49.803361+0000","last_peered":"2023-09-23T19:51:49.803361+0000","last_clean":"2023-09-23T19:51:49.803361+0000","last_became_active":"2023-09-23T19:47:56.789727+0000","last_became_peered":"2023-09-23T19:47:56.789727+0000","last_unstale":"2023-09-23T19:51:49.803361+0000","last_undegraded":"2023-09-23T19:51:49.803361+0000","last_fullsized":"2023-09-23T19:51:49.803361+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"238'2997","reported_seq":3011,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T19:52:19.840973+0000","last_change":"2023-09-23T19:51:24.781980+0000","last_active":"2023-09-23T19:52:19.840973+0000","last_peered":"2023-09-23T19:52:19.840973+0000","last_clean":"2023-09-23T19:52:19.840973+0000","last_became_active":"2023-09-23T19:51:24.781639+0000","last_became_peered":"2023-09-23T19:51:24.781639+0000","last_unstale":"2023-09-23T19:52:19.840973+0000","last_undegraded":"2023-09-23T19:52:19.840973+0000","last_fullsized":"2023-09-23T19:52:19.840973+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":232,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_clean_scrub_stamp":"2023-09-23T19:51:23.737731+0000","objects_scrubbed":0,"log_size":2997,"log_dups_size":0,"ondisk_log_size":2997,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196411392,"num_objects":2997,"num_object_clones":0,"num_object_copies":8991,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2997,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2997,"num_write_kb":191808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":206,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:51.522569+0000","last_change":"2023-09-23T19:22:42.151134+0000","last_active":"2023-09-23T19:51:51.522569+0000","last_peered":"2023-09-23T19:51:51.522569+0000","last_clean":"2023-09-23T19:51:51.522569+0000","last_became_active":"2023-09-23T19:22:42.150553+0000","last_became_peered":"2023-09-23T19:22:42.150553+0000","last_unstale":"2023-09-23T19:51:51.522569+0000","last_undegraded":"2023-09-23T19:51:51.522569+0000","last_fullsized":"2023-09-23T19:51:51.522569+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":217,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:51.523280+0000","last_change":"2023-09-23T19:28:07.899262+0000","last_active":"2023-09-23T19:51:51.523280+0000","last_peered":"2023-09-23T19:51:51.523280+0000","last_clean":"2023-09-23T19:51:51.523280+0000","last_became_active":"2023-09-23T19:28:07.898848+0000","last_became_peered":"2023-09-23T19:28:07.898848+0000","last_unstale":"2023-09-23T19:51:51.523280+0000","last_undegraded":"2023-09-23T19:51:51.523280+0000","last_fullsized":"2023-09-23T19:51:51.523280+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.3","version":"238'2832","reported_seq":2847,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T19:52:17.511133+0000","last_change":"2023-09-23T19:51:24.773320+0000","last_active":"2023-09-23T19:52:17.511133+0000","last_peered":"2023-09-23T19:52:17.511133+0000","last_clean":"2023-09-23T19:52:17.511133+0000","last_became_active":"2023-09-23T19:51:24.772965+0000","last_became_peered":"2023-09-23T19:51:24.772965+0000","last_unstale":"2023-09-23T19:52:17.511133+0000","last_undegraded":"2023-09-23T19:52:17.511133+0000","last_fullsized":"2023-09-23T19:52:17.511133+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":232,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_clean_scrub_stamp":"2023-09-23T19:51:23.737731+0000","objects_scrubbed":0,"log_size":2832,"log_dups_size":0,"ondisk_log_size":2832,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:07.426463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185597952,"num_objects":2832,"num_object_clones":0,"num_object_copies":8496,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2832,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2832,"num_write_kb":181248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":190,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:51.521869+0000","last_change":"2023-09-23T19:05:52.114108+0000","last_active":"2023-09-23T19:51:51.521869+0000","last_peered":"2023-09-23T19:51:51.521869+0000","last_clean":"2023-09-23T19:51:51.521869+0000","last_became_active":"2023-09-23T19:05:52.113609+0000","last_became_peered":"2023-09-23T19:05:52.113609+0000","last_unstale":"2023-09-23T19:51:51.521869+0000","last_undegraded":"2023-09-23T19:51:51.521869+0000","last_fullsized":"2023-09-23T19:51:51.521869+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":203,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:51.357913+0000","last_change":"2023-09-23T19:22:42.156825+0000","last_active":"2023-09-23T19:51:51.357913+0000","last_peered":"2023-09-23T19:51:51.357913+0000","last_clean":"2023-09-23T19:51:51.357913+0000","last_became_active":"2023-09-23T19:22:42.156474+0000","last_became_peered":"2023-09-23T19:22:42.156474+0000","last_unstale":"2023-09-23T19:51:51.357913+0000","last_undegraded":"2023-09-23T19:51:51.357913+0000","last_fullsized":"2023-09-23T19:51:51.357913+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.2","version":"238'3118","reported_seq":3133,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T19:52:21.716489+0000","last_change":"2023-09-23T19:51:24.779170+0000","last_active":"2023-09-23T19:52:21.716489+0000","last_peered":"2023-09-23T19:52:21.716489+0000","last_clean":"2023-09-23T19:52:21.716489+0000","last_became_active":"2023-09-23T19:51:24.778769+0000","last_became_peered":"2023-09-23T19:51:24.778769+0000","last_unstale":"2023-09-23T19:52:21.716489+0000","last_undegraded":"2023-09-23T19:52:21.716489+0000","last_fullsized":"2023-09-23T19:52:21.716489+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":232,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_clean_scrub_stamp":"2023-09-23T19:51:23.737731+0000","objects_scrubbed":0,"log_size":3118,"log_dups_size":0,"ondisk_log_size":3118,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:12:30.926023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":204341248,"num_objects":3118,"num_object_clones":0,"num_object_copies":9354,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3118,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3118,"num_write_kb":199552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":190,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:49.803666+0000","last_change":"2023-09-23T19:47:56.794638+0000","last_active":"2023-09-23T19:51:49.803666+0000","last_peered":"2023-09-23T19:51:49.803666+0000","last_clean":"2023-09-23T19:51:49.803666+0000","last_became_active":"2023-09-23T19:47:56.794305+0000","last_became_peered":"2023-09-23T19:47:56.794305+0000","last_unstale":"2023-09-23T19:51:49.803666+0000","last_undegraded":"2023-09-23T19:51:49.803666+0000","last_fullsized":"2023-09-23T19:51:49.803666+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":204,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:52.548533+0000","last_change":"2023-09-23T19:22:42.135572+0000","last_active":"2023-09-23T19:51:52.548533+0000","last_peered":"2023-09-23T19:51:52.548533+0000","last_clean":"2023-09-23T19:51:52.548533+0000","last_became_active":"2023-09-23T19:22:42.135237+0000","last_became_peered":"2023-09-23T19:22:42.135237+0000","last_unstale":"2023-09-23T19:51:52.548533+0000","last_undegraded":"2023-09-23T19:51:52.548533+0000","last_fullsized":"2023-09-23T19:51:52.548533+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:35.838718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.5","version":"238'2965","reported_seq":2979,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T19:52:19.718994+0000","last_change":"2023-09-23T19:51:24.784350+0000","last_active":"2023-09-23T19:52:19.718994+0000","last_peered":"2023-09-23T19:52:19.718994+0000","last_clean":"2023-09-23T19:52:19.718994+0000","last_became_active":"2023-09-23T19:51:24.783918+0000","last_became_peered":"2023-09-23T19:51:24.783918+0000","last_unstale":"2023-09-23T19:52:19.718994+0000","last_undegraded":"2023-09-23T19:52:19.718994+0000","last_fullsized":"2023-09-23T19:52:19.718994+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":232,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_clean_scrub_stamp":"2023-09-23T19:51:23.737731+0000","objects_scrubbed":0,"log_size":2965,"log_dups_size":0,"ondisk_log_size":2965,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":194314240,"num_objects":2965,"num_object_clones":0,"num_object_copies":8895,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2965,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2965,"num_write_kb":189760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":71,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:51.521044+0000","last_change":"2023-09-23T19:42:55.771907+0000","last_active":"2023-09-23T19:51:51.521044+0000","last_peered":"2023-09-23T19:51:51.521044+0000","last_clean":"2023-09-23T19:51:51.521044+0000","last_became_active":"2023-09-23T19:42:55.771503+0000","last_became_peered":"2023-09-23T19:42:55.771503+0000","last_unstale":"2023-09-23T19:51:51.521044+0000","last_undegraded":"2023-09-23T19:51:51.521044+0000","last_fullsized":"2023-09-23T19:51:51.521044+0000","mapping_epoch":188,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":202,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:51.521443+0000","last_change":"2023-09-23T19:28:07.896803+0000","last_active":"2023-09-23T19:51:51.521443+0000","last_peered":"2023-09-23T19:51:51.521443+0000","last_clean":"2023-09-23T19:51:51.521443+0000","last_became_active":"2023-09-23T19:28:07.896480+0000","last_became_peered":"2023-09-23T19:28:07.896480+0000","last_unstale":"2023-09-23T19:51:51.521443+0000","last_undegraded":"2023-09-23T19:51:51.521443+0000","last_fullsized":"2023-09-23T19:51:51.521443+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.4","version":"238'3019","reported_seq":3033,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T19:52:19.864194+0000","last_change":"2023-09-23T19:51:24.775881+0000","last_active":"2023-09-23T19:52:19.864194+0000","last_peered":"2023-09-23T19:52:19.864194+0000","last_clean":"2023-09-23T19:52:19.864194+0000","last_became_active":"2023-09-23T19:51:24.775309+0000","last_became_peered":"2023-09-23T19:51:24.775309+0000","last_unstale":"2023-09-23T19:52:19.864194+0000","last_undegraded":"2023-09-23T19:52:19.864194+0000","last_fullsized":"2023-09-23T19:52:19.864194+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":232,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_clean_scrub_stamp":"2023-09-23T19:51:23.737731+0000","objects_scrubbed":0,"log_size":3019,"log_dups_size":0,"ondisk_log_size":3019,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197853184,"num_objects":3019,"num_object_clones":0,"num_object_copies":9057,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3019,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3019,"num_write_kb":193216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":204,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:51.357903+0000","last_change":"2023-09-23T19:22:42.156582+0000","last_active":"2023-09-23T19:51:51.357903+0000","last_peered":"2023-09-23T19:51:51.357903+0000","last_clean":"2023-09-23T19:51:51.357903+0000","last_became_active":"2023-09-23T19:22:42.156251+0000","last_became_peered":"2023-09-23T19:22:42.156251+0000","last_unstale":"2023-09-23T19:51:51.357903+0000","last_undegraded":"2023-09-23T19:51:51.357903+0000","last_fullsized":"2023-09-23T19:51:51.357903+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":232,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:51.522750+0000","last_change":"2023-09-23T19:38:43.891593+0000","last_active":"2023-09-23T19:51:51.522750+0000","last_peered":"2023-09-23T19:51:51.522750+0000","last_clean":"2023-09-23T19:51:51.522750+0000","last_became_active":"2023-09-23T19:38:43.891063+0000","last_became_peered":"2023-09-23T19:38:43.891063+0000","last_unstale":"2023-09-23T19:51:51.522750+0000","last_undegraded":"2023-09-23T19:51:51.522750+0000","last_fullsized":"2023-09-23T19:51:51.522750+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.7","version":"238'2922","reported_seq":2936,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T19:52:19.886562+0000","last_change":"2023-09-23T19:51:24.784793+0000","last_active":"2023-09-23T19:52:19.886562+0000","last_peered":"2023-09-23T19:52:19.886562+0000","last_clean":"2023-09-23T19:52:19.886562+0000","last_became_active":"2023-09-23T19:51:24.784503+0000","last_became_peered":"2023-09-23T19:51:24.784503+0000","last_unstale":"2023-09-23T19:52:19.886562+0000","last_undegraded":"2023-09-23T19:52:19.886562+0000","last_fullsized":"2023-09-23T19:52:19.886562+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":232,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_clean_scrub_stamp":"2023-09-23T19:51:23.737731+0000","objects_scrubbed":0,"log_size":2922,"log_dups_size":0,"ondisk_log_size":2922,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191496192,"num_objects":2922,"num_object_clones":0,"num_object_copies":8766,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2922,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2922,"num_write_kb":187008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":218,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:51.520918+0000","last_change":"2023-09-23T19:38:43.892972+0000","last_active":"2023-09-23T19:51:51.520918+0000","last_peered":"2023-09-23T19:51:51.520918+0000","last_clean":"2023-09-23T19:51:51.520918+0000","last_became_active":"2023-09-23T19:38:43.892451+0000","last_became_peered":"2023-09-23T19:38:43.892451+0000","last_unstale":"2023-09-23T19:51:51.520918+0000","last_undegraded":"2023-09-23T19:51:51.520918+0000","last_fullsized":"2023-09-23T19:51:51.520918+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":202,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:51.521956+0000","last_change":"2023-09-23T19:22:42.149670+0000","last_active":"2023-09-23T19:51:51.521956+0000","last_peered":"2023-09-23T19:51:51.521956+0000","last_clean":"2023-09-23T19:51:51.521956+0000","last_became_active":"2023-09-23T19:22:42.149022+0000","last_became_peered":"2023-09-23T19:22:42.149022+0000","last_unstale":"2023-09-23T19:51:51.521956+0000","last_undegraded":"2023-09-23T19:51:51.521956+0000","last_fullsized":"2023-09-23T19:51:51.521956+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.6","version":"238'2933","reported_seq":2947,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T19:52:19.732346+0000","last_change":"2023-09-23T19:51:24.783467+0000","last_active":"2023-09-23T19:52:19.732346+0000","last_peered":"2023-09-23T19:52:19.732346+0000","last_clean":"2023-09-23T19:52:19.732346+0000","last_became_active":"2023-09-23T19:51:24.783123+0000","last_became_peered":"2023-09-23T19:51:24.783123+0000","last_unstale":"2023-09-23T19:52:19.732346+0000","last_undegraded":"2023-09-23T19:52:19.732346+0000","last_fullsized":"2023-09-23T19:52:19.732346+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":232,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_clean_scrub_stamp":"2023-09-23T19:51:23.737731+0000","objects_scrubbed":0,"log_size":2933,"log_dups_size":0,"ondisk_log_size":2933,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192217088,"num_objects":2933,"num_object_clones":0,"num_object_copies":8799,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2933,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2933,"num_write_kb":187712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":201,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T19:52:10.680158+0000","last_change":"2023-09-23T19:32:49.729244+0000","last_active":"2023-09-23T19:52:10.680158+0000","last_peered":"2023-09-23T19:52:10.680158+0000","last_clean":"2023-09-23T19:52:10.680158+0000","last_became_active":"2023-09-23T19:32:49.728810+0000","last_became_peered":"2023-09-23T19:32:49.728810+0000","last_unstale":"2023-09-23T19:52:10.680158+0000","last_undegraded":"2023-09-23T19:52:10.680158+0000","last_fullsized":"2023-09-23T19:52:10.680158+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:57.540826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":188,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:52:01.390613+0000","last_change":"2023-09-23T19:28:07.889095+0000","last_active":"2023-09-23T19:52:01.390613+0000","last_peered":"2023-09-23T19:52:01.390613+0000","last_clean":"2023-09-23T19:52:01.390613+0000","last_became_active":"2023-09-23T19:28:07.888742+0000","last_became_peered":"2023-09-23T19:28:07.888742+0000","last_unstale":"2023-09-23T19:52:01.390613+0000","last_undegraded":"2023-09-23T19:52:01.390613+0000","last_fullsized":"2023-09-23T19:52:01.390613+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:43.929488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.b","version":"238'3052","reported_seq":3066,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T19:52:19.724438+0000","last_change":"2023-09-23T19:51:24.785370+0000","last_active":"2023-09-23T19:52:19.724438+0000","last_peered":"2023-09-23T19:52:19.724438+0000","last_clean":"2023-09-23T19:52:19.724438+0000","last_became_active":"2023-09-23T19:51:24.784883+0000","last_became_peered":"2023-09-23T19:51:24.784883+0000","last_unstale":"2023-09-23T19:52:19.724438+0000","last_undegraded":"2023-09-23T19:52:19.724438+0000","last_fullsized":"2023-09-23T19:52:19.724438+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":232,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_clean_scrub_stamp":"2023-09-23T19:51:23.737731+0000","objects_scrubbed":0,"log_size":3052,"log_dups_size":0,"ondisk_log_size":3052,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":200015872,"num_objects":3052,"num_object_clones":0,"num_object_copies":9156,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3052,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3052,"num_write_kb":195328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":222,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:56.388596+0000","last_change":"2023-09-23T19:22:42.143289+0000","last_active":"2023-09-23T19:51:56.388596+0000","last_peered":"2023-09-23T19:51:56.388596+0000","last_clean":"2023-09-23T19:51:56.388596+0000","last_became_active":"2023-09-23T19:22:42.142656+0000","last_became_peered":"2023-09-23T19:22:42.142656+0000","last_unstale":"2023-09-23T19:51:56.388596+0000","last_undegraded":"2023-09-23T19:51:56.388596+0000","last_fullsized":"2023-09-23T19:51:56.388596+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:38.467441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":220,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:49.804110+0000","last_change":"2023-09-23T19:47:56.794437+0000","last_active":"2023-09-23T19:51:49.804110+0000","last_peered":"2023-09-23T19:51:49.804110+0000","last_clean":"2023-09-23T19:51:49.804110+0000","last_became_active":"2023-09-23T19:47:56.794118+0000","last_became_peered":"2023-09-23T19:47:56.794118+0000","last_unstale":"2023-09-23T19:51:49.804110+0000","last_undegraded":"2023-09-23T19:51:49.804110+0000","last_fullsized":"2023-09-23T19:51:49.804110+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":207,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:51.511112+0000","last_change":"2023-09-23T19:38:43.898519+0000","last_active":"2023-09-23T19:51:51.511112+0000","last_peered":"2023-09-23T19:51:51.511112+0000","last_clean":"2023-09-23T19:51:51.511112+0000","last_became_active":"2023-09-23T19:38:43.896814+0000","last_became_peered":"2023-09-23T19:38:43.896814+0000","last_unstale":"2023-09-23T19:51:51.511112+0000","last_undegraded":"2023-09-23T19:51:51.511112+0000","last_fullsized":"2023-09-23T19:51:51.511112+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:40.465038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.d","version":"238'2971","reported_seq":2985,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T19:52:19.737112+0000","last_change":"2023-09-23T19:51:24.782799+0000","last_active":"2023-09-23T19:52:19.737112+0000","last_peered":"2023-09-23T19:52:19.737112+0000","last_clean":"2023-09-23T19:52:19.737112+0000","last_became_active":"2023-09-23T19:51:24.782419+0000","last_became_peered":"2023-09-23T19:51:24.782419+0000","last_unstale":"2023-09-23T19:52:19.737112+0000","last_undegraded":"2023-09-23T19:52:19.737112+0000","last_fullsized":"2023-09-23T19:52:19.737112+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":232,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_clean_scrub_stamp":"2023-09-23T19:51:23.737731+0000","objects_scrubbed":0,"log_size":2971,"log_dups_size":0,"ondisk_log_size":2971,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":194707456,"num_objects":2971,"num_object_clones":0,"num_object_copies":8913,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2971,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2971,"num_write_kb":190144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":122,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:52:01.390495+0000","last_change":"2023-09-23T19:43:38.612700+0000","last_active":"2023-09-23T19:52:01.390495+0000","last_peered":"2023-09-23T19:52:01.390495+0000","last_clean":"2023-09-23T19:52:01.390495+0000","last_became_active":"2023-09-23T19:43:38.612424+0000","last_became_peered":"2023-09-23T19:43:38.612424+0000","last_unstale":"2023-09-23T19:52:01.390495+0000","last_undegraded":"2023-09-23T19:52:01.390495+0000","last_fullsized":"2023-09-23T19:52:01.390495+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:24.833490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":203,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:51.521215+0000","last_change":"2023-09-23T19:18:09.017525+0000","last_active":"2023-09-23T19:51:51.521215+0000","last_peered":"2023-09-23T19:51:51.521215+0000","last_clean":"2023-09-23T19:51:51.521215+0000","last_became_active":"2023-09-23T19:18:09.017193+0000","last_became_peered":"2023-09-23T19:18:09.017193+0000","last_unstale":"2023-09-23T19:51:51.521215+0000","last_undegraded":"2023-09-23T19:51:51.521215+0000","last_fullsized":"2023-09-23T19:51:51.521215+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":219,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:56.388586+0000","last_change":"2023-09-23T19:38:43.877088+0000","last_active":"2023-09-23T19:51:56.388586+0000","last_peered":"2023-09-23T19:51:56.388586+0000","last_clean":"2023-09-23T19:51:56.388586+0000","last_became_active":"2023-09-23T19:38:43.876719+0000","last_became_peered":"2023-09-23T19:38:43.876719+0000","last_unstale":"2023-09-23T19:51:56.388586+0000","last_undegraded":"2023-09-23T19:51:56.388586+0000","last_fullsized":"2023-09-23T19:51:56.388586+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:05.423256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.e","version":"238'2977","reported_seq":2991,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T19:52:19.879316+0000","last_change":"2023-09-23T19:51:24.785601+0000","last_active":"2023-09-23T19:52:19.879316+0000","last_peered":"2023-09-23T19:52:19.879316+0000","last_clean":"2023-09-23T19:52:19.879316+0000","last_became_active":"2023-09-23T19:51:24.785163+0000","last_became_peered":"2023-09-23T19:51:24.785163+0000","last_unstale":"2023-09-23T19:52:19.879316+0000","last_undegraded":"2023-09-23T19:52:19.879316+0000","last_fullsized":"2023-09-23T19:52:19.879316+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":232,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_clean_scrub_stamp":"2023-09-23T19:51:23.737731+0000","objects_scrubbed":0,"log_size":2977,"log_dups_size":0,"ondisk_log_size":2977,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195100672,"num_objects":2977,"num_object_clones":0,"num_object_copies":8931,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2977,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2977,"num_write_kb":190528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":187,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:51.511181+0000","last_change":"2023-09-23T19:38:43.883663+0000","last_active":"2023-09-23T19:51:51.511181+0000","last_peered":"2023-09-23T19:51:51.511181+0000","last_clean":"2023-09-23T19:51:51.511181+0000","last_became_active":"2023-09-23T19:38:43.882421+0000","last_became_peered":"2023-09-23T19:38:43.882421+0000","last_unstale":"2023-09-23T19:51:51.511181+0000","last_undegraded":"2023-09-23T19:51:51.511181+0000","last_fullsized":"2023-09-23T19:51:51.511181+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:49:01.607659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":204,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:49.516784+0000","last_change":"2023-09-23T19:38:43.890600+0000","last_active":"2023-09-23T19:51:49.516784+0000","last_peered":"2023-09-23T19:51:49.516784+0000","last_clean":"2023-09-23T19:51:49.516784+0000","last_became_active":"2023-09-23T19:38:43.890253+0000","last_became_peered":"2023-09-23T19:38:43.890253+0000","last_unstale":"2023-09-23T19:51:49.516784+0000","last_undegraded":"2023-09-23T19:51:49.516784+0000","last_fullsized":"2023-09-23T19:51:49.516784+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:46.751120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":216,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:49.516944+0000","last_change":"2023-09-23T19:22:42.143733+0000","last_active":"2023-09-23T19:51:49.516944+0000","last_peered":"2023-09-23T19:51:49.516944+0000","last_clean":"2023-09-23T19:51:49.516944+0000","last_became_active":"2023-09-23T19:22:42.143077+0000","last_became_peered":"2023-09-23T19:22:42.143077+0000","last_unstale":"2023-09-23T19:51:49.516944+0000","last_undegraded":"2023-09-23T19:51:49.516944+0000","last_fullsized":"2023-09-23T19:51:49.516944+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:47:33.086994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.c","version":"238'3051","reported_seq":3066,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T19:52:21.386570+0000","last_change":"2023-09-23T19:51:24.778822+0000","last_active":"2023-09-23T19:52:21.386570+0000","last_peered":"2023-09-23T19:52:21.386570+0000","last_clean":"2023-09-23T19:52:21.386570+0000","last_became_active":"2023-09-23T19:51:24.778464+0000","last_became_peered":"2023-09-23T19:51:24.778464+0000","last_unstale":"2023-09-23T19:52:21.386570+0000","last_undegraded":"2023-09-23T19:52:21.386570+0000","last_fullsized":"2023-09-23T19:52:21.386570+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":232,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_clean_scrub_stamp":"2023-09-23T19:51:23.737731+0000","objects_scrubbed":0,"log_size":3051,"log_dups_size":0,"ondisk_log_size":3051,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:47.174791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":199950336,"num_objects":3051,"num_object_clones":0,"num_object_copies":9153,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3051,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3051,"num_write_kb":195264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":209,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:49.803606+0000","last_change":"2023-09-23T19:47:56.791259+0000","last_active":"2023-09-23T19:51:49.803606+0000","last_peered":"2023-09-23T19:51:49.803606+0000","last_clean":"2023-09-23T19:51:49.803606+0000","last_became_active":"2023-09-23T19:47:56.790954+0000","last_became_peered":"2023-09-23T19:47:56.790954+0000","last_unstale":"2023-09-23T19:51:49.803606+0000","last_undegraded":"2023-09-23T19:51:49.803606+0000","last_fullsized":"2023-09-23T19:51:49.803606+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":440,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:51.520893+0000","last_change":"2023-09-23T19:18:09.016297+0000","last_active":"2023-09-23T19:51:51.520893+0000","last_peered":"2023-09-23T19:51:51.520893+0000","last_clean":"2023-09-23T19:51:51.520893+0000","last_became_active":"2023-09-23T19:18:09.015436+0000","last_became_peered":"2023-09-23T19:18:09.015436+0000","last_unstale":"2023-09-23T19:51:51.520893+0000","last_undegraded":"2023-09-23T19:51:51.520893+0000","last_fullsized":"2023-09-23T19:51:51.520893+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:59:33.840751+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:59:33.840751+0000","last_clean_scrub_stamp":"2023-09-23T18:59:33.840751+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":163,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:49.803605+0000","last_change":"2023-09-23T19:47:56.811861+0000","last_active":"2023-09-23T19:51:49.803605+0000","last_peered":"2023-09-23T19:51:49.803605+0000","last_clean":"2023-09-23T19:51:49.803605+0000","last_became_active":"2023-09-23T19:47:56.811527+0000","last_became_peered":"2023-09-23T19:47:56.811527+0000","last_unstale":"2023-09-23T19:51:49.803605+0000","last_undegraded":"2023-09-23T19:51:49.803605+0000","last_fullsized":"2023-09-23T19:51:49.803605+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":188,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:51.358385+0000","last_change":"2023-09-23T19:05:52.786771+0000","last_active":"2023-09-23T19:51:51.358385+0000","last_peered":"2023-09-23T19:51:51.358385+0000","last_clean":"2023-09-23T19:51:51.358385+0000","last_became_active":"2023-09-23T19:05:52.785797+0000","last_became_peered":"2023-09-23T19:05:52.785797+0000","last_unstale":"2023-09-23T19:51:51.358385+0000","last_undegraded":"2023-09-23T19:51:51.358385+0000","last_fullsized":"2023-09-23T19:51:51.358385+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.f","version":"238'3071","reported_seq":3085,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T19:52:19.940679+0000","last_change":"2023-09-23T19:51:24.783251+0000","last_active":"2023-09-23T19:52:19.940679+0000","last_peered":"2023-09-23T19:52:19.940679+0000","last_clean":"2023-09-23T19:52:19.940679+0000","last_became_active":"2023-09-23T19:51:24.783051+0000","last_became_peered":"2023-09-23T19:51:24.783051+0000","last_unstale":"2023-09-23T19:52:19.940679+0000","last_undegraded":"2023-09-23T19:52:19.940679+0000","last_fullsized":"2023-09-23T19:52:19.940679+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":232,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_clean_scrub_stamp":"2023-09-23T19:51:23.737731+0000","objects_scrubbed":0,"log_size":3071,"log_dups_size":0,"ondisk_log_size":3071,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":201261056,"num_objects":3071,"num_object_clones":0,"num_object_copies":9213,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3071,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3071,"num_write_kb":196544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":223,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:51.520789+0000","last_change":"2023-09-23T19:22:42.150636+0000","last_active":"2023-09-23T19:51:51.520789+0000","last_peered":"2023-09-23T19:51:51.520789+0000","last_clean":"2023-09-23T19:51:51.520789+0000","last_became_active":"2023-09-23T19:22:42.150045+0000","last_became_peered":"2023-09-23T19:22:42.150045+0000","last_unstale":"2023-09-23T19:51:51.520789+0000","last_undegraded":"2023-09-23T19:51:51.520789+0000","last_fullsized":"2023-09-23T19:51:51.520789+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":124,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:51.523372+0000","last_change":"2023-09-23T19:42:55.771906+0000","last_active":"2023-09-23T19:51:51.523372+0000","last_peered":"2023-09-23T19:51:51.523372+0000","last_clean":"2023-09-23T19:51:51.523372+0000","last_became_active":"2023-09-23T19:42:55.771599+0000","last_became_peered":"2023-09-23T19:42:55.771599+0000","last_unstale":"2023-09-23T19:51:51.523372+0000","last_undegraded":"2023-09-23T19:51:51.523372+0000","last_fullsized":"2023-09-23T19:51:51.523372+0000","mapping_epoch":188,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":234,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:49.809429+0000","last_change":"2023-09-23T19:38:43.876930+0000","last_active":"2023-09-23T19:51:49.809429+0000","last_peered":"2023-09-23T19:51:49.809429+0000","last_clean":"2023-09-23T19:51:49.809429+0000","last_became_active":"2023-09-23T19:38:43.876571+0000","last_became_peered":"2023-09-23T19:38:43.876571+0000","last_unstale":"2023-09-23T19:51:49.809429+0000","last_undegraded":"2023-09-23T19:51:49.809429+0000","last_fullsized":"2023-09-23T19:51:49.809429+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:30.924536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.a","version":"238'2926","reported_seq":2941,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T19:52:19.149361+0000","last_change":"2023-09-23T19:51:24.773510+0000","last_active":"2023-09-23T19:52:19.149361+0000","last_peered":"2023-09-23T19:52:19.149361+0000","last_clean":"2023-09-23T19:52:19.149361+0000","last_became_active":"2023-09-23T19:51:24.773049+0000","last_became_peered":"2023-09-23T19:51:24.773049+0000","last_unstale":"2023-09-23T19:52:19.149361+0000","last_undegraded":"2023-09-23T19:52:19.149361+0000","last_fullsized":"2023-09-23T19:52:19.149361+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":232,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_clean_scrub_stamp":"2023-09-23T19:51:23.737731+0000","objects_scrubbed":0,"log_size":2926,"log_dups_size":0,"ondisk_log_size":2926,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:19:22.115341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191758336,"num_objects":2926,"num_object_clones":0,"num_object_copies":8778,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2926,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2926,"num_write_kb":187264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":208,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:51.521972+0000","last_change":"2023-09-23T19:18:09.015853+0000","last_active":"2023-09-23T19:51:51.521972+0000","last_peered":"2023-09-23T19:51:51.521972+0000","last_clean":"2023-09-23T19:51:51.521972+0000","last_became_active":"2023-09-23T19:18:09.015338+0000","last_became_peered":"2023-09-23T19:18:09.015338+0000","last_unstale":"2023-09-23T19:51:51.521972+0000","last_undegraded":"2023-09-23T19:51:51.521972+0000","last_fullsized":"2023-09-23T19:51:51.521972+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":204,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:56.388572+0000","last_change":"2023-09-23T19:38:43.881233+0000","last_active":"2023-09-23T19:51:56.388572+0000","last_peered":"2023-09-23T19:51:56.388572+0000","last_clean":"2023-09-23T19:51:56.388572+0000","last_became_active":"2023-09-23T19:38:43.880893+0000","last_became_peered":"2023-09-23T19:38:43.880893+0000","last_unstale":"2023-09-23T19:51:56.388572+0000","last_undegraded":"2023-09-23T19:51:56.388572+0000","last_fullsized":"2023-09-23T19:51:56.388572+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:11.206947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":59,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:49.803585+0000","last_change":"2023-09-23T19:47:56.790340+0000","last_active":"2023-09-23T19:51:49.803585+0000","last_peered":"2023-09-23T19:51:49.803585+0000","last_clean":"2023-09-23T19:51:49.803585+0000","last_became_active":"2023-09-23T19:47:56.789785+0000","last_became_peered":"2023-09-23T19:47:56.789785+0000","last_unstale":"2023-09-23T19:51:49.803585+0000","last_undegraded":"2023-09-23T19:51:49.803585+0000","last_fullsized":"2023-09-23T19:51:49.803585+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.9","version":"238'2977","reported_seq":2991,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T19:52:19.724226+0000","last_change":"2023-09-23T19:51:24.785521+0000","last_active":"2023-09-23T19:52:19.724226+0000","last_peered":"2023-09-23T19:52:19.724226+0000","last_clean":"2023-09-23T19:52:19.724226+0000","last_became_active":"2023-09-23T19:51:24.784935+0000","last_became_peered":"2023-09-23T19:51:24.784935+0000","last_unstale":"2023-09-23T19:52:19.724226+0000","last_undegraded":"2023-09-23T19:52:19.724226+0000","last_fullsized":"2023-09-23T19:52:19.724226+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":232,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_clean_scrub_stamp":"2023-09-23T19:51:23.737731+0000","objects_scrubbed":0,"log_size":2977,"log_dups_size":0,"ondisk_log_size":2977,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195100672,"num_objects":2977,"num_object_clones":0,"num_object_copies":8931,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2977,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2977,"num_write_kb":190528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":142,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:51.522818+0000","last_change":"2023-09-23T19:32:49.737323+0000","last_active":"2023-09-23T19:51:51.522818+0000","last_peered":"2023-09-23T19:51:51.522818+0000","last_clean":"2023-09-23T19:51:51.522818+0000","last_became_active":"2023-09-23T19:32:49.736937+0000","last_became_peered":"2023-09-23T19:32:49.736937+0000","last_unstale":"2023-09-23T19:51:51.522818+0000","last_undegraded":"2023-09-23T19:51:51.522818+0000","last_fullsized":"2023-09-23T19:51:51.522818+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":173,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:49.516668+0000","last_change":"2023-09-23T19:22:42.145506+0000","last_active":"2023-09-23T19:51:49.516668+0000","last_peered":"2023-09-23T19:51:49.516668+0000","last_clean":"2023-09-23T19:51:49.516668+0000","last_became_active":"2023-09-23T19:22:42.143991+0000","last_became_peered":"2023-09-23T19:22:42.143991+0000","last_unstale":"2023-09-23T19:51:49.516668+0000","last_undegraded":"2023-09-23T19:51:49.516668+0000","last_fullsized":"2023-09-23T19:51:49.516668+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:56.875175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":179,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:51.521172+0000","last_change":"2023-09-23T19:32:49.734592+0000","last_active":"2023-09-23T19:51:51.521172+0000","last_peered":"2023-09-23T19:51:51.521172+0000","last_clean":"2023-09-23T19:51:51.521172+0000","last_became_active":"2023-09-23T19:32:49.733905+0000","last_became_peered":"2023-09-23T19:32:49.733905+0000","last_unstale":"2023-09-23T19:51:51.521172+0000","last_undegraded":"2023-09-23T19:51:51.521172+0000","last_fullsized":"2023-09-23T19:51:51.521172+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.8","version":"238'2979","reported_seq":2994,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T19:52:19.122576+0000","last_change":"2023-09-23T19:51:24.780616+0000","last_active":"2023-09-23T19:52:19.122576+0000","last_peered":"2023-09-23T19:52:19.122576+0000","last_clean":"2023-09-23T19:52:19.122576+0000","last_became_active":"2023-09-23T19:51:24.780250+0000","last_became_peered":"2023-09-23T19:51:24.780250+0000","last_unstale":"2023-09-23T19:52:19.122576+0000","last_undegraded":"2023-09-23T19:52:19.122576+0000","last_fullsized":"2023-09-23T19:52:19.122576+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":232,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:51:23.737731+0000","last_clean_scrub_stamp":"2023-09-23T19:51:23.737731+0000","objects_scrubbed":0,"log_size":2979,"log_dups_size":0,"ondisk_log_size":2979,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:10.216601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195231744,"num_objects":2979,"num_object_clones":0,"num_object_copies":8937,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2979,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2979,"num_write_kb":190656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":223,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:51.358622+0000","last_change":"2023-09-23T19:38:43.895887+0000","last_active":"2023-09-23T19:51:51.358622+0000","last_peered":"2023-09-23T19:51:51.358622+0000","last_clean":"2023-09-23T19:51:51.358622+0000","last_became_active":"2023-09-23T19:38:43.895486+0000","last_became_peered":"2023-09-23T19:38:43.895486+0000","last_unstale":"2023-09-23T19:51:51.358622+0000","last_undegraded":"2023-09-23T19:51:51.358622+0000","last_fullsized":"2023-09-23T19:51:51.358622+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":207,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:49.809298+0000","last_change":"2023-09-23T19:22:42.149694+0000","last_active":"2023-09-23T19:51:49.809298+0000","last_peered":"2023-09-23T19:51:49.809298+0000","last_clean":"2023-09-23T19:51:49.809298+0000","last_became_active":"2023-09-23T19:22:42.148194+0000","last_became_peered":"2023-09-23T19:22:42.148194+0000","last_unstale":"2023-09-23T19:51:49.809298+0000","last_undegraded":"2023-09-23T19:51:49.809298+0000","last_fullsized":"2023-09-23T19:51:49.809298+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:47.842383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":187,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:51.521117+0000","last_change":"2023-09-23T19:05:52.138620+0000","last_active":"2023-09-23T19:51:51.521117+0000","last_peered":"2023-09-23T19:51:51.521117+0000","last_clean":"2023-09-23T19:51:51.521117+0000","last_became_active":"2023-09-23T19:05:52.138243+0000","last_became_peered":"2023-09-23T19:05:52.138243+0000","last_unstale":"2023-09-23T19:51:51.521117+0000","last_undegraded":"2023-09-23T19:51:51.521117+0000","last_fullsized":"2023-09-23T19:51:51.521117+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":190,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:49.809534+0000","last_change":"2023-09-23T19:18:09.008811+0000","last_active":"2023-09-23T19:51:49.809534+0000","last_peered":"2023-09-23T19:51:49.809534+0000","last_clean":"2023-09-23T19:51:49.809534+0000","last_became_active":"2023-09-23T19:18:09.007255+0000","last_became_peered":"2023-09-23T19:18:09.007255+0000","last_unstale":"2023-09-23T19:51:49.809534+0000","last_undegraded":"2023-09-23T19:51:49.809534+0000","last_fullsized":"2023-09-23T19:51:49.809534+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:25.874038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":194,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:51.511354+0000","last_change":"2023-09-23T19:38:43.877244+0000","last_active":"2023-09-23T19:51:51.511354+0000","last_peered":"2023-09-23T19:51:51.511354+0000","last_clean":"2023-09-23T19:51:51.511354+0000","last_became_active":"2023-09-23T19:38:43.876896+0000","last_became_peered":"2023-09-23T19:38:43.876896+0000","last_unstale":"2023-09-23T19:51:51.511354+0000","last_undegraded":"2023-09-23T19:51:51.511354+0000","last_fullsized":"2023-09-23T19:51:51.511354+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:22.288276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":67,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:51.521079+0000","last_change":"2023-09-23T19:43:38.620732+0000","last_active":"2023-09-23T19:51:51.521079+0000","last_peered":"2023-09-23T19:51:51.521079+0000","last_clean":"2023-09-23T19:51:51.521079+0000","last_became_active":"2023-09-23T19:43:38.619424+0000","last_became_peered":"2023-09-23T19:43:38.619424+0000","last_unstale":"2023-09-23T19:51:51.521079+0000","last_undegraded":"2023-09-23T19:51:51.521079+0000","last_fullsized":"2023-09-23T19:51:51.521079+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":241,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:51.521808+0000","last_change":"2023-09-23T19:38:43.947451+0000","last_active":"2023-09-23T19:51:51.521808+0000","last_peered":"2023-09-23T19:51:51.521808+0000","last_clean":"2023-09-23T19:51:51.521808+0000","last_became_active":"2023-09-23T19:38:43.946420+0000","last_became_peered":"2023-09-23T19:38:43.946420+0000","last_unstale":"2023-09-23T19:51:51.521808+0000","last_undegraded":"2023-09-23T19:51:51.521808+0000","last_fullsized":"2023-09-23T19:51:51.521808+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":177,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:49.804033+0000","last_change":"2023-09-23T19:47:56.793347+0000","last_active":"2023-09-23T19:51:49.804033+0000","last_peered":"2023-09-23T19:51:49.804033+0000","last_clean":"2023-09-23T19:51:49.804033+0000","last_became_active":"2023-09-23T19:47:56.792866+0000","last_became_peered":"2023-09-23T19:47:56.792866+0000","last_unstale":"2023-09-23T19:51:49.804033+0000","last_undegraded":"2023-09-23T19:51:49.804033+0000","last_fullsized":"2023-09-23T19:51:49.804033+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":217,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:51.521719+0000","last_change":"2023-09-23T19:28:07.896983+0000","last_active":"2023-09-23T19:51:51.521719+0000","last_peered":"2023-09-23T19:51:51.521719+0000","last_clean":"2023-09-23T19:51:51.521719+0000","last_became_active":"2023-09-23T19:28:07.896577+0000","last_became_peered":"2023-09-23T19:28:07.896577+0000","last_unstale":"2023-09-23T19:51:51.521719+0000","last_undegraded":"2023-09-23T19:51:51.521719+0000","last_fullsized":"2023-09-23T19:51:51.521719+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":147,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:51.358424+0000","last_change":"2023-09-23T19:38:43.900225+0000","last_active":"2023-09-23T19:51:51.358424+0000","last_peered":"2023-09-23T19:51:51.358424+0000","last_clean":"2023-09-23T19:51:51.358424+0000","last_became_active":"2023-09-23T19:38:43.899907+0000","last_became_peered":"2023-09-23T19:38:43.899907+0000","last_unstale":"2023-09-23T19:51:51.358424+0000","last_undegraded":"2023-09-23T19:51:51.358424+0000","last_fullsized":"2023-09-23T19:51:51.358424+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":203,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:51.510784+0000","last_change":"2023-09-23T19:38:43.882487+0000","last_active":"2023-09-23T19:51:51.510784+0000","last_peered":"2023-09-23T19:51:51.510784+0000","last_clean":"2023-09-23T19:51:51.510784+0000","last_became_active":"2023-09-23T19:38:43.881931+0000","last_became_peered":"2023-09-23T19:38:43.881931+0000","last_unstale":"2023-09-23T19:51:51.510784+0000","last_undegraded":"2023-09-23T19:51:51.510784+0000","last_fullsized":"2023-09-23T19:51:51.510784+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:02:36.906535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":223,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:49.809561+0000","last_change":"2023-09-23T19:38:43.893983+0000","last_active":"2023-09-23T19:51:49.809561+0000","last_peered":"2023-09-23T19:51:49.809561+0000","last_clean":"2023-09-23T19:51:49.809561+0000","last_became_active":"2023-09-23T19:38:43.893527+0000","last_became_peered":"2023-09-23T19:38:43.893527+0000","last_unstale":"2023-09-23T19:51:49.809561+0000","last_undegraded":"2023-09-23T19:51:49.809561+0000","last_fullsized":"2023-09-23T19:51:49.809561+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:12.156531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":196,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T19:51:51.520681+0000","last_change":"2023-09-23T19:06:36.673002+0000","last_active":"2023-09-23T19:51:51.520681+0000","last_peered":"2023-09-23T19:51:51.520681+0000","last_clean":"2023-09-23T19:51:51.520681+0000","last_became_active":"2023-09-23T19:06:36.671911+0000","last_became_peered":"2023-09-23T19:06:36.671911+0000","last_unstale":"2023-09-23T19:51:51.520681+0000","last_undegraded":"2023-09-23T19:51:51.520681+0000","last_fullsized":"2023-09-23T19:51:51.520681+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":3129147392,"num_objects":47747,"num_object_clones":0,"num_object_copies":143241,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":47747,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":47747,"num_write_kb":3055808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9401794560,"data_stored":9401794560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":47747,"ondisk_log_size":47747,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430802,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1820624,"kb_used_data":932936,"kb_used_omap":0,"kb_used_meta":887680,"kb_avail":91924528,"statfs":{"total":95995035648,"available":94130716672,"internally_reserved":0,"allocated":955326464,"data_stored":954794079,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":908984320},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:52:11 2023","interfaces":[{"interface":"back","average":{"1min":0.91500000000000004,"5min":1.3080000000000001,"15min":1.4950000000000001},"min":{"1min":0.30099999999999999,"5min":0.28699999999999998,"15min":0.26400000000000001},"max":{"1min":1.5660000000000001,"5min":6.8419999999999996,"15min":18.271000000000001},"last":0.308},{"interface":"front","average":{"1min":0.879,"5min":1.2589999999999999,"15min":1.5640000000000001},"min":{"1min":0.33200000000000002,"5min":0.29299999999999998,"15min":0.25600000000000001},"max":{"1min":2.8140000000000001,"5min":12.705,"15min":18.242000000000001},"last":0.34100000000000003}]},{"osd":1,"last update":"Sat Sep 23 19:51:28 2023","interfaces":[{"interface":"back","average":{"1min":1.0640000000000001,"5min":1.5609999999999999,"15min":1.823},"min":{"1min":0.63400000000000001,"5min":0.33900000000000002,"15min":0.28799999999999998},"max":{"1min":2.0720000000000001,"5min":22.611999999999998,"15min":30.780999999999999},"last":0.42999999999999999},{"interface":"front","average":{"1min":1.2310000000000001,"5min":1.4510000000000001,"15min":1.835},"min":{"1min":0.626,"5min":0.314,"15min":0.30599999999999999},"max":{"1min":4.3840000000000003,"5min":22.687999999999999,"15min":30.827999999999999},"last":0.48099999999999998}]},{"osd":2,"last update":"Sat Sep 23 19:52:11 2023","interfaces":[{"interface":"back","average":{"1min":1.3959999999999999,"5min":6.1180000000000003,"15min":3.3119999999999998},"min":{"1min":0.432,"5min":0.311,"15min":0.28100000000000003},"max":{"1min":10.869999999999999,"5min":246.982,"15min":246.982},"last":0.41199999999999998},{"interface":"front","average":{"1min":1.1559999999999999,"5min":5.8339999999999996,"15min":3.0630000000000002},"min":{"1min":0.69899999999999995,"5min":0.35999999999999999,"15min":0.35199999999999998},"max":{"1min":2.9649999999999999,"5min":247.751,"15min":247.751},"last":0.372}]},{"osd":3,"last update":"Sat Sep 23 19:51:59 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":1.5860000000000001,"15min":1.5329999999999999},"min":{"1min":0.54300000000000004,"5min":0.39000000000000001,"15min":0.312},"max":{"1min":1.2589999999999999,"5min":7.593,"15min":8.843},"last":1.016},{"interface":"front","average":{"1min":1.085,"5min":1.6659999999999999,"15min":1.613},"min":{"1min":0.77600000000000002,"5min":0.42999999999999999,"15min":0.34100000000000003},"max":{"1min":1.613,"5min":7.609,"15min":7.609},"last":0.88500000000000001}]},{"osd":4,"last update":"Sat Sep 23 19:51:28 2023","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":0.91800000000000004,"15min":1.145},"min":{"1min":0.40300000000000002,"5min":0.34200000000000003,"15min":0.254},"max":{"1min":1.708,"5min":5.3230000000000004,"15min":8.8010000000000002},"last":1.772},{"interface":"front","average":{"1min":1.1419999999999999,"5min":1.02,"15min":1.3320000000000001},"min":{"1min":0.39100000000000001,"5min":0.34799999999999998,"15min":0.27400000000000002},"max":{"1min":4.4269999999999996,"5min":5.2599999999999998,"15min":13.385},"last":0.45400000000000001}]},{"osd":5,"last update":"Sat Sep 23 19:51:28 2023","interfaces":[{"interface":"back","average":{"1min":0.94099999999999995,"5min":0.996,"15min":1.3149999999999999},"min":{"1min":0.42699999999999999,"5min":0.36399999999999999,"15min":0.34200000000000003},"max":{"1min":1.54,"5min":4.3769999999999998,"15min":13.273999999999999},"last":0.77800000000000002},{"interface":"front","average":{"1min":1.0549999999999999,"5min":0.999,"15min":1.3009999999999999},"min":{"1min":0.59799999999999998,"5min":0.38800000000000001,"15min":0.34200000000000003},"max":{"1min":1.738,"5min":5.4279999999999999,"15min":8.9100000000000001},"last":2.0219999999999998}]},{"osd":6,"last update":"Sat Sep 23 19:51:28 2023","interfaces":[{"interface":"back","average":{"1min":1.1020000000000001,"5min":1.101,"15min":1.6020000000000001},"min":{"1min":0.63600000000000001,"5min":0.33900000000000002,"15min":0.33300000000000002},"max":{"1min":1.8049999999999999,"5min":5.4569999999999999,"15min":22.725000000000001},"last":0.44800000000000001},{"interface":"front","average":{"1min":1.3300000000000001,"5min":1.2609999999999999,"15min":1.8169999999999999},"min":{"1min":0.49099999999999999,"5min":0.38800000000000001,"15min":0.308},"max":{"1min":6.1769999999999996,"5min":9.4510000000000005,"15min":22.707000000000001},"last":0.52100000000000002}]}]},{"osd":6,"up_from":42,"seq":180388627034,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1833152,"kb_used_data":941944,"kb_used_omap":0,"kb_used_meta":891200,"kb_avail":91912000,"statfs":{"total":95995035648,"available":94117888000,"internally_reserved":0,"allocated":964550656,"data_stored":964034194,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":912588800},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[6,0,0,0,1,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:52:15 2023","interfaces":[{"interface":"back","average":{"1min":0.96799999999999997,"5min":1.8799999999999999,"15min":2.008},"min":{"1min":0.49199999999999999,"5min":0.247,"15min":0.24199999999999999},"max":{"1min":2.847,"5min":45.600000000000001,"15min":45.600000000000001},"last":2.847},{"interface":"front","average":{"1min":0.83799999999999997,"5min":1.956,"15min":1.8120000000000001},"min":{"1min":0.40999999999999998,"5min":0.26600000000000001,"15min":0.25800000000000001},"max":{"1min":1.258,"5min":45.347999999999999,"15min":45.347999999999999},"last":0.64600000000000002}]},{"osd":1,"last update":"Sat Sep 23 19:51:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0229999999999999,"5min":1.573,"15min":2.5739999999999998},"min":{"1min":0.70499999999999996,"5min":0.309,"15min":0.26300000000000001},"max":{"1min":1.417,"5min":11.541,"15min":117.58199999999999},"last":2.7730000000000001},{"interface":"front","average":{"1min":1.0620000000000001,"5min":1.8009999999999999,"15min":2.6299999999999999},"min":{"1min":0.54300000000000004,"5min":0.32500000000000001,"15min":0.29199999999999998},"max":{"1min":1.2929999999999999,"5min":11.529,"15min":117.633},"last":0.71999999999999997}]},{"osd":2,"last update":"Sat Sep 23 19:52:05 2023","interfaces":[{"interface":"back","average":{"1min":1.0920000000000001,"5min":5.3570000000000002,"15min":4.3440000000000003},"min":{"1min":0.56699999999999995,"5min":0.308,"15min":0.30299999999999999},"max":{"1min":4.2519999999999998,"5min":165.09800000000001,"15min":165.09800000000001},"last":2.8130000000000002},{"interface":"front","average":{"1min":0.88900000000000001,"5min":5.2510000000000003,"15min":4.2850000000000001},"min":{"1min":0.45400000000000001,"5min":0.38800000000000001,"15min":0.32400000000000001},"max":{"1min":1.2929999999999999,"5min":163.93000000000001,"15min":163.93000000000001},"last":0.59899999999999998}]},{"osd":3,"last update":"Sat Sep 23 19:52:00 2023","interfaces":[{"interface":"back","average":{"1min":0.88800000000000001,"5min":2.0329999999999999,"15min":1.964},"min":{"1min":0.53800000000000003,"5min":0.31,"15min":0.30199999999999999},"max":{"1min":1.177,"5min":16.428000000000001,"15min":16.428000000000001},"last":0.75600000000000001},{"interface":"front","average":{"1min":0.997,"5min":1.7130000000000001,"15min":1.8200000000000001},"min":{"1min":0.60499999999999998,"5min":0.32200000000000001,"15min":0.307},"max":{"1min":1.508,"5min":10.192,"15min":18.917000000000002},"last":2.8799999999999999}]},{"osd":4,"last update":"Sat Sep 23 19:52:00 2023","interfaces":[{"interface":"back","average":{"1min":1.5169999999999999,"5min":1.0489999999999999,"15min":1.4379999999999999},"min":{"1min":0.41599999999999998,"5min":0.33200000000000002,"15min":0.28799999999999998},"max":{"1min":14.401999999999999,"5min":14.401999999999999,"15min":14.401999999999999},"last":1.4810000000000001},{"interface":"front","average":{"1min":1.573,"5min":1.4770000000000001,"15min":1.7230000000000001},"min":{"1min":0.57499999999999996,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":14.603,"5min":14.603,"15min":14.603},"last":0.67700000000000005}]},{"osd":5,"last update":"Sat Sep 23 19:52:00 2023","interfaces":[{"interface":"back","average":{"1min":1.0600000000000001,"5min":1.1100000000000001,"15min":1.54},"min":{"1min":0.54100000000000004,"5min":0.38500000000000001,"15min":0.32000000000000001},"max":{"1min":1.5149999999999999,"5min":6.641,"15min":12.827},"last":2.3119999999999998},{"interface":"front","average":{"1min":0.98899999999999999,"5min":1.373,"15min":1.8160000000000001},"min":{"1min":0.49299999999999999,"5min":0.35299999999999998,"15min":0.33800000000000002},"max":{"1min":1.3580000000000001,"5min":12.138999999999999,"15min":12.138999999999999},"last":2.5009999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:51:24 2023","interfaces":[{"interface":"back","average":{"1min":1.272,"5min":1.25,"15min":1.556},"min":{"1min":0.59899999999999998,"5min":0.40500000000000003,"15min":0.33200000000000002},"max":{"1min":4.4020000000000001,"5min":6.9009999999999998,"15min":11.663},"last":0.84499999999999997},{"interface":"front","average":{"1min":1.554,"5min":1.4850000000000001,"15min":1.95},"min":{"1min":0.46500000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":6.7539999999999996,"5min":12.327999999999999,"15min":12.327999999999999},"last":0.72899999999999998}]}]},{"osd":1,"up_from":98,"seq":420906795366,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1650368,"kb_used_data":980024,"kb_used_omap":4,"kb_used_meta":670331,"kb_avail":92094784,"statfs":{"total":95995035648,"available":94305058816,"internally_reserved":0,"allocated":1003544576,"data_stored":1002558471,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":686419854},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:51:42 2023","interfaces":[{"interface":"back","average":{"1min":0.79600000000000004,"5min":3.0449999999999999,"15min":3.181},"min":{"1min":0.433,"5min":0.221,"15min":0.18099999999999999},"max":{"1min":2.2759999999999998,"5min":84.195999999999998,"15min":84.195999999999998},"last":0.33400000000000002},{"interface":"front","average":{"1min":0.79400000000000004,"5min":2.6949999999999998,"15min":2.9689999999999999},"min":{"1min":0.41099999999999998,"5min":0.22600000000000001,"15min":0.20300000000000001},"max":{"1min":2.2410000000000001,"5min":84.016000000000005,"15min":84.016000000000005},"last":0.72199999999999998}]},{"osd":2,"last update":"Sat Sep 23 19:52:06 2023","interfaces":[{"interface":"back","average":{"1min":0.872,"5min":3.762,"15min":2.5139999999999998},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.23200000000000001},"max":{"1min":3.9510000000000001,"5min":72.859999999999999,"15min":72.859999999999999},"last":2.492},{"interface":"front","average":{"1min":0.82299999999999995,"5min":3.2749999999999999,"15min":2.4630000000000001},"min":{"1min":0.52100000000000002,"5min":0.25800000000000001,"15min":0.23300000000000001},"max":{"1min":1.7070000000000001,"5min":73.665999999999997,"15min":73.665999999999997},"last":0.873}]},{"osd":3,"last update":"Sat Sep 23 19:51:58 2023","interfaces":[{"interface":"back","average":{"1min":0.99399999999999999,"5min":1.96,"15min":1.7250000000000001},"min":{"1min":0.51300000000000001,"5min":0.25,"15min":0.25},"max":{"1min":4.3399999999999999,"5min":10.273999999999999,"15min":15.004},"last":0.29299999999999998},{"interface":"front","average":{"1min":0.94499999999999995,"5min":1.8280000000000001,"15min":1.9139999999999999},"min":{"1min":0.52800000000000002,"5min":0.217,"15min":0.217},"max":{"1min":5.4370000000000003,"5min":11.683999999999999,"15min":48.692},"last":0.53900000000000003}]},{"osd":4,"last update":"Sat Sep 23 19:51:42 2023","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":1.4950000000000001,"15min":1.623},"min":{"1min":0.47899999999999998,"5min":0.313,"15min":0.313},"max":{"1min":1.659,"5min":10.378,"15min":53.128999999999998},"last":1.089},{"interface":"front","average":{"1min":0.92200000000000004,"5min":2.274,"15min":1.9399999999999999},"min":{"1min":0.49199999999999999,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":1.3160000000000001,"5min":82.239999999999995,"15min":82.239999999999995},"last":0.57699999999999996}]},{"osd":5,"last update":"Sat Sep 23 19:51:42 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":1.4730000000000001,"15min":1.673},"min":{"1min":0.432,"5min":0.36199999999999999,"15min":0.34899999999999998},"max":{"1min":1.2529999999999999,"5min":11.750999999999999,"15min":48.738},"last":1.734},{"interface":"front","average":{"1min":1.0489999999999999,"5min":1.675,"15min":1.746},"min":{"1min":0.60099999999999998,"5min":0.36199999999999999,"15min":0.35799999999999998},"max":{"1min":1.552,"5min":10.611000000000001,"15min":48.726999999999997},"last":0.68000000000000005}]},{"osd":6,"last update":"Sat Sep 23 19:51:42 2023","interfaces":[{"interface":"back","average":{"1min":1.129,"5min":2.0979999999999999,"15min":2.2200000000000002},"min":{"1min":0.70099999999999996,"5min":0.41699999999999998,"15min":0.318},"max":{"1min":1.6120000000000001,"5min":10.565,"15min":48.762},"last":2.444},{"interface":"front","average":{"1min":1.149,"5min":1.8300000000000001,"15min":2.0990000000000002},"min":{"1min":0.54900000000000004,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":1.712,"5min":7.8799999999999999,"15min":53.076999999999998},"last":2.46}]},{"osd":7,"last update":"Sat Sep 23 19:51:42 2023","interfaces":[{"interface":"back","average":{"1min":1.167,"5min":1.8520000000000001,"15min":1.859},"min":{"1min":0.51600000000000001,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":1.579,"5min":10.475,"15min":48.677},"last":2.8479999999999999},{"interface":"front","average":{"1min":1.0569999999999999,"5min":1.8460000000000001,"15min":1.869},"min":{"1min":0.46500000000000002,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":1.405,"5min":11.778,"15min":48.805},"last":2.8220000000000001}]}]},{"osd":0,"up_from":85,"seq":365072220572,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1307384,"kb_used_data":529904,"kb_used_omap":5,"kb_used_meta":777466,"kb_avail":92437768,"statfs":{"total":95995035648,"available":94656274432,"internally_reserved":0,"allocated":542621696,"data_stored":541646132,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":796125818},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 19:51:35 2023","interfaces":[{"interface":"back","average":{"1min":0.96599999999999997,"5min":1.6299999999999999,"15min":1.7669999999999999},"min":{"1min":0.34999999999999998,"5min":0.216,"15min":0.20799999999999999},"max":{"1min":3.9580000000000002,"5min":10.111000000000001,"15min":57.234999999999999},"last":0.27100000000000002},{"interface":"front","average":{"1min":0.97299999999999998,"5min":1.702,"15min":1.982},"min":{"1min":0.45600000000000002,"5min":0.20399999999999999,"15min":0.189},"max":{"1min":5.2270000000000003,"5min":8.6750000000000007,"15min":58.825000000000003},"last":0.34599999999999997}]},{"osd":2,"last update":"Sat Sep 23 19:52:09 2023","interfaces":[{"interface":"back","average":{"1min":0.745,"5min":4.4089999999999998,"15min":3.645},"min":{"1min":0.38200000000000001,"5min":0.245,"15min":0.21299999999999999},"max":{"1min":2.2810000000000001,"5min":213.20599999999999,"15min":262.59500000000003},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.61399999999999999,"5min":3.8929999999999998,"15min":3.5579999999999998},"min":{"1min":0.40200000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.92000000000000004,"5min":209.64400000000001,"15min":263.25700000000001},"last":0.92800000000000005}]},{"osd":3,"last update":"Sat Sep 23 19:51:41 2023","interfaces":[{"interface":"back","average":{"1min":0.877,"5min":1.8280000000000001,"15min":1.98},"min":{"1min":0.52200000000000002,"5min":0.222,"15min":0.222},"max":{"1min":2.294,"5min":11.436,"15min":14.212},"last":0.96599999999999997},{"interface":"front","average":{"1min":0.754,"5min":1.909,"15min":1.909},"min":{"1min":0.48799999999999999,"5min":0.28999999999999998,"15min":0.223},"max":{"1min":1.1930000000000001,"5min":8.0809999999999995,"15min":14.84},"last":0.46999999999999997}]},{"osd":4,"last update":"Sat Sep 23 19:51:19 2023","interfaces":[{"interface":"back","average":{"1min":1.1180000000000001,"5min":1.4670000000000001,"15min":1.635},"min":{"1min":0.376,"5min":0.35199999999999998,"15min":0.32800000000000001},"max":{"1min":3.6869999999999998,"5min":9.0679999999999996,"15min":11.792999999999999},"last":0.998},{"interface":"front","average":{"1min":1.581,"5min":1.6930000000000001,"15min":1.732},"min":{"1min":0.50900000000000001,"5min":0.41399999999999998,"15min":0.315},"max":{"1min":5.3140000000000001,"5min":8.6289999999999996,"15min":13.641999999999999},"last":0.43099999999999999}]},{"osd":5,"last update":"Sat Sep 23 19:51:19 2023","interfaces":[{"interface":"back","average":{"1min":1.738,"5min":1.599,"15min":1.7250000000000001},"min":{"1min":0.442,"5min":0.39200000000000002,"15min":0.33700000000000002},"max":{"1min":5.3689999999999998,"5min":6.9370000000000003,"15min":8.6229999999999993},"last":0.50800000000000001},{"interface":"front","average":{"1min":1.1120000000000001,"5min":1.4710000000000001,"15min":1.7450000000000001},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":3.629,"5min":9.0329999999999995,"15min":10.411},"last":1.036}]},{"osd":6,"last update":"Sat Sep 23 19:51:19 2023","interfaces":[{"interface":"back","average":{"1min":1.357,"5min":2.206,"15min":2.1640000000000001},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":3.823,"5min":66.170000000000002,"15min":66.170000000000002},"last":1.0580000000000001},{"interface":"front","average":{"1min":2.004,"5min":2.472,"15min":2.2589999999999999},"min":{"1min":0.47299999999999998,"5min":0.45900000000000002,"15min":0.35299999999999998},"max":{"1min":5.3419999999999996,"5min":66.260999999999996,"15min":66.260999999999996},"last":0.57299999999999995}]},{"osd":7,"last update":"Sat Sep 23 19:51:19 2023","interfaces":[{"interface":"back","average":{"1min":1.5860000000000001,"5min":1.6819999999999999,"15min":1.758},"min":{"1min":0.44800000000000001,"5min":0.40400000000000003,"15min":0.38200000000000001},"max":{"1min":4.6879999999999997,"5min":10.082000000000001,"15min":13.672000000000001},"last":0.60299999999999998},{"interface":"front","average":{"1min":1.411,"5min":1.786,"15min":1.821},"min":{"1min":0.439,"5min":0.41199999999999998,"15min":0.40100000000000002},"max":{"1min":5.5830000000000002,"5min":11.863,"15min":11.863},"last":1.075}]}]},{"osd":2,"up_from":159,"seq":682899800228,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2156864,"kb_used_data":1670008,"kb_used_omap":7,"kb_used_meta":486840,"kb_avail":91588288,"statfs":{"total":95995035648,"available":93786406912,"internally_reserved":0,"allocated":1710088192,"data_stored":1709159143,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":498524241},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:51:21 2023","interfaces":[{"interface":"back","average":{"1min":1.804,"5min":3.0739999999999998,"15min":2.323},"min":{"1min":0.35199999999999998,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":15.249000000000001,"5min":117.77500000000001,"15min":117.77500000000001},"last":0.26800000000000002},{"interface":"front","average":{"1min":1.7050000000000001,"5min":3.8780000000000001,"15min":2.6360000000000001},"min":{"1min":0.40400000000000003,"5min":0.214,"15min":0.214},"max":{"1min":10.914999999999999,"5min":117.91800000000001,"15min":117.91800000000001},"last":0.48699999999999999}]},{"osd":1,"last update":"Sat Sep 23 19:51:21 2023","interfaces":[{"interface":"back","average":{"1min":1.387,"5min":3.4209999999999998,"15min":2.4529999999999998},"min":{"1min":0.34999999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":6.9089999999999998,"5min":114.107,"15min":114.107},"last":0.95499999999999996},{"interface":"front","average":{"1min":1.4379999999999999,"5min":3.044,"15min":2.2770000000000001},"min":{"1min":0.38600000000000001,"5min":0.23699999999999999,"15min":0.22},"max":{"1min":8.6560000000000006,"5min":117.172,"15min":117.172},"last":0.30299999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:51:21 2023","interfaces":[{"interface":"back","average":{"1min":1.5,"5min":2.4569999999999999,"15min":2.1429999999999998},"min":{"1min":0.41799999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":6.9459999999999997,"5min":11.128,"15min":17.736999999999998},"last":1.0009999999999999},{"interface":"front","average":{"1min":1.411,"5min":2.0680000000000001,"15min":2.0830000000000002},"min":{"1min":0.41499999999999998,"5min":0.24099999999999999,"15min":0.21299999999999999},"max":{"1min":9.4009999999999998,"5min":20.286999999999999,"15min":20.286999999999999},"last":0.51100000000000001}]},{"osd":4,"last update":"Sat Sep 23 19:51:21 2023","interfaces":[{"interface":"back","average":{"1min":1.2729999999999999,"5min":2.1160000000000001,"15min":1.887},"min":{"1min":0.52100000000000002,"5min":0.317,"15min":0.317},"max":{"1min":5.6360000000000001,"5min":10.622999999999999,"15min":18.318999999999999},"last":0.621},{"interface":"front","average":{"1min":1.5109999999999999,"5min":2.0680000000000001,"15min":1.776},"min":{"1min":0.51000000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":4.7240000000000002,"5min":12.302,"15min":12.302},"last":1.036}]},{"osd":5,"last update":"Sat Sep 23 19:51:21 2023","interfaces":[{"interface":"back","average":{"1min":1.296,"5min":1.216,"15min":1.542},"min":{"1min":0.47999999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":5.5099999999999998,"5min":5.5099999999999998,"15min":16.577000000000002},"last":1.9550000000000001},{"interface":"front","average":{"1min":1.2749999999999999,"5min":2.0470000000000002,"15min":1.9399999999999999},"min":{"1min":0.48699999999999999,"5min":0.36799999999999999,"15min":0.33600000000000002},"max":{"1min":5.6849999999999996,"5min":10.193,"15min":10.193},"last":0.77100000000000002}]},{"osd":6,"last update":"Sat Sep 23 19:51:21 2023","interfaces":[{"interface":"back","average":{"1min":1.4339999999999999,"5min":2.1920000000000002,"15min":2.1709999999999998},"min":{"1min":0.81899999999999995,"5min":0.38900000000000001,"15min":0.36899999999999999},"max":{"1min":5.7839999999999998,"5min":10.166,"15min":11.851000000000001},"last":0.71199999999999997},{"interface":"front","average":{"1min":1.6990000000000001,"5min":2.3420000000000001,"15min":2.1840000000000002},"min":{"1min":0.48499999999999999,"5min":0.38,"15min":0.36699999999999999},"max":{"1min":5.1109999999999998,"5min":12.256,"15min":12.297000000000001},"last":1.119}]},{"osd":7,"last update":"Sat Sep 23 19:51:21 2023","interfaces":[{"interface":"back","average":{"1min":1.391,"5min":1.9139999999999999,"15min":1.8700000000000001},"min":{"1min":0.45600000000000002,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":5.468,"5min":32.945,"15min":32.945},"last":0.64400000000000002},{"interface":"front","average":{"1min":1.635,"5min":2.3100000000000001,"15min":1.9710000000000001},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.41299999999999998},"max":{"1min":6.7910000000000004,"5min":16.593,"15min":16.593},"last":1.3420000000000001}]}]},{"osd":3,"up_from":116,"seq":498216206629,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1427716,"kb_used_data":799996,"kb_used_omap":2,"kb_used_meta":627709,"kb_avail":92317436,"statfs":{"total":95995035648,"available":94533054464,"internally_reserved":0,"allocated":819195904,"data_stored":818321002,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":642774069},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:51:54 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":2.7559999999999998,"15min":2.0430000000000001},"min":{"1min":0.36599999999999999,"5min":0.23799999999999999,"15min":0.21199999999999999},"max":{"1min":0.89700000000000002,"5min":50.771999999999998,"15min":50.771999999999998},"last":0.72999999999999998},{"interface":"front","average":{"1min":0.60999999999999999,"5min":3.2589999999999999,"15min":2.2690000000000001},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.20100000000000001},"max":{"1min":1.282,"5min":48.887,"15min":48.887},"last":0.61499999999999999}]},{"osd":1,"last update":"Sat Sep 23 19:51:54 2023","interfaces":[{"interface":"back","average":{"1min":0.73599999999999999,"5min":2.4969999999999999,"15min":2.1179999999999999},"min":{"1min":0.28299999999999997,"5min":0.25800000000000001,"15min":0.22600000000000001},"max":{"1min":1.9359999999999999,"5min":48.918999999999997,"15min":92.171000000000006},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.69899999999999995,"5min":2.4540000000000002,"15min":2.2400000000000002},"min":{"1min":0.42699999999999999,"5min":0.25800000000000001,"15min":0.23000000000000001},"max":{"1min":0.92900000000000005,"5min":50.722999999999999,"15min":87.771000000000001},"last":0.68300000000000005}]},{"osd":2,"last update":"Sat Sep 23 19:52:11 2023","interfaces":[{"interface":"back","average":{"1min":0.79800000000000004,"5min":4.8949999999999996,"15min":3.1219999999999999},"min":{"1min":0.32900000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":2.0910000000000002,"5min":116.363,"15min":116.363},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.92600000000000005,"5min":4.6219999999999999,"15min":2.71},"min":{"1min":0.46300000000000002,"5min":0.27300000000000002,"15min":0.252},"max":{"1min":3.504,"5min":118.474,"15min":118.474},"last":1.984}]},{"osd":4,"last update":"Sat Sep 23 19:51:54 2023","interfaces":[{"interface":"back","average":{"1min":1.1140000000000001,"5min":1.5840000000000001,"15min":1.712},"min":{"1min":0.67500000000000004,"5min":0.33100000000000002,"15min":0.32500000000000001},"max":{"1min":2.645,"5min":8.6639999999999997,"15min":11.666},"last":2.0640000000000001},{"interface":"front","average":{"1min":1.1439999999999999,"5min":1.9930000000000001,"15min":1.8180000000000001},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.32300000000000001},"max":{"1min":2.7549999999999999,"5min":9.9990000000000006,"15min":9.9990000000000006},"last":2.0790000000000002}]},{"osd":5,"last update":"Sat Sep 23 19:51:54 2023","interfaces":[{"interface":"back","average":{"1min":0.94499999999999995,"5min":1.998,"15min":1.746},"min":{"1min":0.373,"5min":0.373,"15min":0.35299999999999998},"max":{"1min":1.891,"5min":10.022,"15min":10.022},"last":4.5250000000000004},{"interface":"front","average":{"1min":0.97199999999999998,"5min":1.8160000000000001,"15min":1.758},"min":{"1min":0.46200000000000002,"5min":0.38900000000000001,"15min":0.34000000000000002},"max":{"1min":1.615,"5min":11.507,"15min":13.146000000000001},"last":0.66000000000000003}]},{"osd":6,"last update":"Sat Sep 23 19:51:54 2023","interfaces":[{"interface":"back","average":{"1min":1.0780000000000001,"5min":2.423,"15min":2.2440000000000002},"min":{"1min":0.63,"5min":0.42499999999999999,"15min":0.36299999999999999},"max":{"1min":1.8580000000000001,"5min":10.052,"15min":15.919},"last":0.77500000000000002},{"interface":"front","average":{"1min":1.0129999999999999,"5min":2.0489999999999999,"15min":1.9410000000000001},"min":{"1min":0.44800000000000001,"5min":0.40400000000000003,"15min":0.315},"max":{"1min":1.5309999999999999,"5min":11.456,"15min":11.456},"last":0.70399999999999996}]},{"osd":7,"last update":"Sat Sep 23 19:51:54 2023","interfaces":[{"interface":"back","average":{"1min":1.0549999999999999,"5min":1.8959999999999999,"15min":1.9259999999999999},"min":{"1min":0.42899999999999999,"5min":0.42299999999999999,"15min":0.35799999999999998},"max":{"1min":1.5389999999999999,"5min":11.486000000000001,"15min":14.098000000000001},"last":0.85699999999999998},{"interface":"front","average":{"1min":1.0129999999999999,"5min":1.885,"15min":1.9239999999999999},"min":{"1min":0.70399999999999996,"5min":0.40600000000000003,"15min":0.40300000000000002},"max":{"1min":1.361,"5min":8.7059999999999995,"15min":11.744999999999999},"last":1.236}]}]},{"osd":4,"up_from":30,"seq":128849019499,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2584840,"kb_used_data":1676736,"kb_used_omap":0,"kb_used_meta":908096,"kb_avail":91160312,"statfs":{"total":95995035648,"available":93348159488,"internally_reserved":0,"allocated":1716977664,"data_stored":1716452857,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":929890304},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,2,0,2,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:51:24 2023","interfaces":[{"interface":"back","average":{"1min":1.1839999999999999,"5min":2.9830000000000001,"15min":1.994},"min":{"1min":0.311,"5min":0.25600000000000001,"15min":0.23300000000000001},"max":{"1min":7.694,"5min":150.155,"15min":150.155},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.91800000000000004,"5min":2.8839999999999999,"15min":1.839},"min":{"1min":0.48399999999999999,"5min":0.28100000000000003,"15min":0.27700000000000002},"max":{"1min":2.0019999999999998,"5min":149.47900000000001,"15min":149.47900000000001},"last":0.371}]},{"osd":1,"last update":"Sat Sep 23 19:51:46 2023","interfaces":[{"interface":"back","average":{"1min":0.91400000000000003,"5min":1.27,"15min":1.8300000000000001},"min":{"1min":0.60899999999999999,"5min":0.309,"15min":0.27800000000000002},"max":{"1min":1.3600000000000001,"5min":9.0589999999999993,"15min":101.786},"last":0.438},{"interface":"front","average":{"1min":1.0549999999999999,"5min":1.248,"15min":1.982},"min":{"1min":0.71299999999999997,"5min":0.34899999999999998,"15min":0.28499999999999998},"max":{"1min":1.4490000000000001,"5min":9.0869999999999997,"15min":101.726},"last":0.52300000000000002}]},{"osd":2,"last update":"Sat Sep 23 19:52:09 2023","interfaces":[{"interface":"back","average":{"1min":3.0630000000000002,"5min":7.1349999999999998,"15min":4.1050000000000004},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":40.906999999999996,"5min":219.02799999999999,"15min":219.02799999999999},"last":0.41499999999999998},{"interface":"front","average":{"1min":3.044,"5min":7.181,"15min":3.8490000000000002},"min":{"1min":0.52600000000000002,"5min":0.318,"15min":0.29899999999999999},"max":{"1min":39.210999999999999,"5min":219.65600000000001,"15min":219.65600000000001},"last":0.34200000000000003}]},{"osd":3,"last update":"Sat Sep 23 19:52:09 2023","interfaces":[{"interface":"back","average":{"1min":0.94699999999999995,"5min":1.512,"15min":1.831},"min":{"1min":0.436,"5min":0.32800000000000001,"15min":0.32600000000000001},"max":{"1min":1.9370000000000001,"5min":17.873999999999999,"15min":27.789000000000001},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.89200000000000002,"5min":1.4710000000000001,"15min":1.716},"min":{"1min":0.32500000000000001,"5min":0.32300000000000001,"15min":0.32100000000000001},"max":{"1min":1.1799999999999999,"5min":13.859,"15min":27.808},"last":0.49099999999999999}]},{"osd":5,"last update":"Sat Sep 23 19:51:24 2023","interfaces":[{"interface":"back","average":{"1min":0.86799999999999999,"5min":0.92700000000000005,"15min":1.4219999999999999},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.315},"max":{"1min":1.226,"5min":4.242,"15min":22.442},"last":5.6459999999999999},{"interface":"front","average":{"1min":0.93600000000000005,"5min":1.008,"15min":1.401},"min":{"1min":0.41199999999999998,"5min":0.33900000000000002,"15min":0.28899999999999998},"max":{"1min":2.04,"5min":5.1440000000000001,"15min":10.362},"last":4.2329999999999997}]},{"osd":6,"last update":"Sat Sep 23 19:51:46 2023","interfaces":[{"interface":"back","average":{"1min":1.077,"5min":1.2789999999999999,"15min":1.696},"min":{"1min":0.69999999999999996,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":1.4690000000000001,"5min":18.103000000000002,"15min":22.544},"last":3.907},{"interface":"front","average":{"1min":1.1359999999999999,"5min":1.327,"15min":1.53},"min":{"1min":0.68000000000000005,"5min":0.31,"15min":0.31},"max":{"1min":2.0830000000000002,"5min":19.706,"15min":19.706},"last":2.2709999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:51:46 2023","interfaces":[{"interface":"back","average":{"1min":1.0429999999999999,"5min":1.248,"15min":2.5640000000000001},"min":{"1min":0.78100000000000003,"5min":0.33100000000000002,"15min":0.30299999999999999},"max":{"1min":1.3420000000000001,"5min":8.3399999999999999,"15min":294.76499999999999},"last":0.308},{"interface":"front","average":{"1min":1.1830000000000001,"5min":1.2430000000000001,"15min":2.774},"min":{"1min":0.94099999999999995,"5min":0.39500000000000002,"15min":0.374},"max":{"1min":1.464,"5min":10.371,"15min":294.47699999999998},"last":0.39000000000000001}]}]},{"osd":5,"up_from":36,"seq":154618823266,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2403952,"kb_used_data":1492968,"kb_used_omap":0,"kb_used_meta":910976,"kb_avail":91341200,"statfs":{"total":95995035648,"available":93533388800,"internally_reserved":0,"allocated":1528799232,"data_stored":1528294115,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":932839424},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:52:17 2023","interfaces":[{"interface":"back","average":{"1min":1.1910000000000001,"5min":1.5529999999999999,"15min":1.5649999999999999},"min":{"1min":0.316,"5min":0.28699999999999998,"15min":0.253},"max":{"1min":4.0510000000000002,"5min":28.280000000000001,"15min":43.094000000000001},"last":1.831},{"interface":"front","average":{"1min":1.3839999999999999,"5min":1.966,"15min":1.9410000000000001},"min":{"1min":0.59499999999999997,"5min":0.312,"15min":0.252},"max":{"1min":6.4039999999999999,"5min":28.309999999999999,"15min":43.802999999999997},"last":6.4039999999999999}]},{"osd":1,"last update":"Sat Sep 23 19:51:43 2023","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":1.248,"15min":1.3560000000000001},"min":{"1min":0.60799999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":1.226,"5min":6.9660000000000002,"15min":11.266999999999999},"last":1.9139999999999999},{"interface":"front","average":{"1min":0.92500000000000004,"5min":1.3759999999999999,"15min":1.5820000000000001},"min":{"1min":0.68400000000000005,"5min":0.28799999999999998,"15min":0.27900000000000003},"max":{"1min":1.2549999999999999,"5min":9.0210000000000008,"15min":18.207000000000001},"last":6.4359999999999999}]},{"osd":2,"last update":"Sat Sep 23 19:52:02 2023","interfaces":[{"interface":"back","average":{"1min":0.874,"5min":4.5229999999999997,"15min":3.2839999999999998},"min":{"1min":0.56299999999999994,"5min":0.309,"15min":0.309},"max":{"1min":1.5389999999999999,"5min":241.59700000000001,"15min":241.59700000000001},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.97899999999999998,"5min":4.2190000000000003,"15min":2.931},"min":{"1min":0.64400000000000002,"5min":0.34899999999999998,"15min":0.32000000000000001},"max":{"1min":1.3620000000000001,"5min":242.13999999999999,"15min":242.13999999999999},"last":1.9299999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:51:43 2023","interfaces":[{"interface":"back","average":{"1min":0.98599999999999999,"5min":1.222,"15min":1.4790000000000001},"min":{"1min":0.63200000000000001,"5min":0.35099999999999998,"15min":0.33700000000000002},"max":{"1min":1.226,"5min":7.4580000000000002,"15min":10.686},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.996,"5min":1.234,"15min":1.5700000000000001},"min":{"1min":0.61399999999999999,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":1.284,"5min":6.1870000000000003,"15min":8.5879999999999992},"last":1.9470000000000001}]},{"osd":4,"last update":"Sat Sep 23 19:52:17 2023","interfaces":[{"interface":"back","average":{"1min":1.629,"5min":1.1339999999999999,"15min":1.593},"min":{"1min":0.70099999999999996,"5min":0.32200000000000001,"15min":0.307},"max":{"1min":6.3550000000000004,"5min":6.585,"15min":74.221000000000004},"last":6.3550000000000004},{"interface":"front","average":{"1min":1.1659999999999999,"5min":0.95799999999999996,"15min":1.5860000000000001},"min":{"1min":0.40799999999999997,"5min":0.318,"15min":0.318},"max":{"1min":3.6640000000000001,"5min":6.266,"15min":73.772999999999996},"last":0.40799999999999997}]},{"osd":6,"last update":"Sat Sep 23 19:52:02 2023","interfaces":[{"interface":"back","average":{"1min":1.075,"5min":1.0129999999999999,"15min":1.3979999999999999},"min":{"1min":0.72899999999999998,"5min":0.36099999999999999,"15min":0.35599999999999998},"max":{"1min":1.4039999999999999,"5min":3.8010000000000002,"15min":16.222000000000001},"last":1.859},{"interface":"front","average":{"1min":1.075,"5min":1.1459999999999999,"15min":1.536},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":1.5029999999999999,"5min":5.984,"15min":14.178000000000001},"last":0.59299999999999997}]},{"osd":7,"last update":"Sat Sep 23 19:52:17 2023","interfaces":[{"interface":"back","average":{"1min":1.2669999999999999,"5min":1.1619999999999999,"15min":1.4690000000000001},"min":{"1min":0.40600000000000003,"5min":0.35299999999999998,"15min":0.32100000000000001},"max":{"1min":3.694,"5min":9.8819999999999997,"15min":14.127000000000001},"last":1.389},{"interface":"front","average":{"1min":1.623,"5min":1.2190000000000001,"15min":1.4650000000000001},"min":{"1min":0.68400000000000005,"5min":0.33900000000000002,"15min":0.307},"max":{"1min":4.9539999999999997,"5min":5.5439999999999996,"15min":9.0440000000000005},"last":1.8839999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":550633472,"data_stored":550633472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1016659968,"data_stored":1016659968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1738407936,"data_stored":1738407936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":825425920,"data_stored":825425920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1745879040,"data_stored":1745879040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1561133056,"data_stored":1561133056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":983760896,"data_stored":983760896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":979894272,"data_stored":979894272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T19:52:26.257 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T19:52:26.258 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T19:52:26.258 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2023-09-23T19:52:26.258 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T19:52:26.258 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T19:52:26.258 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 config set filestore_inject_stall 3 2023-09-23T19:52:26.260 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:26.271 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:26 smithi016 bash[75004]: cluster 2023-09-23T19:52:24.456151+0000 mgr.y (mgr.25325) 1353 : cluster 0 pgmap v1356: 89 pgs: 89 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-09-23T19:52:26.271 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:26 smithi016 bash[77995]: cluster 2023-09-23T19:52:24.456151+0000 mgr.y (mgr.25325) 1353 : cluster 0 pgmap v1356: 89 pgs: 89 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-09-23T19:52:26.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:26 smithi092 bash[58855]: cluster 2023-09-23T19:52:24.456151+0000 mgr.y (mgr.25325) 1353 : cluster 0 pgmap v1356: 89 pgs: 89 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-09-23T19:52:26.291 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:26.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:26.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:26.397 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:26.434 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:26.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:26.439 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:26.478 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 34 16 55596 55580 102.16 63.125 0.00680377 0.0097819 2023-09-23T19:52:26.540 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:26.569 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:26.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:26.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:26.675 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:26.708 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:26.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:26.712 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:26.812 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:26.839 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:26.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:26.843 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:52:26.944 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:26.968 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:26.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:26.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:27.076 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:27.103 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:27.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:27.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:27.206 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:27.250 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:27.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:27.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:27.357 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:27.381 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:27.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:27.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:52:27.431 INFO:teuthology.orchestra.run.smithi092.stdout:{ 2023-09-23T19:52:27.431 INFO:teuthology.orchestra.run.smithi092.stdout: "size": 20, 2023-09-23T19:52:27.431 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 600, 2023-09-23T19:52:27.431 INFO:teuthology.orchestra.run.smithi092.stdout: "ops": [ 2023-09-23T19:52:27.431 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.432 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.36243.0:105219 11.e 11:72ef5cba:::benchmark_data_smithi016_160148_object105218:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e190)", 2023-09-23T19:52:27.432 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:43:11.584057+0000", 2023-09-23T19:52:27.432 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 555.84075678299996, 2023-09-23T19:52:27.432 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.14955489999999999, 2023-09-23T19:52:27.432 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:52:27.432 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:52:27.432 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:52:27.432 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.36243", 2023-09-23T19:52:27.432 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/1280038109", 2023-09-23T19:52:27.432 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 105219 2023-09-23T19:52:27.432 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.433 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:52:27.433 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.433 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:52:27.433 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:11.584057+0000", 2023-09-23T19:52:27.433 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:52:27.433 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.433 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.433 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:52:27.433 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:11.584057+0000", 2023-09-23T19:52:27.433 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999976 2023-09-23T19:52:27.433 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.434 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.434 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:52:27.434 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:11.584054+0000", 2023-09-23T19:52:27.434 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.5451999999999998e-05 2023-09-23T19:52:27.434 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.434 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.434 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:52:27.434 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:11.584100+0000", 2023-09-23T19:52:27.434 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.5624000000000001e-05 2023-09-23T19:52:27.434 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.434 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.434 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:52:27.435 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:11.584125+0000", 2023-09-23T19:52:27.435 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.0162e-05 2023-09-23T19:52:27.435 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.435 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.435 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:52:27.435 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:11.584136+0000", 2023-09-23T19:52:27.435 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.6305999999999998e-05 2023-09-23T19:52:27.435 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.435 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.435 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:52:27.435 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:11.584202+0000", 2023-09-23T19:52:27.436 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00032370000000000001 2023-09-23T19:52:27.436 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.436 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.436 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:52:27.436 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:11.584526+0000", 2023-09-23T19:52:27.436 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000216465 2023-09-23T19:52:27.436 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.436 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.436 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,7", 2023-09-23T19:52:27.436 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:11.584742+0000", 2023-09-23T19:52:27.436 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0035166419999999999 2023-09-23T19:52:27.437 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.437 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.437 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:52:27.437 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:11.588259+0000", 2023-09-23T19:52:27.437 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0086689790000000003 2023-09-23T19:52:27.437 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.437 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.437 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:27.437 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:11.596928+0000", 2023-09-23T19:52:27.437 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.13654532699999999 2023-09-23T19:52:27.437 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.438 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.438 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:27.438 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:11.733473+0000", 2023-09-23T19:52:27.438 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.5894000000000002e-05 2023-09-23T19:52:27.438 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.438 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.438 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:52:27.438 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:11.733549+0000", 2023-09-23T19:52:27.438 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.2768999999999995e-05 2023-09-23T19:52:27.438 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.438 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.439 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:52:27.439 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:43:11.733612+0000", 2023-09-23T19:52:27.439 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.14955489999999999 2023-09-23T19:52:27.439 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:52:27.439 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:52:27.439 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:52:27.439 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.439 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.439 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.36480.0:22037 12.0 12:0cbbf7b1:::benchmark_data_smithi016_181952_object22036:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e211)", 2023-09-23T19:52:27.439 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:46:44.776522+0000", 2023-09-23T19:52:27.439 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 342.648291116, 2023-09-23T19:52:27.440 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.20674447900000001, 2023-09-23T19:52:27.440 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:52:27.440 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:52:27.440 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:52:27.440 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.36480", 2023-09-23T19:52:27.440 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/1035295215", 2023-09-23T19:52:27.440 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 22037 2023-09-23T19:52:27.440 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.440 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:52:27.440 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.440 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:52:27.440 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.776522+0000", 2023-09-23T19:52:27.441 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:52:27.441 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.441 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.441 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:52:27.441 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.776522+0000", 2023-09-23T19:52:27.442 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999895 2023-09-23T19:52:27.442 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.442 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.442 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:52:27.442 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.776512+0000", 2023-09-23T19:52:27.442 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.4678000000000002e-05 2023-09-23T19:52:27.442 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.442 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.442 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:52:27.443 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.776557+0000", 2023-09-23T19:52:27.443 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.9806e-05 2023-09-23T19:52:27.443 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.443 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.443 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:52:27.443 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.776576+0000", 2023-09-23T19:52:27.443 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.7700000000000001e-06 2023-09-23T19:52:27.443 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.443 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.443 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:52:27.443 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.776584+0000", 2023-09-23T19:52:27.443 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.1663999999999997e-05 2023-09-23T19:52:27.444 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.444 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.444 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:52:27.444 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.776656+0000", 2023-09-23T19:52:27.444 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00032146900000000001 2023-09-23T19:52:27.444 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.444 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.444 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:52:27.444 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.776977+0000", 2023-09-23T19:52:27.444 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00020193899999999999 2023-09-23T19:52:27.444 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.444 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.445 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 2,6", 2023-09-23T19:52:27.445 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.777179+0000", 2023-09-23T19:52:27.445 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0014039409999999999 2023-09-23T19:52:27.445 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.445 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.445 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:52:27.445 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.778583+0000", 2023-09-23T19:52:27.445 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00032881499999999999 2023-09-23T19:52:27.445 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.445 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.445 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:27.446 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.778912+0000", 2023-09-23T19:52:27.446 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.20420666200000001 2023-09-23T19:52:27.446 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.446 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.446 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:27.446 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.983119+0000", 2023-09-23T19:52:27.446 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.5593000000000006e-05 2023-09-23T19:52:27.446 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.446 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.446 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:52:27.446 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.983194+0000", 2023-09-23T19:52:27.447 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.2633999999999996e-05 2023-09-23T19:52:27.447 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.447 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.447 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:52:27.447 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.983267+0000", 2023-09-23T19:52:27.447 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.20674447900000001 2023-09-23T19:52:27.447 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:52:27.447 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:52:27.447 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:52:27.447 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.447 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.447 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.36480.0:64347 12.0 12:03e5b357:::benchmark_data_smithi016_181952_object64346:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e214)", 2023-09-23T19:52:27.448 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:47:19.786096+0000", 2023-09-23T19:52:27.448 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 307.63871718299998, 2023-09-23T19:52:27.448 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.31235667299999997, 2023-09-23T19:52:27.448 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:52:27.448 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:52:27.448 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:52:27.448 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.36480", 2023-09-23T19:52:27.448 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/1035295215", 2023-09-23T19:52:27.448 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 64347 2023-09-23T19:52:27.448 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.448 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:52:27.448 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.449 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:52:27.449 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:19.786096+0000", 2023-09-23T19:52:27.449 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:52:27.449 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.449 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.449 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:52:27.449 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:19.786096+0000", 2023-09-23T19:52:27.449 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999976 2023-09-23T19:52:27.449 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.449 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.449 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:52:27.449 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:19.786094+0000", 2023-09-23T19:52:27.450 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.9148999999999999e-05 2023-09-23T19:52:27.450 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.450 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.450 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:52:27.450 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:19.786153+0000", 2023-09-23T19:52:27.450 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.9096000000000001e-05 2023-09-23T19:52:27.450 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.450 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.450 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:52:27.450 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:19.786172+0000", 2023-09-23T19:52:27.450 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.4719999999999995e-06 2023-09-23T19:52:27.451 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.451 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.451 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:52:27.451 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:19.786180+0000", 2023-09-23T19:52:27.451 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.8369999999999995e-05 2023-09-23T19:52:27.451 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.451 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.451 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:52:27.451 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:19.786279+0000", 2023-09-23T19:52:27.451 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000186775 2023-09-23T19:52:27.451 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.451 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.452 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:52:27.452 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:19.786466+0000", 2023-09-23T19:52:27.452 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00023910500000000001 2023-09-23T19:52:27.452 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.452 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.452 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 2,6", 2023-09-23T19:52:27.452 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:19.786705+0000", 2023-09-23T19:52:27.452 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0036454830000000001 2023-09-23T19:52:27.452 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.452 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.452 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:52:27.453 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:19.790350+0000", 2023-09-23T19:52:27.453 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00096989400000000001 2023-09-23T19:52:27.453 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.453 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.453 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:27.453 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:19.791320+0000", 2023-09-23T19:52:27.453 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.30701963999999998 2023-09-23T19:52:27.453 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.453 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.453 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:27.453 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:20.098340+0000", 2023-09-23T19:52:27.453 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.5290000000000007e-05 2023-09-23T19:52:27.454 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.454 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.454 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:52:27.454 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:20.098405+0000", 2023-09-23T19:52:27.454 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.7972000000000001e-05 2023-09-23T19:52:27.454 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.454 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.454 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:52:27.454 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:20.098453+0000", 2023-09-23T19:52:27.454 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.31235667299999997 2023-09-23T19:52:27.454 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:52:27.454 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:52:27.455 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:52:27.455 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.455 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.455 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.36480.0:64363 12.0 12:04022993:::benchmark_data_smithi016_181952_object64362:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e214)", 2023-09-23T19:52:27.455 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:47:19.806331+0000", 2023-09-23T19:52:27.455 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 307.61848255400002, 2023-09-23T19:52:27.455 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.299232946, 2023-09-23T19:52:27.455 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:52:27.455 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:52:27.455 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:52:27.455 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.36480", 2023-09-23T19:52:27.456 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/1035295215", 2023-09-23T19:52:27.456 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 64363 2023-09-23T19:52:27.456 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.456 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:52:27.456 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.456 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:52:27.456 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:19.806331+0000", 2023-09-23T19:52:27.456 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:52:27.456 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.456 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.456 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:52:27.456 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:19.806331+0000", 2023-09-23T19:52:27.457 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999976 2023-09-23T19:52:27.457 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.457 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.457 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:52:27.457 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:19.806328+0000", 2023-09-23T19:52:27.457 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.6004000000000001e-05 2023-09-23T19:52:27.457 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.457 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.457 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:52:27.457 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:19.806384+0000", 2023-09-23T19:52:27.457 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.5069e-05 2023-09-23T19:52:27.458 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.458 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.458 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:52:27.458 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:19.806410+0000", 2023-09-23T19:52:27.458 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.0396e-05 2023-09-23T19:52:27.458 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.458 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.458 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:52:27.458 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:19.806420+0000", 2023-09-23T19:52:27.458 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.6582999999999997e-05 2023-09-23T19:52:27.458 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.458 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.459 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:52:27.459 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:19.806476+0000", 2023-09-23T19:52:27.459 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000116307 2023-09-23T19:52:27.459 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.459 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.459 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:52:27.459 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:19.806593+0000", 2023-09-23T19:52:27.459 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000133101 2023-09-23T19:52:27.459 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.459 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.459 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 2,6", 2023-09-23T19:52:27.459 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:19.806726+0000", 2023-09-23T19:52:27.460 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0024214979999999998 2023-09-23T19:52:27.460 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.460 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.460 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:52:27.460 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:19.809147+0000", 2023-09-23T19:52:27.460 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0010249440000000001 2023-09-23T19:52:27.460 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.460 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.460 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:27.460 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:19.810172+0000", 2023-09-23T19:52:27.460 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.295257767 2023-09-23T19:52:27.461 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.461 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.461 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:27.461 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:20.105430+0000", 2023-09-23T19:52:27.461 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.6163999999999998e-05 2023-09-23T19:52:27.461 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.461 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.461 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:52:27.461 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:20.105506+0000", 2023-09-23T19:52:27.461 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.7658000000000003e-05 2023-09-23T19:52:27.461 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.462 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.462 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:52:27.462 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:20.105564+0000", 2023-09-23T19:52:27.462 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.299232946 2023-09-23T19:52:27.462 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:52:27.462 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:52:27.462 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:52:27.462 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.462 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.462 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.36480.0:71262 12.2 12:47ffe377:::benchmark_data_smithi016_181952_object71261:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e214)", 2023-09-23T19:52:27.462 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:47:26.813076+0000", 2023-09-23T19:52:27.463 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 300.61173724700001, 2023-09-23T19:52:27.463 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.21948084400000001, 2023-09-23T19:52:27.463 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:52:27.463 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:52:27.463 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:52:27.463 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.36480", 2023-09-23T19:52:27.463 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/1035295215", 2023-09-23T19:52:27.463 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 71262 2023-09-23T19:52:27.463 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.463 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:52:27.463 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.464 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:52:27.464 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:26.813076+0000", 2023-09-23T19:52:27.464 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:52:27.464 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.464 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.464 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:52:27.464 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:26.813076+0000", 2023-09-23T19:52:27.464 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T19:52:27.464 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.464 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.464 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:52:27.465 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:26.813075+0000", 2023-09-23T19:52:27.465 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.5007999999999998e-05 2023-09-23T19:52:27.465 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.465 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.465 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:52:27.465 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:26.813140+0000", 2023-09-23T19:52:27.465 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.6874000000000001e-05 2023-09-23T19:52:27.465 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.465 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.465 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:52:27.465 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:26.813157+0000", 2023-09-23T19:52:27.466 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.1100000000000003e-06 2023-09-23T19:52:27.466 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.466 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.466 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:52:27.466 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:26.813165+0000", 2023-09-23T19:52:27.466 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.2465000000000001e-05 2023-09-23T19:52:27.466 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.466 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.466 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:52:27.466 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:26.813208+0000", 2023-09-23T19:52:27.466 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00015142199999999999 2023-09-23T19:52:27.467 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.467 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.467 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:52:27.467 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:26.813359+0000", 2023-09-23T19:52:27.467 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00016001000000000001 2023-09-23T19:52:27.467 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.467 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.467 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3,7", 2023-09-23T19:52:27.467 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:26.813519+0000", 2023-09-23T19:52:27.467 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.002497052 2023-09-23T19:52:27.467 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.467 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.468 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:52:27.468 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:26.816016+0000", 2023-09-23T19:52:27.468 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00020807200000000001 2023-09-23T19:52:27.468 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.468 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.468 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:27.468 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:26.816224+0000", 2023-09-23T19:52:27.468 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.21608002000000001 2023-09-23T19:52:27.468 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.468 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.468 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:27.468 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:27.032304+0000", 2023-09-23T19:52:27.468 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00015683599999999999 2023-09-23T19:52:27.469 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.469 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.469 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:52:27.469 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:27.032461+0000", 2023-09-23T19:52:27.469 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.6199999999999994e-05 2023-09-23T19:52:27.469 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.469 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.469 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:52:27.469 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:27.032557+0000", 2023-09-23T19:52:27.469 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.21948084400000001 2023-09-23T19:52:27.469 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:52:27.470 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:52:27.470 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:52:27.470 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.470 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.470 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.36480.0:71285 12.2 12:4e44d41c:::benchmark_data_smithi016_181952_object71284:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e214)", 2023-09-23T19:52:27.470 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:47:26.839449+0000", 2023-09-23T19:52:27.470 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 300.58536502499999, 2023-09-23T19:52:27.470 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.20433733300000001, 2023-09-23T19:52:27.470 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:52:27.470 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:52:27.470 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:52:27.471 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.36480", 2023-09-23T19:52:27.471 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/1035295215", 2023-09-23T19:52:27.471 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 71285 2023-09-23T19:52:27.471 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.471 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:52:27.471 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.471 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:52:27.471 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:26.839449+0000", 2023-09-23T19:52:27.471 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:52:27.471 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.471 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.471 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:52:27.472 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:26.839449+0000", 2023-09-23T19:52:27.472 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.999999 2023-09-23T19:52:27.472 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.472 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.472 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:52:27.472 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:26.839447+0000", 2023-09-23T19:52:27.472 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.1930000000000001e-05 2023-09-23T19:52:27.472 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.472 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.472 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:52:27.472 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:26.839479+0000", 2023-09-23T19:52:27.473 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.0870000000000007e-06 2023-09-23T19:52:27.473 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.473 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.473 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:52:27.473 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:26.839487+0000", 2023-09-23T19:52:27.473 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.276e-06 2023-09-23T19:52:27.473 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.473 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.473 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:52:27.473 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:26.839493+0000", 2023-09-23T19:52:27.473 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.7343999999999996e-05 2023-09-23T19:52:27.474 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.474 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.474 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:52:27.474 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:26.839580+0000", 2023-09-23T19:52:27.474 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000153642 2023-09-23T19:52:27.474 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.474 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.474 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:52:27.474 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:26.839734+0000", 2023-09-23T19:52:27.474 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000166327 2023-09-23T19:52:27.474 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.475 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.475 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3,7", 2023-09-23T19:52:27.475 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:26.839900+0000", 2023-09-23T19:52:27.475 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00350228 2023-09-23T19:52:27.475 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.475 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.475 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:27.475 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:26.843402+0000", 2023-09-23T19:52:27.475 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00026435400000000001 2023-09-23T19:52:27.475 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.475 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.475 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:52:27.475 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:26.843667+0000", 2023-09-23T19:52:27.476 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.20004390999999999 2023-09-23T19:52:27.476 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.476 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.476 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:27.476 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:27.043711+0000", 2023-09-23T19:52:27.476 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.6094999999999998e-05 2023-09-23T19:52:27.476 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.476 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.476 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:52:27.476 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:27.043757+0000", 2023-09-23T19:52:27.476 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.9116999999999999e-05 2023-09-23T19:52:27.476 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.477 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.477 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:52:27.477 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:27.043786+0000", 2023-09-23T19:52:27.477 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.20433733300000001 2023-09-23T19:52:27.477 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:52:27.477 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:52:27.477 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:52:27.477 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.477 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.477 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.36480.0:71300 12.2 12:418201bc:::benchmark_data_smithi016_181952_object71299:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e214)", 2023-09-23T19:52:27.477 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:47:26.856871+0000", 2023-09-23T19:52:27.478 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 300.56794247599998, 2023-09-23T19:52:27.478 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.18777226999999999, 2023-09-23T19:52:27.478 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:52:27.478 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:52:27.478 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:52:27.478 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.36480", 2023-09-23T19:52:27.478 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/1035295215", 2023-09-23T19:52:27.478 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 71300 2023-09-23T19:52:27.478 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.478 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:52:27.478 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.478 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:52:27.479 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:26.856871+0000", 2023-09-23T19:52:27.479 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:52:27.479 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.479 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.479 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:52:27.479 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:26.856871+0000", 2023-09-23T19:52:27.479 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999948 2023-09-23T19:52:27.479 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.479 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.479 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:52:27.479 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:26.856866+0000", 2023-09-23T19:52:27.479 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.8768999999999999e-05 2023-09-23T19:52:27.479 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.480 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.480 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:52:27.480 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:26.856924+0000", 2023-09-23T19:52:27.480 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.1556000000000002e-05 2023-09-23T19:52:27.480 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.480 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.480 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:52:27.480 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:26.856946+0000", 2023-09-23T19:52:27.480 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.023e-05 2023-09-23T19:52:27.480 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.480 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.480 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:52:27.481 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:26.856956+0000", 2023-09-23T19:52:27.481 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.4998000000000003e-05 2023-09-23T19:52:27.481 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.481 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.481 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:52:27.481 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:26.857021+0000", 2023-09-23T19:52:27.481 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00019097499999999999 2023-09-23T19:52:27.481 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.481 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.481 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:52:27.481 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:26.857212+0000", 2023-09-23T19:52:27.481 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000205835 2023-09-23T19:52:27.482 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.482 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.482 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3,7", 2023-09-23T19:52:27.482 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:26.857418+0000", 2023-09-23T19:52:27.482 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0034418600000000001 2023-09-23T19:52:27.482 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.482 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.482 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:52:27.482 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:26.860860+0000", 2023-09-23T19:52:27.482 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00029146199999999998 2023-09-23T19:52:27.482 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.482 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.483 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:27.483 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:26.861151+0000", 2023-09-23T19:52:27.483 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.18337430199999999 2023-09-23T19:52:27.483 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.483 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.483 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:27.483 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:27.044526+0000", 2023-09-23T19:52:27.483 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.9436000000000004e-05 2023-09-23T19:52:27.483 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.483 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.483 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:52:27.483 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:27.044595+0000", 2023-09-23T19:52:27.484 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.8247000000000001e-05 2023-09-23T19:52:27.484 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.484 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.484 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:52:27.484 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:27.044643+0000", 2023-09-23T19:52:27.484 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.18777226999999999 2023-09-23T19:52:27.484 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:52:27.484 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:52:27.484 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:52:27.484 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.484 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.484 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.36480.0:106860 12.0 12:08301b89:::benchmark_data_smithi016_181952_object4179:head [delete] snapc 0=[] ondisk+write+known_if_redirected e217)", 2023-09-23T19:52:27.485 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:48:09.433817+0000", 2023-09-23T19:52:27.485 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 257.99099641800001, 2023-09-23T19:52:27.485 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.174948716, 2023-09-23T19:52:27.485 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:52:27.485 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:52:27.485 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:52:27.485 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.36480", 2023-09-23T19:52:27.485 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/1035295215", 2023-09-23T19:52:27.485 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 106860 2023-09-23T19:52:27.485 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.485 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:52:27.485 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.486 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:52:27.486 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.433817+0000", 2023-09-23T19:52:27.486 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:52:27.486 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.486 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.486 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:52:27.486 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.433817+0000", 2023-09-23T19:52:27.486 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T19:52:27.486 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.486 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.486 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:52:27.486 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.433815+0000", 2023-09-23T19:52:27.487 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.7499999999999998e-06 2023-09-23T19:52:27.487 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.487 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.487 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:52:27.487 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.433825+0000", 2023-09-23T19:52:27.487 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.0953000000000001e-05 2023-09-23T19:52:27.487 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.487 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.487 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:52:27.487 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.433846+0000", 2023-09-23T19:52:27.487 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.2585999999999999e-05 2023-09-23T19:52:27.487 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.488 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.488 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:52:27.488 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.433859+0000", 2023-09-23T19:52:27.488 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.7411999999999999e-05 2023-09-23T19:52:27.488 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.488 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.488 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:52:27.488 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.433916+0000", 2023-09-23T19:52:27.488 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00012736299999999999 2023-09-23T19:52:27.488 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.488 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.488 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:52:27.489 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.434043+0000", 2023-09-23T19:52:27.489 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00013470299999999999 2023-09-23T19:52:27.489 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.489 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.489 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 2,6", 2023-09-23T19:52:27.489 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.434178+0000", 2023-09-23T19:52:27.489 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0080043479999999997 2023-09-23T19:52:27.489 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.489 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.489 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:52:27.489 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.442182+0000", 2023-09-23T19:52:27.489 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0041279810000000002 2023-09-23T19:52:27.490 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.490 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.490 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:27.490 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.446310+0000", 2023-09-23T19:52:27.490 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.162348984 2023-09-23T19:52:27.490 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.490 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.490 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:27.490 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.608659+0000", 2023-09-23T19:52:27.490 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.2477000000000005e-05 2023-09-23T19:52:27.490 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.491 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.491 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:52:27.491 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.608722+0000", 2023-09-23T19:52:27.491 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.4002000000000003e-05 2023-09-23T19:52:27.491 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.491 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.491 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:52:27.491 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.608766+0000", 2023-09-23T19:52:27.491 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.174948716 2023-09-23T19:52:27.491 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:52:27.491 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:52:27.491 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:52:27.491 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.492 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.492 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.36480.0:106863 12.0 12:01fdbc4e:::benchmark_data_smithi016_181952_object4182:head [delete] snapc 0=[] ondisk+write+known_if_redirected e217)", 2023-09-23T19:52:27.492 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:48:09.436692+0000", 2023-09-23T19:52:27.492 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 257.988121041, 2023-09-23T19:52:27.492 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.19540948399999999, 2023-09-23T19:52:27.492 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:52:27.492 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:52:27.492 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:52:27.492 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.36480", 2023-09-23T19:52:27.492 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/1035295215", 2023-09-23T19:52:27.492 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 106863 2023-09-23T19:52:27.492 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.493 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:52:27.493 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.493 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:52:27.493 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.436692+0000", 2023-09-23T19:52:27.493 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:52:27.493 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.493 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.493 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:52:27.493 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.436692+0000", 2023-09-23T19:52:27.493 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T19:52:27.493 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.493 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.494 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:52:27.494 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.436691+0000", 2023-09-23T19:52:27.494 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.3569999999999998e-06 2023-09-23T19:52:27.494 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.494 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.494 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:52:27.494 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.436699+0000", 2023-09-23T19:52:27.494 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.3869000000000001e-05 2023-09-23T19:52:27.494 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.494 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.494 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:52:27.494 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.436723+0000", 2023-09-23T19:52:27.495 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.0686000000000001e-05 2023-09-23T19:52:27.495 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.495 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.495 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:52:27.495 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.436734+0000", 2023-09-23T19:52:27.495 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.001397957 2023-09-23T19:52:27.495 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.495 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.495 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:52:27.495 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.438131+0000", 2023-09-23T19:52:27.495 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000109465 2023-09-23T19:52:27.495 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.496 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.496 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:52:27.496 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.438241+0000", 2023-09-23T19:52:27.496 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000113501 2023-09-23T19:52:27.496 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.496 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.496 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 2,6", 2023-09-23T19:52:27.496 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.438354+0000", 2023-09-23T19:52:27.496 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0078902280000000009 2023-09-23T19:52:27.496 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.496 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.496 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:52:27.497 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.446245+0000", 2023-09-23T19:52:27.497 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.002448034 2023-09-23T19:52:27.497 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.497 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.497 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:27.497 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.448693+0000", 2023-09-23T19:52:27.497 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.18331035900000001 2023-09-23T19:52:27.497 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.497 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.497 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:27.497 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.632003+0000", 2023-09-23T19:52:27.497 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.2883999999999997e-05 2023-09-23T19:52:27.498 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.498 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.498 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:52:27.498 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.632056+0000", 2023-09-23T19:52:27.498 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.6037e-05 2023-09-23T19:52:27.498 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.498 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.498 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:52:27.498 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.632102+0000", 2023-09-23T19:52:27.498 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.19540948399999999 2023-09-23T19:52:27.498 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:52:27.499 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:52:27.499 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:52:27.499 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.499 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.499 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.36480.0:106868 12.0 12:05ea628c:::benchmark_data_smithi016_181952_object4187:head [delete] snapc 0=[] ondisk+write+known_if_redirected e217)", 2023-09-23T19:52:27.499 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:48:09.442356+0000", 2023-09-23T19:52:27.499 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 257.98245750199999, 2023-09-23T19:52:27.499 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.189904034, 2023-09-23T19:52:27.499 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:52:27.499 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:52:27.499 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:52:27.499 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.36480", 2023-09-23T19:52:27.500 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/1035295215", 2023-09-23T19:52:27.500 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 106868 2023-09-23T19:52:27.500 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.500 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:52:27.500 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.500 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:52:27.500 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.442356+0000", 2023-09-23T19:52:27.500 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:52:27.500 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.500 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.500 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:52:27.500 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.442356+0000", 2023-09-23T19:52:27.501 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T19:52:27.501 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.501 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.501 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:52:27.501 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.442354+0000", 2023-09-23T19:52:27.501 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.5189999999999998e-06 2023-09-23T19:52:27.501 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.501 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.501 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:52:27.501 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.442364+0000", 2023-09-23T19:52:27.501 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.3963e-05 2023-09-23T19:52:27.501 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.501 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.502 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:52:27.502 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.442378+0000", 2023-09-23T19:52:27.502 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.5300000000000002e-06 2023-09-23T19:52:27.502 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.502 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.502 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:52:27.502 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.442384+0000", 2023-09-23T19:52:27.502 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.9390000000000001e-05 2023-09-23T19:52:27.502 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.502 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.502 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:52:27.502 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.442424+0000", 2023-09-23T19:52:27.503 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000137794 2023-09-23T19:52:27.503 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.503 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.503 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:52:27.503 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.442562+0000", 2023-09-23T19:52:27.503 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00018243900000000001 2023-09-23T19:52:27.503 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.503 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.503 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 2,6", 2023-09-23T19:52:27.503 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.442744+0000", 2023-09-23T19:52:27.503 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0036423699999999998 2023-09-23T19:52:27.503 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.504 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.504 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:52:27.504 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.446386+0000", 2023-09-23T19:52:27.504 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0024609039999999999 2023-09-23T19:52:27.504 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.504 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.504 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:27.504 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.448847+0000", 2023-09-23T19:52:27.504 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.183318754 2023-09-23T19:52:27.504 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.504 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.504 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:27.505 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.632166+0000", 2023-09-23T19:52:27.505 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.3313999999999998e-05 2023-09-23T19:52:27.505 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.505 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.505 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:52:27.505 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.632219+0000", 2023-09-23T19:52:27.505 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.0722000000000001e-05 2023-09-23T19:52:27.505 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.505 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.505 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:52:27.505 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.632260+0000", 2023-09-23T19:52:27.505 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.189904034 2023-09-23T19:52:27.506 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:52:27.506 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:52:27.506 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:52:27.506 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.506 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.506 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.36480.0:143045 12.0 12:0795d6b0:::benchmark_data_smithi016_181952_object40364:head [delete] snapc 0=[] ondisk+write+known_if_redirected e219)", 2023-09-23T19:52:27.506 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:49:02.051273+0000", 2023-09-23T19:52:27.506 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 205.37354007799999, 2023-09-23T19:52:27.506 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.14356128800000001, 2023-09-23T19:52:27.506 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:52:27.506 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:52:27.507 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:52:27.507 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.36480", 2023-09-23T19:52:27.507 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/1035295215", 2023-09-23T19:52:27.507 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 143045 2023-09-23T19:52:27.507 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.507 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:52:27.507 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.507 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:52:27.507 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:02.051273+0000", 2023-09-23T19:52:27.507 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:52:27.507 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.507 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.508 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:52:27.508 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:02.051273+0000", 2023-09-23T19:52:27.508 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999976 2023-09-23T19:52:27.508 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.508 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.508 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:52:27.508 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:02.051271+0000", 2023-09-23T19:52:27.508 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.1226000000000001e-05 2023-09-23T19:52:27.508 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.508 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.508 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:52:27.508 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:02.051282+0000", 2023-09-23T19:52:27.509 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.9738e-05 2023-09-23T19:52:27.509 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.509 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.509 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:52:27.509 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:02.051302+0000", 2023-09-23T19:52:27.509 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.0210000000000001e-05 2023-09-23T19:52:27.509 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.509 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.509 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:52:27.509 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:02.051312+0000", 2023-09-23T19:52:27.509 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.5131999999999999e-05 2023-09-23T19:52:27.509 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.510 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.510 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:52:27.510 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:02.051377+0000", 2023-09-23T19:52:27.510 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00012707899999999999 2023-09-23T19:52:27.510 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.510 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.510 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:52:27.510 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:02.051504+0000", 2023-09-23T19:52:27.510 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.7738999999999994e-05 2023-09-23T19:52:27.510 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.510 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.511 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 2,6", 2023-09-23T19:52:27.511 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:02.051602+0000", 2023-09-23T19:52:27.511 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0051905739999999999 2023-09-23T19:52:27.511 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.511 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.511 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:52:27.511 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:02.056793+0000", 2023-09-23T19:52:27.511 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000313195 2023-09-23T19:52:27.511 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.511 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.511 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:27.511 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:02.057106+0000", 2023-09-23T19:52:27.512 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.13755663200000001 2023-09-23T19:52:27.512 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.512 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.512 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:27.512 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:02.194663+0000", 2023-09-23T19:52:27.512 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000120642 2023-09-23T19:52:27.512 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.512 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.512 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:52:27.512 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:02.194783+0000", 2023-09-23T19:52:27.512 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.1555999999999999e-05 2023-09-23T19:52:27.513 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.513 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.513 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:52:27.513 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:02.194835+0000", 2023-09-23T19:52:27.513 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.14356128800000001 2023-09-23T19:52:27.513 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:52:27.513 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:52:27.513 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:52:27.513 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.513 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.513 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.36480.0:145019 12.0 12:06dd7ea5:::benchmark_data_smithi016_181952_object42338:head [delete] snapc 0=[] ondisk+write+known_if_redirected e219)", 2023-09-23T19:52:27.514 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:49:05.333650+0000", 2023-09-23T19:52:27.514 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 202.091163526, 2023-09-23T19:52:27.514 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.224586377, 2023-09-23T19:52:27.514 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:52:27.514 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:52:27.514 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:52:27.514 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.36480", 2023-09-23T19:52:27.514 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/1035295215", 2023-09-23T19:52:27.514 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 145019 2023-09-23T19:52:27.514 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.514 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:52:27.515 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.515 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:52:27.515 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:05.333650+0000", 2023-09-23T19:52:27.515 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:52:27.515 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.515 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.515 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:52:27.515 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:05.333650+0000", 2023-09-23T19:52:27.515 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.999999 2023-09-23T19:52:27.515 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.515 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.516 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:52:27.516 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:05.333649+0000", 2023-09-23T19:52:27.516 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.2169999999999997e-06 2023-09-23T19:52:27.516 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.516 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.516 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:52:27.516 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:05.333656+0000", 2023-09-23T19:52:27.516 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.1396999999999999e-05 2023-09-23T19:52:27.516 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.516 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.516 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:52:27.516 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:05.333668+0000", 2023-09-23T19:52:27.517 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.6449999999999999e-06 2023-09-23T19:52:27.517 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.517 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.517 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:52:27.517 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:05.333674+0000", 2023-09-23T19:52:27.517 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0053248119999999999 2023-09-23T19:52:27.517 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.517 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.517 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:52:27.517 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:05.338999+0000", 2023-09-23T19:52:27.517 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00015278500000000001 2023-09-23T19:52:27.517 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.518 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.518 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:52:27.518 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:05.339152+0000", 2023-09-23T19:52:27.518 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00016277500000000001 2023-09-23T19:52:27.518 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.518 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.518 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 2,6", 2023-09-23T19:52:27.518 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:05.339315+0000", 2023-09-23T19:52:27.518 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0045578160000000001 2023-09-23T19:52:27.518 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.519 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.519 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:52:27.519 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:05.343872+0000", 2023-09-23T19:52:27.519 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0030591329999999999 2023-09-23T19:52:27.519 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.519 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.519 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:27.519 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:05.346931+0000", 2023-09-23T19:52:27.519 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.210997132 2023-09-23T19:52:27.519 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.519 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.520 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:27.520 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:05.557929+0000", 2023-09-23T19:52:27.520 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00018207499999999999 2023-09-23T19:52:27.520 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.520 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.520 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:52:27.520 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:05.558111+0000", 2023-09-23T19:52:27.520 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00012569999999999999 2023-09-23T19:52:27.520 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.520 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.520 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:52:27.520 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:05.558236+0000", 2023-09-23T19:52:27.521 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.224586377 2023-09-23T19:52:27.521 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:52:27.521 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:52:27.521 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:52:27.521 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.521 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.521 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.36480.0:145030 12.0 12:0675bd0f:::benchmark_data_smithi016_181952_object42349:head [delete] snapc 0=[] ondisk+write+known_if_redirected e219)", 2023-09-23T19:52:27.521 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:49:05.356739+0000", 2023-09-23T19:52:27.521 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 202.06807502000001, 2023-09-23T19:52:27.521 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.20194063200000001, 2023-09-23T19:52:27.521 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:52:27.522 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:52:27.522 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:52:27.522 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.36480", 2023-09-23T19:52:27.522 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/1035295215", 2023-09-23T19:52:27.522 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 145030 2023-09-23T19:52:27.522 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.522 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:52:27.522 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.522 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:52:27.522 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:05.356739+0000", 2023-09-23T19:52:27.522 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:52:27.523 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.523 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.523 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:52:27.523 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:05.356739+0000", 2023-09-23T19:52:27.523 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999995 2023-09-23T19:52:27.523 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.523 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.523 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:52:27.523 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:05.356738+0000", 2023-09-23T19:52:27.523 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.0029999999999999e-06 2023-09-23T19:52:27.523 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.524 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.524 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:52:27.524 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:05.356741+0000", 2023-09-23T19:52:27.524 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.9970000000000001e-06 2023-09-23T19:52:27.524 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.524 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.524 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:52:27.524 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:05.356748+0000", 2023-09-23T19:52:27.524 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.9790000000000001e-06 2023-09-23T19:52:27.524 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.524 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.525 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:52:27.525 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:05.356753+0000", 2023-09-23T19:52:27.525 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.2879e-05 2023-09-23T19:52:27.525 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.525 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.525 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:52:27.525 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:05.356806+0000", 2023-09-23T19:52:27.525 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000151354 2023-09-23T19:52:27.525 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.525 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.525 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:52:27.525 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:05.356957+0000", 2023-09-23T19:52:27.525 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000166942 2023-09-23T19:52:27.526 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.526 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.526 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 2,6", 2023-09-23T19:52:27.526 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:05.357124+0000", 2023-09-23T19:52:27.526 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0043367350000000004 2023-09-23T19:52:27.526 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.526 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.526 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:52:27.526 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:05.361461+0000", 2023-09-23T19:52:27.526 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.003312798 2023-09-23T19:52:27.526 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.527 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.527 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:27.527 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:05.364774+0000", 2023-09-23T19:52:27.527 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.19363712699999999 2023-09-23T19:52:27.527 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.527 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.527 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:27.527 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:05.558411+0000", 2023-09-23T19:52:27.527 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00014789299999999999 2023-09-23T19:52:27.527 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.527 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.527 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:52:27.528 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:05.558559+0000", 2023-09-23T19:52:27.528 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00012032300000000001 2023-09-23T19:52:27.528 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.528 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.528 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:52:27.528 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:05.558679+0000", 2023-09-23T19:52:27.528 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.20194063200000001 2023-09-23T19:52:27.528 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:52:27.528 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:52:27.528 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:52:27.528 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.529 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.529 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.36480.0:145034 12.0 12:0f724fa8:::benchmark_data_smithi016_181952_object42353:head [delete] snapc 0=[] ondisk+write+known_if_redirected e219)", 2023-09-23T19:52:27.529 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:49:05.365228+0000", 2023-09-23T19:52:27.529 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 202.05958587399999, 2023-09-23T19:52:27.529 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.2170859, 2023-09-23T19:52:27.529 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:52:27.529 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:52:27.529 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:52:27.529 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.36480", 2023-09-23T19:52:27.529 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/1035295215", 2023-09-23T19:52:27.529 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 145034 2023-09-23T19:52:27.529 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.530 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:52:27.530 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.530 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:52:27.530 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:05.365228+0000", 2023-09-23T19:52:27.530 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:52:27.530 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.530 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.530 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:52:27.530 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:05.365228+0000", 2023-09-23T19:52:27.530 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T19:52:27.530 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.531 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.531 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:52:27.531 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:05.365226+0000", 2023-09-23T19:52:27.531 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.9999999999999999e-06 2023-09-23T19:52:27.531 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.531 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.531 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:52:27.531 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:05.365233+0000", 2023-09-23T19:52:27.531 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.0146000000000001e-05 2023-09-23T19:52:27.531 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.531 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.531 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:52:27.531 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:05.365244+0000", 2023-09-23T19:52:27.532 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.7706999999999998e-05 2023-09-23T19:52:27.532 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.532 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.532 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:52:27.532 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:05.365261+0000", 2023-09-23T19:52:27.532 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.2286000000000001e-05 2023-09-23T19:52:27.532 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.532 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.532 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:52:27.532 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:05.365304+0000", 2023-09-23T19:52:27.532 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000120241 2023-09-23T19:52:27.532 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.533 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.533 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:52:27.533 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:05.365424+0000", 2023-09-23T19:52:27.533 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00015740400000000001 2023-09-23T19:52:27.533 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.533 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.533 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 2,6", 2023-09-23T19:52:27.533 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:05.365581+0000", 2023-09-23T19:52:27.533 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0038610060000000002 2023-09-23T19:52:27.533 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.533 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.533 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:52:27.534 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:05.369442+0000", 2023-09-23T19:52:27.534 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00055477200000000003 2023-09-23T19:52:27.534 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.534 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.534 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:27.534 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:05.369997+0000", 2023-09-23T19:52:27.534 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.21221082399999999 2023-09-23T19:52:27.534 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.534 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.534 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:27.534 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:05.582208+0000", 2023-09-23T19:52:27.534 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.1106999999999996e-05 2023-09-23T19:52:27.535 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.535 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.535 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:52:27.535 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:05.582269+0000", 2023-09-23T19:52:27.535 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.4605999999999999e-05 2023-09-23T19:52:27.535 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.535 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.535 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:52:27.535 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:05.582314+0000", 2023-09-23T19:52:27.535 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.2170859 2023-09-23T19:52:27.536 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:52:27.536 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:52:27.536 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:52:27.536 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.536 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.536 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.36480.0:178043 12.0 12:087e92e8:::benchmark_data_smithi016_181952_object75362:head [delete] snapc 0=[] ondisk+write+known_if_redirected e224)", 2023-09-23T19:52:27.536 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:49:58.260360+0000", 2023-09-23T19:52:27.536 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 149.16445366799999, 2023-09-23T19:52:27.536 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.22340005299999999, 2023-09-23T19:52:27.536 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:52:27.536 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:52:27.536 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:52:27.537 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.36480", 2023-09-23T19:52:27.537 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/1035295215", 2023-09-23T19:52:27.537 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 178043 2023-09-23T19:52:27.537 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.537 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:52:27.537 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.537 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:52:27.537 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:58.260360+0000", 2023-09-23T19:52:27.537 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:52:27.537 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.537 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.537 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:52:27.538 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:58.260360+0000", 2023-09-23T19:52:27.538 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T19:52:27.538 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.538 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.538 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:52:27.538 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:58.260358+0000", 2023-09-23T19:52:27.538 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.1654000000000001e-05 2023-09-23T19:52:27.538 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.538 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.538 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:52:27.538 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:58.260370+0000", 2023-09-23T19:52:27.539 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.8156999999999999e-05 2023-09-23T19:52:27.539 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.539 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.539 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:52:27.539 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:58.260388+0000", 2023-09-23T19:52:27.539 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.7910000000000005e-06 2023-09-23T19:52:27.539 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.539 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.539 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:52:27.539 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:58.260396+0000", 2023-09-23T19:52:27.539 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.9910000000000001e-05 2023-09-23T19:52:27.540 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.540 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.540 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:52:27.540 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:58.260456+0000", 2023-09-23T19:52:27.540 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00013244400000000001 2023-09-23T19:52:27.540 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.540 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.540 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:52:27.540 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:58.260588+0000", 2023-09-23T19:52:27.540 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00014522 2023-09-23T19:52:27.540 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.540 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.541 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 2,6", 2023-09-23T19:52:27.541 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:58.260734+0000", 2023-09-23T19:52:27.541 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0092909529999999994 2023-09-23T19:52:27.541 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.541 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.541 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:52:27.541 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:58.270025+0000", 2023-09-23T19:52:27.541 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0070763989999999997 2023-09-23T19:52:27.541 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.541 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.541 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:27.542 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:58.277101+0000", 2023-09-23T19:52:27.542 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.20656934199999999 2023-09-23T19:52:27.542 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.542 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.542 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:27.542 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:58.483670+0000", 2023-09-23T19:52:27.542 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.1418999999999997e-05 2023-09-23T19:52:27.542 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.542 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.542 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:52:27.542 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:58.483722+0000", 2023-09-23T19:52:27.542 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.8235999999999998e-05 2023-09-23T19:52:27.543 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.543 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.543 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:52:27.543 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:58.483760+0000", 2023-09-23T19:52:27.543 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.22340005299999999 2023-09-23T19:52:27.543 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:52:27.543 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:52:27.543 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:52:27.543 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.543 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.543 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.36480.0:178046 12.0 12:0c67f156:::benchmark_data_smithi016_181952_object75365:head [delete] snapc 0=[] ondisk+write+known_if_redirected e224)", 2023-09-23T19:52:27.543 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:49:58.263233+0000", 2023-09-23T19:52:27.544 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 149.1615802, 2023-09-23T19:52:27.544 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.22811294700000001, 2023-09-23T19:52:27.544 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:52:27.544 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:52:27.544 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:52:27.544 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.36480", 2023-09-23T19:52:27.544 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/1035295215", 2023-09-23T19:52:27.544 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 178046 2023-09-23T19:52:27.544 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.544 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:52:27.544 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.544 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:52:27.545 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:58.263233+0000", 2023-09-23T19:52:27.545 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:52:27.545 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.545 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.545 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:52:27.545 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:58.263233+0000", 2023-09-23T19:52:27.545 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T19:52:27.545 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.545 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.545 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:52:27.545 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:58.263231+0000", 2023-09-23T19:52:27.546 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.7759999999999992e-06 2023-09-23T19:52:27.546 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.546 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.546 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:52:27.546 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:58.263241+0000", 2023-09-23T19:52:27.546 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.658e-05 2023-09-23T19:52:27.546 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.546 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.546 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:52:27.546 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:58.263258+0000", 2023-09-23T19:52:27.547 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.6579999999999997e-06 2023-09-23T19:52:27.547 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.547 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.547 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:52:27.547 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:58.263267+0000", 2023-09-23T19:52:27.547 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.002125409 2023-09-23T19:52:27.547 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.547 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.547 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:52:27.547 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:58.265393+0000", 2023-09-23T19:52:27.547 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00015177300000000001 2023-09-23T19:52:27.548 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.548 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.548 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:52:27.548 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:58.265545+0000", 2023-09-23T19:52:27.548 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00011615799999999999 2023-09-23T19:52:27.548 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.548 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.548 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 2,6", 2023-09-23T19:52:27.548 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:58.265661+0000", 2023-09-23T19:52:27.548 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0070840479999999999 2023-09-23T19:52:27.548 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.548 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.549 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:52:27.549 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:58.272745+0000", 2023-09-23T19:52:27.549 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0056782619999999999 2023-09-23T19:52:27.549 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.549 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.549 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:27.549 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:58.278423+0000", 2023-09-23T19:52:27.549 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.212731379 2023-09-23T19:52:27.549 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.549 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.549 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:27.549 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:58.491154+0000", 2023-09-23T19:52:27.549 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000108065 2023-09-23T19:52:27.550 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.550 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.550 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:52:27.550 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:58.491262+0000", 2023-09-23T19:52:27.550 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.3838000000000006e-05 2023-09-23T19:52:27.550 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.550 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.550 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:52:27.550 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:58.491346+0000", 2023-09-23T19:52:27.550 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.22811294700000001 2023-09-23T19:52:27.550 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:52:27.550 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:52:27.551 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:52:27.551 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.551 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.551 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.36480.0:178050 12.0 12:071e7c6c:::benchmark_data_smithi016_181952_object75369:head [delete] snapc 0=[] ondisk+write+known_if_redirected e224)", 2023-09-23T19:52:27.551 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:49:58.272824+0000", 2023-09-23T19:52:27.551 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 149.15198972799999, 2023-09-23T19:52:27.551 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.21871655000000001, 2023-09-23T19:52:27.551 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:52:27.551 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:52:27.551 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:52:27.551 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.36480", 2023-09-23T19:52:27.551 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/1035295215", 2023-09-23T19:52:27.551 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 178050 2023-09-23T19:52:27.552 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.552 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:52:27.552 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.552 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:52:27.552 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:58.272824+0000", 2023-09-23T19:52:27.552 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:52:27.552 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.552 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.552 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:52:27.552 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:58.272824+0000", 2023-09-23T19:52:27.552 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T19:52:27.552 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.552 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.553 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:52:27.553 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:58.272822+0000", 2023-09-23T19:52:27.553 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.7040000000000006e-06 2023-09-23T19:52:27.553 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.553 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.553 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:52:27.553 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:58.272832+0000", 2023-09-23T19:52:27.553 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.7557000000000001e-05 2023-09-23T19:52:27.553 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.553 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.553 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:52:27.553 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:58.272859+0000", 2023-09-23T19:52:27.554 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.8710000000000003e-06 2023-09-23T19:52:27.554 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.554 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.554 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:52:27.554 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:58.272868+0000", 2023-09-23T19:52:27.554 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.9388000000000002e-05 2023-09-23T19:52:27.554 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.554 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.554 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:52:27.554 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:58.272917+0000", 2023-09-23T19:52:27.554 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000176231 2023-09-23T19:52:27.555 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.555 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.555 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:52:27.555 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:58.273094+0000", 2023-09-23T19:52:27.555 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000109535 2023-09-23T19:52:27.555 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.555 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.555 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 2,6", 2023-09-23T19:52:27.555 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:58.273203+0000", 2023-09-23T19:52:27.555 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.003988124 2023-09-23T19:52:27.555 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.555 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.556 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:52:27.556 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:58.277191+0000", 2023-09-23T19:52:27.556 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0014404019999999999 2023-09-23T19:52:27.556 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.556 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.556 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:27.556 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:58.278632+0000", 2023-09-23T19:52:27.556 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.21277889699999999 2023-09-23T19:52:27.556 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.556 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.556 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:27.556 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:58.491411+0000", 2023-09-23T19:52:27.557 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.0610000000000003e-05 2023-09-23T19:52:27.557 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.557 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.557 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:52:27.557 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:58.491481+0000", 2023-09-23T19:52:27.557 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.9228999999999998e-05 2023-09-23T19:52:27.557 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.557 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.557 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:52:27.557 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:49:58.491540+0000", 2023-09-23T19:52:27.557 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.21871655000000001 2023-09-23T19:52:27.557 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:52:27.558 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:52:27.558 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:52:27.558 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.558 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.558 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46640.0:54564 13.d 13:bbf22ffc:::benchmark_data_smithi016_205181_object54563:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e238)", 2023-09-23T19:52:27.558 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:52:25.457472+0000", 2023-09-23T19:52:27.558 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 1.967341899, 2023-09-23T19:52:27.558 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.19337770000000001, 2023-09-23T19:52:27.558 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:52:27.558 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:52:27.558 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:52:27.558 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46640", 2023-09-23T19:52:27.559 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2656794142", 2023-09-23T19:52:27.559 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 54564 2023-09-23T19:52:27.559 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.559 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:52:27.559 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.559 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:52:27.559 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.457472+0000", 2023-09-23T19:52:27.559 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:52:27.559 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.559 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.559 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:52:27.560 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.457472+0000", 2023-09-23T19:52:27.560 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999995 2023-09-23T19:52:27.560 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.560 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.560 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:52:27.560 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.457471+0000", 2023-09-23T19:52:27.560 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.6993000000000001e-05 2023-09-23T19:52:27.560 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.560 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.560 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:52:27.560 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.457498+0000", 2023-09-23T19:52:27.560 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.5569999999999996e-06 2023-09-23T19:52:27.561 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.561 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.561 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:52:27.561 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.457507+0000", 2023-09-23T19:52:27.561 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.8999999999999999e-06 2023-09-23T19:52:27.561 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.561 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.561 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:52:27.561 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.457510+0000", 2023-09-23T19:52:27.561 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.6453000000000003e-05 2023-09-23T19:52:27.561 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.561 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.561 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:52:27.562 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.457567+0000", 2023-09-23T19:52:27.562 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.6142000000000003e-05 2023-09-23T19:52:27.562 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.562 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.562 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:52:27.562 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.457653+0000", 2023-09-23T19:52:27.562 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000228827 2023-09-23T19:52:27.562 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.562 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.562 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,7", 2023-09-23T19:52:27.562 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.457882+0000", 2023-09-23T19:52:27.563 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.001703616 2023-09-23T19:52:27.563 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.563 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.563 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:52:27.563 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.459586+0000", 2023-09-23T19:52:27.563 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0028860719999999999 2023-09-23T19:52:27.563 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.563 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.563 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:27.563 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.462472+0000", 2023-09-23T19:52:27.563 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.18824105199999999 2023-09-23T19:52:27.563 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.563 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.564 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:27.564 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.650713+0000", 2023-09-23T19:52:27.564 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.0278000000000006e-05 2023-09-23T19:52:27.564 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.564 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.564 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:52:27.564 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.650793+0000", 2023-09-23T19:52:27.564 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.6394000000000003e-05 2023-09-23T19:52:27.564 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.564 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.564 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:52:27.564 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.650849+0000", 2023-09-23T19:52:27.564 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.19337770000000001 2023-09-23T19:52:27.565 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:52:27.565 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:52:27.565 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:52:27.565 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.565 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.565 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46640.0:54573 13.d 13:b2ddc232:::benchmark_data_smithi016_205181_object54572:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e238)", 2023-09-23T19:52:27.565 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:52:25.465607+0000", 2023-09-23T19:52:27.565 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 1.9592064200000001, 2023-09-23T19:52:27.565 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.19164104300000001, 2023-09-23T19:52:27.565 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:52:27.565 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:52:27.565 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:52:27.566 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46640", 2023-09-23T19:52:27.566 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2656794142", 2023-09-23T19:52:27.566 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 54573 2023-09-23T19:52:27.566 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.566 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:52:27.566 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.566 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:52:27.566 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.465607+0000", 2023-09-23T19:52:27.566 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:52:27.566 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.567 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.567 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:52:27.567 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.465607+0000", 2023-09-23T19:52:27.567 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.999999 2023-09-23T19:52:27.567 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.567 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.567 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:52:27.567 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.465606+0000", 2023-09-23T19:52:27.567 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.6085999999999998e-05 2023-09-23T19:52:27.567 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.567 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.567 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:52:27.567 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.465662+0000", 2023-09-23T19:52:27.568 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.2879999999999998e-06 2023-09-23T19:52:27.568 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.568 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.568 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:52:27.568 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.465671+0000", 2023-09-23T19:52:27.568 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.7000000000000002e-06 2023-09-23T19:52:27.568 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.568 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.568 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:52:27.568 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.465678+0000", 2023-09-23T19:52:27.568 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00017306900000000001 2023-09-23T19:52:27.568 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.568 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.569 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:52:27.569 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.465851+0000", 2023-09-23T19:52:27.569 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000159183 2023-09-23T19:52:27.569 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.569 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.569 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:52:27.569 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.466010+0000", 2023-09-23T19:52:27.569 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.002332148 2023-09-23T19:52:27.569 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.569 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.569 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,7", 2023-09-23T19:52:27.569 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.468342+0000", 2023-09-23T19:52:27.569 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0048421369999999998 2023-09-23T19:52:27.570 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.570 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.570 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:52:27.570 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.473185+0000", 2023-09-23T19:52:27.570 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0023334929999999999 2023-09-23T19:52:27.570 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.570 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.570 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:27.570 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.475518+0000", 2023-09-23T19:52:27.570 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.18163553499999999 2023-09-23T19:52:27.570 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.570 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.571 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:27.571 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.657154+0000", 2023-09-23T19:52:27.571 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.6647999999999997e-05 2023-09-23T19:52:27.571 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.571 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.571 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:52:27.571 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.657210+0000", 2023-09-23T19:52:27.571 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.7910000000000001e-05 2023-09-23T19:52:27.571 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.571 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.571 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:52:27.571 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.657248+0000", 2023-09-23T19:52:27.572 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.19164104300000001 2023-09-23T19:52:27.572 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:52:27.572 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:52:27.572 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:52:27.572 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.572 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.572 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46640.0:54597 13.d 13:beb437b3:::benchmark_data_smithi016_205181_object54596:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e238)", 2023-09-23T19:52:27.572 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:52:25.496580+0000", 2023-09-23T19:52:27.572 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 1.9282332979999999, 2023-09-23T19:52:27.572 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.16099896299999999, 2023-09-23T19:52:27.572 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:52:27.573 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:52:27.573 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:52:27.573 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46640", 2023-09-23T19:52:27.573 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2656794142", 2023-09-23T19:52:27.573 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 54597 2023-09-23T19:52:27.573 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.573 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:52:27.573 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.573 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:52:27.573 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.496580+0000", 2023-09-23T19:52:27.573 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:52:27.574 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.574 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.574 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:52:27.574 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.496580+0000", 2023-09-23T19:52:27.574 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T19:52:27.574 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.574 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.574 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:52:27.574 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.496578+0000", 2023-09-23T19:52:27.574 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.6666000000000002e-05 2023-09-23T19:52:27.575 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.575 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.575 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:52:27.575 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.496645+0000", 2023-09-23T19:52:27.575 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.5798e-05 2023-09-23T19:52:27.575 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.575 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.575 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:52:27.575 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.496661+0000", 2023-09-23T19:52:27.575 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.3189999999999997e-06 2023-09-23T19:52:27.576 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.576 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.576 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:52:27.576 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.496668+0000", 2023-09-23T19:52:27.576 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.0786999999999998e-05 2023-09-23T19:52:27.576 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.576 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.576 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:52:27.576 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.496739+0000", 2023-09-23T19:52:27.576 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00021146000000000001 2023-09-23T19:52:27.576 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.576 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.577 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:52:27.577 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.496950+0000", 2023-09-23T19:52:27.577 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00025680499999999998 2023-09-23T19:52:27.577 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.577 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.577 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,7", 2023-09-23T19:52:27.577 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.497207+0000", 2023-09-23T19:52:27.577 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0024534209999999999 2023-09-23T19:52:27.577 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.577 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.577 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:52:27.578 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.499661+0000", 2023-09-23T19:52:27.578 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.9648999999999998e-05 2023-09-23T19:52:27.578 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.578 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.578 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:27.578 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.499710+0000", 2023-09-23T19:52:27.578 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.157812219 2023-09-23T19:52:27.578 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.578 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.578 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:27.579 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.657522+0000", 2023-09-23T19:52:27.579 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.2663999999999998e-05 2023-09-23T19:52:27.579 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.579 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.579 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:52:27.579 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.657555+0000", 2023-09-23T19:52:27.579 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.4037e-05 2023-09-23T19:52:27.579 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:52:27.579 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:52:27.579 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:52:27.579 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.657579+0000", 2023-09-23T19:52:27.579 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.16099896299999999 2023-09-23T19:52:27.580 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:52:27.580 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:52:27.580 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:52:27.580 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:52:27.580 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:52:27.580 INFO:teuthology.orchestra.run.smithi092.stdout:} 2023-09-23T19:52:27.580 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:27.581 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 35 16 56811 56795 101.41 75.9375 0.0110137 0.00985324 2023-09-23T19:52:27.602 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:27.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:27.606 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:27.706 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:27.734 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:27.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:27.741 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:27.842 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:27.867 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:27.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:27.874 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:52:27.974 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:28.002 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:28.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:28.006 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:28.107 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:28.133 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:28.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:28.136 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:28.238 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:28.246 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:28 smithi016 bash[75004]: cluster 2023-09-23T19:52:26.456963+0000 mgr.y (mgr.25325) 1354 : cluster 0 pgmap v1357: 89 pgs: 89 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-09-23T19:52:28.247 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:28 smithi016 bash[77995]: cluster 2023-09-23T19:52:26.456963+0000 mgr.y (mgr.25325) 1354 : cluster 0 pgmap v1357: 89 pgs: 89 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-09-23T19:52:28.265 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:28.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:28.267 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:28.368 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:28.395 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:28.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:28.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:28.478 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 36 15 57534 57519 99.8501 45.25 0.0092291 0.0100078 2023-09-23T19:52:28.500 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:28.508 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:28 smithi092 bash[58855]: cluster 2023-09-23T19:52:26.456963+0000 mgr.y (mgr.25325) 1354 : cluster 0 pgmap v1357: 89 pgs: 89 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-09-23T19:52:28.525 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:28.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:28.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:28.628 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T19:52:28.629 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:28.662 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:28.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:28.666 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:28.767 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:28.797 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:28.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:28.800 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:28.901 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:28.943 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:28.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:28.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:29.019 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:52:29.046 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:29.057 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:52:29.074 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:29.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:29.077 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:29.107 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:52:29.145 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:52:29.177 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:29.184 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:52:29.185 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:29 smithi092 bash[58855]: audit 2023-09-23T19:52:28.676078+0000 mon.a (mon.0) 1673 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:52:29.186 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:29 smithi092 bash[58855]: audit 2023-09-23T19:52:28.941906+0000 mon.c (mon.2) 212 : audit 0 from='client.? 172.21.15.16:0/4095450779' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:52:29.207 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:29.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:29.209 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:29.233 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:52:29.282 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:52:29.311 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:29.322 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:52:29.327 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:52:29.327 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:52:29.327 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:52:29.328 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:52:29.328 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:52:29.328 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:52:29.328 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:52:29.328 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:52:29.346 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:29.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:29.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:29.451 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:29.462 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:29 smithi016 bash[75004]: audit 2023-09-23T19:52:28.676078+0000 mon.a (mon.0) 1673 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:52:29.462 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:29 smithi016 bash[75004]: audit 2023-09-23T19:52:28.941906+0000 mon.c (mon.2) 212 : audit 0 from='client.? 172.21.15.16:0/4095450779' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:52:29.462 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:29 smithi016 bash[77995]: audit 2023-09-23T19:52:28.676078+0000 mon.a (mon.0) 1673 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:52:29.462 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:29 smithi016 bash[77995]: audit 2023-09-23T19:52:28.941906+0000 mon.c (mon.2) 212 : audit 0 from='client.? 172.21.15.16:0/4095450779' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:52:29.479 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 37 16 58582 58566 98.9159 65.4375 0.00363976 0.0101037 2023-09-23T19:52:29.481 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:29.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:29.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:52:29.585 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:29.618 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:29.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:29.622 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:52:29.723 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:29.756 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:29.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:29.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:29.864 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:29.893 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:29.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:29.896 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:29.997 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:30.045 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:30.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:30.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:30.148 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:30.176 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:30.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:30.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:30.186 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is unset 2023-09-23T19:52:30.281 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:30.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:30 smithi092 bash[58855]: cluster 2023-09-23T19:52:28.457451+0000 mgr.y (mgr.25325) 1355 : cluster 0 pgmap v1358: 89 pgs: 89 active+clean; 3.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T19:52:30.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:30 smithi092 bash[58855]: audit 2023-09-23T19:52:29.389524+0000 mon.a (mon.0) 1674 : audit 1 from='client.? 172.21.15.16:0/971010369' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:52:30.310 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:30.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:30.314 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:30.415 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:30.442 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:30.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:30.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:30.479 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 38 16 59738 59722 98.214 72.25 0.00644067 0.0101759 2023-09-23T19:52:30.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:30 smithi016 bash[77995]: cluster 2023-09-23T19:52:28.457451+0000 mgr.y (mgr.25325) 1355 : cluster 0 pgmap v1358: 89 pgs: 89 active+clean; 3.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T19:52:30.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:30 smithi016 bash[77995]: audit 2023-09-23T19:52:29.389524+0000 mon.a (mon.0) 1674 : audit 1 from='client.? 172.21.15.16:0/971010369' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:52:30.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:30 smithi016 bash[75004]: cluster 2023-09-23T19:52:28.457451+0000 mgr.y (mgr.25325) 1355 : cluster 0 pgmap v1358: 89 pgs: 89 active+clean; 3.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T19:52:30.538 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:30 smithi016 bash[75004]: audit 2023-09-23T19:52:29.389524+0000 mon.a (mon.0) 1674 : audit 1 from='client.? 172.21.15.16:0/971010369' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:52:30.546 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:30.576 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:30.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:30.579 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:30.680 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:30.706 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:30.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:30.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:30.811 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:30.843 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:30.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:30.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:52:30.949 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:30.979 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:30.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:30.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:31.083 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:31.108 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:31.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:31.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:52:31.212 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:31.232 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:31 smithi016 bash[75004]: cluster 2023-09-23T19:52:30.163086+0000 mon.a (mon.0) 1675 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:52:31.233 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:31 smithi016 bash[75004]: cluster 2023-09-23T19:52:30.163127+0000 mon.a (mon.0) 1676 : cluster 1 Cluster is now healthy 2023-09-23T19:52:31.233 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:31 smithi016 bash[75004]: cluster 2023-09-23T19:52:30.175012+0000 osd.4 (osd.4) 182 : cluster 3 failed to encode map e239 with expected crc 2023-09-23T19:52:31.233 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:31 smithi016 bash[75004]: audit 2023-09-23T19:52:30.176315+0000 mon.a (mon.0) 1677 : audit 1 from='client.? 172.21.15.16:0/971010369' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:52:31.233 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:31 smithi016 bash[75004]: cluster 2023-09-23T19:52:30.176481+0000 mon.a (mon.0) 1678 : cluster 0 osdmap e239: 8 total, 8 up, 8 in 2023-09-23T19:52:31.233 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:31 smithi016 bash[75004]: cluster 2023-09-23T19:52:30.179088+0000 osd.6 (osd.6) 175 : cluster 3 failed to encode map e239 with expected crc 2023-09-23T19:52:31.233 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:31 smithi016 bash[75004]: cluster 2023-09-23T19:52:30.179794+0000 osd.7 (osd.7) 157 : cluster 3 failed to encode map e239 with expected crc 2023-09-23T19:52:31.233 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:31 smithi016 bash[75004]: cluster 2023-09-23T19:52:30.187914+0000 osd.5 (osd.5) 167 : cluster 3 failed to encode map e239 with expected crc 2023-09-23T19:52:31.233 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:31 smithi016 bash[75004]: cluster 2023-09-23T19:52:30.190918+0000 osd.5 (osd.5) 168 : cluster 3 failed to encode map e239 with expected crc 2023-09-23T19:52:31.254 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:31.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:31.256 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:31.357 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:31.365 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:31 smithi092 bash[58855]: cluster 2023-09-23T19:52:30.163086+0000 mon.a (mon.0) 1675 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:52:31.366 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:31 smithi092 bash[58855]: cluster 2023-09-23T19:52:30.163127+0000 mon.a (mon.0) 1676 : cluster 1 Cluster is now healthy 2023-09-23T19:52:31.366 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:31 smithi092 bash[58855]: cluster 2023-09-23T19:52:30.175012+0000 osd.4 (osd.4) 182 : cluster 3 failed to encode map e239 with expected crc 2023-09-23T19:52:31.366 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:31 smithi092 bash[58855]: audit 2023-09-23T19:52:30.176315+0000 mon.a (mon.0) 1677 : audit 1 from='client.? 172.21.15.16:0/971010369' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:52:31.366 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:31 smithi092 bash[58855]: cluster 2023-09-23T19:52:30.176481+0000 mon.a (mon.0) 1678 : cluster 0 osdmap e239: 8 total, 8 up, 8 in 2023-09-23T19:52:31.366 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:31 smithi092 bash[58855]: cluster 2023-09-23T19:52:30.179088+0000 osd.6 (osd.6) 175 : cluster 3 failed to encode map e239 with expected crc 2023-09-23T19:52:31.366 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:31 smithi092 bash[58855]: cluster 2023-09-23T19:52:30.179794+0000 osd.7 (osd.7) 157 : cluster 3 failed to encode map e239 with expected crc 2023-09-23T19:52:31.366 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:31 smithi092 bash[58855]: cluster 2023-09-23T19:52:30.187914+0000 osd.5 (osd.5) 167 : cluster 3 failed to encode map e239 with expected crc 2023-09-23T19:52:31.366 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:31 smithi092 bash[58855]: cluster 2023-09-23T19:52:30.190918+0000 osd.5 (osd.5) 168 : cluster 3 failed to encode map e239 with expected crc 2023-09-23T19:52:31.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:31.391 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:31.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:31.479 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 39 16 60899 60883 97.5562 72.5625 0.0131225 0.0102449 2023-09-23T19:52:31.492 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:31.516 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:31.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:31.522 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:31.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:31 smithi016 bash[77995]: cluster 2023-09-23T19:52:30.163086+0000 mon.a (mon.0) 1675 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:52:31.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:31 smithi016 bash[77995]: cluster 2023-09-23T19:52:30.163127+0000 mon.a (mon.0) 1676 : cluster 1 Cluster is now healthy 2023-09-23T19:52:31.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:31 smithi016 bash[77995]: cluster 2023-09-23T19:52:30.175012+0000 osd.4 (osd.4) 182 : cluster 3 failed to encode map e239 with expected crc 2023-09-23T19:52:31.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:31 smithi016 bash[77995]: audit 2023-09-23T19:52:30.176315+0000 mon.a (mon.0) 1677 : audit 1 from='client.? 172.21.15.16:0/971010369' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:52:31.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:31 smithi016 bash[77995]: cluster 2023-09-23T19:52:30.176481+0000 mon.a (mon.0) 1678 : cluster 0 osdmap e239: 8 total, 8 up, 8 in 2023-09-23T19:52:31.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:31 smithi016 bash[77995]: cluster 2023-09-23T19:52:30.179088+0000 osd.6 (osd.6) 175 : cluster 3 failed to encode map e239 with expected crc 2023-09-23T19:52:31.538 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:31 smithi016 bash[77995]: cluster 2023-09-23T19:52:30.179794+0000 osd.7 (osd.7) 157 : cluster 3 failed to encode map e239 with expected crc 2023-09-23T19:52:31.538 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:31 smithi016 bash[77995]: cluster 2023-09-23T19:52:30.187914+0000 osd.5 (osd.5) 167 : cluster 3 failed to encode map e239 with expected crc 2023-09-23T19:52:31.538 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:31 smithi016 bash[77995]: cluster 2023-09-23T19:52:30.190918+0000 osd.5 (osd.5) 168 : cluster 3 failed to encode map e239 with expected crc 2023-09-23T19:52:31.623 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:31.655 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:31.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:31.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:52:31.758 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:31.786 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:31.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:31.789 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:31.890 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:31.922 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:31.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:31.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:52:32.029 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:32.067 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:32.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:32.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:32.171 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:32.202 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:32.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:32.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:32.308 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:32.318 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:32 smithi016 bash[77995]: cluster 2023-09-23T19:52:30.457902+0000 mgr.y (mgr.25325) 1356 : cluster 0 pgmap v1360: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-09-23T19:52:32.318 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:32 smithi016 bash[75004]: cluster 2023-09-23T19:52:30.457902+0000 mgr.y (mgr.25325) 1356 : cluster 0 pgmap v1360: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-09-23T19:52:32.341 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:32.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:32.343 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:32.444 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:32.453 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:32 smithi092 bash[58855]: cluster 2023-09-23T19:52:30.457902+0000 mgr.y (mgr.25325) 1356 : cluster 0 pgmap v1360: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-09-23T19:52:32.470 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:32.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:32.476 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:32.479 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-09-23T19:52:32.475615+0000 min lat: 0.00144627 max lat: 0.312684 avg lat: 0.0103165 2023-09-23T19:52:32.480 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:52:32.480 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 40 16 62024 62008 96.8749 70.3125 0.0094431 0.0103165 2023-09-23T19:52:32.577 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:32.620 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:32.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:32.623 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:32.724 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:32.749 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:32.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:32.758 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:52:32.859 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:32.889 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:32.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:32.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:32.908 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:52:32.995 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:33.023 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:33.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:33.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:33.129 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:33.155 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:33.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:33.161 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:33.261 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:33.268 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:52:33.286 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:33.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:33.291 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:33.392 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:33.417 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:33.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:33.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:33.480 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 41 16 63139 63123 96.2116 69.6875 0.0128381 0.0103879 2023-09-23T19:52:33.488 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:52:33.521 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:33.547 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:33.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:33.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:33.653 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:33.682 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:33.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:33.688 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:33.776 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T19:52:33.789 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:33.819 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:33.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:33.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:33.929 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:33.970 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:33.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:33.974 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:34.075 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:34.102 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:34.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:34.110 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:34.211 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:34.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:34.245 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:34.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:34.346 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:34.377 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:34.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:34.380 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:34.480 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 42 15 64234 64219 95.5517 68.5 0.0176768 0.0104594 2023-09-23T19:52:34.481 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:34.491 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:34 smithi016 bash[75004]: cluster 2023-09-23T19:52:32.458473+0000 mgr.y (mgr.25325) 1357 : cluster 0 pgmap v1361: 89 pgs: 89 active+clean; 3.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 70 MiB/s wr, 1.13k op/s 2023-09-23T19:52:34.491 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:34 smithi016 bash[77995]: cluster 2023-09-23T19:52:32.458473+0000 mgr.y (mgr.25325) 1357 : cluster 0 pgmap v1361: 89 pgs: 89 active+clean; 3.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 70 MiB/s wr, 1.13k op/s 2023-09-23T19:52:34.520 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:34.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:34.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:52:34.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:34 smithi092 bash[58855]: cluster 2023-09-23T19:52:32.458473+0000 mgr.y (mgr.25325) 1357 : cluster 0 pgmap v1361: 89 pgs: 89 active+clean; 3.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 70 MiB/s wr, 1.13k op/s 2023-09-23T19:52:34.624 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:34.656 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:34.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:34.660 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:34.762 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:34.789 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:34.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:34.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:52:34.892 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:34.918 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:34.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:34.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:35.021 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:35.054 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:35.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:35.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:35.160 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:35.186 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:35.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:35.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:35.293 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:35.323 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:35.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:35.327 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:35.428 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:35.465 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:35.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:35.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:35.480 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 43 16 65345 65329 94.9428 69.375 0.00584197 0.0105262 2023-09-23T19:52:35.569 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:35.594 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:35.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:35.597 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:35.698 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:35.723 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:35.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:35.727 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:35.827 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:35.837 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:52:35 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:52:35] "GET /metrics HTTP/1.1" 200 36299 "" "Prometheus/2.33.4" 2023-09-23T19:52:35.854 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:35.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:35.858 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:35.958 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:35.989 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:35.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:35.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:52:36.090 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:36.126 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:36.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:36.129 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:36.230 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:36.269 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:36.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:36.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:36.372 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:36.399 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:36.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:36.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:36.480 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 44 16 66471 66455 94.3843 70.375 0.0108162 0.0105878 2023-09-23T19:52:36.502 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:36.511 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:36 smithi092 bash[58855]: cluster 2023-09-23T19:52:34.459017+0000 mgr.y (mgr.25325) 1358 : cluster 0 pgmap v1362: 89 pgs: 89 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-09-23T19:52:36.531 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:36.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:36.534 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:36.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:36 smithi016 bash[77995]: cluster 2023-09-23T19:52:34.459017+0000 mgr.y (mgr.25325) 1358 : cluster 0 pgmap v1362: 89 pgs: 89 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-09-23T19:52:36.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:36 smithi016 bash[75004]: cluster 2023-09-23T19:52:34.459017+0000 mgr.y (mgr.25325) 1358 : cluster 0 pgmap v1362: 89 pgs: 89 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-09-23T19:52:36.634 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:36.665 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:36.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:36.669 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:36.770 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:36.805 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:36.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:36.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:36.909 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:36.938 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:36.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:36.941 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:52:37.042 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:37.065 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:37.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:37.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:37.169 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:37.195 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:37.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:37.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:37.302 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:37.337 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:37.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:37.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:37.342 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:52:37.443 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:37.472 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:37.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:37.477 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:37.480 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 45 16 67506 67490 93.7243 64.6875 0.0104372 0.0106631 2023-09-23T19:52:37.577 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:37.610 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:37.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:37.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:37.718 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:37.751 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:37.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:37.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:52:37.858 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:37.893 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:37.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:37.896 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:52:37.997 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:38.030 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:38.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:38.035 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:38.135 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:38.160 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:38.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:38.163 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:38.213 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:38 smithi092 bash[58855]: cluster 2023-09-23T19:52:36.459698+0000 mgr.y (mgr.25325) 1359 : cluster 0 pgmap v1363: 89 pgs: 89 active+clean; 3.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-23T19:52:38.263 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:38.303 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:38.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:38.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:38.364 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:52:38.406 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:38.437 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:38.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:38.440 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:38.481 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 46 16 68539 68523 93.0902 64.5625 0.00854164 0.010736 2023-09-23T19:52:38.527 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:38 smithi016 bash[77995]: cluster 2023-09-23T19:52:36.459698+0000 mgr.y (mgr.25325) 1359 : cluster 0 pgmap v1363: 89 pgs: 89 active+clean; 3.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-23T19:52:38.527 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:38 smithi016 bash[75004]: cluster 2023-09-23T19:52:36.459698+0000 mgr.y (mgr.25325) 1359 : cluster 0 pgmap v1363: 89 pgs: 89 active+clean; 3.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-23T19:52:38.541 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:38.577 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:38.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:38.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:38.682 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:38.705 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-09-23T19:52:38.705 INFO:teuthology.orchestra.run.smithi016.stdout: "success": "" 2023-09-23T19:52:38.705 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-09-23T19:52:38.706 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:38.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:38.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:52:38.814 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:38.839 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:38.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:38.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:38.948 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:38.979 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:38.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:38.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:39.082 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:39.106 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:39.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:39.109 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:52:39.209 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:39.235 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:39.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:39.240 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:39.341 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:39.374 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:39.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:39.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:39.478 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:39.480 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 47 16 69491 69475 92.3753 59.5 0.00629928 0.0108046 2023-09-23T19:52:39.503 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:39.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:39.506 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:39.607 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:39.650 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:39.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:39.653 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:39.754 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:39.777 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:39.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:39.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:39.883 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:39.911 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:39.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:39.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:39.964 INFO:teuthology.orchestra.run.smithi092.stdout:ERROR: (22) Invalid argument 2023-09-23T19:52:39.964 INFO:teuthology.orchestra.run.smithi092.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:52:40.015 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:40.049 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:40.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:40.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:40.157 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:40.182 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:40.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:40.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:40.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:40 smithi016 bash[75004]: cluster 2023-09-23T19:52:38.460207+0000 mgr.y (mgr.25325) 1360 : cluster 0 pgmap v1364: 89 pgs: 89 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-23T19:52:40.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:40 smithi016 bash[77995]: cluster 2023-09-23T19:52:38.460207+0000 mgr.y (mgr.25325) 1360 : cluster 0 pgmap v1364: 89 pgs: 89 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-23T19:52:40.288 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:40.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:40 smithi092 bash[58855]: cluster 2023-09-23T19:52:38.460207+0000 mgr.y (mgr.25325) 1360 : cluster 0 pgmap v1364: 89 pgs: 89 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-23T19:52:40.323 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:40.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:40.326 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:52:40.426 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:40.460 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:40.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:40.463 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:40.480 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 48 16 70473 70457 91.7293 61.375 0.0218177 0.0108956 2023-09-23T19:52:40.564 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:40.611 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:40.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:40.616 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:40.717 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:40.747 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:40.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:40.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:40.851 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:40.876 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:40.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:40.878 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:40.979 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:41.009 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:41.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:41.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:41.116 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:41.149 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:41.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:41.152 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:41.209 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T19:52:41.253 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:41.284 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:41.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:41.287 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:41.388 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:41.420 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:41.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:41.430 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:41.480 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 49 16 71458 71442 91.1135 61.5625 0.0222589 0.0109671 2023-09-23T19:52:41.531 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:41.561 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:41.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:41.564 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:41.665 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:41.695 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:41.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:41.701 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:41.801 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:41.826 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:41.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:41.830 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:41.931 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:41.957 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:41.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:41.960 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:42.060 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:42.094 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:42.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:42.099 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:42.200 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:42.242 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:42.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:42.244 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:42.345 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:42.356 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:42 smithi016 bash[75004]: cluster 2023-09-23T19:52:40.460958+0000 mgr.y (mgr.25325) 1361 : cluster 0 pgmap v1365: 89 pgs: 89 active+clean; 4.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-09-23T19:52:42.356 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:42 smithi016 bash[77995]: cluster 2023-09-23T19:52:40.460958+0000 mgr.y (mgr.25325) 1361 : cluster 0 pgmap v1365: 89 pgs: 89 active+clean; 4.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-09-23T19:52:42.373 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:42.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:42.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:42.480 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:42.481 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 50 16 72481 72465 90.5698 63.9375 0.0117958 0.0110356 2023-09-23T19:52:42.508 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:42.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:42.510 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:52:42.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:42 smithi092 bash[58855]: cluster 2023-09-23T19:52:40.460958+0000 mgr.y (mgr.25325) 1361 : cluster 0 pgmap v1365: 89 pgs: 89 active+clean; 4.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-09-23T19:52:42.611 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:42.637 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:42.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:42.640 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:42.741 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:42.775 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:42.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:42.780 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:42.881 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:42.927 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:42.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:42.931 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:43.032 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:43.060 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:43.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:43.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:52:43.164 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:43.203 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:43.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:43.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:43.308 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:43.334 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:43.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:43.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:43.441 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:43.472 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:43.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:43.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:43.481 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 51 16 73528 73512 90.0769 65.4375 0.0145344 0.0110952 2023-09-23T19:52:43.575 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:43.609 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:43.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:43.614 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:43.715 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:43.743 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:43.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:43.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:43.850 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:43.880 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:43.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:43.883 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:43.984 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:44.008 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:44.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:44.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:44.112 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:44.143 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:44.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:44.147 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:44.248 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:44.277 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:44.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:44.280 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:44.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:44 smithi016 bash[77995]: cluster 2023-09-23T19:52:42.461439+0000 mgr.y (mgr.25325) 1362 : cluster 0 pgmap v1366: 89 pgs: 89 active+clean; 4.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 944 op/s 2023-09-23T19:52:44.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:44 smithi016 bash[77995]: audit 2023-09-23T19:52:43.684785+0000 mon.a (mon.0) 1679 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:52:44.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:44 smithi016 bash[77995]: audit 2023-09-23T19:52:43.686133+0000 mon.a (mon.0) 1680 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:52:44.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:44 smithi016 bash[75004]: cluster 2023-09-23T19:52:42.461439+0000 mgr.y (mgr.25325) 1362 : cluster 0 pgmap v1366: 89 pgs: 89 active+clean; 4.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 944 op/s 2023-09-23T19:52:44.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:44 smithi016 bash[75004]: audit 2023-09-23T19:52:43.684785+0000 mon.a (mon.0) 1679 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:52:44.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:44 smithi016 bash[75004]: audit 2023-09-23T19:52:43.686133+0000 mon.a (mon.0) 1680 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:52:44.381 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:44.418 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:44.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:44.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:44.481 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 52 16 74273 74257 89.24 46.5625 0.00462412 0.0111887 2023-09-23T19:52:44.521 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:44.532 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:44 smithi092 bash[58855]: cluster 2023-09-23T19:52:42.461439+0000 mgr.y (mgr.25325) 1362 : cluster 0 pgmap v1366: 89 pgs: 89 active+clean; 4.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 944 op/s 2023-09-23T19:52:44.532 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:44 smithi092 bash[58855]: audit 2023-09-23T19:52:43.684785+0000 mon.a (mon.0) 1679 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:52:44.532 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:44 smithi092 bash[58855]: audit 2023-09-23T19:52:43.686133+0000 mon.a (mon.0) 1680 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:52:44.551 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:44.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:44.554 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:44.655 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:44.686 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:44.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:44.689 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:44.790 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:44.822 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:44.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:44.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:44.927 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:44.955 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:44.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:44.959 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:52:45.059 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:45.087 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:45.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:45.091 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:45.192 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:45.227 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:45.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:45.231 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:45.332 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:45.358 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:45.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:45.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:52:45.461 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:45.481 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 53 16 75099 75083 88.5302 51.625 0.018007 0.0112898 2023-09-23T19:52:45.495 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:45.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:45.499 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:45.600 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:45.640 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:45.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:45.645 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:52:45.745 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:45.779 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:45.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:45.782 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:45.851 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:52:45.883 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:45.902 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:52:45.918 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:45.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:45.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:45.942 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:52:45.986 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:52:46.021 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:52:46.024 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:46.037 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:52:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:52:45] "GET /metrics HTTP/1.1" 200 36301 "" "Prometheus/2.33.4" 2023-09-23T19:52:46.044 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:52:46.052 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:46.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:46.056 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:46.061 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:52:46.103 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:52:46.144 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:52:46.148 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:52:46.148 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:52:46.148 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:52:46.148 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:52:46.149 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:52:46.149 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:52:46.149 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:52:46.149 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:52:46.157 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:46.197 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:46.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:46.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:46.251 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is set 2023-09-23T19:52:46.300 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:46.332 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:46.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:46.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:46.438 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:46.446 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:46 smithi016 bash[77995]: cluster 2023-09-23T19:52:44.462082+0000 mgr.y (mgr.25325) 1363 : cluster 0 pgmap v1367: 89 pgs: 89 active+clean; 4.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 967 op/s 2023-09-23T19:52:46.446 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:46 smithi016 bash[77995]: audit 2023-09-23T19:52:45.784820+0000 mon.c (mon.2) 213 : audit 0 from='client.? 172.21.15.16:0/1568723265' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:52:46.446 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:46 smithi016 bash[77995]: audit 2023-09-23T19:52:46.179933+0000 mon.c (mon.2) 214 : audit 1 from='client.? 172.21.15.16:0/1203138564' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:52:46.446 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:46 smithi016 bash[77995]: audit 2023-09-23T19:52:46.180639+0000 mon.a (mon.0) 1681 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:52:46.447 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:46 smithi016 bash[75004]: cluster 2023-09-23T19:52:44.462082+0000 mgr.y (mgr.25325) 1363 : cluster 0 pgmap v1367: 89 pgs: 89 active+clean; 4.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 967 op/s 2023-09-23T19:52:46.447 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:46 smithi016 bash[75004]: audit 2023-09-23T19:52:45.784820+0000 mon.c (mon.2) 213 : audit 0 from='client.? 172.21.15.16:0/1568723265' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:52:46.447 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:46 smithi016 bash[75004]: audit 2023-09-23T19:52:46.179933+0000 mon.c (mon.2) 214 : audit 1 from='client.? 172.21.15.16:0/1203138564' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:52:46.447 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:46 smithi016 bash[75004]: audit 2023-09-23T19:52:46.180639+0000 mon.a (mon.0) 1681 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:52:46.463 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:46.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:46.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:46.481 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 54 16 76010 75994 87.9451 56.9375 0.00630149 0.0113638 2023-09-23T19:52:46.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:46 smithi092 bash[58855]: cluster 2023-09-23T19:52:44.462082+0000 mgr.y (mgr.25325) 1363 : cluster 0 pgmap v1367: 89 pgs: 89 active+clean; 4.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 967 op/s 2023-09-23T19:52:46.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:46 smithi092 bash[58855]: audit 2023-09-23T19:52:45.784820+0000 mon.c (mon.2) 213 : audit 0 from='client.? 172.21.15.16:0/1568723265' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:52:46.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:46 smithi092 bash[58855]: audit 2023-09-23T19:52:46.179933+0000 mon.c (mon.2) 214 : audit 1 from='client.? 172.21.15.16:0/1203138564' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:52:46.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:46 smithi092 bash[58855]: audit 2023-09-23T19:52:46.180639+0000 mon.a (mon.0) 1681 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:52:46.568 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:46.597 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:46.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:46.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:46.699 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:46.727 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:46.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:46.730 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:46.803 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-09-23T19:52:46.803 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T19:52:46.804 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2023-09-23T19:52:46.804 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T19:52:46.804 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T19:52:46.804 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 config set filestore_inject_stall 3 2023-09-23T19:52:46.831 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:46.855 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:46.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:46.859 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:46.959 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:46.985 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:46.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:46.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:47.089 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:47.138 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:47.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:47.142 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:47.241 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:47 smithi016 bash[77995]: cluster 2023-09-23T19:52:46.230093+0000 mon.a (mon.0) 1682 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:52:47.241 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:47 smithi016 bash[77995]: audit 2023-09-23T19:52:46.241446+0000 mon.a (mon.0) 1683 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:52:47.241 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:47 smithi016 bash[77995]: cluster 2023-09-23T19:52:46.241582+0000 mon.a (mon.0) 1684 : cluster 0 osdmap e240: 8 total, 8 up, 8 in 2023-09-23T19:52:47.242 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:47 smithi016 bash[77995]: cluster 2023-09-23T19:52:46.244794+0000 osd.4 (osd.4) 183 : cluster 3 failed to encode map e240 with expected crc 2023-09-23T19:52:47.242 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:47 smithi016 bash[77995]: cluster 2023-09-23T19:52:46.250410+0000 osd.6 (osd.6) 176 : cluster 3 failed to encode map e240 with expected crc 2023-09-23T19:52:47.242 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:47 smithi016 bash[77995]: cluster 2023-09-23T19:52:46.252404+0000 osd.5 (osd.5) 169 : cluster 3 failed to encode map e240 with expected crc 2023-09-23T19:52:47.242 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:47 smithi016 bash[77995]: cluster 2023-09-23T19:52:46.254575+0000 osd.7 (osd.7) 158 : cluster 3 failed to encode map e240 with expected crc 2023-09-23T19:52:47.242 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:47 smithi016 bash[77995]: cluster 2023-09-23T19:52:46.255822+0000 osd.7 (osd.7) 159 : cluster 3 failed to encode map e240 with expected crc 2023-09-23T19:52:47.243 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:47 smithi016 bash[75004]: cluster 2023-09-23T19:52:46.230093+0000 mon.a (mon.0) 1682 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:52:47.243 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:47 smithi016 bash[75004]: audit 2023-09-23T19:52:46.241446+0000 mon.a (mon.0) 1683 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:52:47.243 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:47 smithi016 bash[75004]: cluster 2023-09-23T19:52:46.241582+0000 mon.a (mon.0) 1684 : cluster 0 osdmap e240: 8 total, 8 up, 8 in 2023-09-23T19:52:47.243 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:47 smithi016 bash[75004]: cluster 2023-09-23T19:52:46.244794+0000 osd.4 (osd.4) 183 : cluster 3 failed to encode map e240 with expected crc 2023-09-23T19:52:47.243 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:47 smithi016 bash[75004]: cluster 2023-09-23T19:52:46.250410+0000 osd.6 (osd.6) 176 : cluster 3 failed to encode map e240 with expected crc 2023-09-23T19:52:47.243 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:47 smithi016 bash[75004]: cluster 2023-09-23T19:52:46.252404+0000 osd.5 (osd.5) 169 : cluster 3 failed to encode map e240 with expected crc 2023-09-23T19:52:47.244 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:47 smithi016 bash[75004]: cluster 2023-09-23T19:52:46.254575+0000 osd.7 (osd.7) 158 : cluster 3 failed to encode map e240 with expected crc 2023-09-23T19:52:47.244 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:47 smithi016 bash[75004]: cluster 2023-09-23T19:52:46.255822+0000 osd.7 (osd.7) 159 : cluster 3 failed to encode map e240 with expected crc 2023-09-23T19:52:47.244 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:47.270 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:47.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:47.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:47.376 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:47.403 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:47.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:47.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:47.481 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 55 16 77005 76989 87.4766 62.1875 0.00999623 0.0114254 2023-09-23T19:52:47.509 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:47.533 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:47.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:47.536 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:47.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:47 smithi092 bash[58855]: cluster 2023-09-23T19:52:46.230093+0000 mon.a (mon.0) 1682 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:52:47.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:47 smithi092 bash[58855]: audit 2023-09-23T19:52:46.241446+0000 mon.a (mon.0) 1683 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:52:47.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:47 smithi092 bash[58855]: cluster 2023-09-23T19:52:46.241582+0000 mon.a (mon.0) 1684 : cluster 0 osdmap e240: 8 total, 8 up, 8 in 2023-09-23T19:52:47.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:47 smithi092 bash[58855]: cluster 2023-09-23T19:52:46.244794+0000 osd.4 (osd.4) 183 : cluster 3 failed to encode map e240 with expected crc 2023-09-23T19:52:47.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:47 smithi092 bash[58855]: cluster 2023-09-23T19:52:46.250410+0000 osd.6 (osd.6) 176 : cluster 3 failed to encode map e240 with expected crc 2023-09-23T19:52:47.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:47 smithi092 bash[58855]: cluster 2023-09-23T19:52:46.252404+0000 osd.5 (osd.5) 169 : cluster 3 failed to encode map e240 with expected crc 2023-09-23T19:52:47.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:47 smithi092 bash[58855]: cluster 2023-09-23T19:52:46.254575+0000 osd.7 (osd.7) 158 : cluster 3 failed to encode map e240 with expected crc 2023-09-23T19:52:47.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:47 smithi092 bash[58855]: cluster 2023-09-23T19:52:46.255822+0000 osd.7 (osd.7) 159 : cluster 3 failed to encode map e240 with expected crc 2023-09-23T19:52:47.637 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:47.663 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:47.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:47.666 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:47.766 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:47.807 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:47.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:47.811 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:52:47.911 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:47.952 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:47.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:47.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:52:48.056 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:48.083 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:48.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:48.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:48.190 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:48.217 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:48.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:48.221 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:48.322 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:48.331 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:48 smithi016 bash[77995]: cluster 2023-09-23T19:52:46.462575+0000 mgr.y (mgr.25325) 1364 : cluster 0 pgmap v1369: 89 pgs: 89 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 977 op/s 2023-09-23T19:52:48.331 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:48 smithi016 bash[75004]: cluster 2023-09-23T19:52:46.462575+0000 mgr.y (mgr.25325) 1364 : cluster 0 pgmap v1369: 89 pgs: 89 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 977 op/s 2023-09-23T19:52:48.364 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:48.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:48.366 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:52:48.467 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:48.484 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 56 16 78057 78041 87.0886 65.75 0.0175793 0.0114765 2023-09-23T19:52:48.494 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:48.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:48.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:48.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:48 smithi092 bash[58855]: cluster 2023-09-23T19:52:46.462575+0000 mgr.y (mgr.25325) 1364 : cluster 0 pgmap v1369: 89 pgs: 89 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 977 op/s 2023-09-23T19:52:48.604 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:48.643 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:48.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:48.648 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:48.749 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:48.775 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:48.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:48.780 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:48.881 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:48.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:48.915 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:48.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:49.016 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:49.040 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:49.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:49.044 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:49.144 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:49.175 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:49.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:49.176 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:49.277 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:49.314 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:49.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:49.316 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:49.417 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:49.444 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:49.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:49.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:49.481 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 57 16 79061 79045 86.6615 62.75 0.0166401 0.0115333 2023-09-23T19:52:49.548 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:49.576 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:49.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:49.580 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:49.681 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:49.720 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:49.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:49.725 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:49.826 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:49.857 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:49.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:49.862 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:49.962 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:49.994 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:49.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:49.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:50.097 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:50.131 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:50.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:50.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:50.235 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:50.266 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:50.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:50.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:50.371 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:50.380 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:50 smithi016 bash[75004]: cluster 2023-09-23T19:52:48.463046+0000 mgr.y (mgr.25325) 1365 : cluster 0 pgmap v1370: 89 pgs: 89 active+clean; 4.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 966 op/s 2023-09-23T19:52:50.381 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:50 smithi016 bash[77995]: cluster 2023-09-23T19:52:48.463046+0000 mgr.y (mgr.25325) 1365 : cluster 0 pgmap v1370: 89 pgs: 89 active+clean; 4.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 966 op/s 2023-09-23T19:52:50.403 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:50.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:50.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:50.481 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 58 16 80044 80028 86.2265 61.4375 0.0102495 0.0115917 2023-09-23T19:52:50.510 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:50.515 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:52:50.539 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T19:52:50.541 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:50.542 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:50 smithi092 bash[58855]: cluster 2023-09-23T19:52:48.463046+0000 mgr.y (mgr.25325) 1365 : cluster 0 pgmap v1370: 89 pgs: 89 active+clean; 4.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 966 op/s 2023-09-23T19:52:50.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:50.549 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:50.650 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:50.687 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:50.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:50.693 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:50.794 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:50.824 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:50.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:50.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:50.929 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:50.961 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:50.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:50.965 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:51.066 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:51.098 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:51.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:51.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:51.200 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:51.236 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:51.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:51.239 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:52:51.339 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:51.364 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:51.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:51.371 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:51.471 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:51.481 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 59 16 80979 80963 85.7554 58.4375 0.014459 0.0116549 2023-09-23T19:52:51.501 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:51.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:51.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:51.605 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:51.639 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:51.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:51.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:51.743 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:51.772 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:51.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:51.775 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:51.794 INFO:teuthology.orchestra.run.smithi092.stdout:{ 2023-09-23T19:52:51.794 INFO:teuthology.orchestra.run.smithi092.stdout: "ops": [], 2023-09-23T19:52:51.794 INFO:teuthology.orchestra.run.smithi092.stdout: "complaint_time": 30, 2023-09-23T19:52:51.794 INFO:teuthology.orchestra.run.smithi092.stdout: "num_blocked_ops": 0 2023-09-23T19:52:51.794 INFO:teuthology.orchestra.run.smithi092.stdout:} 2023-09-23T19:52:51.876 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:51.905 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:51.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:51.910 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:52.011 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:52.039 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:52.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:52.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:52.143 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:52.179 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:52.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:52.182 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:52.256 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:52 smithi092 bash[58855]: cluster 2023-09-23T19:52:50.464001+0000 mgr.y (mgr.25325) 1366 : cluster 0 pgmap v1371: 89 pgs: 89 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 959 op/s 2023-09-23T19:52:52.282 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:52.312 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:52.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:52.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:52.415 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:52.458 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:52.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:52.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:52.481 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-09-23T19:52:52.477698+0000 min lat: 0.00144627 max lat: 0.312684 avg lat: 0.0117188 2023-09-23T19:52:52.482 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:52:52.482 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 60 16 81903 81887 85.2886 57.75 0.0105125 0.0117188 2023-09-23T19:52:52.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:52 smithi016 bash[75004]: cluster 2023-09-23T19:52:50.464001+0000 mgr.y (mgr.25325) 1366 : cluster 0 pgmap v1371: 89 pgs: 89 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 959 op/s 2023-09-23T19:52:52.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:52 smithi016 bash[77995]: cluster 2023-09-23T19:52:50.464001+0000 mgr.y (mgr.25325) 1366 : cluster 0 pgmap v1371: 89 pgs: 89 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 959 op/s 2023-09-23T19:52:52.564 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:52.593 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:52.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:52.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:52.697 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:52.732 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:52.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:52.736 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:52.838 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:52.869 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:52.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:52.872 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:52.887 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T19:52:52.972 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:53.008 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:53.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:53.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:53.111 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:53.152 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:53.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:53.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:52:53.268 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:53.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:53.302 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:53.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:53.357 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:52:53.403 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:53.453 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:53.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:53.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:52:53.481 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 61 16 82789 82773 84.7981 55.375 0.0101742 0.0117859 2023-09-23T19:52:53.558 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:53.589 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:53.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:53.595 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:53.696 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:53.734 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:53.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:53.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:53.839 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:53.868 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:53.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:53.871 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:53.972 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:54.008 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:54.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:54.014 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:54.115 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:54.141 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:54.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:54.145 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:54.246 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:54.288 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:54.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:54.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:54.392 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:54.403 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:54 smithi016 bash[75004]: cluster 2023-09-23T19:52:52.464540+0000 mgr.y (mgr.25325) 1367 : cluster 0 pgmap v1372: 89 pgs: 89 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 962 op/s 2023-09-23T19:52:54.403 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:54 smithi016 bash[75004]: audit 2023-09-23T19:52:54.123706+0000 mon.a (mon.0) 1685 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:52:54.405 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:54 smithi016 bash[77995]: cluster 2023-09-23T19:52:52.464540+0000 mgr.y (mgr.25325) 1367 : cluster 0 pgmap v1372: 89 pgs: 89 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 962 op/s 2023-09-23T19:52:54.405 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:54 smithi016 bash[77995]: audit 2023-09-23T19:52:54.123706+0000 mon.a (mon.0) 1685 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:52:54.423 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:54.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:54.426 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:54.482 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 62 16 83703 83687 84.3517 57.125 0.0179013 0.011849 2023-09-23T19:52:54.527 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:54.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:54 smithi092 bash[58855]: cluster 2023-09-23T19:52:52.464540+0000 mgr.y (mgr.25325) 1367 : cluster 0 pgmap v1372: 89 pgs: 89 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 962 op/s 2023-09-23T19:52:54.536 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:54 smithi092 bash[58855]: audit 2023-09-23T19:52:54.123706+0000 mon.a (mon.0) 1685 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:52:54.553 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:54.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:54.556 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:54.657 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:54.701 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:54.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:54.704 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:54.805 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:54.837 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:54.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:54.843 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:54.944 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:54.980 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:54.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:54.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:52:55.085 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:55.117 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:55.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:55.119 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:55.220 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:55.263 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:55.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:55.267 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:55.368 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:55.395 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:55.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:55.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:52:55.399 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:52:55.400 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:52:55.485 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 63 15 84613 84598 83.9134 56.9375 0.0211124 0.0119111 2023-09-23T19:52:55.499 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:55.536 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:55.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:55.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:55.642 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:55.670 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:55.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:55.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:55.775 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:55.786 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:52:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:52:55] "GET /metrics HTTP/1.1" 200 36301 "" "Prometheus/2.33.4" 2023-09-23T19:52:55.812 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:55.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:55.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:55.916 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:55.947 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:55.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:55.950 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:52:56.050 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:56.079 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:56.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:56.083 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:56.183 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:56.213 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:56.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:56.216 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:52:56.317 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:56.344 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:56.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:56.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:56.449 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:56.480 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:56.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:56.484 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:56.484 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 64 16 85550 85534 83.5163 58.5 0.0298425 0.0119683 2023-09-23T19:52:56.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:56 smithi016 bash[77995]: cluster 2023-09-23T19:52:54.465166+0000 mgr.y (mgr.25325) 1368 : cluster 0 pgmap v1373: 89 pgs: 89 active+clean; 4.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 960 op/s 2023-09-23T19:52:56.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:56 smithi016 bash[75004]: cluster 2023-09-23T19:52:54.465166+0000 mgr.y (mgr.25325) 1368 : cluster 0 pgmap v1373: 89 pgs: 89 active+clean; 4.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 960 op/s 2023-09-23T19:52:56.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:56 smithi092 bash[58855]: cluster 2023-09-23T19:52:54.465166+0000 mgr.y (mgr.25325) 1368 : cluster 0 pgmap v1373: 89 pgs: 89 active+clean; 4.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 960 op/s 2023-09-23T19:52:56.585 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:56.609 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:56.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:56.613 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:56.714 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:56.753 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:56.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:56.756 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:56.857 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:56.880 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:56.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:56.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:56.986 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:57.016 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:57.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:57.020 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:57.120 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:57.144 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:57.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:57.149 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:57.250 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:57.278 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:57.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:57.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:57.384 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:57.409 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:57.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:57.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:57.431 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:52:57.484 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 65 15 86453 86438 83.1006 56.5 0.0139444 0.0120275 2023-09-23T19:52:57.516 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:57.543 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:57.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:57.544 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:57.645 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:57.685 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:57.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:57.690 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:57.791 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:57.810 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:57.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:57.812 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:57.913 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:57.949 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:57.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:57.954 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:58.055 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:58.090 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:58.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:58.096 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:52:58.197 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:58.223 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:58.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:58.228 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:58.329 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:58.357 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:58.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:58.362 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:58.463 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:58.484 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 66 16 87290 87274 82.6331 52.25 0.0153755 0.0120951 2023-09-23T19:52:58.499 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:58.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:58.502 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:52:58.502 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:58 smithi092 bash[58855]: cluster 2023-09-23T19:52:56.466023+0000 mgr.y (mgr.25325) 1369 : cluster 0 pgmap v1374: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 59 MiB/s wr, 939 op/s 2023-09-23T19:52:58.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:58 smithi016 bash[77995]: cluster 2023-09-23T19:52:56.466023+0000 mgr.y (mgr.25325) 1369 : cluster 0 pgmap v1374: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 59 MiB/s wr, 939 op/s 2023-09-23T19:52:58.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:58 smithi016 bash[75004]: cluster 2023-09-23T19:52:56.466023+0000 mgr.y (mgr.25325) 1369 : cluster 0 pgmap v1374: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 59 MiB/s wr, 939 op/s 2023-09-23T19:52:58.602 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:58.659 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:58.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:58.662 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:58.763 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:58.798 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:58.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:58.805 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:58.887 INFO:teuthology.orchestra.run.smithi092.stdout:{ 2023-09-23T19:52:58.887 INFO:teuthology.orchestra.run.smithi092.stdout: "success": "" 2023-09-23T19:52:58.888 INFO:teuthology.orchestra.run.smithi092.stdout:} 2023-09-23T19:52:58.906 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:58.942 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:58.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:58.950 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:52:59.050 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:59.088 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:59.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:59.091 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:59.191 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:59.224 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:59.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:59.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:52:59.332 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:59.362 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:59.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:59.364 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:59.465 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:59.484 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 67 16 88053 88037 82.1116 47.6875 0.0167556 0.0121721 2023-09-23T19:52:59.499 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:59.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:59.501 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:59.602 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:59.630 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:59.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:59.636 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:59.737 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:59.744 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:59 smithi092 bash[58855]: cluster 2023-09-23T19:52:58.466488+0000 mgr.y (mgr.25325) 1370 : cluster 0 pgmap v1375: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 848 op/s 2023-09-23T19:52:59.744 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:59 smithi092 bash[58855]: audit 2023-09-23T19:52:58.681723+0000 mon.a (mon.0) 1686 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:52:59.744 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:52:59 smithi092 bash[58855]: audit 2023-09-23T19:52:58.683080+0000 mon.a (mon.0) 1687 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:52:59.775 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:59.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:59.779 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:59.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:59 smithi016 bash[75004]: cluster 2023-09-23T19:52:58.466488+0000 mgr.y (mgr.25325) 1370 : cluster 0 pgmap v1375: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 848 op/s 2023-09-23T19:52:59.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:59 smithi016 bash[75004]: audit 2023-09-23T19:52:58.681723+0000 mon.a (mon.0) 1686 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:52:59.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:52:59 smithi016 bash[75004]: audit 2023-09-23T19:52:58.683080+0000 mon.a (mon.0) 1687 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:52:59.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:59 smithi016 bash[77995]: cluster 2023-09-23T19:52:58.466488+0000 mgr.y (mgr.25325) 1370 : cluster 0 pgmap v1375: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 848 op/s 2023-09-23T19:52:59.788 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:59 smithi016 bash[77995]: audit 2023-09-23T19:52:58.681723+0000 mon.a (mon.0) 1686 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:52:59.788 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:52:59 smithi016 bash[77995]: audit 2023-09-23T19:52:58.683080+0000 mon.a (mon.0) 1687 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:52:59.879 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:59.906 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:59.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:52:59.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:00.012 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:00.052 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:00.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:00.056 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:00.157 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:00.216 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:00.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:00.227 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:00.327 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:00.355 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:00.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:00.358 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:00.459 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:00.484 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 68 16 88842 88826 81.6292 49.3125 0.0176213 0.0122432 2023-09-23T19:53:00.491 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:00.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:00.501 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:00.602 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:00.633 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:00.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:00.637 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:00.737 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:00.776 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:00.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:00.778 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:00.879 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:00.915 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:00.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:00.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:01.019 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:01.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:00 smithi016 bash[77995]: audit 2023-09-23T19:52:59.884940+0000 mon.a (mon.0) 1688 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:53:01.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:00 smithi016 bash[77995]: audit 2023-09-23T19:52:59.894537+0000 mon.a (mon.0) 1689 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:53:01.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:00 smithi016 bash[77995]: audit 2023-09-23T19:53:00.035694+0000 mon.a (mon.0) 1690 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:53:01.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:00 smithi016 bash[77995]: audit 2023-09-23T19:53:00.052651+0000 mon.a (mon.0) 1691 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:53:01.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:00 smithi016 bash[77995]: audit 2023-09-23T19:53:00.637592+0000 mon.a (mon.0) 1692 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:53:01.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:00 smithi016 bash[77995]: audit 2023-09-23T19:53:00.642708+0000 mon.a (mon.0) 1693 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:53:01.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:00 smithi016 bash[77995]: audit 2023-09-23T19:53:00.649712+0000 mon.a (mon.0) 1694 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:53:01.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:00 smithi016 bash[75004]: audit 2023-09-23T19:52:59.884940+0000 mon.a (mon.0) 1688 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:53:01.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:00 smithi016 bash[75004]: audit 2023-09-23T19:52:59.894537+0000 mon.a (mon.0) 1689 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:53:01.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:00 smithi016 bash[75004]: audit 2023-09-23T19:53:00.035694+0000 mon.a (mon.0) 1690 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:53:01.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:00 smithi016 bash[75004]: audit 2023-09-23T19:53:00.052651+0000 mon.a (mon.0) 1691 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:53:01.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:00 smithi016 bash[75004]: audit 2023-09-23T19:53:00.637592+0000 mon.a (mon.0) 1692 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:53:01.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:00 smithi016 bash[75004]: audit 2023-09-23T19:53:00.642708+0000 mon.a (mon.0) 1693 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:53:01.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:00 smithi016 bash[75004]: audit 2023-09-23T19:53:00.649712+0000 mon.a (mon.0) 1694 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:53:01.053 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:01.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:01.056 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:01.157 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:01.166 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:00 smithi092 bash[58855]: audit 2023-09-23T19:52:59.884940+0000 mon.a (mon.0) 1688 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:53:01.166 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:00 smithi092 bash[58855]: audit 2023-09-23T19:52:59.894537+0000 mon.a (mon.0) 1689 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:53:01.166 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:00 smithi092 bash[58855]: audit 2023-09-23T19:53:00.035694+0000 mon.a (mon.0) 1690 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:53:01.166 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:00 smithi092 bash[58855]: audit 2023-09-23T19:53:00.052651+0000 mon.a (mon.0) 1691 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:53:01.166 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:00 smithi092 bash[58855]: audit 2023-09-23T19:53:00.637592+0000 mon.a (mon.0) 1692 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:53:01.167 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:00 smithi092 bash[58855]: audit 2023-09-23T19:53:00.642708+0000 mon.a (mon.0) 1693 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:53:01.167 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:00 smithi092 bash[58855]: audit 2023-09-23T19:53:00.649712+0000 mon.a (mon.0) 1694 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:53:01.181 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:01.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:01.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:01.287 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:01.319 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:01.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:01.325 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:01.425 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:01.455 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:01.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:01.458 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:01.484 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 69 16 89723 89707 81.244 55.0625 0.00865534 0.0123018 2023-09-23T19:53:01.559 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:01.584 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:01.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:01.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:01.689 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:01.714 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:01.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:01.717 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:01.817 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:01.850 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:01.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:01.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:01.955 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:01.988 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:01.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:01.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:02.095 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:02.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:02.130 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:02.130 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:02.231 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:02.240 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:02 smithi092 bash[58855]: cluster 2023-09-23T19:53:00.467230+0000 mgr.y (mgr.25325) 1371 : cluster 0 pgmap v1376: 89 pgs: 89 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-09-23T19:53:02.253 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:02.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:02.257 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:02.354 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:02 smithi016 bash[75004]: cluster 2023-09-23T19:53:00.467230+0000 mgr.y (mgr.25325) 1371 : cluster 0 pgmap v1376: 89 pgs: 89 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-09-23T19:53:02.354 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:02 smithi016 bash[77995]: cluster 2023-09-23T19:53:00.467230+0000 mgr.y (mgr.25325) 1371 : cluster 0 pgmap v1376: 89 pgs: 89 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-09-23T19:53:02.357 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:02.378 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:02.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:02.380 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:02.481 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:02.484 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 70 16 90628 90612 80.8914 56.5625 0.0113145 0.0123563 2023-09-23T19:53:02.508 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:02.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:02.512 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:02.613 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:02.629 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:53:02.639 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:02.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:02.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:02.670 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:53:02.711 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:53:02.744 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:02.752 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:53:02.777 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:02.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:02.780 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:02.794 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:53:02.836 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:53:02.876 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:53:02.880 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:02.911 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:02.912 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:53:02.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:02.915 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:02.918 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:53:02.918 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:53:02.918 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:53:02.919 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:53:02.919 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:53:02.919 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:53:02.919 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:53:02.919 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:53:03.015 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:03.045 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:03.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:03.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:03.083 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is set 2023-09-23T19:53:03.151 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:03.179 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:03.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:03.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:03.281 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:03.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:03 smithi092 bash[58855]: audit 2023-09-23T19:53:02.574848+0000 mon.b (mon.1) 56 : audit 0 from='client.? 172.21.15.16:0/1692680267' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:53:03.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:03 smithi092 bash[58855]: audit 2023-09-23T19:53:02.988083+0000 mon.c (mon.2) 215 : audit 1 from='client.? 172.21.15.16:0/2090246544' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:53:03.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:03 smithi092 bash[58855]: audit 2023-09-23T19:53:02.988720+0000 mon.a (mon.0) 1695 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:53:03.307 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:03.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:03.311 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:03.411 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:03.420 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:03 smithi016 bash[75004]: audit 2023-09-23T19:53:02.574848+0000 mon.b (mon.1) 56 : audit 0 from='client.? 172.21.15.16:0/1692680267' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:53:03.420 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:03 smithi016 bash[75004]: audit 2023-09-23T19:53:02.988083+0000 mon.c (mon.2) 215 : audit 1 from='client.? 172.21.15.16:0/2090246544' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:53:03.420 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:03 smithi016 bash[75004]: audit 2023-09-23T19:53:02.988720+0000 mon.a (mon.0) 1695 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:53:03.420 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:03 smithi016 bash[77995]: audit 2023-09-23T19:53:02.574848+0000 mon.b (mon.1) 56 : audit 0 from='client.? 172.21.15.16:0/1692680267' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:53:03.420 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:03 smithi016 bash[77995]: audit 2023-09-23T19:53:02.988083+0000 mon.c (mon.2) 215 : audit 1 from='client.? 172.21.15.16:0/2090246544' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:53:03.420 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:03 smithi016 bash[77995]: audit 2023-09-23T19:53:02.988720+0000 mon.a (mon.0) 1695 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:53:03.440 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:03.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:03.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:03.485 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 71 16 91497 91481 80.517 54.3125 0.00749094 0.0124131 2023-09-23T19:53:03.543 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:03.570 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:03.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:03.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:03.674 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:03.699 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:03.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:03.701 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:03.779 INFO:teuthology.orchestra.run.smithi092.stdout:ERROR: (22) Invalid argument 2023-09-23T19:53:03.779 INFO:teuthology.orchestra.run.smithi092.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:53:03.802 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:03.826 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:03.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:03.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:03.933 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:03.965 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:03.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:03.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:04.070 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:04 smithi092 bash[58855]: cluster 2023-09-23T19:53:02.467697+0000 mgr.y (mgr.25325) 1372 : cluster 0 pgmap v1377: 89 pgs: 89 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 48 MiB/s wr, 773 op/s 2023-09-23T19:53:04.070 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:04 smithi092 bash[58855]: cluster 2023-09-23T19:53:03.055203+0000 mon.a (mon.0) 1696 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:53:04.070 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:04 smithi092 bash[58855]: audit 2023-09-23T19:53:03.067541+0000 mon.a (mon.0) 1697 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:53:04.070 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:04 smithi092 bash[58855]: cluster 2023-09-23T19:53:03.067939+0000 mon.a (mon.0) 1698 : cluster 0 osdmap e241: 8 total, 8 up, 8 in 2023-09-23T19:53:04.070 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:04 smithi092 bash[58855]: cluster 2023-09-23T19:53:03.077628+0000 osd.5 (osd.5) 170 : cluster 3 failed to encode map e241 with expected crc 2023-09-23T19:53:04.070 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:04 smithi092 bash[58855]: cluster 2023-09-23T19:53:03.080466+0000 osd.4 (osd.4) 184 : cluster 3 failed to encode map e241 with expected crc 2023-09-23T19:53:04.070 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:04 smithi092 bash[58855]: cluster 2023-09-23T19:53:03.084211+0000 osd.6 (osd.6) 177 : cluster 3 failed to encode map e241 with expected crc 2023-09-23T19:53:04.070 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:04 smithi092 bash[58855]: cluster 2023-09-23T19:53:03.086191+0000 osd.7 (osd.7) 160 : cluster 3 failed to encode map e241 with expected crc 2023-09-23T19:53:04.072 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:04.096 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:04.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:04.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:04.201 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:04.212 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:04 smithi016 bash[75004]: cluster 2023-09-23T19:53:02.467697+0000 mgr.y (mgr.25325) 1372 : cluster 0 pgmap v1377: 89 pgs: 89 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 48 MiB/s wr, 773 op/s 2023-09-23T19:53:04.212 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:04 smithi016 bash[75004]: cluster 2023-09-23T19:53:03.055203+0000 mon.a (mon.0) 1696 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:53:04.212 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:04 smithi016 bash[75004]: audit 2023-09-23T19:53:03.067541+0000 mon.a (mon.0) 1697 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:53:04.212 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:04 smithi016 bash[75004]: cluster 2023-09-23T19:53:03.067939+0000 mon.a (mon.0) 1698 : cluster 0 osdmap e241: 8 total, 8 up, 8 in 2023-09-23T19:53:04.212 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:04 smithi016 bash[75004]: cluster 2023-09-23T19:53:03.077628+0000 osd.5 (osd.5) 170 : cluster 3 failed to encode map e241 with expected crc 2023-09-23T19:53:04.212 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:04 smithi016 bash[75004]: cluster 2023-09-23T19:53:03.080466+0000 osd.4 (osd.4) 184 : cluster 3 failed to encode map e241 with expected crc 2023-09-23T19:53:04.213 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:04 smithi016 bash[75004]: cluster 2023-09-23T19:53:03.084211+0000 osd.6 (osd.6) 177 : cluster 3 failed to encode map e241 with expected crc 2023-09-23T19:53:04.213 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:04 smithi016 bash[75004]: cluster 2023-09-23T19:53:03.086191+0000 osd.7 (osd.7) 160 : cluster 3 failed to encode map e241 with expected crc 2023-09-23T19:53:04.213 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:04 smithi016 bash[77995]: cluster 2023-09-23T19:53:02.467697+0000 mgr.y (mgr.25325) 1372 : cluster 0 pgmap v1377: 89 pgs: 89 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 48 MiB/s wr, 773 op/s 2023-09-23T19:53:04.213 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:04 smithi016 bash[77995]: cluster 2023-09-23T19:53:03.055203+0000 mon.a (mon.0) 1696 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:53:04.213 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:04 smithi016 bash[77995]: audit 2023-09-23T19:53:03.067541+0000 mon.a (mon.0) 1697 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:53:04.213 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:04 smithi016 bash[77995]: cluster 2023-09-23T19:53:03.067939+0000 mon.a (mon.0) 1698 : cluster 0 osdmap e241: 8 total, 8 up, 8 in 2023-09-23T19:53:04.213 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:04 smithi016 bash[77995]: cluster 2023-09-23T19:53:03.077628+0000 osd.5 (osd.5) 170 : cluster 3 failed to encode map e241 with expected crc 2023-09-23T19:53:04.213 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:04 smithi016 bash[77995]: cluster 2023-09-23T19:53:03.080466+0000 osd.4 (osd.4) 184 : cluster 3 failed to encode map e241 with expected crc 2023-09-23T19:53:04.213 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:04 smithi016 bash[77995]: cluster 2023-09-23T19:53:03.084211+0000 osd.6 (osd.6) 177 : cluster 3 failed to encode map e241 with expected crc 2023-09-23T19:53:04.213 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:04 smithi016 bash[77995]: cluster 2023-09-23T19:53:03.086191+0000 osd.7 (osd.7) 160 : cluster 3 failed to encode map e241 with expected crc 2023-09-23T19:53:04.230 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:04.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:04.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:04.335 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:04.372 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:04.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:04.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:04.480 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:04.485 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 72 16 92422 92406 80.2016 57.8125 0.0104067 0.0124632 2023-09-23T19:53:04.521 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:04.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:04.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:04.628 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:04.652 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:04.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:04.655 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:04.755 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:04.780 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:04.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:04.782 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:04.883 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:04.913 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:04.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:04.916 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:05.017 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:05.041 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:05.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:05.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:05.147 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:05.158 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T19:53:05.178 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:05.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:05.181 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:05.281 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:05.312 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:05.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:05.314 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:05.415 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:05.445 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:05.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:05.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:05.485 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 73 16 93243 93227 79.8058 51.3125 0.0180028 0.0125247 2023-09-23T19:53:05.552 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:05.579 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:05.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:05.582 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:05.682 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:05.714 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:05.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:05.719 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:05.820 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:05.830 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:53:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:53:05] "GET /metrics HTTP/1.1" 200 36305 "" "Prometheus/2.33.4" 2023-09-23T19:53:05.850 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:05.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:05.856 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:05.915 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-09-23T19:53:05.916 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T19:53:05.916 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2023-09-23T19:53:05.916 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T19:53:05.916 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T19:53:05.916 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 config set filestore_inject_stall 3 2023-09-23T19:53:05.956 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:05.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:05.998 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:05.998 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:06.099 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:06.140 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:06.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:06.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:06.244 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:06.254 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:06 smithi016 bash[77995]: cluster 2023-09-23T19:53:04.468186+0000 mgr.y (mgr.25325) 1373 : cluster 0 pgmap v1379: 89 pgs: 89 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 55 MiB/s wr, 884 op/s 2023-09-23T19:53:06.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:06 smithi016 bash[75004]: cluster 2023-09-23T19:53:04.468186+0000 mgr.y (mgr.25325) 1373 : cluster 0 pgmap v1379: 89 pgs: 89 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 55 MiB/s wr, 884 op/s 2023-09-23T19:53:06.275 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:06.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:06.279 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:06.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:06 smithi092 bash[58855]: cluster 2023-09-23T19:53:04.468186+0000 mgr.y (mgr.25325) 1373 : cluster 0 pgmap v1379: 89 pgs: 89 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 55 MiB/s wr, 884 op/s 2023-09-23T19:53:06.380 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:06.407 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:06.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:06.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:06.485 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 74 16 94090 94074 79.4427 52.9375 0.0329643 0.0125813 2023-09-23T19:53:06.513 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:06.539 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:06.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:06.546 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:06.647 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:06.683 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:06.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:06.685 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:06.786 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:06.826 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:06.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:06.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:06.932 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:06.959 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:06.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:06.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:07.063 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:07.097 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:07.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:07.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:07.142 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:53:07.163 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T19:53:07.200 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:07.235 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:07.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:07.238 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:07.339 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:07.373 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:07.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:07.376 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:07.477 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:07.485 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 75 16 94914 94898 79.0699 51.5 0.00989152 0.0126407 2023-09-23T19:53:07.512 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:07.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:07.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:07.618 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:07.647 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:07.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:07.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:07.757 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:07.803 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:07.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:07.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:07.909 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:07.944 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:07.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:07.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:08.047 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:08.082 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:08.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:08.087 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:08.188 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:08.197 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:08 smithi016 bash[75004]: cluster 2023-09-23T19:53:06.469111+0000 mgr.y (mgr.25325) 1374 : cluster 0 pgmap v1380: 89 pgs: 89 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 869 op/s 2023-09-23T19:53:08.197 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:08 smithi016 bash[77995]: cluster 2023-09-23T19:53:06.469111+0000 mgr.y (mgr.25325) 1374 : cluster 0 pgmap v1380: 89 pgs: 89 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 869 op/s 2023-09-23T19:53:08.219 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:08.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:08.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:08.327 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:08.365 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:08.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:08.373 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:08.474 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:08.483 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:08 smithi092 bash[58855]: cluster 2023-09-23T19:53:06.469111+0000 mgr.y (mgr.25325) 1374 : cluster 0 pgmap v1380: 89 pgs: 89 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 869 op/s 2023-09-23T19:53:08.485 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 76 16 95767 95751 78.731 53.3125 0.0137937 0.012696 2023-09-23T19:53:08.506 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:08.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:08.514 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:08.615 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:08.648 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:08.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:08.651 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:08.753 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:08.793 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:08.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:08.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:08.900 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:08.926 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:08.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:08.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:09.030 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:09.058 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:09.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:09.063 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:09.164 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:09.199 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:09.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:09.204 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:09.305 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:09.334 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:09.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:09.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:09.437 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:09.476 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:09.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:09.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:09.485 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 77 16 96626 96610 78.4057 53.6875 0.0288903 0.0127481 2023-09-23T19:53:09.583 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:09.615 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:09.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:09.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:09.720 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:09.747 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:09.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:09.750 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:09.851 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:09.881 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:09.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:09.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:09.988 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:10.017 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:10.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:10.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:10.043 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:53:10.043 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:53:10.125 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:10.134 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:10 smithi092 bash[58855]: cluster 2023-09-23T19:53:08.469646+0000 mgr.y (mgr.25325) 1375 : cluster 0 pgmap v1381: 89 pgs: 89 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 865 op/s 2023-09-23T19:53:10.163 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:10.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:10.164 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:10.264 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:10.292 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:10.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:10.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:10.396 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:10.426 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:10.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:10.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:10.486 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 78 16 97441 97425 78.0535 50.9375 0.0143333 0.012806 2023-09-23T19:53:10.530 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:10.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:10 smithi016 bash[75004]: cluster 2023-09-23T19:53:08.469646+0000 mgr.y (mgr.25325) 1375 : cluster 0 pgmap v1381: 89 pgs: 89 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 865 op/s 2023-09-23T19:53:10.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:10 smithi016 bash[77995]: cluster 2023-09-23T19:53:08.469646+0000 mgr.y (mgr.25325) 1375 : cluster 0 pgmap v1381: 89 pgs: 89 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 865 op/s 2023-09-23T19:53:10.553 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:10.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:10.555 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:10.655 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:10.677 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:10.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:10.680 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:10.781 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:10.813 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:10.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:10.819 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:10.920 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:10.956 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:10.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:10.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:11.059 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:11.085 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:11.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:11.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:11.190 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:11.224 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:11.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:11.228 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:11.329 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:11.371 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:11.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:11.375 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:11.476 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:11.486 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 79 16 98304 98288 77.7482 53.9375 0.0140981 0.0128557 2023-09-23T19:53:11.515 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:11.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:11.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:11.618 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:11.643 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:11.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:11.650 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:11.751 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:11.782 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:11.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:11.785 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:11.886 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:11.912 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:11.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:11.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:11.991 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:53:11.992 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:53:12.018 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:12.056 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:12.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:12.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:12.165 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:12.175 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:12 smithi016 bash[75004]: cluster 2023-09-23T19:53:10.470505+0000 mgr.y (mgr.25325) 1376 : cluster 0 pgmap v1382: 89 pgs: 89 active+clean; 5.9 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 857 op/s 2023-09-23T19:53:12.176 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:12 smithi016 bash[77995]: cluster 2023-09-23T19:53:10.470505+0000 mgr.y (mgr.25325) 1376 : cluster 0 pgmap v1382: 89 pgs: 89 active+clean; 5.9 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 857 op/s 2023-09-23T19:53:12.199 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:12.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:12.203 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:12.304 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:12.329 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:12.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:12.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:12.437 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:12.467 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:12.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:12.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:12.486 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-09-23T19:53:12.482027+0000 min lat: 0.00144627 max lat: 0.312684 avg lat: 0.0129023 2023-09-23T19:53:12.486 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:53:12.486 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 80 16 99192 99176 77.47 55.5 0.00866008 0.0129023 2023-09-23T19:53:12.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:12 smithi092 bash[58855]: cluster 2023-09-23T19:53:10.470505+0000 mgr.y (mgr.25325) 1376 : cluster 0 pgmap v1382: 89 pgs: 89 active+clean; 5.9 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 857 op/s 2023-09-23T19:53:12.576 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:12.598 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:12.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:12.604 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:12.704 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:12.739 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:12.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:12.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:12.844 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:12.873 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:12.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:12.882 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:12.983 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:13.014 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:13.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:13.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:13.117 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:13.143 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:13.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:13.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:13.247 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:13.274 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:13.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:13.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:13.377 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:13.405 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:13.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:13.410 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:13.486 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 81 16 100018 100002 77.1509 51.625 0.0132335 0.0129562 2023-09-23T19:53:13.511 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:13.538 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:13.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:13.540 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:13.641 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:13.668 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:13.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:13.671 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:13.772 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:13.805 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:13.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:13.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:13.909 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:13.934 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:13.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:13.937 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:14.038 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:14.066 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:14.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:14.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:14.172 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:14.181 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:14 smithi092 bash[58855]: cluster 2023-09-23T19:53:12.471009+0000 mgr.y (mgr.25325) 1377 : cluster 0 pgmap v1383: 89 pgs: 89 active+clean; 5.9 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 857 op/s 2023-09-23T19:53:14.181 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:14 smithi092 bash[58855]: audit 2023-09-23T19:53:13.677378+0000 mon.a (mon.0) 1699 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:53:14.204 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:14.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:14.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:14.307 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:14.340 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:14.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:14.343 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:14.350 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:14 smithi016 bash[75004]: cluster 2023-09-23T19:53:12.471009+0000 mgr.y (mgr.25325) 1377 : cluster 0 pgmap v1383: 89 pgs: 89 active+clean; 5.9 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 857 op/s 2023-09-23T19:53:14.350 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:14 smithi016 bash[75004]: audit 2023-09-23T19:53:13.677378+0000 mon.a (mon.0) 1699 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:53:14.350 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:14 smithi016 bash[77995]: cluster 2023-09-23T19:53:12.471009+0000 mgr.y (mgr.25325) 1377 : cluster 0 pgmap v1383: 89 pgs: 89 active+clean; 5.9 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 857 op/s 2023-09-23T19:53:14.350 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:14 smithi016 bash[77995]: audit 2023-09-23T19:53:13.677378+0000 mon.a (mon.0) 1699 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:53:14.444 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:14.471 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:14.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:14.474 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:14.486 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 82 16 100897 100881 76.8799 54.9375 0.0228671 0.0130016 2023-09-23T19:53:14.574 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:14.597 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:14.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:14.601 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:14.702 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:14.726 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:14.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:14.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:14.829 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:14.861 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:14.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:14.863 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:14.964 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:14.988 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:14.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:14.994 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:15.095 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:15.124 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:15.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:15.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:15.231 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:15.255 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:15.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:15.259 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:15.360 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:15.395 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:15.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:15.400 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:15.486 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 83 16 101768 101752 76.6095 54.4375 0.015535 0.0130473 2023-09-23T19:53:15.501 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:15.528 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:15.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:15.533 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:15.634 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:15.661 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:15.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:15.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:15.765 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:15.788 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:15.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:15.792 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:15.893 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:15.902 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:53:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:53:15] "GET /metrics HTTP/1.1" 200 36304 "" "Prometheus/2.33.4" 2023-09-23T19:53:15.924 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:15.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:15.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:16.027 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:16.052 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:16.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:16.055 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:16.155 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:16.169 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:16 smithi016 bash[75004]: cluster 2023-09-23T19:53:14.472751+0000 mgr.y (mgr.25325) 1378 : cluster 0 pgmap v1384: 89 pgs: 89 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 50 MiB/s wr, 797 op/s 2023-09-23T19:53:16.169 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:16 smithi016 bash[77995]: cluster 2023-09-23T19:53:14.472751+0000 mgr.y (mgr.25325) 1378 : cluster 0 pgmap v1384: 89 pgs: 89 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 50 MiB/s wr, 797 op/s 2023-09-23T19:53:16.189 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:16.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:16.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:16.293 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:16.320 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:16.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:16.322 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:16.423 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:16.432 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:16 smithi092 bash[58855]: cluster 2023-09-23T19:53:14.472751+0000 mgr.y (mgr.25325) 1378 : cluster 0 pgmap v1384: 89 pgs: 89 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 50 MiB/s wr, 797 op/s 2023-09-23T19:53:16.457 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:16.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:16.459 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:16.486 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 84 16 102470 102454 76.2198 43.875 0.0165435 0.0131133 2023-09-23T19:53:16.560 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:16.585 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:16.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:16.589 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:16.690 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:16.720 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:16.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:16.724 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:16.825 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:16.856 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:16.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:16.861 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:16.962 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:16.991 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:16.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:16.994 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:17.095 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:17.133 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:17.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:17.135 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:17.236 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:17.264 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:17.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:17.267 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:17.368 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:17.398 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:17.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:17.405 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:17.486 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 85 16 103296 103280 75.9304 51.625 0.00880389 0.0131621 2023-09-23T19:53:17.506 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:17.544 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:17.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:17.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:17.611 INFO:teuthology.orchestra.run.smithi092.stdout:{ 2023-09-23T19:53:17.611 INFO:teuthology.orchestra.run.smithi092.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T19:53:17.611 INFO:teuthology.orchestra.run.smithi092.stdout:} 2023-09-23T19:53:17.628 INFO:teuthology.orchestra.run.smithi092.stdout:ERROR: (22) Invalid argument 2023-09-23T19:53:17.629 INFO:teuthology.orchestra.run.smithi092.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:53:17.649 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:17.681 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:17.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:17.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:17.787 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:17.816 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:17.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:17.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:17.920 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:17.950 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:17.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:17.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:18.057 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:18.084 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:18.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:18.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:18.189 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:18.199 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:18 smithi016 bash[77995]: cluster 2023-09-23T19:53:16.473624+0000 mgr.y (mgr.25325) 1379 : cluster 0 pgmap v1385: 89 pgs: 89 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 62 MiB/s wr, 984 op/s 2023-09-23T19:53:18.199 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:18 smithi016 bash[75004]: cluster 2023-09-23T19:53:16.473624+0000 mgr.y (mgr.25325) 1379 : cluster 0 pgmap v1385: 89 pgs: 89 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 62 MiB/s wr, 984 op/s 2023-09-23T19:53:18.219 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:18.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:18.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:18.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:18 smithi092 bash[58855]: cluster 2023-09-23T19:53:16.473624+0000 mgr.y (mgr.25325) 1379 : cluster 0 pgmap v1385: 89 pgs: 89 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 62 MiB/s wr, 984 op/s 2023-09-23T19:53:18.324 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:18.356 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:18.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:18.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:18.458 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:18.483 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:18.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:18.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:18.486 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 86 16 104099 104083 75.631 50.1875 0.0237241 0.0132165 2023-09-23T19:53:18.586 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:18.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:18.612 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:18.612 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:18.713 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:18.740 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:18.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:18.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:18.849 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:18.888 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:18.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:18.890 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:18.991 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:19.029 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:19.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:19.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:19.136 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:19.166 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:19.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:19.173 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:19.273 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:19.298 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:19.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:19.304 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:19.405 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:19.437 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:19.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:19.443 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:19.486 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 87 16 104940 104924 75.3658 52.5625 0.0282766 0.0132632 2023-09-23T19:53:19.544 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:19.598 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:19.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:19.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:19.705 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:19.746 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:19.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:19.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:19.850 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:19.882 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:19.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:19.887 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:19.987 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T19:53:19.987 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:20.075 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:20.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:20.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:20.181 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:20.211 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:20.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:20.214 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:20.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:20 smithi092 bash[58855]: cluster 2023-09-23T19:53:18.474165+0000 mgr.y (mgr.25325) 1380 : cluster 0 pgmap v1386: 89 pgs: 89 active+clean; 6.2 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 47 MiB/s wr, 755 op/s 2023-09-23T19:53:20.315 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:20.342 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:20.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:20.348 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:20.368 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:20 smithi016 bash[77995]: cluster 2023-09-23T19:53:18.474165+0000 mgr.y (mgr.25325) 1380 : cluster 0 pgmap v1386: 89 pgs: 89 active+clean; 6.2 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 47 MiB/s wr, 755 op/s 2023-09-23T19:53:20.369 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:20 smithi016 bash[75004]: cluster 2023-09-23T19:53:18.474165+0000 mgr.y (mgr.25325) 1380 : cluster 0 pgmap v1386: 89 pgs: 89 active+clean; 6.2 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 47 MiB/s wr, 755 op/s 2023-09-23T19:53:20.448 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:20.487 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 88 16 105697 105681 75.0463 47.3125 0.0159515 0.0133184 2023-09-23T19:53:20.490 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:20.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:20.494 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:20.585 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:53:20.595 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:20.620 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:53:20.626 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:20.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:20.629 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:20.661 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:53:20.700 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:53:20.730 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:20.734 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:53:20.758 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:20.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:20.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:20.776 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:53:20.823 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:53:20.864 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:20.886 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:53:20.896 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:53:20.897 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:53:20.897 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:53:20.897 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:53:20.897 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:53:20.897 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:53:20.897 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:53:20.897 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:53:20.902 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:20.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:20.907 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:21.008 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:21.042 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:21.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:21.047 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:21.138 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is unset 2023-09-23T19:53:21.148 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:21.158 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:21 smithi016 bash[75004]: audit 2023-09-23T19:53:20.516658+0000 mon.a (mon.0) 1700 : audit 0 from='client.? 172.21.15.16:0/597836318' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:53:21.158 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:21 smithi016 bash[75004]: audit 2023-09-23T19:53:21.000159+0000 mon.a (mon.0) 1701 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:53:21.158 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:21 smithi016 bash[75004]: audit 2023-09-23T19:53:21.006144+0000 mon.b (mon.1) 57 : audit 1 from='client.? 172.21.15.16:0/3386462963' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:53:21.158 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:21 smithi016 bash[77995]: audit 2023-09-23T19:53:20.516658+0000 mon.a (mon.0) 1700 : audit 0 from='client.? 172.21.15.16:0/597836318' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:53:21.159 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:21 smithi016 bash[77995]: audit 2023-09-23T19:53:21.000159+0000 mon.a (mon.0) 1701 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:53:21.159 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:21 smithi016 bash[77995]: audit 2023-09-23T19:53:21.006144+0000 mon.b (mon.1) 57 : audit 1 from='client.? 172.21.15.16:0/3386462963' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:53:21.175 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:21.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:21.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:21.283 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:21.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:21 smithi092 bash[58855]: audit 2023-09-23T19:53:20.516658+0000 mon.a (mon.0) 1700 : audit 0 from='client.? 172.21.15.16:0/597836318' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:53:21.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:21 smithi092 bash[58855]: audit 2023-09-23T19:53:21.000159+0000 mon.a (mon.0) 1701 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:53:21.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:21 smithi092 bash[58855]: audit 2023-09-23T19:53:21.006144+0000 mon.b (mon.1) 57 : audit 1 from='client.? 172.21.15.16:0/3386462963' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:53:21.307 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:21.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:21.309 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:21.410 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:21.437 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:21.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:21.440 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:21.487 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 89 15 106470 106455 74.7466 48.375 0.0158334 0.013373 2023-09-23T19:53:21.541 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:21.583 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:21.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:21.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:21.688 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:21.713 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:21.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:21.718 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:21.819 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:21.858 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:21.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:21.865 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:21.966 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:21.989 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:21.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:21.996 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:22.097 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:22.122 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:22.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:22.127 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:22.228 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:22.253 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:22.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:22.258 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:22.358 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:22.387 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:22.390 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:22 smithi092 bash[58855]: cluster 2023-09-23T19:53:20.475072+0000 mgr.y (mgr.25325) 1381 : cluster 0 pgmap v1387: 89 pgs: 89 active+clean; 6.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-09-23T19:53:22.390 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:22 smithi092 bash[58855]: cluster 2023-09-23T19:53:21.106075+0000 mon.a (mon.0) 1702 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:53:22.390 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:22 smithi092 bash[58855]: audit 2023-09-23T19:53:21.116591+0000 mon.a (mon.0) 1703 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:53:22.391 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:22 smithi092 bash[58855]: cluster 2023-09-23T19:53:21.116710+0000 mon.a (mon.0) 1704 : cluster 0 osdmap e242: 8 total, 8 up, 8 in 2023-09-23T19:53:22.391 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:22 smithi092 bash[58855]: cluster 2023-09-23T19:53:21.119378+0000 osd.6 (osd.6) 178 : cluster 3 failed to encode map e242 with expected crc 2023-09-23T19:53:22.391 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:22 smithi092 bash[58855]: cluster 2023-09-23T19:53:21.139715+0000 osd.4 (osd.4) 185 : cluster 3 failed to encode map e242 with expected crc 2023-09-23T19:53:22.391 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:22 smithi092 bash[58855]: cluster 2023-09-23T19:53:21.141743+0000 osd.5 (osd.5) 171 : cluster 3 failed to encode map e242 with expected crc 2023-09-23T19:53:22.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:22.392 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:22.487 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 90 4 107261 107257 74.473 50.125 0.0142535 0.0134222 2023-09-23T19:53:22.493 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:22.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:22 smithi016 bash[75004]: cluster 2023-09-23T19:53:20.475072+0000 mgr.y (mgr.25325) 1381 : cluster 0 pgmap v1387: 89 pgs: 89 active+clean; 6.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-09-23T19:53:22.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:22 smithi016 bash[75004]: cluster 2023-09-23T19:53:21.106075+0000 mon.a (mon.0) 1702 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:53:22.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:22 smithi016 bash[75004]: audit 2023-09-23T19:53:21.116591+0000 mon.a (mon.0) 1703 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:53:22.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:22 smithi016 bash[75004]: cluster 2023-09-23T19:53:21.116710+0000 mon.a (mon.0) 1704 : cluster 0 osdmap e242: 8 total, 8 up, 8 in 2023-09-23T19:53:22.504 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:22 smithi016 bash[75004]: cluster 2023-09-23T19:53:21.119378+0000 osd.6 (osd.6) 178 : cluster 3 failed to encode map e242 with expected crc 2023-09-23T19:53:22.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:22 smithi016 bash[75004]: cluster 2023-09-23T19:53:21.139715+0000 osd.4 (osd.4) 185 : cluster 3 failed to encode map e242 with expected crc 2023-09-23T19:53:22.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:22 smithi016 bash[75004]: cluster 2023-09-23T19:53:21.141743+0000 osd.5 (osd.5) 171 : cluster 3 failed to encode map e242 with expected crc 2023-09-23T19:53:22.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:22 smithi016 bash[77995]: cluster 2023-09-23T19:53:20.475072+0000 mgr.y (mgr.25325) 1381 : cluster 0 pgmap v1387: 89 pgs: 89 active+clean; 6.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-09-23T19:53:22.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:22 smithi016 bash[77995]: cluster 2023-09-23T19:53:21.106075+0000 mon.a (mon.0) 1702 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:53:22.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:22 smithi016 bash[77995]: audit 2023-09-23T19:53:21.116591+0000 mon.a (mon.0) 1703 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:53:22.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:22 smithi016 bash[77995]: cluster 2023-09-23T19:53:21.116710+0000 mon.a (mon.0) 1704 : cluster 0 osdmap e242: 8 total, 8 up, 8 in 2023-09-23T19:53:22.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:22 smithi016 bash[77995]: cluster 2023-09-23T19:53:21.119378+0000 osd.6 (osd.6) 178 : cluster 3 failed to encode map e242 with expected crc 2023-09-23T19:53:22.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:22 smithi016 bash[77995]: cluster 2023-09-23T19:53:21.139715+0000 osd.4 (osd.4) 185 : cluster 3 failed to encode map e242 with expected crc 2023-09-23T19:53:22.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:22 smithi016 bash[77995]: cluster 2023-09-23T19:53:21.141743+0000 osd.5 (osd.5) 171 : cluster 3 failed to encode map e242 with expected crc 2023-09-23T19:53:22.519 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:22.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:22.521 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:22.622 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:22.651 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:22.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:22.654 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:22.755 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:22.787 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:22.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:22.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:22.894 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:22.923 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:22.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:22.926 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:23.027 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:23.057 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:23.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:23.060 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:23.160 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:23.188 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:23.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:23.190 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:23.291 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:23.320 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:23.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:23.323 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:23.424 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:23.454 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:23.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:23.457 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:23.488 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Total time run: 90.0153 2023-09-23T19:53:23.488 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Total writes made: 107261 2023-09-23T19:53:23.488 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Write size: 65536 2023-09-23T19:53:23.488 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Object size: 65536 2023-09-23T19:53:23.488 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Bandwidth (MB/sec): 74.4741 2023-09-23T19:53:23.488 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Stddev Bandwidth: 28.0775 2023-09-23T19:53:23.488 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Max bandwidth (MB/sec): 161.062 2023-09-23T19:53:23.488 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Min bandwidth (MB/sec): 43.875 2023-09-23T19:53:23.489 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Average IOPS: 1191 2023-09-23T19:53:23.489 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Stddev IOPS: 449.241 2023-09-23T19:53:23.489 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Max IOPS: 2577 2023-09-23T19:53:23.489 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Min IOPS: 702 2023-09-23T19:53:23.489 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Average Latency(s): 0.0134225 2023-09-23T19:53:23.489 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Stddev Latency(s): 0.0117205 2023-09-23T19:53:23.489 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Max latency(s): 0.312684 2023-09-23T19:53:23.489 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Min latency(s): 0.00144627 2023-09-23T19:53:23.500 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T19:53:23.558 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:23.583 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:23.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:23.588 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:23.689 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:23.726 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:23.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:23.732 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:23.833 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:23.861 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:23.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:23.863 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:23.964 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:23.995 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:24.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:24.000 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:24.101 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:24.130 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:24.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:24.136 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:24.236 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:24.262 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:24.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:24.269 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:24.370 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:24.379 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:24 smithi092 bash[58855]: cluster 2023-09-23T19:53:22.475497+0000 mgr.y (mgr.25325) 1382 : cluster 0 pgmap v1389: 89 pgs: 89 active+clean; 6.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 52 MiB/s wr, 832 op/s 2023-09-23T19:53:24.401 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:24.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:24.404 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:24.504 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:24.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:24 smithi016 bash[77995]: cluster 2023-09-23T19:53:22.475497+0000 mgr.y (mgr.25325) 1382 : cluster 0 pgmap v1389: 89 pgs: 89 active+clean; 6.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 52 MiB/s wr, 832 op/s 2023-09-23T19:53:24.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:24 smithi016 bash[75004]: cluster 2023-09-23T19:53:22.475497+0000 mgr.y (mgr.25325) 1382 : cluster 0 pgmap v1389: 89 pgs: 89 active+clean; 6.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 52 MiB/s wr, 832 op/s 2023-09-23T19:53:24.538 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:24.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:24.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:24.642 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:24.685 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:24.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:24.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:24.792 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:24.830 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:24.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:24.836 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:24.905 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:53:24.937 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:24.958 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-09-23T19:53:24.958 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T19:53:24.958 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2023-09-23T19:53:24.958 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T19:53:24.958 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T19:53:24.959 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 config set filestore_inject_stall 3 2023-09-23T19:53:24.972 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:24.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:24.978 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:25.079 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:25.104 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:25.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:25.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:25.212 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:25.240 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:25.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:25.246 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:25.330 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:53:25.347 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:25.409 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:25.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:25.412 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:25.513 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:25.535 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:25.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:25.540 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:25.551 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T19:53:25.640 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:53:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:53:25] "GET /metrics HTTP/1.1" 200 36304 "" "Prometheus/2.33.4" 2023-09-23T19:53:25.641 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:25.671 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:25.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:25.675 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:25.776 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:25.814 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:25.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:25.817 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:25.917 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:25.945 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:25.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:25.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:26.049 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:26.077 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:26.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:26.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:26.180 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:26.190 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:26 smithi016 bash[77995]: cluster 2023-09-23T19:53:24.478981+0000 mgr.y (mgr.25325) 1383 : cluster 0 pgmap v1390: 89 pgs: 89 active+clean; 6.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 51 MiB/s wr, 822 op/s 2023-09-23T19:53:26.190 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:26 smithi016 bash[75004]: cluster 2023-09-23T19:53:24.478981+0000 mgr.y (mgr.25325) 1383 : cluster 0 pgmap v1390: 89 pgs: 89 active+clean; 6.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 51 MiB/s wr, 822 op/s 2023-09-23T19:53:26.209 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:26.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:26.213 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:26.313 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:26.345 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:26.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:26.351 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:26.452 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:26.478 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:26.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:26.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:26.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:26 smithi092 bash[58855]: cluster 2023-09-23T19:53:24.478981+0000 mgr.y (mgr.25325) 1383 : cluster 0 pgmap v1390: 89 pgs: 89 active+clean; 6.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 51 MiB/s wr, 822 op/s 2023-09-23T19:53:26.582 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:26.606 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:26.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:26.609 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:26.710 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:26.736 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:26.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:26.740 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:26.841 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:26.872 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:26.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:26.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:26.976 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:27.007 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:27.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:27.009 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:27.110 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:27.141 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:27.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:27.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:27.244 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:27.277 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:27.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:27.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:27.380 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:27.404 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:27.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:27.410 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:27.511 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:27.544 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:27.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:27.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:27.648 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:27.681 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:27.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:27.689 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:27.790 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:27.819 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:27.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:27.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:27.923 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:27.963 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:27.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:27.967 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:28.068 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:28.110 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:28.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:28.114 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:28.139 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:28 smithi016 bash[75004]: cluster 2023-09-23T19:53:26.479740+0000 mgr.y (mgr.25325) 1384 : cluster 0 pgmap v1391: 89 pgs: 89 active+clean; 6.5 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 102 B/s rd, 40 MiB/s wr, 730 op/s 2023-09-23T19:53:28.215 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:28.247 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:28.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:28.253 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:28.353 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:28.381 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:28.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:28.386 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:28.487 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:28.495 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:28 smithi092 bash[58855]: cluster 2023-09-23T19:53:26.479740+0000 mgr.y (mgr.25325) 1384 : cluster 0 pgmap v1391: 89 pgs: 89 active+clean; 6.5 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 102 B/s rd, 40 MiB/s wr, 730 op/s 2023-09-23T19:53:28.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:28.523 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:28.523 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:28.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:28 smithi016 bash[77995]: cluster 2023-09-23T19:53:26.479740+0000 mgr.y (mgr.25325) 1384 : cluster 0 pgmap v1391: 89 pgs: 89 active+clean; 6.5 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 102 B/s rd, 40 MiB/s wr, 730 op/s 2023-09-23T19:53:28.623 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:28.641 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:28.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:28.643 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:28.744 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:28.783 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:28.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:28.786 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:28.887 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:28.913 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:28.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:28.918 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:28.985 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:53:29.019 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:29.067 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:29.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:29.072 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:29.173 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:29.209 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:29.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:29.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:29.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:29 smithi092 bash[58855]: audit 2023-09-23T19:53:28.678114+0000 mon.a (mon.0) 1705 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:53:29.313 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:29.344 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:29.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:29.349 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:29.450 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:29.480 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:29.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:29.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:29.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:29 smithi016 bash[77995]: audit 2023-09-23T19:53:28.678114+0000 mon.a (mon.0) 1705 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:53:29.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:29 smithi016 bash[75004]: audit 2023-09-23T19:53:28.678114+0000 mon.a (mon.0) 1705 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:53:29.584 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:29.609 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:29.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:29.613 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:29.714 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:29.745 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:29.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:29.750 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:29.851 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:29.878 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:29.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:29.883 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:29.984 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:30.002 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:30.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:30.005 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:30.106 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:30.131 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:30.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:30.134 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:30.176 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:53:30.177 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:53:30.235 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:30.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:30.266 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:30.267 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:30.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:30 smithi092 bash[58855]: cluster 2023-09-23T19:53:28.480260+0000 mgr.y (mgr.25325) 1385 : cluster 0 pgmap v1392: 89 pgs: 89 active+clean; 6.5 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 102 B/s rd, 36 MiB/s wr, 718 op/s 2023-09-23T19:53:30.367 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:30.375 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:30 smithi016 bash[77995]: cluster 2023-09-23T19:53:28.480260+0000 mgr.y (mgr.25325) 1385 : cluster 0 pgmap v1392: 89 pgs: 89 active+clean; 6.5 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 102 B/s rd, 36 MiB/s wr, 718 op/s 2023-09-23T19:53:30.375 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:30 smithi016 bash[75004]: cluster 2023-09-23T19:53:28.480260+0000 mgr.y (mgr.25325) 1385 : cluster 0 pgmap v1392: 89 pgs: 89 active+clean; 6.5 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 102 B/s rd, 36 MiB/s wr, 718 op/s 2023-09-23T19:53:30.400 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:30.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:30.404 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:30.505 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:30.531 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:30.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:30.535 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:30.636 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:30.670 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:30.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:30.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:30.779 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:30.813 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:30.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:30.819 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:30.919 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:30.954 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:30.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:30.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:31.059 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:31.092 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:31.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:31.095 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:31.196 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:31.225 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:31.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:31.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:31.330 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:31.375 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:31.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:31.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:31.457 INFO:teuthology.orchestra.run.smithi092.stdout:{ 2023-09-23T19:53:31.457 INFO:teuthology.orchestra.run.smithi092.stdout: "ops": [], 2023-09-23T19:53:31.457 INFO:teuthology.orchestra.run.smithi092.stdout: "complaint_time": 30, 2023-09-23T19:53:31.457 INFO:teuthology.orchestra.run.smithi092.stdout: "num_blocked_ops": 0 2023-09-23T19:53:31.457 INFO:teuthology.orchestra.run.smithi092.stdout:} 2023-09-23T19:53:31.479 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:31.510 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:31.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:31.512 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:31.613 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:31.643 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:31.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:31.647 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:31.748 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:31.771 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:31.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:31.776 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:31.877 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:31.905 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:31.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:31.909 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:32.010 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:32.049 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:32.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:32.053 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:32.154 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:32.161 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:32 smithi016 bash[77995]: cluster 2023-09-23T19:53:30.481046+0000 mgr.y (mgr.25325) 1386 : cluster 0 pgmap v1393: 89 pgs: 89 active+clean; 6.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 102 B/s rd, 14 MiB/s wr, 720 op/s 2023-09-23T19:53:32.162 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:32 smithi016 bash[75004]: cluster 2023-09-23T19:53:30.481046+0000 mgr.y (mgr.25325) 1386 : cluster 0 pgmap v1393: 89 pgs: 89 active+clean; 6.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 102 B/s rd, 14 MiB/s wr, 720 op/s 2023-09-23T19:53:32.182 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:32.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:32.185 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:32.285 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:32 smithi092 bash[58855]: cluster 2023-09-23T19:53:30.481046+0000 mgr.y (mgr.25325) 1386 : cluster 0 pgmap v1393: 89 pgs: 89 active+clean; 6.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 102 B/s rd, 14 MiB/s wr, 720 op/s 2023-09-23T19:53:32.286 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:32.313 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:32.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:32.319 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:32.420 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:32.454 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:32.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:32.458 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:32.559 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:32.594 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:32.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:32.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:32.697 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:32.728 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:32.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:32.731 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:32.831 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:32.855 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:32.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:32.858 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:32.959 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:32.990 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:32.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:32.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:33.094 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:33.124 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:33.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:33.127 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:33.227 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:33.272 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:33.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:33.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:33.377 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:33.403 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:33.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:33.407 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:33.508 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:33.532 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:33.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:33.535 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:33.602 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T19:53:33.636 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:33.673 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:33.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:33.679 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:33.780 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:33.822 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:33.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:33.824 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:33.925 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:33.957 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:33.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:33.960 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:34.061 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:34.110 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:34.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:34.116 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:34.217 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:34.226 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:34 smithi092 bash[58855]: cluster 2023-09-23T19:53:32.481533+0000 mgr.y (mgr.25325) 1387 : cluster 0 pgmap v1394: 89 pgs: 89 active+clean; 6.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 90 B/s rd, 13 MiB/s wr, 656 op/s 2023-09-23T19:53:34.251 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:34.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:34.254 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:34.355 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:34.378 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:34.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:34.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:34.483 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:34.492 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:34 smithi016 bash[77995]: cluster 2023-09-23T19:53:32.481533+0000 mgr.y (mgr.25325) 1387 : cluster 0 pgmap v1394: 89 pgs: 89 active+clean; 6.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 90 B/s rd, 13 MiB/s wr, 656 op/s 2023-09-23T19:53:34.492 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:34 smithi016 bash[75004]: cluster 2023-09-23T19:53:32.481533+0000 mgr.y (mgr.25325) 1387 : cluster 0 pgmap v1394: 89 pgs: 89 active+clean; 6.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 90 B/s rd, 13 MiB/s wr, 656 op/s 2023-09-23T19:53:34.511 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:34.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:34.514 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:34.615 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:34.642 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:34.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:34.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:34.745 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:34.774 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:34.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:34.779 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:34.879 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:34.913 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:34.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:34.917 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:35.017 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:35.054 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:35.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:35.060 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:35.161 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:35.186 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:35.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:35.193 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:35.294 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:35.325 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:35.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:35.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:35.341 INFO:teuthology.orchestra.run.smithi092.stdout:{ 2023-09-23T19:53:35.341 INFO:teuthology.orchestra.run.smithi092.stdout: "success": "" 2023-09-23T19:53:35.341 INFO:teuthology.orchestra.run.smithi092.stdout:} 2023-09-23T19:53:35.429 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:35.460 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:35.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:35.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:35.563 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:35.593 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:35.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:35.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:35.701 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:35.736 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:35.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:35.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:35.843 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:35.880 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:35.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:35.884 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:35.985 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:36.018 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:36.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:36.020 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:36.037 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:53:35 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:53:35] "GET /metrics HTTP/1.1" 200 36308 "" "Prometheus/2.33.4" 2023-09-23T19:53:36.121 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:36.162 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:36.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:36.165 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:36.266 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:36.295 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:36.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:36.300 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:36.401 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:36.409 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:36 smithi092 bash[58855]: cluster 2023-09-23T19:53:34.482020+0000 mgr.y (mgr.25325) 1388 : cluster 0 pgmap v1395: 89 pgs: 89 active+clean; 6.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 11 MiB/s wr, 662 op/s 2023-09-23T19:53:36.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:36.443 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:36.444 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:36.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:36 smithi016 bash[75004]: cluster 2023-09-23T19:53:34.482020+0000 mgr.y (mgr.25325) 1388 : cluster 0 pgmap v1395: 89 pgs: 89 active+clean; 6.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 11 MiB/s wr, 662 op/s 2023-09-23T19:53:36.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:36 smithi016 bash[77995]: cluster 2023-09-23T19:53:34.482020+0000 mgr.y (mgr.25325) 1388 : cluster 0 pgmap v1395: 89 pgs: 89 active+clean; 6.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 11 MiB/s wr, 662 op/s 2023-09-23T19:53:36.544 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:36.569 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:36.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:36.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:36.676 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:36.714 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:36.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:36.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:36.819 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:36.843 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:36.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:36.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:36.949 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:36.977 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:36.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:36.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:37.080 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:37.106 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:37.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:37.108 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:37.209 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:37.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:37.254 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:37.255 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:37.355 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:37.390 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:37.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:37.396 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:37.497 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:37.540 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:37.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:37.544 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:37.645 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:37.693 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:37.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:37.697 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:37.797 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:37.832 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:37.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:37.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:37.935 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:37.966 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:37.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:37.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:38.069 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:38.107 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:38.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:38.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:38.212 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:38.221 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:38 smithi016 bash[75004]: cluster 2023-09-23T19:53:36.484836+0000 mgr.y (mgr.25325) 1389 : cluster 0 pgmap v1396: 89 pgs: 89 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 7.7 MiB/s wr, 862 op/s 2023-09-23T19:53:38.221 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:38 smithi016 bash[77995]: cluster 2023-09-23T19:53:36.484836+0000 mgr.y (mgr.25325) 1389 : cluster 0 pgmap v1396: 89 pgs: 89 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 7.7 MiB/s wr, 862 op/s 2023-09-23T19:53:38.243 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:38.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:38.246 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:38.347 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:38.355 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:38 smithi092 bash[58855]: cluster 2023-09-23T19:53:36.484836+0000 mgr.y (mgr.25325) 1389 : cluster 0 pgmap v1396: 89 pgs: 89 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 7.7 MiB/s wr, 862 op/s 2023-09-23T19:53:38.367 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:53:38.368 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:38.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:38.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:38.404 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:53:38.452 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:53:38.472 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:38.503 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:38.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:38.506 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:38.506 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:53:38.548 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:53:38.591 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:53:38.607 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:38.640 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:53:38.642 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:38.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:38.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:38.680 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:53:38.685 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:53:38.685 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:53:38.685 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:53:38.685 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:53:38.685 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:53:38.685 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:53:38.685 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:53:38.686 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:53:38.749 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:38.781 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:38.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:38.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:38.885 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:38.910 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:38.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:38.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:38.934 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:53:39.016 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:39.046 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:39.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:39.053 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:39.154 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:39.193 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:39.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:39.196 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is unset 2023-09-23T19:53:39.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:39.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:39 smithi092 bash[58855]: audit 2023-09-23T19:53:38.320464+0000 mon.c (mon.2) 216 : audit 0 from='client.? 172.21.15.16:0/737227010' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:53:39.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:39 smithi092 bash[58855]: audit 2023-09-23T19:53:38.850289+0000 mon.c (mon.2) 217 : audit 1 from='client.? 172.21.15.16:0/3725960375' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:53:39.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:39 smithi092 bash[58855]: audit 2023-09-23T19:53:38.850820+0000 mon.a (mon.0) 1706 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:53:39.297 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:39.320 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:39.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:39.325 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:39.425 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:39.466 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:39.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:39.467 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:39.568 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:39.599 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:39.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:39.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:39.705 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:39.738 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:39.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:39.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:39.817 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:39 smithi016 bash[75004]: audit 2023-09-23T19:53:38.320464+0000 mon.c (mon.2) 216 : audit 0 from='client.? 172.21.15.16:0/737227010' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:53:39.817 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:39 smithi016 bash[75004]: audit 2023-09-23T19:53:38.850289+0000 mon.c (mon.2) 217 : audit 1 from='client.? 172.21.15.16:0/3725960375' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:53:39.817 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:39 smithi016 bash[75004]: audit 2023-09-23T19:53:38.850820+0000 mon.a (mon.0) 1706 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:53:39.818 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:39 smithi016 bash[77995]: audit 2023-09-23T19:53:38.320464+0000 mon.c (mon.2) 216 : audit 0 from='client.? 172.21.15.16:0/737227010' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:53:39.818 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:39 smithi016 bash[77995]: audit 2023-09-23T19:53:38.850289+0000 mon.c (mon.2) 217 : audit 1 from='client.? 172.21.15.16:0/3725960375' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:53:39.818 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:39 smithi016 bash[77995]: audit 2023-09-23T19:53:38.850820+0000 mon.a (mon.0) 1706 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:53:39.843 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:39.881 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:39.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:39.884 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:39.985 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:40.013 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:40.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:40.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:40.116 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:40.150 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:40.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:40.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:40.174 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:40 smithi092 bash[58855]: cluster 2023-09-23T19:53:38.485363+0000 mgr.y (mgr.25325) 1390 : cluster 0 pgmap v1397: 89 pgs: 89 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 203 KiB/s wr, 709 op/s 2023-09-23T19:53:40.174 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:40 smithi092 bash[58855]: cluster 2023-09-23T19:53:39.157999+0000 mon.a (mon.0) 1707 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:53:40.174 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:40 smithi092 bash[58855]: cluster 2023-09-23T19:53:39.158048+0000 mon.a (mon.0) 1708 : cluster 1 Cluster is now healthy 2023-09-23T19:53:40.174 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:40 smithi092 bash[58855]: audit 2023-09-23T19:53:39.175177+0000 mon.a (mon.0) 1709 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:53:40.174 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:40 smithi092 bash[58855]: cluster 2023-09-23T19:53:39.175584+0000 mon.a (mon.0) 1710 : cluster 0 osdmap e243: 8 total, 8 up, 8 in 2023-09-23T19:53:40.174 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:40 smithi092 bash[58855]: cluster 2023-09-23T19:53:39.180304+0000 osd.5 (osd.5) 172 : cluster 3 failed to encode map e243 with expected crc 2023-09-23T19:53:40.174 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:40 smithi092 bash[58855]: cluster 2023-09-23T19:53:39.185313+0000 osd.4 (osd.4) 186 : cluster 3 failed to encode map e243 with expected crc 2023-09-23T19:53:40.174 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:40 smithi092 bash[58855]: cluster 2023-09-23T19:53:39.187479+0000 osd.6 (osd.6) 179 : cluster 3 failed to encode map e243 with expected crc 2023-09-23T19:53:40.258 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:40.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:40.289 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:40.290 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:40.391 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:40.422 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:40.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:40.427 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:40.528 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:40.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:40 smithi016 bash[75004]: cluster 2023-09-23T19:53:38.485363+0000 mgr.y (mgr.25325) 1390 : cluster 0 pgmap v1397: 89 pgs: 89 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 203 KiB/s wr, 709 op/s 2023-09-23T19:53:40.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:40 smithi016 bash[75004]: cluster 2023-09-23T19:53:39.157999+0000 mon.a (mon.0) 1707 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:53:40.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:40 smithi016 bash[75004]: cluster 2023-09-23T19:53:39.158048+0000 mon.a (mon.0) 1708 : cluster 1 Cluster is now healthy 2023-09-23T19:53:40.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:40 smithi016 bash[75004]: audit 2023-09-23T19:53:39.175177+0000 mon.a (mon.0) 1709 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:53:40.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:40 smithi016 bash[75004]: cluster 2023-09-23T19:53:39.175584+0000 mon.a (mon.0) 1710 : cluster 0 osdmap e243: 8 total, 8 up, 8 in 2023-09-23T19:53:40.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:40 smithi016 bash[75004]: cluster 2023-09-23T19:53:39.180304+0000 osd.5 (osd.5) 172 : cluster 3 failed to encode map e243 with expected crc 2023-09-23T19:53:40.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:40 smithi016 bash[75004]: cluster 2023-09-23T19:53:39.185313+0000 osd.4 (osd.4) 186 : cluster 3 failed to encode map e243 with expected crc 2023-09-23T19:53:40.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:40 smithi016 bash[75004]: cluster 2023-09-23T19:53:39.187479+0000 osd.6 (osd.6) 179 : cluster 3 failed to encode map e243 with expected crc 2023-09-23T19:53:40.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:40 smithi016 bash[77995]: cluster 2023-09-23T19:53:38.485363+0000 mgr.y (mgr.25325) 1390 : cluster 0 pgmap v1397: 89 pgs: 89 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 203 KiB/s wr, 709 op/s 2023-09-23T19:53:40.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:40 smithi016 bash[77995]: cluster 2023-09-23T19:53:39.157999+0000 mon.a (mon.0) 1707 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:53:40.538 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:40 smithi016 bash[77995]: cluster 2023-09-23T19:53:39.158048+0000 mon.a (mon.0) 1708 : cluster 1 Cluster is now healthy 2023-09-23T19:53:40.538 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:40 smithi016 bash[77995]: audit 2023-09-23T19:53:39.175177+0000 mon.a (mon.0) 1709 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:53:40.538 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:40 smithi016 bash[77995]: cluster 2023-09-23T19:53:39.175584+0000 mon.a (mon.0) 1710 : cluster 0 osdmap e243: 8 total, 8 up, 8 in 2023-09-23T19:53:40.538 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:40 smithi016 bash[77995]: cluster 2023-09-23T19:53:39.180304+0000 osd.5 (osd.5) 172 : cluster 3 failed to encode map e243 with expected crc 2023-09-23T19:53:40.538 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:40 smithi016 bash[77995]: cluster 2023-09-23T19:53:39.185313+0000 osd.4 (osd.4) 186 : cluster 3 failed to encode map e243 with expected crc 2023-09-23T19:53:40.538 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:40 smithi016 bash[77995]: cluster 2023-09-23T19:53:39.187479+0000 osd.6 (osd.6) 179 : cluster 3 failed to encode map e243 with expected crc 2023-09-23T19:53:40.566 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:40.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:40.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:40.672 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:40.702 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:40.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:40.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:40.808 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:40.843 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:40.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:40.849 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:40.950 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:40.983 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:40.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:40.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:41.089 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:41.126 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:41.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:41.139 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:41.240 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:41.273 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:41.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:41.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:41.377 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:41.405 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:41.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:41.406 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:41.427 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-09-23T19:53:41.427 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T19:53:41.427 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T19:53:41.427 INFO:tasks.thrashosds.thrasher:Setting osd 4 primary_affinity to 1.000000 2023-09-23T19:53:41.428 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 4 1 2023-09-23T19:53:41.507 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:41.550 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:41.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:41.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:41.661 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:41.691 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:41.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:41.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:41.796 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:41.819 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:41.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:41.822 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:41.923 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:41.949 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:41.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:41.956 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:42.057 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:42.084 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:42.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:42.088 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:42.189 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:42.225 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:42.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:42.230 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:42.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:42 smithi016 bash[75004]: cluster 2023-09-23T19:53:40.486158+0000 mgr.y (mgr.25325) 1391 : cluster 0 pgmap v1399: 89 pgs: 89 active+clean; 5.8 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 799 op/s 2023-09-23T19:53:42.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:42 smithi016 bash[77995]: cluster 2023-09-23T19:53:40.486158+0000 mgr.y (mgr.25325) 1391 : cluster 0 pgmap v1399: 89 pgs: 89 active+clean; 5.8 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 799 op/s 2023-09-23T19:53:42.296 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:53:42.331 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:42.370 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:42.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:42.376 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:42.477 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:42.486 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:42 smithi092 bash[58855]: cluster 2023-09-23T19:53:40.486158+0000 mgr.y (mgr.25325) 1391 : cluster 0 pgmap v1399: 89 pgs: 89 active+clean; 5.8 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 799 op/s 2023-09-23T19:53:42.514 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:42.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:42.522 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:42.623 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:42.660 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:42.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:42.665 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:42.766 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:42.791 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:42.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:42.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:42.895 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:42.928 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:42.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:42.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:43.030 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:43.064 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:43.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:43.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:43.155 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T19:53:43.171 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:43.196 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:43.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:43.203 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:43.304 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:43.335 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:43.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:43.342 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:43.442 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:43.476 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:43.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:43.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:43.580 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:43.620 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:43.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:43.626 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:43.727 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:43.792 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:43.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:43.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:43.888 INFO:teuthology.orchestra.run.smithi092.stdout:ERROR: (22) Invalid argument 2023-09-23T19:53:43.889 INFO:teuthology.orchestra.run.smithi092.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:53:43.894 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:43.933 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:43.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:43.937 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:44.037 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:44.064 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:44.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:44.070 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:44.171 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:44.200 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:44.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:44.205 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:44.305 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:44.337 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:44.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:44.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:44.443 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:44.480 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:44.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:44.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:44.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:44 smithi016 bash[75004]: cluster 2023-09-23T19:53:42.486632+0000 mgr.y (mgr.25325) 1392 : cluster 0 pgmap v1400: 89 pgs: 89 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 819 op/s 2023-09-23T19:53:44.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:44 smithi016 bash[75004]: audit 2023-09-23T19:53:43.682252+0000 mon.a (mon.0) 1711 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:53:44.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:44 smithi016 bash[75004]: audit 2023-09-23T19:53:43.683518+0000 mon.a (mon.0) 1712 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:53:44.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:44 smithi016 bash[77995]: cluster 2023-09-23T19:53:42.486632+0000 mgr.y (mgr.25325) 1392 : cluster 0 pgmap v1400: 89 pgs: 89 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 819 op/s 2023-09-23T19:53:44.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:44 smithi016 bash[77995]: audit 2023-09-23T19:53:43.682252+0000 mon.a (mon.0) 1711 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:53:44.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:44 smithi016 bash[77995]: audit 2023-09-23T19:53:43.683518+0000 mon.a (mon.0) 1712 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:53:44.585 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:44.608 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:44.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:44.614 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:44.675 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:44 smithi092 bash[58855]: cluster 2023-09-23T19:53:42.486632+0000 mgr.y (mgr.25325) 1392 : cluster 0 pgmap v1400: 89 pgs: 89 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 819 op/s 2023-09-23T19:53:44.675 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:44 smithi092 bash[58855]: audit 2023-09-23T19:53:43.682252+0000 mon.a (mon.0) 1711 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:53:44.675 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:44 smithi092 bash[58855]: audit 2023-09-23T19:53:43.683518+0000 mon.a (mon.0) 1712 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:53:44.715 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:44.746 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:44.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:44.749 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:44.849 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:44.888 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:44.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:44.892 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:44.942 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T19:53:44.992 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:45.025 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:45.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:45.031 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:45.132 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:45.181 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:45.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:45.189 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:45.290 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:45.316 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:45.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:45.320 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:45.421 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:45.458 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:45.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:45.466 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:45.566 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:45.595 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:45.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:45.601 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:45.702 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:45.713 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:53:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:53:45] "GET /metrics HTTP/1.1" 200 36306 "" "Prometheus/2.33.4" 2023-09-23T19:53:45.730 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:45.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:45.732 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:45.833 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:45.870 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:45.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:45.873 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:45.973 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:45.996 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:46.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:46.003 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:46.103 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:46.138 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:46.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:46.141 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:46.242 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:46.250 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:46 smithi092 bash[58855]: cluster 2023-09-23T19:53:44.487123+0000 mgr.y (mgr.25325) 1393 : cluster 0 pgmap v1401: 89 pgs: 89 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 768 op/s 2023-09-23T19:53:46.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:46.275 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:46.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:46.280 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:53:46.285 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:53:46.376 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:46.405 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:46.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:46.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:46.509 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:46.518 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:46 smithi016 bash[77995]: cluster 2023-09-23T19:53:44.487123+0000 mgr.y (mgr.25325) 1393 : cluster 0 pgmap v1401: 89 pgs: 89 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 768 op/s 2023-09-23T19:53:46.518 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:46 smithi016 bash[75004]: cluster 2023-09-23T19:53:44.487123+0000 mgr.y (mgr.25325) 1393 : cluster 0 pgmap v1401: 89 pgs: 89 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 768 op/s 2023-09-23T19:53:46.541 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:46.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:46.545 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:46.646 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:46.684 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:46.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:46.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:46.788 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:46.824 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:46.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:46.827 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:46.928 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:46.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:46.970 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:46.971 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:47.071 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:47.098 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:47.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:47.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:47.203 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:47.248 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:47.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:47.255 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:47.299 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:53:47.356 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:47.380 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:47.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:47.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:47.486 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:47.519 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:47.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:47.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:47.623 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:47.649 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:47.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:47.652 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:47.753 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:47.786 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:47.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:47.792 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:47.893 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:47.929 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:47.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:47.932 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:48.033 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:48.055 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:48.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:48.058 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:48.159 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:48.182 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:48.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:48.185 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:48.286 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:48.313 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:48.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:48.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:48.348 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:48 smithi016 bash[77995]: cluster 2023-09-23T19:53:46.487740+0000 mgr.y (mgr.25325) 1394 : cluster 0 pgmap v1402: 89 pgs: 89 active+clean; 5.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 750 op/s 2023-09-23T19:53:48.349 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:48 smithi016 bash[75004]: cluster 2023-09-23T19:53:46.487740+0000 mgr.y (mgr.25325) 1394 : cluster 0 pgmap v1402: 89 pgs: 89 active+clean; 5.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 750 op/s 2023-09-23T19:53:48.419 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:48.442 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:48.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:48.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:48.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:48 smithi092 bash[58855]: cluster 2023-09-23T19:53:46.487740+0000 mgr.y (mgr.25325) 1394 : cluster 0 pgmap v1402: 89 pgs: 89 active+clean; 5.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 750 op/s 2023-09-23T19:53:48.549 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:48.582 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:48.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:48.588 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:48.688 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:48.718 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:48.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:48.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:48.823 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:48.862 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:48.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:48.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:48.965 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:48.996 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:48.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:48.999 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:49.100 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:49.128 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:49.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:49.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:49.233 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:49.259 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:49.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:49.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:49.364 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:49.395 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:49.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:49.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:49.500 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:49.525 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:49.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:49.528 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:49.629 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:49.678 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:49.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:49.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:49.783 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:49.813 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:53:49.820 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:49.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:49.826 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:49.927 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:49.961 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:49.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:49.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:50.065 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:50.100 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:50.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:50.105 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:50.206 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:50.230 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:50.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:50.236 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:50.337 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:50.369 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:50.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:50.373 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:50.474 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:50.482 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:50 smithi092 bash[58855]: cluster 2023-09-23T19:53:48.488171+0000 mgr.y (mgr.25325) 1395 : cluster 0 pgmap v1403: 89 pgs: 89 active+clean; 5.5 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 791 op/s 2023-09-23T19:53:50.505 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:50.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:50.510 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:50.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:50 smithi016 bash[77995]: cluster 2023-09-23T19:53:48.488171+0000 mgr.y (mgr.25325) 1395 : cluster 0 pgmap v1403: 89 pgs: 89 active+clean; 5.5 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 791 op/s 2023-09-23T19:53:50.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:50 smithi016 bash[75004]: cluster 2023-09-23T19:53:48.488171+0000 mgr.y (mgr.25325) 1395 : cluster 0 pgmap v1403: 89 pgs: 89 active+clean; 5.5 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 791 op/s 2023-09-23T19:53:50.611 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:50.638 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:50.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:50.643 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:50.743 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:50.782 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:50.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:50.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:50.886 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:50.913 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:50.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:50.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:51.016 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:51.039 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:51.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:51.043 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:51.144 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:51.193 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:51.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:51.196 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:51.297 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:51.321 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:51.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:51.326 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:51.427 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:51.462 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:51.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:51.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:51.567 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:51.595 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:51.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:51.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:51.699 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:51.729 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:51.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:51.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:51.833 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:51.865 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:51.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:51.869 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:51.970 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:52.004 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:52.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:52.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:52.112 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:52.141 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:52.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:52.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:52.197 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:52 smithi092 bash[58855]: cluster 2023-09-23T19:53:50.488832+0000 mgr.y (mgr.25325) 1396 : cluster 0 pgmap v1404: 89 pgs: 89 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 946 op/s 2023-09-23T19:53:52.250 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:52.281 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:52.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:52.283 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:52.384 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:52.391 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:52 smithi016 bash[77995]: cluster 2023-09-23T19:53:50.488832+0000 mgr.y (mgr.25325) 1396 : cluster 0 pgmap v1404: 89 pgs: 89 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 946 op/s 2023-09-23T19:53:52.391 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:52 smithi016 bash[75004]: cluster 2023-09-23T19:53:50.488832+0000 mgr.y (mgr.25325) 1396 : cluster 0 pgmap v1404: 89 pgs: 89 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 946 op/s 2023-09-23T19:53:52.405 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:52.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:52.411 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:52.512 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:52.535 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:52.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:52.537 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:52.638 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:52.673 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:52.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:52.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:52.776 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:52.809 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:52.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:52.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:52.915 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:52.937 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:52.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:52.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:53.041 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:53.073 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:53.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:53.078 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:53.179 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:53.203 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:53.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:53.210 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:53.310 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:53.333 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:53.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:53.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:53.436 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:53.473 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:53.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:53.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:53.580 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:53.606 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:53.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:53.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:53.711 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:53.734 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:53.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:53.738 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:53.839 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:53.889 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:53.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:53.892 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:53.992 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:54.025 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:54.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:54.030 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:54.131 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:54.155 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:54.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:54.158 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:54.206 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:54 smithi092 bash[58855]: cluster 2023-09-23T19:53:52.489357+0000 mgr.y (mgr.25325) 1397 : cluster 0 pgmap v1405: 89 pgs: 89 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 661 op/s 2023-09-23T19:53:54.259 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:54.285 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:54.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:54.288 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:54.388 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:54.417 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:54.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:54.420 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:54.521 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:54.548 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:54.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:54.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:54.558 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:54 smithi016 bash[77995]: cluster 2023-09-23T19:53:52.489357+0000 mgr.y (mgr.25325) 1397 : cluster 0 pgmap v1405: 89 pgs: 89 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 661 op/s 2023-09-23T19:53:54.558 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:54 smithi016 bash[75004]: cluster 2023-09-23T19:53:52.489357+0000 mgr.y (mgr.25325) 1397 : cluster 0 pgmap v1405: 89 pgs: 89 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 661 op/s 2023-09-23T19:53:54.653 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:54.680 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:54.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:54.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:54.783 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:54.807 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:54.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:54.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:54.910 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:54.934 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:54.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:54.936 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:55.037 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:55.065 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:55.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:55.067 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:55.167 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:55.195 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:55.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:55.197 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:55.298 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:55.323 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:55.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:55.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:55.428 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:55.456 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:55.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:55.460 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:55.561 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:55.586 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:55.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:55.594 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:55.596 INFO:teuthology.orchestra.run.smithi092.stdout:ERROR: (22) Invalid argument 2023-09-23T19:53:55.596 INFO:teuthology.orchestra.run.smithi092.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:53:55.695 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:55.719 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:55.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:55.722 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:55.822 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:55.847 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:55.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:55.850 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:55.851 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:53:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:53:55] "GET /metrics HTTP/1.1" 200 36306 "" "Prometheus/2.33.4" 2023-09-23T19:53:55.951 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:55.999 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:56.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:56.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:56.105 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:56.150 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:56.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:56.152 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:56.209 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:56 smithi016 bash[75004]: cluster 2023-09-23T19:53:54.489818+0000 mgr.y (mgr.25325) 1398 : cluster 0 pgmap v1406: 89 pgs: 89 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 642 op/s 2023-09-23T19:53:56.253 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:56.277 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:56.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:56.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:56.380 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:56.410 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:56.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:56.414 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:56.515 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:56.522 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:56 smithi092 bash[58855]: cluster 2023-09-23T19:53:54.489818+0000 mgr.y (mgr.25325) 1398 : cluster 0 pgmap v1406: 89 pgs: 89 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 642 op/s 2023-09-23T19:53:56.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:56 smithi016 bash[77995]: cluster 2023-09-23T19:53:54.489818+0000 mgr.y (mgr.25325) 1398 : cluster 0 pgmap v1406: 89 pgs: 89 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 642 op/s 2023-09-23T19:53:56.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:56.554 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:56.555 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:56.656 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:56.692 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:56.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:56.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:56.797 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:56.824 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:56.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:56.828 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:56.842 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T19:53:56.929 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:56.957 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:56.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:56.961 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:57.061 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:57.084 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:57.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:57.090 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:57.191 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:57.224 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:57.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:57.227 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:57.327 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:57.365 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:57.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:57.371 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:57.471 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:57.498 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:57.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:57.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:57.602 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:57.638 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:57.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:57.641 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:57.742 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:57.766 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:57.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:57.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:57.874 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:57.901 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:57.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:57.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:58.004 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:58.028 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:58.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:58.031 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:58.131 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:58.163 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:58.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:58.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:58.269 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:58.301 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:58.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:58.308 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:58.383 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:58 smithi016 bash[77995]: cluster 2023-09-23T19:53:56.490530+0000 mgr.y (mgr.25325) 1399 : cluster 0 pgmap v1407: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 857 op/s 2023-09-23T19:53:58.383 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:58 smithi016 bash[75004]: cluster 2023-09-23T19:53:56.490530+0000 mgr.y (mgr.25325) 1399 : cluster 0 pgmap v1407: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 857 op/s 2023-09-23T19:53:58.409 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:58.434 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:58.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:58.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:58.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:58 smithi092 bash[58855]: cluster 2023-09-23T19:53:56.490530+0000 mgr.y (mgr.25325) 1399 : cluster 0 pgmap v1407: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 857 op/s 2023-09-23T19:53:58.539 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:58.565 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:58.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:58.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:58.670 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:58.698 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:58.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:58.704 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:58.805 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:58.832 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:58.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:58.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:58.940 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:58.978 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:58.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:58.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:59.083 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:59.107 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:59.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:59.112 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:59.213 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:59.263 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:59.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:59.266 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:59.367 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:59.393 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:59.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:59.397 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:59.498 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:59.503 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:53:59.508 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:53:59 smithi016 bash[77995]: audit 2023-09-23T19:53:58.678090+0000 mon.a (mon.0) 1713 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:53:59.508 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:53:59 smithi016 bash[75004]: audit 2023-09-23T19:53:58.678090+0000 mon.a (mon.0) 1713 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:53:59.537 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:59.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:53:59 smithi092 bash[58855]: audit 2023-09-23T19:53:58.678090+0000 mon.a (mon.0) 1713 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:53:59.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:59.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:59.576 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:53:59.635 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:53:59.642 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:59.685 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:59.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:59.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:59.696 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:53:59.750 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:53:59.783 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:53:59.793 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:59.818 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:59.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:59.820 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:59.831 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:53:59.864 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:53:59.875 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:53:59.876 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:53:59.876 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:53:59.876 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:53:59.876 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:53:59.876 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:53:59.876 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:53:59.876 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:53:59.921 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:59.948 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:59.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:53:59.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:00.053 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:00.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:00.090 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:00.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:00.191 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:00.220 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:00.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:00.224 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:00.244 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:00 smithi016 bash[77995]: cluster 2023-09-23T19:53:58.491044+0000 mgr.y (mgr.25325) 1400 : cluster 0 pgmap v1408: 89 pgs: 89 active+clean; 5.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2023-09-23T19:54:00.244 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:00 smithi016 bash[77995]: audit 2023-09-23T19:53:59.450101+0000 mon.c (mon.2) 218 : audit 0 from='client.? 172.21.15.16:0/808924153' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:54:00.244 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:00 smithi016 bash[77995]: audit 2023-09-23T19:53:59.453555+0000 mon.c (mon.2) 219 : audit 1 from='client.? 172.21.15.16:0/2263267336' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T19:54:00.244 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:00 smithi016 bash[77995]: audit 2023-09-23T19:53:59.454039+0000 mon.a (mon.0) 1714 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T19:54:00.245 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:00 smithi016 bash[77995]: audit 2023-09-23T19:53:59.881682+0000 mon.a (mon.0) 1715 : audit 1 from='client.? 172.21.15.16:0/978619524' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:54:00.245 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:00 smithi016 bash[75004]: cluster 2023-09-23T19:53:58.491044+0000 mgr.y (mgr.25325) 1400 : cluster 0 pgmap v1408: 89 pgs: 89 active+clean; 5.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2023-09-23T19:54:00.245 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:00 smithi016 bash[75004]: audit 2023-09-23T19:53:59.450101+0000 mon.c (mon.2) 218 : audit 0 from='client.? 172.21.15.16:0/808924153' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:54:00.245 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:00 smithi016 bash[75004]: audit 2023-09-23T19:53:59.453555+0000 mon.c (mon.2) 219 : audit 1 from='client.? 172.21.15.16:0/2263267336' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T19:54:00.245 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:00 smithi016 bash[75004]: audit 2023-09-23T19:53:59.454039+0000 mon.a (mon.0) 1714 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T19:54:00.245 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:00 smithi016 bash[75004]: audit 2023-09-23T19:53:59.881682+0000 mon.a (mon.0) 1715 : audit 1 from='client.? 172.21.15.16:0/978619524' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:54:00.248 INFO:teuthology.orchestra.run.smithi016.stderr:set osd.4 primary-affinity to 1 (8655362) 2023-09-23T19:54:00.248 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is set 2023-09-23T19:54:00.325 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:00.334 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:00 smithi092 bash[58855]: cluster 2023-09-23T19:53:58.491044+0000 mgr.y (mgr.25325) 1400 : cluster 0 pgmap v1408: 89 pgs: 89 active+clean; 5.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2023-09-23T19:54:00.334 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:00 smithi092 bash[58855]: audit 2023-09-23T19:53:59.450101+0000 mon.c (mon.2) 218 : audit 0 from='client.? 172.21.15.16:0/808924153' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:54:00.334 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:00 smithi092 bash[58855]: audit 2023-09-23T19:53:59.453555+0000 mon.c (mon.2) 219 : audit 1 from='client.? 172.21.15.16:0/2263267336' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T19:54:00.334 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:00 smithi092 bash[58855]: audit 2023-09-23T19:53:59.454039+0000 mon.a (mon.0) 1714 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T19:54:00.334 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:00 smithi092 bash[58855]: audit 2023-09-23T19:53:59.881682+0000 mon.a (mon.0) 1715 : audit 1 from='client.? 172.21.15.16:0/978619524' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:54:00.352 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:00.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:00.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:00.462 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:00.486 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:00.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:00.490 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:00.591 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:00.612 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:00.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:00.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:00.715 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:00.741 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:00.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:00.743 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:54:00.844 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:00.875 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:00.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:00.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:00.982 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:01.031 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:01.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:01.036 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:01.136 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:01.164 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:01.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:01.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:01.267 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:01.298 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:01.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:01.305 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:01.405 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:01.415 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:01 smithi016 bash[75004]: cluster 2023-09-23T19:54:00.227341+0000 mon.a (mon.0) 1716 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:54:01.415 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:01 smithi016 bash[75004]: audit 2023-09-23T19:54:00.233299+0000 mon.a (mon.0) 1717 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-09-23T19:54:01.415 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:01 smithi016 bash[75004]: audit 2023-09-23T19:54:00.233377+0000 mon.a (mon.0) 1718 : audit 1 from='client.? 172.21.15.16:0/978619524' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:54:01.416 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:01 smithi016 bash[75004]: cluster 2023-09-23T19:54:00.233428+0000 mon.a (mon.0) 1719 : cluster 0 osdmap e244: 8 total, 8 up, 8 in 2023-09-23T19:54:01.416 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:01 smithi016 bash[75004]: cluster 2023-09-23T19:54:00.237095+0000 osd.4 (osd.4) 187 : cluster 3 failed to encode map e244 with expected crc 2023-09-23T19:54:01.416 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:01 smithi016 bash[75004]: cluster 2023-09-23T19:54:00.248144+0000 osd.5 (osd.5) 173 : cluster 3 failed to encode map e244 with expected crc 2023-09-23T19:54:01.416 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:01 smithi016 bash[75004]: cluster 2023-09-23T19:54:00.248724+0000 osd.6 (osd.6) 180 : cluster 3 failed to encode map e244 with expected crc 2023-09-23T19:54:01.416 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:01 smithi016 bash[75004]: cluster 2023-09-23T19:54:00.258744+0000 osd.7 (osd.7) 161 : cluster 3 failed to encode map e244 with expected crc 2023-09-23T19:54:01.416 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:01 smithi016 bash[75004]: audit 2023-09-23T19:54:00.740020+0000 mon.a (mon.0) 1720 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:54:01.416 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:01 smithi016 bash[77995]: cluster 2023-09-23T19:54:00.227341+0000 mon.a (mon.0) 1716 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:54:01.416 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:01 smithi016 bash[77995]: audit 2023-09-23T19:54:00.233299+0000 mon.a (mon.0) 1717 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-09-23T19:54:01.416 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:01 smithi016 bash[77995]: audit 2023-09-23T19:54:00.233377+0000 mon.a (mon.0) 1718 : audit 1 from='client.? 172.21.15.16:0/978619524' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:54:01.417 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:01 smithi016 bash[77995]: cluster 2023-09-23T19:54:00.233428+0000 mon.a (mon.0) 1719 : cluster 0 osdmap e244: 8 total, 8 up, 8 in 2023-09-23T19:54:01.417 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:01 smithi016 bash[77995]: cluster 2023-09-23T19:54:00.237095+0000 osd.4 (osd.4) 187 : cluster 3 failed to encode map e244 with expected crc 2023-09-23T19:54:01.417 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:01 smithi016 bash[77995]: cluster 2023-09-23T19:54:00.248144+0000 osd.5 (osd.5) 173 : cluster 3 failed to encode map e244 with expected crc 2023-09-23T19:54:01.417 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:01 smithi016 bash[77995]: cluster 2023-09-23T19:54:00.248724+0000 osd.6 (osd.6) 180 : cluster 3 failed to encode map e244 with expected crc 2023-09-23T19:54:01.417 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:01 smithi016 bash[77995]: cluster 2023-09-23T19:54:00.258744+0000 osd.7 (osd.7) 161 : cluster 3 failed to encode map e244 with expected crc 2023-09-23T19:54:01.417 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:01 smithi016 bash[77995]: audit 2023-09-23T19:54:00.740020+0000 mon.a (mon.0) 1720 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:54:01.436 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:01.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:01.441 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:01.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:01 smithi092 bash[58855]: cluster 2023-09-23T19:54:00.227341+0000 mon.a (mon.0) 1716 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:54:01.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:01 smithi092 bash[58855]: audit 2023-09-23T19:54:00.233299+0000 mon.a (mon.0) 1717 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-09-23T19:54:01.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:01 smithi092 bash[58855]: audit 2023-09-23T19:54:00.233377+0000 mon.a (mon.0) 1718 : audit 1 from='client.? 172.21.15.16:0/978619524' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:54:01.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:01 smithi092 bash[58855]: cluster 2023-09-23T19:54:00.233428+0000 mon.a (mon.0) 1719 : cluster 0 osdmap e244: 8 total, 8 up, 8 in 2023-09-23T19:54:01.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:01 smithi092 bash[58855]: cluster 2023-09-23T19:54:00.237095+0000 osd.4 (osd.4) 187 : cluster 3 failed to encode map e244 with expected crc 2023-09-23T19:54:01.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:01 smithi092 bash[58855]: cluster 2023-09-23T19:54:00.248144+0000 osd.5 (osd.5) 173 : cluster 3 failed to encode map e244 with expected crc 2023-09-23T19:54:01.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:01 smithi092 bash[58855]: cluster 2023-09-23T19:54:00.248724+0000 osd.6 (osd.6) 180 : cluster 3 failed to encode map e244 with expected crc 2023-09-23T19:54:01.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:01 smithi092 bash[58855]: cluster 2023-09-23T19:54:00.258744+0000 osd.7 (osd.7) 161 : cluster 3 failed to encode map e244 with expected crc 2023-09-23T19:54:01.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:01 smithi092 bash[58855]: audit 2023-09-23T19:54:00.740020+0000 mon.a (mon.0) 1720 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:54:01.542 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:01.581 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:01.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:01.586 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:01.671 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:54:01.687 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:01.724 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:01.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:01.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:01.829 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:01.860 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:01.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:01.865 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:01.966 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:01.996 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:02.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:02.001 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:02.102 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:02.142 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:02.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:02.146 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:02.247 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:02.281 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:02.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:02.285 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:02.386 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:02.395 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:02 smithi092 bash[58855]: cluster 2023-09-23T19:54:00.491465+0000 mgr.y (mgr.25325) 1401 : cluster 0 pgmap v1410: 89 pgs: 89 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 737 op/s 2023-09-23T19:54:02.424 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:02.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:02.428 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:02.528 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:02.541 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:02 smithi016 bash[75004]: cluster 2023-09-23T19:54:00.491465+0000 mgr.y (mgr.25325) 1401 : cluster 0 pgmap v1410: 89 pgs: 89 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 737 op/s 2023-09-23T19:54:02.542 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:02 smithi016 bash[77995]: cluster 2023-09-23T19:54:00.491465+0000 mgr.y (mgr.25325) 1401 : cluster 0 pgmap v1410: 89 pgs: 89 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 737 op/s 2023-09-23T19:54:02.558 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:02.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:02.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:02.663 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:02.714 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:02.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:02.719 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:02.819 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:02.846 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:02.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:02.851 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:02.952 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:02.980 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:02.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:02.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:54:03.086 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:03.122 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:03.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:03.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:03.224 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:03.267 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:03.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:03.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:03.371 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:03.393 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:03.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:03.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:03.499 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:03.540 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:03.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:03.545 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:03.646 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:03.681 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:03.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:03.684 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:03.785 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:03.823 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:03.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:03.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:03.879 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:54:03.928 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:03.958 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:03.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:03.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:04.062 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:04.106 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:04.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:04.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:04.212 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:04.250 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:04.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:04.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:04.356 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:04.380 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:04.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:04.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:04.484 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:04.511 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:04.516 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:04 smithi016 bash[75004]: cluster 2023-09-23T19:54:02.491976+0000 mgr.y (mgr.25325) 1402 : cluster 0 pgmap v1411: 89 pgs: 89 active+clean; 4.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2023-09-23T19:54:04.516 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:04 smithi016 bash[77995]: cluster 2023-09-23T19:54:02.491976+0000 mgr.y (mgr.25325) 1402 : cluster 0 pgmap v1411: 89 pgs: 89 active+clean; 4.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2023-09-23T19:54:04.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:04.517 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:04.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:04 smithi092 bash[58855]: cluster 2023-09-23T19:54:02.491976+0000 mgr.y (mgr.25325) 1402 : cluster 0 pgmap v1411: 89 pgs: 89 active+clean; 4.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2023-09-23T19:54:04.618 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:04.648 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:04.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:04.652 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:04.753 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:04.777 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:04.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:04.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:54:04.882 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:04.902 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:04.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:04.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:05.007 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:05.039 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:05.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:05.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:05.146 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:05.171 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:05.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:05.174 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:05.275 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:05.298 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:05.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:05.300 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:05.397 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T19:54:05.401 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:05.475 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:05.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:05.483 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:54:05.583 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:05.612 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:05.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:05.614 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:05.715 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:05.737 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:54:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:54:05] "GET /metrics HTTP/1.1" 200 36306 "" "Prometheus/2.33.4" 2023-09-23T19:54:05.754 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:05.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:05.756 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:05.857 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:05.889 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:05.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:05.891 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:05.992 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:06.016 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:06.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:06.019 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:06.120 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:06.162 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:06.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:06.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:06.265 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:06.303 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:06.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:06.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:06.412 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:06.424 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:06 smithi092 bash[58855]: cluster 2023-09-23T19:54:04.495190+0000 mgr.y (mgr.25325) 1403 : cluster 0 pgmap v1412: 89 pgs: 89 active+clean; 4.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2023-09-23T19:54:06.446 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:06.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:06.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:54:06.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:06 smithi016 bash[75004]: cluster 2023-09-23T19:54:04.495190+0000 mgr.y (mgr.25325) 1403 : cluster 0 pgmap v1412: 89 pgs: 89 active+clean; 4.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2023-09-23T19:54:06.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:06 smithi016 bash[77995]: cluster 2023-09-23T19:54:04.495190+0000 mgr.y (mgr.25325) 1403 : cluster 0 pgmap v1412: 89 pgs: 89 active+clean; 4.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2023-09-23T19:54:06.550 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:06.598 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:06.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:06.611 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:06.711 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:06.741 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:06.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:06.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:06.822 INFO:teuthology.orchestra.run.smithi092.stdout:{ 2023-09-23T19:54:06.822 INFO:teuthology.orchestra.run.smithi092.stdout: "ops": [], 2023-09-23T19:54:06.822 INFO:teuthology.orchestra.run.smithi092.stdout: "complaint_time": 30, 2023-09-23T19:54:06.822 INFO:teuthology.orchestra.run.smithi092.stdout: "num_blocked_ops": 0 2023-09-23T19:54:06.822 INFO:teuthology.orchestra.run.smithi092.stdout:} 2023-09-23T19:54:06.846 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:06.868 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:06.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:06.873 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:06.974 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:07.017 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:07.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:07.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:07.124 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:07.167 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:07.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:07.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:07.272 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:07.299 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:07.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:07.304 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:07.405 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:07.438 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:07.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:07.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:07.543 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:07.583 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:07.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:07.586 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:07.686 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:07.716 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:07.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:07.722 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:07.746 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:54:07.801 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T19:54:07.823 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:07.875 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:07.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:07.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:07.982 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:08.011 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:08.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:08.018 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:08.119 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:08.142 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:08.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:08.145 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:08.247 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:08.256 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:08 smithi092 bash[58855]: cluster 2023-09-23T19:54:06.496099+0000 mgr.y (mgr.25325) 1404 : cluster 0 pgmap v1413: 89 pgs: 89 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 718 op/s 2023-09-23T19:54:08.256 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:08 smithi092 bash[58855]: audit 2023-09-23T19:54:07.045823+0000 mon.a (mon.0) 1721 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:54:08.256 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:08 smithi092 bash[58855]: audit 2023-09-23T19:54:07.052989+0000 mon.a (mon.0) 1722 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:54:08.256 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:08 smithi092 bash[58855]: audit 2023-09-23T19:54:07.206644+0000 mon.a (mon.0) 1723 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:54:08.256 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:08 smithi092 bash[58855]: audit 2023-09-23T19:54:07.216789+0000 mon.a (mon.0) 1724 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:54:08.257 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:08 smithi092 bash[58855]: audit 2023-09-23T19:54:07.813336+0000 mon.a (mon.0) 1725 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:54:08.257 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:08 smithi092 bash[58855]: audit 2023-09-23T19:54:07.814721+0000 mon.a (mon.0) 1726 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:54:08.257 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:08 smithi092 bash[58855]: audit 2023-09-23T19:54:07.823821+0000 mon.a (mon.0) 1727 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:54:08.274 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:08.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:08.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:08.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:08 smithi016 bash[75004]: cluster 2023-09-23T19:54:06.496099+0000 mgr.y (mgr.25325) 1404 : cluster 0 pgmap v1413: 89 pgs: 89 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 718 op/s 2023-09-23T19:54:08.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:08 smithi016 bash[75004]: audit 2023-09-23T19:54:07.045823+0000 mon.a (mon.0) 1721 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:54:08.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:08 smithi016 bash[75004]: audit 2023-09-23T19:54:07.052989+0000 mon.a (mon.0) 1722 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:54:08.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:08 smithi016 bash[75004]: audit 2023-09-23T19:54:07.206644+0000 mon.a (mon.0) 1723 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:54:08.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:08 smithi016 bash[75004]: audit 2023-09-23T19:54:07.216789+0000 mon.a (mon.0) 1724 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:54:08.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:08 smithi016 bash[75004]: audit 2023-09-23T19:54:07.813336+0000 mon.a (mon.0) 1725 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:54:08.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:08 smithi016 bash[75004]: audit 2023-09-23T19:54:07.814721+0000 mon.a (mon.0) 1726 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:54:08.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:08 smithi016 bash[75004]: audit 2023-09-23T19:54:07.823821+0000 mon.a (mon.0) 1727 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:54:08.288 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:08 smithi016 bash[77995]: cluster 2023-09-23T19:54:06.496099+0000 mgr.y (mgr.25325) 1404 : cluster 0 pgmap v1413: 89 pgs: 89 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 718 op/s 2023-09-23T19:54:08.288 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:08 smithi016 bash[77995]: audit 2023-09-23T19:54:07.045823+0000 mon.a (mon.0) 1721 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:54:08.288 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:08 smithi016 bash[77995]: audit 2023-09-23T19:54:07.052989+0000 mon.a (mon.0) 1722 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:54:08.288 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:08 smithi016 bash[77995]: audit 2023-09-23T19:54:07.206644+0000 mon.a (mon.0) 1723 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:54:08.288 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:08 smithi016 bash[77995]: audit 2023-09-23T19:54:07.216789+0000 mon.a (mon.0) 1724 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:54:08.288 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:08 smithi016 bash[77995]: audit 2023-09-23T19:54:07.813336+0000 mon.a (mon.0) 1725 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:54:08.288 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:08 smithi016 bash[77995]: audit 2023-09-23T19:54:07.814721+0000 mon.a (mon.0) 1726 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:54:08.288 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:08 smithi016 bash[77995]: audit 2023-09-23T19:54:07.823821+0000 mon.a (mon.0) 1727 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:54:08.376 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:08.403 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:08.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:08.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:54:08.410 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-09-23T19:54:08.410 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T19:54:08.510 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:08.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:08.547 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:08.547 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:08.649 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:08.679 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:08.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:08.685 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:54:08.786 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:08.814 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:08.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:08.818 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:54:08.919 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:08.947 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:08.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:08.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:09.051 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:09.078 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:09.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:09.083 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:09.184 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:09.217 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:09.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:09.221 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:09.321 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:09.354 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:09.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:09.361 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:09.462 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:09.700 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:09.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:09.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:09.783 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:54:09.805 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:09.840 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:09.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:09.847 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:09.948 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:09.979 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:09.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:09.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:10.085 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:10.095 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:10 smithi016 bash[75004]: cluster 2023-09-23T19:54:08.496692+0000 mgr.y (mgr.25325) 1405 : cluster 0 pgmap v1414: 89 pgs: 89 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2023-09-23T19:54:10.095 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:10 smithi016 bash[77995]: cluster 2023-09-23T19:54:08.496692+0000 mgr.y (mgr.25325) 1405 : cluster 0 pgmap v1414: 89 pgs: 89 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2023-09-23T19:54:10.118 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:10.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:10.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:10.221 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:10.255 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:10.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:10.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:10.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:10 smithi092 bash[58855]: cluster 2023-09-23T19:54:08.496692+0000 mgr.y (mgr.25325) 1405 : cluster 0 pgmap v1414: 89 pgs: 89 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2023-09-23T19:54:10.357 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:10.381 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:10.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:10.384 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:10.485 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:10.516 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:10.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:10.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:10.621 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:10.656 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:10.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:10.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:54:10.764 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:10.800 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:10.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:10.803 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:10.904 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:10.940 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:10.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:10.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:11.044 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:11.070 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:11.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:11.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:11.176 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:11.201 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:11.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:11.206 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:11.307 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:11.338 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:11.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:11.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:11.442 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:11.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:11.478 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:11.479 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:11.579 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:11.603 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:11.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:11.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:11.710 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:11.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:11.747 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:11.747 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:11.848 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:11.881 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:11.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:11.885 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:11.986 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:12.026 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:12.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:12.030 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:12.130 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:12.168 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:12.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:12.172 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:54:12.273 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:12.282 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:12 smithi016 bash[75004]: cluster 2023-09-23T19:54:10.497300+0000 mgr.y (mgr.25325) 1406 : cluster 0 pgmap v1415: 89 pgs: 89 active+clean; 4.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 668 op/s 2023-09-23T19:54:12.282 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:12 smithi016 bash[77995]: cluster 2023-09-23T19:54:10.497300+0000 mgr.y (mgr.25325) 1406 : cluster 0 pgmap v1415: 89 pgs: 89 active+clean; 4.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 668 op/s 2023-09-23T19:54:12.394 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:12.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:12.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:54:12.498 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:12.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:12 smithi092 bash[58855]: cluster 2023-09-23T19:54:10.497300+0000 mgr.y (mgr.25325) 1406 : cluster 0 pgmap v1415: 89 pgs: 89 active+clean; 4.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 668 op/s 2023-09-23T19:54:12.539 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:12.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:12.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:12.645 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:12.654 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:54:12.695 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:12.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:12.698 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:12.799 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:12.835 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:12.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:12.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:12.939 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:12.976 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:12.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:12.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:54:13.081 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:13.108 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:13.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:13.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:13.214 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:13.255 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:13.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:13.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:13.361 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:13.393 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:13.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:13.398 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:13.499 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:13.526 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:13.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:13.530 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:13.630 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:13.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:13.662 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:13.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:13.763 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:13.789 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:13.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:13.792 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:13.893 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:13.941 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:13.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:13.945 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:54:14.046 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:14.078 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:14.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:14.081 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:14.182 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:14.192 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:14 smithi092 bash[58855]: cluster 2023-09-23T19:54:12.497793+0000 mgr.y (mgr.25325) 1407 : cluster 0 pgmap v1416: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 624 op/s 2023-09-23T19:54:14.192 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:14 smithi092 bash[58855]: audit 2023-09-23T19:54:13.685513+0000 mon.a (mon.0) 1728 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:54:14.192 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:14 smithi092 bash[58855]: audit 2023-09-23T19:54:13.686758+0000 mon.a (mon.0) 1729 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:54:14.212 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:14.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:14.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:54:14.316 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:14.324 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:14 smithi016 bash[77995]: cluster 2023-09-23T19:54:12.497793+0000 mgr.y (mgr.25325) 1407 : cluster 0 pgmap v1416: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 624 op/s 2023-09-23T19:54:14.324 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:14 smithi016 bash[77995]: audit 2023-09-23T19:54:13.685513+0000 mon.a (mon.0) 1728 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:54:14.325 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:14 smithi016 bash[77995]: audit 2023-09-23T19:54:13.686758+0000 mon.a (mon.0) 1729 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:54:14.325 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:14 smithi016 bash[75004]: cluster 2023-09-23T19:54:12.497793+0000 mgr.y (mgr.25325) 1407 : cluster 0 pgmap v1416: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 624 op/s 2023-09-23T19:54:14.325 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:14 smithi016 bash[75004]: audit 2023-09-23T19:54:13.685513+0000 mon.a (mon.0) 1728 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:54:14.325 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:14 smithi016 bash[75004]: audit 2023-09-23T19:54:13.686758+0000 mon.a (mon.0) 1729 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:54:14.339 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:14.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:14.342 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:14.443 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:14.484 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:14.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:14.487 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:14.587 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:14.618 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:14.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:14.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:14.721 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:14.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:14.770 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:14.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:14.871 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:14.897 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:14.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:14.898 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:54:14.999 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:15.039 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:15.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:15.045 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:15.146 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:15.176 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:15.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:15.180 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:15.280 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:15.313 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:15.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:15.314 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:15.414 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:15.439 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:15.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:15.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:15.543 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:15.570 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:15.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:15.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:15.673 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:15.708 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:15.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:15.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:54:15.815 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:15.876 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:54:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:54:15] "GET /metrics HTTP/1.1" 200 36306 "" "Prometheus/2.33.4" 2023-09-23T19:54:15.925 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:15.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:15.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:16.031 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:16.048 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:54:16.082 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:16.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:16.085 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:16.186 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:16.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:16.223 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:16.223 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:16.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:16 smithi016 bash[77995]: cluster 2023-09-23T19:54:14.498279+0000 mgr.y (mgr.25325) 1408 : cluster 0 pgmap v1417: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 588 op/s 2023-09-23T19:54:16.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:16 smithi016 bash[75004]: cluster 2023-09-23T19:54:14.498279+0000 mgr.y (mgr.25325) 1408 : cluster 0 pgmap v1417: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 588 op/s 2023-09-23T19:54:16.323 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:16.349 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:16 smithi092 bash[58855]: cluster 2023-09-23T19:54:14.498279+0000 mgr.y (mgr.25325) 1408 : cluster 0 pgmap v1417: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 588 op/s 2023-09-23T19:54:16.351 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:16.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:16.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:54:16.455 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:16.480 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:16.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:16.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:16.592 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:16.622 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:16.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:16.628 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:54:16.729 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:16.756 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:16.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:16.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:16.863 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:16.886 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:16.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:16.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:16.996 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:17.031 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:17.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:17.038 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:17.139 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:17.174 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:17.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:17.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:17.280 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:17.311 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:17.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:17.313 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:54:17.414 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:17.441 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:17.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:17.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:17.545 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:17.570 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:17.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:17.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:17.673 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:17.701 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:17.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:17.706 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:17.784 INFO:teuthology.orchestra.run.smithi092.stdout:{ 2023-09-23T19:54:17.784 INFO:teuthology.orchestra.run.smithi092.stdout: "size": 20, 2023-09-23T19:54:17.784 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 600, 2023-09-23T19:54:17.784 INFO:teuthology.orchestra.run.smithi092.stdout: "ops": [ 2023-09-23T19:54:17.784 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.784 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.36243.0:201246 11.f 11:f50c31f8:::benchmark_data_smithi016_160148_object94043:head [delete] snapc 0=[] ondisk+write+known_if_redirected e201)", 2023-09-23T19:54:17.784 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:45:27.965669+0000", 2023-09-23T19:54:17.785 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 529.81459715100004, 2023-09-23T19:54:17.785 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.15238912700000001, 2023-09-23T19:54:17.785 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:54:17.785 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:17.785 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:54:17.785 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.36243", 2023-09-23T19:54:17.785 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/1280038109", 2023-09-23T19:54:17.785 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 201246 2023-09-23T19:54:17.785 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.785 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:54:17.786 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.786 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:54:17.786 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:45:27.965669+0000", 2023-09-23T19:54:17.786 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:54:17.786 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.786 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.786 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:54:17.786 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:45:27.965669+0000", 2023-09-23T19:54:17.786 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999995 2023-09-23T19:54:17.786 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.787 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.787 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:54:17.787 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:45:27.965669+0000", 2023-09-23T19:54:17.787 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.1479999999999997e-06 2023-09-23T19:54:17.787 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.787 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.787 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:54:17.787 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:45:27.965674+0000", 2023-09-23T19:54:17.787 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.3292000000000001e-05 2023-09-23T19:54:17.787 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.788 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.788 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:54:17.788 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:45:27.965687+0000", 2023-09-23T19:54:17.788 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.3110000000000002e-06 2023-09-23T19:54:17.788 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.788 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.788 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:54:17.788 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:45:27.965693+0000", 2023-09-23T19:54:17.788 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0056683849999999997 2023-09-23T19:54:17.788 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.789 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.789 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:54:17.789 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:45:27.971362+0000", 2023-09-23T19:54:17.789 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000119543 2023-09-23T19:54:17.789 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.789 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.789 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:54:17.789 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:45:27.971481+0000", 2023-09-23T19:54:17.789 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00014001700000000001 2023-09-23T19:54:17.789 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.790 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.790 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,2", 2023-09-23T19:54:17.790 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:45:27.971621+0000", 2023-09-23T19:54:17.790 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0042345580000000002 2023-09-23T19:54:17.790 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.790 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.790 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:54:17.790 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:45:27.975856+0000", 2023-09-23T19:54:17.790 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.125740931 2023-09-23T19:54:17.790 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.791 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.791 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:17.791 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:45:28.101597+0000", 2023-09-23T19:54:17.791 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.016350186999999999 2023-09-23T19:54:17.791 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.791 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.791 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:17.791 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:45:28.117947+0000", 2023-09-23T19:54:17.791 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.2784000000000001e-05 2023-09-23T19:54:17.791 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.791 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.792 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:54:17.792 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:45:28.118010+0000", 2023-09-23T19:54:17.792 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.8683000000000001e-05 2023-09-23T19:54:17.792 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.792 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.792 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:54:17.792 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:45:28.118058+0000", 2023-09-23T19:54:17.792 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.15238912700000001 2023-09-23T19:54:17.792 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:54:17.792 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:54:17.793 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:54:17.793 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.793 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.793 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.36480.0:11088 12.d 12:bdb08a7d:::benchmark_data_smithi016_181952_object11087:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e211)", 2023-09-23T19:54:17.793 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:46:38.754185+0000", 2023-09-23T19:54:17.793 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 459.02608138900001, 2023-09-23T19:54:17.793 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.30593605699999998, 2023-09-23T19:54:17.793 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:54:17.793 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:17.793 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:54:17.793 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.36480", 2023-09-23T19:54:17.793 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/1035295215", 2023-09-23T19:54:17.794 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 11088 2023-09-23T19:54:17.794 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.794 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:54:17.794 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.794 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:54:17.794 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:38.754185+0000", 2023-09-23T19:54:17.794 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:54:17.794 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.794 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.794 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:54:17.794 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:38.754185+0000", 2023-09-23T19:54:17.795 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T19:54:17.795 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.795 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.795 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:54:17.795 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:38.754184+0000", 2023-09-23T19:54:17.795 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.7117000000000003e-05 2023-09-23T19:54:17.795 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.795 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.795 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:54:17.795 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:38.754231+0000", 2023-09-23T19:54:17.795 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.2058e-05 2023-09-23T19:54:17.796 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.796 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.796 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:54:17.796 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:38.754243+0000", 2023-09-23T19:54:17.796 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.0659999999999998e-06 2023-09-23T19:54:17.796 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.796 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.796 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:54:17.796 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:38.754250+0000", 2023-09-23T19:54:17.796 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.9290000000000003e-05 2023-09-23T19:54:17.796 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.797 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.797 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:54:17.797 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:38.754309+0000", 2023-09-23T19:54:17.797 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00015118799999999999 2023-09-23T19:54:17.797 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.797 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.797 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:54:17.797 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:38.754460+0000", 2023-09-23T19:54:17.797 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00018447 2023-09-23T19:54:17.797 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.797 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.797 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,2", 2023-09-23T19:54:17.797 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:38.754645+0000", 2023-09-23T19:54:17.798 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.302783845 2023-09-23T19:54:17.798 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.798 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.798 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:17.798 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:39.057429+0000", 2023-09-23T19:54:17.798 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.6026000000000002e-05 2023-09-23T19:54:17.798 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.798 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.798 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:17.798 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:39.057485+0000", 2023-09-23T19:54:17.798 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0024437439999999999 2023-09-23T19:54:17.799 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.799 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.799 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:54:17.799 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:39.059928+0000", 2023-09-23T19:54:17.799 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00010819699999999999 2023-09-23T19:54:17.799 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.799 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.799 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:54:17.799 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:39.060037+0000", 2023-09-23T19:54:17.799 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.4534000000000001e-05 2023-09-23T19:54:17.799 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.799 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.800 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:54:17.800 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:39.060121+0000", 2023-09-23T19:54:17.800 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.30593605699999998 2023-09-23T19:54:17.800 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:54:17.800 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:54:17.800 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:54:17.800 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.800 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.800 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_repop(client.36480.0:11082 12.7 e211/207 12:edf527c8:::benchmark_data_smithi016_181952_object11081:head v 211'673, mlcod=211'671)", 2023-09-23T19:54:17.800 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:46:38.754648+0000", 2023-09-23T19:54:17.800 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 459.02561823399998, 2023-09-23T19:54:17.800 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.30262623, 2023-09-23T19:54:17.801 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:54:17.801 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:17.801 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:54:17.801 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.801 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:54:17.801 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:38.754648+0000", 2023-09-23T19:54:17.801 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:54:17.801 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.801 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.803 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:54:17.803 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:38.754648+0000", 2023-09-23T19:54:17.803 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.999999 2023-09-23T19:54:17.803 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.803 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.803 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:54:17.804 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:38.754647+0000", 2023-09-23T19:54:17.804 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000150703 2023-09-23T19:54:17.804 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.804 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.804 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:54:17.804 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:38.754798+0000", 2023-09-23T19:54:17.804 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.30083174200000001 2023-09-23T19:54:17.804 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.804 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.804 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:54:17.805 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:39.055630+0000", 2023-09-23T19:54:17.805 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.7614000000000001e-05 2023-09-23T19:54:17.805 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.805 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.805 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:54:17.805 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:39.055647+0000", 2023-09-23T19:54:17.805 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.6595999999999996e-05 2023-09-23T19:54:17.805 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.805 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.805 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:54:17.805 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:39.055724+0000", 2023-09-23T19:54:17.806 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.5590999999999999e-05 2023-09-23T19:54:17.806 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.806 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.806 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:54:17.806 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:39.055750+0000", 2023-09-23T19:54:17.806 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0014905039999999999 2023-09-23T19:54:17.806 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.806 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.806 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:54:17.806 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:39.057240+0000", 2023-09-23T19:54:17.807 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.4326000000000003e-05 2023-09-23T19:54:17.807 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.807 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.807 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:54:17.807 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:39.057274+0000", 2023-09-23T19:54:17.807 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.30262623 2023-09-23T19:54:17.807 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:54:17.807 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:54:17.807 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:54:17.807 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.807 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.808 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_repop(client.36480.0:11084 12.3 e211/207 12:c2885d86:::benchmark_data_smithi016_181952_object11083:head v 211'639, mlcod=211'637)", 2023-09-23T19:54:17.808 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:46:38.754678+0000", 2023-09-23T19:54:17.808 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 459.02558849799999, 2023-09-23T19:54:17.808 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.302533092, 2023-09-23T19:54:17.808 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:54:17.808 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:17.808 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:54:17.808 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.808 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:54:17.808 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:38.754678+0000", 2023-09-23T19:54:17.808 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:54:17.809 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.809 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.809 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:54:17.809 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:38.754678+0000", 2023-09-23T19:54:17.809 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.999999 2023-09-23T19:54:17.809 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.809 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.809 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:54:17.809 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:38.754677+0000", 2023-09-23T19:54:17.809 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00014006700000000001 2023-09-23T19:54:17.810 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.810 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.810 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:54:17.810 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:38.754817+0000", 2023-09-23T19:54:17.810 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.30077571800000003 2023-09-23T19:54:17.810 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.810 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.810 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:54:17.810 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:39.055593+0000", 2023-09-23T19:54:17.810 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.826e-05 2023-09-23T19:54:17.810 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.811 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.811 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:54:17.811 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:39.055621+0000", 2023-09-23T19:54:17.811 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.5353999999999996e-05 2023-09-23T19:54:17.811 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.811 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.811 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:54:17.811 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:39.055706+0000", 2023-09-23T19:54:17.811 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.4102e-05 2023-09-23T19:54:17.811 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.811 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.812 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:54:17.812 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:39.055741+0000", 2023-09-23T19:54:17.812 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0014145710000000001 2023-09-23T19:54:17.812 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.812 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.812 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:54:17.812 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:39.057155+0000", 2023-09-23T19:54:17.812 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.5875000000000003e-05 2023-09-23T19:54:17.812 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.812 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.813 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:54:17.813 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:39.057211+0000", 2023-09-23T19:54:17.813 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.302533092 2023-09-23T19:54:17.813 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:54:17.813 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:54:17.813 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:54:17.813 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.813 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.813 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_repop(client.36480.0:11097 12.1 e211/207 12:82bbe78d:::benchmark_data_smithi016_181952_object11096:head v 211'658, mlcod=211'656)", 2023-09-23T19:54:17.813 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:46:38.763503+0000", 2023-09-23T19:54:17.814 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 459.01676361800003, 2023-09-23T19:54:17.814 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.29536009400000002, 2023-09-23T19:54:17.814 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:54:17.814 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:17.814 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:54:17.814 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.814 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:54:17.814 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:38.763503+0000", 2023-09-23T19:54:17.814 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:54:17.814 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.814 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.815 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:54:17.815 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:38.763503+0000", 2023-09-23T19:54:17.815 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T19:54:17.815 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.815 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.815 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:54:17.815 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:38.763501+0000", 2023-09-23T19:54:17.815 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.3226999999999997e-05 2023-09-23T19:54:17.815 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.815 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.816 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:54:17.816 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:38.763555+0000", 2023-09-23T19:54:17.816 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.29208113499999999 2023-09-23T19:54:17.816 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.816 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.816 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:54:17.816 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:39.055636+0000", 2023-09-23T19:54:17.816 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.2232000000000001e-05 2023-09-23T19:54:17.816 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.816 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.816 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:54:17.817 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:39.055648+0000", 2023-09-23T19:54:17.817 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00022353799999999999 2023-09-23T19:54:17.817 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.817 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.817 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:54:17.817 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:39.055872+0000", 2023-09-23T19:54:17.817 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.0003999999999999e-05 2023-09-23T19:54:17.817 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.817 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.817 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:54:17.817 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:39.055902+0000", 2023-09-23T19:54:17.818 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0029176100000000002 2023-09-23T19:54:17.818 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.818 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.818 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:54:17.818 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:39.058819+0000", 2023-09-23T19:54:17.818 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.3733000000000003e-05 2023-09-23T19:54:17.818 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.818 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.818 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:54:17.818 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:39.058863+0000", 2023-09-23T19:54:17.818 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.29536009400000002 2023-09-23T19:54:17.819 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:54:17.819 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:54:17.819 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:54:17.819 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.819 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.819 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.36480.0:22018 12.d 12:beebae2a:::benchmark_data_smithi016_181952_object22017:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e211)", 2023-09-23T19:54:17.819 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:46:44.766639+0000", 2023-09-23T19:54:17.819 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 453.01362747299999, 2023-09-23T19:54:17.819 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.21884089800000001, 2023-09-23T19:54:17.819 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:54:17.819 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:17.820 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:54:17.820 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.36480", 2023-09-23T19:54:17.820 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/1035295215", 2023-09-23T19:54:17.820 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 22018 2023-09-23T19:54:17.820 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.820 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:54:17.820 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.820 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:54:17.820 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.766639+0000", 2023-09-23T19:54:17.820 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:54:17.820 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.820 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.821 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:54:17.821 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.766639+0000", 2023-09-23T19:54:17.821 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T19:54:17.821 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.821 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.821 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:54:17.821 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.766637+0000", 2023-09-23T19:54:17.821 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.2534999999999998e-05 2023-09-23T19:54:17.821 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.821 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.821 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:54:17.821 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.766690+0000", 2023-09-23T19:54:17.822 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.4758e-05 2023-09-23T19:54:17.822 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.822 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.822 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:54:17.822 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.766705+0000", 2023-09-23T19:54:17.822 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.8669999999999994e-06 2023-09-23T19:54:17.822 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.822 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.822 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:54:17.822 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.766713+0000", 2023-09-23T19:54:17.822 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000184468 2023-09-23T19:54:17.822 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.823 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.823 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:54:17.823 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.766897+0000", 2023-09-23T19:54:17.823 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000107009 2023-09-23T19:54:17.823 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.823 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.823 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:54:17.823 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.767004+0000", 2023-09-23T19:54:17.823 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00019615799999999999 2023-09-23T19:54:17.823 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.823 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.823 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,2", 2023-09-23T19:54:17.824 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.767200+0000", 2023-09-23T19:54:17.824 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.001896355 2023-09-23T19:54:17.824 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.824 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.824 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:54:17.824 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.769097+0000", 2023-09-23T19:54:17.824 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0047745679999999999 2023-09-23T19:54:17.824 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.824 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.824 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:17.824 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.773871+0000", 2023-09-23T19:54:17.824 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.211336463 2023-09-23T19:54:17.825 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.825 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.825 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:17.825 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.985208+0000", 2023-09-23T19:54:17.825 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00015332099999999999 2023-09-23T19:54:17.825 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.825 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.825 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:54:17.825 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.985361+0000", 2023-09-23T19:54:17.825 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00011893 2023-09-23T19:54:17.825 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.825 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.825 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:54:17.826 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.985480+0000", 2023-09-23T19:54:17.826 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.21884089800000001 2023-09-23T19:54:17.826 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:54:17.826 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:54:17.826 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:54:17.826 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.826 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.826 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.36480.0:22026 12.d 12:be1c558a:::benchmark_data_smithi016_181952_object22025:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e211)", 2023-09-23T19:54:17.826 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:46:44.768996+0000", 2023-09-23T19:54:17.826 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 453.01127065700001, 2023-09-23T19:54:17.826 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.21675122199999999, 2023-09-23T19:54:17.827 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:54:17.827 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:17.827 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:54:17.827 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.36480", 2023-09-23T19:54:17.827 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/1035295215", 2023-09-23T19:54:17.827 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 22026 2023-09-23T19:54:17.827 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.827 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:54:17.827 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.827 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:54:17.827 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.768996+0000", 2023-09-23T19:54:17.827 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:54:17.828 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.828 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.828 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:54:17.828 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.768996+0000", 2023-09-23T19:54:17.828 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T19:54:17.828 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.828 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.828 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:54:17.828 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.768994+0000", 2023-09-23T19:54:17.828 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.6185000000000003e-05 2023-09-23T19:54:17.828 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.828 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.829 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:54:17.829 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.769070+0000", 2023-09-23T19:54:17.829 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.0596000000000001e-05 2023-09-23T19:54:17.829 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.829 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.829 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:54:17.829 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.769091+0000", 2023-09-23T19:54:17.829 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.4149999999999999e-06 2023-09-23T19:54:17.829 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.829 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.829 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:54:17.829 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.769099+0000", 2023-09-23T19:54:17.830 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.1843999999999999e-05 2023-09-23T19:54:17.830 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.830 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.830 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:54:17.830 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.769161+0000", 2023-09-23T19:54:17.830 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000117843 2023-09-23T19:54:17.830 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.830 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.830 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:54:17.830 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.769279+0000", 2023-09-23T19:54:17.830 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00012996899999999999 2023-09-23T19:54:17.830 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.831 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.831 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,2", 2023-09-23T19:54:17.831 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.769409+0000", 2023-09-23T19:54:17.831 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00176783 2023-09-23T19:54:17.831 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.831 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.831 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:54:17.831 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.771177+0000", 2023-09-23T19:54:17.831 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0027747639999999999 2023-09-23T19:54:17.831 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.831 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.831 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:17.832 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.773952+0000", 2023-09-23T19:54:17.832 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.21163378399999999 2023-09-23T19:54:17.832 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.832 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.832 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:17.832 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.985585+0000", 2023-09-23T19:54:17.832 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.5916999999999997e-05 2023-09-23T19:54:17.832 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.832 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.832 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:54:17.832 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.985681+0000", 2023-09-23T19:54:17.832 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.5779999999999997e-05 2023-09-23T19:54:17.832 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.833 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.833 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:54:17.833 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.985747+0000", 2023-09-23T19:54:17.833 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.21675122199999999 2023-09-23T19:54:17.833 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:54:17.833 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:54:17.833 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:54:17.833 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.833 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.833 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.36480.0:22027 12.d 12:b647e90f:::benchmark_data_smithi016_181952_object22026:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e211)", 2023-09-23T19:54:17.833 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:46:44.769124+0000", 2023-09-23T19:54:17.833 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 453.011142011, 2023-09-23T19:54:17.834 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.217237874, 2023-09-23T19:54:17.834 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:54:17.834 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:17.834 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:54:17.834 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.36480", 2023-09-23T19:54:17.834 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/1035295215", 2023-09-23T19:54:17.834 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 22027 2023-09-23T19:54:17.834 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.834 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:54:17.834 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.834 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:54:17.835 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.769124+0000", 2023-09-23T19:54:17.835 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:54:17.835 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.835 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.835 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:54:17.835 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.769124+0000", 2023-09-23T19:54:17.835 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.999999 2023-09-23T19:54:17.835 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.835 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.835 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:54:17.835 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.769123+0000", 2023-09-23T19:54:17.835 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.4295999999999997e-05 2023-09-23T19:54:17.836 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.836 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.836 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:54:17.836 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.769168+0000", 2023-09-23T19:54:17.836 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.1542000000000001e-05 2023-09-23T19:54:17.836 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.836 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.836 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:54:17.836 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.769179+0000", 2023-09-23T19:54:17.836 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.0709999999999997e-06 2023-09-23T19:54:17.836 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.837 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.837 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:54:17.837 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.769186+0000", 2023-09-23T19:54:17.837 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.001112619 2023-09-23T19:54:17.837 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.837 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.837 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:54:17.837 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.770299+0000", 2023-09-23T19:54:17.837 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00018189899999999999 2023-09-23T19:54:17.837 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.837 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.837 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:54:17.838 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.770481+0000", 2023-09-23T19:54:17.838 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00012796300000000001 2023-09-23T19:54:17.838 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.838 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.838 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,2", 2023-09-23T19:54:17.838 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.770609+0000", 2023-09-23T19:54:17.838 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0021162249999999998 2023-09-23T19:54:17.838 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.838 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.838 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:54:17.838 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.772725+0000", 2023-09-23T19:54:17.838 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0012887910000000001 2023-09-23T19:54:17.839 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.839 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.839 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:17.839 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.774014+0000", 2023-09-23T19:54:17.839 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.21222712099999999 2023-09-23T19:54:17.839 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.839 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.839 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:17.839 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.986241+0000", 2023-09-23T19:54:17.839 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.0413999999999994e-05 2023-09-23T19:54:17.839 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.839 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.840 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:54:17.840 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.986311+0000", 2023-09-23T19:54:17.840 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.0875000000000003e-05 2023-09-23T19:54:17.840 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.840 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.840 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:54:17.840 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:46:44.986362+0000", 2023-09-23T19:54:17.840 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.217237874 2023-09-23T19:54:17.840 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:54:17.840 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:54:17.840 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:54:17.840 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.841 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.841 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.36480.0:53521 12.d 12:bd742e17:::benchmark_data_smithi016_181952_object53520:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e213)", 2023-09-23T19:54:17.841 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:47:09.160615+0000", 2023-09-23T19:54:17.841 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 428.619651465, 2023-09-23T19:54:17.841 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.15955323199999999, 2023-09-23T19:54:17.841 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:54:17.841 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:17.841 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:54:17.841 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.36480", 2023-09-23T19:54:17.841 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/1035295215", 2023-09-23T19:54:17.841 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 53521 2023-09-23T19:54:17.841 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.842 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:54:17.842 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.842 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:54:17.842 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.160615+0000", 2023-09-23T19:54:17.842 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:54:17.842 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.842 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.842 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:54:17.842 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.160615+0000", 2023-09-23T19:54:17.842 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T19:54:17.842 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.843 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.843 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:54:17.843 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.160614+0000", 2023-09-23T19:54:17.843 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.0057999999999997e-05 2023-09-23T19:54:17.843 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.843 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.843 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:54:17.843 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.160704+0000", 2023-09-23T19:54:17.843 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.2e-05 2023-09-23T19:54:17.843 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.843 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.843 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:54:17.843 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.160716+0000", 2023-09-23T19:54:17.844 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.1059999999999996e-06 2023-09-23T19:54:17.844 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.844 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.844 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:54:17.844 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.160722+0000", 2023-09-23T19:54:17.844 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.3596000000000001e-05 2023-09-23T19:54:17.844 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.844 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.844 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:54:17.844 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.160765+0000", 2023-09-23T19:54:17.844 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000175644 2023-09-23T19:54:17.844 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.845 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.845 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:54:17.845 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.160941+0000", 2023-09-23T19:54:17.845 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00015503199999999999 2023-09-23T19:54:17.845 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.845 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.845 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,2", 2023-09-23T19:54:17.845 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.161096+0000", 2023-09-23T19:54:17.845 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0031243220000000001 2023-09-23T19:54:17.845 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.845 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.845 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:17.846 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.164220+0000", 2023-09-23T19:54:17.846 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0012119920000000001 2023-09-23T19:54:17.846 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.846 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.846 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:54:17.846 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.165432+0000", 2023-09-23T19:54:17.846 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.154565067 2023-09-23T19:54:17.846 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.846 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.846 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:17.846 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.319998+0000", 2023-09-23T19:54:17.846 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.5186999999999995e-05 2023-09-23T19:54:17.847 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.847 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.847 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:54:17.847 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.320093+0000", 2023-09-23T19:54:17.847 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.5507999999999996e-05 2023-09-23T19:54:17.847 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.847 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.847 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:54:17.847 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.320168+0000", 2023-09-23T19:54:17.847 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.15955323199999999 2023-09-23T19:54:17.847 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:54:17.847 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:54:17.848 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:54:17.848 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.848 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.848 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.36480.0:53529 12.d 12:b00f38c8:::benchmark_data_smithi016_181952_object53528:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e213)", 2023-09-23T19:54:17.848 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:47:09.170335+0000", 2023-09-23T19:54:17.848 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 428.60993153999999, 2023-09-23T19:54:17.848 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.15207504199999999, 2023-09-23T19:54:17.848 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:54:17.848 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:17.848 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:54:17.848 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.36480", 2023-09-23T19:54:17.848 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/1035295215", 2023-09-23T19:54:17.849 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 53529 2023-09-23T19:54:17.849 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.849 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:54:17.849 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.849 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:54:17.849 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.170335+0000", 2023-09-23T19:54:17.849 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:54:17.849 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.849 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.849 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:54:17.849 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.170335+0000", 2023-09-23T19:54:17.849 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T19:54:17.850 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.850 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.850 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:54:17.850 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.170334+0000", 2023-09-23T19:54:17.850 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.6759999999999999e-05 2023-09-23T19:54:17.850 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.850 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.850 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:54:17.850 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.170390+0000", 2023-09-23T19:54:17.850 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.2880999999999999e-05 2023-09-23T19:54:17.850 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.850 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.851 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:54:17.851 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.170403+0000", 2023-09-23T19:54:17.851 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.5950000000000001e-06 2023-09-23T19:54:17.851 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.851 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.851 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:54:17.851 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.170411+0000", 2023-09-23T19:54:17.851 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.0958999999999997e-05 2023-09-23T19:54:17.851 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.851 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.851 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:54:17.851 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.170472+0000", 2023-09-23T19:54:17.852 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00016766599999999999 2023-09-23T19:54:17.852 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.852 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.852 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:54:17.852 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.170639+0000", 2023-09-23T19:54:17.852 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00020143299999999999 2023-09-23T19:54:17.852 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.852 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.852 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,2", 2023-09-23T19:54:17.852 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.170841+0000", 2023-09-23T19:54:17.852 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.001808163 2023-09-23T19:54:17.853 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.853 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.853 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:54:17.853 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.172649+0000", 2023-09-23T19:54:17.853 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0025399329999999999 2023-09-23T19:54:17.853 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.853 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.853 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:17.853 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.175189+0000", 2023-09-23T19:54:17.853 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.14696757699999999 2023-09-23T19:54:17.853 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.853 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.854 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:17.854 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.322157+0000", 2023-09-23T19:54:17.854 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00014613199999999999 2023-09-23T19:54:17.854 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.854 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.854 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:54:17.854 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.322303+0000", 2023-09-23T19:54:17.854 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000107285 2023-09-23T19:54:17.854 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.854 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.854 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:54:17.855 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.322410+0000", 2023-09-23T19:54:17.855 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.15207504199999999 2023-09-23T19:54:17.855 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:54:17.855 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:54:17.855 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:54:17.855 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.855 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.855 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.36480.0:53531 12.d 12:bff72a74:::benchmark_data_smithi016_181952_object53530:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e213)", 2023-09-23T19:54:17.855 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:47:09.172760+0000", 2023-09-23T19:54:17.855 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 428.60750616799999, 2023-09-23T19:54:17.855 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.15713658799999999, 2023-09-23T19:54:17.855 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:54:17.856 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:17.857 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:54:17.857 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.36480", 2023-09-23T19:54:17.857 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/1035295215", 2023-09-23T19:54:17.857 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 53531 2023-09-23T19:54:17.857 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.858 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:54:17.858 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.858 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:54:17.858 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.172760+0000", 2023-09-23T19:54:17.858 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:54:17.858 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.858 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.858 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:54:17.858 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.172760+0000", 2023-09-23T19:54:17.858 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T19:54:17.858 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.859 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.859 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:54:17.859 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.172759+0000", 2023-09-23T19:54:17.859 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.8034000000000004e-05 2023-09-23T19:54:17.859 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.859 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.859 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:54:17.859 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.172857+0000", 2023-09-23T19:54:17.859 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.1041e-05 2023-09-23T19:54:17.859 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.860 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.860 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:54:17.860 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.172868+0000", 2023-09-23T19:54:17.860 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.096e-06 2023-09-23T19:54:17.860 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.860 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.860 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:54:17.860 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.172873+0000", 2023-09-23T19:54:17.860 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.6422999999999999e-05 2023-09-23T19:54:17.860 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.860 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.860 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:54:17.861 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.172920+0000", 2023-09-23T19:54:17.861 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000147539 2023-09-23T19:54:17.861 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.861 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.861 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:54:17.861 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.173067+0000", 2023-09-23T19:54:17.861 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00026320500000000003 2023-09-23T19:54:17.861 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.861 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.861 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,2", 2023-09-23T19:54:17.861 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.173330+0000", 2023-09-23T19:54:17.862 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.001940951 2023-09-23T19:54:17.862 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.862 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.862 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:54:17.862 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.175271+0000", 2023-09-23T19:54:17.862 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000198765 2023-09-23T19:54:17.862 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.862 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.862 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:17.862 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.175470+0000", 2023-09-23T19:54:17.862 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.15430743099999999 2023-09-23T19:54:17.863 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.863 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.863 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:17.863 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.329777+0000", 2023-09-23T19:54:17.863 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.0326000000000004e-05 2023-09-23T19:54:17.863 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.863 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.863 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:54:17.863 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.329848+0000", 2023-09-23T19:54:17.863 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.9042999999999998e-05 2023-09-23T19:54:17.863 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.864 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.864 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:54:17.864 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.329897+0000", 2023-09-23T19:54:17.864 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.15713658799999999 2023-09-23T19:54:17.864 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:54:17.864 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:54:17.864 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:54:17.864 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.864 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.864 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.36480.0:53534 12.d 12:bd7593c0:::benchmark_data_smithi016_181952_object53533:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e213)", 2023-09-23T19:54:17.864 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:47:09.179347+0000", 2023-09-23T19:54:17.865 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 428.600919019, 2023-09-23T19:54:17.865 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.150829079, 2023-09-23T19:54:17.865 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:54:17.865 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:17.865 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:54:17.865 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.36480", 2023-09-23T19:54:17.865 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/1035295215", 2023-09-23T19:54:17.865 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 53534 2023-09-23T19:54:17.865 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.865 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:54:17.865 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.865 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:54:17.866 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.179347+0000", 2023-09-23T19:54:17.866 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:54:17.866 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.866 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.866 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:54:17.866 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.179347+0000", 2023-09-23T19:54:17.866 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T19:54:17.866 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.866 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.866 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:54:17.866 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.179346+0000", 2023-09-23T19:54:17.867 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00010075 2023-09-23T19:54:17.867 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.867 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.867 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:54:17.867 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.179447+0000", 2023-09-23T19:54:17.867 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.3382e-05 2023-09-23T19:54:17.867 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.867 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.867 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:54:17.867 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.179460+0000", 2023-09-23T19:54:17.867 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.0399999999999993e-06 2023-09-23T19:54:17.867 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.867 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.868 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:54:17.868 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.179468+0000", 2023-09-23T19:54:17.868 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.9726999999999999e-05 2023-09-23T19:54:17.868 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.868 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.868 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:54:17.868 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.179528+0000", 2023-09-23T19:54:17.868 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00014772 2023-09-23T19:54:17.868 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.868 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.868 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:54:17.868 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.179676+0000", 2023-09-23T19:54:17.869 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00017920499999999999 2023-09-23T19:54:17.869 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.869 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.869 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,2", 2023-09-23T19:54:17.869 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.179855+0000", 2023-09-23T19:54:17.869 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.002016697 2023-09-23T19:54:17.869 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.869 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.869 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:54:17.869 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.181872+0000", 2023-09-23T19:54:17.869 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00062545999999999999 2023-09-23T19:54:17.869 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.870 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.870 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:17.870 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.182497+0000", 2023-09-23T19:54:17.870 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.14751093900000001 2023-09-23T19:54:17.870 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.870 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.870 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:17.870 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.330008+0000", 2023-09-23T19:54:17.870 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00012434600000000001 2023-09-23T19:54:17.870 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.870 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.871 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:54:17.871 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.330132+0000", 2023-09-23T19:54:17.871 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.4181000000000003e-05 2023-09-23T19:54:17.871 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.871 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.871 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:54:17.871 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:09.330176+0000", 2023-09-23T19:54:17.871 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.150829079 2023-09-23T19:54:17.871 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:54:17.871 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:54:17.871 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:54:17.871 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.872 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.872 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.36480.0:64355 12.d 12:b066dbd0:::benchmark_data_smithi016_181952_object64354:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e214)", 2023-09-23T19:54:17.872 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:47:19.791142+0000", 2023-09-23T19:54:17.872 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 417.98912442800003, 2023-09-23T19:54:17.872 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.306341062, 2023-09-23T19:54:17.872 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:54:17.872 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:17.872 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:54:17.872 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.36480", 2023-09-23T19:54:17.872 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/1035295215", 2023-09-23T19:54:17.872 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 64355 2023-09-23T19:54:17.872 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.873 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:54:17.873 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.873 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:54:17.873 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:19.791142+0000", 2023-09-23T19:54:17.873 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:54:17.873 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.873 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.873 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:54:17.873 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:19.791142+0000", 2023-09-23T19:54:17.873 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T19:54:17.873 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.873 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.874 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:54:17.874 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:19.791140+0000", 2023-09-23T19:54:17.874 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.1391000000000003e-05 2023-09-23T19:54:17.874 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.874 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.874 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:54:17.874 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:19.791192+0000", 2023-09-23T19:54:17.874 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.522e-05 2023-09-23T19:54:17.874 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.874 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.874 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:54:17.875 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:19.791207+0000", 2023-09-23T19:54:17.875 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.3289999999999999e-06 2023-09-23T19:54:17.875 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.875 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.875 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:54:17.875 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:19.791213+0000", 2023-09-23T19:54:17.875 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.2302000000000001e-05 2023-09-23T19:54:17.875 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.875 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.875 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:54:17.875 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:19.791275+0000", 2023-09-23T19:54:17.875 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00018195200000000001 2023-09-23T19:54:17.876 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.876 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.876 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:54:17.876 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:19.791457+0000", 2023-09-23T19:54:17.876 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000209645 2023-09-23T19:54:17.876 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.876 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.876 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,2", 2023-09-23T19:54:17.876 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:19.791667+0000", 2023-09-23T19:54:17.876 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0027570609999999999 2023-09-23T19:54:17.876 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.876 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.877 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:17.877 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:19.794424+0000", 2023-09-23T19:54:17.877 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.0900999999999999e-05 2023-09-23T19:54:17.877 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.877 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.877 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:54:17.877 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:19.794435+0000", 2023-09-23T19:54:17.877 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.30292472599999998 2023-09-23T19:54:17.877 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.877 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.877 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:17.877 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:20.097360+0000", 2023-09-23T19:54:17.878 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.5971999999999998e-05 2023-09-23T19:54:17.878 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.878 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.878 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:54:17.878 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:20.097426+0000", 2023-09-23T19:54:17.878 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.7346000000000003e-05 2023-09-23T19:54:17.878 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.878 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.878 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:54:17.878 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:47:20.097483+0000", 2023-09-23T19:54:17.878 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.306341062 2023-09-23T19:54:17.878 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:54:17.879 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:54:17.879 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:54:17.879 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.879 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.879 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.36480.0:106856 12.d 12:b7019e67:::benchmark_data_smithi016_181952_object4175:head [delete] snapc 0=[] ondisk+write+known_if_redirected e217)", 2023-09-23T19:54:17.879 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:48:09.428617+0000", 2023-09-23T19:54:17.879 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 368.35164915500002, 2023-09-23T19:54:17.879 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.16861890199999999, 2023-09-23T19:54:17.879 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:54:17.879 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:17.879 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:54:17.880 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.36480", 2023-09-23T19:54:17.880 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/1035295215", 2023-09-23T19:54:17.880 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 106856 2023-09-23T19:54:17.880 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.880 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:54:17.880 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.880 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:54:17.880 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.428617+0000", 2023-09-23T19:54:17.880 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:54:17.880 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.880 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.880 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:54:17.881 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.428617+0000", 2023-09-23T19:54:17.881 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T19:54:17.881 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.881 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.881 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:54:17.881 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.428616+0000", 2023-09-23T19:54:17.881 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.2549999999999997e-06 2023-09-23T19:54:17.881 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.881 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.881 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:54:17.881 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.428624+0000", 2023-09-23T19:54:17.882 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.4355000000000001e-05 2023-09-23T19:54:17.882 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.882 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.882 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:54:17.882 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.428638+0000", 2023-09-23T19:54:17.882 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.1090000000000003e-06 2023-09-23T19:54:17.882 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.882 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.882 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:54:17.882 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.428645+0000", 2023-09-23T19:54:17.882 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.7475000000000003e-05 2023-09-23T19:54:17.883 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.883 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.883 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:54:17.883 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.428682+0000", 2023-09-23T19:54:17.883 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.4079000000000005e-05 2023-09-23T19:54:17.883 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.883 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.883 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:54:17.883 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.428776+0000", 2023-09-23T19:54:17.883 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000115065 2023-09-23T19:54:17.883 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.883 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.884 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,2", 2023-09-23T19:54:17.884 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.428891+0000", 2023-09-23T19:54:17.884 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0029361389999999999 2023-09-23T19:54:17.884 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.884 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.884 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:54:17.884 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.431827+0000", 2023-09-23T19:54:17.884 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0024404000000000001 2023-09-23T19:54:17.884 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.884 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.884 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:17.885 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.434268+0000", 2023-09-23T19:54:17.885 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.162693846 2023-09-23T19:54:17.885 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.885 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.885 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:17.885 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.596962+0000", 2023-09-23T19:54:17.885 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000150443 2023-09-23T19:54:17.885 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.885 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.885 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:54:17.885 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.597112+0000", 2023-09-23T19:54:17.885 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00012421599999999999 2023-09-23T19:54:17.886 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.886 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.886 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:54:17.886 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:48:09.597236+0000", 2023-09-23T19:54:17.886 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.16861890199999999 2023-09-23T19:54:17.886 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:54:17.886 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:54:17.886 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:54:17.886 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.886 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.886 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46640.0:54566 13.0 13:02cdaa51:::benchmark_data_smithi016_205181_object54565:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e238)", 2023-09-23T19:54:17.887 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:52:25.460372+0000", 2023-09-23T19:54:17.887 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 112.31989466500001, 2023-09-23T19:54:17.887 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.19363842100000001, 2023-09-23T19:54:17.887 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:54:17.887 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:17.887 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:54:17.887 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46640", 2023-09-23T19:54:17.887 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2656794142", 2023-09-23T19:54:17.887 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 54566 2023-09-23T19:54:17.887 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.887 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:54:17.887 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.887 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:54:17.888 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.460372+0000", 2023-09-23T19:54:17.888 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:54:17.888 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.888 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.888 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:54:17.888 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.460372+0000", 2023-09-23T19:54:17.888 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T19:54:17.888 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.888 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.888 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:54:17.888 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.460370+0000", 2023-09-23T19:54:17.888 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.0756999999999999e-05 2023-09-23T19:54:17.889 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.889 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.889 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:54:17.889 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.460451+0000", 2023-09-23T19:54:17.889 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.3703000000000001e-05 2023-09-23T19:54:17.889 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.889 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.889 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:54:17.889 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.460465+0000", 2023-09-23T19:54:17.889 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.5190000000000003e-06 2023-09-23T19:54:17.889 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.890 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.890 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:54:17.890 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.460472+0000", 2023-09-23T19:54:17.890 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.1558999999999995e-05 2023-09-23T19:54:17.890 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.890 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.890 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:54:17.890 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.460564+0000", 2023-09-23T19:54:17.890 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00016508500000000001 2023-09-23T19:54:17.890 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.890 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.890 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:54:17.891 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.460729+0000", 2023-09-23T19:54:17.891 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000180691 2023-09-23T19:54:17.891 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.891 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.891 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,2", 2023-09-23T19:54:17.891 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.460910+0000", 2023-09-23T19:54:17.891 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0025721139999999999 2023-09-23T19:54:17.891 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.891 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.891 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:54:17.891 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.463482+0000", 2023-09-23T19:54:17.891 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00042217200000000001 2023-09-23T19:54:17.892 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.892 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.892 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:17.892 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.463904+0000", 2023-09-23T19:54:17.892 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.18999377100000001 2023-09-23T19:54:17.892 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.892 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.892 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:17.892 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.653898+0000", 2023-09-23T19:54:17.892 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.7748000000000003e-05 2023-09-23T19:54:17.892 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.893 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.893 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:54:17.893 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.653966+0000", 2023-09-23T19:54:17.893 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.4629999999999998e-05 2023-09-23T19:54:17.893 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.893 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.893 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:54:17.893 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.654010+0000", 2023-09-23T19:54:17.893 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.19363842100000001 2023-09-23T19:54:17.893 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:54:17.893 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:54:17.894 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:54:17.894 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.894 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.894 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46640.0:56843 13.0 13:00e212a9:::benchmark_data_smithi016_205181_object56842:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e238)", 2023-09-23T19:54:17.894 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:52:27.511602+0000", 2023-09-23T19:54:17.894 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 110.26866484, 2023-09-23T19:54:17.894 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.17315303900000001, 2023-09-23T19:54:17.894 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:54:17.894 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:17.894 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:54:17.894 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46640", 2023-09-23T19:54:17.895 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2656794142", 2023-09-23T19:54:17.895 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 56843 2023-09-23T19:54:17.895 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.895 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:54:17.895 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.895 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:54:17.895 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:27.511602+0000", 2023-09-23T19:54:17.895 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:54:17.895 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.895 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.895 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:54:17.895 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:27.511602+0000", 2023-09-23T19:54:17.896 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T19:54:17.896 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.896 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.896 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:54:17.896 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:27.511600+0000", 2023-09-23T19:54:17.896 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.9376999999999999e-05 2023-09-23T19:54:17.896 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.896 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.896 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:54:17.896 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:27.511640+0000", 2023-09-23T19:54:17.896 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.3338e-05 2023-09-23T19:54:17.896 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.897 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.897 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:54:17.897 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:27.511653+0000", 2023-09-23T19:54:17.897 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.024e-06 2023-09-23T19:54:17.897 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.897 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.897 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:54:17.897 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:27.511660+0000", 2023-09-23T19:54:17.897 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00015277800000000001 2023-09-23T19:54:17.897 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.897 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.897 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:54:17.897 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:27.511813+0000", 2023-09-23T19:54:17.898 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000149631 2023-09-23T19:54:17.898 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.898 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.898 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:54:17.898 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:27.511962+0000", 2023-09-23T19:54:17.898 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00015164399999999999 2023-09-23T19:54:17.898 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.898 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.898 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,2", 2023-09-23T19:54:17.898 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:27.512114+0000", 2023-09-23T19:54:17.898 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0029355840000000002 2023-09-23T19:54:17.898 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.899 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.899 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:54:17.899 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:27.515050+0000", 2023-09-23T19:54:17.899 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0058118470000000002 2023-09-23T19:54:17.899 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.899 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.899 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:17.899 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:27.520861+0000", 2023-09-23T19:54:17.899 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.163801114 2023-09-23T19:54:17.899 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.899 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.899 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:17.900 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:27.684662+0000", 2023-09-23T19:54:17.900 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.2683999999999998e-05 2023-09-23T19:54:17.900 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.900 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.900 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:54:17.900 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:27.684725+0000", 2023-09-23T19:54:17.900 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.9470000000000001e-05 2023-09-23T19:54:17.900 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.900 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.900 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:54:17.900 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:27.684755+0000", 2023-09-23T19:54:17.900 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.17315303900000001 2023-09-23T19:54:17.901 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:54:17.901 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:54:17.901 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:54:17.901 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.901 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.901 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46640.0:56876 13.0 13:0f368324:::benchmark_data_smithi016_205181_object56875:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e238)", 2023-09-23T19:54:17.901 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:52:27.539341+0000", 2023-09-23T19:54:17.901 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 110.24092523500001, 2023-09-23T19:54:17.901 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.15371965500000001, 2023-09-23T19:54:17.901 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:54:17.901 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:17.901 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:54:17.902 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46640", 2023-09-23T19:54:17.902 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2656794142", 2023-09-23T19:54:17.902 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 56876 2023-09-23T19:54:17.902 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.902 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:54:17.902 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.902 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:54:17.902 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:27.539341+0000", 2023-09-23T19:54:17.902 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:54:17.902 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.902 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.903 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:54:17.903 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:27.539341+0000", 2023-09-23T19:54:17.903 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T19:54:17.903 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.903 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.903 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:54:17.903 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:27.539339+0000", 2023-09-23T19:54:17.903 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.5113999999999999e-05 2023-09-23T19:54:17.903 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.903 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.903 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:54:17.903 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:27.539415+0000", 2023-09-23T19:54:17.904 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.3534e-05 2023-09-23T19:54:17.904 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.904 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.904 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:54:17.904 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:27.539428+0000", 2023-09-23T19:54:17.904 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.9780000000000002e-06 2023-09-23T19:54:17.904 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.904 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.904 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:54:17.904 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:27.539434+0000", 2023-09-23T19:54:17.904 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.9118e-05 2023-09-23T19:54:17.905 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.905 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.905 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:54:17.905 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:27.539483+0000", 2023-09-23T19:54:17.905 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000146067 2023-09-23T19:54:17.905 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.905 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.905 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:54:17.905 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:27.539629+0000", 2023-09-23T19:54:17.905 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000182521 2023-09-23T19:54:17.905 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.905 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.906 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,2", 2023-09-23T19:54:17.906 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:27.539812+0000", 2023-09-23T19:54:17.906 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0021894140000000002 2023-09-23T19:54:17.906 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.906 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.906 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:54:17.906 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:27.542001+0000", 2023-09-23T19:54:17.906 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0016539829999999999 2023-09-23T19:54:17.906 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.906 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.906 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:17.906 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:27.543655+0000", 2023-09-23T19:54:17.907 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.14928767400000001 2023-09-23T19:54:17.907 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.907 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.907 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:17.907 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:27.692943+0000", 2023-09-23T19:54:17.907 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.7736000000000003e-05 2023-09-23T19:54:17.907 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.907 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.907 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:54:17.907 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:27.693011+0000", 2023-09-23T19:54:17.907 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.0321000000000002e-05 2023-09-23T19:54:17.907 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.908 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.908 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:54:17.908 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:27.693061+0000", 2023-09-23T19:54:17.908 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.15371965500000001 2023-09-23T19:54:17.908 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:54:17.908 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:54:17.908 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:54:17.908 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.908 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.908 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46640.0:57342 13.e 13:7fbdd5fe:::benchmark_data_smithi016_205181_object57341:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e238)", 2023-09-23T19:54:17.908 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:52:28.050001+0000", 2023-09-23T19:54:17.909 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 109.730265811, 2023-09-23T19:54:17.909 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.28244245600000001, 2023-09-23T19:54:17.909 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:54:17.909 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:17.909 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:54:17.909 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46640", 2023-09-23T19:54:17.909 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2656794142", 2023-09-23T19:54:17.909 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 57342 2023-09-23T19:54:17.909 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.909 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:54:17.909 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.909 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:54:17.910 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:28.050001+0000", 2023-09-23T19:54:17.910 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:54:17.910 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.910 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.910 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:54:17.910 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:28.050001+0000", 2023-09-23T19:54:17.910 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T19:54:17.910 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.910 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.910 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:54:17.910 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:28.049999+0000", 2023-09-23T19:54:17.911 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.9902999999999998e-05 2023-09-23T19:54:17.911 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.911 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.911 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:54:17.911 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:28.050069+0000", 2023-09-23T19:54:17.911 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.4260999999999999e-05 2023-09-23T19:54:17.911 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.911 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.911 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:54:17.911 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:28.050083+0000", 2023-09-23T19:54:17.911 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.5209999999999999e-06 2023-09-23T19:54:17.911 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.912 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.912 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:54:17.912 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:28.050090+0000", 2023-09-23T19:54:17.912 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.7332e-05 2023-09-23T19:54:17.912 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.912 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.912 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:54:17.912 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:28.050137+0000", 2023-09-23T19:54:17.912 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00014121000000000001 2023-09-23T19:54:17.912 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.912 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.912 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:54:17.913 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:28.050278+0000", 2023-09-23T19:54:17.913 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00012409000000000001 2023-09-23T19:54:17.913 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.913 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.913 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 4,5", 2023-09-23T19:54:17.913 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:28.050402+0000", 2023-09-23T19:54:17.913 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0019525059999999999 2023-09-23T19:54:17.913 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.913 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.913 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:54:17.913 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:28.052355+0000", 2023-09-23T19:54:17.914 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.001069253 2023-09-23T19:54:17.914 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.914 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.914 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:17.914 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:28.053424+0000", 2023-09-23T19:54:17.914 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.27889888800000001 2023-09-23T19:54:17.914 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.914 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.914 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:17.914 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:28.332323+0000", 2023-09-23T19:54:17.914 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.7218999999999995e-05 2023-09-23T19:54:17.914 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.915 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.915 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:54:17.915 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:28.332390+0000", 2023-09-23T19:54:17.915 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.2828999999999999e-05 2023-09-23T19:54:17.915 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.915 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.915 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:54:17.915 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:28.332443+0000", 2023-09-23T19:54:17.915 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.28244245600000001 2023-09-23T19:54:17.915 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:54:17.915 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:54:17.915 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:54:17.916 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.916 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.916 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46640.0:111451 13.e 13:7e51e516:::benchmark_data_smithi016_205181_object4187:head [delete] snapc 0=[] ondisk+write+known_if_redirected e242)", 2023-09-23T19:54:17.916 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:53:28.584486+0000", 2023-09-23T19:54:17.916 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 49.195780145999997, 2023-09-23T19:54:17.916 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.106385576, 2023-09-23T19:54:17.916 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:54:17.916 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:17.916 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:54:17.916 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46640", 2023-09-23T19:54:17.916 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2656794142", 2023-09-23T19:54:17.916 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 111451 2023-09-23T19:54:17.917 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.917 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:54:17.917 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.917 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:54:17.917 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:53:28.584486+0000", 2023-09-23T19:54:17.917 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:54:17.917 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.917 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.917 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:54:17.917 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:53:28.584486+0000", 2023-09-23T19:54:17.917 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T19:54:17.917 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.918 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.918 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:54:17.918 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:53:28.584484+0000", 2023-09-23T19:54:17.918 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.0329000000000001e-05 2023-09-23T19:54:17.918 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.918 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.918 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:54:17.918 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:53:28.584495+0000", 2023-09-23T19:54:17.918 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.9415999999999998e-05 2023-09-23T19:54:17.918 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.918 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.919 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:54:17.919 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:53:28.584514+0000", 2023-09-23T19:54:17.919 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.9889999999999992e-06 2023-09-23T19:54:17.919 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.919 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.919 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:54:17.919 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:53:28.584522+0000", 2023-09-23T19:54:17.919 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00017452799999999999 2023-09-23T19:54:17.919 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.919 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.919 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:54:17.919 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:53:28.584696+0000", 2023-09-23T19:54:17.920 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000109893 2023-09-23T19:54:17.920 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.920 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.920 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:54:17.920 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:53:28.584806+0000", 2023-09-23T19:54:17.920 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000166226 2023-09-23T19:54:17.920 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.920 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.920 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 4,5", 2023-09-23T19:54:17.920 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:53:28.584973+0000", 2023-09-23T19:54:17.920 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.010048756000000001 2023-09-23T19:54:17.920 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.921 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.921 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:54:17.921 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:53:28.595021+0000", 2023-09-23T19:54:17.921 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0041339030000000004 2023-09-23T19:54:17.921 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.921 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.921 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:17.921 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:53:28.599155+0000", 2023-09-23T19:54:17.921 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.091638908000000005 2023-09-23T19:54:17.921 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.921 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.922 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:17.922 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:53:28.690794+0000", 2023-09-23T19:54:17.922 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.1576000000000003e-05 2023-09-23T19:54:17.922 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.922 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.922 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:54:17.922 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:53:28.690836+0000", 2023-09-23T19:54:17.922 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.6106999999999998e-05 2023-09-23T19:54:17.922 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.922 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.923 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:54:17.923 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:53:28.690872+0000", 2023-09-23T19:54:17.923 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.106385576 2023-09-23T19:54:17.923 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:54:17.923 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:54:17.923 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:54:17.923 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.923 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.923 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46640.0:111463 13.e 13:786bc834:::benchmark_data_smithi016_205181_object4199:head [delete] snapc 0=[] ondisk+write+known_if_redirected e242)", 2023-09-23T19:54:17.923 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:53:28.595622+0000", 2023-09-23T19:54:17.923 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 49.184644018, 2023-09-23T19:54:17.923 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.1073984, 2023-09-23T19:54:17.923 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:54:17.924 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:17.924 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:54:17.924 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46640", 2023-09-23T19:54:17.924 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2656794142", 2023-09-23T19:54:17.924 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 111463 2023-09-23T19:54:17.924 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.924 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:54:17.924 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.924 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:54:17.924 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:53:28.595622+0000", 2023-09-23T19:54:17.924 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:54:17.924 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.925 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.925 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:54:17.925 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:53:28.595622+0000", 2023-09-23T19:54:17.925 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.999999 2023-09-23T19:54:17.925 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.925 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.925 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:54:17.925 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:53:28.595622+0000", 2023-09-23T19:54:17.925 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.4309999999999996e-06 2023-09-23T19:54:17.925 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.925 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.925 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:54:17.925 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:53:28.595626+0000", 2023-09-23T19:54:17.926 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.0479000000000001e-05 2023-09-23T19:54:17.926 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.926 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.926 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:54:17.926 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:53:28.595637+0000", 2023-09-23T19:54:17.926 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.4669999999999998e-06 2023-09-23T19:54:17.926 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.926 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.926 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:54:17.926 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:53:28.595641+0000", 2023-09-23T19:54:17.926 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.4159000000000003e-05 2023-09-23T19:54:17.927 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.927 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.927 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:54:17.927 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:53:28.595675+0000", 2023-09-23T19:54:17.927 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00011358600000000001 2023-09-23T19:54:17.927 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.927 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.927 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:54:17.927 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:53:28.595789+0000", 2023-09-23T19:54:17.927 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.3766999999999999e-05 2023-09-23T19:54:17.927 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.928 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.928 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 4,5", 2023-09-23T19:54:17.928 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:53:28.595873+0000", 2023-09-23T19:54:17.928 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0029487409999999999 2023-09-23T19:54:17.928 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.928 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.928 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:54:17.928 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:53:28.598821+0000", 2023-09-23T19:54:17.928 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0041308170000000002 2023-09-23T19:54:17.928 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.928 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.929 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:17.929 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:53:28.602952+0000", 2023-09-23T19:54:17.929 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.099978956999999993 2023-09-23T19:54:17.929 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.929 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.929 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:17.929 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:53:28.702931+0000", 2023-09-23T19:54:17.929 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.8881000000000001e-05 2023-09-23T19:54:17.929 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.929 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.929 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:54:17.929 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:53:28.702990+0000", 2023-09-23T19:54:17.930 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.0882e-05 2023-09-23T19:54:17.930 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:54:17.930 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:54:17.930 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:54:17.930 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:53:28.703021+0000", 2023-09-23T19:54:17.930 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.1073984 2023-09-23T19:54:17.930 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:54:17.930 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:54:17.930 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:54:17.930 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:54:17.930 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:54:17.930 INFO:teuthology.orchestra.run.smithi092.stdout:} 2023-09-23T19:54:17.931 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:17.960 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:17.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:17.965 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:54:18.066 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:18.098 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:18.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:18.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:54:18.200 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:18.219 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:18.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:18.221 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:18.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:18 smithi016 bash[77995]: cluster 2023-09-23T19:54:16.499039+0000 mgr.y (mgr.25325) 1409 : cluster 0 pgmap v1418: 89 pgs: 89 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 785 op/s 2023-09-23T19:54:18.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:18 smithi016 bash[75004]: cluster 2023-09-23T19:54:16.499039+0000 mgr.y (mgr.25325) 1409 : cluster 0 pgmap v1418: 89 pgs: 89 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 785 op/s 2023-09-23T19:54:18.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:18 smithi092 bash[58855]: cluster 2023-09-23T19:54:16.499039+0000 mgr.y (mgr.25325) 1409 : cluster 0 pgmap v1418: 89 pgs: 89 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 785 op/s 2023-09-23T19:54:18.303 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:54:18.322 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:18.339 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:54:18.361 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:18.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:18.368 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:18.396 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:54:18.443 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:54:18.469 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:18.482 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:54:18.527 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:54:18.548 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:18.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:18.553 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:18.563 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:54:18.600 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:54:18.602 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:54:18.602 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:54:18.602 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:54:18.602 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:54:18.602 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:54:18.602 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:54:18.602 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:54:18.602 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:54:18.654 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:18.683 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:18.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:18.688 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:18.789 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:18.821 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:18.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:18.826 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:18.927 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:18.962 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:18.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:18.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:19.045 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T19:54:19.066 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:19.123 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:19.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:19.126 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:19.227 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:19.234 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:19 smithi092 bash[58855]: audit 2023-09-23T19:54:18.254175+0000 mon.c (mon.2) 220 : audit 0 from='client.? 172.21.15.16:0/2528200591' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:54:19.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:19.258 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:19.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:19.359 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:19.369 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:19 smithi016 bash[77995]: audit 2023-09-23T19:54:18.254175+0000 mon.c (mon.2) 220 : audit 0 from='client.? 172.21.15.16:0/2528200591' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:54:19.369 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:19 smithi016 bash[75004]: audit 2023-09-23T19:54:18.254175+0000 mon.c (mon.2) 220 : audit 0 from='client.? 172.21.15.16:0/2528200591' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:54:19.389 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:19.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:19.391 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:19.492 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:19.522 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:19.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:19.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:19.627 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:19.651 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:19.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:19.654 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:19.755 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:19.794 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:19.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:19.799 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:54:19.900 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:19.930 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:19.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:19.933 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:54:20.033 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:20.069 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:20.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:20.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:54:20.172 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:20.217 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:20.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:20.223 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:20.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:20 smithi016 bash[77995]: cluster 2023-09-23T19:54:18.499578+0000 mgr.y (mgr.25325) 1410 : cluster 0 pgmap v1419: 89 pgs: 89 active+clean; 4.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 639 op/s 2023-09-23T19:54:20.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:20 smithi016 bash[75004]: cluster 2023-09-23T19:54:18.499578+0000 mgr.y (mgr.25325) 1410 : cluster 0 pgmap v1419: 89 pgs: 89 active+clean; 4.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 639 op/s 2023-09-23T19:54:20.324 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:20.332 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:20 smithi092 bash[58855]: cluster 2023-09-23T19:54:18.499578+0000 mgr.y (mgr.25325) 1410 : cluster 0 pgmap v1419: 89 pgs: 89 active+clean; 4.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 639 op/s 2023-09-23T19:54:20.352 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:20.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:20.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:20.458 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:20.499 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:20.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:20.503 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:20.604 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:20.647 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:20.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:20.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:20.753 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:20.786 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:20.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:20.792 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:20.893 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:20.923 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:20.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:20.931 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:21.031 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:21.060 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:21.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:21.063 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:21.163 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:21.191 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:21.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:21.199 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:54:21.300 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:21.329 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:21.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:21.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:21.433 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:21.456 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:21.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:21.460 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:21.561 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:21.592 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:21.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:21.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:21.697 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:21.775 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:21.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:21.777 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:21.878 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:21.901 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:21.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:21.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:22.013 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:22.039 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:22.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:22.042 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:54:22.106 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is set 2023-09-23T19:54:22.114 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:22 smithi016 bash[75004]: cluster 2023-09-23T19:54:20.500626+0000 mgr.y (mgr.25325) 1411 : cluster 0 pgmap v1420: 89 pgs: 89 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 779 op/s 2023-09-23T19:54:22.114 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:22 smithi016 bash[75004]: audit 2023-09-23T19:54:21.772285+0000 mon.c (mon.2) 221 : audit 1 from='client.? 172.21.15.16:0/3067132328' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:54:22.114 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:22 smithi016 bash[75004]: audit 2023-09-23T19:54:21.772674+0000 mon.a (mon.0) 1730 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:54:22.115 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:22 smithi016 bash[77995]: cluster 2023-09-23T19:54:20.500626+0000 mgr.y (mgr.25325) 1411 : cluster 0 pgmap v1420: 89 pgs: 89 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 779 op/s 2023-09-23T19:54:22.115 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:22 smithi016 bash[77995]: audit 2023-09-23T19:54:21.772285+0000 mon.c (mon.2) 221 : audit 1 from='client.? 172.21.15.16:0/3067132328' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:54:22.115 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:22 smithi016 bash[77995]: audit 2023-09-23T19:54:21.772674+0000 mon.a (mon.0) 1730 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:54:22.143 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:22.172 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:22.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:22.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:22.279 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:22.317 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:22.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:22.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:22.424 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:22.432 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:22 smithi092 bash[58855]: cluster 2023-09-23T19:54:20.500626+0000 mgr.y (mgr.25325) 1411 : cluster 0 pgmap v1420: 89 pgs: 89 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 779 op/s 2023-09-23T19:54:22.433 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:22 smithi092 bash[58855]: audit 2023-09-23T19:54:21.772285+0000 mon.c (mon.2) 221 : audit 1 from='client.? 172.21.15.16:0/3067132328' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:54:22.433 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:22 smithi092 bash[58855]: audit 2023-09-23T19:54:21.772674+0000 mon.a (mon.0) 1730 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:54:22.449 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:22.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:22.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:22.552 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:22.576 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:22.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:22.581 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:22.682 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:22.719 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:22.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:22.724 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:22.825 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:22.853 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:22.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:22.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:22.956 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:22.998 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:23.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:23.001 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:23.102 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:23.139 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:23.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:23.145 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:54:23.245 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:23.275 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:23.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:23.278 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:23.291 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:23 smithi016 bash[77995]: cluster 2023-09-23T19:54:22.083255+0000 mon.a (mon.0) 1731 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:54:23.291 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:23 smithi016 bash[77995]: audit 2023-09-23T19:54:22.094270+0000 mon.a (mon.0) 1732 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:54:23.291 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:23 smithi016 bash[77995]: cluster 2023-09-23T19:54:22.094357+0000 mon.a (mon.0) 1733 : cluster 0 osdmap e245: 8 total, 8 up, 8 in 2023-09-23T19:54:23.292 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:23 smithi016 bash[77995]: cluster 2023-09-23T19:54:22.099947+0000 osd.5 (osd.5) 174 : cluster 3 failed to encode map e245 with expected crc 2023-09-23T19:54:23.292 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:23 smithi016 bash[77995]: cluster 2023-09-23T19:54:22.103692+0000 osd.4 (osd.4) 188 : cluster 3 failed to encode map e245 with expected crc 2023-09-23T19:54:23.292 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:23 smithi016 bash[77995]: cluster 2023-09-23T19:54:22.115048+0000 osd.6 (osd.6) 181 : cluster 3 failed to encode map e245 with expected crc 2023-09-23T19:54:23.292 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:23 smithi016 bash[75004]: cluster 2023-09-23T19:54:22.083255+0000 mon.a (mon.0) 1731 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:54:23.292 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:23 smithi016 bash[75004]: audit 2023-09-23T19:54:22.094270+0000 mon.a (mon.0) 1732 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:54:23.292 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:23 smithi016 bash[75004]: cluster 2023-09-23T19:54:22.094357+0000 mon.a (mon.0) 1733 : cluster 0 osdmap e245: 8 total, 8 up, 8 in 2023-09-23T19:54:23.292 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:23 smithi016 bash[75004]: cluster 2023-09-23T19:54:22.099947+0000 osd.5 (osd.5) 174 : cluster 3 failed to encode map e245 with expected crc 2023-09-23T19:54:23.292 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:23 smithi016 bash[75004]: cluster 2023-09-23T19:54:22.103692+0000 osd.4 (osd.4) 188 : cluster 3 failed to encode map e245 with expected crc 2023-09-23T19:54:23.293 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:23 smithi016 bash[75004]: cluster 2023-09-23T19:54:22.115048+0000 osd.6 (osd.6) 181 : cluster 3 failed to encode map e245 with expected crc 2023-09-23T19:54:23.370 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:54:23.379 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:23.387 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:23 smithi092 bash[58855]: cluster 2023-09-23T19:54:22.083255+0000 mon.a (mon.0) 1731 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:54:23.387 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:23 smithi092 bash[58855]: audit 2023-09-23T19:54:22.094270+0000 mon.a (mon.0) 1732 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:54:23.387 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:23 smithi092 bash[58855]: cluster 2023-09-23T19:54:22.094357+0000 mon.a (mon.0) 1733 : cluster 0 osdmap e245: 8 total, 8 up, 8 in 2023-09-23T19:54:23.387 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:23 smithi092 bash[58855]: cluster 2023-09-23T19:54:22.099947+0000 osd.5 (osd.5) 174 : cluster 3 failed to encode map e245 with expected crc 2023-09-23T19:54:23.387 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:23 smithi092 bash[58855]: cluster 2023-09-23T19:54:22.103692+0000 osd.4 (osd.4) 188 : cluster 3 failed to encode map e245 with expected crc 2023-09-23T19:54:23.387 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:23 smithi092 bash[58855]: cluster 2023-09-23T19:54:22.115048+0000 osd.6 (osd.6) 181 : cluster 3 failed to encode map e245 with expected crc 2023-09-23T19:54:23.440 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:23.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:23.443 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:54:23.544 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:23.573 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:23.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:23.575 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:23.675 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:23.704 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:23.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:23.710 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:23.811 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:23.838 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:23.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:23.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:23.944 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:23.973 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:23.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:23.978 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:24.079 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:24.098 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:24 smithi016 bash[75004]: cluster 2023-09-23T19:54:22.501102+0000 mgr.y (mgr.25325) 1412 : cluster 0 pgmap v1422: 89 pgs: 89 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 690 op/s 2023-09-23T19:54:24.102 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:24.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:24.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:24.206 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:24.242 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:24.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:24.244 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:54:24.345 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:24.354 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:24 smithi092 bash[58855]: cluster 2023-09-23T19:54:22.501102+0000 mgr.y (mgr.25325) 1412 : cluster 0 pgmap v1422: 89 pgs: 89 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 690 op/s 2023-09-23T19:54:24.366 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:24.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:24.369 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:24.470 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:24.478 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:24 smithi016 bash[77995]: cluster 2023-09-23T19:54:22.501102+0000 mgr.y (mgr.25325) 1412 : cluster 0 pgmap v1422: 89 pgs: 89 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 690 op/s 2023-09-23T19:54:24.499 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:24.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:24.501 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:24.602 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:24.633 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:24.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:24.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:54:24.738 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:24.765 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:24.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:24.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:54:24.871 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:24.910 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:24.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:24.918 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:25.019 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:25.047 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:25.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:25.050 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:25.151 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:25.175 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:25.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:25.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:25.279 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:25.314 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:25.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:25.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:25.421 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:25.456 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:25.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:25.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:25.563 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:25.607 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:25.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:25.613 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:25.713 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:25.740 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:25.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:25.747 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:25.786 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:54:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:54:25] "GET /metrics HTTP/1.1" 200 36306 "" "Prometheus/2.33.4" 2023-09-23T19:54:25.848 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:25.874 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:25.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:25.878 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:25.979 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:26.010 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:26.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:26.018 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:26.118 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:26.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:26.146 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:26.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:26.247 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:26.274 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:26.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:26.280 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:26.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:26 smithi016 bash[77995]: cluster 2023-09-23T19:54:24.501588+0000 mgr.y (mgr.25325) 1413 : cluster 0 pgmap v1423: 89 pgs: 89 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2023-09-23T19:54:26.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:26 smithi016 bash[75004]: cluster 2023-09-23T19:54:24.501588+0000 mgr.y (mgr.25325) 1413 : cluster 0 pgmap v1423: 89 pgs: 89 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2023-09-23T19:54:26.380 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:26.406 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:26.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:26.413 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:26.514 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:26.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:26 smithi092 bash[58855]: cluster 2023-09-23T19:54:24.501588+0000 mgr.y (mgr.25325) 1413 : cluster 0 pgmap v1423: 89 pgs: 89 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2023-09-23T19:54:26.552 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:26.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:26.555 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:26.656 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:26.695 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:26.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:26.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:26.801 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:26.844 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:26.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:26.850 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:26.951 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:26.962 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T19:54:26.988 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:26.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:26.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:27.092 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:27.104 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:27 smithi016 bash[77995]: audit 2023-09-23T19:54:26.107508+0000 mon.a (mon.0) 1734 : audit 1 from='client.? 172.21.15.16:0/3980000507' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T19:54:27.104 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:27 smithi016 bash[75004]: audit 2023-09-23T19:54:26.107508+0000 mon.a (mon.0) 1734 : audit 1 from='client.? 172.21.15.16:0/3980000507' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T19:54:27.117 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:27.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:27.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:27.123 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T19:54:27.221 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:27.251 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:27.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:27.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:54:27.357 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:27.393 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:27.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:27.396 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:27.474 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:54:27.497 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:27.505 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:27 smithi092 bash[58855]: audit 2023-09-23T19:54:26.107508+0000 mon.a (mon.0) 1734 : audit 1 from='client.? 172.21.15.16:0/3980000507' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T19:54:27.522 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:27.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:27.528 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:54:27.629 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:27.682 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:27.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:27.688 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:27.789 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:27.815 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:27.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:27.820 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:27.920 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:27.954 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:27.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:27.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:28.061 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:28.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:28.099 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:28.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:28.200 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:28.234 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:28.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:28.237 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:54:28.337 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:28.363 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:28.368 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:28 smithi016 bash[77995]: cluster 2023-09-23T19:54:26.502292+0000 mgr.y (mgr.25325) 1414 : cluster 0 pgmap v1424: 89 pgs: 89 active+clean; 3.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2023-09-23T19:54:28.368 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:28 smithi016 bash[77995]: audit 2023-09-23T19:54:27.113354+0000 mon.a (mon.0) 1735 : audit 1 from='client.? 172.21.15.16:0/3980000507' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T19:54:28.369 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:28 smithi016 bash[77995]: cluster 2023-09-23T19:54:27.113425+0000 mon.a (mon.0) 1736 : cluster 0 osdmap e246: 8 total, 8 up, 8 in 2023-09-23T19:54:28.369 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:28 smithi016 bash[77995]: cluster 2023-09-23T19:54:27.117121+0000 osd.4 (osd.4) 189 : cluster 3 failed to encode map e246 with expected crc 2023-09-23T19:54:28.369 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:28 smithi016 bash[77995]: cluster 2023-09-23T19:54:27.126182+0000 osd.5 (osd.5) 175 : cluster 3 failed to encode map e246 with expected crc 2023-09-23T19:54:28.369 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:28 smithi016 bash[77995]: cluster 2023-09-23T19:54:27.131877+0000 osd.7 (osd.7) 162 : cluster 3 failed to encode map e246 with expected crc 2023-09-23T19:54:28.369 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:28 smithi016 bash[75004]: cluster 2023-09-23T19:54:26.502292+0000 mgr.y (mgr.25325) 1414 : cluster 0 pgmap v1424: 89 pgs: 89 active+clean; 3.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2023-09-23T19:54:28.370 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:28 smithi016 bash[75004]: audit 2023-09-23T19:54:27.113354+0000 mon.a (mon.0) 1735 : audit 1 from='client.? 172.21.15.16:0/3980000507' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T19:54:28.370 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:28 smithi016 bash[75004]: cluster 2023-09-23T19:54:27.113425+0000 mon.a (mon.0) 1736 : cluster 0 osdmap e246: 8 total, 8 up, 8 in 2023-09-23T19:54:28.370 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:28 smithi016 bash[75004]: cluster 2023-09-23T19:54:27.117121+0000 osd.4 (osd.4) 189 : cluster 3 failed to encode map e246 with expected crc 2023-09-23T19:54:28.370 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:28 smithi016 bash[75004]: cluster 2023-09-23T19:54:27.126182+0000 osd.5 (osd.5) 175 : cluster 3 failed to encode map e246 with expected crc 2023-09-23T19:54:28.370 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:28 smithi016 bash[75004]: cluster 2023-09-23T19:54:27.131877+0000 osd.7 (osd.7) 162 : cluster 3 failed to encode map e246 with expected crc 2023-09-23T19:54:28.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:28.371 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:54:28.472 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:28.501 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:28.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:28.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:28.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:28 smithi092 bash[58855]: cluster 2023-09-23T19:54:26.502292+0000 mgr.y (mgr.25325) 1414 : cluster 0 pgmap v1424: 89 pgs: 89 active+clean; 3.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2023-09-23T19:54:28.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:28 smithi092 bash[58855]: audit 2023-09-23T19:54:27.113354+0000 mon.a (mon.0) 1735 : audit 1 from='client.? 172.21.15.16:0/3980000507' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T19:54:28.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:28 smithi092 bash[58855]: cluster 2023-09-23T19:54:27.113425+0000 mon.a (mon.0) 1736 : cluster 0 osdmap e246: 8 total, 8 up, 8 in 2023-09-23T19:54:28.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:28 smithi092 bash[58855]: cluster 2023-09-23T19:54:27.117121+0000 osd.4 (osd.4) 189 : cluster 3 failed to encode map e246 with expected crc 2023-09-23T19:54:28.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:28 smithi092 bash[58855]: cluster 2023-09-23T19:54:27.126182+0000 osd.5 (osd.5) 175 : cluster 3 failed to encode map e246 with expected crc 2023-09-23T19:54:28.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:28 smithi092 bash[58855]: cluster 2023-09-23T19:54:27.131877+0000 osd.7 (osd.7) 162 : cluster 3 failed to encode map e246 with expected crc 2023-09-23T19:54:28.605 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:28.640 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:28.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:28.647 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:28.748 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:28.786 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:28.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:28.793 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:28.894 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:28.918 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:28.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:28.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:29.021 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:29.069 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:29.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:29.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:54:29.174 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:29.218 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:29.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:29.223 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:29.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:29 smithi092 bash[58855]: audit 2023-09-23T19:54:28.678875+0000 mon.a (mon.0) 1737 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:54:29.324 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:29.355 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:29.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:29.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:29.458 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:29.488 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:29.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:29.496 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:29.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:29 smithi016 bash[77995]: audit 2023-09-23T19:54:28.678875+0000 mon.a (mon.0) 1737 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:54:29.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:29 smithi016 bash[75004]: audit 2023-09-23T19:54:28.678875+0000 mon.a (mon.0) 1737 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:54:29.597 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:29.632 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:29.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:29.638 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:29.664 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T19:54:29.739 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:29.792 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:29.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:29.798 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:29.899 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:29.922 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:29.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:29.925 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:30.026 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:30.056 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:30.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:30.060 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:30.138 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:54:30.161 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:30.235 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:30.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:30.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:30.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:30 smithi092 bash[58855]: cluster 2023-09-23T19:54:28.502793+0000 mgr.y (mgr.25325) 1415 : cluster 0 pgmap v1426: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 535 op/s 2023-09-23T19:54:30.338 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:30.374 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:30.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:30.379 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:54:30.435 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:30 smithi016 bash[75004]: cluster 2023-09-23T19:54:28.502793+0000 mgr.y (mgr.25325) 1415 : cluster 0 pgmap v1426: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 535 op/s 2023-09-23T19:54:30.435 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:30 smithi016 bash[77995]: cluster 2023-09-23T19:54:28.502793+0000 mgr.y (mgr.25325) 1415 : cluster 0 pgmap v1426: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 535 op/s 2023-09-23T19:54:30.480 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:30.509 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:30.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:30.512 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:54:30.613 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:30.639 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:30.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:30.648 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:30.749 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:30.791 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:30.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:30.798 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:54:30.898 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:30.928 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:30.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:30.936 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:31.037 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:31.068 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:31.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:31.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:31.180 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:31.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:31.215 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:31.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:31.316 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:31.429 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:31.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:31.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:54:31.534 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:31.561 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:31.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:31.568 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:54:31.669 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:31.697 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:31.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:31.701 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:31.802 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:31.828 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:31.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:31.832 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:31.933 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:31.969 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:31.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:31.974 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:32.075 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:32.099 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:32.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:32.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:54:32.202 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:32.242 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:32.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:32.246 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:32.339 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:32 smithi016 bash[75004]: cluster 2023-09-23T19:54:30.503683+0000 mgr.y (mgr.25325) 1416 : cluster 0 pgmap v1427: 89 pgs: 89 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 729 op/s 2023-09-23T19:54:32.340 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:32 smithi016 bash[77995]: cluster 2023-09-23T19:54:30.503683+0000 mgr.y (mgr.25325) 1416 : cluster 0 pgmap v1427: 89 pgs: 89 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 729 op/s 2023-09-23T19:54:32.347 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:32.383 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:32.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:32.386 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:54:32.487 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:32.512 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:32.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:32.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:32.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:32 smithi092 bash[58855]: cluster 2023-09-23T19:54:30.503683+0000 mgr.y (mgr.25325) 1416 : cluster 0 pgmap v1427: 89 pgs: 89 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 729 op/s 2023-09-23T19:54:32.616 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:32.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:32.655 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:32.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:32.756 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:32.789 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:32.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:32.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:32.893 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:32.920 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:32.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:32.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:33.027 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:33.059 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:33.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:33.062 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:33.163 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:33.191 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:33.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:33.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:54:33.294 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:33.325 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:33.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:33.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:33.429 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:33.462 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:33.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:33.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:54:33.567 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:33.999 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:34.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:34.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:34.008 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:54:34.013 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:54:34.109 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:34.143 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:34.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:34.144 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:34.245 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:34.277 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:34.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:34.279 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:54:34.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:34 smithi016 bash[75004]: cluster 2023-09-23T19:54:32.504903+0000 mgr.y (mgr.25325) 1417 : cluster 0 pgmap v1428: 89 pgs: 89 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2023-09-23T19:54:34.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:34 smithi016 bash[77995]: cluster 2023-09-23T19:54:32.504903+0000 mgr.y (mgr.25325) 1417 : cluster 0 pgmap v1428: 89 pgs: 89 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2023-09-23T19:54:34.380 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:34.389 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:34 smithi092 bash[58855]: cluster 2023-09-23T19:54:32.504903+0000 mgr.y (mgr.25325) 1417 : cluster 0 pgmap v1428: 89 pgs: 89 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2023-09-23T19:54:34.417 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:34.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:34.420 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:34.520 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:34.547 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:34.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:34.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:34.651 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:34.684 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:34.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:34.686 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:34.787 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:34.812 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:34.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:34.817 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:34.918 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:34.953 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:34.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:34.956 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:35.057 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:35.087 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:35.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:35.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:35.189 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:35.221 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:35.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:35.224 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:35.324 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:35.364 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:35.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:35.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:54:35.468 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:35.502 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:35.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:35.507 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:35.607 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:35.638 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:35.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:35.643 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:35.743 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:35.793 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:35.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:35.796 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:35.897 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:35.906 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:54:35 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:54:35] "GET /metrics HTTP/1.1" 200 36304 "" "Prometheus/2.33.4" 2023-09-23T19:54:35.926 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:35.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:35.930 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:36.031 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:36.061 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:36.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:36.071 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:36.171 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:36.180 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:36 smithi016 bash[75004]: cluster 2023-09-23T19:54:34.505384+0000 mgr.y (mgr.25325) 1418 : cluster 0 pgmap v1429: 89 pgs: 89 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 676 op/s 2023-09-23T19:54:36.181 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:36 smithi016 bash[77995]: cluster 2023-09-23T19:54:34.505384+0000 mgr.y (mgr.25325) 1418 : cluster 0 pgmap v1429: 89 pgs: 89 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 676 op/s 2023-09-23T19:54:36.198 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:36.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:36.201 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:36.301 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:36.325 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:36.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:36.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:36.433 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:36.460 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:36.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:36.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:36.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:36 smithi092 bash[58855]: cluster 2023-09-23T19:54:34.505384+0000 mgr.y (mgr.25325) 1418 : cluster 0 pgmap v1429: 89 pgs: 89 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 676 op/s 2023-09-23T19:54:36.564 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:36.597 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:36.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:36.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:54:36.700 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:36.726 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:36.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:36.730 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:36.830 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:36.857 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:36.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:36.860 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:36.961 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:36.987 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:36.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:36.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:37.090 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:37.178 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:37.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:37.181 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:37.282 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:37.313 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:37.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:37.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:37.421 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:37.444 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:37.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:37.451 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:37.552 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:37.590 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:37.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:37.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:37.697 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:37.723 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:37.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:37.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:37.835 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:37.866 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:37.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:37.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:37.970 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:38.006 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:38.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:38.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:38.111 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:38.142 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:38.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:38.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:38.248 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:38.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:38 smithi016 bash[77995]: cluster 2023-09-23T19:54:36.506178+0000 mgr.y (mgr.25325) 1419 : cluster 0 pgmap v1430: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 668 op/s 2023-09-23T19:54:38.252 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:38 smithi016 bash[75004]: cluster 2023-09-23T19:54:36.506178+0000 mgr.y (mgr.25325) 1419 : cluster 0 pgmap v1430: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 668 op/s 2023-09-23T19:54:38.273 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:38.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:38.275 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:38.376 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:38.385 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:38 smithi092 bash[58855]: cluster 2023-09-23T19:54:36.506178+0000 mgr.y (mgr.25325) 1419 : cluster 0 pgmap v1430: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 668 op/s 2023-09-23T19:54:38.429 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:38.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:38.433 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:38.534 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:38.566 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:38.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:38.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:38.672 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:38.701 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:38.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:38.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:38.804 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:38.837 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:38.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:38.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:54:38.943 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:38.965 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:38.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:38.967 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:39.069 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:39.109 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:39.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:39.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:39.212 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:39.245 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:39.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:39.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:39.350 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:39.387 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:39.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:39.391 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:39.492 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:39.528 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:39.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:39.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:39.633 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:39.641 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-09-23T19:54:39.642 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:54:39.669 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:39.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:39.670 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:54:39.771 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:39.794 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:39.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:39.796 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:39.897 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:39.921 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:39.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:39.924 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:40.024 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:40.048 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:40.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:40.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:40.154 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:40.186 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:40.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:40.192 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:40.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:40 smithi016 bash[75004]: cluster 2023-09-23T19:54:38.506822+0000 mgr.y (mgr.25325) 1420 : cluster 0 pgmap v1431: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 605 op/s 2023-09-23T19:54:40.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:40 smithi016 bash[77995]: cluster 2023-09-23T19:54:38.506822+0000 mgr.y (mgr.25325) 1420 : cluster 0 pgmap v1431: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 605 op/s 2023-09-23T19:54:40.293 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:40.318 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:40.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:40.321 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:40.422 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:40.431 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:40 smithi092 bash[58855]: cluster 2023-09-23T19:54:38.506822+0000 mgr.y (mgr.25325) 1420 : cluster 0 pgmap v1431: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 605 op/s 2023-09-23T19:54:40.446 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:40.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:40.452 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:40.553 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:40.582 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:40.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:40.586 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:40.686 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:40.718 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:40.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:40.724 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:54:40.825 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:40.847 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:40.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:40.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:40.951 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:40.984 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:40.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:40.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:54:41.089 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:41.124 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:41.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:41.131 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:41.232 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:41.313 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:41.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:41.318 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:41.419 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:41.459 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:41.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:41.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:41.563 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:41.595 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:41.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:41.602 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:41.702 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:41.729 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:41.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:41.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:41.836 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:41.867 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:41.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:41.871 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:54:41.971 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:42.005 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:42.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:42.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:42.110 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:42.118 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T19:54:42.131 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:42.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:42.135 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:54:42.235 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:42.238 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:42 smithi016 bash[75004]: cluster 2023-09-23T19:54:40.507723+0000 mgr.y (mgr.25325) 1421 : cluster 0 pgmap v1432: 89 pgs: 89 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 766 op/s 2023-09-23T19:54:42.238 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:42 smithi016 bash[77995]: cluster 2023-09-23T19:54:40.507723+0000 mgr.y (mgr.25325) 1421 : cluster 0 pgmap v1432: 89 pgs: 89 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 766 op/s 2023-09-23T19:54:42.267 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:42.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:42.273 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:54:42.373 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:42.400 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:54:42.409 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:42.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:42.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:42.440 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:54:42.483 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:54:42.517 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:42.533 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:54:42.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:42 smithi092 bash[58855]: cluster 2023-09-23T19:54:40.507723+0000 mgr.y (mgr.25325) 1421 : cluster 0 pgmap v1432: 89 pgs: 89 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 766 op/s 2023-09-23T19:54:42.547 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:42.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:42.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:42.577 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:54:42.627 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:54:42.652 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:42.667 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:54:42.676 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:42.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:42.680 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:42.699 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:54:42.710 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:54:42.710 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:54:42.710 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:54:42.710 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:54:42.710 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:54:42.710 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:54:42.711 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:54:42.711 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:54:42.781 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:42.828 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:42.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:42.834 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:54:42.935 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:42.959 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:42.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:42.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:43.062 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:43.088 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:43.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:43.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:43.195 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:43.223 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:43.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:43.228 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:43.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:43 smithi016 bash[75004]: audit 2023-09-23T19:54:42.343289+0000 mon.c (mon.2) 222 : audit 0 from='client.? 172.21.15.16:0/2325765063' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:54:43.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:43 smithi016 bash[77995]: audit 2023-09-23T19:54:42.343289+0000 mon.c (mon.2) 222 : audit 0 from='client.? 172.21.15.16:0/2325765063' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:54:43.329 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:43.361 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:43.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:43.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:43.464 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:43.473 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:43 smithi092 bash[58855]: audit 2023-09-23T19:54:42.343289+0000 mon.c (mon.2) 222 : audit 0 from='client.? 172.21.15.16:0/2325765063' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:54:43.502 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:43.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:43.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:43.611 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:43.649 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:43.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:43.654 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:54:43.755 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:43.791 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:43.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:43.796 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:43.897 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:43.927 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:43.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:43.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:54:44.031 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:44.064 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:44.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:44.068 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:44.169 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:44.170 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:44 smithi016 bash[75004]: cluster 2023-09-23T19:54:42.508293+0000 mgr.y (mgr.25325) 1422 : cluster 0 pgmap v1433: 89 pgs: 89 active+clean; 3.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 608 op/s 2023-09-23T19:54:44.170 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:44 smithi016 bash[75004]: audit 2023-09-23T19:54:43.679238+0000 mon.a (mon.0) 1738 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:54:44.201 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:44.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:44.203 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:44.304 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:44.313 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:44 smithi092 bash[58855]: cluster 2023-09-23T19:54:42.508293+0000 mgr.y (mgr.25325) 1422 : cluster 0 pgmap v1433: 89 pgs: 89 active+clean; 3.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 608 op/s 2023-09-23T19:54:44.313 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:44 smithi092 bash[58855]: audit 2023-09-23T19:54:43.679238+0000 mon.a (mon.0) 1738 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:54:44.332 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:44.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:44.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:44.437 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:44.447 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:44 smithi016 bash[77995]: cluster 2023-09-23T19:54:42.508293+0000 mgr.y (mgr.25325) 1422 : cluster 0 pgmap v1433: 89 pgs: 89 active+clean; 3.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 608 op/s 2023-09-23T19:54:44.448 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:44 smithi016 bash[77995]: audit 2023-09-23T19:54:43.679238+0000 mon.a (mon.0) 1738 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:54:44.465 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:44.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:44.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:54:44.568 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:44.598 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:44.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:44.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:44.701 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:44.744 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:44.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:44.751 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:44.852 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:44.904 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:44.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:44.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:45.008 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:45.035 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:45.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:45.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:45.141 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:45.179 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:45.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:45.185 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:45.286 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:45.310 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:45.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:45.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:45.414 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:45.445 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:45.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:45.452 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:54:45.553 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:45.585 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:45.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:45.588 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:54:45.689 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:45.720 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:45.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:45.724 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:45.825 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:45.869 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:45.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:45.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:45.977 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:45.985 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:54:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:54:45] "GET /metrics HTTP/1.1" 200 36309 "" "Prometheus/2.33.4" 2023-09-23T19:54:46.003 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:46.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:46.008 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:46.109 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:46.137 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:46.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:46.140 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:46.240 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:46.270 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:46.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:46.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:46.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:46 smithi016 bash[77995]: cluster 2023-09-23T19:54:44.508737+0000 mgr.y (mgr.25325) 1423 : cluster 0 pgmap v1434: 89 pgs: 89 active+clean; 3.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 577 op/s 2023-09-23T19:54:46.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:46 smithi016 bash[75004]: cluster 2023-09-23T19:54:44.508737+0000 mgr.y (mgr.25325) 1423 : cluster 0 pgmap v1434: 89 pgs: 89 active+clean; 3.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 577 op/s 2023-09-23T19:54:46.372 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:46.403 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:46.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:46.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:54:46.508 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:46.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:46 smithi092 bash[58855]: cluster 2023-09-23T19:54:44.508737+0000 mgr.y (mgr.25325) 1423 : cluster 0 pgmap v1434: 89 pgs: 89 active+clean; 3.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 577 op/s 2023-09-23T19:54:46.540 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:46.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:46.543 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:46.643 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:46.672 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:46.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:46.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:46.780 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:46.821 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:46.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:46.824 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:54:46.925 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:46.954 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:46.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:46.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:47.057 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:47.088 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:47.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:47.090 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:54:47.191 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:47.239 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:47.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:47.241 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:47.342 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:47.368 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:54:47.380 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:47.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:47.381 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:54:47.482 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:47.692 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:47.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:47.701 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:47.801 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:47.835 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:47.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:47.841 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:54:47.942 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:47.984 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:47.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:47.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:48.089 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:48.121 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:48.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:48.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:48.224 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:48.266 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:48.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:48.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:48.372 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:48.381 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:48 smithi092 bash[58855]: cluster 2023-09-23T19:54:46.509502+0000 mgr.y (mgr.25325) 1424 : cluster 0 pgmap v1435: 89 pgs: 89 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 760 op/s 2023-09-23T19:54:48.398 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:48.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:48.400 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:48.462 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:48 smithi016 bash[77995]: cluster 2023-09-23T19:54:46.509502+0000 mgr.y (mgr.25325) 1424 : cluster 0 pgmap v1435: 89 pgs: 89 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 760 op/s 2023-09-23T19:54:48.462 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:48 smithi016 bash[75004]: cluster 2023-09-23T19:54:46.509502+0000 mgr.y (mgr.25325) 1424 : cluster 0 pgmap v1435: 89 pgs: 89 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 760 op/s 2023-09-23T19:54:48.501 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:48.535 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:48.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:48.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:48.642 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:48.666 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:48.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:48.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:48.770 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:48.798 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:48.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:48.805 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:54:48.895 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:54:48.905 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:48.943 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:48.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:48.949 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:49.049 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:49.085 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:49.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:49.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:49.189 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:49.190 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:49 smithi092 bash[58855]: audit 2023-09-23T19:54:49.083439+0000 mon.c (mon.2) 223 : audit 1 from='client.? 172.21.15.16:0/3971305789' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:54:49.191 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:49 smithi092 bash[58855]: audit 2023-09-23T19:54:49.084402+0000 mon.a (mon.0) 1739 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:54:49.191 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:49 smithi092 bash[58855]: audit 2023-09-23T19:54:49.120300+0000 mon.a (mon.0) 1740 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T19:54:49.191 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:49 smithi092 bash[58855]: audit 2023-09-23T19:54:49.125230+0000 mon.b (mon.1) 58 : audit 1 from='client.? 172.21.15.16:0/961734755' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T19:54:49.192 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:49 smithi016 bash[77995]: audit 2023-09-23T19:54:49.083439+0000 mon.c (mon.2) 223 : audit 1 from='client.? 172.21.15.16:0/3971305789' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:54:49.192 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:49 smithi016 bash[77995]: audit 2023-09-23T19:54:49.084402+0000 mon.a (mon.0) 1739 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:54:49.192 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:49 smithi016 bash[77995]: audit 2023-09-23T19:54:49.120300+0000 mon.a (mon.0) 1740 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T19:54:49.192 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:49 smithi016 bash[77995]: audit 2023-09-23T19:54:49.125230+0000 mon.b (mon.1) 58 : audit 1 from='client.? 172.21.15.16:0/961734755' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T19:54:49.212 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T19:54:49.212 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is unset 2023-09-23T19:54:49.215 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:49.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:49.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:49.319 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:49.354 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:49.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:49.358 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:49.459 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:49.485 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:49.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:49.489 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:49.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:49 smithi016 bash[75004]: audit 2023-09-23T19:54:49.083439+0000 mon.c (mon.2) 223 : audit 1 from='client.? 172.21.15.16:0/3971305789' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:54:49.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:49 smithi016 bash[75004]: audit 2023-09-23T19:54:49.084402+0000 mon.a (mon.0) 1739 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:54:49.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:49 smithi016 bash[75004]: audit 2023-09-23T19:54:49.120300+0000 mon.a (mon.0) 1740 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T19:54:49.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:49 smithi016 bash[75004]: audit 2023-09-23T19:54:49.125230+0000 mon.b (mon.1) 58 : audit 1 from='client.? 172.21.15.16:0/961734755' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T19:54:49.589 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:49.620 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:49.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:49.625 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:49.726 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:49.750 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:49.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:49.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:54:49.853 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:49.891 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:49.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:49.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:50.003 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:50.028 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:50.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:50.031 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:50.132 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:50.159 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:50.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:50.164 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:50.191 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:50 smithi092 bash[58855]: cluster 2023-09-23T19:54:48.509999+0000 mgr.y (mgr.25325) 1425 : cluster 0 pgmap v1436: 89 pgs: 89 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 621 op/s 2023-09-23T19:54:50.265 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:50.285 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:50.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:50.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:50.363 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:50 smithi016 bash[75004]: cluster 2023-09-23T19:54:48.509999+0000 mgr.y (mgr.25325) 1425 : cluster 0 pgmap v1436: 89 pgs: 89 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 621 op/s 2023-09-23T19:54:50.363 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:50 smithi016 bash[75004]: cluster 2023-09-23T19:54:49.172056+0000 mon.a (mon.0) 1741 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:54:50.363 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:50 smithi016 bash[75004]: cluster 2023-09-23T19:54:49.193900+0000 osd.4 (osd.4) 190 : cluster 3 failed to encode map e247 with expected crc 2023-09-23T19:54:50.363 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:50 smithi016 bash[75004]: audit 2023-09-23T19:54:49.194105+0000 mon.a (mon.0) 1742 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:54:50.363 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:50 smithi016 bash[75004]: audit 2023-09-23T19:54:49.194761+0000 mon.a (mon.0) 1743 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T19:54:50.363 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:50 smithi016 bash[75004]: cluster 2023-09-23T19:54:49.194939+0000 mon.a (mon.0) 1744 : cluster 0 osdmap e247: 8 total, 8 up, 8 in 2023-09-23T19:54:50.364 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:50 smithi016 bash[75004]: cluster 2023-09-23T19:54:49.196923+0000 osd.6 (osd.6) 182 : cluster 3 failed to encode map e247 with expected crc 2023-09-23T19:54:50.364 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:50 smithi016 bash[75004]: cluster 2023-09-23T19:54:49.210590+0000 osd.5 (osd.5) 176 : cluster 3 failed to encode map e247 with expected crc 2023-09-23T19:54:50.364 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:50 smithi016 bash[75004]: cluster 2023-09-23T19:54:49.212679+0000 osd.5 (osd.5) 177 : cluster 3 failed to encode map e247 with expected crc 2023-09-23T19:54:50.364 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:50 smithi016 bash[77995]: cluster 2023-09-23T19:54:48.509999+0000 mgr.y (mgr.25325) 1425 : cluster 0 pgmap v1436: 89 pgs: 89 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 621 op/s 2023-09-23T19:54:50.364 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:50 smithi016 bash[77995]: cluster 2023-09-23T19:54:49.172056+0000 mon.a (mon.0) 1741 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:54:50.364 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:50 smithi016 bash[77995]: cluster 2023-09-23T19:54:49.193900+0000 osd.4 (osd.4) 190 : cluster 3 failed to encode map e247 with expected crc 2023-09-23T19:54:50.364 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:50 smithi016 bash[77995]: audit 2023-09-23T19:54:49.194105+0000 mon.a (mon.0) 1742 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:54:50.364 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:50 smithi016 bash[77995]: audit 2023-09-23T19:54:49.194761+0000 mon.a (mon.0) 1743 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T19:54:50.365 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:50 smithi016 bash[77995]: cluster 2023-09-23T19:54:49.194939+0000 mon.a (mon.0) 1744 : cluster 0 osdmap e247: 8 total, 8 up, 8 in 2023-09-23T19:54:50.365 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:50 smithi016 bash[77995]: cluster 2023-09-23T19:54:49.196923+0000 osd.6 (osd.6) 182 : cluster 3 failed to encode map e247 with expected crc 2023-09-23T19:54:50.365 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:50 smithi016 bash[77995]: cluster 2023-09-23T19:54:49.210590+0000 osd.5 (osd.5) 176 : cluster 3 failed to encode map e247 with expected crc 2023-09-23T19:54:50.365 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:50 smithi016 bash[77995]: cluster 2023-09-23T19:54:49.212679+0000 osd.5 (osd.5) 177 : cluster 3 failed to encode map e247 with expected crc 2023-09-23T19:54:50.392 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:50.444 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:50.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:50.449 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:50 smithi092 bash[58855]: cluster 2023-09-23T19:54:49.172056+0000 mon.a (mon.0) 1741 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:54:50.449 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:50 smithi092 bash[58855]: cluster 2023-09-23T19:54:49.193900+0000 osd.4 (osd.4) 190 : cluster 3 failed to encode map e247 with expected crc 2023-09-23T19:54:50.449 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:50 smithi092 bash[58855]: audit 2023-09-23T19:54:49.194105+0000 mon.a (mon.0) 1742 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:54:50.449 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:50 smithi092 bash[58855]: audit 2023-09-23T19:54:49.194761+0000 mon.a (mon.0) 1743 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T19:54:50.450 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:50 smithi092 bash[58855]: cluster 2023-09-23T19:54:49.194939+0000 mon.a (mon.0) 1744 : cluster 0 osdmap e247: 8 total, 8 up, 8 in 2023-09-23T19:54:50.450 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:50 smithi092 bash[58855]: cluster 2023-09-23T19:54:49.196923+0000 osd.6 (osd.6) 182 : cluster 3 failed to encode map e247 with expected crc 2023-09-23T19:54:50.450 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:50 smithi092 bash[58855]: cluster 2023-09-23T19:54:49.210590+0000 osd.5 (osd.5) 176 : cluster 3 failed to encode map e247 with expected crc 2023-09-23T19:54:50.450 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:50 smithi092 bash[58855]: cluster 2023-09-23T19:54:49.212679+0000 osd.5 (osd.5) 177 : cluster 3 failed to encode map e247 with expected crc 2023-09-23T19:54:50.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:50.551 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:50.577 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:50.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:50.581 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:50.682 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:50.706 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:50.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:50.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:54:50.811 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:50.845 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:50.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:50.849 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:50.950 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:50.985 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:50.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:50.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:51.092 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:51.135 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:51.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:51.139 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:51.240 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:51.278 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:51.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:51.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:51.385 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:51.418 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:51.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:51.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:54:51.521 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:51.551 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:51.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:51.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:51.655 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:51.677 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:51.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:51.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:51.780 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:51.812 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:51.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:51.816 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:51.917 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:51.948 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:51.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:51.953 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:52.053 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:52.088 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:52.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:52.095 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:52.195 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:52.199 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:52 smithi016 bash[75004]: cluster 2023-09-23T19:54:50.510646+0000 mgr.y (mgr.25325) 1426 : cluster 0 pgmap v1438: 89 pgs: 89 active+clean; 2.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2023-09-23T19:54:52.199 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:52 smithi016 bash[77995]: cluster 2023-09-23T19:54:50.510646+0000 mgr.y (mgr.25325) 1426 : cluster 0 pgmap v1438: 89 pgs: 89 active+clean; 2.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2023-09-23T19:54:52.229 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:52.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:52.237 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:54:52.338 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:52.347 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:52 smithi092 bash[58855]: cluster 2023-09-23T19:54:50.510646+0000 mgr.y (mgr.25325) 1426 : cluster 0 pgmap v1438: 89 pgs: 89 active+clean; 2.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2023-09-23T19:54:52.371 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:52.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:52.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:52.477 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:52.511 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:52.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:52.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:54:52.617 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:52.640 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:52.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:52.643 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:52.744 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:52.786 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:52.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:52.787 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:54:52.888 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:52.928 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:52.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:52.937 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:53.028 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T19:54:53.038 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:53.115 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:53.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:53.116 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:53.217 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:53.253 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:53.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:53.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:53.357 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:53.406 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:53.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:53.409 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:53.510 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:53.543 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:53.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:53.545 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:53.646 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:53.672 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:53.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:53.676 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:53.777 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:53.803 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:53.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:53.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:54:53.909 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:53.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:53.934 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:53.934 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:54.035 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:54.062 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:54.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:54.069 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:54.169 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:54.201 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:54.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:54.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:54.308 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:54.314 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:54 smithi016 bash[75004]: cluster 2023-09-23T19:54:52.511165+0000 mgr.y (mgr.25325) 1427 : cluster 0 pgmap v1439: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2023-09-23T19:54:54.314 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:54 smithi016 bash[77995]: cluster 2023-09-23T19:54:52.511165+0000 mgr.y (mgr.25325) 1427 : cluster 0 pgmap v1439: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2023-09-23T19:54:54.340 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:54.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:54.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:54.448 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:54.457 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:54 smithi092 bash[58855]: cluster 2023-09-23T19:54:52.511165+0000 mgr.y (mgr.25325) 1427 : cluster 0 pgmap v1439: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2023-09-23T19:54:54.491 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:54.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:54.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:54:54.594 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:54.630 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:54.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:54.636 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:54.737 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:54.772 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:54.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:54.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:54.883 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:54.917 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:54.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:54.923 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:54.994 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T19:54:55.023 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:55.061 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:55.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:55.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:55.165 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:55.195 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:55.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:55.205 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:55.305 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:55.339 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:55.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:55.342 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:54:55.442 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:55.468 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:55.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:55.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:55.529 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:54:55.572 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:55.615 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:55.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:55.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:55.719 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:55.729 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:54:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:54:55] "GET /metrics HTTP/1.1" 200 36309 "" "Prometheus/2.33.4" 2023-09-23T19:54:55.759 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:55.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:55.764 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:55.865 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:55.900 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:55.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:55.904 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:56.005 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:56.035 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:56.040 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-09-23T19:54:56.040 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [], 2023-09-23T19:54:56.040 INFO:teuthology.orchestra.run.smithi016.stdout: "complaint_time": 30, 2023-09-23T19:54:56.040 INFO:teuthology.orchestra.run.smithi016.stdout: "num_blocked_ops": 0 2023-09-23T19:54:56.040 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-09-23T19:54:56.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:56.041 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:56.142 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:56.176 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:56.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:56.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:56.280 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:56.307 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:56.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:56.312 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:54:56.412 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:56.421 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:56 smithi016 bash[75004]: cluster 2023-09-23T19:54:54.511671+0000 mgr.y (mgr.25325) 1428 : cluster 0 pgmap v1440: 89 pgs: 89 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2023-09-23T19:54:56.421 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:56 smithi016 bash[77995]: cluster 2023-09-23T19:54:54.511671+0000 mgr.y (mgr.25325) 1428 : cluster 0 pgmap v1440: 89 pgs: 89 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2023-09-23T19:54:56.440 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:56.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:56.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:56.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:56 smithi092 bash[58855]: cluster 2023-09-23T19:54:54.511671+0000 mgr.y (mgr.25325) 1428 : cluster 0 pgmap v1440: 89 pgs: 89 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2023-09-23T19:54:56.544 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:56.589 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:56.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:56.592 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:56.693 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:56.721 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:56.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:56.724 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:56.824 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:56.866 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:56.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:56.872 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:56.973 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:56.998 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:57.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:57.006 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:57.106 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:57.139 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:57.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:57.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:57.242 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:57.280 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:57.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:57.286 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:57.386 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:57.420 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:57.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:57.424 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:57.525 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:57.553 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:57.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:57.556 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:54:57.657 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:57.683 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:57.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:57.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:57.786 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:57.814 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:57.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:57.818 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:57.919 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:57.948 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:57.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:57.954 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:58.055 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:58.074 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:58.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:58.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:54:58.177 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:58.209 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:58 smithi092 bash[58855]: cluster 2023-09-23T19:54:56.512444+0000 mgr.y (mgr.25325) 1429 : cluster 0 pgmap v1441: 89 pgs: 89 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2023-09-23T19:54:58.217 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:58.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:58.221 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:58.322 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:58.357 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:58.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:58.363 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:58.464 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:58.474 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:58 smithi016 bash[77995]: cluster 2023-09-23T19:54:56.512444+0000 mgr.y (mgr.25325) 1429 : cluster 0 pgmap v1441: 89 pgs: 89 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2023-09-23T19:54:58.474 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:58 smithi016 bash[75004]: cluster 2023-09-23T19:54:56.512444+0000 mgr.y (mgr.25325) 1429 : cluster 0 pgmap v1441: 89 pgs: 89 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2023-09-23T19:54:58.486 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T19:54:58.489 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:58.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:58.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:58.592 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:58.628 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:58.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:58.633 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:58.734 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:58.766 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:58.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:58.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:58.874 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:58.902 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:58.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:58.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:54:58.950 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:54:59.007 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:59.042 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:59.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:59.044 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:59.145 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:59.179 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:59.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:59.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:59.286 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:59.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:54:59 smithi092 bash[58855]: audit 2023-09-23T19:54:58.684566+0000 mon.a (mon.0) 1745 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:54:59.312 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:59.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:59.318 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:59.418 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:59.441 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:59.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:59.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:59.502 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:54:59 smithi016 bash[75004]: audit 2023-09-23T19:54:58.684566+0000 mon.a (mon.0) 1745 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:54:59.502 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:54:59 smithi016 bash[77995]: audit 2023-09-23T19:54:58.684566+0000 mon.a (mon.0) 1745 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:54:59.544 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:59.576 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:59.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:59.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:59.681 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:59.706 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:59.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:59.709 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:59.809 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:59.841 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:59.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:59.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:59.945 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:59.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:54:59.988 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:59.988 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:00.089 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:00.125 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:00.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:00.135 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:55:00.235 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:00.260 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:00.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:00.266 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:00.367 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:00.394 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:00.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:00.398 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:00.498 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:00.506 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:00 smithi016 bash[77995]: cluster 2023-09-23T19:54:58.513052+0000 mgr.y (mgr.25325) 1430 : cluster 0 pgmap v1442: 89 pgs: 89 active+clean; 2.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 666 op/s 2023-09-23T19:55:00.506 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:00 smithi016 bash[75004]: cluster 2023-09-23T19:54:58.513052+0000 mgr.y (mgr.25325) 1430 : cluster 0 pgmap v1442: 89 pgs: 89 active+clean; 2.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 666 op/s 2023-09-23T19:55:00.529 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:00.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:00.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:00.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:00 smithi092 bash[58855]: cluster 2023-09-23T19:54:58.513052+0000 mgr.y (mgr.25325) 1430 : cluster 0 pgmap v1442: 89 pgs: 89 active+clean; 2.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 666 op/s 2023-09-23T19:55:00.636 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:00.665 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:00.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:00.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:00.769 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:00.801 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:00.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:00.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:00.908 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:00.935 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:00.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:00.938 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:01.039 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:01.057 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:01.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:01.060 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:01.161 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:01.201 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:01.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:01.208 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:01.309 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:01.319 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:55:01.337 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:01.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:01.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:01.441 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:01.470 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:01.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:01.472 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:01.573 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:01.628 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:01.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:01.633 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:55:01.734 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:01.766 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:01.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:01.772 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:01.873 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:01.894 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:01.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:01.897 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:55:01.998 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:02.046 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:02.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:02.050 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:02.151 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:02.175 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:02.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:02.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:02.218 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:02 smithi016 bash[75004]: cluster 2023-09-23T19:55:00.514193+0000 mgr.y (mgr.25325) 1431 : cluster 0 pgmap v1443: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 806 op/s 2023-09-23T19:55:02.278 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:02.310 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:02.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:02.314 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:02.415 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:02.448 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:02.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:02.450 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:02.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:02 smithi016 bash[77995]: cluster 2023-09-23T19:55:00.514193+0000 mgr.y (mgr.25325) 1431 : cluster 0 pgmap v1443: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 806 op/s 2023-09-23T19:55:02.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:02 smithi092 bash[58855]: cluster 2023-09-23T19:55:00.514193+0000 mgr.y (mgr.25325) 1431 : cluster 0 pgmap v1443: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 806 op/s 2023-09-23T19:55:02.551 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:02.574 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:02.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:02.576 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:02.677 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:02.720 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:02.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:02.724 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:02.825 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:02.859 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:02.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:02.865 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:02.966 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:02.998 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:03.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:03.005 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:03.106 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:03.149 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:03.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:03.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:03.252 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:03.274 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:03.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:03.278 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:03.347 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:55:03.379 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:03.419 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:03.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:03.424 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:03.525 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:03.556 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:03.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:03.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:03.664 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:03.699 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:03.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:03.705 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:03.806 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:03.831 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:03.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:03.832 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:03.933 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:03.962 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:03.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:03.966 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:04.067 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:04.111 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:04.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:04.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:04.213 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:04.240 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:04.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:04.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:04.345 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:04.372 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:04.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:04.377 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:04.478 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:04.488 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:04 smithi016 bash[77995]: cluster 2023-09-23T19:55:02.514870+0000 mgr.y (mgr.25325) 1432 : cluster 0 pgmap v1444: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-09-23T19:55:04.488 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:04 smithi016 bash[75004]: cluster 2023-09-23T19:55:02.514870+0000 mgr.y (mgr.25325) 1432 : cluster 0 pgmap v1444: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-09-23T19:55:04.512 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:04.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:04.520 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:04.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:04 smithi092 bash[58855]: cluster 2023-09-23T19:55:02.514870+0000 mgr.y (mgr.25325) 1432 : cluster 0 pgmap v1444: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-09-23T19:55:04.620 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:04.648 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:04.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:04.655 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:04.755 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:04.785 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:04.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:04.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:04.890 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:04.938 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:04.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:04.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:05.047 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:05.079 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:05.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:05.084 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:05.185 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:05.222 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:05.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:05.228 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:05.329 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:05.354 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:05.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:05.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:05.457 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:05.492 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:05.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:05.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:05.600 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:05.634 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:05.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:05.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:05.737 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:05.767 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:05.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:05.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:05.872 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:05.895 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:05.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:05.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:05.988 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:55:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:55:05] "GET /metrics HTTP/1.1" 200 36302 "" "Prometheus/2.33.4" 2023-09-23T19:55:05.997 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:06.034 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:06.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:06.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:06.139 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:06.163 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:06.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:06.169 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:55:06.269 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:06.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:06 smithi016 bash[77995]: cluster 2023-09-23T19:55:04.515343+0000 mgr.y (mgr.25325) 1433 : cluster 0 pgmap v1445: 89 pgs: 89 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 565 op/s 2023-09-23T19:55:06.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:06 smithi016 bash[75004]: cluster 2023-09-23T19:55:04.515343+0000 mgr.y (mgr.25325) 1433 : cluster 0 pgmap v1445: 89 pgs: 89 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 565 op/s 2023-09-23T19:55:06.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:06.318 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:06.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:06.419 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:06.451 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:06.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:06.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:06.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:06 smithi092 bash[58855]: cluster 2023-09-23T19:55:04.515343+0000 mgr.y (mgr.25325) 1433 : cluster 0 pgmap v1445: 89 pgs: 89 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 565 op/s 2023-09-23T19:55:06.554 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:06.600 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:06.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:06.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:06.707 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:06.732 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:06.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:06.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:06.836 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:06.870 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:06.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:06.879 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:06.979 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:07.011 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:07.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:07.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:07.117 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:07.162 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:07.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:07.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:07.266 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:07.303 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:07.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:07.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:07.409 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:07.443 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:07.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:07.448 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:07.449 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:55:07.493 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:55:07.526 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:55:07.549 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:07.580 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:07.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:07.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:07.605 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:55:07.634 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:55:07.672 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:55:07.686 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:07.704 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:55:07.712 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:07.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:07.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:07.730 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:55:07.733 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:55:07.733 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:55:07.733 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:55:07.733 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:55:07.733 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:55:07.733 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:55:07.733 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:55:07.733 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:55:07.817 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:07.851 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:07.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:07.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:07.955 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:07.986 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:07.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:07.989 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:08.090 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:08.133 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:08.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:08.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:08.236 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:08.258 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:08.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:08.265 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:08.365 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:08.394 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:08.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:08.396 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:08.497 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:08.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:08 smithi016 bash[77995]: cluster 2023-09-23T19:55:06.516208+0000 mgr.y (mgr.25325) 1434 : cluster 0 pgmap v1446: 89 pgs: 89 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 755 op/s 2023-09-23T19:55:08.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:08 smithi016 bash[77995]: audit 2023-09-23T19:55:07.413099+0000 mon.a (mon.0) 1746 : audit 0 from='client.? 172.21.15.16:0/3057048622' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:55:08.505 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:08 smithi016 bash[77995]: audit 2023-09-23T19:55:07.907066+0000 mon.a (mon.0) 1747 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:55:08.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:08 smithi016 bash[75004]: cluster 2023-09-23T19:55:06.516208+0000 mgr.y (mgr.25325) 1434 : cluster 0 pgmap v1446: 89 pgs: 89 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 755 op/s 2023-09-23T19:55:08.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:08 smithi016 bash[75004]: audit 2023-09-23T19:55:07.413099+0000 mon.a (mon.0) 1746 : audit 0 from='client.? 172.21.15.16:0/3057048622' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:55:08.505 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:08 smithi016 bash[75004]: audit 2023-09-23T19:55:07.907066+0000 mon.a (mon.0) 1747 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:55:08.520 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:08.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:08.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:08.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:08 smithi092 bash[58855]: cluster 2023-09-23T19:55:06.516208+0000 mgr.y (mgr.25325) 1434 : cluster 0 pgmap v1446: 89 pgs: 89 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 755 op/s 2023-09-23T19:55:08.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:08 smithi092 bash[58855]: audit 2023-09-23T19:55:07.413099+0000 mon.a (mon.0) 1746 : audit 0 from='client.? 172.21.15.16:0/3057048622' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:55:08.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:08 smithi092 bash[58855]: audit 2023-09-23T19:55:07.907066+0000 mon.a (mon.0) 1747 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:55:08.625 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:08.648 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:08.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:08.651 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:08.751 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:08.796 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:08.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:08.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:08.901 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:08.929 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:08.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:08.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:09.034 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:09.066 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:09.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:09.073 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:09.174 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:09.204 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:09.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:09.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:09.309 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:09.342 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:09.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:09.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:09.446 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:09.478 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:09.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:09.482 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:09.582 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:09.625 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:09.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:09.628 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:09.729 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:09.755 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:09.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:09.762 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:09.862 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:09.891 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:09.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:09.898 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:09.998 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:10.024 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:10.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:10.027 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:10.128 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:10.156 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:10.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:10.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:10.263 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:10.302 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:10.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:10.306 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:10.407 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:10.437 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:10.441 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:10 smithi016 bash[77995]: cluster 2023-09-23T19:55:08.516711+0000 mgr.y (mgr.25325) 1435 : cluster 0 pgmap v1447: 89 pgs: 89 active+clean; 2.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 622 op/s 2023-09-23T19:55:10.441 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:10 smithi016 bash[75004]: cluster 2023-09-23T19:55:08.516711+0000 mgr.y (mgr.25325) 1435 : cluster 0 pgmap v1447: 89 pgs: 89 active+clean; 2.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 622 op/s 2023-09-23T19:55:10.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:10.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:10.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:10 smithi092 bash[58855]: cluster 2023-09-23T19:55:08.516711+0000 mgr.y (mgr.25325) 1435 : cluster 0 pgmap v1447: 89 pgs: 89 active+clean; 2.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 622 op/s 2023-09-23T19:55:10.543 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:10.586 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:10.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:10.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:10.692 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:10.728 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:10.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:10.732 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:10.833 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:10.859 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:10.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:10.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:10.964 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:11.002 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:11.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:11.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:11.108 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:11.136 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:11.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:11.142 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:11.243 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:11.272 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:11.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:11.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:11.377 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:11.415 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:11.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:11.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:11.520 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:11.544 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:11.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:11.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:11.652 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:11.685 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:11.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:11.687 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:11.788 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:11.819 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:11.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:11.825 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:11.926 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:11.970 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:11.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:11.975 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:12.076 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:12.114 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:12.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:12.117 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:12.218 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:12.255 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:12.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:12.257 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:12.337 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:12 smithi016 bash[75004]: cluster 2023-09-23T19:55:10.517327+0000 mgr.y (mgr.25325) 1436 : cluster 0 pgmap v1448: 89 pgs: 89 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 753 op/s 2023-09-23T19:55:12.337 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:12 smithi016 bash[77995]: cluster 2023-09-23T19:55:10.517327+0000 mgr.y (mgr.25325) 1436 : cluster 0 pgmap v1448: 89 pgs: 89 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 753 op/s 2023-09-23T19:55:12.358 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:12.391 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:12.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:12.398 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:12.499 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:12.526 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:12.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:12.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:12.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:12 smithi092 bash[58855]: cluster 2023-09-23T19:55:10.517327+0000 mgr.y (mgr.25325) 1436 : cluster 0 pgmap v1448: 89 pgs: 89 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 753 op/s 2023-09-23T19:55:12.630 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:12.653 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:12.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:12.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:12.758 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:12.788 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:12.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:12.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:12.897 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:12.941 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:12.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:12.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:13.046 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:13.098 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:13.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:13.104 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:13.205 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:13.259 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:13.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:13.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:13.364 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:13.416 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:13.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:13.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:13.524 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:13.579 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:13.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:13.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:55:13.684 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:13.723 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:13.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:13.726 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:13.827 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:13.851 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:13.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:13.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:13.955 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:14.035 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:14.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:14.041 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:14.141 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:14.167 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:14.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:14.169 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:14.258 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T19:55:14.270 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:14.293 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:14.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:14.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:14.397 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:14.406 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:14 smithi016 bash[77995]: cluster 2023-09-23T19:55:12.517830+0000 mgr.y (mgr.25325) 1437 : cluster 0 pgmap v1449: 89 pgs: 89 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-09-23T19:55:14.406 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:14 smithi016 bash[77995]: audit 2023-09-23T19:55:13.683631+0000 mon.a (mon.0) 1748 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:55:14.406 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:14 smithi016 bash[77995]: audit 2023-09-23T19:55:13.777261+0000 mon.a (mon.0) 1749 : audit 1 from='client.? 172.21.15.16:0/3332672777' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T19:55:14.406 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:14 smithi016 bash[75004]: cluster 2023-09-23T19:55:12.517830+0000 mgr.y (mgr.25325) 1437 : cluster 0 pgmap v1449: 89 pgs: 89 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-09-23T19:55:14.406 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:14 smithi016 bash[75004]: audit 2023-09-23T19:55:13.683631+0000 mon.a (mon.0) 1748 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:55:14.406 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:14 smithi016 bash[75004]: audit 2023-09-23T19:55:13.777261+0000 mon.a (mon.0) 1749 : audit 1 from='client.? 172.21.15.16:0/3332672777' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T19:55:14.427 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:14.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:14.430 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:14.530 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:14.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:14 smithi092 bash[58855]: cluster 2023-09-23T19:55:12.517830+0000 mgr.y (mgr.25325) 1437 : cluster 0 pgmap v1449: 89 pgs: 89 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-09-23T19:55:14.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:14 smithi092 bash[58855]: audit 2023-09-23T19:55:13.683631+0000 mon.a (mon.0) 1748 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:55:14.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:14 smithi092 bash[58855]: audit 2023-09-23T19:55:13.777261+0000 mon.a (mon.0) 1749 : audit 1 from='client.? 172.21.15.16:0/3332672777' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T19:55:14.569 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:14.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:14.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:55:14.580 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:55:14.675 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:14.708 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:14.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:14.710 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:14.811 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:14.838 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:14.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:14.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:14.942 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:14.972 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:14.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:14.981 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:15.082 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:15.113 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:15.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:15.115 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:15.215 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:15.264 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:15.267 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:15 smithi016 bash[75004]: audit 2023-09-23T19:55:14.246358+0000 mon.a (mon.0) 1750 : audit 1 from='client.? 172.21.15.16:0/3332672777' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T19:55:15.267 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:15 smithi016 bash[75004]: cluster 2023-09-23T19:55:14.246462+0000 mon.a (mon.0) 1751 : cluster 0 osdmap e248: 8 total, 8 up, 8 in 2023-09-23T19:55:15.267 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:15 smithi016 bash[75004]: cluster 2023-09-23T19:55:14.265494+0000 osd.4 (osd.4) 191 : cluster 3 failed to encode map e248 with expected crc 2023-09-23T19:55:15.267 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:15 smithi016 bash[75004]: cluster 2023-09-23T19:55:14.272544+0000 osd.6 (osd.6) 183 : cluster 3 failed to encode map e248 with expected crc 2023-09-23T19:55:15.267 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:15 smithi016 bash[75004]: cluster 2023-09-23T19:55:14.279280+0000 osd.7 (osd.7) 163 : cluster 3 failed to encode map e248 with expected crc 2023-09-23T19:55:15.267 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:15 smithi016 bash[75004]: audit 2023-09-23T19:55:14.759726+0000 mon.a (mon.0) 1752 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:55:15.267 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:15 smithi016 bash[75004]: audit 2023-09-23T19:55:14.771582+0000 mon.a (mon.0) 1753 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:55:15.267 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:15 smithi016 bash[77995]: audit 2023-09-23T19:55:14.246358+0000 mon.a (mon.0) 1750 : audit 1 from='client.? 172.21.15.16:0/3332672777' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T19:55:15.267 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:15 smithi016 bash[77995]: cluster 2023-09-23T19:55:14.246462+0000 mon.a (mon.0) 1751 : cluster 0 osdmap e248: 8 total, 8 up, 8 in 2023-09-23T19:55:15.268 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:15 smithi016 bash[77995]: cluster 2023-09-23T19:55:14.265494+0000 osd.4 (osd.4) 191 : cluster 3 failed to encode map e248 with expected crc 2023-09-23T19:55:15.268 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:15 smithi016 bash[77995]: cluster 2023-09-23T19:55:14.272544+0000 osd.6 (osd.6) 183 : cluster 3 failed to encode map e248 with expected crc 2023-09-23T19:55:15.268 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:15 smithi016 bash[77995]: cluster 2023-09-23T19:55:14.279280+0000 osd.7 (osd.7) 163 : cluster 3 failed to encode map e248 with expected crc 2023-09-23T19:55:15.268 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:15 smithi016 bash[77995]: audit 2023-09-23T19:55:14.759726+0000 mon.a (mon.0) 1752 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:55:15.268 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:15 smithi016 bash[77995]: audit 2023-09-23T19:55:14.771582+0000 mon.a (mon.0) 1753 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:55:15.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:15.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:15.376 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:15.405 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:15.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:15.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:15.509 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:15.517 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:15 smithi092 bash[58855]: audit 2023-09-23T19:55:14.246358+0000 mon.a (mon.0) 1750 : audit 1 from='client.? 172.21.15.16:0/3332672777' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T19:55:15.517 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:15 smithi092 bash[58855]: cluster 2023-09-23T19:55:14.246462+0000 mon.a (mon.0) 1751 : cluster 0 osdmap e248: 8 total, 8 up, 8 in 2023-09-23T19:55:15.517 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:15 smithi092 bash[58855]: cluster 2023-09-23T19:55:14.265494+0000 osd.4 (osd.4) 191 : cluster 3 failed to encode map e248 with expected crc 2023-09-23T19:55:15.518 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:15 smithi092 bash[58855]: cluster 2023-09-23T19:55:14.272544+0000 osd.6 (osd.6) 183 : cluster 3 failed to encode map e248 with expected crc 2023-09-23T19:55:15.518 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:15 smithi092 bash[58855]: cluster 2023-09-23T19:55:14.279280+0000 osd.7 (osd.7) 163 : cluster 3 failed to encode map e248 with expected crc 2023-09-23T19:55:15.518 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:15 smithi092 bash[58855]: audit 2023-09-23T19:55:14.759726+0000 mon.a (mon.0) 1752 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:55:15.518 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:15 smithi092 bash[58855]: audit 2023-09-23T19:55:14.771582+0000 mon.a (mon.0) 1753 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:55:15.555 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:15.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:15.560 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:55:15.640 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:55:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:55:15] "GET /metrics HTTP/1.1" 200 36307 "" "Prometheus/2.33.4" 2023-09-23T19:55:15.660 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:15.687 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:15.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:15.692 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:15.793 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:15.818 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:15.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:15.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:15.922 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:15.958 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:15.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:15.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:16.065 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:16.098 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:16.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:16.104 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:16.205 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:16.234 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:16.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:16.239 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:16.340 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:16.347 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:16 smithi016 bash[77995]: cluster 2023-09-23T19:55:14.518276+0000 mgr.y (mgr.25325) 1438 : cluster 0 pgmap v1451: 89 pgs: 89 active+clean; 1.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2023-09-23T19:55:16.348 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:16 smithi016 bash[75004]: cluster 2023-09-23T19:55:14.518276+0000 mgr.y (mgr.25325) 1438 : cluster 0 pgmap v1451: 89 pgs: 89 active+clean; 1.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2023-09-23T19:55:16.362 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:16.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:16.365 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:16.466 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:16.474 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:16 smithi092 bash[58855]: cluster 2023-09-23T19:55:14.518276+0000 mgr.y (mgr.25325) 1438 : cluster 0 pgmap v1451: 89 pgs: 89 active+clean; 1.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2023-09-23T19:55:16.491 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:16.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:16.498 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:16.599 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:16.631 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:16.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:16.637 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:16.738 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:16.759 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:16.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:16.766 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:16.867 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:16.901 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:16.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:16.906 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:17.006 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:17.041 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:17.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:17.043 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:17.143 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:17.166 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:17.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:17.172 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:17.273 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:17.302 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:17.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:17.305 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:17.406 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:17.438 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:17.441 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T19:55:17.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:17.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:17.543 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:17.572 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:17.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:17.578 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:55:17.678 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:17.714 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:17.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:17.716 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:17.817 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:17.843 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:17.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:17.848 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:17.949 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:17.981 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:17.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:17.990 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:18.090 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:18.127 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:18.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:18.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:18.231 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:18.262 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:18.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:18.264 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:55:18.365 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:18.373 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:18 smithi016 bash[75004]: cluster 2023-09-23T19:55:16.518821+0000 mgr.y (mgr.25325) 1439 : cluster 0 pgmap v1452: 89 pgs: 89 active+clean; 1.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 657 op/s 2023-09-23T19:55:18.373 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:18 smithi016 bash[77995]: cluster 2023-09-23T19:55:16.518821+0000 mgr.y (mgr.25325) 1439 : cluster 0 pgmap v1452: 89 pgs: 89 active+clean; 1.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 657 op/s 2023-09-23T19:55:18.378 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-09-23T19:55:18.378 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:55:18.405 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:18.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:18.414 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:18.515 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:18.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:18 smithi092 bash[58855]: cluster 2023-09-23T19:55:16.518821+0000 mgr.y (mgr.25325) 1439 : cluster 0 pgmap v1452: 89 pgs: 89 active+clean; 1.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 657 op/s 2023-09-23T19:55:18.542 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:18.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:18.545 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:18.645 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:18.684 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:18.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:18.689 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:55:18.789 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:18.830 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:18.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:18.833 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:55:18.934 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:18.964 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:18.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:18.972 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:19.072 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:19.110 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:19.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:19.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:19.219 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:19.257 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:19.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:19.260 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:19.277 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:19 smithi016 bash[77995]: audit 2023-09-23T19:55:18.524169+0000 mon.a (mon.0) 1754 : audit 1 from='client.? 172.21.15.16:0/3478923642' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:55:19.277 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:19 smithi016 bash[75004]: audit 2023-09-23T19:55:18.524169+0000 mon.a (mon.0) 1754 : audit 1 from='client.? 172.21.15.16:0/3478923642' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:55:19.299 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is unset 2023-09-23T19:55:19.361 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:19.390 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:19.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:19.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:19.495 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:19.522 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:19.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:19.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:19.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:19 smithi092 bash[58855]: audit 2023-09-23T19:55:18.524169+0000 mon.a (mon.0) 1754 : audit 1 from='client.? 172.21.15.16:0/3478923642' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:55:19.626 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:19.646 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T19:55:19.663 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:19.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:19.667 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:19.768 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:19.801 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:19.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:19.805 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:19.906 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:19.935 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:19.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:19.940 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:20.041 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:20.071 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:20.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:20.078 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:20.178 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:20.213 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:20.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:20.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:20.280 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:20 smithi016 bash[75004]: cluster 2023-09-23T19:55:18.519372+0000 mgr.y (mgr.25325) 1440 : cluster 0 pgmap v1453: 89 pgs: 89 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 653 op/s 2023-09-23T19:55:20.280 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:20 smithi016 bash[77995]: cluster 2023-09-23T19:55:18.519372+0000 mgr.y (mgr.25325) 1440 : cluster 0 pgmap v1453: 89 pgs: 89 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 653 op/s 2023-09-23T19:55:20.280 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:20 smithi016 bash[77995]: cluster 2023-09-23T19:55:19.268591+0000 mon.a (mon.0) 1755 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:55:20.280 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:20 smithi016 bash[77995]: cluster 2023-09-23T19:55:19.268638+0000 mon.a (mon.0) 1756 : cluster 1 Cluster is now healthy 2023-09-23T19:55:20.280 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:20 smithi016 bash[77995]: audit 2023-09-23T19:55:19.290100+0000 mon.a (mon.0) 1757 : audit 1 from='client.? 172.21.15.16:0/3478923642' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:55:20.281 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:20 smithi016 bash[77995]: cluster 2023-09-23T19:55:19.290429+0000 mon.a (mon.0) 1758 : cluster 0 osdmap e249: 8 total, 8 up, 8 in 2023-09-23T19:55:20.281 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:20 smithi016 bash[77995]: cluster 2023-09-23T19:55:19.297381+0000 osd.5 (osd.5) 178 : cluster 3 failed to encode map e249 with expected crc 2023-09-23T19:55:20.281 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:20 smithi016 bash[77995]: cluster 2023-09-23T19:55:19.302640+0000 osd.4 (osd.4) 192 : cluster 3 failed to encode map e249 with expected crc 2023-09-23T19:55:20.281 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:20 smithi016 bash[77995]: cluster 2023-09-23T19:55:19.304083+0000 osd.6 (osd.6) 184 : cluster 3 failed to encode map e249 with expected crc 2023-09-23T19:55:20.281 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:20 smithi016 bash[77995]: cluster 2023-09-23T19:55:19.304395+0000 osd.4 (osd.4) 193 : cluster 3 failed to encode map e249 with expected crc 2023-09-23T19:55:20.281 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:20 smithi016 bash[77995]: cluster 2023-09-23T19:55:19.306177+0000 osd.6 (osd.6) 185 : cluster 3 failed to encode map e249 with expected crc 2023-09-23T19:55:20.317 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:20.358 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:20.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:20.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:20.464 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:20.506 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:20.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:20.510 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:20.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:20 smithi016 bash[75004]: cluster 2023-09-23T19:55:19.268591+0000 mon.a (mon.0) 1755 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:55:20.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:20 smithi016 bash[75004]: cluster 2023-09-23T19:55:19.268638+0000 mon.a (mon.0) 1756 : cluster 1 Cluster is now healthy 2023-09-23T19:55:20.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:20 smithi016 bash[75004]: audit 2023-09-23T19:55:19.290100+0000 mon.a (mon.0) 1757 : audit 1 from='client.? 172.21.15.16:0/3478923642' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:55:20.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:20 smithi016 bash[75004]: cluster 2023-09-23T19:55:19.290429+0000 mon.a (mon.0) 1758 : cluster 0 osdmap e249: 8 total, 8 up, 8 in 2023-09-23T19:55:20.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:20 smithi016 bash[75004]: cluster 2023-09-23T19:55:19.297381+0000 osd.5 (osd.5) 178 : cluster 3 failed to encode map e249 with expected crc 2023-09-23T19:55:20.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:20 smithi016 bash[75004]: cluster 2023-09-23T19:55:19.302640+0000 osd.4 (osd.4) 192 : cluster 3 failed to encode map e249 with expected crc 2023-09-23T19:55:20.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:20 smithi016 bash[75004]: cluster 2023-09-23T19:55:19.304083+0000 osd.6 (osd.6) 184 : cluster 3 failed to encode map e249 with expected crc 2023-09-23T19:55:20.538 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:20 smithi016 bash[75004]: cluster 2023-09-23T19:55:19.304395+0000 osd.4 (osd.4) 193 : cluster 3 failed to encode map e249 with expected crc 2023-09-23T19:55:20.538 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:20 smithi016 bash[75004]: cluster 2023-09-23T19:55:19.306177+0000 osd.6 (osd.6) 185 : cluster 3 failed to encode map e249 with expected crc 2023-09-23T19:55:20.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:20 smithi092 bash[58855]: cluster 2023-09-23T19:55:18.519372+0000 mgr.y (mgr.25325) 1440 : cluster 0 pgmap v1453: 89 pgs: 89 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 653 op/s 2023-09-23T19:55:20.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:20 smithi092 bash[58855]: cluster 2023-09-23T19:55:19.268591+0000 mon.a (mon.0) 1755 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:55:20.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:20 smithi092 bash[58855]: cluster 2023-09-23T19:55:19.268638+0000 mon.a (mon.0) 1756 : cluster 1 Cluster is now healthy 2023-09-23T19:55:20.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:20 smithi092 bash[58855]: audit 2023-09-23T19:55:19.290100+0000 mon.a (mon.0) 1757 : audit 1 from='client.? 172.21.15.16:0/3478923642' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:55:20.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:20 smithi092 bash[58855]: cluster 2023-09-23T19:55:19.290429+0000 mon.a (mon.0) 1758 : cluster 0 osdmap e249: 8 total, 8 up, 8 in 2023-09-23T19:55:20.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:20 smithi092 bash[58855]: cluster 2023-09-23T19:55:19.297381+0000 osd.5 (osd.5) 178 : cluster 3 failed to encode map e249 with expected crc 2023-09-23T19:55:20.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:20 smithi092 bash[58855]: cluster 2023-09-23T19:55:19.302640+0000 osd.4 (osd.4) 192 : cluster 3 failed to encode map e249 with expected crc 2023-09-23T19:55:20.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:20 smithi092 bash[58855]: cluster 2023-09-23T19:55:19.304083+0000 osd.6 (osd.6) 184 : cluster 3 failed to encode map e249 with expected crc 2023-09-23T19:55:20.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:20 smithi092 bash[58855]: cluster 2023-09-23T19:55:19.304395+0000 osd.4 (osd.4) 193 : cluster 3 failed to encode map e249 with expected crc 2023-09-23T19:55:20.540 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:20 smithi092 bash[58855]: cluster 2023-09-23T19:55:19.306177+0000 osd.6 (osd.6) 185 : cluster 3 failed to encode map e249 with expected crc 2023-09-23T19:55:20.611 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:20.639 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:20.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:20.646 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:20.747 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:20.788 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:20.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:20.795 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:20.896 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:20.922 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:20.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:20.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:21.026 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:21.052 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:21.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:21.058 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:21.158 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:21.204 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:21.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:21.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:21.310 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:21.358 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:21.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:21.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:21.464 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:21.490 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:21.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:21.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:21.595 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:21.627 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:21.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:21.630 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:21.731 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:21.770 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:21.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:21.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:21.880 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:21.915 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:21.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:21.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:55:22.024 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:22.053 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:22.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:22.060 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:22.161 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:22.192 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:22.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:22.197 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:22.297 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:22.349 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:22.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:22.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:22.461 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:22.485 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:22.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:22.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:22.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:22 smithi016 bash[77995]: cluster 2023-09-23T19:55:20.520082+0000 mgr.y (mgr.25325) 1441 : cluster 0 pgmap v1455: 89 pgs: 89 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 747 op/s 2023-09-23T19:55:22.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:22 smithi016 bash[75004]: cluster 2023-09-23T19:55:20.520082+0000 mgr.y (mgr.25325) 1441 : cluster 0 pgmap v1455: 89 pgs: 89 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 747 op/s 2023-09-23T19:55:22.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:22 smithi092 bash[58855]: cluster 2023-09-23T19:55:20.520082+0000 mgr.y (mgr.25325) 1441 : cluster 0 pgmap v1455: 89 pgs: 89 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 747 op/s 2023-09-23T19:55:22.585 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T19:55:22.589 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:22.668 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:22.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:22.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:22.777 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:22.806 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:22.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:22.813 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:55:22.913 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:22.954 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:22.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:22.960 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:23.061 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:23.086 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:23.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:23.094 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:23.187 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:55:23.189 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:55:23.195 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:23.234 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:23.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:23.236 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:23.337 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:23.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:23.376 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:23.376 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:23.477 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:23.514 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:23.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:23.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:23.621 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:23.644 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:23.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:23.647 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:23.747 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:23.774 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:23.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:23.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:23.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:23 smithi092 bash[58855]: cluster 2023-09-23T19:55:22.520653+0000 mgr.y (mgr.25325) 1442 : cluster 0 pgmap v1456: 89 pgs: 89 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 770 op/s 2023-09-23T19:55:23.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:23 smithi092 bash[58855]: audit 2023-09-23T19:55:22.611157+0000 mon.a (mon.0) 1759 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:55:23.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:23 smithi092 bash[58855]: audit 2023-09-23T19:55:22.620384+0000 mon.a (mon.0) 1760 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:55:23.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:23 smithi092 bash[58855]: audit 2023-09-23T19:55:23.288441+0000 mon.a (mon.0) 1761 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:55:23.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:23 smithi092 bash[58855]: audit 2023-09-23T19:55:23.290181+0000 mon.a (mon.0) 1762 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:55:23.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:23 smithi092 bash[58855]: audit 2023-09-23T19:55:23.300284+0000 mon.a (mon.0) 1763 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:55:23.878 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:23.909 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:23.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:23.912 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:24.013 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:24.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:23 smithi016 bash[77995]: cluster 2023-09-23T19:55:22.520653+0000 mgr.y (mgr.25325) 1442 : cluster 0 pgmap v1456: 89 pgs: 89 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 770 op/s 2023-09-23T19:55:24.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:23 smithi016 bash[77995]: audit 2023-09-23T19:55:22.611157+0000 mon.a (mon.0) 1759 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:55:24.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:23 smithi016 bash[77995]: audit 2023-09-23T19:55:22.620384+0000 mon.a (mon.0) 1760 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:55:24.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:23 smithi016 bash[77995]: audit 2023-09-23T19:55:23.288441+0000 mon.a (mon.0) 1761 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:55:24.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:23 smithi016 bash[77995]: audit 2023-09-23T19:55:23.290181+0000 mon.a (mon.0) 1762 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:55:24.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:23 smithi016 bash[77995]: audit 2023-09-23T19:55:23.300284+0000 mon.a (mon.0) 1763 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:55:24.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:23 smithi016 bash[75004]: cluster 2023-09-23T19:55:22.520653+0000 mgr.y (mgr.25325) 1442 : cluster 0 pgmap v1456: 89 pgs: 89 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 770 op/s 2023-09-23T19:55:24.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:23 smithi016 bash[75004]: audit 2023-09-23T19:55:22.611157+0000 mon.a (mon.0) 1759 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:55:24.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:23 smithi016 bash[75004]: audit 2023-09-23T19:55:22.620384+0000 mon.a (mon.0) 1760 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:55:24.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:23 smithi016 bash[75004]: audit 2023-09-23T19:55:23.288441+0000 mon.a (mon.0) 1761 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:55:24.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:23 smithi016 bash[75004]: audit 2023-09-23T19:55:23.290181+0000 mon.a (mon.0) 1762 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:55:24.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:23 smithi016 bash[75004]: audit 2023-09-23T19:55:23.300284+0000 mon.a (mon.0) 1763 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:55:24.040 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:24.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:24.044 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:24.145 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:24.177 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:24.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:24.180 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:24.203 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:55:24.282 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:24.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:24.318 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:24.319 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:24.419 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:24.463 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:24.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:24.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:24.571 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:24.611 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:24.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:24.616 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:24.717 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:24.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:24.741 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:24.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:24.842 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:24.878 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:24.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:24.884 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:24.985 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:25.021 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:25.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:25.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:55:25.128 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:25.152 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:25.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:25.155 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:25.255 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:25.288 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:25.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:25.291 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:25.392 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:25.422 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:25.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:25.424 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:25.525 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:25.559 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:25.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:25.564 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:25.665 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:25.673 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:55:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:55:25] "GET /metrics HTTP/1.1" 200 36307 "" "Prometheus/2.33.4" 2023-09-23T19:55:25.687 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:25.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:25.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:25.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:25 smithi092 bash[58855]: cluster 2023-09-23T19:55:24.521124+0000 mgr.y (mgr.25325) 1443 : cluster 0 pgmap v1457: 89 pgs: 89 active+clean; 1.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2023-09-23T19:55:25.795 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:25.820 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:25.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:25.824 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:25.924 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:25.934 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:25 smithi016 bash[77995]: cluster 2023-09-23T19:55:24.521124+0000 mgr.y (mgr.25325) 1443 : cluster 0 pgmap v1457: 89 pgs: 89 active+clean; 1.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2023-09-23T19:55:25.934 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:25 smithi016 bash[75004]: cluster 2023-09-23T19:55:24.521124+0000 mgr.y (mgr.25325) 1443 : cluster 0 pgmap v1457: 89 pgs: 89 active+clean; 1.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2023-09-23T19:55:25.950 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:25.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:25.954 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:26.055 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:26.085 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:26.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:26.090 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:26.191 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:26.227 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:26.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:26.233 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:55:26.334 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:26.364 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:26.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:26.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:26.470 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:26.493 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:26.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:26.496 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:26.596 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:26.622 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:26.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:26.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:26.727 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:26.757 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:26.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:26.766 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:26.867 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:26.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:26.906 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:26.906 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:27.007 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:27.040 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:27.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:27.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:27.145 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:27.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:27.193 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:27.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:27.259 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:55:27.295 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:27.336 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:27.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:27.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:27.443 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:27.471 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:27.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:27.475 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:27.576 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:27.628 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:27.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:27.635 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:27.732 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:27 smithi016 bash[75004]: cluster 2023-09-23T19:55:26.521817+0000 mgr.y (mgr.25325) 1444 : cluster 0 pgmap v1458: 89 pgs: 89 active+clean; 1.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 640 op/s 2023-09-23T19:55:27.733 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:27 smithi092 bash[58855]: cluster 2023-09-23T19:55:26.521817+0000 mgr.y (mgr.25325) 1444 : cluster 0 pgmap v1458: 89 pgs: 89 active+clean; 1.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 640 op/s 2023-09-23T19:55:27.736 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:27.762 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:27.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:27.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:27.869 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:27.897 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:27.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:27.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:28.000 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:28.019 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:28.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:28.021 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:28.061 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:27 smithi016 bash[77995]: cluster 2023-09-23T19:55:26.521817+0000 mgr.y (mgr.25325) 1444 : cluster 0 pgmap v1458: 89 pgs: 89 active+clean; 1.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 640 op/s 2023-09-23T19:55:28.122 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:28.161 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:28.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:28.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:28.265 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:28.288 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:28.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:28.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:28.396 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:28.428 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:28.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:28.431 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:28.532 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:28.561 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:28.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:28.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:28.663 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:28.698 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:28.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:28.701 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:28.801 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:28.830 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:28.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:28.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:28.936 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:28.986 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:28.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:28.994 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:55:29.095 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:29.131 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:29.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:29.135 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:29.236 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:29.265 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:29.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:29.268 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:29.369 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:29.392 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:29.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:29.402 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:29.502 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:29.537 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:29.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:29.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:29.639 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:29.675 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:29.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:29.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:29.780 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:29.807 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:29.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:29.812 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:29.913 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:29.954 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:29.958 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:29 smithi092 bash[58855]: cluster 2023-09-23T19:55:28.522340+0000 mgr.y (mgr.25325) 1445 : cluster 0 pgmap v1459: 89 pgs: 89 active+clean; 1.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 635 op/s 2023-09-23T19:55:29.958 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:29 smithi092 bash[58855]: audit 2023-09-23T19:55:28.690308+0000 mon.a (mon.0) 1764 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:55:29.959 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:29 smithi092 bash[58855]: audit 2023-09-23T19:55:28.692621+0000 mon.a (mon.0) 1765 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:55:29.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:29.959 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:30.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:29 smithi016 bash[77995]: cluster 2023-09-23T19:55:28.522340+0000 mgr.y (mgr.25325) 1445 : cluster 0 pgmap v1459: 89 pgs: 89 active+clean; 1.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 635 op/s 2023-09-23T19:55:30.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:29 smithi016 bash[77995]: audit 2023-09-23T19:55:28.690308+0000 mon.a (mon.0) 1764 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:55:30.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:29 smithi016 bash[77995]: audit 2023-09-23T19:55:28.692621+0000 mon.a (mon.0) 1765 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:55:30.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:29 smithi016 bash[75004]: cluster 2023-09-23T19:55:28.522340+0000 mgr.y (mgr.25325) 1445 : cluster 0 pgmap v1459: 89 pgs: 89 active+clean; 1.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 635 op/s 2023-09-23T19:55:30.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:29 smithi016 bash[75004]: audit 2023-09-23T19:55:28.690308+0000 mon.a (mon.0) 1764 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:55:30.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:29 smithi016 bash[75004]: audit 2023-09-23T19:55:28.692621+0000 mon.a (mon.0) 1765 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:55:30.060 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:30.094 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:30.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:30.098 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:30.199 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:30.225 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:30.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:30.233 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:30.334 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:30.375 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:30.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:30.383 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:55:30.484 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:30.514 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:30.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:30.516 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:30.617 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:30.657 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:30.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:30.660 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:30.761 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:30.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:30.803 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:30.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:30.904 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:30.942 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:30.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:30.947 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:31.047 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:31.081 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:31.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:31.086 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:31.186 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:31.221 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:31.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:31.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:31.324 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:31.355 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:31.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:31.362 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:31.463 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:31.489 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:31.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:31.494 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:31.595 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:31.723 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:31.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:31.728 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:55:31.829 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:31.838 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:31 smithi016 bash[75004]: cluster 2023-09-23T19:55:30.523161+0000 mgr.y (mgr.25325) 1446 : cluster 0 pgmap v1460: 89 pgs: 89 active+clean; 1.2 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 791 op/s 2023-09-23T19:55:31.839 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:31 smithi016 bash[77995]: cluster 2023-09-23T19:55:30.523161+0000 mgr.y (mgr.25325) 1446 : cluster 0 pgmap v1460: 89 pgs: 89 active+clean; 1.2 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 791 op/s 2023-09-23T19:55:31.869 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:31.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:31.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:31.972 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:31.999 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:32.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:32.002 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:32.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:31 smithi092 bash[58855]: cluster 2023-09-23T19:55:30.523161+0000 mgr.y (mgr.25325) 1446 : cluster 0 pgmap v1460: 89 pgs: 89 active+clean; 1.2 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 791 op/s 2023-09-23T19:55:32.102 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:32.131 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:32.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:32.138 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:32.239 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:32.264 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:32.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:32.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:32.370 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:32.399 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:32.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:32.402 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:32.503 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:32.537 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:32.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:32.543 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:32.643 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:32.674 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:32.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:32.676 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:32.777 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:32.814 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:32.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:32.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:32.920 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:32.961 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:32.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:32.964 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:55:33.065 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:33.084 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:33.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:33.090 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:33.190 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:33.214 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:33.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:33.222 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:33.323 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:33.345 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:33.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:33.352 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:33.453 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:33.486 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:33.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:33.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:33.590 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:33.622 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:33.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:33.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:55:33.725 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:33.763 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:33.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:33.768 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:33.869 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:33.879 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:33 smithi092 bash[58855]: cluster 2023-09-23T19:55:32.523661+0000 mgr.y (mgr.25325) 1447 : cluster 0 pgmap v1461: 89 pgs: 89 active+clean; 1.2 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-09-23T19:55:33.898 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:33.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:33.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:33.929 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:33 smithi016 bash[75004]: cluster 2023-09-23T19:55:32.523661+0000 mgr.y (mgr.25325) 1447 : cluster 0 pgmap v1461: 89 pgs: 89 active+clean; 1.2 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-09-23T19:55:33.930 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:33 smithi016 bash[77995]: cluster 2023-09-23T19:55:32.523661+0000 mgr.y (mgr.25325) 1447 : cluster 0 pgmap v1461: 89 pgs: 89 active+clean; 1.2 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-09-23T19:55:34.003 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:34.034 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:34.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:34.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:34.140 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:34.195 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:34.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:34.201 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:34.302 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:34.329 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:34.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:34.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:34.432 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:34.472 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:34.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:34.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:34.576 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:34.601 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:34.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:34.604 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:34.705 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:34.737 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:34.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:34.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:34.843 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:34.870 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:34.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:34.874 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:34.974 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:35.003 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:35.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:35.006 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:35.107 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:35.144 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:35.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:35.150 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:35.250 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:35.418 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:35.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:35.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:35.527 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:35.549 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:35.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:35.552 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:35.652 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:35.675 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:35.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:35.678 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:35.747 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:35 smithi092 bash[58855]: cluster 2023-09-23T19:55:34.524378+0000 mgr.y (mgr.25325) 1448 : cluster 0 pgmap v1462: 89 pgs: 89 active+clean; 1.2 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 545 op/s 2023-09-23T19:55:35.779 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:35.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:35 smithi016 bash[77995]: cluster 2023-09-23T19:55:34.524378+0000 mgr.y (mgr.25325) 1448 : cluster 0 pgmap v1462: 89 pgs: 89 active+clean; 1.2 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 545 op/s 2023-09-23T19:55:35.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:35 smithi016 bash[75004]: cluster 2023-09-23T19:55:34.524378+0000 mgr.y (mgr.25325) 1448 : cluster 0 pgmap v1462: 89 pgs: 89 active+clean; 1.2 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 545 op/s 2023-09-23T19:55:35.786 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:55:35 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:55:35] "GET /metrics HTTP/1.1" 200 36306 "" "Prometheus/2.33.4" 2023-09-23T19:55:35.812 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:35.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:35.817 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:35.917 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:35.945 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:35.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:35.947 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:36.047 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:36.086 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:36.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:36.090 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:55:36.191 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:36.233 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:36.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:36.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:36.249 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:55:36.280 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:55:36.318 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:55:36.338 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:36.352 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:55:36.382 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:36.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:36.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:36.396 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:55:36.433 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:55:36.481 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:55:36.489 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:36.533 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:36.536 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:55:36.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:36.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:36.543 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:55:36.543 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:55:36.543 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:55:36.543 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:55:36.544 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:55:36.544 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:55:36.544 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:55:36.544 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:55:36.638 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:36.680 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:36.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:36.685 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:36.786 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:36.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:36.820 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:36.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:36.921 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:36.930 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:36 smithi016 bash[75004]: audit 2023-09-23T19:55:36.194593+0000 mon.a (mon.0) 1766 : audit 0 from='client.? 172.21.15.16:0/1653767186' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:55:36.931 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:36 smithi016 bash[77995]: audit 2023-09-23T19:55:36.194593+0000 mon.a (mon.0) 1766 : audit 0 from='client.? 172.21.15.16:0/1653767186' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:55:36.950 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:36.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:36.953 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:37.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:36 smithi092 bash[58855]: audit 2023-09-23T19:55:36.194593+0000 mon.a (mon.0) 1766 : audit 0 from='client.? 172.21.15.16:0/1653767186' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:55:37.053 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:37.085 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:37.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:37.090 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:55:37.191 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:37.220 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:37.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:37.223 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:37.325 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:37.354 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:37.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:37.356 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:37.457 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:37.485 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:37.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:37.487 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:37.588 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:37.620 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:37.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:37.623 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:37.724 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:37.766 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:37.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:37.772 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:37.873 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:37.897 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:37.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:37.899 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:38.000 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:38.009 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:37 smithi092 bash[58855]: cluster 2023-09-23T19:55:36.525222+0000 mgr.y (mgr.25325) 1449 : cluster 0 pgmap v1463: 89 pgs: 89 active+clean; 1021 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 743 op/s 2023-09-23T19:55:38.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:37 smithi016 bash[77995]: cluster 2023-09-23T19:55:36.525222+0000 mgr.y (mgr.25325) 1449 : cluster 0 pgmap v1463: 89 pgs: 89 active+clean; 1021 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 743 op/s 2023-09-23T19:55:38.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:37 smithi016 bash[75004]: cluster 2023-09-23T19:55:36.525222+0000 mgr.y (mgr.25325) 1449 : cluster 0 pgmap v1463: 89 pgs: 89 active+clean; 1021 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 743 op/s 2023-09-23T19:55:38.038 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:38.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:38.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:38.141 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:38.173 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:38.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:38.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:38.276 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:38.300 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:38.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:38.304 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:38.406 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:38.433 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:38.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:38.436 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:38.537 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:38.567 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:38.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:38.573 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:38.673 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:38.702 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:38.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:38.707 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:38.808 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:38.845 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:38.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:38.848 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:38.948 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:38.984 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:38.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:38.989 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:39.090 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:39.163 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:39.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:39.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:39.268 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:39.303 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:39.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:39.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:39.406 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:39.428 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:39.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:39.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:39.530 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:39.560 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:39.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:39.564 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:39.665 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:39.688 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:39.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:39.693 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:39.793 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:39.817 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:39.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:39.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:39.924 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:39.936 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:39 smithi092 bash[58855]: cluster 2023-09-23T19:55:38.525735+0000 mgr.y (mgr.25325) 1450 : cluster 0 pgmap v1464: 89 pgs: 89 active+clean; 967 MiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 625 op/s 2023-09-23T19:55:39.954 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:39.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:39.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:39.970 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:39 smithi016 bash[75004]: cluster 2023-09-23T19:55:38.525735+0000 mgr.y (mgr.25325) 1450 : cluster 0 pgmap v1464: 89 pgs: 89 active+clean; 967 MiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 625 op/s 2023-09-23T19:55:39.970 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:39 smithi016 bash[77995]: cluster 2023-09-23T19:55:38.525735+0000 mgr.y (mgr.25325) 1450 : cluster 0 pgmap v1464: 89 pgs: 89 active+clean; 967 MiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 625 op/s 2023-09-23T19:55:40.057 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:40.092 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:40.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:40.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:40.198 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:40.232 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:40.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:40.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:40.338 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:40.358 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:40.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:40.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:40.462 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:40.500 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:40.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:40.507 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:40.608 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:40.646 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:40.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:40.649 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:40.749 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:40.781 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:40.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:40.786 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:40.887 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:40.926 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:40.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:40.929 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:55:41.030 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:41.058 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:41.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:41.061 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:41.161 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:41.190 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:41.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:41.196 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:41.297 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:41.335 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:41.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:41.343 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:41.444 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:41.476 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-09-23T19:55:41.477 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [], 2023-09-23T19:55:41.477 INFO:teuthology.orchestra.run.smithi016.stdout: "num_ops": 0 2023-09-23T19:55:41.477 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-09-23T19:55:41.483 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:41.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:41.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:41.590 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:41.637 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:41.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:41.640 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:41.741 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:41.761 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:41 smithi016 bash[75004]: cluster 2023-09-23T19:55:40.526436+0000 mgr.y (mgr.25325) 1451 : cluster 0 pgmap v1465: 89 pgs: 89 active+clean; 821 MiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 756 op/s 2023-09-23T19:55:41.762 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:41 smithi016 bash[77995]: cluster 2023-09-23T19:55:40.526436+0000 mgr.y (mgr.25325) 1451 : cluster 0 pgmap v1465: 89 pgs: 89 active+clean; 821 MiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 756 op/s 2023-09-23T19:55:41.776 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:41.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:41.782 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:41.882 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:41.915 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:41.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:41.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:42.021 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:42.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:41 smithi092 bash[58855]: cluster 2023-09-23T19:55:40.526436+0000 mgr.y (mgr.25325) 1451 : cluster 0 pgmap v1465: 89 pgs: 89 active+clean; 821 MiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 756 op/s 2023-09-23T19:55:42.051 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:42.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:42.056 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:42.157 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:42.197 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:42.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:42.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:42.303 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:42.335 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:42.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:42.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:42.440 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:42.481 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:42.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:42.484 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:55:42.585 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:42.610 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:42.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:42.613 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:42.714 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:42.743 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T19:55:42.756 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:42.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:42.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:42.791 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is set 2023-09-23T19:55:42.792 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:55:42.798 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T19:55:42.863 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:42.904 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:42.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:42.909 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:43.009 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:43.018 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:42 smithi092 bash[58855]: audit 2023-09-23T19:55:41.824505+0000 mon.c (mon.2) 224 : audit 1 from='client.? 172.21.15.16:0/342195938' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T19:55:43.018 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:42 smithi092 bash[58855]: audit 2023-09-23T19:55:41.826632+0000 mon.a (mon.0) 1767 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T19:55:43.018 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:42 smithi092 bash[58855]: audit 2023-09-23T19:55:41.972064+0000 mon.a (mon.0) 1768 : audit 1 from='client.? 172.21.15.16:0/3491612239' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:55:43.038 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:43.041 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:42 smithi016 bash[75004]: audit 2023-09-23T19:55:41.824505+0000 mon.c (mon.2) 224 : audit 1 from='client.? 172.21.15.16:0/342195938' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T19:55:43.041 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:42 smithi016 bash[75004]: audit 2023-09-23T19:55:41.826632+0000 mon.a (mon.0) 1767 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T19:55:43.041 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:42 smithi016 bash[75004]: audit 2023-09-23T19:55:41.972064+0000 mon.a (mon.0) 1768 : audit 1 from='client.? 172.21.15.16:0/3491612239' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:55:43.041 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:42 smithi016 bash[77995]: audit 2023-09-23T19:55:41.824505+0000 mon.c (mon.2) 224 : audit 1 from='client.? 172.21.15.16:0/342195938' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T19:55:43.042 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:42 smithi016 bash[77995]: audit 2023-09-23T19:55:41.826632+0000 mon.a (mon.0) 1767 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T19:55:43.042 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:42 smithi016 bash[77995]: audit 2023-09-23T19:55:41.972064+0000 mon.a (mon.0) 1768 : audit 1 from='client.? 172.21.15.16:0/3491612239' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:55:43.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:43.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:43.145 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:43.174 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:43.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:43.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:43.279 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:43.328 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:43.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:43.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:43.433 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:43.467 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:43.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:43.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:55:43.573 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:43.611 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:43.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:43.616 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:43.717 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:43.746 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:43.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:43.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:55:43.771 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:43 smithi016 bash[75004]: cluster 2023-09-23T19:55:42.526938+0000 mgr.y (mgr.25325) 1452 : cluster 0 pgmap v1466: 89 pgs: 89 active+clean; 786 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-09-23T19:55:43.771 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:43 smithi016 bash[75004]: cluster 2023-09-23T19:55:42.757239+0000 mon.a (mon.0) 1769 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:55:43.771 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:43 smithi016 bash[75004]: audit 2023-09-23T19:55:42.773806+0000 mon.a (mon.0) 1770 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T19:55:43.772 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:43 smithi016 bash[75004]: audit 2023-09-23T19:55:42.774079+0000 mon.a (mon.0) 1771 : audit 1 from='client.? 172.21.15.16:0/3491612239' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:55:43.772 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:43 smithi016 bash[75004]: cluster 2023-09-23T19:55:42.774241+0000 mon.a (mon.0) 1772 : cluster 0 osdmap e250: 8 total, 8 up, 8 in 2023-09-23T19:55:43.772 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:43 smithi016 bash[75004]: cluster 2023-09-23T19:55:42.776904+0000 osd.4 (osd.4) 194 : cluster 3 failed to encode map e250 with expected crc 2023-09-23T19:55:43.772 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:43 smithi016 bash[75004]: cluster 2023-09-23T19:55:42.781216+0000 osd.5 (osd.5) 179 : cluster 3 failed to encode map e250 with expected crc 2023-09-23T19:55:43.772 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:43 smithi016 bash[75004]: cluster 2023-09-23T19:55:42.793399+0000 osd.6 (osd.6) 186 : cluster 3 failed to encode map e250 with expected crc 2023-09-23T19:55:43.772 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:43 smithi016 bash[75004]: audit 2023-09-23T19:55:43.688276+0000 mon.a (mon.0) 1773 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:55:43.772 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:43 smithi016 bash[75004]: audit 2023-09-23T19:55:43.689456+0000 mon.a (mon.0) 1774 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:55:43.772 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:43 smithi016 bash[77995]: cluster 2023-09-23T19:55:42.526938+0000 mgr.y (mgr.25325) 1452 : cluster 0 pgmap v1466: 89 pgs: 89 active+clean; 786 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-09-23T19:55:43.772 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:43 smithi016 bash[77995]: cluster 2023-09-23T19:55:42.757239+0000 mon.a (mon.0) 1769 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:55:43.773 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:43 smithi016 bash[77995]: audit 2023-09-23T19:55:42.773806+0000 mon.a (mon.0) 1770 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T19:55:43.773 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:43 smithi016 bash[77995]: audit 2023-09-23T19:55:42.774079+0000 mon.a (mon.0) 1771 : audit 1 from='client.? 172.21.15.16:0/3491612239' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:55:43.773 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:43 smithi016 bash[77995]: cluster 2023-09-23T19:55:42.774241+0000 mon.a (mon.0) 1772 : cluster 0 osdmap e250: 8 total, 8 up, 8 in 2023-09-23T19:55:43.773 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:43 smithi016 bash[77995]: cluster 2023-09-23T19:55:42.776904+0000 osd.4 (osd.4) 194 : cluster 3 failed to encode map e250 with expected crc 2023-09-23T19:55:43.773 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:43 smithi016 bash[77995]: cluster 2023-09-23T19:55:42.781216+0000 osd.5 (osd.5) 179 : cluster 3 failed to encode map e250 with expected crc 2023-09-23T19:55:43.773 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:43 smithi016 bash[77995]: cluster 2023-09-23T19:55:42.793399+0000 osd.6 (osd.6) 186 : cluster 3 failed to encode map e250 with expected crc 2023-09-23T19:55:43.773 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:43 smithi016 bash[77995]: audit 2023-09-23T19:55:43.688276+0000 mon.a (mon.0) 1773 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:55:43.773 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:43 smithi016 bash[77995]: audit 2023-09-23T19:55:43.689456+0000 mon.a (mon.0) 1774 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:55:43.853 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:43.887 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:43.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:43.892 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:43.993 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:44.002 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:43 smithi092 bash[58855]: cluster 2023-09-23T19:55:42.526938+0000 mgr.y (mgr.25325) 1452 : cluster 0 pgmap v1466: 89 pgs: 89 active+clean; 786 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-09-23T19:55:44.002 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:43 smithi092 bash[58855]: cluster 2023-09-23T19:55:42.757239+0000 mon.a (mon.0) 1769 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:55:44.002 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:43 smithi092 bash[58855]: audit 2023-09-23T19:55:42.773806+0000 mon.a (mon.0) 1770 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T19:55:44.002 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:43 smithi092 bash[58855]: audit 2023-09-23T19:55:42.774079+0000 mon.a (mon.0) 1771 : audit 1 from='client.? 172.21.15.16:0/3491612239' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:55:44.002 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:43 smithi092 bash[58855]: cluster 2023-09-23T19:55:42.774241+0000 mon.a (mon.0) 1772 : cluster 0 osdmap e250: 8 total, 8 up, 8 in 2023-09-23T19:55:44.002 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:43 smithi092 bash[58855]: cluster 2023-09-23T19:55:42.776904+0000 osd.4 (osd.4) 194 : cluster 3 failed to encode map e250 with expected crc 2023-09-23T19:55:44.002 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:43 smithi092 bash[58855]: cluster 2023-09-23T19:55:42.781216+0000 osd.5 (osd.5) 179 : cluster 3 failed to encode map e250 with expected crc 2023-09-23T19:55:44.002 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:43 smithi092 bash[58855]: cluster 2023-09-23T19:55:42.793399+0000 osd.6 (osd.6) 186 : cluster 3 failed to encode map e250 with expected crc 2023-09-23T19:55:44.002 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:43 smithi092 bash[58855]: audit 2023-09-23T19:55:43.688276+0000 mon.a (mon.0) 1773 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:55:44.002 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:43 smithi092 bash[58855]: audit 2023-09-23T19:55:43.689456+0000 mon.a (mon.0) 1774 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:55:44.022 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:44.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:44.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:44.129 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:44.153 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:44.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:44.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:44.257 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:44.297 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:44.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:44.301 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:44.319 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T19:55:44.402 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:44.434 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:44.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:44.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:44.539 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:44.570 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:44.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:44.576 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:44.677 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:44.702 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:44.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:44.706 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:44.807 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:44.841 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:44.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:44.846 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:44.947 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:44.976 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:44.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:44.982 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:45.083 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:45.111 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:45.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:45.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:45.215 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:45.261 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:45.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:45.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:45.365 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:45.396 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:45.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:45.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:45.502 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:45.537 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:45.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:45.539 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:45.640 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:45.643 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:55:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:55:45] "GET /metrics HTTP/1.1" 200 36294 "" "Prometheus/2.33.4" 2023-09-23T19:55:45.667 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:45.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:45.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:45.772 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:45.776 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:45 smithi092 bash[58855]: cluster 2023-09-23T19:55:44.527421+0000 mgr.y (mgr.25325) 1453 : cluster 0 pgmap v1468: 89 pgs: 89 active+clean; 774 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-09-23T19:55:45.814 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:45.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:45.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:45.919 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:45.928 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:45 smithi016 bash[75004]: cluster 2023-09-23T19:55:44.527421+0000 mgr.y (mgr.25325) 1453 : cluster 0 pgmap v1468: 89 pgs: 89 active+clean; 774 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-09-23T19:55:45.928 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:45 smithi016 bash[77995]: cluster 2023-09-23T19:55:44.527421+0000 mgr.y (mgr.25325) 1453 : cluster 0 pgmap v1468: 89 pgs: 89 active+clean; 774 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-09-23T19:55:45.947 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:45.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:45.950 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:46.051 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:46.084 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:46.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:46.087 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:46.187 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:46.227 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:46.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:46.229 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:46.296 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T19:55:46.331 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:46.378 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:46.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:46.384 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:46.484 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:46.512 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:46.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:46.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:46.622 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:46.656 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:46.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:46.664 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:46.765 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:46.813 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:46.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:46.817 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:55:46.918 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:46.952 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:46.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:46.958 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:47.059 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:47.096 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:47.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:47.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:47.200 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:47.222 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:47.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:47.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:47.330 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:47.356 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:47.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:47.361 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:47.462 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:47.491 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:47.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:47.496 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:47.597 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:47.634 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:47.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:47.636 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:55:47.645 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:55:47.645 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:55:47.737 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:47.775 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:47.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:47.780 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:55:47.881 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:47.890 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:47 smithi092 bash[58855]: cluster 2023-09-23T19:55:46.528146+0000 mgr.y (mgr.25325) 1454 : cluster 0 pgmap v1469: 89 pgs: 89 active+clean; 622 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 638 op/s 2023-09-23T19:55:47.915 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:47.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:47.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:55:48.020 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:48.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:47 smithi016 bash[77995]: cluster 2023-09-23T19:55:46.528146+0000 mgr.y (mgr.25325) 1454 : cluster 0 pgmap v1469: 89 pgs: 89 active+clean; 622 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 638 op/s 2023-09-23T19:55:48.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:47 smithi016 bash[75004]: cluster 2023-09-23T19:55:46.528146+0000 mgr.y (mgr.25325) 1454 : cluster 0 pgmap v1469: 89 pgs: 89 active+clean; 622 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 638 op/s 2023-09-23T19:55:48.050 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:48.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:48.058 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:48.158 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:48.189 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:48.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:48.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:48.296 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:48.330 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:48.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:48.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:48.434 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:48.468 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:48.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:48.471 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:48.572 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:48.596 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:48.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:48.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:48.663 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:55:48.698 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:48.752 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:48.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:48.756 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:48.857 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:48.894 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:48.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:48.900 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:49.001 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:49.039 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:49.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:49.046 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:49.147 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:49.170 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:49.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:49.172 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:49.273 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:49.297 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:49.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:49.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:49.402 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:49.440 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:49.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:49.443 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:49.543 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:49.575 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:49.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:49.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:49.679 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:49.712 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:49.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:49.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:49.815 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:49.844 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:49.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:49.851 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:49.951 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:49.960 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:49 smithi092 bash[58855]: cluster 2023-09-23T19:55:48.528722+0000 mgr.y (mgr.25325) 1455 : cluster 0 pgmap v1470: 89 pgs: 89 active+clean; 571 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 632 op/s 2023-09-23T19:55:49.978 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:49.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:49.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:50.031 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:49 smithi016 bash[75004]: cluster 2023-09-23T19:55:48.528722+0000 mgr.y (mgr.25325) 1455 : cluster 0 pgmap v1470: 89 pgs: 89 active+clean; 571 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 632 op/s 2023-09-23T19:55:50.031 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:49 smithi016 bash[77995]: cluster 2023-09-23T19:55:48.528722+0000 mgr.y (mgr.25325) 1455 : cluster 0 pgmap v1470: 89 pgs: 89 active+clean; 571 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 632 op/s 2023-09-23T19:55:50.081 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:50.115 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:50.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:50.121 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:50.222 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:50.259 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:50.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:50.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:50.362 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:50.403 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:50.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:50.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:50.508 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:50.542 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:50.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:50.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:50.649 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:50.688 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:50.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:50.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:50.697 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:55:50.792 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:50.835 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:50.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:50.843 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:50.944 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:50.973 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:50.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:50.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:51.076 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:51.105 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:51.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:51.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:51.208 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:51.264 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:51.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:51.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:51.369 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:51.395 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:51.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:51.401 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:51.502 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:51.552 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:51.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:51.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:51.657 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:51.683 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:51.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:51.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:51.793 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:51.822 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:51.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:51.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:51.926 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:51.951 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:51.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:51.959 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:55:52.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:51 smithi016 bash[77995]: cluster 2023-09-23T19:55:50.529473+0000 mgr.y (mgr.25325) 1456 : cluster 0 pgmap v1471: 89 pgs: 89 active+clean; 417 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2023-09-23T19:55:52.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:51 smithi016 bash[75004]: cluster 2023-09-23T19:55:50.529473+0000 mgr.y (mgr.25325) 1456 : cluster 0 pgmap v1471: 89 pgs: 89 active+clean; 417 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2023-09-23T19:55:52.060 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:52.068 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:51 smithi092 bash[58855]: cluster 2023-09-23T19:55:50.529473+0000 mgr.y (mgr.25325) 1456 : cluster 0 pgmap v1471: 89 pgs: 89 active+clean; 417 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2023-09-23T19:55:52.086 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:52.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:52.092 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:52.193 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:52.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:52.230 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:52.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:52.331 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:52.359 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:52.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:52.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:52.463 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:52.504 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:52.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:52.512 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:55:52.612 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:52.642 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:52.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:52.646 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:52.747 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:52.770 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:52.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:52.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:52.875 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:52.903 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:52.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:52.906 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:53.007 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:53.037 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:53.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:53.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:53.140 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:53.165 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:53.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:53.168 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:53.269 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:53.303 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:53.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:53.309 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:53.409 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:53.442 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:53.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:53.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:53.547 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:53.571 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:53.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:53.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:55:53.675 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:53.699 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:53.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:53.705 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:53.806 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:53.815 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:53 smithi092 bash[58855]: cluster 2023-09-23T19:55:52.529975+0000 mgr.y (mgr.25325) 1457 : cluster 0 pgmap v1472: 89 pgs: 89 active+clean; 379 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-09-23T19:55:53.832 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:53.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:53.834 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:53.935 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:53.961 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:53.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:53.964 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:55:54.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:53 smithi016 bash[75004]: cluster 2023-09-23T19:55:52.529975+0000 mgr.y (mgr.25325) 1457 : cluster 0 pgmap v1472: 89 pgs: 89 active+clean; 379 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-09-23T19:55:54.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:53 smithi016 bash[77995]: cluster 2023-09-23T19:55:52.529975+0000 mgr.y (mgr.25325) 1457 : cluster 0 pgmap v1472: 89 pgs: 89 active+clean; 379 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-09-23T19:55:54.065 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:54.098 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:54.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:54.102 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:54.202 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:54.235 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:54.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:54.240 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:54.341 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:54.365 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:54.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:54.372 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:54.472 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:54.509 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:54.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:54.512 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:54.613 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:54.638 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:54.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:54.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:55:54.742 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:54.789 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:54.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:54.792 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:54.893 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:54.922 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:54.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:54.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:55.025 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:55.067 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:55.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:55.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:55:55.173 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:55.199 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:55.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:55.202 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:55.302 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:55.341 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:55.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:55.348 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:55.449 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:55.489 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:55.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:55.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:55.591 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:55.616 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:55.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:55.618 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:55.719 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:55.728 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:55:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:55:55] "GET /metrics HTTP/1.1" 200 36294 "" "Prometheus/2.33.4" 2023-09-23T19:55:55.751 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:55.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:55.758 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:55.859 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:55.895 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:55.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:55.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:55.999 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:56.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:55 smithi016 bash[77995]: cluster 2023-09-23T19:55:54.530573+0000 mgr.y (mgr.25325) 1458 : cluster 0 pgmap v1473: 89 pgs: 89 active+clean; 366 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 555 op/s 2023-09-23T19:55:56.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:55 smithi016 bash[75004]: cluster 2023-09-23T19:55:54.530573+0000 mgr.y (mgr.25325) 1458 : cluster 0 pgmap v1473: 89 pgs: 89 active+clean; 366 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 555 op/s 2023-09-23T19:55:56.046 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:56.049 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:55 smithi092 bash[58855]: cluster 2023-09-23T19:55:54.530573+0000 mgr.y (mgr.25325) 1458 : cluster 0 pgmap v1473: 89 pgs: 89 active+clean; 366 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 555 op/s 2023-09-23T19:55:56.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:56.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:56.151 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:56.185 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:56.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:56.187 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:56.288 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:56.319 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:56.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:56.326 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:56.427 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:56.461 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:56.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:56.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:56.565 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:56.616 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:56.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:56.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:56.722 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:56.757 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:56.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:56.761 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:56.862 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:56.892 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:56.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:56.899 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:57.000 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:57.031 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:57.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:57.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:57.135 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:57.161 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:57.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:57.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:57.266 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:57.306 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:57.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:57.307 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:57.408 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:57.438 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:57.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:57.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:57.545 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:57.576 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:57.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:57.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:57.684 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:57.713 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:57.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:57.719 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:57.820 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:57.851 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:57.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:57.858 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:57.958 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:57.994 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:57.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:57.999 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:58.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:57 smithi092 bash[58855]: cluster 2023-09-23T19:55:56.531285+0000 mgr.y (mgr.25325) 1459 : cluster 0 pgmap v1474: 89 pgs: 89 active+clean; 212 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 748 op/s 2023-09-23T19:55:58.072 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:58 smithi016 bash[77995]: cluster 2023-09-23T19:55:56.531285+0000 mgr.y (mgr.25325) 1459 : cluster 0 pgmap v1474: 89 pgs: 89 active+clean; 212 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 748 op/s 2023-09-23T19:55:58.072 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:58 smithi016 bash[75004]: cluster 2023-09-23T19:55:56.531285+0000 mgr.y (mgr.25325) 1459 : cluster 0 pgmap v1474: 89 pgs: 89 active+clean; 212 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 748 op/s 2023-09-23T19:55:58.100 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:58.133 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:58.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:58.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:58.238 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:58.283 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:58.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:58.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:58.389 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:58.425 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:58.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:58.427 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:58.528 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:58.561 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:58.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:58.569 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:58.670 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:58.701 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:58.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:58.705 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:58.806 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:58.845 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:58.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:58.848 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:58.949 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:58.978 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:58.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:58.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:55:59.081 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:59.092 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:58 smithi092 bash[58855]: audit 2023-09-23T19:55:58.684483+0000 mon.a (mon.0) 1775 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:55:59.110 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:59.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:59.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:59.213 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:59.244 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:59.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:59.247 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:59.348 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:59.379 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:55:59.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:59.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:55:59.395 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:58 smithi016 bash[77995]: audit 2023-09-23T19:55:58.684483+0000 mon.a (mon.0) 1775 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:55:59.395 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:58 smithi016 bash[75004]: audit 2023-09-23T19:55:58.684483+0000 mon.a (mon.0) 1775 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:55:59.484 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:59.506 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:59.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:59.508 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:59.609 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:59.640 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:59.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:59.642 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:59.643 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Removed 107261 objects 2023-09-23T19:55:59.643 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Clean up completed and total clean up time :156.144 2023-09-23T19:55:59.742 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:59.773 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:59.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:59.775 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:59.876 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:59.889 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:55:59 smithi092 bash[58855]: cluster 2023-09-23T19:55:58.532022+0000 mgr.y (mgr.25325) 1460 : cluster 0 pgmap v1475: 89 pgs: 89 active+clean; 160 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 614 op/s 2023-09-23T19:55:59.907 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:59.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:55:59.911 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:00.011 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:00.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:55:59 smithi016 bash[75004]: cluster 2023-09-23T19:55:58.532022+0000 mgr.y (mgr.25325) 1460 : cluster 0 pgmap v1475: 89 pgs: 89 active+clean; 160 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 614 op/s 2023-09-23T19:56:00.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:55:59 smithi016 bash[77995]: cluster 2023-09-23T19:55:58.532022+0000 mgr.y (mgr.25325) 1460 : cluster 0 pgmap v1475: 89 pgs: 89 active+clean; 160 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 614 op/s 2023-09-23T19:56:00.041 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:00.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:00.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:00.146 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:00.177 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:00.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:00.180 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:00.281 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:00.305 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:00.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:00.307 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:00.408 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:00.428 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:00.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:00.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:00.532 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:00.565 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:00.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:00.569 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:00.670 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:00.702 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:00.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:00.705 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:00.763 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-09-23T19:56:00.764 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [], 2023-09-23T19:56:00.764 INFO:teuthology.orchestra.run.smithi016.stdout: "complaint_time": 30, 2023-09-23T19:56:00.764 INFO:teuthology.orchestra.run.smithi016.stdout: "num_blocked_ops": 0 2023-09-23T19:56:00.764 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-09-23T19:56:00.805 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:00.822 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is set 2023-09-23T19:56:00.830 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:00.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:00.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:00.933 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:00.965 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:00.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:00.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:01.032 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:56:01.053 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:56:01.068 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:01.073 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:56:01.078 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:00 smithi016 bash[75004]: audit 2023-09-23T19:56:00.491520+0000 mon.a (mon.0) 1776 : audit 1 from='client.? 172.21.15.16:0/1155801619' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:56:01.078 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:00 smithi016 bash[77995]: audit 2023-09-23T19:56:00.491520+0000 mon.a (mon.0) 1776 : audit 1 from='client.? 172.21.15.16:0/1155801619' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:56:01.093 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:01.096 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:56:01.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:01.097 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:01.123 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:56:01.144 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:56:01.166 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:56:01.189 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:56:01.191 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:56:01.191 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:56:01.191 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:56:01.191 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:56:01.191 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:56:01.191 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:56:01.192 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:56:01.192 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:56:01.198 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:01.211 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:00 smithi092 bash[58855]: audit 2023-09-23T19:56:00.491520+0000 mon.a (mon.0) 1776 : audit 1 from='client.? 172.21.15.16:0/1155801619' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:56:01.230 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:01.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:01.233 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:01.334 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:01.358 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:01.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:01.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:01.461 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:01.492 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:01.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:01.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:01.596 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:01.618 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:01.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:01.620 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:01.721 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:01.748 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:01.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:01.750 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:01.822 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T19:56:01.852 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:01.886 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:01.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:01.889 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:01.989 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:02.025 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:02.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:02.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:02.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:01 smithi016 bash[75004]: cluster 2023-09-23T19:56:00.532713+0000 mgr.y (mgr.25325) 1461 : cluster 0 pgmap v1476: 89 pgs: 89 active+clean; 26 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 726 op/s 2023-09-23T19:56:02.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:01 smithi016 bash[75004]: cluster 2023-09-23T19:56:00.801476+0000 mon.a (mon.0) 1777 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:56:02.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:01 smithi016 bash[75004]: audit 2023-09-23T19:56:00.814462+0000 mon.a (mon.0) 1778 : audit 1 from='client.? 172.21.15.16:0/1155801619' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:56:02.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:01 smithi016 bash[75004]: cluster 2023-09-23T19:56:00.814546+0000 mon.a (mon.0) 1779 : cluster 0 osdmap e251: 8 total, 8 up, 8 in 2023-09-23T19:56:02.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:01 smithi016 bash[75004]: cluster 2023-09-23T19:56:00.815516+0000 osd.7 (osd.7) 164 : cluster 3 failed to encode map e251 with expected crc 2023-09-23T19:56:02.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:01 smithi016 bash[75004]: audit 2023-09-23T19:56:01.003133+0000 mon.c (mon.2) 225 : audit 0 from='client.? 172.21.15.16:0/2311725095' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:56:02.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:01 smithi016 bash[75004]: audit 2023-09-23T19:56:01.244296+0000 mon.a (mon.0) 1780 : audit 1 from='client.? 172.21.15.16:0/1761358283' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T19:56:02.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:01 smithi016 bash[77995]: cluster 2023-09-23T19:56:00.532713+0000 mgr.y (mgr.25325) 1461 : cluster 0 pgmap v1476: 89 pgs: 89 active+clean; 26 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 726 op/s 2023-09-23T19:56:02.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:01 smithi016 bash[77995]: cluster 2023-09-23T19:56:00.801476+0000 mon.a (mon.0) 1777 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:56:02.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:01 smithi016 bash[77995]: audit 2023-09-23T19:56:00.814462+0000 mon.a (mon.0) 1778 : audit 1 from='client.? 172.21.15.16:0/1155801619' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:56:02.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:01 smithi016 bash[77995]: cluster 2023-09-23T19:56:00.814546+0000 mon.a (mon.0) 1779 : cluster 0 osdmap e251: 8 total, 8 up, 8 in 2023-09-23T19:56:02.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:01 smithi016 bash[77995]: cluster 2023-09-23T19:56:00.815516+0000 osd.7 (osd.7) 164 : cluster 3 failed to encode map e251 with expected crc 2023-09-23T19:56:02.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:01 smithi016 bash[77995]: audit 2023-09-23T19:56:01.003133+0000 mon.c (mon.2) 225 : audit 0 from='client.? 172.21.15.16:0/2311725095' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:56:02.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:01 smithi016 bash[77995]: audit 2023-09-23T19:56:01.244296+0000 mon.a (mon.0) 1780 : audit 1 from='client.? 172.21.15.16:0/1761358283' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T19:56:02.128 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:02.142 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:01 smithi092 bash[58855]: cluster 2023-09-23T19:56:00.532713+0000 mgr.y (mgr.25325) 1461 : cluster 0 pgmap v1476: 89 pgs: 89 active+clean; 26 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 726 op/s 2023-09-23T19:56:02.142 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:01 smithi092 bash[58855]: cluster 2023-09-23T19:56:00.801476+0000 mon.a (mon.0) 1777 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:56:02.142 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:01 smithi092 bash[58855]: audit 2023-09-23T19:56:00.814462+0000 mon.a (mon.0) 1778 : audit 1 from='client.? 172.21.15.16:0/1155801619' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:56:02.142 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:01 smithi092 bash[58855]: cluster 2023-09-23T19:56:00.814546+0000 mon.a (mon.0) 1779 : cluster 0 osdmap e251: 8 total, 8 up, 8 in 2023-09-23T19:56:02.143 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:01 smithi092 bash[58855]: cluster 2023-09-23T19:56:00.815516+0000 osd.7 (osd.7) 164 : cluster 3 failed to encode map e251 with expected crc 2023-09-23T19:56:02.143 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:01 smithi092 bash[58855]: audit 2023-09-23T19:56:01.003133+0000 mon.c (mon.2) 225 : audit 0 from='client.? 172.21.15.16:0/2311725095' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:56:02.143 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:01 smithi092 bash[58855]: audit 2023-09-23T19:56:01.244296+0000 mon.a (mon.0) 1780 : audit 1 from='client.? 172.21.15.16:0/1761358283' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T19:56:02.163 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:02.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:02.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:02.267 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:02.301 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:02.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:02.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:02.429 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:02.456 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:02.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:02.458 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:02.559 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:02.582 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_8 2023-09-23T19:56:02.583 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_8 unique_pool_8 --yes-i-really-really-mean-it 2023-09-23T19:56:02.589 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:02.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:02.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:02.693 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:02.726 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:02.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:02.729 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:02.830 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:02.849 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:02.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:02.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:02.952 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:02.980 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:02.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:02.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:03.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:02 smithi092 bash[58855]: audit 2023-09-23T19:56:01.813394+0000 mon.a (mon.0) 1781 : audit 1 from='client.? 172.21.15.16:0/1761358283' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T19:56:03.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:02 smithi092 bash[58855]: cluster 2023-09-23T19:56:01.813446+0000 mon.a (mon.0) 1782 : cluster 0 osdmap e252: 8 total, 8 up, 8 in 2023-09-23T19:56:03.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:02 smithi092 bash[58855]: cluster 2023-09-23T19:56:02.068745+0000 osd.4 (osd.4) 195 : cluster 3 failed to encode map e252 with expected crc 2023-09-23T19:56:03.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:02 smithi092 bash[58855]: cluster 2023-09-23T19:56:02.068915+0000 osd.5 (osd.5) 180 : cluster 3 failed to encode map e252 with expected crc 2023-09-23T19:56:03.084 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:03.114 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:03.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:03.117 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:03.218 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:03.229 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:02 smithi016 bash[77995]: audit 2023-09-23T19:56:01.813394+0000 mon.a (mon.0) 1781 : audit 1 from='client.? 172.21.15.16:0/1761358283' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T19:56:03.230 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:02 smithi016 bash[77995]: cluster 2023-09-23T19:56:01.813446+0000 mon.a (mon.0) 1782 : cluster 0 osdmap e252: 8 total, 8 up, 8 in 2023-09-23T19:56:03.230 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:02 smithi016 bash[77995]: cluster 2023-09-23T19:56:02.068745+0000 osd.4 (osd.4) 195 : cluster 3 failed to encode map e252 with expected crc 2023-09-23T19:56:03.230 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:02 smithi016 bash[77995]: cluster 2023-09-23T19:56:02.068915+0000 osd.5 (osd.5) 180 : cluster 3 failed to encode map e252 with expected crc 2023-09-23T19:56:03.230 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:02 smithi016 bash[75004]: audit 2023-09-23T19:56:01.813394+0000 mon.a (mon.0) 1781 : audit 1 from='client.? 172.21.15.16:0/1761358283' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T19:56:03.230 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:02 smithi016 bash[75004]: cluster 2023-09-23T19:56:01.813446+0000 mon.a (mon.0) 1782 : cluster 0 osdmap e252: 8 total, 8 up, 8 in 2023-09-23T19:56:03.230 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:02 smithi016 bash[75004]: cluster 2023-09-23T19:56:02.068745+0000 osd.4 (osd.4) 195 : cluster 3 failed to encode map e252 with expected crc 2023-09-23T19:56:03.230 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:02 smithi016 bash[75004]: cluster 2023-09-23T19:56:02.068915+0000 osd.5 (osd.5) 180 : cluster 3 failed to encode map e252 with expected crc 2023-09-23T19:56:03.245 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:03.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:03.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:03.349 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:03.380 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:03.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:03.383 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:03.484 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:03.517 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:03.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:03.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:03.620 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:03.650 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:03.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:03.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:03.754 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:03.784 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:03.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:03.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:03.888 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:03.920 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:03.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:03.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:04.023 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:04.046 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:04.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:04.049 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:04.063 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:03 smithi092 bash[58855]: cluster 2023-09-23T19:56:02.068915+0000 osd.7 (osd.7) 165 : cluster 3 failed to encode map e252 with expected crc 2023-09-23T19:56:04.063 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:03 smithi092 bash[58855]: cluster 2023-09-23T19:56:02.069211+0000 osd.6 (osd.6) 187 : cluster 3 failed to encode map e252 with expected crc 2023-09-23T19:56:04.063 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:03 smithi092 bash[58855]: cluster 2023-09-23T19:56:02.533303+0000 mgr.y (mgr.25325) 1462 : cluster 0 pgmap v1479: 89 pgs: 89 active+clean; 5.8 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 720 op/s 2023-09-23T19:56:04.149 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:04.174 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:04.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:04.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:04.278 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:04.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:03 smithi016 bash[77995]: cluster 2023-09-23T19:56:02.068915+0000 osd.7 (osd.7) 165 : cluster 3 failed to encode map e252 with expected crc 2023-09-23T19:56:04.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:03 smithi016 bash[77995]: cluster 2023-09-23T19:56:02.069211+0000 osd.6 (osd.6) 187 : cluster 3 failed to encode map e252 with expected crc 2023-09-23T19:56:04.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:03 smithi016 bash[77995]: cluster 2023-09-23T19:56:02.533303+0000 mgr.y (mgr.25325) 1462 : cluster 0 pgmap v1479: 89 pgs: 89 active+clean; 5.8 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 720 op/s 2023-09-23T19:56:04.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:03 smithi016 bash[75004]: cluster 2023-09-23T19:56:02.068915+0000 osd.7 (osd.7) 165 : cluster 3 failed to encode map e252 with expected crc 2023-09-23T19:56:04.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:03 smithi016 bash[75004]: cluster 2023-09-23T19:56:02.069211+0000 osd.6 (osd.6) 187 : cluster 3 failed to encode map e252 with expected crc 2023-09-23T19:56:04.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:03 smithi016 bash[75004]: cluster 2023-09-23T19:56:02.533303+0000 mgr.y (mgr.25325) 1462 : cluster 0 pgmap v1479: 89 pgs: 89 active+clean; 5.8 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 720 op/s 2023-09-23T19:56:04.293 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T19:56:04.304 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:04.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:04.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:04.407 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:04.433 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:04.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:04.436 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:04.537 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:04.566 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:04.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:04.569 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:04.669 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:04.702 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:04.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:04.704 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:04.747 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T19:56:04.805 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:04.864 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:04.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:04.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:04.968 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:05.002 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:05.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:05.005 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:05.106 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:05.140 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:05.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:05.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:05.243 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:05.275 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:05.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:05.278 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:05.379 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:05.409 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:05.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:05.412 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:05.513 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:05.543 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:05.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:05.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:05.646 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:05.682 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:05.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:05.685 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:05.785 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:05.818 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:05.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:05.822 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:05.922 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:05.936 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:05 smithi016 bash[77995]: cluster 2023-09-23T19:56:04.533696+0000 mgr.y (mgr.25325) 1463 : cluster 0 pgmap v1480: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 423 op/s 2023-09-23T19:56:05.936 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:05 smithi016 bash[75004]: cluster 2023-09-23T19:56:04.533696+0000 mgr.y (mgr.25325) 1463 : cluster 0 pgmap v1480: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 423 op/s 2023-09-23T19:56:05.936 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:56:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:56:05] "GET /metrics HTTP/1.1" 200 36298 "" "Prometheus/2.33.4" 2023-09-23T19:56:05.951 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:05.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:05.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:06.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:05 smithi092 bash[58855]: cluster 2023-09-23T19:56:04.533696+0000 mgr.y (mgr.25325) 1463 : cluster 0 pgmap v1480: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 423 op/s 2023-09-23T19:56:06.054 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:06.088 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:06.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:06.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:06.192 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:06.226 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:06.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:06.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:06.329 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:06.362 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:06.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:06.365 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:06.466 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:06.502 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:06.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:06.506 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:06.607 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:06.638 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:06.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:06.641 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:06.685 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T19:56:06.725 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:56:06.743 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:06.773 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:06.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:06.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:06.876 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:06.906 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:06.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:06.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:07.009 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:07.041 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:07.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:07.045 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:07.145 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:07.178 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:07.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:07.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:07.281 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:07.313 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:07.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:07.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:07.357 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:56:07.417 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:07.441 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:07.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:07.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:07.544 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:07.579 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:07.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:07.582 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:07.683 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:07.714 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:07.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:07.718 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:07.819 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:07.854 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:07.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:07.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:07.957 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:07.989 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:07.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:07.993 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:08.094 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:08.107 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:07 smithi016 bash[75004]: cluster 2023-09-23T19:56:06.534825+0000 mgr.y (mgr.25325) 1464 : cluster 0 pgmap v1481: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 319 op/s 2023-09-23T19:56:08.107 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:07 smithi016 bash[77995]: cluster 2023-09-23T19:56:06.534825+0000 mgr.y (mgr.25325) 1464 : cluster 0 pgmap v1481: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 319 op/s 2023-09-23T19:56:08.124 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:08.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:08.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:08.228 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:08.239 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:07 smithi092 bash[58855]: cluster 2023-09-23T19:56:06.534825+0000 mgr.y (mgr.25325) 1464 : cluster 0 pgmap v1481: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 319 op/s 2023-09-23T19:56:08.260 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:08.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:08.263 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:08.364 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:08.373 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:56:08.392 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:08.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:08.394 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:08.495 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:08.526 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:08.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:08.529 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:08.630 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:08.657 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:08.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:08.660 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:08.760 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:08.790 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:08.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:08.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:08.894 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:08.926 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:08.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:08.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:09.030 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:09.057 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:09.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:09.059 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:09.160 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:09.195 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:09.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:09.199 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:09.299 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:09.333 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:09.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:09.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:09.393 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:56:09.437 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:09.465 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:09.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:09.468 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:09.568 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:09.590 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:09.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:09.593 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:09.693 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:09.725 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:09.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:09.728 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:09.829 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:09.863 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:09.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:09.865 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:09.966 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:10.000 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:10.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:10.004 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:10.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:09 smithi016 bash[77995]: cluster 2023-09-23T19:56:08.535694+0000 mgr.y (mgr.25325) 1465 : cluster 0 pgmap v1482: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 51 op/s 2023-09-23T19:56:10.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:09 smithi016 bash[75004]: cluster 2023-09-23T19:56:08.535694+0000 mgr.y (mgr.25325) 1465 : cluster 0 pgmap v1482: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 51 op/s 2023-09-23T19:56:10.105 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:10.117 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:09 smithi092 bash[58855]: cluster 2023-09-23T19:56:08.535694+0000 mgr.y (mgr.25325) 1465 : cluster 0 pgmap v1482: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 51 op/s 2023-09-23T19:56:10.135 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:10.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:10.139 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:10.240 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:10.270 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:10.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:10.274 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:10.374 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:10.405 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:10.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:10.408 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:10.509 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:10.539 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:10.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:10.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:10.642 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:10.668 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:10.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:10.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:10.771 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:10.793 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:10.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:10.795 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:10.895 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:10.928 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:10.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:10.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:11.032 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:11.063 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:11.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:11.066 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:11.167 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:11.194 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:11.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:11.197 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:11.297 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:11.323 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:11.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:11.326 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:11.422 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:56:11.422 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:56:11.427 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:11.459 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:11.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:11.462 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:11.563 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:11.592 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:11.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:11.595 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:11.696 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:11.728 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:11.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:11.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:11.831 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:11.860 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:11.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:11.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:11.963 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:11.991 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:11.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:11.993 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:12.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:11 smithi092 bash[58855]: cluster 2023-09-23T19:56:10.536779+0000 mgr.y (mgr.25325) 1466 : cluster 0 pgmap v1483: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 42 op/s 2023-09-23T19:56:12.094 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:12.120 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:12.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:12.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:12.223 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:12.252 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:12.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:12.255 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:12.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:11 smithi016 bash[75004]: cluster 2023-09-23T19:56:10.536779+0000 mgr.y (mgr.25325) 1466 : cluster 0 pgmap v1483: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 42 op/s 2023-09-23T19:56:12.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:11 smithi016 bash[77995]: cluster 2023-09-23T19:56:10.536779+0000 mgr.y (mgr.25325) 1466 : cluster 0 pgmap v1483: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 42 op/s 2023-09-23T19:56:12.356 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:12.387 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:12.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:12.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:12.492 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:12.526 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:12.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:12.529 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:12.630 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:12.665 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:12.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:12.670 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:12.771 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:12.805 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:12.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:12.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:12.909 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:12.941 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:12.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:12.944 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:13.045 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:13.072 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:13.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:13.074 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:13.174 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:13.200 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:13.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:13.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:13.303 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:13.329 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:13.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:13.331 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:13.432 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:13.466 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:13.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:13.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:13.569 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:13.599 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:13.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:13.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:13.703 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:13.734 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:13.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:13.738 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:13.838 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:13.875 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:13.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:13.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:13.979 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:14.001 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:14.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:14.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:14.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:13 smithi092 bash[58855]: cluster 2023-09-23T19:56:12.537353+0000 mgr.y (mgr.25325) 1467 : cluster 0 pgmap v1484: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 38 op/s 2023-09-23T19:56:14.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:13 smithi092 bash[58855]: audit 2023-09-23T19:56:13.684745+0000 mon.a (mon.0) 1783 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:56:14.105 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:14.135 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:14.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:14.138 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:14.239 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:14.247 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:13 smithi016 bash[75004]: cluster 2023-09-23T19:56:12.537353+0000 mgr.y (mgr.25325) 1467 : cluster 0 pgmap v1484: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 38 op/s 2023-09-23T19:56:14.247 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:13 smithi016 bash[75004]: audit 2023-09-23T19:56:13.684745+0000 mon.a (mon.0) 1783 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:56:14.247 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:13 smithi016 bash[77995]: cluster 2023-09-23T19:56:12.537353+0000 mgr.y (mgr.25325) 1467 : cluster 0 pgmap v1484: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 38 op/s 2023-09-23T19:56:14.248 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:13 smithi016 bash[77995]: audit 2023-09-23T19:56:13.684745+0000 mon.a (mon.0) 1783 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:56:14.261 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:14.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:14.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:14.365 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:14.393 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:14.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:14.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:14.497 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:14.530 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:14.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:14.532 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:14.633 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:14.663 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:14.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:14.666 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:14.766 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:14.791 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:14.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:14.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:14.894 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:14.918 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:14.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:14.921 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:15.021 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:15.055 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:15.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:15.058 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:15.159 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:15.186 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:15.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:15.189 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:15.289 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:15.315 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:15.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:15.318 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:15.421 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:15.456 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:15.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:15.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:15.560 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:15.594 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:15.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:15.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:15.698 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:15.724 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:15.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:15.727 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:15.828 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:15.858 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:15.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:15.861 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:15.962 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:15.997 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:15.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:15.999 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:16.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:15 smithi016 bash[75004]: cluster 2023-09-23T19:56:14.537980+0000 mgr.y (mgr.25325) 1468 : cluster 0 pgmap v1485: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 6 op/s 2023-09-23T19:56:16.036 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:56:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:56:15] "GET /metrics HTTP/1.1" 200 36253 "" "Prometheus/2.33.4" 2023-09-23T19:56:16.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:15 smithi016 bash[77995]: cluster 2023-09-23T19:56:14.537980+0000 mgr.y (mgr.25325) 1468 : cluster 0 pgmap v1485: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 6 op/s 2023-09-23T19:56:16.100 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:16.113 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:15 smithi092 bash[58855]: cluster 2023-09-23T19:56:14.537980+0000 mgr.y (mgr.25325) 1468 : cluster 0 pgmap v1485: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 6 op/s 2023-09-23T19:56:16.132 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:16.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:16.135 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:16.236 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:16.267 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:16.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:16.270 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:16.370 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:16.395 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:16.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:16.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:16.498 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:16.524 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:16.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:16.526 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:16.627 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:16.660 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:16.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:16.663 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:16.764 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:16.798 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:16.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:16.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:16.902 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:16.937 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:16.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:16.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:17.041 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:17.063 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:17.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:17.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:17.166 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:17.197 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:17.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:17.201 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:17.301 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:17.334 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:17.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:17.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:17.437 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:17.467 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:17.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:17.471 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:17.572 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:17.594 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:17.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:17.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:17.697 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:17.725 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:17.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:17.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:17.829 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:17.855 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:17.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:17.858 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:17.959 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:17.987 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:17.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:17.990 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:18.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:17 smithi092 bash[58855]: cluster 2023-09-23T19:56:16.538712+0000 mgr.y (mgr.25325) 1469 : cluster 0 pgmap v1486: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:18.091 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:18.126 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:18.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:18.129 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:18.230 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:18.261 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:18.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:18.265 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:18.338 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:17 smithi016 bash[77995]: cluster 2023-09-23T19:56:16.538712+0000 mgr.y (mgr.25325) 1469 : cluster 0 pgmap v1486: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:18.339 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:17 smithi016 bash[75004]: cluster 2023-09-23T19:56:16.538712+0000 mgr.y (mgr.25325) 1469 : cluster 0 pgmap v1486: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:18.366 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:18.399 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:18.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:18.402 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:18.502 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:18.535 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:18.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:18.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:18.639 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:18.669 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:18.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:18.672 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:18.773 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:18.801 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:18.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:18.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:18.904 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:18.935 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:18.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:18.939 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:19.039 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:19.076 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:19.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:19.079 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:19.180 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:19.206 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:19.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:19.209 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:19.310 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:19.347 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:19.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:19.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:19.450 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:19.472 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:19.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:19.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:19.575 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:19.600 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:19.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:19.603 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:19.704 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:19.742 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:19.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:19.745 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:19.845 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:19.878 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:19.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:19.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:19.982 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:19.993 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:19 smithi092 bash[58855]: cluster 2023-09-23T19:56:18.539393+0000 mgr.y (mgr.25325) 1470 : cluster 0 pgmap v1487: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:20.011 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:20.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:20.014 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:20.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:19 smithi016 bash[75004]: cluster 2023-09-23T19:56:18.539393+0000 mgr.y (mgr.25325) 1470 : cluster 0 pgmap v1487: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:20.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:19 smithi016 bash[77995]: cluster 2023-09-23T19:56:18.539393+0000 mgr.y (mgr.25325) 1470 : cluster 0 pgmap v1487: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:20.116 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:20.150 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:20.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:20.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:20.254 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:20.292 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:20.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:20.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:20.395 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:20.426 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:20.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:20.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:20.530 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:20.555 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:20.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:20.557 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:20.658 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:20.694 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:20.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:20.697 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:20.797 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:20.830 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:20.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:20.833 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:20.933 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:20.958 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:20.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:20.960 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:21.060 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:21.092 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:21.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:21.096 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:21.196 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:21.229 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:21.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:21.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:21.334 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:21.369 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:21.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:21.372 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:21.473 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:21.494 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:21.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:21.496 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:21.523 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-09-23T19:56:21.523 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:56:21.597 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:21.631 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:21.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:21.634 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:21.735 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:21.757 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:21.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:21.760 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:21.860 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:21.896 INFO:teuthology.orchestra.run.smithi016.stderr:pool 'unique_pool_8' removed 2023-09-23T19:56:21.904 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:21.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:21.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:21.966 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:21 smithi016 bash[75004]: cluster 2023-09-23T19:56:20.540150+0000 mgr.y (mgr.25325) 1471 : cluster 0 pgmap v1488: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:21.966 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:21 smithi016 bash[75004]: audit 2023-09-23T19:56:20.877961+0000 mon.c (mon.2) 226 : audit 1 from='client.? 172.21.15.16:0/1334925011' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T19:56:21.966 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:21 smithi016 bash[75004]: audit 2023-09-23T19:56:20.878290+0000 mon.a (mon.0) 1784 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T19:56:21.967 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:21 smithi016 bash[77995]: cluster 2023-09-23T19:56:20.540150+0000 mgr.y (mgr.25325) 1471 : cluster 0 pgmap v1488: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:21.967 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:21 smithi016 bash[77995]: audit 2023-09-23T19:56:20.877961+0000 mon.c (mon.2) 226 : audit 1 from='client.? 172.21.15.16:0/1334925011' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T19:56:21.967 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:21 smithi016 bash[77995]: audit 2023-09-23T19:56:20.878290+0000 mon.a (mon.0) 1784 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T19:56:22.019 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:22.027 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:21 smithi092 bash[58855]: cluster 2023-09-23T19:56:20.540150+0000 mgr.y (mgr.25325) 1471 : cluster 0 pgmap v1488: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:22.027 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:21 smithi092 bash[58855]: audit 2023-09-23T19:56:20.877961+0000 mon.c (mon.2) 226 : audit 1 from='client.? 172.21.15.16:0/1334925011' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T19:56:22.027 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:21 smithi092 bash[58855]: audit 2023-09-23T19:56:20.878290+0000 mon.a (mon.0) 1784 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T19:56:22.045 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:22.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:22.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:22.148 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:22.181 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:22.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:22.184 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:22.284 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:22.306 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:22.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:22.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:22.410 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:22.443 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:22.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:22.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:22.547 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:22.572 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:22.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:22.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:22.676 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:22.708 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:22.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:22.712 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:22.812 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:22.837 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:22.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:22.840 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:22.941 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:22.953 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:22 smithi092 bash[58855]: audit 2023-09-23T19:56:21.861565+0000 mon.a (mon.0) 1785 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T19:56:22.953 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:22 smithi092 bash[58855]: cluster 2023-09-23T19:56:21.861670+0000 mon.a (mon.0) 1786 : cluster 0 osdmap e253: 8 total, 8 up, 8 in 2023-09-23T19:56:22.953 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:22 smithi092 bash[58855]: cluster 2023-09-23T19:56:21.877135+0000 osd.7 (osd.7) 166 : cluster 3 failed to encode map e253 with expected crc 2023-09-23T19:56:22.953 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:22 smithi092 bash[58855]: cluster 2023-09-23T19:56:21.877135+0000 osd.5 (osd.5) 181 : cluster 3 failed to encode map e253 with expected crc 2023-09-23T19:56:22.953 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:22 smithi092 bash[58855]: cluster 2023-09-23T19:56:21.879561+0000 osd.4 (osd.4) 196 : cluster 3 failed to encode map e253 with expected crc 2023-09-23T19:56:22.971 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:22.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:22.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:23.075 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:23.087 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:22 smithi016 bash[77995]: audit 2023-09-23T19:56:21.861565+0000 mon.a (mon.0) 1785 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T19:56:23.087 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:22 smithi016 bash[77995]: cluster 2023-09-23T19:56:21.861670+0000 mon.a (mon.0) 1786 : cluster 0 osdmap e253: 8 total, 8 up, 8 in 2023-09-23T19:56:23.088 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:22 smithi016 bash[77995]: cluster 2023-09-23T19:56:21.877135+0000 osd.7 (osd.7) 166 : cluster 3 failed to encode map e253 with expected crc 2023-09-23T19:56:23.088 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:22 smithi016 bash[77995]: cluster 2023-09-23T19:56:21.877135+0000 osd.5 (osd.5) 181 : cluster 3 failed to encode map e253 with expected crc 2023-09-23T19:56:23.088 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:22 smithi016 bash[77995]: cluster 2023-09-23T19:56:21.879561+0000 osd.4 (osd.4) 196 : cluster 3 failed to encode map e253 with expected crc 2023-09-23T19:56:23.088 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:22 smithi016 bash[75004]: audit 2023-09-23T19:56:21.861565+0000 mon.a (mon.0) 1785 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T19:56:23.088 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:22 smithi016 bash[75004]: cluster 2023-09-23T19:56:21.861670+0000 mon.a (mon.0) 1786 : cluster 0 osdmap e253: 8 total, 8 up, 8 in 2023-09-23T19:56:23.088 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:22 smithi016 bash[75004]: cluster 2023-09-23T19:56:21.877135+0000 osd.7 (osd.7) 166 : cluster 3 failed to encode map e253 with expected crc 2023-09-23T19:56:23.088 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:22 smithi016 bash[75004]: cluster 2023-09-23T19:56:21.877135+0000 osd.5 (osd.5) 181 : cluster 3 failed to encode map e253 with expected crc 2023-09-23T19:56:23.088 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:22 smithi016 bash[75004]: cluster 2023-09-23T19:56:21.879561+0000 osd.4 (osd.4) 196 : cluster 3 failed to encode map e253 with expected crc 2023-09-23T19:56:23.106 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:23.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:23.109 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:23.210 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:23.234 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:23.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:23.236 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:23.336 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:23.373 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:23.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:23.375 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:23.476 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:23.509 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:23.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:23.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:23.612 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:23.635 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:23.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:23.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:23.738 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:23.762 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:23.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:23.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:23.865 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:23.895 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:23.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:23.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:24.000 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:24.030 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:24.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:24.034 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:24.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:23 smithi016 bash[77995]: cluster 2023-09-23T19:56:21.879363+0000 osd.6 (osd.6) 188 : cluster 3 failed to encode map e253 with expected crc 2023-09-23T19:56:24.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:23 smithi016 bash[77995]: cluster 2023-09-23T19:56:22.540633+0000 mgr.y (mgr.25325) 1472 : cluster 0 pgmap v1490: 73 pgs: 73 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:24.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:23 smithi016 bash[77995]: audit 2023-09-23T19:56:23.396374+0000 mon.a (mon.0) 1787 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:56:24.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:23 smithi016 bash[75004]: cluster 2023-09-23T19:56:21.879363+0000 osd.6 (osd.6) 188 : cluster 3 failed to encode map e253 with expected crc 2023-09-23T19:56:24.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:23 smithi016 bash[75004]: cluster 2023-09-23T19:56:22.540633+0000 mgr.y (mgr.25325) 1472 : cluster 0 pgmap v1490: 73 pgs: 73 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:24.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:23 smithi016 bash[75004]: audit 2023-09-23T19:56:23.396374+0000 mon.a (mon.0) 1787 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:56:24.135 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:24.147 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:23 smithi092 bash[58855]: cluster 2023-09-23T19:56:21.879363+0000 osd.6 (osd.6) 188 : cluster 3 failed to encode map e253 with expected crc 2023-09-23T19:56:24.148 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:23 smithi092 bash[58855]: cluster 2023-09-23T19:56:22.540633+0000 mgr.y (mgr.25325) 1472 : cluster 0 pgmap v1490: 73 pgs: 73 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:24.148 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:23 smithi092 bash[58855]: audit 2023-09-23T19:56:23.396374+0000 mon.a (mon.0) 1787 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:56:24.167 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:24.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:24.170 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:24.270 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:24.304 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:24.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:24.306 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:24.407 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:24.435 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:24.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:24.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:24.538 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:24.569 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:24.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:24.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:24.673 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:24.705 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:24.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:24.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:24.809 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:24.843 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:24.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:24.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:24.948 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:24.969 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:24.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:24.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:25.072 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:25.098 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:25.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:25.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:25.174 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T19:56:25.201 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:25.245 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:25.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:25.248 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:25.349 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:25.368 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:25.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:25.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:25.471 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:25.490 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:25.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:25.493 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:25.594 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:25.618 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:25.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:25.620 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:25.721 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:25.743 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:25.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:25.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:25.846 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:25.880 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:25.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:25.883 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:25.984 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:26.015 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:26.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:26.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:26.058 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:25 smithi016 bash[77995]: cluster 2023-09-23T19:56:24.541215+0000 mgr.y (mgr.25325) 1473 : cluster 0 pgmap v1491: 73 pgs: 73 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:26.059 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:25 smithi016 bash[75004]: cluster 2023-09-23T19:56:24.541215+0000 mgr.y (mgr.25325) 1473 : cluster 0 pgmap v1491: 73 pgs: 73 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:26.059 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:56:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:56:25] "GET /metrics HTTP/1.1" 200 36253 "" "Prometheus/2.33.4" 2023-09-23T19:56:26.119 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:26.131 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:25 smithi092 bash[58855]: cluster 2023-09-23T19:56:24.541215+0000 mgr.y (mgr.25325) 1473 : cluster 0 pgmap v1491: 73 pgs: 73 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:26.149 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:26.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:26.152 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:26.253 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:26.276 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:26.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:26.281 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:26.382 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:26.406 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:26.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:26.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:26.509 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:26.531 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:26.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:26.533 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:26.634 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:26.666 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:26.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:26.669 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:26.770 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:26.794 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:26.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:26.796 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:26.887 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T19:56:26.898 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:26.923 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:26.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:26.925 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:27.026 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:27.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:26 smithi016 bash[75004]: audit 2023-09-23T19:56:26.491602+0000 mon.a (mon.0) 1788 : audit 1 from='client.? 172.21.15.16:0/2680324119' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T19:56:27.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:26 smithi016 bash[77995]: audit 2023-09-23T19:56:26.491602+0000 mon.a (mon.0) 1788 : audit 1 from='client.? 172.21.15.16:0/2680324119' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T19:56:27.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:26 smithi092 bash[58855]: audit 2023-09-23T19:56:26.491602+0000 mon.a (mon.0) 1788 : audit 1 from='client.? 172.21.15.16:0/2680324119' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T19:56:27.060 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:27.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:27.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:27.164 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:27.190 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:27.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:27.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:27.294 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:27.327 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:27.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:27.331 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:27.431 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:27.462 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:27.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:27.465 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:27.566 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:27.594 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:27.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:27.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:27.697 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:27.727 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:27.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:27.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:27.832 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:27.862 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:27.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:27.866 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:27.955 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T19:56:27.955 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T19:56:27.956 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_9 2023-09-23T19:56:27.956 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_9 16 2023-09-23T19:56:27.966 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:27.979 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:27 smithi092 bash[58855]: cluster 2023-09-23T19:56:26.542032+0000 mgr.y (mgr.25325) 1474 : cluster 0 pgmap v1492: 73 pgs: 73 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:27.979 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:27 smithi092 bash[58855]: audit 2023-09-23T19:56:26.880674+0000 mon.a (mon.0) 1789 : audit 1 from='client.? 172.21.15.16:0/2680324119' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T19:56:27.979 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:27 smithi092 bash[58855]: cluster 2023-09-23T19:56:26.880749+0000 mon.a (mon.0) 1790 : cluster 0 osdmap e254: 8 total, 8 up, 8 in 2023-09-23T19:56:27.979 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:27 smithi092 bash[58855]: cluster 2023-09-23T19:56:26.884776+0000 osd.5 (osd.5) 182 : cluster 3 failed to encode map e254 with expected crc 2023-09-23T19:56:27.999 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:28.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:28.002 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:28.103 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:28.132 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:28.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:28.135 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:28.235 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:28.268 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:28.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:28.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:28.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:27 smithi016 bash[75004]: cluster 2023-09-23T19:56:26.542032+0000 mgr.y (mgr.25325) 1474 : cluster 0 pgmap v1492: 73 pgs: 73 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:28.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:27 smithi016 bash[75004]: audit 2023-09-23T19:56:26.880674+0000 mon.a (mon.0) 1789 : audit 1 from='client.? 172.21.15.16:0/2680324119' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T19:56:28.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:27 smithi016 bash[75004]: cluster 2023-09-23T19:56:26.880749+0000 mon.a (mon.0) 1790 : cluster 0 osdmap e254: 8 total, 8 up, 8 in 2023-09-23T19:56:28.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:27 smithi016 bash[75004]: cluster 2023-09-23T19:56:26.884776+0000 osd.5 (osd.5) 182 : cluster 3 failed to encode map e254 with expected crc 2023-09-23T19:56:28.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:27 smithi016 bash[77995]: cluster 2023-09-23T19:56:26.542032+0000 mgr.y (mgr.25325) 1474 : cluster 0 pgmap v1492: 73 pgs: 73 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:28.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:27 smithi016 bash[77995]: audit 2023-09-23T19:56:26.880674+0000 mon.a (mon.0) 1789 : audit 1 from='client.? 172.21.15.16:0/2680324119' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T19:56:28.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:27 smithi016 bash[77995]: cluster 2023-09-23T19:56:26.880749+0000 mon.a (mon.0) 1790 : cluster 0 osdmap e254: 8 total, 8 up, 8 in 2023-09-23T19:56:28.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:27 smithi016 bash[77995]: cluster 2023-09-23T19:56:26.884776+0000 osd.5 (osd.5) 182 : cluster 3 failed to encode map e254 with expected crc 2023-09-23T19:56:28.372 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:28.403 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:28.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:28.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:28.507 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:28.533 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:28.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:28.536 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:28.636 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:28.665 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:28.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:28.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:28.768 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:28.799 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:28.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:28.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:28.904 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:28.915 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:28 smithi092 bash[58855]: cluster 2023-09-23T19:56:27.617531+0000 osd.7 (osd.7) 167 : cluster 3 failed to encode map e254 with expected crc 2023-09-23T19:56:28.915 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:28 smithi092 bash[58855]: cluster 2023-09-23T19:56:28.102156+0000 osd.4 (osd.4) 197 : cluster 3 failed to encode map e254 with expected crc 2023-09-23T19:56:28.915 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:28 smithi092 bash[58855]: audit 2023-09-23T19:56:28.685658+0000 mon.a (mon.0) 1791 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:56:28.931 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:28.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:28.933 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:29.034 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:29.089 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:29.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:29.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:29.193 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:29.219 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:29.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:29.222 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:29.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:28 smithi016 bash[75004]: cluster 2023-09-23T19:56:27.617531+0000 osd.7 (osd.7) 167 : cluster 3 failed to encode map e254 with expected crc 2023-09-23T19:56:29.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:28 smithi016 bash[75004]: cluster 2023-09-23T19:56:28.102156+0000 osd.4 (osd.4) 197 : cluster 3 failed to encode map e254 with expected crc 2023-09-23T19:56:29.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:28 smithi016 bash[75004]: audit 2023-09-23T19:56:28.685658+0000 mon.a (mon.0) 1791 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:56:29.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:28 smithi016 bash[77995]: cluster 2023-09-23T19:56:27.617531+0000 osd.7 (osd.7) 167 : cluster 3 failed to encode map e254 with expected crc 2023-09-23T19:56:29.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:28 smithi016 bash[77995]: cluster 2023-09-23T19:56:28.102156+0000 osd.4 (osd.4) 197 : cluster 3 failed to encode map e254 with expected crc 2023-09-23T19:56:29.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:28 smithi016 bash[77995]: audit 2023-09-23T19:56:28.685658+0000 mon.a (mon.0) 1791 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:56:29.322 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:29.350 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:29.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:29.353 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:29.454 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:29.497 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:29.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:29.500 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:29.519 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T19:56:29.600 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:29.621 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:29.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:29.624 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:29.725 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:29.759 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:29.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:29.763 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:29.863 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:29.891 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:29.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:29.896 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:29.989 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:56:29.996 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:30.022 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:56:30.023 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:30.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:30.026 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:30.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:29 smithi092 bash[58855]: cluster 2023-09-23T19:56:28.542769+0000 mgr.y (mgr.25325) 1475 : cluster 0 pgmap v1494: 73 pgs: 73 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:30.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:29 smithi092 bash[58855]: audit 2023-09-23T19:56:29.693992+0000 mon.a (mon.0) 1792 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:56:30.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:29 smithi092 bash[58855]: audit 2023-09-23T19:56:29.700751+0000 mon.a (mon.0) 1793 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:56:30.048 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:56:30.090 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:56:30.127 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:30.140 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:56:30.163 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:30.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:30.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:30.168 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:56:30.198 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:56:30.225 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:56:30.227 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:56:30.227 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:56:30.227 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:56:30.228 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:56:30.228 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:56:30.228 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:56:30.228 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:56:30.228 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:56:30.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:29 smithi016 bash[77995]: cluster 2023-09-23T19:56:28.542769+0000 mgr.y (mgr.25325) 1475 : cluster 0 pgmap v1494: 73 pgs: 73 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:30.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:29 smithi016 bash[77995]: audit 2023-09-23T19:56:29.693992+0000 mon.a (mon.0) 1792 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:56:30.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:29 smithi016 bash[77995]: audit 2023-09-23T19:56:29.700751+0000 mon.a (mon.0) 1793 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:56:30.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:29 smithi016 bash[75004]: cluster 2023-09-23T19:56:28.542769+0000 mgr.y (mgr.25325) 1475 : cluster 0 pgmap v1494: 73 pgs: 73 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:30.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:29 smithi016 bash[75004]: audit 2023-09-23T19:56:29.693992+0000 mon.a (mon.0) 1792 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:56:30.254 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:29 smithi016 bash[75004]: audit 2023-09-23T19:56:29.700751+0000 mon.a (mon.0) 1793 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:56:30.268 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:30.293 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:30.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:30.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:30.396 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:30.428 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:30.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:30.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:30.531 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:30.567 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:30.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:30.570 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:30.671 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:30.699 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:30.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:30.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:30.803 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:30.831 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:30.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:30.834 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:30.909 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is unset 2023-09-23T19:56:30.935 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:30.946 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:30 smithi016 bash[77995]: audit 2023-09-23T19:56:29.953343+0000 mon.c (mon.2) 227 : audit 0 from='client.? 172.21.15.16:0/3765126529' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:56:30.946 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:30 smithi016 bash[77995]: audit 2023-09-23T19:56:30.230920+0000 mon.a (mon.0) 1794 : audit 1 from='client.? 172.21.15.16:0/2770931865' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:56:30.946 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:30 smithi016 bash[75004]: audit 2023-09-23T19:56:29.953343+0000 mon.c (mon.2) 227 : audit 0 from='client.? 172.21.15.16:0/3765126529' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:56:30.946 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:30 smithi016 bash[75004]: audit 2023-09-23T19:56:30.230920+0000 mon.a (mon.0) 1794 : audit 1 from='client.? 172.21.15.16:0/2770931865' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:56:30.964 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:30.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:30.967 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:31.068 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:31.102 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:31.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:31.104 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:31.205 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:31.219 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:30 smithi092 bash[58855]: audit 2023-09-23T19:56:29.953343+0000 mon.c (mon.2) 227 : audit 0 from='client.? 172.21.15.16:0/3765126529' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:56:31.219 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:30 smithi092 bash[58855]: audit 2023-09-23T19:56:30.230920+0000 mon.a (mon.0) 1794 : audit 1 from='client.? 172.21.15.16:0/2770931865' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:56:31.241 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:31.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:31.244 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:31.345 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:31.374 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:31.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:31.377 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:31.478 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:31.511 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:31.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:31.514 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:31.615 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:31.645 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:31.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:31.648 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:31.749 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:31.777 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:31.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:31.781 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:31.882 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:31.913 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:31.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:31.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:32.017 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:32.032 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:31 smithi016 bash[75004]: cluster 2023-09-23T19:56:30.543489+0000 mgr.y (mgr.25325) 1476 : cluster 0 pgmap v1495: 73 pgs: 73 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:32.032 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:31 smithi016 bash[75004]: cluster 2023-09-23T19:56:30.887436+0000 mon.a (mon.0) 1795 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:56:32.032 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:31 smithi016 bash[75004]: audit 2023-09-23T19:56:30.901529+0000 mon.a (mon.0) 1796 : audit 1 from='client.? 172.21.15.16:0/2770931865' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:56:32.032 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:31 smithi016 bash[75004]: cluster 2023-09-23T19:56:30.901633+0000 mon.a (mon.0) 1797 : cluster 0 osdmap e255: 8 total, 8 up, 8 in 2023-09-23T19:56:32.032 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:31 smithi016 bash[75004]: cluster 2023-09-23T19:56:31.565796+0000 osd.6 (osd.6) 189 : cluster 3 failed to encode map e255 with expected crc 2023-09-23T19:56:32.032 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:31 smithi016 bash[75004]: cluster 2023-09-23T19:56:31.568531+0000 osd.6 (osd.6) 190 : cluster 3 failed to encode map e255 with expected crc 2023-09-23T19:56:32.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:31 smithi016 bash[75004]: cluster 2023-09-23T19:56:31.571220+0000 osd.6 (osd.6) 191 : cluster 3 failed to encode map e255 with expected crc 2023-09-23T19:56:32.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:31 smithi016 bash[77995]: cluster 2023-09-23T19:56:30.543489+0000 mgr.y (mgr.25325) 1476 : cluster 0 pgmap v1495: 73 pgs: 73 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:32.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:31 smithi016 bash[77995]: cluster 2023-09-23T19:56:30.887436+0000 mon.a (mon.0) 1795 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:56:32.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:31 smithi016 bash[77995]: audit 2023-09-23T19:56:30.901529+0000 mon.a (mon.0) 1796 : audit 1 from='client.? 172.21.15.16:0/2770931865' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:56:32.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:31 smithi016 bash[77995]: cluster 2023-09-23T19:56:30.901633+0000 mon.a (mon.0) 1797 : cluster 0 osdmap e255: 8 total, 8 up, 8 in 2023-09-23T19:56:32.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:31 smithi016 bash[77995]: cluster 2023-09-23T19:56:31.565796+0000 osd.6 (osd.6) 189 : cluster 3 failed to encode map e255 with expected crc 2023-09-23T19:56:32.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:31 smithi016 bash[77995]: cluster 2023-09-23T19:56:31.568531+0000 osd.6 (osd.6) 190 : cluster 3 failed to encode map e255 with expected crc 2023-09-23T19:56:32.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:31 smithi016 bash[77995]: cluster 2023-09-23T19:56:31.571220+0000 osd.6 (osd.6) 191 : cluster 3 failed to encode map e255 with expected crc 2023-09-23T19:56:32.051 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:32.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:32.054 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:32.155 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:32.182 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:32.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:32.186 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:32.287 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:32.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:31 smithi092 bash[58855]: cluster 2023-09-23T19:56:30.543489+0000 mgr.y (mgr.25325) 1476 : cluster 0 pgmap v1495: 73 pgs: 73 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:32.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:31 smithi092 bash[58855]: cluster 2023-09-23T19:56:30.887436+0000 mon.a (mon.0) 1795 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:56:32.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:31 smithi092 bash[58855]: audit 2023-09-23T19:56:30.901529+0000 mon.a (mon.0) 1796 : audit 1 from='client.? 172.21.15.16:0/2770931865' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:56:32.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:31 smithi092 bash[58855]: cluster 2023-09-23T19:56:30.901633+0000 mon.a (mon.0) 1797 : cluster 0 osdmap e255: 8 total, 8 up, 8 in 2023-09-23T19:56:32.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:31 smithi092 bash[58855]: cluster 2023-09-23T19:56:31.565796+0000 osd.6 (osd.6) 189 : cluster 3 failed to encode map e255 with expected crc 2023-09-23T19:56:32.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:31 smithi092 bash[58855]: cluster 2023-09-23T19:56:31.568531+0000 osd.6 (osd.6) 190 : cluster 3 failed to encode map e255 with expected crc 2023-09-23T19:56:32.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:31 smithi092 bash[58855]: cluster 2023-09-23T19:56:31.571220+0000 osd.6 (osd.6) 191 : cluster 3 failed to encode map e255 with expected crc 2023-09-23T19:56:32.318 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:32.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:32.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:32.422 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:32.457 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:32.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:32.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:32.561 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:32.597 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:32.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:32.601 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:32.702 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:32.735 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:32.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:32.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:32.839 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:32.875 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:32.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:32.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:32.980 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:32.992 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:32 smithi092 bash[58855]: cluster 2023-09-23T19:56:31.936622+0000 osd.7 (osd.7) 168 : cluster 3 failed to encode map e255 with expected crc 2023-09-23T19:56:32.992 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:32 smithi092 bash[58855]: cluster 2023-09-23T19:56:32.112039+0000 osd.4 (osd.4) 198 : cluster 3 failed to encode map e255 with expected crc 2023-09-23T19:56:32.992 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:32 smithi092 bash[58855]: cluster 2023-09-23T19:56:32.275492+0000 osd.5 (osd.5) 183 : cluster 3 failed to encode map e255 with expected crc 2023-09-23T19:56:33.010 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:33.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:33.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:33.114 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:33.146 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:33.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:33.150 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:33.251 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:33.281 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:33.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:33.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:33.384 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:56:33.384 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:33.396 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:33 smithi016 bash[75004]: cluster 2023-09-23T19:56:31.936622+0000 osd.7 (osd.7) 168 : cluster 3 failed to encode map e255 with expected crc 2023-09-23T19:56:33.396 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:33 smithi016 bash[75004]: cluster 2023-09-23T19:56:32.112039+0000 osd.4 (osd.4) 198 : cluster 3 failed to encode map e255 with expected crc 2023-09-23T19:56:33.397 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:33 smithi016 bash[75004]: cluster 2023-09-23T19:56:32.275492+0000 osd.5 (osd.5) 183 : cluster 3 failed to encode map e255 with expected crc 2023-09-23T19:56:33.397 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:33 smithi016 bash[77995]: cluster 2023-09-23T19:56:31.936622+0000 osd.7 (osd.7) 168 : cluster 3 failed to encode map e255 with expected crc 2023-09-23T19:56:33.397 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:33 smithi016 bash[77995]: cluster 2023-09-23T19:56:32.112039+0000 osd.4 (osd.4) 198 : cluster 3 failed to encode map e255 with expected crc 2023-09-23T19:56:33.397 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:33 smithi016 bash[77995]: cluster 2023-09-23T19:56:32.275492+0000 osd.5 (osd.5) 183 : cluster 3 failed to encode map e255 with expected crc 2023-09-23T19:56:33.413 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:33.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:33.416 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:33.517 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:33.548 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:33.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:33.551 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:33.652 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:33.682 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:33.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:33.685 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:33.766 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:56:33.786 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:33.818 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:33.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:33.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:33.922 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:33.954 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:33.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:33.957 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:34.058 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:34.071 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:33 smithi092 bash[58855]: cluster 2023-09-23T19:56:32.544189+0000 mgr.y (mgr.25325) 1477 : cluster 0 pgmap v1497: 73 pgs: 73 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:34.092 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:34.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:34.095 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:34.196 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:34.230 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:34.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:34.233 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:34.273 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T19:56:34.283 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:33 smithi016 bash[77995]: cluster 2023-09-23T19:56:32.544189+0000 mgr.y (mgr.25325) 1477 : cluster 0 pgmap v1497: 73 pgs: 73 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:34.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:33 smithi016 bash[75004]: cluster 2023-09-23T19:56:32.544189+0000 mgr.y (mgr.25325) 1477 : cluster 0 pgmap v1497: 73 pgs: 73 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:34.334 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:34.363 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:34.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:34.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:34.468 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:34.491 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:34.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:34.494 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:34.595 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:34.626 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:34.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:34.630 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:34.730 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:34.762 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:34.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:34.765 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:34.866 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:34.895 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:34.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:34.899 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:35.000 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:35.030 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:35.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:35.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:35.134 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:35.164 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:35.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:35.169 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:35.270 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:35.302 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:35.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:35.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:35.406 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:35.436 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:35.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:35.440 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:35.540 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:35.569 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:35.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:35.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:35.672 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:35.704 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:35.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:35.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:35.786 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:56:35 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:56:35] "GET /metrics HTTP/1.1" 200 33898 "" "Prometheus/2.33.4" 2023-09-23T19:56:35.808 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:35.842 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:35.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:35.846 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:35.946 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:35.977 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:35.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:35.980 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:36.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:35 smithi092 bash[58855]: cluster 2023-09-23T19:56:34.544791+0000 mgr.y (mgr.25325) 1478 : cluster 0 pgmap v1498: 73 pgs: 73 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:36.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:35 smithi092 bash[58855]: audit 2023-09-23T19:56:34.916741+0000 mon.a (mon.0) 1798 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:56:36.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:35 smithi092 bash[58855]: audit 2023-09-23T19:56:34.923253+0000 mon.a (mon.0) 1799 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:56:36.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:35 smithi092 bash[58855]: audit 2023-09-23T19:56:35.396770+0000 mon.a (mon.0) 1800 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:56:36.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:35 smithi092 bash[58855]: audit 2023-09-23T19:56:35.398218+0000 mon.a (mon.0) 1801 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:56:36.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:35 smithi092 bash[58855]: audit 2023-09-23T19:56:35.404887+0000 mon.a (mon.0) 1802 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:56:36.081 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:36.114 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:36.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:36.118 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:36.218 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:36.227 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:35 smithi016 bash[77995]: cluster 2023-09-23T19:56:34.544791+0000 mgr.y (mgr.25325) 1478 : cluster 0 pgmap v1498: 73 pgs: 73 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:36.227 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:35 smithi016 bash[77995]: audit 2023-09-23T19:56:34.916741+0000 mon.a (mon.0) 1798 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:56:36.227 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:35 smithi016 bash[77995]: audit 2023-09-23T19:56:34.923253+0000 mon.a (mon.0) 1799 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:56:36.227 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:35 smithi016 bash[77995]: audit 2023-09-23T19:56:35.396770+0000 mon.a (mon.0) 1800 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:56:36.227 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:35 smithi016 bash[77995]: audit 2023-09-23T19:56:35.398218+0000 mon.a (mon.0) 1801 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:56:36.227 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:35 smithi016 bash[77995]: audit 2023-09-23T19:56:35.404887+0000 mon.a (mon.0) 1802 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:56:36.227 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:35 smithi016 bash[75004]: cluster 2023-09-23T19:56:34.544791+0000 mgr.y (mgr.25325) 1478 : cluster 0 pgmap v1498: 73 pgs: 73 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:36.227 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:35 smithi016 bash[75004]: audit 2023-09-23T19:56:34.916741+0000 mon.a (mon.0) 1798 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:56:36.228 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:35 smithi016 bash[75004]: audit 2023-09-23T19:56:34.923253+0000 mon.a (mon.0) 1799 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:56:36.228 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:35 smithi016 bash[75004]: audit 2023-09-23T19:56:35.396770+0000 mon.a (mon.0) 1800 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:56:36.228 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:35 smithi016 bash[75004]: audit 2023-09-23T19:56:35.398218+0000 mon.a (mon.0) 1801 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:56:36.228 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:35 smithi016 bash[75004]: audit 2023-09-23T19:56:35.404887+0000 mon.a (mon.0) 1802 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:56:36.245 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:36.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:36.248 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:36.348 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:36.383 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:36.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:36.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:36.487 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:36.515 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:36.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:36.518 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:36.619 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:36.651 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:36.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:36.654 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:36.755 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:36.791 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:36.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:36.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:36.895 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:36.925 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:56:36.929 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:56:36.931 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:36.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:36.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:37.035 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:37.069 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:37.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:37.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:37.174 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:37.200 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:37.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:37.203 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:37.303 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:37.334 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:37.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:37.337 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:37.438 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:37.465 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:37.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:37.468 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:37.569 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:37.606 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:37.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:37.609 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:37.710 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:37.749 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:37.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:37.752 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:37.852 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:37.884 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:37.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:37.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:37.987 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:38.013 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:38.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:38.015 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:38.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:37 smithi092 bash[58855]: cluster 2023-09-23T19:56:36.545836+0000 mgr.y (mgr.25325) 1479 : cluster 0 pgmap v1499: 73 pgs: 73 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:38.116 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:38.150 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:38.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:38.152 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:38.253 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:38.264 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:37 smithi016 bash[75004]: cluster 2023-09-23T19:56:36.545836+0000 mgr.y (mgr.25325) 1479 : cluster 0 pgmap v1499: 73 pgs: 73 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:38.265 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:37 smithi016 bash[77995]: cluster 2023-09-23T19:56:36.545836+0000 mgr.y (mgr.25325) 1479 : cluster 0 pgmap v1499: 73 pgs: 73 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:38.281 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:38.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:38.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:38.385 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:38.416 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:38.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:38.419 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:38.520 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:38.546 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:38.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:38.550 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:38.650 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:38.673 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:38.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:38.676 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:38.777 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:38.803 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:38.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:38.806 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:38.844 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:56:38.848 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:56:38.906 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:38.921 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-09-23T19:56:38.921 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [], 2023-09-23T19:56:38.921 INFO:teuthology.orchestra.run.smithi016.stdout: "num_ops": 0 2023-09-23T19:56:38.921 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-09-23T19:56:38.934 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:38.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:38.936 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:39.037 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:39.058 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:39.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:39.061 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:39.161 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:39.194 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:39.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:39.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:39.298 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:39.323 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:39.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:39.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:39.427 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:39.452 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:39.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:39.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:39.555 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:39.576 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:39.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:39.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:39.679 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:39.701 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:39.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:39.704 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:39.804 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:39.836 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:39.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:39.839 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:39.940 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:39.972 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:39.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:39.976 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:40.077 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:40.088 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:39 smithi092 bash[58855]: cluster 2023-09-23T19:56:38.546340+0000 mgr.y (mgr.25325) 1480 : cluster 0 pgmap v1500: 73 pgs: 73 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:40.106 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:40.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:40.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:40.210 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:40.224 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:39 smithi016 bash[77995]: cluster 2023-09-23T19:56:38.546340+0000 mgr.y (mgr.25325) 1480 : cluster 0 pgmap v1500: 73 pgs: 73 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:40.224 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:39 smithi016 bash[75004]: cluster 2023-09-23T19:56:38.546340+0000 mgr.y (mgr.25325) 1480 : cluster 0 pgmap v1500: 73 pgs: 73 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:40.237 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:40.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:40.240 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:40.341 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:40.367 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:40.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:40.369 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:40.470 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:40.501 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:40.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:40.504 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:40.605 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:40.631 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:40.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:40.634 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:40.734 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:40.759 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:40.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:40.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:40.862 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:40.892 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:40.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:40.895 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:40.996 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:41.017 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:41.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:41.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:41.120 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:41.142 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:41.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:41.144 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:41.244 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:41.275 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:41.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:41.277 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:41.378 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:41.401 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:41.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:41.403 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:41.504 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:41.534 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:41.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:41.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:41.638 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:41.671 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:41.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:41.674 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:41.775 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:41.802 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:41.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:41.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:41.906 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:41.936 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:41.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:41.939 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:42.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:41 smithi016 bash[75004]: cluster 2023-09-23T19:56:40.547267+0000 mgr.y (mgr.25325) 1481 : cluster 0 pgmap v1501: 73 pgs: 73 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:42.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:41 smithi016 bash[77995]: cluster 2023-09-23T19:56:40.547267+0000 mgr.y (mgr.25325) 1481 : cluster 0 pgmap v1501: 73 pgs: 73 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:42.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:41 smithi092 bash[58855]: cluster 2023-09-23T19:56:40.547267+0000 mgr.y (mgr.25325) 1481 : cluster 0 pgmap v1501: 73 pgs: 73 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:42.040 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:42.066 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:42.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:42.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:42.169 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:42.199 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:42.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:42.201 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:42.302 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:42.332 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:42.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:42.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:42.436 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:42.467 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:42.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:42.471 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:42.572 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:42.602 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:42.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:42.604 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:42.705 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:42.736 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:42.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:42.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:42.839 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:42.876 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:42.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:42.879 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:42.980 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:43.006 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:43.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:43.009 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:43.109 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:43.144 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:43.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:43.147 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:43.156 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T19:56:43.248 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:43.280 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:43.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:43.284 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:43.384 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:43.416 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:43.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:43.419 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:43.520 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:43.555 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:43.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:43.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:43.659 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:43.692 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:43.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:43.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:43.796 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:43.821 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:43.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:43.824 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:43.924 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:43.951 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:43.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:43.954 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:44.054 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:44.087 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:44.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:44.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:44.190 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:44.202 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:43 smithi016 bash[75004]: cluster 2023-09-23T19:56:42.547955+0000 mgr.y (mgr.25325) 1482 : cluster 0 pgmap v1502: 73 pgs: 73 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:44.202 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:43 smithi016 bash[75004]: audit 2023-09-23T19:56:43.685421+0000 mon.a (mon.0) 1803 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:56:44.202 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:43 smithi016 bash[77995]: cluster 2023-09-23T19:56:42.547955+0000 mgr.y (mgr.25325) 1482 : cluster 0 pgmap v1502: 73 pgs: 73 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:44.202 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:43 smithi016 bash[77995]: audit 2023-09-23T19:56:43.685421+0000 mon.a (mon.0) 1803 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:56:44.217 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:44.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:44.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:44.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:43 smithi092 bash[58855]: cluster 2023-09-23T19:56:42.547955+0000 mgr.y (mgr.25325) 1482 : cluster 0 pgmap v1502: 73 pgs: 73 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:44.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:43 smithi092 bash[58855]: audit 2023-09-23T19:56:43.685421+0000 mon.a (mon.0) 1803 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:56:44.320 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:44.351 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:44.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:44.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:44.456 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:44.489 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:44.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:44.492 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:44.593 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:44.625 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:44.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:44.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:44.728 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:44.760 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:44.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:44.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:44.864 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:44.891 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:44.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:44.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:44.995 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:45.022 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:45.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:45.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:45.125 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:45.158 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:45.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:45.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:45.261 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:45.291 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:45.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:45.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:45.395 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:45.426 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:45.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:45.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:45.529 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:45.562 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:45.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:45.565 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:45.666 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:45.688 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:45.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:45.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:45.791 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:45.812 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:45.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:45.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:45.916 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:45.917 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:56:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:56:45] "GET /metrics HTTP/1.1" 200 33890 "" "Prometheus/2.33.4" 2023-09-23T19:56:45.937 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:45.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:45.940 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:46.040 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:46.066 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:46.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:46.069 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:46.169 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:46.184 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:45 smithi016 bash[75004]: cluster 2023-09-23T19:56:44.548738+0000 mgr.y (mgr.25325) 1483 : cluster 0 pgmap v1503: 73 pgs: 73 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:46.184 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:45 smithi016 bash[77995]: cluster 2023-09-23T19:56:44.548738+0000 mgr.y (mgr.25325) 1483 : cluster 0 pgmap v1503: 73 pgs: 73 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:46.201 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:46.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:46.203 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:46.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:45 smithi092 bash[58855]: cluster 2023-09-23T19:56:44.548738+0000 mgr.y (mgr.25325) 1483 : cluster 0 pgmap v1503: 73 pgs: 73 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:46.304 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:46.338 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:46.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:46.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:46.441 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:46.469 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:46.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:46.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:46.574 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:46.609 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:46.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:46.612 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:46.713 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:46.743 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:46.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:46.746 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:46.847 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:46.882 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:46.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:46.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:46.985 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:47.011 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:47.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:47.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:47.114 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:47.140 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:47.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:47.142 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:47.243 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:47.275 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:47.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:47.278 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:47.379 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:47.406 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:47.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:47.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:47.510 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:47.541 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:47.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:47.545 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:47.645 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:47.678 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:47.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:47.681 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:47.782 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:47.812 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:47.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:47.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:47.916 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:47.948 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:47.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:47.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:48.052 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:48.063 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:47 smithi092 bash[58855]: cluster 2023-09-23T19:56:46.549723+0000 mgr.y (mgr.25325) 1484 : cluster 0 pgmap v1504: 73 pgs: 73 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:48.086 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:48.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:48.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:48.190 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:48.201 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:47 smithi016 bash[77995]: cluster 2023-09-23T19:56:46.549723+0000 mgr.y (mgr.25325) 1484 : cluster 0 pgmap v1504: 73 pgs: 73 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:48.201 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:47 smithi016 bash[75004]: cluster 2023-09-23T19:56:46.549723+0000 mgr.y (mgr.25325) 1484 : cluster 0 pgmap v1504: 73 pgs: 73 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:48.215 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:48.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:48.218 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:48.318 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:48.348 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:48.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:48.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:48.452 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:48.479 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:48.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:48.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:48.582 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:48.612 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:48.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:48.615 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:48.716 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:48.748 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:48.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:48.751 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:48.852 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:48.884 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:48.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:48.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:48.987 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:49.016 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:49.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:49.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:49.068 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:56:49.119 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:49.137 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:49.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:49.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:49.240 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:49.263 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:49.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:49.266 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:49.351 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T19:56:49.353 INFO:teuthology.orchestra.run.smithi016.stderr:pool 'unique_pool_9' created 2023-09-23T19:56:49.366 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:49.399 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:49.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:49.402 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:49.503 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:49.511 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:49 smithi016 bash[75004]: audit 2023-09-23T19:56:48.830290+0000 mon.a (mon.0) 1804 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]: dispatch 2023-09-23T19:56:49.511 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:49 smithi016 bash[75004]: audit 2023-09-23T19:56:48.835445+0000 mon.b (mon.1) 59 : audit 1 from='client.? 172.21.15.16:0/3482255068' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]: dispatch 2023-09-23T19:56:49.511 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:49 smithi016 bash[75004]: audit 2023-09-23T19:56:48.969056+0000 mon.c (mon.2) 228 : audit 1 from='client.? 172.21.15.16:0/2742436240' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T19:56:49.511 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:49 smithi016 bash[75004]: audit 2023-09-23T19:56:49.283842+0000 mon.a (mon.0) 1805 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T19:56:49.533 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:49.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:49.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:49.637 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:49.651 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:49 smithi092 bash[58855]: audit 2023-09-23T19:56:48.830290+0000 mon.a (mon.0) 1804 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]: dispatch 2023-09-23T19:56:49.651 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:49 smithi092 bash[58855]: audit 2023-09-23T19:56:48.835445+0000 mon.b (mon.1) 59 : audit 1 from='client.? 172.21.15.16:0/3482255068' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]: dispatch 2023-09-23T19:56:49.651 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:49 smithi092 bash[58855]: audit 2023-09-23T19:56:48.969056+0000 mon.c (mon.2) 228 : audit 1 from='client.? 172.21.15.16:0/2742436240' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T19:56:49.651 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:49 smithi092 bash[58855]: audit 2023-09-23T19:56:49.283842+0000 mon.a (mon.0) 1805 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T19:56:49.672 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:49.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:49.676 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:49.776 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:49.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:49 smithi016 bash[77995]: audit 2023-09-23T19:56:48.830290+0000 mon.a (mon.0) 1804 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]: dispatch 2023-09-23T19:56:49.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:49 smithi016 bash[77995]: audit 2023-09-23T19:56:48.835445+0000 mon.b (mon.1) 59 : audit 1 from='client.? 172.21.15.16:0/3482255068' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]: dispatch 2023-09-23T19:56:49.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:49 smithi016 bash[77995]: audit 2023-09-23T19:56:48.969056+0000 mon.c (mon.2) 228 : audit 1 from='client.? 172.21.15.16:0/2742436240' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T19:56:49.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:49 smithi016 bash[77995]: audit 2023-09-23T19:56:49.283842+0000 mon.a (mon.0) 1805 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T19:56:49.812 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:49.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:49.815 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:49.916 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:49.937 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:49.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:49.940 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:50.041 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:50.067 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:50.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:50.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:50.171 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:50.194 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:50.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:50.196 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:50.297 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:50.321 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:50.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:50.324 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:50.424 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:50.453 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:50.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:50.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:50.545 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:50 smithi016 bash[75004]: cluster 2023-09-23T19:56:48.550151+0000 mgr.y (mgr.25325) 1485 : cluster 0 pgmap v1505: 73 pgs: 73 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:50.545 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:50 smithi016 bash[75004]: audit 2023-09-23T19:56:49.338713+0000 mon.a (mon.0) 1806 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]': finished 2023-09-23T19:56:50.545 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:50 smithi016 bash[75004]: audit 2023-09-23T19:56:49.338965+0000 mon.a (mon.0) 1807 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T19:56:50.545 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:50 smithi016 bash[75004]: cluster 2023-09-23T19:56:49.339335+0000 mon.a (mon.0) 1808 : cluster 0 osdmap e256: 8 total, 8 up, 8 in 2023-09-23T19:56:50.545 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:50 smithi016 bash[75004]: cluster 2023-09-23T19:56:49.342068+0000 osd.5 (osd.5) 184 : cluster 3 failed to encode map e256 with expected crc 2023-09-23T19:56:50.545 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:50 smithi016 bash[75004]: cluster 2023-09-23T19:56:49.345247+0000 osd.5 (osd.5) 185 : cluster 3 failed to encode map e256 with expected crc 2023-09-23T19:56:50.545 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:50 smithi016 bash[75004]: cluster 2023-09-23T19:56:49.349023+0000 osd.4 (osd.4) 199 : cluster 3 failed to encode map e256 with expected crc 2023-09-23T19:56:50.545 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:50 smithi016 bash[75004]: cluster 2023-09-23T19:56:49.349164+0000 osd.7 (osd.7) 169 : cluster 3 failed to encode map e256 with expected crc 2023-09-23T19:56:50.546 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:50 smithi016 bash[75004]: cluster 2023-09-23T19:56:50.337688+0000 mon.a (mon.0) 1809 : cluster 0 osdmap e257: 8 total, 8 up, 8 in 2023-09-23T19:56:50.546 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:50 smithi016 bash[77995]: cluster 2023-09-23T19:56:48.550151+0000 mgr.y (mgr.25325) 1485 : cluster 0 pgmap v1505: 73 pgs: 73 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:50.546 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:50 smithi016 bash[77995]: audit 2023-09-23T19:56:49.338713+0000 mon.a (mon.0) 1806 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]': finished 2023-09-23T19:56:50.546 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:50 smithi016 bash[77995]: audit 2023-09-23T19:56:49.338965+0000 mon.a (mon.0) 1807 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T19:56:50.546 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:50 smithi016 bash[77995]: cluster 2023-09-23T19:56:49.339335+0000 mon.a (mon.0) 1808 : cluster 0 osdmap e256: 8 total, 8 up, 8 in 2023-09-23T19:56:50.547 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:50 smithi016 bash[77995]: cluster 2023-09-23T19:56:49.342068+0000 osd.5 (osd.5) 184 : cluster 3 failed to encode map e256 with expected crc 2023-09-23T19:56:50.547 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:50 smithi016 bash[77995]: cluster 2023-09-23T19:56:49.345247+0000 osd.5 (osd.5) 185 : cluster 3 failed to encode map e256 with expected crc 2023-09-23T19:56:50.547 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:50 smithi016 bash[77995]: cluster 2023-09-23T19:56:49.349023+0000 osd.4 (osd.4) 199 : cluster 3 failed to encode map e256 with expected crc 2023-09-23T19:56:50.547 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:50 smithi016 bash[77995]: cluster 2023-09-23T19:56:49.349164+0000 osd.7 (osd.7) 169 : cluster 3 failed to encode map e256 with expected crc 2023-09-23T19:56:50.547 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:50 smithi016 bash[77995]: cluster 2023-09-23T19:56:50.337688+0000 mon.a (mon.0) 1809 : cluster 0 osdmap e257: 8 total, 8 up, 8 in 2023-09-23T19:56:50.556 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:50.581 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:50.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:50.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:50.685 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:50.698 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:50 smithi092 bash[58855]: cluster 2023-09-23T19:56:48.550151+0000 mgr.y (mgr.25325) 1485 : cluster 0 pgmap v1505: 73 pgs: 73 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:50.699 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:50 smithi092 bash[58855]: audit 2023-09-23T19:56:49.338713+0000 mon.a (mon.0) 1806 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]': finished 2023-09-23T19:56:50.699 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:50 smithi092 bash[58855]: audit 2023-09-23T19:56:49.338965+0000 mon.a (mon.0) 1807 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T19:56:50.699 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:50 smithi092 bash[58855]: cluster 2023-09-23T19:56:49.339335+0000 mon.a (mon.0) 1808 : cluster 0 osdmap e256: 8 total, 8 up, 8 in 2023-09-23T19:56:50.699 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:50 smithi092 bash[58855]: cluster 2023-09-23T19:56:49.342068+0000 osd.5 (osd.5) 184 : cluster 3 failed to encode map e256 with expected crc 2023-09-23T19:56:50.699 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:50 smithi092 bash[58855]: cluster 2023-09-23T19:56:49.345247+0000 osd.5 (osd.5) 185 : cluster 3 failed to encode map e256 with expected crc 2023-09-23T19:56:50.699 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:50 smithi092 bash[58855]: cluster 2023-09-23T19:56:49.349023+0000 osd.4 (osd.4) 199 : cluster 3 failed to encode map e256 with expected crc 2023-09-23T19:56:50.699 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:50 smithi092 bash[58855]: cluster 2023-09-23T19:56:49.349164+0000 osd.7 (osd.7) 169 : cluster 3 failed to encode map e256 with expected crc 2023-09-23T19:56:50.699 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:50 smithi092 bash[58855]: cluster 2023-09-23T19:56:50.337688+0000 mon.a (mon.0) 1809 : cluster 0 osdmap e257: 8 total, 8 up, 8 in 2023-09-23T19:56:50.720 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:50.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:50.724 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:50.825 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:50.857 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:50.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:50.859 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:50.960 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:50.996 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:50.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:50.999 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:51.100 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:51.126 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:51.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:51.129 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:51.229 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:51.260 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:51.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:51.263 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:51.363 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:51.373 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:51 smithi016 bash[77995]: cluster 2023-09-23T19:56:50.344142+0000 osd.6 (osd.6) 192 : cluster 3 failed to encode map e257 with expected crc 2023-09-23T19:56:51.374 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:51 smithi016 bash[77995]: cluster 2023-09-23T19:56:50.344182+0000 osd.4 (osd.4) 200 : cluster 3 failed to encode map e257 with expected crc 2023-09-23T19:56:51.374 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:51 smithi016 bash[77995]: cluster 2023-09-23T19:56:50.344209+0000 osd.5 (osd.5) 186 : cluster 3 failed to encode map e257 with expected crc 2023-09-23T19:56:51.374 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:51 smithi016 bash[77995]: cluster 2023-09-23T19:56:50.344539+0000 osd.7 (osd.7) 170 : cluster 3 failed to encode map e257 with expected crc 2023-09-23T19:56:51.374 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:51 smithi016 bash[75004]: cluster 2023-09-23T19:56:50.344142+0000 osd.6 (osd.6) 192 : cluster 3 failed to encode map e257 with expected crc 2023-09-23T19:56:51.374 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:51 smithi016 bash[75004]: cluster 2023-09-23T19:56:50.344182+0000 osd.4 (osd.4) 200 : cluster 3 failed to encode map e257 with expected crc 2023-09-23T19:56:51.374 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:51 smithi016 bash[75004]: cluster 2023-09-23T19:56:50.344209+0000 osd.5 (osd.5) 186 : cluster 3 failed to encode map e257 with expected crc 2023-09-23T19:56:51.374 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:51 smithi016 bash[75004]: cluster 2023-09-23T19:56:50.344539+0000 osd.7 (osd.7) 170 : cluster 3 failed to encode map e257 with expected crc 2023-09-23T19:56:51.395 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:51.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:51.398 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:51.499 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:51.511 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:51 smithi092 bash[58855]: cluster 2023-09-23T19:56:50.344142+0000 osd.6 (osd.6) 192 : cluster 3 failed to encode map e257 with expected crc 2023-09-23T19:56:51.512 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:51 smithi092 bash[58855]: cluster 2023-09-23T19:56:50.344182+0000 osd.4 (osd.4) 200 : cluster 3 failed to encode map e257 with expected crc 2023-09-23T19:56:51.512 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:51 smithi092 bash[58855]: cluster 2023-09-23T19:56:50.344209+0000 osd.5 (osd.5) 186 : cluster 3 failed to encode map e257 with expected crc 2023-09-23T19:56:51.512 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:51 smithi092 bash[58855]: cluster 2023-09-23T19:56:50.344539+0000 osd.7 (osd.7) 170 : cluster 3 failed to encode map e257 with expected crc 2023-09-23T19:56:51.529 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:51.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:51.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:51.633 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:51.663 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:51.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:51.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:51.766 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:51.798 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:51.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:51.801 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:51.902 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:51.929 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:51.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:51.933 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:52.033 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:52.063 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:52.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:52.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:52.166 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:52.199 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:52.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:52.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:52.303 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:52.337 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:52.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:52.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:52.410 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-23T19:56:52.440 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:52.443 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_9 rados --yes-i-really-mean-it || true 2023-09-23T19:56:52.514 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:52.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:52.516 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:52.617 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:52.629 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:52 smithi016 bash[77995]: cluster 2023-09-23T19:56:50.550718+0000 mgr.y (mgr.25325) 1486 : cluster 0 pgmap v1508: 89 pgs: 9 creating+peering, 5 unknown, 75 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:52.630 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:52 smithi016 bash[77995]: cluster 2023-09-23T19:56:51.356415+0000 mon.a (mon.0) 1810 : cluster 0 osdmap e258: 8 total, 8 up, 8 in 2023-09-23T19:56:52.630 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:52 smithi016 bash[77995]: cluster 2023-09-23T19:56:51.357230+0000 osd.7 (osd.7) 171 : cluster 3 failed to encode map e258 with expected crc 2023-09-23T19:56:52.630 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:52 smithi016 bash[77995]: cluster 2023-09-23T19:56:51.470549+0000 osd.6 (osd.6) 193 : cluster 3 failed to encode map e258 with expected crc 2023-09-23T19:56:52.630 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:52 smithi016 bash[77995]: cluster 2023-09-23T19:56:51.473540+0000 osd.6 (osd.6) 194 : cluster 3 failed to encode map e258 with expected crc 2023-09-23T19:56:52.630 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:52 smithi016 bash[75004]: cluster 2023-09-23T19:56:50.550718+0000 mgr.y (mgr.25325) 1486 : cluster 0 pgmap v1508: 89 pgs: 9 creating+peering, 5 unknown, 75 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:52.630 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:52 smithi016 bash[75004]: cluster 2023-09-23T19:56:51.356415+0000 mon.a (mon.0) 1810 : cluster 0 osdmap e258: 8 total, 8 up, 8 in 2023-09-23T19:56:52.630 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:52 smithi016 bash[75004]: cluster 2023-09-23T19:56:51.357230+0000 osd.7 (osd.7) 171 : cluster 3 failed to encode map e258 with expected crc 2023-09-23T19:56:52.631 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:52 smithi016 bash[75004]: cluster 2023-09-23T19:56:51.470549+0000 osd.6 (osd.6) 193 : cluster 3 failed to encode map e258 with expected crc 2023-09-23T19:56:52.631 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:52 smithi016 bash[75004]: cluster 2023-09-23T19:56:51.473540+0000 osd.6 (osd.6) 194 : cluster 3 failed to encode map e258 with expected crc 2023-09-23T19:56:52.648 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:52.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:52.652 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:52.753 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:52.779 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:52.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:52.782 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:52.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:52 smithi092 bash[58855]: cluster 2023-09-23T19:56:50.550718+0000 mgr.y (mgr.25325) 1486 : cluster 0 pgmap v1508: 89 pgs: 9 creating+peering, 5 unknown, 75 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:52.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:52 smithi092 bash[58855]: cluster 2023-09-23T19:56:51.356415+0000 mon.a (mon.0) 1810 : cluster 0 osdmap e258: 8 total, 8 up, 8 in 2023-09-23T19:56:52.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:52 smithi092 bash[58855]: cluster 2023-09-23T19:56:51.357230+0000 osd.7 (osd.7) 171 : cluster 3 failed to encode map e258 with expected crc 2023-09-23T19:56:52.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:52 smithi092 bash[58855]: cluster 2023-09-23T19:56:51.470549+0000 osd.6 (osd.6) 193 : cluster 3 failed to encode map e258 with expected crc 2023-09-23T19:56:52.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:52 smithi092 bash[58855]: cluster 2023-09-23T19:56:51.473540+0000 osd.6 (osd.6) 194 : cluster 3 failed to encode map e258 with expected crc 2023-09-23T19:56:52.883 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:52.911 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:52.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:52.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:53.014 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:53.035 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:53.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:53.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:53.140 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:53.174 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:53.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:53.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:53.232 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:56:53.254 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:56:53.278 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:53.282 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:56:53.305 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:56:53.313 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:53.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:53.316 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:53.329 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:56:53.353 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:56:53.377 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:56:53.400 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:56:53.402 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:56:53.402 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:56:53.402 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:56:53.402 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:56:53.402 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:56:53.402 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:56:53.403 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:56:53.403 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:56:53.417 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:53.430 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:53 smithi092 bash[58855]: cluster 2023-09-23T19:56:51.885208+0000 osd.4 (osd.4) 201 : cluster 3 failed to encode map e258 with expected crc 2023-09-23T19:56:53.430 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:53 smithi092 bash[58855]: cluster 2023-09-23T19:56:51.885278+0000 osd.5 (osd.5) 187 : cluster 3 failed to encode map e258 with expected crc 2023-09-23T19:56:53.430 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:53 smithi092 bash[58855]: cluster 2023-09-23T19:56:52.551470+0000 mgr.y (mgr.25325) 1487 : cluster 0 pgmap v1510: 89 pgs: 9 creating+peering, 2 unknown, 78 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:53.430 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:53 smithi092 bash[58855]: audit 2023-09-23T19:56:53.207127+0000 mon.c (mon.2) 229 : audit 0 from='client.? 172.21.15.16:0/1052618056' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:56:53.450 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:53.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:53.453 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:53.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:53 smithi016 bash[75004]: cluster 2023-09-23T19:56:51.885208+0000 osd.4 (osd.4) 201 : cluster 3 failed to encode map e258 with expected crc 2023-09-23T19:56:53.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:53 smithi016 bash[75004]: cluster 2023-09-23T19:56:51.885278+0000 osd.5 (osd.5) 187 : cluster 3 failed to encode map e258 with expected crc 2023-09-23T19:56:53.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:53 smithi016 bash[75004]: cluster 2023-09-23T19:56:52.551470+0000 mgr.y (mgr.25325) 1487 : cluster 0 pgmap v1510: 89 pgs: 9 creating+peering, 2 unknown, 78 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:53.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:53 smithi016 bash[75004]: audit 2023-09-23T19:56:53.207127+0000 mon.c (mon.2) 229 : audit 0 from='client.? 172.21.15.16:0/1052618056' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:56:53.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:53 smithi016 bash[77995]: cluster 2023-09-23T19:56:51.885208+0000 osd.4 (osd.4) 201 : cluster 3 failed to encode map e258 with expected crc 2023-09-23T19:56:53.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:53 smithi016 bash[77995]: cluster 2023-09-23T19:56:51.885278+0000 osd.5 (osd.5) 187 : cluster 3 failed to encode map e258 with expected crc 2023-09-23T19:56:53.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:53 smithi016 bash[77995]: cluster 2023-09-23T19:56:52.551470+0000 mgr.y (mgr.25325) 1487 : cluster 0 pgmap v1510: 89 pgs: 9 creating+peering, 2 unknown, 78 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:53.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:53 smithi016 bash[77995]: audit 2023-09-23T19:56:53.207127+0000 mon.c (mon.2) 229 : audit 0 from='client.? 172.21.15.16:0/1052618056' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:56:53.553 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:53.585 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:53.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:53.589 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:53.690 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:53.726 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:53.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:53.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:53.829 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:53.862 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:53.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:53.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:53.966 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:53.999 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:54.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:54.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:54.103 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:54.130 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:54.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:54.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:54.233 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:54.262 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:54.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:54.265 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:54.366 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:54.380 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is unset 2023-09-23T19:56:54.392 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:54.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:54.395 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:54.496 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:54.518 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:54.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:54.520 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:54.557 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:54 smithi016 bash[75004]: audit 2023-09-23T19:56:53.379861+0000 mon.a (mon.0) 1811 : audit 1 from='client.? 172.21.15.16:0/1548764833' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:56:54.557 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:54 smithi016 bash[77995]: audit 2023-09-23T19:56:53.379861+0000 mon.a (mon.0) 1811 : audit 1 from='client.? 172.21.15.16:0/1548764833' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:56:54.621 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:54.645 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:54.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:54.648 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:54.748 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:54.760 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:54 smithi092 bash[58855]: audit 2023-09-23T19:56:53.379861+0000 mon.a (mon.0) 1811 : audit 1 from='client.? 172.21.15.16:0/1548764833' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:56:54.779 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:54.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:54.783 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:54.884 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:54.916 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:54.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:54.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:55.020 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:55.049 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:55.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:55.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:55.153 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:55.178 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:55.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:55.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:55.282 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:55.318 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:55.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:55.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:55.422 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:55.448 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:55.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:55.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:55.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:55 smithi016 bash[77995]: cluster 2023-09-23T19:56:54.361038+0000 mon.a (mon.0) 1812 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:56:55.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:55 smithi016 bash[77995]: cluster 2023-09-23T19:56:54.361137+0000 mon.a (mon.0) 1813 : cluster 1 Cluster is now healthy 2023-09-23T19:56:55.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:55 smithi016 bash[77995]: audit 2023-09-23T19:56:54.372561+0000 mon.a (mon.0) 1814 : audit 1 from='client.? 172.21.15.16:0/1548764833' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:56:55.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:55 smithi016 bash[77995]: cluster 2023-09-23T19:56:54.372645+0000 mon.a (mon.0) 1815 : cluster 0 osdmap e259: 8 total, 8 up, 8 in 2023-09-23T19:56:55.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:55 smithi016 bash[77995]: cluster 2023-09-23T19:56:54.375742+0000 osd.5 (osd.5) 188 : cluster 3 failed to encode map e259 with expected crc 2023-09-23T19:56:55.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:55 smithi016 bash[77995]: cluster 2023-09-23T19:56:54.551973+0000 mgr.y (mgr.25325) 1488 : cluster 0 pgmap v1512: 89 pgs: 9 creating+peering, 80 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:55.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:55 smithi016 bash[75004]: cluster 2023-09-23T19:56:54.361038+0000 mon.a (mon.0) 1812 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:56:55.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:55 smithi016 bash[75004]: cluster 2023-09-23T19:56:54.361137+0000 mon.a (mon.0) 1813 : cluster 1 Cluster is now healthy 2023-09-23T19:56:55.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:55 smithi016 bash[75004]: audit 2023-09-23T19:56:54.372561+0000 mon.a (mon.0) 1814 : audit 1 from='client.? 172.21.15.16:0/1548764833' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:56:55.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:55 smithi016 bash[75004]: cluster 2023-09-23T19:56:54.372645+0000 mon.a (mon.0) 1815 : cluster 0 osdmap e259: 8 total, 8 up, 8 in 2023-09-23T19:56:55.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:55 smithi016 bash[75004]: cluster 2023-09-23T19:56:54.375742+0000 osd.5 (osd.5) 188 : cluster 3 failed to encode map e259 with expected crc 2023-09-23T19:56:55.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:55 smithi016 bash[75004]: cluster 2023-09-23T19:56:54.551973+0000 mgr.y (mgr.25325) 1488 : cluster 0 pgmap v1512: 89 pgs: 9 creating+peering, 80 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:55.552 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:55.574 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:55.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:55.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:55.677 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:55.690 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:55 smithi092 bash[58855]: cluster 2023-09-23T19:56:54.361038+0000 mon.a (mon.0) 1812 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:56:55.691 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:55 smithi092 bash[58855]: cluster 2023-09-23T19:56:54.361137+0000 mon.a (mon.0) 1813 : cluster 1 Cluster is now healthy 2023-09-23T19:56:55.691 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:55 smithi092 bash[58855]: audit 2023-09-23T19:56:54.372561+0000 mon.a (mon.0) 1814 : audit 1 from='client.? 172.21.15.16:0/1548764833' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:56:55.691 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:55 smithi092 bash[58855]: cluster 2023-09-23T19:56:54.372645+0000 mon.a (mon.0) 1815 : cluster 0 osdmap e259: 8 total, 8 up, 8 in 2023-09-23T19:56:55.691 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:55 smithi092 bash[58855]: cluster 2023-09-23T19:56:54.375742+0000 osd.5 (osd.5) 188 : cluster 3 failed to encode map e259 with expected crc 2023-09-23T19:56:55.691 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:55 smithi092 bash[58855]: cluster 2023-09-23T19:56:54.551973+0000 mgr.y (mgr.25325) 1488 : cluster 0 pgmap v1512: 89 pgs: 9 creating+peering, 80 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:55.709 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:55.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:55.712 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:55.813 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:55.842 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:55.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:55.846 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:55.947 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:55.960 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:56:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:56:55] "GET /metrics HTTP/1.1" 200 33890 "" "Prometheus/2.33.4" 2023-09-23T19:56:55.977 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:55.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:55.980 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:56.080 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:56.112 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:56.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:56.114 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:56.215 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:56.240 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:56.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:56.244 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:56.345 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:56.377 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:56.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:56.380 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:56.481 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:56.510 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:56.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:56.513 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:56.614 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:56.646 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:56.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:56.649 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:56.749 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:56.779 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:56.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:56.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:56.883 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:56.915 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:56.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:56.918 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:57.019 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:57.051 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:57.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:57.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:57.155 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:57.188 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:57.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:57.190 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:57.291 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:57.322 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:57.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:57.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:57.426 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:57.453 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:57.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:57.456 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:57.557 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:57.586 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:57.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:57.588 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:57.689 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:57.719 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:57.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:57.722 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:57.823 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:57.824 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-09-23T19:56:57.824 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:56:57.849 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:57.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:57.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:57.870 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:57 smithi016 bash[77995]: cluster 2023-09-23T19:56:56.552844+0000 mgr.y (mgr.25325) 1489 : cluster 0 pgmap v1513: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:57.870 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:57 smithi016 bash[75004]: cluster 2023-09-23T19:56:56.552844+0000 mgr.y (mgr.25325) 1489 : cluster 0 pgmap v1513: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:57.953 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:57.964 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:57 smithi092 bash[58855]: cluster 2023-09-23T19:56:56.552844+0000 mgr.y (mgr.25325) 1489 : cluster 0 pgmap v1513: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:57.986 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:57.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:57.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:58.090 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:58.116 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:58.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:58.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:58.219 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:58.249 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:58.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:58.251 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:58.352 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:58.380 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:58.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:58.383 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:58.484 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:58.514 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:58.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:58.516 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:58.617 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:58.649 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:56:58.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:58.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:56:58.667 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:56:58.755 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:58.779 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:58.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:58.782 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:58.882 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:58.911 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:58.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:58.914 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:58.919 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T19:56:59.015 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:59.046 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:59.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:59.050 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:59.150 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:59.174 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:59.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:59.176 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:59.277 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:59.311 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:59.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:59.314 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:59.340 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T19:56:59.415 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:59.447 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:59.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:59.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:59.551 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:59.577 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:59.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:59.580 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:59.681 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:59.695 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:59 smithi016 bash[75004]: cluster 2023-09-23T19:56:58.553364+0000 mgr.y (mgr.25325) 1490 : cluster 0 pgmap v1514: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:59.695 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:59 smithi016 bash[75004]: audit 2023-09-23T19:56:58.689909+0000 mon.a (mon.0) 1816 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:56:59.695 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:56:59 smithi016 bash[75004]: audit 2023-09-23T19:56:58.691762+0000 mon.a (mon.0) 1817 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:56:59.697 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:59 smithi016 bash[77995]: cluster 2023-09-23T19:56:58.553364+0000 mgr.y (mgr.25325) 1490 : cluster 0 pgmap v1514: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:56:59.697 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:59 smithi016 bash[77995]: audit 2023-09-23T19:56:58.689909+0000 mon.a (mon.0) 1816 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:56:59.697 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:56:59 smithi016 bash[77995]: audit 2023-09-23T19:56:58.691762+0000 mon.a (mon.0) 1817 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:56:59.709 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:59.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:59.711 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:59.812 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:59.848 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:59.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:59.851 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:59.952 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:59.985 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:59.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:56:59.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:00.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:59 smithi092 bash[58855]: cluster 2023-09-23T19:56:58.553364+0000 mgr.y (mgr.25325) 1490 : cluster 0 pgmap v1514: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:57:00.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:59 smithi092 bash[58855]: audit 2023-09-23T19:56:58.689909+0000 mon.a (mon.0) 1816 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:57:00.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:56:59 smithi092 bash[58855]: audit 2023-09-23T19:56:58.691762+0000 mon.a (mon.0) 1817 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:57:00.088 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:00.120 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:00.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:00.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:00.207 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:57:00.207 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:57:00.223 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:00.257 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:00.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:00.261 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:00.361 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:00.389 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:00.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:00.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:00.494 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:00.524 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:00.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:00.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:00.628 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:00.654 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:00.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:00.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:00.758 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:00.788 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:00.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:00.792 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:00.892 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:00.921 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:00.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:00.924 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:57:01.025 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:01.056 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:01.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:01.059 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:01.159 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:01.196 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:01.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:01.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:01.300 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:01.333 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:01.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:01.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:01.436 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:01.465 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:01.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:01.468 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:01.569 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:01.600 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:01.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:01.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:01.704 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:01.734 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:01.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:01.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:57:01.837 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:01.864 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:01.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:01.867 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:01.967 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:01.997 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:01.998 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:01 smithi016 bash[77995]: cluster 2023-09-23T19:57:00.554507+0000 mgr.y (mgr.25325) 1491 : cluster 0 pgmap v1515: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:57:01.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:01 smithi016 bash[75004]: cluster 2023-09-23T19:57:00.554507+0000 mgr.y (mgr.25325) 1491 : cluster 0 pgmap v1515: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:57:01.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:02.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:02.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:01 smithi092 bash[58855]: cluster 2023-09-23T19:57:00.554507+0000 mgr.y (mgr.25325) 1491 : cluster 0 pgmap v1515: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:57:02.100 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:02.137 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:02.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:02.140 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:02.240 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:02.271 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:02.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:02.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:02.374 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:02.405 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:02.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:02.408 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:02.509 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:02.536 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:02.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:02.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:02.641 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:02.670 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:02.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:02.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:02.774 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:02.799 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:02.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:02.802 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:02.903 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:02.931 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:02.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:02.935 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:03.035 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:03.065 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:03.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:03.068 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:03.169 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:03.200 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:03.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:03.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:03.304 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:03.329 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:03.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:03.332 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:03.433 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:03.465 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:03.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:03.467 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:03.568 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:03.601 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:03.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:03.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:57:03.705 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:03.735 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:03.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:03.739 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:03.890 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:03.922 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:03.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:03.925 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:04.026 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:04.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:03 smithi016 bash[75004]: cluster 2023-09-23T19:57:02.555001+0000 mgr.y (mgr.25325) 1492 : cluster 0 pgmap v1516: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:57:04.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:03 smithi016 bash[77995]: cluster 2023-09-23T19:57:02.555001+0000 mgr.y (mgr.25325) 1492 : cluster 0 pgmap v1516: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:57:04.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:03 smithi092 bash[58855]: cluster 2023-09-23T19:57:02.555001+0000 mgr.y (mgr.25325) 1492 : cluster 0 pgmap v1516: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:57:04.059 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:04.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:04.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:57:04.163 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:04.198 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:04.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:04.202 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:57:04.302 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:04.337 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:04.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:04.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:57:04.441 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:04.472 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:04.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:04.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:04.575 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:04.607 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:04.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:04.611 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:04.686 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:57:04.686 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:57:04.686 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:57:04.712 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:04.736 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:04.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:04.738 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:04.839 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:04.866 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:04.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:04.869 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:04.970 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:05.000 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:05.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:05.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:05.103 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:05.132 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:05.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:05.135 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:05.236 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:05.261 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:05.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:05.264 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:05.365 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:05.393 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:05.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:05.396 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:05.496 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:05.525 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:05.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:05.528 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:05.628 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:05.658 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:05.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:05.662 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:05.763 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:05.775 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:05 smithi016 bash[75004]: audit 2023-09-23T19:57:04.319312+0000 mon.a (mon.0) 1818 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:57:05.775 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:57:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:57:05] "GET /metrics HTTP/1.1" 200 36237 "" "Prometheus/2.33.4" 2023-09-23T19:57:05.775 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:05 smithi016 bash[77995]: audit 2023-09-23T19:57:04.319312+0000 mon.a (mon.0) 1818 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:57:05.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:05 smithi092 bash[58855]: audit 2023-09-23T19:57:04.319312+0000 mon.a (mon.0) 1818 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:57:05.791 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:05.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:05.795 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:05.896 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:05.930 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:05.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:05.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:06.034 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:06.065 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:06.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:06.068 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:57:06.169 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:06.202 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:06.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:06.205 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:06.306 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:06.338 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:06.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:06.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:06.441 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:06.476 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:06.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:06.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:06.579 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:06.592 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:06 smithi016 bash[75004]: cluster 2023-09-23T19:57:04.555634+0000 mgr.y (mgr.25325) 1493 : cluster 0 pgmap v1517: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:57:06.592 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:06 smithi016 bash[77995]: cluster 2023-09-23T19:57:04.555634+0000 mgr.y (mgr.25325) 1493 : cluster 0 pgmap v1517: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:57:06.610 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:06.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:06.613 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:06.714 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:06.742 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:06.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:06.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:06.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:06 smithi092 bash[58855]: cluster 2023-09-23T19:57:04.555634+0000 mgr.y (mgr.25325) 1493 : cluster 0 pgmap v1517: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:57:06.846 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:06.877 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:06.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:06.880 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:06.980 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:07.012 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:07.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:07.015 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:07.116 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:07.144 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:07.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:07.148 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:07.249 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:07.274 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:07.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:07.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:07.377 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:07.408 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:07.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:07.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:07.512 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:07.546 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:07.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:07.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:07.650 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:07.678 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:07.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:07.682 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:07.782 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:07.816 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:07.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:07.820 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:07.921 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:07.949 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:07.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:07.952 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:08.052 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:08.086 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:08.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:08.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:08.191 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:08.218 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:08.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:08.221 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:08.321 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:08.349 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:08.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:08.352 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:08.453 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:08.480 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:08.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:08.482 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:08.583 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:08.595 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:08 smithi092 bash[58855]: cluster 2023-09-23T19:57:06.556778+0000 mgr.y (mgr.25325) 1494 : cluster 0 pgmap v1518: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:57:08.615 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:08.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:08.617 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:08.718 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:08.752 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:08.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:08.756 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:08.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:08 smithi016 bash[75004]: cluster 2023-09-23T19:57:06.556778+0000 mgr.y (mgr.25325) 1494 : cluster 0 pgmap v1518: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:57:08.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:08 smithi016 bash[77995]: cluster 2023-09-23T19:57:06.556778+0000 mgr.y (mgr.25325) 1494 : cluster 0 pgmap v1518: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:57:08.857 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:08.883 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:08.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:08.886 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:08.986 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:09.016 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:09.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:09.020 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:09.120 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:09.150 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:09.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:09.152 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:09.253 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:09.287 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:09.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:09.290 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:09.391 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:09.426 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:09.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:09.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:09.530 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:09.561 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:09.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:09.564 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:57:09.665 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:09.694 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:09.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:09.696 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:09.797 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:09.826 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:09.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:09.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:09.930 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:09.963 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:09.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:09.967 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:57:10.068 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:10.089 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:10.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:10.093 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:10.194 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:10.217 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:10.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:10.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:10.321 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:10.345 INFO:teuthology.orchestra.run.smithi016.stderr:enabled application 'rados' on pool 'unique_pool_9' 2023-09-23T19:57:10.345 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:10.348 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.7 to 1 (10000) 2023-09-23T19:57:10.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:10.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:10.449 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:10.480 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:10.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:10.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:10.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:10 smithi016 bash[75004]: cluster 2023-09-23T19:57:08.557686+0000 mgr.y (mgr.25325) 1495 : cluster 0 pgmap v1519: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:57:10.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:10 smithi016 bash[75004]: audit 2023-09-23T19:57:09.727004+0000 mon.a (mon.0) 1819 : audit 1 from='client.? 172.21.15.16:0/2248286904' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:57:10.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:10 smithi016 bash[75004]: audit 2023-09-23T19:57:09.735636+0000 mon.a (mon.0) 1820 : audit 1 from='client.? 172.21.15.16:0/1271944277' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T19:57:10.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:10 smithi016 bash[77995]: cluster 2023-09-23T19:57:08.557686+0000 mgr.y (mgr.25325) 1495 : cluster 0 pgmap v1519: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:57:10.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:10 smithi016 bash[77995]: audit 2023-09-23T19:57:09.727004+0000 mon.a (mon.0) 1819 : audit 1 from='client.? 172.21.15.16:0/2248286904' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:57:10.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:10 smithi016 bash[77995]: audit 2023-09-23T19:57:09.735636+0000 mon.a (mon.0) 1820 : audit 1 from='client.? 172.21.15.16:0/1271944277' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T19:57:10.584 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:10.613 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:10.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:10.616 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:10.717 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:10.748 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:10.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:10.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:10.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:10 smithi092 bash[58855]: cluster 2023-09-23T19:57:08.557686+0000 mgr.y (mgr.25325) 1495 : cluster 0 pgmap v1519: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:57:10.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:10 smithi092 bash[58855]: audit 2023-09-23T19:57:09.727004+0000 mon.a (mon.0) 1819 : audit 1 from='client.? 172.21.15.16:0/2248286904' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:57:10.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:10 smithi092 bash[58855]: audit 2023-09-23T19:57:09.735636+0000 mon.a (mon.0) 1820 : audit 1 from='client.? 172.21.15.16:0/1271944277' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T19:57:10.851 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:10.875 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:10.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:10.878 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:10.979 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:11.004 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:11.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:11.007 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:11.108 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:11.142 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:11.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:11.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:11.246 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:11.269 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:11.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:11.272 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:11.373 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:11.400 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:11.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:11.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:11.504 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:11.531 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:11.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:11.533 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:11.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:11 smithi092 bash[58855]: audit 2023-09-23T19:57:10.336039+0000 mon.a (mon.0) 1821 : audit 1 from='client.? 172.21.15.16:0/2248286904' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T19:57:11.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:11 smithi092 bash[58855]: audit 2023-09-23T19:57:10.336141+0000 mon.a (mon.0) 1822 : audit 1 from='client.? 172.21.15.16:0/1271944277' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T19:57:11.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:11 smithi092 bash[58855]: cluster 2023-09-23T19:57:10.336273+0000 mon.a (mon.0) 1823 : cluster 0 osdmap e260: 8 total, 8 up, 8 in 2023-09-23T19:57:11.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:11 smithi092 bash[58855]: cluster 2023-09-23T19:57:10.339155+0000 osd.4 (osd.4) 202 : cluster 3 failed to encode map e260 with expected crc 2023-09-23T19:57:11.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:11 smithi092 bash[58855]: cluster 2023-09-23T19:57:10.424608+0000 osd.7 (osd.7) 172 : cluster 3 failed to encode map e260 with expected crc 2023-09-23T19:57:11.634 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:11.663 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:11.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:11.666 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:11.683 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:11 smithi016 bash[77995]: audit 2023-09-23T19:57:10.336039+0000 mon.a (mon.0) 1821 : audit 1 from='client.? 172.21.15.16:0/2248286904' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T19:57:11.684 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:11 smithi016 bash[77995]: audit 2023-09-23T19:57:10.336141+0000 mon.a (mon.0) 1822 : audit 1 from='client.? 172.21.15.16:0/1271944277' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T19:57:11.684 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:11 smithi016 bash[77995]: cluster 2023-09-23T19:57:10.336273+0000 mon.a (mon.0) 1823 : cluster 0 osdmap e260: 8 total, 8 up, 8 in 2023-09-23T19:57:11.684 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:11 smithi016 bash[77995]: cluster 2023-09-23T19:57:10.339155+0000 osd.4 (osd.4) 202 : cluster 3 failed to encode map e260 with expected crc 2023-09-23T19:57:11.684 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:11 smithi016 bash[77995]: cluster 2023-09-23T19:57:10.424608+0000 osd.7 (osd.7) 172 : cluster 3 failed to encode map e260 with expected crc 2023-09-23T19:57:11.684 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:11 smithi016 bash[75004]: audit 2023-09-23T19:57:10.336039+0000 mon.a (mon.0) 1821 : audit 1 from='client.? 172.21.15.16:0/2248286904' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T19:57:11.684 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:11 smithi016 bash[75004]: audit 2023-09-23T19:57:10.336141+0000 mon.a (mon.0) 1822 : audit 1 from='client.? 172.21.15.16:0/1271944277' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T19:57:11.685 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:11 smithi016 bash[75004]: cluster 2023-09-23T19:57:10.336273+0000 mon.a (mon.0) 1823 : cluster 0 osdmap e260: 8 total, 8 up, 8 in 2023-09-23T19:57:11.685 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:11 smithi016 bash[75004]: cluster 2023-09-23T19:57:10.339155+0000 osd.4 (osd.4) 202 : cluster 3 failed to encode map e260 with expected crc 2023-09-23T19:57:11.685 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:11 smithi016 bash[75004]: cluster 2023-09-23T19:57:10.424608+0000 osd.7 (osd.7) 172 : cluster 3 failed to encode map e260 with expected crc 2023-09-23T19:57:11.766 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:11.794 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:11.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:11.797 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:11.898 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:11.924 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:11.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:11.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:12.028 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:12.059 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:12.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:12.063 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:12.164 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:12.189 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:12.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:12.191 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:12.292 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:12.321 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:12.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:12.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:12.425 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:12.460 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:12.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:12.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:12.563 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:12.593 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:12.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:12.596 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:57:12.688 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:12 smithi016 bash[77995]: cluster 2023-09-23T19:57:10.558225+0000 mgr.y (mgr.25325) 1496 : cluster 0 pgmap v1521: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:57:12.688 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:12 smithi016 bash[75004]: cluster 2023-09-23T19:57:10.558225+0000 mgr.y (mgr.25325) 1496 : cluster 0 pgmap v1521: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:57:12.697 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:12.708 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:12 smithi092 bash[58855]: cluster 2023-09-23T19:57:10.558225+0000 mgr.y (mgr.25325) 1496 : cluster 0 pgmap v1521: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:57:12.726 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:12.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:12.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:12.829 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:12.856 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:12.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:12.860 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:12.961 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:12.990 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:12.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:12.994 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:13.094 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:13.128 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:13.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:13.132 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:13.233 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:13.267 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:13.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:13.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:13.372 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:13.399 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:13.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:13.401 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:13.502 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:13.529 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:13.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:13.532 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:13.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:13 smithi092 bash[58855]: cluster 2023-09-23T19:57:12.558955+0000 mgr.y (mgr.25325) 1497 : cluster 0 pgmap v1522: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:57:13.633 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:13.663 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:13.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:13.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:13.767 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:13.779 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:13 smithi016 bash[77995]: cluster 2023-09-23T19:57:12.558955+0000 mgr.y (mgr.25325) 1497 : cluster 0 pgmap v1522: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:57:13.779 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:13 smithi016 bash[75004]: cluster 2023-09-23T19:57:12.558955+0000 mgr.y (mgr.25325) 1497 : cluster 0 pgmap v1522: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:57:13.801 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:13.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:13.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:13.817 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T19:57:13.817 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T19:57:13.905 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:13.937 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:13.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:13.939 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:14.040 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:14.062 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:14.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:14.064 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:14.165 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:14.188 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:14.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:14.191 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:14.292 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:14.320 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:14.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:14.324 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:14.425 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:14.453 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:14.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:14.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:57:14.557 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:14.569 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:14 smithi016 bash[77995]: audit 2023-09-23T19:57:13.686764+0000 mon.a (mon.0) 1824 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:57:14.569 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:14 smithi016 bash[75004]: audit 2023-09-23T19:57:13.686764+0000 mon.a (mon.0) 1824 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:57:14.583 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:14.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:14.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:14.690 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:14.711 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:14.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:14.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:14.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:14 smithi092 bash[58855]: audit 2023-09-23T19:57:13.686764+0000 mon.a (mon.0) 1824 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:57:14.814 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:14.819 DEBUG:teuthology.orchestra.run.smithi016:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_9 bench 90 write' 2023-09-23T19:57:14.821 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T19:57:14.821 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T19:57:14.844 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:14.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:14.847 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:14.862 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:hints = 1 2023-09-23T19:57:14.862 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T19:57:14.862 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Object prefix: benchmark_data_smithi016_228083 2023-09-23T19:57:14.863 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:57:14.863 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 0 0 0 0 0 0 - 0 2023-09-23T19:57:14.948 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:14.977 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:14.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:14.979 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:15.080 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:15.105 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:15.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:15.109 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:15.209 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:15.235 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:15.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:15.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:15.339 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:15.373 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:15.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:15.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:15.477 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:15.500 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:15.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:15.502 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:15.603 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:15.631 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:15.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:15.634 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:57:15.735 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:15.764 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:15.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:15.766 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:15.863 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 1 16 2346 2330 145.615 145.625 0.0027845 0.00683832 2023-09-23T19:57:15.867 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:15.895 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-09-23T19:57:15.896 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:57:15.896 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:15.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:15.901 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:15.901 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:15 smithi016 bash[75004]: cluster 2023-09-23T19:57:14.559655+0000 mgr.y (mgr.25325) 1498 : cluster 0 pgmap v1523: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:57:15.902 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:57:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:57:15] "GET /metrics HTTP/1.1" 200 36224 "" "Prometheus/2.33.4" 2023-09-23T19:57:16.001 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:16.012 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:15 smithi092 bash[58855]: cluster 2023-09-23T19:57:14.559655+0000 mgr.y (mgr.25325) 1498 : cluster 0 pgmap v1523: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:57:16.030 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:16.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:16.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:16.134 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:16.162 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:16.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:16.165 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:16.266 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:16.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:15 smithi016 bash[77995]: cluster 2023-09-23T19:57:14.559655+0000 mgr.y (mgr.25325) 1498 : cluster 0 pgmap v1523: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:57:16.294 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:16.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:16.298 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:16.399 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:16.436 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:16.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:16.439 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:16.539 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:16.565 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:16.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:16.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:16.671 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:16.701 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:16.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:16.704 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:16.804 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:16.832 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:16.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:16.835 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:16.863 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 2 16 4732 4716 147.363 149.125 0.00748143 0.00676919 2023-09-23T19:57:16.935 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:16.960 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:16.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:16.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:17.063 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:17.091 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:17.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:17.099 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:17.200 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:17.230 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:17.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:17.233 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:17.334 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:17.361 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:17.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:17.364 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:57:17.465 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:17.496 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:17.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:17.499 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:17.600 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:17.627 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:17.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:17.631 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:17.732 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:17.751 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:17 smithi016 bash[75004]: cluster 2023-09-23T19:57:16.560412+0000 mgr.y (mgr.25325) 1499 : cluster 0 pgmap v1524: 89 pgs: 89 active+clean; 45 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 4.5 MiB/s wr, 71 op/s 2023-09-23T19:57:17.763 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:17.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:17.768 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:57:17.863 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 3 16 7417 7401 154.174 167.812 0.00383366 0.00647464 2023-09-23T19:57:17.868 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:17.893 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:17.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:17.896 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:17.997 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:18.007 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:17 smithi092 bash[58855]: cluster 2023-09-23T19:57:16.560412+0000 mgr.y (mgr.25325) 1499 : cluster 0 pgmap v1524: 89 pgs: 89 active+clean; 45 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 4.5 MiB/s wr, 71 op/s 2023-09-23T19:57:18.022 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:18.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:18.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:18.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:17 smithi016 bash[77995]: cluster 2023-09-23T19:57:16.560412+0000 mgr.y (mgr.25325) 1499 : cluster 0 pgmap v1524: 89 pgs: 89 active+clean; 45 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 4.5 MiB/s wr, 71 op/s 2023-09-23T19:57:18.126 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:18.156 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:18.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:18.160 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:18.261 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:18.289 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:18.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:18.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:18.393 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:18.418 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:18.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:18.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:18.521 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:18.563 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:18.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:18.565 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:18.666 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:18.699 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:18.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:18.702 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:18.802 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:18.826 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:18.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:18.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:18.863 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 4 16 9897 9881 154.377 155 0.00403868 0.00646397 2023-09-23T19:57:18.929 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:18.957 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:18.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:18.960 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:19.061 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:19.088 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:19.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:19.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:19.191 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:19.217 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:19.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:19.221 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:19.322 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:19.345 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:19.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:19.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:19.449 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:19.480 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:19.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:19.484 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:19.585 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:19.613 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:19.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:19.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:19.716 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:19.742 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:19.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:19.746 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:19.847 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:19.856 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:19 smithi016 bash[77995]: cluster 2023-09-23T19:57:18.560926+0000 mgr.y (mgr.25325) 1500 : cluster 0 pgmap v1525: 89 pgs: 89 active+clean; 172 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 17 MiB/s wr, 274 op/s 2023-09-23T19:57:19.856 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:19 smithi016 bash[75004]: cluster 2023-09-23T19:57:18.560926+0000 mgr.y (mgr.25325) 1500 : cluster 0 pgmap v1525: 89 pgs: 89 active+clean; 172 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 17 MiB/s wr, 274 op/s 2023-09-23T19:57:19.863 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 5 16 12167 12151 151.856 141.875 0.0158367 0.00657494 2023-09-23T19:57:19.871 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:19.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:19.873 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:19.974 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:20.001 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:20.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:20.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:20.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:19 smithi092 bash[58855]: cluster 2023-09-23T19:57:18.560926+0000 mgr.y (mgr.25325) 1500 : cluster 0 pgmap v1525: 89 pgs: 89 active+clean; 172 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 17 MiB/s wr, 274 op/s 2023-09-23T19:57:20.104 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:20.122 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:20.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:20.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:20.225 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:20.258 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:20.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:20.262 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:57:20.363 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:20.397 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:20.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:20.401 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:20.502 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:20.527 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:20.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:20.530 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:20.631 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:20.656 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:20.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:20.660 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:20.760 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:20.794 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:20.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:20.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:20.838 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T19:57:20.864 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 6 16 13731 13715 142.838 97.75 0.00415674 0.00699122 2023-09-23T19:57:20.897 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:20.924 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:20.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:20.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:21.030 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:21.072 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:21.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:21.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:21.175 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:21.206 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:21.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:21.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:21.307 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:57:21.309 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:21.337 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:21.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:21.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:57:21.346 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:57:21.399 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:57:21.437 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:57:21.441 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:21.468 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:21.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:21.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:21.482 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:57:21.518 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:57:21.555 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:57:21.572 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:21.597 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:57:21.602 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:21.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:21.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:21.634 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:57:21.638 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:57:21.638 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:57:21.638 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:57:21.638 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:57:21.638 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:57:21.639 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:57:21.639 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:57:21.639 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:57:21.710 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:21.743 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:21.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:21.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:21.786 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is set 2023-09-23T19:57:21.840 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:21 smithi016 bash[75004]: cluster 2023-09-23T19:57:20.561678+0000 mgr.y (mgr.25325) 1501 : cluster 0 pgmap v1526: 89 pgs: 89 active+clean; 674 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-09-23T19:57:21.840 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:21 smithi016 bash[75004]: audit 2023-09-23T19:57:21.281285+0000 mon.c (mon.2) 230 : audit 0 from='client.? 172.21.15.16:0/786616113' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:57:21.840 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:21 smithi016 bash[75004]: audit 2023-09-23T19:57:21.460469+0000 mon.a (mon.0) 1825 : audit 1 from='client.? 172.21.15.16:0/640378005' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:57:21.840 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:21 smithi016 bash[77995]: cluster 2023-09-23T19:57:20.561678+0000 mgr.y (mgr.25325) 1501 : cluster 0 pgmap v1526: 89 pgs: 89 active+clean; 674 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-09-23T19:57:21.840 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:21 smithi016 bash[77995]: audit 2023-09-23T19:57:21.281285+0000 mon.c (mon.2) 230 : audit 0 from='client.? 172.21.15.16:0/786616113' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:57:21.841 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:21 smithi016 bash[77995]: audit 2023-09-23T19:57:21.460469+0000 mon.a (mon.0) 1825 : audit 1 from='client.? 172.21.15.16:0/640378005' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:57:21.850 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:21.864 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 7 16 15424 15408 137.548 105.812 0.00949362 0.00725858 2023-09-23T19:57:21.878 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:21.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:21.882 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:21.982 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:22.008 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:22.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:22.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:22.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:21 smithi092 bash[58855]: cluster 2023-09-23T19:57:20.561678+0000 mgr.y (mgr.25325) 1501 : cluster 0 pgmap v1526: 89 pgs: 89 active+clean; 674 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-09-23T19:57:22.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:21 smithi092 bash[58855]: audit 2023-09-23T19:57:21.281285+0000 mon.c (mon.2) 230 : audit 0 from='client.? 172.21.15.16:0/786616113' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:57:22.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:21 smithi092 bash[58855]: audit 2023-09-23T19:57:21.460469+0000 mon.a (mon.0) 1825 : audit 1 from='client.? 172.21.15.16:0/640378005' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:57:22.114 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:22.149 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:22.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:22.153 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:22.253 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:22.281 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:22.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:22.289 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:22.389 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:22.413 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:22.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:22.418 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:57:22.519 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:22.550 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:22.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:22.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:22.652 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:22.682 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:22.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:22.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:22.786 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:22.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:22 smithi092 bash[58855]: cluster 2023-09-23T19:57:21.761011+0000 mon.a (mon.0) 1826 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:57:22.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:22 smithi092 bash[58855]: audit 2023-09-23T19:57:21.775770+0000 mon.a (mon.0) 1827 : audit 1 from='client.? 172.21.15.16:0/640378005' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:57:22.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:22 smithi092 bash[58855]: cluster 2023-09-23T19:57:21.775994+0000 mon.a (mon.0) 1828 : cluster 0 osdmap e261: 8 total, 8 up, 8 in 2023-09-23T19:57:22.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:22 smithi092 bash[58855]: cluster 2023-09-23T19:57:21.786881+0000 osd.6 (osd.6) 195 : cluster 3 failed to encode map e261 with expected crc 2023-09-23T19:57:22.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:22 smithi092 bash[58855]: cluster 2023-09-23T19:57:21.787635+0000 osd.7 (osd.7) 173 : cluster 3 failed to encode map e261 with expected crc 2023-09-23T19:57:22.820 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:22.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:22.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:22.864 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 8 16 17489 17473 136.485 129.062 0.0136181 0.00731948 2023-09-23T19:57:22.924 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:22.950 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:22.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:22.952 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:23.053 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:23.070 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:23.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:23.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:23.107 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:22 smithi016 bash[77995]: cluster 2023-09-23T19:57:21.761011+0000 mon.a (mon.0) 1826 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:57:23.107 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:22 smithi016 bash[77995]: audit 2023-09-23T19:57:21.775770+0000 mon.a (mon.0) 1827 : audit 1 from='client.? 172.21.15.16:0/640378005' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:57:23.107 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:22 smithi016 bash[77995]: cluster 2023-09-23T19:57:21.775994+0000 mon.a (mon.0) 1828 : cluster 0 osdmap e261: 8 total, 8 up, 8 in 2023-09-23T19:57:23.107 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:22 smithi016 bash[77995]: cluster 2023-09-23T19:57:21.786881+0000 osd.6 (osd.6) 195 : cluster 3 failed to encode map e261 with expected crc 2023-09-23T19:57:23.107 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:22 smithi016 bash[77995]: cluster 2023-09-23T19:57:21.787635+0000 osd.7 (osd.7) 173 : cluster 3 failed to encode map e261 with expected crc 2023-09-23T19:57:23.107 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:22 smithi016 bash[75004]: cluster 2023-09-23T19:57:21.761011+0000 mon.a (mon.0) 1826 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:57:23.107 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:22 smithi016 bash[75004]: audit 2023-09-23T19:57:21.775770+0000 mon.a (mon.0) 1827 : audit 1 from='client.? 172.21.15.16:0/640378005' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:57:23.108 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:22 smithi016 bash[75004]: cluster 2023-09-23T19:57:21.775994+0000 mon.a (mon.0) 1828 : cluster 0 osdmap e261: 8 total, 8 up, 8 in 2023-09-23T19:57:23.108 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:22 smithi016 bash[75004]: cluster 2023-09-23T19:57:21.786881+0000 osd.6 (osd.6) 195 : cluster 3 failed to encode map e261 with expected crc 2023-09-23T19:57:23.108 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:22 smithi016 bash[75004]: cluster 2023-09-23T19:57:21.787635+0000 osd.7 (osd.7) 173 : cluster 3 failed to encode map e261 with expected crc 2023-09-23T19:57:23.172 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:23.200 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:23.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:23.202 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:57:23.303 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:23.327 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:23.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:23.333 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:23.433 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:23.466 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:23.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:23.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:23.570 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:23.596 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:23.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:23.599 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:23.700 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:23.732 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:23.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:23.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:23.836 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:23.864 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 9 16 19450 19434 134.937 122.562 0.00781969 0.00740227 2023-09-23T19:57:23.865 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:23.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:23.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:23.969 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:24.001 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:24.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:24.002 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:24.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:23 smithi016 bash[77995]: cluster 2023-09-23T19:57:22.562417+0000 mgr.y (mgr.25325) 1502 : cluster 0 pgmap v1528: 89 pgs: 89 active+clean; 794 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-09-23T19:57:24.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:23 smithi016 bash[75004]: cluster 2023-09-23T19:57:22.562417+0000 mgr.y (mgr.25325) 1502 : cluster 0 pgmap v1528: 89 pgs: 89 active+clean; 794 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-09-23T19:57:24.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:23 smithi092 bash[58855]: cluster 2023-09-23T19:57:22.562417+0000 mgr.y (mgr.25325) 1502 : cluster 0 pgmap v1528: 89 pgs: 89 active+clean; 794 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-09-23T19:57:24.103 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:24.131 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:24.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:24.136 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:24.236 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:24.267 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:24.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:24.270 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:24.371 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:24.395 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:24.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:24.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:24.499 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:24.529 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:24.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:24.533 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:24.634 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:24.662 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:24.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:24.666 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:24.767 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:24.795 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:24.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:24.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:24.864 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 10 16 21088 21072 131.68 102.375 0.0077113 0.0075851 2023-09-23T19:57:24.901 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:24.931 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:24.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:24.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:25.037 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:25.066 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:25.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:25.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:25.169 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:25.190 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:25.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:25.193 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:25.294 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:25.323 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:25.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:25.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:25.428 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:25.454 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:25.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:25.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:25.557 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:25.584 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:25.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:25.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:25.641 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:57:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:57:25] "GET /metrics HTTP/1.1" 200 36224 "" "Prometheus/2.33.4" 2023-09-23T19:57:25.687 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:25.718 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:25.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:25.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:25.783 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:25 smithi092 bash[58855]: cluster 2023-09-23T19:57:24.563114+0000 mgr.y (mgr.25325) 1503 : cluster 0 pgmap v1529: 89 pgs: 89 active+clean; 874 MiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-09-23T19:57:25.822 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:25.850 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:25.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:25.852 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:25.864 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 11 16 22755 22739 129.18 104.188 0.00267163 0.00773306 2023-09-23T19:57:25.953 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:25.981 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:25.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:25.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:57:26.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:25 smithi016 bash[77995]: cluster 2023-09-23T19:57:24.563114+0000 mgr.y (mgr.25325) 1503 : cluster 0 pgmap v1529: 89 pgs: 89 active+clean; 874 MiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-09-23T19:57:26.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:25 smithi016 bash[75004]: cluster 2023-09-23T19:57:24.563114+0000 mgr.y (mgr.25325) 1503 : cluster 0 pgmap v1529: 89 pgs: 89 active+clean; 874 MiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-09-23T19:57:26.084 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:26.113 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:26.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:26.116 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:26.217 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:26.241 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:26.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:26.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:26.344 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:26.369 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:26.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:26.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:26.472 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:26.498 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:26.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:26.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:26.601 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:26.631 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:26.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:26.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:26.737 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:26.779 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:26.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:26.783 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:57:26.864 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 12 16 24457 24441 127.279 106.375 0.0116074 0.00784989 2023-09-23T19:57:26.884 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:26.915 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:26.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:26.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:27.020 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:27.045 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:27.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:27.048 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:27.149 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:27.181 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:27.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:27.184 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:27.286 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:27.317 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:27.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:27.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:57:27.421 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:27.449 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:27.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:27.455 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:27.556 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:27.594 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:27.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:27.598 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:27.698 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:27.729 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:27.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:27.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:27.832 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:27.860 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:27.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:27.864 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:27.864 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 13 16 26150 26134 125.627 105.812 0.00813527 0.0079522 2023-09-23T19:57:27.965 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:27.974 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:27 smithi092 bash[58855]: cluster 2023-09-23T19:57:26.563756+0000 mgr.y (mgr.25325) 1504 : cluster 0 pgmap v1530: 89 pgs: 89 active+clean; 1.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 122 MiB/s wr, 1.95k op/s 2023-09-23T19:57:27.990 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:27.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:27.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:27.995 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:27 smithi016 bash[77995]: cluster 2023-09-23T19:57:26.563756+0000 mgr.y (mgr.25325) 1504 : cluster 0 pgmap v1530: 89 pgs: 89 active+clean; 1.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 122 MiB/s wr, 1.95k op/s 2023-09-23T19:57:27.995 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:27 smithi016 bash[75004]: cluster 2023-09-23T19:57:26.563756+0000 mgr.y (mgr.25325) 1504 : cluster 0 pgmap v1530: 89 pgs: 89 active+clean; 1.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 122 MiB/s wr, 1.95k op/s 2023-09-23T19:57:28.094 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:28.127 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:28.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:28.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:28.232 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:28.261 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:28.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:28.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:28.365 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:28.393 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:28.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:28.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:28.496 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:28.525 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:28.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:28.528 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:28.539 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:57:28.579 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T19:57:28.629 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:28.655 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:28.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:28.658 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:28.758 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:28.789 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:28.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:28.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:28.866 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 14 16 27682 27666 123.476 95.75 0.0272836 0.00809065 2023-09-23T19:57:28.892 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:28.919 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:28.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:28.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:29.023 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:29.048 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:29.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:29.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:29.152 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:29.181 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:29.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:29.185 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:29.286 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:29.313 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:29.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:29.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:29.416 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:29.439 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:29.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:29.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:57:29.542 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:29.568 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:29.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:29.570 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:29.671 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:29.701 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:29.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:29.705 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:57:29.806 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:29.837 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:29.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:29.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:29.868 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 15 16 29363 29347 122.248 105.062 0.0105698 0.00817297 2023-09-23T19:57:29.939 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:29.950 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:29 smithi016 bash[75004]: cluster 2023-09-23T19:57:28.564349+0000 mgr.y (mgr.25325) 1505 : cluster 0 pgmap v1531: 89 pgs: 89 active+clean; 1.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 126 MiB/s wr, 2.01k op/s 2023-09-23T19:57:29.951 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:29 smithi016 bash[75004]: audit 2023-09-23T19:57:28.696120+0000 mon.a (mon.0) 1829 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:57:29.951 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:29 smithi016 bash[75004]: audit 2023-09-23T19:57:28.703478+0000 mon.a (mon.0) 1830 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:57:29.951 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:29 smithi016 bash[77995]: cluster 2023-09-23T19:57:28.564349+0000 mgr.y (mgr.25325) 1505 : cluster 0 pgmap v1531: 89 pgs: 89 active+clean; 1.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 126 MiB/s wr, 2.01k op/s 2023-09-23T19:57:29.951 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:29 smithi016 bash[77995]: audit 2023-09-23T19:57:28.696120+0000 mon.a (mon.0) 1829 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:57:29.951 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:29 smithi016 bash[77995]: audit 2023-09-23T19:57:28.703478+0000 mon.a (mon.0) 1830 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:57:29.968 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:29.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:29.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:30.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:29 smithi092 bash[58855]: cluster 2023-09-23T19:57:28.564349+0000 mgr.y (mgr.25325) 1505 : cluster 0 pgmap v1531: 89 pgs: 89 active+clean; 1.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 126 MiB/s wr, 2.01k op/s 2023-09-23T19:57:30.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:29 smithi092 bash[58855]: audit 2023-09-23T19:57:28.696120+0000 mon.a (mon.0) 1829 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:57:30.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:29 smithi092 bash[58855]: audit 2023-09-23T19:57:28.703478+0000 mon.a (mon.0) 1830 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:57:30.071 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:30.099 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:30.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:30.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:57:30.202 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:30.235 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:30.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:30.240 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:30.340 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:30.377 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:30.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:30.382 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:30.483 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:30.517 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:30.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:30.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:30.622 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:30.651 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:30.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:30.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:30.755 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:30.784 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:30.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:30.786 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:30.866 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 16 16 30923 30907 120.701 97.5 0.00514055 0.00827454 2023-09-23T19:57:30.887 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:30.921 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:30.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:30.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:31.024 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:31.056 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:31.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:31.060 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:57:31.134 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-09-23T19:57:31.141 INFO:teuthology.orchestra.run.smithi016.stderr:dumped all 2023-09-23T19:57:31.161 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:31.195 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:31.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:31.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:31.299 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:31.328 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:31.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:31.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:31.431 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:31.470 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:31.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:31.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:31.574 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:31.620 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:31.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:31.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:31.727 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:31.767 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:31.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:31.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:31.867 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 17 16 32394 32378 119.009 91.9375 0.00926034 0.00839716 2023-09-23T19:57:31.872 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:31.898 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:31.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:31.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:32.004 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:32.011 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:31 smithi092 bash[58855]: cluster 2023-09-23T19:57:30.565105+0000 mgr.y (mgr.25325) 1506 : cluster 0 pgmap v1532: 89 pgs: 89 active+clean; 1.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-09-23T19:57:32.011 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:31 smithi092 bash[58855]: audit 2023-09-23T19:57:31.111076+0000 mgr.y (mgr.25325) 1507 : audit 0 from='client.36942 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:57:32.029 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:32.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:32.031 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:57:32.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:31 smithi016 bash[75004]: cluster 2023-09-23T19:57:30.565105+0000 mgr.y (mgr.25325) 1506 : cluster 0 pgmap v1532: 89 pgs: 89 active+clean; 1.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-09-23T19:57:32.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:31 smithi016 bash[75004]: audit 2023-09-23T19:57:31.111076+0000 mgr.y (mgr.25325) 1507 : audit 0 from='client.36942 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:57:32.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:31 smithi016 bash[77995]: cluster 2023-09-23T19:57:30.565105+0000 mgr.y (mgr.25325) 1506 : cluster 0 pgmap v1532: 89 pgs: 89 active+clean; 1.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-09-23T19:57:32.039 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:31 smithi016 bash[77995]: audit 2023-09-23T19:57:31.111076+0000 mgr.y (mgr.25325) 1507 : audit 0 from='client.36942 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:57:32.132 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:32.159 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:32.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:32.162 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:32.263 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:32.293 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:32.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:32.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:32.397 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:32.430 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:32.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:32.433 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:32.534 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:32.569 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:32.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:32.574 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:32.674 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:32.702 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:32.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:32.705 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:32.805 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:32.829 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:32.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:32.834 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:32.867 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 18 16 33414 33398 115.939 63.75 0.00450262 0.0086178 2023-09-23T19:57:32.935 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:32.963 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:32.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:32.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:33.015 INFO:teuthology.orchestra.run.smithi016.stdout:{"pg_ready":true,"pg_map":{"version":1532,"stamp":"2023-09-23T19:57:30.564565+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1878398880,"num_objects":28677,"num_object_clones":0,"num_object_copies":86031,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28677,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":28914,"num_write_kb":1836238,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":28848,"ondisk_log_size":28848,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":12087304,"kb_used_data":5222216,"kb_used_omap":20,"kb_used_meta":6865003,"kb_avail":737873912,"statfs":{"total":767960285184,"available":755582885888,"internally_reserved":0,"allocated":5347549184,"data_stored":5341319156,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":7029763726},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[6,7,5,3,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1171783680,"num_objects":17880,"num_object_clones":0,"num_object_copies":53640,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":17880,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":17880,"num_write_kb":1144320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003462"},"pg_stats":[{"pgid":"3.1a","version":"59'1","reported_seq":257,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.344959+0000","last_change":"2023-09-23T19:38:43.897548+0000","last_active":"2023-09-23T19:57:10.344959+0000","last_peered":"2023-09-23T19:57:10.344959+0000","last_clean":"2023-09-23T19:57:10.344959+0000","last_became_active":"2023-09-23T19:38:43.896868+0000","last_became_peered":"2023-09-23T19:38:43.896868+0000","last_unstale":"2023-09-23T19:57:10.344959+0000","last_undegraded":"2023-09-23T19:57:10.344959+0000","last_fullsized":"2023-09-23T19:57:10.344959+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":223,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.335384+0000","last_change":"2023-09-23T19:32:49.731128+0000","last_active":"2023-09-23T19:57:10.335384+0000","last_peered":"2023-09-23T19:57:10.335384+0000","last_clean":"2023-09-23T19:57:10.335384+0000","last_became_active":"2023-09-23T19:32:49.730757+0000","last_became_peered":"2023-09-23T19:32:49.730757+0000","last_unstale":"2023-09-23T19:57:10.335384+0000","last_undegraded":"2023-09-23T19:57:10.335384+0000","last_fullsized":"2023-09-23T19:57:10.335384+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:40.183605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":212,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.334980+0000","last_change":"2023-09-23T19:28:07.890462+0000","last_active":"2023-09-23T19:57:10.334980+0000","last_peered":"2023-09-23T19:57:10.334980+0000","last_clean":"2023-09-23T19:57:10.334980+0000","last_became_active":"2023-09-23T19:28:07.890084+0000","last_became_peered":"2023-09-23T19:28:07.890084+0000","last_unstale":"2023-09-23T19:57:10.334980+0000","last_undegraded":"2023-09-23T19:57:10.334980+0000","last_fullsized":"2023-09-23T19:57:10.334980+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:50.458218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":147,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.429581+0000","last_change":"2023-09-23T19:42:55.772143+0000","last_active":"2023-09-23T19:57:10.429581+0000","last_peered":"2023-09-23T19:57:10.429581+0000","last_clean":"2023-09-23T19:57:10.429581+0000","last_became_active":"2023-09-23T19:42:55.771813+0000","last_became_peered":"2023-09-23T19:42:55.771813+0000","last_unstale":"2023-09-23T19:57:10.429581+0000","last_undegraded":"2023-09-23T19:57:10.429581+0000","last_fullsized":"2023-09-23T19:57:10.429581+0000","mapping_epoch":188,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":241,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.921177+0000","last_change":"2023-09-23T19:32:49.729961+0000","last_active":"2023-09-23T19:57:10.921177+0000","last_peered":"2023-09-23T19:57:10.921177+0000","last_clean":"2023-09-23T19:57:10.921177+0000","last_became_active":"2023-09-23T19:32:49.729251+0000","last_became_peered":"2023-09-23T19:32:49.729251+0000","last_unstale":"2023-09-23T19:57:10.921177+0000","last_undegraded":"2023-09-23T19:57:10.921177+0000","last_fullsized":"2023-09-23T19:57:10.921177+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:20:56.595732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":226,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.928780+0000","last_change":"2023-09-23T19:22:42.150582+0000","last_active":"2023-09-23T19:57:10.928780+0000","last_peered":"2023-09-23T19:57:10.928780+0000","last_clean":"2023-09-23T19:57:10.928780+0000","last_became_active":"2023-09-23T19:22:42.150229+0000","last_became_peered":"2023-09-23T19:22:42.150229+0000","last_unstale":"2023-09-23T19:57:10.928780+0000","last_undegraded":"2023-09-23T19:57:10.928780+0000","last_fullsized":"2023-09-23T19:57:10.928780+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":205,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.928962+0000","last_change":"2023-09-23T19:47:56.793944+0000","last_active":"2023-09-23T19:57:10.928962+0000","last_peered":"2023-09-23T19:57:10.928962+0000","last_clean":"2023-09-23T19:57:10.928962+0000","last_became_active":"2023-09-23T19:47:56.793618+0000","last_became_peered":"2023-09-23T19:47:56.793618+0000","last_unstale":"2023-09-23T19:57:10.928962+0000","last_undegraded":"2023-09-23T19:57:10.928962+0000","last_fullsized":"2023-09-23T19:57:10.928962+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":224,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.335053+0000","last_change":"2023-09-23T19:32:49.729765+0000","last_active":"2023-09-23T19:57:10.335053+0000","last_peered":"2023-09-23T19:57:10.335053+0000","last_clean":"2023-09-23T19:57:10.335053+0000","last_became_active":"2023-09-23T19:32:49.729007+0000","last_became_peered":"2023-09-23T19:32:49.729007+0000","last_unstale":"2023-09-23T19:57:10.335053+0000","last_undegraded":"2023-09-23T19:57:10.335053+0000","last_fullsized":"2023-09-23T19:57:10.335053+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:59:31.183753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":213,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.929658+0000","last_change":"2023-09-23T19:47:56.791464+0000","last_active":"2023-09-23T19:57:10.929658+0000","last_peered":"2023-09-23T19:57:10.929658+0000","last_clean":"2023-09-23T19:57:10.929658+0000","last_became_active":"2023-09-23T19:47:56.791144+0000","last_became_peered":"2023-09-23T19:47:56.791144+0000","last_unstale":"2023-09-23T19:57:10.929658+0000","last_undegraded":"2023-09-23T19:57:10.929658+0000","last_fullsized":"2023-09-23T19:57:10.929658+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":201,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.428916+0000","last_change":"2023-09-23T19:32:49.736864+0000","last_active":"2023-09-23T19:57:10.428916+0000","last_peered":"2023-09-23T19:57:10.428916+0000","last_clean":"2023-09-23T19:57:10.428916+0000","last_became_active":"2023-09-23T19:32:49.736523+0000","last_became_peered":"2023-09-23T19:32:49.736523+0000","last_unstale":"2023-09-23T19:57:10.428916+0000","last_undegraded":"2023-09-23T19:57:10.428916+0000","last_fullsized":"2023-09-23T19:57:10.428916+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":230,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.429704+0000","last_change":"2023-09-23T19:28:07.892835+0000","last_active":"2023-09-23T19:57:10.429704+0000","last_peered":"2023-09-23T19:57:10.429704+0000","last_clean":"2023-09-23T19:57:10.429704+0000","last_became_active":"2023-09-23T19:28:07.892338+0000","last_became_peered":"2023-09-23T19:28:07.892338+0000","last_unstale":"2023-09-23T19:57:10.429704+0000","last_undegraded":"2023-09-23T19:57:10.429704+0000","last_fullsized":"2023-09-23T19:57:10.429704+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":227,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.335367+0000","last_change":"2023-09-23T19:38:43.890137+0000","last_active":"2023-09-23T19:57:10.335367+0000","last_peered":"2023-09-23T19:57:10.335367+0000","last_clean":"2023-09-23T19:57:10.335367+0000","last_became_active":"2023-09-23T19:38:43.889800+0000","last_became_peered":"2023-09-23T19:38:43.889800+0000","last_unstale":"2023-09-23T19:57:10.335367+0000","last_undegraded":"2023-09-23T19:57:10.335367+0000","last_fullsized":"2023-09-23T19:57:10.335367+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:59:20.468913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'23","reported_seq":234,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.345837+0000","last_change":"2023-09-23T19:05:52.107727+0000","last_active":"2023-09-23T19:57:10.345837+0000","last_peered":"2023-09-23T19:57:10.345837+0000","last_clean":"2023-09-23T19:57:10.345837+0000","last_became_active":"2023-09-23T19:05:52.107378+0000","last_became_peered":"2023-09-23T19:05:52.107378+0000","last_unstale":"2023-09-23T19:57:10.345837+0000","last_undegraded":"2023-09-23T19:57:10.345837+0000","last_fullsized":"2023-09-23T19:57:10.345837+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":225,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.345130+0000","last_change":"2023-09-23T19:22:42.148497+0000","last_active":"2023-09-23T19:57:10.345130+0000","last_peered":"2023-09-23T19:57:10.345130+0000","last_clean":"2023-09-23T19:57:10.345130+0000","last_became_active":"2023-09-23T19:22:42.148147+0000","last_became_peered":"2023-09-23T19:22:42.148147+0000","last_unstale":"2023-09-23T19:57:10.345130+0000","last_undegraded":"2023-09-23T19:57:10.345130+0000","last_fullsized":"2023-09-23T19:57:10.345130+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":244,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.429323+0000","last_change":"2023-09-23T19:38:43.897378+0000","last_active":"2023-09-23T19:57:10.429323+0000","last_peered":"2023-09-23T19:57:10.429323+0000","last_clean":"2023-09-23T19:57:10.429323+0000","last_became_active":"2023-09-23T19:38:43.897012+0000","last_became_peered":"2023-09-23T19:38:43.897012+0000","last_unstale":"2023-09-23T19:57:10.429323+0000","last_undegraded":"2023-09-23T19:57:10.429323+0000","last_fullsized":"2023-09-23T19:57:10.429323+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":224,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.336041+0000","last_change":"2023-09-23T19:32:49.730549+0000","last_active":"2023-09-23T19:57:10.336041+0000","last_peered":"2023-09-23T19:57:10.336041+0000","last_clean":"2023-09-23T19:57:10.336041+0000","last_became_active":"2023-09-23T19:32:49.730105+0000","last_became_peered":"2023-09-23T19:32:49.730105+0000","last_unstale":"2023-09-23T19:57:10.336041+0000","last_undegraded":"2023-09-23T19:57:10.336041+0000","last_fullsized":"2023-09-23T19:57:10.336041+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:49.751893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":147,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.428813+0000","last_change":"2023-09-23T19:42:55.772691+0000","last_active":"2023-09-23T19:57:10.428813+0000","last_peered":"2023-09-23T19:57:10.428813+0000","last_clean":"2023-09-23T19:57:10.428813+0000","last_became_active":"2023-09-23T19:42:55.772309+0000","last_became_peered":"2023-09-23T19:42:55.772309+0000","last_unstale":"2023-09-23T19:57:10.428813+0000","last_undegraded":"2023-09-23T19:57:10.428813+0000","last_fullsized":"2023-09-23T19:57:10.428813+0000","mapping_epoch":188,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":212,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.929251+0000","last_change":"2023-09-23T19:47:56.794658+0000","last_active":"2023-09-23T19:57:10.929251+0000","last_peered":"2023-09-23T19:57:10.929251+0000","last_clean":"2023-09-23T19:57:10.929251+0000","last_became_active":"2023-09-23T19:47:56.793633+0000","last_became_peered":"2023-09-23T19:47:56.793633+0000","last_unstale":"2023-09-23T19:57:10.929251+0000","last_undegraded":"2023-09-23T19:57:10.929251+0000","last_fullsized":"2023-09-23T19:57:10.929251+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":212,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.429333+0000","last_change":"2023-09-23T19:05:52.114186+0000","last_active":"2023-09-23T19:57:10.429333+0000","last_peered":"2023-09-23T19:57:10.429333+0000","last_clean":"2023-09-23T19:57:10.429333+0000","last_became_active":"2023-09-23T19:05:52.113635+0000","last_became_peered":"2023-09-23T19:05:52.113635+0000","last_unstale":"2023-09-23T19:57:10.429333+0000","last_undegraded":"2023-09-23T19:57:10.429333+0000","last_fullsized":"2023-09-23T19:57:10.429333+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":147,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.429603+0000","last_change":"2023-09-23T19:42:55.772912+0000","last_active":"2023-09-23T19:57:10.429603+0000","last_peered":"2023-09-23T19:57:10.429603+0000","last_clean":"2023-09-23T19:57:10.429603+0000","last_became_active":"2023-09-23T19:42:55.772434+0000","last_became_peered":"2023-09-23T19:42:55.772434+0000","last_unstale":"2023-09-23T19:57:10.429603+0000","last_undegraded":"2023-09-23T19:57:10.429603+0000","last_fullsized":"2023-09-23T19:57:10.429603+0000","mapping_epoch":188,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":235,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.927764+0000","last_change":"2023-09-23T19:18:09.010195+0000","last_active":"2023-09-23T19:57:10.927764+0000","last_peered":"2023-09-23T19:57:10.927764+0000","last_clean":"2023-09-23T19:57:10.927764+0000","last_became_active":"2023-09-23T19:18:09.009687+0000","last_became_peered":"2023-09-23T19:18:09.009687+0000","last_unstale":"2023-09-23T19:57:10.927764+0000","last_undegraded":"2023-09-23T19:57:10.927764+0000","last_fullsized":"2023-09-23T19:57:10.927764+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":233,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.922394+0000","last_change":"2023-09-23T19:28:07.885265+0000","last_active":"2023-09-23T19:57:10.922394+0000","last_peered":"2023-09-23T19:57:10.922394+0000","last_clean":"2023-09-23T19:57:10.922394+0000","last_became_active":"2023-09-23T19:28:07.884353+0000","last_became_peered":"2023-09-23T19:28:07.884353+0000","last_unstale":"2023-09-23T19:57:10.922394+0000","last_undegraded":"2023-09-23T19:57:10.922394+0000","last_fullsized":"2023-09-23T19:57:10.922394+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:56.325352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.2","version":"261'1964","reported_seq":1977,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-09-23T19:57:30.217106+0000","last_change":"2023-09-23T19:56:50.550024+0000","last_active":"2023-09-23T19:57:30.217106+0000","last_peered":"2023-09-23T19:57:30.217106+0000","last_clean":"2023-09-23T19:57:30.217106+0000","last_became_active":"2023-09-23T19:56:50.548890+0000","last_became_peered":"2023-09-23T19:56:50.548890+0000","last_unstale":"2023-09-23T19:57:30.217106+0000","last_undegraded":"2023-09-23T19:57:30.217106+0000","last_fullsized":"2023-09-23T19:57:30.217106+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":256,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_clean_scrub_stamp":"2023-09-23T19:56:49.329294+0000","objects_scrubbed":0,"log_size":1964,"log_dups_size":0,"ondisk_log_size":1964,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128712704,"num_objects":1964,"num_object_clones":0,"num_object_copies":5892,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1964,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1964,"num_write_kb":125696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":228,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.430038+0000","last_change":"2023-09-23T19:18:09.035116+0000","last_active":"2023-09-23T19:57:10.430038+0000","last_peered":"2023-09-23T19:57:10.430038+0000","last_clean":"2023-09-23T19:57:10.430038+0000","last_became_active":"2023-09-23T19:18:09.034416+0000","last_became_peered":"2023-09-23T19:18:09.034416+0000","last_unstale":"2023-09-23T19:57:10.430038+0000","last_undegraded":"2023-09-23T19:57:10.430038+0000","last_fullsized":"2023-09-23T19:57:10.430038+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":197,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.929044+0000","last_change":"2023-09-23T19:47:56.790227+0000","last_active":"2023-09-23T19:57:10.929044+0000","last_peered":"2023-09-23T19:57:10.929044+0000","last_clean":"2023-09-23T19:57:10.929044+0000","last_became_active":"2023-09-23T19:47:56.789727+0000","last_became_peered":"2023-09-23T19:47:56.789727+0000","last_unstale":"2023-09-23T19:57:10.929044+0000","last_undegraded":"2023-09-23T19:57:10.929044+0000","last_fullsized":"2023-09-23T19:57:10.929044+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.3","version":"261'1839","reported_seq":1853,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-09-23T19:57:30.370158+0000","last_change":"2023-09-23T19:56:50.486771+0000","last_active":"2023-09-23T19:57:30.370158+0000","last_peered":"2023-09-23T19:57:30.370158+0000","last_clean":"2023-09-23T19:57:30.370158+0000","last_became_active":"2023-09-23T19:56:50.486200+0000","last_became_peered":"2023-09-23T19:56:50.486200+0000","last_unstale":"2023-09-23T19:57:30.370158+0000","last_undegraded":"2023-09-23T19:57:30.370158+0000","last_fullsized":"2023-09-23T19:57:30.370158+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":256,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_clean_scrub_stamp":"2023-09-23T19:56:49.329294+0000","objects_scrubbed":0,"log_size":1839,"log_dups_size":0,"ondisk_log_size":1839,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:35.217636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120520704,"num_objects":1839,"num_object_clones":0,"num_object_copies":5517,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1839,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1839,"num_write_kb":117696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":229,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.429732+0000","last_change":"2023-09-23T19:22:42.151134+0000","last_active":"2023-09-23T19:57:10.429732+0000","last_peered":"2023-09-23T19:57:10.429732+0000","last_clean":"2023-09-23T19:57:10.429732+0000","last_became_active":"2023-09-23T19:22:42.150553+0000","last_became_peered":"2023-09-23T19:22:42.150553+0000","last_unstale":"2023-09-23T19:57:10.429732+0000","last_undegraded":"2023-09-23T19:57:10.429732+0000","last_fullsized":"2023-09-23T19:57:10.429732+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":240,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.345182+0000","last_change":"2023-09-23T19:28:07.899262+0000","last_active":"2023-09-23T19:57:10.345182+0000","last_peered":"2023-09-23T19:57:10.345182+0000","last_clean":"2023-09-23T19:57:10.345182+0000","last_became_active":"2023-09-23T19:28:07.898848+0000","last_became_peered":"2023-09-23T19:28:07.898848+0000","last_unstale":"2023-09-23T19:57:10.345182+0000","last_undegraded":"2023-09-23T19:57:10.345182+0000","last_fullsized":"2023-09-23T19:57:10.345182+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.0","version":"261'1574","reported_seq":1587,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-09-23T19:57:27.344562+0000","last_change":"2023-09-23T19:56:50.353925+0000","last_active":"2023-09-23T19:57:27.344562+0000","last_peered":"2023-09-23T19:57:27.344562+0000","last_clean":"2023-09-23T19:57:27.344562+0000","last_became_active":"2023-09-23T19:56:50.353546+0000","last_became_peered":"2023-09-23T19:56:50.353546+0000","last_unstale":"2023-09-23T19:57:27.344562+0000","last_undegraded":"2023-09-23T19:57:27.344562+0000","last_fullsized":"2023-09-23T19:57:27.344562+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":256,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_clean_scrub_stamp":"2023-09-23T19:56:49.329294+0000","objects_scrubbed":0,"log_size":1574,"log_dups_size":0,"ondisk_log_size":1574,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:12:56.362116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103153664,"num_objects":1574,"num_object_clones":0,"num_object_copies":4722,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1574,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1574,"num_write_kb":100736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":213,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.429398+0000","last_change":"2023-09-23T19:05:52.114108+0000","last_active":"2023-09-23T19:57:10.429398+0000","last_peered":"2023-09-23T19:57:10.429398+0000","last_clean":"2023-09-23T19:57:10.429398+0000","last_became_active":"2023-09-23T19:05:52.113609+0000","last_became_peered":"2023-09-23T19:05:52.113609+0000","last_unstale":"2023-09-23T19:57:10.429398+0000","last_undegraded":"2023-09-23T19:57:10.429398+0000","last_fullsized":"2023-09-23T19:57:10.429398+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":226,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.929908+0000","last_change":"2023-09-23T19:22:42.156825+0000","last_active":"2023-09-23T19:57:10.929908+0000","last_peered":"2023-09-23T19:57:10.929908+0000","last_clean":"2023-09-23T19:57:10.929908+0000","last_became_active":"2023-09-23T19:22:42.156474+0000","last_became_peered":"2023-09-23T19:22:42.156474+0000","last_unstale":"2023-09-23T19:57:10.929908+0000","last_undegraded":"2023-09-23T19:57:10.929908+0000","last_fullsized":"2023-09-23T19:57:10.929908+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"261'1892","reported_seq":1906,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-09-23T19:57:30.309929+0000","last_change":"2023-09-23T19:56:50.543702+0000","last_active":"2023-09-23T19:57:30.309929+0000","last_peered":"2023-09-23T19:57:30.309929+0000","last_clean":"2023-09-23T19:57:30.309929+0000","last_became_active":"2023-09-23T19:56:50.543330+0000","last_became_peered":"2023-09-23T19:56:50.543330+0000","last_unstale":"2023-09-23T19:57:30.309929+0000","last_undegraded":"2023-09-23T19:57:30.309929+0000","last_fullsized":"2023-09-23T19:57:30.309929+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":256,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_clean_scrub_stamp":"2023-09-23T19:56:49.329294+0000","objects_scrubbed":0,"log_size":1892,"log_dups_size":0,"ondisk_log_size":1892,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:41:26.115634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123994112,"num_objects":1892,"num_object_clones":0,"num_object_copies":5676,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1892,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1892,"num_write_kb":121088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":213,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.928725+0000","last_change":"2023-09-23T19:47:56.794638+0000","last_active":"2023-09-23T19:57:10.928725+0000","last_peered":"2023-09-23T19:57:10.928725+0000","last_clean":"2023-09-23T19:57:10.928725+0000","last_became_active":"2023-09-23T19:47:56.794305+0000","last_became_peered":"2023-09-23T19:47:56.794305+0000","last_unstale":"2023-09-23T19:57:10.928725+0000","last_undegraded":"2023-09-23T19:57:10.928725+0000","last_fullsized":"2023-09-23T19:57:10.928725+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":229,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.921835+0000","last_change":"2023-09-23T19:22:42.135572+0000","last_active":"2023-09-23T19:57:10.921835+0000","last_peered":"2023-09-23T19:57:10.921835+0000","last_clean":"2023-09-23T19:57:10.921835+0000","last_became_active":"2023-09-23T19:22:42.135237+0000","last_became_peered":"2023-09-23T19:22:42.135237+0000","last_unstale":"2023-09-23T19:57:10.921835+0000","last_undegraded":"2023-09-23T19:57:10.921835+0000","last_fullsized":"2023-09-23T19:57:10.921835+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:35.838718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.6","version":"261'1555","reported_seq":1569,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-09-23T19:57:27.337246+0000","last_change":"2023-09-23T19:56:50.350294+0000","last_active":"2023-09-23T19:57:27.337246+0000","last_peered":"2023-09-23T19:57:27.337246+0000","last_clean":"2023-09-23T19:57:27.337246+0000","last_became_active":"2023-09-23T19:56:50.349934+0000","last_became_peered":"2023-09-23T19:56:50.349934+0000","last_unstale":"2023-09-23T19:57:27.337246+0000","last_undegraded":"2023-09-23T19:57:27.337246+0000","last_fullsized":"2023-09-23T19:57:27.337246+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":256,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_clean_scrub_stamp":"2023-09-23T19:56:49.329294+0000","objects_scrubbed":0,"log_size":1555,"log_dups_size":0,"ondisk_log_size":1555,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:52.927510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101908480,"num_objects":1555,"num_object_clones":0,"num_object_copies":4665,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1555,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1555,"num_write_kb":99520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":94,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.344971+0000","last_change":"2023-09-23T19:42:55.771907+0000","last_active":"2023-09-23T19:57:10.344971+0000","last_peered":"2023-09-23T19:57:10.344971+0000","last_clean":"2023-09-23T19:57:10.344971+0000","last_became_active":"2023-09-23T19:42:55.771503+0000","last_became_peered":"2023-09-23T19:42:55.771503+0000","last_unstale":"2023-09-23T19:57:10.344971+0000","last_undegraded":"2023-09-23T19:57:10.344971+0000","last_fullsized":"2023-09-23T19:57:10.344971+0000","mapping_epoch":188,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":225,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.345876+0000","last_change":"2023-09-23T19:28:07.896803+0000","last_active":"2023-09-23T19:57:10.345876+0000","last_peered":"2023-09-23T19:57:10.345876+0000","last_clean":"2023-09-23T19:57:10.345876+0000","last_became_active":"2023-09-23T19:28:07.896480+0000","last_became_peered":"2023-09-23T19:28:07.896480+0000","last_unstale":"2023-09-23T19:57:10.345876+0000","last_undegraded":"2023-09-23T19:57:10.345876+0000","last_fullsized":"2023-09-23T19:57:10.345876+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.7","version":"261'1820","reported_seq":1832,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-09-23T19:57:30.095159+0000","last_change":"2023-09-23T19:56:50.357844+0000","last_active":"2023-09-23T19:57:30.095159+0000","last_peered":"2023-09-23T19:57:30.095159+0000","last_clean":"2023-09-23T19:57:30.095159+0000","last_became_active":"2023-09-23T19:56:50.357465+0000","last_became_peered":"2023-09-23T19:56:50.357465+0000","last_unstale":"2023-09-23T19:57:30.095159+0000","last_undegraded":"2023-09-23T19:57:30.095159+0000","last_fullsized":"2023-09-23T19:57:30.095159+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":256,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_clean_scrub_stamp":"2023-09-23T19:56:49.329294+0000","objects_scrubbed":0,"log_size":1820,"log_dups_size":0,"ondisk_log_size":1820,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119275520,"num_objects":1820,"num_object_clones":0,"num_object_copies":5460,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1820,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1820,"num_write_kb":116480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":227,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.929793+0000","last_change":"2023-09-23T19:22:42.156582+0000","last_active":"2023-09-23T19:57:10.929793+0000","last_peered":"2023-09-23T19:57:10.929793+0000","last_clean":"2023-09-23T19:57:10.929793+0000","last_became_active":"2023-09-23T19:22:42.156251+0000","last_became_peered":"2023-09-23T19:22:42.156251+0000","last_unstale":"2023-09-23T19:57:10.929793+0000","last_undegraded":"2023-09-23T19:57:10.929793+0000","last_fullsized":"2023-09-23T19:57:10.929793+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":255,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.345916+0000","last_change":"2023-09-23T19:38:43.891593+0000","last_active":"2023-09-23T19:57:10.345916+0000","last_peered":"2023-09-23T19:57:10.345916+0000","last_clean":"2023-09-23T19:57:10.345916+0000","last_became_active":"2023-09-23T19:38:43.891063+0000","last_became_peered":"2023-09-23T19:38:43.891063+0000","last_unstale":"2023-09-23T19:57:10.345916+0000","last_undegraded":"2023-09-23T19:57:10.345916+0000","last_fullsized":"2023-09-23T19:57:10.345916+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.4","version":"261'1839","reported_seq":1852,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-09-23T19:57:30.207974+0000","last_change":"2023-09-23T19:56:50.492916+0000","last_active":"2023-09-23T19:57:30.207974+0000","last_peered":"2023-09-23T19:57:30.207974+0000","last_clean":"2023-09-23T19:57:30.207974+0000","last_became_active":"2023-09-23T19:56:50.492066+0000","last_became_peered":"2023-09-23T19:56:50.492066+0000","last_unstale":"2023-09-23T19:57:30.207974+0000","last_undegraded":"2023-09-23T19:57:30.207974+0000","last_fullsized":"2023-09-23T19:57:30.207974+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":256,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_clean_scrub_stamp":"2023-09-23T19:56:49.329294+0000","objects_scrubbed":0,"log_size":1839,"log_dups_size":0,"ondisk_log_size":1839,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120520704,"num_objects":1839,"num_object_clones":0,"num_object_copies":5517,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1839,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1839,"num_write_kb":117696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":241,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.344662+0000","last_change":"2023-09-23T19:38:43.892972+0000","last_active":"2023-09-23T19:57:10.344662+0000","last_peered":"2023-09-23T19:57:10.344662+0000","last_clean":"2023-09-23T19:57:10.344662+0000","last_became_active":"2023-09-23T19:38:43.892451+0000","last_became_peered":"2023-09-23T19:38:43.892451+0000","last_unstale":"2023-09-23T19:57:10.344662+0000","last_undegraded":"2023-09-23T19:57:10.344662+0000","last_fullsized":"2023-09-23T19:57:10.344662+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":225,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.345893+0000","last_change":"2023-09-23T19:22:42.149670+0000","last_active":"2023-09-23T19:57:10.345893+0000","last_peered":"2023-09-23T19:57:10.345893+0000","last_clean":"2023-09-23T19:57:10.345893+0000","last_became_active":"2023-09-23T19:22:42.149022+0000","last_became_peered":"2023-09-23T19:22:42.149022+0000","last_unstale":"2023-09-23T19:57:10.345893+0000","last_undegraded":"2023-09-23T19:57:10.345893+0000","last_fullsized":"2023-09-23T19:57:10.345893+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.5","version":"261'1585","reported_seq":1599,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-09-23T19:57:27.749218+0000","last_change":"2023-09-23T19:56:50.353313+0000","last_active":"2023-09-23T19:57:27.749218+0000","last_peered":"2023-09-23T19:57:27.749218+0000","last_clean":"2023-09-23T19:57:27.749218+0000","last_became_active":"2023-09-23T19:56:50.353091+0000","last_became_peered":"2023-09-23T19:56:50.353091+0000","last_unstale":"2023-09-23T19:57:27.749218+0000","last_undegraded":"2023-09-23T19:57:27.749218+0000","last_fullsized":"2023-09-23T19:57:27.749218+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":256,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_clean_scrub_stamp":"2023-09-23T19:56:49.329294+0000","objects_scrubbed":0,"log_size":1585,"log_dups_size":0,"ondisk_log_size":1585,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:29.940884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103874560,"num_objects":1585,"num_object_clones":0,"num_object_copies":4755,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1585,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1585,"num_write_kb":101440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":226,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.334816+0000","last_change":"2023-09-23T19:32:49.729244+0000","last_active":"2023-09-23T19:57:10.334816+0000","last_peered":"2023-09-23T19:57:10.334816+0000","last_clean":"2023-09-23T19:57:10.334816+0000","last_became_active":"2023-09-23T19:32:49.728810+0000","last_became_peered":"2023-09-23T19:32:49.728810+0000","last_unstale":"2023-09-23T19:57:10.334816+0000","last_undegraded":"2023-09-23T19:57:10.334816+0000","last_fullsized":"2023-09-23T19:57:10.334816+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:57.540826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":213,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.335937+0000","last_change":"2023-09-23T19:28:07.889095+0000","last_active":"2023-09-23T19:57:10.335937+0000","last_peered":"2023-09-23T19:57:10.335937+0000","last_clean":"2023-09-23T19:57:10.335937+0000","last_became_active":"2023-09-23T19:28:07.888742+0000","last_became_peered":"2023-09-23T19:28:07.888742+0000","last_unstale":"2023-09-23T19:57:10.335937+0000","last_undegraded":"2023-09-23T19:57:10.335937+0000","last_fullsized":"2023-09-23T19:57:10.335937+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:43.929488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.8","version":"261'1664","reported_seq":1677,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-09-23T19:57:27.762238+0000","last_change":"2023-09-23T19:56:50.347118+0000","last_active":"2023-09-23T19:57:27.762238+0000","last_peered":"2023-09-23T19:57:27.762238+0000","last_clean":"2023-09-23T19:57:27.762238+0000","last_became_active":"2023-09-23T19:56:50.346834+0000","last_became_peered":"2023-09-23T19:56:50.346834+0000","last_unstale":"2023-09-23T19:57:27.762238+0000","last_undegraded":"2023-09-23T19:57:27.762238+0000","last_fullsized":"2023-09-23T19:57:27.762238+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":256,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_clean_scrub_stamp":"2023-09-23T19:56:49.329294+0000","objects_scrubbed":0,"log_size":1664,"log_dups_size":0,"ondisk_log_size":1664,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:19.591681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109051904,"num_objects":1664,"num_object_clones":0,"num_object_copies":4992,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1664,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1664,"num_write_kb":106496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":247,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.335931+0000","last_change":"2023-09-23T19:22:42.143289+0000","last_active":"2023-09-23T19:57:10.335931+0000","last_peered":"2023-09-23T19:57:10.335931+0000","last_clean":"2023-09-23T19:57:10.335931+0000","last_became_active":"2023-09-23T19:22:42.142656+0000","last_became_peered":"2023-09-23T19:22:42.142656+0000","last_unstale":"2023-09-23T19:57:10.335931+0000","last_undegraded":"2023-09-23T19:57:10.335931+0000","last_fullsized":"2023-09-23T19:57:10.335931+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:38.467441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":243,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.929377+0000","last_change":"2023-09-23T19:47:56.794437+0000","last_active":"2023-09-23T19:57:10.929377+0000","last_peered":"2023-09-23T19:57:10.929377+0000","last_clean":"2023-09-23T19:57:10.929377+0000","last_became_active":"2023-09-23T19:47:56.794118+0000","last_became_peered":"2023-09-23T19:47:56.794118+0000","last_unstale":"2023-09-23T19:57:10.929377+0000","last_undegraded":"2023-09-23T19:57:10.929377+0000","last_fullsized":"2023-09-23T19:57:10.929377+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":233,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-09-23T19:57:21.788957+0000","last_change":"2023-09-23T19:38:43.898519+0000","last_active":"2023-09-23T19:57:21.788957+0000","last_peered":"2023-09-23T19:57:21.788957+0000","last_clean":"2023-09-23T19:57:21.788957+0000","last_became_active":"2023-09-23T19:38:43.896814+0000","last_became_peered":"2023-09-23T19:38:43.896814+0000","last_unstale":"2023-09-23T19:57:21.788957+0000","last_undegraded":"2023-09-23T19:57:21.788957+0000","last_fullsized":"2023-09-23T19:57:21.788957+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:40.465038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.e","version":"261'1875","reported_seq":1886,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-09-23T19:57:30.360957+0000","last_change":"2023-09-23T19:56:50.358245+0000","last_active":"2023-09-23T19:57:30.360957+0000","last_peered":"2023-09-23T19:57:30.360957+0000","last_clean":"2023-09-23T19:57:30.360957+0000","last_became_active":"2023-09-23T19:56:50.357883+0000","last_became_peered":"2023-09-23T19:56:50.357883+0000","last_unstale":"2023-09-23T19:57:30.360957+0000","last_undegraded":"2023-09-23T19:57:30.360957+0000","last_fullsized":"2023-09-23T19:57:30.360957+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":256,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_clean_scrub_stamp":"2023-09-23T19:56:49.329294+0000","objects_scrubbed":0,"log_size":1875,"log_dups_size":0,"ondisk_log_size":1875,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122880000,"num_objects":1875,"num_object_clones":0,"num_object_copies":5625,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1875,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1875,"num_write_kb":120000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":147,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.335248+0000","last_change":"2023-09-23T19:43:38.612700+0000","last_active":"2023-09-23T19:57:10.335248+0000","last_peered":"2023-09-23T19:57:10.335248+0000","last_clean":"2023-09-23T19:57:10.335248+0000","last_became_active":"2023-09-23T19:43:38.612424+0000","last_became_peered":"2023-09-23T19:43:38.612424+0000","last_unstale":"2023-09-23T19:57:10.335248+0000","last_undegraded":"2023-09-23T19:57:10.335248+0000","last_fullsized":"2023-09-23T19:57:10.335248+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:24.833490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":226,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.345029+0000","last_change":"2023-09-23T19:18:09.017525+0000","last_active":"2023-09-23T19:57:10.345029+0000","last_peered":"2023-09-23T19:57:10.345029+0000","last_clean":"2023-09-23T19:57:10.345029+0000","last_became_active":"2023-09-23T19:18:09.017193+0000","last_became_peered":"2023-09-23T19:18:09.017193+0000","last_unstale":"2023-09-23T19:57:10.345029+0000","last_undegraded":"2023-09-23T19:57:10.345029+0000","last_fullsized":"2023-09-23T19:57:10.345029+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":244,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.335813+0000","last_change":"2023-09-23T19:38:43.877088+0000","last_active":"2023-09-23T19:57:10.335813+0000","last_peered":"2023-09-23T19:57:10.335813+0000","last_clean":"2023-09-23T19:57:10.335813+0000","last_became_active":"2023-09-23T19:38:43.876719+0000","last_became_peered":"2023-09-23T19:38:43.876719+0000","last_unstale":"2023-09-23T19:57:10.335813+0000","last_undegraded":"2023-09-23T19:57:10.335813+0000","last_fullsized":"2023-09-23T19:57:10.335813+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:05.423256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.d","version":"261'1866","reported_seq":1879,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-09-23T19:57:30.204595+0000","last_change":"2023-09-23T19:56:50.359619+0000","last_active":"2023-09-23T19:57:30.204595+0000","last_peered":"2023-09-23T19:57:30.204595+0000","last_clean":"2023-09-23T19:57:30.204595+0000","last_became_active":"2023-09-23T19:56:50.359198+0000","last_became_peered":"2023-09-23T19:56:50.359198+0000","last_unstale":"2023-09-23T19:57:30.204595+0000","last_undegraded":"2023-09-23T19:57:30.204595+0000","last_fullsized":"2023-09-23T19:57:30.204595+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":256,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_clean_scrub_stamp":"2023-09-23T19:56:49.329294+0000","objects_scrubbed":0,"log_size":1866,"log_dups_size":0,"ondisk_log_size":1866,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122290176,"num_objects":1866,"num_object_clones":0,"num_object_copies":5598,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1866,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1866,"num_write_kb":119424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":213,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-09-23T19:57:21.789408+0000","last_change":"2023-09-23T19:38:43.883663+0000","last_active":"2023-09-23T19:57:21.789408+0000","last_peered":"2023-09-23T19:57:21.789408+0000","last_clean":"2023-09-23T19:57:21.789408+0000","last_became_active":"2023-09-23T19:38:43.882421+0000","last_became_peered":"2023-09-23T19:38:43.882421+0000","last_unstale":"2023-09-23T19:57:21.789408+0000","last_undegraded":"2023-09-23T19:57:21.789408+0000","last_fullsized":"2023-09-23T19:57:21.789408+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:49:01.607659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":230,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.334826+0000","last_change":"2023-09-23T19:38:43.890600+0000","last_active":"2023-09-23T19:57:10.334826+0000","last_peered":"2023-09-23T19:57:10.334826+0000","last_clean":"2023-09-23T19:57:10.334826+0000","last_became_active":"2023-09-23T19:38:43.890253+0000","last_became_peered":"2023-09-23T19:38:43.890253+0000","last_unstale":"2023-09-23T19:57:10.334826+0000","last_undegraded":"2023-09-23T19:57:10.334826+0000","last_fullsized":"2023-09-23T19:57:10.334826+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:46.751120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":242,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:20.362397+0000","last_change":"2023-09-23T19:22:42.143733+0000","last_active":"2023-09-23T19:57:20.362397+0000","last_peered":"2023-09-23T19:57:20.362397+0000","last_clean":"2023-09-23T19:57:20.362397+0000","last_became_active":"2023-09-23T19:22:42.143077+0000","last_became_peered":"2023-09-23T19:22:42.143077+0000","last_unstale":"2023-09-23T19:57:20.362397+0000","last_undegraded":"2023-09-23T19:57:20.362397+0000","last_fullsized":"2023-09-23T19:57:20.362397+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:47:33.086994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.f","version":"261'1888","reported_seq":1901,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-09-23T19:57:30.234810+0000","last_change":"2023-09-23T19:56:50.493491+0000","last_active":"2023-09-23T19:57:30.234810+0000","last_peered":"2023-09-23T19:57:30.234810+0000","last_clean":"2023-09-23T19:57:30.234810+0000","last_became_active":"2023-09-23T19:56:50.492405+0000","last_became_peered":"2023-09-23T19:56:50.492405+0000","last_unstale":"2023-09-23T19:57:30.234810+0000","last_undegraded":"2023-09-23T19:57:30.234810+0000","last_fullsized":"2023-09-23T19:57:30.234810+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":256,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_clean_scrub_stamp":"2023-09-23T19:56:49.329294+0000","objects_scrubbed":0,"log_size":1888,"log_dups_size":0,"ondisk_log_size":1888,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123731968,"num_objects":1888,"num_object_clones":0,"num_object_copies":5664,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1888,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1888,"num_write_kb":120832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":232,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.928116+0000","last_change":"2023-09-23T19:47:56.791259+0000","last_active":"2023-09-23T19:57:10.928116+0000","last_peered":"2023-09-23T19:57:10.928116+0000","last_clean":"2023-09-23T19:57:10.928116+0000","last_became_active":"2023-09-23T19:47:56.790954+0000","last_became_peered":"2023-09-23T19:47:56.790954+0000","last_unstale":"2023-09-23T19:57:10.928116+0000","last_undegraded":"2023-09-23T19:57:10.928116+0000","last_fullsized":"2023-09-23T19:57:10.928116+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":463,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.428712+0000","last_change":"2023-09-23T19:18:09.016297+0000","last_active":"2023-09-23T19:57:10.428712+0000","last_peered":"2023-09-23T19:57:10.428712+0000","last_clean":"2023-09-23T19:57:10.428712+0000","last_became_active":"2023-09-23T19:18:09.015436+0000","last_became_peered":"2023-09-23T19:18:09.015436+0000","last_unstale":"2023-09-23T19:57:10.428712+0000","last_undegraded":"2023-09-23T19:57:10.428712+0000","last_fullsized":"2023-09-23T19:57:10.428712+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:59:33.840751+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:59:33.840751+0000","last_clean_scrub_stamp":"2023-09-23T18:59:33.840751+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":186,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.929297+0000","last_change":"2023-09-23T19:47:56.811861+0000","last_active":"2023-09-23T19:57:10.929297+0000","last_peered":"2023-09-23T19:57:10.929297+0000","last_clean":"2023-09-23T19:57:10.929297+0000","last_became_active":"2023-09-23T19:47:56.811527+0000","last_became_peered":"2023-09-23T19:47:56.811527+0000","last_unstale":"2023-09-23T19:57:10.929297+0000","last_undegraded":"2023-09-23T19:57:10.929297+0000","last_fullsized":"2023-09-23T19:57:10.929297+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":211,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.929237+0000","last_change":"2023-09-23T19:05:52.786771+0000","last_active":"2023-09-23T19:57:10.929237+0000","last_peered":"2023-09-23T19:57:10.929237+0000","last_clean":"2023-09-23T19:57:10.929237+0000","last_became_active":"2023-09-23T19:05:52.785797+0000","last_became_peered":"2023-09-23T19:05:52.785797+0000","last_unstale":"2023-09-23T19:57:10.929237+0000","last_undegraded":"2023-09-23T19:57:10.929237+0000","last_fullsized":"2023-09-23T19:57:10.929237+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.c","version":"261'1923","reported_seq":1936,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-09-23T19:57:30.376198+0000","last_change":"2023-09-23T19:56:50.358230+0000","last_active":"2023-09-23T19:57:30.376198+0000","last_peered":"2023-09-23T19:57:30.376198+0000","last_clean":"2023-09-23T19:57:30.376198+0000","last_became_active":"2023-09-23T19:56:50.357855+0000","last_became_peered":"2023-09-23T19:56:50.357855+0000","last_unstale":"2023-09-23T19:57:30.376198+0000","last_undegraded":"2023-09-23T19:57:30.376198+0000","last_fullsized":"2023-09-23T19:57:30.376198+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":256,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_clean_scrub_stamp":"2023-09-23T19:56:49.329294+0000","objects_scrubbed":0,"log_size":1923,"log_dups_size":0,"ondisk_log_size":1923,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126025728,"num_objects":1923,"num_object_clones":0,"num_object_copies":5769,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1923,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1923,"num_write_kb":123072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":246,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.428583+0000","last_change":"2023-09-23T19:22:42.150636+0000","last_active":"2023-09-23T19:57:10.428583+0000","last_peered":"2023-09-23T19:57:10.428583+0000","last_clean":"2023-09-23T19:57:10.428583+0000","last_became_active":"2023-09-23T19:22:42.150045+0000","last_became_peered":"2023-09-23T19:22:42.150045+0000","last_unstale":"2023-09-23T19:57:10.428583+0000","last_undegraded":"2023-09-23T19:57:10.428583+0000","last_fullsized":"2023-09-23T19:57:10.428583+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":147,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.345124+0000","last_change":"2023-09-23T19:42:55.771906+0000","last_active":"2023-09-23T19:57:10.345124+0000","last_peered":"2023-09-23T19:57:10.345124+0000","last_clean":"2023-09-23T19:57:10.345124+0000","last_became_active":"2023-09-23T19:42:55.771599+0000","last_became_peered":"2023-09-23T19:42:55.771599+0000","last_unstale":"2023-09-23T19:57:10.345124+0000","last_undegraded":"2023-09-23T19:57:10.345124+0000","last_fullsized":"2023-09-23T19:57:10.345124+0000","mapping_epoch":188,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":260,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.922225+0000","last_change":"2023-09-23T19:38:43.876930+0000","last_active":"2023-09-23T19:57:10.922225+0000","last_peered":"2023-09-23T19:57:10.922225+0000","last_clean":"2023-09-23T19:57:10.922225+0000","last_became_active":"2023-09-23T19:38:43.876571+0000","last_became_peered":"2023-09-23T19:38:43.876571+0000","last_unstale":"2023-09-23T19:57:10.922225+0000","last_undegraded":"2023-09-23T19:57:10.922225+0000","last_fullsized":"2023-09-23T19:57:10.922225+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:30.924536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.9","version":"261'1865","reported_seq":1878,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-09-23T19:57:30.047948+0000","last_change":"2023-09-23T19:56:50.550531+0000","last_active":"2023-09-23T19:57:30.047948+0000","last_peered":"2023-09-23T19:57:30.047948+0000","last_clean":"2023-09-23T19:57:30.047948+0000","last_became_active":"2023-09-23T19:56:50.549713+0000","last_became_peered":"2023-09-23T19:56:50.549713+0000","last_unstale":"2023-09-23T19:57:30.047948+0000","last_undegraded":"2023-09-23T19:57:30.047948+0000","last_fullsized":"2023-09-23T19:57:30.047948+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":256,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_clean_scrub_stamp":"2023-09-23T19:56:49.329294+0000","objects_scrubbed":0,"log_size":1865,"log_dups_size":0,"ondisk_log_size":1865,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122224640,"num_objects":1865,"num_object_clones":0,"num_object_copies":5595,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1865,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1865,"num_write_kb":119360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":231,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.429412+0000","last_change":"2023-09-23T19:18:09.015853+0000","last_active":"2023-09-23T19:57:10.429412+0000","last_peered":"2023-09-23T19:57:10.429412+0000","last_clean":"2023-09-23T19:57:10.429412+0000","last_became_active":"2023-09-23T19:18:09.015338+0000","last_became_peered":"2023-09-23T19:18:09.015338+0000","last_unstale":"2023-09-23T19:57:10.429412+0000","last_undegraded":"2023-09-23T19:57:10.429412+0000","last_fullsized":"2023-09-23T19:57:10.429412+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":229,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.335619+0000","last_change":"2023-09-23T19:38:43.881233+0000","last_active":"2023-09-23T19:57:10.335619+0000","last_peered":"2023-09-23T19:57:10.335619+0000","last_clean":"2023-09-23T19:57:10.335619+0000","last_became_active":"2023-09-23T19:38:43.880893+0000","last_became_peered":"2023-09-23T19:38:43.880893+0000","last_unstale":"2023-09-23T19:57:10.335619+0000","last_undegraded":"2023-09-23T19:57:10.335619+0000","last_fullsized":"2023-09-23T19:57:10.335619+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:11.206947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":82,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.928331+0000","last_change":"2023-09-23T19:47:56.790340+0000","last_active":"2023-09-23T19:57:10.928331+0000","last_peered":"2023-09-23T19:57:10.928331+0000","last_clean":"2023-09-23T19:57:10.928331+0000","last_became_active":"2023-09-23T19:47:56.789785+0000","last_became_peered":"2023-09-23T19:47:56.789785+0000","last_unstale":"2023-09-23T19:57:10.928331+0000","last_undegraded":"2023-09-23T19:57:10.928331+0000","last_fullsized":"2023-09-23T19:57:10.928331+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.a","version":"261'1936","reported_seq":1950,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-09-23T19:57:30.353180+0000","last_change":"2023-09-23T19:56:50.486634+0000","last_active":"2023-09-23T19:57:30.353180+0000","last_peered":"2023-09-23T19:57:30.353180+0000","last_clean":"2023-09-23T19:57:30.353180+0000","last_became_active":"2023-09-23T19:56:50.486290+0000","last_became_peered":"2023-09-23T19:56:50.486290+0000","last_unstale":"2023-09-23T19:57:30.353180+0000","last_undegraded":"2023-09-23T19:57:30.353180+0000","last_fullsized":"2023-09-23T19:57:30.353180+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":256,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_clean_scrub_stamp":"2023-09-23T19:56:49.329294+0000","objects_scrubbed":0,"log_size":1936,"log_dups_size":0,"ondisk_log_size":1936,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:27:04.672284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126877696,"num_objects":1936,"num_object_clones":0,"num_object_copies":5808,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1936,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1936,"num_write_kb":123904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":165,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.429686+0000","last_change":"2023-09-23T19:32:49.737323+0000","last_active":"2023-09-23T19:57:10.429686+0000","last_peered":"2023-09-23T19:57:10.429686+0000","last_clean":"2023-09-23T19:57:10.429686+0000","last_became_active":"2023-09-23T19:32:49.736937+0000","last_became_peered":"2023-09-23T19:32:49.736937+0000","last_unstale":"2023-09-23T19:57:10.429686+0000","last_undegraded":"2023-09-23T19:57:10.429686+0000","last_fullsized":"2023-09-23T19:57:10.429686+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":199,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:20.362362+0000","last_change":"2023-09-23T19:22:42.145506+0000","last_active":"2023-09-23T19:57:20.362362+0000","last_peered":"2023-09-23T19:57:20.362362+0000","last_clean":"2023-09-23T19:57:20.362362+0000","last_became_active":"2023-09-23T19:22:42.143991+0000","last_became_peered":"2023-09-23T19:22:42.143991+0000","last_unstale":"2023-09-23T19:57:20.362362+0000","last_undegraded":"2023-09-23T19:57:20.362362+0000","last_fullsized":"2023-09-23T19:57:20.362362+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:56.875175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":202,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.429086+0000","last_change":"2023-09-23T19:32:49.734592+0000","last_active":"2023-09-23T19:57:10.429086+0000","last_peered":"2023-09-23T19:57:10.429086+0000","last_clean":"2023-09-23T19:57:10.429086+0000","last_became_active":"2023-09-23T19:32:49.733905+0000","last_became_peered":"2023-09-23T19:32:49.733905+0000","last_unstale":"2023-09-23T19:57:10.429086+0000","last_undegraded":"2023-09-23T19:57:10.429086+0000","last_fullsized":"2023-09-23T19:57:10.429086+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.b","version":"261'1568","reported_seq":1582,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-09-23T19:57:27.341376+0000","last_change":"2023-09-23T19:56:50.348570+0000","last_active":"2023-09-23T19:57:27.341376+0000","last_peered":"2023-09-23T19:57:27.341376+0000","last_clean":"2023-09-23T19:57:27.341376+0000","last_became_active":"2023-09-23T19:56:50.348227+0000","last_became_peered":"2023-09-23T19:56:50.348227+0000","last_unstale":"2023-09-23T19:57:27.341376+0000","last_undegraded":"2023-09-23T19:57:27.341376+0000","last_fullsized":"2023-09-23T19:57:27.341376+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":256,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_clean_scrub_stamp":"2023-09-23T19:56:49.329294+0000","objects_scrubbed":0,"log_size":1568,"log_dups_size":0,"ondisk_log_size":1568,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:35:58.887241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102760448,"num_objects":1568,"num_object_clones":0,"num_object_copies":4704,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1568,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1568,"num_write_kb":100352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":246,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.928548+0000","last_change":"2023-09-23T19:38:43.895887+0000","last_active":"2023-09-23T19:57:10.928548+0000","last_peered":"2023-09-23T19:57:10.928548+0000","last_clean":"2023-09-23T19:57:10.928548+0000","last_became_active":"2023-09-23T19:38:43.895486+0000","last_became_peered":"2023-09-23T19:38:43.895486+0000","last_unstale":"2023-09-23T19:57:10.928548+0000","last_undegraded":"2023-09-23T19:57:10.928548+0000","last_fullsized":"2023-09-23T19:57:10.928548+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":233,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.922120+0000","last_change":"2023-09-23T19:22:42.149694+0000","last_active":"2023-09-23T19:57:10.922120+0000","last_peered":"2023-09-23T19:57:10.922120+0000","last_clean":"2023-09-23T19:57:10.922120+0000","last_became_active":"2023-09-23T19:22:42.148194+0000","last_became_peered":"2023-09-23T19:22:42.148194+0000","last_unstale":"2023-09-23T19:57:10.922120+0000","last_undegraded":"2023-09-23T19:57:10.922120+0000","last_fullsized":"2023-09-23T19:57:10.922120+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:47.842383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":210,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.344693+0000","last_change":"2023-09-23T19:05:52.138620+0000","last_active":"2023-09-23T19:57:10.344693+0000","last_peered":"2023-09-23T19:57:10.344693+0000","last_clean":"2023-09-23T19:57:10.344693+0000","last_became_active":"2023-09-23T19:05:52.138243+0000","last_became_peered":"2023-09-23T19:05:52.138243+0000","last_unstale":"2023-09-23T19:57:10.344693+0000","last_undegraded":"2023-09-23T19:57:10.344693+0000","last_fullsized":"2023-09-23T19:57:10.344693+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":215,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.921950+0000","last_change":"2023-09-23T19:18:09.008811+0000","last_active":"2023-09-23T19:57:10.921950+0000","last_peered":"2023-09-23T19:57:10.921950+0000","last_clean":"2023-09-23T19:57:10.921950+0000","last_became_active":"2023-09-23T19:18:09.007255+0000","last_became_peered":"2023-09-23T19:18:09.007255+0000","last_unstale":"2023-09-23T19:57:10.921950+0000","last_undegraded":"2023-09-23T19:57:10.921950+0000","last_fullsized":"2023-09-23T19:57:10.921950+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:25.874038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":219,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.921628+0000","last_change":"2023-09-23T19:38:43.877244+0000","last_active":"2023-09-23T19:57:10.921628+0000","last_peered":"2023-09-23T19:57:10.921628+0000","last_clean":"2023-09-23T19:57:10.921628+0000","last_became_active":"2023-09-23T19:38:43.876896+0000","last_became_peered":"2023-09-23T19:38:43.876896+0000","last_unstale":"2023-09-23T19:57:10.921628+0000","last_undegraded":"2023-09-23T19:57:10.921628+0000","last_fullsized":"2023-09-23T19:57:10.921628+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:22.288276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":90,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.345581+0000","last_change":"2023-09-23T19:43:38.620732+0000","last_active":"2023-09-23T19:57:10.345581+0000","last_peered":"2023-09-23T19:57:10.345581+0000","last_clean":"2023-09-23T19:57:10.345581+0000","last_became_active":"2023-09-23T19:43:38.619424+0000","last_became_peered":"2023-09-23T19:43:38.619424+0000","last_unstale":"2023-09-23T19:57:10.345581+0000","last_undegraded":"2023-09-23T19:57:10.345581+0000","last_fullsized":"2023-09-23T19:57:10.345581+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":264,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.345379+0000","last_change":"2023-09-23T19:38:43.947451+0000","last_active":"2023-09-23T19:57:10.345379+0000","last_peered":"2023-09-23T19:57:10.345379+0000","last_clean":"2023-09-23T19:57:10.345379+0000","last_became_active":"2023-09-23T19:38:43.946420+0000","last_became_peered":"2023-09-23T19:38:43.946420+0000","last_unstale":"2023-09-23T19:57:10.345379+0000","last_undegraded":"2023-09-23T19:57:10.345379+0000","last_fullsized":"2023-09-23T19:57:10.345379+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":200,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.929514+0000","last_change":"2023-09-23T19:47:56.793347+0000","last_active":"2023-09-23T19:57:10.929514+0000","last_peered":"2023-09-23T19:57:10.929514+0000","last_clean":"2023-09-23T19:57:10.929514+0000","last_became_active":"2023-09-23T19:47:56.792866+0000","last_became_peered":"2023-09-23T19:47:56.792866+0000","last_unstale":"2023-09-23T19:57:10.929514+0000","last_undegraded":"2023-09-23T19:57:10.929514+0000","last_fullsized":"2023-09-23T19:57:10.929514+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":240,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.345342+0000","last_change":"2023-09-23T19:28:07.896983+0000","last_active":"2023-09-23T19:57:10.345342+0000","last_peered":"2023-09-23T19:57:10.345342+0000","last_clean":"2023-09-23T19:57:10.345342+0000","last_became_active":"2023-09-23T19:28:07.896577+0000","last_became_peered":"2023-09-23T19:28:07.896577+0000","last_unstale":"2023-09-23T19:57:10.345342+0000","last_undegraded":"2023-09-23T19:57:10.345342+0000","last_fullsized":"2023-09-23T19:57:10.345342+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":170,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.929387+0000","last_change":"2023-09-23T19:38:43.900225+0000","last_active":"2023-09-23T19:57:10.929387+0000","last_peered":"2023-09-23T19:57:10.929387+0000","last_clean":"2023-09-23T19:57:10.929387+0000","last_became_active":"2023-09-23T19:38:43.899907+0000","last_became_peered":"2023-09-23T19:38:43.899907+0000","last_unstale":"2023-09-23T19:57:10.929387+0000","last_undegraded":"2023-09-23T19:57:10.929387+0000","last_fullsized":"2023-09-23T19:57:10.929387+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":228,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.921154+0000","last_change":"2023-09-23T19:38:43.882487+0000","last_active":"2023-09-23T19:57:10.921154+0000","last_peered":"2023-09-23T19:57:10.921154+0000","last_clean":"2023-09-23T19:57:10.921154+0000","last_became_active":"2023-09-23T19:38:43.881931+0000","last_became_peered":"2023-09-23T19:38:43.881931+0000","last_unstale":"2023-09-23T19:57:10.921154+0000","last_undegraded":"2023-09-23T19:57:10.921154+0000","last_fullsized":"2023-09-23T19:57:10.921154+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:02:36.906535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":248,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.922139+0000","last_change":"2023-09-23T19:38:43.893983+0000","last_active":"2023-09-23T19:57:10.922139+0000","last_peered":"2023-09-23T19:57:10.922139+0000","last_clean":"2023-09-23T19:57:10.922139+0000","last_became_active":"2023-09-23T19:38:43.893527+0000","last_became_peered":"2023-09-23T19:38:43.893527+0000","last_unstale":"2023-09-23T19:57:10.922139+0000","last_undegraded":"2023-09-23T19:57:10.922139+0000","last_fullsized":"2023-09-23T19:57:10.922139+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:12.156531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":219,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.344603+0000","last_change":"2023-09-23T19:06:36.673002+0000","last_active":"2023-09-23T19:57:10.344603+0000","last_peered":"2023-09-23T19:57:10.344603+0000","last_clean":"2023-09-23T19:57:10.344603+0000","last_became_active":"2023-09-23T19:06:36.671911+0000","last_became_peered":"2023-09-23T19:06:36.671911+0000","last_unstale":"2023-09-23T19:57:10.344603+0000","last_undegraded":"2023-09-23T19:57:10.344603+0000","last_fullsized":"2023-09-23T19:57:10.344603+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":16,"stat_sum":{"num_bytes":1877803008,"num_objects":28653,"num_object_clones":0,"num_object_copies":85959,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28653,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":28653,"num_write_kb":1833792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5487001600,"data_stored":5487001600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":28653,"ondisk_log_size":28653,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430864,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1245260,"kb_used_data":357572,"kb_used_omap":0,"kb_used_meta":887680,"kb_avail":92499892,"statfs":{"total":95995035648,"available":94719889408,"internally_reserved":0,"allocated":366153728,"data_stored":365607055,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":908984320},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:57:20 2023","interfaces":[{"interface":"back","average":{"1min":3.4049999999999998,"5min":1.8069999999999999,"15min":1.4890000000000001},"min":{"1min":0.34799999999999998,"5min":0.26700000000000002,"15min":0.26400000000000001},"max":{"1min":41.570999999999998,"5min":41.570999999999998,"15min":41.570999999999998},"last":0.27900000000000003},{"interface":"front","average":{"1min":3.665,"5min":2.0899999999999999,"15min":1.6839999999999999},"min":{"1min":0.44500000000000001,"5min":0.28000000000000003,"15min":0.25600000000000001},"max":{"1min":45.634,"5min":45.634,"15min":45.634},"last":2.754}]},{"osd":1,"last update":"Sat Sep 23 19:56:38 2023","interfaces":[{"interface":"back","average":{"1min":7.8600000000000003,"5min":3.0569999999999999,"15min":2.4430000000000001},"min":{"1min":0.33100000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":149.018,"5min":149.018,"15min":149.018},"last":3.407},{"interface":"front","average":{"1min":7.6639999999999997,"5min":3.056,"15min":2.419},"min":{"1min":0.35499999999999998,"5min":0.254,"15min":0.254},"max":{"1min":148.995,"5min":148.995,"15min":148.995},"last":1.391}]},{"osd":2,"last update":"Sat Sep 23 19:57:20 2023","interfaces":[{"interface":"back","average":{"1min":3.6459999999999999,"5min":2.7440000000000002,"15min":3.6320000000000001},"min":{"1min":0.42299999999999999,"5min":0.31900000000000001,"15min":0.28100000000000003},"max":{"1min":44.143999999999998,"5min":53.521000000000001,"15min":246.982},"last":0.39000000000000001},{"interface":"front","average":{"1min":3.5470000000000002,"5min":2.9980000000000002,"15min":3.6429999999999998},"min":{"1min":0.58599999999999997,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":41.790999999999997,"5min":53.466000000000001,"15min":247.751},"last":0.54000000000000004}]},{"osd":3,"last update":"Sat Sep 23 19:57:10 2023","interfaces":[{"interface":"back","average":{"1min":0.92600000000000005,"5min":1.831,"15min":1.7410000000000001},"min":{"1min":0.69699999999999995,"5min":0.33000000000000002,"15min":0.312},"max":{"1min":1.173,"5min":10.615,"15min":10.615},"last":0.87},{"interface":"front","average":{"1min":1.079,"5min":1.708,"15min":1.6899999999999999},"min":{"1min":0.79800000000000004,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":1.3999999999999999,"5min":9.9179999999999993,"15min":9.9179999999999993},"last":0.82799999999999996}]},{"osd":4,"last update":"Sat Sep 23 19:56:38 2023","interfaces":[{"interface":"back","average":{"1min":2.855,"5min":2.5830000000000002,"15min":1.6080000000000001},"min":{"1min":0.53600000000000003,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":13.484999999999999,"5min":13.484999999999999,"15min":13.484999999999999},"last":0.505},{"interface":"front","average":{"1min":1.734,"5min":2.2450000000000001,"15min":1.5549999999999999},"min":{"1min":0.41799999999999998,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":13.116,"5min":13.116,"15min":13.116},"last":0.46600000000000003}]},{"osd":5,"last update":"Sat Sep 23 19:56:38 2023","interfaces":[{"interface":"back","average":{"1min":1.927,"5min":2.1579999999999999,"15min":1.5029999999999999},"min":{"1min":0.65000000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":13.321999999999999,"5min":13.321999999999999,"15min":13.321999999999999},"last":0.56499999999999995},{"interface":"front","average":{"1min":1.327,"5min":2.3250000000000002,"15min":1.5780000000000001},"min":{"1min":0.46700000000000003,"5min":0.315,"15min":0.315},"max":{"1min":4.7050000000000001,"5min":13.478999999999999,"15min":13.478999999999999},"last":3.3540000000000001}]},{"osd":6,"last update":"Sat Sep 23 19:56:38 2023","interfaces":[{"interface":"back","average":{"1min":1.7849999999999999,"5min":2.0169999999999999,"15min":1.6739999999999999},"min":{"1min":0.38,"5min":0.34300000000000003,"15min":0.33300000000000002},"max":{"1min":13.436,"5min":13.436,"15min":13.436},"last":0.53400000000000003},{"interface":"front","average":{"1min":1.1619999999999999,"5min":1.7849999999999999,"15min":1.657},"min":{"1min":0.55900000000000005,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":2.851,"5min":10.766999999999999,"15min":10.766999999999999},"last":0.49399999999999999}]}]},{"osd":6,"up_from":42,"seq":180388627096,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1360444,"kb_used_data":465140,"kb_used_omap":0,"kb_used_meta":895296,"kb_avail":92384708,"statfs":{"total":95995035648,"available":94601940992,"internally_reserved":0,"allocated":476303360,"data_stored":475772610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":916783104},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:57:26 2023","interfaces":[{"interface":"back","average":{"1min":0.88300000000000001,"5min":1.5189999999999999,"15min":1.7749999999999999},"min":{"1min":0.34799999999999998,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":1.393,"5min":9.5700000000000003,"15min":45.600000000000001},"last":0.34799999999999998},{"interface":"front","average":{"1min":1.0509999999999999,"5min":1.581,"15min":1.7769999999999999},"min":{"1min":0.29799999999999999,"5min":0.29599999999999999,"15min":0.25800000000000001},"max":{"1min":3.9990000000000001,"5min":11.340999999999999,"15min":45.347999999999999},"last":0.42799999999999999}]},{"osd":1,"last update":"Sat Sep 23 19:56:51 2023","interfaces":[{"interface":"back","average":{"1min":0.99099999999999999,"5min":1.659,"15min":2.6360000000000001},"min":{"1min":0.32700000000000001,"5min":0.30599999999999999,"15min":0.28399999999999997},"max":{"1min":4.359,"5min":9.4909999999999997,"15min":117.58199999999999},"last":0.40100000000000002},{"interface":"front","average":{"1min":1.0089999999999999,"5min":1.7070000000000001,"15min":2.6960000000000002},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.29199999999999998},"max":{"1min":2.1850000000000001,"5min":17.318999999999999,"15min":117.633},"last":0.38200000000000001}]},{"osd":2,"last update":"Sat Sep 23 19:57:09 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":4.3109999999999999,"15min":4.7119999999999997},"min":{"1min":0.76300000000000001,"5min":0.30499999999999999,"15min":0.30399999999999999},"max":{"1min":1.4330000000000001,"5min":219.453,"15min":219.453},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.98399999999999999,"5min":4.2460000000000004,"15min":4.6299999999999999},"min":{"1min":0.64600000000000002,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":1.3600000000000001,"5min":219.369,"15min":219.369},"last":0.77900000000000003}]},{"osd":3,"last update":"Sat Sep 23 19:57:07 2023","interfaces":[{"interface":"back","average":{"1min":0.91700000000000004,"5min":1.4099999999999999,"15min":1.7949999999999999},"min":{"1min":0.60599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.2609999999999999,"5min":9.1419999999999995,"15min":16.428000000000001},"last":0.877},{"interface":"front","average":{"1min":1.0620000000000001,"5min":1.772,"15min":1.806},"min":{"1min":0.79100000000000004,"5min":0.33100000000000002,"15min":0.307},"max":{"1min":1.5820000000000001,"5min":9.5329999999999995,"15min":18.917000000000002},"last":0.47899999999999998}]},{"osd":4,"last update":"Sat Sep 23 19:57:07 2023","interfaces":[{"interface":"back","average":{"1min":1.01,"5min":2.5459999999999998,"15min":1.6779999999999999},"min":{"1min":0.53800000000000003,"5min":0.33900000000000002,"15min":0.28999999999999998},"max":{"1min":1.379,"5min":11.912000000000001,"15min":14.401999999999999},"last":1.593},{"interface":"front","average":{"1min":1.014,"5min":2.2770000000000001,"15min":1.8220000000000001},"min":{"1min":0.69999999999999996,"5min":0.32200000000000001,"15min":0.28299999999999997},"max":{"1min":1.5269999999999999,"5min":14.239000000000001,"15min":14.603},"last":1.2370000000000001}]},{"osd":5,"last update":"Sat Sep 23 19:57:07 2023","interfaces":[{"interface":"back","average":{"1min":1.1859999999999999,"5min":2.706,"15min":1.833},"min":{"1min":0.82599999999999996,"5min":0.40699999999999997,"15min":0.32000000000000001},"max":{"1min":1.411,"5min":15.045,"15min":15.045},"last":1.637},{"interface":"front","average":{"1min":1.0980000000000001,"5min":2.839,"15min":1.944},"min":{"1min":0.66900000000000004,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":1.383,"5min":16.244,"15min":16.244},"last":0.99099999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:56:31 2023","interfaces":[{"interface":"back","average":{"1min":2.8820000000000001,"5min":2.48,"15min":1.855},"min":{"1min":0.60399999999999998,"5min":0.34899999999999998,"15min":0.33200000000000002},"max":{"1min":11.709,"5min":11.709,"15min":11.709},"last":0.73999999999999999},{"interface":"front","average":{"1min":1.6699999999999999,"5min":2,"15min":1.962},"min":{"1min":0.442,"5min":0.39500000000000002,"15min":0.32200000000000001},"max":{"1min":9.2469999999999999,"5min":9.7599999999999998,"15min":12.327999999999999},"last":0.93500000000000005}]}]},{"osd":1,"up_from":98,"seq":420906795428,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1725060,"kb_used_data":932476,"kb_used_omap":4,"kb_used_meta":792571,"kb_avail":92020092,"statfs":{"total":95995035648,"available":94228574208,"internally_reserved":0,"allocated":954855424,"data_stored":953788400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":811593614},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:56:49 2023","interfaces":[{"interface":"back","average":{"1min":0.877,"5min":2.1859999999999999,"15min":2.3519999999999999},"min":{"1min":0.26300000000000001,"5min":0.221,"15min":0.18099999999999999},"max":{"1min":5.9400000000000004,"5min":134.37700000000001,"15min":134.37700000000001},"last":4.2279999999999998},{"interface":"front","average":{"1min":0.61699999999999999,"5min":2.2469999999999999,"15min":2.274},"min":{"1min":0.33800000000000002,"5min":0.23400000000000001,"15min":0.22600000000000001},"max":{"1min":0.997,"5min":135.79400000000001,"15min":135.79400000000001},"last":3.661}]},{"osd":2,"last update":"Sat Sep 23 19:57:15 2023","interfaces":[{"interface":"back","average":{"1min":0.70799999999999996,"5min":2.0089999999999999,"15min":2.7480000000000002},"min":{"1min":0.46300000000000002,"5min":0.27300000000000002,"15min":0.26800000000000002},"max":{"1min":1.1479999999999999,"5min":48.512,"15min":72.859999999999999},"last":3.629},{"interface":"front","average":{"1min":0.79400000000000004,"5min":1.897,"15min":2.3500000000000001},"min":{"1min":0.42699999999999999,"5min":0.224,"15min":0.224},"max":{"1min":2.4489999999999998,"5min":48.664000000000001,"15min":73.665999999999997},"last":0.53800000000000003}]},{"osd":3,"last update":"Sat Sep 23 19:57:11 2023","interfaces":[{"interface":"back","average":{"1min":0.71099999999999997,"5min":1.365,"15min":1.794},"min":{"1min":0.40400000000000003,"5min":0.26200000000000001,"15min":0.25},"max":{"1min":1.1639999999999999,"5min":20.806999999999999,"15min":20.806999999999999},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.69399999999999995,"5min":1.4850000000000001,"15min":1.8280000000000001},"min":{"1min":0.48499999999999999,"5min":0.25900000000000001,"15min":0.217},"max":{"1min":0.96799999999999997,"5min":19.201000000000001,"15min":19.201000000000001},"last":1.5369999999999999}]},{"osd":4,"last update":"Sat Sep 23 19:56:49 2023","interfaces":[{"interface":"back","average":{"1min":1.159,"5min":2.54,"15min":1.887},"min":{"1min":0.53300000000000003,"5min":0.307,"15min":0.307},"max":{"1min":5.3899999999999997,"5min":48.381,"15min":48.381},"last":3.4390000000000001},{"interface":"front","average":{"1min":1.145,"5min":1.907,"15min":1.879},"min":{"1min":0.69299999999999995,"5min":0.35099999999999998,"15min":0.32200000000000001},"max":{"1min":3.2410000000000001,"5min":9.8219999999999992,"15min":82.239999999999995},"last":1.597}]},{"osd":5,"last update":"Sat Sep 23 19:56:49 2023","interfaces":[{"interface":"back","average":{"1min":1.0049999999999999,"5min":2.278,"15min":1.8069999999999999},"min":{"1min":0.46999999999999997,"5min":0.38200000000000001,"15min":0.34899999999999998},"max":{"1min":2.504,"5min":48.643000000000001,"15min":48.643000000000001},"last":1.504},{"interface":"front","average":{"1min":1.3560000000000001,"5min":1.9299999999999999,"15min":1.772},"min":{"1min":0.64500000000000002,"5min":0.36699999999999999,"15min":0.35799999999999998},"max":{"1min":4.7699999999999996,"5min":48.619999999999997,"15min":48.619999999999997},"last":4.0990000000000002}]},{"osd":6,"last update":"Sat Sep 23 19:56:49 2023","interfaces":[{"interface":"back","average":{"1min":1.258,"5min":2.2599999999999998,"15min":2.198},"min":{"1min":0.76800000000000002,"5min":0.442,"15min":0.318},"max":{"1min":3.3719999999999999,"5min":48.548999999999999,"15min":48.548999999999999},"last":4.149},{"interface":"front","average":{"1min":1.2430000000000001,"5min":1.887,"15min":1.9470000000000001},"min":{"1min":0.67600000000000005,"5min":0.313,"15min":0.313},"max":{"1min":3.3849999999999998,"5min":7.9139999999999997,"15min":11.676},"last":3.4889999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:56:49 2023","interfaces":[{"interface":"back","average":{"1min":1.0960000000000001,"5min":2.4289999999999998,"15min":2.0680000000000001},"min":{"1min":0.81200000000000006,"5min":0.44400000000000001,"15min":0.36799999999999999},"max":{"1min":1.7250000000000001,"5min":48.329999999999998,"15min":48.329999999999998},"last":3.5230000000000001},{"interface":"front","average":{"1min":1.0509999999999999,"5min":2.492,"15min":2.0510000000000002},"min":{"1min":0.74199999999999999,"5min":0.45400000000000001,"15min":0.35299999999999998},"max":{"1min":1.321,"5min":48.579000000000001,"15min":48.579000000000001},"last":1.651}]}]},{"osd":0,"up_from":85,"seq":365072220634,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1710896,"kb_used_data":821032,"kb_used_omap":5,"kb_used_meta":889850,"kb_avail":92034256,"statfs":{"total":95995035648,"available":94243078144,"internally_reserved":0,"allocated":840736768,"data_stored":839691224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":911207034},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 19:56:50 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":3.4929999999999999,"15min":2.5720000000000001},"min":{"1min":0.307,"5min":0.219,"15min":0.216},"max":{"1min":0.89000000000000001,"5min":149.37299999999999,"15min":149.37299999999999},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.63900000000000001,"5min":3.6669999999999998,"15min":2.673},"min":{"1min":0.28100000000000003,"5min":0.26100000000000001,"15min":0.189},"max":{"1min":1.823,"5min":147.119,"15min":147.119},"last":0.36499999999999999}]},{"osd":2,"last update":"Sat Sep 23 19:57:18 2023","interfaces":[{"interface":"back","average":{"1min":0.70299999999999996,"5min":1.2270000000000001,"15min":3.468},"min":{"1min":0.35399999999999998,"5min":0.25600000000000001,"15min":0.21299999999999999},"max":{"1min":1.0429999999999999,"5min":9.0830000000000002,"15min":262.59500000000003},"last":3.3050000000000002},{"interface":"front","average":{"1min":0.84299999999999997,"5min":1.365,"15min":3.327},"min":{"1min":0.378,"5min":0.26300000000000001,"15min":0.24199999999999999},"max":{"1min":4.4160000000000004,"5min":8.9469999999999992,"15min":263.25700000000001},"last":0.41599999999999998}]},{"osd":3,"last update":"Sat Sep 23 19:56:50 2023","interfaces":[{"interface":"back","average":{"1min":0.92900000000000005,"5min":1.339,"15min":1.71},"min":{"1min":0.49199999999999999,"5min":0.33300000000000002,"15min":0.222},"max":{"1min":4.101,"5min":14.983000000000001,"15min":14.983000000000001},"last":2.0190000000000001},{"interface":"front","average":{"1min":0.75800000000000001,"5min":1.163,"15min":1.675},"min":{"1min":0.36499999999999999,"5min":0.33700000000000002,"15min":0.223},"max":{"1min":1.085,"5min":14.843,"15min":14.843},"last":1.986}]},{"osd":4,"last update":"Sat Sep 23 19:56:28 2023","interfaces":[{"interface":"back","average":{"1min":1.4179999999999999,"5min":1.4339999999999999,"15min":1.4299999999999999},"min":{"1min":0.433,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":4.0549999999999997,"5min":6.0469999999999997,"15min":9.0679999999999996},"last":0.69299999999999995},{"interface":"front","average":{"1min":0.95199999999999996,"5min":1.407,"15min":1.4950000000000001},"min":{"1min":0.35999999999999999,"5min":0.34499999999999997,"15min":0.315},"max":{"1min":2.5840000000000001,"5min":7.056,"15min":8.6289999999999996},"last":1.173}]},{"osd":5,"last update":"Sat Sep 23 19:56:28 2023","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":1.5089999999999999,"15min":1.542},"min":{"1min":0.41699999999999998,"5min":0.36099999999999999,"15min":0.33700000000000002},"max":{"1min":2.8740000000000001,"5min":8.5619999999999994,"15min":8.5619999999999994},"last":0.84999999999999998},{"interface":"front","average":{"1min":1.3700000000000001,"5min":1.573,"15min":1.5389999999999999},"min":{"1min":0.40000000000000002,"5min":0.38700000000000001,"15min":0.34100000000000003},"max":{"1min":4.1399999999999997,"5min":8.7910000000000004,"15min":9.0329999999999995},"last":0.78200000000000003}]},{"osd":6,"last update":"Sat Sep 23 19:56:28 2023","interfaces":[{"interface":"back","average":{"1min":1.4059999999999999,"5min":1.5640000000000001,"15min":1.794},"min":{"1min":0.45700000000000002,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":4.2119999999999997,"5min":6.9020000000000001,"15min":66.170000000000002},"last":0.72899999999999998},{"interface":"front","average":{"1min":1.284,"5min":1.627,"15min":1.917},"min":{"1min":0.55300000000000005,"5min":0.433,"15min":0.35299999999999998},"max":{"1min":3.4649999999999999,"5min":6.9189999999999996,"15min":66.260999999999996},"last":0.81799999999999995}]},{"osd":7,"last update":"Sat Sep 23 19:56:28 2023","interfaces":[{"interface":"back","average":{"1min":1.6479999999999999,"5min":3.5059999999999998,"15min":2.3719999999999999},"min":{"1min":0.49299999999999999,"5min":0.42699999999999999,"15min":0.38200000000000001},"max":{"1min":9.3130000000000006,"5min":210.47399999999999,"15min":210.47399999999999},"last":1.8380000000000001},{"interface":"front","average":{"1min":1.47,"5min":3.5870000000000002,"15min":2.3450000000000002},"min":{"1min":0.497,"5min":0.42099999999999999,"15min":0.40100000000000002},"max":{"1min":4.1790000000000003,"5min":210.52199999999999,"15min":210.52199999999999},"last":0.755}]}]},{"osd":2,"up_from":159,"seq":682899800290,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1303188,"kb_used_data":672396,"kb_used_omap":7,"kb_used_meta":630776,"kb_avail":92441964,"statfs":{"total":95995035648,"available":94660571136,"internally_reserved":0,"allocated":688533504,"data_stored":687508144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":645914705},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,3],"upper_bound":4},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:56:29 2023","interfaces":[{"interface":"back","average":{"1min":0.745,"5min":1.9490000000000001,"15min":2.2210000000000001},"min":{"1min":0.27800000000000002,"5min":0.22,"15min":0.17299999999999999},"max":{"1min":2.879,"5min":61.442999999999998,"15min":117.77500000000001},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.85899999999999999,"5min":1.746,"15min":2.5329999999999999},"min":{"1min":0.35499999999999998,"5min":0.23200000000000001,"15min":0.214},"max":{"1min":3.4740000000000002,"5min":61.374000000000002,"15min":117.91800000000001},"last":0.92500000000000004}]},{"osd":1,"last update":"Sat Sep 23 19:56:29 2023","interfaces":[{"interface":"back","average":{"1min":6.9740000000000002,"5min":2.968,"15min":3.105},"min":{"1min":0.42799999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":84.234999999999999,"5min":84.234999999999999,"15min":114.107},"last":1.71},{"interface":"front","average":{"1min":5.9489999999999998,"5min":2.6429999999999998,"15min":2.7040000000000002},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.22},"max":{"1min":84.150000000000006,"5min":84.150000000000006,"15min":117.172},"last":0.36399999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:56:29 2023","interfaces":[{"interface":"back","average":{"1min":1.831,"5min":1.7150000000000001,"15min":2.0760000000000001},"min":{"1min":0.42499999999999999,"5min":0.26300000000000001,"15min":0.24099999999999999},"max":{"1min":10.587999999999999,"5min":17.751000000000001,"15min":17.751000000000001},"last":0.25800000000000001},{"interface":"front","average":{"1min":1.5329999999999999,"5min":1.895,"15min":1.9990000000000001},"min":{"1min":0.38400000000000001,"5min":0.36399999999999999,"15min":0.21299999999999999},"max":{"1min":6.867,"5min":19.596,"15min":20.286999999999999},"last":0.66900000000000004}]},{"osd":4,"last update":"Sat Sep 23 19:56:29 2023","interfaces":[{"interface":"back","average":{"1min":1.748,"5min":2.0630000000000002,"15min":2.0139999999999998},"min":{"1min":0.35099999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":7.4829999999999997,"5min":14.194000000000001,"15min":18.318999999999999},"last":0.98899999999999999},{"interface":"front","average":{"1min":2.2200000000000002,"5min":2.5379999999999998,"15min":2.0699999999999998},"min":{"1min":0.68700000000000006,"5min":0.38700000000000001,"15min":0.31900000000000001},"max":{"1min":6.6420000000000003,"5min":14.271000000000001,"15min":14.271000000000001},"last":0.34200000000000003}]},{"osd":5,"last update":"Sat Sep 23 19:56:29 2023","interfaces":[{"interface":"back","average":{"1min":1.6859999999999999,"5min":2.2810000000000001,"15min":1.724},"min":{"1min":0.44500000000000001,"5min":0.41199999999999998,"15min":0.27600000000000002},"max":{"1min":7.5990000000000002,"5min":9.6110000000000007,"15min":16.577000000000002},"last":0.39500000000000002},{"interface":"front","average":{"1min":1.2689999999999999,"5min":1.3959999999999999,"15min":1.754},"min":{"1min":0.438,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":4.6890000000000001,"5min":7.2939999999999996,"15min":10.193},"last":0.876}]},{"osd":6,"last update":"Sat Sep 23 19:56:29 2023","interfaces":[{"interface":"back","average":{"1min":1.4630000000000001,"5min":1.857,"15min":2.0790000000000002},"min":{"1min":0.53500000000000003,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":4.3639999999999999,"5min":9.8870000000000005,"15min":10.166},"last":0.95899999999999996},{"interface":"front","average":{"1min":1.804,"5min":2.3119999999999998,"15min":2.3330000000000002},"min":{"1min":0.69899999999999995,"5min":0.40799999999999997,"15min":0.38},"max":{"1min":5.2469999999999999,"5min":11.696,"15min":12.297000000000001},"last":0.38300000000000001}]},{"osd":7,"last update":"Sat Sep 23 19:56:29 2023","interfaces":[{"interface":"back","average":{"1min":1.1859999999999999,"5min":1.921,"15min":1.99},"min":{"1min":0.53700000000000003,"5min":0.48299999999999998,"15min":0.38500000000000001},"max":{"1min":3.0920000000000001,"5min":9.4879999999999995,"15min":32.945},"last":0.42299999999999999},{"interface":"front","average":{"1min":2.2200000000000002,"5min":2.105,"15min":2.1779999999999999},"min":{"1min":0.69499999999999995,"5min":0.499,"15min":0.44700000000000001},"max":{"1min":7.0330000000000004,"5min":7.0330000000000004,"15min":16.593},"last":0.41499999999999998}]}]},{"osd":3,"up_from":116,"seq":498216206690,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1358152,"kb_used_data":680384,"kb_used_omap":2,"kb_used_meta":677757,"kb_avail":92387000,"statfs":{"total":95995035648,"available":94604288000,"internally_reserved":0,"allocated":696713216,"data_stored":695757039,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":694023221},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,2,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:57:07 2023","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":0.97499999999999998,"15min":1.851},"min":{"1min":0.42999999999999999,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":1.03,"5min":6.6669999999999998,"15min":50.771999999999998},"last":0.99399999999999999},{"interface":"front","average":{"1min":0.66900000000000004,"5min":0.86399999999999999,"15min":1.996},"min":{"1min":0.34799999999999998,"5min":0.24399999999999999,"15min":0.20100000000000001},"max":{"1min":1.347,"5min":5.9820000000000002,"15min":48.887},"last":0.65900000000000003}]},{"osd":1,"last update":"Sat Sep 23 19:57:07 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":1.119,"15min":2.1320000000000001},"min":{"1min":0.42199999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":1.2130000000000001,"5min":12.542,"15min":92.171000000000006},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.69799999999999995,"5min":1.085,"15min":2.2040000000000002},"min":{"1min":0.47299999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":1.0940000000000001,"5min":6.617,"15min":87.771000000000001},"last":0.90900000000000003}]},{"osd":2,"last update":"Sat Sep 23 19:57:21 2023","interfaces":[{"interface":"back","average":{"1min":0.79500000000000004,"5min":1.105,"15min":2.6389999999999998},"min":{"1min":0.46100000000000002,"5min":0.307,"15min":0.26200000000000001},"max":{"1min":1.6060000000000001,"5min":9.0709999999999997,"15min":116.363},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.66800000000000004,"5min":1.264,"15min":2.5920000000000001},"min":{"1min":0.312,"5min":0.30499999999999999,"15min":0.252},"max":{"1min":0.95499999999999996,"5min":5.7359999999999998,"15min":118.474},"last":3.3199999999999998}]},{"osd":4,"last update":"Sat Sep 23 19:57:07 2023","interfaces":[{"interface":"back","average":{"1min":1.0409999999999999,"5min":1.716,"15min":1.6719999999999999},"min":{"1min":0.66600000000000004,"5min":0.309,"15min":0.309},"max":{"1min":1.7649999999999999,"5min":9.2240000000000002,"15min":11.666},"last":1.0649999999999999},{"interface":"front","average":{"1min":1.077,"5min":1.635,"15min":1.728},"min":{"1min":0.74199999999999999,"5min":0.36199999999999999,"15min":0.33000000000000002},"max":{"1min":1.796,"5min":7.8339999999999996,"15min":9.9990000000000006},"last":0.54400000000000004}]},{"osd":5,"last update":"Sat Sep 23 19:57:07 2023","interfaces":[{"interface":"back","average":{"1min":1.0229999999999999,"5min":1.5980000000000001,"15min":1.708},"min":{"1min":0.76400000000000001,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":1.633,"5min":7.7960000000000003,"15min":10.022},"last":0.50800000000000001},{"interface":"front","average":{"1min":1.052,"5min":1.4530000000000001,"15min":1.726},"min":{"1min":0.68799999999999994,"5min":0.39700000000000002,"15min":0.34000000000000002},"max":{"1min":1.9390000000000001,"5min":10.026,"15min":13.146000000000001},"last":0.42199999999999999}]},{"osd":6,"last update":"Sat Sep 23 19:57:07 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":1.655,"15min":2.1499999999999999},"min":{"1min":0.69799999999999995,"5min":0.41199999999999998,"15min":0.39100000000000001},"max":{"1min":1.718,"5min":8.6059999999999999,"15min":15.919},"last":0.70099999999999996},{"interface":"front","average":{"1min":1.008,"5min":1.6519999999999999,"15min":1.994},"min":{"1min":0.66800000000000004,"5min":0.38900000000000001,"15min":0.36399999999999999},"max":{"1min":1.593,"5min":7.048,"15min":11.456},"last":0.58499999999999996}]},{"osd":7,"last update":"Sat Sep 23 19:57:07 2023","interfaces":[{"interface":"back","average":{"1min":1.1319999999999999,"5min":2.077,"15min":2.153},"min":{"1min":0.70899999999999996,"5min":0.378,"15min":0.378},"max":{"1min":2.8079999999999998,"5min":11.505000000000001,"15min":14.098000000000001},"last":0.63400000000000001},{"interface":"front","average":{"1min":1.139,"5min":1.6890000000000001,"15min":1.9390000000000001},"min":{"1min":0.76300000000000001,"5min":0.41299999999999998,"15min":0.40600000000000003},"max":{"1min":1.6060000000000001,"5min":8.4199999999999999,"15min":11.744999999999999},"last":1.0309999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019561,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1601796,"kb_used_data":701884,"kb_used_omap":0,"kb_used_meta":899904,"kb_avail":92143356,"statfs":{"total":95995035648,"available":94354796544,"internally_reserved":0,"allocated":718729216,"data_stored":718190121,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":921501696},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:56:33 2023","interfaces":[{"interface":"back","average":{"1min":0.99299999999999999,"5min":1.617,"15min":1.8879999999999999},"min":{"1min":0.27700000000000002,"5min":0.247,"15min":0.247},"max":{"1min":3.9620000000000002,"5min":6.641,"15min":150.155},"last":1.4370000000000001},{"interface":"front","average":{"1min":1.071,"5min":1.2869999999999999,"15min":1.6910000000000001},"min":{"1min":0.40000000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":3.6659999999999999,"5min":9.6240000000000006,"15min":149.47900000000001},"last":1.4670000000000001}]},{"osd":1,"last update":"Sat Sep 23 19:56:58 2023","interfaces":[{"interface":"back","average":{"1min":0.92000000000000004,"5min":1.575,"15min":1.8600000000000001},"min":{"1min":0.47699999999999998,"5min":0.29199999999999998,"15min":0.27800000000000002},"max":{"1min":1.4419999999999999,"5min":14.029,"15min":101.786},"last":1.5289999999999999},{"interface":"front","average":{"1min":1.149,"5min":1.74,"15min":2.0379999999999998},"min":{"1min":0.63200000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":4.7729999999999997,"5min":14.090999999999999,"15min":101.726},"last":1.4199999999999999}]},{"osd":2,"last update":"Sat Sep 23 19:57:19 2023","interfaces":[{"interface":"back","average":{"1min":0.86099999999999999,"5min":1.7829999999999999,"15min":3.8300000000000001},"min":{"1min":0.56200000000000006,"5min":0.30399999999999999,"15min":0.27600000000000002},"max":{"1min":1.841,"5min":7.4690000000000003,"15min":219.02799999999999},"last":1.371},{"interface":"front","average":{"1min":1.083,"5min":1.718,"15min":3.7789999999999999},"min":{"1min":0.69899999999999995,"5min":0.308,"15min":0.29899999999999999},"max":{"1min":2.5899999999999999,"5min":9.6780000000000008,"15min":219.65600000000001},"last":2.9780000000000002}]},{"osd":3,"last update":"Sat Sep 23 19:57:19 2023","interfaces":[{"interface":"back","average":{"1min":0.88800000000000001,"5min":1.694,"15min":1.8},"min":{"1min":0.45900000000000002,"5min":0.308,"15min":0.308},"max":{"1min":1.25,"5min":9.4399999999999995,"15min":27.789000000000001},"last":2.694},{"interface":"front","average":{"1min":0.98899999999999999,"5min":1.6890000000000001,"15min":1.6659999999999999},"min":{"1min":0.54300000000000004,"5min":0.33400000000000002,"15min":0.32300000000000001},"max":{"1min":1.853,"5min":7.2220000000000004,"15min":27.808},"last":1.494}]},{"osd":5,"last update":"Sat Sep 23 19:56:33 2023","interfaces":[{"interface":"back","average":{"1min":1.8759999999999999,"5min":2.1720000000000002,"15min":1.5029999999999999},"min":{"1min":0.38300000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.8040000000000003,"5min":9.4740000000000002,"15min":22.442},"last":1.6519999999999999},{"interface":"front","average":{"1min":1.732,"5min":2.399,"15min":1.5329999999999999},"min":{"1min":0.36099999999999999,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":5.2450000000000001,"5min":12.506,"15min":12.506},"last":1.6990000000000001}]},{"osd":6,"last update":"Sat Sep 23 19:56:58 2023","interfaces":[{"interface":"back","average":{"1min":1.242,"5min":2.2120000000000002,"15min":1.6839999999999999},"min":{"1min":0.69699999999999995,"5min":0.35299999999999998,"15min":0.29399999999999998},"max":{"1min":4.9619999999999997,"5min":13.066000000000001,"15min":22.544},"last":1.714},{"interface":"front","average":{"1min":1.2310000000000001,"5min":2.3780000000000001,"15min":1.6319999999999999},"min":{"1min":0.61799999999999999,"5min":0.35499999999999998,"15min":0.31},"max":{"1min":4.4749999999999996,"5min":12.704000000000001,"15min":19.706},"last":1.7370000000000001}]},{"osd":7,"last update":"Sat Sep 23 19:56:58 2023","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":1.9390000000000001,"15min":2.6970000000000001},"min":{"1min":0.63600000000000001,"5min":0.308,"15min":0.308},"max":{"1min":1.5049999999999999,"5min":10.737,"15min":294.76499999999999},"last":1.399},{"interface":"front","average":{"1min":1.2989999999999999,"5min":2.298,"15min":2.9079999999999999},"min":{"1min":0.74399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":4.8239999999999998,"5min":12.169,"15min":294.47699999999998},"last":1.675}]}]},{"osd":5,"up_from":36,"seq":154618823328,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1782508,"kb_used_data":591332,"kb_used_omap":0,"kb_used_meta":1191168,"kb_avail":91962644,"statfs":{"total":95995035648,"available":94169747456,"internally_reserved":0,"allocated":605523968,"data_stored":605004563,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1219756032},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:57:21 2023","interfaces":[{"interface":"back","average":{"1min":0.88500000000000001,"5min":1.3320000000000001,"15min":1.409},"min":{"1min":0.45300000000000001,"5min":0.28399999999999997,"15min":0.253},"max":{"1min":1.216,"5min":9.6720000000000006,"15min":28.280000000000001},"last":0.52000000000000002},{"interface":"front","average":{"1min":1.0720000000000001,"5min":1.9370000000000001,"15min":1.7889999999999999},"min":{"1min":0.61099999999999999,"5min":0.251,"15min":0.251},"max":{"1min":2.8420000000000001,"5min":12.016,"15min":28.309999999999999},"last":0.68999999999999995}]},{"osd":1,"last update":"Sat Sep 23 19:56:54 2023","interfaces":[{"interface":"back","average":{"1min":1.224,"5min":3.7919999999999998,"15min":2.1840000000000002},"min":{"1min":0.56599999999999995,"5min":0.28899999999999998,"15min":0.28000000000000003},"max":{"1min":3.7959999999999998,"5min":143.51499999999999,"15min":143.51499999999999},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.84499999999999997,"5min":4.2869999999999999,"15min":2.492},"min":{"1min":0.34499999999999997,"5min":0.28999999999999998,"15min":0.28100000000000003},"max":{"1min":1.6739999999999999,"5min":144.005,"15min":144.005},"last":0.42999999999999999}]},{"osd":2,"last update":"Sat Sep 23 19:57:09 2023","interfaces":[{"interface":"back","average":{"1min":0.871,"5min":3.2330000000000001,"15min":3.5129999999999999},"min":{"1min":0.628,"5min":0.32400000000000001,"15min":0.309},"max":{"1min":1.4350000000000001,"5min":75.640000000000001,"15min":241.59700000000001},"last":0.32000000000000001},{"interface":"front","average":{"1min":1.008,"5min":3.3010000000000002,"15min":3.3660000000000001},"min":{"1min":0.68700000000000006,"5min":0.34699999999999998,"15min":0.32000000000000001},"max":{"1min":1.3080000000000001,"5min":79.626000000000005,"15min":242.13999999999999},"last":0.71399999999999997}]},{"osd":3,"last update":"Sat Sep 23 19:56:54 2023","interfaces":[{"interface":"back","average":{"1min":1.0049999999999999,"5min":1.734,"15min":1.4319999999999999},"min":{"1min":0.45900000000000002,"5min":0.35599999999999998,"15min":0.33700000000000002},"max":{"1min":3.2669999999999999,"5min":9.3829999999999991,"15min":9.3829999999999991},"last":0.35399999999999998},{"interface":"front","average":{"1min":1.3080000000000001,"5min":1.577,"15min":1.474},"min":{"1min":0.755,"5min":0.34000000000000002,"15min":0.33400000000000002},"max":{"1min":3.7650000000000001,"5min":9.6110000000000007,"15min":9.6110000000000007},"last":0.40799999999999997}]},{"osd":4,"last update":"Sat Sep 23 19:57:21 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":2.573,"15min":1.6220000000000001},"min":{"1min":0.82699999999999996,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":1.367,"5min":12.051,"15min":12.051},"last":0.46600000000000003},{"interface":"front","average":{"1min":1.0489999999999999,"5min":2.2690000000000001,"15min":1.3779999999999999},"min":{"1min":0.57799999999999996,"5min":0.34000000000000002,"15min":0.318},"max":{"1min":1.5740000000000001,"5min":16.462,"15min":16.462},"last":0.53100000000000003}]},{"osd":6,"last update":"Sat Sep 23 19:57:09 2023","interfaces":[{"interface":"back","average":{"1min":1.1459999999999999,"5min":2.0960000000000001,"15min":1.3959999999999999},"min":{"1min":0.81699999999999995,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.431,"5min":9.641,"15min":9.641},"last":0.56499999999999995},{"interface":"front","average":{"1min":1.1100000000000001,"5min":2.0259999999999998,"15min":1.4450000000000001},"min":{"1min":0.73799999999999999,"5min":0.39600000000000002,"15min":0.28999999999999998},"max":{"1min":1.5009999999999999,"5min":11.444000000000001,"15min":11.444000000000001},"last":0.49399999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:57:21 2023","interfaces":[{"interface":"back","average":{"1min":1.0640000000000001,"5min":2.3740000000000001,"15min":1.659},"min":{"1min":0.77300000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.3089999999999999,"5min":9.407,"15min":13.771000000000001},"last":0.44500000000000001},{"interface":"front","average":{"1min":1.1519999999999999,"5min":2.0259999999999998,"15min":1.542},"min":{"1min":0.64400000000000002,"5min":0.38300000000000001,"15min":0.33900000000000002},"max":{"1min":1.3959999999999999,"5min":17.053000000000001,"15min":17.053000000000001},"last":0.623}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":844496896,"data_stored":844496896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":998899712,"data_stored":998899712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":727187456,"data_stored":727187456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":712835072,"data_stored":712835072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":737935360,"data_stored":737935360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":608501760,"data_stored":608501760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":493092864,"data_stored":493092864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":364052480,"data_stored":364052480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T19:57:33.021 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T19:57:33.067 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:33.094 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:33.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:33.098 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:33.199 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:33.226 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:33.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:33.229 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:33.330 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:33.359 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:33.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:33.361 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:33.432 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:57:33.462 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:33.495 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:33.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:33.498 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:33.599 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:33.628 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:33.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:33.632 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:33.733 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:33.763 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:33.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:33.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:33.867 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 19 16 34783 34767 114.34 85.5625 0.0091686 0.00873912 2023-09-23T19:57:33.868 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:33.893 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:33.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:33.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:33.998 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:34.007 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:33 smithi016 bash[77995]: cluster 2023-09-23T19:57:32.565669+0000 mgr.y (mgr.25325) 1508 : cluster 0 pgmap v1533: 89 pgs: 89 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2023-09-23T19:57:34.007 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:33 smithi016 bash[75004]: cluster 2023-09-23T19:57:32.565669+0000 mgr.y (mgr.25325) 1508 : cluster 0 pgmap v1533: 89 pgs: 89 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2023-09-23T19:57:34.028 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:34.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:34.031 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:34.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:33 smithi092 bash[58855]: cluster 2023-09-23T19:57:32.565669+0000 mgr.y (mgr.25325) 1508 : cluster 0 pgmap v1533: 89 pgs: 89 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2023-09-23T19:57:34.132 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:34.169 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:34.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:34.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:34.275 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:34.305 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:34.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:34.309 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:34.410 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:34.436 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:34.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:34.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:57:34.542 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:34.576 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:34.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:34.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:57:34.679 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:34.714 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:34.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:34.717 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:34.818 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:34.851 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:34.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:34.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:34.867 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-09-23T19:57:34.864358+0000 min lat: 0.001237 max lat: 0.327936 avg lat: 0.00881948 2023-09-23T19:57:34.867 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:57:34.867 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 20 16 36163 36147 112.935 86.25 0.00220359 0.00881948 2023-09-23T19:57:34.954 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:34.983 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:35.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:35.062 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:35.163 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:35.190 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:35.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:35.194 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:35.295 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:35.321 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:35.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:35.324 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:35.425 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:35.461 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:35.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:35.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:35.527 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:57:35.529 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:57:35.567 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:35.608 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:35.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:35.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:35.718 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:35.743 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:35.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:35.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:35.849 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:35.860 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:35 smithi016 bash[75004]: cluster 2023-09-23T19:57:34.566264+0000 mgr.y (mgr.25325) 1509 : cluster 0 pgmap v1534: 89 pgs: 89 active+clean; 1.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-09-23T19:57:35.861 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:35 smithi016 bash[75004]: audit 2023-09-23T19:57:35.488426+0000 mon.a (mon.0) 1831 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:57:35.861 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:57:35 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:57:35] "GET /metrics HTTP/1.1" 200 36301 "" "Prometheus/2.33.4" 2023-09-23T19:57:35.861 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:35 smithi016 bash[77995]: cluster 2023-09-23T19:57:34.566264+0000 mgr.y (mgr.25325) 1509 : cluster 0 pgmap v1534: 89 pgs: 89 active+clean; 1.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-09-23T19:57:35.861 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:35 smithi016 bash[77995]: audit 2023-09-23T19:57:35.488426+0000 mon.a (mon.0) 1831 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:57:35.869 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 21 16 37348 37332 111.074 74.0625 0.0211732 0.00899511 2023-09-23T19:57:35.886 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:35.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:35.887 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:35.988 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:36.021 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:36.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:36.024 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:36.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:35 smithi092 bash[58855]: cluster 2023-09-23T19:57:34.566264+0000 mgr.y (mgr.25325) 1509 : cluster 0 pgmap v1534: 89 pgs: 89 active+clean; 1.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-09-23T19:57:36.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:35 smithi092 bash[58855]: audit 2023-09-23T19:57:35.488426+0000 mon.a (mon.0) 1831 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:57:36.124 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:36.150 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:36.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:36.153 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:36.254 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:36.291 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:36.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:36.295 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:36.396 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:36.423 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:36.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:36.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:36.529 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:36.557 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:36.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:36.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:57:36.660 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:36.687 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:36.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:36.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:57:36.792 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:36.822 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:36.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:36.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:36.869 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 22 16 38493 38477 109.278 71.5625 0.0135352 0.00914552 2023-09-23T19:57:36.927 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:36.959 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:36.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:36.961 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:57:37.062 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:37.096 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:37.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:37.098 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:37.199 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:37.224 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:37.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:37.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:37.327 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:37.349 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:37.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:37.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:37.456 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:37.486 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:37.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:37.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:37.593 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:37.617 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:37.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:37.622 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:57:37.723 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:37.752 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:37.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:37.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:37.857 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:37.869 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 23 16 39652 39636 107.676 72.4375 0.00997709 0.00928008 2023-09-23T19:57:37.893 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:37.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:37.897 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:57:37.998 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:38.022 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:38 smithi016 bash[77995]: cluster 2023-09-23T19:57:36.566977+0000 mgr.y (mgr.25325) 1510 : cluster 0 pgmap v1535: 89 pgs: 89 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-09-23T19:57:38.022 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:38 smithi016 bash[75004]: cluster 2023-09-23T19:57:36.566977+0000 mgr.y (mgr.25325) 1510 : cluster 0 pgmap v1535: 89 pgs: 89 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-09-23T19:57:38.030 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:38.034 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:37 smithi092 bash[58855]: cluster 2023-09-23T19:57:36.566977+0000 mgr.y (mgr.25325) 1510 : cluster 0 pgmap v1535: 89 pgs: 89 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-09-23T19:57:38.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:38.035 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:38.135 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:38.157 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:38.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:38.159 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:38.260 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:38.284 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:38.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:38.287 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:38.387 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:38.426 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:38.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:38.432 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:38.533 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:38.560 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:38.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:38.563 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:38.664 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:38.695 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:38.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:38.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:38.799 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:38.827 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:38.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:38.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:38.869 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 24 16 40724 40708 105.981 67 0.00963382 0.0094296 2023-09-23T19:57:38.930 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:38.960 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:38.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:38.964 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:39.065 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:39.092 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:39.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:39.096 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:39.197 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:39.229 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:39.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:39.233 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:39.334 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:39.359 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:39.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:39.366 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:57:39.467 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:39.495 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:39.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:39.497 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:39.597 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:39.626 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:39.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:39.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:39.730 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:39.761 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:39.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:39.768 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:39.770 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:39 smithi016 bash[75004]: cluster 2023-09-23T19:57:38.567476+0000 mgr.y (mgr.25325) 1511 : cluster 0 pgmap v1536: 89 pgs: 89 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-09-23T19:57:39.835 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:57:39.868 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:39.875 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 25 16 42041 42025 105.023 82.3125 0.00803515 0.00951405 2023-09-23T19:57:39.929 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:39.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:39.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:40.033 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:40.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:39 smithi016 bash[77995]: cluster 2023-09-23T19:57:38.567476+0000 mgr.y (mgr.25325) 1511 : cluster 0 pgmap v1536: 89 pgs: 89 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-09-23T19:57:40.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:39 smithi092 bash[58855]: cluster 2023-09-23T19:57:38.567476+0000 mgr.y (mgr.25325) 1511 : cluster 0 pgmap v1536: 89 pgs: 89 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-09-23T19:57:40.061 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:40.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:40.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:40.164 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:40.194 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:40.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:40.201 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:40.302 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:40.327 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:40.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:40.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:40.430 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:40.462 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:40.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:40.465 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:40.565 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:40.611 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:40.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:40.617 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:40.718 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:40.742 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:40.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:40.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:40.845 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:40.872 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 26 16 43026 43010 103.351 61.5625 0.00908193 0.00966817 2023-09-23T19:57:40.877 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:40.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:40.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:41.011 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:41.031 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:41.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:41.033 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:41.134 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:41.170 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:41.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:41.173 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:41.273 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:41.303 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:41.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:41.306 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:41.407 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:41.434 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:41.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:41.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:41.538 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:41.587 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:41.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:41.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:41.690 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:41.721 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:41.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:41.728 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:41.829 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:41.837 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:41 smithi092 bash[58855]: cluster 2023-09-23T19:57:40.568628+0000 mgr.y (mgr.25325) 1512 : cluster 0 pgmap v1537: 89 pgs: 89 active+clean; 2.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-09-23T19:57:41.855 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:41.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:41.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:57:41.872 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 27 16 44062 44046 101.922 64.75 0.002943 0.00979481 2023-09-23T19:57:41.958 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:41.968 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:41 smithi016 bash[75004]: cluster 2023-09-23T19:57:40.568628+0000 mgr.y (mgr.25325) 1512 : cluster 0 pgmap v1537: 89 pgs: 89 active+clean; 2.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-09-23T19:57:41.968 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:41 smithi016 bash[77995]: cluster 2023-09-23T19:57:40.568628+0000 mgr.y (mgr.25325) 1512 : cluster 0 pgmap v1537: 89 pgs: 89 active+clean; 2.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-09-23T19:57:41.989 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:41.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:41.992 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:42.093 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:42.123 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:42.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:42.129 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:42.230 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:42.261 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:42.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:42.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:42.367 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:42.400 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:42.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:42.404 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:57:42.505 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:42.552 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:42.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:42.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:42.656 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:42.686 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:42.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:42.689 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:42.790 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:42.829 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:42.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:42.832 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:42.872 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 28 16 45101 45085 100.601 64.9375 0.00569426 0.00991313 2023-09-23T19:57:42.933 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:42.951 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:42.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:42.953 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:43.054 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:43.064 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:42 smithi016 bash[75004]: audit 2023-09-23T19:57:41.907124+0000 mon.a (mon.0) 1832 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:57:43.064 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:42 smithi016 bash[75004]: audit 2023-09-23T19:57:41.912165+0000 mon.a (mon.0) 1833 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:57:43.064 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:42 smithi016 bash[77995]: audit 2023-09-23T19:57:41.907124+0000 mon.a (mon.0) 1832 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:57:43.065 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:42 smithi016 bash[77995]: audit 2023-09-23T19:57:41.912165+0000 mon.a (mon.0) 1833 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:57:43.084 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:43.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:43.086 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:43.186 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:43.243 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:43.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:43.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:43.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:42 smithi092 bash[58855]: audit 2023-09-23T19:57:41.907124+0000 mon.a (mon.0) 1832 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:57:43.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:42 smithi092 bash[58855]: audit 2023-09-23T19:57:41.912165+0000 mon.a (mon.0) 1833 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:57:43.348 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:43.394 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:43.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:43.400 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:43.501 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:43.528 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:43.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:43.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:43.631 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:43.650 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:43.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:43.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:43.754 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:43.780 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:43.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:43.784 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:43.872 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 29 16 45996 45980 99.0609 55.9375 0.00863694 0.0100872 2023-09-23T19:57:43.885 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:43.916 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:43.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:43.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:43.922 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:43 smithi016 bash[75004]: cluster 2023-09-23T19:57:42.570279+0000 mgr.y (mgr.25325) 1513 : cluster 0 pgmap v1538: 89 pgs: 89 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-09-23T19:57:43.922 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:43 smithi016 bash[75004]: audit 2023-09-23T19:57:43.691572+0000 mon.a (mon.0) 1834 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:57:43.922 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:43 smithi016 bash[75004]: audit 2023-09-23T19:57:43.766864+0000 mon.a (mon.0) 1835 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:57:43.922 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:43 smithi016 bash[75004]: audit 2023-09-23T19:57:43.778357+0000 mon.a (mon.0) 1836 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:57:43.922 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:43 smithi016 bash[77995]: cluster 2023-09-23T19:57:42.570279+0000 mgr.y (mgr.25325) 1513 : cluster 0 pgmap v1538: 89 pgs: 89 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-09-23T19:57:43.923 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:43 smithi016 bash[77995]: audit 2023-09-23T19:57:43.691572+0000 mon.a (mon.0) 1834 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:57:43.923 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:43 smithi016 bash[77995]: audit 2023-09-23T19:57:43.766864+0000 mon.a (mon.0) 1835 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:57:43.923 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:43 smithi016 bash[77995]: audit 2023-09-23T19:57:43.778357+0000 mon.a (mon.0) 1836 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:57:44.020 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:44.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:43 smithi092 bash[58855]: cluster 2023-09-23T19:57:42.570279+0000 mgr.y (mgr.25325) 1513 : cluster 0 pgmap v1538: 89 pgs: 89 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-09-23T19:57:44.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:43 smithi092 bash[58855]: audit 2023-09-23T19:57:43.691572+0000 mon.a (mon.0) 1834 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:57:44.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:43 smithi092 bash[58855]: audit 2023-09-23T19:57:43.766864+0000 mon.a (mon.0) 1835 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:57:44.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:43 smithi092 bash[58855]: audit 2023-09-23T19:57:43.778357+0000 mon.a (mon.0) 1836 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:57:44.051 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:44.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:44.055 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:44.155 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:44.183 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:44.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:44.184 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:44.285 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:44.319 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:44.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:44.323 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:44.423 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:44.455 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:44.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:44.460 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:44.561 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:44.588 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:44.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:44.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:44.695 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:44.722 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:44.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:44.728 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:44.829 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:44.857 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:44.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:44.860 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:57:44.872 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 30 16 47206 47190 98.2796 75.625 0.0106238 0.0101696 2023-09-23T19:57:44.961 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:44.997 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:45.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:45.000 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:45.101 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:45.128 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:45.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:45.130 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:45.231 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:45.245 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:44 smithi092 bash[58855]: audit 2023-09-23T19:57:44.412706+0000 mon.a (mon.0) 1837 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:57:45.245 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:44 smithi092 bash[58855]: audit 2023-09-23T19:57:44.414302+0000 mon.a (mon.0) 1838 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:57:45.245 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:44 smithi092 bash[58855]: audit 2023-09-23T19:57:44.422431+0000 mon.a (mon.0) 1839 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:57:45.260 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:45.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:45.265 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:45.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:44 smithi016 bash[75004]: audit 2023-09-23T19:57:44.412706+0000 mon.a (mon.0) 1837 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:57:45.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:44 smithi016 bash[75004]: audit 2023-09-23T19:57:44.414302+0000 mon.a (mon.0) 1838 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:57:45.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:44 smithi016 bash[75004]: audit 2023-09-23T19:57:44.422431+0000 mon.a (mon.0) 1839 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:57:45.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:44 smithi016 bash[77995]: audit 2023-09-23T19:57:44.412706+0000 mon.a (mon.0) 1837 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:57:45.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:44 smithi016 bash[77995]: audit 2023-09-23T19:57:44.414302+0000 mon.a (mon.0) 1838 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:57:45.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:44 smithi016 bash[77995]: audit 2023-09-23T19:57:44.422431+0000 mon.a (mon.0) 1839 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:57:45.366 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:45.404 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:45.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:45.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:45.511 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:45.539 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:45.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:45.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:45.643 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:45.671 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:45.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:45.675 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:57:45.775 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:45.793 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:45.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:45.798 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:45.873 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 31 16 48226 48210 97.1641 63.75 0.00349544 0.0102482 2023-09-23T19:57:45.898 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:45.908 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:57:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:57:45] "GET /metrics HTTP/1.1" 200 36302 "" "Prometheus/2.33.4" 2023-09-23T19:57:45.930 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:45.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:45.936 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:46.037 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:46.061 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:46.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:46.065 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:46.166 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:46.174 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:45 smithi016 bash[75004]: cluster 2023-09-23T19:57:44.570788+0000 mgr.y (mgr.25325) 1514 : cluster 0 pgmap v1539: 89 pgs: 89 active+clean; 2.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-09-23T19:57:46.175 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:45 smithi016 bash[77995]: cluster 2023-09-23T19:57:44.570788+0000 mgr.y (mgr.25325) 1514 : cluster 0 pgmap v1539: 89 pgs: 89 active+clean; 2.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-09-23T19:57:46.196 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:46.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:46.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:46.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:45 smithi092 bash[58855]: cluster 2023-09-23T19:57:44.570788+0000 mgr.y (mgr.25325) 1514 : cluster 0 pgmap v1539: 89 pgs: 89 active+clean; 2.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-09-23T19:57:46.299 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:46.326 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:46.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:46.331 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:46.432 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:46.463 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:46.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:46.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:46.570 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:46.605 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:46.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:46.608 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:46.709 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:46.735 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:46.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:46.739 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:46.839 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:46.866 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:46.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:46.869 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:46.873 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 32 16 49403 49387 96.4263 73.5625 0.00437801 0.0103509 2023-09-23T19:57:46.970 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:47.001 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:47.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:47.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:47.104 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:47.130 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:47.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:47.134 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:57:47.235 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:47.269 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:47.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:47.273 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:47.348 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-09-23T19:57:47.348 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [ 2023-09-23T19:57:47.348 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:57:47.348 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.46877.0:49813 14.f e261/256 14:f1867885:::benchmark_data_smithi016_228083_object49812:head v 261'3123, mlcod=261'3121)", 2023-09-23T19:57:47.349 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:57:47.336401+0000", 2023-09-23T19:57:47.349 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 0.0065147410000000001, 2023-09-23T19:57:47.349 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0065466700000000001, 2023-09-23T19:57:47.349 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:57:47.349 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "started", 2023-09-23T19:57:47.349 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:57:47.349 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:57:47.349 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:57:47.349 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:47.336401+0000", 2023-09-23T19:57:47.349 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:57:47.349 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:57:47.349 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:57:47.349 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:57:47.350 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:47.336401+0000", 2023-09-23T19:57:47.350 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:57:47.350 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:57:47.350 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:57:47.350 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:57:47.350 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:47.336400+0000", 2023-09-23T19:57:47.350 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:57:47.350 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:57:47.350 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:57:47.350 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:57:47.350 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:47.336443+0000", 2023-09-23T19:57:47.350 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.3597999999999998e-05 2023-09-23T19:57:47.351 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:57:47.351 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:57:47.351 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:57:47.351 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:47.336462+0000", 2023-09-23T19:57:47.351 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.8802000000000001e-05 2023-09-23T19:57:47.351 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:57:47.351 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:57:47.351 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:57:47.351 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:47.336471+0000", 2023-09-23T19:57:47.351 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.354e-06 2023-09-23T19:57:47.351 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:57:47.351 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:57:47.352 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:57:47.352 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:47.336554+0000", 2023-09-23T19:57:47.352 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.2913999999999999e-05 2023-09-23T19:57:47.352 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:57:47.352 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:57:47.352 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:57:47.352 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:47.336575+0000", 2023-09-23T19:57:47.352 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.1798e-05 2023-09-23T19:57:47.352 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:57:47.352 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:57:47.352 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:57:47.352 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:57:47.352 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:57:47.353 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.46877.0:49814 14.4 e261/256 14:23f0df7a:::benchmark_data_smithi016_228083_object49813:head v 261'3157, mlcod=261'3155)", 2023-09-23T19:57:47.353 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:57:47.338694+0000", 2023-09-23T19:57:47.353 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 0.004222127, 2023-09-23T19:57:47.353 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0043413230000000002, 2023-09-23T19:57:47.353 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:57:47.353 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "started", 2023-09-23T19:57:47.353 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:57:47.353 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:57:47.353 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:57:47.353 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:47.338694+0000", 2023-09-23T19:57:47.353 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:57:47.353 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:57:47.353 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:57:47.354 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:57:47.354 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:47.338694+0000", 2023-09-23T19:57:47.354 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:57:47.354 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:57:47.354 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:57:47.354 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:57:47.354 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:47.338692+0000", 2023-09-23T19:57:47.354 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:57:47.354 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:57:47.354 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:57:47.354 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:57:47.354 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:47.338747+0000", 2023-09-23T19:57:47.354 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.4237000000000003e-05 2023-09-23T19:57:47.355 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:57:47.355 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:57:47.355 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:57:47.355 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:47.338765+0000", 2023-09-23T19:57:47.355 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.7867e-05 2023-09-23T19:57:47.355 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:57:47.355 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:57:47.355 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:57:47.355 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:47.338777+0000", 2023-09-23T19:57:47.355 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1953000000000001e-05 2023-09-23T19:57:47.355 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:57:47.355 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:57:47.356 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:57:47.356 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:47.338826+0000", 2023-09-23T19:57:47.356 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.9039000000000003e-05 2023-09-23T19:57:47.356 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:57:47.356 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:57:47.356 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:57:47.356 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:47.338846+0000", 2023-09-23T19:57:47.356 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.0914e-05 2023-09-23T19:57:47.356 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:57:47.356 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:57:47.356 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:57:47.356 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:57:47.356 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:57:47.357 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.46877.0:49817 14.2 e261/256 14:4396bc1e:::benchmark_data_smithi016_228083_object49816:head v 261'3212, mlcod=261'3210)", 2023-09-23T19:57:47.357 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:57:47.338791+0000", 2023-09-23T19:57:47.357 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 0.0041246110000000002, 2023-09-23T19:57:47.357 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0043200110000000003, 2023-09-23T19:57:47.357 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:57:47.357 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "started", 2023-09-23T19:57:47.357 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:57:47.357 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:57:47.357 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:57:47.357 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:47.338791+0000", 2023-09-23T19:57:47.357 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:57:47.357 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:57:47.358 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:57:47.358 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:57:47.358 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:47.338791+0000", 2023-09-23T19:57:47.358 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:57:47.358 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:57:47.358 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:57:47.358 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:57:47.358 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:47.338791+0000", 2023-09-23T19:57:47.358 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T19:57:47.358 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:57:47.358 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:57:47.358 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:57:47.358 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:47.338833+0000", 2023-09-23T19:57:47.358 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.2969999999999997e-05 2023-09-23T19:57:47.359 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:57:47.359 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:57:47.359 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:57:47.359 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:47.338843+0000", 2023-09-23T19:57:47.359 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.3489999999999998e-06 2023-09-23T19:57:47.359 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:57:47.359 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:57:47.359 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:57:47.359 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:47.338849+0000", 2023-09-23T19:57:47.359 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.2700000000000001e-06 2023-09-23T19:57:47.359 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:57:47.359 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:57:47.360 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:57:47.360 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:47.339007+0000", 2023-09-23T19:57:47.360 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000158126 2023-09-23T19:57:47.360 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:57:47.360 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:57:47.360 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:57:47.360 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:47.339028+0000", 2023-09-23T19:57:47.360 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.0925999999999999e-05 2023-09-23T19:57:47.360 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:57:47.360 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:57:47.360 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:57:47.360 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:57:47.360 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:57:47.361 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.46877.0:49816 14.9 e261/256 14:9e15fa5b:::benchmark_data_smithi016_228083_object49815:head v 261'3105, mlcod=261'3103)", 2023-09-23T19:57:47.361 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:57:47.340314+0000", 2023-09-23T19:57:47.361 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 0.0026014480000000001, 2023-09-23T19:57:47.361 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0028711919999999998, 2023-09-23T19:57:47.361 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:57:47.361 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "started", 2023-09-23T19:57:47.361 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:57:47.361 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:57:47.361 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:57:47.361 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:47.340314+0000", 2023-09-23T19:57:47.361 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:57:47.361 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:57:47.361 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:57:47.362 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:57:47.362 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:47.340314+0000", 2023-09-23T19:57:47.362 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:57:47.362 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:57:47.362 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:57:47.362 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:57:47.362 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:47.340314+0000", 2023-09-23T19:57:47.362 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T19:57:47.362 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:57:47.362 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:57:47.363 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:57:47.363 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:47.340348+0000", 2023-09-23T19:57:47.363 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.4727999999999997e-05 2023-09-23T19:57:47.363 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:57:47.363 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:57:47.363 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:57:47.364 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:47.340364+0000", 2023-09-23T19:57:47.364 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.6121999999999999e-05 2023-09-23T19:57:47.364 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:57:47.364 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:57:47.364 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:57:47.364 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:47.340372+0000", 2023-09-23T19:57:47.364 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.1400000000000002e-06 2023-09-23T19:57:47.365 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:57:47.365 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:57:47.365 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:57:47.365 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:47.340684+0000", 2023-09-23T19:57:47.365 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00031277900000000001 2023-09-23T19:57:47.365 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:57:47.366 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:57:47.366 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:57:47.366 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:47.340702+0000", 2023-09-23T19:57:47.366 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.8002000000000001e-05 2023-09-23T19:57:47.366 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:57:47.366 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:57:47.367 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:57:47.367 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:57:47.367 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:57:47.367 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.46877.0:49812 14.3 e261/256)", 2023-09-23T19:57:47.367 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:57:47.340402+0000", 2023-09-23T19:57:47.367 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 0.0025132919999999999, 2023-09-23T19:57:47.367 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0028535470000000001, 2023-09-23T19:57:47.368 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:57:47.368 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "queued for pg", 2023-09-23T19:57:47.368 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:57:47.368 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:57:47.368 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:57:47.368 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:47.340402+0000", 2023-09-23T19:57:47.368 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:57:47.369 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:57:47.369 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:57:47.369 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:57:47.369 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:47.340402+0000", 2023-09-23T19:57:47.369 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:57:47.369 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:57:47.370 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:57:47.370 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:57:47.370 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:47.340402+0000", 2023-09-23T19:57:47.370 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T19:57:47.370 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:57:47.370 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:57:47.370 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:57:47.371 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:47.340427+0000", 2023-09-23T19:57:47.371 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.5174999999999999e-05 2023-09-23T19:57:47.371 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:57:47.371 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:57:47.371 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:57:47.371 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:47.340435+0000", 2023-09-23T19:57:47.371 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.4819999999999997e-06 2023-09-23T19:57:47.372 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:57:47.372 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:57:47.372 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:57:47.372 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:47.340440+0000", 2023-09-23T19:57:47.372 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.1590000000000001e-06 2023-09-23T19:57:47.372 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:57:47.373 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:57:47.373 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:57:47.373 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:57:47.373 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:57:47.373 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.46877.0:49821 14.9 e261/256 14:96090617:::benchmark_data_smithi016_228083_object49820:head v 261'3106, mlcod=261'3103)", 2023-09-23T19:57:47.373 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:57:47.341315+0000", 2023-09-23T19:57:47.373 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 0.001600469, 2023-09-23T19:57:47.374 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0019999940000000002, 2023-09-23T19:57:47.374 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:57:47.374 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "started", 2023-09-23T19:57:47.374 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:57:47.374 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:57:47.374 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:57:47.374 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:47.341315+0000", 2023-09-23T19:57:47.375 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:57:47.375 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:57:47.375 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:57:47.375 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:57:47.375 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:47.341315+0000", 2023-09-23T19:57:47.375 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:57:47.375 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:57:47.376 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:57:47.376 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:57:47.376 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:47.341315+0000", 2023-09-23T19:57:47.376 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T19:57:47.376 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:57:47.376 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:57:47.376 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:57:47.377 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:47.341360+0000", 2023-09-23T19:57:47.377 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.5565000000000002e-05 2023-09-23T19:57:47.377 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:57:47.377 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:57:47.377 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:57:47.377 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:47.341370+0000", 2023-09-23T19:57:47.377 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.9240000000000002e-06 2023-09-23T19:57:47.378 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:57:47.378 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:57:47.378 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:57:47.378 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:47.341377+0000", 2023-09-23T19:57:47.378 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.0820000000000001e-06 2023-09-23T19:57:47.378 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:57:47.378 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:57:47.379 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:57:47.379 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:47.342402+0000", 2023-09-23T19:57:47.379 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0010251640000000001 2023-09-23T19:57:47.379 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:57:47.379 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:57:47.379 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:57:47.379 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:47.342479+0000", 2023-09-23T19:57:47.380 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.7084999999999998e-05 2023-09-23T19:57:47.380 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:57:47.380 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:57:47.380 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:57:47.380 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:57:47.380 INFO:teuthology.orchestra.run.smithi016.stdout: ], 2023-09-23T19:57:47.380 INFO:teuthology.orchestra.run.smithi016.stdout: "num_ops": 6 2023-09-23T19:57:47.381 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-09-23T19:57:47.381 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:47.409 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:47.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:47.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:57:47.514 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:47.541 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:47.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:47.544 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:57:47.645 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:47.679 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:47.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:47.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:47.786 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:47.817 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:47.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:47.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:47.873 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 33 16 50370 50354 95.3359 60.4375 0.0195214 0.0104823 2023-09-23T19:57:47.920 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:47.946 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:47.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:47.950 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:57:48.050 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:48.079 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:48.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:48.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:48.182 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:48.191 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:47 smithi016 bash[75004]: cluster 2023-09-23T19:57:46.571412+0000 mgr.y (mgr.25325) 1515 : cluster 0 pgmap v1540: 89 pgs: 89 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-09-23T19:57:48.191 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:47 smithi016 bash[77995]: cluster 2023-09-23T19:57:46.571412+0000 mgr.y (mgr.25325) 1515 : cluster 0 pgmap v1540: 89 pgs: 89 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-09-23T19:57:48.211 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:48.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:48.215 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:48.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:47 smithi092 bash[58855]: cluster 2023-09-23T19:57:46.571412+0000 mgr.y (mgr.25325) 1515 : cluster 0 pgmap v1540: 89 pgs: 89 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-09-23T19:57:48.316 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:48.339 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:48.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:48.341 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:48.441 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:48.469 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:48.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:48.477 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:48.577 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:48.614 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:48.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:48.618 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:48.718 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:48.750 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:48.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:48.755 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:48.856 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:48.873 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 34 16 51359 51343 94.35 61.8125 0.0149896 0.0105936 2023-09-23T19:57:48.886 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:48.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:48.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:48.989 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:49.019 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:49.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:49.021 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:57:49.122 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:49.152 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:49.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:49.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:49.256 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:49.288 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:49.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:49.293 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:49.394 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:49.426 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:49.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:49.429 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:57:49.530 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:49.554 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:49.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:49.558 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:49.658 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:49.689 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:49.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:49.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:57:49.792 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:49.820 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:49.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:49.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:57:49.874 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 35 16 52571 52555 93.8185 75.75 0.0112228 0.0106517 2023-09-23T19:57:49.923 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:49.958 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:49.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:49.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:50.063 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:50.096 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:50.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:50.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:50.202 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:50.210 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:49 smithi092 bash[58855]: cluster 2023-09-23T19:57:48.572438+0000 mgr.y (mgr.25325) 1516 : cluster 0 pgmap v1541: 89 pgs: 89 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-09-23T19:57:50.226 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:50.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:50.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:57:50.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:49 smithi016 bash[75004]: cluster 2023-09-23T19:57:48.572438+0000 mgr.y (mgr.25325) 1516 : cluster 0 pgmap v1541: 89 pgs: 89 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-09-23T19:57:50.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:49 smithi016 bash[77995]: cluster 2023-09-23T19:57:48.572438+0000 mgr.y (mgr.25325) 1516 : cluster 0 pgmap v1541: 89 pgs: 89 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-09-23T19:57:50.329 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:50.356 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:50.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:50.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:50.459 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:50.485 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:50.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:50.488 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:57:50.589 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:50.615 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:50.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:50.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:57:50.720 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:50.744 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:50.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:50.746 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:50.847 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:50.876 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 36 16 53653 53637 93.0848 67.625 0.0126567 0.0107362 2023-09-23T19:57:50.877 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:50.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:50.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:50.981 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:51.009 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:51.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:51.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:51.112 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:51.137 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:51.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:51.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:51.240 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:51.266 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:51.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:51.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:51.372 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:51.413 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:51.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:51.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:51.517 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:51.551 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:51.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:51.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:51.655 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:51.683 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:51.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:51.686 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:57:51.787 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:51.816 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:51.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:51.824 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:51.876 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 37 16 54808 54792 92.5199 72.1875 0.00313507 0.0108015 2023-09-23T19:57:51.925 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:51.950 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:51.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:51.953 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:52.054 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:52.081 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:52.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:52.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:57:52.185 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:52.195 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:51 smithi016 bash[75004]: cluster 2023-09-23T19:57:50.573285+0000 mgr.y (mgr.25325) 1517 : cluster 0 pgmap v1542: 89 pgs: 89 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2023-09-23T19:57:52.195 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:51 smithi016 bash[77995]: cluster 2023-09-23T19:57:50.573285+0000 mgr.y (mgr.25325) 1517 : cluster 0 pgmap v1542: 89 pgs: 89 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2023-09-23T19:57:52.213 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:52.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:52.217 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:52.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:51 smithi092 bash[58855]: cluster 2023-09-23T19:57:50.573285+0000 mgr.y (mgr.25325) 1517 : cluster 0 pgmap v1542: 89 pgs: 89 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2023-09-23T19:57:52.317 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:52.343 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:52.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:52.345 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:52.446 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:52.471 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:52.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:52.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:52.574 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T19:57:52.580 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:52.607 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:52.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:52.610 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:57:52.711 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:52.755 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:52.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:52.757 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:52.857 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:52.879 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 38 16 55933 55917 91.9295 70.3125 0.0154301 0.0108721 2023-09-23T19:57:52.890 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:52.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:52.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:52.993 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:53.017 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:53.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:53.021 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:53.122 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:53.153 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:53.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:53.156 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:53.257 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:53.283 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:53.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:53.287 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:53.388 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:53.418 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:53.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:53.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:53.522 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:53.547 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:53.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:53.551 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:53.652 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:53.683 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:53.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:53.689 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:53.789 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:53.815 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:53.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:53.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:53.879 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 39 16 57048 57032 91.3592 69.6875 0.00873715 0.0109362 2023-09-23T19:57:53.918 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:53.946 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:57:53.946 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:53.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:53.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:53.987 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:57:54.023 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:57:54.050 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:54.072 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:57:54.077 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:54.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:54.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:54.118 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:57:54.163 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:57:54.181 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:54.190 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:53 smithi016 bash[75004]: cluster 2023-09-23T19:57:52.573767+0000 mgr.y (mgr.25325) 1518 : cluster 0 pgmap v1543: 89 pgs: 89 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 60 MiB/s wr, 954 op/s 2023-09-23T19:57:54.190 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:53 smithi016 bash[75004]: audit 2023-09-23T19:57:53.880911+0000 mon.c (mon.2) 231 : audit 0 from='client.? 172.21.15.16:0/1064651310' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:57:54.190 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:53 smithi016 bash[77995]: cluster 2023-09-23T19:57:52.573767+0000 mgr.y (mgr.25325) 1518 : cluster 0 pgmap v1543: 89 pgs: 89 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 60 MiB/s wr, 954 op/s 2023-09-23T19:57:54.190 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:53 smithi016 bash[77995]: audit 2023-09-23T19:57:53.880911+0000 mon.c (mon.2) 231 : audit 0 from='client.? 172.21.15.16:0/1064651310' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:57:54.198 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:57:54.207 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:54.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:54.210 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:54.237 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:57:54.239 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:57:54.239 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:57:54.240 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:57:54.240 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:57:54.240 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:57:54.240 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:57:54.240 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:57:54.240 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:57:54.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:53 smithi092 bash[58855]: cluster 2023-09-23T19:57:52.573767+0000 mgr.y (mgr.25325) 1518 : cluster 0 pgmap v1543: 89 pgs: 89 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 60 MiB/s wr, 954 op/s 2023-09-23T19:57:54.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:53 smithi092 bash[58855]: audit 2023-09-23T19:57:53.880911+0000 mon.c (mon.2) 231 : audit 0 from='client.? 172.21.15.16:0/1064651310' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:57:54.310 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:54.340 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:54.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:54.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:54.443 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:54.471 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:54.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:54.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:54.577 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:54.610 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:54.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:54.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:54.715 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:54.742 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:54.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:54.750 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:54.851 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:54.878 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:54.879 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-09-23T19:57:54.876542+0000 min lat: 0.001237 max lat: 0.327936 avg lat: 0.0110333 2023-09-23T19:57:54.879 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:57:54.879 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 40 16 58020 58004 90.5939 60.75 0.0215092 0.0110333 2023-09-23T19:57:54.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:54.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:54.984 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:55.003 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:55.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:55.005 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:55.105 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:55.129 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:55.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:55.131 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:57:55.232 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:55.273 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:55.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:55.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:55.380 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:55.418 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:55.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:55.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:55.522 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:55.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:55.553 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:55.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:55.654 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:55.679 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:55.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:55.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:55.782 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:55.785 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:57:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:57:55] "GET /metrics HTTP/1.1" 200 36302 "" "Prometheus/2.33.4" 2023-09-23T19:57:55.826 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:55.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:55.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:55.879 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 41 15 59007 58992 89.8904 61.75 0.00902526 0.0111166 2023-09-23T19:57:55.931 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:55.955 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:55.956 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is set 2023-09-23T19:57:55.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:55.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:56.061 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:56.070 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:55 smithi016 bash[75004]: cluster 2023-09-23T19:57:54.574373+0000 mgr.y (mgr.25325) 1519 : cluster 0 pgmap v1544: 89 pgs: 89 active+clean; 3.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T19:57:56.070 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:55 smithi016 bash[75004]: audit 2023-09-23T19:57:55.653790+0000 mon.a (mon.0) 1840 : audit 1 from='client.? 172.21.15.16:0/895641787' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:57:56.071 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:55 smithi016 bash[77995]: cluster 2023-09-23T19:57:54.574373+0000 mgr.y (mgr.25325) 1519 : cluster 0 pgmap v1544: 89 pgs: 89 active+clean; 3.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T19:57:56.071 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:55 smithi016 bash[77995]: audit 2023-09-23T19:57:55.653790+0000 mon.a (mon.0) 1840 : audit 1 from='client.? 172.21.15.16:0/895641787' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:57:56.088 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:56.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:56.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:56.194 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:56.204 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:55 smithi092 bash[58855]: cluster 2023-09-23T19:57:54.574373+0000 mgr.y (mgr.25325) 1519 : cluster 0 pgmap v1544: 89 pgs: 89 active+clean; 3.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T19:57:56.204 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:55 smithi092 bash[58855]: audit 2023-09-23T19:57:55.653790+0000 mon.a (mon.0) 1840 : audit 1 from='client.? 172.21.15.16:0/895641787' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:57:56.230 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:56.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:56.235 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:57:56.336 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:56.363 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:56.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:56.366 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:56.467 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:56.491 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:56.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:56.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:56.594 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:56.617 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:56.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:56.622 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:56.723 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:56.753 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:56.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:56.758 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:56.858 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:56.879 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 42 16 60019 60003 89.2547 63.1875 0.0213933 0.0111967 2023-09-23T19:57:56.889 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:56.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:56.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:56.993 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:57.019 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:57.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:57.022 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:57.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:56 smithi092 bash[58855]: cluster 2023-09-23T19:57:55.937535+0000 mon.a (mon.0) 1841 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:57:57.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:56 smithi092 bash[58855]: audit 2023-09-23T19:57:55.943173+0000 mon.a (mon.0) 1842 : audit 1 from='client.? 172.21.15.16:0/895641787' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:57:57.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:56 smithi092 bash[58855]: cluster 2023-09-23T19:57:55.943268+0000 mon.a (mon.0) 1843 : cluster 0 osdmap e262: 8 total, 8 up, 8 in 2023-09-23T19:57:57.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:56 smithi092 bash[58855]: cluster 2023-09-23T19:57:55.945385+0000 osd.7 (osd.7) 174 : cluster 3 failed to encode map e262 with expected crc 2023-09-23T19:57:57.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:56 smithi092 bash[58855]: cluster 2023-09-23T19:57:55.957298+0000 osd.5 (osd.5) 189 : cluster 3 failed to encode map e262 with expected crc 2023-09-23T19:57:57.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:56 smithi092 bash[58855]: cluster 2023-09-23T19:57:55.961253+0000 osd.6 (osd.6) 196 : cluster 3 failed to encode map e262 with expected crc 2023-09-23T19:57:57.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:56 smithi092 bash[58855]: cluster 2023-09-23T19:57:55.961521+0000 osd.4 (osd.4) 203 : cluster 3 failed to encode map e262 with expected crc 2023-09-23T19:57:57.123 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:57.145 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:57.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:57.154 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:57.245 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:56 smithi016 bash[75004]: cluster 2023-09-23T19:57:55.937535+0000 mon.a (mon.0) 1841 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:57:57.245 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:56 smithi016 bash[75004]: audit 2023-09-23T19:57:55.943173+0000 mon.a (mon.0) 1842 : audit 1 from='client.? 172.21.15.16:0/895641787' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:57:57.245 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:56 smithi016 bash[75004]: cluster 2023-09-23T19:57:55.943268+0000 mon.a (mon.0) 1843 : cluster 0 osdmap e262: 8 total, 8 up, 8 in 2023-09-23T19:57:57.245 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:56 smithi016 bash[75004]: cluster 2023-09-23T19:57:55.945385+0000 osd.7 (osd.7) 174 : cluster 3 failed to encode map e262 with expected crc 2023-09-23T19:57:57.245 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:56 smithi016 bash[75004]: cluster 2023-09-23T19:57:55.957298+0000 osd.5 (osd.5) 189 : cluster 3 failed to encode map e262 with expected crc 2023-09-23T19:57:57.245 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:56 smithi016 bash[75004]: cluster 2023-09-23T19:57:55.961253+0000 osd.6 (osd.6) 196 : cluster 3 failed to encode map e262 with expected crc 2023-09-23T19:57:57.245 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:56 smithi016 bash[75004]: cluster 2023-09-23T19:57:55.961521+0000 osd.4 (osd.4) 203 : cluster 3 failed to encode map e262 with expected crc 2023-09-23T19:57:57.246 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:56 smithi016 bash[77995]: cluster 2023-09-23T19:57:55.937535+0000 mon.a (mon.0) 1841 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:57:57.246 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:56 smithi016 bash[77995]: audit 2023-09-23T19:57:55.943173+0000 mon.a (mon.0) 1842 : audit 1 from='client.? 172.21.15.16:0/895641787' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:57:57.246 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:56 smithi016 bash[77995]: cluster 2023-09-23T19:57:55.943268+0000 mon.a (mon.0) 1843 : cluster 0 osdmap e262: 8 total, 8 up, 8 in 2023-09-23T19:57:57.246 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:56 smithi016 bash[77995]: cluster 2023-09-23T19:57:55.945385+0000 osd.7 (osd.7) 174 : cluster 3 failed to encode map e262 with expected crc 2023-09-23T19:57:57.246 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:56 smithi016 bash[77995]: cluster 2023-09-23T19:57:55.957298+0000 osd.5 (osd.5) 189 : cluster 3 failed to encode map e262 with expected crc 2023-09-23T19:57:57.246 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:56 smithi016 bash[77995]: cluster 2023-09-23T19:57:55.961253+0000 osd.6 (osd.6) 196 : cluster 3 failed to encode map e262 with expected crc 2023-09-23T19:57:57.246 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:56 smithi016 bash[77995]: cluster 2023-09-23T19:57:55.961521+0000 osd.4 (osd.4) 203 : cluster 3 failed to encode map e262 with expected crc 2023-09-23T19:57:57.255 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:57.287 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:57.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:57.290 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:57.390 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:57.419 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:57.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:57.423 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:57.523 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:57.557 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:57.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:57.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:57.661 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:57.692 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:57.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:57.699 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:57.800 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:57.826 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:57.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:57.833 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:57.879 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 43 16 61011 60995 88.6209 62 0.0110021 0.0112763 2023-09-23T19:57:57.934 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:57.965 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:57.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:57.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:58.071 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:58.080 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:58 smithi016 bash[75004]: cluster 2023-09-23T19:57:56.574802+0000 mgr.y (mgr.25325) 1520 : cluster 0 pgmap v1546: 89 pgs: 89 active+clean; 3.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-09-23T19:57:58.080 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:58 smithi016 bash[77995]: cluster 2023-09-23T19:57:56.574802+0000 mgr.y (mgr.25325) 1520 : cluster 0 pgmap v1546: 89 pgs: 89 active+clean; 3.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-09-23T19:57:58.099 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:58.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:58.102 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:58.203 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:58.212 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:57 smithi092 bash[58855]: cluster 2023-09-23T19:57:56.574802+0000 mgr.y (mgr.25325) 1520 : cluster 0 pgmap v1546: 89 pgs: 89 active+clean; 3.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-09-23T19:57:58.228 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:58.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:58.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:57:58.332 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:58.362 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:58.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:58.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:58.466 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:58.507 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:58.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:58.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:58.612 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:58.632 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:58.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:58.634 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:58.734 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:58.765 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:58.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:58.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:58.869 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:58.879 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 44 16 61887 61871 87.8512 54.75 0.0157656 0.0113764 2023-09-23T19:57:58.901 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:58.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:58.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:59.006 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:59.029 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:58 smithi016 bash[75004]: audit 2023-09-23T19:57:58.691989+0000 mon.a (mon.0) 1844 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:57:59.029 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:58 smithi016 bash[77995]: audit 2023-09-23T19:57:58.691989+0000 mon.a (mon.0) 1844 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:57:59.032 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:59.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:59.034 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:59.135 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:59.159 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:59.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:59.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:59.263 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:59.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:58 smithi092 bash[58855]: audit 2023-09-23T19:57:58.691989+0000 mon.a (mon.0) 1844 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:57:59.299 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:59.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:59.302 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:59.400 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-09-23T19:57:59.402 INFO:teuthology.orchestra.run.smithi016.stderr:dumped all 2023-09-23T19:57:59.403 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:59.439 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:59.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:59.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:59.545 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:59.565 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:57:59.576 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:59.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:59.579 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:59.679 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:59.710 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:59.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:59.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:59.816 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:59.840 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:59.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:59.842 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:59.879 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 45 16 62811 62795 87.1824 57.75 0.00903897 0.0114647 2023-09-23T19:57:59.943 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:59.974 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:57:59 smithi016 bash[77995]: cluster 2023-09-23T19:57:58.575439+0000 mgr.y (mgr.25325) 1521 : cluster 0 pgmap v1547: 89 pgs: 89 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-09-23T19:57:59.975 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:57:59.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:57:59.979 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:00.080 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:00.115 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:00.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:00.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:00.220 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:00.229 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:57:59 smithi016 bash[75004]: cluster 2023-09-23T19:57:58.575439+0000 mgr.y (mgr.25325) 1521 : cluster 0 pgmap v1547: 89 pgs: 89 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-09-23T19:58:00.249 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:00.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:00.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:00.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:57:59 smithi092 bash[58855]: cluster 2023-09-23T19:57:58.575439+0000 mgr.y (mgr.25325) 1521 : cluster 0 pgmap v1547: 89 pgs: 89 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-09-23T19:58:00.353 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:00.381 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:00.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:00.384 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:00.484 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:00.515 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:00.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:00.519 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:00.620 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:00.645 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:00.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:00.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:00.713 INFO:teuthology.orchestra.run.smithi016.stdout:{"pg_ready":true,"pg_map":{"version":1547,"stamp":"2023-09-23T19:57:58.574988+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3865515936,"num_objects":58998,"num_object_clones":0,"num_object_copies":176994,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":58998,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":59235,"num_write_kb":3776782,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":59169,"ondisk_log_size":59169,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":18277928,"kb_used_data":11299688,"kb_used_omap":20,"kb_used_meta":6978155,"kb_avail":731683288,"statfs":{"total":767960285184,"available":749243686912,"internally_reserved":0,"allocated":11570880512,"data_stored":11564641724,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":7145631374},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,3,7,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":703922176,"num_objects":10741,"num_object_clones":0,"num_object_copies":32223,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":10741,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10741,"num_write_kb":687424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003285"},"pg_stats":[{"pgid":"3.1a","version":"59'1","reported_seq":257,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.344959+0000","last_change":"2023-09-23T19:38:43.897548+0000","last_active":"2023-09-23T19:57:10.344959+0000","last_peered":"2023-09-23T19:57:10.344959+0000","last_clean":"2023-09-23T19:57:10.344959+0000","last_became_active":"2023-09-23T19:38:43.896868+0000","last_became_peered":"2023-09-23T19:38:43.896868+0000","last_unstale":"2023-09-23T19:57:10.344959+0000","last_undegraded":"2023-09-23T19:57:10.344959+0000","last_fullsized":"2023-09-23T19:57:10.344959+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":223,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.335384+0000","last_change":"2023-09-23T19:32:49.731128+0000","last_active":"2023-09-23T19:57:10.335384+0000","last_peered":"2023-09-23T19:57:10.335384+0000","last_clean":"2023-09-23T19:57:10.335384+0000","last_became_active":"2023-09-23T19:32:49.730757+0000","last_became_peered":"2023-09-23T19:32:49.730757+0000","last_unstale":"2023-09-23T19:57:10.335384+0000","last_undegraded":"2023-09-23T19:57:10.335384+0000","last_fullsized":"2023-09-23T19:57:10.335384+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:40.183605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":212,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.334980+0000","last_change":"2023-09-23T19:28:07.890462+0000","last_active":"2023-09-23T19:57:10.334980+0000","last_peered":"2023-09-23T19:57:10.334980+0000","last_clean":"2023-09-23T19:57:10.334980+0000","last_became_active":"2023-09-23T19:28:07.890084+0000","last_became_peered":"2023-09-23T19:28:07.890084+0000","last_unstale":"2023-09-23T19:57:10.334980+0000","last_undegraded":"2023-09-23T19:57:10.334980+0000","last_fullsized":"2023-09-23T19:57:10.334980+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:50.458218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":147,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.429581+0000","last_change":"2023-09-23T19:42:55.772143+0000","last_active":"2023-09-23T19:57:10.429581+0000","last_peered":"2023-09-23T19:57:10.429581+0000","last_clean":"2023-09-23T19:57:10.429581+0000","last_became_active":"2023-09-23T19:42:55.771813+0000","last_became_peered":"2023-09-23T19:42:55.771813+0000","last_unstale":"2023-09-23T19:57:10.429581+0000","last_undegraded":"2023-09-23T19:57:10.429581+0000","last_fullsized":"2023-09-23T19:57:10.429581+0000","mapping_epoch":188,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":242,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-09-23T19:57:21.791104+0000","last_change":"2023-09-23T19:32:49.729961+0000","last_active":"2023-09-23T19:57:21.791104+0000","last_peered":"2023-09-23T19:57:21.791104+0000","last_clean":"2023-09-23T19:57:21.791104+0000","last_became_active":"2023-09-23T19:32:49.729251+0000","last_became_peered":"2023-09-23T19:32:49.729251+0000","last_unstale":"2023-09-23T19:57:21.791104+0000","last_undegraded":"2023-09-23T19:57:21.791104+0000","last_fullsized":"2023-09-23T19:57:21.791104+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:20:56.595732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":226,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.928780+0000","last_change":"2023-09-23T19:22:42.150582+0000","last_active":"2023-09-23T19:57:10.928780+0000","last_peered":"2023-09-23T19:57:10.928780+0000","last_clean":"2023-09-23T19:57:10.928780+0000","last_became_active":"2023-09-23T19:22:42.150229+0000","last_became_peered":"2023-09-23T19:22:42.150229+0000","last_unstale":"2023-09-23T19:57:10.928780+0000","last_undegraded":"2023-09-23T19:57:10.928780+0000","last_fullsized":"2023-09-23T19:57:10.928780+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":205,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.928962+0000","last_change":"2023-09-23T19:47:56.793944+0000","last_active":"2023-09-23T19:57:10.928962+0000","last_peered":"2023-09-23T19:57:10.928962+0000","last_clean":"2023-09-23T19:57:10.928962+0000","last_became_active":"2023-09-23T19:47:56.793618+0000","last_became_peered":"2023-09-23T19:47:56.793618+0000","last_unstale":"2023-09-23T19:57:10.928962+0000","last_undegraded":"2023-09-23T19:57:10.928962+0000","last_fullsized":"2023-09-23T19:57:10.928962+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":224,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.335053+0000","last_change":"2023-09-23T19:32:49.729765+0000","last_active":"2023-09-23T19:57:10.335053+0000","last_peered":"2023-09-23T19:57:10.335053+0000","last_clean":"2023-09-23T19:57:10.335053+0000","last_became_active":"2023-09-23T19:32:49.729007+0000","last_became_peered":"2023-09-23T19:32:49.729007+0000","last_unstale":"2023-09-23T19:57:10.335053+0000","last_undegraded":"2023-09-23T19:57:10.335053+0000","last_fullsized":"2023-09-23T19:57:10.335053+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:59:31.183753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":213,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.929658+0000","last_change":"2023-09-23T19:47:56.791464+0000","last_active":"2023-09-23T19:57:10.929658+0000","last_peered":"2023-09-23T19:57:10.929658+0000","last_clean":"2023-09-23T19:57:10.929658+0000","last_became_active":"2023-09-23T19:47:56.791144+0000","last_became_peered":"2023-09-23T19:47:56.791144+0000","last_unstale":"2023-09-23T19:57:10.929658+0000","last_undegraded":"2023-09-23T19:57:10.929658+0000","last_fullsized":"2023-09-23T19:57:10.929658+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":201,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.428916+0000","last_change":"2023-09-23T19:32:49.736864+0000","last_active":"2023-09-23T19:57:10.428916+0000","last_peered":"2023-09-23T19:57:10.428916+0000","last_clean":"2023-09-23T19:57:10.428916+0000","last_became_active":"2023-09-23T19:32:49.736523+0000","last_became_peered":"2023-09-23T19:32:49.736523+0000","last_unstale":"2023-09-23T19:57:10.428916+0000","last_undegraded":"2023-09-23T19:57:10.428916+0000","last_fullsized":"2023-09-23T19:57:10.428916+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":230,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.429704+0000","last_change":"2023-09-23T19:28:07.892835+0000","last_active":"2023-09-23T19:57:10.429704+0000","last_peered":"2023-09-23T19:57:10.429704+0000","last_clean":"2023-09-23T19:57:10.429704+0000","last_became_active":"2023-09-23T19:28:07.892338+0000","last_became_peered":"2023-09-23T19:28:07.892338+0000","last_unstale":"2023-09-23T19:57:10.429704+0000","last_undegraded":"2023-09-23T19:57:10.429704+0000","last_fullsized":"2023-09-23T19:57:10.429704+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":228,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-09-23T19:57:21.784324+0000","last_change":"2023-09-23T19:38:43.890137+0000","last_active":"2023-09-23T19:57:21.784324+0000","last_peered":"2023-09-23T19:57:21.784324+0000","last_clean":"2023-09-23T19:57:21.784324+0000","last_became_active":"2023-09-23T19:38:43.889800+0000","last_became_peered":"2023-09-23T19:38:43.889800+0000","last_unstale":"2023-09-23T19:57:21.784324+0000","last_undegraded":"2023-09-23T19:57:21.784324+0000","last_fullsized":"2023-09-23T19:57:21.784324+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:59:20.468913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'23","reported_seq":234,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.345837+0000","last_change":"2023-09-23T19:05:52.107727+0000","last_active":"2023-09-23T19:57:10.345837+0000","last_peered":"2023-09-23T19:57:10.345837+0000","last_clean":"2023-09-23T19:57:10.345837+0000","last_became_active":"2023-09-23T19:05:52.107378+0000","last_became_peered":"2023-09-23T19:05:52.107378+0000","last_unstale":"2023-09-23T19:57:10.345837+0000","last_undegraded":"2023-09-23T19:57:10.345837+0000","last_fullsized":"2023-09-23T19:57:10.345837+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":225,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.345130+0000","last_change":"2023-09-23T19:22:42.148497+0000","last_active":"2023-09-23T19:57:10.345130+0000","last_peered":"2023-09-23T19:57:10.345130+0000","last_clean":"2023-09-23T19:57:10.345130+0000","last_became_active":"2023-09-23T19:22:42.148147+0000","last_became_peered":"2023-09-23T19:22:42.148147+0000","last_unstale":"2023-09-23T19:57:10.345130+0000","last_undegraded":"2023-09-23T19:57:10.345130+0000","last_fullsized":"2023-09-23T19:57:10.345130+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":244,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.429323+0000","last_change":"2023-09-23T19:38:43.897378+0000","last_active":"2023-09-23T19:57:10.429323+0000","last_peered":"2023-09-23T19:57:10.429323+0000","last_clean":"2023-09-23T19:57:10.429323+0000","last_became_active":"2023-09-23T19:38:43.897012+0000","last_became_peered":"2023-09-23T19:38:43.897012+0000","last_unstale":"2023-09-23T19:57:10.429323+0000","last_undegraded":"2023-09-23T19:57:10.429323+0000","last_fullsized":"2023-09-23T19:57:10.429323+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":224,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.336041+0000","last_change":"2023-09-23T19:32:49.730549+0000","last_active":"2023-09-23T19:57:10.336041+0000","last_peered":"2023-09-23T19:57:10.336041+0000","last_clean":"2023-09-23T19:57:10.336041+0000","last_became_active":"2023-09-23T19:32:49.730105+0000","last_became_peered":"2023-09-23T19:32:49.730105+0000","last_unstale":"2023-09-23T19:57:10.336041+0000","last_undegraded":"2023-09-23T19:57:10.336041+0000","last_fullsized":"2023-09-23T19:57:10.336041+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:49.751893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":147,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.428813+0000","last_change":"2023-09-23T19:42:55.772691+0000","last_active":"2023-09-23T19:57:10.428813+0000","last_peered":"2023-09-23T19:57:10.428813+0000","last_clean":"2023-09-23T19:57:10.428813+0000","last_became_active":"2023-09-23T19:42:55.772309+0000","last_became_peered":"2023-09-23T19:42:55.772309+0000","last_unstale":"2023-09-23T19:57:10.428813+0000","last_undegraded":"2023-09-23T19:57:10.428813+0000","last_fullsized":"2023-09-23T19:57:10.428813+0000","mapping_epoch":188,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":212,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.929251+0000","last_change":"2023-09-23T19:47:56.794658+0000","last_active":"2023-09-23T19:57:10.929251+0000","last_peered":"2023-09-23T19:57:10.929251+0000","last_clean":"2023-09-23T19:57:10.929251+0000","last_became_active":"2023-09-23T19:47:56.793633+0000","last_became_peered":"2023-09-23T19:47:56.793633+0000","last_unstale":"2023-09-23T19:57:10.929251+0000","last_undegraded":"2023-09-23T19:57:10.929251+0000","last_fullsized":"2023-09-23T19:57:10.929251+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":212,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.429333+0000","last_change":"2023-09-23T19:05:52.114186+0000","last_active":"2023-09-23T19:57:10.429333+0000","last_peered":"2023-09-23T19:57:10.429333+0000","last_clean":"2023-09-23T19:57:10.429333+0000","last_became_active":"2023-09-23T19:05:52.113635+0000","last_became_peered":"2023-09-23T19:05:52.113635+0000","last_unstale":"2023-09-23T19:57:10.429333+0000","last_undegraded":"2023-09-23T19:57:10.429333+0000","last_fullsized":"2023-09-23T19:57:10.429333+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":147,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.429603+0000","last_change":"2023-09-23T19:42:55.772912+0000","last_active":"2023-09-23T19:57:10.429603+0000","last_peered":"2023-09-23T19:57:10.429603+0000","last_clean":"2023-09-23T19:57:10.429603+0000","last_became_active":"2023-09-23T19:42:55.772434+0000","last_became_peered":"2023-09-23T19:42:55.772434+0000","last_unstale":"2023-09-23T19:57:10.429603+0000","last_undegraded":"2023-09-23T19:57:10.429603+0000","last_fullsized":"2023-09-23T19:57:10.429603+0000","mapping_epoch":188,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":235,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.927764+0000","last_change":"2023-09-23T19:18:09.010195+0000","last_active":"2023-09-23T19:57:10.927764+0000","last_peered":"2023-09-23T19:57:10.927764+0000","last_clean":"2023-09-23T19:57:10.927764+0000","last_became_active":"2023-09-23T19:18:09.009687+0000","last_became_peered":"2023-09-23T19:18:09.009687+0000","last_unstale":"2023-09-23T19:57:10.927764+0000","last_undegraded":"2023-09-23T19:57:10.927764+0000","last_fullsized":"2023-09-23T19:57:10.927764+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":234,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-09-23T19:57:21.777411+0000","last_change":"2023-09-23T19:28:07.885265+0000","last_active":"2023-09-23T19:57:21.777411+0000","last_peered":"2023-09-23T19:57:21.777411+0000","last_clean":"2023-09-23T19:57:21.777411+0000","last_became_active":"2023-09-23T19:28:07.884353+0000","last_became_peered":"2023-09-23T19:28:07.884353+0000","last_unstale":"2023-09-23T19:57:21.777411+0000","last_undegraded":"2023-09-23T19:57:21.777411+0000","last_fullsized":"2023-09-23T19:57:21.777411+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:56.325352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.2","version":"261'3742","reported_seq":3755,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-09-23T19:57:55.218355+0000","last_change":"2023-09-23T19:56:50.550024+0000","last_active":"2023-09-23T19:57:55.218355+0000","last_peered":"2023-09-23T19:57:55.218355+0000","last_clean":"2023-09-23T19:57:55.218355+0000","last_became_active":"2023-09-23T19:56:50.548890+0000","last_became_peered":"2023-09-23T19:56:50.548890+0000","last_unstale":"2023-09-23T19:57:55.218355+0000","last_undegraded":"2023-09-23T19:57:55.218355+0000","last_fullsized":"2023-09-23T19:57:55.218355+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":256,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_clean_scrub_stamp":"2023-09-23T19:56:49.329294+0000","objects_scrubbed":0,"log_size":3742,"log_dups_size":0,"ondisk_log_size":3742,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":245235712,"num_objects":3742,"num_object_clones":0,"num_object_copies":11226,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3742,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3742,"num_write_kb":239488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":228,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.430038+0000","last_change":"2023-09-23T19:18:09.035116+0000","last_active":"2023-09-23T19:57:10.430038+0000","last_peered":"2023-09-23T19:57:10.430038+0000","last_clean":"2023-09-23T19:57:10.430038+0000","last_became_active":"2023-09-23T19:18:09.034416+0000","last_became_peered":"2023-09-23T19:18:09.034416+0000","last_unstale":"2023-09-23T19:57:10.430038+0000","last_undegraded":"2023-09-23T19:57:10.430038+0000","last_fullsized":"2023-09-23T19:57:10.430038+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":197,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.929044+0000","last_change":"2023-09-23T19:47:56.790227+0000","last_active":"2023-09-23T19:57:10.929044+0000","last_peered":"2023-09-23T19:57:10.929044+0000","last_clean":"2023-09-23T19:57:10.929044+0000","last_became_active":"2023-09-23T19:47:56.789727+0000","last_became_peered":"2023-09-23T19:47:56.789727+0000","last_unstale":"2023-09-23T19:57:10.929044+0000","last_undegraded":"2023-09-23T19:57:10.929044+0000","last_fullsized":"2023-09-23T19:57:10.929044+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.3","version":"261'3627","reported_seq":3641,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-09-23T19:57:55.390913+0000","last_change":"2023-09-23T19:56:50.486771+0000","last_active":"2023-09-23T19:57:55.390913+0000","last_peered":"2023-09-23T19:57:55.390913+0000","last_clean":"2023-09-23T19:57:55.390913+0000","last_became_active":"2023-09-23T19:56:50.486200+0000","last_became_peered":"2023-09-23T19:56:50.486200+0000","last_unstale":"2023-09-23T19:57:55.390913+0000","last_undegraded":"2023-09-23T19:57:55.390913+0000","last_fullsized":"2023-09-23T19:57:55.390913+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":256,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_clean_scrub_stamp":"2023-09-23T19:56:49.329294+0000","objects_scrubbed":0,"log_size":3627,"log_dups_size":0,"ondisk_log_size":3627,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:35.217636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237699072,"num_objects":3627,"num_object_clones":0,"num_object_copies":10881,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3627,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3627,"num_write_kb":232128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":229,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.429732+0000","last_change":"2023-09-23T19:22:42.151134+0000","last_active":"2023-09-23T19:57:10.429732+0000","last_peered":"2023-09-23T19:57:10.429732+0000","last_clean":"2023-09-23T19:57:10.429732+0000","last_became_active":"2023-09-23T19:22:42.150553+0000","last_became_peered":"2023-09-23T19:22:42.150553+0000","last_unstale":"2023-09-23T19:57:10.429732+0000","last_undegraded":"2023-09-23T19:57:10.429732+0000","last_fullsized":"2023-09-23T19:57:10.429732+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":240,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.345182+0000","last_change":"2023-09-23T19:28:07.899262+0000","last_active":"2023-09-23T19:57:10.345182+0000","last_peered":"2023-09-23T19:57:10.345182+0000","last_clean":"2023-09-23T19:57:10.345182+0000","last_became_active":"2023-09-23T19:28:07.898848+0000","last_became_peered":"2023-09-23T19:28:07.898848+0000","last_unstale":"2023-09-23T19:57:10.345182+0000","last_undegraded":"2023-09-23T19:57:10.345182+0000","last_fullsized":"2023-09-23T19:57:10.345182+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.0","version":"262'3736","reported_seq":3751,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T19:57:57.310944+0000","last_change":"2023-09-23T19:56:50.353925+0000","last_active":"2023-09-23T19:57:57.310944+0000","last_peered":"2023-09-23T19:57:57.310944+0000","last_clean":"2023-09-23T19:57:57.310944+0000","last_became_active":"2023-09-23T19:56:50.353546+0000","last_became_peered":"2023-09-23T19:56:50.353546+0000","last_unstale":"2023-09-23T19:57:57.310944+0000","last_undegraded":"2023-09-23T19:57:57.310944+0000","last_fullsized":"2023-09-23T19:57:57.310944+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":256,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_clean_scrub_stamp":"2023-09-23T19:56:49.329294+0000","objects_scrubbed":0,"log_size":3736,"log_dups_size":0,"ondisk_log_size":3736,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:12:56.362116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":244842496,"num_objects":3736,"num_object_clones":0,"num_object_copies":11208,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3736,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3736,"num_write_kb":239104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":213,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.429398+0000","last_change":"2023-09-23T19:05:52.114108+0000","last_active":"2023-09-23T19:57:10.429398+0000","last_peered":"2023-09-23T19:57:10.429398+0000","last_clean":"2023-09-23T19:57:10.429398+0000","last_became_active":"2023-09-23T19:05:52.113609+0000","last_became_peered":"2023-09-23T19:05:52.113609+0000","last_unstale":"2023-09-23T19:57:10.429398+0000","last_undegraded":"2023-09-23T19:57:10.429398+0000","last_fullsized":"2023-09-23T19:57:10.429398+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":226,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.929908+0000","last_change":"2023-09-23T19:22:42.156825+0000","last_active":"2023-09-23T19:57:10.929908+0000","last_peered":"2023-09-23T19:57:10.929908+0000","last_clean":"2023-09-23T19:57:10.929908+0000","last_became_active":"2023-09-23T19:22:42.156474+0000","last_became_peered":"2023-09-23T19:22:42.156474+0000","last_unstale":"2023-09-23T19:57:10.929908+0000","last_undegraded":"2023-09-23T19:57:10.929908+0000","last_fullsized":"2023-09-23T19:57:10.929908+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"261'3624","reported_seq":3638,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-09-23T19:57:55.359512+0000","last_change":"2023-09-23T19:56:50.543702+0000","last_active":"2023-09-23T19:57:55.359512+0000","last_peered":"2023-09-23T19:57:55.359512+0000","last_clean":"2023-09-23T19:57:55.359512+0000","last_became_active":"2023-09-23T19:56:50.543330+0000","last_became_peered":"2023-09-23T19:56:50.543330+0000","last_unstale":"2023-09-23T19:57:55.359512+0000","last_undegraded":"2023-09-23T19:57:55.359512+0000","last_fullsized":"2023-09-23T19:57:55.359512+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":256,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_clean_scrub_stamp":"2023-09-23T19:56:49.329294+0000","objects_scrubbed":0,"log_size":3624,"log_dups_size":0,"ondisk_log_size":3624,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:41:26.115634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237502464,"num_objects":3624,"num_object_clones":0,"num_object_copies":10872,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3624,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3624,"num_write_kb":231936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":213,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.928725+0000","last_change":"2023-09-23T19:47:56.794638+0000","last_active":"2023-09-23T19:57:10.928725+0000","last_peered":"2023-09-23T19:57:10.928725+0000","last_clean":"2023-09-23T19:57:10.928725+0000","last_became_active":"2023-09-23T19:47:56.794305+0000","last_became_peered":"2023-09-23T19:47:56.794305+0000","last_unstale":"2023-09-23T19:57:10.928725+0000","last_undegraded":"2023-09-23T19:57:10.928725+0000","last_fullsized":"2023-09-23T19:57:10.928725+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":230,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-09-23T19:57:21.791410+0000","last_change":"2023-09-23T19:22:42.135572+0000","last_active":"2023-09-23T19:57:21.791410+0000","last_peered":"2023-09-23T19:57:21.791410+0000","last_clean":"2023-09-23T19:57:21.791410+0000","last_became_active":"2023-09-23T19:22:42.135237+0000","last_became_peered":"2023-09-23T19:22:42.135237+0000","last_unstale":"2023-09-23T19:57:21.791410+0000","last_undegraded":"2023-09-23T19:57:21.791410+0000","last_fullsized":"2023-09-23T19:57:21.791410+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:35.838718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.6","version":"262'3768","reported_seq":3780,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T19:57:57.293697+0000","last_change":"2023-09-23T19:56:50.350294+0000","last_active":"2023-09-23T19:57:57.293697+0000","last_peered":"2023-09-23T19:57:57.293697+0000","last_clean":"2023-09-23T19:57:57.293697+0000","last_became_active":"2023-09-23T19:56:50.349934+0000","last_became_peered":"2023-09-23T19:56:50.349934+0000","last_unstale":"2023-09-23T19:57:57.293697+0000","last_undegraded":"2023-09-23T19:57:57.293697+0000","last_fullsized":"2023-09-23T19:57:57.293697+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":256,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_clean_scrub_stamp":"2023-09-23T19:56:49.329294+0000","objects_scrubbed":0,"log_size":3768,"log_dups_size":0,"ondisk_log_size":3768,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:52.927510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246939648,"num_objects":3768,"num_object_clones":0,"num_object_copies":11304,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3768,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3768,"num_write_kb":241152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":94,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.344971+0000","last_change":"2023-09-23T19:42:55.771907+0000","last_active":"2023-09-23T19:57:10.344971+0000","last_peered":"2023-09-23T19:57:10.344971+0000","last_clean":"2023-09-23T19:57:10.344971+0000","last_became_active":"2023-09-23T19:42:55.771503+0000","last_became_peered":"2023-09-23T19:42:55.771503+0000","last_unstale":"2023-09-23T19:57:10.344971+0000","last_undegraded":"2023-09-23T19:57:10.344971+0000","last_fullsized":"2023-09-23T19:57:10.344971+0000","mapping_epoch":188,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":225,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.345876+0000","last_change":"2023-09-23T19:28:07.896803+0000","last_active":"2023-09-23T19:57:10.345876+0000","last_peered":"2023-09-23T19:57:10.345876+0000","last_clean":"2023-09-23T19:57:10.345876+0000","last_became_active":"2023-09-23T19:28:07.896480+0000","last_became_peered":"2023-09-23T19:28:07.896480+0000","last_unstale":"2023-09-23T19:57:10.345876+0000","last_undegraded":"2023-09-23T19:57:10.345876+0000","last_fullsized":"2023-09-23T19:57:10.345876+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.7","version":"261'3491","reported_seq":3504,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-09-23T19:57:54.442399+0000","last_change":"2023-09-23T19:56:50.357844+0000","last_active":"2023-09-23T19:57:54.442399+0000","last_peered":"2023-09-23T19:57:54.442399+0000","last_clean":"2023-09-23T19:57:54.442399+0000","last_became_active":"2023-09-23T19:56:50.357465+0000","last_became_peered":"2023-09-23T19:56:50.357465+0000","last_unstale":"2023-09-23T19:57:54.442399+0000","last_undegraded":"2023-09-23T19:57:54.442399+0000","last_fullsized":"2023-09-23T19:57:54.442399+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":256,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_clean_scrub_stamp":"2023-09-23T19:56:49.329294+0000","objects_scrubbed":0,"log_size":3491,"log_dups_size":0,"ondisk_log_size":3491,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228786176,"num_objects":3491,"num_object_clones":0,"num_object_copies":10473,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3491,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3491,"num_write_kb":223424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":227,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.929793+0000","last_change":"2023-09-23T19:22:42.156582+0000","last_active":"2023-09-23T19:57:10.929793+0000","last_peered":"2023-09-23T19:57:10.929793+0000","last_clean":"2023-09-23T19:57:10.929793+0000","last_became_active":"2023-09-23T19:22:42.156251+0000","last_became_peered":"2023-09-23T19:22:42.156251+0000","last_unstale":"2023-09-23T19:57:10.929793+0000","last_undegraded":"2023-09-23T19:57:10.929793+0000","last_fullsized":"2023-09-23T19:57:10.929793+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":255,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.345916+0000","last_change":"2023-09-23T19:38:43.891593+0000","last_active":"2023-09-23T19:57:10.345916+0000","last_peered":"2023-09-23T19:57:10.345916+0000","last_clean":"2023-09-23T19:57:10.345916+0000","last_became_active":"2023-09-23T19:38:43.891063+0000","last_became_peered":"2023-09-23T19:38:43.891063+0000","last_unstale":"2023-09-23T19:57:10.345916+0000","last_undegraded":"2023-09-23T19:57:10.345916+0000","last_fullsized":"2023-09-23T19:57:10.345916+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.4","version":"261'3699","reported_seq":3712,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-09-23T19:57:55.189953+0000","last_change":"2023-09-23T19:56:50.492916+0000","last_active":"2023-09-23T19:57:55.189953+0000","last_peered":"2023-09-23T19:57:55.189953+0000","last_clean":"2023-09-23T19:57:55.189953+0000","last_became_active":"2023-09-23T19:56:50.492066+0000","last_became_peered":"2023-09-23T19:56:50.492066+0000","last_unstale":"2023-09-23T19:57:55.189953+0000","last_undegraded":"2023-09-23T19:57:55.189953+0000","last_fullsized":"2023-09-23T19:57:55.189953+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":256,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_clean_scrub_stamp":"2023-09-23T19:56:49.329294+0000","objects_scrubbed":0,"log_size":3699,"log_dups_size":0,"ondisk_log_size":3699,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":242417664,"num_objects":3699,"num_object_clones":0,"num_object_copies":11097,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3699,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3699,"num_write_kb":236736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":241,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.344662+0000","last_change":"2023-09-23T19:38:43.892972+0000","last_active":"2023-09-23T19:57:10.344662+0000","last_peered":"2023-09-23T19:57:10.344662+0000","last_clean":"2023-09-23T19:57:10.344662+0000","last_became_active":"2023-09-23T19:38:43.892451+0000","last_became_peered":"2023-09-23T19:38:43.892451+0000","last_unstale":"2023-09-23T19:57:10.344662+0000","last_undegraded":"2023-09-23T19:57:10.344662+0000","last_fullsized":"2023-09-23T19:57:10.344662+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":225,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.345893+0000","last_change":"2023-09-23T19:22:42.149670+0000","last_active":"2023-09-23T19:57:10.345893+0000","last_peered":"2023-09-23T19:57:10.345893+0000","last_clean":"2023-09-23T19:57:10.345893+0000","last_became_active":"2023-09-23T19:22:42.149022+0000","last_became_peered":"2023-09-23T19:22:42.149022+0000","last_unstale":"2023-09-23T19:57:10.345893+0000","last_undegraded":"2023-09-23T19:57:10.345893+0000","last_fullsized":"2023-09-23T19:57:10.345893+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.5","version":"262'3790","reported_seq":3805,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T19:57:57.775502+0000","last_change":"2023-09-23T19:56:50.353313+0000","last_active":"2023-09-23T19:57:57.775502+0000","last_peered":"2023-09-23T19:57:57.775502+0000","last_clean":"2023-09-23T19:57:57.775502+0000","last_became_active":"2023-09-23T19:56:50.353091+0000","last_became_peered":"2023-09-23T19:56:50.353091+0000","last_unstale":"2023-09-23T19:57:57.775502+0000","last_undegraded":"2023-09-23T19:57:57.775502+0000","last_fullsized":"2023-09-23T19:57:57.775502+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":256,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_clean_scrub_stamp":"2023-09-23T19:56:49.329294+0000","objects_scrubbed":0,"log_size":3790,"log_dups_size":0,"ondisk_log_size":3790,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:29.940884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":248381440,"num_objects":3790,"num_object_clones":0,"num_object_copies":11370,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3790,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3790,"num_write_kb":242560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":226,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.334816+0000","last_change":"2023-09-23T19:32:49.729244+0000","last_active":"2023-09-23T19:57:10.334816+0000","last_peered":"2023-09-23T19:57:10.334816+0000","last_clean":"2023-09-23T19:57:10.334816+0000","last_became_active":"2023-09-23T19:32:49.728810+0000","last_became_peered":"2023-09-23T19:32:49.728810+0000","last_unstale":"2023-09-23T19:57:10.334816+0000","last_undegraded":"2023-09-23T19:57:10.334816+0000","last_fullsized":"2023-09-23T19:57:10.334816+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:57.540826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":213,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.335937+0000","last_change":"2023-09-23T19:28:07.889095+0000","last_active":"2023-09-23T19:57:10.335937+0000","last_peered":"2023-09-23T19:57:10.335937+0000","last_clean":"2023-09-23T19:57:10.335937+0000","last_became_active":"2023-09-23T19:28:07.888742+0000","last_became_peered":"2023-09-23T19:28:07.888742+0000","last_unstale":"2023-09-23T19:57:10.335937+0000","last_undegraded":"2023-09-23T19:57:10.335937+0000","last_fullsized":"2023-09-23T19:57:10.335937+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:43.929488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.8","version":"262'3802","reported_seq":3815,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T19:57:57.774346+0000","last_change":"2023-09-23T19:56:50.347118+0000","last_active":"2023-09-23T19:57:57.774346+0000","last_peered":"2023-09-23T19:57:57.774346+0000","last_clean":"2023-09-23T19:57:57.774346+0000","last_became_active":"2023-09-23T19:56:50.346834+0000","last_became_peered":"2023-09-23T19:56:50.346834+0000","last_unstale":"2023-09-23T19:57:57.774346+0000","last_undegraded":"2023-09-23T19:57:57.774346+0000","last_fullsized":"2023-09-23T19:57:57.774346+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":256,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_clean_scrub_stamp":"2023-09-23T19:56:49.329294+0000","objects_scrubbed":0,"log_size":3802,"log_dups_size":0,"ondisk_log_size":3802,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:19.591681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":249167872,"num_objects":3802,"num_object_clones":0,"num_object_copies":11406,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3802,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3802,"num_write_kb":243328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":247,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.335931+0000","last_change":"2023-09-23T19:22:42.143289+0000","last_active":"2023-09-23T19:57:10.335931+0000","last_peered":"2023-09-23T19:57:10.335931+0000","last_clean":"2023-09-23T19:57:10.335931+0000","last_became_active":"2023-09-23T19:22:42.142656+0000","last_became_peered":"2023-09-23T19:22:42.142656+0000","last_unstale":"2023-09-23T19:57:10.335931+0000","last_undegraded":"2023-09-23T19:57:10.335931+0000","last_fullsized":"2023-09-23T19:57:10.335931+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:38.467441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":243,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.929377+0000","last_change":"2023-09-23T19:47:56.794437+0000","last_active":"2023-09-23T19:57:10.929377+0000","last_peered":"2023-09-23T19:57:10.929377+0000","last_clean":"2023-09-23T19:57:10.929377+0000","last_became_active":"2023-09-23T19:47:56.794118+0000","last_became_peered":"2023-09-23T19:47:56.794118+0000","last_unstale":"2023-09-23T19:57:10.929377+0000","last_undegraded":"2023-09-23T19:57:10.929377+0000","last_fullsized":"2023-09-23T19:57:10.929377+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":234,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-09-23T19:57:27.350072+0000","last_change":"2023-09-23T19:38:43.898519+0000","last_active":"2023-09-23T19:57:27.350072+0000","last_peered":"2023-09-23T19:57:27.350072+0000","last_clean":"2023-09-23T19:57:27.350072+0000","last_became_active":"2023-09-23T19:38:43.896814+0000","last_became_peered":"2023-09-23T19:38:43.896814+0000","last_unstale":"2023-09-23T19:57:27.350072+0000","last_undegraded":"2023-09-23T19:57:27.350072+0000","last_fullsized":"2023-09-23T19:57:27.350072+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:40.465038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.e","version":"261'3707","reported_seq":3720,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-09-23T19:57:55.379122+0000","last_change":"2023-09-23T19:56:50.358245+0000","last_active":"2023-09-23T19:57:55.379122+0000","last_peered":"2023-09-23T19:57:55.379122+0000","last_clean":"2023-09-23T19:57:55.379122+0000","last_became_active":"2023-09-23T19:56:50.357883+0000","last_became_peered":"2023-09-23T19:56:50.357883+0000","last_unstale":"2023-09-23T19:57:55.379122+0000","last_undegraded":"2023-09-23T19:57:55.379122+0000","last_fullsized":"2023-09-23T19:57:55.379122+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":256,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_clean_scrub_stamp":"2023-09-23T19:56:49.329294+0000","objects_scrubbed":0,"log_size":3707,"log_dups_size":0,"ondisk_log_size":3707,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":242941952,"num_objects":3707,"num_object_clones":0,"num_object_copies":11121,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3707,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3707,"num_write_kb":237248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":147,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.335248+0000","last_change":"2023-09-23T19:43:38.612700+0000","last_active":"2023-09-23T19:57:10.335248+0000","last_peered":"2023-09-23T19:57:10.335248+0000","last_clean":"2023-09-23T19:57:10.335248+0000","last_became_active":"2023-09-23T19:43:38.612424+0000","last_became_peered":"2023-09-23T19:43:38.612424+0000","last_unstale":"2023-09-23T19:57:10.335248+0000","last_undegraded":"2023-09-23T19:57:10.335248+0000","last_fullsized":"2023-09-23T19:57:10.335248+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:24.833490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":226,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.345029+0000","last_change":"2023-09-23T19:18:09.017525+0000","last_active":"2023-09-23T19:57:10.345029+0000","last_peered":"2023-09-23T19:57:10.345029+0000","last_clean":"2023-09-23T19:57:10.345029+0000","last_became_active":"2023-09-23T19:18:09.017193+0000","last_became_peered":"2023-09-23T19:18:09.017193+0000","last_unstale":"2023-09-23T19:57:10.345029+0000","last_undegraded":"2023-09-23T19:57:10.345029+0000","last_fullsized":"2023-09-23T19:57:10.345029+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":244,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.335813+0000","last_change":"2023-09-23T19:38:43.877088+0000","last_active":"2023-09-23T19:57:10.335813+0000","last_peered":"2023-09-23T19:57:10.335813+0000","last_clean":"2023-09-23T19:57:10.335813+0000","last_became_active":"2023-09-23T19:38:43.876719+0000","last_became_peered":"2023-09-23T19:38:43.876719+0000","last_unstale":"2023-09-23T19:57:10.335813+0000","last_undegraded":"2023-09-23T19:57:10.335813+0000","last_fullsized":"2023-09-23T19:57:10.335813+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:05.423256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.d","version":"261'3628","reported_seq":3641,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-09-23T19:57:55.241581+0000","last_change":"2023-09-23T19:56:50.359619+0000","last_active":"2023-09-23T19:57:55.241581+0000","last_peered":"2023-09-23T19:57:55.241581+0000","last_clean":"2023-09-23T19:57:55.241581+0000","last_became_active":"2023-09-23T19:56:50.359198+0000","last_became_peered":"2023-09-23T19:56:50.359198+0000","last_unstale":"2023-09-23T19:57:55.241581+0000","last_undegraded":"2023-09-23T19:57:55.241581+0000","last_fullsized":"2023-09-23T19:57:55.241581+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":256,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_clean_scrub_stamp":"2023-09-23T19:56:49.329294+0000","objects_scrubbed":0,"log_size":3628,"log_dups_size":0,"ondisk_log_size":3628,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237764608,"num_objects":3628,"num_object_clones":0,"num_object_copies":10884,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3628,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3628,"num_write_kb":232192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":214,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-09-23T19:57:27.350051+0000","last_change":"2023-09-23T19:38:43.883663+0000","last_active":"2023-09-23T19:57:27.350051+0000","last_peered":"2023-09-23T19:57:27.350051+0000","last_clean":"2023-09-23T19:57:27.350051+0000","last_became_active":"2023-09-23T19:38:43.882421+0000","last_became_peered":"2023-09-23T19:38:43.882421+0000","last_unstale":"2023-09-23T19:57:27.350051+0000","last_undegraded":"2023-09-23T19:57:27.350051+0000","last_fullsized":"2023-09-23T19:57:27.350051+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:49:01.607659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":230,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.334826+0000","last_change":"2023-09-23T19:38:43.890600+0000","last_active":"2023-09-23T19:57:10.334826+0000","last_peered":"2023-09-23T19:57:10.334826+0000","last_clean":"2023-09-23T19:57:10.334826+0000","last_became_active":"2023-09-23T19:38:43.890253+0000","last_became_peered":"2023-09-23T19:38:43.890253+0000","last_unstale":"2023-09-23T19:57:10.334826+0000","last_undegraded":"2023-09-23T19:57:10.334826+0000","last_fullsized":"2023-09-23T19:57:10.334826+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:46.751120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":242,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:20.362397+0000","last_change":"2023-09-23T19:22:42.143733+0000","last_active":"2023-09-23T19:57:20.362397+0000","last_peered":"2023-09-23T19:57:20.362397+0000","last_clean":"2023-09-23T19:57:20.362397+0000","last_became_active":"2023-09-23T19:22:42.143077+0000","last_became_peered":"2023-09-23T19:22:42.143077+0000","last_unstale":"2023-09-23T19:57:20.362397+0000","last_undegraded":"2023-09-23T19:57:20.362397+0000","last_fullsized":"2023-09-23T19:57:20.362397+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:47:33.086994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.f","version":"261'3641","reported_seq":3654,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-09-23T19:57:54.485871+0000","last_change":"2023-09-23T19:56:50.493491+0000","last_active":"2023-09-23T19:57:54.485871+0000","last_peered":"2023-09-23T19:57:54.485871+0000","last_clean":"2023-09-23T19:57:54.485871+0000","last_became_active":"2023-09-23T19:56:50.492405+0000","last_became_peered":"2023-09-23T19:56:50.492405+0000","last_unstale":"2023-09-23T19:57:54.485871+0000","last_undegraded":"2023-09-23T19:57:54.485871+0000","last_fullsized":"2023-09-23T19:57:54.485871+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":256,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_clean_scrub_stamp":"2023-09-23T19:56:49.329294+0000","objects_scrubbed":0,"log_size":3641,"log_dups_size":0,"ondisk_log_size":3641,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":238616576,"num_objects":3641,"num_object_clones":0,"num_object_copies":10923,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3641,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3641,"num_write_kb":233024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":232,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.928116+0000","last_change":"2023-09-23T19:47:56.791259+0000","last_active":"2023-09-23T19:57:10.928116+0000","last_peered":"2023-09-23T19:57:10.928116+0000","last_clean":"2023-09-23T19:57:10.928116+0000","last_became_active":"2023-09-23T19:47:56.790954+0000","last_became_peered":"2023-09-23T19:47:56.790954+0000","last_unstale":"2023-09-23T19:57:10.928116+0000","last_undegraded":"2023-09-23T19:57:10.928116+0000","last_fullsized":"2023-09-23T19:57:10.928116+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":463,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.428712+0000","last_change":"2023-09-23T19:18:09.016297+0000","last_active":"2023-09-23T19:57:10.428712+0000","last_peered":"2023-09-23T19:57:10.428712+0000","last_clean":"2023-09-23T19:57:10.428712+0000","last_became_active":"2023-09-23T19:18:09.015436+0000","last_became_peered":"2023-09-23T19:18:09.015436+0000","last_unstale":"2023-09-23T19:57:10.428712+0000","last_undegraded":"2023-09-23T19:57:10.428712+0000","last_fullsized":"2023-09-23T19:57:10.428712+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:59:33.840751+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:59:33.840751+0000","last_clean_scrub_stamp":"2023-09-23T18:59:33.840751+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":186,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.929297+0000","last_change":"2023-09-23T19:47:56.811861+0000","last_active":"2023-09-23T19:57:10.929297+0000","last_peered":"2023-09-23T19:57:10.929297+0000","last_clean":"2023-09-23T19:57:10.929297+0000","last_became_active":"2023-09-23T19:47:56.811527+0000","last_became_peered":"2023-09-23T19:47:56.811527+0000","last_unstale":"2023-09-23T19:57:10.929297+0000","last_undegraded":"2023-09-23T19:57:10.929297+0000","last_fullsized":"2023-09-23T19:57:10.929297+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":211,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.929237+0000","last_change":"2023-09-23T19:05:52.786771+0000","last_active":"2023-09-23T19:57:10.929237+0000","last_peered":"2023-09-23T19:57:10.929237+0000","last_clean":"2023-09-23T19:57:10.929237+0000","last_became_active":"2023-09-23T19:05:52.785797+0000","last_became_peered":"2023-09-23T19:05:52.785797+0000","last_unstale":"2023-09-23T19:57:10.929237+0000","last_undegraded":"2023-09-23T19:57:10.929237+0000","last_fullsized":"2023-09-23T19:57:10.929237+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.c","version":"261'3777","reported_seq":3790,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-09-23T19:57:55.315657+0000","last_change":"2023-09-23T19:56:50.358230+0000","last_active":"2023-09-23T19:57:55.315657+0000","last_peered":"2023-09-23T19:57:55.315657+0000","last_clean":"2023-09-23T19:57:55.315657+0000","last_became_active":"2023-09-23T19:56:50.357855+0000","last_became_peered":"2023-09-23T19:56:50.357855+0000","last_unstale":"2023-09-23T19:57:55.315657+0000","last_undegraded":"2023-09-23T19:57:55.315657+0000","last_fullsized":"2023-09-23T19:57:55.315657+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":256,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_clean_scrub_stamp":"2023-09-23T19:56:49.329294+0000","objects_scrubbed":0,"log_size":3777,"log_dups_size":0,"ondisk_log_size":3777,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":247529472,"num_objects":3777,"num_object_clones":0,"num_object_copies":11331,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3777,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3777,"num_write_kb":241728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":246,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.428583+0000","last_change":"2023-09-23T19:22:42.150636+0000","last_active":"2023-09-23T19:57:10.428583+0000","last_peered":"2023-09-23T19:57:10.428583+0000","last_clean":"2023-09-23T19:57:10.428583+0000","last_became_active":"2023-09-23T19:22:42.150045+0000","last_became_peered":"2023-09-23T19:22:42.150045+0000","last_unstale":"2023-09-23T19:57:10.428583+0000","last_undegraded":"2023-09-23T19:57:10.428583+0000","last_fullsized":"2023-09-23T19:57:10.428583+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":147,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.345124+0000","last_change":"2023-09-23T19:42:55.771906+0000","last_active":"2023-09-23T19:57:10.345124+0000","last_peered":"2023-09-23T19:57:10.345124+0000","last_clean":"2023-09-23T19:57:10.345124+0000","last_became_active":"2023-09-23T19:42:55.771599+0000","last_became_peered":"2023-09-23T19:42:55.771599+0000","last_unstale":"2023-09-23T19:57:10.345124+0000","last_undegraded":"2023-09-23T19:57:10.345124+0000","last_fullsized":"2023-09-23T19:57:10.345124+0000","mapping_epoch":188,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":261,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-09-23T19:57:21.792730+0000","last_change":"2023-09-23T19:38:43.876930+0000","last_active":"2023-09-23T19:57:21.792730+0000","last_peered":"2023-09-23T19:57:21.792730+0000","last_clean":"2023-09-23T19:57:21.792730+0000","last_became_active":"2023-09-23T19:38:43.876571+0000","last_became_peered":"2023-09-23T19:38:43.876571+0000","last_unstale":"2023-09-23T19:57:21.792730+0000","last_undegraded":"2023-09-23T19:57:21.792730+0000","last_fullsized":"2023-09-23T19:57:21.792730+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:30.924536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.9","version":"261'3601","reported_seq":3614,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-09-23T19:57:54.453245+0000","last_change":"2023-09-23T19:56:50.550531+0000","last_active":"2023-09-23T19:57:54.453245+0000","last_peered":"2023-09-23T19:57:54.453245+0000","last_clean":"2023-09-23T19:57:54.453245+0000","last_became_active":"2023-09-23T19:56:50.549713+0000","last_became_peered":"2023-09-23T19:56:50.549713+0000","last_unstale":"2023-09-23T19:57:54.453245+0000","last_undegraded":"2023-09-23T19:57:54.453245+0000","last_fullsized":"2023-09-23T19:57:54.453245+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":256,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_clean_scrub_stamp":"2023-09-23T19:56:49.329294+0000","objects_scrubbed":0,"log_size":3601,"log_dups_size":0,"ondisk_log_size":3601,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":235995136,"num_objects":3601,"num_object_clones":0,"num_object_copies":10803,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3601,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3601,"num_write_kb":230464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":231,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.429412+0000","last_change":"2023-09-23T19:18:09.015853+0000","last_active":"2023-09-23T19:57:10.429412+0000","last_peered":"2023-09-23T19:57:10.429412+0000","last_clean":"2023-09-23T19:57:10.429412+0000","last_became_active":"2023-09-23T19:18:09.015338+0000","last_became_peered":"2023-09-23T19:18:09.015338+0000","last_unstale":"2023-09-23T19:57:10.429412+0000","last_undegraded":"2023-09-23T19:57:10.429412+0000","last_fullsized":"2023-09-23T19:57:10.429412+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":229,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.335619+0000","last_change":"2023-09-23T19:38:43.881233+0000","last_active":"2023-09-23T19:57:10.335619+0000","last_peered":"2023-09-23T19:57:10.335619+0000","last_clean":"2023-09-23T19:57:10.335619+0000","last_became_active":"2023-09-23T19:38:43.880893+0000","last_became_peered":"2023-09-23T19:38:43.880893+0000","last_unstale":"2023-09-23T19:57:10.335619+0000","last_undegraded":"2023-09-23T19:57:10.335619+0000","last_fullsized":"2023-09-23T19:57:10.335619+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:11.206947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":82,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.928331+0000","last_change":"2023-09-23T19:47:56.790340+0000","last_active":"2023-09-23T19:57:10.928331+0000","last_peered":"2023-09-23T19:57:10.928331+0000","last_clean":"2023-09-23T19:57:10.928331+0000","last_became_active":"2023-09-23T19:47:56.789785+0000","last_became_peered":"2023-09-23T19:47:56.789785+0000","last_unstale":"2023-09-23T19:57:10.928331+0000","last_undegraded":"2023-09-23T19:57:10.928331+0000","last_fullsized":"2023-09-23T19:57:10.928331+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.a","version":"261'3627","reported_seq":3641,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-09-23T19:57:55.395699+0000","last_change":"2023-09-23T19:56:50.486634+0000","last_active":"2023-09-23T19:57:55.395699+0000","last_peered":"2023-09-23T19:57:55.395699+0000","last_clean":"2023-09-23T19:57:55.395699+0000","last_became_active":"2023-09-23T19:56:50.486290+0000","last_became_peered":"2023-09-23T19:56:50.486290+0000","last_unstale":"2023-09-23T19:57:55.395699+0000","last_undegraded":"2023-09-23T19:57:55.395699+0000","last_fullsized":"2023-09-23T19:57:55.395699+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":256,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_clean_scrub_stamp":"2023-09-23T19:56:49.329294+0000","objects_scrubbed":0,"log_size":3627,"log_dups_size":0,"ondisk_log_size":3627,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:27:04.672284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237699072,"num_objects":3627,"num_object_clones":0,"num_object_copies":10881,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3627,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3627,"num_write_kb":232128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":165,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.429686+0000","last_change":"2023-09-23T19:32:49.737323+0000","last_active":"2023-09-23T19:57:10.429686+0000","last_peered":"2023-09-23T19:57:10.429686+0000","last_clean":"2023-09-23T19:57:10.429686+0000","last_became_active":"2023-09-23T19:32:49.736937+0000","last_became_peered":"2023-09-23T19:32:49.736937+0000","last_unstale":"2023-09-23T19:57:10.429686+0000","last_undegraded":"2023-09-23T19:57:10.429686+0000","last_fullsized":"2023-09-23T19:57:10.429686+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":199,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:20.362362+0000","last_change":"2023-09-23T19:22:42.145506+0000","last_active":"2023-09-23T19:57:20.362362+0000","last_peered":"2023-09-23T19:57:20.362362+0000","last_clean":"2023-09-23T19:57:20.362362+0000","last_became_active":"2023-09-23T19:22:42.143991+0000","last_became_peered":"2023-09-23T19:22:42.143991+0000","last_unstale":"2023-09-23T19:57:20.362362+0000","last_undegraded":"2023-09-23T19:57:20.362362+0000","last_fullsized":"2023-09-23T19:57:20.362362+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:56.875175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":202,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.429086+0000","last_change":"2023-09-23T19:32:49.734592+0000","last_active":"2023-09-23T19:57:10.429086+0000","last_peered":"2023-09-23T19:57:10.429086+0000","last_clean":"2023-09-23T19:57:10.429086+0000","last_became_active":"2023-09-23T19:32:49.733905+0000","last_became_peered":"2023-09-23T19:32:49.733905+0000","last_unstale":"2023-09-23T19:57:10.429086+0000","last_undegraded":"2023-09-23T19:57:10.429086+0000","last_fullsized":"2023-09-23T19:57:10.429086+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.b","version":"262'3714","reported_seq":3729,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T19:57:57.401272+0000","last_change":"2023-09-23T19:56:50.348570+0000","last_active":"2023-09-23T19:57:57.401272+0000","last_peered":"2023-09-23T19:57:57.401272+0000","last_clean":"2023-09-23T19:57:57.401272+0000","last_became_active":"2023-09-23T19:56:50.348227+0000","last_became_peered":"2023-09-23T19:56:50.348227+0000","last_unstale":"2023-09-23T19:57:57.401272+0000","last_undegraded":"2023-09-23T19:57:57.401272+0000","last_fullsized":"2023-09-23T19:57:57.401272+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":256,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:49.329294+0000","last_clean_scrub_stamp":"2023-09-23T19:56:49.329294+0000","objects_scrubbed":0,"log_size":3714,"log_dups_size":0,"ondisk_log_size":3714,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:35:58.887241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":243400704,"num_objects":3714,"num_object_clones":0,"num_object_copies":11142,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3714,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3714,"num_write_kb":237696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":246,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.928548+0000","last_change":"2023-09-23T19:38:43.895887+0000","last_active":"2023-09-23T19:57:10.928548+0000","last_peered":"2023-09-23T19:57:10.928548+0000","last_clean":"2023-09-23T19:57:10.928548+0000","last_became_active":"2023-09-23T19:38:43.895486+0000","last_became_peered":"2023-09-23T19:38:43.895486+0000","last_unstale":"2023-09-23T19:57:10.928548+0000","last_undegraded":"2023-09-23T19:57:10.928548+0000","last_fullsized":"2023-09-23T19:57:10.928548+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":234,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-09-23T19:57:21.792579+0000","last_change":"2023-09-23T19:22:42.149694+0000","last_active":"2023-09-23T19:57:21.792579+0000","last_peered":"2023-09-23T19:57:21.792579+0000","last_clean":"2023-09-23T19:57:21.792579+0000","last_became_active":"2023-09-23T19:22:42.148194+0000","last_became_peered":"2023-09-23T19:22:42.148194+0000","last_unstale":"2023-09-23T19:57:21.792579+0000","last_undegraded":"2023-09-23T19:57:21.792579+0000","last_fullsized":"2023-09-23T19:57:21.792579+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:47.842383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":210,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.344693+0000","last_change":"2023-09-23T19:05:52.138620+0000","last_active":"2023-09-23T19:57:10.344693+0000","last_peered":"2023-09-23T19:57:10.344693+0000","last_clean":"2023-09-23T19:57:10.344693+0000","last_became_active":"2023-09-23T19:05:52.138243+0000","last_became_peered":"2023-09-23T19:05:52.138243+0000","last_unstale":"2023-09-23T19:57:10.344693+0000","last_undegraded":"2023-09-23T19:57:10.344693+0000","last_fullsized":"2023-09-23T19:57:10.344693+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":216,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-09-23T19:57:21.793090+0000","last_change":"2023-09-23T19:18:09.008811+0000","last_active":"2023-09-23T19:57:21.793090+0000","last_peered":"2023-09-23T19:57:21.793090+0000","last_clean":"2023-09-23T19:57:21.793090+0000","last_became_active":"2023-09-23T19:18:09.007255+0000","last_became_peered":"2023-09-23T19:18:09.007255+0000","last_unstale":"2023-09-23T19:57:21.793090+0000","last_undegraded":"2023-09-23T19:57:21.793090+0000","last_fullsized":"2023-09-23T19:57:21.793090+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:25.874038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":220,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-09-23T19:57:21.789645+0000","last_change":"2023-09-23T19:38:43.877244+0000","last_active":"2023-09-23T19:57:21.789645+0000","last_peered":"2023-09-23T19:57:21.789645+0000","last_clean":"2023-09-23T19:57:21.789645+0000","last_became_active":"2023-09-23T19:38:43.876896+0000","last_became_peered":"2023-09-23T19:38:43.876896+0000","last_unstale":"2023-09-23T19:57:21.789645+0000","last_undegraded":"2023-09-23T19:57:21.789645+0000","last_fullsized":"2023-09-23T19:57:21.789645+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:22.288276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":90,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.345581+0000","last_change":"2023-09-23T19:43:38.620732+0000","last_active":"2023-09-23T19:57:10.345581+0000","last_peered":"2023-09-23T19:57:10.345581+0000","last_clean":"2023-09-23T19:57:10.345581+0000","last_became_active":"2023-09-23T19:43:38.619424+0000","last_became_peered":"2023-09-23T19:43:38.619424+0000","last_unstale":"2023-09-23T19:57:10.345581+0000","last_undegraded":"2023-09-23T19:57:10.345581+0000","last_fullsized":"2023-09-23T19:57:10.345581+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":264,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.345379+0000","last_change":"2023-09-23T19:38:43.947451+0000","last_active":"2023-09-23T19:57:10.345379+0000","last_peered":"2023-09-23T19:57:10.345379+0000","last_clean":"2023-09-23T19:57:10.345379+0000","last_became_active":"2023-09-23T19:38:43.946420+0000","last_became_peered":"2023-09-23T19:38:43.946420+0000","last_unstale":"2023-09-23T19:57:10.345379+0000","last_undegraded":"2023-09-23T19:57:10.345379+0000","last_fullsized":"2023-09-23T19:57:10.345379+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":200,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.929514+0000","last_change":"2023-09-23T19:47:56.793347+0000","last_active":"2023-09-23T19:57:10.929514+0000","last_peered":"2023-09-23T19:57:10.929514+0000","last_clean":"2023-09-23T19:57:10.929514+0000","last_became_active":"2023-09-23T19:47:56.792866+0000","last_became_peered":"2023-09-23T19:47:56.792866+0000","last_unstale":"2023-09-23T19:57:10.929514+0000","last_undegraded":"2023-09-23T19:57:10.929514+0000","last_fullsized":"2023-09-23T19:57:10.929514+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":240,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.345342+0000","last_change":"2023-09-23T19:28:07.896983+0000","last_active":"2023-09-23T19:57:10.345342+0000","last_peered":"2023-09-23T19:57:10.345342+0000","last_clean":"2023-09-23T19:57:10.345342+0000","last_became_active":"2023-09-23T19:28:07.896577+0000","last_became_peered":"2023-09-23T19:28:07.896577+0000","last_unstale":"2023-09-23T19:57:10.345342+0000","last_undegraded":"2023-09-23T19:57:10.345342+0000","last_fullsized":"2023-09-23T19:57:10.345342+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":170,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.929387+0000","last_change":"2023-09-23T19:38:43.900225+0000","last_active":"2023-09-23T19:57:10.929387+0000","last_peered":"2023-09-23T19:57:10.929387+0000","last_clean":"2023-09-23T19:57:10.929387+0000","last_became_active":"2023-09-23T19:38:43.899907+0000","last_became_peered":"2023-09-23T19:38:43.899907+0000","last_unstale":"2023-09-23T19:57:10.929387+0000","last_undegraded":"2023-09-23T19:57:10.929387+0000","last_fullsized":"2023-09-23T19:57:10.929387+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":229,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-09-23T19:57:21.789311+0000","last_change":"2023-09-23T19:38:43.882487+0000","last_active":"2023-09-23T19:57:21.789311+0000","last_peered":"2023-09-23T19:57:21.789311+0000","last_clean":"2023-09-23T19:57:21.789311+0000","last_became_active":"2023-09-23T19:38:43.881931+0000","last_became_peered":"2023-09-23T19:38:43.881931+0000","last_unstale":"2023-09-23T19:57:21.789311+0000","last_undegraded":"2023-09-23T19:57:21.789311+0000","last_fullsized":"2023-09-23T19:57:21.789311+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:02:36.906535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":250,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-09-23T19:57:42.785881+0000","last_change":"2023-09-23T19:38:43.893983+0000","last_active":"2023-09-23T19:57:42.785881+0000","last_peered":"2023-09-23T19:57:42.785881+0000","last_clean":"2023-09-23T19:57:42.785881+0000","last_became_active":"2023-09-23T19:38:43.893527+0000","last_became_peered":"2023-09-23T19:38:43.893527+0000","last_unstale":"2023-09-23T19:57:42.785881+0000","last_undegraded":"2023-09-23T19:57:42.785881+0000","last_fullsized":"2023-09-23T19:57:42.785881+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:12.156531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":219,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.344603+0000","last_change":"2023-09-23T19:06:36.673002+0000","last_active":"2023-09-23T19:57:10.344603+0000","last_peered":"2023-09-23T19:57:10.344603+0000","last_clean":"2023-09-23T19:57:10.344603+0000","last_became_active":"2023-09-23T19:06:36.671911+0000","last_became_peered":"2023-09-23T19:06:36.671911+0000","last_unstale":"2023-09-23T19:57:10.344603+0000","last_undegraded":"2023-09-23T19:57:10.344603+0000","last_fullsized":"2023-09-23T19:57:10.344603+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":16,"stat_sum":{"num_bytes":3864920064,"num_objects":58974,"num_object_clones":0,"num_object_copies":176922,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":58974,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":58974,"num_write_kb":3774336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11654135808,"data_stored":11654135808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":58974,"ondisk_log_size":58974,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430869,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1585484,"kb_used_data":693700,"kb_used_omap":0,"kb_used_meta":891776,"kb_avail":92159668,"statfs":{"total":95995035648,"available":94371500032,"internally_reserved":0,"allocated":710348800,"data_stored":709802127,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":913178624},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:57:20 2023","interfaces":[{"interface":"back","average":{"1min":3.4049999999999998,"5min":1.8069999999999999,"15min":1.4890000000000001},"min":{"1min":0.34799999999999998,"5min":0.26700000000000002,"15min":0.26400000000000001},"max":{"1min":41.570999999999998,"5min":41.570999999999998,"15min":41.570999999999998},"last":0.32300000000000001},{"interface":"front","average":{"1min":3.665,"5min":2.0899999999999999,"15min":1.6839999999999999},"min":{"1min":0.44500000000000001,"5min":0.28000000000000003,"15min":0.25600000000000001},"max":{"1min":45.634,"5min":45.634,"15min":45.634},"last":0.94199999999999995}]},{"osd":1,"last update":"Sat Sep 23 19:57:38 2023","interfaces":[{"interface":"back","average":{"1min":3.4900000000000002,"5min":3.536,"15min":2.5760000000000001},"min":{"1min":0.35399999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":41.433,"5min":149.018,"15min":149.018},"last":0.503},{"interface":"front","average":{"1min":3.379,"5min":3.4990000000000001,"15min":2.5350000000000001},"min":{"1min":0.379,"5min":0.254,"15min":0.254},"max":{"1min":41.372999999999998,"5min":148.995,"15min":148.995},"last":0.52300000000000002}]},{"osd":2,"last update":"Sat Sep 23 19:57:20 2023","interfaces":[{"interface":"back","average":{"1min":3.6459999999999999,"5min":2.7440000000000002,"15min":3.6320000000000001},"min":{"1min":0.42299999999999999,"5min":0.31900000000000001,"15min":0.28100000000000003},"max":{"1min":44.143999999999998,"5min":53.521000000000001,"15min":246.982},"last":1.863},{"interface":"front","average":{"1min":3.5470000000000002,"5min":2.9980000000000002,"15min":3.6429999999999998},"min":{"1min":0.58599999999999997,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":41.790999999999997,"5min":53.466000000000001,"15min":247.751},"last":3.4399999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:57:10 2023","interfaces":[{"interface":"back","average":{"1min":0.92600000000000005,"5min":1.831,"15min":1.7410000000000001},"min":{"1min":0.69699999999999995,"5min":0.33000000000000002,"15min":0.312},"max":{"1min":1.173,"5min":10.615,"15min":10.615},"last":0.42699999999999999},{"interface":"front","average":{"1min":1.079,"5min":1.708,"15min":1.6899999999999999},"min":{"1min":0.79800000000000004,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":1.3999999999999999,"5min":9.9179999999999993,"15min":9.9179999999999993},"last":0.55200000000000005}]},{"osd":4,"last update":"Sat Sep 23 19:57:38 2023","interfaces":[{"interface":"back","average":{"1min":1.353,"5min":2.5139999999999998,"15min":1.631},"min":{"1min":0.48699999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":6.6260000000000003,"5min":13.484999999999999,"15min":13.484999999999999},"last":0.34300000000000003},{"interface":"front","average":{"1min":0.84999999999999998,"5min":2.0590000000000002,"15min":1.5309999999999999},"min":{"1min":0.44700000000000001,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":1.3859999999999999,"5min":13.116,"15min":13.116},"last":0.45300000000000001}]},{"osd":5,"last update":"Sat Sep 23 19:57:38 2023","interfaces":[{"interface":"back","average":{"1min":0.871,"5min":2.044,"15min":1.4830000000000001},"min":{"1min":0.46999999999999997,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":1.3360000000000001,"5min":13.321999999999999,"15min":13.321999999999999},"last":0.40200000000000002},{"interface":"front","average":{"1min":1.2949999999999999,"5min":2.2509999999999999,"15min":1.591},"min":{"1min":0.52800000000000002,"5min":0.315,"15min":0.315},"max":{"1min":3.3540000000000001,"5min":13.478999999999999,"15min":13.478999999999999},"last":0.53700000000000003}]},{"osd":6,"last update":"Sat Sep 23 19:57:38 2023","interfaces":[{"interface":"back","average":{"1min":0.98999999999999999,"5min":1.9930000000000001,"15min":1.667},"min":{"1min":0.44600000000000001,"5min":0.34300000000000003,"15min":0.33900000000000002},"max":{"1min":1.9259999999999999,"5min":13.436,"15min":13.436},"last":0.47799999999999998},{"interface":"front","average":{"1min":1.0109999999999999,"5min":1.754,"15min":1.6240000000000001},"min":{"1min":0.47999999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.732,"5min":10.766999999999999,"15min":10.766999999999999},"last":0.57499999999999996}]}]},{"osd":6,"up_from":42,"seq":180388627101,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1831548,"kb_used_data":940340,"kb_used_omap":0,"kb_used_meta":891200,"kb_avail":91913604,"statfs":{"total":95995035648,"available":94119530496,"internally_reserved":0,"allocated":962908160,"data_stored":962377410,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":912588800},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:57:26 2023","interfaces":[{"interface":"back","average":{"1min":0.88300000000000001,"5min":1.5189999999999999,"15min":1.7749999999999999},"min":{"1min":0.34799999999999998,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":1.393,"5min":9.5700000000000003,"15min":45.600000000000001},"last":0.84699999999999998},{"interface":"front","average":{"1min":1.0509999999999999,"5min":1.581,"15min":1.7769999999999999},"min":{"1min":0.29799999999999999,"5min":0.29599999999999999,"15min":0.25800000000000001},"max":{"1min":3.9990000000000001,"5min":11.340999999999999,"15min":45.347999999999999},"last":0.46600000000000003}]},{"osd":1,"last update":"Sat Sep 23 19:56:51 2023","interfaces":[{"interface":"back","average":{"1min":0.99099999999999999,"5min":1.659,"15min":2.6360000000000001},"min":{"1min":0.32700000000000001,"5min":0.30599999999999999,"15min":0.28399999999999997},"max":{"1min":4.359,"5min":9.4909999999999997,"15min":117.58199999999999},"last":0.57399999999999995},{"interface":"front","average":{"1min":1.0089999999999999,"5min":1.7070000000000001,"15min":2.6960000000000002},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.29199999999999998},"max":{"1min":2.1850000000000001,"5min":17.318999999999999,"15min":117.633},"last":0.58399999999999996}]},{"osd":2,"last update":"Sat Sep 23 19:57:09 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":4.3109999999999999,"15min":4.7119999999999997},"min":{"1min":0.76300000000000001,"5min":0.30499999999999999,"15min":0.30399999999999999},"max":{"1min":1.4330000000000001,"5min":219.453,"15min":219.453},"last":2.2370000000000001},{"interface":"front","average":{"1min":0.98399999999999999,"5min":4.2460000000000004,"15min":4.6299999999999999},"min":{"1min":0.64600000000000002,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":1.3600000000000001,"5min":219.369,"15min":219.369},"last":0.36199999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:57:07 2023","interfaces":[{"interface":"back","average":{"1min":0.91700000000000004,"5min":1.4099999999999999,"15min":1.7949999999999999},"min":{"1min":0.60599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.2609999999999999,"5min":9.1419999999999995,"15min":16.428000000000001},"last":2.254},{"interface":"front","average":{"1min":1.0620000000000001,"5min":1.772,"15min":1.806},"min":{"1min":0.79100000000000004,"5min":0.33100000000000002,"15min":0.307},"max":{"1min":1.5820000000000001,"5min":9.5329999999999995,"15min":18.917000000000002},"last":0.32600000000000001}]},{"osd":4,"last update":"Sat Sep 23 19:57:07 2023","interfaces":[{"interface":"back","average":{"1min":1.01,"5min":2.5459999999999998,"15min":1.6779999999999999},"min":{"1min":0.53800000000000003,"5min":0.33900000000000002,"15min":0.28999999999999998},"max":{"1min":1.379,"5min":11.912000000000001,"15min":14.401999999999999},"last":0.30599999999999999},{"interface":"front","average":{"1min":1.014,"5min":2.2770000000000001,"15min":1.8220000000000001},"min":{"1min":0.69999999999999996,"5min":0.32200000000000001,"15min":0.28299999999999997},"max":{"1min":1.5269999999999999,"5min":14.239000000000001,"15min":14.603},"last":0.33900000000000002}]},{"osd":5,"last update":"Sat Sep 23 19:57:07 2023","interfaces":[{"interface":"back","average":{"1min":1.1859999999999999,"5min":2.706,"15min":1.833},"min":{"1min":0.82599999999999996,"5min":0.40699999999999997,"15min":0.32000000000000001},"max":{"1min":1.411,"5min":15.045,"15min":15.045},"last":0.40799999999999997},{"interface":"front","average":{"1min":1.0980000000000001,"5min":2.839,"15min":1.944},"min":{"1min":0.66900000000000004,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":1.383,"5min":16.244,"15min":16.244},"last":0.45500000000000002}]},{"osd":7,"last update":"Sat Sep 23 19:57:32 2023","interfaces":[{"interface":"back","average":{"1min":1.407,"5min":2.4740000000000002,"15min":1.8839999999999999},"min":{"1min":0.57599999999999996,"5min":0.34899999999999998,"15min":0.33200000000000002},"max":{"1min":4.1369999999999996,"5min":11.709,"15min":11.709},"last":0.435},{"interface":"front","average":{"1min":1.056,"5min":1.875,"15min":1.9550000000000001},"min":{"1min":0.436,"5min":0.39500000000000002,"15min":0.32200000000000001},"max":{"1min":1.7270000000000001,"5min":9.7599999999999998,"15min":12.327999999999999},"last":0.41699999999999998}]}]},{"osd":1,"up_from":98,"seq":420906795433,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2671812,"kb_used_data":1844796,"kb_used_omap":4,"kb_used_meta":827003,"kb_avail":91073340,"statfs":{"total":95995035648,"available":93259100160,"internally_reserved":0,"allocated":1889071104,"data_stored":1888004080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":846851982},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,6,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:57:50 2023","interfaces":[{"interface":"back","average":{"1min":4.6619999999999999,"5min":1.6699999999999999,"15min":2.573},"min":{"1min":0.24199999999999999,"5min":0.221,"15min":0.18099999999999999},"max":{"1min":46.527999999999999,"5min":46.527999999999999,"15min":134.37700000000001},"last":2.6230000000000002},{"interface":"front","average":{"1min":4.4180000000000001,"5min":1.6299999999999999,"15min":2.484},"min":{"1min":0.44400000000000001,"5min":0.23400000000000001,"15min":0.22600000000000001},"max":{"1min":41.683999999999997,"5min":41.683999999999997,"15min":135.79400000000001},"last":3.677}]},{"osd":2,"last update":"Sat Sep 23 19:57:15 2023","interfaces":[{"interface":"back","average":{"1min":0.70799999999999996,"5min":2.0089999999999999,"15min":2.7480000000000002},"min":{"1min":0.46300000000000002,"5min":0.27300000000000002,"15min":0.26800000000000002},"max":{"1min":1.1479999999999999,"5min":48.512,"15min":72.859999999999999},"last":7.3280000000000003},{"interface":"front","average":{"1min":0.79400000000000004,"5min":1.897,"15min":2.3500000000000001},"min":{"1min":0.42699999999999999,"5min":0.224,"15min":0.224},"max":{"1min":2.4489999999999998,"5min":48.664000000000001,"15min":73.665999999999997},"last":2.1859999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:57:11 2023","interfaces":[{"interface":"back","average":{"1min":0.71099999999999997,"5min":1.365,"15min":1.794},"min":{"1min":0.40400000000000003,"5min":0.26200000000000001,"15min":0.25},"max":{"1min":1.1639999999999999,"5min":20.806999999999999,"15min":20.806999999999999},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.69399999999999995,"5min":1.4850000000000001,"15min":1.8280000000000001},"min":{"1min":0.48499999999999999,"5min":0.25900000000000001,"15min":0.217},"max":{"1min":0.96799999999999997,"5min":19.201000000000001,"15min":19.201000000000001},"last":12.962999999999999}]},{"osd":4,"last update":"Sat Sep 23 19:57:50 2023","interfaces":[{"interface":"back","average":{"1min":3.9399999999999999,"5min":3.0449999999999999,"15min":2.0630000000000002},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":39.085999999999999,"5min":48.381,"15min":48.381},"last":2.7280000000000002},{"interface":"front","average":{"1min":3.6360000000000001,"5min":2.3140000000000001,"15min":2.0369999999999999},"min":{"1min":0.36599999999999999,"5min":0.35099999999999998,"15min":0.32200000000000001},"max":{"1min":41.320999999999998,"5min":41.320999999999998,"15min":82.239999999999995},"last":0.46200000000000002}]},{"osd":5,"last update":"Sat Sep 23 19:57:50 2023","interfaces":[{"interface":"back","average":{"1min":3.7370000000000001,"5min":2.6970000000000001,"15min":1.96},"min":{"1min":0.42899999999999999,"5min":0.38200000000000001,"15min":0.34899999999999998},"max":{"1min":41.274999999999999,"5min":48.643000000000001,"15min":48.643000000000001},"last":0.501},{"interface":"front","average":{"1min":4.2249999999999996,"5min":2.516,"15min":1.974},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":43.508000000000003,"5min":48.619999999999997,"15min":48.619999999999997},"last":0.36499999999999999}]},{"osd":6,"last update":"Sat Sep 23 19:57:50 2023","interfaces":[{"interface":"back","average":{"1min":4.5780000000000003,"5min":2.9100000000000001,"15min":2.3849999999999998},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.318},"max":{"1min":46.472000000000001,"5min":48.548999999999999,"15min":48.548999999999999},"last":0.39800000000000002},{"interface":"front","average":{"1min":4.4859999999999998,"5min":2.5150000000000001,"15min":2.1309999999999998},"min":{"1min":0.371,"5min":0.313,"15min":0.313},"max":{"1min":41.648000000000003,"5min":41.648000000000003,"15min":41.648000000000003},"last":2.77}]},{"osd":7,"last update":"Sat Sep 23 19:57:50 2023","interfaces":[{"interface":"back","average":{"1min":4.4000000000000004,"5min":3.0089999999999999,"15min":2.262},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.36799999999999999},"max":{"1min":41.595999999999997,"5min":48.329999999999998,"15min":48.329999999999998},"last":2.7959999999999998},{"interface":"front","average":{"1min":3.6000000000000001,"5min":2.927,"15min":2.198},"min":{"1min":0.499,"5min":0.47399999999999998,"15min":0.35299999999999998},"max":{"1min":41.223999999999997,"5min":48.579000000000001,"15min":48.579000000000001},"last":0.53100000000000003}]}]},{"osd":0,"up_from":85,"seq":365072220640,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2849144,"kb_used_data":1933040,"kb_used_omap":5,"kb_used_meta":916090,"kb_avail":90896008,"statfs":{"total":95995035648,"available":93077512192,"internally_reserved":0,"allocated":1979432960,"data_stored":1978387006,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":938076794},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 19:57:54 2023","interfaces":[{"interface":"back","average":{"1min":1.4910000000000001,"5min":3.6440000000000001,"15min":2.4180000000000001},"min":{"1min":0.39300000000000002,"5min":0.219,"15min":0.216},"max":{"1min":4.8799999999999999,"5min":149.37299999999999,"15min":149.37299999999999},"last":2.3969999999999998},{"interface":"front","average":{"1min":1.9870000000000001,"5min":3.9430000000000001,"15min":2.5329999999999999},"min":{"1min":0.33700000000000002,"5min":0.26100000000000001,"15min":0.189},"max":{"1min":6.649,"5min":147.119,"15min":147.119},"last":0.94699999999999995}]},{"osd":2,"last update":"Sat Sep 23 19:57:18 2023","interfaces":[{"interface":"back","average":{"1min":0.70299999999999996,"5min":1.2270000000000001,"15min":3.468},"min":{"1min":0.35399999999999998,"5min":0.25600000000000001,"15min":0.21299999999999999},"max":{"1min":1.0429999999999999,"5min":9.0830000000000002,"15min":262.59500000000003},"last":4.5359999999999996},{"interface":"front","average":{"1min":0.84299999999999997,"5min":1.365,"15min":3.327},"min":{"1min":0.378,"5min":0.26300000000000001,"15min":0.24199999999999999},"max":{"1min":4.4160000000000004,"5min":8.9469999999999992,"15min":263.25700000000001},"last":1.2949999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:57:54 2023","interfaces":[{"interface":"back","average":{"1min":1.385,"5min":1.429,"15min":1.706},"min":{"1min":0.45900000000000002,"5min":0.33600000000000002,"15min":0.222},"max":{"1min":4.8769999999999998,"5min":14.983000000000001,"15min":14.983000000000001},"last":0.84399999999999997},{"interface":"front","average":{"1min":1.7869999999999999,"5min":1.3580000000000001,"15min":1.704},"min":{"1min":0.497,"5min":0.33800000000000002,"15min":0.26700000000000002},"max":{"1min":5.7549999999999999,"5min":14.843,"15min":14.843},"last":0.97599999999999998}]},{"osd":4,"last update":"Sat Sep 23 19:57:32 2023","interfaces":[{"interface":"back","average":{"1min":1.0309999999999999,"5min":1.3660000000000001,"15min":1.4430000000000001},"min":{"1min":0.39300000000000002,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":4.2359999999999998,"5min":6.0469999999999997,"15min":9.0679999999999996},"last":0.71099999999999997},{"interface":"front","average":{"1min":1.349,"5min":1.4370000000000001,"15min":1.5249999999999999},"min":{"1min":0.63900000000000001,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":5.5579999999999998,"5min":7.056,"15min":8.6289999999999996},"last":0.91200000000000003}]},{"osd":5,"last update":"Sat Sep 23 19:57:32 2023","interfaces":[{"interface":"back","average":{"1min":1.393,"5min":1.4610000000000001,"15min":1.5740000000000001},"min":{"1min":0.66300000000000003,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":5.633,"5min":7.8170000000000002,"15min":8.5619999999999994},"last":1.0089999999999999},{"interface":"front","average":{"1min":1.1619999999999999,"5min":1.4770000000000001,"15min":1.552},"min":{"1min":0.47799999999999998,"5min":0.38700000000000001,"15min":0.34100000000000003},"max":{"1min":4.3129999999999997,"5min":5.9969999999999999,"15min":9.0329999999999995},"last":0.77300000000000002}]},{"osd":6,"last update":"Sat Sep 23 19:57:32 2023","interfaces":[{"interface":"back","average":{"1min":1.083,"5min":1.5920000000000001,"15min":1.788},"min":{"1min":0.436,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":4.2789999999999999,"5min":6.9020000000000001,"15min":66.170000000000002},"last":0.876},{"interface":"front","average":{"1min":1.4630000000000001,"5min":1.7150000000000001,"15min":1.9390000000000001},"min":{"1min":0.60799999999999998,"5min":0.433,"15min":0.35299999999999998},"max":{"1min":5.6749999999999998,"5min":6.9189999999999996,"15min":66.260999999999996},"last":1.0389999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:57:32 2023","interfaces":[{"interface":"back","average":{"1min":1.431,"5min":3.5840000000000001,"15min":2.3879999999999999},"min":{"1min":0.73399999999999999,"5min":0.42699999999999999,"15min":0.38200000000000001},"max":{"1min":4.8280000000000003,"5min":210.47399999999999,"15min":210.47399999999999},"last":2.4390000000000001},{"interface":"front","average":{"1min":1.22,"5min":3.601,"15min":2.3610000000000002},"min":{"1min":0.58699999999999997,"5min":0.42099999999999999,"15min":0.40100000000000002},"max":{"1min":4.3470000000000004,"5min":210.52199999999999,"15min":210.52199999999999},"last":0.81299999999999994}]}]},{"osd":2,"up_from":159,"seq":682899800296,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2328480,"kb_used_data":1675544,"kb_used_omap":7,"kb_used_meta":652920,"kb_avail":91416672,"statfs":{"total":95995035648,"available":93610672128,"internally_reserved":0,"allocated":1715757056,"data_stored":1714727521,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":668590161},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:57:30 2023","interfaces":[{"interface":"back","average":{"1min":0.77000000000000002,"5min":1.1339999999999999,"15min":2.2269999999999999},"min":{"1min":0.32500000000000001,"5min":0.22,"15min":0.17299999999999999},"max":{"1min":2.1070000000000002,"5min":9.532,"15min":117.77500000000001},"last":0.29699999999999999},{"interface":"front","average":{"1min":0.93600000000000005,"5min":0.95799999999999996,"15min":2.544},"min":{"1min":0.375,"5min":0.23200000000000001,"15min":0.214},"max":{"1min":2.4260000000000002,"5min":6.0679999999999996,"15min":117.91800000000001},"last":0.35999999999999999}]},{"osd":1,"last update":"Sat Sep 23 19:57:30 2023","interfaces":[{"interface":"back","average":{"1min":0.86699999999999999,"5min":2.7730000000000001,"15min":3.1150000000000002},"min":{"1min":0.33300000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":3.8969999999999998,"5min":84.234999999999999,"15min":114.107},"last":5.6749999999999998},{"interface":"front","average":{"1min":0.95399999999999996,"5min":2.4900000000000002,"15min":2.7170000000000001},"min":{"1min":0.36399999999999999,"5min":0.26600000000000001,"15min":0.22},"max":{"1min":4.1299999999999999,"5min":84.150000000000006,"15min":117.172},"last":1.1100000000000001}]},{"osd":3,"last update":"Sat Sep 23 19:57:30 2023","interfaces":[{"interface":"back","average":{"1min":0.67000000000000004,"5min":1.4850000000000001,"15min":2.0659999999999998},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.24099999999999999},"max":{"1min":1.044,"5min":16.373999999999999,"15min":17.751000000000001},"last":6.7279999999999998},{"interface":"front","average":{"1min":0.82399999999999995,"5min":1.704,"15min":1.964},"min":{"1min":0.39600000000000002,"5min":0.36399999999999999,"15min":0.21299999999999999},"max":{"1min":1.839,"5min":19.596,"15min":20.286999999999999},"last":0.34799999999999998}]},{"osd":4,"last update":"Sat Sep 23 19:57:30 2023","interfaces":[{"interface":"back","average":{"1min":0.999,"5min":2.077,"15min":1.9930000000000001},"min":{"1min":0.627,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":2.468,"5min":14.194000000000001,"15min":18.318999999999999},"last":0.80900000000000005},{"interface":"front","average":{"1min":0.90900000000000003,"5min":2.4900000000000002,"15min":2.0630000000000002},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.31900000000000001},"max":{"1min":1.26,"5min":14.271000000000001,"15min":14.271000000000001},"last":3.6139999999999999}]},{"osd":5,"last update":"Sat Sep 23 19:57:30 2023","interfaces":[{"interface":"back","average":{"1min":0.92900000000000005,"5min":2.1880000000000002,"15min":1.722},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.27600000000000002},"max":{"1min":1.8799999999999999,"5min":9.6110000000000007,"15min":16.577000000000002},"last":0.40000000000000002},{"interface":"front","average":{"1min":1.008,"5min":1.3999999999999999,"15min":1.7310000000000001},"min":{"1min":0.54400000000000004,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":2.4279999999999999,"5min":7.2939999999999996,"15min":10.193},"last":0.84899999999999998}]},{"osd":6,"last update":"Sat Sep 23 19:57:30 2023","interfaces":[{"interface":"back","average":{"1min":1.054,"5min":1.8280000000000001,"15min":2.052},"min":{"1min":0.59599999999999997,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":2.387,"5min":9.8870000000000005,"15min":10.166},"last":0.89300000000000002},{"interface":"front","average":{"1min":0.98399999999999999,"5min":2.2290000000000001,"15min":2.298},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38},"max":{"1min":1.349,"5min":11.696,"15min":12.297000000000001},"last":3.5710000000000002}]},{"osd":7,"last update":"Sat Sep 23 19:57:30 2023","interfaces":[{"interface":"back","average":{"1min":0.98199999999999998,"5min":1.8620000000000001,"15min":1.9550000000000001},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.38500000000000001},"max":{"1min":1.7969999999999999,"5min":9.4879999999999995,"15min":32.945},"last":1.169},{"interface":"front","average":{"1min":1.0029999999999999,"5min":2.0489999999999999,"15min":2.173},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":1.292,"5min":7.0330000000000004,"15min":16.593},"last":3.8860000000000001}]}]},{"osd":3,"up_from":116,"seq":498216206696,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2382612,"kb_used_data":1682700,"kb_used_omap":2,"kb_used_meta":699901,"kb_avail":91362540,"statfs":{"total":95995035648,"available":93555240960,"internally_reserved":0,"allocated":1723084800,"data_stored":1722124448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":716698677},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:57:07 2023","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":0.97499999999999998,"15min":1.851},"min":{"1min":0.42999999999999999,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":1.03,"5min":6.6669999999999998,"15min":50.771999999999998},"last":1.272},{"interface":"front","average":{"1min":0.66900000000000004,"5min":0.86399999999999999,"15min":1.996},"min":{"1min":0.34799999999999998,"5min":0.24399999999999999,"15min":0.20100000000000001},"max":{"1min":1.347,"5min":5.9820000000000002,"15min":48.887},"last":4.5289999999999999}]},{"osd":1,"last update":"Sat Sep 23 19:57:07 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":1.119,"15min":2.1320000000000001},"min":{"1min":0.42199999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":1.2130000000000001,"5min":12.542,"15min":92.171000000000006},"last":4.2350000000000003},{"interface":"front","average":{"1min":0.69799999999999995,"5min":1.085,"15min":2.2040000000000002},"min":{"1min":0.47299999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":1.0940000000000001,"5min":6.617,"15min":87.771000000000001},"last":1.859}]},{"osd":2,"last update":"Sat Sep 23 19:57:21 2023","interfaces":[{"interface":"back","average":{"1min":0.79500000000000004,"5min":1.105,"15min":2.6389999999999998},"min":{"1min":0.46100000000000002,"5min":0.307,"15min":0.26200000000000001},"max":{"1min":1.6060000000000001,"5min":9.0709999999999997,"15min":116.363},"last":4.4039999999999999},{"interface":"front","average":{"1min":0.66800000000000004,"5min":1.264,"15min":2.5920000000000001},"min":{"1min":0.312,"5min":0.30499999999999999,"15min":0.252},"max":{"1min":0.95499999999999996,"5min":5.7359999999999998,"15min":118.474},"last":2.3250000000000002}]},{"osd":4,"last update":"Sat Sep 23 19:57:07 2023","interfaces":[{"interface":"back","average":{"1min":1.0409999999999999,"5min":1.716,"15min":1.6719999999999999},"min":{"1min":0.66600000000000004,"5min":0.309,"15min":0.309},"max":{"1min":1.7649999999999999,"5min":9.2240000000000002,"15min":11.666},"last":1.234},{"interface":"front","average":{"1min":1.077,"5min":1.635,"15min":1.728},"min":{"1min":0.74199999999999999,"5min":0.36199999999999999,"15min":0.33000000000000002},"max":{"1min":1.796,"5min":7.8339999999999996,"15min":9.9990000000000006},"last":0.32900000000000001}]},{"osd":5,"last update":"Sat Sep 23 19:57:07 2023","interfaces":[{"interface":"back","average":{"1min":1.0229999999999999,"5min":1.5980000000000001,"15min":1.708},"min":{"1min":0.76400000000000001,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":1.633,"5min":7.7960000000000003,"15min":10.022},"last":0.36499999999999999},{"interface":"front","average":{"1min":1.052,"5min":1.4530000000000001,"15min":1.726},"min":{"1min":0.68799999999999994,"5min":0.39700000000000002,"15min":0.34000000000000002},"max":{"1min":1.9390000000000001,"5min":10.026,"15min":13.146000000000001},"last":2.5059999999999998}]},{"osd":6,"last update":"Sat Sep 23 19:57:07 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":1.655,"15min":2.1499999999999999},"min":{"1min":0.69799999999999995,"5min":0.41199999999999998,"15min":0.39100000000000001},"max":{"1min":1.718,"5min":8.6059999999999999,"15min":15.919},"last":2.8149999999999999},{"interface":"front","average":{"1min":1.008,"5min":1.6519999999999999,"15min":1.994},"min":{"1min":0.66800000000000004,"5min":0.38900000000000001,"15min":0.36399999999999999},"max":{"1min":1.593,"5min":7.048,"15min":11.456},"last":3.9060000000000001}]},{"osd":7,"last update":"Sat Sep 23 19:57:07 2023","interfaces":[{"interface":"back","average":{"1min":1.1319999999999999,"5min":2.077,"15min":2.153},"min":{"1min":0.70899999999999996,"5min":0.378,"15min":0.378},"max":{"1min":2.8079999999999998,"5min":11.505000000000001,"15min":14.098000000000001},"last":2.4510000000000001},{"interface":"front","average":{"1min":1.139,"5min":1.6890000000000001,"15min":1.9390000000000001},"min":{"1min":0.76300000000000001,"5min":0.41299999999999998,"15min":0.40600000000000003},"max":{"1min":1.6060000000000001,"5min":8.4199999999999999,"15min":11.744999999999999},"last":1.2989999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019566,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2303364,"kb_used_data":1395260,"kb_used_omap":0,"kb_used_meta":908096,"kb_avail":91441788,"statfs":{"total":95995035648,"available":93636390912,"internally_reserved":0,"allocated":1428746240,"data_stored":1428207145,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":929890304},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:57:33 2023","interfaces":[{"interface":"back","average":{"1min":1.0489999999999999,"5min":1.615,"15min":1.9039999999999999},"min":{"1min":0.437,"5min":0.247,"15min":0.247},"max":{"1min":3.0790000000000002,"5min":6.641,"15min":150.155},"last":0.65400000000000003},{"interface":"front","average":{"1min":1.004,"5min":1.232,"15min":1.7070000000000001},"min":{"1min":0.435,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":2.625,"5min":9.6240000000000006,"15min":149.47900000000001},"last":0.47099999999999997}]},{"osd":1,"last update":"Sat Sep 23 19:56:58 2023","interfaces":[{"interface":"back","average":{"1min":0.92000000000000004,"5min":1.575,"15min":1.8600000000000001},"min":{"1min":0.47699999999999998,"5min":0.29199999999999998,"15min":0.27800000000000002},"max":{"1min":1.4419999999999999,"5min":14.029,"15min":101.786},"last":2.778},{"interface":"front","average":{"1min":1.149,"5min":1.74,"15min":2.0379999999999998},"min":{"1min":0.63200000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":4.7729999999999997,"5min":14.090999999999999,"15min":101.726},"last":2.8250000000000002}]},{"osd":2,"last update":"Sat Sep 23 19:57:19 2023","interfaces":[{"interface":"back","average":{"1min":0.86099999999999999,"5min":1.7829999999999999,"15min":3.8300000000000001},"min":{"1min":0.56200000000000006,"5min":0.30399999999999999,"15min":0.27600000000000002},"max":{"1min":1.841,"5min":7.4690000000000003,"15min":219.02799999999999},"last":0.443},{"interface":"front","average":{"1min":1.083,"5min":1.718,"15min":3.7789999999999999},"min":{"1min":0.69899999999999995,"5min":0.308,"15min":0.29899999999999999},"max":{"1min":2.5899999999999999,"5min":9.6780000000000008,"15min":219.65600000000001},"last":1.8700000000000001}]},{"osd":3,"last update":"Sat Sep 23 19:57:19 2023","interfaces":[{"interface":"back","average":{"1min":0.88800000000000001,"5min":1.694,"15min":1.8},"min":{"1min":0.45900000000000002,"5min":0.308,"15min":0.308},"max":{"1min":1.25,"5min":9.4399999999999995,"15min":27.789000000000001},"last":0.89800000000000002},{"interface":"front","average":{"1min":0.98899999999999999,"5min":1.6890000000000001,"15min":1.6659999999999999},"min":{"1min":0.54300000000000004,"5min":0.33400000000000002,"15min":0.32300000000000001},"max":{"1min":1.853,"5min":7.2220000000000004,"15min":27.808},"last":1.474}]},{"osd":5,"last update":"Sat Sep 23 19:57:33 2023","interfaces":[{"interface":"back","average":{"1min":0.94799999999999995,"5min":2.0510000000000002,"15min":1.508},"min":{"1min":0.40600000000000003,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":1.6519999999999999,"5min":9.4740000000000002,"15min":22.442},"last":1.5860000000000001},{"interface":"front","average":{"1min":1.03,"5min":2.2919999999999998,"15min":1.544},"min":{"1min":0.59599999999999997,"5min":0.36099999999999999,"15min":0.32100000000000001},"max":{"1min":1.6990000000000001,"5min":12.506,"15min":12.506},"last":1.6200000000000001}]},{"osd":6,"last update":"Sat Sep 23 19:56:58 2023","interfaces":[{"interface":"back","average":{"1min":1.242,"5min":2.2120000000000002,"15min":1.6839999999999999},"min":{"1min":0.69699999999999995,"5min":0.35299999999999998,"15min":0.29399999999999998},"max":{"1min":4.9619999999999997,"5min":13.066000000000001,"15min":22.544},"last":0.97499999999999998},{"interface":"front","average":{"1min":1.2310000000000001,"5min":2.3780000000000001,"15min":1.6319999999999999},"min":{"1min":0.61799999999999999,"5min":0.35499999999999998,"15min":0.31},"max":{"1min":4.4749999999999996,"5min":12.704000000000001,"15min":19.706},"last":0.874}]},{"osd":7,"last update":"Sat Sep 23 19:56:58 2023","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":1.9390000000000001,"15min":2.6970000000000001},"min":{"1min":0.63600000000000001,"5min":0.308,"15min":0.308},"max":{"1min":1.5049999999999999,"5min":10.737,"15min":294.76499999999999},"last":0.747},{"interface":"front","average":{"1min":1.2989999999999999,"5min":2.298,"15min":2.9079999999999999},"min":{"1min":0.74399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":4.8239999999999998,"5min":12.169,"15min":294.47699999999998},"last":1.4039999999999999}]}]},{"osd":5,"up_from":36,"seq":154618823333,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2325484,"kb_used_data":1134308,"kb_used_omap":0,"kb_used_meta":1191168,"kb_avail":91419668,"statfs":{"total":95995035648,"available":93613740032,"internally_reserved":0,"allocated":1161531392,"data_stored":1161011987,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1219756032},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:57:21 2023","interfaces":[{"interface":"back","average":{"1min":0.88500000000000001,"5min":1.3320000000000001,"15min":1.409},"min":{"1min":0.45300000000000001,"5min":0.28399999999999997,"15min":0.253},"max":{"1min":1.216,"5min":9.6720000000000006,"15min":28.280000000000001},"last":0.441},{"interface":"front","average":{"1min":1.0720000000000001,"5min":1.9370000000000001,"15min":1.7889999999999999},"min":{"1min":0.61099999999999999,"5min":0.251,"15min":0.251},"max":{"1min":2.8420000000000001,"5min":12.016,"15min":28.309999999999999},"last":0.46800000000000003}]},{"osd":1,"last update":"Sat Sep 23 19:56:54 2023","interfaces":[{"interface":"back","average":{"1min":1.224,"5min":3.7919999999999998,"15min":2.1840000000000002},"min":{"1min":0.56599999999999995,"5min":0.28899999999999998,"15min":0.28000000000000003},"max":{"1min":3.7959999999999998,"5min":143.51499999999999,"15min":143.51499999999999},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.84499999999999997,"5min":4.2869999999999999,"15min":2.492},"min":{"1min":0.34499999999999997,"5min":0.28999999999999998,"15min":0.28100000000000003},"max":{"1min":1.6739999999999999,"5min":144.005,"15min":144.005},"last":0.51900000000000002}]},{"osd":2,"last update":"Sat Sep 23 19:57:09 2023","interfaces":[{"interface":"back","average":{"1min":0.871,"5min":3.2330000000000001,"15min":3.5129999999999999},"min":{"1min":0.628,"5min":0.32400000000000001,"15min":0.309},"max":{"1min":1.4350000000000001,"5min":75.640000000000001,"15min":241.59700000000001},"last":1.3120000000000001},{"interface":"front","average":{"1min":1.008,"5min":3.3010000000000002,"15min":3.3660000000000001},"min":{"1min":0.68700000000000006,"5min":0.34699999999999998,"15min":0.32000000000000001},"max":{"1min":1.3080000000000001,"5min":79.626000000000005,"15min":242.13999999999999},"last":5.0199999999999996}]},{"osd":3,"last update":"Sat Sep 23 19:56:54 2023","interfaces":[{"interface":"back","average":{"1min":1.0049999999999999,"5min":1.734,"15min":1.4319999999999999},"min":{"1min":0.45900000000000002,"5min":0.35599999999999998,"15min":0.33700000000000002},"max":{"1min":3.2669999999999999,"5min":9.3829999999999991,"15min":9.3829999999999991},"last":3.1720000000000002},{"interface":"front","average":{"1min":1.3080000000000001,"5min":1.577,"15min":1.474},"min":{"1min":0.755,"5min":0.34000000000000002,"15min":0.33400000000000002},"max":{"1min":3.7650000000000001,"5min":9.6110000000000007,"15min":9.6110000000000007},"last":0.66300000000000003}]},{"osd":4,"last update":"Sat Sep 23 19:57:21 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":2.573,"15min":1.6220000000000001},"min":{"1min":0.82699999999999996,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":1.367,"5min":12.051,"15min":12.051},"last":0.55800000000000005},{"interface":"front","average":{"1min":1.0489999999999999,"5min":2.2690000000000001,"15min":1.3779999999999999},"min":{"1min":0.57799999999999996,"5min":0.34000000000000002,"15min":0.318},"max":{"1min":1.5740000000000001,"5min":16.462,"15min":16.462},"last":0.61299999999999999}]},{"osd":6,"last update":"Sat Sep 23 19:57:09 2023","interfaces":[{"interface":"back","average":{"1min":1.1459999999999999,"5min":2.0960000000000001,"15min":1.3959999999999999},"min":{"1min":0.81699999999999995,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.431,"5min":9.641,"15min":9.641},"last":0.85599999999999998},{"interface":"front","average":{"1min":1.1100000000000001,"5min":2.0259999999999998,"15min":1.4450000000000001},"min":{"1min":0.73799999999999999,"5min":0.39600000000000002,"15min":0.28999999999999998},"max":{"1min":1.5009999999999999,"5min":11.444000000000001,"15min":11.444000000000001},"last":0.83299999999999996}]},{"osd":7,"last update":"Sat Sep 23 19:57:21 2023","interfaces":[{"interface":"back","average":{"1min":1.0640000000000001,"5min":2.3740000000000001,"15min":1.659},"min":{"1min":0.77300000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.3089999999999999,"5min":9.407,"15min":13.771000000000001},"last":0.68899999999999995},{"interface":"front","average":{"1min":1.1519999999999999,"5min":2.0259999999999998,"15min":1.542},"min":{"1min":0.64400000000000002,"5min":0.38300000000000001,"15min":0.33900000000000002},"max":{"1min":1.3959999999999999,"5min":17.053000000000001,"15min":17.053000000000001},"last":0.70699999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1984495616,"data_stored":1984495616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1915748352,"data_stored":1915748352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1738473472,"data_stored":1738473472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1731723264,"data_stored":1731723264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1438646272,"data_stored":1438646272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1165950976,"data_stored":1165950976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":967770112,"data_stored":967770112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":711327744,"data_stored":711327744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T19:58:00.719 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T19:58:00.720 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T19:58:00.720 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T19:58:00.720 INFO:tasks.thrashosds.thrasher:Setting osd 5 primary_affinity to 0.553664 2023-09-23T19:58:00.720 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 5 0.5536643048890713 2023-09-23T19:58:00.749 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:00.779 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:00.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:00.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:00.880 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 46 16 63896 63880 86.7613 67.8125 0.032294 0.0115192 2023-09-23T19:58:00.884 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:00.929 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:00.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:00.938 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:01.030 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T19:58:01.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:00 smithi092 bash[58855]: audit 2023-09-23T19:57:59.376676+0000 mgr.y (mgr.25325) 1522 : audit 0 from='client.46901 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:58:01.039 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:01.065 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:01.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:01.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:01.168 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:01.179 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:00 smithi016 bash[75004]: audit 2023-09-23T19:57:59.376676+0000 mgr.y (mgr.25325) 1522 : audit 0 from='client.46901 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:58:01.179 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:00 smithi016 bash[77995]: audit 2023-09-23T19:57:59.376676+0000 mgr.y (mgr.25325) 1522 : audit 0 from='client.46901 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:58:01.199 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:01.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:01.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:01.308 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:01.353 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:01.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:01.359 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:01.460 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:01.501 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:01.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:01.504 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:01.539 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:58:01.605 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:01.643 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:01.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:01.647 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:01.747 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:01.778 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:01.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:01.781 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:01.880 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 47 16 64841 64825 86.172 59.0625 0.00811764 0.0115985 2023-09-23T19:58:01.881 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:01.915 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:01.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:01.917 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:02.018 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:02.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:01 smithi092 bash[58855]: cluster 2023-09-23T19:58:00.576243+0000 mgr.y (mgr.25325) 1523 : cluster 0 pgmap v1548: 89 pgs: 89 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T19:58:02.048 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:02.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:02.052 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:02.153 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:02.164 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:01 smithi016 bash[75004]: cluster 2023-09-23T19:58:00.576243+0000 mgr.y (mgr.25325) 1523 : cluster 0 pgmap v1548: 89 pgs: 89 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T19:58:02.164 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:01 smithi016 bash[77995]: cluster 2023-09-23T19:58:00.576243+0000 mgr.y (mgr.25325) 1523 : cluster 0 pgmap v1548: 89 pgs: 89 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T19:58:02.184 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:02.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:02.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:02.295 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:02.329 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:02.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:02.334 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:02.435 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:02.461 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:02.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:02.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:02.564 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:02.592 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:02.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:02.594 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:02.695 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:02.724 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:02.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:02.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:02.829 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:02.858 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:02.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:02.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:02.880 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 48 16 65797 65781 85.6216 59.75 0.0159621 0.0116728 2023-09-23T19:58:02.964 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:02.989 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:02.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:02.992 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:03.093 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:03.117 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:03.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:03.120 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:03.221 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:03.246 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:03.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:03.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:03.350 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:03.387 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:03.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:03.390 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:03.491 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:03.526 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:03.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:03.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:03.585 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:58:03.632 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:03.667 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:03.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:03.671 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:03.772 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:03.796 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:03.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:03.798 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:03.880 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 49 16 66745 66729 85.0834 59.25 0.00391863 0.0117464 2023-09-23T19:58:03.899 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:03.925 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:03.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:03.927 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:04.028 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:04.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:03 smithi092 bash[58855]: cluster 2023-09-23T19:58:02.576721+0000 mgr.y (mgr.25325) 1524 : cluster 0 pgmap v1549: 89 pgs: 89 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-09-23T19:58:04.063 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:04.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:04.067 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:04.168 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:04.209 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:04.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:04.214 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:04.240 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:03 smithi016 bash[77995]: cluster 2023-09-23T19:58:02.576721+0000 mgr.y (mgr.25325) 1524 : cluster 0 pgmap v1549: 89 pgs: 89 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-09-23T19:58:04.241 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:03 smithi016 bash[75004]: cluster 2023-09-23T19:58:02.576721+0000 mgr.y (mgr.25325) 1524 : cluster 0 pgmap v1549: 89 pgs: 89 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-09-23T19:58:04.315 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:04.346 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:04.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:04.350 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:04.451 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:04.485 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:04.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:04.488 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:04.589 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:04.615 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:04.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:04.617 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:04.718 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:04.746 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:04.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:04.751 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:04.852 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:04.882 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 50 16 67719 67703 84.5962 60.875 0.00854302 0.0118143 2023-09-23T19:58:04.893 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:04.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:04.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:04.998 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:05.023 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:05.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:05.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:05.128 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:05.161 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:05.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:05.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:05.266 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:05.291 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:05.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:05.296 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:05.397 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:05.420 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:05.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:05.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:05.523 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:05.550 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:05.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:05.556 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:05.657 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:05.684 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:05.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:05.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:05.789 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:05.823 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:05.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:05.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:05.882 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 51 16 68705 68689 84.1458 61.625 0.00918503 0.0118778 2023-09-23T19:58:05.928 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:05.951 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:58:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:58:05] "GET /metrics HTTP/1.1" 200 36300 "" "Prometheus/2.33.4" 2023-09-23T19:58:05.957 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:05.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:05.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:06.063 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:06.087 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:06.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:06.090 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:06.190 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:06.218 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:06.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:05 smithi016 bash[75004]: cluster 2023-09-23T19:58:04.577352+0000 mgr.y (mgr.25325) 1525 : cluster 0 pgmap v1550: 89 pgs: 89 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T19:58:06.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:05 smithi016 bash[77995]: cluster 2023-09-23T19:58:04.577352+0000 mgr.y (mgr.25325) 1525 : cluster 0 pgmap v1550: 89 pgs: 89 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T19:58:06.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:06.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:06.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:05 smithi092 bash[58855]: cluster 2023-09-23T19:58:04.577352+0000 mgr.y (mgr.25325) 1525 : cluster 0 pgmap v1550: 89 pgs: 89 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T19:58:06.353 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:06.377 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:06.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:06.379 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:06.480 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:06.505 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:06.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:06.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:06.610 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:06.648 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:06.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:06.651 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:06.743 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:58:06.745 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:58:06.751 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:06.775 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:06.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:06.779 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:06.879 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:06.882 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 52 16 69678 69662 83.6971 60.8125 0.00668742 0.011942 2023-09-23T19:58:06.903 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:06.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:06.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:07.006 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:07.045 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:07.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:07.048 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:07.149 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:07.188 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:07.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:07.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:07.293 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:07.334 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:07.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:07.337 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:07.437 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:07.461 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:07.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:07.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:07.571 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:07.601 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:07.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:07.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:07.705 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:07.727 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:07.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:07.729 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:07.830 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:07.864 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:07.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:07.877 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:07.882 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 53 16 70338 70322 82.8964 41.25 0.0113311 0.0120579 2023-09-23T19:58:07.978 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:08.012 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:08.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:08.016 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:08.117 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:08.145 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:08.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:08.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:08.251 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:08.261 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:08 smithi092 bash[58855]: cluster 2023-09-23T19:58:06.578022+0000 mgr.y (mgr.25325) 1526 : cluster 0 pgmap v1551: 89 pgs: 89 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 920 op/s 2023-09-23T19:58:08.280 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:08.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:08.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:08.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:07 smithi016 bash[77995]: cluster 2023-09-23T19:58:06.578022+0000 mgr.y (mgr.25325) 1526 : cluster 0 pgmap v1551: 89 pgs: 89 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 920 op/s 2023-09-23T19:58:08.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:07 smithi016 bash[75004]: cluster 2023-09-23T19:58:06.578022+0000 mgr.y (mgr.25325) 1526 : cluster 0 pgmap v1551: 89 pgs: 89 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 920 op/s 2023-09-23T19:58:08.385 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:08.413 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:08.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:08.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:08.519 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:08.559 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:08.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:08.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:08.664 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:08.691 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:08.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:08.693 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:08.794 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:08.813 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:08.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:08.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:08.882 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 54 16 71182 71166 82.3382 52.75 0.00763708 0.0121148 2023-09-23T19:58:08.916 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:08.939 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:08.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:08.942 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:09.043 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:09.076 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:09.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:09.080 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:09.181 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:09.209 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:09.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:09.212 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:09.313 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:09.330 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:09.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:09.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:09.432 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:09.463 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:09.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:09.466 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:09.567 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:09.599 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:09.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:09.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:09.703 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:09.732 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:09.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:09.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:09.836 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:09.863 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:09.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:09.868 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:09.882 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 55 16 72052 72036 81.8298 54.375 0.00810995 0.0122148 2023-09-23T19:58:09.969 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:10.002 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:10.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:10.005 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:10.106 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:10.115 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:10 smithi092 bash[58855]: cluster 2023-09-23T19:58:08.578552+0000 mgr.y (mgr.25325) 1527 : cluster 0 pgmap v1552: 89 pgs: 89 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 59 MiB/s wr, 939 op/s 2023-09-23T19:58:10.134 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:10.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:10.138 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:10.238 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:10.267 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:10.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:10.271 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:10.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:09 smithi016 bash[75004]: cluster 2023-09-23T19:58:08.578552+0000 mgr.y (mgr.25325) 1527 : cluster 0 pgmap v1552: 89 pgs: 89 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 59 MiB/s wr, 939 op/s 2023-09-23T19:58:10.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:10 smithi016 bash[77995]: cluster 2023-09-23T19:58:08.578552+0000 mgr.y (mgr.25325) 1527 : cluster 0 pgmap v1552: 89 pgs: 89 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 59 MiB/s wr, 939 op/s 2023-09-23T19:58:10.372 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:10.400 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:10.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:10.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:10.504 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:10.540 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:10.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:10.544 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:10.645 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:10.668 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:10.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:10.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:10.772 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:10.803 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:10.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:10.810 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:10.882 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 56 16 72998 72982 81.4243 59.125 0.0201762 0.0122755 2023-09-23T19:58:10.911 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:10.939 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:10.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:10.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:11.045 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:11.074 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:11.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:11.078 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:11.179 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:11.209 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:11.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:11.212 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:11.313 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:11.338 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:11.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:11.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:11.441 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:11.473 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:11.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:11.477 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:11.578 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:11.602 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:11.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:11.605 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:11.706 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:11.742 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:11.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:11.747 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:11.848 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:11.872 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:11.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:11.877 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:11.882 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 57 16 73877 73861 80.9597 54.9375 0.00684215 0.0123454 2023-09-23T19:58:11.978 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:12.006 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:12.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:12.013 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:12.113 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:12.123 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:12 smithi016 bash[75004]: cluster 2023-09-23T19:58:10.579235+0000 mgr.y (mgr.25325) 1528 : cluster 0 pgmap v1553: 89 pgs: 89 active+clean; 4.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 65 MiB/s wr, 1.05k op/s 2023-09-23T19:58:12.123 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:12 smithi016 bash[77995]: cluster 2023-09-23T19:58:10.579235+0000 mgr.y (mgr.25325) 1528 : cluster 0 pgmap v1553: 89 pgs: 89 active+clean; 4.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 65 MiB/s wr, 1.05k op/s 2023-09-23T19:58:12.146 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:12.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:12.150 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:12.251 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:12.276 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:12.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:12.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:12.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:12 smithi092 bash[58855]: cluster 2023-09-23T19:58:10.579235+0000 mgr.y (mgr.25325) 1528 : cluster 0 pgmap v1553: 89 pgs: 89 active+clean; 4.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 65 MiB/s wr, 1.05k op/s 2023-09-23T19:58:12.386 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:12.417 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:12.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:12.423 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:12.523 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:12.557 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:12.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:12.560 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:12.661 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:12.684 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:12.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:12.686 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:12.787 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:12.834 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:12.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:12.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:12.882 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 58 16 74774 74758 80.5305 56.0625 0.0143856 0.0124123 2023-09-23T19:58:12.939 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:12.963 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:12.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:12.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:13.068 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:13.097 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:13.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:13.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:13.204 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:13.230 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:13.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:13.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:13.335 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:13.371 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:13.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:13.374 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:13.474 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:13.502 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:13.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:13.506 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:13.607 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:13.634 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:13.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:13.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:13.737 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:13.768 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:13.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:13.773 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:13.874 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:13.882 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 59 15 75633 75618 80.0766 53.75 0.0154008 0.0124825 2023-09-23T19:58:13.899 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:13.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:13.901 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:14.002 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:14.037 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:14.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:14.042 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:14.143 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:14.151 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:14 smithi092 bash[58855]: cluster 2023-09-23T19:58:12.579695+0000 mgr.y (mgr.25325) 1529 : cluster 0 pgmap v1554: 89 pgs: 89 active+clean; 4.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 835 op/s 2023-09-23T19:58:14.151 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:14 smithi092 bash[58855]: audit 2023-09-23T19:58:13.692688+0000 mon.a (mon.0) 1845 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:58:14.173 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:14.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:14.176 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:14.276 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:14.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:14 smithi016 bash[77995]: cluster 2023-09-23T19:58:12.579695+0000 mgr.y (mgr.25325) 1529 : cluster 0 pgmap v1554: 89 pgs: 89 active+clean; 4.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 835 op/s 2023-09-23T19:58:14.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:14 smithi016 bash[77995]: audit 2023-09-23T19:58:13.692688+0000 mon.a (mon.0) 1845 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:58:14.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:14 smithi016 bash[75004]: cluster 2023-09-23T19:58:12.579695+0000 mgr.y (mgr.25325) 1529 : cluster 0 pgmap v1554: 89 pgs: 89 active+clean; 4.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 835 op/s 2023-09-23T19:58:14.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:14 smithi016 bash[75004]: audit 2023-09-23T19:58:13.692688+0000 mon.a (mon.0) 1845 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:58:14.305 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:14.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:14.308 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:14.409 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:14.442 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:14.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:14.445 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:14.546 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:14.577 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:14.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:14.579 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:14.606 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-09-23T19:58:14.606 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:58:14.681 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:14.706 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:14.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:14.711 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:14.812 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:14.836 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:14.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:14.844 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:14.883 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-09-23T19:58:14.880205+0000 min lat: 0.001237 max lat: 0.327936 avg lat: 0.012547 2023-09-23T19:58:14.883 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:58:14.883 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 60 16 76519 76503 79.6639 55.3125 0.0103135 0.012547 2023-09-23T19:58:14.945 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:14.976 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:14.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:14.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:15.079 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:15.110 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:15.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:15.115 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:15.215 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:15.240 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:15.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:15.248 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:15.349 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:15.383 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:15.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:15.386 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:15.486 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:15.504 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:15.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:15.506 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:15.607 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:15.633 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:15.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:15.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:15.736 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:15.766 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:15.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:15.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:15.870 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:15.883 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 61 16 77470 77454 79.3322 59.4375 0.0164383 0.0125983 2023-09-23T19:58:15.893 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:15.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:15.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:16.000 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:16.001 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:58:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:58:15] "GET /metrics HTTP/1.1" 200 36299 "" "Prometheus/2.33.4" 2023-09-23T19:58:16.041 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:16.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:16.045 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:16.146 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:16.157 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:16 smithi092 bash[58855]: cluster 2023-09-23T19:58:14.580277+0000 mgr.y (mgr.25325) 1530 : cluster 0 pgmap v1555: 89 pgs: 89 active+clean; 4.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 55 MiB/s wr, 882 op/s 2023-09-23T19:58:16.172 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:16.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:16.176 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:16.277 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:16.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:16 smithi016 bash[75004]: cluster 2023-09-23T19:58:14.580277+0000 mgr.y (mgr.25325) 1530 : cluster 0 pgmap v1555: 89 pgs: 89 active+clean; 4.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 55 MiB/s wr, 882 op/s 2023-09-23T19:58:16.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:16 smithi016 bash[77995]: cluster 2023-09-23T19:58:14.580277+0000 mgr.y (mgr.25325) 1530 : cluster 0 pgmap v1555: 89 pgs: 89 active+clean; 4.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 55 MiB/s wr, 882 op/s 2023-09-23T19:58:16.304 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:16.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:16.306 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:16.407 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:16.431 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:16.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:16.436 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:16.537 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:16.581 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:16.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:16.585 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:16.685 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:16.722 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:16.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:16.724 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:16.825 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:16.858 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:16.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:16.860 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:16.883 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 62 16 78303 78287 78.8924 52.0625 0.0230033 0.0126692 2023-09-23T19:58:16.961 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:16.988 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:16.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:16.991 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:17.092 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:17.124 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:17.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:17.127 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:17.228 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:17.252 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:17.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:17.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:17.356 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:17.393 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:17.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:17.395 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:17.496 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:17.530 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:17.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:17.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:17.635 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:17.664 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:17.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:17.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:17.768 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:17.798 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:17.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:17.804 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:17.883 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 63 16 79119 79103 78.4497 51 0.0145126 0.0127413 2023-09-23T19:58:17.905 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:17.930 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:17.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:17.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:18.033 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:18.064 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:18.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:18.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:18.156 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:18 smithi016 bash[75004]: cluster 2023-09-23T19:58:16.580911+0000 mgr.y (mgr.25325) 1531 : cluster 0 pgmap v1556: 89 pgs: 89 active+clean; 4.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 937 op/s 2023-09-23T19:58:18.156 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:18 smithi016 bash[77995]: cluster 2023-09-23T19:58:16.580911+0000 mgr.y (mgr.25325) 1531 : cluster 0 pgmap v1556: 89 pgs: 89 active+clean; 4.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 937 op/s 2023-09-23T19:58:18.169 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:18.196 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:18.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:18.201 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:18.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:18 smithi092 bash[58855]: cluster 2023-09-23T19:58:16.580911+0000 mgr.y (mgr.25325) 1531 : cluster 0 pgmap v1556: 89 pgs: 89 active+clean; 4.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 937 op/s 2023-09-23T19:58:18.302 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:18.325 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:18.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:18.333 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:18.434 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:18.474 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:18.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:18.477 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:18.578 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:18.603 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:18.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:18.608 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:18.708 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:18.742 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:18.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:18.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:18.851 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:18.882 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:18.883 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 64 16 80017 80001 78.1009 56.125 0.0123048 0.0127983 2023-09-23T19:58:18.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:18.889 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:18.990 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:19.017 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:19.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:19.023 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:19.123 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:19.161 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:19.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:19.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:19.268 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:19.275 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T19:58:19.294 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:19.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:19.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:19.397 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:19.441 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:19.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:19.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:19.544 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:19.567 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:19.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:19.573 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:19.674 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:19.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:19.707 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:19.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:19.799 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:58:19.809 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:19.832 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:19.834 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:58:19.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:19.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:19.880 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:58:19.883 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 65 16 80857 80841 77.707 52.5 0.0054193 0.0128621 2023-09-23T19:58:19.929 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:58:19.940 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:19.970 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:19.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:19.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:19.979 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:58:20.012 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:58:20.054 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:58:20.073 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:20.083 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:20 smithi092 bash[58855]: cluster 2023-09-23T19:58:18.581454+0000 mgr.y (mgr.25325) 1532 : cluster 0 pgmap v1557: 89 pgs: 89 active+clean; 4.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 854 op/s 2023-09-23T19:58:20.083 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:20 smithi092 bash[58855]: audit 2023-09-23T19:58:19.729681+0000 mon.b (mon.1) 60 : audit 0 from='client.? 172.21.15.16:0/3770811239' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:58:20.094 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:58:20.096 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:20.096 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:58:20.097 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:58:20.097 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:58:20.097 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:58:20.097 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:58:20.097 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:58:20.097 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:58:20.097 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:58:20.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:20.099 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:20.199 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:20.222 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:20.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:20.224 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:20.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:20 smithi016 bash[77995]: cluster 2023-09-23T19:58:18.581454+0000 mgr.y (mgr.25325) 1532 : cluster 0 pgmap v1557: 89 pgs: 89 active+clean; 4.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 854 op/s 2023-09-23T19:58:20.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:20 smithi016 bash[77995]: audit 2023-09-23T19:58:19.729681+0000 mon.b (mon.1) 60 : audit 0 from='client.? 172.21.15.16:0/3770811239' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:58:20.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:20 smithi016 bash[75004]: cluster 2023-09-23T19:58:18.581454+0000 mgr.y (mgr.25325) 1532 : cluster 0 pgmap v1557: 89 pgs: 89 active+clean; 4.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 854 op/s 2023-09-23T19:58:20.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:20 smithi016 bash[75004]: audit 2023-09-23T19:58:19.729681+0000 mon.b (mon.1) 60 : audit 0 from='client.? 172.21.15.16:0/3770811239' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:58:20.325 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:20.353 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:20.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:20.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:20.456 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:20.489 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:20.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:20.493 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:20.594 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:20.625 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:20.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:20.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:20.731 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:20.769 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:20.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:20.776 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:20.876 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:20.883 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 66 16 81779 81763 77.4027 57.625 0.016135 0.0129133 2023-09-23T19:58:20.904 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:20.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:20.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:21.009 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:21.038 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:21.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:21.045 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:21.146 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:21.175 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:21.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:21.178 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:21.279 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:21.307 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:21.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:21.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:21.410 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:21.443 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:21.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:21.446 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:21.547 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:21.577 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:21.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:21.580 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:21.681 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:21.707 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:21.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:21.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:21.814 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:21.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:21.851 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:21.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:21.883 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 67 16 82639 82623 77.0497 53.75 0.00828052 0.0129729 2023-09-23T19:58:21.953 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:21.981 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:21.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:21.985 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:22.086 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:22.110 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:22.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:22.115 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:22.215 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:22.247 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:22.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:22.249 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:22.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:22 smithi016 bash[77995]: cluster 2023-09-23T19:58:20.582205+0000 mgr.y (mgr.25325) 1533 : cluster 0 pgmap v1558: 89 pgs: 89 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 978 op/s 2023-09-23T19:58:22.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:22 smithi016 bash[75004]: cluster 2023-09-23T19:58:20.582205+0000 mgr.y (mgr.25325) 1533 : cluster 0 pgmap v1558: 89 pgs: 89 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 978 op/s 2023-09-23T19:58:22.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:22 smithi092 bash[58855]: cluster 2023-09-23T19:58:20.582205+0000 mgr.y (mgr.25325) 1533 : cluster 0 pgmap v1558: 89 pgs: 89 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 978 op/s 2023-09-23T19:58:22.350 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:22.374 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:22.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:22.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:22.478 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:22.505 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:22.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:22.509 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:22.609 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:22.637 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:22.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:22.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:22.743 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:22.772 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:22.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:22.778 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:22.878 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:22.883 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 68 16 83583 83567 76.7842 59 0.0272056 0.0130173 2023-09-23T19:58:22.905 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:22.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:22.907 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:23.008 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:23.042 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:23.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:23.046 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:23.148 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:23.172 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:23.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:23.179 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:23.280 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:23.308 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:23.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:23.311 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:23.412 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:23.451 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:23.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:23.454 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:23.555 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:23.587 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:23.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:23.593 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:23.693 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:23.719 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:23.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:23.721 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:23.822 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:23.850 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:23.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:23.855 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:23.884 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 69 16 84419 84403 76.4286 52.25 0.0119346 0.0130768 2023-09-23T19:58:23.955 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:23.982 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:23.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:23.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:24.085 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:24.109 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:24.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:24.112 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:24.212 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:24.258 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:24.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:24.266 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:24.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:24 smithi092 bash[58855]: cluster 2023-09-23T19:58:22.582741+0000 mgr.y (mgr.25325) 1534 : cluster 0 pgmap v1559: 89 pgs: 89 active+clean; 5.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 807 op/s 2023-09-23T19:58:24.366 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:24.376 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:24 smithi016 bash[75004]: cluster 2023-09-23T19:58:22.582741+0000 mgr.y (mgr.25325) 1534 : cluster 0 pgmap v1559: 89 pgs: 89 active+clean; 5.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 807 op/s 2023-09-23T19:58:24.376 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:24 smithi016 bash[77995]: cluster 2023-09-23T19:58:22.582741+0000 mgr.y (mgr.25325) 1534 : cluster 0 pgmap v1559: 89 pgs: 89 active+clean; 5.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 807 op/s 2023-09-23T19:58:24.401 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:24.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:24.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:24.506 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:24.553 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:24.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:24.557 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:24.658 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:24.692 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:24.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:24.696 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:24.797 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:24.826 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:24.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:24.830 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:24.884 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 70 16 85188 85172 76.0234 48.0625 0.0200568 0.0131479 2023-09-23T19:58:24.930 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:24.963 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:24.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:24.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:25.070 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is unset 2023-09-23T19:58:25.070 INFO:teuthology.orchestra.run.smithi016.stderr:set osd.5 primary-affinity to 0.553664 (8362842) 2023-09-23T19:58:25.071 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:25.071 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:25 smithi092 bash[58855]: audit 2023-09-23T19:58:24.356620+0000 mon.c (mon.2) 232 : audit 1 from='client.? 172.21.15.16:0/1101998142' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.5536643048890713}]: dispatch 2023-09-23T19:58:25.072 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:25 smithi092 bash[58855]: audit 2023-09-23T19:58:24.357811+0000 mon.a (mon.0) 1846 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.5536643048890713}]: dispatch 2023-09-23T19:58:25.072 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:25 smithi092 bash[58855]: audit 2023-09-23T19:58:24.365373+0000 mon.c (mon.2) 233 : audit 1 from='client.? 172.21.15.16:0/1620654219' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:58:25.072 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:25 smithi092 bash[58855]: audit 2023-09-23T19:58:24.366302+0000 mon.a (mon.0) 1847 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:58:25.097 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:25.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:25.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:25.200 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:25.229 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:25.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:25.236 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:25.272 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:58:25.282 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:25 smithi016 bash[77995]: audit 2023-09-23T19:58:24.356620+0000 mon.c (mon.2) 232 : audit 1 from='client.? 172.21.15.16:0/1101998142' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.5536643048890713}]: dispatch 2023-09-23T19:58:25.282 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:25 smithi016 bash[77995]: audit 2023-09-23T19:58:24.357811+0000 mon.a (mon.0) 1846 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.5536643048890713}]: dispatch 2023-09-23T19:58:25.282 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:25 smithi016 bash[77995]: audit 2023-09-23T19:58:24.365373+0000 mon.c (mon.2) 233 : audit 1 from='client.? 172.21.15.16:0/1620654219' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:58:25.282 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:25 smithi016 bash[77995]: audit 2023-09-23T19:58:24.366302+0000 mon.a (mon.0) 1847 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:58:25.282 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:25 smithi016 bash[75004]: audit 2023-09-23T19:58:24.356620+0000 mon.c (mon.2) 232 : audit 1 from='client.? 172.21.15.16:0/1101998142' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.5536643048890713}]: dispatch 2023-09-23T19:58:25.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:25 smithi016 bash[75004]: audit 2023-09-23T19:58:24.357811+0000 mon.a (mon.0) 1846 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.5536643048890713}]: dispatch 2023-09-23T19:58:25.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:25 smithi016 bash[75004]: audit 2023-09-23T19:58:24.365373+0000 mon.c (mon.2) 233 : audit 1 from='client.? 172.21.15.16:0/1620654219' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:58:25.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:25 smithi016 bash[75004]: audit 2023-09-23T19:58:24.366302+0000 mon.a (mon.0) 1847 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:58:25.336 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:25.363 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:25.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:25.368 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:25.469 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:25.497 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:25.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:25.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:25.600 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:25.626 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:25.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:25.628 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:25.729 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:25.762 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:25.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:25.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:25.865 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:25.884 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 71 16 85559 85543 75.2793 23.1875 0.00454728 0.0131585 2023-09-23T19:58:25.900 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:25.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:25.903 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:25.988 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:58:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:58:25] "GET /metrics HTTP/1.1" 200 36299 "" "Prometheus/2.33.4" 2023-09-23T19:58:26.004 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:26.038 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:26.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:26.041 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:26.142 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:26.150 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:26 smithi092 bash[58855]: cluster 2023-09-23T19:58:24.583390+0000 mgr.y (mgr.25325) 1535 : cluster 0 pgmap v1560: 89 pgs: 89 active+clean; 5.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 851 op/s 2023-09-23T19:58:26.150 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:26 smithi092 bash[58855]: cluster 2023-09-23T19:58:25.046139+0000 mon.a (mon.0) 1848 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:58:26.150 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:26 smithi092 bash[58855]: audit 2023-09-23T19:58:25.054717+0000 mon.a (mon.0) 1849 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.5536643048890713}]': finished 2023-09-23T19:58:26.151 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:26 smithi092 bash[58855]: audit 2023-09-23T19:58:25.055407+0000 mon.a (mon.0) 1850 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:58:26.151 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:26 smithi092 bash[58855]: cluster 2023-09-23T19:58:25.055757+0000 mon.a (mon.0) 1851 : cluster 0 osdmap e263: 8 total, 8 up, 8 in 2023-09-23T19:58:26.151 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:26 smithi092 bash[58855]: cluster 2023-09-23T19:58:25.063278+0000 osd.7 (osd.7) 175 : cluster 3 failed to encode map e263 with expected crc 2023-09-23T19:58:26.151 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:26 smithi092 bash[58855]: cluster 2023-09-23T19:58:25.064215+0000 osd.6 (osd.6) 197 : cluster 3 failed to encode map e263 with expected crc 2023-09-23T19:58:26.151 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:26 smithi092 bash[58855]: cluster 2023-09-23T19:58:25.065915+0000 osd.4 (osd.4) 204 : cluster 3 failed to encode map e263 with expected crc 2023-09-23T19:58:26.165 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:26.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:26.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:26.269 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:26.277 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:26 smithi016 bash[75004]: cluster 2023-09-23T19:58:24.583390+0000 mgr.y (mgr.25325) 1535 : cluster 0 pgmap v1560: 89 pgs: 89 active+clean; 5.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 851 op/s 2023-09-23T19:58:26.278 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:26 smithi016 bash[75004]: cluster 2023-09-23T19:58:25.046139+0000 mon.a (mon.0) 1848 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:58:26.278 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:26 smithi016 bash[75004]: audit 2023-09-23T19:58:25.054717+0000 mon.a (mon.0) 1849 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.5536643048890713}]': finished 2023-09-23T19:58:26.278 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:26 smithi016 bash[75004]: audit 2023-09-23T19:58:25.055407+0000 mon.a (mon.0) 1850 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:58:26.278 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:26 smithi016 bash[75004]: cluster 2023-09-23T19:58:25.055757+0000 mon.a (mon.0) 1851 : cluster 0 osdmap e263: 8 total, 8 up, 8 in 2023-09-23T19:58:26.278 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:26 smithi016 bash[75004]: cluster 2023-09-23T19:58:25.063278+0000 osd.7 (osd.7) 175 : cluster 3 failed to encode map e263 with expected crc 2023-09-23T19:58:26.278 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:26 smithi016 bash[75004]: cluster 2023-09-23T19:58:25.064215+0000 osd.6 (osd.6) 197 : cluster 3 failed to encode map e263 with expected crc 2023-09-23T19:58:26.278 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:26 smithi016 bash[75004]: cluster 2023-09-23T19:58:25.065915+0000 osd.4 (osd.4) 204 : cluster 3 failed to encode map e263 with expected crc 2023-09-23T19:58:26.279 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:26 smithi016 bash[77995]: cluster 2023-09-23T19:58:24.583390+0000 mgr.y (mgr.25325) 1535 : cluster 0 pgmap v1560: 89 pgs: 89 active+clean; 5.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 851 op/s 2023-09-23T19:58:26.279 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:26 smithi016 bash[77995]: cluster 2023-09-23T19:58:25.046139+0000 mon.a (mon.0) 1848 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:58:26.279 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:26 smithi016 bash[77995]: audit 2023-09-23T19:58:25.054717+0000 mon.a (mon.0) 1849 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.5536643048890713}]': finished 2023-09-23T19:58:26.279 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:26 smithi016 bash[77995]: audit 2023-09-23T19:58:25.055407+0000 mon.a (mon.0) 1850 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:58:26.279 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:26 smithi016 bash[77995]: cluster 2023-09-23T19:58:25.055757+0000 mon.a (mon.0) 1851 : cluster 0 osdmap e263: 8 total, 8 up, 8 in 2023-09-23T19:58:26.279 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:26 smithi016 bash[77995]: cluster 2023-09-23T19:58:25.063278+0000 osd.7 (osd.7) 175 : cluster 3 failed to encode map e263 with expected crc 2023-09-23T19:58:26.279 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:26 smithi016 bash[77995]: cluster 2023-09-23T19:58:25.064215+0000 osd.6 (osd.6) 197 : cluster 3 failed to encode map e263 with expected crc 2023-09-23T19:58:26.280 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:26 smithi016 bash[77995]: cluster 2023-09-23T19:58:25.065915+0000 osd.4 (osd.4) 204 : cluster 3 failed to encode map e263 with expected crc 2023-09-23T19:58:26.298 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:26.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:26.304 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:26.404 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:26.429 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:26.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:26.432 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:26.532 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:26.556 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:26.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:26.560 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:26.661 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:26.689 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:26.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:26.692 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:26.792 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:26.829 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:26.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:26.834 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:26.884 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 72 15 86221 86206 74.8093 41.4375 0.0136768 0.0133608 2023-09-23T19:58:26.934 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:26.958 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:26.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:26.964 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:27.065 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:27.089 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:27.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:27.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:27.094 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:27 smithi016 bash[75004]: cluster 2023-09-23T19:58:26.073386+0000 mon.a (mon.0) 1852 : cluster 0 osdmap e264: 8 total, 8 up, 8 in 2023-09-23T19:58:27.094 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:27 smithi016 bash[75004]: cluster 2023-09-23T19:58:26.077264+0000 osd.4 (osd.4) 205 : cluster 3 failed to encode map e264 with expected crc 2023-09-23T19:58:27.094 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:27 smithi016 bash[75004]: cluster 2023-09-23T19:58:26.079281+0000 osd.7 (osd.7) 176 : cluster 3 failed to encode map e264 with expected crc 2023-09-23T19:58:27.094 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:27 smithi016 bash[75004]: cluster 2023-09-23T19:58:26.079606+0000 osd.6 (osd.6) 198 : cluster 3 failed to encode map e264 with expected crc 2023-09-23T19:58:27.095 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:27 smithi016 bash[75004]: cluster 2023-09-23T19:58:26.080913+0000 osd.4 (osd.4) 206 : cluster 3 failed to encode map e264 with expected crc 2023-09-23T19:58:27.095 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:27 smithi016 bash[75004]: cluster 2023-09-23T19:58:26.082127+0000 osd.7 (osd.7) 177 : cluster 3 failed to encode map e264 with expected crc 2023-09-23T19:58:27.095 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:27 smithi016 bash[75004]: cluster 2023-09-23T19:58:26.083143+0000 osd.6 (osd.6) 199 : cluster 3 failed to encode map e264 with expected crc 2023-09-23T19:58:27.192 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:27.202 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:27 smithi092 bash[58855]: cluster 2023-09-23T19:58:26.073386+0000 mon.a (mon.0) 1852 : cluster 0 osdmap e264: 8 total, 8 up, 8 in 2023-09-23T19:58:27.202 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:27 smithi092 bash[58855]: cluster 2023-09-23T19:58:26.077264+0000 osd.4 (osd.4) 205 : cluster 3 failed to encode map e264 with expected crc 2023-09-23T19:58:27.202 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:27 smithi092 bash[58855]: cluster 2023-09-23T19:58:26.079281+0000 osd.7 (osd.7) 176 : cluster 3 failed to encode map e264 with expected crc 2023-09-23T19:58:27.202 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:27 smithi092 bash[58855]: cluster 2023-09-23T19:58:26.079606+0000 osd.6 (osd.6) 198 : cluster 3 failed to encode map e264 with expected crc 2023-09-23T19:58:27.202 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:27 smithi092 bash[58855]: cluster 2023-09-23T19:58:26.080913+0000 osd.4 (osd.4) 206 : cluster 3 failed to encode map e264 with expected crc 2023-09-23T19:58:27.203 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:27 smithi092 bash[58855]: cluster 2023-09-23T19:58:26.082127+0000 osd.7 (osd.7) 177 : cluster 3 failed to encode map e264 with expected crc 2023-09-23T19:58:27.203 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:27 smithi092 bash[58855]: cluster 2023-09-23T19:58:26.083143+0000 osd.6 (osd.6) 199 : cluster 3 failed to encode map e264 with expected crc 2023-09-23T19:58:27.220 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:27.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:27.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:27.325 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:27.349 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:27.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:27.355 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:27 smithi016 bash[77995]: cluster 2023-09-23T19:58:26.073386+0000 mon.a (mon.0) 1852 : cluster 0 osdmap e264: 8 total, 8 up, 8 in 2023-09-23T19:58:27.355 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:27 smithi016 bash[77995]: cluster 2023-09-23T19:58:26.077264+0000 osd.4 (osd.4) 205 : cluster 3 failed to encode map e264 with expected crc 2023-09-23T19:58:27.355 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:27 smithi016 bash[77995]: cluster 2023-09-23T19:58:26.079281+0000 osd.7 (osd.7) 176 : cluster 3 failed to encode map e264 with expected crc 2023-09-23T19:58:27.355 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:27 smithi016 bash[77995]: cluster 2023-09-23T19:58:26.079606+0000 osd.6 (osd.6) 198 : cluster 3 failed to encode map e264 with expected crc 2023-09-23T19:58:27.355 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:27 smithi016 bash[77995]: cluster 2023-09-23T19:58:26.080913+0000 osd.4 (osd.4) 206 : cluster 3 failed to encode map e264 with expected crc 2023-09-23T19:58:27.355 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:27 smithi016 bash[77995]: cluster 2023-09-23T19:58:26.082127+0000 osd.7 (osd.7) 177 : cluster 3 failed to encode map e264 with expected crc 2023-09-23T19:58:27.355 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:27 smithi016 bash[77995]: cluster 2023-09-23T19:58:26.083143+0000 osd.6 (osd.6) 199 : cluster 3 failed to encode map e264 with expected crc 2023-09-23T19:58:27.356 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:27.457 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:27.489 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:27.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:27.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:27.593 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:27.618 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:27.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:27.622 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:27.723 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:27.760 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:27.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:27.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:27.868 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:27.885 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 73 16 87063 87047 74.5038 52.5625 0.01565 0.0134148 2023-09-23T19:58:27.897 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:27.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:27.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:28.004 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:28.032 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:28.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:28.035 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:28.136 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:28.171 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:28.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:28.173 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:28.274 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:28.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:28 smithi016 bash[77995]: cluster 2023-09-23T19:58:26.584039+0000 mgr.y (mgr.25325) 1536 : cluster 0 pgmap v1563: 89 pgs: 3 peering, 86 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 912 op/s 2023-09-23T19:58:28.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:28 smithi016 bash[77995]: cluster 2023-09-23T19:58:27.075022+0000 mon.a (mon.0) 1853 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T19:58:28.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:28 smithi016 bash[75004]: cluster 2023-09-23T19:58:26.584039+0000 mgr.y (mgr.25325) 1536 : cluster 0 pgmap v1563: 89 pgs: 3 peering, 86 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 912 op/s 2023-09-23T19:58:28.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:28 smithi016 bash[75004]: cluster 2023-09-23T19:58:27.075022+0000 mon.a (mon.0) 1853 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T19:58:28.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:28 smithi092 bash[58855]: cluster 2023-09-23T19:58:26.584039+0000 mgr.y (mgr.25325) 1536 : cluster 0 pgmap v1563: 89 pgs: 3 peering, 86 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 912 op/s 2023-09-23T19:58:28.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:28 smithi092 bash[58855]: cluster 2023-09-23T19:58:27.075022+0000 mon.a (mon.0) 1853 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T19:58:28.303 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:28.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:28.309 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:28.410 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:28.437 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:28.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:28.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:28.541 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:28.574 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:28.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:28.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:28.679 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:28.705 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:28.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:28.708 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:28.808 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:28.833 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:28.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:28.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:28.885 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 74 16 87913 87897 74.2149 53.125 0.0128377 0.0134687 2023-09-23T19:58:28.936 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:28.964 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:28.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:28.968 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:29.069 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:29.096 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:29.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:29.099 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:29.200 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:29.231 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:29.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:29.236 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:29.280 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T19:58:29.338 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:29.385 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:29.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:29.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:29.490 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:29.521 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:29.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:29.525 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:29.625 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:29.650 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:29.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:29.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:29.754 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:29.785 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:29.786 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:58:29.787 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:29 smithi092 bash[58855]: cluster 2023-09-23T19:58:28.585016+0000 mgr.y (mgr.25325) 1537 : cluster 0 pgmap v1564: 89 pgs: 3 peering, 86 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 43 MiB/s wr, 681 op/s 2023-09-23T19:58:29.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:29 smithi092 bash[58855]: audit 2023-09-23T19:58:28.699035+0000 mon.a (mon.0) 1854 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:58:29.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:29 smithi092 bash[58855]: audit 2023-09-23T19:58:28.700434+0000 mon.a (mon.0) 1855 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:58:29.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:29.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:29.888 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 75 16 88660 88644 73.8448 46.6875 0.010981 0.0135368 2023-09-23T19:58:29.897 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:29.905 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:29 smithi016 bash[75004]: cluster 2023-09-23T19:58:28.585016+0000 mgr.y (mgr.25325) 1537 : cluster 0 pgmap v1564: 89 pgs: 3 peering, 86 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 43 MiB/s wr, 681 op/s 2023-09-23T19:58:29.906 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:29 smithi016 bash[75004]: audit 2023-09-23T19:58:28.699035+0000 mon.a (mon.0) 1854 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:58:29.906 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:29 smithi016 bash[75004]: audit 2023-09-23T19:58:28.700434+0000 mon.a (mon.0) 1855 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:58:29.906 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:29 smithi016 bash[77995]: cluster 2023-09-23T19:58:28.585016+0000 mgr.y (mgr.25325) 1537 : cluster 0 pgmap v1564: 89 pgs: 3 peering, 86 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 43 MiB/s wr, 681 op/s 2023-09-23T19:58:29.906 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:29 smithi016 bash[77995]: audit 2023-09-23T19:58:28.699035+0000 mon.a (mon.0) 1854 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:58:29.906 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:29 smithi016 bash[77995]: audit 2023-09-23T19:58:28.700434+0000 mon.a (mon.0) 1855 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:58:29.934 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:29.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:29.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:30.038 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:30.062 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:30.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:30.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:30.168 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:30.198 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:30.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:30.203 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:30.304 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:30.343 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:30.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:30.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:30.447 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:30.476 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:30.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:30.479 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:30.579 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:30.614 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:30.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:30.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:30.720 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:30.745 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:30.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:30.747 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:30.848 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:30.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:30.877 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:30.878 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:30.888 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 76 16 89468 89452 73.5377 50.5 0.0180395 0.013593 2023-09-23T19:58:30.979 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:31.003 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:31.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:31.007 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:31.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:30 smithi016 bash[75004]: cluster 2023-09-23T19:58:30.765943+0000 mon.a (mon.0) 1856 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-09-23T19:58:31.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:30 smithi016 bash[77995]: cluster 2023-09-23T19:58:30.765943+0000 mon.a (mon.0) 1856 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-09-23T19:58:31.107 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:31.131 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:31.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:31.133 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:31.234 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:31.244 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:30 smithi092 bash[58855]: cluster 2023-09-23T19:58:30.765943+0000 mon.a (mon.0) 1856 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-09-23T19:58:31.262 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:31.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:31.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:31.366 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:31.390 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:31.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:31.393 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:31.494 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:31.521 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:31.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:31.524 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:31.625 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:31.654 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:31.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:31.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:31.759 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:31.788 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:31.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:31.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:31.831 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:58:31.891 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 77 16 90265 90249 73.2268 49.8125 0.0227814 0.013651 2023-09-23T19:58:31.894 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:31.941 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:31.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:31.944 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:32.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:31 smithi092 bash[58855]: cluster 2023-09-23T19:58:30.586638+0000 mgr.y (mgr.25325) 1538 : cluster 0 pgmap v1565: 89 pgs: 89 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 55 MiB/s wr, 873 op/s 2023-09-23T19:58:32.045 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:32.067 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:32.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:32.069 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:32.168 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:31 smithi016 bash[75004]: cluster 2023-09-23T19:58:30.586638+0000 mgr.y (mgr.25325) 1538 : cluster 0 pgmap v1565: 89 pgs: 89 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 55 MiB/s wr, 873 op/s 2023-09-23T19:58:32.170 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:31 smithi016 bash[77995]: cluster 2023-09-23T19:58:30.586638+0000 mgr.y (mgr.25325) 1538 : cluster 0 pgmap v1565: 89 pgs: 89 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 55 MiB/s wr, 873 op/s 2023-09-23T19:58:32.170 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:32.200 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:32.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:32.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:32.263 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-09-23T19:58:32.264 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T19:58:32.304 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:32.329 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:32.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:32.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:32.431 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:32.459 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:32.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:32.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:32.562 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:32.586 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:32.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:32.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:32.690 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:32.726 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:32.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:32.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:32.836 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:32.865 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:32.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:32.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:32.891 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 78 15 91005 90990 72.8818 46.3125 0.0206453 0.0137146 2023-09-23T19:58:32.969 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:33.003 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:33.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:33.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:33.107 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:33.135 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:33.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:33.140 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:33.241 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:33.270 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:33.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:33.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:33.374 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:33.406 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:33.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:33.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:33.513 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:33.545 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:33.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:33.549 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:33.650 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:33.683 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:33.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:33.687 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:33.788 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:33.814 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:33.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:33.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:33.891 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:33 smithi016 bash[75004]: cluster 2023-09-23T19:58:32.587143+0000 mgr.y (mgr.25325) 1539 : cluster 0 pgmap v1566: 89 pgs: 89 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 50 MiB/s wr, 801 op/s 2023-09-23T19:58:33.891 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:33 smithi016 bash[77995]: cluster 2023-09-23T19:58:32.587143+0000 mgr.y (mgr.25325) 1539 : cluster 0 pgmap v1566: 89 pgs: 89 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 50 MiB/s wr, 801 op/s 2023-09-23T19:58:33.892 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 79 16 91813 91797 72.5977 50.4375 0.0046944 0.0137682 2023-09-23T19:58:33.919 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:33.956 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:33.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:33.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:34.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:33 smithi092 bash[58855]: cluster 2023-09-23T19:58:32.587143+0000 mgr.y (mgr.25325) 1539 : cluster 0 pgmap v1566: 89 pgs: 89 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 50 MiB/s wr, 801 op/s 2023-09-23T19:58:34.059 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:34.090 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:34.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:34.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:34.193 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:34.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:34.231 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:34.231 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:34.332 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:34.355 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:34.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:34.357 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:34.458 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:34.479 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:34.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:34.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:34.584 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:34.612 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:34.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:34.615 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:34.715 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:34.739 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:34.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:34.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:34.845 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:34.879 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:34.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:34.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:34.891 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:2023-09-23T19:58:34.889005+0000 min lat: 0.001237 max lat: 1.05209 avg lat: 0.0138296 2023-09-23T19:58:34.892 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:58:34.892 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 80 16 92567 92551 72.2793 47.125 0.0279751 0.0138296 2023-09-23T19:58:34.986 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:35.016 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:35.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:35.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:35.122 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:35.150 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:35.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:35.153 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:35.254 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:35.278 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:35.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:35.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:35.351 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:58:35.382 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:35.421 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:35.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:35.425 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:35.525 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:35.574 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:35.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:35.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:35.677 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:35.709 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:35.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:35.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:35.785 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:58:35 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:58:35] "GET /metrics HTTP/1.1" 200 36371 "" "Prometheus/2.33.4" 2023-09-23T19:58:35.814 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:35.841 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:35.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:35.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:35.891 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 81 16 93318 93302 71.9665 46.9375 0.0168378 0.0138897 2023-09-23T19:58:35.948 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:35.973 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:35.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:35.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:36.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:35 smithi092 bash[58855]: cluster 2023-09-23T19:58:34.588607+0000 mgr.y (mgr.25325) 1540 : cluster 0 pgmap v1567: 89 pgs: 89 active+clean; 5.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 49 MiB/s wr, 776 op/s 2023-09-23T19:58:36.077 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:36.103 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:36.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:36.108 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:36.208 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:36.248 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:36.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:36.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:36.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:35 smithi016 bash[75004]: cluster 2023-09-23T19:58:34.588607+0000 mgr.y (mgr.25325) 1540 : cluster 0 pgmap v1567: 89 pgs: 89 active+clean; 5.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 49 MiB/s wr, 776 op/s 2023-09-23T19:58:36.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:35 smithi016 bash[77995]: cluster 2023-09-23T19:58:34.588607+0000 mgr.y (mgr.25325) 1540 : cluster 0 pgmap v1567: 89 pgs: 89 active+clean; 5.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 49 MiB/s wr, 776 op/s 2023-09-23T19:58:36.353 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:36.378 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:36.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:36.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:36.482 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:36.508 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:36.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:36.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:36.613 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:36.641 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:36.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:36.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:36.745 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:36.784 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:36.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:36.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:36.892 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 82 16 94082 94066 71.6712 47.75 0.00931218 0.0139465 2023-09-23T19:58:36.894 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:36.923 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:36.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:36.925 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:37.026 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:37.067 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:37.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:37.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:37.172 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:37.201 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:37.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:37.400 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:37.501 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:37.534 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:37.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:37.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:37.637 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:37.667 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:37.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:37.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:37.772 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:37.807 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:37.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:37.810 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:37.892 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 83 16 94886 94870 71.4131 50.25 0.0205512 0.013997 2023-09-23T19:58:37.910 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:37.937 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:37.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:37.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:38.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:37 smithi016 bash[77995]: cluster 2023-09-23T19:58:36.589296+0000 mgr.y (mgr.25325) 1541 : cluster 0 pgmap v1568: 89 pgs: 89 active+clean; 5.6 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 44 MiB/s wr, 706 op/s 2023-09-23T19:58:38.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:37 smithi016 bash[75004]: cluster 2023-09-23T19:58:36.589296+0000 mgr.y (mgr.25325) 1541 : cluster 0 pgmap v1568: 89 pgs: 89 active+clean; 5.6 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 44 MiB/s wr, 706 op/s 2023-09-23T19:58:38.041 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:38.070 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:38.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:38.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:38.174 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:38.182 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:37 smithi092 bash[58855]: cluster 2023-09-23T19:58:36.589296+0000 mgr.y (mgr.25325) 1541 : cluster 0 pgmap v1568: 89 pgs: 89 active+clean; 5.6 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 44 MiB/s wr, 706 op/s 2023-09-23T19:58:38.204 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:38.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:38.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:38.309 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:38.345 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:38.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:38.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:38.448 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:38.483 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:38.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:38.492 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:38.593 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:38.624 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:38.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:38.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:38.730 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:38.763 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:38.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:38.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:38.866 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:38.892 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 84 16 95554 95538 71.06 41.75 0.0130764 0.0140663 2023-09-23T19:58:38.915 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:38.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:38.918 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:39.019 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:39.046 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:39.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:39.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:39.150 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:39.185 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:39.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:39.188 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:39.289 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:39.313 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:39.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:39.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:39.416 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:39.441 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:39.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:39.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:39.544 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:39.572 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:39.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:39.573 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:39.674 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:39.719 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:39.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:39.722 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:39.787 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:58:39.822 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:39.849 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:39.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:39.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:39.892 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 85 16 96302 96286 70.774 46.75 0.0186371 0.0141238 2023-09-23T19:58:39.951 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:39.974 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:39.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:39.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:40.078 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:40.110 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:40.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:40.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:40.215 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:40.242 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:40.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:40.248 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:40.348 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:40.370 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:40.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:40.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:40.477 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:40.497 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:40.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:40.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:40.600 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:40.608 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:40 smithi016 bash[77995]: cluster 2023-09-23T19:58:38.589827+0000 mgr.y (mgr.25325) 1542 : cluster 0 pgmap v1569: 89 pgs: 89 active+clean; 5.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 45 MiB/s wr, 720 op/s 2023-09-23T19:58:40.608 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:40 smithi016 bash[77995]: audit 2023-09-23T19:58:39.348503+0000 mon.a (mon.0) 1857 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:58:40.609 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:40 smithi016 bash[75004]: cluster 2023-09-23T19:58:38.589827+0000 mgr.y (mgr.25325) 1542 : cluster 0 pgmap v1569: 89 pgs: 89 active+clean; 5.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 45 MiB/s wr, 720 op/s 2023-09-23T19:58:40.609 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:40 smithi016 bash[75004]: audit 2023-09-23T19:58:39.348503+0000 mon.a (mon.0) 1857 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:58:40.631 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:40.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:40.636 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:40.736 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:40.745 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:40 smithi092 bash[58855]: cluster 2023-09-23T19:58:38.589827+0000 mgr.y (mgr.25325) 1542 : cluster 0 pgmap v1569: 89 pgs: 89 active+clean; 5.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 45 MiB/s wr, 720 op/s 2023-09-23T19:58:40.745 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:40 smithi092 bash[58855]: audit 2023-09-23T19:58:39.348503+0000 mon.a (mon.0) 1857 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:58:40.765 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:40.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:40.768 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:40.869 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:40.892 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 86 16 97025 97009 70.4765 45.1875 0.0100348 0.0141827 2023-09-23T19:58:40.897 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:40.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:40.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:41.001 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:41.029 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:41.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:41.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:41.131 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:41.160 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:41.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:41.164 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:41.179 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-09-23T19:58:41.179 INFO:teuthology.orchestra.run.smithi016.stdout: "size": 20, 2023-09-23T19:58:41.180 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 600, 2023-09-23T19:58:41.180 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [ 2023-09-23T19:58:41.180 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.180 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46640.0:57317 13.2 13:493ed141:::benchmark_data_smithi016_205181_object57316:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e238)", 2023-09-23T19:58:41.180 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:52:28.021782+0000", 2023-09-23T19:58:41.180 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 373.15132774800003, 2023-09-23T19:58:41.180 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.294876533, 2023-09-23T19:58:41.180 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:58:41.180 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:41.180 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:58:41.180 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46640", 2023-09-23T19:58:41.180 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2656794142", 2023-09-23T19:58:41.181 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 57317 2023-09-23T19:58:41.181 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.181 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:58:41.181 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.181 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:58:41.181 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.021782+0000", 2023-09-23T19:58:41.181 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:58:41.181 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.181 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.181 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:58:41.181 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.021782+0000", 2023-09-23T19:58:41.181 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:58:41.181 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.182 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.182 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:58:41.182 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.021780+0000", 2023-09-23T19:58:41.182 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-09-23T19:58:41.182 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.182 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.182 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:58:41.182 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.021824+0000", 2023-09-23T19:58:41.182 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.4271999999999999e-05 2023-09-23T19:58:41.182 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.182 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.182 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:58:41.182 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.021841+0000", 2023-09-23T19:58:41.183 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.6612999999999998e-05 2023-09-23T19:58:41.183 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.183 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.183 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:58:41.183 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.021850+0000", 2023-09-23T19:58:41.183 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.2879999999999998e-06 2023-09-23T19:58:41.183 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.183 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.183 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:58:41.183 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.021919+0000", 2023-09-23T19:58:41.183 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.8973999999999994e-05 2023-09-23T19:58:41.183 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.183 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.184 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:58:41.184 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.022104+0000", 2023-09-23T19:58:41.184 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00018429699999999999 2023-09-23T19:58:41.184 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.184 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.184 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 4,5", 2023-09-23T19:58:41.184 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.022342+0000", 2023-09-23T19:58:41.184 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000238139 2023-09-23T19:58:41.184 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.184 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.184 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:58:41.184 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.024499+0000", 2023-09-23T19:58:41.185 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0021574530000000001 2023-09-23T19:58:41.185 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.185 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.185 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:41.185 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.027426+0000", 2023-09-23T19:58:41.185 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0029266560000000001 2023-09-23T19:58:41.185 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.185 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.185 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:41.185 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.316574+0000", 2023-09-23T19:58:41.185 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.28914771900000003 2023-09-23T19:58:41.185 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.185 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.186 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:58:41.186 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.316627+0000", 2023-09-23T19:58:41.186 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.3372999999999999e-05 2023-09-23T19:58:41.186 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.186 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.186 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:58:41.186 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.316659+0000", 2023-09-23T19:58:41.186 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.1739999999999998e-05 2023-09-23T19:58:41.186 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:58:41.186 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:58:41.186 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:58:41.186 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.186 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.187 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46640.0:57321 13.2 13:4a5f05a5:::benchmark_data_smithi016_205181_object57320:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e238)", 2023-09-23T19:58:41.187 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:52:28.024912+0000", 2023-09-23T19:58:41.187 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 373.14819757599997, 2023-09-23T19:58:41.187 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.30132013699999999, 2023-09-23T19:58:41.187 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:58:41.187 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:41.187 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:58:41.187 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46640", 2023-09-23T19:58:41.187 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2656794142", 2023-09-23T19:58:41.187 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 57321 2023-09-23T19:58:41.187 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.187 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:58:41.188 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.188 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:58:41.188 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.024912+0000", 2023-09-23T19:58:41.188 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:58:41.188 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.188 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.188 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:58:41.188 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.024912+0000", 2023-09-23T19:58:41.188 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:58:41.188 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.188 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.188 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:58:41.189 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.024911+0000", 2023-09-23T19:58:41.189 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-09-23T19:58:41.189 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.189 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.189 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:58:41.189 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.024940+0000", 2023-09-23T19:58:41.189 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.9189000000000001e-05 2023-09-23T19:58:41.189 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.189 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.189 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:58:41.189 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.024958+0000", 2023-09-23T19:58:41.189 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.8300000000000001e-05 2023-09-23T19:58:41.190 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.190 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.190 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:58:41.190 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.024969+0000", 2023-09-23T19:58:41.190 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0509999999999999e-05 2023-09-23T19:58:41.190 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.190 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.190 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:58:41.190 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.025010+0000", 2023-09-23T19:58:41.190 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.1109999999999998e-05 2023-09-23T19:58:41.190 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.190 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.191 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:58:41.191 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.025133+0000", 2023-09-23T19:58:41.191 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00012301400000000001 2023-09-23T19:58:41.191 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.191 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.191 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 4,5", 2023-09-23T19:58:41.191 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.025300+0000", 2023-09-23T19:58:41.191 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00016767099999999999 2023-09-23T19:58:41.191 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.191 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.191 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:58:41.191 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.027571+0000", 2023-09-23T19:58:41.192 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0022707640000000002 2023-09-23T19:58:41.192 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.192 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.192 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:41.192 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.028104+0000", 2023-09-23T19:58:41.192 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00053319100000000003 2023-09-23T19:58:41.192 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.192 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.192 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:41.192 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.325639+0000", 2023-09-23T19:58:41.192 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.29753508699999998 2023-09-23T19:58:41.192 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.192 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.193 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:58:41.193 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.325819+0000", 2023-09-23T19:58:41.193 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000179595 2023-09-23T19:58:41.193 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.193 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.193 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:58:41.193 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.326232+0000", 2023-09-23T19:58:41.193 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00041343100000000002 2023-09-23T19:58:41.193 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:58:41.193 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:58:41.193 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:58:41.193 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.193 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.194 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46640.0:57334 13.2 13:4d4c7563:::benchmark_data_smithi016_205181_object57333:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e238)", 2023-09-23T19:58:41.194 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:52:28.034227+0000", 2023-09-23T19:58:41.194 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 373.13888307600001, 2023-09-23T19:58:41.194 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.29222036699999998, 2023-09-23T19:58:41.194 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:58:41.194 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:41.194 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:58:41.194 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46640", 2023-09-23T19:58:41.194 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2656794142", 2023-09-23T19:58:41.194 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 57334 2023-09-23T19:58:41.194 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.194 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:58:41.195 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.195 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:58:41.195 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.034227+0000", 2023-09-23T19:58:41.195 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:58:41.195 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.195 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.195 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:58:41.195 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.034227+0000", 2023-09-23T19:58:41.195 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:58:41.195 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.195 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.195 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:58:41.196 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.034225+0000", 2023-09-23T19:58:41.196 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-09-23T19:58:41.196 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.196 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.196 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:58:41.196 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.034261+0000", 2023-09-23T19:58:41.196 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.6074000000000001e-05 2023-09-23T19:58:41.196 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.196 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.196 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:58:41.196 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.034274+0000", 2023-09-23T19:58:41.197 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2821e-05 2023-09-23T19:58:41.197 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.197 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.197 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:58:41.197 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.034283+0000", 2023-09-23T19:58:41.197 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.2390000000000008e-06 2023-09-23T19:58:41.197 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.197 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.197 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:58:41.197 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.034336+0000", 2023-09-23T19:58:41.197 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.2986999999999999e-05 2023-09-23T19:58:41.197 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.198 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.198 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:58:41.198 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.034489+0000", 2023-09-23T19:58:41.198 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00015313300000000001 2023-09-23T19:58:41.198 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.198 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.198 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 4,5", 2023-09-23T19:58:41.198 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.034665+0000", 2023-09-23T19:58:41.198 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000175711 2023-09-23T19:58:41.198 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.198 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.198 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:58:41.198 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.036963+0000", 2023-09-23T19:58:41.199 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0022978920000000002 2023-09-23T19:58:41.199 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.199 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.199 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:41.199 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.037222+0000", 2023-09-23T19:58:41.199 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00025987900000000002 2023-09-23T19:58:41.199 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.199 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.199 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:41.199 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.326308+0000", 2023-09-23T19:58:41.199 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.28908590899999997 2023-09-23T19:58:41.199 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.200 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.200 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:58:41.200 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.326389+0000", 2023-09-23T19:58:41.200 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.0951999999999993e-05 2023-09-23T19:58:41.200 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.200 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.200 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:58:41.200 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.326447+0000", 2023-09-23T19:58:41.200 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.7831000000000002e-05 2023-09-23T19:58:41.200 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:58:41.200 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:58:41.200 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:58:41.200 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.201 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.201 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46640.0:111449 13.2 13:48621006:::benchmark_data_smithi016_205181_object4185:head [delete] snapc 0=[] ondisk+write+known_if_redirected e242)", 2023-09-23T19:58:41.201 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:53:28.571279+0000", 2023-09-23T19:58:41.201 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 312.60183118899999, 2023-09-23T19:58:41.201 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.101453116, 2023-09-23T19:58:41.201 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:58:41.201 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:41.201 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:58:41.201 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46640", 2023-09-23T19:58:41.201 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2656794142", 2023-09-23T19:58:41.201 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 111449 2023-09-23T19:58:41.201 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.201 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:58:41.202 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.202 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:58:41.202 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:53:28.571279+0000", 2023-09-23T19:58:41.202 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:58:41.202 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.202 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.202 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:58:41.202 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:53:28.571279+0000", 2023-09-23T19:58:41.202 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:58:41.202 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.202 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.202 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:58:41.203 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:53:28.571277+0000", 2023-09-23T19:58:41.203 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-09-23T19:58:41.203 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.203 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.203 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:58:41.203 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:53:28.571285+0000", 2023-09-23T19:58:41.203 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.6330000000000002e-06 2023-09-23T19:58:41.203 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.203 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.203 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:58:41.203 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:53:28.571302+0000", 2023-09-23T19:58:41.203 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.6863000000000001e-05 2023-09-23T19:58:41.204 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.204 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.204 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:58:41.204 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:53:28.571311+0000", 2023-09-23T19:58:41.204 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.0049999999999993e-06 2023-09-23T19:58:41.204 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.204 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.204 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:58:41.204 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:53:28.571367+0000", 2023-09-23T19:58:41.204 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.5844999999999998e-05 2023-09-23T19:58:41.204 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.204 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.205 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:58:41.205 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:53:28.571536+0000", 2023-09-23T19:58:41.205 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000169016 2023-09-23T19:58:41.205 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.205 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.205 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 4,5", 2023-09-23T19:58:41.205 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:53:28.571675+0000", 2023-09-23T19:58:41.205 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00013946499999999999 2023-09-23T19:58:41.205 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.205 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.205 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:41.205 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:53:28.578525+0000", 2023-09-23T19:58:41.206 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.006850002 2023-09-23T19:58:41.206 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.206 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.206 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:58:41.206 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:53:28.580079+0000", 2023-09-23T19:58:41.206 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0015533039999999999 2023-09-23T19:58:41.206 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.206 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.206 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:41.206 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:53:28.672498+0000", 2023-09-23T19:58:41.206 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.092418983999999996 2023-09-23T19:58:41.206 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.207 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.207 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:58:41.207 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:53:28.672629+0000", 2023-09-23T19:58:41.207 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00013154200000000001 2023-09-23T19:58:41.207 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.207 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.207 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:58:41.207 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:53:28.672732+0000", 2023-09-23T19:58:41.207 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00010257 2023-09-23T19:58:41.207 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:58:41.207 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:58:41.207 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:58:41.208 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.208 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.208 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46640.0:111456 13.2 13:43f671e5:::benchmark_data_smithi016_205181_object4192:head [delete] snapc 0=[] ondisk+write+known_if_redirected e242)", 2023-09-23T19:58:41.208 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:53:28.580813+0000", 2023-09-23T19:58:41.208 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 312.592296981, 2023-09-23T19:58:41.208 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.092176757999999998, 2023-09-23T19:58:41.208 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:58:41.208 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:41.208 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:58:41.208 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46640", 2023-09-23T19:58:41.208 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2656794142", 2023-09-23T19:58:41.208 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 111456 2023-09-23T19:58:41.209 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.209 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:58:41.209 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.209 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:58:41.209 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:53:28.580813+0000", 2023-09-23T19:58:41.209 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:58:41.209 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.209 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.209 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:58:41.209 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:53:28.580813+0000", 2023-09-23T19:58:41.209 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:58:41.209 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.210 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.210 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:58:41.210 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:53:28.580811+0000", 2023-09-23T19:58:41.210 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999976 2023-09-23T19:58:41.210 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.210 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.210 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:58:41.210 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:53:28.580821+0000", 2023-09-23T19:58:41.210 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0155e-05 2023-09-23T19:58:41.210 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.210 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.210 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:58:41.211 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:53:28.580841+0000", 2023-09-23T19:58:41.211 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.0157999999999999e-05 2023-09-23T19:58:41.211 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.211 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.211 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:58:41.211 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:53:28.580854+0000", 2023-09-23T19:58:41.211 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2882e-05 2023-09-23T19:58:41.211 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.211 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.211 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:58:41.211 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:53:28.580912+0000", 2023-09-23T19:58:41.211 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.7649000000000003e-05 2023-09-23T19:58:41.212 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.212 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.212 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:58:41.212 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:53:28.581033+0000", 2023-09-23T19:58:41.212 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000121546 2023-09-23T19:58:41.212 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.212 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.212 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 4,5", 2023-09-23T19:58:41.212 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:53:28.581154+0000", 2023-09-23T19:58:41.212 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000121 2023-09-23T19:58:41.212 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.212 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.213 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:58:41.213 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:53:28.584816+0000", 2023-09-23T19:58:41.213 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0036619449999999998 2023-09-23T19:58:41.213 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.213 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.213 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:41.213 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:53:28.592662+0000", 2023-09-23T19:58:41.213 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0078463739999999997 2023-09-23T19:58:41.213 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.213 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.213 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:41.213 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:53:28.672822+0000", 2023-09-23T19:58:41.214 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.080159341999999995 2023-09-23T19:58:41.214 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.214 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.214 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:58:41.214 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:53:28.672911+0000", 2023-09-23T19:58:41.214 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.8778999999999998e-05 2023-09-23T19:58:41.214 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.214 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.214 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:58:41.214 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:53:28.672990+0000", 2023-09-23T19:58:41.214 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.9122999999999994e-05 2023-09-23T19:58:41.215 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:58:41.215 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:58:41.215 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:58:41.215 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.215 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.215 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46640.0:111461 13.2 13:4dd40d74:::benchmark_data_smithi016_205181_object4197:head [delete] snapc 0=[] ondisk+write+known_if_redirected e242)", 2023-09-23T19:58:41.215 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:53:28.587949+0000", 2023-09-23T19:58:41.215 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 312.58516054299997, 2023-09-23T19:58:41.215 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.085314662999999999, 2023-09-23T19:58:41.215 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:58:41.215 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:41.215 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:58:41.216 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46640", 2023-09-23T19:58:41.216 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2656794142", 2023-09-23T19:58:41.216 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 111461 2023-09-23T19:58:41.216 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.216 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:58:41.216 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.216 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:58:41.216 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:53:28.587949+0000", 2023-09-23T19:58:41.216 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:58:41.216 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.216 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.216 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:58:41.217 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:53:28.587949+0000", 2023-09-23T19:58:41.217 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:58:41.217 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.217 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.217 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:58:41.217 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:53:28.587948+0000", 2023-09-23T19:58:41.217 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-09-23T19:58:41.217 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.217 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.217 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:58:41.217 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:53:28.587955+0000", 2023-09-23T19:58:41.217 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.1180000000000003e-06 2023-09-23T19:58:41.217 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.218 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.218 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:58:41.218 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:53:28.587969+0000", 2023-09-23T19:58:41.218 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.4080000000000001e-05 2023-09-23T19:58:41.218 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.218 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.218 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:58:41.218 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:53:28.587977+0000", 2023-09-23T19:58:41.218 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.8790000000000003e-06 2023-09-23T19:58:41.218 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.218 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.218 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:58:41.219 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:53:28.588021+0000", 2023-09-23T19:58:41.219 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.4487000000000003e-05 2023-09-23T19:58:41.219 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.219 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.219 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:58:41.219 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:53:28.588152+0000", 2023-09-23T19:58:41.219 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00013043399999999999 2023-09-23T19:58:41.219 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.219 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.219 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 4,5", 2023-09-23T19:58:41.219 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:53:28.588275+0000", 2023-09-23T19:58:41.219 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000122719 2023-09-23T19:58:41.220 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.220 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.220 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:58:41.220 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:53:28.591825+0000", 2023-09-23T19:58:41.220 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0035499770000000002 2023-09-23T19:58:41.220 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.220 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.220 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:41.220 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:53:28.596485+0000", 2023-09-23T19:58:41.220 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.004660839 2023-09-23T19:58:41.220 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.220 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.221 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:41.221 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:53:28.673076+0000", 2023-09-23T19:58:41.221 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.076590470999999993 2023-09-23T19:58:41.221 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.221 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.221 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:58:41.221 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:53:28.673187+0000", 2023-09-23T19:58:41.221 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00011121899999999999 2023-09-23T19:58:41.221 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.221 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.221 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:58:41.221 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:53:28.673264+0000", 2023-09-23T19:58:41.221 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.6836999999999996e-05 2023-09-23T19:58:41.222 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:58:41.222 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:58:41.222 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:58:41.222 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.222 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.222 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46640.0:213227 13.2 13:4fb90b00:::benchmark_data_smithi016_205181_object105963:head [delete] snapc 0=[] ondisk+write+known_if_redirected e250)", 2023-09-23T19:58:41.222 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:55:57.644746+0000", 2023-09-23T19:58:41.222 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 163.52836400000001, 2023-09-23T19:58:41.222 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.087058025999999997, 2023-09-23T19:58:41.222 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:58:41.222 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:41.223 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:58:41.223 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46640", 2023-09-23T19:58:41.223 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2656794142", 2023-09-23T19:58:41.223 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 213227 2023-09-23T19:58:41.223 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.223 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:58:41.223 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.223 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:58:41.223 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.644746+0000", 2023-09-23T19:58:41.223 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:58:41.223 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.223 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.223 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:58:41.224 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.644746+0000", 2023-09-23T19:58:41.224 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:58:41.224 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.224 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.224 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:58:41.224 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.644745+0000", 2023-09-23T19:58:41.224 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:58:41.224 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.224 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.224 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:58:41.224 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.644751+0000", 2023-09-23T19:58:41.224 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.0179999999999998e-06 2023-09-23T19:58:41.225 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.225 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.225 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:58:41.225 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.644767+0000", 2023-09-23T19:58:41.225 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.5872e-05 2023-09-23T19:58:41.225 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.225 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.225 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:58:41.225 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.644775+0000", 2023-09-23T19:58:41.225 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.0509999999999997e-06 2023-09-23T19:58:41.225 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.225 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.225 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:58:41.226 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.653708+0000", 2023-09-23T19:58:41.226 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0089330680000000006 2023-09-23T19:58:41.226 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.226 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.226 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:58:41.226 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.653872+0000", 2023-09-23T19:58:41.226 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00016362399999999999 2023-09-23T19:58:41.226 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.226 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.226 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 4,5", 2023-09-23T19:58:41.226 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.654024+0000", 2023-09-23T19:58:41.226 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00015267899999999999 2023-09-23T19:58:41.227 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.227 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.227 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:41.227 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.669314+0000", 2023-09-23T19:58:41.227 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.015289412 2023-09-23T19:58:41.227 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.227 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.227 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:58:41.227 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.669333+0000", 2023-09-23T19:58:41.227 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.9429e-05 2023-09-23T19:58:41.227 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.227 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.228 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:41.228 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.731600+0000", 2023-09-23T19:58:41.228 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.062266609000000001 2023-09-23T19:58:41.228 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.228 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.228 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:58:41.228 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.731740+0000", 2023-09-23T19:58:41.228 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000140195 2023-09-23T19:58:41.228 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.228 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.228 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:58:41.228 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.731804+0000", 2023-09-23T19:58:41.229 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.4027000000000002e-05 2023-09-23T19:58:41.229 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:58:41.229 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:58:41.229 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:58:41.229 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.229 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.229 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46640.0:213228 13.2 13:4f494cf0:::benchmark_data_smithi016_205181_object105964:head [delete] snapc 0=[] ondisk+write+known_if_redirected e250)", 2023-09-23T19:58:41.229 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:55:57.646248+0000", 2023-09-23T19:58:41.229 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 163.52686171400001, 2023-09-23T19:58:41.229 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.085739086000000006, 2023-09-23T19:58:41.229 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:58:41.229 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:41.230 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T19:58:41.230 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46640", 2023-09-23T19:58:41.230 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2656794142", 2023-09-23T19:58:41.230 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 213228 2023-09-23T19:58:41.230 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.230 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:58:41.230 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.230 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:58:41.230 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.646248+0000", 2023-09-23T19:58:41.230 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:58:41.230 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.230 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.231 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:58:41.231 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.646248+0000", 2023-09-23T19:58:41.231 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:58:41.231 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.231 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.231 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:58:41.231 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.646247+0000", 2023-09-23T19:58:41.231 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:58:41.231 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.231 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.231 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:58:41.231 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.646254+0000", 2023-09-23T19:58:41.232 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.4949999999999996e-06 2023-09-23T19:58:41.232 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.232 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.232 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:58:41.232 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.646344+0000", 2023-09-23T19:58:41.232 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.0526999999999996e-05 2023-09-23T19:58:41.232 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.232 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.232 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:58:41.232 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.646353+0000", 2023-09-23T19:58:41.232 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.1349999999999998e-06 2023-09-23T19:58:41.232 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.233 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.233 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:58:41.233 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.658892+0000", 2023-09-23T19:58:41.233 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.012538762 2023-09-23T19:58:41.233 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.233 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.233 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:58:41.233 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.659035+0000", 2023-09-23T19:58:41.233 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000142702 2023-09-23T19:58:41.233 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.233 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.233 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 4,5", 2023-09-23T19:58:41.234 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.659177+0000", 2023-09-23T19:58:41.234 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00014189500000000001 2023-09-23T19:58:41.234 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.234 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.234 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T19:58:41.234 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.669341+0000", 2023-09-23T19:58:41.234 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.010164812 2023-09-23T19:58:41.234 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.234 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.234 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:41.234 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.669423+0000", 2023-09-23T19:58:41.234 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.1679000000000002e-05 2023-09-23T19:58:41.234 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.235 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.235 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:41.235 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.731874+0000", 2023-09-23T19:58:41.235 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.062450486999999999 2023-09-23T19:58:41.235 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.235 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.235 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:58:41.235 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.731933+0000", 2023-09-23T19:58:41.235 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.9231000000000003e-05 2023-09-23T19:58:41.235 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.235 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.236 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:58:41.236 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.731987+0000", 2023-09-23T19:58:41.236 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.4435000000000003e-05 2023-09-23T19:58:41.236 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:58:41.236 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:58:41.236 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:58:41.236 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.236 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.236 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.46877.0:54545 14.9 e261/256 14:9dbb7562:::benchmark_data_smithi016_228083_object54544:head v 261'3414, mlcod=261'3412)", 2023-09-23T19:58:41.236 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:57:51.628347+0000", 2023-09-23T19:58:41.236 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 49.544762675999998, 2023-09-23T19:58:41.236 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.055619137999999999, 2023-09-23T19:58:41.236 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:58:41.237 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:41.237 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:58:41.237 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.237 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:58:41.237 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.628347+0000", 2023-09-23T19:58:41.237 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:58:41.237 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.237 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.237 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:58:41.237 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.628347+0000", 2023-09-23T19:58:41.237 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:58:41.238 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.238 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.238 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:58:41.238 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.628347+0000", 2023-09-23T19:58:41.238 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T19:58:41.238 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.238 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.238 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:58:41.238 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.628382+0000", 2023-09-23T19:58:41.238 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.5265000000000002e-05 2023-09-23T19:58:41.238 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.238 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.239 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:58:41.239 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.628389+0000", 2023-09-23T19:58:41.239 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.0330000000000003e-06 2023-09-23T19:58:41.239 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.239 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.239 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:58:41.239 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.628394+0000", 2023-09-23T19:58:41.239 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.096e-06 2023-09-23T19:58:41.239 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.239 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.239 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:58:41.239 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.628436+0000", 2023-09-23T19:58:41.239 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.2317999999999997e-05 2023-09-23T19:58:41.240 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.240 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.240 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:58:41.240 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.628463+0000", 2023-09-23T19:58:41.240 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.6991e-05 2023-09-23T19:58:41.240 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.240 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.240 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:58:41.240 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.636285+0000", 2023-09-23T19:58:41.240 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0078218669999999997 2023-09-23T19:58:41.240 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.240 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.241 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:58:41.241 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.683966+0000", 2023-09-23T19:58:41.241 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.047680955999999997 2023-09-23T19:58:41.241 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:58:41.241 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:58:41.241 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:58:41.241 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.241 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.241 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.46877.0:54549 14.3 e261/256 14:cf0ba4e8:::benchmark_data_smithi016_228083_object54548:head v 261'3351, mlcod=261'3348)", 2023-09-23T19:58:41.241 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:57:51.630003+0000", 2023-09-23T19:58:41.241 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 49.543107317999997, 2023-09-23T19:58:41.241 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.053855074000000003, 2023-09-23T19:58:41.241 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:58:41.242 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:41.242 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:58:41.242 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.242 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:58:41.242 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.630003+0000", 2023-09-23T19:58:41.242 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:58:41.242 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.242 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.242 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:58:41.242 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.630003+0000", 2023-09-23T19:58:41.242 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:58:41.242 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.243 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.243 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:58:41.243 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.630002+0000", 2023-09-23T19:58:41.243 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T19:58:41.243 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.243 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.243 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:58:41.243 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.630021+0000", 2023-09-23T19:58:41.243 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.9360000000000001e-05 2023-09-23T19:58:41.243 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.243 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.243 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:58:41.244 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.630029+0000", 2023-09-23T19:58:41.244 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.8390000000000007e-06 2023-09-23T19:58:41.244 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.244 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.244 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:58:41.244 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.630036+0000", 2023-09-23T19:58:41.244 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.9999999999999999e-06 2023-09-23T19:58:41.244 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.244 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.244 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:58:41.244 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.630468+0000", 2023-09-23T19:58:41.244 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00043158299999999998 2023-09-23T19:58:41.245 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.245 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.245 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:58:41.245 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.630492+0000", 2023-09-23T19:58:41.245 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.3777999999999998e-05 2023-09-23T19:58:41.245 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.245 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.245 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:58:41.245 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.634721+0000", 2023-09-23T19:58:41.245 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0042295859999999996 2023-09-23T19:58:41.245 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.245 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.246 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:58:41.246 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.683858+0000", 2023-09-23T19:58:41.246 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.049136396999999998 2023-09-23T19:58:41.246 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:58:41.246 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:58:41.246 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:58:41.246 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.246 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.246 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.46877.0:54550 14.1 e261/256 14:89020cc7:::benchmark_data_smithi016_228083_object54549:head v 261'3377, mlcod=261'3375)", 2023-09-23T19:58:41.246 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:57:51.630043+0000", 2023-09-23T19:58:41.246 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 49.543067168999997, 2023-09-23T19:58:41.247 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.058685831000000001, 2023-09-23T19:58:41.247 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:58:41.247 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:41.247 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:58:41.247 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.247 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:58:41.247 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.630043+0000", 2023-09-23T19:58:41.247 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:58:41.247 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.247 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.247 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:58:41.247 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.630043+0000", 2023-09-23T19:58:41.247 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:58:41.248 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.248 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.248 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:58:41.248 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.630042+0000", 2023-09-23T19:58:41.248 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T19:58:41.248 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.248 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.248 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:58:41.248 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.630065+0000", 2023-09-23T19:58:41.248 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.2560000000000001e-05 2023-09-23T19:58:41.248 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.248 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.248 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:58:41.249 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.630070+0000", 2023-09-23T19:58:41.249 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.0300000000000001e-06 2023-09-23T19:58:41.249 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.249 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.249 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:58:41.249 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.630073+0000", 2023-09-23T19:58:41.249 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.2780000000000002e-06 2023-09-23T19:58:41.249 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.249 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.249 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:58:41.249 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.630238+0000", 2023-09-23T19:58:41.249 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000164277 2023-09-23T19:58:41.250 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.250 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.250 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:58:41.250 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.630256+0000", 2023-09-23T19:58:41.250 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.7994000000000001e-05 2023-09-23T19:58:41.250 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.250 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.250 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:58:41.250 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.688644+0000", 2023-09-23T19:58:41.250 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.058388566000000003 2023-09-23T19:58:41.250 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.250 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.251 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:58:41.251 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.688729+0000", 2023-09-23T19:58:41.251 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.4421999999999999e-05 2023-09-23T19:58:41.251 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:58:41.251 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:58:41.251 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:58:41.251 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.251 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.251 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.46877.0:54546 14.a e261/256 14:5919b506:::benchmark_data_smithi016_228083_object54545:head v 261'3387, mlcod=261'3384)", 2023-09-23T19:58:41.251 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:57:51.630081+0000", 2023-09-23T19:58:41.251 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 49.543029343999997, 2023-09-23T19:58:41.251 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.053905671000000002, 2023-09-23T19:58:41.252 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:58:41.252 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:41.252 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:58:41.252 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.252 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:58:41.252 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.630081+0000", 2023-09-23T19:58:41.252 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:58:41.252 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.252 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.252 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:58:41.252 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.630081+0000", 2023-09-23T19:58:41.252 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:58:41.253 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.253 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.253 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:58:41.253 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.630080+0000", 2023-09-23T19:58:41.253 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967296 2023-09-23T19:58:41.253 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.253 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.253 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:58:41.253 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.630101+0000", 2023-09-23T19:58:41.253 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.0244000000000001e-05 2023-09-23T19:58:41.253 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.253 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.253 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:58:41.254 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.630104+0000", 2023-09-23T19:58:41.254 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.6940000000000001e-06 2023-09-23T19:58:41.254 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.254 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.254 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:58:41.254 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.630107+0000", 2023-09-23T19:58:41.254 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.6989999999999999e-06 2023-09-23T19:58:41.254 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.254 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.254 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:58:41.254 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.630147+0000", 2023-09-23T19:58:41.254 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.0259000000000002e-05 2023-09-23T19:58:41.254 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.255 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.255 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:58:41.255 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.630168+0000", 2023-09-23T19:58:41.255 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.0446999999999999e-05 2023-09-23T19:58:41.255 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.255 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.255 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:58:41.255 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.683892+0000", 2023-09-23T19:58:41.255 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.053724534999999997 2023-09-23T19:58:41.255 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.255 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.255 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:58:41.256 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.683986+0000", 2023-09-23T19:58:41.256 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.3982999999999998e-05 2023-09-23T19:58:41.256 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:58:41.256 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:58:41.256 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:58:41.256 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.256 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.256 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.46877.0:54548 14.9 e261/256 14:9bec6ce9:::benchmark_data_smithi016_228083_object54547:head v 261'3415, mlcod=261'3412)", 2023-09-23T19:58:41.256 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:57:51.630125+0000", 2023-09-23T19:58:41.256 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 49.542984527000002, 2023-09-23T19:58:41.256 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.062403675999999998, 2023-09-23T19:58:41.257 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:58:41.257 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:41.257 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:58:41.257 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.257 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:58:41.257 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.630125+0000", 2023-09-23T19:58:41.257 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:58:41.257 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.257 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.257 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:58:41.257 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.630125+0000", 2023-09-23T19:58:41.257 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:58:41.258 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.258 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.258 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:58:41.258 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.630125+0000", 2023-09-23T19:58:41.258 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T19:58:41.258 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.258 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.258 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:58:41.258 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.630156+0000", 2023-09-23T19:58:41.258 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.1436999999999997e-05 2023-09-23T19:58:41.258 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.258 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.259 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:58:41.259 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.630164+0000", 2023-09-23T19:58:41.259 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.345e-06 2023-09-23T19:58:41.259 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.259 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.259 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:58:41.259 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.630169+0000", 2023-09-23T19:58:41.259 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.7620000000000003e-06 2023-09-23T19:58:41.259 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.259 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.259 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:58:41.259 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.630730+0000", 2023-09-23T19:58:41.260 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00056141100000000005 2023-09-23T19:58:41.260 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.260 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.260 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:58:41.260 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.630791+0000", 2023-09-23T19:58:41.260 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.1495999999999995e-05 2023-09-23T19:58:41.260 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.260 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.260 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:58:41.260 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.692441+0000", 2023-09-23T19:58:41.260 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.061649029000000001 2023-09-23T19:58:41.260 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.261 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.261 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:58:41.261 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.692529+0000", 2023-09-23T19:58:41.261 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.8478000000000002e-05 2023-09-23T19:58:41.261 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:58:41.261 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:58:41.261 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:58:41.261 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.261 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.261 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.46877.0:54553 14.4 e261/256 14:221fb0e3:::benchmark_data_smithi016_228083_object54552:head v 261'3470, mlcod=261'3468)", 2023-09-23T19:58:41.261 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:57:51.631298+0000", 2023-09-23T19:58:41.261 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 49.541812012000001, 2023-09-23T19:58:41.261 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.056592578999999997, 2023-09-23T19:58:41.262 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:58:41.262 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:41.262 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:58:41.262 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.262 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:58:41.262 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.631298+0000", 2023-09-23T19:58:41.262 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:58:41.262 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.262 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.262 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:58:41.262 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.631298+0000", 2023-09-23T19:58:41.262 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:58:41.263 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.263 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.263 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:58:41.263 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.631297+0000", 2023-09-23T19:58:41.263 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:58:41.263 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.263 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.263 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:58:41.263 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.631346+0000", 2023-09-23T19:58:41.263 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.9193999999999997e-05 2023-09-23T19:58:41.263 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.263 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.263 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:58:41.264 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.631359+0000", 2023-09-23T19:58:41.264 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2424e-05 2023-09-23T19:58:41.264 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.264 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.264 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:58:41.264 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.631365+0000", 2023-09-23T19:58:41.264 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.2999999999999998e-06 2023-09-23T19:58:41.264 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.264 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.264 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:58:41.264 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.631996+0000", 2023-09-23T19:58:41.264 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00063104800000000005 2023-09-23T19:58:41.265 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.265 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.265 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:58:41.265 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.632015+0000", 2023-09-23T19:58:41.265 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.8958e-05 2023-09-23T19:58:41.265 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.265 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.265 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:58:41.265 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.687698+0000", 2023-09-23T19:58:41.265 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.055683379999999998 2023-09-23T19:58:41.265 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.265 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.265 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:58:41.266 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.687890+0000", 2023-09-23T19:58:41.266 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000192036 2023-09-23T19:58:41.266 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:58:41.266 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:58:41.266 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:58:41.266 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.266 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.266 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.46877.0:54552 14.1 e261/256 14:8ebcf13b:::benchmark_data_smithi016_228083_object54551:head v 261'3378, mlcod=261'3375)", 2023-09-23T19:58:41.266 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:57:51.632083+0000", 2023-09-23T19:58:41.266 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 49.541026985000002, 2023-09-23T19:58:41.266 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.06050962, 2023-09-23T19:58:41.267 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:58:41.267 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:41.267 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:58:41.267 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.267 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:58:41.267 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.632083+0000", 2023-09-23T19:58:41.267 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:58:41.267 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.267 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.267 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:58:41.267 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.632083+0000", 2023-09-23T19:58:41.268 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:58:41.268 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.268 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.268 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:58:41.268 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.632082+0000", 2023-09-23T19:58:41.268 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T19:58:41.268 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.268 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.268 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:58:41.268 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.632120+0000", 2023-09-23T19:58:41.268 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.7450000000000002e-05 2023-09-23T19:58:41.268 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.269 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.269 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:58:41.269 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.632130+0000", 2023-09-23T19:58:41.269 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0165e-05 2023-09-23T19:58:41.269 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.269 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.269 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:58:41.269 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.632137+0000", 2023-09-23T19:58:41.269 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.3529999999999997e-06 2023-09-23T19:58:41.269 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.269 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.269 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:58:41.270 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.632702+0000", 2023-09-23T19:58:41.270 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00056454200000000002 2023-09-23T19:58:41.270 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.270 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.270 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:58:41.270 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.683853+0000", 2023-09-23T19:58:41.270 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.051151611999999999 2023-09-23T19:58:41.270 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.270 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.270 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:58:41.270 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.692533+0000", 2023-09-23T19:58:41.270 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0086799700000000004 2023-09-23T19:58:41.270 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.271 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.271 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:58:41.271 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.692592+0000", 2023-09-23T19:58:41.271 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.9290999999999999e-05 2023-09-23T19:58:41.271 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:58:41.271 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:58:41.271 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:58:41.271 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.271 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.271 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.46877.0:54555 14.4 e261/256 14:21c88248:::benchmark_data_smithi016_228083_object54554:head v 261'3471, mlcod=261'3468)", 2023-09-23T19:58:41.271 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:57:51.634447+0000", 2023-09-23T19:58:41.271 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 49.538663047, 2023-09-23T19:58:41.271 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.053531330000000002, 2023-09-23T19:58:41.272 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:58:41.272 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:41.272 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:58:41.272 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.272 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:58:41.272 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.634447+0000", 2023-09-23T19:58:41.272 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:58:41.272 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.272 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.272 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:58:41.272 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.634447+0000", 2023-09-23T19:58:41.272 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:58:41.272 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.272 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.273 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:58:41.273 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.634439+0000", 2023-09-23T19:58:41.273 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999924 2023-09-23T19:58:41.273 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.273 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.273 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:58:41.273 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.634483+0000", 2023-09-23T19:58:41.273 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.4169999999999999e-05 2023-09-23T19:58:41.273 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.273 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.273 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:58:41.273 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.683815+0000", 2023-09-23T19:58:41.274 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.049331549000000002 2023-09-23T19:58:41.274 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.274 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.274 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:58:41.274 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.683844+0000", 2023-09-23T19:58:41.274 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.932e-05 2023-09-23T19:58:41.274 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.274 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.274 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:58:41.274 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.683893+0000", 2023-09-23T19:58:41.274 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.8603000000000002e-05 2023-09-23T19:58:41.275 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.275 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.275 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:58:41.275 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.683915+0000", 2023-09-23T19:58:41.275 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.2118999999999999e-05 2023-09-23T19:58:41.275 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.275 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.275 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:58:41.275 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.687896+0000", 2023-09-23T19:58:41.275 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0039811869999999997 2023-09-23T19:58:41.275 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.276 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.276 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:58:41.276 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.687978+0000", 2023-09-23T19:58:41.276 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.2138999999999994e-05 2023-09-23T19:58:41.276 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:58:41.276 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:58:41.276 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:58:41.276 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.276 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.276 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.46877.0:83933 14.9 e262/256 14:90a2e298:::benchmark_data_smithi016_228083_object83932:head v 262'5239, mlcod=262'5236)", 2023-09-23T19:58:41.276 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:58:23.288509+0000", 2023-09-23T19:58:41.276 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 17.884600655, 2023-09-23T19:58:41.277 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.061785059000000003, 2023-09-23T19:58:41.277 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:58:41.277 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:41.277 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:58:41.277 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.277 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:58:41.277 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:23.288509+0000", 2023-09-23T19:58:41.277 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:58:41.277 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.277 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.277 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:58:41.278 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:23.288509+0000", 2023-09-23T19:58:41.278 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:58:41.278 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.278 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.278 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:58:41.278 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:23.288509+0000", 2023-09-23T19:58:41.278 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T19:58:41.278 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.278 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.278 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:58:41.278 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:23.288543+0000", 2023-09-23T19:58:41.278 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.4434000000000003e-05 2023-09-23T19:58:41.279 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.279 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.279 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:58:41.279 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:23.288555+0000", 2023-09-23T19:58:41.279 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1812e-05 2023-09-23T19:58:41.279 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.279 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.279 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:58:41.279 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:23.288560+0000", 2023-09-23T19:58:41.279 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.1959999999999999e-06 2023-09-23T19:58:41.279 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.279 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.280 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:58:41.280 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:23.290646+0000", 2023-09-23T19:58:41.280 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0020861009999999999 2023-09-23T19:58:41.280 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.280 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.280 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:58:41.280 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:23.290707+0000", 2023-09-23T19:58:41.280 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.1400000000000002e-05 2023-09-23T19:58:41.280 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.280 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.280 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:58:41.280 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:23.350260+0000", 2023-09-23T19:58:41.281 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.059552123999999998 2023-09-23T19:58:41.281 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.281 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.281 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:58:41.281 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:23.350294+0000", 2023-09-23T19:58:41.281 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.4634000000000001e-05 2023-09-23T19:58:41.281 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:58:41.281 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:58:41.281 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:58:41.281 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.281 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.281 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.46877.0:83941 14.9 e262/256 14:9008a354:::benchmark_data_smithi016_228083_object83940:head v 262'5240, mlcod=262'5237)", 2023-09-23T19:58:41.281 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:58:23.296262+0000", 2023-09-23T19:58:41.282 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 17.876847946000002, 2023-09-23T19:58:41.282 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.054093337999999998, 2023-09-23T19:58:41.282 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:58:41.282 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:41.282 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:58:41.282 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.282 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:58:41.282 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:23.296262+0000", 2023-09-23T19:58:41.282 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:58:41.282 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.282 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.283 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:58:41.283 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:23.296262+0000", 2023-09-23T19:58:41.283 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:58:41.283 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.283 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.283 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:58:41.283 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:23.296261+0000", 2023-09-23T19:58:41.283 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T19:58:41.283 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.283 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.283 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:58:41.283 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:23.296300+0000", 2023-09-23T19:58:41.284 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.8794000000000002e-05 2023-09-23T19:58:41.284 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.284 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.284 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:58:41.284 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:23.296311+0000", 2023-09-23T19:58:41.284 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1426999999999999e-05 2023-09-23T19:58:41.284 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.284 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.284 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:58:41.284 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:23.296319+0000", 2023-09-23T19:58:41.284 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.2250000000000003e-06 2023-09-23T19:58:41.284 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.285 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.285 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:58:41.285 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:23.317740+0000", 2023-09-23T19:58:41.285 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.021421341 2023-09-23T19:58:41.285 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.285 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.285 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:58:41.285 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:23.320483+0000", 2023-09-23T19:58:41.285 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.002743012 2023-09-23T19:58:41.285 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.285 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.285 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:58:41.286 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:23.350333+0000", 2023-09-23T19:58:41.286 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.029850031999999999 2023-09-23T19:58:41.286 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.286 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.286 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:58:41.286 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:23.350355+0000", 2023-09-23T19:58:41.286 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.2215e-05 2023-09-23T19:58:41.286 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:58:41.286 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:58:41.286 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:58:41.286 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.287 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.287 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.46877.0:83944 14.1 e262/256 14:868d5ca1:::benchmark_data_smithi016_228083_object83943:head v 262'5176, mlcod=262'5174)", 2023-09-23T19:58:41.287 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:58:23.298727+0000", 2023-09-23T19:58:41.287 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 17.874382600000001, 2023-09-23T19:58:41.287 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.051600808999999997, 2023-09-23T19:58:41.287 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:58:41.287 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:41.287 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:58:41.287 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.287 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:58:41.287 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:23.298727+0000", 2023-09-23T19:58:41.288 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:58:41.288 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.288 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.288 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:58:41.288 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:23.298727+0000", 2023-09-23T19:58:41.288 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:58:41.288 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.288 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.288 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:58:41.288 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:23.298727+0000", 2023-09-23T19:58:41.288 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T19:58:41.288 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.289 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.289 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:58:41.289 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:23.298752+0000", 2023-09-23T19:58:41.289 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.5418e-05 2023-09-23T19:58:41.289 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.289 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.289 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:58:41.289 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:23.298770+0000", 2023-09-23T19:58:41.289 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.8059000000000001e-05 2023-09-23T19:58:41.289 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.289 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.289 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:58:41.290 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:23.298778+0000", 2023-09-23T19:58:41.290 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.9980000000000003e-06 2023-09-23T19:58:41.290 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.290 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.290 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:58:41.290 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:23.317810+0000", 2023-09-23T19:58:41.290 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.019032087 2023-09-23T19:58:41.290 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.290 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.290 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:58:41.290 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:23.317839+0000", 2023-09-23T19:58:41.290 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.8694999999999999e-05 2023-09-23T19:58:41.291 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.291 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.291 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:58:41.291 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:23.350298+0000", 2023-09-23T19:58:41.291 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.032459542000000001 2023-09-23T19:58:41.291 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.291 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.291 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:58:41.291 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:23.350328+0000", 2023-09-23T19:58:41.291 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.968e-05 2023-09-23T19:58:41.291 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:58:41.291 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:58:41.292 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:58:41.292 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.292 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.292 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.46877.0:83943 14.9 e262/256 14:9cde6cf4:::benchmark_data_smithi016_228083_object83942:head v 262'5241, mlcod=262'5237)", 2023-09-23T19:58:41.292 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:58:23.298798+0000", 2023-09-23T19:58:41.292 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 17.874312169, 2023-09-23T19:58:41.292 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.051601735000000003, 2023-09-23T19:58:41.292 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T19:58:41.292 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:41.292 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T19:58:41.292 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.292 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T19:58:41.293 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:23.298798+0000", 2023-09-23T19:58:41.293 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:58:41.293 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.293 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.293 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T19:58:41.293 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:23.298798+0000", 2023-09-23T19:58:41.293 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T19:58:41.293 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.293 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.293 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T19:58:41.293 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:23.298797+0000", 2023-09-23T19:58:41.293 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T19:58:41.294 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.294 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.294 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T19:58:41.294 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:23.298826+0000", 2023-09-23T19:58:41.294 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.9051e-05 2023-09-23T19:58:41.294 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.294 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.294 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T19:58:41.294 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:23.298834+0000", 2023-09-23T19:58:41.294 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.9140000000000008e-06 2023-09-23T19:58:41.294 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.295 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.295 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T19:58:41.295 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:23.298840+0000", 2023-09-23T19:58:41.295 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.5300000000000004e-06 2023-09-23T19:58:41.295 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.295 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.295 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T19:58:41.295 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:23.323795+0000", 2023-09-23T19:58:41.295 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.024955516000000001 2023-09-23T19:58:41.295 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.295 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.295 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T19:58:41.296 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:23.333625+0000", 2023-09-23T19:58:41.296 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0098299939999999999 2023-09-23T19:58:41.296 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.296 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.296 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T19:58:41.296 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:23.350381+0000", 2023-09-23T19:58:41.296 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.016756130000000001 2023-09-23T19:58:41.296 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T19:58:41.296 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T19:58:41.296 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T19:58:41.296 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:23.350399+0000", 2023-09-23T19:58:41.296 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.8042000000000001e-05 2023-09-23T19:58:41.297 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:58:41.297 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:58:41.297 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:58:41.297 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T19:58:41.297 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T19:58:41.297 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-09-23T19:58:41.297 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:41.325 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:41.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:41.328 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:41.428 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:41.462 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:41.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:41.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:41.569 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:41.592 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:41.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:41.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:41.695 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:41.723 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:41.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:41.729 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:41.830 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:41.862 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:41.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:41.869 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:41.892 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 87 16 97794 97778 70.2189 48.0625 0.0237207 0.0142345 2023-09-23T19:58:41.969 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:42.000 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:42.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:42.003 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:42.104 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:42.129 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:42.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:42.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:42.232 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:42.261 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:42.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:42.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:42.365 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:42.394 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:42.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:42.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:42.498 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:42.521 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:42.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:42.526 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:42.537 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:42 smithi092 bash[58855]: cluster 2023-09-23T19:58:40.590477+0000 mgr.y (mgr.25325) 1543 : cluster 0 pgmap v1570: 89 pgs: 89 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 53 MiB/s wr, 845 op/s 2023-09-23T19:58:42.626 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:42.652 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:42.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:42.656 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:42.757 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:42.768 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:42 smithi016 bash[77995]: cluster 2023-09-23T19:58:40.590477+0000 mgr.y (mgr.25325) 1543 : cluster 0 pgmap v1570: 89 pgs: 89 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 53 MiB/s wr, 845 op/s 2023-09-23T19:58:42.769 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:42 smithi016 bash[75004]: cluster 2023-09-23T19:58:40.590477+0000 mgr.y (mgr.25325) 1543 : cluster 0 pgmap v1570: 89 pgs: 89 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 53 MiB/s wr, 845 op/s 2023-09-23T19:58:42.797 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:42.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:42.803 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:42.892 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 88 16 98580 98564 69.9792 49.125 0.0264395 0.014283 2023-09-23T19:58:42.904 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:42.921 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:42.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:42.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:43.024 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:43.058 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:43.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:43.061 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:43.162 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:43.187 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:43.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:43.193 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:43.293 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:43.319 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:43.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:43.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:43.426 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:43.450 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:43.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:43.452 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:43.553 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:43.584 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:43.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:43.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:43.689 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:43.725 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:43.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:43.730 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:43.831 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:43.861 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:43.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:43.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:43.892 INFO:tasks.radosbench.radosbench.0.smithi016.stdout: 89 16 99249 99233 69.6627 41.8125 0.00453438 0.0143359 2023-09-23T19:58:43.965 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:43.991 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:43.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:43.997 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:44.098 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:44.125 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:44.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:44.129 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:44.230 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:44.266 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:44.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:44.268 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:44.369 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:44.403 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:44.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:44.405 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:44.506 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:44.515 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:44 smithi016 bash[75004]: cluster 2023-09-23T19:58:42.590930+0000 mgr.y (mgr.25325) 1544 : cluster 0 pgmap v1571: 89 pgs: 89 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 44 MiB/s wr, 703 op/s 2023-09-23T19:58:44.515 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:44 smithi016 bash[75004]: audit 2023-09-23T19:58:43.698916+0000 mon.a (mon.0) 1858 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:58:44.515 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:44 smithi016 bash[75004]: audit 2023-09-23T19:58:43.700082+0000 mon.a (mon.0) 1859 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:58:44.515 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:44 smithi016 bash[77995]: cluster 2023-09-23T19:58:42.590930+0000 mgr.y (mgr.25325) 1544 : cluster 0 pgmap v1571: 89 pgs: 89 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 44 MiB/s wr, 703 op/s 2023-09-23T19:58:44.515 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:44 smithi016 bash[77995]: audit 2023-09-23T19:58:43.698916+0000 mon.a (mon.0) 1858 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:58:44.515 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:44 smithi016 bash[77995]: audit 2023-09-23T19:58:43.700082+0000 mon.a (mon.0) 1859 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:58:44.550 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:44.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:44.553 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:44.654 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:44.689 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:44.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:44.698 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:44.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:44 smithi092 bash[58855]: cluster 2023-09-23T19:58:42.590930+0000 mgr.y (mgr.25325) 1544 : cluster 0 pgmap v1571: 89 pgs: 89 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 44 MiB/s wr, 703 op/s 2023-09-23T19:58:44.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:44 smithi092 bash[58855]: audit 2023-09-23T19:58:43.698916+0000 mon.a (mon.0) 1858 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:58:44.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:44 smithi092 bash[58855]: audit 2023-09-23T19:58:43.700082+0000 mon.a (mon.0) 1859 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:58:44.798 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:44.822 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:44.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:44.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:44.892 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Total time run: 90.0113 2023-09-23T19:58:44.893 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Total writes made: 99956 2023-09-23T19:58:44.893 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Write size: 65536 2023-09-23T19:58:44.893 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Object size: 65536 2023-09-23T19:58:44.893 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Bandwidth (MB/sec): 69.4051 2023-09-23T19:58:44.893 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Stddev Bandwidth: 28.0515 2023-09-23T19:58:44.893 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Max bandwidth (MB/sec): 167.812 2023-09-23T19:58:44.893 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Min bandwidth (MB/sec): 23.1875 2023-09-23T19:58:44.893 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Average IOPS: 1110 2023-09-23T19:58:44.893 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Stddev IOPS: 448.824 2023-09-23T19:58:44.893 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Max IOPS: 2685 2023-09-23T19:58:44.893 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Min IOPS: 371 2023-09-23T19:58:44.893 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Average Latency(s): 0.0144032 2023-09-23T19:58:44.894 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Stddev Latency(s): 0.0191388 2023-09-23T19:58:44.894 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Max latency(s): 1.05209 2023-09-23T19:58:44.894 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Min latency(s): 0.001237 2023-09-23T19:58:44.901 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T19:58:44.928 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:44.956 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:44.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:44.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:45.060 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:45.093 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:45.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:45.096 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:45.197 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:45.232 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:45.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:45.237 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:45.337 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:45.366 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:45.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:45.369 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:45.470 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:45.498 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:45.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:45.502 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:45.603 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:45.609 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:45 smithi092 bash[58855]: audit 2023-09-23T19:58:44.497113+0000 mon.a (mon.0) 1860 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:58:45.627 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:45.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:45.629 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:45.708 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:45 smithi016 bash[75004]: audit 2023-09-23T19:58:44.497113+0000 mon.a (mon.0) 1860 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:58:45.709 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:58:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:58:45] "GET /metrics HTTP/1.1" 200 36373 "" "Prometheus/2.33.4" 2023-09-23T19:58:45.709 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:45 smithi016 bash[77995]: audit 2023-09-23T19:58:44.497113+0000 mon.a (mon.0) 1860 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:58:45.731 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:45.758 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:45.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:45.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:45.861 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:45.887 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:45.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:45.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:45.990 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:46.020 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:58:46.021 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:46.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:46.025 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:46.062 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:58:46.103 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:58:46.125 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:46.145 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:58:46.156 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:46.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:46.158 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:46.187 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:58:46.231 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:58:46.260 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:46.262 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:58:46.286 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:46.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:46.290 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:46.294 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:58:46.300 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:58:46.300 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:58:46.300 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:58:46.300 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:58:46.300 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:58:46.300 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:58:46.300 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:58:46.300 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:58:46.391 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:46.415 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:46.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:46.422 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:46.523 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:46.549 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:46.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:46.556 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:46.657 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:46.669 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:46 smithi092 bash[58855]: cluster 2023-09-23T19:58:44.591550+0000 mgr.y (mgr.25325) 1545 : cluster 0 pgmap v1572: 89 pgs: 89 active+clean; 6.0 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 45 MiB/s wr, 716 op/s 2023-09-23T19:58:46.669 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:46 smithi092 bash[58855]: audit 2023-09-23T19:58:45.905395+0000 mon.c (mon.2) 234 : audit 0 from='client.? 172.21.15.16:0/2388932200' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:58:46.688 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:46.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:46.692 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:46.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:46 smithi016 bash[75004]: cluster 2023-09-23T19:58:44.591550+0000 mgr.y (mgr.25325) 1545 : cluster 0 pgmap v1572: 89 pgs: 89 active+clean; 6.0 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 45 MiB/s wr, 716 op/s 2023-09-23T19:58:46.749 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:46 smithi016 bash[75004]: audit 2023-09-23T19:58:45.905395+0000 mon.c (mon.2) 234 : audit 0 from='client.? 172.21.15.16:0/2388932200' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:58:46.749 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:46 smithi016 bash[77995]: cluster 2023-09-23T19:58:44.591550+0000 mgr.y (mgr.25325) 1545 : cluster 0 pgmap v1572: 89 pgs: 89 active+clean; 6.0 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 45 MiB/s wr, 716 op/s 2023-09-23T19:58:46.750 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:46 smithi016 bash[77995]: audit 2023-09-23T19:58:45.905395+0000 mon.c (mon.2) 234 : audit 0 from='client.? 172.21.15.16:0/2388932200' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:58:46.793 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:46.799 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T19:58:46.821 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:46.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:46.824 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:46.925 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:46.971 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:46.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:46.979 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:47.080 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:47.102 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:47.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:47.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:47.204 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:47.237 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:47.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:47.241 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:47.342 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:47.382 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:47.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:47.385 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:47.486 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:47.493 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:47 smithi092 bash[58855]: cluster 2023-09-23T19:58:46.592118+0000 mgr.y (mgr.25325) 1546 : cluster 0 pgmap v1573: 89 pgs: 89 active+clean; 6.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 49 MiB/s wr, 804 op/s 2023-09-23T19:58:47.514 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:47.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:47.516 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:47.617 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:47.643 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:47.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:47.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:47.748 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:47.757 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:47 smithi016 bash[75004]: cluster 2023-09-23T19:58:46.592118+0000 mgr.y (mgr.25325) 1546 : cluster 0 pgmap v1573: 89 pgs: 89 active+clean; 6.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 49 MiB/s wr, 804 op/s 2023-09-23T19:58:47.757 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:47 smithi016 bash[77995]: cluster 2023-09-23T19:58:46.592118+0000 mgr.y (mgr.25325) 1546 : cluster 0 pgmap v1573: 89 pgs: 89 active+clean; 6.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 49 MiB/s wr, 804 op/s 2023-09-23T19:58:47.779 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:47.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:47.781 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:47.882 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:47.914 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:47.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:47.916 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:48.019 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:48.051 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:48.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:48.055 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:48.156 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:48.192 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:48.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:48.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:48.295 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:48.325 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:48.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:48.328 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:48.429 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:48.456 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:48.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:48.458 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:48.558 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:48.592 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:48.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:48.595 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:48.696 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:48.722 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:48.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:48.728 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:48.828 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:48.856 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:48.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:48.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:48.963 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:48.989 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:48.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:48.992 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:49.092 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:49.120 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:49.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:49.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:49.225 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:49.256 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:49.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:49.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:49.360 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:49.385 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:49.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:49.388 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:49.488 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:49.511 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:49.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:49.514 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:49.615 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:49.642 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:49.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:49.648 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:49.749 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:49.757 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:49 smithi016 bash[75004]: cluster 2023-09-23T19:58:48.592667+0000 mgr.y (mgr.25325) 1547 : cluster 0 pgmap v1574: 89 pgs: 89 active+clean; 6.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 40 MiB/s wr, 719 op/s 2023-09-23T19:58:49.758 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:49 smithi016 bash[77995]: cluster 2023-09-23T19:58:48.592667+0000 mgr.y (mgr.25325) 1547 : cluster 0 pgmap v1574: 89 pgs: 89 active+clean; 6.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 40 MiB/s wr, 719 op/s 2023-09-23T19:58:49.774 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:49.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:49.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:49.877 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:49.886 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:49 smithi092 bash[58855]: cluster 2023-09-23T19:58:48.592667+0000 mgr.y (mgr.25325) 1547 : cluster 0 pgmap v1574: 89 pgs: 89 active+clean; 6.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 40 MiB/s wr, 719 op/s 2023-09-23T19:58:49.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:49.917 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:49.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:50.017 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:50.068 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:50.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:50.073 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:50.174 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:50.205 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:50.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:50.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:50.312 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:50.336 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:50.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:50.342 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:50.443 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:50.469 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:50.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:50.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:50.573 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:50.609 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:50.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:50.616 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:50.717 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:50.749 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:50.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:50.753 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:50.854 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:50.876 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:50.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:50.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:50.979 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:51.007 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:51.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:51.013 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:51.114 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:51.142 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:51.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:51.146 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:51.247 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:51.278 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:51.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:51.287 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:51.388 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:51.424 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:51.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:51.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:51.475 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:58:51.532 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:51.562 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:51.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:51.564 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:51.665 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:51.690 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:51.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:51.692 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:51.793 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:51.819 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:51.820 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:58:51.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:51.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:51.923 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:51.951 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:51.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:51.956 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:52.057 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:52.087 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:52.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:52.098 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:52 smithi016 bash[75004]: cluster 2023-09-23T19:58:50.596903+0000 mgr.y (mgr.25325) 1548 : cluster 0 pgmap v1575: 89 pgs: 89 active+clean; 5.9 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 34 MiB/s wr, 848 op/s 2023-09-23T19:58:52.098 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:52 smithi016 bash[75004]: audit 2023-09-23T19:58:51.013351+0000 mon.a (mon.0) 1861 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:58:52.098 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:52 smithi016 bash[75004]: audit 2023-09-23T19:58:51.028134+0000 mon.a (mon.0) 1862 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:58:52.098 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:52 smithi016 bash[77995]: cluster 2023-09-23T19:58:50.596903+0000 mgr.y (mgr.25325) 1548 : cluster 0 pgmap v1575: 89 pgs: 89 active+clean; 5.9 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 34 MiB/s wr, 848 op/s 2023-09-23T19:58:52.099 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:52 smithi016 bash[77995]: audit 2023-09-23T19:58:51.013351+0000 mon.a (mon.0) 1861 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:58:52.099 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:52 smithi016 bash[77995]: audit 2023-09-23T19:58:51.028134+0000 mon.a (mon.0) 1862 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:58:52.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:52.200 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:52.243 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:52.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:52.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:52.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:52 smithi092 bash[58855]: cluster 2023-09-23T19:58:50.596903+0000 mgr.y (mgr.25325) 1548 : cluster 0 pgmap v1575: 89 pgs: 89 active+clean; 5.9 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 34 MiB/s wr, 848 op/s 2023-09-23T19:58:52.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:52 smithi092 bash[58855]: audit 2023-09-23T19:58:51.013351+0000 mon.a (mon.0) 1861 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:58:52.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:52 smithi092 bash[58855]: audit 2023-09-23T19:58:51.028134+0000 mon.a (mon.0) 1862 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:58:52.351 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:52.373 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:52.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:52.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:52.477 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:52.501 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:52.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:52.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:52.605 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:52.634 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:52.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:52.637 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:52.738 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:52.766 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:52.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:52.768 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:52.869 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:52.904 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:52.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:52.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:53.012 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:53.033 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is unset 2023-09-23T19:58:53.033 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:53.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:53.036 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:53.041 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T19:58:53.137 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:53.143 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:53 smithi016 bash[75004]: audit 2023-09-23T19:58:52.364604+0000 mon.a (mon.0) 1863 : audit 1 from='client.? 172.21.15.16:0/4104468612' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:58:53.143 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:53 smithi016 bash[75004]: audit 2023-09-23T19:58:52.614747+0000 mon.c (mon.2) 235 : audit 1 from='client.? 172.21.15.16:0/567459264' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T19:58:53.143 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:53 smithi016 bash[75004]: audit 2023-09-23T19:58:52.615674+0000 mon.a (mon.0) 1864 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T19:58:53.143 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:53 smithi016 bash[77995]: audit 2023-09-23T19:58:52.364604+0000 mon.a (mon.0) 1863 : audit 1 from='client.? 172.21.15.16:0/4104468612' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:58:53.144 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:53 smithi016 bash[77995]: audit 2023-09-23T19:58:52.614747+0000 mon.c (mon.2) 235 : audit 1 from='client.? 172.21.15.16:0/567459264' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T19:58:53.144 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:53 smithi016 bash[77995]: audit 2023-09-23T19:58:52.615674+0000 mon.a (mon.0) 1864 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T19:58:53.161 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:53.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:53.163 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:53.263 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:53.271 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:53 smithi092 bash[58855]: audit 2023-09-23T19:58:52.364604+0000 mon.a (mon.0) 1863 : audit 1 from='client.? 172.21.15.16:0/4104468612' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:58:53.271 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:53 smithi092 bash[58855]: audit 2023-09-23T19:58:52.614747+0000 mon.c (mon.2) 235 : audit 1 from='client.? 172.21.15.16:0/567459264' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T19:58:53.271 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:53 smithi092 bash[58855]: audit 2023-09-23T19:58:52.615674+0000 mon.a (mon.0) 1864 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T19:58:53.284 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:53.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:53.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:53.389 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:53.424 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:53.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:53.427 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:53.527 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:53.566 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:53.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:53.572 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:53.673 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:53.718 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:53.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:53.724 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:53.824 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:53.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:53.855 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:53.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:53.956 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:53.992 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:53.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:54.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:54.100 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:54.144 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:54.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:54.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:54.251 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:54.271 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:54 smithi016 bash[75004]: cluster 2023-09-23T19:58:52.597411+0000 mgr.y (mgr.25325) 1549 : cluster 0 pgmap v1576: 89 pgs: 89 active+clean; 5.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 21 MiB/s wr, 687 op/s 2023-09-23T19:58:54.272 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:54 smithi016 bash[75004]: cluster 2023-09-23T19:58:53.009633+0000 mon.a (mon.0) 1865 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:58:54.272 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:54 smithi016 bash[75004]: cluster 2023-09-23T19:58:53.009668+0000 mon.a (mon.0) 1866 : cluster 1 Cluster is now healthy 2023-09-23T19:58:54.272 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:54 smithi016 bash[75004]: audit 2023-09-23T19:58:53.025317+0000 mon.a (mon.0) 1867 : audit 1 from='client.? 172.21.15.16:0/4104468612' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:58:54.272 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:54 smithi016 bash[75004]: audit 2023-09-23T19:58:53.025556+0000 mon.a (mon.0) 1868 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T19:58:54.272 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:54 smithi016 bash[75004]: cluster 2023-09-23T19:58:53.028953+0000 mon.a (mon.0) 1869 : cluster 0 osdmap e265: 8 total, 8 up, 8 in 2023-09-23T19:58:54.272 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:54 smithi016 bash[75004]: cluster 2023-09-23T19:58:53.036361+0000 osd.5 (osd.5) 190 : cluster 3 failed to encode map e265 with expected crc 2023-09-23T19:58:54.272 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:54 smithi016 bash[75004]: cluster 2023-09-23T19:58:53.038778+0000 osd.6 (osd.6) 200 : cluster 3 failed to encode map e265 with expected crc 2023-09-23T19:58:54.272 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:54 smithi016 bash[75004]: cluster 2023-09-23T19:58:53.042752+0000 osd.7 (osd.7) 178 : cluster 3 failed to encode map e265 with expected crc 2023-09-23T19:58:54.272 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:54 smithi016 bash[75004]: cluster 2023-09-23T19:58:53.044302+0000 osd.7 (osd.7) 179 : cluster 3 failed to encode map e265 with expected crc 2023-09-23T19:58:54.272 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:54 smithi016 bash[75004]: cluster 2023-09-23T19:58:53.045493+0000 osd.4 (osd.4) 207 : cluster 3 failed to encode map e265 with expected crc 2023-09-23T19:58:54.273 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:54.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:54.274 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:54.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:54 smithi092 bash[58855]: cluster 2023-09-23T19:58:52.597411+0000 mgr.y (mgr.25325) 1549 : cluster 0 pgmap v1576: 89 pgs: 89 active+clean; 5.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 21 MiB/s wr, 687 op/s 2023-09-23T19:58:54.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:54 smithi092 bash[58855]: cluster 2023-09-23T19:58:53.009633+0000 mon.a (mon.0) 1865 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:58:54.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:54 smithi092 bash[58855]: cluster 2023-09-23T19:58:53.009668+0000 mon.a (mon.0) 1866 : cluster 1 Cluster is now healthy 2023-09-23T19:58:54.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:54 smithi092 bash[58855]: audit 2023-09-23T19:58:53.025317+0000 mon.a (mon.0) 1867 : audit 1 from='client.? 172.21.15.16:0/4104468612' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:58:54.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:54 smithi092 bash[58855]: audit 2023-09-23T19:58:53.025556+0000 mon.a (mon.0) 1868 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T19:58:54.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:54 smithi092 bash[58855]: cluster 2023-09-23T19:58:53.028953+0000 mon.a (mon.0) 1869 : cluster 0 osdmap e265: 8 total, 8 up, 8 in 2023-09-23T19:58:54.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:54 smithi092 bash[58855]: cluster 2023-09-23T19:58:53.036361+0000 osd.5 (osd.5) 190 : cluster 3 failed to encode map e265 with expected crc 2023-09-23T19:58:54.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:54 smithi092 bash[58855]: cluster 2023-09-23T19:58:53.038778+0000 osd.6 (osd.6) 200 : cluster 3 failed to encode map e265 with expected crc 2023-09-23T19:58:54.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:54 smithi092 bash[58855]: cluster 2023-09-23T19:58:53.042752+0000 osd.7 (osd.7) 178 : cluster 3 failed to encode map e265 with expected crc 2023-09-23T19:58:54.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:54 smithi092 bash[58855]: cluster 2023-09-23T19:58:53.044302+0000 osd.7 (osd.7) 179 : cluster 3 failed to encode map e265 with expected crc 2023-09-23T19:58:54.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:54 smithi092 bash[58855]: cluster 2023-09-23T19:58:53.045493+0000 osd.4 (osd.4) 207 : cluster 3 failed to encode map e265 with expected crc 2023-09-23T19:58:54.377 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:54.413 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:54.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:54.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:54.519 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:54.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:54 smithi016 bash[77995]: cluster 2023-09-23T19:58:52.597411+0000 mgr.y (mgr.25325) 1549 : cluster 0 pgmap v1576: 89 pgs: 89 active+clean; 5.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 21 MiB/s wr, 687 op/s 2023-09-23T19:58:54.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:54 smithi016 bash[77995]: cluster 2023-09-23T19:58:53.009633+0000 mon.a (mon.0) 1865 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:58:54.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:54 smithi016 bash[77995]: cluster 2023-09-23T19:58:53.009668+0000 mon.a (mon.0) 1866 : cluster 1 Cluster is now healthy 2023-09-23T19:58:54.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:54 smithi016 bash[77995]: audit 2023-09-23T19:58:53.025317+0000 mon.a (mon.0) 1867 : audit 1 from='client.? 172.21.15.16:0/4104468612' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:58:54.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:54 smithi016 bash[77995]: audit 2023-09-23T19:58:53.025556+0000 mon.a (mon.0) 1868 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T19:58:54.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:54 smithi016 bash[77995]: cluster 2023-09-23T19:58:53.028953+0000 mon.a (mon.0) 1869 : cluster 0 osdmap e265: 8 total, 8 up, 8 in 2023-09-23T19:58:54.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:54 smithi016 bash[77995]: cluster 2023-09-23T19:58:53.036361+0000 osd.5 (osd.5) 190 : cluster 3 failed to encode map e265 with expected crc 2023-09-23T19:58:54.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:54 smithi016 bash[77995]: cluster 2023-09-23T19:58:53.038778+0000 osd.6 (osd.6) 200 : cluster 3 failed to encode map e265 with expected crc 2023-09-23T19:58:54.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:54 smithi016 bash[77995]: cluster 2023-09-23T19:58:53.042752+0000 osd.7 (osd.7) 178 : cluster 3 failed to encode map e265 with expected crc 2023-09-23T19:58:54.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:54 smithi016 bash[77995]: cluster 2023-09-23T19:58:53.044302+0000 osd.7 (osd.7) 179 : cluster 3 failed to encode map e265 with expected crc 2023-09-23T19:58:54.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:54 smithi016 bash[77995]: cluster 2023-09-23T19:58:53.045493+0000 osd.4 (osd.4) 207 : cluster 3 failed to encode map e265 with expected crc 2023-09-23T19:58:54.542 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:54.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:54.544 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:54.645 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:54.682 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:54.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:54.684 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:54.785 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:54.813 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:54.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:54.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:54.919 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:54.942 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:54.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:54.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:55.045 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:55.067 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:55.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:55.070 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:55.170 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:55.194 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:55.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:55.197 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:55.298 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:55.325 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:55.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:55.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:55.365 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T19:58:55.430 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:55.468 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:55.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:55.470 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:55.571 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:55.599 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:55.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:55.601 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:55.701 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:55.724 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:55.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:55.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:55.826 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:55.859 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:55.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:55.861 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:55.962 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:55.997 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:56.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:56.000 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:56.028 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:58:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:58:55] "GET /metrics HTTP/1.1" 200 36373 "" "Prometheus/2.33.4" 2023-09-23T19:58:56.101 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:56.126 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:56.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:56.129 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:56.227 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:56 smithi092 bash[58855]: cluster 2023-09-23T19:58:54.597969+0000 mgr.y (mgr.25325) 1550 : cluster 0 pgmap v1578: 89 pgs: 89 active+clean; 5.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 15 MiB/s wr, 756 op/s 2023-09-23T19:58:56.230 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:56.252 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:56.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:56.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:56.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:56 smithi016 bash[75004]: cluster 2023-09-23T19:58:54.597969+0000 mgr.y (mgr.25325) 1550 : cluster 0 pgmap v1578: 89 pgs: 89 active+clean; 5.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 15 MiB/s wr, 756 op/s 2023-09-23T19:58:56.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:56 smithi016 bash[77995]: cluster 2023-09-23T19:58:54.597969+0000 mgr.y (mgr.25325) 1550 : cluster 0 pgmap v1578: 89 pgs: 89 active+clean; 5.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 15 MiB/s wr, 756 op/s 2023-09-23T19:58:56.355 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:56.393 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:56.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:56.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:56.435 INFO:teuthology.orchestra.run.smithi092.stdout:ERROR: (22) Invalid argument 2023-09-23T19:58:56.435 INFO:teuthology.orchestra.run.smithi092.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:58:56.500 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:56.524 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:56.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:56.529 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:56.630 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:56.665 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:56.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:56.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:56.777 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:56.802 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:56.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:56.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:56.905 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:56.928 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:56.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:56.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:57.031 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:57.062 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:57.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:57.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:57.165 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:57.188 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:57.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:57.190 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:57.291 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:57.313 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:57.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:57.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:57.342 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T19:58:57.416 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:57.478 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:57.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:57.483 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:57.584 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:57.593 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T19:58:57.604 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:57.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:57.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:57.707 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:57.750 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:57.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:57.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:57.857 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:57.874 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:58:57.885 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:57.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:57.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:57.988 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:58.031 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:58.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:58.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:58.138 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:58.163 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:58.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:58.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:58.270 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:58.271 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:58 smithi016 bash[75004]: cluster 2023-09-23T19:58:56.598601+0000 mgr.y (mgr.25325) 1551 : cluster 0 pgmap v1579: 89 pgs: 89 active+clean; 5.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 3.3 MiB/s wr, 763 op/s 2023-09-23T19:58:58.272 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:58 smithi016 bash[75004]: audit 2023-09-23T19:58:57.268557+0000 mon.a (mon.0) 1870 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:58:58.272 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:58 smithi016 bash[75004]: audit 2023-09-23T19:58:57.287006+0000 mon.a (mon.0) 1871 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:58:58.299 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:58.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:58.306 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:58.406 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:58.431 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:58.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:58.435 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:58.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:58 smithi016 bash[77995]: cluster 2023-09-23T19:58:56.598601+0000 mgr.y (mgr.25325) 1551 : cluster 0 pgmap v1579: 89 pgs: 89 active+clean; 5.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 3.3 MiB/s wr, 763 op/s 2023-09-23T19:58:58.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:58 smithi016 bash[77995]: audit 2023-09-23T19:58:57.268557+0000 mon.a (mon.0) 1870 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:58:58.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:58 smithi016 bash[77995]: audit 2023-09-23T19:58:57.287006+0000 mon.a (mon.0) 1871 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:58:58.536 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:58.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:58 smithi092 bash[58855]: cluster 2023-09-23T19:58:56.598601+0000 mgr.y (mgr.25325) 1551 : cluster 0 pgmap v1579: 89 pgs: 89 active+clean; 5.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 3.3 MiB/s wr, 763 op/s 2023-09-23T19:58:58.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:58 smithi092 bash[58855]: audit 2023-09-23T19:58:57.268557+0000 mon.a (mon.0) 1870 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:58:58.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:58 smithi092 bash[58855]: audit 2023-09-23T19:58:57.287006+0000 mon.a (mon.0) 1871 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:58:58.564 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:58.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:58.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:58.668 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:58.693 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:58.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:58.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:58.797 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:58.844 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:58.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:58.851 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:58.952 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:58.984 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:58:58.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:58.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:58:59.089 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:59.118 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:59.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:59.122 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:59.223 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:59.255 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:59.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:59.261 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:59.362 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:59.387 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:59.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:59.389 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:59.459 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:59 smithi016 bash[75004]: audit 2023-09-23T19:58:58.457588+0000 mon.a (mon.0) 1872 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:58:59.459 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:59 smithi016 bash[75004]: audit 2023-09-23T19:58:58.471857+0000 mon.a (mon.0) 1873 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:58:59.459 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:59 smithi016 bash[75004]: cluster 2023-09-23T19:58:58.599208+0000 mgr.y (mgr.25325) 1552 : cluster 0 pgmap v1580: 89 pgs: 89 active+clean; 5.6 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 230 KiB/s wr, 764 op/s 2023-09-23T19:58:59.490 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:59.522 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:59.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:59.527 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:59.627 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:59.664 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:59.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:59.666 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:59.766 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:59.771 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:59 smithi016 bash[75004]: audit 2023-09-23T19:58:58.702540+0000 mon.a (mon.0) 1874 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:58:59.771 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:58:59 smithi016 bash[75004]: audit 2023-09-23T19:58:58.704122+0000 mon.a (mon.0) 1875 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:58:59.771 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:59 smithi016 bash[77995]: audit 2023-09-23T19:58:58.457588+0000 mon.a (mon.0) 1872 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:58:59.771 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:59 smithi016 bash[77995]: audit 2023-09-23T19:58:58.471857+0000 mon.a (mon.0) 1873 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:58:59.772 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:59 smithi016 bash[77995]: cluster 2023-09-23T19:58:58.599208+0000 mgr.y (mgr.25325) 1552 : cluster 0 pgmap v1580: 89 pgs: 89 active+clean; 5.6 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 230 KiB/s wr, 764 op/s 2023-09-23T19:58:59.772 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:59 smithi016 bash[77995]: audit 2023-09-23T19:58:58.702540+0000 mon.a (mon.0) 1874 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:58:59.772 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:58:59 smithi016 bash[77995]: audit 2023-09-23T19:58:58.704122+0000 mon.a (mon.0) 1875 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:58:59.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:59 smithi092 bash[58855]: audit 2023-09-23T19:58:58.457588+0000 mon.a (mon.0) 1872 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:58:59.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:59 smithi092 bash[58855]: audit 2023-09-23T19:58:58.471857+0000 mon.a (mon.0) 1873 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:58:59.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:59 smithi092 bash[58855]: cluster 2023-09-23T19:58:58.599208+0000 mgr.y (mgr.25325) 1552 : cluster 0 pgmap v1580: 89 pgs: 89 active+clean; 5.6 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 230 KiB/s wr, 764 op/s 2023-09-23T19:58:59.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:59 smithi092 bash[58855]: audit 2023-09-23T19:58:58.702540+0000 mon.a (mon.0) 1874 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:58:59.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:58:59 smithi092 bash[58855]: audit 2023-09-23T19:58:58.704122+0000 mon.a (mon.0) 1875 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:58:59.803 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:59.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:59.814 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:59.910 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:58:59.915 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:59.955 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:59.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:58:59.960 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:00.060 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:00.083 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:00.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:00.086 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:00.187 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:00.220 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:00.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:00.224 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:00.324 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:00.350 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:00.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:00.352 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:00.453 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:00.481 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:00.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:00.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:00.588 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:00.618 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:00.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:00.624 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:00.725 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:00.756 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:00.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:00.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:00.859 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:00.887 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:00.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:00.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:00.992 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:01.023 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:01.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:01.027 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:01.130 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:01.155 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:01.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:01.160 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:01.261 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:01.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:01.295 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:01.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:01.395 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:01.433 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:01.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:01.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:01.538 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:01.568 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:01.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:01.572 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:01.673 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:01.697 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:01.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:01.703 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:59:01.804 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:01.833 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:01.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:01.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:59:01.937 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:01.971 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:01.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:01.974 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:02.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:01 smithi016 bash[75004]: cluster 2023-09-23T19:59:00.599944+0000 mgr.y (mgr.25325) 1553 : cluster 0 pgmap v1581: 89 pgs: 89 active+clean; 5.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 658 op/s 2023-09-23T19:59:02.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:01 smithi016 bash[77995]: cluster 2023-09-23T19:59:00.599944+0000 mgr.y (mgr.25325) 1553 : cluster 0 pgmap v1581: 89 pgs: 89 active+clean; 5.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 658 op/s 2023-09-23T19:59:02.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:01 smithi092 bash[58855]: cluster 2023-09-23T19:59:00.599944+0000 mgr.y (mgr.25325) 1553 : cluster 0 pgmap v1581: 89 pgs: 89 active+clean; 5.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 658 op/s 2023-09-23T19:59:02.075 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:02.115 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:02.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:02.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:02.221 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:02.248 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:02.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:02.253 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:02.354 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:02.387 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:02.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:02.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:02.394 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:59:02.490 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:02.515 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:02.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:02.523 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:02.623 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:02.646 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:02.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:02.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:02.753 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:02.768 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:59:02.782 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:02.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:02.786 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:02.886 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:02.916 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:02.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:02.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:03.020 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:03.045 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:03.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:03.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:03.152 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:03.192 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:03.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:03.197 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:03.299 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:03.319 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:03.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:03.322 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:03.423 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:03.452 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:03.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:03.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:03.555 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:03.584 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:03.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:03.592 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:03.693 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:03.715 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:03.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:03.720 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:03.821 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:03.854 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:03.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:03.857 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:03.957 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:03.986 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:03.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:03.993 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:04.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:03 smithi016 bash[75004]: cluster 2023-09-23T19:59:02.600649+0000 mgr.y (mgr.25325) 1554 : cluster 0 pgmap v1582: 89 pgs: 89 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 723 op/s 2023-09-23T19:59:04.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:03 smithi016 bash[77995]: cluster 2023-09-23T19:59:02.600649+0000 mgr.y (mgr.25325) 1554 : cluster 0 pgmap v1582: 89 pgs: 89 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 723 op/s 2023-09-23T19:59:04.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:03 smithi092 bash[58855]: cluster 2023-09-23T19:59:02.600649+0000 mgr.y (mgr.25325) 1554 : cluster 0 pgmap v1582: 89 pgs: 89 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 723 op/s 2023-09-23T19:59:04.094 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:04.122 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:04.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:04.129 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:04.230 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:04.253 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:04.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:04.256 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:59:04.357 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:04.385 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:04.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:04.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:04.492 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:04.520 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:04.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:04.526 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:04.627 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:04.658 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:04.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:04.662 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:04.763 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:04.781 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:04.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:04.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:04.884 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:04.910 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:04.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:04.918 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:05.019 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:05.051 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:05.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:05.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:05.153 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:05.186 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:05.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:05.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:05.288 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:05.328 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:05.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:05.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:05.433 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:05.465 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:05.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:05.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:05.571 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:05.627 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:05.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:05.632 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:05.732 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:05.741 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:59:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:59:05] "GET /metrics HTTP/1.1" 200 36376 "" "Prometheus/2.33.4" 2023-09-23T19:59:05.763 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:05.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:05.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:05.868 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:05.907 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:05.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:05.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:05.943 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:05 smithi016 bash[77995]: cluster 2023-09-23T19:59:04.601195+0000 mgr.y (mgr.25325) 1555 : cluster 0 pgmap v1583: 89 pgs: 89 active+clean; 5.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 703 op/s 2023-09-23T19:59:05.943 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:05 smithi016 bash[75004]: cluster 2023-09-23T19:59:04.601195+0000 mgr.y (mgr.25325) 1555 : cluster 0 pgmap v1583: 89 pgs: 89 active+clean; 5.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 703 op/s 2023-09-23T19:59:06.009 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:06.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:05 smithi092 bash[58855]: cluster 2023-09-23T19:59:04.601195+0000 mgr.y (mgr.25325) 1555 : cluster 0 pgmap v1583: 89 pgs: 89 active+clean; 5.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 703 op/s 2023-09-23T19:59:06.044 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:06.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:06.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:06.148 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:06.174 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:06.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:06.183 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:06.284 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:06.314 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:06.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:06.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:59:06.421 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:06.442 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:06.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:06.444 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:06.544 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:06.572 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:06.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:06.576 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:06.679 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:06.717 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:06.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:06.721 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:06.822 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:06.850 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:06.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:06.852 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:06.953 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:06.975 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:06.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:06.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:06.998 INFO:teuthology.orchestra.run.smithi092.stdout:ERROR: (22) Invalid argument 2023-09-23T19:59:06.999 INFO:teuthology.orchestra.run.smithi092.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:59:07.079 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:07.106 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:07.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:07.114 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:07.215 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:07.240 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:07.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:07.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:59:07.342 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:59:07.345 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:07.370 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:07.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:07.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:59:07.385 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:59:07.431 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:59:07.471 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:59:07.479 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:07.507 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:59:07.527 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:07.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:07.533 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:07.554 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:59:07.584 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:59:07.618 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:59:07.620 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:59:07.620 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:59:07.620 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:59:07.621 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:59:07.621 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:59:07.621 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:59:07.621 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:59:07.621 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:59:07.633 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:07.668 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:07.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:07.672 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:07.773 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:07.798 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:07.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:07.801 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:07.901 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:07.910 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:07 smithi092 bash[58855]: cluster 2023-09-23T19:59:06.602089+0000 mgr.y (mgr.25325) 1556 : cluster 0 pgmap v1584: 89 pgs: 89 active+clean; 5.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 783 op/s 2023-09-23T19:59:07.911 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:07 smithi092 bash[58855]: audit 2023-09-23T19:59:07.287624+0000 mon.a (mon.0) 1876 : audit 0 from='client.? 172.21.15.16:0/547655776' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:59:07.929 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:07.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:07.932 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:59:08.033 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:08.061 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:08.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:08.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:08.087 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T19:59:08.165 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:07 smithi016 bash[77995]: cluster 2023-09-23T19:59:06.602089+0000 mgr.y (mgr.25325) 1556 : cluster 0 pgmap v1584: 89 pgs: 89 active+clean; 5.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 783 op/s 2023-09-23T19:59:08.314 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:07 smithi016 bash[77995]: audit 2023-09-23T19:59:07.287624+0000 mon.a (mon.0) 1876 : audit 0 from='client.? 172.21.15.16:0/547655776' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:59:08.314 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:07 smithi016 bash[75004]: cluster 2023-09-23T19:59:06.602089+0000 mgr.y (mgr.25325) 1556 : cluster 0 pgmap v1584: 89 pgs: 89 active+clean; 5.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 783 op/s 2023-09-23T19:59:08.314 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:07 smithi016 bash[75004]: audit 2023-09-23T19:59:07.287624+0000 mon.a (mon.0) 1876 : audit 0 from='client.? 172.21.15.16:0/547655776' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:59:08.315 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:08.341 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:08.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:08.347 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:59:08.448 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:08.475 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:08.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:08.477 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:08.578 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:08.608 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:08.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:08.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:08.711 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:08.743 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:08.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:08.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:08.848 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:08.874 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:08.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:08.880 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:08.981 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:09.017 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:09.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:09.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:59:09.121 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:09.147 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:09.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:09.150 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:59:09.250 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:09.278 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:09.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:09.281 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:09.382 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:09.415 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:09.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:09.419 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:09.520 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:09.546 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:09.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:09.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:09.649 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:09.684 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:09.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:09.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:09.791 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:09.815 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:09.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:09.819 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:09.920 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:09.928 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:09 smithi092 bash[58855]: cluster 2023-09-23T19:59:08.602570+0000 mgr.y (mgr.25325) 1557 : cluster 0 pgmap v1585: 89 pgs: 89 active+clean; 5.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2023-09-23T19:59:09.945 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:09.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:09.950 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:59:10.050 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:10.078 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:10.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:10.083 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:10.184 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:10.211 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:10.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:10.216 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:59:10.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:09 smithi016 bash[77995]: cluster 2023-09-23T19:59:08.602570+0000 mgr.y (mgr.25325) 1557 : cluster 0 pgmap v1585: 89 pgs: 89 active+clean; 5.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2023-09-23T19:59:10.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:09 smithi016 bash[75004]: cluster 2023-09-23T19:59:08.602570+0000 mgr.y (mgr.25325) 1557 : cluster 0 pgmap v1585: 89 pgs: 89 active+clean; 5.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2023-09-23T19:59:10.317 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:10.354 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:10.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:10.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:10.458 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:10.487 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:10.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:10.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:10.590 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:10.617 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:10.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:10.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:10.723 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:10.754 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:10.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:10.757 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:10.858 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:10.895 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:10.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:10.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:11.004 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:11.032 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:11.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:11.037 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:11.138 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:11.172 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:11.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:11.177 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:11.278 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:11.304 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:11.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:11.308 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:11.408 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:11.436 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:11.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:11.443 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:11.544 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:11.566 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:11.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:11.568 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:11.669 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:11.692 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:11.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:11.695 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:11.795 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:11.819 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:11.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:11.821 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:11.922 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:11.960 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:11.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:11.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:12.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:11 smithi016 bash[75004]: cluster 2023-09-23T19:59:10.603174+0000 mgr.y (mgr.25325) 1558 : cluster 0 pgmap v1586: 89 pgs: 89 active+clean; 5.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 755 op/s 2023-09-23T19:59:12.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:11 smithi016 bash[77995]: cluster 2023-09-23T19:59:10.603174+0000 mgr.y (mgr.25325) 1558 : cluster 0 pgmap v1586: 89 pgs: 89 active+clean; 5.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 755 op/s 2023-09-23T19:59:12.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:11 smithi092 bash[58855]: cluster 2023-09-23T19:59:10.603174+0000 mgr.y (mgr.25325) 1558 : cluster 0 pgmap v1586: 89 pgs: 89 active+clean; 5.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 755 op/s 2023-09-23T19:59:12.063 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:12.103 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:12.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:12.111 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:12.211 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:12.249 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:12.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:12.253 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:12.354 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:12.391 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:12.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:12.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:59:12.495 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:12.524 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:12.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:12.526 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:12.627 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:12.652 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:12.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:12.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:12.758 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:12.800 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:12.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:12.806 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:12.889 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:59:12.906 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:12.943 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:12.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:12.948 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:13.049 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:13.074 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:13.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:13.077 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:13.177 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:13.201 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:13.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:13.203 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:13.304 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:13.327 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:13.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:13.332 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:13.433 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:13.456 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:13.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:13.458 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:13.559 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:13.593 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:13.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:13.601 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:13.702 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:13.726 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:13.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:13.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:13.743 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:59:13.832 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:13.833 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T19:59:13.876 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:13.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:13.878 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:13.979 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:14.025 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:14.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:14.028 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:13 smithi016 bash[75004]: cluster 2023-09-23T19:59:12.603685+0000 mgr.y (mgr.25325) 1559 : cluster 0 pgmap v1587: 89 pgs: 89 active+clean; 4.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 726 op/s 2023-09-23T19:59:14.028 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:13 smithi016 bash[75004]: audit 2023-09-23T19:59:13.692737+0000 mon.c (mon.2) 236 : audit 1 from='client.? 172.21.15.16:0/3167288264' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T19:59:14.028 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:13 smithi016 bash[75004]: audit 2023-09-23T19:59:13.693933+0000 mon.a (mon.0) 1877 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T19:59:14.029 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:13 smithi016 bash[75004]: audit 2023-09-23T19:59:13.694875+0000 mon.a (mon.0) 1878 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:59:14.029 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:13 smithi016 bash[77995]: cluster 2023-09-23T19:59:12.603685+0000 mgr.y (mgr.25325) 1559 : cluster 0 pgmap v1587: 89 pgs: 89 active+clean; 4.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 726 op/s 2023-09-23T19:59:14.029 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:13 smithi016 bash[77995]: audit 2023-09-23T19:59:13.692737+0000 mon.c (mon.2) 236 : audit 1 from='client.? 172.21.15.16:0/3167288264' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T19:59:14.029 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:13 smithi016 bash[77995]: audit 2023-09-23T19:59:13.693933+0000 mon.a (mon.0) 1877 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T19:59:14.029 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:13 smithi016 bash[77995]: audit 2023-09-23T19:59:13.694875+0000 mon.a (mon.0) 1878 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:59:14.030 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:14.037 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:13 smithi092 bash[58855]: cluster 2023-09-23T19:59:12.603685+0000 mgr.y (mgr.25325) 1559 : cluster 0 pgmap v1587: 89 pgs: 89 active+clean; 4.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 726 op/s 2023-09-23T19:59:14.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:13 smithi092 bash[58855]: audit 2023-09-23T19:59:13.692737+0000 mon.c (mon.2) 236 : audit 1 from='client.? 172.21.15.16:0/3167288264' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T19:59:14.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:13 smithi092 bash[58855]: audit 2023-09-23T19:59:13.693933+0000 mon.a (mon.0) 1877 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T19:59:14.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:13 smithi092 bash[58855]: audit 2023-09-23T19:59:13.694875+0000 mon.a (mon.0) 1878 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:59:14.130 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:14.158 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:14.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:14.160 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:14.261 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:14.288 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:14.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:14.292 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:14.393 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:14.428 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:14.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:14.431 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:14.532 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:14.558 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:14.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:14.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:14.661 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:14.694 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:14.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:14.697 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:14.798 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:14.823 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:14.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:14.826 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:14.927 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:14.954 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:14.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:14.957 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:15.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:14 smithi016 bash[75004]: audit 2023-09-23T19:59:13.816222+0000 mon.a (mon.0) 1879 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T19:59:15.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:14 smithi016 bash[75004]: cluster 2023-09-23T19:59:13.816404+0000 mon.a (mon.0) 1880 : cluster 0 osdmap e266: 8 total, 8 up, 8 in 2023-09-23T19:59:15.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:14 smithi016 bash[75004]: cluster 2023-09-23T19:59:13.833528+0000 osd.6 (osd.6) 201 : cluster 3 failed to encode map e266 with expected crc 2023-09-23T19:59:15.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:14 smithi016 bash[75004]: cluster 2023-09-23T19:59:13.845895+0000 osd.7 (osd.7) 180 : cluster 3 failed to encode map e266 with expected crc 2023-09-23T19:59:15.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:14 smithi016 bash[75004]: cluster 2023-09-23T19:59:13.847279+0000 osd.7 (osd.7) 181 : cluster 3 failed to encode map e266 with expected crc 2023-09-23T19:59:15.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:14 smithi016 bash[75004]: cluster 2023-09-23T19:59:13.859334+0000 osd.5 (osd.5) 191 : cluster 3 failed to encode map e266 with expected crc 2023-09-23T19:59:15.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:14 smithi016 bash[77995]: audit 2023-09-23T19:59:13.816222+0000 mon.a (mon.0) 1879 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T19:59:15.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:14 smithi016 bash[77995]: cluster 2023-09-23T19:59:13.816404+0000 mon.a (mon.0) 1880 : cluster 0 osdmap e266: 8 total, 8 up, 8 in 2023-09-23T19:59:15.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:14 smithi016 bash[77995]: cluster 2023-09-23T19:59:13.833528+0000 osd.6 (osd.6) 201 : cluster 3 failed to encode map e266 with expected crc 2023-09-23T19:59:15.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:14 smithi016 bash[77995]: cluster 2023-09-23T19:59:13.845895+0000 osd.7 (osd.7) 180 : cluster 3 failed to encode map e266 with expected crc 2023-09-23T19:59:15.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:14 smithi016 bash[77995]: cluster 2023-09-23T19:59:13.847279+0000 osd.7 (osd.7) 181 : cluster 3 failed to encode map e266 with expected crc 2023-09-23T19:59:15.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:14 smithi016 bash[77995]: cluster 2023-09-23T19:59:13.859334+0000 osd.5 (osd.5) 191 : cluster 3 failed to encode map e266 with expected crc 2023-09-23T19:59:15.058 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:15.067 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:14 smithi092 bash[58855]: audit 2023-09-23T19:59:13.816222+0000 mon.a (mon.0) 1879 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T19:59:15.067 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:14 smithi092 bash[58855]: cluster 2023-09-23T19:59:13.816404+0000 mon.a (mon.0) 1880 : cluster 0 osdmap e266: 8 total, 8 up, 8 in 2023-09-23T19:59:15.067 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:14 smithi092 bash[58855]: cluster 2023-09-23T19:59:13.833528+0000 osd.6 (osd.6) 201 : cluster 3 failed to encode map e266 with expected crc 2023-09-23T19:59:15.067 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:14 smithi092 bash[58855]: cluster 2023-09-23T19:59:13.845895+0000 osd.7 (osd.7) 180 : cluster 3 failed to encode map e266 with expected crc 2023-09-23T19:59:15.067 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:14 smithi092 bash[58855]: cluster 2023-09-23T19:59:13.847279+0000 osd.7 (osd.7) 181 : cluster 3 failed to encode map e266 with expected crc 2023-09-23T19:59:15.068 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:14 smithi092 bash[58855]: cluster 2023-09-23T19:59:13.859334+0000 osd.5 (osd.5) 191 : cluster 3 failed to encode map e266 with expected crc 2023-09-23T19:59:15.085 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:15.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:15.088 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:15.189 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:15.214 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:15.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:15.223 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:15.323 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:15.343 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:15.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:15.350 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:15.450 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:15.484 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:15.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:15.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:15.589 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:15.618 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:15.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:15.620 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:15.721 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:15.730 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:59:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:59:15] "GET /metrics HTTP/1.1" 200 36374 "" "Prometheus/2.33.4" 2023-09-23T19:59:15.749 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:15.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:15.757 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:15.857 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:15.866 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:15 smithi092 bash[58855]: cluster 2023-09-23T19:59:14.604271+0000 mgr.y (mgr.25325) 1560 : cluster 0 pgmap v1589: 89 pgs: 89 active+clean; 4.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2023-09-23T19:59:15.893 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:15.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:15.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:15.997 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:16.030 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:16.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:16.032 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:16.126 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:15 smithi016 bash[75004]: cluster 2023-09-23T19:59:14.604271+0000 mgr.y (mgr.25325) 1560 : cluster 0 pgmap v1589: 89 pgs: 89 active+clean; 4.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2023-09-23T19:59:16.127 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:15 smithi016 bash[77995]: cluster 2023-09-23T19:59:14.604271+0000 mgr.y (mgr.25325) 1560 : cluster 0 pgmap v1589: 89 pgs: 89 active+clean; 4.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2023-09-23T19:59:16.133 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:16.166 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:16.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:16.169 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:16.270 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:16.302 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:16.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:16.307 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:16.407 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:16.442 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:16.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:16.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:16.548 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:16.583 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:16.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:16.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:16.689 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:16.715 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:16.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:16.720 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:16.821 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:16.851 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:16.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:16.856 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:16.957 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:16.983 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:16.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:16.989 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:17.090 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:17.111 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:17.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:17.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:17.133 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T19:59:17.217 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:17.244 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:17.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:17.250 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:59:17.350 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:17.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:17.387 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:17.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:17.454 INFO:teuthology.orchestra.run.smithi092.stdout:{ 2023-09-23T19:59:17.454 INFO:teuthology.orchestra.run.smithi092.stdout: "size": 20, 2023-09-23T19:59:17.454 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 600, 2023-09-23T19:59:17.454 INFO:teuthology.orchestra.run.smithi092.stdout: "ops": [ 2023-09-23T19:59:17.455 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.455 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:32634 14.e 14:7b29bb04:::benchmark_data_smithi016_228083_object32633:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-09-23T19:59:17.455 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:57:32.061510+0000", 2023-09-23T19:59:17.455 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 105.390792163, 2023-09-23T19:59:17.455 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.323430726, 2023-09-23T19:59:17.455 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:59:17.455 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:17.455 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:59:17.455 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T19:59:17.455 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T19:59:17.455 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 32634 2023-09-23T19:59:17.455 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.456 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:59:17.456 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.456 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:59:17.456 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.061510+0000", 2023-09-23T19:59:17.456 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:59:17.456 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.456 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.456 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:59:17.456 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.061510+0000", 2023-09-23T19:59:17.456 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999976 2023-09-23T19:59:17.456 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.456 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.457 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:59:17.457 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.061507+0000", 2023-09-23T19:59:17.457 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.7842e-05 2023-09-23T19:59:17.457 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.457 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.457 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:59:17.457 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.061555+0000", 2023-09-23T19:59:17.457 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.7456999999999999e-05 2023-09-23T19:59:17.457 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.457 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.457 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:59:17.457 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.061573+0000", 2023-09-23T19:59:17.458 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.0223e-05 2023-09-23T19:59:17.458 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.458 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.458 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:59:17.458 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.061583+0000", 2023-09-23T19:59:17.458 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.6848000000000006e-05 2023-09-23T19:59:17.458 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.458 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.458 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:59:17.458 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.061670+0000", 2023-09-23T19:59:17.458 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000230239 2023-09-23T19:59:17.458 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.459 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.459 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:59:17.459 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.061900+0000", 2023-09-23T19:59:17.459 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000202544 2023-09-23T19:59:17.459 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.459 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.459 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 2,6", 2023-09-23T19:59:17.459 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.062103+0000", 2023-09-23T19:59:17.459 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0017556869999999999 2023-09-23T19:59:17.459 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.459 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.459 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:59:17.459 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.063858+0000", 2023-09-23T19:59:17.460 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0080978539999999998 2023-09-23T19:59:17.460 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.460 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.460 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:17.460 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.071956+0000", 2023-09-23T19:59:17.460 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.312810857 2023-09-23T19:59:17.460 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.460 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.460 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:17.460 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.384767+0000", 2023-09-23T19:59:17.460 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000100355 2023-09-23T19:59:17.460 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.461 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.461 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:59:17.461 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.384867+0000", 2023-09-23T19:59:17.461 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.3285000000000001e-05 2023-09-23T19:59:17.461 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.461 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.461 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:59:17.461 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.384941+0000", 2023-09-23T19:59:17.461 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.323430726 2023-09-23T19:59:17.461 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:59:17.461 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:59:17.461 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:59:17.462 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.462 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.462 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:32640 14.e 14:77e29d91:::benchmark_data_smithi016_228083_object32639:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-09-23T19:59:17.462 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:57:32.066749+0000", 2023-09-23T19:59:17.462 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 105.385553491, 2023-09-23T19:59:17.462 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.32743465100000002, 2023-09-23T19:59:17.462 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:59:17.462 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:17.462 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:59:17.462 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T19:59:17.462 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T19:59:17.462 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 32640 2023-09-23T19:59:17.463 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.463 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:59:17.463 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.463 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:59:17.463 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.066749+0000", 2023-09-23T19:59:17.463 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:59:17.463 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.463 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.463 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:59:17.463 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.066749+0000", 2023-09-23T19:59:17.463 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T19:59:17.463 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.464 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.464 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:59:17.464 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.066746+0000", 2023-09-23T19:59:17.464 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.0426e-05 2023-09-23T19:59:17.464 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.464 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.464 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:59:17.464 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.066777+0000", 2023-09-23T19:59:17.464 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.5242000000000001e-05 2023-09-23T19:59:17.464 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.464 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.465 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:59:17.465 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.066792+0000", 2023-09-23T19:59:17.465 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.5299999999999999e-06 2023-09-23T19:59:17.465 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.465 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.465 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:59:17.465 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.066800+0000", 2023-09-23T19:59:17.465 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.4175000000000001e-05 2023-09-23T19:59:17.465 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.465 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.465 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:59:17.465 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.066864+0000", 2023-09-23T19:59:17.466 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00016612800000000001 2023-09-23T19:59:17.466 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.466 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.466 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:59:17.466 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.067030+0000", 2023-09-23T19:59:17.466 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00020505999999999999 2023-09-23T19:59:17.466 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.466 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.466 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 2,6", 2023-09-23T19:59:17.466 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.067235+0000", 2023-09-23T19:59:17.466 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0043600469999999997 2023-09-23T19:59:17.467 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.467 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.467 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:59:17.467 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.071595+0000", 2023-09-23T19:59:17.467 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00042618199999999999 2023-09-23T19:59:17.467 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.467 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.467 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:17.467 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.072021+0000", 2023-09-23T19:59:17.467 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.322009086 2023-09-23T19:59:17.467 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.468 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.468 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:17.468 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.394030+0000", 2023-09-23T19:59:17.468 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.5445999999999995e-05 2023-09-23T19:59:17.468 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.468 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.468 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:59:17.468 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.394116+0000", 2023-09-23T19:59:17.468 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.7471000000000005e-05 2023-09-23T19:59:17.468 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.468 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.469 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:59:17.469 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.394183+0000", 2023-09-23T19:59:17.469 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.32743465100000002 2023-09-23T19:59:17.469 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:59:17.469 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:59:17.469 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:59:17.469 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.469 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.469 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:32643 14.e 14:7adc8930:::benchmark_data_smithi016_228083_object32642:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-09-23T19:59:17.469 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:57:32.066926+0000", 2023-09-23T19:59:17.469 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 105.38537637899999, 2023-09-23T19:59:17.470 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.32744851800000002, 2023-09-23T19:59:17.470 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:59:17.470 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:17.470 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:59:17.470 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T19:59:17.470 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T19:59:17.470 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 32643 2023-09-23T19:59:17.470 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.470 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:59:17.470 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.470 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:59:17.471 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.066926+0000", 2023-09-23T19:59:17.471 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:59:17.471 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.471 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.471 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:59:17.471 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.066926+0000", 2023-09-23T19:59:17.471 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.999999 2023-09-23T19:59:17.471 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.471 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.471 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:59:17.471 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.066925+0000", 2023-09-23T19:59:17.472 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.5062000000000002e-05 2023-09-23T19:59:17.472 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.472 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.472 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:59:17.472 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.066980+0000", 2023-09-23T19:59:17.472 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.1095e-05 2023-09-23T19:59:17.472 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.472 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.472 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:59:17.472 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.066991+0000", 2023-09-23T19:59:17.472 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.5839999999999996e-06 2023-09-23T19:59:17.472 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.473 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.473 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:59:17.473 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.066995+0000", 2023-09-23T19:59:17.473 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0015757010000000001 2023-09-23T19:59:17.473 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.473 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.473 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:59:17.473 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.068571+0000", 2023-09-23T19:59:17.473 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000103711 2023-09-23T19:59:17.473 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.473 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.474 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:59:17.474 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.068675+0000", 2023-09-23T19:59:17.474 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00013845900000000001 2023-09-23T19:59:17.474 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.474 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.474 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 2,6", 2023-09-23T19:59:17.474 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.068813+0000", 2023-09-23T19:59:17.474 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0027960139999999999 2023-09-23T19:59:17.474 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.474 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.474 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:59:17.475 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.071609+0000", 2023-09-23T19:59:17.475 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0012392880000000001 2023-09-23T19:59:17.475 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.475 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.475 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:17.475 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.072848+0000", 2023-09-23T19:59:17.475 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.321397347 2023-09-23T19:59:17.475 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.475 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.475 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:17.475 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.394246+0000", 2023-09-23T19:59:17.476 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.2533000000000005e-05 2023-09-23T19:59:17.476 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.476 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.476 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:59:17.476 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.394318+0000", 2023-09-23T19:59:17.476 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.5840000000000001e-05 2023-09-23T19:59:17.476 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.476 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.476 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:59:17.476 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.394374+0000", 2023-09-23T19:59:17.477 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.32744851800000002 2023-09-23T19:59:17.477 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:59:17.477 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:59:17.477 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:59:17.477 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.477 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.477 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:32651 14.e 14:74226410:::benchmark_data_smithi016_228083_object32650:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-09-23T19:59:17.477 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:57:32.069671+0000", 2023-09-23T19:59:17.477 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 105.38263109499999, 2023-09-23T19:59:17.477 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.32487358399999999, 2023-09-23T19:59:17.477 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:59:17.478 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:17.478 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:59:17.478 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T19:59:17.478 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T19:59:17.478 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 32651 2023-09-23T19:59:17.478 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.478 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:59:17.478 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.478 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:59:17.478 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.069671+0000", 2023-09-23T19:59:17.478 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:59:17.479 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.479 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.479 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:59:17.479 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.069671+0000", 2023-09-23T19:59:17.479 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T19:59:17.479 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.479 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.479 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:59:17.479 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.069669+0000", 2023-09-23T19:59:17.479 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.0265999999999997e-05 2023-09-23T19:59:17.479 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.480 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.480 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:59:17.480 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.069709+0000", 2023-09-23T19:59:17.480 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.1335000000000002e-05 2023-09-23T19:59:17.480 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.480 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.480 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:59:17.480 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.069731+0000", 2023-09-23T19:59:17.480 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.3270000000000001e-06 2023-09-23T19:59:17.480 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.480 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.480 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:59:17.481 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.069739+0000", 2023-09-23T19:59:17.481 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00037404499999999999 2023-09-23T19:59:17.481 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.481 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.481 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:59:17.481 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.070113+0000", 2023-09-23T19:59:17.481 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000118404 2023-09-23T19:59:17.481 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.481 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.481 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:59:17.481 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.070231+0000", 2023-09-23T19:59:17.482 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000145771 2023-09-23T19:59:17.482 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.482 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.482 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 2,6", 2023-09-23T19:59:17.482 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.070377+0000", 2023-09-23T19:59:17.482 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0014109319999999999 2023-09-23T19:59:17.482 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.482 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.482 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:59:17.482 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.071788+0000", 2023-09-23T19:59:17.482 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.001220617 2023-09-23T19:59:17.483 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.483 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.483 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:17.483 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.073009+0000", 2023-09-23T19:59:17.483 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.32142080000000001 2023-09-23T19:59:17.483 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.483 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.483 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:17.483 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.394430+0000", 2023-09-23T19:59:17.483 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.3620000000000004e-05 2023-09-23T19:59:17.483 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.483 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.483 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:59:17.484 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.394493+0000", 2023-09-23T19:59:17.484 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.1310000000000002e-05 2023-09-23T19:59:17.484 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.484 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.484 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:59:17.484 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.394545+0000", 2023-09-23T19:59:17.484 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.32487358399999999 2023-09-23T19:59:17.484 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:59:17.484 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:59:17.484 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:59:17.484 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.484 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.485 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:85324 14.9 14:991f0ae7:::benchmark_data_smithi016_228083_object85323:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e263)", 2023-09-23T19:59:17.485 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:58:25.069847+0000", 2023-09-23T19:59:17.485 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 52.382454697, 2023-09-23T19:59:17.485 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.0295787919999999, 2023-09-23T19:59:17.485 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:59:17.485 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "reached pg", 2023-09-23T19:59:17.485 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:59:17.485 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T19:59:17.485 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T19:59:17.486 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 85324 2023-09-23T19:59:17.486 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.486 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:59:17.486 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.486 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:59:17.486 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.069847+0000", 2023-09-23T19:59:17.486 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:59:17.486 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.486 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.486 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:59:17.486 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.069847+0000", 2023-09-23T19:59:17.486 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999938 2023-09-23T19:59:17.487 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.487 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.487 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:59:17.487 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.069841+0000", 2023-09-23T19:59:17.487 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.8390000000000002e-05 2023-09-23T19:59:17.487 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.487 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.487 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:59:17.487 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.069900+0000", 2023-09-23T19:59:17.487 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.054e-05 2023-09-23T19:59:17.487 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.487 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.488 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:59:17.488 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.069920+0000", 2023-09-23T19:59:17.488 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.7443e-05 2023-09-23T19:59:17.488 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.488 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.488 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:59:17.488 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.069958+0000", 2023-09-23T19:59:17.488 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.4419999999999995e-05 2023-09-23T19:59:17.488 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.488 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.488 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:59:17.488 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.070042+0000", 2023-09-23T19:59:17.489 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.9209999999999992e-06 2023-09-23T19:59:17.489 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.489 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.489 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op must wait for map", 2023-09-23T19:59:17.489 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.070050+0000", 2023-09-23T19:59:17.489 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0081624030000000004 2023-09-23T19:59:17.489 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.489 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.489 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:59:17.489 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.078212+0000", 2023-09-23T19:59:17.489 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.3901e-05 2023-09-23T19:59:17.489 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.490 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.490 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for peered", 2023-09-23T19:59:17.490 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.078236+0000", 2023-09-23T19:59:17.490 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.0194893199999999 2023-09-23T19:59:17.490 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.490 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.490 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:59:17.490 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.097726+0000", 2023-09-23T19:59:17.490 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0017004839999999999 2023-09-23T19:59:17.490 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.490 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.490 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:59:17.491 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.099426+0000", 2023-09-23T19:59:17.491 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.0295787919999999 2023-09-23T19:59:17.491 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:59:17.491 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:59:17.491 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:59:17.491 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.491 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.491 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:85343 14.9 14:97ef8a96:::benchmark_data_smithi016_228083_object85342:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e263)", 2023-09-23T19:59:17.491 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:58:25.081329+0000", 2023-09-23T19:59:17.491 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 52.370973329999998, 2023-09-23T19:59:17.491 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.0315053750000001, 2023-09-23T19:59:17.491 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:59:17.491 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:17.492 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:59:17.492 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T19:59:17.492 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T19:59:17.492 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 85343 2023-09-23T19:59:17.492 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.492 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:59:17.492 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.492 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:59:17.492 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.081329+0000", 2023-09-23T19:59:17.492 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:59:17.492 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.492 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.493 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:59:17.493 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.081329+0000", 2023-09-23T19:59:17.493 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T19:59:17.493 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.493 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.493 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:59:17.493 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.081327+0000", 2023-09-23T19:59:17.493 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.6828e-05 2023-09-23T19:59:17.493 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.493 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.493 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:59:17.493 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.081364+0000", 2023-09-23T19:59:17.493 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.0944000000000001e-05 2023-09-23T19:59:17.494 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.494 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.494 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:59:17.494 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.081375+0000", 2023-09-23T19:59:17.494 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.0110000000000003e-06 2023-09-23T19:59:17.494 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.494 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.494 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:59:17.494 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.081382+0000", 2023-09-23T19:59:17.494 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00090449200000000001 2023-09-23T19:59:17.494 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.494 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.495 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:59:17.495 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.082286+0000", 2023-09-23T19:59:17.495 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.426e-06 2023-09-23T19:59:17.495 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.495 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.495 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for peered", 2023-09-23T19:59:17.495 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.082292+0000", 2023-09-23T19:59:17.495 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.017513431 2023-09-23T19:59:17.495 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.495 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.495 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:59:17.495 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.099806+0000", 2023-09-23T19:59:17.496 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00047541599999999999 2023-09-23T19:59:17.496 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.496 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.496 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:59:17.496 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.100281+0000", 2023-09-23T19:59:17.496 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000148267 2023-09-23T19:59:17.496 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.496 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.496 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3,5", 2023-09-23T19:59:17.496 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.100430+0000", 2023-09-23T19:59:17.496 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.012105301000000001 2023-09-23T19:59:17.496 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.496 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.497 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:17.497 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.112535+0000", 2023-09-23T19:59:17.497 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.6626000000000003e-05 2023-09-23T19:59:17.497 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.497 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.497 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:59:17.497 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.112592+0000", 2023-09-23T19:59:17.497 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.7176999999999996e-05 2023-09-23T19:59:17.497 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.497 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.497 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:17.497 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.112679+0000", 2023-09-23T19:59:17.498 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.7911999999999994e-05 2023-09-23T19:59:17.498 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.498 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.498 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:59:17.498 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.112767+0000", 2023-09-23T19:59:17.498 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.7491999999999997e-05 2023-09-23T19:59:17.498 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.498 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.498 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:59:17.498 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.112834+0000", 2023-09-23T19:59:17.498 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.0315053750000001 2023-09-23T19:59:17.498 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:59:17.499 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:59:17.499 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:59:17.499 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.499 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.499 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:85385 14.9 14:9fce2e80:::benchmark_data_smithi016_228083_object85384:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e263)", 2023-09-23T19:59:17.499 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:58:25.131141+0000", 2023-09-23T19:59:17.499 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 52.321160859000003, 2023-09-23T19:59:17.499 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.98198450999999998, 2023-09-23T19:59:17.499 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:59:17.499 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:17.499 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:59:17.499 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T19:59:17.499 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T19:59:17.500 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 85385 2023-09-23T19:59:17.500 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.500 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:59:17.500 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.500 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:59:17.500 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.131141+0000", 2023-09-23T19:59:17.500 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:59:17.500 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.500 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.500 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:59:17.500 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.131141+0000", 2023-09-23T19:59:17.500 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T19:59:17.501 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.501 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.501 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:59:17.501 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.131140+0000", 2023-09-23T19:59:17.501 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.524e-05 2023-09-23T19:59:17.501 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.501 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.501 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:59:17.501 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.131195+0000", 2023-09-23T19:59:17.501 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.3030000000000001e-05 2023-09-23T19:59:17.501 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.501 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.501 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:59:17.502 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.131208+0000", 2023-09-23T19:59:17.502 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.4240000000000001e-06 2023-09-23T19:59:17.502 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.502 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.502 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:59:17.502 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.131213+0000", 2023-09-23T19:59:17.502 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.2309000000000003e-05 2023-09-23T19:59:17.502 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.502 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.502 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:59:17.502 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.131276+0000", 2023-09-23T19:59:17.503 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.0212e-05 2023-09-23T19:59:17.503 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.503 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.503 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for peered", 2023-09-23T19:59:17.503 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.131286+0000", 2023-09-23T19:59:17.503 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.97170411499999998 2023-09-23T19:59:17.503 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.503 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.503 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:59:17.503 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.102990+0000", 2023-09-23T19:59:17.503 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.8024999999999997e-05 2023-09-23T19:59:17.503 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.504 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.504 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:59:17.504 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.103088+0000", 2023-09-23T19:59:17.504 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000121412 2023-09-23T19:59:17.504 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.504 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.504 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3,5", 2023-09-23T19:59:17.504 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.103210+0000", 2023-09-23T19:59:17.504 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0093902440000000007 2023-09-23T19:59:17.504 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.504 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.504 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:59:17.504 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.112600+0000", 2023-09-23T19:59:17.505 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00031716699999999998 2023-09-23T19:59:17.505 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.505 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.505 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:17.505 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.112917+0000", 2023-09-23T19:59:17.505 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.1812999999999998e-05 2023-09-23T19:59:17.505 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.505 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.505 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:17.505 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.112989+0000", 2023-09-23T19:59:17.505 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.3490000000000003e-05 2023-09-23T19:59:17.505 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.506 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.506 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:59:17.506 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.113062+0000", 2023-09-23T19:59:17.506 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.3431999999999998e-05 2023-09-23T19:59:17.506 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.506 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.506 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:59:17.506 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.113126+0000", 2023-09-23T19:59:17.506 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.98198450999999998 2023-09-23T19:59:17.506 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:59:17.506 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:59:17.506 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:59:17.507 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.507 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.507 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:85401 14.9 14:9e19646f:::benchmark_data_smithi016_228083_object85400:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e263)", 2023-09-23T19:59:17.507 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:58:25.160076+0000", 2023-09-23T19:59:17.507 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 52.292226464000002, 2023-09-23T19:59:17.507 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.95347298599999997, 2023-09-23T19:59:17.507 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:59:17.507 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:17.507 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:59:17.507 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T19:59:17.507 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T19:59:17.507 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 85401 2023-09-23T19:59:17.507 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.508 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:59:17.508 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.508 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:59:17.508 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.160076+0000", 2023-09-23T19:59:17.508 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:59:17.508 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.508 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.508 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:59:17.508 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.160076+0000", 2023-09-23T19:59:17.508 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T19:59:17.508 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.508 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.509 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:59:17.509 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.160074+0000", 2023-09-23T19:59:17.509 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.0092999999999997e-05 2023-09-23T19:59:17.509 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.509 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.509 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:59:17.509 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.160114+0000", 2023-09-23T19:59:17.509 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.9205e-05 2023-09-23T19:59:17.509 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.509 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.509 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:59:17.509 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.160133+0000", 2023-09-23T19:59:17.509 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.3440000000000001e-06 2023-09-23T19:59:17.510 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.510 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.510 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:59:17.510 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.160141+0000", 2023-09-23T19:59:17.510 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.8044e-05 2023-09-23T19:59:17.510 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.510 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.510 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:59:17.510 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.160219+0000", 2023-09-23T19:59:17.510 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.4097000000000001e-05 2023-09-23T19:59:17.510 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.510 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.511 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for peered", 2023-09-23T19:59:17.511 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.160233+0000", 2023-09-23T19:59:17.511 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.945008081 2023-09-23T19:59:17.511 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.511 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.511 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:59:17.511 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.105242+0000", 2023-09-23T19:59:17.511 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00018872800000000001 2023-09-23T19:59:17.511 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.511 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.511 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:59:17.511 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.105430+0000", 2023-09-23T19:59:17.512 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00027045500000000001 2023-09-23T19:59:17.512 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.512 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.512 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3,5", 2023-09-23T19:59:17.512 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.105701+0000", 2023-09-23T19:59:17.512 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0072222060000000001 2023-09-23T19:59:17.512 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.512 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.512 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:59:17.512 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.112923+0000", 2023-09-23T19:59:17.512 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00026887200000000002 2023-09-23T19:59:17.512 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.513 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.513 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:17.513 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.113192+0000", 2023-09-23T19:59:17.513 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.7439000000000002e-05 2023-09-23T19:59:17.513 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.513 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.513 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:17.513 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.113269+0000", 2023-09-23T19:59:17.513 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000166601 2023-09-23T19:59:17.513 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.513 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.513 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:59:17.514 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.113436+0000", 2023-09-23T19:59:17.514 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000112722 2023-09-23T19:59:17.514 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.514 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.514 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:59:17.514 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.113549+0000", 2023-09-23T19:59:17.514 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.95347298599999997 2023-09-23T19:59:17.514 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:59:17.514 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:59:17.514 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:59:17.514 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.514 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.515 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:85402 14.9 14:99e5f598:::benchmark_data_smithi016_228083_object85401:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e263)", 2023-09-23T19:59:17.515 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:58:25.162001+0000", 2023-09-23T19:59:17.515 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 52.290301284000002, 2023-09-23T19:59:17.515 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.95197446799999996, 2023-09-23T19:59:17.515 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:59:17.515 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:17.515 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:59:17.515 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T19:59:17.515 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T19:59:17.515 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 85402 2023-09-23T19:59:17.515 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.515 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:59:17.516 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.516 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:59:17.516 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.162001+0000", 2023-09-23T19:59:17.516 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:59:17.516 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.516 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.516 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:59:17.516 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.162001+0000", 2023-09-23T19:59:17.516 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999976 2023-09-23T19:59:17.516 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.516 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.516 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:59:17.517 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.161998+0000", 2023-09-23T19:59:17.517 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.6414999999999993e-05 2023-09-23T19:59:17.517 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.517 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.517 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:59:17.517 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.162065+0000", 2023-09-23T19:59:17.517 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.9888e-05 2023-09-23T19:59:17.517 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.517 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.517 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:59:17.517 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.162085+0000", 2023-09-23T19:59:17.517 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.5710000000000004e-06 2023-09-23T19:59:17.518 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.518 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.518 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:59:17.518 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.162094+0000", 2023-09-23T19:59:17.518 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.2071999999999997e-05 2023-09-23T19:59:17.518 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.518 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.518 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:59:17.518 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.162166+0000", 2023-09-23T19:59:17.518 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.2556e-05 2023-09-23T19:59:17.518 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.518 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.519 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for peered", 2023-09-23T19:59:17.519 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.162179+0000", 2023-09-23T19:59:17.519 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.94696288500000003 2023-09-23T19:59:17.519 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.519 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.519 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:59:17.519 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.109142+0000", 2023-09-23T19:59:17.519 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00022589299999999999 2023-09-23T19:59:17.519 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.519 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.519 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:59:17.519 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.109368+0000", 2023-09-23T19:59:17.520 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000229898 2023-09-23T19:59:17.520 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.520 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.520 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3,5", 2023-09-23T19:59:17.520 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.109598+0000", 2023-09-23T19:59:17.520 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0033289349999999999 2023-09-23T19:59:17.520 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.520 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.520 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:59:17.520 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.112927+0000", 2023-09-23T19:59:17.520 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00071869200000000005 2023-09-23T19:59:17.520 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.520 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.521 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:17.521 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.113645+0000", 2023-09-23T19:59:17.521 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000106327 2023-09-23T19:59:17.521 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.521 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.521 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:17.521 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.113752+0000", 2023-09-23T19:59:17.521 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00012709 2023-09-23T19:59:17.521 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.521 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.521 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:59:17.521 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.113879+0000", 2023-09-23T19:59:17.522 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.6589000000000006e-05 2023-09-23T19:59:17.522 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.522 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.522 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:59:17.522 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.113975+0000", 2023-09-23T19:59:17.522 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.95197446799999996 2023-09-23T19:59:17.522 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:59:17.522 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:59:17.522 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:59:17.522 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.522 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.522 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:85415 14.9 14:9ffb8655:::benchmark_data_smithi016_228083_object85414:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e263)", 2023-09-23T19:59:17.523 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:58:25.175093+0000", 2023-09-23T19:59:17.523 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 52.277209345000003, 2023-09-23T19:59:17.523 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.96229291100000003, 2023-09-23T19:59:17.523 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:59:17.523 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:17.523 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:59:17.523 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T19:59:17.523 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T19:59:17.523 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 85415 2023-09-23T19:59:17.523 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.523 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:59:17.523 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.523 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:59:17.524 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.175093+0000", 2023-09-23T19:59:17.524 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:59:17.524 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.524 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.524 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:59:17.524 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.175093+0000", 2023-09-23T19:59:17.524 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999976 2023-09-23T19:59:17.524 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.524 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.524 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:59:17.524 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.175091+0000", 2023-09-23T19:59:17.524 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000123707 2023-09-23T19:59:17.525 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.525 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.525 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:59:17.525 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.175214+0000", 2023-09-23T19:59:17.525 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.0718999999999999e-05 2023-09-23T19:59:17.525 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.525 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.525 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:59:17.525 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.175235+0000", 2023-09-23T19:59:17.525 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.2517000000000001e-05 2023-09-23T19:59:17.525 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.525 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.525 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:59:17.526 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.175247+0000", 2023-09-23T19:59:17.526 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.6070999999999996e-05 2023-09-23T19:59:17.526 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.526 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.526 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:59:17.526 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.175334+0000", 2023-09-23T19:59:17.526 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.5437000000000001e-05 2023-09-23T19:59:17.526 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.526 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.526 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for peered", 2023-09-23T19:59:17.526 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.175349+0000", 2023-09-23T19:59:17.527 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.93866744700000004 2023-09-23T19:59:17.527 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.527 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.527 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:59:17.527 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.114016+0000", 2023-09-23T19:59:17.527 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000222554 2023-09-23T19:59:17.527 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.527 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.527 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:59:17.527 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.114239+0000", 2023-09-23T19:59:17.528 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00034813200000000002 2023-09-23T19:59:17.528 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.528 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.528 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3,5", 2023-09-23T19:59:17.528 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.114587+0000", 2023-09-23T19:59:17.528 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0060094249999999997 2023-09-23T19:59:17.528 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.528 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.528 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:17.528 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.120597+0000", 2023-09-23T19:59:17.528 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0069605939999999996 2023-09-23T19:59:17.528 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.529 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.529 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:59:17.529 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.127557+0000", 2023-09-23T19:59:17.529 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0097405860000000007 2023-09-23T19:59:17.529 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.529 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.529 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:17.529 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.137298+0000", 2023-09-23T19:59:17.529 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.1941000000000003e-05 2023-09-23T19:59:17.529 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.530 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.530 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:59:17.530 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.137350+0000", 2023-09-23T19:59:17.530 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.5941e-05 2023-09-23T19:59:17.530 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.530 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.530 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:59:17.530 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.137386+0000", 2023-09-23T19:59:17.530 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.96229291100000003 2023-09-23T19:59:17.530 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:59:17.530 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:59:17.530 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:59:17.531 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.531 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.531 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:85421 14.9 14:91e9f1e1:::benchmark_data_smithi016_228083_object85420:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e263)", 2023-09-23T19:59:17.531 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:58:25.183117+0000", 2023-09-23T19:59:17.531 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 52.269185071999999, 2023-09-23T19:59:17.531 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.954422993, 2023-09-23T19:59:17.531 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:59:17.531 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:17.531 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:59:17.531 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T19:59:17.532 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T19:59:17.532 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 85421 2023-09-23T19:59:17.532 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.532 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:59:17.532 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.532 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:59:17.532 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.183117+0000", 2023-09-23T19:59:17.532 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:59:17.532 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.532 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.532 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:59:17.532 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.183117+0000", 2023-09-23T19:59:17.533 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.999999 2023-09-23T19:59:17.533 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.533 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.533 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:59:17.533 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.183116+0000", 2023-09-23T19:59:17.533 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.1011000000000002e-05 2023-09-23T19:59:17.533 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.533 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.533 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:59:17.533 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.183187+0000", 2023-09-23T19:59:17.533 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.235e-05 2023-09-23T19:59:17.533 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.534 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.534 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:59:17.534 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.183199+0000", 2023-09-23T19:59:17.534 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.17e-06 2023-09-23T19:59:17.534 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.534 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.534 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:59:17.534 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.183206+0000", 2023-09-23T19:59:17.534 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.0999000000000003e-05 2023-09-23T19:59:17.534 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.534 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.534 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:59:17.535 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.183267+0000", 2023-09-23T19:59:17.535 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.7880000000000001e-06 2023-09-23T19:59:17.535 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.535 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.535 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for peered", 2023-09-23T19:59:17.535 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.183277+0000", 2023-09-23T19:59:17.535 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.93473318400000005 2023-09-23T19:59:17.535 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.535 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.535 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:59:17.535 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.118010+0000", 2023-09-23T19:59:17.535 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00018186100000000001 2023-09-23T19:59:17.536 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.536 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.536 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:59:17.536 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.118192+0000", 2023-09-23T19:59:17.536 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00013684600000000001 2023-09-23T19:59:17.536 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.536 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.536 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3,5", 2023-09-23T19:59:17.536 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.118329+0000", 2023-09-23T19:59:17.536 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.012081743000000001 2023-09-23T19:59:17.536 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.536 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.536 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:17.537 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.130411+0000", 2023-09-23T19:59:17.537 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.4800000000000007e-06 2023-09-23T19:59:17.537 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.537 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.537 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:59:17.537 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.130420+0000", 2023-09-23T19:59:17.537 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0070648070000000002 2023-09-23T19:59:17.537 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.537 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.537 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:17.537 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.137485+0000", 2023-09-23T19:59:17.537 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.9612e-05 2023-09-23T19:59:17.538 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.538 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.538 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:59:17.538 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.137515+0000", 2023-09-23T19:59:17.538 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.5318000000000001e-05 2023-09-23T19:59:17.538 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.538 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.538 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:59:17.538 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.137540+0000", 2023-09-23T19:59:17.538 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.954422993 2023-09-23T19:59:17.538 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:59:17.539 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:59:17.539 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:59:17.539 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.539 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.539 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:85441 14.9 14:9c17bf8c:::benchmark_data_smithi016_228083_object85440:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e263)", 2023-09-23T19:59:17.539 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:58:25.209256+0000", 2023-09-23T19:59:17.539 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 52.243046204000002, 2023-09-23T19:59:17.539 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.94155818099999999, 2023-09-23T19:59:17.539 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:59:17.539 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:17.539 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:59:17.539 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T19:59:17.540 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T19:59:17.540 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 85441 2023-09-23T19:59:17.540 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.540 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:59:17.540 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.540 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:59:17.540 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.209256+0000", 2023-09-23T19:59:17.540 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:59:17.540 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.540 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.540 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:59:17.540 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.209256+0000", 2023-09-23T19:59:17.541 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T19:59:17.541 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.541 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.541 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:59:17.541 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.209254+0000", 2023-09-23T19:59:17.541 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.1990999999999996e-05 2023-09-23T19:59:17.541 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.541 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.541 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:59:17.541 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.209326+0000", 2023-09-23T19:59:17.541 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.3918e-05 2023-09-23T19:59:17.541 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.542 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.542 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:59:17.542 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.209340+0000", 2023-09-23T19:59:17.542 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.5520000000000001e-06 2023-09-23T19:59:17.542 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.542 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.542 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:59:17.542 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.209347+0000", 2023-09-23T19:59:17.542 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.1537000000000004e-05 2023-09-23T19:59:17.542 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.542 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.542 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:59:17.543 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.209408+0000", 2023-09-23T19:59:17.543 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.3186e-05 2023-09-23T19:59:17.543 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.543 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.543 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for peered", 2023-09-23T19:59:17.543 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.209422+0000", 2023-09-23T19:59:17.543 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.91123985900000004 2023-09-23T19:59:17.543 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.543 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.543 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:59:17.543 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.120661+0000", 2023-09-23T19:59:17.543 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00015342700000000001 2023-09-23T19:59:17.544 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.544 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.544 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:59:17.544 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.120815+0000", 2023-09-23T19:59:17.544 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000244866 2023-09-23T19:59:17.544 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.544 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.544 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3,5", 2023-09-23T19:59:17.544 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.121060+0000", 2023-09-23T19:59:17.544 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0093679850000000005 2023-09-23T19:59:17.544 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.544 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.545 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:59:17.545 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.130428+0000", 2023-09-23T19:59:17.545 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.4576999999999997e-05 2023-09-23T19:59:17.545 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.545 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.545 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:17.545 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.130472+0000", 2023-09-23T19:59:17.545 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.020209061 2023-09-23T19:59:17.545 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.545 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.545 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:17.545 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.150681+0000", 2023-09-23T19:59:17.545 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.9345000000000006e-05 2023-09-23T19:59:17.546 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.546 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.546 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:59:17.546 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.150771+0000", 2023-09-23T19:59:17.546 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.3334999999999998e-05 2023-09-23T19:59:17.546 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.546 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.546 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:59:17.546 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.150814+0000", 2023-09-23T19:59:17.546 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.94155818099999999 2023-09-23T19:59:17.546 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:59:17.546 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:59:17.547 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:59:17.547 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.547 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.547 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:85442 14.9 14:9e459d4c:::benchmark_data_smithi016_228083_object85441:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e263)", 2023-09-23T19:59:17.547 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:58:25.209357+0000", 2023-09-23T19:59:17.547 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 52.242945462000002, 2023-09-23T19:59:17.547 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.94168814999999995, 2023-09-23T19:59:17.547 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:59:17.547 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:17.547 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:59:17.547 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T19:59:17.547 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T19:59:17.548 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 85442 2023-09-23T19:59:17.548 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.548 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:59:17.548 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.548 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:59:17.548 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.209357+0000", 2023-09-23T19:59:17.548 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:59:17.548 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.548 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.548 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:59:17.548 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.209357+0000", 2023-09-23T19:59:17.548 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999995 2023-09-23T19:59:17.549 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.549 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.549 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:59:17.549 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.209356+0000", 2023-09-23T19:59:17.549 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.3260999999999999e-05 2023-09-23T19:59:17.549 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.549 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.549 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:59:17.549 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.209389+0000", 2023-09-23T19:59:17.549 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.092e-06 2023-09-23T19:59:17.549 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.550 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.550 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:59:17.550 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.209396+0000", 2023-09-23T19:59:17.550 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.2390000000000002e-06 2023-09-23T19:59:17.550 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.550 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.550 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:59:17.550 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.209402+0000", 2023-09-23T19:59:17.550 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.3684999999999999e-05 2023-09-23T19:59:17.550 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.550 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.551 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:59:17.551 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.209445+0000", 2023-09-23T19:59:17.551 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.6279999999999996e-06 2023-09-23T19:59:17.551 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.551 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.551 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for peered", 2023-09-23T19:59:17.551 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.209453+0000", 2023-09-23T19:59:17.551 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.91469508200000005 2023-09-23T19:59:17.551 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.551 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.551 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:59:17.551 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.124148+0000", 2023-09-23T19:59:17.552 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00014875500000000001 2023-09-23T19:59:17.552 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.552 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.552 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:59:17.552 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.124297+0000", 2023-09-23T19:59:17.552 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000203897 2023-09-23T19:59:17.552 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.552 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.552 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3,5", 2023-09-23T19:59:17.552 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.124501+0000", 2023-09-23T19:59:17.552 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0060093730000000001 2023-09-23T19:59:17.552 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.553 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.553 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:17.553 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.130510+0000", 2023-09-23T19:59:17.553 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.2470000000000008e-06 2023-09-23T19:59:17.553 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.553 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.553 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:59:17.553 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.130518+0000", 2023-09-23T19:59:17.553 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.020432044 2023-09-23T19:59:17.553 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.553 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.553 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:17.553 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.150950+0000", 2023-09-23T19:59:17.554 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.5896000000000002e-05 2023-09-23T19:59:17.554 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.554 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.554 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:59:17.554 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.151006+0000", 2023-09-23T19:59:17.554 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.8624999999999997e-05 2023-09-23T19:59:17.554 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.554 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.554 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:59:17.554 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.151045+0000", 2023-09-23T19:59:17.554 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.94168814999999995 2023-09-23T19:59:17.555 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:59:17.555 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:59:17.555 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:59:17.555 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.555 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.555 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:85497 14.9 14:9cb615a6:::benchmark_data_smithi016_228083_object85496:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e263)", 2023-09-23T19:59:17.555 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:58:25.281257+0000", 2023-09-23T19:59:17.555 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 52.171044637999998, 2023-09-23T19:59:17.555 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.88141183400000001, 2023-09-23T19:59:17.555 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:59:17.555 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:17.555 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:59:17.556 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T19:59:17.556 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T19:59:17.556 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 85497 2023-09-23T19:59:17.556 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.556 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:59:17.556 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.556 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:59:17.556 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.281257+0000", 2023-09-23T19:59:17.556 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:59:17.556 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.556 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.556 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:59:17.556 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.281257+0000", 2023-09-23T19:59:17.557 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T19:59:17.557 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.557 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.557 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:59:17.557 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.281255+0000", 2023-09-23T19:59:17.557 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.4725000000000003e-05 2023-09-23T19:59:17.557 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.557 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.557 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:59:17.557 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.281300+0000", 2023-09-23T19:59:17.557 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.0341999999999999e-05 2023-09-23T19:59:17.557 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.558 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.558 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:59:17.558 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.281320+0000", 2023-09-23T19:59:17.558 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.6930000000000002e-06 2023-09-23T19:59:17.558 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.558 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.558 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:59:17.558 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.281330+0000", 2023-09-23T19:59:17.558 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.6923999999999998e-05 2023-09-23T19:59:17.558 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.558 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.558 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:59:17.559 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.281397+0000", 2023-09-23T19:59:17.559 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.5629999999999992e-06 2023-09-23T19:59:17.559 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.559 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.559 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for peered", 2023-09-23T19:59:17.559 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.281406+0000", 2023-09-23T19:59:17.559 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.84619651600000001 2023-09-23T19:59:17.559 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.559 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.559 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:59:17.559 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.127602+0000", 2023-09-23T19:59:17.559 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.2312999999999995e-05 2023-09-23T19:59:17.560 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.560 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.560 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:59:17.560 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.127694+0000", 2023-09-23T19:59:17.560 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00012638600000000001 2023-09-23T19:59:17.560 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.560 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.560 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3,5", 2023-09-23T19:59:17.560 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.127821+0000", 2023-09-23T19:59:17.560 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0094256010000000005 2023-09-23T19:59:17.560 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.560 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.561 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:17.561 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.137246+0000", 2023-09-23T19:59:17.561 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00014908599999999999 2023-09-23T19:59:17.561 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.561 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.561 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:59:17.561 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.137395+0000", 2023-09-23T19:59:17.561 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.025132723999999999 2023-09-23T19:59:17.561 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.561 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.561 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:17.562 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.162528+0000", 2023-09-23T19:59:17.562 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.1192000000000005e-05 2023-09-23T19:59:17.562 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.562 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.562 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:59:17.562 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.162609+0000", 2023-09-23T19:59:17.562 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.9874000000000003e-05 2023-09-23T19:59:17.562 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.562 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.562 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:59:17.562 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.162669+0000", 2023-09-23T19:59:17.562 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.88141183400000001 2023-09-23T19:59:17.562 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:59:17.563 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:59:17.563 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:59:17.563 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.563 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.563 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:85498 14.9 14:941b5f4a:::benchmark_data_smithi016_228083_object85497:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e263)", 2023-09-23T19:59:17.563 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:58:25.285826+0000", 2023-09-23T19:59:17.563 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 52.166476135000003, 2023-09-23T19:59:17.563 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.87697232700000005, 2023-09-23T19:59:17.563 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:59:17.563 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:17.563 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:59:17.563 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T19:59:17.564 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T19:59:17.564 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 85498 2023-09-23T19:59:17.564 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.564 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:59:17.564 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.564 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:59:17.564 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.285826+0000", 2023-09-23T19:59:17.564 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:59:17.564 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.564 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.564 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:59:17.564 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.285826+0000", 2023-09-23T19:59:17.565 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999948 2023-09-23T19:59:17.565 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.565 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.565 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:59:17.565 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.285821+0000", 2023-09-23T19:59:17.565 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00010835 2023-09-23T19:59:17.565 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.565 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.565 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:59:17.565 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.285929+0000", 2023-09-23T19:59:17.565 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.4283000000000001e-05 2023-09-23T19:59:17.565 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.565 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.566 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:59:17.566 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.285953+0000", 2023-09-23T19:59:17.566 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.1270000000000001e-05 2023-09-23T19:59:17.566 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.566 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.566 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:59:17.566 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.285964+0000", 2023-09-23T19:59:17.566 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00012742600000000001 2023-09-23T19:59:17.566 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.566 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.566 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:59:17.566 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.286092+0000", 2023-09-23T19:59:17.567 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.6348999999999999e-05 2023-09-23T19:59:17.567 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.567 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.567 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for peered", 2023-09-23T19:59:17.567 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.286108+0000", 2023-09-23T19:59:17.567 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.844440776 2023-09-23T19:59:17.567 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.567 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.567 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:59:17.567 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.130549+0000", 2023-09-23T19:59:17.567 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000102414 2023-09-23T19:59:17.568 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.568 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.568 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:59:17.568 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.130651+0000", 2023-09-23T19:59:17.568 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000111272 2023-09-23T19:59:17.568 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.568 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.568 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3,5", 2023-09-23T19:59:17.568 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.130763+0000", 2023-09-23T19:59:17.568 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0066374440000000002 2023-09-23T19:59:17.568 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.568 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.569 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:59:17.569 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.137400+0000", 2023-09-23T19:59:17.569 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.7243999999999998e-05 2023-09-23T19:59:17.569 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.569 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.569 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:17.569 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.137437+0000", 2023-09-23T19:59:17.569 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.025275355999999999 2023-09-23T19:59:17.569 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.569 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.569 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:17.569 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.162713+0000", 2023-09-23T19:59:17.570 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.1664e-05 2023-09-23T19:59:17.570 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.570 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.570 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:59:17.570 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.162754+0000", 2023-09-23T19:59:17.570 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.3869000000000003e-05 2023-09-23T19:59:17.570 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.570 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.570 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:59:17.570 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.162798+0000", 2023-09-23T19:59:17.570 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.87697232700000005 2023-09-23T19:59:17.571 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:59:17.571 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:59:17.571 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:59:17.571 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.571 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.571 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:85525 14.9 14:9de208c1:::benchmark_data_smithi016_228083_object85524:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e263)", 2023-09-23T19:59:17.571 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:58:25.332416+0000", 2023-09-23T19:59:17.571 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 52.119885674000002, 2023-09-23T19:59:17.571 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.83059830099999998, 2023-09-23T19:59:17.571 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:59:17.571 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:17.571 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:59:17.572 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T19:59:17.572 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T19:59:17.572 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 85525 2023-09-23T19:59:17.572 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.572 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:59:17.572 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.572 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:59:17.572 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.332416+0000", 2023-09-23T19:59:17.572 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:59:17.572 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.572 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.572 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:59:17.573 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.332416+0000", 2023-09-23T19:59:17.573 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T19:59:17.573 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.573 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.573 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:59:17.573 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.332414+0000", 2023-09-23T19:59:17.573 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.3407e-05 2023-09-23T19:59:17.573 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.573 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.573 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:59:17.573 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.332458+0000", 2023-09-23T19:59:17.573 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.7921e-05 2023-09-23T19:59:17.574 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.574 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.574 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:59:17.574 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.332476+0000", 2023-09-23T19:59:17.574 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.2570000000000007e-06 2023-09-23T19:59:17.574 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.574 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.574 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:59:17.574 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.332484+0000", 2023-09-23T19:59:17.574 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.5447999999999999e-05 2023-09-23T19:59:17.574 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.574 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.575 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:59:17.575 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.332569+0000", 2023-09-23T19:59:17.575 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.2368000000000001e-05 2023-09-23T19:59:17.575 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.575 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.575 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for peered", 2023-09-23T19:59:17.575 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.332582+0000", 2023-09-23T19:59:17.575 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.80044044299999995 2023-09-23T19:59:17.575 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.575 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.575 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:59:17.575 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.133022+0000", 2023-09-23T19:59:17.576 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.2608999999999994e-05 2023-09-23T19:59:17.576 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.576 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.576 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:59:17.576 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.133115+0000", 2023-09-23T19:59:17.576 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00012925100000000001 2023-09-23T19:59:17.576 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.576 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.576 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3,5", 2023-09-23T19:59:17.576 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.133244+0000", 2023-09-23T19:59:17.576 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0043033309999999996 2023-09-23T19:59:17.576 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.576 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.577 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:59:17.577 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.137547+0000", 2023-09-23T19:59:17.577 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0058089120000000003 2023-09-23T19:59:17.577 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.577 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.577 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:17.577 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.143356+0000", 2023-09-23T19:59:17.577 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.019509450000000001 2023-09-23T19:59:17.577 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.577 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.577 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:17.577 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.162866+0000", 2023-09-23T19:59:17.578 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.0210999999999996e-05 2023-09-23T19:59:17.578 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.578 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.578 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:59:17.578 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.162936+0000", 2023-09-23T19:59:17.578 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.8739999999999995e-05 2023-09-23T19:59:17.578 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.578 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.578 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:59:17.578 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.163015+0000", 2023-09-23T19:59:17.578 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.83059830099999998 2023-09-23T19:59:17.579 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:59:17.579 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:59:17.579 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:59:17.579 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.579 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.579 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:85534 14.9 14:9a53a49a:::benchmark_data_smithi016_228083_object85533:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e263)", 2023-09-23T19:59:17.579 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:58:25.353246+0000", 2023-09-23T19:59:17.579 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 52.099055735, 2023-09-23T19:59:17.579 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.80997894299999995, 2023-09-23T19:59:17.579 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:59:17.579 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:17.579 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:59:17.580 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T19:59:17.580 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T19:59:17.580 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 85534 2023-09-23T19:59:17.580 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.580 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:59:17.580 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.580 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:59:17.580 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.353246+0000", 2023-09-23T19:59:17.580 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:59:17.580 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.580 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.580 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:59:17.581 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.353246+0000", 2023-09-23T19:59:17.581 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T19:59:17.581 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.581 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.581 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:59:17.581 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.353244+0000", 2023-09-23T19:59:17.581 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.6495000000000002e-05 2023-09-23T19:59:17.581 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.581 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.581 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:59:17.581 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.353281+0000", 2023-09-23T19:59:17.581 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.9864000000000002e-05 2023-09-23T19:59:17.582 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.582 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.582 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:59:17.582 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.353301+0000", 2023-09-23T19:59:17.582 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.7470000000000005e-06 2023-09-23T19:59:17.582 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.582 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.582 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:59:17.582 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.353308+0000", 2023-09-23T19:59:17.582 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00014357199999999999 2023-09-23T19:59:17.582 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.582 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.583 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:59:17.583 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.353452+0000", 2023-09-23T19:59:17.583 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.1129e-05 2023-09-23T19:59:17.583 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.583 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.583 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for peered", 2023-09-23T19:59:17.583 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.353473+0000", 2023-09-23T19:59:17.583 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.78173484299999996 2023-09-23T19:59:17.583 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.583 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.583 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:59:17.583 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.135208+0000", 2023-09-23T19:59:17.584 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.6490999999999996e-05 2023-09-23T19:59:17.584 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.584 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.584 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:59:17.584 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.135294+0000", 2023-09-23T19:59:17.584 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00013405099999999999 2023-09-23T19:59:17.584 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.584 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.584 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3,5", 2023-09-23T19:59:17.584 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.135428+0000", 2023-09-23T19:59:17.584 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.002123748 2023-09-23T19:59:17.585 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.585 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.585 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:59:17.585 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.137552+0000", 2023-09-23T19:59:17.585 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.012916637999999999 2023-09-23T19:59:17.585 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.585 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.585 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:17.585 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.150469+0000", 2023-09-23T19:59:17.585 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.012647726 2023-09-23T19:59:17.585 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.586 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.586 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:17.586 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.163117+0000", 2023-09-23T19:59:17.586 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.3090999999999996e-05 2023-09-23T19:59:17.586 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.586 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.586 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:59:17.586 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.163180+0000", 2023-09-23T19:59:17.586 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.5578000000000003e-05 2023-09-23T19:59:17.586 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.586 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.587 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:59:17.587 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.163225+0000", 2023-09-23T19:59:17.587 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.80997894299999995 2023-09-23T19:59:17.587 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:59:17.587 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:59:17.587 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:59:17.587 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.587 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.587 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:85537 14.9 14:9e1b9eb0:::benchmark_data_smithi016_228083_object85536:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e263)", 2023-09-23T19:59:17.587 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:58:25.362809+0000", 2023-09-23T19:59:17.587 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 52.089492628999999, 2023-09-23T19:59:17.587 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.80056966299999999, 2023-09-23T19:59:17.588 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:59:17.588 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:17.588 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:59:17.588 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T19:59:17.588 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T19:59:17.588 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 85537 2023-09-23T19:59:17.588 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.588 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:59:17.588 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.588 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:59:17.588 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.362809+0000", 2023-09-23T19:59:17.589 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:59:17.589 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.589 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.589 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:59:17.589 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.362809+0000", 2023-09-23T19:59:17.589 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T19:59:17.589 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.589 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.589 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:59:17.589 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.362808+0000", 2023-09-23T19:59:17.589 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.9162000000000002e-05 2023-09-23T19:59:17.589 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.590 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.590 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:59:17.590 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.362847+0000", 2023-09-23T19:59:17.590 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.7776999999999999e-05 2023-09-23T19:59:17.590 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.590 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.590 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:59:17.590 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.362865+0000", 2023-09-23T19:59:17.590 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.1179999999999992e-06 2023-09-23T19:59:17.590 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.591 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.591 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:59:17.591 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.362873+0000", 2023-09-23T19:59:17.591 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00014888999999999999 2023-09-23T19:59:17.591 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.591 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.591 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:59:17.591 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.363022+0000", 2023-09-23T19:59:17.591 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.7234000000000001e-05 2023-09-23T19:59:17.591 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.591 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.592 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for peered", 2023-09-23T19:59:17.592 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.363039+0000", 2023-09-23T19:59:17.592 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.77452688599999997 2023-09-23T19:59:17.592 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.592 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.592 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:59:17.592 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.137566+0000", 2023-09-23T19:59:17.592 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.9876999999999993e-05 2023-09-23T19:59:17.592 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.593 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.593 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:59:17.593 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.137656+0000", 2023-09-23T19:59:17.593 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00014732300000000001 2023-09-23T19:59:17.593 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.593 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.593 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3,5", 2023-09-23T19:59:17.593 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.137803+0000", 2023-09-23T19:59:17.593 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0055677629999999999 2023-09-23T19:59:17.593 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.593 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.594 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:59:17.594 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.143371+0000", 2023-09-23T19:59:17.594 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0072004060000000003 2023-09-23T19:59:17.594 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.594 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.594 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:17.594 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.150571+0000", 2023-09-23T19:59:17.594 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0127008 2023-09-23T19:59:17.594 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.594 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.595 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:17.595 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.163272+0000", 2023-09-23T19:59:17.595 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.4979999999999999e-05 2023-09-23T19:59:17.595 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.595 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.595 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:59:17.595 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.163327+0000", 2023-09-23T19:59:17.595 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.1836000000000003e-05 2023-09-23T19:59:17.595 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.595 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.595 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:59:17.596 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.163379+0000", 2023-09-23T19:59:17.596 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.80056966299999999 2023-09-23T19:59:17.596 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:59:17.596 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:59:17.596 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:59:17.596 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.596 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.596 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:85542 14.9 14:9a9e0629:::benchmark_data_smithi016_228083_object85541:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e263)", 2023-09-23T19:59:17.596 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:58:25.378606+0000", 2023-09-23T19:59:17.596 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 52.073696009999999, 2023-09-23T19:59:17.597 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.78504788400000003, 2023-09-23T19:59:17.597 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:59:17.597 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:17.597 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:59:17.597 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T19:59:17.597 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T19:59:17.597 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 85542 2023-09-23T19:59:17.597 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.597 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:59:17.598 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.598 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:59:17.598 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.378606+0000", 2023-09-23T19:59:17.598 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:59:17.598 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.598 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.598 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:59:17.598 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.378606+0000", 2023-09-23T19:59:17.599 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999971 2023-09-23T19:59:17.599 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.599 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.599 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:59:17.599 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.378603+0000", 2023-09-23T19:59:17.599 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.7144e-05 2023-09-23T19:59:17.599 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.599 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.599 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:59:17.599 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.378660+0000", 2023-09-23T19:59:17.599 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.9719000000000001e-05 2023-09-23T19:59:17.600 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.600 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.600 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:59:17.600 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.378690+0000", 2023-09-23T19:59:17.600 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.2594e-05 2023-09-23T19:59:17.600 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.600 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.600 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:59:17.600 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.378703+0000", 2023-09-23T19:59:17.601 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00013042800000000001 2023-09-23T19:59:17.601 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.601 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.601 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:59:17.601 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.378833+0000", 2023-09-23T19:59:17.601 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.9785000000000001e-05 2023-09-23T19:59:17.601 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.601 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.601 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for peered", 2023-09-23T19:59:17.602 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.378853+0000", 2023-09-23T19:59:17.602 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.76109633899999996 2023-09-23T19:59:17.602 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.602 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.602 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:59:17.602 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.139949+0000", 2023-09-23T19:59:17.602 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000213799 2023-09-23T19:59:17.602 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.602 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.603 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:59:17.603 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.140163+0000", 2023-09-23T19:59:17.603 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00030738800000000002 2023-09-23T19:59:17.603 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.603 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.603 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3,5", 2023-09-23T19:59:17.603 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.140470+0000", 2023-09-23T19:59:17.603 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.010119651 2023-09-23T19:59:17.604 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.604 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.604 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:59:17.604 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.150590+0000", 2023-09-23T19:59:17.604 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00028541800000000001 2023-09-23T19:59:17.604 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.604 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.604 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:17.604 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.150875+0000", 2023-09-23T19:59:17.605 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.012635431000000001 2023-09-23T19:59:17.605 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.605 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.605 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:17.605 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.163511+0000", 2023-09-23T19:59:17.605 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.7058999999999996e-05 2023-09-23T19:59:17.605 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.605 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.605 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:59:17.605 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.163578+0000", 2023-09-23T19:59:17.605 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.5988999999999994e-05 2023-09-23T19:59:17.605 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.606 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.606 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:59:17.606 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.163654+0000", 2023-09-23T19:59:17.606 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.78504788400000003 2023-09-23T19:59:17.606 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:59:17.606 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:59:17.606 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:59:17.606 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.606 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.606 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:85559 14.9 14:923b20ba:::benchmark_data_smithi016_228083_object85558:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e263)", 2023-09-23T19:59:17.606 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:58:25.463564+0000", 2023-09-23T19:59:17.607 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 51.988738294000001, 2023-09-23T19:59:17.607 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.70025278099999999, 2023-09-23T19:59:17.607 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:59:17.607 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:17.607 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T19:59:17.607 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T19:59:17.607 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T19:59:17.607 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 85559 2023-09-23T19:59:17.607 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.607 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:59:17.607 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.608 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:59:17.608 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.463564+0000", 2023-09-23T19:59:17.608 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:59:17.608 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.608 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.608 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:59:17.608 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.463564+0000", 2023-09-23T19:59:17.608 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999967 2023-09-23T19:59:17.608 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.608 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.608 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:59:17.608 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.463561+0000", 2023-09-23T19:59:17.609 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.0883999999999994e-05 2023-09-23T19:59:17.609 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.609 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.609 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:59:17.609 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.463631+0000", 2023-09-23T19:59:17.609 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.2996000000000001e-05 2023-09-23T19:59:17.609 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.609 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.609 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:59:17.609 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.463664+0000", 2023-09-23T19:59:17.609 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.0069999999999999e-05 2023-09-23T19:59:17.609 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.610 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.610 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:59:17.610 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.463684+0000", 2023-09-23T19:59:17.610 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00019521600000000001 2023-09-23T19:59:17.610 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.610 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.610 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:59:17.610 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.463880+0000", 2023-09-23T19:59:17.610 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.016e-05 2023-09-23T19:59:17.610 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.611 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.611 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for peered", 2023-09-23T19:59:17.611 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.463900+0000", 2023-09-23T19:59:17.611 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.67952702200000004 2023-09-23T19:59:17.611 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.611 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.611 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:59:17.611 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.143427+0000", 2023-09-23T19:59:17.611 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000119385 2023-09-23T19:59:17.611 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.611 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.612 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:59:17.612 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.143546+0000", 2023-09-23T19:59:17.612 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00021470500000000001 2023-09-23T19:59:17.612 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.612 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.612 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3,5", 2023-09-23T19:59:17.612 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.143761+0000", 2023-09-23T19:59:17.612 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0068445800000000003 2023-09-23T19:59:17.612 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.612 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.612 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T19:59:17.612 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.150606+0000", 2023-09-23T19:59:17.613 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00049718400000000002 2023-09-23T19:59:17.613 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.613 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.613 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:17.613 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.151103+0000", 2023-09-23T19:59:17.613 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.012612732999999999 2023-09-23T19:59:17.613 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.613 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.614 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:17.614 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.163715+0000", 2023-09-23T19:59:17.614 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.1959000000000002e-05 2023-09-23T19:59:17.614 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.614 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.614 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T19:59:17.614 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.163767+0000", 2023-09-23T19:59:17.614 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.9100000000000001e-05 2023-09-23T19:59:17.614 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:17.614 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:17.614 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T19:59:17.614 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.163817+0000", 2023-09-23T19:59:17.615 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.70025278099999999 2023-09-23T19:59:17.615 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:59:17.615 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:59:17.615 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:59:17.615 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:59:17.615 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:59:17.615 INFO:teuthology.orchestra.run.smithi092.stdout:} 2023-09-23T19:59:17.615 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:17.648 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:17.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:17.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:59:17.750 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:17.792 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:17.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:17.795 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:17.874 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is set 2023-09-23T19:59:17.896 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:17.903 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:17 smithi092 bash[58855]: cluster 2023-09-23T19:59:16.604961+0000 mgr.y (mgr.25325) 1561 : cluster 0 pgmap v1590: 89 pgs: 89 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 713 op/s 2023-09-23T19:59:17.903 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:17 smithi092 bash[58855]: audit 2023-09-23T19:59:17.764458+0000 mon.c (mon.2) 237 : audit 1 from='client.? 172.21.15.16:0/699779620' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:59:17.903 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:17 smithi092 bash[58855]: audit 2023-09-23T19:59:17.765135+0000 mon.a (mon.0) 1881 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:59:17.915 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:17.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:17.917 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:18.018 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:18.026 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:17 smithi016 bash[77995]: cluster 2023-09-23T19:59:16.604961+0000 mgr.y (mgr.25325) 1561 : cluster 0 pgmap v1590: 89 pgs: 89 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 713 op/s 2023-09-23T19:59:18.027 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:17 smithi016 bash[77995]: audit 2023-09-23T19:59:17.764458+0000 mon.c (mon.2) 237 : audit 1 from='client.? 172.21.15.16:0/699779620' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:59:18.027 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:17 smithi016 bash[77995]: audit 2023-09-23T19:59:17.765135+0000 mon.a (mon.0) 1881 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:59:18.027 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:17 smithi016 bash[75004]: cluster 2023-09-23T19:59:16.604961+0000 mgr.y (mgr.25325) 1561 : cluster 0 pgmap v1590: 89 pgs: 89 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 713 op/s 2023-09-23T19:59:18.027 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:17 smithi016 bash[75004]: audit 2023-09-23T19:59:17.764458+0000 mon.c (mon.2) 237 : audit 1 from='client.? 172.21.15.16:0/699779620' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:59:18.027 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:17 smithi016 bash[75004]: audit 2023-09-23T19:59:17.765135+0000 mon.a (mon.0) 1881 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:59:18.049 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:18.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:18.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:18.152 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:18.177 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:18.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:18.179 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:18.280 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:18.540 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T19:59:18.569 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:18.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:18.572 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:18.673 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:18.690 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:18.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:18.692 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:18.793 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:18.817 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:18.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:18.825 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:18.926 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:18.950 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:18.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:18.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:19.054 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:19.080 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:19.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:19.083 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:19.183 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:19.194 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:18 smithi092 bash[58855]: cluster 2023-09-23T19:59:17.837143+0000 mon.a (mon.0) 1882 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:59:19.195 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:18 smithi092 bash[58855]: audit 2023-09-23T19:59:17.863175+0000 mon.a (mon.0) 1883 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:59:19.195 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:18 smithi092 bash[58855]: cluster 2023-09-23T19:59:17.863425+0000 mon.a (mon.0) 1884 : cluster 0 osdmap e267: 8 total, 8 up, 8 in 2023-09-23T19:59:19.195 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:18 smithi092 bash[58855]: cluster 2023-09-23T19:59:17.863905+0000 osd.5 (osd.5) 192 : cluster 3 failed to encode map e267 with expected crc 2023-09-23T19:59:19.215 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:19.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:19.220 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:19.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:18 smithi016 bash[75004]: cluster 2023-09-23T19:59:17.837143+0000 mon.a (mon.0) 1882 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:59:19.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:18 smithi016 bash[75004]: audit 2023-09-23T19:59:17.863175+0000 mon.a (mon.0) 1883 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:59:19.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:18 smithi016 bash[75004]: cluster 2023-09-23T19:59:17.863425+0000 mon.a (mon.0) 1884 : cluster 0 osdmap e267: 8 total, 8 up, 8 in 2023-09-23T19:59:19.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:18 smithi016 bash[75004]: cluster 2023-09-23T19:59:17.863905+0000 osd.5 (osd.5) 192 : cluster 3 failed to encode map e267 with expected crc 2023-09-23T19:59:19.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:18 smithi016 bash[77995]: cluster 2023-09-23T19:59:17.837143+0000 mon.a (mon.0) 1882 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:59:19.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:18 smithi016 bash[77995]: audit 2023-09-23T19:59:17.863175+0000 mon.a (mon.0) 1883 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:59:19.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:18 smithi016 bash[77995]: cluster 2023-09-23T19:59:17.863425+0000 mon.a (mon.0) 1884 : cluster 0 osdmap e267: 8 total, 8 up, 8 in 2023-09-23T19:59:19.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:18 smithi016 bash[77995]: cluster 2023-09-23T19:59:17.863905+0000 osd.5 (osd.5) 192 : cluster 3 failed to encode map e267 with expected crc 2023-09-23T19:59:19.320 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:19.351 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:19.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:19.356 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:19.457 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:19.486 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:19.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:19.489 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:19.590 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:19.614 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:19.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:19.616 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:19.717 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:19.745 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:19.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:19.750 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:19.850 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:19.880 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:19.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:19.884 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:19.985 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:19.994 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:19 smithi016 bash[75004]: cluster 2023-09-23T19:59:18.605380+0000 mgr.y (mgr.25325) 1562 : cluster 0 pgmap v1592: 89 pgs: 89 active+clean; 4.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-09-23T19:59:19.994 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:19 smithi016 bash[77995]: cluster 2023-09-23T19:59:18.605380+0000 mgr.y (mgr.25325) 1562 : cluster 0 pgmap v1592: 89 pgs: 89 active+clean; 4.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-09-23T19:59:20.034 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:20.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:20.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:20.140 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:20.185 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:20.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:20.191 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:20.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:19 smithi092 bash[58855]: cluster 2023-09-23T19:59:18.605380+0000 mgr.y (mgr.25325) 1562 : cluster 0 pgmap v1592: 89 pgs: 89 active+clean; 4.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-09-23T19:59:20.292 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:20.331 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:20.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:20.335 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:20.435 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:20.472 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:20.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:20.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:20.579 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:20.615 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:20.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:20.618 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:20.719 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:20.749 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:20.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:20.752 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:20.853 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:20.887 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:20.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:20.891 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:20.992 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:21.029 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:21.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:21.036 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:21.137 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:21.161 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:21.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:21.164 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:21.265 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:21.316 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:21.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:21.319 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:21.419 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:21.453 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:21.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:21.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:21.558 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:21.588 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:21.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:21.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:21.693 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:21.718 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:21.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:21.722 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:21.823 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:21.862 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:21.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:21.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:21.966 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:21.974 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:21 smithi092 bash[58855]: cluster 2023-09-23T19:59:20.606043+0000 mgr.y (mgr.25325) 1563 : cluster 0 pgmap v1593: 89 pgs: 89 active+clean; 4.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 690 op/s 2023-09-23T19:59:21.988 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:21.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:21.990 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:22.093 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:22.102 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:21 smithi016 bash[75004]: cluster 2023-09-23T19:59:20.606043+0000 mgr.y (mgr.25325) 1563 : cluster 0 pgmap v1593: 89 pgs: 89 active+clean; 4.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 690 op/s 2023-09-23T19:59:22.103 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:21 smithi016 bash[77995]: cluster 2023-09-23T19:59:20.606043+0000 mgr.y (mgr.25325) 1563 : cluster 0 pgmap v1593: 89 pgs: 89 active+clean; 4.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 690 op/s 2023-09-23T19:59:22.146 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:22.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:22.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:22.251 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:22.277 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T19:59:22.278 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:22.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:22.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:59:22.381 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:22.408 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:22.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:22.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:59:22.513 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:22.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:22.555 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:22.556 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:22.656 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:22.695 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:22.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:22.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:22.800 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:22.826 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:22.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:22.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:22.930 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:22.963 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:22.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:22.966 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:23.068 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:23.092 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:23.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:23.095 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:23.195 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:23.233 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:23.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:23.237 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:23.322 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:59:23.338 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:23.375 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:23.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:23.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:23.484 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:23.510 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:23.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:23.518 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:23.619 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:23.641 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:23.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:23.646 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:59:23.746 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:23.775 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:23.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:23.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:23.880 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:23.887 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:23 smithi092 bash[58855]: cluster 2023-09-23T19:59:22.606592+0000 mgr.y (mgr.25325) 1564 : cluster 0 pgmap v1594: 89 pgs: 89 active+clean; 4.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 648 op/s 2023-09-23T19:59:23.888 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:23 smithi092 bash[58855]: audit 2023-09-23T19:59:23.687952+0000 mon.a (mon.0) 1885 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:59:23.888 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:23 smithi092 bash[58855]: audit 2023-09-23T19:59:23.705564+0000 mon.a (mon.0) 1886 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:59:23.888 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:23 smithi092 bash[58855]: audit 2023-09-23T19:59:23.707291+0000 mon.a (mon.0) 1887 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:59:23.888 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:23 smithi092 bash[58855]: audit 2023-09-23T19:59:23.709179+0000 mon.a (mon.0) 1888 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:59:23.888 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:23 smithi092 bash[58855]: audit 2023-09-23T19:59:23.720618+0000 mon.a (mon.0) 1889 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:59:23.906 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:23.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:23.912 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:24.013 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:24.021 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:23 smithi016 bash[75004]: cluster 2023-09-23T19:59:22.606592+0000 mgr.y (mgr.25325) 1564 : cluster 0 pgmap v1594: 89 pgs: 89 active+clean; 4.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 648 op/s 2023-09-23T19:59:24.021 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:23 smithi016 bash[75004]: audit 2023-09-23T19:59:23.687952+0000 mon.a (mon.0) 1885 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:59:24.022 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:23 smithi016 bash[75004]: audit 2023-09-23T19:59:23.705564+0000 mon.a (mon.0) 1886 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:59:24.022 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:23 smithi016 bash[75004]: audit 2023-09-23T19:59:23.707291+0000 mon.a (mon.0) 1887 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:59:24.022 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:23 smithi016 bash[75004]: audit 2023-09-23T19:59:23.709179+0000 mon.a (mon.0) 1888 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:59:24.022 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:23 smithi016 bash[75004]: audit 2023-09-23T19:59:23.720618+0000 mon.a (mon.0) 1889 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:59:24.022 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:23 smithi016 bash[77995]: cluster 2023-09-23T19:59:22.606592+0000 mgr.y (mgr.25325) 1564 : cluster 0 pgmap v1594: 89 pgs: 89 active+clean; 4.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 648 op/s 2023-09-23T19:59:24.022 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:23 smithi016 bash[77995]: audit 2023-09-23T19:59:23.687952+0000 mon.a (mon.0) 1885 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:59:24.022 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:23 smithi016 bash[77995]: audit 2023-09-23T19:59:23.705564+0000 mon.a (mon.0) 1886 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:59:24.022 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:23 smithi016 bash[77995]: audit 2023-09-23T19:59:23.707291+0000 mon.a (mon.0) 1887 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:59:24.023 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:23 smithi016 bash[77995]: audit 2023-09-23T19:59:23.709179+0000 mon.a (mon.0) 1888 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:59:24.023 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:23 smithi016 bash[77995]: audit 2023-09-23T19:59:23.720618+0000 mon.a (mon.0) 1889 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:59:24.043 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:24.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:24.046 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:24.053 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:59:24.146 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:24.194 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:24.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:24.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:24.302 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:24.329 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:24.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:24.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:24.432 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:24.459 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:24.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:24.465 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:24.566 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:24.601 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:24.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:24.608 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:24.708 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:24.738 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:24.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:24.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:24.842 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:24.868 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:24.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:24.874 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:24.975 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:25.006 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:25.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:25.013 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:25.114 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:25.137 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:25.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:25.140 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:59:25.241 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:25.284 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:25.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:25.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:25.390 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:25.418 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:25.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:25.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:25.524 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:25.549 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:25.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:25.558 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:59:25.659 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:25.707 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:25.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:25.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:25.785 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:59:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:59:25] "GET /metrics HTTP/1.1" 200 36374 "" "Prometheus/2.33.4" 2023-09-23T19:59:25.815 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:25.839 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:25.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:25.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:25.944 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:25.957 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:25 smithi092 bash[58855]: cluster 2023-09-23T19:59:24.607171+0000 mgr.y (mgr.25325) 1565 : cluster 0 pgmap v1595: 89 pgs: 89 active+clean; 4.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 683 op/s 2023-09-23T19:59:25.972 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:25.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:25.979 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:26.079 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:26.103 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:26.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:26.108 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:26.153 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:59:26.185 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:25 smithi016 bash[77995]: cluster 2023-09-23T19:59:24.607171+0000 mgr.y (mgr.25325) 1565 : cluster 0 pgmap v1595: 89 pgs: 89 active+clean; 4.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 683 op/s 2023-09-23T19:59:26.186 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:25 smithi016 bash[75004]: cluster 2023-09-23T19:59:24.607171+0000 mgr.y (mgr.25325) 1565 : cluster 0 pgmap v1595: 89 pgs: 89 active+clean; 4.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 683 op/s 2023-09-23T19:59:26.209 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:26.254 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:26.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:26.258 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:26.359 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:26.384 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:26.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:26.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:26.489 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:26.513 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:26.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:26.519 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:26.620 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:26.653 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:26.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:26.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:26.757 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:26.793 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:26.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:26.798 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:26.899 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:26.930 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:26.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:26.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:27.033 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:27.063 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:27.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:27.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:27.167 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:27.195 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:27.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:27.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:27.299 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:27.322 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:27.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:27.330 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:27.430 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:27.466 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:27.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:27.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:27.570 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:27.611 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:27.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:27.613 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:27.714 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:27.740 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:27.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:27.744 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:27.845 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:27.881 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:27.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:27.885 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:27.986 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:28.013 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:28.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:28.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:28.117 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:28.143 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:28.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:28.153 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:28.156 INFO:teuthology.orchestra.run.smithi092.stdout:{ 2023-09-23T19:59:28.156 INFO:teuthology.orchestra.run.smithi092.stdout: "ops": [ 2023-09-23T19:59:28.156 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:28.156 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_repop(client.46877.0:131084 14.9 e267/263 14:9dbaccb7:::benchmark_data_smithi016_228083_object31125:head v 267'8166, mlcod=267'8163)", 2023-09-23T19:59:28.156 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:59:28.153051+0000", 2023-09-23T19:59:28.156 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 0.0040897370000000004, 2023-09-23T19:59:28.156 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.004113612, 2023-09-23T19:59:28.156 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:59:28.156 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "started", 2023-09-23T19:59:28.156 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:59:28.156 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:28.157 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:59:28.157 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:59:28.153051+0000", 2023-09-23T19:59:28.157 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:59:28.157 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:28.157 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:28.157 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:59:28.157 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:59:28.153051+0000", 2023-09-23T19:59:28.157 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.999999 2023-09-23T19:59:28.157 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:28.157 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:28.157 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:59:28.157 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:59:28.153050+0000", 2023-09-23T19:59:28.158 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.0329999999999997e-06 2023-09-23T19:59:28.158 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:28.158 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:28.158 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:59:28.158 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:59:28.153059+0000", 2023-09-23T19:59:28.158 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.3036e-05 2023-09-23T19:59:28.158 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:28.158 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:28.158 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:59:28.158 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:59:28.153072+0000", 2023-09-23T19:59:28.158 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.2110000000000003e-06 2023-09-23T19:59:28.158 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:28.159 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:28.159 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:59:28.159 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:59:28.153078+0000", 2023-09-23T19:59:28.159 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.8356999999999998e-05 2023-09-23T19:59:28.159 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:28.159 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:28.159 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:59:28.159 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:59:28.153126+0000", 2023-09-23T19:59:28.159 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.6588999999999999e-05 2023-09-23T19:59:28.159 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:28.159 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:28.160 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:59:28.160 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:59:28.153153+0000", 2023-09-23T19:59:28.160 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000102193 2023-09-23T19:59:28.160 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:59:28.160 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:59:28.160 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:59:28.160 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:28.160 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:28.160 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_repop(client.46877.0:131087 14.0 e267/256 14:08fa10bd:::benchmark_data_smithi016_228083_object31128:head v 267'8148, mlcod=267'8146)", 2023-09-23T19:59:28.160 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:59:28.155151+0000", 2023-09-23T19:59:28.160 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 0.0019896670000000001, 2023-09-23T19:59:28.160 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0020669270000000001, 2023-09-23T19:59:28.161 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T19:59:28.161 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "started", 2023-09-23T19:59:28.161 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T19:59:28.161 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:28.161 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T19:59:28.161 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:59:28.155151+0000", 2023-09-23T19:59:28.161 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T19:59:28.161 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:28.161 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:28.161 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T19:59:28.161 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:59:28.155151+0000", 2023-09-23T19:59:28.161 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T19:59:28.161 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:28.162 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:28.162 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T19:59:28.162 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:59:28.155150+0000", 2023-09-23T19:59:28.162 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.1685000000000001e-05 2023-09-23T19:59:28.162 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:28.162 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:28.162 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T19:59:28.162 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:59:28.155161+0000", 2023-09-23T19:59:28.162 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.6847e-05 2023-09-23T19:59:28.162 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:28.162 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:28.162 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T19:59:28.163 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:59:28.155178+0000", 2023-09-23T19:59:28.163 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.8259999999999993e-06 2023-09-23T19:59:28.163 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:28.163 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:28.163 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T19:59:28.163 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:59:28.155186+0000", 2023-09-23T19:59:28.163 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.9440000000000001e-05 2023-09-23T19:59:28.163 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:28.163 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:28.163 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T19:59:28.163 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:59:28.155235+0000", 2023-09-23T19:59:28.163 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.1682999999999999e-05 2023-09-23T19:59:28.163 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T19:59:28.164 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T19:59:28.164 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T19:59:28.164 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:59:28.155257+0000", 2023-09-23T19:59:28.164 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000106235 2023-09-23T19:59:28.164 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:59:28.164 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T19:59:28.164 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:59:28.164 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T19:59:28.164 INFO:teuthology.orchestra.run.smithi092.stdout: ], 2023-09-23T19:59:28.164 INFO:teuthology.orchestra.run.smithi092.stdout: "num_ops": 2 2023-09-23T19:59:28.164 INFO:teuthology.orchestra.run.smithi092.stdout:} 2023-09-23T19:59:28.183 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:27 smithi092 bash[58855]: cluster 2023-09-23T19:59:26.607716+0000 mgr.y (mgr.25325) 1566 : cluster 0 pgmap v1596: 89 pgs: 89 active+clean; 4.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 679 op/s 2023-09-23T19:59:28.190 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:59:28.193 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:27 smithi016 bash[75004]: cluster 2023-09-23T19:59:26.607716+0000 mgr.y (mgr.25325) 1566 : cluster 0 pgmap v1596: 89 pgs: 89 active+clean; 4.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 679 op/s 2023-09-23T19:59:28.193 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:27 smithi016 bash[77995]: cluster 2023-09-23T19:59:26.607716+0000 mgr.y (mgr.25325) 1566 : cluster 0 pgmap v1596: 89 pgs: 89 active+clean; 4.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 679 op/s 2023-09-23T19:59:28.253 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:28.305 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:28.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:28.306 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:28.407 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:28.430 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:28.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:28.434 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:28.535 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:28.562 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:28.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:28.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:28.666 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:28.691 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:28.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:28.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:28.795 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:28.825 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:28.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:28.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:28.929 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:28.957 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:28.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:28.962 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:29.063 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:29.096 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:29.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:29.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:29.201 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:29.211 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T19:59:29.228 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:29.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:29.232 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:29.332 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:29.351 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:29.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:29.353 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:29.454 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:29.493 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:29.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:29.497 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:29.598 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:29.616 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:29.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:29.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:29.718 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:29.746 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:29.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:29.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:29.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:29 smithi092 bash[58855]: cluster 2023-09-23T19:59:28.608211+0000 mgr.y (mgr.25325) 1567 : cluster 0 pgmap v1597: 89 pgs: 89 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 679 op/s 2023-09-23T19:59:29.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:29 smithi092 bash[58855]: audit 2023-09-23T19:59:28.698546+0000 mon.a (mon.0) 1890 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:59:29.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:29 smithi092 bash[58855]: audit 2023-09-23T19:59:28.700846+0000 mon.a (mon.0) 1891 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:59:29.850 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:29.882 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:29.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:29.889 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:29.990 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:29.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:29 smithi016 bash[75004]: cluster 2023-09-23T19:59:28.608211+0000 mgr.y (mgr.25325) 1567 : cluster 0 pgmap v1597: 89 pgs: 89 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 679 op/s 2023-09-23T19:59:29.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:29 smithi016 bash[75004]: audit 2023-09-23T19:59:28.698546+0000 mon.a (mon.0) 1890 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:59:29.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:29 smithi016 bash[75004]: audit 2023-09-23T19:59:28.700846+0000 mon.a (mon.0) 1891 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:59:30.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:29 smithi016 bash[77995]: cluster 2023-09-23T19:59:28.608211+0000 mgr.y (mgr.25325) 1567 : cluster 0 pgmap v1597: 89 pgs: 89 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 679 op/s 2023-09-23T19:59:30.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:29 smithi016 bash[77995]: audit 2023-09-23T19:59:28.698546+0000 mon.a (mon.0) 1890 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T19:59:30.000 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:29 smithi016 bash[77995]: audit 2023-09-23T19:59:28.700846+0000 mon.a (mon.0) 1891 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:59:30.024 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:30.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:30.032 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:30.133 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:30.160 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:30.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:30.163 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:30.263 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:30.293 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:30.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:30.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:30.398 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:30.445 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:30.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:30.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:30.552 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:30.581 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:30.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:30.584 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:59:30.685 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:30.709 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:30.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:30.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:30.816 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:30.852 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:30.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:30.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:30.958 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:30.981 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:30.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:30.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:31.089 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:31.116 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:31.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:31.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:59:31.226 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:31.249 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:31.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:31.254 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:31.355 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:31.396 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:31.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:31.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:31.499 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:31.536 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:59:31.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:31.561 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:31.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:31.587 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:59:31.654 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:59:31.663 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:31.686 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:31.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:31.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:31.694 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:59:31.730 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:59:31.774 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:59:31.791 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:31.812 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:59:31.821 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:31.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:31.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:31.847 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:59:31.855 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:59:31.855 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:59:31.855 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:59:31.855 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:59:31.855 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:59:31.856 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:59:31.856 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:59:31.856 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:59:31.930 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:31.939 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:31 smithi092 bash[58855]: cluster 2023-09-23T19:59:30.609140+0000 mgr.y (mgr.25325) 1568 : cluster 0 pgmap v1598: 89 pgs: 89 active+clean; 4.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 714 op/s 2023-09-23T19:59:31.939 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:31 smithi092 bash[58855]: audit 2023-09-23T19:59:31.484237+0000 mon.c (mon.2) 238 : audit 0 from='client.? 172.21.15.16:0/3418464025' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:59:31.959 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:31.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:31.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:32.065 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:32.074 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:31 smithi016 bash[77995]: cluster 2023-09-23T19:59:30.609140+0000 mgr.y (mgr.25325) 1568 : cluster 0 pgmap v1598: 89 pgs: 89 active+clean; 4.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 714 op/s 2023-09-23T19:59:32.075 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:31 smithi016 bash[77995]: audit 2023-09-23T19:59:31.484237+0000 mon.c (mon.2) 238 : audit 0 from='client.? 172.21.15.16:0/3418464025' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:59:32.075 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:31 smithi016 bash[75004]: cluster 2023-09-23T19:59:30.609140+0000 mgr.y (mgr.25325) 1568 : cluster 0 pgmap v1598: 89 pgs: 89 active+clean; 4.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 714 op/s 2023-09-23T19:59:32.075 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:31 smithi016 bash[75004]: audit 2023-09-23T19:59:31.484237+0000 mon.c (mon.2) 238 : audit 0 from='client.? 172.21.15.16:0/3418464025' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:59:32.105 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:32.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:32.106 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:32.206 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:32.232 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:32.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:32.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:32.338 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:32.363 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:32.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:32.366 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:32.467 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:32.490 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:32.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:32.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:32.596 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:32.626 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:32.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:32.632 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:32.733 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:32.773 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:32.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:32.780 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:32.880 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:32.908 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:32.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:32.911 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:59:33.011 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:33.037 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:33.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:33.041 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:59:33.141 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:33.164 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:33.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:33.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:59:33.267 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:33.291 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:33.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:33.299 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:33.400 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:33.429 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:33.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:33.432 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:33.533 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:33.560 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:33.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:33.563 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:33.664 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:33.692 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:33.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:33.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:33.796 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:33.816 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:33.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:33.819 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:33.920 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:33.932 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:33 smithi016 bash[77995]: cluster 2023-09-23T19:59:32.609767+0000 mgr.y (mgr.25325) 1569 : cluster 0 pgmap v1599: 89 pgs: 89 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 626 op/s 2023-09-23T19:59:33.932 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:33 smithi016 bash[75004]: cluster 2023-09-23T19:59:32.609767+0000 mgr.y (mgr.25325) 1569 : cluster 0 pgmap v1599: 89 pgs: 89 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 626 op/s 2023-09-23T19:59:33.944 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:33.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:33.946 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:33.996 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:59:34.048 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:34.073 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:34.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:34.078 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:34.179 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:34.208 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:34.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:34.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:34.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:33 smithi092 bash[58855]: cluster 2023-09-23T19:59:32.609767+0000 mgr.y (mgr.25325) 1569 : cluster 0 pgmap v1599: 89 pgs: 89 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 626 op/s 2023-09-23T19:59:34.310 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:34.335 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:34.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:34.338 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:59:34.438 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:34.461 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:34.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:34.463 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:34.564 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:34.592 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:34.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:34.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:34.696 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:34.722 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:34.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:34.725 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:34.825 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:34.853 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:34.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:34.860 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:34.961 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:34.996 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:34.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:34.998 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:35.099 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:35.131 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:35.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:35.137 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:35.238 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:35.276 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:35.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:35.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:59:35.384 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:35.410 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:35.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:35.415 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:35.516 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:35.543 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:35.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:35.545 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:35.646 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:35.674 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:35.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:35.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:35.778 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:35.785 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:59:35 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:59:35] "GET /metrics HTTP/1.1" 200 36376 "" "Prometheus/2.33.4" 2023-09-23T19:59:35.821 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:35.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:35.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:35.928 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:35.952 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:35.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:35.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:36.058 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:36.063 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:35 smithi092 bash[58855]: cluster 2023-09-23T19:59:34.610360+0000 mgr.y (mgr.25325) 1570 : cluster 0 pgmap v1600: 89 pgs: 89 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2023-09-23T19:59:36.084 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:36.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:36.089 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:36.190 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:36.226 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:36.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:36.228 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:36.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:35 smithi016 bash[75004]: cluster 2023-09-23T19:59:34.610360+0000 mgr.y (mgr.25325) 1570 : cluster 0 pgmap v1600: 89 pgs: 89 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2023-09-23T19:59:36.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:35 smithi016 bash[77995]: cluster 2023-09-23T19:59:34.610360+0000 mgr.y (mgr.25325) 1570 : cluster 0 pgmap v1600: 89 pgs: 89 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2023-09-23T19:59:36.329 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:36.357 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:36.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:36.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:36.464 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:36.484 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:36.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:36.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:36.587 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:36.617 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:36.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:36.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:36.724 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:36.760 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:36.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:36.763 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:36.864 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:36.886 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:36.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:36.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:36.988 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:37.020 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:37.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:37.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:37.127 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:37.158 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:37.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:37.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:37.261 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:37.293 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:37.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:37.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:37.397 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:37.425 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:37.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:37.433 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:37.534 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:37.565 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:37.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:37.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:37.580 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:59:37.673 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:37.720 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:37.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:37.732 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:37.805 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:37 smithi016 bash[75004]: cluster 2023-09-23T19:59:36.611042+0000 mgr.y (mgr.25325) 1571 : cluster 0 pgmap v1601: 89 pgs: 89 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2023-09-23T19:59:37.833 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:37.874 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:37.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:37.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:37.978 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:38.006 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:38.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:38.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:38.062 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:37 smithi016 bash[77995]: cluster 2023-09-23T19:59:36.611042+0000 mgr.y (mgr.25325) 1571 : cluster 0 pgmap v1601: 89 pgs: 89 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2023-09-23T19:59:38.112 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:38.120 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:37 smithi092 bash[58855]: cluster 2023-09-23T19:59:36.611042+0000 mgr.y (mgr.25325) 1571 : cluster 0 pgmap v1601: 89 pgs: 89 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2023-09-23T19:59:38.144 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:38.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:38.146 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:38.247 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:38.278 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:38.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:38.286 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:38.387 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:38.409 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:38.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:38.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:38.512 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:38.564 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:38.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:38.569 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:38.631 INFO:teuthology.orchestra.run.smithi092.stdout:ERROR: (22) Invalid argument 2023-09-23T19:59:38.632 INFO:teuthology.orchestra.run.smithi092.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:59:38.670 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:38.714 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:38.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:38.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:38.822 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:38.829 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T19:59:38.830 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is set 2023-09-23T19:59:38.852 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:38.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:38.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:38.959 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:38.990 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:38.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:38.993 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:39.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:38 smithi092 bash[58855]: audit 2023-09-23T19:59:37.912434+0000 mon.c (mon.2) 239 : audit 1 from='client.? 172.21.15.16:0/855599891' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T19:59:39.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:38 smithi092 bash[58855]: audit 2023-09-23T19:59:37.913517+0000 mon.a (mon.0) 1892 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T19:59:39.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:38 smithi092 bash[58855]: audit 2023-09-23T19:59:38.729498+0000 mon.c (mon.2) 240 : audit 1 from='client.? 172.21.15.16:0/2216233720' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:59:39.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:38 smithi092 bash[58855]: audit 2023-09-23T19:59:38.730032+0000 mon.a (mon.0) 1893 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:59:39.094 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:39.117 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:39.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:39.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:39.220 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:39.256 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:39.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:39.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:59:39.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:38 smithi016 bash[75004]: audit 2023-09-23T19:59:37.912434+0000 mon.c (mon.2) 239 : audit 1 from='client.? 172.21.15.16:0/855599891' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T19:59:39.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:38 smithi016 bash[75004]: audit 2023-09-23T19:59:37.913517+0000 mon.a (mon.0) 1892 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T19:59:39.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:38 smithi016 bash[75004]: audit 2023-09-23T19:59:38.729498+0000 mon.c (mon.2) 240 : audit 1 from='client.? 172.21.15.16:0/2216233720' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:59:39.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:38 smithi016 bash[75004]: audit 2023-09-23T19:59:38.730032+0000 mon.a (mon.0) 1893 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:59:39.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:38 smithi016 bash[77995]: audit 2023-09-23T19:59:37.912434+0000 mon.c (mon.2) 239 : audit 1 from='client.? 172.21.15.16:0/855599891' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T19:59:39.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:38 smithi016 bash[77995]: audit 2023-09-23T19:59:37.913517+0000 mon.a (mon.0) 1892 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T19:59:39.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:38 smithi016 bash[77995]: audit 2023-09-23T19:59:38.729498+0000 mon.c (mon.2) 240 : audit 1 from='client.? 172.21.15.16:0/2216233720' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:59:39.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:38 smithi016 bash[77995]: audit 2023-09-23T19:59:38.730032+0000 mon.a (mon.0) 1893 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:59:39.361 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:39.388 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:39.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:39.395 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:39.496 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:39.523 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:39.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:39.528 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:39.629 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:39.673 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T19:59:39.681 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:39.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:39.684 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:39.785 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:39.811 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:39.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:39.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:39.918 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:39.926 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:39 smithi092 bash[58855]: cluster 2023-09-23T19:59:38.611543+0000 mgr.y (mgr.25325) 1572 : cluster 0 pgmap v1602: 89 pgs: 89 active+clean; 3.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 640 op/s 2023-09-23T19:59:39.926 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:39 smithi092 bash[58855]: cluster 2023-09-23T19:59:38.801672+0000 mon.a (mon.0) 1894 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:59:39.927 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:39 smithi092 bash[58855]: audit 2023-09-23T19:59:38.816154+0000 mon.a (mon.0) 1895 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T19:59:39.927 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:39 smithi092 bash[58855]: audit 2023-09-23T19:59:38.816431+0000 mon.a (mon.0) 1896 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:59:39.927 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:39 smithi092 bash[58855]: cluster 2023-09-23T19:59:38.816665+0000 mon.a (mon.0) 1897 : cluster 0 osdmap e268: 8 total, 8 up, 8 in 2023-09-23T19:59:39.927 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:39 smithi092 bash[58855]: cluster 2023-09-23T19:59:38.817501+0000 osd.5 (osd.5) 193 : cluster 3 failed to encode map e268 with expected crc 2023-09-23T19:59:39.927 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:39 smithi092 bash[58855]: cluster 2023-09-23T19:59:38.840357+0000 osd.7 (osd.7) 182 : cluster 3 failed to encode map e268 with expected crc 2023-09-23T19:59:39.927 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:39 smithi092 bash[58855]: cluster 2023-09-23T19:59:38.846789+0000 osd.6 (osd.6) 202 : cluster 3 failed to encode map e268 with expected crc 2023-09-23T19:59:39.966 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:39.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:39.968 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:40.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:39 smithi016 bash[75004]: cluster 2023-09-23T19:59:38.611543+0000 mgr.y (mgr.25325) 1572 : cluster 0 pgmap v1602: 89 pgs: 89 active+clean; 3.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 640 op/s 2023-09-23T19:59:40.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:39 smithi016 bash[75004]: cluster 2023-09-23T19:59:38.801672+0000 mon.a (mon.0) 1894 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:59:40.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:39 smithi016 bash[75004]: audit 2023-09-23T19:59:38.816154+0000 mon.a (mon.0) 1895 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T19:59:40.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:39 smithi016 bash[75004]: audit 2023-09-23T19:59:38.816431+0000 mon.a (mon.0) 1896 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:59:40.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:39 smithi016 bash[75004]: cluster 2023-09-23T19:59:38.816665+0000 mon.a (mon.0) 1897 : cluster 0 osdmap e268: 8 total, 8 up, 8 in 2023-09-23T19:59:40.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:39 smithi016 bash[75004]: cluster 2023-09-23T19:59:38.817501+0000 osd.5 (osd.5) 193 : cluster 3 failed to encode map e268 with expected crc 2023-09-23T19:59:40.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:39 smithi016 bash[75004]: cluster 2023-09-23T19:59:38.840357+0000 osd.7 (osd.7) 182 : cluster 3 failed to encode map e268 with expected crc 2023-09-23T19:59:40.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:39 smithi016 bash[75004]: cluster 2023-09-23T19:59:38.846789+0000 osd.6 (osd.6) 202 : cluster 3 failed to encode map e268 with expected crc 2023-09-23T19:59:40.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:39 smithi016 bash[77995]: cluster 2023-09-23T19:59:38.611543+0000 mgr.y (mgr.25325) 1572 : cluster 0 pgmap v1602: 89 pgs: 89 active+clean; 3.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 640 op/s 2023-09-23T19:59:40.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:39 smithi016 bash[77995]: cluster 2023-09-23T19:59:38.801672+0000 mon.a (mon.0) 1894 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:59:40.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:39 smithi016 bash[77995]: audit 2023-09-23T19:59:38.816154+0000 mon.a (mon.0) 1895 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T19:59:40.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:39 smithi016 bash[77995]: audit 2023-09-23T19:59:38.816431+0000 mon.a (mon.0) 1896 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:59:40.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:39 smithi016 bash[77995]: cluster 2023-09-23T19:59:38.816665+0000 mon.a (mon.0) 1897 : cluster 0 osdmap e268: 8 total, 8 up, 8 in 2023-09-23T19:59:40.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:39 smithi016 bash[77995]: cluster 2023-09-23T19:59:38.817501+0000 osd.5 (osd.5) 193 : cluster 3 failed to encode map e268 with expected crc 2023-09-23T19:59:40.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:39 smithi016 bash[77995]: cluster 2023-09-23T19:59:38.840357+0000 osd.7 (osd.7) 182 : cluster 3 failed to encode map e268 with expected crc 2023-09-23T19:59:40.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:39 smithi016 bash[77995]: cluster 2023-09-23T19:59:38.846789+0000 osd.6 (osd.6) 202 : cluster 3 failed to encode map e268 with expected crc 2023-09-23T19:59:40.070 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:40.115 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:40.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:40.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:40.222 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:40.249 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:40.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:40.252 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:40.352 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:40.387 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:40.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:40.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:40.495 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:40.522 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:40.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:40.524 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:40.625 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:40.655 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:40.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:40.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:59:40.761 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:40.790 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:40.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:40.797 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:40.898 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:40.928 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:40.929 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T19:59:40.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:40.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:41.052 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:41.085 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:41.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:41.088 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:41.189 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:41.216 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:41.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:41.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:59:41.327 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:41.356 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:41.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:41.361 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:41.461 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:41.484 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:41.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:41.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:41.592 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:41.617 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:41.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:41.620 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:41.721 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:41.765 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:41.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:41.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:41.871 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:41.900 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:41.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:41.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:42.005 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:42.014 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:41 smithi016 bash[75004]: cluster 2023-09-23T19:59:40.612179+0000 mgr.y (mgr.25325) 1573 : cluster 0 pgmap v1604: 89 pgs: 89 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-09-23T19:59:42.014 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:41 smithi016 bash[77995]: cluster 2023-09-23T19:59:40.612179+0000 mgr.y (mgr.25325) 1573 : cluster 0 pgmap v1604: 89 pgs: 89 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-09-23T19:59:42.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:41 smithi092 bash[58855]: cluster 2023-09-23T19:59:40.612179+0000 mgr.y (mgr.25325) 1573 : cluster 0 pgmap v1604: 89 pgs: 89 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-09-23T19:59:42.043 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:42.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:42.051 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:42.152 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:42.176 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:42.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:42.179 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:42.280 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:42.322 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:42.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:42.325 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:42.426 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:42.465 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:42.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:42.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:42.570 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:42.602 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:42.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:42.613 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:59:42.713 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:42.741 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:42.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:42.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:42.844 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:42.865 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:42.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:42.868 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:59:42.955 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T19:59:42.969 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:43.007 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:43.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:43.011 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:43.112 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:43.134 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:43.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:43.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:59:43.237 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:43.265 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:43.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:43.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:43.369 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:43.419 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:43.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:43.424 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:43.525 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:43.554 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:43.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:43.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:59:43.658 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:43.687 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:43.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:43.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:43.792 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:43.823 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:43.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:43.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:43.930 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:43.939 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:43 smithi016 bash[75004]: cluster 2023-09-23T19:59:42.612716+0000 mgr.y (mgr.25325) 1574 : cluster 0 pgmap v1605: 89 pgs: 89 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-09-23T19:59:43.939 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:43 smithi016 bash[75004]: audit 2023-09-23T19:59:43.695836+0000 mon.a (mon.0) 1898 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:59:43.939 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:43 smithi016 bash[77995]: cluster 2023-09-23T19:59:42.612716+0000 mgr.y (mgr.25325) 1574 : cluster 0 pgmap v1605: 89 pgs: 89 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-09-23T19:59:43.939 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:43 smithi016 bash[77995]: audit 2023-09-23T19:59:43.695836+0000 mon.a (mon.0) 1898 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:59:43.966 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:43.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:43.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:44.072 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:44.079 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:43 smithi092 bash[58855]: cluster 2023-09-23T19:59:42.612716+0000 mgr.y (mgr.25325) 1574 : cluster 0 pgmap v1605: 89 pgs: 89 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-09-23T19:59:44.079 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:43 smithi092 bash[58855]: audit 2023-09-23T19:59:43.695836+0000 mon.a (mon.0) 1898 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:59:44.105 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:44.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:44.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:44.209 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:44.260 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:44.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:44.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:44.365 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:44.394 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:44.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:44.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:44.474 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:59:44.494 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:59:44.500 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:44.648 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:44.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:44.654 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:44.754 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:44.801 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:44.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:44.808 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:44.909 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:44.933 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:44.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:44.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:45.037 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:45.072 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:45.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:45.077 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:45.178 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:45.206 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:45.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:45.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:45.309 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:45.343 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:45.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:45.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:45.448 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:45.477 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:45.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:45.479 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:59:45.517 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:59:45.580 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:45.621 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:45.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:45.632 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:45.733 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:45.771 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:45.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:45.779 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:45.845 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:59:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:59:45] "GET /metrics HTTP/1.1" 200 36373 "" "Prometheus/2.33.4" 2023-09-23T19:59:45.880 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:45.923 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:45.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:45.933 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:46.034 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:46.062 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:46.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:46.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:46.167 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:46.175 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:45 smithi092 bash[58855]: cluster 2023-09-23T19:59:44.613353+0000 mgr.y (mgr.25325) 1575 : cluster 0 pgmap v1606: 89 pgs: 89 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2023-09-23T19:59:46.196 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:46.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:46.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:46.205 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:46 smithi016 bash[77995]: cluster 2023-09-23T19:59:44.613353+0000 mgr.y (mgr.25325) 1575 : cluster 0 pgmap v1606: 89 pgs: 89 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2023-09-23T19:59:46.206 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:45 smithi016 bash[75004]: cluster 2023-09-23T19:59:44.613353+0000 mgr.y (mgr.25325) 1575 : cluster 0 pgmap v1606: 89 pgs: 89 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2023-09-23T19:59:46.302 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:46.325 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:46.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:46.327 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:46.428 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:46.465 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:46.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:46.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:46.572 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:46.599 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:46.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:46.602 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:46.703 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:46.733 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:46.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:46.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:46.839 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:46.893 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:46.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:46.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:46.998 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:47.029 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:47.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:47.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:47.135 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:47.173 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:47.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:47.180 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:47.281 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:47.311 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:47.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:47.319 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:47.420 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:47.460 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:47.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:47.464 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:47.552 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:59:47.565 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:47.594 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:47.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:47.597 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:47.698 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:47.728 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:47.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:47.736 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:47.837 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:47.864 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:47.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:47.870 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:59:47.971 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:48.005 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:48.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:48.013 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:48.114 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:48.127 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:47 smithi016 bash[75004]: cluster 2023-09-23T19:59:46.613863+0000 mgr.y (mgr.25325) 1576 : cluster 0 pgmap v1607: 89 pgs: 89 active+clean; 3.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2023-09-23T19:59:48.128 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:47 smithi016 bash[77995]: cluster 2023-09-23T19:59:46.613863+0000 mgr.y (mgr.25325) 1576 : cluster 0 pgmap v1607: 89 pgs: 89 active+clean; 3.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2023-09-23T19:59:48.151 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:48.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:48.153 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:48.255 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:48.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:47 smithi092 bash[58855]: cluster 2023-09-23T19:59:46.613863+0000 mgr.y (mgr.25325) 1576 : cluster 0 pgmap v1607: 89 pgs: 89 active+clean; 3.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2023-09-23T19:59:48.299 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:48.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:48.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:48.404 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:48.433 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:48.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:48.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:48.537 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:48.574 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:48.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:48.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:48.679 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:48.712 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:48.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:48.715 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:48.816 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:48.848 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:48.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:48.852 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:48.953 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:48.972 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:48.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:48.973 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:49.074 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:49.088 INFO:teuthology.orchestra.run.smithi092.stdout:ERROR: (22) Invalid argument 2023-09-23T19:59:49.089 INFO:teuthology.orchestra.run.smithi092.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:59:49.099 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:49.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:49.106 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:49.206 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:49.229 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:49.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:49.230 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:49.331 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:49.370 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:49.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:49.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:49.476 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:49.504 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:49.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:49.508 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:49.609 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:49.643 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:49.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:49.648 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:49.749 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:49.777 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:49.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:49.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:49.883 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:49.908 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:49.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:49.910 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:49.923 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:49 smithi016 bash[75004]: cluster 2023-09-23T19:59:48.614349+0000 mgr.y (mgr.25325) 1577 : cluster 0 pgmap v1608: 89 pgs: 89 active+clean; 3.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 661 op/s 2023-09-23T19:59:49.923 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:49 smithi016 bash[77995]: cluster 2023-09-23T19:59:48.614349+0000 mgr.y (mgr.25325) 1577 : cluster 0 pgmap v1608: 89 pgs: 89 active+clean; 3.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 661 op/s 2023-09-23T19:59:50.011 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:50.045 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:50.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:50.049 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:50.077 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T19:59:50.126 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:49 smithi092 bash[58855]: cluster 2023-09-23T19:59:48.614349+0000 mgr.y (mgr.25325) 1577 : cluster 0 pgmap v1608: 89 pgs: 89 active+clean; 3.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 661 op/s 2023-09-23T19:59:50.150 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:50.180 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:50.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:50.188 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:50.288 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:50.327 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:50.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:50.333 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:50.434 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:50.461 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:50.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:50.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:59:50.564 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:50.592 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:50.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:50.596 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:50.696 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:50.722 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:50.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:50.724 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:59:50.825 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:50.853 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:50.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:50.862 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:59:50.963 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:50.990 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:50.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:50.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:59:51.093 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:51.133 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:51.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:51.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:51.237 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:51.259 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:51.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:51.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:51.365 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:51.396 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:51.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:51.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:51.499 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:51.531 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:51.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:51.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:51.637 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:51.669 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:51.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:51.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:51.778 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:51.804 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:51.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:51.806 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:51.907 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:51.941 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:51.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:51.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:52.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:51 smithi092 bash[58855]: cluster 2023-09-23T19:59:50.615568+0000 mgr.y (mgr.25325) 1578 : cluster 0 pgmap v1609: 89 pgs: 89 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 706 op/s 2023-09-23T19:59:52.045 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:52.076 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:52.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:52.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:52.181 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:52.209 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:52.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:52.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:52.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:51 smithi016 bash[75004]: cluster 2023-09-23T19:59:50.615568+0000 mgr.y (mgr.25325) 1578 : cluster 0 pgmap v1609: 89 pgs: 89 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 706 op/s 2023-09-23T19:59:52.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:51 smithi016 bash[77995]: cluster 2023-09-23T19:59:50.615568+0000 mgr.y (mgr.25325) 1578 : cluster 0 pgmap v1609: 89 pgs: 89 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 706 op/s 2023-09-23T19:59:52.315 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:52.340 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:52.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:52.345 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:59:52.446 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:52.478 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:52.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:52.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:52.584 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:52.625 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:52.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:52.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:59:52.732 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:52.759 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:52.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:52.762 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:52.863 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:52.888 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:52.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:52.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:52.991 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:53.023 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:53.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:53.025 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:53.126 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:53.156 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:53.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:53.159 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:53.260 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:53.286 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:53.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:53.288 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:53.389 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:53.412 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:53.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:53.414 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:59:53.515 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:53.540 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:53.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:53.543 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:53.644 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:53.671 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:53.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:53.673 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:53.774 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:53.811 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:53.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:53.816 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:53.916 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:53.941 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:53.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:53.944 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:59:54.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:53 smithi016 bash[75004]: cluster 2023-09-23T19:59:52.616125+0000 mgr.y (mgr.25325) 1579 : cluster 0 pgmap v1610: 89 pgs: 89 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 605 op/s 2023-09-23T19:59:54.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:53 smithi016 bash[77995]: cluster 2023-09-23T19:59:52.616125+0000 mgr.y (mgr.25325) 1579 : cluster 0 pgmap v1610: 89 pgs: 89 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 605 op/s 2023-09-23T19:59:54.044 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:54.067 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:54.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:54.072 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:54.173 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:54.181 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:53 smithi092 bash[58855]: cluster 2023-09-23T19:59:52.616125+0000 mgr.y (mgr.25325) 1579 : cluster 0 pgmap v1610: 89 pgs: 89 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 605 op/s 2023-09-23T19:59:54.202 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:54.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:54.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:54.309 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:54.333 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:54.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:54.339 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:59:54.440 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:54.474 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:54.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:54.478 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:54.579 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:54.622 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:54.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:54.624 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:54.725 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:54.749 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:54.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:54.752 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:54.853 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:54.878 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:54.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:54.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:54.927 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:59:54.984 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:55.016 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:55.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:55.018 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:55.119 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:55.141 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:55.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:55.145 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:55.246 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:55.277 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:55.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:55.281 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:55.382 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:55.408 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:55.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:55.410 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:55.511 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:55.535 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:55.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:55.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:55.639 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:55.673 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:55.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:55.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:55.780 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:55.785 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 19:59:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:19:59:55] "GET /metrics HTTP/1.1" 200 36373 "" "Prometheus/2.33.4" 2023-09-23T19:59:55.817 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:55.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:55.818 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:55.918 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:55.952 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:55.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:55.956 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:56.057 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:56.087 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:56.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:56.092 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:56.193 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:56.200 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:55 smithi016 bash[75004]: cluster 2023-09-23T19:59:54.616676+0000 mgr.y (mgr.25325) 1580 : cluster 0 pgmap v1611: 89 pgs: 89 active+clean; 3.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2023-09-23T19:59:56.200 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:55 smithi016 bash[77995]: cluster 2023-09-23T19:59:54.616676+0000 mgr.y (mgr.25325) 1580 : cluster 0 pgmap v1611: 89 pgs: 89 active+clean; 3.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2023-09-23T19:59:56.230 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:56.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:56.233 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:56.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:55 smithi092 bash[58855]: cluster 2023-09-23T19:59:54.616676+0000 mgr.y (mgr.25325) 1580 : cluster 0 pgmap v1611: 89 pgs: 89 active+clean; 3.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2023-09-23T19:59:56.334 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:56.362 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:56.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:56.365 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:56.465 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:56.499 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:56.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:56.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:56.607 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:56.635 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:56.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:56.640 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:56.741 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:56.767 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:56.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:56.772 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:56.873 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:56.898 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:56.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:56.901 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:59:56.980 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T19:59:57.002 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:57.040 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T19:59:57.047 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:57.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:57.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:57.091 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T19:59:57.136 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T19:59:57.153 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:57.178 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:57.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:57.181 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:57.185 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T19:59:57.227 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T19:59:57.267 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T19:59:57.281 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:57.305 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:57.310 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T19:59:57.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:57.312 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:57.316 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:59:57.316 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:59:57.317 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:59:57.317 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:59:57.317 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:59:57.317 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:59:57.317 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:59:57.317 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:59:57.413 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:57.437 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:57.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:57.440 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:57.541 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:57.571 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:57.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:57.573 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:57.673 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:57.702 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:57.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:57.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:57.805 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:57.841 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:57.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:57.845 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:57.887 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:57 smithi016 bash[75004]: cluster 2023-09-23T19:59:56.617528+0000 mgr.y (mgr.25325) 1581 : cluster 0 pgmap v1612: 89 pgs: 89 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2023-09-23T19:59:57.887 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:57 smithi016 bash[75004]: audit 2023-09-23T19:59:56.920373+0000 mon.a (mon.0) 1899 : audit 0 from='client.? 172.21.15.16:0/2959232374' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:59:57.887 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:57 smithi016 bash[75004]: audit 2023-09-23T19:59:57.496517+0000 mon.a (mon.0) 1900 : audit 1 from='client.? 172.21.15.16:0/2068341206' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T19:59:57.887 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:57 smithi016 bash[75004]: audit 2023-09-23T19:59:57.639557+0000 mon.a (mon.0) 1901 : audit 1 from='client.? 172.21.15.16:0/1284769944' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:59:57.905 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is unset 2023-09-23T19:59:57.913 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T19:59:57.946 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:57.980 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:57.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:57.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:58.084 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:58.119 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:58.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:58.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:59:58.226 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:58.235 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:57 smithi016 bash[77995]: cluster 2023-09-23T19:59:56.617528+0000 mgr.y (mgr.25325) 1581 : cluster 0 pgmap v1612: 89 pgs: 89 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2023-09-23T19:59:58.235 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:57 smithi016 bash[77995]: audit 2023-09-23T19:59:56.920373+0000 mon.a (mon.0) 1899 : audit 0 from='client.? 172.21.15.16:0/2959232374' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:59:58.235 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:57 smithi016 bash[77995]: audit 2023-09-23T19:59:57.496517+0000 mon.a (mon.0) 1900 : audit 1 from='client.? 172.21.15.16:0/2068341206' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T19:59:58.235 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:57 smithi016 bash[77995]: audit 2023-09-23T19:59:57.639557+0000 mon.a (mon.0) 1901 : audit 1 from='client.? 172.21.15.16:0/1284769944' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:59:58.268 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:58.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:58.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:58.284 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:57 smithi092 bash[58855]: cluster 2023-09-23T19:59:56.617528+0000 mgr.y (mgr.25325) 1581 : cluster 0 pgmap v1612: 89 pgs: 89 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2023-09-23T19:59:58.284 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:57 smithi092 bash[58855]: audit 2023-09-23T19:59:56.920373+0000 mon.a (mon.0) 1899 : audit 0 from='client.? 172.21.15.16:0/2959232374' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:59:58.284 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:57 smithi092 bash[58855]: audit 2023-09-23T19:59:57.496517+0000 mon.a (mon.0) 1900 : audit 1 from='client.? 172.21.15.16:0/2068341206' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T19:59:58.284 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:57 smithi092 bash[58855]: audit 2023-09-23T19:59:57.639557+0000 mon.a (mon.0) 1901 : audit 1 from='client.? 172.21.15.16:0/1284769944' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:59:58.373 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:58.400 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:58.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:58.404 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:59:58.505 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:58.529 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:58.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:58.535 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:58.636 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:58.675 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:58.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:58.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:58.782 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:58.811 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:58.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:58.816 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:58.888 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:58 smithi016 bash[75004]: cluster 2023-09-23T19:59:57.877860+0000 mon.a (mon.0) 1902 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:59:58.888 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:58 smithi016 bash[75004]: audit 2023-09-23T19:59:57.893649+0000 mon.a (mon.0) 1903 : audit 1 from='client.? 172.21.15.16:0/2068341206' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T19:59:58.888 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:58 smithi016 bash[75004]: audit 2023-09-23T19:59:57.893874+0000 mon.a (mon.0) 1904 : audit 1 from='client.? 172.21.15.16:0/1284769944' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:59:58.916 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:58.941 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:58.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:58.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:59.013 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:58 smithi016 bash[77995]: cluster 2023-09-23T19:59:57.877860+0000 mon.a (mon.0) 1902 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:59:59.013 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:58 smithi016 bash[77995]: audit 2023-09-23T19:59:57.893649+0000 mon.a (mon.0) 1903 : audit 1 from='client.? 172.21.15.16:0/2068341206' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T19:59:59.013 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:58 smithi016 bash[77995]: audit 2023-09-23T19:59:57.893874+0000 mon.a (mon.0) 1904 : audit 1 from='client.? 172.21.15.16:0/1284769944' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:59:59.013 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:58 smithi016 bash[77995]: cluster 2023-09-23T19:59:57.894073+0000 mon.a (mon.0) 1905 : cluster 0 osdmap e269: 8 total, 8 up, 8 in 2023-09-23T19:59:59.013 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:58 smithi016 bash[77995]: cluster 2023-09-23T19:59:57.913914+0000 osd.4 (osd.4) 208 : cluster 3 failed to encode map e269 with expected crc 2023-09-23T19:59:59.014 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:58 smithi016 bash[77995]: cluster 2023-09-23T19:59:57.928267+0000 osd.7 (osd.7) 183 : cluster 3 failed to encode map e269 with expected crc 2023-09-23T19:59:59.014 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:58 smithi016 bash[77995]: audit 2023-09-23T19:59:58.696525+0000 mon.a (mon.0) 1906 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:59:59.014 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:58 smithi016 bash[75004]: cluster 2023-09-23T19:59:57.894073+0000 mon.a (mon.0) 1905 : cluster 0 osdmap e269: 8 total, 8 up, 8 in 2023-09-23T19:59:59.014 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:58 smithi016 bash[75004]: cluster 2023-09-23T19:59:57.913914+0000 osd.4 (osd.4) 208 : cluster 3 failed to encode map e269 with expected crc 2023-09-23T19:59:59.014 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:58 smithi016 bash[75004]: cluster 2023-09-23T19:59:57.928267+0000 osd.7 (osd.7) 183 : cluster 3 failed to encode map e269 with expected crc 2023-09-23T19:59:59.014 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:58 smithi016 bash[75004]: audit 2023-09-23T19:59:58.696525+0000 mon.a (mon.0) 1906 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:59:59.045 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:59.083 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:59.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:59.087 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:59.188 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:59.236 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:59.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:59.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:59.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:58 smithi092 bash[58855]: cluster 2023-09-23T19:59:57.877860+0000 mon.a (mon.0) 1902 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:59:59.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:58 smithi092 bash[58855]: audit 2023-09-23T19:59:57.893649+0000 mon.a (mon.0) 1903 : audit 1 from='client.? 172.21.15.16:0/2068341206' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T19:59:59.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:58 smithi092 bash[58855]: audit 2023-09-23T19:59:57.893874+0000 mon.a (mon.0) 1904 : audit 1 from='client.? 172.21.15.16:0/1284769944' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:59:59.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:58 smithi092 bash[58855]: cluster 2023-09-23T19:59:57.894073+0000 mon.a (mon.0) 1905 : cluster 0 osdmap e269: 8 total, 8 up, 8 in 2023-09-23T19:59:59.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:58 smithi092 bash[58855]: cluster 2023-09-23T19:59:57.913914+0000 osd.4 (osd.4) 208 : cluster 3 failed to encode map e269 with expected crc 2023-09-23T19:59:59.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:58 smithi092 bash[58855]: cluster 2023-09-23T19:59:57.928267+0000 osd.7 (osd.7) 183 : cluster 3 failed to encode map e269 with expected crc 2023-09-23T19:59:59.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:58 smithi092 bash[58855]: audit 2023-09-23T19:59:58.696525+0000 mon.a (mon.0) 1906 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:59:59.342 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:59.367 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:59.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:59.369 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:59.470 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:59.497 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:59.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:59.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:59.602 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:59.630 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:59.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:59.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:59.733 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:59.759 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:59:59.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:59.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:59:59.862 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:59.878 INFO:teuthology.orchestra.run.smithi092.stdout:{ 2023-09-23T19:59:59.878 INFO:teuthology.orchestra.run.smithi092.stdout: "ops": [], 2023-09-23T19:59:59.878 INFO:teuthology.orchestra.run.smithi092.stdout: "num_ops": 0 2023-09-23T19:59:59.878 INFO:teuthology.orchestra.run.smithi092.stdout:} 2023-09-23T19:59:59.898 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:59.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:59:59.902 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:00.002 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:00.025 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:00.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:00.030 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:00.089 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T20:00:00.131 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:00.158 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:00.161 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:59 smithi092 bash[58855]: cluster 2023-09-23T19:59:57.913994+0000 osd.6 (osd.6) 203 : cluster 3 failed to encode map e269 with expected crc 2023-09-23T20:00:00.161 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:59 smithi092 bash[58855]: cluster 2023-09-23T19:59:57.915824+0000 osd.6 (osd.6) 204 : cluster 3 failed to encode map e269 with expected crc 2023-09-23T20:00:00.161 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 19:59:59 smithi092 bash[58855]: cluster 2023-09-23T19:59:58.618033+0000 mgr.y (mgr.25325) 1582 : cluster 0 pgmap v1614: 89 pgs: 89 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 668 op/s 2023-09-23T20:00:00.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:00.162 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:00.263 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:00.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:59 smithi016 bash[75004]: cluster 2023-09-23T19:59:57.913994+0000 osd.6 (osd.6) 203 : cluster 3 failed to encode map e269 with expected crc 2023-09-23T20:00:00.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:59 smithi016 bash[75004]: cluster 2023-09-23T19:59:57.915824+0000 osd.6 (osd.6) 204 : cluster 3 failed to encode map e269 with expected crc 2023-09-23T20:00:00.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 19:59:59 smithi016 bash[75004]: cluster 2023-09-23T19:59:58.618033+0000 mgr.y (mgr.25325) 1582 : cluster 0 pgmap v1614: 89 pgs: 89 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 668 op/s 2023-09-23T20:00:00.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:59 smithi016 bash[77995]: cluster 2023-09-23T19:59:57.913994+0000 osd.6 (osd.6) 203 : cluster 3 failed to encode map e269 with expected crc 2023-09-23T20:00:00.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:59 smithi016 bash[77995]: cluster 2023-09-23T19:59:57.915824+0000 osd.6 (osd.6) 204 : cluster 3 failed to encode map e269 with expected crc 2023-09-23T20:00:00.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 19:59:59 smithi016 bash[77995]: cluster 2023-09-23T19:59:58.618033+0000 mgr.y (mgr.25325) 1582 : cluster 0 pgmap v1614: 89 pgs: 89 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 668 op/s 2023-09-23T20:00:00.296 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:00.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:00.300 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:00.401 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:00.432 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:00.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:00.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:00.535 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:00.570 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:00.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:00.573 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:00.638 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:00:00.674 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:00.726 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:00.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:00.729 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:00.830 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:00.877 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:00.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:00.882 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:00.982 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:01.010 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:01.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:01.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:01.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:00 smithi016 bash[77995]: cluster 2023-09-23T20:00:00.000098+0000 mon.a (mon.0) 1907 : cluster 3 Health detail: HEALTH_WARN nodeep-scrub flag(s) set 2023-09-23T20:00:01.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:00 smithi016 bash[77995]: cluster 2023-09-23T20:00:00.000122+0000 mon.a (mon.0) 1908 : cluster 3 [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-09-23T20:00:01.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:00 smithi016 bash[75004]: cluster 2023-09-23T20:00:00.000098+0000 mon.a (mon.0) 1907 : cluster 3 Health detail: HEALTH_WARN nodeep-scrub flag(s) set 2023-09-23T20:00:01.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:00 smithi016 bash[75004]: cluster 2023-09-23T20:00:00.000122+0000 mon.a (mon.0) 1908 : cluster 3 [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-09-23T20:00:01.112 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:01.131 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T20:00:01.147 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:01.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:01.151 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:00:01.252 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:01.282 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:01.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:01.287 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:01.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:00 smithi092 bash[58855]: cluster 2023-09-23T20:00:00.000098+0000 mon.a (mon.0) 1907 : cluster 3 Health detail: HEALTH_WARN nodeep-scrub flag(s) set 2023-09-23T20:00:01.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:00 smithi092 bash[58855]: cluster 2023-09-23T20:00:00.000122+0000 mon.a (mon.0) 1908 : cluster 3 [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-09-23T20:00:01.387 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:01.416 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:01.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:01.422 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:01.523 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:01.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:01.563 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:01.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:01.664 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:01.704 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:01.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:01.706 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:01.807 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:01.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:01.839 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:01.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:01.940 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:01.966 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:01.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:01.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:02.069 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:02.101 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:02.103 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:00:02.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:02.109 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:00:02.113 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:01 smithi016 bash[75004]: cluster 2023-09-23T20:00:00.618660+0000 mgr.y (mgr.25325) 1583 : cluster 0 pgmap v1615: 89 pgs: 89 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 658 op/s 2023-09-23T20:00:02.114 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:01 smithi016 bash[77995]: cluster 2023-09-23T20:00:00.618660+0000 mgr.y (mgr.25325) 1583 : cluster 0 pgmap v1615: 89 pgs: 89 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 658 op/s 2023-09-23T20:00:02.210 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:02.217 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:01 smithi092 bash[58855]: cluster 2023-09-23T20:00:00.618660+0000 mgr.y (mgr.25325) 1583 : cluster 0 pgmap v1615: 89 pgs: 89 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 658 op/s 2023-09-23T20:00:02.233 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:02.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:02.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:02.336 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:02.361 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:02.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:02.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:02.466 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:02.491 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:02.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:02.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:02.594 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:02.639 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:02.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:02.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:02.744 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:02.770 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:02.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:02.777 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:02.878 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:02.902 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:02.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:02.905 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:00:03.005 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:03.033 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:03.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:03.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:03.142 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:03.171 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:03.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:03.173 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:03.274 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:03.300 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:03.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:03.308 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:03.408 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:03.431 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:03.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:03.435 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:03.536 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:03.567 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:03.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:03.570 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:03.671 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:03.697 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:03.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:03.701 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:03.802 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:03.828 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:03.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:03.830 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:03.931 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:03.933 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:03 smithi016 bash[75004]: cluster 2023-09-23T20:00:02.619217+0000 mgr.y (mgr.25325) 1584 : cluster 0 pgmap v1616: 89 pgs: 89 active+clean; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2023-09-23T20:00:03.962 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:03.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:03.968 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:00:04.069 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:04.119 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:04.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:04.124 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:04.224 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:04.233 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:03 smithi092 bash[58855]: cluster 2023-09-23T20:00:02.619217+0000 mgr.y (mgr.25325) 1584 : cluster 0 pgmap v1616: 89 pgs: 89 active+clean; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2023-09-23T20:00:04.252 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:04.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:04.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:04.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:03 smithi016 bash[77995]: cluster 2023-09-23T20:00:02.619217+0000 mgr.y (mgr.25325) 1584 : cluster 0 pgmap v1616: 89 pgs: 89 active+clean; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2023-09-23T20:00:04.360 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:04.411 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:04.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:04.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:04.518 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:04.541 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:04.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:04.544 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:04.645 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:04.670 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:04.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:04.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:04.774 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:04.806 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:04.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:04.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:04.914 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:04.933 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:00:04.937 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:04.939 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:00:04.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:04.940 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:00:05.041 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:05.066 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:05.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:05.069 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:05.170 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:05.198 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:05.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:05.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:00:05.301 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:05.343 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:05.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:05.347 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:05.448 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:05.488 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:05.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:05.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:05.597 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:05.620 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:05.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:05.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:05.724 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:05.759 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:05.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:05.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:05.866 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:05.897 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:05.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:05.901 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:05.901 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:00:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:20:00:05] "GET /metrics HTTP/1.1" 200 36374 "" "Prometheus/2.33.4" 2023-09-23T20:00:05.946 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:00:06.001 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:06.022 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:06.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:06.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:06.125 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:06.153 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:06.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:06.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:06.157 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:05 smithi016 bash[77995]: cluster 2023-09-23T20:00:04.619874+0000 mgr.y (mgr.25325) 1585 : cluster 0 pgmap v1617: 89 pgs: 89 active+clean; 2.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-09-23T20:00:06.157 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:05 smithi016 bash[75004]: cluster 2023-09-23T20:00:04.619874+0000 mgr.y (mgr.25325) 1585 : cluster 0 pgmap v1617: 89 pgs: 89 active+clean; 2.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-09-23T20:00:06.258 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:06.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:05 smithi092 bash[58855]: cluster 2023-09-23T20:00:04.619874+0000 mgr.y (mgr.25325) 1585 : cluster 0 pgmap v1617: 89 pgs: 89 active+clean; 2.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-09-23T20:00:06.293 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:06.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:06.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:06.397 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:06.428 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:06.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:06.432 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:06.533 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:06.560 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:06.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:06.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:06.665 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:06.698 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:06.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:06.702 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:06.803 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:06.823 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:06.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:06.826 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:06.926 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:06.958 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:06.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:06.960 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:06.969 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:00:07.061 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:07.085 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:07.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:07.093 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:00:07.193 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:07.222 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:07.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:07.228 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:07.329 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:07.361 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:07.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:07.363 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:07.464 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:07.486 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:07.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:07.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:07.593 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:07.626 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:07.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:07.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:07.733 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:07.781 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:07.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:07.785 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:00:07.886 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:07.913 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:07.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:07.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:08.017 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:08.028 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:07 smithi092 bash[58855]: cluster 2023-09-23T20:00:06.620518+0000 mgr.y (mgr.25325) 1586 : cluster 0 pgmap v1618: 89 pgs: 89 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 639 op/s 2023-09-23T20:00:08.050 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:08.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:08.052 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:00:08.153 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:08.176 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:08.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:08.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:08.281 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:08.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:07 smithi016 bash[75004]: cluster 2023-09-23T20:00:06.620518+0000 mgr.y (mgr.25325) 1586 : cluster 0 pgmap v1618: 89 pgs: 89 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 639 op/s 2023-09-23T20:00:08.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:07 smithi016 bash[77995]: cluster 2023-09-23T20:00:06.620518+0000 mgr.y (mgr.25325) 1586 : cluster 0 pgmap v1618: 89 pgs: 89 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 639 op/s 2023-09-23T20:00:08.312 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:08.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:08.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:00:08.419 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:08.450 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:08.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:08.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:08.556 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:08.581 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:08.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:08.585 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:08.686 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:08.709 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:08.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:08.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:08.812 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:08.843 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:08.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:08.846 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:08.946 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:08.978 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:08.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:08.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:09.082 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:09.108 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:09.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:09.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:09.213 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:09.241 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:09.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:09.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:00:09.345 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:09.367 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:09.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:09.370 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:09.471 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:09.493 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:09.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:09.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:09.596 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:09.683 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:09.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:09.689 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:09.790 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:09.819 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:09.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:09.826 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:09.926 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:09.962 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:09.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:09.968 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:10.069 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:10.076 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:09 smithi092 bash[58855]: cluster 2023-09-23T20:00:08.621050+0000 mgr.y (mgr.25325) 1587 : cluster 0 pgmap v1619: 89 pgs: 89 active+clean; 2.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 628 op/s 2023-09-23T20:00:10.092 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:10.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:10.096 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:00:10.197 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:10.220 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:10.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:10.222 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:10.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:09 smithi016 bash[75004]: cluster 2023-09-23T20:00:08.621050+0000 mgr.y (mgr.25325) 1587 : cluster 0 pgmap v1619: 89 pgs: 89 active+clean; 2.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 628 op/s 2023-09-23T20:00:10.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:09 smithi016 bash[77995]: cluster 2023-09-23T20:00:08.621050+0000 mgr.y (mgr.25325) 1587 : cluster 0 pgmap v1619: 89 pgs: 89 active+clean; 2.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 628 op/s 2023-09-23T20:00:10.323 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:10.364 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:10.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:10.375 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:10.475 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:10.498 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:10.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:10.505 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:10.606 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:10.644 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:10.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:10.651 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:10.752 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:10.790 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:10.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:10.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:10.893 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:10.933 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:10.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:10.936 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:10.949 INFO:teuthology.orchestra.run.smithi092.stdout:{ 2023-09-23T20:00:10.950 INFO:teuthology.orchestra.run.smithi092.stdout: "ops": [], 2023-09-23T20:00:10.950 INFO:teuthology.orchestra.run.smithi092.stdout: "complaint_time": 30, 2023-09-23T20:00:10.950 INFO:teuthology.orchestra.run.smithi092.stdout: "num_blocked_ops": 0 2023-09-23T20:00:10.950 INFO:teuthology.orchestra.run.smithi092.stdout:} 2023-09-23T20:00:11.037 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:11.062 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:11.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:11.069 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:00:11.169 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:11.196 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:11.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:11.200 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:11.300 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:11.334 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:11.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:11.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:00:11.440 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:11.467 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:11.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:11.468 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:11.569 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:11.612 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:11.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:11.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:11.715 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:11.752 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:11.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:11.753 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:11.854 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:11.898 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:11.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:11.901 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:11.903 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T20:00:12.002 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:12.024 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:12.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:12.026 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:12.127 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:12.160 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:12.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:12.162 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:12.163 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:11 smithi092 bash[58855]: cluster 2023-09-23T20:00:10.621693+0000 mgr.y (mgr.25325) 1588 : cluster 0 pgmap v1620: 89 pgs: 89 active+clean; 2.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 663 op/s 2023-09-23T20:00:12.263 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:12.272 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:11 smithi016 bash[77995]: cluster 2023-09-23T20:00:10.621693+0000 mgr.y (mgr.25325) 1588 : cluster 0 pgmap v1620: 89 pgs: 89 active+clean; 2.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 663 op/s 2023-09-23T20:00:12.272 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:11 smithi016 bash[75004]: cluster 2023-09-23T20:00:10.621693+0000 mgr.y (mgr.25325) 1588 : cluster 0 pgmap v1620: 89 pgs: 89 active+clean; 2.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 663 op/s 2023-09-23T20:00:12.290 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:12.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:12.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:12.397 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:12.423 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:12.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:12.428 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:12.529 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:12.554 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:12.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:12.556 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:12.657 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:12.686 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:12.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:12.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:12.793 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:13.012 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:13.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:13.016 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:13.117 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:13.151 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:13.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:13.153 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:00:13.254 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:13.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:13.300 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:13.300 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:13.401 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:13.428 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:13.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:13.433 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:13.534 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:13.568 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:13.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:13.571 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:13.672 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:13.698 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:13.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:13.705 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:13.806 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:13.848 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:13.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:13.853 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:13.954 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:13.978 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:13.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:13.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:14.082 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:14.092 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:13 smithi016 bash[75004]: cluster 2023-09-23T20:00:12.622226+0000 mgr.y (mgr.25325) 1589 : cluster 0 pgmap v1621: 89 pgs: 89 active+clean; 2.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 579 op/s 2023-09-23T20:00:14.093 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:13 smithi016 bash[75004]: audit 2023-09-23T20:00:13.696546+0000 mon.a (mon.0) 1909 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:00:14.093 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:13 smithi016 bash[77995]: cluster 2023-09-23T20:00:12.622226+0000 mgr.y (mgr.25325) 1589 : cluster 0 pgmap v1621: 89 pgs: 89 active+clean; 2.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 579 op/s 2023-09-23T20:00:14.093 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:13 smithi016 bash[77995]: audit 2023-09-23T20:00:13.696546+0000 mon.a (mon.0) 1909 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:00:14.110 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:14.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:14.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:14.219 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:14.260 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:14.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:14.265 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:14.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:13 smithi092 bash[58855]: cluster 2023-09-23T20:00:12.622226+0000 mgr.y (mgr.25325) 1589 : cluster 0 pgmap v1621: 89 pgs: 89 active+clean; 2.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 579 op/s 2023-09-23T20:00:14.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:13 smithi092 bash[58855]: audit 2023-09-23T20:00:13.696546+0000 mon.a (mon.0) 1909 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:00:14.367 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:14.395 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:14.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:14.397 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:14.498 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:14.529 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:14.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:14.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:00:14.632 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:14.657 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:14.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:14.663 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:14.764 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:14.801 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:14.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:14.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:14.903 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:14.924 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:14.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:14.927 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:15.028 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:15.061 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:15.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:15.069 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:15.169 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:15.193 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:15.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:15.196 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:15.297 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:15.322 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:15.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:15.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:15.428 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:15.453 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:15.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:15.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:00:15.564 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:15.596 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:15.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:15.598 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:15.699 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:15.747 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:15.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:15.750 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:15.851 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:15.878 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:15.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:15.880 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:15.965 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:00:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:20:00:15] "GET /metrics HTTP/1.1" 200 36374 "" "Prometheus/2.33.4" 2023-09-23T20:00:15.982 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:16.002 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:15 smithi092 bash[58855]: cluster 2023-09-23T20:00:14.622864+0000 mgr.y (mgr.25325) 1590 : cluster 0 pgmap v1622: 89 pgs: 89 active+clean; 2.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 647 op/s 2023-09-23T20:00:16.018 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:16.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:16.020 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:00:16.121 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:16.155 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:16.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:16.157 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:16.258 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:16.273 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:15 smithi016 bash[75004]: cluster 2023-09-23T20:00:14.622864+0000 mgr.y (mgr.25325) 1590 : cluster 0 pgmap v1622: 89 pgs: 89 active+clean; 2.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 647 op/s 2023-09-23T20:00:16.273 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:16 smithi016 bash[77995]: cluster 2023-09-23T20:00:14.622864+0000 mgr.y (mgr.25325) 1590 : cluster 0 pgmap v1622: 89 pgs: 89 active+clean; 2.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 647 op/s 2023-09-23T20:00:16.306 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:16.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:16.321 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:16.422 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:16.457 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:16.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:16.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:16.562 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:16.594 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:16.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:16.599 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:16.694 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:00:16.700 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:16.731 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:16.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:16.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:16.839 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:16.888 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:16.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:16.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:16.993 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:17.037 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:17.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:17.038 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:17.139 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:17.168 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:17.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:17.170 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:17.271 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:17.299 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:17.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:17.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:17.404 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:17.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:17.444 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:17.445 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:17.546 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:17.589 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:17.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:17.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:17.697 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:17.726 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:17.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:17.729 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:17.830 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:17.857 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:17.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:17.859 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:17.960 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:17.994 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:17.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:17.997 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:18.098 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:18.141 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:18.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:18.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:18.243 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:18.253 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:17 smithi016 bash[75004]: cluster 2023-09-23T20:00:16.623502+0000 mgr.y (mgr.25325) 1591 : cluster 0 pgmap v1623: 89 pgs: 89 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2023-09-23T20:00:18.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:18 smithi016 bash[77995]: cluster 2023-09-23T20:00:16.623502+0000 mgr.y (mgr.25325) 1591 : cluster 0 pgmap v1623: 89 pgs: 89 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2023-09-23T20:00:18.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:18.281 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:18.281 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:18.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:17 smithi092 bash[58855]: cluster 2023-09-23T20:00:16.623502+0000 mgr.y (mgr.25325) 1591 : cluster 0 pgmap v1623: 89 pgs: 89 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2023-09-23T20:00:18.382 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:18.405 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:18.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:18.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:18.511 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:18.541 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:18.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:18.544 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:18.644 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:18.666 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:18.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:18.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:18.769 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:18.802 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:18.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:18.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:18.909 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:18.941 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:18.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:18.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:19.015 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T20:00:19.048 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:19.053 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T20:00:19.082 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:19.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:19.085 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:19.094 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T20:00:19.146 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T20:00:19.179 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T20:00:19.186 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:19.194 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:18 smithi016 bash[75004]: audit 2023-09-23T20:00:18.944848+0000 mon.a (mon.0) 1910 : audit 0 from='client.? 172.21.15.16:0/2012577944' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:00:19.195 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:18 smithi016 bash[77995]: audit 2023-09-23T20:00:18.944848+0000 mon.a (mon.0) 1910 : audit 0 from='client.? 172.21.15.16:0/2012577944' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:00:19.211 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T20:00:19.230 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:19.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:19.239 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:19.265 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T20:00:19.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:18 smithi092 bash[58855]: audit 2023-09-23T20:00:18.944848+0000 mon.a (mon.0) 1910 : audit 0 from='client.? 172.21.15.16:0/2012577944' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:00:19.308 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T20:00:19.323 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:00:19.323 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:00:19.323 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:00:19.323 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:00:19.323 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:00:19.323 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:00:19.323 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:00:19.323 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:00:19.340 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:19.369 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:19.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:19.374 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:19.475 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:19.501 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:19.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:19.504 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:19.605 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:19.655 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:19.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:19.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:19.757 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:19.793 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:19.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:19.798 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:19.899 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:19.928 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:19.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:19.930 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:20.031 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:20.057 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:20.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:20.059 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:20.100 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:20 smithi016 bash[77995]: cluster 2023-09-23T20:00:18.623987+0000 mgr.y (mgr.25325) 1592 : cluster 0 pgmap v1624: 89 pgs: 89 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 611 op/s 2023-09-23T20:00:20.100 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:20 smithi016 bash[77995]: audit 2023-09-23T20:00:19.859908+0000 mon.a (mon.0) 1911 : audit 1 from='client.? 172.21.15.16:0/1468777411' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:00:20.100 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:20 smithi016 bash[77995]: audit 2023-09-23T20:00:19.926936+0000 mon.a (mon.0) 1912 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:00:20.100 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:20 smithi016 bash[77995]: audit 2023-09-23T20:00:19.930926+0000 mon.b (mon.1) 61 : audit 1 from='client.? 172.21.15.16:0/991126838' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:00:20.101 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:20 smithi016 bash[75004]: cluster 2023-09-23T20:00:18.623987+0000 mgr.y (mgr.25325) 1592 : cluster 0 pgmap v1624: 89 pgs: 89 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 611 op/s 2023-09-23T20:00:20.101 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:20 smithi016 bash[75004]: audit 2023-09-23T20:00:19.859908+0000 mon.a (mon.0) 1911 : audit 1 from='client.? 172.21.15.16:0/1468777411' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:00:20.101 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:20 smithi016 bash[75004]: audit 2023-09-23T20:00:19.926936+0000 mon.a (mon.0) 1912 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:00:20.101 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:20 smithi016 bash[75004]: audit 2023-09-23T20:00:19.930926+0000 mon.b (mon.1) 61 : audit 1 from='client.? 172.21.15.16:0/991126838' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:00:20.161 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:20.202 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:20.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:20.205 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:20.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:19 smithi092 bash[58855]: cluster 2023-09-23T20:00:18.623987+0000 mgr.y (mgr.25325) 1592 : cluster 0 pgmap v1624: 89 pgs: 89 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 611 op/s 2023-09-23T20:00:20.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:19 smithi092 bash[58855]: audit 2023-09-23T20:00:19.859908+0000 mon.a (mon.0) 1911 : audit 1 from='client.? 172.21.15.16:0/1468777411' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:00:20.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:19 smithi092 bash[58855]: audit 2023-09-23T20:00:19.926936+0000 mon.a (mon.0) 1912 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:00:20.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:19 smithi092 bash[58855]: audit 2023-09-23T20:00:19.930926+0000 mon.b (mon.1) 61 : audit 1 from='client.? 172.21.15.16:0/991126838' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:00:20.306 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:20.345 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:20.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:20.348 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:00:20.449 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:20.479 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:20.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:20.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:20.583 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:20.623 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:20.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:20.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:20.728 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:20.753 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:20.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:20.757 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:20.813 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is unset 2023-09-23T20:00:20.826 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T20:00:20.857 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:20.884 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:20.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:20.889 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:20.989 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:21.019 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:21.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:21.023 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:21.123 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:21.155 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:21.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:21.158 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:21.239 INFO:teuthology.orchestra.run.smithi092.stdout:ERROR: (22) Invalid argument 2023-09-23T20:00:21.239 INFO:teuthology.orchestra.run.smithi092.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:00:21.259 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:21.268 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:20 smithi092 bash[58855]: cluster 2023-09-23T20:00:20.793553+0000 mon.a (mon.0) 1913 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:00:21.268 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:20 smithi092 bash[58855]: cluster 2023-09-23T20:00:20.793585+0000 mon.a (mon.0) 1914 : cluster 1 Cluster is now healthy 2023-09-23T20:00:21.268 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:20 smithi092 bash[58855]: audit 2023-09-23T20:00:20.803757+0000 mon.a (mon.0) 1915 : audit 1 from='client.? 172.21.15.16:0/1468777411' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:00:21.268 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:20 smithi092 bash[58855]: audit 2023-09-23T20:00:20.804048+0000 mon.a (mon.0) 1916 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:00:21.269 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:20 smithi092 bash[58855]: cluster 2023-09-23T20:00:20.804375+0000 mon.a (mon.0) 1917 : cluster 0 osdmap e270: 8 total, 8 up, 8 in 2023-09-23T20:00:21.269 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:20 smithi092 bash[58855]: cluster 2023-09-23T20:00:20.816312+0000 osd.6 (osd.6) 205 : cluster 3 failed to encode map e270 with expected crc 2023-09-23T20:00:21.269 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:20 smithi016 bash[75004]: cluster 2023-09-23T20:00:20.793553+0000 mon.a (mon.0) 1913 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:00:21.269 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:20 smithi016 bash[75004]: cluster 2023-09-23T20:00:20.793585+0000 mon.a (mon.0) 1914 : cluster 1 Cluster is now healthy 2023-09-23T20:00:21.269 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:20 smithi016 bash[75004]: audit 2023-09-23T20:00:20.803757+0000 mon.a (mon.0) 1915 : audit 1 from='client.? 172.21.15.16:0/1468777411' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:00:21.269 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:20 smithi016 bash[75004]: audit 2023-09-23T20:00:20.804048+0000 mon.a (mon.0) 1916 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:00:21.269 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:20 smithi016 bash[75004]: cluster 2023-09-23T20:00:20.804375+0000 mon.a (mon.0) 1917 : cluster 0 osdmap e270: 8 total, 8 up, 8 in 2023-09-23T20:00:21.269 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:20 smithi016 bash[75004]: cluster 2023-09-23T20:00:20.816312+0000 osd.6 (osd.6) 205 : cluster 3 failed to encode map e270 with expected crc 2023-09-23T20:00:21.270 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:20 smithi016 bash[77995]: cluster 2023-09-23T20:00:20.793553+0000 mon.a (mon.0) 1913 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:00:21.270 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:20 smithi016 bash[77995]: cluster 2023-09-23T20:00:20.793585+0000 mon.a (mon.0) 1914 : cluster 1 Cluster is now healthy 2023-09-23T20:00:21.270 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:20 smithi016 bash[77995]: audit 2023-09-23T20:00:20.803757+0000 mon.a (mon.0) 1915 : audit 1 from='client.? 172.21.15.16:0/1468777411' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:00:21.270 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:20 smithi016 bash[77995]: audit 2023-09-23T20:00:20.804048+0000 mon.a (mon.0) 1916 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:00:21.270 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:20 smithi016 bash[77995]: cluster 2023-09-23T20:00:20.804375+0000 mon.a (mon.0) 1917 : cluster 0 osdmap e270: 8 total, 8 up, 8 in 2023-09-23T20:00:21.270 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:20 smithi016 bash[77995]: cluster 2023-09-23T20:00:20.816312+0000 osd.6 (osd.6) 205 : cluster 3 failed to encode map e270 with expected crc 2023-09-23T20:00:21.292 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:21.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:21.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:21.401 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:21.422 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:21.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:21.425 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:21.525 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:21.547 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:21.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:21.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:21.651 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:21.675 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:21.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:21.682 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:21.783 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:21.807 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:21.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:21.809 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:21.910 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:21.937 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:21.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:21.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:00:22.044 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:22.079 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:22.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:22.087 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:22.188 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:22.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:22.225 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:22.225 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:22.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:21 smithi016 bash[77995]: cluster 2023-09-23T20:00:20.624662+0000 mgr.y (mgr.25325) 1593 : cluster 0 pgmap v1625: 89 pgs: 89 active+clean; 2.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2023-09-23T20:00:22.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:21 smithi016 bash[77995]: cluster 2023-09-23T20:00:20.814492+0000 osd.4 (osd.4) 209 : cluster 3 failed to encode map e270 with expected crc 2023-09-23T20:00:22.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:21 smithi016 bash[77995]: cluster 2023-09-23T20:00:20.815923+0000 osd.5 (osd.5) 194 : cluster 3 failed to encode map e270 with expected crc 2023-09-23T20:00:22.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:21 smithi016 bash[77995]: cluster 2023-09-23T20:00:20.821739+0000 osd.7 (osd.7) 184 : cluster 3 failed to encode map e270 with expected crc 2023-09-23T20:00:22.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:21 smithi016 bash[75004]: cluster 2023-09-23T20:00:20.624662+0000 mgr.y (mgr.25325) 1593 : cluster 0 pgmap v1625: 89 pgs: 89 active+clean; 2.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2023-09-23T20:00:22.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:21 smithi016 bash[75004]: cluster 2023-09-23T20:00:20.814492+0000 osd.4 (osd.4) 209 : cluster 3 failed to encode map e270 with expected crc 2023-09-23T20:00:22.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:21 smithi016 bash[75004]: cluster 2023-09-23T20:00:20.815923+0000 osd.5 (osd.5) 194 : cluster 3 failed to encode map e270 with expected crc 2023-09-23T20:00:22.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:21 smithi016 bash[75004]: cluster 2023-09-23T20:00:20.821739+0000 osd.7 (osd.7) 184 : cluster 3 failed to encode map e270 with expected crc 2023-09-23T20:00:22.287 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:22 smithi092 bash[58855]: cluster 2023-09-23T20:00:20.624662+0000 mgr.y (mgr.25325) 1593 : cluster 0 pgmap v1625: 89 pgs: 89 active+clean; 2.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2023-09-23T20:00:22.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:22 smithi092 bash[58855]: cluster 2023-09-23T20:00:20.814492+0000 osd.4 (osd.4) 209 : cluster 3 failed to encode map e270 with expected crc 2023-09-23T20:00:22.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:22 smithi092 bash[58855]: cluster 2023-09-23T20:00:20.815923+0000 osd.5 (osd.5) 194 : cluster 3 failed to encode map e270 with expected crc 2023-09-23T20:00:22.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:22 smithi092 bash[58855]: cluster 2023-09-23T20:00:20.821739+0000 osd.7 (osd.7) 184 : cluster 3 failed to encode map e270 with expected crc 2023-09-23T20:00:22.326 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:22.360 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:22.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:22.368 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:22.407 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T20:00:22.468 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:22.505 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:22.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:22.507 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:22.590 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T20:00:22.608 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:22.638 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:22.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:22.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:22.671 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:00:22.747 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:22.770 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:22.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:22.773 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:22.874 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:22.904 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:22.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:22.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:23.008 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:23.032 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:23.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:23.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:23.136 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:23.162 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:23.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:23.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:23.269 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:23.297 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:23.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:23.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:23.404 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:23.433 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:23.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:23.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:23.537 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:23.572 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:23.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:23.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:23.678 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:23.710 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:23.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:23.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:23.813 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:23.840 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:23.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:23.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:23.947 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:23.985 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:23.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:23.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:24.089 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:24.100 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:23 smithi016 bash[75004]: cluster 2023-09-23T20:00:22.625245+0000 mgr.y (mgr.25325) 1594 : cluster 0 pgmap v1627: 89 pgs: 89 active+clean; 2.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 625 op/s 2023-09-23T20:00:24.101 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:23 smithi016 bash[75004]: audit 2023-09-23T20:00:23.802725+0000 mon.a (mon.0) 1918 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:00:24.101 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:23 smithi016 bash[77995]: cluster 2023-09-23T20:00:22.625245+0000 mgr.y (mgr.25325) 1594 : cluster 0 pgmap v1627: 89 pgs: 89 active+clean; 2.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 625 op/s 2023-09-23T20:00:24.101 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:23 smithi016 bash[77995]: audit 2023-09-23T20:00:23.802725+0000 mon.a (mon.0) 1918 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:00:24.120 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:24.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:24.123 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:24.224 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:24.235 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:24 smithi092 bash[58855]: cluster 2023-09-23T20:00:22.625245+0000 mgr.y (mgr.25325) 1594 : cluster 0 pgmap v1627: 89 pgs: 89 active+clean; 2.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 625 op/s 2023-09-23T20:00:24.235 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:24 smithi092 bash[58855]: audit 2023-09-23T20:00:23.802725+0000 mon.a (mon.0) 1918 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:00:24.251 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:24.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:24.254 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:24.355 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:24.395 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:24.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:24.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:24.504 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:24.534 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:24.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:24.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:00:24.611 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:00:24.637 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:24.667 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:24.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:24.670 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:00:24.770 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:24.796 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:24.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:24.802 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:24.903 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:24.939 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:24.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:24.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:25.044 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:25.073 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:25.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:25.078 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:25.178 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:25.204 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:25.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:25.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:25.310 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:25.319 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:25 smithi092 bash[58855]: audit 2023-09-23T20:00:24.293129+0000 mon.a (mon.0) 1919 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:00:25.319 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:25 smithi092 bash[58855]: audit 2023-09-23T20:00:24.304787+0000 mon.a (mon.0) 1920 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:00:25.319 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:25 smithi092 bash[58855]: audit 2023-09-23T20:00:24.399199+0000 mon.a (mon.0) 1921 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:00:25.319 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:25 smithi092 bash[58855]: audit 2023-09-23T20:00:24.409627+0000 mon.a (mon.0) 1922 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:00:25.340 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:25.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:25.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:25.444 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:25.468 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:25.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:25.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:25.571 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:25.609 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:25.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:25.611 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:00:25.641 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:25 smithi016 bash[75004]: audit 2023-09-23T20:00:24.293129+0000 mon.a (mon.0) 1919 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:00:25.642 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:25 smithi016 bash[75004]: audit 2023-09-23T20:00:24.304787+0000 mon.a (mon.0) 1920 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:00:25.642 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:25 smithi016 bash[75004]: audit 2023-09-23T20:00:24.399199+0000 mon.a (mon.0) 1921 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:00:25.642 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:25 smithi016 bash[75004]: audit 2023-09-23T20:00:24.409627+0000 mon.a (mon.0) 1922 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:00:25.642 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:00:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:20:00:25] "GET /metrics HTTP/1.1" 200 36374 "" "Prometheus/2.33.4" 2023-09-23T20:00:25.642 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:25 smithi016 bash[77995]: audit 2023-09-23T20:00:24.293129+0000 mon.a (mon.0) 1919 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:00:25.642 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:25 smithi016 bash[77995]: audit 2023-09-23T20:00:24.304787+0000 mon.a (mon.0) 1920 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:00:25.642 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:25 smithi016 bash[77995]: audit 2023-09-23T20:00:24.399199+0000 mon.a (mon.0) 1921 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:00:25.643 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:25 smithi016 bash[77995]: audit 2023-09-23T20:00:24.409627+0000 mon.a (mon.0) 1922 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:00:25.712 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:25.756 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:25.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:25.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:25.861 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:25.890 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:25.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:25.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:25.998 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:26.020 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:26.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:26.022 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:26.123 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:26.161 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:26.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:26.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:26.265 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:26.318 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:26.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:26.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:26.422 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:26.434 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:26 smithi016 bash[77995]: cluster 2023-09-23T20:00:24.628324+0000 mgr.y (mgr.25325) 1595 : cluster 0 pgmap v1628: 89 pgs: 89 active+clean; 2.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-09-23T20:00:26.434 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:26 smithi016 bash[75004]: cluster 2023-09-23T20:00:24.628324+0000 mgr.y (mgr.25325) 1595 : cluster 0 pgmap v1628: 89 pgs: 89 active+clean; 2.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-09-23T20:00:26.456 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:26.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:26.462 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:26.563 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:26.586 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:26.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:26.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:00:26.689 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:26.697 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:26 smithi092 bash[58855]: cluster 2023-09-23T20:00:24.628324+0000 mgr.y (mgr.25325) 1595 : cluster 0 pgmap v1628: 89 pgs: 89 active+clean; 2.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-09-23T20:00:26.725 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:26.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:26.728 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:26.829 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:26.854 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:26.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:26.856 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:26.957 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:26.998 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:27.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:27.003 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:27.105 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:27.136 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:27.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:27.140 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:27.217 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:00:27.241 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:27.276 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:27.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:27.282 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:27.383 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:27.402 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:27.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:27.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:27.440 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:00:27.446 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:00:27.512 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:27.534 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:27.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:27.544 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:00:27.644 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:27.673 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:27.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:27.676 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:27.777 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:27.801 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:27.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:27.804 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:00:27.905 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:27.930 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:27.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:27.939 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:28.039 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:28.087 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:28.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:28.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:28.193 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:28.226 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:28.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:28.228 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:28.329 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:28.390 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:28.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:28.397 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:28.418 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:28 smithi016 bash[75004]: cluster 2023-09-23T20:00:26.629026+0000 mgr.y (mgr.25325) 1596 : cluster 0 pgmap v1629: 89 pgs: 89 active+clean; 2.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 618 op/s 2023-09-23T20:00:28.419 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:28 smithi016 bash[77995]: cluster 2023-09-23T20:00:26.629026+0000 mgr.y (mgr.25325) 1596 : cluster 0 pgmap v1629: 89 pgs: 89 active+clean; 2.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 618 op/s 2023-09-23T20:00:28.497 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:28.515 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:28.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:28.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:28.618 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:28.629 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:28 smithi092 bash[58855]: cluster 2023-09-23T20:00:26.629026+0000 mgr.y (mgr.25325) 1596 : cluster 0 pgmap v1629: 89 pgs: 89 active+clean; 2.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 618 op/s 2023-09-23T20:00:28.643 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:28.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:28.646 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:28.747 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:28.775 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:28.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:28.779 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:28.880 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:28.913 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:28.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:28.914 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:29.015 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:29.052 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:29.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:29.056 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:29.156 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:29.189 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:29.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:29.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:29.295 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:29.331 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:29.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:29.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:29.441 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:29.485 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:00:29.489 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:29.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:29.495 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:29.595 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:29.622 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:29.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:29.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:29.725 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:29.761 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:29.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:29.765 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:29.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:29 smithi016 bash[75004]: cluster 2023-09-23T20:00:28.629776+0000 mgr.y (mgr.25325) 1597 : cluster 0 pgmap v1630: 89 pgs: 89 active+clean; 1.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 614 op/s 2023-09-23T20:00:29.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:29 smithi016 bash[75004]: audit 2023-09-23T20:00:28.702073+0000 mon.a (mon.0) 1923 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:00:29.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:29 smithi016 bash[75004]: audit 2023-09-23T20:00:28.708628+0000 mon.a (mon.0) 1924 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:00:29.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:29 smithi016 bash[75004]: audit 2023-09-23T20:00:29.662906+0000 mon.a (mon.0) 1925 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:00:29.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:29 smithi016 bash[75004]: audit 2023-09-23T20:00:29.670562+0000 mon.a (mon.0) 1926 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:00:29.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:29 smithi016 bash[77995]: cluster 2023-09-23T20:00:28.629776+0000 mgr.y (mgr.25325) 1597 : cluster 0 pgmap v1630: 89 pgs: 89 active+clean; 1.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 614 op/s 2023-09-23T20:00:29.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:29 smithi016 bash[77995]: audit 2023-09-23T20:00:28.702073+0000 mon.a (mon.0) 1923 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:00:29.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:29 smithi016 bash[77995]: audit 2023-09-23T20:00:28.708628+0000 mon.a (mon.0) 1924 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:00:29.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:29 smithi016 bash[77995]: audit 2023-09-23T20:00:29.662906+0000 mon.a (mon.0) 1925 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:00:29.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:29 smithi016 bash[77995]: audit 2023-09-23T20:00:29.670562+0000 mon.a (mon.0) 1926 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:00:29.866 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:29.891 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:29.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:29.896 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:29.997 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:30.034 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:30.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:29 smithi092 bash[58855]: cluster 2023-09-23T20:00:28.629776+0000 mgr.y (mgr.25325) 1597 : cluster 0 pgmap v1630: 89 pgs: 89 active+clean; 1.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 614 op/s 2023-09-23T20:00:30.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:29 smithi092 bash[58855]: audit 2023-09-23T20:00:28.702073+0000 mon.a (mon.0) 1923 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:00:30.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:29 smithi092 bash[58855]: audit 2023-09-23T20:00:28.708628+0000 mon.a (mon.0) 1924 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:00:30.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:29 smithi092 bash[58855]: audit 2023-09-23T20:00:29.662906+0000 mon.a (mon.0) 1925 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:00:30.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:29 smithi092 bash[58855]: audit 2023-09-23T20:00:29.670562+0000 mon.a (mon.0) 1926 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:00:30.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:30.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:30.142 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:30.166 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:30.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:30.169 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:30.269 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:30.291 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:30.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:30.299 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:30.400 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:30.421 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:30.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:30.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:30.528 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:30.556 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:30.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:30.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:30.671 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:30.699 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:30.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:30.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:30.803 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:30.826 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:30.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:30.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:30.929 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:30.950 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:30.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:30.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:31.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:30 smithi092 bash[58855]: audit 2023-09-23T20:00:29.823306+0000 mon.a (mon.0) 1927 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:00:31.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:30 smithi092 bash[58855]: audit 2023-09-23T20:00:29.833503+0000 mon.a (mon.0) 1928 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:00:31.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:30 smithi092 bash[58855]: audit 2023-09-23T20:00:30.466380+0000 mon.a (mon.0) 1929 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:00:31.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:30 smithi092 bash[58855]: audit 2023-09-23T20:00:30.467816+0000 mon.a (mon.0) 1930 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:00:31.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:30 smithi092 bash[58855]: audit 2023-09-23T20:00:30.476277+0000 mon.a (mon.0) 1931 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:00:31.058 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:31.093 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:31.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:31.096 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:31.197 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:31.226 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:31.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:31.229 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:31.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:30 smithi016 bash[75004]: audit 2023-09-23T20:00:29.823306+0000 mon.a (mon.0) 1927 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:00:31.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:30 smithi016 bash[75004]: audit 2023-09-23T20:00:29.833503+0000 mon.a (mon.0) 1928 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:00:31.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:30 smithi016 bash[75004]: audit 2023-09-23T20:00:30.466380+0000 mon.a (mon.0) 1929 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:00:31.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:30 smithi016 bash[75004]: audit 2023-09-23T20:00:30.467816+0000 mon.a (mon.0) 1930 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:00:31.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:30 smithi016 bash[75004]: audit 2023-09-23T20:00:30.476277+0000 mon.a (mon.0) 1931 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:00:31.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:30 smithi016 bash[77995]: audit 2023-09-23T20:00:29.823306+0000 mon.a (mon.0) 1927 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:00:31.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:30 smithi016 bash[77995]: audit 2023-09-23T20:00:29.833503+0000 mon.a (mon.0) 1928 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:00:31.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:30 smithi016 bash[77995]: audit 2023-09-23T20:00:30.466380+0000 mon.a (mon.0) 1929 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:00:31.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:30 smithi016 bash[77995]: audit 2023-09-23T20:00:30.467816+0000 mon.a (mon.0) 1930 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:00:31.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:30 smithi016 bash[77995]: audit 2023-09-23T20:00:30.476277+0000 mon.a (mon.0) 1931 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:00:31.330 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:31.352 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:31.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:31.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:31.455 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:31.488 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:31.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:31.490 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:31.591 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:31.619 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:31.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:31.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:31.723 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:31.762 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:31.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:31.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:31.865 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:31.892 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:31.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:31.897 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:31.997 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:32.020 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:32.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:32.022 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:32.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:31 smithi016 bash[77995]: cluster 2023-09-23T20:00:30.630319+0000 mgr.y (mgr.25325) 1598 : cluster 0 pgmap v1631: 89 pgs: 89 active+clean; 1.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 596 op/s 2023-09-23T20:00:32.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:31 smithi016 bash[75004]: cluster 2023-09-23T20:00:30.630319+0000 mgr.y (mgr.25325) 1598 : cluster 0 pgmap v1631: 89 pgs: 89 active+clean; 1.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 596 op/s 2023-09-23T20:00:32.098 INFO:teuthology.orchestra.run.smithi092.stdout:ERROR: (22) Invalid argument 2023-09-23T20:00:32.098 INFO:teuthology.orchestra.run.smithi092.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:00:32.124 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:32.131 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:31 smithi092 bash[58855]: cluster 2023-09-23T20:00:30.630319+0000 mgr.y (mgr.25325) 1598 : cluster 0 pgmap v1631: 89 pgs: 89 active+clean; 1.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 596 op/s 2023-09-23T20:00:32.149 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:32.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:32.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:32.252 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:32.275 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:32.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:32.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:32.379 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:32.406 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:32.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:32.409 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:32.510 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:32.544 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:32.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:32.550 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:32.650 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:32.682 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:32.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:32.683 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:32.784 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:32.811 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:32.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:32.817 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:32.918 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:32.946 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:32.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:32.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:33.050 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:33.084 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:33.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:33.087 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:33.187 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:33.208 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T20:00:33.216 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:33.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:33.221 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:33.322 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:33.359 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:33.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:33.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:33.462 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:33.500 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:33.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:33.505 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:33.606 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:33.629 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:33.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:33.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:00:33.732 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:33.766 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:33.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:33.769 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:00:33.870 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:33.880 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:33 smithi016 bash[75004]: cluster 2023-09-23T20:00:32.630828+0000 mgr.y (mgr.25325) 1599 : cluster 0 pgmap v1632: 89 pgs: 89 active+clean; 1.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 550 op/s 2023-09-23T20:00:33.880 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:33 smithi016 bash[77995]: cluster 2023-09-23T20:00:32.630828+0000 mgr.y (mgr.25325) 1599 : cluster 0 pgmap v1632: 89 pgs: 89 active+clean; 1.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 550 op/s 2023-09-23T20:00:33.905 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:33.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:33.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:34.010 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:34.020 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:33 smithi092 bash[58855]: cluster 2023-09-23T20:00:32.630828+0000 mgr.y (mgr.25325) 1599 : cluster 0 pgmap v1632: 89 pgs: 89 active+clean; 1.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 550 op/s 2023-09-23T20:00:34.041 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:34.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:34.044 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:00:34.144 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:34.172 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:34.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:34.177 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:34.277 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:34.319 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:34.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:34.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:34.425 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:34.453 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:34.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:34.456 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:34.556 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:34.586 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:34.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:34.592 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:34.692 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:34.719 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:34.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:34.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:00:34.825 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:34.868 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:34.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:34.873 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:34.973 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:35.003 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:35.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:35.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:35.108 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:35.151 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:35.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:35.156 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:35.257 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:35.287 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:35.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:35.289 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:35.389 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:35.418 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:35.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:35.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:00:35.521 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:35.560 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:35.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:35.565 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:35.666 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:35.704 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:35.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:35.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:35.814 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:35.847 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:35.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:35.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:35.882 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:00:35 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:20:00:35] "GET /metrics HTTP/1.1" 200 36377 "" "Prometheus/2.33.4" 2023-09-23T20:00:35.952 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:35.978 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:35.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:35.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:36.085 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:36.094 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:35 smithi092 bash[58855]: cluster 2023-09-23T20:00:34.631420+0000 mgr.y (mgr.25325) 1600 : cluster 0 pgmap v1633: 89 pgs: 89 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 605 op/s 2023-09-23T20:00:36.111 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:36.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:36.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:36.214 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:36.241 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:36.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:36.249 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:36.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:35 smithi016 bash[75004]: cluster 2023-09-23T20:00:34.631420+0000 mgr.y (mgr.25325) 1600 : cluster 0 pgmap v1633: 89 pgs: 89 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 605 op/s 2023-09-23T20:00:36.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:35 smithi016 bash[77995]: cluster 2023-09-23T20:00:34.631420+0000 mgr.y (mgr.25325) 1600 : cluster 0 pgmap v1633: 89 pgs: 89 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 605 op/s 2023-09-23T20:00:36.350 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:36.386 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:36.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:36.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:36.490 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:36.529 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:36.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:36.533 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:36.633 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:36.664 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:36.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:36.669 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:36.769 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:36.794 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:36.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:36.798 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:36.899 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:36.922 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:36.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:36.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:00:37.027 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:37.068 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:37.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:37.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:37.172 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:37.199 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:37.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:37.204 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:37.305 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:37.334 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:37.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:37.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:37.441 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:37.465 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:37.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:37.471 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:37.572 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:37.603 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:37.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:37.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:37.710 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:37.744 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:37.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:37.747 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:37.848 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:37.892 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:37.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:37.897 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:00:37.925 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:37 smithi016 bash[77995]: cluster 2023-09-23T20:00:36.632125+0000 mgr.y (mgr.25325) 1601 : cluster 0 pgmap v1634: 89 pgs: 89 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2023-09-23T20:00:37.925 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:37 smithi016 bash[75004]: cluster 2023-09-23T20:00:36.632125+0000 mgr.y (mgr.25325) 1601 : cluster 0 pgmap v1634: 89 pgs: 89 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2023-09-23T20:00:37.998 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:38.010 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:00:38.026 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:38.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:38.030 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:38.130 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:38.138 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:37 smithi092 bash[58855]: cluster 2023-09-23T20:00:36.632125+0000 mgr.y (mgr.25325) 1601 : cluster 0 pgmap v1634: 89 pgs: 89 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2023-09-23T20:00:38.154 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:38.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:38.160 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:38.261 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:38.299 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:38.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:38.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:38.406 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:38.441 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:38.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:38.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:38.545 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:38.573 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:38.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:38.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:38.678 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:38.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:38.712 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:38.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:38.813 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:38.849 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:38.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:38.853 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:38.954 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:38.988 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:38.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:38.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:00:39.094 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:39.123 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:39.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:39.126 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:39.226 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:39.257 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:39.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:39.261 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:39.362 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:39.394 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:39.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:39.397 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:39.498 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:39.525 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:39.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:39.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:39.630 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:39.654 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:39.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:39.657 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:39.758 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:39.784 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:39.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:39.786 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:39.887 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:39.927 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:39.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:39.932 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:39.979 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:39 smithi016 bash[75004]: cluster 2023-09-23T20:00:38.632665+0000 mgr.y (mgr.25325) 1602 : cluster 0 pgmap v1635: 89 pgs: 89 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-09-23T20:00:39.979 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:39 smithi016 bash[77995]: cluster 2023-09-23T20:00:38.632665+0000 mgr.y (mgr.25325) 1602 : cluster 0 pgmap v1635: 89 pgs: 89 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-09-23T20:00:40.033 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:40.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:39 smithi092 bash[58855]: cluster 2023-09-23T20:00:38.632665+0000 mgr.y (mgr.25325) 1602 : cluster 0 pgmap v1635: 89 pgs: 89 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-09-23T20:00:40.068 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T20:00:40.069 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:40.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:40.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:40.140 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T20:00:40.173 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:40.195 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T20:00:40.225 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:40.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:40.232 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:40.245 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T20:00:40.289 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T20:00:40.332 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:40.334 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T20:00:40.373 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:40.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:40.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:40.380 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T20:00:40.428 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T20:00:40.433 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:00:40.433 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:00:40.434 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:00:40.434 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:00:40.434 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:00:40.434 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:00:40.434 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:00:40.434 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:00:40.481 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:40.510 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:40.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:40.515 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:40.615 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:40.663 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:40.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:40.672 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:40.773 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:40.794 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:40.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:40.798 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:40.864 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T20:00:40.864 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is set 2023-09-23T20:00:40.899 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:40.906 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:40 smithi016 bash[75004]: audit 2023-09-23T20:00:39.986212+0000 mon.c (mon.2) 241 : audit 0 from='client.? 172.21.15.16:0/3011110055' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:00:40.906 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:40 smithi016 bash[75004]: audit 2023-09-23T20:00:40.505421+0000 mon.a (mon.0) 1932 : audit 1 from='client.? 172.21.15.16:0/627953810' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:00:40.906 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:40 smithi016 bash[75004]: audit 2023-09-23T20:00:40.622076+0000 mon.a (mon.0) 1933 : audit 1 from='client.? 172.21.15.16:0/993699615' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:00:40.907 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:40 smithi016 bash[77995]: audit 2023-09-23T20:00:39.986212+0000 mon.c (mon.2) 241 : audit 0 from='client.? 172.21.15.16:0/3011110055' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:00:40.907 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:40 smithi016 bash[77995]: audit 2023-09-23T20:00:40.505421+0000 mon.a (mon.0) 1932 : audit 1 from='client.? 172.21.15.16:0/627953810' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:00:40.907 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:40 smithi016 bash[77995]: audit 2023-09-23T20:00:40.622076+0000 mon.a (mon.0) 1933 : audit 1 from='client.? 172.21.15.16:0/993699615' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:00:40.929 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:40.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:40.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:41.032 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:41.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:40 smithi092 bash[58855]: audit 2023-09-23T20:00:39.986212+0000 mon.c (mon.2) 241 : audit 0 from='client.? 172.21.15.16:0/3011110055' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:00:41.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:40 smithi092 bash[58855]: audit 2023-09-23T20:00:40.505421+0000 mon.a (mon.0) 1932 : audit 1 from='client.? 172.21.15.16:0/627953810' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:00:41.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:40 smithi092 bash[58855]: audit 2023-09-23T20:00:40.622076+0000 mon.a (mon.0) 1933 : audit 1 from='client.? 172.21.15.16:0/993699615' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:00:41.064 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:41.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:41.071 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:41.172 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:41.202 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:41.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:41.204 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:41.305 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:41.336 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:41.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:41.342 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:41.443 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:41.479 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:41.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:41.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:41.584 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:41.601 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:41.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:41.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:41.704 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:41.737 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:41.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:41.743 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:41.843 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:41.868 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:41.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:41.875 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:41.976 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:41.997 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:41.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:41.999 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:42.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:41 smithi092 bash[58855]: cluster 2023-09-23T20:00:40.633305+0000 mgr.y (mgr.25325) 1603 : cluster 0 pgmap v1636: 89 pgs: 89 active+clean; 1.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 637 op/s 2023-09-23T20:00:42.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:41 smithi092 bash[58855]: cluster 2023-09-23T20:00:40.837660+0000 mon.a (mon.0) 1934 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:00:42.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:41 smithi092 bash[58855]: audit 2023-09-23T20:00:40.852238+0000 mon.a (mon.0) 1935 : audit 1 from='client.? 172.21.15.16:0/627953810' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T20:00:42.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:41 smithi092 bash[58855]: audit 2023-09-23T20:00:40.852511+0000 mon.a (mon.0) 1936 : audit 1 from='client.? 172.21.15.16:0/993699615' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:00:42.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:41 smithi092 bash[58855]: cluster 2023-09-23T20:00:40.856729+0000 mon.a (mon.0) 1937 : cluster 0 osdmap e271: 8 total, 8 up, 8 in 2023-09-23T20:00:42.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:41 smithi092 bash[58855]: cluster 2023-09-23T20:00:40.862290+0000 osd.4 (osd.4) 210 : cluster 3 failed to encode map e271 with expected crc 2023-09-23T20:00:42.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:41 smithi092 bash[58855]: cluster 2023-09-23T20:00:40.864140+0000 osd.6 (osd.6) 206 : cluster 3 failed to encode map e271 with expected crc 2023-09-23T20:00:42.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:41 smithi092 bash[58855]: cluster 2023-09-23T20:00:40.874905+0000 osd.7 (osd.7) 185 : cluster 3 failed to encode map e271 with expected crc 2023-09-23T20:00:42.100 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:42.116 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:41 smithi016 bash[75004]: cluster 2023-09-23T20:00:40.633305+0000 mgr.y (mgr.25325) 1603 : cluster 0 pgmap v1636: 89 pgs: 89 active+clean; 1.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 637 op/s 2023-09-23T20:00:42.116 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:41 smithi016 bash[75004]: cluster 2023-09-23T20:00:40.837660+0000 mon.a (mon.0) 1934 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:00:42.116 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:41 smithi016 bash[75004]: audit 2023-09-23T20:00:40.852238+0000 mon.a (mon.0) 1935 : audit 1 from='client.? 172.21.15.16:0/627953810' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T20:00:42.116 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:41 smithi016 bash[75004]: audit 2023-09-23T20:00:40.852511+0000 mon.a (mon.0) 1936 : audit 1 from='client.? 172.21.15.16:0/993699615' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:00:42.117 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:41 smithi016 bash[75004]: cluster 2023-09-23T20:00:40.856729+0000 mon.a (mon.0) 1937 : cluster 0 osdmap e271: 8 total, 8 up, 8 in 2023-09-23T20:00:42.117 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:41 smithi016 bash[75004]: cluster 2023-09-23T20:00:40.862290+0000 osd.4 (osd.4) 210 : cluster 3 failed to encode map e271 with expected crc 2023-09-23T20:00:42.117 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:41 smithi016 bash[75004]: cluster 2023-09-23T20:00:40.864140+0000 osd.6 (osd.6) 206 : cluster 3 failed to encode map e271 with expected crc 2023-09-23T20:00:42.117 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:41 smithi016 bash[75004]: cluster 2023-09-23T20:00:40.874905+0000 osd.7 (osd.7) 185 : cluster 3 failed to encode map e271 with expected crc 2023-09-23T20:00:42.117 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:41 smithi016 bash[77995]: cluster 2023-09-23T20:00:40.633305+0000 mgr.y (mgr.25325) 1603 : cluster 0 pgmap v1636: 89 pgs: 89 active+clean; 1.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 637 op/s 2023-09-23T20:00:42.117 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:41 smithi016 bash[77995]: cluster 2023-09-23T20:00:40.837660+0000 mon.a (mon.0) 1934 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:00:42.117 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:41 smithi016 bash[77995]: audit 2023-09-23T20:00:40.852238+0000 mon.a (mon.0) 1935 : audit 1 from='client.? 172.21.15.16:0/627953810' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T20:00:42.117 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:41 smithi016 bash[77995]: audit 2023-09-23T20:00:40.852511+0000 mon.a (mon.0) 1936 : audit 1 from='client.? 172.21.15.16:0/993699615' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:00:42.117 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:41 smithi016 bash[77995]: cluster 2023-09-23T20:00:40.856729+0000 mon.a (mon.0) 1937 : cluster 0 osdmap e271: 8 total, 8 up, 8 in 2023-09-23T20:00:42.118 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:41 smithi016 bash[77995]: cluster 2023-09-23T20:00:40.862290+0000 osd.4 (osd.4) 210 : cluster 3 failed to encode map e271 with expected crc 2023-09-23T20:00:42.118 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:41 smithi016 bash[77995]: cluster 2023-09-23T20:00:40.864140+0000 osd.6 (osd.6) 206 : cluster 3 failed to encode map e271 with expected crc 2023-09-23T20:00:42.118 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:41 smithi016 bash[77995]: cluster 2023-09-23T20:00:40.874905+0000 osd.7 (osd.7) 185 : cluster 3 failed to encode map e271 with expected crc 2023-09-23T20:00:42.122 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:42.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:42.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:42.228 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:42.252 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:42.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:42.255 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:42.356 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:42.397 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:42.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:42.400 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:42.501 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:42.528 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:42.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:42.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:42.632 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:42.660 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:42.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:42.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:42.763 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:42.822 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:42.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:42.827 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:42.928 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:42.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:42.954 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:42.954 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:43.034 INFO:teuthology.orchestra.run.smithi092.stdout:{ 2023-09-23T20:00:43.034 INFO:teuthology.orchestra.run.smithi092.stdout: "ops": [], 2023-09-23T20:00:43.034 INFO:teuthology.orchestra.run.smithi092.stdout: "complaint_time": 30, 2023-09-23T20:00:43.034 INFO:teuthology.orchestra.run.smithi092.stdout: "num_blocked_ops": 0 2023-09-23T20:00:43.034 INFO:teuthology.orchestra.run.smithi092.stdout:} 2023-09-23T20:00:43.055 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:43.090 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:43.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:43.097 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:43.197 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:43.221 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:43.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:43.224 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:43.324 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:43.357 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:43.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:43.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:43.461 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:43.489 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:43.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:43.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:00:43.594 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:43.615 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:43.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:43.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:43.665 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T20:00:43.717 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:43.744 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:43.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:43.747 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:43.847 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:43.856 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:43 smithi092 bash[58855]: cluster 2023-09-23T20:00:42.633874+0000 mgr.y (mgr.25325) 1604 : cluster 0 pgmap v1638: 89 pgs: 89 active+clean; 1.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 630 op/s 2023-09-23T20:00:43.856 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:43 smithi092 bash[58855]: audit 2023-09-23T20:00:43.706429+0000 mon.a (mon.0) 1938 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:00:43.856 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:43 smithi092 bash[58855]: audit 2023-09-23T20:00:43.713355+0000 mon.a (mon.0) 1939 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:00:43.873 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:43.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:43.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:00:43.976 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:44.001 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:44.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:44.003 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:44.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:43 smithi016 bash[77995]: cluster 2023-09-23T20:00:42.633874+0000 mgr.y (mgr.25325) 1604 : cluster 0 pgmap v1638: 89 pgs: 89 active+clean; 1.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 630 op/s 2023-09-23T20:00:44.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:43 smithi016 bash[77995]: audit 2023-09-23T20:00:43.706429+0000 mon.a (mon.0) 1938 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:00:44.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:43 smithi016 bash[77995]: audit 2023-09-23T20:00:43.713355+0000 mon.a (mon.0) 1939 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:00:44.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:43 smithi016 bash[75004]: cluster 2023-09-23T20:00:42.633874+0000 mgr.y (mgr.25325) 1604 : cluster 0 pgmap v1638: 89 pgs: 89 active+clean; 1.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 630 op/s 2023-09-23T20:00:44.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:43 smithi016 bash[75004]: audit 2023-09-23T20:00:43.706429+0000 mon.a (mon.0) 1938 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:00:44.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:43 smithi016 bash[75004]: audit 2023-09-23T20:00:43.713355+0000 mon.a (mon.0) 1939 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:00:44.104 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:44.130 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:44.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:44.136 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:44.146 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:00:44.169 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T20:00:44.237 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:44.269 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:44.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:44.272 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:44.373 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:44.401 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:44.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:44.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:44.506 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:44.545 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:44.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:44.553 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:44.654 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:44.676 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:44.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:44.679 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:00:44.780 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:44.804 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:44.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:44.809 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:44.909 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:44.935 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:44.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:44.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:45.043 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:45.081 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:45.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:45.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:45.185 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:45.213 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:45.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:45.215 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:45.316 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:45.346 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:45.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:45.353 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:45.454 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:45.479 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:45.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:45.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:45.584 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:45.610 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:45.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:45.612 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:45.688 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:00:45.713 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:45.737 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:45.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:45.745 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:45.845 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:45.884 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:45.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:45.892 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:45 smithi016 bash[75004]: cluster 2023-09-23T20:00:44.634497+0000 mgr.y (mgr.25325) 1605 : cluster 0 pgmap v1639: 89 pgs: 89 active+clean; 1.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 630 op/s 2023-09-23T20:00:45.893 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:00:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:20:00:45] "GET /metrics HTTP/1.1" 200 36373 "" "Prometheus/2.33.4" 2023-09-23T20:00:45.893 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:45 smithi016 bash[77995]: cluster 2023-09-23T20:00:44.634497+0000 mgr.y (mgr.25325) 1605 : cluster 0 pgmap v1639: 89 pgs: 89 active+clean; 1.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 630 op/s 2023-09-23T20:00:45.894 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:45.994 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:46.033 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:46.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:46.035 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:46.136 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:46.144 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:45 smithi092 bash[58855]: cluster 2023-09-23T20:00:44.634497+0000 mgr.y (mgr.25325) 1605 : cluster 0 pgmap v1639: 89 pgs: 89 active+clean; 1.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 630 op/s 2023-09-23T20:00:46.170 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:46.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:46.174 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:46.275 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:46.305 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:46.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:46.307 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:46.407 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:46.437 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:46.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:46.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:46.542 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:46.568 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:46.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:46.571 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:46.672 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:46.695 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:46.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:46.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:46.800 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:46.824 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:46.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:46.827 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:46.927 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:46.962 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:46.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:46.966 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:47.067 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:47.091 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:47.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:47.094 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:47.195 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:47.233 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:47.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:47.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:47.337 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:47.367 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:47.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:47.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:47.471 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:47.520 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:47.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:47.524 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:47.625 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:47.651 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:47.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:47.654 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:00:47.755 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:47.799 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:47.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:47.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:47.905 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:47.915 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:47 smithi092 bash[58855]: cluster 2023-09-23T20:00:46.635229+0000 mgr.y (mgr.25325) 1606 : cluster 0 pgmap v1640: 89 pgs: 89 active+clean; 1.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 630 op/s 2023-09-23T20:00:47.938 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:47.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:47.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:48.041 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:48.069 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:48.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:48.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:48.173 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:48.202 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:48.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:48.208 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:48.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:47 smithi016 bash[75004]: cluster 2023-09-23T20:00:46.635229+0000 mgr.y (mgr.25325) 1606 : cluster 0 pgmap v1640: 89 pgs: 89 active+clean; 1.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 630 op/s 2023-09-23T20:00:48.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:47 smithi016 bash[77995]: cluster 2023-09-23T20:00:46.635229+0000 mgr.y (mgr.25325) 1606 : cluster 0 pgmap v1640: 89 pgs: 89 active+clean; 1.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 630 op/s 2023-09-23T20:00:48.309 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:48.343 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:48.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:48.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:48.448 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:48.495 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:48.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:48.500 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:48.526 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:00:48.527 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:00:48.601 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:48.659 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:48.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:48.662 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:48.763 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:48.800 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:48.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:48.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:48.907 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:48.934 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:48.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:48.942 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:00:48.977 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:00:49.043 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:49.070 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:49.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:49.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:49.176 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:49.201 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:49.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:49.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:49.313 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:49.339 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:49.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:49.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:49.442 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:49.466 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:49.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:49.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:00:49.574 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:49.611 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:49.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:49.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:49.712 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:49.747 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:49.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:49.752 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:49.853 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:49.864 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:49 smithi016 bash[77995]: cluster 2023-09-23T20:00:48.635790+0000 mgr.y (mgr.25325) 1607 : cluster 0 pgmap v1641: 89 pgs: 89 active+clean; 1.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 622 op/s 2023-09-23T20:00:49.864 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:49 smithi016 bash[75004]: cluster 2023-09-23T20:00:48.635790+0000 mgr.y (mgr.25325) 1607 : cluster 0 pgmap v1641: 89 pgs: 89 active+clean; 1.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 622 op/s 2023-09-23T20:00:49.897 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:49.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:49.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:50.005 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:50.042 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:50.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:50.045 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:50.145 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:50.153 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:49 smithi092 bash[58855]: cluster 2023-09-23T20:00:48.635790+0000 mgr.y (mgr.25325) 1607 : cluster 0 pgmap v1641: 89 pgs: 89 active+clean; 1.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 622 op/s 2023-09-23T20:00:50.172 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:50.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:50.175 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:50.276 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:50.309 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:50.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:50.312 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:50.412 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:50.450 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:50.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:50.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:50.556 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:50.564 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:00:50.592 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:50.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:50.598 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:50.699 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:50.729 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:50.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:50.733 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:50.834 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:50.868 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:50.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:50.872 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:50.973 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:51.014 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:51.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:51.017 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:51.118 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:51.144 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:51.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:51.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:51.248 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:51.282 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:51.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:51.289 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:51.389 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:51.430 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:51.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:51.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:51.533 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:51.554 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:51.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:51.557 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:51.658 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:51.692 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:51.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:51.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:00:51.797 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:51.817 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:51.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:51.820 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:51.921 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:51.967 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:51.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:51.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:52.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:51 smithi092 bash[58855]: cluster 2023-09-23T20:00:50.636497+0000 mgr.y (mgr.25325) 1608 : cluster 0 pgmap v1642: 89 pgs: 89 active+clean; 1.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 618 op/s 2023-09-23T20:00:52.073 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:52.112 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:52.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:52.115 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:52.216 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:52.226 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:51 smithi016 bash[75004]: cluster 2023-09-23T20:00:50.636497+0000 mgr.y (mgr.25325) 1608 : cluster 0 pgmap v1642: 89 pgs: 89 active+clean; 1.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 618 op/s 2023-09-23T20:00:52.226 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:51 smithi016 bash[77995]: cluster 2023-09-23T20:00:50.636497+0000 mgr.y (mgr.25325) 1608 : cluster 0 pgmap v1642: 89 pgs: 89 active+clean; 1.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 618 op/s 2023-09-23T20:00:52.247 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:52.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:52.256 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:52.356 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:52.377 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:52.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:52.382 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:52.483 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:52.513 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:52.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:52.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:52.617 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:52.657 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:52.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:52.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:52.762 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:52.786 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:52.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:52.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:00:52.894 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:52.919 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:52.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:52.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:53.026 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:53.053 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:53.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:53.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:00:53.156 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:53.179 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:53.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:53.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:53.282 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:53.305 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:53.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:53.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:53.412 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:53.450 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:53.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:53.453 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:53.554 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:53.583 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:53.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:53.587 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:53.688 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:53.721 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:53.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:53.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:00:53.789 INFO:teuthology.orchestra.run.smithi092.stdout:{ 2023-09-23T20:00:53.789 INFO:teuthology.orchestra.run.smithi092.stdout: "size": 20, 2023-09-23T20:00:53.789 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 600, 2023-09-23T20:00:53.790 INFO:teuthology.orchestra.run.smithi092.stdout: "ops": [ 2023-09-23T20:00:53.790 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.790 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46640.0:54566 13.0 13:02cdaa51:::benchmark_data_smithi016_205181_object54565:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e238)", 2023-09-23T20:00:53.790 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:52:25.460372+0000", 2023-09-23T20:00:53.790 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 508.32794258199999, 2023-09-23T20:00:53.790 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.19363842100000001, 2023-09-23T20:00:53.790 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:00:53.790 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:00:53.790 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:00:53.790 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46640", 2023-09-23T20:00:53.790 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2656794142", 2023-09-23T20:00:53.791 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 54566 2023-09-23T20:00:53.791 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.791 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:00:53.791 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.791 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:00:53.791 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.460372+0000", 2023-09-23T20:00:53.791 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:00:53.791 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.791 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.791 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:00:53.791 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.460372+0000", 2023-09-23T20:00:53.792 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T20:00:53.792 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.792 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.792 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:00:53.792 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.460370+0000", 2023-09-23T20:00:53.792 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.0756999999999999e-05 2023-09-23T20:00:53.792 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.792 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.792 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:00:53.792 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.460451+0000", 2023-09-23T20:00:53.792 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.3703000000000001e-05 2023-09-23T20:00:53.792 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.793 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.793 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:00:53.793 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.460465+0000", 2023-09-23T20:00:53.793 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.5190000000000003e-06 2023-09-23T20:00:53.793 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.793 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.793 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:00:53.793 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.460472+0000", 2023-09-23T20:00:53.793 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.1558999999999995e-05 2023-09-23T20:00:53.793 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.793 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.793 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:00:53.794 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.460564+0000", 2023-09-23T20:00:53.794 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00016508500000000001 2023-09-23T20:00:53.794 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.794 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.794 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:00:53.794 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.460729+0000", 2023-09-23T20:00:53.794 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000180691 2023-09-23T20:00:53.794 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.794 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.794 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:00:53.794 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.460910+0000", 2023-09-23T20:00:53.794 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0025721139999999999 2023-09-23T20:00:53.795 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.795 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.795 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:00:53.795 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.463482+0000", 2023-09-23T20:00:53.795 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00042217200000000001 2023-09-23T20:00:53.795 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.795 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.795 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:53.795 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.463904+0000", 2023-09-23T20:00:53.795 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.18999377100000001 2023-09-23T20:00:53.795 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.795 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.796 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:53.796 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.653898+0000", 2023-09-23T20:00:53.796 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.7748000000000003e-05 2023-09-23T20:00:53.796 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.796 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.796 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:00:53.796 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.653966+0000", 2023-09-23T20:00:53.796 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.4629999999999998e-05 2023-09-23T20:00:53.796 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.796 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.796 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:00:53.797 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:25.654010+0000", 2023-09-23T20:00:53.797 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.19363842100000001 2023-09-23T20:00:53.797 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:00:53.797 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:00:53.797 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:00:53.797 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.797 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.797 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46640.0:56843 13.0 13:00e212a9:::benchmark_data_smithi016_205181_object56842:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e238)", 2023-09-23T20:00:53.797 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:52:27.511602+0000", 2023-09-23T20:00:53.797 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 506.27671275699998, 2023-09-23T20:00:53.797 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.17315303900000001, 2023-09-23T20:00:53.797 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:00:53.798 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:00:53.798 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:00:53.798 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46640", 2023-09-23T20:00:53.798 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2656794142", 2023-09-23T20:00:53.798 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 56843 2023-09-23T20:00:53.798 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.798 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:00:53.798 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.798 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:00:53.798 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:27.511602+0000", 2023-09-23T20:00:53.798 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:00:53.798 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.799 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.799 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:00:53.799 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:27.511602+0000", 2023-09-23T20:00:53.799 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T20:00:53.799 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.799 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.799 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:00:53.799 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:27.511600+0000", 2023-09-23T20:00:53.799 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.9376999999999999e-05 2023-09-23T20:00:53.799 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.799 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.799 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:00:53.800 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:27.511640+0000", 2023-09-23T20:00:53.800 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.3338e-05 2023-09-23T20:00:53.800 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.800 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.800 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:00:53.800 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:27.511653+0000", 2023-09-23T20:00:53.800 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.024e-06 2023-09-23T20:00:53.800 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.800 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.800 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:00:53.800 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:27.511660+0000", 2023-09-23T20:00:53.800 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00015277800000000001 2023-09-23T20:00:53.801 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.801 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.801 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:00:53.801 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:27.511813+0000", 2023-09-23T20:00:53.801 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000149631 2023-09-23T20:00:53.801 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.801 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.801 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:00:53.801 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:27.511962+0000", 2023-09-23T20:00:53.801 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00015164399999999999 2023-09-23T20:00:53.801 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.801 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.802 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:00:53.802 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:27.512114+0000", 2023-09-23T20:00:53.802 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0029355840000000002 2023-09-23T20:00:53.802 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.802 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.802 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:00:53.802 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:27.515050+0000", 2023-09-23T20:00:53.802 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0058118470000000002 2023-09-23T20:00:53.802 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.802 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.803 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:53.803 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:27.520861+0000", 2023-09-23T20:00:53.803 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.163801114 2023-09-23T20:00:53.803 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.803 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.803 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:53.803 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:27.684662+0000", 2023-09-23T20:00:53.803 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.2683999999999998e-05 2023-09-23T20:00:53.803 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.803 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.803 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:00:53.803 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:27.684725+0000", 2023-09-23T20:00:53.804 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.9470000000000001e-05 2023-09-23T20:00:53.804 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.804 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.804 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:00:53.804 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:27.684755+0000", 2023-09-23T20:00:53.804 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.17315303900000001 2023-09-23T20:00:53.804 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:00:53.804 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:00:53.804 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:00:53.804 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.804 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.804 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46640.0:57342 13.e 13:7fbdd5fe:::benchmark_data_smithi016_205181_object57341:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e238)", 2023-09-23T20:00:53.805 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:52:28.050001+0000", 2023-09-23T20:00:53.805 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 505.73831372799998, 2023-09-23T20:00:53.805 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.28244245600000001, 2023-09-23T20:00:53.805 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:00:53.805 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:00:53.805 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:00:53.805 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46640", 2023-09-23T20:00:53.805 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2656794142", 2023-09-23T20:00:53.805 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 57342 2023-09-23T20:00:53.805 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.805 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:00:53.806 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.806 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:00:53.806 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:28.050001+0000", 2023-09-23T20:00:53.806 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:00:53.806 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.806 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.806 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:00:53.806 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:28.050001+0000", 2023-09-23T20:00:53.806 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T20:00:53.806 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.806 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.807 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:00:53.807 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:28.049999+0000", 2023-09-23T20:00:53.807 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.9902999999999998e-05 2023-09-23T20:00:53.807 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.807 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.807 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:00:53.807 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:28.050069+0000", 2023-09-23T20:00:53.807 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.4260999999999999e-05 2023-09-23T20:00:53.807 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.807 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.807 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:00:53.807 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:28.050083+0000", 2023-09-23T20:00:53.807 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.5209999999999999e-06 2023-09-23T20:00:53.808 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.808 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.808 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:00:53.808 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:28.050090+0000", 2023-09-23T20:00:53.808 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.7332e-05 2023-09-23T20:00:53.808 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.808 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.808 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:00:53.808 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:28.050137+0000", 2023-09-23T20:00:53.808 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00014121000000000001 2023-09-23T20:00:53.808 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.809 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.809 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:00:53.809 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:28.050278+0000", 2023-09-23T20:00:53.809 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00012409000000000001 2023-09-23T20:00:53.809 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.809 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.809 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 4,5", 2023-09-23T20:00:53.809 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:28.050402+0000", 2023-09-23T20:00:53.809 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0019525059999999999 2023-09-23T20:00:53.809 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.809 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.810 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:00:53.810 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:28.052355+0000", 2023-09-23T20:00:53.810 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.001069253 2023-09-23T20:00:53.810 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.810 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.810 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:53.810 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:28.053424+0000", 2023-09-23T20:00:53.810 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.27889888800000001 2023-09-23T20:00:53.810 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.810 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.810 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:53.811 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:28.332323+0000", 2023-09-23T20:00:53.811 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.7218999999999995e-05 2023-09-23T20:00:53.811 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.811 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.811 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:00:53.811 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:28.332390+0000", 2023-09-23T20:00:53.811 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.2828999999999999e-05 2023-09-23T20:00:53.811 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.811 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.811 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:00:53.811 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:52:28.332443+0000", 2023-09-23T20:00:53.811 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.28244245600000001 2023-09-23T20:00:53.811 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:00:53.812 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:00:53.812 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:00:53.812 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.812 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.812 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46640.0:165456 13.0 13:08905a3e:::benchmark_data_smithi016_205181_object58192:head [delete] snapc 0=[] ondisk+write+known_if_redirected e246)", 2023-09-23T20:00:53.812 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:54:44.454923+0000", 2023-09-23T20:00:53.812 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 369.33339098900001, 2023-09-23T20:00:53.812 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.17939417699999999, 2023-09-23T20:00:53.812 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:00:53.812 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:00:53.812 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:00:53.813 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46640", 2023-09-23T20:00:53.813 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2656794142", 2023-09-23T20:00:53.813 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 165456 2023-09-23T20:00:53.813 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.813 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:00:53.813 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.813 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:00:53.813 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.454923+0000", 2023-09-23T20:00:53.813 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:00:53.813 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.813 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.813 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:00:53.814 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.454923+0000", 2023-09-23T20:00:53.814 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T20:00:53.814 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.814 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.814 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:00:53.814 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.454922+0000", 2023-09-23T20:00:53.814 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.4709999999999996e-06 2023-09-23T20:00:53.814 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.814 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.814 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:00:53.814 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.454931+0000", 2023-09-23T20:00:53.815 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.9559999999999999e-05 2023-09-23T20:00:53.815 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.815 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.815 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:00:53.815 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.454951+0000", 2023-09-23T20:00:53.815 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.3300000000000005e-06 2023-09-23T20:00:53.815 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.815 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.815 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:00:53.815 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.454960+0000", 2023-09-23T20:00:53.815 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.7500999999999997e-05 2023-09-23T20:00:53.815 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.816 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.816 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:00:53.816 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.455017+0000", 2023-09-23T20:00:53.816 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00013139 2023-09-23T20:00:53.816 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.816 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.816 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:00:53.816 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.455149+0000", 2023-09-23T20:00:53.816 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00013308600000000001 2023-09-23T20:00:53.816 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.816 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.816 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:00:53.817 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.455282+0000", 2023-09-23T20:00:53.817 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0083291119999999996 2023-09-23T20:00:53.817 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.817 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.817 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:00:53.817 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.463611+0000", 2023-09-23T20:00:53.817 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.012394275 2023-09-23T20:00:53.817 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.817 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.817 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:53.817 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.476005+0000", 2023-09-23T20:00:53.817 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.15818900599999999 2023-09-23T20:00:53.818 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.818 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.818 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:53.818 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.634194+0000", 2023-09-23T20:00:53.818 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.08e-05 2023-09-23T20:00:53.818 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.818 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.818 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:00:53.818 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.634265+0000", 2023-09-23T20:00:53.818 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.2376999999999997e-05 2023-09-23T20:00:53.818 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.818 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.819 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:00:53.819 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.634318+0000", 2023-09-23T20:00:53.819 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.17939417699999999 2023-09-23T20:00:53.819 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:00:53.819 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:00:53.819 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:00:53.819 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.819 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.819 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46640.0:165461 13.0 13:02a5752a:::benchmark_data_smithi016_205181_object58197:head [delete] snapc 0=[] ondisk+write+known_if_redirected e246)", 2023-09-23T20:00:53.819 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:54:44.465487+0000", 2023-09-23T20:00:53.819 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 369.322827399, 2023-09-23T20:00:53.820 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.183563646, 2023-09-23T20:00:53.820 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:00:53.820 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:00:53.820 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:00:53.820 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46640", 2023-09-23T20:00:53.820 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2656794142", 2023-09-23T20:00:53.820 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 165461 2023-09-23T20:00:53.820 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.820 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:00:53.820 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.820 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:00:53.820 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.465487+0000", 2023-09-23T20:00:53.821 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:00:53.821 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.821 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.821 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:00:53.821 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.465487+0000", 2023-09-23T20:00:53.821 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T20:00:53.821 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.821 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.821 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:00:53.821 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.465485+0000", 2023-09-23T20:00:53.821 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.9760000000000001e-06 2023-09-23T20:00:53.821 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.822 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.822 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:00:53.822 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.465494+0000", 2023-09-23T20:00:53.822 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.4909000000000001e-05 2023-09-23T20:00:53.822 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.822 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.822 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:00:53.822 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.465509+0000", 2023-09-23T20:00:53.822 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.1319999999999996e-06 2023-09-23T20:00:53.822 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.822 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.822 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:00:53.823 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.465516+0000", 2023-09-23T20:00:53.823 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.4416000000000002e-05 2023-09-23T20:00:53.823 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.823 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.823 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:00:53.823 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.465571+0000", 2023-09-23T20:00:53.823 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00013726100000000001 2023-09-23T20:00:53.823 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.823 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.823 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:00:53.823 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.465708+0000", 2023-09-23T20:00:53.823 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00015331899999999999 2023-09-23T20:00:53.824 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.824 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.824 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:00:53.824 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.465861+0000", 2023-09-23T20:00:53.824 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.010089173999999999 2023-09-23T20:00:53.824 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.824 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.824 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:00:53.824 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.475950+0000", 2023-09-23T20:00:53.824 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.4920999999999995e-05 2023-09-23T20:00:53.824 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.824 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.825 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:53.825 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.476045+0000", 2023-09-23T20:00:53.825 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.172886401 2023-09-23T20:00:53.825 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.825 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.825 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:53.825 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.648932+0000", 2023-09-23T20:00:53.825 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.7007000000000004e-05 2023-09-23T20:00:53.825 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.825 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.825 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:00:53.825 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.648999+0000", 2023-09-23T20:00:53.826 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.2006000000000003e-05 2023-09-23T20:00:53.826 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.826 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.826 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:00:53.826 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.649051+0000", 2023-09-23T20:00:53.826 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.183563646 2023-09-23T20:00:53.826 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:00:53.826 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:00:53.826 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:00:53.826 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.826 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.826 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46640.0:165463 13.0 13:08477c1e:::benchmark_data_smithi016_205181_object58199:head [delete] snapc 0=[] ondisk+write+known_if_redirected e246)", 2023-09-23T20:00:53.827 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:54:44.469427+0000", 2023-09-23T20:00:53.827 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 369.31888713000001, 2023-09-23T20:00:53.827 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.179750299, 2023-09-23T20:00:53.827 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:00:53.827 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:00:53.827 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:00:53.827 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46640", 2023-09-23T20:00:53.827 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2656794142", 2023-09-23T20:00:53.827 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 165463 2023-09-23T20:00:53.827 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.827 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:00:53.828 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.828 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:00:53.828 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.469427+0000", 2023-09-23T20:00:53.828 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:00:53.828 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.828 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.828 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:00:53.828 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.469427+0000", 2023-09-23T20:00:53.828 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T20:00:53.828 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.828 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.828 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:00:53.828 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.469425+0000", 2023-09-23T20:00:53.829 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.154e-05 2023-09-23T20:00:53.829 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.829 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.829 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:00:53.829 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.469437+0000", 2023-09-23T20:00:53.829 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.6903999999999999e-05 2023-09-23T20:00:53.829 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.829 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.829 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:00:53.829 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.469454+0000", 2023-09-23T20:00:53.829 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.2250000000000003e-06 2023-09-23T20:00:53.830 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.830 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.830 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:00:53.830 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.469461+0000", 2023-09-23T20:00:53.830 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0010087 2023-09-23T20:00:53.830 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.830 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.830 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:00:53.830 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.470470+0000", 2023-09-23T20:00:53.830 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00013054000000000001 2023-09-23T20:00:53.831 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.831 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.831 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:00:53.831 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.470600+0000", 2023-09-23T20:00:53.831 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00015463599999999999 2023-09-23T20:00:53.831 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.831 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.831 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:00:53.831 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.470755+0000", 2023-09-23T20:00:53.831 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0052573369999999999 2023-09-23T20:00:53.831 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.831 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.831 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:00:53.832 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.476012+0000", 2023-09-23T20:00:53.832 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.4549999999999997e-05 2023-09-23T20:00:53.832 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.832 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.832 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:53.832 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.476077+0000", 2023-09-23T20:00:53.832 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.173024606 2023-09-23T20:00:53.832 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.832 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.832 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:53.832 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.649101+0000", 2023-09-23T20:00:53.832 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.0645000000000001e-05 2023-09-23T20:00:53.832 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.833 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.833 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:00:53.833 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.649142+0000", 2023-09-23T20:00:53.833 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.5698999999999998e-05 2023-09-23T20:00:53.833 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.833 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.833 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:00:53.833 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.649178+0000", 2023-09-23T20:00:53.833 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.179750299 2023-09-23T20:00:53.833 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:00:53.833 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:00:53.833 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:00:53.834 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.834 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.834 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46640.0:165470 13.0 13:0835838a:::benchmark_data_smithi016_205181_object58206:head [delete] snapc 0=[] ondisk+write+known_if_redirected e246)", 2023-09-23T20:00:53.834 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:54:44.479541+0000", 2023-09-23T20:00:53.834 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 369.30877339099999, 2023-09-23T20:00:53.834 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.169743687, 2023-09-23T20:00:53.834 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:00:53.834 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:00:53.834 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:00:53.834 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46640", 2023-09-23T20:00:53.834 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2656794142", 2023-09-23T20:00:53.834 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 165470 2023-09-23T20:00:53.835 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.835 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:00:53.835 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.835 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:00:53.835 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.479541+0000", 2023-09-23T20:00:53.835 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:00:53.835 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.835 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.835 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:00:53.835 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.479541+0000", 2023-09-23T20:00:53.835 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T20:00:53.836 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.836 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.836 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:00:53.836 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.479539+0000", 2023-09-23T20:00:53.836 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.2632e-05 2023-09-23T20:00:53.836 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.836 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.836 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:00:53.836 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.479552+0000", 2023-09-23T20:00:53.836 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.6379e-05 2023-09-23T20:00:53.836 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.837 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.837 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:00:53.837 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.479568+0000", 2023-09-23T20:00:53.837 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.0840000000000003e-06 2023-09-23T20:00:53.837 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.837 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.837 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:00:53.837 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.479575+0000", 2023-09-23T20:00:53.837 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000126654 2023-09-23T20:00:53.837 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.837 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.837 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:00:53.837 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.479702+0000", 2023-09-23T20:00:53.838 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000139104 2023-09-23T20:00:53.838 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.838 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.838 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:00:53.838 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.479841+0000", 2023-09-23T20:00:53.838 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000174115 2023-09-23T20:00:53.838 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.838 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.838 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:00:53.838 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.480015+0000", 2023-09-23T20:00:53.838 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0040933189999999998 2023-09-23T20:00:53.839 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.839 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.839 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:00:53.839 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.484108+0000", 2023-09-23T20:00:53.839 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0013572860000000001 2023-09-23T20:00:53.839 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.839 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.839 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:53.839 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.485466+0000", 2023-09-23T20:00:53.839 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.163744537 2023-09-23T20:00:53.839 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.839 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.839 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:53.840 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.649210+0000", 2023-09-23T20:00:53.840 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.9177000000000001e-05 2023-09-23T20:00:53.840 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.840 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.840 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:00:53.840 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.649249+0000", 2023-09-23T20:00:53.840 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.5178000000000001e-05 2023-09-23T20:00:53.840 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.840 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.840 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:00:53.840 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.649285+0000", 2023-09-23T20:00:53.841 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.169743687 2023-09-23T20:00:53.841 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:00:53.841 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:00:53.841 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:00:53.841 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.841 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.841 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46640.0:165474 13.0 13:08c49eff:::benchmark_data_smithi016_205181_object58210:head [delete] snapc 0=[] ondisk+write+known_if_redirected e246)", 2023-09-23T20:00:53.841 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:54:44.486191+0000", 2023-09-23T20:00:53.841 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 369.30212310600001, 2023-09-23T20:00:53.841 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.16319824899999999, 2023-09-23T20:00:53.841 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:00:53.842 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:00:53.842 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:00:53.842 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46640", 2023-09-23T20:00:53.842 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2656794142", 2023-09-23T20:00:53.842 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 165474 2023-09-23T20:00:53.842 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.842 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:00:53.842 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.842 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:00:53.842 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.486191+0000", 2023-09-23T20:00:53.842 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:00:53.842 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.843 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.843 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:00:53.843 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.486191+0000", 2023-09-23T20:00:53.843 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T20:00:53.843 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.843 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.843 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:00:53.843 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.486190+0000", 2023-09-23T20:00:53.843 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.0152999999999999e-05 2023-09-23T20:00:53.843 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.843 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.843 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:00:53.844 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.486200+0000", 2023-09-23T20:00:53.844 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.1548e-05 2023-09-23T20:00:53.844 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.844 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.844 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:00:53.844 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.486211+0000", 2023-09-23T20:00:53.844 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.1300000000000001e-06 2023-09-23T20:00:53.844 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.844 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.844 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:00:53.844 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.486220+0000", 2023-09-23T20:00:53.844 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.5142999999999997e-05 2023-09-23T20:00:53.845 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.845 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.845 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:00:53.845 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.486275+0000", 2023-09-23T20:00:53.845 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00015813099999999999 2023-09-23T20:00:53.845 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.845 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.845 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:00:53.845 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.486433+0000", 2023-09-23T20:00:53.845 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000196918 2023-09-23T20:00:53.845 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.845 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.846 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:00:53.846 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.486630+0000", 2023-09-23T20:00:53.846 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0050995670000000002 2023-09-23T20:00:53.846 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.846 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.846 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:53.846 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.491729+0000", 2023-09-23T20:00:53.846 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00011642 2023-09-23T20:00:53.846 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.846 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.846 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:00:53.846 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.491846+0000", 2023-09-23T20:00:53.847 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.15747035000000001 2023-09-23T20:00:53.847 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.847 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.847 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:53.847 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.649316+0000", 2023-09-23T20:00:53.847 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.9458000000000001e-05 2023-09-23T20:00:53.847 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.847 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.847 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:00:53.847 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.649356+0000", 2023-09-23T20:00:53.847 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.3955999999999998e-05 2023-09-23T20:00:53.847 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.848 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.848 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:00:53.848 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:54:44.649389+0000", 2023-09-23T20:00:53.848 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.16319824899999999 2023-09-23T20:00:53.848 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:00:53.848 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:00:53.848 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:00:53.848 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.848 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.848 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46640.0:208007 13.0 13:0199449d:::benchmark_data_smithi016_205181_object100743:head [delete] snapc 0=[] ondisk+write+known_if_redirected e250)", 2023-09-23T20:00:53.848 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:55:49.612712+0000", 2023-09-23T20:00:53.848 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 304.17560237499998, 2023-09-23T20:00:53.849 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.15701494799999999, 2023-09-23T20:00:53.849 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:00:53.849 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:00:53.849 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:00:53.849 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46640", 2023-09-23T20:00:53.849 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2656794142", 2023-09-23T20:00:53.849 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 208007 2023-09-23T20:00:53.849 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.849 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:00:53.849 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.849 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:00:53.850 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:49.612712+0000", 2023-09-23T20:00:53.850 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:00:53.850 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.850 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.850 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:00:53.850 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:49.612712+0000", 2023-09-23T20:00:53.850 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T20:00:53.850 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.850 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.850 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:00:53.850 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:49.612710+0000", 2023-09-23T20:00:53.851 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.3270000000000003e-06 2023-09-23T20:00:53.851 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.851 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.851 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:00:53.851 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:49.612718+0000", 2023-09-23T20:00:53.851 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.3315e-05 2023-09-23T20:00:53.851 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.851 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.851 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:00:53.851 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:49.612731+0000", 2023-09-23T20:00:53.851 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.5930000000000002e-06 2023-09-23T20:00:53.852 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.852 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.852 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:00:53.852 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:49.612738+0000", 2023-09-23T20:00:53.852 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.2136999999999999e-05 2023-09-23T20:00:53.852 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.852 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.852 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:00:53.852 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:49.612790+0000", 2023-09-23T20:00:53.852 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000131602 2023-09-23T20:00:53.852 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.852 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.853 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:00:53.853 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:49.612921+0000", 2023-09-23T20:00:53.853 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00013318299999999999 2023-09-23T20:00:53.853 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.853 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.853 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:00:53.853 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:49.613055+0000", 2023-09-23T20:00:53.853 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0057135329999999998 2023-09-23T20:00:53.853 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.853 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.853 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:00:53.853 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:49.618768+0000", 2023-09-23T20:00:53.854 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0035966010000000001 2023-09-23T20:00:53.854 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.854 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.854 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:53.854 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:49.622365+0000", 2023-09-23T20:00:53.854 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.14715002899999999 2023-09-23T20:00:53.854 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.854 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.854 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:53.854 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:49.769515+0000", 2023-09-23T20:00:53.854 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000118269 2023-09-23T20:00:53.855 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.855 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.855 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:00:53.855 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:49.769633+0000", 2023-09-23T20:00:53.855 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.3924000000000004e-05 2023-09-23T20:00:53.855 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.855 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.855 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:00:53.855 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:49.769727+0000", 2023-09-23T20:00:53.855 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.15701494799999999 2023-09-23T20:00:53.855 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:00:53.856 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:00:53.856 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:00:53.856 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.856 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.856 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:36132 14.f 14:f289b9fe:::benchmark_data_smithi016_228083_object36131:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-09-23T20:00:53.856 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:57:34.789453+0000", 2023-09-23T20:00:53.856 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 198.99886132200001, 2023-09-23T20:00:53.856 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.14023466200000001, 2023-09-23T20:00:53.856 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:00:53.856 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:00:53.856 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:00:53.856 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T20:00:53.857 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:00:53.857 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 36132 2023-09-23T20:00:53.857 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.857 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:00:53.857 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.857 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:00:53.857 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.789453+0000", 2023-09-23T20:00:53.857 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:00:53.857 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.857 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.857 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:00:53.857 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.789453+0000", 2023-09-23T20:00:53.858 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T20:00:53.858 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.858 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.858 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:00:53.858 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.789452+0000", 2023-09-23T20:00:53.858 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.7549999999999997e-05 2023-09-23T20:00:53.858 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.858 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.858 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:00:53.858 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.789499+0000", 2023-09-23T20:00:53.858 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.7098000000000001e-05 2023-09-23T20:00:53.858 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.859 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.859 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:00:53.859 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.789516+0000", 2023-09-23T20:00:53.859 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.6249999999999996e-06 2023-09-23T20:00:53.859 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.859 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.859 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:00:53.859 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.789525+0000", 2023-09-23T20:00:53.859 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.9809999999999998e-05 2023-09-23T20:00:53.859 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.859 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.859 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:00:53.859 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.789585+0000", 2023-09-23T20:00:53.859 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00018619700000000001 2023-09-23T20:00:53.860 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.860 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.860 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:00:53.860 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.789771+0000", 2023-09-23T20:00:53.860 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00022020500000000001 2023-09-23T20:00:53.860 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.860 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.860 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:00:53.860 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.789991+0000", 2023-09-23T20:00:53.860 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.002540161 2023-09-23T20:00:53.860 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.860 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.860 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:00:53.860 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.792531+0000", 2023-09-23T20:00:53.861 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0016885349999999999 2023-09-23T20:00:53.861 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.861 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.861 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:53.861 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.794220+0000", 2023-09-23T20:00:53.861 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.13522234299999999 2023-09-23T20:00:53.861 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.861 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.861 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:53.861 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.929442+0000", 2023-09-23T20:00:53.861 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00013912899999999999 2023-09-23T20:00:53.862 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.862 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.862 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:00:53.862 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.929581+0000", 2023-09-23T20:00:53.862 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000106533 2023-09-23T20:00:53.862 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.862 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.862 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:00:53.862 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.929688+0000", 2023-09-23T20:00:53.862 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.14023466200000001 2023-09-23T20:00:53.862 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:00:53.862 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:00:53.863 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:00:53.863 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.863 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.863 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:36148 14.f 14:f2537d57:::benchmark_data_smithi016_228083_object36147:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-09-23T20:00:53.863 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:57:34.799429+0000", 2023-09-23T20:00:53.863 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 198.98888545099999, 2023-09-23T20:00:53.863 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.13068242399999999, 2023-09-23T20:00:53.863 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:00:53.863 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:00:53.863 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:00:53.863 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T20:00:53.863 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:00:53.864 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 36148 2023-09-23T20:00:53.864 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.864 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:00:53.864 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.864 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:00:53.864 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.799429+0000", 2023-09-23T20:00:53.864 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:00:53.864 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.864 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.864 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:00:53.864 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.799429+0000", 2023-09-23T20:00:53.865 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999976 2023-09-23T20:00:53.865 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.865 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.865 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:00:53.865 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.799426+0000", 2023-09-23T20:00:53.865 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.7633999999999998e-05 2023-09-23T20:00:53.865 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.865 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.865 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:00:53.865 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.799474+0000", 2023-09-23T20:00:53.865 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.5546e-05 2023-09-23T20:00:53.866 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.866 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.866 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:00:53.866 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.799490+0000", 2023-09-23T20:00:53.866 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.5249999999999994e-06 2023-09-23T20:00:53.866 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.866 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.866 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:00:53.866 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.799499+0000", 2023-09-23T20:00:53.866 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.0447999999999999e-05 2023-09-23T20:00:53.866 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.866 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.867 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:00:53.867 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.799590+0000", 2023-09-23T20:00:53.867 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00014605400000000001 2023-09-23T20:00:53.867 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.867 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.867 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:00:53.867 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.799736+0000", 2023-09-23T20:00:53.867 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000147152 2023-09-23T20:00:53.867 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.867 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.867 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:00:53.867 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.799883+0000", 2023-09-23T20:00:53.868 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0028179730000000001 2023-09-23T20:00:53.868 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.868 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.868 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:00:53.868 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.802701+0000", 2023-09-23T20:00:53.868 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.0997e-05 2023-09-23T20:00:53.868 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.868 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.868 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:53.868 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.802752+0000", 2023-09-23T20:00:53.868 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.12707763899999999 2023-09-23T20:00:53.868 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.868 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.869 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:53.869 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.929829+0000", 2023-09-23T20:00:53.869 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00016053000000000001 2023-09-23T20:00:53.869 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.869 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.869 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:00:53.869 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.929990+0000", 2023-09-23T20:00:53.869 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000121385 2023-09-23T20:00:53.869 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.869 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.869 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:00:53.869 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.930111+0000", 2023-09-23T20:00:53.870 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.13068242399999999 2023-09-23T20:00:53.870 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:00:53.870 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:00:53.870 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:00:53.870 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.870 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.870 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:38540 14.f 14:f39591d4:::benchmark_data_smithi016_228083_object38539:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-09-23T20:00:53.870 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:57:36.906965+0000", 2023-09-23T20:00:53.870 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 196.88134955800001, 2023-09-23T20:00:53.870 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.214407346, 2023-09-23T20:00:53.871 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:00:53.871 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:00:53.871 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:00:53.871 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T20:00:53.871 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:00:53.871 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 38540 2023-09-23T20:00:53.871 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.871 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:00:53.871 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.871 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:00:53.871 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:36.906965+0000", 2023-09-23T20:00:53.871 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:00:53.872 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.872 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.872 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:00:53.872 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:36.906965+0000", 2023-09-23T20:00:53.872 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T20:00:53.872 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.872 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.872 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:00:53.872 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:36.906963+0000", 2023-09-23T20:00:53.872 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.414e-05 2023-09-23T20:00:53.872 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.872 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.873 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:00:53.873 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:36.907017+0000", 2023-09-23T20:00:53.873 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.2391e-05 2023-09-23T20:00:53.873 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.873 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.873 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:00:53.873 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:36.907030+0000", 2023-09-23T20:00:53.873 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.1810000000000004e-06 2023-09-23T20:00:53.873 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.873 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.873 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:00:53.873 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:36.907037+0000", 2023-09-23T20:00:53.874 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.4173999999999999e-05 2023-09-23T20:00:53.874 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.874 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.874 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:00:53.874 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:36.907101+0000", 2023-09-23T20:00:53.874 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00019921699999999999 2023-09-23T20:00:53.874 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.874 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.874 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:00:53.874 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:36.907300+0000", 2023-09-23T20:00:53.874 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00021943599999999999 2023-09-23T20:00:53.875 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.875 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.875 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:00:53.875 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:36.907520+0000", 2023-09-23T20:00:53.875 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0019506020000000001 2023-09-23T20:00:53.875 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.875 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.875 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:00:53.875 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:36.909470+0000", 2023-09-23T20:00:53.875 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0033401030000000001 2023-09-23T20:00:53.875 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.875 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.876 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:53.876 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:36.912810+0000", 2023-09-23T20:00:53.876 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.20838873299999999 2023-09-23T20:00:53.876 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.876 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.876 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:53.876 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:37.121199+0000", 2023-09-23T20:00:53.876 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.4506999999999995e-05 2023-09-23T20:00:53.876 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.876 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.876 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:00:53.876 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:37.121293+0000", 2023-09-23T20:00:53.877 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.8683999999999994e-05 2023-09-23T20:00:53.877 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.877 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.877 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:00:53.877 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:37.121372+0000", 2023-09-23T20:00:53.877 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.214407346 2023-09-23T20:00:53.877 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:00:53.877 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:00:53.877 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:00:53.877 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.877 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.878 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:49383 14.f 14:ffca6e31:::benchmark_data_smithi016_228083_object49382:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-09-23T20:00:53.878 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:57:46.821698+0000", 2023-09-23T20:00:53.878 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 186.96661658599999, 2023-09-23T20:00:53.878 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.18302286500000001, 2023-09-23T20:00:53.878 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:00:53.878 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:00:53.878 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:00:53.878 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T20:00:53.878 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:00:53.879 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 49383 2023-09-23T20:00:53.879 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.879 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:00:53.879 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.879 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:00:53.879 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:46.821698+0000", 2023-09-23T20:00:53.879 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:00:53.879 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.879 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.880 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:00:53.880 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:46.821698+0000", 2023-09-23T20:00:53.880 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T20:00:53.880 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.880 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.880 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:00:53.880 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:46.821696+0000", 2023-09-23T20:00:53.880 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.0072000000000002e-05 2023-09-23T20:00:53.880 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.880 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.880 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:00:53.881 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:46.821776+0000", 2023-09-23T20:00:53.881 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.2704000000000001e-05 2023-09-23T20:00:53.881 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.881 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.881 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:00:53.881 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:46.821789+0000", 2023-09-23T20:00:53.881 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.447e-06 2023-09-23T20:00:53.881 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.881 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.881 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:00:53.881 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:46.821796+0000", 2023-09-23T20:00:53.881 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00019770899999999999 2023-09-23T20:00:53.881 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.882 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.882 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:00:53.882 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:46.821994+0000", 2023-09-23T20:00:53.882 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000177226 2023-09-23T20:00:53.882 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.882 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.882 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:00:53.882 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:46.822171+0000", 2023-09-23T20:00:53.882 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000214794 2023-09-23T20:00:53.882 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.882 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.882 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:00:53.883 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:46.822386+0000", 2023-09-23T20:00:53.883 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.002034306 2023-09-23T20:00:53.883 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.883 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.883 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:00:53.883 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:46.824420+0000", 2023-09-23T20:00:53.883 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0024117710000000001 2023-09-23T20:00:53.883 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.883 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.883 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:53.883 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:46.826832+0000", 2023-09-23T20:00:53.883 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.17778169499999999 2023-09-23T20:00:53.884 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.884 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.884 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:53.884 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:47.004614+0000", 2023-09-23T20:00:53.884 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.6984000000000002e-05 2023-09-23T20:00:53.884 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.884 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.884 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:00:53.884 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:47.004671+0000", 2023-09-23T20:00:53.884 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.9778999999999999e-05 2023-09-23T20:00:53.884 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.884 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.885 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:00:53.885 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:47.004721+0000", 2023-09-23T20:00:53.885 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.18302286500000001 2023-09-23T20:00:53.885 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:00:53.885 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:00:53.885 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:00:53.885 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.885 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.885 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:92203 14.f 14:f2e46b18:::benchmark_data_smithi016_228083_object92202:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e264)", 2023-09-23T20:00:53.885 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:58:34.347307+0000", 2023-09-23T20:00:53.885 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 139.441007619, 2023-09-23T20:00:53.886 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.13251001000000001, 2023-09-23T20:00:53.886 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:00:53.886 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:00:53.886 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:00:53.886 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T20:00:53.886 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:00:53.886 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 92203 2023-09-23T20:00:53.886 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.886 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:00:53.886 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.887 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:00:53.887 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.347307+0000", 2023-09-23T20:00:53.887 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:00:53.887 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.887 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.887 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:00:53.887 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.347307+0000", 2023-09-23T20:00:53.887 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T20:00:53.887 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.887 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.887 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:00:53.887 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.347305+0000", 2023-09-23T20:00:53.888 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.299e-05 2023-09-23T20:00:53.888 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.888 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.888 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:00:53.888 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.347348+0000", 2023-09-23T20:00:53.888 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.4859999999999999e-05 2023-09-23T20:00:53.888 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.888 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.888 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:00:53.888 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.347373+0000", 2023-09-23T20:00:53.888 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.0611999999999999e-05 2023-09-23T20:00:53.889 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.889 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.889 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:00:53.889 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.347383+0000", 2023-09-23T20:00:53.889 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.3498999999999996e-05 2023-09-23T20:00:53.889 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.889 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.889 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:00:53.889 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.347447+0000", 2023-09-23T20:00:53.889 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00015322799999999999 2023-09-23T20:00:53.889 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.889 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.889 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:00:53.890 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.347600+0000", 2023-09-23T20:00:53.890 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00018437999999999999 2023-09-23T20:00:53.890 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.890 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.890 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:00:53.890 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.347784+0000", 2023-09-23T20:00:53.890 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0038522909999999999 2023-09-23T20:00:53.890 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.890 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.890 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:00:53.890 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.351637+0000", 2023-09-23T20:00:53.891 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000147171 2023-09-23T20:00:53.891 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.891 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.891 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:53.891 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.351784+0000", 2023-09-23T20:00:53.891 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.12785521599999999 2023-09-23T20:00:53.891 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.891 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.891 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:53.891 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.479639+0000", 2023-09-23T20:00:53.891 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000101411 2023-09-23T20:00:53.891 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.892 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.892 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:00:53.892 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.479740+0000", 2023-09-23T20:00:53.892 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.6371000000000005e-05 2023-09-23T20:00:53.892 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.892 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.892 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:00:53.892 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.479817+0000", 2023-09-23T20:00:53.892 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.13251001000000001 2023-09-23T20:00:53.892 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:00:53.892 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:00:53.893 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:00:53.893 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.893 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.893 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:156208 14.f 14:fda8ef3f:::benchmark_data_smithi016_228083_object56249:head [delete] snapc 0=[] ondisk+write+known_if_redirected e269)", 2023-09-23T20:00:53.893 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T20:00:06.689697+0000", 2023-09-23T20:00:53.893 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 47.098617009999998, 2023-09-23T20:00:53.893 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.18813912599999999, 2023-09-23T20:00:53.893 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:00:53.893 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:00:53.893 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:00:53.893 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T20:00:53.893 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:00:53.894 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 156208 2023-09-23T20:00:53.894 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.894 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:00:53.894 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.894 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:00:53.894 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.689697+0000", 2023-09-23T20:00:53.894 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:00:53.894 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.894 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.894 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:00:53.894 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.689697+0000", 2023-09-23T20:00:53.894 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T20:00:53.895 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.895 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.895 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:00:53.895 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.689696+0000", 2023-09-23T20:00:53.895 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.6139999999999992e-06 2023-09-23T20:00:53.895 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.895 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.895 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:00:53.895 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.689704+0000", 2023-09-23T20:00:53.895 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.3042e-05 2023-09-23T20:00:53.895 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.895 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.895 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:00:53.896 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.689717+0000", 2023-09-23T20:00:53.896 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.6159999999999996e-06 2023-09-23T20:00:53.896 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.896 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.896 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:00:53.896 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.689725+0000", 2023-09-23T20:00:53.896 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.8993999999999997e-05 2023-09-23T20:00:53.896 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.896 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.896 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:00:53.896 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.689794+0000", 2023-09-23T20:00:53.897 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000163148 2023-09-23T20:00:53.897 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.897 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.897 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:00:53.897 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.689957+0000", 2023-09-23T20:00:53.897 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00017913599999999999 2023-09-23T20:00:53.897 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.897 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.897 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:00:53.897 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.690136+0000", 2023-09-23T20:00:53.897 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.17782856899999999 2023-09-23T20:00:53.897 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.897 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.898 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:53.898 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.867965+0000", 2023-09-23T20:00:53.898 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.009755962 2023-09-23T20:00:53.898 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.898 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.898 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:00:53.898 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.877721+0000", 2023-09-23T20:00:53.898 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.9586000000000003e-05 2023-09-23T20:00:53.898 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.898 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.898 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:53.899 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.877770+0000", 2023-09-23T20:00:53.899 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.5295e-05 2023-09-23T20:00:53.899 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.899 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.899 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:00:53.899 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.877806+0000", 2023-09-23T20:00:53.899 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.0716999999999997e-05 2023-09-23T20:00:53.899 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.899 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.899 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:00:53.899 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.877836+0000", 2023-09-23T20:00:53.899 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.18813912599999999 2023-09-23T20:00:53.900 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:00:53.900 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:00:53.900 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:00:53.900 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.900 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.900 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_repop(client.46877.0:156207 14.4 e269/256 14:2211dd6c:::benchmark_data_smithi016_228083_object56248:head v 269'9755, mlcod=269'9753)", 2023-09-23T20:00:53.900 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T20:00:06.690168+0000", 2023-09-23T20:00:53.900 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 47.098145914, 2023-09-23T20:00:53.900 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.19565996099999999, 2023-09-23T20:00:53.900 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:00:53.900 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:00:53.901 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:00:53.901 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.901 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:00:53.901 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.690168+0000", 2023-09-23T20:00:53.901 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:00:53.901 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.901 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.901 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:00:53.901 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.690168+0000", 2023-09-23T20:00:53.901 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.999999 2023-09-23T20:00:53.901 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.901 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.902 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:00:53.902 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.690167+0000", 2023-09-23T20:00:53.902 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.2271e-05 2023-09-23T20:00:53.902 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.902 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.902 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:00:53.902 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.690180+0000", 2023-09-23T20:00:53.902 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.4568e-05 2023-09-23T20:00:53.902 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.902 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.902 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:00:53.903 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.690194+0000", 2023-09-23T20:00:53.903 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.5480000000000002e-06 2023-09-23T20:00:53.903 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.903 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.903 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:00:53.903 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.690203+0000", 2023-09-23T20:00:53.903 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.1079e-05 2023-09-23T20:00:53.903 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.903 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.903 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:00:53.903 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.690294+0000", 2023-09-23T20:00:53.904 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.7073999999999999e-05 2023-09-23T20:00:53.904 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.904 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.904 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:00:53.904 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.690321+0000", 2023-09-23T20:00:53.904 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.19545496200000001 2023-09-23T20:00:53.904 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.904 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.904 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:00:53.904 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.885776+0000", 2023-09-23T20:00:53.904 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.2624000000000003e-05 2023-09-23T20:00:53.904 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.905 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.905 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:00:53.905 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.885828+0000", 2023-09-23T20:00:53.905 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.19565996099999999 2023-09-23T20:00:53.905 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:00:53.905 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:00:53.905 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:00:53.905 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.905 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.905 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_repop_reply(client.46877.0:156208 14.f e269/256 ondisk, result = 0)", 2023-09-23T20:00:53.905 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T20:00:06.697831+0000", 2023-09-23T20:00:53.905 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 47.090483482000003, 2023-09-23T20:00:53.906 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.17013848200000001, 2023-09-23T20:00:53.906 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:00:53.906 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "started", 2023-09-23T20:00:53.906 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:00:53.906 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.906 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:00:53.906 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.697831+0000", 2023-09-23T20:00:53.906 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:00:53.906 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.906 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.906 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:00:53.906 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.697831+0000", 2023-09-23T20:00:53.906 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T20:00:53.907 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.907 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.907 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:00:53.907 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.697829+0000", 2023-09-23T20:00:53.907 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.0662000000000001e-05 2023-09-23T20:00:53.907 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.907 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.907 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:00:53.907 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.697840+0000", 2023-09-23T20:00:53.907 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.16987813700000001 2023-09-23T20:00:53.907 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.907 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.908 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:00:53.908 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.867718+0000", 2023-09-23T20:00:53.908 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.7621999999999998e-05 2023-09-23T20:00:53.908 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.908 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.908 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:00:53.908 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.867736+0000", 2023-09-23T20:00:53.908 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000187975 2023-09-23T20:00:53.908 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.908 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.908 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:00:53.908 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.867924+0000", 2023-09-23T20:00:53.908 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.0429999999999999e-05 2023-09-23T20:00:53.909 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.909 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.909 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:00:53.909 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.867944+0000", 2023-09-23T20:00:53.909 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.5139000000000001e-05 2023-09-23T20:00:53.909 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.909 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.909 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:00:53.909 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.867969+0000", 2023-09-23T20:00:53.909 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.17013848200000001 2023-09-23T20:00:53.909 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:00:53.910 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:00:53.910 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:00:53.910 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.910 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.910 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_repop(client.46877.0:156220 14.5 e269/256 14:a095c11c:::benchmark_data_smithi016_228083_object56261:head v 269'9743, mlcod=269'9741)", 2023-09-23T20:00:53.910 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T20:00:06.711990+0000", 2023-09-23T20:00:53.910 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 47.076324692999997, 2023-09-23T20:00:53.910 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.16772410099999999, 2023-09-23T20:00:53.910 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:00:53.910 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:00:53.910 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:00:53.910 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.911 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:00:53.911 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.711990+0000", 2023-09-23T20:00:53.911 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:00:53.911 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.911 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.911 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:00:53.911 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.711990+0000", 2023-09-23T20:00:53.911 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T20:00:53.911 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.911 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.911 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:00:53.911 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.711988+0000", 2023-09-23T20:00:53.911 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.5136999999999999e-05 2023-09-23T20:00:53.912 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.912 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.912 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:00:53.912 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.712004+0000", 2023-09-23T20:00:53.912 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.155722684 2023-09-23T20:00:53.912 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.912 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.912 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:00:53.912 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.867726+0000", 2023-09-23T20:00:53.912 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.4652e-05 2023-09-23T20:00:53.912 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.912 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.913 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:00:53.913 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.867741+0000", 2023-09-23T20:00:53.913 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00012279700000000001 2023-09-23T20:00:53.913 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.913 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.913 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:00:53.913 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.867864+0000", 2023-09-23T20:00:53.913 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.6038000000000001e-05 2023-09-23T20:00:53.913 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.913 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.913 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:00:53.913 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.867920+0000", 2023-09-23T20:00:53.914 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.011769421 2023-09-23T20:00:53.914 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.914 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.914 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:00:53.914 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.879689+0000", 2023-09-23T20:00:53.914 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.4654000000000001e-05 2023-09-23T20:00:53.914 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.914 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.914 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:00:53.914 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.879714+0000", 2023-09-23T20:00:53.914 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.16772410099999999 2023-09-23T20:00:53.914 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:00:53.915 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:00:53.915 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:00:53.915 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.915 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.915 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_repop(client.46877.0:156225 14.4 e269/256 14:2e9b25eb:::benchmark_data_smithi016_228083_object56266:head v 269'9756, mlcod=269'9753)", 2023-09-23T20:00:53.915 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T20:00:06.714431+0000", 2023-09-23T20:00:53.915 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 47.073883197000001, 2023-09-23T20:00:53.915 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.17143608499999999, 2023-09-23T20:00:53.915 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:00:53.915 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:00:53.915 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:00:53.915 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.916 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:00:53.916 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.714431+0000", 2023-09-23T20:00:53.916 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:00:53.916 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.916 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.916 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:00:53.916 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.714431+0000", 2023-09-23T20:00:53.916 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.999999 2023-09-23T20:00:53.916 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.916 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.916 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:00:53.916 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.714430+0000", 2023-09-23T20:00:53.917 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.1928e-05 2023-09-23T20:00:53.917 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.917 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.917 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:00:53.917 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.714442+0000", 2023-09-23T20:00:53.917 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.153322237 2023-09-23T20:00:53.917 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.917 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.917 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:00:53.917 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.867764+0000", 2023-09-23T20:00:53.917 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.4139e-05 2023-09-23T20:00:53.917 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.918 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.918 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:00:53.918 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.867798+0000", 2023-09-23T20:00:53.918 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0098272480000000002 2023-09-23T20:00:53.918 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.918 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.918 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:00:53.918 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.877626+0000", 2023-09-23T20:00:53.918 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.4397999999999999e-05 2023-09-23T20:00:53.918 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.918 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.918 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:00:53.919 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.877650+0000", 2023-09-23T20:00:53.919 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0081816920000000008 2023-09-23T20:00:53.919 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.919 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.919 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:00:53.919 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.885832+0000", 2023-09-23T20:00:53.919 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.5441999999999997e-05 2023-09-23T20:00:53.919 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.919 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.919 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:00:53.919 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.885867+0000", 2023-09-23T20:00:53.919 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.17143608499999999 2023-09-23T20:00:53.920 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:00:53.920 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:00:53.920 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:00:53.920 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.920 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.920 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:177691 14.f 14:ff8ae2d9:::benchmark_data_smithi016_228083_object77732:head [delete] snapc 0=[] ondisk+write+known_if_redirected e271)", 2023-09-23T20:00:53.920 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T20:00:41.523968+0000", 2023-09-23T20:00:53.920 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 12.264345966, 2023-09-23T20:00:53.920 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.176987484, 2023-09-23T20:00:53.920 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:00:53.920 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:00:53.920 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:00:53.921 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T20:00:53.921 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:00:53.921 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 177691 2023-09-23T20:00:53.921 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.921 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:00:53.921 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.921 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:00:53.921 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.523968+0000", 2023-09-23T20:00:53.921 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:00:53.921 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.921 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.921 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:00:53.921 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.523968+0000", 2023-09-23T20:00:53.922 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T20:00:53.922 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.922 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.922 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:00:53.922 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.523967+0000", 2023-09-23T20:00:53.922 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.3499999999999999e-06 2023-09-23T20:00:53.922 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.922 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.922 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:00:53.922 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.523974+0000", 2023-09-23T20:00:53.922 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.3818000000000001e-05 2023-09-23T20:00:53.922 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.923 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.923 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:00:53.923 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.523988+0000", 2023-09-23T20:00:53.923 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.5310000000000004e-06 2023-09-23T20:00:53.923 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.923 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.923 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:00:53.923 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.523995+0000", 2023-09-23T20:00:53.923 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.3820999999999999e-05 2023-09-23T20:00:53.923 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.924 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.924 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:00:53.924 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.524039+0000", 2023-09-23T20:00:53.924 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000131748 2023-09-23T20:00:53.924 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.924 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.924 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:00:53.924 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.524171+0000", 2023-09-23T20:00:53.924 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000159603 2023-09-23T20:00:53.924 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.924 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.924 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:00:53.925 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.524331+0000", 2023-09-23T20:00:53.925 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0050509250000000004 2023-09-23T20:00:53.925 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.925 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.925 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:53.925 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.529382+0000", 2023-09-23T20:00:53.925 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000236911 2023-09-23T20:00:53.925 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.925 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.925 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:00:53.925 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.529618+0000", 2023-09-23T20:00:53.925 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.171243012 2023-09-23T20:00:53.926 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.926 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.926 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:53.926 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.700861+0000", 2023-09-23T20:00:53.926 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.3334000000000001e-05 2023-09-23T20:00:53.926 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.926 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.926 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:00:53.926 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.700915+0000", 2023-09-23T20:00:53.926 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.1083e-05 2023-09-23T20:00:53.926 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:00:53.927 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:00:53.927 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:00:53.927 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.700956+0000", 2023-09-23T20:00:53.927 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.176987484 2023-09-23T20:00:53.927 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:00:53.927 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:00:53.927 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:00:53.927 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:00:53.927 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:00:53.927 INFO:teuthology.orchestra.run.smithi092.stdout:} 2023-09-23T20:00:53.928 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:53.963 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:53.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:53.969 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:53.969 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:53 smithi016 bash[77995]: cluster 2023-09-23T20:00:52.637067+0000 mgr.y (mgr.25325) 1609 : cluster 0 pgmap v1643: 89 pgs: 89 active+clean; 1.0 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 572 op/s 2023-09-23T20:00:53.969 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:53 smithi016 bash[75004]: cluster 2023-09-23T20:00:52.637067+0000 mgr.y (mgr.25325) 1609 : cluster 0 pgmap v1643: 89 pgs: 89 active+clean; 1.0 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 572 op/s 2023-09-23T20:00:54.070 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:54.112 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:54.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:54.114 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:54.215 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:54.223 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:53 smithi092 bash[58855]: cluster 2023-09-23T20:00:52.637067+0000 mgr.y (mgr.25325) 1609 : cluster 0 pgmap v1643: 89 pgs: 89 active+clean; 1.0 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 572 op/s 2023-09-23T20:00:54.253 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:54.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:54.256 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:54.357 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:54.382 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:54.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:54.385 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:54.485 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:54.506 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:54.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:54.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:54.612 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:54.641 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:54.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:54.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:54.749 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:54.786 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:54.791 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T20:00:54.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:54.792 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:54.893 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:54.920 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:54.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:54.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:55.025 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:55.055 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:55.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:55.061 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:55.161 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:55.186 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:55.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:55.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:55.294 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:55.329 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:55.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:55.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:55.435 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:55.466 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:55.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:55.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:00:55.573 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:55.602 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:55.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:55.608 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:55.645 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:00:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:20:00:55] "GET /metrics HTTP/1.1" 200 36373 "" "Prometheus/2.33.4" 2023-09-23T20:00:55.709 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:55.734 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:55.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:55.741 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:55.841 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:55.881 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:55.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:55.885 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:55.896 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:55 smithi016 bash[75004]: cluster 2023-09-23T20:00:54.637678+0000 mgr.y (mgr.25325) 1610 : cluster 0 pgmap v1644: 89 pgs: 89 active+clean; 986 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 631 op/s 2023-09-23T20:00:55.896 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:55 smithi016 bash[77995]: cluster 2023-09-23T20:00:54.637678+0000 mgr.y (mgr.25325) 1610 : cluster 0 pgmap v1644: 89 pgs: 89 active+clean; 986 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 631 op/s 2023-09-23T20:00:55.986 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:55.995 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:55 smithi092 bash[58855]: cluster 2023-09-23T20:00:54.637678+0000 mgr.y (mgr.25325) 1610 : cluster 0 pgmap v1644: 89 pgs: 89 active+clean; 986 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 631 op/s 2023-09-23T20:00:56.009 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:56.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:56.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:56.113 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:56.136 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:56.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:56.142 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:00:56.243 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:56.283 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:56.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:56.285 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:56.386 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:56.416 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:56.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:56.421 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:56.522 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:56.551 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:56.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:56.557 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:56.657 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:56.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:56.691 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:56.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:56.792 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:56.818 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:56.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:56.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:56.922 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:56.945 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:56.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:56.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:57.049 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:57.075 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:57.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:57.078 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:57.179 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:57.216 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:57.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:57.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:57.325 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:57.354 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:57.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:57.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:57.457 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:57.493 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:57.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:57.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:57.601 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:57.639 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:57.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:57.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:57.746 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:57.791 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:57.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:57.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:57.900 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:57.923 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:57.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:57.925 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:58.015 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:57 smithi016 bash[77995]: cluster 2023-09-23T20:00:56.638223+0000 mgr.y (mgr.25325) 1611 : cluster 0 pgmap v1645: 89 pgs: 89 active+clean; 873 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 670 op/s 2023-09-23T20:00:58.015 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:57 smithi016 bash[75004]: cluster 2023-09-23T20:00:56.638223+0000 mgr.y (mgr.25325) 1611 : cluster 0 pgmap v1645: 89 pgs: 89 active+clean; 873 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 670 op/s 2023-09-23T20:00:58.026 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:58.052 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:58.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:58.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:58.155 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:58.182 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:58.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:58.185 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:58.286 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:58.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:57 smithi092 bash[58855]: cluster 2023-09-23T20:00:56.638223+0000 mgr.y (mgr.25325) 1611 : cluster 0 pgmap v1645: 89 pgs: 89 active+clean; 873 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 670 op/s 2023-09-23T20:00:58.319 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:58.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:58.322 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:58.423 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:58.457 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:58.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:58.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:58.561 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:58.600 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:58.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:58.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:58.703 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:58.733 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:58.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:58.737 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:00:58.838 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:58.861 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:00:58.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:58.866 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:00:58.967 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:58.994 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:58.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:58.997 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:59.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:58 smithi092 bash[58855]: audit 2023-09-23T20:00:58.699960+0000 mon.a (mon.0) 1940 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:00:59.097 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:59.123 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:59.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:59.128 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:59.228 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:59.258 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:59.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:59.261 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:59.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:58 smithi016 bash[75004]: audit 2023-09-23T20:00:58.699960+0000 mon.a (mon.0) 1940 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:00:59.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:58 smithi016 bash[77995]: audit 2023-09-23T20:00:58.699960+0000 mon.a (mon.0) 1940 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:00:59.361 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:59.387 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:59.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:59.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:59.493 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:59.514 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:59.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:59.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:59.618 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:59.634 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:00:59.643 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:59.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:59.653 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:59.753 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:59.780 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:59.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:59.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:59.883 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:59.912 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:59.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:00:59.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:00.022 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:00.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:00:59 smithi092 bash[58855]: cluster 2023-09-23T20:00:58.638690+0000 mgr.y (mgr.25325) 1612 : cluster 0 pgmap v1646: 89 pgs: 89 active+clean; 807 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 612 op/s 2023-09-23T20:01:00.057 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:00.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:00.059 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:00.160 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:00.188 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:00.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:00.194 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:00.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:00:59 smithi016 bash[75004]: cluster 2023-09-23T20:00:58.638690+0000 mgr.y (mgr.25325) 1612 : cluster 0 pgmap v1646: 89 pgs: 89 active+clean; 807 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 612 op/s 2023-09-23T20:01:00.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:00:59 smithi016 bash[77995]: cluster 2023-09-23T20:00:58.638690+0000 mgr.y (mgr.25325) 1612 : cluster 0 pgmap v1646: 89 pgs: 89 active+clean; 807 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 612 op/s 2023-09-23T20:01:00.294 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:00.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:00.337 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:00.337 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:00.438 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:00.478 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:00.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:00.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:00.584 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:00.608 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:00.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:00.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:00.712 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:00.730 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:00.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:00.731 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:00.832 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:00.864 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:00.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:00.866 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:01:00.967 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:00.992 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:00.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:00.994 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:01.095 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:01.134 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:01.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:01.141 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:01.242 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:01.269 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:01.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:01.271 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:01.372 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:01.401 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:01.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:01.403 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:01.504 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:01.515 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T20:01:01.533 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:01.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:01.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:01.560 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T20:01:01.613 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T20:01:01.637 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:01.651 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T20:01:01.673 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:01.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:01.680 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:01.696 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T20:01:01.727 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T20:01:01.774 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T20:01:01.781 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:01.814 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T20:01:01.816 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:01.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:01.819 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:01.825 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:01:01.825 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:01:01.825 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:01:01.825 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:01:01.825 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:01:01.825 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:01:01.826 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:01:01.826 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:01:01.920 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:01.948 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:01.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:01.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:02.058 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:02.066 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:01 smithi016 bash[75004]: cluster 2023-09-23T20:01:00.639322+0000 mgr.y (mgr.25325) 1613 : cluster 0 pgmap v1647: 89 pgs: 89 active+clean; 711 MiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2023-09-23T20:01:02.067 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:01 smithi016 bash[75004]: audit 2023-09-23T20:01:01.441767+0000 mon.c (mon.2) 242 : audit 0 from='client.? 172.21.15.16:0/2403709768' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:01:02.067 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:01 smithi016 bash[77995]: cluster 2023-09-23T20:01:00.639322+0000 mgr.y (mgr.25325) 1613 : cluster 0 pgmap v1647: 89 pgs: 89 active+clean; 711 MiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2023-09-23T20:01:02.067 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:01 smithi016 bash[77995]: audit 2023-09-23T20:01:01.441767+0000 mon.c (mon.2) 242 : audit 0 from='client.? 172.21.15.16:0/2403709768' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:01:02.094 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:02.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:02.098 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:02.198 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:02.206 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:01 smithi092 bash[58855]: cluster 2023-09-23T20:01:00.639322+0000 mgr.y (mgr.25325) 1613 : cluster 0 pgmap v1647: 89 pgs: 89 active+clean; 711 MiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2023-09-23T20:01:02.207 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:01 smithi092 bash[58855]: audit 2023-09-23T20:01:01.441767+0000 mon.c (mon.2) 242 : audit 0 from='client.? 172.21.15.16:0/2403709768' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:01:02.220 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:02.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:02.223 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:02.324 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:02.364 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:02.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:02.368 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:02.469 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:02.495 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:02.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:02.500 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:02.601 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:02.631 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:02.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:02.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:02.736 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:02.758 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:02.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:02.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:01:02.861 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:02.894 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:02.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:02.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:02.913 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:02 smithi016 bash[77995]: audit 2023-09-23T20:01:02.607599+0000 mon.c (mon.2) 243 : audit 1 from='client.? 172.21.15.16:0/1145660430' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:01:02.913 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:02 smithi016 bash[77995]: audit 2023-09-23T20:01:02.608079+0000 mon.a (mon.0) 1941 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:01:02.913 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:02 smithi016 bash[77995]: audit 2023-09-23T20:01:02.618668+0000 mon.c (mon.2) 244 : audit 1 from='client.? 172.21.15.16:0/271326033' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:01:02.913 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:02 smithi016 bash[77995]: audit 2023-09-23T20:01:02.623130+0000 mon.a (mon.0) 1942 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:01:02.914 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:02 smithi016 bash[75004]: audit 2023-09-23T20:01:02.607599+0000 mon.c (mon.2) 243 : audit 1 from='client.? 172.21.15.16:0/1145660430' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:01:02.914 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:02 smithi016 bash[75004]: audit 2023-09-23T20:01:02.608079+0000 mon.a (mon.0) 1941 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:01:02.914 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:02 smithi016 bash[75004]: audit 2023-09-23T20:01:02.618668+0000 mon.c (mon.2) 244 : audit 1 from='client.? 172.21.15.16:0/271326033' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:01:02.914 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:02 smithi016 bash[75004]: audit 2023-09-23T20:01:02.623130+0000 mon.a (mon.0) 1942 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:01:02.915 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is set 2023-09-23T20:01:02.915 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T20:01:02.998 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:03.023 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:03.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:03.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:03.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:02 smithi092 bash[58855]: audit 2023-09-23T20:01:02.607599+0000 mon.c (mon.2) 243 : audit 1 from='client.? 172.21.15.16:0/1145660430' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:01:03.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:02 smithi092 bash[58855]: audit 2023-09-23T20:01:02.608079+0000 mon.a (mon.0) 1941 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:01:03.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:02 smithi092 bash[58855]: audit 2023-09-23T20:01:02.618668+0000 mon.c (mon.2) 244 : audit 1 from='client.? 172.21.15.16:0/271326033' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:01:03.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:02 smithi092 bash[58855]: audit 2023-09-23T20:01:02.623130+0000 mon.a (mon.0) 1942 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:01:03.128 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:03.162 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:03.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:03.165 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:03.267 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:03.294 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:03.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:03.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:03.397 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:03.421 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:03.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:03.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:03.524 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:03.545 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:03.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:03.548 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:01:03.649 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:03.678 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:03.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:03.681 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:03.781 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:03.808 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:03.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:03.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:03.914 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:03.930 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:03 smithi016 bash[77995]: cluster 2023-09-23T20:01:02.639822+0000 mgr.y (mgr.25325) 1614 : cluster 0 pgmap v1648: 89 pgs: 89 active+clean; 671 MiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 584 op/s 2023-09-23T20:01:03.930 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:03 smithi016 bash[77995]: cluster 2023-09-23T20:01:02.900781+0000 mon.a (mon.0) 1943 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:01:03.930 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:03 smithi016 bash[77995]: audit 2023-09-23T20:01:02.905000+0000 mon.a (mon.0) 1944 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T20:01:03.930 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:03 smithi016 bash[77995]: audit 2023-09-23T20:01:02.905076+0000 mon.a (mon.0) 1945 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:01:03.930 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:03 smithi016 bash[77995]: cluster 2023-09-23T20:01:02.905153+0000 mon.a (mon.0) 1946 : cluster 0 osdmap e272: 8 total, 8 up, 8 in 2023-09-23T20:01:03.930 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:03 smithi016 bash[77995]: cluster 2023-09-23T20:01:02.908224+0000 osd.4 (osd.4) 211 : cluster 3 failed to encode map e272 with expected crc 2023-09-23T20:01:03.930 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:03 smithi016 bash[77995]: cluster 2023-09-23T20:01:02.951749+0000 osd.7 (osd.7) 186 : cluster 3 failed to encode map e272 with expected crc 2023-09-23T20:01:03.930 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:03 smithi016 bash[75004]: cluster 2023-09-23T20:01:02.639822+0000 mgr.y (mgr.25325) 1614 : cluster 0 pgmap v1648: 89 pgs: 89 active+clean; 671 MiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 584 op/s 2023-09-23T20:01:03.931 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:03 smithi016 bash[75004]: cluster 2023-09-23T20:01:02.900781+0000 mon.a (mon.0) 1943 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:01:03.931 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:03 smithi016 bash[75004]: audit 2023-09-23T20:01:02.905000+0000 mon.a (mon.0) 1944 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T20:01:03.931 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:03 smithi016 bash[75004]: audit 2023-09-23T20:01:02.905076+0000 mon.a (mon.0) 1945 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:01:03.931 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:03 smithi016 bash[75004]: cluster 2023-09-23T20:01:02.905153+0000 mon.a (mon.0) 1946 : cluster 0 osdmap e272: 8 total, 8 up, 8 in 2023-09-23T20:01:03.931 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:03 smithi016 bash[75004]: cluster 2023-09-23T20:01:02.908224+0000 osd.4 (osd.4) 211 : cluster 3 failed to encode map e272 with expected crc 2023-09-23T20:01:03.931 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:03 smithi016 bash[75004]: cluster 2023-09-23T20:01:02.951749+0000 osd.7 (osd.7) 186 : cluster 3 failed to encode map e272 with expected crc 2023-09-23T20:01:03.943 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:03.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:03.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:04.048 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:04.082 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:04.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:04.088 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:04.189 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:04.213 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:04.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:04.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:04.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:03 smithi092 bash[58855]: cluster 2023-09-23T20:01:02.639822+0000 mgr.y (mgr.25325) 1614 : cluster 0 pgmap v1648: 89 pgs: 89 active+clean; 671 MiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 584 op/s 2023-09-23T20:01:04.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:03 smithi092 bash[58855]: cluster 2023-09-23T20:01:02.900781+0000 mon.a (mon.0) 1943 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:01:04.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:03 smithi092 bash[58855]: audit 2023-09-23T20:01:02.905000+0000 mon.a (mon.0) 1944 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T20:01:04.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:03 smithi092 bash[58855]: audit 2023-09-23T20:01:02.905076+0000 mon.a (mon.0) 1945 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:01:04.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:03 smithi092 bash[58855]: cluster 2023-09-23T20:01:02.905153+0000 mon.a (mon.0) 1946 : cluster 0 osdmap e272: 8 total, 8 up, 8 in 2023-09-23T20:01:04.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:03 smithi092 bash[58855]: cluster 2023-09-23T20:01:02.908224+0000 osd.4 (osd.4) 211 : cluster 3 failed to encode map e272 with expected crc 2023-09-23T20:01:04.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:03 smithi092 bash[58855]: cluster 2023-09-23T20:01:02.951749+0000 osd.7 (osd.7) 186 : cluster 3 failed to encode map e272 with expected crc 2023-09-23T20:01:04.317 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:04.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:04.351 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:04.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:04.452 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:04.484 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:04.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:04.491 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:04.592 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:04.617 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:04.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:04.620 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:04.720 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:04.741 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:04.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:04.748 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:04.848 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:04.893 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:04.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:04.896 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:04.997 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:05.019 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:05.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:05.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:05.125 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:05.159 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:05.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:05.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:05.270 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:05.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:05.304 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:05.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:05.405 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:05.431 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:05.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:05.434 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:05.534 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:05.567 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:05.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:05.572 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:05.673 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:05.715 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:05.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:05.722 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:05.823 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:05.831 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:01:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:20:01:05] "GET /metrics HTTP/1.1" 200 36375 "" "Prometheus/2.33.4" 2023-09-23T20:01:05.849 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:05.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:05.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:05.955 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:05.985 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:05.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:05.989 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:06.090 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:06.123 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:06.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:06.126 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:01:06.134 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:01:06.142 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:05 smithi016 bash[77995]: cluster 2023-09-23T20:01:04.640419+0000 mgr.y (mgr.25325) 1615 : cluster 0 pgmap v1650: 89 pgs: 89 active+clean; 591 MiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 632 op/s 2023-09-23T20:01:06.143 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:05 smithi016 bash[75004]: cluster 2023-09-23T20:01:04.640419+0000 mgr.y (mgr.25325) 1615 : cluster 0 pgmap v1650: 89 pgs: 89 active+clean; 591 MiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 632 op/s 2023-09-23T20:01:06.227 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:06.237 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:05 smithi092 bash[58855]: cluster 2023-09-23T20:01:04.640419+0000 mgr.y (mgr.25325) 1615 : cluster 0 pgmap v1650: 89 pgs: 89 active+clean; 591 MiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 632 op/s 2023-09-23T20:01:06.253 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:06.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:06.255 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:06.356 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:06.379 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:06.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:06.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:06.484 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:06.510 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:06.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:06.512 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:06.613 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:06.641 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:06.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:06.644 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:06.745 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:06.775 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:06.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:06.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:06.837 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-23T20:01:06.883 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:06.912 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:06.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:06.918 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:07.019 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:07.055 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:07.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:07.062 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:07.162 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:07.205 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:07.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:07.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:07.312 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:07.360 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:07.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:07.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:07.463 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:07.504 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:07.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:07.507 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:07.608 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:07.655 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:07.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:07.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:07.764 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:07.805 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:07.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:07.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:07.909 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:07.945 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:07.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:07.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:08.048 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:08.075 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:08.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:08.083 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:08.183 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:08.193 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:07 smithi092 bash[58855]: cluster 2023-09-23T20:01:06.641221+0000 mgr.y (mgr.25325) 1616 : cluster 0 pgmap v1651: 89 pgs: 89 active+clean; 492 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 609 op/s 2023-09-23T20:01:08.207 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:08.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:08.209 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:01:08.225 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:07 smithi016 bash[77995]: cluster 2023-09-23T20:01:06.641221+0000 mgr.y (mgr.25325) 1616 : cluster 0 pgmap v1651: 89 pgs: 89 active+clean; 492 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 609 op/s 2023-09-23T20:01:08.225 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:07 smithi016 bash[75004]: cluster 2023-09-23T20:01:06.641221+0000 mgr.y (mgr.25325) 1616 : cluster 0 pgmap v1651: 89 pgs: 89 active+clean; 492 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 609 op/s 2023-09-23T20:01:08.310 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:08.337 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:08.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:08.342 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:08.443 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:08.470 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:08.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:08.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:08.556 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-09-23T20:01:08.557 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:01:08.577 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:08.602 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:08.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:08.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:08.689 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:01:08.705 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:08.727 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:08.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:08.732 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:08.833 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:08.870 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:08.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:08.873 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:08.974 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:09.008 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:09.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:09.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:09.117 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:09.149 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:09.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:09.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:09.252 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:09.289 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:09.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:09.295 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:09.396 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:09.426 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:09.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:09.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:01:09.532 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:09.562 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:09.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:09.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:09.667 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:09.696 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:09.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:09.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:09.804 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:09.839 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:09.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:09.843 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:09.851 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T20:01:09.943 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:09.985 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:09.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:09.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:10.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:09 smithi092 bash[58855]: cluster 2023-09-23T20:01:08.641812+0000 mgr.y (mgr.25325) 1617 : cluster 0 pgmap v1652: 89 pgs: 89 active+clean; 429 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 604 op/s 2023-09-23T20:01:10.090 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:10.112 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:10.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:10.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:10.215 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:10.246 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:10.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:10.249 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:10.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:09 smithi016 bash[75004]: cluster 2023-09-23T20:01:08.641812+0000 mgr.y (mgr.25325) 1617 : cluster 0 pgmap v1652: 89 pgs: 89 active+clean; 429 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 604 op/s 2023-09-23T20:01:10.288 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:09 smithi016 bash[77995]: cluster 2023-09-23T20:01:08.641812+0000 mgr.y (mgr.25325) 1617 : cluster 0 pgmap v1652: 89 pgs: 89 active+clean; 429 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 604 op/s 2023-09-23T20:01:10.349 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:10.399 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:10.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:10.407 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:10.507 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:10.538 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:10.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:10.545 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:10.645 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:10.684 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:10.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:10.688 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:10.789 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:10.817 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:10.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:10.824 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:01:10.925 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:10.965 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:10.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:10.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:11.068 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:11.109 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:11.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:11.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:11.216 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:11.246 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:11.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:11.248 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:11.349 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:11.381 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:11.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:11.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:01:11.485 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:11.510 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:11.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:11.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:11.621 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:11.643 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:11.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:11.645 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:01:11.746 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:11.773 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:11.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:11.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:11.876 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:11.914 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:11.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:11.921 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:12.022 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:12.030 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:11 smithi092 bash[58855]: cluster 2023-09-23T20:01:10.646228+0000 mgr.y (mgr.25325) 1618 : cluster 0 pgmap v1653: 89 pgs: 89 active+clean; 328 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 612 op/s 2023-09-23T20:01:12.049 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:12.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:12.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:12.152 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:12.175 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:12.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:12.177 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:12.278 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:12.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:11 smithi016 bash[77995]: cluster 2023-09-23T20:01:10.646228+0000 mgr.y (mgr.25325) 1618 : cluster 0 pgmap v1653: 89 pgs: 89 active+clean; 328 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 612 op/s 2023-09-23T20:01:12.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:11 smithi016 bash[75004]: cluster 2023-09-23T20:01:10.646228+0000 mgr.y (mgr.25325) 1618 : cluster 0 pgmap v1653: 89 pgs: 89 active+clean; 328 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 612 op/s 2023-09-23T20:01:12.318 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:12.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:12.323 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:12.369 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:01:12.369 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:01:12.424 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:12.451 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:12.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:12.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:01:12.556 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:12.592 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:12.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:12.595 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:12.696 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:12.736 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:12.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:12.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:12.840 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:12.867 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:12.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:12.875 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:01:12.975 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:13.002 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:13.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:13.006 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:13.106 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:13.135 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:13.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:13.138 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:13.239 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:13.283 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:13.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:13.293 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:13.390 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:01:13.394 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:13.437 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:13.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:13.446 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:13.547 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:13.582 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:13.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:13.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:13.692 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:13.726 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:13.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:13.729 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:13.830 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:13.865 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:13.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:13.870 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:01:13.971 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:14.001 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:14.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:14.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:01:14.105 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:14.133 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:14.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:14.138 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:14.239 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:14.247 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:13 smithi092 bash[58855]: cluster 2023-09-23T20:01:12.646761+0000 mgr.y (mgr.25325) 1619 : cluster 0 pgmap v1654: 89 pgs: 89 active+clean; 289 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 609 op/s 2023-09-23T20:01:14.248 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:13 smithi092 bash[58855]: audit 2023-09-23T20:01:13.698142+0000 mon.a (mon.0) 1947 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:01:14.271 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:14.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:14.276 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:01:14.300 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:13 smithi016 bash[75004]: cluster 2023-09-23T20:01:12.646761+0000 mgr.y (mgr.25325) 1619 : cluster 0 pgmap v1654: 89 pgs: 89 active+clean; 289 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 609 op/s 2023-09-23T20:01:14.300 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:13 smithi016 bash[75004]: audit 2023-09-23T20:01:13.698142+0000 mon.a (mon.0) 1947 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:01:14.300 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:13 smithi016 bash[77995]: cluster 2023-09-23T20:01:12.646761+0000 mgr.y (mgr.25325) 1619 : cluster 0 pgmap v1654: 89 pgs: 89 active+clean; 289 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 609 op/s 2023-09-23T20:01:14.300 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:13 smithi016 bash[77995]: audit 2023-09-23T20:01:13.698142+0000 mon.a (mon.0) 1947 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:01:14.377 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:14.408 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:01:14.425 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:14.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:14.429 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:14.530 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:14.589 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:14.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:14.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:14.695 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:14.721 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:14.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:14.724 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:14.824 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:14.857 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:14.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:14.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:14.963 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:14.987 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:14.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:14.990 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:01:15.091 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:15.122 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:15.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:15.129 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:15.229 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:15.252 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:15.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:15.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:01:15.356 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:15.387 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:15.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:15.390 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:15.490 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:15.529 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:15.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:15.533 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:15.634 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:15.683 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:15.687 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:01:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:20:01:15] "GET /metrics HTTP/1.1" 200 36363 "" "Prometheus/2.33.4" 2023-09-23T20:01:15.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:15.689 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:15.790 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:15.826 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:15.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:15.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:15.931 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:15.968 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:15.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:15.973 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:16.074 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:16.105 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:16.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:16.109 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:16.209 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:16.220 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:15 smithi092 bash[58855]: cluster 2023-09-23T20:01:14.647344+0000 mgr.y (mgr.25325) 1620 : cluster 0 pgmap v1655: 89 pgs: 89 active+clean; 207 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 631 op/s 2023-09-23T20:01:16.246 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:16.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:16.249 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:16.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:15 smithi016 bash[75004]: cluster 2023-09-23T20:01:14.647344+0000 mgr.y (mgr.25325) 1620 : cluster 0 pgmap v1655: 89 pgs: 89 active+clean; 207 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 631 op/s 2023-09-23T20:01:16.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:15 smithi016 bash[77995]: cluster 2023-09-23T20:01:14.647344+0000 mgr.y (mgr.25325) 1620 : cluster 0 pgmap v1655: 89 pgs: 89 active+clean; 207 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 631 op/s 2023-09-23T20:01:16.350 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:16.387 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:16.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:16.389 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:16.490 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:16.533 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:16.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:16.536 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:16.636 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:16.666 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:16.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:16.668 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:16.769 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:16.793 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:16.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:16.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:16.896 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:16.923 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:16.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:16.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:01:17.027 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:17.056 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:17.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:17.058 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:17.159 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:17.187 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:17.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:17.191 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:01:17.198 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Removed 99956 objects 2023-09-23T20:01:17.199 INFO:tasks.radosbench.radosbench.0.smithi016.stdout:Clean up completed and total clean up time :152.297 2023-09-23T20:01:17.292 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:17.314 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:17.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:17.316 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:17.417 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:17.443 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:17.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:17.446 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:17.546 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:17.575 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:17.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:17.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:17.678 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:17.708 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:17.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:18.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:18.249 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:17 smithi016 bash[77995]: cluster 2023-09-23T20:01:16.647940+0000 mgr.y (mgr.25325) 1621 : cluster 0 pgmap v1656: 89 pgs: 89 active+clean; 98 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 657 op/s 2023-09-23T20:01:18.249 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:17 smithi016 bash[75004]: cluster 2023-09-23T20:01:16.647940+0000 mgr.y (mgr.25325) 1621 : cluster 0 pgmap v1656: 89 pgs: 89 active+clean; 98 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 657 op/s 2023-09-23T20:01:18.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:17 smithi092 bash[58855]: cluster 2023-09-23T20:01:16.647940+0000 mgr.y (mgr.25325) 1621 : cluster 0 pgmap v1656: 89 pgs: 89 active+clean; 98 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 657 op/s 2023-09-23T20:01:18.350 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:18.374 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:18.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:18.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:18.477 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:18.497 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:18.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:18.499 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:18.600 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:18.622 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:18.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:18.625 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:18.725 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:18.757 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:18.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:18.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:18.860 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:18.870 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_9 2023-09-23T20:01:18.871 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_9 unique_pool_9 --yes-i-really-really-mean-it 2023-09-23T20:01:18.883 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:18.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:18.885 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:18.986 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:19.022 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:19.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:19.026 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:19.126 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:19.154 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:19.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:19.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:19.258 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:19.285 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:19.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:19.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:19.389 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:19.426 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:19.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:19.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:19.530 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:19.555 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:19.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:19.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:19.658 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:19.687 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:19.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:19.690 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:19.791 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:19.827 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:19.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:19.831 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:19.931 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:19.962 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:19.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:19.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:20.066 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:20.096 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:20.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:20.099 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:20.200 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:20.213 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:19 smithi092 bash[58855]: cluster 2023-09-23T20:01:18.648458+0000 mgr.y (mgr.25325) 1622 : cluster 0 pgmap v1657: 89 pgs: 89 active+clean; 49 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 590 op/s 2023-09-23T20:01:20.233 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:20.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:20.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:20.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:19 smithi016 bash[75004]: cluster 2023-09-23T20:01:18.648458+0000 mgr.y (mgr.25325) 1622 : cluster 0 pgmap v1657: 89 pgs: 89 active+clean; 49 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 590 op/s 2023-09-23T20:01:20.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:19 smithi016 bash[77995]: cluster 2023-09-23T20:01:18.648458+0000 mgr.y (mgr.25325) 1622 : cluster 0 pgmap v1657: 89 pgs: 89 active+clean; 49 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 590 op/s 2023-09-23T20:01:20.336 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:20.357 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:20.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:20.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:20.460 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:20.493 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:20.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:20.497 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:20.597 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:20.631 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:20.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:20.634 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:20.734 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:20.764 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:20.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:20.767 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:20.868 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:20.897 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:20.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:20.900 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:01:21.001 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:21.026 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:21.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:21.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:21.129 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:21.157 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:21.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:21.159 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:21.260 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:21.297 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:21.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:21.300 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:01:21.401 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:21.431 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:21.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:21.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:21.535 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:21.567 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:21.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:21.570 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:21.671 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:21.705 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:21.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:21.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:01:21.809 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:21.840 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:21.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:21.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:21.944 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:21.971 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:21.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:21.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:22.075 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:22.083 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:21 smithi016 bash[77995]: cluster 2023-09-23T20:01:20.649367+0000 mgr.y (mgr.25325) 1623 : cluster 0 pgmap v1658: 89 pgs: 89 active+clean; 7.7 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 561 op/s 2023-09-23T20:01:22.083 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:21 smithi016 bash[75004]: cluster 2023-09-23T20:01:20.649367+0000 mgr.y (mgr.25325) 1623 : cluster 0 pgmap v1658: 89 pgs: 89 active+clean; 7.7 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 561 op/s 2023-09-23T20:01:22.098 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:22.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:22.101 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:22.202 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:22.215 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:21 smithi092 bash[58855]: cluster 2023-09-23T20:01:20.649367+0000 mgr.y (mgr.25325) 1623 : cluster 0 pgmap v1658: 89 pgs: 89 active+clean; 7.7 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 561 op/s 2023-09-23T20:01:22.235 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:22.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:22.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:22.340 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:22.369 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:22.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:22.371 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:22.472 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:22.503 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:22.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:22.507 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:22.607 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:22.633 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:22.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:22.635 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:22.736 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:22.766 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:22.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:22.769 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:22.870 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:22.900 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:22.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:22.904 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:23.005 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:23.036 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:23.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:23.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:23.140 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:23.168 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:23.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:23.170 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:23.271 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:23.305 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:23.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:23.307 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:23.408 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:23.443 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:23.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:23.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:23.546 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:23.570 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:23.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:23.572 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:23.673 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:23.706 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:23.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:23.709 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:23.810 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:23.836 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:23.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:23.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:23.940 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:23.966 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:23.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:23.968 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:24.069 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:24.101 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:24.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:24.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:24.205 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:24.213 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:23 smithi016 bash[75004]: cluster 2023-09-23T20:01:22.650036+0000 mgr.y (mgr.25325) 1624 : cluster 0 pgmap v1659: 89 pgs: 89 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 436 op/s 2023-09-23T20:01:24.213 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:23 smithi016 bash[77995]: cluster 2023-09-23T20:01:22.650036+0000 mgr.y (mgr.25325) 1624 : cluster 0 pgmap v1659: 89 pgs: 89 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 436 op/s 2023-09-23T20:01:24.226 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:24.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:24.229 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:24.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:23 smithi092 bash[58855]: cluster 2023-09-23T20:01:22.650036+0000 mgr.y (mgr.25325) 1624 : cluster 0 pgmap v1659: 89 pgs: 89 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 436 op/s 2023-09-23T20:01:24.331 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:24.367 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:24.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:24.370 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:24.471 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:24.503 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:24.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:24.506 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:24.606 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:24.644 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:24.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:24.647 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:01:24.748 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:24.775 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:24.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:24.777 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:24.878 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:24.905 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:24.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:24.907 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:25.008 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:25.034 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:25.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:25.036 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:25.137 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:25.163 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:25.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:25.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:25.266 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:25.295 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:25.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:25.298 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:25.399 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:25.426 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:25.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:25.429 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:25.529 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:25.564 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:25.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:25.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:25.667 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:25.701 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:25.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:25.704 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:25.805 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:25.827 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:25.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:25.830 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:25.907 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:01:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:20:01:25] "GET /metrics HTTP/1.1" 200 36363 "" "Prometheus/2.33.4" 2023-09-23T20:01:25.930 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:25.965 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:25.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:25.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:26.068 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:26.103 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:26.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:26.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:26.206 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:26.219 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:25 smithi092 bash[58855]: cluster 2023-09-23T20:01:24.650922+0000 mgr.y (mgr.25325) 1625 : cluster 0 pgmap v1660: 89 pgs: 89 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 384 op/s 2023-09-23T20:01:26.241 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:26.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:26.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:26.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:25 smithi016 bash[75004]: cluster 2023-09-23T20:01:24.650922+0000 mgr.y (mgr.25325) 1625 : cluster 0 pgmap v1660: 89 pgs: 89 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 384 op/s 2023-09-23T20:01:26.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:25 smithi016 bash[77995]: cluster 2023-09-23T20:01:24.650922+0000 mgr.y (mgr.25325) 1625 : cluster 0 pgmap v1660: 89 pgs: 89 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 384 op/s 2023-09-23T20:01:26.345 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:26.379 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:26.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:26.382 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:01:26.482 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:26.513 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:26.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:26.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:26.516 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T20:01:26.541 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T20:01:26.563 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T20:01:26.585 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T20:01:26.607 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T20:01:26.616 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:26.630 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T20:01:26.648 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:26.650 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T20:01:26.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:26.651 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:26.672 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T20:01:26.674 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:01:26.674 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:01:26.674 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:01:26.674 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:01:26.674 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:01:26.674 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:01:26.674 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:01:26.675 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:01:26.752 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:26.786 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:26.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:26.790 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:26.818 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:01:26.890 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:26.922 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:26.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:26.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:27.017 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:26 smithi016 bash[75004]: audit 2023-09-23T20:01:26.490835+0000 mon.c (mon.2) 245 : audit 0 from='client.? 172.21.15.16:0/3629940912' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:01:27.025 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:27.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:26 smithi092 bash[58855]: audit 2023-09-23T20:01:26.490835+0000 mon.c (mon.2) 245 : audit 0 from='client.? 172.21.15.16:0/3629940912' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:01:27.061 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:27.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:27.064 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:27.164 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:27.185 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:27.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:27.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:27.288 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:27.308 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:27.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:27.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:27.311 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:27 smithi016 bash[77995]: audit 2023-09-23T20:01:26.490835+0000 mon.c (mon.2) 245 : audit 0 from='client.? 172.21.15.16:0/3629940912' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:01:27.412 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:27.446 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:27.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:27.449 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:01:27.550 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:27.584 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:27.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:27.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:27.688 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:27.716 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:27.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:27.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:27.819 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:27.851 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:27.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:27.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:27.956 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:27.989 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:27.990 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.7 to 1 (10000) 2023-09-23T20:01:27.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:27.992 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:28.092 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:28.118 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:28.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:28.120 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:28.212 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:27 smithi016 bash[75004]: cluster 2023-09-23T20:01:26.651619+0000 mgr.y (mgr.25325) 1626 : cluster 0 pgmap v1661: 89 pgs: 89 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 274 op/s 2023-09-23T20:01:28.212 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:27 smithi016 bash[75004]: audit 2023-09-23T20:01:27.212135+0000 mon.a (mon.0) 1948 : audit 1 from='client.? 172.21.15.16:0/4078086324' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:01:28.212 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:27 smithi016 bash[77995]: cluster 2023-09-23T20:01:26.651619+0000 mgr.y (mgr.25325) 1626 : cluster 0 pgmap v1661: 89 pgs: 89 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 274 op/s 2023-09-23T20:01:28.213 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:27 smithi016 bash[77995]: audit 2023-09-23T20:01:27.212135+0000 mon.a (mon.0) 1948 : audit 1 from='client.? 172.21.15.16:0/4078086324' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:01:28.221 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:28.235 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:27 smithi092 bash[58855]: cluster 2023-09-23T20:01:26.651619+0000 mgr.y (mgr.25325) 1626 : cluster 0 pgmap v1661: 89 pgs: 89 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 274 op/s 2023-09-23T20:01:28.235 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:27 smithi092 bash[58855]: audit 2023-09-23T20:01:27.212135+0000 mon.a (mon.0) 1948 : audit 1 from='client.? 172.21.15.16:0/4078086324' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:01:28.251 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:28.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:28.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:01:28.356 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:28.385 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:28.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:28.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:28.488 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:28.515 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:28.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:28.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:28.618 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:28.647 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:28.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:28.650 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:28.750 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:28.782 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:28.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:28.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:28.886 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:28.913 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:28.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:28.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:29.016 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:29.029 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:28 smithi092 bash[58855]: audit 2023-09-23T20:01:27.982857+0000 mon.a (mon.0) 1949 : audit 1 from='client.? 172.21.15.16:0/4078086324' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:01:29.029 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:28 smithi092 bash[58855]: cluster 2023-09-23T20:01:27.982934+0000 mon.a (mon.0) 1950 : cluster 0 osdmap e273: 8 total, 8 up, 8 in 2023-09-23T20:01:29.030 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:28 smithi092 bash[58855]: cluster 2023-09-23T20:01:27.984142+0000 osd.7 (osd.7) 187 : cluster 3 failed to encode map e273 with expected crc 2023-09-23T20:01:29.030 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:28 smithi092 bash[58855]: cluster 2023-09-23T20:01:28.232742+0000 osd.4 (osd.4) 212 : cluster 3 failed to encode map e273 with expected crc 2023-09-23T20:01:29.030 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:28 smithi092 bash[58855]: cluster 2023-09-23T20:01:28.541933+0000 osd.5 (osd.5) 195 : cluster 3 failed to encode map e273 with expected crc 2023-09-23T20:01:29.030 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:28 smithi092 bash[58855]: audit 2023-09-23T20:01:28.698412+0000 mon.a (mon.0) 1951 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:01:29.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:28 smithi016 bash[75004]: audit 2023-09-23T20:01:27.982857+0000 mon.a (mon.0) 1949 : audit 1 from='client.? 172.21.15.16:0/4078086324' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:01:29.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:28 smithi016 bash[75004]: cluster 2023-09-23T20:01:27.982934+0000 mon.a (mon.0) 1950 : cluster 0 osdmap e273: 8 total, 8 up, 8 in 2023-09-23T20:01:29.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:28 smithi016 bash[75004]: cluster 2023-09-23T20:01:27.984142+0000 osd.7 (osd.7) 187 : cluster 3 failed to encode map e273 with expected crc 2023-09-23T20:01:29.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:28 smithi016 bash[75004]: cluster 2023-09-23T20:01:28.232742+0000 osd.4 (osd.4) 212 : cluster 3 failed to encode map e273 with expected crc 2023-09-23T20:01:29.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:28 smithi016 bash[75004]: cluster 2023-09-23T20:01:28.541933+0000 osd.5 (osd.5) 195 : cluster 3 failed to encode map e273 with expected crc 2023-09-23T20:01:29.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:28 smithi016 bash[75004]: audit 2023-09-23T20:01:28.698412+0000 mon.a (mon.0) 1951 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:01:29.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:28 smithi016 bash[77995]: audit 2023-09-23T20:01:27.982857+0000 mon.a (mon.0) 1949 : audit 1 from='client.? 172.21.15.16:0/4078086324' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:01:29.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:28 smithi016 bash[77995]: cluster 2023-09-23T20:01:27.982934+0000 mon.a (mon.0) 1950 : cluster 0 osdmap e273: 8 total, 8 up, 8 in 2023-09-23T20:01:29.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:28 smithi016 bash[77995]: cluster 2023-09-23T20:01:27.984142+0000 osd.7 (osd.7) 187 : cluster 3 failed to encode map e273 with expected crc 2023-09-23T20:01:29.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:28 smithi016 bash[77995]: cluster 2023-09-23T20:01:28.232742+0000 osd.4 (osd.4) 212 : cluster 3 failed to encode map e273 with expected crc 2023-09-23T20:01:29.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:28 smithi016 bash[77995]: cluster 2023-09-23T20:01:28.541933+0000 osd.5 (osd.5) 195 : cluster 3 failed to encode map e273 with expected crc 2023-09-23T20:01:29.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:28 smithi016 bash[77995]: audit 2023-09-23T20:01:28.698412+0000 mon.a (mon.0) 1951 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:01:29.048 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:29.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:29.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:29.152 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:29.178 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:29.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:29.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:29.281 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:29.306 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:29.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:29.308 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:29.409 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:29.433 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:29.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:29.435 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:29.535 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:29.555 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:29.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:29.557 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:29.658 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:29.691 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:29.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:29.694 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:29.796 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:29.832 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:29.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:29.835 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:29.936 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:29.960 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:29.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:29.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:30.006 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is unset 2023-09-23T20:01:30.063 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:30.096 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:30.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:30.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:30.199 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:30.232 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:30.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:30.236 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:01:30.260 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:29 smithi016 bash[75004]: cluster 2023-09-23T20:01:28.652359+0000 mgr.y (mgr.25325) 1627 : cluster 0 pgmap v1663: 89 pgs: 89 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 76 op/s 2023-09-23T20:01:30.260 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:29 smithi016 bash[75004]: audit 2023-09-23T20:01:29.723568+0000 mon.c (mon.2) 246 : audit 1 from='client.? 172.21.15.16:0/296979586' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:01:30.260 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:29 smithi016 bash[75004]: audit 2023-09-23T20:01:29.724245+0000 mon.a (mon.0) 1952 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:01:30.261 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:30 smithi016 bash[77995]: cluster 2023-09-23T20:01:28.652359+0000 mgr.y (mgr.25325) 1627 : cluster 0 pgmap v1663: 89 pgs: 89 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 76 op/s 2023-09-23T20:01:30.261 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:30 smithi016 bash[77995]: audit 2023-09-23T20:01:29.723568+0000 mon.c (mon.2) 246 : audit 1 from='client.? 172.21.15.16:0/296979586' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:01:30.261 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:30 smithi016 bash[77995]: audit 2023-09-23T20:01:29.724245+0000 mon.a (mon.0) 1952 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:01:30.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:30 smithi092 bash[58855]: cluster 2023-09-23T20:01:28.652359+0000 mgr.y (mgr.25325) 1627 : cluster 0 pgmap v1663: 89 pgs: 89 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 76 op/s 2023-09-23T20:01:30.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:30 smithi092 bash[58855]: audit 2023-09-23T20:01:29.723568+0000 mon.c (mon.2) 246 : audit 1 from='client.? 172.21.15.16:0/296979586' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:01:30.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:30 smithi092 bash[58855]: audit 2023-09-23T20:01:29.724245+0000 mon.a (mon.0) 1952 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:01:30.337 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:30.369 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:30.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:30.372 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:30.473 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:30.499 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:30.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:30.501 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:30.602 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:30.624 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:30.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:30.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:30.727 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:30.758 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:30.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:30.761 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:30.837 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-09-23T20:01:30.837 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [], 2023-09-23T20:01:30.838 INFO:teuthology.orchestra.run.smithi016.stdout: "complaint_time": 30, 2023-09-23T20:01:30.838 INFO:teuthology.orchestra.run.smithi016.stdout: "num_blocked_ops": 0 2023-09-23T20:01:30.838 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-09-23T20:01:30.861 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:30.886 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:30.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:30.889 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:01:30.989 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:31.018 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:31.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:31.020 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:31.121 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:31.143 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:31.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:31.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:31.247 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:31.255 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:31 smithi016 bash[75004]: cluster 2023-09-23T20:01:29.986282+0000 mon.a (mon.0) 1953 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:01:31.255 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:31 smithi016 bash[75004]: audit 2023-09-23T20:01:29.994299+0000 mon.a (mon.0) 1954 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:01:31.255 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:31 smithi016 bash[75004]: cluster 2023-09-23T20:01:29.994458+0000 mon.a (mon.0) 1955 : cluster 0 osdmap e274: 8 total, 8 up, 8 in 2023-09-23T20:01:31.255 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:31 smithi016 bash[75004]: cluster 2023-09-23T20:01:29.998743+0000 osd.6 (osd.6) 207 : cluster 3 failed to encode map e274 with expected crc 2023-09-23T20:01:31.256 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:31 smithi016 bash[75004]: cluster 2023-09-23T20:01:29.998873+0000 osd.5 (osd.5) 196 : cluster 3 failed to encode map e274 with expected crc 2023-09-23T20:01:31.256 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:31 smithi016 bash[75004]: cluster 2023-09-23T20:01:29.998938+0000 osd.4 (osd.4) 213 : cluster 3 failed to encode map e274 with expected crc 2023-09-23T20:01:31.256 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:31 smithi016 bash[75004]: cluster 2023-09-23T20:01:29.999194+0000 osd.7 (osd.7) 188 : cluster 3 failed to encode map e274 with expected crc 2023-09-23T20:01:31.256 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:31 smithi016 bash[75004]: audit 2023-09-23T20:01:30.567320+0000 mon.a (mon.0) 1956 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:01:31.256 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:31 smithi016 bash[77995]: cluster 2023-09-23T20:01:29.986282+0000 mon.a (mon.0) 1953 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:01:31.256 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:31 smithi016 bash[77995]: audit 2023-09-23T20:01:29.994299+0000 mon.a (mon.0) 1954 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:01:31.256 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:31 smithi016 bash[77995]: cluster 2023-09-23T20:01:29.994458+0000 mon.a (mon.0) 1955 : cluster 0 osdmap e274: 8 total, 8 up, 8 in 2023-09-23T20:01:31.256 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:31 smithi016 bash[77995]: cluster 2023-09-23T20:01:29.998743+0000 osd.6 (osd.6) 207 : cluster 3 failed to encode map e274 with expected crc 2023-09-23T20:01:31.256 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:31 smithi016 bash[77995]: cluster 2023-09-23T20:01:29.998873+0000 osd.5 (osd.5) 196 : cluster 3 failed to encode map e274 with expected crc 2023-09-23T20:01:31.256 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:31 smithi016 bash[77995]: cluster 2023-09-23T20:01:29.998938+0000 osd.4 (osd.4) 213 : cluster 3 failed to encode map e274 with expected crc 2023-09-23T20:01:31.257 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:31 smithi016 bash[77995]: cluster 2023-09-23T20:01:29.999194+0000 osd.7 (osd.7) 188 : cluster 3 failed to encode map e274 with expected crc 2023-09-23T20:01:31.257 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:31 smithi016 bash[77995]: audit 2023-09-23T20:01:30.567320+0000 mon.a (mon.0) 1956 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:01:31.270 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:31.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:31.273 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:31.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:31 smithi092 bash[58855]: cluster 2023-09-23T20:01:29.986282+0000 mon.a (mon.0) 1953 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:01:31.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:31 smithi092 bash[58855]: audit 2023-09-23T20:01:29.994299+0000 mon.a (mon.0) 1954 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:01:31.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:31 smithi092 bash[58855]: cluster 2023-09-23T20:01:29.994458+0000 mon.a (mon.0) 1955 : cluster 0 osdmap e274: 8 total, 8 up, 8 in 2023-09-23T20:01:31.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:31 smithi092 bash[58855]: cluster 2023-09-23T20:01:29.998743+0000 osd.6 (osd.6) 207 : cluster 3 failed to encode map e274 with expected crc 2023-09-23T20:01:31.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:31 smithi092 bash[58855]: cluster 2023-09-23T20:01:29.998873+0000 osd.5 (osd.5) 196 : cluster 3 failed to encode map e274 with expected crc 2023-09-23T20:01:31.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:31 smithi092 bash[58855]: cluster 2023-09-23T20:01:29.998938+0000 osd.4 (osd.4) 213 : cluster 3 failed to encode map e274 with expected crc 2023-09-23T20:01:31.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:31 smithi092 bash[58855]: cluster 2023-09-23T20:01:29.999194+0000 osd.7 (osd.7) 188 : cluster 3 failed to encode map e274 with expected crc 2023-09-23T20:01:31.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:31 smithi092 bash[58855]: audit 2023-09-23T20:01:30.567320+0000 mon.a (mon.0) 1956 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:01:31.374 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:31.406 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:31.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:31.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:31.510 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:31.546 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:31.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:31.548 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:01:31.649 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:31.671 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:31.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:31.673 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:31.774 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:31.809 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:31.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:31.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:31.912 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:31.946 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:31.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:31.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:32.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:32 smithi092 bash[58855]: cluster 2023-09-23T20:01:30.652844+0000 mgr.y (mgr.25325) 1628 : cluster 0 pgmap v1665: 89 pgs: 89 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:01:32.051 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:32.073 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:32.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:32.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:32.176 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:32.200 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:32.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:32.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:32.264 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:32 smithi016 bash[77995]: cluster 2023-09-23T20:01:30.652844+0000 mgr.y (mgr.25325) 1628 : cluster 0 pgmap v1665: 89 pgs: 89 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:01:32.265 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:32 smithi016 bash[75004]: cluster 2023-09-23T20:01:30.652844+0000 mgr.y (mgr.25325) 1628 : cluster 0 pgmap v1665: 89 pgs: 89 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:01:32.302 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:01:32.302 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:32.333 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:32.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:32.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:32.436 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:32.472 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:32.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:32.475 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:01:32.576 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:32.601 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:32.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:32.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:32.687 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T20:01:32.688 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:01:32.705 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:32.739 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:32.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:32.741 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:32.842 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:32.874 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:32.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:32.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:32.978 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:33.006 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:33.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:33.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:33.109 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:33.141 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:33.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:33.144 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:33.245 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:33.268 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:33.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:33.270 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:33.371 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:33.412 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:33.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:33.414 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:33.515 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:33.547 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:33.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:33.549 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:33.650 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:33.683 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:33.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:33.686 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:01:33.786 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:33.821 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:33.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:33.824 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:33.925 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:33.955 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:33.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:33.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:34.059 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:34.087 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:34.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:34.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:34.191 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:34.227 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:34.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:34.229 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:34.246 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T20:01:34.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:34 smithi092 bash[58855]: cluster 2023-09-23T20:01:32.653397+0000 mgr.y (mgr.25325) 1629 : cluster 0 pgmap v1666: 89 pgs: 89 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:01:34.331 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:34.361 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:34.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:34.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:34.411 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:34 smithi016 bash[77995]: cluster 2023-09-23T20:01:32.653397+0000 mgr.y (mgr.25325) 1629 : cluster 0 pgmap v1666: 89 pgs: 89 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:01:34.411 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:34 smithi016 bash[75004]: cluster 2023-09-23T20:01:32.653397+0000 mgr.y (mgr.25325) 1629 : cluster 0 pgmap v1666: 89 pgs: 89 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:01:34.467 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:34.493 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:34.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:34.496 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:34.596 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:34.627 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:34.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:34.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:34.731 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:34.764 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:34.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:34.766 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:34.867 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:34.894 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:34.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:34.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:34.997 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:35.026 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:35.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:35.029 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:35.130 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:35.163 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:35.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:35.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:35.266 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:35.296 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:35.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:35.300 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:35.400 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:35.429 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:35.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:35.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:35.533 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:35.562 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:35.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:35.564 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:35.665 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:35.695 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:35.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:35.697 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:35.785 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:01:35 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:20:01:35] "GET /metrics HTTP/1.1" 200 36322 "" "Prometheus/2.33.4" 2023-09-23T20:01:35.798 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:35.830 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:35.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:35.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:35.934 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:35.964 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:35.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:35.966 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:36.067 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:36.094 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:36.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:36.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:36.155 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:01:36.168 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:36 smithi016 bash[77995]: cluster 2023-09-23T20:01:34.654350+0000 mgr.y (mgr.25325) 1630 : cluster 0 pgmap v1667: 89 pgs: 89 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:01:36.168 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:36 smithi016 bash[75004]: cluster 2023-09-23T20:01:34.654350+0000 mgr.y (mgr.25325) 1630 : cluster 0 pgmap v1667: 89 pgs: 89 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:01:36.198 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:36.209 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:36 smithi092 bash[58855]: cluster 2023-09-23T20:01:34.654350+0000 mgr.y (mgr.25325) 1630 : cluster 0 pgmap v1667: 89 pgs: 89 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:01:36.250 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:36.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:36.253 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:36.354 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:36.378 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:36.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:36.380 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:36.481 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:36.511 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:36.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:36.515 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:36.615 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:36.653 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:36.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:36.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:36.757 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:36.788 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:36.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:36.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:36.891 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:36.922 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:36.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:36.925 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:37.026 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:37.059 INFO:teuthology.orchestra.run.smithi016.stderr:pool 'unique_pool_9' removed 2023-09-23T20:01:37.063 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:37.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:37.067 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:37.168 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:37.194 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:37.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:37.196 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:37.282 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:37 smithi016 bash[77995]: audit 2023-09-23T20:01:36.747837+0000 mon.a (mon.0) 1957 : audit 1 from='client.? 172.21.15.16:0/4022479871' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:01:37.283 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:37 smithi016 bash[77995]: audit 2023-09-23T20:01:36.812989+0000 mon.a (mon.0) 1958 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:01:37.283 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:37 smithi016 bash[77995]: audit 2023-09-23T20:01:36.820625+0000 mon.a (mon.0) 1959 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:01:37.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:37 smithi016 bash[75004]: audit 2023-09-23T20:01:36.747837+0000 mon.a (mon.0) 1957 : audit 1 from='client.? 172.21.15.16:0/4022479871' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:01:37.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:37 smithi016 bash[75004]: audit 2023-09-23T20:01:36.812989+0000 mon.a (mon.0) 1958 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:01:37.283 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:37 smithi016 bash[75004]: audit 2023-09-23T20:01:36.820625+0000 mon.a (mon.0) 1959 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:01:37.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:37 smithi092 bash[58855]: audit 2023-09-23T20:01:36.747837+0000 mon.a (mon.0) 1957 : audit 1 from='client.? 172.21.15.16:0/4022479871' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:01:37.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:37 smithi092 bash[58855]: audit 2023-09-23T20:01:36.812989+0000 mon.a (mon.0) 1958 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:01:37.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:37 smithi092 bash[58855]: audit 2023-09-23T20:01:36.820625+0000 mon.a (mon.0) 1959 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:01:37.297 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:37.328 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:37.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:37.331 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:37.431 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:37.460 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:37.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:37.462 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:37.563 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:37.600 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:37.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:37.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:37.703 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:37.727 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:37.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:37.730 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:37.831 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:37.861 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:37.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:37.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:37.965 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:37.997 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:38.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:38.000 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:38.101 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:38.113 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:38 smithi016 bash[77995]: cluster 2023-09-23T20:01:36.655356+0000 mgr.y (mgr.25325) 1631 : cluster 0 pgmap v1668: 89 pgs: 89 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:01:38.113 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:38 smithi016 bash[77995]: audit 2023-09-23T20:01:37.029870+0000 mon.a (mon.0) 1960 : audit 1 from='client.? 172.21.15.16:0/4022479871' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T20:01:38.113 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:38 smithi016 bash[77995]: cluster 2023-09-23T20:01:37.029975+0000 mon.a (mon.0) 1961 : cluster 0 osdmap e275: 8 total, 8 up, 8 in 2023-09-23T20:01:38.113 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:38 smithi016 bash[77995]: cluster 2023-09-23T20:01:37.274079+0000 osd.7 (osd.7) 189 : cluster 3 failed to encode map e275 with expected crc 2023-09-23T20:01:38.114 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:38 smithi016 bash[77995]: cluster 2023-09-23T20:01:37.276588+0000 osd.7 (osd.7) 190 : cluster 3 failed to encode map e275 with expected crc 2023-09-23T20:01:38.114 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:38 smithi016 bash[77995]: cluster 2023-09-23T20:01:37.297035+0000 osd.4 (osd.4) 214 : cluster 3 failed to encode map e275 with expected crc 2023-09-23T20:01:38.114 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:38 smithi016 bash[75004]: cluster 2023-09-23T20:01:36.655356+0000 mgr.y (mgr.25325) 1631 : cluster 0 pgmap v1668: 89 pgs: 89 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:01:38.114 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:38 smithi016 bash[75004]: audit 2023-09-23T20:01:37.029870+0000 mon.a (mon.0) 1960 : audit 1 from='client.? 172.21.15.16:0/4022479871' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T20:01:38.115 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:38 smithi016 bash[75004]: cluster 2023-09-23T20:01:37.029975+0000 mon.a (mon.0) 1961 : cluster 0 osdmap e275: 8 total, 8 up, 8 in 2023-09-23T20:01:38.115 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:38 smithi016 bash[75004]: cluster 2023-09-23T20:01:37.274079+0000 osd.7 (osd.7) 189 : cluster 3 failed to encode map e275 with expected crc 2023-09-23T20:01:38.115 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:38 smithi016 bash[75004]: cluster 2023-09-23T20:01:37.276588+0000 osd.7 (osd.7) 190 : cluster 3 failed to encode map e275 with expected crc 2023-09-23T20:01:38.115 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:38 smithi016 bash[75004]: cluster 2023-09-23T20:01:37.297035+0000 osd.4 (osd.4) 214 : cluster 3 failed to encode map e275 with expected crc 2023-09-23T20:01:38.131 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:38.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:38.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:38.235 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:38.258 INFO:teuthology.task.sequential:In sequential, running task print... 2023-09-23T20:01:38.258 INFO:teuthology.task.print:**** done end radosbench.yaml 2023-09-23T20:01:38.259 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-09-23T20:01:38.263 INFO:tasks.workunit:Pulling workunits from ref pacific 2023-09-23T20:01:38.263 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-09-23T20:01:38.264 DEBUG:teuthology.orchestra.run.smithi016:> stat -- /home/ubuntu/cephtest/mnt.0 2023-09-23T20:01:38.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:38.266 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:38.267 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:38.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:38 smithi092 bash[58855]: cluster 2023-09-23T20:01:36.655356+0000 mgr.y (mgr.25325) 1631 : cluster 0 pgmap v1668: 89 pgs: 89 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:01:38.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:38 smithi092 bash[58855]: audit 2023-09-23T20:01:37.029870+0000 mon.a (mon.0) 1960 : audit 1 from='client.? 172.21.15.16:0/4022479871' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T20:01:38.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:38 smithi092 bash[58855]: cluster 2023-09-23T20:01:37.029975+0000 mon.a (mon.0) 1961 : cluster 0 osdmap e275: 8 total, 8 up, 8 in 2023-09-23T20:01:38.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:38 smithi092 bash[58855]: cluster 2023-09-23T20:01:37.274079+0000 osd.7 (osd.7) 189 : cluster 3 failed to encode map e275 with expected crc 2023-09-23T20:01:38.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:38 smithi092 bash[58855]: cluster 2023-09-23T20:01:37.276588+0000 osd.7 (osd.7) 190 : cluster 3 failed to encode map e275 with expected crc 2023-09-23T20:01:38.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:38 smithi092 bash[58855]: cluster 2023-09-23T20:01:37.297035+0000 osd.4 (osd.4) 214 : cluster 3 failed to encode map e275 with expected crc 2023-09-23T20:01:38.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:38.326 INFO:teuthology.orchestra.run.smithi016.stderr:stat: cannot stat '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-09-23T20:01:38.326 DEBUG:teuthology.orchestra.run.smithi016:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T20:01:38.368 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:38.381 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-09-23T20:01:38.381 DEBUG:teuthology.orchestra.run.smithi016:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-09-23T20:01:38.399 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:38.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:38.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:38.429 INFO:tasks.workunit:timeout=3h 2023-09-23T20:01:38.429 INFO:tasks.workunit:cleanup=True 2023-09-23T20:01:38.430 DEBUG:teuthology.orchestra.run.smithi016:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch pacific https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-09-23T20:01:38.503 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:38.535 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:38.536 INFO:tasks.workunit.client.0.smithi016.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-09-23T20:01:38.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:38.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:38.639 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:38.667 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:38.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:38.670 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:38.676 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:01:38.676 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:01:38.737 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:01:38.771 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:38.795 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:38.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:38.798 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:38.899 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:38.923 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:38.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:38.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:39.027 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:39.048 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:39.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:39.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:39.152 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:39.174 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:39.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:39.176 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:39.277 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:39.306 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:39.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:39.309 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:39.409 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:39.443 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:39.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:39.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:39.546 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:39.579 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:39.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:39.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:39.682 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:39.708 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:39.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:39.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:39.813 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:39.836 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:39.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:39.838 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:39.939 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:39.960 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:39.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:39.966 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:40.066 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:40.098 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:40.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:40.105 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:40.206 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:40.242 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:40.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:40.245 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:40.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:40 smithi016 bash[75004]: cluster 2023-09-23T20:01:38.656108+0000 mgr.y (mgr.25325) 1632 : cluster 0 pgmap v1670: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:01:40.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:40 smithi016 bash[77995]: cluster 2023-09-23T20:01:38.656108+0000 mgr.y (mgr.25325) 1632 : cluster 0 pgmap v1670: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:01:40.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:40 smithi092 bash[58855]: cluster 2023-09-23T20:01:38.656108+0000 mgr.y (mgr.25325) 1632 : cluster 0 pgmap v1670: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:01:40.346 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:40.372 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:40.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:40.375 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:40.475 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:40.508 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:40.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:40.511 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:40.612 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:40.640 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:40.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:40.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:01:40.744 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:40.765 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:01:40.769 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:40.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:40.772 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:40.872 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:40.904 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:40.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:40.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:41.008 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:41.042 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:41.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:41.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:01:41.146 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:41.173 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:41.175 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:41 smithi016 bash[77995]: audit 2023-09-23T20:01:40.126369+0000 mon.a (mon.0) 1962 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:01:41.175 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:41 smithi016 bash[77995]: audit 2023-09-23T20:01:40.133378+0000 mon.a (mon.0) 1963 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:01:41.176 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:41 smithi016 bash[77995]: audit 2023-09-23T20:01:40.602711+0000 mon.a (mon.0) 1964 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:01:41.176 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:41 smithi016 bash[77995]: audit 2023-09-23T20:01:40.604596+0000 mon.a (mon.0) 1965 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:01:41.176 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:41 smithi016 bash[77995]: audit 2023-09-23T20:01:40.613702+0000 mon.a (mon.0) 1966 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:01:41.176 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:41 smithi016 bash[75004]: audit 2023-09-23T20:01:40.126369+0000 mon.a (mon.0) 1962 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:01:41.176 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:41 smithi016 bash[75004]: audit 2023-09-23T20:01:40.133378+0000 mon.a (mon.0) 1963 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:01:41.176 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:41 smithi016 bash[75004]: audit 2023-09-23T20:01:40.602711+0000 mon.a (mon.0) 1964 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:01:41.176 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:41 smithi016 bash[75004]: audit 2023-09-23T20:01:40.604596+0000 mon.a (mon.0) 1965 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:01:41.176 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:41 smithi016 bash[75004]: audit 2023-09-23T20:01:40.613702+0000 mon.a (mon.0) 1966 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:01:41.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:41.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:41.278 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:41.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:41 smithi092 bash[58855]: audit 2023-09-23T20:01:40.126369+0000 mon.a (mon.0) 1962 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:01:41.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:41 smithi092 bash[58855]: audit 2023-09-23T20:01:40.133378+0000 mon.a (mon.0) 1963 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:01:41.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:41 smithi092 bash[58855]: audit 2023-09-23T20:01:40.602711+0000 mon.a (mon.0) 1964 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:01:41.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:41 smithi092 bash[58855]: audit 2023-09-23T20:01:40.604596+0000 mon.a (mon.0) 1965 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:01:41.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:41 smithi092 bash[58855]: audit 2023-09-23T20:01:40.613702+0000 mon.a (mon.0) 1966 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:01:41.310 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:41.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:41.313 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:01:41.414 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:41.446 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:41.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:41.450 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:41.552 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:41.575 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:41.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:41.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:41.678 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:41.708 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:41.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:41.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:41.812 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:41.845 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:41.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:41.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:01:41.949 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:41.977 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:41.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:41.979 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:42.080 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:42.111 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:42.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:42.115 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:42.215 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:42.232 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:42 smithi016 bash[77995]: cluster 2023-09-23T20:01:40.656797+0000 mgr.y (mgr.25325) 1633 : cluster 0 pgmap v1671: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:01:42.232 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:42 smithi016 bash[75004]: cluster 2023-09-23T20:01:40.656797+0000 mgr.y (mgr.25325) 1633 : cluster 0 pgmap v1671: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:01:42.237 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:42.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:42.239 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:42.340 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:42.353 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:42 smithi092 bash[58855]: cluster 2023-09-23T20:01:40.656797+0000 mgr.y (mgr.25325) 1633 : cluster 0 pgmap v1671: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:01:42.372 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:42.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:42.375 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:42.475 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:42.509 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:42.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:42.512 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:01:42.612 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:42.644 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:42.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:42.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:42.748 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:42.773 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:42.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:42.775 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:42.876 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:42.896 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:42.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:42.898 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:42.999 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:43.031 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:43.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:43.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:43.135 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:43.156 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:43.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:43.158 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:43.259 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:43.281 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:43.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:43.284 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:43.385 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:43.411 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:43.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:43.414 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:43.515 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:43.544 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:43.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:43.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:43.648 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:43.682 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:43.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:43.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:43.786 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:43.813 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:43.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:43.816 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:43.917 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:43.950 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:43.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:43.954 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:44.054 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:44.087 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:44.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:44.090 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:44.190 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:44.212 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:44.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:44.215 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:44.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:44 smithi092 bash[58855]: cluster 2023-09-23T20:01:42.657531+0000 mgr.y (mgr.25325) 1634 : cluster 0 pgmap v1672: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:01:44.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:44 smithi092 bash[58855]: audit 2023-09-23T20:01:43.698697+0000 mon.a (mon.0) 1967 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:01:44.316 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:44.328 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:44 smithi016 bash[75004]: cluster 2023-09-23T20:01:42.657531+0000 mgr.y (mgr.25325) 1634 : cluster 0 pgmap v1672: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:01:44.329 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:44 smithi016 bash[75004]: audit 2023-09-23T20:01:43.698697+0000 mon.a (mon.0) 1967 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:01:44.329 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:44 smithi016 bash[77995]: cluster 2023-09-23T20:01:42.657531+0000 mgr.y (mgr.25325) 1634 : cluster 0 pgmap v1672: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:01:44.329 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:44 smithi016 bash[77995]: audit 2023-09-23T20:01:43.698697+0000 mon.a (mon.0) 1967 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:01:44.343 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:44.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:44.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:44.446 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:44.473 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:44.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:44.476 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:44.576 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:44.601 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:44.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:44.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:44.707 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:44.727 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:44.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:44.730 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:44.830 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:44.857 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:44.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:44.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:44.960 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:44.985 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:44.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:44.987 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:45.088 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:45.120 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:45.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:45.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:45.224 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:45.246 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:45.247 DEBUG:teuthology.orchestra.run.smithi016:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-09-23T20:01:45.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:45.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:45.349 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:45.379 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:45.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:45.382 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:45.482 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:45.513 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:45.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:45.516 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:01:45.579 INFO:tasks.workunit.client.0.smithi016.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-09-23T20:01:45.594 INFO:tasks.workunit.client.0.smithi016.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T20:01:45.595 INFO:tasks.workunit.client.0.smithi016.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-09-23T20:01:45.617 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:45.643 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:45.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:45.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:45.746 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:45.772 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:45.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:45.775 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:45.875 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:45.886 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:01:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:20:01:45] "GET /metrics HTTP/1.1" 200 33967 "" "Prometheus/2.33.4" 2023-09-23T20:01:45.904 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:45.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:45.907 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:46.008 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:46.039 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:46.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:46.041 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:46.142 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:46.173 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:46.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:46.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:46.276 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:46.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:46 smithi092 bash[58855]: cluster 2023-09-23T20:01:44.658159+0000 mgr.y (mgr.25325) 1635 : cluster 0 pgmap v1673: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:01:46.307 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:46.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:46.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:46.411 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:46.450 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:46.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:46.453 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:46.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:46 smithi016 bash[75004]: cluster 2023-09-23T20:01:44.658159+0000 mgr.y (mgr.25325) 1635 : cluster 0 pgmap v1673: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:01:46.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:46 smithi016 bash[77995]: cluster 2023-09-23T20:01:44.658159+0000 mgr.y (mgr.25325) 1635 : cluster 0 pgmap v1673: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:01:46.554 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:46.585 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:46.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:46.588 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:46.689 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:46.723 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:46.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:46.726 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:46.826 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:46.857 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:46.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:46.860 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:46.961 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:47.029 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:47.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:47.032 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:47.133 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:47.162 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:47.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:47.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:47.266 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:47.294 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:47.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:47.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:47.398 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:47.434 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:47.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:47.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:47.537 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:47.565 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:47.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:47.567 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:47.668 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:47.694 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:47.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:47.697 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:47.797 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:47.830 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:47.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:47.833 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:47.934 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:47.965 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:47.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:47.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:48.069 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:48.093 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:48.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:48.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:48.196 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:48.226 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:48.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:48.229 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:48.330 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:48.341 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:48 smithi016 bash[75004]: cluster 2023-09-23T20:01:46.658821+0000 mgr.y (mgr.25325) 1636 : cluster 0 pgmap v1674: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:01:48.341 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:48 smithi016 bash[77995]: cluster 2023-09-23T20:01:46.658821+0000 mgr.y (mgr.25325) 1636 : cluster 0 pgmap v1674: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:01:48.355 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:48.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:48.357 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:48.458 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:48.471 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:48 smithi092 bash[58855]: cluster 2023-09-23T20:01:46.658821+0000 mgr.y (mgr.25325) 1636 : cluster 0 pgmap v1674: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:01:48.490 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:48.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:48.493 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:01:48.594 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:48.615 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:48.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:48.617 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:48.718 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:48.749 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:48.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:48.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:48.852 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:48.880 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:48.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:48.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:48.983 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:49.015 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:49.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:49.017 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:49.118 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:49.147 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:49.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:49.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:49.250 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:49.275 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:49.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:49.277 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:49.378 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:49.404 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:49.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:49.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:49.507 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:49.535 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:49.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:49.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:49.638 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:49.679 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:49.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:49.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:01:49.784 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:49.825 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:49.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:49.832 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:49.933 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:49.965 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:49.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:49.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:50.069 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:50.103 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:50.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:50.106 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:50.207 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:50.232 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:50.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:50.234 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:50.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:50 smithi092 bash[58855]: cluster 2023-09-23T20:01:48.659238+0000 mgr.y (mgr.25325) 1637 : cluster 0 pgmap v1675: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:01:50.335 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:50.362 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:50.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:50.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:50.466 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:50.498 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:50.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:50.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:50.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:50 smithi016 bash[75004]: cluster 2023-09-23T20:01:48.659238+0000 mgr.y (mgr.25325) 1637 : cluster 0 pgmap v1675: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:01:50.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:50 smithi016 bash[77995]: cluster 2023-09-23T20:01:48.659238+0000 mgr.y (mgr.25325) 1637 : cluster 0 pgmap v1675: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:01:50.602 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:50.633 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:50.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:50.637 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:50.738 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:50.766 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:50.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:50.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:50.870 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:50.903 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:50.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:50.906 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:01:51.007 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:51.042 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:51.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:51.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:51.148 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:51.181 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:51.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:51.185 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:51.286 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:51.321 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:51.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:51.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:51.425 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:51.458 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:51.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:51.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:01:51.561 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:51.593 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:51.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:51.596 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:51.698 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:51.732 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:51.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:51.735 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:51.836 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:51.862 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:51.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:51.865 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:51.966 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:51.997 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:51.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:52.000 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:01:52.101 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:52.134 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:52.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:52.138 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:52.239 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:52.262 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:52.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:52.266 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:52.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:52 smithi092 bash[58855]: cluster 2023-09-23T20:01:50.660181+0000 mgr.y (mgr.25325) 1638 : cluster 0 pgmap v1676: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:01:52.366 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:52.393 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:52.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:52.396 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:52.497 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:52.527 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:52.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:52.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:52.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:52 smithi016 bash[75004]: cluster 2023-09-23T20:01:50.660181+0000 mgr.y (mgr.25325) 1638 : cluster 0 pgmap v1676: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:01:52.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:52 smithi016 bash[77995]: cluster 2023-09-23T20:01:50.660181+0000 mgr.y (mgr.25325) 1638 : cluster 0 pgmap v1676: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:01:52.631 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:52.663 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:52.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:52.666 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:52.769 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:52.788 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:52.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:52.790 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:52.832 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-09-23T20:01:52.832 INFO:teuthology.orchestra.run.smithi016.stdout: "size": 20, 2023-09-23T20:01:52.832 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 600, 2023-09-23T20:01:52.832 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [ 2023-09-23T20:01:52.832 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.832 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46640.0:57325 13.8 13:1e314a4e:::benchmark_data_smithi016_205181_object57324:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e238)", 2023-09-23T20:01:52.833 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:52:28.025973+0000", 2023-09-23T20:01:52.833 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 564.79981824799995, 2023-09-23T20:01:52.833 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.30022919300000001, 2023-09-23T20:01:52.833 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:01:52.833 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:01:52.833 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:01:52.833 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46640", 2023-09-23T20:01:52.833 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2656794142", 2023-09-23T20:01:52.833 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 57325 2023-09-23T20:01:52.834 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.834 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:01:52.834 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.834 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:01:52.834 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.025973+0000", 2023-09-23T20:01:52.834 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:01:52.834 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.835 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.835 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:01:52.835 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.025973+0000", 2023-09-23T20:01:52.835 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:01:52.835 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.835 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.835 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:01:52.835 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.025972+0000", 2023-09-23T20:01:52.835 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T20:01:52.836 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.836 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.836 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:01:52.836 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.026008+0000", 2023-09-23T20:01:52.836 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.6449999999999998e-05 2023-09-23T20:01:52.836 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.837 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.837 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:01:52.837 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.026023+0000", 2023-09-23T20:01:52.837 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.4228e-05 2023-09-23T20:01:52.837 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.837 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.837 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:01:52.837 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.026030+0000", 2023-09-23T20:01:52.837 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.571e-06 2023-09-23T20:01:52.838 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.838 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.838 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:01:52.838 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.026082+0000", 2023-09-23T20:01:52.838 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.1944999999999998e-05 2023-09-23T20:01:52.838 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.839 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.839 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:01:52.839 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.026236+0000", 2023-09-23T20:01:52.839 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00015398000000000001 2023-09-23T20:01:52.839 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.839 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.839 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 5,6", 2023-09-23T20:01:52.840 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.026398+0000", 2023-09-23T20:01:52.840 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000161772 2023-09-23T20:01:52.840 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.840 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.840 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T20:01:52.840 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.029290+0000", 2023-09-23T20:01:52.840 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.002892099 2023-09-23T20:01:52.841 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.841 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.841 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:52.841 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.031242+0000", 2023-09-23T20:01:52.841 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001952304 2023-09-23T20:01:52.841 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.841 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.842 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:52.842 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.326016+0000", 2023-09-23T20:01:52.842 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.29477351200000002 2023-09-23T20:01:52.842 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.842 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.842 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:01:52.843 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.326135+0000", 2023-09-23T20:01:52.843 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00011909 2023-09-23T20:01:52.843 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.843 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.843 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:01:52.843 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.326202+0000", 2023-09-23T20:01:52.843 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.711e-05 2023-09-23T20:01:52.844 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:01:52.844 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:01:52.844 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:01:52.844 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.844 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.844 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46640.0:57343 13.8 13:128d5368:::benchmark_data_smithi016_205181_object57342:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e238)", 2023-09-23T20:01:52.844 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:52:28.043202+0000", 2023-09-23T20:01:52.844 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 564.78258916599998, 2023-09-23T20:01:52.844 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.28672608500000002, 2023-09-23T20:01:52.845 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:01:52.845 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:01:52.845 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:01:52.845 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46640", 2023-09-23T20:01:52.845 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2656794142", 2023-09-23T20:01:52.845 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 57343 2023-09-23T20:01:52.845 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.845 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:01:52.846 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.846 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:01:52.846 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.043202+0000", 2023-09-23T20:01:52.846 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:01:52.846 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.846 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.846 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:01:52.846 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.043202+0000", 2023-09-23T20:01:52.846 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:01:52.846 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.846 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.847 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:01:52.847 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.043201+0000", 2023-09-23T20:01:52.847 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T20:01:52.847 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.847 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.847 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:01:52.847 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.043232+0000", 2023-09-23T20:01:52.847 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.0549999999999997e-05 2023-09-23T20:01:52.847 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.847 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.847 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:01:52.848 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.043243+0000", 2023-09-23T20:01:52.848 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.135e-05 2023-09-23T20:01:52.848 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.848 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.848 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:01:52.848 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.043249+0000", 2023-09-23T20:01:52.848 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.0120000000000002e-06 2023-09-23T20:01:52.848 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.848 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.848 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:01:52.848 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.043303+0000", 2023-09-23T20:01:52.849 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.3511000000000003e-05 2023-09-23T20:01:52.849 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.849 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.849 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:01:52.849 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.043464+0000", 2023-09-23T20:01:52.849 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00016114499999999999 2023-09-23T20:01:52.849 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.849 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.849 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 5,6", 2023-09-23T20:01:52.849 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.043668+0000", 2023-09-23T20:01:52.849 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00020468799999999999 2023-09-23T20:01:52.849 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.850 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.850 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:52.850 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.046186+0000", 2023-09-23T20:01:52.850 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0025178610000000001 2023-09-23T20:01:52.850 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.850 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.850 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T20:01:52.850 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.046206+0000", 2023-09-23T20:01:52.850 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.9286000000000001e-05 2023-09-23T20:01:52.850 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.850 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.850 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:52.851 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.329815+0000", 2023-09-23T20:01:52.851 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.28360949499999999 2023-09-23T20:01:52.851 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.851 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.851 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:01:52.851 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.329876+0000", 2023-09-23T20:01:52.851 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.0569999999999998e-05 2023-09-23T20:01:52.851 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.851 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.851 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:01:52.851 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:52:28.329928+0000", 2023-09-23T20:01:52.851 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.2414999999999999e-05 2023-09-23T20:01:52.852 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:01:52.852 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:01:52.852 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:01:52.852 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.852 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.852 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46877.0:32633 14.6 14:6f340fca:::benchmark_data_smithi016_228083_object32632:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-09-23T20:01:52.852 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:57:32.054419+0000", 2023-09-23T20:01:52.852 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 260.77137173400001, 2023-09-23T20:01:52.852 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.31474886699999999, 2023-09-23T20:01:52.852 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:01:52.852 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:01:52.852 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:01:52.853 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46877", 2023-09-23T20:01:52.853 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:01:52.853 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 32633 2023-09-23T20:01:52.853 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.853 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:01:52.853 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.853 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:01:52.853 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.054419+0000", 2023-09-23T20:01:52.853 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:01:52.853 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.853 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.853 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:01:52.854 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.054419+0000", 2023-09-23T20:01:52.854 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:01:52.854 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.854 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.854 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:01:52.854 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.054418+0000", 2023-09-23T20:01:52.854 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-09-23T20:01:52.854 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.854 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.854 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:01:52.854 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.054444+0000", 2023-09-23T20:01:52.854 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.6101e-05 2023-09-23T20:01:52.855 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.855 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.855 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:01:52.855 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.054457+0000", 2023-09-23T20:01:52.855 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.306e-05 2023-09-23T20:01:52.855 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.855 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.855 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:01:52.855 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.054465+0000", 2023-09-23T20:01:52.855 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.2840000000000006e-06 2023-09-23T20:01:52.855 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.855 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.856 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:01:52.856 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.054518+0000", 2023-09-23T20:01:52.856 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.3016999999999997e-05 2023-09-23T20:01:52.856 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.856 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.856 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:01:52.856 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.054747+0000", 2023-09-23T20:01:52.856 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00022910699999999999 2023-09-23T20:01:52.856 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.856 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.856 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:01:52.856 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.054900+0000", 2023-09-23T20:01:52.857 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00015242399999999999 2023-09-23T20:01:52.857 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.857 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.857 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:52.857 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.369002+0000", 2023-09-23T20:01:52.857 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.31410250000000001 2023-09-23T20:01:52.857 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.857 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.857 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T20:01:52.857 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.369018+0000", 2023-09-23T20:01:52.857 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.5424999999999999e-05 2023-09-23T20:01:52.857 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.858 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.858 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:52.858 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.369058+0000", 2023-09-23T20:01:52.858 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.0711999999999999e-05 2023-09-23T20:01:52.858 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.858 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.858 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:01:52.858 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.369111+0000", 2023-09-23T20:01:52.858 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.2889000000000001e-05 2023-09-23T20:01:52.858 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.858 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.858 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:01:52.859 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.369168+0000", 2023-09-23T20:01:52.859 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.6916999999999997e-05 2023-09-23T20:01:52.859 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:01:52.859 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:01:52.859 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:01:52.859 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.859 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.859 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop_reply(client.46877.0:32633 14.6 e261/256 ondisk, result = 0)", 2023-09-23T20:01:52.859 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:57:32.058418+0000", 2023-09-23T20:01:52.859 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 260.76737274499999, 2023-09-23T20:01:52.859 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.310601026, 2023-09-23T20:01:52.859 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:01:52.860 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "started", 2023-09-23T20:01:52.860 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:01:52.860 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.860 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:01:52.860 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.058418+0000", 2023-09-23T20:01:52.860 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:01:52.860 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.860 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.861 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:01:52.861 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.058418+0000", 2023-09-23T20:01:52.861 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:01:52.861 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.861 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.861 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:01:52.861 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.058417+0000", 2023-09-23T20:01:52.861 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T20:01:52.861 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.861 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.861 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:01:52.861 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.058433+0000", 2023-09-23T20:01:52.862 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.5531000000000001e-05 2023-09-23T20:01:52.862 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.862 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.862 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:01:52.862 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.368773+0000", 2023-09-23T20:01:52.862 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.31034017899999999 2023-09-23T20:01:52.862 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.862 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.862 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:01:52.862 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.368788+0000", 2023-09-23T20:01:52.863 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.4793e-05 2023-09-23T20:01:52.863 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.863 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.863 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:01:52.863 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.368977+0000", 2023-09-23T20:01:52.863 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00018866600000000001 2023-09-23T20:01:52.863 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.863 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.863 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:01:52.863 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.368991+0000", 2023-09-23T20:01:52.863 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.4089e-05 2023-09-23T20:01:52.864 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.864 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.864 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:01:52.864 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.369019+0000", 2023-09-23T20:01:52.864 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.8807999999999999e-05 2023-09-23T20:01:52.864 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:01:52.864 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:01:52.864 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:01:52.865 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.865 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.865 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46877.0:37521 14.0 14:01ed7a9d:::benchmark_data_smithi016_228083_object37520:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-09-23T20:01:52.865 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:57:36.004529+0000", 2023-09-23T20:01:52.865 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 256.82126178300001, 2023-09-23T20:01:52.865 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.242428481, 2023-09-23T20:01:52.865 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:01:52.865 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:01:52.865 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:01:52.865 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46877", 2023-09-23T20:01:52.865 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:01:52.865 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 37521 2023-09-23T20:01:52.866 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.866 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:01:52.866 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.866 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:01:52.866 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.004529+0000", 2023-09-23T20:01:52.866 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:01:52.866 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.867 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.867 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:01:52.867 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.004529+0000", 2023-09-23T20:01:52.867 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:01:52.867 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.867 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.867 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:01:52.867 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.004529+0000", 2023-09-23T20:01:52.868 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T20:01:52.868 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.868 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.868 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:01:52.868 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.004584+0000", 2023-09-23T20:01:52.868 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.5247999999999997e-05 2023-09-23T20:01:52.868 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.868 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.869 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:01:52.869 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.004596+0000", 2023-09-23T20:01:52.869 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1814000000000001e-05 2023-09-23T20:01:52.869 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.869 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.869 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:01:52.870 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.004603+0000", 2023-09-23T20:01:52.870 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.5689999999999999e-06 2023-09-23T20:01:52.870 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.870 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.870 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:01:52.870 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.009703+0000", 2023-09-23T20:01:52.870 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0050999690000000002 2023-09-23T20:01:52.870 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.871 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.871 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:01:52.871 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.009809+0000", 2023-09-23T20:01:52.871 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00010566299999999999 2023-09-23T20:01:52.871 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.871 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.871 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:01:52.872 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.009901+0000", 2023-09-23T20:01:52.872 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.1693000000000005e-05 2023-09-23T20:01:52.872 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.872 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.872 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:52.872 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.245771+0000", 2023-09-23T20:01:52.872 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.235870471 2023-09-23T20:01:52.872 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.873 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.873 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T20:01:52.873 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.245856+0000", 2023-09-23T20:01:52.873 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.5402999999999995e-05 2023-09-23T20:01:52.873 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.873 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.873 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:52.873 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.246872+0000", 2023-09-23T20:01:52.873 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001015515 2023-09-23T20:01:52.873 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.874 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.874 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:01:52.874 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.246921+0000", 2023-09-23T20:01:52.874 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.9431999999999997e-05 2023-09-23T20:01:52.874 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.874 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.874 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:01:52.874 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.246958+0000", 2023-09-23T20:01:52.874 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.6399999999999997e-05 2023-09-23T20:01:52.875 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:01:52.875 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:01:52.875 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:01:52.875 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.875 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.875 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46877.0:37526 14.0 14:03545d35:::benchmark_data_smithi016_228083_object37525:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-09-23T20:01:52.875 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:57:36.004695+0000", 2023-09-23T20:01:52.875 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 256.82109611300001, 2023-09-23T20:01:52.875 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.24465336600000001, 2023-09-23T20:01:52.875 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:01:52.876 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:01:52.876 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:01:52.876 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46877", 2023-09-23T20:01:52.876 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:01:52.876 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 37526 2023-09-23T20:01:52.876 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.876 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:01:52.876 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.876 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:01:52.876 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.004695+0000", 2023-09-23T20:01:52.876 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:01:52.876 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.877 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.877 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:01:52.877 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.004695+0000", 2023-09-23T20:01:52.877 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:01:52.877 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.877 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.877 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:01:52.877 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.004695+0000", 2023-09-23T20:01:52.877 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T20:01:52.877 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.877 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.878 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:01:52.878 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.004714+0000", 2023-09-23T20:01:52.878 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.8773000000000001e-05 2023-09-23T20:01:52.878 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.878 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.878 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:01:52.878 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.004718+0000", 2023-09-23T20:01:52.878 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.2939999999999999e-06 2023-09-23T20:01:52.878 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.879 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.879 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:01:52.879 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.004721+0000", 2023-09-23T20:01:52.879 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.0809999999999998e-06 2023-09-23T20:01:52.879 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.879 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.879 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:01:52.879 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.011190+0000", 2023-09-23T20:01:52.879 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.006469018 2023-09-23T20:01:52.880 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.880 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.880 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:01:52.880 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.011340+0000", 2023-09-23T20:01:52.880 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000150049 2023-09-23T20:01:52.880 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.880 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.880 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:01:52.880 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.011497+0000", 2023-09-23T20:01:52.881 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00015707699999999999 2023-09-23T20:01:52.881 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.881 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.881 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:52.881 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.245840+0000", 2023-09-23T20:01:52.881 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.234342732 2023-09-23T20:01:52.881 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.881 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.881 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T20:01:52.881 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.246835+0000", 2023-09-23T20:01:52.882 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00099527899999999996 2023-09-23T20:01:52.882 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.882 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.882 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:52.882 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.249157+0000", 2023-09-23T20:01:52.882 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0023215419999999998 2023-09-23T20:01:52.882 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.882 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.882 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:01:52.882 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.249214+0000", 2023-09-23T20:01:52.882 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.7711999999999999e-05 2023-09-23T20:01:52.882 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.883 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.883 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:01:52.883 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.249348+0000", 2023-09-23T20:01:52.883 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000134091 2023-09-23T20:01:52.883 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:01:52.883 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:01:52.883 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:01:52.883 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.883 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.883 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46877.0:40016 14.b 14:d86fb8d2:::benchmark_data_smithi016_228083_object40015:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-09-23T20:01:52.883 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:57:38.113494+0000", 2023-09-23T20:01:52.883 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 254.71229751800001, 2023-09-23T20:01:52.884 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.28720668100000002, 2023-09-23T20:01:52.884 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:01:52.884 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:01:52.884 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:01:52.884 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46877", 2023-09-23T20:01:52.884 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:01:52.884 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 40016 2023-09-23T20:01:52.884 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.884 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:01:52.884 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.884 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:01:52.884 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.113494+0000", 2023-09-23T20:01:52.885 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:01:52.885 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.885 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.885 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:01:52.885 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.113494+0000", 2023-09-23T20:01:52.885 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:01:52.885 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.885 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.885 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:01:52.885 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.113492+0000", 2023-09-23T20:01:52.885 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T20:01:52.885 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.886 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.886 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:01:52.886 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.113512+0000", 2023-09-23T20:01:52.886 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.9595000000000001e-05 2023-09-23T20:01:52.886 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.886 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.886 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:01:52.886 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.113523+0000", 2023-09-23T20:01:52.886 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0682e-05 2023-09-23T20:01:52.886 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.886 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.886 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:01:52.887 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.113529+0000", 2023-09-23T20:01:52.887 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.993e-06 2023-09-23T20:01:52.887 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.887 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.887 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:01:52.887 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.114335+0000", 2023-09-23T20:01:52.887 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00080637600000000003 2023-09-23T20:01:52.887 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.887 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.887 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:01:52.887 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.114511+0000", 2023-09-23T20:01:52.887 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00017593 2023-09-23T20:01:52.888 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.888 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.888 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:01:52.888 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.114673+0000", 2023-09-23T20:01:52.888 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00016156300000000001 2023-09-23T20:01:52.888 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.888 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.888 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T20:01:52.888 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.122480+0000", 2023-09-23T20:01:52.888 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0078071590000000001 2023-09-23T20:01:52.888 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.888 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.889 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:52.889 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.123810+0000", 2023-09-23T20:01:52.889 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001330389 2023-09-23T20:01:52.889 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.889 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.889 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:52.889 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.399422+0000", 2023-09-23T20:01:52.889 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.27561206399999999 2023-09-23T20:01:52.889 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.889 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.889 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:01:52.889 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.400595+0000", 2023-09-23T20:01:52.890 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0011730919999999999 2023-09-23T20:01:52.890 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.890 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.890 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:01:52.890 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.400700+0000", 2023-09-23T20:01:52.890 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00010498200000000001 2023-09-23T20:01:52.890 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:01:52.890 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:01:52.890 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:01:52.890 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.891 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.891 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46877.0:40017 14.0 14:0e6089df:::benchmark_data_smithi016_228083_object40016:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-09-23T20:01:52.891 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:57:38.114110+0000", 2023-09-23T20:01:52.891 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 254.711680911, 2023-09-23T20:01:52.891 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.288656941, 2023-09-23T20:01:52.891 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:01:52.891 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:01:52.891 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:01:52.891 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46877", 2023-09-23T20:01:52.892 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:01:52.892 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 40017 2023-09-23T20:01:52.892 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.892 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:01:52.892 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.892 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:01:52.892 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.114110+0000", 2023-09-23T20:01:52.892 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:01:52.892 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.893 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.893 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:01:52.893 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.114110+0000", 2023-09-23T20:01:52.893 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:01:52.893 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.893 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.893 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:01:52.893 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.114110+0000", 2023-09-23T20:01:52.893 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T20:01:52.893 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.894 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.894 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:01:52.894 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.114128+0000", 2023-09-23T20:01:52.894 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.8070000000000001e-05 2023-09-23T20:01:52.894 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.894 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.894 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:01:52.894 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.114137+0000", 2023-09-23T20:01:52.894 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.8990000000000007e-06 2023-09-23T20:01:52.894 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.894 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.895 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:01:52.895 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.114142+0000", 2023-09-23T20:01:52.895 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.1699999999999996e-06 2023-09-23T20:01:52.895 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.895 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.895 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:01:52.895 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.114347+0000", 2023-09-23T20:01:52.895 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000205489 2023-09-23T20:01:52.895 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.895 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.895 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:01:52.895 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.114448+0000", 2023-09-23T20:01:52.896 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00010063700000000001 2023-09-23T20:01:52.896 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.896 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.896 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:01:52.896 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.114556+0000", 2023-09-23T20:01:52.896 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000108142 2023-09-23T20:01:52.896 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.896 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.896 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:52.897 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.118492+0000", 2023-09-23T20:01:52.897 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0039354079999999996 2023-09-23T20:01:52.897 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.897 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.897 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T20:01:52.897 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.119512+0000", 2023-09-23T20:01:52.897 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001020749 2023-09-23T20:01:52.897 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.897 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.898 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:52.898 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.402604+0000", 2023-09-23T20:01:52.898 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.28309172700000002 2023-09-23T20:01:52.898 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.898 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.898 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:01:52.898 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.402722+0000", 2023-09-23T20:01:52.898 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00011809199999999999 2023-09-23T20:01:52.898 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.899 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.899 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:01:52.899 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.402767+0000", 2023-09-23T20:01:52.899 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.507e-05 2023-09-23T20:01:52.899 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:01:52.899 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:01:52.899 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:01:52.899 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.899 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.899 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46877.0:40021 14.0 14:0fc25077:::benchmark_data_smithi016_228083_object40020:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-09-23T20:01:52.900 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:57:38.116613+0000", 2023-09-23T20:01:52.900 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 254.709178065, 2023-09-23T20:01:52.900 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.29051939300000001, 2023-09-23T20:01:52.900 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:01:52.900 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:01:52.900 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:01:52.900 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46877", 2023-09-23T20:01:52.900 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:01:52.900 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 40021 2023-09-23T20:01:52.900 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.901 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:01:52.901 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.901 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:01:52.901 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.116613+0000", 2023-09-23T20:01:52.901 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:01:52.901 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.901 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.901 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:01:52.902 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.116613+0000", 2023-09-23T20:01:52.902 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:01:52.902 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.902 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.902 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:01:52.902 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.116612+0000", 2023-09-23T20:01:52.903 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T20:01:52.903 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.903 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.903 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:01:52.903 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.116631+0000", 2023-09-23T20:01:52.903 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.8822e-05 2023-09-23T20:01:52.903 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.903 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.903 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:01:52.904 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.116641+0000", 2023-09-23T20:01:52.904 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.9669999999999996e-06 2023-09-23T20:01:52.904 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.904 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.904 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:01:52.904 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.116646+0000", 2023-09-23T20:01:52.904 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.2240000000000003e-06 2023-09-23T20:01:52.905 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.905 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.905 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:01:52.905 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.116679+0000", 2023-09-23T20:01:52.905 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.3370000000000001e-05 2023-09-23T20:01:52.905 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.905 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.905 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:01:52.905 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.116795+0000", 2023-09-23T20:01:52.905 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00011593499999999999 2023-09-23T20:01:52.905 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.906 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.906 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:01:52.906 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.116949+0000", 2023-09-23T20:01:52.906 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000153957 2023-09-23T20:01:52.906 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.906 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.906 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T20:01:52.906 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.119519+0000", 2023-09-23T20:01:52.906 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.002569235 2023-09-23T20:01:52.907 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.907 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.907 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:52.907 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.121727+0000", 2023-09-23T20:01:52.907 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.002207966 2023-09-23T20:01:52.907 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.907 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.907 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:52.907 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.406960+0000", 2023-09-23T20:01:52.907 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.28523343899999998 2023-09-23T20:01:52.907 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.908 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.908 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:01:52.908 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.407051+0000", 2023-09-23T20:01:52.908 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.1028999999999999e-05 2023-09-23T20:01:52.908 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.908 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.908 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:01:52.908 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.407132+0000", 2023-09-23T20:01:52.908 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.1416999999999997e-05 2023-09-23T20:01:52.909 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:01:52.909 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:01:52.909 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:01:52.909 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.909 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.909 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46877.0:40030 14.6 14:68474c9b:::benchmark_data_smithi016_228083_object40029:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-09-23T20:01:52.909 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:57:38.124278+0000", 2023-09-23T20:01:52.909 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 254.70151355600001, 2023-09-23T20:01:52.909 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.27846138500000001, 2023-09-23T20:01:52.909 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:01:52.909 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:01:52.910 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:01:52.910 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46877", 2023-09-23T20:01:52.910 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:01:52.910 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 40030 2023-09-23T20:01:52.910 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.910 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:01:52.910 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.910 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:01:52.910 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.124278+0000", 2023-09-23T20:01:52.911 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:01:52.911 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.911 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.911 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:01:52.911 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.124278+0000", 2023-09-23T20:01:52.911 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:01:52.911 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.911 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.911 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:01:52.911 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.124276+0000", 2023-09-23T20:01:52.912 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-09-23T20:01:52.912 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.912 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.912 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:01:52.912 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.124296+0000", 2023-09-23T20:01:52.912 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.9704e-05 2023-09-23T20:01:52.912 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.912 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.912 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:01:52.913 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.124307+0000", 2023-09-23T20:01:52.913 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1165000000000001e-05 2023-09-23T20:01:52.913 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.913 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.913 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:01:52.913 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.124313+0000", 2023-09-23T20:01:52.913 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.7520000000000002e-06 2023-09-23T20:01:52.913 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.913 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.914 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:01:52.914 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.124351+0000", 2023-09-23T20:01:52.914 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.8275000000000002e-05 2023-09-23T20:01:52.914 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.914 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.914 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:01:52.914 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.124463+0000", 2023-09-23T20:01:52.914 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000111705 2023-09-23T20:01:52.914 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.914 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.915 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:01:52.915 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.124596+0000", 2023-09-23T20:01:52.915 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00013313100000000001 2023-09-23T20:01:52.915 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.915 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.915 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:52.915 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.126647+0000", 2023-09-23T20:01:52.915 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0020512400000000002 2023-09-23T20:01:52.915 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.915 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.915 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T20:01:52.916 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.126872+0000", 2023-09-23T20:01:52.916 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00022435599999999999 2023-09-23T20:01:52.916 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.916 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.916 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:52.916 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.402609+0000", 2023-09-23T20:01:52.916 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.27573695199999998 2023-09-23T20:01:52.916 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.916 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.916 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:01:52.916 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.402689+0000", 2023-09-23T20:01:52.916 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.0005000000000004e-05 2023-09-23T20:01:52.917 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.917 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.917 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:01:52.917 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.402739+0000", 2023-09-23T20:01:52.917 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.0300999999999998e-05 2023-09-23T20:01:52.917 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:01:52.917 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:01:52.917 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:01:52.917 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.917 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.917 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46877.0:40033 14.0 14:032ca6e2:::benchmark_data_smithi016_228083_object40032:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-09-23T20:01:52.917 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:57:38.124324+0000", 2023-09-23T20:01:52.917 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 254.701466993, 2023-09-23T20:01:52.918 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.28303245999999999, 2023-09-23T20:01:52.918 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:01:52.918 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:01:52.918 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:01:52.918 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46877", 2023-09-23T20:01:52.918 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:01:52.918 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 40033 2023-09-23T20:01:52.918 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.918 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:01:52.918 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.918 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:01:52.919 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.124324+0000", 2023-09-23T20:01:52.919 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:01:52.919 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.919 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.919 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:01:52.919 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.124324+0000", 2023-09-23T20:01:52.919 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:01:52.919 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.919 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.919 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:01:52.919 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.124324+0000", 2023-09-23T20:01:52.919 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T20:01:52.919 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.920 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.920 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:01:52.920 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.124338+0000", 2023-09-23T20:01:52.920 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.4616e-05 2023-09-23T20:01:52.920 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.920 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.920 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:01:52.920 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.124343+0000", 2023-09-23T20:01:52.920 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.6190000000000002e-06 2023-09-23T20:01:52.920 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.920 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.920 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:01:52.920 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.124346+0000", 2023-09-23T20:01:52.921 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.1020000000000001e-06 2023-09-23T20:01:52.921 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.921 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.921 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:01:52.921 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.124394+0000", 2023-09-23T20:01:52.921 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.7948999999999998e-05 2023-09-23T20:01:52.921 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.921 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.921 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:01:52.921 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.124531+0000", 2023-09-23T20:01:52.921 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000136599 2023-09-23T20:01:52.921 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.922 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.922 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:01:52.922 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.124706+0000", 2023-09-23T20:01:52.922 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00017504599999999999 2023-09-23T20:01:52.922 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.922 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.922 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T20:01:52.922 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.126877+0000", 2023-09-23T20:01:52.922 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0021708460000000001 2023-09-23T20:01:52.923 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.923 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.923 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:52.923 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.127241+0000", 2023-09-23T20:01:52.923 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00036410499999999998 2023-09-23T20:01:52.923 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.923 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.923 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:52.923 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.407205+0000", 2023-09-23T20:01:52.924 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.27996414600000002 2023-09-23T20:01:52.924 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.924 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.924 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:01:52.924 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.407284+0000", 2023-09-23T20:01:52.924 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.9512999999999996e-05 2023-09-23T20:01:52.924 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.924 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.924 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:01:52.924 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.407357+0000", 2023-09-23T20:01:52.925 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.2268999999999995e-05 2023-09-23T20:01:52.925 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:01:52.925 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:01:52.925 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:01:52.925 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.925 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.925 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46877.0:40037 14.0 14:0778a3a7:::benchmark_data_smithi016_228083_object40036:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-09-23T20:01:52.925 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:57:38.127570+0000", 2023-09-23T20:01:52.925 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 254.69822088199999, 2023-09-23T20:01:52.926 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.28006694100000001, 2023-09-23T20:01:52.926 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:01:52.926 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:01:52.926 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:01:52.926 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46877", 2023-09-23T20:01:52.926 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:01:52.926 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 40037 2023-09-23T20:01:52.926 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.926 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:01:52.927 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.927 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:01:52.927 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.127570+0000", 2023-09-23T20:01:52.927 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:01:52.927 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.927 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.927 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:01:52.927 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.127570+0000", 2023-09-23T20:01:52.927 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:01:52.928 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.928 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.928 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:01:52.928 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.127569+0000", 2023-09-23T20:01:52.928 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T20:01:52.928 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.928 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.928 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:01:52.928 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.127588+0000", 2023-09-23T20:01:52.929 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.8281e-05 2023-09-23T20:01:52.929 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.929 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.929 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:01:52.929 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.127598+0000", 2023-09-23T20:01:52.929 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0287e-05 2023-09-23T20:01:52.929 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.929 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.929 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:01:52.930 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.127603+0000", 2023-09-23T20:01:52.930 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.2270000000000001e-06 2023-09-23T20:01:52.930 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.930 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.930 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:01:52.930 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.127648+0000", 2023-09-23T20:01:52.930 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.4895000000000003e-05 2023-09-23T20:01:52.930 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.930 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.931 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:01:52.931 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.127775+0000", 2023-09-23T20:01:52.931 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00012703500000000001 2023-09-23T20:01:52.931 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.931 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.931 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:01:52.931 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.127905+0000", 2023-09-23T20:01:52.931 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00013024799999999999 2023-09-23T20:01:52.932 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.932 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.932 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T20:01:52.932 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.130152+0000", 2023-09-23T20:01:52.932 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.002246287 2023-09-23T20:01:52.932 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.932 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.932 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:52.932 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.130634+0000", 2023-09-23T20:01:52.932 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00048192100000000001 2023-09-23T20:01:52.933 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.933 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.933 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:52.933 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.407450+0000", 2023-09-23T20:01:52.933 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.27681593300000001 2023-09-23T20:01:52.933 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.933 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.933 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:01:52.933 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.407569+0000", 2023-09-23T20:01:52.934 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000119296 2023-09-23T20:01:52.934 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.934 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.934 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:01:52.934 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.407637+0000", 2023-09-23T20:01:52.934 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.8400999999999998e-05 2023-09-23T20:01:52.935 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:01:52.935 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:01:52.935 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:01:52.935 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.935 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.935 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46877.0:40039 14.0 14:08986dac:::benchmark_data_smithi016_228083_object40038:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-09-23T20:01:52.935 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:57:38.127799+0000", 2023-09-23T20:01:52.935 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 254.69799186399999, 2023-09-23T20:01:52.935 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.28031159999999999, 2023-09-23T20:01:52.935 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:01:52.936 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:01:52.936 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:01:52.936 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46877", 2023-09-23T20:01:52.936 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:01:52.936 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 40039 2023-09-23T20:01:52.936 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.936 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:01:52.936 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.937 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:01:52.937 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.127799+0000", 2023-09-23T20:01:52.937 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:01:52.937 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.937 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.937 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:01:52.937 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.127799+0000", 2023-09-23T20:01:52.937 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:01:52.937 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.937 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.937 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:01:52.938 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.127798+0000", 2023-09-23T20:01:52.938 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T20:01:52.938 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.938 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.938 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:01:52.938 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.127819+0000", 2023-09-23T20:01:52.938 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.0835999999999998e-05 2023-09-23T20:01:52.938 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.938 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.939 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:01:52.939 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.127829+0000", 2023-09-23T20:01:52.939 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.6010000000000001e-06 2023-09-23T20:01:52.939 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.939 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.939 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:01:52.939 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.127836+0000", 2023-09-23T20:01:52.939 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.1720000000000001e-06 2023-09-23T20:01:52.939 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.939 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.940 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:01:52.940 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.128976+0000", 2023-09-23T20:01:52.940 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001139506 2023-09-23T20:01:52.940 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.940 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.940 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:01:52.940 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.129079+0000", 2023-09-23T20:01:52.940 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000103671 2023-09-23T20:01:52.940 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.941 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.941 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:01:52.941 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.129279+0000", 2023-09-23T20:01:52.941 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00019996599999999999 2023-09-23T20:01:52.941 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.941 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.941 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:52.941 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.130785+0000", 2023-09-23T20:01:52.941 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001505592 2023-09-23T20:01:52.941 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.941 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.942 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T20:01:52.942 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.131642+0000", 2023-09-23T20:01:52.942 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00085697099999999999 2023-09-23T20:01:52.942 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.942 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.942 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:52.942 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.408035+0000", 2023-09-23T20:01:52.942 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.27639299299999998 2023-09-23T20:01:52.942 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.943 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.943 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:01:52.943 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.408077+0000", 2023-09-23T20:01:52.943 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.2262000000000003e-05 2023-09-23T20:01:52.943 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.943 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.943 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:01:52.943 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.408111+0000", 2023-09-23T20:01:52.943 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.3852e-05 2023-09-23T20:01:52.943 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:01:52.944 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:01:52.944 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:01:52.944 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.944 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.944 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46877.0:40040 14.6 14:63d37f62:::benchmark_data_smithi016_228083_object40039:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-09-23T20:01:52.944 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:57:38.129143+0000", 2023-09-23T20:01:52.944 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 254.696647775, 2023-09-23T20:01:52.944 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.27450606799999999, 2023-09-23T20:01:52.944 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:01:52.944 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:01:52.944 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:01:52.944 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46877", 2023-09-23T20:01:52.945 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:01:52.945 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 40040 2023-09-23T20:01:52.945 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.945 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:01:52.945 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.945 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:01:52.945 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.129143+0000", 2023-09-23T20:01:52.945 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:01:52.945 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.945 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.945 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:01:52.945 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.129143+0000", 2023-09-23T20:01:52.946 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:01:52.946 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.946 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.946 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:01:52.946 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.129143+0000", 2023-09-23T20:01:52.946 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T20:01:52.946 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.946 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.946 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:01:52.947 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.129164+0000", 2023-09-23T20:01:52.947 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.1203999999999999e-05 2023-09-23T20:01:52.947 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.947 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.947 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:01:52.947 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.129172+0000", 2023-09-23T20:01:52.947 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.9899999999999997e-06 2023-09-23T20:01:52.947 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.947 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.948 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:01:52.948 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.129177+0000", 2023-09-23T20:01:52.948 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.0599999999999998e-06 2023-09-23T20:01:52.948 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.948 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.948 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:01:52.948 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.129220+0000", 2023-09-23T20:01:52.948 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.3312000000000001e-05 2023-09-23T20:01:52.949 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.949 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.949 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:01:52.949 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.129362+0000", 2023-09-23T20:01:52.949 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00014192299999999999 2023-09-23T20:01:52.949 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.949 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.949 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:01:52.949 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.129515+0000", 2023-09-23T20:01:52.949 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00015340800000000001 2023-09-23T20:01:52.949 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.949 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.950 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T20:01:52.950 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.131766+0000", 2023-09-23T20:01:52.950 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0022502799999999999 2023-09-23T20:01:52.950 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.950 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.950 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:52.950 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.132138+0000", 2023-09-23T20:01:52.950 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00037191600000000002 2023-09-23T20:01:52.950 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.950 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.950 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:52.950 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.403509+0000", 2023-09-23T20:01:52.951 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.27137124400000001 2023-09-23T20:01:52.951 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.951 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.951 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:01:52.951 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.403608+0000", 2023-09-23T20:01:52.951 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.8888000000000006e-05 2023-09-23T20:01:52.951 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.951 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.951 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:01:52.951 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.403649+0000", 2023-09-23T20:01:52.951 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.1666999999999999e-05 2023-09-23T20:01:52.951 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:01:52.952 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:01:52.952 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:01:52.952 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.952 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.952 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46877.0:40042 14.b 14:daff72e6:::benchmark_data_smithi016_228083_object40041:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-09-23T20:01:52.952 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:57:38.133693+0000", 2023-09-23T20:01:52.952 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 254.69209817699999, 2023-09-23T20:01:52.952 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.27004572799999998, 2023-09-23T20:01:52.952 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:01:52.953 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:01:52.953 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:01:52.953 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46877", 2023-09-23T20:01:52.953 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:01:52.953 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 40042 2023-09-23T20:01:52.953 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.953 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:01:52.953 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.953 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:01:52.953 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.133693+0000", 2023-09-23T20:01:52.954 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:01:52.954 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.954 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.954 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:01:52.954 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.133693+0000", 2023-09-23T20:01:52.954 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:01:52.954 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.954 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.955 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:01:52.955 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.133692+0000", 2023-09-23T20:01:52.955 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T20:01:52.955 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.955 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.955 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:01:52.955 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.133711+0000", 2023-09-23T20:01:52.955 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.8729e-05 2023-09-23T20:01:52.955 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.955 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.955 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:01:52.956 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.133721+0000", 2023-09-23T20:01:52.956 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0477e-05 2023-09-23T20:01:52.956 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.956 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.956 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:01:52.956 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.133727+0000", 2023-09-23T20:01:52.956 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.3870000000000003e-06 2023-09-23T20:01:52.956 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.956 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.956 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:01:52.957 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.133775+0000", 2023-09-23T20:01:52.957 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.8306999999999997e-05 2023-09-23T20:01:52.957 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.957 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.957 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:01:52.957 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.133885+0000", 2023-09-23T20:01:52.957 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000109664 2023-09-23T20:01:52.957 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.957 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.957 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:01:52.958 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.133994+0000", 2023-09-23T20:01:52.958 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000109657 2023-09-23T20:01:52.958 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.958 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.958 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T20:01:52.958 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.135628+0000", 2023-09-23T20:01:52.958 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001633448 2023-09-23T20:01:52.958 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.958 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.959 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:52.959 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.136476+0000", 2023-09-23T20:01:52.959 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00084808400000000003 2023-09-23T20:01:52.959 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.959 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.959 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:52.959 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.403529+0000", 2023-09-23T20:01:52.959 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.26705317699999997 2023-09-23T20:01:52.959 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.959 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.959 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:01:52.959 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.403660+0000", 2023-09-23T20:01:52.960 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00013137899999999999 2023-09-23T20:01:52.960 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.960 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.960 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:01:52.960 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.403739+0000", 2023-09-23T20:01:52.960 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.8256000000000004e-05 2023-09-23T20:01:52.960 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:01:52.960 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:01:52.960 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:01:52.960 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.960 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.961 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46877.0:95176 14.b 14:d73ec9cd:::benchmark_data_smithi016_228083_object95175:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e264)", 2023-09-23T20:01:52.961 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:58:38.226724+0000", 2023-09-23T20:01:52.961 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 194.599066706, 2023-09-23T20:01:52.961 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.24468284900000001, 2023-09-23T20:01:52.961 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:01:52.961 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:01:52.961 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:01:52.961 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46877", 2023-09-23T20:01:52.961 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:01:52.961 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 95176 2023-09-23T20:01:52.961 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.962 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:01:52.962 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.962 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:01:52.962 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:38.226724+0000", 2023-09-23T20:01:52.962 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:01:52.962 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.962 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.962 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:01:52.963 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:38.226724+0000", 2023-09-23T20:01:52.963 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:01:52.963 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.963 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.963 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:01:52.963 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:38.226724+0000", 2023-09-23T20:01:52.963 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T20:01:52.963 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.963 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.964 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:01:52.964 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:38.226744+0000", 2023-09-23T20:01:52.964 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.9792e-05 2023-09-23T20:01:52.964 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.964 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.964 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:01:52.964 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:38.226754+0000", 2023-09-23T20:01:52.964 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.8889999999999997e-06 2023-09-23T20:01:52.964 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.965 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.965 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:01:52.965 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:38.226758+0000", 2023-09-23T20:01:52.965 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.4909999999999999e-06 2023-09-23T20:01:52.965 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.965 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.965 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:01:52.965 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:38.226789+0000", 2023-09-23T20:01:52.965 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.0916e-05 2023-09-23T20:01:52.966 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.966 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.966 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:01:52.966 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:38.226891+0000", 2023-09-23T20:01:52.966 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000101889 2023-09-23T20:01:52.966 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.966 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.966 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:01:52.966 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:38.227068+0000", 2023-09-23T20:01:52.967 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000176634 2023-09-23T20:01:52.967 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.967 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.967 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T20:01:52.967 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:38.230180+0000", 2023-09-23T20:01:52.967 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0031123510000000002 2023-09-23T20:01:52.967 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.967 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.967 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:52.967 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:38.237464+0000", 2023-09-23T20:01:52.968 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.007284095 2023-09-23T20:01:52.968 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.968 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.968 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:52.968 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:38.471223+0000", 2023-09-23T20:01:52.968 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.23375892300000001 2023-09-23T20:01:52.968 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.968 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.968 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:01:52.968 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:38.471277+0000", 2023-09-23T20:01:52.968 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.4287999999999999e-05 2023-09-23T20:01:52.968 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.969 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.969 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:01:52.969 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:38.471407+0000", 2023-09-23T20:01:52.969 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000130106 2023-09-23T20:01:52.969 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:01:52.969 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:01:52.969 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:01:52.969 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.969 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.969 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46877.0:177658 14.b 14:db76f464:::benchmark_data_smithi016_228083_object77699:head [delete] snapc 0=[] ondisk+write+known_if_redirected e271)", 2023-09-23T20:01:52.970 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T20:00:41.475707+0000", 2023-09-23T20:01:52.970 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 71.350084636000005, 2023-09-23T20:01:52.970 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.21980476500000001, 2023-09-23T20:01:52.970 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:01:52.970 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:01:52.970 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:01:52.970 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46877", 2023-09-23T20:01:52.970 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:01:52.970 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 177658 2023-09-23T20:01:52.970 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.970 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:01:52.970 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.971 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:01:52.971 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.475707+0000", 2023-09-23T20:01:52.971 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:01:52.971 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.971 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.971 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:01:52.971 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.475707+0000", 2023-09-23T20:01:52.971 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:01:52.971 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.971 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.971 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:01:52.973 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.475705+0000", 2023-09-23T20:01:52.973 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-09-23T20:01:52.973 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.973 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.973 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:01:52.973 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.475711+0000", 2023-09-23T20:01:52.973 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.1550000000000004e-06 2023-09-23T20:01:52.973 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.974 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.974 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:01:52.974 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.475723+0000", 2023-09-23T20:01:52.974 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2157999999999999e-05 2023-09-23T20:01:52.974 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.974 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.974 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:01:52.974 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.475731+0000", 2023-09-23T20:01:52.974 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.8210000000000002e-06 2023-09-23T20:01:52.974 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.974 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.974 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:01:52.975 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.475787+0000", 2023-09-23T20:01:52.975 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.5390999999999999e-05 2023-09-23T20:01:52.975 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.975 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.975 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:01:52.975 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.476129+0000", 2023-09-23T20:01:52.975 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00034215 2023-09-23T20:01:52.975 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.975 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.975 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:01:52.975 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.476259+0000", 2023-09-23T20:01:52.975 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00013072800000000001 2023-09-23T20:01:52.976 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.976 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.976 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T20:01:52.976 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.487878+0000", 2023-09-23T20:01:52.976 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.011618696 2023-09-23T20:01:52.976 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.976 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.976 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:52.976 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.492295+0000", 2023-09-23T20:01:52.976 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0044171139999999998 2023-09-23T20:01:52.976 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.976 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.977 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:52.977 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.695238+0000", 2023-09-23T20:01:52.977 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.20294247600000001 2023-09-23T20:01:52.977 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.977 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.977 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:01:52.977 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.695372+0000", 2023-09-23T20:01:52.977 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000134356 2023-09-23T20:01:52.977 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.977 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.977 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:01:52.977 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.695511+0000", 2023-09-23T20:01:52.977 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000139185 2023-09-23T20:01:52.978 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:01:52.978 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:01:52.978 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:01:52.978 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.978 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.978 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46877.0:177660 14.b 14:dcbe9ec1:::benchmark_data_smithi016_228083_object77701:head [delete] snapc 0=[] ondisk+write+known_if_redirected e271)", 2023-09-23T20:01:52.978 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T20:00:41.480389+0000", 2023-09-23T20:01:52.978 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 71.345401953999996, 2023-09-23T20:01:52.978 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.21636045000000001, 2023-09-23T20:01:52.978 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:01:52.978 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:01:52.978 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:01:52.979 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46877", 2023-09-23T20:01:52.979 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:01:52.979 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 177660 2023-09-23T20:01:52.979 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.979 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:01:52.979 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.979 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:01:52.979 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.480389+0000", 2023-09-23T20:01:52.979 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:01:52.979 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.979 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.979 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:01:52.980 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.480389+0000", 2023-09-23T20:01:52.980 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:01:52.980 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.980 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.980 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:01:52.980 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.480388+0000", 2023-09-23T20:01:52.980 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-09-23T20:01:52.980 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.980 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.980 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:01:52.980 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.480394+0000", 2023-09-23T20:01:52.980 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.5300000000000002e-06 2023-09-23T20:01:52.981 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.981 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.981 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:01:52.981 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.480406+0000", 2023-09-23T20:01:52.981 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1858000000000001e-05 2023-09-23T20:01:52.981 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.981 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.981 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:01:52.981 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.480413+0000", 2023-09-23T20:01:52.981 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.2629999999999997e-06 2023-09-23T20:01:52.981 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.981 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.981 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:01:52.982 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.481645+0000", 2023-09-23T20:01:52.982 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0012310299999999999 2023-09-23T20:01:52.982 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.982 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.982 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:01:52.982 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.481761+0000", 2023-09-23T20:01:52.982 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00011616399999999999 2023-09-23T20:01:52.982 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.982 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.982 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:01:52.982 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.481874+0000", 2023-09-23T20:01:52.982 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000113335 2023-09-23T20:01:52.983 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.983 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.983 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T20:01:52.983 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.487887+0000", 2023-09-23T20:01:52.983 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0060124879999999999 2023-09-23T20:01:52.983 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.983 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.983 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:52.983 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.492327+0000", 2023-09-23T20:01:52.983 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0044407609999999997 2023-09-23T20:01:52.983 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.983 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.983 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:52.984 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.695899+0000", 2023-09-23T20:01:52.984 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.203571997 2023-09-23T20:01:52.984 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.984 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.984 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:01:52.984 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.696596+0000", 2023-09-23T20:01:52.984 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00069673800000000002 2023-09-23T20:01:52.984 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.984 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.984 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:01:52.984 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.696750+0000", 2023-09-23T20:01:52.984 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000153638 2023-09-23T20:01:52.985 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:01:52.985 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:01:52.985 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:01:52.985 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.985 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.985 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46877.0:177674 14.b 14:d1b71408:::benchmark_data_smithi016_228083_object77715:head [delete] snapc 0=[] ondisk+write+known_if_redirected e271)", 2023-09-23T20:01:52.985 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T20:00:41.494991+0000", 2023-09-23T20:01:52.985 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 71.330800303000004, 2023-09-23T20:01:52.985 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.21355988100000001, 2023-09-23T20:01:52.985 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:01:52.985 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:01:52.985 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:01:52.986 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46877", 2023-09-23T20:01:52.986 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:01:52.986 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 177674 2023-09-23T20:01:52.986 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.986 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:01:52.986 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.986 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:01:52.986 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.494991+0000", 2023-09-23T20:01:52.986 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:01:52.986 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.986 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.986 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:01:52.987 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.494991+0000", 2023-09-23T20:01:52.987 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:01:52.987 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.987 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.987 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:01:52.987 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.494989+0000", 2023-09-23T20:01:52.987 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-09-23T20:01:52.987 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.987 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.987 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:01:52.988 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.494997+0000", 2023-09-23T20:01:52.988 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.4309999999999997e-06 2023-09-23T20:01:52.988 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.988 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.988 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:01:52.988 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.495010+0000", 2023-09-23T20:01:52.988 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2931e-05 2023-09-23T20:01:52.988 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.988 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.989 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:01:52.989 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.495017+0000", 2023-09-23T20:01:52.989 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.8769999999999993e-06 2023-09-23T20:01:52.989 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.989 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.989 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:01:52.989 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.495067+0000", 2023-09-23T20:01:52.989 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.9508000000000001e-05 2023-09-23T20:01:52.989 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.989 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.989 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:01:52.990 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.495190+0000", 2023-09-23T20:01:52.990 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00012350600000000001 2023-09-23T20:01:52.990 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.990 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.990 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:01:52.990 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.495310+0000", 2023-09-23T20:01:52.990 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000119184 2023-09-23T20:01:52.990 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.991 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.991 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T20:01:52.991 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.520552+0000", 2023-09-23T20:01:52.991 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.025242446000000002 2023-09-23T20:01:52.991 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.991 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.991 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:52.991 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.696923+0000", 2023-09-23T20:01:52.991 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.17637043999999999 2023-09-23T20:01:52.991 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.991 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.992 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:52.992 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.708245+0000", 2023-09-23T20:01:52.992 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.011322446 2023-09-23T20:01:52.992 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.992 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.992 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:01:52.992 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.708389+0000", 2023-09-23T20:01:52.992 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000144196 2023-09-23T20:01:52.993 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.993 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.993 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:01:52.993 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.708551+0000", 2023-09-23T20:01:52.993 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000161514 2023-09-23T20:01:52.993 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:01:52.993 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:01:52.993 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:01:52.993 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.994 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.994 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46877.0:177676 14.b 14:d6acdb3e:::benchmark_data_smithi016_228083_object77717:head [delete] snapc 0=[] ondisk+write+known_if_redirected e271)", 2023-09-23T20:01:52.994 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T20:00:41.496202+0000", 2023-09-23T20:01:52.994 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 71.329589279999993, 2023-09-23T20:01:52.994 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.212828088, 2023-09-23T20:01:52.994 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:01:52.994 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:01:52.994 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:01:52.995 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46877", 2023-09-23T20:01:52.995 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:01:52.995 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 177676 2023-09-23T20:01:52.995 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.995 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:01:52.995 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.995 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:01:52.995 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.496202+0000", 2023-09-23T20:01:52.995 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:01:52.996 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.996 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.996 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:01:52.996 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.496202+0000", 2023-09-23T20:01:52.996 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:01:52.996 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.996 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.996 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:01:52.996 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.496201+0000", 2023-09-23T20:01:52.997 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T20:01:52.997 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.997 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.997 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:01:52.997 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.496207+0000", 2023-09-23T20:01:52.997 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.8409999999999996e-06 2023-09-23T20:01:52.997 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.997 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.997 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:01:52.998 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.496217+0000", 2023-09-23T20:01:52.998 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.8940000000000005e-06 2023-09-23T20:01:52.998 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.998 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.998 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:01:52.998 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.496224+0000", 2023-09-23T20:01:52.998 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.0119999999999999e-06 2023-09-23T20:01:52.998 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.999 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.999 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:01:52.999 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.504974+0000", 2023-09-23T20:01:52.999 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0087504910000000009 2023-09-23T20:01:52.999 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:52.999 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:52.999 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:01:52.999 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.505095+0000", 2023-09-23T20:01:52.999 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000121153 2023-09-23T20:01:53.000 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:53.000 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:53.000 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:01:53.000 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.505213+0000", 2023-09-23T20:01:53.000 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000117905 2023-09-23T20:01:53.000 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:53.000 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:53.000 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T20:01:53.000 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.521204+0000", 2023-09-23T20:01:53.000 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.015990652000000001 2023-09-23T20:01:53.000 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:53.001 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:53.001 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:53.001 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.697071+0000", 2023-09-23T20:01:53.001 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.17586681100000001 2023-09-23T20:01:53.001 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:53.001 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:53.001 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:53.001 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.708840+0000", 2023-09-23T20:01:53.001 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.011769595000000001 2023-09-23T20:01:53.001 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:53.001 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:53.001 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:01:53.001 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.708964+0000", 2023-09-23T20:01:53.002 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000123342 2023-09-23T20:01:53.002 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:01:53.002 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:01:53.002 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:01:53.002 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.709030+0000", 2023-09-23T20:01:53.002 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.6239000000000001e-05 2023-09-23T20:01:53.002 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:01:53.002 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:01:53.002 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:01:53.002 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:01:53.002 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:01:53.003 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-09-23T20:01:53.003 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:53.026 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:53.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:53.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:53.128 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:53.155 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:53.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:53.158 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:53.242 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-09-23T20:01:53.243 INFO:teuthology.orchestra.run.smithi016.stderr:dumped all 2023-09-23T20:01:53.259 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:53.293 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:53.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:53.296 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:53.327 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T20:01:53.348 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T20:01:53.367 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T20:01:53.387 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T20:01:53.396 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:53.408 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T20:01:53.428 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:53.430 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T20:01:53.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:53.433 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:53.452 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T20:01:53.477 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T20:01:53.478 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:01:53.478 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:01:53.478 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:01:53.478 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:01:53.479 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:01:53.479 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:01:53.479 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:01:53.479 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:01:53.534 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:53.552 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:53.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:53.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:53.655 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:53.672 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:53.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:53.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:53.776 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:53.801 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:53.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:53.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:53.904 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:53.923 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:53.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:53.925 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:54.025 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:54.059 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:54.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:54.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:54.163 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:54.165 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:54 smithi016 bash[75004]: cluster 2023-09-23T20:01:52.660860+0000 mgr.y (mgr.25325) 1639 : cluster 0 pgmap v1677: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:01:54.165 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:54 smithi016 bash[75004]: audit 2023-09-23T20:01:53.227329+0000 mgr.y (mgr.25325) 1640 : audit 0 from='client.37077 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:01:54.165 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:54 smithi016 bash[75004]: audit 2023-09-23T20:01:53.299360+0000 mon.a (mon.0) 1968 : audit 0 from='client.? 172.21.15.16:0/1400749131' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:01:54.165 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:54 smithi016 bash[75004]: audit 2023-09-23T20:01:53.834253+0000 mon.a (mon.0) 1969 : audit 1 from='client.? 172.21.15.16:0/3475065351' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:01:54.167 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is unset 2023-09-23T20:01:54.186 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:54.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:54.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:54.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:54 smithi016 bash[77995]: cluster 2023-09-23T20:01:52.660860+0000 mgr.y (mgr.25325) 1639 : cluster 0 pgmap v1677: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:01:54.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:54 smithi016 bash[77995]: audit 2023-09-23T20:01:53.227329+0000 mgr.y (mgr.25325) 1640 : audit 0 from='client.37077 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:01:54.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:54 smithi016 bash[77995]: audit 2023-09-23T20:01:53.299360+0000 mon.a (mon.0) 1968 : audit 0 from='client.? 172.21.15.16:0/1400749131' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:01:54.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:54 smithi016 bash[77995]: audit 2023-09-23T20:01:53.834253+0000 mon.a (mon.0) 1969 : audit 1 from='client.? 172.21.15.16:0/3475065351' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:01:54.290 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:54.318 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:54.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:54.320 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:54.421 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:54.435 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:54 smithi092 bash[58855]: cluster 2023-09-23T20:01:52.660860+0000 mgr.y (mgr.25325) 1639 : cluster 0 pgmap v1677: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:01:54.435 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:54 smithi092 bash[58855]: audit 2023-09-23T20:01:53.227329+0000 mgr.y (mgr.25325) 1640 : audit 0 from='client.37077 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:01:54.435 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:54 smithi092 bash[58855]: audit 2023-09-23T20:01:53.299360+0000 mon.a (mon.0) 1968 : audit 0 from='client.? 172.21.15.16:0/1400749131' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:01:54.436 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:54 smithi092 bash[58855]: audit 2023-09-23T20:01:53.834253+0000 mon.a (mon.0) 1969 : audit 1 from='client.? 172.21.15.16:0/3475065351' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:01:54.456 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:54.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:54.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:01:54.560 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:54.594 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:54.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:54.597 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:54.698 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:54.725 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:54.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:54.729 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:54.830 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:54.856 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:54.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:54.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:54.959 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:54.993 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:54.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:54.996 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:01:55.097 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:55.127 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:55.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:55.130 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:01:55.231 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:55.244 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:55 smithi092 bash[58855]: cluster 2023-09-23T20:01:54.149807+0000 mon.a (mon.0) 1970 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:01:55.244 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:55 smithi092 bash[58855]: cluster 2023-09-23T20:01:54.149849+0000 mon.a (mon.0) 1971 : cluster 1 Cluster is now healthy 2023-09-23T20:01:55.244 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:55 smithi092 bash[58855]: audit 2023-09-23T20:01:54.157601+0000 mon.a (mon.0) 1972 : audit 1 from='client.? 172.21.15.16:0/3475065351' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:01:55.244 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:55 smithi092 bash[58855]: cluster 2023-09-23T20:01:54.157683+0000 mon.a (mon.0) 1973 : cluster 0 osdmap e276: 8 total, 8 up, 8 in 2023-09-23T20:01:55.244 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:55 smithi092 bash[58855]: cluster 2023-09-23T20:01:54.304291+0000 osd.5 (osd.5) 197 : cluster 3 failed to encode map e276 with expected crc 2023-09-23T20:01:55.245 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:55 smithi092 bash[58855]: cluster 2023-09-23T20:01:54.304399+0000 osd.7 (osd.7) 191 : cluster 3 failed to encode map e276 with expected crc 2023-09-23T20:01:55.245 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:55 smithi092 bash[58855]: cluster 2023-09-23T20:01:54.304596+0000 osd.4 (osd.4) 215 : cluster 3 failed to encode map e276 with expected crc 2023-09-23T20:01:55.245 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:55 smithi092 bash[58855]: cluster 2023-09-23T20:01:54.304738+0000 osd.6 (osd.6) 208 : cluster 3 failed to encode map e276 with expected crc 2023-09-23T20:01:55.262 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:55.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:55.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:55.365 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:55.391 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:55.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:55.394 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:55.494 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:55.523 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:55.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:55.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:55.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:55 smithi016 bash[75004]: cluster 2023-09-23T20:01:54.149807+0000 mon.a (mon.0) 1970 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:01:55.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:55 smithi016 bash[75004]: cluster 2023-09-23T20:01:54.149849+0000 mon.a (mon.0) 1971 : cluster 1 Cluster is now healthy 2023-09-23T20:01:55.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:55 smithi016 bash[75004]: audit 2023-09-23T20:01:54.157601+0000 mon.a (mon.0) 1972 : audit 1 from='client.? 172.21.15.16:0/3475065351' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:01:55.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:55 smithi016 bash[75004]: cluster 2023-09-23T20:01:54.157683+0000 mon.a (mon.0) 1973 : cluster 0 osdmap e276: 8 total, 8 up, 8 in 2023-09-23T20:01:55.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:55 smithi016 bash[75004]: cluster 2023-09-23T20:01:54.304291+0000 osd.5 (osd.5) 197 : cluster 3 failed to encode map e276 with expected crc 2023-09-23T20:01:55.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:55 smithi016 bash[75004]: cluster 2023-09-23T20:01:54.304399+0000 osd.7 (osd.7) 191 : cluster 3 failed to encode map e276 with expected crc 2023-09-23T20:01:55.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:55 smithi016 bash[75004]: cluster 2023-09-23T20:01:54.304596+0000 osd.4 (osd.4) 215 : cluster 3 failed to encode map e276 with expected crc 2023-09-23T20:01:55.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:55 smithi016 bash[75004]: cluster 2023-09-23T20:01:54.304738+0000 osd.6 (osd.6) 208 : cluster 3 failed to encode map e276 with expected crc 2023-09-23T20:01:55.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:55 smithi016 bash[77995]: cluster 2023-09-23T20:01:54.149807+0000 mon.a (mon.0) 1970 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:01:55.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:55 smithi016 bash[77995]: cluster 2023-09-23T20:01:54.149849+0000 mon.a (mon.0) 1971 : cluster 1 Cluster is now healthy 2023-09-23T20:01:55.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:55 smithi016 bash[77995]: audit 2023-09-23T20:01:54.157601+0000 mon.a (mon.0) 1972 : audit 1 from='client.? 172.21.15.16:0/3475065351' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:01:55.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:55 smithi016 bash[77995]: cluster 2023-09-23T20:01:54.157683+0000 mon.a (mon.0) 1973 : cluster 0 osdmap e276: 8 total, 8 up, 8 in 2023-09-23T20:01:55.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:55 smithi016 bash[77995]: cluster 2023-09-23T20:01:54.304291+0000 osd.5 (osd.5) 197 : cluster 3 failed to encode map e276 with expected crc 2023-09-23T20:01:55.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:55 smithi016 bash[77995]: cluster 2023-09-23T20:01:54.304399+0000 osd.7 (osd.7) 191 : cluster 3 failed to encode map e276 with expected crc 2023-09-23T20:01:55.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:55 smithi016 bash[77995]: cluster 2023-09-23T20:01:54.304596+0000 osd.4 (osd.4) 215 : cluster 3 failed to encode map e276 with expected crc 2023-09-23T20:01:55.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:55 smithi016 bash[77995]: cluster 2023-09-23T20:01:54.304738+0000 osd.6 (osd.6) 208 : cluster 3 failed to encode map e276 with expected crc 2023-09-23T20:01:55.627 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:55.663 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:55.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:55.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:55.767 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:55.794 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:55.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:55.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:55.897 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:55.904 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:01:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:20:01:55] "GET /metrics HTTP/1.1" 200 33967 "" "Prometheus/2.33.4" 2023-09-23T20:01:55.923 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:55.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:55.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:56.026 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:56.058 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:56.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:56.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:56.162 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:56.183 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:56.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:56.186 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:56.287 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:56.314 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:56.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:56.317 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:56.418 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:56.426 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:56 smithi016 bash[75004]: cluster 2023-09-23T20:01:54.661628+0000 mgr.y (mgr.25325) 1641 : cluster 0 pgmap v1679: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:01:56.426 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:56 smithi016 bash[77995]: cluster 2023-09-23T20:01:54.661628+0000 mgr.y (mgr.25325) 1641 : cluster 0 pgmap v1679: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:01:56.443 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:56.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:56.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:56.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:56 smithi092 bash[58855]: cluster 2023-09-23T20:01:54.661628+0000 mgr.y (mgr.25325) 1641 : cluster 0 pgmap v1679: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:01:56.546 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:56.578 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:56.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:56.581 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:56.681 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:56.703 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:56.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:56.705 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:56.806 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:56.838 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:56.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:56.841 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:56.942 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:56.974 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:56.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:56.978 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:57.079 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:57.110 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:57.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:57.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:57.214 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:57.247 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:57.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:57.250 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:57.328 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T20:01:57.351 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:57.394 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:57.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:57.396 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:57.497 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:57.512 INFO:teuthology.orchestra.run.smithi016.stdout:{"pg_ready":true,"pg_map":{"version":1677,"stamp":"2023-09-23T20:01:52.660494+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":261,"num_write_kb":2446,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":195,"ondisk_log_size":195,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":7909844,"kb_used_data":23316,"kb_used_omap":20,"kb_used_meta":7886443,"kb_avail":742051372,"statfs":{"total":767960285184,"available":759860604928,"internally_reserved":0,"allocated":23875584,"data_stored":17405285,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":8075718286},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004152"},"pg_stats":[{"pgid":"3.1a","version":"59'1","reported_seq":271,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:30.007733+0000","last_change":"2023-09-23T19:38:43.897548+0000","last_active":"2023-09-23T20:01:30.007733+0000","last_peered":"2023-09-23T20:01:30.007733+0000","last_clean":"2023-09-23T20:01:30.007733+0000","last_became_active":"2023-09-23T19:38:43.896868+0000","last_became_peered":"2023-09-23T19:38:43.896868+0000","last_unstale":"2023-09-23T20:01:30.007733+0000","last_undegraded":"2023-09-23T20:01:30.007733+0000","last_fullsized":"2023-09-23T20:01:30.007733+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":239,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:29.991845+0000","last_change":"2023-09-23T19:32:49.731128+0000","last_active":"2023-09-23T20:01:29.991845+0000","last_peered":"2023-09-23T20:01:29.991845+0000","last_clean":"2023-09-23T20:01:29.991845+0000","last_became_active":"2023-09-23T19:32:49.730757+0000","last_became_peered":"2023-09-23T19:32:49.730757+0000","last_unstale":"2023-09-23T20:01:29.991845+0000","last_undegraded":"2023-09-23T20:01:29.991845+0000","last_fullsized":"2023-09-23T20:01:29.991845+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:40.183605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":228,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:31.164147+0000","last_change":"2023-09-23T19:28:07.890462+0000","last_active":"2023-09-23T20:01:31.164147+0000","last_peered":"2023-09-23T20:01:31.164147+0000","last_clean":"2023-09-23T20:01:31.164147+0000","last_became_active":"2023-09-23T19:28:07.890084+0000","last_became_peered":"2023-09-23T19:28:07.890084+0000","last_unstale":"2023-09-23T20:01:31.164147+0000","last_undegraded":"2023-09-23T20:01:31.164147+0000","last_fullsized":"2023-09-23T20:01:31.164147+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:50.458218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":161,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:30.005608+0000","last_change":"2023-09-23T19:42:55.772143+0000","last_active":"2023-09-23T20:01:30.005608+0000","last_peered":"2023-09-23T20:01:30.005608+0000","last_clean":"2023-09-23T20:01:30.005608+0000","last_became_active":"2023-09-23T19:42:55.771813+0000","last_became_peered":"2023-09-23T19:42:55.771813+0000","last_unstale":"2023-09-23T20:01:30.005608+0000","last_undegraded":"2023-09-23T20:01:30.005608+0000","last_fullsized":"2023-09-23T20:01:30.005608+0000","mapping_epoch":188,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":256,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:29.995808+0000","last_change":"2023-09-23T19:32:49.729961+0000","last_active":"2023-09-23T20:01:29.995808+0000","last_peered":"2023-09-23T20:01:29.995808+0000","last_clean":"2023-09-23T20:01:29.995808+0000","last_became_active":"2023-09-23T19:32:49.729251+0000","last_became_peered":"2023-09-23T19:32:49.729251+0000","last_unstale":"2023-09-23T20:01:29.995808+0000","last_undegraded":"2023-09-23T20:01:29.995808+0000","last_fullsized":"2023-09-23T20:01:29.995808+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:20:56.595732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":240,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:30.004460+0000","last_change":"2023-09-23T19:22:42.150582+0000","last_active":"2023-09-23T20:01:30.004460+0000","last_peered":"2023-09-23T20:01:30.004460+0000","last_clean":"2023-09-23T20:01:30.004460+0000","last_became_active":"2023-09-23T19:22:42.150229+0000","last_became_peered":"2023-09-23T19:22:42.150229+0000","last_unstale":"2023-09-23T20:01:30.004460+0000","last_undegraded":"2023-09-23T20:01:30.004460+0000","last_fullsized":"2023-09-23T20:01:30.004460+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":219,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:30.004099+0000","last_change":"2023-09-23T19:47:56.793944+0000","last_active":"2023-09-23T20:01:30.004099+0000","last_peered":"2023-09-23T20:01:30.004099+0000","last_clean":"2023-09-23T20:01:30.004099+0000","last_became_active":"2023-09-23T19:47:56.793618+0000","last_became_peered":"2023-09-23T19:47:56.793618+0000","last_unstale":"2023-09-23T20:01:30.004099+0000","last_undegraded":"2023-09-23T20:01:30.004099+0000","last_fullsized":"2023-09-23T20:01:30.004099+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":240,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:29.991738+0000","last_change":"2023-09-23T19:32:49.729765+0000","last_active":"2023-09-23T20:01:29.991738+0000","last_peered":"2023-09-23T20:01:29.991738+0000","last_clean":"2023-09-23T20:01:29.991738+0000","last_became_active":"2023-09-23T19:32:49.729007+0000","last_became_peered":"2023-09-23T19:32:49.729007+0000","last_unstale":"2023-09-23T20:01:29.991738+0000","last_undegraded":"2023-09-23T20:01:29.991738+0000","last_fullsized":"2023-09-23T20:01:29.991738+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:59:31.183753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":227,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:30.003669+0000","last_change":"2023-09-23T19:47:56.791464+0000","last_active":"2023-09-23T20:01:30.003669+0000","last_peered":"2023-09-23T20:01:30.003669+0000","last_clean":"2023-09-23T20:01:30.003669+0000","last_became_active":"2023-09-23T19:47:56.791144+0000","last_became_peered":"2023-09-23T19:47:56.791144+0000","last_unstale":"2023-09-23T20:01:30.003669+0000","last_undegraded":"2023-09-23T20:01:30.003669+0000","last_fullsized":"2023-09-23T20:01:30.003669+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":215,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:30.004686+0000","last_change":"2023-09-23T19:32:49.736864+0000","last_active":"2023-09-23T20:01:30.004686+0000","last_peered":"2023-09-23T20:01:30.004686+0000","last_clean":"2023-09-23T20:01:30.004686+0000","last_became_active":"2023-09-23T19:32:49.736523+0000","last_became_peered":"2023-09-23T19:32:49.736523+0000","last_unstale":"2023-09-23T20:01:30.004686+0000","last_undegraded":"2023-09-23T20:01:30.004686+0000","last_fullsized":"2023-09-23T20:01:30.004686+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":244,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:30.006017+0000","last_change":"2023-09-23T19:28:07.892835+0000","last_active":"2023-09-23T20:01:30.006017+0000","last_peered":"2023-09-23T20:01:30.006017+0000","last_clean":"2023-09-23T20:01:30.006017+0000","last_became_active":"2023-09-23T19:28:07.892338+0000","last_became_peered":"2023-09-23T19:28:07.892338+0000","last_unstale":"2023-09-23T20:01:30.006017+0000","last_undegraded":"2023-09-23T20:01:30.006017+0000","last_fullsized":"2023-09-23T20:01:30.006017+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":243,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:29.992574+0000","last_change":"2023-09-23T19:38:43.890137+0000","last_active":"2023-09-23T20:01:29.992574+0000","last_peered":"2023-09-23T20:01:29.992574+0000","last_clean":"2023-09-23T20:01:29.992574+0000","last_became_active":"2023-09-23T19:38:43.889800+0000","last_became_peered":"2023-09-23T19:38:43.889800+0000","last_unstale":"2023-09-23T20:01:29.992574+0000","last_undegraded":"2023-09-23T20:01:29.992574+0000","last_fullsized":"2023-09-23T20:01:29.992574+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:59:20.468913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'23","reported_seq":248,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:30.008042+0000","last_change":"2023-09-23T19:05:52.107727+0000","last_active":"2023-09-23T20:01:30.008042+0000","last_peered":"2023-09-23T20:01:30.008042+0000","last_clean":"2023-09-23T20:01:30.008042+0000","last_became_active":"2023-09-23T19:05:52.107378+0000","last_became_peered":"2023-09-23T19:05:52.107378+0000","last_unstale":"2023-09-23T20:01:30.008042+0000","last_undegraded":"2023-09-23T20:01:30.008042+0000","last_fullsized":"2023-09-23T20:01:30.008042+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":239,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:30.008184+0000","last_change":"2023-09-23T19:22:42.148497+0000","last_active":"2023-09-23T20:01:30.008184+0000","last_peered":"2023-09-23T20:01:30.008184+0000","last_clean":"2023-09-23T20:01:30.008184+0000","last_became_active":"2023-09-23T19:22:42.148147+0000","last_became_peered":"2023-09-23T19:22:42.148147+0000","last_unstale":"2023-09-23T20:01:30.008184+0000","last_undegraded":"2023-09-23T20:01:30.008184+0000","last_fullsized":"2023-09-23T20:01:30.008184+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":258,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:30.005302+0000","last_change":"2023-09-23T19:38:43.897378+0000","last_active":"2023-09-23T20:01:30.005302+0000","last_peered":"2023-09-23T20:01:30.005302+0000","last_clean":"2023-09-23T20:01:30.005302+0000","last_became_active":"2023-09-23T19:38:43.897012+0000","last_became_peered":"2023-09-23T19:38:43.897012+0000","last_unstale":"2023-09-23T20:01:30.005302+0000","last_undegraded":"2023-09-23T20:01:30.005302+0000","last_fullsized":"2023-09-23T20:01:30.005302+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":240,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:29.992813+0000","last_change":"2023-09-23T19:32:49.730549+0000","last_active":"2023-09-23T20:01:29.992813+0000","last_peered":"2023-09-23T20:01:29.992813+0000","last_clean":"2023-09-23T20:01:29.992813+0000","last_became_active":"2023-09-23T19:32:49.730105+0000","last_became_peered":"2023-09-23T19:32:49.730105+0000","last_unstale":"2023-09-23T20:01:29.992813+0000","last_undegraded":"2023-09-23T20:01:29.992813+0000","last_fullsized":"2023-09-23T20:01:29.992813+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:49.751893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":161,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:30.005006+0000","last_change":"2023-09-23T19:42:55.772691+0000","last_active":"2023-09-23T20:01:30.005006+0000","last_peered":"2023-09-23T20:01:30.005006+0000","last_clean":"2023-09-23T20:01:30.005006+0000","last_became_active":"2023-09-23T19:42:55.772309+0000","last_became_peered":"2023-09-23T19:42:55.772309+0000","last_unstale":"2023-09-23T20:01:30.005006+0000","last_undegraded":"2023-09-23T20:01:30.005006+0000","last_fullsized":"2023-09-23T20:01:30.005006+0000","mapping_epoch":188,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":182,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:30.006123+0000","last_change":"2023-09-23T19:58:26.094387+0000","last_active":"2023-09-23T20:01:30.006123+0000","last_peered":"2023-09-23T20:01:30.006123+0000","last_clean":"2023-09-23T20:01:30.006123+0000","last_became_active":"2023-09-23T19:58:26.093975+0000","last_became_peered":"2023-09-23T19:58:26.093975+0000","last_unstale":"2023-09-23T20:01:30.006123+0000","last_undegraded":"2023-09-23T20:01:30.006123+0000","last_fullsized":"2023-09-23T20:01:30.006123+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":226,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:30.004831+0000","last_change":"2023-09-23T19:05:52.114186+0000","last_active":"2023-09-23T20:01:30.004831+0000","last_peered":"2023-09-23T20:01:30.004831+0000","last_clean":"2023-09-23T20:01:30.004831+0000","last_became_active":"2023-09-23T19:05:52.113635+0000","last_became_peered":"2023-09-23T19:05:52.113635+0000","last_unstale":"2023-09-23T20:01:30.004831+0000","last_undegraded":"2023-09-23T20:01:30.004831+0000","last_fullsized":"2023-09-23T20:01:30.004831+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":161,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:30.005410+0000","last_change":"2023-09-23T19:42:55.772912+0000","last_active":"2023-09-23T20:01:30.005410+0000","last_peered":"2023-09-23T20:01:30.005410+0000","last_clean":"2023-09-23T20:01:30.005410+0000","last_became_active":"2023-09-23T19:42:55.772434+0000","last_became_peered":"2023-09-23T19:42:55.772434+0000","last_unstale":"2023-09-23T20:01:30.005410+0000","last_undegraded":"2023-09-23T20:01:30.005410+0000","last_fullsized":"2023-09-23T20:01:30.005410+0000","mapping_epoch":188,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":249,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:30.003912+0000","last_change":"2023-09-23T19:18:09.010195+0000","last_active":"2023-09-23T20:01:30.003912+0000","last_peered":"2023-09-23T20:01:30.003912+0000","last_clean":"2023-09-23T20:01:30.003912+0000","last_became_active":"2023-09-23T19:18:09.009687+0000","last_became_peered":"2023-09-23T19:18:09.009687+0000","last_unstale":"2023-09-23T20:01:30.003912+0000","last_undegraded":"2023-09-23T20:01:30.003912+0000","last_fullsized":"2023-09-23T20:01:30.003912+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":248,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:29.994663+0000","last_change":"2023-09-23T19:28:07.885265+0000","last_active":"2023-09-23T20:01:29.994663+0000","last_peered":"2023-09-23T20:01:29.994663+0000","last_clean":"2023-09-23T20:01:29.994663+0000","last_became_active":"2023-09-23T19:28:07.884353+0000","last_became_peered":"2023-09-23T19:28:07.884353+0000","last_unstale":"2023-09-23T20:01:29.994663+0000","last_undegraded":"2023-09-23T20:01:29.994663+0000","last_fullsized":"2023-09-23T20:01:29.994663+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:56.325352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":242,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:30.005863+0000","last_change":"2023-09-23T19:18:09.035116+0000","last_active":"2023-09-23T20:01:30.005863+0000","last_peered":"2023-09-23T20:01:30.005863+0000","last_clean":"2023-09-23T20:01:30.005863+0000","last_became_active":"2023-09-23T19:18:09.034416+0000","last_became_peered":"2023-09-23T19:18:09.034416+0000","last_unstale":"2023-09-23T20:01:30.005863+0000","last_undegraded":"2023-09-23T20:01:30.005863+0000","last_fullsized":"2023-09-23T20:01:30.005863+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":211,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:30.003053+0000","last_change":"2023-09-23T19:47:56.790227+0000","last_active":"2023-09-23T20:01:30.003053+0000","last_peered":"2023-09-23T20:01:30.003053+0000","last_clean":"2023-09-23T20:01:30.003053+0000","last_became_active":"2023-09-23T19:47:56.789727+0000","last_became_peered":"2023-09-23T19:47:56.789727+0000","last_unstale":"2023-09-23T20:01:30.003053+0000","last_undegraded":"2023-09-23T20:01:30.003053+0000","last_fullsized":"2023-09-23T20:01:30.003053+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":243,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:30.005515+0000","last_change":"2023-09-23T19:22:42.151134+0000","last_active":"2023-09-23T20:01:30.005515+0000","last_peered":"2023-09-23T20:01:30.005515+0000","last_clean":"2023-09-23T20:01:30.005515+0000","last_became_active":"2023-09-23T19:22:42.150553+0000","last_became_peered":"2023-09-23T19:22:42.150553+0000","last_unstale":"2023-09-23T20:01:30.005515+0000","last_undegraded":"2023-09-23T20:01:30.005515+0000","last_fullsized":"2023-09-23T20:01:30.005515+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":254,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:30.008299+0000","last_change":"2023-09-23T19:28:07.899262+0000","last_active":"2023-09-23T20:01:30.008299+0000","last_peered":"2023-09-23T20:01:30.008299+0000","last_clean":"2023-09-23T20:01:30.008299+0000","last_became_active":"2023-09-23T19:28:07.898848+0000","last_became_peered":"2023-09-23T19:28:07.898848+0000","last_unstale":"2023-09-23T20:01:30.008299+0000","last_undegraded":"2023-09-23T20:01:30.008299+0000","last_fullsized":"2023-09-23T20:01:30.008299+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":227,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:30.005847+0000","last_change":"2023-09-23T19:05:52.114108+0000","last_active":"2023-09-23T20:01:30.005847+0000","last_peered":"2023-09-23T20:01:30.005847+0000","last_clean":"2023-09-23T20:01:30.005847+0000","last_became_active":"2023-09-23T19:05:52.113609+0000","last_became_peered":"2023-09-23T19:05:52.113609+0000","last_unstale":"2023-09-23T20:01:30.005847+0000","last_undegraded":"2023-09-23T20:01:30.005847+0000","last_fullsized":"2023-09-23T20:01:30.005847+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":240,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:30.004653+0000","last_change":"2023-09-23T19:22:42.156825+0000","last_active":"2023-09-23T20:01:30.004653+0000","last_peered":"2023-09-23T20:01:30.004653+0000","last_clean":"2023-09-23T20:01:30.004653+0000","last_became_active":"2023-09-23T19:22:42.156474+0000","last_became_peered":"2023-09-23T19:22:42.156474+0000","last_unstale":"2023-09-23T20:01:30.004653+0000","last_undegraded":"2023-09-23T20:01:30.004653+0000","last_fullsized":"2023-09-23T20:01:30.004653+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":183,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:30.005006+0000","last_change":"2023-09-23T19:58:26.094114+0000","last_active":"2023-09-23T20:01:30.005006+0000","last_peered":"2023-09-23T20:01:30.005006+0000","last_clean":"2023-09-23T20:01:30.005006+0000","last_became_active":"2023-09-23T19:58:26.093615+0000","last_became_peered":"2023-09-23T19:58:26.093615+0000","last_unstale":"2023-09-23T20:01:30.005006+0000","last_undegraded":"2023-09-23T20:01:30.005006+0000","last_fullsized":"2023-09-23T20:01:30.005006+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":245,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:29.996326+0000","last_change":"2023-09-23T19:22:42.135572+0000","last_active":"2023-09-23T20:01:29.996326+0000","last_peered":"2023-09-23T20:01:29.996326+0000","last_clean":"2023-09-23T20:01:29.996326+0000","last_became_active":"2023-09-23T19:22:42.135237+0000","last_became_peered":"2023-09-23T19:22:42.135237+0000","last_unstale":"2023-09-23T20:01:29.996326+0000","last_undegraded":"2023-09-23T20:01:29.996326+0000","last_fullsized":"2023-09-23T20:01:29.996326+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:35.838718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":108,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:30.007964+0000","last_change":"2023-09-23T19:42:55.771907+0000","last_active":"2023-09-23T20:01:30.007964+0000","last_peered":"2023-09-23T20:01:30.007964+0000","last_clean":"2023-09-23T20:01:30.007964+0000","last_became_active":"2023-09-23T19:42:55.771503+0000","last_became_peered":"2023-09-23T19:42:55.771503+0000","last_unstale":"2023-09-23T20:01:30.007964+0000","last_undegraded":"2023-09-23T20:01:30.007964+0000","last_fullsized":"2023-09-23T20:01:30.007964+0000","mapping_epoch":188,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":239,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:30.008219+0000","last_change":"2023-09-23T19:28:07.896803+0000","last_active":"2023-09-23T20:01:30.008219+0000","last_peered":"2023-09-23T20:01:30.008219+0000","last_clean":"2023-09-23T20:01:30.008219+0000","last_became_active":"2023-09-23T19:28:07.896480+0000","last_became_peered":"2023-09-23T19:28:07.896480+0000","last_unstale":"2023-09-23T20:01:30.008219+0000","last_undegraded":"2023-09-23T20:01:30.008219+0000","last_fullsized":"2023-09-23T20:01:30.008219+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":241,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:30.004500+0000","last_change":"2023-09-23T19:22:42.156582+0000","last_active":"2023-09-23T20:01:30.004500+0000","last_peered":"2023-09-23T20:01:30.004500+0000","last_clean":"2023-09-23T20:01:30.004500+0000","last_became_active":"2023-09-23T19:22:42.156251+0000","last_became_peered":"2023-09-23T19:22:42.156251+0000","last_unstale":"2023-09-23T20:01:30.004500+0000","last_undegraded":"2023-09-23T20:01:30.004500+0000","last_fullsized":"2023-09-23T20:01:30.004500+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":269,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:30.008728+0000","last_change":"2023-09-23T19:38:43.891593+0000","last_active":"2023-09-23T20:01:30.008728+0000","last_peered":"2023-09-23T20:01:30.008728+0000","last_clean":"2023-09-23T20:01:30.008728+0000","last_became_active":"2023-09-23T19:38:43.891063+0000","last_became_peered":"2023-09-23T19:38:43.891063+0000","last_unstale":"2023-09-23T20:01:30.008728+0000","last_undegraded":"2023-09-23T20:01:30.008728+0000","last_fullsized":"2023-09-23T20:01:30.008728+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":255,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:30.007550+0000","last_change":"2023-09-23T19:38:43.892972+0000","last_active":"2023-09-23T20:01:30.007550+0000","last_peered":"2023-09-23T20:01:30.007550+0000","last_clean":"2023-09-23T20:01:30.007550+0000","last_became_active":"2023-09-23T19:38:43.892451+0000","last_became_peered":"2023-09-23T19:38:43.892451+0000","last_unstale":"2023-09-23T20:01:30.007550+0000","last_undegraded":"2023-09-23T20:01:30.007550+0000","last_fullsized":"2023-09-23T20:01:30.007550+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":239,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:30.008218+0000","last_change":"2023-09-23T19:22:42.149670+0000","last_active":"2023-09-23T20:01:30.008218+0000","last_peered":"2023-09-23T20:01:30.008218+0000","last_clean":"2023-09-23T20:01:30.008218+0000","last_became_active":"2023-09-23T19:22:42.149022+0000","last_became_peered":"2023-09-23T19:22:42.149022+0000","last_unstale":"2023-09-23T20:01:30.008218+0000","last_undegraded":"2023-09-23T20:01:30.008218+0000","last_fullsized":"2023-09-23T20:01:30.008218+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":242,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:29.991480+0000","last_change":"2023-09-23T19:32:49.729244+0000","last_active":"2023-09-23T20:01:29.991480+0000","last_peered":"2023-09-23T20:01:29.991480+0000","last_clean":"2023-09-23T20:01:29.991480+0000","last_became_active":"2023-09-23T19:32:49.728810+0000","last_became_peered":"2023-09-23T19:32:49.728810+0000","last_unstale":"2023-09-23T20:01:29.991480+0000","last_undegraded":"2023-09-23T20:01:29.991480+0000","last_fullsized":"2023-09-23T20:01:29.991480+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:57.540826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":229,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:29.993030+0000","last_change":"2023-09-23T19:28:07.889095+0000","last_active":"2023-09-23T20:01:29.993030+0000","last_peered":"2023-09-23T20:01:29.993030+0000","last_clean":"2023-09-23T20:01:29.993030+0000","last_became_active":"2023-09-23T19:28:07.888742+0000","last_became_peered":"2023-09-23T19:28:07.888742+0000","last_unstale":"2023-09-23T20:01:29.993030+0000","last_undegraded":"2023-09-23T20:01:29.993030+0000","last_fullsized":"2023-09-23T20:01:29.993030+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:43.929488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":263,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:29.992609+0000","last_change":"2023-09-23T19:22:42.143289+0000","last_active":"2023-09-23T20:01:29.992609+0000","last_peered":"2023-09-23T20:01:29.992609+0000","last_clean":"2023-09-23T20:01:29.992609+0000","last_became_active":"2023-09-23T19:22:42.142656+0000","last_became_peered":"2023-09-23T19:22:42.142656+0000","last_unstale":"2023-09-23T20:01:29.992609+0000","last_undegraded":"2023-09-23T20:01:29.992609+0000","last_fullsized":"2023-09-23T20:01:29.992609+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:38.467441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":257,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:30.004039+0000","last_change":"2023-09-23T19:47:56.794437+0000","last_active":"2023-09-23T20:01:30.004039+0000","last_peered":"2023-09-23T20:01:30.004039+0000","last_clean":"2023-09-23T20:01:30.004039+0000","last_became_active":"2023-09-23T19:47:56.794118+0000","last_became_peered":"2023-09-23T19:47:56.794118+0000","last_unstale":"2023-09-23T20:01:30.004039+0000","last_undegraded":"2023-09-23T20:01:30.004039+0000","last_fullsized":"2023-09-23T20:01:30.004039+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":249,"reported_epoch":275,"state":"active+clean","last_fresh":"2023-09-23T20:01:37.032598+0000","last_change":"2023-09-23T19:38:43.898519+0000","last_active":"2023-09-23T20:01:37.032598+0000","last_peered":"2023-09-23T20:01:37.032598+0000","last_clean":"2023-09-23T20:01:37.032598+0000","last_became_active":"2023-09-23T19:38:43.896814+0000","last_became_peered":"2023-09-23T19:38:43.896814+0000","last_unstale":"2023-09-23T20:01:37.032598+0000","last_undegraded":"2023-09-23T20:01:37.032598+0000","last_fullsized":"2023-09-23T20:01:37.032598+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:40.465038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":163,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:29.992075+0000","last_change":"2023-09-23T19:43:38.612700+0000","last_active":"2023-09-23T20:01:29.992075+0000","last_peered":"2023-09-23T20:01:29.992075+0000","last_clean":"2023-09-23T20:01:29.992075+0000","last_became_active":"2023-09-23T19:43:38.612424+0000","last_became_peered":"2023-09-23T19:43:38.612424+0000","last_unstale":"2023-09-23T20:01:29.992075+0000","last_undegraded":"2023-09-23T20:01:29.992075+0000","last_fullsized":"2023-09-23T20:01:29.992075+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:24.833490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":240,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:30.008043+0000","last_change":"2023-09-23T19:18:09.017525+0000","last_active":"2023-09-23T20:01:30.008043+0000","last_peered":"2023-09-23T20:01:30.008043+0000","last_clean":"2023-09-23T20:01:30.008043+0000","last_became_active":"2023-09-23T19:18:09.017193+0000","last_became_peered":"2023-09-23T19:18:09.017193+0000","last_unstale":"2023-09-23T20:01:30.008043+0000","last_undegraded":"2023-09-23T20:01:30.008043+0000","last_fullsized":"2023-09-23T20:01:30.008043+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":260,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:29.992503+0000","last_change":"2023-09-23T19:38:43.877088+0000","last_active":"2023-09-23T20:01:29.992503+0000","last_peered":"2023-09-23T20:01:29.992503+0000","last_clean":"2023-09-23T20:01:29.992503+0000","last_became_active":"2023-09-23T19:38:43.876719+0000","last_became_peered":"2023-09-23T19:38:43.876719+0000","last_unstale":"2023-09-23T20:01:29.992503+0000","last_undegraded":"2023-09-23T20:01:29.992503+0000","last_fullsized":"2023-09-23T20:01:29.992503+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:05.423256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":229,"reported_epoch":275,"state":"active+clean","last_fresh":"2023-09-23T20:01:37.033210+0000","last_change":"2023-09-23T19:38:43.883663+0000","last_active":"2023-09-23T20:01:37.033210+0000","last_peered":"2023-09-23T20:01:37.033210+0000","last_clean":"2023-09-23T20:01:37.033210+0000","last_became_active":"2023-09-23T19:38:43.882421+0000","last_became_peered":"2023-09-23T19:38:43.882421+0000","last_unstale":"2023-09-23T20:01:37.033210+0000","last_undegraded":"2023-09-23T20:01:37.033210+0000","last_fullsized":"2023-09-23T20:01:37.033210+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:49:01.607659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":246,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:29.991372+0000","last_change":"2023-09-23T19:38:43.890600+0000","last_active":"2023-09-23T20:01:29.991372+0000","last_peered":"2023-09-23T20:01:29.991372+0000","last_clean":"2023-09-23T20:01:29.991372+0000","last_became_active":"2023-09-23T19:38:43.890253+0000","last_became_peered":"2023-09-23T19:38:43.890253+0000","last_unstale":"2023-09-23T20:01:29.991372+0000","last_undegraded":"2023-09-23T20:01:29.991372+0000","last_fullsized":"2023-09-23T20:01:29.991372+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:46.751120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":258,"reported_epoch":275,"state":"active+clean","last_fresh":"2023-09-23T20:01:37.607863+0000","last_change":"2023-09-23T19:22:42.143733+0000","last_active":"2023-09-23T20:01:37.607863+0000","last_peered":"2023-09-23T20:01:37.607863+0000","last_clean":"2023-09-23T20:01:37.607863+0000","last_became_active":"2023-09-23T19:22:42.143077+0000","last_became_peered":"2023-09-23T19:22:42.143077+0000","last_unstale":"2023-09-23T20:01:37.607863+0000","last_undegraded":"2023-09-23T20:01:37.607863+0000","last_fullsized":"2023-09-23T20:01:37.607863+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:47:33.086994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":246,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:30.003870+0000","last_change":"2023-09-23T19:47:56.791259+0000","last_active":"2023-09-23T20:01:30.003870+0000","last_peered":"2023-09-23T20:01:30.003870+0000","last_clean":"2023-09-23T20:01:30.003870+0000","last_became_active":"2023-09-23T19:47:56.790954+0000","last_became_peered":"2023-09-23T19:47:56.790954+0000","last_unstale":"2023-09-23T20:01:30.003870+0000","last_undegraded":"2023-09-23T20:01:30.003870+0000","last_fullsized":"2023-09-23T20:01:30.003870+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":477,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:30.004677+0000","last_change":"2023-09-23T19:18:09.016297+0000","last_active":"2023-09-23T20:01:30.004677+0000","last_peered":"2023-09-23T20:01:30.004677+0000","last_clean":"2023-09-23T20:01:30.004677+0000","last_became_active":"2023-09-23T19:18:09.015436+0000","last_became_peered":"2023-09-23T19:18:09.015436+0000","last_unstale":"2023-09-23T20:01:30.004677+0000","last_undegraded":"2023-09-23T20:01:30.004677+0000","last_fullsized":"2023-09-23T20:01:30.004677+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:59:33.840751+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:59:33.840751+0000","last_clean_scrub_stamp":"2023-09-23T18:59:33.840751+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":200,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:30.003521+0000","last_change":"2023-09-23T19:47:56.811861+0000","last_active":"2023-09-23T20:01:30.003521+0000","last_peered":"2023-09-23T20:01:30.003521+0000","last_clean":"2023-09-23T20:01:30.003521+0000","last_became_active":"2023-09-23T19:47:56.811527+0000","last_became_peered":"2023-09-23T19:47:56.811527+0000","last_unstale":"2023-09-23T20:01:30.003521+0000","last_undegraded":"2023-09-23T20:01:30.003521+0000","last_fullsized":"2023-09-23T20:01:30.003521+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":225,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:30.005441+0000","last_change":"2023-09-23T19:05:52.786771+0000","last_active":"2023-09-23T20:01:30.005441+0000","last_peered":"2023-09-23T20:01:30.005441+0000","last_clean":"2023-09-23T20:01:30.005441+0000","last_became_active":"2023-09-23T19:05:52.785797+0000","last_became_peered":"2023-09-23T19:05:52.785797+0000","last_unstale":"2023-09-23T20:01:30.005441+0000","last_undegraded":"2023-09-23T20:01:30.005441+0000","last_fullsized":"2023-09-23T20:01:30.005441+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":260,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:30.004596+0000","last_change":"2023-09-23T19:22:42.150636+0000","last_active":"2023-09-23T20:01:30.004596+0000","last_peered":"2023-09-23T20:01:30.004596+0000","last_clean":"2023-09-23T20:01:30.004596+0000","last_became_active":"2023-09-23T19:22:42.150045+0000","last_became_peered":"2023-09-23T19:22:42.150045+0000","last_unstale":"2023-09-23T20:01:30.004596+0000","last_undegraded":"2023-09-23T20:01:30.004596+0000","last_fullsized":"2023-09-23T20:01:30.004596+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":161,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:30.008148+0000","last_change":"2023-09-23T19:42:55.771906+0000","last_active":"2023-09-23T20:01:30.008148+0000","last_peered":"2023-09-23T20:01:30.008148+0000","last_clean":"2023-09-23T20:01:30.008148+0000","last_became_active":"2023-09-23T19:42:55.771599+0000","last_became_peered":"2023-09-23T19:42:55.771599+0000","last_unstale":"2023-09-23T20:01:30.008148+0000","last_undegraded":"2023-09-23T20:01:30.008148+0000","last_fullsized":"2023-09-23T20:01:30.008148+0000","mapping_epoch":188,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":276,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:29.996782+0000","last_change":"2023-09-23T19:38:43.876930+0000","last_active":"2023-09-23T20:01:29.996782+0000","last_peered":"2023-09-23T20:01:29.996782+0000","last_clean":"2023-09-23T20:01:29.996782+0000","last_became_active":"2023-09-23T19:38:43.876571+0000","last_became_peered":"2023-09-23T19:38:43.876571+0000","last_unstale":"2023-09-23T20:01:29.996782+0000","last_undegraded":"2023-09-23T20:01:29.996782+0000","last_fullsized":"2023-09-23T20:01:29.996782+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:30.924536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":245,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:30.005945+0000","last_change":"2023-09-23T19:18:09.015853+0000","last_active":"2023-09-23T20:01:30.005945+0000","last_peered":"2023-09-23T20:01:30.005945+0000","last_clean":"2023-09-23T20:01:30.005945+0000","last_became_active":"2023-09-23T19:18:09.015338+0000","last_became_peered":"2023-09-23T19:18:09.015338+0000","last_unstale":"2023-09-23T20:01:30.005945+0000","last_undegraded":"2023-09-23T20:01:30.005945+0000","last_fullsized":"2023-09-23T20:01:30.005945+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":245,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:29.992138+0000","last_change":"2023-09-23T19:38:43.881233+0000","last_active":"2023-09-23T20:01:29.992138+0000","last_peered":"2023-09-23T20:01:29.992138+0000","last_clean":"2023-09-23T20:01:29.992138+0000","last_became_active":"2023-09-23T19:38:43.880893+0000","last_became_peered":"2023-09-23T19:38:43.880893+0000","last_unstale":"2023-09-23T20:01:29.992138+0000","last_undegraded":"2023-09-23T20:01:29.992138+0000","last_fullsized":"2023-09-23T20:01:29.992138+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:11.206947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":96,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:30.004018+0000","last_change":"2023-09-23T19:47:56.790340+0000","last_active":"2023-09-23T20:01:30.004018+0000","last_peered":"2023-09-23T20:01:30.004018+0000","last_clean":"2023-09-23T20:01:30.004018+0000","last_became_active":"2023-09-23T19:47:56.789785+0000","last_became_peered":"2023-09-23T19:47:56.789785+0000","last_unstale":"2023-09-23T20:01:30.004018+0000","last_undegraded":"2023-09-23T20:01:30.004018+0000","last_fullsized":"2023-09-23T20:01:30.004018+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":179,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:30.005662+0000","last_change":"2023-09-23T19:32:49.737323+0000","last_active":"2023-09-23T20:01:30.005662+0000","last_peered":"2023-09-23T20:01:30.005662+0000","last_clean":"2023-09-23T20:01:30.005662+0000","last_became_active":"2023-09-23T19:32:49.736937+0000","last_became_peered":"2023-09-23T19:32:49.736937+0000","last_unstale":"2023-09-23T20:01:30.005662+0000","last_undegraded":"2023-09-23T20:01:30.005662+0000","last_fullsized":"2023-09-23T20:01:30.005662+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":215,"reported_epoch":275,"state":"active+clean","last_fresh":"2023-09-23T20:01:37.607235+0000","last_change":"2023-09-23T19:22:42.145506+0000","last_active":"2023-09-23T20:01:37.607235+0000","last_peered":"2023-09-23T20:01:37.607235+0000","last_clean":"2023-09-23T20:01:37.607235+0000","last_became_active":"2023-09-23T19:22:42.143991+0000","last_became_peered":"2023-09-23T19:22:42.143991+0000","last_unstale":"2023-09-23T20:01:37.607235+0000","last_undegraded":"2023-09-23T20:01:37.607235+0000","last_fullsized":"2023-09-23T20:01:37.607235+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:56.875175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":216,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:30.004802+0000","last_change":"2023-09-23T19:32:49.734592+0000","last_active":"2023-09-23T20:01:30.004802+0000","last_peered":"2023-09-23T20:01:30.004802+0000","last_clean":"2023-09-23T20:01:30.004802+0000","last_became_active":"2023-09-23T19:32:49.733905+0000","last_became_peered":"2023-09-23T19:32:49.733905+0000","last_unstale":"2023-09-23T20:01:30.004802+0000","last_undegraded":"2023-09-23T20:01:30.004802+0000","last_fullsized":"2023-09-23T20:01:30.004802+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":260,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:30.004236+0000","last_change":"2023-09-23T19:38:43.895887+0000","last_active":"2023-09-23T20:01:30.004236+0000","last_peered":"2023-09-23T20:01:30.004236+0000","last_clean":"2023-09-23T20:01:30.004236+0000","last_became_active":"2023-09-23T19:38:43.895486+0000","last_became_peered":"2023-09-23T19:38:43.895486+0000","last_unstale":"2023-09-23T20:01:30.004236+0000","last_undegraded":"2023-09-23T20:01:30.004236+0000","last_fullsized":"2023-09-23T20:01:30.004236+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":249,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:29.996655+0000","last_change":"2023-09-23T19:22:42.149694+0000","last_active":"2023-09-23T20:01:29.996655+0000","last_peered":"2023-09-23T20:01:29.996655+0000","last_clean":"2023-09-23T20:01:29.996655+0000","last_became_active":"2023-09-23T19:22:42.148194+0000","last_became_peered":"2023-09-23T19:22:42.148194+0000","last_unstale":"2023-09-23T20:01:29.996655+0000","last_undegraded":"2023-09-23T20:01:29.996655+0000","last_fullsized":"2023-09-23T20:01:29.996655+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:47.842383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":224,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:30.007640+0000","last_change":"2023-09-23T19:05:52.138620+0000","last_active":"2023-09-23T20:01:30.007640+0000","last_peered":"2023-09-23T20:01:30.007640+0000","last_clean":"2023-09-23T20:01:30.007640+0000","last_became_active":"2023-09-23T19:05:52.138243+0000","last_became_peered":"2023-09-23T19:05:52.138243+0000","last_unstale":"2023-09-23T20:01:30.007640+0000","last_undegraded":"2023-09-23T20:01:30.007640+0000","last_fullsized":"2023-09-23T20:01:30.007640+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":231,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:29.996512+0000","last_change":"2023-09-23T19:18:09.008811+0000","last_active":"2023-09-23T20:01:29.996512+0000","last_peered":"2023-09-23T20:01:29.996512+0000","last_clean":"2023-09-23T20:01:29.996512+0000","last_became_active":"2023-09-23T19:18:09.007255+0000","last_became_peered":"2023-09-23T19:18:09.007255+0000","last_unstale":"2023-09-23T20:01:29.996512+0000","last_undegraded":"2023-09-23T20:01:29.996512+0000","last_fullsized":"2023-09-23T20:01:29.996512+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:25.874038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":235,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:29.997956+0000","last_change":"2023-09-23T19:38:43.877244+0000","last_active":"2023-09-23T20:01:29.997956+0000","last_peered":"2023-09-23T20:01:29.997956+0000","last_clean":"2023-09-23T20:01:29.997956+0000","last_became_active":"2023-09-23T19:38:43.876896+0000","last_became_peered":"2023-09-23T19:38:43.876896+0000","last_unstale":"2023-09-23T20:01:29.997956+0000","last_undegraded":"2023-09-23T20:01:29.997956+0000","last_fullsized":"2023-09-23T20:01:29.997956+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:22.288276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":104,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:30.007794+0000","last_change":"2023-09-23T19:43:38.620732+0000","last_active":"2023-09-23T20:01:30.007794+0000","last_peered":"2023-09-23T20:01:30.007794+0000","last_clean":"2023-09-23T20:01:30.007794+0000","last_became_active":"2023-09-23T19:43:38.619424+0000","last_became_peered":"2023-09-23T19:43:38.619424+0000","last_unstale":"2023-09-23T20:01:30.007794+0000","last_undegraded":"2023-09-23T20:01:30.007794+0000","last_fullsized":"2023-09-23T20:01:30.007794+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":278,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:30.008260+0000","last_change":"2023-09-23T19:38:43.947451+0000","last_active":"2023-09-23T20:01:30.008260+0000","last_peered":"2023-09-23T20:01:30.008260+0000","last_clean":"2023-09-23T20:01:30.008260+0000","last_became_active":"2023-09-23T19:38:43.946420+0000","last_became_peered":"2023-09-23T19:38:43.946420+0000","last_unstale":"2023-09-23T20:01:30.008260+0000","last_undegraded":"2023-09-23T20:01:30.008260+0000","last_fullsized":"2023-09-23T20:01:30.008260+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":66,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:30.008832+0000","last_change":"2023-09-23T19:58:26.092680+0000","last_active":"2023-09-23T20:01:30.008832+0000","last_peered":"2023-09-23T20:01:30.008832+0000","last_clean":"2023-09-23T20:01:30.008832+0000","last_became_active":"2023-09-23T19:58:26.092010+0000","last_became_peered":"2023-09-23T19:58:26.092010+0000","last_unstale":"2023-09-23T20:01:30.008832+0000","last_undegraded":"2023-09-23T20:01:30.008832+0000","last_fullsized":"2023-09-23T20:01:30.008832+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":254,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:30.007912+0000","last_change":"2023-09-23T19:28:07.896983+0000","last_active":"2023-09-23T20:01:30.007912+0000","last_peered":"2023-09-23T20:01:30.007912+0000","last_clean":"2023-09-23T20:01:30.007912+0000","last_became_active":"2023-09-23T19:28:07.896577+0000","last_became_peered":"2023-09-23T19:28:07.896577+0000","last_unstale":"2023-09-23T20:01:30.007912+0000","last_undegraded":"2023-09-23T20:01:30.007912+0000","last_fullsized":"2023-09-23T20:01:30.007912+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":184,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:30.004898+0000","last_change":"2023-09-23T19:38:43.900225+0000","last_active":"2023-09-23T20:01:30.004898+0000","last_peered":"2023-09-23T20:01:30.004898+0000","last_clean":"2023-09-23T20:01:30.004898+0000","last_became_active":"2023-09-23T19:38:43.899907+0000","last_became_peered":"2023-09-23T19:38:43.899907+0000","last_unstale":"2023-09-23T20:01:30.004898+0000","last_undegraded":"2023-09-23T20:01:30.004898+0000","last_fullsized":"2023-09-23T20:01:30.004898+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":244,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:29.997143+0000","last_change":"2023-09-23T19:38:43.882487+0000","last_active":"2023-09-23T20:01:29.997143+0000","last_peered":"2023-09-23T20:01:29.997143+0000","last_clean":"2023-09-23T20:01:29.997143+0000","last_became_active":"2023-09-23T19:38:43.881931+0000","last_became_peered":"2023-09-23T19:38:43.881931+0000","last_unstale":"2023-09-23T20:01:29.997143+0000","last_undegraded":"2023-09-23T20:01:29.997143+0000","last_fullsized":"2023-09-23T20:01:29.997143+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:02:36.906535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":264,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:29.996326+0000","last_change":"2023-09-23T19:38:43.893983+0000","last_active":"2023-09-23T20:01:29.996326+0000","last_peered":"2023-09-23T20:01:29.996326+0000","last_clean":"2023-09-23T20:01:29.996326+0000","last_became_active":"2023-09-23T19:38:43.893527+0000","last_became_peered":"2023-09-23T19:38:43.893527+0000","last_unstale":"2023-09-23T20:01:29.996326+0000","last_undegraded":"2023-09-23T20:01:29.996326+0000","last_fullsized":"2023-09-23T20:01:29.996326+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:12.156531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":233,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T20:01:30.007528+0000","last_change":"2023-09-23T19:06:36.673002+0000","last_active":"2023-09-23T20:01:30.007528+0000","last_peered":"2023-09-23T20:01:30.007528+0000","last_clean":"2023-09-23T20:01:30.007528+0000","last_became_active":"2023-09-23T19:06:36.671911+0000","last_became_peered":"2023-09-23T19:06:36.671911+0000","last_unstale":"2023-09-23T20:01:30.007528+0000","last_undegraded":"2023-09-23T20:01:30.007528+0000","last_fullsized":"2023-09-23T20:01:30.007528+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430916,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1174976,"kb_used_data":3000,"kb_used_omap":0,"kb_used_meta":1171968,"kb_avail":92570176,"statfs":{"total":95995035648,"available":94791860224,"internally_reserved":0,"allocated":3072000,"data_stored":2515491,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1200095232},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:01:26 2023","interfaces":[{"interface":"back","average":{"1min":2.3679999999999999,"5min":1.95,"15min":1.526},"min":{"1min":0.28399999999999997,"5min":0.27400000000000002,"15min":0.26700000000000002},"max":{"1min":10.33,"5min":41.570999999999998,"15min":41.570999999999998},"last":0.755},{"interface":"front","average":{"1min":2.649,"5min":2.234,"15min":1.6799999999999999},"min":{"1min":0.34000000000000002,"5min":0.29699999999999999,"15min":0.28000000000000003},"max":{"1min":7.0960000000000001,"5min":45.634,"15min":45.634},"last":0.91500000000000004}]},{"osd":1,"last update":"Sat Sep 23 20:01:43 2023","interfaces":[{"interface":"back","average":{"1min":1.802,"5min":2.3999999999999999,"15min":2.339},"min":{"1min":0.32600000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":7.7210000000000001,"5min":43.351999999999997,"15min":149.018},"last":1.1950000000000001},{"interface":"front","average":{"1min":1.49,"5min":2.3620000000000001,"15min":2.2890000000000001},"min":{"1min":0.35099999999999998,"5min":0.32100000000000001,"15min":0.254},"max":{"1min":6.1749999999999998,"5min":43.381999999999998,"15min":148.995},"last":0.93899999999999995}]},{"osd":2,"last update":"Sat Sep 23 20:01:26 2023","interfaces":[{"interface":"back","average":{"1min":11.356999999999999,"5min":4.1689999999999996,"15min":4.1559999999999997},"min":{"1min":0.33900000000000002,"5min":0.32300000000000001,"15min":0.308},"max":{"1min":160.655,"5min":160.655,"15min":246.982},"last":0.84799999999999998},{"interface":"front","average":{"1min":12.577999999999999,"5min":4.6580000000000004,"15min":4.3250000000000002},"min":{"1min":0.34000000000000002,"5min":0.30299999999999999,"15min":0.30199999999999999},"max":{"1min":176.428,"5min":176.428,"15min":247.751},"last":0.81899999999999995}]},{"osd":3,"last update":"Sat Sep 23 20:01:16 2023","interfaces":[{"interface":"back","average":{"1min":1.323,"5min":1.2869999999999999,"15min":1.597},"min":{"1min":0.38800000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":7.7590000000000003,"5min":8.4979999999999993,"15min":10.615},"last":1.2549999999999999},{"interface":"front","average":{"1min":1.8260000000000001,"5min":1.423,"15min":1.6040000000000001},"min":{"1min":0.38900000000000001,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":7.298,"5min":8.5220000000000002,"15min":9.9179999999999993},"last":1.0129999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:01:43 2023","interfaces":[{"interface":"back","average":{"1min":1.143,"5min":1.1579999999999999,"15min":1.5529999999999999},"min":{"1min":0.34599999999999997,"5min":0.32500000000000001,"15min":0.27700000000000002},"max":{"1min":3.3889999999999998,"5min":6.9169999999999998,"15min":13.484999999999999},"last":1.1419999999999999},{"interface":"front","average":{"1min":1.2669999999999999,"5min":1.0580000000000001,"15min":1.4410000000000001},"min":{"1min":0.41399999999999998,"5min":0.313,"15min":0.313},"max":{"1min":5.6619999999999999,"5min":7.2030000000000003,"15min":13.116},"last":1.093}]},{"osd":5,"last update":"Sat Sep 23 20:01:43 2023","interfaces":[{"interface":"back","average":{"1min":1.397,"5min":1.1830000000000001,"15min":1.4450000000000001},"min":{"1min":0.378,"5min":0.34799999999999998,"15min":0.28499999999999998},"max":{"1min":4.298,"5min":8.1349999999999998,"15min":13.321999999999999},"last":1.054},{"interface":"front","average":{"1min":0.84099999999999997,"5min":0.995,"15min":1.4399999999999999},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.315},"max":{"1min":1.3240000000000001,"5min":5.1539999999999999,"15min":13.478999999999999},"last":1.2969999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:01:43 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":0.91200000000000003,"15min":1.343},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.33900000000000002},"max":{"1min":2.3029999999999999,"5min":5.1980000000000004,"15min":13.436},"last":1.357},{"interface":"front","average":{"1min":1.3260000000000001,"5min":1.0820000000000001,"15min":1.3759999999999999},"min":{"1min":0.46800000000000003,"5min":0.373,"15min":0.35599999999999998},"max":{"1min":5.6130000000000004,"5min":7.2599999999999998,"15min":10.766999999999999},"last":1.1779999999999999}]}]},{"osd":6,"up_from":42,"seq":180388627148,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":898288,"kb_used_data":2984,"kb_used_omap":0,"kb_used_meta":895296,"kb_avail":92846864,"statfs":{"total":95995035648,"available":95075188736,"internally_reserved":0,"allocated":3055616,"data_stored":2515030,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":916783104},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:01:35 2023","interfaces":[{"interface":"back","average":{"1min":1.901,"5min":5.1520000000000001,"15min":2.863},"min":{"1min":0.29099999999999998,"5min":0.28699999999999998,"15min":0.23799999999999999},"max":{"1min":7.476,"5min":125.34399999999999,"15min":125.34399999999999},"last":0.67400000000000004},{"interface":"front","average":{"1min":1.5,"5min":5.1040000000000001,"15min":2.887},"min":{"1min":0.34799999999999998,"5min":0.27800000000000002,"15min":0.26600000000000001},"max":{"1min":4.6550000000000002,"5min":125.24299999999999,"15min":125.24299999999999},"last":0.93300000000000005}]},{"osd":1,"last update":"Sat Sep 23 20:00:58 2023","interfaces":[{"interface":"back","average":{"1min":1.0409999999999999,"5min":2.4550000000000001,"15min":1.9470000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":4.9039999999999999,"5min":125.51900000000001,"15min":125.51900000000001},"last":0.86099999999999999},{"interface":"front","average":{"1min":1.3009999999999999,"5min":2.6000000000000001,"15min":2.0379999999999998},"min":{"1min":0.315,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.8840000000000003,"5min":125.596,"15min":125.596},"last":1.1539999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:01:21 2023","interfaces":[{"interface":"back","average":{"1min":2.0859999999999999,"5min":2.8439999999999999,"15min":4.1890000000000001},"min":{"1min":0.309,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":5.9589999999999996,"5min":125.01600000000001,"15min":219.453},"last":0.745},{"interface":"front","average":{"1min":2.1739999999999999,"5min":3.0720000000000001,"15min":4.2220000000000004},"min":{"1min":0.33900000000000002,"5min":0.33700000000000002,"15min":0.30199999999999999},"max":{"1min":4.569,"5min":125.122,"15min":219.369},"last":1.224}]},{"osd":3,"last update":"Sat Sep 23 20:01:21 2023","interfaces":[{"interface":"back","average":{"1min":2.109,"5min":1.7729999999999999,"15min":1.7509999999999999},"min":{"1min":0.46800000000000003,"5min":0.32300000000000001,"15min":0.28599999999999998},"max":{"1min":6.109,"5min":13.823,"15min":16.428000000000001},"last":0.83099999999999996},{"interface":"front","average":{"1min":1.903,"5min":1.673,"15min":1.7070000000000001},"min":{"1min":0.39700000000000002,"5min":0.32600000000000001,"15min":0.311},"max":{"1min":9.6910000000000007,"5min":14.42,"15min":14.42},"last":0.96799999999999997}]},{"osd":4,"last update":"Sat Sep 23 20:01:21 2023","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":1.1220000000000001,"15min":1.5600000000000001},"min":{"1min":0.33900000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":4.4390000000000001,"5min":6.1719999999999997,"15min":14.401999999999999},"last":1.02},{"interface":"front","average":{"1min":1.5189999999999999,"5min":1.464,"15min":1.73},"min":{"1min":0.35299999999999998,"5min":0.33900000000000002,"15min":0.28299999999999997},"max":{"1min":6.3289999999999997,"5min":11.08,"15min":14.603},"last":1.0049999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:01:21 2023","interfaces":[{"interface":"back","average":{"1min":0.98399999999999999,"5min":1.1539999999999999,"15min":1.643},"min":{"1min":0.436,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":3.3809999999999998,"5min":6.4539999999999997,"15min":15.045},"last":1.089},{"interface":"front","average":{"1min":1.5289999999999999,"5min":1.3560000000000001,"15min":1.8620000000000001},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.33400000000000002},"max":{"1min":6.3140000000000001,"5min":11.048999999999999,"15min":16.244},"last":1.1759999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:01:39 2023","interfaces":[{"interface":"back","average":{"1min":0.96299999999999997,"5min":1.171,"15min":1.6339999999999999},"min":{"1min":0.36299999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":2.484,"5min":5.0359999999999996,"15min":11.709},"last":1.274},{"interface":"front","average":{"1min":1.6719999999999999,"5min":1.367,"15min":1.617},"min":{"1min":0.43099999999999999,"5min":0.371,"15min":0.32200000000000001},"max":{"1min":6.3499999999999996,"5min":11.000999999999999,"15min":12.327999999999999},"last":1.109}]}]},{"osd":1,"up_from":98,"seq":420906795480,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":917796,"kb_used_data":3036,"kb_used_omap":4,"kb_used_meta":914747,"kb_avail":92827356,"statfs":{"total":95995035648,"available":95055212544,"internally_reserved":0,"allocated":3108864,"data_stored":1990653,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":936701838},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:00:54 2023","interfaces":[{"interface":"back","average":{"1min":2.734,"5min":3.02,"15min":2.7589999999999999},"min":{"1min":0.29799999999999999,"5min":0.24199999999999999,"15min":0.20000000000000001},"max":{"1min":8.7829999999999995,"5min":46.527999999999999,"15min":134.37700000000001},"last":0.86599999999999999},{"interface":"front","average":{"1min":3.2730000000000001,"5min":2.7589999999999999,"15min":2.6179999999999999},"min":{"1min":0.22600000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":15.486000000000001,"5min":41.683999999999997,"15min":135.79400000000001},"last":0.77400000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:01:27 2023","interfaces":[{"interface":"back","average":{"1min":3.4409999999999998,"5min":3.4649999999999999,"15min":3.1349999999999998},"min":{"1min":0.378,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":16,"5min":41.719000000000001,"15min":72.859999999999999},"last":0.89800000000000002},{"interface":"front","average":{"1min":2.6400000000000001,"5min":2.73,"15min":2.6549999999999998},"min":{"1min":0.32100000000000001,"5min":0.28299999999999997,"15min":0.224},"max":{"1min":8.7780000000000005,"5min":47.439,"15min":73.665999999999997},"last":0.79700000000000004}]},{"osd":3,"last update":"Sat Sep 23 20:01:20 2023","interfaces":[{"interface":"back","average":{"1min":2.2709999999999999,"5min":2.6829999999999998,"15min":2.0299999999999998},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.25},"max":{"1min":8.6760000000000002,"5min":41.518000000000001,"15min":41.518000000000001},"last":0.96699999999999997},{"interface":"front","average":{"1min":2.1680000000000001,"5min":3.1360000000000001,"15min":2.1619999999999999},"min":{"1min":0.252,"5min":0.252,"15min":0.217},"max":{"1min":10.558,"5min":41.151000000000003,"15min":41.151000000000003},"last":0.71699999999999997}]},{"osd":4,"last update":"Sat Sep 23 20:00:54 2023","interfaces":[{"interface":"back","average":{"1min":2.6120000000000001,"5min":2.2869999999999999,"15min":2.1349999999999998},"min":{"1min":0.34000000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":9.109,"5min":39.085999999999999,"15min":48.381},"last":0.94799999999999995},{"interface":"front","average":{"1min":1.9670000000000001,"5min":2.3980000000000001,"15min":2.1909999999999998},"min":{"1min":0.40699999999999997,"5min":0.36599999999999999,"15min":0.32200000000000001},"max":{"1min":8.4900000000000002,"5min":41.320999999999998,"15min":82.239999999999995},"last":0.92100000000000004}]},{"osd":5,"last update":"Sat Sep 23 20:00:54 2023","interfaces":[{"interface":"back","average":{"1min":2.0539999999999998,"5min":2.2250000000000001,"15min":2.0009999999999999},"min":{"1min":0.40999999999999998,"5min":0.40699999999999997,"15min":0.36199999999999999},"max":{"1min":8.4540000000000006,"5min":41.274999999999999,"15min":48.643000000000001},"last":0.98699999999999999},{"interface":"front","average":{"1min":2.0259999999999998,"5min":2.5249999999999999,"15min":2.04},"min":{"1min":0.47099999999999997,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":8.8369999999999997,"5min":43.508000000000003,"15min":48.619999999999997},"last":1.012}]},{"osd":6,"last update":"Sat Sep 23 20:00:54 2023","interfaces":[{"interface":"back","average":{"1min":2.3799999999999999,"5min":2.734,"15min":2.3780000000000001},"min":{"1min":0.433,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":8.8119999999999994,"5min":46.472000000000001,"15min":48.548999999999999},"last":1.0409999999999999},{"interface":"front","average":{"1min":3.1299999999999999,"5min":2.8420000000000001,"15min":2.222},"min":{"1min":0.47199999999999998,"5min":0.35199999999999998,"15min":0.313},"max":{"1min":13.010999999999999,"5min":41.648000000000003,"15min":41.648000000000003},"last":1.1579999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:00:54 2023","interfaces":[{"interface":"back","average":{"1min":2.802,"5min":2.5459999999999998,"15min":2.3180000000000001},"min":{"1min":0.434,"5min":0.38300000000000001,"15min":0.36799999999999999},"max":{"1min":10.045,"5min":41.595999999999997,"15min":48.329999999999998},"last":1.288},{"interface":"front","average":{"1min":2.1309999999999998,"5min":2.254,"15min":2.2109999999999999},"min":{"1min":0.433,"5min":0.433,"15min":0.35299999999999998},"max":{"1min":9.2569999999999997,"5min":41.223999999999997,"15min":48.579000000000001},"last":1.133}]}]},{"osd":0,"up_from":85,"seq":365072220686,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":985292,"kb_used_data":3588,"kb_used_omap":5,"kb_used_meta":981690,"kb_avail":92759860,"statfs":{"total":95995035648,"available":94986096640,"internally_reserved":0,"allocated":3674112,"data_stored":2581162,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1005251194},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:00:58 2023","interfaces":[{"interface":"back","average":{"1min":2.8079999999999998,"5min":2.5179999999999998,"15min":2.6099999999999999},"min":{"1min":0.29299999999999998,"5min":0.219,"15min":0.216},"max":{"1min":8.6679999999999993,"5min":14.960000000000001,"15min":149.37299999999999},"last":0.77700000000000002},{"interface":"front","average":{"1min":4.2439999999999998,"5min":2.5339999999999998,"15min":2.6800000000000002},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":13.534000000000001,"5min":13.534000000000001,"15min":147.119},"last":0.75}]},{"osd":2,"last update":"Sat Sep 23 20:01:24 2023","interfaces":[{"interface":"back","average":{"1min":2.5859999999999999,"5min":2.528,"15min":2.75},"min":{"1min":0.46300000000000002,"5min":0.28899999999999998,"15min":0.245},"max":{"1min":8.7089999999999996,"5min":11.646000000000001,"15min":213.20599999999999},"last":0.66600000000000004},{"interface":"front","average":{"1min":2.5859999999999999,"5min":1.891,"15min":2.3839999999999999},"min":{"1min":0.39900000000000002,"5min":0.25700000000000001,"15min":0.24199999999999999},"max":{"1min":8.6899999999999995,"5min":11.542,"15min":209.64400000000001},"last":0.70799999999999996}]},{"osd":3,"last update":"Sat Sep 23 20:00:58 2023","interfaces":[{"interface":"back","average":{"1min":3.2040000000000002,"5min":1.923,"15min":1.7290000000000001},"min":{"1min":0.42099999999999999,"5min":0.28100000000000003,"15min":0.222},"max":{"1min":13.141,"5min":13.141,"15min":14.983000000000001},"last":1.22},{"interface":"front","average":{"1min":3.2930000000000001,"5min":1.966,"15min":1.7010000000000001},"min":{"1min":0.441,"5min":0.29999999999999999,"15min":0.28199999999999997},"max":{"1min":14,"5min":14,"15min":14.843},"last":1.2749999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:01:41 2023","interfaces":[{"interface":"back","average":{"1min":1.7629999999999999,"5min":1.736,"15min":1.546},"min":{"1min":0.495,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":6.6769999999999996,"5min":11.398,"15min":11.398},"last":1.3160000000000001},{"interface":"front","average":{"1min":1.6639999999999999,"5min":1.883,"15min":1.661},"min":{"1min":0.38,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":7.9729999999999999,"5min":10.747,"15min":10.747},"last":1.405}]},{"osd":5,"last update":"Sat Sep 23 20:01:41 2023","interfaces":[{"interface":"back","average":{"1min":1.726,"5min":1.8320000000000001,"15min":1.647},"min":{"1min":0.34599999999999997,"5min":0.316,"15min":0.316},"max":{"1min":5,"5min":10.69,"15min":10.69},"last":1.498},{"interface":"front","average":{"1min":1.8500000000000001,"5min":1.736,"15min":1.5940000000000001},"min":{"1min":0.46300000000000002,"5min":0.36499999999999999,"15min":0.34100000000000003},"max":{"1min":6.6200000000000001,"5min":11.510999999999999,"15min":11.510999999999999},"last":1.544}]},{"osd":6,"last update":"Sat Sep 23 20:01:41 2023","interfaces":[{"interface":"back","average":{"1min":1.821,"5min":1.8480000000000001,"15min":1.873},"min":{"1min":0.46300000000000002,"5min":0.39000000000000001,"15min":0.36699999999999999},"max":{"1min":6.6459999999999999,"5min":11.443,"15min":66.170000000000002},"last":1.6220000000000001},{"interface":"front","average":{"1min":1.748,"5min":2.0099999999999998,"15min":2.036},"min":{"1min":0.41199999999999998,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":4.8680000000000003,"5min":17.050999999999998,"15min":66.260999999999996},"last":1.571}]},{"osd":7,"last update":"Sat Sep 23 20:01:41 2023","interfaces":[{"interface":"back","average":{"1min":1.335,"5min":1.9379999999999999,"15min":2.375},"min":{"1min":0.61099999999999999,"5min":0.42099999999999999,"15min":0.40400000000000003},"max":{"1min":4.4800000000000004,"5min":10.419,"15min":210.47399999999999},"last":1.2549999999999999},{"interface":"front","average":{"1min":1.8779999999999999,"5min":1.9650000000000001,"15min":2.4460000000000002},"min":{"1min":0.432,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":6.7089999999999996,"5min":11.475,"15min":210.52199999999999},"last":1.444}]}]},{"osd":2,"up_from":159,"seq":682899800342,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":732600,"kb_used_data":2992,"kb_used_omap":7,"kb_used_meta":729592,"kb_avail":93012552,"statfs":{"total":95995035648,"available":95244853248,"internally_reserved":0,"allocated":3063808,"data_stored":1983520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":747102289},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:01:39 2023","interfaces":[{"interface":"back","average":{"1min":0.97599999999999998,"5min":2.1819999999999999,"15min":2.4009999999999998},"min":{"1min":0.28399999999999997,"5min":0.19700000000000001,"15min":0.17299999999999999},"max":{"1min":4.3959999999999999,"5min":14.624000000000001,"15min":117.77500000000001},"last":0.53200000000000003},{"interface":"front","average":{"1min":3.2370000000000001,"5min":2.6800000000000002,"15min":2.7679999999999998},"min":{"1min":0.311,"5min":0.223,"15min":0.214},"max":{"1min":40.539000000000001,"5min":40.539000000000001,"15min":117.91800000000001},"last":0.75600000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:01:39 2023","interfaces":[{"interface":"back","average":{"1min":2.6190000000000002,"5min":2.5649999999999999,"15min":2.9849999999999999},"min":{"1min":0.26400000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":13.989000000000001,"5min":13.989000000000001,"15min":114.107},"last":0.505},{"interface":"front","average":{"1min":1.21,"5min":1.976,"15min":2.5539999999999998},"min":{"1min":0.248,"5min":0.248,"15min":0.23699999999999999},"max":{"1min":13.737,"5min":13.737,"15min":117.172},"last":0.36899999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:01:39 2023","interfaces":[{"interface":"back","average":{"1min":2.4550000000000001,"5min":2.2120000000000002,"15min":2.1280000000000001},"min":{"1min":0.25,"5min":0.25,"15min":0.24099999999999999},"max":{"1min":14.045,"5min":14.045,"15min":17.751000000000001},"last":0.56000000000000005},{"interface":"front","average":{"1min":1.339,"5min":1.8169999999999999,"15min":1.927},"min":{"1min":0.32300000000000001,"5min":0.215,"15min":0.215},"max":{"1min":13.798999999999999,"5min":13.798999999999999,"15min":20.286999999999999},"last":0.69599999999999995}]},{"osd":4,"last update":"Sat Sep 23 20:01:39 2023","interfaces":[{"interface":"back","average":{"1min":1.7430000000000001,"5min":1.6000000000000001,"15min":1.9259999999999999},"min":{"1min":0.379,"5min":0.33800000000000002,"15min":0.27100000000000002},"max":{"1min":7.6589999999999998,"5min":8.202,"15min":14.194000000000001},"last":0.64400000000000002},{"interface":"front","average":{"1min":2.444,"5min":1.851,"15min":2.1520000000000001},"min":{"1min":0.33300000000000002,"5min":0.32600000000000001,"15min":0.31900000000000001},"max":{"1min":14.138999999999999,"5min":14.138999999999999,"15min":14.271000000000001},"last":0.94399999999999995}]},{"osd":5,"last update":"Sat Sep 23 20:01:39 2023","interfaces":[{"interface":"back","average":{"1min":0.84999999999999998,"5min":1.5920000000000001,"15min":1.696},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.27600000000000002},"max":{"1min":4.0960000000000001,"5min":14.007,"15min":14.007},"last":0.74099999999999999},{"interface":"front","average":{"1min":1.869,"5min":1.708,"15min":1.7170000000000001},"min":{"1min":0.41199999999999998,"5min":0.377,"15min":0.32100000000000001},"max":{"1min":7.6100000000000003,"5min":8.2720000000000002,"15min":10.193},"last":0.82899999999999996}]},{"osd":6,"last update":"Sat Sep 23 20:01:39 2023","interfaces":[{"interface":"back","average":{"1min":1.748,"5min":1.7130000000000001,"15min":1.9199999999999999},"min":{"1min":0.44,"5min":0.39500000000000002,"15min":0.35699999999999998},"max":{"1min":7.6909999999999998,"5min":8.2390000000000008,"15min":10.166},"last":0.92200000000000004},{"interface":"front","average":{"1min":2.548,"5min":1.909,"15min":2.1869999999999998},"min":{"1min":0.45400000000000001,"5min":0.371,"15min":0.371},"max":{"1min":14.077999999999999,"5min":14.077999999999999,"15min":14.077999999999999},"last":1.083}]},{"osd":7,"last update":"Sat Sep 23 20:01:39 2023","interfaces":[{"interface":"back","average":{"1min":1.1639999999999999,"5min":1.819,"15min":1.8839999999999999},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.38500000000000001},"max":{"1min":9.2680000000000007,"5min":14.058,"15min":32.945},"last":0.81499999999999995},{"interface":"front","average":{"1min":2.782,"5min":2.0350000000000001,"15min":2.1499999999999999},"min":{"1min":0.47299999999999998,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":14.109999999999999,"5min":14.109999999999999,"15min":16.593},"last":1.026}]}]},{"osd":3,"up_from":116,"seq":498216206743,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":824804,"kb_used_data":2908,"kb_used_omap":2,"kb_used_meta":821885,"kb_avail":92920348,"statfs":{"total":95995035648,"available":95150436352,"internally_reserved":0,"allocated":2977792,"data_stored":1974209,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":841610293},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:01:11 2023","interfaces":[{"interface":"back","average":{"1min":2.1850000000000001,"5min":4.7069999999999999,"15min":2.867},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.19600000000000001},"max":{"1min":10.085000000000001,"5min":117.248,"15min":117.248},"last":0.69699999999999995},{"interface":"front","average":{"1min":2.883,"5min":4.4779999999999998,"15min":2.9390000000000001},"min":{"1min":0.26800000000000002,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":12.212,"5min":116.187,"15min":116.187},"last":0.65700000000000003}]},{"osd":1,"last update":"Sat Sep 23 20:01:11 2023","interfaces":[{"interface":"back","average":{"1min":2.8530000000000002,"5min":2.2839999999999998,"15min":1.9870000000000001},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.23400000000000001},"max":{"1min":12.243,"5min":12.243,"15min":48.918999999999997},"last":0.83199999999999996},{"interface":"front","average":{"1min":1.4850000000000001,"5min":1.9159999999999999,"15min":1.835},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":5.8559999999999999,"5min":10.029,"15min":50.722999999999999},"last":0.80700000000000005}]},{"osd":2,"last update":"Sat Sep 23 20:01:32 2023","interfaces":[{"interface":"back","average":{"1min":2.3130000000000002,"5min":2.3580000000000001,"15min":2.8109999999999999},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":14.529,"5min":14.529,"15min":116.363},"last":0.76300000000000001},{"interface":"front","average":{"1min":2.8719999999999999,"5min":2.407,"15min":2.8100000000000001},"min":{"1min":0.35699999999999998,"5min":0.30599999999999999,"15min":0.27300000000000002},"max":{"1min":12.832000000000001,"5min":12.832000000000001,"15min":118.474},"last":0.745}]},{"osd":4,"last update":"Sat Sep 23 20:01:11 2023","interfaces":[{"interface":"back","average":{"1min":1.825,"5min":1.7170000000000001,"15min":1.6839999999999999},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":10.16,"5min":10.16,"15min":10.16},"last":1.306},{"interface":"front","average":{"1min":2.5990000000000002,"5min":1.978,"15min":1.8740000000000001},"min":{"1min":0.40600000000000003,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":12.169,"5min":12.169,"15min":12.169},"last":1.1000000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:01:11 2023","interfaces":[{"interface":"back","average":{"1min":2.5960000000000001,"5min":1.9239999999999999,"15min":1.8500000000000001},"min":{"1min":0.40100000000000002,"5min":0.35599999999999998,"15min":0.33900000000000002},"max":{"1min":11.978,"5min":11.978,"15min":11.978},"last":0.95599999999999996},{"interface":"front","average":{"1min":2.0499999999999998,"5min":1.6850000000000001,"15min":1.6499999999999999},"min":{"1min":0.40000000000000002,"5min":0.34899999999999998,"15min":0.34000000000000002},"max":{"1min":12.741,"5min":12.741,"15min":12.741},"last":1.014}]},{"osd":6,"last update":"Sat Sep 23 20:01:11 2023","interfaces":[{"interface":"back","average":{"1min":3.3180000000000001,"5min":2.262,"15min":2.1240000000000001},"min":{"1min":0.441,"5min":0.42199999999999999,"15min":0.41199999999999998},"max":{"1min":12.135,"5min":12.135,"15min":12.135},"last":1.0509999999999999},{"interface":"front","average":{"1min":2.4820000000000002,"5min":1.76,"15min":1.8260000000000001},"min":{"1min":0.45900000000000002,"5min":0.39700000000000002,"15min":0.38900000000000001},"max":{"1min":12.776999999999999,"5min":12.776999999999999,"15min":12.776999999999999},"last":0.92500000000000004}]},{"osd":7,"last update":"Sat Sep 23 20:01:11 2023","interfaces":[{"interface":"back","average":{"1min":2.4689999999999999,"5min":1.77,"15min":1.923},"min":{"1min":0.38900000000000001,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":12.802,"5min":12.802,"15min":12.802},"last":0.98099999999999998},{"interface":"front","average":{"1min":1.504,"5min":1.6200000000000001,"15min":1.7410000000000001},"min":{"1min":0.45000000000000001,"5min":0.437,"15min":0.40600000000000003},"max":{"1min":5.7519999999999998,"5min":9.9890000000000008,"15min":9.9890000000000008},"last":1.2350000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019613,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1190712,"kb_used_data":2416,"kb_used_omap":0,"kb_used_meta":1188288,"kb_avail":92554440,"statfs":{"total":95995035648,"available":94775746560,"internally_reserved":0,"allocated":2473984,"data_stored":1925053,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1216806912},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:01:37 2023","interfaces":[{"interface":"back","average":{"1min":0.72899999999999998,"5min":1.7509999999999999,"15min":2.117},"min":{"1min":0.27500000000000002,"5min":0.26000000000000001,"15min":0.247},"max":{"1min":1.482,"5min":23.059999999999999,"15min":150.155},"last":0.996},{"interface":"front","average":{"1min":1.157,"5min":2.161,"15min":2.1099999999999999},"min":{"1min":0.441,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":4.6050000000000004,"5min":22.780000000000001,"15min":149.47900000000001},"last":1.0680000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:01:02 2023","interfaces":[{"interface":"back","average":{"1min":2.8370000000000002,"5min":2.4350000000000001,"15min":1.7689999999999999},"min":{"1min":0.34599999999999997,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":17.510000000000002,"5min":30.754000000000001,"15min":30.754000000000001},"last":1.0329999999999999},{"interface":"front","average":{"1min":2.8010000000000002,"5min":2.3940000000000001,"15min":1.7989999999999999},"min":{"1min":0.38100000000000001,"5min":0.36499999999999999,"15min":0.27600000000000002},"max":{"1min":17.550999999999998,"5min":30.777999999999999,"15min":30.777999999999999},"last":1.204}]},{"osd":2,"last update":"Sat Sep 23 20:01:32 2023","interfaces":[{"interface":"back","average":{"1min":1.9319999999999999,"5min":1.925,"15min":3.6309999999999998},"min":{"1min":0.41499999999999998,"5min":0.307,"15min":0.27600000000000002},"max":{"1min":7.2439999999999998,"5min":30.088999999999999,"15min":219.02799999999999},"last":0.78300000000000003},{"interface":"front","average":{"1min":1.756,"5min":5.4690000000000003,"15min":4.806},"min":{"1min":0.34000000000000002,"5min":0.32300000000000001,"15min":0.308},"max":{"1min":9.0700000000000003,"5min":314.31599999999997,"15min":314.31599999999997},"last":0.84599999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:01:32 2023","interfaces":[{"interface":"back","average":{"1min":1.839,"5min":1.538,"15min":1.5860000000000001},"min":{"1min":0.372,"5min":0.35199999999999998,"15min":0.308},"max":{"1min":5.9850000000000003,"5min":8.5329999999999995,"15min":17.873999999999999},"last":0.93999999999999995},{"interface":"front","average":{"1min":2.157,"5min":1.6919999999999999,"15min":1.6180000000000001},"min":{"1min":0.439,"5min":0.35099999999999998,"15min":0.32300000000000001},"max":{"1min":5.3860000000000001,"5min":9.5280000000000005,"15min":13.859},"last":0.91400000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:01:37 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":1.1599999999999999,"15min":1.4199999999999999},"min":{"1min":0.31,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":6.0570000000000004,"5min":6.952,"15min":9.4740000000000002},"last":1.278},{"interface":"front","average":{"1min":0.79500000000000004,"5min":0.92300000000000004,"15min":1.4430000000000001},"min":{"1min":0.41399999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":1.4359999999999999,"5min":5.2729999999999997,"15min":12.506},"last":1.0469999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:01:02 2023","interfaces":[{"interface":"back","average":{"1min":0.88300000000000001,"5min":1.099,"15min":1.5209999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.29399999999999998},"max":{"1min":3.2010000000000001,"5min":4.9619999999999997,"15min":18.103000000000002},"last":1.111},{"interface":"front","average":{"1min":0.72299999999999998,"5min":0.998,"15min":1.5680000000000001},"min":{"1min":0.41099999999999998,"5min":0.36799999999999999,"15min":0.31},"max":{"1min":1.5129999999999999,"5min":5.3150000000000004,"15min":19.706},"last":1.089}]},{"osd":7,"last update":"Sat Sep 23 20:01:02 2023","interfaces":[{"interface":"back","average":{"1min":1.4370000000000001,"5min":1.331,"15min":2.6680000000000001},"min":{"1min":0.40100000000000002,"5min":0.35199999999999998,"15min":0.308},"max":{"1min":12.475,"5min":12.475,"15min":294.76499999999999},"last":0.97599999999999998},{"interface":"front","average":{"1min":0.83999999999999997,"5min":1.1639999999999999,"15min":2.7200000000000002},"min":{"1min":0.42399999999999999,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":3.2469999999999999,"5min":6.9210000000000003,"15min":294.47699999999998},"last":1.3360000000000001}]}]},{"osd":5,"up_from":36,"seq":154618823380,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1185376,"kb_used_data":2392,"kb_used_omap":0,"kb_used_meta":1182976,"kb_avail":92559776,"statfs":{"total":95995035648,"available":94781210624,"internally_reserved":0,"allocated":2449408,"data_stored":1920167,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1211367424},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:01:33 2023","interfaces":[{"interface":"back","average":{"1min":1.843,"5min":5.8470000000000004,"15min":2.915},"min":{"1min":0.29199999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":9.0169999999999995,"5min":201.196,"15min":201.196},"last":0.92700000000000005},{"interface":"front","average":{"1min":0.878,"5min":5.9210000000000003,"15min":3.2690000000000001},"min":{"1min":0.28000000000000003,"5min":0.27700000000000002,"15min":0.251},"max":{"1min":2.8220000000000001,"5min":192.941,"15min":192.941},"last":0.96299999999999997}]},{"osd":1,"last update":"Sat Sep 23 20:01:01 2023","interfaces":[{"interface":"back","average":{"1min":9.4670000000000005,"5min":7.827,"15min":4.2800000000000002},"min":{"1min":0.55000000000000004,"5min":0.28599999999999998,"15min":0.28000000000000003},"max":{"1min":76.453000000000003,"5min":190.298,"15min":190.298},"last":0.82799999999999996},{"interface":"front","average":{"1min":9.1120000000000001,"5min":7.6520000000000001,"15min":4.4950000000000001},"min":{"1min":0.35399999999999998,"5min":0.32700000000000001,"15min":0.28799999999999998},"max":{"1min":76.367000000000004,"5min":190.65799999999999,"15min":190.65799999999999},"last":1.071}]},{"osd":2,"last update":"Sat Sep 23 20:01:17 2023","interfaces":[{"interface":"back","average":{"1min":9.4789999999999992,"5min":4.3369999999999997,"15min":4.056},"min":{"1min":0.36699999999999999,"5min":0.311,"15min":0.309},"max":{"1min":151.322,"5min":151.322,"15min":241.59700000000001},"last":0.85899999999999999},{"interface":"front","average":{"1min":10.257999999999999,"5min":4.6459999999999999,"15min":4.0819999999999999},"min":{"1min":0.34999999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":166.941,"5min":166.941,"15min":242.13999999999999},"last":1.099}]},{"osd":3,"last update":"Sat Sep 23 20:01:01 2023","interfaces":[{"interface":"back","average":{"1min":2.008,"5min":1.4330000000000001,"15min":1.47},"min":{"1min":0.36799999999999999,"5min":0.35399999999999998,"15min":0.34799999999999998},"max":{"1min":7.7290000000000001,"5min":7.7290000000000001,"15min":9.3829999999999991},"last":0.89900000000000002},{"interface":"front","average":{"1min":2.206,"5min":1.5569999999999999,"15min":1.4430000000000001},"min":{"1min":0.35999999999999999,"5min":0.35199999999999998,"15min":0.33400000000000002},"max":{"1min":13.840999999999999,"5min":13.840999999999999,"15min":13.840999999999999},"last":1.002}]},{"osd":4,"last update":"Sat Sep 23 20:01:33 2023","interfaces":[{"interface":"back","average":{"1min":0.82699999999999996,"5min":1.119,"15min":1.601},"min":{"1min":0.318,"5min":0.28399999999999997,"15min":0.26800000000000002},"max":{"1min":2.8519999999999999,"5min":7.2110000000000003,"15min":12.051},"last":1.1519999999999999},{"interface":"front","average":{"1min":0.85999999999999999,"5min":1.0489999999999999,"15min":1.403},"min":{"1min":0.34100000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":2.6850000000000001,"5min":10.388,"15min":16.462},"last":0.77500000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:01:17 2023","interfaces":[{"interface":"back","average":{"1min":1.093,"5min":0.95799999999999996,"15min":1.3460000000000001},"min":{"1min":0.33900000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":5.702,"5min":5.702,"15min":9.641},"last":1.216},{"interface":"front","average":{"1min":1.1120000000000001,"5min":1.0289999999999999,"15min":1.3959999999999999},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.28999999999999998},"max":{"1min":5.6639999999999997,"5min":5.6639999999999997,"15min":11.444000000000001},"last":1.133}]},{"osd":7,"last update":"Sat Sep 23 20:01:33 2023","interfaces":[{"interface":"back","average":{"1min":1.038,"5min":0.96999999999999997,"15min":1.4850000000000001},"min":{"1min":0.40000000000000002,"5min":0.34899999999999998,"15min":0.29999999999999999},"max":{"1min":3.141,"5min":4.5049999999999999,"15min":9.8819999999999997},"last":0.95299999999999996},{"interface":"front","average":{"1min":1.2030000000000001,"5min":1.0489999999999999,"15min":1.407},"min":{"1min":0.42999999999999999,"5min":0.34399999999999997,"15min":0.33900000000000002},"max":{"1min":5.4029999999999996,"5min":7.157,"15min":17.053000000000001},"last":1.171}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:01:57.519 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:01:57.527 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:57.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:57.530 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:01:57.573 INFO:tasks.workunit.client.0.smithi016.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-09-23T20:01:57.630 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:57.658 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:57.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:57.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:57.681 INFO:tasks.workunit.client.0.smithi016.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-09-23T20:01:57.736 INFO:tasks.workunit.client.0.smithi016.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T20:01:57.738 INFO:tasks.workunit.client.0.smithi016.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T20:01:57.739 INFO:tasks.workunit.client.0.smithi016.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-09-23T20:01:57.763 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:57.797 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:01:57.798 INFO:tasks.workunit.client.0.smithi016.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T20:01:57.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:57.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:01:57.805 DEBUG:teuthology.orchestra.run.smithi016:> set -ex 2023-09-23T20:01:57.805 DEBUG:teuthology.orchestra.run.smithi016:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-09-23T20:01:57.867 INFO:tasks.workunit:Running workunits matching cls/test_cls_rbd.sh on client.0... 2023-09-23T20:01:57.868 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2023-09-23T20:01:57.868 DEBUG:teuthology.orchestra.run.smithi016:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=pacific TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2023-09-23T20:01:57.902 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:57.948 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:57.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:57.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:58.006 INFO:tasks.workunit.client.0.smithi016.stdout:Running main() from gmock_main.cc 2023-09-23T20:01:58.006 INFO:tasks.workunit.client.0.smithi016.stdout:[==========] Running 70 tests from 1 test suite. 2023-09-23T20:01:58.007 INFO:tasks.workunit.client.0.smithi016.stdout:[----------] Global test environment set-up. 2023-09-23T20:01:58.007 INFO:tasks.workunit.client.0.smithi016.stdout:[----------] 70 tests from TestClsRbd 2023-09-23T20:01:58.052 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:58.084 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:58.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:58.087 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:58.188 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:58.215 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:58.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:58.219 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:58.319 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:58.350 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:58.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:58.353 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:58.454 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:58.485 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:58.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:58.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:58.489 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:58 smithi016 bash[75004]: cluster 2023-09-23T20:01:56.662397+0000 mgr.y (mgr.25325) 1642 : cluster 0 pgmap v1680: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:01:58.589 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:58.622 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:58.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:58.625 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:58.726 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:58.738 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:58 smithi016 bash[77995]: cluster 2023-09-23T20:01:56.662397+0000 mgr.y (mgr.25325) 1642 : cluster 0 pgmap v1680: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:01:58.762 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:58.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:58.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:58.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:58 smithi092 bash[58855]: cluster 2023-09-23T20:01:56.662397+0000 mgr.y (mgr.25325) 1642 : cluster 0 pgmap v1680: 73 pgs: 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:01:58.865 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:58.896 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:58.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:58.899 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:59.000 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:59.033 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:59.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:59.036 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:59.136 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:59.168 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:59.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:59.171 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:59.272 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:59.305 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:59.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:59.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:59.409 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:59.441 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:59.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:59.444 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:59.521 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:01:59.531 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:01:59.545 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:59.560 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.get_all_features 2023-09-23T20:01:59.579 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:59.580 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.get_all_features (19 ms) 2023-09-23T20:01:59.580 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.copyup 2023-09-23T20:01:59.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:59.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:59.682 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:59.711 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:59.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:59.714 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:59.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:59 smithi016 bash[77995]: cluster 2023-09-23T20:01:58.540797+0000 mon.a (mon.0) 1974 : cluster 0 osdmap e277: 8 total, 8 up, 8 in 2023-09-23T20:01:59.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:59 smithi016 bash[77995]: audit 2023-09-23T20:01:58.543497+0000 mon.a (mon.0) 1975 : audit 1 from='client.? 172.21.15.16:0/1796055372' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi016-248214-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:01:59.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:59 smithi016 bash[77995]: cluster 2023-09-23T20:01:58.545663+0000 osd.7 (osd.7) 192 : cluster 3 failed to encode map e277 with expected crc 2023-09-23T20:01:59.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:59 smithi016 bash[77995]: cluster 2023-09-23T20:01:58.546068+0000 osd.6 (osd.6) 209 : cluster 3 failed to encode map e277 with expected crc 2023-09-23T20:01:59.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:59 smithi016 bash[77995]: cluster 2023-09-23T20:01:58.548552+0000 osd.5 (osd.5) 198 : cluster 3 failed to encode map e277 with expected crc 2023-09-23T20:01:59.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:59 smithi016 bash[77995]: cluster 2023-09-23T20:01:58.550532+0000 osd.4 (osd.4) 216 : cluster 3 failed to encode map e277 with expected crc 2023-09-23T20:01:59.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:59 smithi016 bash[77995]: cluster 2023-09-23T20:01:58.552449+0000 osd.4 (osd.4) 217 : cluster 3 failed to encode map e277 with expected crc 2023-09-23T20:01:59.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:59 smithi016 bash[77995]: cluster 2023-09-23T20:01:58.663088+0000 mgr.y (mgr.25325) 1643 : cluster 0 pgmap v1682: 105 pgs: 1 creating+peering, 31 unknown, 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:01:59.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:59 smithi016 bash[77995]: audit 2023-09-23T20:01:58.705321+0000 mon.a (mon.0) 1976 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:01:59.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:01:59 smithi016 bash[77995]: audit 2023-09-23T20:01:58.707313+0000 mon.a (mon.0) 1977 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:01:59.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:59 smithi016 bash[75004]: cluster 2023-09-23T20:01:58.540797+0000 mon.a (mon.0) 1974 : cluster 0 osdmap e277: 8 total, 8 up, 8 in 2023-09-23T20:01:59.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:59 smithi016 bash[75004]: audit 2023-09-23T20:01:58.543497+0000 mon.a (mon.0) 1975 : audit 1 from='client.? 172.21.15.16:0/1796055372' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi016-248214-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:01:59.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:59 smithi016 bash[75004]: cluster 2023-09-23T20:01:58.545663+0000 osd.7 (osd.7) 192 : cluster 3 failed to encode map e277 with expected crc 2023-09-23T20:01:59.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:59 smithi016 bash[75004]: cluster 2023-09-23T20:01:58.546068+0000 osd.6 (osd.6) 209 : cluster 3 failed to encode map e277 with expected crc 2023-09-23T20:01:59.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:59 smithi016 bash[75004]: cluster 2023-09-23T20:01:58.548552+0000 osd.5 (osd.5) 198 : cluster 3 failed to encode map e277 with expected crc 2023-09-23T20:01:59.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:59 smithi016 bash[75004]: cluster 2023-09-23T20:01:58.550532+0000 osd.4 (osd.4) 216 : cluster 3 failed to encode map e277 with expected crc 2023-09-23T20:01:59.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:59 smithi016 bash[75004]: cluster 2023-09-23T20:01:58.552449+0000 osd.4 (osd.4) 217 : cluster 3 failed to encode map e277 with expected crc 2023-09-23T20:01:59.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:59 smithi016 bash[75004]: cluster 2023-09-23T20:01:58.663088+0000 mgr.y (mgr.25325) 1643 : cluster 0 pgmap v1682: 105 pgs: 1 creating+peering, 31 unknown, 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:01:59.788 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:59 smithi016 bash[75004]: audit 2023-09-23T20:01:58.705321+0000 mon.a (mon.0) 1976 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:01:59.788 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:01:59 smithi016 bash[75004]: audit 2023-09-23T20:01:58.707313+0000 mon.a (mon.0) 1977 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:01:59.812 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.copyup (232 ms) 2023-09-23T20:01:59.812 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.sparse_copyup 2023-09-23T20:01:59.815 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:59.840 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:59.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:59.843 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:59.866 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.sparse_copyup (54 ms) 2023-09-23T20:01:59.866 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.get_and_set_id 2023-09-23T20:01:59.878 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.get_and_set_id (13 ms) 2023-09-23T20:01:59.878 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.add_remove_child 2023-09-23T20:01:59.930 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.add_remove_child (52 ms) 2023-09-23T20:01:59.930 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.directory_methods 2023-09-23T20:01:59.945 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:59.972 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:59.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:01:59.977 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:59.977 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.directory_methods (46 ms) 2023-09-23T20:01:59.977 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.create 2023-09-23T20:02:00.005 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.create (28 ms) 2023-09-23T20:02:00.005 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.get_features 2023-09-23T20:02:00.008 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.get_features (4 ms) 2023-09-23T20:02:00.009 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.get_object_prefix 2023-09-23T20:02:00.012 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.get_object_prefix (3 ms) 2023-09-23T20:02:00.012 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2023-09-23T20:02:00.016 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.get_create_timestamp (3 ms) 2023-09-23T20:02:00.016 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2023-09-23T20:02:00.019 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.get_access_timestamp (4 ms) 2023-09-23T20:02:00.019 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2023-09-23T20:02:00.022 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.get_modify_timestamp (2 ms) 2023-09-23T20:02:00.022 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.get_data_pool 2023-09-23T20:02:00.028 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.get_data_pool (7 ms) 2023-09-23T20:02:00.028 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.get_size 2023-09-23T20:02:00.037 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.get_size (9 ms) 2023-09-23T20:02:00.037 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.set_size 2023-09-23T20:02:00.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:59 smithi092 bash[58855]: cluster 2023-09-23T20:01:58.540797+0000 mon.a (mon.0) 1974 : cluster 0 osdmap e277: 8 total, 8 up, 8 in 2023-09-23T20:02:00.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:59 smithi092 bash[58855]: audit 2023-09-23T20:01:58.543497+0000 mon.a (mon.0) 1975 : audit 1 from='client.? 172.21.15.16:0/1796055372' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi016-248214-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:02:00.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:59 smithi092 bash[58855]: cluster 2023-09-23T20:01:58.545663+0000 osd.7 (osd.7) 192 : cluster 3 failed to encode map e277 with expected crc 2023-09-23T20:02:00.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:59 smithi092 bash[58855]: cluster 2023-09-23T20:01:58.546068+0000 osd.6 (osd.6) 209 : cluster 3 failed to encode map e277 with expected crc 2023-09-23T20:02:00.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:59 smithi092 bash[58855]: cluster 2023-09-23T20:01:58.548552+0000 osd.5 (osd.5) 198 : cluster 3 failed to encode map e277 with expected crc 2023-09-23T20:02:00.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:59 smithi092 bash[58855]: cluster 2023-09-23T20:01:58.550532+0000 osd.4 (osd.4) 216 : cluster 3 failed to encode map e277 with expected crc 2023-09-23T20:02:00.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:59 smithi092 bash[58855]: cluster 2023-09-23T20:01:58.552449+0000 osd.4 (osd.4) 217 : cluster 3 failed to encode map e277 with expected crc 2023-09-23T20:02:00.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:59 smithi092 bash[58855]: cluster 2023-09-23T20:01:58.663088+0000 mgr.y (mgr.25325) 1643 : cluster 0 pgmap v1682: 105 pgs: 1 creating+peering, 31 unknown, 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:02:00.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:59 smithi092 bash[58855]: audit 2023-09-23T20:01:58.705321+0000 mon.a (mon.0) 1976 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:02:00.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:01:59 smithi092 bash[58855]: audit 2023-09-23T20:01:58.707313+0000 mon.a (mon.0) 1977 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:02:00.047 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.set_size (9 ms) 2023-09-23T20:02:00.047 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.protection_status 2023-09-23T20:02:00.078 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:00.083 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.protection_status (36 ms) 2023-09-23T20:02:00.083 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.snapshot_limits 2023-09-23T20:02:00.098 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.snapshot_limits (15 ms) 2023-09-23T20:02:00.098 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.parents_v1 2023-09-23T20:02:00.111 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:00.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:00.115 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:00.178 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.parents_v1 (81 ms) 2023-09-23T20:02:00.179 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.parents_v2 2023-09-23T20:02:00.216 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:00.240 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.parents_v2 (62 ms) 2023-09-23T20:02:00.240 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.snapshots 2023-09-23T20:02:00.247 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:00.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:00.250 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:00.281 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.snapshots (41 ms) 2023-09-23T20:02:00.281 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.snapid_race 2023-09-23T20:02:00.293 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.snapid_race (11 ms) 2023-09-23T20:02:00.293 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.stripingv2 2023-09-23T20:02:00.312 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.stripingv2 (19 ms) 2023-09-23T20:02:00.312 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.object_map_save 2023-09-23T20:02:00.318 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.object_map_save (6 ms) 2023-09-23T20:02:00.318 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.object_map_resize 2023-09-23T20:02:00.332 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.object_map_resize (15 ms) 2023-09-23T20:02:00.332 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.object_map_update 2023-09-23T20:02:00.344 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.object_map_update (11 ms) 2023-09-23T20:02:00.344 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2023-09-23T20:02:00.344 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.object_map_load_enoent (1 ms) 2023-09-23T20:02:00.344 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2023-09-23T20:02:00.351 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:00.355 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.object_map_snap_add (10 ms) 2023-09-23T20:02:00.355 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2023-09-23T20:02:00.364 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.object_map_snap_remove (10 ms) 2023-09-23T20:02:00.364 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.flags 2023-09-23T20:02:00.375 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.flags (10 ms) 2023-09-23T20:02:00.375 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.metadata 2023-09-23T20:02:00.382 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:00.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:00.385 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:00.420 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.metadata (45 ms) 2023-09-23T20:02:00.420 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.set_features 2023-09-23T20:02:00.435 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.set_features (16 ms) 2023-09-23T20:02:00.436 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.mirror 2023-09-23T20:02:00.485 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:00.504 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.mirror (68 ms) 2023-09-23T20:02:00.504 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.mirror_image 2023-09-23T20:02:00.517 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:00.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:00.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:00.531 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.mirror_image (27 ms) 2023-09-23T20:02:00.531 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.mirror_image_status 2023-09-23T20:02:00.621 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:00.653 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:00.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:00.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:00.757 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:00.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:00 smithi016 bash[75004]: audit 2023-09-23T20:01:59.549766+0000 mon.a (mon.0) 1978 : audit 1 from='client.? 172.21.15.16:0/1796055372' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi016-248214-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-09-23T20:02:00.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:00 smithi016 bash[75004]: cluster 2023-09-23T20:01:59.550022+0000 mon.a (mon.0) 1979 : cluster 0 osdmap e278: 8 total, 8 up, 8 in 2023-09-23T20:02:00.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:00 smithi016 bash[75004]: cluster 2023-09-23T20:01:59.550073+0000 osd.4 (osd.4) 218 : cluster 3 failed to encode map e278 with expected crc 2023-09-23T20:02:00.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:00 smithi016 bash[75004]: cluster 2023-09-23T20:01:59.555337+0000 osd.7 (osd.7) 193 : cluster 3 failed to encode map e278 with expected crc 2023-09-23T20:02:00.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:00 smithi016 bash[75004]: cluster 2023-09-23T20:01:59.557282+0000 osd.4 (osd.4) 219 : cluster 3 failed to encode map e278 with expected crc 2023-09-23T20:02:00.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:00 smithi016 bash[75004]: cluster 2023-09-23T20:01:59.557313+0000 osd.6 (osd.6) 210 : cluster 3 failed to encode map e278 with expected crc 2023-09-23T20:02:00.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:00 smithi016 bash[77995]: audit 2023-09-23T20:01:59.549766+0000 mon.a (mon.0) 1978 : audit 1 from='client.? 172.21.15.16:0/1796055372' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi016-248214-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-09-23T20:02:00.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:00 smithi016 bash[77995]: cluster 2023-09-23T20:01:59.550022+0000 mon.a (mon.0) 1979 : cluster 0 osdmap e278: 8 total, 8 up, 8 in 2023-09-23T20:02:00.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:00 smithi016 bash[77995]: cluster 2023-09-23T20:01:59.550073+0000 osd.4 (osd.4) 218 : cluster 3 failed to encode map e278 with expected crc 2023-09-23T20:02:00.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:00 smithi016 bash[77995]: cluster 2023-09-23T20:01:59.555337+0000 osd.7 (osd.7) 193 : cluster 3 failed to encode map e278 with expected crc 2023-09-23T20:02:00.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:00 smithi016 bash[77995]: cluster 2023-09-23T20:01:59.557282+0000 osd.4 (osd.4) 219 : cluster 3 failed to encode map e278 with expected crc 2023-09-23T20:02:00.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:00 smithi016 bash[77995]: cluster 2023-09-23T20:01:59.557313+0000 osd.6 (osd.6) 210 : cluster 3 failed to encode map e278 with expected crc 2023-09-23T20:02:00.791 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:00.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:00.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:00.895 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:00.906 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:00 smithi092 bash[58855]: audit 2023-09-23T20:01:59.549766+0000 mon.a (mon.0) 1978 : audit 1 from='client.? 172.21.15.16:0/1796055372' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi016-248214-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-09-23T20:02:00.906 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:00 smithi092 bash[58855]: cluster 2023-09-23T20:01:59.550022+0000 mon.a (mon.0) 1979 : cluster 0 osdmap e278: 8 total, 8 up, 8 in 2023-09-23T20:02:00.906 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:00 smithi092 bash[58855]: cluster 2023-09-23T20:01:59.550073+0000 osd.4 (osd.4) 218 : cluster 3 failed to encode map e278 with expected crc 2023-09-23T20:02:00.907 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:00 smithi092 bash[58855]: cluster 2023-09-23T20:01:59.555337+0000 osd.7 (osd.7) 193 : cluster 3 failed to encode map e278 with expected crc 2023-09-23T20:02:00.907 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:00 smithi092 bash[58855]: cluster 2023-09-23T20:01:59.557282+0000 osd.4 (osd.4) 219 : cluster 3 failed to encode map e278 with expected crc 2023-09-23T20:02:00.907 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:00 smithi092 bash[58855]: cluster 2023-09-23T20:01:59.557313+0000 osd.6 (osd.6) 210 : cluster 3 failed to encode map e278 with expected crc 2023-09-23T20:02:00.924 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:00.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:00.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:01.028 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:01.055 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:01.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:01.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:01.159 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:01.186 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:01.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:01.189 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:01.290 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:01.319 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:01.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:01.321 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:01.422 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:01.453 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:01.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:01.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:01.557 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:01.558 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:01 smithi016 bash[75004]: cluster 2023-09-23T20:02:00.558532+0000 mon.a (mon.0) 1980 : cluster 0 osdmap e279: 8 total, 8 up, 8 in 2023-09-23T20:02:01.559 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:01 smithi016 bash[75004]: cluster 2023-09-23T20:02:00.564318+0000 osd.5 (osd.5) 199 : cluster 3 failed to encode map e279 with expected crc 2023-09-23T20:02:01.559 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:01 smithi016 bash[75004]: cluster 2023-09-23T20:02:00.663765+0000 mgr.y (mgr.25325) 1644 : cluster 0 pgmap v1685: 105 pgs: 1 creating+activating, 1 creating+peering, 13 unknown, 90 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 1.3 MiB/s rd, 688 KiB/s wr, 19 op/s 2023-09-23T20:02:01.588 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:01.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:01.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:01.691 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:01.719 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:01.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:01.722 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:01.823 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:01.835 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:01 smithi016 bash[77995]: cluster 2023-09-23T20:02:00.558532+0000 mon.a (mon.0) 1980 : cluster 0 osdmap e279: 8 total, 8 up, 8 in 2023-09-23T20:02:01.835 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:01 smithi016 bash[77995]: cluster 2023-09-23T20:02:00.564318+0000 osd.5 (osd.5) 199 : cluster 3 failed to encode map e279 with expected crc 2023-09-23T20:02:01.836 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:01 smithi016 bash[77995]: cluster 2023-09-23T20:02:00.663765+0000 mgr.y (mgr.25325) 1644 : cluster 0 pgmap v1685: 105 pgs: 1 creating+activating, 1 creating+peering, 13 unknown, 90 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 1.3 MiB/s rd, 688 KiB/s wr, 19 op/s 2023-09-23T20:02:01.851 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:01.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:01.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:01.955 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:01.983 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:01.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:01.986 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:02.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:01 smithi092 bash[58855]: cluster 2023-09-23T20:02:00.558532+0000 mon.a (mon.0) 1980 : cluster 0 osdmap e279: 8 total, 8 up, 8 in 2023-09-23T20:02:02.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:01 smithi092 bash[58855]: cluster 2023-09-23T20:02:00.564318+0000 osd.5 (osd.5) 199 : cluster 3 failed to encode map e279 with expected crc 2023-09-23T20:02:02.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:01 smithi092 bash[58855]: cluster 2023-09-23T20:02:00.663765+0000 mgr.y (mgr.25325) 1644 : cluster 0 pgmap v1685: 105 pgs: 1 creating+activating, 1 creating+peering, 13 unknown, 90 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 1.3 MiB/s rd, 688 KiB/s wr, 19 op/s 2023-09-23T20:02:02.087 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:02.115 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:02.116 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:02:02.116 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:02:02.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:02.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:02.219 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:02.243 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:02.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:02.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:02.346 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:02.376 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:02.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:02.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:02.479 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:02.507 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:02.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:02.510 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:02.611 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:02.638 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:02.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:02.641 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:02.741 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:02.767 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:02.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:02.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:02.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:02 smithi092 bash[58855]: cluster 2023-09-23T20:02:01.245374+0000 osd.4 (osd.4) 220 : cluster 3 failed to encode map e279 with expected crc 2023-09-23T20:02:02.870 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:02.879 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:02 smithi016 bash[77995]: cluster 2023-09-23T20:02:01.245374+0000 osd.4 (osd.4) 220 : cluster 3 failed to encode map e279 with expected crc 2023-09-23T20:02:02.879 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:02 smithi016 bash[75004]: cluster 2023-09-23T20:02:01.245374+0000 osd.4 (osd.4) 220 : cluster 3 failed to encode map e279 with expected crc 2023-09-23T20:02:02.894 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:02.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:02.896 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:02.997 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:03.029 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:03.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:03.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:03.133 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:03.162 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:03.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:03.164 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:03.265 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:03.289 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:03.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:03.291 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:03.393 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:03.426 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:03.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:03.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:03.530 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:03.554 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:03.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:03.556 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:03.657 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:03.683 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:03.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:03.685 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:03.787 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:03.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:03 smithi092 bash[58855]: cluster 2023-09-23T20:02:01.986633+0000 osd.6 (osd.6) 211 : cluster 3 failed to encode map e279 with expected crc 2023-09-23T20:02:03.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:03 smithi092 bash[58855]: cluster 2023-09-23T20:02:02.664441+0000 mgr.y (mgr.25325) 1645 : cluster 0 pgmap v1686: 105 pgs: 1 creating+activating, 1 creating+peering, 8 unknown, 95 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 1.4 MiB/s rd, 691 KiB/s wr, 45 op/s 2023-09-23T20:02:03.797 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:03 smithi016 bash[77995]: cluster 2023-09-23T20:02:01.986633+0000 osd.6 (osd.6) 211 : cluster 3 failed to encode map e279 with expected crc 2023-09-23T20:02:03.798 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:03 smithi016 bash[77995]: cluster 2023-09-23T20:02:02.664441+0000 mgr.y (mgr.25325) 1645 : cluster 0 pgmap v1686: 105 pgs: 1 creating+activating, 1 creating+peering, 8 unknown, 95 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 1.4 MiB/s rd, 691 KiB/s wr, 45 op/s 2023-09-23T20:02:03.798 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:03 smithi016 bash[75004]: cluster 2023-09-23T20:02:01.986633+0000 osd.6 (osd.6) 211 : cluster 3 failed to encode map e279 with expected crc 2023-09-23T20:02:03.798 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:03 smithi016 bash[75004]: cluster 2023-09-23T20:02:02.664441+0000 mgr.y (mgr.25325) 1645 : cluster 0 pgmap v1686: 105 pgs: 1 creating+activating, 1 creating+peering, 8 unknown, 95 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 1.4 MiB/s rd, 691 KiB/s wr, 45 op/s 2023-09-23T20:02:03.813 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:03.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:03.817 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:03.917 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:03.947 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:03.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:03.949 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:04.050 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:04.078 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:04.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:04.081 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:04.149 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:02:04.151 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:02:04.181 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:04.217 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:04.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:04.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:04.321 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:04.353 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:04.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:04.355 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:04.456 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:04.483 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:04.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:04.485 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:04.586 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:04.616 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:04.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:04.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:04.720 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:04.747 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:04.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:04.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:04.850 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:04.879 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:04.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:04.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:04.983 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:05.006 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:05.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:05.009 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:05.110 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:05.134 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.mirror_image_status (4604 ms) 2023-09-23T20:02:05.134 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.mirror_image_map 2023-09-23T20:02:05.139 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:05.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:05.141 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:05.242 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:05.274 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:05.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:05.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:05.279 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.mirror_image_map (144 ms) 2023-09-23T20:02:05.279 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.mirror_instances 2023-09-23T20:02:05.292 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.mirror_instances (14 ms) 2023-09-23T20:02:05.293 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2023-09-23T20:02:05.310 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.mirror_snapshot (18 ms) 2023-09-23T20:02:05.310 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_dir_list 2023-09-23T20:02:05.316 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_dir_list (6 ms) 2023-09-23T20:02:05.316 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_dir_add 2023-09-23T20:02:05.321 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_dir_add (5 ms) 2023-09-23T20:02:05.321 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2023-09-23T20:02:05.327 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.dir_add_already_existing (5 ms) 2023-09-23T20:02:05.327 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_dir_rename 2023-09-23T20:02:05.338 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_dir_rename (11 ms) 2023-09-23T20:02:05.338 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_dir_remove 2023-09-23T20:02:05.344 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_dir_remove (6 ms) 2023-09-23T20:02:05.344 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2023-09-23T20:02:05.352 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (8 ms) 2023-09-23T20:02:05.352 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_image_add 2023-09-23T20:02:05.356 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_image_add (4 ms) 2023-09-23T20:02:05.357 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_image_remove 2023-09-23T20:02:05.362 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_image_remove (5 ms) 2023-09-23T20:02:05.362 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_image_list 2023-09-23T20:02:05.366 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_image_list (4 ms) 2023-09-23T20:02:05.366 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_image_clean 2023-09-23T20:02:05.374 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_image_clean (8 ms) 2023-09-23T20:02:05.374 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.image_group_add 2023-09-23T20:02:05.378 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:05.379 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.image_group_add (5 ms) 2023-09-23T20:02:05.379 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.image_group_remove 2023-09-23T20:02:05.385 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.image_group_remove (6 ms) 2023-09-23T20:02:05.385 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.image_group_get 2023-09-23T20:02:05.388 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.image_group_get (4 ms) 2023-09-23T20:02:05.388 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2023-09-23T20:02:05.391 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (3 ms) 2023-09-23T20:02:05.391 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2023-09-23T20:02:05.394 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (3 ms) 2023-09-23T20:02:05.394 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2023-09-23T20:02:05.399 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (4 ms) 2023-09-23T20:02:05.399 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2023-09-23T20:02:05.404 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (5 ms) 2023-09-23T20:02:05.404 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_snap_set 2023-09-23T20:02:05.407 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:05.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:05.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:05.411 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_snap_set (7 ms) 2023-09-23T20:02:05.411 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_snap_list 2023-09-23T20:02:05.416 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_snap_list (5 ms) 2023-09-23T20:02:05.416 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2023-09-23T20:02:05.439 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (24 ms) 2023-09-23T20:02:05.439 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2023-09-23T20:02:05.512 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:05.539 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:05.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:05.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:05.643 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:05.672 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:05.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:05.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:05.775 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:05.803 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:05.804 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (365 ms) 2023-09-23T20:02:05.805 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_snap_remove 2023-09-23T20:02:05.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:05.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:05.810 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_snap_remove (6 ms) 2023-09-23T20:02:05.810 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2023-09-23T20:02:05.814 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (4 ms) 2023-09-23T20:02:05.814 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.trash_methods 2023-09-23T20:02:05.827 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.trash_methods (12 ms) 2023-09-23T20:02:05.827 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.op_features 2023-09-23T20:02:05.838 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.op_features (12 ms) 2023-09-23T20:02:05.838 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.clone_parent 2023-09-23T20:02:05.868 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.clone_parent (29 ms) 2023-09-23T20:02:05.868 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2023-09-23T20:02:05.885 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.clone_parent_ns (18 ms) 2023-09-23T20:02:05.886 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.clone_child 2023-09-23T20:02:05.905 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.clone_child (20 ms) 2023-09-23T20:02:05.906 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.namespace_methods 2023-09-23T20:02:05.907 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:05.917 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:02:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:20:02:05] "GET /metrics HTTP/1.1" 200 36323 "" "Prometheus/2.33.4" 2023-09-23T20:02:05.918 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:05 smithi016 bash[77995]: cluster 2023-09-23T20:02:04.665371+0000 mgr.y (mgr.25325) 1646 : cluster 0 pgmap v1687: 105 pgs: 105 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.0 MiB/s wr, 997 op/s 2023-09-23T20:02:05.918 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:05 smithi016 bash[75004]: cluster 2023-09-23T20:02:04.665371+0000 mgr.y (mgr.25325) 1646 : cluster 0 pgmap v1687: 105 pgs: 105 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.0 MiB/s wr, 997 op/s 2023-09-23T20:02:05.919 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.namespace_methods (12 ms) 2023-09-23T20:02:05.919 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.migration 2023-09-23T20:02:05.933 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.migration (16 ms) 2023-09-23T20:02:05.933 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.migration_v1 2023-09-23T20:02:05.936 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:05.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:05.939 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:05.948 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.migration_v1 (15 ms) 2023-09-23T20:02:05.949 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2023-09-23T20:02:06.040 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:06.080 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:06.082 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:05 smithi092 bash[58855]: cluster 2023-09-23T20:02:04.665371+0000 mgr.y (mgr.25325) 1646 : cluster 0 pgmap v1687: 105 pgs: 105 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.0 MiB/s wr, 997 op/s 2023-09-23T20:02:06.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:06.083 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:06.184 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:06.210 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:06.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:06.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:06.314 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:06.346 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:06.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:06.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:06.449 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:06.478 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:06.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:06.481 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:06.582 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:06.618 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:06.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:06.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:06.721 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:06.755 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:06.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:06.758 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:06.859 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:06.880 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:06.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:06.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:06.983 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:07.019 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:07.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:07.022 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:07.123 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:07.155 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:07.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:07.158 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:07.259 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:07.292 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:07.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:07.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:07.396 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:07.428 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:07.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:07.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:07.532 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:07.559 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:07.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:07.561 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:07.662 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:07.686 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:07.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:07.689 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:07.789 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:07.819 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:07.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:07.822 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:07.845 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1896 ms) 2023-09-23T20:02:07.845 INFO:tasks.workunit.client.0.smithi016.stdout:[ RUN ] TestClsRbd.sparsify 2023-09-23T20:02:07.923 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:07.951 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:07.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:07.953 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:07.968 INFO:tasks.workunit.client.0.smithi016.stdout:[ OK ] TestClsRbd.sparsify (124 ms) 2023-09-23T20:02:08.054 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:08.065 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:07 smithi016 bash[75004]: cluster 2023-09-23T20:02:06.666499+0000 mgr.y (mgr.25325) 1647 : cluster 0 pgmap v1688: 105 pgs: 105 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 1.5 MiB/s rd, 814 KiB/s wr, 807 op/s 2023-09-23T20:02:08.066 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:07 smithi016 bash[75004]: cluster 2023-09-23T20:02:06.839445+0000 mon.a (mon.0) 1981 : cluster 0 osdmap e280: 8 total, 8 up, 8 in 2023-09-23T20:02:08.066 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:07 smithi016 bash[75004]: cluster 2023-09-23T20:02:06.923881+0000 osd.6 (osd.6) 212 : cluster 3 failed to encode map e280 with expected crc 2023-09-23T20:02:08.066 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:07 smithi016 bash[75004]: cluster 2023-09-23T20:02:06.926764+0000 osd.6 (osd.6) 213 : cluster 3 failed to encode map e280 with expected crc 2023-09-23T20:02:08.066 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:07 smithi016 bash[75004]: cluster 2023-09-23T20:02:06.929596+0000 osd.6 (osd.6) 214 : cluster 3 failed to encode map e280 with expected crc 2023-09-23T20:02:08.066 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:07 smithi016 bash[75004]: cluster 2023-09-23T20:02:07.607206+0000 osd.4 (osd.4) 221 : cluster 3 failed to encode map e280 with expected crc 2023-09-23T20:02:08.066 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:07 smithi016 bash[75004]: cluster 2023-09-23T20:02:07.607345+0000 osd.5 (osd.5) 200 : cluster 3 failed to encode map e280 with expected crc 2023-09-23T20:02:08.067 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:07 smithi016 bash[77995]: cluster 2023-09-23T20:02:06.666499+0000 mgr.y (mgr.25325) 1647 : cluster 0 pgmap v1688: 105 pgs: 105 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 1.5 MiB/s rd, 814 KiB/s wr, 807 op/s 2023-09-23T20:02:08.067 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:07 smithi016 bash[77995]: cluster 2023-09-23T20:02:06.839445+0000 mon.a (mon.0) 1981 : cluster 0 osdmap e280: 8 total, 8 up, 8 in 2023-09-23T20:02:08.067 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:07 smithi016 bash[77995]: cluster 2023-09-23T20:02:06.923881+0000 osd.6 (osd.6) 212 : cluster 3 failed to encode map e280 with expected crc 2023-09-23T20:02:08.067 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:07 smithi016 bash[77995]: cluster 2023-09-23T20:02:06.926764+0000 osd.6 (osd.6) 213 : cluster 3 failed to encode map e280 with expected crc 2023-09-23T20:02:08.067 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:07 smithi016 bash[77995]: cluster 2023-09-23T20:02:06.929596+0000 osd.6 (osd.6) 214 : cluster 3 failed to encode map e280 with expected crc 2023-09-23T20:02:08.067 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:07 smithi016 bash[77995]: cluster 2023-09-23T20:02:07.607206+0000 osd.4 (osd.4) 221 : cluster 3 failed to encode map e280 with expected crc 2023-09-23T20:02:08.067 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:07 smithi016 bash[77995]: cluster 2023-09-23T20:02:07.607345+0000 osd.5 (osd.5) 200 : cluster 3 failed to encode map e280 with expected crc 2023-09-23T20:02:08.086 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:08.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:08.088 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:08.189 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:08.201 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:07 smithi092 bash[58855]: cluster 2023-09-23T20:02:06.666499+0000 mgr.y (mgr.25325) 1647 : cluster 0 pgmap v1688: 105 pgs: 105 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 1.5 MiB/s rd, 814 KiB/s wr, 807 op/s 2023-09-23T20:02:08.201 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:07 smithi092 bash[58855]: cluster 2023-09-23T20:02:06.839445+0000 mon.a (mon.0) 1981 : cluster 0 osdmap e280: 8 total, 8 up, 8 in 2023-09-23T20:02:08.201 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:07 smithi092 bash[58855]: cluster 2023-09-23T20:02:06.923881+0000 osd.6 (osd.6) 212 : cluster 3 failed to encode map e280 with expected crc 2023-09-23T20:02:08.201 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:07 smithi092 bash[58855]: cluster 2023-09-23T20:02:06.926764+0000 osd.6 (osd.6) 213 : cluster 3 failed to encode map e280 with expected crc 2023-09-23T20:02:08.201 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:07 smithi092 bash[58855]: cluster 2023-09-23T20:02:06.929596+0000 osd.6 (osd.6) 214 : cluster 3 failed to encode map e280 with expected crc 2023-09-23T20:02:08.201 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:07 smithi092 bash[58855]: cluster 2023-09-23T20:02:07.607206+0000 osd.4 (osd.4) 221 : cluster 3 failed to encode map e280 with expected crc 2023-09-23T20:02:08.202 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:07 smithi092 bash[58855]: cluster 2023-09-23T20:02:07.607345+0000 osd.5 (osd.5) 200 : cluster 3 failed to encode map e280 with expected crc 2023-09-23T20:02:08.218 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:08.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:08.222 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:08.322 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:08.357 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:08.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:08.360 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:08.460 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:08.492 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:08.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:08.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:08.596 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:08.631 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:08.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:08.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:08.735 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:08.766 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:08.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:08.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:08.864 INFO:tasks.workunit.client.0.smithi016.stdout:[----------] 70 tests from TestClsRbd (8408 ms total) 2023-09-23T20:02:08.864 INFO:tasks.workunit.client.0.smithi016.stdout: 2023-09-23T20:02:08.864 INFO:tasks.workunit.client.0.smithi016.stdout:[----------] Global test environment tear-down 2023-09-23T20:02:08.865 INFO:tasks.workunit.client.0.smithi016.stdout:[==========] 70 tests from 1 test suite ran. (10858 ms total) 2023-09-23T20:02:08.865 INFO:tasks.workunit.client.0.smithi016.stdout:[ PASSED ] 70 tests. 2023-09-23T20:02:08.867 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-23T20:02:08.868 DEBUG:teuthology.orchestra.run.smithi016:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-23T20:02:08.870 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:08.878 INFO:tasks.workunit:Stopping ['cls/test_cls_rbd.sh'] on client.0... 2023-09-23T20:02:08.878 DEBUG:teuthology.orchestra.run.smithi016:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-09-23T20:02:08.942 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:08.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:08.945 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:09.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:08 smithi092 bash[58855]: cluster 2023-09-23T20:02:07.283932+0000 osd.7 (osd.7) 194 : cluster 3 failed to encode map e280 with expected crc 2023-09-23T20:02:09.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:08 smithi092 bash[58855]: cluster 2023-09-23T20:02:07.286466+0000 osd.7 (osd.7) 195 : cluster 3 failed to encode map e280 with expected crc 2023-09-23T20:02:09.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:08 smithi092 bash[58855]: cluster 2023-09-23T20:02:07.841175+0000 mon.a (mon.0) 1982 : cluster 0 osdmap e281: 8 total, 8 up, 8 in 2023-09-23T20:02:09.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:08 smithi092 bash[58855]: cluster 2023-09-23T20:02:07.843109+0000 osd.5 (osd.5) 201 : cluster 3 failed to encode map e281 with expected crc 2023-09-23T20:02:09.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:08 smithi092 bash[58855]: cluster 2023-09-23T20:02:07.849059+0000 osd.6 (osd.6) 215 : cluster 3 failed to encode map e281 with expected crc 2023-09-23T20:02:09.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:08 smithi092 bash[58855]: cluster 2023-09-23T20:02:08.337471+0000 osd.4 (osd.4) 222 : cluster 3 failed to encode map e281 with expected crc 2023-09-23T20:02:09.046 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:09.070 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:09.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:09.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:09.154 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:08 smithi016 bash[75004]: cluster 2023-09-23T20:02:07.283932+0000 osd.7 (osd.7) 194 : cluster 3 failed to encode map e280 with expected crc 2023-09-23T20:02:09.154 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:08 smithi016 bash[75004]: cluster 2023-09-23T20:02:07.286466+0000 osd.7 (osd.7) 195 : cluster 3 failed to encode map e280 with expected crc 2023-09-23T20:02:09.154 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:08 smithi016 bash[75004]: cluster 2023-09-23T20:02:07.841175+0000 mon.a (mon.0) 1982 : cluster 0 osdmap e281: 8 total, 8 up, 8 in 2023-09-23T20:02:09.154 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:08 smithi016 bash[75004]: cluster 2023-09-23T20:02:07.843109+0000 osd.5 (osd.5) 201 : cluster 3 failed to encode map e281 with expected crc 2023-09-23T20:02:09.154 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:08 smithi016 bash[75004]: cluster 2023-09-23T20:02:07.849059+0000 osd.6 (osd.6) 215 : cluster 3 failed to encode map e281 with expected crc 2023-09-23T20:02:09.154 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:08 smithi016 bash[75004]: cluster 2023-09-23T20:02:08.337471+0000 osd.4 (osd.4) 222 : cluster 3 failed to encode map e281 with expected crc 2023-09-23T20:02:09.155 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:08 smithi016 bash[77995]: cluster 2023-09-23T20:02:07.283932+0000 osd.7 (osd.7) 194 : cluster 3 failed to encode map e280 with expected crc 2023-09-23T20:02:09.155 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:08 smithi016 bash[77995]: cluster 2023-09-23T20:02:07.286466+0000 osd.7 (osd.7) 195 : cluster 3 failed to encode map e280 with expected crc 2023-09-23T20:02:09.155 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:08 smithi016 bash[77995]: cluster 2023-09-23T20:02:07.841175+0000 mon.a (mon.0) 1982 : cluster 0 osdmap e281: 8 total, 8 up, 8 in 2023-09-23T20:02:09.155 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:08 smithi016 bash[77995]: cluster 2023-09-23T20:02:07.843109+0000 osd.5 (osd.5) 201 : cluster 3 failed to encode map e281 with expected crc 2023-09-23T20:02:09.155 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:08 smithi016 bash[77995]: cluster 2023-09-23T20:02:07.849059+0000 osd.6 (osd.6) 215 : cluster 3 failed to encode map e281 with expected crc 2023-09-23T20:02:09.155 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:08 smithi016 bash[77995]: cluster 2023-09-23T20:02:08.337471+0000 osd.4 (osd.4) 222 : cluster 3 failed to encode map e281 with expected crc 2023-09-23T20:02:09.156 DEBUG:teuthology.parallel:result is None 2023-09-23T20:02:09.156 DEBUG:teuthology.orchestra.run.smithi016:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T20:02:09.168 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T20:02:09.169 DEBUG:teuthology.orchestra.run.smithi016:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T20:02:09.173 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:09.204 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:09.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:09.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:09.282 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T20:02:09.282 INFO:teuthology.task.sequential:In sequential, running task print... 2023-09-23T20:02:09.282 INFO:teuthology.task.print:**** done cls/test_cls_rbd.sh 5-workload 2023-09-23T20:02:09.282 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-09-23T20:02:09.283 INFO:tasks.workunit:Pulling workunits from ref pacific 2023-09-23T20:02:09.283 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-09-23T20:02:09.284 DEBUG:teuthology.orchestra.run.smithi016:> stat -- /home/ubuntu/cephtest/mnt.0 2023-09-23T20:02:09.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:09.289 INFO:teuthology.orchestra.run.smithi016.stderr:stat: cannot stat '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-09-23T20:02:09.289 DEBUG:teuthology.orchestra.run.smithi016:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T20:02:09.309 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:09.336 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-09-23T20:02:09.336 DEBUG:teuthology.orchestra.run.smithi016:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-09-23T20:02:09.344 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:09.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:09.347 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:09.386 INFO:tasks.workunit:timeout=3h 2023-09-23T20:02:09.386 INFO:tasks.workunit:cleanup=True 2023-09-23T20:02:09.387 DEBUG:teuthology.orchestra.run.smithi016:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch pacific https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-09-23T20:02:09.447 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:09.477 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:09.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:09.480 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:09.482 INFO:tasks.workunit.client.0.smithi016.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-09-23T20:02:09.582 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:09.611 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:09.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:09.614 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:09.715 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:09.743 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:09.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:09.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:09.846 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:09.872 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:09.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:09.875 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:09.976 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:09.986 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:09 smithi016 bash[77995]: cluster 2023-09-23T20:02:08.255612+0000 osd.7 (osd.7) 196 : cluster 3 failed to encode map e281 with expected crc 2023-09-23T20:02:09.987 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:09 smithi016 bash[77995]: cluster 2023-09-23T20:02:08.667213+0000 mgr.y (mgr.25325) 1648 : cluster 0 pgmap v1691: 105 pgs: 105 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 1.1 MiB/s rd, 551 KiB/s wr, 1.74k op/s 2023-09-23T20:02:09.987 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:09 smithi016 bash[77995]: cluster 2023-09-23T20:02:08.859002+0000 mon.a (mon.0) 1983 : cluster 0 osdmap e282: 8 total, 8 up, 8 in 2023-09-23T20:02:09.987 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:09 smithi016 bash[77995]: cluster 2023-09-23T20:02:09.115056+0000 osd.4 (osd.4) 223 : cluster 3 failed to encode map e282 with expected crc 2023-09-23T20:02:09.987 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:09 smithi016 bash[77995]: cluster 2023-09-23T20:02:09.117493+0000 osd.4 (osd.4) 224 : cluster 3 failed to encode map e282 with expected crc 2023-09-23T20:02:09.987 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:09 smithi016 bash[77995]: cluster 2023-09-23T20:02:09.205339+0000 osd.5 (osd.5) 202 : cluster 3 failed to encode map e282 with expected crc 2023-09-23T20:02:09.987 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:09 smithi016 bash[77995]: audit 2023-09-23T20:02:09.394495+0000 mon.a (mon.0) 1984 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:02:09.987 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:09 smithi016 bash[75004]: cluster 2023-09-23T20:02:08.255612+0000 osd.7 (osd.7) 196 : cluster 3 failed to encode map e281 with expected crc 2023-09-23T20:02:09.987 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:09 smithi016 bash[75004]: cluster 2023-09-23T20:02:08.667213+0000 mgr.y (mgr.25325) 1648 : cluster 0 pgmap v1691: 105 pgs: 105 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 1.1 MiB/s rd, 551 KiB/s wr, 1.74k op/s 2023-09-23T20:02:09.987 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:09 smithi016 bash[75004]: cluster 2023-09-23T20:02:08.859002+0000 mon.a (mon.0) 1983 : cluster 0 osdmap e282: 8 total, 8 up, 8 in 2023-09-23T20:02:09.988 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:09 smithi016 bash[75004]: cluster 2023-09-23T20:02:09.115056+0000 osd.4 (osd.4) 223 : cluster 3 failed to encode map e282 with expected crc 2023-09-23T20:02:09.988 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:09 smithi016 bash[75004]: cluster 2023-09-23T20:02:09.117493+0000 osd.4 (osd.4) 224 : cluster 3 failed to encode map e282 with expected crc 2023-09-23T20:02:09.988 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:09 smithi016 bash[75004]: cluster 2023-09-23T20:02:09.205339+0000 osd.5 (osd.5) 202 : cluster 3 failed to encode map e282 with expected crc 2023-09-23T20:02:09.988 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:09 smithi016 bash[75004]: audit 2023-09-23T20:02:09.394495+0000 mon.a (mon.0) 1984 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:02:10.005 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:10.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:10.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:10.108 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:10.120 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:09 smithi092 bash[58855]: cluster 2023-09-23T20:02:08.255612+0000 osd.7 (osd.7) 196 : cluster 3 failed to encode map e281 with expected crc 2023-09-23T20:02:10.121 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:09 smithi092 bash[58855]: cluster 2023-09-23T20:02:08.667213+0000 mgr.y (mgr.25325) 1648 : cluster 0 pgmap v1691: 105 pgs: 105 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 1.1 MiB/s rd, 551 KiB/s wr, 1.74k op/s 2023-09-23T20:02:10.121 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:09 smithi092 bash[58855]: cluster 2023-09-23T20:02:08.859002+0000 mon.a (mon.0) 1983 : cluster 0 osdmap e282: 8 total, 8 up, 8 in 2023-09-23T20:02:10.121 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:09 smithi092 bash[58855]: cluster 2023-09-23T20:02:09.115056+0000 osd.4 (osd.4) 223 : cluster 3 failed to encode map e282 with expected crc 2023-09-23T20:02:10.121 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:09 smithi092 bash[58855]: cluster 2023-09-23T20:02:09.117493+0000 osd.4 (osd.4) 224 : cluster 3 failed to encode map e282 with expected crc 2023-09-23T20:02:10.121 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:09 smithi092 bash[58855]: cluster 2023-09-23T20:02:09.205339+0000 osd.5 (osd.5) 202 : cluster 3 failed to encode map e282 with expected crc 2023-09-23T20:02:10.121 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:09 smithi092 bash[58855]: audit 2023-09-23T20:02:09.394495+0000 mon.a (mon.0) 1984 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:02:10.141 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:10.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:10.143 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:10.244 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:10.277 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:10.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:10.280 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:10.381 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:10.411 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:10.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:10.414 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:10.515 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:10.550 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:10.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:10.553 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:10.654 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:10.686 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:10.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:10.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:10.790 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:10.816 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:10.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:10.818 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:10.919 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:10.943 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:10.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:10.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:11.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:10 smithi092 bash[58855]: cluster 2023-09-23T20:02:09.205728+0000 osd.7 (osd.7) 197 : cluster 3 failed to encode map e282 with expected crc 2023-09-23T20:02:11.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:10 smithi092 bash[58855]: cluster 2023-09-23T20:02:09.205803+0000 osd.6 (osd.6) 216 : cluster 3 failed to encode map e282 with expected crc 2023-09-23T20:02:11.048 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:11.081 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:11.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:11.084 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:11.185 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:11.223 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:11.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:11.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:11.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:11 smithi016 bash[75004]: cluster 2023-09-23T20:02:09.205728+0000 osd.7 (osd.7) 197 : cluster 3 failed to encode map e282 with expected crc 2023-09-23T20:02:11.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:11 smithi016 bash[75004]: cluster 2023-09-23T20:02:09.205803+0000 osd.6 (osd.6) 216 : cluster 3 failed to encode map e282 with expected crc 2023-09-23T20:02:11.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:11 smithi016 bash[77995]: cluster 2023-09-23T20:02:09.205728+0000 osd.7 (osd.7) 197 : cluster 3 failed to encode map e282 with expected crc 2023-09-23T20:02:11.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:11 smithi016 bash[77995]: cluster 2023-09-23T20:02:09.205803+0000 osd.6 (osd.6) 216 : cluster 3 failed to encode map e282 with expected crc 2023-09-23T20:02:11.327 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:11.349 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:11.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:11.351 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:11.452 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:11.474 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:11.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:11.477 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:11.578 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:11.611 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:11.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:11.614 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:11.715 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:11.745 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:11.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:11.748 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:11.848 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:11.870 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:11.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:11.876 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:11.977 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:11.984 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:11 smithi016 bash[75004]: cluster 2023-09-23T20:02:10.668116+0000 mgr.y (mgr.25325) 1649 : cluster 0 pgmap v1693: 73 pgs: 73 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:02:11.984 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:11 smithi016 bash[77995]: cluster 2023-09-23T20:02:10.668116+0000 mgr.y (mgr.25325) 1649 : cluster 0 pgmap v1693: 73 pgs: 73 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:02:12.000 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:12.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:12.002 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:12.103 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:12.136 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:12.137 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:11 smithi092 bash[58855]: cluster 2023-09-23T20:02:10.668116+0000 mgr.y (mgr.25325) 1649 : cluster 0 pgmap v1693: 73 pgs: 73 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:02:12.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:12.138 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:12.239 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:12.260 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:12.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:12.262 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:12.363 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:12.393 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:12.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:12.396 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:12.496 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:12.560 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:12.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:12.563 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:12.663 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:12.686 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:12.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:12.689 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:12.790 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:12.815 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:12.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:12.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:12.919 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:12.948 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:12.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:12.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:13.053 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:13.071 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:13.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:13.073 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:13.173 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:13.206 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:13.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:13.209 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:13.309 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:13.331 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:13.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:13.334 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:13.434 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:13.461 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:13.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:13.464 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:13.565 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:13.591 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:13.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:13.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:13.694 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:13.725 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:13.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:13.728 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:13.829 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:13.863 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:13.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:13.866 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:13.874 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:13 smithi016 bash[75004]: cluster 2023-09-23T20:02:12.668731+0000 mgr.y (mgr.25325) 1650 : cluster 0 pgmap v1694: 73 pgs: 73 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:02:13.874 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:13 smithi016 bash[75004]: audit 2023-09-23T20:02:13.699794+0000 mon.a (mon.0) 1985 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:02:13.966 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:13.977 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:13 smithi092 bash[58855]: cluster 2023-09-23T20:02:12.668731+0000 mgr.y (mgr.25325) 1650 : cluster 0 pgmap v1694: 73 pgs: 73 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:02:13.978 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:13 smithi092 bash[58855]: audit 2023-09-23T20:02:13.699794+0000 mon.a (mon.0) 1985 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:02:13.996 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:13.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:13.999 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:14.100 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:14.122 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:14.124 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:13 smithi016 bash[77995]: cluster 2023-09-23T20:02:12.668731+0000 mgr.y (mgr.25325) 1650 : cluster 0 pgmap v1694: 73 pgs: 73 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:02:14.124 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:13 smithi016 bash[77995]: audit 2023-09-23T20:02:13.699794+0000 mon.a (mon.0) 1985 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:02:14.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:14.125 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:14.225 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:14.255 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:14.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:14.258 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:14.359 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:14.391 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:14.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:14.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:14.495 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:14.526 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:14.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:14.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:14.629 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:14.659 DEBUG:teuthology.orchestra.run.smithi016:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-09-23T20:02:14.662 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:14.664 INFO:tasks.workunit.client.0.smithi016.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-09-23T20:02:14.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:14.665 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:14.667 INFO:tasks.workunit.client.0.smithi016.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T20:02:14.667 INFO:tasks.workunit.client.0.smithi016.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-09-23T20:02:14.763 INFO:tasks.workunit.client.0.smithi016.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-09-23T20:02:14.766 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:14.794 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:14.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:14.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:14.886 INFO:tasks.workunit.client.0.smithi016.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-09-23T20:02:14.898 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:14.918 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:14.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:14.921 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:15.004 INFO:tasks.workunit.client.0.smithi016.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T20:02:15.005 INFO:tasks.workunit.client.0.smithi016.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T20:02:15.005 INFO:tasks.workunit.client.0.smithi016.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-09-23T20:02:15.021 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:15.047 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:15.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:15.049 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:15.093 INFO:tasks.workunit.client.0.smithi016.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T20:02:15.099 DEBUG:teuthology.orchestra.run.smithi016:> set -ex 2023-09-23T20:02:15.100 DEBUG:teuthology.orchestra.run.smithi016:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-09-23T20:02:15.106 INFO:tasks.workunit:Running workunits matching rbd/import_export.sh on client.0... 2023-09-23T20:02:15.107 INFO:tasks.workunit:Running workunit rbd/import_export.sh... 2023-09-23T20:02:15.107 DEBUG:teuthology.orchestra.run.smithi016:workunit test rbd/import_export.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=pacific TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_CREATE_ARGS=--new-format adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/import_export.sh 2023-09-23T20:02:15.150 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:15.166 INFO:tasks.workunit.client.0.smithi016.stderr:+ export RBD_FORCE_ALLOW_V1=1 2023-09-23T20:02:15.166 INFO:tasks.workunit.client.0.smithi016.stderr:+ TMPDIR=/tmp/rbd_import_export_249364 2023-09-23T20:02:15.167 INFO:tasks.workunit.client.0.smithi016.stderr:+ rm -rf /tmp/rbd_import_export_249364 2023-09-23T20:02:15.167 INFO:tasks.workunit.client.0.smithi016.stderr:+ mkdir /tmp/rbd_import_export_249364 2023-09-23T20:02:15.168 INFO:tasks.workunit.client.0.smithi016.stderr:+ trap rm -rf /tmp/rbd_import_export_249364 INT TERM EXIT 2023-09-23T20:02:15.168 INFO:tasks.workunit.client.0.smithi016.stderr:+ mkdir foo.249364 2023-09-23T20:02:15.169 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd import foo.249364 foo.dir 2023-09-23T20:02:15.183 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:15.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:15.185 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:15.286 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:15.317 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:15.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:15.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:15.421 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:15.454 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:15.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:15.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:15.558 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:15.584 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:15.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:15.587 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:15.688 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:15.728 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:15.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:15.730 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:15.786 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:02:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:20:02:15] "GET /metrics HTTP/1.1" 200 33960 "" "Prometheus/2.33.4" 2023-09-23T20:02:15.831 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:15.866 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:15.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:15.868 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:15.969 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:15.998 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:16.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:16.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:16.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:15 smithi092 bash[58855]: cluster 2023-09-23T20:02:14.669465+0000 mgr.y (mgr.25325) 1651 : cluster 0 pgmap v1695: 73 pgs: 73 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:02:16.101 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:16.136 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:16.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:16.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:16.240 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:16.280 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:16.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:16.283 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:16.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:15 smithi016 bash[75004]: cluster 2023-09-23T20:02:14.669465+0000 mgr.y (mgr.25325) 1651 : cluster 0 pgmap v1695: 73 pgs: 73 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:02:16.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:15 smithi016 bash[77995]: cluster 2023-09-23T20:02:14.669465+0000 mgr.y (mgr.25325) 1651 : cluster 0 pgmap v1695: 73 pgs: 73 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:02:16.384 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:16.419 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:16.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:16.422 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:16.522 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:16.557 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:16.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:16.560 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:16.661 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:16.702 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:16.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:16.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:16.807 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:16.842 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:16.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:16.845 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:16.946 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:16.988 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:16.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:16.992 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:17.092 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:17.126 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:17.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:17.128 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:17.229 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:17.269 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:17.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:17.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:17.373 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:17.398 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:17.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:17.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:17.502 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:17.526 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:17.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:17.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:17.630 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:17.655 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:17.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:17.658 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:17.759 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:17.781 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:17.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:17.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:17.885 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:17.916 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:17.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:17.920 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:18.020 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:18.033 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:17 smithi016 bash[75004]: cluster 2023-09-23T20:02:16.670375+0000 mgr.y (mgr.25325) 1652 : cluster 0 pgmap v1696: 73 pgs: 73 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:02:18.033 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:17 smithi016 bash[77995]: cluster 2023-09-23T20:02:16.670375+0000 mgr.y (mgr.25325) 1652 : cluster 0 pgmap v1696: 73 pgs: 73 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:02:18.051 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:18.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:18.054 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:18.155 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:18.184 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:18.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:18.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:18.288 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:18.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:17 smithi092 bash[58855]: cluster 2023-09-23T20:02:16.670375+0000 mgr.y (mgr.25325) 1652 : cluster 0 pgmap v1696: 73 pgs: 73 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:02:18.315 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:18.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:18.318 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:18.419 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:18.449 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:18.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:18.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:18.553 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:18.569 INFO:tasks.workunit.client.0.smithi016.stderr:rbd: cannot import a directory 2023-09-23T20:02:18.569 INFO:tasks.workunit.client.0.smithi016.stderr: Importing image: 0% complete...failed. 2023-09-23T20:02:18.569 INFO:tasks.workunit.client.0.smithi016.stderr:rbd: import failed: (21) Is a directory 2023-09-23T20:02:18.583 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:18.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:18.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:18.680 INFO:tasks.workunit.client.0.smithi016.stderr:+ true 2023-09-23T20:02:18.681 INFO:tasks.workunit.client.0.smithi016.stderr:+ rmdir foo.249364 2023-09-23T20:02:18.682 INFO:tasks.workunit.client.0.smithi016.stderr:+ dd if=/bin/sh of=/tmp/rbd_import_export_249364/img bs=1k count=1 seek=10 2023-09-23T20:02:18.683 INFO:tasks.workunit.client.0.smithi016.stderr:1+0 records in 2023-09-23T20:02:18.684 INFO:tasks.workunit.client.0.smithi016.stderr:1+0 records out 2023-09-23T20:02:18.684 INFO:tasks.workunit.client.0.smithi016.stderr:1024 bytes (1.0 kB, 1.0 KiB) copied, 0.000167023 s, 6.1 MB/s 2023-09-23T20:02:18.684 INFO:tasks.workunit.client.0.smithi016.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_249364/img bs=1k count=10 seek=100 2023-09-23T20:02:18.685 INFO:tasks.workunit.client.0.smithi016.stderr:10+0 records in 2023-09-23T20:02:18.685 INFO:tasks.workunit.client.0.smithi016.stderr:10+0 records out 2023-09-23T20:02:18.685 INFO:tasks.workunit.client.0.smithi016.stderr:10240 bytes (10 kB, 10 KiB) copied, 0.000218982 s, 46.8 MB/s 2023-09-23T20:02:18.685 INFO:tasks.workunit.client.0.smithi016.stderr:+ dd if=/bin/rm of=/tmp/rbd_import_export_249364/img bs=1k count=100 seek=1000 2023-09-23T20:02:18.687 INFO:tasks.workunit.client.0.smithi016.stderr:70+1 records in 2023-09-23T20:02:18.687 INFO:tasks.workunit.client.0.smithi016.stderr:70+1 records out 2023-09-23T20:02:18.687 INFO:tasks.workunit.client.0.smithi016.stderr:72056 bytes (72 kB, 70 KiB) copied, 0.000659947 s, 109 MB/s 2023-09-23T20:02:18.687 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:18.688 INFO:tasks.workunit.client.0.smithi016.stderr:+ dd if=/bin/ls of=/tmp/rbd_import_export_249364/img bs=1k seek=10000 2023-09-23T20:02:18.689 INFO:tasks.workunit.client.0.smithi016.stderr:138+1 records in 2023-09-23T20:02:18.689 INFO:tasks.workunit.client.0.smithi016.stderr:138+1 records out 2023-09-23T20:02:18.689 INFO:tasks.workunit.client.0.smithi016.stderr:142144 bytes (142 kB, 139 KiB) copied, 0.000429422 s, 331 MB/s 2023-09-23T20:02:18.689 INFO:tasks.workunit.client.0.smithi016.stderr:+ dd if=/bin/ln of=/tmp/rbd_import_export_249364/img bs=1k seek=100000 2023-09-23T20:02:18.721 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:18.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:18.724 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:18.728 INFO:tasks.workunit.client.0.smithi016.stderr:74+1 records in 2023-09-23T20:02:18.728 INFO:tasks.workunit.client.0.smithi016.stderr:74+1 records out 2023-09-23T20:02:18.728 INFO:tasks.workunit.client.0.smithi016.stderr:76160 bytes (76 kB, 74 KiB) copied, 0.0382765 s, 2.0 MB/s 2023-09-23T20:02:18.728 INFO:tasks.workunit.client.0.smithi016.stderr:+ dd if=/bin/grep of=/tmp/rbd_import_export_249364/img bs=1k seek=1000000 2023-09-23T20:02:18.805 INFO:tasks.workunit.client.0.smithi016.stderr:194+1 records in 2023-09-23T20:02:18.805 INFO:tasks.workunit.client.0.smithi016.stderr:194+1 records out 2023-09-23T20:02:18.805 INFO:tasks.workunit.client.0.smithi016.stderr:199136 bytes (199 kB, 194 KiB) copied, 0.0762938 s, 2.6 MB/s 2023-09-23T20:02:18.805 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd rm testimg 2023-09-23T20:02:18.825 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:18.857 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:18.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:18.860 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:18.933 INFO:tasks.workunit.client.0.smithi016.stderr: Removing image: 0% complete...failed. 2023-09-23T20:02:18.933 INFO:tasks.workunit.client.0.smithi016.stderr:rbd: delete error: (2) No such file or directory 2023-09-23T20:02:18.947 INFO:tasks.workunit.client.0.smithi016.stderr:+ true 2023-09-23T20:02:18.948 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd import --new-format /tmp/rbd_import_export_249364/img testimg 2023-09-23T20:02:18.961 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:18.993 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:18.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:18.996 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:19.096 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:19.128 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:19.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:19.132 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:19.156 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-09-23T20:02:19.156 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:02:19.232 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:19.268 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:19.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:19.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:19.371 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:19.397 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:19.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:19.400 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:19.501 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:19.533 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:19.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:19.536 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:19.637 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:19.669 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:19.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:19.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:19.677 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T20:02:19.706 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T20:02:19.728 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T20:02:19.753 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T20:02:19.773 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:19.792 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T20:02:19.799 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:19.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:19.801 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:19.822 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T20:02:19.849 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T20:02:19.875 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T20:02:19.878 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:02:19.878 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:02:19.878 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:02:19.878 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:02:19.878 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:02:19.878 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:02:19.878 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:02:19.878 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:02:19.902 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:19.903 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is set 2023-09-23T20:02:19.925 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:19.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:19.928 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:19.943 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-09-23T20:02:19.945 INFO:teuthology.orchestra.run.smithi016.stderr:dumped all 2023-09-23T20:02:20.029 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:20.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:19 smithi016 bash[77995]: cluster 2023-09-23T20:02:18.671150+0000 mgr.y (mgr.25325) 1653 : cluster 0 pgmap v1697: 73 pgs: 73 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:02:20.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:19 smithi016 bash[77995]: audit 2023-09-23T20:02:19.633745+0000 mon.c (mon.2) 247 : audit 0 from='client.? 172.21.15.16:0/2412804755' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:02:20.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:19 smithi016 bash[77995]: audit 2023-09-23T20:02:19.721159+0000 mon.a (mon.0) 1986 : audit 1 from='client.? 172.21.15.16:0/3906191299' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:02:20.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:19 smithi016 bash[75004]: cluster 2023-09-23T20:02:18.671150+0000 mgr.y (mgr.25325) 1653 : cluster 0 pgmap v1697: 73 pgs: 73 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:02:20.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:19 smithi016 bash[75004]: audit 2023-09-23T20:02:19.633745+0000 mon.c (mon.2) 247 : audit 0 from='client.? 172.21.15.16:0/2412804755' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:02:20.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:19 smithi016 bash[75004]: audit 2023-09-23T20:02:19.721159+0000 mon.a (mon.0) 1986 : audit 1 from='client.? 172.21.15.16:0/3906191299' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:02:20.057 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:20.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:20.060 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:20.161 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:20.175 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:19 smithi092 bash[58855]: cluster 2023-09-23T20:02:18.671150+0000 mgr.y (mgr.25325) 1653 : cluster 0 pgmap v1697: 73 pgs: 73 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:02:20.175 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:19 smithi092 bash[58855]: audit 2023-09-23T20:02:19.633745+0000 mon.c (mon.2) 247 : audit 0 from='client.? 172.21.15.16:0/2412804755' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:02:20.175 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:19 smithi092 bash[58855]: audit 2023-09-23T20:02:19.721159+0000 mon.a (mon.0) 1986 : audit 1 from='client.? 172.21.15.16:0/3906191299' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:02:20.194 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:20.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:20.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:20.297 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:20.322 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:20.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:20.325 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:20.425 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:20.459 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:20.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:20.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:20.563 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:20.589 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:20.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:20.592 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:20.692 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:20.723 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:20.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:20.726 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:20.827 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:20.862 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:20.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:20.865 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:20.965 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:20.980 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:20 smithi092 bash[58855]: cluster 2023-09-23T20:02:19.879674+0000 mon.a (mon.0) 1987 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:02:20.980 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:20 smithi092 bash[58855]: audit 2023-09-23T20:02:19.894263+0000 mon.a (mon.0) 1988 : audit 1 from='client.? 172.21.15.16:0/3906191299' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:02:20.980 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:20 smithi092 bash[58855]: cluster 2023-09-23T20:02:19.894336+0000 mon.a (mon.0) 1989 : cluster 0 osdmap e283: 8 total, 8 up, 8 in 2023-09-23T20:02:20.980 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:20 smithi092 bash[58855]: cluster 2023-09-23T20:02:19.895526+0000 osd.6 (osd.6) 217 : cluster 3 failed to encode map e283 with expected crc 2023-09-23T20:02:20.980 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:20 smithi092 bash[58855]: cluster 2023-09-23T20:02:19.901402+0000 osd.5 (osd.5) 203 : cluster 3 failed to encode map e283 with expected crc 2023-09-23T20:02:20.980 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:20 smithi092 bash[58855]: audit 2023-09-23T20:02:19.925608+0000 mgr.y (mgr.25325) 1654 : audit 0 from='client.37131 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:02:20.980 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:20 smithi092 bash[58855]: cluster 2023-09-23T20:02:20.333583+0000 osd.4 (osd.4) 225 : cluster 3 failed to encode map e283 with expected crc 2023-09-23T20:02:20.999 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:21.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:21.001 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:21.102 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:21.134 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:21.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:21.138 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:21.239 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:21.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:20 smithi016 bash[77995]: cluster 2023-09-23T20:02:19.879674+0000 mon.a (mon.0) 1987 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:02:21.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:20 smithi016 bash[77995]: audit 2023-09-23T20:02:19.894263+0000 mon.a (mon.0) 1988 : audit 1 from='client.? 172.21.15.16:0/3906191299' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:02:21.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:20 smithi016 bash[77995]: cluster 2023-09-23T20:02:19.894336+0000 mon.a (mon.0) 1989 : cluster 0 osdmap e283: 8 total, 8 up, 8 in 2023-09-23T20:02:21.250 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:20 smithi016 bash[77995]: cluster 2023-09-23T20:02:19.895526+0000 osd.6 (osd.6) 217 : cluster 3 failed to encode map e283 with expected crc 2023-09-23T20:02:21.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:20 smithi016 bash[77995]: cluster 2023-09-23T20:02:19.901402+0000 osd.5 (osd.5) 203 : cluster 3 failed to encode map e283 with expected crc 2023-09-23T20:02:21.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:20 smithi016 bash[77995]: audit 2023-09-23T20:02:19.925608+0000 mgr.y (mgr.25325) 1654 : audit 0 from='client.37131 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:02:21.251 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:20 smithi016 bash[77995]: cluster 2023-09-23T20:02:20.333583+0000 osd.4 (osd.4) 225 : cluster 3 failed to encode map e283 with expected crc 2023-09-23T20:02:21.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:20 smithi016 bash[75004]: cluster 2023-09-23T20:02:19.879674+0000 mon.a (mon.0) 1987 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:02:21.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:20 smithi016 bash[75004]: audit 2023-09-23T20:02:19.894263+0000 mon.a (mon.0) 1988 : audit 1 from='client.? 172.21.15.16:0/3906191299' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:02:21.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:20 smithi016 bash[75004]: cluster 2023-09-23T20:02:19.894336+0000 mon.a (mon.0) 1989 : cluster 0 osdmap e283: 8 total, 8 up, 8 in 2023-09-23T20:02:21.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:20 smithi016 bash[75004]: cluster 2023-09-23T20:02:19.895526+0000 osd.6 (osd.6) 217 : cluster 3 failed to encode map e283 with expected crc 2023-09-23T20:02:21.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:20 smithi016 bash[75004]: cluster 2023-09-23T20:02:19.901402+0000 osd.5 (osd.5) 203 : cluster 3 failed to encode map e283 with expected crc 2023-09-23T20:02:21.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:20 smithi016 bash[75004]: audit 2023-09-23T20:02:19.925608+0000 mgr.y (mgr.25325) 1654 : audit 0 from='client.37131 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:02:21.251 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:20 smithi016 bash[75004]: cluster 2023-09-23T20:02:20.333583+0000 osd.4 (osd.4) 225 : cluster 3 failed to encode map e283 with expected crc 2023-09-23T20:02:21.264 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:21.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:21.267 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:21.368 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:21.400 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:21.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:21.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:21.504 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:21.539 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:21.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:21.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:21.643 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:21.651 INFO:tasks.workunit.client.0.smithi016.stderr: Importing image: 1% complete... Importing image: 2% complete... Importing image: 3% complete... Importing image: 4% complete... Importing image: 5% complete... Importing image: 6% complete... Importing image: 7% complete... Importing image: 8% complete... Importing image: 9% complete... Importing image: 10% complete... Importing image: 11% complete... Importing image: 12% complete... Importing image: 13% complete... Importing image: 14% complete... Importing image: 15% complete... Importing image: 16% complete... Importing image: 17% complete... Importing image: 18% complete... Importing image: 19% complete... Importing image: 20% complete... Importing image: 21% complete... Importing image: 22% complete... Importing image: 23% complete... Importing image: 24% complete... Importing image: 25% complete... Importing image: 26% complete... Importing image: 27% complete... Importing image: 28% complete... Importing image: 29% complete... Importing image: 30% complete... Importing image: 31% complete... Importing image: 32% complete... Importing image: 33% complete... Importing image: 34% complete... Importing image: 35% complete... Importing image: 36% complete... Importing image: 37% complete... Importing image: 38% complete... Importing image: 39% complete... Importing image: 40% complete... Importing image: 41% complete... Importing image: 42% complete... Importing image: 43% complete... Importing image: 44% complete... Importing image: 45% complete... Importing image: 46% complete... Importing image: 47% complete... Importing image: 48% complete... Importing image: 49% complete... Importing image: 50% complete... Importing image: 51% complete... Importing image: 52% complete... Importing image: 53% complete... Importing image: 54% complete... Importing image: 55% complete... Importing image: 56% complete... Importing image: 57% complete... Importing image: 58% complete... Importing image: 59% complete... Importing image: 60% complete... Importing image: 61% complete... Importing image: 62% complete... Importing image: 63% complete... Importing image: 64% complete... Importing image: 65% complete... Importing image: 66% complete... Importing image: 67% complete... Importing image: 68% complete... Importing image: 69% complete... Importing image: 70% complete... Importing image: 71% complete... Importing image: 72% complete... Importing image: 73% complete... Importing image: 74% complete... Importing image: 75% complete... Importing image: 76% complete... Importing image: 77% complete... Importing image: 78% complete... Importing image: 79% complete... Importing image: 80% complete... Importing image: 81% complete... Importing image: 82% complete... Importing image: 83% complete... Importing image: 84% complete... Importing image: 85% complete... Importing image: 86% complete... Importing image: 87% complete... Importing image: 88% complete... Importing image: 89% complete... Importing image: 90% complete... Importing image: 91% complete... Importing image: 92% complete... Importing image: 93% complete... Importing image: 94% complete... Importing image: 95% complete... Importing image: 96% complete... Importing image: 97% complete... Importing image: 98% complete... Importing image: 99% complete... Importing image: 100% complete...done. 2023-09-23T20:02:21.653 INFO:tasks.workunit.client.0.smithi016.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T20:02:21.656 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd export testimg /tmp/rbd_import_export_249364/img2 2023-09-23T20:02:21.662 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:21.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:21.664 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:21.765 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:21.798 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:21.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:21.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:21.851 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T20:02:21.901 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:21.931 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:21.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:21.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:22.027 INFO:tasks.workunit.client.0.smithi016.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T20:02:22.032 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd export testimg - 2023-09-23T20:02:22.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:21 smithi016 bash[77995]: cluster 2023-09-23T20:02:20.672032+0000 mgr.y (mgr.25325) 1655 : cluster 0 pgmap v1699: 73 pgs: 73 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:02:22.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:21 smithi016 bash[77995]: cluster 2023-09-23T20:02:20.897139+0000 mon.a (mon.0) 1990 : cluster 0 osdmap e284: 8 total, 8 up, 8 in 2023-09-23T20:02:22.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:21 smithi016 bash[77995]: cluster 2023-09-23T20:02:20.904666+0000 osd.6 (osd.6) 218 : cluster 3 failed to encode map e284 with expected crc 2023-09-23T20:02:22.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:21 smithi016 bash[77995]: cluster 2023-09-23T20:02:20.979925+0000 osd.5 (osd.5) 204 : cluster 3 failed to encode map e284 with expected crc 2023-09-23T20:02:22.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:21 smithi016 bash[77995]: cluster 2023-09-23T20:02:20.982299+0000 osd.7 (osd.7) 198 : cluster 3 failed to encode map e284 with expected crc 2023-09-23T20:02:22.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:21 smithi016 bash[75004]: cluster 2023-09-23T20:02:20.672032+0000 mgr.y (mgr.25325) 1655 : cluster 0 pgmap v1699: 73 pgs: 73 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:02:22.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:21 smithi016 bash[75004]: cluster 2023-09-23T20:02:20.897139+0000 mon.a (mon.0) 1990 : cluster 0 osdmap e284: 8 total, 8 up, 8 in 2023-09-23T20:02:22.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:21 smithi016 bash[75004]: cluster 2023-09-23T20:02:20.904666+0000 osd.6 (osd.6) 218 : cluster 3 failed to encode map e284 with expected crc 2023-09-23T20:02:22.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:21 smithi016 bash[75004]: cluster 2023-09-23T20:02:20.979925+0000 osd.5 (osd.5) 204 : cluster 3 failed to encode map e284 with expected crc 2023-09-23T20:02:22.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:21 smithi016 bash[75004]: cluster 2023-09-23T20:02:20.982299+0000 osd.7 (osd.7) 198 : cluster 3 failed to encode map e284 with expected crc 2023-09-23T20:02:22.037 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:22.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:21 smithi092 bash[58855]: cluster 2023-09-23T20:02:20.672032+0000 mgr.y (mgr.25325) 1655 : cluster 0 pgmap v1699: 73 pgs: 73 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:02:22.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:21 smithi092 bash[58855]: cluster 2023-09-23T20:02:20.897139+0000 mon.a (mon.0) 1990 : cluster 0 osdmap e284: 8 total, 8 up, 8 in 2023-09-23T20:02:22.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:21 smithi092 bash[58855]: cluster 2023-09-23T20:02:20.904666+0000 osd.6 (osd.6) 218 : cluster 3 failed to encode map e284 with expected crc 2023-09-23T20:02:22.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:21 smithi092 bash[58855]: cluster 2023-09-23T20:02:20.979925+0000 osd.5 (osd.5) 204 : cluster 3 failed to encode map e284 with expected crc 2023-09-23T20:02:22.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:21 smithi092 bash[58855]: cluster 2023-09-23T20:02:20.982299+0000 osd.7 (osd.7) 198 : cluster 3 failed to encode map e284 with expected crc 2023-09-23T20:02:22.063 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:22.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:22.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:22.168 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:22.192 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:22.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:22.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:22.296 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:22.329 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:22.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:22.332 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:22.434 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:22.469 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:22.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:22.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:22.573 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:22.606 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:22.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:22.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:22.710 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:22.745 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:22.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:22.749 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:22.850 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:22.873 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:22.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:22.877 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:22.951 INFO:tasks.workunit.client.0.smithi016.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T20:02:22.959 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd rm testimg 2023-09-23T20:02:22.978 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:23.007 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:23.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:23.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:23.110 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:23.141 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:23.142 INFO:tasks.workunit.client.0.smithi016.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-09-23T20:02:23.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:23.144 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:23.148 INFO:tasks.workunit.client.0.smithi016.stderr:+ cmp /tmp/rbd_import_export_249364/img /tmp/rbd_import_export_249364/img2 2023-09-23T20:02:23.245 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:23.277 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:23.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:23.280 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:23.381 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:23.404 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:23.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:23.407 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:23.508 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:23.538 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:23.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:23.542 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:23.643 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:23.669 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:23.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:23.672 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:23.773 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:23.923 INFO:tasks.workunit.client.0.smithi016.stderr:+ cmp /tmp/rbd_import_export_249364/img /tmp/rbd_import_export_249364/img3 2023-09-23T20:02:24.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:23 smithi016 bash[75004]: cluster 2023-09-23T20:02:22.672572+0000 mgr.y (mgr.25325) 1656 : cluster 0 pgmap v1701: 73 pgs: 73 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 1.1 KiB/s rd, 767 B/s wr, 2 op/s 2023-09-23T20:02:24.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:23 smithi016 bash[77995]: cluster 2023-09-23T20:02:22.672572+0000 mgr.y (mgr.25325) 1656 : cluster 0 pgmap v1701: 73 pgs: 73 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 1.1 KiB/s rd, 767 B/s wr, 2 op/s 2023-09-23T20:02:24.112 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:24.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:24.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:24.216 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:24.227 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:23 smithi092 bash[58855]: cluster 2023-09-23T20:02:22.672572+0000 mgr.y (mgr.25325) 1656 : cluster 0 pgmap v1701: 73 pgs: 73 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 1.1 KiB/s rd, 767 B/s wr, 2 op/s 2023-09-23T20:02:24.247 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:24.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:24.250 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:24.351 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:24.376 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:24.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:24.378 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:24.473 INFO:tasks.workunit.client.0.smithi016.stderr:+ rm /tmp/rbd_import_export_249364/img2 /tmp/rbd_import_export_249364/img3 2023-09-23T20:02:24.479 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:24.511 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:24.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:24.515 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:24.615 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:24.641 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:24.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:24.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:24.744 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:24.775 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:24.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:24.778 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:24.878 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:24.911 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:24.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:24.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:25.015 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:25.047 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:25.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:25.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:25.151 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:25.186 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:25.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:25.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:25.289 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:25.317 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:25.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:25.320 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:25.421 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:25.458 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:25.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:25.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:25.562 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:25.591 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:25.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:25.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:25.695 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:25.726 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:25.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:25.729 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:25.829 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:25.860 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:25.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:25.864 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:25.906 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:02:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:20:02:25] "GET /metrics HTTP/1.1" 200 33960 "" "Prometheus/2.33.4" 2023-09-23T20:02:25.965 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:25.994 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:25.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:25.997 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:26.098 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:26.130 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:26.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:26.134 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:26.235 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:26.247 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:25 smithi092 bash[58855]: cluster 2023-09-23T20:02:24.673372+0000 mgr.y (mgr.25325) 1657 : cluster 0 pgmap v1702: 73 pgs: 73 active+clean; 4.1 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 110 KiB/s rd, 46 KiB/s wr, 27 op/s 2023-09-23T20:02:26.265 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:26.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:26.268 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:26.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:25 smithi016 bash[75004]: cluster 2023-09-23T20:02:24.673372+0000 mgr.y (mgr.25325) 1657 : cluster 0 pgmap v1702: 73 pgs: 73 active+clean; 4.1 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 110 KiB/s rd, 46 KiB/s wr, 27 op/s 2023-09-23T20:02:26.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:25 smithi016 bash[77995]: cluster 2023-09-23T20:02:24.673372+0000 mgr.y (mgr.25325) 1657 : cluster 0 pgmap v1702: 73 pgs: 73 active+clean; 4.1 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 110 KiB/s rd, 46 KiB/s wr, 27 op/s 2023-09-23T20:02:26.369 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:26.397 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:26.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:26.401 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:26.502 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:26.539 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:26.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:26.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:26.642 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:26.672 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:26.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:26.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:26.776 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:26.805 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:26.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:26.809 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:26.910 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:26.941 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:26.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:26.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:26.965 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:02:27.045 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:27.072 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:27.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:27.075 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:27.176 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:27.207 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:27.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:27.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:27.311 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:27.342 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:27.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:27.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:27.446 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:27.476 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:27.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:27.480 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:27.580 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:27.617 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:27.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:27.620 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:27.721 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:27.752 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:27.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:27.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:27.855 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:27.885 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:27.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:27.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:27.989 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:28.022 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:28.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:28.025 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:28.126 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:28.157 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:28.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:28.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:28.261 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:28.273 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:27 smithi092 bash[58855]: cluster 2023-09-23T20:02:26.674341+0000 mgr.y (mgr.25325) 1658 : cluster 0 pgmap v1703: 73 pgs: 73 active+clean; 4.1 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 151 KiB/s rd, 65 KiB/s wr, 37 op/s 2023-09-23T20:02:28.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:27 smithi016 bash[77995]: cluster 2023-09-23T20:02:26.674341+0000 mgr.y (mgr.25325) 1658 : cluster 0 pgmap v1703: 73 pgs: 73 active+clean; 4.1 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 151 KiB/s rd, 65 KiB/s wr, 37 op/s 2023-09-23T20:02:28.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:27 smithi016 bash[75004]: cluster 2023-09-23T20:02:26.674341+0000 mgr.y (mgr.25325) 1658 : cluster 0 pgmap v1703: 73 pgs: 73 active+clean; 4.1 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 151 KiB/s rd, 65 KiB/s wr, 37 op/s 2023-09-23T20:02:28.295 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:28.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:28.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:28.398 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:28.432 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:28.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:28.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:28.536 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:28.569 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:28.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:28.573 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:28.674 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:28.705 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:28.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:28.709 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:28.809 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:28.837 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:28.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:28.841 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:28.942 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:28.967 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:28.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:28.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:29.071 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:29.103 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:29.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:29.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:29.207 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:29.236 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:29.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:29.240 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:29.268 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd import --new-format - testimg 2023-09-23T20:02:29.341 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:29.364 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:29.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:29.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:29.468 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:29.499 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:29.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:29.502 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:29.602 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:29.631 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:29.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:29.633 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:29.734 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:29.741 INFO:tasks.workunit.client.0.smithi016.stderr: Importing image: 100% complete...done. 2023-09-23T20:02:29.741 INFO:tasks.workunit.client.0.smithi016.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T20:02:29.746 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd export testimg /tmp/rbd_import_export_249364/img2 2023-09-23T20:02:29.746 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:29 smithi092 bash[58855]: cluster 2023-09-23T20:02:28.675100+0000 mgr.y (mgr.25325) 1659 : cluster 0 pgmap v1704: 73 pgs: 73 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 137 KiB/s rd, 59 KiB/s wr, 34 op/s 2023-09-23T20:02:29.747 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:29 smithi092 bash[58855]: audit 2023-09-23T20:02:28.706546+0000 mon.a (mon.0) 1991 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:02:29.747 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:29 smithi092 bash[58855]: audit 2023-09-23T20:02:28.707840+0000 mon.a (mon.0) 1992 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:02:29.769 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:29.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:29.772 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:29.873 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:29.903 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:29.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:29.906 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:30.006 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:30.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:29 smithi016 bash[77995]: cluster 2023-09-23T20:02:28.675100+0000 mgr.y (mgr.25325) 1659 : cluster 0 pgmap v1704: 73 pgs: 73 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 137 KiB/s rd, 59 KiB/s wr, 34 op/s 2023-09-23T20:02:30.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:29 smithi016 bash[77995]: audit 2023-09-23T20:02:28.706546+0000 mon.a (mon.0) 1991 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:02:30.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:29 smithi016 bash[77995]: audit 2023-09-23T20:02:28.707840+0000 mon.a (mon.0) 1992 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:02:30.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:29 smithi016 bash[75004]: cluster 2023-09-23T20:02:28.675100+0000 mgr.y (mgr.25325) 1659 : cluster 0 pgmap v1704: 73 pgs: 73 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 137 KiB/s rd, 59 KiB/s wr, 34 op/s 2023-09-23T20:02:30.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:29 smithi016 bash[75004]: audit 2023-09-23T20:02:28.706546+0000 mon.a (mon.0) 1991 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:02:30.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:29 smithi016 bash[75004]: audit 2023-09-23T20:02:28.707840+0000 mon.a (mon.0) 1992 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:02:30.038 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:30.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:30.041 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:30.126 INFO:tasks.workunit.client.0.smithi016.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T20:02:30.130 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd export testimg - 2023-09-23T20:02:30.142 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:30.167 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:30.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:30.170 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:30.223 INFO:teuthology.orchestra.run.smithi016.stdout:{"pg_ready":true,"pg_map":{"version":1698,"stamp":"2023-09-23T20:02:19.890330+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":261,"num_write_kb":2446,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":197,"ondisk_log_size":197,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":7930904,"kb_used_data":23896,"kb_used_omap":20,"kb_used_meta":7906923,"kb_avail":742030312,"statfs":{"total":767960285184,"available":759839039488,"internally_reserved":0,"allocated":24469504,"data_stored":17854308,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":8096689806},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"11.036704"},"pg_stats":[{"pgid":"3.1a","version":"59'1","reported_seq":278,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:08.343791+0000","last_change":"2023-09-23T19:38:43.897548+0000","last_active":"2023-09-23T20:02:08.343791+0000","last_peered":"2023-09-23T20:02:08.343791+0000","last_clean":"2023-09-23T20:02:08.343791+0000","last_became_active":"2023-09-23T19:38:43.896868+0000","last_became_peered":"2023-09-23T19:38:43.896868+0000","last_unstale":"2023-09-23T20:02:08.343791+0000","last_undegraded":"2023-09-23T20:02:08.343791+0000","last_fullsized":"2023-09-23T20:02:08.343791+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":246,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:07.853871+0000","last_change":"2023-09-23T19:32:49.731128+0000","last_active":"2023-09-23T20:02:07.853871+0000","last_peered":"2023-09-23T20:02:07.853871+0000","last_clean":"2023-09-23T20:02:07.853871+0000","last_became_active":"2023-09-23T19:32:49.730757+0000","last_became_peered":"2023-09-23T19:32:49.730757+0000","last_unstale":"2023-09-23T20:02:07.853871+0000","last_undegraded":"2023-09-23T20:02:07.853871+0000","last_fullsized":"2023-09-23T20:02:07.853871+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:40.183605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":235,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:07.901496+0000","last_change":"2023-09-23T19:28:07.890462+0000","last_active":"2023-09-23T20:02:07.901496+0000","last_peered":"2023-09-23T20:02:07.901496+0000","last_clean":"2023-09-23T20:02:07.901496+0000","last_became_active":"2023-09-23T19:28:07.890084+0000","last_became_peered":"2023-09-23T19:28:07.890084+0000","last_unstale":"2023-09-23T20:02:07.901496+0000","last_undegraded":"2023-09-23T20:02:07.901496+0000","last_fullsized":"2023-09-23T20:02:07.901496+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:50.458218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":168,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:08.260427+0000","last_change":"2023-09-23T19:42:55.772143+0000","last_active":"2023-09-23T20:02:08.260427+0000","last_peered":"2023-09-23T20:02:08.260427+0000","last_clean":"2023-09-23T20:02:08.260427+0000","last_became_active":"2023-09-23T19:42:55.771813+0000","last_became_peered":"2023-09-23T19:42:55.771813+0000","last_unstale":"2023-09-23T20:02:08.260427+0000","last_undegraded":"2023-09-23T20:02:08.260427+0000","last_fullsized":"2023-09-23T20:02:08.260427+0000","mapping_epoch":188,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":264,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:08.215297+0000","last_change":"2023-09-23T19:32:49.729961+0000","last_active":"2023-09-23T20:02:08.215297+0000","last_peered":"2023-09-23T20:02:08.215297+0000","last_clean":"2023-09-23T20:02:08.215297+0000","last_became_active":"2023-09-23T19:32:49.729251+0000","last_became_peered":"2023-09-23T19:32:49.729251+0000","last_unstale":"2023-09-23T20:02:08.215297+0000","last_undegraded":"2023-09-23T20:02:08.215297+0000","last_fullsized":"2023-09-23T20:02:08.215297+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:20:56.595732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":247,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:07.856093+0000","last_change":"2023-09-23T19:22:42.150582+0000","last_active":"2023-09-23T20:02:07.856093+0000","last_peered":"2023-09-23T20:02:07.856093+0000","last_clean":"2023-09-23T20:02:07.856093+0000","last_became_active":"2023-09-23T19:22:42.150229+0000","last_became_peered":"2023-09-23T19:22:42.150229+0000","last_unstale":"2023-09-23T20:02:07.856093+0000","last_undegraded":"2023-09-23T20:02:07.856093+0000","last_fullsized":"2023-09-23T20:02:07.856093+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":226,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:07.849340+0000","last_change":"2023-09-23T19:47:56.793944+0000","last_active":"2023-09-23T20:02:07.849340+0000","last_peered":"2023-09-23T20:02:07.849340+0000","last_clean":"2023-09-23T20:02:07.849340+0000","last_became_active":"2023-09-23T19:47:56.793618+0000","last_became_peered":"2023-09-23T19:47:56.793618+0000","last_unstale":"2023-09-23T20:02:07.849340+0000","last_undegraded":"2023-09-23T20:02:07.849340+0000","last_fullsized":"2023-09-23T20:02:07.849340+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":247,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:07.902387+0000","last_change":"2023-09-23T19:32:49.729765+0000","last_active":"2023-09-23T20:02:07.902387+0000","last_peered":"2023-09-23T20:02:07.902387+0000","last_clean":"2023-09-23T20:02:07.902387+0000","last_became_active":"2023-09-23T19:32:49.729007+0000","last_became_peered":"2023-09-23T19:32:49.729007+0000","last_unstale":"2023-09-23T20:02:07.902387+0000","last_undegraded":"2023-09-23T20:02:07.902387+0000","last_fullsized":"2023-09-23T20:02:07.902387+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:59:31.183753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":234,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:07.848904+0000","last_change":"2023-09-23T19:47:56.791464+0000","last_active":"2023-09-23T20:02:07.848904+0000","last_peered":"2023-09-23T20:02:07.848904+0000","last_clean":"2023-09-23T20:02:07.848904+0000","last_became_active":"2023-09-23T19:47:56.791144+0000","last_became_peered":"2023-09-23T19:47:56.791144+0000","last_unstale":"2023-09-23T20:02:07.848904+0000","last_undegraded":"2023-09-23T20:02:07.848904+0000","last_fullsized":"2023-09-23T20:02:07.848904+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":222,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:08.261597+0000","last_change":"2023-09-23T19:32:49.736864+0000","last_active":"2023-09-23T20:02:08.261597+0000","last_peered":"2023-09-23T20:02:08.261597+0000","last_clean":"2023-09-23T20:02:08.261597+0000","last_became_active":"2023-09-23T19:32:49.736523+0000","last_became_peered":"2023-09-23T19:32:49.736523+0000","last_unstale":"2023-09-23T20:02:08.261597+0000","last_undegraded":"2023-09-23T20:02:08.261597+0000","last_fullsized":"2023-09-23T20:02:08.261597+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":251,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:08.260490+0000","last_change":"2023-09-23T19:28:07.892835+0000","last_active":"2023-09-23T20:02:08.260490+0000","last_peered":"2023-09-23T20:02:08.260490+0000","last_clean":"2023-09-23T20:02:08.260490+0000","last_became_active":"2023-09-23T19:28:07.892338+0000","last_became_peered":"2023-09-23T19:28:07.892338+0000","last_unstale":"2023-09-23T20:02:08.260490+0000","last_undegraded":"2023-09-23T20:02:08.260490+0000","last_fullsized":"2023-09-23T20:02:08.260490+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":250,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:07.854179+0000","last_change":"2023-09-23T19:38:43.890137+0000","last_active":"2023-09-23T20:02:07.854179+0000","last_peered":"2023-09-23T20:02:07.854179+0000","last_clean":"2023-09-23T20:02:07.854179+0000","last_became_active":"2023-09-23T19:38:43.889800+0000","last_became_peered":"2023-09-23T19:38:43.889800+0000","last_unstale":"2023-09-23T20:02:07.854179+0000","last_undegraded":"2023-09-23T20:02:07.854179+0000","last_fullsized":"2023-09-23T20:02:07.854179+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:59:20.468913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'23","reported_seq":255,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:08.343759+0000","last_change":"2023-09-23T19:05:52.107727+0000","last_active":"2023-09-23T20:02:08.343759+0000","last_peered":"2023-09-23T20:02:08.343759+0000","last_clean":"2023-09-23T20:02:08.343759+0000","last_became_active":"2023-09-23T19:05:52.107378+0000","last_became_peered":"2023-09-23T19:05:52.107378+0000","last_unstale":"2023-09-23T20:02:08.343759+0000","last_undegraded":"2023-09-23T20:02:08.343759+0000","last_fullsized":"2023-09-23T20:02:08.343759+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":246,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:08.345614+0000","last_change":"2023-09-23T19:22:42.148497+0000","last_active":"2023-09-23T20:02:08.345614+0000","last_peered":"2023-09-23T20:02:08.345614+0000","last_clean":"2023-09-23T20:02:08.345614+0000","last_became_active":"2023-09-23T19:22:42.148147+0000","last_became_peered":"2023-09-23T19:22:42.148147+0000","last_unstale":"2023-09-23T20:02:08.345614+0000","last_undegraded":"2023-09-23T20:02:08.345614+0000","last_fullsized":"2023-09-23T20:02:08.345614+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":265,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:08.262036+0000","last_change":"2023-09-23T19:38:43.897378+0000","last_active":"2023-09-23T20:02:08.262036+0000","last_peered":"2023-09-23T20:02:08.262036+0000","last_clean":"2023-09-23T20:02:08.262036+0000","last_became_active":"2023-09-23T19:38:43.897012+0000","last_became_peered":"2023-09-23T19:38:43.897012+0000","last_unstale":"2023-09-23T20:02:08.262036+0000","last_undegraded":"2023-09-23T20:02:08.262036+0000","last_fullsized":"2023-09-23T20:02:08.262036+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":247,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:07.900932+0000","last_change":"2023-09-23T19:32:49.730549+0000","last_active":"2023-09-23T20:02:07.900932+0000","last_peered":"2023-09-23T20:02:07.900932+0000","last_clean":"2023-09-23T20:02:07.900932+0000","last_became_active":"2023-09-23T19:32:49.730105+0000","last_became_peered":"2023-09-23T19:32:49.730105+0000","last_unstale":"2023-09-23T20:02:07.900932+0000","last_undegraded":"2023-09-23T20:02:07.900932+0000","last_fullsized":"2023-09-23T20:02:07.900932+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:49.751893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":168,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:08.261901+0000","last_change":"2023-09-23T19:42:55.772691+0000","last_active":"2023-09-23T20:02:08.261901+0000","last_peered":"2023-09-23T20:02:08.261901+0000","last_clean":"2023-09-23T20:02:08.261901+0000","last_became_active":"2023-09-23T19:42:55.772309+0000","last_became_peered":"2023-09-23T19:42:55.772309+0000","last_unstale":"2023-09-23T20:02:08.261901+0000","last_undegraded":"2023-09-23T20:02:08.261901+0000","last_fullsized":"2023-09-23T20:02:08.261901+0000","mapping_epoch":188,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":189,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:08.262707+0000","last_change":"2023-09-23T19:58:26.094387+0000","last_active":"2023-09-23T20:02:08.262707+0000","last_peered":"2023-09-23T20:02:08.262707+0000","last_clean":"2023-09-23T20:02:08.262707+0000","last_became_active":"2023-09-23T19:58:26.093975+0000","last_became_peered":"2023-09-23T19:58:26.093975+0000","last_unstale":"2023-09-23T20:02:08.262707+0000","last_undegraded":"2023-09-23T20:02:08.262707+0000","last_fullsized":"2023-09-23T20:02:08.262707+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":233,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:08.261538+0000","last_change":"2023-09-23T19:05:52.114186+0000","last_active":"2023-09-23T20:02:08.261538+0000","last_peered":"2023-09-23T20:02:08.261538+0000","last_clean":"2023-09-23T20:02:08.261538+0000","last_became_active":"2023-09-23T19:05:52.113635+0000","last_became_peered":"2023-09-23T19:05:52.113635+0000","last_unstale":"2023-09-23T20:02:08.261538+0000","last_undegraded":"2023-09-23T20:02:08.261538+0000","last_fullsized":"2023-09-23T20:02:08.261538+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":168,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:08.260932+0000","last_change":"2023-09-23T19:42:55.772912+0000","last_active":"2023-09-23T20:02:08.260932+0000","last_peered":"2023-09-23T20:02:08.260932+0000","last_clean":"2023-09-23T20:02:08.260932+0000","last_became_active":"2023-09-23T19:42:55.772434+0000","last_became_peered":"2023-09-23T19:42:55.772434+0000","last_unstale":"2023-09-23T20:02:08.260932+0000","last_undegraded":"2023-09-23T20:02:08.260932+0000","last_fullsized":"2023-09-23T20:02:08.260932+0000","mapping_epoch":188,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":256,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:07.854727+0000","last_change":"2023-09-23T19:18:09.010195+0000","last_active":"2023-09-23T20:02:07.854727+0000","last_peered":"2023-09-23T20:02:07.854727+0000","last_clean":"2023-09-23T20:02:07.854727+0000","last_became_active":"2023-09-23T19:18:09.009687+0000","last_became_peered":"2023-09-23T19:18:09.009687+0000","last_unstale":"2023-09-23T20:02:07.854727+0000","last_undegraded":"2023-09-23T20:02:07.854727+0000","last_fullsized":"2023-09-23T20:02:07.854727+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":256,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:07.901729+0000","last_change":"2023-09-23T19:28:07.885265+0000","last_active":"2023-09-23T20:02:07.901729+0000","last_peered":"2023-09-23T20:02:07.901729+0000","last_clean":"2023-09-23T20:02:07.901729+0000","last_became_active":"2023-09-23T19:28:07.884353+0000","last_became_peered":"2023-09-23T19:28:07.884353+0000","last_unstale":"2023-09-23T20:02:07.901729+0000","last_undegraded":"2023-09-23T20:02:07.901729+0000","last_fullsized":"2023-09-23T20:02:07.901729+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:56.325352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":249,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:08.262386+0000","last_change":"2023-09-23T19:18:09.035116+0000","last_active":"2023-09-23T20:02:08.262386+0000","last_peered":"2023-09-23T20:02:08.262386+0000","last_clean":"2023-09-23T20:02:08.262386+0000","last_became_active":"2023-09-23T19:18:09.034416+0000","last_became_peered":"2023-09-23T19:18:09.034416+0000","last_unstale":"2023-09-23T20:02:08.262386+0000","last_undegraded":"2023-09-23T20:02:08.262386+0000","last_fullsized":"2023-09-23T20:02:08.262386+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":218,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:07.848248+0000","last_change":"2023-09-23T19:47:56.790227+0000","last_active":"2023-09-23T20:02:07.848248+0000","last_peered":"2023-09-23T20:02:07.848248+0000","last_clean":"2023-09-23T20:02:07.848248+0000","last_became_active":"2023-09-23T19:47:56.789727+0000","last_became_peered":"2023-09-23T19:47:56.789727+0000","last_unstale":"2023-09-23T20:02:07.848248+0000","last_undegraded":"2023-09-23T20:02:07.848248+0000","last_fullsized":"2023-09-23T20:02:07.848248+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":250,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:08.262207+0000","last_change":"2023-09-23T19:22:42.151134+0000","last_active":"2023-09-23T20:02:08.262207+0000","last_peered":"2023-09-23T20:02:08.262207+0000","last_clean":"2023-09-23T20:02:08.262207+0000","last_became_active":"2023-09-23T19:22:42.150553+0000","last_became_peered":"2023-09-23T19:22:42.150553+0000","last_unstale":"2023-09-23T20:02:08.262207+0000","last_undegraded":"2023-09-23T20:02:08.262207+0000","last_fullsized":"2023-09-23T20:02:08.262207+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":261,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:08.342192+0000","last_change":"2023-09-23T19:28:07.899262+0000","last_active":"2023-09-23T20:02:08.342192+0000","last_peered":"2023-09-23T20:02:08.342192+0000","last_clean":"2023-09-23T20:02:08.342192+0000","last_became_active":"2023-09-23T19:28:07.898848+0000","last_became_peered":"2023-09-23T19:28:07.898848+0000","last_unstale":"2023-09-23T20:02:08.342192+0000","last_undegraded":"2023-09-23T20:02:08.342192+0000","last_fullsized":"2023-09-23T20:02:08.342192+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":234,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:08.262555+0000","last_change":"2023-09-23T19:05:52.114108+0000","last_active":"2023-09-23T20:02:08.262555+0000","last_peered":"2023-09-23T20:02:08.262555+0000","last_clean":"2023-09-23T20:02:08.262555+0000","last_became_active":"2023-09-23T19:05:52.113609+0000","last_became_peered":"2023-09-23T19:05:52.113609+0000","last_unstale":"2023-09-23T20:02:08.262555+0000","last_undegraded":"2023-09-23T20:02:08.262555+0000","last_fullsized":"2023-09-23T20:02:08.262555+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":247,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:07.854936+0000","last_change":"2023-09-23T19:22:42.156825+0000","last_active":"2023-09-23T20:02:07.854936+0000","last_peered":"2023-09-23T20:02:07.854936+0000","last_clean":"2023-09-23T20:02:07.854936+0000","last_became_active":"2023-09-23T19:22:42.156474+0000","last_became_peered":"2023-09-23T19:22:42.156474+0000","last_unstale":"2023-09-23T20:02:07.854936+0000","last_undegraded":"2023-09-23T20:02:07.854936+0000","last_fullsized":"2023-09-23T20:02:07.854936+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":190,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:07.855603+0000","last_change":"2023-09-23T19:58:26.094114+0000","last_active":"2023-09-23T20:02:07.855603+0000","last_peered":"2023-09-23T20:02:07.855603+0000","last_clean":"2023-09-23T20:02:07.855603+0000","last_became_active":"2023-09-23T19:58:26.093615+0000","last_became_peered":"2023-09-23T19:58:26.093615+0000","last_unstale":"2023-09-23T20:02:07.855603+0000","last_undegraded":"2023-09-23T20:02:07.855603+0000","last_fullsized":"2023-09-23T20:02:07.855603+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":252,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:08.215144+0000","last_change":"2023-09-23T19:22:42.135572+0000","last_active":"2023-09-23T20:02:08.215144+0000","last_peered":"2023-09-23T20:02:08.215144+0000","last_clean":"2023-09-23T20:02:08.215144+0000","last_became_active":"2023-09-23T19:22:42.135237+0000","last_became_peered":"2023-09-23T19:22:42.135237+0000","last_unstale":"2023-09-23T20:02:08.215144+0000","last_undegraded":"2023-09-23T20:02:08.215144+0000","last_fullsized":"2023-09-23T20:02:08.215144+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:35.838718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":115,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:08.342185+0000","last_change":"2023-09-23T19:42:55.771907+0000","last_active":"2023-09-23T20:02:08.342185+0000","last_peered":"2023-09-23T20:02:08.342185+0000","last_clean":"2023-09-23T20:02:08.342185+0000","last_became_active":"2023-09-23T19:42:55.771503+0000","last_became_peered":"2023-09-23T19:42:55.771503+0000","last_unstale":"2023-09-23T20:02:08.342185+0000","last_undegraded":"2023-09-23T20:02:08.342185+0000","last_fullsized":"2023-09-23T20:02:08.342185+0000","mapping_epoch":188,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":246,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:08.342184+0000","last_change":"2023-09-23T19:28:07.896803+0000","last_active":"2023-09-23T20:02:08.342184+0000","last_peered":"2023-09-23T20:02:08.342184+0000","last_clean":"2023-09-23T20:02:08.342184+0000","last_became_active":"2023-09-23T19:28:07.896480+0000","last_became_peered":"2023-09-23T19:28:07.896480+0000","last_unstale":"2023-09-23T20:02:08.342184+0000","last_undegraded":"2023-09-23T20:02:08.342184+0000","last_fullsized":"2023-09-23T20:02:08.342184+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":248,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:07.854972+0000","last_change":"2023-09-23T19:22:42.156582+0000","last_active":"2023-09-23T20:02:07.854972+0000","last_peered":"2023-09-23T20:02:07.854972+0000","last_clean":"2023-09-23T20:02:07.854972+0000","last_became_active":"2023-09-23T19:22:42.156251+0000","last_became_peered":"2023-09-23T19:22:42.156251+0000","last_unstale":"2023-09-23T20:02:07.854972+0000","last_undegraded":"2023-09-23T20:02:07.854972+0000","last_fullsized":"2023-09-23T20:02:07.854972+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":276,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:08.344229+0000","last_change":"2023-09-23T19:38:43.891593+0000","last_active":"2023-09-23T20:02:08.344229+0000","last_peered":"2023-09-23T20:02:08.344229+0000","last_clean":"2023-09-23T20:02:08.344229+0000","last_became_active":"2023-09-23T19:38:43.891063+0000","last_became_peered":"2023-09-23T19:38:43.891063+0000","last_unstale":"2023-09-23T20:02:08.344229+0000","last_undegraded":"2023-09-23T20:02:08.344229+0000","last_fullsized":"2023-09-23T20:02:08.344229+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":262,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:08.344606+0000","last_change":"2023-09-23T19:38:43.892972+0000","last_active":"2023-09-23T20:02:08.344606+0000","last_peered":"2023-09-23T20:02:08.344606+0000","last_clean":"2023-09-23T20:02:08.344606+0000","last_became_active":"2023-09-23T19:38:43.892451+0000","last_became_peered":"2023-09-23T19:38:43.892451+0000","last_unstale":"2023-09-23T20:02:08.344606+0000","last_undegraded":"2023-09-23T20:02:08.344606+0000","last_fullsized":"2023-09-23T20:02:08.344606+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":246,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:08.344007+0000","last_change":"2023-09-23T19:22:42.149670+0000","last_active":"2023-09-23T20:02:08.344007+0000","last_peered":"2023-09-23T20:02:08.344007+0000","last_clean":"2023-09-23T20:02:08.344007+0000","last_became_active":"2023-09-23T19:22:42.149022+0000","last_became_peered":"2023-09-23T19:22:42.149022+0000","last_unstale":"2023-09-23T20:02:08.344007+0000","last_undegraded":"2023-09-23T20:02:08.344007+0000","last_fullsized":"2023-09-23T20:02:08.344007+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":249,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:07.902276+0000","last_change":"2023-09-23T19:32:49.729244+0000","last_active":"2023-09-23T20:02:07.902276+0000","last_peered":"2023-09-23T20:02:07.902276+0000","last_clean":"2023-09-23T20:02:07.902276+0000","last_became_active":"2023-09-23T19:32:49.728810+0000","last_became_peered":"2023-09-23T19:32:49.728810+0000","last_unstale":"2023-09-23T20:02:07.902276+0000","last_undegraded":"2023-09-23T20:02:07.902276+0000","last_fullsized":"2023-09-23T20:02:07.902276+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:57.540826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":236,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:07.902614+0000","last_change":"2023-09-23T19:28:07.889095+0000","last_active":"2023-09-23T20:02:07.902614+0000","last_peered":"2023-09-23T20:02:07.902614+0000","last_clean":"2023-09-23T20:02:07.902614+0000","last_became_active":"2023-09-23T19:28:07.888742+0000","last_became_peered":"2023-09-23T19:28:07.888742+0000","last_unstale":"2023-09-23T20:02:07.902614+0000","last_undegraded":"2023-09-23T20:02:07.902614+0000","last_fullsized":"2023-09-23T20:02:07.902614+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:43.929488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":270,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:07.900837+0000","last_change":"2023-09-23T19:22:42.143289+0000","last_active":"2023-09-23T20:02:07.900837+0000","last_peered":"2023-09-23T20:02:07.900837+0000","last_clean":"2023-09-23T20:02:07.900837+0000","last_became_active":"2023-09-23T19:22:42.142656+0000","last_became_peered":"2023-09-23T19:22:42.142656+0000","last_unstale":"2023-09-23T20:02:07.900837+0000","last_undegraded":"2023-09-23T20:02:07.900837+0000","last_fullsized":"2023-09-23T20:02:07.900837+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:38.467441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":264,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:07.849448+0000","last_change":"2023-09-23T19:47:56.794437+0000","last_active":"2023-09-23T20:02:07.849448+0000","last_peered":"2023-09-23T20:02:07.849448+0000","last_clean":"2023-09-23T20:02:07.849448+0000","last_became_active":"2023-09-23T19:47:56.794118+0000","last_became_peered":"2023-09-23T19:47:56.794118+0000","last_unstale":"2023-09-23T20:02:07.849448+0000","last_undegraded":"2023-09-23T20:02:07.849448+0000","last_fullsized":"2023-09-23T20:02:07.849448+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":256,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:07.841519+0000","last_change":"2023-09-23T19:38:43.898519+0000","last_active":"2023-09-23T20:02:07.841519+0000","last_peered":"2023-09-23T20:02:07.841519+0000","last_clean":"2023-09-23T20:02:07.841519+0000","last_became_active":"2023-09-23T19:38:43.896814+0000","last_became_peered":"2023-09-23T19:38:43.896814+0000","last_unstale":"2023-09-23T20:02:07.841519+0000","last_undegraded":"2023-09-23T20:02:07.841519+0000","last_fullsized":"2023-09-23T20:02:07.841519+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:40.465038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":170,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:07.854166+0000","last_change":"2023-09-23T19:43:38.612700+0000","last_active":"2023-09-23T20:02:07.854166+0000","last_peered":"2023-09-23T20:02:07.854166+0000","last_clean":"2023-09-23T20:02:07.854166+0000","last_became_active":"2023-09-23T19:43:38.612424+0000","last_became_peered":"2023-09-23T19:43:38.612424+0000","last_unstale":"2023-09-23T20:02:07.854166+0000","last_undegraded":"2023-09-23T20:02:07.854166+0000","last_fullsized":"2023-09-23T20:02:07.854166+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:24.833490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":247,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:08.345484+0000","last_change":"2023-09-23T19:18:09.017525+0000","last_active":"2023-09-23T20:02:08.345484+0000","last_peered":"2023-09-23T20:02:08.345484+0000","last_clean":"2023-09-23T20:02:08.345484+0000","last_became_active":"2023-09-23T19:18:09.017193+0000","last_became_peered":"2023-09-23T19:18:09.017193+0000","last_unstale":"2023-09-23T20:02:08.345484+0000","last_undegraded":"2023-09-23T20:02:08.345484+0000","last_fullsized":"2023-09-23T20:02:08.345484+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":267,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:07.854179+0000","last_change":"2023-09-23T19:38:43.877088+0000","last_active":"2023-09-23T20:02:07.854179+0000","last_peered":"2023-09-23T20:02:07.854179+0000","last_clean":"2023-09-23T20:02:07.854179+0000","last_became_active":"2023-09-23T19:38:43.876719+0000","last_became_peered":"2023-09-23T19:38:43.876719+0000","last_unstale":"2023-09-23T20:02:07.854179+0000","last_undegraded":"2023-09-23T20:02:07.854179+0000","last_fullsized":"2023-09-23T20:02:07.854179+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:05.423256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":236,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:07.840948+0000","last_change":"2023-09-23T19:38:43.883663+0000","last_active":"2023-09-23T20:02:07.840948+0000","last_peered":"2023-09-23T20:02:07.840948+0000","last_clean":"2023-09-23T20:02:07.840948+0000","last_became_active":"2023-09-23T19:38:43.882421+0000","last_became_peered":"2023-09-23T19:38:43.882421+0000","last_unstale":"2023-09-23T20:02:07.840948+0000","last_undegraded":"2023-09-23T20:02:07.840948+0000","last_fullsized":"2023-09-23T20:02:07.840948+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:49:01.607659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":253,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:07.902486+0000","last_change":"2023-09-23T19:38:43.890600+0000","last_active":"2023-09-23T20:02:07.902486+0000","last_peered":"2023-09-23T20:02:07.902486+0000","last_clean":"2023-09-23T20:02:07.902486+0000","last_became_active":"2023-09-23T19:38:43.890253+0000","last_became_peered":"2023-09-23T19:38:43.890253+0000","last_unstale":"2023-09-23T20:02:07.902486+0000","last_undegraded":"2023-09-23T20:02:07.902486+0000","last_fullsized":"2023-09-23T20:02:07.902486+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:46.751120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":265,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:07.902755+0000","last_change":"2023-09-23T19:22:42.143733+0000","last_active":"2023-09-23T20:02:07.902755+0000","last_peered":"2023-09-23T20:02:07.902755+0000","last_clean":"2023-09-23T20:02:07.902755+0000","last_became_active":"2023-09-23T19:22:42.143077+0000","last_became_peered":"2023-09-23T19:22:42.143077+0000","last_unstale":"2023-09-23T20:02:07.902755+0000","last_undegraded":"2023-09-23T20:02:07.902755+0000","last_fullsized":"2023-09-23T20:02:07.902755+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:47:33.086994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"282'1","reported_seq":254,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T20:02:09.209678+0000","last_change":"2023-09-23T19:47:56.791259+0000","last_active":"2023-09-23T20:02:09.209678+0000","last_peered":"2023-09-23T20:02:09.209678+0000","last_clean":"2023-09-23T20:02:09.209678+0000","last_became_active":"2023-09-23T19:47:56.790954+0000","last_became_peered":"2023-09-23T19:47:56.790954+0000","last_unstale":"2023-09-23T20:02:09.209678+0000","last_undegraded":"2023-09-23T20:02:09.209678+0000","last_fullsized":"2023-09-23T20:02:09.209678+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":484,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:08.261377+0000","last_change":"2023-09-23T19:18:09.016297+0000","last_active":"2023-09-23T20:02:08.261377+0000","last_peered":"2023-09-23T20:02:08.261377+0000","last_clean":"2023-09-23T20:02:08.261377+0000","last_became_active":"2023-09-23T19:18:09.015436+0000","last_became_peered":"2023-09-23T19:18:09.015436+0000","last_unstale":"2023-09-23T20:02:08.261377+0000","last_undegraded":"2023-09-23T20:02:08.261377+0000","last_fullsized":"2023-09-23T20:02:08.261377+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:59:33.840751+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:59:33.840751+0000","last_clean_scrub_stamp":"2023-09-23T18:59:33.840751+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":207,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:07.848645+0000","last_change":"2023-09-23T19:47:56.811861+0000","last_active":"2023-09-23T20:02:07.848645+0000","last_peered":"2023-09-23T20:02:07.848645+0000","last_clean":"2023-09-23T20:02:07.848645+0000","last_became_active":"2023-09-23T19:47:56.811527+0000","last_became_peered":"2023-09-23T19:47:56.811527+0000","last_unstale":"2023-09-23T20:02:07.848645+0000","last_undegraded":"2023-09-23T20:02:07.848645+0000","last_fullsized":"2023-09-23T20:02:07.848645+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":232,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:07.856648+0000","last_change":"2023-09-23T19:05:52.786771+0000","last_active":"2023-09-23T20:02:07.856648+0000","last_peered":"2023-09-23T20:02:07.856648+0000","last_clean":"2023-09-23T20:02:07.856648+0000","last_became_active":"2023-09-23T19:05:52.785797+0000","last_became_peered":"2023-09-23T19:05:52.785797+0000","last_unstale":"2023-09-23T20:02:07.856648+0000","last_undegraded":"2023-09-23T20:02:07.856648+0000","last_fullsized":"2023-09-23T20:02:07.856648+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"282'1","reported_seq":268,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T20:02:09.211415+0000","last_change":"2023-09-23T19:22:42.150636+0000","last_active":"2023-09-23T20:02:09.211415+0000","last_peered":"2023-09-23T20:02:09.211415+0000","last_clean":"2023-09-23T20:02:09.211415+0000","last_became_active":"2023-09-23T19:22:42.150045+0000","last_became_peered":"2023-09-23T19:22:42.150045+0000","last_unstale":"2023-09-23T20:02:09.211415+0000","last_undegraded":"2023-09-23T20:02:09.211415+0000","last_fullsized":"2023-09-23T20:02:09.211415+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":168,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:08.341920+0000","last_change":"2023-09-23T19:42:55.771906+0000","last_active":"2023-09-23T20:02:08.341920+0000","last_peered":"2023-09-23T20:02:08.341920+0000","last_clean":"2023-09-23T20:02:08.341920+0000","last_became_active":"2023-09-23T19:42:55.771599+0000","last_became_peered":"2023-09-23T19:42:55.771599+0000","last_unstale":"2023-09-23T20:02:08.341920+0000","last_undegraded":"2023-09-23T20:02:08.341920+0000","last_fullsized":"2023-09-23T20:02:08.341920+0000","mapping_epoch":188,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":283,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:08.215998+0000","last_change":"2023-09-23T19:38:43.876930+0000","last_active":"2023-09-23T20:02:08.215998+0000","last_peered":"2023-09-23T20:02:08.215998+0000","last_clean":"2023-09-23T20:02:08.215998+0000","last_became_active":"2023-09-23T19:38:43.876571+0000","last_became_peered":"2023-09-23T19:38:43.876571+0000","last_unstale":"2023-09-23T20:02:08.215998+0000","last_undegraded":"2023-09-23T20:02:08.215998+0000","last_fullsized":"2023-09-23T20:02:08.215998+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:30.924536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":252,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:08.262035+0000","last_change":"2023-09-23T19:18:09.015853+0000","last_active":"2023-09-23T20:02:08.262035+0000","last_peered":"2023-09-23T20:02:08.262035+0000","last_clean":"2023-09-23T20:02:08.262035+0000","last_became_active":"2023-09-23T19:18:09.015338+0000","last_became_peered":"2023-09-23T19:18:09.015338+0000","last_unstale":"2023-09-23T20:02:08.262035+0000","last_undegraded":"2023-09-23T20:02:08.262035+0000","last_fullsized":"2023-09-23T20:02:08.262035+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":252,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:07.853999+0000","last_change":"2023-09-23T19:38:43.881233+0000","last_active":"2023-09-23T20:02:07.853999+0000","last_peered":"2023-09-23T20:02:07.853999+0000","last_clean":"2023-09-23T20:02:07.853999+0000","last_became_active":"2023-09-23T19:38:43.880893+0000","last_became_peered":"2023-09-23T19:38:43.880893+0000","last_unstale":"2023-09-23T20:02:07.853999+0000","last_undegraded":"2023-09-23T20:02:07.853999+0000","last_fullsized":"2023-09-23T20:02:07.853999+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:11.206947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":103,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:07.848717+0000","last_change":"2023-09-23T19:47:56.790340+0000","last_active":"2023-09-23T20:02:07.848717+0000","last_peered":"2023-09-23T20:02:07.848717+0000","last_clean":"2023-09-23T20:02:07.848717+0000","last_became_active":"2023-09-23T19:47:56.789785+0000","last_became_peered":"2023-09-23T19:47:56.789785+0000","last_unstale":"2023-09-23T20:02:07.848717+0000","last_undegraded":"2023-09-23T20:02:07.848717+0000","last_fullsized":"2023-09-23T20:02:07.848717+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":186,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:08.262099+0000","last_change":"2023-09-23T19:32:49.737323+0000","last_active":"2023-09-23T20:02:08.262099+0000","last_peered":"2023-09-23T20:02:08.262099+0000","last_clean":"2023-09-23T20:02:08.262099+0000","last_became_active":"2023-09-23T19:32:49.736937+0000","last_became_peered":"2023-09-23T19:32:49.736937+0000","last_unstale":"2023-09-23T20:02:08.262099+0000","last_undegraded":"2023-09-23T20:02:08.262099+0000","last_fullsized":"2023-09-23T20:02:08.262099+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":222,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:07.854080+0000","last_change":"2023-09-23T19:22:42.145506+0000","last_active":"2023-09-23T20:02:07.854080+0000","last_peered":"2023-09-23T20:02:07.854080+0000","last_clean":"2023-09-23T20:02:07.854080+0000","last_became_active":"2023-09-23T19:22:42.143991+0000","last_became_peered":"2023-09-23T19:22:42.143991+0000","last_unstale":"2023-09-23T20:02:07.854080+0000","last_undegraded":"2023-09-23T20:02:07.854080+0000","last_fullsized":"2023-09-23T20:02:07.854080+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:56.875175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":223,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:08.261845+0000","last_change":"2023-09-23T19:32:49.734592+0000","last_active":"2023-09-23T20:02:08.261845+0000","last_peered":"2023-09-23T20:02:08.261845+0000","last_clean":"2023-09-23T20:02:08.261845+0000","last_became_active":"2023-09-23T19:32:49.733905+0000","last_became_peered":"2023-09-23T19:32:49.733905+0000","last_unstale":"2023-09-23T20:02:08.261845+0000","last_undegraded":"2023-09-23T20:02:08.261845+0000","last_fullsized":"2023-09-23T20:02:08.261845+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":267,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:07.856003+0000","last_change":"2023-09-23T19:38:43.895887+0000","last_active":"2023-09-23T20:02:07.856003+0000","last_peered":"2023-09-23T20:02:07.856003+0000","last_clean":"2023-09-23T20:02:07.856003+0000","last_became_active":"2023-09-23T19:38:43.895486+0000","last_became_peered":"2023-09-23T19:38:43.895486+0000","last_unstale":"2023-09-23T20:02:07.856003+0000","last_undegraded":"2023-09-23T20:02:07.856003+0000","last_fullsized":"2023-09-23T20:02:07.856003+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":256,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:08.215851+0000","last_change":"2023-09-23T19:22:42.149694+0000","last_active":"2023-09-23T20:02:08.215851+0000","last_peered":"2023-09-23T20:02:08.215851+0000","last_clean":"2023-09-23T20:02:08.215851+0000","last_became_active":"2023-09-23T19:22:42.148194+0000","last_became_peered":"2023-09-23T19:22:42.148194+0000","last_unstale":"2023-09-23T20:02:08.215851+0000","last_undegraded":"2023-09-23T20:02:08.215851+0000","last_fullsized":"2023-09-23T20:02:08.215851+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:47.842383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":231,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:08.344778+0000","last_change":"2023-09-23T19:05:52.138620+0000","last_active":"2023-09-23T20:02:08.344778+0000","last_peered":"2023-09-23T20:02:08.344778+0000","last_clean":"2023-09-23T20:02:08.344778+0000","last_became_active":"2023-09-23T19:05:52.138243+0000","last_became_peered":"2023-09-23T19:05:52.138243+0000","last_unstale":"2023-09-23T20:02:08.344778+0000","last_undegraded":"2023-09-23T20:02:08.344778+0000","last_fullsized":"2023-09-23T20:02:08.344778+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":238,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:08.215484+0000","last_change":"2023-09-23T19:18:09.008811+0000","last_active":"2023-09-23T20:02:08.215484+0000","last_peered":"2023-09-23T20:02:08.215484+0000","last_clean":"2023-09-23T20:02:08.215484+0000","last_became_active":"2023-09-23T19:18:09.007255+0000","last_became_peered":"2023-09-23T19:18:09.007255+0000","last_unstale":"2023-09-23T20:02:08.215484+0000","last_undegraded":"2023-09-23T20:02:08.215484+0000","last_fullsized":"2023-09-23T20:02:08.215484+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:25.874038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":242,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:07.842317+0000","last_change":"2023-09-23T19:38:43.877244+0000","last_active":"2023-09-23T20:02:07.842317+0000","last_peered":"2023-09-23T20:02:07.842317+0000","last_clean":"2023-09-23T20:02:07.842317+0000","last_became_active":"2023-09-23T19:38:43.876896+0000","last_became_peered":"2023-09-23T19:38:43.876896+0000","last_unstale":"2023-09-23T20:02:07.842317+0000","last_undegraded":"2023-09-23T20:02:07.842317+0000","last_fullsized":"2023-09-23T20:02:07.842317+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:22.288276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":111,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:08.343725+0000","last_change":"2023-09-23T19:43:38.620732+0000","last_active":"2023-09-23T20:02:08.343725+0000","last_peered":"2023-09-23T20:02:08.343725+0000","last_clean":"2023-09-23T20:02:08.343725+0000","last_became_active":"2023-09-23T19:43:38.619424+0000","last_became_peered":"2023-09-23T19:43:38.619424+0000","last_unstale":"2023-09-23T20:02:08.343725+0000","last_undegraded":"2023-09-23T20:02:08.343725+0000","last_fullsized":"2023-09-23T20:02:08.343725+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":285,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:08.345747+0000","last_change":"2023-09-23T19:38:43.947451+0000","last_active":"2023-09-23T20:02:08.345747+0000","last_peered":"2023-09-23T20:02:08.345747+0000","last_clean":"2023-09-23T20:02:08.345747+0000","last_became_active":"2023-09-23T19:38:43.946420+0000","last_became_peered":"2023-09-23T19:38:43.946420+0000","last_unstale":"2023-09-23T20:02:08.345747+0000","last_undegraded":"2023-09-23T20:02:08.345747+0000","last_fullsized":"2023-09-23T20:02:08.345747+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":73,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:08.344444+0000","last_change":"2023-09-23T19:58:26.092680+0000","last_active":"2023-09-23T20:02:08.344444+0000","last_peered":"2023-09-23T20:02:08.344444+0000","last_clean":"2023-09-23T20:02:08.344444+0000","last_became_active":"2023-09-23T19:58:26.092010+0000","last_became_peered":"2023-09-23T19:58:26.092010+0000","last_unstale":"2023-09-23T20:02:08.344444+0000","last_undegraded":"2023-09-23T20:02:08.344444+0000","last_fullsized":"2023-09-23T20:02:08.344444+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":261,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:08.342004+0000","last_change":"2023-09-23T19:28:07.896983+0000","last_active":"2023-09-23T20:02:08.342004+0000","last_peered":"2023-09-23T20:02:08.342004+0000","last_clean":"2023-09-23T20:02:08.342004+0000","last_became_active":"2023-09-23T19:28:07.896577+0000","last_became_peered":"2023-09-23T19:28:07.896577+0000","last_unstale":"2023-09-23T20:02:08.342004+0000","last_undegraded":"2023-09-23T20:02:08.342004+0000","last_fullsized":"2023-09-23T20:02:08.342004+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":191,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:07.855896+0000","last_change":"2023-09-23T19:38:43.900225+0000","last_active":"2023-09-23T20:02:07.855896+0000","last_peered":"2023-09-23T20:02:07.855896+0000","last_clean":"2023-09-23T20:02:07.855896+0000","last_became_active":"2023-09-23T19:38:43.899907+0000","last_became_peered":"2023-09-23T19:38:43.899907+0000","last_unstale":"2023-09-23T20:02:07.855896+0000","last_undegraded":"2023-09-23T20:02:07.855896+0000","last_fullsized":"2023-09-23T20:02:07.855896+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":251,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:07.840825+0000","last_change":"2023-09-23T19:38:43.882487+0000","last_active":"2023-09-23T20:02:07.840825+0000","last_peered":"2023-09-23T20:02:07.840825+0000","last_clean":"2023-09-23T20:02:07.840825+0000","last_became_active":"2023-09-23T19:38:43.881931+0000","last_became_peered":"2023-09-23T19:38:43.881931+0000","last_unstale":"2023-09-23T20:02:07.840825+0000","last_undegraded":"2023-09-23T20:02:07.840825+0000","last_fullsized":"2023-09-23T20:02:07.840825+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:02:36.906535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":272,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:08.215755+0000","last_change":"2023-09-23T19:38:43.893983+0000","last_active":"2023-09-23T20:02:08.215755+0000","last_peered":"2023-09-23T20:02:08.215755+0000","last_clean":"2023-09-23T20:02:08.215755+0000","last_became_active":"2023-09-23T19:38:43.893527+0000","last_became_peered":"2023-09-23T19:38:43.893527+0000","last_unstale":"2023-09-23T20:02:08.215755+0000","last_undegraded":"2023-09-23T20:02:08.215755+0000","last_fullsized":"2023-09-23T20:02:08.215755+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:12.156531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":240,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:02:08.342930+0000","last_change":"2023-09-23T19:06:36.673002+0000","last_active":"2023-09-23T20:02:08.342930+0000","last_peered":"2023-09-23T20:02:08.342930+0000","last_clean":"2023-09-23T20:02:08.342930+0000","last_became_active":"2023-09-23T19:06:36.671911+0000","last_became_peered":"2023-09-23T19:06:36.671911+0000","last_unstale":"2023-09-23T20:02:08.342930+0000","last_undegraded":"2023-09-23T20:02:08.342930+0000","last_fullsized":"2023-09-23T20:02:08.342930+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2,"ondisk_log_size":2,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430922,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1175036,"kb_used_data":3060,"kb_used_omap":0,"kb_used_meta":1171968,"kb_avail":92570116,"statfs":{"total":95995035648,"available":94791798784,"internally_reserved":0,"allocated":3133440,"data_stored":2569811,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1200095232},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:01:26 2023","interfaces":[{"interface":"back","average":{"1min":2.3679999999999999,"5min":1.95,"15min":1.526},"min":{"1min":0.28399999999999997,"5min":0.27400000000000002,"15min":0.26700000000000002},"max":{"1min":10.33,"5min":41.570999999999998,"15min":41.570999999999998},"last":0.79800000000000004},{"interface":"front","average":{"1min":2.649,"5min":2.234,"15min":1.6799999999999999},"min":{"1min":0.34000000000000002,"5min":0.29699999999999999,"15min":0.28000000000000003},"max":{"1min":7.0960000000000001,"5min":45.634,"15min":45.634},"last":0.95699999999999996}]},{"osd":1,"last update":"Sat Sep 23 20:01:43 2023","interfaces":[{"interface":"back","average":{"1min":1.802,"5min":2.3999999999999999,"15min":2.339},"min":{"1min":0.32600000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":7.7210000000000001,"5min":43.351999999999997,"15min":149.018},"last":0.86299999999999999},{"interface":"front","average":{"1min":1.49,"5min":2.3620000000000001,"15min":2.2890000000000001},"min":{"1min":0.35099999999999998,"5min":0.32100000000000001,"15min":0.254},"max":{"1min":6.1749999999999998,"5min":43.381999999999998,"15min":148.995},"last":1.119}]},{"osd":2,"last update":"Sat Sep 23 20:01:26 2023","interfaces":[{"interface":"back","average":{"1min":11.356999999999999,"5min":4.1689999999999996,"15min":4.1559999999999997},"min":{"1min":0.33900000000000002,"5min":0.32300000000000001,"15min":0.308},"max":{"1min":160.655,"5min":160.655,"15min":246.982},"last":1.0209999999999999},{"interface":"front","average":{"1min":12.577999999999999,"5min":4.6580000000000004,"15min":4.3250000000000002},"min":{"1min":0.34000000000000002,"5min":0.30299999999999999,"15min":0.30199999999999999},"max":{"1min":176.428,"5min":176.428,"15min":247.751},"last":0.76900000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:02:17 2023","interfaces":[{"interface":"back","average":{"1min":0.98599999999999999,"5min":1.2989999999999999,"15min":1.5720000000000001},"min":{"1min":0.60799999999999998,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.2549999999999999,"5min":8.4979999999999993,"15min":10.615},"last":1.139},{"interface":"front","average":{"1min":1.0740000000000001,"5min":1.4219999999999999,"15min":1.599},"min":{"1min":0.65900000000000003,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":1.3080000000000001,"5min":8.5220000000000002,"15min":9.9179999999999993},"last":1.2290000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:01:43 2023","interfaces":[{"interface":"back","average":{"1min":1.143,"5min":1.1579999999999999,"15min":1.5529999999999999},"min":{"1min":0.34599999999999997,"5min":0.32500000000000001,"15min":0.27700000000000002},"max":{"1min":3.3889999999999998,"5min":6.9169999999999998,"15min":13.484999999999999},"last":1.0780000000000001},{"interface":"front","average":{"1min":1.2669999999999999,"5min":1.0580000000000001,"15min":1.4410000000000001},"min":{"1min":0.41399999999999998,"5min":0.313,"15min":0.313},"max":{"1min":5.6619999999999999,"5min":7.2030000000000003,"15min":13.116},"last":1.1839999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:01:43 2023","interfaces":[{"interface":"back","average":{"1min":1.397,"5min":1.1830000000000001,"15min":1.4450000000000001},"min":{"1min":0.378,"5min":0.34799999999999998,"15min":0.28499999999999998},"max":{"1min":4.298,"5min":8.1349999999999998,"15min":13.321999999999999},"last":1.089},{"interface":"front","average":{"1min":0.84099999999999997,"5min":0.995,"15min":1.4399999999999999},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.315},"max":{"1min":1.3240000000000001,"5min":5.1539999999999999,"15min":13.478999999999999},"last":1.206}]},{"osd":6,"last update":"Sat Sep 23 20:01:43 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":0.91200000000000003,"15min":1.343},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.33900000000000002},"max":{"1min":2.3029999999999999,"5min":5.1980000000000004,"15min":13.436},"last":1.2549999999999999},{"interface":"front","average":{"1min":1.3260000000000001,"5min":1.0820000000000001,"15min":1.3759999999999999},"min":{"1min":0.46800000000000003,"5min":0.373,"15min":0.35599999999999998},"max":{"1min":5.6130000000000004,"5min":7.2599999999999998,"15min":10.766999999999999},"last":1.276}]}]},{"osd":6,"up_from":42,"seq":180388627153,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":886060,"kb_used_data":3044,"kb_used_omap":0,"kb_used_meta":883008,"kb_avail":92859092,"statfs":{"total":95995035648,"available":95087710208,"internally_reserved":0,"allocated":3117056,"data_stored":2569350,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":904200192},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:01:35 2023","interfaces":[{"interface":"back","average":{"1min":1.901,"5min":5.1520000000000001,"15min":2.863},"min":{"1min":0.29099999999999998,"5min":0.28699999999999998,"15min":0.23799999999999999},"max":{"1min":7.476,"5min":125.34399999999999,"15min":125.34399999999999},"last":0.54300000000000004},{"interface":"front","average":{"1min":1.5,"5min":5.1040000000000001,"15min":2.887},"min":{"1min":0.34799999999999998,"5min":0.27800000000000002,"15min":0.26600000000000001},"max":{"1min":4.6550000000000002,"5min":125.24299999999999,"15min":125.24299999999999},"last":1.1930000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:01:59 2023","interfaces":[{"interface":"back","average":{"1min":1.0640000000000001,"5min":2.4700000000000002,"15min":1.901},"min":{"1min":0.35199999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":4.343,"5min":125.51900000000001,"15min":125.51900000000001},"last":1.0569999999999999},{"interface":"front","average":{"1min":0.96299999999999997,"5min":2.5910000000000002,"15min":2.0329999999999999},"min":{"1min":0.442,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":2.4540000000000002,"5min":125.596,"15min":125.596},"last":1.117}]},{"osd":2,"last update":"Sat Sep 23 20:01:21 2023","interfaces":[{"interface":"back","average":{"1min":2.0859999999999999,"5min":2.8439999999999999,"15min":4.1890000000000001},"min":{"1min":0.309,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":5.9589999999999996,"5min":125.01600000000001,"15min":219.453},"last":0.79600000000000004},{"interface":"front","average":{"1min":2.1739999999999999,"5min":3.0720000000000001,"15min":4.2220000000000004},"min":{"1min":0.33900000000000002,"5min":0.33700000000000002,"15min":0.30199999999999999},"max":{"1min":4.569,"5min":125.122,"15min":219.369},"last":0.623}]},{"osd":3,"last update":"Sat Sep 23 20:01:21 2023","interfaces":[{"interface":"back","average":{"1min":2.109,"5min":1.7729999999999999,"15min":1.7509999999999999},"min":{"1min":0.46800000000000003,"5min":0.32300000000000001,"15min":0.28599999999999998},"max":{"1min":6.109,"5min":13.823,"15min":16.428000000000001},"last":0.65400000000000003},{"interface":"front","average":{"1min":1.903,"5min":1.673,"15min":1.7070000000000001},"min":{"1min":0.39700000000000002,"5min":0.32600000000000001,"15min":0.311},"max":{"1min":9.6910000000000007,"5min":14.42,"15min":14.42},"last":0.95699999999999996}]},{"osd":4,"last update":"Sat Sep 23 20:01:21 2023","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":1.1220000000000001,"15min":1.5600000000000001},"min":{"1min":0.33900000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":4.4390000000000001,"5min":6.1719999999999997,"15min":14.401999999999999},"last":1.2589999999999999},{"interface":"front","average":{"1min":1.5189999999999999,"5min":1.464,"15min":1.73},"min":{"1min":0.35299999999999998,"5min":0.33900000000000002,"15min":0.28299999999999997},"max":{"1min":6.3289999999999997,"5min":11.08,"15min":14.603},"last":0.98999999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:01:21 2023","interfaces":[{"interface":"back","average":{"1min":0.98399999999999999,"5min":1.1539999999999999,"15min":1.643},"min":{"1min":0.436,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":3.3809999999999998,"5min":6.4539999999999997,"15min":15.045},"last":1.024},{"interface":"front","average":{"1min":1.5289999999999999,"5min":1.3560000000000001,"15min":1.8620000000000001},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.33400000000000002},"max":{"1min":6.3140000000000001,"5min":11.048999999999999,"15min":16.244},"last":0.84399999999999997}]},{"osd":7,"last update":"Sat Sep 23 20:01:39 2023","interfaces":[{"interface":"back","average":{"1min":0.96299999999999997,"5min":1.171,"15min":1.6339999999999999},"min":{"1min":0.36299999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":2.484,"5min":5.0359999999999996,"15min":11.709},"last":0.87},{"interface":"front","average":{"1min":1.6719999999999999,"5min":1.367,"15min":1.617},"min":{"1min":0.43099999999999999,"5min":0.371,"15min":0.32200000000000001},"max":{"1min":6.3499999999999996,"5min":11.000999999999999,"15min":12.327999999999999},"last":0.77300000000000002}]}]},{"osd":1,"up_from":98,"seq":420906795485,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":921972,"kb_used_data":3116,"kb_used_omap":4,"kb_used_meta":918843,"kb_avail":92823180,"statfs":{"total":95995035648,"available":95050936320,"internally_reserved":0,"allocated":3190784,"data_stored":2047736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":940896142},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:01:56 2023","interfaces":[{"interface":"back","average":{"1min":1.4430000000000001,"5min":3.133,"15min":2.7879999999999998},"min":{"1min":0.26600000000000001,"5min":0.24199999999999999,"15min":0.221},"max":{"1min":8.7959999999999994,"5min":46.527999999999999,"15min":134.37700000000001},"last":0.80400000000000005},{"interface":"front","average":{"1min":1.6459999999999999,"5min":2.9649999999999999,"15min":2.6349999999999998},"min":{"1min":0.36199999999999999,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":9.4100000000000001,"5min":41.683999999999997,"15min":135.79400000000001},"last":0.90600000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:01:27 2023","interfaces":[{"interface":"back","average":{"1min":3.4409999999999998,"5min":3.4649999999999999,"15min":3.1349999999999998},"min":{"1min":0.378,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":16,"5min":41.719000000000001,"15min":72.859999999999999},"last":1.085},{"interface":"front","average":{"1min":2.6400000000000001,"5min":2.73,"15min":2.6549999999999998},"min":{"1min":0.32100000000000001,"5min":0.28299999999999997,"15min":0.224},"max":{"1min":8.7780000000000005,"5min":47.439,"15min":73.665999999999997},"last":1.048}]},{"osd":3,"last update":"Sat Sep 23 20:01:20 2023","interfaces":[{"interface":"back","average":{"1min":2.2709999999999999,"5min":2.6829999999999998,"15min":2.0299999999999998},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.25},"max":{"1min":8.6760000000000002,"5min":41.518000000000001,"15min":41.518000000000001},"last":1.1100000000000001},{"interface":"front","average":{"1min":2.1680000000000001,"5min":3.1360000000000001,"15min":2.1619999999999999},"min":{"1min":0.252,"5min":0.252,"15min":0.217},"max":{"1min":10.558,"5min":41.151000000000003,"15min":41.151000000000003},"last":0.95499999999999996}]},{"osd":4,"last update":"Sat Sep 23 20:01:56 2023","interfaces":[{"interface":"back","average":{"1min":1.774,"5min":2.4100000000000001,"15min":2.1480000000000001},"min":{"1min":0.53700000000000003,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":7.1360000000000001,"5min":39.085999999999999,"15min":48.381},"last":1.153},{"interface":"front","average":{"1min":1.248,"5min":2.419,"15min":2.2000000000000002},"min":{"1min":0.374,"5min":0.36599999999999999,"15min":0.32200000000000001},"max":{"1min":5.0350000000000001,"5min":41.320999999999998,"15min":82.239999999999995},"last":1.071}]},{"osd":5,"last update":"Sat Sep 23 20:01:56 2023","interfaces":[{"interface":"back","average":{"1min":1.0369999999999999,"5min":2.2320000000000002,"15min":1.994},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.36199999999999999},"max":{"1min":3.323,"5min":41.274999999999999,"15min":48.643000000000001},"last":1.0329999999999999},{"interface":"front","average":{"1min":1.7110000000000001,"5min":2.5960000000000001,"15min":2.0670000000000002},"min":{"1min":0.41899999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":8.7539999999999996,"5min":43.508000000000003,"15min":48.619999999999997},"last":1.425}]},{"osd":6,"last update":"Sat Sep 23 20:01:56 2023","interfaces":[{"interface":"back","average":{"1min":2.0790000000000002,"5min":2.899,"15min":2.419},"min":{"1min":0.39500000000000002,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":8.7289999999999992,"5min":46.472000000000001,"15min":48.548999999999999},"last":1.327},{"interface":"front","average":{"1min":2.0880000000000001,"5min":3.0110000000000001,"15min":2.242},"min":{"1min":0.64700000000000002,"5min":0.35199999999999998,"15min":0.313},"max":{"1min":7.2759999999999998,"5min":41.648000000000003,"15min":41.648000000000003},"last":1.506}]},{"osd":7,"last update":"Sat Sep 23 20:01:56 2023","interfaces":[{"interface":"back","average":{"1min":1.923,"5min":2.7120000000000002,"15min":2.331},"min":{"1min":0.749,"5min":0.38300000000000001,"15min":0.36799999999999999},"max":{"1min":7.2220000000000004,"5min":41.595999999999997,"15min":48.329999999999998},"last":1.385},{"interface":"front","average":{"1min":1.337,"5min":2.3119999999999998,"15min":2.2160000000000002},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.35299999999999998},"max":{"1min":4.9989999999999997,"5min":41.223999999999997,"15min":48.579000000000001},"last":1.4650000000000001}]}]},{"osd":0,"up_from":85,"seq":365072220692,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":997668,"kb_used_data":3676,"kb_used_omap":5,"kb_used_meta":993978,"kb_avail":92747484,"statfs":{"total":95995035648,"available":94973423616,"internally_reserved":0,"allocated":3764224,"data_stored":2639615,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1017834106},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:02:02 2023","interfaces":[{"interface":"back","average":{"1min":1.2470000000000001,"5min":2.645,"15min":2.589},"min":{"1min":0.374,"5min":0.219,"15min":0.216},"max":{"1min":7.423,"5min":14.960000000000001,"15min":149.37299999999999},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.96799999999999997,"5min":2.6000000000000001,"15min":2.6560000000000001},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":5.5369999999999999,"5min":13.534000000000001,"15min":147.119},"last":0.42599999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:01:24 2023","interfaces":[{"interface":"back","average":{"1min":2.5859999999999999,"5min":2.528,"15min":2.75},"min":{"1min":0.46300000000000002,"5min":0.28899999999999998,"15min":0.245},"max":{"1min":8.7089999999999996,"5min":11.646000000000001,"15min":213.20599999999999},"last":0.61499999999999999},{"interface":"front","average":{"1min":2.5859999999999999,"5min":1.891,"15min":2.3839999999999999},"min":{"1min":0.39900000000000002,"5min":0.25700000000000001,"15min":0.24199999999999999},"max":{"1min":8.6899999999999995,"5min":11.542,"15min":209.64400000000001},"last":0.50800000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:02:02 2023","interfaces":[{"interface":"back","average":{"1min":1.742,"5min":2.0859999999999999,"15min":1.7509999999999999},"min":{"1min":0.59499999999999997,"5min":0.28100000000000003,"15min":0.222},"max":{"1min":6.5780000000000003,"5min":13.141,"15min":14.983000000000001},"last":0.57999999999999996},{"interface":"front","average":{"1min":1.27,"5min":2.069,"15min":1.714},"min":{"1min":0.48399999999999999,"5min":0.29999999999999999,"15min":0.28999999999999998},"max":{"1min":5.6289999999999996,"5min":14,"15min":14.843},"last":0.59599999999999997}]},{"osd":4,"last update":"Sat Sep 23 20:01:41 2023","interfaces":[{"interface":"back","average":{"1min":1.7629999999999999,"5min":1.736,"15min":1.546},"min":{"1min":0.495,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":6.6769999999999996,"5min":11.398,"15min":11.398},"last":0.71699999999999997},{"interface":"front","average":{"1min":1.6639999999999999,"5min":1.883,"15min":1.661},"min":{"1min":0.38,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":7.9729999999999999,"5min":10.747,"15min":10.747},"last":0.86099999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:01:41 2023","interfaces":[{"interface":"back","average":{"1min":1.726,"5min":1.8320000000000001,"15min":1.647},"min":{"1min":0.34599999999999997,"5min":0.316,"15min":0.316},"max":{"1min":5,"5min":10.69,"15min":10.69},"last":0.98199999999999998},{"interface":"front","average":{"1min":1.8500000000000001,"5min":1.736,"15min":1.5940000000000001},"min":{"1min":0.46300000000000002,"5min":0.36499999999999999,"15min":0.34100000000000003},"max":{"1min":6.6200000000000001,"5min":11.510999999999999,"15min":11.510999999999999},"last":0.75800000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:01:41 2023","interfaces":[{"interface":"back","average":{"1min":1.821,"5min":1.8480000000000001,"15min":1.873},"min":{"1min":0.46300000000000002,"5min":0.39000000000000001,"15min":0.36699999999999999},"max":{"1min":6.6459999999999999,"5min":11.443,"15min":66.170000000000002},"last":0.80600000000000005},{"interface":"front","average":{"1min":1.748,"5min":2.0099999999999998,"15min":2.036},"min":{"1min":0.41199999999999998,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":4.8680000000000003,"5min":17.050999999999998,"15min":66.260999999999996},"last":0.94499999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:01:41 2023","interfaces":[{"interface":"back","average":{"1min":1.335,"5min":1.9379999999999999,"15min":2.375},"min":{"1min":0.61099999999999999,"5min":0.42099999999999999,"15min":0.40400000000000003},"max":{"1min":4.4800000000000004,"5min":10.419,"15min":210.47399999999999},"last":0.88200000000000001},{"interface":"front","average":{"1min":1.8779999999999999,"5min":1.9650000000000001,"15min":2.4460000000000002},"min":{"1min":0.432,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":6.7089999999999996,"5min":11.475,"15min":210.52199999999999},"last":0.78500000000000003}]}]},{"osd":2,"up_from":159,"seq":682899800348,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":732688,"kb_used_data":3080,"kb_used_omap":7,"kb_used_meta":729592,"kb_avail":93012464,"statfs":{"total":95995035648,"available":95244763136,"internally_reserved":0,"allocated":3153920,"data_stored":2041973,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":747102289},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:01:39 2023","interfaces":[{"interface":"back","average":{"1min":0.97599999999999998,"5min":2.1819999999999999,"15min":2.4009999999999998},"min":{"1min":0.28399999999999997,"5min":0.19700000000000001,"15min":0.17299999999999999},"max":{"1min":4.3959999999999999,"5min":14.624000000000001,"15min":117.77500000000001},"last":0.72399999999999998},{"interface":"front","average":{"1min":3.2370000000000001,"5min":2.6800000000000002,"15min":2.7679999999999998},"min":{"1min":0.311,"5min":0.223,"15min":0.214},"max":{"1min":40.539000000000001,"5min":40.539000000000001,"15min":117.91800000000001},"last":0.755}]},{"osd":1,"last update":"Sat Sep 23 20:01:39 2023","interfaces":[{"interface":"back","average":{"1min":2.6190000000000002,"5min":2.5649999999999999,"15min":2.9849999999999999},"min":{"1min":0.26400000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":13.989000000000001,"5min":13.989000000000001,"15min":114.107},"last":0.495},{"interface":"front","average":{"1min":1.21,"5min":1.976,"15min":2.5539999999999998},"min":{"1min":0.248,"5min":0.248,"15min":0.23699999999999999},"max":{"1min":13.737,"5min":13.737,"15min":117.172},"last":0.66300000000000003}]},{"osd":3,"last update":"Sat Sep 23 20:01:39 2023","interfaces":[{"interface":"back","average":{"1min":2.4550000000000001,"5min":2.2120000000000002,"15min":2.1280000000000001},"min":{"1min":0.25,"5min":0.25,"15min":0.24099999999999999},"max":{"1min":14.045,"5min":14.045,"15min":17.751000000000001},"last":0.627},{"interface":"front","average":{"1min":1.339,"5min":1.8169999999999999,"15min":1.927},"min":{"1min":0.32300000000000001,"5min":0.215,"15min":0.215},"max":{"1min":13.798999999999999,"5min":13.798999999999999,"15min":20.286999999999999},"last":0.79500000000000004}]},{"osd":4,"last update":"Sat Sep 23 20:01:39 2023","interfaces":[{"interface":"back","average":{"1min":1.7430000000000001,"5min":1.6000000000000001,"15min":1.9259999999999999},"min":{"1min":0.379,"5min":0.33800000000000002,"15min":0.27100000000000002},"max":{"1min":7.6589999999999998,"5min":8.202,"15min":14.194000000000001},"last":1.0600000000000001},{"interface":"front","average":{"1min":2.444,"5min":1.851,"15min":2.1520000000000001},"min":{"1min":0.33300000000000002,"5min":0.32600000000000001,"15min":0.31900000000000001},"max":{"1min":14.138999999999999,"5min":14.138999999999999,"15min":14.271000000000001},"last":1.0740000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:01:39 2023","interfaces":[{"interface":"back","average":{"1min":0.84999999999999998,"5min":1.5920000000000001,"15min":1.696},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.27600000000000002},"max":{"1min":4.0960000000000001,"5min":14.007,"15min":14.007},"last":0.82799999999999996},{"interface":"front","average":{"1min":1.869,"5min":1.708,"15min":1.7170000000000001},"min":{"1min":0.41199999999999998,"5min":0.377,"15min":0.32100000000000001},"max":{"1min":7.6100000000000003,"5min":8.2720000000000002,"15min":10.193},"last":0.91500000000000004}]},{"osd":6,"last update":"Sat Sep 23 20:01:39 2023","interfaces":[{"interface":"back","average":{"1min":1.748,"5min":1.7130000000000001,"15min":1.9199999999999999},"min":{"1min":0.44,"5min":0.39500000000000002,"15min":0.35699999999999998},"max":{"1min":7.6909999999999998,"5min":8.2390000000000008,"15min":10.166},"last":0.81299999999999994},{"interface":"front","average":{"1min":2.548,"5min":1.909,"15min":2.1869999999999998},"min":{"1min":0.45400000000000001,"5min":0.371,"15min":0.371},"max":{"1min":14.077999999999999,"5min":14.077999999999999,"15min":14.077999999999999},"last":1.0029999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:01:39 2023","interfaces":[{"interface":"back","average":{"1min":1.1639999999999999,"5min":1.819,"15min":1.8839999999999999},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.38500000000000001},"max":{"1min":9.2680000000000007,"5min":14.058,"15min":32.945},"last":0.94499999999999995},{"interface":"front","average":{"1min":2.782,"5min":2.0350000000000001,"15min":2.1499999999999999},"min":{"1min":0.47299999999999998,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":14.109999999999999,"5min":14.109999999999999,"15min":16.593},"last":1.1000000000000001}]}]},{"osd":3,"up_from":116,"seq":498216206748,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":833080,"kb_used_data":2992,"kb_used_omap":2,"kb_used_meta":830077,"kb_avail":92912072,"statfs":{"total":95995035648,"available":95141961728,"internally_reserved":0,"allocated":3063808,"data_stored":2031963,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":849998901},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:02:12 2023","interfaces":[{"interface":"back","average":{"1min":0.67900000000000005,"5min":4.7089999999999996,"15min":2.8130000000000002},"min":{"1min":0.26500000000000001,"5min":0.20699999999999999,"15min":0.19600000000000001},"max":{"1min":3.0950000000000002,"5min":117.248,"15min":117.248},"last":0.45400000000000001},{"interface":"front","average":{"1min":1.29,"5min":4.6020000000000003,"15min":2.9079999999999999},"min":{"1min":0.26800000000000002,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":14.587,"5min":116.187,"15min":116.187},"last":0.37}]},{"osd":1,"last update":"Sat Sep 23 20:02:12 2023","interfaces":[{"interface":"back","average":{"1min":1.2270000000000001,"5min":2.391,"15min":2.0019999999999998},"min":{"1min":0.33700000000000002,"5min":0.28399999999999997,"15min":0.23400000000000001},"max":{"1min":10.066000000000001,"5min":12.243,"15min":48.918999999999997},"last":0.42999999999999999},{"interface":"front","average":{"1min":1.3169999999999999,"5min":2.0390000000000001,"15min":1.859},"min":{"1min":0.27300000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":14.795,"5min":14.795,"15min":50.722999999999999},"last":0.59299999999999997}]},{"osd":2,"last update":"Sat Sep 23 20:01:32 2023","interfaces":[{"interface":"back","average":{"1min":2.3130000000000002,"5min":2.3580000000000001,"15min":2.8109999999999999},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":14.529,"5min":14.529,"15min":116.363},"last":0.52800000000000002},{"interface":"front","average":{"1min":2.8719999999999999,"5min":2.407,"15min":2.8100000000000001},"min":{"1min":0.35699999999999998,"5min":0.30599999999999999,"15min":0.27300000000000002},"max":{"1min":12.832000000000001,"5min":12.832000000000001,"15min":118.474},"last":0.39600000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:02:12 2023","interfaces":[{"interface":"back","average":{"1min":0.879,"5min":1.6850000000000001,"15min":1.6619999999999999},"min":{"1min":0.55800000000000005,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":1.522,"5min":10.16,"15min":10.16},"last":0.88700000000000001},{"interface":"front","average":{"1min":0.93799999999999994,"5min":1.95,"15min":1.859},"min":{"1min":0.501,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":1.6240000000000001,"5min":12.169,"15min":12.169},"last":0.90200000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:02:12 2023","interfaces":[{"interface":"back","average":{"1min":0.88900000000000001,"5min":1.897,"15min":1.831},"min":{"1min":0.39400000000000002,"5min":0.35599999999999998,"15min":0.33900000000000002},"max":{"1min":1.552,"5min":11.978,"15min":11.978},"last":0.84399999999999997},{"interface":"front","average":{"1min":1.1850000000000001,"5min":1.712,"15min":1.6599999999999999},"min":{"1min":0.36299999999999999,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":5.9450000000000003,"5min":12.741,"15min":12.741},"last":0.97699999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:02:12 2023","interfaces":[{"interface":"back","average":{"1min":0.94099999999999995,"5min":2.2410000000000001,"15min":2.1059999999999999},"min":{"1min":0.43099999999999999,"5min":0.42199999999999999,"15min":0.41199999999999998},"max":{"1min":1.417,"5min":12.135,"15min":12.135},"last":0.95499999999999996},{"interface":"front","average":{"1min":1.173,"5min":1.7929999999999999,"15min":1.831},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":6.0179999999999998,"5min":12.776999999999999,"15min":12.776999999999999},"last":0.77800000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:02:12 2023","interfaces":[{"interface":"back","average":{"1min":1.2350000000000001,"5min":1.7909999999999999,"15min":1.921},"min":{"1min":0.64200000000000002,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":6.0510000000000002,"5min":12.802,"15min":12.802},"last":0.82499999999999996},{"interface":"front","average":{"1min":0.92700000000000005,"5min":1.577,"15min":1.7170000000000001},"min":{"1min":0.50600000000000001,"5min":0.437,"15min":0.40600000000000003},"max":{"1min":1.387,"5min":9.9890000000000008,"15min":9.9890000000000008},"last":0.80200000000000005}]}]},{"osd":4,"up_from":30,"seq":128849019618,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1190772,"kb_used_data":2476,"kb_used_omap":0,"kb_used_meta":1188288,"kb_avail":92554380,"statfs":{"total":95995035648,"available":94775685120,"internally_reserved":0,"allocated":2535424,"data_stored":1979373,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1216806912},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:01:37 2023","interfaces":[{"interface":"back","average":{"1min":0.72899999999999998,"5min":1.7509999999999999,"15min":2.117},"min":{"1min":0.27500000000000002,"5min":0.26000000000000001,"15min":0.247},"max":{"1min":1.482,"5min":23.059999999999999,"15min":150.155},"last":1.0669999999999999},{"interface":"front","average":{"1min":1.157,"5min":2.161,"15min":2.1099999999999999},"min":{"1min":0.441,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":4.6050000000000004,"5min":22.780000000000001,"15min":149.47900000000001},"last":0.79400000000000004}]},{"osd":1,"last update":"Sat Sep 23 20:02:06 2023","interfaces":[{"interface":"back","average":{"1min":1.323,"5min":2.5150000000000001,"15min":1.7869999999999999},"min":{"1min":0.55100000000000005,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":5.7069999999999999,"5min":30.754000000000001,"15min":30.754000000000001},"last":1.026},{"interface":"front","average":{"1min":1.6020000000000001,"5min":2.4849999999999999,"15min":1.8240000000000001},"min":{"1min":0.56200000000000006,"5min":0.36499999999999999,"15min":0.27600000000000002},"max":{"1min":6.117,"5min":30.777999999999999,"15min":30.777999999999999},"last":1.006}]},{"osd":2,"last update":"Sat Sep 23 20:01:32 2023","interfaces":[{"interface":"back","average":{"1min":1.9319999999999999,"5min":1.925,"15min":3.6309999999999998},"min":{"1min":0.41499999999999998,"5min":0.307,"15min":0.27600000000000002},"max":{"1min":7.2439999999999998,"5min":30.088999999999999,"15min":219.02799999999999},"last":0.95899999999999996},{"interface":"front","average":{"1min":1.756,"5min":5.4690000000000003,"15min":4.806},"min":{"1min":0.34000000000000002,"5min":0.32300000000000001,"15min":0.308},"max":{"1min":9.0700000000000003,"5min":314.31599999999997,"15min":314.31599999999997},"last":0.69399999999999995}]},{"osd":3,"last update":"Sat Sep 23 20:01:32 2023","interfaces":[{"interface":"back","average":{"1min":1.839,"5min":1.538,"15min":1.5860000000000001},"min":{"1min":0.372,"5min":0.35199999999999998,"15min":0.308},"max":{"1min":5.9850000000000003,"5min":8.5329999999999995,"15min":17.873999999999999},"last":0.65100000000000002},{"interface":"front","average":{"1min":2.157,"5min":1.6919999999999999,"15min":1.6180000000000001},"min":{"1min":0.439,"5min":0.35099999999999998,"15min":0.32300000000000001},"max":{"1min":5.3860000000000001,"5min":9.5280000000000005,"15min":13.859},"last":0.85399999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:01:37 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":1.1599999999999999,"15min":1.4199999999999999},"min":{"1min":0.31,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":6.0570000000000004,"5min":6.952,"15min":9.4740000000000002},"last":0.89400000000000002},{"interface":"front","average":{"1min":0.79500000000000004,"5min":0.92300000000000004,"15min":1.4430000000000001},"min":{"1min":0.41399999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":1.4359999999999999,"5min":5.2729999999999997,"15min":12.506},"last":1.1479999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:02:06 2023","interfaces":[{"interface":"back","average":{"1min":1.3140000000000001,"5min":1.1140000000000001,"15min":1.5349999999999999},"min":{"1min":0.39600000000000002,"5min":0.32400000000000001,"15min":0.29399999999999998},"max":{"1min":6.0890000000000004,"5min":6.0890000000000004,"15min":18.103000000000002},"last":0.746},{"interface":"front","average":{"1min":1.0449999999999999,"5min":0.96099999999999997,"15min":1.5549999999999999},"min":{"1min":0.45900000000000002,"5min":0.36799999999999999,"15min":0.31},"max":{"1min":1.3400000000000001,"5min":5.3150000000000004,"15min":19.706},"last":1.1899999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:02:06 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":1.323,"15min":1.5029999999999999},"min":{"1min":0.38800000000000001,"5min":0.35199999999999998,"15min":0.308},"max":{"1min":1.5429999999999999,"5min":12.475,"15min":12.475},"last":1.095},{"interface":"front","average":{"1min":1.341,"5min":1.1719999999999999,"15min":1.571},"min":{"1min":0.504,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":6.0229999999999997,"5min":6.9210000000000003,"15min":12.169},"last":1.113}]}]},{"osd":5,"up_from":36,"seq":154618823386,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1193628,"kb_used_data":2452,"kb_used_omap":0,"kb_used_meta":1191168,"kb_avail":92551524,"statfs":{"total":95995035648,"available":94772760576,"internally_reserved":0,"allocated":2510848,"data_stored":1974487,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1219756032},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:01:33 2023","interfaces":[{"interface":"back","average":{"1min":1.843,"5min":5.8470000000000004,"15min":2.915},"min":{"1min":0.29199999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":9.0169999999999995,"5min":201.196,"15min":201.196},"last":0.74199999999999999},{"interface":"front","average":{"1min":0.878,"5min":5.9210000000000003,"15min":3.2690000000000001},"min":{"1min":0.28000000000000003,"5min":0.27700000000000002,"15min":0.251},"max":{"1min":2.8220000000000001,"5min":192.941,"15min":192.941},"last":0.70699999999999996}]},{"osd":1,"last update":"Sat Sep 23 20:02:04 2023","interfaces":[{"interface":"back","average":{"1min":1.1659999999999999,"5min":7.8159999999999998,"15min":4.2850000000000001},"min":{"1min":0.54600000000000004,"5min":0.28599999999999998,"15min":0.28000000000000003},"max":{"1min":5.8869999999999996,"5min":190.298,"15min":190.298},"last":0.90500000000000003},{"interface":"front","average":{"1min":0.77600000000000002,"5min":7.6379999999999999,"15min":4.4329999999999998},"min":{"1min":0.34200000000000003,"5min":0.32700000000000001,"15min":0.28799999999999998},"max":{"1min":1.0920000000000001,"5min":190.65799999999999,"15min":190.65799999999999},"last":0.98599999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:02:17 2023","interfaces":[{"interface":"back","average":{"1min":0.79600000000000004,"5min":4.3220000000000001,"15min":4.0259999999999998},"min":{"1min":0.60399999999999998,"5min":0.311,"15min":0.309},"max":{"1min":1.1970000000000001,"5min":151.322,"15min":241.59700000000001},"last":1.0369999999999999},{"interface":"front","average":{"1min":1.0029999999999999,"5min":4.6449999999999996,"15min":4.0549999999999997},"min":{"1min":0.72099999999999997,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":1.2150000000000001,"5min":166.941,"15min":242.13999999999999},"last":1.016}]},{"osd":3,"last update":"Sat Sep 23 20:02:04 2023","interfaces":[{"interface":"back","average":{"1min":0.82299999999999995,"5min":1.397,"15min":1.4510000000000001},"min":{"1min":0.44400000000000001,"5min":0.35399999999999998,"15min":0.35099999999999998},"max":{"1min":1.206,"5min":7.7290000000000001,"15min":9.3829999999999991},"last":0.79100000000000004},{"interface":"front","average":{"1min":1.0800000000000001,"5min":1.5109999999999999,"15min":1.4399999999999999},"min":{"1min":0.51300000000000001,"5min":0.35199999999999998,"15min":0.33400000000000002},"max":{"1min":2.222,"5min":13.840999999999999,"15min":13.840999999999999},"last":1.141}]},{"osd":4,"last update":"Sat Sep 23 20:01:33 2023","interfaces":[{"interface":"back","average":{"1min":0.82699999999999996,"5min":1.119,"15min":1.601},"min":{"1min":0.318,"5min":0.28399999999999997,"15min":0.26800000000000002},"max":{"1min":2.8519999999999999,"5min":7.2110000000000003,"15min":12.051},"last":0.85699999999999998},{"interface":"front","average":{"1min":0.85999999999999999,"5min":1.0489999999999999,"15min":1.403},"min":{"1min":0.34100000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":2.6850000000000001,"5min":10.388,"15min":16.462},"last":0.94599999999999995}]},{"osd":6,"last update":"Sat Sep 23 20:02:17 2023","interfaces":[{"interface":"back","average":{"1min":1.0669999999999999,"5min":0.94199999999999995,"15min":1.3500000000000001},"min":{"1min":0.63400000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":1.4039999999999999,"5min":5.702,"15min":9.641},"last":1.2290000000000001},{"interface":"front","average":{"1min":1.1040000000000001,"5min":1.028,"15min":1.3999999999999999},"min":{"1min":0.83899999999999997,"5min":0.38700000000000001,"15min":0.28999999999999998},"max":{"1min":1.3560000000000001,"5min":5.6639999999999997,"15min":11.444000000000001},"last":1.1919999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:01:33 2023","interfaces":[{"interface":"back","average":{"1min":1.038,"5min":0.96999999999999997,"15min":1.4850000000000001},"min":{"1min":0.40000000000000002,"5min":0.34899999999999998,"15min":0.29999999999999999},"max":{"1min":3.141,"5min":4.5049999999999999,"15min":9.8819999999999997},"last":1.1240000000000001},{"interface":"front","average":{"1min":1.2030000000000001,"5min":1.0489999999999999,"15min":1.407},"min":{"1min":0.42999999999999999,"5min":0.34399999999999997,"15min":0.33900000000000002},"max":{"1min":5.4029999999999996,"5min":7.157,"15min":17.053000000000001},"last":1.292}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:02:30.230 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T20:02:30.230 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:02:30.230 INFO:tasks.thrashosds.thrasher:Removing osd 1, in_osds are: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T20:02:30.230 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd out 1 2023-09-23T20:02:30.272 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:30.306 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:30.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:30.309 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:30.410 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:30.439 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:30.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:30.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:30.542 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:30.572 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:30.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:30.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:30.676 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:30.710 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:30.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:30.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:30.813 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:30.845 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:30.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:30.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:30.949 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:30.981 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:30.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:30.984 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:31.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:30 smithi016 bash[77995]: cluster 2023-09-23T20:02:29.835855+0000 mon.a (mon.0) 1993 : cluster 0 osdmap e285: 8 total, 8 up, 8 in 2023-09-23T20:02:31.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:30 smithi016 bash[77995]: cluster 2023-09-23T20:02:29.854390+0000 osd.7 (osd.7) 199 : cluster 3 failed to encode map e285 with expected crc 2023-09-23T20:02:31.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:30 smithi016 bash[77995]: cluster 2023-09-23T20:02:29.854793+0000 osd.6 (osd.6) 219 : cluster 3 failed to encode map e285 with expected crc 2023-09-23T20:02:31.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:30 smithi016 bash[77995]: cluster 2023-09-23T20:02:30.020019+0000 osd.4 (osd.4) 226 : cluster 3 failed to encode map e285 with expected crc 2023-09-23T20:02:31.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:30 smithi016 bash[77995]: cluster 2023-09-23T20:02:30.022582+0000 osd.4 (osd.4) 227 : cluster 3 failed to encode map e285 with expected crc 2023-09-23T20:02:31.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:30 smithi016 bash[75004]: cluster 2023-09-23T20:02:29.835855+0000 mon.a (mon.0) 1993 : cluster 0 osdmap e285: 8 total, 8 up, 8 in 2023-09-23T20:02:31.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:30 smithi016 bash[75004]: cluster 2023-09-23T20:02:29.854390+0000 osd.7 (osd.7) 199 : cluster 3 failed to encode map e285 with expected crc 2023-09-23T20:02:31.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:30 smithi016 bash[75004]: cluster 2023-09-23T20:02:29.854793+0000 osd.6 (osd.6) 219 : cluster 3 failed to encode map e285 with expected crc 2023-09-23T20:02:31.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:30 smithi016 bash[75004]: cluster 2023-09-23T20:02:30.020019+0000 osd.4 (osd.4) 226 : cluster 3 failed to encode map e285 with expected crc 2023-09-23T20:02:31.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:30 smithi016 bash[75004]: cluster 2023-09-23T20:02:30.022582+0000 osd.4 (osd.4) 227 : cluster 3 failed to encode map e285 with expected crc 2023-09-23T20:02:31.085 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:31.088 INFO:tasks.workunit.client.0.smithi016.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T20:02:31.093 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd rm testimg 2023-09-23T20:02:31.117 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:31.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:31.120 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:31.223 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:31.233 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:30 smithi092 bash[58855]: cluster 2023-09-23T20:02:29.835855+0000 mon.a (mon.0) 1993 : cluster 0 osdmap e285: 8 total, 8 up, 8 in 2023-09-23T20:02:31.233 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:30 smithi092 bash[58855]: cluster 2023-09-23T20:02:29.854390+0000 osd.7 (osd.7) 199 : cluster 3 failed to encode map e285 with expected crc 2023-09-23T20:02:31.233 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:30 smithi092 bash[58855]: cluster 2023-09-23T20:02:29.854793+0000 osd.6 (osd.6) 219 : cluster 3 failed to encode map e285 with expected crc 2023-09-23T20:02:31.233 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:30 smithi092 bash[58855]: cluster 2023-09-23T20:02:30.020019+0000 osd.4 (osd.4) 226 : cluster 3 failed to encode map e285 with expected crc 2023-09-23T20:02:31.233 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:30 smithi092 bash[58855]: cluster 2023-09-23T20:02:30.022582+0000 osd.4 (osd.4) 227 : cluster 3 failed to encode map e285 with expected crc 2023-09-23T20:02:31.242 INFO:tasks.workunit.client.0.smithi016.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-09-23T20:02:31.248 INFO:tasks.workunit.client.0.smithi016.stderr:+ cmp /tmp/rbd_import_export_249364/img /tmp/rbd_import_export_249364/img2 2023-09-23T20:02:31.250 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:31.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:31.253 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:31.354 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:31.386 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:31.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:31.388 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:31.489 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:31.516 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:31.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:31.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:31.620 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:31.652 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:31.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:31.655 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:31.756 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:31.788 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:31.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:31.793 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:31.894 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:31.923 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:31.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:31.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:32.015 INFO:tasks.workunit.client.0.smithi016.stderr:+ cmp /tmp/rbd_import_export_249364/img /tmp/rbd_import_export_249364/img3 2023-09-23T20:02:32.026 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:32.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:31 smithi092 bash[58855]: cluster 2023-09-23T20:02:29.854750+0000 osd.5 (osd.5) 205 : cluster 3 failed to encode map e285 with expected crc 2023-09-23T20:02:32.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:31 smithi092 bash[58855]: cluster 2023-09-23T20:02:30.675622+0000 mgr.y (mgr.25325) 1660 : cluster 0 pgmap v1706: 73 pgs: 73 active+clean; 4.3 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 143 KiB/s rd, 96 KiB/s wr, 32 op/s 2023-09-23T20:02:32.058 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:32.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:32.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:32.161 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:32.173 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:31 smithi016 bash[77995]: cluster 2023-09-23T20:02:29.854750+0000 osd.5 (osd.5) 205 : cluster 3 failed to encode map e285 with expected crc 2023-09-23T20:02:32.173 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:31 smithi016 bash[77995]: cluster 2023-09-23T20:02:30.675622+0000 mgr.y (mgr.25325) 1660 : cluster 0 pgmap v1706: 73 pgs: 73 active+clean; 4.3 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 143 KiB/s rd, 96 KiB/s wr, 32 op/s 2023-09-23T20:02:32.173 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:31 smithi016 bash[75004]: cluster 2023-09-23T20:02:29.854750+0000 osd.5 (osd.5) 205 : cluster 3 failed to encode map e285 with expected crc 2023-09-23T20:02:32.173 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:31 smithi016 bash[75004]: cluster 2023-09-23T20:02:30.675622+0000 mgr.y (mgr.25325) 1660 : cluster 0 pgmap v1706: 73 pgs: 73 active+clean; 4.3 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 143 KiB/s rd, 96 KiB/s wr, 32 op/s 2023-09-23T20:02:32.190 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:32.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:32.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:32.224 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:02:32.294 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:32.325 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:32.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:32.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:32.428 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:32.460 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:32.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:32.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:32.564 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:32.585 INFO:tasks.workunit.client.0.smithi016.stderr:+ rm /tmp/rbd_import_export_249364/img /tmp/rbd_import_export_249364/img2 /tmp/rbd_import_export_249364/img3 2023-09-23T20:02:32.597 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:32.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:32.601 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:32.702 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:32.734 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:32.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:32.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:32.837 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:32.870 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:32.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:32.873 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:32.907 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd help export 2023-09-23T20:02:32.908 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep -q export-format 2023-09-23T20:02:32.932 INFO:tasks.workunit.client.0.smithi016.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_249364/img bs=1k count=10 seek=100 2023-09-23T20:02:32.933 INFO:tasks.workunit.client.0.smithi016.stderr:10+0 records in 2023-09-23T20:02:32.933 INFO:tasks.workunit.client.0.smithi016.stderr:10+0 records out 2023-09-23T20:02:32.933 INFO:tasks.workunit.client.0.smithi016.stderr:10240 bytes (10 kB, 10 KiB) copied, 0.000201612 s, 50.8 MB/s 2023-09-23T20:02:32.933 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd import --new-format /tmp/rbd_import_export_249364/img testimg 2023-09-23T20:02:32.974 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:33.005 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:33.007 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:02:33.008 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:02:33.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:33.009 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:33.021 INFO:tasks.workunit.client.0.smithi016.stderr: Importing image: 100% complete...done. 2023-09-23T20:02:33.021 INFO:tasks.workunit.client.0.smithi016.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T20:02:33.026 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd snap create testimg@snap 2023-09-23T20:02:33.109 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:33.143 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:33.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:33.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:33.247 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:33.279 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:33.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:33.282 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:33.382 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:33.415 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:33.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:33.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:33.519 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:33.541 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:33.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:33.543 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:33.644 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:33.671 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:33.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:33.673 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:33.774 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:33.811 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:33.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:33.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:33.872 INFO:tasks.workunit.client.0.smithi016.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-23T20:02:33.885 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd image-meta set testimg key1 value1 2023-09-23T20:02:33.915 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:33.937 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:33.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:33.941 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:33.957 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd image-meta list testimg 2023-09-23T20:02:34.020 INFO:tasks.workunit.client.0.smithi016.stderr:+ IMAGEMETA_BEFORE=There is 1 metadatum on this image: 2023-09-23T20:02:34.020 INFO:tasks.workunit.client.0.smithi016.stderr: 2023-09-23T20:02:34.020 INFO:tasks.workunit.client.0.smithi016.stderr:Key Value 2023-09-23T20:02:34.020 INFO:tasks.workunit.client.0.smithi016.stderr:key1 value1 2023-09-23T20:02:34.021 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_249364/img_v2 2023-09-23T20:02:34.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:33 smithi092 bash[58855]: cluster 2023-09-23T20:02:32.676389+0000 mgr.y (mgr.25325) 1661 : cluster 0 pgmap v1707: 73 pgs: 73 active+clean; 6.0 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 185 KiB/s rd, 104 KiB/s wr, 68 op/s 2023-09-23T20:02:34.042 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:34.072 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:34.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:34.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:34.077 INFO:tasks.workunit.client.0.smithi016.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T20:02:34.083 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_249364/img_v2 testimg_import 2023-09-23T20:02:34.176 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:34.209 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:34.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:34.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:34.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:33 smithi016 bash[75004]: cluster 2023-09-23T20:02:32.676389+0000 mgr.y (mgr.25325) 1661 : cluster 0 pgmap v1707: 73 pgs: 73 active+clean; 6.0 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 185 KiB/s rd, 104 KiB/s wr, 68 op/s 2023-09-23T20:02:34.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:33 smithi016 bash[77995]: cluster 2023-09-23T20:02:32.676389+0000 mgr.y (mgr.25325) 1661 : cluster 0 pgmap v1707: 73 pgs: 73 active+clean; 6.0 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 185 KiB/s rd, 104 KiB/s wr, 68 op/s 2023-09-23T20:02:34.313 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:34.345 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:34.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:34.348 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:34.449 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:34.477 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:34.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:34.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:34.580 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:34.610 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:34.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:34.614 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:34.715 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:34.744 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:34.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:34.747 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:34.848 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:34.883 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:34.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:34.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:34.889 INFO:tasks.workunit.client.0.smithi016.stderr: Importing image: 50% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T20:02:34.893 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd info testimg_import 2023-09-23T20:02:34.970 INFO:tasks.workunit.client.0.smithi016.stdout:rbd image 'testimg_import': 2023-09-23T20:02:34.970 INFO:tasks.workunit.client.0.smithi016.stdout: size 110 KiB in 1 objects 2023-09-23T20:02:34.970 INFO:tasks.workunit.client.0.smithi016.stdout: order 22 (4 MiB objects) 2023-09-23T20:02:34.971 INFO:tasks.workunit.client.0.smithi016.stdout: snapshot_count: 1 2023-09-23T20:02:34.971 INFO:tasks.workunit.client.0.smithi016.stdout: id: b8405e3620ba 2023-09-23T20:02:34.971 INFO:tasks.workunit.client.0.smithi016.stdout: block_name_prefix: rbd_data.b8405e3620ba 2023-09-23T20:02:34.971 INFO:tasks.workunit.client.0.smithi016.stdout: format: 2 2023-09-23T20:02:34.971 INFO:tasks.workunit.client.0.smithi016.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-09-23T20:02:34.971 INFO:tasks.workunit.client.0.smithi016.stdout: op_features: 2023-09-23T20:02:34.971 INFO:tasks.workunit.client.0.smithi016.stdout: flags: 2023-09-23T20:02:34.971 INFO:tasks.workunit.client.0.smithi016.stdout: create_timestamp: Sat Sep 23 20:02:34 2023 2023-09-23T20:02:34.972 INFO:tasks.workunit.client.0.smithi016.stdout: access_timestamp: Sat Sep 23 20:02:34 2023 2023-09-23T20:02:34.972 INFO:tasks.workunit.client.0.smithi016.stdout: modify_timestamp: Sat Sep 23 20:02:34 2023 2023-09-23T20:02:34.976 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd info testimg_import@snap 2023-09-23T20:02:34.987 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:34.995 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:34 smithi016 bash[75004]: cluster 2023-09-23T20:02:33.846623+0000 mon.a (mon.0) 1994 : cluster 0 osdmap e286: 8 total, 8 up, 8 in 2023-09-23T20:02:34.996 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:34 smithi016 bash[75004]: cluster 2023-09-23T20:02:33.856364+0000 osd.6 (osd.6) 220 : cluster 3 failed to encode map e286 with expected crc 2023-09-23T20:02:34.996 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:34 smithi016 bash[75004]: cluster 2023-09-23T20:02:33.863141+0000 osd.5 (osd.5) 206 : cluster 3 failed to encode map e286 with expected crc 2023-09-23T20:02:34.996 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:34 smithi016 bash[75004]: cluster 2023-09-23T20:02:34.112744+0000 osd.4 (osd.4) 228 : cluster 3 failed to encode map e286 with expected crc 2023-09-23T20:02:34.996 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:34 smithi016 bash[77995]: cluster 2023-09-23T20:02:33.846623+0000 mon.a (mon.0) 1994 : cluster 0 osdmap e286: 8 total, 8 up, 8 in 2023-09-23T20:02:34.996 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:34 smithi016 bash[77995]: cluster 2023-09-23T20:02:33.856364+0000 osd.6 (osd.6) 220 : cluster 3 failed to encode map e286 with expected crc 2023-09-23T20:02:34.997 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:34 smithi016 bash[77995]: cluster 2023-09-23T20:02:33.863141+0000 osd.5 (osd.5) 206 : cluster 3 failed to encode map e286 with expected crc 2023-09-23T20:02:34.997 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:34 smithi016 bash[77995]: cluster 2023-09-23T20:02:34.112744+0000 osd.4 (osd.4) 228 : cluster 3 failed to encode map e286 with expected crc 2023-09-23T20:02:35.009 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:35.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:35.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:35.025 INFO:tasks.workunit.client.0.smithi016.stdout:rbd image 'testimg_import': 2023-09-23T20:02:35.025 INFO:tasks.workunit.client.0.smithi016.stdout: size 110 KiB in 1 objects 2023-09-23T20:02:35.025 INFO:tasks.workunit.client.0.smithi016.stdout: order 22 (4 MiB objects) 2023-09-23T20:02:35.025 INFO:tasks.workunit.client.0.smithi016.stdout: snapshot_count: 1 2023-09-23T20:02:35.026 INFO:tasks.workunit.client.0.smithi016.stdout: id: b8405e3620ba 2023-09-23T20:02:35.026 INFO:tasks.workunit.client.0.smithi016.stdout: block_name_prefix: rbd_data.b8405e3620ba 2023-09-23T20:02:35.026 INFO:tasks.workunit.client.0.smithi016.stdout: format: 2 2023-09-23T20:02:35.026 INFO:tasks.workunit.client.0.smithi016.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-09-23T20:02:35.026 INFO:tasks.workunit.client.0.smithi016.stdout: op_features: 2023-09-23T20:02:35.026 INFO:tasks.workunit.client.0.smithi016.stdout: flags: 2023-09-23T20:02:35.026 INFO:tasks.workunit.client.0.smithi016.stdout: create_timestamp: Sat Sep 23 20:02:34 2023 2023-09-23T20:02:35.026 INFO:tasks.workunit.client.0.smithi016.stdout: access_timestamp: Sat Sep 23 20:02:34 2023 2023-09-23T20:02:35.026 INFO:tasks.workunit.client.0.smithi016.stdout: modify_timestamp: Sat Sep 23 20:02:34 2023 2023-09-23T20:02:35.026 INFO:tasks.workunit.client.0.smithi016.stdout: protected: False 2023-09-23T20:02:35.029 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd image-meta list testimg_import 2023-09-23T20:02:35.034 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:02:35.084 INFO:tasks.workunit.client.0.smithi016.stderr:+ IMAGEMETA_AFTER=There is 1 metadatum on this image: 2023-09-23T20:02:35.085 INFO:tasks.workunit.client.0.smithi016.stderr: 2023-09-23T20:02:35.085 INFO:tasks.workunit.client.0.smithi016.stderr:Key Value 2023-09-23T20:02:35.085 INFO:tasks.workunit.client.0.smithi016.stderr:key1 value1 2023-09-23T20:02:35.085 INFO:tasks.workunit.client.0.smithi016.stderr:+ [ There is 1 metadatum on this image: 2023-09-23T20:02:35.085 INFO:tasks.workunit.client.0.smithi016.stderr: 2023-09-23T20:02:35.085 INFO:tasks.workunit.client.0.smithi016.stderr:Key Value 2023-09-23T20:02:35.085 INFO:tasks.workunit.client.0.smithi016.stderr:key1 value1 = There is 1 metadatum on this image: 2023-09-23T20:02:35.085 INFO:tasks.workunit.client.0.smithi016.stderr: 2023-09-23T20:02:35.085 INFO:tasks.workunit.client.0.smithi016.stderr:Key Value 2023-09-23T20:02:35.085 INFO:tasks.workunit.client.0.smithi016.stderr:key1 value1 ] 2023-09-23T20:02:35.086 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd export testimg_import /tmp/rbd_import_export_249364/img_import 2023-09-23T20:02:35.113 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:35.127 INFO:tasks.workunit.client.0.smithi016.stderr: Exporting image: 100% complete...done. 2023-09-23T20:02:35.131 INFO:tasks.workunit.client.0.smithi016.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_249364/img /tmp/rbd_import_export_249364/img_import 2023-09-23T20:02:35.131 INFO:tasks.workunit.client.0.smithi016.stderr:+ cmp -l /tmp/rbd_import_export_249364/img /tmp/rbd_import_export_249364/img_import 2023-09-23T20:02:35.131 INFO:tasks.workunit.client.0.smithi016.stderr:+ stat /tmp/rbd_import_export_249364/img --format %b 2023-09-23T20:02:35.132 INFO:tasks.workunit.client.0.smithi016.stderr:+ origsize=24 2023-09-23T20:02:35.132 INFO:tasks.workunit.client.0.smithi016.stderr:+ stat /tmp/rbd_import_export_249364/img_import --format %b 2023-09-23T20:02:35.133 INFO:tasks.workunit.client.0.smithi016.stderr:+ exportsize=224 2023-09-23T20:02:35.133 INFO:tasks.workunit.client.0.smithi016.stderr:+ difference=200 2023-09-23T20:02:35.133 INFO:tasks.workunit.client.0.smithi016.stderr:+ difference=200 2023-09-23T20:02:35.133 INFO:tasks.workunit.client.0.smithi016.stderr:+ test 200 -ge 0 -a 200 -lt 4096 2023-09-23T20:02:35.133 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd export testimg@snap /tmp/rbd_import_export_249364/img_snap 2023-09-23T20:02:35.140 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:35.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:35.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:35.177 INFO:tasks.workunit.client.0.smithi016.stderr: Exporting image: 100% complete...done. 2023-09-23T20:02:35.180 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd export testimg_import@snap /tmp/rbd_import_export_249364/img_snap_import 2023-09-23T20:02:35.225 INFO:tasks.workunit.client.0.smithi016.stderr: Exporting image: 100% complete...done. 2023-09-23T20:02:35.230 INFO:tasks.workunit.client.0.smithi016.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_249364/img_snap /tmp/rbd_import_export_249364/img_snap_import 2023-09-23T20:02:35.231 INFO:tasks.workunit.client.0.smithi016.stderr:+ cmp -l /tmp/rbd_import_export_249364/img_snap /tmp/rbd_import_export_249364/img_snap_import 2023-09-23T20:02:35.232 INFO:tasks.workunit.client.0.smithi016.stderr:+ stat /tmp/rbd_import_export_249364/img_snap --format %b 2023-09-23T20:02:35.233 INFO:tasks.workunit.client.0.smithi016.stderr:+ origsize=224 2023-09-23T20:02:35.233 INFO:tasks.workunit.client.0.smithi016.stderr:+ stat /tmp/rbd_import_export_249364/img_snap_import --format %b 2023-09-23T20:02:35.234 INFO:tasks.workunit.client.0.smithi016.stderr:+ exportsize=224 2023-09-23T20:02:35.234 INFO:tasks.workunit.client.0.smithi016.stderr:+ difference=0 2023-09-23T20:02:35.234 INFO:tasks.workunit.client.0.smithi016.stderr:+ difference=0 2023-09-23T20:02:35.234 INFO:tasks.workunit.client.0.smithi016.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T20:02:35.234 INFO:tasks.workunit.client.0.smithi016.stderr:+ rm /tmp/rbd_import_export_249364/img_v2 2023-09-23T20:02:35.234 INFO:tasks.workunit.client.0.smithi016.stderr:+ rm /tmp/rbd_import_export_249364/img_import 2023-09-23T20:02:35.235 INFO:tasks.workunit.client.0.smithi016.stderr:+ rm /tmp/rbd_import_export_249364/img_snap 2023-09-23T20:02:35.236 INFO:tasks.workunit.client.0.smithi016.stderr:+ rm /tmp/rbd_import_export_249364/img_snap_import 2023-09-23T20:02:35.237 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd snap rm testimg_import@snap 2023-09-23T20:02:35.244 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:35.256 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:34 smithi092 bash[58855]: cluster 2023-09-23T20:02:33.846623+0000 mon.a (mon.0) 1994 : cluster 0 osdmap e286: 8 total, 8 up, 8 in 2023-09-23T20:02:35.256 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:34 smithi092 bash[58855]: cluster 2023-09-23T20:02:33.856364+0000 osd.6 (osd.6) 220 : cluster 3 failed to encode map e286 with expected crc 2023-09-23T20:02:35.256 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:34 smithi092 bash[58855]: cluster 2023-09-23T20:02:33.863141+0000 osd.5 (osd.5) 206 : cluster 3 failed to encode map e286 with expected crc 2023-09-23T20:02:35.256 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:34 smithi092 bash[58855]: cluster 2023-09-23T20:02:34.112744+0000 osd.4 (osd.4) 228 : cluster 3 failed to encode map e286 with expected crc 2023-09-23T20:02:35.275 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:35.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:35.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:35.379 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:35.411 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:35.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:35.414 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:35.515 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:35.545 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:35.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:35.547 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:35.648 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:35.673 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:35.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:35.675 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:35.776 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:35.785 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:02:35 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:20:02:35] "GET /metrics HTTP/1.1" 200 33994 "" "Prometheus/2.33.4" 2023-09-23T20:02:35.807 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:35.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:35.809 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:35.882 INFO:tasks.workunit.client.0.smithi016.stderr: Removing snap: 100% complete...done. 2023-09-23T20:02:35.895 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd remove testimg_import 2023-09-23T20:02:35.909 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:35.937 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:35.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:35.939 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:36.033 INFO:tasks.workunit.client.0.smithi016.stderr: Removing image: 100% complete...done. 2023-09-23T20:02:36.037 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd snap rm testimg@snap 2023-09-23T20:02:36.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:35 smithi092 bash[58855]: cluster 2023-09-23T20:02:34.677200+0000 mgr.y (mgr.25325) 1662 : cluster 0 pgmap v1709: 73 pgs: 73 active+clean; 3.1 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 148 KiB/s rd, 67 KiB/s wr, 67 op/s 2023-09-23T20:02:36.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:35 smithi092 bash[58855]: cluster 2023-09-23T20:02:34.860943+0000 mon.a (mon.0) 1995 : cluster 0 osdmap e287: 8 total, 8 up, 8 in 2023-09-23T20:02:36.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:35 smithi092 bash[58855]: cluster 2023-09-23T20:02:34.868067+0000 osd.7 (osd.7) 200 : cluster 3 failed to encode map e287 with expected crc 2023-09-23T20:02:36.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:35 smithi092 bash[58855]: cluster 2023-09-23T20:02:34.878043+0000 osd.6 (osd.6) 221 : cluster 3 failed to encode map e287 with expected crc 2023-09-23T20:02:36.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:35 smithi092 bash[58855]: cluster 2023-09-23T20:02:34.878068+0000 osd.5 (osd.5) 207 : cluster 3 failed to encode map e287 with expected crc 2023-09-23T20:02:36.041 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:36.049 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:02:36.069 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:36.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:36.073 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:36.174 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:36.182 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:35 smithi016 bash[77995]: cluster 2023-09-23T20:02:34.677200+0000 mgr.y (mgr.25325) 1662 : cluster 0 pgmap v1709: 73 pgs: 73 active+clean; 3.1 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 148 KiB/s rd, 67 KiB/s wr, 67 op/s 2023-09-23T20:02:36.182 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:35 smithi016 bash[77995]: cluster 2023-09-23T20:02:34.860943+0000 mon.a (mon.0) 1995 : cluster 0 osdmap e287: 8 total, 8 up, 8 in 2023-09-23T20:02:36.182 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:35 smithi016 bash[77995]: cluster 2023-09-23T20:02:34.868067+0000 osd.7 (osd.7) 200 : cluster 3 failed to encode map e287 with expected crc 2023-09-23T20:02:36.182 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:35 smithi016 bash[77995]: cluster 2023-09-23T20:02:34.878043+0000 osd.6 (osd.6) 221 : cluster 3 failed to encode map e287 with expected crc 2023-09-23T20:02:36.182 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:35 smithi016 bash[77995]: cluster 2023-09-23T20:02:34.878068+0000 osd.5 (osd.5) 207 : cluster 3 failed to encode map e287 with expected crc 2023-09-23T20:02:36.182 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:35 smithi016 bash[75004]: cluster 2023-09-23T20:02:34.677200+0000 mgr.y (mgr.25325) 1662 : cluster 0 pgmap v1709: 73 pgs: 73 active+clean; 3.1 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 148 KiB/s rd, 67 KiB/s wr, 67 op/s 2023-09-23T20:02:36.183 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:35 smithi016 bash[75004]: cluster 2023-09-23T20:02:34.860943+0000 mon.a (mon.0) 1995 : cluster 0 osdmap e287: 8 total, 8 up, 8 in 2023-09-23T20:02:36.183 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:35 smithi016 bash[75004]: cluster 2023-09-23T20:02:34.868067+0000 osd.7 (osd.7) 200 : cluster 3 failed to encode map e287 with expected crc 2023-09-23T20:02:36.183 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:35 smithi016 bash[75004]: cluster 2023-09-23T20:02:34.878043+0000 osd.6 (osd.6) 221 : cluster 3 failed to encode map e287 with expected crc 2023-09-23T20:02:36.183 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:35 smithi016 bash[75004]: cluster 2023-09-23T20:02:34.878068+0000 osd.5 (osd.5) 207 : cluster 3 failed to encode map e287 with expected crc 2023-09-23T20:02:36.198 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:36.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:36.200 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:36.301 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:36.335 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:36.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:36.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:36.438 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:36.472 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:36.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:36.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:36.576 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:36.603 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:36.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:36.606 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:36.707 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:36.741 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:36.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:36.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:36.844 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:36.866 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:36.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:36.870 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:36.894 INFO:tasks.workunit.client.0.smithi016.stderr: Removing snap: 100% complete...done. 2023-09-23T20:02:36.909 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd rm testimg 2023-09-23T20:02:36.970 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:36.998 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:37.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:37.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:37.056 INFO:tasks.workunit.client.0.smithi016.stderr: Removing image: 100% complete...done. 2023-09-23T20:02:37.060 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd import --order 20 /tmp/rbd_import_export_249364/img testimg 2023-09-23T20:02:37.102 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:37.112 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:36 smithi016 bash[75004]: cluster 2023-09-23T20:02:35.865816+0000 mon.a (mon.0) 1996 : cluster 0 osdmap e288: 8 total, 8 up, 8 in 2023-09-23T20:02:37.112 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:36 smithi016 bash[75004]: cluster 2023-09-23T20:02:35.875785+0000 osd.7 (osd.7) 201 : cluster 3 failed to encode map e288 with expected crc 2023-09-23T20:02:37.112 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:36 smithi016 bash[75004]: cluster 2023-09-23T20:02:35.885601+0000 osd.5 (osd.5) 208 : cluster 3 failed to encode map e288 with expected crc 2023-09-23T20:02:37.112 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:36 smithi016 bash[75004]: cluster 2023-09-23T20:02:35.885915+0000 osd.6 (osd.6) 222 : cluster 3 failed to encode map e288 with expected crc 2023-09-23T20:02:37.112 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:36 smithi016 bash[77995]: cluster 2023-09-23T20:02:35.865816+0000 mon.a (mon.0) 1996 : cluster 0 osdmap e288: 8 total, 8 up, 8 in 2023-09-23T20:02:37.113 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:36 smithi016 bash[77995]: cluster 2023-09-23T20:02:35.875785+0000 osd.7 (osd.7) 201 : cluster 3 failed to encode map e288 with expected crc 2023-09-23T20:02:37.113 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:36 smithi016 bash[77995]: cluster 2023-09-23T20:02:35.885601+0000 osd.5 (osd.5) 208 : cluster 3 failed to encode map e288 with expected crc 2023-09-23T20:02:37.113 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:36 smithi016 bash[77995]: cluster 2023-09-23T20:02:35.885915+0000 osd.6 (osd.6) 222 : cluster 3 failed to encode map e288 with expected crc 2023-09-23T20:02:37.127 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:37.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:37.130 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:37.139 INFO:tasks.workunit.client.0.smithi016.stderr: Importing image: 100% complete...done. 2023-09-23T20:02:37.140 INFO:tasks.workunit.client.0.smithi016.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T20:02:37.143 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_249364/img_v2 2023-09-23T20:02:37.220 INFO:tasks.workunit.client.0.smithi016.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T20:02:37.224 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_249364/img_v2 testimg_import 2023-09-23T20:02:37.230 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:37.241 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:36 smithi092 bash[58855]: cluster 2023-09-23T20:02:35.865816+0000 mon.a (mon.0) 1996 : cluster 0 osdmap e288: 8 total, 8 up, 8 in 2023-09-23T20:02:37.242 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:36 smithi092 bash[58855]: cluster 2023-09-23T20:02:35.875785+0000 osd.7 (osd.7) 201 : cluster 3 failed to encode map e288 with expected crc 2023-09-23T20:02:37.242 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:36 smithi092 bash[58855]: cluster 2023-09-23T20:02:35.885601+0000 osd.5 (osd.5) 208 : cluster 3 failed to encode map e288 with expected crc 2023-09-23T20:02:37.242 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:36 smithi092 bash[58855]: cluster 2023-09-23T20:02:35.885915+0000 osd.6 (osd.6) 222 : cluster 3 failed to encode map e288 with expected crc 2023-09-23T20:02:37.259 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:37.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:37.263 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:37.299 INFO:tasks.workunit.client.0.smithi016.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T20:02:37.306 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd info testimg_import 2023-09-23T20:02:37.306 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep order 2023-09-23T20:02:37.306 INFO:tasks.workunit.client.0.smithi016.stderr:+ awk {print $2} 2023-09-23T20:02:37.306 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep 20 2023-09-23T20:02:37.364 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:37.394 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:37.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:37.397 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:37.498 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:37.527 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:37.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:37.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:37.631 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:37.663 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:37.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:37.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:37.759 INFO:tasks.workunit.client.0.smithi016.stdout:20 2023-09-23T20:02:37.760 INFO:tasks.workunit.client.0.smithi016.stderr:+ rm /tmp/rbd_import_export_249364/img_v2 2023-09-23T20:02:37.760 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd remove testimg_import 2023-09-23T20:02:37.767 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:37.800 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:37.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:37.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:37.903 INFO:tasks.workunit.client.0.smithi016.stderr: Removing image: 100% complete...done. 2023-09-23T20:02:37.904 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:37.907 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd remove testimg 2023-09-23T20:02:37.930 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:37.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:37.933 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:38.025 INFO:tasks.workunit.client.0.smithi016.stderr: Removing image: 100% complete...done. 2023-09-23T20:02:38.031 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd import --image-feature layering /tmp/rbd_import_export_249364/img testimg 2023-09-23T20:02:38.034 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:38.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:37 smithi092 bash[58855]: cluster 2023-09-23T20:02:36.678025+0000 mgr.y (mgr.25325) 1663 : cluster 0 pgmap v1712: 73 pgs: 73 active+clean; 692 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 253 KiB/s rd, 27 KiB/s wr, 158 op/s 2023-09-23T20:02:38.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:37 smithi092 bash[58855]: cluster 2023-09-23T20:02:36.881965+0000 mon.a (mon.0) 1997 : cluster 0 osdmap e289: 8 total, 8 up, 8 in 2023-09-23T20:02:38.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:37 smithi092 bash[58855]: cluster 2023-09-23T20:02:36.888153+0000 osd.6 (osd.6) 223 : cluster 3 failed to encode map e289 with expected crc 2023-09-23T20:02:38.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:37 smithi092 bash[58855]: cluster 2023-09-23T20:02:36.897418+0000 osd.5 (osd.5) 209 : cluster 3 failed to encode map e289 with expected crc 2023-09-23T20:02:38.058 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:38.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:38.060 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:38.083 INFO:tasks.workunit.client.0.smithi016.stderr: Importing image: 100% complete...done. 2023-09-23T20:02:38.087 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd info testimg 2023-09-23T20:02:38.087 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep features 2023-09-23T20:02:38.125 INFO:tasks.workunit.client.0.smithi016.stderr:+ FEATURES_BEFORE= features: layering 2023-09-23T20:02:38.125 INFO:tasks.workunit.client.0.smithi016.stderr: op_features: 2023-09-23T20:02:38.125 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_249364/img_v2 2023-09-23T20:02:38.161 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:38.164 INFO:tasks.workunit.client.0.smithi016.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T20:02:38.168 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_249364/img_v2 testimg_import 2023-09-23T20:02:38.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:38.308 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:38.308 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:38.308 INFO:tasks.workunit.client.0.smithi016.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T20:02:38.309 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd info testimg_import 2023-09-23T20:02:38.309 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep features 2023-09-23T20:02:38.309 INFO:tasks.workunit.client.0.smithi016.stderr:+ FEATURES_AFTER= features: layering 2023-09-23T20:02:38.309 INFO:tasks.workunit.client.0.smithi016.stderr: op_features: 2023-09-23T20:02:38.309 INFO:tasks.workunit.client.0.smithi016.stderr:+ [ features: layering 2023-09-23T20:02:38.309 INFO:tasks.workunit.client.0.smithi016.stderr: op_features: != features: layering 2023-09-23T20:02:38.309 INFO:tasks.workunit.client.0.smithi016.stderr: op_features: ] 2023-09-23T20:02:38.309 INFO:tasks.workunit.client.0.smithi016.stderr:+ rm /tmp/rbd_import_export_249364/img_v2 2023-09-23T20:02:38.309 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd remove testimg_import 2023-09-23T20:02:38.310 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:37 smithi016 bash[75004]: cluster 2023-09-23T20:02:36.678025+0000 mgr.y (mgr.25325) 1663 : cluster 0 pgmap v1712: 73 pgs: 73 active+clean; 692 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 253 KiB/s rd, 27 KiB/s wr, 158 op/s 2023-09-23T20:02:38.310 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:37 smithi016 bash[75004]: cluster 2023-09-23T20:02:36.881965+0000 mon.a (mon.0) 1997 : cluster 0 osdmap e289: 8 total, 8 up, 8 in 2023-09-23T20:02:38.310 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:37 smithi016 bash[75004]: cluster 2023-09-23T20:02:36.888153+0000 osd.6 (osd.6) 223 : cluster 3 failed to encode map e289 with expected crc 2023-09-23T20:02:38.310 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:37 smithi016 bash[75004]: cluster 2023-09-23T20:02:36.897418+0000 osd.5 (osd.5) 209 : cluster 3 failed to encode map e289 with expected crc 2023-09-23T20:02:38.310 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:37 smithi016 bash[77995]: cluster 2023-09-23T20:02:36.678025+0000 mgr.y (mgr.25325) 1663 : cluster 0 pgmap v1712: 73 pgs: 73 active+clean; 692 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 253 KiB/s rd, 27 KiB/s wr, 158 op/s 2023-09-23T20:02:38.310 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:37 smithi016 bash[77995]: cluster 2023-09-23T20:02:36.881965+0000 mon.a (mon.0) 1997 : cluster 0 osdmap e289: 8 total, 8 up, 8 in 2023-09-23T20:02:38.311 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:37 smithi016 bash[77995]: cluster 2023-09-23T20:02:36.888153+0000 osd.6 (osd.6) 223 : cluster 3 failed to encode map e289 with expected crc 2023-09-23T20:02:38.311 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:37 smithi016 bash[77995]: cluster 2023-09-23T20:02:36.897418+0000 osd.5 (osd.5) 209 : cluster 3 failed to encode map e289 with expected crc 2023-09-23T20:02:38.391 INFO:tasks.workunit.client.0.smithi016.stderr: Removing image: 100% complete...done. 2023-09-23T20:02:38.396 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd remove testimg 2023-09-23T20:02:38.409 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:38.445 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:38.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:38.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:38.492 INFO:tasks.workunit.client.0.smithi016.stderr: Removing image: 100% complete...done. 2023-09-23T20:02:38.498 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd import --stripe-count 1000 --stripe-unit 4096 /tmp/rbd_import_export_249364/img testimg 2023-09-23T20:02:38.549 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:38.580 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:38.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:38.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:38.587 INFO:tasks.workunit.client.0.smithi016.stderr: Importing image: 100% complete...done. 2023-09-23T20:02:38.591 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_249364/img_v2 2023-09-23T20:02:38.685 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:38.711 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:38.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:38.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:38.814 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:38.832 INFO:tasks.workunit.client.0.smithi016.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T20:02:38.837 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_249364/img_v2 testimg_import 2023-09-23T20:02:38.842 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:38.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:38.845 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:38.926 INFO:tasks.workunit.client.0.smithi016.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T20:02:38.931 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd info testimg_import 2023-09-23T20:02:38.931 INFO:tasks.workunit.client.0.smithi016.stderr:+ + grep stripe unit 2023-09-23T20:02:38.931 INFO:tasks.workunit.client.0.smithi016.stderr:grep -Ei (4 KiB|4096) 2023-09-23T20:02:38.946 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:38.975 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:38.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:38.978 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:38.979 INFO:tasks.workunit.client.0.smithi016.stdout: stripe unit: 4 KiB 2023-09-23T20:02:38.979 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd info testimg_import 2023-09-23T20:02:38.980 INFO:tasks.workunit.client.0.smithi016.stderr:+ awk {print $3}+ 2023-09-23T20:02:38.980 INFO:tasks.workunit.client.0.smithi016.stderr:+ grepgrep stripe count 1000 2023-09-23T20:02:38.980 INFO:tasks.workunit.client.0.smithi016.stderr: 2023-09-23T20:02:39.033 INFO:tasks.workunit.client.0.smithi016.stdout:1000 2023-09-23T20:02:39.033 INFO:tasks.workunit.client.0.smithi016.stderr:+ rm /tmp/rbd_import_export_249364/img_v2 2023-09-23T20:02:39.034 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd remove testimg_import 2023-09-23T20:02:39.078 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:39.111 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:39.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:39.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:39.176 INFO:tasks.workunit.client.0.smithi016.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-09-23T20:02:39.180 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd remove testimg 2023-09-23T20:02:39.214 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:39.244 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:39.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:39.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:39.320 INFO:tasks.workunit.client.0.smithi016.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-09-23T20:02:39.324 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd import --image-format=2 /tmp/rbd_import_export_249364/img testimg 2023-09-23T20:02:39.349 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:39.372 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:39.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:39.374 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:39.396 INFO:tasks.workunit.client.0.smithi016.stderr: Importing image: 100% complete...done. 2023-09-23T20:02:39.399 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd snap create testimg@snap1 2023-09-23T20:02:39.475 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:39.499 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:39.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:39.501 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:39.602 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:39.637 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:39.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:39.640 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:39.741 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:39.773 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:39.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:39.776 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:39.877 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:39.905 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:39.907 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:39 smithi016 bash[75004]: cluster 2023-09-23T20:02:38.678784+0000 mgr.y (mgr.25325) 1664 : cluster 0 pgmap v1714: 73 pgs: 73 active+clean; 692 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 221 KiB/s rd, 11 KiB/s wr, 117 op/s 2023-09-23T20:02:39.907 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:39 smithi016 bash[77995]: cluster 2023-09-23T20:02:38.678784+0000 mgr.y (mgr.25325) 1664 : cluster 0 pgmap v1714: 73 pgs: 73 active+clean; 692 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 221 KiB/s rd, 11 KiB/s wr, 117 op/s 2023-09-23T20:02:39.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:39.908 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:39.916 INFO:tasks.workunit.client.0.smithi016.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-23T20:02:39.926 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd snap create testimg@snap2 2023-09-23T20:02:40.009 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:40.042 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:40.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:40.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:40.146 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:40.171 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:40.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:40.173 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:40.274 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:40.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:39 smithi092 bash[58855]: cluster 2023-09-23T20:02:38.678784+0000 mgr.y (mgr.25325) 1664 : cluster 0 pgmap v1714: 73 pgs: 73 active+clean; 692 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 221 KiB/s rd, 11 KiB/s wr, 117 op/s 2023-09-23T20:02:40.302 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:40.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:40.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:40.406 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:40.442 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:40.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:40.445 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:40.546 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:40.578 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:40.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:40.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:40.682 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:40.704 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:40.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:40.707 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:40.808 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:40.838 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:40.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:40.840 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:40.914 INFO:tasks.workunit.client.0.smithi016.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-23T20:02:40.924 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd snap protect testimg@snap2 2023-09-23T20:02:40.942 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:40.966 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_249364/snap_protect 2023-09-23T20:02:40.970 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:40.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:40.975 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:41.075 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:41.098 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:40 smithi016 bash[75004]: cluster 2023-09-23T20:02:39.890460+0000 mon.a (mon.0) 1998 : cluster 0 osdmap e290: 8 total, 8 up, 8 in 2023-09-23T20:02:41.098 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:40 smithi016 bash[75004]: cluster 2023-09-23T20:02:39.892563+0000 osd.5 (osd.5) 210 : cluster 3 failed to encode map e290 with expected crc 2023-09-23T20:02:41.098 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:40 smithi016 bash[75004]: cluster 2023-09-23T20:02:39.903103+0000 osd.6 (osd.6) 224 : cluster 3 failed to encode map e290 with expected crc 2023-09-23T20:02:41.099 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:40 smithi016 bash[75004]: cluster 2023-09-23T20:02:39.910010+0000 osd.7 (osd.7) 202 : cluster 3 failed to encode map e290 with expected crc 2023-09-23T20:02:41.099 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:40 smithi016 bash[75004]: cluster 2023-09-23T20:02:39.921665+0000 osd.4 (osd.4) 229 : cluster 3 failed to encode map e290 with expected crc 2023-09-23T20:02:41.099 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:40 smithi016 bash[75004]: cluster 2023-09-23T20:02:39.923528+0000 osd.4 (osd.4) 230 : cluster 3 failed to encode map e290 with expected crc 2023-09-23T20:02:41.099 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:40 smithi016 bash[75004]: audit 2023-09-23T20:02:40.696895+0000 mon.a (mon.0) 1999 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:02:41.099 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:40 smithi016 bash[77995]: cluster 2023-09-23T20:02:39.890460+0000 mon.a (mon.0) 1998 : cluster 0 osdmap e290: 8 total, 8 up, 8 in 2023-09-23T20:02:41.099 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:40 smithi016 bash[77995]: cluster 2023-09-23T20:02:39.892563+0000 osd.5 (osd.5) 210 : cluster 3 failed to encode map e290 with expected crc 2023-09-23T20:02:41.099 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:40 smithi016 bash[77995]: cluster 2023-09-23T20:02:39.903103+0000 osd.6 (osd.6) 224 : cluster 3 failed to encode map e290 with expected crc 2023-09-23T20:02:41.099 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:40 smithi016 bash[77995]: cluster 2023-09-23T20:02:39.910010+0000 osd.7 (osd.7) 202 : cluster 3 failed to encode map e290 with expected crc 2023-09-23T20:02:41.099 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:40 smithi016 bash[77995]: cluster 2023-09-23T20:02:39.921665+0000 osd.4 (osd.4) 229 : cluster 3 failed to encode map e290 with expected crc 2023-09-23T20:02:41.100 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:40 smithi016 bash[77995]: cluster 2023-09-23T20:02:39.923528+0000 osd.4 (osd.4) 230 : cluster 3 failed to encode map e290 with expected crc 2023-09-23T20:02:41.100 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:40 smithi016 bash[77995]: audit 2023-09-23T20:02:40.696895+0000 mon.a (mon.0) 1999 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:02:41.102 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:41.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:41.105 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:41.105 INFO:tasks.workunit.client.0.smithi016.stderr: Exporting image: 33% complete... Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T20:02:41.109 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_249364/snap_protect testimg_import 2023-09-23T20:02:41.206 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:41.229 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:41.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:41.232 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:41.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:40 smithi092 bash[58855]: cluster 2023-09-23T20:02:39.890460+0000 mon.a (mon.0) 1998 : cluster 0 osdmap e290: 8 total, 8 up, 8 in 2023-09-23T20:02:41.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:40 smithi092 bash[58855]: cluster 2023-09-23T20:02:39.892563+0000 osd.5 (osd.5) 210 : cluster 3 failed to encode map e290 with expected crc 2023-09-23T20:02:41.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:40 smithi092 bash[58855]: cluster 2023-09-23T20:02:39.903103+0000 osd.6 (osd.6) 224 : cluster 3 failed to encode map e290 with expected crc 2023-09-23T20:02:41.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:40 smithi092 bash[58855]: cluster 2023-09-23T20:02:39.910010+0000 osd.7 (osd.7) 202 : cluster 3 failed to encode map e290 with expected crc 2023-09-23T20:02:41.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:40 smithi092 bash[58855]: cluster 2023-09-23T20:02:39.921665+0000 osd.4 (osd.4) 229 : cluster 3 failed to encode map e290 with expected crc 2023-09-23T20:02:41.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:40 smithi092 bash[58855]: cluster 2023-09-23T20:02:39.923528+0000 osd.4 (osd.4) 230 : cluster 3 failed to encode map e290 with expected crc 2023-09-23T20:02:41.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:40 smithi092 bash[58855]: audit 2023-09-23T20:02:40.696895+0000 mon.a (mon.0) 1999 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:02:41.332 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:41.362 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:41.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:41.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:41.466 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:41.494 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:41.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:41.496 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:41.597 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:41.626 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:41.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:41.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:41.729 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:41.756 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:41.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:41.758 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:41.859 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:41.891 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:41.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:41.894 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:41.995 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:42.022 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:42.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:42.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:42.072 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:41 smithi016 bash[77995]: cluster 2023-09-23T20:02:40.679646+0000 mgr.y (mgr.25325) 1665 : cluster 0 pgmap v1716: 73 pgs: 73 active+clean; 692 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 283 KiB/s rd, 25 KiB/s wr, 321 op/s 2023-09-23T20:02:42.072 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:41 smithi016 bash[77995]: cluster 2023-09-23T20:02:40.898106+0000 mon.a (mon.0) 2000 : cluster 0 osdmap e291: 8 total, 8 up, 8 in 2023-09-23T20:02:42.072 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:41 smithi016 bash[77995]: cluster 2023-09-23T20:02:40.899466+0000 osd.5 (osd.5) 211 : cluster 3 failed to encode map e291 with expected crc 2023-09-23T20:02:42.072 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:41 smithi016 bash[77995]: cluster 2023-09-23T20:02:40.906104+0000 osd.6 (osd.6) 225 : cluster 3 failed to encode map e291 with expected crc 2023-09-23T20:02:42.073 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:41 smithi016 bash[77995]: cluster 2023-09-23T20:02:40.912623+0000 osd.7 (osd.7) 203 : cluster 3 failed to encode map e291 with expected crc 2023-09-23T20:02:42.073 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:41 smithi016 bash[77995]: cluster 2023-09-23T20:02:40.912830+0000 osd.4 (osd.4) 231 : cluster 3 failed to encode map e291 with expected crc 2023-09-23T20:02:42.073 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:41 smithi016 bash[77995]: cluster 2023-09-23T20:02:40.914505+0000 osd.7 (osd.7) 204 : cluster 3 failed to encode map e291 with expected crc 2023-09-23T20:02:42.073 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:41 smithi016 bash[75004]: cluster 2023-09-23T20:02:40.679646+0000 mgr.y (mgr.25325) 1665 : cluster 0 pgmap v1716: 73 pgs: 73 active+clean; 692 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 283 KiB/s rd, 25 KiB/s wr, 321 op/s 2023-09-23T20:02:42.073 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:41 smithi016 bash[75004]: cluster 2023-09-23T20:02:40.898106+0000 mon.a (mon.0) 2000 : cluster 0 osdmap e291: 8 total, 8 up, 8 in 2023-09-23T20:02:42.073 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:41 smithi016 bash[75004]: cluster 2023-09-23T20:02:40.899466+0000 osd.5 (osd.5) 211 : cluster 3 failed to encode map e291 with expected crc 2023-09-23T20:02:42.073 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:41 smithi016 bash[75004]: cluster 2023-09-23T20:02:40.906104+0000 osd.6 (osd.6) 225 : cluster 3 failed to encode map e291 with expected crc 2023-09-23T20:02:42.074 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:41 smithi016 bash[75004]: cluster 2023-09-23T20:02:40.912623+0000 osd.7 (osd.7) 203 : cluster 3 failed to encode map e291 with expected crc 2023-09-23T20:02:42.074 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:41 smithi016 bash[75004]: cluster 2023-09-23T20:02:40.912830+0000 osd.4 (osd.4) 231 : cluster 3 failed to encode map e291 with expected crc 2023-09-23T20:02:42.074 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:41 smithi016 bash[75004]: cluster 2023-09-23T20:02:40.914505+0000 osd.7 (osd.7) 204 : cluster 3 failed to encode map e291 with expected crc 2023-09-23T20:02:42.125 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:42.152 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:42.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:42.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:42.256 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:42.269 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:41 smithi092 bash[58855]: cluster 2023-09-23T20:02:40.679646+0000 mgr.y (mgr.25325) 1665 : cluster 0 pgmap v1716: 73 pgs: 73 active+clean; 692 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 283 KiB/s rd, 25 KiB/s wr, 321 op/s 2023-09-23T20:02:42.270 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:41 smithi092 bash[58855]: cluster 2023-09-23T20:02:40.898106+0000 mon.a (mon.0) 2000 : cluster 0 osdmap e291: 8 total, 8 up, 8 in 2023-09-23T20:02:42.270 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:41 smithi092 bash[58855]: cluster 2023-09-23T20:02:40.899466+0000 osd.5 (osd.5) 211 : cluster 3 failed to encode map e291 with expected crc 2023-09-23T20:02:42.270 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:41 smithi092 bash[58855]: cluster 2023-09-23T20:02:40.906104+0000 osd.6 (osd.6) 225 : cluster 3 failed to encode map e291 with expected crc 2023-09-23T20:02:42.270 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:41 smithi092 bash[58855]: cluster 2023-09-23T20:02:40.912623+0000 osd.7 (osd.7) 203 : cluster 3 failed to encode map e291 with expected crc 2023-09-23T20:02:42.270 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:41 smithi092 bash[58855]: cluster 2023-09-23T20:02:40.912830+0000 osd.4 (osd.4) 231 : cluster 3 failed to encode map e291 with expected crc 2023-09-23T20:02:42.270 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:41 smithi092 bash[58855]: cluster 2023-09-23T20:02:40.914505+0000 osd.7 (osd.7) 204 : cluster 3 failed to encode map e291 with expected crc 2023-09-23T20:02:42.288 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:42.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:42.291 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:42.392 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:42.421 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:42.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:42.424 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:42.525 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:42.551 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:42.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:42.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:42.654 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:42.682 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:42.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:42.684 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:42.785 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:42.819 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:42.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:42.821 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:42.922 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:42.952 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:42.952 INFO:tasks.workunit.client.0.smithi016.stderr: Importing image: 33% complete... Importing image: 66% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T20:02:42.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:42.955 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:42.957 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd info testimg_import@snap1 2023-09-23T20:02:42.957 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep protected: False 2023-09-23T20:02:43.013 INFO:tasks.workunit.client.0.smithi016.stdout: protected: False 2023-09-23T20:02:43.014 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd info testimg_import@snap2 2023-09-23T20:02:43.014 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep protected: True 2023-09-23T20:02:43.056 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:43.070 INFO:tasks.workunit.client.0.smithi016.stdout: protected: True 2023-09-23T20:02:43.071 INFO:tasks.workunit.client.0.smithi016.stderr:+ rm /tmp/rbd_import_export_249364/snap_protect 2023-09-23T20:02:43.072 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd snap unprotect testimg@snap2 2023-09-23T20:02:43.083 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:43.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:43.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:43.122 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd snap unprotect testimg_import@snap2 2023-09-23T20:02:43.184 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd snap purge testimg 2023-09-23T20:02:43.186 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:43.210 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:43.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:43.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:43.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:42 smithi092 bash[58855]: cluster 2023-09-23T20:02:41.905229+0000 mon.a (mon.0) 2001 : cluster 0 osdmap e292: 8 total, 8 up, 8 in 2023-09-23T20:02:43.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:42 smithi092 bash[58855]: cluster 2023-09-23T20:02:41.907927+0000 osd.4 (osd.4) 232 : cluster 3 failed to encode map e292 with expected crc 2023-09-23T20:02:43.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:42 smithi092 bash[58855]: cluster 2023-09-23T20:02:41.911911+0000 osd.7 (osd.7) 205 : cluster 3 failed to encode map e292 with expected crc 2023-09-23T20:02:43.295 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:42 smithi016 bash[75004]: cluster 2023-09-23T20:02:41.905229+0000 mon.a (mon.0) 2001 : cluster 0 osdmap e292: 8 total, 8 up, 8 in 2023-09-23T20:02:43.296 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:42 smithi016 bash[75004]: cluster 2023-09-23T20:02:41.907927+0000 osd.4 (osd.4) 232 : cluster 3 failed to encode map e292 with expected crc 2023-09-23T20:02:43.296 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:42 smithi016 bash[75004]: cluster 2023-09-23T20:02:41.911911+0000 osd.7 (osd.7) 205 : cluster 3 failed to encode map e292 with expected crc 2023-09-23T20:02:43.296 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:42 smithi016 bash[77995]: cluster 2023-09-23T20:02:41.905229+0000 mon.a (mon.0) 2001 : cluster 0 osdmap e292: 8 total, 8 up, 8 in 2023-09-23T20:02:43.296 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:42 smithi016 bash[77995]: cluster 2023-09-23T20:02:41.907927+0000 osd.4 (osd.4) 232 : cluster 3 failed to encode map e292 with expected crc 2023-09-23T20:02:43.296 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:42 smithi016 bash[77995]: cluster 2023-09-23T20:02:41.911911+0000 osd.7 (osd.7) 205 : cluster 3 failed to encode map e292 with expected crc 2023-09-23T20:02:43.315 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:43.341 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:43.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:43.344 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:43.446 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:43.478 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:43.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:43.481 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:43.582 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:43.606 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:43.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:43.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:43.709 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:43.741 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:43.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:43.744 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:43.845 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:43.876 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:43.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:43.880 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:43.915 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:43 smithi016 bash[75004]: cluster 2023-09-23T20:02:41.970973+0000 osd.6 (osd.6) 226 : cluster 3 failed to encode map e292 with expected crc 2023-09-23T20:02:43.915 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:43 smithi016 bash[75004]: cluster 2023-09-23T20:02:42.680135+0000 mgr.y (mgr.25325) 1666 : cluster 0 pgmap v1719: 73 pgs: 73 active+clean; 692 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 278 KiB/s rd, 30 KiB/s wr, 365 op/s 2023-09-23T20:02:43.916 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:43 smithi016 bash[75004]: cluster 2023-09-23T20:02:42.915626+0000 osd.4 (osd.4) 233 : cluster 3 failed to encode map e293 with expected crc 2023-09-23T20:02:43.916 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:43 smithi016 bash[75004]: cluster 2023-09-23T20:02:42.915933+0000 mon.a (mon.0) 2002 : cluster 0 osdmap e293: 8 total, 8 up, 8 in 2023-09-23T20:02:43.916 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:43 smithi016 bash[75004]: cluster 2023-09-23T20:02:42.922398+0000 osd.7 (osd.7) 206 : cluster 3 failed to encode map e293 with expected crc 2023-09-23T20:02:43.916 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:43 smithi016 bash[75004]: audit 2023-09-23T20:02:43.700538+0000 mon.a (mon.0) 2003 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:02:43.980 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:44.006 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:44.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:44.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:44.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:43 smithi092 bash[58855]: cluster 2023-09-23T20:02:41.970973+0000 osd.6 (osd.6) 226 : cluster 3 failed to encode map e292 with expected crc 2023-09-23T20:02:44.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:43 smithi092 bash[58855]: cluster 2023-09-23T20:02:42.680135+0000 mgr.y (mgr.25325) 1666 : cluster 0 pgmap v1719: 73 pgs: 73 active+clean; 692 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 278 KiB/s rd, 30 KiB/s wr, 365 op/s 2023-09-23T20:02:44.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:43 smithi092 bash[58855]: cluster 2023-09-23T20:02:42.915626+0000 osd.4 (osd.4) 233 : cluster 3 failed to encode map e293 with expected crc 2023-09-23T20:02:44.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:43 smithi092 bash[58855]: cluster 2023-09-23T20:02:42.915933+0000 mon.a (mon.0) 2002 : cluster 0 osdmap e293: 8 total, 8 up, 8 in 2023-09-23T20:02:44.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:43 smithi092 bash[58855]: cluster 2023-09-23T20:02:42.922398+0000 osd.7 (osd.7) 206 : cluster 3 failed to encode map e293 with expected crc 2023-09-23T20:02:44.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:43 smithi092 bash[58855]: audit 2023-09-23T20:02:43.700538+0000 mon.a (mon.0) 2003 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:02:44.109 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:44.139 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:44.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:44.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:44.243 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:44.252 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:43 smithi016 bash[77995]: cluster 2023-09-23T20:02:41.970973+0000 osd.6 (osd.6) 226 : cluster 3 failed to encode map e292 with expected crc 2023-09-23T20:02:44.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:43 smithi016 bash[77995]: cluster 2023-09-23T20:02:42.680135+0000 mgr.y (mgr.25325) 1666 : cluster 0 pgmap v1719: 73 pgs: 73 active+clean; 692 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 278 KiB/s rd, 30 KiB/s wr, 365 op/s 2023-09-23T20:02:44.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:43 smithi016 bash[77995]: cluster 2023-09-23T20:02:42.915626+0000 osd.4 (osd.4) 233 : cluster 3 failed to encode map e293 with expected crc 2023-09-23T20:02:44.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:43 smithi016 bash[77995]: cluster 2023-09-23T20:02:42.915933+0000 mon.a (mon.0) 2002 : cluster 0 osdmap e293: 8 total, 8 up, 8 in 2023-09-23T20:02:44.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:43 smithi016 bash[77995]: cluster 2023-09-23T20:02:42.922398+0000 osd.7 (osd.7) 206 : cluster 3 failed to encode map e293 with expected crc 2023-09-23T20:02:44.253 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:43 smithi016 bash[77995]: audit 2023-09-23T20:02:43.700538+0000 mon.a (mon.0) 2003 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:02:44.270 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:44.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:44.273 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:44.374 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:44.397 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:44.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:44.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:44.500 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:44.523 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:44.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:44.526 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:44.627 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:44.662 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:44.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:44.665 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:44.765 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:44.796 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:44.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:44.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:44.851 INFO:tasks.workunit.client.0.smithi016.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-09-23T20:02:44.861 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd snap purge testimg_import 2023-09-23T20:02:44.900 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:44.927 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:44.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:44.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:45.031 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:45.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:44 smithi092 bash[58855]: cluster 2023-09-23T20:02:43.922149+0000 mon.a (mon.0) 2004 : cluster 0 osdmap e294: 8 total, 8 up, 8 in 2023-09-23T20:02:45.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:44 smithi092 bash[58855]: cluster 2023-09-23T20:02:43.923930+0000 osd.7 (osd.7) 207 : cluster 3 failed to encode map e294 with expected crc 2023-09-23T20:02:45.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:44 smithi092 bash[58855]: cluster 2023-09-23T20:02:43.939114+0000 osd.6 (osd.6) 227 : cluster 3 failed to encode map e294 with expected crc 2023-09-23T20:02:45.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:44 smithi092 bash[58855]: cluster 2023-09-23T20:02:44.106247+0000 osd.4 (osd.4) 234 : cluster 3 failed to encode map e294 with expected crc 2023-09-23T20:02:45.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:44 smithi092 bash[58855]: cluster 2023-09-23T20:02:44.573473+0000 osd.5 (osd.5) 212 : cluster 3 failed to encode map e294 with expected crc 2023-09-23T20:02:45.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:44 smithi092 bash[58855]: cluster 2023-09-23T20:02:44.841263+0000 mon.a (mon.0) 2005 : cluster 0 osdmap e295: 8 total, 8 up, 8 in 2023-09-23T20:02:45.062 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:45.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:45.065 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:45.166 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:45.178 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:44 smithi016 bash[77995]: cluster 2023-09-23T20:02:43.922149+0000 mon.a (mon.0) 2004 : cluster 0 osdmap e294: 8 total, 8 up, 8 in 2023-09-23T20:02:45.178 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:44 smithi016 bash[77995]: cluster 2023-09-23T20:02:43.923930+0000 osd.7 (osd.7) 207 : cluster 3 failed to encode map e294 with expected crc 2023-09-23T20:02:45.178 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:44 smithi016 bash[77995]: cluster 2023-09-23T20:02:43.939114+0000 osd.6 (osd.6) 227 : cluster 3 failed to encode map e294 with expected crc 2023-09-23T20:02:45.178 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:44 smithi016 bash[77995]: cluster 2023-09-23T20:02:44.106247+0000 osd.4 (osd.4) 234 : cluster 3 failed to encode map e294 with expected crc 2023-09-23T20:02:45.178 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:44 smithi016 bash[77995]: cluster 2023-09-23T20:02:44.573473+0000 osd.5 (osd.5) 212 : cluster 3 failed to encode map e294 with expected crc 2023-09-23T20:02:45.178 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:44 smithi016 bash[77995]: cluster 2023-09-23T20:02:44.841263+0000 mon.a (mon.0) 2005 : cluster 0 osdmap e295: 8 total, 8 up, 8 in 2023-09-23T20:02:45.179 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:44 smithi016 bash[75004]: cluster 2023-09-23T20:02:43.922149+0000 mon.a (mon.0) 2004 : cluster 0 osdmap e294: 8 total, 8 up, 8 in 2023-09-23T20:02:45.179 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:44 smithi016 bash[75004]: cluster 2023-09-23T20:02:43.923930+0000 osd.7 (osd.7) 207 : cluster 3 failed to encode map e294 with expected crc 2023-09-23T20:02:45.179 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:44 smithi016 bash[75004]: cluster 2023-09-23T20:02:43.939114+0000 osd.6 (osd.6) 227 : cluster 3 failed to encode map e294 with expected crc 2023-09-23T20:02:45.179 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:44 smithi016 bash[75004]: cluster 2023-09-23T20:02:44.106247+0000 osd.4 (osd.4) 234 : cluster 3 failed to encode map e294 with expected crc 2023-09-23T20:02:45.179 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:44 smithi016 bash[75004]: cluster 2023-09-23T20:02:44.573473+0000 osd.5 (osd.5) 212 : cluster 3 failed to encode map e294 with expected crc 2023-09-23T20:02:45.179 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:44 smithi016 bash[75004]: cluster 2023-09-23T20:02:44.841263+0000 mon.a (mon.0) 2005 : cluster 0 osdmap e295: 8 total, 8 up, 8 in 2023-09-23T20:02:45.197 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:45.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:45.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:45.300 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:45.335 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:45.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:45.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:45.440 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:45.470 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:45.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:45.474 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:45.575 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:45.597 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:45.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:45.599 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:45.700 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:45.725 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:45.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:45.727 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:45.828 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:45.844 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:02:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:20:02:45] "GET /metrics HTTP/1.1" 200 34002 "" "Prometheus/2.33.4" 2023-09-23T20:02:45.858 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:45.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:45.862 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:45.964 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:45.994 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:45.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:45.998 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:46.098 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:46.125 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:46.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:46.128 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:46.228 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:46.237 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:45 smithi092 bash[58855]: cluster 2023-09-23T20:02:44.681104+0000 mgr.y (mgr.25325) 1667 : cluster 0 pgmap v1722: 73 pgs: 73 active+clean; 802 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 227 KiB/s rd, 28 KiB/s wr, 308 op/s 2023-09-23T20:02:46.237 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:45 smithi092 bash[58855]: cluster 2023-09-23T20:02:44.843948+0000 osd.6 (osd.6) 228 : cluster 3 failed to encode map e295 with expected crc 2023-09-23T20:02:46.253 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:46.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:46.257 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:46.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:46 smithi016 bash[77995]: cluster 2023-09-23T20:02:44.681104+0000 mgr.y (mgr.25325) 1667 : cluster 0 pgmap v1722: 73 pgs: 73 active+clean; 802 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 227 KiB/s rd, 28 KiB/s wr, 308 op/s 2023-09-23T20:02:46.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:46 smithi016 bash[77995]: cluster 2023-09-23T20:02:44.843948+0000 osd.6 (osd.6) 228 : cluster 3 failed to encode map e295 with expected crc 2023-09-23T20:02:46.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:46 smithi016 bash[75004]: cluster 2023-09-23T20:02:44.681104+0000 mgr.y (mgr.25325) 1667 : cluster 0 pgmap v1722: 73 pgs: 73 active+clean; 802 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 227 KiB/s rd, 28 KiB/s wr, 308 op/s 2023-09-23T20:02:46.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:46 smithi016 bash[75004]: cluster 2023-09-23T20:02:44.843948+0000 osd.6 (osd.6) 228 : cluster 3 failed to encode map e295 with expected crc 2023-09-23T20:02:46.357 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:46.383 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:46.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:46.386 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:46.487 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:46.520 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:46.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:46.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:46.624 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:46.652 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:46.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:46.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:46.755 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:46.812 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:46.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:46.816 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:46.917 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:46.949 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:46.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:46.953 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:46.954 INFO:tasks.workunit.client.0.smithi016.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-09-23T20:02:46.965 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd remove testimg 2023-09-23T20:02:47.053 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:47.063 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:46 smithi092 bash[58855]: cluster 2023-09-23T20:02:45.925695+0000 mon.a (mon.0) 2006 : cluster 0 osdmap e296: 8 total, 8 up, 8 in 2023-09-23T20:02:47.063 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:46 smithi092 bash[58855]: cluster 2023-09-23T20:02:45.927387+0000 osd.7 (osd.7) 208 : cluster 3 failed to encode map e296 with expected crc 2023-09-23T20:02:47.079 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:47.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:47.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:47.106 INFO:tasks.workunit.client.0.smithi016.stderr: Removing image: 100% complete...done. 2023-09-23T20:02:47.110 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd remove testimg_import 2023-09-23T20:02:47.183 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:47.192 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:46 smithi016 bash[77995]: cluster 2023-09-23T20:02:45.925695+0000 mon.a (mon.0) 2006 : cluster 0 osdmap e296: 8 total, 8 up, 8 in 2023-09-23T20:02:47.192 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:46 smithi016 bash[77995]: cluster 2023-09-23T20:02:45.927387+0000 osd.7 (osd.7) 208 : cluster 3 failed to encode map e296 with expected crc 2023-09-23T20:02:47.192 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:46 smithi016 bash[75004]: cluster 2023-09-23T20:02:45.925695+0000 mon.a (mon.0) 2006 : cluster 0 osdmap e296: 8 total, 8 up, 8 in 2023-09-23T20:02:47.192 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:46 smithi016 bash[75004]: cluster 2023-09-23T20:02:45.927387+0000 osd.7 (osd.7) 208 : cluster 3 failed to encode map e296 with expected crc 2023-09-23T20:02:47.210 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:47.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:47.212 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:47.238 INFO:tasks.workunit.client.0.smithi016.stderr: Removing image: 100% complete...done. 2023-09-23T20:02:47.243 INFO:tasks.workunit.client.0.smithi016.stderr:+ tiered=0 2023-09-23T20:02:47.243 INFO:tasks.workunit.client.0.smithi016.stderr:+ ceph osd dump 2023-09-23T20:02:47.243 INFO:tasks.workunit.client.0.smithi016.stderr:+ + grep ^poolgrep 2023-09-23T20:02:47.244 INFO:tasks.workunit.client.0.smithi016.stderr: 'rbd' 2023-09-23T20:02:47.244 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep tier 2023-09-23T20:02:47.313 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:47.338 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:47.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:47.341 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:47.441 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:47.474 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:47.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:47.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:47.578 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:47.607 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:47.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:47.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:47.710 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:47.731 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:47.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:47.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:47.835 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:47.862 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:47.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:47.864 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:47.965 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:47.991 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:47.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:47.993 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:48.094 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:48.102 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:47 smithi016 bash[77995]: cluster 2023-09-23T20:02:45.984154+0000 osd.6 (osd.6) 229 : cluster 3 failed to encode map e296 with expected crc 2023-09-23T20:02:48.102 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:47 smithi016 bash[77995]: cluster 2023-09-23T20:02:46.681775+0000 mgr.y (mgr.25325) 1668 : cluster 0 pgmap v1725: 73 pgs: 73 active+clean; 802 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 112 KiB/s rd, 10 KiB/s wr, 153 op/s 2023-09-23T20:02:48.102 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:47 smithi016 bash[77995]: cluster 2023-09-23T20:02:46.941080+0000 mon.a (mon.0) 2007 : cluster 0 osdmap e297: 8 total, 8 up, 8 in 2023-09-23T20:02:48.102 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:47 smithi016 bash[77995]: audit 2023-09-23T20:02:46.945949+0000 mon.a (mon.0) 2008 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:02:48.102 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:47 smithi016 bash[77995]: cluster 2023-09-23T20:02:46.947666+0000 osd.7 (osd.7) 209 : cluster 3 failed to encode map e297 with expected crc 2023-09-23T20:02:48.102 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:47 smithi016 bash[77995]: audit 2023-09-23T20:02:46.953532+0000 mon.a (mon.0) 2009 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:02:48.102 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:47 smithi016 bash[77995]: cluster 2023-09-23T20:02:47.053431+0000 osd.5 (osd.5) 213 : cluster 3 failed to encode map e297 with expected crc 2023-09-23T20:02:48.103 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:47 smithi016 bash[75004]: cluster 2023-09-23T20:02:45.984154+0000 osd.6 (osd.6) 229 : cluster 3 failed to encode map e296 with expected crc 2023-09-23T20:02:48.103 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:47 smithi016 bash[75004]: cluster 2023-09-23T20:02:46.681775+0000 mgr.y (mgr.25325) 1668 : cluster 0 pgmap v1725: 73 pgs: 73 active+clean; 802 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 112 KiB/s rd, 10 KiB/s wr, 153 op/s 2023-09-23T20:02:48.103 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:47 smithi016 bash[75004]: cluster 2023-09-23T20:02:46.941080+0000 mon.a (mon.0) 2007 : cluster 0 osdmap e297: 8 total, 8 up, 8 in 2023-09-23T20:02:48.103 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:47 smithi016 bash[75004]: audit 2023-09-23T20:02:46.945949+0000 mon.a (mon.0) 2008 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:02:48.103 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:47 smithi016 bash[75004]: cluster 2023-09-23T20:02:46.947666+0000 osd.7 (osd.7) 209 : cluster 3 failed to encode map e297 with expected crc 2023-09-23T20:02:48.103 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:47 smithi016 bash[75004]: audit 2023-09-23T20:02:46.953532+0000 mon.a (mon.0) 2009 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:02:48.103 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:47 smithi016 bash[75004]: cluster 2023-09-23T20:02:47.053431+0000 osd.5 (osd.5) 213 : cluster 3 failed to encode map e297 with expected crc 2023-09-23T20:02:48.119 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:48.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:48.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:48.223 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:48.249 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:48.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:48.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:48.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:47 smithi092 bash[58855]: cluster 2023-09-23T20:02:45.984154+0000 osd.6 (osd.6) 229 : cluster 3 failed to encode map e296 with expected crc 2023-09-23T20:02:48.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:47 smithi092 bash[58855]: cluster 2023-09-23T20:02:46.681775+0000 mgr.y (mgr.25325) 1668 : cluster 0 pgmap v1725: 73 pgs: 73 active+clean; 802 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 112 KiB/s rd, 10 KiB/s wr, 153 op/s 2023-09-23T20:02:48.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:47 smithi092 bash[58855]: cluster 2023-09-23T20:02:46.941080+0000 mon.a (mon.0) 2007 : cluster 0 osdmap e297: 8 total, 8 up, 8 in 2023-09-23T20:02:48.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:47 smithi092 bash[58855]: audit 2023-09-23T20:02:46.945949+0000 mon.a (mon.0) 2008 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:02:48.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:47 smithi092 bash[58855]: cluster 2023-09-23T20:02:46.947666+0000 osd.7 (osd.7) 209 : cluster 3 failed to encode map e297 with expected crc 2023-09-23T20:02:48.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:47 smithi092 bash[58855]: audit 2023-09-23T20:02:46.953532+0000 mon.a (mon.0) 2009 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:02:48.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:47 smithi092 bash[58855]: cluster 2023-09-23T20:02:47.053431+0000 osd.5 (osd.5) 213 : cluster 3 failed to encode map e297 with expected crc 2023-09-23T20:02:48.353 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:48.388 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:48.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:48.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:48.492 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:48.524 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:48.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:48.527 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:48.534 INFO:tasks.workunit.client.0.smithi016.stderr:+ dd if=/dev/urandom bs=1M seek=1 count=1 of=/tmp/rbd_import_export_249364/sparse1 2023-09-23T20:02:48.539 INFO:tasks.workunit.client.0.smithi016.stderr:1+0 records in 2023-09-23T20:02:48.539 INFO:tasks.workunit.client.0.smithi016.stderr:1+0 records out 2023-09-23T20:02:48.539 INFO:tasks.workunit.client.0.smithi016.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00340786 s, 308 MB/s 2023-09-23T20:02:48.539 INFO:tasks.workunit.client.0.smithi016.stderr:+ dd if=/dev/urandom bs=1M count=1 of=/tmp/rbd_import_export_249364/sparse2 2023-09-23T20:02:48.546 INFO:tasks.workunit.client.0.smithi016.stderr:1+0 records in 2023-09-23T20:02:48.546 INFO:tasks.workunit.client.0.smithi016.stderr:1+0 records out 2023-09-23T20:02:48.546 INFO:tasks.workunit.client.0.smithi016.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00644181 s, 163 MB/s 2023-09-23T20:02:48.546 INFO:tasks.workunit.client.0.smithi016.stderr:+ truncate /tmp/rbd_import_export_249364/sparse2 -s 2M 2023-09-23T20:02:48.628 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:48.655 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:48.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:48.658 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:48.733 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd rm sparse1 2023-09-23T20:02:48.760 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:48.782 INFO:tasks.workunit.client.0.smithi016.stderr: Removing image: 0% complete...failed. 2023-09-23T20:02:48.782 INFO:tasks.workunit.client.0.smithi016.stderr:rbd: delete error: (2) No such file or directory 2023-09-23T20:02:48.786 INFO:tasks.workunit.client.0.smithi016.stderr:+ true 2023-09-23T20:02:48.786 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_249364/sparse1 2023-09-23T20:02:48.789 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:48.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:48.793 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:48.869 INFO:tasks.workunit.client.0.smithi016.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-09-23T20:02:48.869 INFO:tasks.workunit.client.0.smithi016.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T20:02:48.869 INFO:tasks.workunit.client.0.smithi016.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T20:02:48.873 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd ls -l 2023-09-23T20:02:48.873 INFO:tasks.workunit.client.0.smithi016.stderr:+ + grep sparse1 2023-09-23T20:02:48.873 INFO:tasks.workunit.client.0.smithi016.stderr:grep -Ei (2 MiB|2048k) 2023-09-23T20:02:48.893 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:48.926 INFO:tasks.workunit.client.0.smithi016.stdout:sparse1 2 MiB 2 2023-09-23T20:02:48.927 INFO:tasks.workunit.client.0.smithi016.stderr:+ objects sparse1 2023-09-23T20:02:48.927 INFO:tasks.workunit.client.0.smithi016.stderr:+ image=sparse1 2023-09-23T20:02:48.927 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:48.928 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep block_name_prefix 2023-09-23T20:02:48.928 INFO:tasks.workunit.client.0.smithi016.stderr:+ awk {print $NF;} 2023-09-23T20:02:48.928 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd info sparse1 2023-09-23T20:02:48.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:48.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:48.977 INFO:tasks.workunit.client.0.smithi016.stderr:+ prefix=rbd_data.9273b5e4519e 2023-09-23T20:02:48.978 INFO:tasks.workunit.client.0.smithi016.stderr:+ + grep rbd_data.9273b5e4519e 2023-09-23T20:02:48.978 INFO:tasks.workunit.client.0.smithi016.stderr:get_image_data_pool sparse1 2023-09-23T20:02:48.978 INFO:tasks.workunit.client.0.smithi016.stderr:+ image=sparse1 2023-09-23T20:02:48.979 INFO:tasks.workunit.client.0.smithi016.stderr:+ sed -e s/rbd_data.9273b5e4519e\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-09-23T20:02:48.979 INFO:tasks.workunit.client.0.smithi016.stderr:+ sort -u 2023-09-23T20:02:48.979 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd info sparse1 2023-09-23T20:02:48.979 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep data_pool: 2023-09-23T20:02:48.979 INFO:tasks.workunit.client.0.smithi016.stderr:+ awk -F: { print $NF } 2023-09-23T20:02:49.029 INFO:tasks.workunit.client.0.smithi016.stderr:+ data_pool= 2023-09-23T20:02:49.029 INFO:tasks.workunit.client.0.smithi016.stderr:+ [ -z ] 2023-09-23T20:02:49.029 INFO:tasks.workunit.client.0.smithi016.stderr:+ data_pool=rbd 2023-09-23T20:02:49.029 INFO:tasks.workunit.client.0.smithi016.stderr:+ echo rbd 2023-09-23T20:02:49.030 INFO:tasks.workunit.client.0.smithi016.stderr:+ rados ls -p rbd 2023-09-23T20:02:49.031 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:49.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:48 smithi092 bash[58855]: cluster 2023-09-23T20:02:47.016023+0000 osd.6 (osd.6) 230 : cluster 3 failed to encode map e297 with expected crc 2023-09-23T20:02:49.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:48 smithi092 bash[58855]: audit 2023-09-23T20:02:48.413721+0000 mon.c (mon.2) 248 : audit 0 from='client.? 172.21.15.16:0/1667327378' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T20:02:49.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:48 smithi092 bash[58855]: audit 2023-09-23T20:02:48.518600+0000 mon.c (mon.2) 249 : audit 0 from='client.? 172.21.15.16:0/1667327378' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T20:02:49.064 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:49.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:49.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:49.071 INFO:tasks.workunit.client.0.smithi016.stderr:+ objects=1 2023-09-23T20:02:49.071 INFO:tasks.workunit.client.0.smithi016.stderr:+ echo 1 2023-09-23T20:02:49.071 INFO:tasks.workunit.client.0.smithi016.stderr:+ [ 0 -eq 1 -o 1 = 1 ] 2023-09-23T20:02:49.071 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd export sparse1 /tmp/rbd_import_export_249364/sparse1.out 2023-09-23T20:02:49.115 INFO:tasks.workunit.client.0.smithi016.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-09-23T20:02:49.121 INFO:tasks.workunit.client.0.smithi016.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_249364/sparse1 /tmp/rbd_import_export_249364/sparse1.out 2023-09-23T20:02:49.121 INFO:tasks.workunit.client.0.smithi016.stderr:+ cmp -l /tmp/rbd_import_export_249364/sparse1 /tmp/rbd_import_export_249364/sparse1.out 2023-09-23T20:02:49.124 INFO:tasks.workunit.client.0.smithi016.stderr:+ stat /tmp/rbd_import_export_249364/sparse1 --format %b 2023-09-23T20:02:49.125 INFO:tasks.workunit.client.0.smithi016.stderr:+ origsize=2048 2023-09-23T20:02:49.125 INFO:tasks.workunit.client.0.smithi016.stderr:+ stat /tmp/rbd_import_export_249364/sparse1.out --format %b 2023-09-23T20:02:49.126 INFO:tasks.workunit.client.0.smithi016.stderr:+ exportsize=2048 2023-09-23T20:02:49.126 INFO:tasks.workunit.client.0.smithi016.stderr:+ difference=0 2023-09-23T20:02:49.126 INFO:tasks.workunit.client.0.smithi016.stderr:+ difference=0 2023-09-23T20:02:49.126 INFO:tasks.workunit.client.0.smithi016.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T20:02:49.126 INFO:tasks.workunit.client.0.smithi016.stderr:+ rm /tmp/rbd_import_export_249364/sparse1.out 2023-09-23T20:02:49.126 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd rm sparse1 2023-09-23T20:02:49.168 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:49.195 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:49.196 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:48 smithi016 bash[77995]: cluster 2023-09-23T20:02:47.016023+0000 osd.6 (osd.6) 230 : cluster 3 failed to encode map e297 with expected crc 2023-09-23T20:02:49.196 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:48 smithi016 bash[77995]: audit 2023-09-23T20:02:48.413721+0000 mon.c (mon.2) 248 : audit 0 from='client.? 172.21.15.16:0/1667327378' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T20:02:49.197 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:48 smithi016 bash[77995]: audit 2023-09-23T20:02:48.518600+0000 mon.c (mon.2) 249 : audit 0 from='client.? 172.21.15.16:0/1667327378' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T20:02:49.197 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:48 smithi016 bash[75004]: cluster 2023-09-23T20:02:47.016023+0000 osd.6 (osd.6) 230 : cluster 3 failed to encode map e297 with expected crc 2023-09-23T20:02:49.197 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:48 smithi016 bash[75004]: audit 2023-09-23T20:02:48.413721+0000 mon.c (mon.2) 248 : audit 0 from='client.? 172.21.15.16:0/1667327378' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T20:02:49.197 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:48 smithi016 bash[75004]: audit 2023-09-23T20:02:48.518600+0000 mon.c (mon.2) 249 : audit 0 from='client.? 172.21.15.16:0/1667327378' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T20:02:49.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:49.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:49.205 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-09-23T20:02:49.205 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [], 2023-09-23T20:02:49.205 INFO:teuthology.orchestra.run.smithi016.stdout: "complaint_time": 30, 2023-09-23T20:02:49.205 INFO:teuthology.orchestra.run.smithi016.stdout: "num_blocked_ops": 0 2023-09-23T20:02:49.205 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-09-23T20:02:49.286 INFO:tasks.workunit.client.0.smithi016.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-09-23T20:02:49.292 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd rm sparse2 2023-09-23T20:02:49.299 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:49.328 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:49.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:49.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:49.336 INFO:tasks.workunit.client.0.smithi016.stderr: Removing image: 0% complete...failed. 2023-09-23T20:02:49.336 INFO:tasks.workunit.client.0.smithi016.stderr:rbd: delete error: (2) No such file or directory 2023-09-23T20:02:49.341 INFO:tasks.workunit.client.0.smithi016.stderr:+ true 2023-09-23T20:02:49.341 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_249364/sparse2 2023-09-23T20:02:49.416 INFO:tasks.workunit.client.0.smithi016.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-09-23T20:02:49.417 INFO:tasks.workunit.client.0.smithi016.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T20:02:49.417 INFO:tasks.workunit.client.0.smithi016.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T20:02:49.420 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd ls -l 2023-09-23T20:02:49.421 INFO:tasks.workunit.client.0.smithi016.stderr:+ + grep -Ei (2 MiB|2048k) 2023-09-23T20:02:49.421 INFO:tasks.workunit.client.0.smithi016.stderr:grep sparse2 2023-09-23T20:02:49.431 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:49.467 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:49.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:49.470 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:49.471 INFO:tasks.workunit.client.0.smithi016.stdout:sparse2 2 MiB 2 2023-09-23T20:02:49.473 INFO:tasks.workunit.client.0.smithi016.stderr:+ objects sparse2 2023-09-23T20:02:49.473 INFO:tasks.workunit.client.0.smithi016.stderr:+ image=sparse2 2023-09-23T20:02:49.473 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd info sparse2 2023-09-23T20:02:49.473 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep block_name_prefix 2023-09-23T20:02:49.473 INFO:tasks.workunit.client.0.smithi016.stderr:+ awk {print $NF;} 2023-09-23T20:02:49.530 INFO:tasks.workunit.client.0.smithi016.stderr:+ prefix=rbd_data.b99063849fa7 2023-09-23T20:02:49.530 INFO:tasks.workunit.client.0.smithi016.stderr:+ + grep rbd_data.b99063849fa7 2023-09-23T20:02:49.530 INFO:tasks.workunit.client.0.smithi016.stderr:+ sed -e s/rbd_data.b99063849fa7\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-09-23T20:02:49.530 INFO:tasks.workunit.client.0.smithi016.stderr:+ get_image_data_pool sparse2 2023-09-23T20:02:49.530 INFO:tasks.workunit.client.0.smithi016.stderr:+ image=sparse2 2023-09-23T20:02:49.530 INFO:tasks.workunit.client.0.smithi016.stderr:sort -u 2023-09-23T20:02:49.530 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd info sparse2+ grep data_pool: 2023-09-23T20:02:49.530 INFO:tasks.workunit.client.0.smithi016.stderr: 2023-09-23T20:02:49.531 INFO:tasks.workunit.client.0.smithi016.stderr:+ awk -F: { print $NF } 2023-09-23T20:02:49.570 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:49.588 INFO:tasks.workunit.client.0.smithi016.stderr:+ data_pool= 2023-09-23T20:02:49.588 INFO:tasks.workunit.client.0.smithi016.stderr:+ [ -z ] 2023-09-23T20:02:49.589 INFO:tasks.workunit.client.0.smithi016.stderr:+ data_pool=rbd 2023-09-23T20:02:49.589 INFO:tasks.workunit.client.0.smithi016.stderr:+ echo rbd 2023-09-23T20:02:49.589 INFO:tasks.workunit.client.0.smithi016.stderr:+ rados ls -p rbd 2023-09-23T20:02:49.591 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:49.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:49.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:49.637 INFO:tasks.workunit.client.0.smithi016.stderr:+ objects=0 2023-09-23T20:02:49.637 INFO:tasks.workunit.client.0.smithi016.stderr:+ echo 0 2023-09-23T20:02:49.637 INFO:tasks.workunit.client.0.smithi016.stderr:+ [ 0 -eq 1 -o 0 = 0 ] 2023-09-23T20:02:49.637 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd export sparse2 /tmp/rbd_import_export_249364/sparse2.out 2023-09-23T20:02:49.676 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T20:02:49.688 INFO:tasks.workunit.client.0.smithi016.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-09-23T20:02:49.693 INFO:tasks.workunit.client.0.smithi016.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_249364/sparse2 /tmp/rbd_import_export_249364/sparse2.out 2023-09-23T20:02:49.693 INFO:tasks.workunit.client.0.smithi016.stderr:+ cmp -l /tmp/rbd_import_export_249364/sparse2 /tmp/rbd_import_export_249364/sparse2.out 2023-09-23T20:02:49.695 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:49.696 INFO:tasks.workunit.client.0.smithi016.stderr:+ stat /tmp/rbd_import_export_249364/sparse2 --format %b 2023-09-23T20:02:49.696 INFO:tasks.workunit.client.0.smithi016.stderr:+ origsize=2048 2023-09-23T20:02:49.696 INFO:tasks.workunit.client.0.smithi016.stderr:+ stat /tmp/rbd_import_export_249364/sparse2.out --format %b 2023-09-23T20:02:49.697 INFO:tasks.workunit.client.0.smithi016.stderr:+ exportsize=2048 2023-09-23T20:02:49.698 INFO:tasks.workunit.client.0.smithi016.stderr:+ difference=0 2023-09-23T20:02:49.698 INFO:tasks.workunit.client.0.smithi016.stderr:+ difference=0 2023-09-23T20:02:49.698 INFO:tasks.workunit.client.0.smithi016.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T20:02:49.698 INFO:tasks.workunit.client.0.smithi016.stderr:+ rm /tmp/rbd_import_export_249364/sparse2.out 2023-09-23T20:02:49.698 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T20:02:49.698 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd rm sparse2 2023-09-23T20:02:49.713 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:49 smithi016 bash[75004]: debug 2023-09-23T20:02:49.705+0000 7f0907c76700 -1 mon.a@0(leader).osd e297 definitely_dead 0 2023-09-23T20:02:49.717 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:49.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:49.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:49.721 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T20:02:49.745 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T20:02:49.767 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T20:02:49.792 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T20:02:49.820 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T20:02:49.820 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:49.833 INFO:tasks.workunit.client.0.smithi016.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-09-23T20:02:49.837 INFO:tasks.workunit.client.0.smithi016.stderr:+ truncate /tmp/rbd_import_export_249364/sparse1 -s 10M 2023-09-23T20:02:49.838 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd import --new-format --order 20 - sparse1 2023-09-23T20:02:49.850 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:49.853 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T20:02:49.857 INFO:teuthology.orchestra.run.smithi016.stderr:marked out osd.1. 2023-09-23T20:02:49.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:49.860 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:49.896 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:02:49.896 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:02:49.897 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:02:49.897 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:02:49.897 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:02:49.897 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:02:49.897 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:02:49.897 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:02:49.898 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is set 2023-09-23T20:02:49.961 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:49.969 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:49 smithi016 bash[77995]: cluster 2023-09-23T20:02:48.682506+0000 mgr.y (mgr.25325) 1669 : cluster 0 pgmap v1727: 73 pgs: 73 active+clean; 802 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 98 KiB/s rd, 8.2 KiB/s wr, 129 op/s 2023-09-23T20:02:49.969 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:49 smithi016 bash[77995]: audit 2023-09-23T20:02:49.528998+0000 mon.a (mon.0) 2010 : audit 1 from='client.? 172.21.15.16:0/757593284' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:02:49.969 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:49 smithi016 bash[77995]: audit 2023-09-23T20:02:49.649074+0000 mon.c (mon.2) 250 : audit 0 from='client.? 172.21.15.16:0/2270319518' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:02:49.969 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:49 smithi016 bash[77995]: audit 2023-09-23T20:02:49.710103+0000 mon.c (mon.2) 251 : audit 1 from='client.? 172.21.15.16:0/3743524925' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-09-23T20:02:49.969 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:49 smithi016 bash[77995]: audit 2023-09-23T20:02:49.710599+0000 mon.a (mon.0) 2011 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-09-23T20:02:49.969 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:49 smithi016 bash[77995]: cluster 2023-09-23T20:02:49.710731+0000 mon.a (mon.0) 2012 : cluster 1 Client client.admin marked osd.1 out, while it was still marked up 2023-09-23T20:02:49.969 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:49 smithi016 bash[77995]: cluster 2023-09-23T20:02:49.838142+0000 mon.a (mon.0) 2013 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:02:49.970 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:49 smithi016 bash[77995]: audit 2023-09-23T20:02:49.843099+0000 mon.a (mon.0) 2014 : audit 1 from='client.? 172.21.15.16:0/757593284' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:02:49.970 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:49 smithi016 bash[77995]: audit 2023-09-23T20:02:49.843173+0000 mon.a (mon.0) 2015 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-09-23T20:02:49.970 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:49 smithi016 bash[77995]: cluster 2023-09-23T20:02:49.843242+0000 mon.a (mon.0) 2016 : cluster 0 osdmap e298: 8 total, 8 up, 7 in 2023-09-23T20:02:49.970 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:49 smithi016 bash[77995]: cluster 2023-09-23T20:02:49.857660+0000 osd.6 (osd.6) 231 : cluster 3 failed to encode map e298 with expected crc 2023-09-23T20:02:49.970 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:49 smithi016 bash[77995]: cluster 2023-09-23T20:02:49.861334+0000 osd.6 (osd.6) 232 : cluster 3 failed to encode map e298 with expected crc 2023-09-23T20:02:49.970 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:49 smithi016 bash[75004]: cluster 2023-09-23T20:02:48.682506+0000 mgr.y (mgr.25325) 1669 : cluster 0 pgmap v1727: 73 pgs: 73 active+clean; 802 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 98 KiB/s rd, 8.2 KiB/s wr, 129 op/s 2023-09-23T20:02:49.970 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:49 smithi016 bash[75004]: audit 2023-09-23T20:02:49.528998+0000 mon.a (mon.0) 2010 : audit 1 from='client.? 172.21.15.16:0/757593284' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:02:49.970 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:49 smithi016 bash[75004]: audit 2023-09-23T20:02:49.649074+0000 mon.c (mon.2) 250 : audit 0 from='client.? 172.21.15.16:0/2270319518' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:02:49.971 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:49 smithi016 bash[75004]: audit 2023-09-23T20:02:49.710103+0000 mon.c (mon.2) 251 : audit 1 from='client.? 172.21.15.16:0/3743524925' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-09-23T20:02:49.971 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:49 smithi016 bash[75004]: audit 2023-09-23T20:02:49.710599+0000 mon.a (mon.0) 2011 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-09-23T20:02:49.971 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:49 smithi016 bash[75004]: cluster 2023-09-23T20:02:49.710731+0000 mon.a (mon.0) 2012 : cluster 1 Client client.admin marked osd.1 out, while it was still marked up 2023-09-23T20:02:49.971 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:49 smithi016 bash[75004]: cluster 2023-09-23T20:02:49.838142+0000 mon.a (mon.0) 2013 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:02:49.971 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:49 smithi016 bash[75004]: audit 2023-09-23T20:02:49.843099+0000 mon.a (mon.0) 2014 : audit 1 from='client.? 172.21.15.16:0/757593284' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:02:49.971 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:49 smithi016 bash[75004]: audit 2023-09-23T20:02:49.843173+0000 mon.a (mon.0) 2015 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-09-23T20:02:49.971 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:49 smithi016 bash[75004]: cluster 2023-09-23T20:02:49.843242+0000 mon.a (mon.0) 2016 : cluster 0 osdmap e298: 8 total, 8 up, 7 in 2023-09-23T20:02:49.971 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:49 smithi016 bash[75004]: cluster 2023-09-23T20:02:49.857660+0000 osd.6 (osd.6) 231 : cluster 3 failed to encode map e298 with expected crc 2023-09-23T20:02:49.971 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:49 smithi016 bash[75004]: cluster 2023-09-23T20:02:49.861334+0000 osd.6 (osd.6) 232 : cluster 3 failed to encode map e298 with expected crc 2023-09-23T20:02:49.985 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:49.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:49.988 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:50.089 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:50.102 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:49 smithi092 bash[58855]: cluster 2023-09-23T20:02:48.682506+0000 mgr.y (mgr.25325) 1669 : cluster 0 pgmap v1727: 73 pgs: 73 active+clean; 802 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 98 KiB/s rd, 8.2 KiB/s wr, 129 op/s 2023-09-23T20:02:50.102 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:49 smithi092 bash[58855]: audit 2023-09-23T20:02:49.528998+0000 mon.a (mon.0) 2010 : audit 1 from='client.? 172.21.15.16:0/757593284' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:02:50.102 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:49 smithi092 bash[58855]: audit 2023-09-23T20:02:49.649074+0000 mon.c (mon.2) 250 : audit 0 from='client.? 172.21.15.16:0/2270319518' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:02:50.102 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:49 smithi092 bash[58855]: audit 2023-09-23T20:02:49.710103+0000 mon.c (mon.2) 251 : audit 1 from='client.? 172.21.15.16:0/3743524925' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-09-23T20:02:50.103 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:49 smithi092 bash[58855]: audit 2023-09-23T20:02:49.710599+0000 mon.a (mon.0) 2011 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-09-23T20:02:50.103 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:49 smithi092 bash[58855]: cluster 2023-09-23T20:02:49.710731+0000 mon.a (mon.0) 2012 : cluster 1 Client client.admin marked osd.1 out, while it was still marked up 2023-09-23T20:02:50.103 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:49 smithi092 bash[58855]: cluster 2023-09-23T20:02:49.838142+0000 mon.a (mon.0) 2013 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:02:50.103 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:49 smithi092 bash[58855]: audit 2023-09-23T20:02:49.843099+0000 mon.a (mon.0) 2014 : audit 1 from='client.? 172.21.15.16:0/757593284' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:02:50.103 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:49 smithi092 bash[58855]: audit 2023-09-23T20:02:49.843173+0000 mon.a (mon.0) 2015 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-09-23T20:02:50.103 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:49 smithi092 bash[58855]: cluster 2023-09-23T20:02:49.843242+0000 mon.a (mon.0) 2016 : cluster 0 osdmap e298: 8 total, 8 up, 7 in 2023-09-23T20:02:50.103 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:49 smithi092 bash[58855]: cluster 2023-09-23T20:02:49.857660+0000 osd.6 (osd.6) 231 : cluster 3 failed to encode map e298 with expected crc 2023-09-23T20:02:50.104 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:49 smithi092 bash[58855]: cluster 2023-09-23T20:02:49.861334+0000 osd.6 (osd.6) 232 : cluster 3 failed to encode map e298 with expected crc 2023-09-23T20:02:50.119 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:50.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:50.122 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:50.223 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:50.245 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:50.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:50.247 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:50.348 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:50.369 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:50.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:50.371 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:50.472 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:50.500 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:50.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:50.503 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:50.604 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:50.634 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:50.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:50.638 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:50.738 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:50.772 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:50.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:50.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:50.875 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:50.908 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:50.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:50.910 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:51.011 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:51.036 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:51.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:51.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:51.138 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:51.164 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:51.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:51.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:51.267 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:51.274 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:50 smithi016 bash[77995]: cluster 2023-09-23T20:02:49.852867+0000 osd.7 (osd.7) 210 : cluster 3 failed to encode map e298 with expected crc 2023-09-23T20:02:51.274 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:50 smithi016 bash[77995]: cluster 2023-09-23T20:02:49.853465+0000 osd.4 (osd.4) 235 : cluster 3 failed to encode map e298 with expected crc 2023-09-23T20:02:51.275 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:50 smithi016 bash[77995]: cluster 2023-09-23T20:02:49.858226+0000 osd.5 (osd.5) 214 : cluster 3 failed to encode map e298 with expected crc 2023-09-23T20:02:51.275 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:50 smithi016 bash[77995]: cluster 2023-09-23T20:02:50.855477+0000 mon.a (mon.0) 2017 : cluster 0 osdmap e299: 8 total, 8 up, 7 in 2023-09-23T20:02:51.275 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:50 smithi016 bash[77995]: cluster 2023-09-23T20:02:50.856998+0000 osd.6 (osd.6) 233 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T20:02:51.275 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:50 smithi016 bash[77995]: cluster 2023-09-23T20:02:50.861594+0000 osd.6 (osd.6) 234 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T20:02:51.275 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:50 smithi016 bash[77995]: cluster 2023-09-23T20:02:50.863565+0000 osd.6 (osd.6) 235 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T20:02:51.275 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:50 smithi016 bash[75004]: cluster 2023-09-23T20:02:49.852867+0000 osd.7 (osd.7) 210 : cluster 3 failed to encode map e298 with expected crc 2023-09-23T20:02:51.275 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:50 smithi016 bash[75004]: cluster 2023-09-23T20:02:49.853465+0000 osd.4 (osd.4) 235 : cluster 3 failed to encode map e298 with expected crc 2023-09-23T20:02:51.275 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:50 smithi016 bash[75004]: cluster 2023-09-23T20:02:49.858226+0000 osd.5 (osd.5) 214 : cluster 3 failed to encode map e298 with expected crc 2023-09-23T20:02:51.276 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:50 smithi016 bash[75004]: cluster 2023-09-23T20:02:50.855477+0000 mon.a (mon.0) 2017 : cluster 0 osdmap e299: 8 total, 8 up, 7 in 2023-09-23T20:02:51.276 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:50 smithi016 bash[75004]: cluster 2023-09-23T20:02:50.856998+0000 osd.6 (osd.6) 233 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T20:02:51.276 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:50 smithi016 bash[75004]: cluster 2023-09-23T20:02:50.861594+0000 osd.6 (osd.6) 234 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T20:02:51.276 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:50 smithi016 bash[75004]: cluster 2023-09-23T20:02:50.863565+0000 osd.6 (osd.6) 235 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T20:02:51.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:50 smithi092 bash[58855]: cluster 2023-09-23T20:02:49.852867+0000 osd.7 (osd.7) 210 : cluster 3 failed to encode map e298 with expected crc 2023-09-23T20:02:51.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:50 smithi092 bash[58855]: cluster 2023-09-23T20:02:49.853465+0000 osd.4 (osd.4) 235 : cluster 3 failed to encode map e298 with expected crc 2023-09-23T20:02:51.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:50 smithi092 bash[58855]: cluster 2023-09-23T20:02:49.858226+0000 osd.5 (osd.5) 214 : cluster 3 failed to encode map e298 with expected crc 2023-09-23T20:02:51.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:50 smithi092 bash[58855]: cluster 2023-09-23T20:02:50.855477+0000 mon.a (mon.0) 2017 : cluster 0 osdmap e299: 8 total, 8 up, 7 in 2023-09-23T20:02:51.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:50 smithi092 bash[58855]: cluster 2023-09-23T20:02:50.856998+0000 osd.6 (osd.6) 233 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T20:02:51.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:50 smithi092 bash[58855]: cluster 2023-09-23T20:02:50.861594+0000 osd.6 (osd.6) 234 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T20:02:51.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:50 smithi092 bash[58855]: cluster 2023-09-23T20:02:50.863565+0000 osd.6 (osd.6) 235 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T20:02:51.289 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:51.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:51.291 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:51.392 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:51.415 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:51.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:51.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:51.506 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T20:02:51.518 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:51.573 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:51.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:51.575 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:51.676 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:51.710 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:51.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:51.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:51.814 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:51.846 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:51.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:51.849 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:51.950 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:51.977 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:51.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:51.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:51.997 INFO:tasks.workunit.client.0.smithi016.stderr: Importing image: 100% complete...done. 2023-09-23T20:02:51.997 INFO:tasks.workunit.client.0.smithi016.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T20:02:51.998 INFO:tasks.workunit.client.0.smithi016.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T20:02:52.001 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd ls -l 2023-09-23T20:02:52.001 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep sparse1 2023-09-23T20:02:52.002 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep -Ei (10 MiB|10240k) 2023-09-23T20:02:52.050 INFO:tasks.workunit.client.0.smithi016.stdout:sparse1 10 MiB 2 2023-09-23T20:02:52.051 INFO:tasks.workunit.client.0.smithi016.stderr:+ objects sparse1 2023-09-23T20:02:52.051 INFO:tasks.workunit.client.0.smithi016.stderr:+ image=sparse1 2023-09-23T20:02:52.051 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd+ info sparse1 2023-09-23T20:02:52.051 INFO:tasks.workunit.client.0.smithi016.stderr:grep block_name_prefix 2023-09-23T20:02:52.051 INFO:tasks.workunit.client.0.smithi016.stderr:+ awk {print $NF;} 2023-09-23T20:02:52.080 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:52.099 INFO:tasks.workunit.client.0.smithi016.stderr:+ prefix=rbd_data.b9c94e2bb9f3 2023-09-23T20:02:52.100 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep rbd_data.b9c94e2bb9f3 2023-09-23T20:02:52.101 INFO:tasks.workunit.client.0.smithi016.stderr:+ get_image_data_pool sparse1 2023-09-23T20:02:52.101 INFO:tasks.workunit.client.0.smithi016.stderr:+ image=sparse1 2023-09-23T20:02:52.101 INFO:tasks.workunit.client.0.smithi016.stderr:+ sed -e s/rbd_data.b9c94e2bb9f3\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-09-23T20:02:52.101 INFO:tasks.workunit.client.0.smithi016.stderr:+ sort -u 2023-09-23T20:02:52.102 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd info sparse1 2023-09-23T20:02:52.102 INFO:tasks.workunit.client.0.smithi016.stderr:+ + awk -F: { print $NF } 2023-09-23T20:02:52.102 INFO:tasks.workunit.client.0.smithi016.stderr:grep data_pool: 2023-09-23T20:02:52.105 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:52.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:52.109 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:52.152 INFO:tasks.workunit.client.0.smithi016.stderr:+ data_pool= 2023-09-23T20:02:52.152 INFO:tasks.workunit.client.0.smithi016.stderr:+ [ -z ] 2023-09-23T20:02:52.153 INFO:tasks.workunit.client.0.smithi016.stderr:+ data_pool=rbd 2023-09-23T20:02:52.153 INFO:tasks.workunit.client.0.smithi016.stderr:+ echo rbd 2023-09-23T20:02:52.153 INFO:tasks.workunit.client.0.smithi016.stderr:+ rados ls -p rbd 2023-09-23T20:02:52.193 INFO:tasks.workunit.client.0.smithi016.stderr:+ objects=1 2023-09-23T20:02:52.194 INFO:tasks.workunit.client.0.smithi016.stderr:+ echo 1 2023-09-23T20:02:52.194 INFO:tasks.workunit.client.0.smithi016.stderr:+ [ 0 -eq 1 -o 1 = 1 ] 2023-09-23T20:02:52.194 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd export sparse1 /tmp/rbd_import_export_249364/sparse1.out 2023-09-23T20:02:52.210 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:52.218 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:51 smithi016 bash[77995]: cluster 2023-09-23T20:02:50.683187+0000 mgr.y (mgr.25325) 1670 : cluster 0 pgmap v1729: 73 pgs: 16 peering, 57 active+clean; 1.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 136 KiB/s rd, 183 KiB/s wr, 179 op/s 2023-09-23T20:02:52.218 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:51 smithi016 bash[77995]: cluster 2023-09-23T20:02:50.853903+0000 osd.4 (osd.4) 236 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T20:02:52.219 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:51 smithi016 bash[77995]: cluster 2023-09-23T20:02:50.858557+0000 osd.5 (osd.5) 215 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T20:02:52.219 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:51 smithi016 bash[77995]: cluster 2023-09-23T20:02:50.858990+0000 osd.7 (osd.7) 211 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T20:02:52.219 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:51 smithi016 bash[77995]: cluster 2023-09-23T20:02:50.860015+0000 osd.4 (osd.4) 237 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T20:02:52.219 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:51 smithi016 bash[77995]: cluster 2023-09-23T20:02:50.861832+0000 osd.5 (osd.5) 216 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T20:02:52.219 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:51 smithi016 bash[77995]: cluster 2023-09-23T20:02:50.862764+0000 osd.7 (osd.7) 212 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T20:02:52.219 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:51 smithi016 bash[77995]: cluster 2023-09-23T20:02:50.863022+0000 osd.4 (osd.4) 238 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T20:02:52.219 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:51 smithi016 bash[77995]: cluster 2023-09-23T20:02:50.865041+0000 osd.7 (osd.7) 213 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T20:02:52.219 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:51 smithi016 bash[77995]: cluster 2023-09-23T20:02:50.947588+0000 mon.a (mon.0) 2018 : cluster 3 Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-09-23T20:02:52.219 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:51 smithi016 bash[77995]: audit 2023-09-23T20:02:51.349947+0000 mon.a (mon.0) 2019 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:02:52.219 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:51 smithi016 bash[77995]: audit 2023-09-23T20:02:51.356184+0000 mon.a (mon.0) 2020 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:02:52.219 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:51 smithi016 bash[77995]: audit 2023-09-23T20:02:51.826623+0000 mon.a (mon.0) 2021 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:02:52.220 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:51 smithi016 bash[77995]: audit 2023-09-23T20:02:51.827842+0000 mon.a (mon.0) 2022 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:02:52.220 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:51 smithi016 bash[77995]: audit 2023-09-23T20:02:51.834839+0000 mon.a (mon.0) 2023 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:02:52.220 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:51 smithi016 bash[77995]: cluster 2023-09-23T20:02:51.852944+0000 mon.a (mon.0) 2024 : cluster 0 osdmap e300: 8 total, 8 up, 7 in 2023-09-23T20:02:52.220 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:51 smithi016 bash[75004]: cluster 2023-09-23T20:02:50.683187+0000 mgr.y (mgr.25325) 1670 : cluster 0 pgmap v1729: 73 pgs: 16 peering, 57 active+clean; 1.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 136 KiB/s rd, 183 KiB/s wr, 179 op/s 2023-09-23T20:02:52.220 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:51 smithi016 bash[75004]: cluster 2023-09-23T20:02:50.853903+0000 osd.4 (osd.4) 236 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T20:02:52.220 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:51 smithi016 bash[75004]: cluster 2023-09-23T20:02:50.858557+0000 osd.5 (osd.5) 215 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T20:02:52.220 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:51 smithi016 bash[75004]: cluster 2023-09-23T20:02:50.858990+0000 osd.7 (osd.7) 211 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T20:02:52.220 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:51 smithi016 bash[75004]: cluster 2023-09-23T20:02:50.860015+0000 osd.4 (osd.4) 237 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T20:02:52.220 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:51 smithi016 bash[75004]: cluster 2023-09-23T20:02:50.861832+0000 osd.5 (osd.5) 216 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T20:02:52.221 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:51 smithi016 bash[75004]: cluster 2023-09-23T20:02:50.862764+0000 osd.7 (osd.7) 212 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T20:02:52.221 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:51 smithi016 bash[75004]: cluster 2023-09-23T20:02:50.863022+0000 osd.4 (osd.4) 238 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T20:02:52.221 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:51 smithi016 bash[75004]: cluster 2023-09-23T20:02:50.865041+0000 osd.7 (osd.7) 213 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T20:02:52.221 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:51 smithi016 bash[75004]: cluster 2023-09-23T20:02:50.947588+0000 mon.a (mon.0) 2018 : cluster 3 Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-09-23T20:02:52.221 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:51 smithi016 bash[75004]: audit 2023-09-23T20:02:51.349947+0000 mon.a (mon.0) 2019 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:02:52.221 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:51 smithi016 bash[75004]: audit 2023-09-23T20:02:51.356184+0000 mon.a (mon.0) 2020 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:02:52.221 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:51 smithi016 bash[75004]: audit 2023-09-23T20:02:51.826623+0000 mon.a (mon.0) 2021 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:02:52.221 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:51 smithi016 bash[75004]: audit 2023-09-23T20:02:51.827842+0000 mon.a (mon.0) 2022 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:02:52.221 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:51 smithi016 bash[75004]: audit 2023-09-23T20:02:51.834839+0000 mon.a (mon.0) 2023 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:02:52.221 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:51 smithi016 bash[75004]: cluster 2023-09-23T20:02:51.852944+0000 mon.a (mon.0) 2024 : cluster 0 osdmap e300: 8 total, 8 up, 7 in 2023-09-23T20:02:52.240 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:52.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:52.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:52.245 INFO:tasks.workunit.client.0.smithi016.stderr: Exporting image: 10% complete... Exporting image: 20% complete... Exporting image: 30% complete... Exporting image: 40% complete... Exporting image: 50% complete... Exporting image: 60% complete... Exporting image: 70% complete... Exporting image: 80% complete... Exporting image: 90% complete... Exporting image: 100% complete...done. 2023-09-23T20:02:52.249 INFO:tasks.workunit.client.0.smithi016.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_249364/sparse1 /tmp/rbd_import_export_249364/sparse1.out 2023-09-23T20:02:52.249 INFO:tasks.workunit.client.0.smithi016.stderr:+ cmp -l /tmp/rbd_import_export_249364/sparse1 /tmp/rbd_import_export_249364/sparse1.out 2023-09-23T20:02:52.258 INFO:tasks.workunit.client.0.smithi016.stderr:+ stat /tmp/rbd_import_export_249364/sparse1 --format %b 2023-09-23T20:02:52.259 INFO:tasks.workunit.client.0.smithi016.stderr:+ origsize=2048 2023-09-23T20:02:52.259 INFO:tasks.workunit.client.0.smithi016.stderr:+ stat /tmp/rbd_import_export_249364/sparse1.out --format %b 2023-09-23T20:02:52.260 INFO:tasks.workunit.client.0.smithi016.stderr:+ exportsize=2048 2023-09-23T20:02:52.260 INFO:tasks.workunit.client.0.smithi016.stderr:+ difference=0 2023-09-23T20:02:52.260 INFO:tasks.workunit.client.0.smithi016.stderr:+ difference=0 2023-09-23T20:02:52.260 INFO:tasks.workunit.client.0.smithi016.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T20:02:52.260 INFO:tasks.workunit.client.0.smithi016.stderr:+ rm /tmp/rbd_import_export_249364/sparse1.out 2023-09-23T20:02:52.261 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd rm sparse1 2023-09-23T20:02:52.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:51 smithi092 bash[58855]: cluster 2023-09-23T20:02:50.683187+0000 mgr.y (mgr.25325) 1670 : cluster 0 pgmap v1729: 73 pgs: 16 peering, 57 active+clean; 1.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 136 KiB/s rd, 183 KiB/s wr, 179 op/s 2023-09-23T20:02:52.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:51 smithi092 bash[58855]: cluster 2023-09-23T20:02:50.853903+0000 osd.4 (osd.4) 236 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T20:02:52.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:51 smithi092 bash[58855]: cluster 2023-09-23T20:02:50.858557+0000 osd.5 (osd.5) 215 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T20:02:52.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:51 smithi092 bash[58855]: cluster 2023-09-23T20:02:50.858990+0000 osd.7 (osd.7) 211 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T20:02:52.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:51 smithi092 bash[58855]: cluster 2023-09-23T20:02:50.860015+0000 osd.4 (osd.4) 237 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T20:02:52.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:51 smithi092 bash[58855]: cluster 2023-09-23T20:02:50.861832+0000 osd.5 (osd.5) 216 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T20:02:52.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:51 smithi092 bash[58855]: cluster 2023-09-23T20:02:50.862764+0000 osd.7 (osd.7) 212 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T20:02:52.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:51 smithi092 bash[58855]: cluster 2023-09-23T20:02:50.863022+0000 osd.4 (osd.4) 238 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T20:02:52.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:51 smithi092 bash[58855]: cluster 2023-09-23T20:02:50.865041+0000 osd.7 (osd.7) 213 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T20:02:52.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:51 smithi092 bash[58855]: cluster 2023-09-23T20:02:50.947588+0000 mon.a (mon.0) 2018 : cluster 3 Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-09-23T20:02:52.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:51 smithi092 bash[58855]: audit 2023-09-23T20:02:51.349947+0000 mon.a (mon.0) 2019 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:02:52.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:51 smithi092 bash[58855]: audit 2023-09-23T20:02:51.356184+0000 mon.a (mon.0) 2020 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:02:52.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:51 smithi092 bash[58855]: audit 2023-09-23T20:02:51.826623+0000 mon.a (mon.0) 2021 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:02:52.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:51 smithi092 bash[58855]: audit 2023-09-23T20:02:51.827842+0000 mon.a (mon.0) 2022 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:02:52.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:51 smithi092 bash[58855]: audit 2023-09-23T20:02:51.834839+0000 mon.a (mon.0) 2023 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:02:52.290 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:51 smithi092 bash[58855]: cluster 2023-09-23T20:02:51.852944+0000 mon.a (mon.0) 2024 : cluster 0 osdmap e300: 8 total, 8 up, 7 in 2023-09-23T20:02:52.344 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:52.369 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:52.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:52.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:52.390 INFO:tasks.workunit.client.0.smithi016.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-09-23T20:02:52.395 INFO:tasks.workunit.client.0.smithi016.stderr:+ dd if=/dev/urandom bs=2M count=1 of=/tmp/rbd_import_export_249364/sparse2 oflag=append conv=notrunc 2023-09-23T20:02:52.407 INFO:tasks.workunit.client.0.smithi016.stderr:1+0 records in 2023-09-23T20:02:52.407 INFO:tasks.workunit.client.0.smithi016.stderr:1+0 records out 2023-09-23T20:02:52.407 INFO:tasks.workunit.client.0.smithi016.stderr:2097152 bytes (2.1 MB, 2.0 MiB) copied, 0.0111327 s, 188 MB/s 2023-09-23T20:02:52.408 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd import --new-format --order 20 - sparse2 2023-09-23T20:02:52.472 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:52.499 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:52.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:52.502 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:52.515 INFO:tasks.workunit.client.0.smithi016.stderr: Importing image: 100% complete...done. 2023-09-23T20:02:52.516 INFO:tasks.workunit.client.0.smithi016.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T20:02:52.516 INFO:tasks.workunit.client.0.smithi016.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T20:02:52.520 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd ls -l 2023-09-23T20:02:52.520 INFO:tasks.workunit.client.0.smithi016.stderr:+ + grep -Ei (4 MiB|4096k) 2023-09-23T20:02:52.520 INFO:tasks.workunit.client.0.smithi016.stderr:grep sparse2 2023-09-23T20:02:52.571 INFO:tasks.workunit.client.0.smithi016.stdout:sparse2 4 MiB 2 2023-09-23T20:02:52.572 INFO:tasks.workunit.client.0.smithi016.stderr:+ objects sparse2 2023-09-23T20:02:52.572 INFO:tasks.workunit.client.0.smithi016.stderr:+ image=sparse2 2023-09-23T20:02:52.573 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd info sparse2 2023-09-23T20:02:52.573 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep block_name_prefix 2023-09-23T20:02:52.573 INFO:tasks.workunit.client.0.smithi016.stderr:+ awk {print $NF;} 2023-09-23T20:02:52.602 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:52.622 INFO:tasks.workunit.client.0.smithi016.stderr:+ prefix=rbd_data.b9f66ffdd43b 2023-09-23T20:02:52.622 INFO:tasks.workunit.client.0.smithi016.stderr:+ get_image_data_pool sparse2 2023-09-23T20:02:52.622 INFO:tasks.workunit.client.0.smithi016.stderr:+ image=sparse2 2023-09-23T20:02:52.623 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep rbd_data.b9f66ffdd43b 2023-09-23T20:02:52.623 INFO:tasks.workunit.client.0.smithi016.stderr:+ sed -e s/rbd_data.b9f66ffdd43b\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-09-23T20:02:52.623 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd info sparse2 2023-09-23T20:02:52.623 INFO:tasks.workunit.client.0.smithi016.stderr:+ + sort -u 2023-09-23T20:02:52.623 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep data_pool: 2023-09-23T20:02:52.623 INFO:tasks.workunit.client.0.smithi016.stderr:awk -F: { print $NF } 2023-09-23T20:02:52.635 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:52.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:52.639 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:52.666 INFO:tasks.workunit.client.0.smithi016.stderr:+ data_pool= 2023-09-23T20:02:52.666 INFO:tasks.workunit.client.0.smithi016.stderr:+ [ -z ] 2023-09-23T20:02:52.666 INFO:tasks.workunit.client.0.smithi016.stderr:+ data_pool=rbd 2023-09-23T20:02:52.667 INFO:tasks.workunit.client.0.smithi016.stderr:+ echo rbd 2023-09-23T20:02:52.667 INFO:tasks.workunit.client.0.smithi016.stderr:+ rados ls -p rbd 2023-09-23T20:02:52.706 INFO:tasks.workunit.client.0.smithi016.stderr:+ objects=0 2023-09-23T20:02:52.706 INFO:tasks.workunit.client.0.smithi016.stderr:2 2023-09-23T20:02:52.706 INFO:tasks.workunit.client.0.smithi016.stderr:3 2023-09-23T20:02:52.706 INFO:tasks.workunit.client.0.smithi016.stderr:+ echo 0 2 3 2023-09-23T20:02:52.707 INFO:tasks.workunit.client.0.smithi016.stderr:+ [ 0 -eq 1 -o 0 2 3 = 0 2 3 ] 2023-09-23T20:02:52.707 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd export sparse2 /tmp/rbd_import_export_249364/sparse2.out 2023-09-23T20:02:52.740 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:52.755 INFO:tasks.workunit.client.0.smithi016.stderr: Exporting image: 25% complete... Exporting image: 50% complete... Exporting image: 75% complete... Exporting image: 100% complete...done. 2023-09-23T20:02:52.760 INFO:tasks.workunit.client.0.smithi016.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_249364/sparse2 /tmp/rbd_import_export_249364/sparse2.out 2023-09-23T20:02:52.760 INFO:tasks.workunit.client.0.smithi016.stderr:+ cmp -l /tmp/rbd_import_export_249364/sparse2 /tmp/rbd_import_export_249364/sparse2.out 2023-09-23T20:02:52.763 INFO:tasks.workunit.client.0.smithi016.stderr:+ stat /tmp/rbd_import_export_249364/sparse2 --format %b 2023-09-23T20:02:52.765 INFO:tasks.workunit.client.0.smithi016.stderr:+ origsize=6144 2023-09-23T20:02:52.765 INFO:tasks.workunit.client.0.smithi016.stderr:+ stat /tmp/rbd_import_export_249364/sparse2.out --format %b 2023-09-23T20:02:52.766 INFO:tasks.workunit.client.0.smithi016.stderr:+ exportsize=6144 2023-09-23T20:02:52.766 INFO:tasks.workunit.client.0.smithi016.stderr:+ difference=0 2023-09-23T20:02:52.766 INFO:tasks.workunit.client.0.smithi016.stderr:+ difference=0 2023-09-23T20:02:52.766 INFO:tasks.workunit.client.0.smithi016.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T20:02:52.766 INFO:tasks.workunit.client.0.smithi016.stderr:+ rm /tmp/rbd_import_export_249364/sparse2.out 2023-09-23T20:02:52.767 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd rm sparse2 2023-09-23T20:02:52.773 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:52.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:52.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:52.876 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:52.905 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:52.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:52.908 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:52.912 INFO:tasks.workunit.client.0.smithi016.stderr: Removing image: 25% complete... Removing image: 50% complete... Removing image: 75% complete... Removing image: 100% complete...done. 2023-09-23T20:02:52.916 INFO:tasks.workunit.client.0.smithi016.stdout:partially-sparse file imports to partially-sparse image 2023-09-23T20:02:52.916 INFO:tasks.workunit.client.0.smithi016.stderr:+ echo partially-sparse file imports to partially-sparse image 2023-09-23T20:02:52.916 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_249364/sparse1 sparse 2023-09-23T20:02:53.009 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:53.041 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:53.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:53.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:53.145 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:53.157 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:52 smithi092 bash[58855]: cluster 2023-09-23T20:02:51.855267+0000 osd.5 (osd.5) 217 : cluster 3 failed to encode map e300 with expected crc 2023-09-23T20:02:53.157 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:52 smithi092 bash[58855]: cluster 2023-09-23T20:02:51.857599+0000 osd.7 (osd.7) 214 : cluster 3 failed to encode map e300 with expected crc 2023-09-23T20:02:53.157 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:52 smithi092 bash[58855]: cluster 2023-09-23T20:02:51.860319+0000 osd.7 (osd.7) 215 : cluster 3 failed to encode map e300 with expected crc 2023-09-23T20:02:53.174 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:53.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:53.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:53.277 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:53.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:52 smithi016 bash[75004]: cluster 2023-09-23T20:02:51.855267+0000 osd.5 (osd.5) 217 : cluster 3 failed to encode map e300 with expected crc 2023-09-23T20:02:53.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:52 smithi016 bash[75004]: cluster 2023-09-23T20:02:51.857599+0000 osd.7 (osd.7) 214 : cluster 3 failed to encode map e300 with expected crc 2023-09-23T20:02:53.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:52 smithi016 bash[75004]: cluster 2023-09-23T20:02:51.860319+0000 osd.7 (osd.7) 215 : cluster 3 failed to encode map e300 with expected crc 2023-09-23T20:02:53.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:52 smithi016 bash[77995]: cluster 2023-09-23T20:02:51.855267+0000 osd.5 (osd.5) 217 : cluster 3 failed to encode map e300 with expected crc 2023-09-23T20:02:53.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:52 smithi016 bash[77995]: cluster 2023-09-23T20:02:51.857599+0000 osd.7 (osd.7) 214 : cluster 3 failed to encode map e300 with expected crc 2023-09-23T20:02:53.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:52 smithi016 bash[77995]: cluster 2023-09-23T20:02:51.860319+0000 osd.7 (osd.7) 215 : cluster 3 failed to encode map e300 with expected crc 2023-09-23T20:02:53.307 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:53.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:53.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:53.411 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:53.442 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:53.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:53.445 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:53.545 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:53.579 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:53.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:53.582 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:53.683 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:53.714 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:53.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:53.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:53.819 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:53.848 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:53.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:53.852 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:53.952 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:53.986 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:53.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:53.988 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:54.041 INFO:tasks.workunit.client.0.smithi016.stderr: Importing image: 10% complete... Importing image: 20% complete... Importing image: 30% complete... Importing image: 40% complete... Importing image: 50% complete... Importing image: 60% complete... Importing image: 70% complete... Importing image: 80% complete... Importing image: 90% complete... Importing image: 100% complete...done. 2023-09-23T20:02:54.042 INFO:tasks.workunit.client.0.smithi016.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T20:02:54.042 INFO:tasks.workunit.client.0.smithi016.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T20:02:54.046 INFO:tasks.workunit.client.0.smithi016.stderr:+ objects sparse 2023-09-23T20:02:54.046 INFO:tasks.workunit.client.0.smithi016.stderr:+ image=sparse 2023-09-23T20:02:54.047 INFO:tasks.workunit.client.0.smithi016.stderr:+ + rbd info sparse 2023-09-23T20:02:54.047 INFO:tasks.workunit.client.0.smithi016.stderr:grep block_name_prefix 2023-09-23T20:02:54.047 INFO:tasks.workunit.client.0.smithi016.stderr:+ awk {print $NF;} 2023-09-23T20:02:54.089 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:54.100 INFO:tasks.workunit.client.0.smithi016.stderr:+ prefix=rbd_data.ba23ebdf1690 2023-09-23T20:02:54.101 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep rbd_data.ba23ebdf1690 2023-09-23T20:02:54.101 INFO:tasks.workunit.client.0.smithi016.stderr:+ get_image_data_pool sparse 2023-09-23T20:02:54.101 INFO:tasks.workunit.client.0.smithi016.stderr:+ image=sparse 2023-09-23T20:02:54.101 INFO:tasks.workunit.client.0.smithi016.stderr:+ sort -u 2023-09-23T20:02:54.101 INFO:tasks.workunit.client.0.smithi016.stderr:+ sed -e s/rbd_data.ba23ebdf1690\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-09-23T20:02:54.101 INFO:tasks.workunit.client.0.smithi016.stderr:+ + rbd info sparse 2023-09-23T20:02:54.102 INFO:tasks.workunit.client.0.smithi016.stderr:+ awk -F: { print $NF } 2023-09-23T20:02:54.102 INFO:tasks.workunit.client.0.smithi016.stderr:grep data_pool: 2023-09-23T20:02:54.113 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:54.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:54.115 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:54.159 INFO:tasks.workunit.client.0.smithi016.stderr:+ data_pool= 2023-09-23T20:02:54.159 INFO:tasks.workunit.client.0.smithi016.stderr:+ [ -z ] 2023-09-23T20:02:54.159 INFO:tasks.workunit.client.0.smithi016.stderr:+ data_pool=rbd 2023-09-23T20:02:54.159 INFO:tasks.workunit.client.0.smithi016.stderr:+ echo rbd 2023-09-23T20:02:54.159 INFO:tasks.workunit.client.0.smithi016.stderr:+ rados ls -p rbd 2023-09-23T20:02:54.205 INFO:tasks.workunit.client.0.smithi016.stderr:+ objects=1 2023-09-23T20:02:54.205 INFO:tasks.workunit.client.0.smithi016.stderr:+ echo 1 2023-09-23T20:02:54.205 INFO:tasks.workunit.client.0.smithi016.stderr:+ [ 0 -eq 1 -o 1 = 1 ] 2023-09-23T20:02:54.205 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd rm sparse 2023-09-23T20:02:54.216 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:54.230 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:53 smithi092 bash[58855]: cluster 2023-09-23T20:02:52.683626+0000 mgr.y (mgr.25325) 1671 : cluster 0 pgmap v1732: 73 pgs: 16 peering, 57 active+clean; 1.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 133 KiB/s rd, 185 KiB/s wr, 173 op/s 2023-09-23T20:02:54.230 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:53 smithi092 bash[58855]: cluster 2023-09-23T20:02:52.968296+0000 mon.a (mon.0) 2025 : cluster 0 osdmap e301: 8 total, 8 up, 7 in 2023-09-23T20:02:54.230 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:53 smithi092 bash[58855]: cluster 2023-09-23T20:02:52.971372+0000 osd.4 (osd.4) 239 : cluster 3 failed to encode map e301 with expected crc 2023-09-23T20:02:54.230 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:53 smithi092 bash[58855]: cluster 2023-09-23T20:02:52.971985+0000 osd.7 (osd.7) 216 : cluster 3 failed to encode map e301 with expected crc 2023-09-23T20:02:54.230 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:53 smithi092 bash[58855]: cluster 2023-09-23T20:02:52.973567+0000 osd.7 (osd.7) 217 : cluster 3 failed to encode map e301 with expected crc 2023-09-23T20:02:54.230 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:53 smithi092 bash[58855]: cluster 2023-09-23T20:02:53.316689+0000 osd.5 (osd.5) 218 : cluster 3 failed to encode map e301 with expected crc 2023-09-23T20:02:54.230 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:53 smithi092 bash[58855]: cluster 2023-09-23T20:02:53.316880+0000 osd.6 (osd.6) 236 : cluster 3 failed to encode map e301 with expected crc 2023-09-23T20:02:54.249 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:54.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:54.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:54.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:53 smithi016 bash[77995]: cluster 2023-09-23T20:02:52.683626+0000 mgr.y (mgr.25325) 1671 : cluster 0 pgmap v1732: 73 pgs: 16 peering, 57 active+clean; 1.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 133 KiB/s rd, 185 KiB/s wr, 173 op/s 2023-09-23T20:02:54.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:53 smithi016 bash[77995]: cluster 2023-09-23T20:02:52.968296+0000 mon.a (mon.0) 2025 : cluster 0 osdmap e301: 8 total, 8 up, 7 in 2023-09-23T20:02:54.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:53 smithi016 bash[77995]: cluster 2023-09-23T20:02:52.971372+0000 osd.4 (osd.4) 239 : cluster 3 failed to encode map e301 with expected crc 2023-09-23T20:02:54.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:53 smithi016 bash[77995]: cluster 2023-09-23T20:02:52.971985+0000 osd.7 (osd.7) 216 : cluster 3 failed to encode map e301 with expected crc 2023-09-23T20:02:54.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:53 smithi016 bash[77995]: cluster 2023-09-23T20:02:52.973567+0000 osd.7 (osd.7) 217 : cluster 3 failed to encode map e301 with expected crc 2023-09-23T20:02:54.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:53 smithi016 bash[77995]: cluster 2023-09-23T20:02:53.316689+0000 osd.5 (osd.5) 218 : cluster 3 failed to encode map e301 with expected crc 2023-09-23T20:02:54.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:53 smithi016 bash[77995]: cluster 2023-09-23T20:02:53.316880+0000 osd.6 (osd.6) 236 : cluster 3 failed to encode map e301 with expected crc 2023-09-23T20:02:54.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:53 smithi016 bash[75004]: cluster 2023-09-23T20:02:52.683626+0000 mgr.y (mgr.25325) 1671 : cluster 0 pgmap v1732: 73 pgs: 16 peering, 57 active+clean; 1.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 133 KiB/s rd, 185 KiB/s wr, 173 op/s 2023-09-23T20:02:54.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:53 smithi016 bash[75004]: cluster 2023-09-23T20:02:52.968296+0000 mon.a (mon.0) 2025 : cluster 0 osdmap e301: 8 total, 8 up, 7 in 2023-09-23T20:02:54.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:53 smithi016 bash[75004]: cluster 2023-09-23T20:02:52.971372+0000 osd.4 (osd.4) 239 : cluster 3 failed to encode map e301 with expected crc 2023-09-23T20:02:54.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:53 smithi016 bash[75004]: cluster 2023-09-23T20:02:52.971985+0000 osd.7 (osd.7) 216 : cluster 3 failed to encode map e301 with expected crc 2023-09-23T20:02:54.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:53 smithi016 bash[75004]: cluster 2023-09-23T20:02:52.973567+0000 osd.7 (osd.7) 217 : cluster 3 failed to encode map e301 with expected crc 2023-09-23T20:02:54.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:53 smithi016 bash[75004]: cluster 2023-09-23T20:02:53.316689+0000 osd.5 (osd.5) 218 : cluster 3 failed to encode map e301 with expected crc 2023-09-23T20:02:54.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:53 smithi016 bash[75004]: cluster 2023-09-23T20:02:53.316880+0000 osd.6 (osd.6) 236 : cluster 3 failed to encode map e301 with expected crc 2023-09-23T20:02:54.353 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:54.360 INFO:tasks.workunit.client.0.smithi016.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-09-23T20:02:54.366 INFO:tasks.workunit.client.0.smithi016.stdout:zeros import through stdin to sparse image 2023-09-23T20:02:54.367 INFO:tasks.workunit.client.0.smithi016.stderr:+ echo zeros import through stdin to sparse image 2023-09-23T20:02:54.367 INFO:tasks.workunit.client.0.smithi016.stderr:+ dd if=/dev/zero bs=1M count=4 2023-09-23T20:02:54.367 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd import --new-format - sparse 2023-09-23T20:02:54.382 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:54.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:54.385 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:54.443 INFO:tasks.workunit.client.0.smithi016.stderr:4+0 records in 2023-09-23T20:02:54.443 INFO:tasks.workunit.client.0.smithi016.stderr:4+0 records out 2023-09-23T20:02:54.443 INFO:tasks.workunit.client.0.smithi016.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0755839 s, 55.5 MB/s 2023-09-23T20:02:54.464 INFO:tasks.workunit.client.0.smithi016.stderr: Importing image: 100% complete...done. 2023-09-23T20:02:54.466 INFO:tasks.workunit.client.0.smithi016.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T20:02:54.470 INFO:tasks.workunit.client.0.smithi016.stderr:+ objects sparse 2023-09-23T20:02:54.471 INFO:tasks.workunit.client.0.smithi016.stderr:+ image=sparse 2023-09-23T20:02:54.471 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd info sparse 2023-09-23T20:02:54.471 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep block_name_prefix 2023-09-23T20:02:54.471 INFO:tasks.workunit.client.0.smithi016.stderr:+ awk {print $NF;} 2023-09-23T20:02:54.486 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:54.516 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:54.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:54.519 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:54.532 INFO:tasks.workunit.client.0.smithi016.stderr:+ prefix=rbd_data.ba442734ec7f 2023-09-23T20:02:54.532 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep rbd_data.ba442734ec7f 2023-09-23T20:02:54.533 INFO:tasks.workunit.client.0.smithi016.stderr:+ get_image_data_pool sparse 2023-09-23T20:02:54.533 INFO:tasks.workunit.client.0.smithi016.stderr:+ image=sparse 2023-09-23T20:02:54.533 INFO:tasks.workunit.client.0.smithi016.stderr:+ + sed -e s/rbd_data.ba442734ec7f\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-09-23T20:02:54.533 INFO:tasks.workunit.client.0.smithi016.stderr:sort -u 2023-09-23T20:02:54.534 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd info sparse 2023-09-23T20:02:54.534 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep data_pool: 2023-09-23T20:02:54.534 INFO:tasks.workunit.client.0.smithi016.stderr:+ awk -F: { print $NF } 2023-09-23T20:02:54.594 INFO:tasks.workunit.client.0.smithi016.stderr:+ data_pool= 2023-09-23T20:02:54.595 INFO:tasks.workunit.client.0.smithi016.stderr:+ [ -z ] 2023-09-23T20:02:54.595 INFO:tasks.workunit.client.0.smithi016.stderr:+ data_pool=rbd 2023-09-23T20:02:54.595 INFO:tasks.workunit.client.0.smithi016.stderr:+ echo rbd 2023-09-23T20:02:54.595 INFO:tasks.workunit.client.0.smithi016.stderr:+ rados ls -p rbd 2023-09-23T20:02:54.619 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:54.644 INFO:tasks.workunit.client.0.smithi016.stderr:+ objects= 2023-09-23T20:02:54.644 INFO:tasks.workunit.client.0.smithi016.stderr:+ echo 2023-09-23T20:02:54.644 INFO:tasks.workunit.client.0.smithi016.stderr:+ [ 0 -eq 1 -o = ] 2023-09-23T20:02:54.644 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd rm sparse 2023-09-23T20:02:54.651 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:54.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:54.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:54.755 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:54.784 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:54.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:54.788 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:54.800 INFO:tasks.workunit.client.0.smithi016.stderr: Removing image: 100% complete...done. 2023-09-23T20:02:54.806 INFO:tasks.workunit.client.0.smithi016.stdout:zeros export to sparse file 2023-09-23T20:02:54.806 INFO:tasks.workunit.client.0.smithi016.stderr:+ echo zeros export to sparse file 2023-09-23T20:02:54.806 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd create --new-format sparse --size 4 2023-09-23T20:02:54.864 INFO:tasks.workunit.client.0.smithi016.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T20:02:54.869 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd info sparse 2023-09-23T20:02:54.869 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep block_name_prefix 2023-09-23T20:02:54.869 INFO:tasks.workunit.client.0.smithi016.stderr:+ awk {print $NF;} 2023-09-23T20:02:54.889 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:54.923 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:54.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:54.925 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:54.934 INFO:tasks.workunit.client.0.smithi016.stderr:+ prefix=rbd_data.ba651b2cc40b 2023-09-23T20:02:54.934 INFO:tasks.workunit.client.0.smithi016.stderr:+ dd if=/dev/zero bs=4M count=1 2023-09-23T20:02:54.935 INFO:tasks.workunit.client.0.smithi016.stderr:+ get_image_data_pool sparse 2023-09-23T20:02:54.935 INFO:tasks.workunit.client.0.smithi016.stderr:+ image=sparse 2023-09-23T20:02:54.935 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd info sparse 2023-09-23T20:02:54.935 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep data_pool: 2023-09-23T20:02:54.935 INFO:tasks.workunit.client.0.smithi016.stderr:+ awk -F: { print $NF } 2023-09-23T20:02:54.996 INFO:tasks.workunit.client.0.smithi016.stderr:+ data_pool= 2023-09-23T20:02:54.996 INFO:tasks.workunit.client.0.smithi016.stderr:+ [ -z ] 2023-09-23T20:02:54.996 INFO:tasks.workunit.client.0.smithi016.stderr:+ data_pool=rbd 2023-09-23T20:02:54.996 INFO:tasks.workunit.client.0.smithi016.stderr:+ echo rbd 2023-09-23T20:02:54.996 INFO:tasks.workunit.client.0.smithi016.stderr:+ rados -p rbd put rbd_data.ba651b2cc40b.000000000000 - 2023-09-23T20:02:55.026 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:55.037 INFO:tasks.workunit.client.0.smithi016.stderr:1+0 records in 2023-09-23T20:02:55.037 INFO:tasks.workunit.client.0.smithi016.stderr:1+0 records out 2023-09-23T20:02:55.038 INFO:tasks.workunit.client.0.smithi016.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.10277 s, 40.8 MB/s 2023-09-23T20:02:55.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:54 smithi092 bash[58855]: cluster 2023-09-23T20:02:53.978075+0000 mon.a (mon.0) 2026 : cluster 0 osdmap e302: 8 total, 8 up, 7 in 2023-09-23T20:02:55.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:54 smithi092 bash[58855]: cluster 2023-09-23T20:02:53.980007+0000 osd.6 (osd.6) 237 : cluster 3 failed to encode map e302 with expected crc 2023-09-23T20:02:55.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:54 smithi092 bash[58855]: cluster 2023-09-23T20:02:53.982297+0000 osd.7 (osd.7) 218 : cluster 3 failed to encode map e302 with expected crc 2023-09-23T20:02:55.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:54 smithi092 bash[58855]: cluster 2023-09-23T20:02:53.984776+0000 osd.7 (osd.7) 219 : cluster 3 failed to encode map e302 with expected crc 2023-09-23T20:02:55.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:54 smithi092 bash[58855]: cluster 2023-09-23T20:02:54.027826+0000 osd.5 (osd.5) 219 : cluster 3 failed to encode map e302 with expected crc 2023-09-23T20:02:55.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:54 smithi092 bash[58855]: cluster 2023-09-23T20:02:54.846764+0000 mon.a (mon.0) 2027 : cluster 0 osdmap e303: 8 total, 8 up, 7 in 2023-09-23T20:02:55.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:54 smithi092 bash[58855]: cluster 2023-09-23T20:02:54.862392+0000 osd.6 (osd.6) 238 : cluster 3 failed to encode map e303 with expected crc 2023-09-23T20:02:55.054 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:55.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:55.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:55.067 INFO:tasks.workunit.client.0.smithi016.stderr:+ objects sparse 2023-09-23T20:02:55.067 INFO:tasks.workunit.client.0.smithi016.stderr:+ image=sparse 2023-09-23T20:02:55.067 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd info sparse 2023-09-23T20:02:55.067 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep block_name_prefix 2023-09-23T20:02:55.067 INFO:tasks.workunit.client.0.smithi016.stderr:+ awk {print $NF;} 2023-09-23T20:02:55.129 INFO:tasks.workunit.client.0.smithi016.stderr:+ prefix=rbd_data.ba651b2cc40b 2023-09-23T20:02:55.131 INFO:tasks.workunit.client.0.smithi016.stderr:+ grep rbd_data.ba651b2cc40b 2023-09-23T20:02:55.131 INFO:tasks.workunit.client.0.smithi016.stderr:+ get_image_data_pool sparse 2023-09-23T20:02:55.131 INFO:tasks.workunit.client.0.smithi016.stderr:+ image=sparse 2023-09-23T20:02:55.131 INFO:tasks.workunit.client.0.smithi016.stderr:+ sed -e s/rbd_data.ba651b2cc40b\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-09-23T20:02:55.131 INFO:tasks.workunit.client.0.smithi016.stderr:+ sort -u 2023-09-23T20:02:55.131 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd info sparse 2023-09-23T20:02:55.131 INFO:tasks.workunit.client.0.smithi016.stderr:+ + grep data_pool: 2023-09-23T20:02:55.131 INFO:tasks.workunit.client.0.smithi016.stderr:awk -F: { print $NF } 2023-09-23T20:02:55.158 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:55.183 INFO:tasks.workunit.client.0.smithi016.stderr:+ data_pool= 2023-09-23T20:02:55.184 INFO:tasks.workunit.client.0.smithi016.stderr:+ [ -z ] 2023-09-23T20:02:55.184 INFO:tasks.workunit.client.0.smithi016.stderr:+ data_pool=rbd 2023-09-23T20:02:55.184 INFO:tasks.workunit.client.0.smithi016.stderr:+ echo rbd 2023-09-23T20:02:55.184 INFO:tasks.workunit.client.0.smithi016.stderr:+ rados ls -p rbd 2023-09-23T20:02:55.188 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:55.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:55.191 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:55.237 INFO:tasks.workunit.client.0.smithi016.stderr:+ objects=0 2023-09-23T20:02:55.238 INFO:tasks.workunit.client.0.smithi016.stderr:+ echo 0 2023-09-23T20:02:55.238 INFO:tasks.workunit.client.0.smithi016.stderr:+ [ 0 -eq 1 -o 0 = 0 ] 2023-09-23T20:02:55.238 INFO:tasks.workunit.client.0.smithi016.stderr:+ rm /tmp/rbd_import_export_249364/sparse 2023-09-23T20:02:55.238 INFO:tasks.workunit.client.0.smithi016.stderr:rm: cannot remove '/tmp/rbd_import_export_249364/sparse': No such file or directory 2023-09-23T20:02:55.239 INFO:tasks.workunit.client.0.smithi016.stderr:+ true 2023-09-23T20:02:55.239 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd export sparse /tmp/rbd_import_export_249364/sparse 2023-09-23T20:02:55.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:54 smithi016 bash[75004]: cluster 2023-09-23T20:02:53.978075+0000 mon.a (mon.0) 2026 : cluster 0 osdmap e302: 8 total, 8 up, 7 in 2023-09-23T20:02:55.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:54 smithi016 bash[75004]: cluster 2023-09-23T20:02:53.980007+0000 osd.6 (osd.6) 237 : cluster 3 failed to encode map e302 with expected crc 2023-09-23T20:02:55.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:54 smithi016 bash[75004]: cluster 2023-09-23T20:02:53.982297+0000 osd.7 (osd.7) 218 : cluster 3 failed to encode map e302 with expected crc 2023-09-23T20:02:55.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:54 smithi016 bash[75004]: cluster 2023-09-23T20:02:53.984776+0000 osd.7 (osd.7) 219 : cluster 3 failed to encode map e302 with expected crc 2023-09-23T20:02:55.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:54 smithi016 bash[75004]: cluster 2023-09-23T20:02:54.027826+0000 osd.5 (osd.5) 219 : cluster 3 failed to encode map e302 with expected crc 2023-09-23T20:02:55.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:54 smithi016 bash[75004]: cluster 2023-09-23T20:02:54.846764+0000 mon.a (mon.0) 2027 : cluster 0 osdmap e303: 8 total, 8 up, 7 in 2023-09-23T20:02:55.287 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:54 smithi016 bash[75004]: cluster 2023-09-23T20:02:54.862392+0000 osd.6 (osd.6) 238 : cluster 3 failed to encode map e303 with expected crc 2023-09-23T20:02:55.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:54 smithi016 bash[77995]: cluster 2023-09-23T20:02:53.978075+0000 mon.a (mon.0) 2026 : cluster 0 osdmap e302: 8 total, 8 up, 7 in 2023-09-23T20:02:55.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:54 smithi016 bash[77995]: cluster 2023-09-23T20:02:53.980007+0000 osd.6 (osd.6) 237 : cluster 3 failed to encode map e302 with expected crc 2023-09-23T20:02:55.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:54 smithi016 bash[77995]: cluster 2023-09-23T20:02:53.982297+0000 osd.7 (osd.7) 218 : cluster 3 failed to encode map e302 with expected crc 2023-09-23T20:02:55.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:54 smithi016 bash[77995]: cluster 2023-09-23T20:02:53.984776+0000 osd.7 (osd.7) 219 : cluster 3 failed to encode map e302 with expected crc 2023-09-23T20:02:55.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:54 smithi016 bash[77995]: cluster 2023-09-23T20:02:54.027826+0000 osd.5 (osd.5) 219 : cluster 3 failed to encode map e302 with expected crc 2023-09-23T20:02:55.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:54 smithi016 bash[77995]: cluster 2023-09-23T20:02:54.846764+0000 mon.a (mon.0) 2027 : cluster 0 osdmap e303: 8 total, 8 up, 7 in 2023-09-23T20:02:55.288 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:54 smithi016 bash[77995]: cluster 2023-09-23T20:02:54.862392+0000 osd.6 (osd.6) 238 : cluster 3 failed to encode map e303 with expected crc 2023-09-23T20:02:55.291 INFO:tasks.workunit.client.0.smithi016.stderr: Exporting image: 100% complete...done. 2023-09-23T20:02:55.292 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:55.297 INFO:tasks.workunit.client.0.smithi016.stderr:+ stat /tmp/rbd_import_export_249364/sparse --format=%b 2023-09-23T20:02:55.298 INFO:tasks.workunit.client.0.smithi016.stderr:+ [ 0 = 0 ] 2023-09-23T20:02:55.298 INFO:tasks.workunit.client.0.smithi016.stderr:+ rbd rm sparse 2023-09-23T20:02:55.318 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:55.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:55.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:55.422 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:55.453 INFO:tasks.workunit.client.0.smithi016.stderr: Removing image: 100% complete...done. 2023-09-23T20:02:55.454 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:55.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:55.457 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:55.459 INFO:tasks.workunit.client.0.smithi016.stderr:+ rm /tmp/rbd_import_export_249364/sparse /tmp/rbd_import_export_249364/sparse1 /tmp/rbd_import_export_249364/sparse2 /tmp/rbd_import_export_249364/sparse3 2023-09-23T20:02:55.463 INFO:tasks.workunit.client.0.smithi016.stderr:rm: cannot remove '/tmp/rbd_import_export_249364/sparse3': No such file or directory 2023-09-23T20:02:55.463 INFO:tasks.workunit.client.0.smithi016.stderr:+ true 2023-09-23T20:02:55.463 INFO:tasks.workunit.client.0.smithi016.stdout:OK 2023-09-23T20:02:55.464 INFO:tasks.workunit.client.0.smithi016.stderr:+ echo OK 2023-09-23T20:02:55.464 INFO:tasks.workunit.client.0.smithi016.stderr:+ rm -rf /tmp/rbd_import_export_249364 2023-09-23T20:02:55.464 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-23T20:02:55.465 DEBUG:teuthology.orchestra.run.smithi016:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-23T20:02:55.520 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:02:55.522 INFO:tasks.workunit:Stopping ['rbd/import_export.sh'] on client.0... 2023-09-23T20:02:55.522 DEBUG:teuthology.orchestra.run.smithi016:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-09-23T20:02:55.539 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:02:55.559 DEBUG:teuthology.orchestra.run.smithi016:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:55.598 INFO:teuthology.orchestra.run.smithi016.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:55.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:55.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi016 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:55.701 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:55.736 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:55.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:55.739 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:55.840 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:55.874 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:55.875 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:02:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:20:02:55] "GET /metrics HTTP/1.1" 200 34002 "" "Prometheus/2.33.4" 2023-09-23T20:02:55.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:55.877 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:55.877 DEBUG:teuthology.parallel:result is None 2023-09-23T20:02:55.878 DEBUG:teuthology.orchestra.run.smithi016:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T20:02:55.891 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T20:02:55.891 DEBUG:teuthology.orchestra.run.smithi016:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T20:02:55.940 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T20:02:55.940 INFO:teuthology.task.sequential:In sequential, running task print... 2023-09-23T20:02:55.941 INFO:teuthology.task.print:**** done rbd/import_export.sh 5-workload 2023-09-23T20:02:55.941 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-09-23T20:02:55.941 INFO:tasks.workunit:Pulling workunits from ref pacific 2023-09-23T20:02:55.941 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-09-23T20:02:55.941 DEBUG:teuthology.orchestra.run.smithi016:> stat -- /home/ubuntu/cephtest/mnt.0 2023-09-23T20:02:55.978 DEBUG:teuthology.orchestra.run.smithi092:> sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:55.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:55.989 INFO:teuthology.orchestra.run.smithi016.stderr:stat: cannot stat '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-09-23T20:02:55.990 DEBUG:teuthology.orchestra.run.smithi016:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T20:02:56.009 INFO:teuthology.orchestra.run.smithi092.stderr:Error response from daemon: Cannot kill container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:56.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:02:56.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi092 with status 1: 'sudo docker kill -s 1 ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:56.037 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-09-23T20:02:56.037 DEBUG:teuthology.orchestra.run.smithi016:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-09-23T20:02:56.090 INFO:tasks.thrashosds:joining thrashosds 2023-09-23T20:02:56.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:55 smithi016 bash[75004]: cluster 2023-09-23T20:02:54.684589+0000 mgr.y (mgr.25325) 1672 : cluster 0 pgmap v1735: 73 pgs: 16 peering, 57 active+clean; 582 KiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 946 KiB/s rd, 787 KiB/s wr, 261 op/s; 0 B/s, 0 keys/s, 4 objects/s recovering 2023-09-23T20:02:56.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:55 smithi016 bash[75004]: cluster 2023-09-23T20:02:54.908939+0000 osd.7 (osd.7) 220 : cluster 3 failed to encode map e303 with expected crc 2023-09-23T20:02:56.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:55 smithi016 bash[75004]: cluster 2023-09-23T20:02:55.019770+0000 osd.4 (osd.4) 240 : cluster 3 failed to encode map e303 with expected crc 2023-09-23T20:02:56.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:55 smithi016 bash[75004]: cluster 2023-09-23T20:02:55.229351+0000 osd.5 (osd.5) 220 : cluster 3 failed to encode map e303 with expected crc 2023-09-23T20:02:56.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:55 smithi016 bash[77995]: cluster 2023-09-23T20:02:54.684589+0000 mgr.y (mgr.25325) 1672 : cluster 0 pgmap v1735: 73 pgs: 16 peering, 57 active+clean; 582 KiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 946 KiB/s rd, 787 KiB/s wr, 261 op/s; 0 B/s, 0 keys/s, 4 objects/s recovering 2023-09-23T20:02:56.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:55 smithi016 bash[77995]: cluster 2023-09-23T20:02:54.908939+0000 osd.7 (osd.7) 220 : cluster 3 failed to encode map e303 with expected crc 2023-09-23T20:02:56.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:55 smithi016 bash[77995]: cluster 2023-09-23T20:02:55.019770+0000 osd.4 (osd.4) 240 : cluster 3 failed to encode map e303 with expected crc 2023-09-23T20:02:56.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:55 smithi016 bash[77995]: cluster 2023-09-23T20:02:55.229351+0000 osd.5 (osd.5) 220 : cluster 3 failed to encode map e303 with expected crc 2023-09-23T20:02:56.288 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:02:56.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:55 smithi092 bash[58855]: cluster 2023-09-23T20:02:54.684589+0000 mgr.y (mgr.25325) 1672 : cluster 0 pgmap v1735: 73 pgs: 16 peering, 57 active+clean; 582 KiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 946 KiB/s rd, 787 KiB/s wr, 261 op/s; 0 B/s, 0 keys/s, 4 objects/s recovering 2023-09-23T20:02:56.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:55 smithi092 bash[58855]: cluster 2023-09-23T20:02:54.908939+0000 osd.7 (osd.7) 220 : cluster 3 failed to encode map e303 with expected crc 2023-09-23T20:02:56.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:55 smithi092 bash[58855]: cluster 2023-09-23T20:02:55.019770+0000 osd.4 (osd.4) 240 : cluster 3 failed to encode map e303 with expected crc 2023-09-23T20:02:56.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:55 smithi092 bash[58855]: cluster 2023-09-23T20:02:55.229351+0000 osd.5 (osd.5) 220 : cluster 3 failed to encode map e303 with expected crc 2023-09-23T20:02:57.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:57 smithi016 bash[75004]: cluster 2023-09-23T20:02:56.983677+0000 mon.a (mon.0) 2028 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2023-09-23T20:02:57.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:57 smithi016 bash[77995]: cluster 2023-09-23T20:02:56.983677+0000 mon.a (mon.0) 2028 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2023-09-23T20:02:57.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:57 smithi092 bash[58855]: cluster 2023-09-23T20:02:56.983677+0000 mon.a (mon.0) 2028 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2023-09-23T20:02:58.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:58 smithi016 bash[75004]: cluster 2023-09-23T20:02:56.685528+0000 mgr.y (mgr.25325) 1673 : cluster 0 pgmap v1737: 73 pgs: 73 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 1.5 MiB/s rd, 2.1 MiB/s wr, 328 op/s; 3 B/s, 0 keys/s, 7 objects/s recovering 2023-09-23T20:02:58.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:58 smithi016 bash[77995]: cluster 2023-09-23T20:02:56.685528+0000 mgr.y (mgr.25325) 1673 : cluster 0 pgmap v1737: 73 pgs: 73 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 1.5 MiB/s rd, 2.1 MiB/s wr, 328 op/s; 3 B/s, 0 keys/s, 7 objects/s recovering 2023-09-23T20:02:58.495 INFO:tasks.thrashosds.thrasher:inning osd 2023-09-23T20:02:58.495 INFO:tasks.thrashosds.thrasher:Adding osd 1 2023-09-23T20:02:58.495 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd in 1 2023-09-23T20:02:58.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:58 smithi092 bash[58855]: cluster 2023-09-23T20:02:56.685528+0000 mgr.y (mgr.25325) 1673 : cluster 0 pgmap v1737: 73 pgs: 73 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 1.5 MiB/s rd, 2.1 MiB/s wr, 328 op/s; 3 B/s, 0 keys/s, 7 objects/s recovering 2023-09-23T20:02:59.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:02:59 smithi016 bash[75004]: audit 2023-09-23T20:02:58.700480+0000 mon.a (mon.0) 2029 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:02:59.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:02:59 smithi016 bash[77995]: audit 2023-09-23T20:02:58.700480+0000 mon.a (mon.0) 2029 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:02:59.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:02:59 smithi092 bash[58855]: audit 2023-09-23T20:02:58.700480+0000 mon.a (mon.0) 2029 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:03:00.274 INFO:teuthology.orchestra.run.smithi016.stdout:ERROR: (22) Invalid argument 2023-09-23T20:03:00.274 INFO:teuthology.orchestra.run.smithi016.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:03:00.296 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:00 smithi016 bash[75004]: cluster 2023-09-23T20:02:58.686076+0000 mgr.y (mgr.25325) 1674 : cluster 0 pgmap v1738: 73 pgs: 73 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 1.2 MiB/s rd, 1.7 MiB/s wr, 264 op/s; 3 B/s, 0 keys/s, 6 objects/s recovering 2023-09-23T20:03:00.296 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:00 smithi016 bash[75004]: cluster 2023-09-23T20:02:59.847754+0000 mon.a (mon.0) 2030 : cluster 0 osdmap e304: 8 total, 8 up, 7 in 2023-09-23T20:03:00.296 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:00 smithi016 bash[77995]: cluster 2023-09-23T20:02:58.686076+0000 mgr.y (mgr.25325) 1674 : cluster 0 pgmap v1738: 73 pgs: 73 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 1.2 MiB/s rd, 1.7 MiB/s wr, 264 op/s; 3 B/s, 0 keys/s, 6 objects/s recovering 2023-09-23T20:03:00.296 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:00 smithi016 bash[77995]: cluster 2023-09-23T20:02:59.847754+0000 mon.a (mon.0) 2030 : cluster 0 osdmap e304: 8 total, 8 up, 7 in 2023-09-23T20:03:00.361 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:03:00.362 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:03:00.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:00 smithi092 bash[58855]: cluster 2023-09-23T20:02:58.686076+0000 mgr.y (mgr.25325) 1674 : cluster 0 pgmap v1738: 73 pgs: 73 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 1.2 MiB/s rd, 1.7 MiB/s wr, 264 op/s; 3 B/s, 0 keys/s, 6 objects/s recovering 2023-09-23T20:03:00.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:00 smithi092 bash[58855]: cluster 2023-09-23T20:02:59.847754+0000 mon.a (mon.0) 2030 : cluster 0 osdmap e304: 8 total, 8 up, 7 in 2023-09-23T20:03:01.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:01 smithi016 bash[75004]: cluster 2023-09-23T20:02:59.999977+0000 osd.5 (osd.5) 221 : cluster 3 failed to encode map e304 with expected crc 2023-09-23T20:03:01.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:01 smithi016 bash[75004]: cluster 2023-09-23T20:03:00.002614+0000 osd.5 (osd.5) 222 : cluster 3 failed to encode map e304 with expected crc 2023-09-23T20:03:01.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:01 smithi016 bash[75004]: cluster 2023-09-23T20:03:00.318477+0000 osd.6 (osd.6) 239 : cluster 3 failed to encode map e304 with expected crc 2023-09-23T20:03:01.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:01 smithi016 bash[75004]: cluster 2023-09-23T20:03:00.318601+0000 osd.4 (osd.4) 241 : cluster 3 failed to encode map e304 with expected crc 2023-09-23T20:03:01.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:01 smithi016 bash[75004]: cluster 2023-09-23T20:03:00.318601+0000 osd.7 (osd.7) 221 : cluster 3 failed to encode map e304 with expected crc 2023-09-23T20:03:01.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:01 smithi016 bash[77995]: cluster 2023-09-23T20:02:59.999977+0000 osd.5 (osd.5) 221 : cluster 3 failed to encode map e304 with expected crc 2023-09-23T20:03:01.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:01 smithi016 bash[77995]: cluster 2023-09-23T20:03:00.002614+0000 osd.5 (osd.5) 222 : cluster 3 failed to encode map e304 with expected crc 2023-09-23T20:03:01.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:01 smithi016 bash[77995]: cluster 2023-09-23T20:03:00.318477+0000 osd.6 (osd.6) 239 : cluster 3 failed to encode map e304 with expected crc 2023-09-23T20:03:01.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:01 smithi016 bash[77995]: cluster 2023-09-23T20:03:00.318601+0000 osd.4 (osd.4) 241 : cluster 3 failed to encode map e304 with expected crc 2023-09-23T20:03:01.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:01 smithi016 bash[77995]: cluster 2023-09-23T20:03:00.318601+0000 osd.7 (osd.7) 221 : cluster 3 failed to encode map e304 with expected crc 2023-09-23T20:03:01.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:01 smithi092 bash[58855]: cluster 2023-09-23T20:02:59.999977+0000 osd.5 (osd.5) 221 : cluster 3 failed to encode map e304 with expected crc 2023-09-23T20:03:01.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:01 smithi092 bash[58855]: cluster 2023-09-23T20:03:00.002614+0000 osd.5 (osd.5) 222 : cluster 3 failed to encode map e304 with expected crc 2023-09-23T20:03:01.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:01 smithi092 bash[58855]: cluster 2023-09-23T20:03:00.318477+0000 osd.6 (osd.6) 239 : cluster 3 failed to encode map e304 with expected crc 2023-09-23T20:03:01.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:01 smithi092 bash[58855]: cluster 2023-09-23T20:03:00.318601+0000 osd.4 (osd.4) 241 : cluster 3 failed to encode map e304 with expected crc 2023-09-23T20:03:01.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:01 smithi092 bash[58855]: cluster 2023-09-23T20:03:00.318601+0000 osd.7 (osd.7) 221 : cluster 3 failed to encode map e304 with expected crc 2023-09-23T20:03:02.017 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T20:03:02.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:02 smithi016 bash[75004]: cluster 2023-09-23T20:03:00.686928+0000 mgr.y (mgr.25325) 1675 : cluster 0 pgmap v1740: 73 pgs: 73 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 1.1 MiB/s rd, 1.5 MiB/s wr, 258 op/s; 2 B/s, 0 keys/s, 5 objects/s recovering 2023-09-23T20:03:02.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:02 smithi016 bash[77995]: cluster 2023-09-23T20:03:00.686928+0000 mgr.y (mgr.25325) 1675 : cluster 0 pgmap v1740: 73 pgs: 73 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 1.1 MiB/s rd, 1.5 MiB/s wr, 258 op/s; 2 B/s, 0 keys/s, 5 objects/s recovering 2023-09-23T20:03:02.492 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:03:02.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:02 smithi092 bash[58855]: cluster 2023-09-23T20:03:00.686928+0000 mgr.y (mgr.25325) 1675 : cluster 0 pgmap v1740: 73 pgs: 73 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 1.1 MiB/s rd, 1.5 MiB/s wr, 258 op/s; 2 B/s, 0 keys/s, 5 objects/s recovering 2023-09-23T20:03:04.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:04 smithi016 bash[77995]: cluster 2023-09-23T20:03:02.687659+0000 mgr.y (mgr.25325) 1676 : cluster 0 pgmap v1741: 73 pgs: 73 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 456 KiB/s rd, 897 KiB/s wr, 90 op/s; 2 B/s, 2 objects/s recovering 2023-09-23T20:03:04.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:04 smithi016 bash[75004]: cluster 2023-09-23T20:03:02.687659+0000 mgr.y (mgr.25325) 1676 : cluster 0 pgmap v1741: 73 pgs: 73 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 456 KiB/s rd, 897 KiB/s wr, 90 op/s; 2 B/s, 2 objects/s recovering 2023-09-23T20:03:04.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:04 smithi092 bash[58855]: cluster 2023-09-23T20:03:02.687659+0000 mgr.y (mgr.25325) 1676 : cluster 0 pgmap v1741: 73 pgs: 73 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 456 KiB/s rd, 897 KiB/s wr, 90 op/s; 2 B/s, 2 objects/s recovering 2023-09-23T20:03:05.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:05 smithi016 bash[75004]: audit 2023-09-23T20:03:04.412964+0000 mon.a (mon.0) 2031 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:03:05.786 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:03:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:20:03:05] "GET /metrics HTTP/1.1" 200 34016 "" "Prometheus/2.33.4" 2023-09-23T20:03:05.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:05 smithi016 bash[77995]: audit 2023-09-23T20:03:04.412964+0000 mon.a (mon.0) 2031 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:03:05.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:05 smithi092 bash[58855]: audit 2023-09-23T20:03:04.412964+0000 mon.a (mon.0) 2031 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:03:06.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:06 smithi016 bash[75004]: cluster 2023-09-23T20:03:04.688746+0000 mgr.y (mgr.25325) 1677 : cluster 0 pgmap v1742: 73 pgs: 73 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 371 KiB/s rd, 729 KiB/s wr, 73 op/s; 1 B/s, 1 objects/s recovering 2023-09-23T20:03:06.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:06 smithi016 bash[77995]: cluster 2023-09-23T20:03:04.688746+0000 mgr.y (mgr.25325) 1677 : cluster 0 pgmap v1742: 73 pgs: 73 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 371 KiB/s rd, 729 KiB/s wr, 73 op/s; 1 B/s, 1 objects/s recovering 2023-09-23T20:03:06.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:06 smithi092 bash[58855]: cluster 2023-09-23T20:03:04.688746+0000 mgr.y (mgr.25325) 1677 : cluster 0 pgmap v1742: 73 pgs: 73 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 371 KiB/s rd, 729 KiB/s wr, 73 op/s; 1 B/s, 1 objects/s recovering 2023-09-23T20:03:08.704 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:08 smithi016 bash[75004]: cluster 2023-09-23T20:03:06.689685+0000 mgr.y (mgr.25325) 1678 : cluster 0 pgmap v1743: 73 pgs: 73 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 12 KiB/s rd, 921 B/s wr, 18 op/s 2023-09-23T20:03:08.704 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:08 smithi016 bash[77995]: cluster 2023-09-23T20:03:06.689685+0000 mgr.y (mgr.25325) 1678 : cluster 0 pgmap v1743: 73 pgs: 73 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 12 KiB/s rd, 921 B/s wr, 18 op/s 2023-09-23T20:03:08.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:08 smithi092 bash[58855]: cluster 2023-09-23T20:03:06.689685+0000 mgr.y (mgr.25325) 1678 : cluster 0 pgmap v1743: 73 pgs: 73 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 12 KiB/s rd, 921 B/s wr, 18 op/s 2023-09-23T20:03:09.445 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is unset 2023-09-23T20:03:09.551 INFO:teuthology.orchestra.run.smithi016.stdout:osd.0: {} 2023-09-23T20:03:09.573 INFO:teuthology.orchestra.run.smithi016.stdout:osd.1: {} 2023-09-23T20:03:09.595 INFO:teuthology.orchestra.run.smithi016.stdout:osd.2: {} 2023-09-23T20:03:09.615 INFO:teuthology.orchestra.run.smithi016.stdout:osd.3: {} 2023-09-23T20:03:09.637 INFO:teuthology.orchestra.run.smithi016.stdout:osd.4: {} 2023-09-23T20:03:09.662 INFO:teuthology.orchestra.run.smithi016.stdout:osd.5: {} 2023-09-23T20:03:09.681 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:03:09.685 INFO:teuthology.orchestra.run.smithi016.stdout:osd.6: {} 2023-09-23T20:03:09.709 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:09 smithi016 bash[77995]: audit 2023-09-23T20:03:09.067011+0000 mon.a (mon.0) 2032 : audit 1 from='client.? 172.21.15.16:0/4032494819' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:03:09.709 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:09 smithi016 bash[75004]: audit 2023-09-23T20:03:09.067011+0000 mon.a (mon.0) 2032 : audit 1 from='client.? 172.21.15.16:0/4032494819' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:03:09.715 INFO:teuthology.orchestra.run.smithi016.stdout:osd.7: {} 2023-09-23T20:03:09.717 INFO:teuthology.orchestra.run.smithi016.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:03:09.717 INFO:teuthology.orchestra.run.smithi016.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:03:09.717 INFO:teuthology.orchestra.run.smithi016.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:03:09.717 INFO:teuthology.orchestra.run.smithi016.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:03:09.718 INFO:teuthology.orchestra.run.smithi016.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:03:09.718 INFO:teuthology.orchestra.run.smithi016.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:03:09.718 INFO:teuthology.orchestra.run.smithi016.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:03:09.718 INFO:teuthology.orchestra.run.smithi016.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:03:09.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:09 smithi092 bash[58855]: audit 2023-09-23T20:03:09.067011+0000 mon.a (mon.0) 2032 : audit 1 from='client.? 172.21.15.16:0/4032494819' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:03:10.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:10 smithi016 bash[75004]: cluster 2023-09-23T20:03:08.690432+0000 mgr.y (mgr.25325) 1679 : cluster 0 pgmap v1744: 73 pgs: 73 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 12 KiB/s rd, 921 B/s wr, 18 op/s 2023-09-23T20:03:10.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:10 smithi016 bash[75004]: cluster 2023-09-23T20:03:09.424355+0000 mon.a (mon.0) 2033 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:03:10.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:10 smithi016 bash[75004]: audit 2023-09-23T20:03:09.438407+0000 mon.a (mon.0) 2034 : audit 1 from='client.? 172.21.15.16:0/4032494819' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:03:10.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:10 smithi016 bash[75004]: cluster 2023-09-23T20:03:09.438483+0000 mon.a (mon.0) 2035 : cluster 0 osdmap e305: 8 total, 8 up, 7 in 2023-09-23T20:03:10.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:10 smithi016 bash[75004]: audit 2023-09-23T20:03:09.524808+0000 mon.c (mon.2) 252 : audit 0 from='client.? 172.21.15.16:0/2677090493' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:03:10.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:10 smithi016 bash[75004]: cluster 2023-09-23T20:03:09.697651+0000 osd.7 (osd.7) 222 : cluster 3 failed to encode map e305 with expected crc 2023-09-23T20:03:10.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:10 smithi016 bash[75004]: cluster 2023-09-23T20:03:09.699689+0000 osd.7 (osd.7) 223 : cluster 3 failed to encode map e305 with expected crc 2023-09-23T20:03:10.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:10 smithi016 bash[77995]: cluster 2023-09-23T20:03:08.690432+0000 mgr.y (mgr.25325) 1679 : cluster 0 pgmap v1744: 73 pgs: 73 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 12 KiB/s rd, 921 B/s wr, 18 op/s 2023-09-23T20:03:10.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:10 smithi016 bash[77995]: cluster 2023-09-23T20:03:09.424355+0000 mon.a (mon.0) 2033 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:03:10.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:10 smithi016 bash[77995]: audit 2023-09-23T20:03:09.438407+0000 mon.a (mon.0) 2034 : audit 1 from='client.? 172.21.15.16:0/4032494819' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:03:10.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:10 smithi016 bash[77995]: cluster 2023-09-23T20:03:09.438483+0000 mon.a (mon.0) 2035 : cluster 0 osdmap e305: 8 total, 8 up, 7 in 2023-09-23T20:03:10.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:10 smithi016 bash[77995]: audit 2023-09-23T20:03:09.524808+0000 mon.c (mon.2) 252 : audit 0 from='client.? 172.21.15.16:0/2677090493' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:03:10.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:10 smithi016 bash[77995]: cluster 2023-09-23T20:03:09.697651+0000 osd.7 (osd.7) 222 : cluster 3 failed to encode map e305 with expected crc 2023-09-23T20:03:10.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:10 smithi016 bash[77995]: cluster 2023-09-23T20:03:09.699689+0000 osd.7 (osd.7) 223 : cluster 3 failed to encode map e305 with expected crc 2023-09-23T20:03:10.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:10 smithi092 bash[58855]: cluster 2023-09-23T20:03:08.690432+0000 mgr.y (mgr.25325) 1679 : cluster 0 pgmap v1744: 73 pgs: 73 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 12 KiB/s rd, 921 B/s wr, 18 op/s 2023-09-23T20:03:10.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:10 smithi092 bash[58855]: cluster 2023-09-23T20:03:09.424355+0000 mon.a (mon.0) 2033 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:03:10.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:10 smithi092 bash[58855]: audit 2023-09-23T20:03:09.438407+0000 mon.a (mon.0) 2034 : audit 1 from='client.? 172.21.15.16:0/4032494819' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:03:10.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:10 smithi092 bash[58855]: cluster 2023-09-23T20:03:09.438483+0000 mon.a (mon.0) 2035 : cluster 0 osdmap e305: 8 total, 8 up, 7 in 2023-09-23T20:03:10.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:10 smithi092 bash[58855]: audit 2023-09-23T20:03:09.524808+0000 mon.c (mon.2) 252 : audit 0 from='client.? 172.21.15.16:0/2677090493' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:03:10.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:10 smithi092 bash[58855]: cluster 2023-09-23T20:03:09.697651+0000 osd.7 (osd.7) 222 : cluster 3 failed to encode map e305 with expected crc 2023-09-23T20:03:10.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:10 smithi092 bash[58855]: cluster 2023-09-23T20:03:09.699689+0000 osd.7 (osd.7) 223 : cluster 3 failed to encode map e305 with expected crc 2023-09-23T20:03:11.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:11 smithi016 bash[75004]: cluster 2023-09-23T20:03:10.019020+0000 osd.4 (osd.4) 242 : cluster 3 failed to encode map e305 with expected crc 2023-09-23T20:03:11.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:11 smithi016 bash[75004]: cluster 2023-09-23T20:03:10.019076+0000 osd.5 (osd.5) 223 : cluster 3 failed to encode map e305 with expected crc 2023-09-23T20:03:11.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:11 smithi016 bash[75004]: cluster 2023-09-23T20:03:10.019359+0000 osd.6 (osd.6) 240 : cluster 3 failed to encode map e305 with expected crc 2023-09-23T20:03:11.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:11 smithi016 bash[77995]: cluster 2023-09-23T20:03:10.019020+0000 osd.4 (osd.4) 242 : cluster 3 failed to encode map e305 with expected crc 2023-09-23T20:03:11.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:11 smithi016 bash[77995]: cluster 2023-09-23T20:03:10.019076+0000 osd.5 (osd.5) 223 : cluster 3 failed to encode map e305 with expected crc 2023-09-23T20:03:11.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:11 smithi016 bash[77995]: cluster 2023-09-23T20:03:10.019359+0000 osd.6 (osd.6) 240 : cluster 3 failed to encode map e305 with expected crc 2023-09-23T20:03:11.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:11 smithi092 bash[58855]: cluster 2023-09-23T20:03:10.019020+0000 osd.4 (osd.4) 242 : cluster 3 failed to encode map e305 with expected crc 2023-09-23T20:03:11.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:11 smithi092 bash[58855]: cluster 2023-09-23T20:03:10.019076+0000 osd.5 (osd.5) 223 : cluster 3 failed to encode map e305 with expected crc 2023-09-23T20:03:11.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:11 smithi092 bash[58855]: cluster 2023-09-23T20:03:10.019359+0000 osd.6 (osd.6) 240 : cluster 3 failed to encode map e305 with expected crc 2023-09-23T20:03:12.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:12 smithi016 bash[77995]: cluster 2023-09-23T20:03:10.691618+0000 mgr.y (mgr.25325) 1680 : cluster 0 pgmap v1746: 73 pgs: 73 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail 2023-09-23T20:03:12.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:12 smithi016 bash[75004]: cluster 2023-09-23T20:03:10.691618+0000 mgr.y (mgr.25325) 1680 : cluster 0 pgmap v1746: 73 pgs: 73 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail 2023-09-23T20:03:12.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:12 smithi092 bash[58855]: cluster 2023-09-23T20:03:10.691618+0000 mgr.y (mgr.25325) 1680 : cluster 0 pgmap v1746: 73 pgs: 73 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail 2023-09-23T20:03:13.448 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:13 smithi016 bash[75004]: debug 2023-09-23T20:03:13.233+0000 7f0907c76700 -1 mon.a@0(leader).osd e305 definitely_dead 0 2023-09-23T20:03:13.475 INFO:teuthology.orchestra.run.smithi016.stderr:marked in osd.1. 2023-09-23T20:03:13.499 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:13 smithi016 bash[75004]: audit 2023-09-23T20:03:13.239326+0000 mon.c (mon.2) 253 : audit 0 from='client.? 172.21.15.16:0/3148091078' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-09-23T20:03:13.500 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:13 smithi016 bash[75004]: audit 2023-09-23T20:03:13.239688+0000 mon.a (mon.0) 2036 : audit 0 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-09-23T20:03:13.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:13 smithi016 bash[77995]: audit 2023-09-23T20:03:13.239326+0000 mon.c (mon.2) 253 : audit 0 from='client.? 172.21.15.16:0/3148091078' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-09-23T20:03:13.500 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:13 smithi016 bash[77995]: audit 2023-09-23T20:03:13.239688+0000 mon.a (mon.0) 2036 : audit 0 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-09-23T20:03:13.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:13 smithi092 bash[58855]: audit 2023-09-23T20:03:13.239326+0000 mon.c (mon.2) 253 : audit 0 from='client.? 172.21.15.16:0/3148091078' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-09-23T20:03:13.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:13 smithi092 bash[58855]: audit 2023-09-23T20:03:13.239688+0000 mon.a (mon.0) 2036 : audit 0 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-09-23T20:03:14.474 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:14 smithi016 bash[77995]: cluster 2023-09-23T20:03:12.692372+0000 mgr.y (mgr.25325) 1681 : cluster 0 pgmap v1747: 73 pgs: 73 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail 2023-09-23T20:03:14.474 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:14 smithi016 bash[77995]: audit 2023-09-23T20:03:13.455293+0000 mon.a (mon.0) 2037 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-09-23T20:03:14.474 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:14 smithi016 bash[77995]: cluster 2023-09-23T20:03:13.455406+0000 mon.a (mon.0) 2038 : cluster 0 osdmap e306: 8 total, 8 up, 8 in 2023-09-23T20:03:14.474 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:14 smithi016 bash[77995]: cluster 2023-09-23T20:03:13.460053+0000 osd.4 (osd.4) 243 : cluster 3 failed to encode map e306 with expected crc 2023-09-23T20:03:14.475 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:14 smithi016 bash[77995]: cluster 2023-09-23T20:03:13.460218+0000 osd.7 (osd.7) 224 : cluster 3 failed to encode map e306 with expected crc 2023-09-23T20:03:14.475 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:14 smithi016 bash[77995]: cluster 2023-09-23T20:03:13.460651+0000 osd.6 (osd.6) 241 : cluster 3 failed to encode map e306 with expected crc 2023-09-23T20:03:14.475 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:14 smithi016 bash[77995]: cluster 2023-09-23T20:03:13.462199+0000 osd.4 (osd.4) 244 : cluster 3 failed to encode map e306 with expected crc 2023-09-23T20:03:14.475 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:14 smithi016 bash[77995]: cluster 2023-09-23T20:03:13.464679+0000 osd.7 (osd.7) 225 : cluster 3 failed to encode map e306 with expected crc 2023-09-23T20:03:14.475 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:14 smithi016 bash[77995]: cluster 2023-09-23T20:03:13.465324+0000 osd.6 (osd.6) 242 : cluster 3 failed to encode map e306 with expected crc 2023-09-23T20:03:14.475 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:14 smithi016 bash[77995]: cluster 2023-09-23T20:03:13.465578+0000 osd.5 (osd.5) 224 : cluster 3 failed to encode map e306 with expected crc 2023-09-23T20:03:14.475 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:14 smithi016 bash[77995]: cluster 2023-09-23T20:03:13.472450+0000 osd.5 (osd.5) 225 : cluster 3 failed to encode map e306 with expected crc 2023-09-23T20:03:14.475 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:14 smithi016 bash[77995]: audit 2023-09-23T20:03:13.700856+0000 mon.a (mon.0) 2039 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:03:14.594 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:03:14.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:14 smithi016 bash[75004]: cluster 2023-09-23T20:03:12.692372+0000 mgr.y (mgr.25325) 1681 : cluster 0 pgmap v1747: 73 pgs: 73 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail 2023-09-23T20:03:14.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:14 smithi016 bash[75004]: audit 2023-09-23T20:03:13.455293+0000 mon.a (mon.0) 2037 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-09-23T20:03:14.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:14 smithi016 bash[75004]: cluster 2023-09-23T20:03:13.455406+0000 mon.a (mon.0) 2038 : cluster 0 osdmap e306: 8 total, 8 up, 8 in 2023-09-23T20:03:14.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:14 smithi016 bash[75004]: cluster 2023-09-23T20:03:13.460053+0000 osd.4 (osd.4) 243 : cluster 3 failed to encode map e306 with expected crc 2023-09-23T20:03:14.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:14 smithi016 bash[75004]: cluster 2023-09-23T20:03:13.460218+0000 osd.7 (osd.7) 224 : cluster 3 failed to encode map e306 with expected crc 2023-09-23T20:03:14.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:14 smithi016 bash[75004]: cluster 2023-09-23T20:03:13.460651+0000 osd.6 (osd.6) 241 : cluster 3 failed to encode map e306 with expected crc 2023-09-23T20:03:14.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:14 smithi016 bash[75004]: cluster 2023-09-23T20:03:13.462199+0000 osd.4 (osd.4) 244 : cluster 3 failed to encode map e306 with expected crc 2023-09-23T20:03:14.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:14 smithi016 bash[75004]: cluster 2023-09-23T20:03:13.464679+0000 osd.7 (osd.7) 225 : cluster 3 failed to encode map e306 with expected crc 2023-09-23T20:03:14.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:14 smithi016 bash[75004]: cluster 2023-09-23T20:03:13.465324+0000 osd.6 (osd.6) 242 : cluster 3 failed to encode map e306 with expected crc 2023-09-23T20:03:14.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:14 smithi016 bash[75004]: cluster 2023-09-23T20:03:13.465578+0000 osd.5 (osd.5) 224 : cluster 3 failed to encode map e306 with expected crc 2023-09-23T20:03:14.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:14 smithi016 bash[75004]: cluster 2023-09-23T20:03:13.472450+0000 osd.5 (osd.5) 225 : cluster 3 failed to encode map e306 with expected crc 2023-09-23T20:03:14.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:14 smithi016 bash[75004]: audit 2023-09-23T20:03:13.700856+0000 mon.a (mon.0) 2039 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:03:14.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:14 smithi092 bash[58855]: cluster 2023-09-23T20:03:12.692372+0000 mgr.y (mgr.25325) 1681 : cluster 0 pgmap v1747: 73 pgs: 73 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail 2023-09-23T20:03:14.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:14 smithi092 bash[58855]: audit 2023-09-23T20:03:13.455293+0000 mon.a (mon.0) 2037 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-09-23T20:03:14.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:14 smithi092 bash[58855]: cluster 2023-09-23T20:03:13.455406+0000 mon.a (mon.0) 2038 : cluster 0 osdmap e306: 8 total, 8 up, 8 in 2023-09-23T20:03:14.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:14 smithi092 bash[58855]: cluster 2023-09-23T20:03:13.460053+0000 osd.4 (osd.4) 243 : cluster 3 failed to encode map e306 with expected crc 2023-09-23T20:03:14.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:14 smithi092 bash[58855]: cluster 2023-09-23T20:03:13.460218+0000 osd.7 (osd.7) 224 : cluster 3 failed to encode map e306 with expected crc 2023-09-23T20:03:14.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:14 smithi092 bash[58855]: cluster 2023-09-23T20:03:13.460651+0000 osd.6 (osd.6) 241 : cluster 3 failed to encode map e306 with expected crc 2023-09-23T20:03:14.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:14 smithi092 bash[58855]: cluster 2023-09-23T20:03:13.462199+0000 osd.4 (osd.4) 244 : cluster 3 failed to encode map e306 with expected crc 2023-09-23T20:03:14.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:14 smithi092 bash[58855]: cluster 2023-09-23T20:03:13.464679+0000 osd.7 (osd.7) 225 : cluster 3 failed to encode map e306 with expected crc 2023-09-23T20:03:14.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:14 smithi092 bash[58855]: cluster 2023-09-23T20:03:13.465324+0000 osd.6 (osd.6) 242 : cluster 3 failed to encode map e306 with expected crc 2023-09-23T20:03:14.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:14 smithi092 bash[58855]: cluster 2023-09-23T20:03:13.465578+0000 osd.5 (osd.5) 224 : cluster 3 failed to encode map e306 with expected crc 2023-09-23T20:03:14.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:14 smithi092 bash[58855]: cluster 2023-09-23T20:03:13.472450+0000 osd.5 (osd.5) 225 : cluster 3 failed to encode map e306 with expected crc 2023-09-23T20:03:14.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:14 smithi092 bash[58855]: audit 2023-09-23T20:03:13.700856+0000 mon.a (mon.0) 2039 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:03:15.564 INFO:tasks.thrashosds.thrasher:Added osd 1 2023-09-23T20:03:15.565 INFO:tasks.thrashosds.thrasher:fixing pg num pool device_health_metrics 2023-09-23T20:03:15.565 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd pool set device_health_metrics pgp_num 21 2023-09-23T20:03:15.785 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:03:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:20:03:15] "GET /metrics HTTP/1.1" 200 34017 "" "Prometheus/2.33.4" 2023-09-23T20:03:15.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:15 smithi016 bash[75004]: cluster 2023-09-23T20:03:14.460281+0000 osd.5 (osd.5) 226 : cluster 3 failed to encode map e307 with expected crc 2023-09-23T20:03:15.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:15 smithi016 bash[75004]: cluster 2023-09-23T20:03:14.460918+0000 mon.a (mon.0) 2040 : cluster 0 osdmap e307: 8 total, 8 up, 8 in 2023-09-23T20:03:15.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:15 smithi016 bash[75004]: cluster 2023-09-23T20:03:14.462807+0000 osd.4 (osd.4) 245 : cluster 3 failed to encode map e307 with expected crc 2023-09-23T20:03:15.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:15 smithi016 bash[75004]: cluster 2023-09-23T20:03:14.465338+0000 osd.4 (osd.4) 246 : cluster 3 failed to encode map e307 with expected crc 2023-09-23T20:03:15.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:15 smithi016 bash[75004]: cluster 2023-09-23T20:03:14.465765+0000 osd.5 (osd.5) 227 : cluster 3 failed to encode map e307 with expected crc 2023-09-23T20:03:15.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:15 smithi016 bash[75004]: cluster 2023-09-23T20:03:14.466048+0000 osd.7 (osd.7) 226 : cluster 3 failed to encode map e307 with expected crc 2023-09-23T20:03:15.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:15 smithi016 bash[75004]: cluster 2023-09-23T20:03:14.466055+0000 osd.6 (osd.6) 243 : cluster 3 failed to encode map e307 with expected crc 2023-09-23T20:03:15.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:15 smithi016 bash[75004]: cluster 2023-09-23T20:03:14.468193+0000 osd.6 (osd.6) 244 : cluster 3 failed to encode map e307 with expected crc 2023-09-23T20:03:15.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:15 smithi016 bash[75004]: cluster 2023-09-23T20:03:14.468267+0000 osd.4 (osd.4) 247 : cluster 3 failed to encode map e307 with expected crc 2023-09-23T20:03:15.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:15 smithi016 bash[75004]: cluster 2023-09-23T20:03:14.469053+0000 osd.5 (osd.5) 228 : cluster 3 failed to encode map e307 with expected crc 2023-09-23T20:03:15.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:15 smithi016 bash[75004]: cluster 2023-09-23T20:03:14.469213+0000 osd.7 (osd.7) 227 : cluster 3 failed to encode map e307 with expected crc 2023-09-23T20:03:15.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:15 smithi016 bash[75004]: cluster 2023-09-23T20:03:14.469679+0000 osd.4 (osd.4) 248 : cluster 3 failed to encode map e307 with expected crc 2023-09-23T20:03:15.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:15 smithi016 bash[75004]: cluster 2023-09-23T20:03:14.470865+0000 osd.7 (osd.7) 228 : cluster 3 failed to encode map e307 with expected crc 2023-09-23T20:03:15.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:15 smithi016 bash[75004]: cluster 2023-09-23T20:03:14.472071+0000 osd.4 (osd.4) 249 : cluster 3 failed to encode map e307 with expected crc 2023-09-23T20:03:15.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:15 smithi016 bash[75004]: cluster 2023-09-23T20:03:14.692840+0000 mgr.y (mgr.25325) 1682 : cluster 0 pgmap v1750: 73 pgs: 1 active+recovering, 72 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s, 0 objects/s recovering 2023-09-23T20:03:15.787 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:15 smithi016 bash[75004]: cluster 2023-09-23T20:03:15.460658+0000 mon.a (mon.0) 2041 : cluster 0 osdmap e308: 8 total, 8 up, 8 in 2023-09-23T20:03:15.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:15 smithi016 bash[77995]: cluster 2023-09-23T20:03:14.460281+0000 osd.5 (osd.5) 226 : cluster 3 failed to encode map e307 with expected crc 2023-09-23T20:03:15.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:15 smithi016 bash[77995]: cluster 2023-09-23T20:03:14.460918+0000 mon.a (mon.0) 2040 : cluster 0 osdmap e307: 8 total, 8 up, 8 in 2023-09-23T20:03:15.788 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:15 smithi016 bash[77995]: cluster 2023-09-23T20:03:14.462807+0000 osd.4 (osd.4) 245 : cluster 3 failed to encode map e307 with expected crc 2023-09-23T20:03:15.788 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:15 smithi016 bash[77995]: cluster 2023-09-23T20:03:14.465338+0000 osd.4 (osd.4) 246 : cluster 3 failed to encode map e307 with expected crc 2023-09-23T20:03:15.788 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:15 smithi016 bash[77995]: cluster 2023-09-23T20:03:14.465765+0000 osd.5 (osd.5) 227 : cluster 3 failed to encode map e307 with expected crc 2023-09-23T20:03:15.788 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:15 smithi016 bash[77995]: cluster 2023-09-23T20:03:14.466048+0000 osd.7 (osd.7) 226 : cluster 3 failed to encode map e307 with expected crc 2023-09-23T20:03:15.788 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:15 smithi016 bash[77995]: cluster 2023-09-23T20:03:14.466055+0000 osd.6 (osd.6) 243 : cluster 3 failed to encode map e307 with expected crc 2023-09-23T20:03:15.788 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:15 smithi016 bash[77995]: cluster 2023-09-23T20:03:14.468193+0000 osd.6 (osd.6) 244 : cluster 3 failed to encode map e307 with expected crc 2023-09-23T20:03:15.788 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:15 smithi016 bash[77995]: cluster 2023-09-23T20:03:14.468267+0000 osd.4 (osd.4) 247 : cluster 3 failed to encode map e307 with expected crc 2023-09-23T20:03:15.788 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:15 smithi016 bash[77995]: cluster 2023-09-23T20:03:14.469053+0000 osd.5 (osd.5) 228 : cluster 3 failed to encode map e307 with expected crc 2023-09-23T20:03:15.788 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:15 smithi016 bash[77995]: cluster 2023-09-23T20:03:14.469213+0000 osd.7 (osd.7) 227 : cluster 3 failed to encode map e307 with expected crc 2023-09-23T20:03:15.788 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:15 smithi016 bash[77995]: cluster 2023-09-23T20:03:14.469679+0000 osd.4 (osd.4) 248 : cluster 3 failed to encode map e307 with expected crc 2023-09-23T20:03:15.788 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:15 smithi016 bash[77995]: cluster 2023-09-23T20:03:14.470865+0000 osd.7 (osd.7) 228 : cluster 3 failed to encode map e307 with expected crc 2023-09-23T20:03:15.788 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:15 smithi016 bash[77995]: cluster 2023-09-23T20:03:14.472071+0000 osd.4 (osd.4) 249 : cluster 3 failed to encode map e307 with expected crc 2023-09-23T20:03:15.789 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:15 smithi016 bash[77995]: cluster 2023-09-23T20:03:14.692840+0000 mgr.y (mgr.25325) 1682 : cluster 0 pgmap v1750: 73 pgs: 1 active+recovering, 72 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s, 0 objects/s recovering 2023-09-23T20:03:15.789 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:15 smithi016 bash[77995]: cluster 2023-09-23T20:03:15.460658+0000 mon.a (mon.0) 2041 : cluster 0 osdmap e308: 8 total, 8 up, 8 in 2023-09-23T20:03:15.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:15 smithi092 bash[58855]: cluster 2023-09-23T20:03:14.460281+0000 osd.5 (osd.5) 226 : cluster 3 failed to encode map e307 with expected crc 2023-09-23T20:03:15.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:15 smithi092 bash[58855]: cluster 2023-09-23T20:03:14.460918+0000 mon.a (mon.0) 2040 : cluster 0 osdmap e307: 8 total, 8 up, 8 in 2023-09-23T20:03:15.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:15 smithi092 bash[58855]: cluster 2023-09-23T20:03:14.462807+0000 osd.4 (osd.4) 245 : cluster 3 failed to encode map e307 with expected crc 2023-09-23T20:03:15.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:15 smithi092 bash[58855]: cluster 2023-09-23T20:03:14.465338+0000 osd.4 (osd.4) 246 : cluster 3 failed to encode map e307 with expected crc 2023-09-23T20:03:15.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:15 smithi092 bash[58855]: cluster 2023-09-23T20:03:14.465765+0000 osd.5 (osd.5) 227 : cluster 3 failed to encode map e307 with expected crc 2023-09-23T20:03:15.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:15 smithi092 bash[58855]: cluster 2023-09-23T20:03:14.466048+0000 osd.7 (osd.7) 226 : cluster 3 failed to encode map e307 with expected crc 2023-09-23T20:03:15.790 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:15 smithi092 bash[58855]: cluster 2023-09-23T20:03:14.466055+0000 osd.6 (osd.6) 243 : cluster 3 failed to encode map e307 with expected crc 2023-09-23T20:03:15.790 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:15 smithi092 bash[58855]: cluster 2023-09-23T20:03:14.468193+0000 osd.6 (osd.6) 244 : cluster 3 failed to encode map e307 with expected crc 2023-09-23T20:03:15.790 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:15 smithi092 bash[58855]: cluster 2023-09-23T20:03:14.468267+0000 osd.4 (osd.4) 247 : cluster 3 failed to encode map e307 with expected crc 2023-09-23T20:03:15.790 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:15 smithi092 bash[58855]: cluster 2023-09-23T20:03:14.469053+0000 osd.5 (osd.5) 228 : cluster 3 failed to encode map e307 with expected crc 2023-09-23T20:03:15.790 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:15 smithi092 bash[58855]: cluster 2023-09-23T20:03:14.469213+0000 osd.7 (osd.7) 227 : cluster 3 failed to encode map e307 with expected crc 2023-09-23T20:03:15.790 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:15 smithi092 bash[58855]: cluster 2023-09-23T20:03:14.469679+0000 osd.4 (osd.4) 248 : cluster 3 failed to encode map e307 with expected crc 2023-09-23T20:03:15.790 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:15 smithi092 bash[58855]: cluster 2023-09-23T20:03:14.470865+0000 osd.7 (osd.7) 228 : cluster 3 failed to encode map e307 with expected crc 2023-09-23T20:03:15.790 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:15 smithi092 bash[58855]: cluster 2023-09-23T20:03:14.472071+0000 osd.4 (osd.4) 249 : cluster 3 failed to encode map e307 with expected crc 2023-09-23T20:03:15.790 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:15 smithi092 bash[58855]: cluster 2023-09-23T20:03:14.692840+0000 mgr.y (mgr.25325) 1682 : cluster 0 pgmap v1750: 73 pgs: 1 active+recovering, 72 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s, 0 objects/s recovering 2023-09-23T20:03:15.790 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:15 smithi092 bash[58855]: cluster 2023-09-23T20:03:15.460658+0000 mon.a (mon.0) 2041 : cluster 0 osdmap e308: 8 total, 8 up, 8 in 2023-09-23T20:03:16.226 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-09-23T20:03:16.226 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [], 2023-09-23T20:03:16.226 INFO:teuthology.orchestra.run.smithi016.stdout: "num_ops": 0 2023-09-23T20:03:16.226 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-09-23T20:03:16.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:16 smithi016 bash[75004]: cluster 2023-09-23T20:03:15.466798+0000 osd.7 (osd.7) 229 : cluster 3 failed to encode map e308 with expected crc 2023-09-23T20:03:16.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:16 smithi016 bash[75004]: cluster 2023-09-23T20:03:15.466847+0000 osd.6 (osd.6) 245 : cluster 3 failed to encode map e308 with expected crc 2023-09-23T20:03:16.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:16 smithi016 bash[75004]: cluster 2023-09-23T20:03:15.467001+0000 osd.4 (osd.4) 250 : cluster 3 failed to encode map e308 with expected crc 2023-09-23T20:03:16.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:16 smithi016 bash[75004]: cluster 2023-09-23T20:03:15.467140+0000 osd.5 (osd.5) 229 : cluster 3 failed to encode map e308 with expected crc 2023-09-23T20:03:16.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:16 smithi016 bash[77995]: cluster 2023-09-23T20:03:15.466798+0000 osd.7 (osd.7) 229 : cluster 3 failed to encode map e308 with expected crc 2023-09-23T20:03:16.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:16 smithi016 bash[77995]: cluster 2023-09-23T20:03:15.466847+0000 osd.6 (osd.6) 245 : cluster 3 failed to encode map e308 with expected crc 2023-09-23T20:03:16.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:16 smithi016 bash[77995]: cluster 2023-09-23T20:03:15.467001+0000 osd.4 (osd.4) 250 : cluster 3 failed to encode map e308 with expected crc 2023-09-23T20:03:16.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:16 smithi016 bash[77995]: cluster 2023-09-23T20:03:15.467140+0000 osd.5 (osd.5) 229 : cluster 3 failed to encode map e308 with expected crc 2023-09-23T20:03:16.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:16 smithi092 bash[58855]: cluster 2023-09-23T20:03:15.466798+0000 osd.7 (osd.7) 229 : cluster 3 failed to encode map e308 with expected crc 2023-09-23T20:03:16.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:16 smithi092 bash[58855]: cluster 2023-09-23T20:03:15.466847+0000 osd.6 (osd.6) 245 : cluster 3 failed to encode map e308 with expected crc 2023-09-23T20:03:16.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:16 smithi092 bash[58855]: cluster 2023-09-23T20:03:15.467001+0000 osd.4 (osd.4) 250 : cluster 3 failed to encode map e308 with expected crc 2023-09-23T20:03:16.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:16 smithi092 bash[58855]: cluster 2023-09-23T20:03:15.467140+0000 osd.5 (osd.5) 229 : cluster 3 failed to encode map e308 with expected crc 2023-09-23T20:03:17.379 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T20:03:17.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:17 smithi016 bash[75004]: cluster 2023-09-23T20:03:16.484486+0000 osd.6 (osd.6) 246 : cluster 3 failed to encode map e309 with expected crc 2023-09-23T20:03:17.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:17 smithi016 bash[75004]: cluster 2023-09-23T20:03:16.485172+0000 mon.a (mon.0) 2042 : cluster 0 osdmap e309: 8 total, 8 up, 8 in 2023-09-23T20:03:17.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:17 smithi016 bash[75004]: cluster 2023-09-23T20:03:16.489125+0000 osd.7 (osd.7) 230 : cluster 3 failed to encode map e309 with expected crc 2023-09-23T20:03:17.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:17 smithi016 bash[75004]: cluster 2023-09-23T20:03:16.489915+0000 osd.6 (osd.6) 247 : cluster 3 failed to encode map e309 with expected crc 2023-09-23T20:03:17.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:17 smithi016 bash[75004]: cluster 2023-09-23T20:03:16.490057+0000 osd.4 (osd.4) 251 : cluster 3 failed to encode map e309 with expected crc 2023-09-23T20:03:17.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:17 smithi016 bash[75004]: cluster 2023-09-23T20:03:16.693462+0000 mgr.y (mgr.25325) 1683 : cluster 0 pgmap v1753: 73 pgs: 1 active+remapped, 1 active+recovering, 71 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 1024 KiB/s, 4 objects/s recovering 2023-09-23T20:03:17.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:17 smithi016 bash[75004]: cluster 2023-09-23T20:03:17.487583+0000 mon.a (mon.0) 2043 : cluster 0 osdmap e310: 8 total, 8 up, 8 in 2023-09-23T20:03:17.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:17 smithi016 bash[77995]: cluster 2023-09-23T20:03:16.484486+0000 osd.6 (osd.6) 246 : cluster 3 failed to encode map e309 with expected crc 2023-09-23T20:03:17.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:17 smithi016 bash[77995]: cluster 2023-09-23T20:03:16.485172+0000 mon.a (mon.0) 2042 : cluster 0 osdmap e309: 8 total, 8 up, 8 in 2023-09-23T20:03:17.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:17 smithi016 bash[77995]: cluster 2023-09-23T20:03:16.489125+0000 osd.7 (osd.7) 230 : cluster 3 failed to encode map e309 with expected crc 2023-09-23T20:03:17.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:17 smithi016 bash[77995]: cluster 2023-09-23T20:03:16.489915+0000 osd.6 (osd.6) 247 : cluster 3 failed to encode map e309 with expected crc 2023-09-23T20:03:17.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:17 smithi016 bash[77995]: cluster 2023-09-23T20:03:16.490057+0000 osd.4 (osd.4) 251 : cluster 3 failed to encode map e309 with expected crc 2023-09-23T20:03:17.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:17 smithi016 bash[77995]: cluster 2023-09-23T20:03:16.693462+0000 mgr.y (mgr.25325) 1683 : cluster 0 pgmap v1753: 73 pgs: 1 active+remapped, 1 active+recovering, 71 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 1024 KiB/s, 4 objects/s recovering 2023-09-23T20:03:17.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:17 smithi016 bash[77995]: cluster 2023-09-23T20:03:17.487583+0000 mon.a (mon.0) 2043 : cluster 0 osdmap e310: 8 total, 8 up, 8 in 2023-09-23T20:03:17.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:17 smithi092 bash[58855]: cluster 2023-09-23T20:03:16.484486+0000 osd.6 (osd.6) 246 : cluster 3 failed to encode map e309 with expected crc 2023-09-23T20:03:17.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:17 smithi092 bash[58855]: cluster 2023-09-23T20:03:16.485172+0000 mon.a (mon.0) 2042 : cluster 0 osdmap e309: 8 total, 8 up, 8 in 2023-09-23T20:03:17.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:17 smithi092 bash[58855]: cluster 2023-09-23T20:03:16.489125+0000 osd.7 (osd.7) 230 : cluster 3 failed to encode map e309 with expected crc 2023-09-23T20:03:17.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:17 smithi092 bash[58855]: cluster 2023-09-23T20:03:16.489915+0000 osd.6 (osd.6) 247 : cluster 3 failed to encode map e309 with expected crc 2023-09-23T20:03:17.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:17 smithi092 bash[58855]: cluster 2023-09-23T20:03:16.490057+0000 osd.4 (osd.4) 251 : cluster 3 failed to encode map e309 with expected crc 2023-09-23T20:03:17.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:17 smithi092 bash[58855]: cluster 2023-09-23T20:03:16.693462+0000 mgr.y (mgr.25325) 1683 : cluster 0 pgmap v1753: 73 pgs: 1 active+remapped, 1 active+recovering, 71 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 1024 KiB/s, 4 objects/s recovering 2023-09-23T20:03:17.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:17 smithi092 bash[58855]: cluster 2023-09-23T20:03:17.487583+0000 mon.a (mon.0) 2043 : cluster 0 osdmap e310: 8 total, 8 up, 8 in 2023-09-23T20:03:18.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:18 smithi016 bash[75004]: cluster 2023-09-23T20:03:17.493502+0000 osd.7 (osd.7) 231 : cluster 3 failed to encode map e310 with expected crc 2023-09-23T20:03:18.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:18 smithi016 bash[75004]: cluster 2023-09-23T20:03:17.494424+0000 osd.4 (osd.4) 252 : cluster 3 failed to encode map e310 with expected crc 2023-09-23T20:03:18.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:18 smithi016 bash[75004]: cluster 2023-09-23T20:03:17.494515+0000 osd.6 (osd.6) 248 : cluster 3 failed to encode map e310 with expected crc 2023-09-23T20:03:18.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:18 smithi016 bash[77995]: cluster 2023-09-23T20:03:17.493502+0000 osd.7 (osd.7) 231 : cluster 3 failed to encode map e310 with expected crc 2023-09-23T20:03:18.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:18 smithi016 bash[77995]: cluster 2023-09-23T20:03:17.494424+0000 osd.4 (osd.4) 252 : cluster 3 failed to encode map e310 with expected crc 2023-09-23T20:03:18.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:18 smithi016 bash[77995]: cluster 2023-09-23T20:03:17.494515+0000 osd.6 (osd.6) 248 : cluster 3 failed to encode map e310 with expected crc 2023-09-23T20:03:18.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:18 smithi092 bash[58855]: cluster 2023-09-23T20:03:17.493502+0000 osd.7 (osd.7) 231 : cluster 3 failed to encode map e310 with expected crc 2023-09-23T20:03:18.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:18 smithi092 bash[58855]: cluster 2023-09-23T20:03:17.494424+0000 osd.4 (osd.4) 252 : cluster 3 failed to encode map e310 with expected crc 2023-09-23T20:03:18.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:18 smithi092 bash[58855]: cluster 2023-09-23T20:03:17.494515+0000 osd.6 (osd.6) 248 : cluster 3 failed to encode map e310 with expected crc 2023-09-23T20:03:19.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:19 smithi016 bash[75004]: cluster 2023-09-23T20:03:18.694186+0000 mgr.y (mgr.25325) 1684 : cluster 0 pgmap v1755: 73 pgs: 2 peering, 1 active+recovering, 70 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 966 KiB/s, 10 objects/s recovering 2023-09-23T20:03:19.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:19 smithi016 bash[77995]: cluster 2023-09-23T20:03:18.694186+0000 mgr.y (mgr.25325) 1684 : cluster 0 pgmap v1755: 73 pgs: 2 peering, 1 active+recovering, 70 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 966 KiB/s, 10 objects/s recovering 2023-09-23T20:03:19.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:19 smithi092 bash[58855]: cluster 2023-09-23T20:03:18.694186+0000 mgr.y (mgr.25325) 1684 : cluster 0 pgmap v1755: 73 pgs: 2 peering, 1 active+recovering, 70 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 966 KiB/s, 10 objects/s recovering 2023-09-23T20:03:19.893 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:03:19.893 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:03:21.927 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:03:21.949 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:21 smithi016 bash[75004]: cluster 2023-09-23T20:03:20.695146+0000 mgr.y (mgr.25325) 1685 : cluster 0 pgmap v1756: 73 pgs: 2 peering, 71 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 682 KiB/s, 7 objects/s recovering 2023-09-23T20:03:21.949 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:21 smithi016 bash[77995]: cluster 2023-09-23T20:03:20.695146+0000 mgr.y (mgr.25325) 1685 : cluster 0 pgmap v1756: 73 pgs: 2 peering, 71 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 682 KiB/s, 7 objects/s recovering 2023-09-23T20:03:22.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:21 smithi092 bash[58855]: cluster 2023-09-23T20:03:20.695146+0000 mgr.y (mgr.25325) 1685 : cluster 0 pgmap v1756: 73 pgs: 2 peering, 71 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 682 KiB/s, 7 objects/s recovering 2023-09-23T20:03:24.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:23 smithi016 bash[77995]: cluster 2023-09-23T20:03:22.695987+0000 mgr.y (mgr.25325) 1686 : cluster 0 pgmap v1757: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 566 KiB/s, 5 objects/s recovering 2023-09-23T20:03:24.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:23 smithi016 bash[75004]: cluster 2023-09-23T20:03:22.695987+0000 mgr.y (mgr.25325) 1686 : cluster 0 pgmap v1757: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 566 KiB/s, 5 objects/s recovering 2023-09-23T20:03:24.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:23 smithi092 bash[58855]: cluster 2023-09-23T20:03:22.695987+0000 mgr.y (mgr.25325) 1686 : cluster 0 pgmap v1757: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 566 KiB/s, 5 objects/s recovering 2023-09-23T20:03:25.785 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:03:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:20:03:25] "GET /metrics HTTP/1.1" 200 34017 "" "Prometheus/2.33.4" 2023-09-23T20:03:25.844 INFO:teuthology.orchestra.run.smithi016.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-09-23T20:03:26.123 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:25 smithi016 bash[75004]: cluster 2023-09-23T20:03:24.696839+0000 mgr.y (mgr.25325) 1687 : cluster 0 pgmap v1758: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 2 B/s, 3 objects/s recovering 2023-09-23T20:03:26.123 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:25 smithi016 bash[75004]: audit 2023-09-23T20:03:25.837288+0000 mon.c (mon.2) 254 : audit 1 from='client.? 172.21.15.16:0/1791417831' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T20:03:26.123 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:25 smithi016 bash[75004]: audit 2023-09-23T20:03:25.837673+0000 mon.a (mon.0) 2044 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T20:03:26.123 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:25 smithi016 bash[77995]: cluster 2023-09-23T20:03:24.696839+0000 mgr.y (mgr.25325) 1687 : cluster 0 pgmap v1758: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 2 B/s, 3 objects/s recovering 2023-09-23T20:03:26.124 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:25 smithi016 bash[77995]: audit 2023-09-23T20:03:25.837288+0000 mon.c (mon.2) 254 : audit 1 from='client.? 172.21.15.16:0/1791417831' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T20:03:26.124 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:25 smithi016 bash[77995]: audit 2023-09-23T20:03:25.837673+0000 mon.a (mon.0) 2044 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T20:03:26.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:25 smithi092 bash[58855]: cluster 2023-09-23T20:03:24.696839+0000 mgr.y (mgr.25325) 1687 : cluster 0 pgmap v1758: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 2 B/s, 3 objects/s recovering 2023-09-23T20:03:26.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:25 smithi092 bash[58855]: audit 2023-09-23T20:03:25.837288+0000 mon.c (mon.2) 254 : audit 1 from='client.? 172.21.15.16:0/1791417831' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T20:03:26.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:25 smithi092 bash[58855]: audit 2023-09-23T20:03:25.837673+0000 mon.a (mon.0) 2044 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T20:03:26.881 INFO:teuthology.orchestra.run.smithi016.stderr:noscrub is unset 2023-09-23T20:03:27.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:26 smithi016 bash[75004]: audit 2023-09-23T20:03:26.196195+0000 mon.c (mon.2) 255 : audit 1 from='client.? 172.21.15.16:0/2343679481' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:03:27.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:26 smithi016 bash[75004]: audit 2023-09-23T20:03:26.196570+0000 mon.a (mon.0) 2045 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:03:27.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:26 smithi016 bash[77995]: audit 2023-09-23T20:03:26.196195+0000 mon.c (mon.2) 255 : audit 1 from='client.? 172.21.15.16:0/2343679481' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:03:27.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:26 smithi016 bash[77995]: audit 2023-09-23T20:03:26.196570+0000 mon.a (mon.0) 2045 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:03:27.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:26 smithi092 bash[58855]: audit 2023-09-23T20:03:26.196195+0000 mon.c (mon.2) 255 : audit 1 from='client.? 172.21.15.16:0/2343679481' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:03:27.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:26 smithi092 bash[58855]: audit 2023-09-23T20:03:26.196570+0000 mon.a (mon.0) 2045 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:03:27.853 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-09-23T20:03:27.854 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=600 2023-09-23T20:03:28.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:27 smithi016 bash[75004]: cluster 2023-09-23T20:03:26.697689+0000 mgr.y (mgr.25325) 1688 : cluster 0 pgmap v1759: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 1 B/s, 2 objects/s recovering 2023-09-23T20:03:28.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:27 smithi016 bash[75004]: audit 2023-09-23T20:03:26.873166+0000 mon.a (mon.0) 2046 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:03:28.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:27 smithi016 bash[75004]: cluster 2023-09-23T20:03:26.873251+0000 mon.a (mon.0) 2047 : cluster 0 osdmap e311: 8 total, 8 up, 8 in 2023-09-23T20:03:28.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:27 smithi016 bash[75004]: cluster 2023-09-23T20:03:26.875622+0000 osd.5 (osd.5) 230 : cluster 3 failed to encode map e311 with expected crc 2023-09-23T20:03:28.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:27 smithi016 bash[75004]: cluster 2023-09-23T20:03:26.938266+0000 osd.6 (osd.6) 249 : cluster 3 failed to encode map e311 with expected crc 2023-09-23T20:03:28.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:27 smithi016 bash[75004]: cluster 2023-09-23T20:03:27.534226+0000 osd.4 (osd.4) 253 : cluster 3 failed to encode map e311 with expected crc 2023-09-23T20:03:28.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:27 smithi016 bash[75004]: cluster 2023-09-23T20:03:27.536981+0000 osd.4 (osd.4) 254 : cluster 3 failed to encode map e311 with expected crc 2023-09-23T20:03:28.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:27 smithi016 bash[77995]: cluster 2023-09-23T20:03:26.697689+0000 mgr.y (mgr.25325) 1688 : cluster 0 pgmap v1759: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 1 B/s, 2 objects/s recovering 2023-09-23T20:03:28.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:27 smithi016 bash[77995]: audit 2023-09-23T20:03:26.873166+0000 mon.a (mon.0) 2046 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:03:28.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:27 smithi016 bash[77995]: cluster 2023-09-23T20:03:26.873251+0000 mon.a (mon.0) 2047 : cluster 0 osdmap e311: 8 total, 8 up, 8 in 2023-09-23T20:03:28.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:27 smithi016 bash[77995]: cluster 2023-09-23T20:03:26.875622+0000 osd.5 (osd.5) 230 : cluster 3 failed to encode map e311 with expected crc 2023-09-23T20:03:28.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:27 smithi016 bash[77995]: cluster 2023-09-23T20:03:26.938266+0000 osd.6 (osd.6) 249 : cluster 3 failed to encode map e311 with expected crc 2023-09-23T20:03:28.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:27 smithi016 bash[77995]: cluster 2023-09-23T20:03:27.534226+0000 osd.4 (osd.4) 253 : cluster 3 failed to encode map e311 with expected crc 2023-09-23T20:03:28.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:27 smithi016 bash[77995]: cluster 2023-09-23T20:03:27.536981+0000 osd.4 (osd.4) 254 : cluster 3 failed to encode map e311 with expected crc 2023-09-23T20:03:28.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:27 smithi092 bash[58855]: cluster 2023-09-23T20:03:26.697689+0000 mgr.y (mgr.25325) 1688 : cluster 0 pgmap v1759: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 1 B/s, 2 objects/s recovering 2023-09-23T20:03:28.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:27 smithi092 bash[58855]: audit 2023-09-23T20:03:26.873166+0000 mon.a (mon.0) 2046 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:03:28.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:27 smithi092 bash[58855]: cluster 2023-09-23T20:03:26.873251+0000 mon.a (mon.0) 2047 : cluster 0 osdmap e311: 8 total, 8 up, 8 in 2023-09-23T20:03:28.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:27 smithi092 bash[58855]: cluster 2023-09-23T20:03:26.875622+0000 osd.5 (osd.5) 230 : cluster 3 failed to encode map e311 with expected crc 2023-09-23T20:03:28.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:27 smithi092 bash[58855]: cluster 2023-09-23T20:03:26.938266+0000 osd.6 (osd.6) 249 : cluster 3 failed to encode map e311 with expected crc 2023-09-23T20:03:28.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:27 smithi092 bash[58855]: cluster 2023-09-23T20:03:27.534226+0000 osd.4 (osd.4) 253 : cluster 3 failed to encode map e311 with expected crc 2023-09-23T20:03:28.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:27 smithi092 bash[58855]: cluster 2023-09-23T20:03:27.536981+0000 osd.4 (osd.4) 254 : cluster 3 failed to encode map e311 with expected crc 2023-09-23T20:03:28.855 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:03:29.246 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-09-23T20:03:29.246 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [], 2023-09-23T20:03:29.246 INFO:teuthology.orchestra.run.smithi016.stdout: "complaint_time": 30, 2023-09-23T20:03:29.246 INFO:teuthology.orchestra.run.smithi016.stdout: "num_blocked_ops": 0 2023-09-23T20:03:29.246 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-09-23T20:03:29.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:28 smithi016 bash[77995]: audit 2023-09-23T20:03:28.701333+0000 mon.a (mon.0) 2048 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:03:29.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:28 smithi016 bash[75004]: audit 2023-09-23T20:03:28.701333+0000 mon.a (mon.0) 2048 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:03:29.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:28 smithi092 bash[58855]: audit 2023-09-23T20:03:28.701333+0000 mon.a (mon.0) 2048 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:03:30.335 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T20:03:30.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:30 smithi016 bash[77995]: cluster 2023-09-23T20:03:28.698283+0000 mgr.y (mgr.25325) 1689 : cluster 0 pgmap v1761: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-09-23T20:03:30.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:30 smithi016 bash[77995]: audit 2023-09-23T20:03:29.434225+0000 mon.a (mon.0) 2049 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:03:30.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:30 smithi016 bash[75004]: cluster 2023-09-23T20:03:28.698283+0000 mgr.y (mgr.25325) 1689 : cluster 0 pgmap v1761: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-09-23T20:03:30.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:30 smithi016 bash[75004]: audit 2023-09-23T20:03:29.434225+0000 mon.a (mon.0) 2049 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:03:30.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:30 smithi092 bash[58855]: cluster 2023-09-23T20:03:28.698283+0000 mgr.y (mgr.25325) 1689 : cluster 0 pgmap v1761: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-09-23T20:03:30.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:30 smithi092 bash[58855]: audit 2023-09-23T20:03:29.434225+0000 mon.a (mon.0) 2049 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:03:32.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:32 smithi016 bash[75004]: cluster 2023-09-23T20:03:30.699231+0000 mgr.y (mgr.25325) 1690 : cluster 0 pgmap v1762: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:03:32.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:32 smithi016 bash[77995]: cluster 2023-09-23T20:03:30.699231+0000 mgr.y (mgr.25325) 1690 : cluster 0 pgmap v1762: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:03:32.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:32 smithi092 bash[58855]: cluster 2023-09-23T20:03:30.699231+0000 mgr.y (mgr.25325) 1690 : cluster 0 pgmap v1762: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:03:32.804 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:03:33.821 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:03:34.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:34 smithi016 bash[77995]: cluster 2023-09-23T20:03:32.700163+0000 mgr.y (mgr.25325) 1691 : cluster 0 pgmap v1763: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:03:34.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:34 smithi016 bash[75004]: cluster 2023-09-23T20:03:32.700163+0000 mgr.y (mgr.25325) 1691 : cluster 0 pgmap v1763: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:03:34.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:34 smithi092 bash[58855]: cluster 2023-09-23T20:03:32.700163+0000 mgr.y (mgr.25325) 1691 : cluster 0 pgmap v1763: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:03:35.892 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:03:35 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:20:03:35] "GET /metrics HTTP/1.1" 200 34015 "" "Prometheus/2.33.4" 2023-09-23T20:03:36.335 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:03:36.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:36 smithi092 bash[58855]: cluster 2023-09-23T20:03:34.700797+0000 mgr.y (mgr.25325) 1692 : cluster 0 pgmap v1764: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:03:36.861 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:36 smithi016 bash[75004]: cluster 2023-09-23T20:03:34.700797+0000 mgr.y (mgr.25325) 1692 : cluster 0 pgmap v1764: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:03:36.862 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:36 smithi016 bash[77995]: cluster 2023-09-23T20:03:34.700797+0000 mgr.y (mgr.25325) 1692 : cluster 0 pgmap v1764: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:03:38.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:38 smithi016 bash[75004]: cluster 2023-09-23T20:03:36.701296+0000 mgr.y (mgr.25325) 1693 : cluster 0 pgmap v1765: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:03:38.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:38 smithi016 bash[77995]: cluster 2023-09-23T20:03:36.701296+0000 mgr.y (mgr.25325) 1693 : cluster 0 pgmap v1765: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:03:38.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:38 smithi092 bash[58855]: cluster 2023-09-23T20:03:36.701296+0000 mgr.y (mgr.25325) 1693 : cluster 0 pgmap v1765: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:03:39.195 INFO:teuthology.orchestra.run.smithi016.stdout:mon.a: {} 2023-09-23T20:03:39.209 INFO:teuthology.orchestra.run.smithi016.stdout:mon.b: {} 2023-09-23T20:03:39.221 INFO:teuthology.orchestra.run.smithi016.stdout:mon.c: {} 2023-09-23T20:03:39.222 INFO:teuthology.orchestra.run.smithi016.stderr:mon.a: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-09-23T20:03:39.222 INFO:teuthology.orchestra.run.smithi016.stderr:mon.b: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-09-23T20:03:39.222 INFO:teuthology.orchestra.run.smithi016.stderr:mon.c: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-09-23T20:03:39.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:39 smithi016 bash[75004]: audit 2023-09-23T20:03:39.180724+0000 mon.c (mon.2) 256 : audit 0 from='client.? 172.21.15.16:0/2151796618' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T20:03:39.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:39 smithi016 bash[77995]: audit 2023-09-23T20:03:39.180724+0000 mon.c (mon.2) 256 : audit 0 from='client.? 172.21.15.16:0/2151796618' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T20:03:39.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:39 smithi092 bash[58855]: audit 2023-09-23T20:03:39.180724+0000 mon.c (mon.2) 256 : audit 0 from='client.? 172.21.15.16:0/2151796618' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T20:03:40.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:40 smithi016 bash[75004]: cluster 2023-09-23T20:03:38.702107+0000 mgr.y (mgr.25325) 1694 : cluster 0 pgmap v1766: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:03:40.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:40 smithi016 bash[77995]: cluster 2023-09-23T20:03:38.702107+0000 mgr.y (mgr.25325) 1694 : cluster 0 pgmap v1766: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:03:40.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:40 smithi092 bash[58855]: cluster 2023-09-23T20:03:38.702107+0000 mgr.y (mgr.25325) 1694 : cluster 0 pgmap v1766: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:03:41.442 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T20:03:42.478 INFO:teuthology.orchestra.run.smithi016.stderr:nodeep-scrub is unset 2023-09-23T20:03:42.730 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:42 smithi016 bash[75004]: cluster 2023-09-23T20:03:40.703010+0000 mgr.y (mgr.25325) 1695 : cluster 0 pgmap v1767: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:03:42.731 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:42 smithi016 bash[75004]: audit 2023-09-23T20:03:41.957312+0000 mon.c (mon.2) 257 : audit 1 from='client.? 172.21.15.16:0/3480724824' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:03:42.731 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:42 smithi016 bash[75004]: audit 2023-09-23T20:03:41.957875+0000 mon.a (mon.0) 2050 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:03:42.731 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:42 smithi016 bash[77995]: cluster 2023-09-23T20:03:40.703010+0000 mgr.y (mgr.25325) 1695 : cluster 0 pgmap v1767: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:03:42.731 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:42 smithi016 bash[77995]: audit 2023-09-23T20:03:41.957312+0000 mon.c (mon.2) 257 : audit 1 from='client.? 172.21.15.16:0/3480724824' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:03:42.731 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:42 smithi016 bash[77995]: audit 2023-09-23T20:03:41.957875+0000 mon.a (mon.0) 2050 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:03:42.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:42 smithi092 bash[58855]: cluster 2023-09-23T20:03:40.703010+0000 mgr.y (mgr.25325) 1695 : cluster 0 pgmap v1767: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:03:42.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:42 smithi092 bash[58855]: audit 2023-09-23T20:03:41.957312+0000 mon.c (mon.2) 257 : audit 1 from='client.? 172.21.15.16:0/3480724824' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:03:42.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:42 smithi092 bash[58855]: audit 2023-09-23T20:03:41.957875+0000 mon.a (mon.0) 2050 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:03:43.146 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-09-23T20:03:43.147 INFO:teuthology.orchestra.run.smithi016.stdout: "size": 20, 2023-09-23T20:03:43.147 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 600, 2023-09-23T20:03:43.147 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [ 2023-09-23T20:03:43.147 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.147 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46877.0:32655 14.1 14:8db6c3c9:::benchmark_data_smithi016_228083_object32654:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-09-23T20:03:43.147 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:57:32.068031+0000", 2023-09-23T20:03:43.147 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 371.07234957700001, 2023-09-23T20:03:43.147 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.304007572, 2023-09-23T20:03:43.147 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:03:43.147 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:03:43.148 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:03:43.148 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46877", 2023-09-23T20:03:43.148 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:03:43.148 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 32655 2023-09-23T20:03:43.148 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.148 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:03:43.148 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.148 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:03:43.148 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.068031+0000", 2023-09-23T20:03:43.148 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:03:43.148 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.148 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.148 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:03:43.149 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.068031+0000", 2023-09-23T20:03:43.149 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:03:43.149 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.149 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.149 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:03:43.149 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.068030+0000", 2023-09-23T20:03:43.149 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-09-23T20:03:43.149 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.149 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.149 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:03:43.149 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.068051+0000", 2023-09-23T20:03:43.149 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.1004000000000001e-05 2023-09-23T20:03:43.150 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.150 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.150 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:03:43.150 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.068061+0000", 2023-09-23T20:03:43.150 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0108e-05 2023-09-23T20:03:43.150 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.150 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.150 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:03:43.150 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.068067+0000", 2023-09-23T20:03:43.150 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.3029999999999996e-06 2023-09-23T20:03:43.150 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.150 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.150 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:03:43.151 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.068134+0000", 2023-09-23T20:03:43.151 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.6894e-05 2023-09-23T20:03:43.151 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.151 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.151 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:03:43.151 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.068274+0000", 2023-09-23T20:03:43.151 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000139574 2023-09-23T20:03:43.151 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.151 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.151 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:03:43.151 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.068415+0000", 2023-09-23T20:03:43.151 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000141113 2023-09-23T20:03:43.152 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.152 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.152 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T20:03:43.152 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.070063+0000", 2023-09-23T20:03:43.152 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001647548 2023-09-23T20:03:43.152 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.152 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.152 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:43.152 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.070265+0000", 2023-09-23T20:03:43.152 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00020254299999999999 2023-09-23T20:03:43.152 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.152 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.152 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:43.153 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.371937+0000", 2023-09-23T20:03:43.153 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.30167186099999999 2023-09-23T20:03:43.153 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.153 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.153 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:03:43.153 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.371991+0000", 2023-09-23T20:03:43.153 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.3737999999999999e-05 2023-09-23T20:03:43.153 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.153 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.153 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:03:43.153 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.372039+0000", 2023-09-23T20:03:43.153 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.8103999999999999e-05 2023-09-23T20:03:43.153 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:03:43.154 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:03:43.154 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:03:43.154 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.154 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.154 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46877.0:37532 14.3 14:c820bd98:::benchmark_data_smithi016_228083_object37531:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-09-23T20:03:43.154 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:57:36.015427+0000", 2023-09-23T20:03:43.154 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 367.12495360399998, 2023-09-23T20:03:43.154 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.23718687899999999, 2023-09-23T20:03:43.154 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:03:43.154 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:03:43.154 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:03:43.154 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46877", 2023-09-23T20:03:43.155 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:03:43.155 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 37532 2023-09-23T20:03:43.155 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.155 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:03:43.155 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.155 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:03:43.155 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.015427+0000", 2023-09-23T20:03:43.155 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:03:43.155 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.155 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.155 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:03:43.155 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.015427+0000", 2023-09-23T20:03:43.156 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:03:43.156 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.156 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.156 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:03:43.156 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.015425+0000", 2023-09-23T20:03:43.156 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-09-23T20:03:43.156 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.156 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.156 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:03:43.156 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.015463+0000", 2023-09-23T20:03:43.156 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.7901000000000002e-05 2023-09-23T20:03:43.156 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.157 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.157 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:03:43.157 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.015476+0000", 2023-09-23T20:03:43.157 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2842e-05 2023-09-23T20:03:43.157 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.157 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.157 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:03:43.157 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.015489+0000", 2023-09-23T20:03:43.157 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3390000000000001e-05 2023-09-23T20:03:43.157 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.157 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.157 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:03:43.157 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.015542+0000", 2023-09-23T20:03:43.158 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.2621000000000003e-05 2023-09-23T20:03:43.158 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.158 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.158 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:03:43.158 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.015687+0000", 2023-09-23T20:03:43.158 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00014543100000000001 2023-09-23T20:03:43.158 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.158 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.158 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:03:43.158 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.015858+0000", 2023-09-23T20:03:43.158 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00017042399999999999 2023-09-23T20:03:43.158 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.158 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.159 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:43.159 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.243332+0000", 2023-09-23T20:03:43.159 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.22747421800000001 2023-09-23T20:03:43.159 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.159 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.159 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T20:03:43.159 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.244743+0000", 2023-09-23T20:03:43.159 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001410913 2023-09-23T20:03:43.159 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.159 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.159 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:43.159 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.252506+0000", 2023-09-23T20:03:43.159 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0077628469999999998 2023-09-23T20:03:43.160 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.160 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.160 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:03:43.160 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.252563+0000", 2023-09-23T20:03:43.160 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.7045e-05 2023-09-23T20:03:43.160 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.160 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.160 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:03:43.160 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.252614+0000", 2023-09-23T20:03:43.160 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.1193000000000003e-05 2023-09-23T20:03:43.160 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:03:43.160 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:03:43.160 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:03:43.161 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.161 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.161 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46877.0:37533 14.a 14:51168917:::benchmark_data_smithi016_228083_object37532:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-09-23T20:03:43.161 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:57:36.015506+0000", 2023-09-23T20:03:43.161 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 367.124875256, 2023-09-23T20:03:43.161 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.229738686, 2023-09-23T20:03:43.161 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:03:43.161 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:03:43.161 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:03:43.161 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46877", 2023-09-23T20:03:43.161 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:03:43.161 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 37533 2023-09-23T20:03:43.161 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.162 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:03:43.162 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.162 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:03:43.162 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.015506+0000", 2023-09-23T20:03:43.162 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:03:43.162 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.162 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.162 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:03:43.162 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.015506+0000", 2023-09-23T20:03:43.162 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:03:43.162 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.162 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.163 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:03:43.163 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.015505+0000", 2023-09-23T20:03:43.163 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T20:03:43.163 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.163 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.163 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:03:43.163 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.015532+0000", 2023-09-23T20:03:43.163 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.6763000000000001e-05 2023-09-23T20:03:43.163 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.163 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.163 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:03:43.163 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.015540+0000", 2023-09-23T20:03:43.163 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.2239999999999995e-06 2023-09-23T20:03:43.164 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.164 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.164 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:03:43.164 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.015546+0000", 2023-09-23T20:03:43.164 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.0560000000000001e-06 2023-09-23T20:03:43.164 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.164 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.164 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:03:43.164 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.015599+0000", 2023-09-23T20:03:43.164 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.2751999999999999e-05 2023-09-23T20:03:43.164 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.164 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.164 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:03:43.165 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.015729+0000", 2023-09-23T20:03:43.165 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00013033899999999999 2023-09-23T20:03:43.165 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.165 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.165 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 3,5", 2023-09-23T20:03:43.165 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.015858+0000", 2023-09-23T20:03:43.165 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00012875199999999999 2023-09-23T20:03:43.165 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.165 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.165 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:43.165 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.244759+0000", 2023-09-23T20:03:43.165 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.228900669 2023-09-23T20:03:43.165 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.166 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.166 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T20:03:43.166 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.244790+0000", 2023-09-23T20:03:43.166 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.1622000000000003e-05 2023-09-23T20:03:43.166 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.166 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.166 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:43.166 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.244845+0000", 2023-09-23T20:03:43.166 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.4527000000000002e-05 2023-09-23T20:03:43.166 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.166 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.166 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:03:43.167 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.245167+0000", 2023-09-23T20:03:43.167 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00032201199999999999 2023-09-23T20:03:43.167 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.167 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.167 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:03:43.167 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.245244+0000", 2023-09-23T20:03:43.167 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.7542999999999999e-05 2023-09-23T20:03:43.167 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:03:43.167 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:03:43.167 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:03:43.167 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.167 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.167 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.46877.0:37536 14.8 e261/256 14:130018db:::benchmark_data_smithi016_228083_object37535:head v 261'2381, mlcod=261'2379)", 2023-09-23T20:03:43.168 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:57:36.016044+0000", 2023-09-23T20:03:43.168 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 367.12433678600001, 2023-09-23T20:03:43.168 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.229959211, 2023-09-23T20:03:43.168 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:03:43.168 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:03:43.168 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:03:43.168 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.168 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:03:43.168 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.016044+0000", 2023-09-23T20:03:43.168 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:03:43.168 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.168 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.168 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:03:43.169 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.016044+0000", 2023-09-23T20:03:43.169 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:03:43.169 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.169 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.169 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:03:43.169 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.016043+0000", 2023-09-23T20:03:43.169 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T20:03:43.169 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.169 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.169 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:03:43.169 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.016070+0000", 2023-09-23T20:03:43.169 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.6616999999999999e-05 2023-09-23T20:03:43.169 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.170 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.170 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:03:43.170 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.016081+0000", 2023-09-23T20:03:43.170 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1277000000000001e-05 2023-09-23T20:03:43.170 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.170 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.170 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:03:43.170 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.016088+0000", 2023-09-23T20:03:43.170 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.0639999999999996e-06 2023-09-23T20:03:43.170 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.170 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.170 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:03:43.170 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.016136+0000", 2023-09-23T20:03:43.171 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.7893999999999999e-05 2023-09-23T20:03:43.171 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.171 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.171 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:03:43.171 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.016159+0000", 2023-09-23T20:03:43.171 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.2625000000000001e-05 2023-09-23T20:03:43.171 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.171 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.171 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:03:43.171 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.245695+0000", 2023-09-23T20:03:43.171 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.22953663699999999 2023-09-23T20:03:43.171 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.171 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.172 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:03:43.172 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.246003+0000", 2023-09-23T20:03:43.172 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00030801500000000002 2023-09-23T20:03:43.172 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:03:43.172 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:03:43.172 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:03:43.172 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.172 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.172 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop_reply(client.46877.0:37533 14.a e261/256 ondisk, result = 0)", 2023-09-23T20:03:43.172 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:57:36.018175+0000", 2023-09-23T20:03:43.172 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 367.12220583200002, 2023-09-23T20:03:43.172 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.226599404, 2023-09-23T20:03:43.172 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:03:43.173 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "started", 2023-09-23T20:03:43.173 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:03:43.173 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.173 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:03:43.173 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.018175+0000", 2023-09-23T20:03:43.173 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:03:43.173 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.173 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.173 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:03:43.173 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.018175+0000", 2023-09-23T20:03:43.173 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:03:43.173 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.173 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.174 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:03:43.174 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.018174+0000", 2023-09-23T20:03:43.174 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T20:03:43.174 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.174 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.174 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:03:43.174 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.018180+0000", 2023-09-23T20:03:43.174 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.947e-06 2023-09-23T20:03:43.174 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.174 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.174 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:03:43.174 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.242162+0000", 2023-09-23T20:03:43.174 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.22398196200000001 2023-09-23T20:03:43.175 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.175 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.175 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:03:43.175 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.242187+0000", 2023-09-23T20:03:43.175 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.5287999999999999e-05 2023-09-23T20:03:43.175 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.175 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.175 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:03:43.175 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.244730+0000", 2023-09-23T20:03:43.175 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0025429749999999998 2023-09-23T20:03:43.175 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.175 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.176 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:03:43.176 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.244746+0000", 2023-09-23T20:03:43.176 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.5988e-05 2023-09-23T20:03:43.176 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.176 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.176 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:03:43.176 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.244774+0000", 2023-09-23T20:03:43.176 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.8113999999999999e-05 2023-09-23T20:03:43.176 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:03:43.176 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:03:43.176 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:03:43.176 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.176 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.177 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop_reply(client.46877.0:37533 14.a e261/256 ondisk, result = 0)", 2023-09-23T20:03:43.177 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:57:36.018436+0000", 2023-09-23T20:03:43.177 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 367.121945074, 2023-09-23T20:03:43.177 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.226822362, 2023-09-23T20:03:43.177 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:03:43.177 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "started", 2023-09-23T20:03:43.177 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:03:43.177 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.177 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:03:43.177 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.018436+0000", 2023-09-23T20:03:43.177 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:03:43.177 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.178 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.178 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:03:43.178 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.018436+0000", 2023-09-23T20:03:43.178 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:03:43.178 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.178 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.178 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:03:43.178 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.018435+0000", 2023-09-23T20:03:43.178 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T20:03:43.178 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.178 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.178 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:03:43.178 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.018439+0000", 2023-09-23T20:03:43.179 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.6030000000000001e-06 2023-09-23T20:03:43.179 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.179 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.179 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:03:43.179 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.243145+0000", 2023-09-23T20:03:43.179 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.224706238 2023-09-23T20:03:43.179 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.179 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.179 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:03:43.179 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.243158+0000", 2023-09-23T20:03:43.179 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3285000000000001e-05 2023-09-23T20:03:43.179 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.180 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.180 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:03:43.180 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.244824+0000", 2023-09-23T20:03:43.180 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0016655870000000001 2023-09-23T20:03:43.180 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.180 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.180 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:03:43.180 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.244835+0000", 2023-09-23T20:03:43.180 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1452e-05 2023-09-23T20:03:43.180 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.180 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.180 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:03:43.180 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.245258+0000", 2023-09-23T20:03:43.181 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00042266899999999997 2023-09-23T20:03:43.181 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:03:43.181 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:03:43.181 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:03:43.181 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.181 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.181 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46877.0:37541 14.1 14:8e82dad6:::benchmark_data_smithi016_228083_object37540:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-09-23T20:03:43.181 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:57:36.023783+0000", 2023-09-23T20:03:43.181 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 367.11659749099999, 2023-09-23T20:03:43.181 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.22998958999999999, 2023-09-23T20:03:43.181 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:03:43.181 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:03:43.181 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:03:43.182 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46877", 2023-09-23T20:03:43.182 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:03:43.182 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 37541 2023-09-23T20:03:43.182 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.182 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:03:43.182 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.182 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:03:43.182 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.023783+0000", 2023-09-23T20:03:43.182 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:03:43.182 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.182 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.182 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:03:43.182 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.023783+0000", 2023-09-23T20:03:43.183 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:03:43.183 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.183 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.183 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:03:43.183 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.023781+0000", 2023-09-23T20:03:43.183 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999976 2023-09-23T20:03:43.183 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.183 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.183 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:03:43.183 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.023812+0000", 2023-09-23T20:03:43.183 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.0793000000000001e-05 2023-09-23T20:03:43.183 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.183 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.184 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:03:43.184 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.243277+0000", 2023-09-23T20:03:43.184 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.21946533700000001 2023-09-23T20:03:43.184 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.184 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.184 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:03:43.184 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.243290+0000", 2023-09-23T20:03:43.184 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3267e-05 2023-09-23T20:03:43.184 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.184 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.184 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:03:43.184 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.243482+0000", 2023-09-23T20:03:43.184 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00019202299999999999 2023-09-23T20:03:43.185 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.185 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.185 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:03:43.185 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.243689+0000", 2023-09-23T20:03:43.185 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00020658300000000001 2023-09-23T20:03:43.185 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.185 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.185 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:03:43.185 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.243923+0000", 2023-09-23T20:03:43.185 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00023409799999999999 2023-09-23T20:03:43.185 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.185 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.186 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T20:03:43.186 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.250376+0000", 2023-09-23T20:03:43.186 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0064529649999999997 2023-09-23T20:03:43.186 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.186 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.186 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:43.186 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.253628+0000", 2023-09-23T20:03:43.186 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0032516469999999999 2023-09-23T20:03:43.186 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.186 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.186 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:43.187 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.253677+0000", 2023-09-23T20:03:43.187 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.9425999999999997e-05 2023-09-23T20:03:43.187 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.187 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.187 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:03:43.187 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.253729+0000", 2023-09-23T20:03:43.187 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.1613999999999997e-05 2023-09-23T20:03:43.187 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.187 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.187 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:03:43.187 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.253773+0000", 2023-09-23T20:03:43.187 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.4175999999999999e-05 2023-09-23T20:03:43.187 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:03:43.188 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:03:43.188 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:03:43.188 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.188 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.188 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46877.0:38528 14.3 14:cf0a5521:::benchmark_data_smithi016_228083_object38527:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-09-23T20:03:43.188 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:57:36.895018+0000", 2023-09-23T20:03:43.188 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 366.24536318600002, 2023-09-23T20:03:43.188 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.22428244, 2023-09-23T20:03:43.188 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:03:43.188 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:03:43.188 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:03:43.188 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46877", 2023-09-23T20:03:43.188 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:03:43.189 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 38528 2023-09-23T20:03:43.189 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.189 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:03:43.189 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.189 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:03:43.189 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.895018+0000", 2023-09-23T20:03:43.189 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:03:43.189 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.189 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.189 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:03:43.189 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.895018+0000", 2023-09-23T20:03:43.189 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:03:43.189 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.190 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.190 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:03:43.190 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.895017+0000", 2023-09-23T20:03:43.190 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T20:03:43.190 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.190 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.190 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:03:43.190 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.895037+0000", 2023-09-23T20:03:43.190 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.9898000000000002e-05 2023-09-23T20:03:43.190 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.190 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.190 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:03:43.190 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.895047+0000", 2023-09-23T20:03:43.191 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.6299999999999993e-06 2023-09-23T20:03:43.191 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.191 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.191 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:03:43.191 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.895053+0000", 2023-09-23T20:03:43.191 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.7379999999999997e-06 2023-09-23T20:03:43.191 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.191 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.191 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:03:43.191 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.896064+0000", 2023-09-23T20:03:43.191 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001010326 2023-09-23T20:03:43.191 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.191 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.192 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:03:43.192 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.896192+0000", 2023-09-23T20:03:43.192 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00012806300000000001 2023-09-23T20:03:43.192 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.192 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.192 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:03:43.192 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.896440+0000", 2023-09-23T20:03:43.192 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00024817099999999999 2023-09-23T20:03:43.192 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.192 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.192 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:43.192 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.901744+0000", 2023-09-23T20:03:43.193 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0053043999999999999 2023-09-23T20:03:43.193 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.193 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.193 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T20:03:43.193 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.905480+0000", 2023-09-23T20:03:43.193 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0037355420000000001 2023-09-23T20:03:43.193 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.193 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.193 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:43.193 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:37.118349+0000", 2023-09-23T20:03:43.193 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.212869432 2023-09-23T20:03:43.193 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.193 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.194 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:03:43.194 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:37.119227+0000", 2023-09-23T20:03:43.194 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00087753899999999999 2023-09-23T20:03:43.194 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.194 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.194 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:03:43.194 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:37.119300+0000", 2023-09-23T20:03:43.194 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.3373000000000004e-05 2023-09-23T20:03:43.194 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:03:43.194 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:03:43.194 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:03:43.194 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.194 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.195 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46877.0:95179 14.3 14:cca641ee:::benchmark_data_smithi016_228083_object95178:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e264)", 2023-09-23T20:03:43.195 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:58:38.230375+0000", 2023-09-23T20:03:43.195 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 304.91000619099998, 2023-09-23T20:03:43.195 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.23062196900000001, 2023-09-23T20:03:43.195 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:03:43.195 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:03:43.195 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:03:43.195 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46877", 2023-09-23T20:03:43.195 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:03:43.195 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 95179 2023-09-23T20:03:43.195 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.195 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:03:43.196 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.196 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:03:43.196 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:38.230375+0000", 2023-09-23T20:03:43.196 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:03:43.196 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.196 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.196 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:03:43.196 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:38.230375+0000", 2023-09-23T20:03:43.196 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:03:43.196 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.196 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.196 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:03:43.197 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:38.230373+0000", 2023-09-23T20:03:43.197 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-09-23T20:03:43.197 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.197 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.197 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:03:43.197 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:38.230397+0000", 2023-09-23T20:03:43.197 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.3569000000000001e-05 2023-09-23T20:03:43.197 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.197 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.197 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:03:43.197 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:38.230409+0000", 2023-09-23T20:03:43.197 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2058e-05 2023-09-23T20:03:43.198 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.198 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.198 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:03:43.198 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:38.230417+0000", 2023-09-23T20:03:43.198 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.8709999999999997e-06 2023-09-23T20:03:43.198 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.198 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.198 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:03:43.198 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:38.230623+0000", 2023-09-23T20:03:43.198 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000205649 2023-09-23T20:03:43.198 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.198 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.198 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:03:43.199 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:38.230782+0000", 2023-09-23T20:03:43.199 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00015977299999999999 2023-09-23T20:03:43.199 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.199 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.199 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:03:43.199 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:38.230930+0000", 2023-09-23T20:03:43.199 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00014735099999999999 2023-09-23T20:03:43.199 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.199 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.199 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:43.199 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:38.234867+0000", 2023-09-23T20:03:43.199 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0039370170000000001 2023-09-23T20:03:43.199 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.200 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.200 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T20:03:43.200 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:38.234926+0000", 2023-09-23T20:03:43.200 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.9202e-05 2023-09-23T20:03:43.200 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.200 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.200 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:43.200 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:38.460900+0000", 2023-09-23T20:03:43.200 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.225974117 2023-09-23T20:03:43.200 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.200 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.200 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:03:43.200 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:38.460953+0000", 2023-09-23T20:03:43.201 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.3399999999999997e-05 2023-09-23T20:03:43.201 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.201 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.201 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:03:43.201 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:38.460997+0000", 2023-09-23T20:03:43.201 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.3158000000000003e-05 2023-09-23T20:03:43.201 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:03:43.201 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:03:43.201 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:03:43.201 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.201 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.201 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46877.0:131767 14.a 14:50040954:::benchmark_data_smithi016_228083_object31808:head [delete] snapc 0=[] ondisk+write+known_if_redirected e267)", 2023-09-23T20:03:43.202 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:59:29.208730+0000", 2023-09-23T20:03:43.202 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 253.93165079299999, 2023-09-23T20:03:43.202 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.23015328500000001, 2023-09-23T20:03:43.202 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:03:43.202 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:03:43.202 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:03:43.202 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46877", 2023-09-23T20:03:43.202 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:03:43.202 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 131767 2023-09-23T20:03:43.202 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.202 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:03:43.202 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.202 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:03:43.203 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.208730+0000", 2023-09-23T20:03:43.203 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:03:43.203 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.203 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.203 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:03:43.203 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.208730+0000", 2023-09-23T20:03:43.203 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:03:43.203 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.203 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.203 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:03:43.203 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.208728+0000", 2023-09-23T20:03:43.203 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-09-23T20:03:43.203 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.204 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.204 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:03:43.204 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.208735+0000", 2023-09-23T20:03:43.204 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.4599999999999997e-06 2023-09-23T20:03:43.204 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.204 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.204 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:03:43.204 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.208748+0000", 2023-09-23T20:03:43.204 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2765999999999999e-05 2023-09-23T20:03:43.204 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.204 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.204 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:03:43.205 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.208757+0000", 2023-09-23T20:03:43.205 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.4479999999999994e-06 2023-09-23T20:03:43.205 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.205 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.205 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:03:43.205 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.208901+0000", 2023-09-23T20:03:43.205 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00014417499999999999 2023-09-23T20:03:43.205 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.205 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.205 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:03:43.205 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.209018+0000", 2023-09-23T20:03:43.205 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00011719500000000001 2023-09-23T20:03:43.205 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.206 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.206 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 3,5", 2023-09-23T20:03:43.206 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.209141+0000", 2023-09-23T20:03:43.206 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00012313999999999999 2023-09-23T20:03:43.206 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.206 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.206 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T20:03:43.206 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.217135+0000", 2023-09-23T20:03:43.206 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0079940259999999996 2023-09-23T20:03:43.206 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.206 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.206 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:43.207 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.217792+0000", 2023-09-23T20:03:43.207 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00065676199999999997 2023-09-23T20:03:43.207 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.207 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.207 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:43.207 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.228600+0000", 2023-09-23T20:03:43.207 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.010808464 2023-09-23T20:03:43.207 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.207 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.207 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:03:43.207 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.438828+0000", 2023-09-23T20:03:43.207 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.21022755000000001 2023-09-23T20:03:43.208 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.208 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.208 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:03:43.208 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.438883+0000", 2023-09-23T20:03:43.208 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.5417000000000002e-05 2023-09-23T20:03:43.208 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:03:43.208 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:03:43.208 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:03:43.208 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.208 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.208 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46877.0:131770 14.3 14:c6ddca92:::benchmark_data_smithi016_228083_object31811:head [delete] snapc 0=[] ondisk+write+known_if_redirected e267)", 2023-09-23T20:03:43.208 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:59:29.212565+0000", 2023-09-23T20:03:43.208 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 253.92781575399999, 2023-09-23T20:03:43.209 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.22753303499999999, 2023-09-23T20:03:43.209 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:03:43.209 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:03:43.209 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:03:43.209 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46877", 2023-09-23T20:03:43.209 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:03:43.209 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 131770 2023-09-23T20:03:43.209 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.209 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:03:43.209 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.209 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:03:43.209 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.212565+0000", 2023-09-23T20:03:43.210 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:03:43.210 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.210 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.210 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:03:43.210 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.212565+0000", 2023-09-23T20:03:43.210 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:03:43.210 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.210 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.210 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:03:43.210 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.212564+0000", 2023-09-23T20:03:43.210 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-09-23T20:03:43.210 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.210 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.211 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:03:43.211 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.212570+0000", 2023-09-23T20:03:43.211 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.75e-06 2023-09-23T20:03:43.211 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.211 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.211 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:03:43.211 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.212579+0000", 2023-09-23T20:03:43.211 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.8570000000000007e-06 2023-09-23T20:03:43.211 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.211 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.211 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:03:43.211 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.212586+0000", 2023-09-23T20:03:43.211 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.8850000000000001e-06 2023-09-23T20:03:43.212 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.212 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.212 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:03:43.212 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.212752+0000", 2023-09-23T20:03:43.212 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00016594400000000001 2023-09-23T20:03:43.212 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.212 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.212 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:03:43.212 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.212863+0000", 2023-09-23T20:03:43.212 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00011061 2023-09-23T20:03:43.212 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.212 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.213 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:03:43.213 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.212979+0000", 2023-09-23T20:03:43.213 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000116228 2023-09-23T20:03:43.213 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.213 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.213 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T20:03:43.213 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.217288+0000", 2023-09-23T20:03:43.213 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0043092879999999997 2023-09-23T20:03:43.213 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.213 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.213 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:43.213 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.228496+0000", 2023-09-23T20:03:43.214 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.011207299 2023-09-23T20:03:43.214 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.214 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.214 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:43.214 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.440012+0000", 2023-09-23T20:03:43.214 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.211516168 2023-09-23T20:03:43.214 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.214 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.214 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:03:43.214 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.440058+0000", 2023-09-23T20:03:43.214 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.6536000000000003e-05 2023-09-23T20:03:43.214 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.214 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.215 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:03:43.215 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.440098+0000", 2023-09-23T20:03:43.215 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.9733000000000001e-05 2023-09-23T20:03:43.215 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:03:43.215 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:03:43.215 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:03:43.215 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.215 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.215 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46877.0:131777 14.a 14:506c399e:::benchmark_data_smithi016_228083_object31818:head [delete] snapc 0=[] ondisk+write+known_if_redirected e267)", 2023-09-23T20:03:43.215 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:59:29.220345+0000", 2023-09-23T20:03:43.215 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 253.92003564500001, 2023-09-23T20:03:43.215 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.218703649, 2023-09-23T20:03:43.216 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:03:43.216 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:03:43.216 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:03:43.216 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46877", 2023-09-23T20:03:43.216 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:03:43.216 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 131777 2023-09-23T20:03:43.216 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.216 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:03:43.216 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.216 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:03:43.216 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.220345+0000", 2023-09-23T20:03:43.216 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:03:43.217 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.217 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.217 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:03:43.217 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.220345+0000", 2023-09-23T20:03:43.217 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:03:43.217 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.217 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.217 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:03:43.217 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.220344+0000", 2023-09-23T20:03:43.217 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-09-23T20:03:43.217 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.217 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.217 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:03:43.218 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.220363+0000", 2023-09-23T20:03:43.218 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.8870999999999999e-05 2023-09-23T20:03:43.218 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.218 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.218 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:03:43.218 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.220374+0000", 2023-09-23T20:03:43.218 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0638e-05 2023-09-23T20:03:43.218 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.218 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.218 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:03:43.218 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.220381+0000", 2023-09-23T20:03:43.218 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.2710000000000003e-06 2023-09-23T20:03:43.219 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.219 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.219 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:03:43.219 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.220488+0000", 2023-09-23T20:03:43.219 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000107278 2023-09-23T20:03:43.219 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.219 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.219 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:03:43.219 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.220613+0000", 2023-09-23T20:03:43.219 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000124876 2023-09-23T20:03:43.219 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.219 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.219 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 3,5", 2023-09-23T20:03:43.220 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.220729+0000", 2023-09-23T20:03:43.220 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000116518 2023-09-23T20:03:43.220 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.220 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.220 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:43.220 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.228449+0000", 2023-09-23T20:03:43.220 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0077197530000000002 2023-09-23T20:03:43.220 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.220 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.220 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T20:03:43.220 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.438907+0000", 2023-09-23T20:03:43.220 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.210458162 2023-09-23T20:03:43.220 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.221 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.221 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:43.221 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.438966+0000", 2023-09-23T20:03:43.221 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.8217999999999997e-05 2023-09-23T20:03:43.221 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.221 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.221 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:03:43.221 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.439011+0000", 2023-09-23T20:03:43.221 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.5170000000000003e-05 2023-09-23T20:03:43.221 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.221 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.221 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:03:43.221 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.439049+0000", 2023-09-23T20:03:43.222 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.8093999999999998e-05 2023-09-23T20:03:43.222 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:03:43.222 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:03:43.222 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:03:43.222 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.222 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.222 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46877.0:131778 14.3 14:cd91677c:::benchmark_data_smithi016_228083_object31819:head [delete] snapc 0=[] ondisk+write+known_if_redirected e267)", 2023-09-23T20:03:43.222 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:59:29.220544+0000", 2023-09-23T20:03:43.222 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 253.91983688100001, 2023-09-23T20:03:43.222 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.220218846, 2023-09-23T20:03:43.222 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:03:43.222 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:03:43.222 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:03:43.223 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46877", 2023-09-23T20:03:43.223 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:03:43.223 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 131778 2023-09-23T20:03:43.223 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.223 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:03:43.223 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.223 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:03:43.223 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.220544+0000", 2023-09-23T20:03:43.223 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:03:43.223 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.223 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.223 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:03:43.224 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.220544+0000", 2023-09-23T20:03:43.224 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:03:43.224 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.224 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.224 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:03:43.224 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.220543+0000", 2023-09-23T20:03:43.224 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T20:03:43.224 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.224 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.224 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:03:43.224 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.220548+0000", 2023-09-23T20:03:43.224 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.7269999999999998e-06 2023-09-23T20:03:43.225 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.225 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.225 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:03:43.225 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.220557+0000", 2023-09-23T20:03:43.225 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.3810000000000005e-06 2023-09-23T20:03:43.225 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.225 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.225 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:03:43.225 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.220564+0000", 2023-09-23T20:03:43.225 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.5760000000000002e-06 2023-09-23T20:03:43.225 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.225 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.226 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:03:43.226 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.220612+0000", 2023-09-23T20:03:43.226 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.8294999999999998e-05 2023-09-23T20:03:43.226 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.226 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.226 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:03:43.226 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.220720+0000", 2023-09-23T20:03:43.226 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000107633 2023-09-23T20:03:43.226 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.226 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.226 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:03:43.226 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.220830+0000", 2023-09-23T20:03:43.227 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000110672 2023-09-23T20:03:43.227 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.227 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.227 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T20:03:43.227 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.226892+0000", 2023-09-23T20:03:43.227 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0060614800000000002 2023-09-23T20:03:43.227 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.227 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.227 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:43.227 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.440515+0000", 2023-09-23T20:03:43.227 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.21362266299999999 2023-09-23T20:03:43.227 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.227 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.228 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:43.228 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.440553+0000", 2023-09-23T20:03:43.228 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.8176000000000002e-05 2023-09-23T20:03:43.228 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.228 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.228 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:03:43.228 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.440722+0000", 2023-09-23T20:03:43.228 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00016940799999999999 2023-09-23T20:03:43.228 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.228 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.228 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:03:43.228 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.440763+0000", 2023-09-23T20:03:43.229 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.0651000000000001e-05 2023-09-23T20:03:43.229 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:03:43.229 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:03:43.229 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:03:43.229 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.229 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.229 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46877.0:131779 14.1 14:823fe7b5:::benchmark_data_smithi016_228083_object31820:head [delete] snapc 0=[] ondisk+write+known_if_redirected e267)", 2023-09-23T20:03:43.229 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:59:29.227105+0000", 2023-09-23T20:03:43.229 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 253.91327573199999, 2023-09-23T20:03:43.229 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.213056209, 2023-09-23T20:03:43.229 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:03:43.229 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:03:43.229 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:03:43.230 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46877", 2023-09-23T20:03:43.230 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:03:43.230 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 131779 2023-09-23T20:03:43.230 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.230 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:03:43.230 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.230 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:03:43.230 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.227105+0000", 2023-09-23T20:03:43.230 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:03:43.230 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.230 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.230 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:03:43.230 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.227105+0000", 2023-09-23T20:03:43.231 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:03:43.231 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.231 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.231 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:03:43.231 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.227104+0000", 2023-09-23T20:03:43.231 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-09-23T20:03:43.231 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.231 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.231 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:03:43.231 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.227110+0000", 2023-09-23T20:03:43.231 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.6529999999999997e-06 2023-09-23T20:03:43.231 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.231 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.232 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:03:43.232 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.227120+0000", 2023-09-23T20:03:43.232 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.9879999999999999e-06 2023-09-23T20:03:43.232 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.232 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.232 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:03:43.232 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.227126+0000", 2023-09-23T20:03:43.232 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.8919999999999997e-06 2023-09-23T20:03:43.232 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.232 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.232 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:03:43.232 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.227205+0000", 2023-09-23T20:03:43.233 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.8756000000000003e-05 2023-09-23T20:03:43.233 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.233 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.233 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:03:43.233 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.227292+0000", 2023-09-23T20:03:43.233 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.6821000000000001e-05 2023-09-23T20:03:43.233 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.233 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.233 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:03:43.233 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.227366+0000", 2023-09-23T20:03:43.233 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.3658999999999994e-05 2023-09-23T20:03:43.233 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.233 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.234 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:43.234 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.440020+0000", 2023-09-23T20:03:43.234 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.212654486 2023-09-23T20:03:43.234 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.234 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.234 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:43.234 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.440065+0000", 2023-09-23T20:03:43.234 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.5058999999999997e-05 2023-09-23T20:03:43.234 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.234 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.234 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T20:03:43.234 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.440075+0000", 2023-09-23T20:03:43.234 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.5599999999999999e-06 2023-09-23T20:03:43.235 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.235 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.235 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:03:43.235 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.440121+0000", 2023-09-23T20:03:43.235 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.6180000000000002e-05 2023-09-23T20:03:43.235 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.235 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.235 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:03:43.235 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.440161+0000", 2023-09-23T20:03:43.235 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.0487e-05 2023-09-23T20:03:43.235 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:03:43.235 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:03:43.236 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:03:43.236 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.236 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.236 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46877.0:131782 14.a 14:5720411b:::benchmark_data_smithi016_228083_object31823:head [delete] snapc 0=[] ondisk+write+known_if_redirected e267)", 2023-09-23T20:03:43.236 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:59:29.227519+0000", 2023-09-23T20:03:43.236 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 253.91286230099999, 2023-09-23T20:03:43.236 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.242046343, 2023-09-23T20:03:43.236 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:03:43.236 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:03:43.236 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:03:43.236 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46877", 2023-09-23T20:03:43.236 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:03:43.237 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 131782 2023-09-23T20:03:43.237 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.237 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:03:43.237 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.237 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:03:43.237 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.227519+0000", 2023-09-23T20:03:43.237 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:03:43.237 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.237 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.237 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:03:43.237 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.227519+0000", 2023-09-23T20:03:43.237 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:03:43.237 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.238 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.238 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:03:43.238 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.227518+0000", 2023-09-23T20:03:43.238 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T20:03:43.238 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.238 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.238 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:03:43.238 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.227522+0000", 2023-09-23T20:03:43.238 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.481e-06 2023-09-23T20:03:43.238 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.238 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.238 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:03:43.239 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.227530+0000", 2023-09-23T20:03:43.239 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.576e-06 2023-09-23T20:03:43.239 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.239 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.239 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:03:43.239 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.227535+0000", 2023-09-23T20:03:43.239 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.702e-06 2023-09-23T20:03:43.239 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.239 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.239 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:03:43.239 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.440326+0000", 2023-09-23T20:03:43.239 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.212791219 2023-09-23T20:03:43.239 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.240 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.240 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:03:43.240 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.440790+0000", 2023-09-23T20:03:43.240 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00046456900000000002 2023-09-23T20:03:43.240 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.240 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.240 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 3,5", 2023-09-23T20:03:43.240 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.440898+0000", 2023-09-23T20:03:43.240 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000107447 2023-09-23T20:03:43.240 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.240 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.240 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:43.240 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.453351+0000", 2023-09-23T20:03:43.241 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.012453296000000001 2023-09-23T20:03:43.241 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.241 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.241 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:43.241 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.463165+0000", 2023-09-23T20:03:43.241 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0098135679999999999 2023-09-23T20:03:43.241 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.241 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.241 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T20:03:43.241 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.469476+0000", 2023-09-23T20:03:43.241 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0063114 2023-09-23T20:03:43.241 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.241 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.242 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:03:43.242 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.469526+0000", 2023-09-23T20:03:43.242 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.9836000000000002e-05 2023-09-23T20:03:43.242 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.242 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.242 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:03:43.242 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.469565+0000", 2023-09-23T20:03:43.242 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.8937999999999999e-05 2023-09-23T20:03:43.242 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:03:43.242 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:03:43.242 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:03:43.242 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.242 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.243 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.46877.0:131783 14.6 e267/256 14:6617160e:::benchmark_data_smithi016_228083_object31824:head v 267'8242, mlcod=267'8240)", 2023-09-23T20:03:43.243 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:59:29.228671+0000", 2023-09-23T20:03:43.243 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 253.91170946599999, 2023-09-23T20:03:43.243 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.23690403199999999, 2023-09-23T20:03:43.243 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:03:43.243 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:03:43.243 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:03:43.243 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.243 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:03:43.243 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.228671+0000", 2023-09-23T20:03:43.243 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:03:43.243 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.244 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.244 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:03:43.244 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.228671+0000", 2023-09-23T20:03:43.244 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:03:43.244 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.244 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.244 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:03:43.244 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.228671+0000", 2023-09-23T20:03:43.244 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T20:03:43.244 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.244 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.244 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:03:43.244 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.228676+0000", 2023-09-23T20:03:43.245 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.5180000000000004e-06 2023-09-23T20:03:43.245 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.245 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.245 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:03:43.245 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.228686+0000", 2023-09-23T20:03:43.245 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.3190000000000001e-06 2023-09-23T20:03:43.245 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.245 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.245 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:03:43.245 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.228693+0000", 2023-09-23T20:03:43.245 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.0110000000000003e-06 2023-09-23T20:03:43.245 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.246 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.246 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:03:43.246 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.439985+0000", 2023-09-23T20:03:43.246 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.21129251299999999 2023-09-23T20:03:43.246 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.246 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.246 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:03:43.246 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.440053+0000", 2023-09-23T20:03:43.246 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.8131000000000002e-05 2023-09-23T20:03:43.246 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.246 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.247 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:03:43.247 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.465539+0000", 2023-09-23T20:03:43.247 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.025485409000000001 2023-09-23T20:03:43.247 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.247 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.247 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:03:43.247 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:29.465575+0000", 2023-09-23T20:03:43.247 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.6680000000000001e-05 2023-09-23T20:03:43.247 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:03:43.247 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:03:43.247 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:03:43.247 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.247 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.248 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46877.0:177665 14.3 14:cb306a57:::benchmark_data_smithi016_228083_object77706:head [delete] snapc 0=[] ondisk+write+known_if_redirected e271)", 2023-09-23T20:03:43.248 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T20:00:41.484908+0000", 2023-09-23T20:03:43.248 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 181.65547239200001, 2023-09-23T20:03:43.248 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.222479387, 2023-09-23T20:03:43.248 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:03:43.248 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:03:43.248 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:03:43.248 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46877", 2023-09-23T20:03:43.248 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:03:43.248 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 177665 2023-09-23T20:03:43.248 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.248 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:03:43.248 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.249 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:03:43.249 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.484908+0000", 2023-09-23T20:03:43.249 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:03:43.249 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.249 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.249 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:03:43.249 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.484908+0000", 2023-09-23T20:03:43.249 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:03:43.249 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.249 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.249 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:03:43.249 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.484908+0000", 2023-09-23T20:03:43.249 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T20:03:43.250 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.250 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.250 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:03:43.250 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.484911+0000", 2023-09-23T20:03:43.250 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.6690000000000002e-06 2023-09-23T20:03:43.250 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.250 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.250 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:03:43.250 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.484917+0000", 2023-09-23T20:03:43.250 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.1460000000000001e-06 2023-09-23T20:03:43.250 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.250 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.251 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:03:43.251 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.484922+0000", 2023-09-23T20:03:43.251 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.7249999999999997e-06 2023-09-23T20:03:43.251 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.251 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.251 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:03:43.251 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.492494+0000", 2023-09-23T20:03:43.251 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0075719170000000001 2023-09-23T20:03:43.251 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.251 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.251 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:03:43.251 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.492612+0000", 2023-09-23T20:03:43.251 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000118533 2023-09-23T20:03:43.252 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.252 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.252 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:03:43.252 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.492723+0000", 2023-09-23T20:03:43.252 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00011123 2023-09-23T20:03:43.252 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.252 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.252 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:43.252 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.516309+0000", 2023-09-23T20:03:43.252 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.023585677999999999 2023-09-23T20:03:43.252 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.252 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.252 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:43.253 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.693997+0000", 2023-09-23T20:03:43.253 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.17768841499999999 2023-09-23T20:03:43.253 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.253 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.253 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T20:03:43.253 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.707297+0000", 2023-09-23T20:03:43.253 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.013299223000000001 2023-09-23T20:03:43.253 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.253 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.253 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:03:43.253 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.707347+0000", 2023-09-23T20:03:43.253 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.0692999999999997e-05 2023-09-23T20:03:43.254 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.254 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.254 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:03:43.254 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.707388+0000", 2023-09-23T20:03:43.254 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.0537999999999997e-05 2023-09-23T20:03:43.254 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:03:43.254 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:03:43.254 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:03:43.254 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.254 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.254 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46877.0:177678 14.3 14:c68a01f5:::benchmark_data_smithi016_228083_object77719:head [delete] snapc 0=[] ondisk+write+known_if_redirected e271)", 2023-09-23T20:03:43.254 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T20:00:41.496687+0000", 2023-09-23T20:03:43.254 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 181.64369348, 2023-09-23T20:03:43.255 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.21333376900000001, 2023-09-23T20:03:43.255 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:03:43.255 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:03:43.255 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:03:43.255 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46877", 2023-09-23T20:03:43.255 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:03:43.255 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 177678 2023-09-23T20:03:43.255 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.255 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:03:43.255 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.255 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:03:43.255 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.496687+0000", 2023-09-23T20:03:43.256 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:03:43.256 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.256 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.256 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:03:43.256 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.496687+0000", 2023-09-23T20:03:43.256 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:03:43.256 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.256 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.256 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:03:43.256 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.496687+0000", 2023-09-23T20:03:43.256 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T20:03:43.256 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.257 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.257 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:03:43.257 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.496690+0000", 2023-09-23T20:03:43.257 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.6929999999999999e-06 2023-09-23T20:03:43.257 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.257 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.257 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:03:43.257 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.496698+0000", 2023-09-23T20:03:43.257 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.8089999999999993e-06 2023-09-23T20:03:43.257 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.257 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.257 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:03:43.258 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.496703+0000", 2023-09-23T20:03:43.258 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.2730000000000002e-06 2023-09-23T20:03:43.258 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.258 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.258 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:03:43.258 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.694210+0000", 2023-09-23T20:03:43.258 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.19750701000000001 2023-09-23T20:03:43.258 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.258 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.258 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:03:43.258 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.694548+0000", 2023-09-23T20:03:43.258 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000338673 2023-09-23T20:03:43.258 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.259 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.259 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:03:43.259 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.694665+0000", 2023-09-23T20:03:43.259 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000116772 2023-09-23T20:03:43.259 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.259 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.259 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:43.259 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.707428+0000", 2023-09-23T20:03:43.259 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.012762529 2023-09-23T20:03:43.259 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.259 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.259 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T20:03:43.260 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.707483+0000", 2023-09-23T20:03:43.260 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.5325000000000003e-05 2023-09-23T20:03:43.260 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.260 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.260 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:43.260 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.709935+0000", 2023-09-23T20:03:43.260 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0024523700000000002 2023-09-23T20:03:43.260 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.260 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.260 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:03:43.260 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.709981+0000", 2023-09-23T20:03:43.260 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.5527e-05 2023-09-23T20:03:43.260 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.260 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.261 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:03:43.261 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.710021+0000", 2023-09-23T20:03:43.261 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.0148999999999998e-05 2023-09-23T20:03:43.261 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:03:43.261 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:03:43.261 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:03:43.261 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.261 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.261 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.37491.0:29 2.4 2:3949466f:::rbd_data.9273b5e4519e.0000000000000001:head [set-alloc-hint object_size 1048576 write_size 1048576,writefull 0~1048576 [fadvise_nocache]] snapc 0=[] ondisk+write+known_if_redirected e297)", 2023-09-23T20:03:43.261 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T20:02:48.854472+0000", 2023-09-23T20:03:43.261 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 54.285909308000001, 2023-09-23T20:03:43.261 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0055452940000000001, 2023-09-23T20:03:43.262 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:03:43.262 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:03:43.262 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:03:43.262 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.37491", 2023-09-23T20:03:43.262 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2677854898", 2023-09-23T20:03:43.262 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 29 2023-09-23T20:03:43.262 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.262 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:03:43.262 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.262 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:03:43.262 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:48.854472+0000", 2023-09-23T20:03:43.262 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:03:43.263 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.263 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.263 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:03:43.263 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:48.854472+0000", 2023-09-23T20:03:43.263 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:03:43.263 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.263 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.263 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:03:43.263 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:48.854471+0000", 2023-09-23T20:03:43.263 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T20:03:43.263 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.263 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.263 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:03:43.264 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:48.855285+0000", 2023-09-23T20:03:43.264 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00081387899999999999 2023-09-23T20:03:43.264 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.264 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.264 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:03:43.264 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:48.855298+0000", 2023-09-23T20:03:43.264 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3304e-05 2023-09-23T20:03:43.264 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.264 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.264 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:03:43.264 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:48.855304+0000", 2023-09-23T20:03:43.264 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.3609999999999997e-06 2023-09-23T20:03:43.265 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.265 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.265 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:03:43.265 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:48.855370+0000", 2023-09-23T20:03:43.265 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.6157999999999999e-05 2023-09-23T20:03:43.265 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.265 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.265 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:03:43.265 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:48.855548+0000", 2023-09-23T20:03:43.265 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00017772299999999999 2023-09-23T20:03:43.265 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.265 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.265 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:03:43.266 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:48.855784+0000", 2023-09-23T20:03:43.266 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000236009 2023-09-23T20:03:43.266 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.266 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.266 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T20:03:43.266 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:48.858550+0000", 2023-09-23T20:03:43.266 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.002765633 2023-09-23T20:03:43.266 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.266 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.266 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:43.266 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:48.858932+0000", 2023-09-23T20:03:43.266 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00038221000000000002 2023-09-23T20:03:43.267 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.267 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.267 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:43.267 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:48.859898+0000", 2023-09-23T20:03:43.267 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00096641299999999995 2023-09-23T20:03:43.267 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.267 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.267 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:03:43.267 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:48.859968+0000", 2023-09-23T20:03:43.267 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.9937000000000005e-05 2023-09-23T20:03:43.267 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.267 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.267 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:03:43.268 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:48.860017+0000", 2023-09-23T20:03:43.268 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.8483000000000003e-05 2023-09-23T20:03:43.268 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:03:43.268 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:03:43.268 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:03:43.268 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.268 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.268 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.47489.0:81 2.4 2:318d1043:::rbd_object_map.9273b5e4519e:head [call lock.assert_locked in=17b,call rbd.object_map_update in=20b] snapc 0=[] ondisk+write+known_if_redirected+full_try e297)", 2023-09-23T20:03:43.268 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T20:02:49.255667+0000", 2023-09-23T20:03:43.268 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 53.884713406000003, 2023-09-23T20:03:43.268 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0080909789999999999, 2023-09-23T20:03:43.268 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:03:43.268 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:03:43.269 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:03:43.269 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.47489", 2023-09-23T20:03:43.269 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/3995923599", 2023-09-23T20:03:43.269 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 81 2023-09-23T20:03:43.269 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.269 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:03:43.269 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.269 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:03:43.269 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:49.255667+0000", 2023-09-23T20:03:43.269 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:03:43.269 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.269 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.270 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:03:43.270 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:49.255667+0000", 2023-09-23T20:03:43.270 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:03:43.270 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.270 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.270 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:03:43.270 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:49.255666+0000", 2023-09-23T20:03:43.270 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-09-23T20:03:43.270 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.270 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.270 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:03:43.270 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:49.255674+0000", 2023-09-23T20:03:43.270 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.6940000000000001e-06 2023-09-23T20:03:43.271 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.271 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.271 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:03:43.271 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:49.255689+0000", 2023-09-23T20:03:43.271 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.488e-05 2023-09-23T20:03:43.271 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.271 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.271 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:03:43.271 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:49.255698+0000", 2023-09-23T20:03:43.271 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.1279999999999997e-06 2023-09-23T20:03:43.271 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.271 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.272 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:03:43.272 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:49.255753+0000", 2023-09-23T20:03:43.272 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.4973999999999999e-05 2023-09-23T20:03:43.272 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.272 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.272 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:03:43.272 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:49.255885+0000", 2023-09-23T20:03:43.272 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00013208600000000001 2023-09-23T20:03:43.272 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.272 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.272 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:03:43.272 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:49.256172+0000", 2023-09-23T20:03:43.272 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00028688500000000002 2023-09-23T20:03:43.273 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.273 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.273 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:43.273 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:49.257442+0000", 2023-09-23T20:03:43.273 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0012700529999999999 2023-09-23T20:03:43.273 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.273 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.273 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:43.273 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:49.259967+0000", 2023-09-23T20:03:43.273 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0025249249999999999 2023-09-23T20:03:43.273 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.273 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.274 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T20:03:43.274 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:49.263633+0000", 2023-09-23T20:03:43.274 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0036653419999999998 2023-09-23T20:03:43.274 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.274 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.274 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:03:43.274 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:49.263702+0000", 2023-09-23T20:03:43.274 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.9363000000000007e-05 2023-09-23T20:03:43.274 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:03:43.274 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:03:43.274 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:03:43.274 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:49.263758+0000", 2023-09-23T20:03:43.275 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.6443000000000002e-05 2023-09-23T20:03:43.275 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:03:43.275 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:03:43.275 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:03:43.275 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:03:43.275 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:03:43.275 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-09-23T20:03:43.758 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:43 smithi016 bash[75004]: cluster 2023-09-23T20:03:42.461938+0000 mon.a (mon.0) 2051 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:03:43.758 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:43 smithi016 bash[75004]: cluster 2023-09-23T20:03:42.461986+0000 mon.a (mon.0) 2052 : cluster 1 Cluster is now healthy 2023-09-23T20:03:43.758 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:43 smithi016 bash[75004]: audit 2023-09-23T20:03:42.469876+0000 mon.a (mon.0) 2053 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:03:43.758 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:43 smithi016 bash[75004]: cluster 2023-09-23T20:03:42.470013+0000 mon.a (mon.0) 2054 : cluster 0 osdmap e312: 8 total, 8 up, 8 in 2023-09-23T20:03:43.758 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:43 smithi016 bash[75004]: cluster 2023-09-23T20:03:42.703638+0000 mgr.y (mgr.25325) 1696 : cluster 0 pgmap v1769: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:03:43.758 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:43 smithi016 bash[77995]: cluster 2023-09-23T20:03:42.461938+0000 mon.a (mon.0) 2051 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:03:43.758 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:43 smithi016 bash[77995]: cluster 2023-09-23T20:03:42.461986+0000 mon.a (mon.0) 2052 : cluster 1 Cluster is now healthy 2023-09-23T20:03:43.759 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:43 smithi016 bash[77995]: audit 2023-09-23T20:03:42.469876+0000 mon.a (mon.0) 2053 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:03:43.759 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:43 smithi016 bash[77995]: cluster 2023-09-23T20:03:42.470013+0000 mon.a (mon.0) 2054 : cluster 0 osdmap e312: 8 total, 8 up, 8 in 2023-09-23T20:03:43.759 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:43 smithi016 bash[77995]: cluster 2023-09-23T20:03:42.703638+0000 mgr.y (mgr.25325) 1696 : cluster 0 pgmap v1769: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:03:43.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:43 smithi092 bash[58855]: cluster 2023-09-23T20:03:42.461938+0000 mon.a (mon.0) 2051 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:03:43.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:43 smithi092 bash[58855]: cluster 2023-09-23T20:03:42.461986+0000 mon.a (mon.0) 2052 : cluster 1 Cluster is now healthy 2023-09-23T20:03:43.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:43 smithi092 bash[58855]: audit 2023-09-23T20:03:42.469876+0000 mon.a (mon.0) 2053 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:03:43.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:43 smithi092 bash[58855]: cluster 2023-09-23T20:03:42.470013+0000 mon.a (mon.0) 2054 : cluster 0 osdmap e312: 8 total, 8 up, 8 in 2023-09-23T20:03:43.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:43 smithi092 bash[58855]: cluster 2023-09-23T20:03:42.703638+0000 mgr.y (mgr.25325) 1696 : cluster 0 pgmap v1769: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:03:44.573 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T20:03:45.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:44 smithi016 bash[75004]: cluster 2023-09-23T20:03:43.337519+0000 osd.4 (osd.4) 255 : cluster 3 failed to encode map e312 with expected crc 2023-09-23T20:03:45.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:44 smithi016 bash[75004]: cluster 2023-09-23T20:03:43.339925+0000 osd.4 (osd.4) 256 : cluster 3 failed to encode map e312 with expected crc 2023-09-23T20:03:45.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:44 smithi016 bash[75004]: audit 2023-09-23T20:03:43.708364+0000 mon.a (mon.0) 2055 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:03:45.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:44 smithi016 bash[75004]: audit 2023-09-23T20:03:43.709815+0000 mon.a (mon.0) 2056 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:03:45.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:44 smithi016 bash[75004]: audit 2023-09-23T20:03:43.737653+0000 mon.a (mon.0) 2057 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2023-09-23T20:03:45.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:44 smithi016 bash[75004]: audit 2023-09-23T20:03:43.738155+0000 mon.a (mon.0) 2058 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2023-09-23T20:03:45.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:44 smithi016 bash[75004]: audit 2023-09-23T20:03:43.738577+0000 mon.a (mon.0) 2059 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2023-09-23T20:03:45.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:44 smithi016 bash[75004]: cluster 2023-09-23T20:03:43.873559+0000 osd.6 (osd.6) 250 : cluster 3 failed to encode map e312 with expected crc 2023-09-23T20:03:45.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:44 smithi016 bash[75004]: cluster 2023-09-23T20:03:43.873750+0000 osd.7 (osd.7) 232 : cluster 3 failed to encode map e312 with expected crc 2023-09-23T20:03:45.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:44 smithi016 bash[77995]: cluster 2023-09-23T20:03:43.337519+0000 osd.4 (osd.4) 255 : cluster 3 failed to encode map e312 with expected crc 2023-09-23T20:03:45.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:44 smithi016 bash[77995]: cluster 2023-09-23T20:03:43.339925+0000 osd.4 (osd.4) 256 : cluster 3 failed to encode map e312 with expected crc 2023-09-23T20:03:45.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:44 smithi016 bash[77995]: audit 2023-09-23T20:03:43.708364+0000 mon.a (mon.0) 2055 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:03:45.037 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:44 smithi016 bash[77995]: audit 2023-09-23T20:03:43.709815+0000 mon.a (mon.0) 2056 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:03:45.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:44 smithi016 bash[77995]: audit 2023-09-23T20:03:43.737653+0000 mon.a (mon.0) 2057 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2023-09-23T20:03:45.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:44 smithi016 bash[77995]: audit 2023-09-23T20:03:43.738155+0000 mon.a (mon.0) 2058 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2023-09-23T20:03:45.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:44 smithi016 bash[77995]: audit 2023-09-23T20:03:43.738577+0000 mon.a (mon.0) 2059 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2023-09-23T20:03:45.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:44 smithi016 bash[77995]: cluster 2023-09-23T20:03:43.873559+0000 osd.6 (osd.6) 250 : cluster 3 failed to encode map e312 with expected crc 2023-09-23T20:03:45.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:44 smithi016 bash[77995]: cluster 2023-09-23T20:03:43.873750+0000 osd.7 (osd.7) 232 : cluster 3 failed to encode map e312 with expected crc 2023-09-23T20:03:45.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:44 smithi092 bash[58855]: cluster 2023-09-23T20:03:43.337519+0000 osd.4 (osd.4) 255 : cluster 3 failed to encode map e312 with expected crc 2023-09-23T20:03:45.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:44 smithi092 bash[58855]: cluster 2023-09-23T20:03:43.339925+0000 osd.4 (osd.4) 256 : cluster 3 failed to encode map e312 with expected crc 2023-09-23T20:03:45.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:44 smithi092 bash[58855]: audit 2023-09-23T20:03:43.708364+0000 mon.a (mon.0) 2055 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:03:45.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:44 smithi092 bash[58855]: audit 2023-09-23T20:03:43.709815+0000 mon.a (mon.0) 2056 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:03:45.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:44 smithi092 bash[58855]: audit 2023-09-23T20:03:43.737653+0000 mon.a (mon.0) 2057 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2023-09-23T20:03:45.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:44 smithi092 bash[58855]: audit 2023-09-23T20:03:43.738155+0000 mon.a (mon.0) 2058 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2023-09-23T20:03:45.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:44 smithi092 bash[58855]: audit 2023-09-23T20:03:43.738577+0000 mon.a (mon.0) 2059 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2023-09-23T20:03:45.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:44 smithi092 bash[58855]: cluster 2023-09-23T20:03:43.873559+0000 osd.6 (osd.6) 250 : cluster 3 failed to encode map e312 with expected crc 2023-09-23T20:03:45.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:44 smithi092 bash[58855]: cluster 2023-09-23T20:03:43.873750+0000 osd.7 (osd.7) 232 : cluster 3 failed to encode map e312 with expected crc 2023-09-23T20:03:46.036 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:03:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:20:03:45] "GET /metrics HTTP/1.1" 200 34016 "" "Prometheus/2.33.4" 2023-09-23T20:03:46.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:45 smithi016 bash[75004]: cluster 2023-09-23T20:03:43.873766+0000 osd.5 (osd.5) 231 : cluster 3 failed to encode map e312 with expected crc 2023-09-23T20:03:46.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:45 smithi016 bash[75004]: cluster 2023-09-23T20:03:44.704279+0000 mgr.y (mgr.25325) 1697 : cluster 0 pgmap v1770: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:03:46.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:45 smithi016 bash[75004]: audit 2023-09-23T20:03:44.716659+0000 mon.a (mon.0) 2060 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2023-09-23T20:03:46.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:45 smithi016 bash[75004]: audit 2023-09-23T20:03:44.716777+0000 mon.a (mon.0) 2061 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2023-09-23T20:03:46.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:45 smithi016 bash[75004]: audit 2023-09-23T20:03:44.716866+0000 mon.a (mon.0) 2062 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2023-09-23T20:03:46.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:45 smithi016 bash[75004]: cluster 2023-09-23T20:03:44.716940+0000 mon.a (mon.0) 2063 : cluster 0 osdmap e313: 8 total, 8 up, 8 in 2023-09-23T20:03:46.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:45 smithi016 bash[75004]: cluster 2023-09-23T20:03:44.722660+0000 osd.4 (osd.4) 257 : cluster 3 failed to encode map e313 with expected crc 2023-09-23T20:03:46.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:45 smithi016 bash[75004]: cluster 2023-09-23T20:03:44.724556+0000 osd.7 (osd.7) 233 : cluster 3 failed to encode map e313 with expected crc 2023-09-23T20:03:46.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:45 smithi016 bash[75004]: cluster 2023-09-23T20:03:44.725423+0000 osd.4 (osd.4) 258 : cluster 3 failed to encode map e313 with expected crc 2023-09-23T20:03:46.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:45 smithi016 bash[77995]: cluster 2023-09-23T20:03:43.873766+0000 osd.5 (osd.5) 231 : cluster 3 failed to encode map e312 with expected crc 2023-09-23T20:03:46.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:45 smithi016 bash[77995]: cluster 2023-09-23T20:03:44.704279+0000 mgr.y (mgr.25325) 1697 : cluster 0 pgmap v1770: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:03:46.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:45 smithi016 bash[77995]: audit 2023-09-23T20:03:44.716659+0000 mon.a (mon.0) 2060 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2023-09-23T20:03:46.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:45 smithi016 bash[77995]: audit 2023-09-23T20:03:44.716777+0000 mon.a (mon.0) 2061 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2023-09-23T20:03:46.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:45 smithi016 bash[77995]: audit 2023-09-23T20:03:44.716866+0000 mon.a (mon.0) 2062 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2023-09-23T20:03:46.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:45 smithi016 bash[77995]: cluster 2023-09-23T20:03:44.716940+0000 mon.a (mon.0) 2063 : cluster 0 osdmap e313: 8 total, 8 up, 8 in 2023-09-23T20:03:46.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:45 smithi016 bash[77995]: cluster 2023-09-23T20:03:44.722660+0000 osd.4 (osd.4) 257 : cluster 3 failed to encode map e313 with expected crc 2023-09-23T20:03:46.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:45 smithi016 bash[77995]: cluster 2023-09-23T20:03:44.724556+0000 osd.7 (osd.7) 233 : cluster 3 failed to encode map e313 with expected crc 2023-09-23T20:03:46.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:45 smithi016 bash[77995]: cluster 2023-09-23T20:03:44.725423+0000 osd.4 (osd.4) 258 : cluster 3 failed to encode map e313 with expected crc 2023-09-23T20:03:46.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:45 smithi092 bash[58855]: cluster 2023-09-23T20:03:43.873766+0000 osd.5 (osd.5) 231 : cluster 3 failed to encode map e312 with expected crc 2023-09-23T20:03:46.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:45 smithi092 bash[58855]: cluster 2023-09-23T20:03:44.704279+0000 mgr.y (mgr.25325) 1697 : cluster 0 pgmap v1770: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:03:46.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:45 smithi092 bash[58855]: audit 2023-09-23T20:03:44.716659+0000 mon.a (mon.0) 2060 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2023-09-23T20:03:46.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:45 smithi092 bash[58855]: audit 2023-09-23T20:03:44.716777+0000 mon.a (mon.0) 2061 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2023-09-23T20:03:46.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:45 smithi092 bash[58855]: audit 2023-09-23T20:03:44.716866+0000 mon.a (mon.0) 2062 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2023-09-23T20:03:46.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:45 smithi092 bash[58855]: cluster 2023-09-23T20:03:44.716940+0000 mon.a (mon.0) 2063 : cluster 0 osdmap e313: 8 total, 8 up, 8 in 2023-09-23T20:03:46.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:45 smithi092 bash[58855]: cluster 2023-09-23T20:03:44.722660+0000 osd.4 (osd.4) 257 : cluster 3 failed to encode map e313 with expected crc 2023-09-23T20:03:46.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:45 smithi092 bash[58855]: cluster 2023-09-23T20:03:44.724556+0000 osd.7 (osd.7) 233 : cluster 3 failed to encode map e313 with expected crc 2023-09-23T20:03:46.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:45 smithi092 bash[58855]: cluster 2023-09-23T20:03:44.725423+0000 osd.4 (osd.4) 258 : cluster 3 failed to encode map e313 with expected crc 2023-09-23T20:03:47.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:46 smithi016 bash[75004]: cluster 2023-09-23T20:03:45.642124+0000 osd.6 (osd.6) 251 : cluster 3 failed to encode map e313 with expected crc 2023-09-23T20:03:47.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:46 smithi016 bash[75004]: cluster 2023-09-23T20:03:45.725867+0000 mon.a (mon.0) 2064 : cluster 0 osdmap e314: 8 total, 8 up, 8 in 2023-09-23T20:03:47.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:46 smithi016 bash[75004]: cluster 2023-09-23T20:03:45.727736+0000 osd.4 (osd.4) 259 : cluster 3 failed to encode map e314 with expected crc 2023-09-23T20:03:47.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:46 smithi016 bash[75004]: cluster 2023-09-23T20:03:45.730512+0000 osd.4 (osd.4) 260 : cluster 3 failed to encode map e314 with expected crc 2023-09-23T20:03:47.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:46 smithi016 bash[75004]: cluster 2023-09-23T20:03:45.733041+0000 osd.4 (osd.4) 261 : cluster 3 failed to encode map e314 with expected crc 2023-09-23T20:03:47.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:46 smithi016 bash[75004]: cluster 2023-09-23T20:03:45.927419+0000 osd.7 (osd.7) 234 : cluster 3 failed to encode map e314 with expected crc 2023-09-23T20:03:47.037 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:46 smithi016 bash[75004]: cluster 2023-09-23T20:03:45.927497+0000 osd.6 (osd.6) 252 : cluster 3 failed to encode map e314 with expected crc 2023-09-23T20:03:47.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:46 smithi016 bash[77995]: cluster 2023-09-23T20:03:45.642124+0000 osd.6 (osd.6) 251 : cluster 3 failed to encode map e313 with expected crc 2023-09-23T20:03:47.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:46 smithi016 bash[77995]: cluster 2023-09-23T20:03:45.725867+0000 mon.a (mon.0) 2064 : cluster 0 osdmap e314: 8 total, 8 up, 8 in 2023-09-23T20:03:47.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:46 smithi016 bash[77995]: cluster 2023-09-23T20:03:45.727736+0000 osd.4 (osd.4) 259 : cluster 3 failed to encode map e314 with expected crc 2023-09-23T20:03:47.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:46 smithi016 bash[77995]: cluster 2023-09-23T20:03:45.730512+0000 osd.4 (osd.4) 260 : cluster 3 failed to encode map e314 with expected crc 2023-09-23T20:03:47.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:46 smithi016 bash[77995]: cluster 2023-09-23T20:03:45.733041+0000 osd.4 (osd.4) 261 : cluster 3 failed to encode map e314 with expected crc 2023-09-23T20:03:47.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:46 smithi016 bash[77995]: cluster 2023-09-23T20:03:45.927419+0000 osd.7 (osd.7) 234 : cluster 3 failed to encode map e314 with expected crc 2023-09-23T20:03:47.038 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:46 smithi016 bash[77995]: cluster 2023-09-23T20:03:45.927497+0000 osd.6 (osd.6) 252 : cluster 3 failed to encode map e314 with expected crc 2023-09-23T20:03:47.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:46 smithi092 bash[58855]: cluster 2023-09-23T20:03:45.642124+0000 osd.6 (osd.6) 251 : cluster 3 failed to encode map e313 with expected crc 2023-09-23T20:03:47.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:46 smithi092 bash[58855]: cluster 2023-09-23T20:03:45.725867+0000 mon.a (mon.0) 2064 : cluster 0 osdmap e314: 8 total, 8 up, 8 in 2023-09-23T20:03:47.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:46 smithi092 bash[58855]: cluster 2023-09-23T20:03:45.727736+0000 osd.4 (osd.4) 259 : cluster 3 failed to encode map e314 with expected crc 2023-09-23T20:03:47.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:46 smithi092 bash[58855]: cluster 2023-09-23T20:03:45.730512+0000 osd.4 (osd.4) 260 : cluster 3 failed to encode map e314 with expected crc 2023-09-23T20:03:47.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:46 smithi092 bash[58855]: cluster 2023-09-23T20:03:45.733041+0000 osd.4 (osd.4) 261 : cluster 3 failed to encode map e314 with expected crc 2023-09-23T20:03:47.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:46 smithi092 bash[58855]: cluster 2023-09-23T20:03:45.927419+0000 osd.7 (osd.7) 234 : cluster 3 failed to encode map e314 with expected crc 2023-09-23T20:03:47.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:46 smithi092 bash[58855]: cluster 2023-09-23T20:03:45.927497+0000 osd.6 (osd.6) 252 : cluster 3 failed to encode map e314 with expected crc 2023-09-23T20:03:47.250 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:03:48.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:47 smithi016 bash[77995]: cluster 2023-09-23T20:03:45.926819+0000 osd.5 (osd.5) 232 : cluster 3 failed to encode map e314 with expected crc 2023-09-23T20:03:48.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:47 smithi016 bash[77995]: cluster 2023-09-23T20:03:46.704876+0000 mgr.y (mgr.25325) 1698 : cluster 0 pgmap v1773: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:03:48.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:47 smithi016 bash[75004]: cluster 2023-09-23T20:03:45.926819+0000 osd.5 (osd.5) 232 : cluster 3 failed to encode map e314 with expected crc 2023-09-23T20:03:48.036 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:47 smithi016 bash[75004]: cluster 2023-09-23T20:03:46.704876+0000 mgr.y (mgr.25325) 1698 : cluster 0 pgmap v1773: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:03:48.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:47 smithi092 bash[58855]: cluster 2023-09-23T20:03:45.926819+0000 osd.5 (osd.5) 232 : cluster 3 failed to encode map e314 with expected crc 2023-09-23T20:03:48.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:47 smithi092 bash[58855]: cluster 2023-09-23T20:03:46.704876+0000 mgr.y (mgr.25325) 1698 : cluster 0 pgmap v1773: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:03:49.918 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:03:50.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:49 smithi016 bash[75004]: cluster 2023-09-23T20:03:48.705505+0000 mgr.y (mgr.25325) 1699 : cluster 0 pgmap v1774: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:03:50.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:49 smithi016 bash[77995]: cluster 2023-09-23T20:03:48.705505+0000 mgr.y (mgr.25325) 1699 : cluster 0 pgmap v1774: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:03:50.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:49 smithi092 bash[58855]: cluster 2023-09-23T20:03:48.705505+0000 mgr.y (mgr.25325) 1699 : cluster 0 pgmap v1774: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:03:51.882 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T20:03:51.907 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:51 smithi016 bash[75004]: cluster 2023-09-23T20:03:50.706238+0000 mgr.y (mgr.25325) 1700 : cluster 0 pgmap v1775: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:03:51.908 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:51 smithi016 bash[75004]: audit 2023-09-23T20:03:51.731011+0000 mon.a (mon.0) 2065 : audit 1 from='client.? 172.21.15.16:0/1970891798' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:03:51.908 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:51 smithi016 bash[77995]: cluster 2023-09-23T20:03:50.706238+0000 mgr.y (mgr.25325) 1700 : cluster 0 pgmap v1775: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:03:51.908 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:51 smithi016 bash[77995]: audit 2023-09-23T20:03:51.731011+0000 mon.a (mon.0) 2065 : audit 1 from='client.? 172.21.15.16:0/1970891798' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:03:52.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:51 smithi092 bash[58855]: cluster 2023-09-23T20:03:50.706238+0000 mgr.y (mgr.25325) 1700 : cluster 0 pgmap v1775: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:03:52.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:51 smithi092 bash[58855]: audit 2023-09-23T20:03:51.731011+0000 mon.a (mon.0) 2065 : audit 1 from='client.? 172.21.15.16:0/1970891798' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:03:53.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:52 smithi016 bash[77995]: audit 2023-09-23T20:03:51.874959+0000 mon.a (mon.0) 2066 : audit 1 from='client.? 172.21.15.16:0/1970891798' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:03:53.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:52 smithi016 bash[77995]: cluster 2023-09-23T20:03:51.875042+0000 mon.a (mon.0) 2067 : cluster 0 osdmap e315: 8 total, 8 up, 8 in 2023-09-23T20:03:53.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:52 smithi016 bash[77995]: cluster 2023-09-23T20:03:51.875796+0000 osd.4 (osd.4) 262 : cluster 3 failed to encode map e315 with expected crc 2023-09-23T20:03:53.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:52 smithi016 bash[77995]: audit 2023-09-23T20:03:51.896344+0000 mon.a (mon.0) 2068 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:03:53.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:52 smithi016 bash[77995]: cluster 2023-09-23T20:03:52.329165+0000 osd.5 (osd.5) 233 : cluster 3 failed to encode map e315 with expected crc 2023-09-23T20:03:53.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:52 smithi016 bash[75004]: audit 2023-09-23T20:03:51.874959+0000 mon.a (mon.0) 2066 : audit 1 from='client.? 172.21.15.16:0/1970891798' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:03:53.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:52 smithi016 bash[75004]: cluster 2023-09-23T20:03:51.875042+0000 mon.a (mon.0) 2067 : cluster 0 osdmap e315: 8 total, 8 up, 8 in 2023-09-23T20:03:53.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:52 smithi016 bash[75004]: cluster 2023-09-23T20:03:51.875796+0000 osd.4 (osd.4) 262 : cluster 3 failed to encode map e315 with expected crc 2023-09-23T20:03:53.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:52 smithi016 bash[75004]: audit 2023-09-23T20:03:51.896344+0000 mon.a (mon.0) 2068 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:03:53.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:52 smithi016 bash[75004]: cluster 2023-09-23T20:03:52.329165+0000 osd.5 (osd.5) 233 : cluster 3 failed to encode map e315 with expected crc 2023-09-23T20:03:53.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:52 smithi092 bash[58855]: audit 2023-09-23T20:03:51.874959+0000 mon.a (mon.0) 2066 : audit 1 from='client.? 172.21.15.16:0/1970891798' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:03:53.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:52 smithi092 bash[58855]: cluster 2023-09-23T20:03:51.875042+0000 mon.a (mon.0) 2067 : cluster 0 osdmap e315: 8 total, 8 up, 8 in 2023-09-23T20:03:53.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:52 smithi092 bash[58855]: cluster 2023-09-23T20:03:51.875796+0000 osd.4 (osd.4) 262 : cluster 3 failed to encode map e315 with expected crc 2023-09-23T20:03:53.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:52 smithi092 bash[58855]: audit 2023-09-23T20:03:51.896344+0000 mon.a (mon.0) 2068 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:03:53.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:52 smithi092 bash[58855]: cluster 2023-09-23T20:03:52.329165+0000 osd.5 (osd.5) 233 : cluster 3 failed to encode map e315 with expected crc 2023-09-23T20:03:53.677 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 0 1 2023-09-23T20:03:54.004 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:53 smithi016 bash[77995]: cluster 2023-09-23T20:03:52.328544+0000 osd.6 (osd.6) 253 : cluster 3 failed to encode map e315 with expected crc 2023-09-23T20:03:54.005 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:53 smithi016 bash[77995]: cluster 2023-09-23T20:03:52.329184+0000 osd.7 (osd.7) 235 : cluster 3 failed to encode map e315 with expected crc 2023-09-23T20:03:54.005 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:53 smithi016 bash[77995]: cluster 2023-09-23T20:03:52.706781+0000 mgr.y (mgr.25325) 1701 : cluster 0 pgmap v1777: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:03:54.005 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:53 smithi016 bash[75004]: cluster 2023-09-23T20:03:52.328544+0000 osd.6 (osd.6) 253 : cluster 3 failed to encode map e315 with expected crc 2023-09-23T20:03:54.005 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:53 smithi016 bash[75004]: cluster 2023-09-23T20:03:52.329184+0000 osd.7 (osd.7) 235 : cluster 3 failed to encode map e315 with expected crc 2023-09-23T20:03:54.005 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:53 smithi016 bash[75004]: cluster 2023-09-23T20:03:52.706781+0000 mgr.y (mgr.25325) 1701 : cluster 0 pgmap v1777: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:03:54.187 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-09-23T20:03:54.187 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [], 2023-09-23T20:03:54.187 INFO:teuthology.orchestra.run.smithi016.stdout: "num_ops": 0 2023-09-23T20:03:54.187 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-09-23T20:03:54.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:53 smithi092 bash[58855]: cluster 2023-09-23T20:03:52.328544+0000 osd.6 (osd.6) 253 : cluster 3 failed to encode map e315 with expected crc 2023-09-23T20:03:54.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:53 smithi092 bash[58855]: cluster 2023-09-23T20:03:52.329184+0000 osd.7 (osd.7) 235 : cluster 3 failed to encode map e315 with expected crc 2023-09-23T20:03:54.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:53 smithi092 bash[58855]: cluster 2023-09-23T20:03:52.706781+0000 mgr.y (mgr.25325) 1701 : cluster 0 pgmap v1777: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:03:55.222 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T20:03:56.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:55 smithi016 bash[75004]: cluster 2023-09-23T20:03:54.707227+0000 mgr.y (mgr.25325) 1702 : cluster 0 pgmap v1778: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:03:56.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:55 smithi016 bash[77995]: cluster 2023-09-23T20:03:54.707227+0000 mgr.y (mgr.25325) 1702 : cluster 0 pgmap v1778: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:03:56.036 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:03:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:20:03:55] "GET /metrics HTTP/1.1" 200 34016 "" "Prometheus/2.33.4" 2023-09-23T20:03:56.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:55 smithi092 bash[58855]: cluster 2023-09-23T20:03:54.707227+0000 mgr.y (mgr.25325) 1702 : cluster 0 pgmap v1778: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:03:58.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:57 smithi016 bash[77995]: cluster 2023-09-23T20:03:56.708050+0000 mgr.y (mgr.25325) 1703 : cluster 0 pgmap v1779: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:03:58.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:57 smithi016 bash[75004]: cluster 2023-09-23T20:03:56.708050+0000 mgr.y (mgr.25325) 1703 : cluster 0 pgmap v1779: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:03:58.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:57 smithi092 bash[58855]: cluster 2023-09-23T20:03:56.708050+0000 mgr.y (mgr.25325) 1703 : cluster 0 pgmap v1779: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:03:58.779 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:03:59.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:59 smithi016 bash[77995]: audit 2023-09-23T20:03:58.090790+0000 mon.a (mon.0) 2069 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:03:59.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:59 smithi016 bash[77995]: audit 2023-09-23T20:03:58.099926+0000 mon.a (mon.0) 2070 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:03:59.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:59 smithi016 bash[77995]: audit 2023-09-23T20:03:58.702923+0000 mon.a (mon.0) 2071 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:03:59.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:59 smithi016 bash[77995]: audit 2023-09-23T20:03:59.002356+0000 mon.a (mon.0) 2072 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:03:59.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:03:59 smithi016 bash[77995]: audit 2023-09-23T20:03:59.008850+0000 mon.a (mon.0) 2073 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:03:59.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:59 smithi016 bash[75004]: audit 2023-09-23T20:03:58.090790+0000 mon.a (mon.0) 2069 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:03:59.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:59 smithi016 bash[75004]: audit 2023-09-23T20:03:58.099926+0000 mon.a (mon.0) 2070 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:03:59.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:59 smithi016 bash[75004]: audit 2023-09-23T20:03:58.702923+0000 mon.a (mon.0) 2071 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:03:59.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:59 smithi016 bash[75004]: audit 2023-09-23T20:03:59.002356+0000 mon.a (mon.0) 2072 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:03:59.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:03:59 smithi016 bash[75004]: audit 2023-09-23T20:03:59.008850+0000 mon.a (mon.0) 2073 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:03:59.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:59 smithi092 bash[58855]: audit 2023-09-23T20:03:58.090790+0000 mon.a (mon.0) 2069 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:03:59.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:59 smithi092 bash[58855]: audit 2023-09-23T20:03:58.099926+0000 mon.a (mon.0) 2070 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:03:59.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:59 smithi092 bash[58855]: audit 2023-09-23T20:03:58.702923+0000 mon.a (mon.0) 2071 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:03:59.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:59 smithi092 bash[58855]: audit 2023-09-23T20:03:59.002356+0000 mon.a (mon.0) 2072 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:03:59.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:03:59 smithi092 bash[58855]: audit 2023-09-23T20:03:59.008850+0000 mon.a (mon.0) 2073 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:03:59.763 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:04:00.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:00 smithi016 bash[77995]: cluster 2023-09-23T20:03:58.710767+0000 mgr.y (mgr.25325) 1704 : cluster 0 pgmap v1780: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:00.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:00 smithi016 bash[77995]: audit 2023-09-23T20:03:59.462748+0000 mon.a (mon.0) 2074 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:04:00.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:00 smithi016 bash[77995]: audit 2023-09-23T20:03:59.480636+0000 mon.a (mon.0) 2075 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:04:00.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:00 smithi016 bash[77995]: audit 2023-09-23T20:03:59.482097+0000 mon.a (mon.0) 2076 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:04:00.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:00 smithi016 bash[77995]: audit 2023-09-23T20:03:59.491129+0000 mon.a (mon.0) 2077 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:04:00.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:00 smithi016 bash[75004]: cluster 2023-09-23T20:03:58.710767+0000 mgr.y (mgr.25325) 1704 : cluster 0 pgmap v1780: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:00.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:00 smithi016 bash[75004]: audit 2023-09-23T20:03:59.462748+0000 mon.a (mon.0) 2074 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:04:00.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:00 smithi016 bash[75004]: audit 2023-09-23T20:03:59.480636+0000 mon.a (mon.0) 2075 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:04:00.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:00 smithi016 bash[75004]: audit 2023-09-23T20:03:59.482097+0000 mon.a (mon.0) 2076 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:04:00.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:00 smithi016 bash[75004]: audit 2023-09-23T20:03:59.491129+0000 mon.a (mon.0) 2077 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:04:00.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:00 smithi092 bash[58855]: cluster 2023-09-23T20:03:58.710767+0000 mgr.y (mgr.25325) 1704 : cluster 0 pgmap v1780: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:00.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:00 smithi092 bash[58855]: audit 2023-09-23T20:03:59.462748+0000 mon.a (mon.0) 2074 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:04:00.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:00 smithi092 bash[58855]: audit 2023-09-23T20:03:59.480636+0000 mon.a (mon.0) 2075 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:04:00.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:00 smithi092 bash[58855]: audit 2023-09-23T20:03:59.482097+0000 mon.a (mon.0) 2076 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:04:00.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:00 smithi092 bash[58855]: audit 2023-09-23T20:03:59.491129+0000 mon.a (mon.0) 2077 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:04:02.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:02 smithi016 bash[75004]: cluster 2023-09-23T20:04:00.711374+0000 mgr.y (mgr.25325) 1705 : cluster 0 pgmap v1781: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:02.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:02 smithi016 bash[77995]: cluster 2023-09-23T20:04:00.711374+0000 mgr.y (mgr.25325) 1705 : cluster 0 pgmap v1781: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:02.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:02 smithi092 bash[58855]: cluster 2023-09-23T20:04:00.711374+0000 mgr.y (mgr.25325) 1705 : cluster 0 pgmap v1781: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:04.558 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-09-23T20:04:04.559 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [], 2023-09-23T20:04:04.559 INFO:teuthology.orchestra.run.smithi016.stdout: "complaint_time": 30, 2023-09-23T20:04:04.559 INFO:teuthology.orchestra.run.smithi016.stdout: "num_blocked_ops": 0 2023-09-23T20:04:04.559 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-09-23T20:04:04.773 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:04 smithi016 bash[75004]: cluster 2023-09-23T20:04:02.711858+0000 mgr.y (mgr.25325) 1706 : cluster 0 pgmap v1782: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:04.773 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:04 smithi016 bash[77995]: cluster 2023-09-23T20:04:02.711858+0000 mgr.y (mgr.25325) 1706 : cluster 0 pgmap v1782: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:04.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:04 smithi092 bash[58855]: cluster 2023-09-23T20:04:02.711858+0000 mgr.y (mgr.25325) 1706 : cluster 0 pgmap v1782: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:05.491 INFO:teuthology.orchestra.run.smithi016.stderr:set osd.0 primary-affinity to 1 (8655362) 2023-09-23T20:04:05.509 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T20:04:05.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:05 smithi016 bash[75004]: audit 2023-09-23T20:04:04.651882+0000 mon.a (mon.0) 2078 : audit 1 from='client.? 172.21.15.16:0/2386805258' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:04:05.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:05 smithi016 bash[77995]: audit 2023-09-23T20:04:04.651882+0000 mon.a (mon.0) 2078 : audit 1 from='client.? 172.21.15.16:0/2386805258' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:04:05.786 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:04:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:20:04:05] "GET /metrics HTTP/1.1" 200 34014 "" "Prometheus/2.33.4" 2023-09-23T20:04:05.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:05 smithi092 bash[58855]: audit 2023-09-23T20:04:04.651882+0000 mon.a (mon.0) 2078 : audit 1 from='client.? 172.21.15.16:0/2386805258' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:04:06.431 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T20:04:06.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:06 smithi016 bash[75004]: cluster 2023-09-23T20:04:04.712652+0000 mgr.y (mgr.25325) 1707 : cluster 0 pgmap v1783: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:06.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:06 smithi016 bash[75004]: audit 2023-09-23T20:04:05.480471+0000 mon.a (mon.0) 2079 : audit 1 from='client.? 172.21.15.16:0/2386805258' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:04:06.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:06 smithi016 bash[75004]: cluster 2023-09-23T20:04:05.480854+0000 mon.a (mon.0) 2080 : cluster 0 osdmap e316: 8 total, 8 up, 8 in 2023-09-23T20:04:06.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:06 smithi016 bash[75004]: cluster 2023-09-23T20:04:05.491404+0000 osd.7 (osd.7) 236 : cluster 3 failed to encode map e316 with expected crc 2023-09-23T20:04:06.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:06 smithi016 bash[75004]: cluster 2023-09-23T20:04:05.491629+0000 osd.4 (osd.4) 263 : cluster 3 failed to encode map e316 with expected crc 2023-09-23T20:04:06.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:06 smithi016 bash[75004]: cluster 2023-09-23T20:04:05.492293+0000 osd.5 (osd.5) 234 : cluster 3 failed to encode map e316 with expected crc 2023-09-23T20:04:06.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:06 smithi016 bash[75004]: cluster 2023-09-23T20:04:05.494168+0000 osd.7 (osd.7) 237 : cluster 3 failed to encode map e316 with expected crc 2023-09-23T20:04:06.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:06 smithi016 bash[75004]: cluster 2023-09-23T20:04:05.885801+0000 osd.6 (osd.6) 254 : cluster 3 failed to encode map e316 with expected crc 2023-09-23T20:04:06.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:06 smithi016 bash[77995]: cluster 2023-09-23T20:04:04.712652+0000 mgr.y (mgr.25325) 1707 : cluster 0 pgmap v1783: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:06.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:06 smithi016 bash[77995]: audit 2023-09-23T20:04:05.480471+0000 mon.a (mon.0) 2079 : audit 1 from='client.? 172.21.15.16:0/2386805258' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:04:06.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:06 smithi016 bash[77995]: cluster 2023-09-23T20:04:05.480854+0000 mon.a (mon.0) 2080 : cluster 0 osdmap e316: 8 total, 8 up, 8 in 2023-09-23T20:04:06.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:06 smithi016 bash[77995]: cluster 2023-09-23T20:04:05.491404+0000 osd.7 (osd.7) 236 : cluster 3 failed to encode map e316 with expected crc 2023-09-23T20:04:06.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:06 smithi016 bash[77995]: cluster 2023-09-23T20:04:05.491629+0000 osd.4 (osd.4) 263 : cluster 3 failed to encode map e316 with expected crc 2023-09-23T20:04:06.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:06 smithi016 bash[77995]: cluster 2023-09-23T20:04:05.492293+0000 osd.5 (osd.5) 234 : cluster 3 failed to encode map e316 with expected crc 2023-09-23T20:04:06.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:06 smithi016 bash[77995]: cluster 2023-09-23T20:04:05.494168+0000 osd.7 (osd.7) 237 : cluster 3 failed to encode map e316 with expected crc 2023-09-23T20:04:06.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:06 smithi016 bash[77995]: cluster 2023-09-23T20:04:05.885801+0000 osd.6 (osd.6) 254 : cluster 3 failed to encode map e316 with expected crc 2023-09-23T20:04:06.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:06 smithi092 bash[58855]: cluster 2023-09-23T20:04:04.712652+0000 mgr.y (mgr.25325) 1707 : cluster 0 pgmap v1783: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:06.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:06 smithi092 bash[58855]: audit 2023-09-23T20:04:05.480471+0000 mon.a (mon.0) 2079 : audit 1 from='client.? 172.21.15.16:0/2386805258' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:04:06.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:06 smithi092 bash[58855]: cluster 2023-09-23T20:04:05.480854+0000 mon.a (mon.0) 2080 : cluster 0 osdmap e316: 8 total, 8 up, 8 in 2023-09-23T20:04:06.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:06 smithi092 bash[58855]: cluster 2023-09-23T20:04:05.491404+0000 osd.7 (osd.7) 236 : cluster 3 failed to encode map e316 with expected crc 2023-09-23T20:04:06.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:06 smithi092 bash[58855]: cluster 2023-09-23T20:04:05.491629+0000 osd.4 (osd.4) 263 : cluster 3 failed to encode map e316 with expected crc 2023-09-23T20:04:06.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:06 smithi092 bash[58855]: cluster 2023-09-23T20:04:05.492293+0000 osd.5 (osd.5) 234 : cluster 3 failed to encode map e316 with expected crc 2023-09-23T20:04:06.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:06 smithi092 bash[58855]: cluster 2023-09-23T20:04:05.494168+0000 osd.7 (osd.7) 237 : cluster 3 failed to encode map e316 with expected crc 2023-09-23T20:04:06.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:06 smithi092 bash[58855]: cluster 2023-09-23T20:04:05.885801+0000 osd.6 (osd.6) 254 : cluster 3 failed to encode map e316 with expected crc 2023-09-23T20:04:07.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:07 smithi016 bash[75004]: cluster 2023-09-23T20:04:06.482588+0000 mon.a (mon.0) 2081 : cluster 0 osdmap e317: 8 total, 8 up, 8 in 2023-09-23T20:04:07.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:07 smithi016 bash[75004]: cluster 2023-09-23T20:04:06.484702+0000 osd.6 (osd.6) 255 : cluster 3 failed to encode map e317 with expected crc 2023-09-23T20:04:07.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:07 smithi016 bash[75004]: cluster 2023-09-23T20:04:06.490692+0000 osd.4 (osd.4) 264 : cluster 3 failed to encode map e317 with expected crc 2023-09-23T20:04:07.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:07 smithi016 bash[75004]: cluster 2023-09-23T20:04:06.516565+0000 osd.5 (osd.5) 235 : cluster 3 failed to encode map e317 with expected crc 2023-09-23T20:04:07.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:07 smithi016 bash[75004]: cluster 2023-09-23T20:04:06.710617+0000 osd.7 (osd.7) 238 : cluster 3 failed to encode map e317 with expected crc 2023-09-23T20:04:07.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:07 smithi016 bash[75004]: cluster 2023-09-23T20:04:06.713279+0000 mgr.y (mgr.25325) 1708 : cluster 0 pgmap v1786: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:07.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:07 smithi016 bash[75004]: cluster 2023-09-23T20:04:06.713414+0000 osd.7 (osd.7) 239 : cluster 3 failed to encode map e317 with expected crc 2023-09-23T20:04:07.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:07 smithi016 bash[75004]: cluster 2023-09-23T20:04:06.716136+0000 osd.7 (osd.7) 240 : cluster 3 failed to encode map e317 with expected crc 2023-09-23T20:04:07.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:07 smithi016 bash[77995]: cluster 2023-09-23T20:04:06.482588+0000 mon.a (mon.0) 2081 : cluster 0 osdmap e317: 8 total, 8 up, 8 in 2023-09-23T20:04:07.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:07 smithi016 bash[77995]: cluster 2023-09-23T20:04:06.484702+0000 osd.6 (osd.6) 255 : cluster 3 failed to encode map e317 with expected crc 2023-09-23T20:04:07.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:07 smithi016 bash[77995]: cluster 2023-09-23T20:04:06.490692+0000 osd.4 (osd.4) 264 : cluster 3 failed to encode map e317 with expected crc 2023-09-23T20:04:07.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:07 smithi016 bash[77995]: cluster 2023-09-23T20:04:06.516565+0000 osd.5 (osd.5) 235 : cluster 3 failed to encode map e317 with expected crc 2023-09-23T20:04:07.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:07 smithi016 bash[77995]: cluster 2023-09-23T20:04:06.710617+0000 osd.7 (osd.7) 238 : cluster 3 failed to encode map e317 with expected crc 2023-09-23T20:04:07.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:07 smithi016 bash[77995]: cluster 2023-09-23T20:04:06.713279+0000 mgr.y (mgr.25325) 1708 : cluster 0 pgmap v1786: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:07.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:07 smithi016 bash[77995]: cluster 2023-09-23T20:04:06.713414+0000 osd.7 (osd.7) 239 : cluster 3 failed to encode map e317 with expected crc 2023-09-23T20:04:07.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:07 smithi016 bash[77995]: cluster 2023-09-23T20:04:06.716136+0000 osd.7 (osd.7) 240 : cluster 3 failed to encode map e317 with expected crc 2023-09-23T20:04:07.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:07 smithi092 bash[58855]: cluster 2023-09-23T20:04:06.482588+0000 mon.a (mon.0) 2081 : cluster 0 osdmap e317: 8 total, 8 up, 8 in 2023-09-23T20:04:07.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:07 smithi092 bash[58855]: cluster 2023-09-23T20:04:06.484702+0000 osd.6 (osd.6) 255 : cluster 3 failed to encode map e317 with expected crc 2023-09-23T20:04:07.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:07 smithi092 bash[58855]: cluster 2023-09-23T20:04:06.490692+0000 osd.4 (osd.4) 264 : cluster 3 failed to encode map e317 with expected crc 2023-09-23T20:04:07.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:07 smithi092 bash[58855]: cluster 2023-09-23T20:04:06.516565+0000 osd.5 (osd.5) 235 : cluster 3 failed to encode map e317 with expected crc 2023-09-23T20:04:07.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:07 smithi092 bash[58855]: cluster 2023-09-23T20:04:06.710617+0000 osd.7 (osd.7) 238 : cluster 3 failed to encode map e317 with expected crc 2023-09-23T20:04:07.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:07 smithi092 bash[58855]: cluster 2023-09-23T20:04:06.713279+0000 mgr.y (mgr.25325) 1708 : cluster 0 pgmap v1786: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:07.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:07 smithi092 bash[58855]: cluster 2023-09-23T20:04:06.713414+0000 osd.7 (osd.7) 239 : cluster 3 failed to encode map e317 with expected crc 2023-09-23T20:04:07.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:07 smithi092 bash[58855]: cluster 2023-09-23T20:04:06.716136+0000 osd.7 (osd.7) 240 : cluster 3 failed to encode map e317 with expected crc 2023-09-23T20:04:10.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:09 smithi016 bash[75004]: cluster 2023-09-23T20:04:08.714177+0000 mgr.y (mgr.25325) 1709 : cluster 0 pgmap v1787: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:10.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:09 smithi016 bash[77995]: cluster 2023-09-23T20:04:08.714177+0000 mgr.y (mgr.25325) 1709 : cluster 0 pgmap v1787: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:10.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:09 smithi092 bash[58855]: cluster 2023-09-23T20:04:08.714177+0000 mgr.y (mgr.25325) 1709 : cluster 0 pgmap v1787: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:10.291 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:04:10.292 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:04:12.280 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:11 smithi016 bash[77995]: cluster 2023-09-23T20:04:10.714704+0000 mgr.y (mgr.25325) 1710 : cluster 0 pgmap v1788: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:12.280 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:11 smithi016 bash[75004]: cluster 2023-09-23T20:04:10.714704+0000 mgr.y (mgr.25325) 1710 : cluster 0 pgmap v1788: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:12.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:11 smithi092 bash[58855]: cluster 2023-09-23T20:04:10.714704+0000 mgr.y (mgr.25325) 1710 : cluster 0 pgmap v1788: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:14.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:13 smithi016 bash[77995]: cluster 2023-09-23T20:04:12.715453+0000 mgr.y (mgr.25325) 1711 : cluster 0 pgmap v1789: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:14.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:13 smithi016 bash[77995]: audit 2023-09-23T20:04:13.702966+0000 mon.a (mon.0) 2082 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:04:14.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:13 smithi016 bash[75004]: cluster 2023-09-23T20:04:12.715453+0000 mgr.y (mgr.25325) 1711 : cluster 0 pgmap v1789: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:14.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:13 smithi016 bash[75004]: audit 2023-09-23T20:04:13.702966+0000 mon.a (mon.0) 2082 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:04:14.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:13 smithi092 bash[58855]: cluster 2023-09-23T20:04:12.715453+0000 mgr.y (mgr.25325) 1711 : cluster 0 pgmap v1789: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:14.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:13 smithi092 bash[58855]: audit 2023-09-23T20:04:13.702966+0000 mon.a (mon.0) 2082 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:04:15.162 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-09-23T20:04:15.162 INFO:teuthology.orchestra.run.smithi016.stdout: "size": 20, 2023-09-23T20:04:15.162 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 600, 2023-09-23T20:04:15.163 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [ 2023-09-23T20:04:15.163 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.163 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46640.0:213227 13.2 13:4fb90b00:::benchmark_data_smithi016_205181_object105963:head [delete] snapc 0=[] ondisk+write+known_if_redirected e250)", 2023-09-23T20:04:15.163 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:55:57.644746+0000", 2023-09-23T20:04:15.163 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 497.51123161800001, 2023-09-23T20:04:15.163 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.087058025999999997, 2023-09-23T20:04:15.163 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:04:15.163 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:15.163 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:04:15.163 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46640", 2023-09-23T20:04:15.163 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2656794142", 2023-09-23T20:04:15.164 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 213227 2023-09-23T20:04:15.164 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.164 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:04:15.164 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.164 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:04:15.164 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.644746+0000", 2023-09-23T20:04:15.164 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:04:15.164 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.164 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.164 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:04:15.164 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.644746+0000", 2023-09-23T20:04:15.164 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:04:15.165 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.165 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.165 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:04:15.165 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.644745+0000", 2023-09-23T20:04:15.165 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T20:04:15.165 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.165 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.165 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:04:15.165 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.644751+0000", 2023-09-23T20:04:15.165 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.0179999999999998e-06 2023-09-23T20:04:15.165 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.165 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.166 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:04:15.166 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.644767+0000", 2023-09-23T20:04:15.166 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.5872e-05 2023-09-23T20:04:15.166 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.166 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.166 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:04:15.166 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.644775+0000", 2023-09-23T20:04:15.166 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.0509999999999997e-06 2023-09-23T20:04:15.166 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.166 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.166 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:04:15.166 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.653708+0000", 2023-09-23T20:04:15.167 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0089330680000000006 2023-09-23T20:04:15.167 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.167 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.167 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:04:15.167 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.653872+0000", 2023-09-23T20:04:15.167 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00016362399999999999 2023-09-23T20:04:15.167 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.167 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.167 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 4,5", 2023-09-23T20:04:15.167 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.654024+0000", 2023-09-23T20:04:15.167 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00015267899999999999 2023-09-23T20:04:15.167 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.168 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.168 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:15.168 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.669314+0000", 2023-09-23T20:04:15.168 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.015289412 2023-09-23T20:04:15.168 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.168 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.168 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T20:04:15.168 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.669333+0000", 2023-09-23T20:04:15.168 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.9429e-05 2023-09-23T20:04:15.168 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.168 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.168 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:15.168 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.731600+0000", 2023-09-23T20:04:15.169 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.062266609000000001 2023-09-23T20:04:15.169 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.169 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.169 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:04:15.169 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.731740+0000", 2023-09-23T20:04:15.169 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000140195 2023-09-23T20:04:15.169 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.169 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.169 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:04:15.169 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.731804+0000", 2023-09-23T20:04:15.169 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.4027000000000002e-05 2023-09-23T20:04:15.169 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:04:15.170 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:04:15.170 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:04:15.170 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.170 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.170 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46640.0:213228 13.2 13:4f494cf0:::benchmark_data_smithi016_205181_object105964:head [delete] snapc 0=[] ondisk+write+known_if_redirected e250)", 2023-09-23T20:04:15.170 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:55:57.646248+0000", 2023-09-23T20:04:15.170 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 497.50972933200001, 2023-09-23T20:04:15.170 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.085739086000000006, 2023-09-23T20:04:15.170 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:04:15.170 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:15.170 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:04:15.170 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46640", 2023-09-23T20:04:15.171 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/2656794142", 2023-09-23T20:04:15.171 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 213228 2023-09-23T20:04:15.171 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.171 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:04:15.171 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.171 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:04:15.171 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.646248+0000", 2023-09-23T20:04:15.171 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:04:15.171 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.171 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.171 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:04:15.171 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.646248+0000", 2023-09-23T20:04:15.172 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:04:15.172 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.172 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.172 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:04:15.172 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.646247+0000", 2023-09-23T20:04:15.172 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T20:04:15.172 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.172 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.172 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:04:15.172 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.646254+0000", 2023-09-23T20:04:15.172 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.4949999999999996e-06 2023-09-23T20:04:15.172 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.173 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.173 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:04:15.173 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.646344+0000", 2023-09-23T20:04:15.173 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.0526999999999996e-05 2023-09-23T20:04:15.173 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.173 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.173 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:04:15.173 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.646353+0000", 2023-09-23T20:04:15.173 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.1349999999999998e-06 2023-09-23T20:04:15.173 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.173 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.173 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:04:15.174 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.658892+0000", 2023-09-23T20:04:15.174 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.012538762 2023-09-23T20:04:15.174 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.174 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.174 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:04:15.174 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.659035+0000", 2023-09-23T20:04:15.174 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000142702 2023-09-23T20:04:15.174 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.174 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.174 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 4,5", 2023-09-23T20:04:15.174 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.659177+0000", 2023-09-23T20:04:15.174 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00014189500000000001 2023-09-23T20:04:15.175 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.175 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.175 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T20:04:15.175 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.669341+0000", 2023-09-23T20:04:15.175 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.010164812 2023-09-23T20:04:15.175 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.175 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.175 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:15.175 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.669423+0000", 2023-09-23T20:04:15.175 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.1679000000000002e-05 2023-09-23T20:04:15.175 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.175 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.176 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:15.176 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.731874+0000", 2023-09-23T20:04:15.176 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.062450486999999999 2023-09-23T20:04:15.176 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.176 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.176 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:04:15.176 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.731933+0000", 2023-09-23T20:04:15.176 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.9231000000000003e-05 2023-09-23T20:04:15.176 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.176 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.176 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:04:15.176 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:55:57.731987+0000", 2023-09-23T20:04:15.177 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.4435000000000003e-05 2023-09-23T20:04:15.177 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:04:15.177 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:04:15.177 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:04:15.177 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.177 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.177 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.46877.0:54550 14.1 e261/256 14:89020cc7:::benchmark_data_smithi016_228083_object54549:head v 261'3377, mlcod=261'3375)", 2023-09-23T20:04:15.177 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:57:51.630043+0000", 2023-09-23T20:04:15.177 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 383.52593478699998, 2023-09-23T20:04:15.177 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.058685831000000001, 2023-09-23T20:04:15.177 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:04:15.177 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:15.177 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:04:15.178 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.178 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:04:15.178 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.630043+0000", 2023-09-23T20:04:15.178 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:04:15.178 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.178 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.178 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:04:15.178 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.630043+0000", 2023-09-23T20:04:15.178 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:04:15.178 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.178 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.179 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:04:15.179 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.630042+0000", 2023-09-23T20:04:15.179 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T20:04:15.179 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.179 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.179 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:04:15.179 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.630065+0000", 2023-09-23T20:04:15.179 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.2560000000000001e-05 2023-09-23T20:04:15.179 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.179 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.179 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:04:15.179 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.630070+0000", 2023-09-23T20:04:15.180 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.0300000000000001e-06 2023-09-23T20:04:15.180 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.180 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.180 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:04:15.180 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.630073+0000", 2023-09-23T20:04:15.180 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.2780000000000002e-06 2023-09-23T20:04:15.180 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.180 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.180 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:04:15.180 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.630238+0000", 2023-09-23T20:04:15.180 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000164277 2023-09-23T20:04:15.180 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.181 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.181 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:04:15.181 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.630256+0000", 2023-09-23T20:04:15.181 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.7994000000000001e-05 2023-09-23T20:04:15.181 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.181 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.181 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:04:15.181 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.688644+0000", 2023-09-23T20:04:15.181 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.058388566000000003 2023-09-23T20:04:15.181 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.181 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.181 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:04:15.182 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.688729+0000", 2023-09-23T20:04:15.182 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.4421999999999999e-05 2023-09-23T20:04:15.182 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:04:15.182 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:04:15.182 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:04:15.182 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.182 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.182 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.46877.0:54548 14.9 e261/256 14:9bec6ce9:::benchmark_data_smithi016_228083_object54547:head v 261'3415, mlcod=261'3412)", 2023-09-23T20:04:15.182 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:57:51.630125+0000", 2023-09-23T20:04:15.182 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 383.52585214499999, 2023-09-23T20:04:15.182 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.062403675999999998, 2023-09-23T20:04:15.182 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:04:15.182 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:15.183 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:04:15.183 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.183 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:04:15.183 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.630125+0000", 2023-09-23T20:04:15.183 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:04:15.183 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.183 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.183 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:04:15.183 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.630125+0000", 2023-09-23T20:04:15.183 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:04:15.183 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.183 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.184 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:04:15.184 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.630125+0000", 2023-09-23T20:04:15.184 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T20:04:15.184 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.184 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.184 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:04:15.184 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.630156+0000", 2023-09-23T20:04:15.184 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.1436999999999997e-05 2023-09-23T20:04:15.184 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.184 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.184 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:04:15.184 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.630164+0000", 2023-09-23T20:04:15.184 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.345e-06 2023-09-23T20:04:15.185 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.185 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.185 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:04:15.185 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.630169+0000", 2023-09-23T20:04:15.185 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.7620000000000003e-06 2023-09-23T20:04:15.185 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.185 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.185 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:04:15.185 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.630730+0000", 2023-09-23T20:04:15.185 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00056141100000000005 2023-09-23T20:04:15.185 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.185 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.185 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:04:15.186 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.630791+0000", 2023-09-23T20:04:15.186 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.1495999999999995e-05 2023-09-23T20:04:15.186 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.186 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.186 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:04:15.186 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.692441+0000", 2023-09-23T20:04:15.186 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.061649029000000001 2023-09-23T20:04:15.186 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.186 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.186 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:04:15.186 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.692529+0000", 2023-09-23T20:04:15.186 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.8478000000000002e-05 2023-09-23T20:04:15.187 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:04:15.187 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:04:15.187 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:04:15.187 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.187 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.187 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.46877.0:54553 14.4 e261/256 14:221fb0e3:::benchmark_data_smithi016_228083_object54552:head v 261'3470, mlcod=261'3468)", 2023-09-23T20:04:15.187 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:57:51.631298+0000", 2023-09-23T20:04:15.187 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 383.52467962999998, 2023-09-23T20:04:15.187 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.056592578999999997, 2023-09-23T20:04:15.187 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:04:15.187 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:15.187 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:04:15.188 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.188 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:04:15.188 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.631298+0000", 2023-09-23T20:04:15.188 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:04:15.188 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.188 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.188 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:04:15.188 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.631298+0000", 2023-09-23T20:04:15.188 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:04:15.188 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.188 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.188 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:04:15.188 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.631297+0000", 2023-09-23T20:04:15.189 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T20:04:15.189 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.189 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.189 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:04:15.189 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.631346+0000", 2023-09-23T20:04:15.189 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.9193999999999997e-05 2023-09-23T20:04:15.189 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.189 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.189 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:04:15.189 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.631359+0000", 2023-09-23T20:04:15.189 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2424e-05 2023-09-23T20:04:15.189 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.190 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.190 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:04:15.190 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.631365+0000", 2023-09-23T20:04:15.190 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.2999999999999998e-06 2023-09-23T20:04:15.190 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.190 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.190 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:04:15.190 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.631996+0000", 2023-09-23T20:04:15.190 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00063104800000000005 2023-09-23T20:04:15.190 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.190 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.190 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:04:15.191 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.632015+0000", 2023-09-23T20:04:15.191 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.8958e-05 2023-09-23T20:04:15.191 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.191 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.191 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:04:15.191 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.687698+0000", 2023-09-23T20:04:15.191 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.055683379999999998 2023-09-23T20:04:15.191 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.191 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.191 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:04:15.191 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.687890+0000", 2023-09-23T20:04:15.191 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000192036 2023-09-23T20:04:15.192 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:04:15.192 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:04:15.192 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:04:15.192 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.192 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.192 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.46877.0:54552 14.1 e261/256 14:8ebcf13b:::benchmark_data_smithi016_228083_object54551:head v 261'3378, mlcod=261'3375)", 2023-09-23T20:04:15.192 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:57:51.632083+0000", 2023-09-23T20:04:15.192 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 383.52389460299997, 2023-09-23T20:04:15.192 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.06050962, 2023-09-23T20:04:15.192 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:04:15.192 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:15.192 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:04:15.192 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.193 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:04:15.193 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.632083+0000", 2023-09-23T20:04:15.193 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:04:15.193 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.193 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.193 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:04:15.193 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.632083+0000", 2023-09-23T20:04:15.193 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:04:15.193 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.193 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.193 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:04:15.193 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.632082+0000", 2023-09-23T20:04:15.194 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T20:04:15.194 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.194 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.194 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:04:15.194 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.632120+0000", 2023-09-23T20:04:15.194 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.7450000000000002e-05 2023-09-23T20:04:15.194 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.194 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.194 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:04:15.194 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.632130+0000", 2023-09-23T20:04:15.194 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0165e-05 2023-09-23T20:04:15.194 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.195 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.195 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:04:15.195 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.632137+0000", 2023-09-23T20:04:15.195 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.3529999999999997e-06 2023-09-23T20:04:15.195 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.195 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.195 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:04:15.195 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.632702+0000", 2023-09-23T20:04:15.195 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00056454200000000002 2023-09-23T20:04:15.195 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.195 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.196 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:04:15.196 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.683853+0000", 2023-09-23T20:04:15.196 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.051151611999999999 2023-09-23T20:04:15.196 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.196 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.196 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:04:15.196 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.692533+0000", 2023-09-23T20:04:15.196 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0086799700000000004 2023-09-23T20:04:15.196 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.196 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.196 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:04:15.196 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:51.692592+0000", 2023-09-23T20:04:15.196 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.9290999999999999e-05 2023-09-23T20:04:15.197 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:04:15.197 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:04:15.197 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:04:15.197 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.197 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.197 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.46877.0:83933 14.9 e262/256 14:90a2e298:::benchmark_data_smithi016_228083_object83932:head v 262'5239, mlcod=262'5236)", 2023-09-23T20:04:15.197 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:58:23.288509+0000", 2023-09-23T20:04:15.197 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 351.86746827299999, 2023-09-23T20:04:15.197 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.061785059000000003, 2023-09-23T20:04:15.197 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:04:15.197 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:15.197 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:04:15.198 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.198 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:04:15.198 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:23.288509+0000", 2023-09-23T20:04:15.198 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:04:15.198 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.198 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.198 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:04:15.198 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:23.288509+0000", 2023-09-23T20:04:15.198 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:04:15.198 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.198 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.198 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:04:15.199 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:23.288509+0000", 2023-09-23T20:04:15.199 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T20:04:15.199 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.199 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.199 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:04:15.199 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:23.288543+0000", 2023-09-23T20:04:15.199 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.4434000000000003e-05 2023-09-23T20:04:15.199 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.199 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.199 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:04:15.199 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:23.288555+0000", 2023-09-23T20:04:15.199 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1812e-05 2023-09-23T20:04:15.199 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.200 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.200 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:04:15.200 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:23.288560+0000", 2023-09-23T20:04:15.200 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.1959999999999999e-06 2023-09-23T20:04:15.200 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.200 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.200 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:04:15.200 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:23.290646+0000", 2023-09-23T20:04:15.200 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0020861009999999999 2023-09-23T20:04:15.200 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.200 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.200 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:04:15.201 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:23.290707+0000", 2023-09-23T20:04:15.201 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.1400000000000002e-05 2023-09-23T20:04:15.201 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.201 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.201 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:04:15.201 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:23.350260+0000", 2023-09-23T20:04:15.201 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.059552123999999998 2023-09-23T20:04:15.201 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.201 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.201 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:04:15.201 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:23.350294+0000", 2023-09-23T20:04:15.201 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.4634000000000001e-05 2023-09-23T20:04:15.202 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:04:15.202 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:04:15.202 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:04:15.202 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.202 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.202 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.46877.0:150230 14.4 e268/256 14:2088cce9:::benchmark_data_smithi016_228083_object50271:head v 268'9358, mlcod=268'9355)", 2023-09-23T20:04:15.202 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:59:57.445860+0000", 2023-09-23T20:04:15.202 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 257.71011736299999, 2023-09-23T20:04:15.202 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.059924799000000001, 2023-09-23T20:04:15.202 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:04:15.202 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:15.202 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:04:15.203 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.203 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:04:15.203 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:57.445860+0000", 2023-09-23T20:04:15.203 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:04:15.203 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.203 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.203 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:04:15.203 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:57.445860+0000", 2023-09-23T20:04:15.203 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:04:15.203 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.203 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.203 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:04:15.203 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:57.445859+0000", 2023-09-23T20:04:15.204 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T20:04:15.204 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.204 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.204 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:04:15.204 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:57.445874+0000", 2023-09-23T20:04:15.204 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.4341e-05 2023-09-23T20:04:15.204 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.204 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.204 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:04:15.204 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:57.445892+0000", 2023-09-23T20:04:15.204 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.8051e-05 2023-09-23T20:04:15.204 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.205 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.205 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:04:15.205 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:57.445901+0000", 2023-09-23T20:04:15.205 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.2750000000000001e-06 2023-09-23T20:04:15.205 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.205 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.205 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:04:15.205 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:57.449537+0000", 2023-09-23T20:04:15.205 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00363592 2023-09-23T20:04:15.205 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.205 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.205 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:04:15.206 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:57.449560+0000", 2023-09-23T20:04:15.206 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.3127999999999999e-05 2023-09-23T20:04:15.206 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.206 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.206 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:04:15.206 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:57.505741+0000", 2023-09-23T20:04:15.206 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.056180764000000001 2023-09-23T20:04:15.206 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.206 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.206 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:04:15.206 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:59:57.505785+0000", 2023-09-23T20:04:15.206 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.4169999999999999e-05 2023-09-23T20:04:15.207 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:04:15.207 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:04:15.207 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:04:15.207 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.207 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.207 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.46877.0:154473 14.2 e269/256 14:497720c1:::benchmark_data_smithi016_228083_object54514:head v 269'9797, mlcod=269'9795)", 2023-09-23T20:04:15.207 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T20:00:04.024268+0000", 2023-09-23T20:04:15.207 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 251.13170976800001, 2023-09-23T20:04:15.207 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.065530372000000003, 2023-09-23T20:04:15.207 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:04:15.207 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:15.207 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:04:15.208 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.208 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:04:15.208 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.024268+0000", 2023-09-23T20:04:15.208 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:04:15.208 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.208 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.208 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:04:15.208 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.024268+0000", 2023-09-23T20:04:15.208 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:04:15.208 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.208 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.208 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:04:15.208 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.024266+0000", 2023-09-23T20:04:15.209 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-09-23T20:04:15.209 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.209 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.209 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:04:15.209 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.024277+0000", 2023-09-23T20:04:15.209 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0164e-05 2023-09-23T20:04:15.209 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.209 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.209 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:04:15.209 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.024293+0000", 2023-09-23T20:04:15.209 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.6594000000000001e-05 2023-09-23T20:04:15.209 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.209 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.210 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:04:15.210 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.024302+0000", 2023-09-23T20:04:15.210 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.2480000000000002e-06 2023-09-23T20:04:15.210 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.210 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.210 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:04:15.210 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.029755+0000", 2023-09-23T20:04:15.210 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0054523599999999998 2023-09-23T20:04:15.210 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.210 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.210 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:04:15.211 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.029926+0000", 2023-09-23T20:04:15.211 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00017156300000000001 2023-09-23T20:04:15.211 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.211 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.211 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:04:15.211 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.089501+0000", 2023-09-23T20:04:15.211 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.059574237000000002 2023-09-23T20:04:15.211 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.211 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.211 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:04:15.211 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.089798+0000", 2023-09-23T20:04:15.211 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000297547 2023-09-23T20:04:15.211 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:04:15.212 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:04:15.212 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:04:15.212 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.212 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.212 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.46877.0:154482 14.2 e269/256 14:44c77acc:::benchmark_data_smithi016_228083_object54523:head v 269'9798, mlcod=269'9795)", 2023-09-23T20:04:15.212 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T20:00:04.028190+0000", 2023-09-23T20:04:15.212 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 251.12778776299999, 2023-09-23T20:04:15.212 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.077841320000000006, 2023-09-23T20:04:15.212 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:04:15.212 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:15.212 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:04:15.212 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.213 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:04:15.213 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.028190+0000", 2023-09-23T20:04:15.213 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:04:15.213 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.213 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.213 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:04:15.213 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.028190+0000", 2023-09-23T20:04:15.213 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:04:15.213 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.213 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.213 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:04:15.213 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.028189+0000", 2023-09-23T20:04:15.214 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T20:04:15.214 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.214 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.214 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:04:15.214 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.028197+0000", 2023-09-23T20:04:15.214 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.0690000000000002e-06 2023-09-23T20:04:15.214 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.214 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.214 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:04:15.214 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.028213+0000", 2023-09-23T20:04:15.214 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.5829999999999999e-05 2023-09-23T20:04:15.215 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.215 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.215 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:04:15.215 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.028220+0000", 2023-09-23T20:04:15.215 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.5569999999999998e-06 2023-09-23T20:04:15.215 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.215 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.215 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:04:15.215 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.036948+0000", 2023-09-23T20:04:15.215 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0087276049999999994 2023-09-23T20:04:15.215 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.215 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.215 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:04:15.216 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.037004+0000", 2023-09-23T20:04:15.216 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.6475999999999999e-05 2023-09-23T20:04:15.216 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.216 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.216 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:04:15.216 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.105910+0000", 2023-09-23T20:04:15.216 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.068905575999999996 2023-09-23T20:04:15.216 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.216 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.216 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:04:15.216 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.106031+0000", 2023-09-23T20:04:15.216 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000120967 2023-09-23T20:04:15.217 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:04:15.217 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:04:15.217 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:04:15.217 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.217 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.217 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.46877.0:154486 14.2 e269/256 14:416aed1f:::benchmark_data_smithi016_228083_object54527:head v 269'9799, mlcod=269'9795)", 2023-09-23T20:04:15.217 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T20:00:04.032795+0000", 2023-09-23T20:04:15.217 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 251.12318264199999, 2023-09-23T20:04:15.217 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.073542451999999994, 2023-09-23T20:04:15.217 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:04:15.217 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:15.217 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:04:15.218 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.218 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:04:15.218 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.032795+0000", 2023-09-23T20:04:15.218 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:04:15.218 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.218 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.218 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:04:15.218 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.032795+0000", 2023-09-23T20:04:15.218 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:04:15.218 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.218 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.218 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:04:15.219 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.032794+0000", 2023-09-23T20:04:15.219 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T20:04:15.219 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.219 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.219 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:04:15.219 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.032828+0000", 2023-09-23T20:04:15.219 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.4489999999999997e-05 2023-09-23T20:04:15.219 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.219 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.219 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:04:15.219 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.032840+0000", 2023-09-23T20:04:15.219 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1693e-05 2023-09-23T20:04:15.219 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.220 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.220 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:04:15.220 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.032858+0000", 2023-09-23T20:04:15.220 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.7951000000000001e-05 2023-09-23T20:04:15.220 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.220 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.220 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:04:15.220 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.049113+0000", 2023-09-23T20:04:15.220 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.016254807999999999 2023-09-23T20:04:15.220 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.220 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.220 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:04:15.221 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.049168+0000", 2023-09-23T20:04:15.221 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.5520999999999999e-05 2023-09-23T20:04:15.221 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.221 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.221 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:04:15.221 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.106159+0000", 2023-09-23T20:04:15.221 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.056990681000000001 2023-09-23T20:04:15.221 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.221 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.221 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:04:15.221 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.106337+0000", 2023-09-23T20:04:15.221 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000178178 2023-09-23T20:04:15.222 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:04:15.222 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:04:15.222 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:04:15.222 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.222 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.222 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.46877.0:154484 14.a e269/256 14:5bcad579:::benchmark_data_smithi016_228083_object54525:head v 269'9573, mlcod=269'9571)", 2023-09-23T20:04:15.222 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T20:00:04.033012+0000", 2023-09-23T20:04:15.222 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 251.12296551, 2023-09-23T20:04:15.222 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.073113250000000005, 2023-09-23T20:04:15.222 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:04:15.222 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:15.222 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:04:15.223 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.223 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:04:15.223 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.033012+0000", 2023-09-23T20:04:15.223 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:04:15.223 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.223 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.223 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:04:15.223 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.033012+0000", 2023-09-23T20:04:15.223 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:04:15.223 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.223 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.223 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:04:15.224 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.033011+0000", 2023-09-23T20:04:15.224 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T20:04:15.224 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.224 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.224 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:04:15.224 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.033019+0000", 2023-09-23T20:04:15.224 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.6769999999999995e-06 2023-09-23T20:04:15.224 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.224 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.224 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:04:15.224 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.033036+0000", 2023-09-23T20:04:15.224 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.6714e-05 2023-09-23T20:04:15.225 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.225 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.225 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:04:15.225 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.033044+0000", 2023-09-23T20:04:15.225 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.0600000000000008e-06 2023-09-23T20:04:15.225 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.225 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.225 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:04:15.225 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.049311+0000", 2023-09-23T20:04:15.225 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.016267835000000001 2023-09-23T20:04:15.225 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.225 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.226 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:04:15.226 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.049450+0000", 2023-09-23T20:04:15.226 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00013821300000000001 2023-09-23T20:04:15.226 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.226 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.226 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:04:15.226 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.106034+0000", 2023-09-23T20:04:15.226 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.056584809999999999 2023-09-23T20:04:15.226 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.226 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.226 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:04:15.226 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.106125+0000", 2023-09-23T20:04:15.227 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.0725999999999998e-05 2023-09-23T20:04:15.227 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:04:15.227 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:04:15.227 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:04:15.227 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.227 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.227 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.46877.0:154488 14.a e269/256 14:54686451:::benchmark_data_smithi016_228083_object54529:head v 269'9574, mlcod=269'9571)", 2023-09-23T20:04:15.227 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T20:00:04.038029+0000", 2023-09-23T20:04:15.227 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 251.11794821300001, 2023-09-23T20:04:15.227 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.068671248000000004, 2023-09-23T20:04:15.227 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:04:15.227 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:15.227 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:04:15.228 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.228 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:04:15.228 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.038029+0000", 2023-09-23T20:04:15.228 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:04:15.228 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.228 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.228 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:04:15.228 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.038029+0000", 2023-09-23T20:04:15.228 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:04:15.228 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.228 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.228 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:04:15.229 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.038029+0000", 2023-09-23T20:04:15.229 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T20:04:15.229 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.229 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.229 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:04:15.229 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.038036+0000", 2023-09-23T20:04:15.229 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.9500000000000001e-06 2023-09-23T20:04:15.229 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.229 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.229 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:04:15.229 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.038051+0000", 2023-09-23T20:04:15.229 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.4503e-05 2023-09-23T20:04:15.230 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.230 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.230 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:04:15.230 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.038060+0000", 2023-09-23T20:04:15.230 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.5860000000000005e-06 2023-09-23T20:04:15.230 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.230 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.230 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:04:15.230 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.089954+0000", 2023-09-23T20:04:15.230 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.051894653999999998 2023-09-23T20:04:15.230 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.230 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.231 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:04:15.231 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.090009+0000", 2023-09-23T20:04:15.231 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.4729999999999999e-05 2023-09-23T20:04:15.231 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.231 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.231 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:04:15.231 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.106545+0000", 2023-09-23T20:04:15.231 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.016536449000000002 2023-09-23T20:04:15.231 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.231 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.231 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:04:15.231 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.106701+0000", 2023-09-23T20:04:15.232 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000155117 2023-09-23T20:04:15.232 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:04:15.232 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:04:15.232 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:04:15.232 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.232 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.232 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.46877.0:154495 14.2 e269/256 14:4e64f39b:::benchmark_data_smithi016_228083_object54536:head v 269'9800, mlcod=269'9795)", 2023-09-23T20:04:15.232 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T20:00:04.046908+0000", 2023-09-23T20:04:15.232 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 251.10906919600001, 2023-09-23T20:04:15.232 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.059628992999999998, 2023-09-23T20:04:15.232 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:04:15.232 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:15.233 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:04:15.233 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.233 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:04:15.233 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.046908+0000", 2023-09-23T20:04:15.233 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:04:15.233 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.233 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.233 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:04:15.233 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.046908+0000", 2023-09-23T20:04:15.233 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:04:15.233 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.233 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.233 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:04:15.234 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.046907+0000", 2023-09-23T20:04:15.234 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T20:04:15.234 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.234 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.234 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:04:15.234 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.046916+0000", 2023-09-23T20:04:15.234 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.4319999999999999e-06 2023-09-23T20:04:15.234 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.234 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.234 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:04:15.234 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.046932+0000", 2023-09-23T20:04:15.234 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.6422e-05 2023-09-23T20:04:15.235 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.235 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.235 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:04:15.235 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.046941+0000", 2023-09-23T20:04:15.235 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.653e-06 2023-09-23T20:04:15.235 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.235 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.235 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:04:15.235 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.075440+0000", 2023-09-23T20:04:15.235 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.028498684999999999 2023-09-23T20:04:15.235 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.235 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.236 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:04:15.236 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.078303+0000", 2023-09-23T20:04:15.236 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0028637530000000001 2023-09-23T20:04:15.236 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.236 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.236 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:04:15.236 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.106343+0000", 2023-09-23T20:04:15.236 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.028039135999999999 2023-09-23T20:04:15.236 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.236 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.236 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:04:15.236 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:04.106537+0000", 2023-09-23T20:04:15.237 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00019472399999999999 2023-09-23T20:04:15.237 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:04:15.237 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:04:15.237 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:04:15.237 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.237 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.237 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop(client.47312.0:35 2.1 e289/159 2:924df93f:::rbd_data.b8d031d2592e.000000000000001b:head v 289'21, mlcod=289'19)", 2023-09-23T20:04:15.237 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T20:02:38.577484+0000", 2023-09-23T20:04:15.237 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 96.578493676999997, 2023-09-23T20:04:15.237 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00091127500000000002, 2023-09-23T20:04:15.237 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:04:15.237 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:15.237 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:04:15.238 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.238 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:04:15.238 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:38.577484+0000", 2023-09-23T20:04:15.238 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:04:15.238 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.238 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.238 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:04:15.238 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:38.577484+0000", 2023-09-23T20:04:15.238 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:04:15.238 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.238 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.238 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:04:15.239 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:38.577482+0000", 2023-09-23T20:04:15.239 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999981 2023-09-23T20:04:15.239 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.239 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.239 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:04:15.239 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:38.577506+0000", 2023-09-23T20:04:15.239 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.4386999999999999e-05 2023-09-23T20:04:15.239 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.239 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.239 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:04:15.239 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:38.577546+0000", 2023-09-23T20:04:15.239 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.9464e-05 2023-09-23T20:04:15.240 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.240 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.240 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:04:15.240 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:38.577567+0000", 2023-09-23T20:04:15.240 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.0934999999999999e-05 2023-09-23T20:04:15.240 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.240 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.240 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:04:15.240 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:38.577686+0000", 2023-09-23T20:04:15.240 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000119381 2023-09-23T20:04:15.240 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.240 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.240 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:04:15.241 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:38.577723+0000", 2023-09-23T20:04:15.241 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.7004e-05 2023-09-23T20:04:15.241 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.241 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.241 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:04:15.241 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:38.578322+0000", 2023-09-23T20:04:15.241 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00059925699999999996 2023-09-23T20:04:15.241 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.241 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.241 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:04:15.241 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:38.578395+0000", 2023-09-23T20:04:15.241 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.2788999999999997e-05 2023-09-23T20:04:15.242 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:04:15.242 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:04:15.242 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:04:15.242 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.242 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.242 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "MOSDPGPushReply(2.6 308/307 [PushReplyOp(2:6cb36536:::rbd_data.ba651b2cc40b.000000000000:head)])", 2023-09-23T20:04:15.242 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T20:03:15.505762+0000", 2023-09-23T20:04:15.242 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 59.650215858999999, 2023-09-23T20:04:15.242 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0010563, 2023-09-23T20:04:15.242 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:04:15.242 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "reached pg", 2023-09-23T20:04:15.242 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:04:15.243 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.243 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:04:15.243 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:03:15.505762+0000", 2023-09-23T20:04:15.243 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:04:15.243 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.243 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.243 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:04:15.243 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:03:15.505762+0000", 2023-09-23T20:04:15.243 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:04:15.243 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.243 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.243 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:04:15.244 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:03:15.505761+0000", 2023-09-23T20:04:15.244 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T20:04:15.244 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.244 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.244 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:04:15.244 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:03:15.505767+0000", 2023-09-23T20:04:15.244 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.4189999999999999e-06 2023-09-23T20:04:15.244 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.244 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.244 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:04:15.244 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:03:15.505781+0000", 2023-09-23T20:04:15.244 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3929e-05 2023-09-23T20:04:15.244 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.245 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.245 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:04:15.245 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:03:15.505789+0000", 2023-09-23T20:04:15.245 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.7740000000000004e-06 2023-09-23T20:04:15.245 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.245 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.245 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:04:15.245 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:03:15.505843+0000", 2023-09-23T20:04:15.245 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.4562999999999999e-05 2023-09-23T20:04:15.245 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.245 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.245 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:04:15.246 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:03:15.506818+0000", 2023-09-23T20:04:15.246 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00097446100000000003 2023-09-23T20:04:15.246 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:04:15.246 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:04:15.246 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:04:15.246 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.246 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.246 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "MOSDPGPushReply(2.6 308/307 [PushReplyOp(2:6cb36536:::rbd_data.ba651b2cc40b.000000000000:head)])", 2023-09-23T20:04:15.246 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T20:03:15.512228+0000", 2023-09-23T20:04:15.246 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 59.643749548000002, 2023-09-23T20:04:15.246 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001021725, 2023-09-23T20:04:15.246 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:04:15.247 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "reached pg", 2023-09-23T20:04:15.247 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:04:15.247 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.247 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:04:15.247 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:03:15.512228+0000", 2023-09-23T20:04:15.247 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:04:15.247 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.247 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.247 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:04:15.247 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:03:15.512228+0000", 2023-09-23T20:04:15.247 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:04:15.247 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.248 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.248 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:04:15.248 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:03:15.512227+0000", 2023-09-23T20:04:15.248 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T20:04:15.248 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.248 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.248 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:04:15.248 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:03:15.512246+0000", 2023-09-23T20:04:15.248 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.8681999999999998e-05 2023-09-23T20:04:15.248 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.248 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.248 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:04:15.248 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:03:15.512260+0000", 2023-09-23T20:04:15.249 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.3879e-05 2023-09-23T20:04:15.249 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.249 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.249 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:04:15.249 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:03:15.512266+0000", 2023-09-23T20:04:15.249 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.5230000000000001e-06 2023-09-23T20:04:15.249 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.249 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.249 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:04:15.249 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:03:15.512334+0000", 2023-09-23T20:04:15.249 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.7719999999999995e-05 2023-09-23T20:04:15.249 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:04:15.250 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:04:15.250 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:04:15.250 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:03:15.513250+0000", 2023-09-23T20:04:15.250 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00091573299999999996 2023-09-23T20:04:15.250 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:04:15.250 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:04:15.250 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:04:15.250 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:04:15.250 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:04:15.250 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-09-23T20:04:15.896 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:04:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:20:04:15] "GET /metrics HTTP/1.1" 200 34012 "" "Prometheus/2.33.4" 2023-09-23T20:04:15.911 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T20:04:15.965 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T20:04:16.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:15 smithi016 bash[77995]: cluster 2023-09-23T20:04:14.716486+0000 mgr.y (mgr.25325) 1712 : cluster 0 pgmap v1790: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:16.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:15 smithi016 bash[77995]: audit 2023-09-23T20:04:15.450347+0000 mon.c (mon.2) 258 : audit 1 from='client.? 172.21.15.16:0/637091873' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:04:16.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:15 smithi016 bash[77995]: audit 2023-09-23T20:04:15.450807+0000 mon.a (mon.0) 2083 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:04:16.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:15 smithi016 bash[75004]: cluster 2023-09-23T20:04:14.716486+0000 mgr.y (mgr.25325) 1712 : cluster 0 pgmap v1790: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:16.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:15 smithi016 bash[75004]: audit 2023-09-23T20:04:15.450347+0000 mon.c (mon.2) 258 : audit 1 from='client.? 172.21.15.16:0/637091873' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:04:16.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:15 smithi016 bash[75004]: audit 2023-09-23T20:04:15.450807+0000 mon.a (mon.0) 2083 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:04:16.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:15 smithi092 bash[58855]: cluster 2023-09-23T20:04:14.716486+0000 mgr.y (mgr.25325) 1712 : cluster 0 pgmap v1790: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:16.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:15 smithi092 bash[58855]: audit 2023-09-23T20:04:15.450347+0000 mon.c (mon.2) 258 : audit 1 from='client.? 172.21.15.16:0/637091873' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:04:16.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:15 smithi092 bash[58855]: audit 2023-09-23T20:04:15.450807+0000 mon.a (mon.0) 2083 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:04:16.863 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 1 1 2023-09-23T20:04:17.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:16 smithi016 bash[75004]: audit 2023-09-23T20:04:15.903556+0000 mon.a (mon.0) 2084 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:04:17.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:16 smithi016 bash[75004]: cluster 2023-09-23T20:04:15.903711+0000 mon.a (mon.0) 2085 : cluster 0 osdmap e318: 8 total, 8 up, 8 in 2023-09-23T20:04:17.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:16 smithi016 bash[75004]: cluster 2023-09-23T20:04:15.905887+0000 osd.4 (osd.4) 265 : cluster 3 failed to encode map e318 with expected crc 2023-09-23T20:04:17.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:16 smithi016 bash[75004]: cluster 2023-09-23T20:04:16.759539+0000 osd.5 (osd.5) 236 : cluster 3 failed to encode map e318 with expected crc 2023-09-23T20:04:17.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:16 smithi016 bash[77995]: audit 2023-09-23T20:04:15.903556+0000 mon.a (mon.0) 2084 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:04:17.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:16 smithi016 bash[77995]: cluster 2023-09-23T20:04:15.903711+0000 mon.a (mon.0) 2085 : cluster 0 osdmap e318: 8 total, 8 up, 8 in 2023-09-23T20:04:17.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:16 smithi016 bash[77995]: cluster 2023-09-23T20:04:15.905887+0000 osd.4 (osd.4) 265 : cluster 3 failed to encode map e318 with expected crc 2023-09-23T20:04:17.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:16 smithi016 bash[77995]: cluster 2023-09-23T20:04:16.759539+0000 osd.5 (osd.5) 236 : cluster 3 failed to encode map e318 with expected crc 2023-09-23T20:04:17.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:16 smithi092 bash[58855]: audit 2023-09-23T20:04:15.903556+0000 mon.a (mon.0) 2084 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:04:17.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:16 smithi092 bash[58855]: cluster 2023-09-23T20:04:15.903711+0000 mon.a (mon.0) 2085 : cluster 0 osdmap e318: 8 total, 8 up, 8 in 2023-09-23T20:04:17.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:16 smithi092 bash[58855]: cluster 2023-09-23T20:04:15.905887+0000 osd.4 (osd.4) 265 : cluster 3 failed to encode map e318 with expected crc 2023-09-23T20:04:17.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:16 smithi092 bash[58855]: cluster 2023-09-23T20:04:16.759539+0000 osd.5 (osd.5) 236 : cluster 3 failed to encode map e318 with expected crc 2023-09-23T20:04:18.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:17 smithi016 bash[75004]: cluster 2023-09-23T20:04:16.717171+0000 mgr.y (mgr.25325) 1713 : cluster 0 pgmap v1792: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:18.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:17 smithi016 bash[77995]: cluster 2023-09-23T20:04:16.717171+0000 mgr.y (mgr.25325) 1713 : cluster 0 pgmap v1792: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:18.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:17 smithi092 bash[58855]: cluster 2023-09-23T20:04:16.717171+0000 mgr.y (mgr.25325) 1713 : cluster 0 pgmap v1792: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:19.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:18 smithi016 bash[75004]: cluster 2023-09-23T20:04:16.984477+0000 osd.6 (osd.6) 256 : cluster 3 failed to encode map e318 with expected crc 2023-09-23T20:04:19.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:18 smithi016 bash[75004]: cluster 2023-09-23T20:04:17.072343+0000 osd.7 (osd.7) 241 : cluster 3 failed to encode map e318 with expected crc 2023-09-23T20:04:19.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:18 smithi016 bash[77995]: cluster 2023-09-23T20:04:16.984477+0000 osd.6 (osd.6) 256 : cluster 3 failed to encode map e318 with expected crc 2023-09-23T20:04:19.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:18 smithi016 bash[77995]: cluster 2023-09-23T20:04:17.072343+0000 osd.7 (osd.7) 241 : cluster 3 failed to encode map e318 with expected crc 2023-09-23T20:04:19.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:18 smithi092 bash[58855]: cluster 2023-09-23T20:04:16.984477+0000 osd.6 (osd.6) 256 : cluster 3 failed to encode map e318 with expected crc 2023-09-23T20:04:19.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:18 smithi092 bash[58855]: cluster 2023-09-23T20:04:17.072343+0000 osd.7 (osd.7) 241 : cluster 3 failed to encode map e318 with expected crc 2023-09-23T20:04:20.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:19 smithi016 bash[75004]: cluster 2023-09-23T20:04:18.718076+0000 mgr.y (mgr.25325) 1714 : cluster 0 pgmap v1793: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:20.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:19 smithi016 bash[77995]: cluster 2023-09-23T20:04:18.718076+0000 mgr.y (mgr.25325) 1714 : cluster 0 pgmap v1793: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:20.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:19 smithi092 bash[58855]: cluster 2023-09-23T20:04:18.718076+0000 mgr.y (mgr.25325) 1714 : cluster 0 pgmap v1793: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:20.744 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:04:21.648 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:04:22.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:21 smithi016 bash[75004]: cluster 2023-09-23T20:04:20.719015+0000 mgr.y (mgr.25325) 1715 : cluster 0 pgmap v1794: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:22.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:21 smithi016 bash[77995]: cluster 2023-09-23T20:04:20.719015+0000 mgr.y (mgr.25325) 1715 : cluster 0 pgmap v1794: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:22.194 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:21 smithi092 bash[58855]: cluster 2023-09-23T20:04:20.719015+0000 mgr.y (mgr.25325) 1715 : cluster 0 pgmap v1794: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:24.066 INFO:teuthology.orchestra.run.smithi092.stdout:{ 2023-09-23T20:04:24.066 INFO:teuthology.orchestra.run.smithi092.stdout: "ops": [], 2023-09-23T20:04:24.066 INFO:teuthology.orchestra.run.smithi092.stdout: "num_ops": 0 2023-09-23T20:04:24.067 INFO:teuthology.orchestra.run.smithi092.stdout:} 2023-09-23T20:04:24.118 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:23 smithi092 bash[58855]: cluster 2023-09-23T20:04:22.719542+0000 mgr.y (mgr.25325) 1716 : cluster 0 pgmap v1795: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:24.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:23 smithi016 bash[75004]: cluster 2023-09-23T20:04:22.719542+0000 mgr.y (mgr.25325) 1716 : cluster 0 pgmap v1795: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:24.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:23 smithi016 bash[77995]: cluster 2023-09-23T20:04:22.719542+0000 mgr.y (mgr.25325) 1716 : cluster 0 pgmap v1795: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:24.916 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T20:04:25.943 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:04:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:20:04:25] "GET /metrics HTTP/1.1" 200 34012 "" "Prometheus/2.33.4" 2023-09-23T20:04:25.944 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:25 smithi016 bash[75004]: cluster 2023-09-23T20:04:24.720286+0000 mgr.y (mgr.25325) 1717 : cluster 0 pgmap v1796: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:25.944 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:25 smithi016 bash[75004]: audit 2023-09-23T20:04:25.358995+0000 mon.a (mon.0) 2086 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:04:25.944 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:25 smithi016 bash[75004]: audit 2023-09-23T20:04:25.362409+0000 mon.b (mon.1) 62 : audit 1 from='client.? 172.21.15.16:0/3535949864' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:04:25.946 INFO:teuthology.orchestra.run.smithi016.stderr:set osd.1 primary-affinity to 1 (8655362) 2023-09-23T20:04:26.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:25 smithi016 bash[77995]: cluster 2023-09-23T20:04:24.720286+0000 mgr.y (mgr.25325) 1717 : cluster 0 pgmap v1796: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:26.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:25 smithi016 bash[77995]: audit 2023-09-23T20:04:25.358995+0000 mon.a (mon.0) 2086 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:04:26.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:25 smithi016 bash[77995]: audit 2023-09-23T20:04:25.362409+0000 mon.b (mon.1) 62 : audit 1 from='client.? 172.21.15.16:0/3535949864' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:04:26.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:25 smithi092 bash[58855]: cluster 2023-09-23T20:04:24.720286+0000 mgr.y (mgr.25325) 1717 : cluster 0 pgmap v1796: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:26.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:25 smithi092 bash[58855]: audit 2023-09-23T20:04:25.358995+0000 mon.a (mon.0) 2086 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:04:26.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:25 smithi092 bash[58855]: audit 2023-09-23T20:04:25.362409+0000 mon.b (mon.1) 62 : audit 1 from='client.? 172.21.15.16:0/3535949864' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:04:26.833 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T20:04:27.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:26 smithi016 bash[75004]: audit 2023-09-23T20:04:25.937947+0000 mon.a (mon.0) 2087 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:04:27.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:26 smithi016 bash[75004]: cluster 2023-09-23T20:04:25.938217+0000 mon.a (mon.0) 2088 : cluster 0 osdmap e319: 8 total, 8 up, 8 in 2023-09-23T20:04:27.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:26 smithi016 bash[75004]: cluster 2023-09-23T20:04:26.221663+0000 osd.5 (osd.5) 237 : cluster 3 failed to encode map e319 with expected crc 2023-09-23T20:04:27.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:26 smithi016 bash[75004]: cluster 2023-09-23T20:04:26.223916+0000 osd.5 (osd.5) 238 : cluster 3 failed to encode map e319 with expected crc 2023-09-23T20:04:27.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:26 smithi016 bash[77995]: audit 2023-09-23T20:04:25.937947+0000 mon.a (mon.0) 2087 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:04:27.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:26 smithi016 bash[77995]: cluster 2023-09-23T20:04:25.938217+0000 mon.a (mon.0) 2088 : cluster 0 osdmap e319: 8 total, 8 up, 8 in 2023-09-23T20:04:27.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:26 smithi016 bash[77995]: cluster 2023-09-23T20:04:26.221663+0000 osd.5 (osd.5) 237 : cluster 3 failed to encode map e319 with expected crc 2023-09-23T20:04:27.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:26 smithi016 bash[77995]: cluster 2023-09-23T20:04:26.223916+0000 osd.5 (osd.5) 238 : cluster 3 failed to encode map e319 with expected crc 2023-09-23T20:04:27.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:26 smithi092 bash[58855]: audit 2023-09-23T20:04:25.937947+0000 mon.a (mon.0) 2087 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:04:27.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:26 smithi092 bash[58855]: cluster 2023-09-23T20:04:25.938217+0000 mon.a (mon.0) 2088 : cluster 0 osdmap e319: 8 total, 8 up, 8 in 2023-09-23T20:04:27.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:26 smithi092 bash[58855]: cluster 2023-09-23T20:04:26.221663+0000 osd.5 (osd.5) 237 : cluster 3 failed to encode map e319 with expected crc 2023-09-23T20:04:27.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:26 smithi092 bash[58855]: cluster 2023-09-23T20:04:26.223916+0000 osd.5 (osd.5) 238 : cluster 3 failed to encode map e319 with expected crc 2023-09-23T20:04:28.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:27 smithi016 bash[77995]: cluster 2023-09-23T20:04:26.036539+0000 osd.7 (osd.7) 242 : cluster 3 failed to encode map e319 with expected crc 2023-09-23T20:04:28.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:27 smithi016 bash[77995]: cluster 2023-09-23T20:04:26.720949+0000 mgr.y (mgr.25325) 1718 : cluster 0 pgmap v1798: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:28.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:27 smithi016 bash[77995]: cluster 2023-09-23T20:04:26.833889+0000 osd.4 (osd.4) 266 : cluster 3 failed to encode map e319 with expected crc 2023-09-23T20:04:28.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:27 smithi016 bash[77995]: cluster 2023-09-23T20:04:26.881162+0000 osd.6 (osd.6) 257 : cluster 3 failed to encode map e319 with expected crc 2023-09-23T20:04:28.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:27 smithi016 bash[75004]: cluster 2023-09-23T20:04:26.036539+0000 osd.7 (osd.7) 242 : cluster 3 failed to encode map e319 with expected crc 2023-09-23T20:04:28.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:27 smithi016 bash[75004]: cluster 2023-09-23T20:04:26.720949+0000 mgr.y (mgr.25325) 1718 : cluster 0 pgmap v1798: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:28.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:27 smithi016 bash[75004]: cluster 2023-09-23T20:04:26.833889+0000 osd.4 (osd.4) 266 : cluster 3 failed to encode map e319 with expected crc 2023-09-23T20:04:28.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:27 smithi016 bash[75004]: cluster 2023-09-23T20:04:26.881162+0000 osd.6 (osd.6) 257 : cluster 3 failed to encode map e319 with expected crc 2023-09-23T20:04:28.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:27 smithi092 bash[58855]: cluster 2023-09-23T20:04:26.036539+0000 osd.7 (osd.7) 242 : cluster 3 failed to encode map e319 with expected crc 2023-09-23T20:04:28.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:27 smithi092 bash[58855]: cluster 2023-09-23T20:04:26.720949+0000 mgr.y (mgr.25325) 1718 : cluster 0 pgmap v1798: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:28.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:27 smithi092 bash[58855]: cluster 2023-09-23T20:04:26.833889+0000 osd.4 (osd.4) 266 : cluster 3 failed to encode map e319 with expected crc 2023-09-23T20:04:28.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:27 smithi092 bash[58855]: cluster 2023-09-23T20:04:26.881162+0000 osd.6 (osd.6) 257 : cluster 3 failed to encode map e319 with expected crc 2023-09-23T20:04:29.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:28 smithi016 bash[75004]: audit 2023-09-23T20:04:28.703613+0000 mon.a (mon.0) 2089 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:04:29.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:28 smithi016 bash[77995]: audit 2023-09-23T20:04:28.703613+0000 mon.a (mon.0) 2089 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:04:29.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:28 smithi092 bash[58855]: audit 2023-09-23T20:04:28.703613+0000 mon.a (mon.0) 2089 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:04:29.698 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:04:30.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:29 smithi016 bash[75004]: cluster 2023-09-23T20:04:28.722792+0000 mgr.y (mgr.25325) 1719 : cluster 0 pgmap v1799: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:30.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:29 smithi016 bash[77995]: cluster 2023-09-23T20:04:28.722792+0000 mgr.y (mgr.25325) 1719 : cluster 0 pgmap v1799: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:30.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:29 smithi092 bash[58855]: cluster 2023-09-23T20:04:28.722792+0000 mgr.y (mgr.25325) 1719 : cluster 0 pgmap v1799: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:31.623 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:04:32.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:31 smithi016 bash[75004]: cluster 2023-09-23T20:04:30.723665+0000 mgr.y (mgr.25325) 1720 : cluster 0 pgmap v1800: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:32.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:31 smithi016 bash[77995]: cluster 2023-09-23T20:04:30.723665+0000 mgr.y (mgr.25325) 1720 : cluster 0 pgmap v1800: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:32.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:31 smithi092 bash[58855]: cluster 2023-09-23T20:04:30.723665+0000 mgr.y (mgr.25325) 1720 : cluster 0 pgmap v1800: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:32.879 INFO:teuthology.orchestra.run.smithi092.stdout:{ 2023-09-23T20:04:32.879 INFO:teuthology.orchestra.run.smithi092.stdout: "ops": [], 2023-09-23T20:04:32.879 INFO:teuthology.orchestra.run.smithi092.stdout: "complaint_time": 30, 2023-09-23T20:04:32.879 INFO:teuthology.orchestra.run.smithi092.stdout: "num_blocked_ops": 0 2023-09-23T20:04:32.879 INFO:teuthology.orchestra.run.smithi092.stdout:} 2023-09-23T20:04:33.722 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T20:04:34.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:33 smithi016 bash[75004]: cluster 2023-09-23T20:04:32.724496+0000 mgr.y (mgr.25325) 1721 : cluster 0 pgmap v1801: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:34.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:33 smithi016 bash[77995]: cluster 2023-09-23T20:04:32.724496+0000 mgr.y (mgr.25325) 1721 : cluster 0 pgmap v1801: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:34.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:33 smithi092 bash[58855]: cluster 2023-09-23T20:04:32.724496+0000 mgr.y (mgr.25325) 1721 : cluster 0 pgmap v1801: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:35.975 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:04:35 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:20:04:35] "GET /metrics HTTP/1.1" 200 34016 "" "Prometheus/2.33.4" 2023-09-23T20:04:35.990 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T20:04:36.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:35 smithi016 bash[77995]: cluster 2023-09-23T20:04:34.725273+0000 mgr.y (mgr.25325) 1722 : cluster 0 pgmap v1802: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:36.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:35 smithi016 bash[77995]: audit 2023-09-23T20:04:35.299257+0000 mon.a (mon.0) 2090 : audit 1 from='client.? 172.21.15.16:0/4011740608' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:04:36.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:35 smithi016 bash[75004]: cluster 2023-09-23T20:04:34.725273+0000 mgr.y (mgr.25325) 1722 : cluster 0 pgmap v1802: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:36.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:35 smithi016 bash[75004]: audit 2023-09-23T20:04:35.299257+0000 mon.a (mon.0) 2090 : audit 1 from='client.? 172.21.15.16:0/4011740608' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:04:36.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:35 smithi092 bash[58855]: cluster 2023-09-23T20:04:34.725273+0000 mgr.y (mgr.25325) 1722 : cluster 0 pgmap v1802: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:36.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:35 smithi092 bash[58855]: audit 2023-09-23T20:04:35.299257+0000 mon.a (mon.0) 2090 : audit 1 from='client.? 172.21.15.16:0/4011740608' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:04:36.922 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 3 1 2023-09-23T20:04:37.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:36 smithi016 bash[75004]: audit 2023-09-23T20:04:35.983474+0000 mon.a (mon.0) 2091 : audit 1 from='client.? 172.21.15.16:0/4011740608' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:04:37.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:36 smithi016 bash[75004]: cluster 2023-09-23T20:04:35.983672+0000 mon.a (mon.0) 2092 : cluster 0 osdmap e320: 8 total, 8 up, 8 in 2023-09-23T20:04:37.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:36 smithi016 bash[75004]: cluster 2023-09-23T20:04:35.984577+0000 osd.7 (osd.7) 243 : cluster 3 failed to encode map e320 with expected crc 2023-09-23T20:04:37.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:36 smithi016 bash[75004]: cluster 2023-09-23T20:04:36.253365+0000 osd.6 (osd.6) 258 : cluster 3 failed to encode map e320 with expected crc 2023-09-23T20:04:37.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:36 smithi016 bash[75004]: cluster 2023-09-23T20:04:36.539871+0000 osd.4 (osd.4) 267 : cluster 3 failed to encode map e320 with expected crc 2023-09-23T20:04:37.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:36 smithi016 bash[75004]: cluster 2023-09-23T20:04:36.540255+0000 osd.5 (osd.5) 239 : cluster 3 failed to encode map e320 with expected crc 2023-09-23T20:04:37.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:36 smithi016 bash[77995]: audit 2023-09-23T20:04:35.983474+0000 mon.a (mon.0) 2091 : audit 1 from='client.? 172.21.15.16:0/4011740608' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:04:37.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:36 smithi016 bash[77995]: cluster 2023-09-23T20:04:35.983672+0000 mon.a (mon.0) 2092 : cluster 0 osdmap e320: 8 total, 8 up, 8 in 2023-09-23T20:04:37.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:36 smithi016 bash[77995]: cluster 2023-09-23T20:04:35.984577+0000 osd.7 (osd.7) 243 : cluster 3 failed to encode map e320 with expected crc 2023-09-23T20:04:37.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:36 smithi016 bash[77995]: cluster 2023-09-23T20:04:36.253365+0000 osd.6 (osd.6) 258 : cluster 3 failed to encode map e320 with expected crc 2023-09-23T20:04:37.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:36 smithi016 bash[77995]: cluster 2023-09-23T20:04:36.539871+0000 osd.4 (osd.4) 267 : cluster 3 failed to encode map e320 with expected crc 2023-09-23T20:04:37.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:36 smithi016 bash[77995]: cluster 2023-09-23T20:04:36.540255+0000 osd.5 (osd.5) 239 : cluster 3 failed to encode map e320 with expected crc 2023-09-23T20:04:37.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:36 smithi092 bash[58855]: audit 2023-09-23T20:04:35.983474+0000 mon.a (mon.0) 2091 : audit 1 from='client.? 172.21.15.16:0/4011740608' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:04:37.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:36 smithi092 bash[58855]: cluster 2023-09-23T20:04:35.983672+0000 mon.a (mon.0) 2092 : cluster 0 osdmap e320: 8 total, 8 up, 8 in 2023-09-23T20:04:37.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:36 smithi092 bash[58855]: cluster 2023-09-23T20:04:35.984577+0000 osd.7 (osd.7) 243 : cluster 3 failed to encode map e320 with expected crc 2023-09-23T20:04:37.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:36 smithi092 bash[58855]: cluster 2023-09-23T20:04:36.253365+0000 osd.6 (osd.6) 258 : cluster 3 failed to encode map e320 with expected crc 2023-09-23T20:04:37.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:36 smithi092 bash[58855]: cluster 2023-09-23T20:04:36.539871+0000 osd.4 (osd.4) 267 : cluster 3 failed to encode map e320 with expected crc 2023-09-23T20:04:37.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:36 smithi092 bash[58855]: cluster 2023-09-23T20:04:36.540255+0000 osd.5 (osd.5) 239 : cluster 3 failed to encode map e320 with expected crc 2023-09-23T20:04:38.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:37 smithi016 bash[75004]: cluster 2023-09-23T20:04:36.725942+0000 mgr.y (mgr.25325) 1723 : cluster 0 pgmap v1804: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:38.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:37 smithi016 bash[77995]: cluster 2023-09-23T20:04:36.725942+0000 mgr.y (mgr.25325) 1723 : cluster 0 pgmap v1804: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:38.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:37 smithi092 bash[58855]: cluster 2023-09-23T20:04:36.725942+0000 mgr.y (mgr.25325) 1723 : cluster 0 pgmap v1804: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:38.492 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:04:40.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:39 smithi016 bash[75004]: cluster 2023-09-23T20:04:38.726856+0000 mgr.y (mgr.25325) 1724 : cluster 0 pgmap v1805: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:40.528 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:39 smithi016 bash[77995]: cluster 2023-09-23T20:04:38.726856+0000 mgr.y (mgr.25325) 1724 : cluster 0 pgmap v1805: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:40.528 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:39 smithi092 bash[58855]: cluster 2023-09-23T20:04:38.726856+0000 mgr.y (mgr.25325) 1724 : cluster 0 pgmap v1805: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:41.708 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:04:41.759 INFO:teuthology.orchestra.run.smithi092.stdout:{ 2023-09-23T20:04:41.759 INFO:teuthology.orchestra.run.smithi092.stdout: "size": 20, 2023-09-23T20:04:41.759 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 600, 2023-09-23T20:04:41.760 INFO:teuthology.orchestra.run.smithi092.stdout: "ops": [ 2023-09-23T20:04:41.760 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.760 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:32634 14.e 14:7b29bb04:::benchmark_data_smithi016_228083_object32633:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-09-23T20:04:41.760 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:57:32.061510+0000", 2023-09-23T20:04:41.760 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 429.69362343099999, 2023-09-23T20:04:41.760 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.323430726, 2023-09-23T20:04:41.760 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:04:41.760 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:41.760 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:04:41.760 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T20:04:41.760 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:04:41.761 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 32634 2023-09-23T20:04:41.761 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.761 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:04:41.761 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.761 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:04:41.761 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.061510+0000", 2023-09-23T20:04:41.761 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:04:41.761 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.761 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.761 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:04:41.761 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.061510+0000", 2023-09-23T20:04:41.761 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999976 2023-09-23T20:04:41.761 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.762 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.762 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:04:41.762 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.061507+0000", 2023-09-23T20:04:41.762 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.7842e-05 2023-09-23T20:04:41.762 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.762 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.762 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:04:41.762 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.061555+0000", 2023-09-23T20:04:41.762 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.7456999999999999e-05 2023-09-23T20:04:41.762 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.762 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.763 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:04:41.763 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.061573+0000", 2023-09-23T20:04:41.763 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.0223e-05 2023-09-23T20:04:41.763 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.763 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.763 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:04:41.763 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.061583+0000", 2023-09-23T20:04:41.763 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.6848000000000006e-05 2023-09-23T20:04:41.763 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.763 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.763 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:04:41.763 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.061670+0000", 2023-09-23T20:04:41.763 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000230239 2023-09-23T20:04:41.764 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.764 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.764 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:04:41.764 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.061900+0000", 2023-09-23T20:04:41.764 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000202544 2023-09-23T20:04:41.764 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.764 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.764 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:04:41.764 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.062103+0000", 2023-09-23T20:04:41.764 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0017556869999999999 2023-09-23T20:04:41.764 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.764 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.765 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:04:41.765 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.063858+0000", 2023-09-23T20:04:41.765 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0080978539999999998 2023-09-23T20:04:41.765 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.765 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.765 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:41.765 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.071956+0000", 2023-09-23T20:04:41.765 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.312810857 2023-09-23T20:04:41.765 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.765 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.765 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:41.765 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.384767+0000", 2023-09-23T20:04:41.766 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000100355 2023-09-23T20:04:41.766 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.766 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.766 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:04:41.766 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.384867+0000", 2023-09-23T20:04:41.766 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.3285000000000001e-05 2023-09-23T20:04:41.766 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.766 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.766 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:04:41.766 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.384941+0000", 2023-09-23T20:04:41.766 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.323430726 2023-09-23T20:04:41.766 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:04:41.767 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:04:41.767 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:04:41.767 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.767 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.767 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:32640 14.e 14:77e29d91:::benchmark_data_smithi016_228083_object32639:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-09-23T20:04:41.767 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:57:32.066749+0000", 2023-09-23T20:04:41.767 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 429.68838475899997, 2023-09-23T20:04:41.767 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.32743465100000002, 2023-09-23T20:04:41.767 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:04:41.767 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:41.767 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:04:41.767 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T20:04:41.768 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:04:41.768 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 32640 2023-09-23T20:04:41.768 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.768 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:04:41.768 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.768 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:04:41.768 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.066749+0000", 2023-09-23T20:04:41.768 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:04:41.768 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.768 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.768 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:04:41.768 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.066749+0000", 2023-09-23T20:04:41.769 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T20:04:41.769 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.769 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.769 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:04:41.769 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.066746+0000", 2023-09-23T20:04:41.769 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.0426e-05 2023-09-23T20:04:41.769 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.769 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.769 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:04:41.769 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.066777+0000", 2023-09-23T20:04:41.769 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.5242000000000001e-05 2023-09-23T20:04:41.769 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.770 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.770 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:04:41.770 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.066792+0000", 2023-09-23T20:04:41.770 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.5299999999999999e-06 2023-09-23T20:04:41.770 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.770 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.770 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:04:41.770 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.066800+0000", 2023-09-23T20:04:41.770 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.4175000000000001e-05 2023-09-23T20:04:41.770 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.770 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.770 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:04:41.771 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.066864+0000", 2023-09-23T20:04:41.771 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00016612800000000001 2023-09-23T20:04:41.771 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.771 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.771 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:04:41.771 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.067030+0000", 2023-09-23T20:04:41.771 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00020505999999999999 2023-09-23T20:04:41.771 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.771 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.771 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:04:41.771 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.067235+0000", 2023-09-23T20:04:41.771 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0043600469999999997 2023-09-23T20:04:41.772 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.772 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.772 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:04:41.772 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.071595+0000", 2023-09-23T20:04:41.772 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00042618199999999999 2023-09-23T20:04:41.772 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.772 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.772 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:41.772 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.072021+0000", 2023-09-23T20:04:41.772 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.322009086 2023-09-23T20:04:41.772 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.772 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.773 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:41.773 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.394030+0000", 2023-09-23T20:04:41.773 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.5445999999999995e-05 2023-09-23T20:04:41.773 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.773 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.773 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:04:41.773 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.394116+0000", 2023-09-23T20:04:41.773 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.7471000000000005e-05 2023-09-23T20:04:41.773 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.773 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.773 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:04:41.773 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.394183+0000", 2023-09-23T20:04:41.774 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.32743465100000002 2023-09-23T20:04:41.774 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:04:41.774 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:04:41.774 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:04:41.774 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.774 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.774 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:32643 14.e 14:7adc8930:::benchmark_data_smithi016_228083_object32642:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-09-23T20:04:41.774 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:57:32.066926+0000", 2023-09-23T20:04:41.774 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 429.68820764700001, 2023-09-23T20:04:41.774 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.32744851800000002, 2023-09-23T20:04:41.774 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:04:41.774 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:41.775 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:04:41.775 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T20:04:41.775 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:04:41.775 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 32643 2023-09-23T20:04:41.775 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.775 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:04:41.775 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.775 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:04:41.775 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.066926+0000", 2023-09-23T20:04:41.775 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:04:41.775 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.776 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.776 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:04:41.776 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.066926+0000", 2023-09-23T20:04:41.776 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.999999 2023-09-23T20:04:41.776 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.776 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.776 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:04:41.776 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.066925+0000", 2023-09-23T20:04:41.776 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.5062000000000002e-05 2023-09-23T20:04:41.776 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.776 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.776 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:04:41.776 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.066980+0000", 2023-09-23T20:04:41.777 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.1095e-05 2023-09-23T20:04:41.777 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.777 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.777 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:04:41.777 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.066991+0000", 2023-09-23T20:04:41.777 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.5839999999999996e-06 2023-09-23T20:04:41.777 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.777 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.777 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:04:41.777 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.066995+0000", 2023-09-23T20:04:41.777 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0015757010000000001 2023-09-23T20:04:41.777 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.778 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.778 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:04:41.778 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.068571+0000", 2023-09-23T20:04:41.778 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000103711 2023-09-23T20:04:41.778 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.778 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.778 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:04:41.778 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.068675+0000", 2023-09-23T20:04:41.778 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00013845900000000001 2023-09-23T20:04:41.778 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.778 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.778 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:04:41.779 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.068813+0000", 2023-09-23T20:04:41.779 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0027960139999999999 2023-09-23T20:04:41.779 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.779 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.779 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:04:41.779 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.071609+0000", 2023-09-23T20:04:41.779 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0012392880000000001 2023-09-23T20:04:41.779 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.779 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.779 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:41.779 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.072848+0000", 2023-09-23T20:04:41.779 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.321397347 2023-09-23T20:04:41.780 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.780 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.780 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:41.780 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.394246+0000", 2023-09-23T20:04:41.780 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.2533000000000005e-05 2023-09-23T20:04:41.780 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.780 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.780 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:04:41.780 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.394318+0000", 2023-09-23T20:04:41.780 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.5840000000000001e-05 2023-09-23T20:04:41.780 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.780 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.781 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:04:41.781 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.394374+0000", 2023-09-23T20:04:41.781 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.32744851800000002 2023-09-23T20:04:41.781 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:04:41.781 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:04:41.781 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:04:41.781 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.781 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.781 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:32651 14.e 14:74226410:::benchmark_data_smithi016_228083_object32650:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-09-23T20:04:41.781 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:57:32.069671+0000", 2023-09-23T20:04:41.781 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 429.685462363, 2023-09-23T20:04:41.781 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.32487358399999999, 2023-09-23T20:04:41.782 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:04:41.782 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:41.782 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:04:41.782 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T20:04:41.782 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:04:41.782 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 32651 2023-09-23T20:04:41.782 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.782 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:04:41.782 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.782 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:04:41.782 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.069671+0000", 2023-09-23T20:04:41.782 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:04:41.783 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.783 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.783 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:04:41.783 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.069671+0000", 2023-09-23T20:04:41.783 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T20:04:41.783 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.783 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.783 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:04:41.783 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.069669+0000", 2023-09-23T20:04:41.783 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.0265999999999997e-05 2023-09-23T20:04:41.783 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.783 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.784 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:04:41.784 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.069709+0000", 2023-09-23T20:04:41.784 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.1335000000000002e-05 2023-09-23T20:04:41.784 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.784 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.784 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:04:41.784 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.069731+0000", 2023-09-23T20:04:41.784 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.3270000000000001e-06 2023-09-23T20:04:41.784 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.784 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.784 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:04:41.784 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.069739+0000", 2023-09-23T20:04:41.785 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00037404499999999999 2023-09-23T20:04:41.785 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.785 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.785 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:04:41.785 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.070113+0000", 2023-09-23T20:04:41.785 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000118404 2023-09-23T20:04:41.785 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.785 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.785 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:04:41.785 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.070231+0000", 2023-09-23T20:04:41.785 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000145771 2023-09-23T20:04:41.785 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.785 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.786 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:04:41.786 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.070377+0000", 2023-09-23T20:04:41.786 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0014109319999999999 2023-09-23T20:04:41.786 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.786 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.786 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:04:41.786 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.071788+0000", 2023-09-23T20:04:41.786 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.001220617 2023-09-23T20:04:41.786 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.786 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.786 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:41.787 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.073009+0000", 2023-09-23T20:04:41.787 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.32142080000000001 2023-09-23T20:04:41.787 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.787 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.787 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:41.787 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.394430+0000", 2023-09-23T20:04:41.787 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.3620000000000004e-05 2023-09-23T20:04:41.787 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.787 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.787 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:04:41.787 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.394493+0000", 2023-09-23T20:04:41.787 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.1310000000000002e-05 2023-09-23T20:04:41.788 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.788 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.788 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:04:41.788 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:32.394545+0000", 2023-09-23T20:04:41.788 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.32487358399999999 2023-09-23T20:04:41.788 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:04:41.788 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:04:41.788 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:04:41.788 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.788 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.788 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:85324 14.9 14:991f0ae7:::benchmark_data_smithi016_228083_object85323:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e263)", 2023-09-23T20:04:41.788 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:58:25.069847+0000", 2023-09-23T20:04:41.789 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 376.68528596499999, 2023-09-23T20:04:41.789 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.0295787919999999, 2023-09-23T20:04:41.789 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:04:41.789 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "reached pg", 2023-09-23T20:04:41.789 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:04:41.789 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T20:04:41.789 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:04:41.789 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 85324 2023-09-23T20:04:41.789 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.789 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:04:41.789 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.789 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:04:41.790 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.069847+0000", 2023-09-23T20:04:41.790 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:04:41.790 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.790 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.790 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:04:41.790 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.069847+0000", 2023-09-23T20:04:41.790 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999938 2023-09-23T20:04:41.790 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.790 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.790 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:04:41.790 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.069841+0000", 2023-09-23T20:04:41.790 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.8390000000000002e-05 2023-09-23T20:04:41.791 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.791 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.791 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:04:41.791 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.069900+0000", 2023-09-23T20:04:41.791 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.054e-05 2023-09-23T20:04:41.791 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.791 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.791 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:04:41.791 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.069920+0000", 2023-09-23T20:04:41.791 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.7443e-05 2023-09-23T20:04:41.791 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.791 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.792 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:04:41.792 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.069958+0000", 2023-09-23T20:04:41.792 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.4419999999999995e-05 2023-09-23T20:04:41.792 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.792 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.792 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:04:41.792 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.070042+0000", 2023-09-23T20:04:41.792 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.9209999999999992e-06 2023-09-23T20:04:41.792 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.792 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.792 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op must wait for map", 2023-09-23T20:04:41.793 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.070050+0000", 2023-09-23T20:04:41.793 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0081624030000000004 2023-09-23T20:04:41.793 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.793 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.793 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:04:41.793 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.078212+0000", 2023-09-23T20:04:41.793 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.3901e-05 2023-09-23T20:04:41.793 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.793 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.793 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for peered", 2023-09-23T20:04:41.793 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.078236+0000", 2023-09-23T20:04:41.793 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.0194893199999999 2023-09-23T20:04:41.794 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.794 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.794 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:04:41.794 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.097726+0000", 2023-09-23T20:04:41.794 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0017004839999999999 2023-09-23T20:04:41.794 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.794 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.794 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:04:41.794 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.099426+0000", 2023-09-23T20:04:41.794 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.0295787919999999 2023-09-23T20:04:41.794 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:04:41.794 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:04:41.795 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:04:41.795 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.795 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.795 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:85343 14.9 14:97ef8a96:::benchmark_data_smithi016_228083_object85342:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e263)", 2023-09-23T20:04:41.795 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:58:25.081329+0000", 2023-09-23T20:04:41.795 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 376.673804598, 2023-09-23T20:04:41.795 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.0315053750000001, 2023-09-23T20:04:41.795 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:04:41.795 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:41.795 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:04:41.795 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T20:04:41.795 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:04:41.796 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 85343 2023-09-23T20:04:41.796 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.796 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:04:41.796 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.796 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:04:41.796 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.081329+0000", 2023-09-23T20:04:41.796 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:04:41.796 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.796 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.796 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:04:41.796 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.081329+0000", 2023-09-23T20:04:41.796 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T20:04:41.797 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.797 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.797 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:04:41.797 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.081327+0000", 2023-09-23T20:04:41.797 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.6828e-05 2023-09-23T20:04:41.797 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.797 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.797 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:04:41.797 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.081364+0000", 2023-09-23T20:04:41.797 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.0944000000000001e-05 2023-09-23T20:04:41.797 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.797 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.798 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:04:41.798 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.081375+0000", 2023-09-23T20:04:41.798 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.0110000000000003e-06 2023-09-23T20:04:41.798 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.798 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.798 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:04:41.798 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.081382+0000", 2023-09-23T20:04:41.798 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00090449200000000001 2023-09-23T20:04:41.798 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.798 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.798 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:04:41.798 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.082286+0000", 2023-09-23T20:04:41.799 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.426e-06 2023-09-23T20:04:41.799 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.799 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.799 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for peered", 2023-09-23T20:04:41.799 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.082292+0000", 2023-09-23T20:04:41.799 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.017513431 2023-09-23T20:04:41.799 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.799 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.799 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:04:41.799 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.099806+0000", 2023-09-23T20:04:41.799 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00047541599999999999 2023-09-23T20:04:41.800 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.800 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.800 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:04:41.800 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.100281+0000", 2023-09-23T20:04:41.800 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000148267 2023-09-23T20:04:41.800 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.800 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.800 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3,5", 2023-09-23T20:04:41.800 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.100430+0000", 2023-09-23T20:04:41.800 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.012105301000000001 2023-09-23T20:04:41.800 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.800 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.801 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:41.801 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.112535+0000", 2023-09-23T20:04:41.801 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.6626000000000003e-05 2023-09-23T20:04:41.801 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.801 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.801 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:04:41.801 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.112592+0000", 2023-09-23T20:04:41.801 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.7176999999999996e-05 2023-09-23T20:04:41.801 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.801 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.801 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:41.801 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.112679+0000", 2023-09-23T20:04:41.801 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.7911999999999994e-05 2023-09-23T20:04:41.802 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.802 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.802 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:04:41.802 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.112767+0000", 2023-09-23T20:04:41.802 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.7491999999999997e-05 2023-09-23T20:04:41.802 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.802 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.802 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:04:41.802 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.112834+0000", 2023-09-23T20:04:41.802 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.0315053750000001 2023-09-23T20:04:41.802 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:04:41.803 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:04:41.803 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:04:41.803 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.803 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.803 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:85385 14.9 14:9fce2e80:::benchmark_data_smithi016_228083_object85384:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e263)", 2023-09-23T20:04:41.803 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:58:25.131141+0000", 2023-09-23T20:04:41.803 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 376.62399212700001, 2023-09-23T20:04:41.803 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.98198450999999998, 2023-09-23T20:04:41.803 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:04:41.803 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:41.803 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:04:41.803 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T20:04:41.804 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:04:41.804 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 85385 2023-09-23T20:04:41.804 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.804 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:04:41.804 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.804 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:04:41.804 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.131141+0000", 2023-09-23T20:04:41.804 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:04:41.804 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.804 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.804 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:04:41.804 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.131141+0000", 2023-09-23T20:04:41.804 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T20:04:41.805 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.805 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.805 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:04:41.805 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.131140+0000", 2023-09-23T20:04:41.805 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.524e-05 2023-09-23T20:04:41.805 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.805 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.805 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:04:41.805 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.131195+0000", 2023-09-23T20:04:41.805 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.3030000000000001e-05 2023-09-23T20:04:41.805 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.805 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.806 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:04:41.806 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.131208+0000", 2023-09-23T20:04:41.806 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.4240000000000001e-06 2023-09-23T20:04:41.806 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.806 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.806 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:04:41.806 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.131213+0000", 2023-09-23T20:04:41.806 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.2309000000000003e-05 2023-09-23T20:04:41.806 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.806 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.806 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:04:41.806 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.131276+0000", 2023-09-23T20:04:41.807 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.0212e-05 2023-09-23T20:04:41.807 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.807 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.807 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for peered", 2023-09-23T20:04:41.807 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.131286+0000", 2023-09-23T20:04:41.807 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.97170411499999998 2023-09-23T20:04:41.807 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.807 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.807 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:04:41.807 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.102990+0000", 2023-09-23T20:04:41.807 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.8024999999999997e-05 2023-09-23T20:04:41.807 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.808 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.808 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:04:41.808 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.103088+0000", 2023-09-23T20:04:41.808 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000121412 2023-09-23T20:04:41.808 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.808 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.808 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3,5", 2023-09-23T20:04:41.808 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.103210+0000", 2023-09-23T20:04:41.808 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0093902440000000007 2023-09-23T20:04:41.808 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.808 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.809 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:04:41.809 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.112600+0000", 2023-09-23T20:04:41.809 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00031716699999999998 2023-09-23T20:04:41.809 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.809 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.809 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:41.809 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.112917+0000", 2023-09-23T20:04:41.809 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.1812999999999998e-05 2023-09-23T20:04:41.809 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.809 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.810 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:41.810 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.112989+0000", 2023-09-23T20:04:41.810 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.3490000000000003e-05 2023-09-23T20:04:41.810 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.810 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.810 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:04:41.810 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.113062+0000", 2023-09-23T20:04:41.810 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.3431999999999998e-05 2023-09-23T20:04:41.810 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.810 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.810 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:04:41.811 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.113126+0000", 2023-09-23T20:04:41.811 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.98198450999999998 2023-09-23T20:04:41.811 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:04:41.811 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:04:41.811 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:04:41.811 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.811 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.811 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:85401 14.9 14:9e19646f:::benchmark_data_smithi016_228083_object85400:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e263)", 2023-09-23T20:04:41.811 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:58:25.160076+0000", 2023-09-23T20:04:41.811 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 376.59505773199999, 2023-09-23T20:04:41.811 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.95347298599999997, 2023-09-23T20:04:41.812 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:04:41.812 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:41.812 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:04:41.812 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T20:04:41.812 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:04:41.812 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 85401 2023-09-23T20:04:41.812 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.812 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:04:41.812 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.812 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:04:41.812 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.160076+0000", 2023-09-23T20:04:41.812 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:04:41.813 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.813 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.813 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:04:41.813 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.160076+0000", 2023-09-23T20:04:41.813 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T20:04:41.813 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.813 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.813 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:04:41.813 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.160074+0000", 2023-09-23T20:04:41.813 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.0092999999999997e-05 2023-09-23T20:04:41.813 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.813 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.814 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:04:41.814 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.160114+0000", 2023-09-23T20:04:41.814 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.9205e-05 2023-09-23T20:04:41.814 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.814 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.814 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:04:41.814 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.160133+0000", 2023-09-23T20:04:41.814 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.3440000000000001e-06 2023-09-23T20:04:41.814 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.814 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.814 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:04:41.814 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.160141+0000", 2023-09-23T20:04:41.815 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.8044e-05 2023-09-23T20:04:41.815 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.815 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.815 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:04:41.815 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.160219+0000", 2023-09-23T20:04:41.815 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.4097000000000001e-05 2023-09-23T20:04:41.815 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.815 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.815 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for peered", 2023-09-23T20:04:41.815 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.160233+0000", 2023-09-23T20:04:41.815 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.945008081 2023-09-23T20:04:41.815 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.816 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.816 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:04:41.816 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.105242+0000", 2023-09-23T20:04:41.816 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00018872800000000001 2023-09-23T20:04:41.816 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.816 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.816 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:04:41.816 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.105430+0000", 2023-09-23T20:04:41.816 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00027045500000000001 2023-09-23T20:04:41.816 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.816 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.816 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3,5", 2023-09-23T20:04:41.817 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.105701+0000", 2023-09-23T20:04:41.817 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0072222060000000001 2023-09-23T20:04:41.817 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.817 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.817 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:04:41.817 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.112923+0000", 2023-09-23T20:04:41.817 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00026887200000000002 2023-09-23T20:04:41.817 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.817 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.817 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:41.817 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.113192+0000", 2023-09-23T20:04:41.817 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.7439000000000002e-05 2023-09-23T20:04:41.818 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.818 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.818 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:41.818 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.113269+0000", 2023-09-23T20:04:41.818 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000166601 2023-09-23T20:04:41.818 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.818 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.818 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:04:41.818 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.113436+0000", 2023-09-23T20:04:41.818 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000112722 2023-09-23T20:04:41.818 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.819 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.819 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:04:41.819 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.113549+0000", 2023-09-23T20:04:41.819 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.95347298599999997 2023-09-23T20:04:41.819 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:04:41.819 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:04:41.819 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:04:41.819 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.819 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.819 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:85402 14.9 14:99e5f598:::benchmark_data_smithi016_228083_object85401:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e263)", 2023-09-23T20:04:41.819 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:58:25.162001+0000", 2023-09-23T20:04:41.819 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 376.59313255199999, 2023-09-23T20:04:41.820 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.95197446799999996, 2023-09-23T20:04:41.820 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:04:41.820 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:41.820 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:04:41.820 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T20:04:41.820 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:04:41.820 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 85402 2023-09-23T20:04:41.820 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.820 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:04:41.820 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.820 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:04:41.820 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.162001+0000", 2023-09-23T20:04:41.820 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:04:41.821 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.821 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.821 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:04:41.821 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.162001+0000", 2023-09-23T20:04:41.821 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999976 2023-09-23T20:04:41.821 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.821 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.821 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:04:41.821 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.161998+0000", 2023-09-23T20:04:41.821 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.6414999999999993e-05 2023-09-23T20:04:41.821 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.821 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.822 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:04:41.822 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.162065+0000", 2023-09-23T20:04:41.822 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.9888e-05 2023-09-23T20:04:41.822 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.822 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.822 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:04:41.822 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.162085+0000", 2023-09-23T20:04:41.822 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.5710000000000004e-06 2023-09-23T20:04:41.822 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.822 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.822 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:04:41.822 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.162094+0000", 2023-09-23T20:04:41.823 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.2071999999999997e-05 2023-09-23T20:04:41.823 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.823 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.823 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:04:41.823 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.162166+0000", 2023-09-23T20:04:41.823 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.2556e-05 2023-09-23T20:04:41.823 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.823 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.823 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for peered", 2023-09-23T20:04:41.823 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.162179+0000", 2023-09-23T20:04:41.823 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.94696288500000003 2023-09-23T20:04:41.823 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.824 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.824 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:04:41.824 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.109142+0000", 2023-09-23T20:04:41.824 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00022589299999999999 2023-09-23T20:04:41.824 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.824 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.824 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:04:41.824 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.109368+0000", 2023-09-23T20:04:41.824 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000229898 2023-09-23T20:04:41.824 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.824 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.824 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3,5", 2023-09-23T20:04:41.825 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.109598+0000", 2023-09-23T20:04:41.825 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0033289349999999999 2023-09-23T20:04:41.825 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.825 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.825 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:04:41.825 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.112927+0000", 2023-09-23T20:04:41.825 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00071869200000000005 2023-09-23T20:04:41.825 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.825 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.825 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:41.825 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.113645+0000", 2023-09-23T20:04:41.825 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000106327 2023-09-23T20:04:41.826 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.826 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.826 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:41.826 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.113752+0000", 2023-09-23T20:04:41.826 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00012709 2023-09-23T20:04:41.826 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.826 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.826 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:04:41.826 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.113879+0000", 2023-09-23T20:04:41.826 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.6589000000000006e-05 2023-09-23T20:04:41.826 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.826 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.827 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:04:41.827 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.113975+0000", 2023-09-23T20:04:41.827 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.95197446799999996 2023-09-23T20:04:41.827 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:04:41.827 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:04:41.827 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:04:41.827 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.827 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.827 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:85415 14.9 14:9ffb8655:::benchmark_data_smithi016_228083_object85414:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e263)", 2023-09-23T20:04:41.827 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:58:25.175093+0000", 2023-09-23T20:04:41.827 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 376.58004061299999, 2023-09-23T20:04:41.827 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.96229291100000003, 2023-09-23T20:04:41.828 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:04:41.828 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:41.828 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:04:41.828 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T20:04:41.828 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:04:41.828 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 85415 2023-09-23T20:04:41.828 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.828 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:04:41.828 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.828 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:04:41.828 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.175093+0000", 2023-09-23T20:04:41.828 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:04:41.828 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.829 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.829 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:04:41.829 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.175093+0000", 2023-09-23T20:04:41.829 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999976 2023-09-23T20:04:41.829 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.829 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.829 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:04:41.829 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.175091+0000", 2023-09-23T20:04:41.829 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000123707 2023-09-23T20:04:41.829 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.829 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.829 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:04:41.830 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.175214+0000", 2023-09-23T20:04:41.830 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.0718999999999999e-05 2023-09-23T20:04:41.830 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.830 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.830 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:04:41.830 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.175235+0000", 2023-09-23T20:04:41.830 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.2517000000000001e-05 2023-09-23T20:04:41.830 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.830 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.830 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:04:41.830 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.175247+0000", 2023-09-23T20:04:41.831 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.6070999999999996e-05 2023-09-23T20:04:41.831 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.831 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.831 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:04:41.831 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.175334+0000", 2023-09-23T20:04:41.831 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.5437000000000001e-05 2023-09-23T20:04:41.831 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.831 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.831 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for peered", 2023-09-23T20:04:41.831 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.175349+0000", 2023-09-23T20:04:41.831 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.93866744700000004 2023-09-23T20:04:41.831 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.831 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.832 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:04:41.832 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.114016+0000", 2023-09-23T20:04:41.832 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000222554 2023-09-23T20:04:41.832 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.832 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.832 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:04:41.832 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.114239+0000", 2023-09-23T20:04:41.832 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00034813200000000002 2023-09-23T20:04:41.832 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.832 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.832 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3,5", 2023-09-23T20:04:41.832 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.114587+0000", 2023-09-23T20:04:41.833 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0060094249999999997 2023-09-23T20:04:41.833 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.833 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.833 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:41.833 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.120597+0000", 2023-09-23T20:04:41.833 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0069605939999999996 2023-09-23T20:04:41.833 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.833 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.833 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:04:41.833 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.127557+0000", 2023-09-23T20:04:41.833 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0097405860000000007 2023-09-23T20:04:41.833 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.834 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.834 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:41.834 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.137298+0000", 2023-09-23T20:04:41.834 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.1941000000000003e-05 2023-09-23T20:04:41.834 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.834 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.834 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:04:41.834 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.137350+0000", 2023-09-23T20:04:41.834 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.5941e-05 2023-09-23T20:04:41.834 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.834 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.834 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:04:41.834 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.137386+0000", 2023-09-23T20:04:41.835 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.96229291100000003 2023-09-23T20:04:41.835 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:04:41.835 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:04:41.835 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:04:41.835 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.835 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.835 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:85421 14.9 14:91e9f1e1:::benchmark_data_smithi016_228083_object85420:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e263)", 2023-09-23T20:04:41.835 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:58:25.183117+0000", 2023-09-23T20:04:41.835 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 376.57201634, 2023-09-23T20:04:41.835 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.954422993, 2023-09-23T20:04:41.835 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:04:41.835 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:41.836 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:04:41.836 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T20:04:41.836 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:04:41.836 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 85421 2023-09-23T20:04:41.836 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.836 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:04:41.836 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.836 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:04:41.836 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.183117+0000", 2023-09-23T20:04:41.836 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:04:41.836 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.836 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.837 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:04:41.837 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.183117+0000", 2023-09-23T20:04:41.837 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.999999 2023-09-23T20:04:41.837 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.837 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.837 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:04:41.837 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.183116+0000", 2023-09-23T20:04:41.837 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.1011000000000002e-05 2023-09-23T20:04:41.837 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.837 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.837 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:04:41.837 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.183187+0000", 2023-09-23T20:04:41.837 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.235e-05 2023-09-23T20:04:41.838 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.838 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.838 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:04:41.838 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.183199+0000", 2023-09-23T20:04:41.838 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.17e-06 2023-09-23T20:04:41.838 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.838 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.838 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:04:41.838 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.183206+0000", 2023-09-23T20:04:41.838 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.0999000000000003e-05 2023-09-23T20:04:41.838 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.839 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.839 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:04:41.839 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.183267+0000", 2023-09-23T20:04:41.839 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.7880000000000001e-06 2023-09-23T20:04:41.839 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.839 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.839 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for peered", 2023-09-23T20:04:41.839 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.183277+0000", 2023-09-23T20:04:41.839 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.93473318400000005 2023-09-23T20:04:41.839 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.839 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.839 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:04:41.839 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.118010+0000", 2023-09-23T20:04:41.840 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00018186100000000001 2023-09-23T20:04:41.840 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.840 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.840 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:04:41.840 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.118192+0000", 2023-09-23T20:04:41.840 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00013684600000000001 2023-09-23T20:04:41.840 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.840 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.840 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3,5", 2023-09-23T20:04:41.840 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.118329+0000", 2023-09-23T20:04:41.840 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.012081743000000001 2023-09-23T20:04:41.840 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.841 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.841 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:41.841 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.130411+0000", 2023-09-23T20:04:41.841 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.4800000000000007e-06 2023-09-23T20:04:41.841 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.841 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.841 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:04:41.841 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.130420+0000", 2023-09-23T20:04:41.841 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0070648070000000002 2023-09-23T20:04:41.841 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.841 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.841 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:41.842 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.137485+0000", 2023-09-23T20:04:41.842 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.9612e-05 2023-09-23T20:04:41.842 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.842 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.842 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:04:41.842 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.137515+0000", 2023-09-23T20:04:41.842 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.5318000000000001e-05 2023-09-23T20:04:41.842 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.842 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.842 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:04:41.842 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.137540+0000", 2023-09-23T20:04:41.842 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.954422993 2023-09-23T20:04:41.843 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:04:41.843 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:04:41.843 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:04:41.843 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.843 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.843 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:85441 14.9 14:9c17bf8c:::benchmark_data_smithi016_228083_object85440:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e263)", 2023-09-23T20:04:41.843 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:58:25.209256+0000", 2023-09-23T20:04:41.843 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 376.54587747199997, 2023-09-23T20:04:41.843 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.94155818099999999, 2023-09-23T20:04:41.843 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:04:41.843 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:41.843 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:04:41.844 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T20:04:41.844 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:04:41.844 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 85441 2023-09-23T20:04:41.844 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.844 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:04:41.844 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.844 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:04:41.844 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.209256+0000", 2023-09-23T20:04:41.844 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:04:41.844 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.844 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.844 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:04:41.845 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.209256+0000", 2023-09-23T20:04:41.845 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T20:04:41.845 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.845 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.845 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:04:41.845 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.209254+0000", 2023-09-23T20:04:41.845 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.1990999999999996e-05 2023-09-23T20:04:41.845 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.845 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.845 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:04:41.845 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.209326+0000", 2023-09-23T20:04:41.845 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.3918e-05 2023-09-23T20:04:41.845 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.846 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.846 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:04:41.846 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.209340+0000", 2023-09-23T20:04:41.846 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.5520000000000001e-06 2023-09-23T20:04:41.846 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.846 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.846 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:04:41.846 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.209347+0000", 2023-09-23T20:04:41.846 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.1537000000000004e-05 2023-09-23T20:04:41.846 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.847 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.847 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:04:41.847 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.209408+0000", 2023-09-23T20:04:41.847 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.3186e-05 2023-09-23T20:04:41.847 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.847 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.847 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for peered", 2023-09-23T20:04:41.847 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.209422+0000", 2023-09-23T20:04:41.847 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.91123985900000004 2023-09-23T20:04:41.847 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.847 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.847 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:04:41.848 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.120661+0000", 2023-09-23T20:04:41.848 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00015342700000000001 2023-09-23T20:04:41.848 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.848 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.848 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:04:41.848 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.120815+0000", 2023-09-23T20:04:41.848 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000244866 2023-09-23T20:04:41.848 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.848 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.848 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3,5", 2023-09-23T20:04:41.848 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.121060+0000", 2023-09-23T20:04:41.848 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0093679850000000005 2023-09-23T20:04:41.849 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.849 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.849 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:04:41.849 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.130428+0000", 2023-09-23T20:04:41.849 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.4576999999999997e-05 2023-09-23T20:04:41.849 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.849 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.849 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:41.849 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.130472+0000", 2023-09-23T20:04:41.849 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.020209061 2023-09-23T20:04:41.849 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.849 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.850 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:41.850 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.150681+0000", 2023-09-23T20:04:41.850 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.9345000000000006e-05 2023-09-23T20:04:41.850 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.850 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.850 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:04:41.850 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.150771+0000", 2023-09-23T20:04:41.850 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.3334999999999998e-05 2023-09-23T20:04:41.850 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.850 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.850 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:04:41.850 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.150814+0000", 2023-09-23T20:04:41.851 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.94155818099999999 2023-09-23T20:04:41.851 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:04:41.851 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:04:41.851 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:04:41.851 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.851 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.851 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:85442 14.9 14:9e459d4c:::benchmark_data_smithi016_228083_object85441:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e263)", 2023-09-23T20:04:41.851 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:58:25.209357+0000", 2023-09-23T20:04:41.851 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 376.54577673, 2023-09-23T20:04:41.851 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.94168814999999995, 2023-09-23T20:04:41.851 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:04:41.852 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:41.852 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:04:41.852 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T20:04:41.852 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:04:41.852 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 85442 2023-09-23T20:04:41.852 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.852 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:04:41.852 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.852 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:04:41.852 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.209357+0000", 2023-09-23T20:04:41.852 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:04:41.852 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.853 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.853 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:04:41.853 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.209357+0000", 2023-09-23T20:04:41.853 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999995 2023-09-23T20:04:41.853 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.853 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.853 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:04:41.853 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.209356+0000", 2023-09-23T20:04:41.853 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.3260999999999999e-05 2023-09-23T20:04:41.853 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.853 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.853 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:04:41.854 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.209389+0000", 2023-09-23T20:04:41.854 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.092e-06 2023-09-23T20:04:41.854 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.854 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.854 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:04:41.854 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.209396+0000", 2023-09-23T20:04:41.854 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.2390000000000002e-06 2023-09-23T20:04:41.854 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.854 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.854 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:04:41.854 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.209402+0000", 2023-09-23T20:04:41.854 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.3684999999999999e-05 2023-09-23T20:04:41.855 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.855 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.855 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:04:41.855 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.209445+0000", 2023-09-23T20:04:41.855 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.6279999999999996e-06 2023-09-23T20:04:41.855 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.855 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.855 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for peered", 2023-09-23T20:04:41.855 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.209453+0000", 2023-09-23T20:04:41.855 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.91469508200000005 2023-09-23T20:04:41.855 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.855 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.856 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:04:41.856 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.124148+0000", 2023-09-23T20:04:41.856 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00014875500000000001 2023-09-23T20:04:41.856 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.856 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.856 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:04:41.856 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.124297+0000", 2023-09-23T20:04:41.856 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000203897 2023-09-23T20:04:41.856 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.856 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.856 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3,5", 2023-09-23T20:04:41.856 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.124501+0000", 2023-09-23T20:04:41.857 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0060093730000000001 2023-09-23T20:04:41.857 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.857 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.857 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:41.857 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.130510+0000", 2023-09-23T20:04:41.857 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.2470000000000008e-06 2023-09-23T20:04:41.857 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.857 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.857 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:04:41.857 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.130518+0000", 2023-09-23T20:04:41.857 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.020432044 2023-09-23T20:04:41.857 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.857 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.858 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:41.858 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.150950+0000", 2023-09-23T20:04:41.858 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.5896000000000002e-05 2023-09-23T20:04:41.858 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.858 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.858 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:04:41.858 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.151006+0000", 2023-09-23T20:04:41.858 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.8624999999999997e-05 2023-09-23T20:04:41.858 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.858 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.858 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:04:41.859 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.151045+0000", 2023-09-23T20:04:41.859 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.94168814999999995 2023-09-23T20:04:41.859 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:04:41.859 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:04:41.859 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:04:41.859 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.859 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.859 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:85497 14.9 14:9cb615a6:::benchmark_data_smithi016_228083_object85496:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e263)", 2023-09-23T20:04:41.859 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:58:25.281257+0000", 2023-09-23T20:04:41.859 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 376.47387590599999, 2023-09-23T20:04:41.859 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.88141183400000001, 2023-09-23T20:04:41.859 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:04:41.860 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:41.860 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:04:41.860 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T20:04:41.860 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:04:41.860 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 85497 2023-09-23T20:04:41.860 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.860 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:04:41.860 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.860 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:04:41.860 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.281257+0000", 2023-09-23T20:04:41.860 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:04:41.860 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.861 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.861 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:04:41.861 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.281257+0000", 2023-09-23T20:04:41.861 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T20:04:41.861 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.861 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.861 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:04:41.861 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.281255+0000", 2023-09-23T20:04:41.861 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.4725000000000003e-05 2023-09-23T20:04:41.861 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.861 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.861 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:04:41.862 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.281300+0000", 2023-09-23T20:04:41.862 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.0341999999999999e-05 2023-09-23T20:04:41.862 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.862 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.862 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:04:41.862 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.281320+0000", 2023-09-23T20:04:41.862 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.6930000000000002e-06 2023-09-23T20:04:41.862 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.862 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.862 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:04:41.862 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.281330+0000", 2023-09-23T20:04:41.862 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.6923999999999998e-05 2023-09-23T20:04:41.863 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.863 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.863 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:04:41.863 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.281397+0000", 2023-09-23T20:04:41.863 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.5629999999999992e-06 2023-09-23T20:04:41.863 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.863 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.863 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for peered", 2023-09-23T20:04:41.863 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.281406+0000", 2023-09-23T20:04:41.863 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.84619651600000001 2023-09-23T20:04:41.863 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.863 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.864 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:04:41.864 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.127602+0000", 2023-09-23T20:04:41.864 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.2312999999999995e-05 2023-09-23T20:04:41.864 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.864 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.864 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:04:41.864 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.127694+0000", 2023-09-23T20:04:41.864 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00012638600000000001 2023-09-23T20:04:41.864 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.864 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.864 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3,5", 2023-09-23T20:04:41.864 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.127821+0000", 2023-09-23T20:04:41.865 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0094256010000000005 2023-09-23T20:04:41.865 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.865 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.865 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:41.865 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.137246+0000", 2023-09-23T20:04:41.865 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00014908599999999999 2023-09-23T20:04:41.865 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.865 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.865 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:04:41.865 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.137395+0000", 2023-09-23T20:04:41.865 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.025132723999999999 2023-09-23T20:04:41.865 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.866 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.866 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:41.866 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.162528+0000", 2023-09-23T20:04:41.866 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.1192000000000005e-05 2023-09-23T20:04:41.866 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.866 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.866 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:04:41.866 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.162609+0000", 2023-09-23T20:04:41.866 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.9874000000000003e-05 2023-09-23T20:04:41.866 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.866 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.866 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:04:41.867 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.162669+0000", 2023-09-23T20:04:41.867 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.88141183400000001 2023-09-23T20:04:41.867 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:04:41.867 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:04:41.867 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:04:41.867 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.867 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.867 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:85498 14.9 14:941b5f4a:::benchmark_data_smithi016_228083_object85497:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e263)", 2023-09-23T20:04:41.867 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:58:25.285826+0000", 2023-09-23T20:04:41.867 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 376.46930740300002, 2023-09-23T20:04:41.867 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.87697232700000005, 2023-09-23T20:04:41.867 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:04:41.868 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:41.868 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:04:41.868 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T20:04:41.868 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:04:41.868 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 85498 2023-09-23T20:04:41.868 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.868 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:04:41.868 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.868 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:04:41.868 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.285826+0000", 2023-09-23T20:04:41.868 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:04:41.868 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.869 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.869 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:04:41.869 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.285826+0000", 2023-09-23T20:04:41.869 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999948 2023-09-23T20:04:41.869 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.869 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.869 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:04:41.869 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.285821+0000", 2023-09-23T20:04:41.869 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00010835 2023-09-23T20:04:41.869 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.869 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.869 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:04:41.870 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.285929+0000", 2023-09-23T20:04:41.870 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.4283000000000001e-05 2023-09-23T20:04:41.870 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.870 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.870 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:04:41.870 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.285953+0000", 2023-09-23T20:04:41.870 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.1270000000000001e-05 2023-09-23T20:04:41.870 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.870 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.870 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:04:41.870 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.285964+0000", 2023-09-23T20:04:41.870 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00012742600000000001 2023-09-23T20:04:41.871 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.871 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.871 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:04:41.871 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.286092+0000", 2023-09-23T20:04:41.871 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.6348999999999999e-05 2023-09-23T20:04:41.871 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.871 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.871 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for peered", 2023-09-23T20:04:41.871 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.286108+0000", 2023-09-23T20:04:41.871 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.844440776 2023-09-23T20:04:41.871 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.873 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.873 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:04:41.873 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.130549+0000", 2023-09-23T20:04:41.873 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000102414 2023-09-23T20:04:41.873 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.873 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.873 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:04:41.873 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.130651+0000", 2023-09-23T20:04:41.873 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000111272 2023-09-23T20:04:41.873 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.874 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.874 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3,5", 2023-09-23T20:04:41.874 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.130763+0000", 2023-09-23T20:04:41.874 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0066374440000000002 2023-09-23T20:04:41.874 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.874 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.874 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:04:41.874 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.137400+0000", 2023-09-23T20:04:41.874 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.7243999999999998e-05 2023-09-23T20:04:41.874 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.874 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.875 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:41.875 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.137437+0000", 2023-09-23T20:04:41.875 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.025275355999999999 2023-09-23T20:04:41.875 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.875 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.875 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:41.875 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.162713+0000", 2023-09-23T20:04:41.875 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.1664e-05 2023-09-23T20:04:41.875 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.875 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.875 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:04:41.875 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.162754+0000", 2023-09-23T20:04:41.876 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.3869000000000003e-05 2023-09-23T20:04:41.876 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.876 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.876 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:04:41.876 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.162798+0000", 2023-09-23T20:04:41.876 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.87697232700000005 2023-09-23T20:04:41.876 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:04:41.876 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:04:41.876 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:04:41.876 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.876 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.876 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:85525 14.9 14:9de208c1:::benchmark_data_smithi016_228083_object85524:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e263)", 2023-09-23T20:04:41.877 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:58:25.332416+0000", 2023-09-23T20:04:41.877 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 376.42271694200002, 2023-09-23T20:04:41.877 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.83059830099999998, 2023-09-23T20:04:41.877 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:04:41.877 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:41.877 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:04:41.877 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T20:04:41.877 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:04:41.877 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 85525 2023-09-23T20:04:41.877 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.877 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:04:41.877 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.878 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:04:41.878 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.332416+0000", 2023-09-23T20:04:41.878 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:04:41.878 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.878 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.878 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:04:41.878 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.332416+0000", 2023-09-23T20:04:41.878 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T20:04:41.878 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.878 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.878 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:04:41.878 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.332414+0000", 2023-09-23T20:04:41.879 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.3407e-05 2023-09-23T20:04:41.879 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.879 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.879 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:04:41.879 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.332458+0000", 2023-09-23T20:04:41.879 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.7921e-05 2023-09-23T20:04:41.879 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.879 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.879 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:04:41.879 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.332476+0000", 2023-09-23T20:04:41.879 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.2570000000000007e-06 2023-09-23T20:04:41.879 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.880 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.880 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:04:41.880 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.332484+0000", 2023-09-23T20:04:41.880 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.5447999999999999e-05 2023-09-23T20:04:41.880 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.880 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.880 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:04:41.880 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.332569+0000", 2023-09-23T20:04:41.880 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.2368000000000001e-05 2023-09-23T20:04:41.880 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.880 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.880 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for peered", 2023-09-23T20:04:41.880 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.332582+0000", 2023-09-23T20:04:41.881 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.80044044299999995 2023-09-23T20:04:41.881 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.881 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.881 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:04:41.881 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.133022+0000", 2023-09-23T20:04:41.881 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.2608999999999994e-05 2023-09-23T20:04:41.881 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.881 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.881 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:04:41.881 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.133115+0000", 2023-09-23T20:04:41.881 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00012925100000000001 2023-09-23T20:04:41.881 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.882 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.882 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3,5", 2023-09-23T20:04:41.882 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.133244+0000", 2023-09-23T20:04:41.882 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0043033309999999996 2023-09-23T20:04:41.882 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.882 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.882 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:04:41.882 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.137547+0000", 2023-09-23T20:04:41.882 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0058089120000000003 2023-09-23T20:04:41.882 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.882 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.882 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:41.883 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.143356+0000", 2023-09-23T20:04:41.883 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.019509450000000001 2023-09-23T20:04:41.883 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.883 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.883 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:41.883 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.162866+0000", 2023-09-23T20:04:41.883 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.0210999999999996e-05 2023-09-23T20:04:41.883 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.883 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.883 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:04:41.883 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.162936+0000", 2023-09-23T20:04:41.884 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.8739999999999995e-05 2023-09-23T20:04:41.884 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.884 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.884 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:04:41.884 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.163015+0000", 2023-09-23T20:04:41.884 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.83059830099999998 2023-09-23T20:04:41.884 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:04:41.884 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:04:41.884 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:04:41.884 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.884 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.884 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:85534 14.9 14:9a53a49a:::benchmark_data_smithi016_228083_object85533:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e263)", 2023-09-23T20:04:41.885 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:58:25.353246+0000", 2023-09-23T20:04:41.885 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 376.40188700300001, 2023-09-23T20:04:41.885 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.80997894299999995, 2023-09-23T20:04:41.885 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:04:41.885 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:41.885 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:04:41.885 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T20:04:41.885 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:04:41.885 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 85534 2023-09-23T20:04:41.885 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.885 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:04:41.885 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.885 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:04:41.886 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.353246+0000", 2023-09-23T20:04:41.886 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:04:41.886 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.886 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.886 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:04:41.886 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.353246+0000", 2023-09-23T20:04:41.886 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T20:04:41.886 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.886 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.886 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:04:41.886 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.353244+0000", 2023-09-23T20:04:41.886 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.6495000000000002e-05 2023-09-23T20:04:41.887 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.887 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.887 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:04:41.887 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.353281+0000", 2023-09-23T20:04:41.887 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.9864000000000002e-05 2023-09-23T20:04:41.887 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.887 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.887 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:04:41.887 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.353301+0000", 2023-09-23T20:04:41.887 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.7470000000000005e-06 2023-09-23T20:04:41.887 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.887 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.888 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:04:41.888 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.353308+0000", 2023-09-23T20:04:41.888 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00014357199999999999 2023-09-23T20:04:41.888 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.888 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.888 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:04:41.888 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.353452+0000", 2023-09-23T20:04:41.888 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.1129e-05 2023-09-23T20:04:41.888 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.888 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.888 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for peered", 2023-09-23T20:04:41.889 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.353473+0000", 2023-09-23T20:04:41.889 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.78173484299999996 2023-09-23T20:04:41.889 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.889 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.889 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:04:41.889 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.135208+0000", 2023-09-23T20:04:41.889 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.6490999999999996e-05 2023-09-23T20:04:41.889 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.889 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.889 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:04:41.889 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.135294+0000", 2023-09-23T20:04:41.889 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00013405099999999999 2023-09-23T20:04:41.890 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.890 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.890 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3,5", 2023-09-23T20:04:41.890 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.135428+0000", 2023-09-23T20:04:41.890 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.002123748 2023-09-23T20:04:41.890 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.890 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.890 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:04:41.890 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.137552+0000", 2023-09-23T20:04:41.890 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.012916637999999999 2023-09-23T20:04:41.890 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.890 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.891 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:41.891 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.150469+0000", 2023-09-23T20:04:41.891 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.012647726 2023-09-23T20:04:41.891 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.891 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.891 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:41.891 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.163117+0000", 2023-09-23T20:04:41.891 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.3090999999999996e-05 2023-09-23T20:04:41.891 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.891 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.891 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:04:41.891 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.163180+0000", 2023-09-23T20:04:41.892 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.5578000000000003e-05 2023-09-23T20:04:41.892 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.892 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.892 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:04:41.892 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.163225+0000", 2023-09-23T20:04:41.892 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.80997894299999995 2023-09-23T20:04:41.892 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:04:41.892 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:04:41.892 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:04:41.892 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.892 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.892 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:85537 14.9 14:9e1b9eb0:::benchmark_data_smithi016_228083_object85536:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e263)", 2023-09-23T20:04:41.893 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:58:25.362809+0000", 2023-09-23T20:04:41.893 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 376.39232389699998, 2023-09-23T20:04:41.893 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.80056966299999999, 2023-09-23T20:04:41.893 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:04:41.893 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:41.893 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:04:41.893 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T20:04:41.893 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:04:41.893 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 85537 2023-09-23T20:04:41.893 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.893 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:04:41.893 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.894 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:04:41.894 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.362809+0000", 2023-09-23T20:04:41.894 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:04:41.894 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.894 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.894 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:04:41.894 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.362809+0000", 2023-09-23T20:04:41.894 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T20:04:41.894 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.894 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.894 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:04:41.894 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.362808+0000", 2023-09-23T20:04:41.895 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.9162000000000002e-05 2023-09-23T20:04:41.895 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.895 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.895 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:04:41.895 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.362847+0000", 2023-09-23T20:04:41.895 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.7776999999999999e-05 2023-09-23T20:04:41.895 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.895 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.895 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:04:41.895 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.362865+0000", 2023-09-23T20:04:41.895 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.1179999999999992e-06 2023-09-23T20:04:41.895 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.895 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.896 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:04:41.896 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.362873+0000", 2023-09-23T20:04:41.896 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00014888999999999999 2023-09-23T20:04:41.896 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.896 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.896 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:04:41.896 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.363022+0000", 2023-09-23T20:04:41.896 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.7234000000000001e-05 2023-09-23T20:04:41.896 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.896 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.896 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for peered", 2023-09-23T20:04:41.896 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.363039+0000", 2023-09-23T20:04:41.897 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.77452688599999997 2023-09-23T20:04:41.897 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.897 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.897 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:04:41.897 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.137566+0000", 2023-09-23T20:04:41.897 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.9876999999999993e-05 2023-09-23T20:04:41.897 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.897 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.897 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:04:41.897 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.137656+0000", 2023-09-23T20:04:41.897 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00014732300000000001 2023-09-23T20:04:41.897 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.898 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.898 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3,5", 2023-09-23T20:04:41.898 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.137803+0000", 2023-09-23T20:04:41.898 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0055677629999999999 2023-09-23T20:04:41.898 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.898 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.898 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:04:41.898 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.143371+0000", 2023-09-23T20:04:41.898 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0072004060000000003 2023-09-23T20:04:41.898 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.898 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.899 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:41.899 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.150571+0000", 2023-09-23T20:04:41.899 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0127008 2023-09-23T20:04:41.899 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.899 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.899 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:41.899 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.163272+0000", 2023-09-23T20:04:41.899 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.4979999999999999e-05 2023-09-23T20:04:41.899 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.899 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.899 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:04:41.899 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.163327+0000", 2023-09-23T20:04:41.900 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.1836000000000003e-05 2023-09-23T20:04:41.900 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.900 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.900 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:04:41.900 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.163379+0000", 2023-09-23T20:04:41.900 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.80056966299999999 2023-09-23T20:04:41.900 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:04:41.900 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:04:41.900 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:04:41.900 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.900 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.900 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:85542 14.9 14:9a9e0629:::benchmark_data_smithi016_228083_object85541:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e263)", 2023-09-23T20:04:41.900 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:58:25.378606+0000", 2023-09-23T20:04:41.901 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 376.37652727800003, 2023-09-23T20:04:41.901 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.78504788400000003, 2023-09-23T20:04:41.901 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:04:41.901 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:41.901 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:04:41.901 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T20:04:41.901 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:04:41.901 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 85542 2023-09-23T20:04:41.901 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.901 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:04:41.901 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.901 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:04:41.902 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.378606+0000", 2023-09-23T20:04:41.902 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:04:41.902 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.902 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.902 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:04:41.902 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.378606+0000", 2023-09-23T20:04:41.902 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999971 2023-09-23T20:04:41.902 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.902 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.902 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:04:41.902 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.378603+0000", 2023-09-23T20:04:41.903 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.7144e-05 2023-09-23T20:04:41.903 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.903 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.903 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:04:41.903 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.378660+0000", 2023-09-23T20:04:41.903 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.9719000000000001e-05 2023-09-23T20:04:41.903 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.903 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.903 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:04:41.903 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.378690+0000", 2023-09-23T20:04:41.903 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.2594e-05 2023-09-23T20:04:41.903 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.903 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.904 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:04:41.904 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.378703+0000", 2023-09-23T20:04:41.904 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00013042800000000001 2023-09-23T20:04:41.904 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.904 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.904 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:04:41.904 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.378833+0000", 2023-09-23T20:04:41.904 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.9785000000000001e-05 2023-09-23T20:04:41.904 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.904 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.904 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for peered", 2023-09-23T20:04:41.904 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.378853+0000", 2023-09-23T20:04:41.905 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.76109633899999996 2023-09-23T20:04:41.905 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.905 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.905 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:04:41.905 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.139949+0000", 2023-09-23T20:04:41.905 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000213799 2023-09-23T20:04:41.905 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.905 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.905 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:04:41.905 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.140163+0000", 2023-09-23T20:04:41.905 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00030738800000000002 2023-09-23T20:04:41.905 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.905 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.906 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3,5", 2023-09-23T20:04:41.906 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.140470+0000", 2023-09-23T20:04:41.906 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.010119651 2023-09-23T20:04:41.906 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.906 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.906 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:04:41.906 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.150590+0000", 2023-09-23T20:04:41.906 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00028541800000000001 2023-09-23T20:04:41.906 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.906 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.906 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:41.906 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.150875+0000", 2023-09-23T20:04:41.907 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.012635431000000001 2023-09-23T20:04:41.907 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.907 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.907 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:41.907 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.163511+0000", 2023-09-23T20:04:41.907 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.7058999999999996e-05 2023-09-23T20:04:41.907 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.907 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.907 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:04:41.907 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.163578+0000", 2023-09-23T20:04:41.907 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.5988999999999994e-05 2023-09-23T20:04:41.907 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.908 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.908 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:04:41.908 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.163654+0000", 2023-09-23T20:04:41.908 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.78504788400000003 2023-09-23T20:04:41.908 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:04:41.908 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:04:41.908 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:04:41.908 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.908 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.908 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:85559 14.9 14:923b20ba:::benchmark_data_smithi016_228083_object85558:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e263)", 2023-09-23T20:04:41.908 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:58:25.463564+0000", 2023-09-23T20:04:41.908 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 376.29156956200001, 2023-09-23T20:04:41.909 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.70025278099999999, 2023-09-23T20:04:41.909 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:04:41.909 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:41.909 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:04:41.909 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T20:04:41.909 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:04:41.909 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 85559 2023-09-23T20:04:41.909 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.909 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:04:41.909 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.909 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:04:41.909 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.463564+0000", 2023-09-23T20:04:41.910 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:04:41.910 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.910 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.910 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:04:41.910 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.463564+0000", 2023-09-23T20:04:41.910 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999967 2023-09-23T20:04:41.910 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.910 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.910 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:04:41.910 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.463561+0000", 2023-09-23T20:04:41.910 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.0883999999999994e-05 2023-09-23T20:04:41.910 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.911 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.911 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:04:41.911 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.463631+0000", 2023-09-23T20:04:41.911 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.2996000000000001e-05 2023-09-23T20:04:41.911 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.911 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.911 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:04:41.911 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.463664+0000", 2023-09-23T20:04:41.911 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.0069999999999999e-05 2023-09-23T20:04:41.911 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.911 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.911 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:04:41.911 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.463684+0000", 2023-09-23T20:04:41.912 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00019521600000000001 2023-09-23T20:04:41.912 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.912 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.912 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:04:41.912 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.463880+0000", 2023-09-23T20:04:41.912 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.016e-05 2023-09-23T20:04:41.912 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.912 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.912 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for peered", 2023-09-23T20:04:41.912 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:25.463900+0000", 2023-09-23T20:04:41.912 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.67952702200000004 2023-09-23T20:04:41.913 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.913 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.913 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:04:41.913 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.143427+0000", 2023-09-23T20:04:41.913 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000119385 2023-09-23T20:04:41.913 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.913 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.913 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:04:41.913 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.143546+0000", 2023-09-23T20:04:41.913 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00021470500000000001 2023-09-23T20:04:41.913 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.913 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.914 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3,5", 2023-09-23T20:04:41.914 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.143761+0000", 2023-09-23T20:04:41.914 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0068445800000000003 2023-09-23T20:04:41.914 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.914 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.914 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:04:41.914 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.150606+0000", 2023-09-23T20:04:41.914 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00049718400000000002 2023-09-23T20:04:41.914 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.914 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.914 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:41.915 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.151103+0000", 2023-09-23T20:04:41.915 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.012612732999999999 2023-09-23T20:04:41.915 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.915 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.915 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:41.915 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.163715+0000", 2023-09-23T20:04:41.915 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.1959000000000002e-05 2023-09-23T20:04:41.915 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.915 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.915 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:04:41.915 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.163767+0000", 2023-09-23T20:04:41.915 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.9100000000000001e-05 2023-09-23T20:04:41.915 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:04:41.916 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:04:41.916 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:04:41.916 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:26.163817+0000", 2023-09-23T20:04:41.916 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.70025278099999999 2023-09-23T20:04:41.916 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:04:41.916 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:04:41.916 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:04:41.916 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:04:41.916 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:04:41.916 INFO:teuthology.orchestra.run.smithi092.stdout:} 2023-09-23T20:04:41.999 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:41 smithi016 bash[75004]: cluster 2023-09-23T20:04:40.727810+0000 mgr.y (mgr.25325) 1725 : cluster 0 pgmap v1806: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:42.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:42 smithi092 bash[58855]: cluster 2023-09-23T20:04:40.727810+0000 mgr.y (mgr.25325) 1725 : cluster 0 pgmap v1806: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:42.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:41 smithi016 bash[77995]: cluster 2023-09-23T20:04:40.727810+0000 mgr.y (mgr.25325) 1725 : cluster 0 pgmap v1806: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:42.676 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T20:04:44.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:43 smithi016 bash[75004]: cluster 2023-09-23T20:04:42.728694+0000 mgr.y (mgr.25325) 1726 : cluster 0 pgmap v1807: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:44.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:43 smithi016 bash[75004]: audit 2023-09-23T20:04:43.703192+0000 mon.a (mon.0) 2093 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:04:44.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:44 smithi016 bash[77995]: cluster 2023-09-23T20:04:42.728694+0000 mgr.y (mgr.25325) 1726 : cluster 0 pgmap v1807: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:44.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:44 smithi016 bash[77995]: audit 2023-09-23T20:04:43.703192+0000 mon.a (mon.0) 2093 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:04:44.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:44 smithi092 bash[58855]: cluster 2023-09-23T20:04:42.728694+0000 mgr.y (mgr.25325) 1726 : cluster 0 pgmap v1807: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:44.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:44 smithi092 bash[58855]: audit 2023-09-23T20:04:43.703192+0000 mon.a (mon.0) 2093 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:04:46.020 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:46 smithi016 bash[77995]: cluster 2023-09-23T20:04:44.729414+0000 mgr.y (mgr.25325) 1727 : cluster 0 pgmap v1808: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:46.021 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:46 smithi016 bash[77995]: audit 2023-09-23T20:04:45.483697+0000 mon.a (mon.0) 2094 : audit 1 from='client.? 172.21.15.16:0/1545229384' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:04:46.021 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:04:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:20:04:45] "GET /metrics HTTP/1.1" 200 34016 "" "Prometheus/2.33.4" 2023-09-23T20:04:46.024 INFO:teuthology.orchestra.run.smithi016.stderr:set osd.3 primary-affinity to 1 (8655362) 2023-09-23T20:04:46.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:46 smithi016 bash[75004]: cluster 2023-09-23T20:04:44.729414+0000 mgr.y (mgr.25325) 1727 : cluster 0 pgmap v1808: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:46.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:46 smithi016 bash[75004]: audit 2023-09-23T20:04:45.483697+0000 mon.a (mon.0) 2094 : audit 1 from='client.? 172.21.15.16:0/1545229384' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:04:46.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:46 smithi092 bash[58855]: cluster 2023-09-23T20:04:44.729414+0000 mgr.y (mgr.25325) 1727 : cluster 0 pgmap v1808: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:46.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:46 smithi092 bash[58855]: audit 2023-09-23T20:04:45.483697+0000 mon.a (mon.0) 2094 : audit 1 from='client.? 172.21.15.16:0/1545229384' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:04:46.982 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T20:04:47.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:47 smithi016 bash[75004]: audit 2023-09-23T20:04:46.010875+0000 mon.a (mon.0) 2095 : audit 1 from='client.? 172.21.15.16:0/1545229384' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:04:47.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:47 smithi016 bash[75004]: cluster 2023-09-23T20:04:46.011030+0000 mon.a (mon.0) 2096 : cluster 0 osdmap e321: 8 total, 8 up, 8 in 2023-09-23T20:04:47.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:47 smithi016 bash[75004]: cluster 2023-09-23T20:04:46.014954+0000 osd.7 (osd.7) 244 : cluster 3 failed to encode map e321 with expected crc 2023-09-23T20:04:47.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:47 smithi016 bash[75004]: cluster 2023-09-23T20:04:46.018396+0000 osd.6 (osd.6) 259 : cluster 3 failed to encode map e321 with expected crc 2023-09-23T20:04:47.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:47 smithi016 bash[75004]: cluster 2023-09-23T20:04:46.018997+0000 osd.4 (osd.4) 268 : cluster 3 failed to encode map e321 with expected crc 2023-09-23T20:04:47.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:47 smithi016 bash[75004]: cluster 2023-09-23T20:04:46.019292+0000 osd.5 (osd.5) 240 : cluster 3 failed to encode map e321 with expected crc 2023-09-23T20:04:47.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:47 smithi016 bash[75004]: cluster 2023-09-23T20:04:47.013437+0000 mon.a (mon.0) 2097 : cluster 0 osdmap e322: 8 total, 8 up, 8 in 2023-09-23T20:04:47.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:47 smithi016 bash[77995]: audit 2023-09-23T20:04:46.010875+0000 mon.a (mon.0) 2095 : audit 1 from='client.? 172.21.15.16:0/1545229384' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:04:47.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:47 smithi016 bash[77995]: cluster 2023-09-23T20:04:46.011030+0000 mon.a (mon.0) 2096 : cluster 0 osdmap e321: 8 total, 8 up, 8 in 2023-09-23T20:04:47.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:47 smithi016 bash[77995]: cluster 2023-09-23T20:04:46.014954+0000 osd.7 (osd.7) 244 : cluster 3 failed to encode map e321 with expected crc 2023-09-23T20:04:47.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:47 smithi016 bash[77995]: cluster 2023-09-23T20:04:46.018396+0000 osd.6 (osd.6) 259 : cluster 3 failed to encode map e321 with expected crc 2023-09-23T20:04:47.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:47 smithi016 bash[77995]: cluster 2023-09-23T20:04:46.018997+0000 osd.4 (osd.4) 268 : cluster 3 failed to encode map e321 with expected crc 2023-09-23T20:04:47.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:47 smithi016 bash[77995]: cluster 2023-09-23T20:04:46.019292+0000 osd.5 (osd.5) 240 : cluster 3 failed to encode map e321 with expected crc 2023-09-23T20:04:47.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:47 smithi016 bash[77995]: cluster 2023-09-23T20:04:47.013437+0000 mon.a (mon.0) 2097 : cluster 0 osdmap e322: 8 total, 8 up, 8 in 2023-09-23T20:04:47.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:47 smithi092 bash[58855]: audit 2023-09-23T20:04:46.010875+0000 mon.a (mon.0) 2095 : audit 1 from='client.? 172.21.15.16:0/1545229384' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:04:47.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:47 smithi092 bash[58855]: cluster 2023-09-23T20:04:46.011030+0000 mon.a (mon.0) 2096 : cluster 0 osdmap e321: 8 total, 8 up, 8 in 2023-09-23T20:04:47.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:47 smithi092 bash[58855]: cluster 2023-09-23T20:04:46.014954+0000 osd.7 (osd.7) 244 : cluster 3 failed to encode map e321 with expected crc 2023-09-23T20:04:47.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:47 smithi092 bash[58855]: cluster 2023-09-23T20:04:46.018396+0000 osd.6 (osd.6) 259 : cluster 3 failed to encode map e321 with expected crc 2023-09-23T20:04:47.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:47 smithi092 bash[58855]: cluster 2023-09-23T20:04:46.018997+0000 osd.4 (osd.4) 268 : cluster 3 failed to encode map e321 with expected crc 2023-09-23T20:04:47.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:47 smithi092 bash[58855]: cluster 2023-09-23T20:04:46.019292+0000 osd.5 (osd.5) 240 : cluster 3 failed to encode map e321 with expected crc 2023-09-23T20:04:47.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:47 smithi092 bash[58855]: cluster 2023-09-23T20:04:47.013437+0000 mon.a (mon.0) 2097 : cluster 0 osdmap e322: 8 total, 8 up, 8 in 2023-09-23T20:04:47.456 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:04:48.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:48 smithi016 bash[75004]: cluster 2023-09-23T20:04:46.730480+0000 mgr.y (mgr.25325) 1728 : cluster 0 pgmap v1810: 73 pgs: 13 peering, 60 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:48.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:48 smithi016 bash[75004]: cluster 2023-09-23T20:04:47.015956+0000 mon.a (mon.0) 2098 : cluster 3 Health check failed: Reduced data availability: 10 pgs inactive, 13 pgs peering (PG_AVAILABILITY) 2023-09-23T20:04:48.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:48 smithi016 bash[75004]: cluster 2023-09-23T20:04:47.021243+0000 osd.7 (osd.7) 245 : cluster 3 failed to encode map e322 with expected crc 2023-09-23T20:04:48.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:48 smithi016 bash[75004]: cluster 2023-09-23T20:04:47.021261+0000 osd.4 (osd.4) 269 : cluster 3 failed to encode map e322 with expected crc 2023-09-23T20:04:48.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:48 smithi016 bash[75004]: cluster 2023-09-23T20:04:47.021508+0000 osd.6 (osd.6) 260 : cluster 3 failed to encode map e322 with expected crc 2023-09-23T20:04:48.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:48 smithi016 bash[77995]: cluster 2023-09-23T20:04:46.730480+0000 mgr.y (mgr.25325) 1728 : cluster 0 pgmap v1810: 73 pgs: 13 peering, 60 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:48.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:48 smithi016 bash[77995]: cluster 2023-09-23T20:04:47.015956+0000 mon.a (mon.0) 2098 : cluster 3 Health check failed: Reduced data availability: 10 pgs inactive, 13 pgs peering (PG_AVAILABILITY) 2023-09-23T20:04:48.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:48 smithi016 bash[77995]: cluster 2023-09-23T20:04:47.021243+0000 osd.7 (osd.7) 245 : cluster 3 failed to encode map e322 with expected crc 2023-09-23T20:04:48.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:48 smithi016 bash[77995]: cluster 2023-09-23T20:04:47.021261+0000 osd.4 (osd.4) 269 : cluster 3 failed to encode map e322 with expected crc 2023-09-23T20:04:48.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:48 smithi016 bash[77995]: cluster 2023-09-23T20:04:47.021508+0000 osd.6 (osd.6) 260 : cluster 3 failed to encode map e322 with expected crc 2023-09-23T20:04:48.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:48 smithi092 bash[58855]: cluster 2023-09-23T20:04:46.730480+0000 mgr.y (mgr.25325) 1728 : cluster 0 pgmap v1810: 73 pgs: 13 peering, 60 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:48.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:48 smithi092 bash[58855]: cluster 2023-09-23T20:04:47.015956+0000 mon.a (mon.0) 2098 : cluster 3 Health check failed: Reduced data availability: 10 pgs inactive, 13 pgs peering (PG_AVAILABILITY) 2023-09-23T20:04:48.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:48 smithi092 bash[58855]: cluster 2023-09-23T20:04:47.021243+0000 osd.7 (osd.7) 245 : cluster 3 failed to encode map e322 with expected crc 2023-09-23T20:04:48.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:48 smithi092 bash[58855]: cluster 2023-09-23T20:04:47.021261+0000 osd.4 (osd.4) 269 : cluster 3 failed to encode map e322 with expected crc 2023-09-23T20:04:48.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:48 smithi092 bash[58855]: cluster 2023-09-23T20:04:47.021508+0000 osd.6 (osd.6) 260 : cluster 3 failed to encode map e322 with expected crc 2023-09-23T20:04:50.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:50 smithi016 bash[75004]: cluster 2023-09-23T20:04:48.731685+0000 mgr.y (mgr.25325) 1729 : cluster 0 pgmap v1812: 73 pgs: 13 peering, 60 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:50.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:50 smithi016 bash[77995]: cluster 2023-09-23T20:04:48.731685+0000 mgr.y (mgr.25325) 1729 : cluster 0 pgmap v1812: 73 pgs: 13 peering, 60 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:50.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:50 smithi092 bash[58855]: cluster 2023-09-23T20:04:48.731685+0000 mgr.y (mgr.25325) 1729 : cluster 0 pgmap v1812: 73 pgs: 13 peering, 60 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:50.686 INFO:teuthology.orchestra.run.smithi092.stdout:{ 2023-09-23T20:04:50.687 INFO:teuthology.orchestra.run.smithi092.stdout: "ops": [], 2023-09-23T20:04:50.687 INFO:teuthology.orchestra.run.smithi092.stdout: "num_ops": 0 2023-09-23T20:04:50.687 INFO:teuthology.orchestra.run.smithi092.stdout:} 2023-09-23T20:04:51.520 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T20:04:51.779 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:04:52.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:52 smithi016 bash[75004]: cluster 2023-09-23T20:04:50.732913+0000 mgr.y (mgr.25325) 1730 : cluster 0 pgmap v1813: 73 pgs: 13 peering, 60 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:52.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:52 smithi016 bash[77995]: cluster 2023-09-23T20:04:50.732913+0000 mgr.y (mgr.25325) 1730 : cluster 0 pgmap v1813: 73 pgs: 13 peering, 60 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:52.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:52 smithi092 bash[58855]: cluster 2023-09-23T20:04:50.732913+0000 mgr.y (mgr.25325) 1730 : cluster 0 pgmap v1813: 73 pgs: 13 peering, 60 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:53.415 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:53 smithi016 bash[75004]: cluster 2023-09-23T20:04:53.035448+0000 mon.a (mon.0) 2099 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 10 pgs inactive, 13 pgs peering) 2023-09-23T20:04:53.415 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:53 smithi016 bash[75004]: cluster 2023-09-23T20:04:53.035532+0000 mon.a (mon.0) 2100 : cluster 1 Cluster is now healthy 2023-09-23T20:04:53.416 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:53 smithi016 bash[77995]: cluster 2023-09-23T20:04:53.035448+0000 mon.a (mon.0) 2099 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 10 pgs inactive, 13 pgs peering) 2023-09-23T20:04:53.416 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:53 smithi016 bash[77995]: cluster 2023-09-23T20:04:53.035532+0000 mon.a (mon.0) 2100 : cluster 1 Cluster is now healthy 2023-09-23T20:04:53.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:53 smithi092 bash[58855]: cluster 2023-09-23T20:04:53.035448+0000 mon.a (mon.0) 2099 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 10 pgs inactive, 13 pgs peering) 2023-09-23T20:04:53.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:53 smithi092 bash[58855]: cluster 2023-09-23T20:04:53.035532+0000 mon.a (mon.0) 2100 : cluster 1 Cluster is now healthy 2023-09-23T20:04:54.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:54 smithi016 bash[75004]: cluster 2023-09-23T20:04:52.733811+0000 mgr.y (mgr.25325) 1731 : cluster 0 pgmap v1814: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:54.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:54 smithi016 bash[77995]: cluster 2023-09-23T20:04:52.733811+0000 mgr.y (mgr.25325) 1731 : cluster 0 pgmap v1814: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:54.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:54 smithi092 bash[58855]: cluster 2023-09-23T20:04:52.733811+0000 mgr.y (mgr.25325) 1731 : cluster 0 pgmap v1814: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:56.035 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:04:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:20:04:55] "GET /metrics HTTP/1.1" 200 34016 "" "Prometheus/2.33.4" 2023-09-23T20:04:56.109 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T20:04:56.296 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:04:56.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:56 smithi016 bash[75004]: cluster 2023-09-23T20:04:54.734633+0000 mgr.y (mgr.25325) 1732 : cluster 0 pgmap v1815: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:56.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:56 smithi016 bash[75004]: audit 2023-09-23T20:04:55.595977+0000 mon.a (mon.0) 2101 : audit 1 from='client.? 172.21.15.16:0/240325137' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:04:56.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:56 smithi016 bash[77995]: cluster 2023-09-23T20:04:54.734633+0000 mgr.y (mgr.25325) 1732 : cluster 0 pgmap v1815: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:56.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:56 smithi016 bash[77995]: audit 2023-09-23T20:04:55.595977+0000 mon.a (mon.0) 2101 : audit 1 from='client.? 172.21.15.16:0/240325137' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:04:56.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:56 smithi092 bash[58855]: cluster 2023-09-23T20:04:54.734633+0000 mgr.y (mgr.25325) 1732 : cluster 0 pgmap v1815: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:56.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:56 smithi092 bash[58855]: audit 2023-09-23T20:04:55.595977+0000 mon.a (mon.0) 2101 : audit 1 from='client.? 172.21.15.16:0/240325137' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:04:57.081 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 4 1 2023-09-23T20:04:57.371 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:57 smithi092 bash[58855]: cluster 2023-09-23T20:04:56.100859+0000 osd.4 (osd.4) 270 : cluster 3 failed to encode map e323 with expected crc 2023-09-23T20:04:57.371 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:57 smithi092 bash[58855]: audit 2023-09-23T20:04:56.100922+0000 mon.a (mon.0) 2102 : audit 1 from='client.? 172.21.15.16:0/240325137' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:04:57.371 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:57 smithi092 bash[58855]: cluster 2023-09-23T20:04:56.101019+0000 mon.a (mon.0) 2103 : cluster 0 osdmap e323: 8 total, 8 up, 8 in 2023-09-23T20:04:57.371 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:57 smithi092 bash[58855]: cluster 2023-09-23T20:04:56.146059+0000 osd.6 (osd.6) 261 : cluster 3 failed to encode map e323 with expected crc 2023-09-23T20:04:57.371 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:57 smithi092 bash[58855]: cluster 2023-09-23T20:04:56.244265+0000 osd.7 (osd.7) 246 : cluster 3 failed to encode map e323 with expected crc 2023-09-23T20:04:57.372 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:57 smithi092 bash[58855]: cluster 2023-09-23T20:04:56.343545+0000 osd.5 (osd.5) 241 : cluster 3 failed to encode map e323 with expected crc 2023-09-23T20:04:57.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:57 smithi016 bash[77995]: cluster 2023-09-23T20:04:56.100859+0000 osd.4 (osd.4) 270 : cluster 3 failed to encode map e323 with expected crc 2023-09-23T20:04:57.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:57 smithi016 bash[77995]: audit 2023-09-23T20:04:56.100922+0000 mon.a (mon.0) 2102 : audit 1 from='client.? 172.21.15.16:0/240325137' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:04:57.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:57 smithi016 bash[77995]: cluster 2023-09-23T20:04:56.101019+0000 mon.a (mon.0) 2103 : cluster 0 osdmap e323: 8 total, 8 up, 8 in 2023-09-23T20:04:57.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:57 smithi016 bash[77995]: cluster 2023-09-23T20:04:56.146059+0000 osd.6 (osd.6) 261 : cluster 3 failed to encode map e323 with expected crc 2023-09-23T20:04:57.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:57 smithi016 bash[77995]: cluster 2023-09-23T20:04:56.244265+0000 osd.7 (osd.7) 246 : cluster 3 failed to encode map e323 with expected crc 2023-09-23T20:04:57.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:57 smithi016 bash[77995]: cluster 2023-09-23T20:04:56.343545+0000 osd.5 (osd.5) 241 : cluster 3 failed to encode map e323 with expected crc 2023-09-23T20:04:57.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:57 smithi016 bash[75004]: cluster 2023-09-23T20:04:56.100859+0000 osd.4 (osd.4) 270 : cluster 3 failed to encode map e323 with expected crc 2023-09-23T20:04:57.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:57 smithi016 bash[75004]: audit 2023-09-23T20:04:56.100922+0000 mon.a (mon.0) 2102 : audit 1 from='client.? 172.21.15.16:0/240325137' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:04:57.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:57 smithi016 bash[75004]: cluster 2023-09-23T20:04:56.101019+0000 mon.a (mon.0) 2103 : cluster 0 osdmap e323: 8 total, 8 up, 8 in 2023-09-23T20:04:57.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:57 smithi016 bash[75004]: cluster 2023-09-23T20:04:56.146059+0000 osd.6 (osd.6) 261 : cluster 3 failed to encode map e323 with expected crc 2023-09-23T20:04:57.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:57 smithi016 bash[75004]: cluster 2023-09-23T20:04:56.244265+0000 osd.7 (osd.7) 246 : cluster 3 failed to encode map e323 with expected crc 2023-09-23T20:04:57.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:57 smithi016 bash[75004]: cluster 2023-09-23T20:04:56.343545+0000 osd.5 (osd.5) 241 : cluster 3 failed to encode map e323 with expected crc 2023-09-23T20:04:58.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:58 smithi016 bash[75004]: cluster 2023-09-23T20:04:56.735523+0000 mgr.y (mgr.25325) 1733 : cluster 0 pgmap v1817: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:58.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:58 smithi016 bash[77995]: cluster 2023-09-23T20:04:56.735523+0000 mgr.y (mgr.25325) 1733 : cluster 0 pgmap v1817: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:58.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:58 smithi092 bash[58855]: cluster 2023-09-23T20:04:56.735523+0000 mgr.y (mgr.25325) 1733 : cluster 0 pgmap v1817: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:04:59.380 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:04:59 smithi092 bash[58855]: audit 2023-09-23T20:04:58.704513+0000 mon.a (mon.0) 2104 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:04:59.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:04:59 smithi016 bash[75004]: audit 2023-09-23T20:04:58.704513+0000 mon.a (mon.0) 2104 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:04:59.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:04:59 smithi016 bash[77995]: audit 2023-09-23T20:04:58.704513+0000 mon.a (mon.0) 2104 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:04:59.636 INFO:teuthology.orchestra.run.smithi092.stdout:{ 2023-09-23T20:04:59.636 INFO:teuthology.orchestra.run.smithi092.stdout: "ops": [], 2023-09-23T20:04:59.636 INFO:teuthology.orchestra.run.smithi092.stdout: "complaint_time": 30, 2023-09-23T20:04:59.636 INFO:teuthology.orchestra.run.smithi092.stdout: "num_blocked_ops": 0 2023-09-23T20:04:59.636 INFO:teuthology.orchestra.run.smithi092.stdout:} 2023-09-23T20:05:00.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:00 smithi016 bash[77995]: cluster 2023-09-23T20:04:58.736410+0000 mgr.y (mgr.25325) 1734 : cluster 0 pgmap v1818: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:00.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:00 smithi016 bash[77995]: audit 2023-09-23T20:04:59.492847+0000 mon.a (mon.0) 2105 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:05:00.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:00 smithi016 bash[77995]: audit 2023-09-23T20:04:59.576110+0000 mon.a (mon.0) 2106 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:05:00.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:00 smithi016 bash[75004]: cluster 2023-09-23T20:04:58.736410+0000 mgr.y (mgr.25325) 1734 : cluster 0 pgmap v1818: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:00.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:00 smithi016 bash[75004]: audit 2023-09-23T20:04:59.492847+0000 mon.a (mon.0) 2105 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:05:00.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:00 smithi016 bash[75004]: audit 2023-09-23T20:04:59.576110+0000 mon.a (mon.0) 2106 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:05:00.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:00 smithi092 bash[58855]: cluster 2023-09-23T20:04:58.736410+0000 mgr.y (mgr.25325) 1734 : cluster 0 pgmap v1818: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:00.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:00 smithi092 bash[58855]: audit 2023-09-23T20:04:59.492847+0000 mon.a (mon.0) 2105 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:05:00.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:00 smithi092 bash[58855]: audit 2023-09-23T20:04:59.576110+0000 mon.a (mon.0) 2106 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:05:00.665 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T20:05:01.872 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:05:02.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:02 smithi016 bash[75004]: cluster 2023-09-23T20:05:00.737217+0000 mgr.y (mgr.25325) 1735 : cluster 0 pgmap v1819: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:02.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:02 smithi016 bash[77995]: cluster 2023-09-23T20:05:00.737217+0000 mgr.y (mgr.25325) 1735 : cluster 0 pgmap v1819: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:02.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:02 smithi092 bash[58855]: cluster 2023-09-23T20:05:00.737217+0000 mgr.y (mgr.25325) 1735 : cluster 0 pgmap v1819: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:04.459 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:05:04.486 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:04 smithi092 bash[58855]: cluster 2023-09-23T20:05:02.738022+0000 mgr.y (mgr.25325) 1736 : cluster 0 pgmap v1820: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:04.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:04 smithi016 bash[75004]: cluster 2023-09-23T20:05:02.738022+0000 mgr.y (mgr.25325) 1736 : cluster 0 pgmap v1820: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:04.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:04 smithi016 bash[77995]: cluster 2023-09-23T20:05:02.738022+0000 mgr.y (mgr.25325) 1736 : cluster 0 pgmap v1820: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:06.035 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:05:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:20:05:05] "GET /metrics HTTP/1.1" 200 34013 "" "Prometheus/2.33.4" 2023-09-23T20:05:06.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:06 smithi092 bash[58855]: cluster 2023-09-23T20:05:04.738703+0000 mgr.y (mgr.25325) 1737 : cluster 0 pgmap v1821: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:06.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:06 smithi092 bash[58855]: audit 2023-09-23T20:05:05.732557+0000 mon.a (mon.0) 2107 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:05:06.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:06 smithi092 bash[58855]: audit 2023-09-23T20:05:05.735383+0000 mon.a (mon.0) 2108 : audit 1 from='client.? 172.21.15.16:0/3492365049' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:05:06.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:06 smithi092 bash[58855]: audit 2023-09-23T20:05:05.739051+0000 mon.a (mon.0) 2109 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:05:06.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:06 smithi092 bash[58855]: audit 2023-09-23T20:05:05.769142+0000 mon.a (mon.0) 2110 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:05:06.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:06 smithi092 bash[58855]: audit 2023-09-23T20:05:05.776977+0000 mon.a (mon.0) 2111 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:05:06.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:06 smithi016 bash[75004]: cluster 2023-09-23T20:05:04.738703+0000 mgr.y (mgr.25325) 1737 : cluster 0 pgmap v1821: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:06.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:06 smithi016 bash[75004]: audit 2023-09-23T20:05:05.732557+0000 mon.a (mon.0) 2107 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:05:06.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:06 smithi016 bash[75004]: audit 2023-09-23T20:05:05.735383+0000 mon.a (mon.0) 2108 : audit 1 from='client.? 172.21.15.16:0/3492365049' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:05:06.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:06 smithi016 bash[75004]: audit 2023-09-23T20:05:05.739051+0000 mon.a (mon.0) 2109 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:05:06.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:06 smithi016 bash[75004]: audit 2023-09-23T20:05:05.769142+0000 mon.a (mon.0) 2110 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:05:06.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:06 smithi016 bash[75004]: audit 2023-09-23T20:05:05.776977+0000 mon.a (mon.0) 2111 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:05:06.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:06 smithi016 bash[77995]: cluster 2023-09-23T20:05:04.738703+0000 mgr.y (mgr.25325) 1737 : cluster 0 pgmap v1821: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:06.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:06 smithi016 bash[77995]: audit 2023-09-23T20:05:05.732557+0000 mon.a (mon.0) 2107 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:05:06.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:06 smithi016 bash[77995]: audit 2023-09-23T20:05:05.735383+0000 mon.a (mon.0) 2108 : audit 1 from='client.? 172.21.15.16:0/3492365049' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:05:06.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:06 smithi016 bash[77995]: audit 2023-09-23T20:05:05.739051+0000 mon.a (mon.0) 2109 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:05:06.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:06 smithi016 bash[77995]: audit 2023-09-23T20:05:05.769142+0000 mon.a (mon.0) 2110 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:05:06.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:06 smithi016 bash[77995]: audit 2023-09-23T20:05:05.776977+0000 mon.a (mon.0) 2111 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:05:06.747 INFO:teuthology.orchestra.run.smithi016.stderr:set osd.4 primary-affinity to 1 (8655362) 2023-09-23T20:05:07.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:07 smithi016 bash[75004]: audit 2023-09-23T20:05:06.309761+0000 mon.a (mon.0) 2112 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:05:07.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:07 smithi016 bash[75004]: audit 2023-09-23T20:05:06.311521+0000 mon.a (mon.0) 2113 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:05:07.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:07 smithi016 bash[75004]: audit 2023-09-23T20:05:06.320639+0000 mon.a (mon.0) 2114 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:05:07.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:07 smithi016 bash[75004]: cluster 2023-09-23T20:05:06.738454+0000 osd.6 (osd.6) 262 : cluster 3 failed to encode map e324 with expected crc 2023-09-23T20:05:07.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:07 smithi016 bash[75004]: audit 2023-09-23T20:05:06.738537+0000 mon.a (mon.0) 2115 : audit 1 from='client.? 172.21.15.16:0/3492365049' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:05:07.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:07 smithi016 bash[75004]: cluster 2023-09-23T20:05:06.738683+0000 mon.a (mon.0) 2116 : cluster 0 osdmap e324: 8 total, 8 up, 8 in 2023-09-23T20:05:07.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:07 smithi016 bash[77995]: audit 2023-09-23T20:05:06.309761+0000 mon.a (mon.0) 2112 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:05:07.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:07 smithi016 bash[77995]: audit 2023-09-23T20:05:06.311521+0000 mon.a (mon.0) 2113 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:05:07.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:07 smithi016 bash[77995]: audit 2023-09-23T20:05:06.320639+0000 mon.a (mon.0) 2114 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:05:07.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:07 smithi016 bash[77995]: cluster 2023-09-23T20:05:06.738454+0000 osd.6 (osd.6) 262 : cluster 3 failed to encode map e324 with expected crc 2023-09-23T20:05:07.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:07 smithi016 bash[77995]: audit 2023-09-23T20:05:06.738537+0000 mon.a (mon.0) 2115 : audit 1 from='client.? 172.21.15.16:0/3492365049' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:05:07.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:07 smithi016 bash[77995]: cluster 2023-09-23T20:05:06.738683+0000 mon.a (mon.0) 2116 : cluster 0 osdmap e324: 8 total, 8 up, 8 in 2023-09-23T20:05:07.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:07 smithi092 bash[58855]: audit 2023-09-23T20:05:06.309761+0000 mon.a (mon.0) 2112 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:05:07.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:07 smithi092 bash[58855]: audit 2023-09-23T20:05:06.311521+0000 mon.a (mon.0) 2113 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:05:07.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:07 smithi092 bash[58855]: audit 2023-09-23T20:05:06.320639+0000 mon.a (mon.0) 2114 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:05:07.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:07 smithi092 bash[58855]: cluster 2023-09-23T20:05:06.738454+0000 osd.6 (osd.6) 262 : cluster 3 failed to encode map e324 with expected crc 2023-09-23T20:05:07.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:07 smithi092 bash[58855]: audit 2023-09-23T20:05:06.738537+0000 mon.a (mon.0) 2115 : audit 1 from='client.? 172.21.15.16:0/3492365049' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:05:07.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:07 smithi092 bash[58855]: cluster 2023-09-23T20:05:06.738683+0000 mon.a (mon.0) 2116 : cluster 0 osdmap e324: 8 total, 8 up, 8 in 2023-09-23T20:05:07.808 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T20:05:07.889 INFO:teuthology.orchestra.run.smithi092.stdout:{ 2023-09-23T20:05:07.889 INFO:teuthology.orchestra.run.smithi092.stdout: "size": 20, 2023-09-23T20:05:07.890 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 600, 2023-09-23T20:05:07.890 INFO:teuthology.orchestra.run.smithi092.stdout: "ops": [ 2023-09-23T20:05:07.890 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.890 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46640.0:200209 13.b 13:db8d8303:::benchmark_data_smithi016_205181_object92945:head [delete] snapc 0=[] ondisk+write+known_if_redirected e249)", 2023-09-23T20:05:07.890 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:55:37.480424+0000", 2023-09-23T20:05:07.890 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 570.40812675899997, 2023-09-23T20:05:07.890 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.181297509, 2023-09-23T20:05:07.890 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:05:07.890 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:07.890 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:05:07.891 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46640", 2023-09-23T20:05:07.891 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2656794142", 2023-09-23T20:05:07.891 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 200209 2023-09-23T20:05:07.891 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.891 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:05:07.891 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.891 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:05:07.891 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.480424+0000", 2023-09-23T20:05:07.891 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:05:07.891 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.891 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.891 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:05:07.892 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.480424+0000", 2023-09-23T20:05:07.892 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T20:05:07.892 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.892 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.892 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:05:07.892 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.480422+0000", 2023-09-23T20:05:07.892 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.1600000000000003e-06 2023-09-23T20:05:07.892 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.892 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.892 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:05:07.892 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.480428+0000", 2023-09-23T20:05:07.892 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.2449999999999999e-05 2023-09-23T20:05:07.893 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.893 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.893 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:05:07.893 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.480440+0000", 2023-09-23T20:05:07.893 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.5250000000000002e-06 2023-09-23T20:05:07.893 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.893 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.893 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:05:07.893 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.480447+0000", 2023-09-23T20:05:07.893 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.014180157000000001 2023-09-23T20:05:07.893 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.893 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.894 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:05:07.894 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.494627+0000", 2023-09-23T20:05:07.894 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000118114 2023-09-23T20:05:07.894 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.894 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.894 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:05:07.894 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.494745+0000", 2023-09-23T20:05:07.894 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00014008799999999999 2023-09-23T20:05:07.894 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.894 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.894 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:05:07.895 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.494885+0000", 2023-09-23T20:05:07.895 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.019398823999999999 2023-09-23T20:05:07.895 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.895 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.895 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:07.895 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.514284+0000", 2023-09-23T20:05:07.895 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00011607699999999999 2023-09-23T20:05:07.895 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.895 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.895 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:05:07.895 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.514400+0000", 2023-09-23T20:05:07.895 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.147232276 2023-09-23T20:05:07.895 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.896 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.896 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:07.896 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.661633+0000", 2023-09-23T20:05:07.896 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.8930000000000001e-05 2023-09-23T20:05:07.896 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.896 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.896 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:05:07.896 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.661681+0000", 2023-09-23T20:05:07.896 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.9533000000000002e-05 2023-09-23T20:05:07.896 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.896 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.896 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:05:07.897 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.661721+0000", 2023-09-23T20:05:07.897 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.181297509 2023-09-23T20:05:07.897 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:07.897 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:05:07.897 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:07.897 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.897 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.897 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46640.0:200210 13.b 13:d11b0604:::benchmark_data_smithi016_205181_object92946:head [delete] snapc 0=[] ondisk+write+known_if_redirected e249)", 2023-09-23T20:05:07.897 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:55:37.480453+0000", 2023-09-23T20:05:07.897 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 570.40809698199996, 2023-09-23T20:05:07.897 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.183861895, 2023-09-23T20:05:07.898 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:05:07.898 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:07.898 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:05:07.898 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46640", 2023-09-23T20:05:07.898 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2656794142", 2023-09-23T20:05:07.898 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 200210 2023-09-23T20:05:07.898 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.898 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:05:07.898 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.898 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:05:07.898 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.480453+0000", 2023-09-23T20:05:07.898 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:05:07.899 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.899 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.899 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:05:07.899 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.480453+0000", 2023-09-23T20:05:07.899 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999995 2023-09-23T20:05:07.899 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.899 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.899 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:05:07.899 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.480453+0000", 2023-09-23T20:05:07.899 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.458e-06 2023-09-23T20:05:07.899 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.899 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.900 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:05:07.900 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.480455+0000", 2023-09-23T20:05:07.900 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.5979999999999996e-06 2023-09-23T20:05:07.900 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.900 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.900 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:05:07.900 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.480461+0000", 2023-09-23T20:05:07.900 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.7720000000000001e-06 2023-09-23T20:05:07.900 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.900 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.900 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:05:07.900 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.480465+0000", 2023-09-23T20:05:07.901 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.019905068000000001 2023-09-23T20:05:07.901 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.901 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.901 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:05:07.901 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.500370+0000", 2023-09-23T20:05:07.901 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00010952599999999999 2023-09-23T20:05:07.901 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.901 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.901 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:05:07.901 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.500479+0000", 2023-09-23T20:05:07.901 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0040133870000000002 2023-09-23T20:05:07.901 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.902 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.902 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:05:07.902 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.504493+0000", 2023-09-23T20:05:07.902 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0099118620000000004 2023-09-23T20:05:07.902 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.902 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.902 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:05:07.902 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.514405+0000", 2023-09-23T20:05:07.902 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.7217000000000004e-05 2023-09-23T20:05:07.902 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.902 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.902 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:07.903 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.514472+0000", 2023-09-23T20:05:07.903 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.14964140500000001 2023-09-23T20:05:07.903 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.903 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.903 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:07.903 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.664113+0000", 2023-09-23T20:05:07.903 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00014186799999999999 2023-09-23T20:05:07.903 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.903 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.903 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:05:07.903 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.664255+0000", 2023-09-23T20:05:07.904 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.0152999999999998e-05 2023-09-23T20:05:07.904 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.904 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.904 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:05:07.904 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.664315+0000", 2023-09-23T20:05:07.904 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.183861895 2023-09-23T20:05:07.904 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:07.904 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:05:07.904 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:07.904 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.904 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.904 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46640.0:200218 13.b 13:d3307db0:::benchmark_data_smithi016_205181_object92954:head [delete] snapc 0=[] ondisk+write+known_if_redirected e249)", 2023-09-23T20:05:07.905 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:55:37.492440+0000", 2023-09-23T20:05:07.905 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 570.39611055299997, 2023-09-23T20:05:07.905 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.192990945, 2023-09-23T20:05:07.905 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:05:07.905 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:07.905 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:05:07.905 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46640", 2023-09-23T20:05:07.905 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2656794142", 2023-09-23T20:05:07.905 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 200218 2023-09-23T20:05:07.905 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.905 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:05:07.905 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.906 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:05:07.906 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.492440+0000", 2023-09-23T20:05:07.906 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:05:07.906 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.906 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.906 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:05:07.906 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.492440+0000", 2023-09-23T20:05:07.906 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T20:05:07.906 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.906 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.906 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:05:07.907 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.492438+0000", 2023-09-23T20:05:07.907 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.967e-06 2023-09-23T20:05:07.907 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.907 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.907 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:05:07.907 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.492443+0000", 2023-09-23T20:05:07.907 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.8300000000000002e-06 2023-09-23T20:05:07.907 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.907 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.907 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:05:07.907 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.492452+0000", 2023-09-23T20:05:07.907 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.6859999999999997e-06 2023-09-23T20:05:07.907 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.908 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.908 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:05:07.908 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.492457+0000", 2023-09-23T20:05:07.908 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.017479604999999999 2023-09-23T20:05:07.908 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.908 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.908 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:05:07.908 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.509936+0000", 2023-09-23T20:05:07.908 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.7836999999999994e-05 2023-09-23T20:05:07.908 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.908 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.909 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:05:07.909 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.510014+0000", 2023-09-23T20:05:07.909 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.2588000000000003e-05 2023-09-23T20:05:07.909 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.909 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.909 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:05:07.909 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.510097+0000", 2023-09-23T20:05:07.909 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0043997139999999999 2023-09-23T20:05:07.909 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.909 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.909 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:07.909 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.514496+0000", 2023-09-23T20:05:07.910 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.5040000000000004e-06 2023-09-23T20:05:07.910 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.910 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.910 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:05:07.910 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.514501+0000", 2023-09-23T20:05:07.910 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.17083277999999999 2023-09-23T20:05:07.910 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.910 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.910 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:07.910 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.685334+0000", 2023-09-23T20:05:07.910 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.9605000000000003e-05 2023-09-23T20:05:07.910 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.911 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.911 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:05:07.911 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.685393+0000", 2023-09-23T20:05:07.911 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.7320000000000002e-05 2023-09-23T20:05:07.911 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.911 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.911 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:05:07.911 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.685431+0000", 2023-09-23T20:05:07.911 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.192990945 2023-09-23T20:05:07.911 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:07.911 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:05:07.911 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:07.912 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.912 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.912 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46640.0:200231 13.b 13:d7930137:::benchmark_data_smithi016_205181_object92967:head [delete] snapc 0=[] ondisk+write+known_if_redirected e249)", 2023-09-23T20:05:07.912 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:55:37.510046+0000", 2023-09-23T20:05:07.912 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 570.37850406899997, 2023-09-23T20:05:07.912 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.17547036199999999, 2023-09-23T20:05:07.912 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:05:07.912 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:07.912 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:05:07.912 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46640", 2023-09-23T20:05:07.912 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2656794142", 2023-09-23T20:05:07.912 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 200231 2023-09-23T20:05:07.912 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.913 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:05:07.913 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.913 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:05:07.913 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.510046+0000", 2023-09-23T20:05:07.913 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:05:07.913 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.913 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.913 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:05:07.913 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.510046+0000", 2023-09-23T20:05:07.913 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T20:05:07.913 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.913 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.914 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:05:07.914 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.510045+0000", 2023-09-23T20:05:07.914 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.2670000000000003e-06 2023-09-23T20:05:07.914 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.914 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.914 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:05:07.914 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.510051+0000", 2023-09-23T20:05:07.914 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.0893999999999999e-05 2023-09-23T20:05:07.914 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.914 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.914 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:05:07.914 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.510062+0000", 2023-09-23T20:05:07.915 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.2379999999999997e-06 2023-09-23T20:05:07.915 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.915 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.915 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:05:07.915 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.510067+0000", 2023-09-23T20:05:07.915 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0044560160000000001 2023-09-23T20:05:07.915 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.915 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.915 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:05:07.915 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.514523+0000", 2023-09-23T20:05:07.915 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000137243 2023-09-23T20:05:07.915 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.915 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.916 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:05:07.916 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.514660+0000", 2023-09-23T20:05:07.916 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00016373099999999999 2023-09-23T20:05:07.916 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.916 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.916 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:05:07.916 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.514824+0000", 2023-09-23T20:05:07.916 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0061843319999999999 2023-09-23T20:05:07.916 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.916 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.916 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:07.916 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.521009+0000", 2023-09-23T20:05:07.917 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00015000500000000001 2023-09-23T20:05:07.917 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.917 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.917 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:05:07.917 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.521159+0000", 2023-09-23T20:05:07.917 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.16430768100000001 2023-09-23T20:05:07.917 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.917 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.917 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:07.917 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.685466+0000", 2023-09-23T20:05:07.917 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.4414e-05 2023-09-23T20:05:07.917 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.918 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.918 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:05:07.918 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.685491+0000", 2023-09-23T20:05:07.918 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.5914e-05 2023-09-23T20:05:07.918 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.918 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.918 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:05:07.918 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:37.685517+0000", 2023-09-23T20:05:07.918 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.17547036199999999 2023-09-23T20:05:07.918 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:07.918 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:05:07.918 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:07.919 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.919 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.919 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46640.0:201637 13.b 13:d6e519b4:::benchmark_data_smithi016_205181_object94373:head [delete] snapc 0=[] ondisk+write+known_if_redirected e249)", 2023-09-23T20:05:07.919 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:55:39.837955+0000", 2023-09-23T20:05:07.919 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 568.05059494, 2023-09-23T20:05:07.919 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.15928557600000001, 2023-09-23T20:05:07.919 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:05:07.919 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:07.919 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:05:07.919 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46640", 2023-09-23T20:05:07.919 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2656794142", 2023-09-23T20:05:07.919 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 201637 2023-09-23T20:05:07.920 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.920 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:05:07.920 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.920 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:05:07.920 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:39.837955+0000", 2023-09-23T20:05:07.920 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:05:07.920 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.920 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.920 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:05:07.920 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:39.837955+0000", 2023-09-23T20:05:07.920 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T20:05:07.920 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.920 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.921 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:05:07.921 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:39.837954+0000", 2023-09-23T20:05:07.921 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.9460000000000003e-06 2023-09-23T20:05:07.921 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.921 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.921 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:05:07.921 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:39.837960+0000", 2023-09-23T20:05:07.921 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.6560000000000004e-06 2023-09-23T20:05:07.921 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.921 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.921 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:05:07.921 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:39.837969+0000", 2023-09-23T20:05:07.922 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.091e-06 2023-09-23T20:05:07.922 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.922 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.922 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:05:07.922 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:39.837974+0000", 2023-09-23T20:05:07.922 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00042364200000000001 2023-09-23T20:05:07.922 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.922 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.922 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:05:07.922 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:39.838398+0000", 2023-09-23T20:05:07.922 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00012949200000000001 2023-09-23T20:05:07.923 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.923 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.923 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:05:07.923 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:39.838528+0000", 2023-09-23T20:05:07.923 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000152547 2023-09-23T20:05:07.923 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.923 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.923 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:05:07.923 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:39.838680+0000", 2023-09-23T20:05:07.923 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0068553970000000001 2023-09-23T20:05:07.923 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.923 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.924 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:05:07.924 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:39.845536+0000", 2023-09-23T20:05:07.924 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.003723456 2023-09-23T20:05:07.924 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.924 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.924 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:07.924 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:39.849259+0000", 2023-09-23T20:05:07.924 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.14786917799999999 2023-09-23T20:05:07.924 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.924 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.924 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:07.924 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:39.997128+0000", 2023-09-23T20:05:07.924 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.3937999999999997e-05 2023-09-23T20:05:07.925 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.925 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.925 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:05:07.925 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:39.997192+0000", 2023-09-23T20:05:07.925 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.8813000000000002e-05 2023-09-23T20:05:07.925 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.925 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.925 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:05:07.925 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:39.997241+0000", 2023-09-23T20:05:07.925 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.15928557600000001 2023-09-23T20:05:07.925 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:07.925 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:05:07.926 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:07.926 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.926 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.926 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:37537 14.7 14:eb329858:::benchmark_data_smithi016_228083_object37536:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-09-23T20:05:07.926 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:57:36.021187+0000", 2023-09-23T20:05:07.926 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 451.867363458, 2023-09-23T20:05:07.926 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.235362606, 2023-09-23T20:05:07.926 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:05:07.926 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:07.926 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:05:07.926 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T20:05:07.926 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:05:07.927 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 37537 2023-09-23T20:05:07.927 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.927 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:05:07.927 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.927 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:05:07.927 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:36.021187+0000", 2023-09-23T20:05:07.927 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:05:07.927 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.927 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.927 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:05:07.927 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:36.021187+0000", 2023-09-23T20:05:07.927 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T20:05:07.928 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.928 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.928 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:05:07.928 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:36.021185+0000", 2023-09-23T20:05:07.928 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.6517000000000001e-05 2023-09-23T20:05:07.928 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.928 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.928 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:05:07.928 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:36.021241+0000", 2023-09-23T20:05:07.928 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.2512999999999999e-05 2023-09-23T20:05:07.928 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.928 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.928 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:05:07.929 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:36.021264+0000", 2023-09-23T20:05:07.929 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.2049999999999996e-06 2023-09-23T20:05:07.929 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.929 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.929 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:05:07.929 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:36.021271+0000", 2023-09-23T20:05:07.929 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.9954000000000001e-05 2023-09-23T20:05:07.929 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.929 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.929 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:05:07.929 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:36.021341+0000", 2023-09-23T20:05:07.929 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00016290599999999999 2023-09-23T20:05:07.930 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.930 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.930 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:05:07.930 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:36.021504+0000", 2023-09-23T20:05:07.930 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00021503399999999999 2023-09-23T20:05:07.930 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.930 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.930 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:05:07.930 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:36.021719+0000", 2023-09-23T20:05:07.930 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0032425079999999998 2023-09-23T20:05:07.930 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.931 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.931 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:05:07.931 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:36.024962+0000", 2023-09-23T20:05:07.931 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00030831499999999998 2023-09-23T20:05:07.931 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.931 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.931 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:07.931 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:36.025270+0000", 2023-09-23T20:05:07.931 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.231014566 2023-09-23T20:05:07.931 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.931 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.931 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:07.932 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:36.256284+0000", 2023-09-23T20:05:07.932 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00016937300000000001 2023-09-23T20:05:07.932 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.932 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.932 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:05:07.932 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:36.256454+0000", 2023-09-23T20:05:07.932 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.5574000000000003e-05 2023-09-23T20:05:07.932 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.932 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.932 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:05:07.932 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:36.256549+0000", 2023-09-23T20:05:07.932 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.235362606 2023-09-23T20:05:07.933 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:07.933 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:05:07.933 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:07.933 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.933 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.933 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:40018 14.7 14:e59bff90:::benchmark_data_smithi016_228083_object40017:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-09-23T20:05:07.933 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:57:38.120073+0000", 2023-09-23T20:05:07.933 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 449.76847738100003, 2023-09-23T20:05:07.933 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.28751564400000001, 2023-09-23T20:05:07.933 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:05:07.933 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:07.933 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:05:07.934 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T20:05:07.934 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:05:07.934 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 40018 2023-09-23T20:05:07.934 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.934 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:05:07.934 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.934 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:05:07.934 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:38.120073+0000", 2023-09-23T20:05:07.934 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:05:07.934 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.934 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.935 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:05:07.935 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:38.120073+0000", 2023-09-23T20:05:07.935 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T20:05:07.935 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.935 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.935 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:05:07.935 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:38.120072+0000", 2023-09-23T20:05:07.935 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.7407000000000002e-05 2023-09-23T20:05:07.935 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.935 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.935 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:05:07.935 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:38.120119+0000", 2023-09-23T20:05:07.936 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.3012e-05 2023-09-23T20:05:07.936 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.936 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.936 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:05:07.936 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:38.120132+0000", 2023-09-23T20:05:07.936 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.0839999999999997e-06 2023-09-23T20:05:07.936 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.936 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.936 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:05:07.936 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:38.120138+0000", 2023-09-23T20:05:07.936 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.8371999999999997e-05 2023-09-23T20:05:07.936 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.936 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.937 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:05:07.937 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:38.120186+0000", 2023-09-23T20:05:07.937 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00015543800000000001 2023-09-23T20:05:07.937 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.937 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.937 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:05:07.937 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:38.120342+0000", 2023-09-23T20:05:07.937 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0018517399999999999 2023-09-23T20:05:07.937 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.937 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.937 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:05:07.937 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:38.122194+0000", 2023-09-23T20:05:07.938 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0033080090000000002 2023-09-23T20:05:07.938 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.938 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.938 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:05:07.938 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:38.125502+0000", 2023-09-23T20:05:07.938 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.5063e-05 2023-09-23T20:05:07.938 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.938 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.938 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:07.938 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:38.125537+0000", 2023-09-23T20:05:07.938 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.28182004100000002 2023-09-23T20:05:07.939 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.939 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.939 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:07.939 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:38.407357+0000", 2023-09-23T20:05:07.939 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00013066999999999999 2023-09-23T20:05:07.939 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.939 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.939 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:05:07.939 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:38.407487+0000", 2023-09-23T20:05:07.939 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00010109 2023-09-23T20:05:07.939 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.939 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.939 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:05:07.940 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:38.407589+0000", 2023-09-23T20:05:07.940 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.28751564400000001 2023-09-23T20:05:07.940 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:07.940 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:05:07.940 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:07.940 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.940 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.940 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:40020 14.7 14:e8af5206:::benchmark_data_smithi016_228083_object40019:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-09-23T20:05:07.940 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:57:38.121992+0000", 2023-09-23T20:05:07.940 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 449.766558116, 2023-09-23T20:05:07.940 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.28583325900000001, 2023-09-23T20:05:07.940 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:05:07.941 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:07.941 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:05:07.941 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T20:05:07.941 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:05:07.941 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 40020 2023-09-23T20:05:07.941 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.941 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:05:07.941 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.941 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:05:07.941 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:38.121992+0000", 2023-09-23T20:05:07.941 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:05:07.941 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.942 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.942 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:05:07.942 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:38.121992+0000", 2023-09-23T20:05:07.942 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T20:05:07.942 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.942 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.942 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:05:07.942 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:38.121991+0000", 2023-09-23T20:05:07.942 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.2562000000000001e-05 2023-09-23T20:05:07.942 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.942 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.942 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:05:07.943 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:38.122063+0000", 2023-09-23T20:05:07.943 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.8042000000000001e-05 2023-09-23T20:05:07.943 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.943 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.943 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:05:07.943 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:38.122081+0000", 2023-09-23T20:05:07.943 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.5089999999999993e-06 2023-09-23T20:05:07.943 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.943 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.943 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:05:07.943 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:38.122090+0000", 2023-09-23T20:05:07.943 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0019927819999999998 2023-09-23T20:05:07.944 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.944 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.944 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:05:07.944 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:38.124083+0000", 2023-09-23T20:05:07.944 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.8430000000000005e-05 2023-09-23T20:05:07.944 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.944 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.944 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:05:07.944 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:38.124181+0000", 2023-09-23T20:05:07.944 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000115394 2023-09-23T20:05:07.944 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.944 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.945 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:05:07.945 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:38.124297+0000", 2023-09-23T20:05:07.945 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0028549640000000002 2023-09-23T20:05:07.945 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.945 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.945 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:05:07.945 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:38.127152+0000", 2023-09-23T20:05:07.945 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00065275000000000001 2023-09-23T20:05:07.945 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.945 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.945 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:07.945 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:38.127804+0000", 2023-09-23T20:05:07.945 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.279934562 2023-09-23T20:05:07.946 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.946 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.946 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:07.946 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:38.407739+0000", 2023-09-23T20:05:07.946 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.9829000000000001e-05 2023-09-23T20:05:07.946 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.946 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.946 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:05:07.946 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:38.407779+0000", 2023-09-23T20:05:07.946 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.6703000000000003e-05 2023-09-23T20:05:07.946 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.946 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.947 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:05:07.947 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:38.407825+0000", 2023-09-23T20:05:07.947 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.28583325900000001 2023-09-23T20:05:07.947 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:07.947 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:05:07.947 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:07.947 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.947 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.947 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:49397 14.7 14:eb4777c9:::benchmark_data_smithi016_228083_object49396:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-09-23T20:05:07.947 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:57:46.832972+0000", 2023-09-23T20:05:07.947 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 441.05557789199997, 2023-09-23T20:05:07.947 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.171797699, 2023-09-23T20:05:07.947 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:05:07.948 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:07.948 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:05:07.948 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T20:05:07.948 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:05:07.948 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 49397 2023-09-23T20:05:07.948 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.948 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:05:07.948 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.948 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:05:07.948 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:46.832972+0000", 2023-09-23T20:05:07.948 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:05:07.948 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.948 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.949 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:05:07.949 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:46.832972+0000", 2023-09-23T20:05:07.949 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.999999 2023-09-23T20:05:07.949 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.949 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.949 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:05:07.949 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:46.832971+0000", 2023-09-23T20:05:07.949 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.1427999999999994e-05 2023-09-23T20:05:07.949 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.949 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.949 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:05:07.949 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:46.833053+0000", 2023-09-23T20:05:07.950 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.1839999999999999e-06 2023-09-23T20:05:07.950 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.950 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.950 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:05:07.950 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:46.833061+0000", 2023-09-23T20:05:07.950 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.4489999999999998e-06 2023-09-23T20:05:07.950 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.950 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.950 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:05:07.950 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:46.833066+0000", 2023-09-23T20:05:07.950 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0021409979999999999 2023-09-23T20:05:07.950 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.951 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.951 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:05:07.951 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:46.835207+0000", 2023-09-23T20:05:07.951 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000108371 2023-09-23T20:05:07.951 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.951 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.951 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:05:07.951 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:46.835316+0000", 2023-09-23T20:05:07.951 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00018373099999999999 2023-09-23T20:05:07.951 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.951 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.951 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:05:07.951 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:46.835500+0000", 2023-09-23T20:05:07.952 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0021555350000000001 2023-09-23T20:05:07.952 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.952 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.952 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:05:07.952 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:46.837655+0000", 2023-09-23T20:05:07.952 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000302422 2023-09-23T20:05:07.952 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.952 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.952 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:07.952 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:46.837958+0000", 2023-09-23T20:05:07.952 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.16662668999999999 2023-09-23T20:05:07.952 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.953 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.953 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:07.953 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:47.004584+0000", 2023-09-23T20:05:07.953 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000104331 2023-09-23T20:05:07.953 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.953 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.953 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:05:07.953 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:47.004689+0000", 2023-09-23T20:05:07.953 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.1521999999999996e-05 2023-09-23T20:05:07.953 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.953 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.953 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:05:07.954 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:47.004770+0000", 2023-09-23T20:05:07.954 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.171797699 2023-09-23T20:05:07.954 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:07.954 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:05:07.954 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:07.954 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.954 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.954 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:95171 14.7 14:e6ad3080:::benchmark_data_smithi016_228083_object95170:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e264)", 2023-09-23T20:05:07.954 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:58:38.231438+0000", 2023-09-23T20:05:07.954 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 389.657112176, 2023-09-23T20:05:07.954 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.237089573, 2023-09-23T20:05:07.954 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:05:07.955 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:07.955 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:05:07.955 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T20:05:07.955 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:05:07.955 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 95171 2023-09-23T20:05:07.955 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.955 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:05:07.955 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.955 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:05:07.955 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:38.231438+0000", 2023-09-23T20:05:07.955 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:05:07.955 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.956 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.956 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:05:07.956 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:38.231438+0000", 2023-09-23T20:05:07.956 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T20:05:07.956 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.956 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.956 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:05:07.956 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:38.231436+0000", 2023-09-23T20:05:07.956 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.8737000000000003e-05 2023-09-23T20:05:07.956 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.956 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.956 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:05:07.957 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:38.231505+0000", 2023-09-23T20:05:07.957 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.1761999999999999e-05 2023-09-23T20:05:07.957 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.957 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.957 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:05:07.957 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:38.231526+0000", 2023-09-23T20:05:07.957 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.5211000000000001e-05 2023-09-23T20:05:07.957 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.957 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.957 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:05:07.957 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:38.231542+0000", 2023-09-23T20:05:07.957 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.5178000000000005e-05 2023-09-23T20:05:07.957 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.958 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.958 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:05:07.958 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:38.231607+0000", 2023-09-23T20:05:07.958 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00020528899999999999 2023-09-23T20:05:07.958 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.958 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.958 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:05:07.958 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:38.231812+0000", 2023-09-23T20:05:07.958 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00034610499999999998 2023-09-23T20:05:07.958 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.958 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.958 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:05:07.959 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:38.232158+0000", 2023-09-23T20:05:07.959 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0054900360000000002 2023-09-23T20:05:07.959 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.959 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.959 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:05:07.959 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:38.237648+0000", 2023-09-23T20:05:07.959 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0012128060000000001 2023-09-23T20:05:07.959 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.959 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.959 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:07.959 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:38.238861+0000", 2023-09-23T20:05:07.959 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.22949417 2023-09-23T20:05:07.960 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.960 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.960 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:07.960 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:38.468355+0000", 2023-09-23T20:05:07.960 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.9131999999999999e-05 2023-09-23T20:05:07.960 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.960 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.960 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:05:07.960 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:38.468454+0000", 2023-09-23T20:05:07.960 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.3250000000000005e-05 2023-09-23T20:05:07.960 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.960 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.960 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:05:07.961 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:38.468528+0000", 2023-09-23T20:05:07.961 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.237089573 2023-09-23T20:05:07.961 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:07.961 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:05:07.961 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:07.961 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.961 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.961 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:95183 14.7 14:eaa9693c:::benchmark_data_smithi016_228083_object95182:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e264)", 2023-09-23T20:05:07.961 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:58:38.241094+0000", 2023-09-23T20:05:07.961 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 389.64745615599998, 2023-09-23T20:05:07.961 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.22765014, 2023-09-23T20:05:07.962 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:05:07.962 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:07.962 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:05:07.962 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T20:05:07.962 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:05:07.962 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 95183 2023-09-23T20:05:07.962 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.962 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:05:07.962 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.962 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:05:07.962 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:38.241094+0000", 2023-09-23T20:05:07.962 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:05:07.963 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.963 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.963 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:05:07.963 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:38.241094+0000", 2023-09-23T20:05:07.963 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T20:05:07.963 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.963 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.963 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:05:07.963 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:38.241092+0000", 2023-09-23T20:05:07.963 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.5741000000000001e-05 2023-09-23T20:05:07.963 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.963 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.963 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:05:07.964 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:38.241138+0000", 2023-09-23T20:05:07.964 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.4874e-05 2023-09-23T20:05:07.964 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.964 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.964 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:05:07.964 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:38.241153+0000", 2023-09-23T20:05:07.964 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.7429999999999996e-06 2023-09-23T20:05:07.964 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.964 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.964 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:05:07.964 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:38.241160+0000", 2023-09-23T20:05:07.964 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.6246000000000003e-05 2023-09-23T20:05:07.965 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.965 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.965 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:05:07.965 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:38.241216+0000", 2023-09-23T20:05:07.965 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000118099 2023-09-23T20:05:07.965 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.965 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.965 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:05:07.965 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:38.241334+0000", 2023-09-23T20:05:07.965 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00013826100000000001 2023-09-23T20:05:07.965 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.965 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.966 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:05:07.966 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:38.241472+0000", 2023-09-23T20:05:07.966 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.002856889 2023-09-23T20:05:07.966 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.966 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.966 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:05:07.966 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:38.244329+0000", 2023-09-23T20:05:07.966 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.9928999999999997e-05 2023-09-23T20:05:07.966 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.966 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.966 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:07.966 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:38.244369+0000", 2023-09-23T20:05:07.967 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.224231343 2023-09-23T20:05:07.967 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.967 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.967 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:07.967 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:38.468601+0000", 2023-09-23T20:05:07.967 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.1069999999999995e-05 2023-09-23T20:05:07.967 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.967 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.967 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:05:07.967 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:38.468682+0000", 2023-09-23T20:05:07.967 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.2609999999999999e-05 2023-09-23T20:05:07.967 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.968 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.968 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:05:07.968 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:38.468744+0000", 2023-09-23T20:05:07.968 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.22765014 2023-09-23T20:05:07.968 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:07.968 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:05:07.968 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:07.968 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.968 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.968 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:177686 14.7 14:e28ca462:::benchmark_data_smithi016_228083_object77727:head [delete] snapc 0=[] ondisk+write+known_if_redirected e271)", 2023-09-23T20:05:07.968 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T20:00:41.513839+0000", 2023-09-23T20:05:07.968 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 266.37471122900001, 2023-09-23T20:05:07.969 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.17888817600000001, 2023-09-23T20:05:07.969 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:05:07.969 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:07.969 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:05:07.969 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T20:05:07.969 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:05:07.969 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 177686 2023-09-23T20:05:07.969 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.969 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:05:07.969 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.969 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:05:07.969 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.513839+0000", 2023-09-23T20:05:07.970 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:05:07.970 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.970 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.970 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:05:07.970 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.513839+0000", 2023-09-23T20:05:07.970 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T20:05:07.970 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.970 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.970 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:05:07.970 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.513837+0000", 2023-09-23T20:05:07.970 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.6240000000000008e-06 2023-09-23T20:05:07.970 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.971 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.971 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:05:07.971 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.513846+0000", 2023-09-23T20:05:07.971 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.5750999999999999e-05 2023-09-23T20:05:07.971 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.971 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.971 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:05:07.971 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.513862+0000", 2023-09-23T20:05:07.971 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.6580000000000002e-06 2023-09-23T20:05:07.971 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.971 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.971 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:05:07.971 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.513869+0000", 2023-09-23T20:05:07.972 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.5924000000000001e-05 2023-09-23T20:05:07.972 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.972 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.972 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:05:07.972 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.513935+0000", 2023-09-23T20:05:07.972 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00017360300000000001 2023-09-23T20:05:07.972 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.972 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.972 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:05:07.972 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.514109+0000", 2023-09-23T20:05:07.972 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00020305799999999999 2023-09-23T20:05:07.972 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.972 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.973 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:05:07.973 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.514312+0000", 2023-09-23T20:05:07.973 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0060048669999999997 2023-09-23T20:05:07.973 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.973 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.973 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:07.973 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.520317+0000", 2023-09-23T20:05:07.973 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.9914999999999995e-05 2023-09-23T20:05:07.973 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.973 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.973 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:05:07.973 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.520417+0000", 2023-09-23T20:05:07.974 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.17223017099999999 2023-09-23T20:05:07.974 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.974 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.974 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:07.974 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.692647+0000", 2023-09-23T20:05:07.974 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.6347999999999998e-05 2023-09-23T20:05:07.974 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.974 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.974 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:05:07.974 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.692683+0000", 2023-09-23T20:05:07.974 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.4138000000000003e-05 2023-09-23T20:05:07.974 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.975 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.975 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:05:07.975 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.692727+0000", 2023-09-23T20:05:07.975 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.17888817600000001 2023-09-23T20:05:07.975 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:07.975 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:05:07.975 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:07.975 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.975 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.975 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.37161.0:62 2.3 2:cefee11b:::rbd_data.b80d7e31e638.0000000000000031:head [sparse-read 0~4194304 [fadvise_nocache]] snapc 0=[] ondisk+read+known_if_redirected e285)", 2023-09-23T20:05:07.975 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T20:02:29.853336+0000", 2023-09-23T20:05:07.976 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 158.03521377499999, 2023-09-23T20:05:07.976 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.010939589, 2023-09-23T20:05:07.976 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:05:07.976 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "reached pg", 2023-09-23T20:05:07.976 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:05:07.976 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.37161", 2023-09-23T20:05:07.976 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/407694934", 2023-09-23T20:05:07.976 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 62 2023-09-23T20:05:07.976 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.976 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:05:07.976 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.976 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:05:07.976 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:29.853336+0000", 2023-09-23T20:05:07.977 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:05:07.977 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.977 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.977 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:05:07.977 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:29.853336+0000", 2023-09-23T20:05:07.977 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999971 2023-09-23T20:05:07.977 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.977 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.977 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:05:07.977 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:29.853334+0000", 2023-09-23T20:05:07.977 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.5325999999999999e-05 2023-09-23T20:05:07.977 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.978 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.978 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:05:07.978 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:29.853349+0000", 2023-09-23T20:05:07.978 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.5669e-05 2023-09-23T20:05:07.978 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.978 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.978 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:05:07.978 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:29.853385+0000", 2023-09-23T20:05:07.978 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.8026e-05 2023-09-23T20:05:07.978 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.978 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.978 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:05:07.979 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:29.853403+0000", 2023-09-23T20:05:07.979 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00019367700000000001 2023-09-23T20:05:07.979 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.979 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.979 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:05:07.979 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:29.853596+0000", 2023-09-23T20:05:07.979 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.6560000000000001e-05 2023-09-23T20:05:07.979 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.979 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.979 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op must wait for map", 2023-09-23T20:05:07.979 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:29.853613+0000", 2023-09-23T20:05:07.979 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.010539682999999999 2023-09-23T20:05:07.980 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.980 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.980 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:05:07.980 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:29.864153+0000", 2023-09-23T20:05:07.980 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00012343400000000001 2023-09-23T20:05:07.980 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.980 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.980 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:05:07.980 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:29.864276+0000", 2023-09-23T20:05:07.980 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.010939589 2023-09-23T20:05:07.980 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:07.980 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:05:07.980 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:07.981 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.981 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.981 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.37161.0:64 2.3 2:c751de2b:::rbd_data.b80d7e31e638.0000000000000035:head [sparse-read 0~4194304 [fadvise_nocache]] snapc 0=[] ondisk+read+known_if_redirected e285)", 2023-09-23T20:05:07.981 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T20:02:29.853439+0000", 2023-09-23T20:05:07.981 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 158.03511136099999, 2023-09-23T20:05:07.981 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.010955055999999999, 2023-09-23T20:05:07.981 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:05:07.981 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "reached pg", 2023-09-23T20:05:07.981 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:05:07.981 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.37161", 2023-09-23T20:05:07.981 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/407694934", 2023-09-23T20:05:07.981 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 64 2023-09-23T20:05:07.982 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.982 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:05:07.982 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.982 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:05:07.982 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:29.853439+0000", 2023-09-23T20:05:07.982 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:05:07.982 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.982 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.982 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:05:07.982 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:29.853439+0000", 2023-09-23T20:05:07.982 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T20:05:07.982 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.983 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.983 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:05:07.983 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:29.853437+0000", 2023-09-23T20:05:07.983 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.7930000000000003e-06 2023-09-23T20:05:07.983 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.983 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.983 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:05:07.983 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:29.853446+0000", 2023-09-23T20:05:07.983 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.3716000000000001e-05 2023-09-23T20:05:07.983 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.983 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.983 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:05:07.984 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:29.853470+0000", 2023-09-23T20:05:07.984 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.3456e-05 2023-09-23T20:05:07.984 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.984 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.984 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:05:07.984 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:29.853483+0000", 2023-09-23T20:05:07.984 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000250117 2023-09-23T20:05:07.984 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.984 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.984 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:05:07.984 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:29.853734+0000", 2023-09-23T20:05:07.985 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.3549e-05 2023-09-23T20:05:07.985 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.985 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.985 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:05:07.985 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:29.853747+0000", 2023-09-23T20:05:07.985 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.010558988 2023-09-23T20:05:07.985 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.985 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.985 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:05:07.985 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:29.864306+0000", 2023-09-23T20:05:07.985 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.7919999999999998e-05 2023-09-23T20:05:07.985 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.985 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.986 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:05:07.986 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:29.864394+0000", 2023-09-23T20:05:07.986 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.010955055999999999 2023-09-23T20:05:07.986 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:07.986 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:05:07.986 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:07.986 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.986 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.986 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.47144.0:23 2.3 2:dd15fc62:::rbd_object_map.b8226285b1bf:head [read 0~35] snapc 0=[] ondisk+read+known_if_redirected e286)", 2023-09-23T20:05:07.986 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T20:02:33.862116+0000", 2023-09-23T20:05:07.986 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 154.02643426200001, 2023-09-23T20:05:07.986 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.003856562, 2023-09-23T20:05:07.987 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:05:07.987 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "started", 2023-09-23T20:05:07.987 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:05:07.987 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.47144", 2023-09-23T20:05:07.987 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/1636197388", 2023-09-23T20:05:07.987 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 23 2023-09-23T20:05:07.987 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.987 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:05:07.987 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.987 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:05:07.987 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:33.862116+0000", 2023-09-23T20:05:07.987 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:05:07.988 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.988 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.988 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:05:07.988 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:33.862116+0000", 2023-09-23T20:05:07.988 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999967 2023-09-23T20:05:07.988 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.988 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.988 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:05:07.988 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:33.862113+0000", 2023-09-23T20:05:07.988 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.5928000000000001e-05 2023-09-23T20:05:07.988 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.988 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.988 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:05:07.989 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:33.862129+0000", 2023-09-23T20:05:07.989 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.9717999999999999e-05 2023-09-23T20:05:07.989 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.989 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.989 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:05:07.989 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:33.862159+0000", 2023-09-23T20:05:07.989 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.7510000000000003e-06 2023-09-23T20:05:07.989 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.989 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.989 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:05:07.989 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:33.862168+0000", 2023-09-23T20:05:07.989 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.3032000000000002e-05 2023-09-23T20:05:07.989 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.990 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.990 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:05:07.990 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:33.862231+0000", 2023-09-23T20:05:07.990 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.5750000000000001e-06 2023-09-23T20:05:07.990 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.990 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.990 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op must wait for map", 2023-09-23T20:05:07.990 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:33.862240+0000", 2023-09-23T20:05:07.990 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0034958340000000002 2023-09-23T20:05:07.990 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.990 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.990 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:05:07.991 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:33.865736+0000", 2023-09-23T20:05:07.991 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00012716299999999999 2023-09-23T20:05:07.991 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.991 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.991 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:05:07.991 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:33.865863+0000", 2023-09-23T20:05:07.991 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000109669 2023-09-23T20:05:07.991 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.991 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.991 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:05:07.991 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:33.865973+0000", 2023-09-23T20:05:07.991 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.003856562 2023-09-23T20:05:07.992 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:07.992 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:05:07.992 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:07.992 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.992 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.992 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.47222.0:29 2.3 2:dd15fc62:::rbd_object_map.b8226285b1bf:head [call lock.unlock in=10b] snapc 0=[] ondisk+write+known_if_redirected+full_try e289)", 2023-09-23T20:05:07.992 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T20:02:36.896639+0000", 2023-09-23T20:05:07.992 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 150.99191149999999, 2023-09-23T20:05:07.992 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0054772900000000001, 2023-09-23T20:05:07.992 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:05:07.992 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:07.992 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:05:07.993 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.47222", 2023-09-23T20:05:07.993 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3175549456", 2023-09-23T20:05:07.993 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 29 2023-09-23T20:05:07.993 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.993 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:05:07.993 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.993 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:05:07.993 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:36.896639+0000", 2023-09-23T20:05:07.993 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:05:07.993 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.993 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.993 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:05:07.994 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:36.896639+0000", 2023-09-23T20:05:07.994 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999962 2023-09-23T20:05:07.994 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.994 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.994 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:05:07.994 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:36.896635+0000", 2023-09-23T20:05:07.994 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.6336000000000001e-05 2023-09-23T20:05:07.994 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.994 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.994 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:05:07.994 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:36.896651+0000", 2023-09-23T20:05:07.994 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.1090000000000002e-05 2023-09-23T20:05:07.995 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.995 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.995 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:05:07.995 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:36.896682+0000", 2023-09-23T20:05:07.995 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.5549e-05 2023-09-23T20:05:07.995 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.995 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.995 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:05:07.995 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:36.896698+0000", 2023-09-23T20:05:07.995 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.5407000000000003e-05 2023-09-23T20:05:07.995 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.995 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.996 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:05:07.996 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:36.896793+0000", 2023-09-23T20:05:07.996 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.1765e-05 2023-09-23T20:05:07.996 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.996 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.996 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op must wait for map", 2023-09-23T20:05:07.996 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:36.896805+0000", 2023-09-23T20:05:07.996 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0037982609999999998 2023-09-23T20:05:07.996 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.996 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.996 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:05:07.996 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:36.900603+0000", 2023-09-23T20:05:07.996 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.4709999999999995e-05 2023-09-23T20:05:07.997 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.997 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.997 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:05:07.997 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:36.900678+0000", 2023-09-23T20:05:07.997 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000205999 2023-09-23T20:05:07.997 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.997 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.997 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 2,7", 2023-09-23T20:05:07.997 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:36.900884+0000", 2023-09-23T20:05:07.997 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00074037799999999998 2023-09-23T20:05:07.997 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.997 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.998 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:05:07.998 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:36.901625+0000", 2023-09-23T20:05:07.998 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00025618400000000001 2023-09-23T20:05:07.998 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.998 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.998 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:07.998 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:36.901881+0000", 2023-09-23T20:05:07.998 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00014067100000000001 2023-09-23T20:05:07.998 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.998 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.998 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:07.998 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:36.902021+0000", 2023-09-23T20:05:07.999 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.8430999999999997e-05 2023-09-23T20:05:07.999 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.999 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.999 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:05:07.999 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:36.902080+0000", 2023-09-23T20:05:07.999 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.6214000000000003e-05 2023-09-23T20:05:07.999 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:07.999 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:07.999 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:05:07.999 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:36.902116+0000", 2023-09-23T20:05:07.999 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0054772900000000001 2023-09-23T20:05:07.999 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:08.000 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:05:08.000 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:08.000 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.000 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.000 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.47312.0:8 2.3 2:ca16ec50:::rbd_header.b8d031d2592e:head [create,call rbd.create in=9b] snapc 0=[] ondisk+write+known_if_redirected e289)", 2023-09-23T20:05:08.000 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T20:02:38.546467+0000", 2023-09-23T20:05:08.000 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 149.342082938, 2023-09-23T20:05:08.000 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0022935949999999998, 2023-09-23T20:05:08.000 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:05:08.000 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:08.000 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:05:08.000 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.47312", 2023-09-23T20:05:08.000 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/1922906319", 2023-09-23T20:05:08.001 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 8 2023-09-23T20:05:08.001 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.001 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:05:08.001 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.001 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:05:08.001 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:38.546467+0000", 2023-09-23T20:05:08.001 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:05:08.001 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.001 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.001 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:05:08.001 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:38.546467+0000", 2023-09-23T20:05:08.001 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999976 2023-09-23T20:05:08.002 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.002 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.002 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:05:08.002 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:38.546465+0000", 2023-09-23T20:05:08.002 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.2531000000000001e-05 2023-09-23T20:05:08.002 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.002 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.002 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:05:08.002 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:38.546478+0000", 2023-09-23T20:05:08.002 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.0781999999999999e-05 2023-09-23T20:05:08.002 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.002 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.003 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:05:08.003 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:38.546498+0000", 2023-09-23T20:05:08.003 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.2140000000000001e-05 2023-09-23T20:05:08.003 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.003 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.003 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:05:08.003 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:38.546511+0000", 2023-09-23T20:05:08.003 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.3283999999999997e-05 2023-09-23T20:05:08.003 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.003 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.003 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:05:08.003 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:38.546604+0000", 2023-09-23T20:05:08.004 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00019171999999999999 2023-09-23T20:05:08.004 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.004 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.004 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:05:08.004 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:38.546796+0000", 2023-09-23T20:05:08.004 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00036764399999999998 2023-09-23T20:05:08.004 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.004 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.004 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 2,7", 2023-09-23T20:05:08.004 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:38.547163+0000", 2023-09-23T20:05:08.004 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00072700300000000005 2023-09-23T20:05:08.004 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.005 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.005 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:05:08.005 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:38.547890+0000", 2023-09-23T20:05:08.005 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00029605300000000001 2023-09-23T20:05:08.005 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.005 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.005 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:08.005 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:38.548186+0000", 2023-09-23T20:05:08.005 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00033232800000000001 2023-09-23T20:05:08.005 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.005 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.005 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:08.006 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:38.548519+0000", 2023-09-23T20:05:08.006 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00013531 2023-09-23T20:05:08.006 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.006 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.006 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:05:08.006 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:38.548654+0000", 2023-09-23T20:05:08.006 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000107043 2023-09-23T20:05:08.006 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.006 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.006 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:05:08.006 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:38.548761+0000", 2023-09-23T20:05:08.006 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0022935949999999998 2023-09-23T20:05:08.007 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:08.007 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:05:08.007 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:08.007 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.007 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.007 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.47312.0:9 2.3 2:ca16ec50:::rbd_header.b8d031d2592e:head [call rbd.set_stripe_unit_count in=24b] snapc 0=[] ondisk+write+known_if_redirected e289)", 2023-09-23T20:05:08.007 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T20:02:38.548889+0000", 2023-09-23T20:05:08.007 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 149.339661525, 2023-09-23T20:05:08.007 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.002472924, 2023-09-23T20:05:08.007 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:05:08.007 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:08.007 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:05:08.008 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.47312", 2023-09-23T20:05:08.008 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/1922906319", 2023-09-23T20:05:08.008 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 9 2023-09-23T20:05:08.008 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.008 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:05:08.008 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.008 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:05:08.008 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:38.548889+0000", 2023-09-23T20:05:08.008 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:05:08.008 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.008 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.008 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:05:08.009 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:38.548889+0000", 2023-09-23T20:05:08.009 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T20:05:08.009 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.009 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.009 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:05:08.009 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:38.548887+0000", 2023-09-23T20:05:08.009 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.1039999999999999e-05 2023-09-23T20:05:08.009 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.009 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.009 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:05:08.009 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:38.548898+0000", 2023-09-23T20:05:08.009 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.2096e-05 2023-09-23T20:05:08.010 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.010 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.010 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:05:08.010 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:38.548920+0000", 2023-09-23T20:05:08.010 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.7229999999999999e-06 2023-09-23T20:05:08.010 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.010 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.010 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:05:08.010 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:38.548930+0000", 2023-09-23T20:05:08.010 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.8145000000000004e-05 2023-09-23T20:05:08.010 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.010 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.011 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:05:08.011 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:38.549018+0000", 2023-09-23T20:05:08.011 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000212642 2023-09-23T20:05:08.011 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.011 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.011 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:05:08.011 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:38.549230+0000", 2023-09-23T20:05:08.011 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00066238400000000002 2023-09-23T20:05:08.011 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.011 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.011 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 2,7", 2023-09-23T20:05:08.011 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:38.549893+0000", 2023-09-23T20:05:08.012 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00069118499999999998 2023-09-23T20:05:08.012 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.012 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.012 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:05:08.012 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:38.550584+0000", 2023-09-23T20:05:08.012 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00024863100000000003 2023-09-23T20:05:08.012 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.012 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.012 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:08.012 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:38.550833+0000", 2023-09-23T20:05:08.012 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000296211 2023-09-23T20:05:08.012 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.013 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.013 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:08.013 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:38.551129+0000", 2023-09-23T20:05:08.013 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00012848800000000001 2023-09-23T20:05:08.013 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.013 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.013 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:05:08.013 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:38.551257+0000", 2023-09-23T20:05:08.013 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00010433399999999999 2023-09-23T20:05:08.013 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.013 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.013 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:05:08.014 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:38.551362+0000", 2023-09-23T20:05:08.014 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.002472924 2023-09-23T20:05:08.014 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:08.014 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:05:08.014 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:08.014 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.014 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.014 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.37467.0:51 2.3 2:c4c92e5a:::rbd_trash:head [call rbd.trash_add in=12b] snapc 0=[] ondisk+write+known_if_redirected+full_try e297)", 2023-09-23T20:05:08.014 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T20:02:47.052596+0000", 2023-09-23T20:05:08.014 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 140.835954106, 2023-09-23T20:05:08.014 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.007273809, 2023-09-23T20:05:08.015 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:05:08.015 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:08.015 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:05:08.015 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.37467", 2023-09-23T20:05:08.015 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/790524163", 2023-09-23T20:05:08.015 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 51 2023-09-23T20:05:08.015 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.015 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:05:08.015 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.015 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:05:08.015 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:47.052596+0000", 2023-09-23T20:05:08.015 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:05:08.015 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.016 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.016 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:05:08.016 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:47.052596+0000", 2023-09-23T20:05:08.016 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999976 2023-09-23T20:05:08.016 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.016 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.016 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:05:08.016 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:47.052594+0000", 2023-09-23T20:05:08.016 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.1914e-05 2023-09-23T20:05:08.016 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.016 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.017 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:05:08.017 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:47.052606+0000", 2023-09-23T20:05:08.017 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.9752e-05 2023-09-23T20:05:08.017 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.017 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.017 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:05:08.017 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:47.052625+0000", 2023-09-23T20:05:08.017 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.6455000000000001e-05 2023-09-23T20:05:08.017 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.017 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.017 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:05:08.017 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:47.052642+0000", 2023-09-23T20:05:08.017 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.8367000000000003e-05 2023-09-23T20:05:08.018 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.018 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.018 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:05:08.018 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:47.052730+0000", 2023-09-23T20:05:08.018 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.2507000000000001e-05 2023-09-23T20:05:08.018 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.018 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.018 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op must wait for map", 2023-09-23T20:05:08.018 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:47.052743+0000", 2023-09-23T20:05:08.018 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0049522990000000003 2023-09-23T20:05:08.018 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.018 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.019 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:05:08.019 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:47.057695+0000", 2023-09-23T20:05:08.019 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00011051 2023-09-23T20:05:08.019 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.019 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.019 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:05:08.019 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:47.057806+0000", 2023-09-23T20:05:08.019 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00026958700000000001 2023-09-23T20:05:08.019 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.019 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.019 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 2,7", 2023-09-23T20:05:08.019 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:47.058075+0000", 2023-09-23T20:05:08.019 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00086280299999999999 2023-09-23T20:05:08.020 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.020 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.020 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:05:08.020 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:47.058938+0000", 2023-09-23T20:05:08.020 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00049688499999999997 2023-09-23T20:05:08.021 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.021 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.021 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:08.021 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:47.059435+0000", 2023-09-23T20:05:08.021 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00033292199999999998 2023-09-23T20:05:08.021 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.021 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.021 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:08.021 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:47.059768+0000", 2023-09-23T20:05:08.021 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.7073000000000001e-05 2023-09-23T20:05:08.022 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.022 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.022 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:05:08.022 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:47.059825+0000", 2023-09-23T20:05:08.022 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.5175e-05 2023-09-23T20:05:08.022 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.022 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.022 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:05:08.022 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:47.059870+0000", 2023-09-23T20:05:08.022 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.007273809 2023-09-23T20:05:08.022 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:08.022 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:05:08.023 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:08.023 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.023 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.023 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.47489.0:51 2.3 2:c4c92e5a:::rbd_trash:head [call rbd.trash_add in=12b] snapc 0=[] ondisk+write+known_if_redirected+full_try e297)", 2023-09-23T20:05:08.023 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T20:02:49.221791+0000", 2023-09-23T20:05:08.023 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 138.66675908900001, 2023-09-23T20:05:08.023 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0023441849999999999, 2023-09-23T20:05:08.023 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:05:08.023 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:08.023 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:05:08.023 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.47489", 2023-09-23T20:05:08.023 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3995923599", 2023-09-23T20:05:08.024 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 51 2023-09-23T20:05:08.024 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.024 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:05:08.024 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.024 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:05:08.024 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:49.221791+0000", 2023-09-23T20:05:08.024 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:05:08.024 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.024 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.024 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:05:08.024 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:49.221791+0000", 2023-09-23T20:05:08.024 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999971 2023-09-23T20:05:08.025 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.025 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.025 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:05:08.025 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:49.221788+0000", 2023-09-23T20:05:08.025 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.3365e-05 2023-09-23T20:05:08.025 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.025 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.025 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:05:08.025 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:49.221802+0000", 2023-09-23T20:05:08.025 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.2739e-05 2023-09-23T20:05:08.025 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.025 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.026 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:05:08.026 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:49.221824+0000", 2023-09-23T20:05:08.026 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.3128e-05 2023-09-23T20:05:08.026 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.026 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.026 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:05:08.026 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:49.221838+0000", 2023-09-23T20:05:08.026 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00012639800000000001 2023-09-23T20:05:08.026 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.026 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.026 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:05:08.026 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:49.221964+0000", 2023-09-23T20:05:08.027 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00021777100000000001 2023-09-23T20:05:08.027 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.027 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.027 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:05:08.027 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:49.222182+0000", 2023-09-23T20:05:08.027 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000544317 2023-09-23T20:05:08.027 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.027 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.027 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 2,7", 2023-09-23T20:05:08.027 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:49.222726+0000", 2023-09-23T20:05:08.027 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00085235300000000003 2023-09-23T20:05:08.027 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.028 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.028 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:05:08.028 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:49.223578+0000", 2023-09-23T20:05:08.028 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00028404899999999998 2023-09-23T20:05:08.028 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.028 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.028 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:08.028 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:49.223862+0000", 2023-09-23T20:05:08.028 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00017988499999999999 2023-09-23T20:05:08.028 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.028 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.028 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:08.029 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:49.224042+0000", 2023-09-23T20:05:08.029 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.1770000000000001e-05 2023-09-23T20:05:08.029 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.029 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.029 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:05:08.029 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:49.224094+0000", 2023-09-23T20:05:08.029 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.1232000000000001e-05 2023-09-23T20:05:08.029 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:08.029 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:08.029 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:05:08.029 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:49.224135+0000", 2023-09-23T20:05:08.029 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0023441849999999999 2023-09-23T20:05:08.029 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:08.030 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:05:08.030 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:08.030 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:08.030 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:05:08.030 INFO:teuthology.orchestra.run.smithi092.stdout:} 2023-09-23T20:05:08.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:08 smithi092 bash[58855]: cluster 2023-09-23T20:05:06.739253+0000 mgr.y (mgr.25325) 1738 : cluster 0 pgmap v1823: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:08.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:08 smithi016 bash[75004]: cluster 2023-09-23T20:05:06.739253+0000 mgr.y (mgr.25325) 1738 : cluster 0 pgmap v1823: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:08.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:08 smithi016 bash[77995]: cluster 2023-09-23T20:05:06.739253+0000 mgr.y (mgr.25325) 1738 : cluster 0 pgmap v1823: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:08.780 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T20:05:10.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:10 smithi016 bash[75004]: cluster 2023-09-23T20:05:08.740015+0000 mgr.y (mgr.25325) 1739 : cluster 0 pgmap v1824: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:10.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:10 smithi016 bash[77995]: cluster 2023-09-23T20:05:08.740015+0000 mgr.y (mgr.25325) 1739 : cluster 0 pgmap v1824: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:10.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:10 smithi092 bash[58855]: cluster 2023-09-23T20:05:08.740015+0000 mgr.y (mgr.25325) 1739 : cluster 0 pgmap v1824: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:12.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:12 smithi016 bash[75004]: cluster 2023-09-23T20:05:10.740918+0000 mgr.y (mgr.25325) 1740 : cluster 0 pgmap v1825: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:12.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:12 smithi016 bash[77995]: cluster 2023-09-23T20:05:10.740918+0000 mgr.y (mgr.25325) 1740 : cluster 0 pgmap v1825: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:12.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:12 smithi092 bash[58855]: cluster 2023-09-23T20:05:10.740918+0000 mgr.y (mgr.25325) 1740 : cluster 0 pgmap v1825: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:12.593 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:05:13.560 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:05:14.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:14 smithi016 bash[77995]: cluster 2023-09-23T20:05:12.741557+0000 mgr.y (mgr.25325) 1741 : cluster 0 pgmap v1826: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:14.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:14 smithi016 bash[77995]: audit 2023-09-23T20:05:13.704850+0000 mon.a (mon.0) 2117 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:05:14.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:14 smithi016 bash[75004]: cluster 2023-09-23T20:05:12.741557+0000 mgr.y (mgr.25325) 1741 : cluster 0 pgmap v1826: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:14.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:14 smithi016 bash[75004]: audit 2023-09-23T20:05:13.704850+0000 mon.a (mon.0) 2117 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:05:14.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:14 smithi092 bash[58855]: cluster 2023-09-23T20:05:12.741557+0000 mgr.y (mgr.25325) 1741 : cluster 0 pgmap v1826: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:14.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:14 smithi092 bash[58855]: audit 2023-09-23T20:05:13.704850+0000 mon.a (mon.0) 2117 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:05:16.035 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:05:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:20:05:15] "GET /metrics HTTP/1.1" 200 34015 "" "Prometheus/2.33.4" 2023-09-23T20:05:16.428 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:16 smithi092 bash[58855]: cluster 2023-09-23T20:05:14.742148+0000 mgr.y (mgr.25325) 1742 : cluster 0 pgmap v1827: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:16.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:16 smithi016 bash[75004]: cluster 2023-09-23T20:05:14.742148+0000 mgr.y (mgr.25325) 1742 : cluster 0 pgmap v1827: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:16.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:16 smithi016 bash[77995]: cluster 2023-09-23T20:05:14.742148+0000 mgr.y (mgr.25325) 1742 : cluster 0 pgmap v1827: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:16.590 INFO:teuthology.orchestra.run.smithi092.stdout:{ 2023-09-23T20:05:16.590 INFO:teuthology.orchestra.run.smithi092.stdout: "ops": [], 2023-09-23T20:05:16.590 INFO:teuthology.orchestra.run.smithi092.stdout: "num_ops": 0 2023-09-23T20:05:16.590 INFO:teuthology.orchestra.run.smithi092.stdout:} 2023-09-23T20:05:17.181 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T20:05:17.386 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T20:05:17.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:17 smithi016 bash[77995]: audit 2023-09-23T20:05:16.270030+0000 mon.c (mon.2) 259 : audit 1 from='client.? 172.21.15.16:0/2940349315' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:05:17.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:17 smithi016 bash[77995]: audit 2023-09-23T20:05:16.270601+0000 mon.a (mon.0) 2118 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:05:17.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:17 smithi016 bash[75004]: audit 2023-09-23T20:05:16.270030+0000 mon.c (mon.2) 259 : audit 1 from='client.? 172.21.15.16:0/2940349315' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:05:17.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:17 smithi016 bash[75004]: audit 2023-09-23T20:05:16.270601+0000 mon.a (mon.0) 2118 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:05:17.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:17 smithi092 bash[58855]: audit 2023-09-23T20:05:16.270030+0000 mon.c (mon.2) 259 : audit 1 from='client.? 172.21.15.16:0/2940349315' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:05:17.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:17 smithi092 bash[58855]: audit 2023-09-23T20:05:16.270601+0000 mon.a (mon.0) 2118 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:05:18.067 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 5 1 2023-09-23T20:05:18.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:18 smithi016 bash[75004]: cluster 2023-09-23T20:05:16.743017+0000 mgr.y (mgr.25325) 1743 : cluster 0 pgmap v1828: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:18.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:18 smithi016 bash[75004]: cluster 2023-09-23T20:05:17.172893+0000 osd.7 (osd.7) 247 : cluster 3 failed to encode map e325 with expected crc 2023-09-23T20:05:18.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:18 smithi016 bash[75004]: audit 2023-09-23T20:05:17.173650+0000 mon.a (mon.0) 2119 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T20:05:18.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:18 smithi016 bash[75004]: cluster 2023-09-23T20:05:17.173802+0000 mon.a (mon.0) 2120 : cluster 0 osdmap e325: 8 total, 8 up, 8 in 2023-09-23T20:05:18.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:18 smithi016 bash[75004]: cluster 2023-09-23T20:05:17.348411+0000 osd.4 (osd.4) 271 : cluster 3 failed to encode map e325 with expected crc 2023-09-23T20:05:18.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:18 smithi016 bash[75004]: cluster 2023-09-23T20:05:17.348416+0000 osd.5 (osd.5) 242 : cluster 3 failed to encode map e325 with expected crc 2023-09-23T20:05:18.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:18 smithi016 bash[75004]: cluster 2023-09-23T20:05:17.349010+0000 osd.6 (osd.6) 263 : cluster 3 failed to encode map e325 with expected crc 2023-09-23T20:05:18.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:18 smithi016 bash[77995]: cluster 2023-09-23T20:05:16.743017+0000 mgr.y (mgr.25325) 1743 : cluster 0 pgmap v1828: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:18.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:18 smithi016 bash[77995]: cluster 2023-09-23T20:05:17.172893+0000 osd.7 (osd.7) 247 : cluster 3 failed to encode map e325 with expected crc 2023-09-23T20:05:18.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:18 smithi016 bash[77995]: audit 2023-09-23T20:05:17.173650+0000 mon.a (mon.0) 2119 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T20:05:18.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:18 smithi016 bash[77995]: cluster 2023-09-23T20:05:17.173802+0000 mon.a (mon.0) 2120 : cluster 0 osdmap e325: 8 total, 8 up, 8 in 2023-09-23T20:05:18.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:18 smithi016 bash[77995]: cluster 2023-09-23T20:05:17.348411+0000 osd.4 (osd.4) 271 : cluster 3 failed to encode map e325 with expected crc 2023-09-23T20:05:18.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:18 smithi016 bash[77995]: cluster 2023-09-23T20:05:17.348416+0000 osd.5 (osd.5) 242 : cluster 3 failed to encode map e325 with expected crc 2023-09-23T20:05:18.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:18 smithi016 bash[77995]: cluster 2023-09-23T20:05:17.349010+0000 osd.6 (osd.6) 263 : cluster 3 failed to encode map e325 with expected crc 2023-09-23T20:05:18.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:18 smithi092 bash[58855]: cluster 2023-09-23T20:05:16.743017+0000 mgr.y (mgr.25325) 1743 : cluster 0 pgmap v1828: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:18.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:18 smithi092 bash[58855]: cluster 2023-09-23T20:05:17.172893+0000 osd.7 (osd.7) 247 : cluster 3 failed to encode map e325 with expected crc 2023-09-23T20:05:18.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:18 smithi092 bash[58855]: audit 2023-09-23T20:05:17.173650+0000 mon.a (mon.0) 2119 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T20:05:18.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:18 smithi092 bash[58855]: cluster 2023-09-23T20:05:17.173802+0000 mon.a (mon.0) 2120 : cluster 0 osdmap e325: 8 total, 8 up, 8 in 2023-09-23T20:05:18.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:18 smithi092 bash[58855]: cluster 2023-09-23T20:05:17.348411+0000 osd.4 (osd.4) 271 : cluster 3 failed to encode map e325 with expected crc 2023-09-23T20:05:18.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:18 smithi092 bash[58855]: cluster 2023-09-23T20:05:17.348416+0000 osd.5 (osd.5) 242 : cluster 3 failed to encode map e325 with expected crc 2023-09-23T20:05:18.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:18 smithi092 bash[58855]: cluster 2023-09-23T20:05:17.349010+0000 osd.6 (osd.6) 263 : cluster 3 failed to encode map e325 with expected crc 2023-09-23T20:05:20.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:20 smithi016 bash[75004]: cluster 2023-09-23T20:05:18.743707+0000 mgr.y (mgr.25325) 1744 : cluster 0 pgmap v1830: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:20.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:20 smithi016 bash[77995]: cluster 2023-09-23T20:05:18.743707+0000 mgr.y (mgr.25325) 1744 : cluster 0 pgmap v1830: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:20.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:20 smithi092 bash[58855]: cluster 2023-09-23T20:05:18.743707+0000 mgr.y (mgr.25325) 1744 : cluster 0 pgmap v1830: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:22.162 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:05:22.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:22 smithi016 bash[75004]: cluster 2023-09-23T20:05:20.744637+0000 mgr.y (mgr.25325) 1745 : cluster 0 pgmap v1831: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:22.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:22 smithi016 bash[77995]: cluster 2023-09-23T20:05:20.744637+0000 mgr.y (mgr.25325) 1745 : cluster 0 pgmap v1831: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:22.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:22 smithi092 bash[58855]: cluster 2023-09-23T20:05:20.744637+0000 mgr.y (mgr.25325) 1745 : cluster 0 pgmap v1831: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:22.856 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:05:24.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:24 smithi016 bash[75004]: cluster 2023-09-23T20:05:22.745533+0000 mgr.y (mgr.25325) 1746 : cluster 0 pgmap v1832: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:24.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:24 smithi016 bash[77995]: cluster 2023-09-23T20:05:22.745533+0000 mgr.y (mgr.25325) 1746 : cluster 0 pgmap v1832: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:24.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:24 smithi092 bash[58855]: cluster 2023-09-23T20:05:22.745533+0000 mgr.y (mgr.25325) 1746 : cluster 0 pgmap v1832: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:25.513 INFO:teuthology.orchestra.run.smithi092.stdout:{ 2023-09-23T20:05:25.513 INFO:teuthology.orchestra.run.smithi092.stdout: "ops": [], 2023-09-23T20:05:25.513 INFO:teuthology.orchestra.run.smithi092.stdout: "complaint_time": 30, 2023-09-23T20:05:25.513 INFO:teuthology.orchestra.run.smithi092.stdout: "num_blocked_ops": 0 2023-09-23T20:05:25.513 INFO:teuthology.orchestra.run.smithi092.stdout:} 2023-09-23T20:05:25.942 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:05:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:20:05:25] "GET /metrics HTTP/1.1" 200 34015 "" "Prometheus/2.33.4" 2023-09-23T20:05:26.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:26 smithi016 bash[77995]: cluster 2023-09-23T20:05:24.746211+0000 mgr.y (mgr.25325) 1747 : cluster 0 pgmap v1833: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:26.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:26 smithi016 bash[75004]: cluster 2023-09-23T20:05:24.746211+0000 mgr.y (mgr.25325) 1747 : cluster 0 pgmap v1833: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:26.330 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T20:05:26.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:26 smithi092 bash[58855]: cluster 2023-09-23T20:05:24.746211+0000 mgr.y (mgr.25325) 1747 : cluster 0 pgmap v1833: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:27.207 INFO:teuthology.orchestra.run.smithi016.stderr:set osd.5 primary-affinity to 1 (8655362) 2023-09-23T20:05:27.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:27 smithi016 bash[75004]: audit 2023-09-23T20:05:26.396700+0000 mon.a (mon.0) 2121 : audit 1 from='client.? 172.21.15.16:0/1373105471' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:05:27.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:27 smithi016 bash[77995]: audit 2023-09-23T20:05:26.396700+0000 mon.a (mon.0) 2121 : audit 1 from='client.? 172.21.15.16:0/1373105471' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:05:27.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:27 smithi092 bash[58855]: audit 2023-09-23T20:05:26.396700+0000 mon.a (mon.0) 2121 : audit 1 from='client.? 172.21.15.16:0/1373105471' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:05:28.134 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T20:05:28.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:28 smithi016 bash[75004]: cluster 2023-09-23T20:05:26.747080+0000 mgr.y (mgr.25325) 1748 : cluster 0 pgmap v1834: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:28.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:28 smithi016 bash[75004]: audit 2023-09-23T20:05:27.196997+0000 mon.a (mon.0) 2122 : audit 1 from='client.? 172.21.15.16:0/1373105471' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-09-23T20:05:28.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:28 smithi016 bash[75004]: cluster 2023-09-23T20:05:27.197086+0000 mon.a (mon.0) 2123 : cluster 0 osdmap e326: 8 total, 8 up, 8 in 2023-09-23T20:05:28.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:28 smithi016 bash[75004]: cluster 2023-09-23T20:05:27.202576+0000 osd.5 (osd.5) 243 : cluster 3 failed to encode map e326 with expected crc 2023-09-23T20:05:28.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:28 smithi016 bash[75004]: cluster 2023-09-23T20:05:27.205067+0000 osd.5 (osd.5) 244 : cluster 3 failed to encode map e326 with expected crc 2023-09-23T20:05:28.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:28 smithi016 bash[77995]: cluster 2023-09-23T20:05:26.747080+0000 mgr.y (mgr.25325) 1748 : cluster 0 pgmap v1834: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:28.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:28 smithi016 bash[77995]: audit 2023-09-23T20:05:27.196997+0000 mon.a (mon.0) 2122 : audit 1 from='client.? 172.21.15.16:0/1373105471' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-09-23T20:05:28.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:28 smithi016 bash[77995]: cluster 2023-09-23T20:05:27.197086+0000 mon.a (mon.0) 2123 : cluster 0 osdmap e326: 8 total, 8 up, 8 in 2023-09-23T20:05:28.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:28 smithi016 bash[77995]: cluster 2023-09-23T20:05:27.202576+0000 osd.5 (osd.5) 243 : cluster 3 failed to encode map e326 with expected crc 2023-09-23T20:05:28.537 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:28 smithi016 bash[77995]: cluster 2023-09-23T20:05:27.205067+0000 osd.5 (osd.5) 244 : cluster 3 failed to encode map e326 with expected crc 2023-09-23T20:05:28.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:28 smithi092 bash[58855]: cluster 2023-09-23T20:05:26.747080+0000 mgr.y (mgr.25325) 1748 : cluster 0 pgmap v1834: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:28.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:28 smithi092 bash[58855]: audit 2023-09-23T20:05:27.196997+0000 mon.a (mon.0) 2122 : audit 1 from='client.? 172.21.15.16:0/1373105471' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-09-23T20:05:28.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:28 smithi092 bash[58855]: cluster 2023-09-23T20:05:27.197086+0000 mon.a (mon.0) 2123 : cluster 0 osdmap e326: 8 total, 8 up, 8 in 2023-09-23T20:05:28.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:28 smithi092 bash[58855]: cluster 2023-09-23T20:05:27.202576+0000 osd.5 (osd.5) 243 : cluster 3 failed to encode map e326 with expected crc 2023-09-23T20:05:28.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:28 smithi092 bash[58855]: cluster 2023-09-23T20:05:27.205067+0000 osd.5 (osd.5) 244 : cluster 3 failed to encode map e326 with expected crc 2023-09-23T20:05:29.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:29 smithi016 bash[77995]: cluster 2023-09-23T20:05:28.206467+0000 mon.a (mon.0) 2124 : cluster 0 osdmap e327: 8 total, 8 up, 8 in 2023-09-23T20:05:29.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:29 smithi016 bash[77995]: cluster 2023-09-23T20:05:28.208050+0000 osd.5 (osd.5) 245 : cluster 3 failed to encode map e327 with expected crc 2023-09-23T20:05:29.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:29 smithi016 bash[77995]: cluster 2023-09-23T20:05:28.210159+0000 osd.5 (osd.5) 246 : cluster 3 failed to encode map e327 with expected crc 2023-09-23T20:05:29.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:29 smithi016 bash[77995]: cluster 2023-09-23T20:05:28.211399+0000 osd.5 (osd.5) 247 : cluster 3 failed to encode map e327 with expected crc 2023-09-23T20:05:29.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:29 smithi016 bash[77995]: cluster 2023-09-23T20:05:28.398485+0000 osd.7 (osd.7) 248 : cluster 3 failed to encode map e327 with expected crc 2023-09-23T20:05:29.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:29 smithi016 bash[77995]: audit 2023-09-23T20:05:28.705170+0000 mon.a (mon.0) 2125 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:05:29.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:29 smithi016 bash[75004]: cluster 2023-09-23T20:05:28.206467+0000 mon.a (mon.0) 2124 : cluster 0 osdmap e327: 8 total, 8 up, 8 in 2023-09-23T20:05:29.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:29 smithi016 bash[75004]: cluster 2023-09-23T20:05:28.208050+0000 osd.5 (osd.5) 245 : cluster 3 failed to encode map e327 with expected crc 2023-09-23T20:05:29.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:29 smithi016 bash[75004]: cluster 2023-09-23T20:05:28.210159+0000 osd.5 (osd.5) 246 : cluster 3 failed to encode map e327 with expected crc 2023-09-23T20:05:29.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:29 smithi016 bash[75004]: cluster 2023-09-23T20:05:28.211399+0000 osd.5 (osd.5) 247 : cluster 3 failed to encode map e327 with expected crc 2023-09-23T20:05:29.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:29 smithi016 bash[75004]: cluster 2023-09-23T20:05:28.398485+0000 osd.7 (osd.7) 248 : cluster 3 failed to encode map e327 with expected crc 2023-09-23T20:05:29.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:29 smithi016 bash[75004]: audit 2023-09-23T20:05:28.705170+0000 mon.a (mon.0) 2125 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:05:29.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:29 smithi092 bash[58855]: cluster 2023-09-23T20:05:28.206467+0000 mon.a (mon.0) 2124 : cluster 0 osdmap e327: 8 total, 8 up, 8 in 2023-09-23T20:05:29.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:29 smithi092 bash[58855]: cluster 2023-09-23T20:05:28.208050+0000 osd.5 (osd.5) 245 : cluster 3 failed to encode map e327 with expected crc 2023-09-23T20:05:29.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:29 smithi092 bash[58855]: cluster 2023-09-23T20:05:28.210159+0000 osd.5 (osd.5) 246 : cluster 3 failed to encode map e327 with expected crc 2023-09-23T20:05:29.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:29 smithi092 bash[58855]: cluster 2023-09-23T20:05:28.211399+0000 osd.5 (osd.5) 247 : cluster 3 failed to encode map e327 with expected crc 2023-09-23T20:05:29.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:29 smithi092 bash[58855]: cluster 2023-09-23T20:05:28.398485+0000 osd.7 (osd.7) 248 : cluster 3 failed to encode map e327 with expected crc 2023-09-23T20:05:29.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:29 smithi092 bash[58855]: audit 2023-09-23T20:05:28.705170+0000 mon.a (mon.0) 2125 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:05:30.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:30 smithi016 bash[77995]: cluster 2023-09-23T20:05:28.747694+0000 mgr.y (mgr.25325) 1749 : cluster 0 pgmap v1837: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:30.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:30 smithi016 bash[75004]: cluster 2023-09-23T20:05:28.747694+0000 mgr.y (mgr.25325) 1749 : cluster 0 pgmap v1837: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:30.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:30 smithi092 bash[58855]: cluster 2023-09-23T20:05:28.747694+0000 mgr.y (mgr.25325) 1749 : cluster 0 pgmap v1837: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:31.105 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:05:32.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:32 smithi016 bash[75004]: cluster 2023-09-23T20:05:30.748620+0000 mgr.y (mgr.25325) 1750 : cluster 0 pgmap v1838: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:32.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:32 smithi016 bash[77995]: cluster 2023-09-23T20:05:30.748620+0000 mgr.y (mgr.25325) 1750 : cluster 0 pgmap v1838: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:32.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:32 smithi092 bash[58855]: cluster 2023-09-23T20:05:30.748620+0000 mgr.y (mgr.25325) 1750 : cluster 0 pgmap v1838: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:32.930 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:05:34.351 INFO:teuthology.orchestra.run.smithi092.stdout:{ 2023-09-23T20:05:34.351 INFO:teuthology.orchestra.run.smithi092.stdout: "size": 20, 2023-09-23T20:05:34.351 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 600, 2023-09-23T20:05:34.351 INFO:teuthology.orchestra.run.smithi092.stdout: "ops": [ 2023-09-23T20:05:34.351 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.351 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46640.0:207993 13.4 13:26dbc7d7:::benchmark_data_smithi016_205181_object100729:head [delete] snapc 0=[] ondisk+write+known_if_redirected e250)", 2023-09-23T20:05:34.351 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:55:49.591852+0000", 2023-09-23T20:05:34.352 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 584.75590718000001, 2023-09-23T20:05:34.352 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.18285876700000001, 2023-09-23T20:05:34.352 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:05:34.352 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:34.352 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:05:34.352 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46640", 2023-09-23T20:05:34.352 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2656794142", 2023-09-23T20:05:34.352 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 207993 2023-09-23T20:05:34.352 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.352 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:05:34.352 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.353 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:05:34.353 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:49.591852+0000", 2023-09-23T20:05:34.353 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:05:34.353 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.353 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.353 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:05:34.353 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:49.591852+0000", 2023-09-23T20:05:34.353 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T20:05:34.353 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.353 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.353 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:05:34.353 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:49.591851+0000", 2023-09-23T20:05:34.354 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.7619999999999995e-06 2023-09-23T20:05:34.354 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.354 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.354 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:05:34.354 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:49.591859+0000", 2023-09-23T20:05:34.354 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.4579999999999999e-05 2023-09-23T20:05:34.354 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.354 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.354 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:05:34.354 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:49.591873+0000", 2023-09-23T20:05:34.354 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.5710000000000003e-06 2023-09-23T20:05:34.355 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.355 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.355 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:05:34.355 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:49.591880+0000", 2023-09-23T20:05:34.355 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.1532000000000001e-05 2023-09-23T20:05:34.355 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.355 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.355 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:05:34.355 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:49.591931+0000", 2023-09-23T20:05:34.355 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000169412 2023-09-23T20:05:34.355 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.355 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.356 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:05:34.356 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:49.592101+0000", 2023-09-23T20:05:34.356 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00016271600000000001 2023-09-23T20:05:34.356 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.356 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.356 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:05:34.356 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:49.592263+0000", 2023-09-23T20:05:34.356 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.032553717000000003 2023-09-23T20:05:34.356 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.356 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.356 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:05:34.357 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:49.624817+0000", 2023-09-23T20:05:34.357 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000137229 2023-09-23T20:05:34.357 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.357 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.357 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:34.357 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:49.624954+0000", 2023-09-23T20:05:34.357 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.14960952199999999 2023-09-23T20:05:34.357 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.357 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.357 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:34.357 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:49.774564+0000", 2023-09-23T20:05:34.357 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.5525000000000005e-05 2023-09-23T20:05:34.358 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.358 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.358 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:05:34.358 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:49.774649+0000", 2023-09-23T20:05:34.358 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.1805999999999997e-05 2023-09-23T20:05:34.358 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.358 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.358 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:05:34.358 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:55:49.774711+0000", 2023-09-23T20:05:34.358 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.18285876700000001 2023-09-23T20:05:34.358 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:34.358 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:05:34.359 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:34.359 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.359 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.359 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:43213 14.2 14:409d622b:::benchmark_data_smithi016_228083_object43212:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-09-23T20:05:34.359 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:57:41.006238+0000", 2023-09-23T20:05:34.359 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 473.34152205100003, 2023-09-23T20:05:34.360 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.13062058100000001, 2023-09-23T20:05:34.360 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:05:34.360 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:34.360 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:05:34.360 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T20:05:34.360 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:05:34.360 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 43213 2023-09-23T20:05:34.360 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.361 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:05:34.361 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.361 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:05:34.361 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:41.006238+0000", 2023-09-23T20:05:34.361 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:05:34.361 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.362 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.362 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:05:34.362 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:41.006238+0000", 2023-09-23T20:05:34.362 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.999999 2023-09-23T20:05:34.362 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.362 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.362 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:05:34.362 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:41.006236+0000", 2023-09-23T20:05:34.362 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.0263999999999999e-05 2023-09-23T20:05:34.363 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.363 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.363 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:05:34.363 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:41.006287+0000", 2023-09-23T20:05:34.363 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.6359999999999994e-06 2023-09-23T20:05:34.363 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.363 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.363 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:05:34.363 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:41.006294+0000", 2023-09-23T20:05:34.363 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.3390000000000003e-06 2023-09-23T20:05:34.363 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.363 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.364 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:05:34.364 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:41.006299+0000", 2023-09-23T20:05:34.364 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.5705999999999999e-05 2023-09-23T20:05:34.364 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.364 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.364 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:05:34.364 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:41.006344+0000", 2023-09-23T20:05:34.364 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000109337 2023-09-23T20:05:34.364 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.364 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.364 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:05:34.365 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:41.006454+0000", 2023-09-23T20:05:34.365 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00015609200000000001 2023-09-23T20:05:34.365 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.365 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.365 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:05:34.365 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:41.006610+0000", 2023-09-23T20:05:34.365 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0020831030000000002 2023-09-23T20:05:34.365 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.365 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.365 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:05:34.365 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:41.008693+0000", 2023-09-23T20:05:34.365 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0026887959999999998 2023-09-23T20:05:34.366 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.366 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.366 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:34.366 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:41.011382+0000", 2023-09-23T20:05:34.366 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.12531683699999999 2023-09-23T20:05:34.366 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.366 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.366 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:34.366 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:41.136699+0000", 2023-09-23T20:05:34.366 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.0168999999999998e-05 2023-09-23T20:05:34.366 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.366 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.367 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:05:34.367 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:41.136779+0000", 2023-09-23T20:05:34.367 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.9425000000000006e-05 2023-09-23T20:05:34.367 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.367 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.367 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:05:34.367 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:41.136858+0000", 2023-09-23T20:05:34.367 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.13062058100000001 2023-09-23T20:05:34.367 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:34.367 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:05:34.367 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:34.367 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.368 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.368 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:92183 14.2 14:4b1b4797:::benchmark_data_smithi016_228083_object92182:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e264)", 2023-09-23T20:05:34.368 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:58:34.325235+0000", 2023-09-23T20:05:34.368 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 420.02252458800001, 2023-09-23T20:05:34.368 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.16326417600000001, 2023-09-23T20:05:34.368 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:05:34.368 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:34.368 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:05:34.368 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T20:05:34.368 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:05:34.368 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 92183 2023-09-23T20:05:34.368 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.369 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:05:34.369 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.369 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:05:34.369 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.325235+0000", 2023-09-23T20:05:34.369 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:05:34.369 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.369 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.369 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:05:34.369 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.325235+0000", 2023-09-23T20:05:34.369 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T20:05:34.369 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.369 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.370 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:05:34.370 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.325234+0000", 2023-09-23T20:05:34.370 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.3367000000000005e-05 2023-09-23T20:05:34.370 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.370 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.370 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:05:34.370 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.325297+0000", 2023-09-23T20:05:34.370 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.1691e-05 2023-09-23T20:05:34.370 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.370 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.370 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:05:34.370 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.325309+0000", 2023-09-23T20:05:34.371 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.5110000000000003e-06 2023-09-23T20:05:34.371 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.371 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.371 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:05:34.371 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.325314+0000", 2023-09-23T20:05:34.371 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.9453999999999997e-05 2023-09-23T20:05:34.371 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.371 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.371 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:05:34.371 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.325374+0000", 2023-09-23T20:05:34.371 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000182485 2023-09-23T20:05:34.371 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.372 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.372 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:05:34.372 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.325556+0000", 2023-09-23T20:05:34.372 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000235515 2023-09-23T20:05:34.372 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.372 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.372 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:05:34.372 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.325792+0000", 2023-09-23T20:05:34.372 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0040707800000000004 2023-09-23T20:05:34.372 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.372 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.373 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:05:34.373 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.329863+0000", 2023-09-23T20:05:34.373 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.005011934 2023-09-23T20:05:34.373 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.373 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.373 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:34.373 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.334874+0000", 2023-09-23T20:05:34.373 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.153458495 2023-09-23T20:05:34.373 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.373 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.373 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:34.373 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.488333+0000", 2023-09-23T20:05:34.374 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.3057e-05 2023-09-23T20:05:34.374 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.374 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.374 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:05:34.374 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.488426+0000", 2023-09-23T20:05:34.374 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.3187000000000002e-05 2023-09-23T20:05:34.374 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.374 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.374 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:05:34.374 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.488499+0000", 2023-09-23T20:05:34.374 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.16326417600000001 2023-09-23T20:05:34.374 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:34.375 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:05:34.375 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:34.375 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.375 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.375 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:92189 14.2 14:4802e9bb:::benchmark_data_smithi016_228083_object92188:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e264)", 2023-09-23T20:05:34.375 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:58:34.330412+0000", 2023-09-23T20:05:34.375 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 420.01734798899997, 2023-09-23T20:05:34.375 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.158235811, 2023-09-23T20:05:34.375 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:05:34.375 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:34.375 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:05:34.375 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T20:05:34.376 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:05:34.376 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 92189 2023-09-23T20:05:34.376 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.376 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:05:34.376 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.376 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:05:34.376 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.330412+0000", 2023-09-23T20:05:34.376 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:05:34.376 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.376 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.376 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:05:34.376 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.330412+0000", 2023-09-23T20:05:34.377 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T20:05:34.377 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.377 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.377 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:05:34.377 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.330410+0000", 2023-09-23T20:05:34.377 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.6081999999999993e-05 2023-09-23T20:05:34.377 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.377 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.377 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:05:34.377 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.330496+0000", 2023-09-23T20:05:34.377 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.6662000000000001e-05 2023-09-23T20:05:34.377 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.378 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.378 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:05:34.378 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.330513+0000", 2023-09-23T20:05:34.378 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.145e-05 2023-09-23T20:05:34.378 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.378 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.378 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:05:34.378 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.330524+0000", 2023-09-23T20:05:34.378 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.2972000000000006e-05 2023-09-23T20:05:34.378 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.378 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.378 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:05:34.379 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.330587+0000", 2023-09-23T20:05:34.379 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00019310099999999999 2023-09-23T20:05:34.379 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.379 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.379 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:05:34.379 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.330780+0000", 2023-09-23T20:05:34.379 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000206589 2023-09-23T20:05:34.379 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.379 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.379 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:05:34.379 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.330987+0000", 2023-09-23T20:05:34.380 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0077610200000000004 2023-09-23T20:05:34.380 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.380 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.380 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:05:34.380 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.338748+0000", 2023-09-23T20:05:34.380 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00063484700000000002 2023-09-23T20:05:34.380 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.380 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.380 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:34.380 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.339383+0000", 2023-09-23T20:05:34.380 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.14917364499999999 2023-09-23T20:05:34.380 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.381 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.381 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:34.381 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.488556+0000", 2023-09-23T20:05:34.381 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.6814000000000002e-05 2023-09-23T20:05:34.381 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.381 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.381 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:05:34.381 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.488613+0000", 2023-09-23T20:05:34.381 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.4261999999999998e-05 2023-09-23T20:05:34.381 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.381 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.381 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:05:34.382 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.488647+0000", 2023-09-23T20:05:34.382 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.158235811 2023-09-23T20:05:34.382 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:34.382 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:05:34.382 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:34.382 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.382 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.382 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:92202 14.2 14:40ca811c:::benchmark_data_smithi016_228083_object92201:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e264)", 2023-09-23T20:05:34.382 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:58:34.347175+0000", 2023-09-23T20:05:34.382 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 420.00058489600002, 2023-09-23T20:05:34.382 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.14168755599999999, 2023-09-23T20:05:34.383 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:05:34.383 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:34.383 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:05:34.383 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T20:05:34.383 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:05:34.383 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 92202 2023-09-23T20:05:34.383 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.383 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:05:34.383 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.383 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:05:34.383 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.347175+0000", 2023-09-23T20:05:34.384 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:05:34.384 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.384 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.384 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:05:34.384 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.347175+0000", 2023-09-23T20:05:34.384 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T20:05:34.384 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.384 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.384 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:05:34.384 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.347173+0000", 2023-09-23T20:05:34.384 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.5488999999999996e-05 2023-09-23T20:05:34.384 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.385 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.385 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:05:34.385 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.347238+0000", 2023-09-23T20:05:34.385 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.1846e-05 2023-09-23T20:05:34.385 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.385 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.385 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:05:34.385 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.347250+0000", 2023-09-23T20:05:34.385 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.3360000000000002e-06 2023-09-23T20:05:34.385 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.385 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.385 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:05:34.386 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.347255+0000", 2023-09-23T20:05:34.386 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.1162000000000003e-05 2023-09-23T20:05:34.386 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.386 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.386 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:05:34.386 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.347296+0000", 2023-09-23T20:05:34.386 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000140457 2023-09-23T20:05:34.386 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.386 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.386 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:05:34.386 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.347437+0000", 2023-09-23T20:05:34.387 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00026646900000000003 2023-09-23T20:05:34.387 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.387 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.387 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:05:34.387 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.347703+0000", 2023-09-23T20:05:34.387 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0039196470000000001 2023-09-23T20:05:34.387 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.387 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.387 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:05:34.387 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.351623+0000", 2023-09-23T20:05:34.387 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00023723900000000001 2023-09-23T20:05:34.387 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.387 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.388 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:34.388 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.351860+0000", 2023-09-23T20:05:34.388 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.13684961000000001 2023-09-23T20:05:34.388 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.388 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.388 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:34.388 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.488710+0000", 2023-09-23T20:05:34.388 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.7148000000000001e-05 2023-09-23T20:05:34.388 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.388 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.388 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:05:34.388 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.488787+0000", 2023-09-23T20:05:34.388 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.5240000000000005e-05 2023-09-23T20:05:34.389 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.389 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.389 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:05:34.389 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.488862+0000", 2023-09-23T20:05:34.389 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.14168755599999999 2023-09-23T20:05:34.389 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:34.389 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:05:34.389 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:34.389 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.389 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.389 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:93512 14.2 14:47463c3f:::benchmark_data_smithi016_228083_object93511:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e264)", 2023-09-23T20:05:34.389 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:58:36.104635+0000", 2023-09-23T20:05:34.390 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 418.24312501100002, 2023-09-23T20:05:34.390 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.130115076, 2023-09-23T20:05:34.390 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:05:34.390 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:34.390 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:05:34.390 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T20:05:34.390 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:05:34.390 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 93512 2023-09-23T20:05:34.390 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.390 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:05:34.390 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.390 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:05:34.391 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:36.104635+0000", 2023-09-23T20:05:34.391 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:05:34.391 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.391 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.391 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:05:34.391 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:36.104635+0000", 2023-09-23T20:05:34.391 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T20:05:34.391 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.391 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.391 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:05:34.391 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:36.104633+0000", 2023-09-23T20:05:34.391 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.8351999999999999e-05 2023-09-23T20:05:34.392 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.392 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.392 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:05:34.392 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:36.104702+0000", 2023-09-23T20:05:34.392 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.1873e-05 2023-09-23T20:05:34.392 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.392 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.392 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:05:34.392 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:36.104714+0000", 2023-09-23T20:05:34.392 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.6749999999999999e-06 2023-09-23T20:05:34.392 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.392 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.392 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:05:34.393 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:36.104719+0000", 2023-09-23T20:05:34.393 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.5256999999999997e-05 2023-09-23T20:05:34.393 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.393 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.393 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:05:34.393 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:36.104764+0000", 2023-09-23T20:05:34.393 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00016432799999999999 2023-09-23T20:05:34.393 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.393 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.393 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:05:34.393 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:36.104929+0000", 2023-09-23T20:05:34.393 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00021613200000000001 2023-09-23T20:05:34.393 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.394 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.394 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:05:34.394 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:36.105145+0000", 2023-09-23T20:05:34.394 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.003813795 2023-09-23T20:05:34.394 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.394 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.394 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:05:34.394 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:36.108959+0000", 2023-09-23T20:05:34.394 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.002553254 2023-09-23T20:05:34.394 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.394 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.395 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:34.395 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:36.111512+0000", 2023-09-23T20:05:34.395 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.123000211 2023-09-23T20:05:34.395 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.395 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.395 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:34.395 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:36.234512+0000", 2023-09-23T20:05:34.395 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000145456 2023-09-23T20:05:34.395 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.395 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.395 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:05:34.395 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:36.234658+0000", 2023-09-23T20:05:34.396 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.2038000000000002e-05 2023-09-23T20:05:34.396 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.396 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.396 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:05:34.396 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:36.234750+0000", 2023-09-23T20:05:34.396 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.130115076 2023-09-23T20:05:34.396 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:34.396 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:05:34.396 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:34.396 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.396 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.396 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:98578 14.2 14:44836486:::benchmark_data_smithi016_228083_object98577:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e264)", 2023-09-23T20:05:34.397 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:58:42.892630+0000", 2023-09-23T20:05:34.397 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 411.455129591, 2023-09-23T20:05:34.397 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.13868059299999999, 2023-09-23T20:05:34.397 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:05:34.397 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:34.397 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:05:34.397 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T20:05:34.397 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:05:34.397 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 98578 2023-09-23T20:05:34.397 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.397 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:05:34.397 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.398 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:05:34.398 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:42.892630+0000", 2023-09-23T20:05:34.398 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:05:34.398 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.398 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.398 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:05:34.398 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:42.892630+0000", 2023-09-23T20:05:34.398 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T20:05:34.398 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.398 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.398 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:05:34.398 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:42.892629+0000", 2023-09-23T20:05:34.398 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.9437e-05 2023-09-23T20:05:34.399 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.399 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.399 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:05:34.399 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:42.892688+0000", 2023-09-23T20:05:34.399 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.3705e-05 2023-09-23T20:05:34.399 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.399 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.399 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:05:34.399 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:42.892702+0000", 2023-09-23T20:05:34.399 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.6110000000000001e-06 2023-09-23T20:05:34.399 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.399 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.399 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:05:34.400 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:42.892707+0000", 2023-09-23T20:05:34.400 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.7040999999999998e-05 2023-09-23T20:05:34.400 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.400 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.400 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:05:34.400 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:42.892764+0000", 2023-09-23T20:05:34.400 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00014083999999999999 2023-09-23T20:05:34.400 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.400 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.400 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:05:34.400 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:42.892905+0000", 2023-09-23T20:05:34.401 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00013424300000000001 2023-09-23T20:05:34.401 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.401 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.401 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:05:34.401 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:42.893040+0000", 2023-09-23T20:05:34.401 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0032106830000000002 2023-09-23T20:05:34.401 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.401 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.401 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:05:34.401 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:42.896250+0000", 2023-09-23T20:05:34.401 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0072382649999999998 2023-09-23T20:05:34.401 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.401 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.402 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:34.402 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:42.903488+0000", 2023-09-23T20:05:34.402 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.12764887599999999 2023-09-23T20:05:34.402 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.402 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.402 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:34.402 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:43.031137+0000", 2023-09-23T20:05:34.402 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00010245400000000001 2023-09-23T20:05:34.402 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.402 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.402 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:05:34.402 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:43.031240+0000", 2023-09-23T20:05:34.403 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.0840999999999995e-05 2023-09-23T20:05:34.403 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.403 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.403 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:05:34.403 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:43.031311+0000", 2023-09-23T20:05:34.403 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.13868059299999999 2023-09-23T20:05:34.403 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:34.403 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:05:34.403 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:34.403 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.403 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.403 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:98581 14.2 14:4d3dd6de:::benchmark_data_smithi016_228083_object98580:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e264)", 2023-09-23T20:05:34.404 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:58:42.896573+0000", 2023-09-23T20:05:34.404 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 411.45118613699998, 2023-09-23T20:05:34.404 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.151797139, 2023-09-23T20:05:34.404 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:05:34.404 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:34.404 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:05:34.404 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T20:05:34.404 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:05:34.404 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 98581 2023-09-23T20:05:34.404 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.404 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:05:34.405 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.405 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:05:34.405 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:42.896573+0000", 2023-09-23T20:05:34.405 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:05:34.405 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.405 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.405 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:05:34.405 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:42.896573+0000", 2023-09-23T20:05:34.405 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T20:05:34.405 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.405 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.405 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:05:34.405 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:42.896572+0000", 2023-09-23T20:05:34.406 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.1119999999999998e-05 2023-09-23T20:05:34.406 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.406 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.406 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:05:34.406 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:42.896633+0000", 2023-09-23T20:05:34.406 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.3960000000000001e-05 2023-09-23T20:05:34.406 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.406 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.406 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:05:34.406 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:42.896657+0000", 2023-09-23T20:05:34.406 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.0029999999999997e-06 2023-09-23T20:05:34.407 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.407 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.407 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:05:34.407 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:42.896664+0000", 2023-09-23T20:05:34.407 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.7967999999999995e-05 2023-09-23T20:05:34.407 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.407 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.407 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:05:34.407 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:42.896752+0000", 2023-09-23T20:05:34.407 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00018203499999999999 2023-09-23T20:05:34.407 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.407 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.407 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:05:34.408 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:42.896934+0000", 2023-09-23T20:05:34.408 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00021912900000000001 2023-09-23T20:05:34.408 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.408 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.408 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:05:34.408 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:42.897153+0000", 2023-09-23T20:05:34.408 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0064041860000000001 2023-09-23T20:05:34.408 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.408 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.408 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:34.408 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:42.903557+0000", 2023-09-23T20:05:34.408 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0032486170000000001 2023-09-23T20:05:34.409 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.409 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.409 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:05:34.409 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:42.906806+0000", 2023-09-23T20:05:34.409 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.141463333 2023-09-23T20:05:34.409 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.409 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.409 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:34.409 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:43.048269+0000", 2023-09-23T20:05:34.409 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.1909999999999995e-05 2023-09-23T20:05:34.409 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.409 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.410 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:05:34.410 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:43.048331+0000", 2023-09-23T20:05:34.410 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.9378999999999997e-05 2023-09-23T20:05:34.410 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.410 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.410 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:05:34.410 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:43.048371+0000", 2023-09-23T20:05:34.410 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.151797139 2023-09-23T20:05:34.410 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:34.410 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:05:34.410 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:34.411 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.411 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.411 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:98582 14.2 14:43d85a91:::benchmark_data_smithi016_228083_object98581:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e264)", 2023-09-23T20:05:34.411 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:58:42.896687+0000", 2023-09-23T20:05:34.411 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 411.45107283800002, 2023-09-23T20:05:34.411 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.15182257399999999, 2023-09-23T20:05:34.411 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:05:34.411 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:34.411 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:05:34.411 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T20:05:34.411 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:05:34.411 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 98582 2023-09-23T20:05:34.412 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.412 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:05:34.412 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.412 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:05:34.412 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:42.896687+0000", 2023-09-23T20:05:34.412 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:05:34.412 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.412 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.412 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:05:34.412 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:42.896687+0000", 2023-09-23T20:05:34.412 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999995 2023-09-23T20:05:34.412 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.412 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.413 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:05:34.413 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:42.896686+0000", 2023-09-23T20:05:34.413 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.2604999999999997e-05 2023-09-23T20:05:34.413 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.413 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.413 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:05:34.413 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:42.896719+0000", 2023-09-23T20:05:34.413 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.3000000000000004e-06 2023-09-23T20:05:34.413 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.413 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.413 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:05:34.413 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:42.896726+0000", 2023-09-23T20:05:34.414 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.9600000000000002e-06 2023-09-23T20:05:34.414 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.414 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.414 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:05:34.414 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:42.896730+0000", 2023-09-23T20:05:34.414 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.003285741 2023-09-23T20:05:34.414 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.414 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.414 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:05:34.414 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:42.900016+0000", 2023-09-23T20:05:34.414 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00013740000000000001 2023-09-23T20:05:34.414 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.415 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.415 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:05:34.415 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:42.900153+0000", 2023-09-23T20:05:34.415 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00015567700000000001 2023-09-23T20:05:34.415 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.415 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.415 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:05:34.415 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:42.900309+0000", 2023-09-23T20:05:34.415 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0065014590000000002 2023-09-23T20:05:34.415 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.415 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.416 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:05:34.416 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:42.906810+0000", 2023-09-23T20:05:34.416 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00078754699999999999 2023-09-23T20:05:34.416 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.416 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.416 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:34.416 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:42.907598+0000", 2023-09-23T20:05:34.416 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.140834351 2023-09-23T20:05:34.416 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.416 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.416 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:34.417 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:43.048432+0000", 2023-09-23T20:05:34.417 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.3866999999999999e-05 2023-09-23T20:05:34.417 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.417 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.417 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:05:34.417 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:43.048476+0000", 2023-09-23T20:05:34.417 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.3302000000000001e-05 2023-09-23T20:05:34.417 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.417 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.417 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:05:34.417 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:43.048509+0000", 2023-09-23T20:05:34.417 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.15182257399999999 2023-09-23T20:05:34.417 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:34.418 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:05:34.418 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:34.418 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.418 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.418 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:98583 14.2 14:4e3128f7:::benchmark_data_smithi016_228083_object98582:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e264)", 2023-09-23T20:05:34.418 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:58:42.896754+0000", 2023-09-23T20:05:34.418 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 411.45100590999999, 2023-09-23T20:05:34.418 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.15186918999999999, 2023-09-23T20:05:34.418 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:05:34.418 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:34.418 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:05:34.418 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T20:05:34.419 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:05:34.419 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 98583 2023-09-23T20:05:34.419 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.419 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:05:34.419 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.419 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:05:34.419 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:42.896754+0000", 2023-09-23T20:05:34.419 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:05:34.419 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.419 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.419 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:05:34.419 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:42.896754+0000", 2023-09-23T20:05:34.419 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999995 2023-09-23T20:05:34.420 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.420 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.420 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:05:34.420 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:42.896753+0000", 2023-09-23T20:05:34.420 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.7939000000000001e-05 2023-09-23T20:05:34.420 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.420 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.420 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:05:34.420 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:42.896781+0000", 2023-09-23T20:05:34.420 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.3919999999999992e-06 2023-09-23T20:05:34.421 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.421 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.421 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:05:34.421 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:42.896790+0000", 2023-09-23T20:05:34.421 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.4880000000000004e-06 2023-09-23T20:05:34.421 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.421 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.421 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:05:34.421 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:42.896797+0000", 2023-09-23T20:05:34.421 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0068005380000000001 2023-09-23T20:05:34.421 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.422 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.422 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:05:34.422 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:42.903598+0000", 2023-09-23T20:05:34.422 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00013098600000000001 2023-09-23T20:05:34.422 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.422 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.422 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:05:34.422 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:42.903728+0000", 2023-09-23T20:05:34.422 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00015259 2023-09-23T20:05:34.422 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.422 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.423 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:05:34.423 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:42.903881+0000", 2023-09-23T20:05:34.423 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0031460749999999999 2023-09-23T20:05:34.423 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.423 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.423 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:05:34.423 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:42.907027+0000", 2023-09-23T20:05:34.423 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00069755800000000003 2023-09-23T20:05:34.423 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.423 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.423 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:34.423 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:42.907725+0000", 2023-09-23T20:05:34.424 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.14082932000000001 2023-09-23T20:05:34.424 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.424 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.424 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:34.424 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:43.048554+0000", 2023-09-23T20:05:34.424 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.0062999999999999e-05 2023-09-23T20:05:34.424 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.424 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.424 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:05:34.424 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:43.048594+0000", 2023-09-23T20:05:34.424 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.8802999999999998e-05 2023-09-23T20:05:34.424 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.424 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.425 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:05:34.425 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:43.048623+0000", 2023-09-23T20:05:34.425 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.15186918999999999 2023-09-23T20:05:34.425 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:34.425 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:05:34.425 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:34.425 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.425 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.425 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:131789 14.2 14:4d10793d:::benchmark_data_smithi016_228083_object31830:head [delete] snapc 0=[] ondisk+write+known_if_redirected e267)", 2023-09-23T20:05:34.425 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:59:29.239057+0000", 2023-09-23T20:05:34.425 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 365.10870294599999, 2023-09-23T20:05:34.425 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.229620459, 2023-09-23T20:05:34.426 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:05:34.426 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:34.426 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:05:34.426 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T20:05:34.426 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:05:34.426 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 131789 2023-09-23T20:05:34.426 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.426 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:05:34.426 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.426 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:05:34.426 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:59:29.239057+0000", 2023-09-23T20:05:34.427 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:05:34.427 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.427 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.427 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:05:34.427 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:59:29.239057+0000", 2023-09-23T20:05:34.427 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T20:05:34.427 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.427 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.427 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:05:34.427 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:59:29.239055+0000", 2023-09-23T20:05:34.427 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.2390000000000002e-06 2023-09-23T20:05:34.427 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.427 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.428 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:05:34.428 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:59:29.239063+0000", 2023-09-23T20:05:34.428 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.3703000000000001e-05 2023-09-23T20:05:34.428 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.428 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.428 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:05:34.428 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:59:29.239077+0000", 2023-09-23T20:05:34.428 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.8720000000000002e-06 2023-09-23T20:05:34.428 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.428 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.428 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:05:34.428 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:59:29.239085+0000", 2023-09-23T20:05:34.428 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000108242 2023-09-23T20:05:34.429 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.429 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.429 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:05:34.429 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:59:29.239193+0000", 2023-09-23T20:05:34.429 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000163902 2023-09-23T20:05:34.429 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.429 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.429 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:05:34.429 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:59:29.239357+0000", 2023-09-23T20:05:34.429 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00018038900000000001 2023-09-23T20:05:34.429 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.429 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.429 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:05:34.430 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:59:29.239537+0000", 2023-09-23T20:05:34.430 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0043455020000000002 2023-09-23T20:05:34.430 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.430 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.430 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:34.430 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:59:29.243883+0000", 2023-09-23T20:05:34.430 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00016622699999999999 2023-09-23T20:05:34.430 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.430 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.430 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:05:34.430 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:59:29.244049+0000", 2023-09-23T20:05:34.430 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.224551108 2023-09-23T20:05:34.431 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.431 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.431 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:34.431 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:59:29.468600+0000", 2023-09-23T20:05:34.431 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.2425000000000001e-05 2023-09-23T20:05:34.431 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.431 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.431 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:05:34.431 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:59:29.468643+0000", 2023-09-23T20:05:34.431 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.4399e-05 2023-09-23T20:05:34.431 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.431 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.432 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:05:34.432 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:59:29.468677+0000", 2023-09-23T20:05:34.432 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.229620459 2023-09-23T20:05:34.432 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:34.432 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:05:34.432 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:34.432 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.432 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.432 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:156207 14.4 14:2211dd6c:::benchmark_data_smithi016_228083_object56248:head [delete] snapc 0=[] ondisk+write+known_if_redirected e269)", 2023-09-23T20:05:34.432 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T20:00:06.689672+0000", 2023-09-23T20:05:34.432 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 327.65808789099998, 2023-09-23T20:05:34.432 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.19658257400000001, 2023-09-23T20:05:34.432 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:05:34.433 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:34.433 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:05:34.433 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T20:05:34.433 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:05:34.433 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 156207 2023-09-23T20:05:34.433 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.433 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:05:34.433 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.433 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:05:34.433 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.689672+0000", 2023-09-23T20:05:34.433 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:05:34.433 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.434 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.434 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:05:34.434 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.689672+0000", 2023-09-23T20:05:34.434 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.999999 2023-09-23T20:05:34.434 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.434 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.434 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:05:34.434 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.689671+0000", 2023-09-23T20:05:34.434 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.1210000000000003e-06 2023-09-23T20:05:34.434 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.434 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.434 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:05:34.435 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.689677+0000", 2023-09-23T20:05:34.435 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.9400000000000008e-06 2023-09-23T20:05:34.435 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.435 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.435 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:05:34.435 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.689686+0000", 2023-09-23T20:05:34.435 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.3689999999999998e-06 2023-09-23T20:05:34.435 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.435 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.435 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:05:34.435 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.689691+0000", 2023-09-23T20:05:34.435 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.0745999999999998e-05 2023-09-23T20:05:34.436 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.436 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.436 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:05:34.436 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.689742+0000", 2023-09-23T20:05:34.436 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00013108299999999999 2023-09-23T20:05:34.436 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.436 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.436 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:05:34.436 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.689873+0000", 2023-09-23T20:05:34.436 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000156336 2023-09-23T20:05:34.436 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.436 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.437 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:05:34.437 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.690029+0000", 2023-09-23T20:05:34.437 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0055085969999999996 2023-09-23T20:05:34.437 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.437 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.437 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:05:34.437 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.695538+0000", 2023-09-23T20:05:34.437 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0028824559999999998 2023-09-23T20:05:34.437 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.437 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.437 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:34.438 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.698420+0000", 2023-09-23T20:05:34.438 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.187674441 2023-09-23T20:05:34.438 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.438 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.438 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:34.438 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.886095+0000", 2023-09-23T20:05:34.438 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.7641999999999998e-05 2023-09-23T20:05:34.438 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.438 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.438 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:05:34.438 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.886193+0000", 2023-09-23T20:05:34.439 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.1764e-05 2023-09-23T20:05:34.439 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.439 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.439 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:05:34.439 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.886254+0000", 2023-09-23T20:05:34.439 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.19658257400000001 2023-09-23T20:05:34.439 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:34.439 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:05:34.439 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:34.439 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.439 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.439 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:156225 14.4 14:2e9b25eb:::benchmark_data_smithi016_228083_object56266:head [delete] snapc 0=[] ondisk+write+known_if_redirected e269)", 2023-09-23T20:05:34.440 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T20:00:06.714083+0000", 2023-09-23T20:05:34.440 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 327.63367706100001, 2023-09-23T20:05:34.440 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.172301016, 2023-09-23T20:05:34.440 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:05:34.440 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:34.440 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:05:34.440 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T20:05:34.440 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:05:34.440 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 156225 2023-09-23T20:05:34.440 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.440 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:05:34.440 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.440 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:05:34.440 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.714083+0000", 2023-09-23T20:05:34.441 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:05:34.441 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.441 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.441 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:05:34.441 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.714083+0000", 2023-09-23T20:05:34.441 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T20:05:34.441 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.441 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.441 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:05:34.441 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.714081+0000", 2023-09-23T20:05:34.441 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.3780000000000003e-06 2023-09-23T20:05:34.441 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.442 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.442 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:05:34.442 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.714089+0000", 2023-09-23T20:05:34.442 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.3226e-05 2023-09-23T20:05:34.442 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.442 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.442 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:05:34.442 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.714102+0000", 2023-09-23T20:05:34.442 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.516e-06 2023-09-23T20:05:34.442 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.442 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.443 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:05:34.443 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.714108+0000", 2023-09-23T20:05:34.443 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.1022000000000001e-05 2023-09-23T20:05:34.443 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.443 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.443 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:05:34.443 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.714149+0000", 2023-09-23T20:05:34.443 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.6254000000000005e-05 2023-09-23T20:05:34.443 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.443 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.443 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:05:34.443 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.714236+0000", 2023-09-23T20:05:34.443 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.624e-05 2023-09-23T20:05:34.444 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.444 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.444 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:05:34.444 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.714332+0000", 2023-09-23T20:05:34.444 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0042403700000000003 2023-09-23T20:05:34.444 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.444 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.444 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:05:34.444 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.718572+0000", 2023-09-23T20:05:34.444 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.003762208 2023-09-23T20:05:34.444 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.444 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.445 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:34.445 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.722334+0000", 2023-09-23T20:05:34.445 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.16397790000000001 2023-09-23T20:05:34.445 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.445 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.445 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:34.445 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.886312+0000", 2023-09-23T20:05:34.445 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.4023999999999998e-05 2023-09-23T20:05:34.445 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.445 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.445 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:05:34.446 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.886346+0000", 2023-09-23T20:05:34.446 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.7286000000000002e-05 2023-09-23T20:05:34.446 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.446 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.446 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:05:34.446 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.886384+0000", 2023-09-23T20:05:34.446 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.172301016 2023-09-23T20:05:34.446 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:34.446 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:05:34.446 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:34.446 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.446 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.446 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:177688 14.2 14:44561758:::benchmark_data_smithi016_228083_object77729:head [delete] snapc 0=[] ondisk+write+known_if_redirected e271)", 2023-09-23T20:05:34.447 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T20:00:41.521621+0000", 2023-09-23T20:05:34.447 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 292.82613859100002, 2023-09-23T20:05:34.447 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.171759614, 2023-09-23T20:05:34.447 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:05:34.447 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:34.447 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:05:34.447 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T20:05:34.447 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:05:34.447 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 177688 2023-09-23T20:05:34.447 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.447 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:05:34.447 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.447 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:05:34.448 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.521621+0000", 2023-09-23T20:05:34.448 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:05:34.448 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.448 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.448 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:05:34.448 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.521621+0000", 2023-09-23T20:05:34.448 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.999999 2023-09-23T20:05:34.448 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.448 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.448 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:05:34.448 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.521620+0000", 2023-09-23T20:05:34.448 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.7219999999999999e-06 2023-09-23T20:05:34.448 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.449 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.449 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:05:34.449 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.521625+0000", 2023-09-23T20:05:34.449 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.7839999999999998e-06 2023-09-23T20:05:34.449 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.449 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.449 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:05:34.449 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.521632+0000", 2023-09-23T20:05:34.449 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.0749999999999999e-06 2023-09-23T20:05:34.449 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.449 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.449 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:05:34.450 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.521636+0000", 2023-09-23T20:05:34.450 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.5818000000000001e-05 2023-09-23T20:05:34.450 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.450 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.450 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:05:34.450 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.521682+0000", 2023-09-23T20:05:34.450 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.7113999999999994e-05 2023-09-23T20:05:34.450 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.450 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.450 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:05:34.450 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.521759+0000", 2023-09-23T20:05:34.451 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.0841999999999996e-05 2023-09-23T20:05:34.451 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.451 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.451 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:05:34.451 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.521840+0000", 2023-09-23T20:05:34.451 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0072993449999999996 2023-09-23T20:05:34.451 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.451 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.451 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:05:34.451 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.529139+0000", 2023-09-23T20:05:34.451 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.3352999999999999e-05 2023-09-23T20:05:34.451 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.452 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.452 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:34.452 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.529232+0000", 2023-09-23T20:05:34.452 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.164050107 2023-09-23T20:05:34.452 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.452 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.452 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:34.452 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.693282+0000", 2023-09-23T20:05:34.452 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.4555999999999997e-05 2023-09-23T20:05:34.452 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.452 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.452 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:05:34.453 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.693337+0000", 2023-09-23T20:05:34.453 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.3736999999999998e-05 2023-09-23T20:05:34.453 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.453 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.453 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:05:34.453 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.693381+0000", 2023-09-23T20:05:34.453 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.171759614 2023-09-23T20:05:34.453 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:34.453 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:05:34.453 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:34.453 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.453 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.454 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.37161.0:59 2.7 2:e7e8af32:::rbd_data.b80d7e31e638.000000000000002f:head [sparse-read 0~4194304 [fadvise_nocache]] snapc 0=[] ondisk+read+known_if_redirected e285)", 2023-09-23T20:05:34.454 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T20:02:29.853316+0000", 2023-09-23T20:05:34.454 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 184.494443437, 2023-09-23T20:05:34.454 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.010554554000000001, 2023-09-23T20:05:34.454 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:05:34.454 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "reached pg", 2023-09-23T20:05:34.454 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:05:34.454 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.37161", 2023-09-23T20:05:34.454 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/407694934", 2023-09-23T20:05:34.454 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 59 2023-09-23T20:05:34.454 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.455 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:05:34.455 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.455 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:05:34.455 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:29.853316+0000", 2023-09-23T20:05:34.455 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:05:34.455 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.455 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.455 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:05:34.455 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:29.853316+0000", 2023-09-23T20:05:34.455 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999971 2023-09-23T20:05:34.455 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.455 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.455 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:05:34.456 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:29.853313+0000", 2023-09-23T20:05:34.456 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.5928999999999999e-05 2023-09-23T20:05:34.456 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.456 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.456 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:05:34.456 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:29.853329+0000", 2023-09-23T20:05:34.456 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.1585999999999999e-05 2023-09-23T20:05:34.456 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.456 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.456 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:05:34.456 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:29.853361+0000", 2023-09-23T20:05:34.456 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.4606999999999999e-05 2023-09-23T20:05:34.457 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.457 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.457 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:05:34.457 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:29.853376+0000", 2023-09-23T20:05:34.457 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00017684800000000001 2023-09-23T20:05:34.457 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.457 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.457 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:05:34.458 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:29.853552+0000", 2023-09-23T20:05:34.458 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.0706000000000001e-05 2023-09-23T20:05:34.458 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.458 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.458 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op must wait for map", 2023-09-23T20:05:34.459 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:29.853573+0000", 2023-09-23T20:05:34.459 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.010194328000000001 2023-09-23T20:05:34.459 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.459 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.459 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:05:34.459 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:29.863767+0000", 2023-09-23T20:05:34.459 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000103266 2023-09-23T20:05:34.459 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.459 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.459 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:05:34.460 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:29.863871+0000", 2023-09-23T20:05:34.460 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.010554554000000001 2023-09-23T20:05:34.460 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:34.460 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:05:34.460 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:34.460 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.460 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.460 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.37161.0:61 2.7 2:e8bfd45a:::rbd_data.b80d7e31e638.0000000000000032:head [sparse-read 0~4194304 [fadvise_nocache]] snapc 0=[] ondisk+read+known_if_redirected e285)", 2023-09-23T20:05:34.460 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T20:02:29.853405+0000", 2023-09-23T20:05:34.460 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 184.494355066, 2023-09-23T20:05:34.460 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.010595711000000001, 2023-09-23T20:05:34.460 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:05:34.460 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "reached pg", 2023-09-23T20:05:34.461 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:05:34.461 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.37161", 2023-09-23T20:05:34.461 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/407694934", 2023-09-23T20:05:34.461 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 61 2023-09-23T20:05:34.461 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.461 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:05:34.461 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.461 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:05:34.461 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:29.853405+0000", 2023-09-23T20:05:34.461 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:05:34.461 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.461 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.461 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:05:34.462 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:29.853405+0000", 2023-09-23T20:05:34.462 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T20:05:34.462 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.462 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.462 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:05:34.462 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:29.853403+0000", 2023-09-23T20:05:34.463 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.5429999999999994e-06 2023-09-23T20:05:34.463 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.463 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.463 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:05:34.463 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:29.853412+0000", 2023-09-23T20:05:34.463 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.1086000000000001e-05 2023-09-23T20:05:34.463 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.463 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.464 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:05:34.464 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:29.853433+0000", 2023-09-23T20:05:34.464 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.4450000000000001e-05 2023-09-23T20:05:34.464 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.464 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.464 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:05:34.464 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:29.853447+0000", 2023-09-23T20:05:34.465 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00057206200000000003 2023-09-23T20:05:34.465 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.465 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.465 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:05:34.465 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:29.854019+0000", 2023-09-23T20:05:34.465 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.9261999999999998e-05 2023-09-23T20:05:34.465 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.465 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.465 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:05:34.465 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:29.854058+0000", 2023-09-23T20:05:34.465 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0098479350000000004 2023-09-23T20:05:34.466 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.466 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.466 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:05:34.466 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:29.863906+0000", 2023-09-23T20:05:34.466 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.3916e-05 2023-09-23T20:05:34.466 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.466 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.466 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:05:34.466 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:29.864000+0000", 2023-09-23T20:05:34.466 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.010595711000000001 2023-09-23T20:05:34.466 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:34.466 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:05:34.467 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:34.467 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.467 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.467 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.37161.0:67 2.7 2:e24ef17d:::rbd_data.b80d7e31e638.0000000000000037:head [sparse-read 0~4194304 [fadvise_nocache]] snapc 0=[] ondisk+read+known_if_redirected e285)", 2023-09-23T20:05:34.467 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T20:02:29.853464+0000", 2023-09-23T20:05:34.467 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 184.494295728, 2023-09-23T20:05:34.467 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.010631470000000001, 2023-09-23T20:05:34.467 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:05:34.467 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "reached pg", 2023-09-23T20:05:34.467 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:05:34.467 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.37161", 2023-09-23T20:05:34.467 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/407694934", 2023-09-23T20:05:34.468 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 67 2023-09-23T20:05:34.468 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.468 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:05:34.468 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.468 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:05:34.468 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:29.853464+0000", 2023-09-23T20:05:34.468 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:05:34.468 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.468 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.468 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:05:34.468 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:29.853464+0000", 2023-09-23T20:05:34.468 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T20:05:34.468 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.469 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.469 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:05:34.469 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:29.853463+0000", 2023-09-23T20:05:34.469 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.5630000000000003e-06 2023-09-23T20:05:34.469 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.469 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.469 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:05:34.470 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:29.853470+0000", 2023-09-23T20:05:34.470 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.1759e-05 2023-09-23T20:05:34.470 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.470 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.470 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:05:34.470 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:29.853492+0000", 2023-09-23T20:05:34.470 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.1437000000000001e-05 2023-09-23T20:05:34.470 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.470 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.470 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:05:34.470 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:29.853503+0000", 2023-09-23T20:05:34.471 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00087668700000000002 2023-09-23T20:05:34.471 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.471 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.471 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:05:34.471 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:29.854380+0000", 2023-09-23T20:05:34.471 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.5165e-05 2023-09-23T20:05:34.471 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.471 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.471 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:05:34.471 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:29.854395+0000", 2023-09-23T20:05:34.471 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0096228189999999995 2023-09-23T20:05:34.471 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.471 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.472 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:05:34.472 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:29.864018+0000", 2023-09-23T20:05:34.472 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.7316000000000003e-05 2023-09-23T20:05:34.472 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.472 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.472 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:05:34.472 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:29.864095+0000", 2023-09-23T20:05:34.472 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.010631470000000001 2023-09-23T20:05:34.472 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:34.472 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:05:34.472 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:34.472 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.473 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.473 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.47144.0:22 2.7 2:ed24fc4d:::rbd_header.b8226285b1bf:head [call rbd.snapshot_add in=15b] snapc 0=[] ondisk+write+known_if_redirected e286)", 2023-09-23T20:05:34.473 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T20:02:33.852364+0000", 2023-09-23T20:05:34.473 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 180.495395675, 2023-09-23T20:05:34.473 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0093623019999999994, 2023-09-23T20:05:34.473 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:05:34.473 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:34.473 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:05:34.473 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.47144", 2023-09-23T20:05:34.473 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/1636197388", 2023-09-23T20:05:34.473 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 22 2023-09-23T20:05:34.473 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.473 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:05:34.474 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.474 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:05:34.474 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:33.852364+0000", 2023-09-23T20:05:34.474 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:05:34.474 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.474 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.474 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:05:34.474 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:33.852364+0000", 2023-09-23T20:05:34.474 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999962 2023-09-23T20:05:34.474 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.474 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.474 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:05:34.475 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:33.852360+0000", 2023-09-23T20:05:34.475 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.4655999999999999e-05 2023-09-23T20:05:34.475 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.475 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.475 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:05:34.475 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:33.852385+0000", 2023-09-23T20:05:34.475 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.8231e-05 2023-09-23T20:05:34.475 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.475 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.475 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:05:34.475 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:33.852423+0000", 2023-09-23T20:05:34.475 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.5858999999999998e-05 2023-09-23T20:05:34.475 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.476 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.476 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:05:34.476 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:33.852439+0000", 2023-09-23T20:05:34.476 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00010963000000000001 2023-09-23T20:05:34.476 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.476 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.476 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:05:34.476 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:33.852548+0000", 2023-09-23T20:05:34.476 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.0868e-05 2023-09-23T20:05:34.476 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.476 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.476 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op must wait for map", 2023-09-23T20:05:34.476 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:33.852559+0000", 2023-09-23T20:05:34.477 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0074678629999999999 2023-09-23T20:05:34.477 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.477 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.477 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:05:34.477 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:33.860027+0000", 2023-09-23T20:05:34.477 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.9425000000000005e-05 2023-09-23T20:05:34.477 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.477 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.477 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:05:34.477 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:33.860116+0000", 2023-09-23T20:05:34.477 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00029232699999999999 2023-09-23T20:05:34.477 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.477 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.478 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 2,7", 2023-09-23T20:05:34.478 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:33.860409+0000", 2023-09-23T20:05:34.478 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00073122699999999998 2023-09-23T20:05:34.478 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.478 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.478 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:05:34.478 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:33.861140+0000", 2023-09-23T20:05:34.478 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00038013199999999999 2023-09-23T20:05:34.478 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.478 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.478 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:34.478 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:33.861520+0000", 2023-09-23T20:05:34.479 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000128307 2023-09-23T20:05:34.479 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.479 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.479 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:34.479 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:33.861648+0000", 2023-09-23T20:05:34.479 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.5902999999999997e-05 2023-09-23T20:05:34.479 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.479 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.479 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:05:34.479 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:33.861694+0000", 2023-09-23T20:05:34.479 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.1917e-05 2023-09-23T20:05:34.479 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:05:34.480 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:05:34.480 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:05:34.480 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:33.861726+0000", 2023-09-23T20:05:34.480 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0093623019999999994 2023-09-23T20:05:34.480 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:34.480 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:05:34.480 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:34.480 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:05:34.480 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:05:34.480 INFO:teuthology.orchestra.run.smithi092.stdout:} 2023-09-23T20:05:34.481 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:34 smithi016 bash[77995]: cluster 2023-09-23T20:05:32.749470+0000 mgr.y (mgr.25325) 1751 : cluster 0 pgmap v1839: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:34.481 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:34 smithi092 bash[58855]: cluster 2023-09-23T20:05:32.749470+0000 mgr.y (mgr.25325) 1751 : cluster 0 pgmap v1839: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:34.481 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:34 smithi016 bash[75004]: cluster 2023-09-23T20:05:32.749470+0000 mgr.y (mgr.25325) 1751 : cluster 0 pgmap v1839: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:35.193 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T20:05:35.982 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:05:35 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:20:05:35] "GET /metrics HTTP/1.1" 200 34012 "" "Prometheus/2.33.4" 2023-09-23T20:05:36.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:36 smithi016 bash[75004]: cluster 2023-09-23T20:05:34.750217+0000 mgr.y (mgr.25325) 1752 : cluster 0 pgmap v1840: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:36.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:36 smithi016 bash[77995]: cluster 2023-09-23T20:05:34.750217+0000 mgr.y (mgr.25325) 1752 : cluster 0 pgmap v1840: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:36.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:36 smithi092 bash[58855]: cluster 2023-09-23T20:05:34.750217+0000 mgr.y (mgr.25325) 1752 : cluster 0 pgmap v1840: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:37.246 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T20:05:37.493 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:37 smithi016 bash[77995]: audit 2023-09-23T20:05:36.433946+0000 mon.a (mon.0) 2126 : audit 1 from='client.? 172.21.15.16:0/868345718' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:05:37.494 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:37 smithi016 bash[75004]: audit 2023-09-23T20:05:36.433946+0000 mon.a (mon.0) 2126 : audit 1 from='client.? 172.21.15.16:0/868345718' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:05:37.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:37 smithi092 bash[58855]: audit 2023-09-23T20:05:36.433946+0000 mon.a (mon.0) 2126 : audit 1 from='client.? 172.21.15.16:0/868345718' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:05:38.254 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 6 1 2023-09-23T20:05:38.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:38 smithi016 bash[75004]: cluster 2023-09-23T20:05:36.751099+0000 mgr.y (mgr.25325) 1753 : cluster 0 pgmap v1841: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:38.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:38 smithi016 bash[75004]: audit 2023-09-23T20:05:37.237626+0000 mon.a (mon.0) 2127 : audit 1 from='client.? 172.21.15.16:0/868345718' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T20:05:38.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:38 smithi016 bash[75004]: cluster 2023-09-23T20:05:37.237706+0000 mon.a (mon.0) 2128 : cluster 0 osdmap e328: 8 total, 8 up, 8 in 2023-09-23T20:05:38.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:38 smithi016 bash[77995]: cluster 2023-09-23T20:05:36.751099+0000 mgr.y (mgr.25325) 1753 : cluster 0 pgmap v1841: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:38.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:38 smithi016 bash[77995]: audit 2023-09-23T20:05:37.237626+0000 mon.a (mon.0) 2127 : audit 1 from='client.? 172.21.15.16:0/868345718' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T20:05:38.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:38 smithi016 bash[77995]: cluster 2023-09-23T20:05:37.237706+0000 mon.a (mon.0) 2128 : cluster 0 osdmap e328: 8 total, 8 up, 8 in 2023-09-23T20:05:38.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:38 smithi092 bash[58855]: cluster 2023-09-23T20:05:36.751099+0000 mgr.y (mgr.25325) 1753 : cluster 0 pgmap v1841: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:38.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:38 smithi092 bash[58855]: audit 2023-09-23T20:05:37.237626+0000 mon.a (mon.0) 2127 : audit 1 from='client.? 172.21.15.16:0/868345718' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T20:05:38.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:38 smithi092 bash[58855]: cluster 2023-09-23T20:05:37.237706+0000 mon.a (mon.0) 2128 : cluster 0 osdmap e328: 8 total, 8 up, 8 in 2023-09-23T20:05:39.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:39 smithi016 bash[77995]: cluster 2023-09-23T20:05:38.331263+0000 osd.7 (osd.7) 249 : cluster 3 failed to encode map e328 with expected crc 2023-09-23T20:05:39.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:39 smithi016 bash[77995]: cluster 2023-09-23T20:05:38.735109+0000 osd.5 (osd.5) 248 : cluster 3 failed to encode map e328 with expected crc 2023-09-23T20:05:39.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:39 smithi016 bash[77995]: cluster 2023-09-23T20:05:38.737795+0000 osd.5 (osd.5) 249 : cluster 3 failed to encode map e328 with expected crc 2023-09-23T20:05:39.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:39 smithi016 bash[77995]: cluster 2023-09-23T20:05:38.868826+0000 osd.6 (osd.6) 264 : cluster 3 failed to encode map e328 with expected crc 2023-09-23T20:05:39.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:39 smithi016 bash[77995]: cluster 2023-09-23T20:05:38.871612+0000 osd.6 (osd.6) 265 : cluster 3 failed to encode map e328 with expected crc 2023-09-23T20:05:39.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:39 smithi016 bash[77995]: cluster 2023-09-23T20:05:38.873987+0000 osd.6 (osd.6) 266 : cluster 3 failed to encode map e328 with expected crc 2023-09-23T20:05:39.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:39 smithi016 bash[77995]: cluster 2023-09-23T20:05:38.875483+0000 osd.6 (osd.6) 267 : cluster 3 failed to encode map e328 with expected crc 2023-09-23T20:05:39.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:39 smithi016 bash[75004]: cluster 2023-09-23T20:05:38.331263+0000 osd.7 (osd.7) 249 : cluster 3 failed to encode map e328 with expected crc 2023-09-23T20:05:39.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:39 smithi016 bash[75004]: cluster 2023-09-23T20:05:38.735109+0000 osd.5 (osd.5) 248 : cluster 3 failed to encode map e328 with expected crc 2023-09-23T20:05:39.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:39 smithi016 bash[75004]: cluster 2023-09-23T20:05:38.737795+0000 osd.5 (osd.5) 249 : cluster 3 failed to encode map e328 with expected crc 2023-09-23T20:05:39.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:39 smithi016 bash[75004]: cluster 2023-09-23T20:05:38.868826+0000 osd.6 (osd.6) 264 : cluster 3 failed to encode map e328 with expected crc 2023-09-23T20:05:39.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:39 smithi016 bash[75004]: cluster 2023-09-23T20:05:38.871612+0000 osd.6 (osd.6) 265 : cluster 3 failed to encode map e328 with expected crc 2023-09-23T20:05:39.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:39 smithi016 bash[75004]: cluster 2023-09-23T20:05:38.873987+0000 osd.6 (osd.6) 266 : cluster 3 failed to encode map e328 with expected crc 2023-09-23T20:05:39.537 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:39 smithi016 bash[75004]: cluster 2023-09-23T20:05:38.875483+0000 osd.6 (osd.6) 267 : cluster 3 failed to encode map e328 with expected crc 2023-09-23T20:05:39.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:39 smithi092 bash[58855]: cluster 2023-09-23T20:05:38.331263+0000 osd.7 (osd.7) 249 : cluster 3 failed to encode map e328 with expected crc 2023-09-23T20:05:39.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:39 smithi092 bash[58855]: cluster 2023-09-23T20:05:38.735109+0000 osd.5 (osd.5) 248 : cluster 3 failed to encode map e328 with expected crc 2023-09-23T20:05:39.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:39 smithi092 bash[58855]: cluster 2023-09-23T20:05:38.737795+0000 osd.5 (osd.5) 249 : cluster 3 failed to encode map e328 with expected crc 2023-09-23T20:05:39.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:39 smithi092 bash[58855]: cluster 2023-09-23T20:05:38.868826+0000 osd.6 (osd.6) 264 : cluster 3 failed to encode map e328 with expected crc 2023-09-23T20:05:39.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:39 smithi092 bash[58855]: cluster 2023-09-23T20:05:38.871612+0000 osd.6 (osd.6) 265 : cluster 3 failed to encode map e328 with expected crc 2023-09-23T20:05:39.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:39 smithi092 bash[58855]: cluster 2023-09-23T20:05:38.873987+0000 osd.6 (osd.6) 266 : cluster 3 failed to encode map e328 with expected crc 2023-09-23T20:05:39.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:39 smithi092 bash[58855]: cluster 2023-09-23T20:05:38.875483+0000 osd.6 (osd.6) 267 : cluster 3 failed to encode map e328 with expected crc 2023-09-23T20:05:39.974 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:05:40.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:40 smithi092 bash[58855]: cluster 2023-09-23T20:05:38.751709+0000 mgr.y (mgr.25325) 1754 : cluster 0 pgmap v1843: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:40.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:40 smithi016 bash[75004]: cluster 2023-09-23T20:05:38.751709+0000 mgr.y (mgr.25325) 1754 : cluster 0 pgmap v1843: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:40.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:40 smithi016 bash[77995]: cluster 2023-09-23T20:05:38.751709+0000 mgr.y (mgr.25325) 1754 : cluster 0 pgmap v1843: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:42.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:42 smithi016 bash[75004]: cluster 2023-09-23T20:05:40.752576+0000 mgr.y (mgr.25325) 1755 : cluster 0 pgmap v1844: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:42.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:42 smithi016 bash[77995]: cluster 2023-09-23T20:05:40.752576+0000 mgr.y (mgr.25325) 1755 : cluster 0 pgmap v1844: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:42.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:42 smithi092 bash[58855]: cluster 2023-09-23T20:05:40.752576+0000 mgr.y (mgr.25325) 1755 : cluster 0 pgmap v1844: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:43.038 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:05:43.312 INFO:teuthology.orchestra.run.smithi092.stdout:{ 2023-09-23T20:05:43.312 INFO:teuthology.orchestra.run.smithi092.stdout: "ops": [], 2023-09-23T20:05:43.312 INFO:teuthology.orchestra.run.smithi092.stdout: "num_ops": 0 2023-09-23T20:05:43.312 INFO:teuthology.orchestra.run.smithi092.stdout:} 2023-09-23T20:05:44.090 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T20:05:44.526 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:44 smithi016 bash[77995]: cluster 2023-09-23T20:05:42.753481+0000 mgr.y (mgr.25325) 1756 : cluster 0 pgmap v1845: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:44.526 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:44 smithi016 bash[77995]: audit 2023-09-23T20:05:43.705532+0000 mon.a (mon.0) 2129 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:05:44.526 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:44 smithi016 bash[75004]: cluster 2023-09-23T20:05:42.753481+0000 mgr.y (mgr.25325) 1756 : cluster 0 pgmap v1845: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:44.526 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:44 smithi016 bash[75004]: audit 2023-09-23T20:05:43.705532+0000 mon.a (mon.0) 2129 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:05:44.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:44 smithi092 bash[58855]: cluster 2023-09-23T20:05:42.753481+0000 mgr.y (mgr.25325) 1756 : cluster 0 pgmap v1845: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:44.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:44 smithi092 bash[58855]: audit 2023-09-23T20:05:43.705532+0000 mon.a (mon.0) 2129 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:05:46.035 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:05:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:20:05:45] "GET /metrics HTTP/1.1" 200 34012 "" "Prometheus/2.33.4" 2023-09-23T20:05:46.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:46 smithi016 bash[77995]: cluster 2023-09-23T20:05:44.754121+0000 mgr.y (mgr.25325) 1757 : cluster 0 pgmap v1846: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:46.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:46 smithi016 bash[75004]: cluster 2023-09-23T20:05:44.754121+0000 mgr.y (mgr.25325) 1757 : cluster 0 pgmap v1846: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:46.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:46 smithi092 bash[58855]: cluster 2023-09-23T20:05:44.754121+0000 mgr.y (mgr.25325) 1757 : cluster 0 pgmap v1846: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:47.278 INFO:teuthology.orchestra.run.smithi016.stderr:set osd.6 primary-affinity to 1 (8655362) 2023-09-23T20:05:47.513 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:47 smithi016 bash[75004]: audit 2023-09-23T20:05:46.577581+0000 mon.a (mon.0) 2130 : audit 1 from='client.? 172.21.15.16:0/55322206' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:05:47.513 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:47 smithi016 bash[77995]: audit 2023-09-23T20:05:46.577581+0000 mon.a (mon.0) 2130 : audit 1 from='client.? 172.21.15.16:0/55322206' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:05:47.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:47 smithi092 bash[58855]: audit 2023-09-23T20:05:46.577581+0000 mon.a (mon.0) 2130 : audit 1 from='client.? 172.21.15.16:0/55322206' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:05:48.207 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-23T20:05:48.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:48 smithi016 bash[77995]: cluster 2023-09-23T20:05:46.754968+0000 mgr.y (mgr.25325) 1758 : cluster 0 pgmap v1847: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:48.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:48 smithi016 bash[77995]: audit 2023-09-23T20:05:47.270709+0000 mon.a (mon.0) 2131 : audit 1 from='client.? 172.21.15.16:0/55322206' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-09-23T20:05:48.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:48 smithi016 bash[77995]: cluster 2023-09-23T20:05:47.270795+0000 mon.a (mon.0) 2132 : cluster 0 osdmap e329: 8 total, 8 up, 8 in 2023-09-23T20:05:48.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:48 smithi016 bash[77995]: cluster 2023-09-23T20:05:47.272455+0000 osd.5 (osd.5) 250 : cluster 3 failed to encode map e329 with expected crc 2023-09-23T20:05:48.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:48 smithi016 bash[75004]: cluster 2023-09-23T20:05:46.754968+0000 mgr.y (mgr.25325) 1758 : cluster 0 pgmap v1847: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:48.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:48 smithi016 bash[75004]: audit 2023-09-23T20:05:47.270709+0000 mon.a (mon.0) 2131 : audit 1 from='client.? 172.21.15.16:0/55322206' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-09-23T20:05:48.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:48 smithi016 bash[75004]: cluster 2023-09-23T20:05:47.270795+0000 mon.a (mon.0) 2132 : cluster 0 osdmap e329: 8 total, 8 up, 8 in 2023-09-23T20:05:48.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:48 smithi016 bash[75004]: cluster 2023-09-23T20:05:47.272455+0000 osd.5 (osd.5) 250 : cluster 3 failed to encode map e329 with expected crc 2023-09-23T20:05:48.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:48 smithi092 bash[58855]: cluster 2023-09-23T20:05:46.754968+0000 mgr.y (mgr.25325) 1758 : cluster 0 pgmap v1847: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:48.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:48 smithi092 bash[58855]: audit 2023-09-23T20:05:47.270709+0000 mon.a (mon.0) 2131 : audit 1 from='client.? 172.21.15.16:0/55322206' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-09-23T20:05:48.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:48 smithi092 bash[58855]: cluster 2023-09-23T20:05:47.270795+0000 mon.a (mon.0) 2132 : cluster 0 osdmap e329: 8 total, 8 up, 8 in 2023-09-23T20:05:48.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:48 smithi092 bash[58855]: cluster 2023-09-23T20:05:47.272455+0000 osd.5 (osd.5) 250 : cluster 3 failed to encode map e329 with expected crc 2023-09-23T20:05:48.863 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:05:50.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:50 smithi016 bash[77995]: cluster 2023-09-23T20:05:48.755635+0000 mgr.y (mgr.25325) 1759 : cluster 0 pgmap v1849: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:50.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:50 smithi016 bash[75004]: cluster 2023-09-23T20:05:48.755635+0000 mgr.y (mgr.25325) 1759 : cluster 0 pgmap v1849: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:50.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:50 smithi092 bash[58855]: cluster 2023-09-23T20:05:48.755635+0000 mgr.y (mgr.25325) 1759 : cluster 0 pgmap v1849: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:52.049 INFO:teuthology.orchestra.run.smithi092.stdout:{ 2023-09-23T20:05:52.050 INFO:teuthology.orchestra.run.smithi092.stdout: "ops": [], 2023-09-23T20:05:52.050 INFO:teuthology.orchestra.run.smithi092.stdout: "complaint_time": 30, 2023-09-23T20:05:52.050 INFO:teuthology.orchestra.run.smithi092.stdout: "num_blocked_ops": 0 2023-09-23T20:05:52.050 INFO:teuthology.orchestra.run.smithi092.stdout:} 2023-09-23T20:05:52.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:52 smithi016 bash[75004]: cluster 2023-09-23T20:05:50.756549+0000 mgr.y (mgr.25325) 1760 : cluster 0 pgmap v1850: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:52.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:52 smithi016 bash[77995]: cluster 2023-09-23T20:05:50.756549+0000 mgr.y (mgr.25325) 1760 : cluster 0 pgmap v1850: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:52.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:52 smithi092 bash[58855]: cluster 2023-09-23T20:05:50.756549+0000 mgr.y (mgr.25325) 1760 : cluster 0 pgmap v1850: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:52.905 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T20:05:52.996 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:05:54.406 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:54 smithi016 bash[75004]: cluster 2023-09-23T20:05:52.757432+0000 mgr.y (mgr.25325) 1761 : cluster 0 pgmap v1851: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:54.406 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:54 smithi016 bash[77995]: cluster 2023-09-23T20:05:52.757432+0000 mgr.y (mgr.25325) 1761 : cluster 0 pgmap v1851: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:54.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:54 smithi092 bash[58855]: cluster 2023-09-23T20:05:52.757432+0000 mgr.y (mgr.25325) 1761 : cluster 0 pgmap v1851: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:56.035 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:05:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:20:05:55] "GET /metrics HTTP/1.1" 200 34012 "" "Prometheus/2.33.4" 2023-09-23T20:05:56.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:56 smithi016 bash[75004]: cluster 2023-09-23T20:05:54.758211+0000 mgr.y (mgr.25325) 1762 : cluster 0 pgmap v1852: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:56.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:56 smithi016 bash[77995]: cluster 2023-09-23T20:05:54.758211+0000 mgr.y (mgr.25325) 1762 : cluster 0 pgmap v1852: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:56.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:56 smithi092 bash[58855]: cluster 2023-09-23T20:05:54.758211+0000 mgr.y (mgr.25325) 1762 : cluster 0 pgmap v1852: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:57.307 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.7 to 1 (10000) 2023-09-23T20:05:57.676 INFO:teuthology.orchestra.run.smithi092.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:05:57.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:57 smithi016 bash[75004]: audit 2023-09-23T20:05:56.493288+0000 mon.c (mon.2) 260 : audit 1 from='client.? 172.21.15.16:0/4111531436' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:05:57.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:57 smithi016 bash[75004]: audit 2023-09-23T20:05:56.493987+0000 mon.a (mon.0) 2133 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:05:57.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:57 smithi016 bash[77995]: audit 2023-09-23T20:05:56.493288+0000 mon.c (mon.2) 260 : audit 1 from='client.? 172.21.15.16:0/4111531436' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:05:57.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:57 smithi016 bash[77995]: audit 2023-09-23T20:05:56.493987+0000 mon.a (mon.0) 2133 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:05:57.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:57 smithi092 bash[58855]: audit 2023-09-23T20:05:56.493288+0000 mon.c (mon.2) 260 : audit 1 from='client.? 172.21.15.16:0/4111531436' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:05:57.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:57 smithi092 bash[58855]: audit 2023-09-23T20:05:56.493987+0000 mon.a (mon.0) 2133 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:05:58.321 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 7 1 2023-09-23T20:05:58.635 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:58 smithi092 bash[58855]: cluster 2023-09-23T20:05:56.759078+0000 mgr.y (mgr.25325) 1763 : cluster 0 pgmap v1853: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:58.635 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:58 smithi092 bash[58855]: audit 2023-09-23T20:05:57.298998+0000 mon.a (mon.0) 2134 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:05:58.635 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:58 smithi092 bash[58855]: cluster 2023-09-23T20:05:57.299129+0000 mon.a (mon.0) 2135 : cluster 0 osdmap e330: 8 total, 8 up, 8 in 2023-09-23T20:05:58.635 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:58 smithi092 bash[58855]: cluster 2023-09-23T20:05:57.856608+0000 osd.5 (osd.5) 251 : cluster 3 failed to encode map e330 with expected crc 2023-09-23T20:05:58.635 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:58 smithi092 bash[58855]: cluster 2023-09-23T20:05:57.856943+0000 osd.6 (osd.6) 268 : cluster 3 failed to encode map e330 with expected crc 2023-09-23T20:05:58.635 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:58 smithi092 bash[58855]: cluster 2023-09-23T20:05:57.857211+0000 osd.7 (osd.7) 250 : cluster 3 failed to encode map e330 with expected crc 2023-09-23T20:05:58.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:58 smithi016 bash[75004]: cluster 2023-09-23T20:05:56.759078+0000 mgr.y (mgr.25325) 1763 : cluster 0 pgmap v1853: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:58.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:58 smithi016 bash[75004]: audit 2023-09-23T20:05:57.298998+0000 mon.a (mon.0) 2134 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:05:58.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:58 smithi016 bash[75004]: cluster 2023-09-23T20:05:57.299129+0000 mon.a (mon.0) 2135 : cluster 0 osdmap e330: 8 total, 8 up, 8 in 2023-09-23T20:05:58.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:58 smithi016 bash[75004]: cluster 2023-09-23T20:05:57.856608+0000 osd.5 (osd.5) 251 : cluster 3 failed to encode map e330 with expected crc 2023-09-23T20:05:58.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:58 smithi016 bash[75004]: cluster 2023-09-23T20:05:57.856943+0000 osd.6 (osd.6) 268 : cluster 3 failed to encode map e330 with expected crc 2023-09-23T20:05:58.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:58 smithi016 bash[75004]: cluster 2023-09-23T20:05:57.857211+0000 osd.7 (osd.7) 250 : cluster 3 failed to encode map e330 with expected crc 2023-09-23T20:05:58.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:58 smithi016 bash[77995]: cluster 2023-09-23T20:05:56.759078+0000 mgr.y (mgr.25325) 1763 : cluster 0 pgmap v1853: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:05:58.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:58 smithi016 bash[77995]: audit 2023-09-23T20:05:57.298998+0000 mon.a (mon.0) 2134 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:05:58.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:58 smithi016 bash[77995]: cluster 2023-09-23T20:05:57.299129+0000 mon.a (mon.0) 2135 : cluster 0 osdmap e330: 8 total, 8 up, 8 in 2023-09-23T20:05:58.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:58 smithi016 bash[77995]: cluster 2023-09-23T20:05:57.856608+0000 osd.5 (osd.5) 251 : cluster 3 failed to encode map e330 with expected crc 2023-09-23T20:05:58.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:58 smithi016 bash[77995]: cluster 2023-09-23T20:05:57.856943+0000 osd.6 (osd.6) 268 : cluster 3 failed to encode map e330 with expected crc 2023-09-23T20:05:58.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:58 smithi016 bash[77995]: cluster 2023-09-23T20:05:57.857211+0000 osd.7 (osd.7) 250 : cluster 3 failed to encode map e330 with expected crc 2023-09-23T20:05:59.596 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:59 smithi092 bash[58855]: cluster 2023-09-23T20:05:57.856783+0000 osd.4 (osd.4) 272 : cluster 3 failed to encode map e330 with expected crc 2023-09-23T20:05:59.596 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:05:59 smithi092 bash[58855]: audit 2023-09-23T20:05:58.706219+0000 mon.a (mon.0) 2136 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:05:59.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:59 smithi016 bash[77995]: cluster 2023-09-23T20:05:57.856783+0000 osd.4 (osd.4) 272 : cluster 3 failed to encode map e330 with expected crc 2023-09-23T20:05:59.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:05:59 smithi016 bash[77995]: audit 2023-09-23T20:05:58.706219+0000 mon.a (mon.0) 2136 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:05:59.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:59 smithi016 bash[75004]: cluster 2023-09-23T20:05:57.856783+0000 osd.4 (osd.4) 272 : cluster 3 failed to encode map e330 with expected crc 2023-09-23T20:05:59.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:05:59 smithi016 bash[75004]: audit 2023-09-23T20:05:58.706219+0000 mon.a (mon.0) 2136 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:06:00.684 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:00 smithi092 bash[58855]: cluster 2023-09-23T20:05:58.759820+0000 mgr.y (mgr.25325) 1764 : cluster 0 pgmap v1855: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:00.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:00 smithi016 bash[75004]: cluster 2023-09-23T20:05:58.759820+0000 mgr.y (mgr.25325) 1764 : cluster 0 pgmap v1855: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:00.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:00 smithi016 bash[77995]: cluster 2023-09-23T20:05:58.759820+0000 mgr.y (mgr.25325) 1764 : cluster 0 pgmap v1855: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:00.922 INFO:teuthology.orchestra.run.smithi092.stdout:{ 2023-09-23T20:06:00.922 INFO:teuthology.orchestra.run.smithi092.stdout: "size": 20, 2023-09-23T20:06:00.922 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 600, 2023-09-23T20:06:00.922 INFO:teuthology.orchestra.run.smithi092.stdout: "ops": [ 2023-09-23T20:06:00.922 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.922 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:36132 14.f 14:f289b9fe:::benchmark_data_smithi016_228083_object36131:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-09-23T20:06:00.922 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:57:34.789453+0000", 2023-09-23T20:06:00.922 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 506.130298681, 2023-09-23T20:06:00.922 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.14023466200000001, 2023-09-23T20:06:00.922 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:06:00.922 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:00.923 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:06:00.923 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T20:06:00.923 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:06:00.923 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 36132 2023-09-23T20:06:00.923 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.923 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:06:00.923 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.923 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:06:00.923 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.789453+0000", 2023-09-23T20:06:00.923 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:06:00.923 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.923 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.924 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:06:00.924 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.789453+0000", 2023-09-23T20:06:00.924 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T20:06:00.924 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.924 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.924 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:06:00.924 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.789452+0000", 2023-09-23T20:06:00.924 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.7549999999999997e-05 2023-09-23T20:06:00.924 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.924 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.924 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:06:00.924 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.789499+0000", 2023-09-23T20:06:00.925 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.7098000000000001e-05 2023-09-23T20:06:00.925 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.925 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.925 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:06:00.925 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.789516+0000", 2023-09-23T20:06:00.925 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.6249999999999996e-06 2023-09-23T20:06:00.925 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.925 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.925 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:06:00.925 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.789525+0000", 2023-09-23T20:06:00.925 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.9809999999999998e-05 2023-09-23T20:06:00.925 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.926 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.926 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:06:00.926 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.789585+0000", 2023-09-23T20:06:00.926 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00018619700000000001 2023-09-23T20:06:00.926 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.926 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.926 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:06:00.926 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.789771+0000", 2023-09-23T20:06:00.926 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00022020500000000001 2023-09-23T20:06:00.926 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.926 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.926 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:06:00.927 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.789991+0000", 2023-09-23T20:06:00.927 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.002540161 2023-09-23T20:06:00.927 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.927 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.927 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:06:00.927 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.792531+0000", 2023-09-23T20:06:00.927 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0016885349999999999 2023-09-23T20:06:00.927 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.927 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.927 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:00.927 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.794220+0000", 2023-09-23T20:06:00.927 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.13522234299999999 2023-09-23T20:06:00.928 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.928 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.928 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:00.928 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.929442+0000", 2023-09-23T20:06:00.928 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00013912899999999999 2023-09-23T20:06:00.928 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.928 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.928 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:06:00.928 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.929581+0000", 2023-09-23T20:06:00.928 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000106533 2023-09-23T20:06:00.928 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.928 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.928 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:06:00.929 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.929688+0000", 2023-09-23T20:06:00.929 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.14023466200000001 2023-09-23T20:06:00.929 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:06:00.929 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:06:00.929 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:06:00.929 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.929 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.929 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:36148 14.f 14:f2537d57:::benchmark_data_smithi016_228083_object36147:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-09-23T20:06:00.929 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:57:34.799429+0000", 2023-09-23T20:06:00.929 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 506.12032281, 2023-09-23T20:06:00.929 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.13068242399999999, 2023-09-23T20:06:00.929 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:06:00.930 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:00.930 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:06:00.930 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T20:06:00.930 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:06:00.930 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 36148 2023-09-23T20:06:00.930 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.930 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:06:00.930 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.930 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:06:00.930 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.799429+0000", 2023-09-23T20:06:00.930 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:06:00.930 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.931 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.931 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:06:00.931 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.799429+0000", 2023-09-23T20:06:00.931 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999976 2023-09-23T20:06:00.931 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.931 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.931 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:06:00.931 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.799426+0000", 2023-09-23T20:06:00.931 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.7633999999999998e-05 2023-09-23T20:06:00.931 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.931 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.931 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:06:00.932 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.799474+0000", 2023-09-23T20:06:00.932 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.5546e-05 2023-09-23T20:06:00.932 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.932 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.932 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:06:00.932 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.799490+0000", 2023-09-23T20:06:00.932 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.5249999999999994e-06 2023-09-23T20:06:00.932 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.932 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.932 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:06:00.932 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.799499+0000", 2023-09-23T20:06:00.932 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.0447999999999999e-05 2023-09-23T20:06:00.933 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.933 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.933 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:06:00.933 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.799590+0000", 2023-09-23T20:06:00.933 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00014605400000000001 2023-09-23T20:06:00.933 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.933 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.933 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:06:00.933 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.799736+0000", 2023-09-23T20:06:00.933 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000147152 2023-09-23T20:06:00.933 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.933 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.934 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:06:00.934 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.799883+0000", 2023-09-23T20:06:00.934 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0028179730000000001 2023-09-23T20:06:00.934 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.934 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.934 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:06:00.934 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.802701+0000", 2023-09-23T20:06:00.934 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.0997e-05 2023-09-23T20:06:00.934 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.934 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.934 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:00.934 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.802752+0000", 2023-09-23T20:06:00.935 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.12707763899999999 2023-09-23T20:06:00.935 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.935 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.935 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:00.935 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.929829+0000", 2023-09-23T20:06:00.935 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00016053000000000001 2023-09-23T20:06:00.935 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.935 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.935 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:06:00.935 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.929990+0000", 2023-09-23T20:06:00.935 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000121385 2023-09-23T20:06:00.935 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.936 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.936 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:06:00.936 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:34.930111+0000", 2023-09-23T20:06:00.936 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.13068242399999999 2023-09-23T20:06:00.936 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:06:00.936 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:06:00.936 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:06:00.936 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.936 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.936 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:38540 14.f 14:f39591d4:::benchmark_data_smithi016_228083_object38539:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-09-23T20:06:00.936 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:57:36.906965+0000", 2023-09-23T20:06:00.936 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 504.01278691700003, 2023-09-23T20:06:00.937 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.214407346, 2023-09-23T20:06:00.937 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:06:00.937 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:00.937 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:06:00.937 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T20:06:00.937 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:06:00.937 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 38540 2023-09-23T20:06:00.937 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.937 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:06:00.937 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.937 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:06:00.938 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:36.906965+0000", 2023-09-23T20:06:00.938 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:06:00.938 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.938 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.938 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:06:00.938 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:36.906965+0000", 2023-09-23T20:06:00.938 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T20:06:00.938 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.938 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.938 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:06:00.938 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:36.906963+0000", 2023-09-23T20:06:00.938 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.414e-05 2023-09-23T20:06:00.938 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.939 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.939 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:06:00.939 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:36.907017+0000", 2023-09-23T20:06:00.939 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.2391e-05 2023-09-23T20:06:00.939 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.939 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.939 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:06:00.939 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:36.907030+0000", 2023-09-23T20:06:00.939 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.1810000000000004e-06 2023-09-23T20:06:00.939 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.939 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.939 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:06:00.940 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:36.907037+0000", 2023-09-23T20:06:00.940 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.4173999999999999e-05 2023-09-23T20:06:00.940 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.940 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.940 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:06:00.940 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:36.907101+0000", 2023-09-23T20:06:00.940 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00019921699999999999 2023-09-23T20:06:00.940 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.940 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.940 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:06:00.940 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:36.907300+0000", 2023-09-23T20:06:00.940 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00021943599999999999 2023-09-23T20:06:00.940 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.941 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.941 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:06:00.941 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:36.907520+0000", 2023-09-23T20:06:00.941 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0019506020000000001 2023-09-23T20:06:00.941 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.941 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.941 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:06:00.941 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:36.909470+0000", 2023-09-23T20:06:00.941 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0033401030000000001 2023-09-23T20:06:00.941 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.941 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.941 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:00.941 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:36.912810+0000", 2023-09-23T20:06:00.942 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.20838873299999999 2023-09-23T20:06:00.942 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.942 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.942 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:00.942 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:37.121199+0000", 2023-09-23T20:06:00.942 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.4506999999999995e-05 2023-09-23T20:06:00.942 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.942 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.942 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:06:00.942 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:37.121293+0000", 2023-09-23T20:06:00.942 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.8683999999999994e-05 2023-09-23T20:06:00.942 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.942 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.943 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:06:00.943 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:37.121372+0000", 2023-09-23T20:06:00.943 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.214407346 2023-09-23T20:06:00.943 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:06:00.943 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:06:00.943 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:06:00.943 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.943 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.943 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:49383 14.f 14:ffca6e31:::benchmark_data_smithi016_228083_object49382:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-09-23T20:06:00.943 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:57:46.821698+0000", 2023-09-23T20:06:00.943 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 494.098053945, 2023-09-23T20:06:00.943 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.18302286500000001, 2023-09-23T20:06:00.944 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:06:00.944 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:00.944 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:06:00.944 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T20:06:00.944 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:06:00.944 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 49383 2023-09-23T20:06:00.944 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.944 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:06:00.944 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.944 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:06:00.944 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:46.821698+0000", 2023-09-23T20:06:00.944 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:06:00.944 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.944 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.945 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:06:00.945 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:46.821698+0000", 2023-09-23T20:06:00.945 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T20:06:00.945 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.945 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.945 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:06:00.945 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:46.821696+0000", 2023-09-23T20:06:00.945 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.0072000000000002e-05 2023-09-23T20:06:00.945 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.945 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.945 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:06:00.945 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:46.821776+0000", 2023-09-23T20:06:00.946 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.2704000000000001e-05 2023-09-23T20:06:00.946 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.946 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.946 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:06:00.946 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:46.821789+0000", 2023-09-23T20:06:00.946 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.447e-06 2023-09-23T20:06:00.946 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.946 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.946 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:06:00.946 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:46.821796+0000", 2023-09-23T20:06:00.946 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00019770899999999999 2023-09-23T20:06:00.946 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.947 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.947 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:06:00.947 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:46.821994+0000", 2023-09-23T20:06:00.947 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000177226 2023-09-23T20:06:00.947 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.947 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.947 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:06:00.947 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:46.822171+0000", 2023-09-23T20:06:00.947 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000214794 2023-09-23T20:06:00.947 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.947 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.947 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:06:00.947 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:46.822386+0000", 2023-09-23T20:06:00.948 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.002034306 2023-09-23T20:06:00.948 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.948 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.948 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:06:00.948 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:46.824420+0000", 2023-09-23T20:06:00.948 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0024117710000000001 2023-09-23T20:06:00.948 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.948 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.948 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:00.948 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:46.826832+0000", 2023-09-23T20:06:00.948 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.17778169499999999 2023-09-23T20:06:00.948 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.948 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.949 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:00.949 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:47.004614+0000", 2023-09-23T20:06:00.949 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.6984000000000002e-05 2023-09-23T20:06:00.949 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.949 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.949 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:06:00.949 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:47.004671+0000", 2023-09-23T20:06:00.949 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.9778999999999999e-05 2023-09-23T20:06:00.949 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.949 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.949 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:06:00.949 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:57:47.004721+0000", 2023-09-23T20:06:00.950 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.18302286500000001 2023-09-23T20:06:00.950 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:06:00.950 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:06:00.950 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:06:00.950 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.950 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.950 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:92203 14.f 14:f2e46b18:::benchmark_data_smithi016_228083_object92202:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e264)", 2023-09-23T20:06:00.950 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T19:58:34.347307+0000", 2023-09-23T20:06:00.950 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 446.57244497800002, 2023-09-23T20:06:00.950 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.13251001000000001, 2023-09-23T20:06:00.950 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:06:00.950 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:00.951 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:06:00.951 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T20:06:00.951 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:06:00.951 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 92203 2023-09-23T20:06:00.951 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.951 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:06:00.951 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.951 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:06:00.951 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.347307+0000", 2023-09-23T20:06:00.951 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:06:00.951 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.951 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.951 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:06:00.952 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.347307+0000", 2023-09-23T20:06:00.952 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999981 2023-09-23T20:06:00.952 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.952 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.952 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:06:00.952 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.347305+0000", 2023-09-23T20:06:00.952 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.299e-05 2023-09-23T20:06:00.952 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.952 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.952 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:06:00.952 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.347348+0000", 2023-09-23T20:06:00.952 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.4859999999999999e-05 2023-09-23T20:06:00.952 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.953 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.953 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:06:00.953 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.347373+0000", 2023-09-23T20:06:00.953 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.0611999999999999e-05 2023-09-23T20:06:00.953 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.953 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.953 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:06:00.953 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.347383+0000", 2023-09-23T20:06:00.953 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.3498999999999996e-05 2023-09-23T20:06:00.953 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.953 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.953 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:06:00.953 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.347447+0000", 2023-09-23T20:06:00.954 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00015322799999999999 2023-09-23T20:06:00.954 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.954 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.954 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:06:00.954 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.347600+0000", 2023-09-23T20:06:00.954 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00018437999999999999 2023-09-23T20:06:00.954 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.954 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.954 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:06:00.954 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.347784+0000", 2023-09-23T20:06:00.954 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0038522909999999999 2023-09-23T20:06:00.954 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.955 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.955 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:06:00.955 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.351637+0000", 2023-09-23T20:06:00.955 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000147171 2023-09-23T20:06:00.955 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.955 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.955 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:00.955 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.351784+0000", 2023-09-23T20:06:00.955 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.12785521599999999 2023-09-23T20:06:00.955 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.955 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.955 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:00.956 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.479639+0000", 2023-09-23T20:06:00.956 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000101411 2023-09-23T20:06:00.956 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.956 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.956 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:06:00.956 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.479740+0000", 2023-09-23T20:06:00.956 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.6371000000000005e-05 2023-09-23T20:06:00.956 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.956 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.956 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:06:00.956 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T19:58:34.479817+0000", 2023-09-23T20:06:00.956 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.13251001000000001 2023-09-23T20:06:00.957 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:06:00.957 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:06:00.957 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:06:00.957 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.957 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.957 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:156208 14.f 14:fda8ef3f:::benchmark_data_smithi016_228083_object56249:head [delete] snapc 0=[] ondisk+write+known_if_redirected e269)", 2023-09-23T20:06:00.957 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T20:00:06.689697+0000", 2023-09-23T20:06:00.957 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 354.23005436900002, 2023-09-23T20:06:00.957 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.18813912599999999, 2023-09-23T20:06:00.957 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:06:00.957 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:00.957 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:06:00.957 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T20:06:00.958 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:06:00.958 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 156208 2023-09-23T20:06:00.958 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.958 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:06:00.958 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.958 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:06:00.958 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.689697+0000", 2023-09-23T20:06:00.958 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:06:00.958 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.958 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.958 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:06:00.958 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.689697+0000", 2023-09-23T20:06:00.958 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T20:06:00.959 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.959 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.959 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:06:00.959 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.689696+0000", 2023-09-23T20:06:00.959 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.6139999999999992e-06 2023-09-23T20:06:00.959 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.959 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.959 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:06:00.959 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.689704+0000", 2023-09-23T20:06:00.959 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.3042e-05 2023-09-23T20:06:00.959 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.959 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.959 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:06:00.960 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.689717+0000", 2023-09-23T20:06:00.960 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.6159999999999996e-06 2023-09-23T20:06:00.960 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.960 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.960 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:06:00.960 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.689725+0000", 2023-09-23T20:06:00.960 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.8993999999999997e-05 2023-09-23T20:06:00.960 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.960 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.960 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:06:00.960 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.689794+0000", 2023-09-23T20:06:00.960 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000163148 2023-09-23T20:06:00.961 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.961 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.961 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:06:00.961 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.689957+0000", 2023-09-23T20:06:00.961 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00017913599999999999 2023-09-23T20:06:00.961 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.961 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.961 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:06:00.961 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.690136+0000", 2023-09-23T20:06:00.961 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.17782856899999999 2023-09-23T20:06:00.961 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.961 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.962 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:00.962 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.867965+0000", 2023-09-23T20:06:00.962 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.009755962 2023-09-23T20:06:00.962 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.962 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.962 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:06:00.962 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.877721+0000", 2023-09-23T20:06:00.962 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.9586000000000003e-05 2023-09-23T20:06:00.962 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.962 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.962 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:00.962 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.877770+0000", 2023-09-23T20:06:00.962 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.5295e-05 2023-09-23T20:06:00.963 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.963 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.963 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:06:00.963 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.877806+0000", 2023-09-23T20:06:00.963 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.0716999999999997e-05 2023-09-23T20:06:00.963 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.963 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.963 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:06:00.963 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.877836+0000", 2023-09-23T20:06:00.963 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.18813912599999999 2023-09-23T20:06:00.963 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:06:00.963 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:06:00.964 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:06:00.964 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.964 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.964 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_repop(client.46877.0:156207 14.4 e269/256 14:2211dd6c:::benchmark_data_smithi016_228083_object56248:head v 269'9755, mlcod=269'9753)", 2023-09-23T20:06:00.964 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T20:00:06.690168+0000", 2023-09-23T20:06:00.964 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 354.229583273, 2023-09-23T20:06:00.964 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.19565996099999999, 2023-09-23T20:06:00.964 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:06:00.964 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:00.964 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:06:00.964 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.964 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:06:00.965 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.690168+0000", 2023-09-23T20:06:00.965 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:06:00.965 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.965 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.965 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:06:00.965 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.690168+0000", 2023-09-23T20:06:00.965 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.999999 2023-09-23T20:06:00.965 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.965 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.965 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:06:00.965 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.690167+0000", 2023-09-23T20:06:00.965 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.2271e-05 2023-09-23T20:06:00.966 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.966 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.966 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:06:00.966 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.690180+0000", 2023-09-23T20:06:00.966 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.4568e-05 2023-09-23T20:06:00.966 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.966 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.966 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:06:00.966 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.690194+0000", 2023-09-23T20:06:00.966 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.5480000000000002e-06 2023-09-23T20:06:00.966 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.966 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.966 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:06:00.967 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.690203+0000", 2023-09-23T20:06:00.967 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.1079e-05 2023-09-23T20:06:00.967 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.967 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.967 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:06:00.967 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.690294+0000", 2023-09-23T20:06:00.967 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.7073999999999999e-05 2023-09-23T20:06:00.967 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.967 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.967 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:06:00.967 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.690321+0000", 2023-09-23T20:06:00.967 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.19545496200000001 2023-09-23T20:06:00.967 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.968 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.968 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:06:00.968 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.885776+0000", 2023-09-23T20:06:00.968 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.2624000000000003e-05 2023-09-23T20:06:00.968 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.968 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.968 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:06:00.968 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.885828+0000", 2023-09-23T20:06:00.968 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.19565996099999999 2023-09-23T20:06:00.968 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:06:00.968 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:06:00.968 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:06:00.968 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.969 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.969 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_repop_reply(client.46877.0:156208 14.f e269/256 ondisk, result = 0)", 2023-09-23T20:06:00.969 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T20:00:06.697831+0000", 2023-09-23T20:06:00.969 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 354.22192084099999, 2023-09-23T20:06:00.969 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.17013848200000001, 2023-09-23T20:06:00.969 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:06:00.969 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "started", 2023-09-23T20:06:00.969 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:06:00.969 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.969 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:06:00.969 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.697831+0000", 2023-09-23T20:06:00.969 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:06:00.969 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.970 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.970 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:06:00.970 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.697831+0000", 2023-09-23T20:06:00.970 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T20:06:00.970 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.970 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.970 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:06:00.970 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.697829+0000", 2023-09-23T20:06:00.970 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.0662000000000001e-05 2023-09-23T20:06:00.970 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.970 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.970 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:06:00.971 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.697840+0000", 2023-09-23T20:06:00.971 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.16987813700000001 2023-09-23T20:06:00.971 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.971 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.971 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:06:00.971 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.867718+0000", 2023-09-23T20:06:00.971 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.7621999999999998e-05 2023-09-23T20:06:00.971 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.971 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.971 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:06:00.971 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.867736+0000", 2023-09-23T20:06:00.971 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000187975 2023-09-23T20:06:00.971 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.971 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.972 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:06:00.972 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.867924+0000", 2023-09-23T20:06:00.972 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.0429999999999999e-05 2023-09-23T20:06:00.972 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.972 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.972 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:06:00.972 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.867944+0000", 2023-09-23T20:06:00.972 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.5139000000000001e-05 2023-09-23T20:06:00.972 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.972 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.972 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:06:00.972 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.867969+0000", 2023-09-23T20:06:00.973 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.17013848200000001 2023-09-23T20:06:00.973 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:06:00.973 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:06:00.973 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:06:00.973 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.973 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.973 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_repop(client.46877.0:156220 14.5 e269/256 14:a095c11c:::benchmark_data_smithi016_228083_object56261:head v 269'9743, mlcod=269'9741)", 2023-09-23T20:06:00.973 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T20:00:06.711990+0000", 2023-09-23T20:06:00.973 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 354.20776205200002, 2023-09-23T20:06:00.973 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.16772410099999999, 2023-09-23T20:06:00.973 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:06:00.973 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:00.974 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:06:00.974 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.974 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:06:00.974 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.711990+0000", 2023-09-23T20:06:00.974 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:06:00.974 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.974 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.974 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:06:00.974 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.711990+0000", 2023-09-23T20:06:00.974 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T20:06:00.974 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.974 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.974 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:06:00.975 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.711988+0000", 2023-09-23T20:06:00.975 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.5136999999999999e-05 2023-09-23T20:06:00.975 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.975 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.975 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:06:00.975 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.712004+0000", 2023-09-23T20:06:00.975 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.155722684 2023-09-23T20:06:00.975 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.975 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.975 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:06:00.975 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.867726+0000", 2023-09-23T20:06:00.975 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.4652e-05 2023-09-23T20:06:00.976 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.976 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.976 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:06:00.976 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.867741+0000", 2023-09-23T20:06:00.976 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00012279700000000001 2023-09-23T20:06:00.976 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.976 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.976 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:06:00.976 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.867864+0000", 2023-09-23T20:06:00.976 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.6038000000000001e-05 2023-09-23T20:06:00.976 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.976 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.977 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:06:00.977 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.867920+0000", 2023-09-23T20:06:00.977 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.011769421 2023-09-23T20:06:00.977 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.977 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.977 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:06:00.977 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.879689+0000", 2023-09-23T20:06:00.977 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.4654000000000001e-05 2023-09-23T20:06:00.977 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.977 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.977 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:06:00.977 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.879714+0000", 2023-09-23T20:06:00.977 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.16772410099999999 2023-09-23T20:06:00.978 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:06:00.978 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:06:00.978 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:06:00.978 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.978 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.978 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_repop(client.46877.0:156225 14.4 e269/256 14:2e9b25eb:::benchmark_data_smithi016_228083_object56266:head v 269'9756, mlcod=269'9753)", 2023-09-23T20:06:00.978 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T20:00:06.714431+0000", 2023-09-23T20:06:00.978 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 354.205320556, 2023-09-23T20:06:00.978 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.17143608499999999, 2023-09-23T20:06:00.978 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:06:00.978 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:00.978 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:06:00.979 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.979 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:06:00.979 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.714431+0000", 2023-09-23T20:06:00.979 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:06:00.979 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.979 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.979 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:06:00.979 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.714431+0000", 2023-09-23T20:06:00.979 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.999999 2023-09-23T20:06:00.979 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.979 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.979 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:06:00.980 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.714430+0000", 2023-09-23T20:06:00.980 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.1928e-05 2023-09-23T20:06:00.980 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.980 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.980 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:06:00.980 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.714442+0000", 2023-09-23T20:06:00.980 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.153322237 2023-09-23T20:06:00.980 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.980 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.980 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:06:00.980 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.867764+0000", 2023-09-23T20:06:00.980 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.4139e-05 2023-09-23T20:06:00.981 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.981 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.981 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:06:00.981 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.867798+0000", 2023-09-23T20:06:00.981 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0098272480000000002 2023-09-23T20:06:00.981 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.981 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.981 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:06:00.981 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.877626+0000", 2023-09-23T20:06:00.981 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.4397999999999999e-05 2023-09-23T20:06:00.981 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.981 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.981 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:06:00.982 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.877650+0000", 2023-09-23T20:06:00.982 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0081816920000000008 2023-09-23T20:06:00.982 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.982 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.982 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:06:00.982 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.885832+0000", 2023-09-23T20:06:00.982 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.5441999999999997e-05 2023-09-23T20:06:00.982 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.982 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.982 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:06:00.982 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:06.885867+0000", 2023-09-23T20:06:00.982 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.17143608499999999 2023-09-23T20:06:00.982 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:06:00.983 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:06:00.983 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:06:00.983 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.983 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.983 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.46877.0:177691 14.f 14:ff8ae2d9:::benchmark_data_smithi016_228083_object77732:head [delete] snapc 0=[] ondisk+write+known_if_redirected e271)", 2023-09-23T20:06:00.983 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T20:00:41.523968+0000", 2023-09-23T20:06:00.983 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 319.39578332500002, 2023-09-23T20:06:00.983 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.176987484, 2023-09-23T20:06:00.983 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:06:00.983 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:00.983 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:06:00.983 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.46877", 2023-09-23T20:06:00.983 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:06:00.984 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 177691 2023-09-23T20:06:00.984 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.984 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:06:00.984 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.984 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:06:00.984 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.523968+0000", 2023-09-23T20:06:00.984 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:06:00.984 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.984 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.984 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:06:00.984 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.523968+0000", 2023-09-23T20:06:00.985 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999986 2023-09-23T20:06:00.985 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.985 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.985 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:06:00.985 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.523967+0000", 2023-09-23T20:06:00.985 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.3499999999999999e-06 2023-09-23T20:06:00.985 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.985 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.985 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:06:00.985 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.523974+0000", 2023-09-23T20:06:00.985 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.3818000000000001e-05 2023-09-23T20:06:00.985 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.986 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.986 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:06:00.986 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.523988+0000", 2023-09-23T20:06:00.986 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.5310000000000004e-06 2023-09-23T20:06:00.986 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.986 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.986 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:06:00.986 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.523995+0000", 2023-09-23T20:06:00.986 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.3820999999999999e-05 2023-09-23T20:06:00.986 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.986 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.986 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:06:00.987 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.524039+0000", 2023-09-23T20:06:00.987 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000131748 2023-09-23T20:06:00.987 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.987 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.987 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:06:00.987 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.524171+0000", 2023-09-23T20:06:00.987 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000159603 2023-09-23T20:06:00.987 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.987 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.987 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:06:00.987 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.524331+0000", 2023-09-23T20:06:00.987 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0050509250000000004 2023-09-23T20:06:00.988 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.988 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.988 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:00.988 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.529382+0000", 2023-09-23T20:06:00.988 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000236911 2023-09-23T20:06:00.988 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.988 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.988 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:06:00.988 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.529618+0000", 2023-09-23T20:06:00.988 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.171243012 2023-09-23T20:06:00.988 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.988 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.988 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:00.989 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.700861+0000", 2023-09-23T20:06:00.989 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.3334000000000001e-05 2023-09-23T20:06:00.989 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.989 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.989 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:06:00.989 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.700915+0000", 2023-09-23T20:06:00.989 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.1083e-05 2023-09-23T20:06:00.989 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.989 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.989 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:06:00.989 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:00:41.700956+0000", 2023-09-23T20:06:00.989 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.176987484 2023-09-23T20:06:00.990 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:06:00.990 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:06:00.990 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:06:00.990 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.990 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.990 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.47168.0:49 2.5 2:b9e1c49d:::rbd_header.b8405e3620ba:head [call rbd.snapshot_add in=15b] snapc 0=[] ondisk+write+known_if_redirected e287)", 2023-09-23T20:06:00.990 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T20:02:34.866878+0000", 2023-09-23T20:06:00.990 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 206.05287383800001, 2023-09-23T20:06:00.990 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0077713879999999997, 2023-09-23T20:06:00.990 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:06:00.990 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:00.990 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:06:00.990 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.47168", 2023-09-23T20:06:00.991 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/3330635919", 2023-09-23T20:06:00.991 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 49 2023-09-23T20:06:00.991 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.991 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:06:00.991 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.991 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:06:00.991 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:34.866878+0000", 2023-09-23T20:06:00.991 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:06:00.991 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.991 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.991 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:06:00.991 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:34.866878+0000", 2023-09-23T20:06:00.992 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999976 2023-09-23T20:06:00.992 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.992 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.992 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:06:00.992 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:34.866875+0000", 2023-09-23T20:06:00.992 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.411e-05 2023-09-23T20:06:00.992 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.992 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.992 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:06:00.992 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:34.866889+0000", 2023-09-23T20:06:00.992 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.3910999999999999e-05 2023-09-23T20:06:00.992 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.992 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.993 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:06:00.993 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:34.866913+0000", 2023-09-23T20:06:00.993 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.4025e-05 2023-09-23T20:06:00.993 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.993 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.993 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:06:00.993 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:34.866927+0000", 2023-09-23T20:06:00.993 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000173191 2023-09-23T20:06:00.993 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.993 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.993 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:06:00.993 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:34.867101+0000", 2023-09-23T20:06:00.994 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.4239e-05 2023-09-23T20:06:00.994 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.994 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.994 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op must wait for map", 2023-09-23T20:06:00.994 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:34.867125+0000", 2023-09-23T20:06:00.994 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0057355560000000002 2023-09-23T20:06:00.994 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.994 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.994 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:06:00.994 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:34.872860+0000", 2023-09-23T20:06:00.994 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00012960000000000001 2023-09-23T20:06:00.994 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.994 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.995 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:06:00.995 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:34.872990+0000", 2023-09-23T20:06:00.995 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00032343399999999999 2023-09-23T20:06:00.995 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.995 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.995 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:06:00.995 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:34.873313+0000", 2023-09-23T20:06:00.995 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00059030699999999998 2023-09-23T20:06:00.995 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.995 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.995 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:06:00.996 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:34.873904+0000", 2023-09-23T20:06:00.996 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00058329200000000001 2023-09-23T20:06:00.996 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.996 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.996 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:00.996 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:34.874487+0000", 2023-09-23T20:06:00.996 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.7385000000000005e-05 2023-09-23T20:06:00.996 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.996 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.996 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:00.996 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:34.874564+0000", 2023-09-23T20:06:00.996 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4.5630999999999997e-05 2023-09-23T20:06:00.997 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.997 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.997 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:06:00.997 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:34.874610+0000", 2023-09-23T20:06:00.997 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.9283000000000003e-05 2023-09-23T20:06:00.997 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.997 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.997 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:06:00.997 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:34.874649+0000", 2023-09-23T20:06:00.997 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0077713879999999997 2023-09-23T20:06:00.997 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:06:00.997 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:06:00.997 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:06:00.998 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.998 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.998 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.37251.0:26 2.5 2:b9e1c49d:::rbd_header.b8405e3620ba:head [call rbd.snapshot_remove in=18b] snapc 0=[] ondisk+write+known_if_redirected+full_try e288)", 2023-09-23T20:06:00.998 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T20:02:35.871807+0000", 2023-09-23T20:06:00.998 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 205.04794462999999, 2023-09-23T20:06:00.998 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.010707242, 2023-09-23T20:06:00.998 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:06:00.998 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:00.998 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:06:00.998 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.37251", 2023-09-23T20:06:00.998 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/2162705294", 2023-09-23T20:06:00.998 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 26 2023-09-23T20:06:00.999 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.999 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:06:00.999 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.999 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:06:00.999 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:35.871807+0000", 2023-09-23T20:06:00.999 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:06:00.999 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.999 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:00.999 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:06:00.999 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:35.871807+0000", 2023-09-23T20:06:00.999 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999971 2023-09-23T20:06:00.999 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:00.999 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:01.000 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:06:01.000 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:35.871804+0000", 2023-09-23T20:06:01.000 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.5325e-05 2023-09-23T20:06:01.000 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:01.000 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:01.000 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:06:01.000 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:35.871819+0000", 2023-09-23T20:06:01.000 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 2.845e-05 2023-09-23T20:06:01.000 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:01.000 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:01.000 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:06:01.000 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:35.871848+0000", 2023-09-23T20:06:01.001 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.3723000000000001e-05 2023-09-23T20:06:01.001 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:01.001 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:01.001 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:06:01.001 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:35.871862+0000", 2023-09-23T20:06:01.001 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.2276999999999998e-05 2023-09-23T20:06:01.001 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:01.001 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:01.001 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:06:01.001 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:35.871954+0000", 2023-09-23T20:06:01.001 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.731e-05 2023-09-23T20:06:01.001 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:01.002 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:01.002 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op must wait for map", 2023-09-23T20:06:01.002 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:35.871991+0000", 2023-09-23T20:06:01.002 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0085192719999999996 2023-09-23T20:06:01.002 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:01.002 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:01.002 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:06:01.002 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:35.880510+0000", 2023-09-23T20:06:01.002 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000119939 2023-09-23T20:06:01.002 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:01.002 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:01.002 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:06:01.003 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:35.880630+0000", 2023-09-23T20:06:01.003 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00038135799999999999 2023-09-23T20:06:01.003 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:01.003 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:01.003 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:06:01.003 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:35.881012+0000", 2023-09-23T20:06:01.003 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00091180999999999999 2023-09-23T20:06:01.003 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:01.003 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:01.003 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:06:01.003 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:35.881924+0000", 2023-09-23T20:06:01.003 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000130412 2023-09-23T20:06:01.004 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:01.004 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:01.004 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:01.004 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:35.882054+0000", 2023-09-23T20:06:01.004 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000338442 2023-09-23T20:06:01.004 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:01.004 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:01.004 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:01.004 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:35.882392+0000", 2023-09-23T20:06:01.004 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 6.6165000000000001e-05 2023-09-23T20:06:01.004 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:01.004 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:01.005 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:06:01.005 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:35.882459+0000", 2023-09-23T20:06:01.005 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 5.5738000000000001e-05 2023-09-23T20:06:01.005 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:01.005 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:01.005 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:06:01.005 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:35.882514+0000", 2023-09-23T20:06:01.005 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.010707242 2023-09-23T20:06:01.005 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:06:01.005 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:06:01.005 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:06:01.006 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:01.006 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:01.006 INFO:teuthology.orchestra.run.smithi092.stdout: "description": "osd_op(client.37461.0:41 2.0 2:0da5187f:::rbd_header.b918373bb2bb:head [call rbd.snapshot_remove in=18b] snapc 0=[] ondisk+write+known_if_redirected+full_try e297)", 2023-09-23T20:06:01.006 INFO:teuthology.orchestra.run.smithi092.stdout: "initiated_at": "2023-09-23T20:02:46.946798+0000", 2023-09-23T20:06:01.006 INFO:teuthology.orchestra.run.smithi092.stdout: "age": 193.972953671, 2023-09-23T20:06:01.006 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0081012830000000008, 2023-09-23T20:06:01.006 INFO:teuthology.orchestra.run.smithi092.stdout: "type_data": { 2023-09-23T20:06:01.006 INFO:teuthology.orchestra.run.smithi092.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:01.006 INFO:teuthology.orchestra.run.smithi092.stdout: "client_info": { 2023-09-23T20:06:01.006 INFO:teuthology.orchestra.run.smithi092.stdout: "client": "client.37461", 2023-09-23T20:06:01.006 INFO:teuthology.orchestra.run.smithi092.stdout: "client_addr": "172.21.15.16:0/4192399385", 2023-09-23T20:06:01.006 INFO:teuthology.orchestra.run.smithi092.stdout: "tid": 41 2023-09-23T20:06:01.007 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:01.007 INFO:teuthology.orchestra.run.smithi092.stdout: "events": [ 2023-09-23T20:06:01.007 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:01.007 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "initiated", 2023-09-23T20:06:01.007 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:46.946798+0000", 2023-09-23T20:06:01.007 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0 2023-09-23T20:06:01.007 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:01.007 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:01.007 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "throttled", 2023-09-23T20:06:01.007 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:46.946798+0000", 2023-09-23T20:06:01.007 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 4294967295.9999976 2023-09-23T20:06:01.007 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:01.008 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:01.008 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "header_read", 2023-09-23T20:06:01.008 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:46.946795+0000", 2023-09-23T20:06:01.008 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.5707999999999999e-05 2023-09-23T20:06:01.008 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:01.008 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:01.008 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "all_read", 2023-09-23T20:06:01.008 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:46.946811+0000", 2023-09-23T20:06:01.008 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 3.0039000000000002e-05 2023-09-23T20:06:01.008 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:01.008 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:01.008 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "dispatched", 2023-09-23T20:06:01.009 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:46.946841+0000", 2023-09-23T20:06:01.009 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 1.7374e-05 2023-09-23T20:06:01.009 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:01.009 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:01.009 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "queued_for_pg", 2023-09-23T20:06:01.009 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:46.946859+0000", 2023-09-23T20:06:01.009 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 8.9040000000000001e-05 2023-09-23T20:06:01.009 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:01.009 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:01.009 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:06:01.009 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:46.946948+0000", 2023-09-23T20:06:01.009 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.2529999999999993e-06 2023-09-23T20:06:01.010 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:01.010 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:01.010 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op must wait for map", 2023-09-23T20:06:01.010 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:46.946957+0000", 2023-09-23T20:06:01.010 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0054277290000000001 2023-09-23T20:06:01.010 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:01.010 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:01.010 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "reached_pg", 2023-09-23T20:06:01.010 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:46.952385+0000", 2023-09-23T20:06:01.010 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00019615499999999999 2023-09-23T20:06:01.010 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:01.010 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:01.011 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "started", 2023-09-23T20:06:01.011 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:46.952581+0000", 2023-09-23T20:06:01.011 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00059112100000000001 2023-09-23T20:06:01.011 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:01.011 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:01.011 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:06:01.011 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:46.953172+0000", 2023-09-23T20:06:01.011 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0010499400000000001 2023-09-23T20:06:01.011 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:01.011 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:01.011 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "op_commit", 2023-09-23T20:06:01.012 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:46.954222+0000", 2023-09-23T20:06:01.012 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.00032888600000000002 2023-09-23T20:06:01.012 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:01.012 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:01.012 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:01.012 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:46.954551+0000", 2023-09-23T20:06:01.012 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.000181805 2023-09-23T20:06:01.012 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:01.012 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:01.012 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:01.012 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:46.954733+0000", 2023-09-23T20:06:01.012 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 7.2503999999999996e-05 2023-09-23T20:06:01.013 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:01.013 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:01.013 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "commit_sent", 2023-09-23T20:06:01.013 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:46.954805+0000", 2023-09-23T20:06:01.013 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 9.4283000000000005e-05 2023-09-23T20:06:01.013 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2023-09-23T20:06:01.013 INFO:teuthology.orchestra.run.smithi092.stdout: { 2023-09-23T20:06:01.013 INFO:teuthology.orchestra.run.smithi092.stdout: "event": "done", 2023-09-23T20:06:01.013 INFO:teuthology.orchestra.run.smithi092.stdout: "time": "2023-09-23T20:02:46.954899+0000", 2023-09-23T20:06:01.013 INFO:teuthology.orchestra.run.smithi092.stdout: "duration": 0.0081012830000000008 2023-09-23T20:06:01.013 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:06:01.013 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:06:01.013 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:06:01.014 INFO:teuthology.orchestra.run.smithi092.stdout: } 2023-09-23T20:06:01.014 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2023-09-23T20:06:01.014 INFO:teuthology.orchestra.run.smithi092.stdout:} 2023-09-23T20:06:01.807 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T20:06:02.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:02 smithi016 bash[75004]: cluster 2023-09-23T20:06:00.760846+0000 mgr.y (mgr.25325) 1765 : cluster 0 pgmap v1856: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:02.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:02 smithi016 bash[77995]: cluster 2023-09-23T20:06:00.760846+0000 mgr.y (mgr.25325) 1765 : cluster 0 pgmap v1856: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:02.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:02 smithi092 bash[58855]: cluster 2023-09-23T20:06:00.760846+0000 mgr.y (mgr.25325) 1765 : cluster 0 pgmap v1856: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:03.101 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:06:04.524 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:04 smithi016 bash[75004]: cluster 2023-09-23T20:06:02.761847+0000 mgr.y (mgr.25325) 1766 : cluster 0 pgmap v1857: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:04.524 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:04 smithi016 bash[77995]: cluster 2023-09-23T20:06:02.761847+0000 mgr.y (mgr.25325) 1766 : cluster 0 pgmap v1857: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:04.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:04 smithi092 bash[58855]: cluster 2023-09-23T20:06:02.761847+0000 mgr.y (mgr.25325) 1766 : cluster 0 pgmap v1857: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:06.035 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:06:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:20:06:05] "GET /metrics HTTP/1.1" 200 34014 "" "Prometheus/2.33.4" 2023-09-23T20:06:06.149 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:06:06.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:06 smithi016 bash[77995]: cluster 2023-09-23T20:06:04.762670+0000 mgr.y (mgr.25325) 1767 : cluster 0 pgmap v1858: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:06.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:06 smithi016 bash[75004]: cluster 2023-09-23T20:06:04.762670+0000 mgr.y (mgr.25325) 1767 : cluster 0 pgmap v1858: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:06.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:06 smithi092 bash[58855]: cluster 2023-09-23T20:06:04.762670+0000 mgr.y (mgr.25325) 1767 : cluster 0 pgmap v1858: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:07.337 INFO:teuthology.orchestra.run.smithi016.stderr:set osd.7 primary-affinity to 1 (8655362) 2023-09-23T20:06:07.608 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:07 smithi016 bash[75004]: audit 2023-09-23T20:06:06.402410+0000 mon.a (mon.0) 2137 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:06:07.608 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:07 smithi016 bash[75004]: audit 2023-09-23T20:06:06.520453+0000 mon.c (mon.2) 261 : audit 1 from='client.? 172.21.15.16:0/4080680640' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:06:07.608 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:07 smithi016 bash[75004]: audit 2023-09-23T20:06:06.521037+0000 mon.a (mon.0) 2138 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:06:07.608 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:07 smithi016 bash[77995]: audit 2023-09-23T20:06:06.402410+0000 mon.a (mon.0) 2137 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:06:07.609 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:07 smithi016 bash[77995]: audit 2023-09-23T20:06:06.520453+0000 mon.c (mon.2) 261 : audit 1 from='client.? 172.21.15.16:0/4080680640' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:06:07.609 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:07 smithi016 bash[77995]: audit 2023-09-23T20:06:06.521037+0000 mon.a (mon.0) 2138 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:06:07.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:07 smithi092 bash[58855]: audit 2023-09-23T20:06:06.402410+0000 mon.a (mon.0) 2137 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:06:07.790 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:07 smithi092 bash[58855]: audit 2023-09-23T20:06:06.520453+0000 mon.c (mon.2) 261 : audit 1 from='client.? 172.21.15.16:0/4080680640' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:06:07.790 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:07 smithi092 bash[58855]: audit 2023-09-23T20:06:06.521037+0000 mon.a (mon.0) 2138 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:06:08.305 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T20:06:08.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:08 smithi016 bash[75004]: cluster 2023-09-23T20:06:06.763489+0000 mgr.y (mgr.25325) 1768 : cluster 0 pgmap v1859: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:08.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:08 smithi016 bash[75004]: audit 2023-09-23T20:06:07.325428+0000 mon.a (mon.0) 2139 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:06:08.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:08 smithi016 bash[75004]: cluster 2023-09-23T20:06:07.325502+0000 mon.a (mon.0) 2140 : cluster 0 osdmap e331: 8 total, 8 up, 8 in 2023-09-23T20:06:08.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:08 smithi016 bash[75004]: cluster 2023-09-23T20:06:07.327718+0000 osd.6 (osd.6) 269 : cluster 3 failed to encode map e331 with expected crc 2023-09-23T20:06:08.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:08 smithi016 bash[75004]: cluster 2023-09-23T20:06:07.742939+0000 osd.5 (osd.5) 252 : cluster 3 failed to encode map e331 with expected crc 2023-09-23T20:06:08.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:08 smithi016 bash[77995]: cluster 2023-09-23T20:06:06.763489+0000 mgr.y (mgr.25325) 1768 : cluster 0 pgmap v1859: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:08.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:08 smithi016 bash[77995]: audit 2023-09-23T20:06:07.325428+0000 mon.a (mon.0) 2139 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:06:08.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:08 smithi016 bash[77995]: cluster 2023-09-23T20:06:07.325502+0000 mon.a (mon.0) 2140 : cluster 0 osdmap e331: 8 total, 8 up, 8 in 2023-09-23T20:06:08.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:08 smithi016 bash[77995]: cluster 2023-09-23T20:06:07.327718+0000 osd.6 (osd.6) 269 : cluster 3 failed to encode map e331 with expected crc 2023-09-23T20:06:08.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:08 smithi016 bash[77995]: cluster 2023-09-23T20:06:07.742939+0000 osd.5 (osd.5) 252 : cluster 3 failed to encode map e331 with expected crc 2023-09-23T20:06:08.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:08 smithi092 bash[58855]: cluster 2023-09-23T20:06:06.763489+0000 mgr.y (mgr.25325) 1768 : cluster 0 pgmap v1859: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:08.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:08 smithi092 bash[58855]: audit 2023-09-23T20:06:07.325428+0000 mon.a (mon.0) 2139 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:06:08.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:08 smithi092 bash[58855]: cluster 2023-09-23T20:06:07.325502+0000 mon.a (mon.0) 2140 : cluster 0 osdmap e331: 8 total, 8 up, 8 in 2023-09-23T20:06:08.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:08 smithi092 bash[58855]: cluster 2023-09-23T20:06:07.327718+0000 osd.6 (osd.6) 269 : cluster 3 failed to encode map e331 with expected crc 2023-09-23T20:06:08.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:08 smithi092 bash[58855]: cluster 2023-09-23T20:06:07.742939+0000 osd.5 (osd.5) 252 : cluster 3 failed to encode map e331 with expected crc 2023-09-23T20:06:10.292 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-09-23T20:06:10.292 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [], 2023-09-23T20:06:10.292 INFO:teuthology.orchestra.run.smithi016.stdout: "num_ops": 0 2023-09-23T20:06:10.292 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-09-23T20:06:10.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:10 smithi016 bash[75004]: cluster 2023-09-23T20:06:08.764151+0000 mgr.y (mgr.25325) 1769 : cluster 0 pgmap v1861: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:10.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:10 smithi016 bash[77995]: cluster 2023-09-23T20:06:08.764151+0000 mgr.y (mgr.25325) 1769 : cluster 0 pgmap v1861: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:10.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:10 smithi092 bash[58855]: cluster 2023-09-23T20:06:08.764151+0000 mgr.y (mgr.25325) 1769 : cluster 0 pgmap v1861: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:11.192 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T20:06:12.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:12 smithi016 bash[77995]: cluster 2023-09-23T20:06:10.764814+0000 mgr.y (mgr.25325) 1770 : cluster 0 pgmap v1862: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:12.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:12 smithi016 bash[75004]: cluster 2023-09-23T20:06:10.764814+0000 mgr.y (mgr.25325) 1770 : cluster 0 pgmap v1862: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:12.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:12 smithi092 bash[58855]: cluster 2023-09-23T20:06:10.764814+0000 mgr.y (mgr.25325) 1770 : cluster 0 pgmap v1862: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:13.868 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:06:13.901 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:13 smithi016 bash[75004]: audit 2023-09-23T20:06:12.645506+0000 mon.a (mon.0) 2141 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:06:13.901 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:13 smithi016 bash[75004]: audit 2023-09-23T20:06:12.654041+0000 mon.a (mon.0) 2142 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:06:13.901 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:13 smithi016 bash[75004]: cluster 2023-09-23T20:06:12.765444+0000 mgr.y (mgr.25325) 1771 : cluster 0 pgmap v1863: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:13.902 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:13 smithi016 bash[75004]: audit 2023-09-23T20:06:13.150414+0000 mon.a (mon.0) 2143 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:06:13.902 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:13 smithi016 bash[75004]: audit 2023-09-23T20:06:13.158404+0000 mon.a (mon.0) 2144 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:06:13.902 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:13 smithi016 bash[77995]: audit 2023-09-23T20:06:12.645506+0000 mon.a (mon.0) 2141 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:06:13.902 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:13 smithi016 bash[77995]: audit 2023-09-23T20:06:12.654041+0000 mon.a (mon.0) 2142 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:06:13.902 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:13 smithi016 bash[77995]: cluster 2023-09-23T20:06:12.765444+0000 mgr.y (mgr.25325) 1771 : cluster 0 pgmap v1863: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:13.902 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:13 smithi016 bash[77995]: audit 2023-09-23T20:06:13.150414+0000 mon.a (mon.0) 2143 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:06:13.903 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:13 smithi016 bash[77995]: audit 2023-09-23T20:06:13.158404+0000 mon.a (mon.0) 2144 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:06:14.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:13 smithi092 bash[58855]: audit 2023-09-23T20:06:12.645506+0000 mon.a (mon.0) 2141 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:06:14.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:13 smithi092 bash[58855]: audit 2023-09-23T20:06:12.654041+0000 mon.a (mon.0) 2142 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:06:14.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:13 smithi092 bash[58855]: cluster 2023-09-23T20:06:12.765444+0000 mgr.y (mgr.25325) 1771 : cluster 0 pgmap v1863: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:14.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:13 smithi092 bash[58855]: audit 2023-09-23T20:06:13.150414+0000 mon.a (mon.0) 2143 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:06:14.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:13 smithi092 bash[58855]: audit 2023-09-23T20:06:13.158404+0000 mon.a (mon.0) 2144 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:06:15.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:14 smithi016 bash[75004]: audit 2023-09-23T20:06:13.680204+0000 mon.a (mon.0) 2145 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:06:15.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:14 smithi016 bash[75004]: audit 2023-09-23T20:06:13.682152+0000 mon.a (mon.0) 2146 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:06:15.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:14 smithi016 bash[75004]: audit 2023-09-23T20:06:13.691018+0000 mon.a (mon.0) 2147 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:06:15.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:14 smithi016 bash[75004]: audit 2023-09-23T20:06:13.705845+0000 mon.a (mon.0) 2148 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:06:15.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:14 smithi016 bash[77995]: audit 2023-09-23T20:06:13.680204+0000 mon.a (mon.0) 2145 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:06:15.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:14 smithi016 bash[77995]: audit 2023-09-23T20:06:13.682152+0000 mon.a (mon.0) 2146 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:06:15.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:14 smithi016 bash[77995]: audit 2023-09-23T20:06:13.691018+0000 mon.a (mon.0) 2147 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:06:15.036 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:14 smithi016 bash[77995]: audit 2023-09-23T20:06:13.705845+0000 mon.a (mon.0) 2148 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:06:15.038 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:14 smithi092 bash[58855]: audit 2023-09-23T20:06:13.680204+0000 mon.a (mon.0) 2145 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:06:15.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:14 smithi092 bash[58855]: audit 2023-09-23T20:06:13.682152+0000 mon.a (mon.0) 2146 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:06:15.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:14 smithi092 bash[58855]: audit 2023-09-23T20:06:13.691018+0000 mon.a (mon.0) 2147 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:06:15.039 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:14 smithi092 bash[58855]: audit 2023-09-23T20:06:13.705845+0000 mon.a (mon.0) 2148 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:06:15.785 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:06:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:20:06:15] "GET /metrics HTTP/1.1" 200 34012 "" "Prometheus/2.33.4" 2023-09-23T20:06:16.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:15 smithi016 bash[75004]: cluster 2023-09-23T20:06:14.766218+0000 mgr.y (mgr.25325) 1772 : cluster 0 pgmap v1864: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:16.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:15 smithi016 bash[77995]: cluster 2023-09-23T20:06:14.766218+0000 mgr.y (mgr.25325) 1772 : cluster 0 pgmap v1864: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:16.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:15 smithi092 bash[58855]: cluster 2023-09-23T20:06:14.766218+0000 mgr.y (mgr.25325) 1772 : cluster 0 pgmap v1864: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:16.769 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:06:18.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:17 smithi016 bash[77995]: cluster 2023-09-23T20:06:16.767077+0000 mgr.y (mgr.25325) 1773 : cluster 0 pgmap v1865: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:18.038 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:17 smithi016 bash[75004]: cluster 2023-09-23T20:06:16.767077+0000 mgr.y (mgr.25325) 1773 : cluster 0 pgmap v1865: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:18.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:17 smithi092 bash[58855]: cluster 2023-09-23T20:06:16.767077+0000 mgr.y (mgr.25325) 1773 : cluster 0 pgmap v1865: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:18.941 INFO:teuthology.orchestra.run.smithi016.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T20:06:19.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:18 smithi016 bash[75004]: audit 2023-09-23T20:06:18.252703+0000 mon.a (mon.0) 2149 : audit 1 from='client.? 172.21.15.16:0/661028189' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:06:19.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:18 smithi016 bash[77995]: audit 2023-09-23T20:06:18.252703+0000 mon.a (mon.0) 2149 : audit 1 from='client.? 172.21.15.16:0/661028189' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:06:19.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:18 smithi092 bash[58855]: audit 2023-09-23T20:06:18.252703+0000 mon.a (mon.0) 2149 : audit 1 from='client.? 172.21.15.16:0/661028189' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:06:20.266 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:19 smithi016 bash[75004]: cluster 2023-09-23T20:06:18.767927+0000 mgr.y (mgr.25325) 1774 : cluster 0 pgmap v1866: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:20.266 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:19 smithi016 bash[75004]: audit 2023-09-23T20:06:18.934156+0000 mon.a (mon.0) 2150 : audit 1 from='client.? 172.21.15.16:0/661028189' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:06:20.266 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:19 smithi016 bash[75004]: cluster 2023-09-23T20:06:18.934322+0000 mon.a (mon.0) 2151 : cluster 0 osdmap e332: 8 total, 8 up, 8 in 2023-09-23T20:06:20.266 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:19 smithi016 bash[75004]: cluster 2023-09-23T20:06:19.227635+0000 osd.5 (osd.5) 253 : cluster 3 failed to encode map e332 with expected crc 2023-09-23T20:06:20.267 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:19 smithi016 bash[77995]: cluster 2023-09-23T20:06:18.767927+0000 mgr.y (mgr.25325) 1774 : cluster 0 pgmap v1866: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:20.267 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:19 smithi016 bash[77995]: audit 2023-09-23T20:06:18.934156+0000 mon.a (mon.0) 2150 : audit 1 from='client.? 172.21.15.16:0/661028189' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:06:20.267 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:19 smithi016 bash[77995]: cluster 2023-09-23T20:06:18.934322+0000 mon.a (mon.0) 2151 : cluster 0 osdmap e332: 8 total, 8 up, 8 in 2023-09-23T20:06:20.267 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:19 smithi016 bash[77995]: cluster 2023-09-23T20:06:19.227635+0000 osd.5 (osd.5) 253 : cluster 3 failed to encode map e332 with expected crc 2023-09-23T20:06:20.268 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 2 1 2023-09-23T20:06:20.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:19 smithi092 bash[58855]: cluster 2023-09-23T20:06:18.767927+0000 mgr.y (mgr.25325) 1774 : cluster 0 pgmap v1866: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:20.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:19 smithi092 bash[58855]: audit 2023-09-23T20:06:18.934156+0000 mon.a (mon.0) 2150 : audit 1 from='client.? 172.21.15.16:0/661028189' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:06:20.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:19 smithi092 bash[58855]: cluster 2023-09-23T20:06:18.934322+0000 mon.a (mon.0) 2151 : cluster 0 osdmap e332: 8 total, 8 up, 8 in 2023-09-23T20:06:20.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:19 smithi092 bash[58855]: cluster 2023-09-23T20:06:19.227635+0000 osd.5 (osd.5) 253 : cluster 3 failed to encode map e332 with expected crc 2023-09-23T20:06:20.609 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-09-23T20:06:20.610 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [], 2023-09-23T20:06:20.610 INFO:teuthology.orchestra.run.smithi016.stdout: "complaint_time": 30, 2023-09-23T20:06:20.610 INFO:teuthology.orchestra.run.smithi016.stdout: "num_blocked_ops": 0 2023-09-23T20:06:20.610 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-09-23T20:06:21.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:20 smithi016 bash[75004]: cluster 2023-09-23T20:06:19.177367+0000 osd.6 (osd.6) 270 : cluster 3 failed to encode map e332 with expected crc 2023-09-23T20:06:21.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:20 smithi016 bash[75004]: cluster 2023-09-23T20:06:19.179999+0000 osd.6 (osd.6) 271 : cluster 3 failed to encode map e332 with expected crc 2023-09-23T20:06:21.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:20 smithi016 bash[75004]: cluster 2023-09-23T20:06:19.227481+0000 osd.7 (osd.7) 251 : cluster 3 failed to encode map e332 with expected crc 2023-09-23T20:06:21.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:20 smithi016 bash[77995]: cluster 2023-09-23T20:06:19.177367+0000 osd.6 (osd.6) 270 : cluster 3 failed to encode map e332 with expected crc 2023-09-23T20:06:21.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:20 smithi016 bash[77995]: cluster 2023-09-23T20:06:19.179999+0000 osd.6 (osd.6) 271 : cluster 3 failed to encode map e332 with expected crc 2023-09-23T20:06:21.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:20 smithi016 bash[77995]: cluster 2023-09-23T20:06:19.227481+0000 osd.7 (osd.7) 251 : cluster 3 failed to encode map e332 with expected crc 2023-09-23T20:06:21.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:20 smithi092 bash[58855]: cluster 2023-09-23T20:06:19.177367+0000 osd.6 (osd.6) 270 : cluster 3 failed to encode map e332 with expected crc 2023-09-23T20:06:21.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:20 smithi092 bash[58855]: cluster 2023-09-23T20:06:19.179999+0000 osd.6 (osd.6) 271 : cluster 3 failed to encode map e332 with expected crc 2023-09-23T20:06:21.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:20 smithi092 bash[58855]: cluster 2023-09-23T20:06:19.227481+0000 osd.7 (osd.7) 251 : cluster 3 failed to encode map e332 with expected crc 2023-09-23T20:06:21.626 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T20:06:22.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:21 smithi016 bash[75004]: cluster 2023-09-23T20:06:20.768539+0000 mgr.y (mgr.25325) 1775 : cluster 0 pgmap v1868: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:22.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:21 smithi016 bash[77995]: cluster 2023-09-23T20:06:20.768539+0000 mgr.y (mgr.25325) 1775 : cluster 0 pgmap v1868: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:22.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:21 smithi092 bash[58855]: cluster 2023-09-23T20:06:20.768539+0000 mgr.y (mgr.25325) 1775 : cluster 0 pgmap v1868: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:24.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:23 smithi016 bash[75004]: cluster 2023-09-23T20:06:22.769324+0000 mgr.y (mgr.25325) 1776 : cluster 0 pgmap v1869: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:24.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:23 smithi016 bash[77995]: cluster 2023-09-23T20:06:22.769324+0000 mgr.y (mgr.25325) 1776 : cluster 0 pgmap v1869: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:24.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:23 smithi092 bash[58855]: cluster 2023-09-23T20:06:22.769324+0000 mgr.y (mgr.25325) 1776 : cluster 0 pgmap v1869: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:25.150 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:06:25.951 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:06:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:20:06:25] "GET /metrics HTTP/1.1" 200 34012 "" "Prometheus/2.33.4" 2023-09-23T20:06:26.170 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:06:26.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:25 smithi016 bash[77995]: cluster 2023-09-23T20:06:24.770074+0000 mgr.y (mgr.25325) 1777 : cluster 0 pgmap v1870: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:26.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:25 smithi016 bash[75004]: cluster 2023-09-23T20:06:24.770074+0000 mgr.y (mgr.25325) 1777 : cluster 0 pgmap v1870: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:26.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:25 smithi092 bash[58855]: cluster 2023-09-23T20:06:24.770074+0000 mgr.y (mgr.25325) 1777 : cluster 0 pgmap v1870: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:28.035 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:27 smithi016 bash[75004]: cluster 2023-09-23T20:06:26.770708+0000 mgr.y (mgr.25325) 1778 : cluster 0 pgmap v1871: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:28.035 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:27 smithi016 bash[77995]: cluster 2023-09-23T20:06:26.770708+0000 mgr.y (mgr.25325) 1778 : cluster 0 pgmap v1871: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:28.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:27 smithi092 bash[58855]: cluster 2023-09-23T20:06:26.770708+0000 mgr.y (mgr.25325) 1778 : cluster 0 pgmap v1871: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:29.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:28 smithi016 bash[75004]: audit 2023-09-23T20:06:28.706675+0000 mon.a (mon.0) 2152 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:06:29.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:28 smithi016 bash[77995]: audit 2023-09-23T20:06:28.706675+0000 mon.a (mon.0) 2152 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:06:29.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:28 smithi092 bash[58855]: audit 2023-09-23T20:06:28.706675+0000 mon.a (mon.0) 2152 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:06:30.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:29 smithi016 bash[75004]: cluster 2023-09-23T20:06:28.771487+0000 mgr.y (mgr.25325) 1779 : cluster 0 pgmap v1872: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:30.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:29 smithi016 bash[77995]: cluster 2023-09-23T20:06:28.771487+0000 mgr.y (mgr.25325) 1779 : cluster 0 pgmap v1872: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:30.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:29 smithi092 bash[58855]: cluster 2023-09-23T20:06:28.771487+0000 mgr.y (mgr.25325) 1779 : cluster 0 pgmap v1872: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:31.181 INFO:teuthology.orchestra.run.smithi016.stdout:{ 2023-09-23T20:06:31.181 INFO:teuthology.orchestra.run.smithi016.stdout: "size": 20, 2023-09-23T20:06:31.181 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 600, 2023-09-23T20:06:31.181 INFO:teuthology.orchestra.run.smithi016.stdout: "ops": [ 2023-09-23T20:06:31.181 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.181 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46877.0:32633 14.6 14:6f340fca:::benchmark_data_smithi016_228083_object32632:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-09-23T20:06:31.182 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:57:32.054419+0000", 2023-09-23T20:06:31.182 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 539.11857769300002, 2023-09-23T20:06:31.182 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.31474886699999999, 2023-09-23T20:06:31.182 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:06:31.182 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:31.182 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:06:31.182 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46877", 2023-09-23T20:06:31.182 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:06:31.182 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 32633 2023-09-23T20:06:31.182 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.182 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:06:31.183 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.183 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:06:31.183 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.054419+0000", 2023-09-23T20:06:31.183 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:06:31.183 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.183 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.183 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:06:31.183 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.054419+0000", 2023-09-23T20:06:31.183 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:06:31.183 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.183 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.183 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:06:31.184 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.054418+0000", 2023-09-23T20:06:31.184 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-09-23T20:06:31.184 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.184 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.184 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:06:31.184 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.054444+0000", 2023-09-23T20:06:31.184 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.6101e-05 2023-09-23T20:06:31.184 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.184 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.184 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:06:31.184 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.054457+0000", 2023-09-23T20:06:31.185 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.306e-05 2023-09-23T20:06:31.185 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.185 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.185 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:06:31.185 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.054465+0000", 2023-09-23T20:06:31.185 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.2840000000000006e-06 2023-09-23T20:06:31.185 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.185 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.185 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:06:31.185 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.054518+0000", 2023-09-23T20:06:31.185 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.3016999999999997e-05 2023-09-23T20:06:31.185 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.186 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.186 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:06:31.186 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.054747+0000", 2023-09-23T20:06:31.186 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00022910699999999999 2023-09-23T20:06:31.186 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.186 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.186 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:06:31.186 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.054900+0000", 2023-09-23T20:06:31.186 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00015242399999999999 2023-09-23T20:06:31.186 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.186 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.186 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:31.187 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.369002+0000", 2023-09-23T20:06:31.187 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.31410250000000001 2023-09-23T20:06:31.187 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.187 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.187 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T20:06:31.187 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.369018+0000", 2023-09-23T20:06:31.187 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.5424999999999999e-05 2023-09-23T20:06:31.187 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.187 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.187 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:31.187 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.369058+0000", 2023-09-23T20:06:31.187 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.0711999999999999e-05 2023-09-23T20:06:31.187 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.188 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.188 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:06:31.188 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.369111+0000", 2023-09-23T20:06:31.188 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.2889000000000001e-05 2023-09-23T20:06:31.188 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.188 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.188 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:06:31.188 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.369168+0000", 2023-09-23T20:06:31.188 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.6916999999999997e-05 2023-09-23T20:06:31.188 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:06:31.188 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:06:31.188 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:06:31.189 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.189 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.189 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_repop_reply(client.46877.0:32633 14.6 e261/256 ondisk, result = 0)", 2023-09-23T20:06:31.189 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:57:32.058418+0000", 2023-09-23T20:06:31.189 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 539.114578704, 2023-09-23T20:06:31.189 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.310601026, 2023-09-23T20:06:31.189 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:06:31.189 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "started", 2023-09-23T20:06:31.189 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:06:31.189 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.189 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:06:31.189 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.058418+0000", 2023-09-23T20:06:31.190 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:06:31.190 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.190 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.190 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:06:31.190 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.058418+0000", 2023-09-23T20:06:31.190 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:06:31.190 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.190 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.190 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:06:31.190 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.058417+0000", 2023-09-23T20:06:31.190 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T20:06:31.190 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.191 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.191 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:06:31.191 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.058433+0000", 2023-09-23T20:06:31.191 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.5531000000000001e-05 2023-09-23T20:06:31.191 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.191 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.191 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:06:31.191 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.368773+0000", 2023-09-23T20:06:31.191 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.31034017899999999 2023-09-23T20:06:31.191 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.191 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.191 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:06:31.192 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.368788+0000", 2023-09-23T20:06:31.192 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.4793e-05 2023-09-23T20:06:31.192 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.192 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.192 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:06:31.192 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.368977+0000", 2023-09-23T20:06:31.192 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00018866600000000001 2023-09-23T20:06:31.192 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.192 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.192 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:06:31.192 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.368991+0000", 2023-09-23T20:06:31.192 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.4089e-05 2023-09-23T20:06:31.193 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.193 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.193 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:06:31.193 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:32.369019+0000", 2023-09-23T20:06:31.193 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.8807999999999999e-05 2023-09-23T20:06:31.193 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:06:31.193 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:06:31.193 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:06:31.193 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.193 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.193 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46877.0:37521 14.0 14:01ed7a9d:::benchmark_data_smithi016_228083_object37520:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-09-23T20:06:31.193 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:57:36.004529+0000", 2023-09-23T20:06:31.194 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 535.16846774199996, 2023-09-23T20:06:31.194 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.242428481, 2023-09-23T20:06:31.194 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:06:31.194 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:31.194 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:06:31.194 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46877", 2023-09-23T20:06:31.194 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:06:31.194 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 37521 2023-09-23T20:06:31.194 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.194 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:06:31.194 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.194 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:06:31.195 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.004529+0000", 2023-09-23T20:06:31.195 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:06:31.195 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.195 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.195 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:06:31.195 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.004529+0000", 2023-09-23T20:06:31.195 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:06:31.195 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.195 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.195 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:06:31.195 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.004529+0000", 2023-09-23T20:06:31.195 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T20:06:31.196 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.196 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.196 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:06:31.196 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.004584+0000", 2023-09-23T20:06:31.196 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.5247999999999997e-05 2023-09-23T20:06:31.196 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.196 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.196 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:06:31.196 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.004596+0000", 2023-09-23T20:06:31.196 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1814000000000001e-05 2023-09-23T20:06:31.196 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.196 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.196 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:06:31.197 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.004603+0000", 2023-09-23T20:06:31.197 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.5689999999999999e-06 2023-09-23T20:06:31.197 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.197 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.197 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:06:31.197 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.009703+0000", 2023-09-23T20:06:31.197 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0050999690000000002 2023-09-23T20:06:31.197 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.197 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.197 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:06:31.197 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.009809+0000", 2023-09-23T20:06:31.198 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00010566299999999999 2023-09-23T20:06:31.198 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.198 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.198 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:06:31.198 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.009901+0000", 2023-09-23T20:06:31.198 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.1693000000000005e-05 2023-09-23T20:06:31.198 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.198 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.198 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:31.198 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.245771+0000", 2023-09-23T20:06:31.198 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.235870471 2023-09-23T20:06:31.198 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.199 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.199 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T20:06:31.199 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.245856+0000", 2023-09-23T20:06:31.199 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.5402999999999995e-05 2023-09-23T20:06:31.199 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.199 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.199 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:31.199 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.246872+0000", 2023-09-23T20:06:31.199 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001015515 2023-09-23T20:06:31.199 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.199 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.199 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:06:31.200 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.246921+0000", 2023-09-23T20:06:31.200 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.9431999999999997e-05 2023-09-23T20:06:31.200 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.200 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.200 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:06:31.200 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.246958+0000", 2023-09-23T20:06:31.200 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.6399999999999997e-05 2023-09-23T20:06:31.200 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:06:31.200 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:06:31.200 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:06:31.200 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.200 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.201 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46877.0:37526 14.0 14:03545d35:::benchmark_data_smithi016_228083_object37525:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-09-23T20:06:31.201 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:57:36.004695+0000", 2023-09-23T20:06:31.201 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 535.16830207199996, 2023-09-23T20:06:31.201 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.24465336600000001, 2023-09-23T20:06:31.201 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:06:31.201 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:31.201 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:06:31.201 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46877", 2023-09-23T20:06:31.201 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:06:31.201 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 37526 2023-09-23T20:06:31.201 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.201 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:06:31.201 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.202 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:06:31.202 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.004695+0000", 2023-09-23T20:06:31.202 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:06:31.202 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.202 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.202 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:06:31.202 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.004695+0000", 2023-09-23T20:06:31.202 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:06:31.202 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.202 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.202 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:06:31.202 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.004695+0000", 2023-09-23T20:06:31.203 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T20:06:31.203 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.203 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.203 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:06:31.203 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.004714+0000", 2023-09-23T20:06:31.203 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.8773000000000001e-05 2023-09-23T20:06:31.203 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.203 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.203 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:06:31.203 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.004718+0000", 2023-09-23T20:06:31.203 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.2939999999999999e-06 2023-09-23T20:06:31.203 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.203 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.204 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:06:31.204 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.004721+0000", 2023-09-23T20:06:31.204 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.0809999999999998e-06 2023-09-23T20:06:31.204 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.204 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.204 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:06:31.204 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.011190+0000", 2023-09-23T20:06:31.204 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.006469018 2023-09-23T20:06:31.204 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.204 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.204 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:06:31.204 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.011340+0000", 2023-09-23T20:06:31.205 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000150049 2023-09-23T20:06:31.205 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.205 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.205 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:06:31.205 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.011497+0000", 2023-09-23T20:06:31.205 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00015707699999999999 2023-09-23T20:06:31.205 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.205 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.205 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:31.205 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.245840+0000", 2023-09-23T20:06:31.205 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.234342732 2023-09-23T20:06:31.205 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.205 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.206 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T20:06:31.206 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.246835+0000", 2023-09-23T20:06:31.206 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00099527899999999996 2023-09-23T20:06:31.206 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.206 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.206 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:31.206 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.249157+0000", 2023-09-23T20:06:31.206 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0023215419999999998 2023-09-23T20:06:31.206 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.206 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.207 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:06:31.207 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.249214+0000", 2023-09-23T20:06:31.207 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.7711999999999999e-05 2023-09-23T20:06:31.207 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.207 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.207 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:06:31.207 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:36.249348+0000", 2023-09-23T20:06:31.207 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000134091 2023-09-23T20:06:31.207 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:06:31.207 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:06:31.207 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:06:31.207 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.207 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.208 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46877.0:40016 14.b 14:d86fb8d2:::benchmark_data_smithi016_228083_object40015:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-09-23T20:06:31.208 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:57:38.113494+0000", 2023-09-23T20:06:31.208 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 533.05950347700002, 2023-09-23T20:06:31.208 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.28720668100000002, 2023-09-23T20:06:31.208 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:06:31.208 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:31.208 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:06:31.208 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46877", 2023-09-23T20:06:31.208 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:06:31.208 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 40016 2023-09-23T20:06:31.208 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.208 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:06:31.209 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.209 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:06:31.209 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.113494+0000", 2023-09-23T20:06:31.209 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:06:31.209 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.209 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.209 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:06:31.209 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.113494+0000", 2023-09-23T20:06:31.209 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:06:31.209 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.209 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.209 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:06:31.209 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.113492+0000", 2023-09-23T20:06:31.209 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T20:06:31.210 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.210 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.210 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:06:31.210 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.113512+0000", 2023-09-23T20:06:31.210 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.9595000000000001e-05 2023-09-23T20:06:31.210 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.210 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.210 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:06:31.210 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.113523+0000", 2023-09-23T20:06:31.210 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0682e-05 2023-09-23T20:06:31.210 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.210 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.211 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:06:31.211 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.113529+0000", 2023-09-23T20:06:31.211 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.993e-06 2023-09-23T20:06:31.211 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.211 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.211 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:06:31.211 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.114335+0000", 2023-09-23T20:06:31.211 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00080637600000000003 2023-09-23T20:06:31.211 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.211 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.211 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:06:31.211 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.114511+0000", 2023-09-23T20:06:31.212 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00017593 2023-09-23T20:06:31.212 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.212 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.212 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:06:31.212 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.114673+0000", 2023-09-23T20:06:31.212 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00016156300000000001 2023-09-23T20:06:31.212 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.212 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.212 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T20:06:31.212 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.122480+0000", 2023-09-23T20:06:31.212 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0078071590000000001 2023-09-23T20:06:31.212 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.212 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.213 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:31.213 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.123810+0000", 2023-09-23T20:06:31.213 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001330389 2023-09-23T20:06:31.213 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.213 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.213 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:31.213 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.399422+0000", 2023-09-23T20:06:31.213 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.27561206399999999 2023-09-23T20:06:31.213 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.213 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.213 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:06:31.213 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.400595+0000", 2023-09-23T20:06:31.213 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0011730919999999999 2023-09-23T20:06:31.213 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.214 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.214 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:06:31.214 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.400700+0000", 2023-09-23T20:06:31.214 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00010498200000000001 2023-09-23T20:06:31.214 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:06:31.214 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:06:31.214 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:06:31.214 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.214 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.214 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46877.0:40017 14.0 14:0e6089df:::benchmark_data_smithi016_228083_object40016:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-09-23T20:06:31.214 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:57:38.114110+0000", 2023-09-23T20:06:31.214 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 533.05888687000004, 2023-09-23T20:06:31.215 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.288656941, 2023-09-23T20:06:31.215 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:06:31.215 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:31.215 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:06:31.215 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46877", 2023-09-23T20:06:31.215 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:06:31.215 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 40017 2023-09-23T20:06:31.215 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.215 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:06:31.215 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.215 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:06:31.215 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.114110+0000", 2023-09-23T20:06:31.215 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:06:31.216 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.216 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.216 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:06:31.216 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.114110+0000", 2023-09-23T20:06:31.216 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:06:31.216 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.216 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.216 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:06:31.216 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.114110+0000", 2023-09-23T20:06:31.216 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T20:06:31.216 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.216 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.216 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:06:31.217 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.114128+0000", 2023-09-23T20:06:31.217 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.8070000000000001e-05 2023-09-23T20:06:31.217 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.217 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.217 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:06:31.217 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.114137+0000", 2023-09-23T20:06:31.217 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.8990000000000007e-06 2023-09-23T20:06:31.217 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.217 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.217 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:06:31.217 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.114142+0000", 2023-09-23T20:06:31.217 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.1699999999999996e-06 2023-09-23T20:06:31.217 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.217 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.218 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:06:31.218 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.114347+0000", 2023-09-23T20:06:31.218 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000205489 2023-09-23T20:06:31.218 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.218 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.218 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:06:31.218 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.114448+0000", 2023-09-23T20:06:31.218 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00010063700000000001 2023-09-23T20:06:31.218 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.218 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.218 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:06:31.218 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.114556+0000", 2023-09-23T20:06:31.218 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000108142 2023-09-23T20:06:31.219 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.219 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.219 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:31.219 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.118492+0000", 2023-09-23T20:06:31.219 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0039354079999999996 2023-09-23T20:06:31.219 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.219 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.219 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T20:06:31.219 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.119512+0000", 2023-09-23T20:06:31.219 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001020749 2023-09-23T20:06:31.219 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.219 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.220 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:31.220 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.402604+0000", 2023-09-23T20:06:31.220 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.28309172700000002 2023-09-23T20:06:31.220 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.220 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.220 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:06:31.220 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.402722+0000", 2023-09-23T20:06:31.220 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00011809199999999999 2023-09-23T20:06:31.220 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.220 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.220 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:06:31.220 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.402767+0000", 2023-09-23T20:06:31.220 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.507e-05 2023-09-23T20:06:31.221 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:06:31.221 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:06:31.221 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:06:31.221 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.221 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.221 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46877.0:40021 14.0 14:0fc25077:::benchmark_data_smithi016_228083_object40020:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-09-23T20:06:31.221 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:57:38.116613+0000", 2023-09-23T20:06:31.221 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 533.05638402399995, 2023-09-23T20:06:31.221 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.29051939300000001, 2023-09-23T20:06:31.221 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:06:31.221 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:31.221 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:06:31.221 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46877", 2023-09-23T20:06:31.221 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:06:31.222 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 40021 2023-09-23T20:06:31.222 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.222 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:06:31.222 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.222 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:06:31.222 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.116613+0000", 2023-09-23T20:06:31.222 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:06:31.222 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.222 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.222 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:06:31.222 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.116613+0000", 2023-09-23T20:06:31.222 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:06:31.222 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.223 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.223 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:06:31.223 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.116612+0000", 2023-09-23T20:06:31.223 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T20:06:31.223 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.223 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.223 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:06:31.223 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.116631+0000", 2023-09-23T20:06:31.223 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.8822e-05 2023-09-23T20:06:31.223 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.223 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.223 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:06:31.223 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.116641+0000", 2023-09-23T20:06:31.224 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.9669999999999996e-06 2023-09-23T20:06:31.224 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.224 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.224 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:06:31.224 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.116646+0000", 2023-09-23T20:06:31.224 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.2240000000000003e-06 2023-09-23T20:06:31.224 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.224 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.224 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:06:31.224 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.116679+0000", 2023-09-23T20:06:31.224 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.3370000000000001e-05 2023-09-23T20:06:31.224 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.224 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.225 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:06:31.225 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.116795+0000", 2023-09-23T20:06:31.225 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00011593499999999999 2023-09-23T20:06:31.225 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.225 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.225 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:06:31.225 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.116949+0000", 2023-09-23T20:06:31.225 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000153957 2023-09-23T20:06:31.225 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.225 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.225 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T20:06:31.225 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.119519+0000", 2023-09-23T20:06:31.225 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.002569235 2023-09-23T20:06:31.226 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.226 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.226 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:31.226 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.121727+0000", 2023-09-23T20:06:31.226 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.002207966 2023-09-23T20:06:31.226 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.226 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.226 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:31.226 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.406960+0000", 2023-09-23T20:06:31.226 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.28523343899999998 2023-09-23T20:06:31.226 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.226 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.226 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:06:31.226 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.407051+0000", 2023-09-23T20:06:31.227 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.1028999999999999e-05 2023-09-23T20:06:31.227 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.227 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.227 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:06:31.227 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.407132+0000", 2023-09-23T20:06:31.227 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.1416999999999997e-05 2023-09-23T20:06:31.227 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:06:31.227 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:06:31.227 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:06:31.227 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.227 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.227 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46877.0:40030 14.6 14:68474c9b:::benchmark_data_smithi016_228083_object40029:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-09-23T20:06:31.228 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:57:38.124278+0000", 2023-09-23T20:06:31.228 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 533.04871951500002, 2023-09-23T20:06:31.228 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.27846138500000001, 2023-09-23T20:06:31.228 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:06:31.228 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:31.228 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:06:31.228 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46877", 2023-09-23T20:06:31.228 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:06:31.228 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 40030 2023-09-23T20:06:31.228 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.228 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:06:31.228 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.228 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:06:31.229 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.124278+0000", 2023-09-23T20:06:31.229 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:06:31.229 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.229 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.229 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:06:31.229 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.124278+0000", 2023-09-23T20:06:31.229 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:06:31.229 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.229 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.229 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:06:31.229 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.124276+0000", 2023-09-23T20:06:31.229 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-09-23T20:06:31.230 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.230 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.230 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:06:31.230 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.124296+0000", 2023-09-23T20:06:31.230 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.9704e-05 2023-09-23T20:06:31.230 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.230 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.230 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:06:31.230 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.124307+0000", 2023-09-23T20:06:31.230 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1165000000000001e-05 2023-09-23T20:06:31.230 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.230 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.230 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:06:31.231 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.124313+0000", 2023-09-23T20:06:31.231 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.7520000000000002e-06 2023-09-23T20:06:31.231 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.231 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.231 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:06:31.231 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.124351+0000", 2023-09-23T20:06:31.231 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.8275000000000002e-05 2023-09-23T20:06:31.231 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.231 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.231 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:06:31.231 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.124463+0000", 2023-09-23T20:06:31.231 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000111705 2023-09-23T20:06:31.232 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.232 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.232 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:06:31.232 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.124596+0000", 2023-09-23T20:06:31.232 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00013313100000000001 2023-09-23T20:06:31.232 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.232 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.232 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:31.232 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.126647+0000", 2023-09-23T20:06:31.232 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0020512400000000002 2023-09-23T20:06:31.232 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.232 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.232 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T20:06:31.232 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.126872+0000", 2023-09-23T20:06:31.233 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00022435599999999999 2023-09-23T20:06:31.233 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.233 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.233 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:31.233 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.402609+0000", 2023-09-23T20:06:31.233 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.27573695199999998 2023-09-23T20:06:31.233 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.233 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.233 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:06:31.233 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.402689+0000", 2023-09-23T20:06:31.233 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 8.0005000000000004e-05 2023-09-23T20:06:31.233 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.233 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.234 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:06:31.234 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.402739+0000", 2023-09-23T20:06:31.234 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.0300999999999998e-05 2023-09-23T20:06:31.234 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:06:31.234 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:06:31.234 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:06:31.234 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.234 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.234 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46877.0:40033 14.0 14:032ca6e2:::benchmark_data_smithi016_228083_object40032:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-09-23T20:06:31.234 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:57:38.124324+0000", 2023-09-23T20:06:31.234 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 533.04867295199995, 2023-09-23T20:06:31.234 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.28303245999999999, 2023-09-23T20:06:31.235 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:06:31.235 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:31.235 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:06:31.235 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46877", 2023-09-23T20:06:31.235 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:06:31.235 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 40033 2023-09-23T20:06:31.235 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.235 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:06:31.235 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.235 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:06:31.235 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.124324+0000", 2023-09-23T20:06:31.235 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:06:31.236 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.236 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.236 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:06:31.236 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.124324+0000", 2023-09-23T20:06:31.236 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:06:31.236 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.236 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.236 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:06:31.236 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.124324+0000", 2023-09-23T20:06:31.236 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T20:06:31.236 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.237 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.237 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:06:31.237 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.124338+0000", 2023-09-23T20:06:31.237 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.4616e-05 2023-09-23T20:06:31.237 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.237 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.237 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:06:31.237 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.124343+0000", 2023-09-23T20:06:31.237 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.6190000000000002e-06 2023-09-23T20:06:31.237 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.237 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.237 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:06:31.238 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.124346+0000", 2023-09-23T20:06:31.238 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.1020000000000001e-06 2023-09-23T20:06:31.238 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.238 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.238 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:06:31.238 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.124394+0000", 2023-09-23T20:06:31.238 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.7948999999999998e-05 2023-09-23T20:06:31.238 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.238 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.238 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:06:31.238 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.124531+0000", 2023-09-23T20:06:31.238 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000136599 2023-09-23T20:06:31.238 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.239 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.239 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:06:31.239 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.124706+0000", 2023-09-23T20:06:31.239 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00017504599999999999 2023-09-23T20:06:31.239 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.239 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.239 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T20:06:31.239 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.126877+0000", 2023-09-23T20:06:31.239 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0021708460000000001 2023-09-23T20:06:31.239 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.239 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.239 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:31.239 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.127241+0000", 2023-09-23T20:06:31.240 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00036410499999999998 2023-09-23T20:06:31.240 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.240 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.240 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:31.240 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.407205+0000", 2023-09-23T20:06:31.240 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.27996414600000002 2023-09-23T20:06:31.240 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.240 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.240 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:06:31.240 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.407284+0000", 2023-09-23T20:06:31.240 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.9512999999999996e-05 2023-09-23T20:06:31.240 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.240 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.241 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:06:31.241 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.407357+0000", 2023-09-23T20:06:31.241 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.2268999999999995e-05 2023-09-23T20:06:31.241 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:06:31.241 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:06:31.241 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:06:31.241 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.241 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.241 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46877.0:40037 14.0 14:0778a3a7:::benchmark_data_smithi016_228083_object40036:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-09-23T20:06:31.241 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:57:38.127570+0000", 2023-09-23T20:06:31.241 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 533.04542684099999, 2023-09-23T20:06:31.241 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.28006694100000001, 2023-09-23T20:06:31.241 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:06:31.242 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:31.242 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:06:31.242 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46877", 2023-09-23T20:06:31.242 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:06:31.242 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 40037 2023-09-23T20:06:31.242 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.242 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:06:31.242 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.242 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:06:31.242 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.127570+0000", 2023-09-23T20:06:31.242 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:06:31.242 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.242 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.243 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:06:31.243 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.127570+0000", 2023-09-23T20:06:31.243 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:06:31.243 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.243 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.243 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:06:31.243 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.127569+0000", 2023-09-23T20:06:31.243 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T20:06:31.243 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.243 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.243 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:06:31.243 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.127588+0000", 2023-09-23T20:06:31.243 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.8281e-05 2023-09-23T20:06:31.244 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.244 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.244 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:06:31.244 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.127598+0000", 2023-09-23T20:06:31.244 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0287e-05 2023-09-23T20:06:31.244 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.244 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.244 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:06:31.244 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.127603+0000", 2023-09-23T20:06:31.244 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.2270000000000001e-06 2023-09-23T20:06:31.244 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.244 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.244 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:06:31.244 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.127648+0000", 2023-09-23T20:06:31.245 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.4895000000000003e-05 2023-09-23T20:06:31.245 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.245 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.245 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:06:31.245 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.127775+0000", 2023-09-23T20:06:31.245 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00012703500000000001 2023-09-23T20:06:31.245 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.245 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.245 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:06:31.245 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.127905+0000", 2023-09-23T20:06:31.245 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00013024799999999999 2023-09-23T20:06:31.245 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.245 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.246 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T20:06:31.246 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.130152+0000", 2023-09-23T20:06:31.246 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.002246287 2023-09-23T20:06:31.246 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.246 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.246 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:31.246 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.130634+0000", 2023-09-23T20:06:31.246 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00048192100000000001 2023-09-23T20:06:31.246 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.246 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.246 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:31.246 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.407450+0000", 2023-09-23T20:06:31.247 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.27681593300000001 2023-09-23T20:06:31.247 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.247 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.247 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:06:31.247 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.407569+0000", 2023-09-23T20:06:31.247 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000119296 2023-09-23T20:06:31.247 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.247 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.247 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:06:31.247 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.407637+0000", 2023-09-23T20:06:31.247 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.8400999999999998e-05 2023-09-23T20:06:31.247 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:06:31.247 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:06:31.247 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:06:31.248 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.248 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.248 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46877.0:40039 14.0 14:08986dac:::benchmark_data_smithi016_228083_object40038:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-09-23T20:06:31.248 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:57:38.127799+0000", 2023-09-23T20:06:31.248 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 533.04519782299997, 2023-09-23T20:06:31.248 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.28031159999999999, 2023-09-23T20:06:31.248 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:06:31.248 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:31.248 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:06:31.248 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46877", 2023-09-23T20:06:31.248 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:06:31.248 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 40039 2023-09-23T20:06:31.249 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.249 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:06:31.249 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.249 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:06:31.249 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.127799+0000", 2023-09-23T20:06:31.249 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:06:31.249 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.249 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.249 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:06:31.249 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.127799+0000", 2023-09-23T20:06:31.249 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:06:31.249 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.249 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.250 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:06:31.250 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.127798+0000", 2023-09-23T20:06:31.250 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T20:06:31.250 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.250 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.250 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:06:31.250 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.127819+0000", 2023-09-23T20:06:31.250 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.0835999999999998e-05 2023-09-23T20:06:31.250 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.250 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.250 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:06:31.250 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.127829+0000", 2023-09-23T20:06:31.250 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.6010000000000001e-06 2023-09-23T20:06:31.251 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.251 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.251 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:06:31.251 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.127836+0000", 2023-09-23T20:06:31.251 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.1720000000000001e-06 2023-09-23T20:06:31.251 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.251 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.251 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:06:31.251 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.128976+0000", 2023-09-23T20:06:31.251 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001139506 2023-09-23T20:06:31.251 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.251 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.251 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:06:31.252 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.129079+0000", 2023-09-23T20:06:31.252 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000103671 2023-09-23T20:06:31.252 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.252 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.252 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:06:31.252 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.129279+0000", 2023-09-23T20:06:31.252 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00019996599999999999 2023-09-23T20:06:31.252 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.252 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.252 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:31.252 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.130785+0000", 2023-09-23T20:06:31.252 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001505592 2023-09-23T20:06:31.252 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.253 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.253 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T20:06:31.253 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.131642+0000", 2023-09-23T20:06:31.253 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00085697099999999999 2023-09-23T20:06:31.253 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.253 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.253 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:31.253 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.408035+0000", 2023-09-23T20:06:31.253 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.27639299299999998 2023-09-23T20:06:31.253 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.253 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.253 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:06:31.253 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.408077+0000", 2023-09-23T20:06:31.254 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.2262000000000003e-05 2023-09-23T20:06:31.254 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.254 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.254 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:06:31.254 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.408111+0000", 2023-09-23T20:06:31.254 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.3852e-05 2023-09-23T20:06:31.254 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:06:31.254 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:06:31.254 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:06:31.254 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.254 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.254 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46877.0:40040 14.6 14:63d37f62:::benchmark_data_smithi016_228083_object40039:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-09-23T20:06:31.255 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:57:38.129143+0000", 2023-09-23T20:06:31.255 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 533.04385373399998, 2023-09-23T20:06:31.255 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.27450606799999999, 2023-09-23T20:06:31.255 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:06:31.255 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:31.255 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:06:31.255 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46877", 2023-09-23T20:06:31.255 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:06:31.255 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 40040 2023-09-23T20:06:31.255 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.255 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:06:31.255 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.256 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:06:31.256 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.129143+0000", 2023-09-23T20:06:31.256 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:06:31.256 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.256 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.256 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:06:31.256 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.129143+0000", 2023-09-23T20:06:31.256 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:06:31.256 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.256 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.256 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:06:31.256 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.129143+0000", 2023-09-23T20:06:31.257 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T20:06:31.257 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.257 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.257 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:06:31.257 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.129164+0000", 2023-09-23T20:06:31.257 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 2.1203999999999999e-05 2023-09-23T20:06:31.257 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.257 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.257 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:06:31.257 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.129172+0000", 2023-09-23T20:06:31.257 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.9899999999999997e-06 2023-09-23T20:06:31.257 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.257 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.258 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:06:31.258 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.129177+0000", 2023-09-23T20:06:31.258 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.0599999999999998e-06 2023-09-23T20:06:31.258 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.258 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.258 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:06:31.258 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.129220+0000", 2023-09-23T20:06:31.258 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.3312000000000001e-05 2023-09-23T20:06:31.258 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.258 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.258 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:06:31.258 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.129362+0000", 2023-09-23T20:06:31.258 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00014192299999999999 2023-09-23T20:06:31.259 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.259 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.259 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:06:31.259 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.129515+0000", 2023-09-23T20:06:31.259 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00015340800000000001 2023-09-23T20:06:31.259 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.259 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.259 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T20:06:31.259 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.131766+0000", 2023-09-23T20:06:31.259 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0022502799999999999 2023-09-23T20:06:31.259 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.259 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.259 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:31.260 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.132138+0000", 2023-09-23T20:06:31.260 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00037191600000000002 2023-09-23T20:06:31.260 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.260 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.260 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:31.260 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.403509+0000", 2023-09-23T20:06:31.260 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.27137124400000001 2023-09-23T20:06:31.260 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.260 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.260 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:06:31.260 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.403608+0000", 2023-09-23T20:06:31.260 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.8888000000000006e-05 2023-09-23T20:06:31.260 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.261 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.261 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:06:31.261 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.403649+0000", 2023-09-23T20:06:31.261 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.1666999999999999e-05 2023-09-23T20:06:31.261 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:06:31.261 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:06:31.261 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:06:31.261 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.261 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.261 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46877.0:40042 14.b 14:daff72e6:::benchmark_data_smithi016_228083_object40041:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-09-23T20:06:31.261 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:57:38.133693+0000", 2023-09-23T20:06:31.261 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 533.03930413600006, 2023-09-23T20:06:31.261 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.27004572799999998, 2023-09-23T20:06:31.262 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:06:31.262 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:31.262 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:06:31.262 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46877", 2023-09-23T20:06:31.262 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:06:31.262 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 40042 2023-09-23T20:06:31.262 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.262 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:06:31.262 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.262 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:06:31.262 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.133693+0000", 2023-09-23T20:06:31.262 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:06:31.262 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.263 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.263 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:06:31.263 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.133693+0000", 2023-09-23T20:06:31.263 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:06:31.263 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.263 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.263 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:06:31.263 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.133692+0000", 2023-09-23T20:06:31.263 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T20:06:31.263 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.263 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.263 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:06:31.263 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.133711+0000", 2023-09-23T20:06:31.264 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.8729e-05 2023-09-23T20:06:31.264 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.264 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.264 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:06:31.264 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.133721+0000", 2023-09-23T20:06:31.264 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.0477e-05 2023-09-23T20:06:31.264 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.264 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.264 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:06:31.264 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.133727+0000", 2023-09-23T20:06:31.264 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.3870000000000003e-06 2023-09-23T20:06:31.264 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.264 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.265 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:06:31.265 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.133775+0000", 2023-09-23T20:06:31.265 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.8306999999999997e-05 2023-09-23T20:06:31.265 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.265 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.265 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:06:31.265 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.133885+0000", 2023-09-23T20:06:31.265 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000109664 2023-09-23T20:06:31.265 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.265 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.265 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:06:31.265 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.133994+0000", 2023-09-23T20:06:31.265 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000109657 2023-09-23T20:06:31.265 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.266 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.266 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T20:06:31.266 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.135628+0000", 2023-09-23T20:06:31.266 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.001633448 2023-09-23T20:06:31.266 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.266 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.266 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:31.266 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.136476+0000", 2023-09-23T20:06:31.266 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00084808400000000003 2023-09-23T20:06:31.266 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.266 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.266 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:31.267 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.403529+0000", 2023-09-23T20:06:31.267 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.26705317699999997 2023-09-23T20:06:31.267 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.267 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.267 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:06:31.267 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.403660+0000", 2023-09-23T20:06:31.267 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00013137899999999999 2023-09-23T20:06:31.267 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.267 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.267 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:06:31.267 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:57:38.403739+0000", 2023-09-23T20:06:31.267 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.8256000000000004e-05 2023-09-23T20:06:31.267 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:06:31.268 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:06:31.268 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:06:31.268 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.268 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.268 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46877.0:95176 14.b 14:d73ec9cd:::benchmark_data_smithi016_228083_object95175:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e264)", 2023-09-23T20:06:31.268 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T19:58:38.226724+0000", 2023-09-23T20:06:31.268 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 472.94627266499998, 2023-09-23T20:06:31.268 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.24468284900000001, 2023-09-23T20:06:31.268 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:06:31.268 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:31.268 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:06:31.268 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46877", 2023-09-23T20:06:31.268 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:06:31.268 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 95176 2023-09-23T20:06:31.269 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.269 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:06:31.269 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.269 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:06:31.269 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:38.226724+0000", 2023-09-23T20:06:31.269 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:06:31.269 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.269 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.269 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:06:31.269 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:38.226724+0000", 2023-09-23T20:06:31.269 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:06:31.269 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.269 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.270 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:06:31.270 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:38.226724+0000", 2023-09-23T20:06:31.270 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T20:06:31.270 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.270 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.270 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:06:31.270 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:38.226744+0000", 2023-09-23T20:06:31.270 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.9792e-05 2023-09-23T20:06:31.270 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.270 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.270 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:06:31.270 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:38.226754+0000", 2023-09-23T20:06:31.270 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.8889999999999997e-06 2023-09-23T20:06:31.271 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.271 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.271 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:06:31.271 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:38.226758+0000", 2023-09-23T20:06:31.271 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.4909999999999999e-06 2023-09-23T20:06:31.271 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.271 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.271 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:06:31.271 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:38.226789+0000", 2023-09-23T20:06:31.271 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.0916e-05 2023-09-23T20:06:31.271 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.271 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.271 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:06:31.272 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:38.226891+0000", 2023-09-23T20:06:31.272 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000101889 2023-09-23T20:06:31.272 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.272 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.272 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:06:31.272 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:38.227068+0000", 2023-09-23T20:06:31.272 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000176634 2023-09-23T20:06:31.272 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.272 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.272 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T20:06:31.272 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:38.230180+0000", 2023-09-23T20:06:31.272 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0031123510000000002 2023-09-23T20:06:31.272 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.272 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.273 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:31.273 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:38.237464+0000", 2023-09-23T20:06:31.273 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.007284095 2023-09-23T20:06:31.273 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.273 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.273 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:31.273 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:38.471223+0000", 2023-09-23T20:06:31.273 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.23375892300000001 2023-09-23T20:06:31.273 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.273 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.273 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:06:31.273 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:38.471277+0000", 2023-09-23T20:06:31.273 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.4287999999999999e-05 2023-09-23T20:06:31.274 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.274 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.274 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:06:31.274 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T19:58:38.471407+0000", 2023-09-23T20:06:31.274 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000130106 2023-09-23T20:06:31.274 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:06:31.274 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:06:31.274 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:06:31.274 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.274 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.274 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46877.0:177658 14.b 14:db76f464:::benchmark_data_smithi016_228083_object77699:head [delete] snapc 0=[] ondisk+write+known_if_redirected e271)", 2023-09-23T20:06:31.274 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T20:00:41.475707+0000", 2023-09-23T20:06:31.275 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 349.69729059500003, 2023-09-23T20:06:31.275 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.21980476500000001, 2023-09-23T20:06:31.275 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:06:31.275 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:31.275 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:06:31.275 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46877", 2023-09-23T20:06:31.275 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:06:31.275 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 177658 2023-09-23T20:06:31.275 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.275 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:06:31.275 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.275 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:06:31.275 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.475707+0000", 2023-09-23T20:06:31.276 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:06:31.276 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.276 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.276 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:06:31.276 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.475707+0000", 2023-09-23T20:06:31.276 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:06:31.276 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.276 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.276 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:06:31.276 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.475705+0000", 2023-09-23T20:06:31.276 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-09-23T20:06:31.276 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.276 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.277 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:06:31.277 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.475711+0000", 2023-09-23T20:06:31.277 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.1550000000000004e-06 2023-09-23T20:06:31.277 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.277 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.277 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:06:31.277 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.475723+0000", 2023-09-23T20:06:31.277 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2157999999999999e-05 2023-09-23T20:06:31.277 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.277 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.277 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:06:31.277 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.475731+0000", 2023-09-23T20:06:31.277 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.8210000000000002e-06 2023-09-23T20:06:31.278 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.278 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.278 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:06:31.278 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.475787+0000", 2023-09-23T20:06:31.278 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.5390999999999999e-05 2023-09-23T20:06:31.278 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.278 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.278 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:06:31.278 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.476129+0000", 2023-09-23T20:06:31.278 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00034215 2023-09-23T20:06:31.278 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.278 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.278 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:06:31.279 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.476259+0000", 2023-09-23T20:06:31.279 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00013072800000000001 2023-09-23T20:06:31.279 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.279 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.279 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T20:06:31.279 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.487878+0000", 2023-09-23T20:06:31.279 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.011618696 2023-09-23T20:06:31.279 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.279 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.279 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:31.279 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.492295+0000", 2023-09-23T20:06:31.279 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0044171139999999998 2023-09-23T20:06:31.280 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.280 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.280 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:31.280 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.695238+0000", 2023-09-23T20:06:31.280 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.20294247600000001 2023-09-23T20:06:31.280 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.280 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.280 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:06:31.280 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.695372+0000", 2023-09-23T20:06:31.280 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000134356 2023-09-23T20:06:31.280 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.280 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.280 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:06:31.281 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.695511+0000", 2023-09-23T20:06:31.281 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000139185 2023-09-23T20:06:31.281 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:06:31.281 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:06:31.281 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:06:31.281 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.281 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.281 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46877.0:177660 14.b 14:dcbe9ec1:::benchmark_data_smithi016_228083_object77701:head [delete] snapc 0=[] ondisk+write+known_if_redirected e271)", 2023-09-23T20:06:31.281 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T20:00:41.480389+0000", 2023-09-23T20:06:31.281 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 349.69260791300002, 2023-09-23T20:06:31.281 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.21636045000000001, 2023-09-23T20:06:31.281 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:06:31.281 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:31.282 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:06:31.282 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46877", 2023-09-23T20:06:31.282 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:06:31.282 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 177660 2023-09-23T20:06:31.282 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.282 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:06:31.282 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.282 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:06:31.282 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.480389+0000", 2023-09-23T20:06:31.282 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:06:31.282 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.282 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.282 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:06:31.282 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.480389+0000", 2023-09-23T20:06:31.283 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:06:31.283 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.283 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.283 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:06:31.283 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.480388+0000", 2023-09-23T20:06:31.283 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-09-23T20:06:31.283 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.283 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.283 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:06:31.283 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.480394+0000", 2023-09-23T20:06:31.283 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.5300000000000002e-06 2023-09-23T20:06:31.283 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.283 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.284 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:06:31.284 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.480406+0000", 2023-09-23T20:06:31.284 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.1858000000000001e-05 2023-09-23T20:06:31.284 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.284 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.284 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:06:31.284 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.480413+0000", 2023-09-23T20:06:31.284 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.2629999999999997e-06 2023-09-23T20:06:31.284 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.284 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.284 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:06:31.284 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.481645+0000", 2023-09-23T20:06:31.284 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0012310299999999999 2023-09-23T20:06:31.285 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.285 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.285 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:06:31.285 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.481761+0000", 2023-09-23T20:06:31.285 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00011616399999999999 2023-09-23T20:06:31.285 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.285 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.285 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:06:31.285 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.481874+0000", 2023-09-23T20:06:31.285 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000113335 2023-09-23T20:06:31.285 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.285 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.285 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T20:06:31.286 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.487887+0000", 2023-09-23T20:06:31.286 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0060124879999999999 2023-09-23T20:06:31.286 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.286 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.286 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:31.286 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.492327+0000", 2023-09-23T20:06:31.286 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0044407609999999997 2023-09-23T20:06:31.286 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.286 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.286 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:31.286 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.695899+0000", 2023-09-23T20:06:31.287 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.203571997 2023-09-23T20:06:31.287 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.287 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.287 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:06:31.287 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.696596+0000", 2023-09-23T20:06:31.287 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00069673800000000002 2023-09-23T20:06:31.287 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.287 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.287 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:06:31.287 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.696750+0000", 2023-09-23T20:06:31.287 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000153638 2023-09-23T20:06:31.287 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:06:31.287 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:06:31.287 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:06:31.288 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.288 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.288 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46877.0:177674 14.b 14:d1b71408:::benchmark_data_smithi016_228083_object77715:head [delete] snapc 0=[] ondisk+write+known_if_redirected e271)", 2023-09-23T20:06:31.288 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T20:00:41.494991+0000", 2023-09-23T20:06:31.288 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 349.678006262, 2023-09-23T20:06:31.288 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.21355988100000001, 2023-09-23T20:06:31.288 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:06:31.288 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:31.288 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:06:31.288 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46877", 2023-09-23T20:06:31.288 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:06:31.288 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 177674 2023-09-23T20:06:31.289 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.289 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:06:31.289 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.289 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:06:31.289 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.494991+0000", 2023-09-23T20:06:31.289 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:06:31.289 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.289 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.289 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:06:31.289 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.494991+0000", 2023-09-23T20:06:31.289 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:06:31.289 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.289 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.290 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:06:31.290 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.494989+0000", 2023-09-23T20:06:31.290 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999986 2023-09-23T20:06:31.290 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.290 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.290 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:06:31.290 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.494997+0000", 2023-09-23T20:06:31.290 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.4309999999999997e-06 2023-09-23T20:06:31.290 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.290 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.290 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:06:31.290 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.495010+0000", 2023-09-23T20:06:31.291 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 1.2931e-05 2023-09-23T20:06:31.291 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.291 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.291 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:06:31.291 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.495017+0000", 2023-09-23T20:06:31.291 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.8769999999999993e-06 2023-09-23T20:06:31.291 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.291 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.291 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:06:31.291 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.495067+0000", 2023-09-23T20:06:31.291 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.9508000000000001e-05 2023-09-23T20:06:31.291 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.291 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.292 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:06:31.292 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.495190+0000", 2023-09-23T20:06:31.292 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00012350600000000001 2023-09-23T20:06:31.292 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.292 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.292 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:06:31.292 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.495310+0000", 2023-09-23T20:06:31.292 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000119184 2023-09-23T20:06:31.292 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.292 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.292 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T20:06:31.292 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.520552+0000", 2023-09-23T20:06:31.292 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.025242446000000002 2023-09-23T20:06:31.292 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.293 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.293 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:31.293 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.696923+0000", 2023-09-23T20:06:31.293 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.17637043999999999 2023-09-23T20:06:31.293 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.293 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.293 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:31.293 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.708245+0000", 2023-09-23T20:06:31.293 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.011322446 2023-09-23T20:06:31.293 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.293 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.293 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:06:31.294 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.708389+0000", 2023-09-23T20:06:31.294 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000144196 2023-09-23T20:06:31.294 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.294 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.294 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:06:31.294 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.708551+0000", 2023-09-23T20:06:31.294 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000161514 2023-09-23T20:06:31.294 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:06:31.294 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:06:31.294 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:06:31.294 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.294 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.294 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.46877.0:177676 14.b 14:d6acdb3e:::benchmark_data_smithi016_228083_object77717:head [delete] snapc 0=[] ondisk+write+known_if_redirected e271)", 2023-09-23T20:06:31.295 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T20:00:41.496202+0000", 2023-09-23T20:06:31.295 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 349.676795239, 2023-09-23T20:06:31.295 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.212828088, 2023-09-23T20:06:31.295 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:06:31.295 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:31.295 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:06:31.295 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.46877", 2023-09-23T20:06:31.295 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/3443127126", 2023-09-23T20:06:31.295 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 177676 2023-09-23T20:06:31.295 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.295 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:06:31.295 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.296 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:06:31.296 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.496202+0000", 2023-09-23T20:06:31.296 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:06:31.296 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.296 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.296 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:06:31.296 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.496202+0000", 2023-09-23T20:06:31.296 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:06:31.296 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.296 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.296 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:06:31.296 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.496201+0000", 2023-09-23T20:06:31.296 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.999999 2023-09-23T20:06:31.296 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.297 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.297 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:06:31.297 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.496207+0000", 2023-09-23T20:06:31.297 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 5.8409999999999996e-06 2023-09-23T20:06:31.297 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.297 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.297 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:06:31.297 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.496217+0000", 2023-09-23T20:06:31.297 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.8940000000000005e-06 2023-09-23T20:06:31.297 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.297 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.297 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:06:31.298 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.496224+0000", 2023-09-23T20:06:31.298 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.0119999999999999e-06 2023-09-23T20:06:31.298 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.298 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.298 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:06:31.298 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.504974+0000", 2023-09-23T20:06:31.298 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.0087504910000000009 2023-09-23T20:06:31.298 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.298 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.298 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:06:31.298 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.505095+0000", 2023-09-23T20:06:31.298 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000121153 2023-09-23T20:06:31.299 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.299 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.299 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:06:31.299 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.505213+0000", 2023-09-23T20:06:31.299 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000117905 2023-09-23T20:06:31.299 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.299 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.299 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "op_commit", 2023-09-23T20:06:31.299 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.521204+0000", 2023-09-23T20:06:31.299 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.015990652000000001 2023-09-23T20:06:31.299 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.299 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.299 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:31.300 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.697071+0000", 2023-09-23T20:06:31.300 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.17586681100000001 2023-09-23T20:06:31.300 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.300 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.300 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:31.300 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.708840+0000", 2023-09-23T20:06:31.300 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.011769595000000001 2023-09-23T20:06:31.300 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.300 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.300 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "commit_sent", 2023-09-23T20:06:31.300 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.708964+0000", 2023-09-23T20:06:31.300 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.000123342 2023-09-23T20:06:31.300 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.301 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.301 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:06:31.301 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:00:41.709030+0000", 2023-09-23T20:06:31.301 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.6239000000000001e-05 2023-09-23T20:06:31.301 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:06:31.301 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:06:31.301 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:06:31.301 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.301 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.301 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.37455.0:20 2.1 2:9a5f4aab:::rbd_header.92133bbd078e:head [notify-ack] snapc 0=[] ondisk+read+known_if_redirected+full_try e293)", 2023-09-23T20:06:31.301 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T20:02:43.223653+0000", 2023-09-23T20:06:31.301 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 227.94934399499999, 2023-09-23T20:06:31.302 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.069929127999999993, 2023-09-23T20:06:31.302 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:06:31.302 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "started", 2023-09-23T20:06:31.302 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:06:31.302 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.37455", 2023-09-23T20:06:31.302 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/3691728143", 2023-09-23T20:06:31.302 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 20 2023-09-23T20:06:31.302 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.302 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:06:31.302 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.302 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:06:31.302 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:43.223653+0000", 2023-09-23T20:06:31.302 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:06:31.303 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.303 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.303 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:06:31.303 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:43.223653+0000", 2023-09-23T20:06:31.303 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:06:31.303 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.303 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.303 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:06:31.303 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:43.223652+0000", 2023-09-23T20:06:31.303 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T20:06:31.303 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.303 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.303 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:06:31.304 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:43.223656+0000", 2023-09-23T20:06:31.304 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.574e-06 2023-09-23T20:06:31.304 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.304 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.304 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:06:31.304 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:43.223664+0000", 2023-09-23T20:06:31.304 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 7.8820000000000001e-06 2023-09-23T20:06:31.304 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.304 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.304 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:06:31.304 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:43.223668+0000", 2023-09-23T20:06:31.304 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 3.9920000000000001e-06 2023-09-23T20:06:31.304 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.305 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.305 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:06:31.305 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:43.224556+0000", 2023-09-23T20:06:31.305 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00088783099999999995 2023-09-23T20:06:31.305 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.305 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.305 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "waiting for rw locks", 2023-09-23T20:06:31.305 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:43.224601+0000", 2023-09-23T20:06:31.305 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.5726999999999998e-05 2023-09-23T20:06:31.305 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.305 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.305 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:06:31.306 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:43.224906+0000", 2023-09-23T20:06:31.306 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.00030472300000000002 2023-09-23T20:06:31.306 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.306 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.306 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:06:31.306 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:43.224971+0000", 2023-09-23T20:06:31.306 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.4558000000000001e-05 2023-09-23T20:06:31.306 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.306 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.306 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:06:31.306 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:43.293582+0000", 2023-09-23T20:06:31.306 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.068611501000000005 2023-09-23T20:06:31.306 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:06:31.307 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:06:31.307 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:06:31.307 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.307 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.307 INFO:teuthology.orchestra.run.smithi016.stdout: "description": "osd_op(client.37455.0:21 2.1 2:9a5f4aab:::rbd_header.92133bbd078e:head [call rbd.snapshot_get in=15b] snapc 0=[] ondisk+read+known_if_redirected+full_try e293)", 2023-09-23T20:06:31.307 INFO:teuthology.orchestra.run.smithi016.stdout: "initiated_at": "2023-09-23T20:02:43.224953+0000", 2023-09-23T20:06:31.307 INFO:teuthology.orchestra.run.smithi016.stdout: "age": 227.948044012, 2023-09-23T20:06:31.307 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.068837161999999993, 2023-09-23T20:06:31.307 INFO:teuthology.orchestra.run.smithi016.stdout: "type_data": { 2023-09-23T20:06:31.307 INFO:teuthology.orchestra.run.smithi016.stdout: "flag_point": "started", 2023-09-23T20:06:31.307 INFO:teuthology.orchestra.run.smithi016.stdout: "client_info": { 2023-09-23T20:06:31.307 INFO:teuthology.orchestra.run.smithi016.stdout: "client": "client.37455", 2023-09-23T20:06:31.307 INFO:teuthology.orchestra.run.smithi016.stdout: "client_addr": "172.21.15.16:0/3691728143", 2023-09-23T20:06:31.308 INFO:teuthology.orchestra.run.smithi016.stdout: "tid": 21 2023-09-23T20:06:31.308 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.308 INFO:teuthology.orchestra.run.smithi016.stdout: "events": [ 2023-09-23T20:06:31.308 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.308 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "initiated", 2023-09-23T20:06:31.308 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:43.224953+0000", 2023-09-23T20:06:31.308 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:06:31.308 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.308 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.308 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "throttled", 2023-09-23T20:06:31.308 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:43.224953+0000", 2023-09-23T20:06:31.308 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0 2023-09-23T20:06:31.308 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.309 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.309 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "header_read", 2023-09-23T20:06:31.309 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:43.224952+0000", 2023-09-23T20:06:31.309 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4294967295.9999995 2023-09-23T20:06:31.309 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.309 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.309 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "all_read", 2023-09-23T20:06:31.309 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:43.224957+0000", 2023-09-23T20:06:31.309 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.3059999999999999e-06 2023-09-23T20:06:31.309 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.309 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.309 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "dispatched", 2023-09-23T20:06:31.310 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:43.224964+0000", 2023-09-23T20:06:31.310 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.9920000000000001e-06 2023-09-23T20:06:31.310 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.310 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.310 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "queued_for_pg", 2023-09-23T20:06:31.310 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:43.224968+0000", 2023-09-23T20:06:31.310 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 4.4580000000000003e-06 2023-09-23T20:06:31.310 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.310 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.310 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "reached_pg", 2023-09-23T20:06:31.310 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:43.293628+0000", 2023-09-23T20:06:31.310 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 0.068659355000000005 2023-09-23T20:06:31.310 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.311 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.311 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "started", 2023-09-23T20:06:31.311 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:43.293694+0000", 2023-09-23T20:06:31.311 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 6.6007999999999995e-05 2023-09-23T20:06:31.311 INFO:teuthology.orchestra.run.smithi016.stdout: }, 2023-09-23T20:06:31.311 INFO:teuthology.orchestra.run.smithi016.stdout: { 2023-09-23T20:06:31.311 INFO:teuthology.orchestra.run.smithi016.stdout: "event": "done", 2023-09-23T20:06:31.311 INFO:teuthology.orchestra.run.smithi016.stdout: "time": "2023-09-23T20:02:43.293790+0000", 2023-09-23T20:06:31.311 INFO:teuthology.orchestra.run.smithi016.stdout: "duration": 9.6741000000000001e-05 2023-09-23T20:06:31.311 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:06:31.311 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:06:31.311 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:06:31.311 INFO:teuthology.orchestra.run.smithi016.stdout: } 2023-09-23T20:06:31.312 INFO:teuthology.orchestra.run.smithi016.stdout: ] 2023-09-23T20:06:31.312 INFO:teuthology.orchestra.run.smithi016.stdout:} 2023-09-23T20:06:31.984 INFO:teuthology.orchestra.run.smithi016.stderr:set osd.2 primary-affinity to 1 (8655362) 2023-09-23T20:06:32.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:31 smithi016 bash[75004]: cluster 2023-09-23T20:06:30.772334+0000 mgr.y (mgr.25325) 1780 : cluster 0 pgmap v1873: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:32.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:31 smithi016 bash[75004]: audit 2023-09-23T20:06:31.390972+0000 mon.a (mon.0) 2153 : audit 1 from='client.? 172.21.15.16:0/2249604870' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:06:32.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:31 smithi016 bash[77995]: cluster 2023-09-23T20:06:30.772334+0000 mgr.y (mgr.25325) 1780 : cluster 0 pgmap v1873: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:32.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:31 smithi016 bash[77995]: audit 2023-09-23T20:06:31.390972+0000 mon.a (mon.0) 2153 : audit 1 from='client.? 172.21.15.16:0/2249604870' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:06:32.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:31 smithi092 bash[58855]: cluster 2023-09-23T20:06:30.772334+0000 mgr.y (mgr.25325) 1780 : cluster 0 pgmap v1873: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:32.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:31 smithi092 bash[58855]: audit 2023-09-23T20:06:31.390972+0000 mon.a (mon.0) 2153 : audit 1 from='client.? 172.21.15.16:0/2249604870' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:06:32.891 INFO:tasks.thrashosds.thrasher:joining the do_sighup greenlet 2023-09-23T20:06:32.892 INFO:tasks.thrashosds.thrasher:joining the do_optrack_toggle greenlet 2023-09-23T20:06:32.892 INFO:tasks.thrashosds.thrasher:joining the do_dump_ops greenlet 2023-09-23T20:06:32.892 INFO:tasks.thrashosds.thrasher:joining the do_noscrub_toggle greenlet 2023-09-23T20:06:32.892 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-09-23T20:06:32.892 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T20:06:33.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:32 smithi016 bash[75004]: audit 2023-09-23T20:06:31.976669+0000 mon.a (mon.0) 2154 : audit 1 from='client.? 172.21.15.16:0/2249604870' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:06:33.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:32 smithi016 bash[75004]: cluster 2023-09-23T20:06:31.976752+0000 mon.a (mon.0) 2155 : cluster 0 osdmap e333: 8 total, 8 up, 8 in 2023-09-23T20:06:33.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:32 smithi016 bash[75004]: cluster 2023-09-23T20:06:32.247812+0000 osd.5 (osd.5) 254 : cluster 3 failed to encode map e333 with expected crc 2023-09-23T20:06:33.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:32 smithi016 bash[75004]: cluster 2023-09-23T20:06:32.250266+0000 osd.5 (osd.5) 255 : cluster 3 failed to encode map e333 with expected crc 2023-09-23T20:06:33.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:32 smithi016 bash[77995]: audit 2023-09-23T20:06:31.976669+0000 mon.a (mon.0) 2154 : audit 1 from='client.? 172.21.15.16:0/2249604870' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:06:33.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:32 smithi016 bash[77995]: cluster 2023-09-23T20:06:31.976752+0000 mon.a (mon.0) 2155 : cluster 0 osdmap e333: 8 total, 8 up, 8 in 2023-09-23T20:06:33.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:32 smithi016 bash[77995]: cluster 2023-09-23T20:06:32.247812+0000 osd.5 (osd.5) 254 : cluster 3 failed to encode map e333 with expected crc 2023-09-23T20:06:33.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:32 smithi016 bash[77995]: cluster 2023-09-23T20:06:32.250266+0000 osd.5 (osd.5) 255 : cluster 3 failed to encode map e333 with expected crc 2023-09-23T20:06:33.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:32 smithi092 bash[58855]: audit 2023-09-23T20:06:31.976669+0000 mon.a (mon.0) 2154 : audit 1 from='client.? 172.21.15.16:0/2249604870' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:06:33.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:32 smithi092 bash[58855]: cluster 2023-09-23T20:06:31.976752+0000 mon.a (mon.0) 2155 : cluster 0 osdmap e333: 8 total, 8 up, 8 in 2023-09-23T20:06:33.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:32 smithi092 bash[58855]: cluster 2023-09-23T20:06:32.247812+0000 osd.5 (osd.5) 254 : cluster 3 failed to encode map e333 with expected crc 2023-09-23T20:06:33.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:32 smithi092 bash[58855]: cluster 2023-09-23T20:06:32.250266+0000 osd.5 (osd.5) 255 : cluster 3 failed to encode map e333 with expected crc 2023-09-23T20:06:34.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:33 smithi016 bash[75004]: cluster 2023-09-23T20:06:32.245605+0000 osd.7 (osd.7) 252 : cluster 3 failed to encode map e333 with expected crc 2023-09-23T20:06:34.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:33 smithi016 bash[75004]: cluster 2023-09-23T20:06:32.249032+0000 osd.7 (osd.7) 253 : cluster 3 failed to encode map e333 with expected crc 2023-09-23T20:06:34.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:33 smithi016 bash[75004]: cluster 2023-09-23T20:06:32.772995+0000 mgr.y (mgr.25325) 1781 : cluster 0 pgmap v1875: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:34.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:33 smithi016 bash[75004]: cluster 2023-09-23T20:06:32.875810+0000 osd.4 (osd.4) 273 : cluster 3 failed to encode map e333 with expected crc 2023-09-23T20:06:34.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:33 smithi016 bash[75004]: cluster 2023-09-23T20:06:32.878503+0000 osd.4 (osd.4) 274 : cluster 3 failed to encode map e333 with expected crc 2023-09-23T20:06:34.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:34 smithi016 bash[75004]: cluster 2023-09-23T20:06:33.195048+0000 osd.6 (osd.6) 272 : cluster 3 failed to encode map e333 with expected crc 2023-09-23T20:06:34.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:34 smithi016 bash[77995]: cluster 2023-09-23T20:06:32.245605+0000 osd.7 (osd.7) 252 : cluster 3 failed to encode map e333 with expected crc 2023-09-23T20:06:34.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:34 smithi016 bash[77995]: cluster 2023-09-23T20:06:32.249032+0000 osd.7 (osd.7) 253 : cluster 3 failed to encode map e333 with expected crc 2023-09-23T20:06:34.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:34 smithi016 bash[77995]: cluster 2023-09-23T20:06:32.772995+0000 mgr.y (mgr.25325) 1781 : cluster 0 pgmap v1875: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:34.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:34 smithi016 bash[77995]: cluster 2023-09-23T20:06:32.875810+0000 osd.4 (osd.4) 273 : cluster 3 failed to encode map e333 with expected crc 2023-09-23T20:06:34.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:34 smithi016 bash[77995]: cluster 2023-09-23T20:06:32.878503+0000 osd.4 (osd.4) 274 : cluster 3 failed to encode map e333 with expected crc 2023-09-23T20:06:34.287 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:34 smithi016 bash[77995]: cluster 2023-09-23T20:06:33.195048+0000 osd.6 (osd.6) 272 : cluster 3 failed to encode map e333 with expected crc 2023-09-23T20:06:34.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:33 smithi092 bash[58855]: cluster 2023-09-23T20:06:32.245605+0000 osd.7 (osd.7) 252 : cluster 3 failed to encode map e333 with expected crc 2023-09-23T20:06:34.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:33 smithi092 bash[58855]: cluster 2023-09-23T20:06:32.249032+0000 osd.7 (osd.7) 253 : cluster 3 failed to encode map e333 with expected crc 2023-09-23T20:06:34.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:33 smithi092 bash[58855]: cluster 2023-09-23T20:06:32.772995+0000 mgr.y (mgr.25325) 1781 : cluster 0 pgmap v1875: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:34.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:33 smithi092 bash[58855]: cluster 2023-09-23T20:06:32.875810+0000 osd.4 (osd.4) 273 : cluster 3 failed to encode map e333 with expected crc 2023-09-23T20:06:34.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:33 smithi092 bash[58855]: cluster 2023-09-23T20:06:32.878503+0000 osd.4 (osd.4) 274 : cluster 3 failed to encode map e333 with expected crc 2023-09-23T20:06:34.289 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:33 smithi092 bash[58855]: cluster 2023-09-23T20:06:33.195048+0000 osd.6 (osd.6) 272 : cluster 3 failed to encode map e333 with expected crc 2023-09-23T20:06:35.991 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:06:35 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:20:06:35] "GET /metrics HTTP/1.1" 200 34011 "" "Prometheus/2.33.4" 2023-09-23T20:06:36.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:35 smithi016 bash[75004]: cluster 2023-09-23T20:06:34.773746+0000 mgr.y (mgr.25325) 1782 : cluster 0 pgmap v1876: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:36.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:35 smithi016 bash[77995]: cluster 2023-09-23T20:06:34.773746+0000 mgr.y (mgr.25325) 1782 : cluster 0 pgmap v1876: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:36.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:35 smithi092 bash[58855]: cluster 2023-09-23T20:06:34.773746+0000 mgr.y (mgr.25325) 1782 : cluster 0 pgmap v1876: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:37.666 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:06:38.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:37 smithi016 bash[75004]: cluster 2023-09-23T20:06:36.774637+0000 mgr.y (mgr.25325) 1783 : cluster 0 pgmap v1877: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:38.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:37 smithi016 bash[77995]: cluster 2023-09-23T20:06:36.774637+0000 mgr.y (mgr.25325) 1783 : cluster 0 pgmap v1877: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:38.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:37 smithi092 bash[58855]: cluster 2023-09-23T20:06:36.774637+0000 mgr.y (mgr.25325) 1783 : cluster 0 pgmap v1877: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:40.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:39 smithi016 bash[77995]: cluster 2023-09-23T20:06:38.775506+0000 mgr.y (mgr.25325) 1784 : cluster 0 pgmap v1878: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:40.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:39 smithi016 bash[75004]: cluster 2023-09-23T20:06:38.775506+0000 mgr.y (mgr.25325) 1784 : cluster 0 pgmap v1878: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:40.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:40 smithi092 bash[58855]: cluster 2023-09-23T20:06:38.775506+0000 mgr.y (mgr.25325) 1784 : cluster 0 pgmap v1878: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:41.490 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-09-23T20:06:41.490 INFO:teuthology.orchestra.run.smithi016.stdout:{"epoch":333,"fsid":"95ce9eb6-5a42-11ee-8db4-212e2dc638e7","created":"2023-09-23T18:55:23.802934+0000","modified":"2023-09-23T20:06:31.964654+0000","last_up_change":"2023-09-23T19:38:42.842616+0000","last_in_change":"2023-09-23T20:03:13.443789+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":23,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":15,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T18:59:33.315416+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{},"mgr_devicehealth":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T19:04:01.440427+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"297","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":297,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T19:05:50.982524+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T19:05:51.538373+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"a4e8ed09-1195-4878-928a-13881bc04dc7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":81,"up_from":85,"up_thru":316,"down_at":82,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6802","nonce":3034120466},{"type":"v1","addr":"172.21.15.16:6803","nonce":3034120466}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6804","nonce":3034120466},{"type":"v1","addr":"172.21.15.16:6805","nonce":3034120466}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6808","nonce":3034120466},{"type":"v1","addr":"172.21.15.16:6809","nonce":3034120466}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6806","nonce":3034120466},{"type":"v1","addr":"172.21.15.16:6807","nonce":3034120466}]},"public_addr":"172.21.15.16:6803/3034120466","cluster_addr":"172.21.15.16:6805/3034120466","heartbeat_back_addr":"172.21.15.16:6809/3034120466","heartbeat_front_addr":"172.21.15.16:6807/3034120466","state":["exists","up"]},{"osd":1,"uuid":"c5ff2879-0d1c-481a-b847-248433c12aa4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":93,"up_from":98,"up_thru":309,"down_at":94,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6810","nonce":3881357617},{"type":"v1","addr":"172.21.15.16:6811","nonce":3881357617}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6812","nonce":3881357617},{"type":"v1","addr":"172.21.15.16:6813","nonce":3881357617}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6816","nonce":3881357617},{"type":"v1","addr":"172.21.15.16:6817","nonce":3881357617}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6814","nonce":3881357617},{"type":"v1","addr":"172.21.15.16:6815","nonce":3881357617}]},"public_addr":"172.21.15.16:6811/3881357617","cluster_addr":"172.21.15.16:6813/3881357617","heartbeat_back_addr":"172.21.15.16:6817/3881357617","heartbeat_front_addr":"172.21.15.16:6815/3881357617","state":["exists","up"]},{"osd":2,"uuid":"7974ce4d-9ed8-4a72-b44b-d500b34da8cf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":111,"last_clean_end":154,"up_from":159,"up_thru":313,"down_at":155,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6818","nonce":4244964590},{"type":"v1","addr":"172.21.15.16:6819","nonce":4244964590}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6820","nonce":4244964590},{"type":"v1","addr":"172.21.15.16:6821","nonce":4244964590}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6824","nonce":4244964590},{"type":"v1","addr":"172.21.15.16:6825","nonce":4244964590}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6822","nonce":4244964590},{"type":"v1","addr":"172.21.15.16:6823","nonce":4244964590}]},"public_addr":"172.21.15.16:6819/4244964590","cluster_addr":"172.21.15.16:6821/4244964590","heartbeat_back_addr":"172.21.15.16:6825/4244964590","heartbeat_front_addr":"172.21.15.16:6823/4244964590","state":["exists","up"]},{"osd":3,"uuid":"f5e0d57b-d629-4453-a527-ad77e6c210ad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":112,"up_from":116,"up_thru":321,"down_at":113,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6826","nonce":3155744043},{"type":"v1","addr":"172.21.15.16:6827","nonce":3155744043}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6828","nonce":3155744043},{"type":"v1","addr":"172.21.15.16:6829","nonce":3155744043}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6832","nonce":3155744043},{"type":"v1","addr":"172.21.15.16:6833","nonce":3155744043}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6830","nonce":3155744043},{"type":"v1","addr":"172.21.15.16:6831","nonce":3155744043}]},"public_addr":"172.21.15.16:6827/3155744043","cluster_addr":"172.21.15.16:6829/3155744043","heartbeat_back_addr":"172.21.15.16:6833/3155744043","heartbeat_front_addr":"172.21.15.16:6831/3155744043","state":["exists","up"]},{"osd":4,"uuid":"55a8dab2-6393-44ab-943e-cb7ab2cb7c31","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":313,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6800","nonce":99745912},{"type":"v1","addr":"172.21.15.92:6801","nonce":99745912}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6802","nonce":99745912},{"type":"v1","addr":"172.21.15.92:6803","nonce":99745912}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6806","nonce":99745912},{"type":"v1","addr":"172.21.15.92:6807","nonce":99745912}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6804","nonce":99745912},{"type":"v1","addr":"172.21.15.92:6805","nonce":99745912}]},"public_addr":"172.21.15.92:6801/99745912","cluster_addr":"172.21.15.92:6803/99745912","heartbeat_back_addr":"172.21.15.92:6807/99745912","heartbeat_front_addr":"172.21.15.92:6805/99745912","state":["exists","up"]},{"osd":5,"uuid":"b29b73b1-51d7-4484-bd79-2334c650cb62","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":326,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6808","nonce":1987214502},{"type":"v1","addr":"172.21.15.92:6809","nonce":1987214502}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6810","nonce":1987214502},{"type":"v1","addr":"172.21.15.92:6811","nonce":1987214502}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6814","nonce":1987214502},{"type":"v1","addr":"172.21.15.92:6815","nonce":1987214502}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6812","nonce":1987214502},{"type":"v1","addr":"172.21.15.92:6813","nonce":1987214502}]},"public_addr":"172.21.15.92:6809/1987214502","cluster_addr":"172.21.15.92:6811/1987214502","heartbeat_back_addr":"172.21.15.92:6815/1987214502","heartbeat_front_addr":"172.21.15.92:6813/1987214502","state":["exists","up"]},{"osd":6,"uuid":"9fe16cc9-f0d3-4b79-bc68-38305c55b551","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":306,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6816","nonce":2820798178},{"type":"v1","addr":"172.21.15.92:6817","nonce":2820798178}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6818","nonce":2820798178},{"type":"v1","addr":"172.21.15.92:6819","nonce":2820798178}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6822","nonce":2820798178},{"type":"v1","addr":"172.21.15.92:6823","nonce":2820798178}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6820","nonce":2820798178},{"type":"v1","addr":"172.21.15.92:6821","nonce":2820798178}]},"public_addr":"172.21.15.92:6817/2820798178","cluster_addr":"172.21.15.92:6819/2820798178","heartbeat_back_addr":"172.21.15.92:6823/2820798178","heartbeat_front_addr":"172.21.15.92:6821/2820798178","state":["exists","up"]},{"osd":7,"uuid":"d086b076-127f-42de-bee9-157bf579d92d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":306,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6824","nonce":98624761},{"type":"v1","addr":"172.21.15.92:6825","nonce":98624761}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6826","nonce":98624761},{"type":"v1","addr":"172.21.15.92:6827","nonce":98624761}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6830","nonce":98624761},{"type":"v1","addr":"172.21.15.92:6831","nonce":98624761}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6828","nonce":98624761},{"type":"v1","addr":"172.21.15.92:6829","nonce":98624761}]},"public_addr":"172.21.15.92:6825/98624761","cluster_addr":"172.21.15.92:6827/98624761","heartbeat_back_addr":"172.21.15.92:6831/98624761","heartbeat_front_addr":"172.21.15.92:6829/98624761","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:58:07.335980+0000","dead_epoch":81},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:58:49.342213+0000","dead_epoch":93},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:59:30.939986+0000","dead_epoch":154},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:00:12.926364+0000","dead_epoch":112},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:00:53.466692+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:01:36.332957+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:02:18.593854+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:03:00.817448+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.d","mappings":[{"from":1,"to":5}]},{"pgid":"4.17","mappings":[{"from":1,"to":2}]},{"pgid":"4.1d","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.16:6800/2756103674":"2023-09-24T18:56:20.574625+0000","172.21.15.16:0/2843459289":"2023-09-24T18:56:20.574625+0000","172.21.15.16:6801/2756103674":"2023-09-24T18:56:20.574625+0000","172.21.15.92:0/1841502917":"2023-09-24T19:11:32.247480+0000","172.21.15.16:6801/777573315":"2023-09-24T19:12:43.452315+0000","172.21.15.16:6800/777573315":"2023-09-24T19:12:43.452315+0000","172.21.15.16:6801/2615282850":"2023-09-24T18:55:50.550546+0000","172.21.15.16:6801/1259630240":"2023-09-24T19:03:34.846451+0000","172.21.15.16:6800/1259630240":"2023-09-24T19:03:34.846451+0000","172.21.15.16:0/79169563":"2023-09-24T18:55:50.550546+0000","172.21.15.16:0/3595835147":"2023-09-24T19:12:43.452315+0000","172.21.15.92:0/3173267635":"2023-09-24T19:11:32.247480+0000","172.21.15.16:0/3394254122":"2023-09-24T19:12:43.452315+0000","172.21.15.16:0/272242693":"2023-09-24T18:56:20.574625+0000","172.21.15.16:0/736761601":"2023-09-24T19:12:43.452315+0000","172.21.15.92:0/3338210669":"2023-09-24T19:11:32.247480+0000","172.21.15.92:6835/2057654371":"2023-09-24T19:11:32.247480+0000","172.21.15.16:0/3767844203":"2023-09-24T19:03:34.846451+0000","172.21.15.16:0/2334334390":"2023-09-24T19:12:43.452315+0000","172.21.15.16:0/4196794739":"2023-09-24T19:12:43.452315+0000","172.21.15.16:6800/1501185908":"2023-09-24T19:09:35.764967+0000","172.21.15.16:0/3765291224":"2023-09-24T19:09:35.764967+0000","172.21.15.16:6801/1501185908":"2023-09-24T19:09:35.764967+0000","172.21.15.16:0/647003663":"2023-09-24T19:12:43.452315+0000","172.21.15.16:0/2669353350":"2023-09-24T19:03:34.846451+0000","172.21.15.92:6834/2057654371":"2023-09-24T19:11:32.247480+0000","172.21.15.92:0/561988499":"2023-09-24T19:11:32.247480+0000","172.21.15.16:0/4152413938":"2023-09-24T19:09:35.764967+0000","172.21.15.16:6800/2615282850":"2023-09-24T18:55:50.550546+0000","172.21.15.16:0/349446544":"2023-09-24T18:55:50.550546+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":15,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T20:06:42.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:42 smithi016 bash[77995]: cluster 2023-09-23T20:06:40.776452+0000 mgr.y (mgr.25325) 1785 : cluster 0 pgmap v1879: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:42.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:42 smithi016 bash[77995]: audit 2023-09-23T20:06:41.484766+0000 mon.c (mon.2) 262 : audit 0 from='client.? 172.21.15.16:0/956368011' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:06:42.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:41 smithi016 bash[75004]: cluster 2023-09-23T20:06:40.776452+0000 mgr.y (mgr.25325) 1785 : cluster 0 pgmap v1879: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:42.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:41 smithi016 bash[75004]: audit 2023-09-23T20:06:41.484766+0000 mon.c (mon.2) 262 : audit 0 from='client.? 172.21.15.16:0/956368011' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:06:42.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:42 smithi092 bash[58855]: cluster 2023-09-23T20:06:40.776452+0000 mgr.y (mgr.25325) 1785 : cluster 0 pgmap v1879: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:42.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:42 smithi092 bash[58855]: audit 2023-09-23T20:06:41.484766+0000 mon.c (mon.2) 262 : audit 0 from='client.? 172.21.15.16:0/956368011' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:06:42.395 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-09-23T20:06:42.395 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T20:06:44.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:44 smithi016 bash[75004]: cluster 2023-09-23T20:06:42.777256+0000 mgr.y (mgr.25325) 1786 : cluster 0 pgmap v1880: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:44.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:44 smithi016 bash[75004]: audit 2023-09-23T20:06:43.707171+0000 mon.a (mon.0) 2156 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:06:44.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:44 smithi016 bash[77995]: cluster 2023-09-23T20:06:42.777256+0000 mgr.y (mgr.25325) 1786 : cluster 0 pgmap v1880: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:44.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:44 smithi016 bash[77995]: audit 2023-09-23T20:06:43.707171+0000 mon.a (mon.0) 2156 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:06:44.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:44 smithi092 bash[58855]: cluster 2023-09-23T20:06:42.777256+0000 mgr.y (mgr.25325) 1786 : cluster 0 pgmap v1880: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:44.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:44 smithi092 bash[58855]: audit 2023-09-23T20:06:43.707171+0000 mon.a (mon.0) 2156 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:06:46.009 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:06:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:20:06:45] "GET /metrics HTTP/1.1" 200 34011 "" "Prometheus/2.33.4" 2023-09-23T20:06:46.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:46 smithi016 bash[75004]: cluster 2023-09-23T20:06:44.778152+0000 mgr.y (mgr.25325) 1787 : cluster 0 pgmap v1881: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:46.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:46 smithi016 bash[77995]: cluster 2023-09-23T20:06:44.778152+0000 mgr.y (mgr.25325) 1787 : cluster 0 pgmap v1881: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:46.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:46 smithi092 bash[58855]: cluster 2023-09-23T20:06:44.778152+0000 mgr.y (mgr.25325) 1787 : cluster 0 pgmap v1881: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:47.186 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:06:48.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:48 smithi016 bash[75004]: cluster 2023-09-23T20:06:46.779177+0000 mgr.y (mgr.25325) 1788 : cluster 0 pgmap v1882: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:48.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:48 smithi016 bash[77995]: cluster 2023-09-23T20:06:46.779177+0000 mgr.y (mgr.25325) 1788 : cluster 0 pgmap v1882: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:48.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:48 smithi092 bash[58855]: cluster 2023-09-23T20:06:46.779177+0000 mgr.y (mgr.25325) 1788 : cluster 0 pgmap v1882: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:50.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:50 smithi016 bash[75004]: cluster 2023-09-23T20:06:48.779924+0000 mgr.y (mgr.25325) 1789 : cluster 0 pgmap v1883: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:50.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:50 smithi016 bash[77995]: cluster 2023-09-23T20:06:48.779924+0000 mgr.y (mgr.25325) 1789 : cluster 0 pgmap v1883: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:50.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:50 smithi092 bash[58855]: cluster 2023-09-23T20:06:48.779924+0000 mgr.y (mgr.25325) 1789 : cluster 0 pgmap v1883: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:50.682 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-09-23T20:06:50.682 INFO:teuthology.orchestra.run.smithi016.stdout:{"epoch":333,"fsid":"95ce9eb6-5a42-11ee-8db4-212e2dc638e7","created":"2023-09-23T18:55:23.802934+0000","modified":"2023-09-23T20:06:31.964654+0000","last_up_change":"2023-09-23T19:38:42.842616+0000","last_in_change":"2023-09-23T20:03:13.443789+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":23,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":15,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T18:59:33.315416+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{},"mgr_devicehealth":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T19:04:01.440427+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"297","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":297,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T19:05:50.982524+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T19:05:51.538373+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"a4e8ed09-1195-4878-928a-13881bc04dc7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":81,"up_from":85,"up_thru":316,"down_at":82,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6802","nonce":3034120466},{"type":"v1","addr":"172.21.15.16:6803","nonce":3034120466}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6804","nonce":3034120466},{"type":"v1","addr":"172.21.15.16:6805","nonce":3034120466}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6808","nonce":3034120466},{"type":"v1","addr":"172.21.15.16:6809","nonce":3034120466}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6806","nonce":3034120466},{"type":"v1","addr":"172.21.15.16:6807","nonce":3034120466}]},"public_addr":"172.21.15.16:6803/3034120466","cluster_addr":"172.21.15.16:6805/3034120466","heartbeat_back_addr":"172.21.15.16:6809/3034120466","heartbeat_front_addr":"172.21.15.16:6807/3034120466","state":["exists","up"]},{"osd":1,"uuid":"c5ff2879-0d1c-481a-b847-248433c12aa4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":93,"up_from":98,"up_thru":309,"down_at":94,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6810","nonce":3881357617},{"type":"v1","addr":"172.21.15.16:6811","nonce":3881357617}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6812","nonce":3881357617},{"type":"v1","addr":"172.21.15.16:6813","nonce":3881357617}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6816","nonce":3881357617},{"type":"v1","addr":"172.21.15.16:6817","nonce":3881357617}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6814","nonce":3881357617},{"type":"v1","addr":"172.21.15.16:6815","nonce":3881357617}]},"public_addr":"172.21.15.16:6811/3881357617","cluster_addr":"172.21.15.16:6813/3881357617","heartbeat_back_addr":"172.21.15.16:6817/3881357617","heartbeat_front_addr":"172.21.15.16:6815/3881357617","state":["exists","up"]},{"osd":2,"uuid":"7974ce4d-9ed8-4a72-b44b-d500b34da8cf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":111,"last_clean_end":154,"up_from":159,"up_thru":313,"down_at":155,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6818","nonce":4244964590},{"type":"v1","addr":"172.21.15.16:6819","nonce":4244964590}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6820","nonce":4244964590},{"type":"v1","addr":"172.21.15.16:6821","nonce":4244964590}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6824","nonce":4244964590},{"type":"v1","addr":"172.21.15.16:6825","nonce":4244964590}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6822","nonce":4244964590},{"type":"v1","addr":"172.21.15.16:6823","nonce":4244964590}]},"public_addr":"172.21.15.16:6819/4244964590","cluster_addr":"172.21.15.16:6821/4244964590","heartbeat_back_addr":"172.21.15.16:6825/4244964590","heartbeat_front_addr":"172.21.15.16:6823/4244964590","state":["exists","up"]},{"osd":3,"uuid":"f5e0d57b-d629-4453-a527-ad77e6c210ad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":112,"up_from":116,"up_thru":321,"down_at":113,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6826","nonce":3155744043},{"type":"v1","addr":"172.21.15.16:6827","nonce":3155744043}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6828","nonce":3155744043},{"type":"v1","addr":"172.21.15.16:6829","nonce":3155744043}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6832","nonce":3155744043},{"type":"v1","addr":"172.21.15.16:6833","nonce":3155744043}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6830","nonce":3155744043},{"type":"v1","addr":"172.21.15.16:6831","nonce":3155744043}]},"public_addr":"172.21.15.16:6827/3155744043","cluster_addr":"172.21.15.16:6829/3155744043","heartbeat_back_addr":"172.21.15.16:6833/3155744043","heartbeat_front_addr":"172.21.15.16:6831/3155744043","state":["exists","up"]},{"osd":4,"uuid":"55a8dab2-6393-44ab-943e-cb7ab2cb7c31","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":313,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6800","nonce":99745912},{"type":"v1","addr":"172.21.15.92:6801","nonce":99745912}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6802","nonce":99745912},{"type":"v1","addr":"172.21.15.92:6803","nonce":99745912}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6806","nonce":99745912},{"type":"v1","addr":"172.21.15.92:6807","nonce":99745912}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6804","nonce":99745912},{"type":"v1","addr":"172.21.15.92:6805","nonce":99745912}]},"public_addr":"172.21.15.92:6801/99745912","cluster_addr":"172.21.15.92:6803/99745912","heartbeat_back_addr":"172.21.15.92:6807/99745912","heartbeat_front_addr":"172.21.15.92:6805/99745912","state":["exists","up"]},{"osd":5,"uuid":"b29b73b1-51d7-4484-bd79-2334c650cb62","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":326,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6808","nonce":1987214502},{"type":"v1","addr":"172.21.15.92:6809","nonce":1987214502}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6810","nonce":1987214502},{"type":"v1","addr":"172.21.15.92:6811","nonce":1987214502}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6814","nonce":1987214502},{"type":"v1","addr":"172.21.15.92:6815","nonce":1987214502}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6812","nonce":1987214502},{"type":"v1","addr":"172.21.15.92:6813","nonce":1987214502}]},"public_addr":"172.21.15.92:6809/1987214502","cluster_addr":"172.21.15.92:6811/1987214502","heartbeat_back_addr":"172.21.15.92:6815/1987214502","heartbeat_front_addr":"172.21.15.92:6813/1987214502","state":["exists","up"]},{"osd":6,"uuid":"9fe16cc9-f0d3-4b79-bc68-38305c55b551","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":306,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6816","nonce":2820798178},{"type":"v1","addr":"172.21.15.92:6817","nonce":2820798178}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6818","nonce":2820798178},{"type":"v1","addr":"172.21.15.92:6819","nonce":2820798178}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6822","nonce":2820798178},{"type":"v1","addr":"172.21.15.92:6823","nonce":2820798178}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6820","nonce":2820798178},{"type":"v1","addr":"172.21.15.92:6821","nonce":2820798178}]},"public_addr":"172.21.15.92:6817/2820798178","cluster_addr":"172.21.15.92:6819/2820798178","heartbeat_back_addr":"172.21.15.92:6823/2820798178","heartbeat_front_addr":"172.21.15.92:6821/2820798178","state":["exists","up"]},{"osd":7,"uuid":"d086b076-127f-42de-bee9-157bf579d92d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":306,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6824","nonce":98624761},{"type":"v1","addr":"172.21.15.92:6825","nonce":98624761}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6826","nonce":98624761},{"type":"v1","addr":"172.21.15.92:6827","nonce":98624761}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6830","nonce":98624761},{"type":"v1","addr":"172.21.15.92:6831","nonce":98624761}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6828","nonce":98624761},{"type":"v1","addr":"172.21.15.92:6829","nonce":98624761}]},"public_addr":"172.21.15.92:6825/98624761","cluster_addr":"172.21.15.92:6827/98624761","heartbeat_back_addr":"172.21.15.92:6831/98624761","heartbeat_front_addr":"172.21.15.92:6829/98624761","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:58:07.335980+0000","dead_epoch":81},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:58:49.342213+0000","dead_epoch":93},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:59:30.939986+0000","dead_epoch":154},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:00:12.926364+0000","dead_epoch":112},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:00:53.466692+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:01:36.332957+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:02:18.593854+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:03:00.817448+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.d","mappings":[{"from":1,"to":5}]},{"pgid":"4.17","mappings":[{"from":1,"to":2}]},{"pgid":"4.1d","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.16:6801/777573315":"2023-09-24T19:12:43.452315+0000","172.21.15.16:6800/777573315":"2023-09-24T19:12:43.452315+0000","172.21.15.16:0/2334334390":"2023-09-24T19:12:43.452315+0000","172.21.15.16:0/4196794739":"2023-09-24T19:12:43.452315+0000","172.21.15.16:0/736761601":"2023-09-24T19:12:43.452315+0000","172.21.15.16:6800/2756103674":"2023-09-24T18:56:20.574625+0000","172.21.15.16:0/2843459289":"2023-09-24T18:56:20.574625+0000","172.21.15.16:6801/2756103674":"2023-09-24T18:56:20.574625+0000","172.21.15.92:0/1841502917":"2023-09-24T19:11:32.247480+0000","172.21.15.16:0/3767844203":"2023-09-24T19:03:34.846451+0000","172.21.15.16:6800/1259630240":"2023-09-24T19:03:34.846451+0000","172.21.15.92:0/3338210669":"2023-09-24T19:11:32.247480+0000","172.21.15.92:6835/2057654371":"2023-09-24T19:11:32.247480+0000","172.21.15.16:0/272242693":"2023-09-24T18:56:20.574625+0000","172.21.15.16:0/3595835147":"2023-09-24T19:12:43.452315+0000","172.21.15.16:0/79169563":"2023-09-24T18:55:50.550546+0000","172.21.15.16:6800/1501185908":"2023-09-24T19:09:35.764967+0000","172.21.15.16:6801/1259630240":"2023-09-24T19:03:34.846451+0000","172.21.15.16:0/3765291224":"2023-09-24T19:09:35.764967+0000","172.21.15.16:6801/1501185908":"2023-09-24T19:09:35.764967+0000","172.21.15.16:0/647003663":"2023-09-24T19:12:43.452315+0000","172.21.15.16:0/2669353350":"2023-09-24T19:03:34.846451+0000","172.21.15.92:6834/2057654371":"2023-09-24T19:11:32.247480+0000","172.21.15.92:0/561988499":"2023-09-24T19:11:32.247480+0000","172.21.15.16:6801/2615282850":"2023-09-24T18:55:50.550546+0000","172.21.15.16:0/3394254122":"2023-09-24T19:12:43.452315+0000","172.21.15.92:0/3173267635":"2023-09-24T19:11:32.247480+0000","172.21.15.16:0/4152413938":"2023-09-24T19:09:35.764967+0000","172.21.15.16:6800/2615282850":"2023-09-24T18:55:50.550546+0000","172.21.15.16:0/349446544":"2023-09-24T18:55:50.550546+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":15,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T20:06:51.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:51 smithi016 bash[77995]: audit 2023-09-23T20:06:50.675348+0000 mon.a (mon.0) 2157 : audit 0 from='client.? 172.21.15.16:0/3790508164' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:06:51.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:51 smithi016 bash[75004]: audit 2023-09-23T20:06:50.675348+0000 mon.a (mon.0) 2157 : audit 0 from='client.? 172.21.15.16:0/3790508164' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:06:51.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:51 smithi092 bash[58855]: audit 2023-09-23T20:06:50.675348+0000 mon.a (mon.0) 2157 : audit 0 from='client.? 172.21.15.16:0/3790508164' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:06:51.632 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-09-23T20:06:51.633 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-09-23T20:06:51.633 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-09-23T20:06:51.634 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-09-23T20:06:51.634 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-09-23T20:06:51.635 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-09-23T20:06:51.635 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-09-23T20:06:51.636 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-09-23T20:06:52.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:52 smithi016 bash[77995]: cluster 2023-09-23T20:06:50.780850+0000 mgr.y (mgr.25325) 1790 : cluster 0 pgmap v1884: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:52.286 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:52 smithi016 bash[75004]: cluster 2023-09-23T20:06:50.780850+0000 mgr.y (mgr.25325) 1790 : cluster 0 pgmap v1884: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:52.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:52 smithi092 bash[58855]: cluster 2023-09-23T20:06:50.780850+0000 mgr.y (mgr.25325) 1790 : cluster 0 pgmap v1884: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:54.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:54 smithi016 bash[75004]: cluster 2023-09-23T20:06:52.781740+0000 mgr.y (mgr.25325) 1791 : cluster 0 pgmap v1885: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:54.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:54 smithi016 bash[77995]: cluster 2023-09-23T20:06:52.781740+0000 mgr.y (mgr.25325) 1791 : cluster 0 pgmap v1885: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:54.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:54 smithi092 bash[58855]: cluster 2023-09-23T20:06:52.781740+0000 mgr.y (mgr.25325) 1791 : cluster 0 pgmap v1885: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:56.034 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:06:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:20:06:55] "GET /metrics HTTP/1.1" 200 34011 "" "Prometheus/2.33.4" 2023-09-23T20:06:56.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:56 smithi016 bash[77995]: cluster 2023-09-23T20:06:54.782525+0000 mgr.y (mgr.25325) 1792 : cluster 0 pgmap v1886: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:56.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:56 smithi016 bash[75004]: cluster 2023-09-23T20:06:54.782525+0000 mgr.y (mgr.25325) 1792 : cluster 0 pgmap v1886: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:56.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:56 smithi092 bash[58855]: cluster 2023-09-23T20:06:54.782525+0000 mgr.y (mgr.25325) 1792 : cluster 0 pgmap v1886: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:56.532 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:06:56.532 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:06:56.534 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:06:56.536 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:06:56.536 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:06:56.537 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:06:56.538 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:06:56.539 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:06:58.288 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:58 smithi092 bash[58855]: cluster 2023-09-23T20:06:56.783352+0000 mgr.y (mgr.25325) 1793 : cluster 0 pgmap v1887: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:58.445 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:58 smithi016 bash[75004]: cluster 2023-09-23T20:06:56.783352+0000 mgr.y (mgr.25325) 1793 : cluster 0 pgmap v1887: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:58.446 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:58 smithi016 bash[77995]: cluster 2023-09-23T20:06:56.783352+0000 mgr.y (mgr.25325) 1793 : cluster 0 pgmap v1887: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:06:59.354 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:06:59 smithi016 bash[75004]: audit 2023-09-23T20:06:58.707156+0000 mon.a (mon.0) 2158 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:06:59.355 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:06:59 smithi016 bash[77995]: audit 2023-09-23T20:06:58.707156+0000 mon.a (mon.0) 2158 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:06:59.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:06:59 smithi092 bash[58855]: audit 2023-09-23T20:06:58.707156+0000 mon.a (mon.0) 2158 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:07:00.367 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:07:00 smithi016 bash[77995]: cluster 2023-09-23T20:06:58.783889+0000 mgr.y (mgr.25325) 1794 : cluster 0 pgmap v1888: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:00.367 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:07:00 smithi016 bash[75004]: cluster 2023-09-23T20:06:58.783889+0000 mgr.y (mgr.25325) 1794 : cluster 0 pgmap v1888: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:00.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:07:00 smithi092 bash[58855]: cluster 2023-09-23T20:06:58.783889+0000 mgr.y (mgr.25325) 1794 : cluster 0 pgmap v1888: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:02.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:07:02 smithi016 bash[75004]: cluster 2023-09-23T20:07:00.784865+0000 mgr.y (mgr.25325) 1795 : cluster 0 pgmap v1889: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:02.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:07:02 smithi016 bash[77995]: cluster 2023-09-23T20:07:00.784865+0000 mgr.y (mgr.25325) 1795 : cluster 0 pgmap v1889: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:02.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:07:02 smithi092 bash[58855]: cluster 2023-09-23T20:07:00.784865+0000 mgr.y (mgr.25325) 1795 : cluster 0 pgmap v1889: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:04.513 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:07:04 smithi016 bash[75004]: cluster 2023-09-23T20:07:02.785661+0000 mgr.y (mgr.25325) 1796 : cluster 0 pgmap v1890: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:04.513 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:07:04 smithi016 bash[77995]: cluster 2023-09-23T20:07:02.785661+0000 mgr.y (mgr.25325) 1796 : cluster 0 pgmap v1890: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:04.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:07:04 smithi092 bash[58855]: cluster 2023-09-23T20:07:02.785661+0000 mgr.y (mgr.25325) 1796 : cluster 0 pgmap v1890: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:06.035 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:07:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:20:07:05] "GET /metrics HTTP/1.1" 200 34012 "" "Prometheus/2.33.4" 2023-09-23T20:07:06.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:07:06 smithi016 bash[75004]: cluster 2023-09-23T20:07:04.786267+0000 mgr.y (mgr.25325) 1797 : cluster 0 pgmap v1891: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:06.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:07:06 smithi016 bash[77995]: cluster 2023-09-23T20:07:04.786267+0000 mgr.y (mgr.25325) 1797 : cluster 0 pgmap v1891: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:06.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:07:06 smithi092 bash[58855]: cluster 2023-09-23T20:07:04.786267+0000 mgr.y (mgr.25325) 1797 : cluster 0 pgmap v1891: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:08.284 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:07:08 smithi016 bash[75004]: cluster 2023-09-23T20:07:06.787092+0000 mgr.y (mgr.25325) 1798 : cluster 0 pgmap v1892: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:08.284 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:07:08 smithi016 bash[77995]: cluster 2023-09-23T20:07:06.787092+0000 mgr.y (mgr.25325) 1798 : cluster 0 pgmap v1892: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:08.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:07:08 smithi092 bash[58855]: cluster 2023-09-23T20:07:06.787092+0000 mgr.y (mgr.25325) 1798 : cluster 0 pgmap v1892: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:10.303 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:07:10 smithi016 bash[75004]: cluster 2023-09-23T20:07:08.787736+0000 mgr.y (mgr.25325) 1799 : cluster 0 pgmap v1893: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:10.303 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:07:10 smithi016 bash[77995]: cluster 2023-09-23T20:07:08.787736+0000 mgr.y (mgr.25325) 1799 : cluster 0 pgmap v1893: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:10.304 INFO:teuthology.orchestra.run.smithi016.stdout:498216206807 2023-09-23T20:07:10.304 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-09-23T20:07:10.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:07:10 smithi092 bash[58855]: cluster 2023-09-23T20:07:08.787736+0000 mgr.y (mgr.25325) 1799 : cluster 0 pgmap v1893: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:12.104 INFO:teuthology.orchestra.run.smithi016.stdout:420906795544 2023-09-23T20:07:12.104 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-09-23T20:07:12.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:07:12 smithi016 bash[75004]: cluster 2023-09-23T20:07:10.788668+0000 mgr.y (mgr.25325) 1800 : cluster 0 pgmap v1894: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:12.285 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:07:12 smithi016 bash[77995]: cluster 2023-09-23T20:07:10.788668+0000 mgr.y (mgr.25325) 1800 : cluster 0 pgmap v1894: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:12.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:07:12 smithi092 bash[58855]: cluster 2023-09-23T20:07:10.788668+0000 mgr.y (mgr.25325) 1800 : cluster 0 pgmap v1894: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:14.269 INFO:teuthology.orchestra.run.smithi016.stdout:180388627212 2023-09-23T20:07:14.270 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-09-23T20:07:14.381 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:07:14 smithi016 bash[75004]: cluster 2023-09-23T20:07:12.789641+0000 mgr.y (mgr.25325) 1801 : cluster 0 pgmap v1895: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:14.381 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:07:14 smithi016 bash[75004]: audit 2023-09-23T20:07:13.707998+0000 mon.a (mon.0) 2159 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:07:14.381 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:07:14 smithi016 bash[75004]: audit 2023-09-23T20:07:13.809401+0000 mon.a (mon.0) 2160 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:07:14.381 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:07:14 smithi016 bash[77995]: cluster 2023-09-23T20:07:12.789641+0000 mgr.y (mgr.25325) 1801 : cluster 0 pgmap v1895: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:14.382 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:07:14 smithi016 bash[77995]: audit 2023-09-23T20:07:13.707998+0000 mon.a (mon.0) 2159 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:07:14.382 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:07:14 smithi016 bash[77995]: audit 2023-09-23T20:07:13.809401+0000 mon.a (mon.0) 2160 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:07:14.383 INFO:teuthology.orchestra.run.smithi016.stdout:206158430980 2023-09-23T20:07:14.383 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-09-23T20:07:14.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:07:14 smithi092 bash[58855]: cluster 2023-09-23T20:07:12.789641+0000 mgr.y (mgr.25325) 1801 : cluster 0 pgmap v1895: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:14.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:07:14 smithi092 bash[58855]: audit 2023-09-23T20:07:13.707998+0000 mon.a (mon.0) 2159 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:07:14.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:07:14 smithi092 bash[58855]: audit 2023-09-23T20:07:13.809401+0000 mon.a (mon.0) 2160 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:07:14.634 INFO:teuthology.orchestra.run.smithi016.stdout:154618823445 2023-09-23T20:07:14.635 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-09-23T20:07:15.254 INFO:teuthology.orchestra.run.smithi016.stdout:128849019678 2023-09-23T20:07:15.255 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-09-23T20:07:15.898 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:07:15 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:20:07:15] "GET /metrics HTTP/1.1" 200 34016 "" "Prometheus/2.33.4" 2023-09-23T20:07:16.285 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:07:16 smithi016 bash[75004]: cluster 2023-09-23T20:07:14.790362+0000 mgr.y (mgr.25325) 1802 : cluster 0 pgmap v1896: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:16.286 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:07:16 smithi016 bash[77995]: cluster 2023-09-23T20:07:14.790362+0000 mgr.y (mgr.25325) 1802 : cluster 0 pgmap v1896: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:16.333 INFO:teuthology.orchestra.run.smithi016.stdout:365072220752 2023-09-23T20:07:16.334 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-09-23T20:07:16.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:07:16 smithi092 bash[58855]: cluster 2023-09-23T20:07:14.790362+0000 mgr.y (mgr.25325) 1802 : cluster 0 pgmap v1896: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:16.823 INFO:teuthology.orchestra.run.smithi016.stdout:682899800408 2023-09-23T20:07:16.823 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-09-23T20:07:18.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:07:18 smithi016 bash[75004]: cluster 2023-09-23T20:07:16.791181+0000 mgr.y (mgr.25325) 1803 : cluster 0 pgmap v1897: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:18.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:07:18 smithi016 bash[77995]: cluster 2023-09-23T20:07:16.791181+0000 mgr.y (mgr.25325) 1803 : cluster 0 pgmap v1897: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:18.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:07:18 smithi092 bash[58855]: cluster 2023-09-23T20:07:16.791181+0000 mgr.y (mgr.25325) 1803 : cluster 0 pgmap v1897: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:19.131 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:07:19.133 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:07:19.134 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:07:19.134 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:07:19.134 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:07:19.137 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:07:20.144 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:07:20.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:07:20 smithi016 bash[77995]: cluster 2023-09-23T20:07:18.792102+0000 mgr.y (mgr.25325) 1804 : cluster 0 pgmap v1898: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:20.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:07:20 smithi016 bash[77995]: audit 2023-09-23T20:07:20.048853+0000 mon.a (mon.0) 2161 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:07:20.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:07:20 smithi016 bash[77995]: audit 2023-09-23T20:07:20.055820+0000 mon.a (mon.0) 2162 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:07:20.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:07:20 smithi016 bash[75004]: cluster 2023-09-23T20:07:18.792102+0000 mgr.y (mgr.25325) 1804 : cluster 0 pgmap v1898: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:20.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:07:20 smithi016 bash[75004]: audit 2023-09-23T20:07:20.048853+0000 mon.a (mon.0) 2161 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:07:20.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:07:20 smithi016 bash[75004]: audit 2023-09-23T20:07:20.055820+0000 mon.a (mon.0) 2162 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:07:20.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:07:20 smithi092 bash[58855]: cluster 2023-09-23T20:07:18.792102+0000 mgr.y (mgr.25325) 1804 : cluster 0 pgmap v1898: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:20.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:07:20 smithi092 bash[58855]: audit 2023-09-23T20:07:20.048853+0000 mon.a (mon.0) 2161 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:07:20.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:07:20 smithi092 bash[58855]: audit 2023-09-23T20:07:20.055820+0000 mon.a (mon.0) 2162 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:07:21.156 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:07:21.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:07:21 smithi092 bash[58855]: audit 2023-09-23T20:07:20.248640+0000 mon.a (mon.0) 2163 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:07:21.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:07:21 smithi092 bash[58855]: audit 2023-09-23T20:07:20.256102+0000 mon.a (mon.0) 2164 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:07:21.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:07:21 smithi092 bash[58855]: audit 2023-09-23T20:07:20.736112+0000 mon.a (mon.0) 2165 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:07:21.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:07:21 smithi092 bash[58855]: audit 2023-09-23T20:07:20.737569+0000 mon.a (mon.0) 2166 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:07:21.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:07:21 smithi092 bash[58855]: audit 2023-09-23T20:07:20.748256+0000 mon.a (mon.0) 2167 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:07:21.590 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:07:21 smithi016 bash[75004]: audit 2023-09-23T20:07:20.248640+0000 mon.a (mon.0) 2163 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:07:21.590 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:07:21 smithi016 bash[75004]: audit 2023-09-23T20:07:20.256102+0000 mon.a (mon.0) 2164 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:07:21.590 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:07:21 smithi016 bash[75004]: audit 2023-09-23T20:07:20.736112+0000 mon.a (mon.0) 2165 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:07:21.590 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:07:21 smithi016 bash[75004]: audit 2023-09-23T20:07:20.737569+0000 mon.a (mon.0) 2166 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:07:21.591 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:07:21 smithi016 bash[75004]: audit 2023-09-23T20:07:20.748256+0000 mon.a (mon.0) 2167 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:07:21.591 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:07:21 smithi016 bash[77995]: audit 2023-09-23T20:07:20.248640+0000 mon.a (mon.0) 2163 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:07:21.591 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:07:21 smithi016 bash[77995]: audit 2023-09-23T20:07:20.256102+0000 mon.a (mon.0) 2164 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:07:21.591 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:07:21 smithi016 bash[77995]: audit 2023-09-23T20:07:20.736112+0000 mon.a (mon.0) 2165 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:07:21.591 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:07:21 smithi016 bash[77995]: audit 2023-09-23T20:07:20.737569+0000 mon.a (mon.0) 2166 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:07:21.591 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:07:21 smithi016 bash[77995]: audit 2023-09-23T20:07:20.748256+0000 mon.a (mon.0) 2167 : audit 1 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' 2023-09-23T20:07:22.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:07:22 smithi016 bash[75004]: cluster 2023-09-23T20:07:20.793024+0000 mgr.y (mgr.25325) 1805 : cluster 0 pgmap v1899: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:22.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:07:22 smithi016 bash[77995]: cluster 2023-09-23T20:07:20.793024+0000 mgr.y (mgr.25325) 1805 : cluster 0 pgmap v1899: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:22.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:07:22 smithi092 bash[58855]: cluster 2023-09-23T20:07:20.793024+0000 mgr.y (mgr.25325) 1805 : cluster 0 pgmap v1899: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:24.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:07:24 smithi092 bash[58855]: cluster 2023-09-23T20:07:22.793996+0000 mgr.y (mgr.25325) 1806 : cluster 0 pgmap v1900: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:24.562 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:07:24 smithi016 bash[75004]: cluster 2023-09-23T20:07:22.793996+0000 mgr.y (mgr.25325) 1806 : cluster 0 pgmap v1900: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:24.563 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:07:24 smithi016 bash[77995]: cluster 2023-09-23T20:07:22.793996+0000 mgr.y (mgr.25325) 1806 : cluster 0 pgmap v1900: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:25.980 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:07:25 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:20:07:25] "GET /metrics HTTP/1.1" 200 34016 "" "Prometheus/2.33.4" 2023-09-23T20:07:26.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:07:26 smithi016 bash[77995]: cluster 2023-09-23T20:07:24.794519+0000 mgr.y (mgr.25325) 1807 : cluster 0 pgmap v1901: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:26.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:07:26 smithi016 bash[75004]: cluster 2023-09-23T20:07:24.794519+0000 mgr.y (mgr.25325) 1807 : cluster 0 pgmap v1901: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:26.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:07:26 smithi092 bash[58855]: cluster 2023-09-23T20:07:24.794519+0000 mgr.y (mgr.25325) 1807 : cluster 0 pgmap v1901: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:28.432 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:07:28 smithi016 bash[77995]: cluster 2023-09-23T20:07:26.795249+0000 mgr.y (mgr.25325) 1808 : cluster 0 pgmap v1902: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:28.432 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:07:28 smithi016 bash[75004]: cluster 2023-09-23T20:07:26.795249+0000 mgr.y (mgr.25325) 1808 : cluster 0 pgmap v1902: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:28.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:07:28 smithi092 bash[58855]: cluster 2023-09-23T20:07:26.795249+0000 mgr.y (mgr.25325) 1808 : cluster 0 pgmap v1902: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:29.536 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:07:29 smithi016 bash[75004]: audit 2023-09-23T20:07:28.707754+0000 mon.a (mon.0) 2168 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:07:29.536 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:07:29 smithi016 bash[77995]: audit 2023-09-23T20:07:28.707754+0000 mon.a (mon.0) 2168 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:07:29.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:07:29 smithi092 bash[58855]: audit 2023-09-23T20:07:28.707754+0000 mon.a (mon.0) 2168 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:07:30.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:07:30 smithi016 bash[75004]: cluster 2023-09-23T20:07:28.796042+0000 mgr.y (mgr.25325) 1809 : cluster 0 pgmap v1903: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:30.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:07:30 smithi016 bash[77995]: cluster 2023-09-23T20:07:28.796042+0000 mgr.y (mgr.25325) 1809 : cluster 0 pgmap v1903: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:30.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:07:30 smithi092 bash[58855]: cluster 2023-09-23T20:07:28.796042+0000 mgr.y (mgr.25325) 1809 : cluster 0 pgmap v1903: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:32.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:07:32 smithi016 bash[75004]: cluster 2023-09-23T20:07:30.796927+0000 mgr.y (mgr.25325) 1810 : cluster 0 pgmap v1904: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:32.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:07:32 smithi016 bash[77995]: cluster 2023-09-23T20:07:30.796927+0000 mgr.y (mgr.25325) 1810 : cluster 0 pgmap v1904: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:32.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:07:32 smithi092 bash[58855]: cluster 2023-09-23T20:07:30.796927+0000 mgr.y (mgr.25325) 1810 : cluster 0 pgmap v1904: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:34.288 INFO:teuthology.orchestra.run.smithi016.stdout:128849019682 2023-09-23T20:07:34.538 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:07:34 smithi092 bash[58855]: cluster 2023-09-23T20:07:32.797806+0000 mgr.y (mgr.25325) 1811 : cluster 0 pgmap v1905: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:34.589 INFO:teuthology.orchestra.run.smithi016.stdout:180388627217 2023-09-23T20:07:34.636 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:07:34 smithi016 bash[75004]: cluster 2023-09-23T20:07:32.797806+0000 mgr.y (mgr.25325) 1811 : cluster 0 pgmap v1905: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:34.637 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:07:34 smithi016 bash[77995]: cluster 2023-09-23T20:07:32.797806+0000 mgr.y (mgr.25325) 1811 : cluster 0 pgmap v1905: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:35.641 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:07:35 smithi016 bash[75004]: audit 2023-09-23T20:07:34.285211+0000 mon.c (mon.2) 263 : audit 0 from='client.? 172.21.15.16:0/1094799498' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T20:07:35.641 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:07:35 smithi016 bash[75004]: audit 2023-09-23T20:07:34.586853+0000 mon.a (mon.0) 2169 : audit 0 from='client.? 172.21.15.16:0/2845196980' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T20:07:35.641 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:07:35 smithi016 bash[77995]: audit 2023-09-23T20:07:34.285211+0000 mon.c (mon.2) 263 : audit 0 from='client.? 172.21.15.16:0/1094799498' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T20:07:35.641 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:07:35 smithi016 bash[77995]: audit 2023-09-23T20:07:34.586853+0000 mon.a (mon.0) 2169 : audit 0 from='client.? 172.21.15.16:0/2845196980' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T20:07:35.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:07:35 smithi092 bash[58855]: audit 2023-09-23T20:07:34.285211+0000 mon.c (mon.2) 263 : audit 0 from='client.? 172.21.15.16:0/1094799498' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T20:07:35.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:07:35 smithi092 bash[58855]: audit 2023-09-23T20:07:34.586853+0000 mon.a (mon.0) 2169 : audit 0 from='client.? 172.21.15.16:0/2845196980' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T20:07:36.035 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:07:35 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:20:07:35] "GET /metrics HTTP/1.1" 200 34014 "" "Prometheus/2.33.4" 2023-09-23T20:07:36.469 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:07:36 smithi016 bash[75004]: cluster 2023-09-23T20:07:34.798432+0000 mgr.y (mgr.25325) 1812 : cluster 0 pgmap v1906: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:36.469 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:07:36 smithi016 bash[77995]: cluster 2023-09-23T20:07:34.798432+0000 mgr.y (mgr.25325) 1812 : cluster 0 pgmap v1906: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:36.535 INFO:teuthology.orchestra.run.smithi016.stdout:154618823450 2023-09-23T20:07:36.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:07:36 smithi092 bash[58855]: cluster 2023-09-23T20:07:34.798432+0000 mgr.y (mgr.25325) 1812 : cluster 0 pgmap v1906: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:37.558 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:07:37 smithi016 bash[75004]: audit 2023-09-23T20:07:36.532486+0000 mon.c (mon.2) 264 : audit 0 from='client.? 172.21.15.16:0/602345235' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T20:07:37.558 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:07:37 smithi016 bash[77995]: audit 2023-09-23T20:07:36.532486+0000 mon.c (mon.2) 264 : audit 0 from='client.? 172.21.15.16:0/602345235' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T20:07:37.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:07:37 smithi092 bash[58855]: audit 2023-09-23T20:07:36.532486+0000 mon.c (mon.2) 264 : audit 0 from='client.? 172.21.15.16:0/602345235' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T20:07:37.893 INFO:teuthology.orchestra.run.smithi016.stdout:206158430986 2023-09-23T20:07:38.617 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:07:38 smithi016 bash[75004]: cluster 2023-09-23T20:07:36.799039+0000 mgr.y (mgr.25325) 1813 : cluster 0 pgmap v1907: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:38.617 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:07:38 smithi016 bash[75004]: audit 2023-09-23T20:07:37.890361+0000 mon.c (mon.2) 265 : audit 0 from='client.? 172.21.15.16:0/2695688438' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T20:07:38.617 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:07:38 smithi016 bash[77995]: cluster 2023-09-23T20:07:36.799039+0000 mgr.y (mgr.25325) 1813 : cluster 0 pgmap v1907: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:38.617 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:07:38 smithi016 bash[77995]: audit 2023-09-23T20:07:37.890361+0000 mon.c (mon.2) 265 : audit 0 from='client.? 172.21.15.16:0/2695688438' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T20:07:38.618 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388627212 got 180388627217 for osd.6 2023-09-23T20:07:38.618 DEBUG:teuthology.parallel:result is None 2023-09-23T20:07:38.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:07:38 smithi092 bash[58855]: cluster 2023-09-23T20:07:36.799039+0000 mgr.y (mgr.25325) 1813 : cluster 0 pgmap v1907: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:38.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:07:38 smithi092 bash[58855]: audit 2023-09-23T20:07:37.890361+0000 mon.c (mon.2) 265 : audit 0 from='client.? 172.21.15.16:0/2695688438' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T20:07:38.903 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849019678 got 128849019682 for osd.4 2023-09-23T20:07:38.903 DEBUG:teuthology.parallel:result is None 2023-09-23T20:07:39.376 INFO:teuthology.orchestra.run.smithi016.stdout:498216206813 2023-09-23T20:07:39.815 INFO:teuthology.orchestra.run.smithi016.stdout:420906795550 2023-09-23T20:07:40.293 INFO:teuthology.orchestra.run.smithi016.stdout:365072220757 2023-09-23T20:07:40.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:07:40 smithi016 bash[75004]: cluster 2023-09-23T20:07:38.799815+0000 mgr.y (mgr.25325) 1814 : cluster 0 pgmap v1908: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:40.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:07:40 smithi016 bash[75004]: audit 2023-09-23T20:07:39.373823+0000 mon.c (mon.2) 266 : audit 0 from='client.? 172.21.15.16:0/4196797702' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T20:07:40.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:07:40 smithi016 bash[75004]: audit 2023-09-23T20:07:39.812129+0000 mon.c (mon.2) 267 : audit 0 from='client.? 172.21.15.16:0/87523315' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T20:07:40.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:07:40 smithi016 bash[75004]: audit 2023-09-23T20:07:40.290052+0000 mon.a (mon.0) 2170 : audit 0 from='client.? 172.21.15.16:0/4061395958' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T20:07:40.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:07:40 smithi016 bash[77995]: cluster 2023-09-23T20:07:38.799815+0000 mgr.y (mgr.25325) 1814 : cluster 0 pgmap v1908: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:40.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:07:40 smithi016 bash[77995]: audit 2023-09-23T20:07:39.373823+0000 mon.c (mon.2) 266 : audit 0 from='client.? 172.21.15.16:0/4196797702' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T20:07:40.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:07:40 smithi016 bash[77995]: audit 2023-09-23T20:07:39.812129+0000 mon.c (mon.2) 267 : audit 0 from='client.? 172.21.15.16:0/87523315' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T20:07:40.787 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:07:40 smithi016 bash[77995]: audit 2023-09-23T20:07:40.290052+0000 mon.a (mon.0) 2170 : audit 0 from='client.? 172.21.15.16:0/4061395958' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T20:07:40.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:07:40 smithi092 bash[58855]: cluster 2023-09-23T20:07:38.799815+0000 mgr.y (mgr.25325) 1814 : cluster 0 pgmap v1908: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:40.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:07:40 smithi092 bash[58855]: audit 2023-09-23T20:07:39.373823+0000 mon.c (mon.2) 266 : audit 0 from='client.? 172.21.15.16:0/4196797702' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T20:07:40.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:07:40 smithi092 bash[58855]: audit 2023-09-23T20:07:39.812129+0000 mon.c (mon.2) 267 : audit 0 from='client.? 172.21.15.16:0/87523315' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T20:07:40.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:07:40 smithi092 bash[58855]: audit 2023-09-23T20:07:40.290052+0000 mon.a (mon.0) 2170 : audit 0 from='client.? 172.21.15.16:0/4061395958' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T20:07:41.286 INFO:teuthology.orchestra.run.smithi016.stdout:682899800413 2023-09-23T20:07:41.566 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:07:41 smithi016 bash[75004]: audit 2023-09-23T20:07:41.283510+0000 mon.a (mon.0) 2171 : audit 0 from='client.? 172.21.15.16:0/4174539804' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T20:07:41.567 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:07:41 smithi016 bash[77995]: audit 2023-09-23T20:07:41.283510+0000 mon.a (mon.0) 2171 : audit 0 from='client.? 172.21.15.16:0/4174539804' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T20:07:41.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:07:41 smithi092 bash[58855]: audit 2023-09-23T20:07:41.283510+0000 mon.a (mon.0) 2171 : audit 0 from='client.? 172.21.15.16:0/4174539804' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T20:07:42.252 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430980 got 206158430986 for osd.7 2023-09-23T20:07:42.252 DEBUG:teuthology.parallel:result is None 2023-09-23T20:07:42.547 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:07:42 smithi016 bash[75004]: cluster 2023-09-23T20:07:40.800399+0000 mgr.y (mgr.25325) 1815 : cluster 0 pgmap v1909: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:42.547 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:07:42 smithi016 bash[77995]: cluster 2023-09-23T20:07:40.800399+0000 mgr.y (mgr.25325) 1815 : cluster 0 pgmap v1909: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:42.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:07:42 smithi092 bash[58855]: cluster 2023-09-23T20:07:40.800399+0000 mgr.y (mgr.25325) 1815 : cluster 0 pgmap v1909: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:42.976 INFO:tasks.cephadm.ceph_manager.ceph:need seq 498216206807 got 498216206813 for osd.3 2023-09-23T20:07:42.976 DEBUG:teuthology.parallel:result is None 2023-09-23T20:07:43.154 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618823445 got 154618823450 for osd.5 2023-09-23T20:07:43.155 DEBUG:teuthology.parallel:result is None 2023-09-23T20:07:44.141 INFO:tasks.cephadm.ceph_manager.ceph:need seq 420906795544 got 420906795550 for osd.1 2023-09-23T20:07:44.141 DEBUG:teuthology.parallel:result is None 2023-09-23T20:07:44.350 INFO:tasks.cephadm.ceph_manager.ceph:need seq 365072220752 got 365072220757 for osd.0 2023-09-23T20:07:44.351 DEBUG:teuthology.parallel:result is None 2023-09-23T20:07:44.392 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:07:44 smithi016 bash[77995]: cluster 2023-09-23T20:07:42.801147+0000 mgr.y (mgr.25325) 1816 : cluster 0 pgmap v1910: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:44.392 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:07:44 smithi016 bash[77995]: audit 2023-09-23T20:07:43.708157+0000 mon.a (mon.0) 2172 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:07:44.393 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:07:44 smithi016 bash[75004]: cluster 2023-09-23T20:07:42.801147+0000 mgr.y (mgr.25325) 1816 : cluster 0 pgmap v1910: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:44.393 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:07:44 smithi016 bash[75004]: audit 2023-09-23T20:07:43.708157+0000 mon.a (mon.0) 2172 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:07:44.393 INFO:tasks.cephadm.ceph_manager.ceph:need seq 682899800408 got 682899800413 for osd.2 2023-09-23T20:07:44.394 DEBUG:teuthology.parallel:result is None 2023-09-23T20:07:44.394 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T20:07:44.394 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:07:44.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:07:44 smithi092 bash[58855]: cluster 2023-09-23T20:07:42.801147+0000 mgr.y (mgr.25325) 1816 : cluster 0 pgmap v1910: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:44.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:07:44 smithi092 bash[58855]: audit 2023-09-23T20:07:43.708157+0000 mon.a (mon.0) 2172 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:07:46.035 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:07:45 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:20:07:45] "GET /metrics HTTP/1.1" 200 34016 "" "Prometheus/2.33.4" 2023-09-23T20:07:46.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:07:46 smithi016 bash[75004]: cluster 2023-09-23T20:07:44.801881+0000 mgr.y (mgr.25325) 1817 : cluster 0 pgmap v1911: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:46.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:07:46 smithi016 bash[77995]: cluster 2023-09-23T20:07:44.801881+0000 mgr.y (mgr.25325) 1817 : cluster 0 pgmap v1911: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:46.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:07:46 smithi092 bash[58855]: cluster 2023-09-23T20:07:44.801881+0000 mgr.y (mgr.25325) 1817 : cluster 0 pgmap v1911: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:48.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:07:48 smithi016 bash[75004]: cluster 2023-09-23T20:07:46.802876+0000 mgr.y (mgr.25325) 1818 : cluster 0 pgmap v1912: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:48.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:07:48 smithi016 bash[77995]: cluster 2023-09-23T20:07:46.802876+0000 mgr.y (mgr.25325) 1818 : cluster 0 pgmap v1912: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:48.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:07:48 smithi092 bash[58855]: cluster 2023-09-23T20:07:46.802876+0000 mgr.y (mgr.25325) 1818 : cluster 0 pgmap v1912: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:49.176 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:07:50.592 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:07:50 smithi016 bash[77995]: cluster 2023-09-23T20:07:48.803658+0000 mgr.y (mgr.25325) 1819 : cluster 0 pgmap v1913: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:50.592 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:07:50 smithi016 bash[75004]: cluster 2023-09-23T20:07:48.803658+0000 mgr.y (mgr.25325) 1819 : cluster 0 pgmap v1913: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:50.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:07:50 smithi092 bash[58855]: cluster 2023-09-23T20:07:48.803658+0000 mgr.y (mgr.25325) 1819 : cluster 0 pgmap v1913: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:52.767 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-09-23T20:07:52.770 INFO:teuthology.orchestra.run.smithi016.stderr:dumped all 2023-09-23T20:07:52.785 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:07:52 smithi016 bash[77995]: cluster 2023-09-23T20:07:50.804660+0000 mgr.y (mgr.25325) 1820 : cluster 0 pgmap v1914: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:52.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:07:52 smithi016 bash[75004]: cluster 2023-09-23T20:07:50.804660+0000 mgr.y (mgr.25325) 1820 : cluster 0 pgmap v1914: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:52.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:07:52 smithi092 bash[58855]: cluster 2023-09-23T20:07:50.804660+0000 mgr.y (mgr.25325) 1820 : cluster 0 pgmap v1914: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:53.681 INFO:teuthology.orchestra.run.smithi016.stdout:{"pg_ready":true,"pg_map":{"version":1914,"stamp":"2023-09-23T20:07:50.803983+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6198,"num_read_kb":12633,"num_write":1107,"num_write_kb":15160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":73,"num_bytes_recovered":4194323,"num_keys_recovered":2,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1171,"ondisk_log_size":1171,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":7951224,"kb_used_data":40120,"kb_used_omap":20,"kb_used_meta":7911019,"kb_avail":742009992,"statfs":{"total":767960285184,"available":759818231808,"internally_reserved":0,"allocated":41082880,"data_stored":33545994,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":8100884110},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004657"},"pg_stats":[{"pgid":"3.1a","version":"59'1","reported_seq":347,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.642865+0000","last_change":"2023-09-23T20:03:14.491154+0000","last_active":"2023-09-23T20:06:19.642865+0000","last_peered":"2023-09-23T20:06:19.642865+0000","last_clean":"2023-09-23T20:06:19.642865+0000","last_became_active":"2023-09-23T20:03:14.490786+0000","last_became_peered":"2023-09-23T20:03:14.490786+0000","last_unstale":"2023-09-23T20:06:19.642865+0000","last_undegraded":"2023-09-23T20:06:19.642865+0000","last_fullsized":"2023-09-23T20:06:19.642865+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":109,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:06:32.811282+0000","last_change":"2023-09-23T20:04:47.030349+0000","last_active":"2023-09-23T20:06:32.811282+0000","last_peered":"2023-09-23T20:06:32.811282+0000","last_clean":"2023-09-23T20:06:32.811282+0000","last_became_active":"2023-09-23T20:04:47.029966+0000","last_became_peered":"2023-09-23T20:04:47.029966+0000","last_unstale":"2023-09-23T20:06:32.811282+0000","last_undegraded":"2023-09-23T20:06:32.811282+0000","last_fullsized":"2023-09-23T20:06:32.811282+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":322,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:46.639171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":38,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:18.930726+0000","last_change":"2023-09-23T20:03:14.480773+0000","last_active":"2023-09-23T20:06:18.930726+0000","last_peered":"2023-09-23T20:06:18.930726+0000","last_clean":"2023-09-23T20:06:18.930726+0000","last_became_active":"2023-09-23T20:03:14.480401+0000","last_became_peered":"2023-09-23T20:03:14.480401+0000","last_unstale":"2023-09-23T20:06:18.930726+0000","last_undegraded":"2023-09-23T20:06:18.930726+0000","last_fullsized":"2023-09-23T20:06:18.930726+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:52:53.701767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":178,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.223566+0000","last_change":"2023-09-23T20:04:06.716877+0000","last_active":"2023-09-23T20:06:19.223566+0000","last_peered":"2023-09-23T20:06:19.223566+0000","last_clean":"2023-09-23T20:06:19.223566+0000","last_became_active":"2023-09-23T20:04:06.716273+0000","last_became_peered":"2023-09-23T20:04:06.716273+0000","last_unstale":"2023-09-23T20:06:19.223566+0000","last_undegraded":"2023-09-23T20:06:19.223566+0000","last_fullsized":"2023-09-23T20:06:19.223566+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:54:45.068172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":322,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:06:32.810631+0000","last_change":"2023-09-23T20:04:47.051743+0000","last_active":"2023-09-23T20:06:32.810631+0000","last_peered":"2023-09-23T20:06:32.810631+0000","last_clean":"2023-09-23T20:06:32.810631+0000","last_became_active":"2023-09-23T20:04:47.050630+0000","last_became_peered":"2023-09-23T20:04:47.050630+0000","last_unstale":"2023-09-23T20:06:32.810631+0000","last_undegraded":"2023-09-23T20:06:32.810631+0000","last_fullsized":"2023-09-23T20:06:32.810631+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":322,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:53:53.798779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":316,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.184793+0000","last_change":"2023-09-23T20:03:14.487145+0000","last_active":"2023-09-23T20:06:19.184793+0000","last_peered":"2023-09-23T20:06:19.184793+0000","last_clean":"2023-09-23T20:06:19.184793+0000","last_became_active":"2023-09-23T20:03:14.486753+0000","last_became_peered":"2023-09-23T20:03:14.486753+0000","last_unstale":"2023-09-23T20:06:19.184793+0000","last_undegraded":"2023-09-23T20:06:19.184793+0000","last_fullsized":"2023-09-23T20:06:19.184793+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":210,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:06:31.976185+0000","last_change":"2023-09-23T20:04:06.516984+0000","last_active":"2023-09-23T20:06:31.976185+0000","last_peered":"2023-09-23T20:06:31.976185+0000","last_clean":"2023-09-23T20:06:31.976185+0000","last_became_active":"2023-09-23T20:04:06.516558+0000","last_became_peered":"2023-09-23T20:04:06.516558+0000","last_unstale":"2023-09-23T20:06:31.976185+0000","last_undegraded":"2023-09-23T20:06:31.976185+0000","last_fullsized":"2023-09-23T20:06:31.976185+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:13.722150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":110,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:06:32.810663+0000","last_change":"2023-09-23T20:04:47.051162+0000","last_active":"2023-09-23T20:06:32.810663+0000","last_peered":"2023-09-23T20:06:32.810663+0000","last_clean":"2023-09-23T20:06:32.810663+0000","last_became_active":"2023-09-23T20:04:47.049811+0000","last_became_peered":"2023-09-23T20:04:47.049811+0000","last_unstale":"2023-09-23T20:06:32.810663+0000","last_undegraded":"2023-09-23T20:06:32.810663+0000","last_fullsized":"2023-09-23T20:06:32.810663+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":322,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:44.815308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":303,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.233076+0000","last_change":"2023-09-23T20:03:14.488499+0000","last_active":"2023-09-23T20:06:19.233076+0000","last_peered":"2023-09-23T20:06:19.233076+0000","last_clean":"2023-09-23T20:06:19.233076+0000","last_became_active":"2023-09-23T20:03:14.488096+0000","last_became_peered":"2023-09-23T20:03:14.488096+0000","last_unstale":"2023-09-23T20:06:19.233076+0000","last_undegraded":"2023-09-23T20:06:19.233076+0000","last_fullsized":"2023-09-23T20:06:19.233076+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":96,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:06:32.810778+0000","last_change":"2023-09-23T20:04:47.029088+0000","last_active":"2023-09-23T20:06:32.810778+0000","last_peered":"2023-09-23T20:06:32.810778+0000","last_clean":"2023-09-23T20:06:32.810778+0000","last_became_active":"2023-09-23T20:04:47.028684+0000","last_became_peered":"2023-09-23T20:04:47.028684+0000","last_unstale":"2023-09-23T20:06:32.810778+0000","last_undegraded":"2023-09-23T20:06:32.810778+0000","last_fullsized":"2023-09-23T20:06:32.810778+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":322,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:36:34.533261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":302,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.232242+0000","last_change":"2023-09-23T19:28:07.892835+0000","last_active":"2023-09-23T20:06:19.232242+0000","last_peered":"2023-09-23T20:06:19.232242+0000","last_clean":"2023-09-23T20:06:19.232242+0000","last_became_active":"2023-09-23T19:28:07.892338+0000","last_became_peered":"2023-09-23T19:28:07.892338+0000","last_unstale":"2023-09-23T20:06:19.232242+0000","last_undegraded":"2023-09-23T20:06:19.232242+0000","last_fullsized":"2023-09-23T20:06:19.232242+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":38,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:18.931507+0000","last_change":"2023-09-23T20:03:14.483257+0000","last_active":"2023-09-23T20:06:18.931507+0000","last_peered":"2023-09-23T20:06:18.931507+0000","last_clean":"2023-09-23T20:06:18.931507+0000","last_became_active":"2023-09-23T20:03:14.482217+0000","last_became_peered":"2023-09-23T20:03:14.482217+0000","last_unstale":"2023-09-23T20:06:18.931507+0000","last_undegraded":"2023-09-23T20:06:18.931507+0000","last_fullsized":"2023-09-23T20:06:18.931507+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:24:19.853891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'23","reported_seq":306,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.643166+0000","last_change":"2023-09-23T19:05:52.107727+0000","last_active":"2023-09-23T20:06:19.643166+0000","last_peered":"2023-09-23T20:06:19.643166+0000","last_clean":"2023-09-23T20:06:19.643166+0000","last_became_active":"2023-09-23T19:05:52.107378+0000","last_became_peered":"2023-09-23T19:05:52.107378+0000","last_unstale":"2023-09-23T20:06:19.643166+0000","last_undegraded":"2023-09-23T20:06:19.643166+0000","last_fullsized":"2023-09-23T20:06:19.643166+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":315,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.643461+0000","last_change":"2023-09-23T20:03:14.490617+0000","last_active":"2023-09-23T20:06:19.643461+0000","last_peered":"2023-09-23T20:06:19.643461+0000","last_clean":"2023-09-23T20:06:19.643461+0000","last_became_active":"2023-09-23T20:03:14.490198+0000","last_became_peered":"2023-09-23T20:03:14.490198+0000","last_unstale":"2023-09-23T20:06:19.643461+0000","last_undegraded":"2023-09-23T20:06:19.643461+0000","last_fullsized":"2023-09-23T20:06:19.643461+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":316,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.231985+0000","last_change":"2023-09-23T19:38:43.897378+0000","last_active":"2023-09-23T20:06:19.231985+0000","last_peered":"2023-09-23T20:06:19.231985+0000","last_clean":"2023-09-23T20:06:19.231985+0000","last_became_active":"2023-09-23T19:38:43.897012+0000","last_became_peered":"2023-09-23T19:38:43.897012+0000","last_unstale":"2023-09-23T20:06:19.231985+0000","last_undegraded":"2023-09-23T20:06:19.231985+0000","last_fullsized":"2023-09-23T20:06:19.231985+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":110,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:06:32.811215+0000","last_change":"2023-09-23T20:04:47.051062+0000","last_active":"2023-09-23T20:06:32.811215+0000","last_peered":"2023-09-23T20:06:32.811215+0000","last_clean":"2023-09-23T20:06:32.811215+0000","last_became_active":"2023-09-23T20:04:47.050134+0000","last_became_peered":"2023-09-23T20:04:47.050134+0000","last_unstale":"2023-09-23T20:06:32.811215+0000","last_undegraded":"2023-09-23T20:06:32.811215+0000","last_fullsized":"2023-09-23T20:06:32.811215+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":322,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:31:20.560311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":176,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:27.140676+0000","last_change":"2023-09-23T20:04:06.716624+0000","last_active":"2023-09-23T20:06:27.140676+0000","last_peered":"2023-09-23T20:06:27.140676+0000","last_clean":"2023-09-23T20:06:27.140676+0000","last_became_active":"2023-09-23T20:04:06.715866+0000","last_became_peered":"2023-09-23T20:04:06.715866+0000","last_unstale":"2023-09-23T20:06:27.140676+0000","last_undegraded":"2023-09-23T20:06:27.140676+0000","last_fullsized":"2023-09-23T20:06:27.140676+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:35:05.633686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":230,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.232877+0000","last_change":"2023-09-23T20:05:28.472296+0000","last_active":"2023-09-23T20:06:19.232877+0000","last_peered":"2023-09-23T20:06:19.232877+0000","last_clean":"2023-09-23T20:06:19.232877+0000","last_became_active":"2023-09-23T20:05:28.471242+0000","last_became_peered":"2023-09-23T20:05:28.471242+0000","last_unstale":"2023-09-23T20:06:19.232877+0000","last_undegraded":"2023-09-23T20:06:19.232877+0000","last_fullsized":"2023-09-23T20:06:19.232877+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":284,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.231337+0000","last_change":"2023-09-23T19:05:52.114186+0000","last_active":"2023-09-23T20:06:19.231337+0000","last_peered":"2023-09-23T20:06:19.231337+0000","last_clean":"2023-09-23T20:06:19.231337+0000","last_became_active":"2023-09-23T19:05:52.113635+0000","last_became_peered":"2023-09-23T19:05:52.113635+0000","last_unstale":"2023-09-23T20:06:19.231337+0000","last_undegraded":"2023-09-23T20:06:19.231337+0000","last_fullsized":"2023-09-23T20:06:19.231337+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":176,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:06:31.976012+0000","last_change":"2023-09-23T20:04:06.718614+0000","last_active":"2023-09-23T20:06:31.976012+0000","last_peered":"2023-09-23T20:06:31.976012+0000","last_clean":"2023-09-23T20:06:31.976012+0000","last_became_active":"2023-09-23T20:04:06.716485+0000","last_became_peered":"2023-09-23T20:04:06.716485+0000","last_unstale":"2023-09-23T20:06:31.976012+0000","last_undegraded":"2023-09-23T20:06:31.976012+0000","last_fullsized":"2023-09-23T20:06:31.976012+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:49.567649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":307,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.183953+0000","last_change":"2023-09-23T19:18:09.010195+0000","last_active":"2023-09-23T20:06:19.183953+0000","last_peered":"2023-09-23T20:06:19.183953+0000","last_clean":"2023-09-23T20:06:19.183953+0000","last_became_active":"2023-09-23T19:18:09.009687+0000","last_became_peered":"2023-09-23T19:18:09.009687+0000","last_unstale":"2023-09-23T20:06:19.183953+0000","last_undegraded":"2023-09-23T20:06:19.183953+0000","last_fullsized":"2023-09-23T20:06:19.183953+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":336,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:06:32.811499+0000","last_change":"2023-09-23T20:03:45.931421+0000","last_active":"2023-09-23T20:06:32.811499+0000","last_peered":"2023-09-23T20:06:32.811499+0000","last_clean":"2023-09-23T20:06:32.811499+0000","last_became_active":"2023-09-23T20:03:45.930726+0000","last_became_peered":"2023-09-23T20:03:45.930726+0000","last_unstale":"2023-09-23T20:06:32.811499+0000","last_undegraded":"2023-09-23T20:06:32.811499+0000","last_fullsized":"2023-09-23T20:06:32.811499+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:28:33.412847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":300,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.231987+0000","last_change":"2023-09-23T19:18:09.035116+0000","last_active":"2023-09-23T20:06:19.231987+0000","last_peered":"2023-09-23T20:06:19.231987+0000","last_clean":"2023-09-23T20:06:19.231987+0000","last_became_active":"2023-09-23T19:18:09.034416+0000","last_became_peered":"2023-09-23T19:18:09.034416+0000","last_unstale":"2023-09-23T20:06:19.231987+0000","last_undegraded":"2023-09-23T20:06:19.231987+0000","last_fullsized":"2023-09-23T20:06:19.231987+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":269,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.232445+0000","last_change":"2023-09-23T19:47:56.790227+0000","last_active":"2023-09-23T20:06:19.232445+0000","last_peered":"2023-09-23T20:06:19.232445+0000","last_clean":"2023-09-23T20:06:19.232445+0000","last_became_active":"2023-09-23T19:47:56.789727+0000","last_became_peered":"2023-09-23T19:47:56.789727+0000","last_unstale":"2023-09-23T20:06:19.232445+0000","last_undegraded":"2023-09-23T20:06:19.232445+0000","last_fullsized":"2023-09-23T20:06:19.232445+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":327,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.231991+0000","last_change":"2023-09-23T20:03:14.500188+0000","last_active":"2023-09-23T20:06:19.231991+0000","last_peered":"2023-09-23T20:06:19.231991+0000","last_clean":"2023-09-23T20:06:19.231991+0000","last_became_active":"2023-09-23T20:03:14.489458+0000","last_became_peered":"2023-09-23T20:03:14.489458+0000","last_unstale":"2023-09-23T20:06:19.231991+0000","last_undegraded":"2023-09-23T20:06:19.231991+0000","last_fullsized":"2023-09-23T20:06:19.231991+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":330,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.643130+0000","last_change":"2023-09-23T20:03:14.488674+0000","last_active":"2023-09-23T20:06:19.643130+0000","last_peered":"2023-09-23T20:06:19.643130+0000","last_clean":"2023-09-23T20:06:19.643130+0000","last_became_active":"2023-09-23T20:03:14.488290+0000","last_became_peered":"2023-09-23T20:03:14.488290+0000","last_unstale":"2023-09-23T20:06:19.643130+0000","last_undegraded":"2023-09-23T20:06:19.643130+0000","last_fullsized":"2023-09-23T20:06:19.643130+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":285,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.231719+0000","last_change":"2023-09-23T19:05:52.114108+0000","last_active":"2023-09-23T20:06:19.231719+0000","last_peered":"2023-09-23T20:06:19.231719+0000","last_clean":"2023-09-23T20:06:19.231719+0000","last_became_active":"2023-09-23T19:05:52.113609+0000","last_became_peered":"2023-09-23T19:05:52.113609+0000","last_unstale":"2023-09-23T20:06:19.231719+0000","last_undegraded":"2023-09-23T20:06:19.231719+0000","last_fullsized":"2023-09-23T20:06:19.231719+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":316,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.184438+0000","last_change":"2023-09-23T20:03:14.493455+0000","last_active":"2023-09-23T20:06:19.184438+0000","last_peered":"2023-09-23T20:06:19.184438+0000","last_clean":"2023-09-23T20:06:19.184438+0000","last_became_active":"2023-09-23T20:03:14.493053+0000","last_became_peered":"2023-09-23T20:03:14.493053+0000","last_unstale":"2023-09-23T20:06:19.184438+0000","last_undegraded":"2023-09-23T20:06:19.184438+0000","last_fullsized":"2023-09-23T20:06:19.184438+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":231,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.232879+0000","last_change":"2023-09-23T20:05:28.472196+0000","last_active":"2023-09-23T20:06:19.232879+0000","last_peered":"2023-09-23T20:06:19.232879+0000","last_clean":"2023-09-23T20:06:19.232879+0000","last_became_active":"2023-09-23T20:05:28.471259+0000","last_became_peered":"2023-09-23T20:05:28.471259+0000","last_unstale":"2023-09-23T20:06:19.232879+0000","last_undegraded":"2023-09-23T20:06:19.232879+0000","last_fullsized":"2023-09-23T20:06:19.232879+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":324,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:06:31.975993+0000","last_change":"2023-09-23T20:03:14.484103+0000","last_active":"2023-09-23T20:06:31.975993+0000","last_peered":"2023-09-23T20:06:31.975993+0000","last_clean":"2023-09-23T20:06:31.975993+0000","last_became_active":"2023-09-23T20:03:14.483704+0000","last_became_peered":"2023-09-23T20:03:14.483704+0000","last_unstale":"2023-09-23T20:06:31.975993+0000","last_undegraded":"2023-09-23T20:06:31.975993+0000","last_fullsized":"2023-09-23T20:06:31.975993+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:30:14.811132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":100,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:06:32.811190+0000","last_change":"2023-09-23T20:04:47.030508+0000","last_active":"2023-09-23T20:06:32.811190+0000","last_peered":"2023-09-23T20:06:32.811190+0000","last_clean":"2023-09-23T20:06:32.811190+0000","last_became_active":"2023-09-23T20:04:47.030061+0000","last_became_peered":"2023-09-23T20:04:47.030061+0000","last_unstale":"2023-09-23T20:06:32.811190+0000","last_undegraded":"2023-09-23T20:06:32.811190+0000","last_fullsized":"2023-09-23T20:06:32.811190+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":322,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:21.053552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":297,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.643314+0000","last_change":"2023-09-23T19:28:07.896803+0000","last_active":"2023-09-23T20:06:19.643314+0000","last_peered":"2023-09-23T20:06:19.643314+0000","last_clean":"2023-09-23T20:06:19.643314+0000","last_became_active":"2023-09-23T19:28:07.896480+0000","last_became_peered":"2023-09-23T19:28:07.896480+0000","last_unstale":"2023-09-23T20:06:19.643314+0000","last_undegraded":"2023-09-23T20:06:19.643314+0000","last_fullsized":"2023-09-23T20:06:19.643314+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":317,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.184414+0000","last_change":"2023-09-23T20:03:14.493451+0000","last_active":"2023-09-23T20:06:19.184414+0000","last_peered":"2023-09-23T20:06:19.184414+0000","last_clean":"2023-09-23T20:06:19.184414+0000","last_became_active":"2023-09-23T20:03:14.492920+0000","last_became_peered":"2023-09-23T20:03:14.492920+0000","last_unstale":"2023-09-23T20:06:19.184414+0000","last_undegraded":"2023-09-23T20:06:19.184414+0000","last_fullsized":"2023-09-23T20:06:19.184414+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":354,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.643292+0000","last_change":"2023-09-23T20:03:45.935493+0000","last_active":"2023-09-23T20:06:19.643292+0000","last_peered":"2023-09-23T20:06:19.643292+0000","last_clean":"2023-09-23T20:06:19.643292+0000","last_became_active":"2023-09-23T20:03:45.934909+0000","last_became_peered":"2023-09-23T20:03:45.934909+0000","last_unstale":"2023-09-23T20:06:19.643292+0000","last_undegraded":"2023-09-23T20:06:19.643292+0000","last_fullsized":"2023-09-23T20:06:19.643292+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":313,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.642665+0000","last_change":"2023-09-23T19:38:43.892972+0000","last_active":"2023-09-23T20:06:19.642665+0000","last_peered":"2023-09-23T20:06:19.642665+0000","last_clean":"2023-09-23T20:06:19.642665+0000","last_became_active":"2023-09-23T19:38:43.892451+0000","last_became_peered":"2023-09-23T19:38:43.892451+0000","last_unstale":"2023-09-23T20:06:19.642665+0000","last_undegraded":"2023-09-23T20:06:19.642665+0000","last_fullsized":"2023-09-23T20:06:19.642665+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":315,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.643815+0000","last_change":"2023-09-23T20:03:14.490494+0000","last_active":"2023-09-23T20:06:19.643815+0000","last_peered":"2023-09-23T20:06:19.643815+0000","last_clean":"2023-09-23T20:06:19.643815+0000","last_became_active":"2023-09-23T20:03:14.490147+0000","last_became_peered":"2023-09-23T20:03:14.490147+0000","last_unstale":"2023-09-23T20:06:19.643815+0000","last_undegraded":"2023-09-23T20:06:19.643815+0000","last_fullsized":"2023-09-23T20:06:19.643815+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":111,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:06:32.810886+0000","last_change":"2023-09-23T20:04:47.051293+0000","last_active":"2023-09-23T20:06:32.810886+0000","last_peered":"2023-09-23T20:06:32.810886+0000","last_clean":"2023-09-23T20:06:32.810886+0000","last_became_active":"2023-09-23T20:04:47.050289+0000","last_became_peered":"2023-09-23T20:04:47.050289+0000","last_unstale":"2023-09-23T20:06:32.810886+0000","last_undegraded":"2023-09-23T20:06:32.810886+0000","last_fullsized":"2023-09-23T20:06:32.810886+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":322,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:29:02.007222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":39,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:06:33.375211+0000","last_change":"2023-09-23T20:03:14.483670+0000","last_active":"2023-09-23T20:06:33.375211+0000","last_peered":"2023-09-23T20:06:33.375211+0000","last_clean":"2023-09-23T20:06:33.375211+0000","last_became_active":"2023-09-23T20:03:14.483318+0000","last_became_peered":"2023-09-23T20:03:14.483318+0000","last_unstale":"2023-09-23T20:06:33.375211+0000","last_undegraded":"2023-09-23T20:06:33.375211+0000","last_fullsized":"2023-09-23T20:06:33.375211+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:47.398395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"303'174","reported_seq":883,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:06:33.375295+0000","last_change":"2023-09-23T20:03:17.498400+0000","last_active":"2023-09-23T20:06:33.375295+0000","last_peered":"2023-09-23T20:06:33.375295+0000","last_clean":"2023-09-23T20:06:33.375295+0000","last_became_active":"2023-09-23T20:03:17.497416+0000","last_became_peered":"2023-09-23T20:03:17.497416+0000","last_unstale":"2023-09-23T20:06:33.375295+0000","last_undegraded":"2023-09-23T20:06:33.375295+0000","last_fullsized":"2023-09-23T20:06:33.375295+0000","mapping_epoch":309,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":310,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:12.558962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":804,"num_read_kb":2995,"num_write":184,"num_write_kb":2268,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":0,"num_keys_recovered":2,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":315,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.232948+0000","last_change":"2023-09-23T19:47:56.794437+0000","last_active":"2023-09-23T20:06:19.232948+0000","last_peered":"2023-09-23T20:06:19.232948+0000","last_clean":"2023-09-23T20:06:19.232948+0000","last_became_active":"2023-09-23T19:47:56.794118+0000","last_became_peered":"2023-09-23T19:47:56.794118+0000","last_unstale":"2023-09-23T20:06:19.232948+0000","last_undegraded":"2023-09-23T20:06:19.232948+0000","last_fullsized":"2023-09-23T20:06:19.232948+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":39,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:06:33.374519+0000","last_change":"2023-09-23T20:03:14.483749+0000","last_active":"2023-09-23T20:06:33.374519+0000","last_peered":"2023-09-23T20:06:33.374519+0000","last_clean":"2023-09-23T20:06:33.374519+0000","last_became_active":"2023-09-23T20:03:14.483398+0000","last_became_peered":"2023-09-23T20:03:14.483398+0000","last_unstale":"2023-09-23T20:06:33.374519+0000","last_undegraded":"2023-09-23T20:06:33.374519+0000","last_fullsized":"2023-09-23T20:06:33.374519+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:40:43.487777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"300'87","reported_seq":599,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.232615+0000","last_change":"2023-09-23T20:05:28.221260+0000","last_active":"2023-09-23T20:06:19.232615+0000","last_peered":"2023-09-23T20:06:19.232615+0000","last_clean":"2023-09-23T20:06:19.232615+0000","last_became_active":"2023-09-23T20:05:28.220457+0000","last_became_peered":"2023-09-23T20:05:28.220457+0000","last_unstale":"2023-09-23T20:06:19.232615+0000","last_undegraded":"2023-09-23T20:06:19.232615+0000","last_fullsized":"2023-09-23T20:06:19.232615+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":562,"num_read_kb":518,"num_write":90,"num_write_kb":1091,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":298,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.643102+0000","last_change":"2023-09-23T19:18:09.017525+0000","last_active":"2023-09-23T20:06:19.643102+0000","last_peered":"2023-09-23T20:06:19.643102+0000","last_clean":"2023-09-23T20:06:19.643102+0000","last_became_active":"2023-09-23T19:18:09.017193+0000","last_became_peered":"2023-09-23T19:18:09.017193+0000","last_unstale":"2023-09-23T20:06:19.643102+0000","last_undegraded":"2023-09-23T20:06:19.643102+0000","last_fullsized":"2023-09-23T20:06:19.643102+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":39,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:06:33.375254+0000","last_change":"2023-09-23T20:03:14.471701+0000","last_active":"2023-09-23T20:06:33.375254+0000","last_peered":"2023-09-23T20:06:33.375254+0000","last_clean":"2023-09-23T20:06:33.375254+0000","last_became_active":"2023-09-23T20:03:14.469272+0000","last_became_peered":"2023-09-23T20:03:14.469272+0000","last_unstale":"2023-09-23T20:06:33.375254+0000","last_undegraded":"2023-09-23T20:06:33.375254+0000","last_fullsized":"2023-09-23T20:06:33.375254+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:01.913425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"300'76","reported_seq":634,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.357481+0000","last_change":"2023-09-23T20:02:46.940411+0000","last_active":"2023-09-23T20:06:19.357481+0000","last_peered":"2023-09-23T20:06:19.357481+0000","last_clean":"2023-09-23T20:06:19.357481+0000","last_became_active":"2023-09-23T19:38:43.882421+0000","last_became_peered":"2023-09-23T19:38:43.882421+0000","last_unstale":"2023-09-23T20:06:19.357481+0000","last_undegraded":"2023-09-23T20:06:19.357481+0000","last_fullsized":"2023-09-23T20:06:19.357481+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":76,"log_dups_size":0,"ondisk_log_size":76,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:49:01.607659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099725700000000009,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":533,"num_read_kb":1101,"num_write":65,"num_write_kb":387,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":39,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:06:33.374588+0000","last_change":"2023-09-23T20:03:14.481700+0000","last_active":"2023-09-23T20:06:33.374588+0000","last_peered":"2023-09-23T20:06:33.374588+0000","last_clean":"2023-09-23T20:06:33.374588+0000","last_became_active":"2023-09-23T20:03:14.480935+0000","last_became_peered":"2023-09-23T20:03:14.480935+0000","last_unstale":"2023-09-23T20:06:33.374588+0000","last_undegraded":"2023-09-23T20:06:33.374588+0000","last_fullsized":"2023-09-23T20:06:33.374588+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:13.224166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":39,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:06:33.375353+0000","last_change":"2023-09-23T20:03:14.480704+0000","last_active":"2023-09-23T20:06:33.375353+0000","last_peered":"2023-09-23T20:06:33.375353+0000","last_clean":"2023-09-23T20:06:33.375353+0000","last_became_active":"2023-09-23T20:03:14.480216+0000","last_became_peered":"2023-09-23T20:03:14.480216+0000","last_unstale":"2023-09-23T20:06:33.375353+0000","last_undegraded":"2023-09-23T20:06:33.375353+0000","last_fullsized":"2023-09-23T20:06:33.375353+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:40:51.272381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"303'139","reported_seq":640,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.232694+0000","last_change":"2023-09-23T20:02:47.058854+0000","last_active":"2023-09-23T20:06:19.232694+0000","last_peered":"2023-09-23T20:06:19.232694+0000","last_clean":"2023-09-23T20:06:19.232694+0000","last_became_active":"2023-09-23T19:47:56.790954+0000","last_became_peered":"2023-09-23T19:47:56.790954+0000","last_unstale":"2023-09-23T20:06:19.232694+0000","last_undegraded":"2023-09-23T20:06:19.232694+0000","last_fullsized":"2023-09-23T20:06:19.232694+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":388,"num_read_kb":340,"num_write":136,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":535,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.231173+0000","last_change":"2023-09-23T19:18:09.016297+0000","last_active":"2023-09-23T20:06:19.231173+0000","last_peered":"2023-09-23T20:06:19.231173+0000","last_clean":"2023-09-23T20:06:19.231173+0000","last_became_active":"2023-09-23T19:18:09.015436+0000","last_became_peered":"2023-09-23T19:18:09.015436+0000","last_unstale":"2023-09-23T20:06:19.231173+0000","last_undegraded":"2023-09-23T20:06:19.231173+0000","last_fullsized":"2023-09-23T20:06:19.231173+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:59:33.840751+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:59:33.840751+0000","last_clean_scrub_stamp":"2023-09-23T18:59:33.840751+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":157,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:06:32.811612+0000","last_change":"2023-09-23T20:04:47.050132+0000","last_active":"2023-09-23T20:06:32.811612+0000","last_peered":"2023-09-23T20:06:32.811612+0000","last_clean":"2023-09-23T20:06:32.811612+0000","last_became_active":"2023-09-23T20:04:47.049460+0000","last_became_peered":"2023-09-23T20:04:47.049460+0000","last_unstale":"2023-09-23T20:06:32.811612+0000","last_undegraded":"2023-09-23T20:06:32.811612+0000","last_fullsized":"2023-09-23T20:06:32.811612+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":322,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:56.748778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":283,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.184926+0000","last_change":"2023-09-23T19:05:52.786771+0000","last_active":"2023-09-23T20:06:19.184926+0000","last_peered":"2023-09-23T20:06:19.184926+0000","last_clean":"2023-09-23T20:06:19.184926+0000","last_became_active":"2023-09-23T19:05:52.785797+0000","last_became_peered":"2023-09-23T19:05:52.785797+0000","last_unstale":"2023-09-23T20:06:19.184926+0000","last_undegraded":"2023-09-23T20:06:19.184926+0000","last_fullsized":"2023-09-23T20:06:19.184926+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"302'89","reported_seq":740,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.231144+0000","last_change":"2023-09-23T20:03:14.895983+0000","last_active":"2023-09-23T20:06:19.231144+0000","last_peered":"2023-09-23T20:06:19.231144+0000","last_clean":"2023-09-23T20:06:19.231144+0000","last_became_active":"2023-09-23T20:03:14.499439+0000","last_became_peered":"2023-09-23T20:03:14.499439+0000","last_unstale":"2023-09-23T20:06:19.231144+0000","last_undegraded":"2023-09-23T20:06:19.231144+0000","last_fullsized":"2023-09-23T20:06:19.231144+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":89,"log_dups_size":0,"ondisk_log_size":89,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":702,"num_read_kb":610,"num_write":76,"num_write_kb":51,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":177,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.223014+0000","last_change":"2023-09-23T20:04:06.491671+0000","last_active":"2023-09-23T20:06:19.223014+0000","last_peered":"2023-09-23T20:06:19.223014+0000","last_clean":"2023-09-23T20:06:19.223014+0000","last_became_active":"2023-09-23T20:04:06.491297+0000","last_became_peered":"2023-09-23T20:04:06.491297+0000","last_unstale":"2023-09-23T20:06:19.223014+0000","last_undegraded":"2023-09-23T20:06:19.223014+0000","last_fullsized":"2023-09-23T20:06:19.223014+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:30.618468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":354,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.223739+0000","last_change":"2023-09-23T20:03:14.485798+0000","last_active":"2023-09-23T20:06:19.223739+0000","last_peered":"2023-09-23T20:06:19.223739+0000","last_clean":"2023-09-23T20:06:19.223739+0000","last_became_active":"2023-09-23T20:03:14.485232+0000","last_became_peered":"2023-09-23T20:03:14.485232+0000","last_unstale":"2023-09-23T20:06:19.223739+0000","last_undegraded":"2023-09-23T20:06:19.223739+0000","last_fullsized":"2023-09-23T20:06:19.223739+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:37.416281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"303'146","reported_seq":896,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.231975+0000","last_change":"2023-09-23T20:02:46.954392+0000","last_active":"2023-09-23T20:06:19.231975+0000","last_peered":"2023-09-23T20:06:19.231975+0000","last_clean":"2023-09-23T20:06:19.231975+0000","last_became_active":"2023-09-23T19:18:09.015338+0000","last_became_peered":"2023-09-23T19:18:09.015338+0000","last_unstale":"2023-09-23T20:06:19.231975+0000","last_undegraded":"2023-09-23T20:06:19.231975+0000","last_fullsized":"2023-09-23T20:06:19.231975+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":146,"log_dups_size":0,"ondisk_log_size":146,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1086,"num_read_kb":2403,"num_write":113,"num_write_kb":1312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":39,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:06:33.374948+0000","last_change":"2023-09-23T20:03:14.476900+0000","last_active":"2023-09-23T20:06:33.374948+0000","last_peered":"2023-09-23T20:06:33.374948+0000","last_clean":"2023-09-23T20:06:33.374948+0000","last_became_active":"2023-09-23T20:03:14.475909+0000","last_became_peered":"2023-09-23T20:03:14.475909+0000","last_unstale":"2023-09-23T20:06:33.374948+0000","last_undegraded":"2023-09-23T20:06:33.374948+0000","last_fullsized":"2023-09-23T20:06:33.374948+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:29.191512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":162,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.223375+0000","last_change":"2023-09-23T20:04:06.716754+0000","last_active":"2023-09-23T20:06:19.223375+0000","last_peered":"2023-09-23T20:06:19.223375+0000","last_clean":"2023-09-23T20:06:19.223375+0000","last_became_active":"2023-09-23T20:04:06.716156+0000","last_became_peered":"2023-09-23T20:04:06.716156+0000","last_unstale":"2023-09-23T20:06:19.223375+0000","last_undegraded":"2023-09-23T20:06:19.223375+0000","last_fullsized":"2023-09-23T20:06:19.223375+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:54:57.478605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":100,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:06:32.811567+0000","last_change":"2023-09-23T20:04:47.027885+0000","last_active":"2023-09-23T20:06:32.811567+0000","last_peered":"2023-09-23T20:06:32.811567+0000","last_clean":"2023-09-23T20:06:32.811567+0000","last_became_active":"2023-09-23T20:04:47.027383+0000","last_became_peered":"2023-09-23T20:04:47.027383+0000","last_unstale":"2023-09-23T20:06:32.811567+0000","last_undegraded":"2023-09-23T20:06:32.811567+0000","last_fullsized":"2023-09-23T20:06:32.811567+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":322,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:43.621489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"303'124","reported_seq":793,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:06:33.375315+0000","last_change":"2023-09-23T20:03:17.500586+0000","last_active":"2023-09-23T20:06:33.375315+0000","last_peered":"2023-09-23T20:06:33.375315+0000","last_clean":"2023-09-23T20:06:33.375315+0000","last_became_active":"2023-09-23T20:03:17.499921+0000","last_became_peered":"2023-09-23T20:03:17.499921+0000","last_unstale":"2023-09-23T20:06:33.375315+0000","last_undegraded":"2023-09-23T20:06:33.375315+0000","last_fullsized":"2023-09-23T20:06:33.375315+0000","mapping_epoch":309,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":310,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:06:40.172512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":811,"num_read_kb":1848,"num_write":72,"num_write_kb":5235,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":97,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:06:32.810736+0000","last_change":"2023-09-23T20:04:47.030013+0000","last_active":"2023-09-23T20:06:32.810736+0000","last_peered":"2023-09-23T20:06:32.810736+0000","last_clean":"2023-09-23T20:06:32.810736+0000","last_became_active":"2023-09-23T20:04:47.029620+0000","last_became_peered":"2023-09-23T20:04:47.029620+0000","last_unstale":"2023-09-23T20:06:32.810736+0000","last_undegraded":"2023-09-23T20:06:32.810736+0000","last_fullsized":"2023-09-23T20:06:32.810736+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":322,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:19:55.728860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"300'141","reported_seq":896,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.184683+0000","last_change":"2023-09-23T20:02:47.022756+0000","last_active":"2023-09-23T20:06:19.184683+0000","last_peered":"2023-09-23T20:06:19.184683+0000","last_clean":"2023-09-23T20:06:19.184683+0000","last_became_active":"2023-09-23T19:38:43.895486+0000","last_became_peered":"2023-09-23T19:38:43.895486+0000","last_unstale":"2023-09-23T20:06:19.184683+0000","last_undegraded":"2023-09-23T20:06:19.184683+0000","last_fullsized":"2023-09-23T20:06:19.184683+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":141,"log_dups_size":0,"ondisk_log_size":141,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1004,"num_read_kb":2306,"num_write":110,"num_write_kb":1264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":335,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.223538+0000","last_change":"2023-09-23T20:03:14.547047+0000","last_active":"2023-09-23T20:06:19.223538+0000","last_peered":"2023-09-23T20:06:19.223538+0000","last_clean":"2023-09-23T20:06:19.223538+0000","last_became_active":"2023-09-23T20:03:14.486040+0000","last_became_peered":"2023-09-23T20:03:14.486040+0000","last_unstale":"2023-09-23T20:06:19.223538+0000","last_undegraded":"2023-09-23T20:06:19.223538+0000","last_fullsized":"2023-09-23T20:06:19.223538+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:14.044993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":282,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.642706+0000","last_change":"2023-09-23T19:05:52.138620+0000","last_active":"2023-09-23T20:06:19.642706+0000","last_peered":"2023-09-23T20:06:19.642706+0000","last_clean":"2023-09-23T20:06:19.642706+0000","last_became_active":"2023-09-23T19:05:52.138243+0000","last_became_peered":"2023-09-23T19:05:52.138243+0000","last_unstale":"2023-09-23T20:06:19.642706+0000","last_undegraded":"2023-09-23T20:06:19.642706+0000","last_fullsized":"2023-09-23T20:06:19.642706+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":291,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.223139+0000","last_change":"2023-09-23T19:18:09.008811+0000","last_active":"2023-09-23T20:06:19.223139+0000","last_peered":"2023-09-23T20:06:19.223139+0000","last_clean":"2023-09-23T20:06:19.223139+0000","last_became_active":"2023-09-23T19:18:09.007255+0000","last_became_peered":"2023-09-23T19:18:09.007255+0000","last_unstale":"2023-09-23T20:06:19.223139+0000","last_undegraded":"2023-09-23T20:06:19.223139+0000","last_fullsized":"2023-09-23T20:06:19.223139+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:25.874038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":313,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.357659+0000","last_change":"2023-09-23T20:03:14.469248+0000","last_active":"2023-09-23T20:06:19.357659+0000","last_peered":"2023-09-23T20:06:19.357659+0000","last_clean":"2023-09-23T20:06:19.357659+0000","last_became_active":"2023-09-23T20:03:14.468878+0000","last_became_peered":"2023-09-23T20:03:14.468878+0000","last_unstale":"2023-09-23T20:06:19.357659+0000","last_undegraded":"2023-09-23T20:06:19.357659+0000","last_fullsized":"2023-09-23T20:06:19.357659+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:02:59.063744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":177,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.232922+0000","last_change":"2023-09-23T20:05:28.218979+0000","last_active":"2023-09-23T20:06:19.232922+0000","last_peered":"2023-09-23T20:06:19.232922+0000","last_clean":"2023-09-23T20:06:19.232922+0000","last_became_active":"2023-09-23T20:05:28.218478+0000","last_became_peered":"2023-09-23T20:05:28.218478+0000","last_unstale":"2023-09-23T20:06:19.232922+0000","last_undegraded":"2023-09-23T20:06:19.232922+0000","last_fullsized":"2023-09-23T20:06:19.232922+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":345,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.643557+0000","last_change":"2023-09-23T20:03:14.488715+0000","last_active":"2023-09-23T20:06:19.643557+0000","last_peered":"2023-09-23T20:06:19.643557+0000","last_clean":"2023-09-23T20:06:19.643557+0000","last_became_active":"2023-09-23T20:03:14.488323+0000","last_became_peered":"2023-09-23T20:03:14.488323+0000","last_unstale":"2023-09-23T20:06:19.643557+0000","last_undegraded":"2023-09-23T20:06:19.643557+0000","last_fullsized":"2023-09-23T20:06:19.643557+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":218,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.233090+0000","last_change":"2023-09-23T20:05:28.220932+0000","last_active":"2023-09-23T20:06:19.233090+0000","last_peered":"2023-09-23T20:06:19.233090+0000","last_clean":"2023-09-23T20:06:19.233090+0000","last_became_active":"2023-09-23T20:05:28.220462+0000","last_became_peered":"2023-09-23T20:05:28.220462+0000","last_unstale":"2023-09-23T20:06:19.233090+0000","last_undegraded":"2023-09-23T20:06:19.233090+0000","last_fullsized":"2023-09-23T20:06:19.233090+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":312,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.643068+0000","last_change":"2023-09-23T19:28:07.896983+0000","last_active":"2023-09-23T20:06:19.643068+0000","last_peered":"2023-09-23T20:06:19.643068+0000","last_clean":"2023-09-23T20:06:19.643068+0000","last_became_active":"2023-09-23T19:28:07.896577+0000","last_became_peered":"2023-09-23T19:28:07.896577+0000","last_unstale":"2023-09-23T20:06:19.643068+0000","last_undegraded":"2023-09-23T20:06:19.643068+0000","last_fullsized":"2023-09-23T20:06:19.643068+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":124,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:06:32.811414+0000","last_change":"2023-09-23T20:04:47.027195+0000","last_active":"2023-09-23T20:06:32.811414+0000","last_peered":"2023-09-23T20:06:32.811414+0000","last_clean":"2023-09-23T20:06:32.811414+0000","last_became_active":"2023-09-23T20:04:47.026583+0000","last_became_peered":"2023-09-23T20:04:47.026583+0000","last_unstale":"2023-09-23T20:06:32.811414+0000","last_undegraded":"2023-09-23T20:06:32.811414+0000","last_fullsized":"2023-09-23T20:06:32.811414+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":322,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:17:45.781926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":136,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:06:32.810791+0000","last_change":"2023-09-23T20:04:47.019080+0000","last_active":"2023-09-23T20:06:32.810791+0000","last_peered":"2023-09-23T20:06:32.810791+0000","last_clean":"2023-09-23T20:06:32.810791+0000","last_became_active":"2023-09-23T20:04:47.018529+0000","last_became_peered":"2023-09-23T20:04:47.018529+0000","last_unstale":"2023-09-23T20:06:32.810791+0000","last_undegraded":"2023-09-23T20:06:32.810791+0000","last_fullsized":"2023-09-23T20:06:32.810791+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":322,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:13.056894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":325,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.223344+0000","last_change":"2023-09-23T19:38:43.893983+0000","last_active":"2023-09-23T20:06:19.223344+0000","last_peered":"2023-09-23T20:06:19.223344+0000","last_clean":"2023-09-23T20:06:19.223344+0000","last_became_active":"2023-09-23T19:38:43.893527+0000","last_became_peered":"2023-09-23T19:38:43.893527+0000","last_unstale":"2023-09-23T20:06:19.223344+0000","last_undegraded":"2023-09-23T20:06:19.223344+0000","last_fullsized":"2023-09-23T20:06:19.223344+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:12.156531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":309,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.642628+0000","last_change":"2023-09-23T20:03:14.488733+0000","last_active":"2023-09-23T20:06:19.642628+0000","last_peered":"2023-09-23T20:06:19.642628+0000","last_clean":"2023-09-23T20:06:19.642628+0000","last_became_active":"2023-09-23T20:03:14.488370+0000","last_became_peered":"2023-09-23T20:03:14.488370+0000","last_unstale":"2023-09-23T20:06:19.642628+0000","last_undegraded":"2023-09-23T20:06:19.642628+0000","last_fullsized":"2023-09-23T20:06:19.642628+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5890,"num_read_kb":12121,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":69,"num_bytes_recovered":4194323,"num_keys_recovered":2,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430989,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1179540,"kb_used_data":3468,"kb_used_omap":0,"kb_used_meta":1176064,"kb_avail":92565612,"statfs":{"total":95995035648,"available":94787186688,"internally_reserved":0,"allocated":3551232,"data_stored":2949211,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1204289536},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:07:32 2023","interfaces":[{"interface":"back","average":{"1min":0.83099999999999996,"5min":0.86399999999999999,"15min":1.3680000000000001},"min":{"1min":0.56200000000000006,"5min":0.52700000000000002,"15min":0.26700000000000002},"max":{"1min":1.252,"5min":1.3779999999999999,"15min":41.570999999999998},"last":0.73699999999999999},{"interface":"front","average":{"1min":0.88400000000000001,"5min":0.86399999999999999,"15min":1.542},"min":{"1min":0.60799999999999998,"5min":0.46000000000000002,"15min":0.28000000000000003},"max":{"1min":1.1180000000000001,"5min":1.4630000000000001,"15min":45.634},"last":0.84799999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:06:50 2023","interfaces":[{"interface":"back","average":{"1min":1.054,"5min":0.97899999999999998,"15min":2.145},"min":{"1min":0.69799999999999995,"5min":0.56100000000000005,"15min":0.28299999999999997},"max":{"1min":1.702,"5min":1.702,"15min":149.018},"last":1.0900000000000001},{"interface":"front","average":{"1min":1.095,"5min":1.048,"15min":2.1549999999999998},"min":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.254},"max":{"1min":1.6140000000000001,"5min":1.6140000000000001,"15min":148.995},"last":1.0620000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:07:32 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":0.93300000000000005,"15min":2.4329999999999998},"min":{"1min":0.68300000000000005,"5min":0.60699999999999998,"15min":0.31900000000000001},"max":{"1min":1.133,"5min":1.405,"15min":160.655},"last":0.89400000000000002},{"interface":"front","average":{"1min":0.88400000000000001,"5min":0.88900000000000001,"15min":2.6720000000000002},"min":{"1min":0.624,"5min":0.57699999999999996,"15min":0.30199999999999999},"max":{"1min":1.175,"5min":1.3049999999999999,"15min":176.428},"last":0.97899999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:07:26 2023","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":1.018,"15min":1.383},"min":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.32600000000000001},"max":{"1min":1.1679999999999999,"5min":1.405,"15min":10.615},"last":1.008},{"interface":"front","average":{"1min":1.085,"5min":1.0980000000000001,"15min":1.4099999999999999},"min":{"1min":0.85499999999999998,"5min":0.66000000000000003,"15min":0.33800000000000002},"max":{"1min":1.337,"5min":1.4330000000000001,"15min":9.9179999999999993},"last":1.2090000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:06:50 2023","interfaces":[{"interface":"back","average":{"1min":0.89200000000000002,"5min":0.92200000000000004,"15min":1.554},"min":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.27700000000000002},"max":{"1min":1.4750000000000001,"5min":1.4750000000000001,"15min":13.484999999999999},"last":0.91300000000000003},{"interface":"front","average":{"1min":0.98299999999999998,"5min":1.004,"15min":1.4350000000000001},"min":{"1min":0.66900000000000004,"5min":0.53400000000000003,"15min":0.313},"max":{"1min":1.5620000000000001,"5min":1.5620000000000001,"15min":13.116},"last":1.153}]},{"osd":5,"last update":"Sat Sep 23 20:06:50 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":0.95299999999999996,"15min":1.431},"min":{"1min":0.69199999999999995,"5min":0.54400000000000004,"15min":0.28499999999999998},"max":{"1min":1.1299999999999999,"5min":1.595,"15min":13.321999999999999},"last":1.125},{"interface":"front","average":{"1min":1.042,"5min":1.077,"15min":1.466},"min":{"1min":0.69499999999999995,"5min":0.69399999999999995,"15min":0.315},"max":{"1min":1.407,"5min":1.538,"15min":13.478999999999999},"last":1.254}]},{"osd":6,"last update":"Sat Sep 23 20:06:50 2023","interfaces":[{"interface":"back","average":{"1min":1.091,"5min":1.095,"15min":1.341},"min":{"1min":0.73699999999999999,"5min":0.59999999999999998,"15min":0.34300000000000003},"max":{"1min":1.4099999999999999,"5min":1.4970000000000001,"15min":13.436},"last":1.1759999999999999},{"interface":"front","average":{"1min":1.131,"5min":1.1100000000000001,"15min":1.3260000000000001},"min":{"1min":0.76100000000000001,"5min":0.49299999999999999,"15min":0.35599999999999998},"max":{"1min":1.702,"5min":1.702,"15min":10.766999999999999},"last":1.286}]}]},{"osd":6,"up_from":42,"seq":180388627221,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":890568,"kb_used_data":7552,"kb_used_omap":0,"kb_used_meta":883008,"kb_avail":92854584,"statfs":{"total":95995035648,"available":95083094016,"internally_reserved":0,"allocated":7733248,"data_stored":7143073,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":904200192},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:06:48 2023","interfaces":[{"interface":"back","average":{"1min":0.83099999999999996,"5min":0.83999999999999997,"15min":2.5099999999999998},"min":{"1min":0.66900000000000004,"5min":0.38800000000000001,"15min":0.23799999999999999},"max":{"1min":1.204,"5min":1.347,"15min":125.34399999999999},"last":0.73699999999999999},{"interface":"front","average":{"1min":0.98799999999999999,"5min":0.88300000000000001,"15min":2.5089999999999999},"min":{"1min":0.77000000000000002,"5min":0.38100000000000001,"15min":0.27800000000000002},"max":{"1min":1.3280000000000001,"5min":1.4570000000000001,"15min":125.24299999999999},"last":1.1399999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:07:12 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":0.96699999999999997,"15min":1.6990000000000001},"min":{"1min":0.65500000000000003,"5min":0.40600000000000003,"15min":0.29699999999999999},"max":{"1min":1.327,"5min":2.1629999999999998,"15min":125.51900000000001},"last":1.1739999999999999},{"interface":"front","average":{"1min":1.1830000000000001,"5min":1.075,"15min":1.7909999999999999},"min":{"1min":0.92700000000000005,"5min":0.53100000000000003,"15min":0.27700000000000002},"max":{"1min":1.4470000000000001,"5min":2.181,"15min":125.596},"last":1.343}]},{"osd":2,"last update":"Sat Sep 23 20:07:27 2023","interfaces":[{"interface":"back","average":{"1min":0.97299999999999998,"5min":0.97499999999999998,"15min":2.7000000000000002},"min":{"1min":0.70299999999999996,"5min":0.57299999999999995,"15min":0.29799999999999999},"max":{"1min":1.137,"5min":2.6230000000000002,"15min":219.453},"last":0.85599999999999998},{"interface":"front","average":{"1min":0.97499999999999998,"5min":1.0049999999999999,"15min":2.7650000000000001},"min":{"1min":0.63800000000000001,"5min":0.56899999999999995,"15min":0.30199999999999999},"max":{"1min":1.3069999999999999,"5min":2.1139999999999999,"15min":219.369},"last":0.88}]},{"osd":3,"last update":"Sat Sep 23 20:07:27 2023","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":0.94399999999999995,"15min":1.369},"min":{"1min":0.60799999999999998,"5min":0.56299999999999994,"15min":0.28599999999999998},"max":{"1min":1.284,"5min":1.3089999999999999,"15min":13.823},"last":1.282},{"interface":"front","average":{"1min":0.96699999999999997,"5min":1.014,"15min":1.478},"min":{"1min":0.66600000000000004,"5min":0.44600000000000001,"15min":0.32600000000000001},"max":{"1min":1.248,"5min":1.6870000000000001,"15min":14.42},"last":0.97799999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:07:27 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":0.98699999999999999,"15min":1.5509999999999999},"min":{"1min":0.63500000000000001,"5min":0.52200000000000002,"15min":0.30599999999999999},"max":{"1min":1.4890000000000001,"5min":1.643,"15min":11.912000000000001},"last":1.042},{"interface":"front","average":{"1min":1.002,"5min":0.98699999999999999,"15min":1.5780000000000001},"min":{"1min":0.71599999999999997,"5min":0.47699999999999998,"15min":0.32200000000000001},"max":{"1min":1.2,"5min":1.3560000000000001,"15min":14.239000000000001},"last":1.204}]},{"osd":5,"last update":"Sat Sep 23 20:07:27 2023","interfaces":[{"interface":"back","average":{"1min":1.079,"5min":1.0880000000000001,"15min":1.6379999999999999},"min":{"1min":0.72399999999999998,"5min":0.57199999999999995,"15min":0.33300000000000002},"max":{"1min":1.381,"5min":1.5149999999999999,"15min":15.045},"last":1.248},{"interface":"front","average":{"1min":1.0649999999999999,"5min":1.0609999999999999,"15min":1.7430000000000001},"min":{"1min":0.79300000000000004,"5min":0.51600000000000001,"15min":0.33400000000000002},"max":{"1min":1.2829999999999999,"5min":1.597,"15min":16.244},"last":1.0800000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:06:50 2023","interfaces":[{"interface":"back","average":{"1min":1.266,"5min":1.137,"15min":1.5960000000000001},"min":{"1min":0.94499999999999995,"5min":0.52300000000000002,"15min":0.34599999999999997},"max":{"1min":1.6279999999999999,"5min":1.946,"15min":11.709},"last":1.411},{"interface":"front","average":{"1min":1.161,"5min":1.1040000000000001,"15min":1.49},"min":{"1min":0.90200000000000002,"5min":0.58399999999999996,"15min":0.371},"max":{"1min":1.71,"5min":1.71,"15min":11.000999999999999},"last":1.329}]}]},{"osd":1,"up_from":98,"seq":420906795553,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":930748,"kb_used_data":7796,"kb_used_omap":4,"kb_used_meta":922939,"kb_avail":92814404,"statfs":{"total":95995035648,"available":95041949696,"internally_reserved":0,"allocated":7983104,"data_stored":6640805,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":945090446},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:07:08 2023","interfaces":[{"interface":"back","average":{"1min":0.748,"5min":0.75900000000000001,"15min":2.0259999999999998},"min":{"1min":0.47899999999999998,"5min":0.247,"15min":0.221},"max":{"1min":1.544,"5min":1.544,"15min":134.37700000000001},"last":0.97099999999999997},{"interface":"front","average":{"1min":0.879,"5min":0.78700000000000003,"15min":2},"min":{"1min":0.42799999999999999,"5min":0.32900000000000001,"15min":0.20200000000000001},"max":{"1min":2.3940000000000001,"5min":2.3940000000000001,"15min":135.79400000000001},"last":0.51000000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:07:39 2023","interfaces":[{"interface":"back","average":{"1min":0.79700000000000004,"5min":0.80900000000000005,"15min":2.1070000000000002},"min":{"1min":0.443,"5min":0.28599999999999998,"15min":0.23300000000000001},"max":{"1min":2.0489999999999999,"5min":2.0489999999999999,"15min":48.512},"last":0.69299999999999995},{"interface":"front","average":{"1min":0.73999999999999999,"5min":0.78700000000000003,"15min":1.8029999999999999},"min":{"1min":0.434,"5min":0.27600000000000002,"15min":0.224},"max":{"1min":2.0830000000000002,"5min":2.0830000000000002,"15min":48.664000000000001},"last":0.875}]},{"osd":3,"last update":"Sat Sep 23 20:07:39 2023","interfaces":[{"interface":"back","average":{"1min":0.78200000000000003,"5min":0.878,"15min":1.6599999999999999},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.26200000000000001},"max":{"1min":2.339,"5min":2.339,"15min":41.518000000000001},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.79900000000000004,"5min":0.80100000000000005,"15min":1.8200000000000001},"min":{"1min":0.45600000000000002,"5min":0.38500000000000001,"15min":0.252},"max":{"1min":2.0230000000000001,"5min":2.0230000000000001,"15min":41.151000000000003},"last":0.80300000000000005}]},{"osd":4,"last update":"Sat Sep 23 20:07:08 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":1.0569999999999999,"15min":2.0019999999999998},"min":{"1min":0.71899999999999997,"5min":0.51900000000000002,"15min":0.28499999999999998},"max":{"1min":2.4529999999999998,"5min":2.4529999999999998,"15min":48.381},"last":0.77300000000000002},{"interface":"front","average":{"1min":1.0429999999999999,"5min":1.0109999999999999,"15min":1.7789999999999999},"min":{"1min":0.70099999999999996,"5min":0.58099999999999996,"15min":0.35099999999999998},"max":{"1min":2.4140000000000001,"5min":2.4140000000000001,"15min":41.320999999999998},"last":0.93700000000000006}]},{"osd":5,"last update":"Sat Sep 23 20:07:08 2023","interfaces":[{"interface":"back","average":{"1min":1.101,"5min":1.0349999999999999,"15min":1.8480000000000001},"min":{"1min":0.61899999999999999,"5min":0.54700000000000004,"15min":0.38200000000000001},"max":{"1min":2.2149999999999999,"5min":2.2149999999999999,"15min":48.643000000000001},"last":1.1299999999999999},{"interface":"front","average":{"1min":1.177,"5min":1.1639999999999999,"15min":1.897},"min":{"1min":0.70999999999999996,"5min":0.44600000000000001,"15min":0.33300000000000002},"max":{"1min":2.4940000000000002,"5min":2.4940000000000002,"15min":48.619999999999997},"last":1.0720000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:07:08 2023","interfaces":[{"interface":"back","average":{"1min":1.206,"5min":1.2,"15min":2.1200000000000001},"min":{"1min":0.75700000000000001,"5min":0.61499999999999999,"15min":0.35199999999999998},"max":{"1min":2.4300000000000002,"5min":2.4300000000000002,"15min":48.548999999999999},"last":1.1930000000000001},{"interface":"front","average":{"1min":1.2330000000000001,"5min":1.1830000000000001,"15min":2.0270000000000001},"min":{"1min":0.80900000000000005,"5min":0.57199999999999995,"15min":0.313},"max":{"1min":2.2770000000000001,"5min":2.2770000000000001,"15min":41.648000000000003},"last":1.097}]},{"osd":7,"last update":"Sat Sep 23 20:07:08 2023","interfaces":[{"interface":"back","average":{"1min":1.276,"5min":1.208,"15min":2.1160000000000001},"min":{"1min":0.85199999999999998,"5min":0.56699999999999995,"15min":0.38300000000000001},"max":{"1min":2.5099999999999998,"5min":2.5099999999999998,"15min":48.329999999999998},"last":1.1639999999999999},{"interface":"front","average":{"1min":1.2290000000000001,"5min":1.1779999999999999,"15min":1.994},"min":{"1min":0.66800000000000004,"5min":0.63900000000000001,"15min":0.42699999999999999},"max":{"1min":2.4740000000000002,"5min":2.4740000000000002,"15min":48.579000000000001},"last":1.024}]}]},{"osd":0,"up_from":85,"seq":365072220759,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":981848,"kb_used_data":4240,"kb_used_omap":5,"kb_used_meta":977594,"kb_avail":92763304,"statfs":{"total":95995035648,"available":94989623296,"internally_reserved":0,"allocated":4341760,"data_stored":3036322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1001056890},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:07:13 2023","interfaces":[{"interface":"back","average":{"1min":0.70699999999999996,"5min":0.71899999999999997,"15min":2.286},"min":{"1min":0.318,"5min":0.318,"15min":0.219},"max":{"1min":1.0640000000000001,"5min":1.3899999999999999,"15min":149.37299999999999},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.72599999999999998,"5min":0.72099999999999997,"15min":2.3290000000000002},"min":{"1min":0.45700000000000002,"5min":0.33000000000000002,"15min":0.191},"max":{"1min":0.96899999999999997,"5min":1.47,"15min":147.119},"last":0.69399999999999995}]},{"osd":2,"last update":"Sat Sep 23 20:07:30 2023","interfaces":[{"interface":"back","average":{"1min":0.80400000000000005,"5min":0.77600000000000002,"15min":1.5089999999999999},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.25600000000000001},"max":{"1min":2.4660000000000002,"5min":2.4660000000000002,"15min":11.646000000000001},"last":0.73899999999999999},{"interface":"front","average":{"1min":0.82599999999999996,"5min":0.79400000000000004,"15min":1.339},"min":{"1min":0.38700000000000001,"5min":0.36499999999999999,"15min":0.25700000000000001},"max":{"1min":2.1960000000000002,"5min":2.1960000000000002,"15min":11.542},"last":0.79300000000000004}]},{"osd":3,"last update":"Sat Sep 23 20:07:13 2023","interfaces":[{"interface":"back","average":{"1min":0.95099999999999996,"5min":0.93799999999999994,"15min":1.454},"min":{"1min":0.499,"5min":0.41299999999999998,"15min":0.28100000000000003},"max":{"1min":1.702,"5min":1.702,"15min":14.983000000000001},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.90600000000000003,"5min":0.90400000000000003,"15min":1.379},"min":{"1min":0.47599999999999998,"5min":0.46800000000000003,"15min":0.29999999999999999},"max":{"1min":1.3380000000000001,"5min":1.431,"15min":14.843},"last":0.59499999999999997}]},{"osd":4,"last update":"Sat Sep 23 20:06:49 2023","interfaces":[{"interface":"back","average":{"1min":1.0620000000000001,"5min":1.0609999999999999,"15min":1.4099999999999999},"min":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.32000000000000001},"max":{"1min":1.393,"5min":1.79,"15min":11.398},"last":1.016},{"interface":"front","average":{"1min":1.0449999999999999,"5min":1.0349999999999999,"15min":1.4419999999999999},"min":{"1min":0.70899999999999996,"5min":0.59999999999999998,"15min":0.30199999999999999},"max":{"1min":1.4259999999999999,"5min":1.444,"15min":10.747},"last":0.81799999999999995}]},{"osd":5,"last update":"Sat Sep 23 20:06:49 2023","interfaces":[{"interface":"back","average":{"1min":1.1599999999999999,"5min":1.1220000000000001,"15min":1.488},"min":{"1min":0.83599999999999997,"5min":0.63300000000000001,"15min":0.316},"max":{"1min":1.522,"5min":1.579,"15min":10.69},"last":0.84999999999999998},{"interface":"front","average":{"1min":1.248,"5min":1.198,"15min":1.502},"min":{"1min":0.84899999999999998,"5min":0.755,"15min":0.36499999999999999},"max":{"1min":1.526,"5min":2.0550000000000002,"15min":11.510999999999999},"last":0.89900000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:06:49 2023","interfaces":[{"interface":"back","average":{"1min":1.2230000000000001,"5min":1.2150000000000001,"15min":1.542},"min":{"1min":0.80500000000000005,"5min":0.72499999999999998,"15min":0.36699999999999999},"max":{"1min":1.8260000000000001,"5min":1.9950000000000001,"15min":11.443},"last":1.1339999999999999},{"interface":"front","average":{"1min":1.2629999999999999,"5min":1.2,"15min":1.6120000000000001},"min":{"1min":0.88300000000000001,"5min":0.82199999999999995,"15min":0.36599999999999999},"max":{"1min":1.5840000000000001,"5min":1.5840000000000001,"15min":17.050999999999998},"last":0.89100000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:06:49 2023","interfaces":[{"interface":"back","average":{"1min":1.181,"5min":1.1459999999999999,"15min":2.1970000000000001},"min":{"1min":0.85099999999999998,"5min":0.54500000000000004,"15min":0.42099999999999999},"max":{"1min":1.5680000000000001,"5min":2.0099999999999998,"15min":210.47399999999999},"last":1.038},{"interface":"front","average":{"1min":1.264,"5min":1.2070000000000001,"15min":2.2530000000000001},"min":{"1min":0.79400000000000004,"5min":0.77400000000000002,"15min":0.41099999999999998},"max":{"1min":1.569,"5min":1.909,"15min":210.52199999999999},"last":0.95899999999999996}]}]},{"osd":2,"up_from":159,"seq":682899800415,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":737364,"kb_used_data":3660,"kb_used_omap":7,"kb_used_meta":733688,"kb_avail":93007788,"statfs":{"total":95995035648,"available":95239974912,"internally_reserved":0,"allocated":3747840,"data_stored":2440299,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":751296593},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:06:46 2023","interfaces":[{"interface":"back","average":{"1min":0.78700000000000003,"5min":0.78500000000000003,"15min":1.6379999999999999},"min":{"1min":0.44900000000000001,"5min":0.30099999999999999,"15min":0.19700000000000001},"max":{"1min":1.1879999999999999,"5min":1.661,"15min":61.442999999999998},"last":0.83199999999999996},{"interface":"front","average":{"1min":0.82999999999999996,"5min":0.80100000000000005,"15min":1.742},"min":{"1min":0.44600000000000001,"5min":0.374,"15min":0.223},"max":{"1min":1.3899999999999999,"5min":1.5309999999999999,"15min":61.374000000000002},"last":0.88600000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:06:46 2023","interfaces":[{"interface":"back","average":{"1min":0.83599999999999997,"5min":0.74299999999999999,"15min":2.0920000000000001},"min":{"1min":0.51900000000000002,"5min":0.36399999999999999,"15min":0.22600000000000001},"max":{"1min":1.143,"5min":1.4330000000000001,"15min":84.234999999999999},"last":0.94599999999999995},{"interface":"front","average":{"1min":0.90900000000000003,"5min":0.83399999999999996,"15min":1.8169999999999999},"min":{"1min":0.65500000000000003,"5min":0.32900000000000001,"15min":0.248},"max":{"1min":1.4470000000000001,"5min":1.4470000000000001,"15min":84.150000000000006},"last":1.0620000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:06:46 2023","interfaces":[{"interface":"back","average":{"1min":0.88300000000000001,"5min":0.81599999999999995,"15min":1.581},"min":{"1min":0.47699999999999998,"5min":0.38200000000000001,"15min":0.25},"max":{"1min":1.4950000000000001,"5min":1.6890000000000001,"15min":17.751000000000001},"last":1.1850000000000001},{"interface":"front","average":{"1min":0.98899999999999999,"5min":0.879,"15min":1.53},"min":{"1min":0.60599999999999998,"5min":0.373,"15min":0.215},"max":{"1min":1.569,"5min":1.569,"15min":19.596},"last":1.1499999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:06:46 2023","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":0.98899999999999999,"15min":1.55},"min":{"1min":0.78300000000000003,"5min":0.52000000000000002,"15min":0.27100000000000002},"max":{"1min":1.1160000000000001,"5min":1.5549999999999999,"15min":14.194000000000001},"last":1.46},{"interface":"front","average":{"1min":1.1080000000000001,"5min":1.0840000000000001,"15min":1.8240000000000001},"min":{"1min":0.75800000000000001,"5min":0.63,"15min":0.32600000000000001},"max":{"1min":1.4159999999999999,"5min":1.754,"15min":14.271000000000001},"last":1.2709999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:06:46 2023","interfaces":[{"interface":"back","average":{"1min":1.145,"5min":1.1060000000000001,"15min":1.6599999999999999},"min":{"1min":0.69899999999999995,"5min":0.60799999999999998,"15min":0.32400000000000001},"max":{"1min":1.806,"5min":2.206,"15min":14.007},"last":1.3740000000000001},{"interface":"front","average":{"1min":1.1319999999999999,"5min":1.0880000000000001,"15min":1.397},"min":{"1min":0.81599999999999995,"5min":0.62,"15min":0.32100000000000001},"max":{"1min":1.7969999999999999,"5min":1.974,"15min":8.2720000000000002},"last":1.3300000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:06:46 2023","interfaces":[{"interface":"back","average":{"1min":1.1140000000000001,"5min":1.1299999999999999,"15min":1.5660000000000001},"min":{"1min":0.81999999999999995,"5min":0.55000000000000004,"15min":0.35699999999999998},"max":{"1min":1.661,"5min":2.1019999999999999,"15min":9.8870000000000005},"last":1.2310000000000001},{"interface":"front","average":{"1min":1.2470000000000001,"5min":1.1870000000000001,"15min":1.802},"min":{"1min":0.97099999999999997,"5min":0.64600000000000002,"15min":0.371},"max":{"1min":1.6479999999999999,"5min":2.0649999999999999,"15min":14.077999999999999},"last":1.5089999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:06:46 2023","interfaces":[{"interface":"back","average":{"1min":1.288,"5min":1.2070000000000001,"15min":1.649},"min":{"1min":0.85099999999999998,"5min":0.66900000000000004,"15min":0.40000000000000002},"max":{"1min":1.873,"5min":2.1339999999999999,"15min":14.058},"last":1.5369999999999999},{"interface":"front","average":{"1min":1.268,"5min":1.1859999999999999,"15min":1.7749999999999999},"min":{"1min":0.91200000000000003,"5min":0.63600000000000001,"15min":0.38700000000000001},"max":{"1min":1.766,"5min":1.9430000000000001,"15min":14.109999999999999},"last":1.4319999999999999}]}]},{"osd":3,"up_from":116,"seq":498216206815,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":837744,"kb_used_data":3560,"kb_used_omap":2,"kb_used_meta":834173,"kb_avail":92907408,"statfs":{"total":95995035648,"available":95137185792,"internally_reserved":0,"allocated":3645440,"data_stored":2429301,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":854193205},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:07:21 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":0.872,"15min":2.1850000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.19600000000000001},"max":{"1min":1.024,"5min":15.679,"15min":117.248},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.72799999999999998,"5min":0.81699999999999995,"15min":2.0939999999999999},"min":{"1min":0.37,"5min":0.37,"15min":0.20899999999999999},"max":{"1min":1.268,"5min":4.516,"15min":116.187},"last":0.55700000000000005}]},{"osd":1,"last update":"Sat Sep 23 20:07:21 2023","interfaces":[{"interface":"back","average":{"1min":0.747,"5min":0.96499999999999997,"15min":1.4910000000000001},"min":{"1min":0.52200000000000002,"5min":0.34000000000000002,"15min":0.23400000000000001},"max":{"1min":0.98599999999999999,"5min":8.2140000000000004,"15min":12.542},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.754,"5min":0.96999999999999997,"15min":1.365},"min":{"1min":0.42999999999999999,"5min":0.309,"15min":0.23100000000000001},"max":{"1min":0.98499999999999999,"5min":17.187000000000001,"15min":17.187000000000001},"last":0.497}]},{"osd":2,"last update":"Sat Sep 23 20:07:38 2023","interfaces":[{"interface":"back","average":{"1min":0.74199999999999999,"5min":0.89800000000000002,"15min":1.446},"min":{"1min":0.42099999999999999,"5min":0.40400000000000003,"15min":0.246},"max":{"1min":1.3100000000000001,"5min":2.2650000000000001,"15min":14.529},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.93899999999999995,"15min":1.5389999999999999},"min":{"1min":0.438,"5min":0.32100000000000001,"15min":0.30499999999999999},"max":{"1min":0.98099999999999998,"5min":15.5,"15min":15.5},"last":0.41999999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:07:21 2023","interfaces":[{"interface":"back","average":{"1min":1.0580000000000001,"5min":1.1930000000000001,"15min":1.5309999999999999},"min":{"1min":0.59799999999999998,"5min":0.56899999999999995,"15min":0.29699999999999999},"max":{"1min":1.51,"5min":15.461,"15min":15.461},"last":0.80300000000000005},{"interface":"front","average":{"1min":1.1140000000000001,"5min":1.204,"15min":1.5960000000000001},"min":{"1min":0.81499999999999995,"5min":0.54900000000000004,"15min":0.32900000000000001},"max":{"1min":1.484,"5min":5.7149999999999999,"15min":12.169},"last":0.752}]},{"osd":5,"last update":"Sat Sep 23 20:07:21 2023","interfaces":[{"interface":"back","average":{"1min":1.0860000000000001,"5min":1.1479999999999999,"15min":1.548},"min":{"1min":0.64600000000000002,"5min":0.60199999999999998,"15min":0.33900000000000002},"max":{"1min":1.446,"5min":2.3380000000000001,"15min":11.978},"last":0.69499999999999995},{"interface":"front","average":{"1min":0.96399999999999997,"5min":1.2230000000000001,"15min":1.4630000000000001},"min":{"1min":0.747,"5min":0.67700000000000005,"15min":0.34899999999999998},"max":{"1min":1.206,"5min":14.715,"15min":14.715},"last":0.93999999999999995}]},{"osd":6,"last update":"Sat Sep 23 20:07:21 2023","interfaces":[{"interface":"back","average":{"1min":1.1299999999999999,"5min":1.196,"15min":1.6970000000000001},"min":{"1min":0.77100000000000002,"5min":0.77000000000000002,"15min":0.41199999999999998},"max":{"1min":1.5269999999999999,"5min":2.3849999999999998,"15min":12.135},"last":0.72299999999999998},{"interface":"front","average":{"1min":1.0229999999999999,"5min":1.2809999999999999,"15min":1.575},"min":{"1min":0.66000000000000003,"5min":0.61799999999999999,"15min":0.38500000000000001},"max":{"1min":1.278,"5min":15.537000000000001,"15min":15.537000000000001},"last":0.83699999999999997}]},{"osd":7,"last update":"Sat Sep 23 20:07:21 2023","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":1.329,"15min":1.732},"min":{"1min":0.69499999999999995,"5min":0.61899999999999999,"15min":0.35799999999999998},"max":{"1min":1.337,"5min":14.754,"15min":14.754},"last":0.78400000000000003},{"interface":"front","average":{"1min":1.111,"5min":1.3320000000000001,"15min":1.5329999999999999},"min":{"1min":0.83299999999999996,"5min":0.72399999999999998,"15min":0.41299999999999998},"max":{"1min":1.343,"5min":17.145,"15min":17.145},"last":0.85899999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019686,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1195276,"kb_used_data":6980,"kb_used_omap":0,"kb_used_meta":1188288,"kb_avail":92549876,"statfs":{"total":95995035648,"available":94771073024,"internally_reserved":0,"allocated":7147520,"data_stored":6553077,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1216806912},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:06:50 2023","interfaces":[{"interface":"back","average":{"1min":0.89200000000000002,"5min":0.86599999999999999,"15min":1.411},"min":{"1min":0.69099999999999995,"5min":0.443,"15min":0.247},"max":{"1min":1.131,"5min":2.0099999999999998,"15min":23.059999999999999},"last":0.77500000000000002},{"interface":"front","average":{"1min":0.84699999999999998,"5min":0.84099999999999997,"15min":1.429},"min":{"1min":0.61199999999999999,"5min":0.39900000000000002,"15min":0.26300000000000001},"max":{"1min":1.266,"5min":1.306,"15min":22.780000000000001},"last":1.113}]},{"osd":1,"last update":"Sat Sep 23 20:07:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0049999999999999,"5min":0.96899999999999997,"15min":1.6859999999999999},"min":{"1min":0.76300000000000001,"5min":0.44400000000000001,"15min":0.28100000000000003},"max":{"1min":1.2390000000000001,"5min":1.4450000000000001,"15min":30.754000000000001},"last":1.3080000000000001},{"interface":"front","average":{"1min":1.0589999999999999,"5min":1.006,"15min":1.7430000000000001},"min":{"1min":0.66600000000000004,"5min":0.51400000000000001,"15min":0.27600000000000002},"max":{"1min":1.3109999999999999,"5min":1.321,"15min":30.777999999999999},"last":1.1579999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:06:47 2023","interfaces":[{"interface":"back","average":{"1min":0.94399999999999995,"5min":0.95099999999999996,"15min":1.7},"min":{"1min":0.72199999999999998,"5min":0.48399999999999999,"15min":0.27600000000000002},"max":{"1min":1.246,"5min":5.8860000000000001,"15min":40.906999999999996},"last":0.93200000000000005},{"interface":"front","average":{"1min":0.97899999999999998,"5min":0.95599999999999996,"15min":2.8450000000000002},"min":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.308},"max":{"1min":1.3620000000000001,"5min":2.6880000000000002,"15min":314.31599999999997},"last":1.236}]},{"osd":3,"last update":"Sat Sep 23 20:06:47 2023","interfaces":[{"interface":"back","average":{"1min":0.93700000000000006,"5min":1.012,"15min":1.4179999999999999},"min":{"1min":0.751,"5min":0.60199999999999998,"15min":0.308},"max":{"1min":1.2849999999999999,"5min":8.8219999999999992,"15min":9.4399999999999995},"last":0.86799999999999999},{"interface":"front","average":{"1min":1.038,"5min":1.0049999999999999,"15min":1.4550000000000001},"min":{"1min":0.59799999999999998,"5min":0.47599999999999998,"15min":0.32500000000000001},"max":{"1min":1.3540000000000001,"5min":3.266,"15min":9.5280000000000005},"last":1.0109999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:06:50 2023","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":0.97799999999999998,"15min":1.4370000000000001},"min":{"1min":0.63200000000000001,"5min":0.54200000000000004,"15min":0.26800000000000002},"max":{"1min":1.3340000000000001,"5min":1.548,"15min":9.4740000000000002},"last":0.95399999999999996},{"interface":"front","average":{"1min":1.097,"5min":1.0529999999999999,"15min":1.4590000000000001},"min":{"1min":0.68600000000000005,"5min":0.57599999999999996,"15min":0.29299999999999998},"max":{"1min":1.468,"5min":1.7609999999999999,"15min":12.506},"last":1.206}]},{"osd":6,"last update":"Sat Sep 23 20:07:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0600000000000001,"5min":1.052,"15min":1.4590000000000001},"min":{"1min":0.65200000000000002,"5min":0.60899999999999999,"15min":0.32400000000000001},"max":{"1min":1.4099999999999999,"5min":1.5900000000000001,"15min":13.066000000000001},"last":1.0569999999999999},{"interface":"front","average":{"1min":1.1799999999999999,"5min":1.117,"15min":1.4850000000000001},"min":{"1min":0.94299999999999995,"5min":0.51500000000000001,"15min":0.35499999999999998},"max":{"1min":1.4330000000000001,"5min":2.8149999999999999,"15min":12.704000000000001},"last":1.093}]},{"osd":7,"last update":"Sat Sep 23 20:07:15 2023","interfaces":[{"interface":"back","average":{"1min":1.131,"5min":1.038,"15min":1.4330000000000001},"min":{"1min":0.66800000000000004,"5min":0.47899999999999998,"15min":0.308},"max":{"1min":1.6299999999999999,"5min":1.6299999999999999,"15min":12.475},"last":1.359},{"interface":"front","average":{"1min":1.1719999999999999,"5min":1.135,"15min":1.5349999999999999},"min":{"1min":0.83699999999999997,"5min":0.58599999999999997,"15min":0.35199999999999998},"max":{"1min":1.4630000000000001,"5min":3.121,"15min":12.169},"last":1.2629999999999999}]}]},{"osd":5,"up_from":36,"seq":154618823453,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1198136,"kb_used_data":2864,"kb_used_omap":0,"kb_used_meta":1195264,"kb_avail":92547016,"statfs":{"total":95995035648,"available":94768144384,"internally_reserved":0,"allocated":2932736,"data_stored":2353906,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1223950336},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:07:46 2023","interfaces":[{"interface":"back","average":{"1min":0.94099999999999995,"5min":0.92700000000000005,"15min":2.6960000000000002},"min":{"1min":0.58299999999999996,"5min":0.53400000000000003,"15min":0.27800000000000002},"max":{"1min":1.2210000000000001,"5min":1.3600000000000001,"15min":201.196},"last":0.93500000000000005},{"interface":"front","average":{"1min":0.91600000000000004,"5min":0.91200000000000003,"15min":2.9089999999999998},"min":{"1min":0.50900000000000001,"5min":0.499,"15min":0.251},"max":{"1min":1.1679999999999999,"5min":1.411,"15min":192.941},"last":1.0149999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:07:11 2023","interfaces":[{"interface":"back","average":{"1min":0.93700000000000006,"5min":0.94199999999999995,"15min":4.1829999999999998},"min":{"1min":0.55200000000000005,"5min":0.46999999999999997,"15min":0.28599999999999998},"max":{"1min":1.282,"5min":1.5129999999999999,"15min":190.298},"last":0.82199999999999995},{"interface":"front","average":{"1min":0.90200000000000002,"5min":0.92700000000000005,"15min":4.2839999999999998},"min":{"1min":0.65200000000000002,"5min":0.629,"15min":0.28999999999999998},"max":{"1min":1.2470000000000001,"5min":1.5029999999999999,"15min":190.65799999999999},"last":1.1339999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:07:32 2023","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":0.93799999999999994,"15min":2.831},"min":{"1min":0.68100000000000005,"5min":0.49399999999999999,"15min":0.311},"max":{"1min":1.3260000000000001,"5min":1.5449999999999999,"15min":151.322},"last":1.0680000000000001},{"interface":"front","average":{"1min":1.0349999999999999,"5min":0.98299999999999998,"15min":2.976},"min":{"1min":0.70099999999999996,"5min":0.55400000000000005,"15min":0.34000000000000002},"max":{"1min":1.3200000000000001,"5min":1.387,"15min":166.941},"last":0.996}]},{"osd":3,"last update":"Sat Sep 23 20:07:11 2023","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":0.97499999999999998,"15min":1.369},"min":{"1min":0.69899999999999995,"5min":0.59999999999999998,"15min":0.35399999999999998},"max":{"1min":1.2829999999999999,"5min":1.6299999999999999,"15min":9.3829999999999991},"last":1.226},{"interface":"front","average":{"1min":1.042,"5min":1.0389999999999999,"15min":1.3759999999999999},"min":{"1min":0.78100000000000003,"5min":0.61899999999999999,"15min":0.34000000000000002},"max":{"1min":1.2390000000000001,"5min":1.375,"15min":13.840999999999999},"last":1.101}]},{"osd":4,"last update":"Sat Sep 23 20:07:46 2023","interfaces":[{"interface":"back","average":{"1min":0.874,"5min":0.88800000000000001,"15min":1.5269999999999999},"min":{"1min":0.64300000000000002,"5min":0.47799999999999998,"15min":0.26800000000000002},"max":{"1min":1.04,"5min":1.593,"15min":12.051},"last":0.90200000000000002},{"interface":"front","average":{"1min":1.0429999999999999,"5min":0.98799999999999999,"15min":1.431},"min":{"1min":0.754,"5min":0.56999999999999995,"15min":0.27900000000000003},"max":{"1min":1.2789999999999999,"5min":1.8,"15min":16.462},"last":1.2789999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:07:32 2023","interfaces":[{"interface":"back","average":{"1min":1.024,"5min":1.036,"15min":1.3580000000000001},"min":{"1min":0.63900000000000001,"5min":0.45700000000000002,"15min":0.32100000000000001},"max":{"1min":1.375,"5min":1.589,"15min":9.641},"last":1.151},{"interface":"front","average":{"1min":1.1140000000000001,"5min":1.1419999999999999,"15min":1.399},"min":{"1min":0.78900000000000003,"5min":0.67100000000000004,"15min":0.38700000000000001},"max":{"1min":1.262,"5min":1.6799999999999999,"15min":11.444000000000001},"last":1.327}]},{"osd":7,"last update":"Sat Sep 23 20:07:46 2023","interfaces":[{"interface":"back","average":{"1min":1.119,"5min":1.085,"15min":1.4770000000000001},"min":{"1min":0.84299999999999997,"5min":0.70199999999999996,"15min":0.29999999999999999},"max":{"1min":1.3580000000000001,"5min":1.7829999999999999,"15min":9.407},"last":1.159},{"interface":"front","average":{"1min":1.165,"5min":1.129,"15min":1.403},"min":{"1min":0.91100000000000003,"5min":0.622,"15min":0.34399999999999997},"max":{"1min":1.3200000000000001,"5min":1.643,"15min":17.053000000000001},"last":1.1970000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:07:53.687 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:07:54.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:07:54 smithi016 bash[75004]: audit 2023-09-23T20:07:52.747034+0000 mgr.y (mgr.25325) 1821 : audit 0 from='client.38061 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:07:54.786 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:07:54 smithi016 bash[75004]: cluster 2023-09-23T20:07:52.805181+0000 mgr.y (mgr.25325) 1822 : cluster 0 pgmap v1915: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:54.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:07:54 smithi016 bash[77995]: audit 2023-09-23T20:07:52.747034+0000 mgr.y (mgr.25325) 1821 : audit 0 from='client.38061 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:07:54.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:07:54 smithi016 bash[77995]: cluster 2023-09-23T20:07:52.805181+0000 mgr.y (mgr.25325) 1822 : cluster 0 pgmap v1915: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:54.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:07:54 smithi092 bash[58855]: audit 2023-09-23T20:07:52.747034+0000 mgr.y (mgr.25325) 1821 : audit 0 from='client.38061 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:07:54.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:07:54 smithi092 bash[58855]: cluster 2023-09-23T20:07:52.805181+0000 mgr.y (mgr.25325) 1822 : cluster 0 pgmap v1915: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:56.035 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:07:55 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:20:07:55] "GET /metrics HTTP/1.1" 200 34016 "" "Prometheus/2.33.4" 2023-09-23T20:07:56.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:07:56 smithi016 bash[75004]: cluster 2023-09-23T20:07:54.805923+0000 mgr.y (mgr.25325) 1823 : cluster 0 pgmap v1916: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:56.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:07:56 smithi016 bash[77995]: cluster 2023-09-23T20:07:54.805923+0000 mgr.y (mgr.25325) 1823 : cluster 0 pgmap v1916: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:56.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:07:56 smithi092 bash[58855]: cluster 2023-09-23T20:07:54.805923+0000 mgr.y (mgr.25325) 1823 : cluster 0 pgmap v1916: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:58.511 INFO:teuthology.orchestra.run.smithi016.stderr:Inferring config /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:07:58.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:07:58 smithi016 bash[75004]: cluster 2023-09-23T20:07:56.806884+0000 mgr.y (mgr.25325) 1824 : cluster 0 pgmap v1917: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:58.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:07:58 smithi016 bash[77995]: cluster 2023-09-23T20:07:56.806884+0000 mgr.y (mgr.25325) 1824 : cluster 0 pgmap v1917: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:58.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:07:58 smithi092 bash[58855]: cluster 2023-09-23T20:07:56.806884+0000 mgr.y (mgr.25325) 1824 : cluster 0 pgmap v1917: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:07:59.659 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:07:59 smithi016 bash[77995]: audit 2023-09-23T20:07:58.708509+0000 mon.a (mon.0) 2173 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:07:59.660 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:07:59 smithi016 bash[75004]: audit 2023-09-23T20:07:58.708509+0000 mon.a (mon.0) 2173 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:07:59.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:07:59 smithi092 bash[58855]: audit 2023-09-23T20:07:58.708509+0000 mon.a (mon.0) 2173 : audit 0 from='mgr.25325 172.21.15.16:0/3249536797' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:08:00.785 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:08:00 smithi016 bash[75004]: cluster 2023-09-23T20:07:58.807603+0000 mgr.y (mgr.25325) 1825 : cluster 0 pgmap v1918: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:08:00.786 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:08:00 smithi016 bash[77995]: cluster 2023-09-23T20:07:58.807603+0000 mgr.y (mgr.25325) 1825 : cluster 0 pgmap v1918: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:08:00.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:08:00 smithi092 bash[58855]: cluster 2023-09-23T20:07:58.807603+0000 mgr.y (mgr.25325) 1825 : cluster 0 pgmap v1918: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:08:02.082 INFO:teuthology.orchestra.run.smithi016.stdout: 2023-09-23T20:08:02.086 INFO:teuthology.orchestra.run.smithi016.stderr:dumped all 2023-09-23T20:08:02.535 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:08:02 smithi016 bash[77995]: cluster 2023-09-23T20:08:00.808681+0000 mgr.y (mgr.25325) 1826 : cluster 0 pgmap v1919: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:08:02.535 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:08:02 smithi016 bash[75004]: cluster 2023-09-23T20:08:00.808681+0000 mgr.y (mgr.25325) 1826 : cluster 0 pgmap v1919: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:08:02.788 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:08:02 smithi092 bash[58855]: cluster 2023-09-23T20:08:00.808681+0000 mgr.y (mgr.25325) 1826 : cluster 0 pgmap v1919: 73 pgs: 73 active+clean; 4.6 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:08:03.049 INFO:teuthology.orchestra.run.smithi016.stdout:{"pg_ready":true,"pg_map":{"version":1919,"stamp":"2023-09-23T20:08:00.807953+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6198,"num_read_kb":12633,"num_write":1107,"num_write_kb":15160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":73,"num_bytes_recovered":4194323,"num_keys_recovered":2,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1171,"ondisk_log_size":1171,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":7951224,"kb_used_data":40120,"kb_used_omap":20,"kb_used_meta":7911019,"kb_avail":742009992,"statfs":{"total":767960285184,"available":759818231808,"internally_reserved":0,"allocated":41082880,"data_stored":33545994,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":8100884110},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004771"},"pg_stats":[{"pgid":"3.1a","version":"59'1","reported_seq":347,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.642865+0000","last_change":"2023-09-23T20:03:14.491154+0000","last_active":"2023-09-23T20:06:19.642865+0000","last_peered":"2023-09-23T20:06:19.642865+0000","last_clean":"2023-09-23T20:06:19.642865+0000","last_became_active":"2023-09-23T20:03:14.490786+0000","last_became_peered":"2023-09-23T20:03:14.490786+0000","last_unstale":"2023-09-23T20:06:19.642865+0000","last_undegraded":"2023-09-23T20:06:19.642865+0000","last_fullsized":"2023-09-23T20:06:19.642865+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":109,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:06:32.811282+0000","last_change":"2023-09-23T20:04:47.030349+0000","last_active":"2023-09-23T20:06:32.811282+0000","last_peered":"2023-09-23T20:06:32.811282+0000","last_clean":"2023-09-23T20:06:32.811282+0000","last_became_active":"2023-09-23T20:04:47.029966+0000","last_became_peered":"2023-09-23T20:04:47.029966+0000","last_unstale":"2023-09-23T20:06:32.811282+0000","last_undegraded":"2023-09-23T20:06:32.811282+0000","last_fullsized":"2023-09-23T20:06:32.811282+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":322,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:46.639171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":38,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:18.930726+0000","last_change":"2023-09-23T20:03:14.480773+0000","last_active":"2023-09-23T20:06:18.930726+0000","last_peered":"2023-09-23T20:06:18.930726+0000","last_clean":"2023-09-23T20:06:18.930726+0000","last_became_active":"2023-09-23T20:03:14.480401+0000","last_became_peered":"2023-09-23T20:03:14.480401+0000","last_unstale":"2023-09-23T20:06:18.930726+0000","last_undegraded":"2023-09-23T20:06:18.930726+0000","last_fullsized":"2023-09-23T20:06:18.930726+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:52:53.701767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":178,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.223566+0000","last_change":"2023-09-23T20:04:06.716877+0000","last_active":"2023-09-23T20:06:19.223566+0000","last_peered":"2023-09-23T20:06:19.223566+0000","last_clean":"2023-09-23T20:06:19.223566+0000","last_became_active":"2023-09-23T20:04:06.716273+0000","last_became_peered":"2023-09-23T20:04:06.716273+0000","last_unstale":"2023-09-23T20:06:19.223566+0000","last_undegraded":"2023-09-23T20:06:19.223566+0000","last_fullsized":"2023-09-23T20:06:19.223566+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:54:45.068172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":322,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:06:32.810631+0000","last_change":"2023-09-23T20:04:47.051743+0000","last_active":"2023-09-23T20:06:32.810631+0000","last_peered":"2023-09-23T20:06:32.810631+0000","last_clean":"2023-09-23T20:06:32.810631+0000","last_became_active":"2023-09-23T20:04:47.050630+0000","last_became_peered":"2023-09-23T20:04:47.050630+0000","last_unstale":"2023-09-23T20:06:32.810631+0000","last_undegraded":"2023-09-23T20:06:32.810631+0000","last_fullsized":"2023-09-23T20:06:32.810631+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":322,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:53:53.798779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":316,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.184793+0000","last_change":"2023-09-23T20:03:14.487145+0000","last_active":"2023-09-23T20:06:19.184793+0000","last_peered":"2023-09-23T20:06:19.184793+0000","last_clean":"2023-09-23T20:06:19.184793+0000","last_became_active":"2023-09-23T20:03:14.486753+0000","last_became_peered":"2023-09-23T20:03:14.486753+0000","last_unstale":"2023-09-23T20:06:19.184793+0000","last_undegraded":"2023-09-23T20:06:19.184793+0000","last_fullsized":"2023-09-23T20:06:19.184793+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":210,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:06:31.976185+0000","last_change":"2023-09-23T20:04:06.516984+0000","last_active":"2023-09-23T20:06:31.976185+0000","last_peered":"2023-09-23T20:06:31.976185+0000","last_clean":"2023-09-23T20:06:31.976185+0000","last_became_active":"2023-09-23T20:04:06.516558+0000","last_became_peered":"2023-09-23T20:04:06.516558+0000","last_unstale":"2023-09-23T20:06:31.976185+0000","last_undegraded":"2023-09-23T20:06:31.976185+0000","last_fullsized":"2023-09-23T20:06:31.976185+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:13.722150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":110,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:06:32.810663+0000","last_change":"2023-09-23T20:04:47.051162+0000","last_active":"2023-09-23T20:06:32.810663+0000","last_peered":"2023-09-23T20:06:32.810663+0000","last_clean":"2023-09-23T20:06:32.810663+0000","last_became_active":"2023-09-23T20:04:47.049811+0000","last_became_peered":"2023-09-23T20:04:47.049811+0000","last_unstale":"2023-09-23T20:06:32.810663+0000","last_undegraded":"2023-09-23T20:06:32.810663+0000","last_fullsized":"2023-09-23T20:06:32.810663+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":322,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:44.815308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":303,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.233076+0000","last_change":"2023-09-23T20:03:14.488499+0000","last_active":"2023-09-23T20:06:19.233076+0000","last_peered":"2023-09-23T20:06:19.233076+0000","last_clean":"2023-09-23T20:06:19.233076+0000","last_became_active":"2023-09-23T20:03:14.488096+0000","last_became_peered":"2023-09-23T20:03:14.488096+0000","last_unstale":"2023-09-23T20:06:19.233076+0000","last_undegraded":"2023-09-23T20:06:19.233076+0000","last_fullsized":"2023-09-23T20:06:19.233076+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":96,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:06:32.810778+0000","last_change":"2023-09-23T20:04:47.029088+0000","last_active":"2023-09-23T20:06:32.810778+0000","last_peered":"2023-09-23T20:06:32.810778+0000","last_clean":"2023-09-23T20:06:32.810778+0000","last_became_active":"2023-09-23T20:04:47.028684+0000","last_became_peered":"2023-09-23T20:04:47.028684+0000","last_unstale":"2023-09-23T20:06:32.810778+0000","last_undegraded":"2023-09-23T20:06:32.810778+0000","last_fullsized":"2023-09-23T20:06:32.810778+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":322,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:36:34.533261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":302,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.232242+0000","last_change":"2023-09-23T19:28:07.892835+0000","last_active":"2023-09-23T20:06:19.232242+0000","last_peered":"2023-09-23T20:06:19.232242+0000","last_clean":"2023-09-23T20:06:19.232242+0000","last_became_active":"2023-09-23T19:28:07.892338+0000","last_became_peered":"2023-09-23T19:28:07.892338+0000","last_unstale":"2023-09-23T20:06:19.232242+0000","last_undegraded":"2023-09-23T20:06:19.232242+0000","last_fullsized":"2023-09-23T20:06:19.232242+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":39,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:06:33.374753+0000","last_change":"2023-09-23T20:03:14.483257+0000","last_active":"2023-09-23T20:06:33.374753+0000","last_peered":"2023-09-23T20:06:33.374753+0000","last_clean":"2023-09-23T20:06:33.374753+0000","last_became_active":"2023-09-23T20:03:14.482217+0000","last_became_peered":"2023-09-23T20:03:14.482217+0000","last_unstale":"2023-09-23T20:06:33.374753+0000","last_undegraded":"2023-09-23T20:06:33.374753+0000","last_fullsized":"2023-09-23T20:06:33.374753+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:24:19.853891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'23","reported_seq":306,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.643166+0000","last_change":"2023-09-23T19:05:52.107727+0000","last_active":"2023-09-23T20:06:19.643166+0000","last_peered":"2023-09-23T20:06:19.643166+0000","last_clean":"2023-09-23T20:06:19.643166+0000","last_became_active":"2023-09-23T19:05:52.107378+0000","last_became_peered":"2023-09-23T19:05:52.107378+0000","last_unstale":"2023-09-23T20:06:19.643166+0000","last_undegraded":"2023-09-23T20:06:19.643166+0000","last_fullsized":"2023-09-23T20:06:19.643166+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":315,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.643461+0000","last_change":"2023-09-23T20:03:14.490617+0000","last_active":"2023-09-23T20:06:19.643461+0000","last_peered":"2023-09-23T20:06:19.643461+0000","last_clean":"2023-09-23T20:06:19.643461+0000","last_became_active":"2023-09-23T20:03:14.490198+0000","last_became_peered":"2023-09-23T20:03:14.490198+0000","last_unstale":"2023-09-23T20:06:19.643461+0000","last_undegraded":"2023-09-23T20:06:19.643461+0000","last_fullsized":"2023-09-23T20:06:19.643461+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":316,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.231985+0000","last_change":"2023-09-23T19:38:43.897378+0000","last_active":"2023-09-23T20:06:19.231985+0000","last_peered":"2023-09-23T20:06:19.231985+0000","last_clean":"2023-09-23T20:06:19.231985+0000","last_became_active":"2023-09-23T19:38:43.897012+0000","last_became_peered":"2023-09-23T19:38:43.897012+0000","last_unstale":"2023-09-23T20:06:19.231985+0000","last_undegraded":"2023-09-23T20:06:19.231985+0000","last_fullsized":"2023-09-23T20:06:19.231985+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":110,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:06:32.811215+0000","last_change":"2023-09-23T20:04:47.051062+0000","last_active":"2023-09-23T20:06:32.811215+0000","last_peered":"2023-09-23T20:06:32.811215+0000","last_clean":"2023-09-23T20:06:32.811215+0000","last_became_active":"2023-09-23T20:04:47.050134+0000","last_became_peered":"2023-09-23T20:04:47.050134+0000","last_unstale":"2023-09-23T20:06:32.811215+0000","last_undegraded":"2023-09-23T20:06:32.811215+0000","last_fullsized":"2023-09-23T20:06:32.811215+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":322,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:31:20.560311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":176,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:27.140676+0000","last_change":"2023-09-23T20:04:06.716624+0000","last_active":"2023-09-23T20:06:27.140676+0000","last_peered":"2023-09-23T20:06:27.140676+0000","last_clean":"2023-09-23T20:06:27.140676+0000","last_became_active":"2023-09-23T20:04:06.715866+0000","last_became_peered":"2023-09-23T20:04:06.715866+0000","last_unstale":"2023-09-23T20:06:27.140676+0000","last_undegraded":"2023-09-23T20:06:27.140676+0000","last_fullsized":"2023-09-23T20:06:27.140676+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:35:05.633686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":230,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.232877+0000","last_change":"2023-09-23T20:05:28.472296+0000","last_active":"2023-09-23T20:06:19.232877+0000","last_peered":"2023-09-23T20:06:19.232877+0000","last_clean":"2023-09-23T20:06:19.232877+0000","last_became_active":"2023-09-23T20:05:28.471242+0000","last_became_peered":"2023-09-23T20:05:28.471242+0000","last_unstale":"2023-09-23T20:06:19.232877+0000","last_undegraded":"2023-09-23T20:06:19.232877+0000","last_fullsized":"2023-09-23T20:06:19.232877+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":284,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.231337+0000","last_change":"2023-09-23T19:05:52.114186+0000","last_active":"2023-09-23T20:06:19.231337+0000","last_peered":"2023-09-23T20:06:19.231337+0000","last_clean":"2023-09-23T20:06:19.231337+0000","last_became_active":"2023-09-23T19:05:52.113635+0000","last_became_peered":"2023-09-23T19:05:52.113635+0000","last_unstale":"2023-09-23T20:06:19.231337+0000","last_undegraded":"2023-09-23T20:06:19.231337+0000","last_fullsized":"2023-09-23T20:06:19.231337+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":176,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:06:31.976012+0000","last_change":"2023-09-23T20:04:06.718614+0000","last_active":"2023-09-23T20:06:31.976012+0000","last_peered":"2023-09-23T20:06:31.976012+0000","last_clean":"2023-09-23T20:06:31.976012+0000","last_became_active":"2023-09-23T20:04:06.716485+0000","last_became_peered":"2023-09-23T20:04:06.716485+0000","last_unstale":"2023-09-23T20:06:31.976012+0000","last_undegraded":"2023-09-23T20:06:31.976012+0000","last_fullsized":"2023-09-23T20:06:31.976012+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:49.567649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":307,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.183953+0000","last_change":"2023-09-23T19:18:09.010195+0000","last_active":"2023-09-23T20:06:19.183953+0000","last_peered":"2023-09-23T20:06:19.183953+0000","last_clean":"2023-09-23T20:06:19.183953+0000","last_became_active":"2023-09-23T19:18:09.009687+0000","last_became_peered":"2023-09-23T19:18:09.009687+0000","last_unstale":"2023-09-23T20:06:19.183953+0000","last_undegraded":"2023-09-23T20:06:19.183953+0000","last_fullsized":"2023-09-23T20:06:19.183953+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":336,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:06:32.811499+0000","last_change":"2023-09-23T20:03:45.931421+0000","last_active":"2023-09-23T20:06:32.811499+0000","last_peered":"2023-09-23T20:06:32.811499+0000","last_clean":"2023-09-23T20:06:32.811499+0000","last_became_active":"2023-09-23T20:03:45.930726+0000","last_became_peered":"2023-09-23T20:03:45.930726+0000","last_unstale":"2023-09-23T20:06:32.811499+0000","last_undegraded":"2023-09-23T20:06:32.811499+0000","last_fullsized":"2023-09-23T20:06:32.811499+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:28:33.412847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":300,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.231987+0000","last_change":"2023-09-23T19:18:09.035116+0000","last_active":"2023-09-23T20:06:19.231987+0000","last_peered":"2023-09-23T20:06:19.231987+0000","last_clean":"2023-09-23T20:06:19.231987+0000","last_became_active":"2023-09-23T19:18:09.034416+0000","last_became_peered":"2023-09-23T19:18:09.034416+0000","last_unstale":"2023-09-23T20:06:19.231987+0000","last_undegraded":"2023-09-23T20:06:19.231987+0000","last_fullsized":"2023-09-23T20:06:19.231987+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":269,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.232445+0000","last_change":"2023-09-23T19:47:56.790227+0000","last_active":"2023-09-23T20:06:19.232445+0000","last_peered":"2023-09-23T20:06:19.232445+0000","last_clean":"2023-09-23T20:06:19.232445+0000","last_became_active":"2023-09-23T19:47:56.789727+0000","last_became_peered":"2023-09-23T19:47:56.789727+0000","last_unstale":"2023-09-23T20:06:19.232445+0000","last_undegraded":"2023-09-23T20:06:19.232445+0000","last_fullsized":"2023-09-23T20:06:19.232445+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":327,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.231991+0000","last_change":"2023-09-23T20:03:14.500188+0000","last_active":"2023-09-23T20:06:19.231991+0000","last_peered":"2023-09-23T20:06:19.231991+0000","last_clean":"2023-09-23T20:06:19.231991+0000","last_became_active":"2023-09-23T20:03:14.489458+0000","last_became_peered":"2023-09-23T20:03:14.489458+0000","last_unstale":"2023-09-23T20:06:19.231991+0000","last_undegraded":"2023-09-23T20:06:19.231991+0000","last_fullsized":"2023-09-23T20:06:19.231991+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":330,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.643130+0000","last_change":"2023-09-23T20:03:14.488674+0000","last_active":"2023-09-23T20:06:19.643130+0000","last_peered":"2023-09-23T20:06:19.643130+0000","last_clean":"2023-09-23T20:06:19.643130+0000","last_became_active":"2023-09-23T20:03:14.488290+0000","last_became_peered":"2023-09-23T20:03:14.488290+0000","last_unstale":"2023-09-23T20:06:19.643130+0000","last_undegraded":"2023-09-23T20:06:19.643130+0000","last_fullsized":"2023-09-23T20:06:19.643130+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":285,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.231719+0000","last_change":"2023-09-23T19:05:52.114108+0000","last_active":"2023-09-23T20:06:19.231719+0000","last_peered":"2023-09-23T20:06:19.231719+0000","last_clean":"2023-09-23T20:06:19.231719+0000","last_became_active":"2023-09-23T19:05:52.113609+0000","last_became_peered":"2023-09-23T19:05:52.113609+0000","last_unstale":"2023-09-23T20:06:19.231719+0000","last_undegraded":"2023-09-23T20:06:19.231719+0000","last_fullsized":"2023-09-23T20:06:19.231719+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":316,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.184438+0000","last_change":"2023-09-23T20:03:14.493455+0000","last_active":"2023-09-23T20:06:19.184438+0000","last_peered":"2023-09-23T20:06:19.184438+0000","last_clean":"2023-09-23T20:06:19.184438+0000","last_became_active":"2023-09-23T20:03:14.493053+0000","last_became_peered":"2023-09-23T20:03:14.493053+0000","last_unstale":"2023-09-23T20:06:19.184438+0000","last_undegraded":"2023-09-23T20:06:19.184438+0000","last_fullsized":"2023-09-23T20:06:19.184438+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":231,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.232879+0000","last_change":"2023-09-23T20:05:28.472196+0000","last_active":"2023-09-23T20:06:19.232879+0000","last_peered":"2023-09-23T20:06:19.232879+0000","last_clean":"2023-09-23T20:06:19.232879+0000","last_became_active":"2023-09-23T20:05:28.471259+0000","last_became_peered":"2023-09-23T20:05:28.471259+0000","last_unstale":"2023-09-23T20:06:19.232879+0000","last_undegraded":"2023-09-23T20:06:19.232879+0000","last_fullsized":"2023-09-23T20:06:19.232879+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":324,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:06:31.975993+0000","last_change":"2023-09-23T20:03:14.484103+0000","last_active":"2023-09-23T20:06:31.975993+0000","last_peered":"2023-09-23T20:06:31.975993+0000","last_clean":"2023-09-23T20:06:31.975993+0000","last_became_active":"2023-09-23T20:03:14.483704+0000","last_became_peered":"2023-09-23T20:03:14.483704+0000","last_unstale":"2023-09-23T20:06:31.975993+0000","last_undegraded":"2023-09-23T20:06:31.975993+0000","last_fullsized":"2023-09-23T20:06:31.975993+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:30:14.811132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":100,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:06:32.811190+0000","last_change":"2023-09-23T20:04:47.030508+0000","last_active":"2023-09-23T20:06:32.811190+0000","last_peered":"2023-09-23T20:06:32.811190+0000","last_clean":"2023-09-23T20:06:32.811190+0000","last_became_active":"2023-09-23T20:04:47.030061+0000","last_became_peered":"2023-09-23T20:04:47.030061+0000","last_unstale":"2023-09-23T20:06:32.811190+0000","last_undegraded":"2023-09-23T20:06:32.811190+0000","last_fullsized":"2023-09-23T20:06:32.811190+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":322,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:21.053552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":297,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.643314+0000","last_change":"2023-09-23T19:28:07.896803+0000","last_active":"2023-09-23T20:06:19.643314+0000","last_peered":"2023-09-23T20:06:19.643314+0000","last_clean":"2023-09-23T20:06:19.643314+0000","last_became_active":"2023-09-23T19:28:07.896480+0000","last_became_peered":"2023-09-23T19:28:07.896480+0000","last_unstale":"2023-09-23T20:06:19.643314+0000","last_undegraded":"2023-09-23T20:06:19.643314+0000","last_fullsized":"2023-09-23T20:06:19.643314+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":317,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.184414+0000","last_change":"2023-09-23T20:03:14.493451+0000","last_active":"2023-09-23T20:06:19.184414+0000","last_peered":"2023-09-23T20:06:19.184414+0000","last_clean":"2023-09-23T20:06:19.184414+0000","last_became_active":"2023-09-23T20:03:14.492920+0000","last_became_peered":"2023-09-23T20:03:14.492920+0000","last_unstale":"2023-09-23T20:06:19.184414+0000","last_undegraded":"2023-09-23T20:06:19.184414+0000","last_fullsized":"2023-09-23T20:06:19.184414+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":354,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.643292+0000","last_change":"2023-09-23T20:03:45.935493+0000","last_active":"2023-09-23T20:06:19.643292+0000","last_peered":"2023-09-23T20:06:19.643292+0000","last_clean":"2023-09-23T20:06:19.643292+0000","last_became_active":"2023-09-23T20:03:45.934909+0000","last_became_peered":"2023-09-23T20:03:45.934909+0000","last_unstale":"2023-09-23T20:06:19.643292+0000","last_undegraded":"2023-09-23T20:06:19.643292+0000","last_fullsized":"2023-09-23T20:06:19.643292+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":313,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.642665+0000","last_change":"2023-09-23T19:38:43.892972+0000","last_active":"2023-09-23T20:06:19.642665+0000","last_peered":"2023-09-23T20:06:19.642665+0000","last_clean":"2023-09-23T20:06:19.642665+0000","last_became_active":"2023-09-23T19:38:43.892451+0000","last_became_peered":"2023-09-23T19:38:43.892451+0000","last_unstale":"2023-09-23T20:06:19.642665+0000","last_undegraded":"2023-09-23T20:06:19.642665+0000","last_fullsized":"2023-09-23T20:06:19.642665+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":315,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.643815+0000","last_change":"2023-09-23T20:03:14.490494+0000","last_active":"2023-09-23T20:06:19.643815+0000","last_peered":"2023-09-23T20:06:19.643815+0000","last_clean":"2023-09-23T20:06:19.643815+0000","last_became_active":"2023-09-23T20:03:14.490147+0000","last_became_peered":"2023-09-23T20:03:14.490147+0000","last_unstale":"2023-09-23T20:06:19.643815+0000","last_undegraded":"2023-09-23T20:06:19.643815+0000","last_fullsized":"2023-09-23T20:06:19.643815+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":111,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:06:32.810886+0000","last_change":"2023-09-23T20:04:47.051293+0000","last_active":"2023-09-23T20:06:32.810886+0000","last_peered":"2023-09-23T20:06:32.810886+0000","last_clean":"2023-09-23T20:06:32.810886+0000","last_became_active":"2023-09-23T20:04:47.050289+0000","last_became_peered":"2023-09-23T20:04:47.050289+0000","last_unstale":"2023-09-23T20:06:32.810886+0000","last_undegraded":"2023-09-23T20:06:32.810886+0000","last_fullsized":"2023-09-23T20:06:32.810886+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":322,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:29:02.007222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":39,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:06:33.375211+0000","last_change":"2023-09-23T20:03:14.483670+0000","last_active":"2023-09-23T20:06:33.375211+0000","last_peered":"2023-09-23T20:06:33.375211+0000","last_clean":"2023-09-23T20:06:33.375211+0000","last_became_active":"2023-09-23T20:03:14.483318+0000","last_became_peered":"2023-09-23T20:03:14.483318+0000","last_unstale":"2023-09-23T20:06:33.375211+0000","last_undegraded":"2023-09-23T20:06:33.375211+0000","last_fullsized":"2023-09-23T20:06:33.375211+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:47.398395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"303'174","reported_seq":883,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:06:33.375295+0000","last_change":"2023-09-23T20:03:17.498400+0000","last_active":"2023-09-23T20:06:33.375295+0000","last_peered":"2023-09-23T20:06:33.375295+0000","last_clean":"2023-09-23T20:06:33.375295+0000","last_became_active":"2023-09-23T20:03:17.497416+0000","last_became_peered":"2023-09-23T20:03:17.497416+0000","last_unstale":"2023-09-23T20:06:33.375295+0000","last_undegraded":"2023-09-23T20:06:33.375295+0000","last_fullsized":"2023-09-23T20:06:33.375295+0000","mapping_epoch":309,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":310,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:12.558962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":804,"num_read_kb":2995,"num_write":184,"num_write_kb":2268,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":0,"num_keys_recovered":2,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":315,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.232948+0000","last_change":"2023-09-23T19:47:56.794437+0000","last_active":"2023-09-23T20:06:19.232948+0000","last_peered":"2023-09-23T20:06:19.232948+0000","last_clean":"2023-09-23T20:06:19.232948+0000","last_became_active":"2023-09-23T19:47:56.794118+0000","last_became_peered":"2023-09-23T19:47:56.794118+0000","last_unstale":"2023-09-23T20:06:19.232948+0000","last_undegraded":"2023-09-23T20:06:19.232948+0000","last_fullsized":"2023-09-23T20:06:19.232948+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":39,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:06:33.374519+0000","last_change":"2023-09-23T20:03:14.483749+0000","last_active":"2023-09-23T20:06:33.374519+0000","last_peered":"2023-09-23T20:06:33.374519+0000","last_clean":"2023-09-23T20:06:33.374519+0000","last_became_active":"2023-09-23T20:03:14.483398+0000","last_became_peered":"2023-09-23T20:03:14.483398+0000","last_unstale":"2023-09-23T20:06:33.374519+0000","last_undegraded":"2023-09-23T20:06:33.374519+0000","last_fullsized":"2023-09-23T20:06:33.374519+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:40:43.487777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"300'87","reported_seq":599,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.232615+0000","last_change":"2023-09-23T20:05:28.221260+0000","last_active":"2023-09-23T20:06:19.232615+0000","last_peered":"2023-09-23T20:06:19.232615+0000","last_clean":"2023-09-23T20:06:19.232615+0000","last_became_active":"2023-09-23T20:05:28.220457+0000","last_became_peered":"2023-09-23T20:05:28.220457+0000","last_unstale":"2023-09-23T20:06:19.232615+0000","last_undegraded":"2023-09-23T20:06:19.232615+0000","last_fullsized":"2023-09-23T20:06:19.232615+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":562,"num_read_kb":518,"num_write":90,"num_write_kb":1091,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":298,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.643102+0000","last_change":"2023-09-23T19:18:09.017525+0000","last_active":"2023-09-23T20:06:19.643102+0000","last_peered":"2023-09-23T20:06:19.643102+0000","last_clean":"2023-09-23T20:06:19.643102+0000","last_became_active":"2023-09-23T19:18:09.017193+0000","last_became_peered":"2023-09-23T19:18:09.017193+0000","last_unstale":"2023-09-23T20:06:19.643102+0000","last_undegraded":"2023-09-23T20:06:19.643102+0000","last_fullsized":"2023-09-23T20:06:19.643102+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":39,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:06:33.375254+0000","last_change":"2023-09-23T20:03:14.471701+0000","last_active":"2023-09-23T20:06:33.375254+0000","last_peered":"2023-09-23T20:06:33.375254+0000","last_clean":"2023-09-23T20:06:33.375254+0000","last_became_active":"2023-09-23T20:03:14.469272+0000","last_became_peered":"2023-09-23T20:03:14.469272+0000","last_unstale":"2023-09-23T20:06:33.375254+0000","last_undegraded":"2023-09-23T20:06:33.375254+0000","last_fullsized":"2023-09-23T20:06:33.375254+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:01.913425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"300'76","reported_seq":635,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:06:31.976664+0000","last_change":"2023-09-23T20:02:46.940411+0000","last_active":"2023-09-23T20:06:31.976664+0000","last_peered":"2023-09-23T20:06:31.976664+0000","last_clean":"2023-09-23T20:06:31.976664+0000","last_became_active":"2023-09-23T19:38:43.882421+0000","last_became_peered":"2023-09-23T19:38:43.882421+0000","last_unstale":"2023-09-23T20:06:31.976664+0000","last_undegraded":"2023-09-23T20:06:31.976664+0000","last_fullsized":"2023-09-23T20:06:31.976664+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":76,"log_dups_size":0,"ondisk_log_size":76,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:49:01.607659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099725700000000009,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":533,"num_read_kb":1101,"num_write":65,"num_write_kb":387,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":39,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:06:33.374588+0000","last_change":"2023-09-23T20:03:14.481700+0000","last_active":"2023-09-23T20:06:33.374588+0000","last_peered":"2023-09-23T20:06:33.374588+0000","last_clean":"2023-09-23T20:06:33.374588+0000","last_became_active":"2023-09-23T20:03:14.480935+0000","last_became_peered":"2023-09-23T20:03:14.480935+0000","last_unstale":"2023-09-23T20:06:33.374588+0000","last_undegraded":"2023-09-23T20:06:33.374588+0000","last_fullsized":"2023-09-23T20:06:33.374588+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:13.224166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":39,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:06:33.375353+0000","last_change":"2023-09-23T20:03:14.480704+0000","last_active":"2023-09-23T20:06:33.375353+0000","last_peered":"2023-09-23T20:06:33.375353+0000","last_clean":"2023-09-23T20:06:33.375353+0000","last_became_active":"2023-09-23T20:03:14.480216+0000","last_became_peered":"2023-09-23T20:03:14.480216+0000","last_unstale":"2023-09-23T20:06:33.375353+0000","last_undegraded":"2023-09-23T20:06:33.375353+0000","last_fullsized":"2023-09-23T20:06:33.375353+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:40:51.272381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"303'139","reported_seq":640,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.232694+0000","last_change":"2023-09-23T20:02:47.058854+0000","last_active":"2023-09-23T20:06:19.232694+0000","last_peered":"2023-09-23T20:06:19.232694+0000","last_clean":"2023-09-23T20:06:19.232694+0000","last_became_active":"2023-09-23T19:47:56.790954+0000","last_became_peered":"2023-09-23T19:47:56.790954+0000","last_unstale":"2023-09-23T20:06:19.232694+0000","last_undegraded":"2023-09-23T20:06:19.232694+0000","last_fullsized":"2023-09-23T20:06:19.232694+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":388,"num_read_kb":340,"num_write":136,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":535,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.231173+0000","last_change":"2023-09-23T19:18:09.016297+0000","last_active":"2023-09-23T20:06:19.231173+0000","last_peered":"2023-09-23T20:06:19.231173+0000","last_clean":"2023-09-23T20:06:19.231173+0000","last_became_active":"2023-09-23T19:18:09.015436+0000","last_became_peered":"2023-09-23T19:18:09.015436+0000","last_unstale":"2023-09-23T20:06:19.231173+0000","last_undegraded":"2023-09-23T20:06:19.231173+0000","last_fullsized":"2023-09-23T20:06:19.231173+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:59:33.840751+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:59:33.840751+0000","last_clean_scrub_stamp":"2023-09-23T18:59:33.840751+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":157,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:06:32.811612+0000","last_change":"2023-09-23T20:04:47.050132+0000","last_active":"2023-09-23T20:06:32.811612+0000","last_peered":"2023-09-23T20:06:32.811612+0000","last_clean":"2023-09-23T20:06:32.811612+0000","last_became_active":"2023-09-23T20:04:47.049460+0000","last_became_peered":"2023-09-23T20:04:47.049460+0000","last_unstale":"2023-09-23T20:06:32.811612+0000","last_undegraded":"2023-09-23T20:06:32.811612+0000","last_fullsized":"2023-09-23T20:06:32.811612+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":322,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:56.748778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":283,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.184926+0000","last_change":"2023-09-23T19:05:52.786771+0000","last_active":"2023-09-23T20:06:19.184926+0000","last_peered":"2023-09-23T20:06:19.184926+0000","last_clean":"2023-09-23T20:06:19.184926+0000","last_became_active":"2023-09-23T19:05:52.785797+0000","last_became_peered":"2023-09-23T19:05:52.785797+0000","last_unstale":"2023-09-23T20:06:19.184926+0000","last_undegraded":"2023-09-23T20:06:19.184926+0000","last_fullsized":"2023-09-23T20:06:19.184926+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"302'89","reported_seq":740,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.231144+0000","last_change":"2023-09-23T20:03:14.895983+0000","last_active":"2023-09-23T20:06:19.231144+0000","last_peered":"2023-09-23T20:06:19.231144+0000","last_clean":"2023-09-23T20:06:19.231144+0000","last_became_active":"2023-09-23T20:03:14.499439+0000","last_became_peered":"2023-09-23T20:03:14.499439+0000","last_unstale":"2023-09-23T20:06:19.231144+0000","last_undegraded":"2023-09-23T20:06:19.231144+0000","last_fullsized":"2023-09-23T20:06:19.231144+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":89,"log_dups_size":0,"ondisk_log_size":89,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":702,"num_read_kb":610,"num_write":76,"num_write_kb":51,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":177,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.223014+0000","last_change":"2023-09-23T20:04:06.491671+0000","last_active":"2023-09-23T20:06:19.223014+0000","last_peered":"2023-09-23T20:06:19.223014+0000","last_clean":"2023-09-23T20:06:19.223014+0000","last_became_active":"2023-09-23T20:04:06.491297+0000","last_became_peered":"2023-09-23T20:04:06.491297+0000","last_unstale":"2023-09-23T20:06:19.223014+0000","last_undegraded":"2023-09-23T20:06:19.223014+0000","last_fullsized":"2023-09-23T20:06:19.223014+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:30.618468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":355,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:06:31.975976+0000","last_change":"2023-09-23T20:03:14.485798+0000","last_active":"2023-09-23T20:06:31.975976+0000","last_peered":"2023-09-23T20:06:31.975976+0000","last_clean":"2023-09-23T20:06:31.975976+0000","last_became_active":"2023-09-23T20:03:14.485232+0000","last_became_peered":"2023-09-23T20:03:14.485232+0000","last_unstale":"2023-09-23T20:06:31.975976+0000","last_undegraded":"2023-09-23T20:06:31.975976+0000","last_fullsized":"2023-09-23T20:06:31.975976+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:37.416281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"303'146","reported_seq":896,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.231975+0000","last_change":"2023-09-23T20:02:46.954392+0000","last_active":"2023-09-23T20:06:19.231975+0000","last_peered":"2023-09-23T20:06:19.231975+0000","last_clean":"2023-09-23T20:06:19.231975+0000","last_became_active":"2023-09-23T19:18:09.015338+0000","last_became_peered":"2023-09-23T19:18:09.015338+0000","last_unstale":"2023-09-23T20:06:19.231975+0000","last_undegraded":"2023-09-23T20:06:19.231975+0000","last_fullsized":"2023-09-23T20:06:19.231975+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":146,"log_dups_size":0,"ondisk_log_size":146,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1086,"num_read_kb":2403,"num_write":113,"num_write_kb":1312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":39,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:06:33.374948+0000","last_change":"2023-09-23T20:03:14.476900+0000","last_active":"2023-09-23T20:06:33.374948+0000","last_peered":"2023-09-23T20:06:33.374948+0000","last_clean":"2023-09-23T20:06:33.374948+0000","last_became_active":"2023-09-23T20:03:14.475909+0000","last_became_peered":"2023-09-23T20:03:14.475909+0000","last_unstale":"2023-09-23T20:06:33.374948+0000","last_undegraded":"2023-09-23T20:06:33.374948+0000","last_fullsized":"2023-09-23T20:06:33.374948+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:29.191512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":162,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.223375+0000","last_change":"2023-09-23T20:04:06.716754+0000","last_active":"2023-09-23T20:06:19.223375+0000","last_peered":"2023-09-23T20:06:19.223375+0000","last_clean":"2023-09-23T20:06:19.223375+0000","last_became_active":"2023-09-23T20:04:06.716156+0000","last_became_peered":"2023-09-23T20:04:06.716156+0000","last_unstale":"2023-09-23T20:06:19.223375+0000","last_undegraded":"2023-09-23T20:06:19.223375+0000","last_fullsized":"2023-09-23T20:06:19.223375+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:54:57.478605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":100,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:06:32.811567+0000","last_change":"2023-09-23T20:04:47.027885+0000","last_active":"2023-09-23T20:06:32.811567+0000","last_peered":"2023-09-23T20:06:32.811567+0000","last_clean":"2023-09-23T20:06:32.811567+0000","last_became_active":"2023-09-23T20:04:47.027383+0000","last_became_peered":"2023-09-23T20:04:47.027383+0000","last_unstale":"2023-09-23T20:06:32.811567+0000","last_undegraded":"2023-09-23T20:06:32.811567+0000","last_fullsized":"2023-09-23T20:06:32.811567+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":322,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:43.621489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"303'124","reported_seq":793,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:06:33.375315+0000","last_change":"2023-09-23T20:03:17.500586+0000","last_active":"2023-09-23T20:06:33.375315+0000","last_peered":"2023-09-23T20:06:33.375315+0000","last_clean":"2023-09-23T20:06:33.375315+0000","last_became_active":"2023-09-23T20:03:17.499921+0000","last_became_peered":"2023-09-23T20:03:17.499921+0000","last_unstale":"2023-09-23T20:06:33.375315+0000","last_undegraded":"2023-09-23T20:06:33.375315+0000","last_fullsized":"2023-09-23T20:06:33.375315+0000","mapping_epoch":309,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":310,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:06:40.172512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":811,"num_read_kb":1848,"num_write":72,"num_write_kb":5235,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":97,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:06:32.810736+0000","last_change":"2023-09-23T20:04:47.030013+0000","last_active":"2023-09-23T20:06:32.810736+0000","last_peered":"2023-09-23T20:06:32.810736+0000","last_clean":"2023-09-23T20:06:32.810736+0000","last_became_active":"2023-09-23T20:04:47.029620+0000","last_became_peered":"2023-09-23T20:04:47.029620+0000","last_unstale":"2023-09-23T20:06:32.810736+0000","last_undegraded":"2023-09-23T20:06:32.810736+0000","last_fullsized":"2023-09-23T20:06:32.810736+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":322,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:19:55.728860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"300'141","reported_seq":896,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.184683+0000","last_change":"2023-09-23T20:02:47.022756+0000","last_active":"2023-09-23T20:06:19.184683+0000","last_peered":"2023-09-23T20:06:19.184683+0000","last_clean":"2023-09-23T20:06:19.184683+0000","last_became_active":"2023-09-23T19:38:43.895486+0000","last_became_peered":"2023-09-23T19:38:43.895486+0000","last_unstale":"2023-09-23T20:06:19.184683+0000","last_undegraded":"2023-09-23T20:06:19.184683+0000","last_fullsized":"2023-09-23T20:06:19.184683+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:04:01.490591+0000","last_clean_scrub_stamp":"2023-09-23T19:04:01.490591+0000","objects_scrubbed":0,"log_size":141,"log_dups_size":0,"ondisk_log_size":141,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1004,"num_read_kb":2306,"num_write":110,"num_write_kb":1264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":336,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:06:31.975780+0000","last_change":"2023-09-23T20:03:14.547047+0000","last_active":"2023-09-23T20:06:31.975780+0000","last_peered":"2023-09-23T20:06:31.975780+0000","last_clean":"2023-09-23T20:06:31.975780+0000","last_became_active":"2023-09-23T20:03:14.486040+0000","last_became_peered":"2023-09-23T20:03:14.486040+0000","last_unstale":"2023-09-23T20:06:31.975780+0000","last_undegraded":"2023-09-23T20:06:31.975780+0000","last_fullsized":"2023-09-23T20:06:31.975780+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:14.044993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":282,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.642706+0000","last_change":"2023-09-23T19:05:52.138620+0000","last_active":"2023-09-23T20:06:19.642706+0000","last_peered":"2023-09-23T20:06:19.642706+0000","last_clean":"2023-09-23T20:06:19.642706+0000","last_became_active":"2023-09-23T19:05:52.138243+0000","last_became_peered":"2023-09-23T19:05:52.138243+0000","last_unstale":"2023-09-23T20:06:19.642706+0000","last_undegraded":"2023-09-23T20:06:19.642706+0000","last_fullsized":"2023-09-23T20:06:19.642706+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":291,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.223139+0000","last_change":"2023-09-23T19:18:09.008811+0000","last_active":"2023-09-23T20:06:19.223139+0000","last_peered":"2023-09-23T20:06:19.223139+0000","last_clean":"2023-09-23T20:06:19.223139+0000","last_became_active":"2023-09-23T19:18:09.007255+0000","last_became_peered":"2023-09-23T19:18:09.007255+0000","last_unstale":"2023-09-23T20:06:19.223139+0000","last_undegraded":"2023-09-23T20:06:19.223139+0000","last_fullsized":"2023-09-23T20:06:19.223139+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:25.874038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":313,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.357659+0000","last_change":"2023-09-23T20:03:14.469248+0000","last_active":"2023-09-23T20:06:19.357659+0000","last_peered":"2023-09-23T20:06:19.357659+0000","last_clean":"2023-09-23T20:06:19.357659+0000","last_became_active":"2023-09-23T20:03:14.468878+0000","last_became_peered":"2023-09-23T20:03:14.468878+0000","last_unstale":"2023-09-23T20:06:19.357659+0000","last_undegraded":"2023-09-23T20:06:19.357659+0000","last_fullsized":"2023-09-23T20:06:19.357659+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:02:59.063744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":177,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.232922+0000","last_change":"2023-09-23T20:05:28.218979+0000","last_active":"2023-09-23T20:06:19.232922+0000","last_peered":"2023-09-23T20:06:19.232922+0000","last_clean":"2023-09-23T20:06:19.232922+0000","last_became_active":"2023-09-23T20:05:28.218478+0000","last_became_peered":"2023-09-23T20:05:28.218478+0000","last_unstale":"2023-09-23T20:06:19.232922+0000","last_undegraded":"2023-09-23T20:06:19.232922+0000","last_fullsized":"2023-09-23T20:06:19.232922+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":345,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.643557+0000","last_change":"2023-09-23T20:03:14.488715+0000","last_active":"2023-09-23T20:06:19.643557+0000","last_peered":"2023-09-23T20:06:19.643557+0000","last_clean":"2023-09-23T20:06:19.643557+0000","last_became_active":"2023-09-23T20:03:14.488323+0000","last_became_peered":"2023-09-23T20:03:14.488323+0000","last_unstale":"2023-09-23T20:06:19.643557+0000","last_undegraded":"2023-09-23T20:06:19.643557+0000","last_fullsized":"2023-09-23T20:06:19.643557+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":218,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.233090+0000","last_change":"2023-09-23T20:05:28.220932+0000","last_active":"2023-09-23T20:06:19.233090+0000","last_peered":"2023-09-23T20:06:19.233090+0000","last_clean":"2023-09-23T20:06:19.233090+0000","last_became_active":"2023-09-23T20:05:28.220462+0000","last_became_peered":"2023-09-23T20:05:28.220462+0000","last_unstale":"2023-09-23T20:06:19.233090+0000","last_undegraded":"2023-09-23T20:06:19.233090+0000","last_fullsized":"2023-09-23T20:06:19.233090+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":312,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.643068+0000","last_change":"2023-09-23T19:28:07.896983+0000","last_active":"2023-09-23T20:06:19.643068+0000","last_peered":"2023-09-23T20:06:19.643068+0000","last_clean":"2023-09-23T20:06:19.643068+0000","last_became_active":"2023-09-23T19:28:07.896577+0000","last_became_peered":"2023-09-23T19:28:07.896577+0000","last_unstale":"2023-09-23T20:06:19.643068+0000","last_undegraded":"2023-09-23T20:06:19.643068+0000","last_fullsized":"2023-09-23T20:06:19.643068+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":124,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:06:32.811414+0000","last_change":"2023-09-23T20:04:47.027195+0000","last_active":"2023-09-23T20:06:32.811414+0000","last_peered":"2023-09-23T20:06:32.811414+0000","last_clean":"2023-09-23T20:06:32.811414+0000","last_became_active":"2023-09-23T20:04:47.026583+0000","last_became_peered":"2023-09-23T20:04:47.026583+0000","last_unstale":"2023-09-23T20:06:32.811414+0000","last_undegraded":"2023-09-23T20:06:32.811414+0000","last_fullsized":"2023-09-23T20:06:32.811414+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":322,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:17:45.781926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":136,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:06:32.810791+0000","last_change":"2023-09-23T20:04:47.019080+0000","last_active":"2023-09-23T20:06:32.810791+0000","last_peered":"2023-09-23T20:06:32.810791+0000","last_clean":"2023-09-23T20:06:32.810791+0000","last_became_active":"2023-09-23T20:04:47.018529+0000","last_became_peered":"2023-09-23T20:04:47.018529+0000","last_unstale":"2023-09-23T20:06:32.810791+0000","last_undegraded":"2023-09-23T20:06:32.810791+0000","last_fullsized":"2023-09-23T20:06:32.810791+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":322,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:13.056894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":325,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.223344+0000","last_change":"2023-09-23T19:38:43.893983+0000","last_active":"2023-09-23T20:06:19.223344+0000","last_peered":"2023-09-23T20:06:19.223344+0000","last_clean":"2023-09-23T20:06:19.223344+0000","last_became_active":"2023-09-23T19:38:43.893527+0000","last_became_peered":"2023-09-23T19:38:43.893527+0000","last_unstale":"2023-09-23T20:06:19.223344+0000","last_undegraded":"2023-09-23T20:06:19.223344+0000","last_fullsized":"2023-09-23T20:06:19.223344+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:51.067106+0000","last_clean_scrub_stamp":"2023-09-23T19:05:51.067106+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:12.156531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":309,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:06:19.642628+0000","last_change":"2023-09-23T20:03:14.488733+0000","last_active":"2023-09-23T20:06:19.642628+0000","last_peered":"2023-09-23T20:06:19.642628+0000","last_clean":"2023-09-23T20:06:19.642628+0000","last_became_active":"2023-09-23T20:03:14.488370+0000","last_became_peered":"2023-09-23T20:03:14.488370+0000","last_unstale":"2023-09-23T20:06:19.642628+0000","last_undegraded":"2023-09-23T20:06:19.642628+0000","last_fullsized":"2023-09-23T20:06:19.642628+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:05:52.075332+0000","last_clean_scrub_stamp":"2023-09-23T19:05:52.075332+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5890,"num_read_kb":12121,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":69,"num_bytes_recovered":4194323,"num_keys_recovered":2,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430991,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1179540,"kb_used_data":3468,"kb_used_omap":0,"kb_used_meta":1176064,"kb_avail":92565612,"statfs":{"total":95995035648,"available":94787186688,"internally_reserved":0,"allocated":3551232,"data_stored":2949211,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1204289536},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:07:32 2023","interfaces":[{"interface":"back","average":{"1min":0.83099999999999996,"5min":0.86399999999999999,"15min":1.3680000000000001},"min":{"1min":0.56200000000000006,"5min":0.52700000000000002,"15min":0.26700000000000002},"max":{"1min":1.252,"5min":1.3779999999999999,"15min":41.570999999999998},"last":0.85099999999999998},{"interface":"front","average":{"1min":0.88400000000000001,"5min":0.86399999999999999,"15min":1.542},"min":{"1min":0.60799999999999998,"5min":0.46000000000000002,"15min":0.28000000000000003},"max":{"1min":1.1180000000000001,"5min":1.4630000000000001,"15min":45.634},"last":1.036}]},{"osd":1,"last update":"Sat Sep 23 20:07:51 2023","interfaces":[{"interface":"back","average":{"1min":0.94399999999999995,"5min":0.99299999999999999,"15min":2.1349999999999998},"min":{"1min":0.628,"5min":0.60099999999999998,"15min":0.28299999999999997},"max":{"1min":1.2030000000000001,"5min":1.702,"15min":149.018},"last":1.2789999999999999},{"interface":"front","average":{"1min":1.0069999999999999,"5min":1.0409999999999999,"15min":2.145},"min":{"1min":0.79900000000000004,"5min":0.55600000000000005,"15min":0.254},"max":{"1min":1.2230000000000001,"5min":1.6140000000000001,"15min":148.995},"last":1.0880000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:07:32 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":0.93300000000000005,"15min":2.4329999999999998},"min":{"1min":0.68300000000000005,"5min":0.60699999999999998,"15min":0.31900000000000001},"max":{"1min":1.133,"5min":1.405,"15min":160.655},"last":1.1859999999999999},{"interface":"front","average":{"1min":0.88400000000000001,"5min":0.88900000000000001,"15min":2.6720000000000002},"min":{"1min":0.624,"5min":0.57699999999999996,"15min":0.30199999999999999},"max":{"1min":1.175,"5min":1.3049999999999999,"15min":176.428},"last":1.1619999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:07:26 2023","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":1.018,"15min":1.383},"min":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.32600000000000001},"max":{"1min":1.1679999999999999,"5min":1.405,"15min":10.615},"last":0.88200000000000001},{"interface":"front","average":{"1min":1.085,"5min":1.0980000000000001,"15min":1.4099999999999999},"min":{"1min":0.85499999999999998,"5min":0.66000000000000003,"15min":0.33800000000000002},"max":{"1min":1.337,"5min":1.4330000000000001,"15min":9.9179999999999993},"last":1.3999999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:07:51 2023","interfaces":[{"interface":"back","average":{"1min":0.93200000000000005,"5min":0.91700000000000004,"15min":1.5029999999999999},"min":{"1min":0.63900000000000001,"5min":0.48799999999999999,"15min":0.27700000000000002},"max":{"1min":1.226,"5min":1.4750000000000001,"15min":13.484999999999999},"last":1.1240000000000001},{"interface":"front","average":{"1min":1.071,"5min":0.998,"15min":1.3879999999999999},"min":{"1min":0.57599999999999996,"5min":0.53400000000000003,"15min":0.313},"max":{"1min":1.2430000000000001,"5min":1.5620000000000001,"15min":13.116},"last":1.306}]},{"osd":5,"last update":"Sat Sep 23 20:07:51 2023","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":0.93899999999999995,"15min":1.399},"min":{"1min":0.77900000000000003,"5min":0.54400000000000004,"15min":0.28499999999999998},"max":{"1min":1.196,"5min":1.595,"15min":13.321999999999999},"last":1.254},{"interface":"front","average":{"1min":1.1000000000000001,"5min":1.0669999999999999,"15min":1.4279999999999999},"min":{"1min":0.76200000000000001,"5min":0.69399999999999995,"15min":0.315},"max":{"1min":1.325,"5min":1.4359999999999999,"15min":13.478999999999999},"last":1.331}]},{"osd":6,"last update":"Sat Sep 23 20:07:51 2023","interfaces":[{"interface":"back","average":{"1min":1.0840000000000001,"5min":1.077,"15min":1.339},"min":{"1min":0.72799999999999998,"5min":0.59999999999999998,"15min":0.34300000000000003},"max":{"1min":1.3009999999999999,"5min":1.4970000000000001,"15min":13.436},"last":1.218},{"interface":"front","average":{"1min":1.1779999999999999,"5min":1.1080000000000001,"15min":1.3260000000000001},"min":{"1min":0.95999999999999996,"5min":0.49299999999999999,"15min":0.35599999999999998},"max":{"1min":1.3819999999999999,"5min":1.702,"15min":10.766999999999999},"last":1.3580000000000001}]}]},{"osd":6,"up_from":42,"seq":180388627223,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":890568,"kb_used_data":7552,"kb_used_omap":0,"kb_used_meta":883008,"kb_avail":92854584,"statfs":{"total":95995035648,"available":95083094016,"internally_reserved":0,"allocated":7733248,"data_stored":7143073,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":904200192},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:07:53 2023","interfaces":[{"interface":"back","average":{"1min":0.86899999999999999,"5min":0.86599999999999999,"15min":2.5030000000000001},"min":{"1min":0.55900000000000005,"5min":0.39100000000000001,"15min":0.23799999999999999},"max":{"1min":1.339,"5min":1.347,"15min":125.34399999999999},"last":0.95799999999999996},{"interface":"front","average":{"1min":0.91100000000000003,"5min":0.91600000000000004,"15min":2.5129999999999999},"min":{"1min":0.628,"5min":0.57599999999999996,"15min":0.27800000000000002},"max":{"1min":1.3129999999999999,"5min":1.4570000000000001,"15min":125.24299999999999},"last":1.456}]},{"osd":1,"last update":"Sat Sep 23 20:07:12 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":0.96699999999999997,"15min":1.6990000000000001},"min":{"1min":0.65500000000000003,"5min":0.40600000000000003,"15min":0.29699999999999999},"max":{"1min":1.327,"5min":2.1629999999999998,"15min":125.51900000000001},"last":1.091},{"interface":"front","average":{"1min":1.1830000000000001,"5min":1.075,"15min":1.7909999999999999},"min":{"1min":0.92700000000000005,"5min":0.53100000000000003,"15min":0.27700000000000002},"max":{"1min":1.4470000000000001,"5min":2.181,"15min":125.596},"last":1.2330000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:07:27 2023","interfaces":[{"interface":"back","average":{"1min":0.97299999999999998,"5min":0.97499999999999998,"15min":2.7000000000000002},"min":{"1min":0.70299999999999996,"5min":0.57299999999999995,"15min":0.29799999999999999},"max":{"1min":1.137,"5min":2.6230000000000002,"15min":219.453},"last":0.78600000000000003},{"interface":"front","average":{"1min":0.97499999999999998,"5min":1.0049999999999999,"15min":2.7650000000000001},"min":{"1min":0.63800000000000001,"5min":0.56899999999999995,"15min":0.30199999999999999},"max":{"1min":1.3069999999999999,"5min":2.1139999999999999,"15min":219.369},"last":0.997}]},{"osd":3,"last update":"Sat Sep 23 20:07:27 2023","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":0.94399999999999995,"15min":1.369},"min":{"1min":0.60799999999999998,"5min":0.56299999999999994,"15min":0.28599999999999998},"max":{"1min":1.284,"5min":1.3089999999999999,"15min":13.823},"last":1.0509999999999999},{"interface":"front","average":{"1min":0.96699999999999997,"5min":1.014,"15min":1.478},"min":{"1min":0.66600000000000004,"5min":0.44600000000000001,"15min":0.32600000000000001},"max":{"1min":1.248,"5min":1.6870000000000001,"15min":14.42},"last":1.1879999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:07:27 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":0.98699999999999999,"15min":1.5509999999999999},"min":{"1min":0.63500000000000001,"5min":0.52200000000000002,"15min":0.30599999999999999},"max":{"1min":1.4890000000000001,"5min":1.643,"15min":11.912000000000001},"last":1.3839999999999999},{"interface":"front","average":{"1min":1.002,"5min":0.98699999999999999,"15min":1.5780000000000001},"min":{"1min":0.71599999999999997,"5min":0.47699999999999998,"15min":0.32200000000000001},"max":{"1min":1.2,"5min":1.3560000000000001,"15min":14.239000000000001},"last":1.151}]},{"osd":5,"last update":"Sat Sep 23 20:07:27 2023","interfaces":[{"interface":"back","average":{"1min":1.079,"5min":1.0880000000000001,"15min":1.6379999999999999},"min":{"1min":0.72399999999999998,"5min":0.57199999999999995,"15min":0.33300000000000002},"max":{"1min":1.381,"5min":1.5149999999999999,"15min":15.045},"last":1.125},{"interface":"front","average":{"1min":1.0649999999999999,"5min":1.0609999999999999,"15min":1.7430000000000001},"min":{"1min":0.79300000000000004,"5min":0.51600000000000001,"15min":0.33400000000000002},"max":{"1min":1.2829999999999999,"5min":1.597,"15min":16.244},"last":1.264}]},{"osd":7,"last update":"Sat Sep 23 20:07:53 2023","interfaces":[{"interface":"back","average":{"1min":1.1279999999999999,"5min":1.1479999999999999,"15min":1.575},"min":{"1min":0.75,"5min":0.58599999999999997,"15min":0.34599999999999997},"max":{"1min":1.411,"5min":1.946,"15min":11.709},"last":1.3200000000000001},{"interface":"front","average":{"1min":1.0960000000000001,"5min":1.123,"15min":1.4510000000000001},"min":{"1min":0.85899999999999999,"5min":0.59399999999999997,"15min":0.371},"max":{"1min":1.488,"5min":1.71,"15min":11.000999999999999},"last":1.4279999999999999}]}]},{"osd":1,"up_from":98,"seq":420906795555,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":930748,"kb_used_data":7796,"kb_used_omap":4,"kb_used_meta":922939,"kb_avail":92814404,"statfs":{"total":95995035648,"available":95041949696,"internally_reserved":0,"allocated":7983104,"data_stored":6640805,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":945090446},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:07:08 2023","interfaces":[{"interface":"back","average":{"1min":0.748,"5min":0.75900000000000001,"15min":2.0259999999999998},"min":{"1min":0.47899999999999998,"5min":0.247,"15min":0.221},"max":{"1min":1.544,"5min":1.544,"15min":134.37700000000001},"last":0.92600000000000005},{"interface":"front","average":{"1min":0.879,"5min":0.78700000000000003,"15min":2},"min":{"1min":0.42799999999999999,"5min":0.32900000000000001,"15min":0.20200000000000001},"max":{"1min":2.3940000000000001,"5min":2.3940000000000001,"15min":135.79400000000001},"last":0.89000000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:07:39 2023","interfaces":[{"interface":"back","average":{"1min":0.79700000000000004,"5min":0.80900000000000005,"15min":2.1070000000000002},"min":{"1min":0.443,"5min":0.28599999999999998,"15min":0.23300000000000001},"max":{"1min":2.0489999999999999,"5min":2.0489999999999999,"15min":48.512},"last":0.77400000000000002},{"interface":"front","average":{"1min":0.73999999999999999,"5min":0.78700000000000003,"15min":1.8029999999999999},"min":{"1min":0.434,"5min":0.27600000000000002,"15min":0.224},"max":{"1min":2.0830000000000002,"5min":2.0830000000000002,"15min":48.664000000000001},"last":0.81299999999999994}]},{"osd":3,"last update":"Sat Sep 23 20:07:39 2023","interfaces":[{"interface":"back","average":{"1min":0.78200000000000003,"5min":0.878,"15min":1.6599999999999999},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.26200000000000001},"max":{"1min":2.339,"5min":2.339,"15min":41.518000000000001},"last":0.99199999999999999},{"interface":"front","average":{"1min":0.79900000000000004,"5min":0.80100000000000005,"15min":1.8200000000000001},"min":{"1min":0.45600000000000002,"5min":0.38500000000000001,"15min":0.252},"max":{"1min":2.0230000000000001,"5min":2.0230000000000001,"15min":41.151000000000003},"last":0.51200000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:07:08 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":1.0569999999999999,"15min":2.0019999999999998},"min":{"1min":0.71899999999999997,"5min":0.51900000000000002,"15min":0.28499999999999998},"max":{"1min":2.4529999999999998,"5min":2.4529999999999998,"15min":48.381},"last":1.04},{"interface":"front","average":{"1min":1.0429999999999999,"5min":1.0109999999999999,"15min":1.7789999999999999},"min":{"1min":0.70099999999999996,"5min":0.58099999999999996,"15min":0.35099999999999998},"max":{"1min":2.4140000000000001,"5min":2.4140000000000001,"15min":41.320999999999998},"last":1.016}]},{"osd":5,"last update":"Sat Sep 23 20:07:08 2023","interfaces":[{"interface":"back","average":{"1min":1.101,"5min":1.0349999999999999,"15min":1.8480000000000001},"min":{"1min":0.61899999999999999,"5min":0.54700000000000004,"15min":0.38200000000000001},"max":{"1min":2.2149999999999999,"5min":2.2149999999999999,"15min":48.643000000000001},"last":0.85199999999999998},{"interface":"front","average":{"1min":1.177,"5min":1.1639999999999999,"15min":1.897},"min":{"1min":0.70999999999999996,"5min":0.44600000000000001,"15min":0.33300000000000002},"max":{"1min":2.4940000000000002,"5min":2.4940000000000002,"15min":48.619999999999997},"last":1.139}]},{"osd":6,"last update":"Sat Sep 23 20:07:08 2023","interfaces":[{"interface":"back","average":{"1min":1.206,"5min":1.2,"15min":2.1200000000000001},"min":{"1min":0.75700000000000001,"5min":0.61499999999999999,"15min":0.35199999999999998},"max":{"1min":2.4300000000000002,"5min":2.4300000000000002,"15min":48.548999999999999},"last":1.0840000000000001},{"interface":"front","average":{"1min":1.2330000000000001,"5min":1.1830000000000001,"15min":2.0270000000000001},"min":{"1min":0.80900000000000005,"5min":0.57199999999999995,"15min":0.313},"max":{"1min":2.2770000000000001,"5min":2.2770000000000001,"15min":41.648000000000003},"last":1.2769999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:07:08 2023","interfaces":[{"interface":"back","average":{"1min":1.276,"5min":1.208,"15min":2.1160000000000001},"min":{"1min":0.85199999999999998,"5min":0.56699999999999995,"15min":0.38300000000000001},"max":{"1min":2.5099999999999998,"5min":2.5099999999999998,"15min":48.329999999999998},"last":1.345},{"interface":"front","average":{"1min":1.2290000000000001,"5min":1.1779999999999999,"15min":1.994},"min":{"1min":0.66800000000000004,"5min":0.63900000000000001,"15min":0.42699999999999999},"max":{"1min":2.4740000000000002,"5min":2.4740000000000002,"15min":48.579000000000001},"last":0.95299999999999996}]}]},{"osd":0,"up_from":85,"seq":365072220761,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":981848,"kb_used_data":4240,"kb_used_omap":5,"kb_used_meta":977594,"kb_avail":92763304,"statfs":{"total":95995035648,"available":94989623296,"internally_reserved":0,"allocated":4341760,"data_stored":3036322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1001056890},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:07:13 2023","interfaces":[{"interface":"back","average":{"1min":0.70699999999999996,"5min":0.71899999999999997,"15min":2.286},"min":{"1min":0.318,"5min":0.318,"15min":0.219},"max":{"1min":1.0640000000000001,"5min":1.3899999999999999,"15min":149.37299999999999},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.72599999999999998,"5min":0.72099999999999997,"15min":2.3290000000000002},"min":{"1min":0.45700000000000002,"5min":0.33000000000000002,"15min":0.191},"max":{"1min":0.96899999999999997,"5min":1.47,"15min":147.119},"last":0.59199999999999997}]},{"osd":2,"last update":"Sat Sep 23 20:07:30 2023","interfaces":[{"interface":"back","average":{"1min":0.80400000000000005,"5min":0.77600000000000002,"15min":1.5089999999999999},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.25600000000000001},"max":{"1min":2.4660000000000002,"5min":2.4660000000000002,"15min":11.646000000000001},"last":0.73499999999999999},{"interface":"front","average":{"1min":0.82599999999999996,"5min":0.79400000000000004,"15min":1.339},"min":{"1min":0.38700000000000001,"5min":0.36499999999999999,"15min":0.25700000000000001},"max":{"1min":2.1960000000000002,"5min":2.1960000000000002,"15min":11.542},"last":1.0229999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:07:13 2023","interfaces":[{"interface":"back","average":{"1min":0.95099999999999996,"5min":0.93799999999999994,"15min":1.454},"min":{"1min":0.499,"5min":0.41299999999999998,"15min":0.28100000000000003},"max":{"1min":1.702,"5min":1.702,"15min":14.983000000000001},"last":1.169},{"interface":"front","average":{"1min":0.90600000000000003,"5min":0.90400000000000003,"15min":1.379},"min":{"1min":0.47599999999999998,"5min":0.46800000000000003,"15min":0.29999999999999999},"max":{"1min":1.3380000000000001,"5min":1.431,"15min":14.843},"last":0.65500000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:07:49 2023","interfaces":[{"interface":"back","average":{"1min":0.97099999999999997,"5min":1.0529999999999999,"15min":1.3839999999999999},"min":{"1min":0.626,"5min":0.626,"15min":0.32000000000000001},"max":{"1min":1.5169999999999999,"5min":1.79,"15min":11.398},"last":0.85499999999999998},{"interface":"front","average":{"1min":1.0029999999999999,"5min":1.0309999999999999,"15min":1.429},"min":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.30199999999999999},"max":{"1min":1.3759999999999999,"5min":1.4259999999999999,"15min":10.747},"last":0.83499999999999996}]},{"osd":5,"last update":"Sat Sep 23 20:07:49 2023","interfaces":[{"interface":"back","average":{"1min":1.073,"5min":1.119,"15min":1.45},"min":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.316},"max":{"1min":1.4299999999999999,"5min":1.579,"15min":10.69},"last":0.79300000000000004},{"interface":"front","average":{"1min":1.1140000000000001,"5min":1.194,"15min":1.4670000000000001},"min":{"1min":0.77400000000000002,"5min":0.755,"15min":0.36499999999999999},"max":{"1min":1.637,"5min":2.0550000000000002,"15min":11.510999999999999},"last":1.3120000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:07:49 2023","interfaces":[{"interface":"back","average":{"1min":1.1759999999999999,"5min":1.218,"15min":1.5580000000000001},"min":{"1min":0.82099999999999995,"5min":0.753,"15min":0.36699999999999999},"max":{"1min":1.8320000000000001,"5min":1.9950000000000001,"15min":11.443},"last":1.232},{"interface":"front","average":{"1min":1.1200000000000001,"5min":1.1879999999999999,"15min":1.619},"min":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.36599999999999999},"max":{"1min":1.478,"5min":1.5840000000000001,"15min":17.050999999999998},"last":0.97899999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:07:49 2023","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":1.1379999999999999,"15min":2.1989999999999998},"min":{"1min":0.61099999999999999,"5min":0.54500000000000004,"15min":0.42099999999999999},"max":{"1min":1.5660000000000001,"5min":1.5680000000000001,"15min":210.47399999999999},"last":1.006},{"interface":"front","average":{"1min":1.117,"5min":1.2110000000000001,"15min":2.2509999999999999},"min":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.41099999999999998},"max":{"1min":1.7470000000000001,"5min":1.909,"15min":210.52199999999999},"last":1.3740000000000001}]}]},{"osd":2,"up_from":159,"seq":682899800417,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":737364,"kb_used_data":3660,"kb_used_omap":7,"kb_used_meta":733688,"kb_avail":93007788,"statfs":{"total":95995035648,"available":95239974912,"internally_reserved":0,"allocated":3747840,"data_stored":2440299,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":751296593},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:07:48 2023","interfaces":[{"interface":"back","average":{"1min":0.77900000000000003,"5min":0.81599999999999995,"15min":1.367},"min":{"1min":0.33300000000000002,"5min":0.30099999999999999,"15min":0.19700000000000001},"max":{"1min":3.0550000000000002,"5min":3.0550000000000002,"15min":14.624000000000001},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.81100000000000005,"5min":0.83699999999999997,"15min":1.4710000000000001},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.223},"max":{"1min":3.0270000000000001,"5min":3.0270000000000001,"15min":40.539000000000001},"last":0.57699999999999996}]},{"osd":1,"last update":"Sat Sep 23 20:07:48 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":0.80200000000000005,"15min":2.0259999999999998},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.22600000000000001},"max":{"1min":2.8180000000000001,"5min":2.8180000000000001,"15min":84.234999999999999},"last":0.70399999999999996},{"interface":"front","average":{"1min":0.71099999999999997,"5min":0.84799999999999998,"15min":1.75},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.248},"max":{"1min":1.1120000000000001,"5min":1.4470000000000001,"15min":84.150000000000006},"last":0.84599999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:07:48 2023","interfaces":[{"interface":"back","average":{"1min":0.81899999999999995,"5min":0.85899999999999999,"15min":1.514},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.25},"max":{"1min":1.3129999999999999,"5min":1.6890000000000001,"15min":16.373999999999999},"last":0.87},{"interface":"front","average":{"1min":0.78700000000000003,"5min":0.89500000000000002,"15min":1.464},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.215},"max":{"1min":1.3129999999999999,"5min":1.569,"15min":19.596},"last":0.77800000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:07:48 2023","interfaces":[{"interface":"back","average":{"1min":0.88600000000000001,"5min":0.98799999999999999,"15min":1.5469999999999999},"min":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.27100000000000002},"max":{"1min":1.46,"5min":1.5549999999999999,"15min":14.194000000000001},"last":0.91800000000000004},{"interface":"front","average":{"1min":1.087,"5min":1.113,"15min":1.8200000000000001},"min":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.32600000000000001},"max":{"1min":1.573,"5min":1.754,"15min":14.271000000000001},"last":1.01}]},{"osd":5,"last update":"Sat Sep 23 20:07:48 2023","interfaces":[{"interface":"back","average":{"1min":1.0389999999999999,"5min":1.1339999999999999,"15min":1.6359999999999999},"min":{"1min":0.64200000000000002,"5min":0.60799999999999998,"15min":0.32400000000000001},"max":{"1min":1.4019999999999999,"5min":2.206,"15min":14.007},"last":0.98099999999999998},{"interface":"front","average":{"1min":0.96499999999999997,"5min":1.1040000000000001,"15min":1.3959999999999999},"min":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.32100000000000001},"max":{"1min":1.3500000000000001,"5min":1.974,"15min":8.2720000000000002},"last":0.82499999999999996}]},{"osd":6,"last update":"Sat Sep 23 20:07:48 2023","interfaces":[{"interface":"back","average":{"1min":1.0129999999999999,"5min":1.145,"15min":1.554},"min":{"1min":0.71699999999999997,"5min":0.65900000000000003,"15min":0.35699999999999998},"max":{"1min":1.278,"5min":2.1019999999999999,"15min":9.8870000000000005},"last":1.0700000000000001},{"interface":"front","average":{"1min":1.151,"5min":1.222,"15min":1.786},"min":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.371},"max":{"1min":1.5089999999999999,"5min":2.0649999999999999,"15min":14.077999999999999},"last":1.171}]},{"osd":7,"last update":"Sat Sep 23 20:07:48 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":1.2330000000000001,"15min":1.6379999999999999},"min":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.40000000000000002},"max":{"1min":1.5369999999999999,"5min":2.1339999999999999,"15min":14.058},"last":1.046},{"interface":"front","average":{"1min":1.113,"5min":1.2070000000000001,"15min":1.764},"min":{"1min":0.64800000000000002,"5min":0.63600000000000001,"15min":0.38700000000000001},"max":{"1min":1.635,"5min":1.9430000000000001,"15min":14.109999999999999},"last":1.2569999999999999}]}]},{"osd":3,"up_from":116,"seq":498216206817,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":837744,"kb_used_data":3560,"kb_used_omap":2,"kb_used_meta":834173,"kb_avail":92907408,"statfs":{"total":95995035648,"available":95137185792,"internally_reserved":0,"allocated":3645440,"data_stored":2429301,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":854193205},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:07:21 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":0.872,"15min":2.1850000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.19600000000000001},"max":{"1min":1.024,"5min":15.679,"15min":117.248},"last":0.78700000000000003},{"interface":"front","average":{"1min":0.72799999999999998,"5min":0.81699999999999995,"15min":2.0939999999999999},"min":{"1min":0.37,"5min":0.37,"15min":0.20899999999999999},"max":{"1min":1.268,"5min":4.516,"15min":116.187},"last":1.016}]},{"osd":1,"last update":"Sat Sep 23 20:07:21 2023","interfaces":[{"interface":"back","average":{"1min":0.747,"5min":0.96499999999999997,"15min":1.4910000000000001},"min":{"1min":0.52200000000000002,"5min":0.34000000000000002,"15min":0.23400000000000001},"max":{"1min":0.98599999999999999,"5min":8.2140000000000004,"15min":12.542},"last":1.0940000000000001},{"interface":"front","average":{"1min":0.754,"5min":0.96999999999999997,"15min":1.365},"min":{"1min":0.42999999999999999,"5min":0.309,"15min":0.23100000000000001},"max":{"1min":0.98499999999999999,"5min":17.187000000000001,"15min":17.187000000000001},"last":0.92700000000000005}]},{"osd":2,"last update":"Sat Sep 23 20:07:38 2023","interfaces":[{"interface":"back","average":{"1min":0.74199999999999999,"5min":0.89800000000000002,"15min":1.446},"min":{"1min":0.42099999999999999,"5min":0.40400000000000003,"15min":0.246},"max":{"1min":1.3100000000000001,"5min":2.2650000000000001,"15min":14.529},"last":0.93799999999999994},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.93899999999999995,"15min":1.5389999999999999},"min":{"1min":0.438,"5min":0.32100000000000001,"15min":0.30499999999999999},"max":{"1min":0.98099999999999998,"5min":15.5,"15min":15.5},"last":0.96699999999999997}]},{"osd":4,"last update":"Sat Sep 23 20:07:21 2023","interfaces":[{"interface":"back","average":{"1min":1.0580000000000001,"5min":1.1930000000000001,"15min":1.5309999999999999},"min":{"1min":0.59799999999999998,"5min":0.56899999999999995,"15min":0.29699999999999999},"max":{"1min":1.51,"5min":15.461,"15min":15.461},"last":1.2070000000000001},{"interface":"front","average":{"1min":1.1140000000000001,"5min":1.204,"15min":1.5960000000000001},"min":{"1min":0.81499999999999995,"5min":0.54900000000000004,"15min":0.32900000000000001},"max":{"1min":1.484,"5min":5.7149999999999999,"15min":12.169},"last":1.3080000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:07:21 2023","interfaces":[{"interface":"back","average":{"1min":1.0860000000000001,"5min":1.1479999999999999,"15min":1.548},"min":{"1min":0.64600000000000002,"5min":0.60199999999999998,"15min":0.33900000000000002},"max":{"1min":1.446,"5min":2.3380000000000001,"15min":11.978},"last":1.3520000000000001},{"interface":"front","average":{"1min":0.96399999999999997,"5min":1.2230000000000001,"15min":1.4630000000000001},"min":{"1min":0.747,"5min":0.67700000000000005,"15min":0.34899999999999998},"max":{"1min":1.206,"5min":14.715,"15min":14.715},"last":1.1910000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:07:21 2023","interfaces":[{"interface":"back","average":{"1min":1.1299999999999999,"5min":1.196,"15min":1.6970000000000001},"min":{"1min":0.77100000000000002,"5min":0.77000000000000002,"15min":0.41199999999999998},"max":{"1min":1.5269999999999999,"5min":2.3849999999999998,"15min":12.135},"last":1.2410000000000001},{"interface":"front","average":{"1min":1.0229999999999999,"5min":1.2809999999999999,"15min":1.575},"min":{"1min":0.66000000000000003,"5min":0.61799999999999999,"15min":0.38500000000000001},"max":{"1min":1.278,"5min":15.537000000000001,"15min":15.537000000000001},"last":1.1120000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:07:21 2023","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":1.329,"15min":1.732},"min":{"1min":0.69499999999999995,"5min":0.61899999999999999,"15min":0.35799999999999998},"max":{"1min":1.337,"5min":14.754,"15min":14.754},"last":1.274},{"interface":"front","average":{"1min":1.111,"5min":1.3320000000000001,"15min":1.5329999999999999},"min":{"1min":0.83299999999999996,"5min":0.72399999999999998,"15min":0.41299999999999998},"max":{"1min":1.343,"5min":17.145,"15min":17.145},"last":1.0569999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019688,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1195276,"kb_used_data":6980,"kb_used_omap":0,"kb_used_meta":1188288,"kb_avail":92549876,"statfs":{"total":95995035648,"available":94771073024,"internally_reserved":0,"allocated":7147520,"data_stored":6553077,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1216806912},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:07:52 2023","interfaces":[{"interface":"back","average":{"1min":0.80500000000000005,"5min":0.84599999999999997,"15min":1.3939999999999999},"min":{"1min":0.57699999999999996,"5min":0.443,"15min":0.247},"max":{"1min":1.077,"5min":1.431,"15min":23.059999999999999},"last":0.79800000000000004},{"interface":"front","average":{"1min":0.88800000000000001,"5min":0.87,"15min":1.403},"min":{"1min":0.56999999999999995,"5min":0.39900000000000002,"15min":0.26300000000000001},"max":{"1min":1.1200000000000001,"5min":1.306,"15min":22.780000000000001},"last":0.76600000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:07:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0049999999999999,"5min":0.96899999999999997,"15min":1.6859999999999999},"min":{"1min":0.76300000000000001,"5min":0.44400000000000001,"15min":0.28100000000000003},"max":{"1min":1.2390000000000001,"5min":1.4450000000000001,"15min":30.754000000000001},"last":0.88200000000000001},{"interface":"front","average":{"1min":1.0589999999999999,"5min":1.006,"15min":1.7430000000000001},"min":{"1min":0.66600000000000004,"5min":0.51400000000000001,"15min":0.27600000000000002},"max":{"1min":1.3109999999999999,"5min":1.321,"15min":30.777999999999999},"last":1.0189999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:07:52 2023","interfaces":[{"interface":"back","average":{"1min":0.90300000000000002,"5min":0.90100000000000002,"15min":1.556},"min":{"1min":0.53400000000000003,"5min":0.48399999999999999,"15min":0.30399999999999999},"max":{"1min":1.2090000000000001,"5min":1.556,"15min":30.088999999999999},"last":0.872},{"interface":"front","average":{"1min":0.91400000000000003,"5min":0.93200000000000005,"15min":2.7029999999999998},"min":{"1min":0.61599999999999999,"5min":0.56899999999999995,"15min":0.308},"max":{"1min":1.236,"5min":1.3620000000000001,"15min":314.31599999999997},"last":0.98299999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:07:52 2023","interfaces":[{"interface":"back","average":{"1min":0.875,"5min":0.90600000000000003,"15min":1.4139999999999999},"min":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.308},"max":{"1min":1.1220000000000001,"5min":1.3600000000000001,"15min":9.4399999999999995},"last":1.179},{"interface":"front","average":{"1min":0.94199999999999995,"5min":0.97699999999999998,"15min":1.4590000000000001},"min":{"1min":0.71099999999999997,"5min":0.47599999999999998,"15min":0.33400000000000002},"max":{"1min":1.2090000000000001,"5min":1.6579999999999999,"15min":9.5280000000000005},"last":0.90900000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:07:52 2023","interfaces":[{"interface":"back","average":{"1min":0.99099999999999999,"5min":0.96999999999999997,"15min":1.3999999999999999},"min":{"1min":0.73099999999999998,"5min":0.60999999999999999,"15min":0.26800000000000002},"max":{"1min":1.2410000000000001,"5min":1.548,"15min":9.4740000000000002},"last":1.0940000000000001},{"interface":"front","average":{"1min":1.0389999999999999,"5min":1.048,"15min":1.423},"min":{"1min":0.67300000000000004,"5min":0.57599999999999996,"15min":0.29299999999999998},"max":{"1min":1.3069999999999999,"5min":1.7609999999999999,"15min":12.506},"last":0.94499999999999995}]},{"osd":6,"last update":"Sat Sep 23 20:07:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0600000000000001,"5min":1.052,"15min":1.4590000000000001},"min":{"1min":0.65200000000000002,"5min":0.60899999999999999,"15min":0.32400000000000001},"max":{"1min":1.4099999999999999,"5min":1.5900000000000001,"15min":13.066000000000001},"last":1.214},{"interface":"front","average":{"1min":1.1799999999999999,"5min":1.117,"15min":1.4850000000000001},"min":{"1min":0.94299999999999995,"5min":0.51500000000000001,"15min":0.35499999999999998},"max":{"1min":1.4330000000000001,"5min":2.8149999999999999,"15min":12.704000000000001},"last":0.97299999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:07:15 2023","interfaces":[{"interface":"back","average":{"1min":1.131,"5min":1.038,"15min":1.4330000000000001},"min":{"1min":0.66800000000000004,"5min":0.47899999999999998,"15min":0.308},"max":{"1min":1.6299999999999999,"5min":1.6299999999999999,"15min":12.475},"last":1.0780000000000001},{"interface":"front","average":{"1min":1.1719999999999999,"5min":1.135,"15min":1.5349999999999999},"min":{"1min":0.83699999999999997,"5min":0.58599999999999997,"15min":0.35199999999999998},"max":{"1min":1.4630000000000001,"5min":3.121,"15min":12.169},"last":1.1399999999999999}]}]},{"osd":5,"up_from":36,"seq":154618823455,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1198136,"kb_used_data":2864,"kb_used_omap":0,"kb_used_meta":1195264,"kb_avail":92547016,"statfs":{"total":95995035648,"available":94768144384,"internally_reserved":0,"allocated":2932736,"data_stored":2353906,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1223950336},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:07:46 2023","interfaces":[{"interface":"back","average":{"1min":0.94099999999999995,"5min":0.92700000000000005,"15min":2.6960000000000002},"min":{"1min":0.58299999999999996,"5min":0.53400000000000003,"15min":0.27800000000000002},"max":{"1min":1.2210000000000001,"5min":1.3600000000000001,"15min":201.196},"last":1.0580000000000001},{"interface":"front","average":{"1min":0.91600000000000004,"5min":0.91200000000000003,"15min":2.9089999999999998},"min":{"1min":0.50900000000000001,"5min":0.499,"15min":0.251},"max":{"1min":1.1679999999999999,"5min":1.411,"15min":192.941},"last":1.03}]},{"osd":1,"last update":"Sat Sep 23 20:07:11 2023","interfaces":[{"interface":"back","average":{"1min":0.93700000000000006,"5min":0.94199999999999995,"15min":4.1829999999999998},"min":{"1min":0.55200000000000005,"5min":0.46999999999999997,"15min":0.28599999999999998},"max":{"1min":1.282,"5min":1.5129999999999999,"15min":190.298},"last":0.94599999999999995},{"interface":"front","average":{"1min":0.90200000000000002,"5min":0.92700000000000005,"15min":4.2839999999999998},"min":{"1min":0.65200000000000002,"5min":0.629,"15min":0.28999999999999998},"max":{"1min":1.2470000000000001,"5min":1.5029999999999999,"15min":190.65799999999999},"last":1.0840000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:07:32 2023","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":0.93799999999999994,"15min":2.831},"min":{"1min":0.68100000000000005,"5min":0.49399999999999999,"15min":0.311},"max":{"1min":1.3260000000000001,"5min":1.5449999999999999,"15min":151.322},"last":1.1220000000000001},{"interface":"front","average":{"1min":1.0349999999999999,"5min":0.98299999999999998,"15min":2.976},"min":{"1min":0.70099999999999996,"5min":0.55400000000000005,"15min":0.34000000000000002},"max":{"1min":1.3200000000000001,"5min":1.387,"15min":166.941},"last":1.228}]},{"osd":3,"last update":"Sat Sep 23 20:07:11 2023","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":0.97499999999999998,"15min":1.369},"min":{"1min":0.69899999999999995,"5min":0.59999999999999998,"15min":0.35399999999999998},"max":{"1min":1.2829999999999999,"5min":1.6299999999999999,"15min":9.3829999999999991},"last":0.997},{"interface":"front","average":{"1min":1.042,"5min":1.0389999999999999,"15min":1.3759999999999999},"min":{"1min":0.78100000000000003,"5min":0.61899999999999999,"15min":0.34000000000000002},"max":{"1min":1.2390000000000001,"5min":1.375,"15min":13.840999999999999},"last":1.3859999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:07:46 2023","interfaces":[{"interface":"back","average":{"1min":0.874,"5min":0.88800000000000001,"15min":1.5269999999999999},"min":{"1min":0.64300000000000002,"5min":0.47799999999999998,"15min":0.26800000000000002},"max":{"1min":1.04,"5min":1.593,"15min":12.051},"last":0.89300000000000002},{"interface":"front","average":{"1min":1.0429999999999999,"5min":0.98799999999999999,"15min":1.431},"min":{"1min":0.754,"5min":0.56999999999999995,"15min":0.27900000000000003},"max":{"1min":1.2789999999999999,"5min":1.8,"15min":16.462},"last":0.84899999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:07:32 2023","interfaces":[{"interface":"back","average":{"1min":1.024,"5min":1.036,"15min":1.3580000000000001},"min":{"1min":0.63900000000000001,"5min":0.45700000000000002,"15min":0.32100000000000001},"max":{"1min":1.375,"5min":1.589,"15min":9.641},"last":1.1439999999999999},{"interface":"front","average":{"1min":1.1140000000000001,"5min":1.1419999999999999,"15min":1.399},"min":{"1min":0.78900000000000003,"5min":0.67100000000000004,"15min":0.38700000000000001},"max":{"1min":1.262,"5min":1.6799999999999999,"15min":11.444000000000001},"last":1.29}]},{"osd":7,"last update":"Sat Sep 23 20:07:46 2023","interfaces":[{"interface":"back","average":{"1min":1.119,"5min":1.085,"15min":1.4770000000000001},"min":{"1min":0.84299999999999997,"5min":0.70199999999999996,"15min":0.29999999999999999},"max":{"1min":1.3580000000000001,"5min":1.7829999999999999,"15min":9.407},"last":1.2030000000000001},{"interface":"front","average":{"1min":1.165,"5min":1.129,"15min":1.403},"min":{"1min":0.91100000000000003,"5min":0.622,"15min":0.34399999999999997},"max":{"1min":1.3200000000000001,"5min":1.643,"15min":17.053000000000001},"last":1.3200000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:08:03.055 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T20:08:03.056 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_bb0daeff97f020f686596fca6eba20c310e26fb7/qa/tasks/workunit.py", line 119, in task assert 'client' in role AssertionError 2023-09-23T20:08:03.125 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=bc53a3edf7294239ae12e428011f0c20 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_bb0daeff97f020f686596fca6eba20c310e26fb7/qa/tasks/workunit.py", line 119, in task assert 'client' in role AssertionError 2023-09-23T20:08:03.128 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2023-09-23T20:08:03.138 INFO:tasks.cephadm:Teardown begin 2023-09-23T20:08:03.139 DEBUG:teuthology.orchestra.run.smithi016:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-09-23T20:08:03.180 DEBUG:teuthology.orchestra.run.smithi092:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-09-23T20:08:03.206 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2023-09-23T20:08:03.207 DEBUG:teuthology.orchestra.run.smithi016:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-09-23T20:08:03.229 DEBUG:teuthology.orchestra.run.smithi092:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-09-23T20:08:03.257 INFO:tasks.cephadm:Stopping all daemons... 2023-09-23T20:08:03.257 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2023-09-23T20:08:03.257 DEBUG:teuthology.orchestra.run.smithi016:> sudo systemctl stop ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@mon.a 2023-09-23T20:08:03.363 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:08:03 smithi016 systemd[1]: Stopping Ceph mon.a for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7... 2023-09-23T20:08:03.539 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:08:03 smithi092 bash[58855]: audit 2023-09-23T20:08:02.060816+0000 mgr.y (mgr.25325) 1827 : audit 0 from='client.48035 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:08:03.733 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:08:03 smithi016 bash[77995]: audit 2023-09-23T20:08:02.060816+0000 mgr.y (mgr.25325) 1827 : audit 0 from='client.48035 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:08:03.733 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:08:03 smithi016 bash[75004]: audit 2023-09-23T20:08:02.060816+0000 mgr.y (mgr.25325) 1827 : audit 0 from='client.48035 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:08:03.734 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:08:03 smithi016 bash[75004]: debug 2023-09-23T20:08:03.452+0000 7f090d481700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-09-23T20:08:03.734 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:08:03 smithi016 bash[75004]: debug 2023-09-23T20:08:03.452+0000 7f090d481700 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2023-09-23T20:08:03.734 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:08:03 smithi016 bash[70992]: [23/Sep/2023:20:08:03] ENGINE Bus STOPPING 2023-09-23T20:08:03.734 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:08:03 smithi016 bash[70992]: [23/Sep/2023:20:08:03] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T20:08:03.735 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:08:03 smithi016 bash[70992]: [23/Sep/2023:20:08:03] ENGINE Bus STOPPED 2023-09-23T20:08:03.735 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:08:03 smithi016 bash[70992]: [23/Sep/2023:20:08:03] ENGINE Bus STARTING 2023-09-23T20:08:03.735 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:08:03 smithi016 bash[70992]: [23/Sep/2023:20:08:03] ENGINE Serving on http://:::9283 2023-09-23T20:08:03.735 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:08:03 smithi016 bash[70992]: [23/Sep/2023:20:08:03] ENGINE Bus STARTED 2023-09-23T20:08:04.571 INFO:journalctl@ceph.mon.a.smithi016.stdout:Sep 23 20:08:04 smithi016 bash[267393]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-mon-a 2023-09-23T20:08:04.577 DEBUG:teuthology.orchestra.run.smithi016:> sudo pkill -f 'journalctl -f -n 0 -u ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@mon.a.service' 2023-09-23T20:08:04.686 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T20:08:04.687 INFO:tasks.cephadm.mon.a:Stopped mon.a 2023-09-23T20:08:04.687 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2023-09-23T20:08:04.687 DEBUG:teuthology.orchestra.run.smithi016:> sudo systemctl stop ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@mon.c 2023-09-23T20:08:04.869 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:08:04 smithi016 systemd[1]: Stopping Ceph mon.c for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7... 2023-09-23T20:08:05.150 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:08:04 smithi016 bash[77995]: debug 2023-09-23T20:08:04.860+0000 7f3950d3c700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-09-23T20:08:05.151 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:08:04 smithi016 bash[77995]: debug 2023-09-23T20:08:04.860+0000 7f3950d3c700 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2023-09-23T20:08:05.965 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:08:05 smithi016 bash[70992]: ::ffff:172.21.15.92 - - [23/Sep/2023:20:08:05] "GET /metrics HTTP/1.1" 200 34009 "" "Prometheus/2.33.4" 2023-09-23T20:08:06.232 INFO:journalctl@ceph.mon.c.smithi016.stdout:Sep 23 20:08:05 smithi016 bash[267506]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-mon-c 2023-09-23T20:08:06.237 DEBUG:teuthology.orchestra.run.smithi016:> sudo pkill -f 'journalctl -f -n 0 -u ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@mon.c.service' 2023-09-23T20:08:06.305 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T20:08:06.305 INFO:tasks.cephadm.mon.b:Stopped mon.c 2023-09-23T20:08:06.305 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2023-09-23T20:08:06.306 DEBUG:teuthology.orchestra.run.smithi092:> sudo systemctl stop ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@mon.b 2023-09-23T20:08:06.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:08:06 smithi092 systemd[1]: Stopping Ceph mon.b for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7... 2023-09-23T20:08:06.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:08:06 smithi092 bash[58855]: debug 2023-09-23T20:08:06.512+0000 7ff66edef700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-09-23T20:08:06.789 INFO:journalctl@ceph.mon.b.smithi092.stdout:Sep 23 20:08:06 smithi092 bash[58855]: debug 2023-09-23T20:08:06.512+0000 7ff66edef700 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2023-09-23T20:08:07.837 DEBUG:teuthology.orchestra.run.smithi092:> sudo pkill -f 'journalctl -f -n 0 -u ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@mon.b.service' 2023-09-23T20:08:07.879 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T20:08:07.879 INFO:tasks.cephadm.mon.b:Stopped mon.b 2023-09-23T20:08:07.879 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2023-09-23T20:08:07.879 DEBUG:teuthology.orchestra.run.smithi016:> sudo systemctl stop ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@mgr.y 2023-09-23T20:08:08.282 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:08:07 smithi016 systemd[1]: Stopping Ceph mgr.y for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7... 2023-09-23T20:08:08.283 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:08:08 smithi016 bash[70992]: debug 2023-09-23T20:08:08.032+0000 7fb2e258e700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-09-23T20:08:08.283 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:08:08 smithi016 bash[70992]: debug 2023-09-23T20:08:08.032+0000 7fb2e258e700 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2023-09-23T20:08:09.055 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:08:08 smithi016 bash[267608]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-mgr-y 2023-09-23T20:08:09.055 INFO:journalctl@ceph.mgr.y.smithi016.stdout:Sep 23 20:08:09 smithi016 bash[267673]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-mgr-y 2023-09-23T20:08:09.059 DEBUG:teuthology.orchestra.run.smithi016:> sudo pkill -f 'journalctl -f -n 0 -u ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@mgr.y.service' 2023-09-23T20:08:09.140 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T20:08:09.140 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2023-09-23T20:08:09.140 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2023-09-23T20:08:09.141 DEBUG:teuthology.orchestra.run.smithi092:> sudo systemctl stop ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@mgr.x 2023-09-23T20:08:09.426 INFO:journalctl@ceph.mgr.x.smithi092.stdout:Sep 23 20:08:09 smithi092 systemd[1]: Stopping Ceph mgr.x for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7... 2023-09-23T20:08:10.307 DEBUG:teuthology.orchestra.run.smithi092:> sudo pkill -f 'journalctl -f -n 0 -u ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@mgr.x.service' 2023-09-23T20:08:10.337 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T20:08:10.338 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2023-09-23T20:08:10.338 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2023-09-23T20:08:10.338 DEBUG:teuthology.orchestra.run.smithi016:> sudo systemctl stop ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@osd.0 2023-09-23T20:08:10.797 INFO:journalctl@ceph.osd.0.smithi016.stdout:Sep 23 20:08:10 smithi016 systemd[1]: Stopping Ceph osd.0 for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7... 2023-09-23T20:08:10.798 INFO:journalctl@ceph.osd.0.smithi016.stdout:Sep 23 20:08:10 smithi016 bash[84502]: debug 2023-09-23T20:08:10.504+0000 7f728b876700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-09-23T20:08:10.798 INFO:journalctl@ceph.osd.0.smithi016.stdout:Sep 23 20:08:10 smithi016 bash[84502]: debug 2023-09-23T20:08:10.504+0000 7f728b876700 -1 osd.0 333 *** Got signal Terminated *** 2023-09-23T20:08:10.798 INFO:journalctl@ceph.osd.0.smithi016.stdout:Sep 23 20:08:10 smithi016 bash[84502]: debug 2023-09-23T20:08:10.504+0000 7f728b876700 -1 osd.0 333 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T20:08:17.248 INFO:journalctl@ceph.osd.0.smithi016.stdout:Sep 23 20:08:16 smithi016 bash[267712]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd-0 2023-09-23T20:08:17.248 INFO:journalctl@ceph.osd.0.smithi016.stdout:Sep 23 20:08:17 smithi016 bash[267779]: Error response from daemon: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd-0 2023-09-23T20:08:19.696 DEBUG:teuthology.orchestra.run.smithi016:> sudo pkill -f 'journalctl -f -n 0 -u ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@osd.0.service' 2023-09-23T20:08:19.728 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T20:08:19.728 INFO:tasks.cephadm.osd.0:Stopped osd.0 2023-09-23T20:08:19.729 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2023-09-23T20:08:19.729 DEBUG:teuthology.orchestra.run.smithi016:> sudo systemctl stop ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@osd.1 2023-09-23T20:08:20.035 INFO:journalctl@ceph.osd.1.smithi016.stdout:Sep 23 20:08:19 smithi016 systemd[1]: Stopping Ceph osd.1 for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7... 2023-09-23T20:08:20.036 INFO:journalctl@ceph.osd.1.smithi016.stdout:Sep 23 20:08:19 smithi016 bash[91347]: debug 2023-09-23T20:08:19.924+0000 7f8ee983a700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-09-23T20:08:20.036 INFO:journalctl@ceph.osd.1.smithi016.stdout:Sep 23 20:08:19 smithi016 bash[91347]: debug 2023-09-23T20:08:19.924+0000 7f8ee983a700 -1 osd.1 333 *** Got signal Terminated *** 2023-09-23T20:08:20.036 INFO:journalctl@ceph.osd.1.smithi016.stdout:Sep 23 20:08:19 smithi016 bash[91347]: debug 2023-09-23T20:08:19.924+0000 7f8ee983a700 -1 osd.1 333 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T20:08:26.535 INFO:journalctl@ceph.osd.1.smithi016.stdout:Sep 23 20:08:26 smithi016 bash[267920]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd-1 2023-09-23T20:08:26.535 INFO:journalctl@ceph.osd.1.smithi016.stdout:Sep 23 20:08:26 smithi016 bash[267987]: Error response from daemon: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd-1 2023-09-23T20:08:28.611 DEBUG:teuthology.orchestra.run.smithi016:> sudo pkill -f 'journalctl -f -n 0 -u ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@osd.1.service' 2023-09-23T20:08:28.645 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T20:08:28.646 INFO:tasks.cephadm.osd.1:Stopped osd.1 2023-09-23T20:08:28.646 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2023-09-23T20:08:28.646 DEBUG:teuthology.orchestra.run.smithi016:> sudo systemctl stop ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@osd.2 2023-09-23T20:08:28.872 INFO:journalctl@ceph.osd.2.smithi016.stdout:Sep 23 20:08:28 smithi016 systemd[1]: Stopping Ceph osd.2 for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7... 2023-09-23T20:08:29.285 INFO:journalctl@ceph.osd.2.smithi016.stdout:Sep 23 20:08:28 smithi016 bash[145232]: debug 2023-09-23T20:08:28.868+0000 7f316ca9e700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-09-23T20:08:29.286 INFO:journalctl@ceph.osd.2.smithi016.stdout:Sep 23 20:08:28 smithi016 bash[145232]: debug 2023-09-23T20:08:28.868+0000 7f316ca9e700 -1 osd.2 333 *** Got signal Terminated *** 2023-09-23T20:08:29.286 INFO:journalctl@ceph.osd.2.smithi016.stdout:Sep 23 20:08:28 smithi016 bash[145232]: debug 2023-09-23T20:08:28.868+0000 7f316ca9e700 -1 osd.2 333 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T20:08:35.300 INFO:journalctl@ceph.osd.2.smithi016.stdout:Sep 23 20:08:35 smithi016 bash[268125]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd-2 2023-09-23T20:08:35.785 INFO:journalctl@ceph.osd.2.smithi016.stdout:Sep 23 20:08:35 smithi016 bash[268194]: Error response from daemon: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd-2 2023-09-23T20:08:37.039 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:08:36 smithi092 bash[43686]: debug 2023-09-23T20:08:36.823+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:12.262854+0000 front 2023-09-23T20:08:12.263069+0000 (oldest deadline 2023-09-23T20:08:36.263834+0000) 2023-09-23T20:08:37.039 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:08:36 smithi092 bash[39474]: debug 2023-09-23T20:08:36.787+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.009391+0000 front 2023-09-23T20:08:13.009812+0000 (oldest deadline 2023-09-23T20:08:36.509203+0000) 2023-09-23T20:08:37.359 DEBUG:teuthology.orchestra.run.smithi016:> sudo pkill -f 'journalctl -f -n 0 -u ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@osd.2.service' 2023-09-23T20:08:37.394 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T20:08:37.394 INFO:tasks.cephadm.osd.2:Stopped osd.2 2023-09-23T20:08:37.395 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2023-09-23T20:08:37.395 DEBUG:teuthology.orchestra.run.smithi016:> sudo systemctl stop ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@osd.3 2023-09-23T20:08:37.616 INFO:journalctl@ceph.osd.3.smithi016.stdout:Sep 23 20:08:37 smithi016 systemd[1]: Stopping Ceph osd.3 for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7... 2023-09-23T20:08:38.035 INFO:journalctl@ceph.osd.3.smithi016.stdout:Sep 23 20:08:37 smithi016 bash[100899]: debug 2023-09-23T20:08:37.612+0000 7f3b95804700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-09-23T20:08:38.035 INFO:journalctl@ceph.osd.3.smithi016.stdout:Sep 23 20:08:37 smithi016 bash[100899]: debug 2023-09-23T20:08:37.612+0000 7f3b95804700 -1 osd.3 333 *** Got signal Terminated *** 2023-09-23T20:08:38.036 INFO:journalctl@ceph.osd.3.smithi016.stdout:Sep 23 20:08:37 smithi016 bash[100899]: debug 2023-09-23T20:08:37.612+0000 7f3b95804700 -1 osd.3 333 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T20:08:38.039 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:08:37 smithi092 bash[39474]: debug 2023-09-23T20:08:37.759+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.009391+0000 front 2023-09-23T20:08:13.009812+0000 (oldest deadline 2023-09-23T20:08:36.509203+0000) 2023-09-23T20:08:38.039 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:08:37 smithi092 bash[43686]: debug 2023-09-23T20:08:37.803+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:12.262854+0000 front 2023-09-23T20:08:12.263069+0000 (oldest deadline 2023-09-23T20:08:36.263834+0000) 2023-09-23T20:08:38.789 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:08:38 smithi092 bash[39474]: debug 2023-09-23T20:08:38.767+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.009391+0000 front 2023-09-23T20:08:13.009812+0000 (oldest deadline 2023-09-23T20:08:36.509203+0000) 2023-09-23T20:08:38.790 INFO:journalctl@ceph.osd.5.smithi092.stdout:Sep 23 20:08:38 smithi092 bash[35401]: debug 2023-09-23T20:08:38.519+0000 7f5c63d35700 -1 osd.5 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.668168+0000 front 2023-09-23T20:08:13.667873+0000 (oldest deadline 2023-09-23T20:08:37.767622+0000) 2023-09-23T20:08:39.125 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:08:38 smithi092 bash[43686]: debug 2023-09-23T20:08:38.799+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:12.262854+0000 front 2023-09-23T20:08:12.263069+0000 (oldest deadline 2023-09-23T20:08:36.263834+0000) 2023-09-23T20:08:39.285 INFO:journalctl@ceph.osd.3.smithi016.stdout:Sep 23 20:08:38 smithi016 bash[100899]: debug 2023-09-23T20:08:38.916+0000 7f3b91dfa700 -1 osd.3 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.435918+0000 front 2023-09-23T20:08:13.436213+0000 (oldest deadline 2023-09-23T20:08:38.735718+0000) 2023-09-23T20:08:39.496 INFO:journalctl@ceph.osd.4.smithi092.stdout:Sep 23 20:08:39 smithi092 bash[31327]: debug 2023-09-23T20:08:39.123+0000 7f35396ac700 -1 osd.4 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:14.699141+0000 front 2023-09-23T20:08:14.698926+0000 (oldest deadline 2023-09-23T20:08:38.798560+0000) 2023-09-23T20:08:39.788 INFO:journalctl@ceph.osd.5.smithi092.stdout:Sep 23 20:08:39 smithi092 bash[35401]: debug 2023-09-23T20:08:39.495+0000 7f5c63d35700 -1 osd.5 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.668168+0000 front 2023-09-23T20:08:13.667873+0000 (oldest deadline 2023-09-23T20:08:37.767622+0000) 2023-09-23T20:08:39.789 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:08:39 smithi092 bash[39474]: debug 2023-09-23T20:08:39.731+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.009391+0000 front 2023-09-23T20:08:13.009812+0000 (oldest deadline 2023-09-23T20:08:36.509203+0000) 2023-09-23T20:08:40.130 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:08:39 smithi092 bash[43686]: debug 2023-09-23T20:08:39.795+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:12.262854+0000 front 2023-09-23T20:08:12.263069+0000 (oldest deadline 2023-09-23T20:08:36.263834+0000) 2023-09-23T20:08:40.285 INFO:journalctl@ceph.osd.3.smithi016.stdout:Sep 23 20:08:39 smithi016 bash[100899]: debug 2023-09-23T20:08:39.948+0000 7f3b91dfa700 -1 osd.3 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.435918+0000 front 2023-09-23T20:08:13.436213+0000 (oldest deadline 2023-09-23T20:08:38.735718+0000) 2023-09-23T20:08:40.505 INFO:journalctl@ceph.osd.4.smithi092.stdout:Sep 23 20:08:40 smithi092 bash[31327]: debug 2023-09-23T20:08:40.127+0000 7f35396ac700 -1 osd.4 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:14.699141+0000 front 2023-09-23T20:08:14.698926+0000 (oldest deadline 2023-09-23T20:08:38.798560+0000) 2023-09-23T20:08:40.771 INFO:journalctl@ceph.osd.5.smithi092.stdout:Sep 23 20:08:40 smithi092 bash[35401]: debug 2023-09-23T20:08:40.503+0000 7f5c63d35700 -1 osd.5 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.668168+0000 front 2023-09-23T20:08:13.667873+0000 (oldest deadline 2023-09-23T20:08:37.767622+0000) 2023-09-23T20:08:41.038 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:08:40 smithi092 bash[39474]: debug 2023-09-23T20:08:40.771+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.009391+0000 front 2023-09-23T20:08:13.009812+0000 (oldest deadline 2023-09-23T20:08:36.509203+0000) 2023-09-23T20:08:41.039 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:08:40 smithi092 bash[43686]: debug 2023-09-23T20:08:40.835+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:12.262854+0000 front 2023-09-23T20:08:12.263069+0000 (oldest deadline 2023-09-23T20:08:36.263834+0000) 2023-09-23T20:08:41.285 INFO:journalctl@ceph.osd.3.smithi016.stdout:Sep 23 20:08:40 smithi016 bash[100899]: debug 2023-09-23T20:08:40.984+0000 7f3b91dfa700 -1 osd.3 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.435918+0000 front 2023-09-23T20:08:13.436213+0000 (oldest deadline 2023-09-23T20:08:38.735718+0000) 2023-09-23T20:08:41.467 INFO:journalctl@ceph.osd.4.smithi092.stdout:Sep 23 20:08:41 smithi092 bash[31327]: debug 2023-09-23T20:08:41.103+0000 7f35396ac700 -1 osd.4 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:14.699141+0000 front 2023-09-23T20:08:14.698926+0000 (oldest deadline 2023-09-23T20:08:38.798560+0000) 2023-09-23T20:08:41.764 INFO:journalctl@ceph.osd.5.smithi092.stdout:Sep 23 20:08:41 smithi092 bash[35401]: debug 2023-09-23T20:08:41.467+0000 7f5c63d35700 -1 osd.5 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.668168+0000 front 2023-09-23T20:08:13.667873+0000 (oldest deadline 2023-09-23T20:08:37.767622+0000) 2023-09-23T20:08:42.038 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:08:41 smithi092 bash[39474]: debug 2023-09-23T20:08:41.763+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.009391+0000 front 2023-09-23T20:08:13.009812+0000 (oldest deadline 2023-09-23T20:08:36.509203+0000) 2023-09-23T20:08:42.039 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:08:41 smithi092 bash[43686]: debug 2023-09-23T20:08:41.791+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:12.262854+0000 front 2023-09-23T20:08:12.263069+0000 (oldest deadline 2023-09-23T20:08:36.263834+0000) 2023-09-23T20:08:42.285 INFO:journalctl@ceph.osd.3.smithi016.stdout:Sep 23 20:08:41 smithi016 bash[100899]: debug 2023-09-23T20:08:41.972+0000 7f3b91dfa700 -1 osd.3 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.435918+0000 front 2023-09-23T20:08:13.436213+0000 (oldest deadline 2023-09-23T20:08:38.735718+0000) 2023-09-23T20:08:42.483 INFO:journalctl@ceph.osd.4.smithi092.stdout:Sep 23 20:08:42 smithi092 bash[31327]: debug 2023-09-23T20:08:42.079+0000 7f35396ac700 -1 osd.4 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:14.699141+0000 front 2023-09-23T20:08:14.698926+0000 (oldest deadline 2023-09-23T20:08:38.798560+0000) 2023-09-23T20:08:42.754 INFO:journalctl@ceph.osd.5.smithi092.stdout:Sep 23 20:08:42 smithi092 bash[35401]: debug 2023-09-23T20:08:42.483+0000 7f5c63d35700 -1 osd.5 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.668168+0000 front 2023-09-23T20:08:13.667873+0000 (oldest deadline 2023-09-23T20:08:37.767622+0000) 2023-09-23T20:08:43.038 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:08:42 smithi092 bash[39474]: debug 2023-09-23T20:08:42.779+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.009391+0000 front 2023-09-23T20:08:13.009812+0000 (oldest deadline 2023-09-23T20:08:36.509203+0000) 2023-09-23T20:08:43.039 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:08:42 smithi092 bash[43686]: debug 2023-09-23T20:08:42.751+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:12.262854+0000 front 2023-09-23T20:08:12.263069+0000 (oldest deadline 2023-09-23T20:08:36.263834+0000) 2023-09-23T20:08:43.497 INFO:journalctl@ceph.osd.4.smithi092.stdout:Sep 23 20:08:43 smithi092 bash[31327]: debug 2023-09-23T20:08:43.039+0000 7f35396ac700 -1 osd.4 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:14.699141+0000 front 2023-09-23T20:08:14.698926+0000 (oldest deadline 2023-09-23T20:08:38.798560+0000) 2023-09-23T20:08:43.760 INFO:journalctl@ceph.osd.5.smithi092.stdout:Sep 23 20:08:43 smithi092 bash[35401]: debug 2023-09-23T20:08:43.495+0000 7f5c63d35700 -1 osd.5 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.668168+0000 front 2023-09-23T20:08:13.667873+0000 (oldest deadline 2023-09-23T20:08:37.767622+0000) 2023-09-23T20:08:44.038 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:08:43 smithi092 bash[39474]: debug 2023-09-23T20:08:43.787+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.009391+0000 front 2023-09-23T20:08:13.009812+0000 (oldest deadline 2023-09-23T20:08:36.509203+0000) 2023-09-23T20:08:44.039 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:08:43 smithi092 bash[43686]: debug 2023-09-23T20:08:43.759+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:12.262854+0000 front 2023-09-23T20:08:12.263069+0000 (oldest deadline 2023-09-23T20:08:36.263834+0000) 2023-09-23T20:08:44.096 INFO:journalctl@ceph.osd.3.smithi016.stdout:Sep 23 20:08:43 smithi016 bash[268333]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd-3 2023-09-23T20:08:44.096 INFO:journalctl@ceph.osd.3.smithi016.stdout:Sep 23 20:08:44 smithi016 bash[268402]: Error response from daemon: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd-3 2023-09-23T20:08:44.522 INFO:journalctl@ceph.osd.4.smithi092.stdout:Sep 23 20:08:44 smithi092 bash[31327]: debug 2023-09-23T20:08:44.087+0000 7f35396ac700 -1 osd.4 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:14.699141+0000 front 2023-09-23T20:08:14.698926+0000 (oldest deadline 2023-09-23T20:08:38.798560+0000) 2023-09-23T20:08:44.782 INFO:journalctl@ceph.osd.5.smithi092.stdout:Sep 23 20:08:44 smithi092 bash[35401]: debug 2023-09-23T20:08:44.519+0000 7f5c63d35700 -1 osd.5 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.668168+0000 front 2023-09-23T20:08:13.667873+0000 (oldest deadline 2023-09-23T20:08:37.767622+0000) 2023-09-23T20:08:44.782 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:08:44 smithi092 bash[43686]: debug 2023-09-23T20:08:44.755+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:12.262854+0000 front 2023-09-23T20:08:12.263069+0000 (oldest deadline 2023-09-23T20:08:36.263834+0000) 2023-09-23T20:08:45.038 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:08:44 smithi092 bash[39474]: debug 2023-09-23T20:08:44.779+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.009391+0000 front 2023-09-23T20:08:13.009812+0000 (oldest deadline 2023-09-23T20:08:36.509203+0000) 2023-09-23T20:08:45.538 INFO:journalctl@ceph.osd.4.smithi092.stdout:Sep 23 20:08:45 smithi092 bash[31327]: debug 2023-09-23T20:08:45.055+0000 7f35396ac700 -1 osd.4 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:14.699141+0000 front 2023-09-23T20:08:14.698926+0000 (oldest deadline 2023-09-23T20:08:38.798560+0000) 2023-09-23T20:08:46.038 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:08:45 smithi092 bash[43686]: debug 2023-09-23T20:08:45.763+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:12.262854+0000 front 2023-09-23T20:08:12.263069+0000 (oldest deadline 2023-09-23T20:08:36.263834+0000) 2023-09-23T20:08:46.039 INFO:journalctl@ceph.osd.5.smithi092.stdout:Sep 23 20:08:45 smithi092 bash[35401]: debug 2023-09-23T20:08:45.547+0000 7f5c63d35700 -1 osd.5 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.668168+0000 front 2023-09-23T20:08:13.667873+0000 (oldest deadline 2023-09-23T20:08:37.767622+0000) 2023-09-23T20:08:46.040 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:08:45 smithi092 bash[39474]: debug 2023-09-23T20:08:45.735+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.009391+0000 front 2023-09-23T20:08:13.009812+0000 (oldest deadline 2023-09-23T20:08:36.509203+0000) 2023-09-23T20:08:46.138 DEBUG:teuthology.orchestra.run.smithi016:> sudo pkill -f 'journalctl -f -n 0 -u ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@osd.3.service' 2023-09-23T20:08:46.175 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T20:08:46.175 INFO:tasks.cephadm.osd.3:Stopped osd.3 2023-09-23T20:08:46.175 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2023-09-23T20:08:46.175 DEBUG:teuthology.orchestra.run.smithi092:> sudo systemctl stop ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@osd.4 2023-09-23T20:08:46.342 INFO:journalctl@ceph.osd.4.smithi092.stdout:Sep 23 20:08:46 smithi092 bash[31327]: debug 2023-09-23T20:08:46.083+0000 7f35396ac700 -1 osd.4 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:14.699141+0000 front 2023-09-23T20:08:14.698926+0000 (oldest deadline 2023-09-23T20:08:38.798560+0000) 2023-09-23T20:08:46.343 INFO:journalctl@ceph.osd.4.smithi092.stdout:Sep 23 20:08:46 smithi092 bash[31327]: debug 2023-09-23T20:08:46.083+0000 7f35396ac700 -1 osd.4 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:21.600586+0000 front 2023-09-23T20:08:21.600913+0000 (oldest deadline 2023-09-23T20:08:45.700331+0000) 2023-09-23T20:08:46.343 INFO:journalctl@ceph.osd.4.smithi092.stdout:Sep 23 20:08:46 smithi092 systemd[1]: Stopping Ceph osd.4 for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7... 2023-09-23T20:08:46.752 INFO:journalctl@ceph.osd.5.smithi092.stdout:Sep 23 20:08:46 smithi092 bash[35401]: debug 2023-09-23T20:08:46.511+0000 7f5c63d35700 -1 osd.5 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.668168+0000 front 2023-09-23T20:08:13.667873+0000 (oldest deadline 2023-09-23T20:08:37.767622+0000) 2023-09-23T20:08:46.753 INFO:journalctl@ceph.osd.4.smithi092.stdout:Sep 23 20:08:46 smithi092 bash[31327]: debug 2023-09-23T20:08:46.339+0000 7f353bf2e700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-09-23T20:08:46.753 INFO:journalctl@ceph.osd.4.smithi092.stdout:Sep 23 20:08:46 smithi092 bash[31327]: debug 2023-09-23T20:08:46.339+0000 7f353bf2e700 -1 osd.4 333 *** Got signal Terminated *** 2023-09-23T20:08:46.753 INFO:journalctl@ceph.osd.4.smithi092.stdout:Sep 23 20:08:46 smithi092 bash[31327]: debug 2023-09-23T20:08:46.339+0000 7f353bf2e700 -1 osd.4 333 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T20:08:47.039 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:08:46 smithi092 bash[39474]: debug 2023-09-23T20:08:46.751+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.009391+0000 front 2023-09-23T20:08:13.009812+0000 (oldest deadline 2023-09-23T20:08:36.509203+0000) 2023-09-23T20:08:47.039 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:08:46 smithi092 bash[43686]: debug 2023-09-23T20:08:46.755+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:12.262854+0000 front 2023-09-23T20:08:12.263069+0000 (oldest deadline 2023-09-23T20:08:36.263834+0000) 2023-09-23T20:08:47.512 INFO:journalctl@ceph.osd.4.smithi092.stdout:Sep 23 20:08:47 smithi092 bash[31327]: debug 2023-09-23T20:08:47.087+0000 7f35396ac700 -1 osd.4 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:14.699141+0000 front 2023-09-23T20:08:14.698926+0000 (oldest deadline 2023-09-23T20:08:38.798560+0000) 2023-09-23T20:08:47.513 INFO:journalctl@ceph.osd.4.smithi092.stdout:Sep 23 20:08:47 smithi092 bash[31327]: debug 2023-09-23T20:08:47.087+0000 7f35396ac700 -1 osd.4 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:21.600586+0000 front 2023-09-23T20:08:21.600913+0000 (oldest deadline 2023-09-23T20:08:45.700331+0000) 2023-09-23T20:08:47.788 INFO:journalctl@ceph.osd.5.smithi092.stdout:Sep 23 20:08:47 smithi092 bash[35401]: debug 2023-09-23T20:08:47.511+0000 7f5c63d35700 -1 osd.5 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.668168+0000 front 2023-09-23T20:08:13.667873+0000 (oldest deadline 2023-09-23T20:08:37.767622+0000) 2023-09-23T20:08:47.789 INFO:journalctl@ceph.osd.5.smithi092.stdout:Sep 23 20:08:47 smithi092 bash[35401]: debug 2023-09-23T20:08:47.511+0000 7f5c63d35700 -1 osd.5 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:21.769834+0000 front 2023-09-23T20:08:21.769861+0000 (oldest deadline 2023-09-23T20:08:47.069534+0000) 2023-09-23T20:08:47.789 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:08:47 smithi092 bash[39474]: debug 2023-09-23T20:08:47.759+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.009391+0000 front 2023-09-23T20:08:13.009812+0000 (oldest deadline 2023-09-23T20:08:36.509203+0000) 2023-09-23T20:08:48.047 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:08:47 smithi092 bash[43686]: debug 2023-09-23T20:08:47.787+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:12.262854+0000 front 2023-09-23T20:08:12.263069+0000 (oldest deadline 2023-09-23T20:08:36.263834+0000) 2023-09-23T20:08:48.048 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:08:47 smithi092 bash[43686]: debug 2023-09-23T20:08:47.787+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:22.666136+0000 front 2023-09-23T20:08:22.666165+0000 (oldest deadline 2023-09-23T20:08:47.365645+0000) 2023-09-23T20:08:48.467 INFO:journalctl@ceph.osd.4.smithi092.stdout:Sep 23 20:08:48 smithi092 bash[31327]: debug 2023-09-23T20:08:48.043+0000 7f35396ac700 -1 osd.4 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:14.699141+0000 front 2023-09-23T20:08:14.698926+0000 (oldest deadline 2023-09-23T20:08:38.798560+0000) 2023-09-23T20:08:48.467 INFO:journalctl@ceph.osd.4.smithi092.stdout:Sep 23 20:08:48 smithi092 bash[31327]: debug 2023-09-23T20:08:48.043+0000 7f35396ac700 -1 osd.4 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:21.600586+0000 front 2023-09-23T20:08:21.600913+0000 (oldest deadline 2023-09-23T20:08:45.700331+0000) 2023-09-23T20:08:48.767 INFO:journalctl@ceph.osd.5.smithi092.stdout:Sep 23 20:08:48 smithi092 bash[35401]: debug 2023-09-23T20:08:48.463+0000 7f5c63d35700 -1 osd.5 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.668168+0000 front 2023-09-23T20:08:13.667873+0000 (oldest deadline 2023-09-23T20:08:37.767622+0000) 2023-09-23T20:08:48.767 INFO:journalctl@ceph.osd.5.smithi092.stdout:Sep 23 20:08:48 smithi092 bash[35401]: debug 2023-09-23T20:08:48.463+0000 7f5c63d35700 -1 osd.5 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:21.769834+0000 front 2023-09-23T20:08:21.769861+0000 (oldest deadline 2023-09-23T20:08:47.069534+0000) 2023-09-23T20:08:49.038 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:08:48 smithi092 bash[39474]: debug 2023-09-23T20:08:48.763+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.009391+0000 front 2023-09-23T20:08:13.009812+0000 (oldest deadline 2023-09-23T20:08:36.509203+0000) 2023-09-23T20:08:49.039 INFO:journalctl@ceph.osd.4.smithi092.stdout:Sep 23 20:08:49 smithi092 bash[31327]: debug 2023-09-23T20:08:49.003+0000 7f35396ac700 -1 osd.4 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:14.699141+0000 front 2023-09-23T20:08:14.698926+0000 (oldest deadline 2023-09-23T20:08:38.798560+0000) 2023-09-23T20:08:49.039 INFO:journalctl@ceph.osd.4.smithi092.stdout:Sep 23 20:08:49 smithi092 bash[31327]: debug 2023-09-23T20:08:49.003+0000 7f35396ac700 -1 osd.4 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:21.600586+0000 front 2023-09-23T20:08:21.600913+0000 (oldest deadline 2023-09-23T20:08:45.700331+0000) 2023-09-23T20:08:49.039 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:08:48 smithi092 bash[43686]: debug 2023-09-23T20:08:48.815+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:12.262854+0000 front 2023-09-23T20:08:12.263069+0000 (oldest deadline 2023-09-23T20:08:36.263834+0000) 2023-09-23T20:08:49.040 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:08:48 smithi092 bash[43686]: debug 2023-09-23T20:08:48.815+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:22.666136+0000 front 2023-09-23T20:08:22.666165+0000 (oldest deadline 2023-09-23T20:08:47.365645+0000) 2023-09-23T20:08:49.788 INFO:journalctl@ceph.osd.5.smithi092.stdout:Sep 23 20:08:49 smithi092 bash[35401]: debug 2023-09-23T20:08:49.471+0000 7f5c63d35700 -1 osd.5 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.668168+0000 front 2023-09-23T20:08:13.667873+0000 (oldest deadline 2023-09-23T20:08:37.767622+0000) 2023-09-23T20:08:49.789 INFO:journalctl@ceph.osd.5.smithi092.stdout:Sep 23 20:08:49 smithi092 bash[35401]: debug 2023-09-23T20:08:49.471+0000 7f5c63d35700 -1 osd.5 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:21.769834+0000 front 2023-09-23T20:08:21.769861+0000 (oldest deadline 2023-09-23T20:08:47.069534+0000) 2023-09-23T20:08:50.289 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:08:49 smithi092 bash[39474]: debug 2023-09-23T20:08:49.795+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.009391+0000 front 2023-09-23T20:08:13.009812+0000 (oldest deadline 2023-09-23T20:08:36.509203+0000) 2023-09-23T20:08:50.289 INFO:journalctl@ceph.osd.4.smithi092.stdout:Sep 23 20:08:50 smithi092 bash[31327]: debug 2023-09-23T20:08:50.015+0000 7f35396ac700 -1 osd.4 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:14.699141+0000 front 2023-09-23T20:08:14.698926+0000 (oldest deadline 2023-09-23T20:08:38.798560+0000) 2023-09-23T20:08:50.289 INFO:journalctl@ceph.osd.4.smithi092.stdout:Sep 23 20:08:50 smithi092 bash[31327]: debug 2023-09-23T20:08:50.015+0000 7f35396ac700 -1 osd.4 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:21.600586+0000 front 2023-09-23T20:08:21.600913+0000 (oldest deadline 2023-09-23T20:08:45.700331+0000) 2023-09-23T20:08:50.290 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:08:49 smithi092 bash[43686]: debug 2023-09-23T20:08:49.843+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:12.262854+0000 front 2023-09-23T20:08:12.263069+0000 (oldest deadline 2023-09-23T20:08:36.263834+0000) 2023-09-23T20:08:50.290 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:08:49 smithi092 bash[43686]: debug 2023-09-23T20:08:49.843+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:22.666136+0000 front 2023-09-23T20:08:22.666165+0000 (oldest deadline 2023-09-23T20:08:47.365645+0000) 2023-09-23T20:08:50.789 INFO:journalctl@ceph.osd.5.smithi092.stdout:Sep 23 20:08:50 smithi092 bash[35401]: debug 2023-09-23T20:08:50.491+0000 7f5c63d35700 -1 osd.5 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.668168+0000 front 2023-09-23T20:08:13.667873+0000 (oldest deadline 2023-09-23T20:08:37.767622+0000) 2023-09-23T20:08:50.789 INFO:journalctl@ceph.osd.5.smithi092.stdout:Sep 23 20:08:50 smithi092 bash[35401]: debug 2023-09-23T20:08:50.491+0000 7f5c63d35700 -1 osd.5 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:21.769834+0000 front 2023-09-23T20:08:21.769861+0000 (oldest deadline 2023-09-23T20:08:47.069534+0000) 2023-09-23T20:08:51.288 INFO:journalctl@ceph.osd.4.smithi092.stdout:Sep 23 20:08:50 smithi092 bash[31327]: debug 2023-09-23T20:08:50.967+0000 7f35396ac700 -1 osd.4 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:14.699141+0000 front 2023-09-23T20:08:14.698926+0000 (oldest deadline 2023-09-23T20:08:38.798560+0000) 2023-09-23T20:08:51.289 INFO:journalctl@ceph.osd.4.smithi092.stdout:Sep 23 20:08:50 smithi092 bash[31327]: debug 2023-09-23T20:08:50.967+0000 7f35396ac700 -1 osd.4 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:21.600586+0000 front 2023-09-23T20:08:21.600913+0000 (oldest deadline 2023-09-23T20:08:45.700331+0000) 2023-09-23T20:08:51.290 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:08:50 smithi092 bash[39474]: debug 2023-09-23T20:08:50.819+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.009391+0000 front 2023-09-23T20:08:13.009812+0000 (oldest deadline 2023-09-23T20:08:36.509203+0000) 2023-09-23T20:08:51.290 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:08:50 smithi092 bash[39474]: debug 2023-09-23T20:08:50.819+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:24.111476+0000 front 2023-09-23T20:08:24.111596+0000 (oldest deadline 2023-09-23T20:08:50.511939+0000) 2023-09-23T20:08:51.290 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:08:50 smithi092 bash[43686]: debug 2023-09-23T20:08:50.843+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:12.262854+0000 front 2023-09-23T20:08:12.263069+0000 (oldest deadline 2023-09-23T20:08:36.263834+0000) 2023-09-23T20:08:51.290 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:08:50 smithi092 bash[43686]: debug 2023-09-23T20:08:50.843+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:22.666136+0000 front 2023-09-23T20:08:22.666165+0000 (oldest deadline 2023-09-23T20:08:47.365645+0000) 2023-09-23T20:08:51.788 INFO:journalctl@ceph.osd.5.smithi092.stdout:Sep 23 20:08:51 smithi092 bash[35401]: debug 2023-09-23T20:08:51.511+0000 7f5c63d35700 -1 osd.5 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.668168+0000 front 2023-09-23T20:08:13.667873+0000 (oldest deadline 2023-09-23T20:08:37.767622+0000) 2023-09-23T20:08:51.789 INFO:journalctl@ceph.osd.5.smithi092.stdout:Sep 23 20:08:51 smithi092 bash[35401]: debug 2023-09-23T20:08:51.511+0000 7f5c63d35700 -1 osd.5 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:21.769834+0000 front 2023-09-23T20:08:21.769861+0000 (oldest deadline 2023-09-23T20:08:47.069534+0000) 2023-09-23T20:08:52.289 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:08:51 smithi092 bash[39474]: debug 2023-09-23T20:08:51.791+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.009391+0000 front 2023-09-23T20:08:13.009812+0000 (oldest deadline 2023-09-23T20:08:36.509203+0000) 2023-09-23T20:08:52.289 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:08:51 smithi092 bash[39474]: debug 2023-09-23T20:08:51.791+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:24.111476+0000 front 2023-09-23T20:08:24.111596+0000 (oldest deadline 2023-09-23T20:08:50.511939+0000) 2023-09-23T20:08:52.289 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:08:51 smithi092 bash[43686]: debug 2023-09-23T20:08:51.839+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:12.262854+0000 front 2023-09-23T20:08:12.263069+0000 (oldest deadline 2023-09-23T20:08:36.263834+0000) 2023-09-23T20:08:52.289 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:08:51 smithi092 bash[43686]: debug 2023-09-23T20:08:51.839+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:22.666136+0000 front 2023-09-23T20:08:22.666165+0000 (oldest deadline 2023-09-23T20:08:47.365645+0000) 2023-09-23T20:08:52.687 INFO:journalctl@ceph.osd.4.smithi092.stdout:Sep 23 20:08:52 smithi092 bash[155232]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd-4 2023-09-23T20:08:52.688 INFO:journalctl@ceph.osd.4.smithi092.stdout:Sep 23 20:08:52 smithi092 bash[155299]: Error response from daemon: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd-4 2023-09-23T20:08:52.688 INFO:journalctl@ceph.osd.5.smithi092.stdout:Sep 23 20:08:52 smithi092 bash[35401]: debug 2023-09-23T20:08:52.491+0000 7f5c63d35700 -1 osd.5 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.668168+0000 front 2023-09-23T20:08:13.667873+0000 (oldest deadline 2023-09-23T20:08:37.767622+0000) 2023-09-23T20:08:52.688 INFO:journalctl@ceph.osd.5.smithi092.stdout:Sep 23 20:08:52 smithi092 bash[35401]: debug 2023-09-23T20:08:52.491+0000 7f5c63d35700 -1 osd.5 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:21.769834+0000 front 2023-09-23T20:08:21.769861+0000 (oldest deadline 2023-09-23T20:08:47.069534+0000) 2023-09-23T20:08:53.038 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:08:52 smithi092 bash[39474]: debug 2023-09-23T20:08:52.831+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.009391+0000 front 2023-09-23T20:08:13.009812+0000 (oldest deadline 2023-09-23T20:08:36.509203+0000) 2023-09-23T20:08:53.039 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:08:52 smithi092 bash[39474]: debug 2023-09-23T20:08:52.831+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:24.111476+0000 front 2023-09-23T20:08:24.111596+0000 (oldest deadline 2023-09-23T20:08:50.511939+0000) 2023-09-23T20:08:53.039 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:08:52 smithi092 bash[43686]: debug 2023-09-23T20:08:52.811+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:12.262854+0000 front 2023-09-23T20:08:12.263069+0000 (oldest deadline 2023-09-23T20:08:36.263834+0000) 2023-09-23T20:08:53.039 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:08:52 smithi092 bash[43686]: debug 2023-09-23T20:08:52.811+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:22.666136+0000 front 2023-09-23T20:08:22.666165+0000 (oldest deadline 2023-09-23T20:08:47.365645+0000) 2023-09-23T20:08:53.835 INFO:journalctl@ceph.osd.5.smithi092.stdout:Sep 23 20:08:53 smithi092 bash[35401]: debug 2023-09-23T20:08:53.539+0000 7f5c63d35700 -1 osd.5 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.668168+0000 front 2023-09-23T20:08:13.667873+0000 (oldest deadline 2023-09-23T20:08:37.767622+0000) 2023-09-23T20:08:53.835 INFO:journalctl@ceph.osd.5.smithi092.stdout:Sep 23 20:08:53 smithi092 bash[35401]: debug 2023-09-23T20:08:53.539+0000 7f5c63d35700 -1 osd.5 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:21.769834+0000 front 2023-09-23T20:08:21.769861+0000 (oldest deadline 2023-09-23T20:08:47.069534+0000) 2023-09-23T20:08:54.289 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:08:53 smithi092 bash[39474]: debug 2023-09-23T20:08:53.839+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.009391+0000 front 2023-09-23T20:08:13.009812+0000 (oldest deadline 2023-09-23T20:08:36.509203+0000) 2023-09-23T20:08:54.289 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:08:53 smithi092 bash[39474]: debug 2023-09-23T20:08:53.839+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:24.111476+0000 front 2023-09-23T20:08:24.111596+0000 (oldest deadline 2023-09-23T20:08:50.511939+0000) 2023-09-23T20:08:54.289 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:08:53 smithi092 bash[43686]: debug 2023-09-23T20:08:53.831+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:12.262854+0000 front 2023-09-23T20:08:12.263069+0000 (oldest deadline 2023-09-23T20:08:36.263834+0000) 2023-09-23T20:08:54.290 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:08:53 smithi092 bash[43686]: debug 2023-09-23T20:08:53.831+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:22.666136+0000 front 2023-09-23T20:08:22.666165+0000 (oldest deadline 2023-09-23T20:08:47.365645+0000) 2023-09-23T20:08:54.789 INFO:journalctl@ceph.osd.5.smithi092.stdout:Sep 23 20:08:54 smithi092 bash[35401]: debug 2023-09-23T20:08:54.531+0000 7f5c63d35700 -1 osd.5 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.668168+0000 front 2023-09-23T20:08:13.667873+0000 (oldest deadline 2023-09-23T20:08:37.767622+0000) 2023-09-23T20:08:54.789 INFO:journalctl@ceph.osd.5.smithi092.stdout:Sep 23 20:08:54 smithi092 bash[35401]: debug 2023-09-23T20:08:54.531+0000 7f5c63d35700 -1 osd.5 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:21.769834+0000 front 2023-09-23T20:08:21.769861+0000 (oldest deadline 2023-09-23T20:08:47.069534+0000) 2023-09-23T20:08:54.843 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:08:54 smithi092 bash[43686]: debug 2023-09-23T20:08:54.791+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:12.262854+0000 front 2023-09-23T20:08:12.263069+0000 (oldest deadline 2023-09-23T20:08:36.263834+0000) 2023-09-23T20:08:54.843 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:08:54 smithi092 bash[43686]: debug 2023-09-23T20:08:54.791+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:22.666136+0000 front 2023-09-23T20:08:22.666165+0000 (oldest deadline 2023-09-23T20:08:47.365645+0000) 2023-09-23T20:08:55.042 INFO:journalctl@ceph.osd.4.smithi092.stdout:Sep 23 20:08:55 smithi092 systemd[1]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@osd.4.service: Succeeded. 2023-09-23T20:08:55.042 INFO:journalctl@ceph.osd.4.smithi092.stdout:Sep 23 20:08:55 smithi092 systemd[1]: Stopped Ceph osd.4 for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7. 2023-09-23T20:08:55.043 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:08:54 smithi092 bash[39474]: debug 2023-09-23T20:08:54.843+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.009391+0000 front 2023-09-23T20:08:13.009812+0000 (oldest deadline 2023-09-23T20:08:36.509203+0000) 2023-09-23T20:08:55.043 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:08:54 smithi092 bash[39474]: debug 2023-09-23T20:08:54.843+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:24.111476+0000 front 2023-09-23T20:08:24.111596+0000 (oldest deadline 2023-09-23T20:08:50.511939+0000) 2023-09-23T20:08:55.044 DEBUG:teuthology.orchestra.run.smithi092:> sudo pkill -f 'journalctl -f -n 0 -u ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@osd.4.service' 2023-09-23T20:08:55.074 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T20:08:55.074 INFO:tasks.cephadm.osd.4:Stopped osd.4 2023-09-23T20:08:55.074 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2023-09-23T20:08:55.074 DEBUG:teuthology.orchestra.run.smithi092:> sudo systemctl stop ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@osd.5 2023-09-23T20:08:55.297 INFO:journalctl@ceph.osd.5.smithi092.stdout:Sep 23 20:08:55 smithi092 systemd[1]: Stopping Ceph osd.5 for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7... 2023-09-23T20:08:55.556 INFO:journalctl@ceph.osd.5.smithi092.stdout:Sep 23 20:08:55 smithi092 bash[35401]: debug 2023-09-23T20:08:55.295+0000 7f5c66db8700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-09-23T20:08:55.556 INFO:journalctl@ceph.osd.5.smithi092.stdout:Sep 23 20:08:55 smithi092 bash[35401]: debug 2023-09-23T20:08:55.295+0000 7f5c66db8700 -1 osd.5 333 *** Got signal Terminated *** 2023-09-23T20:08:55.556 INFO:journalctl@ceph.osd.5.smithi092.stdout:Sep 23 20:08:55 smithi092 bash[35401]: debug 2023-09-23T20:08:55.295+0000 7f5c66db8700 -1 osd.5 333 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T20:08:55.556 INFO:journalctl@ceph.osd.5.smithi092.stdout:Sep 23 20:08:55 smithi092 bash[35401]: debug 2023-09-23T20:08:55.555+0000 7f5c63d35700 -1 osd.5 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.668168+0000 front 2023-09-23T20:08:13.667873+0000 (oldest deadline 2023-09-23T20:08:37.767622+0000) 2023-09-23T20:08:55.822 INFO:journalctl@ceph.osd.5.smithi092.stdout:Sep 23 20:08:55 smithi092 bash[35401]: debug 2023-09-23T20:08:55.555+0000 7f5c63d35700 -1 osd.5 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:21.769834+0000 front 2023-09-23T20:08:21.769861+0000 (oldest deadline 2023-09-23T20:08:47.069534+0000) 2023-09-23T20:08:55.822 INFO:journalctl@ceph.osd.5.smithi092.stdout:Sep 23 20:08:55 smithi092 bash[35401]: debug 2023-09-23T20:08:55.555+0000 7f5c63d35700 -1 osd.5 333 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-09-23T20:08:32.871835+0000 front 2023-09-23T20:08:32.871948+0000 (oldest deadline 2023-09-23T20:08:55.171429+0000) 2023-09-23T20:08:56.288 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:08:55 smithi092 bash[43686]: debug 2023-09-23T20:08:55.819+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:12.262854+0000 front 2023-09-23T20:08:12.263069+0000 (oldest deadline 2023-09-23T20:08:36.263834+0000) 2023-09-23T20:08:56.289 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:08:55 smithi092 bash[43686]: debug 2023-09-23T20:08:55.819+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:22.666136+0000 front 2023-09-23T20:08:22.666165+0000 (oldest deadline 2023-09-23T20:08:47.365645+0000) 2023-09-23T20:08:56.289 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:08:55 smithi092 bash[39474]: debug 2023-09-23T20:08:55.859+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.009391+0000 front 2023-09-23T20:08:13.009812+0000 (oldest deadline 2023-09-23T20:08:36.509203+0000) 2023-09-23T20:08:56.289 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:08:55 smithi092 bash[39474]: debug 2023-09-23T20:08:55.859+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:24.111476+0000 front 2023-09-23T20:08:24.111596+0000 (oldest deadline 2023-09-23T20:08:50.511939+0000) 2023-09-23T20:08:56.788 INFO:journalctl@ceph.osd.5.smithi092.stdout:Sep 23 20:08:56 smithi092 bash[35401]: debug 2023-09-23T20:08:56.510+0000 7f5c63d35700 -1 osd.5 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.668168+0000 front 2023-09-23T20:08:13.667873+0000 (oldest deadline 2023-09-23T20:08:37.767622+0000) 2023-09-23T20:08:56.789 INFO:journalctl@ceph.osd.5.smithi092.stdout:Sep 23 20:08:56 smithi092 bash[35401]: debug 2023-09-23T20:08:56.510+0000 7f5c63d35700 -1 osd.5 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:21.769834+0000 front 2023-09-23T20:08:21.769861+0000 (oldest deadline 2023-09-23T20:08:47.069534+0000) 2023-09-23T20:08:56.789 INFO:journalctl@ceph.osd.5.smithi092.stdout:Sep 23 20:08:56 smithi092 bash[35401]: debug 2023-09-23T20:08:56.510+0000 7f5c63d35700 -1 osd.5 333 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-09-23T20:08:32.871835+0000 front 2023-09-23T20:08:32.871948+0000 (oldest deadline 2023-09-23T20:08:55.171429+0000) 2023-09-23T20:08:57.288 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:08:56 smithi092 bash[39474]: debug 2023-09-23T20:08:56.842+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.009391+0000 front 2023-09-23T20:08:13.009812+0000 (oldest deadline 2023-09-23T20:08:36.509203+0000) 2023-09-23T20:08:57.289 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:08:56 smithi092 bash[39474]: debug 2023-09-23T20:08:56.842+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:24.111476+0000 front 2023-09-23T20:08:24.111596+0000 (oldest deadline 2023-09-23T20:08:50.511939+0000) 2023-09-23T20:08:57.289 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:08:56 smithi092 bash[43686]: debug 2023-09-23T20:08:56.798+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:12.262854+0000 front 2023-09-23T20:08:12.263069+0000 (oldest deadline 2023-09-23T20:08:36.263834+0000) 2023-09-23T20:08:57.289 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:08:56 smithi092 bash[43686]: debug 2023-09-23T20:08:56.798+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:22.666136+0000 front 2023-09-23T20:08:22.666165+0000 (oldest deadline 2023-09-23T20:08:47.365645+0000) 2023-09-23T20:08:57.789 INFO:journalctl@ceph.osd.5.smithi092.stdout:Sep 23 20:08:57 smithi092 bash[35401]: debug 2023-09-23T20:08:57.522+0000 7f5c63d35700 -1 osd.5 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.668168+0000 front 2023-09-23T20:08:13.667873+0000 (oldest deadline 2023-09-23T20:08:37.767622+0000) 2023-09-23T20:08:57.789 INFO:journalctl@ceph.osd.5.smithi092.stdout:Sep 23 20:08:57 smithi092 bash[35401]: debug 2023-09-23T20:08:57.522+0000 7f5c63d35700 -1 osd.5 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:21.769834+0000 front 2023-09-23T20:08:21.769861+0000 (oldest deadline 2023-09-23T20:08:47.069534+0000) 2023-09-23T20:08:57.789 INFO:journalctl@ceph.osd.5.smithi092.stdout:Sep 23 20:08:57 smithi092 bash[35401]: debug 2023-09-23T20:08:57.522+0000 7f5c63d35700 -1 osd.5 333 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-09-23T20:08:32.871835+0000 front 2023-09-23T20:08:32.871948+0000 (oldest deadline 2023-09-23T20:08:55.171429+0000) 2023-09-23T20:08:58.289 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:08:57 smithi092 bash[39474]: debug 2023-09-23T20:08:57.814+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.009391+0000 front 2023-09-23T20:08:13.009812+0000 (oldest deadline 2023-09-23T20:08:36.509203+0000) 2023-09-23T20:08:58.289 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:08:57 smithi092 bash[39474]: debug 2023-09-23T20:08:57.814+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:24.111476+0000 front 2023-09-23T20:08:24.111596+0000 (oldest deadline 2023-09-23T20:08:50.511939+0000) 2023-09-23T20:08:58.289 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:08:57 smithi092 bash[39474]: debug 2023-09-23T20:08:57.814+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-09-23T20:08:32.714224+0000 front 2023-09-23T20:08:32.714297+0000 (oldest deadline 2023-09-23T20:08:57.414051+0000) 2023-09-23T20:08:58.290 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:08:57 smithi092 bash[43686]: debug 2023-09-23T20:08:57.786+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:12.262854+0000 front 2023-09-23T20:08:12.263069+0000 (oldest deadline 2023-09-23T20:08:36.263834+0000) 2023-09-23T20:08:58.290 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:08:57 smithi092 bash[43686]: debug 2023-09-23T20:08:57.786+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:22.666136+0000 front 2023-09-23T20:08:22.666165+0000 (oldest deadline 2023-09-23T20:08:47.365645+0000) 2023-09-23T20:08:58.290 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:08:57 smithi092 bash[43686]: debug 2023-09-23T20:08:57.786+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-09-23T20:08:31.367819+0000 front 2023-09-23T20:08:31.367841+0000 (oldest deadline 2023-09-23T20:08:57.267753+0000) 2023-09-23T20:08:58.808 INFO:journalctl@ceph.osd.5.smithi092.stdout:Sep 23 20:08:58 smithi092 bash[35401]: debug 2023-09-23T20:08:58.554+0000 7f5c63d35700 -1 osd.5 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.668168+0000 front 2023-09-23T20:08:13.667873+0000 (oldest deadline 2023-09-23T20:08:37.767622+0000) 2023-09-23T20:08:58.808 INFO:journalctl@ceph.osd.5.smithi092.stdout:Sep 23 20:08:58 smithi092 bash[35401]: debug 2023-09-23T20:08:58.554+0000 7f5c63d35700 -1 osd.5 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:21.769834+0000 front 2023-09-23T20:08:21.769861+0000 (oldest deadline 2023-09-23T20:08:47.069534+0000) 2023-09-23T20:08:58.808 INFO:journalctl@ceph.osd.5.smithi092.stdout:Sep 23 20:08:58 smithi092 bash[35401]: debug 2023-09-23T20:08:58.554+0000 7f5c63d35700 -1 osd.5 333 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-09-23T20:08:32.871835+0000 front 2023-09-23T20:08:32.871948+0000 (oldest deadline 2023-09-23T20:08:55.171429+0000) 2023-09-23T20:08:59.289 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:08:58 smithi092 bash[39474]: debug 2023-09-23T20:08:58.806+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.009391+0000 front 2023-09-23T20:08:13.009812+0000 (oldest deadline 2023-09-23T20:08:36.509203+0000) 2023-09-23T20:08:59.289 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:08:58 smithi092 bash[39474]: debug 2023-09-23T20:08:58.806+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:24.111476+0000 front 2023-09-23T20:08:24.111596+0000 (oldest deadline 2023-09-23T20:08:50.511939+0000) 2023-09-23T20:08:59.289 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:08:58 smithi092 bash[39474]: debug 2023-09-23T20:08:58.806+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-09-23T20:08:32.714224+0000 front 2023-09-23T20:08:32.714297+0000 (oldest deadline 2023-09-23T20:08:57.414051+0000) 2023-09-23T20:08:59.289 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:08:58 smithi092 bash[43686]: debug 2023-09-23T20:08:58.830+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:12.262854+0000 front 2023-09-23T20:08:12.263069+0000 (oldest deadline 2023-09-23T20:08:36.263834+0000) 2023-09-23T20:08:59.290 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:08:58 smithi092 bash[43686]: debug 2023-09-23T20:08:58.830+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:22.666136+0000 front 2023-09-23T20:08:22.666165+0000 (oldest deadline 2023-09-23T20:08:47.365645+0000) 2023-09-23T20:08:59.290 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:08:58 smithi092 bash[43686]: debug 2023-09-23T20:08:58.830+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-09-23T20:08:31.367819+0000 front 2023-09-23T20:08:31.367841+0000 (oldest deadline 2023-09-23T20:08:57.267753+0000) 2023-09-23T20:08:59.805 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:08:59 smithi092 bash[39474]: debug 2023-09-23T20:08:59.770+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.009391+0000 front 2023-09-23T20:08:13.009812+0000 (oldest deadline 2023-09-23T20:08:36.509203+0000) 2023-09-23T20:08:59.806 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:08:59 smithi092 bash[39474]: debug 2023-09-23T20:08:59.770+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:24.111476+0000 front 2023-09-23T20:08:24.111596+0000 (oldest deadline 2023-09-23T20:08:50.511939+0000) 2023-09-23T20:08:59.806 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:08:59 smithi092 bash[39474]: debug 2023-09-23T20:08:59.770+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-09-23T20:08:32.714224+0000 front 2023-09-23T20:08:32.714297+0000 (oldest deadline 2023-09-23T20:08:57.414051+0000) 2023-09-23T20:08:59.806 INFO:journalctl@ceph.osd.5.smithi092.stdout:Sep 23 20:08:59 smithi092 bash[35401]: debug 2023-09-23T20:08:59.538+0000 7f5c63d35700 -1 osd.5 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.668168+0000 front 2023-09-23T20:08:13.667873+0000 (oldest deadline 2023-09-23T20:08:37.767622+0000) 2023-09-23T20:08:59.806 INFO:journalctl@ceph.osd.5.smithi092.stdout:Sep 23 20:08:59 smithi092 bash[35401]: debug 2023-09-23T20:08:59.538+0000 7f5c63d35700 -1 osd.5 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:21.769834+0000 front 2023-09-23T20:08:21.769861+0000 (oldest deadline 2023-09-23T20:08:47.069534+0000) 2023-09-23T20:08:59.807 INFO:journalctl@ceph.osd.5.smithi092.stdout:Sep 23 20:08:59 smithi092 bash[35401]: debug 2023-09-23T20:08:59.538+0000 7f5c63d35700 -1 osd.5 333 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-09-23T20:08:32.871835+0000 front 2023-09-23T20:08:32.871948+0000 (oldest deadline 2023-09-23T20:08:55.171429+0000) 2023-09-23T20:09:00.289 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:08:59 smithi092 bash[43686]: debug 2023-09-23T20:08:59.802+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:12.262854+0000 front 2023-09-23T20:08:12.263069+0000 (oldest deadline 2023-09-23T20:08:36.263834+0000) 2023-09-23T20:09:00.289 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:08:59 smithi092 bash[43686]: debug 2023-09-23T20:08:59.802+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:22.666136+0000 front 2023-09-23T20:08:22.666165+0000 (oldest deadline 2023-09-23T20:08:47.365645+0000) 2023-09-23T20:09:00.289 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:08:59 smithi092 bash[43686]: debug 2023-09-23T20:08:59.802+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-09-23T20:08:31.367819+0000 front 2023-09-23T20:08:31.367841+0000 (oldest deadline 2023-09-23T20:08:57.267753+0000) 2023-09-23T20:09:01.039 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:09:00 smithi092 bash[39474]: debug 2023-09-23T20:09:00.814+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.009391+0000 front 2023-09-23T20:08:13.009812+0000 (oldest deadline 2023-09-23T20:08:36.509203+0000) 2023-09-23T20:09:01.039 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:09:00 smithi092 bash[39474]: debug 2023-09-23T20:09:00.814+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:24.111476+0000 front 2023-09-23T20:08:24.111596+0000 (oldest deadline 2023-09-23T20:08:50.511939+0000) 2023-09-23T20:09:01.039 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:09:00 smithi092 bash[39474]: debug 2023-09-23T20:09:00.814+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-09-23T20:08:32.714224+0000 front 2023-09-23T20:08:32.714297+0000 (oldest deadline 2023-09-23T20:08:57.414051+0000) 2023-09-23T20:09:01.039 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:00 smithi092 bash[43686]: debug 2023-09-23T20:09:00.842+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:12.262854+0000 front 2023-09-23T20:08:12.263069+0000 (oldest deadline 2023-09-23T20:08:36.263834+0000) 2023-09-23T20:09:01.040 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:00 smithi092 bash[43686]: debug 2023-09-23T20:09:00.842+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:22.666136+0000 front 2023-09-23T20:08:22.666165+0000 (oldest deadline 2023-09-23T20:08:47.365645+0000) 2023-09-23T20:09:01.040 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:00 smithi092 bash[43686]: debug 2023-09-23T20:09:00.842+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-09-23T20:08:31.367819+0000 front 2023-09-23T20:08:31.367841+0000 (oldest deadline 2023-09-23T20:08:57.267753+0000) 2023-09-23T20:09:01.373 INFO:journalctl@ceph.osd.5.smithi092.stdout:Sep 23 20:09:01 smithi092 bash[155441]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd-5 2023-09-23T20:09:01.373 INFO:journalctl@ceph.osd.5.smithi092.stdout:Sep 23 20:09:01 smithi092 bash[155506]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd-5 2023-09-23T20:09:02.261 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:09:01 smithi092 bash[39474]: debug 2023-09-23T20:09:01.814+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.009391+0000 front 2023-09-23T20:08:13.009812+0000 (oldest deadline 2023-09-23T20:08:36.509203+0000) 2023-09-23T20:09:02.261 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:09:01 smithi092 bash[39474]: debug 2023-09-23T20:09:01.814+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:24.111476+0000 front 2023-09-23T20:08:24.111596+0000 (oldest deadline 2023-09-23T20:08:50.511939+0000) 2023-09-23T20:09:02.261 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:09:01 smithi092 bash[39474]: debug 2023-09-23T20:09:01.814+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-09-23T20:08:32.714224+0000 front 2023-09-23T20:08:32.714297+0000 (oldest deadline 2023-09-23T20:08:57.414051+0000) 2023-09-23T20:09:02.262 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:01 smithi092 bash[43686]: debug 2023-09-23T20:09:01.866+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:12.262854+0000 front 2023-09-23T20:08:12.263069+0000 (oldest deadline 2023-09-23T20:08:36.263834+0000) 2023-09-23T20:09:02.262 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:01 smithi092 bash[43686]: debug 2023-09-23T20:09:01.866+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:22.666136+0000 front 2023-09-23T20:08:22.666165+0000 (oldest deadline 2023-09-23T20:08:47.365645+0000) 2023-09-23T20:09:02.262 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:01 smithi092 bash[43686]: debug 2023-09-23T20:09:01.866+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-09-23T20:08:31.367819+0000 front 2023-09-23T20:08:31.367841+0000 (oldest deadline 2023-09-23T20:08:57.267753+0000) 2023-09-23T20:09:03.038 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:09:02 smithi092 bash[39474]: debug 2023-09-23T20:09:02.766+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.009391+0000 front 2023-09-23T20:08:13.009812+0000 (oldest deadline 2023-09-23T20:08:36.509203+0000) 2023-09-23T20:09:03.039 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:09:02 smithi092 bash[39474]: debug 2023-09-23T20:09:02.766+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:24.111476+0000 front 2023-09-23T20:08:24.111596+0000 (oldest deadline 2023-09-23T20:08:50.511939+0000) 2023-09-23T20:09:03.039 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:09:02 smithi092 bash[39474]: debug 2023-09-23T20:09:02.766+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-09-23T20:08:32.714224+0000 front 2023-09-23T20:08:32.714297+0000 (oldest deadline 2023-09-23T20:08:57.414051+0000) 2023-09-23T20:09:03.039 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:02 smithi092 bash[43686]: debug 2023-09-23T20:09:02.866+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:12.262854+0000 front 2023-09-23T20:08:12.263069+0000 (oldest deadline 2023-09-23T20:08:36.263834+0000) 2023-09-23T20:09:03.039 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:02 smithi092 bash[43686]: debug 2023-09-23T20:09:02.866+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:22.666136+0000 front 2023-09-23T20:08:22.666165+0000 (oldest deadline 2023-09-23T20:08:47.365645+0000) 2023-09-23T20:09:03.039 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:02 smithi092 bash[43686]: debug 2023-09-23T20:09:02.866+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-09-23T20:08:31.367819+0000 front 2023-09-23T20:08:31.367841+0000 (oldest deadline 2023-09-23T20:08:57.267753+0000) 2023-09-23T20:09:03.454 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 20:09:03 smithi092 bash[54653]: ts=2023-09-23T20:09:03.150Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.16:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.16:8765: connect: connection refused" 2023-09-23T20:09:03.455 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 20:09:03 smithi092 bash[54653]: ts=2023-09-23T20:09:03.154Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.16:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.16:8765: connect: connection refused" 2023-09-23T20:09:03.455 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 20:09:03 smithi092 bash[54653]: ts=2023-09-23T20:09:03.152Z caller=refresh.go:98 level=error component="discovery manager notify" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.16:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.16:8765: connect: connection refused" 2023-09-23T20:09:03.455 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 20:09:03 smithi092 bash[54653]: ts=2023-09-23T20:09:03.154Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.16:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.16:8765: connect: connection refused" 2023-09-23T20:09:03.458 DEBUG:teuthology.orchestra.run.smithi092:> sudo pkill -f 'journalctl -f -n 0 -u ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@osd.5.service' 2023-09-23T20:09:03.489 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T20:09:03.490 INFO:tasks.cephadm.osd.5:Stopped osd.5 2023-09-23T20:09:03.490 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2023-09-23T20:09:03.490 DEBUG:teuthology.orchestra.run.smithi092:> sudo systemctl stop ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@osd.6 2023-09-23T20:09:03.748 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:09:03 smithi092 systemd[1]: Stopping Ceph osd.6 for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7... 2023-09-23T20:09:03.748 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:09:03 smithi092 bash[39474]: debug 2023-09-23T20:09:03.698+0000 7fcae32af700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-09-23T20:09:03.748 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:09:03 smithi092 bash[39474]: debug 2023-09-23T20:09:03.698+0000 7fcae32af700 -1 osd.6 333 *** Got signal Terminated *** 2023-09-23T20:09:03.748 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:09:03 smithi092 bash[39474]: debug 2023-09-23T20:09:03.698+0000 7fcae32af700 -1 osd.6 333 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T20:09:03.749 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:09:03 smithi092 bash[39474]: debug 2023-09-23T20:09:03.746+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.009391+0000 front 2023-09-23T20:08:13.009812+0000 (oldest deadline 2023-09-23T20:08:36.509203+0000) 2023-09-23T20:09:04.039 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:09:03 smithi092 bash[39474]: debug 2023-09-23T20:09:03.746+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:24.111476+0000 front 2023-09-23T20:08:24.111596+0000 (oldest deadline 2023-09-23T20:08:50.511939+0000) 2023-09-23T20:09:04.039 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:09:03 smithi092 bash[39474]: debug 2023-09-23T20:09:03.746+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-09-23T20:08:32.714224+0000 front 2023-09-23T20:08:32.714297+0000 (oldest deadline 2023-09-23T20:08:57.414051+0000) 2023-09-23T20:09:04.039 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:03 smithi092 bash[43686]: debug 2023-09-23T20:09:03.870+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:12.262854+0000 front 2023-09-23T20:08:12.263069+0000 (oldest deadline 2023-09-23T20:08:36.263834+0000) 2023-09-23T20:09:04.040 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:03 smithi092 bash[43686]: debug 2023-09-23T20:09:03.870+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:22.666136+0000 front 2023-09-23T20:08:22.666165+0000 (oldest deadline 2023-09-23T20:08:47.365645+0000) 2023-09-23T20:09:04.040 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:03 smithi092 bash[43686]: debug 2023-09-23T20:09:03.870+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-09-23T20:08:31.367819+0000 front 2023-09-23T20:08:31.367841+0000 (oldest deadline 2023-09-23T20:08:57.267753+0000) 2023-09-23T20:09:05.039 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:09:04 smithi092 bash[39474]: debug 2023-09-23T20:09:04.710+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.009391+0000 front 2023-09-23T20:08:13.009812+0000 (oldest deadline 2023-09-23T20:08:36.509203+0000) 2023-09-23T20:09:05.039 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:09:04 smithi092 bash[39474]: debug 2023-09-23T20:09:04.710+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:24.111476+0000 front 2023-09-23T20:08:24.111596+0000 (oldest deadline 2023-09-23T20:08:50.511939+0000) 2023-09-23T20:09:05.039 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:09:04 smithi092 bash[39474]: debug 2023-09-23T20:09:04.710+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-09-23T20:08:32.714224+0000 front 2023-09-23T20:08:32.714297+0000 (oldest deadline 2023-09-23T20:08:57.414051+0000) 2023-09-23T20:09:05.039 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:04 smithi092 bash[43686]: debug 2023-09-23T20:09:04.822+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:12.262854+0000 front 2023-09-23T20:08:12.263069+0000 (oldest deadline 2023-09-23T20:08:36.263834+0000) 2023-09-23T20:09:05.039 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:04 smithi092 bash[43686]: debug 2023-09-23T20:09:04.822+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:22.666136+0000 front 2023-09-23T20:08:22.666165+0000 (oldest deadline 2023-09-23T20:08:47.365645+0000) 2023-09-23T20:09:05.040 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:04 smithi092 bash[43686]: debug 2023-09-23T20:09:04.822+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-09-23T20:08:31.367819+0000 front 2023-09-23T20:08:31.367841+0000 (oldest deadline 2023-09-23T20:08:57.267753+0000) 2023-09-23T20:09:05.040 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:04 smithi092 bash[43686]: debug 2023-09-23T20:09:04.822+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6830 osd.3 since back 2023-09-23T20:08:40.769255+0000 front 2023-09-23T20:08:40.769222+0000 (oldest deadline 2023-09-23T20:09:04.268916+0000) 2023-09-23T20:09:06.039 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:09:05 smithi092 bash[39474]: debug 2023-09-23T20:09:05.738+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.009391+0000 front 2023-09-23T20:08:13.009812+0000 (oldest deadline 2023-09-23T20:08:36.509203+0000) 2023-09-23T20:09:06.039 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:09:05 smithi092 bash[39474]: debug 2023-09-23T20:09:05.738+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:24.111476+0000 front 2023-09-23T20:08:24.111596+0000 (oldest deadline 2023-09-23T20:08:50.511939+0000) 2023-09-23T20:09:06.039 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:09:05 smithi092 bash[39474]: debug 2023-09-23T20:09:05.738+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-09-23T20:08:32.714224+0000 front 2023-09-23T20:08:32.714297+0000 (oldest deadline 2023-09-23T20:08:57.414051+0000) 2023-09-23T20:09:06.040 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:05 smithi092 bash[43686]: debug 2023-09-23T20:09:05.834+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:12.262854+0000 front 2023-09-23T20:08:12.263069+0000 (oldest deadline 2023-09-23T20:08:36.263834+0000) 2023-09-23T20:09:06.040 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:05 smithi092 bash[43686]: debug 2023-09-23T20:09:05.834+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:22.666136+0000 front 2023-09-23T20:08:22.666165+0000 (oldest deadline 2023-09-23T20:08:47.365645+0000) 2023-09-23T20:09:06.040 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:05 smithi092 bash[43686]: debug 2023-09-23T20:09:05.834+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-09-23T20:08:31.367819+0000 front 2023-09-23T20:08:31.367841+0000 (oldest deadline 2023-09-23T20:08:57.267753+0000) 2023-09-23T20:09:06.040 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:05 smithi092 bash[43686]: debug 2023-09-23T20:09:05.834+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6830 osd.3 since back 2023-09-23T20:08:40.769255+0000 front 2023-09-23T20:08:40.769222+0000 (oldest deadline 2023-09-23T20:09:04.268916+0000) 2023-09-23T20:09:07.038 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:06 smithi092 bash[43686]: debug 2023-09-23T20:09:06.870+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:12.262854+0000 front 2023-09-23T20:08:12.263069+0000 (oldest deadline 2023-09-23T20:08:36.263834+0000) 2023-09-23T20:09:07.039 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:06 smithi092 bash[43686]: debug 2023-09-23T20:09:06.870+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:22.666136+0000 front 2023-09-23T20:08:22.666165+0000 (oldest deadline 2023-09-23T20:08:47.365645+0000) 2023-09-23T20:09:07.039 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:06 smithi092 bash[43686]: debug 2023-09-23T20:09:06.870+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-09-23T20:08:31.367819+0000 front 2023-09-23T20:08:31.367841+0000 (oldest deadline 2023-09-23T20:08:57.267753+0000) 2023-09-23T20:09:07.039 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:06 smithi092 bash[43686]: debug 2023-09-23T20:09:06.870+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6830 osd.3 since back 2023-09-23T20:08:40.769255+0000 front 2023-09-23T20:08:40.769222+0000 (oldest deadline 2023-09-23T20:09:04.268916+0000) 2023-09-23T20:09:07.039 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:09:06 smithi092 bash[39474]: debug 2023-09-23T20:09:06.786+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.009391+0000 front 2023-09-23T20:08:13.009812+0000 (oldest deadline 2023-09-23T20:08:36.509203+0000) 2023-09-23T20:09:07.040 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:09:06 smithi092 bash[39474]: debug 2023-09-23T20:09:06.786+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:24.111476+0000 front 2023-09-23T20:08:24.111596+0000 (oldest deadline 2023-09-23T20:08:50.511939+0000) 2023-09-23T20:09:07.040 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:09:06 smithi092 bash[39474]: debug 2023-09-23T20:09:06.786+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-09-23T20:08:32.714224+0000 front 2023-09-23T20:08:32.714297+0000 (oldest deadline 2023-09-23T20:08:57.414051+0000) 2023-09-23T20:09:08.039 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:07 smithi092 bash[43686]: debug 2023-09-23T20:09:07.858+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:12.262854+0000 front 2023-09-23T20:08:12.263069+0000 (oldest deadline 2023-09-23T20:08:36.263834+0000) 2023-09-23T20:09:08.039 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:07 smithi092 bash[43686]: debug 2023-09-23T20:09:07.858+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:22.666136+0000 front 2023-09-23T20:08:22.666165+0000 (oldest deadline 2023-09-23T20:08:47.365645+0000) 2023-09-23T20:09:08.039 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:07 smithi092 bash[43686]: debug 2023-09-23T20:09:07.858+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-09-23T20:08:31.367819+0000 front 2023-09-23T20:08:31.367841+0000 (oldest deadline 2023-09-23T20:08:57.267753+0000) 2023-09-23T20:09:08.039 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:07 smithi092 bash[43686]: debug 2023-09-23T20:09:07.858+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6830 osd.3 since back 2023-09-23T20:08:40.769255+0000 front 2023-09-23T20:08:40.769222+0000 (oldest deadline 2023-09-23T20:09:04.268916+0000) 2023-09-23T20:09:08.040 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:09:07 smithi092 bash[39474]: debug 2023-09-23T20:09:07.746+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:13.009391+0000 front 2023-09-23T20:08:13.009812+0000 (oldest deadline 2023-09-23T20:08:36.509203+0000) 2023-09-23T20:09:08.040 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:09:07 smithi092 bash[39474]: debug 2023-09-23T20:09:07.746+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:24.111476+0000 front 2023-09-23T20:08:24.111596+0000 (oldest deadline 2023-09-23T20:08:50.511939+0000) 2023-09-23T20:09:08.040 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:09:07 smithi092 bash[39474]: debug 2023-09-23T20:09:07.746+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-09-23T20:08:32.714224+0000 front 2023-09-23T20:08:32.714297+0000 (oldest deadline 2023-09-23T20:08:57.414051+0000) 2023-09-23T20:09:08.040 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:09:07 smithi092 bash[39474]: debug 2023-09-23T20:09:07.746+0000 7fcade228700 -1 osd.6 333 heartbeat_check: no reply from 172.21.15.16:6830 osd.3 since back 2023-09-23T20:08:41.515439+0000 front 2023-09-23T20:08:41.515412+0000 (oldest deadline 2023-09-23T20:09:07.415232+0000) 2023-09-23T20:09:09.288 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:08 smithi092 bash[43686]: debug 2023-09-23T20:09:08.910+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:12.262854+0000 front 2023-09-23T20:08:12.263069+0000 (oldest deadline 2023-09-23T20:08:36.263834+0000) 2023-09-23T20:09:09.289 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:08 smithi092 bash[43686]: debug 2023-09-23T20:09:08.910+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:22.666136+0000 front 2023-09-23T20:08:22.666165+0000 (oldest deadline 2023-09-23T20:08:47.365645+0000) 2023-09-23T20:09:09.289 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:08 smithi092 bash[43686]: debug 2023-09-23T20:09:08.910+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-09-23T20:08:31.367819+0000 front 2023-09-23T20:08:31.367841+0000 (oldest deadline 2023-09-23T20:08:57.267753+0000) 2023-09-23T20:09:09.289 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:08 smithi092 bash[43686]: debug 2023-09-23T20:09:08.910+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6830 osd.3 since back 2023-09-23T20:08:40.769255+0000 front 2023-09-23T20:08:40.769222+0000 (oldest deadline 2023-09-23T20:09:04.268916+0000) 2023-09-23T20:09:09.903 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:09:09 smithi092 bash[155648]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd-6 2023-09-23T20:09:09.903 INFO:journalctl@ceph.osd.6.smithi092.stdout:Sep 23 20:09:09 smithi092 bash[155715]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd-6 2023-09-23T20:09:10.288 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:09 smithi092 bash[43686]: debug 2023-09-23T20:09:09.898+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:12.262854+0000 front 2023-09-23T20:08:12.263069+0000 (oldest deadline 2023-09-23T20:08:36.263834+0000) 2023-09-23T20:09:10.289 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:09 smithi092 bash[43686]: debug 2023-09-23T20:09:09.898+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:22.666136+0000 front 2023-09-23T20:08:22.666165+0000 (oldest deadline 2023-09-23T20:08:47.365645+0000) 2023-09-23T20:09:10.289 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:09 smithi092 bash[43686]: debug 2023-09-23T20:09:09.898+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-09-23T20:08:31.367819+0000 front 2023-09-23T20:08:31.367841+0000 (oldest deadline 2023-09-23T20:08:57.267753+0000) 2023-09-23T20:09:10.289 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:09 smithi092 bash[43686]: debug 2023-09-23T20:09:09.898+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6830 osd.3 since back 2023-09-23T20:08:40.769255+0000 front 2023-09-23T20:08:40.769222+0000 (oldest deadline 2023-09-23T20:09:04.268916+0000) 2023-09-23T20:09:11.289 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:10 smithi092 bash[43686]: debug 2023-09-23T20:09:10.934+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:12.262854+0000 front 2023-09-23T20:08:12.263069+0000 (oldest deadline 2023-09-23T20:08:36.263834+0000) 2023-09-23T20:09:11.289 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:10 smithi092 bash[43686]: debug 2023-09-23T20:09:10.934+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:22.666136+0000 front 2023-09-23T20:08:22.666165+0000 (oldest deadline 2023-09-23T20:08:47.365645+0000) 2023-09-23T20:09:11.289 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:10 smithi092 bash[43686]: debug 2023-09-23T20:09:10.934+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-09-23T20:08:31.367819+0000 front 2023-09-23T20:08:31.367841+0000 (oldest deadline 2023-09-23T20:08:57.267753+0000) 2023-09-23T20:09:11.289 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:10 smithi092 bash[43686]: debug 2023-09-23T20:09:10.934+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6830 osd.3 since back 2023-09-23T20:08:40.769255+0000 front 2023-09-23T20:08:40.769222+0000 (oldest deadline 2023-09-23T20:09:04.268916+0000) 2023-09-23T20:09:11.737 DEBUG:teuthology.orchestra.run.smithi092:> sudo pkill -f 'journalctl -f -n 0 -u ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@osd.6.service' 2023-09-23T20:09:11.767 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T20:09:11.768 INFO:tasks.cephadm.osd.6:Stopped osd.6 2023-09-23T20:09:11.768 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2023-09-23T20:09:11.768 DEBUG:teuthology.orchestra.run.smithi092:> sudo systemctl stop ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@osd.7 2023-09-23T20:09:12.001 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:11 smithi092 systemd[1]: Stopping Ceph osd.7 for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7... 2023-09-23T20:09:12.001 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:11 smithi092 bash[43686]: debug 2023-09-23T20:09:11.890+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:12.262854+0000 front 2023-09-23T20:08:12.263069+0000 (oldest deadline 2023-09-23T20:08:36.263834+0000) 2023-09-23T20:09:12.001 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:11 smithi092 bash[43686]: debug 2023-09-23T20:09:11.890+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:22.666136+0000 front 2023-09-23T20:08:22.666165+0000 (oldest deadline 2023-09-23T20:08:47.365645+0000) 2023-09-23T20:09:12.001 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:11 smithi092 bash[43686]: debug 2023-09-23T20:09:11.890+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-09-23T20:08:31.367819+0000 front 2023-09-23T20:08:31.367841+0000 (oldest deadline 2023-09-23T20:08:57.267753+0000) 2023-09-23T20:09:12.001 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:11 smithi092 bash[43686]: debug 2023-09-23T20:09:11.890+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6830 osd.3 since back 2023-09-23T20:08:40.769255+0000 front 2023-09-23T20:08:40.769222+0000 (oldest deadline 2023-09-23T20:09:04.268916+0000) 2023-09-23T20:09:12.288 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:12 smithi092 bash[43686]: debug 2023-09-23T20:09:11.998+0000 7f79c7016700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-09-23T20:09:12.289 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:12 smithi092 bash[43686]: debug 2023-09-23T20:09:11.998+0000 7f79c7016700 -1 osd.7 333 *** Got signal Terminated *** 2023-09-23T20:09:12.289 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:12 smithi092 bash[43686]: debug 2023-09-23T20:09:11.998+0000 7f79c7016700 -1 osd.7 333 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T20:09:13.289 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:12 smithi092 bash[43686]: debug 2023-09-23T20:09:12.862+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:12.262854+0000 front 2023-09-23T20:08:12.263069+0000 (oldest deadline 2023-09-23T20:08:36.263834+0000) 2023-09-23T20:09:13.289 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:12 smithi092 bash[43686]: debug 2023-09-23T20:09:12.862+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:22.666136+0000 front 2023-09-23T20:08:22.666165+0000 (oldest deadline 2023-09-23T20:08:47.365645+0000) 2023-09-23T20:09:13.289 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:12 smithi092 bash[43686]: debug 2023-09-23T20:09:12.862+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-09-23T20:08:31.367819+0000 front 2023-09-23T20:08:31.367841+0000 (oldest deadline 2023-09-23T20:08:57.267753+0000) 2023-09-23T20:09:13.289 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:12 smithi092 bash[43686]: debug 2023-09-23T20:09:12.862+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6830 osd.3 since back 2023-09-23T20:08:40.769255+0000 front 2023-09-23T20:08:40.769222+0000 (oldest deadline 2023-09-23T20:09:04.268916+0000) 2023-09-23T20:09:13.289 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:12 smithi092 bash[43686]: debug 2023-09-23T20:09:12.862+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.92:6804 osd.4 since back 2023-09-23T20:08:47.671118+0000 front 2023-09-23T20:08:47.671151+0000 (oldest deadline 2023-09-23T20:09:12.370950+0000) 2023-09-23T20:09:14.289 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:13 smithi092 bash[43686]: debug 2023-09-23T20:09:13.842+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:12.262854+0000 front 2023-09-23T20:08:12.263069+0000 (oldest deadline 2023-09-23T20:08:36.263834+0000) 2023-09-23T20:09:14.289 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:13 smithi092 bash[43686]: debug 2023-09-23T20:09:13.842+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:22.666136+0000 front 2023-09-23T20:08:22.666165+0000 (oldest deadline 2023-09-23T20:08:47.365645+0000) 2023-09-23T20:09:14.289 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:13 smithi092 bash[43686]: debug 2023-09-23T20:09:13.842+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-09-23T20:08:31.367819+0000 front 2023-09-23T20:08:31.367841+0000 (oldest deadline 2023-09-23T20:08:57.267753+0000) 2023-09-23T20:09:14.289 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:13 smithi092 bash[43686]: debug 2023-09-23T20:09:13.842+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6830 osd.3 since back 2023-09-23T20:08:40.769255+0000 front 2023-09-23T20:08:40.769222+0000 (oldest deadline 2023-09-23T20:09:04.268916+0000) 2023-09-23T20:09:14.289 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:13 smithi092 bash[43686]: debug 2023-09-23T20:09:13.842+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.92:6804 osd.4 since back 2023-09-23T20:08:47.671118+0000 front 2023-09-23T20:08:47.671151+0000 (oldest deadline 2023-09-23T20:09:12.370950+0000) 2023-09-23T20:09:15.289 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:14 smithi092 bash[43686]: debug 2023-09-23T20:09:14.858+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:12.262854+0000 front 2023-09-23T20:08:12.263069+0000 (oldest deadline 2023-09-23T20:08:36.263834+0000) 2023-09-23T20:09:15.289 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:14 smithi092 bash[43686]: debug 2023-09-23T20:09:14.858+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:22.666136+0000 front 2023-09-23T20:08:22.666165+0000 (oldest deadline 2023-09-23T20:08:47.365645+0000) 2023-09-23T20:09:15.289 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:14 smithi092 bash[43686]: debug 2023-09-23T20:09:14.858+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-09-23T20:08:31.367819+0000 front 2023-09-23T20:08:31.367841+0000 (oldest deadline 2023-09-23T20:08:57.267753+0000) 2023-09-23T20:09:15.289 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:14 smithi092 bash[43686]: debug 2023-09-23T20:09:14.858+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6830 osd.3 since back 2023-09-23T20:08:40.769255+0000 front 2023-09-23T20:08:40.769222+0000 (oldest deadline 2023-09-23T20:09:04.268916+0000) 2023-09-23T20:09:15.289 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:14 smithi092 bash[43686]: debug 2023-09-23T20:09:14.858+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.92:6804 osd.4 since back 2023-09-23T20:08:47.671118+0000 front 2023-09-23T20:08:47.671151+0000 (oldest deadline 2023-09-23T20:09:12.370950+0000) 2023-09-23T20:09:16.288 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:15 smithi092 bash[43686]: debug 2023-09-23T20:09:15.814+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:12.262854+0000 front 2023-09-23T20:08:12.263069+0000 (oldest deadline 2023-09-23T20:08:36.263834+0000) 2023-09-23T20:09:16.289 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:15 smithi092 bash[43686]: debug 2023-09-23T20:09:15.814+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:22.666136+0000 front 2023-09-23T20:08:22.666165+0000 (oldest deadline 2023-09-23T20:08:47.365645+0000) 2023-09-23T20:09:16.289 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:15 smithi092 bash[43686]: debug 2023-09-23T20:09:15.814+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-09-23T20:08:31.367819+0000 front 2023-09-23T20:08:31.367841+0000 (oldest deadline 2023-09-23T20:08:57.267753+0000) 2023-09-23T20:09:16.289 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:15 smithi092 bash[43686]: debug 2023-09-23T20:09:15.814+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6830 osd.3 since back 2023-09-23T20:08:40.769255+0000 front 2023-09-23T20:08:40.769222+0000 (oldest deadline 2023-09-23T20:09:04.268916+0000) 2023-09-23T20:09:16.289 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:15 smithi092 bash[43686]: debug 2023-09-23T20:09:15.814+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.92:6804 osd.4 since back 2023-09-23T20:08:47.671118+0000 front 2023-09-23T20:08:47.671151+0000 (oldest deadline 2023-09-23T20:09:12.370950+0000) 2023-09-23T20:09:17.039 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:16 smithi092 bash[43686]: debug 2023-09-23T20:09:16.786+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6806 osd.0 since back 2023-09-23T20:08:12.262854+0000 front 2023-09-23T20:08:12.263069+0000 (oldest deadline 2023-09-23T20:08:36.263834+0000) 2023-09-23T20:09:17.039 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:16 smithi092 bash[43686]: debug 2023-09-23T20:09:16.786+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6814 osd.1 since back 2023-09-23T20:08:22.666136+0000 front 2023-09-23T20:08:22.666165+0000 (oldest deadline 2023-09-23T20:08:47.365645+0000) 2023-09-23T20:09:17.039 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:16 smithi092 bash[43686]: debug 2023-09-23T20:09:16.786+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6822 osd.2 since back 2023-09-23T20:08:31.367819+0000 front 2023-09-23T20:08:31.367841+0000 (oldest deadline 2023-09-23T20:08:57.267753+0000) 2023-09-23T20:09:17.039 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:16 smithi092 bash[43686]: debug 2023-09-23T20:09:16.786+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.16:6830 osd.3 since back 2023-09-23T20:08:40.769255+0000 front 2023-09-23T20:08:40.769222+0000 (oldest deadline 2023-09-23T20:09:04.268916+0000) 2023-09-23T20:09:17.039 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:16 smithi092 bash[43686]: debug 2023-09-23T20:09:16.786+0000 7f79c0f8d700 -1 osd.7 333 heartbeat_check: no reply from 172.21.15.92:6804 osd.4 since back 2023-09-23T20:08:47.671118+0000 front 2023-09-23T20:08:47.671151+0000 (oldest deadline 2023-09-23T20:09:12.370950+0000) 2023-09-23T20:09:18.038 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:17 smithi092 bash[155854]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd-7 2023-09-23T20:09:18.039 INFO:journalctl@ceph.osd.7.smithi092.stdout:Sep 23 20:09:17 smithi092 bash[155935]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-osd-7 2023-09-23T20:09:19.862 DEBUG:teuthology.orchestra.run.smithi092:> sudo pkill -f 'journalctl -f -n 0 -u ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@osd.7.service' 2023-09-23T20:09:19.894 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T20:09:19.895 INFO:tasks.cephadm.osd.7:Stopped osd.7 2023-09-23T20:09:19.895 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2023-09-23T20:09:19.895 DEBUG:teuthology.orchestra.run.smithi092:> sudo systemctl stop ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@prometheus.a 2023-09-23T20:09:20.124 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 20:09:19 smithi092 systemd[1]: Stopping Ceph prometheus.a for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7... 2023-09-23T20:09:20.539 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 20:09:20 smithi092 bash[54653]: ts=2023-09-23T20:09:20.125Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-09-23T20:09:20.539 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 20:09:20 smithi092 bash[54653]: ts=2023-09-23T20:09:20.125Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-09-23T20:09:20.539 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 20:09:20 smithi092 bash[54653]: ts=2023-09-23T20:09:20.125Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-09-23T20:09:20.539 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 20:09:20 smithi092 bash[54653]: ts=2023-09-23T20:09:20.125Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-09-23T20:09:20.539 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 20:09:20 smithi092 bash[54653]: ts=2023-09-23T20:09:20.125Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-09-23T20:09:20.540 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 20:09:20 smithi092 bash[54653]: ts=2023-09-23T20:09:20.125Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-09-23T20:09:20.540 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 20:09:20 smithi092 bash[54653]: ts=2023-09-23T20:09:20.126Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-09-23T20:09:20.540 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 20:09:20 smithi092 bash[54653]: ts=2023-09-23T20:09:20.126Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-09-23T20:09:20.540 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 20:09:20 smithi092 bash[54653]: ts=2023-09-23T20:09:20.126Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-09-23T20:09:20.540 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 20:09:20 smithi092 bash[54653]: ts=2023-09-23T20:09:20.129Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-09-23T20:09:20.540 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 20:09:20 smithi092 bash[54653]: ts=2023-09-23T20:09:20.129Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-09-23T20:09:20.540 INFO:journalctl@ceph.prometheus.a.smithi092.stdout:Sep 23 20:09:20 smithi092 bash[54653]: ts=2023-09-23T20:09:20.129Z caller=main.go:1066 level=info msg="See you next time!" 2023-09-23T20:09:20.967 DEBUG:teuthology.orchestra.run.smithi092:> sudo pkill -f 'journalctl -f -n 0 -u ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@prometheus.a.service' 2023-09-23T20:09:20.996 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T20:09:20.996 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2023-09-23T20:09:20.996 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 --force --keep-logs 2023-09-23T20:09:27.996 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 20:09:27 smithi016 systemd[1]: Stopping Ceph node-exporter.a for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7... 2023-09-23T20:09:28.779 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 20:09:28 smithi016 bash[268761]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-node-exporter-a 2023-09-23T20:09:28.780 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 20:09:28 smithi016 bash[268823]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-node-exporter-a 2023-09-23T20:09:28.780 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 20:09:28 smithi016 systemd[1]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-09-23T20:09:28.780 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 20:09:28 smithi016 systemd[1]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@node-exporter.a.service: Failed with result 'exit-code'. 2023-09-23T20:09:28.781 INFO:journalctl@ceph.node-exporter.a.smithi016.stdout:Sep 23 20:09:28 smithi016 systemd[1]: Stopped Ceph node-exporter.a for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7. 2023-09-23T20:09:29.035 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Sep 23 20:09:28 smithi016 systemd[1]: Stopping Ceph alertmanager.a for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7... 2023-09-23T20:09:29.036 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Sep 23 20:09:28 smithi016 bash[70090]: level=info ts=2023-09-23T20:09:28.929Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2023-09-23T20:09:30.011 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Sep 23 20:09:29 smithi016 bash[268885]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-alertmanager-a 2023-09-23T20:09:30.011 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Sep 23 20:09:29 smithi016 bash[268947]: Error response from daemon: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-alertmanager-a 2023-09-23T20:09:30.011 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Sep 23 20:09:29 smithi016 systemd[1]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-09-23T20:09:30.011 INFO:journalctl@ceph.alertmanager.a.smithi016.stdout:Sep 23 20:09:29 smithi016 systemd[1]: Stopped Ceph alertmanager.a for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7. 2023-09-23T20:09:40.885 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 --force --keep-logs 2023-09-23T20:09:47.789 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 20:09:47 smithi092 systemd[1]: Stopping Ceph node-exporter.b for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7... 2023-09-23T20:09:48.509 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 20:09:48 smithi092 bash[156457]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-node-exporter-b 2023-09-23T20:09:48.510 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 20:09:48 smithi092 systemd[1]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-09-23T20:09:48.510 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 20:09:48 smithi092 bash[156523]: Error response from daemon: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-node-exporter-b 2023-09-23T20:09:48.510 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 20:09:48 smithi092 systemd[1]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@node-exporter.b.service: Failed with result 'exit-code'. 2023-09-23T20:09:48.510 INFO:journalctl@ceph.node-exporter.b.smithi092.stdout:Sep 23 20:09:48 smithi092 systemd[1]: Stopped Ceph node-exporter.b for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7. 2023-09-23T20:09:48.789 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 20:09:48 smithi092 systemd[1]: Stopping Ceph grafana.a for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7... 2023-09-23T20:09:48.789 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 20:09:48 smithi092 bash[53391]: t=2023-09-23T20:09:48+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2023-09-23T20:09:49.696 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 20:09:49 smithi092 bash[156586]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-grafana-a 2023-09-23T20:09:49.697 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 20:09:49 smithi092 bash[156649]: Error response from daemon: No such container: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7-grafana-a 2023-09-23T20:09:49.697 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 20:09:49 smithi092 systemd[1]: ceph-95ce9eb6-5a42-11ee-8db4-212e2dc638e7@grafana.a.service: Succeeded. 2023-09-23T20:09:49.697 INFO:journalctl@ceph.grafana.a.smithi092.stdout:Sep 23 20:09:49 smithi092 systemd[1]: Stopped Ceph grafana.a for 95ce9eb6-5a42-11ee-8db4-212e2dc638e7. 2023-09-23T20:09:52.942 DEBUG:teuthology.orchestra.run.smithi016:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-09-23T20:09:52.960 DEBUG:teuthology.orchestra.run.smithi092:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-09-23T20:09:52.975 INFO:tasks.cephadm:Archiving crash dumps... 2023-09-23T20:09:52.977 DEBUG:teuthology.misc:Transferring archived files from smithi016:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/crash to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:pacific-x-reef-distro-default-smithi/7401794/remote/smithi016/crash 2023-09-23T20:09:52.978 DEBUG:teuthology.orchestra.run.smithi016:> sudo tar c -f - -C /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/crash -- . 2023-09-23T20:09:53.030 INFO:teuthology.orchestra.run.smithi016.stderr:tar: /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/crash: Cannot open: No such file or directory 2023-09-23T20:09:53.030 INFO:teuthology.orchestra.run.smithi016.stderr:tar: Error is not recoverable: exiting now 2023-09-23T20:09:53.032 DEBUG:teuthology.misc:Transferring archived files from smithi092:/var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/crash to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:pacific-x-reef-distro-default-smithi/7401794/remote/smithi092/crash 2023-09-23T20:09:53.033 DEBUG:teuthology.orchestra.run.smithi092:> sudo tar c -f - -C /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/crash -- . 2023-09-23T20:09:53.068 INFO:teuthology.orchestra.run.smithi092.stderr:tar: /var/lib/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/crash: Cannot open: No such file or directory 2023-09-23T20:09:53.068 INFO:teuthology.orchestra.run.smithi092.stderr:tar: Error is not recoverable: exiting now 2023-09-23T20:09:53.069 INFO:tasks.cephadm:Checking cluster log for badness... 2023-09-23T20:09:53.070 DEBUG:teuthology.orchestra.run.smithi016:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2023-09-23T20:09:53.091 INFO:teuthology.orchestra.run.smithi016.stderr:grep: /var/log/ceph/95ce9eb6-5a42-11ee-8db4-212e2dc638e7/ceph.log: No such file or directory 2023-09-23T20:09:53.093 INFO:tasks.cephadm:Compressing logs... 2023-09-23T20:09:53.093 DEBUG:teuthology.orchestra.run.smithi016:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-09-23T20:09:53.136 DEBUG:teuthology.orchestra.run.smithi092:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-09-23T20:09:53.149 INFO:teuthology.orchestra.run.smithi016.stderr:find: '/var/log/rbd-target-api': No such file or directory 2023-09-23T20:09:53.152 INFO:teuthology.orchestra.run.smithi092.stderr:find: '/var/log/rbd-target-api': No such file or directory 2023-09-23T20:18:56.781 INFO:tasks.cephadm:Archiving logs... 2023-09-23T20:18:56.782 DEBUG:teuthology.misc:Transferring archived files from smithi016:/var/log/ceph to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:pacific-x-reef-distro-default-smithi/7401794/remote/smithi016/log 2023-09-23T20:18:56.783 DEBUG:teuthology.orchestra.run.smithi016:> sudo tar c -f - -C /var/log/ceph -- . 2023-09-23T20:19:51.143 DEBUG:teuthology.misc:Transferring archived files from smithi092:/var/log/ceph to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:pacific-x-reef-distro-default-smithi/7401794/remote/smithi092/log 2023-09-23T20:19:51.143 DEBUG:teuthology.orchestra.run.smithi092:> sudo tar c -f - -C /var/log/ceph -- . 2023-09-23T20:20:40.669 INFO:tasks.cephadm:Removing cluster... 2023-09-23T20:20:40.670 DEBUG:teuthology.orchestra.run.smithi016:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 --force 2023-09-23T20:20:42.733 DEBUG:teuthology.orchestra.run.smithi092:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 95ce9eb6-5a42-11ee-8db4-212e2dc638e7 --force 2023-09-23T20:20:45.231 INFO:tasks.cephadm:Removing cephadm ... 2023-09-23T20:20:45.231 DEBUG:teuthology.orchestra.run.smithi016:> rm -rf /home/ubuntu/cephtest/cephadm 2023-09-23T20:20:45.238 DEBUG:teuthology.orchestra.run.smithi092:> rm -rf /home/ubuntu/cephtest/cephadm 2023-09-23T20:20:45.245 INFO:tasks.cephadm:Teardown complete 2023-09-23T20:20:45.246 DEBUG:teuthology.run_tasks:Unwinding manager install 2023-09-23T20:20:45.256 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2023-09-23T20:20:45.257 DEBUG:teuthology.orchestra.run.smithi016:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-09-23T20:20:45.282 DEBUG:teuthology.orchestra.run.smithi092:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-09-23T20:20:45.362 DEBUG:teuthology.run_tasks:Unwinding manager clock 2023-09-23T20:20:45.371 INFO:teuthology.task.clock:Checking final clock skew... 2023-09-23T20:20:45.372 DEBUG:teuthology.orchestra.run.smithi016:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-09-23T20:20:45.374 DEBUG:teuthology.orchestra.run.smithi092:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-09-23T20:20:45.428 INFO:teuthology.orchestra.run.smithi016.stdout: remote refid st t when poll reach delay offset jitter 2023-09-23T20:20:45.428 INFO:teuthology.orchestra.run.smithi016.stdout:============================================================================== 2023-09-23T20:20:45.428 INFO:teuthology.orchestra.run.smithi016.stdout:*hv01.front.sepi 44.4.53.2 2 u 108 128 377 0.190 4.789 1.059 2023-09-23T20:20:45.428 INFO:teuthology.orchestra.run.smithi016.stdout:+hv02.front.sepi 63.231.80.2 3 u 94 128 377 0.164 1.971 1.601 2023-09-23T20:20:45.428 INFO:teuthology.orchestra.run.smithi016.stdout:+hv03.front.sepi 74.6.168.72 3 u 77 128 377 0.164 0.644 0.694 2023-09-23T20:20:45.429 INFO:teuthology.orchestra.run.smithi016.stdout: hv04.front.sepi .INIT. 16 u - 1024 0 0.000 0.000 0.000 2023-09-23T20:20:45.433 INFO:teuthology.orchestra.run.smithi092.stdout: remote refid st t when poll reach delay offset jitter 2023-09-23T20:20:45.434 INFO:teuthology.orchestra.run.smithi092.stdout:============================================================================== 2023-09-23T20:20:45.434 INFO:teuthology.orchestra.run.smithi092.stdout:*hv01.front.sepi 44.4.53.2 2 u 118 256 377 0.189 0.101 0.107 2023-09-23T20:20:45.434 INFO:teuthology.orchestra.run.smithi092.stdout:+hv02.front.sepi 63.231.80.2 3 u 79 256 377 0.148 -0.543 0.094 2023-09-23T20:20:45.434 INFO:teuthology.orchestra.run.smithi092.stdout:+hv03.front.sepi 74.6.168.72 3 u 66 256 377 0.161 -2.835 0.221 2023-09-23T20:20:45.434 INFO:teuthology.orchestra.run.smithi092.stdout: hv04.front.sepi .INIT. 16 u - 1024 0 0.000 0.000 0.000 2023-09-23T20:20:45.435 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2023-09-23T20:20:45.445 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2023-09-23T20:20:45.445 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2023-09-23T20:20:45.454 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2023-09-23T20:20:45.463 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2023-09-23T20:20:45.471 INFO:teuthology.task.internal:Duration was 5652.467862 seconds 2023-09-23T20:20:45.471 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2023-09-23T20:20:45.480 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2023-09-23T20:20:45.480 DEBUG:teuthology.orchestra.run.smithi016:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-09-23T20:20:45.482 DEBUG:teuthology.orchestra.run.smithi092:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-09-23T20:20:45.588 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2023-09-23T20:20:45.588 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi016.front.sepia.ceph.com 2023-09-23T20:20:45.588 DEBUG:teuthology.orchestra.run.smithi016:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-09-23T20:20:45.602 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi092.front.sepia.ceph.com 2023-09-23T20:20:45.603 DEBUG:teuthology.orchestra.run.smithi092:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-09-23T20:20:45.619 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2023-09-23T20:20:45.619 DEBUG:teuthology.orchestra.run.smithi016:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-09-23T20:20:45.649 DEBUG:teuthology.orchestra.run.smithi092:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-09-23T20:20:45.868 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2023-09-23T20:20:45.868 DEBUG:teuthology.orchestra.run.smithi016:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-09-23T20:20:45.872 DEBUG:teuthology.orchestra.run.smithi092:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-09-23T20:20:46.438 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2023-09-23T20:20:46.450 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2023-09-23T20:20:46.450 DEBUG:teuthology.orchestra.run.smithi016:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-09-23T20:20:46.483 DEBUG:teuthology.orchestra.run.smithi092:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-09-23T20:20:46.501 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2023-09-23T20:20:46.511 DEBUG:teuthology.orchestra.run.smithi016:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-09-23T20:20:46.526 DEBUG:teuthology.orchestra.run.smithi092:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-09-23T20:20:46.537 INFO:teuthology.orchestra.run.smithi016.stdout:kernel.core_pattern = core 2023-09-23T20:20:46.568 INFO:teuthology.orchestra.run.smithi092.stdout:kernel.core_pattern = core 2023-09-23T20:20:46.614 DEBUG:teuthology.orchestra.run.smithi016:> test -e /home/ubuntu/cephtest/archive/coredump 2023-09-23T20:20:46.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:20:46.622 DEBUG:teuthology.orchestra.run.smithi092:> test -e /home/ubuntu/cephtest/archive/coredump 2023-09-23T20:20:46.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:20:46.661 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2023-09-23T20:20:46.670 INFO:teuthology.task.internal:Transferring archived files... 2023-09-23T20:20:46.671 DEBUG:teuthology.misc:Transferring archived files from smithi016:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:pacific-x-reef-distro-default-smithi/7401794/remote/smithi016 2023-09-23T20:20:46.671 DEBUG:teuthology.orchestra.run.smithi016:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-09-23T20:20:46.719 DEBUG:teuthology.misc:Transferring archived files from smithi092:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:pacific-x-reef-distro-default-smithi/7401794/remote/smithi092 2023-09-23T20:20:46.720 DEBUG:teuthology.orchestra.run.smithi092:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-09-23T20:20:46.756 INFO:teuthology.task.internal:Removing archive directory... 2023-09-23T20:20:46.756 DEBUG:teuthology.orchestra.run.smithi016:> rm -rf -- /home/ubuntu/cephtest/archive 2023-09-23T20:20:46.758 DEBUG:teuthology.orchestra.run.smithi092:> rm -rf -- /home/ubuntu/cephtest/archive 2023-09-23T20:20:46.797 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2023-09-23T20:20:46.809 INFO:teuthology.task.internal:Not uploading archives. 2023-09-23T20:20:46.809 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2023-09-23T20:20:46.821 INFO:teuthology.task.internal:Tidying up after the test... 2023-09-23T20:20:46.821 DEBUG:teuthology.orchestra.run.smithi016:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-09-23T20:20:46.824 DEBUG:teuthology.orchestra.run.smithi092:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-09-23T20:20:46.829 INFO:teuthology.orchestra.run.smithi016.stdout: 1048609 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Sep 23 20:20 /home/ubuntu/cephtest 2023-09-23T20:20:46.830 INFO:teuthology.orchestra.run.smithi016.stdout: 1311525 4 drwxrwxr-x 3 ubuntu ubuntu 4096 Sep 23 20:02 /home/ubuntu/cephtest/mnt.0 2023-09-23T20:20:46.830 INFO:teuthology.orchestra.run.smithi016.stdout: 1311528 4 drwxrwxr-x 2 ubuntu ubuntu 4096 Sep 23 20:02 /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T20:20:46.831 INFO:teuthology.orchestra.run.smithi016.stderr:rmdir: failed to remove '/home/ubuntu/cephtest': Directory not empty 2023-09-23T20:20:46.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:20:46.841 ERROR:teuthology.run_tasks:Manager failed: internal.base Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 154, in run_tasks suppress = manager.__exit__(*exc_info) File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__ next(self.gen) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/internal/__init__.py", line 53, in base run.wait( File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 479, in wait proc.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi016 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-09-23T20:20:46.847 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2023-09-23T20:20:46.860 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2023-09-23T20:20:46.870 DEBUG:teuthology.run_tasks:Exception was not quenched, exiting: CommandFailedError: Command failed on smithi016 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-09-23T20:20:46.871 INFO:teuthology.nuke:Checking targets against current locks 2023-09-23T20:20:46.877 INFO:teuthology.orchestra.run.smithi092.stdout: 1048600 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 23 20:20 /home/ubuntu/cephtest 2023-09-23T20:20:46.900 DEBUG:teuthology.nuke:shortname: smithi016 2023-09-23T20:20:46.900 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-09-23T20:20:46.924 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi016.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:pacific-x-reef-distro-default-smithi/7401794', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '20.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-09-23 18:36:55.411726', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFyT/r7xuKfmg50w0OdQebr4A/ptJo1Zk6DyZNZNZM/q5rW+sznh+wvHqeT3e4YtCdZLaXyvMms8o8u8H5nXOYM='} 2023-09-23T20:20:46.927 DEBUG:teuthology.nuke:shortname: smithi092 2023-09-23T20:20:46.927 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-09-23T20:20:46.950 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi092.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:pacific-x-reef-distro-default-smithi/7401794', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '20.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-09-23 18:36:55.413131', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFyT/r7xuKfmg50w0OdQebr4A/ptJo1Zk6DyZNZNZM/q5rW+sznh+wvHqeT3e4YtCdZLaXyvMms8o8u8H5nXOYM='} 2023-09-23T20:20:46.978 INFO:teuthology.orchestra.console.smithi016:Power off 2023-09-23T20:20:46.978 DEBUG:teuthology.orchestra.console.smithi016:pexpect command: ipmitool -H smithi016.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-09-23T20:20:47.000 DEBUG:teuthology.orchestra.console.smithi016:power off output: Chassis Power Control: Down/Off 2023-09-23T20:20:47.001 DEBUG:teuthology.orchestra.console.smithi016:pexpect command: ipmitool -H smithi016.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T20:20:47.015 INFO:teuthology.orchestra.console.smithi092:Power off 2023-09-23T20:20:47.015 DEBUG:teuthology.orchestra.console.smithi092:pexpect command: ipmitool -H smithi092.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-09-23T20:20:47.027 DEBUG:teuthology.orchestra.console.smithi016:check power output: Chassis Power is on 2023-09-23T20:20:47.037 DEBUG:teuthology.orchestra.console.smithi092:power off output: Chassis Power Control: Down/Off 2023-09-23T20:20:47.037 DEBUG:teuthology.orchestra.console.smithi092:pexpect command: ipmitool -H smithi092.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T20:20:47.059 DEBUG:teuthology.orchestra.console.smithi092:check power output: Chassis Power is on 2023-09-23T20:20:51.029 DEBUG:teuthology.orchestra.console.smithi016:pexpect command: ipmitool -H smithi016.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T20:20:51.059 DEBUG:teuthology.orchestra.console.smithi092:pexpect command: ipmitool -H smithi092.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T20:20:51.138 DEBUG:teuthology.orchestra.console.smithi016:check power output: Chassis Power is on 2023-09-23T20:20:51.168 DEBUG:teuthology.orchestra.console.smithi092:check power output: Chassis Power is on 2023-09-23T20:20:55.141 DEBUG:teuthology.orchestra.console.smithi016:pexpect command: ipmitool -H smithi016.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T20:20:55.169 DEBUG:teuthology.orchestra.console.smithi092:pexpect command: ipmitool -H smithi092.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T20:20:55.252 DEBUG:teuthology.orchestra.console.smithi016:check power output: Chassis Power is off 2023-09-23T20:20:55.278 DEBUG:teuthology.orchestra.console.smithi092:check power output: Chassis Power is on 2023-09-23T20:20:55.353 INFO:teuthology.orchestra.console.smithi016:Power off completed 2023-09-23T20:20:59.282 DEBUG:teuthology.orchestra.console.smithi092:pexpect command: ipmitool -H smithi092.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T20:20:59.393 DEBUG:teuthology.orchestra.console.smithi092:check power output: Chassis Power is off 2023-09-23T20:20:59.493 INFO:teuthology.orchestra.console.smithi092:Power off completed 2023-09-23T20:20:59.598 INFO:teuthology.run:Summary data: description: upgrade:pacific-x/stress-split/{0-distro/ubuntu_20.04 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} duration: 5652.467861890793 failure_reason: '' flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=bc53a3edf7294239ae12e428011f0c20 status: fail success: false 2023-09-23T20:20:59.598 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-09-23T20:20:59.675 INFO:teuthology.run:FAIL